/src/aom/aom_dsp/x86/variance_impl_avx2.c
Line | Count | Source |
1 | | /* |
2 | | * Copyright (c) 2016, Alliance for Open Media. All rights reserved. |
3 | | * |
4 | | * This source code is subject to the terms of the BSD 2 Clause License and |
5 | | * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
6 | | * was not distributed with this source code in the LICENSE file, you can |
7 | | * obtain it at www.aomedia.org/license/software. If the Alliance for Open |
8 | | * Media Patent License 1.0 was not distributed with this source code in the |
9 | | * PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
10 | | */ |
11 | | |
12 | | #include <immintrin.h> // AVX2 |
13 | | |
14 | | #include "config/aom_dsp_rtcd.h" |
15 | | |
16 | | #include "aom_ports/mem.h" |
17 | | |
18 | | /* clang-format off */ |
19 | | DECLARE_ALIGNED(32, static const uint8_t, bilinear_filters_avx2[512]) = { |
20 | | 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, |
21 | | 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, 16, 0, |
22 | | 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, |
23 | | 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, |
24 | | 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, |
25 | | 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, |
26 | | 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, |
27 | | 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, |
28 | | 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, |
29 | | 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, |
30 | | 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, |
31 | | 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, 6, 10, |
32 | | 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, |
33 | | 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, 4, 12, |
34 | | 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, |
35 | | 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, 2, 14, |
36 | | }; |
37 | | /* clang-format on */ |
38 | | |
39 | | #define FILTER_SRC(filter) \ |
40 | | /* filter the source */ \ |
41 | 0 | exp_src_lo = _mm256_maddubs_epi16(exp_src_lo, filter); \ |
42 | 0 | exp_src_hi = _mm256_maddubs_epi16(exp_src_hi, filter); \ |
43 | 0 | \ |
44 | 0 | /* add 8 to source */ \ |
45 | 0 | exp_src_lo = _mm256_add_epi16(exp_src_lo, pw8); \ |
46 | 0 | exp_src_hi = _mm256_add_epi16(exp_src_hi, pw8); \ |
47 | 0 | \ |
48 | 0 | /* divide source by 16 */ \ |
49 | 0 | exp_src_lo = _mm256_srai_epi16(exp_src_lo, 4); \ |
50 | 0 | exp_src_hi = _mm256_srai_epi16(exp_src_hi, 4); |
51 | | |
52 | | #define MERGE_WITH_SRC(src_reg, reg) \ |
53 | 0 | exp_src_lo = _mm256_unpacklo_epi8(src_reg, reg); \ |
54 | 0 | exp_src_hi = _mm256_unpackhi_epi8(src_reg, reg); |
55 | | |
56 | | #define LOAD_SRC_DST \ |
57 | | /* load source and destination */ \ |
58 | 0 | src_reg = _mm256_loadu_si256((__m256i const *)(src)); \ |
59 | 0 | dst_reg = _mm256_loadu_si256((__m256i const *)(dst)); |
60 | | |
61 | | #define AVG_NEXT_SRC(src_reg, size_stride) \ |
62 | 0 | src_next_reg = _mm256_loadu_si256((__m256i const *)(src + size_stride)); \ |
63 | 0 | /* average between current and next stride source */ \ |
64 | 0 | src_reg = _mm256_avg_epu8(src_reg, src_next_reg); |
65 | | |
66 | | #define MERGE_NEXT_SRC(src_reg, size_stride) \ |
67 | 0 | src_next_reg = _mm256_loadu_si256((__m256i const *)(src + size_stride)); \ |
68 | 0 | MERGE_WITH_SRC(src_reg, src_next_reg) |
69 | | |
70 | | #define CALC_SUM_SSE_INSIDE_LOOP \ |
71 | | /* expand each byte to 2 bytes */ \ |
72 | 0 | exp_dst_lo = _mm256_unpacklo_epi8(dst_reg, zero_reg); \ |
73 | 0 | exp_dst_hi = _mm256_unpackhi_epi8(dst_reg, zero_reg); \ |
74 | 0 | /* source - dest */ \ |
75 | 0 | exp_src_lo = _mm256_sub_epi16(exp_src_lo, exp_dst_lo); \ |
76 | 0 | exp_src_hi = _mm256_sub_epi16(exp_src_hi, exp_dst_hi); \ |
77 | 0 | /* caculate sum */ \ |
78 | 0 | sum_reg = _mm256_add_epi16(sum_reg, exp_src_lo); \ |
79 | 0 | exp_src_lo = _mm256_madd_epi16(exp_src_lo, exp_src_lo); \ |
80 | 0 | sum_reg = _mm256_add_epi16(sum_reg, exp_src_hi); \ |
81 | 0 | exp_src_hi = _mm256_madd_epi16(exp_src_hi, exp_src_hi); \ |
82 | 0 | /* calculate sse */ \ |
83 | 0 | sse_reg = _mm256_add_epi32(sse_reg, exp_src_lo); \ |
84 | 0 | sse_reg = _mm256_add_epi32(sse_reg, exp_src_hi); |
85 | | |
86 | | // final calculation to sum and sse |
87 | | #define CALC_SUM_AND_SSE \ |
88 | 0 | res_cmp = _mm256_cmpgt_epi16(zero_reg, sum_reg); \ |
89 | 0 | sse_reg_hi = _mm256_srli_si256(sse_reg, 8); \ |
90 | 0 | sum_reg_lo = _mm256_unpacklo_epi16(sum_reg, res_cmp); \ |
91 | 0 | sum_reg_hi = _mm256_unpackhi_epi16(sum_reg, res_cmp); \ |
92 | 0 | sse_reg = _mm256_add_epi32(sse_reg, sse_reg_hi); \ |
93 | 0 | sum_reg = _mm256_add_epi32(sum_reg_lo, sum_reg_hi); \ |
94 | 0 | \ |
95 | 0 | sse_reg_hi = _mm256_srli_si256(sse_reg, 4); \ |
96 | 0 | sum_reg_hi = _mm256_srli_si256(sum_reg, 8); \ |
97 | 0 | \ |
98 | 0 | sse_reg = _mm256_add_epi32(sse_reg, sse_reg_hi); \ |
99 | 0 | sum_reg = _mm256_add_epi32(sum_reg, sum_reg_hi); \ |
100 | 0 | *((int *)sse) = _mm_cvtsi128_si32(_mm256_castsi256_si128(sse_reg)) + \ |
101 | 0 | _mm_cvtsi128_si32(_mm256_extractf128_si256(sse_reg, 1)); \ |
102 | 0 | sum_reg_hi = _mm256_srli_si256(sum_reg, 4); \ |
103 | 0 | sum_reg = _mm256_add_epi32(sum_reg, sum_reg_hi); \ |
104 | 0 | sum = _mm_cvtsi128_si32(_mm256_castsi256_si128(sum_reg)) + \ |
105 | 0 | _mm_cvtsi128_si32(_mm256_extractf128_si256(sum_reg, 1)); |
106 | | |
107 | | // Functions related to sub pixel variance width 16 |
108 | | #define LOAD_SRC_DST_INSERT(src_stride, dst_stride) \ |
109 | | /* load source and destination of 2 rows and insert*/ \ |
110 | 0 | src_reg = _mm256_inserti128_si256( \ |
111 | 0 | _mm256_castsi128_si256(_mm_loadu_si128((__m128i *)(src))), \ |
112 | 0 | _mm_loadu_si128((__m128i *)(src + src_stride)), 1); \ |
113 | 0 | dst_reg = _mm256_inserti128_si256( \ |
114 | 0 | _mm256_castsi128_si256(_mm_loadu_si128((__m128i *)(dst))), \ |
115 | 0 | _mm_loadu_si128((__m128i *)(dst + dst_stride)), 1); |
116 | | |
117 | | #define AVG_NEXT_SRC_INSERT(src_reg, size_stride) \ |
118 | 0 | src_next_reg = _mm256_inserti128_si256( \ |
119 | 0 | _mm256_castsi128_si256(_mm_loadu_si128((__m128i *)(src + size_stride))), \ |
120 | 0 | _mm_loadu_si128((__m128i *)(src + (size_stride << 1))), 1); \ |
121 | 0 | /* average between current and next stride source */ \ |
122 | 0 | src_reg = _mm256_avg_epu8(src_reg, src_next_reg); |
123 | | |
124 | | #define MERGE_NEXT_SRC_INSERT(src_reg, size_stride) \ |
125 | 0 | src_next_reg = _mm256_inserti128_si256( \ |
126 | 0 | _mm256_castsi128_si256(_mm_loadu_si128((__m128i *)(src + size_stride))), \ |
127 | 0 | _mm_loadu_si128((__m128i *)(src + (src_stride + size_stride))), 1); \ |
128 | 0 | MERGE_WITH_SRC(src_reg, src_next_reg) |
129 | | |
130 | | #define LOAD_SRC_NEXT_BYTE_INSERT \ |
131 | | /* load source and another source from next row */ \ |
132 | 0 | src_reg = _mm256_inserti128_si256( \ |
133 | 0 | _mm256_castsi128_si256(_mm_loadu_si128((__m128i *)(src))), \ |
134 | 0 | _mm_loadu_si128((__m128i *)(src + src_stride)), 1); \ |
135 | 0 | /* load source and next row source from 1 byte onwards */ \ |
136 | 0 | src_next_reg = _mm256_inserti128_si256( \ |
137 | 0 | _mm256_castsi128_si256(_mm_loadu_si128((__m128i *)(src + 1))), \ |
138 | 0 | _mm_loadu_si128((__m128i *)(src + src_stride + 1)), 1); |
139 | | |
140 | | #define LOAD_DST_INSERT \ |
141 | 0 | dst_reg = _mm256_inserti128_si256( \ |
142 | 0 | _mm256_castsi128_si256(_mm_loadu_si128((__m128i *)(dst))), \ |
143 | 0 | _mm_loadu_si128((__m128i *)(dst + dst_stride)), 1); |
144 | | |
145 | | #define LOAD_SRC_MERGE_128BIT(filter) \ |
146 | 0 | __m128i src_reg_0 = _mm_loadu_si128((__m128i *)(src)); \ |
147 | 0 | __m128i src_reg_1 = _mm_loadu_si128((__m128i *)(src + 1)); \ |
148 | 0 | __m128i src_lo = _mm_unpacklo_epi8(src_reg_0, src_reg_1); \ |
149 | 0 | __m128i src_hi = _mm_unpackhi_epi8(src_reg_0, src_reg_1); \ |
150 | 0 | __m128i filter_128bit = _mm256_castsi256_si128(filter); \ |
151 | 0 | __m128i pw8_128bit = _mm256_castsi256_si128(pw8); |
152 | | |
153 | | #define FILTER_SRC_128BIT(filter) \ |
154 | | /* filter the source */ \ |
155 | 0 | src_lo = _mm_maddubs_epi16(src_lo, filter); \ |
156 | 0 | src_hi = _mm_maddubs_epi16(src_hi, filter); \ |
157 | 0 | \ |
158 | 0 | /* add 8 to source */ \ |
159 | 0 | src_lo = _mm_add_epi16(src_lo, pw8_128bit); \ |
160 | 0 | src_hi = _mm_add_epi16(src_hi, pw8_128bit); \ |
161 | 0 | \ |
162 | 0 | /* divide source by 16 */ \ |
163 | 0 | src_lo = _mm_srai_epi16(src_lo, 4); \ |
164 | 0 | src_hi = _mm_srai_epi16(src_hi, 4); |
165 | | |
166 | | // TODO(chiyotsai@google.com): These variance functions are macro-fied so we |
167 | | // don't have to manually optimize the individual for-loops. We could save some |
168 | | // binary size by optimizing the loops more carefully without duplicating the |
169 | | // codes with a macro. |
170 | | #define MAKE_SUB_PIXEL_VAR_32XH(height, log2height) \ |
171 | | static inline int aom_sub_pixel_variance32x##height##_imp_avx2( \ |
172 | | const uint8_t *src, int src_stride, int x_offset, int y_offset, \ |
173 | 0 | const uint8_t *dst, int dst_stride, unsigned int *sse) { \ |
174 | 0 | __m256i src_reg, dst_reg, exp_src_lo, exp_src_hi, exp_dst_lo, exp_dst_hi; \ |
175 | 0 | __m256i sse_reg, sum_reg, sse_reg_hi, res_cmp, sum_reg_lo, sum_reg_hi; \ |
176 | 0 | __m256i zero_reg; \ |
177 | 0 | int i, sum; \ |
178 | 0 | sum_reg = _mm256_setzero_si256(); \ |
179 | 0 | sse_reg = _mm256_setzero_si256(); \ |
180 | 0 | zero_reg = _mm256_setzero_si256(); \ |
181 | 0 | \ |
182 | 0 | /* x_offset = 0 and y_offset = 0 */ \ |
183 | 0 | if (x_offset == 0) { \ |
184 | 0 | if (y_offset == 0) { \ |
185 | 0 | for (i = 0; i < height; i++) { \ |
186 | 0 | LOAD_SRC_DST \ |
187 | 0 | /* expend each byte to 2 bytes */ \ |
188 | 0 | MERGE_WITH_SRC(src_reg, zero_reg) \ |
189 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
190 | 0 | src += src_stride; \ |
191 | 0 | dst += dst_stride; \ |
192 | 0 | } \ |
193 | 0 | /* x_offset = 0 and y_offset = 4 */ \ |
194 | 0 | } else if (y_offset == 4) { \ |
195 | 0 | __m256i src_next_reg; \ |
196 | 0 | for (i = 0; i < height; i++) { \ |
197 | 0 | LOAD_SRC_DST \ |
198 | 0 | AVG_NEXT_SRC(src_reg, src_stride) \ |
199 | 0 | /* expend each byte to 2 bytes */ \ |
200 | 0 | MERGE_WITH_SRC(src_reg, zero_reg) \ |
201 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
202 | 0 | src += src_stride; \ |
203 | 0 | dst += dst_stride; \ |
204 | 0 | } \ |
205 | 0 | /* x_offset = 0 and y_offset = bilin interpolation */ \ |
206 | 0 | } else { \ |
207 | 0 | __m256i filter, pw8, src_next_reg; \ |
208 | 0 | \ |
209 | 0 | y_offset <<= 5; \ |
210 | 0 | filter = _mm256_load_si256( \ |
211 | 0 | (__m256i const *)(bilinear_filters_avx2 + y_offset)); \ |
212 | 0 | pw8 = _mm256_set1_epi16(8); \ |
213 | 0 | for (i = 0; i < height; i++) { \ |
214 | 0 | LOAD_SRC_DST \ |
215 | 0 | MERGE_NEXT_SRC(src_reg, src_stride) \ |
216 | 0 | FILTER_SRC(filter) \ |
217 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
218 | 0 | src += src_stride; \ |
219 | 0 | dst += dst_stride; \ |
220 | 0 | } \ |
221 | 0 | } \ |
222 | 0 | /* x_offset = 4 and y_offset = 0 */ \ |
223 | 0 | } else if (x_offset == 4) { \ |
224 | 0 | if (y_offset == 0) { \ |
225 | 0 | __m256i src_next_reg; \ |
226 | 0 | for (i = 0; i < height; i++) { \ |
227 | 0 | LOAD_SRC_DST \ |
228 | 0 | AVG_NEXT_SRC(src_reg, 1) \ |
229 | 0 | /* expand each byte to 2 bytes */ \ |
230 | 0 | MERGE_WITH_SRC(src_reg, zero_reg) \ |
231 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
232 | 0 | src += src_stride; \ |
233 | 0 | dst += dst_stride; \ |
234 | 0 | } \ |
235 | 0 | /* x_offset = 4 and y_offset = 4 */ \ |
236 | 0 | } else if (y_offset == 4) { \ |
237 | 0 | __m256i src_next_reg, src_avg; \ |
238 | 0 | /* load source and another source starting from the next */ \ |
239 | 0 | /* following byte */ \ |
240 | 0 | src_reg = _mm256_loadu_si256((__m256i const *)(src)); \ |
241 | 0 | AVG_NEXT_SRC(src_reg, 1) \ |
242 | 0 | for (i = 0; i < height; i++) { \ |
243 | 0 | src_avg = src_reg; \ |
244 | 0 | src += src_stride; \ |
245 | 0 | LOAD_SRC_DST \ |
246 | 0 | AVG_NEXT_SRC(src_reg, 1) \ |
247 | 0 | /* average between previous average to current average */ \ |
248 | 0 | src_avg = _mm256_avg_epu8(src_avg, src_reg); \ |
249 | 0 | /* expand each byte to 2 bytes */ \ |
250 | 0 | MERGE_WITH_SRC(src_avg, zero_reg) \ |
251 | 0 | /* save current source average */ \ |
252 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
253 | 0 | dst += dst_stride; \ |
254 | 0 | } \ |
255 | 0 | /* x_offset = 4 and y_offset = bilin interpolation */ \ |
256 | 0 | } else { \ |
257 | 0 | __m256i filter, pw8, src_next_reg, src_avg; \ |
258 | 0 | y_offset <<= 5; \ |
259 | 0 | filter = _mm256_load_si256( \ |
260 | 0 | (__m256i const *)(bilinear_filters_avx2 + y_offset)); \ |
261 | 0 | pw8 = _mm256_set1_epi16(8); \ |
262 | 0 | /* load source and another source starting from the next */ \ |
263 | 0 | /* following byte */ \ |
264 | 0 | src_reg = _mm256_loadu_si256((__m256i const *)(src)); \ |
265 | 0 | AVG_NEXT_SRC(src_reg, 1) \ |
266 | 0 | for (i = 0; i < height; i++) { \ |
267 | 0 | /* save current source average */ \ |
268 | 0 | src_avg = src_reg; \ |
269 | 0 | src += src_stride; \ |
270 | 0 | LOAD_SRC_DST \ |
271 | 0 | AVG_NEXT_SRC(src_reg, 1) \ |
272 | 0 | MERGE_WITH_SRC(src_avg, src_reg) \ |
273 | 0 | FILTER_SRC(filter) \ |
274 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
275 | 0 | dst += dst_stride; \ |
276 | 0 | } \ |
277 | 0 | } \ |
278 | 0 | /* x_offset = bilin interpolation and y_offset = 0 */ \ |
279 | 0 | } else { \ |
280 | 0 | if (y_offset == 0) { \ |
281 | 0 | __m256i filter, pw8, src_next_reg; \ |
282 | 0 | x_offset <<= 5; \ |
283 | 0 | filter = _mm256_load_si256( \ |
284 | 0 | (__m256i const *)(bilinear_filters_avx2 + x_offset)); \ |
285 | 0 | pw8 = _mm256_set1_epi16(8); \ |
286 | 0 | for (i = 0; i < height; i++) { \ |
287 | 0 | LOAD_SRC_DST \ |
288 | 0 | MERGE_NEXT_SRC(src_reg, 1) \ |
289 | 0 | FILTER_SRC(filter) \ |
290 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
291 | 0 | src += src_stride; \ |
292 | 0 | dst += dst_stride; \ |
293 | 0 | } \ |
294 | 0 | /* x_offset = bilin interpolation and y_offset = 4 */ \ |
295 | 0 | } else if (y_offset == 4) { \ |
296 | 0 | __m256i filter, pw8, src_next_reg, src_pack; \ |
297 | 0 | x_offset <<= 5; \ |
298 | 0 | filter = _mm256_load_si256( \ |
299 | 0 | (__m256i const *)(bilinear_filters_avx2 + x_offset)); \ |
300 | 0 | pw8 = _mm256_set1_epi16(8); \ |
301 | 0 | src_reg = _mm256_loadu_si256((__m256i const *)(src)); \ |
302 | 0 | MERGE_NEXT_SRC(src_reg, 1) \ |
303 | 0 | FILTER_SRC(filter) \ |
304 | 0 | /* convert each 16 bit to 8 bit to each low and high lane source */ \ |
305 | 0 | src_pack = _mm256_packus_epi16(exp_src_lo, exp_src_hi); \ |
306 | 0 | for (i = 0; i < height; i++) { \ |
307 | 0 | src += src_stride; \ |
308 | 0 | LOAD_SRC_DST \ |
309 | 0 | MERGE_NEXT_SRC(src_reg, 1) \ |
310 | 0 | FILTER_SRC(filter) \ |
311 | 0 | src_reg = _mm256_packus_epi16(exp_src_lo, exp_src_hi); \ |
312 | 0 | /* average between previous pack to the current */ \ |
313 | 0 | src_pack = _mm256_avg_epu8(src_pack, src_reg); \ |
314 | 0 | MERGE_WITH_SRC(src_pack, zero_reg) \ |
315 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
316 | 0 | src_pack = src_reg; \ |
317 | 0 | dst += dst_stride; \ |
318 | 0 | } \ |
319 | 0 | /* x_offset = bilin interpolation and y_offset = bilin interpolation \ |
320 | 0 | */ \ |
321 | 0 | } else { \ |
322 | 0 | __m256i xfilter, yfilter, pw8, mask_00ff; \ |
323 | 0 | __m256i p0, p1, p2; \ |
324 | 0 | const uint8_t *src_ptr = src; \ |
325 | 0 | x_offset <<= 5; \ |
326 | 0 | xfilter = _mm256_load_si256( \ |
327 | 0 | (__m256i const *)(bilinear_filters_avx2 + x_offset)); \ |
328 | 0 | y_offset <<= 5; \ |
329 | 0 | yfilter = _mm256_load_si256( \ |
330 | 0 | (__m256i const *)(bilinear_filters_avx2 + y_offset)); \ |
331 | 0 | pw8 = _mm256_set1_epi16(8); \ |
332 | 0 | mask_00ff = _mm256_set1_epi16(0x00ff); \ |
333 | 0 | \ |
334 | 0 | { \ |
335 | 0 | __m256i s0 = _mm256_loadu_si256((__m256i const *)(src_ptr)); \ |
336 | 0 | __m256i s1 = _mm256_loadu_si256((__m256i const *)(src_ptr + 1)); \ |
337 | 0 | __m256i he = _mm256_maddubs_epi16(s0, xfilter); \ |
338 | 0 | __m256i ho = _mm256_maddubs_epi16(s1, xfilter); \ |
339 | 0 | he = _mm256_srai_epi16(_mm256_add_epi16(he, pw8), 4); \ |
340 | 0 | ho = _mm256_srai_epi16(_mm256_add_epi16(ho, pw8), 4); \ |
341 | 0 | p0 = _mm256_packus_epi16(he, ho); \ |
342 | 0 | src_ptr += src_stride; \ |
343 | 0 | } \ |
344 | 0 | { \ |
345 | 0 | __m256i s0 = _mm256_loadu_si256((__m256i const *)(src_ptr)); \ |
346 | 0 | __m256i s1 = _mm256_loadu_si256((__m256i const *)(src_ptr + 1)); \ |
347 | 0 | __m256i he = _mm256_maddubs_epi16(s0, xfilter); \ |
348 | 0 | __m256i ho = _mm256_maddubs_epi16(s1, xfilter); \ |
349 | 0 | he = _mm256_srai_epi16(_mm256_add_epi16(he, pw8), 4); \ |
350 | 0 | ho = _mm256_srai_epi16(_mm256_add_epi16(ho, pw8), 4); \ |
351 | 0 | p1 = _mm256_packus_epi16(he, ho); \ |
352 | 0 | src_ptr += src_stride; \ |
353 | 0 | } \ |
354 | 0 | { \ |
355 | 0 | __m256i s0 = _mm256_loadu_si256((__m256i const *)(src_ptr)); \ |
356 | 0 | __m256i s1 = _mm256_loadu_si256((__m256i const *)(src_ptr + 1)); \ |
357 | 0 | __m256i he = _mm256_maddubs_epi16(s0, xfilter); \ |
358 | 0 | __m256i ho = _mm256_maddubs_epi16(s1, xfilter); \ |
359 | 0 | he = _mm256_srai_epi16(_mm256_add_epi16(he, pw8), 4); \ |
360 | 0 | ho = _mm256_srai_epi16(_mm256_add_epi16(ho, pw8), 4); \ |
361 | 0 | p2 = _mm256_packus_epi16(he, ho); \ |
362 | 0 | src_ptr += src_stride; \ |
363 | 0 | } \ |
364 | 0 | \ |
365 | 0 | for (i = 0; i < height - 2; i += 2) { \ |
366 | 0 | __m256i p3, p4, v_ev_A, v_od_A, dst_A, dst_A_ev, dst_A_od; \ |
367 | 0 | __m256i diff_A_ev, diff_A_od, v_ev_B, v_od_B, dst_B, dst_B_ev; \ |
368 | 0 | __m256i dst_B_od, diff_B_ev, diff_B_od, sum_comb, sse_comb; \ |
369 | 0 | \ |
370 | 0 | { \ |
371 | 0 | __m256i s0 = _mm256_loadu_si256((__m256i const *)(src_ptr)); \ |
372 | 0 | __m256i s1 = _mm256_loadu_si256((__m256i const *)(src_ptr + 1)); \ |
373 | 0 | __m256i he = _mm256_maddubs_epi16(s0, xfilter); \ |
374 | 0 | __m256i ho = _mm256_maddubs_epi16(s1, xfilter); \ |
375 | 0 | he = _mm256_srai_epi16(_mm256_add_epi16(he, pw8), 4); \ |
376 | 0 | ho = _mm256_srai_epi16(_mm256_add_epi16(ho, pw8), 4); \ |
377 | 0 | p3 = _mm256_packus_epi16(he, ho); \ |
378 | 0 | src_ptr += src_stride; \ |
379 | 0 | } \ |
380 | 0 | { \ |
381 | 0 | __m256i s0 = _mm256_loadu_si256((__m256i const *)(src_ptr)); \ |
382 | 0 | __m256i s1 = _mm256_loadu_si256((__m256i const *)(src_ptr + 1)); \ |
383 | 0 | __m256i he = _mm256_maddubs_epi16(s0, xfilter); \ |
384 | 0 | __m256i ho = _mm256_maddubs_epi16(s1, xfilter); \ |
385 | 0 | he = _mm256_srai_epi16(_mm256_add_epi16(he, pw8), 4); \ |
386 | 0 | ho = _mm256_srai_epi16(_mm256_add_epi16(ho, pw8), 4); \ |
387 | 0 | p4 = _mm256_packus_epi16(he, ho); \ |
388 | 0 | src_ptr += src_stride; \ |
389 | 0 | } \ |
390 | 0 | \ |
391 | 0 | v_ev_A = \ |
392 | 0 | _mm256_maddubs_epi16(_mm256_unpacklo_epi8(p0, p1), yfilter); \ |
393 | 0 | v_od_A = \ |
394 | 0 | _mm256_maddubs_epi16(_mm256_unpackhi_epi8(p0, p1), yfilter); \ |
395 | 0 | v_ev_A = _mm256_srai_epi16(_mm256_add_epi16(v_ev_A, pw8), 4); \ |
396 | 0 | v_od_A = _mm256_srai_epi16(_mm256_add_epi16(v_od_A, pw8), 4); \ |
397 | 0 | \ |
398 | 0 | dst_A = _mm256_loadu_si256((__m256i const *)(dst)); \ |
399 | 0 | dst += dst_stride; \ |
400 | 0 | dst_A_ev = _mm256_and_si256(dst_A, mask_00ff); \ |
401 | 0 | dst_A_od = _mm256_srli_epi16(dst_A, 8); \ |
402 | 0 | diff_A_ev = _mm256_sub_epi16(v_ev_A, dst_A_ev); \ |
403 | 0 | diff_A_od = _mm256_sub_epi16(v_od_A, dst_A_od); \ |
404 | 0 | \ |
405 | 0 | v_ev_B = \ |
406 | 0 | _mm256_maddubs_epi16(_mm256_unpacklo_epi8(p1, p2), yfilter); \ |
407 | 0 | v_od_B = \ |
408 | 0 | _mm256_maddubs_epi16(_mm256_unpackhi_epi8(p1, p2), yfilter); \ |
409 | 0 | v_ev_B = _mm256_srai_epi16(_mm256_add_epi16(v_ev_B, pw8), 4); \ |
410 | 0 | v_od_B = _mm256_srai_epi16(_mm256_add_epi16(v_od_B, pw8), 4); \ |
411 | 0 | \ |
412 | 0 | dst_B = _mm256_loadu_si256((__m256i const *)(dst)); \ |
413 | 0 | dst += dst_stride; \ |
414 | 0 | dst_B_ev = _mm256_and_si256(dst_B, mask_00ff); \ |
415 | 0 | dst_B_od = _mm256_srli_epi16(dst_B, 8); \ |
416 | 0 | diff_B_ev = _mm256_sub_epi16(v_ev_B, dst_B_ev); \ |
417 | 0 | diff_B_od = _mm256_sub_epi16(v_od_B, dst_B_od); \ |
418 | 0 | \ |
419 | 0 | sum_comb = _mm256_add_epi16(_mm256_add_epi16(diff_A_ev, diff_A_od), \ |
420 | 0 | _mm256_add_epi16(diff_B_ev, diff_B_od)); \ |
421 | 0 | sum_reg = _mm256_add_epi16(sum_reg, sum_comb); \ |
422 | 0 | \ |
423 | 0 | sse_comb = _mm256_add_epi32( \ |
424 | 0 | _mm256_add_epi32(_mm256_madd_epi16(diff_A_ev, diff_A_ev), \ |
425 | 0 | _mm256_madd_epi16(diff_A_od, diff_A_od)), \ |
426 | 0 | _mm256_add_epi32(_mm256_madd_epi16(diff_B_ev, diff_B_ev), \ |
427 | 0 | _mm256_madd_epi16(diff_B_od, diff_B_od))); \ |
428 | 0 | sse_reg = _mm256_add_epi32(sse_reg, sse_comb); \ |
429 | 0 | \ |
430 | 0 | p0 = p2; \ |
431 | 0 | p1 = p3; \ |
432 | 0 | p2 = p4; \ |
433 | 0 | } \ |
434 | 0 | \ |
435 | 0 | { \ |
436 | 0 | __m256i v_ev_A, v_od_A, dst_A, dst_A_ev, dst_A_od; \ |
437 | 0 | __m256i diff_A_ev, diff_A_od, v_ev_B, v_od_B, dst_B, dst_B_ev; \ |
438 | 0 | __m256i dst_B_od, diff_B_ev, diff_B_od, sum_comb, sse_comb; \ |
439 | 0 | \ |
440 | 0 | v_ev_A = \ |
441 | 0 | _mm256_maddubs_epi16(_mm256_unpacklo_epi8(p0, p1), yfilter); \ |
442 | 0 | v_od_A = \ |
443 | 0 | _mm256_maddubs_epi16(_mm256_unpackhi_epi8(p0, p1), yfilter); \ |
444 | 0 | v_ev_A = _mm256_srai_epi16(_mm256_add_epi16(v_ev_A, pw8), 4); \ |
445 | 0 | v_od_A = _mm256_srai_epi16(_mm256_add_epi16(v_od_A, pw8), 4); \ |
446 | 0 | \ |
447 | 0 | dst_A = _mm256_loadu_si256((__m256i const *)(dst)); \ |
448 | 0 | dst += dst_stride; \ |
449 | 0 | dst_A_ev = _mm256_and_si256(dst_A, mask_00ff); \ |
450 | 0 | dst_A_od = _mm256_srli_epi16(dst_A, 8); \ |
451 | 0 | diff_A_ev = _mm256_sub_epi16(v_ev_A, dst_A_ev); \ |
452 | 0 | diff_A_od = _mm256_sub_epi16(v_od_A, dst_A_od); \ |
453 | 0 | \ |
454 | 0 | v_ev_B = \ |
455 | 0 | _mm256_maddubs_epi16(_mm256_unpacklo_epi8(p1, p2), yfilter); \ |
456 | 0 | v_od_B = \ |
457 | 0 | _mm256_maddubs_epi16(_mm256_unpackhi_epi8(p1, p2), yfilter); \ |
458 | 0 | v_ev_B = _mm256_srai_epi16(_mm256_add_epi16(v_ev_B, pw8), 4); \ |
459 | 0 | v_od_B = _mm256_srai_epi16(_mm256_add_epi16(v_od_B, pw8), 4); \ |
460 | 0 | \ |
461 | 0 | dst_B = _mm256_loadu_si256((__m256i const *)(dst)); \ |
462 | 0 | dst += dst_stride; \ |
463 | 0 | dst_B_ev = _mm256_and_si256(dst_B, mask_00ff); \ |
464 | 0 | dst_B_od = _mm256_srli_epi16(dst_B, 8); \ |
465 | 0 | diff_B_ev = _mm256_sub_epi16(v_ev_B, dst_B_ev); \ |
466 | 0 | diff_B_od = _mm256_sub_epi16(v_od_B, dst_B_od); \ |
467 | 0 | \ |
468 | 0 | sum_comb = _mm256_add_epi16(_mm256_add_epi16(diff_A_ev, diff_A_od), \ |
469 | 0 | _mm256_add_epi16(diff_B_ev, diff_B_od)); \ |
470 | 0 | sum_reg = _mm256_add_epi16(sum_reg, sum_comb); \ |
471 | 0 | \ |
472 | 0 | sse_comb = _mm256_add_epi32( \ |
473 | 0 | _mm256_add_epi32(_mm256_madd_epi16(diff_A_ev, diff_A_ev), \ |
474 | 0 | _mm256_madd_epi16(diff_A_od, diff_A_od)), \ |
475 | 0 | _mm256_add_epi32(_mm256_madd_epi16(diff_B_ev, diff_B_ev), \ |
476 | 0 | _mm256_madd_epi16(diff_B_od, diff_B_od))); \ |
477 | 0 | sse_reg = _mm256_add_epi32(sse_reg, sse_comb); \ |
478 | 0 | } \ |
479 | 0 | \ |
480 | 0 | { \ |
481 | 0 | __m256i sse_hi, sum_hi; \ |
482 | 0 | int f_sse, f_sum; \ |
483 | 0 | sum_reg = _mm256_madd_epi16(sum_reg, _mm256_set1_epi16(1)); \ |
484 | 0 | sse_hi = _mm256_srli_si256(sse_reg, 8); \ |
485 | 0 | sum_hi = _mm256_srli_si256(sum_reg, 8); \ |
486 | 0 | sse_reg = _mm256_add_epi32(sse_reg, sse_hi); \ |
487 | 0 | sum_reg = _mm256_add_epi32(sum_reg, sum_hi); \ |
488 | 0 | sse_hi = _mm256_srli_si256(sse_reg, 4); \ |
489 | 0 | sum_hi = _mm256_srli_si256(sum_reg, 4); \ |
490 | 0 | sse_reg = _mm256_add_epi32(sse_reg, sse_hi); \ |
491 | 0 | sum_reg = _mm256_add_epi32(sum_reg, sum_hi); \ |
492 | 0 | f_sse = _mm_cvtsi128_si32(_mm256_castsi256_si128(sse_reg)) + \ |
493 | 0 | _mm_cvtsi128_si32(_mm256_extractf128_si256(sse_reg, 1)); \ |
494 | 0 | f_sum = _mm_cvtsi128_si32(_mm256_castsi256_si128(sum_reg)) + \ |
495 | 0 | _mm_cvtsi128_si32(_mm256_extractf128_si256(sum_reg, 1)); \ |
496 | 0 | *sse = f_sse; \ |
497 | 0 | _mm256_zeroupper(); \ |
498 | 0 | return f_sum; \ |
499 | 0 | } \ |
500 | 0 | } \ |
501 | 0 | } \ |
502 | 0 | CALC_SUM_AND_SSE \ |
503 | 0 | _mm256_zeroupper(); \ |
504 | 0 | return sum; \ |
505 | 0 | } \ |
506 | | unsigned int aom_sub_pixel_variance32x##height##_avx2( \ |
507 | | const uint8_t *src, int src_stride, int x_offset, int y_offset, \ |
508 | 0 | const uint8_t *dst, int dst_stride, unsigned int *sse) { \ |
509 | 0 | const int sum = aom_sub_pixel_variance32x##height##_imp_avx2( \ |
510 | 0 | src, src_stride, x_offset, y_offset, dst, dst_stride, sse); \ |
511 | 0 | return *sse - (unsigned int)(((int64_t)sum * sum) >> (5 + log2height)); \ |
512 | 0 | } Unexecuted instantiation: aom_sub_pixel_variance32x64_avx2 Unexecuted instantiation: aom_sub_pixel_variance32x32_avx2 Unexecuted instantiation: aom_sub_pixel_variance32x16_avx2 |
513 | | |
514 | 0 | MAKE_SUB_PIXEL_VAR_32XH(64, 6) |
515 | 0 | MAKE_SUB_PIXEL_VAR_32XH(32, 5) |
516 | 0 | MAKE_SUB_PIXEL_VAR_32XH(16, 4) |
517 | | |
518 | | #define AOM_SUB_PIXEL_VAR_AVX2(w, h, wf, hf, wlog2, hlog2) \ |
519 | | unsigned int aom_sub_pixel_variance##w##x##h##_avx2( \ |
520 | | const uint8_t *src, int src_stride, int x_offset, int y_offset, \ |
521 | 0 | const uint8_t *dst, int dst_stride, unsigned int *sse_ptr) { \ |
522 | 0 | unsigned int sse = 0; \ |
523 | 0 | int se = 0; \ |
524 | 0 | for (int i = 0; i < (w / wf); ++i) { \ |
525 | 0 | const uint8_t *src_ptr = src; \ |
526 | 0 | const uint8_t *dst_ptr = dst; \ |
527 | 0 | for (int j = 0; j < (h / hf); ++j) { \ |
528 | 0 | unsigned int sse2; \ |
529 | 0 | const int se2 = aom_sub_pixel_variance##wf##x##hf##_imp_avx2( \ |
530 | 0 | src_ptr, src_stride, x_offset, y_offset, dst_ptr, dst_stride, \ |
531 | 0 | &sse2); \ |
532 | 0 | dst_ptr += hf * dst_stride; \ |
533 | 0 | src_ptr += hf * src_stride; \ |
534 | 0 | se += se2; \ |
535 | 0 | sse += sse2; \ |
536 | 0 | } \ |
537 | 0 | src += wf; \ |
538 | 0 | dst += wf; \ |
539 | 0 | } \ |
540 | 0 | *sse_ptr = sse; \ |
541 | 0 | return sse - (unsigned int)(((int64_t)se * se) >> (wlog2 + hlog2)); \ |
542 | 0 | } Unexecuted instantiation: aom_sub_pixel_variance128x128_avx2 Unexecuted instantiation: aom_sub_pixel_variance128x64_avx2 Unexecuted instantiation: aom_sub_pixel_variance64x128_avx2 Unexecuted instantiation: aom_sub_pixel_variance64x64_avx2 Unexecuted instantiation: aom_sub_pixel_variance64x32_avx2 |
543 | | |
544 | | // Note: hf = AOMMIN(h, 64) to avoid overflow in helper by capping height. |
545 | | AOM_SUB_PIXEL_VAR_AVX2(128, 128, 32, 64, 7, 7) |
546 | | AOM_SUB_PIXEL_VAR_AVX2(128, 64, 32, 64, 7, 6) |
547 | | AOM_SUB_PIXEL_VAR_AVX2(64, 128, 32, 64, 6, 7) |
548 | | AOM_SUB_PIXEL_VAR_AVX2(64, 64, 32, 64, 6, 6) |
549 | | AOM_SUB_PIXEL_VAR_AVX2(64, 32, 32, 32, 6, 5) |
550 | | |
551 | | #define MAKE_SUB_PIXEL_VAR_16XH(height, log2height) \ |
552 | | unsigned int aom_sub_pixel_variance16x##height##_avx2( \ |
553 | | const uint8_t *src, int src_stride, int x_offset, int y_offset, \ |
554 | 0 | const uint8_t *dst, int dst_stride, unsigned int *sse) { \ |
555 | 0 | __m256i src_reg, dst_reg, exp_src_lo, exp_src_hi, exp_dst_lo, exp_dst_hi; \ |
556 | 0 | __m256i sse_reg, sum_reg, sse_reg_hi, res_cmp, sum_reg_lo, sum_reg_hi; \ |
557 | 0 | __m256i zero_reg; \ |
558 | 0 | int i, sum; \ |
559 | 0 | sum_reg = _mm256_setzero_si256(); \ |
560 | 0 | sse_reg = _mm256_setzero_si256(); \ |
561 | 0 | zero_reg = _mm256_setzero_si256(); \ |
562 | 0 | \ |
563 | 0 | /* x_offset = 0 and y_offset = 0 */ \ |
564 | 0 | if (x_offset == 0) { \ |
565 | 0 | if (y_offset == 0) { \ |
566 | 0 | for (i = 0; i < height; i += 2) { \ |
567 | 0 | LOAD_SRC_DST_INSERT(src_stride, dst_stride) \ |
568 | 0 | /* expend each byte to 2 bytes */ \ |
569 | 0 | MERGE_WITH_SRC(src_reg, zero_reg) \ |
570 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
571 | 0 | src += (src_stride << 1); \ |
572 | 0 | dst += (dst_stride << 1); \ |
573 | 0 | } \ |
574 | 0 | /* x_offset = 0 and y_offset = 4 */ \ |
575 | 0 | } else if (y_offset == 4) { \ |
576 | 0 | __m256i src_next_reg; \ |
577 | 0 | for (i = 0; i < height; i += 2) { \ |
578 | 0 | LOAD_SRC_DST_INSERT(src_stride, dst_stride) \ |
579 | 0 | AVG_NEXT_SRC_INSERT(src_reg, src_stride) \ |
580 | 0 | /* expend each byte to 2 bytes */ \ |
581 | 0 | MERGE_WITH_SRC(src_reg, zero_reg) \ |
582 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
583 | 0 | src += (src_stride << 1); \ |
584 | 0 | dst += (dst_stride << 1); \ |
585 | 0 | } \ |
586 | 0 | /* x_offset = 0 and y_offset = bilin interpolation */ \ |
587 | 0 | } else { \ |
588 | 0 | __m256i filter, pw8, src_next_reg; \ |
589 | 0 | y_offset <<= 5; \ |
590 | 0 | filter = _mm256_load_si256( \ |
591 | 0 | (__m256i const *)(bilinear_filters_avx2 + y_offset)); \ |
592 | 0 | pw8 = _mm256_set1_epi16(8); \ |
593 | 0 | for (i = 0; i < height; i += 2) { \ |
594 | 0 | LOAD_SRC_DST_INSERT(src_stride, dst_stride) \ |
595 | 0 | MERGE_NEXT_SRC_INSERT(src_reg, src_stride) \ |
596 | 0 | FILTER_SRC(filter) \ |
597 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
598 | 0 | src += (src_stride << 1); \ |
599 | 0 | dst += (dst_stride << 1); \ |
600 | 0 | } \ |
601 | 0 | } \ |
602 | 0 | /* x_offset = 4 and y_offset = 0 */ \ |
603 | 0 | } else if (x_offset == 4) { \ |
604 | 0 | if (y_offset == 0) { \ |
605 | 0 | __m256i src_next_reg; \ |
606 | 0 | for (i = 0; i < height; i += 2) { \ |
607 | 0 | LOAD_SRC_NEXT_BYTE_INSERT \ |
608 | 0 | LOAD_DST_INSERT \ |
609 | 0 | /* average between current and next stride source */ \ |
610 | 0 | src_reg = _mm256_avg_epu8(src_reg, src_next_reg); \ |
611 | 0 | /* expand each byte to 2 bytes */ \ |
612 | 0 | MERGE_WITH_SRC(src_reg, zero_reg) \ |
613 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
614 | 0 | src += (src_stride << 1); \ |
615 | 0 | dst += (dst_stride << 1); \ |
616 | 0 | } \ |
617 | 0 | /* x_offset = 4 and y_offset = 4 */ \ |
618 | 0 | } else if (y_offset == 4) { \ |
619 | 0 | __m256i src_next_reg, src_avg, src_temp; \ |
620 | 0 | /* load and insert source and next row source */ \ |
621 | 0 | LOAD_SRC_NEXT_BYTE_INSERT \ |
622 | 0 | src_avg = _mm256_avg_epu8(src_reg, src_next_reg); \ |
623 | 0 | src += src_stride << 1; \ |
624 | 0 | for (i = 0; i < height - 2; i += 2) { \ |
625 | 0 | LOAD_SRC_NEXT_BYTE_INSERT \ |
626 | 0 | src_next_reg = _mm256_avg_epu8(src_reg, src_next_reg); \ |
627 | 0 | src_temp = _mm256_permute2x128_si256(src_avg, src_next_reg, 0x21); \ |
628 | 0 | src_temp = _mm256_avg_epu8(src_avg, src_temp); \ |
629 | 0 | LOAD_DST_INSERT \ |
630 | 0 | /* expand each byte to 2 bytes */ \ |
631 | 0 | MERGE_WITH_SRC(src_temp, zero_reg) \ |
632 | 0 | /* save current source average */ \ |
633 | 0 | src_avg = src_next_reg; \ |
634 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
635 | 0 | dst += dst_stride << 1; \ |
636 | 0 | src += src_stride << 1; \ |
637 | 0 | } \ |
638 | 0 | /* last 2 rows processing happens here */ \ |
639 | 0 | __m128i src_reg_0 = _mm_loadu_si128((__m128i *)(src)); \ |
640 | 0 | __m128i src_reg_1 = _mm_loadu_si128((__m128i *)(src + 1)); \ |
641 | 0 | src_reg_0 = _mm_avg_epu8(src_reg_0, src_reg_1); \ |
642 | 0 | src_next_reg = _mm256_permute2x128_si256( \ |
643 | 0 | src_avg, _mm256_castsi128_si256(src_reg_0), 0x21); \ |
644 | 0 | LOAD_DST_INSERT \ |
645 | 0 | src_avg = _mm256_avg_epu8(src_avg, src_next_reg); \ |
646 | 0 | MERGE_WITH_SRC(src_avg, zero_reg) \ |
647 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
648 | 0 | } else { \ |
649 | 0 | /* x_offset = 4 and y_offset = bilin interpolation */ \ |
650 | 0 | __m256i filter, pw8, src_next_reg, src_avg, src_temp; \ |
651 | 0 | y_offset <<= 5; \ |
652 | 0 | filter = _mm256_load_si256( \ |
653 | 0 | (__m256i const *)(bilinear_filters_avx2 + y_offset)); \ |
654 | 0 | pw8 = _mm256_set1_epi16(8); \ |
655 | 0 | /* load and insert source and next row source */ \ |
656 | 0 | LOAD_SRC_NEXT_BYTE_INSERT \ |
657 | 0 | src_avg = _mm256_avg_epu8(src_reg, src_next_reg); \ |
658 | 0 | src += src_stride << 1; \ |
659 | 0 | for (i = 0; i < height - 2; i += 2) { \ |
660 | 0 | LOAD_SRC_NEXT_BYTE_INSERT \ |
661 | 0 | src_next_reg = _mm256_avg_epu8(src_reg, src_next_reg); \ |
662 | 0 | src_temp = _mm256_permute2x128_si256(src_avg, src_next_reg, 0x21); \ |
663 | 0 | LOAD_DST_INSERT \ |
664 | 0 | MERGE_WITH_SRC(src_avg, src_temp) \ |
665 | 0 | /* save current source average */ \ |
666 | 0 | src_avg = src_next_reg; \ |
667 | 0 | FILTER_SRC(filter) \ |
668 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
669 | 0 | dst += dst_stride << 1; \ |
670 | 0 | src += src_stride << 1; \ |
671 | 0 | } \ |
672 | 0 | /* last 2 rows processing happens here */ \ |
673 | 0 | __m128i src_reg_0 = _mm_loadu_si128((__m128i *)(src)); \ |
674 | 0 | __m128i src_reg_1 = _mm_loadu_si128((__m128i *)(src + 1)); \ |
675 | 0 | src_reg_0 = _mm_avg_epu8(src_reg_0, src_reg_1); \ |
676 | 0 | src_next_reg = _mm256_permute2x128_si256( \ |
677 | 0 | src_avg, _mm256_castsi128_si256(src_reg_0), 0x21); \ |
678 | 0 | LOAD_DST_INSERT \ |
679 | 0 | MERGE_WITH_SRC(src_avg, src_next_reg) \ |
680 | 0 | FILTER_SRC(filter) \ |
681 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
682 | 0 | } \ |
683 | 0 | /* x_offset = bilin interpolation and y_offset = 0 */ \ |
684 | 0 | } else { \ |
685 | 0 | if (y_offset == 0) { \ |
686 | 0 | __m256i filter, pw8, src_next_reg; \ |
687 | 0 | x_offset <<= 5; \ |
688 | 0 | filter = _mm256_load_si256( \ |
689 | 0 | (__m256i const *)(bilinear_filters_avx2 + x_offset)); \ |
690 | 0 | pw8 = _mm256_set1_epi16(8); \ |
691 | 0 | for (i = 0; i < height; i += 2) { \ |
692 | 0 | LOAD_SRC_DST_INSERT(src_stride, dst_stride) \ |
693 | 0 | MERGE_NEXT_SRC_INSERT(src_reg, 1) \ |
694 | 0 | FILTER_SRC(filter) \ |
695 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
696 | 0 | src += (src_stride << 1); \ |
697 | 0 | dst += (dst_stride << 1); \ |
698 | 0 | } \ |
699 | 0 | /* x_offset = bilin interpolation and y_offset = 4 */ \ |
700 | 0 | } else if (y_offset == 4) { \ |
701 | 0 | __m256i filter, pw8, src_next_reg, src_pack; \ |
702 | 0 | x_offset <<= 5; \ |
703 | 0 | filter = _mm256_load_si256( \ |
704 | 0 | (__m256i const *)(bilinear_filters_avx2 + x_offset)); \ |
705 | 0 | pw8 = _mm256_set1_epi16(8); \ |
706 | 0 | /* load and insert source and next row source */ \ |
707 | 0 | LOAD_SRC_NEXT_BYTE_INSERT \ |
708 | 0 | MERGE_WITH_SRC(src_reg, src_next_reg) \ |
709 | 0 | FILTER_SRC(filter) \ |
710 | 0 | /* convert each 16 bit to 8 bit to each low and high lane source */ \ |
711 | 0 | src_pack = _mm256_packus_epi16(exp_src_lo, exp_src_hi); \ |
712 | 0 | src += src_stride << 1; \ |
713 | 0 | for (i = 0; i < height - 2; i += 2) { \ |
714 | 0 | LOAD_SRC_NEXT_BYTE_INSERT \ |
715 | 0 | LOAD_DST_INSERT \ |
716 | 0 | MERGE_WITH_SRC(src_reg, src_next_reg) \ |
717 | 0 | FILTER_SRC(filter) \ |
718 | 0 | src_reg = _mm256_packus_epi16(exp_src_lo, exp_src_hi); \ |
719 | 0 | src_next_reg = _mm256_permute2x128_si256(src_pack, src_reg, 0x21); \ |
720 | 0 | /* average between previous pack to the current */ \ |
721 | 0 | src_pack = _mm256_avg_epu8(src_pack, src_next_reg); \ |
722 | 0 | MERGE_WITH_SRC(src_pack, zero_reg) \ |
723 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
724 | 0 | src_pack = src_reg; \ |
725 | 0 | src += src_stride << 1; \ |
726 | 0 | dst += dst_stride << 1; \ |
727 | 0 | } \ |
728 | 0 | /* last 2 rows processing happens here */ \ |
729 | 0 | LOAD_SRC_MERGE_128BIT(filter) \ |
730 | 0 | LOAD_DST_INSERT \ |
731 | 0 | FILTER_SRC_128BIT(filter_128bit) \ |
732 | 0 | src_reg_0 = _mm_packus_epi16(src_lo, src_hi); \ |
733 | 0 | src_next_reg = _mm256_permute2x128_si256( \ |
734 | 0 | src_pack, _mm256_castsi128_si256(src_reg_0), 0x21); \ |
735 | 0 | /* average between previous pack to the current */ \ |
736 | 0 | src_pack = _mm256_avg_epu8(src_pack, src_next_reg); \ |
737 | 0 | MERGE_WITH_SRC(src_pack, zero_reg) \ |
738 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
739 | 0 | } else { \ |
740 | 0 | __m256i xfilter, yfilter, pw8, mask_00ff; \ |
741 | 0 | __m256i p0, p1; \ |
742 | 0 | const uint8_t *src_ptr = src; \ |
743 | 0 | x_offset <<= 5; \ |
744 | 0 | xfilter = _mm256_load_si256( \ |
745 | 0 | (__m256i const *)(bilinear_filters_avx2 + x_offset)); \ |
746 | 0 | y_offset <<= 5; \ |
747 | 0 | yfilter = _mm256_load_si256( \ |
748 | 0 | (__m256i const *)(bilinear_filters_avx2 + y_offset)); \ |
749 | 0 | pw8 = _mm256_set1_epi16(8); \ |
750 | 0 | mask_00ff = _mm256_set1_epi16(0x00ff); \ |
751 | 0 | \ |
752 | 0 | { \ |
753 | 0 | __m256i s0 = _mm256_inserti128_si256( \ |
754 | 0 | _mm256_castsi128_si256( \ |
755 | 0 | _mm_loadu_si128((__m128i const *)(src_ptr))), \ |
756 | 0 | _mm_loadu_si128((__m128i const *)(src_ptr + src_stride)), 1); \ |
757 | 0 | __m256i s1 = _mm256_inserti128_si256( \ |
758 | 0 | _mm256_castsi128_si256( \ |
759 | 0 | _mm_loadu_si128((__m128i const *)(src_ptr + 1))), \ |
760 | 0 | _mm_loadu_si128((__m128i const *)(src_ptr + src_stride + 1)), \ |
761 | 0 | 1); \ |
762 | 0 | __m256i he = _mm256_maddubs_epi16(s0, xfilter); \ |
763 | 0 | __m256i ho = _mm256_maddubs_epi16(s1, xfilter); \ |
764 | 0 | he = _mm256_srai_epi16(_mm256_add_epi16(he, pw8), 4); \ |
765 | 0 | ho = _mm256_srai_epi16(_mm256_add_epi16(ho, pw8), 4); \ |
766 | 0 | p0 = _mm256_packus_epi16(he, ho); \ |
767 | 0 | src_ptr += src_stride << 1; \ |
768 | 0 | } \ |
769 | 0 | { \ |
770 | 0 | __m256i s0 = _mm256_inserti128_si256( \ |
771 | 0 | _mm256_castsi128_si256( \ |
772 | 0 | _mm_loadu_si128((__m128i const *)(src_ptr))), \ |
773 | 0 | _mm_loadu_si128((__m128i const *)(src_ptr + src_stride)), 1); \ |
774 | 0 | __m256i s1 = _mm256_inserti128_si256( \ |
775 | 0 | _mm256_castsi128_si256( \ |
776 | 0 | _mm_loadu_si128((__m128i const *)(src_ptr + 1))), \ |
777 | 0 | _mm_loadu_si128((__m128i const *)(src_ptr + src_stride + 1)), \ |
778 | 0 | 1); \ |
779 | 0 | __m256i he = _mm256_maddubs_epi16(s0, xfilter); \ |
780 | 0 | __m256i ho = _mm256_maddubs_epi16(s1, xfilter); \ |
781 | 0 | he = _mm256_srai_epi16(_mm256_add_epi16(he, pw8), 4); \ |
782 | 0 | ho = _mm256_srai_epi16(_mm256_add_epi16(ho, pw8), 4); \ |
783 | 0 | p1 = _mm256_packus_epi16(he, ho); \ |
784 | 0 | src_ptr += src_stride << 1; \ |
785 | 0 | } \ |
786 | 0 | \ |
787 | 0 | for (i = 0; i < height - 4; i += 2) { \ |
788 | 0 | __m256i p2, p_mix, v_ev, v_od, dst_ev, dst_od; \ |
789 | 0 | __m256i diff_ev, diff_od, sum_comb, sse_comb; \ |
790 | 0 | \ |
791 | 0 | { \ |
792 | 0 | __m256i s0 = _mm256_inserti128_si256( \ |
793 | 0 | _mm256_castsi128_si256( \ |
794 | 0 | _mm_loadu_si128((__m128i const *)(src_ptr))), \ |
795 | 0 | _mm_loadu_si128((__m128i const *)(src_ptr + src_stride)), 1); \ |
796 | 0 | __m256i s1 = _mm256_inserti128_si256( \ |
797 | 0 | _mm256_castsi128_si256( \ |
798 | 0 | _mm_loadu_si128((__m128i const *)(src_ptr + 1))), \ |
799 | 0 | _mm_loadu_si128((__m128i const *)(src_ptr + src_stride + 1)), \ |
800 | 0 | 1); \ |
801 | 0 | __m256i he = _mm256_maddubs_epi16(s0, xfilter); \ |
802 | 0 | __m256i ho = _mm256_maddubs_epi16(s1, xfilter); \ |
803 | 0 | he = _mm256_srai_epi16(_mm256_add_epi16(he, pw8), 4); \ |
804 | 0 | ho = _mm256_srai_epi16(_mm256_add_epi16(ho, pw8), 4); \ |
805 | 0 | p2 = _mm256_packus_epi16(he, ho); \ |
806 | 0 | src_ptr += src_stride << 1; \ |
807 | 0 | } \ |
808 | 0 | \ |
809 | 0 | p_mix = _mm256_permute2x128_si256(p0, p1, 0x21); \ |
810 | 0 | v_ev = \ |
811 | 0 | _mm256_maddubs_epi16(_mm256_unpacklo_epi8(p0, p_mix), yfilter); \ |
812 | 0 | v_od = \ |
813 | 0 | _mm256_maddubs_epi16(_mm256_unpackhi_epi8(p0, p_mix), yfilter); \ |
814 | 0 | v_ev = _mm256_srai_epi16(_mm256_add_epi16(v_ev, pw8), 4); \ |
815 | 0 | v_od = _mm256_srai_epi16(_mm256_add_epi16(v_od, pw8), 4); \ |
816 | 0 | \ |
817 | 0 | dst_reg = _mm256_inserti128_si256( \ |
818 | 0 | _mm256_castsi128_si256(_mm_loadu_si128((__m128i const *)(dst))), \ |
819 | 0 | _mm_loadu_si128((__m128i const *)(dst + dst_stride)), 1); \ |
820 | 0 | dst += dst_stride << 1; \ |
821 | 0 | dst_ev = _mm256_and_si256(dst_reg, mask_00ff); \ |
822 | 0 | dst_od = _mm256_srli_epi16(dst_reg, 8); \ |
823 | 0 | diff_ev = _mm256_sub_epi16(v_ev, dst_ev); \ |
824 | 0 | diff_od = _mm256_sub_epi16(v_od, dst_od); \ |
825 | 0 | \ |
826 | 0 | sum_comb = _mm256_add_epi16(diff_ev, diff_od); \ |
827 | 0 | sum_reg = _mm256_add_epi16(sum_reg, sum_comb); \ |
828 | 0 | \ |
829 | 0 | sse_comb = _mm256_add_epi32(_mm256_madd_epi16(diff_ev, diff_ev), \ |
830 | 0 | _mm256_madd_epi16(diff_od, diff_od)); \ |
831 | 0 | sse_reg = _mm256_add_epi32(sse_reg, sse_comb); \ |
832 | 0 | \ |
833 | 0 | p0 = p1; \ |
834 | 0 | p1 = p2; \ |
835 | 0 | } \ |
836 | 0 | \ |
837 | 0 | { \ |
838 | 0 | __m256i p_mix = _mm256_permute2x128_si256(p0, p1, 0x21); \ |
839 | 0 | __m256i v_ev = \ |
840 | 0 | _mm256_maddubs_epi16(_mm256_unpacklo_epi8(p0, p_mix), yfilter); \ |
841 | 0 | __m256i v_od = \ |
842 | 0 | _mm256_maddubs_epi16(_mm256_unpackhi_epi8(p0, p_mix), yfilter); \ |
843 | 0 | __m256i dst_ev, dst_od, diff_ev, diff_od, sum_comb, sse_comb; \ |
844 | 0 | v_ev = _mm256_srai_epi16(_mm256_add_epi16(v_ev, pw8), 4); \ |
845 | 0 | v_od = _mm256_srai_epi16(_mm256_add_epi16(v_od, pw8), 4); \ |
846 | 0 | \ |
847 | 0 | dst_reg = _mm256_inserti128_si256( \ |
848 | 0 | _mm256_castsi128_si256(_mm_loadu_si128((__m128i const *)(dst))), \ |
849 | 0 | _mm_loadu_si128((__m128i const *)(dst + dst_stride)), 1); \ |
850 | 0 | dst += dst_stride << 1; \ |
851 | 0 | dst_ev = _mm256_and_si256(dst_reg, mask_00ff); \ |
852 | 0 | dst_od = _mm256_srli_epi16(dst_reg, 8); \ |
853 | 0 | diff_ev = _mm256_sub_epi16(v_ev, dst_ev); \ |
854 | 0 | diff_od = _mm256_sub_epi16(v_od, dst_od); \ |
855 | 0 | \ |
856 | 0 | sum_comb = _mm256_add_epi16(diff_ev, diff_od); \ |
857 | 0 | sum_reg = _mm256_add_epi16(sum_reg, sum_comb); \ |
858 | 0 | \ |
859 | 0 | sse_comb = _mm256_add_epi32(_mm256_madd_epi16(diff_ev, diff_ev), \ |
860 | 0 | _mm256_madd_epi16(diff_od, diff_od)); \ |
861 | 0 | sse_reg = _mm256_add_epi32(sse_reg, sse_comb); \ |
862 | 0 | } \ |
863 | 0 | { \ |
864 | 0 | __m128i s0 = _mm_loadu_si128((__m128i const *)(src_ptr)); \ |
865 | 0 | __m128i s1 = _mm_loadu_si128((__m128i const *)(src_ptr + 1)); \ |
866 | 0 | __m128i he = _mm_maddubs_epi16(s0, _mm256_castsi256_si128(xfilter)); \ |
867 | 0 | __m128i ho = _mm_maddubs_epi16(s1, _mm256_castsi256_si128(xfilter)); \ |
868 | 0 | __m256i p_last, p_mix, v_ev, v_od, dst_ev, dst_od; \ |
869 | 0 | __m256i diff_ev, diff_od, sum_comb, sse_comb; \ |
870 | 0 | he = _mm_srai_epi16(_mm_add_epi16(he, _mm256_castsi256_si128(pw8)), \ |
871 | 0 | 4); \ |
872 | 0 | ho = _mm_srai_epi16(_mm_add_epi16(ho, _mm256_castsi256_si128(pw8)), \ |
873 | 0 | 4); \ |
874 | 0 | p_last = _mm256_castsi128_si256(_mm_packus_epi16(he, ho)); \ |
875 | 0 | \ |
876 | 0 | p_mix = _mm256_permute2x128_si256(p1, p_last, 0x21); \ |
877 | 0 | v_ev = \ |
878 | 0 | _mm256_maddubs_epi16(_mm256_unpacklo_epi8(p1, p_mix), yfilter); \ |
879 | 0 | v_od = \ |
880 | 0 | _mm256_maddubs_epi16(_mm256_unpackhi_epi8(p1, p_mix), yfilter); \ |
881 | 0 | v_ev = _mm256_srai_epi16(_mm256_add_epi16(v_ev, pw8), 4); \ |
882 | 0 | v_od = _mm256_srai_epi16(_mm256_add_epi16(v_od, pw8), 4); \ |
883 | 0 | \ |
884 | 0 | dst_reg = _mm256_inserti128_si256( \ |
885 | 0 | _mm256_castsi128_si256(_mm_loadu_si128((__m128i const *)(dst))), \ |
886 | 0 | _mm_loadu_si128((__m128i const *)(dst + dst_stride)), 1); \ |
887 | 0 | dst_ev = _mm256_and_si256(dst_reg, mask_00ff); \ |
888 | 0 | dst_od = _mm256_srli_epi16(dst_reg, 8); \ |
889 | 0 | diff_ev = _mm256_sub_epi16(v_ev, dst_ev); \ |
890 | 0 | diff_od = _mm256_sub_epi16(v_od, dst_od); \ |
891 | 0 | \ |
892 | 0 | sum_comb = _mm256_add_epi16(diff_ev, diff_od); \ |
893 | 0 | sum_reg = _mm256_add_epi16(sum_reg, sum_comb); \ |
894 | 0 | \ |
895 | 0 | sse_comb = _mm256_add_epi32(_mm256_madd_epi16(diff_ev, diff_ev), \ |
896 | 0 | _mm256_madd_epi16(diff_od, diff_od)); \ |
897 | 0 | sse_reg = _mm256_add_epi32(sse_reg, sse_comb); \ |
898 | 0 | } \ |
899 | 0 | \ |
900 | 0 | { \ |
901 | 0 | __m256i sse_hi, sum_hi; \ |
902 | 0 | int f_sse, f_sum; \ |
903 | 0 | sum_reg = _mm256_madd_epi16(sum_reg, _mm256_set1_epi16(1)); \ |
904 | 0 | sse_hi = _mm256_srli_si256(sse_reg, 8); \ |
905 | 0 | sum_hi = _mm256_srli_si256(sum_reg, 8); \ |
906 | 0 | sse_reg = _mm256_add_epi32(sse_reg, sse_hi); \ |
907 | 0 | sum_reg = _mm256_add_epi32(sum_reg, sum_hi); \ |
908 | 0 | sse_hi = _mm256_srli_si256(sse_reg, 4); \ |
909 | 0 | sum_hi = _mm256_srli_si256(sum_reg, 4); \ |
910 | 0 | sse_reg = _mm256_add_epi32(sse_reg, sse_hi); \ |
911 | 0 | sum_reg = _mm256_add_epi32(sum_reg, sum_hi); \ |
912 | 0 | f_sse = _mm_cvtsi128_si32(_mm256_castsi256_si128(sse_reg)) + \ |
913 | 0 | _mm_cvtsi128_si32(_mm256_extractf128_si256(sse_reg, 1)); \ |
914 | 0 | f_sum = _mm_cvtsi128_si32(_mm256_castsi256_si128(sum_reg)) + \ |
915 | 0 | _mm_cvtsi128_si32(_mm256_extractf128_si256(sum_reg, 1)); \ |
916 | 0 | *sse = f_sse; \ |
917 | 0 | _mm256_zeroupper(); \ |
918 | 0 | return f_sse - \ |
919 | 0 | (unsigned int)(((int64_t)f_sum * f_sum) >> (4 + log2height)); \ |
920 | 0 | } \ |
921 | 0 | } \ |
922 | 0 | } \ |
923 | 0 | CALC_SUM_AND_SSE \ |
924 | 0 | _mm256_zeroupper(); \ |
925 | 0 | return *sse - (unsigned int)(((int64_t)sum * sum) >> (4 + log2height)); \ |
926 | 0 | } |
927 | | |
928 | 0 | MAKE_SUB_PIXEL_VAR_16XH(32, 5) |
929 | 0 | MAKE_SUB_PIXEL_VAR_16XH(16, 4) |
930 | 0 | MAKE_SUB_PIXEL_VAR_16XH(8, 3) |
931 | | #if !CONFIG_REALTIME_ONLY |
932 | 0 | MAKE_SUB_PIXEL_VAR_16XH(64, 6) |
933 | 0 | MAKE_SUB_PIXEL_VAR_16XH(4, 2) |
934 | | #endif |
935 | | |
936 | | #define MAKE_SUB_PIXEL_AVG_VAR_32XH(height, log2height) \ |
937 | | static int sub_pixel_avg_variance32x##height##_imp_avx2( \ |
938 | | const uint8_t *src, int src_stride, int x_offset, int y_offset, \ |
939 | | const uint8_t *dst, int dst_stride, const uint8_t *sec, int sec_stride, \ |
940 | 0 | unsigned int *sse) { \ |
941 | 0 | __m256i sec_reg; \ |
942 | 0 | __m256i src_reg, dst_reg, exp_src_lo, exp_src_hi, exp_dst_lo, exp_dst_hi; \ |
943 | 0 | __m256i sse_reg, sum_reg, sse_reg_hi, res_cmp, sum_reg_lo, sum_reg_hi; \ |
944 | 0 | __m256i zero_reg; \ |
945 | 0 | int i, sum; \ |
946 | 0 | sum_reg = _mm256_setzero_si256(); \ |
947 | 0 | sse_reg = _mm256_setzero_si256(); \ |
948 | 0 | zero_reg = _mm256_setzero_si256(); \ |
949 | 0 | \ |
950 | 0 | /* x_offset = 0 and y_offset = 0 */ \ |
951 | 0 | if (x_offset == 0) { \ |
952 | 0 | if (y_offset == 0) { \ |
953 | 0 | for (i = 0; i < height; i++) { \ |
954 | 0 | LOAD_SRC_DST \ |
955 | 0 | sec_reg = _mm256_loadu_si256((__m256i const *)(sec)); \ |
956 | 0 | src_reg = _mm256_avg_epu8(src_reg, sec_reg); \ |
957 | 0 | sec += sec_stride; \ |
958 | 0 | /* expend each byte to 2 bytes */ \ |
959 | 0 | MERGE_WITH_SRC(src_reg, zero_reg) \ |
960 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
961 | 0 | src += src_stride; \ |
962 | 0 | dst += dst_stride; \ |
963 | 0 | } \ |
964 | 0 | } else if (y_offset == 4) { \ |
965 | 0 | __m256i src_next_reg; \ |
966 | 0 | for (i = 0; i < height; i++) { \ |
967 | 0 | LOAD_SRC_DST \ |
968 | 0 | AVG_NEXT_SRC(src_reg, src_stride) \ |
969 | 0 | sec_reg = _mm256_loadu_si256((__m256i const *)(sec)); \ |
970 | 0 | src_reg = _mm256_avg_epu8(src_reg, sec_reg); \ |
971 | 0 | sec += sec_stride; \ |
972 | 0 | /* expend each byte to 2 bytes */ \ |
973 | 0 | MERGE_WITH_SRC(src_reg, zero_reg) \ |
974 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
975 | 0 | src += src_stride; \ |
976 | 0 | dst += dst_stride; \ |
977 | 0 | } \ |
978 | 0 | /* x_offset = 0 and y_offset = bilin interpolation */ \ |
979 | 0 | } else { \ |
980 | 0 | __m256i filter, pw8, src_next_reg; \ |
981 | 0 | \ |
982 | 0 | y_offset <<= 5; \ |
983 | 0 | filter = _mm256_load_si256( \ |
984 | 0 | (__m256i const *)(bilinear_filters_avx2 + y_offset)); \ |
985 | 0 | pw8 = _mm256_set1_epi16(8); \ |
986 | 0 | for (i = 0; i < height; i++) { \ |
987 | 0 | LOAD_SRC_DST \ |
988 | 0 | MERGE_NEXT_SRC(src_reg, src_stride) \ |
989 | 0 | FILTER_SRC(filter) \ |
990 | 0 | src_reg = _mm256_packus_epi16(exp_src_lo, exp_src_hi); \ |
991 | 0 | sec_reg = _mm256_loadu_si256((__m256i const *)(sec)); \ |
992 | 0 | src_reg = _mm256_avg_epu8(src_reg, sec_reg); \ |
993 | 0 | sec += sec_stride; \ |
994 | 0 | MERGE_WITH_SRC(src_reg, zero_reg) \ |
995 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
996 | 0 | src += src_stride; \ |
997 | 0 | dst += dst_stride; \ |
998 | 0 | } \ |
999 | 0 | } \ |
1000 | 0 | /* x_offset = 4 and y_offset = 0 */ \ |
1001 | 0 | } else if (x_offset == 4) { \ |
1002 | 0 | if (y_offset == 0) { \ |
1003 | 0 | __m256i src_next_reg; \ |
1004 | 0 | for (i = 0; i < height; i++) { \ |
1005 | 0 | LOAD_SRC_DST \ |
1006 | 0 | AVG_NEXT_SRC(src_reg, 1) \ |
1007 | 0 | sec_reg = _mm256_loadu_si256((__m256i const *)(sec)); \ |
1008 | 0 | src_reg = _mm256_avg_epu8(src_reg, sec_reg); \ |
1009 | 0 | sec += sec_stride; \ |
1010 | 0 | /* expand each byte to 2 bytes */ \ |
1011 | 0 | MERGE_WITH_SRC(src_reg, zero_reg) \ |
1012 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
1013 | 0 | src += src_stride; \ |
1014 | 0 | dst += dst_stride; \ |
1015 | 0 | } \ |
1016 | 0 | /* x_offset = 4 and y_offset = 4 */ \ |
1017 | 0 | } else if (y_offset == 4) { \ |
1018 | 0 | __m256i src_next_reg, src_avg; \ |
1019 | 0 | /* load source and another source starting from the next */ \ |
1020 | 0 | /* following byte */ \ |
1021 | 0 | src_reg = _mm256_loadu_si256((__m256i const *)(src)); \ |
1022 | 0 | AVG_NEXT_SRC(src_reg, 1) \ |
1023 | 0 | for (i = 0; i < height; i++) { \ |
1024 | 0 | /* save current source average */ \ |
1025 | 0 | src_avg = src_reg; \ |
1026 | 0 | src += src_stride; \ |
1027 | 0 | LOAD_SRC_DST \ |
1028 | 0 | AVG_NEXT_SRC(src_reg, 1) \ |
1029 | 0 | /* average between previous average to current average */ \ |
1030 | 0 | src_avg = _mm256_avg_epu8(src_avg, src_reg); \ |
1031 | 0 | sec_reg = _mm256_loadu_si256((__m256i const *)(sec)); \ |
1032 | 0 | src_avg = _mm256_avg_epu8(src_avg, sec_reg); \ |
1033 | 0 | sec += sec_stride; \ |
1034 | 0 | /* expand each byte to 2 bytes */ \ |
1035 | 0 | MERGE_WITH_SRC(src_avg, zero_reg) \ |
1036 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
1037 | 0 | dst += dst_stride; \ |
1038 | 0 | } \ |
1039 | 0 | /* x_offset = 4 and y_offset = bilin interpolation */ \ |
1040 | 0 | } else { \ |
1041 | 0 | __m256i filter, pw8, src_next_reg, src_avg; \ |
1042 | 0 | y_offset <<= 5; \ |
1043 | 0 | filter = _mm256_load_si256( \ |
1044 | 0 | (__m256i const *)(bilinear_filters_avx2 + y_offset)); \ |
1045 | 0 | pw8 = _mm256_set1_epi16(8); \ |
1046 | 0 | /* load source and another source starting from the next */ \ |
1047 | 0 | /* following byte */ \ |
1048 | 0 | src_reg = _mm256_loadu_si256((__m256i const *)(src)); \ |
1049 | 0 | AVG_NEXT_SRC(src_reg, 1) \ |
1050 | 0 | for (i = 0; i < height; i++) { \ |
1051 | 0 | /* save current source average */ \ |
1052 | 0 | src_avg = src_reg; \ |
1053 | 0 | src += src_stride; \ |
1054 | 0 | LOAD_SRC_DST \ |
1055 | 0 | AVG_NEXT_SRC(src_reg, 1) \ |
1056 | 0 | MERGE_WITH_SRC(src_avg, src_reg) \ |
1057 | 0 | FILTER_SRC(filter) \ |
1058 | 0 | src_avg = _mm256_packus_epi16(exp_src_lo, exp_src_hi); \ |
1059 | 0 | sec_reg = _mm256_loadu_si256((__m256i const *)(sec)); \ |
1060 | 0 | src_avg = _mm256_avg_epu8(src_avg, sec_reg); \ |
1061 | 0 | /* expand each byte to 2 bytes */ \ |
1062 | 0 | MERGE_WITH_SRC(src_avg, zero_reg) \ |
1063 | 0 | sec += sec_stride; \ |
1064 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
1065 | 0 | dst += dst_stride; \ |
1066 | 0 | } \ |
1067 | 0 | } \ |
1068 | 0 | /* x_offset = bilin interpolation and y_offset = 0 */ \ |
1069 | 0 | } else { \ |
1070 | 0 | if (y_offset == 0) { \ |
1071 | 0 | __m256i filter, pw8, src_next_reg; \ |
1072 | 0 | x_offset <<= 5; \ |
1073 | 0 | filter = _mm256_load_si256( \ |
1074 | 0 | (__m256i const *)(bilinear_filters_avx2 + x_offset)); \ |
1075 | 0 | pw8 = _mm256_set1_epi16(8); \ |
1076 | 0 | for (i = 0; i < height; i++) { \ |
1077 | 0 | LOAD_SRC_DST \ |
1078 | 0 | MERGE_NEXT_SRC(src_reg, 1) \ |
1079 | 0 | FILTER_SRC(filter) \ |
1080 | 0 | src_reg = _mm256_packus_epi16(exp_src_lo, exp_src_hi); \ |
1081 | 0 | sec_reg = _mm256_loadu_si256((__m256i const *)(sec)); \ |
1082 | 0 | src_reg = _mm256_avg_epu8(src_reg, sec_reg); \ |
1083 | 0 | MERGE_WITH_SRC(src_reg, zero_reg) \ |
1084 | 0 | sec += sec_stride; \ |
1085 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
1086 | 0 | src += src_stride; \ |
1087 | 0 | dst += dst_stride; \ |
1088 | 0 | } \ |
1089 | 0 | /* x_offset = bilin interpolation and y_offset = 4 */ \ |
1090 | 0 | } else if (y_offset == 4) { \ |
1091 | 0 | __m256i filter, pw8, src_next_reg, src_pack; \ |
1092 | 0 | x_offset <<= 5; \ |
1093 | 0 | filter = _mm256_load_si256( \ |
1094 | 0 | (__m256i const *)(bilinear_filters_avx2 + x_offset)); \ |
1095 | 0 | pw8 = _mm256_set1_epi16(8); \ |
1096 | 0 | src_reg = _mm256_loadu_si256((__m256i const *)(src)); \ |
1097 | 0 | MERGE_NEXT_SRC(src_reg, 1) \ |
1098 | 0 | FILTER_SRC(filter) \ |
1099 | 0 | /* convert each 16 bit to 8 bit to each low and high lane source */ \ |
1100 | 0 | src_pack = _mm256_packus_epi16(exp_src_lo, exp_src_hi); \ |
1101 | 0 | for (i = 0; i < height; i++) { \ |
1102 | 0 | src += src_stride; \ |
1103 | 0 | LOAD_SRC_DST \ |
1104 | 0 | MERGE_NEXT_SRC(src_reg, 1) \ |
1105 | 0 | FILTER_SRC(filter) \ |
1106 | 0 | src_reg = _mm256_packus_epi16(exp_src_lo, exp_src_hi); \ |
1107 | 0 | /* average between previous pack to the current */ \ |
1108 | 0 | src_pack = _mm256_avg_epu8(src_pack, src_reg); \ |
1109 | 0 | sec_reg = _mm256_loadu_si256((__m256i const *)(sec)); \ |
1110 | 0 | src_pack = _mm256_avg_epu8(src_pack, sec_reg); \ |
1111 | 0 | sec += sec_stride; \ |
1112 | 0 | MERGE_WITH_SRC(src_pack, zero_reg) \ |
1113 | 0 | src_pack = src_reg; \ |
1114 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
1115 | 0 | dst += dst_stride; \ |
1116 | 0 | } \ |
1117 | 0 | /* x_offset = bilin interpolation and y_offset = bilin interpolation \ |
1118 | 0 | */ \ |
1119 | 0 | } else { \ |
1120 | 0 | __m256i xfilter, yfilter, pw8, src_next_reg, src_pack; \ |
1121 | 0 | x_offset <<= 5; \ |
1122 | 0 | xfilter = _mm256_load_si256( \ |
1123 | 0 | (__m256i const *)(bilinear_filters_avx2 + x_offset)); \ |
1124 | 0 | y_offset <<= 5; \ |
1125 | 0 | yfilter = _mm256_load_si256( \ |
1126 | 0 | (__m256i const *)(bilinear_filters_avx2 + y_offset)); \ |
1127 | 0 | pw8 = _mm256_set1_epi16(8); \ |
1128 | 0 | /* load source and another source starting from the next */ \ |
1129 | 0 | /* following byte */ \ |
1130 | 0 | src_reg = _mm256_loadu_si256((__m256i const *)(src)); \ |
1131 | 0 | MERGE_NEXT_SRC(src_reg, 1) \ |
1132 | 0 | \ |
1133 | 0 | FILTER_SRC(xfilter) \ |
1134 | 0 | /* convert each 16 bit to 8 bit to each low and high lane source */ \ |
1135 | 0 | src_pack = _mm256_packus_epi16(exp_src_lo, exp_src_hi); \ |
1136 | 0 | for (i = 0; i < height; i++) { \ |
1137 | 0 | src += src_stride; \ |
1138 | 0 | LOAD_SRC_DST \ |
1139 | 0 | MERGE_NEXT_SRC(src_reg, 1) \ |
1140 | 0 | FILTER_SRC(xfilter) \ |
1141 | 0 | src_reg = _mm256_packus_epi16(exp_src_lo, exp_src_hi); \ |
1142 | 0 | /* merge previous pack to current pack source */ \ |
1143 | 0 | MERGE_WITH_SRC(src_pack, src_reg) \ |
1144 | 0 | /* filter the source */ \ |
1145 | 0 | FILTER_SRC(yfilter) \ |
1146 | 0 | src_pack = _mm256_packus_epi16(exp_src_lo, exp_src_hi); \ |
1147 | 0 | sec_reg = _mm256_loadu_si256((__m256i const *)(sec)); \ |
1148 | 0 | src_pack = _mm256_avg_epu8(src_pack, sec_reg); \ |
1149 | 0 | MERGE_WITH_SRC(src_pack, zero_reg) \ |
1150 | 0 | src_pack = src_reg; \ |
1151 | 0 | sec += sec_stride; \ |
1152 | 0 | CALC_SUM_SSE_INSIDE_LOOP \ |
1153 | 0 | dst += dst_stride; \ |
1154 | 0 | } \ |
1155 | 0 | } \ |
1156 | 0 | } \ |
1157 | 0 | CALC_SUM_AND_SSE \ |
1158 | 0 | _mm256_zeroupper(); \ |
1159 | 0 | return sum; \ |
1160 | 0 | } \ |
1161 | | unsigned int aom_sub_pixel_avg_variance32x##height##_avx2( \ |
1162 | | const uint8_t *src, int src_stride, int x_offset, int y_offset, \ |
1163 | | const uint8_t *dst, int dst_stride, unsigned int *sse, \ |
1164 | 0 | const uint8_t *sec_ptr) { \ |
1165 | 0 | const int sum = sub_pixel_avg_variance32x##height##_imp_avx2( \ |
1166 | 0 | src, src_stride, x_offset, y_offset, dst, dst_stride, sec_ptr, 32, \ |
1167 | 0 | sse); \ |
1168 | 0 | return *sse - (unsigned int)(((int64_t)sum * sum) >> (5 + log2height)); \ |
1169 | 0 | } Unexecuted instantiation: aom_sub_pixel_avg_variance32x64_avx2 Unexecuted instantiation: aom_sub_pixel_avg_variance32x32_avx2 Unexecuted instantiation: aom_sub_pixel_avg_variance32x16_avx2 |
1170 | | |
1171 | 0 | MAKE_SUB_PIXEL_AVG_VAR_32XH(64, 6) |
1172 | 0 | MAKE_SUB_PIXEL_AVG_VAR_32XH(32, 5) |
1173 | 0 | MAKE_SUB_PIXEL_AVG_VAR_32XH(16, 4) |
1174 | | |
1175 | | #define AOM_SUB_PIXEL_AVG_VAR_AVX2(w, h, wf, hf, wlog2, hlog2) \ |
1176 | | unsigned int aom_sub_pixel_avg_variance##w##x##h##_avx2( \ |
1177 | | const uint8_t *src, int src_stride, int x_offset, int y_offset, \ |
1178 | | const uint8_t *dst, int dst_stride, unsigned int *sse_ptr, \ |
1179 | 0 | const uint8_t *sec) { \ |
1180 | 0 | unsigned int sse = 0; \ |
1181 | 0 | int se = 0; \ |
1182 | 0 | for (int i = 0; i < (w / wf); ++i) { \ |
1183 | 0 | const uint8_t *src_ptr = src; \ |
1184 | 0 | const uint8_t *dst_ptr = dst; \ |
1185 | 0 | const uint8_t *sec_ptr = sec; \ |
1186 | 0 | for (int j = 0; j < (h / hf); ++j) { \ |
1187 | 0 | unsigned int sse2; \ |
1188 | 0 | const int se2 = sub_pixel_avg_variance##wf##x##hf##_imp_avx2( \ |
1189 | 0 | src_ptr, src_stride, x_offset, y_offset, dst_ptr, dst_stride, \ |
1190 | 0 | sec_ptr, w, &sse2); \ |
1191 | 0 | dst_ptr += hf * dst_stride; \ |
1192 | 0 | src_ptr += hf * src_stride; \ |
1193 | 0 | sec_ptr += hf * w; \ |
1194 | 0 | se += se2; \ |
1195 | 0 | sse += sse2; \ |
1196 | 0 | } \ |
1197 | 0 | src += wf; \ |
1198 | 0 | dst += wf; \ |
1199 | 0 | sec += wf; \ |
1200 | 0 | } \ |
1201 | 0 | *sse_ptr = sse; \ |
1202 | 0 | return sse - (unsigned int)(((int64_t)se * se) >> (wlog2 + hlog2)); \ |
1203 | 0 | } Unexecuted instantiation: aom_sub_pixel_avg_variance128x128_avx2 Unexecuted instantiation: aom_sub_pixel_avg_variance128x64_avx2 Unexecuted instantiation: aom_sub_pixel_avg_variance64x128_avx2 Unexecuted instantiation: aom_sub_pixel_avg_variance64x64_avx2 Unexecuted instantiation: aom_sub_pixel_avg_variance64x32_avx2 |
1204 | | |
1205 | | // Note: hf = AOMMIN(h, 64) to avoid overflow in helper by capping height. |
1206 | | AOM_SUB_PIXEL_AVG_VAR_AVX2(128, 128, 32, 64, 7, 7) |
1207 | | AOM_SUB_PIXEL_AVG_VAR_AVX2(128, 64, 32, 64, 7, 6) |
1208 | | AOM_SUB_PIXEL_AVG_VAR_AVX2(64, 128, 32, 64, 6, 7) |
1209 | | AOM_SUB_PIXEL_AVG_VAR_AVX2(64, 64, 32, 64, 6, 6) |
1210 | | AOM_SUB_PIXEL_AVG_VAR_AVX2(64, 32, 32, 32, 6, 5) |