/src/aom/aom_dsp/x86/highbd_variance_avx2.c
Line | Count | Source |
1 | | /* |
2 | | * Copyright (c) 2020, Alliance for Open Media. All rights reserved. |
3 | | * |
4 | | * This source code is subject to the terms of the BSD 2 Clause License and |
5 | | * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
6 | | * was not distributed with this source code in the LICENSE file, you can |
7 | | * obtain it at www.aomedia.org/license/software. If the Alliance for Open |
8 | | * Media Patent License 1.0 was not distributed with this source code in the |
9 | | * PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
10 | | */ |
11 | | |
12 | | #include <assert.h> |
13 | | #include <immintrin.h> // AVX2 |
14 | | |
15 | | #include "config/aom_dsp_rtcd.h" |
16 | | #include "aom_dsp/aom_filter.h" |
17 | | #include "aom_dsp/x86/synonyms.h" |
18 | | |
19 | | typedef void (*high_variance_fn_t)(const uint16_t *src, int src_stride, |
20 | | const uint16_t *ref, int ref_stride, |
21 | | uint32_t *sse, int *sum); |
22 | | |
23 | | static uint32_t aom_highbd_var_filter_block2d_bil_avx2( |
24 | | const uint8_t *src_ptr8, unsigned int src_pixels_per_line, int pixel_step, |
25 | | unsigned int output_height, unsigned int output_width, |
26 | | const uint32_t xoffset, const uint32_t yoffset, const uint8_t *dst_ptr8, |
27 | 0 | int dst_stride, uint32_t *sse) { |
28 | 0 | const __m256i filter1 = |
29 | 0 | _mm256_set1_epi32((int)(bilinear_filters_2t[xoffset][1] << 16) | |
30 | 0 | bilinear_filters_2t[xoffset][0]); |
31 | 0 | const __m256i filter2 = |
32 | 0 | _mm256_set1_epi32((int)(bilinear_filters_2t[yoffset][1] << 16) | |
33 | 0 | bilinear_filters_2t[yoffset][0]); |
34 | 0 | const __m256i one = _mm256_set1_epi16(1); |
35 | 0 | const int bitshift = 0x40; |
36 | 0 | (void)pixel_step; |
37 | 0 | unsigned int i, j, prev = 0, curr = 2; |
38 | 0 | uint16_t *src_ptr = CONVERT_TO_SHORTPTR(src_ptr8); |
39 | 0 | uint16_t *dst_ptr = CONVERT_TO_SHORTPTR(dst_ptr8); |
40 | 0 | uint16_t *src_ptr_ref = src_ptr; |
41 | 0 | uint16_t *dst_ptr_ref = dst_ptr; |
42 | 0 | int64_t sum_long = 0; |
43 | 0 | uint64_t sse_long = 0; |
44 | 0 | unsigned int rshift = 0, inc = 1; |
45 | 0 | __m256i rbias = _mm256_set1_epi32(bitshift); |
46 | 0 | __m256i opointer[8]; |
47 | 0 | unsigned int range; |
48 | 0 | if (xoffset == 0) { |
49 | 0 | if (yoffset == 0) { // xoffset==0 && yoffset==0 |
50 | 0 | range = output_width / 16; |
51 | 0 | if (output_height == 8) inc = 2; |
52 | 0 | if (output_height == 4) inc = 4; |
53 | 0 | for (j = 0; j < range * output_height * inc / 16; j++) { |
54 | 0 | if (j % (output_height * inc / 16) == 0) { |
55 | 0 | src_ptr = src_ptr_ref; |
56 | 0 | src_ptr_ref += 16; |
57 | 0 | dst_ptr = dst_ptr_ref; |
58 | 0 | dst_ptr_ref += 16; |
59 | 0 | } |
60 | 0 | __m256i sum1 = _mm256_setzero_si256(); |
61 | 0 | __m256i sse1 = _mm256_setzero_si256(); |
62 | 0 | for (i = 0; i < 16 / inc; ++i) { |
63 | 0 | __m256i V_S_SRC = _mm256_loadu_si256((const __m256i *)src_ptr); |
64 | 0 | src_ptr += src_pixels_per_line; |
65 | 0 | __m256i V_D_DST = _mm256_loadu_si256((const __m256i *)dst_ptr); |
66 | 0 | dst_ptr += dst_stride; |
67 | |
|
68 | 0 | __m256i V_R_SUB = _mm256_sub_epi16(V_S_SRC, V_D_DST); |
69 | 0 | __m256i V_R_MAD = _mm256_madd_epi16(V_R_SUB, V_R_SUB); |
70 | |
|
71 | 0 | sum1 = _mm256_add_epi16(sum1, V_R_SUB); |
72 | 0 | sse1 = _mm256_add_epi32(sse1, V_R_MAD); |
73 | 0 | } |
74 | |
|
75 | 0 | __m256i v_sum0 = _mm256_madd_epi16(sum1, one); |
76 | 0 | __m256i v_d_l = _mm256_unpacklo_epi32(v_sum0, sse1); |
77 | 0 | __m256i v_d_h = _mm256_unpackhi_epi32(v_sum0, sse1); |
78 | 0 | __m256i v_d_lh = _mm256_add_epi32(v_d_l, v_d_h); |
79 | 0 | const __m128i v_d0_d = _mm256_castsi256_si128(v_d_lh); |
80 | 0 | const __m128i v_d1_d = _mm256_extracti128_si256(v_d_lh, 1); |
81 | 0 | __m128i v_d = _mm_add_epi32(v_d0_d, v_d1_d); |
82 | 0 | v_d = _mm_add_epi32(v_d, _mm_srli_si128(v_d, 8)); |
83 | 0 | sum_long += _mm_extract_epi32(v_d, 0); |
84 | 0 | sse_long += _mm_extract_epi32(v_d, 1); |
85 | 0 | } |
86 | |
|
87 | 0 | rshift = get_msb(output_height) + get_msb(output_width); |
88 | |
|
89 | 0 | } else if (yoffset == 4) { // xoffset==0 && yoffset==4 |
90 | 0 | range = output_width / 16; |
91 | 0 | if (output_height == 8) inc = 2; |
92 | 0 | if (output_height == 4) inc = 4; |
93 | 0 | for (j = 0; j < range * output_height * inc / 16; j++) { |
94 | 0 | if (j % (output_height * inc / 16) == 0) { |
95 | 0 | src_ptr = src_ptr_ref; |
96 | 0 | src_ptr_ref += 16; |
97 | 0 | dst_ptr = dst_ptr_ref; |
98 | 0 | dst_ptr_ref += 16; |
99 | |
|
100 | 0 | opointer[0] = _mm256_loadu_si256((const __m256i *)src_ptr); |
101 | 0 | src_ptr += src_pixels_per_line; |
102 | 0 | curr = 0; |
103 | 0 | } |
104 | |
|
105 | 0 | __m256i sum1 = _mm256_setzero_si256(); |
106 | 0 | __m256i sse1 = _mm256_setzero_si256(); |
107 | |
|
108 | 0 | for (i = 0; i < 16 / inc; ++i) { |
109 | 0 | prev = curr; |
110 | 0 | curr = (curr == 0) ? 1 : 0; |
111 | 0 | opointer[curr] = _mm256_loadu_si256((const __m256i *)src_ptr); |
112 | 0 | src_ptr += src_pixels_per_line; |
113 | |
|
114 | 0 | __m256i V_S_SRC = _mm256_avg_epu16(opointer[curr], opointer[prev]); |
115 | |
|
116 | 0 | __m256i V_D_DST = _mm256_loadu_si256((const __m256i *)dst_ptr); |
117 | 0 | dst_ptr += dst_stride; |
118 | 0 | __m256i V_R_SUB = _mm256_sub_epi16(V_S_SRC, V_D_DST); |
119 | 0 | __m256i V_R_MAD = _mm256_madd_epi16(V_R_SUB, V_R_SUB); |
120 | 0 | sum1 = _mm256_add_epi16(sum1, V_R_SUB); |
121 | 0 | sse1 = _mm256_add_epi32(sse1, V_R_MAD); |
122 | 0 | } |
123 | |
|
124 | 0 | __m256i v_sum0 = _mm256_madd_epi16(sum1, one); |
125 | 0 | __m256i v_d_l = _mm256_unpacklo_epi32(v_sum0, sse1); |
126 | 0 | __m256i v_d_h = _mm256_unpackhi_epi32(v_sum0, sse1); |
127 | 0 | __m256i v_d_lh = _mm256_add_epi32(v_d_l, v_d_h); |
128 | 0 | const __m128i v_d0_d = _mm256_castsi256_si128(v_d_lh); |
129 | 0 | const __m128i v_d1_d = _mm256_extracti128_si256(v_d_lh, 1); |
130 | 0 | __m128i v_d = _mm_add_epi32(v_d0_d, v_d1_d); |
131 | 0 | v_d = _mm_add_epi32(v_d, _mm_srli_si128(v_d, 8)); |
132 | 0 | sum_long += _mm_extract_epi32(v_d, 0); |
133 | 0 | sse_long += _mm_extract_epi32(v_d, 1); |
134 | 0 | } |
135 | |
|
136 | 0 | rshift = get_msb(output_height) + get_msb(output_width); |
137 | |
|
138 | 0 | } else { // xoffset==0 && yoffset==1,2,3,5,6,7 |
139 | 0 | range = output_width / 16; |
140 | 0 | if (output_height == 8) inc = 2; |
141 | 0 | if (output_height == 4) inc = 4; |
142 | 0 | for (j = 0; j < range * output_height * inc / 16; j++) { |
143 | 0 | if (j % (output_height * inc / 16) == 0) { |
144 | 0 | src_ptr = src_ptr_ref; |
145 | 0 | src_ptr_ref += 16; |
146 | 0 | dst_ptr = dst_ptr_ref; |
147 | 0 | dst_ptr_ref += 16; |
148 | |
|
149 | 0 | opointer[0] = _mm256_loadu_si256((const __m256i *)src_ptr); |
150 | 0 | src_ptr += src_pixels_per_line; |
151 | 0 | curr = 0; |
152 | 0 | } |
153 | |
|
154 | 0 | __m256i sum1 = _mm256_setzero_si256(); |
155 | 0 | __m256i sse1 = _mm256_setzero_si256(); |
156 | |
|
157 | 0 | for (i = 0; i < 16 / inc; ++i) { |
158 | 0 | prev = curr; |
159 | 0 | curr = (curr == 0) ? 1 : 0; |
160 | 0 | opointer[curr] = _mm256_loadu_si256((const __m256i *)src_ptr); |
161 | 0 | src_ptr += src_pixels_per_line; |
162 | |
|
163 | 0 | __m256i V_S_M1 = |
164 | 0 | _mm256_unpacklo_epi16(opointer[prev], opointer[curr]); |
165 | 0 | __m256i V_S_M2 = |
166 | 0 | _mm256_unpackhi_epi16(opointer[prev], opointer[curr]); |
167 | |
|
168 | 0 | __m256i V_S_MAD1 = _mm256_madd_epi16(V_S_M1, filter2); |
169 | 0 | __m256i V_S_MAD2 = _mm256_madd_epi16(V_S_M2, filter2); |
170 | |
|
171 | 0 | __m256i V_S_S1 = |
172 | 0 | _mm256_srli_epi32(_mm256_add_epi32(V_S_MAD1, rbias), 7); |
173 | 0 | __m256i V_S_S2 = |
174 | 0 | _mm256_srli_epi32(_mm256_add_epi32(V_S_MAD2, rbias), 7); |
175 | |
|
176 | 0 | __m256i V_S_SRC = _mm256_packus_epi32(V_S_S1, V_S_S2); |
177 | |
|
178 | 0 | __m256i V_D_DST = _mm256_loadu_si256((const __m256i *)dst_ptr); |
179 | 0 | dst_ptr += dst_stride; |
180 | |
|
181 | 0 | __m256i V_R_SUB = _mm256_sub_epi16(V_S_SRC, V_D_DST); |
182 | 0 | __m256i V_R_MAD = _mm256_madd_epi16(V_R_SUB, V_R_SUB); |
183 | |
|
184 | 0 | sum1 = _mm256_add_epi16(sum1, V_R_SUB); |
185 | 0 | sse1 = _mm256_add_epi32(sse1, V_R_MAD); |
186 | 0 | } |
187 | |
|
188 | 0 | __m256i v_sum0 = _mm256_madd_epi16(sum1, one); |
189 | 0 | __m256i v_d_l = _mm256_unpacklo_epi32(v_sum0, sse1); |
190 | 0 | __m256i v_d_h = _mm256_unpackhi_epi32(v_sum0, sse1); |
191 | 0 | __m256i v_d_lh = _mm256_add_epi32(v_d_l, v_d_h); |
192 | 0 | const __m128i v_d0_d = _mm256_castsi256_si128(v_d_lh); |
193 | 0 | const __m128i v_d1_d = _mm256_extracti128_si256(v_d_lh, 1); |
194 | 0 | __m128i v_d = _mm_add_epi32(v_d0_d, v_d1_d); |
195 | 0 | v_d = _mm_add_epi32(v_d, _mm_srli_si128(v_d, 8)); |
196 | 0 | sum_long += _mm_extract_epi32(v_d, 0); |
197 | 0 | sse_long += _mm_extract_epi32(v_d, 1); |
198 | 0 | } |
199 | |
|
200 | 0 | rshift = get_msb(output_height) + get_msb(output_width); |
201 | 0 | } |
202 | 0 | } else if (xoffset == 4) { |
203 | 0 | if (yoffset == 0) { // xoffset==4 && yoffset==0 |
204 | 0 | range = output_width / 16; |
205 | 0 | if (output_height == 8) inc = 2; |
206 | 0 | if (output_height == 4) inc = 4; |
207 | 0 | for (j = 0; j < range * output_height * inc / 16; j++) { |
208 | 0 | if (j % (output_height * inc / 16) == 0) { |
209 | 0 | src_ptr = src_ptr_ref; |
210 | 0 | src_ptr_ref += 16; |
211 | 0 | dst_ptr = dst_ptr_ref; |
212 | 0 | dst_ptr_ref += 16; |
213 | 0 | __m256i V_H_D1 = _mm256_loadu_si256((const __m256i *)src_ptr); |
214 | 0 | __m256i V_H_D2 = _mm256_loadu_si256((const __m256i *)(src_ptr + 1)); |
215 | 0 | src_ptr += src_pixels_per_line; |
216 | |
|
217 | 0 | opointer[0] = _mm256_avg_epu16(V_H_D1, V_H_D2); |
218 | |
|
219 | 0 | curr = 0; |
220 | 0 | } |
221 | |
|
222 | 0 | __m256i sum1 = _mm256_setzero_si256(); |
223 | 0 | __m256i sse1 = _mm256_setzero_si256(); |
224 | |
|
225 | 0 | for (i = 0; i < 16 / inc; ++i) { |
226 | 0 | prev = curr; |
227 | 0 | curr = (curr == 0) ? 1 : 0; |
228 | 0 | __m256i V_V_D1 = _mm256_loadu_si256((const __m256i *)src_ptr); |
229 | 0 | __m256i V_V_D2 = _mm256_loadu_si256((const __m256i *)(src_ptr + 1)); |
230 | 0 | src_ptr += src_pixels_per_line; |
231 | |
|
232 | 0 | opointer[curr] = _mm256_avg_epu16(V_V_D1, V_V_D2); |
233 | |
|
234 | 0 | __m256i V_S_M1 = |
235 | 0 | _mm256_unpacklo_epi16(opointer[prev], opointer[curr]); |
236 | 0 | __m256i V_S_M2 = |
237 | 0 | _mm256_unpackhi_epi16(opointer[prev], opointer[curr]); |
238 | |
|
239 | 0 | __m256i V_S_MAD1 = _mm256_madd_epi16(V_S_M1, filter2); |
240 | 0 | __m256i V_S_MAD2 = _mm256_madd_epi16(V_S_M2, filter2); |
241 | |
|
242 | 0 | __m256i V_S_S1 = |
243 | 0 | _mm256_srli_epi32(_mm256_add_epi32(V_S_MAD1, rbias), 7); |
244 | 0 | __m256i V_S_S2 = |
245 | 0 | _mm256_srli_epi32(_mm256_add_epi32(V_S_MAD2, rbias), 7); |
246 | |
|
247 | 0 | __m256i V_S_SRC = _mm256_packus_epi32(V_S_S1, V_S_S2); |
248 | |
|
249 | 0 | __m256i V_D_DST = _mm256_loadu_si256((const __m256i *)dst_ptr); |
250 | 0 | dst_ptr += dst_stride; |
251 | |
|
252 | 0 | __m256i V_R_SUB = _mm256_sub_epi16(V_S_SRC, V_D_DST); |
253 | 0 | __m256i V_R_MAD = _mm256_madd_epi16(V_R_SUB, V_R_SUB); |
254 | |
|
255 | 0 | sum1 = _mm256_add_epi16(sum1, V_R_SUB); |
256 | 0 | sse1 = _mm256_add_epi32(sse1, V_R_MAD); |
257 | 0 | } |
258 | |
|
259 | 0 | __m256i v_sum0 = _mm256_madd_epi16(sum1, one); |
260 | 0 | __m256i v_d_l = _mm256_unpacklo_epi32(v_sum0, sse1); |
261 | 0 | __m256i v_d_h = _mm256_unpackhi_epi32(v_sum0, sse1); |
262 | 0 | __m256i v_d_lh = _mm256_add_epi32(v_d_l, v_d_h); |
263 | 0 | const __m128i v_d0_d = _mm256_castsi256_si128(v_d_lh); |
264 | 0 | const __m128i v_d1_d = _mm256_extracti128_si256(v_d_lh, 1); |
265 | 0 | __m128i v_d = _mm_add_epi32(v_d0_d, v_d1_d); |
266 | 0 | v_d = _mm_add_epi32(v_d, _mm_srli_si128(v_d, 8)); |
267 | 0 | sum_long += _mm_extract_epi32(v_d, 0); |
268 | 0 | sse_long += _mm_extract_epi32(v_d, 1); |
269 | 0 | } |
270 | |
|
271 | 0 | rshift = get_msb(output_height) + get_msb(output_width); |
272 | |
|
273 | 0 | } else if (yoffset == 4) { // xoffset==4 && yoffset==4 |
274 | 0 | range = output_width / 16; |
275 | 0 | if (output_height == 8) inc = 2; |
276 | 0 | if (output_height == 4) inc = 4; |
277 | 0 | for (j = 0; j < range * output_height * inc / 16; j++) { |
278 | 0 | if (j % (output_height * inc / 16) == 0) { |
279 | 0 | src_ptr = src_ptr_ref; |
280 | 0 | src_ptr_ref += 16; |
281 | 0 | dst_ptr = dst_ptr_ref; |
282 | 0 | dst_ptr_ref += 16; |
283 | |
|
284 | 0 | __m256i V_H_D1 = _mm256_loadu_si256((const __m256i *)src_ptr); |
285 | 0 | __m256i V_H_D2 = _mm256_loadu_si256((const __m256i *)(src_ptr + 1)); |
286 | 0 | src_ptr += src_pixels_per_line; |
287 | 0 | opointer[0] = _mm256_avg_epu16(V_H_D1, V_H_D2); |
288 | 0 | curr = 0; |
289 | 0 | } |
290 | |
|
291 | 0 | __m256i sum1 = _mm256_setzero_si256(); |
292 | 0 | __m256i sse1 = _mm256_setzero_si256(); |
293 | |
|
294 | 0 | for (i = 0; i < 16 / inc; ++i) { |
295 | 0 | prev = curr; |
296 | 0 | curr = (curr == 0) ? 1 : 0; |
297 | 0 | __m256i V_V_D1 = _mm256_loadu_si256((const __m256i *)src_ptr); |
298 | 0 | __m256i V_V_D2 = _mm256_loadu_si256((const __m256i *)(src_ptr + 1)); |
299 | 0 | src_ptr += src_pixels_per_line; |
300 | 0 | opointer[curr] = _mm256_avg_epu16(V_V_D1, V_V_D2); |
301 | 0 | __m256i V_S_SRC = _mm256_avg_epu16(opointer[curr], opointer[prev]); |
302 | |
|
303 | 0 | __m256i V_D_DST = _mm256_loadu_si256((const __m256i *)dst_ptr); |
304 | 0 | dst_ptr += dst_stride; |
305 | 0 | __m256i V_R_SUB = _mm256_sub_epi16(V_S_SRC, V_D_DST); |
306 | 0 | __m256i V_R_MAD = _mm256_madd_epi16(V_R_SUB, V_R_SUB); |
307 | 0 | sum1 = _mm256_add_epi16(sum1, V_R_SUB); |
308 | 0 | sse1 = _mm256_add_epi32(sse1, V_R_MAD); |
309 | 0 | } |
310 | |
|
311 | 0 | __m256i v_sum0 = _mm256_madd_epi16(sum1, one); |
312 | 0 | __m256i v_d_l = _mm256_unpacklo_epi32(v_sum0, sse1); |
313 | 0 | __m256i v_d_h = _mm256_unpackhi_epi32(v_sum0, sse1); |
314 | 0 | __m256i v_d_lh = _mm256_add_epi32(v_d_l, v_d_h); |
315 | 0 | const __m128i v_d0_d = _mm256_castsi256_si128(v_d_lh); |
316 | 0 | const __m128i v_d1_d = _mm256_extracti128_si256(v_d_lh, 1); |
317 | 0 | __m128i v_d = _mm_add_epi32(v_d0_d, v_d1_d); |
318 | 0 | v_d = _mm_add_epi32(v_d, _mm_srli_si128(v_d, 8)); |
319 | 0 | sum_long += _mm_extract_epi32(v_d, 0); |
320 | 0 | sse_long += _mm_extract_epi32(v_d, 1); |
321 | 0 | } |
322 | |
|
323 | 0 | rshift = get_msb(output_height) + get_msb(output_width); |
324 | |
|
325 | 0 | } else { // xoffset==4 && yoffset==1,2,3,5,6,7 |
326 | 0 | range = output_width / 16; |
327 | 0 | if (output_height == 8) inc = 2; |
328 | 0 | if (output_height == 4) inc = 4; |
329 | 0 | for (j = 0; j < range * output_height * inc / 16; j++) { |
330 | 0 | if (j % (output_height * inc / 16) == 0) { |
331 | 0 | src_ptr = src_ptr_ref; |
332 | 0 | src_ptr_ref += 16; |
333 | 0 | dst_ptr = dst_ptr_ref; |
334 | 0 | dst_ptr_ref += 16; |
335 | |
|
336 | 0 | __m256i V_H_D1 = _mm256_loadu_si256((const __m256i *)src_ptr); |
337 | 0 | __m256i V_H_D2 = _mm256_loadu_si256((const __m256i *)(src_ptr + 1)); |
338 | 0 | src_ptr += src_pixels_per_line; |
339 | 0 | opointer[0] = _mm256_avg_epu16(V_H_D1, V_H_D2); |
340 | 0 | curr = 0; |
341 | 0 | } |
342 | |
|
343 | 0 | __m256i sum1 = _mm256_setzero_si256(); |
344 | 0 | __m256i sse1 = _mm256_setzero_si256(); |
345 | |
|
346 | 0 | for (i = 0; i < 16 / inc; ++i) { |
347 | 0 | prev = curr; |
348 | 0 | curr = (curr == 0) ? 1 : 0; |
349 | 0 | __m256i V_V_D1 = _mm256_loadu_si256((const __m256i *)src_ptr); |
350 | 0 | __m256i V_V_D2 = _mm256_loadu_si256((const __m256i *)(src_ptr + 1)); |
351 | 0 | src_ptr += src_pixels_per_line; |
352 | 0 | opointer[curr] = _mm256_avg_epu16(V_V_D1, V_V_D2); |
353 | |
|
354 | 0 | __m256i V_S_M1 = |
355 | 0 | _mm256_unpacklo_epi16(opointer[prev], opointer[curr]); |
356 | 0 | __m256i V_S_M2 = |
357 | 0 | _mm256_unpackhi_epi16(opointer[prev], opointer[curr]); |
358 | |
|
359 | 0 | __m256i V_S_MAD1 = _mm256_madd_epi16(V_S_M1, filter2); |
360 | 0 | __m256i V_S_MAD2 = _mm256_madd_epi16(V_S_M2, filter2); |
361 | |
|
362 | 0 | __m256i V_S_S1 = |
363 | 0 | _mm256_srli_epi32(_mm256_add_epi32(V_S_MAD1, rbias), 7); |
364 | 0 | __m256i V_S_S2 = |
365 | 0 | _mm256_srli_epi32(_mm256_add_epi32(V_S_MAD2, rbias), 7); |
366 | |
|
367 | 0 | __m256i V_S_SRC = _mm256_packus_epi32(V_S_S1, V_S_S2); |
368 | |
|
369 | 0 | __m256i V_D_DST = _mm256_loadu_si256((const __m256i *)dst_ptr); |
370 | 0 | dst_ptr += dst_stride; |
371 | |
|
372 | 0 | __m256i V_R_SUB = _mm256_sub_epi16(V_S_SRC, V_D_DST); |
373 | 0 | __m256i V_R_MAD = _mm256_madd_epi16(V_R_SUB, V_R_SUB); |
374 | |
|
375 | 0 | sum1 = _mm256_add_epi16(sum1, V_R_SUB); |
376 | 0 | sse1 = _mm256_add_epi32(sse1, V_R_MAD); |
377 | 0 | } |
378 | |
|
379 | 0 | __m256i v_sum0 = _mm256_madd_epi16(sum1, one); |
380 | 0 | __m256i v_d_l = _mm256_unpacklo_epi32(v_sum0, sse1); |
381 | 0 | __m256i v_d_h = _mm256_unpackhi_epi32(v_sum0, sse1); |
382 | 0 | __m256i v_d_lh = _mm256_add_epi32(v_d_l, v_d_h); |
383 | 0 | const __m128i v_d0_d = _mm256_castsi256_si128(v_d_lh); |
384 | 0 | const __m128i v_d1_d = _mm256_extracti128_si256(v_d_lh, 1); |
385 | 0 | __m128i v_d = _mm_add_epi32(v_d0_d, v_d1_d); |
386 | 0 | v_d = _mm_add_epi32(v_d, _mm_srli_si128(v_d, 8)); |
387 | 0 | sum_long += _mm_extract_epi32(v_d, 0); |
388 | 0 | sse_long += _mm_extract_epi32(v_d, 1); |
389 | 0 | } |
390 | |
|
391 | 0 | rshift = get_msb(output_height) + get_msb(output_width); |
392 | 0 | } |
393 | 0 | } else if (yoffset == 0) { // xoffset==1,2,3,5,6,7 && yoffset==0 |
394 | 0 | range = output_width / 16; |
395 | 0 | if (output_height == 8) inc = 2; |
396 | 0 | if (output_height == 4) inc = 4; |
397 | 0 | for (j = 0; j < range * output_height * inc / 16; j++) { |
398 | 0 | if (j % (output_height * inc / 16) == 0) { |
399 | 0 | src_ptr = src_ptr_ref; |
400 | 0 | src_ptr_ref += 16; |
401 | 0 | dst_ptr = dst_ptr_ref; |
402 | 0 | dst_ptr_ref += 16; |
403 | |
|
404 | 0 | curr = 0; |
405 | 0 | } |
406 | |
|
407 | 0 | __m256i sum1 = _mm256_setzero_si256(); |
408 | 0 | __m256i sse1 = _mm256_setzero_si256(); |
409 | |
|
410 | 0 | for (i = 0; i < 16 / inc; ++i) { |
411 | 0 | __m256i V_V_D1 = _mm256_loadu_si256((const __m256i *)src_ptr); |
412 | 0 | __m256i V_V_D2 = _mm256_loadu_si256((const __m256i *)(src_ptr + 1)); |
413 | 0 | src_ptr += src_pixels_per_line; |
414 | 0 | __m256i V_V_M1 = _mm256_unpacklo_epi16(V_V_D1, V_V_D2); |
415 | 0 | __m256i V_V_M2 = _mm256_unpackhi_epi16(V_V_D1, V_V_D2); |
416 | 0 | __m256i V_V_MAD1 = _mm256_madd_epi16(V_V_M1, filter1); |
417 | 0 | __m256i V_V_MAD2 = _mm256_madd_epi16(V_V_M2, filter1); |
418 | 0 | __m256i V_V_S1 = |
419 | 0 | _mm256_srli_epi32(_mm256_add_epi32(V_V_MAD1, rbias), 7); |
420 | 0 | __m256i V_V_S2 = |
421 | 0 | _mm256_srli_epi32(_mm256_add_epi32(V_V_MAD2, rbias), 7); |
422 | 0 | opointer[curr] = _mm256_packus_epi32(V_V_S1, V_V_S2); |
423 | |
|
424 | 0 | __m256i V_D_DST = _mm256_loadu_si256((const __m256i *)dst_ptr); |
425 | 0 | dst_ptr += dst_stride; |
426 | 0 | __m256i V_R_SUB = _mm256_sub_epi16(opointer[curr], V_D_DST); |
427 | 0 | __m256i V_R_MAD = _mm256_madd_epi16(V_R_SUB, V_R_SUB); |
428 | |
|
429 | 0 | sum1 = _mm256_add_epi16(sum1, V_R_SUB); |
430 | 0 | sse1 = _mm256_add_epi32(sse1, V_R_MAD); |
431 | 0 | } |
432 | |
|
433 | 0 | __m256i v_sum0 = _mm256_madd_epi16(sum1, one); |
434 | 0 | __m256i v_d_l = _mm256_unpacklo_epi32(v_sum0, sse1); |
435 | 0 | __m256i v_d_h = _mm256_unpackhi_epi32(v_sum0, sse1); |
436 | 0 | __m256i v_d_lh = _mm256_add_epi32(v_d_l, v_d_h); |
437 | 0 | const __m128i v_d0_d = _mm256_castsi256_si128(v_d_lh); |
438 | 0 | const __m128i v_d1_d = _mm256_extracti128_si256(v_d_lh, 1); |
439 | 0 | __m128i v_d = _mm_add_epi32(v_d0_d, v_d1_d); |
440 | 0 | v_d = _mm_add_epi32(v_d, _mm_srli_si128(v_d, 8)); |
441 | 0 | sum_long += _mm_extract_epi32(v_d, 0); |
442 | 0 | sse_long += _mm_extract_epi32(v_d, 1); |
443 | 0 | } |
444 | |
|
445 | 0 | rshift = get_msb(output_height) + get_msb(output_width); |
446 | |
|
447 | 0 | } else if (yoffset == 4) { // xoffset==1,2,3,5,6,7 && yoffset==4 |
448 | |
|
449 | 0 | range = output_width / 16; |
450 | 0 | if (output_height == 8) inc = 2; |
451 | 0 | if (output_height == 4) inc = 4; |
452 | 0 | for (j = 0; j < range * output_height * inc / 16; j++) { |
453 | 0 | if (j % (output_height * inc / 16) == 0) { |
454 | 0 | src_ptr = src_ptr_ref; |
455 | 0 | src_ptr_ref += 16; |
456 | 0 | dst_ptr = dst_ptr_ref; |
457 | 0 | dst_ptr_ref += 16; |
458 | |
|
459 | 0 | __m256i V_H_D1 = _mm256_loadu_si256((const __m256i *)src_ptr); |
460 | 0 | __m256i V_H_D2 = _mm256_loadu_si256((const __m256i *)(src_ptr + 1)); |
461 | 0 | src_ptr += src_pixels_per_line; |
462 | |
|
463 | 0 | __m256i V_H_M1 = _mm256_unpacklo_epi16(V_H_D1, V_H_D2); |
464 | 0 | __m256i V_H_M2 = _mm256_unpackhi_epi16(V_H_D1, V_H_D2); |
465 | |
|
466 | 0 | __m256i V_H_MAD1 = _mm256_madd_epi16(V_H_M1, filter1); |
467 | 0 | __m256i V_H_MAD2 = _mm256_madd_epi16(V_H_M2, filter1); |
468 | |
|
469 | 0 | __m256i V_H_S1 = |
470 | 0 | _mm256_srli_epi32(_mm256_add_epi32(V_H_MAD1, rbias), 7); |
471 | 0 | __m256i V_H_S2 = |
472 | 0 | _mm256_srli_epi32(_mm256_add_epi32(V_H_MAD2, rbias), 7); |
473 | |
|
474 | 0 | opointer[0] = _mm256_packus_epi32(V_H_S1, V_H_S2); |
475 | |
|
476 | 0 | curr = 0; |
477 | 0 | } |
478 | |
|
479 | 0 | __m256i sum1 = _mm256_setzero_si256(); |
480 | 0 | __m256i sse1 = _mm256_setzero_si256(); |
481 | |
|
482 | 0 | for (i = 0; i < 16 / inc; ++i) { |
483 | 0 | prev = curr; |
484 | 0 | curr = (curr == 0) ? 1 : 0; |
485 | 0 | __m256i V_V_D1 = _mm256_loadu_si256((const __m256i *)src_ptr); |
486 | 0 | __m256i V_V_D2 = _mm256_loadu_si256((const __m256i *)(src_ptr + 1)); |
487 | 0 | src_ptr += src_pixels_per_line; |
488 | 0 | __m256i V_V_M1 = _mm256_unpacklo_epi16(V_V_D1, V_V_D2); |
489 | 0 | __m256i V_V_M2 = _mm256_unpackhi_epi16(V_V_D1, V_V_D2); |
490 | 0 | __m256i V_V_MAD1 = _mm256_madd_epi16(V_V_M1, filter1); |
491 | 0 | __m256i V_V_MAD2 = _mm256_madd_epi16(V_V_M2, filter1); |
492 | 0 | __m256i V_V_S1 = |
493 | 0 | _mm256_srli_epi32(_mm256_add_epi32(V_V_MAD1, rbias), 7); |
494 | 0 | __m256i V_V_S2 = |
495 | 0 | _mm256_srli_epi32(_mm256_add_epi32(V_V_MAD2, rbias), 7); |
496 | 0 | opointer[curr] = _mm256_packus_epi32(V_V_S1, V_V_S2); |
497 | |
|
498 | 0 | __m256i V_S_SRC = _mm256_avg_epu16(opointer[prev], opointer[curr]); |
499 | |
|
500 | 0 | __m256i V_D_DST = _mm256_loadu_si256((const __m256i *)dst_ptr); |
501 | 0 | dst_ptr += dst_stride; |
502 | |
|
503 | 0 | __m256i V_R_SUB = _mm256_sub_epi16(V_S_SRC, V_D_DST); |
504 | 0 | __m256i V_R_MAD = _mm256_madd_epi16(V_R_SUB, V_R_SUB); |
505 | |
|
506 | 0 | sum1 = _mm256_add_epi16(sum1, V_R_SUB); |
507 | 0 | sse1 = _mm256_add_epi32(sse1, V_R_MAD); |
508 | 0 | } |
509 | |
|
510 | 0 | __m256i v_sum0 = _mm256_madd_epi16(sum1, one); |
511 | 0 | __m256i v_d_l = _mm256_unpacklo_epi32(v_sum0, sse1); |
512 | 0 | __m256i v_d_h = _mm256_unpackhi_epi32(v_sum0, sse1); |
513 | 0 | __m256i v_d_lh = _mm256_add_epi32(v_d_l, v_d_h); |
514 | 0 | const __m128i v_d0_d = _mm256_castsi256_si128(v_d_lh); |
515 | 0 | const __m128i v_d1_d = _mm256_extracti128_si256(v_d_lh, 1); |
516 | 0 | __m128i v_d = _mm_add_epi32(v_d0_d, v_d1_d); |
517 | 0 | v_d = _mm_add_epi32(v_d, _mm_srli_si128(v_d, 8)); |
518 | 0 | sum_long += _mm_extract_epi32(v_d, 0); |
519 | 0 | sse_long += _mm_extract_epi32(v_d, 1); |
520 | 0 | } |
521 | |
|
522 | 0 | rshift = get_msb(output_height) + get_msb(output_width); |
523 | |
|
524 | 0 | } else { // xoffset==1,2,3,5,6,7 && yoffset==1,2,3,5,6,7 |
525 | 0 | range = output_width / 16; |
526 | 0 | if (output_height == 8) inc = 2; |
527 | 0 | if (output_height == 4) inc = 4; |
528 | 0 | unsigned int nloop = 16 / inc; |
529 | 0 | for (j = 0; j < range * output_height * inc / 16; j++) { |
530 | 0 | if (j % (output_height * inc / 16) == 0) { |
531 | 0 | src_ptr = src_ptr_ref; |
532 | 0 | src_ptr_ref += 16; |
533 | 0 | dst_ptr = dst_ptr_ref; |
534 | 0 | dst_ptr_ref += 16; |
535 | |
|
536 | 0 | __m256i V_H_D1 = _mm256_loadu_si256((const __m256i *)src_ptr); |
537 | 0 | __m256i V_H_D2 = _mm256_loadu_si256((const __m256i *)(src_ptr + 1)); |
538 | 0 | src_ptr += src_pixels_per_line; |
539 | |
|
540 | 0 | __m256i V_H_M1 = _mm256_unpacklo_epi16(V_H_D1, V_H_D2); |
541 | 0 | __m256i V_H_M2 = _mm256_unpackhi_epi16(V_H_D1, V_H_D2); |
542 | |
|
543 | 0 | __m256i V_H_MAD1 = _mm256_madd_epi16(V_H_M1, filter1); |
544 | 0 | __m256i V_H_MAD2 = _mm256_madd_epi16(V_H_M2, filter1); |
545 | |
|
546 | 0 | __m256i V_H_S1 = |
547 | 0 | _mm256_srli_epi32(_mm256_add_epi32(V_H_MAD1, rbias), 7); |
548 | 0 | __m256i V_H_S2 = |
549 | 0 | _mm256_srli_epi32(_mm256_add_epi32(V_H_MAD2, rbias), 7); |
550 | |
|
551 | 0 | opointer[0] = _mm256_packus_epi32(V_H_S1, V_H_S2); |
552 | |
|
553 | 0 | curr = 0; |
554 | 0 | } |
555 | |
|
556 | 0 | __m256i sum1 = _mm256_setzero_si256(); |
557 | 0 | __m256i sse1 = _mm256_setzero_si256(); |
558 | |
|
559 | 0 | for (i = 0; i < nloop; ++i) { |
560 | 0 | prev = curr; |
561 | 0 | curr = !curr; |
562 | 0 | __m256i V_V_D1 = _mm256_loadu_si256((const __m256i *)src_ptr); |
563 | 0 | __m256i V_V_D2 = _mm256_loadu_si256((const __m256i *)(src_ptr + 1)); |
564 | 0 | src_ptr += src_pixels_per_line; |
565 | 0 | __m256i V_V_M1 = _mm256_unpacklo_epi16(V_V_D1, V_V_D2); |
566 | 0 | __m256i V_V_M2 = _mm256_unpackhi_epi16(V_V_D1, V_V_D2); |
567 | 0 | __m256i V_V_MAD1 = _mm256_madd_epi16(V_V_M1, filter1); |
568 | 0 | __m256i V_V_MAD2 = _mm256_madd_epi16(V_V_M2, filter1); |
569 | 0 | __m256i V_V_S1 = |
570 | 0 | _mm256_srli_epi32(_mm256_add_epi32(V_V_MAD1, rbias), 7); |
571 | 0 | __m256i V_V_S2 = |
572 | 0 | _mm256_srli_epi32(_mm256_add_epi32(V_V_MAD2, rbias), 7); |
573 | 0 | opointer[curr] = _mm256_packus_epi32(V_V_S1, V_V_S2); |
574 | |
|
575 | 0 | __m256i V_S_M1 = _mm256_unpacklo_epi16(opointer[prev], opointer[curr]); |
576 | 0 | __m256i V_S_M2 = _mm256_unpackhi_epi16(opointer[prev], opointer[curr]); |
577 | |
|
578 | 0 | __m256i V_S_MAD1 = _mm256_madd_epi16(V_S_M1, filter2); |
579 | 0 | __m256i V_S_MAD2 = _mm256_madd_epi16(V_S_M2, filter2); |
580 | |
|
581 | 0 | __m256i V_S_S1 = |
582 | 0 | _mm256_srli_epi32(_mm256_add_epi32(V_S_MAD1, rbias), 7); |
583 | 0 | __m256i V_S_S2 = |
584 | 0 | _mm256_srli_epi32(_mm256_add_epi32(V_S_MAD2, rbias), 7); |
585 | |
|
586 | 0 | __m256i V_S_SRC = _mm256_packus_epi32(V_S_S1, V_S_S2); |
587 | |
|
588 | 0 | __m256i V_D_DST = _mm256_loadu_si256((const __m256i *)dst_ptr); |
589 | 0 | dst_ptr += dst_stride; |
590 | |
|
591 | 0 | __m256i V_R_SUB = _mm256_sub_epi16(V_S_SRC, V_D_DST); |
592 | 0 | __m256i V_R_MAD = _mm256_madd_epi16(V_R_SUB, V_R_SUB); |
593 | |
|
594 | 0 | sum1 = _mm256_add_epi16(sum1, V_R_SUB); |
595 | 0 | sse1 = _mm256_add_epi32(sse1, V_R_MAD); |
596 | 0 | } |
597 | |
|
598 | 0 | __m256i v_sum0 = _mm256_madd_epi16(sum1, one); |
599 | 0 | __m256i v_d_l = _mm256_unpacklo_epi32(v_sum0, sse1); |
600 | 0 | __m256i v_d_h = _mm256_unpackhi_epi32(v_sum0, sse1); |
601 | 0 | __m256i v_d_lh = _mm256_add_epi32(v_d_l, v_d_h); |
602 | 0 | const __m128i v_d0_d = _mm256_castsi256_si128(v_d_lh); |
603 | 0 | const __m128i v_d1_d = _mm256_extracti128_si256(v_d_lh, 1); |
604 | 0 | __m128i v_d = _mm_add_epi32(v_d0_d, v_d1_d); |
605 | 0 | v_d = _mm_add_epi32(v_d, _mm_srli_si128(v_d, 8)); |
606 | 0 | sum_long += _mm_extract_epi32(v_d, 0); |
607 | 0 | sse_long += _mm_extract_epi32(v_d, 1); |
608 | 0 | } |
609 | |
|
610 | 0 | rshift = get_msb(output_height) + get_msb(output_width); |
611 | 0 | } |
612 | |
|
613 | 0 | *sse = (uint32_t)ROUND_POWER_OF_TWO(sse_long, 4); |
614 | 0 | int sum = (int)ROUND_POWER_OF_TWO(sum_long, 2); |
615 | |
|
616 | 0 | int32_t var = *sse - (uint32_t)(((int64_t)sum * sum) >> rshift); |
617 | |
|
618 | 0 | return (var > 0) ? var : 0; |
619 | 0 | } |
620 | | |
621 | | static void highbd_calc8x8var_avx2(const uint16_t *src, int src_stride, |
622 | | const uint16_t *ref, int ref_stride, |
623 | 0 | uint32_t *sse, int *sum) { |
624 | 0 | __m256i v_sum_d = _mm256_setzero_si256(); |
625 | 0 | __m256i v_sse_d = _mm256_setzero_si256(); |
626 | 0 | for (int i = 0; i < 8; i += 2) { |
627 | 0 | const __m128i v_p_a0 = _mm_loadu_si128((const __m128i *)src); |
628 | 0 | const __m128i v_p_a1 = _mm_loadu_si128((const __m128i *)(src + src_stride)); |
629 | 0 | const __m128i v_p_b0 = _mm_loadu_si128((const __m128i *)ref); |
630 | 0 | const __m128i v_p_b1 = _mm_loadu_si128((const __m128i *)(ref + ref_stride)); |
631 | 0 | __m256i v_p_a = _mm256_castsi128_si256(v_p_a0); |
632 | 0 | __m256i v_p_b = _mm256_castsi128_si256(v_p_b0); |
633 | 0 | v_p_a = _mm256_inserti128_si256(v_p_a, v_p_a1, 1); |
634 | 0 | v_p_b = _mm256_inserti128_si256(v_p_b, v_p_b1, 1); |
635 | 0 | const __m256i v_diff = _mm256_sub_epi16(v_p_a, v_p_b); |
636 | 0 | const __m256i v_sqrdiff = _mm256_madd_epi16(v_diff, v_diff); |
637 | 0 | v_sum_d = _mm256_add_epi16(v_sum_d, v_diff); |
638 | 0 | v_sse_d = _mm256_add_epi32(v_sse_d, v_sqrdiff); |
639 | 0 | src += src_stride * 2; |
640 | 0 | ref += ref_stride * 2; |
641 | 0 | } |
642 | 0 | __m256i v_sum00 = _mm256_cvtepi16_epi32(_mm256_castsi256_si128(v_sum_d)); |
643 | 0 | __m256i v_sum01 = _mm256_cvtepi16_epi32(_mm256_extracti128_si256(v_sum_d, 1)); |
644 | 0 | __m256i v_sum0 = _mm256_add_epi32(v_sum00, v_sum01); |
645 | 0 | __m256i v_d_l = _mm256_unpacklo_epi32(v_sum0, v_sse_d); |
646 | 0 | __m256i v_d_h = _mm256_unpackhi_epi32(v_sum0, v_sse_d); |
647 | 0 | __m256i v_d_lh = _mm256_add_epi32(v_d_l, v_d_h); |
648 | 0 | const __m128i v_d0_d = _mm256_castsi256_si128(v_d_lh); |
649 | 0 | const __m128i v_d1_d = _mm256_extracti128_si256(v_d_lh, 1); |
650 | 0 | __m128i v_d = _mm_add_epi32(v_d0_d, v_d1_d); |
651 | 0 | v_d = _mm_add_epi32(v_d, _mm_srli_si128(v_d, 8)); |
652 | 0 | *sum = _mm_extract_epi32(v_d, 0); |
653 | 0 | *sse = _mm_extract_epi32(v_d, 1); |
654 | 0 | } |
655 | | |
656 | | static void highbd_calc16x16var_avx2(const uint16_t *src, int src_stride, |
657 | | const uint16_t *ref, int ref_stride, |
658 | 0 | uint32_t *sse, int *sum) { |
659 | 0 | __m256i v_sum_d = _mm256_setzero_si256(); |
660 | 0 | __m256i v_sse_d = _mm256_setzero_si256(); |
661 | 0 | const __m256i one = _mm256_set1_epi16(1); |
662 | 0 | for (int i = 0; i < 16; ++i) { |
663 | 0 | const __m256i v_p_a = _mm256_loadu_si256((const __m256i *)src); |
664 | 0 | const __m256i v_p_b = _mm256_loadu_si256((const __m256i *)ref); |
665 | 0 | const __m256i v_diff = _mm256_sub_epi16(v_p_a, v_p_b); |
666 | 0 | const __m256i v_sqrdiff = _mm256_madd_epi16(v_diff, v_diff); |
667 | 0 | v_sum_d = _mm256_add_epi16(v_sum_d, v_diff); |
668 | 0 | v_sse_d = _mm256_add_epi32(v_sse_d, v_sqrdiff); |
669 | 0 | src += src_stride; |
670 | 0 | ref += ref_stride; |
671 | 0 | } |
672 | 0 | __m256i v_sum0 = _mm256_madd_epi16(v_sum_d, one); |
673 | 0 | __m256i v_d_l = _mm256_unpacklo_epi32(v_sum0, v_sse_d); |
674 | 0 | __m256i v_d_h = _mm256_unpackhi_epi32(v_sum0, v_sse_d); |
675 | 0 | __m256i v_d_lh = _mm256_add_epi32(v_d_l, v_d_h); |
676 | 0 | const __m128i v_d0_d = _mm256_castsi256_si128(v_d_lh); |
677 | 0 | const __m128i v_d1_d = _mm256_extracti128_si256(v_d_lh, 1); |
678 | 0 | __m128i v_d = _mm_add_epi32(v_d0_d, v_d1_d); |
679 | 0 | v_d = _mm_add_epi32(v_d, _mm_srli_si128(v_d, 8)); |
680 | 0 | *sum = _mm_extract_epi32(v_d, 0); |
681 | 0 | *sse = _mm_extract_epi32(v_d, 1); |
682 | 0 | } |
683 | | |
684 | | static void highbd_10_variance_avx2(const uint16_t *src, int src_stride, |
685 | | const uint16_t *ref, int ref_stride, int w, |
686 | | int h, uint32_t *sse, int *sum, |
687 | 0 | high_variance_fn_t var_fn, int block_size) { |
688 | 0 | int i, j; |
689 | 0 | uint64_t sse_long = 0; |
690 | 0 | int32_t sum_long = 0; |
691 | |
|
692 | 0 | for (i = 0; i < h; i += block_size) { |
693 | 0 | for (j = 0; j < w; j += block_size) { |
694 | 0 | unsigned int sse0; |
695 | 0 | int sum0; |
696 | 0 | var_fn(src + src_stride * i + j, src_stride, ref + ref_stride * i + j, |
697 | 0 | ref_stride, &sse0, &sum0); |
698 | 0 | sse_long += sse0; |
699 | 0 | sum_long += sum0; |
700 | 0 | } |
701 | 0 | } |
702 | 0 | *sum = ROUND_POWER_OF_TWO(sum_long, 2); |
703 | 0 | *sse = (uint32_t)ROUND_POWER_OF_TWO(sse_long, 4); |
704 | 0 | } |
705 | | |
706 | | #define VAR_FN(w, h, block_size, shift) \ |
707 | | uint32_t aom_highbd_10_variance##w##x##h##_avx2( \ |
708 | | const uint8_t *src8, int src_stride, const uint8_t *ref8, \ |
709 | 0 | int ref_stride, uint32_t *sse) { \ |
710 | 0 | int sum; \ |
711 | 0 | int64_t var; \ |
712 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
713 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ |
714 | 0 | highbd_10_variance_avx2(src, src_stride, ref, ref_stride, w, h, sse, &sum, \ |
715 | 0 | highbd_calc##block_size##x##block_size##var_avx2, \ |
716 | 0 | block_size); \ |
717 | 0 | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> shift); \ |
718 | 0 | return (var >= 0) ? (uint32_t)var : 0; \ |
719 | 0 | } Unexecuted instantiation: aom_highbd_10_variance128x128_avx2 Unexecuted instantiation: aom_highbd_10_variance128x64_avx2 Unexecuted instantiation: aom_highbd_10_variance64x128_avx2 Unexecuted instantiation: aom_highbd_10_variance64x64_avx2 Unexecuted instantiation: aom_highbd_10_variance64x32_avx2 Unexecuted instantiation: aom_highbd_10_variance32x64_avx2 Unexecuted instantiation: aom_highbd_10_variance32x32_avx2 Unexecuted instantiation: aom_highbd_10_variance32x16_avx2 Unexecuted instantiation: aom_highbd_10_variance16x32_avx2 Unexecuted instantiation: aom_highbd_10_variance16x16_avx2 Unexecuted instantiation: aom_highbd_10_variance16x8_avx2 Unexecuted instantiation: aom_highbd_10_variance8x16_avx2 Unexecuted instantiation: aom_highbd_10_variance8x8_avx2 Unexecuted instantiation: aom_highbd_10_variance16x64_avx2 Unexecuted instantiation: aom_highbd_10_variance32x8_avx2 Unexecuted instantiation: aom_highbd_10_variance64x16_avx2 Unexecuted instantiation: aom_highbd_10_variance8x32_avx2 |
720 | | |
721 | | VAR_FN(128, 128, 16, 14) |
722 | | VAR_FN(128, 64, 16, 13) |
723 | | VAR_FN(64, 128, 16, 13) |
724 | | VAR_FN(64, 64, 16, 12) |
725 | | VAR_FN(64, 32, 16, 11) |
726 | | VAR_FN(32, 64, 16, 11) |
727 | | VAR_FN(32, 32, 16, 10) |
728 | | VAR_FN(32, 16, 16, 9) |
729 | | VAR_FN(16, 32, 16, 9) |
730 | | VAR_FN(16, 16, 16, 8) |
731 | | VAR_FN(16, 8, 8, 7) |
732 | | VAR_FN(8, 16, 8, 7) |
733 | | VAR_FN(8, 8, 8, 6) |
734 | | |
735 | | #if !CONFIG_REALTIME_ONLY |
736 | | VAR_FN(16, 64, 16, 10) |
737 | | VAR_FN(32, 8, 8, 8) |
738 | | VAR_FN(64, 16, 16, 10) |
739 | | VAR_FN(8, 32, 8, 8) |
740 | | #endif // !CONFIG_REALTIME_ONLY |
741 | | |
742 | | #undef VAR_FN |
743 | | |
744 | | unsigned int aom_highbd_10_mse16x16_avx2(const uint8_t *src8, int src_stride, |
745 | | const uint8_t *ref8, int ref_stride, |
746 | 0 | unsigned int *sse) { |
747 | 0 | int sum; |
748 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); |
749 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); |
750 | 0 | highbd_10_variance_avx2(src, src_stride, ref, ref_stride, 16, 16, sse, &sum, |
751 | 0 | highbd_calc16x16var_avx2, 16); |
752 | 0 | return *sse; |
753 | 0 | } |
754 | | |
755 | | #define SSE2_HEIGHT(H) \ |
756 | | uint32_t aom_highbd_10_sub_pixel_variance8x##H##_sse2( \ |
757 | | const uint8_t *src8, int src_stride, int x_offset, int y_offset, \ |
758 | | const uint8_t *dst8, int dst_stride, uint32_t *sse_ptr); |
759 | | |
760 | | SSE2_HEIGHT(8) |
761 | | SSE2_HEIGHT(16) |
762 | | |
763 | | #undef SSE2_HEIGHT |
764 | | |
765 | | #define HIGHBD_SUBPIX_VAR(W, H) \ |
766 | | uint32_t aom_highbd_10_sub_pixel_variance##W##x##H##_avx2( \ |
767 | | const uint8_t *src, int src_stride, int xoffset, int yoffset, \ |
768 | 0 | const uint8_t *dst, int dst_stride, uint32_t *sse) { \ |
769 | 0 | if (W == 8 && H == 16) \ |
770 | 0 | return aom_highbd_10_sub_pixel_variance8x16_sse2( \ |
771 | 0 | src, src_stride, xoffset, yoffset, dst, dst_stride, sse); \ |
772 | 0 | else if (W == 8 && H == 8) \ |
773 | 0 | return aom_highbd_10_sub_pixel_variance8x8_sse2( \ |
774 | 0 | src, src_stride, xoffset, yoffset, dst, dst_stride, sse); \ |
775 | 0 | else \ |
776 | 0 | return aom_highbd_var_filter_block2d_bil_avx2( \ |
777 | 0 | src, src_stride, 1, H, W, xoffset, yoffset, dst, dst_stride, sse); \ |
778 | 0 | } Unexecuted instantiation: aom_highbd_10_sub_pixel_variance16x8_avx2 Unexecuted instantiation: aom_highbd_10_sub_pixel_variance8x8_avx2 |
779 | | |
780 | | HIGHBD_SUBPIX_VAR(128, 128) |
781 | | HIGHBD_SUBPIX_VAR(128, 64) |
782 | | HIGHBD_SUBPIX_VAR(64, 128) |
783 | | HIGHBD_SUBPIX_VAR(64, 64) |
784 | | HIGHBD_SUBPIX_VAR(64, 32) |
785 | | HIGHBD_SUBPIX_VAR(32, 64) |
786 | | HIGHBD_SUBPIX_VAR(32, 32) |
787 | | HIGHBD_SUBPIX_VAR(32, 16) |
788 | | HIGHBD_SUBPIX_VAR(16, 32) |
789 | | HIGHBD_SUBPIX_VAR(16, 16) |
790 | | HIGHBD_SUBPIX_VAR(16, 8) |
791 | | HIGHBD_SUBPIX_VAR(8, 16) |
792 | | HIGHBD_SUBPIX_VAR(8, 8) |
793 | | |
794 | | #undef HIGHBD_SUBPIX_VAR |
795 | | |
796 | | static uint64_t mse_4xh_16bit_highbd_avx2(uint16_t *dst, int dstride, |
797 | 0 | uint16_t *src, int sstride, int h) { |
798 | 0 | uint64_t sum = 0; |
799 | 0 | __m128i reg0_4x16, reg1_4x16, reg2_4x16, reg3_4x16; |
800 | 0 | __m256i src0_8x16, src1_8x16, src_16x16; |
801 | 0 | __m256i dst0_8x16, dst1_8x16, dst_16x16; |
802 | 0 | __m256i res0_4x64, res1_4x64, res2_4x64, res3_4x64; |
803 | 0 | __m256i sub_result; |
804 | 0 | const __m256i zeros = _mm256_broadcastsi128_si256(_mm_setzero_si128()); |
805 | 0 | __m256i square_result = _mm256_broadcastsi128_si256(_mm_setzero_si128()); |
806 | 0 | for (int i = 0; i < h; i += 4) { |
807 | 0 | reg0_4x16 = _mm_loadl_epi64((__m128i const *)(&dst[(i + 0) * dstride])); |
808 | 0 | reg1_4x16 = _mm_loadl_epi64((__m128i const *)(&dst[(i + 1) * dstride])); |
809 | 0 | reg2_4x16 = _mm_loadl_epi64((__m128i const *)(&dst[(i + 2) * dstride])); |
810 | 0 | reg3_4x16 = _mm_loadl_epi64((__m128i const *)(&dst[(i + 3) * dstride])); |
811 | 0 | dst0_8x16 = |
812 | 0 | _mm256_castsi128_si256(_mm_unpacklo_epi64(reg0_4x16, reg1_4x16)); |
813 | 0 | dst1_8x16 = |
814 | 0 | _mm256_castsi128_si256(_mm_unpacklo_epi64(reg2_4x16, reg3_4x16)); |
815 | 0 | dst_16x16 = _mm256_permute2x128_si256(dst0_8x16, dst1_8x16, 0x20); |
816 | |
|
817 | 0 | reg0_4x16 = _mm_loadl_epi64((__m128i const *)(&src[(i + 0) * sstride])); |
818 | 0 | reg1_4x16 = _mm_loadl_epi64((__m128i const *)(&src[(i + 1) * sstride])); |
819 | 0 | reg2_4x16 = _mm_loadl_epi64((__m128i const *)(&src[(i + 2) * sstride])); |
820 | 0 | reg3_4x16 = _mm_loadl_epi64((__m128i const *)(&src[(i + 3) * sstride])); |
821 | 0 | src0_8x16 = |
822 | 0 | _mm256_castsi128_si256(_mm_unpacklo_epi64(reg0_4x16, reg1_4x16)); |
823 | 0 | src1_8x16 = |
824 | 0 | _mm256_castsi128_si256(_mm_unpacklo_epi64(reg2_4x16, reg3_4x16)); |
825 | 0 | src_16x16 = _mm256_permute2x128_si256(src0_8x16, src1_8x16, 0x20); |
826 | |
|
827 | 0 | sub_result = _mm256_abs_epi16(_mm256_sub_epi16(src_16x16, dst_16x16)); |
828 | |
|
829 | 0 | src_16x16 = _mm256_unpacklo_epi16(sub_result, zeros); |
830 | 0 | dst_16x16 = _mm256_unpackhi_epi16(sub_result, zeros); |
831 | |
|
832 | 0 | src_16x16 = _mm256_madd_epi16(src_16x16, src_16x16); |
833 | 0 | dst_16x16 = _mm256_madd_epi16(dst_16x16, dst_16x16); |
834 | |
|
835 | 0 | res0_4x64 = _mm256_unpacklo_epi32(src_16x16, zeros); |
836 | 0 | res1_4x64 = _mm256_unpackhi_epi32(src_16x16, zeros); |
837 | 0 | res2_4x64 = _mm256_unpacklo_epi32(dst_16x16, zeros); |
838 | 0 | res3_4x64 = _mm256_unpackhi_epi32(dst_16x16, zeros); |
839 | |
|
840 | 0 | square_result = _mm256_add_epi64( |
841 | 0 | square_result, |
842 | 0 | _mm256_add_epi64( |
843 | 0 | _mm256_add_epi64(_mm256_add_epi64(res0_4x64, res1_4x64), res2_4x64), |
844 | 0 | res3_4x64)); |
845 | 0 | } |
846 | 0 | const __m128i sum_2x64 = |
847 | 0 | _mm_add_epi64(_mm256_castsi256_si128(square_result), |
848 | 0 | _mm256_extracti128_si256(square_result, 1)); |
849 | 0 | const __m128i sum_1x64 = _mm_add_epi64(sum_2x64, _mm_srli_si128(sum_2x64, 8)); |
850 | 0 | xx_storel_64(&sum, sum_1x64); |
851 | 0 | return sum; |
852 | 0 | } |
853 | | |
854 | | static uint64_t mse_8xh_16bit_highbd_avx2(uint16_t *dst, int dstride, |
855 | 0 | uint16_t *src, int sstride, int h) { |
856 | 0 | uint64_t sum = 0; |
857 | 0 | __m256i src0_8x16, src1_8x16, src_16x16; |
858 | 0 | __m256i dst0_8x16, dst1_8x16, dst_16x16; |
859 | 0 | __m256i res0_4x64, res1_4x64, res2_4x64, res3_4x64; |
860 | 0 | __m256i sub_result; |
861 | 0 | const __m256i zeros = _mm256_broadcastsi128_si256(_mm_setzero_si128()); |
862 | 0 | __m256i square_result = _mm256_broadcastsi128_si256(_mm_setzero_si128()); |
863 | |
|
864 | 0 | for (int i = 0; i < h; i += 2) { |
865 | 0 | dst0_8x16 = |
866 | 0 | _mm256_castsi128_si256(_mm_loadu_si128((__m128i *)&dst[i * dstride])); |
867 | 0 | dst1_8x16 = _mm256_castsi128_si256( |
868 | 0 | _mm_loadu_si128((__m128i *)&dst[(i + 1) * dstride])); |
869 | 0 | dst_16x16 = _mm256_permute2x128_si256(dst0_8x16, dst1_8x16, 0x20); |
870 | |
|
871 | 0 | src0_8x16 = |
872 | 0 | _mm256_castsi128_si256(_mm_loadu_si128((__m128i *)&src[i * sstride])); |
873 | 0 | src1_8x16 = _mm256_castsi128_si256( |
874 | 0 | _mm_loadu_si128((__m128i *)&src[(i + 1) * sstride])); |
875 | 0 | src_16x16 = _mm256_permute2x128_si256(src0_8x16, src1_8x16, 0x20); |
876 | |
|
877 | 0 | sub_result = _mm256_abs_epi16(_mm256_sub_epi16(src_16x16, dst_16x16)); |
878 | |
|
879 | 0 | src_16x16 = _mm256_unpacklo_epi16(sub_result, zeros); |
880 | 0 | dst_16x16 = _mm256_unpackhi_epi16(sub_result, zeros); |
881 | |
|
882 | 0 | src_16x16 = _mm256_madd_epi16(src_16x16, src_16x16); |
883 | 0 | dst_16x16 = _mm256_madd_epi16(dst_16x16, dst_16x16); |
884 | |
|
885 | 0 | res0_4x64 = _mm256_unpacklo_epi32(src_16x16, zeros); |
886 | 0 | res1_4x64 = _mm256_unpackhi_epi32(src_16x16, zeros); |
887 | 0 | res2_4x64 = _mm256_unpacklo_epi32(dst_16x16, zeros); |
888 | 0 | res3_4x64 = _mm256_unpackhi_epi32(dst_16x16, zeros); |
889 | |
|
890 | 0 | square_result = _mm256_add_epi64( |
891 | 0 | square_result, |
892 | 0 | _mm256_add_epi64( |
893 | 0 | _mm256_add_epi64(_mm256_add_epi64(res0_4x64, res1_4x64), res2_4x64), |
894 | 0 | res3_4x64)); |
895 | 0 | } |
896 | |
|
897 | 0 | const __m128i sum_2x64 = |
898 | 0 | _mm_add_epi64(_mm256_castsi256_si128(square_result), |
899 | 0 | _mm256_extracti128_si256(square_result, 1)); |
900 | 0 | const __m128i sum_1x64 = _mm_add_epi64(sum_2x64, _mm_srli_si128(sum_2x64, 8)); |
901 | 0 | xx_storel_64(&sum, sum_1x64); |
902 | 0 | return sum; |
903 | 0 | } |
904 | | |
905 | | uint64_t aom_mse_wxh_16bit_highbd_avx2(uint16_t *dst, int dstride, |
906 | | uint16_t *src, int sstride, int w, |
907 | 0 | int h) { |
908 | 0 | assert((w == 8 || w == 4) && (h == 8 || h == 4) && |
909 | 0 | "w=8/4 and h=8/4 must satisfy"); |
910 | 0 | switch (w) { |
911 | 0 | case 4: return mse_4xh_16bit_highbd_avx2(dst, dstride, src, sstride, h); |
912 | 0 | case 8: return mse_8xh_16bit_highbd_avx2(dst, dstride, src, sstride, h); |
913 | 0 | default: assert(0 && "unsupported width"); return -1; |
914 | 0 | } |
915 | 0 | } |