/src/aom/aom_dsp/x86/highbd_variance_sse2.c
Line | Count | Source |
1 | | /* |
2 | | * Copyright (c) 2016, Alliance for Open Media. All rights reserved. |
3 | | * |
4 | | * This source code is subject to the terms of the BSD 2 Clause License and |
5 | | * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
6 | | * was not distributed with this source code in the LICENSE file, you can |
7 | | * obtain it at www.aomedia.org/license/software. If the Alliance for Open |
8 | | * Media Patent License 1.0 was not distributed with this source code in the |
9 | | * PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
10 | | */ |
11 | | |
12 | | #include <assert.h> |
13 | | #include <emmintrin.h> // SSE2 |
14 | | |
15 | | #include "config/aom_config.h" |
16 | | #include "config/aom_dsp_rtcd.h" |
17 | | |
18 | | #include "aom_dsp/x86/synonyms.h" |
19 | | #include "aom_ports/mem.h" |
20 | | |
21 | | #include "av1/common/filter.h" |
22 | | #include "av1/common/reconinter.h" |
23 | | |
24 | | #if HAVE_X86_ASM |
25 | | typedef uint32_t (*high_variance_fn_t)(const uint16_t *src, int src_stride, |
26 | | const uint16_t *ref, int ref_stride, |
27 | | uint32_t *sse, int *sum); |
28 | | |
29 | | uint32_t aom_highbd_calc8x8var_sse2(const uint16_t *src, int src_stride, |
30 | | const uint16_t *ref, int ref_stride, |
31 | | uint32_t *sse, int *sum); |
32 | | |
33 | | uint32_t aom_highbd_calc16x16var_sse2(const uint16_t *src, int src_stride, |
34 | | const uint16_t *ref, int ref_stride, |
35 | | uint32_t *sse, int *sum); |
36 | | |
37 | | static void highbd_8_variance_sse2(const uint16_t *src, int src_stride, |
38 | | const uint16_t *ref, int ref_stride, int w, |
39 | | int h, uint32_t *sse, int *sum, |
40 | 0 | high_variance_fn_t var_fn, int block_size) { |
41 | 0 | int i, j; |
42 | |
|
43 | 0 | *sse = 0; |
44 | 0 | *sum = 0; |
45 | |
|
46 | 0 | for (i = 0; i < h; i += block_size) { |
47 | 0 | for (j = 0; j < w; j += block_size) { |
48 | 0 | unsigned int sse0; |
49 | 0 | int sum0; |
50 | 0 | var_fn(src + src_stride * i + j, src_stride, ref + ref_stride * i + j, |
51 | 0 | ref_stride, &sse0, &sum0); |
52 | 0 | *sse += sse0; |
53 | 0 | *sum += sum0; |
54 | 0 | } |
55 | 0 | } |
56 | 0 | } |
57 | | |
58 | | static void highbd_10_variance_sse2(const uint16_t *src, int src_stride, |
59 | | const uint16_t *ref, int ref_stride, int w, |
60 | | int h, uint32_t *sse, int *sum, |
61 | 0 | high_variance_fn_t var_fn, int block_size) { |
62 | 0 | int i, j; |
63 | 0 | uint64_t sse_long = 0; |
64 | 0 | int32_t sum_long = 0; |
65 | |
|
66 | 0 | for (i = 0; i < h; i += block_size) { |
67 | 0 | for (j = 0; j < w; j += block_size) { |
68 | 0 | unsigned int sse0; |
69 | 0 | int sum0; |
70 | 0 | var_fn(src + src_stride * i + j, src_stride, ref + ref_stride * i + j, |
71 | 0 | ref_stride, &sse0, &sum0); |
72 | 0 | sse_long += sse0; |
73 | 0 | sum_long += sum0; |
74 | 0 | } |
75 | 0 | } |
76 | 0 | *sum = ROUND_POWER_OF_TWO(sum_long, 2); |
77 | 0 | *sse = (uint32_t)ROUND_POWER_OF_TWO(sse_long, 4); |
78 | 0 | } |
79 | | |
80 | | static void highbd_12_variance_sse2(const uint16_t *src, int src_stride, |
81 | | const uint16_t *ref, int ref_stride, int w, |
82 | | int h, uint32_t *sse, int *sum, |
83 | 0 | high_variance_fn_t var_fn, int block_size) { |
84 | 0 | int i, j; |
85 | 0 | uint64_t sse_long = 0; |
86 | 0 | int32_t sum_long = 0; |
87 | |
|
88 | 0 | for (i = 0; i < h; i += block_size) { |
89 | 0 | for (j = 0; j < w; j += block_size) { |
90 | 0 | unsigned int sse0; |
91 | 0 | int sum0; |
92 | 0 | var_fn(src + src_stride * i + j, src_stride, ref + ref_stride * i + j, |
93 | 0 | ref_stride, &sse0, &sum0); |
94 | 0 | sse_long += sse0; |
95 | 0 | sum_long += sum0; |
96 | 0 | } |
97 | 0 | } |
98 | 0 | *sum = ROUND_POWER_OF_TWO(sum_long, 4); |
99 | 0 | *sse = (uint32_t)ROUND_POWER_OF_TWO(sse_long, 8); |
100 | 0 | } |
101 | | |
102 | | #define VAR_FN(w, h, block_size, shift) \ |
103 | | uint32_t aom_highbd_8_variance##w##x##h##_sse2( \ |
104 | | const uint8_t *src8, int src_stride, const uint8_t *ref8, \ |
105 | 0 | int ref_stride, uint32_t *sse) { \ |
106 | 0 | int sum; \ |
107 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
108 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ |
109 | 0 | highbd_8_variance_sse2( \ |
110 | 0 | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ |
111 | 0 | aom_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ |
112 | 0 | return *sse - (uint32_t)(((int64_t)sum * sum) >> shift); \ |
113 | 0 | } \ Unexecuted instantiation: aom_highbd_8_variance128x128_sse2 Unexecuted instantiation: aom_highbd_8_variance128x64_sse2 Unexecuted instantiation: aom_highbd_8_variance64x128_sse2 Unexecuted instantiation: aom_highbd_8_variance64x64_sse2 Unexecuted instantiation: aom_highbd_8_variance64x32_sse2 Unexecuted instantiation: aom_highbd_8_variance32x64_sse2 Unexecuted instantiation: aom_highbd_8_variance32x32_sse2 Unexecuted instantiation: aom_highbd_8_variance32x16_sse2 Unexecuted instantiation: aom_highbd_8_variance16x32_sse2 Unexecuted instantiation: aom_highbd_8_variance16x16_sse2 Unexecuted instantiation: aom_highbd_8_variance16x8_sse2 Unexecuted instantiation: aom_highbd_8_variance8x16_sse2 Unexecuted instantiation: aom_highbd_8_variance8x8_sse2 Unexecuted instantiation: aom_highbd_8_variance8x32_sse2 Unexecuted instantiation: aom_highbd_8_variance32x8_sse2 Unexecuted instantiation: aom_highbd_8_variance16x64_sse2 Unexecuted instantiation: aom_highbd_8_variance64x16_sse2 |
114 | | \ |
115 | | uint32_t aom_highbd_10_variance##w##x##h##_sse2( \ |
116 | | const uint8_t *src8, int src_stride, const uint8_t *ref8, \ |
117 | 0 | int ref_stride, uint32_t *sse) { \ |
118 | 0 | int sum; \ |
119 | 0 | int64_t var; \ |
120 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
121 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ |
122 | 0 | highbd_10_variance_sse2( \ |
123 | 0 | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ |
124 | 0 | aom_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ |
125 | 0 | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> shift); \ |
126 | 0 | return (var >= 0) ? (uint32_t)var : 0; \ |
127 | 0 | } \ Unexecuted instantiation: aom_highbd_10_variance128x128_sse2 Unexecuted instantiation: aom_highbd_10_variance128x64_sse2 Unexecuted instantiation: aom_highbd_10_variance64x128_sse2 Unexecuted instantiation: aom_highbd_10_variance64x64_sse2 Unexecuted instantiation: aom_highbd_10_variance64x32_sse2 Unexecuted instantiation: aom_highbd_10_variance32x64_sse2 Unexecuted instantiation: aom_highbd_10_variance32x32_sse2 Unexecuted instantiation: aom_highbd_10_variance32x16_sse2 Unexecuted instantiation: aom_highbd_10_variance16x32_sse2 Unexecuted instantiation: aom_highbd_10_variance16x16_sse2 Unexecuted instantiation: aom_highbd_10_variance16x8_sse2 Unexecuted instantiation: aom_highbd_10_variance8x16_sse2 Unexecuted instantiation: aom_highbd_10_variance8x8_sse2 Unexecuted instantiation: aom_highbd_10_variance8x32_sse2 Unexecuted instantiation: aom_highbd_10_variance32x8_sse2 Unexecuted instantiation: aom_highbd_10_variance16x64_sse2 Unexecuted instantiation: aom_highbd_10_variance64x16_sse2 |
128 | | \ |
129 | | uint32_t aom_highbd_12_variance##w##x##h##_sse2( \ |
130 | | const uint8_t *src8, int src_stride, const uint8_t *ref8, \ |
131 | 0 | int ref_stride, uint32_t *sse) { \ |
132 | 0 | int sum; \ |
133 | 0 | int64_t var; \ |
134 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
135 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); \ |
136 | 0 | highbd_12_variance_sse2( \ |
137 | 0 | src, src_stride, ref, ref_stride, w, h, sse, &sum, \ |
138 | 0 | aom_highbd_calc##block_size##x##block_size##var_sse2, block_size); \ |
139 | 0 | var = (int64_t)(*sse) - (((int64_t)sum * sum) >> shift); \ |
140 | 0 | return (var >= 0) ? (uint32_t)var : 0; \ |
141 | 0 | } Unexecuted instantiation: aom_highbd_12_variance128x128_sse2 Unexecuted instantiation: aom_highbd_12_variance128x64_sse2 Unexecuted instantiation: aom_highbd_12_variance64x128_sse2 Unexecuted instantiation: aom_highbd_12_variance64x64_sse2 Unexecuted instantiation: aom_highbd_12_variance64x32_sse2 Unexecuted instantiation: aom_highbd_12_variance32x64_sse2 Unexecuted instantiation: aom_highbd_12_variance32x32_sse2 Unexecuted instantiation: aom_highbd_12_variance32x16_sse2 Unexecuted instantiation: aom_highbd_12_variance16x32_sse2 Unexecuted instantiation: aom_highbd_12_variance16x16_sse2 Unexecuted instantiation: aom_highbd_12_variance16x8_sse2 Unexecuted instantiation: aom_highbd_12_variance8x16_sse2 Unexecuted instantiation: aom_highbd_12_variance8x8_sse2 Unexecuted instantiation: aom_highbd_12_variance8x32_sse2 Unexecuted instantiation: aom_highbd_12_variance32x8_sse2 Unexecuted instantiation: aom_highbd_12_variance16x64_sse2 Unexecuted instantiation: aom_highbd_12_variance64x16_sse2 |
142 | | |
143 | | VAR_FN(128, 128, 16, 14) |
144 | | VAR_FN(128, 64, 16, 13) |
145 | | VAR_FN(64, 128, 16, 13) |
146 | | VAR_FN(64, 64, 16, 12) |
147 | | VAR_FN(64, 32, 16, 11) |
148 | | VAR_FN(32, 64, 16, 11) |
149 | | VAR_FN(32, 32, 16, 10) |
150 | | VAR_FN(32, 16, 16, 9) |
151 | | VAR_FN(16, 32, 16, 9) |
152 | | VAR_FN(16, 16, 16, 8) |
153 | | VAR_FN(16, 8, 8, 7) |
154 | | VAR_FN(8, 16, 8, 7) |
155 | | VAR_FN(8, 8, 8, 6) |
156 | | |
157 | | #if !CONFIG_REALTIME_ONLY |
158 | | VAR_FN(8, 32, 8, 8) |
159 | | VAR_FN(32, 8, 8, 8) |
160 | | VAR_FN(16, 64, 16, 10) |
161 | | VAR_FN(64, 16, 16, 10) |
162 | | #endif // !CONFIG_REALTIME_ONLY |
163 | | |
164 | | #undef VAR_FN |
165 | | |
166 | | unsigned int aom_highbd_8_mse16x16_sse2(const uint8_t *src8, int src_stride, |
167 | | const uint8_t *ref8, int ref_stride, |
168 | 0 | unsigned int *sse) { |
169 | 0 | int sum; |
170 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); |
171 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); |
172 | 0 | highbd_8_variance_sse2(src, src_stride, ref, ref_stride, 16, 16, sse, &sum, |
173 | 0 | aom_highbd_calc16x16var_sse2, 16); |
174 | 0 | return *sse; |
175 | 0 | } |
176 | | |
177 | | unsigned int aom_highbd_10_mse16x16_sse2(const uint8_t *src8, int src_stride, |
178 | | const uint8_t *ref8, int ref_stride, |
179 | 0 | unsigned int *sse) { |
180 | 0 | int sum; |
181 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); |
182 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); |
183 | 0 | highbd_10_variance_sse2(src, src_stride, ref, ref_stride, 16, 16, sse, &sum, |
184 | 0 | aom_highbd_calc16x16var_sse2, 16); |
185 | 0 | return *sse; |
186 | 0 | } |
187 | | |
188 | | unsigned int aom_highbd_12_mse16x16_sse2(const uint8_t *src8, int src_stride, |
189 | | const uint8_t *ref8, int ref_stride, |
190 | 0 | unsigned int *sse) { |
191 | 0 | int sum; |
192 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); |
193 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); |
194 | 0 | highbd_12_variance_sse2(src, src_stride, ref, ref_stride, 16, 16, sse, &sum, |
195 | 0 | aom_highbd_calc16x16var_sse2, 16); |
196 | 0 | return *sse; |
197 | 0 | } |
198 | | |
199 | | unsigned int aom_highbd_8_mse8x8_sse2(const uint8_t *src8, int src_stride, |
200 | | const uint8_t *ref8, int ref_stride, |
201 | 0 | unsigned int *sse) { |
202 | 0 | int sum; |
203 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); |
204 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); |
205 | 0 | highbd_8_variance_sse2(src, src_stride, ref, ref_stride, 8, 8, sse, &sum, |
206 | 0 | aom_highbd_calc8x8var_sse2, 8); |
207 | 0 | return *sse; |
208 | 0 | } |
209 | | |
210 | | unsigned int aom_highbd_10_mse8x8_sse2(const uint8_t *src8, int src_stride, |
211 | | const uint8_t *ref8, int ref_stride, |
212 | 0 | unsigned int *sse) { |
213 | 0 | int sum; |
214 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); |
215 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); |
216 | 0 | highbd_10_variance_sse2(src, src_stride, ref, ref_stride, 8, 8, sse, &sum, |
217 | 0 | aom_highbd_calc8x8var_sse2, 8); |
218 | 0 | return *sse; |
219 | 0 | } |
220 | | |
221 | | unsigned int aom_highbd_12_mse8x8_sse2(const uint8_t *src8, int src_stride, |
222 | | const uint8_t *ref8, int ref_stride, |
223 | 0 | unsigned int *sse) { |
224 | 0 | int sum; |
225 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); |
226 | 0 | uint16_t *ref = CONVERT_TO_SHORTPTR(ref8); |
227 | 0 | highbd_12_variance_sse2(src, src_stride, ref, ref_stride, 8, 8, sse, &sum, |
228 | 0 | aom_highbd_calc8x8var_sse2, 8); |
229 | 0 | return *sse; |
230 | 0 | } |
231 | | |
232 | | // The 2 unused parameters are place holders for PIC enabled build. |
233 | | // These definitions are for functions defined in |
234 | | // highbd_subpel_variance_impl_sse2.asm |
235 | | #define DECL(w, opt) \ |
236 | | int aom_highbd_sub_pixel_variance##w##xh_##opt( \ |
237 | | const uint16_t *src, ptrdiff_t src_stride, int x_offset, int y_offset, \ |
238 | | const uint16_t *dst, ptrdiff_t dst_stride, int height, \ |
239 | | unsigned int *sse, void *unused0, void *unused); |
240 | | #define DECLS(opt) \ |
241 | | DECL(8, opt) \ |
242 | | DECL(16, opt) |
243 | | |
244 | | DECLS(sse2) |
245 | | |
246 | | #undef DECLS |
247 | | #undef DECL |
248 | | |
249 | | #define FN(w, h, wf, wlog2, hlog2, opt, cast) \ |
250 | | uint32_t aom_highbd_8_sub_pixel_variance##w##x##h##_##opt( \ |
251 | | const uint8_t *src8, int src_stride, int x_offset, int y_offset, \ |
252 | 0 | const uint8_t *dst8, int dst_stride, uint32_t *sse_ptr) { \ |
253 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
254 | 0 | uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); \ |
255 | 0 | int se = 0; \ |
256 | 0 | unsigned int sse = 0; \ |
257 | 0 | unsigned int sse2; \ |
258 | 0 | int row_rep = (w > 64) ? 2 : 1; \ |
259 | 0 | for (int wd_64 = 0; wd_64 < row_rep; wd_64++) { \ |
260 | 0 | src += wd_64 * 64; \ |
261 | 0 | dst += wd_64 * 64; \ |
262 | 0 | int se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ |
263 | 0 | src, src_stride, x_offset, y_offset, dst, dst_stride, h, &sse2, \ |
264 | 0 | NULL, NULL); \ |
265 | 0 | se += se2; \ |
266 | 0 | sse += sse2; \ |
267 | 0 | if (w > wf) { \ |
268 | 0 | se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ |
269 | 0 | src + wf, src_stride, x_offset, y_offset, dst + wf, dst_stride, h, \ |
270 | 0 | &sse2, NULL, NULL); \ |
271 | 0 | se += se2; \ |
272 | 0 | sse += sse2; \ |
273 | 0 | if (w > wf * 2) { \ |
274 | 0 | se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ |
275 | 0 | src + 2 * wf, src_stride, x_offset, y_offset, dst + 2 * wf, \ |
276 | 0 | dst_stride, h, &sse2, NULL, NULL); \ |
277 | 0 | se += se2; \ |
278 | 0 | sse += sse2; \ |
279 | 0 | se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ |
280 | 0 | src + 3 * wf, src_stride, x_offset, y_offset, dst + 3 * wf, \ |
281 | 0 | dst_stride, h, &sse2, NULL, NULL); \ |
282 | 0 | se += se2; \ |
283 | 0 | sse += sse2; \ |
284 | 0 | } \ |
285 | 0 | } \ |
286 | 0 | } \ |
287 | 0 | *sse_ptr = sse; \ |
288 | 0 | return sse - (uint32_t)((cast se * se) >> (wlog2 + hlog2)); \ |
289 | 0 | } \ |
290 | | \ |
291 | | uint32_t aom_highbd_10_sub_pixel_variance##w##x##h##_##opt( \ |
292 | | const uint8_t *src8, int src_stride, int x_offset, int y_offset, \ |
293 | 0 | const uint8_t *dst8, int dst_stride, uint32_t *sse_ptr) { \ |
294 | 0 | int64_t var; \ |
295 | 0 | uint32_t sse; \ |
296 | 0 | uint64_t long_sse = 0; \ |
297 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
298 | 0 | uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); \ |
299 | 0 | int se = 0; \ |
300 | 0 | int row_rep = (w > 64) ? 2 : 1; \ |
301 | 0 | for (int wd_64 = 0; wd_64 < row_rep; wd_64++) { \ |
302 | 0 | src += wd_64 * 64; \ |
303 | 0 | dst += wd_64 * 64; \ |
304 | 0 | int se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ |
305 | 0 | src, src_stride, x_offset, y_offset, dst, dst_stride, h, &sse, NULL, \ |
306 | 0 | NULL); \ |
307 | 0 | se += se2; \ |
308 | 0 | long_sse += sse; \ |
309 | 0 | if (w > wf) { \ |
310 | 0 | uint32_t sse2; \ |
311 | 0 | se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ |
312 | 0 | src + wf, src_stride, x_offset, y_offset, dst + wf, dst_stride, h, \ |
313 | 0 | &sse2, NULL, NULL); \ |
314 | 0 | se += se2; \ |
315 | 0 | long_sse += sse2; \ |
316 | 0 | if (w > wf * 2) { \ |
317 | 0 | se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ |
318 | 0 | src + 2 * wf, src_stride, x_offset, y_offset, dst + 2 * wf, \ |
319 | 0 | dst_stride, h, &sse2, NULL, NULL); \ |
320 | 0 | se += se2; \ |
321 | 0 | long_sse += sse2; \ |
322 | 0 | se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ |
323 | 0 | src + 3 * wf, src_stride, x_offset, y_offset, dst + 3 * wf, \ |
324 | 0 | dst_stride, h, &sse2, NULL, NULL); \ |
325 | 0 | se += se2; \ |
326 | 0 | long_sse += sse2; \ |
327 | 0 | } \ |
328 | 0 | } \ |
329 | 0 | } \ |
330 | 0 | se = ROUND_POWER_OF_TWO(se, 2); \ |
331 | 0 | sse = (uint32_t)ROUND_POWER_OF_TWO(long_sse, 4); \ |
332 | 0 | *sse_ptr = sse; \ |
333 | 0 | var = (int64_t)(sse) - ((cast se * se) >> (wlog2 + hlog2)); \ |
334 | 0 | return (var >= 0) ? (uint32_t)var : 0; \ |
335 | 0 | } \ |
336 | | \ |
337 | | uint32_t aom_highbd_12_sub_pixel_variance##w##x##h##_##opt( \ |
338 | | const uint8_t *src8, int src_stride, int x_offset, int y_offset, \ |
339 | 0 | const uint8_t *dst8, int dst_stride, uint32_t *sse_ptr) { \ |
340 | 0 | int start_row; \ |
341 | 0 | uint32_t sse; \ |
342 | 0 | int se = 0; \ |
343 | 0 | int64_t var; \ |
344 | 0 | uint64_t long_sse = 0; \ |
345 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
346 | 0 | uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); \ |
347 | 0 | int row_rep = (w > 64) ? 2 : 1; \ |
348 | 0 | for (start_row = 0; start_row < h; start_row += 16) { \ |
349 | 0 | uint32_t sse2; \ |
350 | 0 | int height = h - start_row < 16 ? h - start_row : 16; \ |
351 | 0 | uint16_t *src_tmp = src + (start_row * src_stride); \ |
352 | 0 | uint16_t *dst_tmp = dst + (start_row * dst_stride); \ |
353 | 0 | for (int wd_64 = 0; wd_64 < row_rep; wd_64++) { \ |
354 | 0 | src_tmp += wd_64 * 64; \ |
355 | 0 | dst_tmp += wd_64 * 64; \ |
356 | 0 | int se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ |
357 | 0 | src_tmp, src_stride, x_offset, y_offset, dst_tmp, dst_stride, \ |
358 | 0 | height, &sse2, NULL, NULL); \ |
359 | 0 | se += se2; \ |
360 | 0 | long_sse += sse2; \ |
361 | 0 | if (w > wf) { \ |
362 | 0 | se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ |
363 | 0 | src_tmp + wf, src_stride, x_offset, y_offset, dst_tmp + wf, \ |
364 | 0 | dst_stride, height, &sse2, NULL, NULL); \ |
365 | 0 | se += se2; \ |
366 | 0 | long_sse += sse2; \ |
367 | 0 | if (w > wf * 2) { \ |
368 | 0 | se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ |
369 | 0 | src_tmp + 2 * wf, src_stride, x_offset, y_offset, \ |
370 | 0 | dst_tmp + 2 * wf, dst_stride, height, &sse2, NULL, NULL); \ |
371 | 0 | se += se2; \ |
372 | 0 | long_sse += sse2; \ |
373 | 0 | se2 = aom_highbd_sub_pixel_variance##wf##xh_##opt( \ |
374 | 0 | src_tmp + 3 * wf, src_stride, x_offset, y_offset, \ |
375 | 0 | dst_tmp + 3 * wf, dst_stride, height, &sse2, NULL, NULL); \ |
376 | 0 | se += se2; \ |
377 | 0 | long_sse += sse2; \ |
378 | 0 | } \ |
379 | 0 | } \ |
380 | 0 | } \ |
381 | 0 | } \ |
382 | 0 | se = ROUND_POWER_OF_TWO(se, 4); \ |
383 | 0 | sse = (uint32_t)ROUND_POWER_OF_TWO(long_sse, 8); \ |
384 | 0 | *sse_ptr = sse; \ |
385 | 0 | var = (int64_t)(sse) - ((cast se * se) >> (wlog2 + hlog2)); \ |
386 | 0 | return (var >= 0) ? (uint32_t)var : 0; \ |
387 | 0 | } |
388 | | |
389 | | #if CONFIG_REALTIME_ONLY |
390 | | #define FNS(opt) \ |
391 | | FN(128, 128, 16, 7, 7, opt, (int64_t)) \ |
392 | | FN(128, 64, 16, 7, 6, opt, (int64_t)) \ |
393 | | FN(64, 128, 16, 6, 7, opt, (int64_t)) \ |
394 | | FN(64, 64, 16, 6, 6, opt, (int64_t)) \ |
395 | | FN(64, 32, 16, 6, 5, opt, (int64_t)) \ |
396 | | FN(32, 64, 16, 5, 6, opt, (int64_t)) \ |
397 | | FN(32, 32, 16, 5, 5, opt, (int64_t)) \ |
398 | | FN(32, 16, 16, 5, 4, opt, (int64_t)) \ |
399 | | FN(16, 32, 16, 4, 5, opt, (int64_t)) \ |
400 | | FN(16, 16, 16, 4, 4, opt, (int64_t)) \ |
401 | | FN(16, 8, 16, 4, 3, opt, (int64_t)) \ |
402 | | FN(8, 16, 8, 3, 4, opt, (int64_t)) \ |
403 | | FN(8, 8, 8, 3, 3, opt, (int64_t)) \ |
404 | | FN(8, 4, 8, 3, 2, opt, (int64_t)) |
405 | | #else // !CONFIG_REALTIME_ONLY |
406 | | #define FNS(opt) \ |
407 | | FN(128, 128, 16, 7, 7, opt, (int64_t)) \ |
408 | | FN(128, 64, 16, 7, 6, opt, (int64_t)) \ |
409 | | FN(64, 128, 16, 6, 7, opt, (int64_t)) \ |
410 | | FN(64, 64, 16, 6, 6, opt, (int64_t)) \ |
411 | | FN(64, 32, 16, 6, 5, opt, (int64_t)) \ |
412 | | FN(32, 64, 16, 5, 6, opt, (int64_t)) \ |
413 | | FN(32, 32, 16, 5, 5, opt, (int64_t)) \ |
414 | | FN(32, 16, 16, 5, 4, opt, (int64_t)) \ |
415 | | FN(16, 32, 16, 4, 5, opt, (int64_t)) \ |
416 | | FN(16, 16, 16, 4, 4, opt, (int64_t)) \ |
417 | | FN(16, 8, 16, 4, 3, opt, (int64_t)) \ |
418 | | FN(8, 16, 8, 3, 4, opt, (int64_t)) \ |
419 | | FN(8, 8, 8, 3, 3, opt, (int64_t)) \ |
420 | | FN(8, 4, 8, 3, 2, opt, (int64_t)) \ |
421 | | FN(16, 4, 16, 4, 2, opt, (int64_t)) \ |
422 | | FN(8, 32, 8, 3, 5, opt, (int64_t)) \ |
423 | | FN(32, 8, 16, 5, 3, opt, (int64_t)) \ |
424 | | FN(16, 64, 16, 4, 6, opt, (int64_t)) \ |
425 | | FN(64, 16, 16, 6, 4, opt, (int64_t)) |
426 | | #endif // CONFIG_REALTIME_ONLY |
427 | | |
428 | 0 | FNS(sse2) Unexecuted instantiation: aom_highbd_8_sub_pixel_variance128x128_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_variance128x128_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_variance128x128_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_variance128x64_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_variance128x64_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_variance128x64_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_variance64x128_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_variance64x128_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_variance64x128_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_variance64x64_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_variance64x64_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_variance64x64_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_variance64x32_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_variance64x32_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_variance64x32_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_variance32x64_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_variance32x64_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_variance32x64_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_variance32x32_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_variance32x32_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_variance32x32_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_variance32x16_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_variance32x16_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_variance32x16_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_variance16x32_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_variance16x32_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_variance16x32_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_variance16x16_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_variance16x16_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_variance16x16_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_variance16x8_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_variance16x8_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_variance16x8_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_variance8x16_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_variance8x16_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_variance8x16_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_variance8x8_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_variance8x8_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_variance8x8_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_variance8x4_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_variance8x4_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_variance8x4_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_variance16x4_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_variance16x4_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_variance16x4_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_variance8x32_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_variance8x32_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_variance8x32_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_variance32x8_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_variance32x8_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_variance32x8_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_variance16x64_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_variance16x64_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_variance16x64_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_variance64x16_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_variance64x16_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_variance64x16_sse2 |
429 | 0 |
|
430 | 0 | #undef FNS |
431 | 0 | #undef FN |
432 | 0 |
|
433 | 0 | // The 2 unused parameters are place holders for PIC enabled build. |
434 | 0 | #define DECL(w, opt) \ |
435 | 0 | int aom_highbd_sub_pixel_avg_variance##w##xh_##opt( \ |
436 | 0 | const uint16_t *src, ptrdiff_t src_stride, int x_offset, int y_offset, \ |
437 | 0 | const uint16_t *dst, ptrdiff_t dst_stride, const uint16_t *sec, \ |
438 | 0 | ptrdiff_t sec_stride, int height, unsigned int *sse, void *unused0, \ |
439 | 0 | void *unused); |
440 | 0 | #define DECLS(opt) \ |
441 | 0 | DECL(16, opt) \ |
442 | 0 | DECL(8, opt) |
443 | 0 |
|
444 | 0 | DECLS(sse2) |
445 | 0 | #undef DECL |
446 | 0 | #undef DECLS |
447 | 0 |
|
448 | 0 | #define FN(w, h, wf, wlog2, hlog2, opt, cast) \ |
449 | 0 | uint32_t aom_highbd_8_sub_pixel_avg_variance##w##x##h##_##opt( \ |
450 | 0 | const uint8_t *src8, int src_stride, int x_offset, int y_offset, \ |
451 | 0 | const uint8_t *dst8, int dst_stride, uint32_t *sse_ptr, \ |
452 | 0 | const uint8_t *sec8) { \ |
453 | 0 | uint32_t sse; \ |
454 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
455 | 0 | uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); \ |
456 | 0 | uint16_t *sec = CONVERT_TO_SHORTPTR(sec8); \ |
457 | 0 | int se = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
458 | 0 | src, src_stride, x_offset, y_offset, dst, dst_stride, sec, w, h, &sse, \ |
459 | 0 | NULL, NULL); \ |
460 | 0 | if (w > wf) { \ |
461 | 0 | uint32_t sse2; \ |
462 | 0 | int se2 = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
463 | 0 | src + wf, src_stride, x_offset, y_offset, dst + wf, dst_stride, \ |
464 | 0 | sec + wf, w, h, &sse2, NULL, NULL); \ |
465 | 0 | se += se2; \ |
466 | 0 | sse += sse2; \ |
467 | 0 | if (w > wf * 2) { \ |
468 | 0 | se2 = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
469 | 0 | src + 2 * wf, src_stride, x_offset, y_offset, dst + 2 * wf, \ |
470 | 0 | dst_stride, sec + 2 * wf, w, h, &sse2, NULL, NULL); \ |
471 | 0 | se += se2; \ |
472 | 0 | sse += sse2; \ |
473 | 0 | se2 = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
474 | 0 | src + 3 * wf, src_stride, x_offset, y_offset, dst + 3 * wf, \ |
475 | 0 | dst_stride, sec + 3 * wf, w, h, &sse2, NULL, NULL); \ |
476 | 0 | se += se2; \ |
477 | 0 | sse += sse2; \ |
478 | 0 | } \ |
479 | 0 | } \ |
480 | 0 | *sse_ptr = sse; \ |
481 | 0 | return sse - (uint32_t)((cast se * se) >> (wlog2 + hlog2)); \ |
482 | 0 | } \ |
483 | | \ |
484 | | uint32_t aom_highbd_10_sub_pixel_avg_variance##w##x##h##_##opt( \ |
485 | | const uint8_t *src8, int src_stride, int x_offset, int y_offset, \ |
486 | | const uint8_t *dst8, int dst_stride, uint32_t *sse_ptr, \ |
487 | 0 | const uint8_t *sec8) { \ |
488 | 0 | int64_t var; \ |
489 | 0 | uint32_t sse; \ |
490 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
491 | 0 | uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); \ |
492 | 0 | uint16_t *sec = CONVERT_TO_SHORTPTR(sec8); \ |
493 | 0 | int se = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
494 | 0 | src, src_stride, x_offset, y_offset, dst, dst_stride, sec, w, h, &sse, \ |
495 | 0 | NULL, NULL); \ |
496 | 0 | if (w > wf) { \ |
497 | 0 | uint32_t sse2; \ |
498 | 0 | int se2 = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
499 | 0 | src + wf, src_stride, x_offset, y_offset, dst + wf, dst_stride, \ |
500 | 0 | sec + wf, w, h, &sse2, NULL, NULL); \ |
501 | 0 | se += se2; \ |
502 | 0 | sse += sse2; \ |
503 | 0 | if (w > wf * 2) { \ |
504 | 0 | se2 = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
505 | 0 | src + 2 * wf, src_stride, x_offset, y_offset, dst + 2 * wf, \ |
506 | 0 | dst_stride, sec + 2 * wf, w, h, &sse2, NULL, NULL); \ |
507 | 0 | se += se2; \ |
508 | 0 | sse += sse2; \ |
509 | 0 | se2 = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
510 | 0 | src + 3 * wf, src_stride, x_offset, y_offset, dst + 3 * wf, \ |
511 | 0 | dst_stride, sec + 3 * wf, w, h, &sse2, NULL, NULL); \ |
512 | 0 | se += se2; \ |
513 | 0 | sse += sse2; \ |
514 | 0 | } \ |
515 | 0 | } \ |
516 | 0 | se = ROUND_POWER_OF_TWO(se, 2); \ |
517 | 0 | sse = ROUND_POWER_OF_TWO(sse, 4); \ |
518 | 0 | *sse_ptr = sse; \ |
519 | 0 | var = (int64_t)(sse) - ((cast se * se) >> (wlog2 + hlog2)); \ |
520 | 0 | return (var >= 0) ? (uint32_t)var : 0; \ |
521 | 0 | } \ |
522 | | \ |
523 | | uint32_t aom_highbd_12_sub_pixel_avg_variance##w##x##h##_##opt( \ |
524 | | const uint8_t *src8, int src_stride, int x_offset, int y_offset, \ |
525 | | const uint8_t *dst8, int dst_stride, uint32_t *sse_ptr, \ |
526 | 0 | const uint8_t *sec8) { \ |
527 | 0 | int start_row; \ |
528 | 0 | int64_t var; \ |
529 | 0 | uint32_t sse; \ |
530 | 0 | int se = 0; \ |
531 | 0 | uint64_t long_sse = 0; \ |
532 | 0 | uint16_t *src = CONVERT_TO_SHORTPTR(src8); \ |
533 | 0 | uint16_t *dst = CONVERT_TO_SHORTPTR(dst8); \ |
534 | 0 | uint16_t *sec = CONVERT_TO_SHORTPTR(sec8); \ |
535 | 0 | for (start_row = 0; start_row < h; start_row += 16) { \ |
536 | 0 | uint32_t sse2; \ |
537 | 0 | int height = h - start_row < 16 ? h - start_row : 16; \ |
538 | 0 | int se2 = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
539 | 0 | src + (start_row * src_stride), src_stride, x_offset, y_offset, \ |
540 | 0 | dst + (start_row * dst_stride), dst_stride, sec + (start_row * w), \ |
541 | 0 | w, height, &sse2, NULL, NULL); \ |
542 | 0 | se += se2; \ |
543 | 0 | long_sse += sse2; \ |
544 | 0 | if (w > wf) { \ |
545 | 0 | se2 = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
546 | 0 | src + wf + (start_row * src_stride), src_stride, x_offset, \ |
547 | 0 | y_offset, dst + wf + (start_row * dst_stride), dst_stride, \ |
548 | 0 | sec + wf + (start_row * w), w, height, &sse2, NULL, NULL); \ |
549 | 0 | se += se2; \ |
550 | 0 | long_sse += sse2; \ |
551 | 0 | if (w > wf * 2) { \ |
552 | 0 | se2 = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
553 | 0 | src + 2 * wf + (start_row * src_stride), src_stride, x_offset, \ |
554 | 0 | y_offset, dst + 2 * wf + (start_row * dst_stride), dst_stride, \ |
555 | 0 | sec + 2 * wf + (start_row * w), w, height, &sse2, NULL, NULL); \ |
556 | 0 | se += se2; \ |
557 | 0 | long_sse += sse2; \ |
558 | 0 | se2 = aom_highbd_sub_pixel_avg_variance##wf##xh_##opt( \ |
559 | 0 | src + 3 * wf + (start_row * src_stride), src_stride, x_offset, \ |
560 | 0 | y_offset, dst + 3 * wf + (start_row * dst_stride), dst_stride, \ |
561 | 0 | sec + 3 * wf + (start_row * w), w, height, &sse2, NULL, NULL); \ |
562 | 0 | se += se2; \ |
563 | 0 | long_sse += sse2; \ |
564 | 0 | } \ |
565 | 0 | } \ |
566 | 0 | } \ |
567 | 0 | se = ROUND_POWER_OF_TWO(se, 4); \ |
568 | 0 | sse = (uint32_t)ROUND_POWER_OF_TWO(long_sse, 8); \ |
569 | 0 | *sse_ptr = sse; \ |
570 | 0 | var = (int64_t)(sse) - ((cast se * se) >> (wlog2 + hlog2)); \ |
571 | 0 | return (var >= 0) ? (uint32_t)var : 0; \ |
572 | 0 | } |
573 | | |
574 | | #if CONFIG_REALTIME_ONLY |
575 | | #define FNS(opt) \ |
576 | | FN(64, 64, 16, 6, 6, opt, (int64_t)) \ |
577 | | FN(64, 32, 16, 6, 5, opt, (int64_t)) \ |
578 | | FN(32, 64, 16, 5, 6, opt, (int64_t)) \ |
579 | | FN(32, 32, 16, 5, 5, opt, (int64_t)) \ |
580 | | FN(32, 16, 16, 5, 4, opt, (int64_t)) \ |
581 | | FN(16, 32, 16, 4, 5, opt, (int64_t)) \ |
582 | | FN(16, 16, 16, 4, 4, opt, (int64_t)) \ |
583 | | FN(16, 8, 16, 4, 3, opt, (int64_t)) \ |
584 | | FN(8, 16, 8, 3, 4, opt, (int64_t)) \ |
585 | | FN(8, 8, 8, 3, 3, opt, (int64_t)) \ |
586 | | FN(8, 4, 8, 3, 2, opt, (int64_t)) |
587 | | #else // !CONFIG_REALTIME_ONLY |
588 | | #define FNS(opt) \ |
589 | | FN(64, 64, 16, 6, 6, opt, (int64_t)) \ |
590 | | FN(64, 32, 16, 6, 5, opt, (int64_t)) \ |
591 | | FN(32, 64, 16, 5, 6, opt, (int64_t)) \ |
592 | | FN(32, 32, 16, 5, 5, opt, (int64_t)) \ |
593 | | FN(32, 16, 16, 5, 4, opt, (int64_t)) \ |
594 | | FN(16, 32, 16, 4, 5, opt, (int64_t)) \ |
595 | | FN(16, 16, 16, 4, 4, opt, (int64_t)) \ |
596 | | FN(16, 8, 16, 4, 3, opt, (int64_t)) \ |
597 | | FN(8, 16, 8, 3, 4, opt, (int64_t)) \ |
598 | | FN(8, 8, 8, 3, 3, opt, (int64_t)) \ |
599 | | FN(8, 4, 8, 3, 2, opt, (int64_t)) \ |
600 | | FN(16, 4, 16, 4, 2, opt, (int64_t)) \ |
601 | | FN(8, 32, 8, 3, 5, opt, (int64_t)) \ |
602 | | FN(32, 8, 16, 5, 3, opt, (int64_t)) \ |
603 | | FN(16, 64, 16, 4, 6, opt, (int64_t)) \ |
604 | | FN(64, 16, 16, 6, 4, opt, (int64_t)) |
605 | | #endif // CONFIG_REALTIME_ONLY |
606 | | |
607 | 0 | FNS(sse2) Unexecuted instantiation: aom_highbd_8_sub_pixel_avg_variance64x64_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_avg_variance64x64_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_avg_variance64x64_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_avg_variance64x32_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_avg_variance64x32_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_avg_variance64x32_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_avg_variance32x64_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_avg_variance32x64_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_avg_variance32x64_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_avg_variance32x32_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_avg_variance32x32_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_avg_variance32x32_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_avg_variance32x16_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_avg_variance32x16_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_avg_variance32x16_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_avg_variance16x32_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_avg_variance16x32_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_avg_variance16x32_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_avg_variance16x16_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_avg_variance16x16_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_avg_variance16x16_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_avg_variance16x8_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_avg_variance16x8_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_avg_variance16x8_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_avg_variance8x16_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_avg_variance8x16_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_avg_variance8x16_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_avg_variance8x8_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_avg_variance8x8_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_avg_variance8x8_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_avg_variance8x4_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_avg_variance8x4_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_avg_variance8x4_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_avg_variance16x4_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_avg_variance16x4_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_avg_variance16x4_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_avg_variance8x32_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_avg_variance8x32_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_avg_variance8x32_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_avg_variance32x8_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_avg_variance32x8_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_avg_variance32x8_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_avg_variance16x64_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_avg_variance16x64_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_avg_variance16x64_sse2 Unexecuted instantiation: aom_highbd_8_sub_pixel_avg_variance64x16_sse2 Unexecuted instantiation: aom_highbd_10_sub_pixel_avg_variance64x16_sse2 Unexecuted instantiation: aom_highbd_12_sub_pixel_avg_variance64x16_sse2 |
608 | 0 |
|
609 | 0 | #undef FNS |
610 | 0 | #undef FN |
611 | 0 | #endif // HAVE_X86_ASM |
612 | 0 |
|
613 | 0 | static uint64_t mse_4xh_16bit_highbd_sse2(uint16_t *dst, int dstride, |
614 | 0 | uint16_t *src, int sstride, int h) { |
615 | 0 | uint64_t sum = 0; |
616 | 0 | __m128i reg0_4x16, reg1_4x16; |
617 | 0 | __m128i src_8x16; |
618 | 0 | __m128i dst_8x16; |
619 | 0 | __m128i res0_4x32, res1_4x32, res0_4x64, res1_4x64, res2_4x64, res3_4x64; |
620 | 0 | __m128i sub_result_8x16; |
621 | 0 | const __m128i zeros = _mm_setzero_si128(); |
622 | 0 | __m128i square_result = _mm_setzero_si128(); |
623 | 0 | for (int i = 0; i < h; i += 2) { |
624 | 0 | reg0_4x16 = _mm_loadl_epi64((__m128i const *)(&dst[(i + 0) * dstride])); |
625 | 0 | reg1_4x16 = _mm_loadl_epi64((__m128i const *)(&dst[(i + 1) * dstride])); |
626 | 0 | dst_8x16 = _mm_unpacklo_epi64(reg0_4x16, reg1_4x16); |
627 | |
|
628 | 0 | reg0_4x16 = _mm_loadl_epi64((__m128i const *)(&src[(i + 0) * sstride])); |
629 | 0 | reg1_4x16 = _mm_loadl_epi64((__m128i const *)(&src[(i + 1) * sstride])); |
630 | 0 | src_8x16 = _mm_unpacklo_epi64(reg0_4x16, reg1_4x16); |
631 | |
|
632 | 0 | sub_result_8x16 = _mm_sub_epi16(src_8x16, dst_8x16); |
633 | |
|
634 | 0 | res0_4x32 = _mm_unpacklo_epi16(sub_result_8x16, zeros); |
635 | 0 | res1_4x32 = _mm_unpackhi_epi16(sub_result_8x16, zeros); |
636 | |
|
637 | 0 | res0_4x32 = _mm_madd_epi16(res0_4x32, res0_4x32); |
638 | 0 | res1_4x32 = _mm_madd_epi16(res1_4x32, res1_4x32); |
639 | |
|
640 | 0 | res0_4x64 = _mm_unpacklo_epi32(res0_4x32, zeros); |
641 | 0 | res1_4x64 = _mm_unpackhi_epi32(res0_4x32, zeros); |
642 | 0 | res2_4x64 = _mm_unpacklo_epi32(res1_4x32, zeros); |
643 | 0 | res3_4x64 = _mm_unpackhi_epi32(res1_4x32, zeros); |
644 | |
|
645 | 0 | square_result = _mm_add_epi64( |
646 | 0 | square_result, |
647 | 0 | _mm_add_epi64( |
648 | 0 | _mm_add_epi64(_mm_add_epi64(res0_4x64, res1_4x64), res2_4x64), |
649 | 0 | res3_4x64)); |
650 | 0 | } |
651 | |
|
652 | 0 | const __m128i sum_1x64 = |
653 | 0 | _mm_add_epi64(square_result, _mm_srli_si128(square_result, 8)); |
654 | 0 | xx_storel_64(&sum, sum_1x64); |
655 | 0 | return sum; |
656 | 0 | } |
657 | | |
658 | | static uint64_t mse_8xh_16bit_highbd_sse2(uint16_t *dst, int dstride, |
659 | 0 | uint16_t *src, int sstride, int h) { |
660 | 0 | uint64_t sum = 0; |
661 | 0 | __m128i src_8x16; |
662 | 0 | __m128i dst_8x16; |
663 | 0 | __m128i res0_4x32, res1_4x32, res0_4x64, res1_4x64, res2_4x64, res3_4x64; |
664 | 0 | __m128i sub_result_8x16; |
665 | 0 | const __m128i zeros = _mm_setzero_si128(); |
666 | 0 | __m128i square_result = _mm_setzero_si128(); |
667 | |
|
668 | 0 | for (int i = 0; i < h; i++) { |
669 | 0 | dst_8x16 = _mm_loadu_si128((__m128i *)&dst[i * dstride]); |
670 | 0 | src_8x16 = _mm_loadu_si128((__m128i *)&src[i * sstride]); |
671 | |
|
672 | 0 | sub_result_8x16 = _mm_sub_epi16(src_8x16, dst_8x16); |
673 | |
|
674 | 0 | res0_4x32 = _mm_unpacklo_epi16(sub_result_8x16, zeros); |
675 | 0 | res1_4x32 = _mm_unpackhi_epi16(sub_result_8x16, zeros); |
676 | |
|
677 | 0 | res0_4x32 = _mm_madd_epi16(res0_4x32, res0_4x32); |
678 | 0 | res1_4x32 = _mm_madd_epi16(res1_4x32, res1_4x32); |
679 | |
|
680 | 0 | res0_4x64 = _mm_unpacklo_epi32(res0_4x32, zeros); |
681 | 0 | res1_4x64 = _mm_unpackhi_epi32(res0_4x32, zeros); |
682 | 0 | res2_4x64 = _mm_unpacklo_epi32(res1_4x32, zeros); |
683 | 0 | res3_4x64 = _mm_unpackhi_epi32(res1_4x32, zeros); |
684 | |
|
685 | 0 | square_result = _mm_add_epi64( |
686 | 0 | square_result, |
687 | 0 | _mm_add_epi64( |
688 | 0 | _mm_add_epi64(_mm_add_epi64(res0_4x64, res1_4x64), res2_4x64), |
689 | 0 | res3_4x64)); |
690 | 0 | } |
691 | |
|
692 | 0 | const __m128i sum_1x64 = |
693 | 0 | _mm_add_epi64(square_result, _mm_srli_si128(square_result, 8)); |
694 | 0 | xx_storel_64(&sum, sum_1x64); |
695 | 0 | return sum; |
696 | 0 | } |
697 | | |
698 | | uint64_t aom_mse_wxh_16bit_highbd_sse2(uint16_t *dst, int dstride, |
699 | | uint16_t *src, int sstride, int w, |
700 | 0 | int h) { |
701 | 0 | assert((w == 8 || w == 4) && (h == 8 || h == 4) && |
702 | 0 | "w=8/4 and h=8/4 must satisfy"); |
703 | 0 | switch (w) { |
704 | 0 | case 4: return mse_4xh_16bit_highbd_sse2(dst, dstride, src, sstride, h); |
705 | 0 | case 8: return mse_8xh_16bit_highbd_sse2(dst, dstride, src, sstride, h); |
706 | 0 | default: assert(0 && "unsupported width"); return -1; |
707 | 0 | } |
708 | 0 | } |