/src/aom/av1/common/x86/jnt_convolve_sse2.c
Line | Count | Source (jump to first uncovered line) |
1 | | /* |
2 | | * Copyright (c) 2018, Alliance for Open Media. All rights reserved. |
3 | | * |
4 | | * This source code is subject to the terms of the BSD 2 Clause License and |
5 | | * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
6 | | * was not distributed with this source code in the LICENSE file, you can |
7 | | * obtain it at www.aomedia.org/license/software. If the Alliance for Open |
8 | | * Media Patent License 1.0 was not distributed with this source code in the |
9 | | * PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
10 | | */ |
11 | | |
12 | | #include <emmintrin.h> |
13 | | |
14 | | #include "config/av1_rtcd.h" |
15 | | |
16 | | #include "aom_dsp/aom_filter.h" |
17 | | #include "aom_dsp/x86/convolve_sse2.h" |
18 | | #include "aom_dsp/x86/synonyms.h" |
19 | | |
20 | | void av1_dist_wtd_convolve_x_sse2(const uint8_t *src, int src_stride, |
21 | | uint8_t *dst0, int dst_stride0, int w, int h, |
22 | | const InterpFilterParams *filter_params_x, |
23 | | const int subpel_x_qn, |
24 | 0 | ConvolveParams *conv_params) { |
25 | 0 | const int bd = 8; |
26 | 0 | CONV_BUF_TYPE *dst = conv_params->dst; |
27 | 0 | const int dst_stride = conv_params->dst_stride; |
28 | 0 | const int fo_horiz = filter_params_x->taps / 2 - 1; |
29 | 0 | const uint8_t *src_ptr = src - fo_horiz; |
30 | 0 | const int bits = FILTER_BITS - conv_params->round_1; |
31 | 0 | const __m128i left_shift = _mm_cvtsi32_si128(bits); |
32 | 0 | const __m128i round_const = _mm_set1_epi32((1 << conv_params->round_0) >> 1); |
33 | 0 | const __m128i round_shift = _mm_cvtsi32_si128(conv_params->round_0); |
34 | 0 | const int w0 = conv_params->fwd_offset; |
35 | 0 | const int w1 = conv_params->bck_offset; |
36 | 0 | const __m128i wt0 = _mm_set1_epi16(w0); |
37 | 0 | const __m128i wt1 = _mm_set1_epi16(w1); |
38 | 0 | const __m128i wt = _mm_unpacklo_epi16(wt0, wt1); |
39 | 0 | const int do_average = conv_params->do_average; |
40 | 0 | const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; |
41 | 0 | const int offset_0 = |
42 | 0 | bd + 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; |
43 | 0 | const int offset = (1 << offset_0) + (1 << (offset_0 - 1)); |
44 | 0 | const __m128i offset_const = _mm_set1_epi16(offset); |
45 | 0 | const int rounding_shift = |
46 | 0 | 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; |
47 | 0 | const __m128i rounding_const = _mm_set1_epi16((1 << rounding_shift) >> 1); |
48 | 0 | __m128i coeffs[4]; |
49 | |
|
50 | 0 | prepare_coeffs(filter_params_x, subpel_x_qn, coeffs); |
51 | |
|
52 | 0 | if (w == 4) { |
53 | 0 | do { |
54 | 0 | const __m128i data = _mm_loadu_si128((__m128i *)src_ptr); |
55 | 0 | __m128i s[4]; |
56 | |
|
57 | 0 | s[0] = _mm_unpacklo_epi8(data, _mm_srli_si128(data, 1)); |
58 | 0 | s[1] = |
59 | 0 | _mm_unpacklo_epi8(_mm_srli_si128(data, 2), _mm_srli_si128(data, 3)); |
60 | 0 | s[2] = |
61 | 0 | _mm_unpacklo_epi8(_mm_srli_si128(data, 4), _mm_srli_si128(data, 5)); |
62 | 0 | s[3] = |
63 | 0 | _mm_unpacklo_epi8(_mm_srli_si128(data, 6), _mm_srli_si128(data, 7)); |
64 | 0 | const __m128i res_lo = convolve_lo_x(s, coeffs); |
65 | 0 | const __m128i res_lo_round = |
66 | 0 | _mm_sra_epi32(_mm_add_epi32(res_lo, round_const), round_shift); |
67 | 0 | const __m128i res_lo_shift = _mm_sll_epi32(res_lo_round, left_shift); |
68 | |
|
69 | 0 | const __m128i res_16b = _mm_packs_epi32(res_lo_shift, res_lo_shift); |
70 | 0 | const __m128i res_unsigned = _mm_add_epi16(res_16b, offset_const); |
71 | | |
72 | | // Accumulate values into the destination buffer |
73 | 0 | if (do_average) { |
74 | 0 | const __m128i data_ref_0 = _mm_loadu_si128((__m128i *)dst); |
75 | |
|
76 | 0 | const __m128i comp_avg_res = |
77 | 0 | comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg); |
78 | |
|
79 | 0 | const __m128i round_result = convolve_rounding( |
80 | 0 | &comp_avg_res, &offset_const, &rounding_const, rounding_shift); |
81 | |
|
82 | 0 | const __m128i res_8 = _mm_packus_epi16(round_result, round_result); |
83 | 0 | *(int *)(&dst0[0]) = _mm_cvtsi128_si32(res_8); |
84 | 0 | } else { |
85 | 0 | _mm_store_si128((__m128i *)(&dst[0]), res_unsigned); |
86 | 0 | } |
87 | 0 | src_ptr += src_stride; |
88 | 0 | dst += dst_stride; |
89 | 0 | dst0 += dst_stride0; |
90 | 0 | } while (--h); |
91 | 0 | } else { |
92 | 0 | assert(!(w % 8)); |
93 | 0 | int i = 0; |
94 | 0 | do { |
95 | 0 | int j = 0; |
96 | 0 | do { |
97 | 0 | const __m128i data = |
98 | 0 | _mm_loadu_si128((__m128i *)&src_ptr[i * src_stride + j]); |
99 | 0 | __m128i s[4]; |
100 | | |
101 | | // Filter even-index pixels |
102 | 0 | s[0] = data; |
103 | 0 | s[1] = _mm_srli_si128(data, 2); |
104 | 0 | s[2] = _mm_srli_si128(data, 4); |
105 | 0 | s[3] = _mm_srli_si128(data, 6); |
106 | 0 | const __m128i res_even = convolve_lo_x(s, coeffs); |
107 | | |
108 | | // Filter odd-index pixels |
109 | 0 | s[0] = _mm_srli_si128(data, 1); |
110 | 0 | s[1] = _mm_srli_si128(data, 3); |
111 | 0 | s[2] = _mm_srli_si128(data, 5); |
112 | 0 | s[3] = _mm_srli_si128(data, 7); |
113 | 0 | const __m128i res_odd = convolve_lo_x(s, coeffs); |
114 | | |
115 | | // Rearrange pixels back into the order 0 ... 7 |
116 | 0 | const __m128i res_lo = _mm_unpacklo_epi32(res_even, res_odd); |
117 | 0 | const __m128i res_hi = _mm_unpackhi_epi32(res_even, res_odd); |
118 | 0 | const __m128i res_lo_round = |
119 | 0 | _mm_sra_epi32(_mm_add_epi32(res_lo, round_const), round_shift); |
120 | 0 | const __m128i res_hi_round = |
121 | 0 | _mm_sra_epi32(_mm_add_epi32(res_hi, round_const), round_shift); |
122 | 0 | const __m128i res_lo_shift = _mm_sll_epi32(res_lo_round, left_shift); |
123 | 0 | const __m128i res_hi_shift = _mm_sll_epi32(res_hi_round, left_shift); |
124 | |
|
125 | 0 | const __m128i res_16b = _mm_packs_epi32(res_lo_shift, res_hi_shift); |
126 | 0 | const __m128i res_unsigned = _mm_add_epi16(res_16b, offset_const); |
127 | | |
128 | | // Accumulate values into the destination buffer |
129 | 0 | if (do_average) { |
130 | 0 | const __m128i data_ref_0 = |
131 | 0 | _mm_loadu_si128((__m128i *)(&dst[i * dst_stride + j])); |
132 | |
|
133 | 0 | const __m128i comp_avg_res = |
134 | 0 | comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg); |
135 | |
|
136 | 0 | const __m128i round_result = convolve_rounding( |
137 | 0 | &comp_avg_res, &offset_const, &rounding_const, rounding_shift); |
138 | |
|
139 | 0 | const __m128i res_8 = _mm_packus_epi16(round_result, round_result); |
140 | 0 | _mm_storel_epi64((__m128i *)(&dst0[i * dst_stride0 + j]), res_8); |
141 | 0 | } else { |
142 | 0 | _mm_store_si128((__m128i *)(&dst[i * dst_stride + j]), res_unsigned); |
143 | 0 | } |
144 | 0 | j += 8; |
145 | 0 | } while (j < w); |
146 | 0 | } while (++i < h); |
147 | 0 | } |
148 | 0 | } |
149 | | |
150 | | void av1_dist_wtd_convolve_y_sse2(const uint8_t *src, int src_stride, |
151 | | uint8_t *dst0, int dst_stride0, int w, int h, |
152 | | const InterpFilterParams *filter_params_y, |
153 | | const int subpel_y_qn, |
154 | 0 | ConvolveParams *conv_params) { |
155 | 0 | const int bd = 8; |
156 | 0 | CONV_BUF_TYPE *dst = conv_params->dst; |
157 | 0 | const int dst_stride = conv_params->dst_stride; |
158 | 0 | const int fo_vert = filter_params_y->taps / 2 - 1; |
159 | 0 | const uint8_t *src_ptr = src - fo_vert * src_stride; |
160 | 0 | const int bits = FILTER_BITS - conv_params->round_0; |
161 | 0 | const __m128i left_shift = _mm_cvtsi32_si128(bits); |
162 | 0 | const __m128i wt0 = _mm_set1_epi16(conv_params->fwd_offset); |
163 | 0 | const __m128i wt1 = _mm_set1_epi16(conv_params->bck_offset); |
164 | 0 | const __m128i wt = _mm_unpacklo_epi16(wt0, wt1); |
165 | 0 | const int do_average = conv_params->do_average; |
166 | 0 | const int use_dist_wtd_comp_avg = conv_params->use_dist_wtd_comp_avg; |
167 | 0 | const int offset_0 = |
168 | 0 | bd + 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; |
169 | 0 | const int offset = (1 << offset_0) + (1 << (offset_0 - 1)); |
170 | 0 | const __m128i offset_const = _mm_set1_epi16(offset); |
171 | 0 | const int rounding_shift = |
172 | 0 | 2 * FILTER_BITS - conv_params->round_0 - conv_params->round_1; |
173 | 0 | const __m128i rounding_const = _mm_set1_epi16((1 << rounding_shift) >> 1); |
174 | 0 | const __m128i round_const = _mm_set1_epi32((1 << conv_params->round_1) >> 1); |
175 | 0 | const __m128i round_shift = _mm_cvtsi32_si128(conv_params->round_1); |
176 | 0 | __m128i coeffs[4]; |
177 | |
|
178 | 0 | prepare_coeffs(filter_params_y, subpel_y_qn, coeffs); |
179 | |
|
180 | 0 | if (w == 4) { |
181 | 0 | __m128i s[8], src6, res, res_shift; |
182 | 0 | s[0] = _mm_unpacklo_epi8(xx_loadl_32(src_ptr + 0 * src_stride), |
183 | 0 | xx_loadl_32(src_ptr + 1 * src_stride)); |
184 | 0 | s[1] = _mm_unpacklo_epi8(xx_loadl_32(src_ptr + 1 * src_stride), |
185 | 0 | xx_loadl_32(src_ptr + 2 * src_stride)); |
186 | 0 | s[2] = _mm_unpacklo_epi8(xx_loadl_32(src_ptr + 2 * src_stride), |
187 | 0 | xx_loadl_32(src_ptr + 3 * src_stride)); |
188 | 0 | s[3] = _mm_unpacklo_epi8(xx_loadl_32(src_ptr + 3 * src_stride), |
189 | 0 | xx_loadl_32(src_ptr + 4 * src_stride)); |
190 | 0 | s[4] = _mm_unpacklo_epi8(xx_loadl_32(src_ptr + 4 * src_stride), |
191 | 0 | xx_loadl_32(src_ptr + 5 * src_stride)); |
192 | 0 | src6 = xx_loadl_32(src_ptr + 6 * src_stride); |
193 | 0 | s[5] = _mm_unpacklo_epi8(xx_loadl_32(src_ptr + 5 * src_stride), src6); |
194 | |
|
195 | 0 | do { |
196 | 0 | s[6] = _mm_unpacklo_epi8(src6, xx_loadl_32(src_ptr + 7 * src_stride)); |
197 | 0 | src6 = xx_loadl_32(src_ptr + 8 * src_stride); |
198 | 0 | s[7] = _mm_unpacklo_epi8(xx_loadl_32(src_ptr + 7 * src_stride), src6); |
199 | |
|
200 | 0 | res = convolve_lo_y(s + 0, coeffs); |
201 | 0 | res_shift = _mm_sll_epi32(res, left_shift); |
202 | 0 | res_shift = |
203 | 0 | _mm_sra_epi32(_mm_add_epi32(res_shift, round_const), round_shift); |
204 | |
|
205 | 0 | __m128i res_16b = _mm_packs_epi32(res_shift, res_shift); |
206 | 0 | __m128i res_unsigned = _mm_add_epi16(res_16b, offset_const); |
207 | | |
208 | | // Accumulate values into the destination buffer |
209 | 0 | if (do_average) { |
210 | 0 | const __m128i data_ref_0 = _mm_loadu_si128((__m128i *)dst); |
211 | |
|
212 | 0 | const __m128i comp_avg_res = |
213 | 0 | comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg); |
214 | |
|
215 | 0 | const __m128i round_result = convolve_rounding( |
216 | 0 | &comp_avg_res, &offset_const, &rounding_const, rounding_shift); |
217 | |
|
218 | 0 | const __m128i res_8 = _mm_packus_epi16(round_result, round_result); |
219 | 0 | *(int *)(&dst0[0]) = _mm_cvtsi128_si32(res_8); |
220 | |
|
221 | 0 | } else { |
222 | 0 | _mm_store_si128((__m128i *)dst, res_unsigned); |
223 | 0 | } |
224 | |
|
225 | 0 | src_ptr += src_stride; |
226 | 0 | dst += dst_stride; |
227 | 0 | dst0 += dst_stride0; |
228 | |
|
229 | 0 | res = convolve_lo_y(s + 1, coeffs); |
230 | 0 | res_shift = _mm_sll_epi32(res, left_shift); |
231 | 0 | res_shift = |
232 | 0 | _mm_sra_epi32(_mm_add_epi32(res_shift, round_const), round_shift); |
233 | |
|
234 | 0 | res_16b = _mm_packs_epi32(res_shift, res_shift); |
235 | 0 | res_unsigned = _mm_add_epi16(res_16b, offset_const); |
236 | | |
237 | | // Accumulate values into the destination buffer |
238 | 0 | if (do_average) { |
239 | 0 | const __m128i data_ref_0 = _mm_loadu_si128((__m128i *)dst); |
240 | |
|
241 | 0 | const __m128i comp_avg_res = |
242 | 0 | comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg); |
243 | |
|
244 | 0 | const __m128i round_result = convolve_rounding( |
245 | 0 | &comp_avg_res, &offset_const, &rounding_const, rounding_shift); |
246 | |
|
247 | 0 | const __m128i res_8 = _mm_packus_epi16(round_result, round_result); |
248 | 0 | *(int *)(&dst0[0]) = _mm_cvtsi128_si32(res_8); |
249 | |
|
250 | 0 | } else { |
251 | 0 | _mm_store_si128((__m128i *)dst, res_unsigned); |
252 | 0 | } |
253 | |
|
254 | 0 | src_ptr += src_stride; |
255 | 0 | dst += dst_stride; |
256 | 0 | dst0 += dst_stride0; |
257 | |
|
258 | 0 | s[0] = s[2]; |
259 | 0 | s[1] = s[3]; |
260 | 0 | s[2] = s[4]; |
261 | 0 | s[3] = s[5]; |
262 | 0 | s[4] = s[6]; |
263 | 0 | s[5] = s[7]; |
264 | 0 | h -= 2; |
265 | 0 | } while (h); |
266 | 0 | } else { |
267 | 0 | assert(!(w % 8)); |
268 | 0 | int j = 0; |
269 | 0 | do { |
270 | 0 | __m128i s[8], src6, res_lo, res_hi, res_lo_shift, res_hi_shift; |
271 | 0 | const uint8_t *data = &src_ptr[j]; |
272 | |
|
273 | 0 | src6 = _mm_loadl_epi64((__m128i *)(data + 6 * src_stride)); |
274 | 0 | s[0] = _mm_unpacklo_epi8( |
275 | 0 | _mm_loadl_epi64((__m128i *)(data + 0 * src_stride)), |
276 | 0 | _mm_loadl_epi64((__m128i *)(data + 1 * src_stride))); |
277 | 0 | s[1] = _mm_unpacklo_epi8( |
278 | 0 | _mm_loadl_epi64((__m128i *)(data + 1 * src_stride)), |
279 | 0 | _mm_loadl_epi64((__m128i *)(data + 2 * src_stride))); |
280 | 0 | s[2] = _mm_unpacklo_epi8( |
281 | 0 | _mm_loadl_epi64((__m128i *)(data + 2 * src_stride)), |
282 | 0 | _mm_loadl_epi64((__m128i *)(data + 3 * src_stride))); |
283 | 0 | s[3] = _mm_unpacklo_epi8( |
284 | 0 | _mm_loadl_epi64((__m128i *)(data + 3 * src_stride)), |
285 | 0 | _mm_loadl_epi64((__m128i *)(data + 4 * src_stride))); |
286 | 0 | s[4] = _mm_unpacklo_epi8( |
287 | 0 | _mm_loadl_epi64((__m128i *)(data + 4 * src_stride)), |
288 | 0 | _mm_loadl_epi64((__m128i *)(data + 5 * src_stride))); |
289 | 0 | s[5] = _mm_unpacklo_epi8( |
290 | 0 | _mm_loadl_epi64((__m128i *)(data + 5 * src_stride)), src6); |
291 | |
|
292 | 0 | int i = 0; |
293 | 0 | do { |
294 | 0 | data = &src_ptr[i * src_stride + j]; |
295 | 0 | s[6] = _mm_unpacklo_epi8( |
296 | 0 | src6, _mm_loadl_epi64((__m128i *)(data + 7 * src_stride))); |
297 | 0 | src6 = _mm_loadl_epi64((__m128i *)(data + 8 * src_stride)); |
298 | 0 | s[7] = _mm_unpacklo_epi8( |
299 | 0 | _mm_loadl_epi64((__m128i *)(data + 7 * src_stride)), src6); |
300 | |
|
301 | 0 | res_lo = convolve_lo_y(s, coeffs); // Filter low index pixels |
302 | 0 | res_hi = convolve_hi_y(s, coeffs); // Filter high index pixels |
303 | 0 | res_lo_shift = _mm_sll_epi32(res_lo, left_shift); |
304 | 0 | res_hi_shift = _mm_sll_epi32(res_hi, left_shift); |
305 | 0 | res_lo_shift = _mm_sra_epi32(_mm_add_epi32(res_lo_shift, round_const), |
306 | 0 | round_shift); |
307 | 0 | res_hi_shift = _mm_sra_epi32(_mm_add_epi32(res_hi_shift, round_const), |
308 | 0 | round_shift); |
309 | |
|
310 | 0 | __m128i res_16b = _mm_packs_epi32(res_lo_shift, res_hi_shift); |
311 | 0 | __m128i res_unsigned = _mm_add_epi16(res_16b, offset_const); |
312 | | |
313 | | // Accumulate values into the destination buffer |
314 | 0 | if (do_average) { |
315 | 0 | const __m128i data_ref_0 = |
316 | 0 | _mm_loadu_si128((__m128i *)(&dst[i * dst_stride + j])); |
317 | |
|
318 | 0 | const __m128i comp_avg_res = |
319 | 0 | comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg); |
320 | |
|
321 | 0 | const __m128i round_result = convolve_rounding( |
322 | 0 | &comp_avg_res, &offset_const, &rounding_const, rounding_shift); |
323 | |
|
324 | 0 | const __m128i res_8 = _mm_packus_epi16(round_result, round_result); |
325 | 0 | _mm_storel_epi64((__m128i *)(&dst0[i * dst_stride0 + j]), res_8); |
326 | 0 | } else { |
327 | 0 | _mm_store_si128((__m128i *)(&dst[i * dst_stride + j]), res_unsigned); |
328 | 0 | } |
329 | 0 | i++; |
330 | |
|
331 | 0 | res_lo = convolve_lo_y(s + 1, coeffs); // Filter low index pixels |
332 | 0 | res_hi = convolve_hi_y(s + 1, coeffs); // Filter high index pixels |
333 | 0 | res_lo_shift = _mm_sll_epi32(res_lo, left_shift); |
334 | 0 | res_hi_shift = _mm_sll_epi32(res_hi, left_shift); |
335 | 0 | res_lo_shift = _mm_sra_epi32(_mm_add_epi32(res_lo_shift, round_const), |
336 | 0 | round_shift); |
337 | 0 | res_hi_shift = _mm_sra_epi32(_mm_add_epi32(res_hi_shift, round_const), |
338 | 0 | round_shift); |
339 | 0 | res_16b = _mm_packs_epi32(res_lo_shift, res_hi_shift); |
340 | 0 | res_unsigned = _mm_add_epi16(res_16b, offset_const); |
341 | | |
342 | | // Accumulate values into the destination buffer |
343 | 0 | if (do_average) { |
344 | 0 | __m128i data_ref_0 = |
345 | 0 | _mm_loadu_si128((__m128i *)(&dst[i * dst_stride + j])); |
346 | |
|
347 | 0 | const __m128i comp_avg_res = |
348 | 0 | comp_avg(&data_ref_0, &res_unsigned, &wt, use_dist_wtd_comp_avg); |
349 | |
|
350 | 0 | const __m128i round_result = convolve_rounding( |
351 | 0 | &comp_avg_res, &offset_const, &rounding_const, rounding_shift); |
352 | |
|
353 | 0 | const __m128i res_8 = _mm_packus_epi16(round_result, round_result); |
354 | 0 | _mm_storel_epi64((__m128i *)(&dst0[i * dst_stride0 + j]), res_8); |
355 | 0 | } else { |
356 | 0 | _mm_store_si128((__m128i *)(&dst[i * dst_stride + j]), res_unsigned); |
357 | 0 | } |
358 | 0 | i++; |
359 | |
|
360 | 0 | s[0] = s[2]; |
361 | 0 | s[1] = s[3]; |
362 | 0 | s[2] = s[4]; |
363 | 0 | s[3] = s[5]; |
364 | 0 | s[4] = s[6]; |
365 | 0 | s[5] = s[7]; |
366 | 0 | } while (i < h); |
367 | 0 | j += 8; |
368 | 0 | } while (j < w); |
369 | 0 | } |
370 | 0 | } |