/src/aom/av1/common/x86/convolve_sse2.c
Line | Count | Source (jump to first uncovered line) |
1 | | /* |
2 | | * Copyright (c) 2017, Alliance for Open Media. All rights reserved. |
3 | | * |
4 | | * This source code is subject to the terms of the BSD 2 Clause License and |
5 | | * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
6 | | * was not distributed with this source code in the LICENSE file, you can |
7 | | * obtain it at www.aomedia.org/license/software. If the Alliance for Open |
8 | | * Media Patent License 1.0 was not distributed with this source code in the |
9 | | * PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
10 | | */ |
11 | | |
12 | | #include <emmintrin.h> |
13 | | |
14 | | #include "config/av1_rtcd.h" |
15 | | |
16 | | #include "aom_dsp/aom_dsp_common.h" |
17 | | #include "aom_dsp/aom_filter.h" |
18 | | #include "aom_dsp/x86/convolve_common_intrin.h" |
19 | | #include "aom_dsp/x86/synonyms.h" |
20 | | #include "av1/common/convolve.h" |
21 | | |
22 | | static inline void prepare_coeffs(const InterpFilterParams *const filter_params, |
23 | | const int subpel_q4, |
24 | 0 | __m128i *const coeffs /* [4] */) { |
25 | 0 | const int16_t *const y_filter = av1_get_interp_filter_subpel_kernel( |
26 | 0 | filter_params, subpel_q4 & SUBPEL_MASK); |
27 | 0 | const __m128i coeffs_y = _mm_loadu_si128((__m128i *)y_filter); |
28 | | // coeffs 0 1 0 1 2 3 2 3 |
29 | 0 | const __m128i tmp_0 = _mm_unpacklo_epi32(coeffs_y, coeffs_y); |
30 | | // coeffs 4 5 4 5 6 7 6 7 |
31 | 0 | const __m128i tmp_1 = _mm_unpackhi_epi32(coeffs_y, coeffs_y); |
32 | |
|
33 | 0 | coeffs[0] = _mm_unpacklo_epi64(tmp_0, tmp_0); // coeffs 0 1 0 1 0 1 0 1 |
34 | 0 | coeffs[1] = _mm_unpackhi_epi64(tmp_0, tmp_0); // coeffs 2 3 2 3 2 3 2 3 |
35 | 0 | coeffs[2] = _mm_unpacklo_epi64(tmp_1, tmp_1); // coeffs 4 5 4 5 4 5 4 5 |
36 | 0 | coeffs[3] = _mm_unpackhi_epi64(tmp_1, tmp_1); // coeffs 6 7 6 7 6 7 6 7 |
37 | 0 | } |
38 | | |
39 | | static inline __m128i convolve(const __m128i *const s, |
40 | 0 | const __m128i *const coeffs) { |
41 | 0 | const __m128i d0 = _mm_madd_epi16(s[0], coeffs[0]); |
42 | 0 | const __m128i d1 = _mm_madd_epi16(s[1], coeffs[1]); |
43 | 0 | const __m128i d2 = _mm_madd_epi16(s[2], coeffs[2]); |
44 | 0 | const __m128i d3 = _mm_madd_epi16(s[3], coeffs[3]); |
45 | 0 | const __m128i d = _mm_add_epi32(_mm_add_epi32(d0, d1), _mm_add_epi32(d2, d3)); |
46 | 0 | return d; |
47 | 0 | } |
48 | | |
49 | | static inline __m128i convolve_lo_x(const __m128i *const s, |
50 | 0 | const __m128i *const coeffs) { |
51 | 0 | __m128i ss[4]; |
52 | 0 | ss[0] = _mm_unpacklo_epi8(s[0], _mm_setzero_si128()); |
53 | 0 | ss[1] = _mm_unpacklo_epi8(s[1], _mm_setzero_si128()); |
54 | 0 | ss[2] = _mm_unpacklo_epi8(s[2], _mm_setzero_si128()); |
55 | 0 | ss[3] = _mm_unpacklo_epi8(s[3], _mm_setzero_si128()); |
56 | 0 | return convolve(ss, coeffs); |
57 | 0 | } |
58 | | |
59 | | static inline __m128i convolve_lo_y(const __m128i *const s, |
60 | 0 | const __m128i *const coeffs) { |
61 | 0 | __m128i ss[4]; |
62 | 0 | ss[0] = _mm_unpacklo_epi8(s[0], _mm_setzero_si128()); |
63 | 0 | ss[1] = _mm_unpacklo_epi8(s[2], _mm_setzero_si128()); |
64 | 0 | ss[2] = _mm_unpacklo_epi8(s[4], _mm_setzero_si128()); |
65 | 0 | ss[3] = _mm_unpacklo_epi8(s[6], _mm_setzero_si128()); |
66 | 0 | return convolve(ss, coeffs); |
67 | 0 | } |
68 | | |
69 | | static inline __m128i convolve_hi_y(const __m128i *const s, |
70 | 0 | const __m128i *const coeffs) { |
71 | 0 | __m128i ss[4]; |
72 | 0 | ss[0] = _mm_unpackhi_epi8(s[0], _mm_setzero_si128()); |
73 | 0 | ss[1] = _mm_unpackhi_epi8(s[2], _mm_setzero_si128()); |
74 | 0 | ss[2] = _mm_unpackhi_epi8(s[4], _mm_setzero_si128()); |
75 | 0 | ss[3] = _mm_unpackhi_epi8(s[6], _mm_setzero_si128()); |
76 | 0 | return convolve(ss, coeffs); |
77 | 0 | } |
78 | | |
79 | | static void convolve_y_sr_12tap_sse2(const uint8_t *src, int src_stride, |
80 | | uint8_t *dst, int dst_stride, int w, int h, |
81 | | const InterpFilterParams *filter_params_y, |
82 | 0 | int subpel_y_qn) { |
83 | 0 | const int fo_vert = filter_params_y->taps / 2 - 1; |
84 | 0 | const uint8_t *src_ptr = src - fo_vert * src_stride; |
85 | 0 | const __m128i round_const = _mm_set1_epi32((1 << FILTER_BITS) >> 1); |
86 | 0 | const __m128i round_shift = _mm_cvtsi32_si128(FILTER_BITS); |
87 | 0 | __m128i coeffs[6]; |
88 | |
|
89 | 0 | prepare_coeffs_12tap(filter_params_y, subpel_y_qn, coeffs); |
90 | |
|
91 | 0 | int j = 0; |
92 | 0 | do { |
93 | 0 | __m128i s[12], src10, res_lo, res_hi; |
94 | 0 | __m128i res_lo_round, res_hi_round, res16, res; |
95 | 0 | const uint8_t *data = &src_ptr[j]; |
96 | |
|
97 | 0 | src10 = _mm_loadl_epi64((__m128i *)(data + 10 * src_stride)); |
98 | 0 | s[0] = |
99 | 0 | _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(data + 0 * src_stride)), |
100 | 0 | _mm_loadl_epi64((__m128i *)(data + 1 * src_stride))); |
101 | 0 | s[1] = |
102 | 0 | _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(data + 1 * src_stride)), |
103 | 0 | _mm_loadl_epi64((__m128i *)(data + 2 * src_stride))); |
104 | 0 | s[2] = |
105 | 0 | _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(data + 2 * src_stride)), |
106 | 0 | _mm_loadl_epi64((__m128i *)(data + 3 * src_stride))); |
107 | 0 | s[3] = |
108 | 0 | _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(data + 3 * src_stride)), |
109 | 0 | _mm_loadl_epi64((__m128i *)(data + 4 * src_stride))); |
110 | 0 | s[4] = |
111 | 0 | _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(data + 4 * src_stride)), |
112 | 0 | _mm_loadl_epi64((__m128i *)(data + 5 * src_stride))); |
113 | 0 | s[5] = |
114 | 0 | _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(data + 5 * src_stride)), |
115 | 0 | _mm_loadl_epi64((__m128i *)(data + 6 * src_stride))); |
116 | 0 | s[6] = |
117 | 0 | _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(data + 6 * src_stride)), |
118 | 0 | _mm_loadl_epi64((__m128i *)(data + 7 * src_stride))); |
119 | 0 | s[7] = |
120 | 0 | _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(data + 7 * src_stride)), |
121 | 0 | _mm_loadl_epi64((__m128i *)(data + 8 * src_stride))); |
122 | 0 | s[8] = |
123 | 0 | _mm_unpacklo_epi8(_mm_loadl_epi64((__m128i *)(data + 8 * src_stride)), |
124 | 0 | _mm_loadl_epi64((__m128i *)(data + 9 * src_stride))); |
125 | 0 | s[9] = _mm_unpacklo_epi8( |
126 | 0 | _mm_loadl_epi64((__m128i *)(data + 9 * src_stride)), src10); |
127 | |
|
128 | 0 | int i = 0; |
129 | 0 | do { |
130 | 0 | data = &src_ptr[i * src_stride + j]; |
131 | 0 | s[10] = _mm_unpacklo_epi8( |
132 | 0 | src10, _mm_loadl_epi64((__m128i *)(data + 11 * src_stride))); |
133 | 0 | src10 = _mm_loadl_epi64((__m128i *)(data + 12 * src_stride)); |
134 | 0 | s[11] = _mm_unpacklo_epi8( |
135 | 0 | _mm_loadl_epi64((__m128i *)(data + 11 * src_stride)), src10); |
136 | |
|
137 | 0 | res_lo = convolve_lo_y_12tap(s, coeffs); // Filter low index pixels |
138 | 0 | res_hi = convolve_hi_y_12tap(s, coeffs); // Filter high index pixels |
139 | |
|
140 | 0 | res_lo_round = |
141 | 0 | _mm_sra_epi32(_mm_add_epi32(res_lo, round_const), round_shift); |
142 | 0 | res_hi_round = |
143 | 0 | _mm_sra_epi32(_mm_add_epi32(res_hi, round_const), round_shift); |
144 | |
|
145 | 0 | res16 = _mm_packs_epi32(res_lo_round, res_hi_round); |
146 | 0 | res = _mm_packus_epi16(res16, res16); |
147 | |
|
148 | 0 | _mm_storel_epi64((__m128i *)(dst + i * dst_stride + j), res); |
149 | 0 | i++; |
150 | |
|
151 | 0 | res_lo = convolve_lo_y_12tap(s + 1, coeffs); // Filter low index pixels |
152 | 0 | res_hi = convolve_hi_y_12tap(s + 1, coeffs); // Filter high index pixels |
153 | |
|
154 | 0 | res_lo_round = |
155 | 0 | _mm_sra_epi32(_mm_add_epi32(res_lo, round_const), round_shift); |
156 | 0 | res_hi_round = |
157 | 0 | _mm_sra_epi32(_mm_add_epi32(res_hi, round_const), round_shift); |
158 | |
|
159 | 0 | res16 = _mm_packs_epi32(res_lo_round, res_hi_round); |
160 | 0 | res = _mm_packus_epi16(res16, res16); |
161 | |
|
162 | 0 | _mm_storel_epi64((__m128i *)(dst + i * dst_stride + j), res); |
163 | 0 | i++; |
164 | |
|
165 | 0 | s[0] = s[2]; |
166 | 0 | s[1] = s[3]; |
167 | 0 | s[2] = s[4]; |
168 | 0 | s[3] = s[5]; |
169 | 0 | s[4] = s[6]; |
170 | 0 | s[5] = s[7]; |
171 | 0 | s[6] = s[8]; |
172 | 0 | s[7] = s[9]; |
173 | 0 | s[8] = s[10]; |
174 | 0 | s[9] = s[11]; |
175 | 0 | } while (i < h); |
176 | 0 | j += 8; |
177 | 0 | } while (j < w); |
178 | 0 | } |
179 | | |
180 | | void av1_convolve_y_sr_sse2(const uint8_t *src, int src_stride, uint8_t *dst, |
181 | | int dst_stride, int w, int h, |
182 | | const InterpFilterParams *filter_params_y, |
183 | 0 | const int subpel_y_qn) { |
184 | 0 | if (filter_params_y->taps > 8) { |
185 | 0 | if (w < 8) { |
186 | 0 | av1_convolve_y_sr_c(src, src_stride, dst, dst_stride, w, h, |
187 | 0 | filter_params_y, subpel_y_qn); |
188 | 0 | } else { |
189 | 0 | convolve_y_sr_12tap_sse2(src, src_stride, dst, dst_stride, w, h, |
190 | 0 | filter_params_y, subpel_y_qn); |
191 | 0 | } |
192 | 0 | } else { |
193 | 0 | const int fo_vert = filter_params_y->taps / 2 - 1; |
194 | 0 | const uint8_t *src_ptr = src - fo_vert * src_stride; |
195 | 0 | const __m128i round_const = _mm_set1_epi32((1 << FILTER_BITS) >> 1); |
196 | 0 | const __m128i round_shift = _mm_cvtsi32_si128(FILTER_BITS); |
197 | 0 | __m128i coeffs[4]; |
198 | |
|
199 | 0 | prepare_coeffs(filter_params_y, subpel_y_qn, coeffs); |
200 | |
|
201 | 0 | if (w <= 4) { |
202 | 0 | __m128i s[8], src6, res, res_round, res16; |
203 | 0 | int res_int; |
204 | 0 | s[0] = _mm_unpacklo_epi8(xx_loadl_32(src_ptr + 0 * src_stride), |
205 | 0 | xx_loadl_32(src_ptr + 1 * src_stride)); |
206 | 0 | s[1] = _mm_unpacklo_epi8(xx_loadl_32(src_ptr + 1 * src_stride), |
207 | 0 | xx_loadl_32(src_ptr + 2 * src_stride)); |
208 | 0 | s[2] = _mm_unpacklo_epi8(xx_loadl_32(src_ptr + 2 * src_stride), |
209 | 0 | xx_loadl_32(src_ptr + 3 * src_stride)); |
210 | 0 | s[3] = _mm_unpacklo_epi8(xx_loadl_32(src_ptr + 3 * src_stride), |
211 | 0 | xx_loadl_32(src_ptr + 4 * src_stride)); |
212 | 0 | s[4] = _mm_unpacklo_epi8(xx_loadl_32(src_ptr + 4 * src_stride), |
213 | 0 | xx_loadl_32(src_ptr + 5 * src_stride)); |
214 | 0 | src6 = xx_loadl_32(src_ptr + 6 * src_stride); |
215 | 0 | s[5] = _mm_unpacklo_epi8(xx_loadl_32(src_ptr + 5 * src_stride), src6); |
216 | |
|
217 | 0 | do { |
218 | 0 | s[6] = _mm_unpacklo_epi8(src6, xx_loadl_32(src_ptr + 7 * src_stride)); |
219 | 0 | src6 = xx_loadl_32(src_ptr + 8 * src_stride); |
220 | 0 | s[7] = _mm_unpacklo_epi8(xx_loadl_32(src_ptr + 7 * src_stride), src6); |
221 | |
|
222 | 0 | res = convolve_lo_y(s + 0, coeffs); |
223 | 0 | res_round = _mm_sra_epi32(_mm_add_epi32(res, round_const), round_shift); |
224 | 0 | res16 = _mm_packs_epi32(res_round, res_round); |
225 | 0 | res_int = _mm_cvtsi128_si32(_mm_packus_epi16(res16, res16)); |
226 | |
|
227 | 0 | if (w == 2) |
228 | 0 | *(uint16_t *)dst = (uint16_t)res_int; |
229 | 0 | else |
230 | 0 | *(int *)dst = res_int; |
231 | |
|
232 | 0 | src_ptr += src_stride; |
233 | 0 | dst += dst_stride; |
234 | |
|
235 | 0 | res = convolve_lo_y(s + 1, coeffs); |
236 | 0 | res_round = _mm_sra_epi32(_mm_add_epi32(res, round_const), round_shift); |
237 | 0 | res16 = _mm_packs_epi32(res_round, res_round); |
238 | 0 | res_int = _mm_cvtsi128_si32(_mm_packus_epi16(res16, res16)); |
239 | |
|
240 | 0 | if (w == 2) |
241 | 0 | *(uint16_t *)dst = (uint16_t)res_int; |
242 | 0 | else |
243 | 0 | *(int *)dst = res_int; |
244 | |
|
245 | 0 | src_ptr += src_stride; |
246 | 0 | dst += dst_stride; |
247 | |
|
248 | 0 | s[0] = s[2]; |
249 | 0 | s[1] = s[3]; |
250 | 0 | s[2] = s[4]; |
251 | 0 | s[3] = s[5]; |
252 | 0 | s[4] = s[6]; |
253 | 0 | s[5] = s[7]; |
254 | 0 | h -= 2; |
255 | 0 | } while (h); |
256 | 0 | } else { |
257 | 0 | assert(!(w % 8)); |
258 | 0 | int j = 0; |
259 | 0 | do { |
260 | 0 | __m128i s[8], src6, res_lo, res_hi; |
261 | 0 | __m128i res_lo_round, res_hi_round, res16, res; |
262 | 0 | const uint8_t *data = &src_ptr[j]; |
263 | |
|
264 | 0 | src6 = _mm_loadl_epi64((__m128i *)(data + 6 * src_stride)); |
265 | 0 | s[0] = _mm_unpacklo_epi8( |
266 | 0 | _mm_loadl_epi64((__m128i *)(data + 0 * src_stride)), |
267 | 0 | _mm_loadl_epi64((__m128i *)(data + 1 * src_stride))); |
268 | 0 | s[1] = _mm_unpacklo_epi8( |
269 | 0 | _mm_loadl_epi64((__m128i *)(data + 1 * src_stride)), |
270 | 0 | _mm_loadl_epi64((__m128i *)(data + 2 * src_stride))); |
271 | 0 | s[2] = _mm_unpacklo_epi8( |
272 | 0 | _mm_loadl_epi64((__m128i *)(data + 2 * src_stride)), |
273 | 0 | _mm_loadl_epi64((__m128i *)(data + 3 * src_stride))); |
274 | 0 | s[3] = _mm_unpacklo_epi8( |
275 | 0 | _mm_loadl_epi64((__m128i *)(data + 3 * src_stride)), |
276 | 0 | _mm_loadl_epi64((__m128i *)(data + 4 * src_stride))); |
277 | 0 | s[4] = _mm_unpacklo_epi8( |
278 | 0 | _mm_loadl_epi64((__m128i *)(data + 4 * src_stride)), |
279 | 0 | _mm_loadl_epi64((__m128i *)(data + 5 * src_stride))); |
280 | 0 | s[5] = _mm_unpacklo_epi8( |
281 | 0 | _mm_loadl_epi64((__m128i *)(data + 5 * src_stride)), src6); |
282 | |
|
283 | 0 | int i = 0; |
284 | 0 | do { |
285 | 0 | data = &src_ptr[i * src_stride + j]; |
286 | 0 | s[6] = _mm_unpacklo_epi8( |
287 | 0 | src6, _mm_loadl_epi64((__m128i *)(data + 7 * src_stride))); |
288 | 0 | src6 = _mm_loadl_epi64((__m128i *)(data + 8 * src_stride)); |
289 | 0 | s[7] = _mm_unpacklo_epi8( |
290 | 0 | _mm_loadl_epi64((__m128i *)(data + 7 * src_stride)), src6); |
291 | |
|
292 | 0 | res_lo = convolve_lo_y(s, coeffs); // Filter low index pixels |
293 | 0 | res_hi = convolve_hi_y(s, coeffs); // Filter high index pixels |
294 | |
|
295 | 0 | res_lo_round = |
296 | 0 | _mm_sra_epi32(_mm_add_epi32(res_lo, round_const), round_shift); |
297 | 0 | res_hi_round = |
298 | 0 | _mm_sra_epi32(_mm_add_epi32(res_hi, round_const), round_shift); |
299 | |
|
300 | 0 | res16 = _mm_packs_epi32(res_lo_round, res_hi_round); |
301 | 0 | res = _mm_packus_epi16(res16, res16); |
302 | |
|
303 | 0 | _mm_storel_epi64((__m128i *)(dst + i * dst_stride + j), res); |
304 | 0 | i++; |
305 | |
|
306 | 0 | res_lo = convolve_lo_y(s + 1, coeffs); // Filter low index pixels |
307 | 0 | res_hi = convolve_hi_y(s + 1, coeffs); // Filter high index pixels |
308 | |
|
309 | 0 | res_lo_round = |
310 | 0 | _mm_sra_epi32(_mm_add_epi32(res_lo, round_const), round_shift); |
311 | 0 | res_hi_round = |
312 | 0 | _mm_sra_epi32(_mm_add_epi32(res_hi, round_const), round_shift); |
313 | |
|
314 | 0 | res16 = _mm_packs_epi32(res_lo_round, res_hi_round); |
315 | 0 | res = _mm_packus_epi16(res16, res16); |
316 | |
|
317 | 0 | _mm_storel_epi64((__m128i *)(dst + i * dst_stride + j), res); |
318 | 0 | i++; |
319 | |
|
320 | 0 | s[0] = s[2]; |
321 | 0 | s[1] = s[3]; |
322 | 0 | s[2] = s[4]; |
323 | 0 | s[3] = s[5]; |
324 | 0 | s[4] = s[6]; |
325 | 0 | s[5] = s[7]; |
326 | 0 | } while (i < h); |
327 | 0 | j += 8; |
328 | 0 | } while (j < w); |
329 | 0 | } |
330 | 0 | } |
331 | 0 | } |
332 | | |
333 | | static void convolve_x_sr_12tap_sse2(const uint8_t *src, int src_stride, |
334 | | uint8_t *dst, int dst_stride, int w, int h, |
335 | | const InterpFilterParams *filter_params_x, |
336 | | int subpel_x_qn, |
337 | 0 | ConvolveParams *conv_params) { |
338 | 0 | const int fo_horiz = filter_params_x->taps / 2 - 1; |
339 | 0 | const uint8_t *src_ptr = src - fo_horiz; |
340 | 0 | const int bits = FILTER_BITS - conv_params->round_0; |
341 | 0 | const __m128i round_0_const = |
342 | 0 | _mm_set1_epi32((1 << conv_params->round_0) >> 1); |
343 | 0 | const __m128i round_const = _mm_set1_epi32((1 << bits) >> 1); |
344 | 0 | const __m128i round_0_shift = _mm_cvtsi32_si128(conv_params->round_0); |
345 | 0 | const __m128i round_shift = _mm_cvtsi32_si128(bits); |
346 | 0 | const __m128i zero = _mm_setzero_si128(); |
347 | 0 | __m128i coeffs[6]; |
348 | |
|
349 | 0 | assert(bits >= 0); |
350 | 0 | assert((FILTER_BITS - conv_params->round_1) >= 0 || |
351 | 0 | ((conv_params->round_0 + conv_params->round_1) == 2 * FILTER_BITS)); |
352 | | |
353 | 0 | prepare_coeffs_12tap(filter_params_x, subpel_x_qn, coeffs); |
354 | |
|
355 | 0 | int i = 0; |
356 | 0 | do { |
357 | 0 | int j = 0; |
358 | 0 | do { |
359 | 0 | const __m128i data = |
360 | 0 | _mm_loadu_si128((__m128i *)&src_ptr[i * src_stride + j]); |
361 | 0 | __m128i s[4]; |
362 | |
|
363 | 0 | s[0] = _mm_unpacklo_epi16(data, _mm_srli_si128(data, 1)); |
364 | 0 | s[1] = |
365 | 0 | _mm_unpacklo_epi16(_mm_srli_si128(data, 2), _mm_srli_si128(data, 3)); |
366 | 0 | s[2] = |
367 | 0 | _mm_unpacklo_epi16(_mm_srli_si128(data, 4), _mm_srli_si128(data, 5)); |
368 | 0 | s[3] = |
369 | 0 | _mm_unpacklo_epi16(_mm_srli_si128(data, 6), _mm_srli_si128(data, 7)); |
370 | |
|
371 | 0 | const __m128i res32 = convolve_lo_x_12tap(s, coeffs, zero); |
372 | |
|
373 | 0 | __m128i res32_round = |
374 | 0 | _mm_sra_epi32(_mm_add_epi32(res32, round_0_const), round_0_shift); |
375 | 0 | res32_round = |
376 | 0 | _mm_sra_epi32(_mm_add_epi32(res32_round, round_const), round_shift); |
377 | |
|
378 | 0 | const __m128i res16 = _mm_packs_epi32(res32_round, zero); |
379 | 0 | const __m128i res = _mm_packus_epi16(res16, zero); |
380 | |
|
381 | 0 | const int val = _mm_cvtsi128_si32(res); |
382 | 0 | memcpy((dst + i * dst_stride + j), &val, sizeof(val)); |
383 | 0 | j += 4; |
384 | 0 | } while (j < w); |
385 | 0 | } while (++i < h); |
386 | 0 | } |
387 | | |
388 | | void av1_convolve_x_sr_sse2(const uint8_t *src, int src_stride, uint8_t *dst, |
389 | | int dst_stride, int w, int h, |
390 | | const InterpFilterParams *filter_params_x, |
391 | | const int subpel_x_qn, |
392 | 0 | ConvolveParams *conv_params) { |
393 | 0 | if (filter_params_x->taps > 8) { |
394 | 0 | if (w < 4) { |
395 | 0 | av1_convolve_x_sr_c(src, src_stride, dst, dst_stride, w, h, |
396 | 0 | filter_params_x, subpel_x_qn, conv_params); |
397 | 0 | } else { |
398 | 0 | convolve_x_sr_12tap_sse2(src, src_stride, dst, dst_stride, w, h, |
399 | 0 | filter_params_x, subpel_x_qn, conv_params); |
400 | 0 | } |
401 | 0 | } else { |
402 | 0 | const int fo_horiz = filter_params_x->taps / 2 - 1; |
403 | 0 | const uint8_t *src_ptr = src - fo_horiz; |
404 | 0 | const int bits = FILTER_BITS - conv_params->round_0; |
405 | 0 | const __m128i round_0_const = |
406 | 0 | _mm_set1_epi32((1 << conv_params->round_0) >> 1); |
407 | 0 | const __m128i round_const = _mm_set1_epi32((1 << bits) >> 1); |
408 | 0 | const __m128i round_0_shift = _mm_cvtsi32_si128(conv_params->round_0); |
409 | 0 | const __m128i round_shift = _mm_cvtsi32_si128(bits); |
410 | 0 | __m128i coeffs[4]; |
411 | |
|
412 | 0 | assert(bits >= 0); |
413 | 0 | assert((FILTER_BITS - conv_params->round_1) >= 0 || |
414 | 0 | ((conv_params->round_0 + conv_params->round_1) == 2 * FILTER_BITS)); |
415 | | |
416 | 0 | prepare_coeffs(filter_params_x, subpel_x_qn, coeffs); |
417 | |
|
418 | 0 | if (w <= 4) { |
419 | 0 | do { |
420 | 0 | const __m128i data = _mm_loadu_si128((__m128i *)src_ptr); |
421 | 0 | __m128i s[4]; |
422 | |
|
423 | 0 | s[0] = _mm_unpacklo_epi8(data, _mm_srli_si128(data, 1)); |
424 | 0 | s[1] = |
425 | 0 | _mm_unpacklo_epi8(_mm_srli_si128(data, 2), _mm_srli_si128(data, 3)); |
426 | 0 | s[2] = |
427 | 0 | _mm_unpacklo_epi8(_mm_srli_si128(data, 4), _mm_srli_si128(data, 5)); |
428 | 0 | s[3] = |
429 | 0 | _mm_unpacklo_epi8(_mm_srli_si128(data, 6), _mm_srli_si128(data, 7)); |
430 | 0 | const __m128i res_lo = convolve_lo_x(s, coeffs); |
431 | 0 | __m128i res_lo_round = |
432 | 0 | _mm_sra_epi32(_mm_add_epi32(res_lo, round_0_const), round_0_shift); |
433 | 0 | res_lo_round = _mm_sra_epi32(_mm_add_epi32(res_lo_round, round_const), |
434 | 0 | round_shift); |
435 | |
|
436 | 0 | const __m128i res16 = _mm_packs_epi32(res_lo_round, res_lo_round); |
437 | 0 | const __m128i res = _mm_packus_epi16(res16, res16); |
438 | |
|
439 | 0 | int r = _mm_cvtsi128_si32(res); |
440 | 0 | if (w == 2) |
441 | 0 | *(uint16_t *)dst = (uint16_t)r; |
442 | 0 | else |
443 | 0 | *(int *)dst = r; |
444 | |
|
445 | 0 | src_ptr += src_stride; |
446 | 0 | dst += dst_stride; |
447 | 0 | } while (--h); |
448 | 0 | } else { |
449 | 0 | assert(!(w % 8)); |
450 | 0 | int i = 0; |
451 | 0 | do { |
452 | 0 | int j = 0; |
453 | 0 | do { |
454 | 0 | const __m128i data = |
455 | 0 | _mm_loadu_si128((__m128i *)&src_ptr[i * src_stride + j]); |
456 | 0 | __m128i s[4]; |
457 | | |
458 | | // Filter even-index pixels |
459 | 0 | s[0] = data; |
460 | 0 | s[1] = _mm_srli_si128(data, 2); |
461 | 0 | s[2] = _mm_srli_si128(data, 4); |
462 | 0 | s[3] = _mm_srli_si128(data, 6); |
463 | 0 | const __m128i res_even = convolve_lo_x(s, coeffs); |
464 | | |
465 | | // Filter odd-index pixels |
466 | 0 | s[0] = _mm_srli_si128(data, 1); |
467 | 0 | s[1] = _mm_srli_si128(data, 3); |
468 | 0 | s[2] = _mm_srli_si128(data, 5); |
469 | 0 | s[3] = _mm_srli_si128(data, 7); |
470 | 0 | const __m128i res_odd = convolve_lo_x(s, coeffs); |
471 | | |
472 | | // Rearrange pixels back into the order 0 ... 7 |
473 | 0 | const __m128i res_lo = _mm_unpacklo_epi32(res_even, res_odd); |
474 | 0 | const __m128i res_hi = _mm_unpackhi_epi32(res_even, res_odd); |
475 | 0 | __m128i res_lo_round = _mm_sra_epi32( |
476 | 0 | _mm_add_epi32(res_lo, round_0_const), round_0_shift); |
477 | 0 | res_lo_round = _mm_sra_epi32(_mm_add_epi32(res_lo_round, round_const), |
478 | 0 | round_shift); |
479 | 0 | __m128i res_hi_round = _mm_sra_epi32( |
480 | 0 | _mm_add_epi32(res_hi, round_0_const), round_0_shift); |
481 | 0 | res_hi_round = _mm_sra_epi32(_mm_add_epi32(res_hi_round, round_const), |
482 | 0 | round_shift); |
483 | |
|
484 | 0 | const __m128i res16 = _mm_packs_epi32(res_lo_round, res_hi_round); |
485 | 0 | const __m128i res = _mm_packus_epi16(res16, res16); |
486 | |
|
487 | 0 | _mm_storel_epi64((__m128i *)(dst + i * dst_stride + j), res); |
488 | 0 | j += 8; |
489 | 0 | } while (j < w); |
490 | 0 | } while (++i < h); |
491 | 0 | } |
492 | 0 | } |
493 | 0 | } |