/src/avm/avm_dsp/x86/convolve_avx2.h
Line | Count | Source |
1 | | /* |
2 | | * Copyright (c) 2021, Alliance for Open Media. All rights reserved |
3 | | * |
4 | | * This source code is subject to the terms of the BSD 3-Clause Clear License |
5 | | * and the Alliance for Open Media Patent License 1.0. If the BSD 3-Clause Clear |
6 | | * License was not distributed with this source code in the LICENSE file, you |
7 | | * can obtain it at aomedia.org/license/software-license/bsd-3-c-c/. If the |
8 | | * Alliance for Open Media Patent License 1.0 was not distributed with this |
9 | | * source code in the PATENTS file, you can obtain it at |
10 | | * aomedia.org/license/patent-license/. |
11 | | */ |
12 | | |
13 | | #ifndef AVM_AVM_DSP_X86_CONVOLVE_AVX2_H_ |
14 | | #define AVM_AVM_DSP_X86_CONVOLVE_AVX2_H_ |
15 | | |
16 | | // filters for 16 |
17 | | DECLARE_ALIGNED(32, static const uint8_t, filt_global_avx2[]) = { |
18 | | 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 0, 1, 1, |
19 | | 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 2, 3, 3, 4, 4, 5, |
20 | | 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 2, 3, 3, 4, 4, 5, 5, 6, 6, |
21 | | 7, 7, 8, 8, 9, 9, 10, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, |
22 | | 10, 11, 11, 12, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, |
23 | | 12, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 6, 7, |
24 | | 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14 |
25 | | }; |
26 | | |
27 | | DECLARE_ALIGNED(32, static const uint8_t, filt_d4_global_avx2[]) = { |
28 | | 0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, 0, 1, 2, 3, 1, 2, |
29 | | 3, 4, 2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, |
30 | | 7, 8, 9, 10, 4, 5, 6, 7, 5, 6, 7, 8, 6, 7, 8, 9, 7, 8, 9, 10, |
31 | | }; |
32 | | |
33 | | DECLARE_ALIGNED(32, static const uint8_t, filt4_d4_global_avx2[]) = { |
34 | | 2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8, |
35 | | 2, 3, 4, 5, 3, 4, 5, 6, 4, 5, 6, 7, 5, 6, 7, 8, |
36 | | }; |
37 | | |
38 | | DECLARE_ALIGNED(32, static const uint8_t, filt_center_global_avx2[32]) = { |
39 | | 3, 255, 4, 255, 5, 255, 6, 255, 7, 255, 8, 255, 9, 255, 10, 255, |
40 | | 3, 255, 4, 255, 5, 255, 6, 255, 7, 255, 8, 255, 9, 255, 10, 255 |
41 | | }; |
42 | | |
43 | | DECLARE_ALIGNED(32, static const uint8_t, |
44 | | filt1_global_avx2[32]) = { 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, |
45 | | 6, 6, 7, 7, 8, 0, 1, 1, 2, 2, 3, |
46 | | 3, 4, 4, 5, 5, 6, 6, 7, 7, 8 }; |
47 | | |
48 | | DECLARE_ALIGNED(32, static const uint8_t, |
49 | | filt2_global_avx2[32]) = { 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, |
50 | | 8, 8, 9, 9, 10, 2, 3, 3, 4, 4, 5, |
51 | | 5, 6, 6, 7, 7, 8, 8, 9, 9, 10 }; |
52 | | |
53 | | DECLARE_ALIGNED(32, static const uint8_t, filt3_global_avx2[32]) = { |
54 | | 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, |
55 | | 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12 |
56 | | }; |
57 | | |
58 | | DECLARE_ALIGNED(32, static const uint8_t, filt4_global_avx2[32]) = { |
59 | | 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, |
60 | | 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14 |
61 | | }; |
62 | | |
63 | | #define CONVOLVE_HORIZ_FILTER_8TAP \ |
64 | 0 | for (i = 0; i < im_h; i += 2) { \ |
65 | 0 | const __m256i row0 = \ |
66 | 0 | _mm256_loadu_si256((__m256i *)&src_ptr[i * src_stride + j]); \ |
67 | 0 | __m256i row1 = _mm256_set1_epi16(0); \ |
68 | 0 | if (i + 1 < im_h) \ |
69 | 0 | row1 = \ |
70 | 0 | _mm256_loadu_si256((__m256i *)&src_ptr[(i + 1) * src_stride + j]); \ |
71 | 0 | \ |
72 | 0 | const __m256i r0 = _mm256_permute2x128_si256(row0, row1, 0x20); \ |
73 | 0 | const __m256i r1 = _mm256_permute2x128_si256(row0, row1, 0x31); \ |
74 | 0 | \ |
75 | 0 | s[0] = r0; \ |
76 | 0 | s[1] = _mm256_alignr_epi8(r1, r0, 4); \ |
77 | 0 | s[2] = _mm256_alignr_epi8(r1, r0, 8); \ |
78 | 0 | s[3] = _mm256_alignr_epi8(r1, r0, 12); \ |
79 | 0 | \ |
80 | 0 | __m256i res_even = convolve(s, coeffs_x); \ |
81 | 0 | res_even = _mm256_sra_epi32(_mm256_add_epi32(res_even, round_const_x), \ |
82 | 0 | round_shift_x); \ |
83 | 0 | \ |
84 | 0 | s[0] = _mm256_alignr_epi8(r1, r0, 2); \ |
85 | 0 | s[1] = _mm256_alignr_epi8(r1, r0, 6); \ |
86 | 0 | s[2] = _mm256_alignr_epi8(r1, r0, 10); \ |
87 | 0 | s[3] = _mm256_alignr_epi8(r1, r0, 14); \ |
88 | 0 | \ |
89 | 0 | __m256i res_odd = convolve(s, coeffs_x); \ |
90 | 0 | res_odd = _mm256_sra_epi32(_mm256_add_epi32(res_odd, round_const_x), \ |
91 | 0 | round_shift_x); \ |
92 | 0 | \ |
93 | 0 | const __m256i res_even1 = _mm256_packs_epi32(res_even, res_even); \ |
94 | 0 | const __m256i res_odd1 = _mm256_packs_epi32(res_odd, res_odd); \ |
95 | 0 | const __m256i res = _mm256_unpacklo_epi16(res_even1, res_odd1); \ |
96 | 0 | \ |
97 | 0 | _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); \ |
98 | 0 | } |
99 | | |
100 | | #define CONVOLVE_HORIZ_FILTER_6TAP \ |
101 | 0 | for (i = 0; i < im_h; i += 2) { \ |
102 | 0 | const __m256i row0 = \ |
103 | 0 | _mm256_loadu_si256((__m256i *)&src_ptr[i * src_stride + j]); \ |
104 | 0 | __m256i row1 = _mm256_set1_epi16(0); \ |
105 | 0 | if (i + 1 < im_h) \ |
106 | 0 | row1 = \ |
107 | 0 | _mm256_loadu_si256((__m256i *)&src_ptr[(i + 1) * src_stride + j]); \ |
108 | 0 | \ |
109 | 0 | const __m256i r0 = _mm256_permute2x128_si256(row0, row1, 0x20); \ |
110 | 0 | const __m256i r1 = _mm256_permute2x128_si256(row0, row1, 0x31); \ |
111 | 0 | \ |
112 | 0 | s[0] = r0; \ |
113 | 0 | s[1] = _mm256_alignr_epi8(r1, r0, 4); \ |
114 | 0 | s[2] = _mm256_alignr_epi8(r1, r0, 8); \ |
115 | 0 | \ |
116 | 0 | __m256i res_even = convolve_6tap(s, coeffs_x); \ |
117 | 0 | res_even = _mm256_sra_epi32(_mm256_add_epi32(res_even, round_const_x), \ |
118 | 0 | round_shift_x); \ |
119 | 0 | \ |
120 | 0 | s[0] = _mm256_alignr_epi8(r1, r0, 2); \ |
121 | 0 | s[1] = _mm256_alignr_epi8(r1, r0, 6); \ |
122 | 0 | s[2] = _mm256_alignr_epi8(r1, r0, 10); \ |
123 | 0 | \ |
124 | 0 | __m256i res_odd = convolve_6tap(s, coeffs_x); \ |
125 | 0 | res_odd = _mm256_sra_epi32(_mm256_add_epi32(res_odd, round_const_x), \ |
126 | 0 | round_shift_x); \ |
127 | 0 | \ |
128 | 0 | const __m256i res_even1 = _mm256_packs_epi32(res_even, res_even); \ |
129 | 0 | const __m256i res_odd1 = _mm256_packs_epi32(res_odd, res_odd); \ |
130 | 0 | const __m256i res = _mm256_unpacklo_epi16(res_even1, res_odd1); \ |
131 | 0 | \ |
132 | 0 | _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); \ |
133 | 0 | } |
134 | | |
135 | | #define CONVOLVE_HORIZ_FILTER_4TAP \ |
136 | 0 | for (i = 0; i < im_h; i += 2) { \ |
137 | 0 | const __m256i row0 = \ |
138 | 0 | _mm256_loadu_si256((__m256i *)&src_ptr[i * src_stride + j]); \ |
139 | 0 | __m256i row1 = _mm256_set1_epi16(0); \ |
140 | 0 | if (i + 1 < im_h) \ |
141 | 0 | row1 = \ |
142 | 0 | _mm256_loadu_si256((__m256i *)&src_ptr[(i + 1) * src_stride + j]); \ |
143 | 0 | \ |
144 | 0 | const __m256i r0 = _mm256_permute2x128_si256(row0, row1, 0x20); \ |
145 | 0 | const __m256i r1 = _mm256_permute2x128_si256(row0, row1, 0x31); \ |
146 | 0 | \ |
147 | 0 | s[0] = r0; \ |
148 | 0 | s[1] = _mm256_alignr_epi8(r1, r0, 4); \ |
149 | 0 | \ |
150 | 0 | __m256i res_even = convolve_4tap(s, coeffs_x); \ |
151 | 0 | res_even = _mm256_sra_epi32(_mm256_add_epi32(res_even, round_const_x), \ |
152 | 0 | round_shift_x); \ |
153 | 0 | \ |
154 | 0 | s[0] = _mm256_alignr_epi8(r1, r0, 2); \ |
155 | 0 | s[1] = _mm256_alignr_epi8(r1, r0, 6); \ |
156 | 0 | \ |
157 | 0 | __m256i res_odd = convolve_4tap(s, coeffs_x); \ |
158 | 0 | res_odd = _mm256_sra_epi32(_mm256_add_epi32(res_odd, round_const_x), \ |
159 | 0 | round_shift_x); \ |
160 | 0 | \ |
161 | 0 | const __m256i res_even1 = _mm256_packs_epi32(res_even, res_even); \ |
162 | 0 | const __m256i res_odd1 = _mm256_packs_epi32(res_odd, res_odd); \ |
163 | 0 | const __m256i res = _mm256_unpacklo_epi16(res_even1, res_odd1); \ |
164 | 0 | \ |
165 | 0 | _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); \ |
166 | 0 | } |
167 | | |
168 | | #define CONVOLVE_HORIZ_FILTER_2TAP \ |
169 | 0 | for (i = 0; i < im_h; i += 2) { \ |
170 | 0 | const __m256i row0 = \ |
171 | 0 | _mm256_loadu_si256((__m256i *)&src_ptr[i * src_stride + j]); \ |
172 | 0 | __m256i row1 = _mm256_set1_epi16(0); \ |
173 | 0 | if (i + 1 < im_h) \ |
174 | 0 | row1 = \ |
175 | 0 | _mm256_loadu_si256((__m256i *)&src_ptr[(i + 1) * src_stride + j]); \ |
176 | 0 | \ |
177 | 0 | const __m256i r0 = _mm256_permute2x128_si256(row0, row1, 0x20); \ |
178 | 0 | const __m256i r1 = _mm256_permute2x128_si256(row0, row1, 0x31); \ |
179 | 0 | \ |
180 | 0 | s[0] = r0; \ |
181 | 0 | s[1] = _mm256_alignr_epi8(r1, r0, 2); \ |
182 | 0 | \ |
183 | 0 | __m256i res_0 = _mm256_madd_epi16(s[0], coeffs_x[0]); \ |
184 | 0 | __m256i res_1 = _mm256_madd_epi16(s[1], coeffs_x[0]); \ |
185 | 0 | \ |
186 | 0 | res_0 = _mm256_sra_epi32(_mm256_add_epi32(res_0, round_const_x), \ |
187 | 0 | round_shift_x); \ |
188 | 0 | res_1 = _mm256_sra_epi32(_mm256_add_epi32(res_1, round_const_x), \ |
189 | 0 | round_shift_x); \ |
190 | 0 | \ |
191 | 0 | res_0 = _mm256_packs_epi32(res_0, res_0); \ |
192 | 0 | res_1 = _mm256_packs_epi32(res_1, res_1); \ |
193 | 0 | __m256i res = _mm256_unpacklo_epi16(res_0, res_1); \ |
194 | 0 | \ |
195 | 0 | _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); \ |
196 | 0 | } |
197 | | |
198 | | #define CONVOLVE_SR_HORIZONTAL_FILTER_4TAP \ |
199 | | for (i = 0; i < (im_h - 2); i += 2) { \ |
200 | | __m256i data = _mm256_castsi128_si256( \ |
201 | | _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); \ |
202 | | data = _mm256_inserti128_si256( \ |
203 | | data, \ |
204 | | _mm_loadu_si128( \ |
205 | | (__m128i *)&src_ptr[(i * src_stride) + j + src_stride]), \ |
206 | | 1); \ |
207 | | __m256i res = convolve_lowbd_x_4tap(data, coeffs_h + 1, filt); \ |
208 | | res = \ |
209 | | _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h); \ |
210 | | _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); \ |
211 | | } \ |
212 | | __m256i data_1 = _mm256_castsi128_si256( \ |
213 | | _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); \ |
214 | | __m256i res = convolve_lowbd_x_4tap(data_1, coeffs_h + 1, filt); \ |
215 | | res = _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h); \ |
216 | | _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); |
217 | | |
218 | | #define CONVOLVE_SR_VERTICAL_FILTER_4TAP \ |
219 | | __m256i s[6]; \ |
220 | | __m256i src_0 = _mm256_loadu_si256((__m256i *)(im_block + 0 * im_stride)); \ |
221 | | __m256i src_1 = _mm256_loadu_si256((__m256i *)(im_block + 1 * im_stride)); \ |
222 | | __m256i src_2 = _mm256_loadu_si256((__m256i *)(im_block + 2 * im_stride)); \ |
223 | | __m256i src_3 = _mm256_loadu_si256((__m256i *)(im_block + 3 * im_stride)); \ |
224 | | \ |
225 | | s[0] = _mm256_unpacklo_epi16(src_0, src_1); \ |
226 | | s[1] = _mm256_unpacklo_epi16(src_2, src_3); \ |
227 | | s[3] = _mm256_unpackhi_epi16(src_0, src_1); \ |
228 | | s[4] = _mm256_unpackhi_epi16(src_2, src_3); \ |
229 | | \ |
230 | | for (i = 0; i < h; i += 2) { \ |
231 | | const int16_t *data = &im_block[i * im_stride]; \ |
232 | | const __m256i s4 = _mm256_loadu_si256((__m256i *)(data + 4 * im_stride)); \ |
233 | | const __m256i s5 = _mm256_loadu_si256((__m256i *)(data + 5 * im_stride)); \ |
234 | | s[2] = _mm256_unpacklo_epi16(s4, s5); \ |
235 | | s[5] = _mm256_unpackhi_epi16(s4, s5); \ |
236 | | \ |
237 | | __m256i res_a = convolve_4tap(s, coeffs_v + 1); \ |
238 | | __m256i res_b = convolve_4tap(s + 3, coeffs_v + 1); \ |
239 | | \ |
240 | | res_a = \ |
241 | | _mm256_sra_epi32(_mm256_add_epi32(res_a, sum_round_v), sum_shift_v); \ |
242 | | res_b = \ |
243 | | _mm256_sra_epi32(_mm256_add_epi32(res_b, sum_round_v), sum_shift_v); \ |
244 | | const __m256i res_a_round = _mm256_sra_epi32( \ |
245 | | _mm256_add_epi32(res_a, round_const_v), round_shift_v); \ |
246 | | const __m256i res_b_round = _mm256_sra_epi32( \ |
247 | | _mm256_add_epi32(res_b, round_const_v), round_shift_v); \ |
248 | | const __m256i res_16bit = _mm256_packs_epi32(res_a_round, res_b_round); \ |
249 | | const __m256i res_8b = _mm256_packus_epi16(res_16bit, res_16bit); \ |
250 | | const __m128i res_0 = _mm256_castsi256_si128(res_8b); \ |
251 | | const __m128i res_1 = _mm256_extracti128_si256(res_8b, 1); \ |
252 | | \ |
253 | | __m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j]; \ |
254 | | __m128i *const p_1 = (__m128i *)&dst[i * dst_stride + j + dst_stride]; \ |
255 | | if (w - j > 4) { \ |
256 | | _mm_storel_epi64(p_0, res_0); \ |
257 | | _mm_storel_epi64(p_1, res_1); \ |
258 | | } else if (w == 4) { \ |
259 | | xx_storel_32(p_0, res_0); \ |
260 | | xx_storel_32(p_1, res_1); \ |
261 | | } else { \ |
262 | | *(uint16_t *)p_0 = (uint16_t)_mm_cvtsi128_si32(res_0); \ |
263 | | *(uint16_t *)p_1 = (uint16_t)_mm_cvtsi128_si32(res_1); \ |
264 | | } \ |
265 | | \ |
266 | | s[0] = s[1]; \ |
267 | | s[1] = s[2]; \ |
268 | | s[3] = s[4]; \ |
269 | | s[4] = s[5]; \ |
270 | | } |
271 | | |
272 | | #define CONVOLVE_SR_HORIZONTAL_FILTER_6TAP \ |
273 | | for (i = 0; i < (im_h - 2); i += 2) { \ |
274 | | __m256i data = _mm256_castsi128_si256( \ |
275 | | _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); \ |
276 | | data = _mm256_inserti128_si256( \ |
277 | | data, \ |
278 | | _mm_loadu_si128( \ |
279 | | (__m128i *)&src_ptr[(i * src_stride) + j + src_stride]), \ |
280 | | 1); \ |
281 | | \ |
282 | | __m256i res = convolve_lowbd_x_6tap(data, coeffs_h, filt); \ |
283 | | res = \ |
284 | | _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h); \ |
285 | | _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); \ |
286 | | } \ |
287 | | \ |
288 | | __m256i data_1 = _mm256_castsi128_si256( \ |
289 | | _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); \ |
290 | | \ |
291 | | __m256i res = convolve_lowbd_x_6tap(data_1, coeffs_h, filt); \ |
292 | | \ |
293 | | res = _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h); \ |
294 | | \ |
295 | | _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); |
296 | | |
297 | | #define CONVOLVE_SR_VERTICAL_FILTER_6TAP \ |
298 | | __m256i src_0 = _mm256_loadu_si256((__m256i *)(im_block + 0 * im_stride)); \ |
299 | | __m256i src_1 = _mm256_loadu_si256((__m256i *)(im_block + 1 * im_stride)); \ |
300 | | __m256i src_2 = _mm256_loadu_si256((__m256i *)(im_block + 2 * im_stride)); \ |
301 | | __m256i src_3 = _mm256_loadu_si256((__m256i *)(im_block + 3 * im_stride)); \ |
302 | | \ |
303 | | __m256i s[8]; \ |
304 | | s[0] = _mm256_unpacklo_epi16(src_0, src_1); \ |
305 | | s[1] = _mm256_unpacklo_epi16(src_2, src_3); \ |
306 | | \ |
307 | | s[3] = _mm256_unpackhi_epi16(src_0, src_1); \ |
308 | | s[4] = _mm256_unpackhi_epi16(src_2, src_3); \ |
309 | | \ |
310 | | for (i = 0; i < h; i += 2) { \ |
311 | | const int16_t *data = &im_block[i * im_stride]; \ |
312 | | \ |
313 | | const __m256i s6 = _mm256_loadu_si256((__m256i *)(data + 4 * im_stride)); \ |
314 | | const __m256i s7 = _mm256_loadu_si256((__m256i *)(data + 5 * im_stride)); \ |
315 | | \ |
316 | | s[2] = _mm256_unpacklo_epi16(s6, s7); \ |
317 | | s[5] = _mm256_unpackhi_epi16(s6, s7); \ |
318 | | \ |
319 | | __m256i res_a = convolve_6tap(s, coeffs_v); \ |
320 | | __m256i res_b = convolve_6tap(s + 3, coeffs_v); \ |
321 | | \ |
322 | | res_a = \ |
323 | | _mm256_sra_epi32(_mm256_add_epi32(res_a, sum_round_v), sum_shift_v); \ |
324 | | res_b = \ |
325 | | _mm256_sra_epi32(_mm256_add_epi32(res_b, sum_round_v), sum_shift_v); \ |
326 | | \ |
327 | | const __m256i res_a_round = _mm256_sra_epi32( \ |
328 | | _mm256_add_epi32(res_a, round_const_v), round_shift_v); \ |
329 | | const __m256i res_b_round = _mm256_sra_epi32( \ |
330 | | _mm256_add_epi32(res_b, round_const_v), round_shift_v); \ |
331 | | \ |
332 | | const __m256i res_16bit = _mm256_packs_epi32(res_a_round, res_b_round); \ |
333 | | const __m256i res_8b = _mm256_packus_epi16(res_16bit, res_16bit); \ |
334 | | \ |
335 | | const __m128i res_0 = _mm256_castsi256_si128(res_8b); \ |
336 | | const __m128i res_1 = _mm256_extracti128_si256(res_8b, 1); \ |
337 | | \ |
338 | | __m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j]; \ |
339 | | __m128i *const p_1 = (__m128i *)&dst[i * dst_stride + j + dst_stride]; \ |
340 | | if (w - j > 4) { \ |
341 | | _mm_storel_epi64(p_0, res_0); \ |
342 | | _mm_storel_epi64(p_1, res_1); \ |
343 | | } else if (w == 4) { \ |
344 | | xx_storel_32(p_0, res_0); \ |
345 | | xx_storel_32(p_1, res_1); \ |
346 | | } else { \ |
347 | | *(uint16_t *)p_0 = (uint16_t)_mm_cvtsi128_si32(res_0); \ |
348 | | *(uint16_t *)p_1 = (uint16_t)_mm_cvtsi128_si32(res_1); \ |
349 | | } \ |
350 | | \ |
351 | | s[0] = s[1]; \ |
352 | | s[1] = s[2]; \ |
353 | | \ |
354 | | s[3] = s[4]; \ |
355 | | s[4] = s[5]; \ |
356 | | } |
357 | | |
358 | | #define CONVOLVE_SR_HORIZONTAL_FILTER_8TAP \ |
359 | | for (i = 0; i < (im_h - 2); i += 2) { \ |
360 | | __m256i data = _mm256_castsi128_si256( \ |
361 | | _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); \ |
362 | | data = _mm256_inserti128_si256( \ |
363 | | data, \ |
364 | | _mm_loadu_si128( \ |
365 | | (__m128i *)&src_ptr[(i * src_stride) + j + src_stride]), \ |
366 | | 1); \ |
367 | | \ |
368 | | __m256i res = convolve_lowbd_x(data, coeffs_h, filt); \ |
369 | | res = \ |
370 | | _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h); \ |
371 | | _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); \ |
372 | | } \ |
373 | | \ |
374 | | __m256i data_1 = _mm256_castsi128_si256( \ |
375 | | _mm_loadu_si128((__m128i *)&src_ptr[(i * src_stride) + j])); \ |
376 | | \ |
377 | | __m256i res = convolve_lowbd_x(data_1, coeffs_h, filt); \ |
378 | | \ |
379 | | res = _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h); \ |
380 | | \ |
381 | | _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); |
382 | | |
383 | | #define CONVOLVE_SR_VERTICAL_FILTER_8TAP \ |
384 | | __m256i src_0 = _mm256_loadu_si256((__m256i *)(im_block + 0 * im_stride)); \ |
385 | | __m256i src_1 = _mm256_loadu_si256((__m256i *)(im_block + 1 * im_stride)); \ |
386 | | __m256i src_2 = _mm256_loadu_si256((__m256i *)(im_block + 2 * im_stride)); \ |
387 | | __m256i src_3 = _mm256_loadu_si256((__m256i *)(im_block + 3 * im_stride)); \ |
388 | | __m256i src_4 = _mm256_loadu_si256((__m256i *)(im_block + 4 * im_stride)); \ |
389 | | __m256i src_5 = _mm256_loadu_si256((__m256i *)(im_block + 5 * im_stride)); \ |
390 | | \ |
391 | | __m256i s[8]; \ |
392 | | s[0] = _mm256_unpacklo_epi16(src_0, src_1); \ |
393 | | s[1] = _mm256_unpacklo_epi16(src_2, src_3); \ |
394 | | s[2] = _mm256_unpacklo_epi16(src_4, src_5); \ |
395 | | \ |
396 | | s[4] = _mm256_unpackhi_epi16(src_0, src_1); \ |
397 | | s[5] = _mm256_unpackhi_epi16(src_2, src_3); \ |
398 | | s[6] = _mm256_unpackhi_epi16(src_4, src_5); \ |
399 | | \ |
400 | | for (i = 0; i < h; i += 2) { \ |
401 | | const int16_t *data = &im_block[i * im_stride]; \ |
402 | | \ |
403 | | const __m256i s6 = _mm256_loadu_si256((__m256i *)(data + 6 * im_stride)); \ |
404 | | const __m256i s7 = _mm256_loadu_si256((__m256i *)(data + 7 * im_stride)); \ |
405 | | \ |
406 | | s[3] = _mm256_unpacklo_epi16(s6, s7); \ |
407 | | s[7] = _mm256_unpackhi_epi16(s6, s7); \ |
408 | | \ |
409 | | __m256i res_a = convolve(s, coeffs_v); \ |
410 | | __m256i res_b = convolve(s + 4, coeffs_v); \ |
411 | | \ |
412 | | res_a = \ |
413 | | _mm256_sra_epi32(_mm256_add_epi32(res_a, sum_round_v), sum_shift_v); \ |
414 | | res_b = \ |
415 | | _mm256_sra_epi32(_mm256_add_epi32(res_b, sum_round_v), sum_shift_v); \ |
416 | | \ |
417 | | const __m256i res_a_round = _mm256_sra_epi32( \ |
418 | | _mm256_add_epi32(res_a, round_const_v), round_shift_v); \ |
419 | | const __m256i res_b_round = _mm256_sra_epi32( \ |
420 | | _mm256_add_epi32(res_b, round_const_v), round_shift_v); \ |
421 | | \ |
422 | | const __m256i res_16bit = _mm256_packs_epi32(res_a_round, res_b_round); \ |
423 | | const __m256i res_8b = _mm256_packus_epi16(res_16bit, res_16bit); \ |
424 | | \ |
425 | | const __m128i res_0 = _mm256_castsi256_si128(res_8b); \ |
426 | | const __m128i res_1 = _mm256_extracti128_si256(res_8b, 1); \ |
427 | | \ |
428 | | __m128i *const p_0 = (__m128i *)&dst[i * dst_stride + j]; \ |
429 | | __m128i *const p_1 = (__m128i *)&dst[i * dst_stride + j + dst_stride]; \ |
430 | | if (w - j > 4) { \ |
431 | | _mm_storel_epi64(p_0, res_0); \ |
432 | | _mm_storel_epi64(p_1, res_1); \ |
433 | | } else if (w == 4) { \ |
434 | | xx_storel_32(p_0, res_0); \ |
435 | | xx_storel_32(p_1, res_1); \ |
436 | | } else { \ |
437 | | *(uint16_t *)p_0 = (uint16_t)_mm_cvtsi128_si32(res_0); \ |
438 | | *(uint16_t *)p_1 = (uint16_t)_mm_cvtsi128_si32(res_1); \ |
439 | | } \ |
440 | | \ |
441 | | s[0] = s[1]; \ |
442 | | s[1] = s[2]; \ |
443 | | s[2] = s[3]; \ |
444 | | \ |
445 | | s[4] = s[5]; \ |
446 | | s[5] = s[6]; \ |
447 | | s[6] = s[7]; \ |
448 | | } |
449 | | |
450 | | #define DIST_WTD_CONVOLVE_HORIZONTAL_FILTER_8TAP \ |
451 | | for (i = 0; i < im_h; i += 2) { \ |
452 | | __m256i data = _mm256_castsi128_si256(_mm_loadu_si128((__m128i *)src_h)); \ |
453 | | if (i + 1 < im_h) \ |
454 | | data = _mm256_inserti128_si256( \ |
455 | | data, _mm_loadu_si128((__m128i *)(src_h + src_stride)), 1); \ |
456 | | src_h += (src_stride << 1); \ |
457 | | __m256i res = convolve_lowbd_x(data, coeffs_x, filt); \ |
458 | | \ |
459 | | res = \ |
460 | | _mm256_sra_epi16(_mm256_add_epi16(res, round_const_h), round_shift_h); \ |
461 | | \ |
462 | | _mm256_store_si256((__m256i *)&im_block[i * im_stride], res); \ |
463 | | } |
464 | | |
465 | | // In optical flow MV refinement, unaligned store (_mm_storeu_si128) is used |
466 | | // in this function to avoid a subblock boundary error. |
467 | | #define DIST_WTD_CONVOLVE_VERTICAL_FILTER_8TAP \ |
468 | | __m256i s[8]; \ |
469 | | __m256i s0 = _mm256_loadu_si256((__m256i *)(im_block + 0 * im_stride)); \ |
470 | | __m256i s1 = _mm256_loadu_si256((__m256i *)(im_block + 1 * im_stride)); \ |
471 | | __m256i s2 = _mm256_loadu_si256((__m256i *)(im_block + 2 * im_stride)); \ |
472 | | __m256i s3 = _mm256_loadu_si256((__m256i *)(im_block + 3 * im_stride)); \ |
473 | | __m256i s4 = _mm256_loadu_si256((__m256i *)(im_block + 4 * im_stride)); \ |
474 | | __m256i s5 = _mm256_loadu_si256((__m256i *)(im_block + 5 * im_stride)); \ |
475 | | \ |
476 | | s[0] = _mm256_unpacklo_epi16(s0, s1); \ |
477 | | s[1] = _mm256_unpacklo_epi16(s2, s3); \ |
478 | | s[2] = _mm256_unpacklo_epi16(s4, s5); \ |
479 | | \ |
480 | | s[4] = _mm256_unpackhi_epi16(s0, s1); \ |
481 | | s[5] = _mm256_unpackhi_epi16(s2, s3); \ |
482 | | s[6] = _mm256_unpackhi_epi16(s4, s5); \ |
483 | | \ |
484 | | for (i = 0; i < h; i += 2) { \ |
485 | | const int16_t *data = &im_block[i * im_stride]; \ |
486 | | \ |
487 | | const __m256i s6 = _mm256_loadu_si256((__m256i *)(data + 6 * im_stride)); \ |
488 | | const __m256i s7 = _mm256_loadu_si256((__m256i *)(data + 7 * im_stride)); \ |
489 | | \ |
490 | | s[3] = _mm256_unpacklo_epi16(s6, s7); \ |
491 | | s[7] = _mm256_unpackhi_epi16(s6, s7); \ |
492 | | \ |
493 | | const __m256i res_a = convolve(s, coeffs_y); \ |
494 | | const __m256i res_a_round = _mm256_sra_epi32( \ |
495 | | _mm256_add_epi32(res_a, round_const_v), round_shift_v); \ |
496 | | \ |
497 | | if (w - j > 4) { \ |
498 | | const __m256i res_b = convolve(s + 4, coeffs_y); \ |
499 | | const __m256i res_b_round = _mm256_sra_epi32( \ |
500 | | _mm256_add_epi32(res_b, round_const_v), round_shift_v); \ |
501 | | const __m256i res_16b = _mm256_packs_epi32(res_a_round, res_b_round); \ |
502 | | const __m256i res_unsigned = _mm256_add_epi16(res_16b, offset_const); \ |
503 | | \ |
504 | | if (do_average) { \ |
505 | | const __m256i data_ref_0 = load_line2_avx2( \ |
506 | | &dst[i * dst_stride + j], &dst[i * dst_stride + j + dst_stride]); \ |
507 | | const __m256i comp_avg_res = \ |
508 | | comp_avg(&data_ref_0, &res_unsigned, &wt, use_wtd_comp_avg); \ |
509 | | \ |
510 | | const __m256i round_result = convolve_rounding( \ |
511 | | &comp_avg_res, &offset_const, &rounding_const, rounding_shift); \ |
512 | | \ |
513 | | const __m256i res_8 = _mm256_packus_epi16(round_result, round_result); \ |
514 | | const __m128i res_0 = _mm256_castsi256_si128(res_8); \ |
515 | | const __m128i res_1 = _mm256_extracti128_si256(res_8, 1); \ |
516 | | \ |
517 | | _mm_storel_epi64((__m128i *)(&dst0[i * dst_stride0 + j]), res_0); \ |
518 | | _mm_storel_epi64( \ |
519 | | (__m128i *)((&dst0[i * dst_stride0 + j + dst_stride0])), res_1); \ |
520 | | } else { \ |
521 | | const __m128i res_0 = _mm256_castsi256_si128(res_unsigned); \ |
522 | | _mm_storeu_si128((__m128i *)(&dst[i * dst_stride + j]), res_0); \ |
523 | | \ |
524 | | const __m128i res_1 = _mm256_extracti128_si256(res_unsigned, 1); \ |
525 | | _mm_storeu_si128((__m128i *)(&dst[i * dst_stride + j + dst_stride]), \ |
526 | | res_1); \ |
527 | | } \ |
528 | | } else { \ |
529 | | const __m256i res_16b = _mm256_packs_epi32(res_a_round, res_a_round); \ |
530 | | const __m256i res_unsigned = _mm256_add_epi16(res_16b, offset_const); \ |
531 | | \ |
532 | | if (do_average) { \ |
533 | | const __m256i data_ref_0 = load_line2_avx2( \ |
534 | | &dst[i * dst_stride + j], &dst[i * dst_stride + j + dst_stride]); \ |
535 | | \ |
536 | | const __m256i comp_avg_res = \ |
537 | | comp_avg(&data_ref_0, &res_unsigned, &wt, use_wtd_comp_avg); \ |
538 | | \ |
539 | | const __m256i round_result = convolve_rounding( \ |
540 | | &comp_avg_res, &offset_const, &rounding_const, rounding_shift); \ |
541 | | \ |
542 | | const __m256i res_8 = _mm256_packus_epi16(round_result, round_result); \ |
543 | | const __m128i res_0 = _mm256_castsi256_si128(res_8); \ |
544 | | const __m128i res_1 = _mm256_extracti128_si256(res_8, 1); \ |
545 | | \ |
546 | | *(uint32_t *)(&dst0[i * dst_stride0 + j]) = _mm_cvtsi128_si32(res_0); \ |
547 | | *(uint32_t *)(&dst0[i * dst_stride0 + j + dst_stride0]) = \ |
548 | | _mm_cvtsi128_si32(res_1); \ |
549 | | \ |
550 | | } else { \ |
551 | | const __m128i res_0 = _mm256_castsi256_si128(res_unsigned); \ |
552 | | _mm_storeu_si128((__m128i *)(&dst[i * dst_stride + j]), res_0); \ |
553 | | \ |
554 | | const __m128i res_1 = _mm256_extracti128_si256(res_unsigned, 1); \ |
555 | | _mm_storeu_si128((__m128i *)(&dst[i * dst_stride + j + dst_stride]), \ |
556 | | res_1); \ |
557 | | } \ |
558 | | } \ |
559 | | \ |
560 | | s[0] = s[1]; \ |
561 | | s[1] = s[2]; \ |
562 | | s[2] = s[3]; \ |
563 | | \ |
564 | | s[4] = s[5]; \ |
565 | | s[5] = s[6]; \ |
566 | | s[6] = s[7]; \ |
567 | | } |
568 | | |
569 | | static INLINE void prepare_coeffs_lowbd( |
570 | | const InterpFilterParams *const filter_params, const int subpel_q4, |
571 | 0 | __m256i *const coeffs /* [4] */) { |
572 | 0 | const int16_t *const filter = av2_get_interp_filter_subpel_kernel( |
573 | 0 | filter_params, subpel_q4 & SUBPEL_MASK); |
574 | 0 | const __m128i coeffs_8 = _mm_loadu_si128((__m128i *)filter); |
575 | 0 | const __m256i filter_coeffs = _mm256_broadcastsi128_si256(coeffs_8); |
576 | 0 |
|
577 | 0 | // right shift all filter co-efficients by 1 to reduce the bits required. |
578 | 0 | // This extra right shift will be taken care of at the end while rounding |
579 | 0 | // the result. |
580 | 0 | // Since all filter co-efficients are even, this change will not affect the |
581 | 0 | // end result |
582 | 0 | assert(_mm_test_all_zeros(_mm_and_si128(coeffs_8, _mm_set1_epi16(1)), |
583 | 0 | _mm_set1_epi16((short)0xffff))); |
584 | 0 |
|
585 | 0 | const __m256i coeffs_1 = _mm256_srai_epi16(filter_coeffs, 1); |
586 | 0 |
|
587 | 0 | // coeffs 0 1 0 1 0 1 0 1 |
588 | 0 | coeffs[0] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0200u)); |
589 | 0 | // coeffs 2 3 2 3 2 3 2 3 |
590 | 0 | coeffs[1] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0604u)); |
591 | 0 | // coeffs 4 5 4 5 4 5 4 5 |
592 | 0 | coeffs[2] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0a08u)); |
593 | 0 | // coeffs 6 7 6 7 6 7 6 7 |
594 | 0 | coeffs[3] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0e0cu)); |
595 | 0 | } Unexecuted instantiation: highbd_convolve_avx2.c:prepare_coeffs_lowbd Unexecuted instantiation: highbd_convolve_2d_avx2.c:prepare_coeffs_lowbd Unexecuted instantiation: highbd_jnt_convolve_avx2.c:prepare_coeffs_lowbd |
596 | | |
597 | | static INLINE void prepare_coeffs_6t_lowbd( |
598 | | const InterpFilterParams *const filter_params, const int subpel_q4, |
599 | 0 | __m256i *const coeffs /* [4] */) { |
600 | 0 | const int16_t *const filter = av2_get_interp_filter_subpel_kernel( |
601 | 0 | filter_params, subpel_q4 & SUBPEL_MASK); |
602 | 0 | const __m128i coeffs_8 = _mm_loadu_si128((__m128i *)filter); |
603 | 0 | const __m256i filter_coeffs = _mm256_broadcastsi128_si256(coeffs_8); |
604 | 0 |
|
605 | 0 | // right shift all filter co-efficients by 1 to reduce the bits required. |
606 | 0 | // This extra right shift will be taken care of at the end while rounding |
607 | 0 | // the result. |
608 | 0 | // Since all filter co-efficients are even, this change will not affect the |
609 | 0 | // end result |
610 | 0 | assert(_mm_test_all_zeros(_mm_and_si128(coeffs_8, _mm_set1_epi16(1)), |
611 | 0 | _mm_set1_epi16((int16_t)0xffff))); |
612 | 0 |
|
613 | 0 | const __m256i coeffs_1 = _mm256_srai_epi16(filter_coeffs, 1); |
614 | 0 |
|
615 | 0 | // coeffs 1 2 1 2 1 2 1 2 |
616 | 0 | coeffs[0] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0402u)); |
617 | 0 | // coeffs 3 4 3 4 3 4 3 4 |
618 | 0 | coeffs[1] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0806u)); |
619 | 0 | // coeffs 5 6 5 6 5 6 5 6 |
620 | 0 | coeffs[2] = _mm256_shuffle_epi8(coeffs_1, _mm256_set1_epi16(0x0c0au)); |
621 | 0 | } Unexecuted instantiation: highbd_convolve_avx2.c:prepare_coeffs_6t_lowbd Unexecuted instantiation: highbd_convolve_2d_avx2.c:prepare_coeffs_6t_lowbd Unexecuted instantiation: highbd_jnt_convolve_avx2.c:prepare_coeffs_6t_lowbd |
622 | | |
623 | | static INLINE void prepare_coeffs_6t( |
624 | | const InterpFilterParams *const filter_params, const int subpel_q4, |
625 | 0 | __m256i *const coeffs /* [4] */) { |
626 | 0 | const int16_t *filter = av2_get_interp_filter_subpel_kernel( |
627 | 0 | filter_params, subpel_q4 & SUBPEL_MASK); |
628 | |
|
629 | 0 | const __m128i coeff_8 = _mm_loadu_si128((__m128i *)(filter + 1)); |
630 | 0 | const __m256i coeff = _mm256_broadcastsi128_si256(coeff_8); |
631 | | |
632 | | // coeffs 1 2 1 2 1 2 1 2 |
633 | 0 | coeffs[0] = _mm256_shuffle_epi32(coeff, 0x00); |
634 | | // coeffs 3 4 3 4 3 4 3 4 |
635 | 0 | coeffs[1] = _mm256_shuffle_epi32(coeff, 0x55); |
636 | | // coeffs 5 6 5 6 5 6 5 6 |
637 | 0 | coeffs[2] = _mm256_shuffle_epi32(coeff, 0xaa); |
638 | 0 | } Unexecuted instantiation: highbd_convolve_avx2.c:prepare_coeffs_6t Unexecuted instantiation: highbd_convolve_2d_avx2.c:prepare_coeffs_6t Unexecuted instantiation: highbd_jnt_convolve_avx2.c:prepare_coeffs_6t |
639 | | |
640 | | static INLINE void prepare_coeffs(const InterpFilterParams *const filter_params, |
641 | | const int subpel_q4, |
642 | 0 | __m256i *const coeffs /* [4] */) { |
643 | 0 | const int16_t *filter = av2_get_interp_filter_subpel_kernel( |
644 | 0 | filter_params, subpel_q4 & SUBPEL_MASK); |
645 | |
|
646 | 0 | const __m128i coeff_8 = _mm_loadu_si128((__m128i *)filter); |
647 | 0 | const __m256i coeff = _mm256_broadcastsi128_si256(coeff_8); |
648 | | |
649 | | // coeffs 0 1 0 1 0 1 0 1 |
650 | 0 | coeffs[0] = _mm256_shuffle_epi32(coeff, 0x00); |
651 | | // coeffs 2 3 2 3 2 3 2 3 |
652 | 0 | coeffs[1] = _mm256_shuffle_epi32(coeff, 0x55); |
653 | | // coeffs 4 5 4 5 4 5 4 5 |
654 | 0 | coeffs[2] = _mm256_shuffle_epi32(coeff, 0xaa); |
655 | | // coeffs 6 7 6 7 6 7 6 7 |
656 | 0 | coeffs[3] = _mm256_shuffle_epi32(coeff, 0xff); |
657 | 0 | } Unexecuted instantiation: highbd_convolve_avx2.c:prepare_coeffs Unexecuted instantiation: highbd_convolve_2d_avx2.c:prepare_coeffs Unexecuted instantiation: highbd_jnt_convolve_avx2.c:prepare_coeffs |
658 | | |
659 | | static INLINE void prepare_coeffs_4t( |
660 | | const InterpFilterParams *const filter_params, const int subpel_q4, |
661 | 0 | __m256i *const coeffs /* [4] */) { |
662 | 0 | const int16_t *filter = av2_get_interp_filter_subpel_kernel( |
663 | 0 | filter_params, subpel_q4 & SUBPEL_MASK); |
664 | |
|
665 | 0 | const __m128i coeff_8 = _mm_loadu_si128((__m128i *)filter); |
666 | 0 | const __m256i coeff = _mm256_broadcastsi128_si256(coeff_8); |
667 | | |
668 | | // coeffs 2 3 2 3 2 3 2 3 |
669 | 0 | coeffs[0] = _mm256_shuffle_epi32(coeff, 0x55); |
670 | | // coeffs 4 5 4 5 4 5 4 5 |
671 | 0 | coeffs[1] = _mm256_shuffle_epi32(coeff, 0xaa); |
672 | 0 | } Unexecuted instantiation: highbd_convolve_avx2.c:prepare_coeffs_4t Unexecuted instantiation: highbd_convolve_2d_avx2.c:prepare_coeffs_4t Unexecuted instantiation: highbd_jnt_convolve_avx2.c:prepare_coeffs_4t |
673 | | |
674 | | // Preparation of filter coefficient registers for BILINEAR interpolation |
675 | | // filter. |
676 | | static INLINE __m256i prepare_coeffs_bilinear( |
677 | 0 | const InterpFilterParams *const filter_params, const int subpel_q4) { |
678 | 0 | const int16_t *filter = av2_get_interp_filter_subpel_kernel( |
679 | 0 | filter_params, subpel_q4 & SUBPEL_MASK); |
680 | 0 | const int16_t *filter_pos = filter; |
681 | 0 | if (filter_params->taps == 8) filter_pos = filter + 3; |
682 | 0 | const int filter_coeff = ((filter_pos[1] << 16) | filter_pos[0]); |
683 | 0 | return _mm256_set1_epi32(filter_coeff); |
684 | 0 | } Unexecuted instantiation: highbd_convolve_avx2.c:prepare_coeffs_bilinear Unexecuted instantiation: highbd_convolve_2d_avx2.c:prepare_coeffs_bilinear Unexecuted instantiation: highbd_jnt_convolve_avx2.c:prepare_coeffs_bilinear |
685 | | |
686 | | static INLINE __m256i convolve_lowbd(const __m256i *const s, |
687 | 0 | const __m256i *const coeffs) { |
688 | 0 | const __m256i res_01 = _mm256_maddubs_epi16(s[0], coeffs[0]); |
689 | 0 | const __m256i res_23 = _mm256_maddubs_epi16(s[1], coeffs[1]); |
690 | 0 | const __m256i res_45 = _mm256_maddubs_epi16(s[2], coeffs[2]); |
691 | 0 | const __m256i res_67 = _mm256_maddubs_epi16(s[3], coeffs[3]); |
692 | 0 |
|
693 | 0 | // order: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
694 | 0 | const __m256i res = _mm256_add_epi16(_mm256_add_epi16(res_01, res_45), |
695 | 0 | _mm256_add_epi16(res_23, res_67)); |
696 | 0 |
|
697 | 0 | return res; |
698 | 0 | } Unexecuted instantiation: highbd_convolve_avx2.c:convolve_lowbd Unexecuted instantiation: highbd_convolve_2d_avx2.c:convolve_lowbd Unexecuted instantiation: highbd_jnt_convolve_avx2.c:convolve_lowbd |
699 | | |
700 | | static INLINE __m256i convolve_lowbd_6tap(const __m256i *const s, |
701 | 0 | const __m256i *const coeffs) { |
702 | 0 | const __m256i res_01 = _mm256_maddubs_epi16(s[0], coeffs[0]); |
703 | 0 | const __m256i res_23 = _mm256_maddubs_epi16(s[1], coeffs[1]); |
704 | 0 | const __m256i res_45 = _mm256_maddubs_epi16(s[2], coeffs[2]); |
705 | 0 |
|
706 | 0 | // order: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
707 | 0 | const __m256i res = |
708 | 0 | _mm256_add_epi16(_mm256_add_epi16(res_01, res_45), res_23); |
709 | 0 |
|
710 | 0 | return res; |
711 | 0 | } Unexecuted instantiation: highbd_convolve_avx2.c:convolve_lowbd_6tap Unexecuted instantiation: highbd_convolve_2d_avx2.c:convolve_lowbd_6tap Unexecuted instantiation: highbd_jnt_convolve_avx2.c:convolve_lowbd_6tap |
712 | | |
713 | | static INLINE __m256i convolve_lowbd_4tap(const __m256i *const s, |
714 | 0 | const __m256i *const coeffs) { |
715 | 0 | const __m256i res_23 = _mm256_maddubs_epi16(s[0], coeffs[0]); |
716 | 0 | const __m256i res_45 = _mm256_maddubs_epi16(s[1], coeffs[1]); |
717 | 0 |
|
718 | 0 | // order: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 |
719 | 0 | const __m256i res = _mm256_add_epi16(res_45, res_23); |
720 | 0 |
|
721 | 0 | return res; |
722 | 0 | } Unexecuted instantiation: highbd_convolve_avx2.c:convolve_lowbd_4tap Unexecuted instantiation: highbd_convolve_2d_avx2.c:convolve_lowbd_4tap Unexecuted instantiation: highbd_jnt_convolve_avx2.c:convolve_lowbd_4tap |
723 | | |
724 | | static INLINE __m256i convolve_6tap(const __m256i *const s, |
725 | 0 | const __m256i *const coeffs) { |
726 | 0 | const __m256i res_0 = _mm256_madd_epi16(s[0], coeffs[0]); |
727 | 0 | const __m256i res_1 = _mm256_madd_epi16(s[1], coeffs[1]); |
728 | 0 | const __m256i res_2 = _mm256_madd_epi16(s[2], coeffs[2]); |
729 | |
|
730 | 0 | const __m256i res = _mm256_add_epi32(_mm256_add_epi32(res_0, res_1), res_2); |
731 | |
|
732 | 0 | return res; |
733 | 0 | } Unexecuted instantiation: highbd_convolve_avx2.c:convolve_6tap Unexecuted instantiation: highbd_convolve_2d_avx2.c:convolve_6tap Unexecuted instantiation: highbd_jnt_convolve_avx2.c:convolve_6tap |
734 | | |
735 | | static INLINE __m256i convolve(const __m256i *const s, |
736 | 0 | const __m256i *const coeffs) { |
737 | 0 | const __m256i res_0 = _mm256_madd_epi16(s[0], coeffs[0]); |
738 | 0 | const __m256i res_1 = _mm256_madd_epi16(s[1], coeffs[1]); |
739 | 0 | const __m256i res_2 = _mm256_madd_epi16(s[2], coeffs[2]); |
740 | 0 | const __m256i res_3 = _mm256_madd_epi16(s[3], coeffs[3]); |
741 | |
|
742 | 0 | const __m256i res = _mm256_add_epi32(_mm256_add_epi32(res_0, res_1), |
743 | 0 | _mm256_add_epi32(res_2, res_3)); |
744 | |
|
745 | 0 | return res; |
746 | 0 | } Unexecuted instantiation: highbd_convolve_avx2.c:convolve Unexecuted instantiation: highbd_convolve_2d_avx2.c:convolve Unexecuted instantiation: highbd_jnt_convolve_avx2.c:convolve |
747 | | |
748 | | static INLINE __m256i convolve_4tap(const __m256i *const s, |
749 | 0 | const __m256i *const coeffs) { |
750 | 0 | const __m256i res_1 = _mm256_madd_epi16(s[0], coeffs[0]); |
751 | 0 | const __m256i res_2 = _mm256_madd_epi16(s[1], coeffs[1]); |
752 | |
|
753 | 0 | const __m256i res = _mm256_add_epi32(res_1, res_2); |
754 | 0 | return res; |
755 | 0 | } Unexecuted instantiation: highbd_convolve_avx2.c:convolve_4tap Unexecuted instantiation: highbd_convolve_2d_avx2.c:convolve_4tap Unexecuted instantiation: highbd_jnt_convolve_avx2.c:convolve_4tap |
756 | | |
757 | | static INLINE __m256i convolve_lowbd_x(const __m256i data, |
758 | | const __m256i *const coeffs, |
759 | 0 | const __m256i *const filt) { |
760 | 0 | __m256i s[4]; |
761 | 0 |
|
762 | 0 | s[0] = _mm256_shuffle_epi8(data, filt[0]); |
763 | 0 | s[1] = _mm256_shuffle_epi8(data, filt[1]); |
764 | 0 | s[2] = _mm256_shuffle_epi8(data, filt[2]); |
765 | 0 | s[3] = _mm256_shuffle_epi8(data, filt[3]); |
766 | 0 |
|
767 | 0 | return convolve_lowbd(s, coeffs); |
768 | 0 | } Unexecuted instantiation: highbd_convolve_avx2.c:convolve_lowbd_x Unexecuted instantiation: highbd_convolve_2d_avx2.c:convolve_lowbd_x Unexecuted instantiation: highbd_jnt_convolve_avx2.c:convolve_lowbd_x |
769 | | |
770 | | static INLINE __m256i convolve_lowbd_x_6tap(const __m256i data, |
771 | | const __m256i *const coeffs, |
772 | 0 | const __m256i *const filt) { |
773 | 0 | __m256i s[4]; |
774 | 0 |
|
775 | 0 | s[0] = _mm256_shuffle_epi8(data, filt[0]); |
776 | 0 | s[1] = _mm256_shuffle_epi8(data, filt[1]); |
777 | 0 | s[2] = _mm256_shuffle_epi8(data, filt[2]); |
778 | 0 |
|
779 | 0 | return convolve_lowbd_6tap(s, coeffs); |
780 | 0 | } Unexecuted instantiation: highbd_convolve_avx2.c:convolve_lowbd_x_6tap Unexecuted instantiation: highbd_convolve_2d_avx2.c:convolve_lowbd_x_6tap Unexecuted instantiation: highbd_jnt_convolve_avx2.c:convolve_lowbd_x_6tap |
781 | | |
782 | | static INLINE __m256i convolve_lowbd_x_4tap(const __m256i data, |
783 | | const __m256i *const coeffs, |
784 | 0 | const __m256i *const filt) { |
785 | 0 | __m256i s[2]; |
786 | 0 |
|
787 | 0 | s[0] = _mm256_shuffle_epi8(data, filt[0]); |
788 | 0 | s[1] = _mm256_shuffle_epi8(data, filt[1]); |
789 | 0 |
|
790 | 0 | return convolve_lowbd_4tap(s, coeffs); |
791 | 0 | } Unexecuted instantiation: highbd_convolve_avx2.c:convolve_lowbd_x_4tap Unexecuted instantiation: highbd_convolve_2d_avx2.c:convolve_lowbd_x_4tap Unexecuted instantiation: highbd_jnt_convolve_avx2.c:convolve_lowbd_x_4tap |
792 | | |
793 | | static INLINE void add_store_aligned_256(CONV_BUF_TYPE *const dst, |
794 | | const __m256i *const res, |
795 | 0 | const int do_average) { |
796 | 0 | __m256i d; |
797 | 0 | if (do_average) { |
798 | 0 | d = _mm256_loadu_si256((__m256i *)dst); |
799 | 0 | d = _mm256_add_epi32(d, *res); |
800 | 0 | d = _mm256_srai_epi32(d, 1); |
801 | 0 | } else { |
802 | 0 | d = *res; |
803 | 0 | } |
804 | 0 | _mm256_store_si256((__m256i *)dst, d); |
805 | 0 | } Unexecuted instantiation: highbd_convolve_avx2.c:add_store_aligned_256 Unexecuted instantiation: highbd_convolve_2d_avx2.c:add_store_aligned_256 Unexecuted instantiation: highbd_jnt_convolve_avx2.c:add_store_aligned_256 |
806 | | |
807 | | static INLINE __m256i comp_avg(const __m256i *const data_ref_0, |
808 | | const __m256i *const res_unsigned, |
809 | | const __m256i *const wt, |
810 | 0 | const int use_wtd_comp_avg) { |
811 | 0 | __m256i res; |
812 | 0 | if (use_wtd_comp_avg) { |
813 | 0 | const __m256i data_lo = _mm256_unpacklo_epi16(*data_ref_0, *res_unsigned); |
814 | 0 | const __m256i data_hi = _mm256_unpackhi_epi16(*data_ref_0, *res_unsigned); |
815 | 0 |
|
816 | 0 | const __m256i wt_res_lo = _mm256_madd_epi16(data_lo, *wt); |
817 | 0 | const __m256i wt_res_hi = _mm256_madd_epi16(data_hi, *wt); |
818 | 0 |
|
819 | 0 | const __m256i res_lo = _mm256_srai_epi32(wt_res_lo, DIST_PRECISION_BITS); |
820 | 0 | const __m256i res_hi = _mm256_srai_epi32(wt_res_hi, DIST_PRECISION_BITS); |
821 | 0 |
|
822 | 0 | res = _mm256_packs_epi32(res_lo, res_hi); |
823 | 0 | } else { |
824 | 0 | const __m256i wt_res = _mm256_add_epi16(*data_ref_0, *res_unsigned); |
825 | 0 | res = _mm256_srai_epi16(wt_res, 1); |
826 | 0 | } |
827 | 0 | return res; |
828 | 0 | } Unexecuted instantiation: highbd_convolve_avx2.c:comp_avg Unexecuted instantiation: highbd_convolve_2d_avx2.c:comp_avg Unexecuted instantiation: highbd_jnt_convolve_avx2.c:comp_avg |
829 | | |
830 | | static INLINE __m256i convolve_rounding(const __m256i *const res_unsigned, |
831 | | const __m256i *const offset_const, |
832 | | const __m256i *const round_const, |
833 | 0 | const int round_shift) { |
834 | 0 | const __m256i res_signed = _mm256_sub_epi16(*res_unsigned, *offset_const); |
835 | 0 | const __m256i res_round = _mm256_srai_epi16( |
836 | 0 | _mm256_add_epi16(res_signed, *round_const), round_shift); |
837 | 0 | return res_round; |
838 | 0 | } Unexecuted instantiation: highbd_convolve_avx2.c:convolve_rounding Unexecuted instantiation: highbd_convolve_2d_avx2.c:convolve_rounding Unexecuted instantiation: highbd_jnt_convolve_avx2.c:convolve_rounding |
839 | | |
840 | | static INLINE __m256i highbd_comp_avg(const __m256i *const data_ref_0, |
841 | | const __m256i *const res_unsigned, |
842 | | const __m256i *const wt0, |
843 | | const __m256i *const wt1, |
844 | 0 | const int use_wtd_comp_avg) { |
845 | 0 | __m256i res; |
846 | 0 | if (use_wtd_comp_avg) { |
847 | 0 | const __m256i wt0_res = _mm256_mullo_epi32(*data_ref_0, *wt0); |
848 | 0 | const __m256i wt1_res = _mm256_mullo_epi32(*res_unsigned, *wt1); |
849 | 0 | const __m256i wt_res = _mm256_add_epi32(wt0_res, wt1_res); |
850 | 0 | res = _mm256_srai_epi32(wt_res, DIST_PRECISION_BITS); |
851 | 0 | } else { |
852 | 0 | const __m256i wt_res = _mm256_add_epi32(*data_ref_0, *res_unsigned); |
853 | 0 | res = _mm256_srai_epi32(wt_res, 1); |
854 | 0 | } |
855 | 0 | return res; |
856 | 0 | } Unexecuted instantiation: highbd_convolve_avx2.c:highbd_comp_avg Unexecuted instantiation: highbd_convolve_2d_avx2.c:highbd_comp_avg Unexecuted instantiation: highbd_jnt_convolve_avx2.c:highbd_comp_avg |
857 | | |
858 | | static INLINE __m256i highbd_convolve_rounding( |
859 | | const __m256i *const res_unsigned, const __m256i *const offset_const, |
860 | 0 | const __m256i *const round_const, const int round_shift) { |
861 | 0 | const __m256i res_signed = _mm256_sub_epi32(*res_unsigned, *offset_const); |
862 | 0 | const __m256i res_round = _mm256_srai_epi32( |
863 | 0 | _mm256_add_epi32(res_signed, *round_const), round_shift); |
864 | |
|
865 | 0 | return res_round; |
866 | 0 | } Unexecuted instantiation: highbd_convolve_avx2.c:highbd_convolve_rounding Unexecuted instantiation: highbd_convolve_2d_avx2.c:highbd_convolve_rounding Unexecuted instantiation: highbd_jnt_convolve_avx2.c:highbd_convolve_rounding |
867 | | |
868 | | #endif // AVM_AVM_DSP_X86_CONVOLVE_AVX2_H_ |