/src/libvpx/vpx_dsp/x86/highbd_sad4d_avx2.c
Line | Count | Source |
1 | | /* |
2 | | * Copyright (c) 2022 The WebM project authors. All Rights Reserved. |
3 | | * |
4 | | * Use of this source code is governed by a BSD-style license |
5 | | * that can be found in the LICENSE file in the root of the source |
6 | | * tree. An additional intellectual property rights grant can be found |
7 | | * in the file PATENTS. All contributing project authors may |
8 | | * be found in the AUTHORS file in the root of the source tree. |
9 | | */ |
10 | | #include <immintrin.h> // AVX2 |
11 | | #include "./vpx_dsp_rtcd.h" |
12 | | #include "vpx/vpx_integer.h" |
13 | | |
14 | | static VPX_FORCE_INLINE void calc_final_4(const __m256i *const sums /*[4]*/, |
15 | 194k | uint32_t sad_array[4]) { |
16 | 194k | const __m256i t0 = _mm256_hadd_epi32(sums[0], sums[1]); |
17 | 194k | const __m256i t1 = _mm256_hadd_epi32(sums[2], sums[3]); |
18 | 194k | const __m256i t2 = _mm256_hadd_epi32(t0, t1); |
19 | 194k | const __m128i sum = _mm_add_epi32(_mm256_castsi256_si128(t2), |
20 | 194k | _mm256_extractf128_si256(t2, 1)); |
21 | 194k | _mm_storeu_si128((__m128i *)sad_array, sum); |
22 | 194k | } |
23 | | |
24 | | static VPX_FORCE_INLINE void highbd_sad64xHx4d(__m256i *sums_16 /*[4]*/, |
25 | | const uint16_t *src, |
26 | | int src_stride, |
27 | | uint16_t *refs[4], |
28 | 1.12M | int ref_stride, int height) { |
29 | 1.12M | int i; |
30 | 3.36M | for (i = 0; i < height; ++i) { |
31 | | // load src and all ref[] |
32 | 2.24M | const __m256i s0 = _mm256_load_si256((const __m256i *)src); |
33 | 2.24M | const __m256i s1 = _mm256_load_si256((const __m256i *)(src + 16)); |
34 | 2.24M | const __m256i s2 = _mm256_load_si256((const __m256i *)(src + 32)); |
35 | 2.24M | const __m256i s3 = _mm256_load_si256((const __m256i *)(src + 48)); |
36 | 2.24M | int x; |
37 | | |
38 | 11.2M | for (x = 0; x < 4; ++x) { |
39 | 8.96M | __m256i r[4]; |
40 | 8.96M | r[0] = _mm256_loadu_si256((const __m256i *)refs[x]); |
41 | 8.96M | r[1] = _mm256_loadu_si256((const __m256i *)(refs[x] + 16)); |
42 | 8.96M | r[2] = _mm256_loadu_si256((const __m256i *)(refs[x] + 32)); |
43 | 8.96M | r[3] = _mm256_loadu_si256((const __m256i *)(refs[x] + 48)); |
44 | | |
45 | | // absolute differences between every ref[] to src |
46 | 8.96M | r[0] = _mm256_abs_epi16(_mm256_sub_epi16(r[0], s0)); |
47 | 8.96M | r[1] = _mm256_abs_epi16(_mm256_sub_epi16(r[1], s1)); |
48 | 8.96M | r[2] = _mm256_abs_epi16(_mm256_sub_epi16(r[2], s2)); |
49 | 8.96M | r[3] = _mm256_abs_epi16(_mm256_sub_epi16(r[3], s3)); |
50 | | |
51 | | // sum every abs diff |
52 | 8.96M | sums_16[x] = _mm256_add_epi16(sums_16[x], _mm256_add_epi16(r[0], r[1])); |
53 | 8.96M | sums_16[x] = _mm256_add_epi16(sums_16[x], _mm256_add_epi16(r[2], r[3])); |
54 | 8.96M | } |
55 | | |
56 | 2.24M | src += src_stride; |
57 | 2.24M | refs[0] += ref_stride; |
58 | 2.24M | refs[1] += ref_stride; |
59 | 2.24M | refs[2] += ref_stride; |
60 | 2.24M | refs[3] += ref_stride; |
61 | 2.24M | } |
62 | 1.12M | } |
63 | | |
64 | | static VPX_FORCE_INLINE void highbd_sad64xNx4d_avx2( |
65 | | const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], |
66 | 50.3k | int ref_stride, uint32_t sad_array[4], int n) { |
67 | 50.3k | const uint16_t *src = CONVERT_TO_SHORTPTR(src_ptr); |
68 | 50.3k | uint16_t *refs[4]; |
69 | 50.3k | __m256i sums_16[4]; |
70 | 50.3k | __m256i sums_32[4]; |
71 | 50.3k | int i; |
72 | | |
73 | 50.3k | refs[0] = CONVERT_TO_SHORTPTR(ref_array[0]); |
74 | 50.3k | refs[1] = CONVERT_TO_SHORTPTR(ref_array[1]); |
75 | 50.3k | refs[2] = CONVERT_TO_SHORTPTR(ref_array[2]); |
76 | 50.3k | refs[3] = CONVERT_TO_SHORTPTR(ref_array[3]); |
77 | 50.3k | sums_32[0] = _mm256_setzero_si256(); |
78 | 50.3k | sums_32[1] = _mm256_setzero_si256(); |
79 | 50.3k | sums_32[2] = _mm256_setzero_si256(); |
80 | 50.3k | sums_32[3] = _mm256_setzero_si256(); |
81 | | |
82 | 1.17M | for (i = 0; i < (n / 2); ++i) { |
83 | 1.12M | sums_16[0] = _mm256_setzero_si256(); |
84 | 1.12M | sums_16[1] = _mm256_setzero_si256(); |
85 | 1.12M | sums_16[2] = _mm256_setzero_si256(); |
86 | 1.12M | sums_16[3] = _mm256_setzero_si256(); |
87 | | |
88 | 1.12M | highbd_sad64xHx4d(sums_16, src, src_stride, refs, ref_stride, 2); |
89 | | |
90 | | /* sums_16 will outrange after 2 rows, so add current sums_16 to |
91 | | * sums_32*/ |
92 | 1.12M | sums_32[0] = _mm256_add_epi32( |
93 | 1.12M | sums_32[0], |
94 | 1.12M | _mm256_add_epi32( |
95 | 1.12M | _mm256_cvtepu16_epi32(_mm256_castsi256_si128(sums_16[0])), |
96 | 1.12M | _mm256_cvtepu16_epi32(_mm256_extractf128_si256(sums_16[0], 1)))); |
97 | 1.12M | sums_32[1] = _mm256_add_epi32( |
98 | 1.12M | sums_32[1], |
99 | 1.12M | _mm256_add_epi32( |
100 | 1.12M | _mm256_cvtepu16_epi32(_mm256_castsi256_si128(sums_16[1])), |
101 | 1.12M | _mm256_cvtepu16_epi32(_mm256_extractf128_si256(sums_16[1], 1)))); |
102 | 1.12M | sums_32[2] = _mm256_add_epi32( |
103 | 1.12M | sums_32[2], |
104 | 1.12M | _mm256_add_epi32( |
105 | 1.12M | _mm256_cvtepu16_epi32(_mm256_castsi256_si128(sums_16[2])), |
106 | 1.12M | _mm256_cvtepu16_epi32(_mm256_extractf128_si256(sums_16[2], 1)))); |
107 | 1.12M | sums_32[3] = _mm256_add_epi32( |
108 | 1.12M | sums_32[3], |
109 | 1.12M | _mm256_add_epi32( |
110 | 1.12M | _mm256_cvtepu16_epi32(_mm256_castsi256_si128(sums_16[3])), |
111 | 1.12M | _mm256_cvtepu16_epi32(_mm256_extractf128_si256(sums_16[3], 1)))); |
112 | | |
113 | 1.12M | src += src_stride << 1; |
114 | 1.12M | } |
115 | 50.3k | calc_final_4(sums_32, sad_array); |
116 | 50.3k | } |
117 | | |
118 | | #define HIGHBD_SAD64XNX4D(n) \ |
119 | | void vpx_highbd_sad64x##n##x4d_avx2(const uint8_t *src, int src_stride, \ |
120 | | const uint8_t *const ref_array[4], \ |
121 | 26.2k | int ref_stride, uint32_t sad_array[4]) { \ |
122 | 26.2k | highbd_sad64xNx4d_avx2(src, src_stride, ref_array, ref_stride, sad_array, \ |
123 | 26.2k | n); \ |
124 | 26.2k | } vpx_highbd_sad64x64x4d_avx2 Line | Count | Source | 121 | 20.2k | int ref_stride, uint32_t sad_array[4]) { \ | 122 | 20.2k | highbd_sad64xNx4d_avx2(src, src_stride, ref_array, ref_stride, sad_array, \ | 123 | 20.2k | n); \ | 124 | 20.2k | } |
vpx_highbd_sad64x32x4d_avx2 Line | Count | Source | 121 | 6.05k | int ref_stride, uint32_t sad_array[4]) { \ | 122 | 6.05k | highbd_sad64xNx4d_avx2(src, src_stride, ref_array, ref_stride, sad_array, \ | 123 | 6.05k | n); \ | 124 | 6.05k | } |
|
125 | | |
126 | | #define HIGHBD_SADSKIP64XNx4D(n) \ |
127 | | void vpx_highbd_sad_skip_64x##n##x4d_avx2( \ |
128 | | const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \ |
129 | 24.1k | int ref_stride, uint32_t sad_array[4]) { \ |
130 | 24.1k | highbd_sad64xNx4d_avx2(src, 2 * src_stride, ref_array, 2 * ref_stride, \ |
131 | 24.1k | sad_array, n / 2); \ |
132 | 24.1k | sad_array[0] <<= 1; \ |
133 | 24.1k | sad_array[1] <<= 1; \ |
134 | 24.1k | sad_array[2] <<= 1; \ |
135 | 24.1k | sad_array[3] <<= 1; \ |
136 | 24.1k | } vpx_highbd_sad_skip_64x64x4d_avx2 Line | Count | Source | 129 | 23.0k | int ref_stride, uint32_t sad_array[4]) { \ | 130 | 23.0k | highbd_sad64xNx4d_avx2(src, 2 * src_stride, ref_array, 2 * ref_stride, \ | 131 | 23.0k | sad_array, n / 2); \ | 132 | 23.0k | sad_array[0] <<= 1; \ | 133 | 23.0k | sad_array[1] <<= 1; \ | 134 | 23.0k | sad_array[2] <<= 1; \ | 135 | 23.0k | sad_array[3] <<= 1; \ | 136 | 23.0k | } |
vpx_highbd_sad_skip_64x32x4d_avx2 Line | Count | Source | 129 | 1.02k | int ref_stride, uint32_t sad_array[4]) { \ | 130 | 1.02k | highbd_sad64xNx4d_avx2(src, 2 * src_stride, ref_array, 2 * ref_stride, \ | 131 | 1.02k | sad_array, n / 2); \ | 132 | 1.02k | sad_array[0] <<= 1; \ | 133 | 1.02k | sad_array[1] <<= 1; \ | 134 | 1.02k | sad_array[2] <<= 1; \ | 135 | 1.02k | sad_array[3] <<= 1; \ | 136 | 1.02k | } |
|
137 | | |
138 | | static VPX_FORCE_INLINE void highbd_sad32xHx4d(__m256i *sums_16 /*[4]*/, |
139 | | const uint16_t *src, |
140 | | int src_stride, |
141 | | uint16_t *refs[4], |
142 | 170k | int ref_stride, int height) { |
143 | 170k | int i; |
144 | 1.53M | for (i = 0; i < height; i++) { |
145 | 1.36M | __m256i r[8]; |
146 | | |
147 | | // load src and all ref[] |
148 | 1.36M | const __m256i s = _mm256_load_si256((const __m256i *)src); |
149 | 1.36M | const __m256i s2 = _mm256_load_si256((const __m256i *)(src + 16)); |
150 | 1.36M | r[0] = _mm256_loadu_si256((const __m256i *)refs[0]); |
151 | 1.36M | r[1] = _mm256_loadu_si256((const __m256i *)(refs[0] + 16)); |
152 | 1.36M | r[2] = _mm256_loadu_si256((const __m256i *)refs[1]); |
153 | 1.36M | r[3] = _mm256_loadu_si256((const __m256i *)(refs[1] + 16)); |
154 | 1.36M | r[4] = _mm256_loadu_si256((const __m256i *)refs[2]); |
155 | 1.36M | r[5] = _mm256_loadu_si256((const __m256i *)(refs[2] + 16)); |
156 | 1.36M | r[6] = _mm256_loadu_si256((const __m256i *)refs[3]); |
157 | 1.36M | r[7] = _mm256_loadu_si256((const __m256i *)(refs[3] + 16)); |
158 | | |
159 | | // absolute differences between every ref[] to src |
160 | 1.36M | r[0] = _mm256_abs_epi16(_mm256_sub_epi16(r[0], s)); |
161 | 1.36M | r[1] = _mm256_abs_epi16(_mm256_sub_epi16(r[1], s2)); |
162 | 1.36M | r[2] = _mm256_abs_epi16(_mm256_sub_epi16(r[2], s)); |
163 | 1.36M | r[3] = _mm256_abs_epi16(_mm256_sub_epi16(r[3], s2)); |
164 | 1.36M | r[4] = _mm256_abs_epi16(_mm256_sub_epi16(r[4], s)); |
165 | 1.36M | r[5] = _mm256_abs_epi16(_mm256_sub_epi16(r[5], s2)); |
166 | 1.36M | r[6] = _mm256_abs_epi16(_mm256_sub_epi16(r[6], s)); |
167 | 1.36M | r[7] = _mm256_abs_epi16(_mm256_sub_epi16(r[7], s2)); |
168 | | |
169 | | // sum every abs diff |
170 | 1.36M | sums_16[0] = _mm256_add_epi16(sums_16[0], _mm256_add_epi16(r[0], r[1])); |
171 | 1.36M | sums_16[1] = _mm256_add_epi16(sums_16[1], _mm256_add_epi16(r[2], r[3])); |
172 | 1.36M | sums_16[2] = _mm256_add_epi16(sums_16[2], _mm256_add_epi16(r[4], r[5])); |
173 | 1.36M | sums_16[3] = _mm256_add_epi16(sums_16[3], _mm256_add_epi16(r[6], r[7])); |
174 | | |
175 | 1.36M | src += src_stride; |
176 | 1.36M | refs[0] += ref_stride; |
177 | 1.36M | refs[1] += ref_stride; |
178 | 1.36M | refs[2] += ref_stride; |
179 | 1.36M | refs[3] += ref_stride; |
180 | 1.36M | } |
181 | 170k | } |
182 | | |
183 | | static VPX_FORCE_INLINE void highbd_sad32xNx4d_avx2( |
184 | | const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], |
185 | 54.3k | int ref_stride, uint32_t sad_array[4], int n) { |
186 | 54.3k | const uint16_t *src = CONVERT_TO_SHORTPTR(src_ptr); |
187 | 54.3k | uint16_t *refs[4]; |
188 | 54.3k | __m256i sums_16[4]; |
189 | 54.3k | __m256i sums_32[4]; |
190 | 54.3k | int i; |
191 | | |
192 | 54.3k | refs[0] = CONVERT_TO_SHORTPTR(ref_array[0]); |
193 | 54.3k | refs[1] = CONVERT_TO_SHORTPTR(ref_array[1]); |
194 | 54.3k | refs[2] = CONVERT_TO_SHORTPTR(ref_array[2]); |
195 | 54.3k | refs[3] = CONVERT_TO_SHORTPTR(ref_array[3]); |
196 | 54.3k | sums_32[0] = _mm256_setzero_si256(); |
197 | 54.3k | sums_32[1] = _mm256_setzero_si256(); |
198 | 54.3k | sums_32[2] = _mm256_setzero_si256(); |
199 | 54.3k | sums_32[3] = _mm256_setzero_si256(); |
200 | | |
201 | 224k | for (i = 0; i < (n / 8); ++i) { |
202 | 170k | sums_16[0] = _mm256_setzero_si256(); |
203 | 170k | sums_16[1] = _mm256_setzero_si256(); |
204 | 170k | sums_16[2] = _mm256_setzero_si256(); |
205 | 170k | sums_16[3] = _mm256_setzero_si256(); |
206 | | |
207 | 170k | highbd_sad32xHx4d(sums_16, src, src_stride, refs, ref_stride, 8); |
208 | | |
209 | | /* sums_16 will outrange after 8 rows, so add current sums_16 to |
210 | | * sums_32*/ |
211 | 170k | sums_32[0] = _mm256_add_epi32( |
212 | 170k | sums_32[0], |
213 | 170k | _mm256_add_epi32( |
214 | 170k | _mm256_cvtepu16_epi32(_mm256_castsi256_si128(sums_16[0])), |
215 | 170k | _mm256_cvtepu16_epi32(_mm256_extractf128_si256(sums_16[0], 1)))); |
216 | 170k | sums_32[1] = _mm256_add_epi32( |
217 | 170k | sums_32[1], |
218 | 170k | _mm256_add_epi32( |
219 | 170k | _mm256_cvtepu16_epi32(_mm256_castsi256_si128(sums_16[1])), |
220 | 170k | _mm256_cvtepu16_epi32(_mm256_extractf128_si256(sums_16[1], 1)))); |
221 | 170k | sums_32[2] = _mm256_add_epi32( |
222 | 170k | sums_32[2], |
223 | 170k | _mm256_add_epi32( |
224 | 170k | _mm256_cvtepu16_epi32(_mm256_castsi256_si128(sums_16[2])), |
225 | 170k | _mm256_cvtepu16_epi32(_mm256_extractf128_si256(sums_16[2], 1)))); |
226 | 170k | sums_32[3] = _mm256_add_epi32( |
227 | 170k | sums_32[3], |
228 | 170k | _mm256_add_epi32( |
229 | 170k | _mm256_cvtepu16_epi32(_mm256_castsi256_si128(sums_16[3])), |
230 | 170k | _mm256_cvtepu16_epi32(_mm256_extractf128_si256(sums_16[3], 1)))); |
231 | | |
232 | 170k | src += src_stride << 3; |
233 | 170k | } |
234 | 54.3k | calc_final_4(sums_32, sad_array); |
235 | 54.3k | } |
236 | | |
237 | | #define HIGHBD_SAD32XNX4D(n) \ |
238 | | void vpx_highbd_sad32x##n##x4d_avx2(const uint8_t *src, int src_stride, \ |
239 | | const uint8_t *const ref_array[4], \ |
240 | 34.1k | int ref_stride, uint32_t sad_array[4]) { \ |
241 | 34.1k | highbd_sad32xNx4d_avx2(src, src_stride, ref_array, ref_stride, sad_array, \ |
242 | 34.1k | n); \ |
243 | 34.1k | } vpx_highbd_sad32x64x4d_avx2 Line | Count | Source | 240 | 2.94k | int ref_stride, uint32_t sad_array[4]) { \ | 241 | 2.94k | highbd_sad32xNx4d_avx2(src, src_stride, ref_array, ref_stride, sad_array, \ | 242 | 2.94k | n); \ | 243 | 2.94k | } |
vpx_highbd_sad32x32x4d_avx2 Line | Count | Source | 240 | 23.3k | int ref_stride, uint32_t sad_array[4]) { \ | 241 | 23.3k | highbd_sad32xNx4d_avx2(src, src_stride, ref_array, ref_stride, sad_array, \ | 242 | 23.3k | n); \ | 243 | 23.3k | } |
vpx_highbd_sad32x16x4d_avx2 Line | Count | Source | 240 | 7.87k | int ref_stride, uint32_t sad_array[4]) { \ | 241 | 7.87k | highbd_sad32xNx4d_avx2(src, src_stride, ref_array, ref_stride, sad_array, \ | 242 | 7.87k | n); \ | 243 | 7.87k | } |
|
244 | | |
245 | | #define HIGHBD_SADSKIP32XNx4D(n) \ |
246 | | void vpx_highbd_sad_skip_32x##n##x4d_avx2( \ |
247 | | const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \ |
248 | 20.2k | int ref_stride, uint32_t sad_array[4]) { \ |
249 | 20.2k | highbd_sad32xNx4d_avx2(src, 2 * src_stride, ref_array, 2 * ref_stride, \ |
250 | 20.2k | sad_array, n / 2); \ |
251 | 20.2k | sad_array[0] <<= 1; \ |
252 | 20.2k | sad_array[1] <<= 1; \ |
253 | 20.2k | sad_array[2] <<= 1; \ |
254 | 20.2k | sad_array[3] <<= 1; \ |
255 | 20.2k | } Unexecuted instantiation: vpx_highbd_sad_skip_32x64x4d_avx2 vpx_highbd_sad_skip_32x32x4d_avx2 Line | Count | Source | 248 | 17.7k | int ref_stride, uint32_t sad_array[4]) { \ | 249 | 17.7k | highbd_sad32xNx4d_avx2(src, 2 * src_stride, ref_array, 2 * ref_stride, \ | 250 | 17.7k | sad_array, n / 2); \ | 251 | 17.7k | sad_array[0] <<= 1; \ | 252 | 17.7k | sad_array[1] <<= 1; \ | 253 | 17.7k | sad_array[2] <<= 1; \ | 254 | 17.7k | sad_array[3] <<= 1; \ | 255 | 17.7k | } |
vpx_highbd_sad_skip_32x16x4d_avx2 Line | Count | Source | 248 | 2.52k | int ref_stride, uint32_t sad_array[4]) { \ | 249 | 2.52k | highbd_sad32xNx4d_avx2(src, 2 * src_stride, ref_array, 2 * ref_stride, \ | 250 | 2.52k | sad_array, n / 2); \ | 251 | 2.52k | sad_array[0] <<= 1; \ | 252 | 2.52k | sad_array[1] <<= 1; \ | 253 | 2.52k | sad_array[2] <<= 1; \ | 254 | 2.52k | sad_array[3] <<= 1; \ | 255 | 2.52k | } |
|
256 | | |
257 | | static VPX_FORCE_INLINE void highbd_sad16xHx4d(__m256i *sums_16 /*[4]*/, |
258 | | const uint16_t *src, |
259 | | int src_stride, |
260 | | uint16_t *refs[4], |
261 | 93.3k | int ref_stride, int height) { |
262 | 93.3k | int i; |
263 | 1.27M | for (i = 0; i < height; i++) { |
264 | 1.18M | __m256i r[4]; |
265 | | |
266 | | // load src and all ref[] |
267 | 1.18M | const __m256i s = _mm256_load_si256((const __m256i *)src); |
268 | 1.18M | r[0] = _mm256_loadu_si256((const __m256i *)refs[0]); |
269 | 1.18M | r[1] = _mm256_loadu_si256((const __m256i *)refs[1]); |
270 | 1.18M | r[2] = _mm256_loadu_si256((const __m256i *)refs[2]); |
271 | 1.18M | r[3] = _mm256_loadu_si256((const __m256i *)refs[3]); |
272 | | |
273 | | // absolute differences between every ref[] to src |
274 | 1.18M | r[0] = _mm256_abs_epi16(_mm256_sub_epi16(r[0], s)); |
275 | 1.18M | r[1] = _mm256_abs_epi16(_mm256_sub_epi16(r[1], s)); |
276 | 1.18M | r[2] = _mm256_abs_epi16(_mm256_sub_epi16(r[2], s)); |
277 | 1.18M | r[3] = _mm256_abs_epi16(_mm256_sub_epi16(r[3], s)); |
278 | | |
279 | | // sum every abs diff |
280 | 1.18M | sums_16[0] = _mm256_add_epi16(sums_16[0], r[0]); |
281 | 1.18M | sums_16[1] = _mm256_add_epi16(sums_16[1], r[1]); |
282 | 1.18M | sums_16[2] = _mm256_add_epi16(sums_16[2], r[2]); |
283 | 1.18M | sums_16[3] = _mm256_add_epi16(sums_16[3], r[3]); |
284 | | |
285 | 1.18M | src += src_stride; |
286 | 1.18M | refs[0] += ref_stride; |
287 | 1.18M | refs[1] += ref_stride; |
288 | 1.18M | refs[2] += ref_stride; |
289 | 1.18M | refs[3] += ref_stride; |
290 | 1.18M | } |
291 | 93.3k | } |
292 | | |
293 | | static VPX_FORCE_INLINE void highbd_sad16xNx4d_avx2( |
294 | | const uint8_t *src_ptr, int src_stride, const uint8_t *const ref_array[4], |
295 | 28.0k | int ref_stride, uint32_t sad_array[4], int n) { |
296 | 28.0k | const uint16_t *src = CONVERT_TO_SHORTPTR(src_ptr); |
297 | 28.0k | uint16_t *refs[4]; |
298 | 28.0k | __m256i sums_16[4]; |
299 | 28.0k | __m256i sums_32[4]; |
300 | 28.0k | const int height = VPXMIN(16, n); |
301 | 28.0k | const int num_iters = n / height; |
302 | 28.0k | int i; |
303 | | |
304 | 28.0k | refs[0] = CONVERT_TO_SHORTPTR(ref_array[0]); |
305 | 28.0k | refs[1] = CONVERT_TO_SHORTPTR(ref_array[1]); |
306 | 28.0k | refs[2] = CONVERT_TO_SHORTPTR(ref_array[2]); |
307 | 28.0k | refs[3] = CONVERT_TO_SHORTPTR(ref_array[3]); |
308 | 28.0k | sums_32[0] = _mm256_setzero_si256(); |
309 | 28.0k | sums_32[1] = _mm256_setzero_si256(); |
310 | 28.0k | sums_32[2] = _mm256_setzero_si256(); |
311 | 28.0k | sums_32[3] = _mm256_setzero_si256(); |
312 | | |
313 | 59.9k | for (i = 0; i < num_iters; ++i) { |
314 | 31.8k | sums_16[0] = _mm256_setzero_si256(); |
315 | 31.8k | sums_16[1] = _mm256_setzero_si256(); |
316 | 31.8k | sums_16[2] = _mm256_setzero_si256(); |
317 | 31.8k | sums_16[3] = _mm256_setzero_si256(); |
318 | | |
319 | 31.8k | highbd_sad16xHx4d(sums_16, src, src_stride, refs, ref_stride, height); |
320 | | |
321 | | // sums_16 will outrange after 16 rows, so add current sums_16 to sums_32 |
322 | 31.8k | sums_32[0] = _mm256_add_epi32( |
323 | 31.8k | sums_32[0], |
324 | 31.8k | _mm256_add_epi32( |
325 | 31.8k | _mm256_cvtepu16_epi32(_mm256_castsi256_si128(sums_16[0])), |
326 | 31.8k | _mm256_cvtepu16_epi32(_mm256_extractf128_si256(sums_16[0], 1)))); |
327 | 31.8k | sums_32[1] = _mm256_add_epi32( |
328 | 31.8k | sums_32[1], |
329 | 31.8k | _mm256_add_epi32( |
330 | 31.8k | _mm256_cvtepu16_epi32(_mm256_castsi256_si128(sums_16[1])), |
331 | 31.8k | _mm256_cvtepu16_epi32(_mm256_extractf128_si256(sums_16[1], 1)))); |
332 | 31.8k | sums_32[2] = _mm256_add_epi32( |
333 | 31.8k | sums_32[2], |
334 | 31.8k | _mm256_add_epi32( |
335 | 31.8k | _mm256_cvtepu16_epi32(_mm256_castsi256_si128(sums_16[2])), |
336 | 31.8k | _mm256_cvtepu16_epi32(_mm256_extractf128_si256(sums_16[2], 1)))); |
337 | 31.8k | sums_32[3] = _mm256_add_epi32( |
338 | 31.8k | sums_32[3], |
339 | 31.8k | _mm256_add_epi32( |
340 | 31.8k | _mm256_cvtepu16_epi32(_mm256_castsi256_si128(sums_16[3])), |
341 | 31.8k | _mm256_cvtepu16_epi32(_mm256_extractf128_si256(sums_16[3], 1)))); |
342 | | |
343 | 31.8k | src += src_stride << 4; |
344 | 31.8k | } |
345 | 28.0k | calc_final_4(sums_32, sad_array); |
346 | 28.0k | } |
347 | | |
348 | | #define HIGHBD_SAD16XNX4D(n) \ |
349 | | void vpx_highbd_sad16x##n##x4d_avx2(const uint8_t *src, int src_stride, \ |
350 | | const uint8_t *const ref_array[4], \ |
351 | 3.82k | int ref_stride, uint32_t sad_array[4]) { \ |
352 | 3.82k | highbd_sad16xNx4d_avx2(src, src_stride, ref_array, ref_stride, sad_array, \ |
353 | 3.82k | n); \ |
354 | 3.82k | } |
355 | | |
356 | | #define HIGHBD_SADSKIP16XNx4D(n) \ |
357 | | void vpx_highbd_sad_skip_16x##n##x4d_avx2( \ |
358 | | const uint8_t *src, int src_stride, const uint8_t *const ref_array[4], \ |
359 | 24.2k | int ref_stride, uint32_t sad_array[4]) { \ |
360 | 24.2k | highbd_sad16xNx4d_avx2(src, 2 * src_stride, ref_array, 2 * ref_stride, \ |
361 | 24.2k | sad_array, n / 2); \ |
362 | 24.2k | sad_array[0] <<= 1; \ |
363 | 24.2k | sad_array[1] <<= 1; \ |
364 | 24.2k | sad_array[2] <<= 1; \ |
365 | 24.2k | sad_array[3] <<= 1; \ |
366 | 24.2k | } vpx_highbd_sad_skip_16x32x4d_avx2 Line | Count | Source | 359 | 107 | int ref_stride, uint32_t sad_array[4]) { \ | 360 | 107 | highbd_sad16xNx4d_avx2(src, 2 * src_stride, ref_array, 2 * ref_stride, \ | 361 | 107 | sad_array, n / 2); \ | 362 | 107 | sad_array[0] <<= 1; \ | 363 | 107 | sad_array[1] <<= 1; \ | 364 | 107 | sad_array[2] <<= 1; \ | 365 | 107 | sad_array[3] <<= 1; \ | 366 | 107 | } |
vpx_highbd_sad_skip_16x16x4d_avx2 Line | Count | Source | 359 | 20.0k | int ref_stride, uint32_t sad_array[4]) { \ | 360 | 20.0k | highbd_sad16xNx4d_avx2(src, 2 * src_stride, ref_array, 2 * ref_stride, \ | 361 | 20.0k | sad_array, n / 2); \ | 362 | 20.0k | sad_array[0] <<= 1; \ | 363 | 20.0k | sad_array[1] <<= 1; \ | 364 | 20.0k | sad_array[2] <<= 1; \ | 365 | 20.0k | sad_array[3] <<= 1; \ | 366 | 20.0k | } |
vpx_highbd_sad_skip_16x8x4d_avx2 Line | Count | Source | 359 | 4.06k | int ref_stride, uint32_t sad_array[4]) { \ | 360 | 4.06k | highbd_sad16xNx4d_avx2(src, 2 * src_stride, ref_array, 2 * ref_stride, \ | 361 | 4.06k | sad_array, n / 2); \ | 362 | 4.06k | sad_array[0] <<= 1; \ | 363 | 4.06k | sad_array[1] <<= 1; \ | 364 | 4.06k | sad_array[2] <<= 1; \ | 365 | 4.06k | sad_array[3] <<= 1; \ | 366 | 4.06k | } |
|
367 | | |
368 | | void vpx_highbd_sad16x16x4d_avx2(const uint8_t *src_ptr, int src_stride, |
369 | | const uint8_t *const ref_array[4], |
370 | 49.1k | int ref_stride, uint32_t sad_array[4]) { |
371 | 49.1k | const uint16_t *src = CONVERT_TO_SHORTPTR(src_ptr); |
372 | 49.1k | uint16_t *refs[4]; |
373 | 49.1k | __m256i sums_16[4]; |
374 | | |
375 | 49.1k | refs[0] = CONVERT_TO_SHORTPTR(ref_array[0]); |
376 | 49.1k | refs[1] = CONVERT_TO_SHORTPTR(ref_array[1]); |
377 | 49.1k | refs[2] = CONVERT_TO_SHORTPTR(ref_array[2]); |
378 | 49.1k | refs[3] = CONVERT_TO_SHORTPTR(ref_array[3]); |
379 | 49.1k | sums_16[0] = _mm256_setzero_si256(); |
380 | 49.1k | sums_16[1] = _mm256_setzero_si256(); |
381 | 49.1k | sums_16[2] = _mm256_setzero_si256(); |
382 | 49.1k | sums_16[3] = _mm256_setzero_si256(); |
383 | | |
384 | 49.1k | highbd_sad16xHx4d(sums_16, src, src_stride, refs, ref_stride, 16); |
385 | | |
386 | 49.1k | { |
387 | 49.1k | __m256i sums_32[4]; |
388 | 49.1k | sums_32[0] = _mm256_add_epi32( |
389 | 49.1k | _mm256_cvtepu16_epi32(_mm256_castsi256_si128(sums_16[0])), |
390 | 49.1k | _mm256_cvtepu16_epi32(_mm256_extractf128_si256(sums_16[0], 1))); |
391 | 49.1k | sums_32[1] = _mm256_add_epi32( |
392 | 49.1k | _mm256_cvtepu16_epi32(_mm256_castsi256_si128(sums_16[1])), |
393 | 49.1k | _mm256_cvtepu16_epi32(_mm256_extractf128_si256(sums_16[1], 1))); |
394 | 49.1k | sums_32[2] = _mm256_add_epi32( |
395 | 49.1k | _mm256_cvtepu16_epi32(_mm256_castsi256_si128(sums_16[2])), |
396 | 49.1k | _mm256_cvtepu16_epi32(_mm256_extractf128_si256(sums_16[2], 1))); |
397 | 49.1k | sums_32[3] = _mm256_add_epi32( |
398 | 49.1k | _mm256_cvtepu16_epi32(_mm256_castsi256_si128(sums_16[3])), |
399 | 49.1k | _mm256_cvtepu16_epi32(_mm256_extractf128_si256(sums_16[3], 1))); |
400 | 49.1k | calc_final_4(sums_32, sad_array); |
401 | 49.1k | } |
402 | 49.1k | } |
403 | | |
404 | | void vpx_highbd_sad16x8x4d_avx2(const uint8_t *src_ptr, int src_stride, |
405 | | const uint8_t *const ref_array[4], |
406 | 12.3k | int ref_stride, uint32_t sad_array[4]) { |
407 | 12.3k | const uint16_t *src = CONVERT_TO_SHORTPTR(src_ptr); |
408 | 12.3k | uint16_t *refs[4]; |
409 | 12.3k | __m256i sums_16[4]; |
410 | | |
411 | 12.3k | refs[0] = CONVERT_TO_SHORTPTR(ref_array[0]); |
412 | 12.3k | refs[1] = CONVERT_TO_SHORTPTR(ref_array[1]); |
413 | 12.3k | refs[2] = CONVERT_TO_SHORTPTR(ref_array[2]); |
414 | 12.3k | refs[3] = CONVERT_TO_SHORTPTR(ref_array[3]); |
415 | 12.3k | sums_16[0] = _mm256_setzero_si256(); |
416 | 12.3k | sums_16[1] = _mm256_setzero_si256(); |
417 | 12.3k | sums_16[2] = _mm256_setzero_si256(); |
418 | 12.3k | sums_16[3] = _mm256_setzero_si256(); |
419 | | |
420 | 12.3k | highbd_sad16xHx4d(sums_16, src, src_stride, refs, ref_stride, 8); |
421 | | |
422 | 12.3k | { |
423 | 12.3k | __m256i sums_32[4]; |
424 | 12.3k | sums_32[0] = _mm256_add_epi32( |
425 | 12.3k | _mm256_cvtepu16_epi32(_mm256_castsi256_si128(sums_16[0])), |
426 | 12.3k | _mm256_cvtepu16_epi32(_mm256_extractf128_si256(sums_16[0], 1))); |
427 | 12.3k | sums_32[1] = _mm256_add_epi32( |
428 | 12.3k | _mm256_cvtepu16_epi32(_mm256_castsi256_si128(sums_16[1])), |
429 | 12.3k | _mm256_cvtepu16_epi32(_mm256_extractf128_si256(sums_16[1], 1))); |
430 | 12.3k | sums_32[2] = _mm256_add_epi32( |
431 | 12.3k | _mm256_cvtepu16_epi32(_mm256_castsi256_si128(sums_16[2])), |
432 | 12.3k | _mm256_cvtepu16_epi32(_mm256_extractf128_si256(sums_16[2], 1))); |
433 | 12.3k | sums_32[3] = _mm256_add_epi32( |
434 | 12.3k | _mm256_cvtepu16_epi32(_mm256_castsi256_si128(sums_16[3])), |
435 | | _mm256_cvtepu16_epi32(_mm256_extractf128_si256(sums_16[3], 1))); |
436 | 12.3k | calc_final_4(sums_32, sad_array); |
437 | 12.3k | } |
438 | 12.3k | } |
439 | | |
440 | | // clang-format off |
441 | | HIGHBD_SAD64XNX4D(64) |
442 | | HIGHBD_SADSKIP64XNx4D(64) |
443 | | |
444 | | HIGHBD_SAD64XNX4D(32) |
445 | | HIGHBD_SADSKIP64XNx4D(32) |
446 | | |
447 | | HIGHBD_SAD32XNX4D(64) |
448 | | HIGHBD_SADSKIP32XNx4D(64) |
449 | | |
450 | | HIGHBD_SAD32XNX4D(32) |
451 | | HIGHBD_SADSKIP32XNx4D(32) |
452 | | |
453 | | HIGHBD_SAD32XNX4D(16) |
454 | | HIGHBD_SADSKIP32XNx4D(16) |
455 | | |
456 | | HIGHBD_SAD16XNX4D(32) |
457 | | HIGHBD_SADSKIP16XNx4D(32) |
458 | | |
459 | | HIGHBD_SADSKIP16XNx4D(16) |
460 | | |
461 | | HIGHBD_SADSKIP16XNx4D(8) |
462 | | // clang-format on |