/src/aom/aom_dsp/x86/sad_avx2.c
Line | Count | Source |
1 | | /* |
2 | | * Copyright (c) 2016, Alliance for Open Media. All rights reserved. |
3 | | * |
4 | | * This source code is subject to the terms of the BSD 2 Clause License and |
5 | | * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
6 | | * was not distributed with this source code in the LICENSE file, you can |
7 | | * obtain it at www.aomedia.org/license/software. If the Alliance for Open |
8 | | * Media Patent License 1.0 was not distributed with this source code in the |
9 | | * PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
10 | | */ |
11 | | #include <immintrin.h> |
12 | | #include <stdint.h> |
13 | | |
14 | | #include "config/aom_dsp_rtcd.h" |
15 | | |
16 | | #include "aom_ports/mem.h" |
17 | | |
18 | | // SAD, SAD_SKIP and SAD_AVG for 64xh blocks |
19 | | #if !CONFIG_HIGHWAY |
20 | | static inline unsigned int sad64xh_avx2(const uint8_t *src_ptr, int src_stride, |
21 | | const uint8_t *ref_ptr, int ref_stride, |
22 | 0 | int h) { |
23 | 0 | int i; |
24 | 0 | __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg; |
25 | 0 | __m256i sum_sad = _mm256_setzero_si256(); |
26 | 0 | __m256i sum_sad_h; |
27 | 0 | __m128i sum_sad128; |
28 | 0 | for (i = 0; i < h; i++) { |
29 | 0 | ref1_reg = _mm256_loadu_si256((__m256i const *)ref_ptr); |
30 | 0 | ref2_reg = _mm256_loadu_si256((__m256i const *)(ref_ptr + 32)); |
31 | 0 | sad1_reg = |
32 | 0 | _mm256_sad_epu8(ref1_reg, _mm256_loadu_si256((__m256i const *)src_ptr)); |
33 | 0 | sad2_reg = _mm256_sad_epu8( |
34 | 0 | ref2_reg, _mm256_loadu_si256((__m256i const *)(src_ptr + 32))); |
35 | 0 | sum_sad = _mm256_add_epi32(sum_sad, _mm256_add_epi32(sad1_reg, sad2_reg)); |
36 | 0 | ref_ptr += ref_stride; |
37 | 0 | src_ptr += src_stride; |
38 | 0 | } |
39 | 0 | sum_sad_h = _mm256_srli_si256(sum_sad, 8); |
40 | 0 | sum_sad = _mm256_add_epi32(sum_sad, sum_sad_h); |
41 | 0 | sum_sad128 = _mm256_extracti128_si256(sum_sad, 1); |
42 | 0 | sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128); |
43 | 0 | unsigned int res = (unsigned int)_mm_cvtsi128_si32(sum_sad128); |
44 | 0 | _mm256_zeroupper(); |
45 | 0 | return res; |
46 | 0 | } |
47 | | |
48 | | #define FSAD64_H(h) \ |
49 | | unsigned int aom_sad64x##h##_avx2(const uint8_t *src_ptr, int src_stride, \ |
50 | 0 | const uint8_t *ref_ptr, int ref_stride) { \ |
51 | 0 | return sad64xh_avx2(src_ptr, src_stride, ref_ptr, ref_stride, h); \ |
52 | 0 | } Unexecuted instantiation: aom_sad64x64_avx2 Unexecuted instantiation: aom_sad64x32_avx2 |
53 | | |
54 | | #define FSADS64_H(h) \ |
55 | | unsigned int aom_sad_skip_64x##h##_avx2( \ |
56 | | const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ |
57 | 0 | int ref_stride) { \ |
58 | 0 | return 2 * sad64xh_avx2(src_ptr, src_stride * 2, ref_ptr, ref_stride * 2, \ |
59 | 0 | h / 2); \ |
60 | 0 | } Unexecuted instantiation: aom_sad_skip_64x64_avx2 Unexecuted instantiation: aom_sad_skip_64x32_avx2 |
61 | | |
62 | | #define FSAD64 \ |
63 | | FSAD64_H(64) \ |
64 | | FSAD64_H(32) \ |
65 | | FSADS64_H(64) \ |
66 | | FSADS64_H(32) |
67 | | |
68 | | /* clang-format off */ |
69 | | FSAD64 |
70 | | /* clang-format on */ |
71 | | |
72 | | #undef FSAD64 |
73 | | #undef FSAD64_H |
74 | | |
75 | | #define FSADAVG64_H(h) \ |
76 | | unsigned int aom_sad64x##h##_avg_avx2( \ |
77 | | const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ |
78 | 0 | int ref_stride, const uint8_t *second_pred) { \ |
79 | 0 | int i; \ |
80 | 0 | __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg; \ |
81 | 0 | __m256i sum_sad = _mm256_setzero_si256(); \ |
82 | 0 | __m256i sum_sad_h; \ |
83 | 0 | __m128i sum_sad128; \ |
84 | 0 | for (i = 0; i < h; i++) { \ |
85 | 0 | ref1_reg = _mm256_loadu_si256((__m256i const *)ref_ptr); \ |
86 | 0 | ref2_reg = _mm256_loadu_si256((__m256i const *)(ref_ptr + 32)); \ |
87 | 0 | ref1_reg = _mm256_avg_epu8( \ |
88 | 0 | ref1_reg, _mm256_loadu_si256((__m256i const *)second_pred)); \ |
89 | 0 | ref2_reg = _mm256_avg_epu8( \ |
90 | 0 | ref2_reg, _mm256_loadu_si256((__m256i const *)(second_pred + 32))); \ |
91 | 0 | sad1_reg = _mm256_sad_epu8( \ |
92 | 0 | ref1_reg, _mm256_loadu_si256((__m256i const *)src_ptr)); \ |
93 | 0 | sad2_reg = _mm256_sad_epu8( \ |
94 | 0 | ref2_reg, _mm256_loadu_si256((__m256i const *)(src_ptr + 32))); \ |
95 | 0 | sum_sad = \ |
96 | 0 | _mm256_add_epi32(sum_sad, _mm256_add_epi32(sad1_reg, sad2_reg)); \ |
97 | 0 | ref_ptr += ref_stride; \ |
98 | 0 | src_ptr += src_stride; \ |
99 | 0 | second_pred += 64; \ |
100 | 0 | } \ |
101 | 0 | sum_sad_h = _mm256_srli_si256(sum_sad, 8); \ |
102 | 0 | sum_sad = _mm256_add_epi32(sum_sad, sum_sad_h); \ |
103 | 0 | sum_sad128 = _mm256_extracti128_si256(sum_sad, 1); \ |
104 | 0 | sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128); \ |
105 | 0 | unsigned int res = (unsigned int)_mm_cvtsi128_si32(sum_sad128); \ |
106 | 0 | _mm256_zeroupper(); \ |
107 | 0 | return res; \ |
108 | 0 | } |
109 | | |
110 | | #define FSADAVG64 \ |
111 | | FSADAVG64_H(64) \ |
112 | | FSADAVG64_H(32) |
113 | | |
114 | | /* clang-format off */ |
115 | 0 | FSADAVG64 Unexecuted instantiation: aom_sad64x64_avg_avx2 Unexecuted instantiation: aom_sad64x32_avg_avx2 |
116 | 0 | /* clang-format on */ |
117 | 0 |
|
118 | 0 | #undef FSADAVG64 |
119 | 0 | #undef FSADAVG64_H |
120 | 0 | #endif // !CONFIG_HIGHWAY |
121 | 0 |
|
122 | 0 | // SAD, SAD_SKIP and SAD_AVG for 32xh blocks |
123 | 0 | static inline unsigned int sad32xh_avx2(const uint8_t *src_ptr, int src_stride, |
124 | 0 | const uint8_t *ref_ptr, int ref_stride, |
125 | 0 | int h) { |
126 | 0 | int i; |
127 | 0 | __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg; |
128 | 0 | __m256i sum_sad = _mm256_setzero_si256(); |
129 | 0 | __m256i sum_sad_h; |
130 | 0 | __m128i sum_sad128; |
131 | 0 | int ref2_stride = ref_stride << 1; |
132 | 0 | int src2_stride = src_stride << 1; |
133 | 0 | int max = h >> 1; |
134 | 0 | for (i = 0; i < max; i++) { |
135 | 0 | ref1_reg = _mm256_loadu_si256((__m256i const *)ref_ptr); |
136 | 0 | ref2_reg = _mm256_loadu_si256((__m256i const *)(ref_ptr + ref_stride)); |
137 | 0 | sad1_reg = |
138 | 0 | _mm256_sad_epu8(ref1_reg, _mm256_loadu_si256((__m256i const *)src_ptr)); |
139 | 0 | sad2_reg = _mm256_sad_epu8( |
140 | 0 | ref2_reg, _mm256_loadu_si256((__m256i const *)(src_ptr + src_stride))); |
141 | 0 | sum_sad = _mm256_add_epi32(sum_sad, _mm256_add_epi32(sad1_reg, sad2_reg)); |
142 | 0 | ref_ptr += ref2_stride; |
143 | 0 | src_ptr += src2_stride; |
144 | 0 | } |
145 | 0 | sum_sad_h = _mm256_srli_si256(sum_sad, 8); |
146 | 0 | sum_sad = _mm256_add_epi32(sum_sad, sum_sad_h); |
147 | 0 | sum_sad128 = _mm256_extracti128_si256(sum_sad, 1); |
148 | 0 | sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128); |
149 | 0 | unsigned int res = (unsigned int)_mm_cvtsi128_si32(sum_sad128); |
150 | 0 | _mm256_zeroupper(); |
151 | 0 | return res; |
152 | 0 | } |
153 | | |
154 | | #define FSAD32_H(h) \ |
155 | | unsigned int aom_sad32x##h##_avx2(const uint8_t *src_ptr, int src_stride, \ |
156 | 0 | const uint8_t *ref_ptr, int ref_stride) { \ |
157 | 0 | return sad32xh_avx2(src_ptr, src_stride, ref_ptr, ref_stride, h); \ |
158 | 0 | } Unexecuted instantiation: aom_sad32x64_avx2 Unexecuted instantiation: aom_sad32x32_avx2 Unexecuted instantiation: aom_sad32x16_avx2 |
159 | | |
160 | | #define FSADS32_H(h) \ |
161 | | unsigned int aom_sad_skip_32x##h##_avx2( \ |
162 | | const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ |
163 | 0 | int ref_stride) { \ |
164 | 0 | return 2 * sad32xh_avx2(src_ptr, src_stride * 2, ref_ptr, ref_stride * 2, \ |
165 | 0 | h / 2); \ |
166 | 0 | } Unexecuted instantiation: aom_sad_skip_32x64_avx2 Unexecuted instantiation: aom_sad_skip_32x32_avx2 Unexecuted instantiation: aom_sad_skip_32x16_avx2 |
167 | | |
168 | | #define FSAD32 \ |
169 | | FSAD32_H(64) \ |
170 | | FSAD32_H(32) \ |
171 | | FSAD32_H(16) \ |
172 | | FSADS32_H(64) \ |
173 | | FSADS32_H(32) \ |
174 | | FSADS32_H(16) |
175 | | |
176 | | /* clang-format off */ |
177 | | FSAD32 |
178 | | /* clang-format on */ |
179 | | |
180 | | #undef FSAD32 |
181 | | #undef FSAD32_H |
182 | | |
183 | | #define FSADAVG32_H(h) \ |
184 | | unsigned int aom_sad32x##h##_avg_avx2( \ |
185 | | const uint8_t *src_ptr, int src_stride, const uint8_t *ref_ptr, \ |
186 | 0 | int ref_stride, const uint8_t *second_pred) { \ |
187 | 0 | int i; \ |
188 | 0 | __m256i sad1_reg, sad2_reg, ref1_reg, ref2_reg; \ |
189 | 0 | __m256i sum_sad = _mm256_setzero_si256(); \ |
190 | 0 | __m256i sum_sad_h; \ |
191 | 0 | __m128i sum_sad128; \ |
192 | 0 | int ref2_stride = ref_stride << 1; \ |
193 | 0 | int src2_stride = src_stride << 1; \ |
194 | 0 | int max = h >> 1; \ |
195 | 0 | for (i = 0; i < max; i++) { \ |
196 | 0 | ref1_reg = _mm256_loadu_si256((__m256i const *)ref_ptr); \ |
197 | 0 | ref2_reg = _mm256_loadu_si256((__m256i const *)(ref_ptr + ref_stride)); \ |
198 | 0 | ref1_reg = _mm256_avg_epu8( \ |
199 | 0 | ref1_reg, _mm256_loadu_si256((__m256i const *)second_pred)); \ |
200 | 0 | ref2_reg = _mm256_avg_epu8( \ |
201 | 0 | ref2_reg, _mm256_loadu_si256((__m256i const *)(second_pred + 32))); \ |
202 | 0 | sad1_reg = _mm256_sad_epu8( \ |
203 | 0 | ref1_reg, _mm256_loadu_si256((__m256i const *)src_ptr)); \ |
204 | 0 | sad2_reg = _mm256_sad_epu8( \ |
205 | 0 | ref2_reg, \ |
206 | 0 | _mm256_loadu_si256((__m256i const *)(src_ptr + src_stride))); \ |
207 | 0 | sum_sad = \ |
208 | 0 | _mm256_add_epi32(sum_sad, _mm256_add_epi32(sad1_reg, sad2_reg)); \ |
209 | 0 | ref_ptr += ref2_stride; \ |
210 | 0 | src_ptr += src2_stride; \ |
211 | 0 | second_pred += 64; \ |
212 | 0 | } \ |
213 | 0 | sum_sad_h = _mm256_srli_si256(sum_sad, 8); \ |
214 | 0 | sum_sad = _mm256_add_epi32(sum_sad, sum_sad_h); \ |
215 | 0 | sum_sad128 = _mm256_extracti128_si256(sum_sad, 1); \ |
216 | 0 | sum_sad128 = _mm_add_epi32(_mm256_castsi256_si128(sum_sad), sum_sad128); \ |
217 | 0 | unsigned int res = (unsigned int)_mm_cvtsi128_si32(sum_sad128); \ |
218 | 0 | _mm256_zeroupper(); \ |
219 | 0 | return res; \ |
220 | 0 | } |
221 | | |
222 | | #define FSADAVG32 \ |
223 | | FSADAVG32_H(64) \ |
224 | | FSADAVG32_H(32) \ |
225 | | FSADAVG32_H(16) |
226 | | |
227 | | /* clang-format off */ |
228 | | FSADAVG32 Unexecuted instantiation: aom_sad32x64_avg_avx2 Unexecuted instantiation: aom_sad32x32_avg_avx2 Unexecuted instantiation: aom_sad32x16_avg_avx2 |
229 | | /* clang-format on */ |
230 | | |
231 | | #undef FSADAVG32 |
232 | | #undef FSADAVG32_H |