/src/libvpx/vpx_dsp/x86/sad4d_avx2.c
Line | Count | Source |
1 | | /* |
2 | | * Copyright (c) 2014 The WebM project authors. All Rights Reserved. |
3 | | * |
4 | | * Use of this source code is governed by a BSD-style license |
5 | | * that can be found in the LICENSE file in the root of the source |
6 | | * tree. An additional intellectual property rights grant can be found |
7 | | * in the file PATENTS. All contributing project authors may |
8 | | * be found in the AUTHORS file in the root of the source tree. |
9 | | */ |
10 | | #include <immintrin.h> // AVX2 |
11 | | #include "./vpx_dsp_rtcd.h" |
12 | | #include "vpx/vpx_integer.h" |
13 | | |
14 | | // Note with sums[4] some versions of Visual Studio may fail due to parameter |
15 | | // alignment, though the functions should be equivalent: |
16 | | // error C2719: 'sums': formal parameter with requested alignment of 32 won't be |
17 | | // aligned |
18 | | static INLINE void calc_final_4(const __m256i *const sums /*[4]*/, |
19 | 7.95M | uint32_t sad_array[4]) { |
20 | 7.95M | const __m256i t0 = _mm256_hadd_epi32(sums[0], sums[1]); |
21 | 7.95M | const __m256i t1 = _mm256_hadd_epi32(sums[2], sums[3]); |
22 | 7.95M | const __m256i t2 = _mm256_hadd_epi32(t0, t1); |
23 | 7.95M | const __m128i sum = _mm_add_epi32(_mm256_castsi256_si128(t2), |
24 | 7.95M | _mm256_extractf128_si256(t2, 1)); |
25 | 7.95M | _mm_storeu_si128((__m128i *)sad_array, sum); |
26 | 7.95M | } |
27 | | |
28 | | static INLINE void sad32xhx4d_avx2(const uint8_t *src_ptr, int src_stride, |
29 | | const uint8_t *const ref_array[4], |
30 | | int ref_stride, int h, |
31 | 6.22M | uint32_t sad_array[4]) { |
32 | 6.22M | int i; |
33 | 6.22M | const uint8_t *refs[4]; |
34 | 6.22M | __m256i sums[4]; |
35 | | |
36 | 6.22M | refs[0] = ref_array[0]; |
37 | 6.22M | refs[1] = ref_array[1]; |
38 | 6.22M | refs[2] = ref_array[2]; |
39 | 6.22M | refs[3] = ref_array[3]; |
40 | 6.22M | sums[0] = _mm256_setzero_si256(); |
41 | 6.22M | sums[1] = _mm256_setzero_si256(); |
42 | 6.22M | sums[2] = _mm256_setzero_si256(); |
43 | 6.22M | sums[3] = _mm256_setzero_si256(); |
44 | | |
45 | 159M | for (i = 0; i < h; i++) { |
46 | 153M | __m256i r[4]; |
47 | | |
48 | | // load src and all ref[] |
49 | 153M | const __m256i s = _mm256_load_si256((const __m256i *)src_ptr); |
50 | 153M | r[0] = _mm256_loadu_si256((const __m256i *)refs[0]); |
51 | 153M | r[1] = _mm256_loadu_si256((const __m256i *)refs[1]); |
52 | 153M | r[2] = _mm256_loadu_si256((const __m256i *)refs[2]); |
53 | 153M | r[3] = _mm256_loadu_si256((const __m256i *)refs[3]); |
54 | | |
55 | | // sum of the absolute differences between every ref[] to src |
56 | 153M | r[0] = _mm256_sad_epu8(r[0], s); |
57 | 153M | r[1] = _mm256_sad_epu8(r[1], s); |
58 | 153M | r[2] = _mm256_sad_epu8(r[2], s); |
59 | 153M | r[3] = _mm256_sad_epu8(r[3], s); |
60 | | |
61 | | // sum every ref[] |
62 | 153M | sums[0] = _mm256_add_epi32(sums[0], r[0]); |
63 | 153M | sums[1] = _mm256_add_epi32(sums[1], r[1]); |
64 | 153M | sums[2] = _mm256_add_epi32(sums[2], r[2]); |
65 | 153M | sums[3] = _mm256_add_epi32(sums[3], r[3]); |
66 | | |
67 | 153M | src_ptr += src_stride; |
68 | 153M | refs[0] += ref_stride; |
69 | 153M | refs[1] += ref_stride; |
70 | 153M | refs[2] += ref_stride; |
71 | 153M | refs[3] += ref_stride; |
72 | 153M | } |
73 | | |
74 | 6.22M | calc_final_4(sums, sad_array); |
75 | 6.22M | } |
76 | | |
77 | | static INLINE void sad64xhx4d_avx2(const uint8_t *src_ptr, int src_stride, |
78 | | const uint8_t *const ref_array[4], |
79 | | int ref_stride, int h, |
80 | 1.73M | uint32_t sad_array[4]) { |
81 | 1.73M | __m256i sums[4]; |
82 | 1.73M | int i; |
83 | 1.73M | const uint8_t *refs[4]; |
84 | | |
85 | 1.73M | refs[0] = ref_array[0]; |
86 | 1.73M | refs[1] = ref_array[1]; |
87 | 1.73M | refs[2] = ref_array[2]; |
88 | 1.73M | refs[3] = ref_array[3]; |
89 | 1.73M | sums[0] = _mm256_setzero_si256(); |
90 | 1.73M | sums[1] = _mm256_setzero_si256(); |
91 | 1.73M | sums[2] = _mm256_setzero_si256(); |
92 | 1.73M | sums[3] = _mm256_setzero_si256(); |
93 | | |
94 | 86.5M | for (i = 0; i < h; i++) { |
95 | 84.8M | __m256i r_lo[4], r_hi[4]; |
96 | | // load 64 bytes from src and all ref[] |
97 | 84.8M | const __m256i s_lo = _mm256_load_si256((const __m256i *)src_ptr); |
98 | 84.8M | const __m256i s_hi = _mm256_load_si256((const __m256i *)(src_ptr + 32)); |
99 | 84.8M | r_lo[0] = _mm256_loadu_si256((const __m256i *)refs[0]); |
100 | 84.8M | r_hi[0] = _mm256_loadu_si256((const __m256i *)(refs[0] + 32)); |
101 | 84.8M | r_lo[1] = _mm256_loadu_si256((const __m256i *)refs[1]); |
102 | 84.8M | r_hi[1] = _mm256_loadu_si256((const __m256i *)(refs[1] + 32)); |
103 | 84.8M | r_lo[2] = _mm256_loadu_si256((const __m256i *)refs[2]); |
104 | 84.8M | r_hi[2] = _mm256_loadu_si256((const __m256i *)(refs[2] + 32)); |
105 | 84.8M | r_lo[3] = _mm256_loadu_si256((const __m256i *)refs[3]); |
106 | 84.8M | r_hi[3] = _mm256_loadu_si256((const __m256i *)(refs[3] + 32)); |
107 | | |
108 | | // sum of the absolute differences between every ref[] to src |
109 | 84.8M | r_lo[0] = _mm256_sad_epu8(r_lo[0], s_lo); |
110 | 84.8M | r_lo[1] = _mm256_sad_epu8(r_lo[1], s_lo); |
111 | 84.8M | r_lo[2] = _mm256_sad_epu8(r_lo[2], s_lo); |
112 | 84.8M | r_lo[3] = _mm256_sad_epu8(r_lo[3], s_lo); |
113 | 84.8M | r_hi[0] = _mm256_sad_epu8(r_hi[0], s_hi); |
114 | 84.8M | r_hi[1] = _mm256_sad_epu8(r_hi[1], s_hi); |
115 | 84.8M | r_hi[2] = _mm256_sad_epu8(r_hi[2], s_hi); |
116 | 84.8M | r_hi[3] = _mm256_sad_epu8(r_hi[3], s_hi); |
117 | | |
118 | | // sum every ref[] |
119 | 84.8M | sums[0] = _mm256_add_epi32(sums[0], r_lo[0]); |
120 | 84.8M | sums[1] = _mm256_add_epi32(sums[1], r_lo[1]); |
121 | 84.8M | sums[2] = _mm256_add_epi32(sums[2], r_lo[2]); |
122 | 84.8M | sums[3] = _mm256_add_epi32(sums[3], r_lo[3]); |
123 | 84.8M | sums[0] = _mm256_add_epi32(sums[0], r_hi[0]); |
124 | 84.8M | sums[1] = _mm256_add_epi32(sums[1], r_hi[1]); |
125 | 84.8M | sums[2] = _mm256_add_epi32(sums[2], r_hi[2]); |
126 | 84.8M | sums[3] = _mm256_add_epi32(sums[3], r_hi[3]); |
127 | | |
128 | 84.8M | src_ptr += src_stride; |
129 | 84.8M | refs[0] += ref_stride; |
130 | 84.8M | refs[1] += ref_stride; |
131 | 84.8M | refs[2] += ref_stride; |
132 | 84.8M | refs[3] += ref_stride; |
133 | 84.8M | } |
134 | | |
135 | 1.73M | calc_final_4(sums, sad_array); |
136 | 1.73M | } |
137 | | |
138 | | #define SAD64_H(h) \ |
139 | | void vpx_sad64x##h##x4d_avx2(const uint8_t *src, int src_stride, \ |
140 | | const uint8_t *const ref_array[4], \ |
141 | 992k | int ref_stride, uint32_t sad_array[4]) { \ |
142 | 992k | sad64xhx4d_avx2(src, src_stride, ref_array, ref_stride, h, sad_array); \ |
143 | 992k | } |
144 | | |
145 | | #define SAD32_H(h) \ |
146 | | void vpx_sad32x##h##x4d_avx2(const uint8_t *src, int src_stride, \ |
147 | | const uint8_t *const ref_array[4], \ |
148 | 3.61M | int ref_stride, uint32_t sad_array[4]) { \ |
149 | 3.61M | sad32xhx4d_avx2(src, src_stride, ref_array, ref_stride, h, sad_array); \ |
150 | 3.61M | } |
151 | | |
152 | | SAD64_H(64) |
153 | | SAD32_H(32) |
154 | | |
155 | | #define SADS64_H(h) \ |
156 | | void vpx_sad_skip_64x##h##x4d_avx2(const uint8_t *src, int src_stride, \ |
157 | | const uint8_t *const ref_array[4], \ |
158 | 738k | int ref_stride, uint32_t sad_array[4]) { \ |
159 | 738k | sad64xhx4d_avx2(src, 2 * src_stride, ref_array, 2 * ref_stride, \ |
160 | 738k | ((h) >> 1), sad_array); \ |
161 | 738k | sad_array[0] <<= 1; \ |
162 | 738k | sad_array[1] <<= 1; \ |
163 | 738k | sad_array[2] <<= 1; \ |
164 | 738k | sad_array[3] <<= 1; \ |
165 | 738k | } vpx_sad_skip_64x64x4d_avx2 Line | Count | Source | 158 | 592k | int ref_stride, uint32_t sad_array[4]) { \ | 159 | 592k | sad64xhx4d_avx2(src, 2 * src_stride, ref_array, 2 * ref_stride, \ | 160 | 592k | ((h) >> 1), sad_array); \ | 161 | 592k | sad_array[0] <<= 1; \ | 162 | 592k | sad_array[1] <<= 1; \ | 163 | 592k | sad_array[2] <<= 1; \ | 164 | 592k | sad_array[3] <<= 1; \ | 165 | 592k | } |
vpx_sad_skip_64x32x4d_avx2 Line | Count | Source | 158 | 145k | int ref_stride, uint32_t sad_array[4]) { \ | 159 | 145k | sad64xhx4d_avx2(src, 2 * src_stride, ref_array, 2 * ref_stride, \ | 160 | 145k | ((h) >> 1), sad_array); \ | 161 | 145k | sad_array[0] <<= 1; \ | 162 | 145k | sad_array[1] <<= 1; \ | 163 | 145k | sad_array[2] <<= 1; \ | 164 | 145k | sad_array[3] <<= 1; \ | 165 | 145k | } |
|
166 | | |
167 | | #define SADS32_H(h) \ |
168 | | void vpx_sad_skip_32x##h##x4d_avx2(const uint8_t *src, int src_stride, \ |
169 | | const uint8_t *const ref_array[4], \ |
170 | 2.60M | int ref_stride, uint32_t sad_array[4]) { \ |
171 | 2.60M | sad32xhx4d_avx2(src, 2 * src_stride, ref_array, 2 * ref_stride, \ |
172 | 2.60M | ((h) >> 1), sad_array); \ |
173 | 2.60M | sad_array[0] <<= 1; \ |
174 | 2.60M | sad_array[1] <<= 1; \ |
175 | 2.60M | sad_array[2] <<= 1; \ |
176 | 2.60M | sad_array[3] <<= 1; \ |
177 | 2.60M | } vpx_sad_skip_32x64x4d_avx2 Line | Count | Source | 170 | 16.9k | int ref_stride, uint32_t sad_array[4]) { \ | 171 | 16.9k | sad32xhx4d_avx2(src, 2 * src_stride, ref_array, 2 * ref_stride, \ | 172 | 16.9k | ((h) >> 1), sad_array); \ | 173 | 16.9k | sad_array[0] <<= 1; \ | 174 | 16.9k | sad_array[1] <<= 1; \ | 175 | 16.9k | sad_array[2] <<= 1; \ | 176 | 16.9k | sad_array[3] <<= 1; \ | 177 | 16.9k | } |
vpx_sad_skip_32x32x4d_avx2 Line | Count | Source | 170 | 1.99M | int ref_stride, uint32_t sad_array[4]) { \ | 171 | 1.99M | sad32xhx4d_avx2(src, 2 * src_stride, ref_array, 2 * ref_stride, \ | 172 | 1.99M | ((h) >> 1), sad_array); \ | 173 | 1.99M | sad_array[0] <<= 1; \ | 174 | 1.99M | sad_array[1] <<= 1; \ | 175 | 1.99M | sad_array[2] <<= 1; \ | 176 | 1.99M | sad_array[3] <<= 1; \ | 177 | 1.99M | } |
vpx_sad_skip_32x16x4d_avx2 Line | Count | Source | 170 | 592k | int ref_stride, uint32_t sad_array[4]) { \ | 171 | 592k | sad32xhx4d_avx2(src, 2 * src_stride, ref_array, 2 * ref_stride, \ | 172 | 592k | ((h) >> 1), sad_array); \ | 173 | 592k | sad_array[0] <<= 1; \ | 174 | 592k | sad_array[1] <<= 1; \ | 175 | 592k | sad_array[2] <<= 1; \ | 176 | 592k | sad_array[3] <<= 1; \ | 177 | 592k | } |
|
178 | | |
179 | | SADS64_H(64) |
180 | | SADS64_H(32) |
181 | | |
182 | | SADS32_H(64) |
183 | | SADS32_H(32) |
184 | | SADS32_H(16) |