/src/Simd/src/Simd/SimdAvx2ColSums.cpp
Line | Count | Source |
1 | | /* |
2 | | * Simd Library (http://ermig1979.github.io/Simd). |
3 | | * |
4 | | * Copyright (c) 2011-2026 Yermalayeu Ihar. |
5 | | * |
6 | | * Permission is hereby granted, free of charge, to any person obtaining a copy |
7 | | * of this software and associated documentation files (the "Software"), to deal |
8 | | * in the Software without restriction, including without limitation the rights |
9 | | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
10 | | * copies of the Software, and to permit persons to whom the Software is |
11 | | * furnished to do so, subject to the following conditions: |
12 | | * |
13 | | * The above copyright notice and this permission notice shall be included in |
14 | | * all copies or substantial portions of the Software. |
15 | | * |
16 | | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
17 | | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
18 | | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
19 | | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
20 | | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
21 | | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
22 | | * SOFTWARE. |
23 | | */ |
24 | | #include "Simd/SimdMemory.h" |
25 | | #include "Simd/SimdStore.h" |
26 | | |
27 | | namespace Simd |
28 | | { |
29 | | #ifdef SIMD_AVX2_ENABLE |
30 | | namespace Avx2 |
31 | | { |
32 | | namespace |
33 | | { |
34 | | struct Buffer |
35 | | { |
36 | | Buffer(size_t width) |
37 | 0 | { |
38 | 0 | _p = Allocate(sizeof(uint16_t)*width + sizeof(uint32_t)*width); |
39 | 0 | sums16 = (uint16_t*)_p; |
40 | 0 | sums32 = (uint32_t*)(sums16 + width); |
41 | 0 | } |
42 | | |
43 | | ~Buffer() |
44 | 0 | { |
45 | 0 | Free(_p); |
46 | 0 | } |
47 | | |
48 | | uint16_t * sums16; |
49 | | uint32_t * sums32; |
50 | | private: |
51 | | void *_p; |
52 | | }; |
53 | | } |
54 | | |
55 | | template <bool align> SIMD_INLINE void Sum16x1(const uint8_t * src, size_t, uint16_t * dst) |
56 | 0 | { |
57 | 0 | __m256i sum0 = Load<true>((__m256i*)dst + 0); |
58 | 0 | __m256i sum1 = Load<true>((__m256i*)dst + 1); |
59 | 0 | __m256i src0 = Load<align>((__m256i*)src); |
60 | 0 | sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO)); |
61 | 0 | sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO)); |
62 | 0 | Store<true>((__m256i*)dst + 0, sum0); |
63 | 0 | Store<true>((__m256i*)dst + 1, sum1); |
64 | 0 | } Unexecuted instantiation: void Simd::Avx2::Sum16x1<true>(unsigned char const*, unsigned long, unsigned short*) Unexecuted instantiation: void Simd::Avx2::Sum16x1<false>(unsigned char const*, unsigned long, unsigned short*) |
65 | | |
66 | | template <bool align> SIMD_INLINE void Sum16x4(const uint8_t * src, size_t stride, uint16_t * dst) |
67 | 0 | { |
68 | 0 | __m256i sum0 = Load<true>((__m256i*)dst + 0); |
69 | 0 | __m256i sum1 = Load<true>((__m256i*)dst + 1); |
70 | 0 | __m256i src0 = Load<align>((__m256i*)src); |
71 | 0 | sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO)); |
72 | 0 | sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO)); |
73 | 0 | src0 = Load<align>((__m256i*)(src + 1 * stride)); |
74 | 0 | sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO)); |
75 | 0 | sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO)); |
76 | 0 | src0 = Load<align>((__m256i*)(src + 2 * stride)); |
77 | 0 | sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO)); |
78 | 0 | sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO)); |
79 | 0 | src0 = Load<align>((__m256i*)(src + 3 * stride)); |
80 | 0 | sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO)); |
81 | 0 | sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO)); |
82 | 0 | Store<true>((__m256i*)dst + 0, sum0); |
83 | 0 | Store<true>((__m256i*)dst + 1, sum1); |
84 | 0 | } Unexecuted instantiation: void Simd::Avx2::Sum16x4<true>(unsigned char const*, unsigned long, unsigned short*) Unexecuted instantiation: void Simd::Avx2::Sum16x4<false>(unsigned char const*, unsigned long, unsigned short*) |
85 | | |
86 | | template <bool align> SIMD_INLINE void Sum16x8(const uint8_t * src, size_t stride, uint16_t * dst) |
87 | 0 | { |
88 | 0 | __m256i sum0 = Load<true>((__m256i*)dst + 0); |
89 | 0 | __m256i sum1 = Load<true>((__m256i*)dst + 1); |
90 | 0 | __m256i src0 = Load<align>((__m256i*)src); |
91 | 0 | sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO)); |
92 | 0 | sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO)); |
93 | 0 | src0 = Load<align>((__m256i*)(src + 1 * stride)); |
94 | 0 | sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO)); |
95 | 0 | sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO)); |
96 | 0 | src0 = Load<align>((__m256i*)(src + 2 * stride)); |
97 | 0 | sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO)); |
98 | 0 | sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO)); |
99 | 0 | src0 = Load<align>((__m256i*)(src + 3 * stride)); |
100 | 0 | sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO)); |
101 | 0 | sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO)); |
102 | 0 | src0 = Load<align>((__m256i*)(src + 4 * stride)); |
103 | 0 | sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO)); |
104 | 0 | sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO)); |
105 | 0 | src0 = Load<align>((__m256i*)(src + 5 * stride)); |
106 | 0 | sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO)); |
107 | 0 | sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO)); |
108 | 0 | src0 = Load<align>((__m256i*)(src + 6 * stride)); |
109 | 0 | sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO)); |
110 | 0 | sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO)); |
111 | 0 | src0 = Load<align>((__m256i*)(src + 7 * stride)); |
112 | 0 | sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO)); |
113 | 0 | sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO)); |
114 | 0 | Store<true>((__m256i*)dst + 0, sum0); |
115 | 0 | Store<true>((__m256i*)dst + 1, sum1); |
116 | 0 | } Unexecuted instantiation: void Simd::Avx2::Sum16x8<true>(unsigned char const*, unsigned long, unsigned short*) Unexecuted instantiation: void Simd::Avx2::Sum16x8<false>(unsigned char const*, unsigned long, unsigned short*) |
117 | | |
118 | | SIMD_INLINE void Sum16(__m256i src8, uint16_t * sums16) |
119 | 0 | { |
120 | 0 | Store<true>((__m256i*)sums16 + 0, _mm256_add_epi16(Load<true>((__m256i*)sums16 + 0), _mm256_unpacklo_epi8(src8, K_ZERO))); |
121 | 0 | Store<true>((__m256i*)sums16 + 1, _mm256_add_epi16(Load<true>((__m256i*)sums16 + 1), _mm256_unpackhi_epi8(src8, K_ZERO))); |
122 | 0 | } |
123 | | |
124 | | SIMD_INLINE void Sum16To32(const uint16_t * src, uint32_t * dst) |
125 | 0 | { |
126 | 0 | __m256i lo = LoadPermuted<true>((__m256i*)src + 0); |
127 | 0 | __m256i hi = LoadPermuted<true>((__m256i*)src + 1); |
128 | 0 | Store<true>((__m256i*)dst + 0, _mm256_add_epi32(Load<true>((__m256i*)dst + 0), _mm256_unpacklo_epi16(lo, K_ZERO))); |
129 | 0 | Store<true>((__m256i*)dst + 1, _mm256_add_epi32(Load<true>((__m256i*)dst + 1), _mm256_unpacklo_epi16(hi, K_ZERO))); |
130 | 0 | Store<true>((__m256i*)dst + 2, _mm256_add_epi32(Load<true>((__m256i*)dst + 2), _mm256_unpackhi_epi16(lo, K_ZERO))); |
131 | 0 | Store<true>((__m256i*)dst + 3, _mm256_add_epi32(Load<true>((__m256i*)dst + 3), _mm256_unpackhi_epi16(hi, K_ZERO))); |
132 | 0 | } |
133 | | |
134 | | template <bool align> void GetColSums(const uint8_t * src, size_t stride, size_t width, size_t height, uint32_t * sums) |
135 | 0 | { |
136 | 0 | size_t alignedLoWidth = AlignLo(width, A); |
137 | 0 | size_t alignedHiWidth = AlignHi(width, A); |
138 | 0 | size_t stepSize = SCHAR_MAX + 1; |
139 | 0 | size_t stepCount = (height + SCHAR_MAX) / stepSize; |
140 | |
|
141 | 0 | Buffer buffer(alignedHiWidth); |
142 | 0 | memset(buffer.sums32, 0, sizeof(uint32_t)*alignedHiWidth); |
143 | 0 | for (size_t step = 0; step < stepCount; ++step) |
144 | 0 | { |
145 | 0 | size_t rowStart = step*stepSize; |
146 | 0 | size_t rowEnd = Min(rowStart + stepSize, height); |
147 | |
|
148 | 0 | size_t rowEnd4 = AlignLo(rowEnd, 4); |
149 | 0 | size_t rowEnd8 = AlignLo(rowEnd, 8); |
150 | |
|
151 | 0 | memset(buffer.sums16, 0, sizeof(uint16_t)*alignedHiWidth); |
152 | 0 | size_t row = rowStart; |
153 | 0 | for (; row < rowEnd8; row += 8) |
154 | 0 | { |
155 | 0 | for (size_t col = 0; col < alignedLoWidth; col += A) |
156 | 0 | Sum16x8<align>(src + col, stride, buffer.sums16 + col); |
157 | 0 | if (alignedLoWidth != width) |
158 | 0 | Sum16x8<false>(src + width - A, stride, buffer.sums16 + alignedLoWidth); |
159 | 0 | src += 8 * stride; |
160 | 0 | } |
161 | 0 | for (; row < rowEnd4; row += 4) |
162 | 0 | { |
163 | 0 | for (size_t col = 0; col < alignedLoWidth; col += A) |
164 | 0 | Sum16x4<align>(src + col, stride, buffer.sums16 + col); |
165 | 0 | if (alignedLoWidth != width) |
166 | 0 | Sum16x4<false>(src + width - A, stride, buffer.sums16 + alignedLoWidth); |
167 | 0 | src += 4 * stride; |
168 | 0 | } |
169 | 0 | for (; row < rowEnd; ++row) |
170 | 0 | { |
171 | 0 | for (size_t col = 0; col < alignedLoWidth; col += A) |
172 | 0 | Sum16x1<align>(src + col, stride, buffer.sums16 + col); |
173 | 0 | if (alignedLoWidth != width) |
174 | 0 | Sum16x1<false>(src + width - A, stride, buffer.sums16 + alignedLoWidth); |
175 | 0 | src += stride; |
176 | 0 | } |
177 | |
|
178 | 0 | for (size_t col = 0; col < alignedHiWidth; col += A) |
179 | 0 | Sum16To32(buffer.sums16 + col, buffer.sums32 + col); |
180 | 0 | } |
181 | 0 | memcpy(sums, buffer.sums32, sizeof(uint32_t)*alignedLoWidth); |
182 | 0 | if (alignedLoWidth != width) |
183 | 0 | memcpy(sums + alignedLoWidth, buffer.sums32 + alignedLoWidth + alignedHiWidth - width, sizeof(uint32_t)*(width - alignedLoWidth)); |
184 | 0 | } Unexecuted instantiation: void Simd::Avx2::GetColSums<true>(unsigned char const*, unsigned long, unsigned long, unsigned long, unsigned int*) Unexecuted instantiation: void Simd::Avx2::GetColSums<false>(unsigned char const*, unsigned long, unsigned long, unsigned long, unsigned int*) |
185 | | |
186 | | void GetColSums(const uint8_t * src, size_t stride, size_t width, size_t height, uint32_t * sums) |
187 | 0 | { |
188 | 0 | if (Aligned(src) && Aligned(stride)) |
189 | 0 | GetColSums<true>(src, stride, width, height, sums); |
190 | 0 | else |
191 | 0 | GetColSums<false>(src, stride, width, height, sums); |
192 | 0 | } |
193 | | |
194 | | //----------------------------------------------------------------------------------------- |
195 | | |
196 | | template <bool align> void GetAbsDxColSums(const uint8_t * src, size_t stride, size_t width, size_t height, uint32_t * sums) |
197 | 0 | { |
198 | 0 | width--; |
199 | 0 | size_t alignedLoWidth = AlignLo(width, A); |
200 | 0 | size_t alignedHiWidth = AlignHi(width, A); |
201 | 0 | size_t stepSize = SCHAR_MAX + 1; |
202 | 0 | size_t stepCount = (height + SCHAR_MAX) / stepSize; |
203 | |
|
204 | 0 | Buffer buffer(alignedHiWidth); |
205 | 0 | memset(buffer.sums32, 0, sizeof(uint32_t)*alignedHiWidth); |
206 | 0 | for (size_t step = 0; step < stepCount; ++step) |
207 | 0 | { |
208 | 0 | size_t rowStart = step*stepSize; |
209 | 0 | size_t rowEnd = Min(rowStart + stepSize, height); |
210 | |
|
211 | 0 | memset(buffer.sums16, 0, sizeof(uint16_t)*alignedHiWidth); |
212 | 0 | for (size_t row = rowStart; row < rowEnd; ++row) |
213 | 0 | { |
214 | 0 | for (size_t col = 0; col < alignedLoWidth; col += A) |
215 | 0 | { |
216 | 0 | __m256i _src0 = Load<align>((__m256i*)(src + col + 0)); |
217 | 0 | __m256i _src1 = Load<false>((__m256i*)(src + col + 1)); |
218 | 0 | Sum16(AbsDifferenceU8(_src0, _src1), buffer.sums16 + col); |
219 | 0 | } |
220 | 0 | if (alignedLoWidth != width) |
221 | 0 | { |
222 | 0 | __m256i _src0 = Load<false>((__m256i*)(src + width - A + 0)); |
223 | 0 | __m256i _src1 = Load<false>((__m256i*)(src + width - A + 1)); |
224 | 0 | Sum16(AbsDifferenceU8(_src0, _src1), buffer.sums16 + alignedLoWidth); |
225 | 0 | } |
226 | 0 | src += stride; |
227 | 0 | } |
228 | |
|
229 | 0 | for (size_t col = 0; col < alignedHiWidth; col += A) |
230 | 0 | Sum16To32(buffer.sums16 + col, buffer.sums32 + col); |
231 | 0 | } |
232 | 0 | memcpy(sums, buffer.sums32, sizeof(uint32_t)*alignedLoWidth); |
233 | 0 | if (alignedLoWidth != width) |
234 | 0 | memcpy(sums + alignedLoWidth, buffer.sums32 + alignedLoWidth + alignedHiWidth - width, sizeof(uint32_t)*(width - alignedLoWidth)); |
235 | 0 | sums[width] = 0; |
236 | 0 | } Unexecuted instantiation: void Simd::Avx2::GetAbsDxColSums<true>(unsigned char const*, unsigned long, unsigned long, unsigned long, unsigned int*) Unexecuted instantiation: void Simd::Avx2::GetAbsDxColSums<false>(unsigned char const*, unsigned long, unsigned long, unsigned long, unsigned int*) |
237 | | |
238 | | void GetAbsDxColSums(const uint8_t * src, size_t stride, size_t width, size_t height, uint32_t * sums) |
239 | 0 | { |
240 | 0 | if (Aligned(src) && Aligned(stride)) |
241 | 0 | GetAbsDxColSums<true>(src, stride, width, height, sums); |
242 | 0 | else |
243 | 0 | GetAbsDxColSums<false>(src, stride, width, height, sums); |
244 | 0 | } |
245 | | } |
246 | | #endif |
247 | | } |