Coverage Report

Created: 2026-10-31 07:15

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/Simd/src/Simd/SimdAvx2ColSums.cpp
Line
Count
Source
1
/*
2
* Simd Library (http://ermig1979.github.io/Simd).
3
*
4
* Copyright (c) 2011-2026 Yermalayeu Ihar.
5
*
6
* Permission is hereby granted, free of charge, to any person obtaining a copy
7
* of this software and associated documentation files (the "Software"), to deal
8
* in the Software without restriction, including without limitation the rights
9
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10
* copies of the Software, and to permit persons to whom the Software is
11
* furnished to do so, subject to the following conditions:
12
*
13
* The above copyright notice and this permission notice shall be included in
14
* all copies or substantial portions of the Software.
15
*
16
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22
* SOFTWARE.
23
*/
24
#include "Simd/SimdMemory.h"
25
#include "Simd/SimdStore.h"
26
27
namespace Simd
28
{
29
#ifdef SIMD_AVX2_ENABLE    
30
    namespace Avx2
31
    {
32
        namespace
33
        {
34
            struct Buffer
35
            {
36
                Buffer(size_t width)
37
0
                {
38
0
                    _p = Allocate(sizeof(uint16_t)*width + sizeof(uint32_t)*width);
39
0
                    sums16 = (uint16_t*)_p;
40
0
                    sums32 = (uint32_t*)(sums16 + width);
41
0
                }
42
43
                ~Buffer()
44
0
                {
45
0
                    Free(_p);
46
0
                }
47
48
                uint16_t * sums16;
49
                uint32_t * sums32;
50
            private:
51
                void *_p;
52
            };
53
        }
54
55
        template <bool align> SIMD_INLINE void Sum16x1(const uint8_t * src, size_t, uint16_t * dst)
56
0
        {
57
0
            __m256i sum0 = Load<true>((__m256i*)dst + 0);
58
0
            __m256i sum1 = Load<true>((__m256i*)dst + 1);
59
0
            __m256i src0 = Load<align>((__m256i*)src);
60
0
            sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO));
61
0
            sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO));
62
0
            Store<true>((__m256i*)dst + 0, sum0);
63
0
            Store<true>((__m256i*)dst + 1, sum1);
64
0
        }
Unexecuted instantiation: void Simd::Avx2::Sum16x1<true>(unsigned char const*, unsigned long, unsigned short*)
Unexecuted instantiation: void Simd::Avx2::Sum16x1<false>(unsigned char const*, unsigned long, unsigned short*)
65
66
        template <bool align> SIMD_INLINE void Sum16x4(const uint8_t * src, size_t stride, uint16_t * dst)
67
0
        {
68
0
            __m256i sum0 = Load<true>((__m256i*)dst + 0);
69
0
            __m256i sum1 = Load<true>((__m256i*)dst + 1);
70
0
            __m256i src0 = Load<align>((__m256i*)src);
71
0
            sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO));
72
0
            sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO));
73
0
            src0 = Load<align>((__m256i*)(src + 1 * stride));
74
0
            sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO));
75
0
            sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO));
76
0
            src0 = Load<align>((__m256i*)(src + 2 * stride));
77
0
            sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO));
78
0
            sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO));
79
0
            src0 = Load<align>((__m256i*)(src + 3 * stride));
80
0
            sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO));
81
0
            sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO));
82
0
            Store<true>((__m256i*)dst + 0, sum0);
83
0
            Store<true>((__m256i*)dst + 1, sum1);
84
0
        }
Unexecuted instantiation: void Simd::Avx2::Sum16x4<true>(unsigned char const*, unsigned long, unsigned short*)
Unexecuted instantiation: void Simd::Avx2::Sum16x4<false>(unsigned char const*, unsigned long, unsigned short*)
85
86
        template <bool align> SIMD_INLINE void Sum16x8(const uint8_t * src, size_t stride, uint16_t * dst)
87
0
        {
88
0
            __m256i sum0 = Load<true>((__m256i*)dst + 0);
89
0
            __m256i sum1 = Load<true>((__m256i*)dst + 1);
90
0
            __m256i src0 = Load<align>((__m256i*)src);
91
0
            sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO));
92
0
            sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO));
93
0
            src0 = Load<align>((__m256i*)(src + 1 * stride));
94
0
            sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO));
95
0
            sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO));
96
0
            src0 = Load<align>((__m256i*)(src + 2 * stride));
97
0
            sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO));
98
0
            sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO));
99
0
            src0 = Load<align>((__m256i*)(src + 3 * stride));
100
0
            sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO));
101
0
            sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO));
102
0
            src0 = Load<align>((__m256i*)(src + 4 * stride));
103
0
            sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO));
104
0
            sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO));
105
0
            src0 = Load<align>((__m256i*)(src + 5 * stride));
106
0
            sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO));
107
0
            sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO));
108
0
            src0 = Load<align>((__m256i*)(src + 6 * stride));
109
0
            sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO));
110
0
            sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO));
111
0
            src0 = Load<align>((__m256i*)(src + 7 * stride));
112
0
            sum0 = _mm256_add_epi16(sum0, _mm256_unpacklo_epi8(src0, K_ZERO));
113
0
            sum1 = _mm256_add_epi16(sum1, _mm256_unpackhi_epi8(src0, K_ZERO));
114
0
            Store<true>((__m256i*)dst + 0, sum0);
115
0
            Store<true>((__m256i*)dst + 1, sum1);
116
0
        }
Unexecuted instantiation: void Simd::Avx2::Sum16x8<true>(unsigned char const*, unsigned long, unsigned short*)
Unexecuted instantiation: void Simd::Avx2::Sum16x8<false>(unsigned char const*, unsigned long, unsigned short*)
117
118
        SIMD_INLINE void Sum16(__m256i src8, uint16_t * sums16)
119
0
        {
120
0
            Store<true>((__m256i*)sums16 + 0, _mm256_add_epi16(Load<true>((__m256i*)sums16 + 0), _mm256_unpacklo_epi8(src8, K_ZERO)));
121
0
            Store<true>((__m256i*)sums16 + 1, _mm256_add_epi16(Load<true>((__m256i*)sums16 + 1), _mm256_unpackhi_epi8(src8, K_ZERO)));
122
0
        }
123
124
        SIMD_INLINE void Sum16To32(const uint16_t * src, uint32_t * dst)
125
0
        {
126
0
            __m256i lo = LoadPermuted<true>((__m256i*)src + 0);
127
0
            __m256i hi = LoadPermuted<true>((__m256i*)src + 1);
128
0
            Store<true>((__m256i*)dst + 0, _mm256_add_epi32(Load<true>((__m256i*)dst + 0), _mm256_unpacklo_epi16(lo, K_ZERO)));
129
0
            Store<true>((__m256i*)dst + 1, _mm256_add_epi32(Load<true>((__m256i*)dst + 1), _mm256_unpacklo_epi16(hi, K_ZERO)));
130
0
            Store<true>((__m256i*)dst + 2, _mm256_add_epi32(Load<true>((__m256i*)dst + 2), _mm256_unpackhi_epi16(lo, K_ZERO)));
131
0
            Store<true>((__m256i*)dst + 3, _mm256_add_epi32(Load<true>((__m256i*)dst + 3), _mm256_unpackhi_epi16(hi, K_ZERO)));
132
0
        }
133
134
        template <bool align> void GetColSums(const uint8_t * src, size_t stride, size_t width, size_t height, uint32_t * sums)
135
0
        {
136
0
            size_t alignedLoWidth = AlignLo(width, A);
137
0
            size_t alignedHiWidth = AlignHi(width, A);
138
0
            size_t stepSize = SCHAR_MAX + 1;
139
0
            size_t stepCount = (height + SCHAR_MAX) / stepSize;
140
141
0
            Buffer buffer(alignedHiWidth);
142
0
            memset(buffer.sums32, 0, sizeof(uint32_t)*alignedHiWidth);
143
0
            for (size_t step = 0; step < stepCount; ++step)
144
0
            {
145
0
                size_t rowStart = step*stepSize;
146
0
                size_t rowEnd = Min(rowStart + stepSize, height);
147
148
0
                size_t rowEnd4 = AlignLo(rowEnd, 4);
149
0
                size_t rowEnd8 = AlignLo(rowEnd, 8);
150
151
0
                memset(buffer.sums16, 0, sizeof(uint16_t)*alignedHiWidth);
152
0
                size_t row = rowStart;
153
0
                for (; row < rowEnd8; row += 8)
154
0
                {
155
0
                    for (size_t col = 0; col < alignedLoWidth; col += A)
156
0
                        Sum16x8<align>(src + col, stride, buffer.sums16 + col);
157
0
                    if (alignedLoWidth != width)
158
0
                        Sum16x8<false>(src + width - A, stride, buffer.sums16 + alignedLoWidth);
159
0
                    src += 8 * stride;
160
0
                }
161
0
                for (; row < rowEnd4; row += 4)
162
0
                {
163
0
                    for (size_t col = 0; col < alignedLoWidth; col += A)
164
0
                        Sum16x4<align>(src + col, stride, buffer.sums16 + col);
165
0
                    if (alignedLoWidth != width)
166
0
                        Sum16x4<false>(src + width - A, stride, buffer.sums16 + alignedLoWidth);
167
0
                    src += 4 * stride;
168
0
                }
169
0
                for (; row < rowEnd; ++row)
170
0
                {
171
0
                    for (size_t col = 0; col < alignedLoWidth; col += A)
172
0
                        Sum16x1<align>(src + col, stride, buffer.sums16 + col);
173
0
                    if (alignedLoWidth != width)
174
0
                        Sum16x1<false>(src + width - A, stride, buffer.sums16 + alignedLoWidth);
175
0
                    src += stride;
176
0
                }
177
178
0
                for (size_t col = 0; col < alignedHiWidth; col += A)
179
0
                    Sum16To32(buffer.sums16 + col, buffer.sums32 + col);
180
0
            }
181
0
            memcpy(sums, buffer.sums32, sizeof(uint32_t)*alignedLoWidth);
182
0
            if (alignedLoWidth != width)
183
0
                memcpy(sums + alignedLoWidth, buffer.sums32 + alignedLoWidth + alignedHiWidth - width, sizeof(uint32_t)*(width - alignedLoWidth));
184
0
        }
Unexecuted instantiation: void Simd::Avx2::GetColSums<true>(unsigned char const*, unsigned long, unsigned long, unsigned long, unsigned int*)
Unexecuted instantiation: void Simd::Avx2::GetColSums<false>(unsigned char const*, unsigned long, unsigned long, unsigned long, unsigned int*)
185
186
        void GetColSums(const uint8_t * src, size_t stride, size_t width, size_t height, uint32_t * sums)
187
0
        {
188
0
            if (Aligned(src) && Aligned(stride))
189
0
                GetColSums<true>(src, stride, width, height, sums);
190
0
            else
191
0
                GetColSums<false>(src, stride, width, height, sums);
192
0
        }
193
194
        //-----------------------------------------------------------------------------------------
195
196
        template <bool align> void GetAbsDxColSums(const uint8_t * src, size_t stride, size_t width, size_t height, uint32_t * sums)
197
0
        {
198
0
            width--;
199
0
            size_t alignedLoWidth = AlignLo(width, A);
200
0
            size_t alignedHiWidth = AlignHi(width, A);
201
0
            size_t stepSize = SCHAR_MAX + 1;
202
0
            size_t stepCount = (height + SCHAR_MAX) / stepSize;
203
204
0
            Buffer buffer(alignedHiWidth);
205
0
            memset(buffer.sums32, 0, sizeof(uint32_t)*alignedHiWidth);
206
0
            for (size_t step = 0; step < stepCount; ++step)
207
0
            {
208
0
                size_t rowStart = step*stepSize;
209
0
                size_t rowEnd = Min(rowStart + stepSize, height);
210
211
0
                memset(buffer.sums16, 0, sizeof(uint16_t)*alignedHiWidth);
212
0
                for (size_t row = rowStart; row < rowEnd; ++row)
213
0
                {
214
0
                    for (size_t col = 0; col < alignedLoWidth; col += A)
215
0
                    {
216
0
                        __m256i _src0 = Load<align>((__m256i*)(src + col + 0));
217
0
                        __m256i _src1 = Load<false>((__m256i*)(src + col + 1));
218
0
                        Sum16(AbsDifferenceU8(_src0, _src1), buffer.sums16 + col);
219
0
                    }
220
0
                    if (alignedLoWidth != width)
221
0
                    {
222
0
                        __m256i _src0 = Load<false>((__m256i*)(src + width - A + 0));
223
0
                        __m256i _src1 = Load<false>((__m256i*)(src + width - A + 1));
224
0
                        Sum16(AbsDifferenceU8(_src0, _src1), buffer.sums16 + alignedLoWidth);
225
0
                    }
226
0
                    src += stride;
227
0
                }
228
229
0
                for (size_t col = 0; col < alignedHiWidth; col += A)
230
0
                    Sum16To32(buffer.sums16 + col, buffer.sums32 + col);
231
0
            }
232
0
            memcpy(sums, buffer.sums32, sizeof(uint32_t)*alignedLoWidth);
233
0
            if (alignedLoWidth != width)
234
0
                memcpy(sums + alignedLoWidth, buffer.sums32 + alignedLoWidth + alignedHiWidth - width, sizeof(uint32_t)*(width - alignedLoWidth));
235
0
            sums[width] = 0;
236
0
        }
Unexecuted instantiation: void Simd::Avx2::GetAbsDxColSums<true>(unsigned char const*, unsigned long, unsigned long, unsigned long, unsigned int*)
Unexecuted instantiation: void Simd::Avx2::GetAbsDxColSums<false>(unsigned char const*, unsigned long, unsigned long, unsigned long, unsigned int*)
237
238
        void GetAbsDxColSums(const uint8_t * src, size_t stride, size_t width, size_t height, uint32_t * sums)
239
0
        {
240
0
            if (Aligned(src) && Aligned(stride))
241
0
                GetAbsDxColSums<true>(src, stride, width, height, sums);
242
0
            else
243
0
                GetAbsDxColSums<false>(src, stride, width, height, sums);
244
0
        }
245
    }
246
#endif
247
}