/src/libwebp/sharpyuv/sharpyuv_sse2.c
Line | Count | Source (jump to first uncovered line) |
1 | | // Copyright 2022 Google Inc. All Rights Reserved. |
2 | | // |
3 | | // Use of this source code is governed by a BSD-style license |
4 | | // that can be found in the COPYING file in the root of the source |
5 | | // tree. An additional intellectual property rights grant can be found |
6 | | // in the file PATENTS. All contributing project authors may |
7 | | // be found in the AUTHORS file in the root of the source tree. |
8 | | // ----------------------------------------------------------------------------- |
9 | | // |
10 | | // Speed-critical functions for Sharp YUV. |
11 | | // |
12 | | // Author: Skal (pascal.massimino@gmail.com) |
13 | | |
14 | | #include "sharpyuv/sharpyuv_dsp.h" |
15 | | |
16 | | #if defined(WEBP_USE_SSE2) |
17 | | #include <stdlib.h> |
18 | | #include <emmintrin.h> |
19 | | |
20 | 0 | static uint16_t clip_SSE2(int v, int max) { |
21 | 0 | return (v < 0) ? 0 : (v > max) ? max : (uint16_t)v; |
22 | 0 | } |
23 | | |
24 | | static uint64_t SharpYuvUpdateY_SSE2(const uint16_t* ref, const uint16_t* src, |
25 | 0 | uint16_t* dst, int len, int bit_depth) { |
26 | 0 | const int max_y = (1 << bit_depth) - 1; |
27 | 0 | uint64_t diff = 0; |
28 | 0 | uint32_t tmp[4]; |
29 | 0 | int i; |
30 | 0 | const __m128i zero = _mm_setzero_si128(); |
31 | 0 | const __m128i max = _mm_set1_epi16(max_y); |
32 | 0 | const __m128i one = _mm_set1_epi16(1); |
33 | 0 | __m128i sum = zero; |
34 | |
|
35 | 0 | for (i = 0; i + 8 <= len; i += 8) { |
36 | 0 | const __m128i A = _mm_loadu_si128((const __m128i*)(ref + i)); |
37 | 0 | const __m128i B = _mm_loadu_si128((const __m128i*)(src + i)); |
38 | 0 | const __m128i C = _mm_loadu_si128((const __m128i*)(dst + i)); |
39 | 0 | const __m128i D = _mm_sub_epi16(A, B); // diff_y |
40 | 0 | const __m128i E = _mm_cmpgt_epi16(zero, D); // sign (-1 or 0) |
41 | 0 | const __m128i F = _mm_add_epi16(C, D); // new_y |
42 | 0 | const __m128i G = _mm_or_si128(E, one); // -1 or 1 |
43 | 0 | const __m128i H = _mm_max_epi16(_mm_min_epi16(F, max), zero); |
44 | 0 | const __m128i I = _mm_madd_epi16(D, G); // sum(abs(...)) |
45 | 0 | _mm_storeu_si128((__m128i*)(dst + i), H); |
46 | 0 | sum = _mm_add_epi32(sum, I); |
47 | 0 | } |
48 | 0 | _mm_storeu_si128((__m128i*)tmp, sum); |
49 | 0 | diff = tmp[3] + tmp[2] + tmp[1] + tmp[0]; |
50 | 0 | for (; i < len; ++i) { |
51 | 0 | const int diff_y = ref[i] - src[i]; |
52 | 0 | const int new_y = (int)dst[i] + diff_y; |
53 | 0 | dst[i] = clip_SSE2(new_y, max_y); |
54 | 0 | diff += (uint64_t)abs(diff_y); |
55 | 0 | } |
56 | 0 | return diff; |
57 | 0 | } |
58 | | |
59 | | static void SharpYuvUpdateRGB_SSE2(const int16_t* ref, const int16_t* src, |
60 | 0 | int16_t* dst, int len) { |
61 | 0 | int i = 0; |
62 | 0 | for (i = 0; i + 8 <= len; i += 8) { |
63 | 0 | const __m128i A = _mm_loadu_si128((const __m128i*)(ref + i)); |
64 | 0 | const __m128i B = _mm_loadu_si128((const __m128i*)(src + i)); |
65 | 0 | const __m128i C = _mm_loadu_si128((const __m128i*)(dst + i)); |
66 | 0 | const __m128i D = _mm_sub_epi16(A, B); // diff_uv |
67 | 0 | const __m128i E = _mm_add_epi16(C, D); // new_uv |
68 | 0 | _mm_storeu_si128((__m128i*)(dst + i), E); |
69 | 0 | } |
70 | 0 | for (; i < len; ++i) { |
71 | 0 | const int diff_uv = ref[i] - src[i]; |
72 | 0 | dst[i] += diff_uv; |
73 | 0 | } |
74 | 0 | } |
75 | | |
76 | | static void SharpYuvFilterRow16_SSE2(const int16_t* A, const int16_t* B, |
77 | | int len, const uint16_t* best_y, |
78 | 0 | uint16_t* out, int bit_depth) { |
79 | 0 | const int max_y = (1 << bit_depth) - 1; |
80 | 0 | int i; |
81 | 0 | const __m128i kCst8 = _mm_set1_epi16(8); |
82 | 0 | const __m128i max = _mm_set1_epi16(max_y); |
83 | 0 | const __m128i zero = _mm_setzero_si128(); |
84 | 0 | for (i = 0; i + 8 <= len; i += 8) { |
85 | 0 | const __m128i a0 = _mm_loadu_si128((const __m128i*)(A + i + 0)); |
86 | 0 | const __m128i a1 = _mm_loadu_si128((const __m128i*)(A + i + 1)); |
87 | 0 | const __m128i b0 = _mm_loadu_si128((const __m128i*)(B + i + 0)); |
88 | 0 | const __m128i b1 = _mm_loadu_si128((const __m128i*)(B + i + 1)); |
89 | 0 | const __m128i a0b1 = _mm_add_epi16(a0, b1); |
90 | 0 | const __m128i a1b0 = _mm_add_epi16(a1, b0); |
91 | 0 | const __m128i a0a1b0b1 = _mm_add_epi16(a0b1, a1b0); // A0+A1+B0+B1 |
92 | 0 | const __m128i a0a1b0b1_8 = _mm_add_epi16(a0a1b0b1, kCst8); |
93 | 0 | const __m128i a0b1_2 = _mm_add_epi16(a0b1, a0b1); // 2*(A0+B1) |
94 | 0 | const __m128i a1b0_2 = _mm_add_epi16(a1b0, a1b0); // 2*(A1+B0) |
95 | 0 | const __m128i c0 = _mm_srai_epi16(_mm_add_epi16(a0b1_2, a0a1b0b1_8), 3); |
96 | 0 | const __m128i c1 = _mm_srai_epi16(_mm_add_epi16(a1b0_2, a0a1b0b1_8), 3); |
97 | 0 | const __m128i d0 = _mm_add_epi16(c1, a0); |
98 | 0 | const __m128i d1 = _mm_add_epi16(c0, a1); |
99 | 0 | const __m128i e0 = _mm_srai_epi16(d0, 1); |
100 | 0 | const __m128i e1 = _mm_srai_epi16(d1, 1); |
101 | 0 | const __m128i f0 = _mm_unpacklo_epi16(e0, e1); |
102 | 0 | const __m128i f1 = _mm_unpackhi_epi16(e0, e1); |
103 | 0 | const __m128i g0 = _mm_loadu_si128((const __m128i*)(best_y + 2 * i + 0)); |
104 | 0 | const __m128i g1 = _mm_loadu_si128((const __m128i*)(best_y + 2 * i + 8)); |
105 | 0 | const __m128i h0 = _mm_add_epi16(g0, f0); |
106 | 0 | const __m128i h1 = _mm_add_epi16(g1, f1); |
107 | 0 | const __m128i i0 = _mm_max_epi16(_mm_min_epi16(h0, max), zero); |
108 | 0 | const __m128i i1 = _mm_max_epi16(_mm_min_epi16(h1, max), zero); |
109 | 0 | _mm_storeu_si128((__m128i*)(out + 2 * i + 0), i0); |
110 | 0 | _mm_storeu_si128((__m128i*)(out + 2 * i + 8), i1); |
111 | 0 | } |
112 | 0 | for (; i < len; ++i) { |
113 | | // (9 * A0 + 3 * A1 + 3 * B0 + B1 + 8) >> 4 = |
114 | | // = (8 * A0 + 2 * (A1 + B0) + (A0 + A1 + B0 + B1 + 8)) >> 4 |
115 | | // We reuse the common sub-expressions. |
116 | 0 | const int a0b1 = A[i + 0] + B[i + 1]; |
117 | 0 | const int a1b0 = A[i + 1] + B[i + 0]; |
118 | 0 | const int a0a1b0b1 = a0b1 + a1b0 + 8; |
119 | 0 | const int v0 = (8 * A[i + 0] + 2 * a1b0 + a0a1b0b1) >> 4; |
120 | 0 | const int v1 = (8 * A[i + 1] + 2 * a0b1 + a0a1b0b1) >> 4; |
121 | 0 | out[2 * i + 0] = clip_SSE2(best_y[2 * i + 0] + v0, max_y); |
122 | 0 | out[2 * i + 1] = clip_SSE2(best_y[2 * i + 1] + v1, max_y); |
123 | 0 | } |
124 | 0 | } |
125 | | |
126 | 0 | static WEBP_INLINE __m128i s16_to_s32(__m128i in) { |
127 | 0 | return _mm_srai_epi32(_mm_unpacklo_epi16(in, in), 16); |
128 | 0 | } |
129 | | |
130 | | static void SharpYuvFilterRow32_SSE2(const int16_t* A, const int16_t* B, |
131 | | int len, const uint16_t* best_y, |
132 | 0 | uint16_t* out, int bit_depth) { |
133 | 0 | const int max_y = (1 << bit_depth) - 1; |
134 | 0 | int i; |
135 | 0 | const __m128i kCst8 = _mm_set1_epi32(8); |
136 | 0 | const __m128i max = _mm_set1_epi16(max_y); |
137 | 0 | const __m128i zero = _mm_setzero_si128(); |
138 | 0 | for (i = 0; i + 4 <= len; i += 4) { |
139 | 0 | const __m128i a0 = s16_to_s32(_mm_loadl_epi64((const __m128i*)(A + i + 0))); |
140 | 0 | const __m128i a1 = s16_to_s32(_mm_loadl_epi64((const __m128i*)(A + i + 1))); |
141 | 0 | const __m128i b0 = s16_to_s32(_mm_loadl_epi64((const __m128i*)(B + i + 0))); |
142 | 0 | const __m128i b1 = s16_to_s32(_mm_loadl_epi64((const __m128i*)(B + i + 1))); |
143 | 0 | const __m128i a0b1 = _mm_add_epi32(a0, b1); |
144 | 0 | const __m128i a1b0 = _mm_add_epi32(a1, b0); |
145 | 0 | const __m128i a0a1b0b1 = _mm_add_epi32(a0b1, a1b0); // A0+A1+B0+B1 |
146 | 0 | const __m128i a0a1b0b1_8 = _mm_add_epi32(a0a1b0b1, kCst8); |
147 | 0 | const __m128i a0b1_2 = _mm_add_epi32(a0b1, a0b1); // 2*(A0+B1) |
148 | 0 | const __m128i a1b0_2 = _mm_add_epi32(a1b0, a1b0); // 2*(A1+B0) |
149 | 0 | const __m128i c0 = _mm_srai_epi32(_mm_add_epi32(a0b1_2, a0a1b0b1_8), 3); |
150 | 0 | const __m128i c1 = _mm_srai_epi32(_mm_add_epi32(a1b0_2, a0a1b0b1_8), 3); |
151 | 0 | const __m128i d0 = _mm_add_epi32(c1, a0); |
152 | 0 | const __m128i d1 = _mm_add_epi32(c0, a1); |
153 | 0 | const __m128i e0 = _mm_srai_epi32(d0, 1); |
154 | 0 | const __m128i e1 = _mm_srai_epi32(d1, 1); |
155 | 0 | const __m128i f0 = _mm_unpacklo_epi32(e0, e1); |
156 | 0 | const __m128i f1 = _mm_unpackhi_epi32(e0, e1); |
157 | 0 | const __m128i g = _mm_loadu_si128((const __m128i*)(best_y + 2 * i + 0)); |
158 | 0 | const __m128i h_16 = _mm_add_epi16(g, _mm_packs_epi32(f0, f1)); |
159 | 0 | const __m128i final = _mm_max_epi16(_mm_min_epi16(h_16, max), zero); |
160 | 0 | _mm_storeu_si128((__m128i*)(out + 2 * i + 0), final); |
161 | 0 | } |
162 | 0 | for (; i < len; ++i) { |
163 | | // (9 * A0 + 3 * A1 + 3 * B0 + B1 + 8) >> 4 = |
164 | | // = (8 * A0 + 2 * (A1 + B0) + (A0 + A1 + B0 + B1 + 8)) >> 4 |
165 | | // We reuse the common sub-expressions. |
166 | 0 | const int a0b1 = A[i + 0] + B[i + 1]; |
167 | 0 | const int a1b0 = A[i + 1] + B[i + 0]; |
168 | 0 | const int a0a1b0b1 = a0b1 + a1b0 + 8; |
169 | 0 | const int v0 = (8 * A[i + 0] + 2 * a1b0 + a0a1b0b1) >> 4; |
170 | 0 | const int v1 = (8 * A[i + 1] + 2 * a0b1 + a0a1b0b1) >> 4; |
171 | 0 | out[2 * i + 0] = clip_SSE2(best_y[2 * i + 0] + v0, max_y); |
172 | 0 | out[2 * i + 1] = clip_SSE2(best_y[2 * i + 1] + v1, max_y); |
173 | 0 | } |
174 | 0 | } |
175 | | |
176 | | static void SharpYuvFilterRow_SSE2(const int16_t* A, const int16_t* B, int len, |
177 | | const uint16_t* best_y, uint16_t* out, |
178 | 0 | int bit_depth) { |
179 | 0 | if (bit_depth <= 10) { |
180 | 0 | SharpYuvFilterRow16_SSE2(A, B, len, best_y, out, bit_depth); |
181 | 0 | } else { |
182 | 0 | SharpYuvFilterRow32_SSE2(A, B, len, best_y, out, bit_depth); |
183 | 0 | } |
184 | 0 | } |
185 | | |
186 | | //------------------------------------------------------------------------------ |
187 | | |
188 | | extern void InitSharpYuvSSE2(void); |
189 | | |
190 | 0 | WEBP_TSAN_IGNORE_FUNCTION void InitSharpYuvSSE2(void) { |
191 | 0 | SharpYuvUpdateY = SharpYuvUpdateY_SSE2; |
192 | 0 | SharpYuvUpdateRGB = SharpYuvUpdateRGB_SSE2; |
193 | 0 | SharpYuvFilterRow = SharpYuvFilterRow_SSE2; |
194 | 0 | } |
195 | | #else // !WEBP_USE_SSE2 |
196 | | |
197 | | extern void InitSharpYuvSSE2(void); |
198 | | |
199 | | void InitSharpYuvSSE2(void) {} |
200 | | |
201 | | #endif // WEBP_USE_SSE2 |