/src/libwebp/src/dsp/lossless_sse2.c
Line | Count | Source (jump to first uncovered line) |
1 | | // Copyright 2014 Google Inc. All Rights Reserved. |
2 | | // |
3 | | // Use of this source code is governed by a BSD-style license |
4 | | // that can be found in the COPYING file in the root of the source |
5 | | // tree. An additional intellectual property rights grant can be found |
6 | | // in the file PATENTS. All contributing project authors may |
7 | | // be found in the AUTHORS file in the root of the source tree. |
8 | | // ----------------------------------------------------------------------------- |
9 | | // |
10 | | // SSE2 variant of methods for lossless decoder |
11 | | // |
12 | | // Author: Skal (pascal.massimino@gmail.com) |
13 | | |
14 | | #include "src/dsp/dsp.h" |
15 | | |
16 | | #if defined(WEBP_USE_SSE2) |
17 | | |
18 | | #include <emmintrin.h> |
19 | | #include <string.h> |
20 | | |
21 | | #include "src/dsp/common_sse2.h" |
22 | | #include "src/dsp/cpu.h" |
23 | | #include "src/dsp/lossless.h" |
24 | | #include "src/dsp/lossless_common.h" |
25 | | #include "src/webp/format_constants.h" |
26 | | #include "src/webp/types.h" |
27 | | |
28 | | //------------------------------------------------------------------------------ |
29 | | // Predictor Transform |
30 | | |
31 | | static WEBP_INLINE uint32_t ClampedAddSubtractFull_SSE2(uint32_t c0, |
32 | | uint32_t c1, |
33 | 0 | uint32_t c2) { |
34 | 0 | const __m128i zero = _mm_setzero_si128(); |
35 | 0 | const __m128i C0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)c0), zero); |
36 | 0 | const __m128i C1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)c1), zero); |
37 | 0 | const __m128i C2 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)c2), zero); |
38 | 0 | const __m128i V1 = _mm_add_epi16(C0, C1); |
39 | 0 | const __m128i V2 = _mm_sub_epi16(V1, C2); |
40 | 0 | const __m128i b = _mm_packus_epi16(V2, V2); |
41 | 0 | return (uint32_t)_mm_cvtsi128_si32(b); |
42 | 0 | } |
43 | | |
44 | | static WEBP_INLINE uint32_t ClampedAddSubtractHalf_SSE2(uint32_t c0, |
45 | | uint32_t c1, |
46 | 0 | uint32_t c2) { |
47 | 0 | const __m128i zero = _mm_setzero_si128(); |
48 | 0 | const __m128i C0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)c0), zero); |
49 | 0 | const __m128i C1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)c1), zero); |
50 | 0 | const __m128i B0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)c2), zero); |
51 | 0 | const __m128i avg = _mm_add_epi16(C1, C0); |
52 | 0 | const __m128i A0 = _mm_srli_epi16(avg, 1); |
53 | 0 | const __m128i A1 = _mm_sub_epi16(A0, B0); |
54 | 0 | const __m128i BgtA = _mm_cmpgt_epi16(B0, A0); |
55 | 0 | const __m128i A2 = _mm_sub_epi16(A1, BgtA); |
56 | 0 | const __m128i A3 = _mm_srai_epi16(A2, 1); |
57 | 0 | const __m128i A4 = _mm_add_epi16(A0, A3); |
58 | 0 | const __m128i A5 = _mm_packus_epi16(A4, A4); |
59 | 0 | return (uint32_t)_mm_cvtsi128_si32(A5); |
60 | 0 | } |
61 | | |
62 | 0 | static WEBP_INLINE uint32_t Select_SSE2(uint32_t a, uint32_t b, uint32_t c) { |
63 | 0 | int pa_minus_pb; |
64 | 0 | const __m128i zero = _mm_setzero_si128(); |
65 | 0 | const __m128i A0 = _mm_cvtsi32_si128((int)a); |
66 | 0 | const __m128i B0 = _mm_cvtsi32_si128((int)b); |
67 | 0 | const __m128i C0 = _mm_cvtsi32_si128((int)c); |
68 | 0 | const __m128i AC0 = _mm_subs_epu8(A0, C0); |
69 | 0 | const __m128i CA0 = _mm_subs_epu8(C0, A0); |
70 | 0 | const __m128i BC0 = _mm_subs_epu8(B0, C0); |
71 | 0 | const __m128i CB0 = _mm_subs_epu8(C0, B0); |
72 | 0 | const __m128i AC = _mm_or_si128(AC0, CA0); |
73 | 0 | const __m128i BC = _mm_or_si128(BC0, CB0); |
74 | 0 | const __m128i pa = _mm_unpacklo_epi8(AC, zero); // |a - c| |
75 | 0 | const __m128i pb = _mm_unpacklo_epi8(BC, zero); // |b - c| |
76 | 0 | const __m128i diff = _mm_sub_epi16(pb, pa); |
77 | 0 | { |
78 | 0 | int16_t out[8]; |
79 | 0 | _mm_storeu_si128((__m128i*)out, diff); |
80 | 0 | pa_minus_pb = out[0] + out[1] + out[2] + out[3]; |
81 | 0 | } |
82 | 0 | return (pa_minus_pb <= 0) ? a : b; |
83 | 0 | } |
84 | | |
85 | | static WEBP_INLINE void Average2_m128i(const __m128i* const a0, |
86 | | const __m128i* const a1, |
87 | 0 | __m128i* const avg) { |
88 | | // (a + b) >> 1 = ((a + b + 1) >> 1) - ((a ^ b) & 1) |
89 | 0 | const __m128i ones = _mm_set1_epi8(1); |
90 | 0 | const __m128i avg1 = _mm_avg_epu8(*a0, *a1); |
91 | 0 | const __m128i one = _mm_and_si128(_mm_xor_si128(*a0, *a1), ones); |
92 | 0 | *avg = _mm_sub_epi8(avg1, one); |
93 | 0 | } |
94 | | |
95 | | static WEBP_INLINE void Average2_uint32_SSE2(const uint32_t a0, |
96 | | const uint32_t a1, |
97 | 0 | __m128i* const avg) { |
98 | | // (a + b) >> 1 = ((a + b + 1) >> 1) - ((a ^ b) & 1) |
99 | 0 | const __m128i ones = _mm_set1_epi8(1); |
100 | 0 | const __m128i A0 = _mm_cvtsi32_si128((int)a0); |
101 | 0 | const __m128i A1 = _mm_cvtsi32_si128((int)a1); |
102 | 0 | const __m128i avg1 = _mm_avg_epu8(A0, A1); |
103 | 0 | const __m128i one = _mm_and_si128(_mm_xor_si128(A0, A1), ones); |
104 | 0 | *avg = _mm_sub_epi8(avg1, one); |
105 | 0 | } |
106 | | |
107 | 0 | static WEBP_INLINE __m128i Average2_uint32_16_SSE2(uint32_t a0, uint32_t a1) { |
108 | 0 | const __m128i zero = _mm_setzero_si128(); |
109 | 0 | const __m128i A0 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)a0), zero); |
110 | 0 | const __m128i A1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)a1), zero); |
111 | 0 | const __m128i sum = _mm_add_epi16(A1, A0); |
112 | 0 | return _mm_srli_epi16(sum, 1); |
113 | 0 | } |
114 | | |
115 | 0 | static WEBP_INLINE uint32_t Average2_SSE2(uint32_t a0, uint32_t a1) { |
116 | 0 | __m128i output; |
117 | 0 | Average2_uint32_SSE2(a0, a1, &output); |
118 | 0 | return (uint32_t)_mm_cvtsi128_si32(output); |
119 | 0 | } |
120 | | |
121 | | static WEBP_INLINE uint32_t Average3_SSE2(uint32_t a0, uint32_t a1, |
122 | 0 | uint32_t a2) { |
123 | 0 | const __m128i zero = _mm_setzero_si128(); |
124 | 0 | const __m128i avg1 = Average2_uint32_16_SSE2(a0, a2); |
125 | 0 | const __m128i A1 = _mm_unpacklo_epi8(_mm_cvtsi32_si128((int)a1), zero); |
126 | 0 | const __m128i sum = _mm_add_epi16(avg1, A1); |
127 | 0 | const __m128i avg2 = _mm_srli_epi16(sum, 1); |
128 | 0 | const __m128i A2 = _mm_packus_epi16(avg2, avg2); |
129 | 0 | return (uint32_t)_mm_cvtsi128_si32(A2); |
130 | 0 | } |
131 | | |
132 | | static WEBP_INLINE uint32_t Average4_SSE2(uint32_t a0, uint32_t a1, |
133 | 0 | uint32_t a2, uint32_t a3) { |
134 | 0 | const __m128i avg1 = Average2_uint32_16_SSE2(a0, a1); |
135 | 0 | const __m128i avg2 = Average2_uint32_16_SSE2(a2, a3); |
136 | 0 | const __m128i sum = _mm_add_epi16(avg2, avg1); |
137 | 0 | const __m128i avg3 = _mm_srli_epi16(sum, 1); |
138 | 0 | const __m128i A0 = _mm_packus_epi16(avg3, avg3); |
139 | 0 | return (uint32_t)_mm_cvtsi128_si32(A0); |
140 | 0 | } |
141 | | |
142 | | static uint32_t Predictor5_SSE2(const uint32_t* const left, |
143 | 0 | const uint32_t* const top) { |
144 | 0 | const uint32_t pred = Average3_SSE2(*left, top[0], top[1]); |
145 | 0 | return pred; |
146 | 0 | } |
147 | | static uint32_t Predictor6_SSE2(const uint32_t* const left, |
148 | 0 | const uint32_t* const top) { |
149 | 0 | const uint32_t pred = Average2_SSE2(*left, top[-1]); |
150 | 0 | return pred; |
151 | 0 | } |
152 | | static uint32_t Predictor7_SSE2(const uint32_t* const left, |
153 | 0 | const uint32_t* const top) { |
154 | 0 | const uint32_t pred = Average2_SSE2(*left, top[0]); |
155 | 0 | return pred; |
156 | 0 | } |
157 | | static uint32_t Predictor8_SSE2(const uint32_t* const left, |
158 | 0 | const uint32_t* const top) { |
159 | 0 | const uint32_t pred = Average2_SSE2(top[-1], top[0]); |
160 | 0 | (void)left; |
161 | 0 | return pred; |
162 | 0 | } |
163 | | static uint32_t Predictor9_SSE2(const uint32_t* const left, |
164 | 0 | const uint32_t* const top) { |
165 | 0 | const uint32_t pred = Average2_SSE2(top[0], top[1]); |
166 | 0 | (void)left; |
167 | 0 | return pred; |
168 | 0 | } |
169 | | static uint32_t Predictor10_SSE2(const uint32_t* const left, |
170 | 0 | const uint32_t* const top) { |
171 | 0 | const uint32_t pred = Average4_SSE2(*left, top[-1], top[0], top[1]); |
172 | 0 | return pred; |
173 | 0 | } |
174 | | static uint32_t Predictor11_SSE2(const uint32_t* const left, |
175 | 0 | const uint32_t* const top) { |
176 | 0 | const uint32_t pred = Select_SSE2(top[0], *left, top[-1]); |
177 | 0 | return pred; |
178 | 0 | } |
179 | | static uint32_t Predictor12_SSE2(const uint32_t* const left, |
180 | 0 | const uint32_t* const top) { |
181 | 0 | const uint32_t pred = ClampedAddSubtractFull_SSE2(*left, top[0], top[-1]); |
182 | 0 | return pred; |
183 | 0 | } |
184 | | static uint32_t Predictor13_SSE2(const uint32_t* const left, |
185 | 0 | const uint32_t* const top) { |
186 | 0 | const uint32_t pred = ClampedAddSubtractHalf_SSE2(*left, top[0], top[-1]); |
187 | 0 | return pred; |
188 | 0 | } |
189 | | |
190 | | // Batch versions of those functions. |
191 | | |
192 | | // Predictor0: ARGB_BLACK. |
193 | | static void PredictorAdd0_SSE2(const uint32_t* in, const uint32_t* upper, |
194 | 0 | int num_pixels, uint32_t* WEBP_RESTRICT out) { |
195 | 0 | int i; |
196 | 0 | const __m128i black = _mm_set1_epi32((int)ARGB_BLACK); |
197 | 0 | for (i = 0; i + 4 <= num_pixels; i += 4) { |
198 | 0 | const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); |
199 | 0 | const __m128i res = _mm_add_epi8(src, black); |
200 | 0 | _mm_storeu_si128((__m128i*)&out[i], res); |
201 | 0 | } |
202 | 0 | if (i != num_pixels) { |
203 | 0 | VP8LPredictorsAdd_C[0](in + i, NULL, num_pixels - i, out + i); |
204 | 0 | } |
205 | 0 | (void)upper; |
206 | 0 | } |
207 | | |
208 | | // Predictor1: left. |
209 | | static void PredictorAdd1_SSE2(const uint32_t* in, const uint32_t* upper, |
210 | 0 | int num_pixels, uint32_t* WEBP_RESTRICT out) { |
211 | 0 | int i; |
212 | 0 | __m128i prev = _mm_set1_epi32((int)out[-1]); |
213 | 0 | for (i = 0; i + 4 <= num_pixels; i += 4) { |
214 | | // a | b | c | d |
215 | 0 | const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); |
216 | | // 0 | a | b | c |
217 | 0 | const __m128i shift0 = _mm_slli_si128(src, 4); |
218 | | // a | a + b | b + c | c + d |
219 | 0 | const __m128i sum0 = _mm_add_epi8(src, shift0); |
220 | | // 0 | 0 | a | a + b |
221 | 0 | const __m128i shift1 = _mm_slli_si128(sum0, 8); |
222 | | // a | a + b | a + b + c | a + b + c + d |
223 | 0 | const __m128i sum1 = _mm_add_epi8(sum0, shift1); |
224 | 0 | const __m128i res = _mm_add_epi8(sum1, prev); |
225 | 0 | _mm_storeu_si128((__m128i*)&out[i], res); |
226 | | // replicate prev output on the four lanes |
227 | 0 | prev = _mm_shuffle_epi32(res, (3 << 0) | (3 << 2) | (3 << 4) | (3 << 6)); |
228 | 0 | } |
229 | 0 | if (i != num_pixels) { |
230 | 0 | VP8LPredictorsAdd_C[1](in + i, upper + i, num_pixels - i, out + i); |
231 | 0 | } |
232 | 0 | } |
233 | | |
234 | | // Macro that adds 32-bit integers from IN using mod 256 arithmetic |
235 | | // per 8 bit channel. |
236 | | #define GENERATE_PREDICTOR_1(X, IN) \ |
237 | | static void PredictorAdd##X##_SSE2(const uint32_t* in, const uint32_t* upper, \ |
238 | | int num_pixels, \ |
239 | 0 | uint32_t* WEBP_RESTRICT out) { \ |
240 | 0 | int i; \ |
241 | 0 | for (i = 0; i + 4 <= num_pixels; i += 4) { \ |
242 | 0 | const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); \ |
243 | 0 | const __m128i other = _mm_loadu_si128((const __m128i*)&(IN)); \ |
244 | 0 | const __m128i res = _mm_add_epi8(src, other); \ |
245 | 0 | _mm_storeu_si128((__m128i*)&out[i], res); \ |
246 | 0 | } \ |
247 | 0 | if (i != num_pixels) { \ |
248 | 0 | VP8LPredictorsAdd_C[(X)](in + i, upper + i, num_pixels - i, out + i); \ |
249 | 0 | } \ |
250 | 0 | } Unexecuted instantiation: lossless_sse2.c:PredictorAdd2_SSE2 Unexecuted instantiation: lossless_sse2.c:PredictorAdd3_SSE2 Unexecuted instantiation: lossless_sse2.c:PredictorAdd4_SSE2 |
251 | | |
252 | | // Predictor2: Top. |
253 | | GENERATE_PREDICTOR_1(2, upper[i]) |
254 | | // Predictor3: Top-right. |
255 | | GENERATE_PREDICTOR_1(3, upper[i + 1]) |
256 | | // Predictor4: Top-left. |
257 | | GENERATE_PREDICTOR_1(4, upper[i - 1]) |
258 | | #undef GENERATE_PREDICTOR_1 |
259 | | |
260 | | // Due to averages with integers, values cannot be accumulated in parallel for |
261 | | // predictors 5 to 7. |
262 | | GENERATE_PREDICTOR_ADD(Predictor5_SSE2, PredictorAdd5_SSE2) |
263 | | GENERATE_PREDICTOR_ADD(Predictor6_SSE2, PredictorAdd6_SSE2) |
264 | | GENERATE_PREDICTOR_ADD(Predictor7_SSE2, PredictorAdd7_SSE2) |
265 | | |
266 | | #define GENERATE_PREDICTOR_2(X, IN) \ |
267 | | static void PredictorAdd##X##_SSE2(const uint32_t* in, const uint32_t* upper, \ |
268 | | int num_pixels, \ |
269 | 0 | uint32_t* WEBP_RESTRICT out) { \ |
270 | 0 | int i; \ |
271 | 0 | for (i = 0; i + 4 <= num_pixels; i += 4) { \ |
272 | 0 | const __m128i Tother = _mm_loadu_si128((const __m128i*)&(IN)); \ |
273 | 0 | const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]); \ |
274 | 0 | const __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); \ |
275 | 0 | __m128i avg, res; \ |
276 | 0 | Average2_m128i(&T, &Tother, &avg); \ |
277 | 0 | res = _mm_add_epi8(avg, src); \ |
278 | 0 | _mm_storeu_si128((__m128i*)&out[i], res); \ |
279 | 0 | } \ |
280 | 0 | if (i != num_pixels) { \ |
281 | 0 | VP8LPredictorsAdd_C[(X)](in + i, upper + i, num_pixels - i, out + i); \ |
282 | 0 | } \ |
283 | 0 | } Unexecuted instantiation: lossless_sse2.c:PredictorAdd8_SSE2 Unexecuted instantiation: lossless_sse2.c:PredictorAdd9_SSE2 |
284 | | // Predictor8: average TL T. |
285 | | GENERATE_PREDICTOR_2(8, upper[i - 1]) |
286 | | // Predictor9: average T TR. |
287 | | GENERATE_PREDICTOR_2(9, upper[i + 1]) |
288 | | #undef GENERATE_PREDICTOR_2 |
289 | | |
290 | | // Predictor10: average of (average of (L,TL), average of (T, TR)). |
291 | 0 | #define DO_PRED10(OUT) do { \ |
292 | 0 | __m128i avgLTL, avg; \ |
293 | 0 | Average2_m128i(&L, &TL, &avgLTL); \ |
294 | 0 | Average2_m128i(&avgTTR, &avgLTL, &avg); \ |
295 | 0 | L = _mm_add_epi8(avg, src); \ |
296 | 0 | out[i + (OUT)] = (uint32_t)_mm_cvtsi128_si32(L); \ |
297 | 0 | } while (0) |
298 | | |
299 | 0 | #define DO_PRED10_SHIFT do { \ |
300 | 0 | /* Rotate the pre-computed values for the next iteration.*/ \ |
301 | 0 | avgTTR = _mm_srli_si128(avgTTR, 4); \ |
302 | 0 | TL = _mm_srli_si128(TL, 4); \ |
303 | 0 | src = _mm_srli_si128(src, 4); \ |
304 | 0 | } while (0) |
305 | | |
306 | | static void PredictorAdd10_SSE2(const uint32_t* in, const uint32_t* upper, |
307 | 0 | int num_pixels, uint32_t* WEBP_RESTRICT out) { |
308 | 0 | int i; |
309 | 0 | __m128i L = _mm_cvtsi32_si128((int)out[-1]); |
310 | 0 | for (i = 0; i + 4 <= num_pixels; i += 4) { |
311 | 0 | __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); |
312 | 0 | __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]); |
313 | 0 | const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]); |
314 | 0 | const __m128i TR = _mm_loadu_si128((const __m128i*)&upper[i + 1]); |
315 | 0 | __m128i avgTTR; |
316 | 0 | Average2_m128i(&T, &TR, &avgTTR); |
317 | 0 | DO_PRED10(0); |
318 | 0 | DO_PRED10_SHIFT; |
319 | 0 | DO_PRED10(1); |
320 | 0 | DO_PRED10_SHIFT; |
321 | 0 | DO_PRED10(2); |
322 | 0 | DO_PRED10_SHIFT; |
323 | 0 | DO_PRED10(3); |
324 | 0 | } |
325 | 0 | if (i != num_pixels) { |
326 | 0 | VP8LPredictorsAdd_C[10](in + i, upper + i, num_pixels - i, out + i); |
327 | 0 | } |
328 | 0 | } |
329 | | #undef DO_PRED10 |
330 | | #undef DO_PRED10_SHIFT |
331 | | |
332 | | // Predictor11: select. |
333 | 0 | #define DO_PRED11(OUT) do { \ |
334 | 0 | const __m128i L_lo = _mm_unpacklo_epi32(L, T); \ |
335 | 0 | const __m128i TL_lo = _mm_unpacklo_epi32(TL, T); \ |
336 | 0 | const __m128i pb = _mm_sad_epu8(L_lo, TL_lo); /* pb = sum |L-TL|*/ \ |
337 | 0 | const __m128i mask = _mm_cmpgt_epi32(pb, pa); \ |
338 | 0 | const __m128i A = _mm_and_si128(mask, L); \ |
339 | 0 | const __m128i B = _mm_andnot_si128(mask, T); \ |
340 | 0 | const __m128i pred = _mm_or_si128(A, B); /* pred = (pa > b)? L : T*/ \ |
341 | 0 | L = _mm_add_epi8(src, pred); \ |
342 | 0 | out[i + (OUT)] = (uint32_t)_mm_cvtsi128_si32(L); \ |
343 | 0 | } while (0) |
344 | | |
345 | 0 | #define DO_PRED11_SHIFT do { \ |
346 | 0 | /* Shift the pre-computed value for the next iteration.*/ \ |
347 | 0 | T = _mm_srli_si128(T, 4); \ |
348 | 0 | TL = _mm_srli_si128(TL, 4); \ |
349 | 0 | src = _mm_srli_si128(src, 4); \ |
350 | 0 | pa = _mm_srli_si128(pa, 4); \ |
351 | 0 | } while (0) |
352 | | |
353 | | static void PredictorAdd11_SSE2(const uint32_t* in, const uint32_t* upper, |
354 | 0 | int num_pixels, uint32_t* WEBP_RESTRICT out) { |
355 | 0 | int i; |
356 | 0 | __m128i pa; |
357 | 0 | __m128i L = _mm_cvtsi32_si128((int)out[-1]); |
358 | 0 | for (i = 0; i + 4 <= num_pixels; i += 4) { |
359 | 0 | __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]); |
360 | 0 | __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]); |
361 | 0 | __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); |
362 | 0 | { |
363 | | // We can unpack with any value on the upper 32 bits, provided it's the |
364 | | // same on both operands (so that their sum of abs diff is zero). Here we |
365 | | // use T. |
366 | 0 | const __m128i T_lo = _mm_unpacklo_epi32(T, T); |
367 | 0 | const __m128i TL_lo = _mm_unpacklo_epi32(TL, T); |
368 | 0 | const __m128i T_hi = _mm_unpackhi_epi32(T, T); |
369 | 0 | const __m128i TL_hi = _mm_unpackhi_epi32(TL, T); |
370 | 0 | const __m128i s_lo = _mm_sad_epu8(T_lo, TL_lo); |
371 | 0 | const __m128i s_hi = _mm_sad_epu8(T_hi, TL_hi); |
372 | 0 | pa = _mm_packs_epi32(s_lo, s_hi); // pa = sum |T-TL| |
373 | 0 | } |
374 | 0 | DO_PRED11(0); |
375 | 0 | DO_PRED11_SHIFT; |
376 | 0 | DO_PRED11(1); |
377 | 0 | DO_PRED11_SHIFT; |
378 | 0 | DO_PRED11(2); |
379 | 0 | DO_PRED11_SHIFT; |
380 | 0 | DO_PRED11(3); |
381 | 0 | } |
382 | 0 | if (i != num_pixels) { |
383 | 0 | VP8LPredictorsAdd_C[11](in + i, upper + i, num_pixels - i, out + i); |
384 | 0 | } |
385 | 0 | } |
386 | | #undef DO_PRED11 |
387 | | #undef DO_PRED11_SHIFT |
388 | | |
389 | | // Predictor12: ClampedAddSubtractFull. |
390 | 0 | #define DO_PRED12(DIFF, LANE, OUT) do { \ |
391 | 0 | const __m128i all = _mm_add_epi16(L, (DIFF)); \ |
392 | 0 | const __m128i alls = _mm_packus_epi16(all, all); \ |
393 | 0 | const __m128i res = _mm_add_epi8(src, alls); \ |
394 | 0 | out[i + (OUT)] = (uint32_t)_mm_cvtsi128_si32(res); \ |
395 | 0 | L = _mm_unpacklo_epi8(res, zero); \ |
396 | 0 | } while (0) |
397 | | |
398 | 0 | #define DO_PRED12_SHIFT(DIFF, LANE) do { \ |
399 | 0 | /* Shift the pre-computed value for the next iteration.*/ \ |
400 | 0 | if ((LANE) == 0) (DIFF) = _mm_srli_si128((DIFF), 8); \ |
401 | 0 | src = _mm_srli_si128(src, 4); \ |
402 | 0 | } while (0) |
403 | | |
404 | | static void PredictorAdd12_SSE2(const uint32_t* in, const uint32_t* upper, |
405 | 0 | int num_pixels, uint32_t* WEBP_RESTRICT out) { |
406 | 0 | int i; |
407 | 0 | const __m128i zero = _mm_setzero_si128(); |
408 | 0 | const __m128i L8 = _mm_cvtsi32_si128((int)out[-1]); |
409 | 0 | __m128i L = _mm_unpacklo_epi8(L8, zero); |
410 | 0 | for (i = 0; i + 4 <= num_pixels; i += 4) { |
411 | | // Load 4 pixels at a time. |
412 | 0 | __m128i src = _mm_loadu_si128((const __m128i*)&in[i]); |
413 | 0 | const __m128i T = _mm_loadu_si128((const __m128i*)&upper[i]); |
414 | 0 | const __m128i T_lo = _mm_unpacklo_epi8(T, zero); |
415 | 0 | const __m128i T_hi = _mm_unpackhi_epi8(T, zero); |
416 | 0 | const __m128i TL = _mm_loadu_si128((const __m128i*)&upper[i - 1]); |
417 | 0 | const __m128i TL_lo = _mm_unpacklo_epi8(TL, zero); |
418 | 0 | const __m128i TL_hi = _mm_unpackhi_epi8(TL, zero); |
419 | 0 | __m128i diff_lo = _mm_sub_epi16(T_lo, TL_lo); |
420 | 0 | __m128i diff_hi = _mm_sub_epi16(T_hi, TL_hi); |
421 | 0 | DO_PRED12(diff_lo, 0, 0); |
422 | 0 | DO_PRED12_SHIFT(diff_lo, 0); |
423 | 0 | DO_PRED12(diff_lo, 1, 1); |
424 | 0 | DO_PRED12_SHIFT(diff_lo, 1); |
425 | 0 | DO_PRED12(diff_hi, 0, 2); |
426 | 0 | DO_PRED12_SHIFT(diff_hi, 0); |
427 | 0 | DO_PRED12(diff_hi, 1, 3); |
428 | 0 | } |
429 | 0 | if (i != num_pixels) { |
430 | 0 | VP8LPredictorsAdd_C[12](in + i, upper + i, num_pixels - i, out + i); |
431 | 0 | } |
432 | 0 | } |
433 | | #undef DO_PRED12 |
434 | | #undef DO_PRED12_SHIFT |
435 | | |
436 | | // Due to averages with integers, values cannot be accumulated in parallel for |
437 | | // predictors 13. |
438 | | GENERATE_PREDICTOR_ADD(Predictor13_SSE2, PredictorAdd13_SSE2) |
439 | | |
440 | | //------------------------------------------------------------------------------ |
441 | | // Subtract-Green Transform |
442 | | |
443 | | static void AddGreenToBlueAndRed_SSE2(const uint32_t* const src, int num_pixels, |
444 | 0 | uint32_t* dst) { |
445 | 0 | int i; |
446 | 0 | for (i = 0; i + 4 <= num_pixels; i += 4) { |
447 | 0 | const __m128i in = _mm_loadu_si128((const __m128i*)&src[i]); // argb |
448 | 0 | const __m128i A = _mm_srli_epi16(in, 8); // 0 a 0 g |
449 | 0 | const __m128i B = _mm_shufflelo_epi16(A, _MM_SHUFFLE(2, 2, 0, 0)); |
450 | 0 | const __m128i C = _mm_shufflehi_epi16(B, _MM_SHUFFLE(2, 2, 0, 0)); // 0g0g |
451 | 0 | const __m128i out = _mm_add_epi8(in, C); |
452 | 0 | _mm_storeu_si128((__m128i*)&dst[i], out); |
453 | 0 | } |
454 | | // fallthrough and finish off with plain-C |
455 | 0 | if (i != num_pixels) { |
456 | 0 | VP8LAddGreenToBlueAndRed_C(src + i, num_pixels - i, dst + i); |
457 | 0 | } |
458 | 0 | } |
459 | | |
460 | | //------------------------------------------------------------------------------ |
461 | | // Color Transform |
462 | | |
463 | | static void TransformColorInverse_SSE2(const VP8LMultipliers* const m, |
464 | | const uint32_t* const src, |
465 | 0 | int num_pixels, uint32_t* dst) { |
466 | | // sign-extended multiplying constants, pre-shifted by 5. |
467 | 0 | #define CST(X) (((int16_t)(m->X << 8)) >> 5) // sign-extend |
468 | 0 | #define MK_CST_16(HI, LO) \ |
469 | 0 | _mm_set1_epi32((int)(((uint32_t)(HI) << 16) | ((LO) & 0xffff))) |
470 | 0 | const __m128i mults_rb = MK_CST_16(CST(green_to_red), CST(green_to_blue)); |
471 | 0 | const __m128i mults_b2 = MK_CST_16(CST(red_to_blue), 0); |
472 | 0 | #undef MK_CST_16 |
473 | 0 | #undef CST |
474 | 0 | const __m128i mask_ag = _mm_set1_epi32((int)0xff00ff00); // alpha-green masks |
475 | 0 | int i; |
476 | 0 | for (i = 0; i + 4 <= num_pixels; i += 4) { |
477 | 0 | const __m128i in = _mm_loadu_si128((const __m128i*)&src[i]); // argb |
478 | 0 | const __m128i A = _mm_and_si128(in, mask_ag); // a 0 g 0 |
479 | 0 | const __m128i B = _mm_shufflelo_epi16(A, _MM_SHUFFLE(2, 2, 0, 0)); |
480 | 0 | const __m128i C = _mm_shufflehi_epi16(B, _MM_SHUFFLE(2, 2, 0, 0)); // g0g0 |
481 | 0 | const __m128i D = _mm_mulhi_epi16(C, mults_rb); // x dr x db1 |
482 | 0 | const __m128i E = _mm_add_epi8(in, D); // x r' x b' |
483 | 0 | const __m128i F = _mm_slli_epi16(E, 8); // r' 0 b' 0 |
484 | 0 | const __m128i G = _mm_mulhi_epi16(F, mults_b2); // x db2 0 0 |
485 | 0 | const __m128i H = _mm_srli_epi32(G, 8); // 0 x db2 0 |
486 | 0 | const __m128i I = _mm_add_epi8(H, F); // r' x b'' 0 |
487 | 0 | const __m128i J = _mm_srli_epi16(I, 8); // 0 r' 0 b'' |
488 | 0 | const __m128i out = _mm_or_si128(J, A); |
489 | 0 | _mm_storeu_si128((__m128i*)&dst[i], out); |
490 | 0 | } |
491 | | // Fall-back to C-version for left-overs. |
492 | 0 | if (i != num_pixels) { |
493 | 0 | VP8LTransformColorInverse_C(m, src + i, num_pixels - i, dst + i); |
494 | 0 | } |
495 | 0 | } |
496 | | |
497 | | //------------------------------------------------------------------------------ |
498 | | // Color-space conversion functions |
499 | | |
500 | | static void ConvertBGRAToRGB_SSE2(const uint32_t* WEBP_RESTRICT src, |
501 | 0 | int num_pixels, uint8_t* WEBP_RESTRICT dst) { |
502 | 0 | const __m128i* in = (const __m128i*)src; |
503 | 0 | __m128i* out = (__m128i*)dst; |
504 | |
|
505 | 0 | while (num_pixels >= 32) { |
506 | | // Load the BGRA buffers. |
507 | 0 | __m128i in0 = _mm_loadu_si128(in + 0); |
508 | 0 | __m128i in1 = _mm_loadu_si128(in + 1); |
509 | 0 | __m128i in2 = _mm_loadu_si128(in + 2); |
510 | 0 | __m128i in3 = _mm_loadu_si128(in + 3); |
511 | 0 | __m128i in4 = _mm_loadu_si128(in + 4); |
512 | 0 | __m128i in5 = _mm_loadu_si128(in + 5); |
513 | 0 | __m128i in6 = _mm_loadu_si128(in + 6); |
514 | 0 | __m128i in7 = _mm_loadu_si128(in + 7); |
515 | 0 | VP8L32bToPlanar_SSE2(&in0, &in1, &in2, &in3); |
516 | 0 | VP8L32bToPlanar_SSE2(&in4, &in5, &in6, &in7); |
517 | | // At this points, in1/in5 contains red only, in2/in6 green only ... |
518 | | // Pack the colors in 24b RGB. |
519 | 0 | VP8PlanarTo24b_SSE2(&in1, &in5, &in2, &in6, &in3, &in7); |
520 | 0 | _mm_storeu_si128(out + 0, in1); |
521 | 0 | _mm_storeu_si128(out + 1, in5); |
522 | 0 | _mm_storeu_si128(out + 2, in2); |
523 | 0 | _mm_storeu_si128(out + 3, in6); |
524 | 0 | _mm_storeu_si128(out + 4, in3); |
525 | 0 | _mm_storeu_si128(out + 5, in7); |
526 | 0 | in += 8; |
527 | 0 | out += 6; |
528 | 0 | num_pixels -= 32; |
529 | 0 | } |
530 | | // left-overs |
531 | 0 | if (num_pixels > 0) { |
532 | 0 | VP8LConvertBGRAToRGB_C((const uint32_t*)in, num_pixels, (uint8_t*)out); |
533 | 0 | } |
534 | 0 | } |
535 | | |
536 | | static void ConvertBGRAToRGBA_SSE2(const uint32_t* WEBP_RESTRICT src, |
537 | 0 | int num_pixels, uint8_t* WEBP_RESTRICT dst) { |
538 | 0 | const __m128i red_blue_mask = _mm_set1_epi32(0x00ff00ff); |
539 | 0 | const __m128i* in = (const __m128i*)src; |
540 | 0 | __m128i* out = (__m128i*)dst; |
541 | 0 | while (num_pixels >= 8) { |
542 | 0 | const __m128i A1 = _mm_loadu_si128(in++); |
543 | 0 | const __m128i A2 = _mm_loadu_si128(in++); |
544 | 0 | const __m128i B1 = _mm_and_si128(A1, red_blue_mask); // R 0 B 0 |
545 | 0 | const __m128i B2 = _mm_and_si128(A2, red_blue_mask); // R 0 B 0 |
546 | 0 | const __m128i C1 = _mm_andnot_si128(red_blue_mask, A1); // 0 G 0 A |
547 | 0 | const __m128i C2 = _mm_andnot_si128(red_blue_mask, A2); // 0 G 0 A |
548 | 0 | const __m128i D1 = _mm_shufflelo_epi16(B1, _MM_SHUFFLE(2, 3, 0, 1)); |
549 | 0 | const __m128i D2 = _mm_shufflelo_epi16(B2, _MM_SHUFFLE(2, 3, 0, 1)); |
550 | 0 | const __m128i E1 = _mm_shufflehi_epi16(D1, _MM_SHUFFLE(2, 3, 0, 1)); |
551 | 0 | const __m128i E2 = _mm_shufflehi_epi16(D2, _MM_SHUFFLE(2, 3, 0, 1)); |
552 | 0 | const __m128i F1 = _mm_or_si128(E1, C1); |
553 | 0 | const __m128i F2 = _mm_or_si128(E2, C2); |
554 | 0 | _mm_storeu_si128(out++, F1); |
555 | 0 | _mm_storeu_si128(out++, F2); |
556 | 0 | num_pixels -= 8; |
557 | 0 | } |
558 | | // left-overs |
559 | 0 | if (num_pixels > 0) { |
560 | 0 | VP8LConvertBGRAToRGBA_C((const uint32_t*)in, num_pixels, (uint8_t*)out); |
561 | 0 | } |
562 | 0 | } |
563 | | |
564 | | static void ConvertBGRAToRGBA4444_SSE2(const uint32_t* WEBP_RESTRICT src, |
565 | | int num_pixels, |
566 | 0 | uint8_t* WEBP_RESTRICT dst) { |
567 | 0 | const __m128i mask_0x0f = _mm_set1_epi8(0x0f); |
568 | 0 | const __m128i mask_0xf0 = _mm_set1_epi8((char)0xf0); |
569 | 0 | const __m128i* in = (const __m128i*)src; |
570 | 0 | __m128i* out = (__m128i*)dst; |
571 | 0 | while (num_pixels >= 8) { |
572 | 0 | const __m128i bgra0 = _mm_loadu_si128(in++); // bgra0|bgra1|bgra2|bgra3 |
573 | 0 | const __m128i bgra4 = _mm_loadu_si128(in++); // bgra4|bgra5|bgra6|bgra7 |
574 | 0 | const __m128i v0l = _mm_unpacklo_epi8(bgra0, bgra4); // b0b4g0g4r0r4a0a4... |
575 | 0 | const __m128i v0h = _mm_unpackhi_epi8(bgra0, bgra4); // b2b6g2g6r2r6a2a6... |
576 | 0 | const __m128i v1l = _mm_unpacklo_epi8(v0l, v0h); // b0b2b4b6g0g2g4g6... |
577 | 0 | const __m128i v1h = _mm_unpackhi_epi8(v0l, v0h); // b1b3b5b7g1g3g5g7... |
578 | 0 | const __m128i v2l = _mm_unpacklo_epi8(v1l, v1h); // b0...b7 | g0...g7 |
579 | 0 | const __m128i v2h = _mm_unpackhi_epi8(v1l, v1h); // r0...r7 | a0...a7 |
580 | 0 | const __m128i ga0 = _mm_unpackhi_epi64(v2l, v2h); // g0...g7 | a0...a7 |
581 | 0 | const __m128i rb0 = _mm_unpacklo_epi64(v2h, v2l); // r0...r7 | b0...b7 |
582 | 0 | const __m128i ga1 = _mm_srli_epi16(ga0, 4); // g0-|g1-|...|a6-|a7- |
583 | 0 | const __m128i rb1 = _mm_and_si128(rb0, mask_0xf0); // -r0|-r1|...|-b6|-a7 |
584 | 0 | const __m128i ga2 = _mm_and_si128(ga1, mask_0x0f); // g0-|g1-|...|a6-|a7- |
585 | 0 | const __m128i rgba0 = _mm_or_si128(ga2, rb1); // rg0..rg7 | ba0..ba7 |
586 | 0 | const __m128i rgba1 = _mm_srli_si128(rgba0, 8); // ba0..ba7 | 0 |
587 | | #if (WEBP_SWAP_16BIT_CSP == 1) |
588 | | const __m128i rgba = _mm_unpacklo_epi8(rgba1, rgba0); // barg0...barg7 |
589 | | #else |
590 | 0 | const __m128i rgba = _mm_unpacklo_epi8(rgba0, rgba1); // rgba0...rgba7 |
591 | 0 | #endif |
592 | 0 | _mm_storeu_si128(out++, rgba); |
593 | 0 | num_pixels -= 8; |
594 | 0 | } |
595 | | // left-overs |
596 | 0 | if (num_pixels > 0) { |
597 | 0 | VP8LConvertBGRAToRGBA4444_C((const uint32_t*)in, num_pixels, (uint8_t*)out); |
598 | 0 | } |
599 | 0 | } |
600 | | |
601 | | static void ConvertBGRAToRGB565_SSE2(const uint32_t* WEBP_RESTRICT src, |
602 | | int num_pixels, |
603 | 0 | uint8_t* WEBP_RESTRICT dst) { |
604 | 0 | const __m128i mask_0xe0 = _mm_set1_epi8((char)0xe0); |
605 | 0 | const __m128i mask_0xf8 = _mm_set1_epi8((char)0xf8); |
606 | 0 | const __m128i mask_0x07 = _mm_set1_epi8(0x07); |
607 | 0 | const __m128i* in = (const __m128i*)src; |
608 | 0 | __m128i* out = (__m128i*)dst; |
609 | 0 | while (num_pixels >= 8) { |
610 | 0 | const __m128i bgra0 = _mm_loadu_si128(in++); // bgra0|bgra1|bgra2|bgra3 |
611 | 0 | const __m128i bgra4 = _mm_loadu_si128(in++); // bgra4|bgra5|bgra6|bgra7 |
612 | 0 | const __m128i v0l = _mm_unpacklo_epi8(bgra0, bgra4); // b0b4g0g4r0r4a0a4... |
613 | 0 | const __m128i v0h = _mm_unpackhi_epi8(bgra0, bgra4); // b2b6g2g6r2r6a2a6... |
614 | 0 | const __m128i v1l = _mm_unpacklo_epi8(v0l, v0h); // b0b2b4b6g0g2g4g6... |
615 | 0 | const __m128i v1h = _mm_unpackhi_epi8(v0l, v0h); // b1b3b5b7g1g3g5g7... |
616 | 0 | const __m128i v2l = _mm_unpacklo_epi8(v1l, v1h); // b0...b7 | g0...g7 |
617 | 0 | const __m128i v2h = _mm_unpackhi_epi8(v1l, v1h); // r0...r7 | a0...a7 |
618 | 0 | const __m128i ga0 = _mm_unpackhi_epi64(v2l, v2h); // g0...g7 | a0...a7 |
619 | 0 | const __m128i rb0 = _mm_unpacklo_epi64(v2h, v2l); // r0...r7 | b0...b7 |
620 | 0 | const __m128i rb1 = _mm_and_si128(rb0, mask_0xf8); // -r0..-r7|-b0..-b7 |
621 | 0 | const __m128i g_lo1 = _mm_srli_epi16(ga0, 5); |
622 | 0 | const __m128i g_lo2 = _mm_and_si128(g_lo1, mask_0x07); // g0-...g7-|xx (3b) |
623 | 0 | const __m128i g_hi1 = _mm_slli_epi16(ga0, 3); |
624 | 0 | const __m128i g_hi2 = _mm_and_si128(g_hi1, mask_0xe0); // -g0...-g7|xx (3b) |
625 | 0 | const __m128i b0 = _mm_srli_si128(rb1, 8); // -b0...-b7|0 |
626 | 0 | const __m128i rg1 = _mm_or_si128(rb1, g_lo2); // gr0...gr7|xx |
627 | 0 | const __m128i b1 = _mm_srli_epi16(b0, 3); |
628 | 0 | const __m128i gb1 = _mm_or_si128(b1, g_hi2); // bg0...bg7|xx |
629 | | #if (WEBP_SWAP_16BIT_CSP == 1) |
630 | | const __m128i rgba = _mm_unpacklo_epi8(gb1, rg1); // rggb0...rggb7 |
631 | | #else |
632 | 0 | const __m128i rgba = _mm_unpacklo_epi8(rg1, gb1); // bgrb0...bgrb7 |
633 | 0 | #endif |
634 | 0 | _mm_storeu_si128(out++, rgba); |
635 | 0 | num_pixels -= 8; |
636 | 0 | } |
637 | | // left-overs |
638 | 0 | if (num_pixels > 0) { |
639 | 0 | VP8LConvertBGRAToRGB565_C((const uint32_t*)in, num_pixels, (uint8_t*)out); |
640 | 0 | } |
641 | 0 | } |
642 | | |
643 | | static void ConvertBGRAToBGR_SSE2(const uint32_t* WEBP_RESTRICT src, |
644 | 0 | int num_pixels, uint8_t* WEBP_RESTRICT dst) { |
645 | 0 | const __m128i mask_l = _mm_set_epi32(0, 0x00ffffff, 0, 0x00ffffff); |
646 | 0 | const __m128i mask_h = _mm_set_epi32(0x00ffffff, 0, 0x00ffffff, 0); |
647 | 0 | const __m128i* in = (const __m128i*)src; |
648 | 0 | const uint8_t* const end = dst + num_pixels * 3; |
649 | | // the last storel_epi64 below writes 8 bytes starting at offset 18 |
650 | 0 | while (dst + 26 <= end) { |
651 | 0 | const __m128i bgra0 = _mm_loadu_si128(in++); // bgra0|bgra1|bgra2|bgra3 |
652 | 0 | const __m128i bgra4 = _mm_loadu_si128(in++); // bgra4|bgra5|bgra6|bgra7 |
653 | 0 | const __m128i a0l = _mm_and_si128(bgra0, mask_l); // bgr0|0|bgr0|0 |
654 | 0 | const __m128i a4l = _mm_and_si128(bgra4, mask_l); // bgr0|0|bgr0|0 |
655 | 0 | const __m128i a0h = _mm_and_si128(bgra0, mask_h); // 0|bgr0|0|bgr0 |
656 | 0 | const __m128i a4h = _mm_and_si128(bgra4, mask_h); // 0|bgr0|0|bgr0 |
657 | 0 | const __m128i b0h = _mm_srli_epi64(a0h, 8); // 000b|gr00|000b|gr00 |
658 | 0 | const __m128i b4h = _mm_srli_epi64(a4h, 8); // 000b|gr00|000b|gr00 |
659 | 0 | const __m128i c0 = _mm_or_si128(a0l, b0h); // rgbrgb00|rgbrgb00 |
660 | 0 | const __m128i c4 = _mm_or_si128(a4l, b4h); // rgbrgb00|rgbrgb00 |
661 | 0 | const __m128i c2 = _mm_srli_si128(c0, 8); |
662 | 0 | const __m128i c6 = _mm_srli_si128(c4, 8); |
663 | 0 | _mm_storel_epi64((__m128i*)(dst + 0), c0); |
664 | 0 | _mm_storel_epi64((__m128i*)(dst + 6), c2); |
665 | 0 | _mm_storel_epi64((__m128i*)(dst + 12), c4); |
666 | 0 | _mm_storel_epi64((__m128i*)(dst + 18), c6); |
667 | 0 | dst += 24; |
668 | 0 | num_pixels -= 8; |
669 | 0 | } |
670 | | // left-overs |
671 | 0 | if (num_pixels > 0) { |
672 | 0 | VP8LConvertBGRAToBGR_C((const uint32_t*)in, num_pixels, dst); |
673 | 0 | } |
674 | 0 | } |
675 | | |
676 | | //------------------------------------------------------------------------------ |
677 | | // Entry point |
678 | | |
679 | | extern void VP8LDspInitSSE2(void); |
680 | | |
681 | 0 | WEBP_TSAN_IGNORE_FUNCTION void VP8LDspInitSSE2(void) { |
682 | 0 | VP8LPredictors[5] = Predictor5_SSE2; |
683 | 0 | VP8LPredictors[6] = Predictor6_SSE2; |
684 | 0 | VP8LPredictors[7] = Predictor7_SSE2; |
685 | 0 | VP8LPredictors[8] = Predictor8_SSE2; |
686 | 0 | VP8LPredictors[9] = Predictor9_SSE2; |
687 | 0 | VP8LPredictors[10] = Predictor10_SSE2; |
688 | 0 | VP8LPredictors[11] = Predictor11_SSE2; |
689 | 0 | VP8LPredictors[12] = Predictor12_SSE2; |
690 | 0 | VP8LPredictors[13] = Predictor13_SSE2; |
691 | |
|
692 | 0 | VP8LPredictorsAdd[0] = PredictorAdd0_SSE2; |
693 | 0 | VP8LPredictorsAdd[1] = PredictorAdd1_SSE2; |
694 | 0 | VP8LPredictorsAdd[2] = PredictorAdd2_SSE2; |
695 | 0 | VP8LPredictorsAdd[3] = PredictorAdd3_SSE2; |
696 | 0 | VP8LPredictorsAdd[4] = PredictorAdd4_SSE2; |
697 | 0 | VP8LPredictorsAdd[5] = PredictorAdd5_SSE2; |
698 | 0 | VP8LPredictorsAdd[6] = PredictorAdd6_SSE2; |
699 | 0 | VP8LPredictorsAdd[7] = PredictorAdd7_SSE2; |
700 | 0 | VP8LPredictorsAdd[8] = PredictorAdd8_SSE2; |
701 | 0 | VP8LPredictorsAdd[9] = PredictorAdd9_SSE2; |
702 | 0 | VP8LPredictorsAdd[10] = PredictorAdd10_SSE2; |
703 | 0 | VP8LPredictorsAdd[11] = PredictorAdd11_SSE2; |
704 | 0 | VP8LPredictorsAdd[12] = PredictorAdd12_SSE2; |
705 | 0 | VP8LPredictorsAdd[13] = PredictorAdd13_SSE2; |
706 | |
|
707 | 0 | VP8LAddGreenToBlueAndRed = AddGreenToBlueAndRed_SSE2; |
708 | 0 | VP8LTransformColorInverse = TransformColorInverse_SSE2; |
709 | |
|
710 | 0 | VP8LConvertBGRAToRGB = ConvertBGRAToRGB_SSE2; |
711 | 0 | VP8LConvertBGRAToRGBA = ConvertBGRAToRGBA_SSE2; |
712 | 0 | VP8LConvertBGRAToRGBA4444 = ConvertBGRAToRGBA4444_SSE2; |
713 | 0 | VP8LConvertBGRAToRGB565 = ConvertBGRAToRGB565_SSE2; |
714 | 0 | VP8LConvertBGRAToBGR = ConvertBGRAToBGR_SSE2; |
715 | | |
716 | | // SSE exports for AVX and above. |
717 | 0 | memcpy(VP8LPredictorsAdd_SSE, VP8LPredictorsAdd, sizeof(VP8LPredictorsAdd)); |
718 | |
|
719 | 0 | VP8LAddGreenToBlueAndRed_SSE = AddGreenToBlueAndRed_SSE2; |
720 | 0 | VP8LTransformColorInverse_SSE = TransformColorInverse_SSE2; |
721 | |
|
722 | 0 | VP8LConvertBGRAToRGB_SSE = ConvertBGRAToRGB_SSE2; |
723 | 0 | VP8LConvertBGRAToRGBA_SSE = ConvertBGRAToRGBA_SSE2; |
724 | 0 | } |
725 | | |
726 | | #else // !WEBP_USE_SSE2 |
727 | | |
728 | | WEBP_DSP_INIT_STUB(VP8LDspInitSSE2) |
729 | | |
730 | | #endif // WEBP_USE_SSE2 |