Coverage Report

Created: 2024-07-27 06:28

/src/libwebp/sharpyuv/sharpyuv_sse2.c
Line
Count
Source (jump to first uncovered line)
1
// Copyright 2022 Google Inc. All Rights Reserved.
2
//
3
// Use of this source code is governed by a BSD-style license
4
// that can be found in the COPYING file in the root of the source
5
// tree. An additional intellectual property rights grant can be found
6
// in the file PATENTS. All contributing project authors may
7
// be found in the AUTHORS file in the root of the source tree.
8
// -----------------------------------------------------------------------------
9
//
10
// Speed-critical functions for Sharp YUV.
11
//
12
// Author: Skal (pascal.massimino@gmail.com)
13
14
#include "sharpyuv/sharpyuv_dsp.h"
15
16
#if defined(WEBP_USE_SSE2)
17
#include <stdlib.h>
18
#include <emmintrin.h>
19
20
0
static uint16_t clip_SSE2(int v, int max) {
21
0
  return (v < 0) ? 0 : (v > max) ? max : (uint16_t)v;
22
0
}
23
24
static uint64_t SharpYuvUpdateY_SSE2(const uint16_t* ref, const uint16_t* src,
25
0
                                     uint16_t* dst, int len, int bit_depth) {
26
0
  const int max_y = (1 << bit_depth) - 1;
27
0
  uint64_t diff = 0;
28
0
  uint32_t tmp[4];
29
0
  int i;
30
0
  const __m128i zero = _mm_setzero_si128();
31
0
  const __m128i max = _mm_set1_epi16(max_y);
32
0
  const __m128i one = _mm_set1_epi16(1);
33
0
  __m128i sum = zero;
34
35
0
  for (i = 0; i + 8 <= len; i += 8) {
36
0
    const __m128i A = _mm_loadu_si128((const __m128i*)(ref + i));
37
0
    const __m128i B = _mm_loadu_si128((const __m128i*)(src + i));
38
0
    const __m128i C = _mm_loadu_si128((const __m128i*)(dst + i));
39
0
    const __m128i D = _mm_sub_epi16(A, B);       // diff_y
40
0
    const __m128i E = _mm_cmpgt_epi16(zero, D);  // sign (-1 or 0)
41
0
    const __m128i F = _mm_add_epi16(C, D);       // new_y
42
0
    const __m128i G = _mm_or_si128(E, one);      // -1 or 1
43
0
    const __m128i H = _mm_max_epi16(_mm_min_epi16(F, max), zero);
44
0
    const __m128i I = _mm_madd_epi16(D, G);      // sum(abs(...))
45
0
    _mm_storeu_si128((__m128i*)(dst + i), H);
46
0
    sum = _mm_add_epi32(sum, I);
47
0
  }
48
0
  _mm_storeu_si128((__m128i*)tmp, sum);
49
0
  diff = tmp[3] + tmp[2] + tmp[1] + tmp[0];
50
0
  for (; i < len; ++i) {
51
0
    const int diff_y = ref[i] - src[i];
52
0
    const int new_y = (int)dst[i] + diff_y;
53
0
    dst[i] = clip_SSE2(new_y, max_y);
54
0
    diff += (uint64_t)abs(diff_y);
55
0
  }
56
0
  return diff;
57
0
}
58
59
static void SharpYuvUpdateRGB_SSE2(const int16_t* ref, const int16_t* src,
60
0
                                   int16_t* dst, int len) {
61
0
  int i = 0;
62
0
  for (i = 0; i + 8 <= len; i += 8) {
63
0
    const __m128i A = _mm_loadu_si128((const __m128i*)(ref + i));
64
0
    const __m128i B = _mm_loadu_si128((const __m128i*)(src + i));
65
0
    const __m128i C = _mm_loadu_si128((const __m128i*)(dst + i));
66
0
    const __m128i D = _mm_sub_epi16(A, B);   // diff_uv
67
0
    const __m128i E = _mm_add_epi16(C, D);   // new_uv
68
0
    _mm_storeu_si128((__m128i*)(dst + i), E);
69
0
  }
70
0
  for (; i < len; ++i) {
71
0
    const int diff_uv = ref[i] - src[i];
72
0
    dst[i] += diff_uv;
73
0
  }
74
0
}
75
76
static void SharpYuvFilterRow16_SSE2(const int16_t* A, const int16_t* B,
77
                                     int len, const uint16_t* best_y,
78
0
                                     uint16_t* out, int bit_depth) {
79
0
  const int max_y = (1 << bit_depth) - 1;
80
0
  int i;
81
0
  const __m128i kCst8 = _mm_set1_epi16(8);
82
0
  const __m128i max = _mm_set1_epi16(max_y);
83
0
  const __m128i zero = _mm_setzero_si128();
84
0
  for (i = 0; i + 8 <= len; i += 8) {
85
0
    const __m128i a0 = _mm_loadu_si128((const __m128i*)(A + i + 0));
86
0
    const __m128i a1 = _mm_loadu_si128((const __m128i*)(A + i + 1));
87
0
    const __m128i b0 = _mm_loadu_si128((const __m128i*)(B + i + 0));
88
0
    const __m128i b1 = _mm_loadu_si128((const __m128i*)(B + i + 1));
89
0
    const __m128i a0b1 = _mm_add_epi16(a0, b1);
90
0
    const __m128i a1b0 = _mm_add_epi16(a1, b0);
91
0
    const __m128i a0a1b0b1 = _mm_add_epi16(a0b1, a1b0);  // A0+A1+B0+B1
92
0
    const __m128i a0a1b0b1_8 = _mm_add_epi16(a0a1b0b1, kCst8);
93
0
    const __m128i a0b1_2 = _mm_add_epi16(a0b1, a0b1);    // 2*(A0+B1)
94
0
    const __m128i a1b0_2 = _mm_add_epi16(a1b0, a1b0);    // 2*(A1+B0)
95
0
    const __m128i c0 = _mm_srai_epi16(_mm_add_epi16(a0b1_2, a0a1b0b1_8), 3);
96
0
    const __m128i c1 = _mm_srai_epi16(_mm_add_epi16(a1b0_2, a0a1b0b1_8), 3);
97
0
    const __m128i d0 = _mm_add_epi16(c1, a0);
98
0
    const __m128i d1 = _mm_add_epi16(c0, a1);
99
0
    const __m128i e0 = _mm_srai_epi16(d0, 1);
100
0
    const __m128i e1 = _mm_srai_epi16(d1, 1);
101
0
    const __m128i f0 = _mm_unpacklo_epi16(e0, e1);
102
0
    const __m128i f1 = _mm_unpackhi_epi16(e0, e1);
103
0
    const __m128i g0 = _mm_loadu_si128((const __m128i*)(best_y + 2 * i + 0));
104
0
    const __m128i g1 = _mm_loadu_si128((const __m128i*)(best_y + 2 * i + 8));
105
0
    const __m128i h0 = _mm_add_epi16(g0, f0);
106
0
    const __m128i h1 = _mm_add_epi16(g1, f1);
107
0
    const __m128i i0 = _mm_max_epi16(_mm_min_epi16(h0, max), zero);
108
0
    const __m128i i1 = _mm_max_epi16(_mm_min_epi16(h1, max), zero);
109
0
    _mm_storeu_si128((__m128i*)(out + 2 * i + 0), i0);
110
0
    _mm_storeu_si128((__m128i*)(out + 2 * i + 8), i1);
111
0
  }
112
0
  for (; i < len; ++i) {
113
    //   (9 * A0 + 3 * A1 + 3 * B0 + B1 + 8) >> 4 =
114
    // = (8 * A0 + 2 * (A1 + B0) + (A0 + A1 + B0 + B1 + 8)) >> 4
115
    // We reuse the common sub-expressions.
116
0
    const int a0b1 = A[i + 0] + B[i + 1];
117
0
    const int a1b0 = A[i + 1] + B[i + 0];
118
0
    const int a0a1b0b1 = a0b1 + a1b0 + 8;
119
0
    const int v0 = (8 * A[i + 0] + 2 * a1b0 + a0a1b0b1) >> 4;
120
0
    const int v1 = (8 * A[i + 1] + 2 * a0b1 + a0a1b0b1) >> 4;
121
0
    out[2 * i + 0] = clip_SSE2(best_y[2 * i + 0] + v0, max_y);
122
0
    out[2 * i + 1] = clip_SSE2(best_y[2 * i + 1] + v1, max_y);
123
0
  }
124
0
}
125
126
0
static WEBP_INLINE __m128i s16_to_s32(__m128i in) {
127
0
  return _mm_srai_epi32(_mm_unpacklo_epi16(in, in), 16);
128
0
}
129
130
static void SharpYuvFilterRow32_SSE2(const int16_t* A, const int16_t* B,
131
                                     int len, const uint16_t* best_y,
132
0
                                     uint16_t* out, int bit_depth) {
133
0
  const int max_y = (1 << bit_depth) - 1;
134
0
  int i;
135
0
  const __m128i kCst8 = _mm_set1_epi32(8);
136
0
  const __m128i max = _mm_set1_epi16(max_y);
137
0
  const __m128i zero = _mm_setzero_si128();
138
0
  for (i = 0; i + 4 <= len; i += 4) {
139
0
    const __m128i a0 = s16_to_s32(_mm_loadl_epi64((const __m128i*)(A + i + 0)));
140
0
    const __m128i a1 = s16_to_s32(_mm_loadl_epi64((const __m128i*)(A + i + 1)));
141
0
    const __m128i b0 = s16_to_s32(_mm_loadl_epi64((const __m128i*)(B + i + 0)));
142
0
    const __m128i b1 = s16_to_s32(_mm_loadl_epi64((const __m128i*)(B + i + 1)));
143
0
    const __m128i a0b1 = _mm_add_epi32(a0, b1);
144
0
    const __m128i a1b0 = _mm_add_epi32(a1, b0);
145
0
    const __m128i a0a1b0b1 = _mm_add_epi32(a0b1, a1b0);  // A0+A1+B0+B1
146
0
    const __m128i a0a1b0b1_8 = _mm_add_epi32(a0a1b0b1, kCst8);
147
0
    const __m128i a0b1_2 = _mm_add_epi32(a0b1, a0b1);  // 2*(A0+B1)
148
0
    const __m128i a1b0_2 = _mm_add_epi32(a1b0, a1b0);  // 2*(A1+B0)
149
0
    const __m128i c0 = _mm_srai_epi32(_mm_add_epi32(a0b1_2, a0a1b0b1_8), 3);
150
0
    const __m128i c1 = _mm_srai_epi32(_mm_add_epi32(a1b0_2, a0a1b0b1_8), 3);
151
0
    const __m128i d0 = _mm_add_epi32(c1, a0);
152
0
    const __m128i d1 = _mm_add_epi32(c0, a1);
153
0
    const __m128i e0 = _mm_srai_epi32(d0, 1);
154
0
    const __m128i e1 = _mm_srai_epi32(d1, 1);
155
0
    const __m128i f0 = _mm_unpacklo_epi32(e0, e1);
156
0
    const __m128i f1 = _mm_unpackhi_epi32(e0, e1);
157
0
    const __m128i g = _mm_loadu_si128((const __m128i*)(best_y + 2 * i + 0));
158
0
    const __m128i h_16 = _mm_add_epi16(g, _mm_packs_epi32(f0, f1));
159
0
    const __m128i final = _mm_max_epi16(_mm_min_epi16(h_16, max), zero);
160
0
    _mm_storeu_si128((__m128i*)(out + 2 * i + 0), final);
161
0
  }
162
0
  for (; i < len; ++i) {
163
    //   (9 * A0 + 3 * A1 + 3 * B0 + B1 + 8) >> 4 =
164
    // = (8 * A0 + 2 * (A1 + B0) + (A0 + A1 + B0 + B1 + 8)) >> 4
165
    // We reuse the common sub-expressions.
166
0
    const int a0b1 = A[i + 0] + B[i + 1];
167
0
    const int a1b0 = A[i + 1] + B[i + 0];
168
0
    const int a0a1b0b1 = a0b1 + a1b0 + 8;
169
0
    const int v0 = (8 * A[i + 0] + 2 * a1b0 + a0a1b0b1) >> 4;
170
0
    const int v1 = (8 * A[i + 1] + 2 * a0b1 + a0a1b0b1) >> 4;
171
0
    out[2 * i + 0] = clip_SSE2(best_y[2 * i + 0] + v0, max_y);
172
0
    out[2 * i + 1] = clip_SSE2(best_y[2 * i + 1] + v1, max_y);
173
0
  }
174
0
}
175
176
static void SharpYuvFilterRow_SSE2(const int16_t* A, const int16_t* B, int len,
177
                                   const uint16_t* best_y, uint16_t* out,
178
0
                                   int bit_depth) {
179
0
  if (bit_depth <= 10) {
180
0
    SharpYuvFilterRow16_SSE2(A, B, len, best_y, out, bit_depth);
181
0
  } else {
182
0
    SharpYuvFilterRow32_SSE2(A, B, len, best_y, out, bit_depth);
183
0
  }
184
0
}
185
186
//------------------------------------------------------------------------------
187
188
extern void InitSharpYuvSSE2(void);
189
190
0
WEBP_TSAN_IGNORE_FUNCTION void InitSharpYuvSSE2(void) {
191
0
  SharpYuvUpdateY = SharpYuvUpdateY_SSE2;
192
0
  SharpYuvUpdateRGB = SharpYuvUpdateRGB_SSE2;
193
0
  SharpYuvFilterRow = SharpYuvFilterRow_SSE2;
194
0
}
195
#else  // !WEBP_USE_SSE2
196
197
extern void InitSharpYuvSSE2(void);
198
199
void InitSharpYuvSSE2(void) {}
200
201
#endif  // WEBP_USE_SSE2