Coverage Report

Created: 2026-07-16 06:32

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/work/svt-av1/Source/Lib/Codec/cdef.c
Line
Count
Source
1
/*
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved
3
 *
4
 * This source code is subject to the terms of the BSD 2 Clause License and
5
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6
 * was not distributed with this source code in the LICENSE file, you can
7
 * obtain it at https://www.aomedia.org/license/software-license. If the Alliance for Open
8
 * Media Patent License 1.0 was not distributed with this source code in the
9
 * PATENTS file, you can obtain it at https://www.aomedia.org/license/patent-license.
10
 */
11
12
#include "cdef.h"
13
#include "common_dsp_rtcd.h"
14
#include "bitstream_unit.h"
15
16
0
static INLINE int32_t sign(int32_t i) {
17
0
    return i < 0 ? -1 : 1;
18
0
}
19
20
0
static INLINE int32_t constrain(int32_t diff, int32_t threshold, int32_t damping) {
21
0
    if (!threshold) {
22
0
        return 0;
23
0
    }
24
25
0
    const int32_t shift = AOMMAX(0, damping - get_msb(threshold));
26
0
    return sign(diff) * AOMMIN(abs(diff), AOMMAX(0, threshold - (abs(diff) >> shift)));
27
0
}
28
29
/*
30
This is Cdef_Directions (section 7.15.3) with 2 padding entries at the
31
beginning and end of the table. The cdef direction range is [0, 7] and the
32
first index is offset +/-2. This removes the need to constrain the first
33
index to the same range using e.g., & 7.
34
*/
35
DECLARE_ALIGNED(16, const int, eb_cdef_directions_padded[12][2]) = {
36
    /* Padding: svt_aom_eb_cdef_directions[6] */
37
    {1 * CDEF_BSTRIDE + 0, 2 * CDEF_BSTRIDE + 0},
38
    /* Padding: svt_aom_eb_cdef_directions[7] */
39
    {1 * CDEF_BSTRIDE + 0, 2 * CDEF_BSTRIDE - 1},
40
41
    /* Begin svt_aom_eb_cdef_directions */
42
    {-1 * CDEF_BSTRIDE + 1, -2 * CDEF_BSTRIDE + 2},
43
    {0 * CDEF_BSTRIDE + 1, -1 * CDEF_BSTRIDE + 2},
44
    {0 * CDEF_BSTRIDE + 1, 0 * CDEF_BSTRIDE + 2},
45
    {0 * CDEF_BSTRIDE + 1, 1 * CDEF_BSTRIDE + 2},
46
    {1 * CDEF_BSTRIDE + 1, 2 * CDEF_BSTRIDE + 2},
47
    {1 * CDEF_BSTRIDE + 0, 2 * CDEF_BSTRIDE + 1},
48
    {1 * CDEF_BSTRIDE + 0, 2 * CDEF_BSTRIDE + 0},
49
    {1 * CDEF_BSTRIDE + 0, 2 * CDEF_BSTRIDE - 1},
50
    /* End svt_aom_eb_cdef_directions */
51
52
    /* Padding: svt_aom_eb_cdef_directions[0] */
53
    {-1 * CDEF_BSTRIDE + 1, -2 * CDEF_BSTRIDE + 2},
54
    /* Padding: svt_aom_eb_cdef_directions[1] */
55
    {0 * CDEF_BSTRIDE + 1, -1 * CDEF_BSTRIDE + 2},
56
};
57
58
const int (*const svt_aom_eb_cdef_directions)[2] = eb_cdef_directions_padded + 2;
59
60
/* Compute the primary filter strength for an 8x8 block based on the
61
directional variance difference. A high variance difference means
62
that we have a highly directional pattern (e.g. a high contrast
63
edge), so we can apply more deringing. A low variance means that we
64
either have a low contrast edge, or a non-directional texture, so
65
we want to be careful not to blur. */
66
0
static INLINE int32_t adjust_strength(int32_t strength, int32_t var) {
67
0
    const int32_t i = (var >> 6) ? AOMMIN(get_msb(var >> 6), 12) : 0;
68
    /* We use the variance of 8x8 blocks to adjust the strength. */
69
0
    return var ? (strength * (4 + i) + 8) >> 4 : 0;
70
0
}
71
72
void svt_aom_copy_rect8_8bit_to_16bit_c(uint16_t* dst, int32_t dstride, const uint8_t* src, int32_t sstride, int32_t v,
73
0
                                        int32_t h) {
74
0
    for (int32_t i = 0; i < v; i++) {
75
0
        for (int32_t j = 0; j < h; j++) {
76
0
            dst[i * dstride + j] = src[i * sstride + j];
77
0
        }
78
0
    }
79
0
}
80
81
/* Detect direction. 0 means 45-degree up-right, 2 is horizontal, and so on.
82
The search minimizes the weighted variance along all the lines in a
83
particular direction, i.e. the squared error between the input and a
84
"predicted" block where each pixel is replaced by the average along a line
85
in a particular direction. Since each direction have the same sum(x^2) term,
86
that term is never computed. See Section 2, step 2, of:
87
http://jmvalin.ca/notes/intra_paint.pdf */
88
0
uint8_t svt_aom_cdef_find_dir_c(const uint16_t* img, int32_t stride, int32_t* var, int32_t coeff_shift) {
89
0
    int32_t cost[8]        = {0};
90
0
    int32_t partial[8][15] = {{0}};
91
0
    int32_t best_cost      = 0;
92
0
    uint8_t i;
93
0
    uint8_t best_dir = 0;
94
    /* Instead of dividing by n between 2 and 8, we multiply by 3*5*7*8/n.
95
    The output is then 840 times larger, but we don't care for finding
96
    the max. */
97
0
    static const int32_t div_table[] = {0, 840, 420, 280, 210, 168, 140, 120, 105};
98
0
    for (i = 0; i < 8; i++) {
99
0
        int32_t j;
100
0
        for (j = 0; j < 8; j++) {
101
0
            int32_t x;
102
            /* We subtract 128 here to reduce the maximum range of the squared
103
            partial sums. */
104
0
            x = (img[i * stride + j] >> coeff_shift) - 128;
105
0
            partial[0][i + j] += x;
106
0
            partial[1][i + j / 2] += x;
107
0
            partial[2][i] += x;
108
0
            partial[3][3 + i - j / 2] += x;
109
0
            partial[4][7 + i - j] += x;
110
0
            partial[5][3 - i / 2 + j] += x;
111
0
            partial[6][j] += x;
112
0
            partial[7][i / 2 + j] += x;
113
0
        }
114
0
    }
115
0
    for (i = 0; i < 8; i++) {
116
0
        cost[2] += partial[2][i] * partial[2][i];
117
0
        cost[6] += partial[6][i] * partial[6][i];
118
0
    }
119
0
    cost[2] *= div_table[8];
120
0
    cost[6] *= div_table[8];
121
0
    for (i = 0; i < 7; i++) {
122
0
        cost[0] += (partial[0][i] * partial[0][i] + partial[0][14 - i] * partial[0][14 - i]) * div_table[i + 1];
123
0
        cost[4] += (partial[4][i] * partial[4][i] + partial[4][14 - i] * partial[4][14 - i]) * div_table[i + 1];
124
0
    }
125
0
    cost[0] += partial[0][7] * partial[0][7] * div_table[8];
126
0
    cost[4] += partial[4][7] * partial[4][7] * div_table[8];
127
0
    for (i = 1; i < 8; i += 2) {
128
0
        int32_t j;
129
0
        for (j = 0; j < 4 + 1; j++) {
130
0
            cost[i] += partial[i][3 + j] * partial[i][3 + j];
131
0
        }
132
0
        cost[i] *= div_table[8];
133
0
        for (j = 0; j < 4 - 1; j++) {
134
0
            cost[i] += (partial[i][j] * partial[i][j] + partial[i][10 - j] * partial[i][10 - j]) * div_table[2 * j + 2];
135
0
        }
136
0
    }
137
0
    for (i = 0; i < 8; i++) {
138
0
        if (cost[i] > best_cost) {
139
0
            best_cost = cost[i];
140
0
            best_dir  = i;
141
0
        }
142
0
    }
143
    /* Difference between the optimal variance and the variance along the
144
    orthogonal direction. Again, the sum(x^2) terms cancel out. */
145
0
    *var = best_cost - cost[(best_dir + 4) & 7];
146
    /* We'd normally divide by 840, but dividing by 1024 is close enough
147
    for what we're going to do with this. */
148
0
    *var >>= 10;
149
0
    return best_dir;
150
0
}
151
152
void svt_aom_cdef_find_dir_dual_c(const uint16_t* img1, const uint16_t* img2, int stride, int32_t* var1, int32_t* var2,
153
0
                                  int32_t coeff_shift, uint8_t* out1, uint8_t* out2) {
154
0
    *out1 = svt_aom_cdef_find_dir_c(img1, stride, var1, coeff_shift);
155
0
    *out2 = svt_aom_cdef_find_dir_c(img2, stride, var2, coeff_shift);
156
0
}
157
158
static AOM_INLINE void cdef_find_dir(uint16_t* in, CdefList* dlist, int32_t var[CDEF_NBLOCKS][CDEF_NBLOCKS],
159
0
                                     int32_t cdef_count, int32_t coeff_shift, uint8_t dir[CDEF_NBLOCKS][CDEF_NBLOCKS]) {
160
0
    int bi;
161
162
    // Find direction of two 8x8 blocks together.
163
0
    for (bi = 0; bi < cdef_count - 1; bi += 2) {
164
0
        const uint8_t by   = dlist[bi].by;
165
0
        const uint8_t bx   = dlist[bi].bx;
166
0
        const uint8_t by2  = dlist[bi + 1].by;
167
0
        const uint8_t bx2  = dlist[bi + 1].bx;
168
0
        const int     pos1 = 8 * by * CDEF_BSTRIDE + 8 * bx;
169
0
        const int     pos2 = 8 * by2 * CDEF_BSTRIDE + 8 * bx2;
170
0
        svt_aom_cdef_find_dir_dual(&in[pos1],
171
0
                                   &in[pos2],
172
0
                                   CDEF_BSTRIDE,
173
0
                                   &var[by][bx],
174
0
                                   &var[by2][bx2],
175
0
                                   coeff_shift,
176
0
                                   &dir[by][bx],
177
0
                                   &dir[by2][bx2]);
178
0
    }
179
180
    // Process remaining 8x8 blocks here. One 8x8 at a time.
181
0
    if (cdef_count % 2) {
182
0
        const uint8_t by = dlist[bi].by;
183
0
        const uint8_t bx = dlist[bi].bx;
184
0
        dir[by][bx]      = svt_aom_cdef_find_dir(
185
0
            &in[8 * by * CDEF_BSTRIDE + 8 * bx], CDEF_BSTRIDE, &var[by][bx], coeff_shift);
186
0
    }
187
0
}
188
189
const int32_t svt_aom_eb_cdef_pri_taps[2][2] = {{4, 2}, {3, 3}};
190
const int32_t svt_aom_eb_cdef_sec_taps[2][2] = {{2, 1}, {2, 1}};
191
192
/* Smooth in the direction detected. */
193
void svt_cdef_filter_block_c(uint8_t* dst8, uint16_t* dst16, int32_t dstride, const uint16_t* in, int32_t pri_strength,
194
                             int32_t sec_strength, int32_t dir, int32_t pri_damping, int32_t sec_damping, int32_t bsize,
195
0
                             int32_t coeff_shift, uint8_t subsampling_factor) {
196
0
    int32_t        i, j, k;
197
0
    const int32_t  s        = CDEF_BSTRIDE;
198
0
    const int32_t* pri_taps = svt_aom_eb_cdef_pri_taps[(pri_strength >> coeff_shift) & 1];
199
0
    const int32_t* sec_taps = svt_aom_eb_cdef_sec_taps[(pri_strength >> coeff_shift) & 1];
200
201
0
    for (i = 0; i < (4 << (int32_t)(bsize == BLOCK_8X8 || bsize == BLOCK_4X8)); i += subsampling_factor) {
202
0
        for (j = 0; j < (4 << (int32_t)(bsize == BLOCK_8X8 || bsize == BLOCK_8X4)); j++) {
203
0
            int16_t sum = 0;
204
0
            int16_t y;
205
0
            int16_t x   = in[i * s + j];
206
0
            int32_t max = x;
207
0
            int32_t min = x;
208
0
            for (k = 0; k < 2; k++) {
209
0
                int16_t p0 = in[i * s + j + svt_aom_eb_cdef_directions[dir][k]];
210
0
                int16_t p1 = in[i * s + j - svt_aom_eb_cdef_directions[dir][k]];
211
0
                sum += (int16_t)(pri_taps[k] * constrain(p0 - x, pri_strength, pri_damping));
212
0
                sum += (int16_t)(pri_taps[k] * constrain(p1 - x, pri_strength, pri_damping));
213
0
                if (p0 != CDEF_VERY_LARGE) {
214
0
                    max = AOMMAX(p0, max);
215
0
                }
216
0
                if (p1 != CDEF_VERY_LARGE) {
217
0
                    max = AOMMAX(p1, max);
218
0
                }
219
0
                min        = AOMMIN(p0, min);
220
0
                min        = AOMMIN(p1, min);
221
0
                int16_t s0 = in[i * s + j + svt_aom_eb_cdef_directions[(dir + 2)][k]];
222
0
                int16_t s1 = in[i * s + j - svt_aom_eb_cdef_directions[(dir + 2)][k]];
223
0
                int16_t s2 = in[i * s + j + svt_aom_eb_cdef_directions[(dir - 2)][k]];
224
0
                int16_t s3 = in[i * s + j - svt_aom_eb_cdef_directions[(dir - 2)][k]];
225
0
                if (s0 != CDEF_VERY_LARGE) {
226
0
                    max = AOMMAX(s0, max);
227
0
                }
228
0
                if (s1 != CDEF_VERY_LARGE) {
229
0
                    max = AOMMAX(s1, max);
230
0
                }
231
0
                if (s2 != CDEF_VERY_LARGE) {
232
0
                    max = AOMMAX(s2, max);
233
0
                }
234
0
                if (s3 != CDEF_VERY_LARGE) {
235
0
                    max = AOMMAX(s3, max);
236
0
                }
237
0
                min = AOMMIN(s0, min);
238
0
                min = AOMMIN(s1, min);
239
0
                min = AOMMIN(s2, min);
240
0
                min = AOMMIN(s3, min);
241
0
                sum += (int16_t)(sec_taps[k] * constrain(s0 - x, sec_strength, sec_damping));
242
0
                sum += (int16_t)(sec_taps[k] * constrain(s1 - x, sec_strength, sec_damping));
243
0
                sum += (int16_t)(sec_taps[k] * constrain(s2 - x, sec_strength, sec_damping));
244
0
                sum += (int16_t)(sec_taps[k] * constrain(s3 - x, sec_strength, sec_damping));
245
0
            }
246
0
            y = (int16_t)clamp((int16_t)x + ((8 + sum - (sum < 0)) >> 4), min, max);
247
0
            if (dst8) {
248
0
                dst8[i * dstride + j] = (uint8_t)y;
249
0
            } else {
250
0
                dst16[i * dstride + j] = (uint16_t)y;
251
0
            }
252
0
        }
253
0
    }
254
0
}
255
256
// C reference for the native 8-bit interior filter (no off-frame sentinel).
257
void svt_cdef_filter_block_8bit_c(uint8_t* dst, int32_t dstride, const uint8_t* in, int32_t pri_strength,
258
                                  int32_t sec_strength, int32_t dir, int32_t damping, int32_t bsize,
259
0
                                  int32_t coeff_shift, uint8_t subsampling_factor) {
260
0
    const int32_t  s        = CDEF_BSTRIDE;
261
0
    const int32_t* pri_taps = svt_aom_eb_cdef_pri_taps[(pri_strength >> coeff_shift) & 1];
262
0
    const int32_t* sec_taps = svt_aom_eb_cdef_sec_taps[(pri_strength >> coeff_shift) & 1];
263
0
    const int32_t  rows     = (bsize == BLOCK_8X8 || bsize == BLOCK_4X8) ? 8 : 4;
264
0
    const int32_t  cols     = (bsize == BLOCK_8X8 || bsize == BLOCK_8X4) ? 8 : 4;
265
0
    const int32_t  sub      = (bsize == BLOCK_4X4) ? 1 : subsampling_factor;
266
0
    for (int32_t i = 0; i < rows; i += sub) {
267
0
        for (int32_t j = 0; j < cols; j++) {
268
0
            const int16_t x   = in[i * s + j];
269
0
            int16_t       sum = 0;
270
0
            int32_t       max = x, min = x;
271
0
            for (int32_t k = 0; k < 2; k++) {
272
0
                const int16_t p0 = in[i * s + j + svt_aom_eb_cdef_directions[dir][k]];
273
0
                const int16_t p1 = in[i * s + j - svt_aom_eb_cdef_directions[dir][k]];
274
0
                sum += (int16_t)(pri_taps[k] * constrain(p0 - x, pri_strength, damping));
275
0
                sum += (int16_t)(pri_taps[k] * constrain(p1 - x, pri_strength, damping));
276
0
                max              = AOMMAX(p0, max);
277
0
                max              = AOMMAX(p1, max);
278
0
                min              = AOMMIN(p0, min);
279
0
                min              = AOMMIN(p1, min);
280
0
                const int16_t s0 = in[i * s + j + svt_aom_eb_cdef_directions[dir + 2][k]];
281
0
                const int16_t s1 = in[i * s + j - svt_aom_eb_cdef_directions[dir + 2][k]];
282
0
                const int16_t s2 = in[i * s + j + svt_aom_eb_cdef_directions[dir - 2][k]];
283
0
                const int16_t s3 = in[i * s + j - svt_aom_eb_cdef_directions[dir - 2][k]];
284
0
                sum += (int16_t)(sec_taps[k] * constrain(s0 - x, sec_strength, damping));
285
0
                sum += (int16_t)(sec_taps[k] * constrain(s1 - x, sec_strength, damping));
286
0
                sum += (int16_t)(sec_taps[k] * constrain(s2 - x, sec_strength, damping));
287
0
                sum += (int16_t)(sec_taps[k] * constrain(s3 - x, sec_strength, damping));
288
0
                max = AOMMAX(s0, max);
289
0
                max = AOMMAX(s1, max);
290
0
                max = AOMMAX(s2, max);
291
0
                max = AOMMAX(s3, max);
292
0
                min = AOMMIN(s0, min);
293
0
                min = AOMMIN(s1, min);
294
0
                min = AOMMIN(s2, min);
295
0
                min = AOMMIN(s3, min);
296
0
            }
297
0
            dst[i * dstride + j] = (uint8_t)clamp((int16_t)x + ((8 + sum - (sum < 0)) >> 4), min, max);
298
0
        }
299
0
    }
300
0
}
301
302
// C reference for the 8-bit find_dir (widen to 16-bit, delegate to the spec).
303
0
uint8_t svt_aom_cdef_find_dir_8bit_c(const uint8_t* img, int32_t stride, int32_t* var, int32_t coeff_shift) {
304
0
    uint16_t img16[8 * 8];
305
0
    for (int32_t i = 0; i < 8; i++) {
306
0
        for (int32_t j = 0; j < 8; j++) {
307
0
            img16[i * 8 + j] = img[i * stride + j];
308
0
        }
309
0
    }
310
0
    return svt_aom_cdef_find_dir_c(img16, 8, var, coeff_shift);
311
0
}
312
313
void svt_aom_cdef_find_dir_dual_8bit_c(const uint8_t* img1, const uint8_t* img2, int stride, int32_t* var1,
314
0
                                       int32_t* var2, int32_t coeff_shift, uint8_t* out1, uint8_t* out2) {
315
0
    *out1 = svt_aom_cdef_find_dir_8bit_c(img1, stride, var1, coeff_shift);
316
0
    *out2 = svt_aom_cdef_find_dir_8bit_c(img2, stride, var2, coeff_shift);
317
0
}
318
319
/*
320
 * Loop over the non-skip 8x8 blocks.  For each block, find the CDEF direction, then apply the specified filter.
321
*/
322
void svt_cdef_filter_fb(uint8_t* dst8, uint16_t* dst16, int32_t dstride, uint16_t* in, int32_t xdec, int32_t ydec,
323
                        uint8_t dir[CDEF_NBLOCKS][CDEF_NBLOCKS], int32_t* dirinit,
324
                        int32_t var[CDEF_NBLOCKS][CDEF_NBLOCKS], int32_t pli, CdefList* dlist, int32_t cdef_count,
325
0
                        int32_t cdef_strength, int32_t damping, int32_t coeff_shift, uint8_t subsampling_factor) {
326
0
    int32_t bi;
327
0
    int32_t pri_strength = (cdef_strength / CDEF_SEC_STRENGTHS) << coeff_shift;
328
0
    int32_t sec          = cdef_strength % CDEF_SEC_STRENGTHS;
329
0
    int32_t sec_strength = (sec + (sec == 3)) << coeff_shift;
330
0
    damping += coeff_shift - (pli != PLANE_Y);
331
332
0
    int32_t bsize  = ydec ? (xdec ? BLOCK_4X4 : BLOCK_8X4) : (xdec ? BLOCK_4X8 : BLOCK_8X8);
333
0
    int32_t bsizex = 3 - xdec;
334
0
    int32_t bsizey = 3 - ydec;
335
336
0
    if (!dstride && cdef_strength == 0) {
337
        // If we're here, both primary and secondary strengths are 0, and
338
        // we still haven't written anything to y[] yet, so we just copy
339
        // the input to y[]. This is necessary only for svt_av1_cdef_search()
340
        // and only svt_av1_cdef_search() sets dirinit.
341
0
        for (bi = 0; bi < cdef_count; bi++) {
342
0
            int32_t   by = dlist[bi].by << bsizey;
343
0
            int32_t   bx = dlist[bi].bx << bsizex;
344
0
            int32_t   iy;
345
0
            uint16_t* src_16 = in + (by * CDEF_BSTRIDE + bx);
346
0
            if (dst8) {
347
0
                uint8_t* dst_8 = dst8 + (bi << (bsizex + bsizey));
348
                //size 2x2 and 3x3, no gain to use SIMD
349
0
                for (iy = 0; iy < 1 << bsizey; iy += subsampling_factor) {
350
0
                    for (int32_t ix = 0; ix < 1 << bsizex; ix++) {
351
0
                        dst_8[(iy << bsizex) + ix] = (uint8_t)src_16[iy * CDEF_BSTRIDE + ix];
352
0
                    }
353
0
                }
354
0
            } else {
355
0
                uint16_t* dst_16 = dst16 + (bi << (bsizex + bsizey));
356
0
                for (iy = 0; iy < 1 << bsizey; iy += subsampling_factor) {
357
0
                    memcpy(dst_16 + (iy << bsizex),
358
0
                           src_16 + iy * CDEF_BSTRIDE,
359
0
                           (uint32_t)(1 << bsizex) * sizeof(uint16_t));
360
0
                }
361
0
            }
362
0
        }
363
0
        return;
364
0
    }
365
366
0
    if (pli == 0) {
367
0
        if (!dirinit || !*dirinit) {
368
0
            cdef_find_dir(in, dlist, var, cdef_count, coeff_shift, dir);
369
0
            if (dirinit) {
370
0
                *dirinit = 1;
371
0
            }
372
0
        }
373
0
    } else if (pli == 1 && xdec != ydec) {
374
0
        for (bi = 0; bi < cdef_count; bi++) {
375
0
            static const uint8_t conv422[8] = {7, 0, 2, 4, 5, 6, 6, 6};
376
0
            static const uint8_t conv440[8] = {1, 2, 2, 2, 3, 4, 6, 0};
377
378
0
            int32_t by  = dlist[bi].by;
379
0
            int32_t bx  = dlist[bi].bx;
380
0
            dir[by][bx] = (xdec ? conv422 : conv440)[dir[by][bx]];
381
0
        }
382
0
    }
383
384
0
    for (bi = 0; bi < cdef_count; bi++) {
385
0
        int32_t by = dlist[bi].by;
386
0
        int32_t bx = dlist[bi].bx;
387
0
        int32_t t  = pli ? pri_strength : adjust_strength(pri_strength, var[by][bx]);
388
0
        int32_t k  = dstride ? (by << bsizey) * dstride + (bx << bsizex) : bi << (bsizex + bsizey);
389
0
        svt_cdef_filter_block(dst8 ? &dst8[k] : NULL,
390
0
                              dst8 ? NULL : &dst16[k],
391
0
                              dstride ? dstride : 1 << bsizex,
392
0
                              &in[(by * CDEF_BSTRIDE << bsizey) + (bx << bsizex)],
393
0
                              t,
394
0
                              sec_strength,
395
0
                              pri_strength ? dir[by][bx] : 0,
396
0
                              damping,
397
0
                              damping,
398
0
                              bsize,
399
0
                              coeff_shift,
400
0
                              subsampling_factor);
401
0
    }
402
0
}
403
404
// Per-tap (drow,dcol) decode of eb_cdef_directions_padded, with the same +2-offset indexing as
405
// svt_aom_eb_cdef_directions. The boundary-aware kernels (C and NEON) use these to test, per tap,
406
// whether the tap lands off-frame (geometry) instead of reading an in-band 16-bit sentinel. Decoded
407
// directly from the flat-offset literals: each {r*CDEF_BSTRIDE + c} entry maps to {r, c}.
408
DECLARE_ALIGNED(16, const int8_t, eb_cdef_directions_padded_rc[12][2][2]) = {
409
    {{1, 0}, {2, 0}},
410
    {{1, 0}, {2, -1}},
411
    {{-1, 1}, {-2, 2}},
412
    {{0, 1}, {-1, 2}},
413
    {{0, 1}, {0, 2}},
414
    {{0, 1}, {1, 2}},
415
    {{1, 1}, {2, 2}},
416
    {{1, 0}, {2, 1}},
417
    {{1, 0}, {2, 0}},
418
    {{1, 0}, {2, -1}},
419
    {{-1, 1}, {-2, 2}},
420
    {{0, 1}, {-1, 2}},
421
};
422
const int8_t (*const svt_aom_eb_cdef_directions_rc)[2][2] = eb_cdef_directions_padded_rc + 2;
423
424
// True when tap (r,c), expressed in block-local coords, lands outside the frame: it crosses a
425
// block edge that is also a frame edge. Internal tile/SB boundaries are NOT off-frame (the
426
// caller only sets edge_* on true frame borders).
427
static INLINE int cdef_tap_off_frame(int r, int c, int rows, int cols, int edge_top, int edge_bottom, int edge_left,
428
0
                                     int edge_right) {
429
0
    return (edge_top && r < 0) || (edge_bottom && r >= rows) || (edge_left && c < 0) || (edge_right && c >= cols);
430
0
}
431
432
// Boundary-aware native 8-bit kernel: identical math to svt_cdef_filter_block_8bit_c, but excludes
433
// off-frame taps by geometry (the 4 per-block edge flags) instead of an in-band 16-bit sentinel.
434
// An off-frame tap is excluded from sum, max AND min, which exactly reproduces the 16-bit sentinel
435
// kernel (svt_cdef_filter_block_c): sentinel -> constrain==0 (sum), guarded for max, and large so it
436
// never wins min. Lets 8-bit content filter frame-perimeter ring blocks without a 16-bit buffer.
437
void svt_cdef_filter_block_8bit_bounded_c(uint8_t* dst, int32_t dstride, const uint8_t* in, int32_t pri_strength,
438
                                          int32_t sec_strength, int32_t dir, int32_t damping, int32_t bsize,
439
                                          int32_t coeff_shift, uint8_t subsampling_factor, int edge_top, int edge_left,
440
0
                                          int edge_bottom, int edge_right) {
441
0
    const int32_t  s        = CDEF_BSTRIDE;
442
0
    const int32_t* pri_taps = svt_aom_eb_cdef_pri_taps[(pri_strength >> coeff_shift) & 1];
443
0
    const int32_t* sec_taps = svt_aom_eb_cdef_sec_taps[(pri_strength >> coeff_shift) & 1];
444
0
    const int32_t  rows     = (bsize == BLOCK_8X8 || bsize == BLOCK_4X8) ? 8 : 4;
445
0
    const int32_t  cols     = (bsize == BLOCK_8X8 || bsize == BLOCK_8X4) ? 8 : 4;
446
0
    const int32_t  sub      = (bsize == BLOCK_4X4) ? 1 : subsampling_factor;
447
0
    for (int32_t i = 0; i < rows; i += sub) {
448
0
        for (int32_t j = 0; j < cols; j++) {
449
0
            const int16_t x   = in[i * s + j];
450
0
            int16_t       sum = 0;
451
0
            int32_t       max = x, min = x;
452
0
            for (int32_t k = 0; k < 2; k++) {
453
                // Primary taps p0 at (i+dr, j+dc), p1 at (i-dr, j-dc).
454
0
                const int     pdr  = svt_aom_eb_cdef_directions_rc[dir][k][0];
455
0
                const int     pdc  = svt_aom_eb_cdef_directions_rc[dir][k][1];
456
0
                const int32_t poff = svt_aom_eb_cdef_directions[dir][k];
457
0
                if (!cdef_tap_off_frame(i + pdr, j + pdc, rows, cols, edge_top, edge_bottom, edge_left, edge_right)) {
458
0
                    const int16_t p0 = in[i * s + j + poff];
459
0
                    sum += (int16_t)(pri_taps[k] * constrain(p0 - x, pri_strength, damping));
460
0
                    max = AOMMAX(p0, max);
461
0
                    min = AOMMIN(p0, min);
462
0
                }
463
0
                if (!cdef_tap_off_frame(i - pdr, j - pdc, rows, cols, edge_top, edge_bottom, edge_left, edge_right)) {
464
0
                    const int16_t p1 = in[i * s + j - poff];
465
0
                    sum += (int16_t)(pri_taps[k] * constrain(p1 - x, pri_strength, damping));
466
0
                    max = AOMMAX(p1, max);
467
0
                    min = AOMMIN(p1, min);
468
0
                }
469
                // Secondary taps from directions dir+2 and dir-2 (each used with +/- offset).
470
0
                const int     s0dr  = svt_aom_eb_cdef_directions_rc[dir + 2][k][0];
471
0
                const int     s0dc  = svt_aom_eb_cdef_directions_rc[dir + 2][k][1];
472
0
                const int32_t s0off = svt_aom_eb_cdef_directions[dir + 2][k];
473
0
                const int     s2dr  = svt_aom_eb_cdef_directions_rc[dir - 2][k][0];
474
0
                const int     s2dc  = svt_aom_eb_cdef_directions_rc[dir - 2][k][1];
475
0
                const int32_t s2off = svt_aom_eb_cdef_directions[dir - 2][k];
476
0
                if (!cdef_tap_off_frame(i + s0dr, j + s0dc, rows, cols, edge_top, edge_bottom, edge_left, edge_right)) {
477
0
                    const int16_t s0 = in[i * s + j + s0off];
478
0
                    sum += (int16_t)(sec_taps[k] * constrain(s0 - x, sec_strength, damping));
479
0
                    max = AOMMAX(s0, max);
480
0
                    min = AOMMIN(s0, min);
481
0
                }
482
0
                if (!cdef_tap_off_frame(i - s0dr, j - s0dc, rows, cols, edge_top, edge_bottom, edge_left, edge_right)) {
483
0
                    const int16_t s1 = in[i * s + j - s0off];
484
0
                    sum += (int16_t)(sec_taps[k] * constrain(s1 - x, sec_strength, damping));
485
0
                    max = AOMMAX(s1, max);
486
0
                    min = AOMMIN(s1, min);
487
0
                }
488
0
                if (!cdef_tap_off_frame(i + s2dr, j + s2dc, rows, cols, edge_top, edge_bottom, edge_left, edge_right)) {
489
0
                    const int16_t s2 = in[i * s + j + s2off];
490
0
                    sum += (int16_t)(sec_taps[k] * constrain(s2 - x, sec_strength, damping));
491
0
                    max = AOMMAX(s2, max);
492
0
                    min = AOMMIN(s2, min);
493
0
                }
494
0
                if (!cdef_tap_off_frame(i - s2dr, j - s2dc, rows, cols, edge_top, edge_bottom, edge_left, edge_right)) {
495
0
                    const int16_t s3 = in[i * s + j - s2off];
496
0
                    sum += (int16_t)(sec_taps[k] * constrain(s3 - x, sec_strength, damping));
497
0
                    max = AOMMAX(s3, max);
498
0
                    min = AOMMIN(s3, min);
499
0
                }
500
0
            }
501
0
            dst[i * dstride + j] = (uint8_t)clamp((int16_t)x + ((8 + sum - (sum < 0)) >> 4), min, max);
502
0
        }
503
0
    }
504
0
}
505
506
#if CDEF_8BITS_PATH
507
#include "cdef_copy.h"
508
509
static AOM_INLINE void cdef_find_dir_8bit(const uint8_t* in, CdefList* dlist, int32_t var[CDEF_NBLOCKS][CDEF_NBLOCKS],
510
                                          int32_t cdef_count, int32_t coeff_shift,
511
                                          uint8_t dir[CDEF_NBLOCKS][CDEF_NBLOCKS]) {
512
    int bi;
513
    for (bi = 0; bi < cdef_count - 1; bi += 2) {
514
        const uint8_t by   = dlist[bi].by;
515
        const uint8_t bx   = dlist[bi].bx;
516
        const uint8_t by2  = dlist[bi + 1].by;
517
        const uint8_t bx2  = dlist[bi + 1].bx;
518
        const int     pos1 = 8 * by * CDEF_BSTRIDE + 8 * bx;
519
        const int     pos2 = 8 * by2 * CDEF_BSTRIDE + 8 * bx2;
520
        svt_aom_cdef_find_dir_dual_8bit(&in[pos1],
521
                                        &in[pos2],
522
                                        CDEF_BSTRIDE,
523
                                        &var[by][bx],
524
                                        &var[by2][bx2],
525
                                        coeff_shift,
526
                                        &dir[by][bx],
527
                                        &dir[by2][bx2]);
528
    }
529
    if (cdef_count % 2) {
530
        const uint8_t by = dlist[bi].by;
531
        const uint8_t bx = dlist[bi].bx;
532
        dir[by][bx]      = svt_aom_cdef_find_dir_8bit(
533
            &in[8 * by * CDEF_BSTRIDE + 8 * bx], CDEF_BSTRIDE, &var[by][bx], coeff_shift);
534
    }
535
}
536
537
// in8: 8-bit padded buffer for all blocks. Interior-vs-frame-edge dispatch is decided from the
538
// frame_* geometry flags; frame-perimeter ring blocks use the boundary-aware kernel on in8.
539
// Body of svt_cdef_filter_fb_lbd, force-inlined so the per-plane call sites below specialize it with
540
// compile-time-constant bsize/bsizex/bsizey/is_luma. That constant block size then propagates into the
541
// (PGO/LTO-inlined) per-block kernels, which otherwise only see a runtime bsize.
542
static AOM_FORCE_INLINE void cdef_filter_fb_lbd_impl(uint8_t* dst8, int32_t dstride, const uint8_t* in8, int frame_top,
543
                                                     int frame_left, int frame_bottom, int frame_right, int vsize,
544
                                                     int hsize, int32_t xdec, int32_t ydec,
545
                                                     uint8_t dir[CDEF_NBLOCKS][CDEF_NBLOCKS], int32_t* dirinit,
546
                                                     int32_t var[CDEF_NBLOCKS][CDEF_NBLOCKS], int32_t pli,
547
                                                     CdefList* dlist, int32_t cdef_count, int32_t cdef_strength,
548
                                                     int32_t damping, int32_t coeff_shift, uint8_t subsampling_factor,
549
                                                     int32_t bsize, int32_t bsizex, int32_t bsizey, int is_luma) {
550
    int32_t bi;
551
    int32_t pri_strength = (cdef_strength / CDEF_SEC_STRENGTHS) << coeff_shift;
552
    int32_t sec          = cdef_strength % CDEF_SEC_STRENGTHS;
553
    int32_t sec_strength = (sec + (sec == 3)) << coeff_shift;
554
    damping += coeff_shift - (is_luma ? 0 : 1);
555
556
    // Last block row/col index within the fb (frame-edge ring detection).
557
    const int32_t by_last = (vsize >> bsizey) - 1;
558
    const int32_t bx_last = (hsize >> bsizex) - 1;
559
560
    if (!dstride && cdef_strength == 0) {
561
        // Zero-strength search candidate: copy input straight to packed output (4/8-byte rows).
562
        const int w = 1 << bsizex;
563
        if (w == 8) {
564
            for (bi = 0; bi < cdef_count; bi++) {
565
                const uint8_t* src_8 = in8 + ((dlist[bi].by << bsizey) * CDEF_BSTRIDE + (dlist[bi].bx << bsizex));
566
                uint8_t*       dst_b = dst8 + (bi << (bsizex + bsizey));
567
                for (int32_t iy = 0; iy < 1 << bsizey; iy += subsampling_factor) {
568
                    memcpy(dst_b + (iy << bsizex), src_8 + iy * CDEF_BSTRIDE, 8);
569
                }
570
            }
571
        } else {
572
            for (bi = 0; bi < cdef_count; bi++) {
573
                const uint8_t* src_8 = in8 + ((dlist[bi].by << bsizey) * CDEF_BSTRIDE + (dlist[bi].bx << bsizex));
574
                uint8_t*       dst_b = dst8 + (bi << (bsizex + bsizey));
575
                for (int32_t iy = 0; iy < 1 << bsizey; iy += subsampling_factor) {
576
                    memcpy(dst_b + (iy << bsizex), src_8 + iy * CDEF_BSTRIDE, 4);
577
                }
578
            }
579
        }
580
        return;
581
    }
582
583
    if (is_luma) {
584
        if (!dirinit || !*dirinit) {
585
            cdef_find_dir_8bit(in8, dlist, var, cdef_count, coeff_shift, dir);
586
            if (dirinit) {
587
                *dirinit = 1;
588
            }
589
        }
590
    } else if (pli == 1 && xdec != ydec) {
591
        for (bi = 0; bi < cdef_count; bi++) {
592
            static const uint8_t conv422[8] = {7, 0, 2, 4, 5, 6, 6, 6};
593
            static const uint8_t conv440[8] = {1, 2, 2, 2, 3, 4, 6, 0};
594
            int32_t              by         = dlist[bi].by;
595
            int32_t              bx         = dlist[bi].bx;
596
            dir[by][bx]                     = (xdec ? conv422 : conv440)[dir[by][bx]];
597
        }
598
    }
599
600
    // Strength 0 reaches here only as the "compute dir for later planes" pass; identity passthrough.
601
    if (cdef_strength == 0) {
602
        return;
603
    }
604
605
    bool edge_fb = frame_top || frame_left || frame_bottom || frame_right;
606
    for (bi = 0; bi < cdef_count; bi++) {
607
        int32_t   by          = dlist[bi].by;
608
        int32_t   bx          = dlist[bi].bx;
609
        int32_t   t           = is_luma ? adjust_strength(pri_strength, var[by][bx]) : pri_strength;
610
        int32_t   k           = dstride ? (by << bsizey) * dstride + (bx << bsizex) : bi << (bsizex + bsizey);
611
        int32_t   ds          = dstride ? dstride : 1 << bsizex;
612
        int32_t   off         = (by * CDEF_BSTRIDE << bsizey) + (bx << bsizex);
613
        int32_t   bdir        = pri_strength ? dir[by][bx] : 0;
614
        const int edge_top    = frame_top && by == 0;
615
        const int edge_bottom = frame_bottom && by == by_last;
616
        const int edge_left   = frame_left && bx == 0;
617
        const int edge_right  = frame_right && bx == bx_last;
618
        if (edge_fb && (edge_top || edge_bottom || edge_left || edge_right)) {
619
            svt_cdef_filter_block_8bit_bounded(&dst8[k],
620
                                               ds,
621
                                               &in8[off],
622
                                               t,
623
                                               sec_strength,
624
                                               bdir,
625
                                               damping,
626
                                               bsize,
627
                                               coeff_shift,
628
                                               subsampling_factor,
629
                                               edge_top,
630
                                               edge_left,
631
                                               edge_bottom,
632
                                               edge_right);
633
        } else {
634
            svt_cdef_filter_block_8bit(
635
                &dst8[k], ds, &in8[off], t, sec_strength, bdir, damping, bsize, coeff_shift, subsampling_factor);
636
        }
637
    }
638
}
639
640
static void cdef_filter_fb_lbd_luma(uint8_t* dst8, int32_t dstride, const uint8_t* in8, int frame_top, int frame_left,
641
                                    int frame_bottom, int frame_right, int vsize, int hsize,
642
                                    uint8_t dir[CDEF_NBLOCKS][CDEF_NBLOCKS], int32_t* dirinit,
643
                                    int32_t var[CDEF_NBLOCKS][CDEF_NBLOCKS], CdefList* dlist, int32_t cdef_count,
644
                                    int32_t cdef_strength, int32_t damping, int32_t coeff_shift,
645
                                    uint8_t subsampling_factor) {
646
    cdef_filter_fb_lbd_impl(dst8,
647
                            dstride,
648
                            in8,
649
                            frame_top,
650
                            frame_left,
651
                            frame_bottom,
652
                            frame_right,
653
                            vsize,
654
                            hsize,
655
                            0,
656
                            0,
657
                            dir,
658
                            dirinit,
659
                            var,
660
                            /*pli*/ 0,
661
                            dlist,
662
                            cdef_count,
663
                            cdef_strength,
664
                            damping,
665
                            coeff_shift,
666
                            subsampling_factor,
667
                            BLOCK_8X8,
668
                            3,
669
                            3,
670
                            /*is_luma*/ 1);
671
}
672
673
static void cdef_filter_fb_lbd_chroma(uint8_t* dst8, int32_t dstride, const uint8_t* in8, int frame_top, int frame_left,
674
                                      int frame_bottom, int frame_right, int vsize, int hsize, int32_t pli,
675
                                      uint8_t dir[CDEF_NBLOCKS][CDEF_NBLOCKS], int32_t* dirinit,
676
                                      int32_t var[CDEF_NBLOCKS][CDEF_NBLOCKS], CdefList* dlist, int32_t cdef_count,
677
                                      int32_t cdef_strength, int32_t damping, int32_t coeff_shift,
678
                                      uint8_t subsampling_factor) {
679
    cdef_filter_fb_lbd_impl(dst8,
680
                            dstride,
681
                            in8,
682
                            frame_top,
683
                            frame_left,
684
                            frame_bottom,
685
                            frame_right,
686
                            vsize,
687
                            hsize,
688
                            1,
689
                            1,
690
                            dir,
691
                            dirinit,
692
                            var,
693
                            pli,
694
                            dlist,
695
                            cdef_count,
696
                            cdef_strength,
697
                            damping,
698
                            coeff_shift,
699
                            subsampling_factor,
700
                            BLOCK_4X4,
701
                            2,
702
                            2,
703
                            /*is_luma*/ 0);
704
}
705
706
void svt_cdef_filter_fb_lbd(uint8_t* dst8, int32_t dstride, const uint8_t* in8, int frame_top, int frame_left,
707
                            int frame_bottom, int frame_right, int vsize, int hsize, int32_t xdec, int32_t ydec,
708
                            uint8_t dir[CDEF_NBLOCKS][CDEF_NBLOCKS], int32_t* dirinit,
709
                            int32_t var[CDEF_NBLOCKS][CDEF_NBLOCKS], int32_t pli, CdefList* dlist, int32_t cdef_count,
710
                            int32_t cdef_strength, int32_t damping, int32_t coeff_shift, uint8_t subsampling_factor) {
711
    if (xdec == 0 && ydec == 0) { // luma: BLOCK_8X8
712
        cdef_filter_fb_lbd_luma(dst8,
713
                                dstride,
714
                                in8,
715
                                frame_top,
716
                                frame_left,
717
                                frame_bottom,
718
                                frame_right,
719
                                vsize,
720
                                hsize,
721
                                dir,
722
                                dirinit,
723
                                var,
724
                                dlist,
725
                                cdef_count,
726
                                cdef_strength,
727
                                damping,
728
                                coeff_shift,
729
                                subsampling_factor);
730
    } else if (xdec == 1 && ydec == 1) { // chroma 4:2:0: BLOCK_4X4
731
        cdef_filter_fb_lbd_chroma(dst8,
732
                                  dstride,
733
                                  in8,
734
                                  frame_top,
735
                                  frame_left,
736
                                  frame_bottom,
737
                                  frame_right,
738
                                  vsize,
739
                                  hsize,
740
                                  pli,
741
                                  dir,
742
                                  dirinit,
743
                                  var,
744
                                  dlist,
745
                                  cdef_count,
746
                                  cdef_strength,
747
                                  damping,
748
                                  coeff_shift,
749
                                  subsampling_factor);
750
    } else { // 4:2:2 / 4:4:4: runtime block size
751
        const int32_t bsize = ydec ? (xdec ? BLOCK_4X4 : BLOCK_8X4) : (xdec ? BLOCK_4X8 : BLOCK_8X8);
752
        cdef_filter_fb_lbd_impl(dst8,
753
                                dstride,
754
                                in8,
755
                                frame_top,
756
                                frame_left,
757
                                frame_bottom,
758
                                frame_right,
759
                                vsize,
760
                                hsize,
761
                                xdec,
762
                                ydec,
763
                                dir,
764
                                dirinit,
765
                                var,
766
                                pli,
767
                                dlist,
768
                                cdef_count,
769
                                cdef_strength,
770
                                damping,
771
                                coeff_shift,
772
                                subsampling_factor,
773
                                bsize,
774
                                3 - xdec,
775
                                3 - ydec,
776
                                pli == 0);
777
    }
778
}
779
#endif // CDEF_8BITS_PATH