Coverage Report

Created: 2026-08-31 07:01

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/libavif/ext/aom/av1/encoder/pickcdef.c
Line
Count
Source
1
/*
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved.
3
 *
4
 * This source code is subject to the terms of the BSD 2 Clause License and
5
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6
 * was not distributed with this source code in the LICENSE file, you can
7
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8
 * Media Patent License 1.0 was not distributed with this source code in the
9
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10
 */
11
12
#include <math.h>
13
#include <stdbool.h>
14
#include <stddef.h>
15
#include <string.h>
16
17
#include "config/aom_dsp_rtcd.h"
18
#include "config/aom_scale_rtcd.h"
19
20
#include "aom/aom_integer.h"
21
#include "av1/common/av1_common_int.h"
22
#include "av1/common/reconinter.h"
23
#include "av1/encoder/encoder.h"
24
#include "av1/encoder/ethread.h"
25
#include "av1/encoder/pickcdef.h"
26
27
// Get primary and secondary filter strength for the given strength index and
28
// search method
29
static inline void get_cdef_filter_strengths(CDEF_PICK_METHOD pick_method,
30
                                             int *pri_strength,
31
                                             int *sec_strength,
32
1.70M
                                             int strength_idx) {
33
1.70M
  const int tot_sec_filter =
34
1.70M
      (pick_method == CDEF_FAST_SEARCH_LVL5)
35
1.70M
          ? REDUCED_SEC_STRENGTHS_LVL5
36
1.70M
          : ((pick_method >= CDEF_FAST_SEARCH_LVL3) ? REDUCED_SEC_STRENGTHS_LVL3
37
18.4E
                                                    : CDEF_SEC_STRENGTHS);
38
1.70M
  const int pri_idx = strength_idx / tot_sec_filter;
39
1.70M
  const int sec_idx = strength_idx % tot_sec_filter;
40
1.70M
  *pri_strength = pri_idx;
41
1.70M
  *sec_strength = sec_idx;
42
1.70M
  if (pick_method == CDEF_FULL_SEARCH) return;
43
44
1.70M
  switch (pick_method) {
45
0
    case CDEF_FAST_SEARCH_LVL1:
46
0
      assert(pri_idx < REDUCED_PRI_STRENGTHS_LVL1);
47
0
      *pri_strength = priconv_lvl1[pri_idx];
48
0
      break;
49
0
    case CDEF_FAST_SEARCH_LVL2:
50
0
      assert(pri_idx < REDUCED_PRI_STRENGTHS_LVL2);
51
0
      *pri_strength = priconv_lvl2[pri_idx];
52
0
      break;
53
0
    case CDEF_FAST_SEARCH_LVL3:
54
0
      assert(pri_idx < REDUCED_PRI_STRENGTHS_LVL2);
55
0
      assert(sec_idx < REDUCED_SEC_STRENGTHS_LVL3);
56
0
      *pri_strength = priconv_lvl2[pri_idx];
57
0
      *sec_strength = secconv_lvl3[sec_idx];
58
0
      break;
59
1.70M
    case CDEF_FAST_SEARCH_LVL4:
60
1.70M
      assert(pri_idx < REDUCED_PRI_STRENGTHS_LVL4);
61
1.70M
      assert(sec_idx < REDUCED_SEC_STRENGTHS_LVL3);
62
1.70M
      *pri_strength = priconv_lvl4[pri_idx];
63
1.70M
      *sec_strength = secconv_lvl3[sec_idx];
64
1.70M
      break;
65
0
    case CDEF_FAST_SEARCH_LVL5:
66
0
      assert(pri_idx < REDUCED_PRI_STRENGTHS_LVL4);
67
0
      assert(sec_idx < REDUCED_SEC_STRENGTHS_LVL5);
68
0
      *pri_strength = priconv_lvl5[pri_idx];
69
0
      *sec_strength = secconv_lvl5[sec_idx];
70
0
      break;
71
0
    default: assert(0 && "Invalid CDEF search method");
72
1.70M
  }
73
1.70M
}
74
75
// Store CDEF filter strength calculated from strength index for given search
76
// method
77
#define STORE_CDEF_FILTER_STRENGTH(cdef_strength, pick_method, strength_idx) \
78
120k
  do {                                                                       \
79
120k
    get_cdef_filter_strengths((pick_method), &pri_strength, &sec_strength,   \
80
120k
                              (strength_idx));                               \
81
120k
    cdef_strength = pri_strength * CDEF_SEC_STRENGTHS + sec_strength;        \
82
120k
  } while (0)
83
84
/* Search for the best strength to add as an option, knowing we
85
   already selected nb_strengths options. */
86
static uint64_t search_one(int *lev, int nb_strengths,
87
                           uint64_t mse[][TOTAL_STRENGTHS], int sb_count,
88
240k
                           CDEF_PICK_METHOD pick_method) {
89
240k
  uint64_t tot_mse[TOTAL_STRENGTHS];
90
240k
  const int total_strengths = nb_cdef_strengths[pick_method];
91
240k
  int i, j;
92
240k
  uint64_t best_tot_mse = (uint64_t)1 << 63;
93
240k
  int best_id = 0;
94
240k
  memset(tot_mse, 0, sizeof(tot_mse));
95
978k
  for (i = 0; i < sb_count; i++) {
96
737k
    int gi;
97
737k
    uint64_t best_mse = (uint64_t)1 << 63;
98
    /* Find best mse among already selected options. */
99
1.49M
    for (gi = 0; gi < nb_strengths; gi++) {
100
759k
      if (mse[i][lev[gi]] < best_mse) {
101
493k
        best_mse = mse[i][lev[gi]];
102
493k
      }
103
759k
    }
104
    /* Find best mse when adding each possible new option. */
105
3.68M
    for (j = 0; j < total_strengths; j++) {
106
2.95M
      uint64_t best = best_mse;
107
2.95M
      if (mse[i][j] < best) best = mse[i][j];
108
2.95M
      tot_mse[j] += best;
109
2.95M
    }
110
737k
  }
111
1.20M
  for (j = 0; j < total_strengths; j++) {
112
962k
    if (tot_mse[j] < best_tot_mse) {
113
405k
      best_tot_mse = tot_mse[j];
114
405k
      best_id = j;
115
405k
    }
116
962k
  }
117
240k
  lev[nb_strengths] = best_id;
118
240k
  return best_tot_mse;
119
240k
}
120
121
/* Search for the best luma+chroma strength to add as an option, knowing we
122
   already selected nb_strengths options. */
123
static uint64_t search_one_dual(int *lev0, int *lev1, int nb_strengths,
124
                                uint64_t (**mse)[TOTAL_STRENGTHS], int sb_count,
125
2.02M
                                CDEF_PICK_METHOD pick_method) {
126
2.02M
  uint64_t tot_mse[TOTAL_STRENGTHS][TOTAL_STRENGTHS];
127
2.02M
  int i, j;
128
2.02M
  uint64_t best_tot_mse = (uint64_t)1 << 63;
129
2.02M
  int best_id0 = 0;
130
2.02M
  int best_id1 = 0;
131
2.02M
  const int total_strengths = nb_cdef_strengths[pick_method];
132
2.02M
  memset(tot_mse, 0, sizeof(tot_mse));
133
9.19M
  for (i = 0; i < sb_count; i++) {
134
7.16M
    int gi;
135
7.16M
    uint64_t best_mse = (uint64_t)1 << 63;
136
    /* Find best mse among already selected options. */
137
37.6M
    for (gi = 0; gi < nb_strengths; gi++) {
138
30.5M
      uint64_t curr = mse[0][i][lev0[gi]];
139
30.5M
      curr += mse[1][i][lev1[gi]];
140
30.5M
      if (curr < best_mse) {
141
11.0M
        best_mse = curr;
142
11.0M
      }
143
30.5M
    }
144
    /* Find best mse when adding each possible new option. */
145
35.8M
    for (j = 0; j < total_strengths; j++) {
146
28.6M
      int k;
147
143M
      for (k = 0; k < total_strengths; k++) {
148
114M
        uint64_t best = best_mse;
149
114M
        uint64_t curr = mse[0][i][j];
150
114M
        curr += mse[1][i][k];
151
114M
        if (curr < best) best = curr;
152
114M
        tot_mse[j][k] += best;
153
114M
      }
154
28.6M
    }
155
7.16M
  }
156
10.1M
  for (j = 0; j < total_strengths; j++) {
157
8.10M
    int k;
158
40.5M
    for (k = 0; k < total_strengths; k++) {
159
32.4M
      if (tot_mse[j][k] < best_tot_mse) {
160
4.34M
        best_tot_mse = tot_mse[j][k];
161
4.34M
        best_id0 = j;
162
4.34M
        best_id1 = k;
163
4.34M
      }
164
32.4M
    }
165
8.10M
  }
166
2.02M
  lev0[nb_strengths] = best_id0;
167
2.02M
  lev1[nb_strengths] = best_id1;
168
2.02M
  return best_tot_mse;
169
2.02M
}
170
171
/* Search for the set of strengths that minimizes mse. */
172
static uint64_t joint_strength_search(int *best_lev, int nb_strengths,
173
                                      uint64_t mse[][TOTAL_STRENGTHS],
174
                                      int sb_count,
175
100k
                                      CDEF_PICK_METHOD pick_method) {
176
100k
  uint64_t best_tot_mse;
177
100k
  int fast = (pick_method >= CDEF_FAST_SEARCH_LVL1 &&
178
100k
              pick_method <= CDEF_FAST_SEARCH_LVL5);
179
100k
  int i;
180
100k
  best_tot_mse = (uint64_t)1 << 63;
181
  /* Greedy search: add one strength options at a time. */
182
341k
  for (i = 0; i < nb_strengths; i++) {
183
240k
    best_tot_mse = search_one(best_lev, i, mse, sb_count, pick_method);
184
240k
  }
185
  /* Trying to refine the greedy search by reconsidering each
186
     already-selected option. */
187
100k
  if (!fast) {
188
0
    for (i = 0; i < 4 * nb_strengths; i++) {
189
0
      int j;
190
0
      for (j = 0; j < nb_strengths - 1; j++) best_lev[j] = best_lev[j + 1];
191
0
      best_tot_mse =
192
0
          search_one(best_lev, nb_strengths - 1, mse, sb_count, pick_method);
193
0
    }
194
0
  }
195
100k
  return best_tot_mse;
196
100k
}
197
198
/* Search for the set of luma+chroma strengths that minimizes mse. */
199
static uint64_t joint_strength_search_dual(int *best_lev0, int *best_lev1,
200
                                           int nb_strengths,
201
                                           uint64_t (**mse)[TOTAL_STRENGTHS],
202
                                           int sb_count,
203
104k
                                           CDEF_PICK_METHOD pick_method) {
204
104k
  uint64_t best_tot_mse;
205
104k
  int i;
206
104k
  best_tot_mse = (uint64_t)1 << 63;
207
  /* Greedy search: add one strength options at a time. */
208
510k
  for (i = 0; i < nb_strengths; i++) {
209
405k
    best_tot_mse =
210
405k
        search_one_dual(best_lev0, best_lev1, i, mse, sb_count, pick_method);
211
405k
  }
212
  /* Trying to refine the greedy search by reconsidering each
213
     already-selected option. */
214
1.72M
  for (i = 0; i < 4 * nb_strengths; i++) {
215
1.62M
    int j;
216
9.27M
    for (j = 0; j < nb_strengths - 1; j++) {
217
7.65M
      best_lev0[j] = best_lev0[j + 1];
218
7.65M
      best_lev1[j] = best_lev1[j + 1];
219
7.65M
    }
220
1.62M
    best_tot_mse = search_one_dual(best_lev0, best_lev1, nb_strengths - 1, mse,
221
1.62M
                                   sb_count, pick_method);
222
1.62M
  }
223
104k
  return best_tot_mse;
224
104k
}
225
226
static inline void init_src_params(int *src_stride, int *width, int *height,
227
                                   int *width_log2, int *height_log2,
228
367k
                                   BLOCK_SIZE bsize) {
229
367k
  *src_stride = block_size_wide[bsize];
230
367k
  *width = block_size_wide[bsize];
231
367k
  *height = block_size_high[bsize];
232
367k
  *width_log2 = MI_SIZE_LOG2 + mi_size_wide_log2[bsize];
233
367k
  *height_log2 = MI_SIZE_LOG2 + mi_size_high_log2[bsize];
234
367k
}
235
#if CONFIG_AV1_HIGHBITDEPTH
236
/* Compute MSE only on the blocks we filtered. */
237
static uint64_t compute_cdef_dist_highbd(void *dst, int dstride, uint16_t *src,
238
                                         cdef_list *dlist, int cdef_count,
239
                                         BLOCK_SIZE bsize, int coeff_shift,
240
367k
                                         int row, int col) {
241
367k
  assert(bsize == BLOCK_4X4 || bsize == BLOCK_4X8 || bsize == BLOCK_8X4 ||
242
367k
         bsize == BLOCK_8X8);
243
367k
  uint64_t sum = 0;
244
367k
  int bi, bx, by;
245
367k
  uint16_t *dst16 = CONVERT_TO_SHORTPTR((uint8_t *)dst);
246
367k
  uint16_t *dst_buff = &dst16[row * dstride + col];
247
367k
  int src_stride, width, height, width_log2, height_log2;
248
367k
  init_src_params(&src_stride, &width, &height, &width_log2, &height_log2,
249
367k
                  bsize);
250
7.06M
  for (bi = 0; bi < cdef_count; bi++) {
251
6.69M
    by = dlist[bi].by;
252
6.69M
    bx = dlist[bi].bx;
253
6.69M
    sum += aom_mse_wxh_16bit_highbd(
254
6.69M
        &dst_buff[(by << height_log2) * dstride + (bx << width_log2)], dstride,
255
6.69M
        &src[bi << (height_log2 + width_log2)], src_stride, width, height);
256
6.69M
  }
257
367k
  return sum >> 2 * coeff_shift;
258
367k
}
259
#endif
260
261
// Checks dual and quad block processing is applicable for block widths 8 and 4
262
// respectively.
263
static inline int is_dual_or_quad_applicable(cdef_list *dlist, int width,
264
0
                                             int cdef_count, int bi, int iter) {
265
0
  assert(width == 8 || width == 4);
266
0
  const int blk_offset = (width == 8) ? 1 : 3;
267
0
  if ((iter + blk_offset) >= cdef_count) return 0;
268
269
0
  if (dlist[bi].by == dlist[bi + blk_offset].by &&
270
0
      dlist[bi].bx + blk_offset == dlist[bi + blk_offset].bx)
271
0
    return 1;
272
273
0
  return 0;
274
0
}
275
276
static uint64_t compute_cdef_dist(void *dst, int dstride, uint16_t *src,
277
                                  cdef_list *dlist, int cdef_count,
278
                                  BLOCK_SIZE bsize, int coeff_shift, int row,
279
0
                                  int col) {
280
0
  assert(bsize == BLOCK_4X4 || bsize == BLOCK_4X8 || bsize == BLOCK_8X4 ||
281
0
         bsize == BLOCK_8X8);
282
0
  uint64_t sum = 0;
283
0
  int bi, bx, by;
284
0
  int iter = 0;
285
0
  int inc = 1;
286
0
  uint8_t *dst8 = (uint8_t *)dst;
287
0
  uint8_t *dst_buff = &dst8[row * dstride + col];
288
0
  int src_stride, width, height, width_log2, height_log2;
289
0
  init_src_params(&src_stride, &width, &height, &width_log2, &height_log2,
290
0
                  bsize);
291
292
0
  const int num_blks = 16 / width;
293
0
  for (bi = 0; bi < cdef_count; bi += inc) {
294
0
    by = dlist[bi].by;
295
0
    bx = dlist[bi].bx;
296
0
    uint16_t *src_tmp = &src[bi << (height_log2 + width_log2)];
297
0
    uint8_t *dst_tmp =
298
0
        &dst_buff[(by << height_log2) * dstride + (bx << width_log2)];
299
300
0
    if (is_dual_or_quad_applicable(dlist, width, cdef_count, bi, iter)) {
301
0
      sum += aom_mse_16xh_16bit(dst_tmp, dstride, src_tmp, width, height);
302
0
      iter += num_blks;
303
0
      inc = num_blks;
304
0
    } else {
305
0
      sum += aom_mse_wxh_16bit(dst_tmp, dstride, src_tmp, src_stride, width,
306
0
                               height);
307
0
      iter += 1;
308
0
      inc = 1;
309
0
    }
310
0
  }
311
312
0
  return sum >> 2 * coeff_shift;
313
0
}
314
315
// Fill the boundary regions of the block with CDEF_VERY_LARGE, only if the
316
// region is outside frame boundary
317
static inline void fill_borders_for_fbs_on_frame_boundary(
318
    uint16_t *inbuf, int hfilt_size, int vfilt_size,
319
    bool is_fb_on_frm_left_boundary, bool is_fb_on_frm_right_boundary,
320
397k
    bool is_fb_on_frm_top_boundary, bool is_fb_on_frm_bottom_boundary) {
321
397k
  if (!is_fb_on_frm_left_boundary && !is_fb_on_frm_right_boundary &&
322
122k
      !is_fb_on_frm_top_boundary && !is_fb_on_frm_bottom_boundary)
323
43.0k
    return;
324
354k
  if (is_fb_on_frm_bottom_boundary) {
325
    // Fill bottom region of the block
326
209k
    const int buf_offset =
327
209k
        (vfilt_size + CDEF_VBORDER) * CDEF_BSTRIDE + CDEF_HBORDER;
328
209k
    fill_rect(&inbuf[buf_offset], CDEF_BSTRIDE, CDEF_VBORDER, hfilt_size,
329
209k
              CDEF_VERY_LARGE);
330
209k
  }
331
354k
  if (is_fb_on_frm_bottom_boundary || is_fb_on_frm_left_boundary) {
332
300k
    const int buf_offset = (vfilt_size + CDEF_VBORDER) * CDEF_BSTRIDE;
333
    // Fill bottom-left region of the block
334
300k
    fill_rect(&inbuf[buf_offset], CDEF_BSTRIDE, CDEF_VBORDER, CDEF_HBORDER,
335
300k
              CDEF_VERY_LARGE);
336
300k
  }
337
354k
  if (is_fb_on_frm_bottom_boundary || is_fb_on_frm_right_boundary) {
338
300k
    const int buf_offset =
339
300k
        (vfilt_size + CDEF_VBORDER) * CDEF_BSTRIDE + hfilt_size + CDEF_HBORDER;
340
    // Fill bottom-right region of the block
341
300k
    fill_rect(&inbuf[buf_offset], CDEF_BSTRIDE, CDEF_VBORDER, CDEF_HBORDER,
342
300k
              CDEF_VERY_LARGE);
343
300k
  }
344
354k
  if (is_fb_on_frm_top_boundary) {
345
    // Fill top region of the block
346
209k
    fill_rect(&inbuf[CDEF_HBORDER], CDEF_BSTRIDE, CDEF_VBORDER, hfilt_size,
347
209k
              CDEF_VERY_LARGE);
348
209k
  }
349
354k
  if (is_fb_on_frm_top_boundary || is_fb_on_frm_left_boundary) {
350
    // Fill top-left region of the block
351
300k
    fill_rect(inbuf, CDEF_BSTRIDE, CDEF_VBORDER, CDEF_HBORDER, CDEF_VERY_LARGE);
352
300k
  }
353
354k
  if (is_fb_on_frm_top_boundary || is_fb_on_frm_right_boundary) {
354
300k
    const int buf_offset = hfilt_size + CDEF_HBORDER;
355
    // Fill top-right region of the block
356
300k
    fill_rect(&inbuf[buf_offset], CDEF_BSTRIDE, CDEF_VBORDER, CDEF_HBORDER,
357
300k
              CDEF_VERY_LARGE);
358
300k
  }
359
354k
  if (is_fb_on_frm_left_boundary) {
360
202k
    const int buf_offset = CDEF_VBORDER * CDEF_BSTRIDE;
361
    // Fill left region of the block
362
202k
    fill_rect(&inbuf[buf_offset], CDEF_BSTRIDE, vfilt_size, CDEF_HBORDER,
363
202k
              CDEF_VERY_LARGE);
364
202k
  }
365
354k
  if (is_fb_on_frm_right_boundary) {
366
202k
    const int buf_offset = CDEF_VBORDER * CDEF_BSTRIDE;
367
    // Fill right region of the block
368
202k
    fill_rect(&inbuf[buf_offset + hfilt_size + CDEF_HBORDER], CDEF_BSTRIDE,
369
202k
              vfilt_size, CDEF_HBORDER, CDEF_VERY_LARGE);
370
202k
  }
371
354k
}
372
373
// Calculate the number of 8x8/4x4 filter units for which SSE can be calculated
374
// after CDEF filtering in single function call
375
static AOM_FORCE_INLINE int get_error_calc_width_in_filt_units(
376
    cdef_list *dlist, int cdef_count, int bi, int subsampling_x,
377
5.83M
    int subsampling_y) {
378
  // TODO(Ranjit): Extend the optimization for 422
379
5.83M
  if (subsampling_x != subsampling_y) return 1;
380
381
  // Combining more blocks seems to increase encode time due to increase in
382
  // control code
383
4.22M
  if (bi + 3 < cdef_count && dlist[bi].by == dlist[bi + 3].by &&
384
1.60M
      dlist[bi].bx + 3 == dlist[bi + 3].bx) {
385
    /* Calculate error for four 8x8/4x4 blocks using 32x8/16x4 block specific
386
     * logic if y co-ordinates match and x co-ordinates are
387
     * separated by 3 for first and fourth 8x8/4x4 blocks in dlist[]. */
388
1.60M
    return 4;
389
1.60M
  }
390
2.61M
  if (bi + 1 < cdef_count && dlist[bi].by == dlist[bi + 1].by &&
391
847k
      dlist[bi].bx + 1 == dlist[bi + 1].bx) {
392
    /* Calculate error for two 8x8/4x4 blocks using 16x8/8x4 block specific
393
     * logic if their y co-ordinates match and x co-ordinates are
394
     * separated by 1 for first and second 8x8/4x4 blocks in dlist[]. */
395
844k
    return 2;
396
844k
  }
397
1.77M
  return 1;
398
2.61M
}
399
400
// Returns the block error after CDEF filtering for a given strength
401
static inline uint64_t get_filt_error(
402
    const CdefSearchCtx *cdef_search_ctx, const struct macroblockd_plane *pd,
403
    cdef_list *dlist, int dir[CDEF_NBLOCKS][CDEF_NBLOCKS], int *dirinit,
404
    int var[CDEF_NBLOCKS][CDEF_NBLOCKS], uint16_t *in, uint8_t *ref_buffer,
405
    int ref_stride, int row, int col, int pri_strength, int sec_strength,
406
1.58M
    int cdef_count, int pli, int coeff_shift, BLOCK_SIZE bs) {
407
1.58M
  uint64_t curr_sse = 0;
408
1.58M
  const BLOCK_SIZE plane_bsize =
409
1.58M
      get_plane_block_size(bs, pd->subsampling_x, pd->subsampling_y);
410
1.58M
  const int bw_log2 = 3 - pd->subsampling_x;
411
1.58M
  const int bh_log2 = 3 - pd->subsampling_y;
412
413
  // TODO(Ranjit): Extend this optimization for HBD
414
1.58M
  if (!cdef_search_ctx->use_highbitdepth) {
415
    // If all 8x8/4x4 blocks in CDEF block need to be filtered, calculate the
416
    // error at CDEF block level
417
1.21M
    const int tot_blk_count =
418
1.21M
        (block_size_wide[plane_bsize] * block_size_high[plane_bsize]) >>
419
1.21M
        (bw_log2 + bh_log2);
420
1.21M
    if (cdef_count == tot_blk_count) {
421
426k
      const ptrdiff_t buf_offset = (ptrdiff_t)row * ref_stride + col;
422
426k
      const ptrdiff_t dst_offset = (ptrdiff_t)row * pd->dst.stride + col;
423
426k
      if (pri_strength == 0 && sec_strength == 0) {
424
        // When CDEF strength is zero, filtering is not applied. Hence
425
        // error is calculated between source and unfiltered pixels
426
106k
        curr_sse =
427
106k
            aom_sse(&ref_buffer[buf_offset], ref_stride,
428
106k
                    &pd->dst.buf[dst_offset], pd->dst.stride,
429
106k
                    block_size_wide[plane_bsize], block_size_high[plane_bsize]);
430
319k
      } else {
431
319k
        DECLARE_ALIGNED(32, uint8_t, tmp_dst8[1 << (MAX_SB_SIZE_LOG2 * 2)]);
432
433
319k
        av1_cdef_filter_fb(tmp_dst8, NULL, (1 << MAX_SB_SIZE_LOG2), in,
434
319k
                           cdef_search_ctx->xdec[pli],
435
319k
                           cdef_search_ctx->ydec[pli], dir, dirinit, var, pli,
436
319k
                           dlist, cdef_count, pri_strength,
437
319k
                           sec_strength + (sec_strength == 3),
438
319k
                           cdef_search_ctx->damping, coeff_shift);
439
319k
        curr_sse =
440
319k
            aom_sse(&ref_buffer[buf_offset], ref_stride, tmp_dst8,
441
319k
                    (1 << MAX_SB_SIZE_LOG2), block_size_wide[plane_bsize],
442
319k
                    block_size_high[plane_bsize]);
443
319k
      }
444
791k
    } else {
445
      // If few 8x8/4x4 blocks in CDEF block need to be filtered, filtering
446
      // functions produce 8-bit output and the error is calculated in 8-bit
447
      // domain
448
791k
      if (pri_strength == 0 && sec_strength == 0) {
449
198k
        int num_error_calc_filt_units = 1;
450
1.67M
        for (int bi = 0; bi < cdef_count; bi = bi + num_error_calc_filt_units) {
451
1.47M
          const uint8_t by = dlist[bi].by;
452
1.47M
          const uint8_t bx = dlist[bi].bx;
453
1.47M
          const int by_pos = by << bh_log2;
454
1.47M
          const int bx_pos = bx << bw_log2;
455
1.47M
          const ptrdiff_t buf_offset =
456
1.47M
              (ptrdiff_t)(row + by_pos) * ref_stride + (col + bx_pos);
457
1.47M
          const ptrdiff_t dst_offset =
458
1.47M
              (ptrdiff_t)(row + by_pos) * pd->dst.stride + (col + bx_pos);
459
1.47M
          num_error_calc_filt_units = get_error_calc_width_in_filt_units(
460
1.47M
              dlist, cdef_count, bi, pd->subsampling_x, pd->subsampling_y);
461
1.47M
          curr_sse +=
462
1.47M
              aom_sse(&ref_buffer[buf_offset], ref_stride,
463
1.47M
                      &pd->dst.buf[dst_offset], pd->dst.stride,
464
1.47M
                      num_error_calc_filt_units * (1 << bw_log2), 1 << bh_log2);
465
1.47M
        }
466
593k
      } else {
467
593k
        DECLARE_ALIGNED(32, uint8_t, tmp_dst8[1 << (MAX_SB_SIZE_LOG2 * 2)]);
468
593k
        av1_cdef_filter_fb(tmp_dst8, NULL, (1 << MAX_SB_SIZE_LOG2), in,
469
593k
                           cdef_search_ctx->xdec[pli],
470
593k
                           cdef_search_ctx->ydec[pli], dir, dirinit, var, pli,
471
593k
                           dlist, cdef_count, pri_strength,
472
593k
                           sec_strength + (sec_strength == 3),
473
593k
                           cdef_search_ctx->damping, coeff_shift);
474
593k
        int num_error_calc_filt_units = 1;
475
4.98M
        for (int bi = 0; bi < cdef_count; bi = bi + num_error_calc_filt_units) {
476
4.39M
          const uint8_t by = dlist[bi].by;
477
4.39M
          const uint8_t bx = dlist[bi].bx;
478
4.39M
          const int by_pos = by << bh_log2;
479
4.39M
          const int bx_pos = bx << bw_log2;
480
4.39M
          const ptrdiff_t buf_offset =
481
4.39M
              (ptrdiff_t)(row + by_pos) * ref_stride + (col + bx_pos);
482
4.39M
          const ptrdiff_t tmp_buf_offset =
483
4.39M
              by_pos * (1 << MAX_SB_SIZE_LOG2) + bx_pos;
484
4.39M
          num_error_calc_filt_units = get_error_calc_width_in_filt_units(
485
4.39M
              dlist, cdef_count, bi, pd->subsampling_x, pd->subsampling_y);
486
4.39M
          curr_sse += aom_sse(
487
4.39M
              &ref_buffer[buf_offset], ref_stride, &tmp_dst8[tmp_buf_offset],
488
4.39M
              (1 << MAX_SB_SIZE_LOG2),
489
4.39M
              num_error_calc_filt_units * (1 << bw_log2), (1 << bh_log2));
490
4.39M
        }
491
593k
      }
492
791k
    }
493
1.21M
  } else {
494
369k
    DECLARE_ALIGNED(32, uint16_t, tmp_dst[1 << (MAX_SB_SIZE_LOG2 * 2)]);
495
496
369k
    av1_cdef_filter_fb(NULL, tmp_dst, CDEF_BSTRIDE, in,
497
369k
                       cdef_search_ctx->xdec[pli], cdef_search_ctx->ydec[pli],
498
369k
                       dir, dirinit, var, pli, dlist, cdef_count, pri_strength,
499
369k
                       sec_strength + (sec_strength == 3),
500
369k
                       cdef_search_ctx->damping, coeff_shift);
501
369k
    curr_sse = cdef_search_ctx->compute_cdef_dist_fn(
502
369k
        ref_buffer, ref_stride, tmp_dst, dlist, cdef_count,
503
369k
        cdef_search_ctx->bsize[pli], coeff_shift, row, col);
504
369k
  }
505
1.58M
  return curr_sse;
506
1.58M
}
507
508
// Calculates MSE at block level.
509
// Inputs:
510
//   cdef_search_ctx: Pointer to the structure containing parameters related to
511
//   CDEF search context.
512
//   fbr: Row index in units of 64x64 block
513
//   fbc: Column index in units of 64x64 block
514
//   adaptive_cdef_mode: Speed feature to control CDEF adaptively.
515
// Returns:
516
//   Nothing will be returned. Contents of cdef_search_ctx will be modified.
517
void av1_cdef_mse_calc_block(CdefSearchCtx *cdef_search_ctx,
518
                             struct aom_internal_error_info *error_info,
519
                             int fbr, int fbc, int sb_count,
520
205k
                             int adaptive_cdef_mode) {
521
  // TODO(aomedia:3276): Pass error_info to the low-level functions as required
522
  // in future to handle error propagation.
523
205k
  (void)error_info;
524
205k
  const CommonModeInfoParams *const mi_params = cdef_search_ctx->mi_params;
525
205k
  const YV12_BUFFER_CONFIG *ref = cdef_search_ctx->ref;
526
205k
  const int coeff_shift = cdef_search_ctx->coeff_shift;
527
205k
  const int *mi_wide_l2 = cdef_search_ctx->mi_wide_l2;
528
205k
  const int *mi_high_l2 = cdef_search_ctx->mi_high_l2;
529
530
  // Declare and initialize the temporary buffers.
531
205k
  DECLARE_ALIGNED(32, uint16_t, inbuf[CDEF_INBUF_SIZE]);
532
205k
  cdef_list dlist[MI_SIZE_128X128 * MI_SIZE_128X128];
533
205k
  int dir[CDEF_NBLOCKS][CDEF_NBLOCKS] = { { 0 } };
534
205k
  int var[CDEF_NBLOCKS][CDEF_NBLOCKS] = { { 0 } };
535
205k
  uint16_t *const in = inbuf + CDEF_VBORDER * CDEF_BSTRIDE + CDEF_HBORDER;
536
205k
  int nhb = AOMMIN(MI_SIZE_64X64, mi_params->mi_cols - MI_SIZE_64X64 * fbc);
537
205k
  int nvb = AOMMIN(MI_SIZE_64X64, mi_params->mi_rows - MI_SIZE_64X64 * fbr);
538
205k
  int hb_step = 1, vb_step = 1;
539
205k
  BLOCK_SIZE bs;
540
541
205k
  const MB_MODE_INFO *const mbmi =
542
205k
      mi_params->mi_grid_base[MI_SIZE_64X64 * fbr * mi_params->mi_stride +
543
205k
                              MI_SIZE_64X64 * fbc];
544
545
205k
  uint8_t *ref_buffer[MAX_MB_PLANE] = { ref->y_buffer, ref->u_buffer,
546
205k
                                        ref->v_buffer };
547
205k
  int ref_stride[MAX_MB_PLANE] = { ref->y_stride, ref->uv_stride,
548
205k
                                   ref->uv_stride };
549
550
205k
  if (mbmi->bsize == BLOCK_128X128 || mbmi->bsize == BLOCK_128X64 ||
551
204k
      mbmi->bsize == BLOCK_64X128) {
552
0
    bs = mbmi->bsize;
553
0
    if (bs == BLOCK_128X128 || bs == BLOCK_128X64) {
554
0
      nhb = AOMMIN(MI_SIZE_128X128, mi_params->mi_cols - MI_SIZE_64X64 * fbc);
555
0
      hb_step = 2;
556
0
    }
557
0
    if (bs == BLOCK_128X128 || bs == BLOCK_64X128) {
558
0
      nvb = AOMMIN(MI_SIZE_128X128, mi_params->mi_rows - MI_SIZE_64X64 * fbr);
559
0
      vb_step = 2;
560
0
    }
561
205k
  } else {
562
205k
    bs = BLOCK_64X64;
563
205k
  }
564
  // Get number of 8x8 blocks which are not skip. Cdef processing happens for
565
  // 8x8 blocks which are not skip.
566
205k
  const int cdef_count = av1_cdef_compute_sb_list(
567
205k
      mi_params, fbr * MI_SIZE_64X64, fbc * MI_SIZE_64X64, dlist, bs);
568
205k
  const bool is_fb_on_frm_left_boundary = (fbc == 0);
569
205k
  const bool is_fb_on_frm_right_boundary =
570
205k
      (fbc + hb_step == cdef_search_ctx->nhfb);
571
205k
  const bool is_fb_on_frm_top_boundary = (fbr == 0);
572
205k
  const bool is_fb_on_frm_bottom_boundary =
573
205k
      (fbr + vb_step == cdef_search_ctx->nvfb);
574
205k
  const int yoff = CDEF_VBORDER * (!is_fb_on_frm_top_boundary);
575
205k
  const int xoff = CDEF_HBORDER * (!is_fb_on_frm_left_boundary);
576
205k
  int dirinit = 0;
577
603k
  for (int pli = 0; pli < cdef_search_ctx->num_planes; pli++) {
578
    // To disable CDEF filter of chroma, set MSE of chroma strength index 0 to
579
    // zero and all non-zero strength indices to 1, such that the joint
580
    // luma-chroma strength search always chooses filter strength 0 for chroma
581
    // due to least cost.
582
398k
    if (adaptive_cdef_mode > 0 && pli > 0) {
583
0
      cdef_search_ctx->mse[pli][sb_count][0] = 0;
584
0
      for (int gi = 1; gi < cdef_search_ctx->total_strengths; gi++)
585
0
        cdef_search_ctx->mse[pli][sb_count][gi] = 1;
586
0
      break;
587
0
    }
588
589
    /* We avoid filtering the pixels for which some of the pixels to
590
    average are outside the frame. We could change the filter instead,
591
    but it would add special cases for any future vectorization. */
592
398k
    const int hfilt_size = (nhb << mi_wide_l2[pli]);
593
398k
    const int vfilt_size = (nvb << mi_high_l2[pli]);
594
398k
    const int ysize =
595
398k
        vfilt_size + CDEF_VBORDER * (!is_fb_on_frm_bottom_boundary) + yoff;
596
398k
    const int xsize =
597
398k
        hfilt_size + CDEF_HBORDER * (!is_fb_on_frm_right_boundary) + xoff;
598
398k
    const int row = fbr * MI_SIZE_64X64 << mi_high_l2[pli];
599
398k
    const int col = fbc * MI_SIZE_64X64 << mi_wide_l2[pli];
600
398k
    struct macroblockd_plane pd = cdef_search_ctx->plane[pli];
601
398k
    cdef_search_ctx->copy_fn(&in[(-yoff * CDEF_BSTRIDE - xoff)], CDEF_BSTRIDE,
602
398k
                             pd.dst.buf, row - yoff, col - xoff, pd.dst.stride,
603
398k
                             ysize, xsize);
604
398k
    fill_borders_for_fbs_on_frame_boundary(
605
398k
        inbuf, hfilt_size, vfilt_size, is_fb_on_frm_left_boundary,
606
398k
        is_fb_on_frm_right_boundary, is_fb_on_frm_top_boundary,
607
398k
        is_fb_on_frm_bottom_boundary);
608
1.98M
    for (int gi = 0; gi < cdef_search_ctx->total_strengths; gi++) {
609
1.58M
      int pri_strength, sec_strength;
610
1.58M
      get_cdef_filter_strengths(cdef_search_ctx->pick_method, &pri_strength,
611
1.58M
                                &sec_strength, gi);
612
1.58M
      const uint64_t curr_mse = get_filt_error(
613
1.58M
          cdef_search_ctx, &pd, dlist, dir, &dirinit, var, in, ref_buffer[pli],
614
1.58M
          ref_stride[pli], row, col, pri_strength, sec_strength, cdef_count,
615
1.58M
          pli, coeff_shift, bs);
616
1.58M
      if (pli < 2)
617
1.20M
        cdef_search_ctx->mse[pli][sb_count][gi] = curr_mse;
618
383k
      else
619
383k
        cdef_search_ctx->mse[1][sb_count][gi] += curr_mse;
620
1.58M
    }
621
398k
  }
622
205k
  cdef_search_ctx->sb_index[sb_count] =
623
205k
      MI_SIZE_64X64 * fbr * mi_params->mi_stride + MI_SIZE_64X64 * fbc;
624
205k
}
625
626
// MSE calculation at frame level.
627
// Inputs:
628
//   cdef_search_ctx: Pointer to the structure containing parameters related to
629
//   CDEF search context.
630
//   adaptive_cdef_mode: Speed feature to control CDEF adaptively.
631
// Returns:
632
//   Nothing will be returned. Contents of cdef_search_ctx will be modified.
633
static void cdef_mse_calc_frame(CdefSearchCtx *cdef_search_ctx,
634
                                struct aom_internal_error_info *error_info,
635
21.8k
                                int adaptive_cdef_mode) {
636
  // Loop over each sb.
637
50.5k
  for (int fbr = 0; fbr < cdef_search_ctx->nvfb; ++fbr) {
638
77.3k
    for (int fbc = 0; fbc < cdef_search_ctx->nhfb; ++fbc) {
639
      // Checks if cdef processing can be skipped for particular sb.
640
48.6k
      if (cdef_sb_skip(cdef_search_ctx->mi_params, fbr, fbc)) continue;
641
      // Calculate mse for each sb and store the relevant sb index.
642
45.8k
      av1_cdef_mse_calc_block(cdef_search_ctx, error_info, fbr, fbc,
643
45.8k
                              cdef_search_ctx->sb_count, adaptive_cdef_mode);
644
45.8k
      cdef_search_ctx->sb_count++;
645
45.8k
    }
646
28.6k
  }
647
21.8k
}
648
649
// Allocates memory for members of CdefSearchCtx.
650
// Inputs:
651
//   cdef_search_ctx: Pointer to the structure containing parameters
652
//   related to CDEF search context.
653
// Returns:
654
//   Nothing will be returned. Contents of cdef_search_ctx will be modified.
655
62.3k
static void cdef_alloc_data(AV1_COMMON *cm, CdefSearchCtx *cdef_search_ctx) {
656
62.3k
  const int nvfb = cdef_search_ctx->nvfb;
657
62.3k
  const int nhfb = cdef_search_ctx->nhfb;
658
62.3k
  CHECK_MEM_ERROR(
659
62.3k
      cm, cdef_search_ctx->sb_index,
660
62.3k
      aom_malloc(nvfb * nhfb * sizeof(cdef_search_ctx->sb_index[0])));
661
62.3k
  cdef_search_ctx->sb_count = 0;
662
62.3k
  CHECK_MEM_ERROR(cm, cdef_search_ctx->mse[0],
663
62.3k
                  aom_malloc(sizeof(**cdef_search_ctx->mse) * nvfb * nhfb));
664
62.3k
  CHECK_MEM_ERROR(cm, cdef_search_ctx->mse[1],
665
62.3k
                  aom_malloc(sizeof(**cdef_search_ctx->mse) * nvfb * nhfb));
666
62.3k
}
667
668
// Deallocates the memory allocated for members of CdefSearchCtx.
669
// Inputs:
670
//   cdef_search_ctx: Pointer to the structure containing parameters
671
//   related to CDEF search context.
672
// Returns:
673
//   Nothing will be returned.
674
158k
void av1_cdef_dealloc_data(CdefSearchCtx *cdef_search_ctx) {
675
158k
  if (cdef_search_ctx) {
676
101k
    aom_free(cdef_search_ctx->mse[0]);
677
101k
    cdef_search_ctx->mse[0] = NULL;
678
101k
    aom_free(cdef_search_ctx->mse[1]);
679
101k
    cdef_search_ctx->mse[1] = NULL;
680
101k
    aom_free(cdef_search_ctx->sb_index);
681
101k
    cdef_search_ctx->sb_index = NULL;
682
101k
  }
683
158k
}
684
685
// Initialize the parameters related to CDEF search context.
686
// Inputs:
687
//   frame: Pointer to compressed frame buffer
688
//   ref: Pointer to the frame buffer holding the source frame
689
//   cm: Pointer to top level common structure
690
//   xd: Pointer to common current coding block structure
691
//   cdef_search_ctx: Pointer to the structure containing parameters related to
692
//   CDEF search context.
693
//   pick_method: Search method used to select CDEF parameters
694
// Returns:
695
//   Nothing will be returned. Contents of cdef_search_ctx will be modified.
696
static inline void cdef_params_init(const YV12_BUFFER_CONFIG *frame,
697
                                    const YV12_BUFFER_CONFIG *ref,
698
                                    AV1_COMMON *cm, MACROBLOCKD *xd,
699
                                    CdefSearchCtx *cdef_search_ctx,
700
62.3k
                                    CDEF_PICK_METHOD pick_method) {
701
62.3k
  const CommonModeInfoParams *const mi_params = &cm->mi_params;
702
62.3k
  const int num_planes = av1_num_planes(cm);
703
62.3k
  cdef_search_ctx->mi_params = &cm->mi_params;
704
62.3k
  cdef_search_ctx->ref = ref;
705
62.3k
  cdef_search_ctx->nvfb =
706
62.3k
      (mi_params->mi_rows + MI_SIZE_64X64 - 1) / MI_SIZE_64X64;
707
62.3k
  cdef_search_ctx->nhfb =
708
62.3k
      (mi_params->mi_cols + MI_SIZE_64X64 - 1) / MI_SIZE_64X64;
709
62.3k
  cdef_search_ctx->coeff_shift = AOMMAX(cm->seq_params->bit_depth - 8, 0);
710
62.3k
  cdef_search_ctx->damping = 3 + (cm->quant_params.base_qindex >> 6);
711
62.3k
  cdef_search_ctx->total_strengths = nb_cdef_strengths[pick_method];
712
62.3k
  cdef_search_ctx->num_planes = num_planes;
713
62.3k
  cdef_search_ctx->pick_method = pick_method;
714
62.3k
  cdef_search_ctx->sb_count = 0;
715
62.3k
  cdef_search_ctx->use_highbitdepth = cm->seq_params->use_highbitdepth;
716
62.3k
  av1_setup_dst_planes(xd->plane, cm->seq_params->sb_size, frame, 0, 0, 0,
717
62.3k
                       num_planes);
718
  // Initialize plane wise information.
719
179k
  for (int pli = 0; pli < num_planes; pli++) {
720
117k
    cdef_search_ctx->xdec[pli] = xd->plane[pli].subsampling_x;
721
117k
    cdef_search_ctx->ydec[pli] = xd->plane[pli].subsampling_y;
722
117k
    cdef_search_ctx->bsize[pli] =
723
117k
        cdef_search_ctx->ydec[pli]
724
117k
            ? (cdef_search_ctx->xdec[pli] ? BLOCK_4X4 : BLOCK_8X4)
725
117k
            : (cdef_search_ctx->xdec[pli] ? BLOCK_4X8 : BLOCK_8X8);
726
117k
    cdef_search_ctx->mi_wide_l2[pli] =
727
117k
        MI_SIZE_LOG2 - xd->plane[pli].subsampling_x;
728
117k
    cdef_search_ctx->mi_high_l2[pli] =
729
117k
        MI_SIZE_LOG2 - xd->plane[pli].subsampling_y;
730
117k
    cdef_search_ctx->plane[pli] = xd->plane[pli];
731
117k
  }
732
  // Function pointer initialization.
733
62.3k
#if CONFIG_AV1_HIGHBITDEPTH
734
62.3k
  if (cm->seq_params->use_highbitdepth) {
735
19.9k
    cdef_search_ctx->copy_fn = av1_cdef_copy_sb8_16_highbd;
736
19.9k
    cdef_search_ctx->compute_cdef_dist_fn = compute_cdef_dist_highbd;
737
42.4k
  } else {
738
42.4k
    cdef_search_ctx->copy_fn = av1_cdef_copy_sb8_16_lowbd;
739
42.4k
    cdef_search_ctx->compute_cdef_dist_fn = compute_cdef_dist;
740
42.4k
  }
741
#else
742
  cdef_search_ctx->copy_fn = av1_cdef_copy_sb8_16_lowbd;
743
  cdef_search_ctx->compute_cdef_dist_fn = compute_cdef_dist;
744
#endif
745
62.3k
}
746
747
void av1_pick_cdef_from_qp(AV1_COMMON *const cm, int skip_cdef,
748
37.6k
                           int is_screen_content, bool avoid_uv_cdef) {
749
37.6k
  const int bd = cm->seq_params->bit_depth;
750
37.6k
  const int q =
751
37.6k
      av1_ac_quant_QTX(cm->quant_params.base_qindex, 0, bd) >> (bd - 8);
752
37.6k
  CdefInfo *const cdef_info = &cm->cdef_info;
753
  // Check the speed feature to avoid extra signaling.
754
37.6k
  if (skip_cdef) {
755
0
    cdef_info->cdef_bits = 1;
756
0
    cdef_info->nb_cdef_strengths = 2;
757
37.6k
  } else {
758
37.6k
    cdef_info->cdef_bits = 0;
759
37.6k
    cdef_info->nb_cdef_strengths = 1;
760
37.6k
  }
761
37.6k
  cdef_info->cdef_damping = 3 + (cm->quant_params.base_qindex >> 6);
762
763
37.6k
  int predicted_y_f1 = 0;
764
37.6k
  int predicted_y_f2 = 0;
765
37.6k
  int predicted_uv_f1 = 0;
766
37.6k
  int predicted_uv_f2 = 0;
767
37.6k
  if (is_screen_content) {
768
0
    predicted_y_f1 =
769
0
        (int)(5.88217781e-06 * q * q + 6.10391455e-03 * q + 9.95043102e-02);
770
0
    predicted_y_f2 =
771
0
        (int)(-7.79934857e-06 * q * q + 6.58957830e-03 * q + 8.81045025e-01);
772
0
    predicted_uv_f1 =
773
0
        (int)(-6.79500136e-06 * q * q + 1.02695586e-02 * q + 1.36126802e-01);
774
0
    predicted_uv_f2 =
775
0
        (int)(-9.99613695e-08 * q * q - 1.79361339e-05 * q + 1.17022324e+0);
776
0
    predicted_y_f1 = clamp(predicted_y_f1, 0, 15);
777
0
    predicted_y_f2 = clamp(predicted_y_f2, 0, 3);
778
0
    predicted_uv_f1 = clamp(predicted_uv_f1, 0, 15);
779
0
    predicted_uv_f2 = clamp(predicted_uv_f2, 0, 3);
780
37.6k
  } else {
781
37.6k
    if (!frame_is_intra_only(cm)) {
782
9.31k
      predicted_y_f1 = clamp((int)roundf(q * q * -0.0000023593946f +
783
9.31k
                                         q * 0.0068615186f + 0.02709886f),
784
9.31k
                             0, 15);
785
9.31k
      predicted_y_f2 = clamp((int)roundf(q * q * -0.00000057629734f +
786
9.31k
                                         q * 0.0013993345f + 0.03831067f),
787
9.31k
                             0, 3);
788
9.31k
      predicted_uv_f1 = clamp((int)roundf(q * q * -0.0000007095069f +
789
9.31k
                                          q * 0.0034628846f + 0.00887099f),
790
9.31k
                              0, 15);
791
9.31k
      predicted_uv_f2 = clamp((int)roundf(q * q * 0.00000023874085f +
792
9.31k
                                          q * 0.00028223585f + 0.05576307f),
793
9.31k
                              0, 3);
794
28.3k
    } else {
795
28.3k
      predicted_y_f1 = clamp(
796
28.3k
          (int)roundf(q * q * 0.0000033731974f + q * 0.008070594f + 0.0187634f),
797
28.3k
          0, 15);
798
28.3k
      predicted_y_f2 = clamp((int)roundf(q * q * 0.0000029167343f +
799
28.3k
                                         q * 0.0027798624f + 0.0079405f),
800
28.3k
                             0, 3);
801
28.3k
      predicted_uv_f1 = clamp((int)roundf(q * q * -0.0000130790995f +
802
28.3k
                                          q * 0.012892405f - 0.00748388f),
803
28.3k
                              0, 15);
804
28.3k
      predicted_uv_f2 = clamp((int)roundf(q * q * 0.0000032651783f +
805
28.3k
                                          q * 0.00035520183f + 0.00228092f),
806
28.3k
                              0, 3);
807
28.3k
    }
808
37.6k
  }
809
37.6k
  cdef_info->cdef_strengths[0] =
810
37.6k
      predicted_y_f1 * CDEF_SEC_STRENGTHS + predicted_y_f2;
811
37.6k
  cdef_info->cdef_uv_strengths[0] =
812
37.6k
      avoid_uv_cdef ? 0
813
37.6k
                    : predicted_uv_f1 * CDEF_SEC_STRENGTHS + predicted_uv_f2;
814
815
  // mbmi->cdef_strength is already set in the encoding stage. We don't need to
816
  // set it again here.
817
37.6k
  if (skip_cdef) {
818
0
    cdef_info->cdef_strengths[1] = 0;
819
0
    cdef_info->cdef_uv_strengths[1] = 0;
820
0
    return;
821
0
  }
822
823
37.6k
  const CommonModeInfoParams *const mi_params = &cm->mi_params;
824
37.6k
  const int nvfb = (mi_params->mi_rows + MI_SIZE_64X64 - 1) / MI_SIZE_64X64;
825
37.6k
  const int nhfb = (mi_params->mi_cols + MI_SIZE_64X64 - 1) / MI_SIZE_64X64;
826
37.6k
  MB_MODE_INFO **mbmi = mi_params->mi_grid_base;
827
  // mbmi is NULL when real-time rate control library is used.
828
37.6k
  if (!mbmi) return;
829
108k
  for (int r = 0; r < nvfb; ++r) {
830
213k
    for (int c = 0; c < nhfb; ++c) {
831
142k
      MB_MODE_INFO *current_mbmi = mbmi[MI_SIZE_64X64 * c];
832
142k
      current_mbmi->cdef_strength = 0;
833
142k
    }
834
70.3k
    mbmi += MI_SIZE_64X64 * mi_params->mi_stride;
835
70.3k
  }
836
37.6k
}
837
838
105k
void av1_cdef_search(AV1_COMP *cpi) {
839
105k
  AV1_COMMON *cm = &cpi->common;
840
105k
  CDEF_CONTROL cdef_control = cpi->oxcf.tool_cfg.cdef_control;
841
105k
  const bool apply_adaptive_cdef =
842
105k
      cdef_control == CDEF_ADAPTIVE &&
843
51.8k
      (cpi->oxcf.rc_cfg.mode == AOM_Q || cpi->oxcf.rc_cfg.mode == AOM_CQ);
844
845
105k
  assert(cdef_control != CDEF_NONE);
846
  // For CDEF_ADAPTIVE, turning off CDEF around qindex 32 was best for still
847
  // pictures
848
105k
  if ((cdef_control == CDEF_REFERENCE &&
849
0
       cpi->ppi->rtc_ref.non_reference_frame) ||
850
105k
      (apply_adaptive_cdef && cpi->oxcf.rc_cfg.cq_level <= 32)) {
851
5.52k
    CdefInfo *const cdef_info = &cm->cdef_info;
852
5.52k
    cdef_info->nb_cdef_strengths = 1;
853
5.52k
    cdef_info->cdef_bits = 0;
854
5.52k
    cdef_info->cdef_strengths[0] = 0;
855
5.52k
    cdef_info->cdef_uv_strengths[0] = 0;
856
5.52k
    return;
857
5.52k
  }
858
859
  // Indicate if external RC is used for testing
860
100k
  const int rtc_ext_rc = cpi->rc.rtc_external_ratectrl;
861
100k
  if (rtc_ext_rc) {
862
0
    av1_pick_cdef_from_qp(cm, /*skip_cdef=*/0, /*is_screen_content=*/0,
863
0
                          /*avoid_uv_cdef=*/false);
864
0
    return;
865
0
  }
866
100k
  CDEF_PICK_METHOD pick_method = cpi->sf.lpf_sf.cdef_pick_method;
867
100k
  if (pick_method == CDEF_PICK_FROM_Q) {
868
37.6k
    const int use_screen_content_model =
869
37.6k
        cm->quant_params.base_qindex >
870
37.6k
            AOMMAX(cpi->sf.rt_sf.screen_content_cdef_filter_qindex_thresh,
871
37.6k
                   cpi->rc.best_quality + 5) &&
872
18.2k
        cpi->oxcf.tune_cfg.content == AOM_CONTENT_SCREEN;
873
874
    // For adaptive CDEF, do not apply CDEF to chroma channels.
875
    // This is done to reduce decode time, as CDEF is a relatively-expensive
876
    // filter to compute.
877
37.6k
    const bool avoid_uv_cdef = apply_adaptive_cdef;
878
879
37.6k
    av1_pick_cdef_from_qp(cm, cpi->sf.rt_sf.skip_cdef_sb != 0,
880
37.6k
                          use_screen_content_model, avoid_uv_cdef);
881
37.6k
    return;
882
37.6k
  }
883
62.3k
  const CommonModeInfoParams *const mi_params = &cm->mi_params;
884
62.3k
  const int damping = 3 + (cm->quant_params.base_qindex >> 6);
885
62.3k
  const int fast = (pick_method >= CDEF_FAST_SEARCH_LVL1 &&
886
62.3k
                    pick_method <= CDEF_FAST_SEARCH_LVL5);
887
62.3k
  const int adaptive_cdef_mode = cpi->sf.lpf_sf.adaptive_cdef_mode;
888
62.3k
  const int num_planes = av1_num_planes(cm);
889
62.3k
  MACROBLOCKD *xd = &cpi->td.mb.e_mbd;
890
891
62.3k
  if (!cpi->cdef_search_ctx)
892
62.3k
    CHECK_MEM_ERROR(cm, cpi->cdef_search_ctx,
893
62.3k
                    aom_calloc(1, sizeof(*cpi->cdef_search_ctx)));
894
62.3k
  CdefSearchCtx *cdef_search_ctx = cpi->cdef_search_ctx;
895
896
  // Initialize parameters related to CDEF search context.
897
62.3k
  cdef_params_init(&cm->cur_frame->buf, cpi->source, cm, xd, cdef_search_ctx,
898
62.3k
                   pick_method);
899
  // Allocate CDEF search context buffers.
900
62.3k
  cdef_alloc_data(cm, cdef_search_ctx);
901
  // Frame level mse calculation.
902
62.3k
  if (cpi->mt_info.num_workers > 1) {
903
40.5k
    av1_cdef_mse_calc_frame_mt(cpi);
904
40.5k
  } else {
905
21.8k
    cdef_mse_calc_frame(cdef_search_ctx, cm->error, adaptive_cdef_mode);
906
21.8k
  }
907
908
  /* Search for different number of signaling bits. */
909
62.3k
  int nb_strength_bits = 0;
910
62.3k
  uint64_t best_rd = UINT64_MAX;
911
62.3k
  CdefInfo *const cdef_info = &cm->cdef_info;
912
62.3k
  int sb_count = cdef_search_ctx->sb_count;
913
62.3k
  uint64_t(*mse[2])[TOTAL_STRENGTHS];
914
62.3k
  mse[0] = cdef_search_ctx->mse[0];
915
62.3k
  mse[1] = cdef_search_ctx->mse[1];
916
  /* Calculate the maximum number of bits required to signal CDEF strengths at
917
   * block level */
918
62.3k
  const int total_strengths = nb_cdef_strengths[pick_method];
919
62.3k
  const int joint_strengths =
920
62.3k
      num_planes > 1 ? total_strengths * total_strengths : total_strengths;
921
62.3k
  const int max_signaling_bits =
922
62.3k
      joint_strengths == 1 ? 0 : get_msb(joint_strengths - 1) + 1;
923
62.3k
  int rdmult = cpi->td.mb.rdmult;
924
925
  // For adaptive CDEF, reduce primary and secondary CDEF strengths for
926
  // qindexes up to 220.
927
62.3k
  const bool should_reduce_cdef_strengths =
928
62.3k
      apply_adaptive_cdef && cpi->oxcf.rc_cfg.cq_level <= 220;
929
  // For adaptive CDEF with strength reduction, zero out CDEF strengths with
930
  // low values (luma and/or chroma). This is done to reduce decode time, as
931
  // CDEF is a relatively-expensive filter to compute.
932
62.3k
  const bool should_zero_cdef_strengths =
933
62.3k
      should_reduce_cdef_strengths && cpi->sf.lpf_sf.zero_low_cdef_strengths;
934
  // If running adaptive CDEF with strength zeroing, let search derive at least
935
  // two CDEF strengths (i.e. at least 1 CDEF signaling bit), unless search was
936
  // explicitly set to search for 1 strength only (i.e. 0 CDEF signaling bits).
937
  // Doing so will help find opportunities to zero out low strengths to reduce
938
  // overall decode time.
939
62.3k
  const int min_signaling_bits =
940
62.3k
      (should_zero_cdef_strengths && max_signaling_bits > 0) ? 1 : 0;
941
942
267k
  for (int i = min_signaling_bits; i <= 3; i++) {
943
240k
    if (i > max_signaling_bits) break;
944
205k
    int best_lev0[CDEF_MAX_STRENGTHS] = { 0 };
945
205k
    int best_lev1[CDEF_MAX_STRENGTHS] = { 0 };
946
205k
    const int nb_strengths = 1 << i;
947
205k
    uint64_t tot_mse;
948
205k
    if (num_planes > 1) {
949
104k
      tot_mse = joint_strength_search_dual(best_lev0, best_lev1, nb_strengths,
950
104k
                                           mse, sb_count, pick_method);
951
104k
    } else {
952
100k
      tot_mse = joint_strength_search(best_lev0, nb_strengths, mse[0], sb_count,
953
100k
                                      pick_method);
954
100k
    }
955
956
205k
    const int total_bits = sb_count * i + nb_strengths * CDEF_STRENGTH_BITS *
957
205k
                                              (num_planes > 1 ? 2 : 1);
958
205k
    const int rate_cost = av1_cost_literal(total_bits);
959
205k
    const uint64_t dist = tot_mse * 16;
960
205k
    const uint64_t rd = RDCOST(rdmult, rate_cost, dist);
961
205k
    if (rd < best_rd) {
962
72.1k
      best_rd = rd;
963
72.1k
      nb_strength_bits = i;
964
72.1k
      memcpy(cdef_info->cdef_strengths, best_lev0,
965
72.1k
             nb_strengths * sizeof(best_lev0[0]));
966
72.1k
      if (num_planes > 1) {
967
31.8k
        memcpy(cdef_info->cdef_uv_strengths, best_lev1,
968
31.8k
               nb_strengths * sizeof(best_lev1[0]));
969
31.8k
      }
970
72.1k
    }
971
205k
  }
972
973
62.3k
  cdef_info->cdef_bits = nb_strength_bits;
974
62.3k
  cdef_info->nb_cdef_strengths = 1 << nb_strength_bits;
975
62.3k
  uint64_t tot_best_mse_y = 0, tot_zero_mse_y = 0;
976
267k
  for (int i = 0; i < sb_count; i++) {
977
205k
    uint64_t best_mse = UINT64_MAX;
978
205k
    int best_gi = 0;
979
205k
    tot_zero_mse_y += mse[0][i][0];
980
553k
    for (int gi = 0; gi < cdef_info->nb_cdef_strengths; gi++) {
981
347k
      uint64_t curr = mse[0][i][cdef_info->cdef_strengths[gi]];
982
347k
      if (num_planes > 1) curr += mse[1][i][cdef_info->cdef_uv_strengths[gi]];
983
347k
      if (curr < best_mse) {
984
244k
        best_gi = gi;
985
244k
        best_mse = curr;
986
244k
      }
987
347k
    }
988
    // When adaptive_cdef_mode is enabled, CDEF filter of chroma is disabled by
989
    // setting the MSE at chroma strength index 0 to zero, ensuring zero filter
990
    // strength always wins for chroma. Hence, best_mse reflects luma MSE only.
991
205k
    tot_best_mse_y += best_mse;
992
205k
    mi_params->mi_grid_base[cdef_search_ctx->sb_index[i]]->cdef_strength =
993
205k
        best_gi;
994
205k
  }
995
996
  // Disable CDEF filter of luma if the MSE improvement over zero filter
997
  // strength is below the threshold.
998
62.3k
  if (cm->current_frame.pyramid_level > 1 &&
999
12.0k
      cpi->sf.lpf_sf.adaptive_cdef_mode > 0 && sb_count > 0) {
1000
0
    const double cdef_mse_pct_imp_thresh = 4.0;
1001
0
    double luma_mse_gain_pct = 0.0;
1002
    // Percentage reduction in luma MSE with best CDEF strength vs zero
1003
    // strength.
1004
0
    luma_mse_gain_pct =
1005
0
        (tot_zero_mse_y - tot_best_mse_y) * 100.0 / tot_zero_mse_y;
1006
0
    if (luma_mse_gain_pct < cdef_mse_pct_imp_thresh) {
1007
0
      cdef_info->nb_cdef_strengths = 1;
1008
0
      cdef_info->cdef_bits = 0;
1009
0
      cdef_info->cdef_strengths[0] = 0;
1010
0
      cdef_info->cdef_uv_strengths[0] = 0;
1011
0
      for (int i = 0; i < sb_count; i++) {
1012
0
        mi_params->mi_grid_base[cdef_search_ctx->sb_index[i]]->cdef_strength =
1013
0
            0;
1014
0
      }
1015
0
    }
1016
0
  }
1017
1018
62.3k
  if (fast) {
1019
145k
    for (int j = 0; j < cdef_info->nb_cdef_strengths; j++) {
1020
83.2k
      const int luma_strength = cdef_info->cdef_strengths[j];
1021
83.2k
      int pri_strength, sec_strength;
1022
1023
83.2k
      STORE_CDEF_FILTER_STRENGTH(cdef_info->cdef_strengths[j], pick_method,
1024
83.2k
                                 luma_strength);
1025
83.2k
      if (num_planes > 1) {
1026
37.5k
        const int chroma_strength = cdef_info->cdef_uv_strengths[j];
1027
37.5k
        STORE_CDEF_FILTER_STRENGTH(cdef_info->cdef_uv_strengths[j], pick_method,
1028
37.5k
                                   chroma_strength);
1029
37.5k
      }
1030
83.2k
    }
1031
62.3k
  }
1032
1033
  // Perform CDEF strength reduction.
1034
  // Note 1: for odd strengths, the 0.5 discarded by ">> 1" is a significant
1035
  // part of the strength when the strength is small, and because there are
1036
  // few strength levels, odd strengths are reduced significantly more than a
1037
  // half. This is intended behavior for reduced strength.
1038
  // For example: a pri strength of 3 becomes 1, and a sec strength of 1
1039
  // becomes 0.
1040
  // Note 2: a (signaled) sec strength value of 3 is special as it results in an
1041
  // actual sec strength of 4. We tried adding +1 to the sec strength 3 so it
1042
  // maps to a reduced sec strength of 2. However, on Daala's subset1, the
1043
  // resulting SSIMULACRA 2 scores were either exactly the same (at cpu-used 6),
1044
  // or within noise level (at cpu-used 3). Given that there were no discernible
1045
  // improvements, this special mapping was left out for reduced strength.
1046
62.3k
  if (should_reduce_cdef_strengths) {
1047
53.0k
    for (int j = 0; j < cdef_info->nb_cdef_strengths; j++) {
1048
32.1k
      const int luma_strength = cdef_info->cdef_strengths[j];
1049
32.1k
      const int new_pri_luma_strength =
1050
32.1k
          (luma_strength / CDEF_SEC_STRENGTHS) >> 1;
1051
32.1k
      const int new_sec_luma_strength =
1052
32.1k
          (luma_strength % CDEF_SEC_STRENGTHS) >> 1;
1053
1054
32.1k
      cdef_info->cdef_strengths[j] =
1055
32.1k
          new_pri_luma_strength * CDEF_SEC_STRENGTHS + new_sec_luma_strength;
1056
1057
32.1k
      int new_pri_chroma_strength;
1058
32.1k
      int new_sec_chroma_strength;
1059
1060
32.1k
      if (num_planes > 1) {
1061
15.9k
        const int chroma_strength = cdef_info->cdef_uv_strengths[j];
1062
15.9k
        new_pri_chroma_strength = (chroma_strength / CDEF_SEC_STRENGTHS) >> 1;
1063
15.9k
        new_sec_chroma_strength = (chroma_strength % CDEF_SEC_STRENGTHS) >> 1;
1064
1065
15.9k
        cdef_info->cdef_uv_strengths[j] =
1066
15.9k
            new_pri_chroma_strength * CDEF_SEC_STRENGTHS +
1067
15.9k
            new_sec_chroma_strength;
1068
15.9k
      }
1069
1070
      // Zero out entries with low CDEF luma (and optional chroma) strengths.
1071
      // The low-strength thresholds were empirically derived from subjective
1072
      // testing and SSIMULACRA 2 scores. These strike a balance between
1073
      // perceptual quality gains and a reasonable single-threaded decode time
1074
      // increase (~10%) over --enable-cdef 0. There's an overall 0.18 point
1075
      // loss in SSIMULACRA 2 scores over no CDEF strength zeroing at speed 6,
1076
      // QP 30 on the CLIC 2020 dataset.
1077
32.1k
      if (should_zero_cdef_strengths) {
1078
18.0k
        const bool is_low_luma_strength =
1079
18.0k
            new_pri_luma_strength <= 4 && new_sec_luma_strength <= 1;
1080
1081
18.0k
        if (is_low_luma_strength) {
1082
11.0k
          cdef_info->cdef_strengths[j] = 0;
1083
11.0k
        }
1084
18.0k
        if (num_planes > 1) {
1085
8.89k
          const bool is_low_chroma_strength =
1086
8.89k
              new_pri_chroma_strength <= 4 && new_sec_chroma_strength <= 1;
1087
1088
8.89k
          if (is_low_luma_strength || is_low_chroma_strength) {
1089
            // Disable CDEF on chroma if we've disabled it on luma
1090
6.19k
            cdef_info->cdef_uv_strengths[j] = 0;
1091
6.19k
          }
1092
8.89k
        }
1093
18.0k
      }
1094
32.1k
    }
1095
20.8k
  }
1096
1097
62.3k
  cdef_info->cdef_damping = damping;
1098
  // Deallocate CDEF search context buffers.
1099
62.3k
  av1_cdef_dealloc_data(cdef_search_ctx);
1100
62.3k
}