Coverage Report

Created: 2022-08-24 06:11

/src/aom/av1/encoder/txb_rdopt.c
Line
Count
Source (jump to first uncovered line)
1
/*
2
 * Copyright (c) 2021, Alliance for Open Media. All rights reserved
3
 *
4
 * This source code is subject to the terms of the BSD 2 Clause License and
5
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6
 * was not distributed with this source code in the LICENSE file, you can
7
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8
 * Media Patent License 1.0 was not distributed with this source code in the
9
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10
 */
11
12
#include "av1/encoder/txb_rdopt.h"
13
#include "av1/encoder/txb_rdopt_utils.h"
14
15
#include "av1/common/idct.h"
16
17
static INLINE void update_coeff_general(
18
    int *accu_rate, int64_t *accu_dist, int si, int eob, TX_SIZE tx_size,
19
    TX_CLASS tx_class, int bwl, int height, int64_t rdmult, int shift,
20
    int dc_sign_ctx, const int16_t *dequant, const int16_t *scan,
21
    const LV_MAP_COEFF_COST *txb_costs, const tran_low_t *tcoeff,
22
    tran_low_t *qcoeff, tran_low_t *dqcoeff, uint8_t *levels,
23
0
    const qm_val_t *iqmatrix) {
24
0
  const int dqv = get_dqv(dequant, scan[si], iqmatrix);
25
0
  const int ci = scan[si];
26
0
  const tran_low_t qc = qcoeff[ci];
27
0
  const int is_last = si == (eob - 1);
28
0
  const int coeff_ctx = get_lower_levels_ctx_general(
29
0
      is_last, si, bwl, height, levels, ci, tx_size, tx_class);
30
0
  if (qc == 0) {
31
0
    *accu_rate += txb_costs->base_cost[coeff_ctx][0];
32
0
  } else {
33
0
    const int sign = (qc < 0) ? 1 : 0;
34
0
    const tran_low_t abs_qc = abs(qc);
35
0
    const tran_low_t tqc = tcoeff[ci];
36
0
    const tran_low_t dqc = dqcoeff[ci];
37
0
    const int64_t dist = get_coeff_dist(tqc, dqc, shift);
38
0
    const int64_t dist0 = get_coeff_dist(tqc, 0, shift);
39
0
    const int rate =
40
0
        get_coeff_cost_general(is_last, ci, abs_qc, sign, coeff_ctx,
41
0
                               dc_sign_ctx, txb_costs, bwl, tx_class, levels);
42
0
    const int64_t rd = RDCOST(rdmult, rate, dist);
43
44
0
    tran_low_t qc_low, dqc_low;
45
0
    tran_low_t abs_qc_low;
46
0
    int64_t dist_low, rd_low;
47
0
    int rate_low;
48
0
    if (abs_qc == 1) {
49
0
      abs_qc_low = qc_low = dqc_low = 0;
50
0
      dist_low = dist0;
51
0
      rate_low = txb_costs->base_cost[coeff_ctx][0];
52
0
    } else {
53
0
      get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low);
54
0
      abs_qc_low = abs_qc - 1;
55
0
      dist_low = get_coeff_dist(tqc, dqc_low, shift);
56
0
      rate_low =
57
0
          get_coeff_cost_general(is_last, ci, abs_qc_low, sign, coeff_ctx,
58
0
                                 dc_sign_ctx, txb_costs, bwl, tx_class, levels);
59
0
    }
60
61
0
    rd_low = RDCOST(rdmult, rate_low, dist_low);
62
0
    if (rd_low < rd) {
63
0
      qcoeff[ci] = qc_low;
64
0
      dqcoeff[ci] = dqc_low;
65
0
      levels[get_padded_idx(ci, bwl)] = AOMMIN(abs_qc_low, INT8_MAX);
66
0
      *accu_rate += rate_low;
67
0
      *accu_dist += dist_low - dist0;
68
0
    } else {
69
0
      *accu_rate += rate;
70
0
      *accu_dist += dist - dist0;
71
0
    }
72
0
  }
73
0
}
74
75
static AOM_FORCE_INLINE void update_coeff_simple(
76
    int *accu_rate, int si, int eob, TX_SIZE tx_size, TX_CLASS tx_class,
77
    int bwl, int64_t rdmult, int shift, const int16_t *dequant,
78
    const int16_t *scan, const LV_MAP_COEFF_COST *txb_costs,
79
    const tran_low_t *tcoeff, tran_low_t *qcoeff, tran_low_t *dqcoeff,
80
0
    uint8_t *levels, const qm_val_t *iqmatrix) {
81
0
  const int dqv = get_dqv(dequant, scan[si], iqmatrix);
82
0
  (void)eob;
83
  // this simple version assumes the coeff's scan_idx is not DC (scan_idx != 0)
84
  // and not the last (scan_idx != eob - 1)
85
0
  assert(si != eob - 1);
86
0
  assert(si > 0);
87
0
  const int ci = scan[si];
88
0
  const tran_low_t qc = qcoeff[ci];
89
0
  const int coeff_ctx =
90
0
      get_lower_levels_ctx(levels, ci, bwl, tx_size, tx_class);
91
0
  if (qc == 0) {
92
0
    *accu_rate += txb_costs->base_cost[coeff_ctx][0];
93
0
  } else {
94
0
    const tran_low_t abs_qc = abs(qc);
95
0
    const tran_low_t abs_tqc = abs(tcoeff[ci]);
96
0
    const tran_low_t abs_dqc = abs(dqcoeff[ci]);
97
0
    int rate_low = 0;
98
0
    const int rate = get_two_coeff_cost_simple(
99
0
        ci, abs_qc, coeff_ctx, txb_costs, bwl, tx_class, levels, &rate_low);
100
0
    if (abs_dqc < abs_tqc) {
101
0
      *accu_rate += rate;
102
0
      return;
103
0
    }
104
105
0
    const int64_t dist = get_coeff_dist(abs_tqc, abs_dqc, shift);
106
0
    const int64_t rd = RDCOST(rdmult, rate, dist);
107
108
0
    const tran_low_t abs_qc_low = abs_qc - 1;
109
0
    const tran_low_t abs_dqc_low = (abs_qc_low * dqv) >> shift;
110
0
    const int64_t dist_low = get_coeff_dist(abs_tqc, abs_dqc_low, shift);
111
0
    const int64_t rd_low = RDCOST(rdmult, rate_low, dist_low);
112
113
0
    if (rd_low < rd) {
114
0
      const int sign = (qc < 0) ? 1 : 0;
115
0
      qcoeff[ci] = (-sign ^ abs_qc_low) + sign;
116
0
      dqcoeff[ci] = (-sign ^ abs_dqc_low) + sign;
117
0
      levels[get_padded_idx(ci, bwl)] = AOMMIN(abs_qc_low, INT8_MAX);
118
0
      *accu_rate += rate_low;
119
0
    } else {
120
0
      *accu_rate += rate;
121
0
    }
122
0
  }
123
0
}
124
125
static AOM_FORCE_INLINE void update_coeff_eob(
126
    int *accu_rate, int64_t *accu_dist, int *eob, int *nz_num, int *nz_ci,
127
    int si, TX_SIZE tx_size, TX_CLASS tx_class, int bwl, int height,
128
    int dc_sign_ctx, int64_t rdmult, int shift, const int16_t *dequant,
129
    const int16_t *scan, const LV_MAP_EOB_COST *txb_eob_costs,
130
    const LV_MAP_COEFF_COST *txb_costs, const tran_low_t *tcoeff,
131
    tran_low_t *qcoeff, tran_low_t *dqcoeff, uint8_t *levels, int sharpness,
132
0
    const qm_val_t *iqmatrix) {
133
0
  const int dqv = get_dqv(dequant, scan[si], iqmatrix);
134
0
  assert(si != *eob - 1);
135
0
  const int ci = scan[si];
136
0
  const tran_low_t qc = qcoeff[ci];
137
0
  const int coeff_ctx =
138
0
      get_lower_levels_ctx(levels, ci, bwl, tx_size, tx_class);
139
0
  if (qc == 0) {
140
0
    *accu_rate += txb_costs->base_cost[coeff_ctx][0];
141
0
  } else {
142
0
    int lower_level = 0;
143
0
    const tran_low_t abs_qc = abs(qc);
144
0
    const tran_low_t tqc = tcoeff[ci];
145
0
    const tran_low_t dqc = dqcoeff[ci];
146
0
    const int sign = (qc < 0) ? 1 : 0;
147
0
    const int64_t dist0 = get_coeff_dist(tqc, 0, shift);
148
0
    int64_t dist = get_coeff_dist(tqc, dqc, shift) - dist0;
149
0
    int rate =
150
0
        get_coeff_cost_general(0, ci, abs_qc, sign, coeff_ctx, dc_sign_ctx,
151
0
                               txb_costs, bwl, tx_class, levels);
152
0
    int64_t rd = RDCOST(rdmult, *accu_rate + rate, *accu_dist + dist);
153
154
0
    tran_low_t qc_low, dqc_low;
155
0
    tran_low_t abs_qc_low;
156
0
    int64_t dist_low, rd_low;
157
0
    int rate_low;
158
159
0
    if (abs_qc == 1) {
160
0
      abs_qc_low = 0;
161
0
      dqc_low = qc_low = 0;
162
0
      dist_low = 0;
163
0
      rate_low = txb_costs->base_cost[coeff_ctx][0];
164
0
      rd_low = RDCOST(rdmult, *accu_rate + rate_low, *accu_dist);
165
0
    } else {
166
0
      get_qc_dqc_low(abs_qc, sign, dqv, shift, &qc_low, &dqc_low);
167
0
      abs_qc_low = abs_qc - 1;
168
0
      dist_low = get_coeff_dist(tqc, dqc_low, shift) - dist0;
169
0
      rate_low =
170
0
          get_coeff_cost_general(0, ci, abs_qc_low, sign, coeff_ctx,
171
0
                                 dc_sign_ctx, txb_costs, bwl, tx_class, levels);
172
0
      rd_low = RDCOST(rdmult, *accu_rate + rate_low, *accu_dist + dist_low);
173
0
    }
174
175
0
    int lower_level_new_eob = 0;
176
0
    const int new_eob = si + 1;
177
0
    const int coeff_ctx_new_eob = get_lower_levels_ctx_eob(bwl, height, si);
178
0
    const int new_eob_cost =
179
0
        get_eob_cost(new_eob, txb_eob_costs, txb_costs, tx_class);
180
0
    int rate_coeff_eob =
181
0
        new_eob_cost + get_coeff_cost_eob(ci, abs_qc, sign, coeff_ctx_new_eob,
182
0
                                          dc_sign_ctx, txb_costs, bwl,
183
0
                                          tx_class);
184
0
    int64_t dist_new_eob = dist;
185
0
    int64_t rd_new_eob = RDCOST(rdmult, rate_coeff_eob, dist_new_eob);
186
187
0
    if (abs_qc_low > 0) {
188
0
      const int rate_coeff_eob_low =
189
0
          new_eob_cost + get_coeff_cost_eob(ci, abs_qc_low, sign,
190
0
                                            coeff_ctx_new_eob, dc_sign_ctx,
191
0
                                            txb_costs, bwl, tx_class);
192
0
      const int64_t dist_new_eob_low = dist_low;
193
0
      const int64_t rd_new_eob_low =
194
0
          RDCOST(rdmult, rate_coeff_eob_low, dist_new_eob_low);
195
0
      if (rd_new_eob_low < rd_new_eob) {
196
0
        lower_level_new_eob = 1;
197
0
        rd_new_eob = rd_new_eob_low;
198
0
        rate_coeff_eob = rate_coeff_eob_low;
199
0
        dist_new_eob = dist_new_eob_low;
200
0
      }
201
0
    }
202
203
0
    if (sharpness == 0 || abs_qc > 1) {
204
0
      if (rd_low < rd) {
205
0
        lower_level = 1;
206
0
        rd = rd_low;
207
0
        rate = rate_low;
208
0
        dist = dist_low;
209
0
      }
210
0
    }
211
212
0
    if (sharpness == 0 && rd_new_eob < rd) {
213
0
      for (int ni = 0; ni < *nz_num; ++ni) {
214
0
        int last_ci = nz_ci[ni];
215
0
        levels[get_padded_idx(last_ci, bwl)] = 0;
216
0
        qcoeff[last_ci] = 0;
217
0
        dqcoeff[last_ci] = 0;
218
0
      }
219
0
      *eob = new_eob;
220
0
      *nz_num = 0;
221
0
      *accu_rate = rate_coeff_eob;
222
0
      *accu_dist = dist_new_eob;
223
0
      lower_level = lower_level_new_eob;
224
0
    } else {
225
0
      *accu_rate += rate;
226
0
      *accu_dist += dist;
227
0
    }
228
229
0
    if (lower_level) {
230
0
      qcoeff[ci] = qc_low;
231
0
      dqcoeff[ci] = dqc_low;
232
0
      levels[get_padded_idx(ci, bwl)] = AOMMIN(abs_qc_low, INT8_MAX);
233
0
    }
234
0
    if (qcoeff[ci]) {
235
0
      nz_ci[*nz_num] = ci;
236
0
      ++*nz_num;
237
0
    }
238
0
  }
239
0
}
240
241
static INLINE void update_skip(int *accu_rate, int64_t accu_dist, int *eob,
242
                               int nz_num, int *nz_ci, int64_t rdmult,
243
                               int skip_cost, int non_skip_cost,
244
0
                               tran_low_t *qcoeff, tran_low_t *dqcoeff) {
245
0
  const int64_t rd = RDCOST(rdmult, *accu_rate + non_skip_cost, accu_dist);
246
0
  const int64_t rd_new_eob = RDCOST(rdmult, skip_cost, 0);
247
0
  if (rd_new_eob < rd) {
248
0
    for (int i = 0; i < nz_num; ++i) {
249
0
      const int ci = nz_ci[i];
250
0
      qcoeff[ci] = 0;
251
0
      dqcoeff[ci] = 0;
252
      // no need to set up levels because this is the last step
253
      // levels[get_padded_idx(ci, bwl)] = 0;
254
0
    }
255
0
    *accu_rate = 0;
256
0
    *eob = 0;
257
0
  }
258
0
}
259
260
// TODO(angiebird): use this function whenever it's possible
261
static int get_tx_type_cost(const MACROBLOCK *x, const MACROBLOCKD *xd,
262
                            int plane, TX_SIZE tx_size, TX_TYPE tx_type,
263
0
                            int reduced_tx_set_used) {
264
0
  if (plane > 0) return 0;
265
266
0
  const TX_SIZE square_tx_size = txsize_sqr_map[tx_size];
267
268
0
  const MB_MODE_INFO *mbmi = xd->mi[0];
269
0
  const int is_inter = is_inter_block(mbmi);
270
0
  if (get_ext_tx_types(tx_size, is_inter, reduced_tx_set_used) > 1 &&
271
0
      !xd->lossless[xd->mi[0]->segment_id]) {
272
0
    const int ext_tx_set =
273
0
        get_ext_tx_set(tx_size, is_inter, reduced_tx_set_used);
274
0
    if (is_inter) {
275
0
      if (ext_tx_set > 0)
276
0
        return x->mode_costs
277
0
            .inter_tx_type_costs[ext_tx_set][square_tx_size][tx_type];
278
0
    } else {
279
0
      if (ext_tx_set > 0) {
280
0
        PREDICTION_MODE intra_dir;
281
0
        if (mbmi->filter_intra_mode_info.use_filter_intra)
282
0
          intra_dir = fimode_to_intradir[mbmi->filter_intra_mode_info
283
0
                                             .filter_intra_mode];
284
0
        else
285
0
          intra_dir = mbmi->mode;
286
0
        return x->mode_costs.intra_tx_type_costs[ext_tx_set][square_tx_size]
287
0
                                                [intra_dir][tx_type];
288
0
      }
289
0
    }
290
0
  }
291
0
  return 0;
292
0
}
293
294
int av1_optimize_txb(const struct AV1_COMP *cpi, MACROBLOCK *x, int plane,
295
                     int block, TX_SIZE tx_size, TX_TYPE tx_type,
296
                     const TXB_CTX *const txb_ctx, int *rate_cost,
297
0
                     int sharpness) {
298
0
  MACROBLOCKD *xd = &x->e_mbd;
299
0
  const struct macroblock_plane *p = &x->plane[plane];
300
0
  const SCAN_ORDER *scan_order = get_scan(tx_size, tx_type);
301
0
  const int16_t *scan = scan_order->scan;
302
0
  const int shift = av1_get_tx_scale(tx_size);
303
0
  int eob = p->eobs[block];
304
0
  const int16_t *dequant = p->dequant_QTX;
305
0
  const qm_val_t *iqmatrix =
306
0
      av1_get_iqmatrix(&cpi->common.quant_params, xd, plane, tx_size, tx_type);
307
0
  const int block_offset = BLOCK_OFFSET(block);
308
0
  tran_low_t *qcoeff = p->qcoeff + block_offset;
309
0
  tran_low_t *dqcoeff = p->dqcoeff + block_offset;
310
0
  const tran_low_t *tcoeff = p->coeff + block_offset;
311
0
  const CoeffCosts *coeff_costs = &x->coeff_costs;
312
313
  // This function is not called if eob = 0.
314
0
  assert(eob > 0);
315
316
0
  const AV1_COMMON *cm = &cpi->common;
317
0
  const PLANE_TYPE plane_type = get_plane_type(plane);
318
0
  const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size);
319
0
  const TX_CLASS tx_class = tx_type_to_class[tx_type];
320
0
  const MB_MODE_INFO *mbmi = xd->mi[0];
321
0
  const int bwl = get_txb_bwl(tx_size);
322
0
  const int width = get_txb_wide(tx_size);
323
0
  const int height = get_txb_high(tx_size);
324
0
  assert(width == (1 << bwl));
325
0
  const int is_inter = is_inter_block(mbmi);
326
0
  const LV_MAP_COEFF_COST *txb_costs =
327
0
      &coeff_costs->coeff_costs[txs_ctx][plane_type];
328
0
  const int eob_multi_size = txsize_log2_minus4[tx_size];
329
0
  const LV_MAP_EOB_COST *txb_eob_costs =
330
0
      &coeff_costs->eob_costs[eob_multi_size][plane_type];
331
332
0
  const int rshift = 2;
333
334
0
  const int64_t rdmult =
335
0
      (((int64_t)x->rdmult *
336
0
        (plane_rd_mult[is_inter][plane_type] << (2 * (xd->bd - 8)))) +
337
0
       2) >>
338
0
      rshift;
339
340
0
  uint8_t levels_buf[TX_PAD_2D];
341
0
  uint8_t *const levels = set_levels(levels_buf, width);
342
343
0
  if (eob > 1) av1_txb_init_levels(qcoeff, width, height, levels);
344
345
  // TODO(angirbird): check iqmatrix
346
347
0
  const int non_skip_cost = txb_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][0];
348
0
  const int skip_cost = txb_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1];
349
0
  const int eob_cost = get_eob_cost(eob, txb_eob_costs, txb_costs, tx_class);
350
0
  int accu_rate = eob_cost;
351
0
  int64_t accu_dist = 0;
352
0
  int si = eob - 1;
353
0
  const int ci = scan[si];
354
0
  const tran_low_t qc = qcoeff[ci];
355
0
  const tran_low_t abs_qc = abs(qc);
356
0
  const int sign = qc < 0;
357
0
  const int max_nz_num = 2;
358
0
  int nz_num = 1;
359
0
  int nz_ci[3] = { ci, 0, 0 };
360
0
  if (abs_qc >= 2) {
361
0
    update_coeff_general(&accu_rate, &accu_dist, si, eob, tx_size, tx_class,
362
0
                         bwl, height, rdmult, shift, txb_ctx->dc_sign_ctx,
363
0
                         dequant, scan, txb_costs, tcoeff, qcoeff, dqcoeff,
364
0
                         levels, iqmatrix);
365
0
    --si;
366
0
  } else {
367
0
    assert(abs_qc == 1);
368
0
    const int coeff_ctx = get_lower_levels_ctx_eob(bwl, height, si);
369
0
    accu_rate +=
370
0
        get_coeff_cost_eob(ci, abs_qc, sign, coeff_ctx, txb_ctx->dc_sign_ctx,
371
0
                           txb_costs, bwl, tx_class);
372
0
    const tran_low_t tqc = tcoeff[ci];
373
0
    const tran_low_t dqc = dqcoeff[ci];
374
0
    const int64_t dist = get_coeff_dist(tqc, dqc, shift);
375
0
    const int64_t dist0 = get_coeff_dist(tqc, 0, shift);
376
0
    accu_dist += dist - dist0;
377
0
    --si;
378
0
  }
379
380
0
#define UPDATE_COEFF_EOB_CASE(tx_class_literal)                            \
381
0
  case tx_class_literal:                                                   \
382
0
    for (; si >= 0 && nz_num <= max_nz_num; --si) {                        \
383
0
      update_coeff_eob(&accu_rate, &accu_dist, &eob, &nz_num, nz_ci, si,   \
384
0
                       tx_size, tx_class_literal, bwl, height,             \
385
0
                       txb_ctx->dc_sign_ctx, rdmult, shift, dequant, scan, \
386
0
                       txb_eob_costs, txb_costs, tcoeff, qcoeff, dqcoeff,  \
387
0
                       levels, sharpness, iqmatrix);                       \
388
0
    }                                                                      \
389
0
    break;
390
0
  switch (tx_class) {
391
0
    UPDATE_COEFF_EOB_CASE(TX_CLASS_2D);
392
0
    UPDATE_COEFF_EOB_CASE(TX_CLASS_HORIZ);
393
0
    UPDATE_COEFF_EOB_CASE(TX_CLASS_VERT);
394
0
#undef UPDATE_COEFF_EOB_CASE
395
0
    default: assert(false);
396
0
  }
397
398
0
  if (si == -1 && nz_num <= max_nz_num && sharpness == 0) {
399
0
    update_skip(&accu_rate, accu_dist, &eob, nz_num, nz_ci, rdmult, skip_cost,
400
0
                non_skip_cost, qcoeff, dqcoeff);
401
0
  }
402
403
0
#define UPDATE_COEFF_SIMPLE_CASE(tx_class_literal)                             \
404
0
  case tx_class_literal:                                                       \
405
0
    for (; si >= 1; --si) {                                                    \
406
0
      update_coeff_simple(&accu_rate, si, eob, tx_size, tx_class_literal, bwl, \
407
0
                          rdmult, shift, dequant, scan, txb_costs, tcoeff,     \
408
0
                          qcoeff, dqcoeff, levels, iqmatrix);                  \
409
0
    }                                                                          \
410
0
    break;
411
0
  switch (tx_class) {
412
0
    UPDATE_COEFF_SIMPLE_CASE(TX_CLASS_2D);
413
0
    UPDATE_COEFF_SIMPLE_CASE(TX_CLASS_HORIZ);
414
0
    UPDATE_COEFF_SIMPLE_CASE(TX_CLASS_VERT);
415
0
#undef UPDATE_COEFF_SIMPLE_CASE
416
0
    default: assert(false);
417
0
  }
418
419
  // DC position
420
0
  if (si == 0) {
421
    // no need to update accu_dist because it's not used after this point
422
0
    int64_t dummy_dist = 0;
423
0
    update_coeff_general(&accu_rate, &dummy_dist, si, eob, tx_size, tx_class,
424
0
                         bwl, height, rdmult, shift, txb_ctx->dc_sign_ctx,
425
0
                         dequant, scan, txb_costs, tcoeff, qcoeff, dqcoeff,
426
0
                         levels, iqmatrix);
427
0
  }
428
429
0
  const int tx_type_cost = get_tx_type_cost(x, xd, plane, tx_size, tx_type,
430
0
                                            cm->features.reduced_tx_set_used);
431
0
  if (eob == 0)
432
0
    accu_rate += skip_cost;
433
0
  else
434
0
    accu_rate += non_skip_cost + tx_type_cost;
435
436
0
  p->eobs[block] = eob;
437
0
  p->txb_entropy_ctx[block] =
438
0
      av1_get_txb_entropy_context(qcoeff, scan_order, p->eobs[block]);
439
440
0
  *rate_cost = accu_rate;
441
0
  return eob;
442
0
}
443
444
static AOM_FORCE_INLINE int warehouse_efficients_txb(
445
    const MACROBLOCK *x, const int plane, const int block,
446
    const TX_SIZE tx_size, const TXB_CTX *const txb_ctx,
447
    const struct macroblock_plane *p, const int eob,
448
    const PLANE_TYPE plane_type, const LV_MAP_COEFF_COST *const coeff_costs,
449
    const MACROBLOCKD *const xd, const TX_TYPE tx_type, const TX_CLASS tx_class,
450
0
    int reduced_tx_set_used) {
451
0
  const tran_low_t *const qcoeff = p->qcoeff + BLOCK_OFFSET(block);
452
0
  const int txb_skip_ctx = txb_ctx->txb_skip_ctx;
453
0
  const int bwl = get_txb_bwl(tx_size);
454
0
  const int width = get_txb_wide(tx_size);
455
0
  const int height = get_txb_high(tx_size);
456
0
  const SCAN_ORDER *const scan_order = get_scan(tx_size, tx_type);
457
0
  const int16_t *const scan = scan_order->scan;
458
0
  uint8_t levels_buf[TX_PAD_2D];
459
0
  uint8_t *const levels = set_levels(levels_buf, width);
460
0
  DECLARE_ALIGNED(16, int8_t, coeff_contexts[MAX_TX_SQUARE]);
461
0
  const int eob_multi_size = txsize_log2_minus4[tx_size];
462
0
  const LV_MAP_EOB_COST *const eob_costs =
463
0
      &x->coeff_costs.eob_costs[eob_multi_size][plane_type];
464
0
  int cost = coeff_costs->txb_skip_cost[txb_skip_ctx][0];
465
466
0
  av1_txb_init_levels(qcoeff, width, height, levels);
467
468
0
  cost += get_tx_type_cost(x, xd, plane, tx_size, tx_type, reduced_tx_set_used);
469
470
0
  cost += get_eob_cost(eob, eob_costs, coeff_costs, tx_class);
471
472
0
  av1_get_nz_map_contexts(levels, scan, eob, tx_size, tx_class, coeff_contexts);
473
474
0
  const int(*lps_cost)[COEFF_BASE_RANGE + 1 + COEFF_BASE_RANGE + 1] =
475
0
      coeff_costs->lps_cost;
476
0
  int c = eob - 1;
477
0
  {
478
0
    const int pos = scan[c];
479
0
    const tran_low_t v = qcoeff[pos];
480
0
    const int sign = AOMSIGN(v);
481
0
    const int level = (v ^ sign) - sign;
482
0
    const int coeff_ctx = coeff_contexts[pos];
483
0
    cost += coeff_costs->base_eob_cost[coeff_ctx][AOMMIN(level, 3) - 1];
484
485
0
    if (v) {
486
      // sign bit cost
487
0
      if (level > NUM_BASE_LEVELS) {
488
0
        const int ctx = get_br_ctx_eob(pos, bwl, tx_class);
489
0
        cost += get_br_cost(level, lps_cost[ctx]);
490
0
      }
491
0
      if (c) {
492
0
        cost += av1_cost_literal(1);
493
0
      } else {
494
0
        const int sign01 = (sign ^ sign) - sign;
495
0
        const int dc_sign_ctx = txb_ctx->dc_sign_ctx;
496
0
        cost += coeff_costs->dc_sign_cost[dc_sign_ctx][sign01];
497
0
        return cost;
498
0
      }
499
0
    }
500
0
  }
501
0
  const int(*base_cost)[8] = coeff_costs->base_cost;
502
0
  for (c = eob - 2; c >= 1; --c) {
503
0
    const int pos = scan[c];
504
0
    const int coeff_ctx = coeff_contexts[pos];
505
0
    const tran_low_t v = qcoeff[pos];
506
0
    const int level = abs(v);
507
0
    cost += base_cost[coeff_ctx][AOMMIN(level, 3)];
508
0
    if (v) {
509
      // sign bit cost
510
0
      cost += av1_cost_literal(1);
511
0
      if (level > NUM_BASE_LEVELS) {
512
0
        const int ctx = get_br_ctx(levels, pos, bwl, tx_class);
513
0
        cost += get_br_cost(level, lps_cost[ctx]);
514
0
      }
515
0
    }
516
0
  }
517
  // c == 0 after previous loop
518
0
  {
519
0
    const int pos = scan[c];
520
0
    const tran_low_t v = qcoeff[pos];
521
0
    const int coeff_ctx = coeff_contexts[pos];
522
0
    const int sign = AOMSIGN(v);
523
0
    const int level = (v ^ sign) - sign;
524
0
    cost += base_cost[coeff_ctx][AOMMIN(level, 3)];
525
526
0
    if (v) {
527
      // sign bit cost
528
0
      const int sign01 = (sign ^ sign) - sign;
529
0
      const int dc_sign_ctx = txb_ctx->dc_sign_ctx;
530
0
      cost += coeff_costs->dc_sign_cost[dc_sign_ctx][sign01];
531
0
      if (level > NUM_BASE_LEVELS) {
532
0
        const int ctx = get_br_ctx(levels, pos, bwl, tx_class);
533
0
        cost += get_br_cost(level, lps_cost[ctx]);
534
0
      }
535
0
    }
536
0
  }
537
0
  return cost;
538
0
}
539
540
int av1_cost_coeffs_txb_estimate(const MACROBLOCK *x, const int plane,
541
                                 const int block, const TX_SIZE tx_size,
542
0
                                 const TX_TYPE tx_type) {
543
0
  assert(plane == 0);
544
545
0
  int cost = 0;
546
0
  const struct macroblock_plane *p = &x->plane[plane];
547
0
  const SCAN_ORDER *scan_order = get_scan(tx_size, tx_type);
548
0
  const int16_t *scan = scan_order->scan;
549
0
  tran_low_t *qcoeff = p->qcoeff + BLOCK_OFFSET(block);
550
551
0
  int eob = p->eobs[block];
552
553
  // coeffs
554
0
  int c = eob - 1;
555
  // eob
556
0
  {
557
0
    const int pos = scan[c];
558
0
    const tran_low_t v = abs(qcoeff[pos]) - 1;
559
0
    cost += (v << (AV1_PROB_COST_SHIFT + 2));
560
0
  }
561
  // other coeffs
562
0
  for (c = eob - 2; c >= 0; c--) {
563
0
    const int pos = scan[c];
564
0
    const tran_low_t v = abs(qcoeff[pos]);
565
0
    const int idx = AOMMIN(v, 14);
566
567
0
    cost += costLUT[idx];
568
0
  }
569
570
  // const_term does not contain DC, and log(e) does not contain eob, so both
571
  // (eob-1)
572
0
  cost += (const_term + loge_par) * (eob - 1);
573
574
0
  return cost;
575
0
}
576
577
static AOM_FORCE_INLINE int warehouse_efficients_txb_laplacian(
578
    const MACROBLOCK *x, const int plane, const int block,
579
    const TX_SIZE tx_size, const TXB_CTX *const txb_ctx, const int eob,
580
    const PLANE_TYPE plane_type, const LV_MAP_COEFF_COST *const coeff_costs,
581
    const MACROBLOCKD *const xd, const TX_TYPE tx_type, const TX_CLASS tx_class,
582
0
    int reduced_tx_set_used) {
583
0
  const int txb_skip_ctx = txb_ctx->txb_skip_ctx;
584
585
0
  const int eob_multi_size = txsize_log2_minus4[tx_size];
586
0
  const LV_MAP_EOB_COST *const eob_costs =
587
0
      &x->coeff_costs.eob_costs[eob_multi_size][plane_type];
588
0
  int cost = coeff_costs->txb_skip_cost[txb_skip_ctx][0];
589
590
0
  cost += get_tx_type_cost(x, xd, plane, tx_size, tx_type, reduced_tx_set_used);
591
592
0
  cost += get_eob_cost(eob, eob_costs, coeff_costs, tx_class);
593
594
0
  cost += av1_cost_coeffs_txb_estimate(x, plane, block, tx_size, tx_type);
595
0
  return cost;
596
0
}
597
598
int av1_cost_coeffs_txb(const MACROBLOCK *x, const int plane, const int block,
599
                        const TX_SIZE tx_size, const TX_TYPE tx_type,
600
0
                        const TXB_CTX *const txb_ctx, int reduced_tx_set_used) {
601
0
  const struct macroblock_plane *p = &x->plane[plane];
602
0
  const int eob = p->eobs[block];
603
0
  const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size);
604
0
  const PLANE_TYPE plane_type = get_plane_type(plane);
605
0
  const LV_MAP_COEFF_COST *const coeff_costs =
606
0
      &x->coeff_costs.coeff_costs[txs_ctx][plane_type];
607
0
  if (eob == 0) {
608
0
    return coeff_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1];
609
0
  }
610
611
0
  const MACROBLOCKD *const xd = &x->e_mbd;
612
0
  const TX_CLASS tx_class = tx_type_to_class[tx_type];
613
614
0
  return warehouse_efficients_txb(x, plane, block, tx_size, txb_ctx, p, eob,
615
0
                                  plane_type, coeff_costs, xd, tx_type,
616
0
                                  tx_class, reduced_tx_set_used);
617
0
}
618
619
int av1_cost_coeffs_txb_laplacian(const MACROBLOCK *x, const int plane,
620
                                  const int block, const TX_SIZE tx_size,
621
                                  const TX_TYPE tx_type,
622
                                  const TXB_CTX *const txb_ctx,
623
                                  const int reduced_tx_set_used,
624
0
                                  const int adjust_eob) {
625
0
  const struct macroblock_plane *p = &x->plane[plane];
626
0
  int eob = p->eobs[block];
627
628
0
  if (adjust_eob) {
629
0
    const SCAN_ORDER *scan_order = get_scan(tx_size, tx_type);
630
0
    const int16_t *scan = scan_order->scan;
631
0
    tran_low_t *tcoeff = p->coeff + BLOCK_OFFSET(block);
632
0
    tran_low_t *qcoeff = p->qcoeff + BLOCK_OFFSET(block);
633
0
    tran_low_t *dqcoeff = p->dqcoeff + BLOCK_OFFSET(block);
634
0
    update_coeff_eob_fast(&eob, av1_get_tx_scale(tx_size), p->dequant_QTX, scan,
635
0
                          tcoeff, qcoeff, dqcoeff);
636
0
    p->eobs[block] = eob;
637
0
  }
638
639
0
  const TX_SIZE txs_ctx = get_txsize_entropy_ctx(tx_size);
640
0
  const PLANE_TYPE plane_type = get_plane_type(plane);
641
0
  const LV_MAP_COEFF_COST *const coeff_costs =
642
0
      &x->coeff_costs.coeff_costs[txs_ctx][plane_type];
643
0
  if (eob == 0) {
644
0
    return coeff_costs->txb_skip_cost[txb_ctx->txb_skip_ctx][1];
645
0
  }
646
647
0
  const MACROBLOCKD *const xd = &x->e_mbd;
648
0
  const TX_CLASS tx_class = tx_type_to_class[tx_type];
649
650
0
  return warehouse_efficients_txb_laplacian(
651
0
      x, plane, block, tx_size, txb_ctx, eob, plane_type, coeff_costs, xd,
652
0
      tx_type, tx_class, reduced_tx_set_used);
653
0
}