Coverage Report

Created: 2026-11-12 08:20

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/aom/av1/encoder/txb_rdopt_utils.h
Line
Count
Source
1
/*
2
 * Copyright (c) 2021, Alliance for Open Media. All rights reserved.
3
 *
4
 * This source code is subject to the terms of the BSD 2 Clause License and
5
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6
 * was not distributed with this source code in the LICENSE file, you can
7
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8
 * Media Patent License 1.0 was not distributed with this source code in the
9
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10
 */
11
12
#ifndef AOM_AV1_ENCODER_TXB_RDOPT_UTILS_H_
13
#define AOM_AV1_ENCODER_TXB_RDOPT_UTILS_H_
14
15
#include "av1/encoder/encodetxb.h"
16
17
static const int golomb_bits_cost[32] = {
18
  0,       512,     512 * 3, 512 * 3, 512 * 5, 512 * 5, 512 * 5, 512 * 5,
19
  512 * 7, 512 * 7, 512 * 7, 512 * 7, 512 * 7, 512 * 7, 512 * 7, 512 * 7,
20
  512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9,
21
  512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9, 512 * 9
22
};
23
24
static const int golomb_cost_diff[32] = {
25
  0,       512, 512 * 2, 0, 512 * 2, 0, 0, 0, 512 * 2, 0, 0, 0, 0, 0, 0, 0,
26
  512 * 2, 0,   0,       0, 0,       0, 0, 0, 0,       0, 0, 0, 0, 0, 0, 0
27
};
28
29
// Look up table of individual cost of coefficient by its quantization level.
30
// determined based on Laplacian distribution conditioned on estimated context
31
static const int costLUT[15] = { -1143, 53,   545,  825,  1031,
32
                                 1209,  1393, 1577, 1763, 1947,
33
                                 2132,  2317, 2501, 2686, 2871 };
34
35
static const int const_term = (1 << AV1_PROB_COST_SHIFT);
36
37
static const int loge_par = ((14427 << AV1_PROB_COST_SHIFT) + 5000) / 10000;
38
39
static inline int get_dqv(const int16_t *dequant, int coeff_idx,
40
0
                          const qm_val_t *iqmatrix) {
41
0
  int dqv = dequant[!!coeff_idx];
42
0
  if (iqmatrix != NULL)
43
0
    dqv =
44
0
        ((iqmatrix[coeff_idx] * dqv) + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS;
45
0
  return dqv;
46
0
}
47
48
static inline int64_t get_coeff_dist_qm(tran_low_t tcoeff, tran_low_t dqcoeff,
49
0
                                        int shift, int64_t qm_val) {
50
0
  const int64_t diff = (int64_t)(tcoeff - dqcoeff) * (1 << shift) * qm_val;
51
0
  return (diff * diff + (1 << (2 * AOM_QM_BITS - 1))) >> (2 * AOM_QM_BITS);
52
0
}
53
54
static inline int64_t get_coeff_dist(tran_low_t tcoeff, tran_low_t dqcoeff,
55
                                     int shift, const qm_val_t *qmatrix,
56
0
                                     int coeff_idx) {
57
0
  const int64_t diff = (int64_t)(tcoeff - dqcoeff) * (1 << shift);
58
0
  if (qmatrix == NULL) {
59
0
    return diff * diff;
60
0
  }
61
0
  const int64_t wdiff = diff * qmatrix[coeff_idx];
62
0
  return (wdiff * wdiff + (1 << (2 * AOM_QM_BITS - 1))) >> (2 * AOM_QM_BITS);
63
0
}
64
65
static inline int get_eob_cost(int eob, const LV_MAP_EOB_COST *txb_eob_costs,
66
                               const LV_MAP_COEFF_COST *txb_costs,
67
0
                               TX_CLASS tx_class) {
68
0
  const int eob_multi_ctx = (tx_class == TX_CLASS_2D) ? 0 : 1;
69
0
  if (eob < 3) {
70
0
    return txb_eob_costs->eob_cost[eob_multi_ctx][eob - 1];
71
0
  }
72
  // For eob >= 3, eob_pt = k + 2 where k = get_msb(eob - 1), and offset_bits =
73
  // k. The top offset bit uses context-coded eob_extra_cost[k - 1][bit], and
74
  // the remaining (k - 1) offset bits are coded as literals.
75
0
  const int d = eob - 1;
76
0
  const int k = get_msb(d);
77
0
  const int bit = (d >> (k - 1)) & 1;
78
0
  return txb_eob_costs->eob_cost[eob_multi_ctx][k + 1] +
79
0
         txb_costs->eob_extra_cost[k - 1][bit] + av1_cost_literal(k - 1);
80
0
}
81
82
0
static inline int get_golomb_cost(int abs_qc) {
83
0
  if (abs_qc >= 1 + NUM_BASE_LEVELS + COEFF_BASE_RANGE) {
84
0
    const int r = abs_qc - COEFF_BASE_RANGE - NUM_BASE_LEVELS;
85
0
    if (r < 32) return golomb_bits_cost[r];
86
0
    const int length = get_msb(r) + 1;
87
0
    return av1_cost_literal(2 * length - 1);
88
0
  }
89
0
  return 0;
90
0
}
91
92
0
static inline int get_br_cost(tran_low_t level, const int *coeff_lps) {
93
0
  if (level < 1 + NUM_BASE_LEVELS + COEFF_BASE_RANGE) {
94
0
    return coeff_lps[level - 1 - NUM_BASE_LEVELS];
95
0
  }
96
0
  const int r = level - COEFF_BASE_RANGE - NUM_BASE_LEVELS;
97
0
  return coeff_lps[COEFF_BASE_RANGE] + av1_cost_literal((get_msb(r) << 1) + 1);
98
0
}
99
100
static inline int get_br_cost_with_diff(tran_low_t level, const int *coeff_lps,
101
0
                                        int *diff) {
102
0
  if (level < 1 + NUM_BASE_LEVELS + COEFF_BASE_RANGE) {
103
0
    const int base_range = level - 1 - NUM_BASE_LEVELS;
104
0
    *diff += coeff_lps[base_range + COEFF_BASE_RANGE + 1];
105
0
    return coeff_lps[base_range];
106
0
  }
107
0
  int golomb_bits = 0;
108
0
  if (level == COEFF_BASE_RANGE + 1 + NUM_BASE_LEVELS)
109
0
    *diff += coeff_lps[COEFF_BASE_RANGE + COEFF_BASE_RANGE + 1];
110
0
111
0
  const int r = level - COEFF_BASE_RANGE - NUM_BASE_LEVELS;
112
0
  if (r < 32) {
113
0
    golomb_bits = golomb_bits_cost[r];
114
0
    *diff += golomb_cost_diff[r];
115
0
  } else {
116
0
    golomb_bits = av1_cost_literal((get_msb(r) << 1) + 1);
117
0
    *diff += (r & (r - 1)) == 0 ? 1024 : 0;
118
0
  }
119
0
120
0
  return coeff_lps[COEFF_BASE_RANGE] + golomb_bits;
121
0
}
122
123
static AOM_FORCE_INLINE int get_two_coeff_cost_simple(
124
    int ci, tran_low_t abs_qc, int coeff_ctx,
125
    const LV_MAP_COEFF_COST *txb_costs, int bhl, TX_CLASS tx_class,
126
0
    const uint8_t *levels, int *cost_low) {
127
0
  // this simple version assumes the coeff's scan_idx is not DC (scan_idx != 0)
128
0
  // and not the last (scan_idx != eob - 1)
129
0
  assert(ci > 0);
130
0
  int cost = txb_costs->base_cost[coeff_ctx][AOMMIN(abs_qc, 3)];
131
0
  int diff = 0;
132
0
  if (abs_qc <= 3) diff = txb_costs->base_cost[coeff_ctx][abs_qc + 4];
133
0
  if (abs_qc) {
134
0
    cost += av1_cost_literal(1);
135
0
    if (abs_qc > NUM_BASE_LEVELS) {
136
0
      const int br_ctx = get_br_ctx_simple(levels, ci, bhl, tx_class);
137
0
      int brcost_diff = 0;
138
0
      cost += get_br_cost_with_diff(abs_qc, txb_costs->lps_cost[br_ctx],
139
0
                                    &brcost_diff);
140
0
      diff += brcost_diff;
141
0
    }
142
0
  }
143
0
  *cost_low = cost - diff;
144
0
145
0
  return cost;
146
0
}
147
148
static inline int get_coeff_cost_eob(int ci, tran_low_t abs_qc, int sign,
149
                                     int coeff_ctx, int dc_sign_ctx,
150
                                     const LV_MAP_COEFF_COST *txb_costs,
151
0
                                     int bhl, TX_CLASS tx_class) {
152
0
  int cost = 0;
153
0
  cost += txb_costs->base_eob_cost[coeff_ctx][AOMMIN(abs_qc, 3) - 1];
154
0
  if (abs_qc != 0) {
155
0
    if (ci == 0) {
156
0
      cost += txb_costs->dc_sign_cost[dc_sign_ctx][sign];
157
0
    } else {
158
0
      cost += av1_cost_literal(1);
159
0
    }
160
0
    if (abs_qc > NUM_BASE_LEVELS) {
161
0
      int br_ctx;
162
0
      br_ctx = get_br_ctx_eob(ci, bhl, tx_class);
163
0
      cost += get_br_cost(abs_qc, txb_costs->lps_cost[br_ctx]);
164
0
    }
165
0
  }
166
0
  return cost;
167
0
}
168
169
static inline int get_coeff_cost_general(int is_last, int ci, tran_low_t abs_qc,
170
                                         int sign, int coeff_ctx,
171
                                         int dc_sign_ctx,
172
                                         const LV_MAP_COEFF_COST *txb_costs,
173
                                         int bhl, TX_CLASS tx_class,
174
0
                                         const uint8_t *levels) {
175
0
  int cost = 0;
176
0
  if (is_last) {
177
0
    cost += txb_costs->base_eob_cost[coeff_ctx][AOMMIN(abs_qc, 3) - 1];
178
0
  } else {
179
0
    cost += txb_costs->base_cost[coeff_ctx][AOMMIN(abs_qc, 3)];
180
0
  }
181
0
  if (abs_qc != 0) {
182
0
    if (ci == 0) {
183
0
      cost += txb_costs->dc_sign_cost[dc_sign_ctx][sign];
184
0
    } else {
185
0
      cost += av1_cost_literal(1);
186
0
    }
187
0
    if (abs_qc > NUM_BASE_LEVELS) {
188
0
      int br_ctx;
189
0
      if (is_last)
190
0
        br_ctx = get_br_ctx_eob(ci, bhl, tx_class);
191
0
      else
192
0
        br_ctx = get_br_ctx(levels, ci, bhl, tx_class);
193
0
      cost += get_br_cost(abs_qc, txb_costs->lps_cost[br_ctx]);
194
0
    }
195
0
  }
196
0
  return cost;
197
0
}
198
199
static inline void get_qc_dqc_low(tran_low_t abs_qc, int sign, int dqv,
200
                                  int shift, tran_low_t *qc_low,
201
0
                                  tran_low_t *dqc_low) {
202
0
  tran_low_t abs_qc_low = abs_qc - 1;
203
0
  *qc_low = (-sign ^ abs_qc_low) + sign;
204
0
  assert((sign ? -abs_qc_low : abs_qc_low) == *qc_low);
205
0
  tran_low_t abs_dqc_low = (abs_qc_low * dqv) >> shift;
206
0
  *dqc_low = (-sign ^ abs_dqc_low) + sign;
207
0
  assert((sign ? -abs_dqc_low : abs_dqc_low) == *dqc_low);
208
0
}
209
210
static inline void update_coeff_eob_fast(int *eob, int shift,
211
                                         const int16_t *dequant_ptr,
212
                                         const int16_t *scan,
213
                                         const tran_low_t *coeff_ptr,
214
                                         tran_low_t *qcoeff_ptr,
215
0
                                         tran_low_t *dqcoeff_ptr) {
216
  // TODO(sarahparker) make this work for aomqm
217
0
  int eob_out = *eob;
218
0
  int zbin[2] = { dequant_ptr[0] + ROUND_POWER_OF_TWO(dequant_ptr[0] * 70, 7),
219
0
                  dequant_ptr[1] + ROUND_POWER_OF_TWO(dequant_ptr[1] * 70, 7) };
220
221
0
  for (int i = *eob - 1; i >= 0; i--) {
222
0
    const int rc = scan[i];
223
0
    const int qcoeff = qcoeff_ptr[rc];
224
0
    const int coeff = coeff_ptr[rc];
225
0
    const int coeff_sign = AOMSIGN(coeff);
226
0
    int64_t abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
227
228
0
    if (((abs_coeff << (1 + shift)) < zbin[rc != 0]) || (qcoeff == 0)) {
229
0
      eob_out--;
230
0
      qcoeff_ptr[rc] = 0;
231
0
      dqcoeff_ptr[rc] = 0;
232
0
    } else {
233
0
      break;
234
0
    }
235
0
  }
236
237
0
  *eob = eob_out;
238
0
}
239
#endif  // AOM_AV1_ENCODER_TXB_RDOPT_UTILS_H_