Coverage Report

Created: 2026-10-27 06:32

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/libavif/ext/aom/av1/encoder/av1_quantize.c
Line
Count
Source
1
/*
2
 * Copyright (c) 2016, Alliance for Open Media. All rights reserved.
3
 *
4
 * This source code is subject to the terms of the BSD 2 Clause License and
5
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6
 * was not distributed with this source code in the LICENSE file, you can
7
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8
 * Media Patent License 1.0 was not distributed with this source code in the
9
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10
 */
11
12
#include <math.h>
13
14
#include "config/aom_dsp_rtcd.h"
15
16
#include "aom/aomcx.h"
17
#include "aom_dsp/quantize.h"
18
#include "aom_mem/aom_mem.h"
19
#include "aom_ports/bitops.h"
20
#include "aom_ports/mem.h"
21
22
#include "av1/common/idct.h"
23
#include "av1/common/quant_common.h"
24
#include "av1/common/scan.h"
25
#include "av1/common/seg_common.h"
26
27
#include "av1/encoder/av1_quantize.h"
28
#include "av1/encoder/encoder.h"
29
#include "av1/encoder/rd.h"
30
31
void av1_quantize_skip(intptr_t n_coeffs, tran_low_t *qcoeff_ptr,
32
0
                       tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr) {
33
0
  memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
34
0
  memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
35
0
  *eob_ptr = 0;
36
0
}
37
38
int av1_quantize_fp_no_qmatrix(const int16_t quant_ptr[2],
39
                               const int16_t dequant_ptr[2],
40
                               const int16_t round_ptr[2], int log_scale,
41
                               const int16_t *scan, int coeff_count,
42
                               const tran_low_t *coeff_ptr,
43
                               tran_low_t *qcoeff_ptr,
44
0
                               tran_low_t *dqcoeff_ptr) {
45
0
  memset(qcoeff_ptr, 0, coeff_count * sizeof(*qcoeff_ptr));
46
0
  memset(dqcoeff_ptr, 0, coeff_count * sizeof(*dqcoeff_ptr));
47
0
  const int rounding[2] = { ROUND_POWER_OF_TWO(round_ptr[0], log_scale),
48
0
                            ROUND_POWER_OF_TWO(round_ptr[1], log_scale) };
49
0
  int eob = 0;
50
0
  for (int i = 0; i < coeff_count; i++) {
51
0
    const int rc = scan[i];
52
0
    const int32_t thresh = (int32_t)(dequant_ptr[rc != 0]);
53
0
    const int coeff = coeff_ptr[rc];
54
0
    const int coeff_sign = AOMSIGN(coeff);
55
0
    int64_t abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
56
0
    int tmp32 = 0;
57
0
    if ((abs_coeff << (1 + log_scale)) >= thresh) {
58
0
      abs_coeff = clamp64(abs_coeff + rounding[rc != 0], INT16_MIN, INT16_MAX);
59
0
      tmp32 = (int)((abs_coeff * quant_ptr[rc != 0]) >> (16 - log_scale));
60
0
      if (tmp32) {
61
0
        qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign;
62
0
        const tran_low_t abs_dqcoeff =
63
0
            (tmp32 * dequant_ptr[rc != 0]) >> log_scale;
64
0
        dqcoeff_ptr[rc] = (abs_dqcoeff ^ coeff_sign) - coeff_sign;
65
0
      }
66
0
    }
67
0
    if (tmp32) eob = i + 1;
68
0
  }
69
0
  return eob;
70
0
}
71
72
static void quantize_fp_helper_c(
73
    const tran_low_t *coeff_ptr, intptr_t n_coeffs, const int16_t *zbin_ptr,
74
    const int16_t *round_ptr, const int16_t *quant_ptr,
75
    const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
76
    tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
77
    const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr,
78
137M
    const qm_val_t *iqm_ptr, int log_scale) {
79
137M
  int i, eob = -1;
80
137M
  const int rounding[2] = { ROUND_POWER_OF_TWO(round_ptr[0], log_scale),
81
137M
                            ROUND_POWER_OF_TWO(round_ptr[1], log_scale) };
82
  // TODO(jingning) Decide the need of these arguments after the
83
  // quantization process is completed.
84
137M
  (void)zbin_ptr;
85
137M
  (void)quant_shift_ptr;
86
137M
  (void)iscan;
87
88
137M
  memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
89
137M
  memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
90
91
137M
  if (qm_ptr == NULL && iqm_ptr == NULL) {
92
0
    *eob_ptr = av1_quantize_fp_no_qmatrix(quant_ptr, dequant_ptr, round_ptr,
93
0
                                          log_scale, scan, (int)n_coeffs,
94
0
                                          coeff_ptr, qcoeff_ptr, dqcoeff_ptr);
95
137M
  } else {
96
    // Quantization pass: All coefficients with index >= zero_flag are
97
    // skippable. Note: zero_flag can be zero.
98
7.15G
    for (i = 0; i < n_coeffs; i++) {
99
7.01G
      const int rc = scan[i];
100
7.01G
      const int coeff = coeff_ptr[rc];
101
7.01G
      const qm_val_t wt = qm_ptr ? qm_ptr[rc] : (1 << AOM_QM_BITS);
102
7.01G
      const qm_val_t iwt = iqm_ptr ? iqm_ptr[rc] : (1 << AOM_QM_BITS);
103
7.01G
      const int dequant =
104
7.01G
          (dequant_ptr[rc != 0] * iwt + (1 << (AOM_QM_BITS - 1))) >>
105
7.01G
          AOM_QM_BITS;
106
7.01G
      const int coeff_sign = AOMSIGN(coeff);
107
7.01G
      int64_t abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
108
7.01G
      int tmp32 = 0;
109
7.01G
      if (abs_coeff * wt >=
110
7.01G
          (dequant_ptr[rc != 0] << (AOM_QM_BITS - (1 + log_scale)))) {
111
3.21G
        abs_coeff += rounding[rc != 0];
112
3.21G
        abs_coeff = clamp64(abs_coeff, INT16_MIN, INT16_MAX);
113
3.21G
        tmp32 = (int)((abs_coeff * wt * quant_ptr[rc != 0]) >>
114
3.21G
                      (16 - log_scale + AOM_QM_BITS));
115
3.21G
        qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign;
116
3.21G
        const tran_low_t abs_dqcoeff = (tmp32 * dequant) >> log_scale;
117
3.21G
        dqcoeff_ptr[rc] = (abs_dqcoeff ^ coeff_sign) - coeff_sign;
118
3.21G
      }
119
120
7.01G
      if (tmp32) eob = i;
121
7.01G
    }
122
137M
    *eob_ptr = eob + 1;
123
137M
  }
124
137M
}
125
126
#if CONFIG_AV1_HIGHBITDEPTH
127
static void highbd_quantize_fp_helper_c(
128
    const tran_low_t *coeff_ptr, intptr_t count, const int16_t *zbin_ptr,
129
    const int16_t *round_ptr, const int16_t *quant_ptr,
130
    const int16_t *quant_shift_ptr, tran_low_t *qcoeff_ptr,
131
    tran_low_t *dqcoeff_ptr, const int16_t *dequant_ptr, uint16_t *eob_ptr,
132
    const int16_t *scan, const int16_t *iscan, const qm_val_t *qm_ptr,
133
43.2M
    const qm_val_t *iqm_ptr, int log_scale) {
134
43.2M
  int i;
135
43.2M
  int eob = -1;
136
43.2M
  const int shift = 16 - log_scale;
137
  // TODO(jingning) Decide the need of these arguments after the
138
  // quantization process is completed.
139
43.2M
  (void)zbin_ptr;
140
43.2M
  (void)quant_shift_ptr;
141
43.2M
  (void)iscan;
142
143
43.2M
  if (qm_ptr || iqm_ptr) {
144
    // Quantization pass: All coefficients with index >= zero_flag are
145
    // skippable. Note: zero_flag can be zero.
146
1.78G
    for (i = 0; i < count; i++) {
147
1.73G
      const int rc = scan[i];
148
1.73G
      const int coeff = coeff_ptr[rc];
149
18.4E
      const qm_val_t wt = qm_ptr != NULL ? qm_ptr[rc] : (1 << AOM_QM_BITS);
150
18.4E
      const qm_val_t iwt = iqm_ptr != NULL ? iqm_ptr[rc] : (1 << AOM_QM_BITS);
151
1.73G
      const int dequant =
152
1.73G
          (dequant_ptr[rc != 0] * iwt + (1 << (AOM_QM_BITS - 1))) >>
153
1.73G
          AOM_QM_BITS;
154
1.73G
      const int coeff_sign = AOMSIGN(coeff);
155
1.73G
      const int64_t abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
156
1.73G
      int abs_qcoeff = 0;
157
1.73G
      if (abs_coeff * wt >=
158
1.73G
          (dequant_ptr[rc != 0] << (AOM_QM_BITS - (1 + log_scale)))) {
159
689M
        const int64_t tmp =
160
689M
            abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], log_scale);
161
689M
        abs_qcoeff =
162
689M
            (int)((tmp * quant_ptr[rc != 0] * wt) >> (shift + AOM_QM_BITS));
163
689M
        qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
164
689M
        const tran_low_t abs_dqcoeff = (abs_qcoeff * dequant) >> log_scale;
165
689M
        dqcoeff_ptr[rc] = (tran_low_t)((abs_dqcoeff ^ coeff_sign) - coeff_sign);
166
689M
        if (abs_qcoeff) eob = i;
167
1.04G
      } else {
168
1.04G
        qcoeff_ptr[rc] = 0;
169
1.04G
        dqcoeff_ptr[rc] = 0;
170
1.04G
      }
171
1.73G
    }
172
18.4E
  } else {
173
18.4E
    const int log_scaled_round_arr[2] = {
174
18.4E
      ROUND_POWER_OF_TWO(round_ptr[0], log_scale),
175
18.4E
      ROUND_POWER_OF_TWO(round_ptr[1], log_scale),
176
18.4E
    };
177
18.4E
    for (i = 0; i < count; i++) {
178
0
      const int rc = scan[i];
179
0
      const int coeff = coeff_ptr[rc];
180
0
      const int rc01 = (rc != 0);
181
0
      const int coeff_sign = AOMSIGN(coeff);
182
0
      const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
183
0
      const int log_scaled_round = log_scaled_round_arr[rc01];
184
0
      if ((abs_coeff << (1 + log_scale)) >= dequant_ptr[rc01]) {
185
0
        const int quant = quant_ptr[rc01];
186
0
        const int dequant = dequant_ptr[rc01];
187
0
        const int64_t tmp = (int64_t)abs_coeff + log_scaled_round;
188
0
        const int abs_qcoeff = (int)((tmp * quant) >> shift);
189
0
        qcoeff_ptr[rc] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
190
0
        const tran_low_t abs_dqcoeff = (abs_qcoeff * dequant) >> log_scale;
191
0
        if (abs_qcoeff) eob = i;
192
0
        dqcoeff_ptr[rc] = (tran_low_t)((abs_dqcoeff ^ coeff_sign) - coeff_sign);
193
0
      } else {
194
0
        qcoeff_ptr[rc] = 0;
195
0
        dqcoeff_ptr[rc] = 0;
196
0
      }
197
0
    }
198
18.4E
  }
199
43.2M
  *eob_ptr = eob + 1;
200
43.2M
}
201
#endif  // CONFIG_AV1_HIGHBITDEPTH
202
203
void av1_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
204
                       const int16_t *zbin_ptr, const int16_t *round_ptr,
205
                       const int16_t *quant_ptr, const int16_t *quant_shift_ptr,
206
                       tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
207
                       const int16_t *dequant_ptr, uint16_t *eob_ptr,
208
0
                       const int16_t *scan, const int16_t *iscan) {
209
0
  quantize_fp_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr, quant_ptr,
210
0
                       quant_shift_ptr, qcoeff_ptr, dqcoeff_ptr, dequant_ptr,
211
0
                       eob_ptr, scan, iscan, NULL, NULL, 0);
212
0
}
213
214
void av1_quantize_lp_c(const int16_t *coeff_ptr, intptr_t n_coeffs,
215
                       const int16_t *round_ptr, const int16_t *quant_ptr,
216
                       int16_t *qcoeff_ptr, int16_t *dqcoeff_ptr,
217
                       const int16_t *dequant_ptr, uint16_t *eob_ptr,
218
0
                       const int16_t *scan, const int16_t *iscan) {
219
0
  (void)iscan;
220
0
  int eob = -1;
221
222
0
  memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
223
0
  memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
224
225
  // Quantization pass: All coefficients with index >= zero_flag are
226
  // skippable. Note: zero_flag can be zero.
227
0
  for (int i = 0; i < n_coeffs; i++) {
228
0
    const int rc = scan[i];
229
0
    const int coeff = coeff_ptr[rc];
230
0
    const int coeff_sign = AOMSIGN(coeff);
231
0
    const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
232
233
0
    int tmp = clamp(abs_coeff + round_ptr[rc != 0], INT16_MIN, INT16_MAX);
234
0
    tmp = (tmp * quant_ptr[rc != 0]) >> 16;
235
236
0
    qcoeff_ptr[rc] = (tmp ^ coeff_sign) - coeff_sign;
237
0
    dqcoeff_ptr[rc] = qcoeff_ptr[rc] * dequant_ptr[rc != 0];
238
239
0
    if (tmp) eob = i;
240
0
  }
241
0
  *eob_ptr = eob + 1;
242
0
}
243
244
void av1_quantize_fp_32x32_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
245
                             const int16_t *zbin_ptr, const int16_t *round_ptr,
246
                             const int16_t *quant_ptr,
247
                             const int16_t *quant_shift_ptr,
248
                             tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
249
                             const int16_t *dequant_ptr, uint16_t *eob_ptr,
250
0
                             const int16_t *scan, const int16_t *iscan) {
251
0
  quantize_fp_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr, quant_ptr,
252
0
                       quant_shift_ptr, qcoeff_ptr, dqcoeff_ptr, dequant_ptr,
253
0
                       eob_ptr, scan, iscan, NULL, NULL, 1);
254
0
}
255
256
void av1_quantize_fp_64x64_c(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
257
                             const int16_t *zbin_ptr, const int16_t *round_ptr,
258
                             const int16_t *quant_ptr,
259
                             const int16_t *quant_shift_ptr,
260
                             tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
261
                             const int16_t *dequant_ptr, uint16_t *eob_ptr,
262
0
                             const int16_t *scan, const int16_t *iscan) {
263
0
  quantize_fp_helper_c(coeff_ptr, n_coeffs, zbin_ptr, round_ptr, quant_ptr,
264
0
                       quant_shift_ptr, qcoeff_ptr, dqcoeff_ptr, dequant_ptr,
265
0
                       eob_ptr, scan, iscan, NULL, NULL, 2);
266
0
}
267
268
void av1_quantize_fp_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
269
                            const MACROBLOCK_PLANE *p, tran_low_t *qcoeff_ptr,
270
                            tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
271
282M
                            const SCAN_ORDER *sc, const QUANT_PARAM *qparam) {
272
282M
  const qm_val_t *qm_ptr = qparam->qmatrix;
273
282M
  const qm_val_t *iqm_ptr = qparam->iqmatrix;
274
282M
  if (qm_ptr != NULL && iqm_ptr != NULL) {
275
137M
    quantize_fp_helper_c(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_fp_QTX,
276
137M
                         p->quant_fp_QTX, p->quant_shift_QTX, qcoeff_ptr,
277
137M
                         dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan,
278
137M
                         sc->iscan, qm_ptr, iqm_ptr, qparam->log_scale);
279
144M
  } else {
280
144M
    switch (qparam->log_scale) {
281
144M
      case 0:
282
144M
        av1_quantize_fp(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_fp_QTX,
283
144M
                        p->quant_fp_QTX, p->quant_shift_QTX, qcoeff_ptr,
284
144M
                        dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan,
285
144M
                        sc->iscan);
286
144M
        break;
287
1.14M
      case 1:
288
1.14M
        av1_quantize_fp_32x32(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_fp_QTX,
289
1.14M
                              p->quant_fp_QTX, p->quant_shift_QTX, qcoeff_ptr,
290
1.14M
                              dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan,
291
1.14M
                              sc->iscan);
292
1.14M
        break;
293
26.3k
      case 2:
294
26.3k
        av1_quantize_fp_64x64(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_fp_QTX,
295
26.3k
                              p->quant_fp_QTX, p->quant_shift_QTX, qcoeff_ptr,
296
26.3k
                              dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan,
297
26.3k
                              sc->iscan);
298
26.3k
        break;
299
0
      default: assert(0);
300
144M
    }
301
144M
  }
302
282M
}
303
304
void av1_quantize_b_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
305
                           const MACROBLOCK_PLANE *p, tran_low_t *qcoeff_ptr,
306
                           tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
307
257M
                           const SCAN_ORDER *sc, const QUANT_PARAM *qparam) {
308
257M
  const qm_val_t *qm_ptr = qparam->qmatrix;
309
257M
  const qm_val_t *iqm_ptr = qparam->iqmatrix;
310
257M
#if !CONFIG_REALTIME_ONLY
311
257M
  if (qparam->use_quant_b_adapt) {
312
    // TODO(sarahparker) These quantize_b optimizations need SIMD
313
    // implementations
314
0
    if (qm_ptr != NULL && iqm_ptr != NULL) {
315
0
      aom_quantize_b_adaptive_helper_c(
316
0
          coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX,
317
0
          p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, eob_ptr,
318
0
          sc->scan, sc->iscan, qm_ptr, iqm_ptr, qparam->log_scale);
319
0
    } else {
320
0
      switch (qparam->log_scale) {
321
0
        case 0:
322
0
          aom_quantize_b_adaptive(coeff_ptr, n_coeffs, p->zbin_QTX,
323
0
                                  p->round_QTX, p->quant_QTX,
324
0
                                  p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr,
325
0
                                  p->dequant_QTX, eob_ptr, sc->scan, sc->iscan);
326
0
          break;
327
0
        case 1:
328
0
          aom_quantize_b_32x32_adaptive(
329
0
              coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX,
330
0
              p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX,
331
0
              eob_ptr, sc->scan, sc->iscan);
332
0
          break;
333
0
        case 2:
334
0
          aom_quantize_b_64x64_adaptive(
335
0
              coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX,
336
0
              p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX,
337
0
              eob_ptr, sc->scan, sc->iscan);
338
0
          break;
339
0
        default: assert(0);
340
0
      }
341
0
    }
342
0
    return;
343
0
  }
344
257M
#endif  // !CONFIG_REALTIME_ONLY
345
346
257M
  if (qm_ptr != NULL && iqm_ptr != NULL) {
347
87.8M
    aom_quantize_b_helper_c(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX,
348
87.8M
                            p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr,
349
87.8M
                            dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan,
350
87.8M
                            sc->iscan, qm_ptr, iqm_ptr, qparam->log_scale);
351
169M
  } else {
352
169M
    switch (qparam->log_scale) {
353
167M
      case 0:
354
167M
        aom_quantize_b(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX,
355
167M
                       p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr,
356
167M
                       dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan,
357
167M
                       sc->iscan);
358
167M
        break;
359
1.29M
      case 1:
360
1.29M
        aom_quantize_b_32x32(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX,
361
1.29M
                             p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr,
362
1.29M
                             dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan,
363
1.29M
                             sc->iscan);
364
1.29M
        break;
365
58.2k
      case 2:
366
58.2k
        aom_quantize_b_64x64(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX,
367
58.2k
                             p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr,
368
58.2k
                             dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan,
369
58.2k
                             sc->iscan);
370
58.2k
        break;
371
0
      default: assert(0);
372
169M
    }
373
169M
  }
374
257M
}
375
376
static void quantize_dc(const tran_low_t *coeff_ptr, int n_coeffs,
377
                        int skip_block, const int16_t *round_ptr,
378
                        const int16_t quant, tran_low_t *qcoeff_ptr,
379
                        tran_low_t *dqcoeff_ptr, const int16_t dequant_ptr,
380
                        uint16_t *eob_ptr, const qm_val_t *qm_ptr,
381
0
                        const qm_val_t *iqm_ptr, const int log_scale) {
382
0
  const int rc = 0;
383
0
  const int coeff = coeff_ptr[rc];
384
0
  const int coeff_sign = AOMSIGN(coeff);
385
0
  const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
386
0
  int64_t tmp;
387
0
  int eob = -1;
388
0
  int32_t tmp32;
389
0
  int dequant;
390
391
0
  memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
392
0
  memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
393
394
0
  if (!skip_block) {
395
0
    const int wt = qm_ptr != NULL ? qm_ptr[rc] : (1 << AOM_QM_BITS);
396
0
    const int iwt = iqm_ptr != NULL ? iqm_ptr[rc] : (1 << AOM_QM_BITS);
397
0
    tmp = clamp(abs_coeff + ROUND_POWER_OF_TWO(round_ptr[rc != 0], log_scale),
398
0
                INT16_MIN, INT16_MAX);
399
0
    tmp32 = (int32_t)((tmp * wt * quant) >> (16 - log_scale + AOM_QM_BITS));
400
0
    qcoeff_ptr[rc] = (tmp32 ^ coeff_sign) - coeff_sign;
401
0
    dequant = (dequant_ptr * iwt + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS;
402
0
    const tran_low_t abs_dqcoeff = (tmp32 * dequant) >> log_scale;
403
0
    dqcoeff_ptr[rc] = (tran_low_t)((abs_dqcoeff ^ coeff_sign) - coeff_sign);
404
0
    if (tmp32) eob = 0;
405
0
  }
406
0
  *eob_ptr = eob + 1;
407
0
}
408
409
void av1_quantize_dc_facade(const tran_low_t *coeff_ptr, intptr_t n_coeffs,
410
                            const MACROBLOCK_PLANE *p, tran_low_t *qcoeff_ptr,
411
                            tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
412
0
                            const SCAN_ORDER *sc, const QUANT_PARAM *qparam) {
413
  // obsolete skip_block
414
0
  const int skip_block = 0;
415
0
  (void)sc;
416
0
  assert(qparam->log_scale >= 0 && qparam->log_scale < (3));
417
0
  const qm_val_t *qm_ptr = qparam->qmatrix;
418
0
  const qm_val_t *iqm_ptr = qparam->iqmatrix;
419
0
  quantize_dc(coeff_ptr, (int)n_coeffs, skip_block, p->round_QTX,
420
0
              p->quant_fp_QTX[0], qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX[0],
421
0
              eob_ptr, qm_ptr, iqm_ptr, qparam->log_scale);
422
0
}
423
424
#if CONFIG_AV1_HIGHBITDEPTH
425
void av1_highbd_quantize_fp_facade(const tran_low_t *coeff_ptr,
426
                                   intptr_t n_coeffs, const MACROBLOCK_PLANE *p,
427
                                   tran_low_t *qcoeff_ptr,
428
                                   tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
429
                                   const SCAN_ORDER *sc,
430
86.5M
                                   const QUANT_PARAM *qparam) {
431
86.5M
  const qm_val_t *qm_ptr = qparam->qmatrix;
432
86.5M
  const qm_val_t *iqm_ptr = qparam->iqmatrix;
433
86.5M
  if (qm_ptr != NULL && iqm_ptr != NULL) {
434
42.9M
    highbd_quantize_fp_helper_c(
435
42.9M
        coeff_ptr, n_coeffs, p->zbin_QTX, p->round_fp_QTX, p->quant_fp_QTX,
436
42.9M
        p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, eob_ptr,
437
42.9M
        sc->scan, sc->iscan, qm_ptr, iqm_ptr, qparam->log_scale);
438
43.6M
  } else {
439
43.6M
    av1_highbd_quantize_fp(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_fp_QTX,
440
43.6M
                           p->quant_fp_QTX, p->quant_shift_QTX, qcoeff_ptr,
441
43.6M
                           dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan,
442
43.6M
                           sc->iscan, qparam->log_scale);
443
43.6M
  }
444
86.5M
}
445
446
void av1_highbd_quantize_b_facade(const tran_low_t *coeff_ptr,
447
                                  intptr_t n_coeffs, const MACROBLOCK_PLANE *p,
448
                                  tran_low_t *qcoeff_ptr,
449
                                  tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
450
                                  const SCAN_ORDER *sc,
451
87.2M
                                  const QUANT_PARAM *qparam) {
452
87.2M
  const qm_val_t *qm_ptr = qparam->qmatrix;
453
87.2M
  const qm_val_t *iqm_ptr = qparam->iqmatrix;
454
87.2M
#if !CONFIG_REALTIME_ONLY
455
87.2M
  if (qparam->use_quant_b_adapt) {
456
0
    if (qm_ptr != NULL && iqm_ptr != NULL) {
457
0
      aom_highbd_quantize_b_adaptive_helper_c(
458
0
          coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX,
459
0
          p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, eob_ptr,
460
0
          sc->scan, sc->iscan, qm_ptr, iqm_ptr, qparam->log_scale);
461
0
    } else {
462
0
      switch (qparam->log_scale) {
463
0
        case 0:
464
0
          aom_highbd_quantize_b_adaptive(
465
0
              coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX,
466
0
              p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX,
467
0
              eob_ptr, sc->scan, sc->iscan);
468
0
          break;
469
0
        case 1:
470
0
          aom_highbd_quantize_b_32x32_adaptive(
471
0
              coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX,
472
0
              p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX,
473
0
              eob_ptr, sc->scan, sc->iscan);
474
0
          break;
475
0
        case 2:
476
0
          aom_highbd_quantize_b_64x64_adaptive(
477
0
              coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX,
478
0
              p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX,
479
0
              eob_ptr, sc->scan, sc->iscan);
480
0
          break;
481
0
        default: assert(0);
482
0
      }
483
0
    }
484
0
    return;
485
0
  }
486
87.2M
#endif  // !CONFIG_REALTIME_ONLY
487
488
87.2M
  if (qm_ptr != NULL && iqm_ptr != NULL) {
489
53.0M
    aom_highbd_quantize_b_helper_c(
490
53.0M
        coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX,
491
53.0M
        p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX, eob_ptr,
492
53.0M
        sc->scan, sc->iscan, qm_ptr, iqm_ptr, qparam->log_scale);
493
53.0M
  } else {
494
34.2M
    switch (qparam->log_scale) {
495
33.9M
      case 0:
496
33.9M
        aom_highbd_quantize_b(coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX,
497
33.9M
                              p->quant_QTX, p->quant_shift_QTX, qcoeff_ptr,
498
33.9M
                              dqcoeff_ptr, p->dequant_QTX, eob_ptr, sc->scan,
499
33.9M
                              sc->iscan);
500
33.9M
        break;
501
339k
      case 1:
502
339k
        aom_highbd_quantize_b_32x32(
503
339k
            coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX,
504
339k
            p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX,
505
339k
            eob_ptr, sc->scan, sc->iscan);
506
339k
        break;
507
15.7k
      case 2:
508
15.7k
        aom_highbd_quantize_b_64x64(
509
15.7k
            coeff_ptr, n_coeffs, p->zbin_QTX, p->round_QTX, p->quant_QTX,
510
15.7k
            p->quant_shift_QTX, qcoeff_ptr, dqcoeff_ptr, p->dequant_QTX,
511
15.7k
            eob_ptr, sc->scan, sc->iscan);
512
15.7k
        break;
513
0
      default: assert(0);
514
34.2M
    }
515
34.2M
  }
516
87.2M
}
517
518
static inline void highbd_quantize_dc(
519
    const tran_low_t *coeff_ptr, int n_coeffs, int skip_block,
520
    const int16_t *round_ptr, const int16_t quant, tran_low_t *qcoeff_ptr,
521
    tran_low_t *dqcoeff_ptr, const int16_t dequant_ptr, uint16_t *eob_ptr,
522
0
    const qm_val_t *qm_ptr, const qm_val_t *iqm_ptr, const int log_scale) {
523
0
  int eob = -1;
524
525
0
  memset(qcoeff_ptr, 0, n_coeffs * sizeof(*qcoeff_ptr));
526
0
  memset(dqcoeff_ptr, 0, n_coeffs * sizeof(*dqcoeff_ptr));
527
528
0
  if (!skip_block) {
529
0
    const qm_val_t wt = qm_ptr != NULL ? qm_ptr[0] : (1 << AOM_QM_BITS);
530
0
    const qm_val_t iwt = iqm_ptr != NULL ? iqm_ptr[0] : (1 << AOM_QM_BITS);
531
0
    const int coeff = coeff_ptr[0];
532
0
    const int coeff_sign = AOMSIGN(coeff);
533
0
    const int abs_coeff = (coeff ^ coeff_sign) - coeff_sign;
534
0
    const int64_t tmp = abs_coeff + ROUND_POWER_OF_TWO(round_ptr[0], log_scale);
535
0
    const int64_t tmpw = tmp * wt;
536
0
    const int abs_qcoeff =
537
0
        (int)((tmpw * quant) >> (16 - log_scale + AOM_QM_BITS));
538
0
    qcoeff_ptr[0] = (tran_low_t)((abs_qcoeff ^ coeff_sign) - coeff_sign);
539
0
    const int dequant =
540
0
        (dequant_ptr * iwt + (1 << (AOM_QM_BITS - 1))) >> AOM_QM_BITS;
541
542
0
    const tran_low_t abs_dqcoeff = (abs_qcoeff * dequant) >> log_scale;
543
0
    dqcoeff_ptr[0] = (tran_low_t)((abs_dqcoeff ^ coeff_sign) - coeff_sign);
544
0
    if (abs_qcoeff) eob = 0;
545
0
  }
546
0
  *eob_ptr = eob + 1;
547
0
}
548
549
void av1_highbd_quantize_dc_facade(const tran_low_t *coeff_ptr,
550
                                   intptr_t n_coeffs, const MACROBLOCK_PLANE *p,
551
                                   tran_low_t *qcoeff_ptr,
552
                                   tran_low_t *dqcoeff_ptr, uint16_t *eob_ptr,
553
                                   const SCAN_ORDER *sc,
554
0
                                   const QUANT_PARAM *qparam) {
555
  // obsolete skip_block
556
0
  const int skip_block = 0;
557
0
  const qm_val_t *qm_ptr = qparam->qmatrix;
558
0
  const qm_val_t *iqm_ptr = qparam->iqmatrix;
559
0
  (void)sc;
560
561
0
  highbd_quantize_dc(coeff_ptr, (int)n_coeffs, skip_block, p->round_QTX,
562
0
                     p->quant_fp_QTX[0], qcoeff_ptr, dqcoeff_ptr,
563
0
                     p->dequant_QTX[0], eob_ptr, qm_ptr, iqm_ptr,
564
0
                     qparam->log_scale);
565
0
}
566
567
void av1_highbd_quantize_fp_c(const tran_low_t *coeff_ptr, intptr_t count,
568
                              const int16_t *zbin_ptr, const int16_t *round_ptr,
569
                              const int16_t *quant_ptr,
570
                              const int16_t *quant_shift_ptr,
571
                              tran_low_t *qcoeff_ptr, tran_low_t *dqcoeff_ptr,
572
                              const int16_t *dequant_ptr, uint16_t *eob_ptr,
573
                              const int16_t *scan, const int16_t *iscan,
574
0
                              int log_scale) {
575
0
  highbd_quantize_fp_helper_c(coeff_ptr, count, zbin_ptr, round_ptr, quant_ptr,
576
0
                              quant_shift_ptr, qcoeff_ptr, dqcoeff_ptr,
577
0
                              dequant_ptr, eob_ptr, scan, iscan, NULL, NULL,
578
0
                              log_scale);
579
0
}
580
#endif  // CONFIG_AV1_HIGHBITDEPTH
581
582
291M
static void invert_quant(int16_t *quant, int16_t *shift, int d) {
583
291M
  uint32_t t;
584
291M
  int l, m;
585
291M
  t = d;
586
291M
  l = get_msb(t);
587
291M
  m = 1 + (1 << (16 + l)) / d;
588
291M
  *quant = (int16_t)(m - (1 << 16));
589
291M
  *shift = 1 << (16 - l);
590
291M
}
591
592
48.6M
static int get_qzbin_factor(int q, aom_bit_depth_t bit_depth) {
593
48.6M
  const int quant = av1_dc_quant_QTX(q, 0, bit_depth);
594
48.6M
  switch (bit_depth) {
595
32.9M
    case AOM_BITS_8: return q == 0 ? 64 : (quant < 148 ? 84 : 80);
596
7.48M
    case AOM_BITS_10: return q == 0 ? 64 : (quant < 592 ? 84 : 80);
597
8.24M
    case AOM_BITS_12: return q == 0 ? 64 : (quant < 2368 ? 84 : 80);
598
0
    default:
599
0
      assert(0 && "bit_depth should be AOM_BITS_8, AOM_BITS_10 or AOM_BITS_12");
600
0
      return -1;
601
48.6M
  }
602
48.6M
}
603
604
void av1_build_quantizer(aom_bit_depth_t bit_depth, int y_dc_delta_q,
605
                         int u_dc_delta_q, int u_ac_delta_q, int v_dc_delta_q,
606
                         int v_ac_delta_q, QUANTS *const quants,
607
190k
                         Dequants *const deq, int sharpness) {
608
190k
  int i, q, quant_QTX;
609
190k
  const int sharpness_adjustment = 16 * (7 - sharpness) / 7;
610
611
48.8M
  for (q = 0; q < QINDEX_RANGE; q++) {
612
48.6M
    const int qzbin_factor = get_qzbin_factor(q, bit_depth);
613
48.6M
    int qrounding_factor = q == 0 ? 64 : 48;
614
615
145M
    for (i = 0; i < 2; ++i) {
616
97.2M
      int qrounding_factor_fp = 64;
617
618
97.2M
      if (sharpness != 0 && q != 0) {
619
37.0M
        qrounding_factor = 64 - sharpness_adjustment;
620
37.0M
        qrounding_factor_fp = 64 - sharpness_adjustment;
621
37.0M
      }
622
623
      // y quantizer with TX scale
624
97.2M
      quant_QTX = i == 0 ? av1_dc_quant_QTX(q, y_dc_delta_q, bit_depth)
625
97.2M
                         : av1_ac_quant_QTX(q, 0, bit_depth);
626
97.2M
      invert_quant(&quants->y_quant[q][i], &quants->y_quant_shift[q][i],
627
97.2M
                   quant_QTX);
628
97.2M
      quants->y_quant_fp[q][i] = (1 << 16) / quant_QTX;
629
97.2M
      quants->y_round_fp[q][i] = (qrounding_factor_fp * quant_QTX) >> 7;
630
97.2M
      quants->y_zbin[q][i] = ROUND_POWER_OF_TWO(qzbin_factor * quant_QTX, 7);
631
97.2M
      quants->y_round[q][i] = (qrounding_factor * quant_QTX) >> 7;
632
97.2M
      deq->y_dequant_QTX[q][i] = quant_QTX;
633
634
      // u quantizer with TX scale
635
97.2M
      quant_QTX = i == 0 ? av1_dc_quant_QTX(q, u_dc_delta_q, bit_depth)
636
97.2M
                         : av1_ac_quant_QTX(q, u_ac_delta_q, bit_depth);
637
97.2M
      invert_quant(&quants->u_quant[q][i], &quants->u_quant_shift[q][i],
638
97.2M
                   quant_QTX);
639
97.2M
      quants->u_quant_fp[q][i] = (1 << 16) / quant_QTX;
640
97.2M
      quants->u_round_fp[q][i] = (qrounding_factor_fp * quant_QTX) >> 7;
641
97.2M
      quants->u_zbin[q][i] = ROUND_POWER_OF_TWO(qzbin_factor * quant_QTX, 7);
642
97.2M
      quants->u_round[q][i] = (qrounding_factor * quant_QTX) >> 7;
643
97.2M
      deq->u_dequant_QTX[q][i] = quant_QTX;
644
645
      // v quantizer with TX scale
646
97.2M
      quant_QTX = i == 0 ? av1_dc_quant_QTX(q, v_dc_delta_q, bit_depth)
647
97.2M
                         : av1_ac_quant_QTX(q, v_ac_delta_q, bit_depth);
648
97.2M
      invert_quant(&quants->v_quant[q][i], &quants->v_quant_shift[q][i],
649
97.2M
                   quant_QTX);
650
97.2M
      quants->v_quant_fp[q][i] = (1 << 16) / quant_QTX;
651
97.2M
      quants->v_round_fp[q][i] = (qrounding_factor_fp * quant_QTX) >> 7;
652
97.2M
      quants->v_zbin[q][i] = ROUND_POWER_OF_TWO(qzbin_factor * quant_QTX, 7);
653
97.2M
      quants->v_round[q][i] = (qrounding_factor * quant_QTX) >> 7;
654
97.2M
      deq->v_dequant_QTX[q][i] = quant_QTX;
655
97.2M
    }
656
657
340M
    for (i = 2; i < 8; i++) {  // 8: SIMD width
658
291M
      quants->y_quant[q][i] = quants->y_quant[q][1];
659
291M
      quants->y_quant_fp[q][i] = quants->y_quant_fp[q][1];
660
291M
      quants->y_round_fp[q][i] = quants->y_round_fp[q][1];
661
291M
      quants->y_quant_shift[q][i] = quants->y_quant_shift[q][1];
662
291M
      quants->y_zbin[q][i] = quants->y_zbin[q][1];
663
291M
      quants->y_round[q][i] = quants->y_round[q][1];
664
291M
      deq->y_dequant_QTX[q][i] = deq->y_dequant_QTX[q][1];
665
666
291M
      quants->u_quant[q][i] = quants->u_quant[q][1];
667
291M
      quants->u_quant_fp[q][i] = quants->u_quant_fp[q][1];
668
291M
      quants->u_round_fp[q][i] = quants->u_round_fp[q][1];
669
291M
      quants->u_quant_shift[q][i] = quants->u_quant_shift[q][1];
670
291M
      quants->u_zbin[q][i] = quants->u_zbin[q][1];
671
291M
      quants->u_round[q][i] = quants->u_round[q][1];
672
291M
      deq->u_dequant_QTX[q][i] = deq->u_dequant_QTX[q][1];
673
674
291M
      quants->v_quant[q][i] = quants->v_quant[q][1];
675
291M
      quants->v_quant_fp[q][i] = quants->v_quant_fp[q][1];
676
291M
      quants->v_round_fp[q][i] = quants->v_round_fp[q][1];
677
291M
      quants->v_quant_shift[q][i] = quants->v_quant_shift[q][1];
678
291M
      quants->v_zbin[q][i] = quants->v_zbin[q][1];
679
291M
      quants->v_round[q][i] = quants->v_round[q][1];
680
291M
      deq->v_dequant_QTX[q][i] = deq->v_dequant_QTX[q][1];
681
291M
    }
682
48.6M
  }
683
190k
}
684
685
static inline bool deltaq_params_have_changed(
686
    const DeltaQuantParams *prev_deltaq_params,
687
366k
    const CommonQuantParams *quant_params) {
688
366k
  return (prev_deltaq_params->y_dc_delta_q != quant_params->y_dc_delta_q ||
689
248k
          prev_deltaq_params->u_dc_delta_q != quant_params->u_dc_delta_q ||
690
205k
          prev_deltaq_params->v_dc_delta_q != quant_params->v_dc_delta_q ||
691
205k
          prev_deltaq_params->u_ac_delta_q != quant_params->u_ac_delta_q ||
692
182k
          prev_deltaq_params->v_ac_delta_q != quant_params->v_ac_delta_q ||
693
182k
          prev_deltaq_params->sharpness != quant_params->sharpness);
694
366k
}
695
696
void av1_init_quantizer(EncQuantDequantParams *const enc_quant_dequant_params,
697
                        CommonQuantParams *quant_params,
698
366k
                        aom_bit_depth_t bit_depth, int sharpness) {
699
366k
  DeltaQuantParams *const prev_deltaq_params =
700
366k
      &enc_quant_dequant_params->prev_deltaq_params;
701
366k
  quant_params->sharpness = sharpness;
702
703
  // Re-initialize the quantizer only if any of the dc/ac deltaq parameters
704
  // change.
705
366k
  if (!deltaq_params_have_changed(prev_deltaq_params, quant_params)) return;
706
190k
  QUANTS *const quants = &enc_quant_dequant_params->quants;
707
190k
  Dequants *const dequants = &enc_quant_dequant_params->dequants;
708
190k
  av1_build_quantizer(bit_depth, quant_params->y_dc_delta_q,
709
190k
                      quant_params->u_dc_delta_q, quant_params->u_ac_delta_q,
710
190k
                      quant_params->v_dc_delta_q, quant_params->v_ac_delta_q,
711
190k
                      quants, dequants, sharpness);
712
713
  // Record the state of deltaq parameters.
714
190k
  prev_deltaq_params->y_dc_delta_q = quant_params->y_dc_delta_q;
715
190k
  prev_deltaq_params->u_dc_delta_q = quant_params->u_dc_delta_q;
716
190k
  prev_deltaq_params->v_dc_delta_q = quant_params->v_dc_delta_q;
717
190k
  prev_deltaq_params->u_ac_delta_q = quant_params->u_ac_delta_q;
718
190k
  prev_deltaq_params->v_ac_delta_q = quant_params->v_ac_delta_q;
719
190k
  prev_deltaq_params->sharpness = sharpness;
720
190k
}
721
722
/*!\brief Update quantize parameters in MACROBLOCK
723
 *
724
 * \param[in]  enc_quant_dequant_params This parameter cached the quantize and
725
 *                                      dequantize parameters for all q
726
 *                                      indices.
727
 * \param[in]  qindex                   Quantize index used for the current
728
 *                                      superblock.
729
 * \param[out] x                        A superblock data structure for
730
 *                                      encoder.
731
 */
732
static void set_q_index(const EncQuantDequantParams *enc_quant_dequant_params,
733
362k
                        int qindex, MACROBLOCK *x) {
734
362k
  const QUANTS *const quants = &enc_quant_dequant_params->quants;
735
362k
  const Dequants *const dequants = &enc_quant_dequant_params->dequants;
736
362k
  x->qindex = qindex;
737
362k
  x->seg_skip_block =
738
362k
      0;  // TODO(angiebird): Find a proper place to init this variable.
739
740
  // Y
741
362k
  x->plane[0].quant_QTX = quants->y_quant[qindex];
742
362k
  x->plane[0].quant_fp_QTX = quants->y_quant_fp[qindex];
743
362k
  x->plane[0].round_fp_QTX = quants->y_round_fp[qindex];
744
362k
  x->plane[0].quant_shift_QTX = quants->y_quant_shift[qindex];
745
362k
  x->plane[0].zbin_QTX = quants->y_zbin[qindex];
746
362k
  x->plane[0].round_QTX = quants->y_round[qindex];
747
362k
  x->plane[0].dequant_QTX = dequants->y_dequant_QTX[qindex];
748
749
  // U
750
362k
  x->plane[1].quant_QTX = quants->u_quant[qindex];
751
362k
  x->plane[1].quant_fp_QTX = quants->u_quant_fp[qindex];
752
362k
  x->plane[1].round_fp_QTX = quants->u_round_fp[qindex];
753
362k
  x->plane[1].quant_shift_QTX = quants->u_quant_shift[qindex];
754
362k
  x->plane[1].zbin_QTX = quants->u_zbin[qindex];
755
362k
  x->plane[1].round_QTX = quants->u_round[qindex];
756
362k
  x->plane[1].dequant_QTX = dequants->u_dequant_QTX[qindex];
757
758
  // V
759
362k
  x->plane[2].quant_QTX = quants->v_quant[qindex];
760
362k
  x->plane[2].quant_fp_QTX = quants->v_quant_fp[qindex];
761
362k
  x->plane[2].round_fp_QTX = quants->v_round_fp[qindex];
762
362k
  x->plane[2].quant_shift_QTX = quants->v_quant_shift[qindex];
763
362k
  x->plane[2].zbin_QTX = quants->v_zbin[qindex];
764
362k
  x->plane[2].round_QTX = quants->v_round[qindex];
765
362k
  x->plane[2].dequant_QTX = dequants->v_dequant_QTX[qindex];
766
362k
}
767
768
/*!\brief Update quantize matrix in MACROBLOCKD based on segment id
769
 *
770
 * \param[in]  quant_params  Quantize parameters used by encoder and decoder
771
 * \param[in]  segment_id    Segment id.
772
 * \param[out] xd            A superblock data structure used by encoder and
773
 * decoder.
774
 */
775
static void set_qmatrix(const CommonQuantParams *quant_params, int segment_id,
776
729k
                        MACROBLOCKD *xd) {
777
729k
  const int use_qmatrix = av1_use_qmatrix(quant_params, xd, segment_id);
778
729k
  const int qmlevel_y =
779
729k
      use_qmatrix ? quant_params->qmatrix_level_y : NUM_QM_LEVELS - 1;
780
729k
  const int qmlevel_u =
781
729k
      use_qmatrix ? quant_params->qmatrix_level_u : NUM_QM_LEVELS - 1;
782
729k
  const int qmlevel_v =
783
729k
      use_qmatrix ? quant_params->qmatrix_level_v : NUM_QM_LEVELS - 1;
784
729k
  const int qmlevel_ls[MAX_MB_PLANE] = { qmlevel_y, qmlevel_u, qmlevel_v };
785
2.90M
  for (int i = 0; i < MAX_MB_PLANE; ++i) {
786
2.18M
    const int qmlevel = qmlevel_ls[i];
787
2.18M
    memcpy(&xd->plane[i].seg_qmatrix[segment_id],
788
2.18M
           quant_params->gqmatrix[qmlevel][i],
789
2.18M
           sizeof(quant_params->gqmatrix[qmlevel][i]));
790
2.18M
    memcpy(&xd->plane[i].seg_iqmatrix[segment_id],
791
2.18M
           quant_params->giqmatrix[qmlevel][i],
792
2.18M
           sizeof(quant_params->giqmatrix[qmlevel][i]));
793
2.18M
  }
794
729k
}
795
796
void av1_init_plane_quantizers(const AV1_COMP *cpi, MACROBLOCK *x,
797
732k
                               int segment_id, const int do_update) {
798
732k
  const AV1_COMMON *const cm = &cpi->common;
799
732k
  const CommonQuantParams *const quant_params = &cm->quant_params;
800
732k
  const GF_GROUP *const gf_group = &cpi->ppi->gf_group;
801
732k
  const int boost_index = AOMMIN(15, (cpi->ppi->p_rc.gfu_boost / 100));
802
732k
  const int layer_depth = AOMMIN(gf_group->layer_depth[cpi->gf_frame_index], 6);
803
732k
  const FRAME_TYPE frame_type = cm->current_frame.frame_type;
804
732k
  int qindex_rd;
805
806
732k
  const int current_qindex =
807
732k
      clamp(cm->delta_q_info.delta_q_present_flag
808
732k
                ? quant_params->base_qindex + x->delta_qindex
809
732k
                : quant_params->base_qindex,
810
732k
            0, QINDEX_RANGE - 1);
811
732k
  const int qindex = av1_get_qindex(&cm->seg, segment_id, current_qindex);
812
813
732k
  if (cpi->oxcf.sb_qp_sweep) {
814
0
    const int current_rd_qindex =
815
0
        clamp(cm->delta_q_info.delta_q_present_flag
816
0
                  ? quant_params->base_qindex + x->rdmult_delta_qindex
817
0
                  : quant_params->base_qindex,
818
0
              0, QINDEX_RANGE - 1);
819
0
    qindex_rd = av1_get_qindex(&cm->seg, segment_id, current_rd_qindex);
820
732k
  } else {
821
732k
    if (cpi->cb_delta_rdmult_enabled) {
822
601k
      const int current_rd_qindex =
823
601k
          clamp(quant_params->base_qindex + x->rdmult_delta_qindex, 0,
824
601k
                QINDEX_RANGE - 1);
825
601k
      qindex_rd = av1_get_qindex(&cm->seg, segment_id, current_rd_qindex);
826
601k
    } else {
827
131k
      qindex_rd = qindex;
828
131k
    }
829
732k
  }
830
831
732k
  const int qindex_rdmult = qindex_rd + quant_params->y_dc_delta_q;
832
732k
  const int rdmult = av1_compute_rd_mult(
833
732k
      qindex_rdmult, cm->seq_params->bit_depth,
834
732k
      cpi->ppi->gf_group.update_type[cpi->gf_frame_index], layer_depth,
835
732k
      boost_index, frame_type, cpi->oxcf.q_cfg.use_fixed_qp_offsets,
836
732k
      is_stat_consumption_stage(cpi), cpi->oxcf.tune_cfg.tuning,
837
732k
      cpi->oxcf.mode);
838
839
732k
  const int qindex_change = x->qindex != qindex;
840
732k
  if (qindex_change || do_update) {
841
362k
    set_q_index(&cpi->enc_quant_dequant_params, qindex, x);
842
362k
  }
843
844
732k
  MACROBLOCKD *const xd = &x->e_mbd;
845
732k
  if ((segment_id != x->prev_segment_id) ||
846
729k
      av1_use_qmatrix(quant_params, xd, segment_id)) {
847
729k
    set_qmatrix(quant_params, segment_id, xd);
848
729k
  }
849
850
732k
  x->seg_skip_block = segfeature_active(&cm->seg, segment_id, SEG_LVL_SKIP);
851
852
732k
  av1_set_error_per_bit(&x->errorperbit, rdmult);
853
732k
  av1_set_sad_per_bit(cpi, &x->sadperbit, qindex_rd);
854
855
732k
  x->prev_segment_id = segment_id;
856
732k
}
857
858
276k
void av1_frame_init_quantizer(AV1_COMP *cpi) {
859
276k
  MACROBLOCK *const x = &cpi->td.mb;
860
276k
  MACROBLOCKD *const xd = &x->e_mbd;
861
276k
  x->prev_segment_id = -1;
862
276k
  av1_init_plane_quantizers(cpi, x, xd->mi[0]->segment_id, 1);
863
276k
}
864
865
0
static int adjust_hdr_cb_deltaq(int base_qindex) {
866
0
  double baseQp = base_qindex / QP_SCALE_FACTOR;
867
0
  const double chromaQp = CHROMA_QP_SCALE * baseQp + CHROMA_QP_OFFSET;
868
0
  const double dcbQP = CHROMA_CB_QP_SCALE * chromaQp * QP_SCALE_FACTOR;
869
0
  int dqpCb = (int)(dcbQP + (dcbQP < 0 ? -0.5 : 0.5));
870
0
  dqpCb = AOMMIN(0, dqpCb);
871
0
  dqpCb = (int)CLIP(dqpCb, -12 * QP_SCALE_FACTOR, 12 * QP_SCALE_FACTOR);
872
0
  return dqpCb;
873
0
}
874
875
0
static int adjust_hdr_cr_deltaq(int base_qindex) {
876
0
  double baseQp = base_qindex / QP_SCALE_FACTOR;
877
0
  const double chromaQp = CHROMA_QP_SCALE * baseQp + CHROMA_QP_OFFSET;
878
0
  const double dcrQP = CHROMA_CR_QP_SCALE * chromaQp * QP_SCALE_FACTOR;
879
0
  int dqpCr = (int)(dcrQP + (dcrQP < 0 ? -0.5 : 0.5));
880
0
  dqpCr = AOMMIN(0, dqpCr);
881
0
  dqpCr = (int)CLIP(dqpCr, -12 * QP_SCALE_FACTOR, 12 * QP_SCALE_FACTOR);
882
0
  return dqpCr;
883
0
}
884
885
void av1_set_quantizer(AV1_COMMON *const cm, int min_qmlevel, int max_qmlevel,
886
                       int q, int enable_chroma_deltaq, int enable_hdr_deltaq,
887
262k
                       bool is_allintra, aom_tune_metric tuning) {
888
  // quantizer has to be reinitialized with av1_init_quantizer() if any
889
  // delta_q changes.
890
262k
  CommonQuantParams *quant_params = &cm->quant_params;
891
262k
  quant_params->base_qindex = AOMMAX(cm->delta_q_info.delta_q_present_flag, q);
892
262k
  quant_params->y_dc_delta_q = 0;
893
894
  // Disable deltaq in lossless mode.
895
262k
  if (enable_chroma_deltaq && q) {
896
144k
    if (tuning == AOM_TUNE_IQ || tuning == AOM_TUNE_SSIMULACRA2) {
897
144k
      int chroma_dc_delta_q = 0;
898
144k
      int chroma_ac_delta_q = 0;
899
900
144k
      if (cm->seq_params->subsampling_x == 1 &&
901
124k
          cm->seq_params->subsampling_y == 1) {
902
        // 4:2:0 subsampling: Constant chroma delta_q decrease (i.e. improved
903
        // chroma quality relative to luma) with gradual ramp-down for very low
904
        // qindexes.
905
        // Lowering chroma delta_q by 16 was found to improve SSIMULACRA 2
906
        // BD-Rate by 1.5-2% on Daala's subset1, as well as reducing chroma
907
        // artifacts (smudging, discoloration) during subjective quality
908
        // evaluations.
909
        // The ramp-down of chroma increase was determined by generating the
910
        // convex hull of SSIMULACRA 2 scores (for all boosts from 0-16), and
911
        // finding a linear equation that fits the convex hull.
912
        // For the SSIMULACRA 2 tune only: a convex hull of SSIMULACRA 2 scores
913
        // (mean, 10th percentile) was generated by decreasing chroma delta_q
914
        // by 3 different values (16, 20 and 24). 20 was selected as it had the
915
        // best 10th percentile performance.
916
102k
        int offset = (tuning == AOM_TUNE_SSIMULACRA2) ? 20 : 16;
917
102k
        chroma_dc_delta_q =
918
102k
            -clamp((quant_params->base_qindex / 2) - 14, 0, offset);
919
102k
        chroma_ac_delta_q = chroma_dc_delta_q;
920
102k
      } else if (cm->seq_params->subsampling_x == 1 &&
921
21.5k
                 cm->seq_params->subsampling_y == 0) {
922
        // 4:2:2 subsampling: Constant chroma AC delta_q increase (i.e. improved
923
        // luma quality relative to chroma) with gradual ramp-down for very low
924
        // qindexes.
925
        // SSIMULACRA 2 appears to have some issues correctly scoring 4:2:2
926
        // material. Solely optimizing for maximum scores suggests a chroma AC
927
        // delta_q of 12 is the most efficient. However, visual inspection on
928
        // difficult-to-encode material resulted in chroma quality degrading too
929
        // much relative to luma, and chroma channels ending up being too small
930
        // compared to equivalent 4:4:4 or 4:2:0 encodes.
931
        // A chroma AC delta_q of 6 was selected because encoded chroma channels
932
        // have a much closer size to 4:4:4 and 4:2:0 encodes, and have more
933
        // favorable visual quality characteristics.
934
        // The ramp-down of chroma decrease was put into place to match 4:2:0
935
        // and 4:4:4 behavior. There were no special considerations on
936
        // SSIMULACRA 2 scores.
937
21.5k
        chroma_dc_delta_q = 0;
938
21.5k
        chroma_ac_delta_q = clamp((quant_params->base_qindex / 2), 0, 6);
939
21.5k
      } else if (cm->seq_params->subsampling_x == 0 &&
940
20.1k
                 cm->seq_params->subsampling_y == 0) {
941
        // 4:4:4 subsampling: Constant chroma AC delta_q increase (i.e. improved
942
        // luma quality relative to chroma) with gradual ramp-down for very low
943
        // qindexes.
944
        // Raising chroma AC delta_q by 24 was found to improve SSIMULACRA 2
945
        // BD-Rate by 2.5-3% on Daala's subset1, as well as providing a more
946
        // balanced bit allocation between the (relatively-starved) luma and
947
        // chroma channels.
948
        // Raising chroma DC delta_q appears to be harmful, both for SSIMULACRA
949
        // 2 scores and subjective quality (harshens blocking artifacts).
950
        // The ramp-down of chroma decrease was put into place so (lossy) QP 0
951
        // encodes still score within 0.1 SSIMULACRA 2 points of the equivalent
952
        // with no chroma delta_q (with a small efficiency improvement), while
953
        // encodes in the SSIMULACRA 2 <=90 range yield full benefits from this
954
        // adjustment.
955
20.1k
        chroma_dc_delta_q = 0;
956
20.1k
        chroma_ac_delta_q = clamp((quant_params->base_qindex / 2), 0, 24);
957
20.1k
      }
958
959
      // TODO: bug https://crbug.com/aomedia/375221136 - find chroma_delta_q
960
      // values for 4:2:2 subsampling mode.
961
144k
      quant_params->u_dc_delta_q = chroma_dc_delta_q;
962
144k
      quant_params->u_ac_delta_q = chroma_ac_delta_q;
963
144k
      quant_params->v_dc_delta_q = chroma_dc_delta_q;
964
144k
      quant_params->v_ac_delta_q = chroma_ac_delta_q;
965
144k
    } else {
966
      // TODO(aomedia:2717): need to design better delta
967
0
      quant_params->u_dc_delta_q = 2;
968
0
      quant_params->u_ac_delta_q = 2;
969
0
      quant_params->v_dc_delta_q = 2;
970
0
      quant_params->v_ac_delta_q = 2;
971
0
    }
972
144k
  } else {
973
118k
    quant_params->u_dc_delta_q = 0;
974
118k
    quant_params->u_ac_delta_q = 0;
975
118k
    quant_params->v_dc_delta_q = 0;
976
118k
    quant_params->v_ac_delta_q = 0;
977
118k
  }
978
979
  // following section 8.3.2 in T-REC-H.Sup15 document
980
  // to apply to AV1 qindex in the range of [0, 255]
981
262k
  if (enable_hdr_deltaq && q) {
982
0
    int dqpCb = adjust_hdr_cb_deltaq(quant_params->base_qindex);
983
0
    int dqpCr = adjust_hdr_cr_deltaq(quant_params->base_qindex);
984
0
    quant_params->u_dc_delta_q = quant_params->u_ac_delta_q = dqpCb;
985
0
    quant_params->v_dc_delta_q = quant_params->v_ac_delta_q = dqpCr;
986
0
    if (dqpCb != dqpCr) {
987
0
      cm->seq_params->separate_uv_delta_q = 1;
988
0
    }
989
0
  }
990
991
  // Select the best luma and chroma QM formulas based on encoding mode and
992
  // tuning
993
262k
  int (*get_luma_qmlevel)(int, int, int);
994
262k
  int (*get_chroma_qmlevel)(int, int, int);
995
996
262k
  if (tuning == AOM_TUNE_IQ || tuning == AOM_TUNE_SSIMULACRA2) {
997
145k
    if (tuning == AOM_TUNE_SSIMULACRA2) {
998
      // Use luma QM formula specifically tailored for tune SSIMULACRA2
999
0
      get_luma_qmlevel = aom_get_qmlevel_luma_ssimulacra2;
1000
145k
    } else {
1001
145k
      get_luma_qmlevel = aom_get_qmlevel_allintra;
1002
145k
    }
1003
1004
145k
    if (cm->seq_params->subsampling_x == 0 &&
1005
20.1k
        cm->seq_params->subsampling_y == 0) {
1006
      // 4:4:4 subsampling mode has 4x the number of chroma coefficients
1007
      // compared to 4:2:0 (2x on each dimension). This means the encoder
1008
      // should use lower chroma QM levels that more closely match the scaling
1009
      // of an equivalent 4:2:0 chroma QM.
1010
20.1k
      get_chroma_qmlevel = aom_get_qmlevel_444_chroma;
1011
124k
    } else {
1012
      // For all other chroma subsampling modes, use the all intra QM formula
1013
124k
      get_chroma_qmlevel = aom_get_qmlevel_allintra;
1014
124k
    }
1015
145k
  } else if (is_allintra) {
1016
18.4k
    get_luma_qmlevel = aom_get_qmlevel_allintra;
1017
18.4k
    get_chroma_qmlevel = aom_get_qmlevel_allintra;
1018
99.2k
  } else {
1019
99.2k
    get_luma_qmlevel = aom_get_qmlevel;
1020
99.2k
    get_chroma_qmlevel = aom_get_qmlevel;
1021
99.2k
  }
1022
1023
262k
  quant_params->qmatrix_level_y =
1024
262k
      get_luma_qmlevel(quant_params->base_qindex, min_qmlevel, max_qmlevel);
1025
262k
  quant_params->qmatrix_level_u =
1026
262k
      get_chroma_qmlevel(quant_params->base_qindex + quant_params->u_ac_delta_q,
1027
262k
                         min_qmlevel, max_qmlevel);
1028
1029
262k
  if (cm->seq_params->separate_uv_delta_q) {
1030
0
    quant_params->qmatrix_level_v = get_chroma_qmlevel(
1031
0
        quant_params->base_qindex + quant_params->v_ac_delta_q, min_qmlevel,
1032
0
        max_qmlevel);
1033
262k
  } else {
1034
262k
    quant_params->qmatrix_level_v = quant_params->qmatrix_level_u;
1035
262k
  }
1036
262k
}
1037
1038
// Table that converts 0-63 Q-range values passed in outside to the Qindex
1039
// range used internally.
1040
static const int quantizer_to_qindex[] = {
1041
  0,   4,   8,   12,  16,  20,  24,  28,  32,  36,  40,  44,  48,
1042
  52,  56,  60,  64,  68,  72,  76,  80,  84,  88,  92,  96,  100,
1043
  104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144, 148, 152,
1044
  156, 160, 164, 168, 172, 176, 180, 184, 188, 192, 196, 200, 204,
1045
  208, 212, 216, 220, 224, 228, 232, 236, 240, 244, 249, 255,
1046
};
1047
1048
4.45M
int av1_quantizer_to_qindex(int quantizer) {
1049
4.45M
  return quantizer_to_qindex[quantizer];
1050
4.45M
}
1051
1052
0
int av1_qindex_to_quantizer(int qindex) {
1053
0
  int quantizer;
1054
1055
0
  for (quantizer = 0; quantizer < 64; ++quantizer)
1056
0
    if (quantizer_to_qindex[quantizer] >= qindex) return quantizer;
1057
1058
0
  return 63;
1059
0
}