Coverage Report

Created: 2022-08-24 06:17

/src/aom/av1/encoder/model_rd.h
Line
Count
Source (jump to first uncovered line)
1
/*
2
 * Copyright (c) 2020, Alliance for Open Media. All rights reserved
3
 *
4
 * This source code is subject to the terms of the BSD 2 Clause License and
5
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6
 * was not distributed with this source code in the LICENSE file, you can
7
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8
 * Media Patent License 1.0 was not distributed with this source code in the
9
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10
 */
11
12
#ifndef AOM_AV1_ENCODER_MODEL_RD_H_
13
#define AOM_AV1_ENCODER_MODEL_RD_H_
14
15
#include "aom/aom_integer.h"
16
#include "av1/encoder/block.h"
17
#include "av1/encoder/encoder.h"
18
#include "av1/encoder/pustats.h"
19
#include "av1/encoder/rdopt_utils.h"
20
#include "config/aom_dsp_rtcd.h"
21
22
#ifdef __cplusplus
23
extern "C" {
24
#endif
25
26
// 0: Legacy model
27
// 1: Curve fit model
28
// 2: Surface fit model
29
// 3: DNN regression model
30
// 4: Full rd model
31
0
#define MODELRD_TYPE_INTERP_FILTER 1
32
0
#define MODELRD_TYPE_TX_SEARCH_PRUNE 1
33
0
#define MODELRD_TYPE_MASKED_COMPOUND 1
34
0
#define MODELRD_TYPE_INTERINTRA 1
35
#define MODELRD_TYPE_INTRA 1
36
0
#define MODELRD_TYPE_MOTION_MODE_RD 1
37
38
typedef void (*model_rd_for_sb_type)(const AV1_COMP *const cpi,
39
                                     BLOCK_SIZE bsize, MACROBLOCK *x,
40
                                     MACROBLOCKD *xd, int plane_from,
41
                                     int plane_to, int *out_rate_sum,
42
                                     int64_t *out_dist_sum, int *skip_txfm_sb,
43
                                     int64_t *skip_sse_sb, int *plane_rate,
44
                                     int64_t *plane_sse, int64_t *plane_dist);
45
typedef void (*model_rd_from_sse_type)(const AV1_COMP *const cpi,
46
                                       const MACROBLOCK *const x,
47
                                       BLOCK_SIZE plane_bsize, int plane,
48
                                       int64_t sse, int num_samples, int *rate,
49
                                       int64_t *dist);
50
51
static int64_t calculate_sse(MACROBLOCKD *const xd,
52
                             const struct macroblock_plane *p,
53
                             struct macroblockd_plane *pd, const int bw,
54
0
                             const int bh) {
55
0
  int64_t sse = 0;
56
0
  const int shift = xd->bd - 8;
57
0
#if CONFIG_AV1_HIGHBITDEPTH
58
0
  if (is_cur_buf_hbd(xd)) {
59
0
    sse = aom_highbd_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride,
60
0
                         bw, bh);
61
0
  } else {
62
0
    sse =
63
0
        aom_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, bw, bh);
64
0
  }
65
#else
66
  sse = aom_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, bw, bh);
67
#endif
68
0
  sse = ROUND_POWER_OF_TWO(sse, shift * 2);
69
0
  return sse;
70
0
}
Unexecuted instantiation: encoder.c:calculate_sse
Unexecuted instantiation: encoder_utils.c:calculate_sse
Unexecuted instantiation: ethread.c:calculate_sse
Unexecuted instantiation: palette.c:calculate_sse
Unexecuted instantiation: rdopt.c:calculate_sse
Unexecuted instantiation: superres_scale.c:calculate_sse
Unexecuted instantiation: svc_layercontext.c:calculate_sse
Unexecuted instantiation: tx_search.c:calculate_sse
Unexecuted instantiation: intra_mode_search.c:calculate_sse
Unexecuted instantiation: allintra_vis.c:calculate_sse
Unexecuted instantiation: compound_type.c:calculate_sse
Unexecuted instantiation: encodeframe.c:calculate_sse
Unexecuted instantiation: encode_strategy.c:calculate_sse
Unexecuted instantiation: interp_search.c:calculate_sse
Unexecuted instantiation: nonrd_pickmode.c:calculate_sse
71
72
static AOM_INLINE int64_t compute_sse_plane(MACROBLOCK *x, MACROBLOCKD *xd,
73
0
                                            int plane, const BLOCK_SIZE bsize) {
74
0
  struct macroblockd_plane *const pd = &xd->plane[plane];
75
0
  const BLOCK_SIZE plane_bsize =
76
0
      get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y);
77
0
  int bw, bh;
78
0
  const struct macroblock_plane *const p = &x->plane[plane];
79
0
  get_txb_dimensions(xd, plane, plane_bsize, 0, 0, plane_bsize, NULL, NULL, &bw,
80
0
                     &bh);
81
82
0
  int64_t sse = calculate_sse(xd, p, pd, bw, bh);
83
84
0
  return sse;
85
0
}
Unexecuted instantiation: encoder.c:compute_sse_plane
Unexecuted instantiation: encoder_utils.c:compute_sse_plane
Unexecuted instantiation: ethread.c:compute_sse_plane
Unexecuted instantiation: palette.c:compute_sse_plane
Unexecuted instantiation: rdopt.c:compute_sse_plane
Unexecuted instantiation: superres_scale.c:compute_sse_plane
Unexecuted instantiation: svc_layercontext.c:compute_sse_plane
Unexecuted instantiation: tx_search.c:compute_sse_plane
Unexecuted instantiation: intra_mode_search.c:compute_sse_plane
Unexecuted instantiation: allintra_vis.c:compute_sse_plane
Unexecuted instantiation: compound_type.c:compute_sse_plane
Unexecuted instantiation: encodeframe.c:compute_sse_plane
Unexecuted instantiation: encode_strategy.c:compute_sse_plane
Unexecuted instantiation: interp_search.c:compute_sse_plane
Unexecuted instantiation: nonrd_pickmode.c:compute_sse_plane
86
87
static AOM_INLINE void model_rd_from_sse(const AV1_COMP *const cpi,
88
                                         const MACROBLOCK *const x,
89
                                         BLOCK_SIZE plane_bsize, int plane,
90
                                         int64_t sse, int num_samples,
91
0
                                         int *rate, int64_t *dist) {
92
0
  (void)num_samples;
93
0
  const MACROBLOCKD *const xd = &x->e_mbd;
94
0
  const struct macroblock_plane *const p = &x->plane[plane];
95
0
  const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3;
96
97
  // Fast approximate the modelling function.
98
0
  if (cpi->sf.rd_sf.simple_model_rd_from_var) {
99
0
    const int64_t square_error = sse;
100
0
    int quantizer = p->dequant_QTX[1] >> dequant_shift;
101
0
    if (quantizer < 120)
102
0
      *rate = (int)AOMMIN(
103
0
          (square_error * (280 - quantizer)) >> (16 - AV1_PROB_COST_SHIFT),
104
0
          INT_MAX);
105
0
    else
106
0
      *rate = 0;
107
0
    assert(*rate >= 0);
108
0
    *dist = (square_error * quantizer) >> 8;
109
0
  } else {
110
0
    av1_model_rd_from_var_lapndz(sse, num_pels_log2_lookup[plane_bsize],
111
0
                                 p->dequant_QTX[1] >> dequant_shift, rate,
112
0
                                 dist);
113
0
  }
114
0
  *dist <<= 4;
115
0
}
Unexecuted instantiation: encoder.c:model_rd_from_sse
Unexecuted instantiation: encoder_utils.c:model_rd_from_sse
Unexecuted instantiation: ethread.c:model_rd_from_sse
Unexecuted instantiation: palette.c:model_rd_from_sse
Unexecuted instantiation: rdopt.c:model_rd_from_sse
Unexecuted instantiation: superres_scale.c:model_rd_from_sse
Unexecuted instantiation: svc_layercontext.c:model_rd_from_sse
Unexecuted instantiation: tx_search.c:model_rd_from_sse
Unexecuted instantiation: intra_mode_search.c:model_rd_from_sse
Unexecuted instantiation: allintra_vis.c:model_rd_from_sse
Unexecuted instantiation: compound_type.c:model_rd_from_sse
Unexecuted instantiation: encodeframe.c:model_rd_from_sse
Unexecuted instantiation: encode_strategy.c:model_rd_from_sse
Unexecuted instantiation: interp_search.c:model_rd_from_sse
Unexecuted instantiation: nonrd_pickmode.c:model_rd_from_sse
116
117
// Fits a curve for rate and distortion using as feature:
118
// log2(sse_norm/qstep^2)
119
static AOM_INLINE void model_rd_with_curvfit(const AV1_COMP *const cpi,
120
                                             const MACROBLOCK *const x,
121
                                             BLOCK_SIZE plane_bsize, int plane,
122
                                             int64_t sse, int num_samples,
123
0
                                             int *rate, int64_t *dist) {
124
0
  (void)cpi;
125
0
  (void)plane_bsize;
126
0
  const MACROBLOCKD *const xd = &x->e_mbd;
127
0
  const struct macroblock_plane *const p = &x->plane[plane];
128
0
  const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3;
129
0
  const int qstep = AOMMAX(p->dequant_QTX[1] >> dequant_shift, 1);
130
131
0
  if (sse == 0) {
132
0
    if (rate) *rate = 0;
133
0
    if (dist) *dist = 0;
134
0
    return;
135
0
  }
136
0
  const double sse_norm = (double)sse / num_samples;
137
0
  const double qstepsqr = (double)qstep * qstep;
138
0
  const double xqr = log2(sse_norm / qstepsqr);
139
0
  double rate_f, dist_by_sse_norm_f;
140
0
  av1_model_rd_curvfit(plane_bsize, sse_norm, xqr, &rate_f,
141
0
                       &dist_by_sse_norm_f);
142
143
0
  const double dist_f = dist_by_sse_norm_f * sse_norm;
144
0
  int rate_i = (int)(AOMMAX(0.0, rate_f * num_samples) + 0.5);
145
0
  int64_t dist_i = (int64_t)(AOMMAX(0.0, dist_f * num_samples) + 0.5);
146
147
  // Check if skip is better
148
0
  if (rate_i == 0) {
149
0
    dist_i = sse << 4;
150
0
  } else if (RDCOST(x->rdmult, rate_i, dist_i) >=
151
0
             RDCOST(x->rdmult, 0, sse << 4)) {
152
0
    rate_i = 0;
153
0
    dist_i = sse << 4;
154
0
  }
155
156
0
  if (rate) *rate = rate_i;
157
0
  if (dist) *dist = dist_i;
158
0
}
Unexecuted instantiation: encoder.c:model_rd_with_curvfit
Unexecuted instantiation: encoder_utils.c:model_rd_with_curvfit
Unexecuted instantiation: ethread.c:model_rd_with_curvfit
Unexecuted instantiation: palette.c:model_rd_with_curvfit
Unexecuted instantiation: rdopt.c:model_rd_with_curvfit
Unexecuted instantiation: superres_scale.c:model_rd_with_curvfit
Unexecuted instantiation: svc_layercontext.c:model_rd_with_curvfit
Unexecuted instantiation: tx_search.c:model_rd_with_curvfit
Unexecuted instantiation: intra_mode_search.c:model_rd_with_curvfit
Unexecuted instantiation: allintra_vis.c:model_rd_with_curvfit
Unexecuted instantiation: compound_type.c:model_rd_with_curvfit
Unexecuted instantiation: encodeframe.c:model_rd_with_curvfit
Unexecuted instantiation: encode_strategy.c:model_rd_with_curvfit
Unexecuted instantiation: interp_search.c:model_rd_with_curvfit
Unexecuted instantiation: nonrd_pickmode.c:model_rd_with_curvfit
159
160
static AOM_INLINE void model_rd_for_sb(
161
    const AV1_COMP *const cpi, BLOCK_SIZE bsize, MACROBLOCK *x, MACROBLOCKD *xd,
162
    int plane_from, int plane_to, int *out_rate_sum, int64_t *out_dist_sum,
163
    int *skip_txfm_sb, int64_t *skip_sse_sb, int *plane_rate,
164
0
    int64_t *plane_sse, int64_t *plane_dist) {
165
  // Note our transform coeffs are 8 times an orthogonal transform.
166
  // Hence quantizer step is also 8 times. To get effective quantizer
167
  // we need to divide by 8 before sending to modeling function.
168
0
  int plane;
169
0
  const int ref = xd->mi[0]->ref_frame[0];
170
171
0
  int64_t rate_sum = 0;
172
0
  int64_t dist_sum = 0;
173
0
  int64_t total_sse = 0;
174
175
0
  assert(bsize < BLOCK_SIZES_ALL);
176
177
0
  for (plane = plane_from; plane <= plane_to; ++plane) {
178
0
    if (plane && !xd->is_chroma_ref) break;
179
0
    struct macroblock_plane *const p = &x->plane[plane];
180
0
    struct macroblockd_plane *const pd = &xd->plane[plane];
181
0
    const BLOCK_SIZE plane_bsize =
182
0
        get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y);
183
0
    assert(plane_bsize < BLOCK_SIZES_ALL);
184
0
    const int bw = block_size_wide[plane_bsize];
185
0
    const int bh = block_size_high[plane_bsize];
186
0
    int64_t sse;
187
0
    int rate;
188
0
    int64_t dist;
189
190
0
    sse = calculate_sse(xd, p, pd, bw, bh);
191
192
0
    model_rd_from_sse(cpi, x, plane_bsize, plane, sse, bw * bh, &rate, &dist);
193
194
0
    if (plane == 0) x->pred_sse[ref] = (unsigned int)AOMMIN(sse, UINT_MAX);
195
196
0
    total_sse += sse;
197
0
    rate_sum += rate;
198
0
    dist_sum += dist;
199
0
    if (plane_rate) plane_rate[plane] = rate;
200
0
    if (plane_sse) plane_sse[plane] = sse;
201
0
    if (plane_dist) plane_dist[plane] = dist;
202
0
    assert(rate_sum >= 0);
203
0
  }
204
205
0
  if (skip_txfm_sb) *skip_txfm_sb = total_sse == 0;
206
0
  if (skip_sse_sb) *skip_sse_sb = total_sse << 4;
207
0
  rate_sum = AOMMIN(rate_sum, INT_MAX);
208
0
  *out_rate_sum = (int)rate_sum;
209
0
  *out_dist_sum = dist_sum;
210
0
}
Unexecuted instantiation: encoder.c:model_rd_for_sb
Unexecuted instantiation: encoder_utils.c:model_rd_for_sb
Unexecuted instantiation: ethread.c:model_rd_for_sb
Unexecuted instantiation: palette.c:model_rd_for_sb
Unexecuted instantiation: rdopt.c:model_rd_for_sb
Unexecuted instantiation: superres_scale.c:model_rd_for_sb
Unexecuted instantiation: svc_layercontext.c:model_rd_for_sb
Unexecuted instantiation: tx_search.c:model_rd_for_sb
Unexecuted instantiation: intra_mode_search.c:model_rd_for_sb
Unexecuted instantiation: allintra_vis.c:model_rd_for_sb
Unexecuted instantiation: compound_type.c:model_rd_for_sb
Unexecuted instantiation: encodeframe.c:model_rd_for_sb
Unexecuted instantiation: encode_strategy.c:model_rd_for_sb
Unexecuted instantiation: interp_search.c:model_rd_for_sb
Unexecuted instantiation: nonrd_pickmode.c:model_rd_for_sb
211
212
static AOM_INLINE void model_rd_for_sb_with_curvfit(
213
    const AV1_COMP *const cpi, BLOCK_SIZE bsize, MACROBLOCK *x, MACROBLOCKD *xd,
214
    int plane_from, int plane_to, int *out_rate_sum, int64_t *out_dist_sum,
215
    int *skip_txfm_sb, int64_t *skip_sse_sb, int *plane_rate,
216
0
    int64_t *plane_sse, int64_t *plane_dist) {
217
  // Note our transform coeffs are 8 times an orthogonal transform.
218
  // Hence quantizer step is also 8 times. To get effective quantizer
219
  // we need to divide by 8 before sending to modeling function.
220
0
  const int ref = xd->mi[0]->ref_frame[0];
221
222
0
  int64_t rate_sum = 0;
223
0
  int64_t dist_sum = 0;
224
0
  int64_t total_sse = 0;
225
226
0
  for (int plane = plane_from; plane <= plane_to; ++plane) {
227
0
    if (plane && !xd->is_chroma_ref) break;
228
0
    struct macroblockd_plane *const pd = &xd->plane[plane];
229
0
    const BLOCK_SIZE plane_bsize =
230
0
        get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y);
231
0
    int64_t dist, sse;
232
0
    int rate;
233
0
    int bw, bh;
234
0
    const struct macroblock_plane *const p = &x->plane[plane];
235
0
    get_txb_dimensions(xd, plane, plane_bsize, 0, 0, plane_bsize, NULL, NULL,
236
0
                       &bw, &bh);
237
238
0
    sse = calculate_sse(xd, p, pd, bw, bh);
239
0
    model_rd_with_curvfit(cpi, x, plane_bsize, plane, sse, bw * bh, &rate,
240
0
                          &dist);
241
242
0
    if (plane == 0) x->pred_sse[ref] = (unsigned int)AOMMIN(sse, UINT_MAX);
243
244
0
    total_sse += sse;
245
0
    rate_sum += rate;
246
0
    dist_sum += dist;
247
248
0
    if (plane_rate) plane_rate[plane] = rate;
249
0
    if (plane_sse) plane_sse[plane] = sse;
250
0
    if (plane_dist) plane_dist[plane] = dist;
251
0
  }
252
253
0
  if (skip_txfm_sb) *skip_txfm_sb = rate_sum == 0;
254
0
  if (skip_sse_sb) *skip_sse_sb = total_sse << 4;
255
0
  *out_rate_sum = (int)rate_sum;
256
0
  *out_dist_sum = dist_sum;
257
0
}
Unexecuted instantiation: encoder.c:model_rd_for_sb_with_curvfit
Unexecuted instantiation: encoder_utils.c:model_rd_for_sb_with_curvfit
Unexecuted instantiation: ethread.c:model_rd_for_sb_with_curvfit
Unexecuted instantiation: palette.c:model_rd_for_sb_with_curvfit
Unexecuted instantiation: rdopt.c:model_rd_for_sb_with_curvfit
Unexecuted instantiation: superres_scale.c:model_rd_for_sb_with_curvfit
Unexecuted instantiation: svc_layercontext.c:model_rd_for_sb_with_curvfit
Unexecuted instantiation: tx_search.c:model_rd_for_sb_with_curvfit
Unexecuted instantiation: intra_mode_search.c:model_rd_for_sb_with_curvfit
Unexecuted instantiation: allintra_vis.c:model_rd_for_sb_with_curvfit
Unexecuted instantiation: compound_type.c:model_rd_for_sb_with_curvfit
Unexecuted instantiation: encodeframe.c:model_rd_for_sb_with_curvfit
Unexecuted instantiation: encode_strategy.c:model_rd_for_sb_with_curvfit
Unexecuted instantiation: interp_search.c:model_rd_for_sb_with_curvfit
Unexecuted instantiation: nonrd_pickmode.c:model_rd_for_sb_with_curvfit
258
259
enum { MODELRD_LEGACY, MODELRD_CURVFIT, MODELRD_TYPES } UENUM1BYTE(ModelRdType);
260
261
static const model_rd_for_sb_type model_rd_sb_fn[MODELRD_TYPES] = {
262
  model_rd_for_sb, model_rd_for_sb_with_curvfit
263
};
264
265
static const model_rd_from_sse_type model_rd_sse_fn[MODELRD_TYPES] = {
266
  model_rd_from_sse, model_rd_with_curvfit
267
};
268
269
#ifdef __cplusplus
270
}  // extern "C"
271
#endif
272
#endif  // AOM_AV1_ENCODER_MODEL_RD_H_