/src/aom/av1/encoder/model_rd.h

Source
/*
 * Copyright (c) 2020, Alliance for Open Media. All rights reserved.
 *
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
 */

#ifndef AOM_AV1_ENCODER_MODEL_RD_H_
#define AOM_AV1_ENCODER_MODEL_RD_H_

#include "aom/aom_integer.h"
#include "av1/encoder/block.h"
#include "av1/encoder/encoder.h"
#include "av1/encoder/pustats.h"
#include "av1/encoder/rdopt_utils.h"
#include "config/aom_dsp_rtcd.h"

#ifdef __cplusplus
extern "C" {
#endif

// 0: Legacy model
// 1: Curve fit model
// 2: Surface fit model
// 3: DNN regression model
// 4: Full rd model
#define MODELRD_TYPE_INTERP_FILTER 1
#define MODELRD_TYPE_TX_SEARCH_PRUNE 1
#define MODELRD_TYPE_MASKED_COMPOUND 1
#define MODELRD_TYPE_INTERINTRA 1
#define MODELRD_TYPE_INTRA 1
#define MODELRD_TYPE_MOTION_MODE_RD 1

typedef void (*model_rd_for_sb_type)(
    const AV1_COMP *const cpi, BLOCK_SIZE bsize, MACROBLOCK *x, MACROBLOCKD *xd,
    int plane_from, int plane_to, int *out_rate_sum, int64_t *out_dist_sum,
    uint8_t *skip_txfm_sb, int64_t *skip_sse_sb, int *plane_rate,
    int64_t *plane_sse, int64_t *plane_dist);
typedef void (*model_rd_from_sse_type)(const AV1_COMP *const cpi,
                                       const MACROBLOCK *const x,
                                       BLOCK_SIZE plane_bsize, int plane,
                                       int64_t sse, int num_samples, int *rate,
                                       int64_t *dist);

static int64_t calculate_sse(MACROBLOCKD *const xd,
                             const struct macroblock_plane *p,
                             struct macroblockd_plane *pd, const int bw,
                             const int bh) {
  int64_t sse = 0;
  const int shift = xd->bd - 8;
#if CONFIG_AV1_HIGHBITDEPTH
  if (is_cur_buf_hbd(xd)) {
    sse = aom_highbd_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride,
                         bw, bh);
  } else {
    sse =
        aom_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, bw, bh);
  }
#else
  sse = aom_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, bw, bh);
#endif
  sse = ROUND_POWER_OF_TWO(sse, shift * 2);
  return sse;
}

static unsigned pixel_dist_visible_only(
    const AV1_COMP *const cpi, const MACROBLOCK *x, const uint8_t *src,
    const int src_stride, const uint8_t *dst, const int dst_stride,
    const BLOCK_SIZE tx_bsize, int txb_rows, int txb_cols, int visible_rows,
    int visible_cols) {
  if (visible_rows == 0 || visible_cols == 0) return 0;

  unsigned sse;
  if (txb_rows == visible_rows && txb_cols == visible_cols) {
    cpi->ppi->fn_ptr[tx_bsize].vf(src, src_stride, dst, dst_stride, &sse);
    return sse;
  }

#if CONFIG_AV1_HIGHBITDEPTH
  const MACROBLOCKD *xd = &x->e_mbd;
  if (is_cur_buf_hbd(xd)) {
    uint64_t sse64;
    if (!(visible_rows % 4) && !(visible_cols % 4)) {
      sse64 = aom_highbd_sse(src, src_stride, dst, dst_stride, visible_cols,
                             visible_rows);
    } else {
      sse64 = aom_highbd_sse_odd_size(src, src_stride, dst, dst_stride,
                                      visible_cols, visible_rows);
    }
    return (unsigned int)ROUND_POWER_OF_TWO(sse64, (xd->bd - 8) * 2);
  }
#else
  (void)x;
#endif
  if (!(visible_rows % 4) && !(visible_cols % 4)) {
    sse = (unsigned)aom_sse(src, src_stride, dst, dst_stride, visible_cols,
                            visible_rows);
  } else {
    sse = aom_sse_odd_size(src, src_stride, dst, dst_stride, visible_cols,
                           visible_rows);
  }
  return sse;
}

static inline int64_t compute_sse_plane(const AV1_COMP *cpi, MACROBLOCK *x,
                                        MACROBLOCKD *xd, int plane,
                                        const BLOCK_SIZE bsize) {
  struct macroblockd_plane *const pd = &xd->plane[plane];
  const BLOCK_SIZE plane_bsize =
      get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y);
  int bw, bh;
  const struct macroblock_plane *const p = &x->plane[plane];
  const int block_width = block_size_wide[plane_bsize];
  const int block_height = block_size_high[plane_bsize];

  get_visible_dimensions(x, plane, plane_bsize, 0, 0, block_width, block_height,
                         /*clip_dims=*/true, &bw, &bh);

  int64_t sse = pixel_dist_visible_only(
      cpi, x, p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride,
      plane_bsize, block_height, block_width, bh, bw);

  return sse;
}

static inline void model_rd_from_sse(const AV1_COMP *const cpi,
                                     const MACROBLOCK *const x,
                                     BLOCK_SIZE plane_bsize, int plane,
                                     int64_t sse, int num_samples, int *rate,
                                     int64_t *dist) {
  (void)num_samples;
  const MACROBLOCKD *const xd = &x->e_mbd;
  const struct macroblock_plane *const p = &x->plane[plane];
  const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3;

  // Fast approximate the modelling function.
  if (cpi->sf.rd_sf.simple_model_rd_from_var) {
    const int64_t square_error = sse;
    int quantizer = p->dequant_QTX[1] >> dequant_shift;
    if (quantizer < 120)
      *rate = (int)AOMMIN(
          (square_error * (280 - quantizer)) >> (16 - AV1_PROB_COST_SHIFT),
          INT_MAX);
    else
      *rate = 0;
    assert(*rate >= 0);
    *dist = (square_error * quantizer) >> 8;
  } else {
    av1_model_rd_from_var_lapndz(sse, num_pels_log2_lookup[plane_bsize],
                                 p->dequant_QTX[1] >> dequant_shift, rate,
                                 dist);
  }
  *dist <<= 4;
}

// Fits a curve for rate and distortion using as feature:
// log2(sse_norm/qstep^2)
static inline void model_rd_with_curvfit(const AV1_COMP *const cpi,
                                         const MACROBLOCK *const x,
                                         BLOCK_SIZE plane_bsize, int plane,
                                         int64_t sse, int num_samples,
                                         int *rate, int64_t *dist) {
  (void)cpi;
  (void)plane_bsize;
  const MACROBLOCKD *const xd = &x->e_mbd;
  const struct macroblock_plane *const p = &x->plane[plane];
  const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3;
  const int qstep = AOMMAX(p->dequant_QTX[1] >> dequant_shift, 1);

  if (sse == 0) {
    if (rate) *rate = 0;
    if (dist) *dist = 0;
    return;
  }
  const double sse_norm = (double)sse / num_samples;
  const double qstepsqr = (double)qstep * qstep;
  const double xqr = log2(sse_norm / qstepsqr);
  double rate_dist_f[2];
  av1_model_rd_curvfit(plane_bsize, sse_norm, xqr, rate_dist_f);

  const double dist_f = rate_dist_f[1] * sse_norm;
  int rate_i = (int)(AOMMAX(0.0, rate_dist_f[0] * num_samples) + 0.5);
  int64_t dist_i = (int64_t)(AOMMAX(0.0, dist_f * num_samples) + 0.5);

  // Check if skip is better
  if (rate_i == 0) {
    dist_i = sse << 4;
  } else if (RDCOST(x->rdmult, rate_i, dist_i) >=
             RDCOST(x->rdmult, 0, sse << 4)) {
    rate_i = 0;
    dist_i = sse << 4;
  }

  if (rate) *rate = rate_i;
  if (dist) *dist = dist_i;
}

static inline void model_rd_for_sb(const AV1_COMP *const cpi, BLOCK_SIZE bsize,
                                   MACROBLOCK *x, MACROBLOCKD *xd,
                                   int plane_from, int plane_to,
                                   int *out_rate_sum, int64_t *out_dist_sum,
                                   uint8_t *skip_txfm_sb, int64_t *skip_sse_sb,
                                   int *plane_rate, int64_t *plane_sse,
                                   int64_t *plane_dist) {
  // Note our transform coeffs are 8 times an orthogonal transform.
  // Hence quantizer step is also 8 times. To get effective quantizer
  // we need to divide by 8 before sending to modeling function.
  int plane;
  const int ref = xd->mi[0]->ref_frame[0];

  int64_t rate_sum = 0;
  int64_t dist_sum = 0;
  int64_t total_sse = 0;

  assert(bsize < BLOCK_SIZES_ALL);

  for (plane = plane_from; plane <= plane_to; ++plane) {
    if (plane && !xd->is_chroma_ref) break;
    struct macroblock_plane *const p = &x->plane[plane];
    struct macroblockd_plane *const pd = &xd->plane[plane];
    const BLOCK_SIZE plane_bsize =
        get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y);
    assert(plane_bsize < BLOCK_SIZES_ALL);
    const int bw = block_size_wide[plane_bsize];
    const int bh = block_size_high[plane_bsize];
    int64_t sse;
    int rate;
    int64_t dist;

    sse = calculate_sse(xd, p, pd, bw, bh);

    model_rd_from_sse(cpi, x, plane_bsize, plane, sse, bw * bh, &rate, &dist);

    if (plane == 0) x->pred_sse[ref] = (unsigned int)AOMMIN(sse, UINT_MAX);

    total_sse += sse;
    rate_sum += rate;
    dist_sum += dist;
    if (plane_rate) plane_rate[plane] = rate;
    if (plane_sse) plane_sse[plane] = sse;
    if (plane_dist) plane_dist[plane] = dist;
    assert(rate_sum >= 0);
  }

  if (skip_txfm_sb) *skip_txfm_sb = total_sse == 0;
  if (skip_sse_sb) *skip_sse_sb = total_sse << 4;
  rate_sum = AOMMIN(rate_sum, INT_MAX);
  *out_rate_sum = (int)rate_sum;
  *out_dist_sum = dist_sum;
}

static inline void model_rd_for_sb_with_curvfit(
    const AV1_COMP *const cpi, BLOCK_SIZE bsize, MACROBLOCK *x, MACROBLOCKD *xd,
    int plane_from, int plane_to, int *out_rate_sum, int64_t *out_dist_sum,
    uint8_t *skip_txfm_sb, int64_t *skip_sse_sb, int *plane_rate,
    int64_t *plane_sse, int64_t *plane_dist) {
  // Note our transform coeffs are 8 times an orthogonal transform.
  // Hence quantizer step is also 8 times. To get effective quantizer
  // we need to divide by 8 before sending to modeling function.
  const int ref = xd->mi[0]->ref_frame[0];

  int64_t rate_sum = 0;
  int64_t dist_sum = 0;
  int64_t total_sse = 0;

  for (int plane = plane_from; plane <= plane_to; ++plane) {
    if (plane && !xd->is_chroma_ref) break;
    struct macroblockd_plane *const pd = &xd->plane[plane];
    const BLOCK_SIZE plane_bsize =
        get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y);
    int64_t dist, sse;
    int rate;
    int bw, bh;
    const struct macroblock_plane *const p = &x->plane[plane];
    const int block_width = block_size_wide[plane_bsize];
    const int block_height = block_size_high[plane_bsize];

    get_visible_dimensions(x, plane, plane_bsize, 0, 0, block_width,
                           block_height, /*clip_dims=*/true, &bw, &bh);

    sse = pixel_dist_visible_only(cpi, x, p->src.buf, p->src.stride,
                                  pd->dst.buf, pd->dst.stride, plane_bsize,
                                  block_height, block_width, bh, bw);

    model_rd_with_curvfit(cpi, x, plane_bsize, plane, sse, bw * bh, &rate,
                          &dist);

    if (plane == 0) x->pred_sse[ref] = (unsigned int)AOMMIN(sse, UINT_MAX);

    total_sse += sse;
    rate_sum += rate;
    dist_sum += dist;

    if (plane_rate) plane_rate[plane] = rate;
    if (plane_sse) plane_sse[plane] = sse;
    if (plane_dist) plane_dist[plane] = dist;
  }

  if (skip_txfm_sb) *skip_txfm_sb = rate_sum == 0;
  if (skip_sse_sb) *skip_sse_sb = total_sse << 4;
  *out_rate_sum = (int)rate_sum;
  *out_dist_sum = dist_sum;
}

enum { MODELRD_LEGACY, MODELRD_CURVFIT, MODELRD_TYPES } UENUM1BYTE(ModelRdType);

static const model_rd_for_sb_type model_rd_sb_fn[MODELRD_TYPES] = {
  model_rd_for_sb, model_rd_for_sb_with_curvfit
};

static const model_rd_from_sse_type model_rd_sse_fn[MODELRD_TYPES] = {
  model_rd_from_sse, model_rd_with_curvfit
};

#ifdef __cplusplus
}  // extern "C"
#endif
#endif  // AOM_AV1_ENCODER_MODEL_RD_H_

Coverage Report

Created: 2026-07-25 07:06

Line	Count	Source
1		/*
2		* Copyright (c) 2020, Alliance for Open Media. All rights reserved.
3		*
4		* This source code is subject to the terms of the BSD 2 Clause License and
5		* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6		* was not distributed with this source code in the LICENSE file, you can
7		* obtain it at www.aomedia.org/license/software. If the Alliance for Open
8		* Media Patent License 1.0 was not distributed with this source code in the
9		* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10		*/
11
12		#ifndef AOM_AV1_ENCODER_MODEL_RD_H_
13		#define AOM_AV1_ENCODER_MODEL_RD_H_
14
15		#include "aom/aom_integer.h"
16		#include "av1/encoder/block.h"
17		#include "av1/encoder/encoder.h"
18		#include "av1/encoder/pustats.h"
19		#include "av1/encoder/rdopt_utils.h"
20		#include "config/aom_dsp_rtcd.h"
21
22		#ifdef __cplusplus
23		extern "C" {
24		#endif
25
26		// 0: Legacy model
27		// 1: Curve fit model
28		// 2: Surface fit model
29		// 3: DNN regression model
30		// 4: Full rd model
31	0	#define MODELRD_TYPE_INTERP_FILTER 1
32	0	#define MODELRD_TYPE_TX_SEARCH_PRUNE 1
33	0	#define MODELRD_TYPE_MASKED_COMPOUND 1
34	0	#define MODELRD_TYPE_INTERINTRA 1
35		#define MODELRD_TYPE_INTRA 1
36	0	#define MODELRD_TYPE_MOTION_MODE_RD 1
37
38		typedef void (*model_rd_for_sb_type)(
39		const AV1_COMP const cpi, BLOCK_SIZE bsize, MACROBLOCK x, MACROBLOCKD *xd,
40		int plane_from, int plane_to, int out_rate_sum, int64_t out_dist_sum,
41		uint8_t skip_txfm_sb, int64_t skip_sse_sb, int *plane_rate,
42		int64_t plane_sse, int64_t plane_dist);
43		typedef void (model_rd_from_sse_type)(const AV1_COMP const cpi,
44		const MACROBLOCK *const x,
45		BLOCK_SIZE plane_bsize, int plane,
46		int64_t sse, int num_samples, int *rate,
47		int64_t *dist);
48
49		static int64_t calculate_sse(MACROBLOCKD *const xd,
50		const struct macroblock_plane *p,
51		struct macroblockd_plane *pd, const int bw,
52	0	const int bh) {
53	0	int64_t sse = 0;
54	0	const int shift = xd->bd - 8;
55	0	#if CONFIG_AV1_HIGHBITDEPTH
56	0	if (is_cur_buf_hbd(xd)) {
57	0	sse = aom_highbd_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride,
58	0	bw, bh);
59	0	} else {
60	0	sse =
61	0	aom_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, bw, bh);
62	0	}
63		#else
64		sse = aom_sse(p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride, bw, bh);
65		#endif
66	0	sse = ROUND_POWER_OF_TWO(sse, shift * 2);
67	0	return sse;
68	0	} Unexecuted instantiation: av1_cx_iface.c:calculate_sse Unexecuted instantiation: allintra_vis.c:calculate_sse Unexecuted instantiation: encodeframe.c:calculate_sse Unexecuted instantiation: encoder.c:calculate_sse Unexecuted instantiation: encoder_utils.c:calculate_sse Unexecuted instantiation: ethread.c:calculate_sse Unexecuted instantiation: palette.c:calculate_sse Unexecuted instantiation: partition_search.c:calculate_sse Unexecuted instantiation: rdopt.c:calculate_sse Unexecuted instantiation: nonrd_pickmode.c:calculate_sse Unexecuted instantiation: superres_scale.c:calculate_sse Unexecuted instantiation: svc_layercontext.c:calculate_sse Unexecuted instantiation: temporal_filter.c:calculate_sse Unexecuted instantiation: tx_search.c:calculate_sse Unexecuted instantiation: intra_mode_search.c:calculate_sse Unexecuted instantiation: compound_type.c:calculate_sse Unexecuted instantiation: encode_strategy.c:calculate_sse Unexecuted instantiation: interp_search.c:calculate_sse
69
70		static unsigned pixel_dist_visible_only(
71		const AV1_COMP const cpi, const MACROBLOCK x, const uint8_t *src,
72		const int src_stride, const uint8_t *dst, const int dst_stride,
73		const BLOCK_SIZE tx_bsize, int txb_rows, int txb_cols, int visible_rows,
74	0	int visible_cols) {
75	0	if (visible_rows == 0 \|\| visible_cols == 0) return 0;
76
77	0	unsigned sse;
78	0	if (txb_rows == visible_rows && txb_cols == visible_cols) {
79	0	cpi->ppi->fn_ptr[tx_bsize].vf(src, src_stride, dst, dst_stride, &sse);
80	0	return sse;
81	0	}
82
83	0	#if CONFIG_AV1_HIGHBITDEPTH
84	0	const MACROBLOCKD *xd = &x->e_mbd;
85	0	if (is_cur_buf_hbd(xd)) {
86	0	uint64_t sse64;
87	0	if (!(visible_rows % 4) && !(visible_cols % 4)) {
88	0	sse64 = aom_highbd_sse(src, src_stride, dst, dst_stride, visible_cols,
89	0	visible_rows);
90	0	} else {
91	0	sse64 = aom_highbd_sse_odd_size(src, src_stride, dst, dst_stride,
92	0	visible_cols, visible_rows);
93	0	}
94	0	return (unsigned int)ROUND_POWER_OF_TWO(sse64, (xd->bd - 8) * 2);
95	0	}
96		#else
97		(void)x;
98		#endif
99	0	if (!(visible_rows % 4) && !(visible_cols % 4)) {
100	0	sse = (unsigned)aom_sse(src, src_stride, dst, dst_stride, visible_cols,
101	0	visible_rows);
102	0	} else {
103	0	sse = aom_sse_odd_size(src, src_stride, dst, dst_stride, visible_cols,
104	0	visible_rows);
105	0	}
106	0	return sse;
107	0	} Unexecuted instantiation: av1_cx_iface.c:pixel_dist_visible_only Unexecuted instantiation: allintra_vis.c:pixel_dist_visible_only Unexecuted instantiation: encodeframe.c:pixel_dist_visible_only Unexecuted instantiation: encoder.c:pixel_dist_visible_only Unexecuted instantiation: encoder_utils.c:pixel_dist_visible_only Unexecuted instantiation: ethread.c:pixel_dist_visible_only Unexecuted instantiation: palette.c:pixel_dist_visible_only Unexecuted instantiation: partition_search.c:pixel_dist_visible_only Unexecuted instantiation: rdopt.c:pixel_dist_visible_only Unexecuted instantiation: nonrd_pickmode.c:pixel_dist_visible_only Unexecuted instantiation: superres_scale.c:pixel_dist_visible_only Unexecuted instantiation: svc_layercontext.c:pixel_dist_visible_only Unexecuted instantiation: temporal_filter.c:pixel_dist_visible_only Unexecuted instantiation: tx_search.c:pixel_dist_visible_only Unexecuted instantiation: intra_mode_search.c:pixel_dist_visible_only Unexecuted instantiation: compound_type.c:pixel_dist_visible_only Unexecuted instantiation: encode_strategy.c:pixel_dist_visible_only Unexecuted instantiation: interp_search.c:pixel_dist_visible_only
108
109		static inline int64_t compute_sse_plane(const AV1_COMP cpi, MACROBLOCK x,
110		MACROBLOCKD *xd, int plane,
111	0	const BLOCK_SIZE bsize) {
112	0	struct macroblockd_plane *const pd = &xd->plane[plane];
113	0	const BLOCK_SIZE plane_bsize =
114	0	get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y);
115	0	int bw, bh;
116	0	const struct macroblock_plane *const p = &x->plane[plane];
117	0	const int block_width = block_size_wide[plane_bsize];
118	0	const int block_height = block_size_high[plane_bsize];
119
120	0	get_visible_dimensions(x, plane, plane_bsize, 0, 0, block_width, block_height,
121	0	/clip_dims=/true, &bw, &bh);
122
123	0	int64_t sse = pixel_dist_visible_only(
124	0	cpi, x, p->src.buf, p->src.stride, pd->dst.buf, pd->dst.stride,
125	0	plane_bsize, block_height, block_width, bh, bw);
126
127	0	return sse;
128	0	} Unexecuted instantiation: av1_cx_iface.c:compute_sse_plane Unexecuted instantiation: allintra_vis.c:compute_sse_plane Unexecuted instantiation: encodeframe.c:compute_sse_plane Unexecuted instantiation: encoder.c:compute_sse_plane Unexecuted instantiation: encoder_utils.c:compute_sse_plane Unexecuted instantiation: ethread.c:compute_sse_plane Unexecuted instantiation: palette.c:compute_sse_plane Unexecuted instantiation: partition_search.c:compute_sse_plane Unexecuted instantiation: rdopt.c:compute_sse_plane Unexecuted instantiation: nonrd_pickmode.c:compute_sse_plane Unexecuted instantiation: superres_scale.c:compute_sse_plane Unexecuted instantiation: svc_layercontext.c:compute_sse_plane Unexecuted instantiation: temporal_filter.c:compute_sse_plane Unexecuted instantiation: tx_search.c:compute_sse_plane Unexecuted instantiation: intra_mode_search.c:compute_sse_plane Unexecuted instantiation: compound_type.c:compute_sse_plane Unexecuted instantiation: encode_strategy.c:compute_sse_plane Unexecuted instantiation: interp_search.c:compute_sse_plane
129
130		static inline void model_rd_from_sse(const AV1_COMP *const cpi,
131		const MACROBLOCK *const x,
132		BLOCK_SIZE plane_bsize, int plane,
133		int64_t sse, int num_samples, int *rate,
134	0	int64_t *dist) {
135	0	(void)num_samples;
136	0	const MACROBLOCKD *const xd = &x->e_mbd;
137	0	const struct macroblock_plane *const p = &x->plane[plane];
138	0	const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3;
139
140		// Fast approximate the modelling function.
141	0	if (cpi->sf.rd_sf.simple_model_rd_from_var) {
142	0	const int64_t square_error = sse;
143	0	int quantizer = p->dequant_QTX[1] >> dequant_shift;
144	0	if (quantizer < 120)
145	0	*rate = (int)AOMMIN(
146	0	(square_error * (280 - quantizer)) >> (16 - AV1_PROB_COST_SHIFT),
147	0	INT_MAX);
148	0	else
149	0	*rate = 0;
150	0	assert(*rate >= 0);
151	0	dist = (square_error quantizer) >> 8;
152	0	} else {
153	0	av1_model_rd_from_var_lapndz(sse, num_pels_log2_lookup[plane_bsize],
154	0	p->dequant_QTX[1] >> dequant_shift, rate,
155	0	dist);
156	0	}
157	0	*dist <<= 4;
158	0	} Unexecuted instantiation: av1_cx_iface.c:model_rd_from_sse Unexecuted instantiation: allintra_vis.c:model_rd_from_sse Unexecuted instantiation: encodeframe.c:model_rd_from_sse Unexecuted instantiation: encoder.c:model_rd_from_sse Unexecuted instantiation: encoder_utils.c:model_rd_from_sse Unexecuted instantiation: ethread.c:model_rd_from_sse Unexecuted instantiation: palette.c:model_rd_from_sse Unexecuted instantiation: partition_search.c:model_rd_from_sse Unexecuted instantiation: rdopt.c:model_rd_from_sse Unexecuted instantiation: nonrd_pickmode.c:model_rd_from_sse Unexecuted instantiation: superres_scale.c:model_rd_from_sse Unexecuted instantiation: svc_layercontext.c:model_rd_from_sse Unexecuted instantiation: temporal_filter.c:model_rd_from_sse Unexecuted instantiation: tx_search.c:model_rd_from_sse Unexecuted instantiation: intra_mode_search.c:model_rd_from_sse Unexecuted instantiation: compound_type.c:model_rd_from_sse Unexecuted instantiation: encode_strategy.c:model_rd_from_sse Unexecuted instantiation: interp_search.c:model_rd_from_sse
159
160		// Fits a curve for rate and distortion using as feature:
161		// log2(sse_norm/qstep^2)
162		static inline void model_rd_with_curvfit(const AV1_COMP *const cpi,
163		const MACROBLOCK *const x,
164		BLOCK_SIZE plane_bsize, int plane,
165		int64_t sse, int num_samples,
166	0	int rate, int64_t dist) {
167	0	(void)cpi;
168	0	(void)plane_bsize;
169	0	const MACROBLOCKD *const xd = &x->e_mbd;
170	0	const struct macroblock_plane *const p = &x->plane[plane];
171	0	const int dequant_shift = (is_cur_buf_hbd(xd)) ? xd->bd - 5 : 3;
172	0	const int qstep = AOMMAX(p->dequant_QTX[1] >> dequant_shift, 1);
173
174	0	if (sse == 0) {
175	0	if (rate) *rate = 0;
176	0	if (dist) *dist = 0;
177	0	return;
178	0	}
179	0	const double sse_norm = (double)sse / num_samples;
180	0	const double qstepsqr = (double)qstep * qstep;
181	0	const double xqr = log2(sse_norm / qstepsqr);
182	0	double rate_dist_f[2];
183	0	av1_model_rd_curvfit(plane_bsize, sse_norm, xqr, rate_dist_f);
184
185	0	const double dist_f = rate_dist_f[1] * sse_norm;
186	0	int rate_i = (int)(AOMMAX(0.0, rate_dist_f[0] * num_samples) + 0.5);
187	0	int64_t dist_i = (int64_t)(AOMMAX(0.0, dist_f * num_samples) + 0.5);
188
189		// Check if skip is better
190	0	if (rate_i == 0) {
191	0	dist_i = sse << 4;
192	0	} else if (RDCOST(x->rdmult, rate_i, dist_i) >=
193	0	RDCOST(x->rdmult, 0, sse << 4)) {
194	0	rate_i = 0;
195	0	dist_i = sse << 4;
196	0	}
197
198	0	if (rate) *rate = rate_i;
199	0	if (dist) *dist = dist_i;
200	0	} Unexecuted instantiation: av1_cx_iface.c:model_rd_with_curvfit Unexecuted instantiation: allintra_vis.c:model_rd_with_curvfit Unexecuted instantiation: encodeframe.c:model_rd_with_curvfit Unexecuted instantiation: encoder.c:model_rd_with_curvfit Unexecuted instantiation: encoder_utils.c:model_rd_with_curvfit Unexecuted instantiation: ethread.c:model_rd_with_curvfit Unexecuted instantiation: palette.c:model_rd_with_curvfit Unexecuted instantiation: partition_search.c:model_rd_with_curvfit Unexecuted instantiation: rdopt.c:model_rd_with_curvfit Unexecuted instantiation: nonrd_pickmode.c:model_rd_with_curvfit Unexecuted instantiation: superres_scale.c:model_rd_with_curvfit Unexecuted instantiation: svc_layercontext.c:model_rd_with_curvfit Unexecuted instantiation: temporal_filter.c:model_rd_with_curvfit Unexecuted instantiation: tx_search.c:model_rd_with_curvfit Unexecuted instantiation: intra_mode_search.c:model_rd_with_curvfit Unexecuted instantiation: compound_type.c:model_rd_with_curvfit Unexecuted instantiation: encode_strategy.c:model_rd_with_curvfit Unexecuted instantiation: interp_search.c:model_rd_with_curvfit
201
202		static inline void model_rd_for_sb(const AV1_COMP *const cpi, BLOCK_SIZE bsize,
203		MACROBLOCK x, MACROBLOCKD xd,
204		int plane_from, int plane_to,
205		int out_rate_sum, int64_t out_dist_sum,
206		uint8_t skip_txfm_sb, int64_t skip_sse_sb,
207		int plane_rate, int64_t plane_sse,
208	0	int64_t *plane_dist) {
209		// Note our transform coeffs are 8 times an orthogonal transform.
210		// Hence quantizer step is also 8 times. To get effective quantizer
211		// we need to divide by 8 before sending to modeling function.
212	0	int plane;
213	0	const int ref = xd->mi[0]->ref_frame[0];
214
215	0	int64_t rate_sum = 0;
216	0	int64_t dist_sum = 0;
217	0	int64_t total_sse = 0;
218
219	0	assert(bsize < BLOCK_SIZES_ALL);
220
221	0	for (plane = plane_from; plane <= plane_to; ++plane) {
222	0	if (plane && !xd->is_chroma_ref) break;
223	0	struct macroblock_plane *const p = &x->plane[plane];
224	0	struct macroblockd_plane *const pd = &xd->plane[plane];
225	0	const BLOCK_SIZE plane_bsize =
226	0	get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y);
227	0	assert(plane_bsize < BLOCK_SIZES_ALL);
228	0	const int bw = block_size_wide[plane_bsize];
229	0	const int bh = block_size_high[plane_bsize];
230	0	int64_t sse;
231	0	int rate;
232	0	int64_t dist;
233
234	0	sse = calculate_sse(xd, p, pd, bw, bh);
235
236	0	model_rd_from_sse(cpi, x, plane_bsize, plane, sse, bw * bh, &rate, &dist);
237
238	0	if (plane == 0) x->pred_sse[ref] = (unsigned int)AOMMIN(sse, UINT_MAX);
239
240	0	total_sse += sse;
241	0	rate_sum += rate;
242	0	dist_sum += dist;
243	0	if (plane_rate) plane_rate[plane] = rate;
244	0	if (plane_sse) plane_sse[plane] = sse;
245	0	if (plane_dist) plane_dist[plane] = dist;
246	0	assert(rate_sum >= 0);
247	0	}
248
249	0	if (skip_txfm_sb) *skip_txfm_sb = total_sse == 0;
250	0	if (skip_sse_sb) *skip_sse_sb = total_sse << 4;
251	0	rate_sum = AOMMIN(rate_sum, INT_MAX);
252	0	*out_rate_sum = (int)rate_sum;
253	0	*out_dist_sum = dist_sum;
254	0	} Unexecuted instantiation: av1_cx_iface.c:model_rd_for_sb Unexecuted instantiation: allintra_vis.c:model_rd_for_sb Unexecuted instantiation: encodeframe.c:model_rd_for_sb Unexecuted instantiation: encoder.c:model_rd_for_sb Unexecuted instantiation: encoder_utils.c:model_rd_for_sb Unexecuted instantiation: ethread.c:model_rd_for_sb Unexecuted instantiation: palette.c:model_rd_for_sb Unexecuted instantiation: partition_search.c:model_rd_for_sb Unexecuted instantiation: rdopt.c:model_rd_for_sb Unexecuted instantiation: nonrd_pickmode.c:model_rd_for_sb Unexecuted instantiation: superres_scale.c:model_rd_for_sb Unexecuted instantiation: svc_layercontext.c:model_rd_for_sb Unexecuted instantiation: temporal_filter.c:model_rd_for_sb Unexecuted instantiation: tx_search.c:model_rd_for_sb Unexecuted instantiation: intra_mode_search.c:model_rd_for_sb Unexecuted instantiation: compound_type.c:model_rd_for_sb Unexecuted instantiation: encode_strategy.c:model_rd_for_sb Unexecuted instantiation: interp_search.c:model_rd_for_sb
255
256		static inline void model_rd_for_sb_with_curvfit(
257		const AV1_COMP const cpi, BLOCK_SIZE bsize, MACROBLOCK x, MACROBLOCKD *xd,
258		int plane_from, int plane_to, int out_rate_sum, int64_t out_dist_sum,
259		uint8_t skip_txfm_sb, int64_t skip_sse_sb, int *plane_rate,
260	0	int64_t plane_sse, int64_t plane_dist) {
261		// Note our transform coeffs are 8 times an orthogonal transform.
262		// Hence quantizer step is also 8 times. To get effective quantizer
263		// we need to divide by 8 before sending to modeling function.
264	0	const int ref = xd->mi[0]->ref_frame[0];
265
266	0	int64_t rate_sum = 0;
267	0	int64_t dist_sum = 0;
268	0	int64_t total_sse = 0;
269
270	0	for (int plane = plane_from; plane <= plane_to; ++plane) {
271	0	if (plane && !xd->is_chroma_ref) break;
272	0	struct macroblockd_plane *const pd = &xd->plane[plane];
273	0	const BLOCK_SIZE plane_bsize =
274	0	get_plane_block_size(bsize, pd->subsampling_x, pd->subsampling_y);
275	0	int64_t dist, sse;
276	0	int rate;
277	0	int bw, bh;
278	0	const struct macroblock_plane *const p = &x->plane[plane];
279	0	const int block_width = block_size_wide[plane_bsize];
280	0	const int block_height = block_size_high[plane_bsize];
281
282	0	get_visible_dimensions(x, plane, plane_bsize, 0, 0, block_width,
283	0	block_height, /clip_dims=/true, &bw, &bh);
284
285	0	sse = pixel_dist_visible_only(cpi, x, p->src.buf, p->src.stride,
286	0	pd->dst.buf, pd->dst.stride, plane_bsize,
287	0	block_height, block_width, bh, bw);
288
289	0	model_rd_with_curvfit(cpi, x, plane_bsize, plane, sse, bw * bh, &rate,
290	0	&dist);
291
292	0	if (plane == 0) x->pred_sse[ref] = (unsigned int)AOMMIN(sse, UINT_MAX);
293
294	0	total_sse += sse;
295	0	rate_sum += rate;
296	0	dist_sum += dist;
297
298	0	if (plane_rate) plane_rate[plane] = rate;
299	0	if (plane_sse) plane_sse[plane] = sse;
300	0	if (plane_dist) plane_dist[plane] = dist;
301	0	}
302
303	0	if (skip_txfm_sb) *skip_txfm_sb = rate_sum == 0;
304	0	if (skip_sse_sb) *skip_sse_sb = total_sse << 4;
305	0	*out_rate_sum = (int)rate_sum;
306	0	*out_dist_sum = dist_sum;
307	0	} Unexecuted instantiation: av1_cx_iface.c:model_rd_for_sb_with_curvfit Unexecuted instantiation: allintra_vis.c:model_rd_for_sb_with_curvfit Unexecuted instantiation: encodeframe.c:model_rd_for_sb_with_curvfit Unexecuted instantiation: encoder.c:model_rd_for_sb_with_curvfit Unexecuted instantiation: encoder_utils.c:model_rd_for_sb_with_curvfit Unexecuted instantiation: ethread.c:model_rd_for_sb_with_curvfit Unexecuted instantiation: palette.c:model_rd_for_sb_with_curvfit Unexecuted instantiation: partition_search.c:model_rd_for_sb_with_curvfit Unexecuted instantiation: rdopt.c:model_rd_for_sb_with_curvfit Unexecuted instantiation: nonrd_pickmode.c:model_rd_for_sb_with_curvfit Unexecuted instantiation: superres_scale.c:model_rd_for_sb_with_curvfit Unexecuted instantiation: svc_layercontext.c:model_rd_for_sb_with_curvfit Unexecuted instantiation: temporal_filter.c:model_rd_for_sb_with_curvfit Unexecuted instantiation: tx_search.c:model_rd_for_sb_with_curvfit Unexecuted instantiation: intra_mode_search.c:model_rd_for_sb_with_curvfit Unexecuted instantiation: compound_type.c:model_rd_for_sb_with_curvfit Unexecuted instantiation: encode_strategy.c:model_rd_for_sb_with_curvfit Unexecuted instantiation: interp_search.c:model_rd_for_sb_with_curvfit
308
309		enum { MODELRD_LEGACY, MODELRD_CURVFIT, MODELRD_TYPES } UENUM1BYTE(ModelRdType);
310
311		static const model_rd_for_sb_type model_rd_sb_fn[MODELRD_TYPES] = {
312		model_rd_for_sb, model_rd_for_sb_with_curvfit
313		};
314
315		static const model_rd_from_sse_type model_rd_sse_fn[MODELRD_TYPES] = {
316		model_rd_from_sse, model_rd_with_curvfit
317		};
318
319		#ifdef __cplusplus
320		} // extern "C"
321		#endif
322		#endif // AOM_AV1_ENCODER_MODEL_RD_H_