/src/aom/av1/encoder/tx_search.h

Source (jump to first uncovered line)
/*
 * Copyright (c) 2020, Alliance for Open Media. All rights reserved
 *
 * This source code is subject to the terms of the BSD 2 Clause License and
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
 * was not distributed with this source code in the LICENSE file, you can
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
 * Media Patent License 1.0 was not distributed with this source code in the
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
 */

#ifndef AOM_AV1_ENCODER_TRANSFORM_SEARCH_H_
#define AOM_AV1_ENCODER_TRANSFORM_SEARCH_H_

#include "av1/common/pred_common.h"
#include "av1/encoder/encoder.h"

#ifdef __cplusplus
extern "C" {
#endif

// Set this macro as 1 to collect data about tx size selection.
#define COLLECT_TX_SIZE_DATA 0

#if COLLECT_TX_SIZE_DATA
static const char av1_tx_size_data_output_file[] = "tx_size_data.txt";
#endif

enum {
  FTXS_NONE = 0,
  FTXS_DCT_AND_1D_DCT_ONLY = 1 << 0,
  FTXS_DISABLE_TRELLIS_OPT = 1 << 1,
  FTXS_USE_TRANSFORM_DOMAIN = 1 << 2
} UENUM1BYTE(FAST_TX_SEARCH_MODE);

static AOM_INLINE int tx_size_cost(const MACROBLOCK *const x, BLOCK_SIZE bsize,
                                   TX_SIZE tx_size) {
  assert(bsize == x->e_mbd.mi[0]->bsize);
  if (x->txfm_search_params.tx_mode_search_type != TX_MODE_SELECT ||
      !block_signals_txsize(bsize))
    return 0;

  const int32_t tx_size_cat = bsize_to_tx_size_cat(bsize);
  const int depth = tx_size_to_depth(tx_size, bsize);
  const MACROBLOCKD *const xd = &x->e_mbd;
  const int tx_size_ctx = get_tx_size_context(xd);
  return x->mode_costs.tx_size_cost[tx_size_cat][tx_size_ctx][depth];
}

int64_t av1_estimate_txfm_yrd(const AV1_COMP *const cpi, MACROBLOCK *x,
                              RD_STATS *rd_stats, int64_t ref_best_rd,
                              BLOCK_SIZE bs, TX_SIZE tx_size);

/*!\brief Transform type search for luma macroblock with fixed transform size.
 *
 * \ingroup transform_search
 * Search for the best transform type and return the transform coefficients RD
 * cost of current luma macroblock with the given uniform transform size.
 *
 * \param[in]    x              Pointer to structure holding the data for the
                                current encoding macroblock
 * \param[in]    cpi            Top-level encoder structure
 * \param[in]    rd_stats       Pointer to struct to keep track of the RD stats
 * \param[in]    ref_best_rd    Best RD cost seen for this block so far
 * \param[in]    bs             Size of the current macroblock
 * \param[in]    tx_size        The given transform size
 * \param[in]    ftxs_mode      Transform search mode specifying desired speed
                                and quality tradeoff
 * \param[in]    skip_trellis   Binary flag indicating if trellis optimization
                                should be skipped
 * \return       An int64_t value that is the best RD cost found.
 */
int64_t av1_uniform_txfm_yrd(const AV1_COMP *const cpi, MACROBLOCK *x,
                             RD_STATS *rd_stats, int64_t ref_best_rd,
                             BLOCK_SIZE bs, TX_SIZE tx_size,
                             FAST_TX_SEARCH_MODE ftxs_mode, int skip_trellis);

/*!\brief Recursive transform size and type search.
 *
 * \ingroup transform_search
 * Search for best transform size and type for luma inter blocks. The transform
 * block partitioning can be recursive resulting in non-uniform transform sizes.
 * The best transform size and type, if found, will be saved in the MB_MODE_INFO
 * structure, and the corresponding RD stats will be saved in rd_stats.
 *
 * \param[in]    cpi            Top-level encoder structure
 * \param[in]    x              Pointer to structure holding the data for the
                                current encoding macroblock
 * \param[in]    rd_stats       Pointer to struct to keep track of the RD stats
 * \param[in]    bsize          Current macroblock size
 * \param[in]    ref_best_rd    Best RD cost seen for this block so far
 * \return       Nothing is returned. The selected transform size and type will
                 be saved in the MB_MODE_INFO structure
 */
void av1_pick_recursive_tx_size_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
                                         RD_STATS *rd_stats, BLOCK_SIZE bsize,
                                         int64_t ref_best_rd);

/*!\brief Uniform transform size and type search.
 *
 * \ingroup transform_search
 * Search for the best transform size and type for current macroblock block,
 * with the assumption that all the transform blocks have a uniform size
 * (VP9 style). The selected transform size and type will be saved in the
 * MB_MODE_INFO structure; the corresponding RD stats will be saved in rd_stats.
 * This function may be used for both intra and inter predicted blocks.
 *
 * \param[in]    cpi            Top-level encoder structure
 * \param[in]    x              Pointer to structure holding the data for the
                                current encoding macroblock
 * \param[in]    rd_stats       Pointer to struct to keep track of the RD stats
 * \param[in]    bs             Current macroblock size
 * \param[in]    ref_best_rd    Best RD cost seen for this block so far
 * \return       Nothing is returned. The selected transform size and type will
                 be saved in the MB_MODE_INFO structure
 */
void av1_pick_uniform_tx_size_type_yrd(const AV1_COMP *const cpi, MACROBLOCK *x,
                                       RD_STATS *rd_stats, BLOCK_SIZE bs,
                                       int64_t ref_best_rd);

/*!\brief Chroma block transform search.
 *
 * \ingroup transform_search
 * Calculate the transform coefficient RD cost for the given chroma macroblock
 * If the current mode is intra, then this function will compute the predictor.
 *
 * \param[in]    cpi            Top-level encoder structure
 * \param[in]    x              Pointer to structure holding the data for the
                                current encoding macroblock
 * \param[in]    rd_stats       Pointer to struct to keep track of the RD stats
 * \param[in]    bsize          Current macroblock size
 * \param[in]    ref_best_rd    Best RD cost seen for this block so far
 * \return       An integer value is returned. 0: early termination triggered,
                 no valid rd cost available; 1: rd cost values are valid.
 */
int av1_txfm_uvrd(const AV1_COMP *const cpi, MACROBLOCK *x, RD_STATS *rd_stats,
                  BLOCK_SIZE bsize, int64_t ref_best_rd);

/*!\brief Transform type search with fixed transform size.
 *
 * \ingroup transform_search
 * Search for the best transform type and calculate the transform coefficients
 * RD cost of the current transform block with the specified (uniform) transform
 * size and plane. The RD results will be saved in rd_stats.
 *
 * \param[in]    x              Pointer to structure holding the data for the
                                current encoding macroblock
 * \param[in]    cpi            Top-level encoder structure
 * \param[in]    rd_stats       Pointer to struct to keep track of the RD stats
 * \param[in]    ref_best_rd    Best RD cost seen for this block so far
 * \param[in]    current_rd     Current RD cost for this block so far
 * \param[in]    plane          Plane index
 * \param[in]    plane_bsize    Size of the current macroblock considering
                                sup-sampling
 * \param[in]    tx_size        The given transform size
 * \param[in]    ftxs_mode      Transform search mode specifying desired speed
                                and quality tradeoff
 * \param[in]    skip_trellis   Binary flag indicating if trellis optimization
                                should be skipped
 *
 * \return       Nothing is returned. The RD results will be saved in rd_stats.
 */
void av1_txfm_rd_in_plane(MACROBLOCK *x, const AV1_COMP *cpi,
                          RD_STATS *rd_stats, int64_t ref_best_rd,
                          int64_t current_rd, int plane, BLOCK_SIZE plane_bsize,
                          TX_SIZE tx_size, FAST_TX_SEARCH_MODE ftxs_mode,
                          int skip_trellis);

/*!\brief Recursive transform size and type search.
 *
 * \ingroup transform_search
 * This function combines y and uv planes' transform search processes together
 * for inter-predicted blocks (including IntraBC), when the prediction is
 * already generated. It first does subtraction to obtain the prediction error.
 * Then it calls
 * av1_pick_recursive_tx_size_type_yrd/av1_pick_uniform_tx_size_type_yrd and
 * av1_txfm_uvrd sequentially and handles possible early terminations.
 * The RD metrics are calculated and stored in rd_stats/_y/_uv.
 *
 * \param[in]    cpi            Top-level encoder structure
 * \param[in]    x              Pointer to structure holding the data for the
                                current encoding macroblock
 * \param[in]    bsize          Current macroblock size
 * \param[in]    rd_stats       Pointer to struct to keep track of the overal RD
                                stats
 * \param[in]    rd_stats_y     Pointer to struct to keep track of the RD
                                stats for the luma plane
 * \param[in]    rd_stats_uv    Pointer to struct to keep track of the RD
                                stats for the chroma planes
 * \param[in]    mode_rate      Rate cost to encode the prediction mode info. of
                                the current macroblock
 * \param[in]    ref_best_rd    Best RD cost seen for this block so far
 *
 * \return       An integer value is returned indicating if a valid transform
                 candidate is found (1) or not (0).
 */
int av1_txfm_search(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize,
                    RD_STATS *rd_stats, RD_STATS *rd_stats_y,
                    RD_STATS *rd_stats_uv, int mode_rate, int64_t ref_best_rd);

#ifdef __cplusplus
}  // extern "C"
#endif

#endif  // AOM_AV1_ENCODER_TRANSFORM_SEARCH_H_

Coverage Report

Created: 2022-08-24 06:11

Line	Count	Source (jump to first uncovered line)
1		/*
2		* Copyright (c) 2020, Alliance for Open Media. All rights reserved
3		*
4		* This source code is subject to the terms of the BSD 2 Clause License and
5		* the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6		* was not distributed with this source code in the LICENSE file, you can
7		* obtain it at www.aomedia.org/license/software. If the Alliance for Open
8		* Media Patent License 1.0 was not distributed with this source code in the
9		* PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10		*/
11
12		#ifndef AOM_AV1_ENCODER_TRANSFORM_SEARCH_H_
13		#define AOM_AV1_ENCODER_TRANSFORM_SEARCH_H_
14
15		#include "av1/common/pred_common.h"
16		#include "av1/encoder/encoder.h"
17
18		#ifdef __cplusplus
19		extern "C" {
20		#endif
21
22		// Set this macro as 1 to collect data about tx size selection.
23		#define COLLECT_TX_SIZE_DATA 0
24
25		#if COLLECT_TX_SIZE_DATA
26		static const char av1_tx_size_data_output_file[] = "tx_size_data.txt";
27		#endif
28
29		enum {
30		FTXS_NONE = 0,
31		FTXS_DCT_AND_1D_DCT_ONLY = 1 << 0,
32		FTXS_DISABLE_TRELLIS_OPT = 1 << 1,
33		FTXS_USE_TRANSFORM_DOMAIN = 1 << 2
34		} UENUM1BYTE(FAST_TX_SEARCH_MODE);
35
36		static AOM_INLINE int tx_size_cost(const MACROBLOCK *const x, BLOCK_SIZE bsize,
37	0	TX_SIZE tx_size) {
38	0	assert(bsize == x->e_mbd.mi[0]->bsize);
39	0	if (x->txfm_search_params.tx_mode_search_type != TX_MODE_SELECT \|\|
40	0	!block_signals_txsize(bsize))
41	0	return 0;
42
43	0	const int32_t tx_size_cat = bsize_to_tx_size_cat(bsize);
44	0	const int depth = tx_size_to_depth(tx_size, bsize);
45	0	const MACROBLOCKD *const xd = &x->e_mbd;
46	0	const int tx_size_ctx = get_tx_size_context(xd);
47	0	return x->mode_costs.tx_size_cost[tx_size_cat][tx_size_ctx][depth];
48	0	} Unexecuted instantiation: palette.c:tx_size_cost Unexecuted instantiation: rdopt.c:tx_size_cost Unexecuted instantiation: tx_search.c:tx_size_cost Unexecuted instantiation: intra_mode_search.c:tx_size_cost Unexecuted instantiation: compound_type.c:tx_size_cost Unexecuted instantiation: motion_search_facade.c:tx_size_cost
49
50		int64_t av1_estimate_txfm_yrd(const AV1_COMP const cpi, MACROBLOCK x,
51		RD_STATS *rd_stats, int64_t ref_best_rd,
52		BLOCK_SIZE bs, TX_SIZE tx_size);
53
54		/*!\brief Transform type search for luma macroblock with fixed transform size.
55		*
56		* \ingroup transform_search
57		* Search for the best transform type and return the transform coefficients RD
58		* cost of current luma macroblock with the given uniform transform size.
59		*
60		* \param[in] x Pointer to structure holding the data for the
61		current encoding macroblock
62		* \param[in] cpi Top-level encoder structure
63		* \param[in] rd_stats Pointer to struct to keep track of the RD stats
64		* \param[in] ref_best_rd Best RD cost seen for this block so far
65		* \param[in] bs Size of the current macroblock
66		* \param[in] tx_size The given transform size
67		* \param[in] ftxs_mode Transform search mode specifying desired speed
68		and quality tradeoff
69		* \param[in] skip_trellis Binary flag indicating if trellis optimization
70		should be skipped
71		* \return An int64_t value that is the best RD cost found.
72		*/
73		int64_t av1_uniform_txfm_yrd(const AV1_COMP const cpi, MACROBLOCK x,
74		RD_STATS *rd_stats, int64_t ref_best_rd,
75		BLOCK_SIZE bs, TX_SIZE tx_size,
76		FAST_TX_SEARCH_MODE ftxs_mode, int skip_trellis);
77
78		/*!\brief Recursive transform size and type search.
79		*
80		* \ingroup transform_search
81		* Search for best transform size and type for luma inter blocks. The transform
82		* block partitioning can be recursive resulting in non-uniform transform sizes.
83		* The best transform size and type, if found, will be saved in the MB_MODE_INFO
84		* structure, and the corresponding RD stats will be saved in rd_stats.
85		*
86		* \param[in] cpi Top-level encoder structure
87		* \param[in] x Pointer to structure holding the data for the
88		current encoding macroblock
89		* \param[in] rd_stats Pointer to struct to keep track of the RD stats
90		* \param[in] bsize Current macroblock size
91		* \param[in] ref_best_rd Best RD cost seen for this block so far
92		* \return Nothing is returned. The selected transform size and type will
93		be saved in the MB_MODE_INFO structure
94		*/
95		void av1_pick_recursive_tx_size_type_yrd(const AV1_COMP cpi, MACROBLOCK x,
96		RD_STATS *rd_stats, BLOCK_SIZE bsize,
97		int64_t ref_best_rd);
98
99		/*!\brief Uniform transform size and type search.
100		*
101		* \ingroup transform_search
102		* Search for the best transform size and type for current macroblock block,
103		* with the assumption that all the transform blocks have a uniform size
104		* (VP9 style). The selected transform size and type will be saved in the
105		* MB_MODE_INFO structure; the corresponding RD stats will be saved in rd_stats.
106		* This function may be used for both intra and inter predicted blocks.
107		*
108		* \param[in] cpi Top-level encoder structure
109		* \param[in] x Pointer to structure holding the data for the
110		current encoding macroblock
111		* \param[in] rd_stats Pointer to struct to keep track of the RD stats
112		* \param[in] bs Current macroblock size
113		* \param[in] ref_best_rd Best RD cost seen for this block so far
114		* \return Nothing is returned. The selected transform size and type will
115		be saved in the MB_MODE_INFO structure
116		*/
117		void av1_pick_uniform_tx_size_type_yrd(const AV1_COMP const cpi, MACROBLOCK x,
118		RD_STATS *rd_stats, BLOCK_SIZE bs,
119		int64_t ref_best_rd);
120
121		/*!\brief Chroma block transform search.
122		*
123		* \ingroup transform_search
124		* Calculate the transform coefficient RD cost for the given chroma macroblock
125		* If the current mode is intra, then this function will compute the predictor.
126		*
127		* \param[in] cpi Top-level encoder structure
128		* \param[in] x Pointer to structure holding the data for the
129		current encoding macroblock
130		* \param[in] rd_stats Pointer to struct to keep track of the RD stats
131		* \param[in] bsize Current macroblock size
132		* \param[in] ref_best_rd Best RD cost seen for this block so far
133		* \return An integer value is returned. 0: early termination triggered,
134		no valid rd cost available; 1: rd cost values are valid.
135		*/
136		int av1_txfm_uvrd(const AV1_COMP const cpi, MACROBLOCK x, RD_STATS *rd_stats,
137		BLOCK_SIZE bsize, int64_t ref_best_rd);
138
139		/*!\brief Transform type search with fixed transform size.
140		*
141		* \ingroup transform_search
142		* Search for the best transform type and calculate the transform coefficients
143		* RD cost of the current transform block with the specified (uniform) transform
144		* size and plane. The RD results will be saved in rd_stats.
145		*
146		* \param[in] x Pointer to structure holding the data for the
147		current encoding macroblock
148		* \param[in] cpi Top-level encoder structure
149		* \param[in] rd_stats Pointer to struct to keep track of the RD stats
150		* \param[in] ref_best_rd Best RD cost seen for this block so far
151		* \param[in] current_rd Current RD cost for this block so far
152		* \param[in] plane Plane index
153		* \param[in] plane_bsize Size of the current macroblock considering
154		sup-sampling
155		* \param[in] tx_size The given transform size
156		* \param[in] ftxs_mode Transform search mode specifying desired speed
157		and quality tradeoff
158		* \param[in] skip_trellis Binary flag indicating if trellis optimization
159		should be skipped
160		*
161		* \return Nothing is returned. The RD results will be saved in rd_stats.
162		*/
163		void av1_txfm_rd_in_plane(MACROBLOCK x, const AV1_COMP cpi,
164		RD_STATS *rd_stats, int64_t ref_best_rd,
165		int64_t current_rd, int plane, BLOCK_SIZE plane_bsize,
166		TX_SIZE tx_size, FAST_TX_SEARCH_MODE ftxs_mode,
167		int skip_trellis);
168
169		/*!\brief Recursive transform size and type search.
170		*
171		* \ingroup transform_search
172		* This function combines y and uv planes' transform search processes together
173		* for inter-predicted blocks (including IntraBC), when the prediction is
174		* already generated. It first does subtraction to obtain the prediction error.
175		* Then it calls
176		* av1_pick_recursive_tx_size_type_yrd/av1_pick_uniform_tx_size_type_yrd and
177		* av1_txfm_uvrd sequentially and handles possible early terminations.
178		* The RD metrics are calculated and stored in rd_stats/_y/_uv.
179		*
180		* \param[in] cpi Top-level encoder structure
181		* \param[in] x Pointer to structure holding the data for the
182		current encoding macroblock
183		* \param[in] bsize Current macroblock size
184		* \param[in] rd_stats Pointer to struct to keep track of the overal RD
185		stats
186		* \param[in] rd_stats_y Pointer to struct to keep track of the RD
187		stats for the luma plane
188		* \param[in] rd_stats_uv Pointer to struct to keep track of the RD
189		stats for the chroma planes
190		* \param[in] mode_rate Rate cost to encode the prediction mode info. of
191		the current macroblock
192		* \param[in] ref_best_rd Best RD cost seen for this block so far
193		*
194		* \return An integer value is returned indicating if a valid transform
195		candidate is found (1) or not (0).
196		*/
197		int av1_txfm_search(const AV1_COMP cpi, MACROBLOCK x, BLOCK_SIZE bsize,
198		RD_STATS rd_stats, RD_STATS rd_stats_y,
199		RD_STATS *rd_stats_uv, int mode_rate, int64_t ref_best_rd);
200
201		#ifdef __cplusplus
202		} // extern "C"
203		#endif
204
205		#endif // AOM_AV1_ENCODER_TRANSFORM_SEARCH_H_