Coverage Report

Created: 2022-08-24 06:11

/src/aom/av1/encoder/tx_search.h
Line
Count
Source (jump to first uncovered line)
1
/*
2
 * Copyright (c) 2020, Alliance for Open Media. All rights reserved
3
 *
4
 * This source code is subject to the terms of the BSD 2 Clause License and
5
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6
 * was not distributed with this source code in the LICENSE file, you can
7
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8
 * Media Patent License 1.0 was not distributed with this source code in the
9
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10
 */
11
12
#ifndef AOM_AV1_ENCODER_TRANSFORM_SEARCH_H_
13
#define AOM_AV1_ENCODER_TRANSFORM_SEARCH_H_
14
15
#include "av1/common/pred_common.h"
16
#include "av1/encoder/encoder.h"
17
18
#ifdef __cplusplus
19
extern "C" {
20
#endif
21
22
// Set this macro as 1 to collect data about tx size selection.
23
#define COLLECT_TX_SIZE_DATA 0
24
25
#if COLLECT_TX_SIZE_DATA
26
static const char av1_tx_size_data_output_file[] = "tx_size_data.txt";
27
#endif
28
29
enum {
30
  FTXS_NONE = 0,
31
  FTXS_DCT_AND_1D_DCT_ONLY = 1 << 0,
32
  FTXS_DISABLE_TRELLIS_OPT = 1 << 1,
33
  FTXS_USE_TRANSFORM_DOMAIN = 1 << 2
34
} UENUM1BYTE(FAST_TX_SEARCH_MODE);
35
36
static AOM_INLINE int tx_size_cost(const MACROBLOCK *const x, BLOCK_SIZE bsize,
37
0
                                   TX_SIZE tx_size) {
38
0
  assert(bsize == x->e_mbd.mi[0]->bsize);
39
0
  if (x->txfm_search_params.tx_mode_search_type != TX_MODE_SELECT ||
40
0
      !block_signals_txsize(bsize))
41
0
    return 0;
42
43
0
  const int32_t tx_size_cat = bsize_to_tx_size_cat(bsize);
44
0
  const int depth = tx_size_to_depth(tx_size, bsize);
45
0
  const MACROBLOCKD *const xd = &x->e_mbd;
46
0
  const int tx_size_ctx = get_tx_size_context(xd);
47
0
  return x->mode_costs.tx_size_cost[tx_size_cat][tx_size_ctx][depth];
48
0
}
Unexecuted instantiation: palette.c:tx_size_cost
Unexecuted instantiation: rdopt.c:tx_size_cost
Unexecuted instantiation: tx_search.c:tx_size_cost
Unexecuted instantiation: intra_mode_search.c:tx_size_cost
Unexecuted instantiation: compound_type.c:tx_size_cost
Unexecuted instantiation: motion_search_facade.c:tx_size_cost
49
50
int64_t av1_estimate_txfm_yrd(const AV1_COMP *const cpi, MACROBLOCK *x,
51
                              RD_STATS *rd_stats, int64_t ref_best_rd,
52
                              BLOCK_SIZE bs, TX_SIZE tx_size);
53
54
/*!\brief Transform type search for luma macroblock with fixed transform size.
55
 *
56
 * \ingroup transform_search
57
 * Search for the best transform type and return the transform coefficients RD
58
 * cost of current luma macroblock with the given uniform transform size.
59
 *
60
 * \param[in]    x              Pointer to structure holding the data for the
61
                                current encoding macroblock
62
 * \param[in]    cpi            Top-level encoder structure
63
 * \param[in]    rd_stats       Pointer to struct to keep track of the RD stats
64
 * \param[in]    ref_best_rd    Best RD cost seen for this block so far
65
 * \param[in]    bs             Size of the current macroblock
66
 * \param[in]    tx_size        The given transform size
67
 * \param[in]    ftxs_mode      Transform search mode specifying desired speed
68
                                and quality tradeoff
69
 * \param[in]    skip_trellis   Binary flag indicating if trellis optimization
70
                                should be skipped
71
 * \return       An int64_t value that is the best RD cost found.
72
 */
73
int64_t av1_uniform_txfm_yrd(const AV1_COMP *const cpi, MACROBLOCK *x,
74
                             RD_STATS *rd_stats, int64_t ref_best_rd,
75
                             BLOCK_SIZE bs, TX_SIZE tx_size,
76
                             FAST_TX_SEARCH_MODE ftxs_mode, int skip_trellis);
77
78
/*!\brief Recursive transform size and type search.
79
 *
80
 * \ingroup transform_search
81
 * Search for best transform size and type for luma inter blocks. The transform
82
 * block partitioning can be recursive resulting in non-uniform transform sizes.
83
 * The best transform size and type, if found, will be saved in the MB_MODE_INFO
84
 * structure, and the corresponding RD stats will be saved in rd_stats.
85
 *
86
 * \param[in]    cpi            Top-level encoder structure
87
 * \param[in]    x              Pointer to structure holding the data for the
88
                                current encoding macroblock
89
 * \param[in]    rd_stats       Pointer to struct to keep track of the RD stats
90
 * \param[in]    bsize          Current macroblock size
91
 * \param[in]    ref_best_rd    Best RD cost seen for this block so far
92
 * \return       Nothing is returned. The selected transform size and type will
93
                 be saved in the MB_MODE_INFO structure
94
 */
95
void av1_pick_recursive_tx_size_type_yrd(const AV1_COMP *cpi, MACROBLOCK *x,
96
                                         RD_STATS *rd_stats, BLOCK_SIZE bsize,
97
                                         int64_t ref_best_rd);
98
99
/*!\brief Uniform transform size and type search.
100
 *
101
 * \ingroup transform_search
102
 * Search for the best transform size and type for current macroblock block,
103
 * with the assumption that all the transform blocks have a uniform size
104
 * (VP9 style). The selected transform size and type will be saved in the
105
 * MB_MODE_INFO structure; the corresponding RD stats will be saved in rd_stats.
106
 * This function may be used for both intra and inter predicted blocks.
107
 *
108
 * \param[in]    cpi            Top-level encoder structure
109
 * \param[in]    x              Pointer to structure holding the data for the
110
                                current encoding macroblock
111
 * \param[in]    rd_stats       Pointer to struct to keep track of the RD stats
112
 * \param[in]    bs             Current macroblock size
113
 * \param[in]    ref_best_rd    Best RD cost seen for this block so far
114
 * \return       Nothing is returned. The selected transform size and type will
115
                 be saved in the MB_MODE_INFO structure
116
 */
117
void av1_pick_uniform_tx_size_type_yrd(const AV1_COMP *const cpi, MACROBLOCK *x,
118
                                       RD_STATS *rd_stats, BLOCK_SIZE bs,
119
                                       int64_t ref_best_rd);
120
121
/*!\brief Chroma block transform search.
122
 *
123
 * \ingroup transform_search
124
 * Calculate the transform coefficient RD cost for the given chroma macroblock
125
 * If the current mode is intra, then this function will compute the predictor.
126
 *
127
 * \param[in]    cpi            Top-level encoder structure
128
 * \param[in]    x              Pointer to structure holding the data for the
129
                                current encoding macroblock
130
 * \param[in]    rd_stats       Pointer to struct to keep track of the RD stats
131
 * \param[in]    bsize          Current macroblock size
132
 * \param[in]    ref_best_rd    Best RD cost seen for this block so far
133
 * \return       An integer value is returned. 0: early termination triggered,
134
                 no valid rd cost available; 1: rd cost values are valid.
135
 */
136
int av1_txfm_uvrd(const AV1_COMP *const cpi, MACROBLOCK *x, RD_STATS *rd_stats,
137
                  BLOCK_SIZE bsize, int64_t ref_best_rd);
138
139
/*!\brief Transform type search with fixed transform size.
140
 *
141
 * \ingroup transform_search
142
 * Search for the best transform type and calculate the transform coefficients
143
 * RD cost of the current transform block with the specified (uniform) transform
144
 * size and plane. The RD results will be saved in rd_stats.
145
 *
146
 * \param[in]    x              Pointer to structure holding the data for the
147
                                current encoding macroblock
148
 * \param[in]    cpi            Top-level encoder structure
149
 * \param[in]    rd_stats       Pointer to struct to keep track of the RD stats
150
 * \param[in]    ref_best_rd    Best RD cost seen for this block so far
151
 * \param[in]    current_rd     Current RD cost for this block so far
152
 * \param[in]    plane          Plane index
153
 * \param[in]    plane_bsize    Size of the current macroblock considering
154
                                sup-sampling
155
 * \param[in]    tx_size        The given transform size
156
 * \param[in]    ftxs_mode      Transform search mode specifying desired speed
157
                                and quality tradeoff
158
 * \param[in]    skip_trellis   Binary flag indicating if trellis optimization
159
                                should be skipped
160
 *
161
 * \return       Nothing is returned. The RD results will be saved in rd_stats.
162
 */
163
void av1_txfm_rd_in_plane(MACROBLOCK *x, const AV1_COMP *cpi,
164
                          RD_STATS *rd_stats, int64_t ref_best_rd,
165
                          int64_t current_rd, int plane, BLOCK_SIZE plane_bsize,
166
                          TX_SIZE tx_size, FAST_TX_SEARCH_MODE ftxs_mode,
167
                          int skip_trellis);
168
169
/*!\brief Recursive transform size and type search.
170
 *
171
 * \ingroup transform_search
172
 * This function combines y and uv planes' transform search processes together
173
 * for inter-predicted blocks (including IntraBC), when the prediction is
174
 * already generated. It first does subtraction to obtain the prediction error.
175
 * Then it calls
176
 * av1_pick_recursive_tx_size_type_yrd/av1_pick_uniform_tx_size_type_yrd and
177
 * av1_txfm_uvrd sequentially and handles possible early terminations.
178
 * The RD metrics are calculated and stored in rd_stats/_y/_uv.
179
 *
180
 * \param[in]    cpi            Top-level encoder structure
181
 * \param[in]    x              Pointer to structure holding the data for the
182
                                current encoding macroblock
183
 * \param[in]    bsize          Current macroblock size
184
 * \param[in]    rd_stats       Pointer to struct to keep track of the overal RD
185
                                stats
186
 * \param[in]    rd_stats_y     Pointer to struct to keep track of the RD
187
                                stats for the luma plane
188
 * \param[in]    rd_stats_uv    Pointer to struct to keep track of the RD
189
                                stats for the chroma planes
190
 * \param[in]    mode_rate      Rate cost to encode the prediction mode info. of
191
                                the current macroblock
192
 * \param[in]    ref_best_rd    Best RD cost seen for this block so far
193
 *
194
 * \return       An integer value is returned indicating if a valid transform
195
                 candidate is found (1) or not (0).
196
 */
197
int av1_txfm_search(const AV1_COMP *cpi, MACROBLOCK *x, BLOCK_SIZE bsize,
198
                    RD_STATS *rd_stats, RD_STATS *rd_stats_y,
199
                    RD_STATS *rd_stats_uv, int mode_rate, int64_t ref_best_rd);
200
201
#ifdef __cplusplus
202
}  // extern "C"
203
#endif
204
205
#endif  // AOM_AV1_ENCODER_TRANSFORM_SEARCH_H_