Coverage Report

Created: 2022-08-24 06:11

/src/aom/av1/encoder/intra_mode_search.h
Line
Count
Source (jump to first uncovered line)
1
/*
2
 * Copyright (c) 2020, Alliance for Open Media. All rights reserved
3
 *
4
 * This source code is subject to the terms of the BSD 2 Clause License and
5
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6
 * was not distributed with this source code in the LICENSE file, you can
7
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8
 * Media Patent License 1.0 was not distributed with this source code in the
9
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10
 */
11
12
/*!\file
13
 * \brief Declares high level functions to search through intra modes.
14
 */
15
#ifndef AOM_AV1_ENCODER_INTRA_MODE_SEARCH_H_
16
#define AOM_AV1_ENCODER_INTRA_MODE_SEARCH_H_
17
18
#include "av1/encoder/encoder.h"
19
20
#ifdef __cplusplus
21
extern "C" {
22
#endif
23
24
/*! \brief Variables related to intra-mode search during inter frame coding.
25
 *
26
 * \ingroup intra_mode_search
27
 * This is a set of variables used during intra-mode search for inter frames.
28
 * This includes an histogram of gradient speed features and a cache of uv
29
 * prediction to avoid repeated search of chroma prediction.
30
 */
31
typedef struct IntraModeSearchState {
32
  /*!
33
   * \brief The best luma intra-mode found so far
34
   */
35
  PREDICTION_MODE best_intra_mode;
36
37
  /** \name Speed feature variables
38
   * Variables to help with pruning some luma intra-modes during inter frame
39
   * coding process.
40
   */
41
  /**@{*/
42
  /*!
43
   * \brief Whether to terminate all intra mode search.
44
   */
45
  int skip_intra_modes;
46
  /*!
47
   * \brief Whether a directional mode is pruned.
48
   */
49
  uint8_t directional_mode_skip_mask[INTRA_MODES];
50
  /*!
51
   * \brief Whether \ref directional_mode_skip_mask is valid for pruning.
52
   */
53
  int dir_mode_skip_mask_ready;
54
  /**@}*/
55
56
  /** \name Chroma mode search cache
57
   * A cache of the best chroma prediction mode to avoid having to search for
58
   * chroma predictions repeatedly in \ref
59
   * av1_search_intra_uv_modes_in_interframe()
60
   */
61
  /**@{*/
62
  int rate_uv_intra;          /*!< \brief Total rate to transmit uv_mode */
63
  int rate_uv_tokenonly;      /*!< \brief Rate transmit txfm tokens */
64
  int64_t dist_uvs;           /*!< \brief Distortion of the uv_mode's recon */
65
  int skip_uvs;               /*!< \brief Whether the uv txfm is skippable */
66
  UV_PREDICTION_MODE mode_uv; /*!< \brief The best uv mode */
67
  PALETTE_MODE_INFO pmi_uv;   /*!< \brief Color map if mode_uv is palette */
68
  int8_t uv_angle_delta;      /*!< \brief Angle delta if mode_uv directional */
69
  /**@}*/
70
} IntraModeSearchState;
71
72
/*!\brief Evaluate a given luma intra-mode for inter frames.
73
 *
74
 * \ingroup intra_mode_search
75
 * \callgraph
76
 * \callergraph
77
 * This function handles an intra-mode luma prediction when the current frame
78
 * is an inter frame. This is the intra-mode counterpart of handle_inter_mode.
79
 * This function performs an intra luma prediction using the mode specified by
80
 * x->e_mbd.mi[0]->mode. This function does *not* support palette mode
81
 * prediction in the luma channel.
82
 *
83
 * \param[in,out]    intra_search_state Structure to intra search state.
84
 * \param[in]        cpi                Top-level encoder structure.
85
 * \param[in,out]    x                  Pointer to structure holding all the
86
 *                                      data for the current macroblock.
87
 * \param[in]        bsize              Current partition block size.
88
 * \param[in]        ref_frame_cost     The entropy cost for signaling that the
89
 *                                      current ref frame is an intra frame.
90
 * \param[in]        ctx                Structure to hold the number of 4x4 blks
91
 *                                      to copy tx_type and txfm_skip arrays.
92
 * \param[out]       rd_stats_y         Struct to keep track of the current
93
 *                                      intra-mode's rd_stats (luma only).
94
 * \param[in]        best_rd            Best RD seen for this block so far.
95
 * \param[out]       mode_cost_y        The cost needed to signal the current
96
 *                                      intra mode.
97
 * \param[out]       rd_y               The rdcost of the chosen mode.
98
 * \param[in]        best_model_rd      Best model RD seen for this block so far
99
 * \param[in]        top_intra_model_rd Top intra model RD seen for this
100
 *                                      block so far.
101
 *
102
 * \return Returns 1 if a valid intra mode is found, 0 otherwise.
103
 * The corresponding values in x->e_mbd.mi[0], rd_stats_y, mode_cost_y, and
104
 * rd_y are also updated. Moreover, in the first evaluation with directional
105
 * mode, a prune_mask computed with histogram of gradient is also stored in
106
 * intra_search_state.
107
 */
108
int av1_handle_intra_y_mode(IntraModeSearchState *intra_search_state,
109
                            const AV1_COMP *cpi, MACROBLOCK *x,
110
                            BLOCK_SIZE bsize, unsigned int ref_frame_cost,
111
                            const PICK_MODE_CONTEXT *ctx, RD_STATS *rd_stats_y,
112
                            int64_t best_rd, int *mode_cost_y, int64_t *rd_y,
113
                            int64_t *best_model_rd,
114
                            int64_t top_intra_model_rd[]);
115
116
/*!\brief Search through all chroma intra-modes for inter frames.
117
 *
118
 * \ingroup intra_mode_search
119
 * \callgraph
120
 * \callergraph
121
 * This function handles intra-mode chroma prediction when the current frame
122
 * is an inter frame. This is done by calling \ref av1_rd_pick_intra_sbuv_mode
123
 * with some additional book-keeping.
124
 *
125
 * \param[in,out]    intra_search_state Structure to intra search state.
126
 * \param[in]        cpi                Top-level encoder structure.
127
 * \param[in,out]    x                  Pointer to structure holding all the
128
 *                                      data for the current macroblock.
129
 * \param[in]        bsize              Current partition block size.
130
 * \param[out]       rd_stats           Struct to keep track of the current
131
 *                                      intra-mode's rd_stats (all planes).
132
 * \param[out]       rd_stats_y         Struct to keep track of the current
133
 *                                      intra-mode's rd_stats (luma only).
134
 * \param[out]       rd_stats_uv        Struct to keep track of the current
135
 *                                      intra-mode's rd_stats (chroma only).
136
 * \param[in]        best_rd            Best RD seen for this block so far.
137
 *
138
 * \return Returns 1 if a valid intra mode is found, 0 otherwise.
139
 * The corresponding values in x->e_mbd.mi[0], rd_stats(_y|_uv)  are also
140
 * updated. Moreover, in the first evocation of the function, the chroma intra
141
 * mode result is cached in intra_search_state to be used in subsequent calls.
142
 */
143
int av1_search_intra_uv_modes_in_interframe(
144
    IntraModeSearchState *intra_search_state, const AV1_COMP *cpi,
145
    MACROBLOCK *x, BLOCK_SIZE bsize, RD_STATS *rd_stats,
146
    const RD_STATS *rd_stats_y, RD_STATS *rd_stats_uv, int64_t best_rd);
147
148
/*!\brief Evaluate luma palette mode for inter frames.
149
 *
150
 * \ingroup intra_mode_search
151
 * \callergraph
152
 * \callgraph
153
 * This function handles luma palette mode when the current frame is an
154
 * inter frame.
155
 *
156
 * \param[in]    intra_search_state Structure to hold the best luma intra mode
157
 *                                  and cache chroma prediction for speed up.
158
 * \param[in]    cpi                Top-level encoder structure.
159
 * \param[in]    x                  Pointer to structure holding all the data
160
 *                                  for the current macroblock.
161
 * \param[in]    bsize              Current partition block size.
162
 * \param[in]    ref_frame_cost     The entropy cost for signaling that the
163
 *                                  current ref frame is an intra frame.
164
 * \param[in]    ctx                Structure to hold the number of 4x4 blks to
165
 *                                  copy the tx_type and txfm_skip arrays.
166
 * \param[in]    this_rd_cost       Struct to keep track of palette mode's
167
 *                                  rd_stats.
168
 * \param[in]    best_rd            Best RD seen for this block so far.
169
 *
170
 * \return Returns whether luma palette mode can skip the txfm. The
171
 * corresponding mbmi, this_rd_costs, intra_search_state, and tx_type arrays in
172
 * ctx are also updated.
173
 */
174
int av1_search_palette_mode(IntraModeSearchState *intra_search_state,
175
                            const AV1_COMP *cpi, MACROBLOCK *x,
176
                            BLOCK_SIZE bsize, unsigned int ref_frame_cost,
177
                            PICK_MODE_CONTEXT *ctx, RD_STATS *this_rd_cost,
178
                            int64_t best_rd);
179
180
/*!\brief Perform intra-mode search on luma channels for intra frames.
181
 *
182
 * \ingroup intra_mode_search
183
 * \callgraph
184
 * \callergraph
185
 * This function performs intra-mode search on the luma channel when the
186
 * current frame is intra-only. This function does not search intrabc mode,
187
 * but it does search palette and filter_intra.
188
 *
189
 * \param[in]    cpi                Top-level encoder structure.
190
 * \param[in]    x                  Pointer to structure holding all the data
191
 *                                  for the current macroblock.
192
 * \param[in]    rate               The total rate needed to predict the current
193
 *                                  chroma block.
194
 * \param[in]    rate_tokenonly     The rate without the cost of sending the
195
 *                                  prediction modes.
196
 *                                  chroma block.
197
 *                                  after the reconstruction.
198
 * \param[in]    distortion         The chroma distortion of the best prediction
199
 *                                  after the reconstruction.
200
 * \param[in]    skippable          Whether we can skip txfm process.
201
 * \param[in]    bsize              Current partition block size.
202
 * \param[in]    best_rd            Best RD seen for this block so far.
203
 * \param[in]    ctx                Structure to hold the number of 4x4 blks to
204
 *                                  copy the tx_type and txfm_skip arrays.
205
 *
206
 * \return Returns the rd_cost if this function finds a mode better than
207
 * best_rd, otherwise returns INT64_MAX. This also updates the mbmi, the rate
208
 * and distortion, and the tx_type arrays in ctx.
209
 */
210
int64_t av1_rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
211
                                   int *rate, int *rate_tokenonly,
212
                                   int64_t *distortion, int *skippable,
213
                                   BLOCK_SIZE bsize, int64_t best_rd,
214
                                   PICK_MODE_CONTEXT *ctx);
215
216
/*!\brief Perform intra-mode search on chroma channels.
217
 *
218
 * \ingroup intra_mode_search
219
 * \callergraph
220
 * \callgraph
221
 * This function performs intra-mode search on the chroma channels. Just like
222
 * \ref av1_rd_pick_intra_sby_mode(), this function searches over palette mode
223
 * (filter_intra is not available on chroma planes). Unlike \ref
224
 * av1_rd_pick_intra_sby_mode() this function is used by both inter and intra
225
 * frames.
226
 *
227
 * \param[in]    cpi                Top-level encoder structure.
228
 * \param[in]    x                  Pointer to structure holding all the data
229
 *                                  for the current macroblock.
230
 * \param[in]    rate               The total rate needed to predict the current
231
 *                                  chroma block.
232
 * \param[in]    rate_tokenonly     The rate without the cost of sending the
233
 *                                  prediction modes.
234
 *                                  chroma block.
235
 *                                  after the reconstruction.
236
 * \param[in]    distortion         The chroma distortion of the best prediction
237
 *                                  after the reconstruction.
238
 * \param[in]    skippable          Whether we can skip txfm process.
239
 * \param[in]    bsize              Current partition block size.
240
 * \param[in]    max_tx_size        The maximum tx_size available
241
 *
242
 * \return Returns the rd_cost of the best uv mode found. This also updates the
243
 * mbmi, the rate and distortion, distortion.
244
 */
245
int64_t av1_rd_pick_intra_sbuv_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
246
                                    int *rate, int *rate_tokenonly,
247
                                    int64_t *distortion, int *skippable,
248
                                    BLOCK_SIZE bsize, TX_SIZE max_tx_size);
249
250
/*! \brief Return the number of colors in src. Used by palette mode.
251
 */
252
void av1_count_colors(const uint8_t *src, int stride, int rows, int cols,
253
                      int *val_count, int *num_colors);
254
255
/*! \brief See \ref av1_count_colors(), but for highbd.
256
 */
257
void av1_count_colors_highbd(const uint8_t *src8, int stride, int rows,
258
                             int cols, int bit_depth, int *val_count,
259
                             int *val_count_8bit, int *num_color_bins,
260
                             int *num_colors);
261
262
/*! \brief Initializes the \ref IntraModeSearchState struct.
263
 */
264
static AOM_INLINE void init_intra_mode_search_state(
265
0
    IntraModeSearchState *intra_search_state) {
266
0
  memset(intra_search_state, 0, sizeof(*intra_search_state));
267
0
  intra_search_state->rate_uv_intra = INT_MAX;
268
0
}
Unexecuted instantiation: encoder.c:init_intra_mode_search_state
Unexecuted instantiation: palette.c:init_intra_mode_search_state
Unexecuted instantiation: rdopt.c:init_intra_mode_search_state
Unexecuted instantiation: intra_mode_search.c:init_intra_mode_search_state
269
270
/*! \brief set the luma intra mode and delta angles for a given mode index.
271
 * The total number of luma intra mode is LUMA_MODE_COUNT = 61.
272
 * The first 13 modes are from DC_PRED to PAETH_PRED, followed by directional
273
 * modes. Each of the main 8 directional modes have 6 = MAX_ANGLE_DELTA * 2
274
 * delta angles.
275
 * \param[in]    mode_idx           mode index in intra mode decision
276
 *                                  process.
277
 * \param[in]    mbmi               Pointer to structure holding
278
 *                                  the mode info for the current macroblock.
279
 */
280
void set_y_mode_and_delta_angle(const int mode_idx, MB_MODE_INFO *const mbmi);
281
282
/*! \brief prune luma intra mode based on the model rd.
283
 * \param[in]    this_model_rd              model rd for current mode.
284
 * \param[in]    best_model_rd              Best model RD seen for this block so
285
 *                                          far.
286
 * \param[in]    top_intra_model_rd         Top intra model RD seen for this
287
 *                                          block so far.
288
 * \param[in]    max_model_cnt_allowed      The maximum number of top intra
289
 *                                          model RD allowed.
290
 * \param[in]    model_rd_index_for_pruning Index of the candidate used for
291
 *                                          pruning based on model rd.
292
 */
293
int prune_intra_y_mode(int64_t this_model_rd, int64_t *best_model_rd,
294
                       int64_t top_intra_model_rd[], int max_model_cnt_allowed,
295
                       int model_rd_index_for_pruning);
296
297
#ifdef __cplusplus
298
}  // extern "C"
299
#endif
300
301
#endif  // AOM_AV1_ENCODER_INTRA_MODE_SEARCH_H_