Coverage Report

Created: 2026-04-01 07:24

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/aom/av1/encoder/intra_mode_search.h
Line
Count
Source
1
/*
2
 * Copyright (c) 2020, Alliance for Open Media. All rights reserved.
3
 *
4
 * This source code is subject to the terms of the BSD 2 Clause License and
5
 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6
 * was not distributed with this source code in the LICENSE file, you can
7
 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8
 * Media Patent License 1.0 was not distributed with this source code in the
9
 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10
 */
11
12
/*!\file
13
 * \brief Declares high level functions to search through intra modes.
14
 */
15
#ifndef AOM_AV1_ENCODER_INTRA_MODE_SEARCH_H_
16
#define AOM_AV1_ENCODER_INTRA_MODE_SEARCH_H_
17
18
#include <stdbool.h>
19
20
#include "av1/encoder/encoder.h"
21
22
#ifdef __cplusplus
23
extern "C" {
24
#endif
25
26
/*! \brief Variables related to intra-mode search during inter frame coding.
27
 *
28
 * \ingroup intra_mode_search
29
 * This is a set of variables used during intra-mode search for inter frames.
30
 * This includes an histogram of gradient speed features and a cache of uv
31
 * prediction to avoid repeated search of chroma prediction.
32
 */
33
typedef struct IntraModeSearchState {
34
  /*!
35
   * \brief The best luma intra-mode found so far
36
   */
37
  PREDICTION_MODE best_intra_mode;
38
39
  /** \name Speed feature variables
40
   * Variables to help with pruning some luma intra-modes during inter frame
41
   * coding process.
42
   */
43
  /**@{*/
44
  /*!
45
   * \brief Whether to terminate all intra mode search.
46
   */
47
  int skip_intra_modes;
48
  /*!
49
   * \brief Whether a directional mode is pruned.
50
   */
51
  uint8_t directional_mode_skip_mask[INTRA_MODES];
52
  /*!
53
   * \brief Whether \ref directional_mode_skip_mask is valid for pruning.
54
   */
55
  int dir_mode_skip_mask_ready;
56
  /**@}*/
57
58
  /** \name Chroma mode search cache
59
   * A cache of the best chroma prediction mode to avoid having to search for
60
   * chroma predictions repeatedly in \ref
61
   * av1_search_intra_uv_modes_in_interframe()
62
   */
63
  /**@{*/
64
  int rate_uv_intra;          /*!< \brief Total rate to transmit uv_mode */
65
  int rate_uv_tokenonly;      /*!< \brief Rate transmit txfm tokens */
66
  int64_t dist_uvs;           /*!< \brief Distortion of the uv_mode's recon */
67
  uint8_t skip_uvs;           /*!< \brief Whether the uv txfm is skippable */
68
  UV_PREDICTION_MODE mode_uv; /*!< \brief The best uv mode */
69
  PALETTE_MODE_INFO pmi_uv;   /*!< \brief Color map if mode_uv is palette */
70
  int8_t uv_angle_delta;      /*!< \brief Angle delta if mode_uv directional */
71
  /**@}*/
72
} IntraModeSearchState;
73
74
/*!\brief Evaluate a given luma intra-mode for inter frames.
75
 *
76
 * \ingroup intra_mode_search
77
 * \callgraph
78
 * \callergraph
79
 * This function handles an intra-mode luma prediction when the current frame
80
 * is an inter frame. This is the intra-mode counterpart of handle_inter_mode.
81
 * This function performs an intra luma prediction using the mode specified by
82
 * x->e_mbd.mi[0]->mode. This function does *not* support palette mode
83
 * prediction in the luma channel.
84
 *
85
 * \param[in,out]    intra_search_state Structure to intra search state.
86
 * \param[in]        cpi                Top-level encoder structure.
87
 * \param[in,out]    x                  Pointer to structure holding all the
88
 *                                      data for the current macroblock.
89
 * \param[in]        bsize              Current partition block size.
90
 * \param[in]        ref_frame_cost     The entropy cost for signaling that the
91
 *                                      current ref frame is an intra frame.
92
 * \param[in]        ctx                Structure to hold the number of 4x4 blks
93
 *                                      to copy tx_type and txfm_skip arrays.
94
 * \param[out]       rd_stats_y         Struct to keep track of the current
95
 *                                      intra-mode's rd_stats (luma only).
96
 * \param[in]        best_rd            Best RD seen for this block so far.
97
 * \param[out]       mode_cost_y        The cost needed to signal the current
98
 *                                      intra mode.
99
 * \param[out]       rd_y               The rdcost of the chosen mode.
100
 * \param[in]        best_model_rd      Best model RD seen for this block so far
101
 * \param[in]        top_intra_model_rd Top intra model RD seen for this
102
 *                                      block so far.
103
 *
104
 * \return Returns 1 if a valid intra mode is found, 0 otherwise.
105
 * The corresponding values in x->e_mbd.mi[0], rd_stats_y, mode_cost_y, and
106
 * rd_y are also updated. Moreover, in the first evaluation with directional
107
 * mode, a prune_mask computed with histogram of gradient is also stored in
108
 * intra_search_state.
109
 */
110
int av1_handle_intra_y_mode(IntraModeSearchState *intra_search_state,
111
                            const AV1_COMP *cpi, MACROBLOCK *x,
112
                            BLOCK_SIZE bsize, unsigned int ref_frame_cost,
113
                            const PICK_MODE_CONTEXT *ctx, RD_STATS *rd_stats_y,
114
                            int64_t best_rd, int *mode_cost_y, int64_t *rd_y,
115
                            int64_t *best_model_rd,
116
                            int64_t top_intra_model_rd[]);
117
118
/*!\brief Search through all chroma intra-modes for inter frames.
119
 *
120
 * \ingroup intra_mode_search
121
 * \callgraph
122
 * \callergraph
123
 * This function handles intra-mode chroma prediction when the current frame
124
 * is an inter frame. This is done by calling \ref av1_rd_pick_intra_sbuv_mode
125
 * with some additional book-keeping.
126
 *
127
 * \param[in,out]    intra_search_state Structure to intra search state.
128
 * \param[in]        cpi                Top-level encoder structure.
129
 * \param[in,out]    x                  Pointer to structure holding all the
130
 *                                      data for the current macroblock.
131
 * \param[in]        bsize              Current partition block size.
132
 * \param[out]       rd_stats           Struct to keep track of the current
133
 *                                      intra-mode's rd_stats (all planes).
134
 * \param[out]       rd_stats_y         Struct to keep track of the current
135
 *                                      intra-mode's rd_stats (luma only).
136
 * \param[out]       rd_stats_uv        Struct to keep track of the current
137
 *                                      intra-mode's rd_stats (chroma only).
138
 * \param[in]        best_rd            Best RD seen for this block so far.
139
 *
140
 * \return Returns 1 if a valid intra mode is found, 0 otherwise.
141
 * The corresponding values in x->e_mbd.mi[0], rd_stats(_y|_uv)  are also
142
 * updated. Moreover, in the first evocation of the function, the chroma intra
143
 * mode result is cached in intra_search_state to be used in subsequent calls.
144
 */
145
int av1_search_intra_uv_modes_in_interframe(
146
    IntraModeSearchState *intra_search_state, const AV1_COMP *cpi,
147
    MACROBLOCK *x, BLOCK_SIZE bsize, RD_STATS *rd_stats,
148
    const RD_STATS *rd_stats_y, RD_STATS *rd_stats_uv, int64_t best_rd);
149
150
/*!\brief Evaluate luma palette mode for inter frames.
151
 *
152
 * \ingroup intra_mode_search
153
 * \callergraph
154
 * \callgraph
155
 * This function handles luma palette mode when the current frame is an
156
 * inter frame.
157
 *
158
 * \param[in]    intra_search_state Structure to hold the best luma intra mode
159
 *                                  and cache chroma prediction for speed up.
160
 * \param[in]    cpi                Top-level encoder structure.
161
 * \param[in]    x                  Pointer to structure holding all the data
162
 *                                  for the current macroblock.
163
 * \param[in]    bsize              Current partition block size.
164
 * \param[in]    ref_frame_cost     The entropy cost for signaling that the
165
 *                                  current ref frame is an intra frame.
166
 * \param[in]    ctx                Structure to hold the number of 4x4 blks to
167
 *                                  copy the tx_type and txfm_skip arrays.
168
 * \param[in]    this_rd_cost       Struct to keep track of palette mode's
169
 *                                  rd_stats.
170
 * \param[in]    best_rd            Best RD seen for this block so far.
171
 *
172
 * \return Returns whether luma palette mode can skip the txfm. The
173
 * corresponding mbmi, this_rd_costs, intra_search_state, and tx_type arrays in
174
 * ctx are also updated.
175
 */
176
int av1_search_palette_mode(IntraModeSearchState *intra_search_state,
177
                            const AV1_COMP *cpi, MACROBLOCK *x,
178
                            BLOCK_SIZE bsize, unsigned int ref_frame_cost,
179
                            PICK_MODE_CONTEXT *ctx, RD_STATS *this_rd_cost,
180
                            int64_t best_rd);
181
182
/*!\brief Evaluate luma palette mode for inter frames.
183
 *
184
 * \ingroup intra_mode_search
185
 * \callergraph
186
 * \callgraph
187
 * This function handles luma palette mode when the current frame is an
188
 * inter frame.
189
 *
190
 * \param[in]    cpi                Top-level encoder structure.
191
 * \param[in]    x                  Pointer to structure holding all the data
192
 *                                  for the current macroblock.
193
 * \param[in]    bsize              Current partition block size.
194
 * \param[in]    ref_frame_cost     The entropy cost for signaling that the
195
 *                                  current ref frame is an intra frame.
196
 * \param[in]    ctx                Structure to hold the number of 4x4 blks to
197
 *                                  copy the tx_type and txfm_skip arrays.
198
 * \param[in]    this_rd_cost       Struct to keep track of palette mode's
199
 *                                  rd_stats.
200
 * \param[in]    best_rd            Best RD seen for this block so far.
201
 */
202
void av1_search_palette_mode_luma(const AV1_COMP *cpi, MACROBLOCK *x,
203
                                  BLOCK_SIZE bsize, unsigned int ref_frame_cost,
204
                                  PICK_MODE_CONTEXT *ctx,
205
                                  RD_STATS *this_rd_cost, int64_t best_rd);
206
207
/*!\brief Perform intra-mode search on luma channels for intra frames.
208
 *
209
 * \ingroup intra_mode_search
210
 * \callgraph
211
 * \callergraph
212
 * This function performs intra-mode search on the luma channel when the
213
 * current frame is intra-only. This function does not search intrabc mode,
214
 * but it does search palette and filter_intra.
215
 *
216
 * \param[in]    cpi                Top-level encoder structure.
217
 * \param[in]    x                  Pointer to structure holding all the data
218
 *                                  for the current macroblock.
219
 * \param[in]    rate               The total rate needed to predict the current
220
 *                                  chroma block.
221
 * \param[in]    rate_tokenonly     The rate without the cost of sending the
222
 *                                  prediction modes.
223
 *                                  chroma block.
224
 *                                  after the reconstruction.
225
 * \param[in]    distortion         The chroma distortion of the best prediction
226
 *                                  after the reconstruction.
227
 * \param[in]    skippable          Whether we can skip txfm process.
228
 * \param[in]    bsize              Current partition block size.
229
 * \param[in]    best_rd            Best RD seen for this block so far.
230
 * \param[in]    ctx                Structure to hold the number of 4x4 blks to
231
 *                                  copy the tx_type and txfm_skip arrays.
232
 *
233
 * \return Returns the rd_cost if this function finds a mode better than
234
 * best_rd, otherwise returns INT64_MAX. This also updates the mbmi, the rate
235
 * and distortion, and the tx_type arrays in ctx.
236
 */
237
int64_t av1_rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
238
                                   int *rate, int *rate_tokenonly,
239
                                   int64_t *distortion, uint8_t *skippable,
240
                                   BLOCK_SIZE bsize, int64_t best_rd,
241
                                   PICK_MODE_CONTEXT *ctx);
242
243
/*!\brief Perform intra-mode search on chroma channels.
244
 *
245
 * \ingroup intra_mode_search
246
 * \callergraph
247
 * \callgraph
248
 * This function performs intra-mode search on the chroma channels. Just like
249
 * \ref av1_rd_pick_intra_sby_mode(), this function searches over palette mode
250
 * (filter_intra is not available on chroma planes). Unlike \ref
251
 * av1_rd_pick_intra_sby_mode() this function is used by both inter and intra
252
 * frames.
253
 *
254
 * \param[in]    cpi                Top-level encoder structure.
255
 * \param[in]    x                  Pointer to structure holding all the data
256
 *                                  for the current macroblock.
257
 * \param[in]    rate               The total rate needed to predict the current
258
 *                                  chroma block.
259
 * \param[in]    rate_tokenonly     The rate without the cost of sending the
260
 *                                  prediction modes.
261
 *                                  chroma block.
262
 *                                  after the reconstruction.
263
 * \param[in]    distortion         The chroma distortion of the best prediction
264
 *                                  after the reconstruction.
265
 * \param[in]    skippable          Whether we can skip txfm process.
266
 * \param[in]    bsize              Current partition block size.
267
 * \param[in]    max_tx_size        The maximum tx_size available
268
 *
269
 * \return Returns the rd_cost of the best uv mode found. This also updates the
270
 * mbmi, the rate and distortion, distortion.
271
 */
272
int64_t av1_rd_pick_intra_sbuv_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
273
                                    int *rate, int *rate_tokenonly,
274
                                    int64_t *distortion, uint8_t *skippable,
275
                                    BLOCK_SIZE bsize, TX_SIZE max_tx_size);
276
277
/*! \brief Return the number of colors in src. Used by palette mode.
278
 */
279
void av1_count_colors(const uint8_t *src, int stride, int rows, int cols,
280
                      int *val_count, int *num_colors);
281
282
/*! \brief See \ref av1_count_colors(), but for highbd.
283
 */
284
void av1_count_colors_highbd(const uint8_t *src8, int stride, int rows,
285
                             int cols, int bit_depth, int *val_count,
286
                             int *val_count_8bit, int *num_color_bins,
287
                             int *num_colors);
288
289
/*! \brief Set \a *num_colors to the number of colors in src.
290
291
   Exits early if the number of colors exceeds a specified threshold.
292
   Used by screen content detection mode 2.
293
   \return Returns true if the number of colors does not exceed the threshold.
294
   \note If the function returns true, \a *num_colors is accurate (and is <=
295
   \a num_colors_threshold). If the function returns false, the function exited
296
   early, and \a *num_colors may be smaller than the actual number of colors
297
   (and is > \a num_colors_threshold).
298
 */
299
bool av1_count_colors_with_threshold(const uint8_t *src, int stride, int rows,
300
                                     int cols, int num_colors_threshold,
301
                                     int *num_colors);
302
303
/*! \brief Initializes the \ref IntraModeSearchState struct.
304
 */
305
static inline void init_intra_mode_search_state(
306
0
    IntraModeSearchState *intra_search_state) {
307
0
  memset(intra_search_state, 0, sizeof(*intra_search_state));
308
  intra_search_state->rate_uv_intra = INT_MAX;
309
0
}
Unexecuted instantiation: encoder.c:init_intra_mode_search_state
Unexecuted instantiation: palette.c:init_intra_mode_search_state
Unexecuted instantiation: rdopt.c:init_intra_mode_search_state
Unexecuted instantiation: nonrd_pickmode.c:init_intra_mode_search_state
Unexecuted instantiation: intra_mode_search.c:init_intra_mode_search_state
310
311
/*! \brief set the luma intra mode and delta angles for a given mode index.
312
 *
313
 * The total number of luma intra mode is LUMA_MODE_COUNT = 61.
314
 * The first 13 modes are from DC_PRED to PAETH_PRED, followed by directional
315
 * modes. Each of the main 8 directional modes have 6 = MAX_ANGLE_DELTA * 2
316
 * delta angles.
317
 * \param[in]    mode_idx                  mode index in intra mode decision
318
 *                                         process.
319
 * \param[in]    mbmi                      Pointer to structure holding the mode
320
 *                                         info for the current macroblock.
321
 * \param[in]    reorder_delta_angle_eval  Indicates whether to reorder the
322
 *                                         evaluation of delta angle modes.
323
 */
324
void set_y_mode_and_delta_angle(const int mode_idx, MB_MODE_INFO *const mbmi,
325
                                int reorder_delta_angle_eval);
326
#ifdef __cplusplus
327
}  // extern "C"
328
#endif
329
330
#endif  // AOM_AV1_ENCODER_INTRA_MODE_SEARCH_H_