/src/aom/av1/encoder/intra_mode_search.h
Line | Count | Source (jump to first uncovered line) |
1 | | /* |
2 | | * Copyright (c) 2020, Alliance for Open Media. All rights reserved |
3 | | * |
4 | | * This source code is subject to the terms of the BSD 2 Clause License and |
5 | | * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License |
6 | | * was not distributed with this source code in the LICENSE file, you can |
7 | | * obtain it at www.aomedia.org/license/software. If the Alliance for Open |
8 | | * Media Patent License 1.0 was not distributed with this source code in the |
9 | | * PATENTS file, you can obtain it at www.aomedia.org/license/patent. |
10 | | */ |
11 | | |
12 | | /*!\file |
13 | | * \brief Declares high level functions to search through intra modes. |
14 | | */ |
15 | | #ifndef AOM_AV1_ENCODER_INTRA_MODE_SEARCH_H_ |
16 | | #define AOM_AV1_ENCODER_INTRA_MODE_SEARCH_H_ |
17 | | |
18 | | #include "av1/encoder/encoder.h" |
19 | | |
20 | | #ifdef __cplusplus |
21 | | extern "C" { |
22 | | #endif |
23 | | |
24 | | /*! \brief Variables related to intra-mode search during inter frame coding. |
25 | | * |
26 | | * \ingroup intra_mode_search |
27 | | * This is a set of variables used during intra-mode search for inter frames. |
28 | | * This includes an histogram of gradient speed features and a cache of uv |
29 | | * prediction to avoid repeated search of chroma prediction. |
30 | | */ |
31 | | typedef struct IntraModeSearchState { |
32 | | /*! |
33 | | * \brief The best luma intra-mode found so far |
34 | | */ |
35 | | PREDICTION_MODE best_intra_mode; |
36 | | |
37 | | /** \name Speed feature variables |
38 | | * Variables to help with pruning some luma intra-modes during inter frame |
39 | | * coding process. |
40 | | */ |
41 | | /**@{*/ |
42 | | /*! |
43 | | * \brief Whether to terminate all intra mode search. |
44 | | */ |
45 | | int skip_intra_modes; |
46 | | /*! |
47 | | * \brief Whether a directional mode is pruned. |
48 | | */ |
49 | | uint8_t directional_mode_skip_mask[INTRA_MODES]; |
50 | | /*! |
51 | | * \brief Whether \ref directional_mode_skip_mask is valid for pruning. |
52 | | */ |
53 | | int dir_mode_skip_mask_ready; |
54 | | /**@}*/ |
55 | | |
56 | | /** \name Chroma mode search cache |
57 | | * A cache of the best chroma prediction mode to avoid having to search for |
58 | | * chroma predictions repeatedly in \ref |
59 | | * av1_search_intra_uv_modes_in_interframe() |
60 | | */ |
61 | | /**@{*/ |
62 | | int rate_uv_intra; /*!< \brief Total rate to transmit uv_mode */ |
63 | | int rate_uv_tokenonly; /*!< \brief Rate transmit txfm tokens */ |
64 | | int64_t dist_uvs; /*!< \brief Distortion of the uv_mode's recon */ |
65 | | int skip_uvs; /*!< \brief Whether the uv txfm is skippable */ |
66 | | UV_PREDICTION_MODE mode_uv; /*!< \brief The best uv mode */ |
67 | | PALETTE_MODE_INFO pmi_uv; /*!< \brief Color map if mode_uv is palette */ |
68 | | int8_t uv_angle_delta; /*!< \brief Angle delta if mode_uv directional */ |
69 | | /**@}*/ |
70 | | } IntraModeSearchState; |
71 | | |
72 | | /*!\brief Evaluate a given luma intra-mode for inter frames. |
73 | | * |
74 | | * \ingroup intra_mode_search |
75 | | * \callgraph |
76 | | * \callergraph |
77 | | * This function handles an intra-mode luma prediction when the current frame |
78 | | * is an inter frame. This is the intra-mode counterpart of handle_inter_mode. |
79 | | * This function performs an intra luma prediction using the mode specified by |
80 | | * x->e_mbd.mi[0]->mode. This function does *not* support palette mode |
81 | | * prediction in the luma channel. |
82 | | * |
83 | | * \param[in,out] intra_search_state Structure to intra search state. |
84 | | * \param[in] cpi Top-level encoder structure. |
85 | | * \param[in,out] x Pointer to structure holding all the |
86 | | * data for the current macroblock. |
87 | | * \param[in] bsize Current partition block size. |
88 | | * \param[in] ref_frame_cost The entropy cost for signaling that the |
89 | | * current ref frame is an intra frame. |
90 | | * \param[in] ctx Structure to hold the number of 4x4 blks |
91 | | * to copy tx_type and txfm_skip arrays. |
92 | | * \param[out] rd_stats_y Struct to keep track of the current |
93 | | * intra-mode's rd_stats (luma only). |
94 | | * \param[in] best_rd Best RD seen for this block so far. |
95 | | * \param[out] mode_cost_y The cost needed to signal the current |
96 | | * intra mode. |
97 | | * \param[out] rd_y The rdcost of the chosen mode. |
98 | | * \param[in] best_model_rd Best model RD seen for this block so far |
99 | | * \param[in] top_intra_model_rd Top intra model RD seen for this |
100 | | * block so far. |
101 | | * |
102 | | * \return Returns 1 if a valid intra mode is found, 0 otherwise. |
103 | | * The corresponding values in x->e_mbd.mi[0], rd_stats_y, mode_cost_y, and |
104 | | * rd_y are also updated. Moreover, in the first evaluation with directional |
105 | | * mode, a prune_mask computed with histogram of gradient is also stored in |
106 | | * intra_search_state. |
107 | | */ |
108 | | int av1_handle_intra_y_mode(IntraModeSearchState *intra_search_state, |
109 | | const AV1_COMP *cpi, MACROBLOCK *x, |
110 | | BLOCK_SIZE bsize, unsigned int ref_frame_cost, |
111 | | const PICK_MODE_CONTEXT *ctx, RD_STATS *rd_stats_y, |
112 | | int64_t best_rd, int *mode_cost_y, int64_t *rd_y, |
113 | | int64_t *best_model_rd, |
114 | | int64_t top_intra_model_rd[]); |
115 | | |
116 | | /*!\brief Search through all chroma intra-modes for inter frames. |
117 | | * |
118 | | * \ingroup intra_mode_search |
119 | | * \callgraph |
120 | | * \callergraph |
121 | | * This function handles intra-mode chroma prediction when the current frame |
122 | | * is an inter frame. This is done by calling \ref av1_rd_pick_intra_sbuv_mode |
123 | | * with some additional book-keeping. |
124 | | * |
125 | | * \param[in,out] intra_search_state Structure to intra search state. |
126 | | * \param[in] cpi Top-level encoder structure. |
127 | | * \param[in,out] x Pointer to structure holding all the |
128 | | * data for the current macroblock. |
129 | | * \param[in] bsize Current partition block size. |
130 | | * \param[out] rd_stats Struct to keep track of the current |
131 | | * intra-mode's rd_stats (all planes). |
132 | | * \param[out] rd_stats_y Struct to keep track of the current |
133 | | * intra-mode's rd_stats (luma only). |
134 | | * \param[out] rd_stats_uv Struct to keep track of the current |
135 | | * intra-mode's rd_stats (chroma only). |
136 | | * \param[in] best_rd Best RD seen for this block so far. |
137 | | * |
138 | | * \return Returns 1 if a valid intra mode is found, 0 otherwise. |
139 | | * The corresponding values in x->e_mbd.mi[0], rd_stats(_y|_uv) are also |
140 | | * updated. Moreover, in the first evocation of the function, the chroma intra |
141 | | * mode result is cached in intra_search_state to be used in subsequent calls. |
142 | | */ |
143 | | int av1_search_intra_uv_modes_in_interframe( |
144 | | IntraModeSearchState *intra_search_state, const AV1_COMP *cpi, |
145 | | MACROBLOCK *x, BLOCK_SIZE bsize, RD_STATS *rd_stats, |
146 | | const RD_STATS *rd_stats_y, RD_STATS *rd_stats_uv, int64_t best_rd); |
147 | | |
148 | | /*!\brief Evaluate luma palette mode for inter frames. |
149 | | * |
150 | | * \ingroup intra_mode_search |
151 | | * \callergraph |
152 | | * \callgraph |
153 | | * This function handles luma palette mode when the current frame is an |
154 | | * inter frame. |
155 | | * |
156 | | * \param[in] intra_search_state Structure to hold the best luma intra mode |
157 | | * and cache chroma prediction for speed up. |
158 | | * \param[in] cpi Top-level encoder structure. |
159 | | * \param[in] x Pointer to structure holding all the data |
160 | | * for the current macroblock. |
161 | | * \param[in] bsize Current partition block size. |
162 | | * \param[in] ref_frame_cost The entropy cost for signaling that the |
163 | | * current ref frame is an intra frame. |
164 | | * \param[in] ctx Structure to hold the number of 4x4 blks to |
165 | | * copy the tx_type and txfm_skip arrays. |
166 | | * \param[in] this_rd_cost Struct to keep track of palette mode's |
167 | | * rd_stats. |
168 | | * \param[in] best_rd Best RD seen for this block so far. |
169 | | * |
170 | | * \return Returns whether luma palette mode can skip the txfm. The |
171 | | * corresponding mbmi, this_rd_costs, intra_search_state, and tx_type arrays in |
172 | | * ctx are also updated. |
173 | | */ |
174 | | int av1_search_palette_mode(IntraModeSearchState *intra_search_state, |
175 | | const AV1_COMP *cpi, MACROBLOCK *x, |
176 | | BLOCK_SIZE bsize, unsigned int ref_frame_cost, |
177 | | PICK_MODE_CONTEXT *ctx, RD_STATS *this_rd_cost, |
178 | | int64_t best_rd); |
179 | | |
180 | | /*!\brief Perform intra-mode search on luma channels for intra frames. |
181 | | * |
182 | | * \ingroup intra_mode_search |
183 | | * \callgraph |
184 | | * \callergraph |
185 | | * This function performs intra-mode search on the luma channel when the |
186 | | * current frame is intra-only. This function does not search intrabc mode, |
187 | | * but it does search palette and filter_intra. |
188 | | * |
189 | | * \param[in] cpi Top-level encoder structure. |
190 | | * \param[in] x Pointer to structure holding all the data |
191 | | * for the current macroblock. |
192 | | * \param[in] rate The total rate needed to predict the current |
193 | | * chroma block. |
194 | | * \param[in] rate_tokenonly The rate without the cost of sending the |
195 | | * prediction modes. |
196 | | * chroma block. |
197 | | * after the reconstruction. |
198 | | * \param[in] distortion The chroma distortion of the best prediction |
199 | | * after the reconstruction. |
200 | | * \param[in] skippable Whether we can skip txfm process. |
201 | | * \param[in] bsize Current partition block size. |
202 | | * \param[in] best_rd Best RD seen for this block so far. |
203 | | * \param[in] ctx Structure to hold the number of 4x4 blks to |
204 | | * copy the tx_type and txfm_skip arrays. |
205 | | * |
206 | | * \return Returns the rd_cost if this function finds a mode better than |
207 | | * best_rd, otherwise returns INT64_MAX. This also updates the mbmi, the rate |
208 | | * and distortion, and the tx_type arrays in ctx. |
209 | | */ |
210 | | int64_t av1_rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x, |
211 | | int *rate, int *rate_tokenonly, |
212 | | int64_t *distortion, int *skippable, |
213 | | BLOCK_SIZE bsize, int64_t best_rd, |
214 | | PICK_MODE_CONTEXT *ctx); |
215 | | |
216 | | /*!\brief Perform intra-mode search on chroma channels. |
217 | | * |
218 | | * \ingroup intra_mode_search |
219 | | * \callergraph |
220 | | * \callgraph |
221 | | * This function performs intra-mode search on the chroma channels. Just like |
222 | | * \ref av1_rd_pick_intra_sby_mode(), this function searches over palette mode |
223 | | * (filter_intra is not available on chroma planes). Unlike \ref |
224 | | * av1_rd_pick_intra_sby_mode() this function is used by both inter and intra |
225 | | * frames. |
226 | | * |
227 | | * \param[in] cpi Top-level encoder structure. |
228 | | * \param[in] x Pointer to structure holding all the data |
229 | | * for the current macroblock. |
230 | | * \param[in] rate The total rate needed to predict the current |
231 | | * chroma block. |
232 | | * \param[in] rate_tokenonly The rate without the cost of sending the |
233 | | * prediction modes. |
234 | | * chroma block. |
235 | | * after the reconstruction. |
236 | | * \param[in] distortion The chroma distortion of the best prediction |
237 | | * after the reconstruction. |
238 | | * \param[in] skippable Whether we can skip txfm process. |
239 | | * \param[in] bsize Current partition block size. |
240 | | * \param[in] max_tx_size The maximum tx_size available |
241 | | * |
242 | | * \return Returns the rd_cost of the best uv mode found. This also updates the |
243 | | * mbmi, the rate and distortion, distortion. |
244 | | */ |
245 | | int64_t av1_rd_pick_intra_sbuv_mode(const AV1_COMP *const cpi, MACROBLOCK *x, |
246 | | int *rate, int *rate_tokenonly, |
247 | | int64_t *distortion, int *skippable, |
248 | | BLOCK_SIZE bsize, TX_SIZE max_tx_size); |
249 | | |
250 | | /*! \brief Return the number of colors in src. Used by palette mode. |
251 | | */ |
252 | | void av1_count_colors(const uint8_t *src, int stride, int rows, int cols, |
253 | | int *val_count, int *num_colors); |
254 | | |
255 | | /*! \brief See \ref av1_count_colors(), but for highbd. |
256 | | */ |
257 | | void av1_count_colors_highbd(const uint8_t *src8, int stride, int rows, |
258 | | int cols, int bit_depth, int *val_count, |
259 | | int *val_count_8bit, int *num_color_bins, |
260 | | int *num_colors); |
261 | | |
262 | | /*! \brief Initializes the \ref IntraModeSearchState struct. |
263 | | */ |
264 | | static AOM_INLINE void init_intra_mode_search_state( |
265 | 0 | IntraModeSearchState *intra_search_state) { |
266 | 0 | memset(intra_search_state, 0, sizeof(*intra_search_state)); |
267 | 0 | intra_search_state->rate_uv_intra = INT_MAX; |
268 | 0 | } Unexecuted instantiation: encoder.c:init_intra_mode_search_state Unexecuted instantiation: palette.c:init_intra_mode_search_state Unexecuted instantiation: rdopt.c:init_intra_mode_search_state Unexecuted instantiation: intra_mode_search.c:init_intra_mode_search_state |
269 | | |
270 | | /*! \brief set the luma intra mode and delta angles for a given mode index. |
271 | | * The total number of luma intra mode is LUMA_MODE_COUNT = 61. |
272 | | * The first 13 modes are from DC_PRED to PAETH_PRED, followed by directional |
273 | | * modes. Each of the main 8 directional modes have 6 = MAX_ANGLE_DELTA * 2 |
274 | | * delta angles. |
275 | | * \param[in] mode_idx mode index in intra mode decision |
276 | | * process. |
277 | | * \param[in] mbmi Pointer to structure holding |
278 | | * the mode info for the current macroblock. |
279 | | */ |
280 | | void set_y_mode_and_delta_angle(const int mode_idx, MB_MODE_INFO *const mbmi); |
281 | | |
282 | | /*! \brief prune luma intra mode based on the model rd. |
283 | | * \param[in] this_model_rd model rd for current mode. |
284 | | * \param[in] best_model_rd Best model RD seen for this block so |
285 | | * far. |
286 | | * \param[in] top_intra_model_rd Top intra model RD seen for this |
287 | | * block so far. |
288 | | * \param[in] max_model_cnt_allowed The maximum number of top intra |
289 | | * model RD allowed. |
290 | | * \param[in] model_rd_index_for_pruning Index of the candidate used for |
291 | | * pruning based on model rd. |
292 | | */ |
293 | | int prune_intra_y_mode(int64_t this_model_rd, int64_t *best_model_rd, |
294 | | int64_t top_intra_model_rd[], int max_model_cnt_allowed, |
295 | | int model_rd_index_for_pruning); |
296 | | |
297 | | #ifdef __cplusplus |
298 | | } // extern "C" |
299 | | #endif |
300 | | |
301 | | #endif // AOM_AV1_ENCODER_INTRA_MODE_SEARCH_H_ |