1 /*
2 * Copyright (c) 2020, Alliance for Open Media. All rights reserved.
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12 /*!\file
13 * \brief Declares high level functions to search through intra modes.
14 */
15 #ifndef AOM_AV1_ENCODER_INTRA_MODE_SEARCH_H_
16 #define AOM_AV1_ENCODER_INTRA_MODE_SEARCH_H_
17
18 #include "av1/encoder/encoder.h"
19
20 #ifdef __cplusplus
21 extern "C" {
22 #endif
23
24 /*! \brief Variables related to intra-mode search during inter frame coding.
25 *
26 * \ingroup intra_mode_search
27 * This is a set of variables used during intra-mode search for inter frames.
28 * This includes an histogram of gradient speed features and a cache of uv
29 * prediction to avoid repeated search of chroma prediction.
30 */
31 typedef struct IntraModeSearchState {
32 /*!
33 * \brief The best luma intra-mode found so far
34 */
35 PREDICTION_MODE best_intra_mode;
36
37 /** \name Speed feature variables
38 * Variables to help with pruning some luma intra-modes during inter frame
39 * coding process.
40 */
41 /**@{*/
42 /*!
43 * \brief Whether to terminate all intra mode search.
44 */
45 int skip_intra_modes;
46 /*!
47 * \brief Whether a directional mode is pruned.
48 */
49 uint8_t directional_mode_skip_mask[INTRA_MODES];
50 /*!
51 * \brief Whether \ref directional_mode_skip_mask is valid for pruning.
52 */
53 int dir_mode_skip_mask_ready;
54 /**@}*/
55
56 /** \name Chroma mode search cache
57 * A cache of the best chroma prediction mode to avoid having to search for
58 * chroma predictions repeatedly in \ref
59 * av1_search_intra_uv_modes_in_interframe()
60 */
61 /**@{*/
62 int rate_uv_intra; /*!< \brief Total rate to transmit uv_mode */
63 int rate_uv_tokenonly; /*!< \brief Rate transmit txfm tokens */
64 int64_t dist_uvs; /*!< \brief Distortion of the uv_mode's recon */
65 uint8_t skip_uvs; /*!< \brief Whether the uv txfm is skippable */
66 UV_PREDICTION_MODE mode_uv; /*!< \brief The best uv mode */
67 PALETTE_MODE_INFO pmi_uv; /*!< \brief Color map if mode_uv is palette */
68 int8_t uv_angle_delta; /*!< \brief Angle delta if mode_uv directional */
69 /**@}*/
70 } IntraModeSearchState;
71
72 /*!\brief Evaluate a given luma intra-mode for inter frames.
73 *
74 * \ingroup intra_mode_search
75 * \callgraph
76 * \callergraph
77 * This function handles an intra-mode luma prediction when the current frame
78 * is an inter frame. This is the intra-mode counterpart of handle_inter_mode.
79 * This function performs an intra luma prediction using the mode specified by
80 * x->e_mbd.mi[0]->mode. This function does *not* support palette mode
81 * prediction in the luma channel.
82 *
83 * \param[in,out] intra_search_state Structure to intra search state.
84 * \param[in] cpi Top-level encoder structure.
85 * \param[in,out] x Pointer to structure holding all the
86 * data for the current macroblock.
87 * \param[in] bsize Current partition block size.
88 * \param[in] ref_frame_cost The entropy cost for signaling that the
89 * current ref frame is an intra frame.
90 * \param[in] ctx Structure to hold the number of 4x4 blks
91 * to copy tx_type and txfm_skip arrays.
92 * \param[out] rd_stats_y Struct to keep track of the current
93 * intra-mode's rd_stats (luma only).
94 * \param[in] best_rd Best RD seen for this block so far.
95 * \param[out] mode_cost_y The cost needed to signal the current
96 * intra mode.
97 * \param[out] rd_y The rdcost of the chosen mode.
98 * \param[in] best_model_rd Best model RD seen for this block so far
99 * \param[in] top_intra_model_rd Top intra model RD seen for this
100 * block so far.
101 *
102 * \return Returns 1 if a valid intra mode is found, 0 otherwise.
103 * The corresponding values in x->e_mbd.mi[0], rd_stats_y, mode_cost_y, and
104 * rd_y are also updated. Moreover, in the first evaluation with directional
105 * mode, a prune_mask computed with histogram of gradient is also stored in
106 * intra_search_state.
107 */
108 int av1_handle_intra_y_mode(IntraModeSearchState *intra_search_state,
109 const AV1_COMP *cpi, MACROBLOCK *x,
110 BLOCK_SIZE bsize, unsigned int ref_frame_cost,
111 const PICK_MODE_CONTEXT *ctx, RD_STATS *rd_stats_y,
112 int64_t best_rd, int *mode_cost_y, int64_t *rd_y,
113 int64_t *best_model_rd,
114 int64_t top_intra_model_rd[]);
115
116 /*!\brief Search through all chroma intra-modes for inter frames.
117 *
118 * \ingroup intra_mode_search
119 * \callgraph
120 * \callergraph
121 * This function handles intra-mode chroma prediction when the current frame
122 * is an inter frame. This is done by calling \ref av1_rd_pick_intra_sbuv_mode
123 * with some additional book-keeping.
124 *
125 * \param[in,out] intra_search_state Structure to intra search state.
126 * \param[in] cpi Top-level encoder structure.
127 * \param[in,out] x Pointer to structure holding all the
128 * data for the current macroblock.
129 * \param[in] bsize Current partition block size.
130 * \param[out] rd_stats Struct to keep track of the current
131 * intra-mode's rd_stats (all planes).
132 * \param[out] rd_stats_y Struct to keep track of the current
133 * intra-mode's rd_stats (luma only).
134 * \param[out] rd_stats_uv Struct to keep track of the current
135 * intra-mode's rd_stats (chroma only).
136 * \param[in] best_rd Best RD seen for this block so far.
137 *
138 * \return Returns 1 if a valid intra mode is found, 0 otherwise.
139 * The corresponding values in x->e_mbd.mi[0], rd_stats(_y|_uv) are also
140 * updated. Moreover, in the first evocation of the function, the chroma intra
141 * mode result is cached in intra_search_state to be used in subsequent calls.
142 */
143 int av1_search_intra_uv_modes_in_interframe(
144 IntraModeSearchState *intra_search_state, const AV1_COMP *cpi,
145 MACROBLOCK *x, BLOCK_SIZE bsize, RD_STATS *rd_stats,
146 const RD_STATS *rd_stats_y, RD_STATS *rd_stats_uv, int64_t best_rd);
147
148 /*!\brief Evaluate luma palette mode for inter frames.
149 *
150 * \ingroup intra_mode_search
151 * \callergraph
152 * \callgraph
153 * This function handles luma palette mode when the current frame is an
154 * inter frame.
155 *
156 * \param[in] intra_search_state Structure to hold the best luma intra mode
157 * and cache chroma prediction for speed up.
158 * \param[in] cpi Top-level encoder structure.
159 * \param[in] x Pointer to structure holding all the data
160 * for the current macroblock.
161 * \param[in] bsize Current partition block size.
162 * \param[in] ref_frame_cost The entropy cost for signaling that the
163 * current ref frame is an intra frame.
164 * \param[in] ctx Structure to hold the number of 4x4 blks to
165 * copy the tx_type and txfm_skip arrays.
166 * \param[in] this_rd_cost Struct to keep track of palette mode's
167 * rd_stats.
168 * \param[in] best_rd Best RD seen for this block so far.
169 *
170 * \return Returns whether luma palette mode can skip the txfm. The
171 * corresponding mbmi, this_rd_costs, intra_search_state, and tx_type arrays in
172 * ctx are also updated.
173 */
174 int av1_search_palette_mode(IntraModeSearchState *intra_search_state,
175 const AV1_COMP *cpi, MACROBLOCK *x,
176 BLOCK_SIZE bsize, unsigned int ref_frame_cost,
177 PICK_MODE_CONTEXT *ctx, RD_STATS *this_rd_cost,
178 int64_t best_rd);
179
180 /*!\brief Evaluate luma palette mode for inter frames.
181 *
182 * \ingroup intra_mode_search
183 * \callergraph
184 * \callgraph
185 * This function handles luma palette mode when the current frame is an
186 * inter frame.
187 *
188 * \param[in] cpi Top-level encoder structure.
189 * \param[in] x Pointer to structure holding all the data
190 * for the current macroblock.
191 * \param[in] bsize Current partition block size.
192 * \param[in] ref_frame_cost The entropy cost for signaling that the
193 * current ref frame is an intra frame.
194 * \param[in] ctx Structure to hold the number of 4x4 blks to
195 * copy the tx_type and txfm_skip arrays.
196 * \param[in] this_rd_cost Struct to keep track of palette mode's
197 * rd_stats.
198 * \param[in] best_rd Best RD seen for this block so far.
199 */
200 void av1_search_palette_mode_luma(const AV1_COMP *cpi, MACROBLOCK *x,
201 BLOCK_SIZE bsize, unsigned int ref_frame_cost,
202 PICK_MODE_CONTEXT *ctx,
203 RD_STATS *this_rd_cost, int64_t best_rd);
204
205 /*!\brief Perform intra-mode search on luma channels for intra frames.
206 *
207 * \ingroup intra_mode_search
208 * \callgraph
209 * \callergraph
210 * This function performs intra-mode search on the luma channel when the
211 * current frame is intra-only. This function does not search intrabc mode,
212 * but it does search palette and filter_intra.
213 *
214 * \param[in] cpi Top-level encoder structure.
215 * \param[in] x Pointer to structure holding all the data
216 * for the current macroblock.
217 * \param[in] rate The total rate needed to predict the current
218 * chroma block.
219 * \param[in] rate_tokenonly The rate without the cost of sending the
220 * prediction modes.
221 * chroma block.
222 * after the reconstruction.
223 * \param[in] distortion The chroma distortion of the best prediction
224 * after the reconstruction.
225 * \param[in] skippable Whether we can skip txfm process.
226 * \param[in] bsize Current partition block size.
227 * \param[in] best_rd Best RD seen for this block so far.
228 * \param[in] ctx Structure to hold the number of 4x4 blks to
229 * copy the tx_type and txfm_skip arrays.
230 *
231 * \return Returns the rd_cost if this function finds a mode better than
232 * best_rd, otherwise returns INT64_MAX. This also updates the mbmi, the rate
233 * and distortion, and the tx_type arrays in ctx.
234 */
235 int64_t av1_rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
236 int *rate, int *rate_tokenonly,
237 int64_t *distortion, uint8_t *skippable,
238 BLOCK_SIZE bsize, int64_t best_rd,
239 PICK_MODE_CONTEXT *ctx);
240
241 /*!\brief Perform intra-mode search on chroma channels.
242 *
243 * \ingroup intra_mode_search
244 * \callergraph
245 * \callgraph
246 * This function performs intra-mode search on the chroma channels. Just like
247 * \ref av1_rd_pick_intra_sby_mode(), this function searches over palette mode
248 * (filter_intra is not available on chroma planes). Unlike \ref
249 * av1_rd_pick_intra_sby_mode() this function is used by both inter and intra
250 * frames.
251 *
252 * \param[in] cpi Top-level encoder structure.
253 * \param[in] x Pointer to structure holding all the data
254 * for the current macroblock.
255 * \param[in] rate The total rate needed to predict the current
256 * chroma block.
257 * \param[in] rate_tokenonly The rate without the cost of sending the
258 * prediction modes.
259 * chroma block.
260 * after the reconstruction.
261 * \param[in] distortion The chroma distortion of the best prediction
262 * after the reconstruction.
263 * \param[in] skippable Whether we can skip txfm process.
264 * \param[in] bsize Current partition block size.
265 * \param[in] max_tx_size The maximum tx_size available
266 *
267 * \return Returns the rd_cost of the best uv mode found. This also updates the
268 * mbmi, the rate and distortion, distortion.
269 */
270 int64_t av1_rd_pick_intra_sbuv_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
271 int *rate, int *rate_tokenonly,
272 int64_t *distortion, uint8_t *skippable,
273 BLOCK_SIZE bsize, TX_SIZE max_tx_size);
274
275 /*! \brief Return the number of colors in src. Used by palette mode.
276 */
277 void av1_count_colors(const uint8_t *src, int stride, int rows, int cols,
278 int *val_count, int *num_colors);
279
280 /*! \brief See \ref av1_count_colors(), but for highbd.
281 */
282 void av1_count_colors_highbd(const uint8_t *src8, int stride, int rows,
283 int cols, int bit_depth, int *val_count,
284 int *val_count_8bit, int *num_color_bins,
285 int *num_colors);
286
287 /*! \brief Initializes the \ref IntraModeSearchState struct.
288 */
init_intra_mode_search_state(IntraModeSearchState * intra_search_state)289 static inline void init_intra_mode_search_state(
290 IntraModeSearchState *intra_search_state) {
291 memset(intra_search_state, 0, sizeof(*intra_search_state));
292 intra_search_state->rate_uv_intra = INT_MAX;
293 }
294
295 /*! \brief set the luma intra mode and delta angles for a given mode index.
296 * The total number of luma intra mode is LUMA_MODE_COUNT = 61.
297 * The first 13 modes are from DC_PRED to PAETH_PRED, followed by directional
298 * modes. Each of the main 8 directional modes have 6 = MAX_ANGLE_DELTA * 2
299 * delta angles.
300 * \param[in] mode_idx mode index in intra mode decision
301 * process.
302 * \param[in] mbmi Pointer to structure holding the mode
303 * info for the current macroblock.
304 * \param[in] reorder_delta_angle_eval Indicates whether to reorder the
305 * evaluation of delta angle modes.
306 */
307 void set_y_mode_and_delta_angle(const int mode_idx, MB_MODE_INFO *const mbmi,
308 int reorder_delta_angle_eval);
309 #ifdef __cplusplus
310 } // extern "C"
311 #endif
312
313 #endif // AOM_AV1_ENCODER_INTRA_MODE_SEARCH_H_
314