xref: /aosp_15_r20/external/libaom/av1/encoder/intra_mode_search.c (revision 77c1e3ccc04c968bd2bc212e87364f250e820521)
1 /*
2  * Copyright (c) 2020, Alliance for Open Media. All rights reserved.
3  *
4  * This source code is subject to the terms of the BSD 2 Clause License and
5  * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6  * was not distributed with this source code in the LICENSE file, you can
7  * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8  * Media Patent License 1.0 was not distributed with this source code in the
9  * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10  */
11 
12 #include "av1/common/av1_common_int.h"
13 #include "av1/common/cfl.h"
14 #include "av1/common/reconintra.h"
15 
16 #include "av1/encoder/intra_mode_search.h"
17 #include "av1/encoder/intra_mode_search_utils.h"
18 #include "av1/encoder/palette.h"
19 #include "av1/encoder/speed_features.h"
20 #include "av1/encoder/tx_search.h"
21 
22 // Even though there are 7 delta angles, this macro is set to 9 to facilitate
23 // the rd threshold check to prune -3 and 3 delta angles.
24 #define SIZE_OF_ANGLE_DELTA_RD_COST_ARRAY (2 * MAX_ANGLE_DELTA + 3)
25 
26 // The order for evaluating delta angles while processing the luma directional
27 // intra modes. Currently, this order of evaluation is applicable only when
28 // speed feature prune_luma_odd_delta_angles_in_intra is enabled. In this case,
29 // even angles are evaluated first in order to facilitate the pruning of odd
30 // delta angles based on the rd costs of the neighboring delta angles.
31 static const int8_t luma_delta_angles_order[2 * MAX_ANGLE_DELTA] = {
32   -2, 2, -3, -1, 1, 3,
33 };
34 
35 /*!\cond */
36 static const PREDICTION_MODE intra_rd_search_mode_order[INTRA_MODES] = {
37   DC_PRED,       H_PRED,        V_PRED,    SMOOTH_PRED, PAETH_PRED,
38   SMOOTH_V_PRED, SMOOTH_H_PRED, D135_PRED, D203_PRED,   D157_PRED,
39   D67_PRED,      D113_PRED,     D45_PRED,
40 };
41 
42 static const UV_PREDICTION_MODE uv_rd_search_mode_order[UV_INTRA_MODES] = {
43   UV_DC_PRED,     UV_CFL_PRED,   UV_H_PRED,        UV_V_PRED,
44   UV_SMOOTH_PRED, UV_PAETH_PRED, UV_SMOOTH_V_PRED, UV_SMOOTH_H_PRED,
45   UV_D135_PRED,   UV_D203_PRED,  UV_D157_PRED,     UV_D67_PRED,
46   UV_D113_PRED,   UV_D45_PRED,
47 };
48 
49 // The bitmask corresponds to the filter intra modes as defined in enums.h
50 // FILTER_INTRA_MODE enumeration type. Setting a bit to 0 in the mask means to
51 // disable the evaluation of corresponding filter intra mode. The table
52 // av1_derived_filter_intra_mode_used_flag is used when speed feature
53 // prune_filter_intra_level is 1. The evaluated filter intra modes are union
54 // of the following:
55 // 1) FILTER_DC_PRED
56 // 2) mode that corresponds to best mode so far of DC_PRED, V_PRED, H_PRED,
57 // D157_PRED and PAETH_PRED. (Eg: FILTER_V_PRED if best mode so far is V_PRED).
58 static const uint8_t av1_derived_filter_intra_mode_used_flag[INTRA_MODES] = {
59   0x01,  // DC_PRED:           0000 0001
60   0x03,  // V_PRED:            0000 0011
61   0x05,  // H_PRED:            0000 0101
62   0x01,  // D45_PRED:          0000 0001
63   0x01,  // D135_PRED:         0000 0001
64   0x01,  // D113_PRED:         0000 0001
65   0x09,  // D157_PRED:         0000 1001
66   0x01,  // D203_PRED:         0000 0001
67   0x01,  // D67_PRED:          0000 0001
68   0x01,  // SMOOTH_PRED:       0000 0001
69   0x01,  // SMOOTH_V_PRED:     0000 0001
70   0x01,  // SMOOTH_H_PRED:     0000 0001
71   0x11   // PAETH_PRED:        0001 0001
72 };
73 
74 // The bitmask corresponds to the chroma intra modes as defined in enums.h
75 // UV_PREDICTION_MODE enumeration type. Setting a bit to 0 in the mask means to
76 // disable the evaluation of corresponding chroma intra mode. The table
77 // av1_derived_chroma_intra_mode_used_flag is used when speed feature
78 // prune_chroma_modes_using_luma_winner is enabled. The evaluated chroma
79 // intra modes are union of the following:
80 // 1) UV_DC_PRED
81 // 2) UV_SMOOTH_PRED
82 // 3) UV_CFL_PRED
83 // 4) mode that corresponds to luma intra mode winner (Eg : UV_V_PRED if luma
84 // intra mode winner is V_PRED).
85 static const uint16_t av1_derived_chroma_intra_mode_used_flag[INTRA_MODES] = {
86   0x2201,  // DC_PRED:           0010 0010 0000 0001
87   0x2203,  // V_PRED:            0010 0010 0000 0011
88   0x2205,  // H_PRED:            0010 0010 0000 0101
89   0x2209,  // D45_PRED:          0010 0010 0000 1001
90   0x2211,  // D135_PRED:         0010 0010 0001 0001
91   0x2221,  // D113_PRED:         0010 0010 0010 0001
92   0x2241,  // D157_PRED:         0010 0010 0100 0001
93   0x2281,  // D203_PRED:         0010 0010 1000 0001
94   0x2301,  // D67_PRED:          0010 0011 0000 0001
95   0x2201,  // SMOOTH_PRED:       0010 0010 0000 0001
96   0x2601,  // SMOOTH_V_PRED:     0010 0110 0000 0001
97   0x2a01,  // SMOOTH_H_PRED:     0010 1010 0000 0001
98   0x3201   // PAETH_PRED:        0011 0010 0000 0001
99 };
100 
101 DECLARE_ALIGNED(16, static const uint8_t, all_zeros[MAX_SB_SIZE]) = { 0 };
102 DECLARE_ALIGNED(16, static const uint16_t,
103                 highbd_all_zeros[MAX_SB_SIZE]) = { 0 };
104 
av1_calc_normalized_variance(aom_variance_fn_t vf,const uint8_t * const buf,const int stride,const int is_hbd)105 int av1_calc_normalized_variance(aom_variance_fn_t vf, const uint8_t *const buf,
106                                  const int stride, const int is_hbd) {
107   unsigned int sse;
108 
109   if (is_hbd)
110     return vf(buf, stride, CONVERT_TO_BYTEPTR(highbd_all_zeros), 0, &sse);
111   else
112     return vf(buf, stride, all_zeros, 0, &sse);
113 }
114 
115 // Computes average of log(1 + variance) across 4x4 sub-blocks for source and
116 // reconstructed blocks.
compute_avg_log_variance(const AV1_COMP * const cpi,MACROBLOCK * x,const BLOCK_SIZE bs,double * avg_log_src_variance,double * avg_log_recon_variance)117 static void compute_avg_log_variance(const AV1_COMP *const cpi, MACROBLOCK *x,
118                                      const BLOCK_SIZE bs,
119                                      double *avg_log_src_variance,
120                                      double *avg_log_recon_variance) {
121   const MACROBLOCKD *const xd = &x->e_mbd;
122   const BLOCK_SIZE sb_size = cpi->common.seq_params->sb_size;
123   const int mi_row_in_sb = x->e_mbd.mi_row & (mi_size_high[sb_size] - 1);
124   const int mi_col_in_sb = x->e_mbd.mi_col & (mi_size_wide[sb_size] - 1);
125   const int right_overflow =
126       (xd->mb_to_right_edge < 0) ? ((-xd->mb_to_right_edge) >> 3) : 0;
127   const int bottom_overflow =
128       (xd->mb_to_bottom_edge < 0) ? ((-xd->mb_to_bottom_edge) >> 3) : 0;
129   const int bw = (MI_SIZE * mi_size_wide[bs] - right_overflow);
130   const int bh = (MI_SIZE * mi_size_high[bs] - bottom_overflow);
131   const int is_hbd = is_cur_buf_hbd(xd);
132 
133   for (int i = 0; i < bh; i += MI_SIZE) {
134     const int r = mi_row_in_sb + (i >> MI_SIZE_LOG2);
135     for (int j = 0; j < bw; j += MI_SIZE) {
136       const int c = mi_col_in_sb + (j >> MI_SIZE_LOG2);
137       const int mi_offset = r * mi_size_wide[sb_size] + c;
138       Block4x4VarInfo *block_4x4_var_info =
139           &x->src_var_info_of_4x4_sub_blocks[mi_offset];
140       int src_var = block_4x4_var_info->var;
141       double log_src_var = block_4x4_var_info->log_var;
142       // Compute average of log(1 + variance) for the source block from 4x4
143       // sub-block variance values. Calculate and store 4x4 sub-block variance
144       // and log(1 + variance), if the values present in
145       // src_var_of_4x4_sub_blocks are invalid. Reuse the same if it is readily
146       // available with valid values.
147       if (src_var < 0) {
148         src_var = av1_calc_normalized_variance(
149             cpi->ppi->fn_ptr[BLOCK_4X4].vf,
150             x->plane[0].src.buf + i * x->plane[0].src.stride + j,
151             x->plane[0].src.stride, is_hbd);
152         block_4x4_var_info->var = src_var;
153         log_src_var = log1p(src_var / 16.0);
154         block_4x4_var_info->log_var = log_src_var;
155       } else {
156         // When source variance is already calculated and available for
157         // retrieval, check if log(1 + variance) is also available. If it is
158         // available, then retrieve from buffer. Else, calculate the same and
159         // store to the buffer.
160         if (log_src_var < 0) {
161           log_src_var = log1p(src_var / 16.0);
162           block_4x4_var_info->log_var = log_src_var;
163         }
164       }
165       *avg_log_src_variance += log_src_var;
166 
167       const int recon_var = av1_calc_normalized_variance(
168           cpi->ppi->fn_ptr[BLOCK_4X4].vf,
169           xd->plane[0].dst.buf + i * xd->plane[0].dst.stride + j,
170           xd->plane[0].dst.stride, is_hbd);
171       *avg_log_recon_variance += log1p(recon_var / 16.0);
172     }
173   }
174 
175   const int blocks = (bw * bh) / 16;
176   *avg_log_src_variance /= (double)blocks;
177   *avg_log_recon_variance /= (double)blocks;
178 }
179 
180 // Returns a factor to be applied to the RD value based on how well the
181 // reconstructed block variance matches the source variance.
intra_rd_variance_factor(const AV1_COMP * cpi,MACROBLOCK * x,BLOCK_SIZE bs)182 static double intra_rd_variance_factor(const AV1_COMP *cpi, MACROBLOCK *x,
183                                        BLOCK_SIZE bs) {
184   double threshold = INTRA_RD_VAR_THRESH(cpi->oxcf.speed);
185   // For non-positive threshold values, the comparison of source and
186   // reconstructed variances with threshold evaluates to false
187   // (src_var < threshold/rec_var < threshold) as these metrics are greater than
188   // than 0. Hence further calculations are skipped.
189   if (threshold <= 0) return 1.0;
190 
191   double variance_rd_factor = 1.0;
192   double avg_log_src_variance = 0.0;
193   double avg_log_recon_variance = 0.0;
194   double var_diff = 0.0;
195 
196   compute_avg_log_variance(cpi, x, bs, &avg_log_src_variance,
197                            &avg_log_recon_variance);
198 
199   // Dont allow 0 to prevent / 0 below.
200   avg_log_src_variance += 0.000001;
201   avg_log_recon_variance += 0.000001;
202 
203   if (avg_log_src_variance >= avg_log_recon_variance) {
204     var_diff = (avg_log_src_variance - avg_log_recon_variance);
205     if ((var_diff > 0.5) && (avg_log_recon_variance < threshold)) {
206       variance_rd_factor = 1.0 + ((var_diff * 2) / avg_log_src_variance);
207     }
208   } else {
209     var_diff = (avg_log_recon_variance - avg_log_src_variance);
210     if ((var_diff > 0.5) && (avg_log_src_variance < threshold)) {
211       variance_rd_factor = 1.0 + (var_diff / (2 * avg_log_src_variance));
212     }
213   }
214 
215   // Limit adjustment;
216   variance_rd_factor = AOMMIN(3.0, variance_rd_factor);
217 
218   return variance_rd_factor;
219 }
220 /*!\endcond */
221 
222 /*!\brief Search for the best filter_intra mode when coding intra frame.
223  *
224  * \ingroup intra_mode_search
225  * \callergraph
226  * This function loops through all filter_intra modes to find the best one.
227  *
228  * \return Returns 1 if a new filter_intra mode is selected; 0 otherwise.
229  */
rd_pick_filter_intra_sby(const AV1_COMP * const cpi,MACROBLOCK * x,int * rate,int * rate_tokenonly,int64_t * distortion,uint8_t * skippable,BLOCK_SIZE bsize,int mode_cost,PREDICTION_MODE best_mode_so_far,int64_t * best_rd,int64_t * best_model_rd,PICK_MODE_CONTEXT * ctx)230 static int rd_pick_filter_intra_sby(const AV1_COMP *const cpi, MACROBLOCK *x,
231                                     int *rate, int *rate_tokenonly,
232                                     int64_t *distortion, uint8_t *skippable,
233                                     BLOCK_SIZE bsize, int mode_cost,
234                                     PREDICTION_MODE best_mode_so_far,
235                                     int64_t *best_rd, int64_t *best_model_rd,
236                                     PICK_MODE_CONTEXT *ctx) {
237   // Skip the evaluation of filter intra modes.
238   if (cpi->sf.intra_sf.prune_filter_intra_level == 2) return 0;
239 
240   MACROBLOCKD *const xd = &x->e_mbd;
241   MB_MODE_INFO *mbmi = xd->mi[0];
242   int filter_intra_selected_flag = 0;
243   FILTER_INTRA_MODE mode;
244   TX_SIZE best_tx_size = TX_8X8;
245   FILTER_INTRA_MODE_INFO filter_intra_mode_info;
246   uint8_t best_tx_type_map[MAX_MIB_SIZE * MAX_MIB_SIZE];
247   av1_zero(filter_intra_mode_info);
248   mbmi->filter_intra_mode_info.use_filter_intra = 1;
249   mbmi->mode = DC_PRED;
250   mbmi->palette_mode_info.palette_size[0] = 0;
251 
252   // Skip the evaluation of filter-intra if cached MB_MODE_INFO does not have
253   // filter-intra as winner.
254   if (x->use_mb_mode_cache &&
255       !x->mb_mode_cache->filter_intra_mode_info.use_filter_intra)
256     return 0;
257 
258   for (mode = 0; mode < FILTER_INTRA_MODES; ++mode) {
259     int64_t this_rd;
260     RD_STATS tokenonly_rd_stats;
261     mbmi->filter_intra_mode_info.filter_intra_mode = mode;
262 
263     if ((cpi->sf.intra_sf.prune_filter_intra_level == 1) &&
264         !(av1_derived_filter_intra_mode_used_flag[best_mode_so_far] &
265           (1 << mode)))
266       continue;
267 
268     // Skip the evaluation of modes that do not match with the winner mode in
269     // x->mb_mode_cache.
270     if (x->use_mb_mode_cache &&
271         mode != x->mb_mode_cache->filter_intra_mode_info.filter_intra_mode)
272       continue;
273 
274     if (model_intra_yrd_and_prune(cpi, x, bsize, best_model_rd)) {
275       continue;
276     }
277     av1_pick_uniform_tx_size_type_yrd(cpi, x, &tokenonly_rd_stats, bsize,
278                                       *best_rd);
279     if (tokenonly_rd_stats.rate == INT_MAX) continue;
280     const int this_rate =
281         tokenonly_rd_stats.rate +
282         intra_mode_info_cost_y(cpi, x, mbmi, bsize, mode_cost, 0);
283     this_rd = RDCOST(x->rdmult, this_rate, tokenonly_rd_stats.dist);
284 
285     // Visual quality adjustment based on recon vs source variance.
286     if ((cpi->oxcf.mode == ALLINTRA) && (this_rd != INT64_MAX)) {
287       this_rd = (int64_t)(this_rd * intra_rd_variance_factor(cpi, x, bsize));
288     }
289 
290     // Collect mode stats for multiwinner mode processing
291     const int txfm_search_done = 1;
292     store_winner_mode_stats(
293         &cpi->common, x, mbmi, NULL, NULL, NULL, 0, NULL, bsize, this_rd,
294         cpi->sf.winner_mode_sf.multi_winner_mode_type, txfm_search_done);
295     if (this_rd < *best_rd) {
296       *best_rd = this_rd;
297       best_tx_size = mbmi->tx_size;
298       filter_intra_mode_info = mbmi->filter_intra_mode_info;
299       av1_copy_array(best_tx_type_map, xd->tx_type_map, ctx->num_4x4_blk);
300       memcpy(ctx->blk_skip, x->txfm_search_info.blk_skip,
301              sizeof(x->txfm_search_info.blk_skip[0]) * ctx->num_4x4_blk);
302       *rate = this_rate;
303       *rate_tokenonly = tokenonly_rd_stats.rate;
304       *distortion = tokenonly_rd_stats.dist;
305       *skippable = tokenonly_rd_stats.skip_txfm;
306       filter_intra_selected_flag = 1;
307     }
308   }
309 
310   if (filter_intra_selected_flag) {
311     mbmi->mode = DC_PRED;
312     mbmi->tx_size = best_tx_size;
313     mbmi->filter_intra_mode_info = filter_intra_mode_info;
314     av1_copy_array(ctx->tx_type_map, best_tx_type_map, ctx->num_4x4_blk);
315     return 1;
316   } else {
317     return 0;
318   }
319 }
320 
av1_count_colors(const uint8_t * src,int stride,int rows,int cols,int * val_count,int * num_colors)321 void av1_count_colors(const uint8_t *src, int stride, int rows, int cols,
322                       int *val_count, int *num_colors) {
323   const int max_pix_val = 1 << 8;
324   memset(val_count, 0, max_pix_val * sizeof(val_count[0]));
325   for (int r = 0; r < rows; ++r) {
326     for (int c = 0; c < cols; ++c) {
327       const int this_val = src[r * stride + c];
328       assert(this_val < max_pix_val);
329       ++val_count[this_val];
330     }
331   }
332   int n = 0;
333   for (int i = 0; i < max_pix_val; ++i) {
334     if (val_count[i]) ++n;
335   }
336   *num_colors = n;
337 }
338 
av1_count_colors_highbd(const uint8_t * src8,int stride,int rows,int cols,int bit_depth,int * val_count,int * bin_val_count,int * num_color_bins,int * num_colors)339 void av1_count_colors_highbd(const uint8_t *src8, int stride, int rows,
340                              int cols, int bit_depth, int *val_count,
341                              int *bin_val_count, int *num_color_bins,
342                              int *num_colors) {
343   assert(bit_depth <= 12);
344   const int max_bin_val = 1 << 8;
345   const int max_pix_val = 1 << bit_depth;
346   const uint16_t *src = CONVERT_TO_SHORTPTR(src8);
347   memset(bin_val_count, 0, max_bin_val * sizeof(val_count[0]));
348   if (val_count != NULL)
349     memset(val_count, 0, max_pix_val * sizeof(val_count[0]));
350   for (int r = 0; r < rows; ++r) {
351     for (int c = 0; c < cols; ++c) {
352       /*
353        * Down-convert the pixels to 8-bit domain before counting.
354        * This provides consistency of behavior for palette search
355        * between lbd and hbd encodes. This down-converted pixels
356        * are only used for calculating the threshold (n).
357        */
358       const int this_val = ((src[r * stride + c]) >> (bit_depth - 8));
359       assert(this_val < max_bin_val);
360       if (this_val >= max_bin_val) continue;
361       ++bin_val_count[this_val];
362       if (val_count != NULL) ++val_count[(src[r * stride + c])];
363     }
364   }
365   int n = 0;
366   // Count the colors based on 8-bit domain used to gate the palette path
367   for (int i = 0; i < max_bin_val; ++i) {
368     if (bin_val_count[i]) ++n;
369   }
370   *num_color_bins = n;
371 
372   // Count the actual hbd colors used to create top_colors
373   n = 0;
374   if (val_count != NULL) {
375     for (int i = 0; i < max_pix_val; ++i) {
376       if (val_count[i]) ++n;
377     }
378     *num_colors = n;
379   }
380 }
381 
set_y_mode_and_delta_angle(const int mode_idx,MB_MODE_INFO * const mbmi,int reorder_delta_angle_eval)382 void set_y_mode_and_delta_angle(const int mode_idx, MB_MODE_INFO *const mbmi,
383                                 int reorder_delta_angle_eval) {
384   if (mode_idx < INTRA_MODE_END) {
385     mbmi->mode = intra_rd_search_mode_order[mode_idx];
386     mbmi->angle_delta[PLANE_TYPE_Y] = 0;
387   } else {
388     mbmi->mode = (mode_idx - INTRA_MODE_END) / (MAX_ANGLE_DELTA * 2) + V_PRED;
389     int delta_angle_eval_idx =
390         (mode_idx - INTRA_MODE_END) % (MAX_ANGLE_DELTA * 2);
391     if (reorder_delta_angle_eval) {
392       mbmi->angle_delta[PLANE_TYPE_Y] =
393           luma_delta_angles_order[delta_angle_eval_idx];
394     } else {
395       mbmi->angle_delta[PLANE_TYPE_Y] =
396           (delta_angle_eval_idx < 3 ? (delta_angle_eval_idx - 3)
397                                     : (delta_angle_eval_idx - 2));
398     }
399   }
400 }
401 
get_model_rd_index_for_pruning(const MACROBLOCK * const x,const INTRA_MODE_SPEED_FEATURES * const intra_sf)402 static inline int get_model_rd_index_for_pruning(
403     const MACROBLOCK *const x,
404     const INTRA_MODE_SPEED_FEATURES *const intra_sf) {
405   const int top_intra_model_count_allowed =
406       intra_sf->top_intra_model_count_allowed;
407   if (!intra_sf->adapt_top_model_rd_count_using_neighbors)
408     return top_intra_model_count_allowed - 1;
409 
410   const MACROBLOCKD *const xd = &x->e_mbd;
411   const PREDICTION_MODE mode = xd->mi[0]->mode;
412   int model_rd_index_for_pruning = top_intra_model_count_allowed - 1;
413   int is_left_mode_neq_cur_mode = 0, is_above_mode_neq_cur_mode = 0;
414   if (xd->left_available)
415     is_left_mode_neq_cur_mode = xd->left_mbmi->mode != mode;
416   if (xd->up_available)
417     is_above_mode_neq_cur_mode = xd->above_mbmi->mode != mode;
418   // The pruning of luma intra modes is made more aggressive at lower quantizers
419   // and vice versa. The value for model_rd_index_for_pruning is derived as
420   // follows.
421   // qidx 0 to 127: Reduce the index of a candidate used for comparison only if
422   // the current mode does not match either of the available neighboring modes.
423   // qidx 128 to 255: Reduce the index of a candidate used for comparison only
424   // if the current mode does not match both the available neighboring modes.
425   if (x->qindex <= 127) {
426     if (is_left_mode_neq_cur_mode || is_above_mode_neq_cur_mode)
427       model_rd_index_for_pruning = AOMMAX(model_rd_index_for_pruning - 1, 0);
428   } else {
429     if (is_left_mode_neq_cur_mode && is_above_mode_neq_cur_mode)
430       model_rd_index_for_pruning = AOMMAX(model_rd_index_for_pruning - 1, 0);
431   }
432   return model_rd_index_for_pruning;
433 }
434 
435 /*! \brief prune luma intra mode based on the model rd.
436  * \param[in]    this_model_rd              model rd for current mode.
437  * \param[in]    best_model_rd              Best model RD seen for this block so
438  *                                          far.
439  * \param[in]    top_intra_model_rd         Top intra model RD seen for this
440  *                                          block so far.
441  * \param[in]    max_model_cnt_allowed      The maximum number of top intra
442  *                                          model RD allowed.
443  * \param[in]    model_rd_index_for_pruning Index of the candidate used for
444  *                                          pruning based on model rd.
445  */
prune_intra_y_mode(int64_t this_model_rd,int64_t * best_model_rd,int64_t top_intra_model_rd[],int max_model_cnt_allowed,int model_rd_index_for_pruning)446 static int prune_intra_y_mode(int64_t this_model_rd, int64_t *best_model_rd,
447                               int64_t top_intra_model_rd[],
448                               int max_model_cnt_allowed,
449                               int model_rd_index_for_pruning) {
450   const double thresh_best = 1.50;
451   const double thresh_top = 1.00;
452   for (int i = 0; i < max_model_cnt_allowed; i++) {
453     if (this_model_rd < top_intra_model_rd[i]) {
454       for (int j = max_model_cnt_allowed - 1; j > i; j--) {
455         top_intra_model_rd[j] = top_intra_model_rd[j - 1];
456       }
457       top_intra_model_rd[i] = this_model_rd;
458       break;
459     }
460   }
461   if (top_intra_model_rd[model_rd_index_for_pruning] != INT64_MAX &&
462       this_model_rd >
463           thresh_top * top_intra_model_rd[model_rd_index_for_pruning])
464     return 1;
465 
466   if (this_model_rd != INT64_MAX &&
467       this_model_rd > thresh_best * (*best_model_rd))
468     return 1;
469   if (this_model_rd < *best_model_rd) *best_model_rd = this_model_rd;
470   return 0;
471 }
472 
473 // Run RD calculation with given chroma intra prediction angle., and return
474 // the RD cost. Update the best mode info. if the RD cost is the best so far.
pick_intra_angle_routine_sbuv(const AV1_COMP * const cpi,MACROBLOCK * x,BLOCK_SIZE bsize,int rate_overhead,int64_t best_rd_in,int * rate,RD_STATS * rd_stats,int * best_angle_delta,int64_t * best_rd)475 static int64_t pick_intra_angle_routine_sbuv(
476     const AV1_COMP *const cpi, MACROBLOCK *x, BLOCK_SIZE bsize,
477     int rate_overhead, int64_t best_rd_in, int *rate, RD_STATS *rd_stats,
478     int *best_angle_delta, int64_t *best_rd) {
479   MB_MODE_INFO *mbmi = x->e_mbd.mi[0];
480   assert(!is_inter_block(mbmi));
481   int this_rate;
482   int64_t this_rd;
483   RD_STATS tokenonly_rd_stats;
484 
485   if (!av1_txfm_uvrd(cpi, x, &tokenonly_rd_stats, bsize, best_rd_in))
486     return INT64_MAX;
487   this_rate = tokenonly_rd_stats.rate +
488               intra_mode_info_cost_uv(cpi, x, mbmi, bsize, rate_overhead);
489   this_rd = RDCOST(x->rdmult, this_rate, tokenonly_rd_stats.dist);
490   if (this_rd < *best_rd) {
491     *best_rd = this_rd;
492     *best_angle_delta = mbmi->angle_delta[PLANE_TYPE_UV];
493     *rate = this_rate;
494     rd_stats->rate = tokenonly_rd_stats.rate;
495     rd_stats->dist = tokenonly_rd_stats.dist;
496     rd_stats->skip_txfm = tokenonly_rd_stats.skip_txfm;
497   }
498   return this_rd;
499 }
500 
501 /*!\brief Search for the best angle delta for chroma prediction
502  *
503  * \ingroup intra_mode_search
504  * \callergraph
505  * Given a chroma directional intra prediction mode, this function will try to
506  * estimate the best delta_angle.
507  *
508  * \returns Return if there is a new mode with smaller rdcost than best_rd.
509  */
rd_pick_intra_angle_sbuv(const AV1_COMP * const cpi,MACROBLOCK * x,BLOCK_SIZE bsize,int rate_overhead,int64_t best_rd,int * rate,RD_STATS * rd_stats)510 static int rd_pick_intra_angle_sbuv(const AV1_COMP *const cpi, MACROBLOCK *x,
511                                     BLOCK_SIZE bsize, int rate_overhead,
512                                     int64_t best_rd, int *rate,
513                                     RD_STATS *rd_stats) {
514   MACROBLOCKD *const xd = &x->e_mbd;
515   MB_MODE_INFO *mbmi = xd->mi[0];
516   assert(!is_inter_block(mbmi));
517   int i, angle_delta, best_angle_delta = 0;
518   int64_t this_rd, best_rd_in, rd_cost[2 * (MAX_ANGLE_DELTA + 2)];
519 
520   rd_stats->rate = INT_MAX;
521   rd_stats->skip_txfm = 0;
522   rd_stats->dist = INT64_MAX;
523   for (i = 0; i < 2 * (MAX_ANGLE_DELTA + 2); ++i) rd_cost[i] = INT64_MAX;
524 
525   for (angle_delta = 0; angle_delta <= MAX_ANGLE_DELTA; angle_delta += 2) {
526     for (i = 0; i < 2; ++i) {
527       best_rd_in = (best_rd == INT64_MAX)
528                        ? INT64_MAX
529                        : (best_rd + (best_rd >> ((angle_delta == 0) ? 3 : 5)));
530       mbmi->angle_delta[PLANE_TYPE_UV] = (1 - 2 * i) * angle_delta;
531       this_rd = pick_intra_angle_routine_sbuv(cpi, x, bsize, rate_overhead,
532                                               best_rd_in, rate, rd_stats,
533                                               &best_angle_delta, &best_rd);
534       rd_cost[2 * angle_delta + i] = this_rd;
535       if (angle_delta == 0) {
536         if (this_rd == INT64_MAX) return 0;
537         rd_cost[1] = this_rd;
538         break;
539       }
540     }
541   }
542 
543   assert(best_rd != INT64_MAX);
544   for (angle_delta = 1; angle_delta <= MAX_ANGLE_DELTA; angle_delta += 2) {
545     int64_t rd_thresh;
546     for (i = 0; i < 2; ++i) {
547       int skip_search = 0;
548       rd_thresh = best_rd + (best_rd >> 5);
549       if (rd_cost[2 * (angle_delta + 1) + i] > rd_thresh &&
550           rd_cost[2 * (angle_delta - 1) + i] > rd_thresh)
551         skip_search = 1;
552       if (!skip_search) {
553         mbmi->angle_delta[PLANE_TYPE_UV] = (1 - 2 * i) * angle_delta;
554         pick_intra_angle_routine_sbuv(cpi, x, bsize, rate_overhead, best_rd,
555                                       rate, rd_stats, &best_angle_delta,
556                                       &best_rd);
557       }
558     }
559   }
560 
561   mbmi->angle_delta[PLANE_TYPE_UV] = best_angle_delta;
562   return rd_stats->rate != INT_MAX;
563 }
564 
565 #define PLANE_SIGN_TO_JOINT_SIGN(plane, a, b) \
566   (plane == CFL_PRED_U ? a * CFL_SIGNS + b - 1 : b * CFL_SIGNS + a - 1)
567 
cfl_idx_to_sign_and_alpha(int cfl_idx,CFL_SIGN_TYPE * cfl_sign,int * cfl_alpha)568 static void cfl_idx_to_sign_and_alpha(int cfl_idx, CFL_SIGN_TYPE *cfl_sign,
569                                       int *cfl_alpha) {
570   int cfl_linear_idx = cfl_idx - CFL_INDEX_ZERO;
571   if (cfl_linear_idx == 0) {
572     *cfl_sign = CFL_SIGN_ZERO;
573     *cfl_alpha = 0;
574   } else {
575     *cfl_sign = cfl_linear_idx > 0 ? CFL_SIGN_POS : CFL_SIGN_NEG;
576     *cfl_alpha = abs(cfl_linear_idx) - 1;
577   }
578 }
579 
cfl_compute_rd(const AV1_COMP * const cpi,MACROBLOCK * x,int plane,TX_SIZE tx_size,BLOCK_SIZE plane_bsize,int cfl_idx,int fast_mode,RD_STATS * rd_stats)580 static int64_t cfl_compute_rd(const AV1_COMP *const cpi, MACROBLOCK *x,
581                               int plane, TX_SIZE tx_size,
582                               BLOCK_SIZE plane_bsize, int cfl_idx,
583                               int fast_mode, RD_STATS *rd_stats) {
584   assert(IMPLIES(fast_mode, rd_stats == NULL));
585   const AV1_COMMON *const cm = &cpi->common;
586   MACROBLOCKD *const xd = &x->e_mbd;
587   MB_MODE_INFO *const mbmi = xd->mi[0];
588   int cfl_plane = get_cfl_pred_type(plane);
589   CFL_SIGN_TYPE cfl_sign;
590   int cfl_alpha;
591   cfl_idx_to_sign_and_alpha(cfl_idx, &cfl_sign, &cfl_alpha);
592   // We conly build CFL for a given plane, the other plane's sign is dummy
593   int dummy_sign = CFL_SIGN_NEG;
594   const int8_t orig_cfl_alpha_signs = mbmi->cfl_alpha_signs;
595   const uint8_t orig_cfl_alpha_idx = mbmi->cfl_alpha_idx;
596   mbmi->cfl_alpha_signs =
597       PLANE_SIGN_TO_JOINT_SIGN(cfl_plane, cfl_sign, dummy_sign);
598   mbmi->cfl_alpha_idx = (cfl_alpha << CFL_ALPHABET_SIZE_LOG2) + cfl_alpha;
599   int64_t cfl_cost;
600   if (fast_mode) {
601     cfl_cost =
602         intra_model_rd(cm, x, plane, plane_bsize, tx_size, /*use_hadamard=*/0);
603   } else {
604     av1_init_rd_stats(rd_stats);
605     av1_txfm_rd_in_plane(x, cpi, rd_stats, INT64_MAX, 0, plane, plane_bsize,
606                          tx_size, FTXS_NONE, 0);
607     av1_rd_cost_update(x->rdmult, rd_stats);
608     cfl_cost = rd_stats->rdcost;
609   }
610   mbmi->cfl_alpha_signs = orig_cfl_alpha_signs;
611   mbmi->cfl_alpha_idx = orig_cfl_alpha_idx;
612   return cfl_cost;
613 }
614 
615 static const int cfl_dir_ls[2] = { 1, -1 };
616 
617 // If cfl_search_range is CFL_MAGS_SIZE, return zero. Otherwise return the index
618 // of the best alpha found using intra_model_rd().
cfl_pick_plane_parameter(const AV1_COMP * const cpi,MACROBLOCK * x,int plane,TX_SIZE tx_size,int cfl_search_range)619 static int cfl_pick_plane_parameter(const AV1_COMP *const cpi, MACROBLOCK *x,
620                                     int plane, TX_SIZE tx_size,
621                                     int cfl_search_range) {
622   assert(cfl_search_range >= 1 && cfl_search_range <= CFL_MAGS_SIZE);
623 
624   if (cfl_search_range == CFL_MAGS_SIZE) return CFL_INDEX_ZERO;
625 
626   const MACROBLOCKD *const xd = &x->e_mbd;
627   const MB_MODE_INFO *const mbmi = xd->mi[0];
628   assert(mbmi->uv_mode == UV_CFL_PRED);
629   const MACROBLOCKD_PLANE *pd = &xd->plane[plane];
630   const BLOCK_SIZE plane_bsize =
631       get_plane_block_size(mbmi->bsize, pd->subsampling_x, pd->subsampling_y);
632 
633   int est_best_cfl_idx = CFL_INDEX_ZERO;
634   int fast_mode = 1;
635   int start_cfl_idx = CFL_INDEX_ZERO;
636   int64_t best_cfl_cost = cfl_compute_rd(cpi, x, plane, tx_size, plane_bsize,
637                                          start_cfl_idx, fast_mode, NULL);
638   for (int si = 0; si < 2; ++si) {
639     const int dir = cfl_dir_ls[si];
640     for (int i = 1; i < CFL_MAGS_SIZE; ++i) {
641       int cfl_idx = start_cfl_idx + dir * i;
642       if (cfl_idx < 0 || cfl_idx >= CFL_MAGS_SIZE) break;
643       int64_t cfl_cost = cfl_compute_rd(cpi, x, plane, tx_size, plane_bsize,
644                                         cfl_idx, fast_mode, NULL);
645       if (cfl_cost < best_cfl_cost) {
646         best_cfl_cost = cfl_cost;
647         est_best_cfl_idx = cfl_idx;
648       } else {
649         break;
650       }
651     }
652   }
653   return est_best_cfl_idx;
654 }
655 
set_invalid_cfl_parameters(uint8_t * best_cfl_alpha_idx,int8_t * best_cfl_alpha_signs)656 static inline void set_invalid_cfl_parameters(uint8_t *best_cfl_alpha_idx,
657                                               int8_t *best_cfl_alpha_signs) {
658   *best_cfl_alpha_idx = 0;
659   *best_cfl_alpha_signs = 0;
660 }
661 
cfl_pick_plane_rd(const AV1_COMP * const cpi,MACROBLOCK * x,int plane,TX_SIZE tx_size,int cfl_search_range,RD_STATS cfl_rd_arr[CFL_MAGS_SIZE],int est_best_cfl_idx)662 static void cfl_pick_plane_rd(const AV1_COMP *const cpi, MACROBLOCK *x,
663                               int plane, TX_SIZE tx_size, int cfl_search_range,
664                               RD_STATS cfl_rd_arr[CFL_MAGS_SIZE],
665                               int est_best_cfl_idx) {
666   assert(cfl_search_range >= 1 && cfl_search_range <= CFL_MAGS_SIZE);
667   const MACROBLOCKD *const xd = &x->e_mbd;
668   const MB_MODE_INFO *const mbmi = xd->mi[0];
669   assert(mbmi->uv_mode == UV_CFL_PRED);
670   const MACROBLOCKD_PLANE *pd = &xd->plane[plane];
671   const BLOCK_SIZE plane_bsize =
672       get_plane_block_size(mbmi->bsize, pd->subsampling_x, pd->subsampling_y);
673 
674   for (int cfl_idx = 0; cfl_idx < CFL_MAGS_SIZE; ++cfl_idx) {
675     av1_invalid_rd_stats(&cfl_rd_arr[cfl_idx]);
676   }
677 
678   int fast_mode = 0;
679   int start_cfl_idx = est_best_cfl_idx;
680   cfl_compute_rd(cpi, x, plane, tx_size, plane_bsize, start_cfl_idx, fast_mode,
681                  &cfl_rd_arr[start_cfl_idx]);
682 
683   if (cfl_search_range == 1) return;
684 
685   for (int si = 0; si < 2; ++si) {
686     const int dir = cfl_dir_ls[si];
687     for (int i = 1; i < cfl_search_range; ++i) {
688       int cfl_idx = start_cfl_idx + dir * i;
689       if (cfl_idx < 0 || cfl_idx >= CFL_MAGS_SIZE) break;
690       cfl_compute_rd(cpi, x, plane, tx_size, plane_bsize, cfl_idx, fast_mode,
691                      &cfl_rd_arr[cfl_idx]);
692     }
693   }
694 }
695 
696 /*!\brief Pick the optimal parameters for Chroma to Luma (CFL) component
697  *
698  * \ingroup intra_mode_search
699  * \callergraph
700  *
701  * This function will use DCT_DCT followed by computing SATD (sum of absolute
702  * transformed differences) to estimate the RD score and find the best possible
703  * CFL parameter.
704  *
705  * Then the function will apply a full RD search near the best possible CFL
706  * parameter to find the best actual CFL parameter.
707  *
708  * Side effect:
709  * We use ths buffers in x->plane[] and xd->plane[] as throw-away buffers for RD
710  * search.
711  *
712  * \param[in] x                Encoder prediction block structure.
713  * \param[in] cpi              Top-level encoder instance structure.
714  * \param[in] tx_size          Transform size.
715  * \param[in] ref_best_rd      Reference best RD.
716  * \param[in] cfl_search_range The search range of full RD search near the
717  *                             estimated best CFL parameter.
718  *
719  * \param[out]   best_rd_stats          RD stats of the best CFL parameter
720  * \param[out]   best_cfl_alpha_idx     Best CFL alpha index
721  * \param[out]   best_cfl_alpha_signs   Best CFL joint signs
722  *
723  */
cfl_rd_pick_alpha(MACROBLOCK * const x,const AV1_COMP * const cpi,TX_SIZE tx_size,int64_t ref_best_rd,int cfl_search_range,RD_STATS * best_rd_stats,uint8_t * best_cfl_alpha_idx,int8_t * best_cfl_alpha_signs)724 static int cfl_rd_pick_alpha(MACROBLOCK *const x, const AV1_COMP *const cpi,
725                              TX_SIZE tx_size, int64_t ref_best_rd,
726                              int cfl_search_range, RD_STATS *best_rd_stats,
727                              uint8_t *best_cfl_alpha_idx,
728                              int8_t *best_cfl_alpha_signs) {
729   assert(cfl_search_range >= 1 && cfl_search_range <= CFL_MAGS_SIZE);
730   const ModeCosts *mode_costs = &x->mode_costs;
731   RD_STATS cfl_rd_arr_u[CFL_MAGS_SIZE];
732   RD_STATS cfl_rd_arr_v[CFL_MAGS_SIZE];
733   MACROBLOCKD *const xd = &x->e_mbd;
734   int est_best_cfl_idx_u, est_best_cfl_idx_v;
735 
736   av1_invalid_rd_stats(best_rd_stats);
737 
738   // As the dc pred data is same for different values of alpha, enable the
739   // caching of dc pred data. Call clear_cfl_dc_pred_cache_flags() before
740   // returning to avoid the unintentional usage of cached dc pred data.
741   xd->cfl.use_dc_pred_cache = true;
742   // Evaluate alpha parameter of each chroma plane.
743   est_best_cfl_idx_u =
744       cfl_pick_plane_parameter(cpi, x, 1, tx_size, cfl_search_range);
745   est_best_cfl_idx_v =
746       cfl_pick_plane_parameter(cpi, x, 2, tx_size, cfl_search_range);
747 
748   if (cfl_search_range == 1) {
749     // For cfl_search_range=1, further refinement of alpha is not enabled. Hence
750     // CfL index=0 for both the chroma planes implies invalid CfL mode.
751     if (est_best_cfl_idx_u == CFL_INDEX_ZERO &&
752         est_best_cfl_idx_v == CFL_INDEX_ZERO) {
753       set_invalid_cfl_parameters(best_cfl_alpha_idx, best_cfl_alpha_signs);
754       clear_cfl_dc_pred_cache_flags(&xd->cfl);
755       return 0;
756     }
757 
758     int cfl_alpha_u, cfl_alpha_v;
759     CFL_SIGN_TYPE cfl_sign_u, cfl_sign_v;
760     const MB_MODE_INFO *mbmi = xd->mi[0];
761     cfl_idx_to_sign_and_alpha(est_best_cfl_idx_u, &cfl_sign_u, &cfl_alpha_u);
762     cfl_idx_to_sign_and_alpha(est_best_cfl_idx_v, &cfl_sign_v, &cfl_alpha_v);
763     const int joint_sign = cfl_sign_u * CFL_SIGNS + cfl_sign_v - 1;
764     // Compute alpha and mode signaling rate.
765     const int rate_overhead =
766         mode_costs->cfl_cost[joint_sign][CFL_PRED_U][cfl_alpha_u] +
767         mode_costs->cfl_cost[joint_sign][CFL_PRED_V][cfl_alpha_v] +
768         mode_costs
769             ->intra_uv_mode_cost[is_cfl_allowed(xd)][mbmi->mode][UV_CFL_PRED];
770     // Skip the CfL mode evaluation if the RD cost derived using the rate needed
771     // to signal the CfL mode and alpha parameter exceeds the ref_best_rd.
772     if (RDCOST(x->rdmult, rate_overhead, 0) > ref_best_rd) {
773       set_invalid_cfl_parameters(best_cfl_alpha_idx, best_cfl_alpha_signs);
774       clear_cfl_dc_pred_cache_flags(&xd->cfl);
775       return 0;
776     }
777   }
778 
779   // Compute the rd cost of each chroma plane using the alpha parameters which
780   // were already evaluated.
781   cfl_pick_plane_rd(cpi, x, 1, tx_size, cfl_search_range, cfl_rd_arr_u,
782                     est_best_cfl_idx_u);
783   cfl_pick_plane_rd(cpi, x, 2, tx_size, cfl_search_range, cfl_rd_arr_v,
784                     est_best_cfl_idx_v);
785 
786   clear_cfl_dc_pred_cache_flags(&xd->cfl);
787 
788   for (int ui = 0; ui < CFL_MAGS_SIZE; ++ui) {
789     if (cfl_rd_arr_u[ui].rate == INT_MAX) continue;
790     int cfl_alpha_u;
791     CFL_SIGN_TYPE cfl_sign_u;
792     cfl_idx_to_sign_and_alpha(ui, &cfl_sign_u, &cfl_alpha_u);
793     for (int vi = 0; vi < CFL_MAGS_SIZE; ++vi) {
794       if (cfl_rd_arr_v[vi].rate == INT_MAX) continue;
795       int cfl_alpha_v;
796       CFL_SIGN_TYPE cfl_sign_v;
797       cfl_idx_to_sign_and_alpha(vi, &cfl_sign_v, &cfl_alpha_v);
798       // cfl_sign_u == CFL_SIGN_ZERO && cfl_sign_v == CFL_SIGN_ZERO is not a
799       // valid parameter for CFL
800       if (cfl_sign_u == CFL_SIGN_ZERO && cfl_sign_v == CFL_SIGN_ZERO) continue;
801       int joint_sign = cfl_sign_u * CFL_SIGNS + cfl_sign_v - 1;
802       RD_STATS rd_stats = cfl_rd_arr_u[ui];
803       av1_merge_rd_stats(&rd_stats, &cfl_rd_arr_v[vi]);
804       if (rd_stats.rate != INT_MAX) {
805         rd_stats.rate +=
806             mode_costs->cfl_cost[joint_sign][CFL_PRED_U][cfl_alpha_u];
807         rd_stats.rate +=
808             mode_costs->cfl_cost[joint_sign][CFL_PRED_V][cfl_alpha_v];
809       }
810       av1_rd_cost_update(x->rdmult, &rd_stats);
811       if (rd_stats.rdcost < best_rd_stats->rdcost) {
812         *best_rd_stats = rd_stats;
813         *best_cfl_alpha_idx =
814             (cfl_alpha_u << CFL_ALPHABET_SIZE_LOG2) + cfl_alpha_v;
815         *best_cfl_alpha_signs = joint_sign;
816       }
817     }
818   }
819   if (best_rd_stats->rdcost >= ref_best_rd) {
820     av1_invalid_rd_stats(best_rd_stats);
821     // Set invalid CFL parameters here since the rdcost is not better than
822     // ref_best_rd.
823     set_invalid_cfl_parameters(best_cfl_alpha_idx, best_cfl_alpha_signs);
824     return 0;
825   }
826   return 1;
827 }
828 
should_prune_chroma_smooth_pred_based_on_source_variance(const AV1_COMP * cpi,const MACROBLOCK * x,BLOCK_SIZE bsize)829 static bool should_prune_chroma_smooth_pred_based_on_source_variance(
830     const AV1_COMP *cpi, const MACROBLOCK *x, BLOCK_SIZE bsize) {
831   if (!cpi->sf.intra_sf.prune_smooth_intra_mode_for_chroma) return false;
832 
833   // If the source variance of both chroma planes is less than 20 (empirically
834   // derived), prune UV_SMOOTH_PRED.
835   for (int i = AOM_PLANE_U; i < av1_num_planes(&cpi->common); i++) {
836     const unsigned int variance = av1_get_perpixel_variance_facade(
837         cpi, &x->e_mbd, &x->plane[i].src, bsize, i);
838     if (variance >= 20) return false;
839   }
840   return true;
841 }
842 
av1_rd_pick_intra_sbuv_mode(const AV1_COMP * const cpi,MACROBLOCK * x,int * rate,int * rate_tokenonly,int64_t * distortion,uint8_t * skippable,BLOCK_SIZE bsize,TX_SIZE max_tx_size)843 int64_t av1_rd_pick_intra_sbuv_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
844                                     int *rate, int *rate_tokenonly,
845                                     int64_t *distortion, uint8_t *skippable,
846                                     BLOCK_SIZE bsize, TX_SIZE max_tx_size) {
847   const AV1_COMMON *const cm = &cpi->common;
848   MACROBLOCKD *xd = &x->e_mbd;
849   MB_MODE_INFO *mbmi = xd->mi[0];
850   assert(!is_inter_block(mbmi));
851   MB_MODE_INFO best_mbmi = *mbmi;
852   int64_t best_rd = INT64_MAX, this_rd;
853   const ModeCosts *mode_costs = &x->mode_costs;
854   const IntraModeCfg *const intra_mode_cfg = &cpi->oxcf.intra_mode_cfg;
855 
856   init_sbuv_mode(mbmi);
857 
858   // Return if the current block does not correspond to a chroma block.
859   if (!xd->is_chroma_ref) {
860     *rate = 0;
861     *rate_tokenonly = 0;
862     *distortion = 0;
863     *skippable = 1;
864     return INT64_MAX;
865   }
866 
867   // Only store reconstructed luma when there's chroma RDO. When there's no
868   // chroma RDO, the reconstructed luma will be stored in encode_superblock().
869   xd->cfl.store_y = store_cfl_required_rdo(cm, x);
870   if (xd->cfl.store_y) {
871     // Restore reconstructed luma values.
872     // TODO([email protected]): right now we are re-computing the txfm in
873     // this function everytime we search through uv modes. There is some
874     // potential speed up here if we cache the result to avoid redundant
875     // computation.
876     av1_encode_intra_block_plane(cpi, x, mbmi->bsize, AOM_PLANE_Y,
877                                  DRY_RUN_NORMAL,
878                                  cpi->optimize_seg_arr[mbmi->segment_id]);
879     xd->cfl.store_y = 0;
880   }
881   IntraModeSearchState intra_search_state;
882   init_intra_mode_search_state(&intra_search_state);
883   const CFL_ALLOWED_TYPE cfl_allowed = is_cfl_allowed(xd);
884 
885   // Search through all non-palette modes.
886   for (int mode_idx = 0; mode_idx < UV_INTRA_MODES; ++mode_idx) {
887     int this_rate;
888     RD_STATS tokenonly_rd_stats;
889     UV_PREDICTION_MODE uv_mode = uv_rd_search_mode_order[mode_idx];
890 
891     // Skip the current mode evaluation if the RD cost derived using the mode
892     // signaling rate exceeds the best_rd so far.
893     const int mode_rate =
894         mode_costs->intra_uv_mode_cost[cfl_allowed][mbmi->mode][uv_mode];
895     if (RDCOST(x->rdmult, mode_rate, 0) > best_rd) continue;
896 
897     PREDICTION_MODE intra_mode = get_uv_mode(uv_mode);
898     const int is_diagonal_mode = av1_is_diagonal_mode(intra_mode);
899     const int is_directional_mode = av1_is_directional_mode(intra_mode);
900 
901     if (is_diagonal_mode && !cpi->oxcf.intra_mode_cfg.enable_diagonal_intra)
902       continue;
903     if (is_directional_mode &&
904         !cpi->oxcf.intra_mode_cfg.enable_directional_intra)
905       continue;
906 
907     if (!(cpi->sf.intra_sf.intra_uv_mode_mask[txsize_sqr_up_map[max_tx_size]] &
908           (1 << uv_mode)))
909       continue;
910     if (!intra_mode_cfg->enable_smooth_intra && uv_mode >= UV_SMOOTH_PRED &&
911         uv_mode <= UV_SMOOTH_H_PRED)
912       continue;
913 
914     if (!intra_mode_cfg->enable_paeth_intra && uv_mode == UV_PAETH_PRED)
915       continue;
916 
917     assert(mbmi->mode < INTRA_MODES);
918     if (cpi->sf.intra_sf.prune_chroma_modes_using_luma_winner &&
919         !(av1_derived_chroma_intra_mode_used_flag[mbmi->mode] & (1 << uv_mode)))
920       continue;
921 
922     mbmi->uv_mode = uv_mode;
923 
924     // Init variables for cfl and angle delta
925     const SPEED_FEATURES *sf = &cpi->sf;
926     mbmi->angle_delta[PLANE_TYPE_UV] = 0;
927     if (uv_mode == UV_CFL_PRED) {
928       if (!cfl_allowed || !intra_mode_cfg->enable_cfl_intra) continue;
929       assert(!is_directional_mode);
930       const TX_SIZE uv_tx_size = av1_get_tx_size(AOM_PLANE_U, xd);
931       if (!cfl_rd_pick_alpha(x, cpi, uv_tx_size, best_rd,
932                              sf->intra_sf.cfl_search_range, &tokenonly_rd_stats,
933                              &mbmi->cfl_alpha_idx, &mbmi->cfl_alpha_signs)) {
934         continue;
935       }
936     } else if (is_directional_mode && av1_use_angle_delta(mbmi->bsize) &&
937                intra_mode_cfg->enable_angle_delta) {
938       if (sf->intra_sf.chroma_intra_pruning_with_hog &&
939           !intra_search_state.dir_mode_skip_mask_ready) {
940         static const float thresh[2][4] = {
941           { -1.2f, 0.0f, 0.0f, 1.2f },    // Interframe
942           { -1.2f, -1.2f, -0.6f, 0.4f },  // Intraframe
943         };
944         const int is_chroma = 1;
945         const int is_intra_frame = frame_is_intra_only(cm);
946         prune_intra_mode_with_hog(
947             x, bsize, cm->seq_params->sb_size,
948             thresh[is_intra_frame]
949                   [sf->intra_sf.chroma_intra_pruning_with_hog - 1],
950             intra_search_state.directional_mode_skip_mask, is_chroma);
951         intra_search_state.dir_mode_skip_mask_ready = 1;
952       }
953       if (intra_search_state.directional_mode_skip_mask[uv_mode]) {
954         continue;
955       }
956 
957       // Search through angle delta
958       const int rate_overhead =
959           mode_costs->intra_uv_mode_cost[cfl_allowed][mbmi->mode][uv_mode];
960       if (!rd_pick_intra_angle_sbuv(cpi, x, bsize, rate_overhead, best_rd,
961                                     &this_rate, &tokenonly_rd_stats))
962         continue;
963     } else {
964       if (uv_mode == UV_SMOOTH_PRED &&
965           should_prune_chroma_smooth_pred_based_on_source_variance(cpi, x,
966                                                                    bsize))
967         continue;
968 
969       // Predict directly if we don't need to search for angle delta.
970       if (!av1_txfm_uvrd(cpi, x, &tokenonly_rd_stats, bsize, best_rd)) {
971         continue;
972       }
973     }
974     const int mode_cost =
975         mode_costs->intra_uv_mode_cost[cfl_allowed][mbmi->mode][uv_mode];
976     this_rate = tokenonly_rd_stats.rate +
977                 intra_mode_info_cost_uv(cpi, x, mbmi, bsize, mode_cost);
978     this_rd = RDCOST(x->rdmult, this_rate, tokenonly_rd_stats.dist);
979 
980     if (this_rd < best_rd) {
981       best_mbmi = *mbmi;
982       best_rd = this_rd;
983       *rate = this_rate;
984       *rate_tokenonly = tokenonly_rd_stats.rate;
985       *distortion = tokenonly_rd_stats.dist;
986       *skippable = tokenonly_rd_stats.skip_txfm;
987     }
988   }
989 
990   // Search palette mode
991   const int try_palette =
992       cpi->oxcf.tool_cfg.enable_palette &&
993       av1_allow_palette(cpi->common.features.allow_screen_content_tools,
994                         mbmi->bsize);
995   if (try_palette) {
996     uint8_t *best_palette_color_map = x->palette_buffer->best_palette_color_map;
997     av1_rd_pick_palette_intra_sbuv(
998         cpi, x,
999         mode_costs->intra_uv_mode_cost[cfl_allowed][mbmi->mode][UV_DC_PRED],
1000         best_palette_color_map, &best_mbmi, &best_rd, rate, rate_tokenonly,
1001         distortion, skippable);
1002   }
1003 
1004   *mbmi = best_mbmi;
1005   // Make sure we actually chose a mode
1006   assert(best_rd < INT64_MAX);
1007   return best_rd;
1008 }
1009 
1010 // Searches palette mode for luma channel in inter frame.
av1_search_palette_mode(IntraModeSearchState * intra_search_state,const AV1_COMP * cpi,MACROBLOCK * x,BLOCK_SIZE bsize,unsigned int ref_frame_cost,PICK_MODE_CONTEXT * ctx,RD_STATS * this_rd_cost,int64_t best_rd)1011 int av1_search_palette_mode(IntraModeSearchState *intra_search_state,
1012                             const AV1_COMP *cpi, MACROBLOCK *x,
1013                             BLOCK_SIZE bsize, unsigned int ref_frame_cost,
1014                             PICK_MODE_CONTEXT *ctx, RD_STATS *this_rd_cost,
1015                             int64_t best_rd) {
1016   const AV1_COMMON *const cm = &cpi->common;
1017   MB_MODE_INFO *const mbmi = x->e_mbd.mi[0];
1018   PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
1019   const int num_planes = av1_num_planes(cm);
1020   MACROBLOCKD *const xd = &x->e_mbd;
1021   int rate2 = 0;
1022   int64_t distortion2 = 0, best_rd_palette = best_rd, this_rd;
1023   int skippable = 0;
1024   uint8_t *const best_palette_color_map =
1025       x->palette_buffer->best_palette_color_map;
1026   uint8_t *const color_map = xd->plane[0].color_index_map;
1027   MB_MODE_INFO best_mbmi_palette = *mbmi;
1028   uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE];
1029   uint8_t best_tx_type_map[MAX_MIB_SIZE * MAX_MIB_SIZE];
1030   const ModeCosts *mode_costs = &x->mode_costs;
1031   const int *const intra_mode_cost =
1032       mode_costs->mbmode_cost[size_group_lookup[bsize]];
1033   const int rows = block_size_high[bsize];
1034   const int cols = block_size_wide[bsize];
1035 
1036   mbmi->mode = DC_PRED;
1037   mbmi->uv_mode = UV_DC_PRED;
1038   mbmi->ref_frame[0] = INTRA_FRAME;
1039   mbmi->ref_frame[1] = NONE_FRAME;
1040   av1_zero(pmi->palette_size);
1041 
1042   RD_STATS rd_stats_y;
1043   av1_invalid_rd_stats(&rd_stats_y);
1044   av1_rd_pick_palette_intra_sby(cpi, x, bsize, intra_mode_cost[DC_PRED],
1045                                 &best_mbmi_palette, best_palette_color_map,
1046                                 &best_rd_palette, &rd_stats_y.rate, NULL,
1047                                 &rd_stats_y.dist, &rd_stats_y.skip_txfm, NULL,
1048                                 ctx, best_blk_skip, best_tx_type_map);
1049   if (rd_stats_y.rate == INT_MAX || pmi->palette_size[0] == 0) {
1050     this_rd_cost->rdcost = INT64_MAX;
1051     return skippable;
1052   }
1053 
1054   memcpy(x->txfm_search_info.blk_skip, best_blk_skip,
1055          sizeof(best_blk_skip[0]) * bsize_to_num_blk(bsize));
1056   av1_copy_array(xd->tx_type_map, best_tx_type_map, ctx->num_4x4_blk);
1057   memcpy(color_map, best_palette_color_map,
1058          rows * cols * sizeof(best_palette_color_map[0]));
1059 
1060   skippable = rd_stats_y.skip_txfm;
1061   distortion2 = rd_stats_y.dist;
1062   rate2 = rd_stats_y.rate + ref_frame_cost;
1063   if (num_planes > 1) {
1064     if (intra_search_state->rate_uv_intra == INT_MAX) {
1065       // We have not found any good uv mode yet, so we need to search for it.
1066       TX_SIZE uv_tx = av1_get_tx_size(AOM_PLANE_U, xd);
1067       av1_rd_pick_intra_sbuv_mode(cpi, x, &intra_search_state->rate_uv_intra,
1068                                   &intra_search_state->rate_uv_tokenonly,
1069                                   &intra_search_state->dist_uvs,
1070                                   &intra_search_state->skip_uvs, bsize, uv_tx);
1071       intra_search_state->mode_uv = mbmi->uv_mode;
1072       intra_search_state->pmi_uv = *pmi;
1073       intra_search_state->uv_angle_delta = mbmi->angle_delta[PLANE_TYPE_UV];
1074     }
1075 
1076     // We have found at least one good uv mode before, so copy and paste it
1077     // over.
1078     mbmi->uv_mode = intra_search_state->mode_uv;
1079     pmi->palette_size[1] = intra_search_state->pmi_uv.palette_size[1];
1080     if (pmi->palette_size[1] > 0) {
1081       memcpy(pmi->palette_colors + PALETTE_MAX_SIZE,
1082              intra_search_state->pmi_uv.palette_colors + PALETTE_MAX_SIZE,
1083              2 * PALETTE_MAX_SIZE * sizeof(pmi->palette_colors[0]));
1084     }
1085     mbmi->angle_delta[PLANE_TYPE_UV] = intra_search_state->uv_angle_delta;
1086     skippable = skippable && intra_search_state->skip_uvs;
1087     distortion2 += intra_search_state->dist_uvs;
1088     rate2 += intra_search_state->rate_uv_intra;
1089   }
1090 
1091   if (skippable) {
1092     rate2 -= rd_stats_y.rate;
1093     if (num_planes > 1) rate2 -= intra_search_state->rate_uv_tokenonly;
1094     rate2 += mode_costs->skip_txfm_cost[av1_get_skip_txfm_context(xd)][1];
1095   } else {
1096     rate2 += mode_costs->skip_txfm_cost[av1_get_skip_txfm_context(xd)][0];
1097   }
1098   this_rd = RDCOST(x->rdmult, rate2, distortion2);
1099   this_rd_cost->rate = rate2;
1100   this_rd_cost->dist = distortion2;
1101   this_rd_cost->rdcost = this_rd;
1102   return skippable;
1103 }
1104 
av1_search_palette_mode_luma(const AV1_COMP * cpi,MACROBLOCK * x,BLOCK_SIZE bsize,unsigned int ref_frame_cost,PICK_MODE_CONTEXT * ctx,RD_STATS * this_rd_cost,int64_t best_rd)1105 void av1_search_palette_mode_luma(const AV1_COMP *cpi, MACROBLOCK *x,
1106                                   BLOCK_SIZE bsize, unsigned int ref_frame_cost,
1107                                   PICK_MODE_CONTEXT *ctx,
1108                                   RD_STATS *this_rd_cost, int64_t best_rd) {
1109   MB_MODE_INFO *const mbmi = x->e_mbd.mi[0];
1110   PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
1111   MACROBLOCKD *const xd = &x->e_mbd;
1112   int64_t best_rd_palette = best_rd, this_rd;
1113   uint8_t *const best_palette_color_map =
1114       x->palette_buffer->best_palette_color_map;
1115   uint8_t *const color_map = xd->plane[0].color_index_map;
1116   MB_MODE_INFO best_mbmi_palette = *mbmi;
1117   uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE];
1118   uint8_t best_tx_type_map[MAX_MIB_SIZE * MAX_MIB_SIZE];
1119   const ModeCosts *mode_costs = &x->mode_costs;
1120   const int *const intra_mode_cost =
1121       mode_costs->mbmode_cost[size_group_lookup[bsize]];
1122   const int rows = block_size_high[bsize];
1123   const int cols = block_size_wide[bsize];
1124 
1125   mbmi->mode = DC_PRED;
1126   mbmi->uv_mode = UV_DC_PRED;
1127   mbmi->ref_frame[0] = INTRA_FRAME;
1128   mbmi->ref_frame[1] = NONE_FRAME;
1129   av1_zero(pmi->palette_size);
1130 
1131   RD_STATS rd_stats_y;
1132   av1_invalid_rd_stats(&rd_stats_y);
1133   av1_rd_pick_palette_intra_sby(cpi, x, bsize, intra_mode_cost[DC_PRED],
1134                                 &best_mbmi_palette, best_palette_color_map,
1135                                 &best_rd_palette, &rd_stats_y.rate, NULL,
1136                                 &rd_stats_y.dist, &rd_stats_y.skip_txfm, NULL,
1137                                 ctx, best_blk_skip, best_tx_type_map);
1138   if (rd_stats_y.rate == INT_MAX || pmi->palette_size[0] == 0) {
1139     this_rd_cost->rdcost = INT64_MAX;
1140     return;
1141   }
1142 
1143   memcpy(x->txfm_search_info.blk_skip, best_blk_skip,
1144          sizeof(best_blk_skip[0]) * bsize_to_num_blk(bsize));
1145   av1_copy_array(xd->tx_type_map, best_tx_type_map, ctx->num_4x4_blk);
1146   memcpy(color_map, best_palette_color_map,
1147          rows * cols * sizeof(best_palette_color_map[0]));
1148 
1149   rd_stats_y.rate += ref_frame_cost;
1150 
1151   if (rd_stats_y.skip_txfm) {
1152     rd_stats_y.rate =
1153         ref_frame_cost +
1154         mode_costs->skip_txfm_cost[av1_get_skip_txfm_context(xd)][1];
1155   } else {
1156     rd_stats_y.rate +=
1157         mode_costs->skip_txfm_cost[av1_get_skip_txfm_context(xd)][0];
1158   }
1159   this_rd = RDCOST(x->rdmult, rd_stats_y.rate, rd_stats_y.dist);
1160   this_rd_cost->rate = rd_stats_y.rate;
1161   this_rd_cost->dist = rd_stats_y.dist;
1162   this_rd_cost->rdcost = this_rd;
1163   this_rd_cost->skip_txfm = rd_stats_y.skip_txfm;
1164 }
1165 
1166 /*!\brief Get the intra prediction by searching through tx_type and tx_size.
1167  *
1168  * \ingroup intra_mode_search
1169  * \callergraph
1170  * Currently this function is only used in the intra frame code path for
1171  * winner-mode processing.
1172  *
1173  * \return Returns whether the current mode is an improvement over best_rd.
1174  */
intra_block_yrd(const AV1_COMP * const cpi,MACROBLOCK * x,BLOCK_SIZE bsize,const int * bmode_costs,int64_t * best_rd,int * rate,int * rate_tokenonly,int64_t * distortion,uint8_t * skippable,MB_MODE_INFO * best_mbmi,PICK_MODE_CONTEXT * ctx)1175 static inline int intra_block_yrd(const AV1_COMP *const cpi, MACROBLOCK *x,
1176                                   BLOCK_SIZE bsize, const int *bmode_costs,
1177                                   int64_t *best_rd, int *rate,
1178                                   int *rate_tokenonly, int64_t *distortion,
1179                                   uint8_t *skippable, MB_MODE_INFO *best_mbmi,
1180                                   PICK_MODE_CONTEXT *ctx) {
1181   MACROBLOCKD *const xd = &x->e_mbd;
1182   MB_MODE_INFO *const mbmi = xd->mi[0];
1183   RD_STATS rd_stats;
1184   // In order to improve txfm search, avoid rd based breakouts during winner
1185   // mode evaluation. Hence passing ref_best_rd as INT64_MAX by default when the
1186   // speed feature use_rd_based_breakout_for_intra_tx_search is disabled.
1187   int64_t ref_best_rd = cpi->sf.tx_sf.use_rd_based_breakout_for_intra_tx_search
1188                             ? *best_rd
1189                             : INT64_MAX;
1190   av1_pick_uniform_tx_size_type_yrd(cpi, x, &rd_stats, bsize, ref_best_rd);
1191   if (rd_stats.rate == INT_MAX) return 0;
1192   int this_rate_tokenonly = rd_stats.rate;
1193   if (!xd->lossless[mbmi->segment_id] && block_signals_txsize(mbmi->bsize)) {
1194     // av1_pick_uniform_tx_size_type_yrd above includes the cost of the tx_size
1195     // in the tokenonly rate, but for intra blocks, tx_size is always coded
1196     // (prediction granularity), so we account for it in the full rate,
1197     // not the tokenonly rate.
1198     this_rate_tokenonly -= tx_size_cost(x, bsize, mbmi->tx_size);
1199   }
1200   const int this_rate =
1201       rd_stats.rate +
1202       intra_mode_info_cost_y(cpi, x, mbmi, bsize, bmode_costs[mbmi->mode], 0);
1203   const int64_t this_rd = RDCOST(x->rdmult, this_rate, rd_stats.dist);
1204   if (this_rd < *best_rd) {
1205     *best_mbmi = *mbmi;
1206     *best_rd = this_rd;
1207     *rate = this_rate;
1208     *rate_tokenonly = this_rate_tokenonly;
1209     *distortion = rd_stats.dist;
1210     *skippable = rd_stats.skip_txfm;
1211     av1_copy_array(ctx->blk_skip, x->txfm_search_info.blk_skip,
1212                    ctx->num_4x4_blk);
1213     av1_copy_array(ctx->tx_type_map, xd->tx_type_map, ctx->num_4x4_blk);
1214     return 1;
1215   }
1216   return 0;
1217 }
1218 
1219 /*!\brief Search for the best filter_intra mode when coding inter frame.
1220  *
1221  * \ingroup intra_mode_search
1222  * \callergraph
1223  * This function loops through all filter_intra modes to find the best one.
1224  *
1225  * \remark Returns nothing, but updates the mbmi and rd_stats.
1226  */
handle_filter_intra_mode(const AV1_COMP * cpi,MACROBLOCK * x,BLOCK_SIZE bsize,const PICK_MODE_CONTEXT * ctx,RD_STATS * rd_stats_y,int mode_cost,int64_t best_rd,int64_t best_rd_so_far)1227 static inline void handle_filter_intra_mode(const AV1_COMP *cpi, MACROBLOCK *x,
1228                                             BLOCK_SIZE bsize,
1229                                             const PICK_MODE_CONTEXT *ctx,
1230                                             RD_STATS *rd_stats_y, int mode_cost,
1231                                             int64_t best_rd,
1232                                             int64_t best_rd_so_far) {
1233   MACROBLOCKD *const xd = &x->e_mbd;
1234   MB_MODE_INFO *const mbmi = xd->mi[0];
1235   assert(mbmi->mode == DC_PRED &&
1236          av1_filter_intra_allowed_bsize(&cpi->common, bsize));
1237 
1238   RD_STATS rd_stats_y_fi;
1239   int filter_intra_selected_flag = 0;
1240   TX_SIZE best_tx_size = mbmi->tx_size;
1241   FILTER_INTRA_MODE best_fi_mode = FILTER_DC_PRED;
1242   uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE];
1243   memcpy(best_blk_skip, x->txfm_search_info.blk_skip,
1244          sizeof(best_blk_skip[0]) * ctx->num_4x4_blk);
1245   uint8_t best_tx_type_map[MAX_MIB_SIZE * MAX_MIB_SIZE];
1246   av1_copy_array(best_tx_type_map, xd->tx_type_map, ctx->num_4x4_blk);
1247   mbmi->filter_intra_mode_info.use_filter_intra = 1;
1248   for (FILTER_INTRA_MODE fi_mode = FILTER_DC_PRED; fi_mode < FILTER_INTRA_MODES;
1249        ++fi_mode) {
1250     mbmi->filter_intra_mode_info.filter_intra_mode = fi_mode;
1251     av1_pick_uniform_tx_size_type_yrd(cpi, x, &rd_stats_y_fi, bsize, best_rd);
1252     if (rd_stats_y_fi.rate == INT_MAX) continue;
1253     const int this_rate_tmp =
1254         rd_stats_y_fi.rate +
1255         intra_mode_info_cost_y(cpi, x, mbmi, bsize, mode_cost, 0);
1256     const int64_t this_rd_tmp =
1257         RDCOST(x->rdmult, this_rate_tmp, rd_stats_y_fi.dist);
1258 
1259     if (this_rd_tmp != INT64_MAX && this_rd_tmp / 2 > best_rd) {
1260       break;
1261     }
1262     if (this_rd_tmp < best_rd_so_far) {
1263       best_tx_size = mbmi->tx_size;
1264       av1_copy_array(best_tx_type_map, xd->tx_type_map, ctx->num_4x4_blk);
1265       memcpy(best_blk_skip, x->txfm_search_info.blk_skip,
1266              sizeof(best_blk_skip[0]) * ctx->num_4x4_blk);
1267       best_fi_mode = fi_mode;
1268       *rd_stats_y = rd_stats_y_fi;
1269       filter_intra_selected_flag = 1;
1270       best_rd_so_far = this_rd_tmp;
1271     }
1272   }
1273 
1274   mbmi->tx_size = best_tx_size;
1275   av1_copy_array(xd->tx_type_map, best_tx_type_map, ctx->num_4x4_blk);
1276   memcpy(x->txfm_search_info.blk_skip, best_blk_skip,
1277          sizeof(x->txfm_search_info.blk_skip[0]) * ctx->num_4x4_blk);
1278 
1279   if (filter_intra_selected_flag) {
1280     mbmi->filter_intra_mode_info.use_filter_intra = 1;
1281     mbmi->filter_intra_mode_info.filter_intra_mode = best_fi_mode;
1282   } else {
1283     mbmi->filter_intra_mode_info.use_filter_intra = 0;
1284   }
1285 }
1286 
1287 // Evaluate a given luma intra-mode in inter frames.
av1_handle_intra_y_mode(IntraModeSearchState * intra_search_state,const AV1_COMP * cpi,MACROBLOCK * x,BLOCK_SIZE bsize,unsigned int ref_frame_cost,const PICK_MODE_CONTEXT * ctx,RD_STATS * rd_stats_y,int64_t best_rd,int * mode_cost_y,int64_t * rd_y,int64_t * best_model_rd,int64_t top_intra_model_rd[])1288 int av1_handle_intra_y_mode(IntraModeSearchState *intra_search_state,
1289                             const AV1_COMP *cpi, MACROBLOCK *x,
1290                             BLOCK_SIZE bsize, unsigned int ref_frame_cost,
1291                             const PICK_MODE_CONTEXT *ctx, RD_STATS *rd_stats_y,
1292                             int64_t best_rd, int *mode_cost_y, int64_t *rd_y,
1293                             int64_t *best_model_rd,
1294                             int64_t top_intra_model_rd[]) {
1295   const AV1_COMMON *cm = &cpi->common;
1296   const INTRA_MODE_SPEED_FEATURES *const intra_sf = &cpi->sf.intra_sf;
1297   MACROBLOCKD *const xd = &x->e_mbd;
1298   MB_MODE_INFO *const mbmi = xd->mi[0];
1299   assert(mbmi->ref_frame[0] == INTRA_FRAME);
1300   const PREDICTION_MODE mode = mbmi->mode;
1301   const ModeCosts *mode_costs = &x->mode_costs;
1302   const int mode_cost =
1303       mode_costs->mbmode_cost[size_group_lookup[bsize]][mode] + ref_frame_cost;
1304   const int skip_ctx = av1_get_skip_txfm_context(xd);
1305 
1306   int known_rate = mode_cost;
1307   const int intra_cost_penalty = av1_get_intra_cost_penalty(
1308       cm->quant_params.base_qindex, cm->quant_params.y_dc_delta_q,
1309       cm->seq_params->bit_depth);
1310 
1311   if (mode != DC_PRED && mode != PAETH_PRED) known_rate += intra_cost_penalty;
1312   known_rate += AOMMIN(mode_costs->skip_txfm_cost[skip_ctx][0],
1313                        mode_costs->skip_txfm_cost[skip_ctx][1]);
1314   const int64_t known_rd = RDCOST(x->rdmult, known_rate, 0);
1315   if (known_rd > best_rd) {
1316     intra_search_state->skip_intra_modes = 1;
1317     return 0;
1318   }
1319 
1320   const int is_directional_mode = av1_is_directional_mode(mode);
1321   if (is_directional_mode && av1_use_angle_delta(bsize) &&
1322       cpi->oxcf.intra_mode_cfg.enable_angle_delta) {
1323     if (intra_sf->intra_pruning_with_hog &&
1324         !intra_search_state->dir_mode_skip_mask_ready) {
1325       const float thresh[4] = { -1.2f, 0.0f, 0.0f, 1.2f };
1326       const int is_chroma = 0;
1327       prune_intra_mode_with_hog(x, bsize, cm->seq_params->sb_size,
1328                                 thresh[intra_sf->intra_pruning_with_hog - 1],
1329                                 intra_search_state->directional_mode_skip_mask,
1330                                 is_chroma);
1331       intra_search_state->dir_mode_skip_mask_ready = 1;
1332     }
1333     if (intra_search_state->directional_mode_skip_mask[mode]) return 0;
1334   }
1335   const TX_SIZE tx_size = AOMMIN(TX_32X32, max_txsize_lookup[bsize]);
1336   const int64_t this_model_rd =
1337       intra_model_rd(&cpi->common, x, 0, bsize, tx_size, /*use_hadamard=*/1);
1338 
1339   const int model_rd_index_for_pruning =
1340       get_model_rd_index_for_pruning(x, intra_sf);
1341 
1342   if (prune_intra_y_mode(this_model_rd, best_model_rd, top_intra_model_rd,
1343                          intra_sf->top_intra_model_count_allowed,
1344                          model_rd_index_for_pruning))
1345     return 0;
1346   av1_init_rd_stats(rd_stats_y);
1347   av1_pick_uniform_tx_size_type_yrd(cpi, x, rd_stats_y, bsize, best_rd);
1348 
1349   // Pick filter intra modes.
1350   if (mode == DC_PRED && av1_filter_intra_allowed_bsize(cm, bsize)) {
1351     int try_filter_intra = 1;
1352     int64_t best_rd_so_far = INT64_MAX;
1353     if (rd_stats_y->rate != INT_MAX) {
1354       // best_rd_so_far is the rdcost of DC_PRED without using filter_intra.
1355       // Later, in filter intra search, best_rd_so_far is used for comparison.
1356       mbmi->filter_intra_mode_info.use_filter_intra = 0;
1357       const int tmp_rate =
1358           rd_stats_y->rate +
1359           intra_mode_info_cost_y(cpi, x, mbmi, bsize, mode_cost, 0);
1360       best_rd_so_far = RDCOST(x->rdmult, tmp_rate, rd_stats_y->dist);
1361       try_filter_intra = (best_rd_so_far / 2) <= best_rd;
1362     } else if (intra_sf->skip_filter_intra_in_inter_frames >= 1) {
1363       // As rd cost of luma intra dc mode is more than best_rd (i.e.,
1364       // rd_stats_y->rate = INT_MAX), skip the evaluation of filter intra modes.
1365       try_filter_intra = 0;
1366     }
1367 
1368     if (try_filter_intra) {
1369       handle_filter_intra_mode(cpi, x, bsize, ctx, rd_stats_y, mode_cost,
1370                                best_rd, best_rd_so_far);
1371     }
1372   }
1373 
1374   if (rd_stats_y->rate == INT_MAX) return 0;
1375 
1376   *mode_cost_y = intra_mode_info_cost_y(cpi, x, mbmi, bsize, mode_cost, 0);
1377   const int rate_y = rd_stats_y->skip_txfm
1378                          ? mode_costs->skip_txfm_cost[skip_ctx][1]
1379                          : rd_stats_y->rate;
1380   *rd_y = RDCOST(x->rdmult, rate_y + *mode_cost_y, rd_stats_y->dist);
1381   if (best_rd < (INT64_MAX / 2) && *rd_y > (best_rd + (best_rd >> 2))) {
1382     intra_search_state->skip_intra_modes = 1;
1383     return 0;
1384   }
1385 
1386   return 1;
1387 }
1388 
av1_search_intra_uv_modes_in_interframe(IntraModeSearchState * intra_search_state,const AV1_COMP * cpi,MACROBLOCK * x,BLOCK_SIZE bsize,RD_STATS * rd_stats,const RD_STATS * rd_stats_y,RD_STATS * rd_stats_uv,int64_t best_rd)1389 int av1_search_intra_uv_modes_in_interframe(
1390     IntraModeSearchState *intra_search_state, const AV1_COMP *cpi,
1391     MACROBLOCK *x, BLOCK_SIZE bsize, RD_STATS *rd_stats,
1392     const RD_STATS *rd_stats_y, RD_STATS *rd_stats_uv, int64_t best_rd) {
1393   const AV1_COMMON *cm = &cpi->common;
1394   MACROBLOCKD *const xd = &x->e_mbd;
1395   MB_MODE_INFO *const mbmi = xd->mi[0];
1396   assert(mbmi->ref_frame[0] == INTRA_FRAME);
1397 
1398   // TODO([email protected]): Consolidate the chroma search code here with
1399   // the one in av1_search_palette_mode.
1400   PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
1401   const int try_palette =
1402       cpi->oxcf.tool_cfg.enable_palette &&
1403       av1_allow_palette(cm->features.allow_screen_content_tools, mbmi->bsize);
1404 
1405   assert(intra_search_state->rate_uv_intra == INT_MAX);
1406   if (intra_search_state->rate_uv_intra == INT_MAX) {
1407     // If no good uv-predictor had been found, search for it.
1408     const TX_SIZE uv_tx = av1_get_tx_size(AOM_PLANE_U, xd);
1409     av1_rd_pick_intra_sbuv_mode(cpi, x, &intra_search_state->rate_uv_intra,
1410                                 &intra_search_state->rate_uv_tokenonly,
1411                                 &intra_search_state->dist_uvs,
1412                                 &intra_search_state->skip_uvs, bsize, uv_tx);
1413     intra_search_state->mode_uv = mbmi->uv_mode;
1414     if (try_palette) intra_search_state->pmi_uv = *pmi;
1415     intra_search_state->uv_angle_delta = mbmi->angle_delta[PLANE_TYPE_UV];
1416 
1417     const int uv_rate = intra_search_state->rate_uv_tokenonly;
1418     const int64_t uv_dist = intra_search_state->dist_uvs;
1419     const int64_t uv_rd = RDCOST(x->rdmult, uv_rate, uv_dist);
1420     if (uv_rd > best_rd) {
1421       // If there is no good intra uv-mode available, we can skip all intra
1422       // modes.
1423       intra_search_state->skip_intra_modes = 1;
1424       return 0;
1425     }
1426   }
1427 
1428   // If we are here, then the encoder has found at least one good intra uv
1429   // predictor, so we can directly copy its statistics over.
1430   // TODO(any): the stats here is not right if the best uv mode is CFL but the
1431   // best y mode is palette.
1432   rd_stats_uv->rate = intra_search_state->rate_uv_tokenonly;
1433   rd_stats_uv->dist = intra_search_state->dist_uvs;
1434   rd_stats_uv->skip_txfm = intra_search_state->skip_uvs;
1435   rd_stats->skip_txfm = rd_stats_y->skip_txfm && rd_stats_uv->skip_txfm;
1436   mbmi->uv_mode = intra_search_state->mode_uv;
1437   if (try_palette) {
1438     pmi->palette_size[1] = intra_search_state->pmi_uv.palette_size[1];
1439     memcpy(pmi->palette_colors + PALETTE_MAX_SIZE,
1440            intra_search_state->pmi_uv.palette_colors + PALETTE_MAX_SIZE,
1441            2 * PALETTE_MAX_SIZE * sizeof(pmi->palette_colors[0]));
1442   }
1443   mbmi->angle_delta[PLANE_TYPE_UV] = intra_search_state->uv_angle_delta;
1444 
1445   return 1;
1446 }
1447 
1448 // Checks if odd delta angles can be pruned based on rdcosts of even delta
1449 // angles of the corresponding directional mode.
prune_luma_odd_delta_angles_using_rd_cost(const MB_MODE_INFO * const mbmi,const int64_t * const intra_modes_rd_cost,int64_t best_rd,int prune_luma_odd_delta_angles_in_intra)1450 static inline int prune_luma_odd_delta_angles_using_rd_cost(
1451     const MB_MODE_INFO *const mbmi, const int64_t *const intra_modes_rd_cost,
1452     int64_t best_rd, int prune_luma_odd_delta_angles_in_intra) {
1453   const int luma_delta_angle = mbmi->angle_delta[PLANE_TYPE_Y];
1454   if (!prune_luma_odd_delta_angles_in_intra ||
1455       !av1_is_directional_mode(mbmi->mode) || !(abs(luma_delta_angle) & 1) ||
1456       best_rd == INT64_MAX)
1457     return 0;
1458 
1459   const int64_t rd_thresh = best_rd + (best_rd >> 3);
1460 
1461   // Neighbour rdcosts are considered for pruning of odd delta angles as
1462   // mentioned below:
1463   // Delta angle      Delta angle rdcost
1464   // to be pruned     to be considered
1465   //    -3                   -2
1466   //    -1                -2, 0
1467   //     1                 0, 2
1468   //     3                    2
1469   return intra_modes_rd_cost[luma_delta_angle + MAX_ANGLE_DELTA] > rd_thresh &&
1470          intra_modes_rd_cost[luma_delta_angle + MAX_ANGLE_DELTA + 2] >
1471              rd_thresh;
1472 }
1473 
1474 // Finds the best non-intrabc mode on an intra frame.
av1_rd_pick_intra_sby_mode(const AV1_COMP * const cpi,MACROBLOCK * x,int * rate,int * rate_tokenonly,int64_t * distortion,uint8_t * skippable,BLOCK_SIZE bsize,int64_t best_rd,PICK_MODE_CONTEXT * ctx)1475 int64_t av1_rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
1476                                    int *rate, int *rate_tokenonly,
1477                                    int64_t *distortion, uint8_t *skippable,
1478                                    BLOCK_SIZE bsize, int64_t best_rd,
1479                                    PICK_MODE_CONTEXT *ctx) {
1480   MACROBLOCKD *const xd = &x->e_mbd;
1481   MB_MODE_INFO *const mbmi = xd->mi[0];
1482   assert(!is_inter_block(mbmi));
1483   int64_t best_model_rd = INT64_MAX;
1484   int is_directional_mode;
1485   uint8_t directional_mode_skip_mask[INTRA_MODES] = { 0 };
1486   // Flag to check rd of any intra mode is better than best_rd passed to this
1487   // function
1488   int beat_best_rd = 0;
1489   const int *bmode_costs;
1490   const IntraModeCfg *const intra_mode_cfg = &cpi->oxcf.intra_mode_cfg;
1491   PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
1492   const int try_palette =
1493       cpi->oxcf.tool_cfg.enable_palette &&
1494       av1_allow_palette(cpi->common.features.allow_screen_content_tools,
1495                         mbmi->bsize);
1496   uint8_t *best_palette_color_map =
1497       try_palette ? x->palette_buffer->best_palette_color_map : NULL;
1498   const MB_MODE_INFO *above_mi = xd->above_mbmi;
1499   const MB_MODE_INFO *left_mi = xd->left_mbmi;
1500   const PREDICTION_MODE A = av1_above_block_mode(above_mi);
1501   const PREDICTION_MODE L = av1_left_block_mode(left_mi);
1502   const int above_ctx = intra_mode_context[A];
1503   const int left_ctx = intra_mode_context[L];
1504   bmode_costs = x->mode_costs.y_mode_costs[above_ctx][left_ctx];
1505 
1506   mbmi->angle_delta[PLANE_TYPE_Y] = 0;
1507   const INTRA_MODE_SPEED_FEATURES *const intra_sf = &cpi->sf.intra_sf;
1508   if (intra_sf->intra_pruning_with_hog) {
1509     // Less aggressive thresholds are used here than those used in inter frame
1510     // encoding in av1_handle_intra_y_mode() because we want key frames/intra
1511     // frames to have higher quality.
1512     const float thresh[4] = { -1.2f, -1.2f, -0.6f, 0.4f };
1513     const int is_chroma = 0;
1514     prune_intra_mode_with_hog(x, bsize, cpi->common.seq_params->sb_size,
1515                               thresh[intra_sf->intra_pruning_with_hog - 1],
1516                               directional_mode_skip_mask, is_chroma);
1517   }
1518   mbmi->filter_intra_mode_info.use_filter_intra = 0;
1519   pmi->palette_size[0] = 0;
1520 
1521   // Set params for mode evaluation
1522   set_mode_eval_params(cpi, x, MODE_EVAL);
1523 
1524   MB_MODE_INFO best_mbmi = *mbmi;
1525   const int max_winner_mode_count =
1526       winner_mode_count_allowed[cpi->sf.winner_mode_sf.multi_winner_mode_type];
1527   zero_winner_mode_stats(bsize, max_winner_mode_count, x->winner_mode_stats);
1528   x->winner_mode_count = 0;
1529 
1530   // Searches the intra-modes except for intrabc, palette, and filter_intra.
1531   int64_t top_intra_model_rd[TOP_INTRA_MODEL_COUNT];
1532   for (int i = 0; i < TOP_INTRA_MODEL_COUNT; i++) {
1533     top_intra_model_rd[i] = INT64_MAX;
1534   }
1535 
1536   // Initialize the rdcost corresponding to all the directional and
1537   // non-directional intra modes.
1538   // 1. For directional modes, it stores the rdcost values for delta angles -4,
1539   // -3, ..., 3, 4.
1540   // 2. The rdcost value for luma_delta_angle is stored at index
1541   // luma_delta_angle + MAX_ANGLE_DELTA + 1.
1542   // 3. The rdcost values for fictitious/nonexistent luma_delta_angle -4 and 4
1543   // (array indices 0 and 8) are always set to INT64_MAX (the initial value).
1544   int64_t intra_modes_rd_cost[INTRA_MODE_END]
1545                              [SIZE_OF_ANGLE_DELTA_RD_COST_ARRAY];
1546   for (int i = 0; i < INTRA_MODE_END; i++) {
1547     for (int j = 0; j < SIZE_OF_ANGLE_DELTA_RD_COST_ARRAY; j++) {
1548       intra_modes_rd_cost[i][j] = INT64_MAX;
1549     }
1550   }
1551 
1552   for (int mode_idx = INTRA_MODE_START; mode_idx < LUMA_MODE_COUNT;
1553        ++mode_idx) {
1554     set_y_mode_and_delta_angle(mode_idx, mbmi,
1555                                intra_sf->prune_luma_odd_delta_angles_in_intra);
1556     RD_STATS this_rd_stats;
1557     int this_rate, this_rate_tokenonly, s;
1558     int is_diagonal_mode;
1559     int64_t this_distortion, this_rd;
1560     const int luma_delta_angle = mbmi->angle_delta[PLANE_TYPE_Y];
1561 
1562     is_diagonal_mode = av1_is_diagonal_mode(mbmi->mode);
1563     if (is_diagonal_mode && !intra_mode_cfg->enable_diagonal_intra) continue;
1564     if (av1_is_directional_mode(mbmi->mode) &&
1565         !intra_mode_cfg->enable_directional_intra)
1566       continue;
1567 
1568     // The smooth prediction mode appears to be more frequently picked
1569     // than horizontal / vertical smooth prediction modes. Hence treat
1570     // them differently in speed features.
1571     if ((!intra_mode_cfg->enable_smooth_intra ||
1572          intra_sf->disable_smooth_intra) &&
1573         (mbmi->mode == SMOOTH_H_PRED || mbmi->mode == SMOOTH_V_PRED))
1574       continue;
1575     if (!intra_mode_cfg->enable_smooth_intra && mbmi->mode == SMOOTH_PRED)
1576       continue;
1577 
1578     // The functionality of filter intra modes and smooth prediction
1579     // overlap. Hence smooth prediction is pruned only if all the
1580     // filter intra modes are enabled.
1581     if (intra_sf->disable_smooth_intra &&
1582         intra_sf->prune_filter_intra_level == 0 && mbmi->mode == SMOOTH_PRED)
1583       continue;
1584     if (!intra_mode_cfg->enable_paeth_intra && mbmi->mode == PAETH_PRED)
1585       continue;
1586 
1587     // Skip the evaluation of modes that do not match with the winner mode in
1588     // x->mb_mode_cache.
1589     if (x->use_mb_mode_cache && mbmi->mode != x->mb_mode_cache->mode) continue;
1590 
1591     is_directional_mode = av1_is_directional_mode(mbmi->mode);
1592     if (is_directional_mode && directional_mode_skip_mask[mbmi->mode]) continue;
1593     if (is_directional_mode &&
1594         !(av1_use_angle_delta(bsize) && intra_mode_cfg->enable_angle_delta) &&
1595         luma_delta_angle != 0)
1596       continue;
1597 
1598     // Use intra_y_mode_mask speed feature to skip intra mode evaluation.
1599     if (!(intra_sf->intra_y_mode_mask[max_txsize_lookup[bsize]] &
1600           (1 << mbmi->mode)))
1601       continue;
1602 
1603     if (prune_luma_odd_delta_angles_using_rd_cost(
1604             mbmi, intra_modes_rd_cost[mbmi->mode], best_rd,
1605             intra_sf->prune_luma_odd_delta_angles_in_intra))
1606       continue;
1607 
1608     const TX_SIZE tx_size = AOMMIN(TX_32X32, max_txsize_lookup[bsize]);
1609     const int64_t this_model_rd =
1610         intra_model_rd(&cpi->common, x, 0, bsize, tx_size, /*use_hadamard=*/1);
1611 
1612     const int model_rd_index_for_pruning =
1613         get_model_rd_index_for_pruning(x, intra_sf);
1614 
1615     if (prune_intra_y_mode(this_model_rd, &best_model_rd, top_intra_model_rd,
1616                            intra_sf->top_intra_model_count_allowed,
1617                            model_rd_index_for_pruning))
1618       continue;
1619 
1620     // Builds the actual prediction. The prediction from
1621     // model_intra_yrd_and_prune was just an estimation that did not take into
1622     // account the effect of txfm pipeline, so we need to redo it for real
1623     // here.
1624     av1_pick_uniform_tx_size_type_yrd(cpi, x, &this_rd_stats, bsize, best_rd);
1625     this_rate_tokenonly = this_rd_stats.rate;
1626     this_distortion = this_rd_stats.dist;
1627     s = this_rd_stats.skip_txfm;
1628 
1629     if (this_rate_tokenonly == INT_MAX) continue;
1630 
1631     if (!xd->lossless[mbmi->segment_id] && block_signals_txsize(mbmi->bsize)) {
1632       // av1_pick_uniform_tx_size_type_yrd above includes the cost of the
1633       // tx_size in the tokenonly rate, but for intra blocks, tx_size is always
1634       // coded (prediction granularity), so we account for it in the full rate,
1635       // not the tokenonly rate.
1636       this_rate_tokenonly -= tx_size_cost(x, bsize, mbmi->tx_size);
1637     }
1638     this_rate =
1639         this_rd_stats.rate +
1640         intra_mode_info_cost_y(cpi, x, mbmi, bsize, bmode_costs[mbmi->mode], 0);
1641     this_rd = RDCOST(x->rdmult, this_rate, this_distortion);
1642 
1643     // Visual quality adjustment based on recon vs source variance.
1644     if ((cpi->oxcf.mode == ALLINTRA) && (this_rd != INT64_MAX)) {
1645       this_rd = (int64_t)(this_rd * intra_rd_variance_factor(cpi, x, bsize));
1646     }
1647 
1648     intra_modes_rd_cost[mbmi->mode][luma_delta_angle + MAX_ANGLE_DELTA + 1] =
1649         this_rd;
1650 
1651     // Collect mode stats for multiwinner mode processing
1652     const int txfm_search_done = 1;
1653     store_winner_mode_stats(
1654         &cpi->common, x, mbmi, NULL, NULL, NULL, 0, NULL, bsize, this_rd,
1655         cpi->sf.winner_mode_sf.multi_winner_mode_type, txfm_search_done);
1656     if (this_rd < best_rd) {
1657       best_mbmi = *mbmi;
1658       best_rd = this_rd;
1659       // Setting beat_best_rd flag because current mode rd is better than
1660       // best_rd passed to this function
1661       beat_best_rd = 1;
1662       *rate = this_rate;
1663       *rate_tokenonly = this_rate_tokenonly;
1664       *distortion = this_distortion;
1665       *skippable = s;
1666       memcpy(ctx->blk_skip, x->txfm_search_info.blk_skip,
1667              sizeof(x->txfm_search_info.blk_skip[0]) * ctx->num_4x4_blk);
1668       av1_copy_array(ctx->tx_type_map, xd->tx_type_map, ctx->num_4x4_blk);
1669     }
1670   }
1671 
1672   // Searches palette
1673   if (try_palette) {
1674     av1_rd_pick_palette_intra_sby(
1675         cpi, x, bsize, bmode_costs[DC_PRED], &best_mbmi, best_palette_color_map,
1676         &best_rd, rate, rate_tokenonly, distortion, skippable, &beat_best_rd,
1677         ctx, ctx->blk_skip, ctx->tx_type_map);
1678   }
1679 
1680   // Searches filter_intra
1681   if (beat_best_rd && av1_filter_intra_allowed_bsize(&cpi->common, bsize)) {
1682     if (rd_pick_filter_intra_sby(cpi, x, rate, rate_tokenonly, distortion,
1683                                  skippable, bsize, bmode_costs[DC_PRED],
1684                                  best_mbmi.mode, &best_rd, &best_model_rd,
1685                                  ctx)) {
1686       best_mbmi = *mbmi;
1687     }
1688   }
1689 
1690   // No mode is identified with less rd value than best_rd passed to this
1691   // function. In such cases winner mode processing is not necessary and return
1692   // best_rd as INT64_MAX to indicate best mode is not identified
1693   if (!beat_best_rd) return INT64_MAX;
1694 
1695   // In multi-winner mode processing, perform tx search for few best modes
1696   // identified during mode evaluation. Winner mode processing uses best tx
1697   // configuration for tx search.
1698   if (cpi->sf.winner_mode_sf.multi_winner_mode_type) {
1699     int best_mode_idx = 0;
1700     int block_width, block_height;
1701     uint8_t *color_map_dst = xd->plane[PLANE_TYPE_Y].color_index_map;
1702     av1_get_block_dimensions(bsize, AOM_PLANE_Y, xd, &block_width,
1703                              &block_height, NULL, NULL);
1704 
1705     for (int mode_idx = 0; mode_idx < x->winner_mode_count; mode_idx++) {
1706       *mbmi = x->winner_mode_stats[mode_idx].mbmi;
1707       if (is_winner_mode_processing_enabled(cpi, x, mbmi, 0)) {
1708         // Restore color_map of palette mode before winner mode processing
1709         if (mbmi->palette_mode_info.palette_size[0] > 0) {
1710           uint8_t *color_map_src =
1711               x->winner_mode_stats[mode_idx].color_index_map;
1712           memcpy(color_map_dst, color_map_src,
1713                  block_width * block_height * sizeof(*color_map_src));
1714         }
1715         // Set params for winner mode evaluation
1716         set_mode_eval_params(cpi, x, WINNER_MODE_EVAL);
1717 
1718         // Winner mode processing
1719         // If previous searches use only the default tx type/no R-D optimization
1720         // of quantized coeffs, do an extra search for the best tx type/better
1721         // R-D optimization of quantized coeffs
1722         if (intra_block_yrd(cpi, x, bsize, bmode_costs, &best_rd, rate,
1723                             rate_tokenonly, distortion, skippable, &best_mbmi,
1724                             ctx))
1725           best_mode_idx = mode_idx;
1726       }
1727     }
1728     // Copy color_map of palette mode for final winner mode
1729     if (best_mbmi.palette_mode_info.palette_size[0] > 0) {
1730       uint8_t *color_map_src =
1731           x->winner_mode_stats[best_mode_idx].color_index_map;
1732       memcpy(color_map_dst, color_map_src,
1733              block_width * block_height * sizeof(*color_map_src));
1734     }
1735   } else {
1736     // If previous searches use only the default tx type/no R-D optimization of
1737     // quantized coeffs, do an extra search for the best tx type/better R-D
1738     // optimization of quantized coeffs
1739     if (is_winner_mode_processing_enabled(cpi, x, mbmi, 0)) {
1740       // Set params for winner mode evaluation
1741       set_mode_eval_params(cpi, x, WINNER_MODE_EVAL);
1742       *mbmi = best_mbmi;
1743       intra_block_yrd(cpi, x, bsize, bmode_costs, &best_rd, rate,
1744                       rate_tokenonly, distortion, skippable, &best_mbmi, ctx);
1745     }
1746   }
1747   *mbmi = best_mbmi;
1748   av1_copy_array(xd->tx_type_map, ctx->tx_type_map, ctx->num_4x4_blk);
1749   return best_rd;
1750 }
1751