1*77c1e3ccSAndroid Build Coastguard Worker /*
2*77c1e3ccSAndroid Build Coastguard Worker * Copyright (c) 2020, Alliance for Open Media. All rights reserved.
3*77c1e3ccSAndroid Build Coastguard Worker *
4*77c1e3ccSAndroid Build Coastguard Worker * This source code is subject to the terms of the BSD 2 Clause License and
5*77c1e3ccSAndroid Build Coastguard Worker * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6*77c1e3ccSAndroid Build Coastguard Worker * was not distributed with this source code in the LICENSE file, you can
7*77c1e3ccSAndroid Build Coastguard Worker * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8*77c1e3ccSAndroid Build Coastguard Worker * Media Patent License 1.0 was not distributed with this source code in the
9*77c1e3ccSAndroid Build Coastguard Worker * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10*77c1e3ccSAndroid Build Coastguard Worker */
11*77c1e3ccSAndroid Build Coastguard Worker
12*77c1e3ccSAndroid Build Coastguard Worker #include "av1/common/av1_common_int.h"
13*77c1e3ccSAndroid Build Coastguard Worker #include "av1/common/cfl.h"
14*77c1e3ccSAndroid Build Coastguard Worker #include "av1/common/reconintra.h"
15*77c1e3ccSAndroid Build Coastguard Worker
16*77c1e3ccSAndroid Build Coastguard Worker #include "av1/encoder/intra_mode_search.h"
17*77c1e3ccSAndroid Build Coastguard Worker #include "av1/encoder/intra_mode_search_utils.h"
18*77c1e3ccSAndroid Build Coastguard Worker #include "av1/encoder/palette.h"
19*77c1e3ccSAndroid Build Coastguard Worker #include "av1/encoder/speed_features.h"
20*77c1e3ccSAndroid Build Coastguard Worker #include "av1/encoder/tx_search.h"
21*77c1e3ccSAndroid Build Coastguard Worker
22*77c1e3ccSAndroid Build Coastguard Worker // Even though there are 7 delta angles, this macro is set to 9 to facilitate
23*77c1e3ccSAndroid Build Coastguard Worker // the rd threshold check to prune -3 and 3 delta angles.
24*77c1e3ccSAndroid Build Coastguard Worker #define SIZE_OF_ANGLE_DELTA_RD_COST_ARRAY (2 * MAX_ANGLE_DELTA + 3)
25*77c1e3ccSAndroid Build Coastguard Worker
26*77c1e3ccSAndroid Build Coastguard Worker // The order for evaluating delta angles while processing the luma directional
27*77c1e3ccSAndroid Build Coastguard Worker // intra modes. Currently, this order of evaluation is applicable only when
28*77c1e3ccSAndroid Build Coastguard Worker // speed feature prune_luma_odd_delta_angles_in_intra is enabled. In this case,
29*77c1e3ccSAndroid Build Coastguard Worker // even angles are evaluated first in order to facilitate the pruning of odd
30*77c1e3ccSAndroid Build Coastguard Worker // delta angles based on the rd costs of the neighboring delta angles.
31*77c1e3ccSAndroid Build Coastguard Worker static const int8_t luma_delta_angles_order[2 * MAX_ANGLE_DELTA] = {
32*77c1e3ccSAndroid Build Coastguard Worker -2, 2, -3, -1, 1, 3,
33*77c1e3ccSAndroid Build Coastguard Worker };
34*77c1e3ccSAndroid Build Coastguard Worker
35*77c1e3ccSAndroid Build Coastguard Worker /*!\cond */
36*77c1e3ccSAndroid Build Coastguard Worker static const PREDICTION_MODE intra_rd_search_mode_order[INTRA_MODES] = {
37*77c1e3ccSAndroid Build Coastguard Worker DC_PRED, H_PRED, V_PRED, SMOOTH_PRED, PAETH_PRED,
38*77c1e3ccSAndroid Build Coastguard Worker SMOOTH_V_PRED, SMOOTH_H_PRED, D135_PRED, D203_PRED, D157_PRED,
39*77c1e3ccSAndroid Build Coastguard Worker D67_PRED, D113_PRED, D45_PRED,
40*77c1e3ccSAndroid Build Coastguard Worker };
41*77c1e3ccSAndroid Build Coastguard Worker
42*77c1e3ccSAndroid Build Coastguard Worker static const UV_PREDICTION_MODE uv_rd_search_mode_order[UV_INTRA_MODES] = {
43*77c1e3ccSAndroid Build Coastguard Worker UV_DC_PRED, UV_CFL_PRED, UV_H_PRED, UV_V_PRED,
44*77c1e3ccSAndroid Build Coastguard Worker UV_SMOOTH_PRED, UV_PAETH_PRED, UV_SMOOTH_V_PRED, UV_SMOOTH_H_PRED,
45*77c1e3ccSAndroid Build Coastguard Worker UV_D135_PRED, UV_D203_PRED, UV_D157_PRED, UV_D67_PRED,
46*77c1e3ccSAndroid Build Coastguard Worker UV_D113_PRED, UV_D45_PRED,
47*77c1e3ccSAndroid Build Coastguard Worker };
48*77c1e3ccSAndroid Build Coastguard Worker
49*77c1e3ccSAndroid Build Coastguard Worker // The bitmask corresponds to the filter intra modes as defined in enums.h
50*77c1e3ccSAndroid Build Coastguard Worker // FILTER_INTRA_MODE enumeration type. Setting a bit to 0 in the mask means to
51*77c1e3ccSAndroid Build Coastguard Worker // disable the evaluation of corresponding filter intra mode. The table
52*77c1e3ccSAndroid Build Coastguard Worker // av1_derived_filter_intra_mode_used_flag is used when speed feature
53*77c1e3ccSAndroid Build Coastguard Worker // prune_filter_intra_level is 1. The evaluated filter intra modes are union
54*77c1e3ccSAndroid Build Coastguard Worker // of the following:
55*77c1e3ccSAndroid Build Coastguard Worker // 1) FILTER_DC_PRED
56*77c1e3ccSAndroid Build Coastguard Worker // 2) mode that corresponds to best mode so far of DC_PRED, V_PRED, H_PRED,
57*77c1e3ccSAndroid Build Coastguard Worker // D157_PRED and PAETH_PRED. (Eg: FILTER_V_PRED if best mode so far is V_PRED).
58*77c1e3ccSAndroid Build Coastguard Worker static const uint8_t av1_derived_filter_intra_mode_used_flag[INTRA_MODES] = {
59*77c1e3ccSAndroid Build Coastguard Worker 0x01, // DC_PRED: 0000 0001
60*77c1e3ccSAndroid Build Coastguard Worker 0x03, // V_PRED: 0000 0011
61*77c1e3ccSAndroid Build Coastguard Worker 0x05, // H_PRED: 0000 0101
62*77c1e3ccSAndroid Build Coastguard Worker 0x01, // D45_PRED: 0000 0001
63*77c1e3ccSAndroid Build Coastguard Worker 0x01, // D135_PRED: 0000 0001
64*77c1e3ccSAndroid Build Coastguard Worker 0x01, // D113_PRED: 0000 0001
65*77c1e3ccSAndroid Build Coastguard Worker 0x09, // D157_PRED: 0000 1001
66*77c1e3ccSAndroid Build Coastguard Worker 0x01, // D203_PRED: 0000 0001
67*77c1e3ccSAndroid Build Coastguard Worker 0x01, // D67_PRED: 0000 0001
68*77c1e3ccSAndroid Build Coastguard Worker 0x01, // SMOOTH_PRED: 0000 0001
69*77c1e3ccSAndroid Build Coastguard Worker 0x01, // SMOOTH_V_PRED: 0000 0001
70*77c1e3ccSAndroid Build Coastguard Worker 0x01, // SMOOTH_H_PRED: 0000 0001
71*77c1e3ccSAndroid Build Coastguard Worker 0x11 // PAETH_PRED: 0001 0001
72*77c1e3ccSAndroid Build Coastguard Worker };
73*77c1e3ccSAndroid Build Coastguard Worker
74*77c1e3ccSAndroid Build Coastguard Worker // The bitmask corresponds to the chroma intra modes as defined in enums.h
75*77c1e3ccSAndroid Build Coastguard Worker // UV_PREDICTION_MODE enumeration type. Setting a bit to 0 in the mask means to
76*77c1e3ccSAndroid Build Coastguard Worker // disable the evaluation of corresponding chroma intra mode. The table
77*77c1e3ccSAndroid Build Coastguard Worker // av1_derived_chroma_intra_mode_used_flag is used when speed feature
78*77c1e3ccSAndroid Build Coastguard Worker // prune_chroma_modes_using_luma_winner is enabled. The evaluated chroma
79*77c1e3ccSAndroid Build Coastguard Worker // intra modes are union of the following:
80*77c1e3ccSAndroid Build Coastguard Worker // 1) UV_DC_PRED
81*77c1e3ccSAndroid Build Coastguard Worker // 2) UV_SMOOTH_PRED
82*77c1e3ccSAndroid Build Coastguard Worker // 3) UV_CFL_PRED
83*77c1e3ccSAndroid Build Coastguard Worker // 4) mode that corresponds to luma intra mode winner (Eg : UV_V_PRED if luma
84*77c1e3ccSAndroid Build Coastguard Worker // intra mode winner is V_PRED).
85*77c1e3ccSAndroid Build Coastguard Worker static const uint16_t av1_derived_chroma_intra_mode_used_flag[INTRA_MODES] = {
86*77c1e3ccSAndroid Build Coastguard Worker 0x2201, // DC_PRED: 0010 0010 0000 0001
87*77c1e3ccSAndroid Build Coastguard Worker 0x2203, // V_PRED: 0010 0010 0000 0011
88*77c1e3ccSAndroid Build Coastguard Worker 0x2205, // H_PRED: 0010 0010 0000 0101
89*77c1e3ccSAndroid Build Coastguard Worker 0x2209, // D45_PRED: 0010 0010 0000 1001
90*77c1e3ccSAndroid Build Coastguard Worker 0x2211, // D135_PRED: 0010 0010 0001 0001
91*77c1e3ccSAndroid Build Coastguard Worker 0x2221, // D113_PRED: 0010 0010 0010 0001
92*77c1e3ccSAndroid Build Coastguard Worker 0x2241, // D157_PRED: 0010 0010 0100 0001
93*77c1e3ccSAndroid Build Coastguard Worker 0x2281, // D203_PRED: 0010 0010 1000 0001
94*77c1e3ccSAndroid Build Coastguard Worker 0x2301, // D67_PRED: 0010 0011 0000 0001
95*77c1e3ccSAndroid Build Coastguard Worker 0x2201, // SMOOTH_PRED: 0010 0010 0000 0001
96*77c1e3ccSAndroid Build Coastguard Worker 0x2601, // SMOOTH_V_PRED: 0010 0110 0000 0001
97*77c1e3ccSAndroid Build Coastguard Worker 0x2a01, // SMOOTH_H_PRED: 0010 1010 0000 0001
98*77c1e3ccSAndroid Build Coastguard Worker 0x3201 // PAETH_PRED: 0011 0010 0000 0001
99*77c1e3ccSAndroid Build Coastguard Worker };
100*77c1e3ccSAndroid Build Coastguard Worker
101*77c1e3ccSAndroid Build Coastguard Worker DECLARE_ALIGNED(16, static const uint8_t, all_zeros[MAX_SB_SIZE]) = { 0 };
102*77c1e3ccSAndroid Build Coastguard Worker DECLARE_ALIGNED(16, static const uint16_t,
103*77c1e3ccSAndroid Build Coastguard Worker highbd_all_zeros[MAX_SB_SIZE]) = { 0 };
104*77c1e3ccSAndroid Build Coastguard Worker
av1_calc_normalized_variance(aom_variance_fn_t vf,const uint8_t * const buf,const int stride,const int is_hbd)105*77c1e3ccSAndroid Build Coastguard Worker int av1_calc_normalized_variance(aom_variance_fn_t vf, const uint8_t *const buf,
106*77c1e3ccSAndroid Build Coastguard Worker const int stride, const int is_hbd) {
107*77c1e3ccSAndroid Build Coastguard Worker unsigned int sse;
108*77c1e3ccSAndroid Build Coastguard Worker
109*77c1e3ccSAndroid Build Coastguard Worker if (is_hbd)
110*77c1e3ccSAndroid Build Coastguard Worker return vf(buf, stride, CONVERT_TO_BYTEPTR(highbd_all_zeros), 0, &sse);
111*77c1e3ccSAndroid Build Coastguard Worker else
112*77c1e3ccSAndroid Build Coastguard Worker return vf(buf, stride, all_zeros, 0, &sse);
113*77c1e3ccSAndroid Build Coastguard Worker }
114*77c1e3ccSAndroid Build Coastguard Worker
115*77c1e3ccSAndroid Build Coastguard Worker // Computes average of log(1 + variance) across 4x4 sub-blocks for source and
116*77c1e3ccSAndroid Build Coastguard Worker // reconstructed blocks.
compute_avg_log_variance(const AV1_COMP * const cpi,MACROBLOCK * x,const BLOCK_SIZE bs,double * avg_log_src_variance,double * avg_log_recon_variance)117*77c1e3ccSAndroid Build Coastguard Worker static void compute_avg_log_variance(const AV1_COMP *const cpi, MACROBLOCK *x,
118*77c1e3ccSAndroid Build Coastguard Worker const BLOCK_SIZE bs,
119*77c1e3ccSAndroid Build Coastguard Worker double *avg_log_src_variance,
120*77c1e3ccSAndroid Build Coastguard Worker double *avg_log_recon_variance) {
121*77c1e3ccSAndroid Build Coastguard Worker const MACROBLOCKD *const xd = &x->e_mbd;
122*77c1e3ccSAndroid Build Coastguard Worker const BLOCK_SIZE sb_size = cpi->common.seq_params->sb_size;
123*77c1e3ccSAndroid Build Coastguard Worker const int mi_row_in_sb = x->e_mbd.mi_row & (mi_size_high[sb_size] - 1);
124*77c1e3ccSAndroid Build Coastguard Worker const int mi_col_in_sb = x->e_mbd.mi_col & (mi_size_wide[sb_size] - 1);
125*77c1e3ccSAndroid Build Coastguard Worker const int right_overflow =
126*77c1e3ccSAndroid Build Coastguard Worker (xd->mb_to_right_edge < 0) ? ((-xd->mb_to_right_edge) >> 3) : 0;
127*77c1e3ccSAndroid Build Coastguard Worker const int bottom_overflow =
128*77c1e3ccSAndroid Build Coastguard Worker (xd->mb_to_bottom_edge < 0) ? ((-xd->mb_to_bottom_edge) >> 3) : 0;
129*77c1e3ccSAndroid Build Coastguard Worker const int bw = (MI_SIZE * mi_size_wide[bs] - right_overflow);
130*77c1e3ccSAndroid Build Coastguard Worker const int bh = (MI_SIZE * mi_size_high[bs] - bottom_overflow);
131*77c1e3ccSAndroid Build Coastguard Worker const int is_hbd = is_cur_buf_hbd(xd);
132*77c1e3ccSAndroid Build Coastguard Worker
133*77c1e3ccSAndroid Build Coastguard Worker for (int i = 0; i < bh; i += MI_SIZE) {
134*77c1e3ccSAndroid Build Coastguard Worker const int r = mi_row_in_sb + (i >> MI_SIZE_LOG2);
135*77c1e3ccSAndroid Build Coastguard Worker for (int j = 0; j < bw; j += MI_SIZE) {
136*77c1e3ccSAndroid Build Coastguard Worker const int c = mi_col_in_sb + (j >> MI_SIZE_LOG2);
137*77c1e3ccSAndroid Build Coastguard Worker const int mi_offset = r * mi_size_wide[sb_size] + c;
138*77c1e3ccSAndroid Build Coastguard Worker Block4x4VarInfo *block_4x4_var_info =
139*77c1e3ccSAndroid Build Coastguard Worker &x->src_var_info_of_4x4_sub_blocks[mi_offset];
140*77c1e3ccSAndroid Build Coastguard Worker int src_var = block_4x4_var_info->var;
141*77c1e3ccSAndroid Build Coastguard Worker double log_src_var = block_4x4_var_info->log_var;
142*77c1e3ccSAndroid Build Coastguard Worker // Compute average of log(1 + variance) for the source block from 4x4
143*77c1e3ccSAndroid Build Coastguard Worker // sub-block variance values. Calculate and store 4x4 sub-block variance
144*77c1e3ccSAndroid Build Coastguard Worker // and log(1 + variance), if the values present in
145*77c1e3ccSAndroid Build Coastguard Worker // src_var_of_4x4_sub_blocks are invalid. Reuse the same if it is readily
146*77c1e3ccSAndroid Build Coastguard Worker // available with valid values.
147*77c1e3ccSAndroid Build Coastguard Worker if (src_var < 0) {
148*77c1e3ccSAndroid Build Coastguard Worker src_var = av1_calc_normalized_variance(
149*77c1e3ccSAndroid Build Coastguard Worker cpi->ppi->fn_ptr[BLOCK_4X4].vf,
150*77c1e3ccSAndroid Build Coastguard Worker x->plane[0].src.buf + i * x->plane[0].src.stride + j,
151*77c1e3ccSAndroid Build Coastguard Worker x->plane[0].src.stride, is_hbd);
152*77c1e3ccSAndroid Build Coastguard Worker block_4x4_var_info->var = src_var;
153*77c1e3ccSAndroid Build Coastguard Worker log_src_var = log1p(src_var / 16.0);
154*77c1e3ccSAndroid Build Coastguard Worker block_4x4_var_info->log_var = log_src_var;
155*77c1e3ccSAndroid Build Coastguard Worker } else {
156*77c1e3ccSAndroid Build Coastguard Worker // When source variance is already calculated and available for
157*77c1e3ccSAndroid Build Coastguard Worker // retrieval, check if log(1 + variance) is also available. If it is
158*77c1e3ccSAndroid Build Coastguard Worker // available, then retrieve from buffer. Else, calculate the same and
159*77c1e3ccSAndroid Build Coastguard Worker // store to the buffer.
160*77c1e3ccSAndroid Build Coastguard Worker if (log_src_var < 0) {
161*77c1e3ccSAndroid Build Coastguard Worker log_src_var = log1p(src_var / 16.0);
162*77c1e3ccSAndroid Build Coastguard Worker block_4x4_var_info->log_var = log_src_var;
163*77c1e3ccSAndroid Build Coastguard Worker }
164*77c1e3ccSAndroid Build Coastguard Worker }
165*77c1e3ccSAndroid Build Coastguard Worker *avg_log_src_variance += log_src_var;
166*77c1e3ccSAndroid Build Coastguard Worker
167*77c1e3ccSAndroid Build Coastguard Worker const int recon_var = av1_calc_normalized_variance(
168*77c1e3ccSAndroid Build Coastguard Worker cpi->ppi->fn_ptr[BLOCK_4X4].vf,
169*77c1e3ccSAndroid Build Coastguard Worker xd->plane[0].dst.buf + i * xd->plane[0].dst.stride + j,
170*77c1e3ccSAndroid Build Coastguard Worker xd->plane[0].dst.stride, is_hbd);
171*77c1e3ccSAndroid Build Coastguard Worker *avg_log_recon_variance += log1p(recon_var / 16.0);
172*77c1e3ccSAndroid Build Coastguard Worker }
173*77c1e3ccSAndroid Build Coastguard Worker }
174*77c1e3ccSAndroid Build Coastguard Worker
175*77c1e3ccSAndroid Build Coastguard Worker const int blocks = (bw * bh) / 16;
176*77c1e3ccSAndroid Build Coastguard Worker *avg_log_src_variance /= (double)blocks;
177*77c1e3ccSAndroid Build Coastguard Worker *avg_log_recon_variance /= (double)blocks;
178*77c1e3ccSAndroid Build Coastguard Worker }
179*77c1e3ccSAndroid Build Coastguard Worker
180*77c1e3ccSAndroid Build Coastguard Worker // Returns a factor to be applied to the RD value based on how well the
181*77c1e3ccSAndroid Build Coastguard Worker // reconstructed block variance matches the source variance.
intra_rd_variance_factor(const AV1_COMP * cpi,MACROBLOCK * x,BLOCK_SIZE bs)182*77c1e3ccSAndroid Build Coastguard Worker static double intra_rd_variance_factor(const AV1_COMP *cpi, MACROBLOCK *x,
183*77c1e3ccSAndroid Build Coastguard Worker BLOCK_SIZE bs) {
184*77c1e3ccSAndroid Build Coastguard Worker double threshold = INTRA_RD_VAR_THRESH(cpi->oxcf.speed);
185*77c1e3ccSAndroid Build Coastguard Worker // For non-positive threshold values, the comparison of source and
186*77c1e3ccSAndroid Build Coastguard Worker // reconstructed variances with threshold evaluates to false
187*77c1e3ccSAndroid Build Coastguard Worker // (src_var < threshold/rec_var < threshold) as these metrics are greater than
188*77c1e3ccSAndroid Build Coastguard Worker // than 0. Hence further calculations are skipped.
189*77c1e3ccSAndroid Build Coastguard Worker if (threshold <= 0) return 1.0;
190*77c1e3ccSAndroid Build Coastguard Worker
191*77c1e3ccSAndroid Build Coastguard Worker double variance_rd_factor = 1.0;
192*77c1e3ccSAndroid Build Coastguard Worker double avg_log_src_variance = 0.0;
193*77c1e3ccSAndroid Build Coastguard Worker double avg_log_recon_variance = 0.0;
194*77c1e3ccSAndroid Build Coastguard Worker double var_diff = 0.0;
195*77c1e3ccSAndroid Build Coastguard Worker
196*77c1e3ccSAndroid Build Coastguard Worker compute_avg_log_variance(cpi, x, bs, &avg_log_src_variance,
197*77c1e3ccSAndroid Build Coastguard Worker &avg_log_recon_variance);
198*77c1e3ccSAndroid Build Coastguard Worker
199*77c1e3ccSAndroid Build Coastguard Worker // Dont allow 0 to prevent / 0 below.
200*77c1e3ccSAndroid Build Coastguard Worker avg_log_src_variance += 0.000001;
201*77c1e3ccSAndroid Build Coastguard Worker avg_log_recon_variance += 0.000001;
202*77c1e3ccSAndroid Build Coastguard Worker
203*77c1e3ccSAndroid Build Coastguard Worker if (avg_log_src_variance >= avg_log_recon_variance) {
204*77c1e3ccSAndroid Build Coastguard Worker var_diff = (avg_log_src_variance - avg_log_recon_variance);
205*77c1e3ccSAndroid Build Coastguard Worker if ((var_diff > 0.5) && (avg_log_recon_variance < threshold)) {
206*77c1e3ccSAndroid Build Coastguard Worker variance_rd_factor = 1.0 + ((var_diff * 2) / avg_log_src_variance);
207*77c1e3ccSAndroid Build Coastguard Worker }
208*77c1e3ccSAndroid Build Coastguard Worker } else {
209*77c1e3ccSAndroid Build Coastguard Worker var_diff = (avg_log_recon_variance - avg_log_src_variance);
210*77c1e3ccSAndroid Build Coastguard Worker if ((var_diff > 0.5) && (avg_log_src_variance < threshold)) {
211*77c1e3ccSAndroid Build Coastguard Worker variance_rd_factor = 1.0 + (var_diff / (2 * avg_log_src_variance));
212*77c1e3ccSAndroid Build Coastguard Worker }
213*77c1e3ccSAndroid Build Coastguard Worker }
214*77c1e3ccSAndroid Build Coastguard Worker
215*77c1e3ccSAndroid Build Coastguard Worker // Limit adjustment;
216*77c1e3ccSAndroid Build Coastguard Worker variance_rd_factor = AOMMIN(3.0, variance_rd_factor);
217*77c1e3ccSAndroid Build Coastguard Worker
218*77c1e3ccSAndroid Build Coastguard Worker return variance_rd_factor;
219*77c1e3ccSAndroid Build Coastguard Worker }
220*77c1e3ccSAndroid Build Coastguard Worker /*!\endcond */
221*77c1e3ccSAndroid Build Coastguard Worker
222*77c1e3ccSAndroid Build Coastguard Worker /*!\brief Search for the best filter_intra mode when coding intra frame.
223*77c1e3ccSAndroid Build Coastguard Worker *
224*77c1e3ccSAndroid Build Coastguard Worker * \ingroup intra_mode_search
225*77c1e3ccSAndroid Build Coastguard Worker * \callergraph
226*77c1e3ccSAndroid Build Coastguard Worker * This function loops through all filter_intra modes to find the best one.
227*77c1e3ccSAndroid Build Coastguard Worker *
228*77c1e3ccSAndroid Build Coastguard Worker * \return Returns 1 if a new filter_intra mode is selected; 0 otherwise.
229*77c1e3ccSAndroid Build Coastguard Worker */
rd_pick_filter_intra_sby(const AV1_COMP * const cpi,MACROBLOCK * x,int * rate,int * rate_tokenonly,int64_t * distortion,uint8_t * skippable,BLOCK_SIZE bsize,int mode_cost,PREDICTION_MODE best_mode_so_far,int64_t * best_rd,int64_t * best_model_rd,PICK_MODE_CONTEXT * ctx)230*77c1e3ccSAndroid Build Coastguard Worker static int rd_pick_filter_intra_sby(const AV1_COMP *const cpi, MACROBLOCK *x,
231*77c1e3ccSAndroid Build Coastguard Worker int *rate, int *rate_tokenonly,
232*77c1e3ccSAndroid Build Coastguard Worker int64_t *distortion, uint8_t *skippable,
233*77c1e3ccSAndroid Build Coastguard Worker BLOCK_SIZE bsize, int mode_cost,
234*77c1e3ccSAndroid Build Coastguard Worker PREDICTION_MODE best_mode_so_far,
235*77c1e3ccSAndroid Build Coastguard Worker int64_t *best_rd, int64_t *best_model_rd,
236*77c1e3ccSAndroid Build Coastguard Worker PICK_MODE_CONTEXT *ctx) {
237*77c1e3ccSAndroid Build Coastguard Worker // Skip the evaluation of filter intra modes.
238*77c1e3ccSAndroid Build Coastguard Worker if (cpi->sf.intra_sf.prune_filter_intra_level == 2) return 0;
239*77c1e3ccSAndroid Build Coastguard Worker
240*77c1e3ccSAndroid Build Coastguard Worker MACROBLOCKD *const xd = &x->e_mbd;
241*77c1e3ccSAndroid Build Coastguard Worker MB_MODE_INFO *mbmi = xd->mi[0];
242*77c1e3ccSAndroid Build Coastguard Worker int filter_intra_selected_flag = 0;
243*77c1e3ccSAndroid Build Coastguard Worker FILTER_INTRA_MODE mode;
244*77c1e3ccSAndroid Build Coastguard Worker TX_SIZE best_tx_size = TX_8X8;
245*77c1e3ccSAndroid Build Coastguard Worker FILTER_INTRA_MODE_INFO filter_intra_mode_info;
246*77c1e3ccSAndroid Build Coastguard Worker uint8_t best_tx_type_map[MAX_MIB_SIZE * MAX_MIB_SIZE];
247*77c1e3ccSAndroid Build Coastguard Worker av1_zero(filter_intra_mode_info);
248*77c1e3ccSAndroid Build Coastguard Worker mbmi->filter_intra_mode_info.use_filter_intra = 1;
249*77c1e3ccSAndroid Build Coastguard Worker mbmi->mode = DC_PRED;
250*77c1e3ccSAndroid Build Coastguard Worker mbmi->palette_mode_info.palette_size[0] = 0;
251*77c1e3ccSAndroid Build Coastguard Worker
252*77c1e3ccSAndroid Build Coastguard Worker // Skip the evaluation of filter-intra if cached MB_MODE_INFO does not have
253*77c1e3ccSAndroid Build Coastguard Worker // filter-intra as winner.
254*77c1e3ccSAndroid Build Coastguard Worker if (x->use_mb_mode_cache &&
255*77c1e3ccSAndroid Build Coastguard Worker !x->mb_mode_cache->filter_intra_mode_info.use_filter_intra)
256*77c1e3ccSAndroid Build Coastguard Worker return 0;
257*77c1e3ccSAndroid Build Coastguard Worker
258*77c1e3ccSAndroid Build Coastguard Worker for (mode = 0; mode < FILTER_INTRA_MODES; ++mode) {
259*77c1e3ccSAndroid Build Coastguard Worker int64_t this_rd;
260*77c1e3ccSAndroid Build Coastguard Worker RD_STATS tokenonly_rd_stats;
261*77c1e3ccSAndroid Build Coastguard Worker mbmi->filter_intra_mode_info.filter_intra_mode = mode;
262*77c1e3ccSAndroid Build Coastguard Worker
263*77c1e3ccSAndroid Build Coastguard Worker if ((cpi->sf.intra_sf.prune_filter_intra_level == 1) &&
264*77c1e3ccSAndroid Build Coastguard Worker !(av1_derived_filter_intra_mode_used_flag[best_mode_so_far] &
265*77c1e3ccSAndroid Build Coastguard Worker (1 << mode)))
266*77c1e3ccSAndroid Build Coastguard Worker continue;
267*77c1e3ccSAndroid Build Coastguard Worker
268*77c1e3ccSAndroid Build Coastguard Worker // Skip the evaluation of modes that do not match with the winner mode in
269*77c1e3ccSAndroid Build Coastguard Worker // x->mb_mode_cache.
270*77c1e3ccSAndroid Build Coastguard Worker if (x->use_mb_mode_cache &&
271*77c1e3ccSAndroid Build Coastguard Worker mode != x->mb_mode_cache->filter_intra_mode_info.filter_intra_mode)
272*77c1e3ccSAndroid Build Coastguard Worker continue;
273*77c1e3ccSAndroid Build Coastguard Worker
274*77c1e3ccSAndroid Build Coastguard Worker if (model_intra_yrd_and_prune(cpi, x, bsize, best_model_rd)) {
275*77c1e3ccSAndroid Build Coastguard Worker continue;
276*77c1e3ccSAndroid Build Coastguard Worker }
277*77c1e3ccSAndroid Build Coastguard Worker av1_pick_uniform_tx_size_type_yrd(cpi, x, &tokenonly_rd_stats, bsize,
278*77c1e3ccSAndroid Build Coastguard Worker *best_rd);
279*77c1e3ccSAndroid Build Coastguard Worker if (tokenonly_rd_stats.rate == INT_MAX) continue;
280*77c1e3ccSAndroid Build Coastguard Worker const int this_rate =
281*77c1e3ccSAndroid Build Coastguard Worker tokenonly_rd_stats.rate +
282*77c1e3ccSAndroid Build Coastguard Worker intra_mode_info_cost_y(cpi, x, mbmi, bsize, mode_cost, 0);
283*77c1e3ccSAndroid Build Coastguard Worker this_rd = RDCOST(x->rdmult, this_rate, tokenonly_rd_stats.dist);
284*77c1e3ccSAndroid Build Coastguard Worker
285*77c1e3ccSAndroid Build Coastguard Worker // Visual quality adjustment based on recon vs source variance.
286*77c1e3ccSAndroid Build Coastguard Worker if ((cpi->oxcf.mode == ALLINTRA) && (this_rd != INT64_MAX)) {
287*77c1e3ccSAndroid Build Coastguard Worker this_rd = (int64_t)(this_rd * intra_rd_variance_factor(cpi, x, bsize));
288*77c1e3ccSAndroid Build Coastguard Worker }
289*77c1e3ccSAndroid Build Coastguard Worker
290*77c1e3ccSAndroid Build Coastguard Worker // Collect mode stats for multiwinner mode processing
291*77c1e3ccSAndroid Build Coastguard Worker const int txfm_search_done = 1;
292*77c1e3ccSAndroid Build Coastguard Worker store_winner_mode_stats(
293*77c1e3ccSAndroid Build Coastguard Worker &cpi->common, x, mbmi, NULL, NULL, NULL, 0, NULL, bsize, this_rd,
294*77c1e3ccSAndroid Build Coastguard Worker cpi->sf.winner_mode_sf.multi_winner_mode_type, txfm_search_done);
295*77c1e3ccSAndroid Build Coastguard Worker if (this_rd < *best_rd) {
296*77c1e3ccSAndroid Build Coastguard Worker *best_rd = this_rd;
297*77c1e3ccSAndroid Build Coastguard Worker best_tx_size = mbmi->tx_size;
298*77c1e3ccSAndroid Build Coastguard Worker filter_intra_mode_info = mbmi->filter_intra_mode_info;
299*77c1e3ccSAndroid Build Coastguard Worker av1_copy_array(best_tx_type_map, xd->tx_type_map, ctx->num_4x4_blk);
300*77c1e3ccSAndroid Build Coastguard Worker memcpy(ctx->blk_skip, x->txfm_search_info.blk_skip,
301*77c1e3ccSAndroid Build Coastguard Worker sizeof(x->txfm_search_info.blk_skip[0]) * ctx->num_4x4_blk);
302*77c1e3ccSAndroid Build Coastguard Worker *rate = this_rate;
303*77c1e3ccSAndroid Build Coastguard Worker *rate_tokenonly = tokenonly_rd_stats.rate;
304*77c1e3ccSAndroid Build Coastguard Worker *distortion = tokenonly_rd_stats.dist;
305*77c1e3ccSAndroid Build Coastguard Worker *skippable = tokenonly_rd_stats.skip_txfm;
306*77c1e3ccSAndroid Build Coastguard Worker filter_intra_selected_flag = 1;
307*77c1e3ccSAndroid Build Coastguard Worker }
308*77c1e3ccSAndroid Build Coastguard Worker }
309*77c1e3ccSAndroid Build Coastguard Worker
310*77c1e3ccSAndroid Build Coastguard Worker if (filter_intra_selected_flag) {
311*77c1e3ccSAndroid Build Coastguard Worker mbmi->mode = DC_PRED;
312*77c1e3ccSAndroid Build Coastguard Worker mbmi->tx_size = best_tx_size;
313*77c1e3ccSAndroid Build Coastguard Worker mbmi->filter_intra_mode_info = filter_intra_mode_info;
314*77c1e3ccSAndroid Build Coastguard Worker av1_copy_array(ctx->tx_type_map, best_tx_type_map, ctx->num_4x4_blk);
315*77c1e3ccSAndroid Build Coastguard Worker return 1;
316*77c1e3ccSAndroid Build Coastguard Worker } else {
317*77c1e3ccSAndroid Build Coastguard Worker return 0;
318*77c1e3ccSAndroid Build Coastguard Worker }
319*77c1e3ccSAndroid Build Coastguard Worker }
320*77c1e3ccSAndroid Build Coastguard Worker
av1_count_colors(const uint8_t * src,int stride,int rows,int cols,int * val_count,int * num_colors)321*77c1e3ccSAndroid Build Coastguard Worker void av1_count_colors(const uint8_t *src, int stride, int rows, int cols,
322*77c1e3ccSAndroid Build Coastguard Worker int *val_count, int *num_colors) {
323*77c1e3ccSAndroid Build Coastguard Worker const int max_pix_val = 1 << 8;
324*77c1e3ccSAndroid Build Coastguard Worker memset(val_count, 0, max_pix_val * sizeof(val_count[0]));
325*77c1e3ccSAndroid Build Coastguard Worker for (int r = 0; r < rows; ++r) {
326*77c1e3ccSAndroid Build Coastguard Worker for (int c = 0; c < cols; ++c) {
327*77c1e3ccSAndroid Build Coastguard Worker const int this_val = src[r * stride + c];
328*77c1e3ccSAndroid Build Coastguard Worker assert(this_val < max_pix_val);
329*77c1e3ccSAndroid Build Coastguard Worker ++val_count[this_val];
330*77c1e3ccSAndroid Build Coastguard Worker }
331*77c1e3ccSAndroid Build Coastguard Worker }
332*77c1e3ccSAndroid Build Coastguard Worker int n = 0;
333*77c1e3ccSAndroid Build Coastguard Worker for (int i = 0; i < max_pix_val; ++i) {
334*77c1e3ccSAndroid Build Coastguard Worker if (val_count[i]) ++n;
335*77c1e3ccSAndroid Build Coastguard Worker }
336*77c1e3ccSAndroid Build Coastguard Worker *num_colors = n;
337*77c1e3ccSAndroid Build Coastguard Worker }
338*77c1e3ccSAndroid Build Coastguard Worker
av1_count_colors_highbd(const uint8_t * src8,int stride,int rows,int cols,int bit_depth,int * val_count,int * bin_val_count,int * num_color_bins,int * num_colors)339*77c1e3ccSAndroid Build Coastguard Worker void av1_count_colors_highbd(const uint8_t *src8, int stride, int rows,
340*77c1e3ccSAndroid Build Coastguard Worker int cols, int bit_depth, int *val_count,
341*77c1e3ccSAndroid Build Coastguard Worker int *bin_val_count, int *num_color_bins,
342*77c1e3ccSAndroid Build Coastguard Worker int *num_colors) {
343*77c1e3ccSAndroid Build Coastguard Worker assert(bit_depth <= 12);
344*77c1e3ccSAndroid Build Coastguard Worker const int max_bin_val = 1 << 8;
345*77c1e3ccSAndroid Build Coastguard Worker const int max_pix_val = 1 << bit_depth;
346*77c1e3ccSAndroid Build Coastguard Worker const uint16_t *src = CONVERT_TO_SHORTPTR(src8);
347*77c1e3ccSAndroid Build Coastguard Worker memset(bin_val_count, 0, max_bin_val * sizeof(val_count[0]));
348*77c1e3ccSAndroid Build Coastguard Worker if (val_count != NULL)
349*77c1e3ccSAndroid Build Coastguard Worker memset(val_count, 0, max_pix_val * sizeof(val_count[0]));
350*77c1e3ccSAndroid Build Coastguard Worker for (int r = 0; r < rows; ++r) {
351*77c1e3ccSAndroid Build Coastguard Worker for (int c = 0; c < cols; ++c) {
352*77c1e3ccSAndroid Build Coastguard Worker /*
353*77c1e3ccSAndroid Build Coastguard Worker * Down-convert the pixels to 8-bit domain before counting.
354*77c1e3ccSAndroid Build Coastguard Worker * This provides consistency of behavior for palette search
355*77c1e3ccSAndroid Build Coastguard Worker * between lbd and hbd encodes. This down-converted pixels
356*77c1e3ccSAndroid Build Coastguard Worker * are only used for calculating the threshold (n).
357*77c1e3ccSAndroid Build Coastguard Worker */
358*77c1e3ccSAndroid Build Coastguard Worker const int this_val = ((src[r * stride + c]) >> (bit_depth - 8));
359*77c1e3ccSAndroid Build Coastguard Worker assert(this_val < max_bin_val);
360*77c1e3ccSAndroid Build Coastguard Worker if (this_val >= max_bin_val) continue;
361*77c1e3ccSAndroid Build Coastguard Worker ++bin_val_count[this_val];
362*77c1e3ccSAndroid Build Coastguard Worker if (val_count != NULL) ++val_count[(src[r * stride + c])];
363*77c1e3ccSAndroid Build Coastguard Worker }
364*77c1e3ccSAndroid Build Coastguard Worker }
365*77c1e3ccSAndroid Build Coastguard Worker int n = 0;
366*77c1e3ccSAndroid Build Coastguard Worker // Count the colors based on 8-bit domain used to gate the palette path
367*77c1e3ccSAndroid Build Coastguard Worker for (int i = 0; i < max_bin_val; ++i) {
368*77c1e3ccSAndroid Build Coastguard Worker if (bin_val_count[i]) ++n;
369*77c1e3ccSAndroid Build Coastguard Worker }
370*77c1e3ccSAndroid Build Coastguard Worker *num_color_bins = n;
371*77c1e3ccSAndroid Build Coastguard Worker
372*77c1e3ccSAndroid Build Coastguard Worker // Count the actual hbd colors used to create top_colors
373*77c1e3ccSAndroid Build Coastguard Worker n = 0;
374*77c1e3ccSAndroid Build Coastguard Worker if (val_count != NULL) {
375*77c1e3ccSAndroid Build Coastguard Worker for (int i = 0; i < max_pix_val; ++i) {
376*77c1e3ccSAndroid Build Coastguard Worker if (val_count[i]) ++n;
377*77c1e3ccSAndroid Build Coastguard Worker }
378*77c1e3ccSAndroid Build Coastguard Worker *num_colors = n;
379*77c1e3ccSAndroid Build Coastguard Worker }
380*77c1e3ccSAndroid Build Coastguard Worker }
381*77c1e3ccSAndroid Build Coastguard Worker
set_y_mode_and_delta_angle(const int mode_idx,MB_MODE_INFO * const mbmi,int reorder_delta_angle_eval)382*77c1e3ccSAndroid Build Coastguard Worker void set_y_mode_and_delta_angle(const int mode_idx, MB_MODE_INFO *const mbmi,
383*77c1e3ccSAndroid Build Coastguard Worker int reorder_delta_angle_eval) {
384*77c1e3ccSAndroid Build Coastguard Worker if (mode_idx < INTRA_MODE_END) {
385*77c1e3ccSAndroid Build Coastguard Worker mbmi->mode = intra_rd_search_mode_order[mode_idx];
386*77c1e3ccSAndroid Build Coastguard Worker mbmi->angle_delta[PLANE_TYPE_Y] = 0;
387*77c1e3ccSAndroid Build Coastguard Worker } else {
388*77c1e3ccSAndroid Build Coastguard Worker mbmi->mode = (mode_idx - INTRA_MODE_END) / (MAX_ANGLE_DELTA * 2) + V_PRED;
389*77c1e3ccSAndroid Build Coastguard Worker int delta_angle_eval_idx =
390*77c1e3ccSAndroid Build Coastguard Worker (mode_idx - INTRA_MODE_END) % (MAX_ANGLE_DELTA * 2);
391*77c1e3ccSAndroid Build Coastguard Worker if (reorder_delta_angle_eval) {
392*77c1e3ccSAndroid Build Coastguard Worker mbmi->angle_delta[PLANE_TYPE_Y] =
393*77c1e3ccSAndroid Build Coastguard Worker luma_delta_angles_order[delta_angle_eval_idx];
394*77c1e3ccSAndroid Build Coastguard Worker } else {
395*77c1e3ccSAndroid Build Coastguard Worker mbmi->angle_delta[PLANE_TYPE_Y] =
396*77c1e3ccSAndroid Build Coastguard Worker (delta_angle_eval_idx < 3 ? (delta_angle_eval_idx - 3)
397*77c1e3ccSAndroid Build Coastguard Worker : (delta_angle_eval_idx - 2));
398*77c1e3ccSAndroid Build Coastguard Worker }
399*77c1e3ccSAndroid Build Coastguard Worker }
400*77c1e3ccSAndroid Build Coastguard Worker }
401*77c1e3ccSAndroid Build Coastguard Worker
get_model_rd_index_for_pruning(const MACROBLOCK * const x,const INTRA_MODE_SPEED_FEATURES * const intra_sf)402*77c1e3ccSAndroid Build Coastguard Worker static inline int get_model_rd_index_for_pruning(
403*77c1e3ccSAndroid Build Coastguard Worker const MACROBLOCK *const x,
404*77c1e3ccSAndroid Build Coastguard Worker const INTRA_MODE_SPEED_FEATURES *const intra_sf) {
405*77c1e3ccSAndroid Build Coastguard Worker const int top_intra_model_count_allowed =
406*77c1e3ccSAndroid Build Coastguard Worker intra_sf->top_intra_model_count_allowed;
407*77c1e3ccSAndroid Build Coastguard Worker if (!intra_sf->adapt_top_model_rd_count_using_neighbors)
408*77c1e3ccSAndroid Build Coastguard Worker return top_intra_model_count_allowed - 1;
409*77c1e3ccSAndroid Build Coastguard Worker
410*77c1e3ccSAndroid Build Coastguard Worker const MACROBLOCKD *const xd = &x->e_mbd;
411*77c1e3ccSAndroid Build Coastguard Worker const PREDICTION_MODE mode = xd->mi[0]->mode;
412*77c1e3ccSAndroid Build Coastguard Worker int model_rd_index_for_pruning = top_intra_model_count_allowed - 1;
413*77c1e3ccSAndroid Build Coastguard Worker int is_left_mode_neq_cur_mode = 0, is_above_mode_neq_cur_mode = 0;
414*77c1e3ccSAndroid Build Coastguard Worker if (xd->left_available)
415*77c1e3ccSAndroid Build Coastguard Worker is_left_mode_neq_cur_mode = xd->left_mbmi->mode != mode;
416*77c1e3ccSAndroid Build Coastguard Worker if (xd->up_available)
417*77c1e3ccSAndroid Build Coastguard Worker is_above_mode_neq_cur_mode = xd->above_mbmi->mode != mode;
418*77c1e3ccSAndroid Build Coastguard Worker // The pruning of luma intra modes is made more aggressive at lower quantizers
419*77c1e3ccSAndroid Build Coastguard Worker // and vice versa. The value for model_rd_index_for_pruning is derived as
420*77c1e3ccSAndroid Build Coastguard Worker // follows.
421*77c1e3ccSAndroid Build Coastguard Worker // qidx 0 to 127: Reduce the index of a candidate used for comparison only if
422*77c1e3ccSAndroid Build Coastguard Worker // the current mode does not match either of the available neighboring modes.
423*77c1e3ccSAndroid Build Coastguard Worker // qidx 128 to 255: Reduce the index of a candidate used for comparison only
424*77c1e3ccSAndroid Build Coastguard Worker // if the current mode does not match both the available neighboring modes.
425*77c1e3ccSAndroid Build Coastguard Worker if (x->qindex <= 127) {
426*77c1e3ccSAndroid Build Coastguard Worker if (is_left_mode_neq_cur_mode || is_above_mode_neq_cur_mode)
427*77c1e3ccSAndroid Build Coastguard Worker model_rd_index_for_pruning = AOMMAX(model_rd_index_for_pruning - 1, 0);
428*77c1e3ccSAndroid Build Coastguard Worker } else {
429*77c1e3ccSAndroid Build Coastguard Worker if (is_left_mode_neq_cur_mode && is_above_mode_neq_cur_mode)
430*77c1e3ccSAndroid Build Coastguard Worker model_rd_index_for_pruning = AOMMAX(model_rd_index_for_pruning - 1, 0);
431*77c1e3ccSAndroid Build Coastguard Worker }
432*77c1e3ccSAndroid Build Coastguard Worker return model_rd_index_for_pruning;
433*77c1e3ccSAndroid Build Coastguard Worker }
434*77c1e3ccSAndroid Build Coastguard Worker
435*77c1e3ccSAndroid Build Coastguard Worker /*! \brief prune luma intra mode based on the model rd.
436*77c1e3ccSAndroid Build Coastguard Worker * \param[in] this_model_rd model rd for current mode.
437*77c1e3ccSAndroid Build Coastguard Worker * \param[in] best_model_rd Best model RD seen for this block so
438*77c1e3ccSAndroid Build Coastguard Worker * far.
439*77c1e3ccSAndroid Build Coastguard Worker * \param[in] top_intra_model_rd Top intra model RD seen for this
440*77c1e3ccSAndroid Build Coastguard Worker * block so far.
441*77c1e3ccSAndroid Build Coastguard Worker * \param[in] max_model_cnt_allowed The maximum number of top intra
442*77c1e3ccSAndroid Build Coastguard Worker * model RD allowed.
443*77c1e3ccSAndroid Build Coastguard Worker * \param[in] model_rd_index_for_pruning Index of the candidate used for
444*77c1e3ccSAndroid Build Coastguard Worker * pruning based on model rd.
445*77c1e3ccSAndroid Build Coastguard Worker */
prune_intra_y_mode(int64_t this_model_rd,int64_t * best_model_rd,int64_t top_intra_model_rd[],int max_model_cnt_allowed,int model_rd_index_for_pruning)446*77c1e3ccSAndroid Build Coastguard Worker static int prune_intra_y_mode(int64_t this_model_rd, int64_t *best_model_rd,
447*77c1e3ccSAndroid Build Coastguard Worker int64_t top_intra_model_rd[],
448*77c1e3ccSAndroid Build Coastguard Worker int max_model_cnt_allowed,
449*77c1e3ccSAndroid Build Coastguard Worker int model_rd_index_for_pruning) {
450*77c1e3ccSAndroid Build Coastguard Worker const double thresh_best = 1.50;
451*77c1e3ccSAndroid Build Coastguard Worker const double thresh_top = 1.00;
452*77c1e3ccSAndroid Build Coastguard Worker for (int i = 0; i < max_model_cnt_allowed; i++) {
453*77c1e3ccSAndroid Build Coastguard Worker if (this_model_rd < top_intra_model_rd[i]) {
454*77c1e3ccSAndroid Build Coastguard Worker for (int j = max_model_cnt_allowed - 1; j > i; j--) {
455*77c1e3ccSAndroid Build Coastguard Worker top_intra_model_rd[j] = top_intra_model_rd[j - 1];
456*77c1e3ccSAndroid Build Coastguard Worker }
457*77c1e3ccSAndroid Build Coastguard Worker top_intra_model_rd[i] = this_model_rd;
458*77c1e3ccSAndroid Build Coastguard Worker break;
459*77c1e3ccSAndroid Build Coastguard Worker }
460*77c1e3ccSAndroid Build Coastguard Worker }
461*77c1e3ccSAndroid Build Coastguard Worker if (top_intra_model_rd[model_rd_index_for_pruning] != INT64_MAX &&
462*77c1e3ccSAndroid Build Coastguard Worker this_model_rd >
463*77c1e3ccSAndroid Build Coastguard Worker thresh_top * top_intra_model_rd[model_rd_index_for_pruning])
464*77c1e3ccSAndroid Build Coastguard Worker return 1;
465*77c1e3ccSAndroid Build Coastguard Worker
466*77c1e3ccSAndroid Build Coastguard Worker if (this_model_rd != INT64_MAX &&
467*77c1e3ccSAndroid Build Coastguard Worker this_model_rd > thresh_best * (*best_model_rd))
468*77c1e3ccSAndroid Build Coastguard Worker return 1;
469*77c1e3ccSAndroid Build Coastguard Worker if (this_model_rd < *best_model_rd) *best_model_rd = this_model_rd;
470*77c1e3ccSAndroid Build Coastguard Worker return 0;
471*77c1e3ccSAndroid Build Coastguard Worker }
472*77c1e3ccSAndroid Build Coastguard Worker
473*77c1e3ccSAndroid Build Coastguard Worker // Run RD calculation with given chroma intra prediction angle., and return
474*77c1e3ccSAndroid Build Coastguard Worker // the RD cost. Update the best mode info. if the RD cost is the best so far.
pick_intra_angle_routine_sbuv(const AV1_COMP * const cpi,MACROBLOCK * x,BLOCK_SIZE bsize,int rate_overhead,int64_t best_rd_in,int * rate,RD_STATS * rd_stats,int * best_angle_delta,int64_t * best_rd)475*77c1e3ccSAndroid Build Coastguard Worker static int64_t pick_intra_angle_routine_sbuv(
476*77c1e3ccSAndroid Build Coastguard Worker const AV1_COMP *const cpi, MACROBLOCK *x, BLOCK_SIZE bsize,
477*77c1e3ccSAndroid Build Coastguard Worker int rate_overhead, int64_t best_rd_in, int *rate, RD_STATS *rd_stats,
478*77c1e3ccSAndroid Build Coastguard Worker int *best_angle_delta, int64_t *best_rd) {
479*77c1e3ccSAndroid Build Coastguard Worker MB_MODE_INFO *mbmi = x->e_mbd.mi[0];
480*77c1e3ccSAndroid Build Coastguard Worker assert(!is_inter_block(mbmi));
481*77c1e3ccSAndroid Build Coastguard Worker int this_rate;
482*77c1e3ccSAndroid Build Coastguard Worker int64_t this_rd;
483*77c1e3ccSAndroid Build Coastguard Worker RD_STATS tokenonly_rd_stats;
484*77c1e3ccSAndroid Build Coastguard Worker
485*77c1e3ccSAndroid Build Coastguard Worker if (!av1_txfm_uvrd(cpi, x, &tokenonly_rd_stats, bsize, best_rd_in))
486*77c1e3ccSAndroid Build Coastguard Worker return INT64_MAX;
487*77c1e3ccSAndroid Build Coastguard Worker this_rate = tokenonly_rd_stats.rate +
488*77c1e3ccSAndroid Build Coastguard Worker intra_mode_info_cost_uv(cpi, x, mbmi, bsize, rate_overhead);
489*77c1e3ccSAndroid Build Coastguard Worker this_rd = RDCOST(x->rdmult, this_rate, tokenonly_rd_stats.dist);
490*77c1e3ccSAndroid Build Coastguard Worker if (this_rd < *best_rd) {
491*77c1e3ccSAndroid Build Coastguard Worker *best_rd = this_rd;
492*77c1e3ccSAndroid Build Coastguard Worker *best_angle_delta = mbmi->angle_delta[PLANE_TYPE_UV];
493*77c1e3ccSAndroid Build Coastguard Worker *rate = this_rate;
494*77c1e3ccSAndroid Build Coastguard Worker rd_stats->rate = tokenonly_rd_stats.rate;
495*77c1e3ccSAndroid Build Coastguard Worker rd_stats->dist = tokenonly_rd_stats.dist;
496*77c1e3ccSAndroid Build Coastguard Worker rd_stats->skip_txfm = tokenonly_rd_stats.skip_txfm;
497*77c1e3ccSAndroid Build Coastguard Worker }
498*77c1e3ccSAndroid Build Coastguard Worker return this_rd;
499*77c1e3ccSAndroid Build Coastguard Worker }
500*77c1e3ccSAndroid Build Coastguard Worker
501*77c1e3ccSAndroid Build Coastguard Worker /*!\brief Search for the best angle delta for chroma prediction
502*77c1e3ccSAndroid Build Coastguard Worker *
503*77c1e3ccSAndroid Build Coastguard Worker * \ingroup intra_mode_search
504*77c1e3ccSAndroid Build Coastguard Worker * \callergraph
505*77c1e3ccSAndroid Build Coastguard Worker * Given a chroma directional intra prediction mode, this function will try to
506*77c1e3ccSAndroid Build Coastguard Worker * estimate the best delta_angle.
507*77c1e3ccSAndroid Build Coastguard Worker *
508*77c1e3ccSAndroid Build Coastguard Worker * \returns Return if there is a new mode with smaller rdcost than best_rd.
509*77c1e3ccSAndroid Build Coastguard Worker */
rd_pick_intra_angle_sbuv(const AV1_COMP * const cpi,MACROBLOCK * x,BLOCK_SIZE bsize,int rate_overhead,int64_t best_rd,int * rate,RD_STATS * rd_stats)510*77c1e3ccSAndroid Build Coastguard Worker static int rd_pick_intra_angle_sbuv(const AV1_COMP *const cpi, MACROBLOCK *x,
511*77c1e3ccSAndroid Build Coastguard Worker BLOCK_SIZE bsize, int rate_overhead,
512*77c1e3ccSAndroid Build Coastguard Worker int64_t best_rd, int *rate,
513*77c1e3ccSAndroid Build Coastguard Worker RD_STATS *rd_stats) {
514*77c1e3ccSAndroid Build Coastguard Worker MACROBLOCKD *const xd = &x->e_mbd;
515*77c1e3ccSAndroid Build Coastguard Worker MB_MODE_INFO *mbmi = xd->mi[0];
516*77c1e3ccSAndroid Build Coastguard Worker assert(!is_inter_block(mbmi));
517*77c1e3ccSAndroid Build Coastguard Worker int i, angle_delta, best_angle_delta = 0;
518*77c1e3ccSAndroid Build Coastguard Worker int64_t this_rd, best_rd_in, rd_cost[2 * (MAX_ANGLE_DELTA + 2)];
519*77c1e3ccSAndroid Build Coastguard Worker
520*77c1e3ccSAndroid Build Coastguard Worker rd_stats->rate = INT_MAX;
521*77c1e3ccSAndroid Build Coastguard Worker rd_stats->skip_txfm = 0;
522*77c1e3ccSAndroid Build Coastguard Worker rd_stats->dist = INT64_MAX;
523*77c1e3ccSAndroid Build Coastguard Worker for (i = 0; i < 2 * (MAX_ANGLE_DELTA + 2); ++i) rd_cost[i] = INT64_MAX;
524*77c1e3ccSAndroid Build Coastguard Worker
525*77c1e3ccSAndroid Build Coastguard Worker for (angle_delta = 0; angle_delta <= MAX_ANGLE_DELTA; angle_delta += 2) {
526*77c1e3ccSAndroid Build Coastguard Worker for (i = 0; i < 2; ++i) {
527*77c1e3ccSAndroid Build Coastguard Worker best_rd_in = (best_rd == INT64_MAX)
528*77c1e3ccSAndroid Build Coastguard Worker ? INT64_MAX
529*77c1e3ccSAndroid Build Coastguard Worker : (best_rd + (best_rd >> ((angle_delta == 0) ? 3 : 5)));
530*77c1e3ccSAndroid Build Coastguard Worker mbmi->angle_delta[PLANE_TYPE_UV] = (1 - 2 * i) * angle_delta;
531*77c1e3ccSAndroid Build Coastguard Worker this_rd = pick_intra_angle_routine_sbuv(cpi, x, bsize, rate_overhead,
532*77c1e3ccSAndroid Build Coastguard Worker best_rd_in, rate, rd_stats,
533*77c1e3ccSAndroid Build Coastguard Worker &best_angle_delta, &best_rd);
534*77c1e3ccSAndroid Build Coastguard Worker rd_cost[2 * angle_delta + i] = this_rd;
535*77c1e3ccSAndroid Build Coastguard Worker if (angle_delta == 0) {
536*77c1e3ccSAndroid Build Coastguard Worker if (this_rd == INT64_MAX) return 0;
537*77c1e3ccSAndroid Build Coastguard Worker rd_cost[1] = this_rd;
538*77c1e3ccSAndroid Build Coastguard Worker break;
539*77c1e3ccSAndroid Build Coastguard Worker }
540*77c1e3ccSAndroid Build Coastguard Worker }
541*77c1e3ccSAndroid Build Coastguard Worker }
542*77c1e3ccSAndroid Build Coastguard Worker
543*77c1e3ccSAndroid Build Coastguard Worker assert(best_rd != INT64_MAX);
544*77c1e3ccSAndroid Build Coastguard Worker for (angle_delta = 1; angle_delta <= MAX_ANGLE_DELTA; angle_delta += 2) {
545*77c1e3ccSAndroid Build Coastguard Worker int64_t rd_thresh;
546*77c1e3ccSAndroid Build Coastguard Worker for (i = 0; i < 2; ++i) {
547*77c1e3ccSAndroid Build Coastguard Worker int skip_search = 0;
548*77c1e3ccSAndroid Build Coastguard Worker rd_thresh = best_rd + (best_rd >> 5);
549*77c1e3ccSAndroid Build Coastguard Worker if (rd_cost[2 * (angle_delta + 1) + i] > rd_thresh &&
550*77c1e3ccSAndroid Build Coastguard Worker rd_cost[2 * (angle_delta - 1) + i] > rd_thresh)
551*77c1e3ccSAndroid Build Coastguard Worker skip_search = 1;
552*77c1e3ccSAndroid Build Coastguard Worker if (!skip_search) {
553*77c1e3ccSAndroid Build Coastguard Worker mbmi->angle_delta[PLANE_TYPE_UV] = (1 - 2 * i) * angle_delta;
554*77c1e3ccSAndroid Build Coastguard Worker pick_intra_angle_routine_sbuv(cpi, x, bsize, rate_overhead, best_rd,
555*77c1e3ccSAndroid Build Coastguard Worker rate, rd_stats, &best_angle_delta,
556*77c1e3ccSAndroid Build Coastguard Worker &best_rd);
557*77c1e3ccSAndroid Build Coastguard Worker }
558*77c1e3ccSAndroid Build Coastguard Worker }
559*77c1e3ccSAndroid Build Coastguard Worker }
560*77c1e3ccSAndroid Build Coastguard Worker
561*77c1e3ccSAndroid Build Coastguard Worker mbmi->angle_delta[PLANE_TYPE_UV] = best_angle_delta;
562*77c1e3ccSAndroid Build Coastguard Worker return rd_stats->rate != INT_MAX;
563*77c1e3ccSAndroid Build Coastguard Worker }
564*77c1e3ccSAndroid Build Coastguard Worker
565*77c1e3ccSAndroid Build Coastguard Worker #define PLANE_SIGN_TO_JOINT_SIGN(plane, a, b) \
566*77c1e3ccSAndroid Build Coastguard Worker (plane == CFL_PRED_U ? a * CFL_SIGNS + b - 1 : b * CFL_SIGNS + a - 1)
567*77c1e3ccSAndroid Build Coastguard Worker
cfl_idx_to_sign_and_alpha(int cfl_idx,CFL_SIGN_TYPE * cfl_sign,int * cfl_alpha)568*77c1e3ccSAndroid Build Coastguard Worker static void cfl_idx_to_sign_and_alpha(int cfl_idx, CFL_SIGN_TYPE *cfl_sign,
569*77c1e3ccSAndroid Build Coastguard Worker int *cfl_alpha) {
570*77c1e3ccSAndroid Build Coastguard Worker int cfl_linear_idx = cfl_idx - CFL_INDEX_ZERO;
571*77c1e3ccSAndroid Build Coastguard Worker if (cfl_linear_idx == 0) {
572*77c1e3ccSAndroid Build Coastguard Worker *cfl_sign = CFL_SIGN_ZERO;
573*77c1e3ccSAndroid Build Coastguard Worker *cfl_alpha = 0;
574*77c1e3ccSAndroid Build Coastguard Worker } else {
575*77c1e3ccSAndroid Build Coastguard Worker *cfl_sign = cfl_linear_idx > 0 ? CFL_SIGN_POS : CFL_SIGN_NEG;
576*77c1e3ccSAndroid Build Coastguard Worker *cfl_alpha = abs(cfl_linear_idx) - 1;
577*77c1e3ccSAndroid Build Coastguard Worker }
578*77c1e3ccSAndroid Build Coastguard Worker }
579*77c1e3ccSAndroid Build Coastguard Worker
cfl_compute_rd(const AV1_COMP * const cpi,MACROBLOCK * x,int plane,TX_SIZE tx_size,BLOCK_SIZE plane_bsize,int cfl_idx,int fast_mode,RD_STATS * rd_stats)580*77c1e3ccSAndroid Build Coastguard Worker static int64_t cfl_compute_rd(const AV1_COMP *const cpi, MACROBLOCK *x,
581*77c1e3ccSAndroid Build Coastguard Worker int plane, TX_SIZE tx_size,
582*77c1e3ccSAndroid Build Coastguard Worker BLOCK_SIZE plane_bsize, int cfl_idx,
583*77c1e3ccSAndroid Build Coastguard Worker int fast_mode, RD_STATS *rd_stats) {
584*77c1e3ccSAndroid Build Coastguard Worker assert(IMPLIES(fast_mode, rd_stats == NULL));
585*77c1e3ccSAndroid Build Coastguard Worker const AV1_COMMON *const cm = &cpi->common;
586*77c1e3ccSAndroid Build Coastguard Worker MACROBLOCKD *const xd = &x->e_mbd;
587*77c1e3ccSAndroid Build Coastguard Worker MB_MODE_INFO *const mbmi = xd->mi[0];
588*77c1e3ccSAndroid Build Coastguard Worker int cfl_plane = get_cfl_pred_type(plane);
589*77c1e3ccSAndroid Build Coastguard Worker CFL_SIGN_TYPE cfl_sign;
590*77c1e3ccSAndroid Build Coastguard Worker int cfl_alpha;
591*77c1e3ccSAndroid Build Coastguard Worker cfl_idx_to_sign_and_alpha(cfl_idx, &cfl_sign, &cfl_alpha);
592*77c1e3ccSAndroid Build Coastguard Worker // We conly build CFL for a given plane, the other plane's sign is dummy
593*77c1e3ccSAndroid Build Coastguard Worker int dummy_sign = CFL_SIGN_NEG;
594*77c1e3ccSAndroid Build Coastguard Worker const int8_t orig_cfl_alpha_signs = mbmi->cfl_alpha_signs;
595*77c1e3ccSAndroid Build Coastguard Worker const uint8_t orig_cfl_alpha_idx = mbmi->cfl_alpha_idx;
596*77c1e3ccSAndroid Build Coastguard Worker mbmi->cfl_alpha_signs =
597*77c1e3ccSAndroid Build Coastguard Worker PLANE_SIGN_TO_JOINT_SIGN(cfl_plane, cfl_sign, dummy_sign);
598*77c1e3ccSAndroid Build Coastguard Worker mbmi->cfl_alpha_idx = (cfl_alpha << CFL_ALPHABET_SIZE_LOG2) + cfl_alpha;
599*77c1e3ccSAndroid Build Coastguard Worker int64_t cfl_cost;
600*77c1e3ccSAndroid Build Coastguard Worker if (fast_mode) {
601*77c1e3ccSAndroid Build Coastguard Worker cfl_cost =
602*77c1e3ccSAndroid Build Coastguard Worker intra_model_rd(cm, x, plane, plane_bsize, tx_size, /*use_hadamard=*/0);
603*77c1e3ccSAndroid Build Coastguard Worker } else {
604*77c1e3ccSAndroid Build Coastguard Worker av1_init_rd_stats(rd_stats);
605*77c1e3ccSAndroid Build Coastguard Worker av1_txfm_rd_in_plane(x, cpi, rd_stats, INT64_MAX, 0, plane, plane_bsize,
606*77c1e3ccSAndroid Build Coastguard Worker tx_size, FTXS_NONE, 0);
607*77c1e3ccSAndroid Build Coastguard Worker av1_rd_cost_update(x->rdmult, rd_stats);
608*77c1e3ccSAndroid Build Coastguard Worker cfl_cost = rd_stats->rdcost;
609*77c1e3ccSAndroid Build Coastguard Worker }
610*77c1e3ccSAndroid Build Coastguard Worker mbmi->cfl_alpha_signs = orig_cfl_alpha_signs;
611*77c1e3ccSAndroid Build Coastguard Worker mbmi->cfl_alpha_idx = orig_cfl_alpha_idx;
612*77c1e3ccSAndroid Build Coastguard Worker return cfl_cost;
613*77c1e3ccSAndroid Build Coastguard Worker }
614*77c1e3ccSAndroid Build Coastguard Worker
615*77c1e3ccSAndroid Build Coastguard Worker static const int cfl_dir_ls[2] = { 1, -1 };
616*77c1e3ccSAndroid Build Coastguard Worker
617*77c1e3ccSAndroid Build Coastguard Worker // If cfl_search_range is CFL_MAGS_SIZE, return zero. Otherwise return the index
618*77c1e3ccSAndroid Build Coastguard Worker // of the best alpha found using intra_model_rd().
cfl_pick_plane_parameter(const AV1_COMP * const cpi,MACROBLOCK * x,int plane,TX_SIZE tx_size,int cfl_search_range)619*77c1e3ccSAndroid Build Coastguard Worker static int cfl_pick_plane_parameter(const AV1_COMP *const cpi, MACROBLOCK *x,
620*77c1e3ccSAndroid Build Coastguard Worker int plane, TX_SIZE tx_size,
621*77c1e3ccSAndroid Build Coastguard Worker int cfl_search_range) {
622*77c1e3ccSAndroid Build Coastguard Worker assert(cfl_search_range >= 1 && cfl_search_range <= CFL_MAGS_SIZE);
623*77c1e3ccSAndroid Build Coastguard Worker
624*77c1e3ccSAndroid Build Coastguard Worker if (cfl_search_range == CFL_MAGS_SIZE) return CFL_INDEX_ZERO;
625*77c1e3ccSAndroid Build Coastguard Worker
626*77c1e3ccSAndroid Build Coastguard Worker const MACROBLOCKD *const xd = &x->e_mbd;
627*77c1e3ccSAndroid Build Coastguard Worker const MB_MODE_INFO *const mbmi = xd->mi[0];
628*77c1e3ccSAndroid Build Coastguard Worker assert(mbmi->uv_mode == UV_CFL_PRED);
629*77c1e3ccSAndroid Build Coastguard Worker const MACROBLOCKD_PLANE *pd = &xd->plane[plane];
630*77c1e3ccSAndroid Build Coastguard Worker const BLOCK_SIZE plane_bsize =
631*77c1e3ccSAndroid Build Coastguard Worker get_plane_block_size(mbmi->bsize, pd->subsampling_x, pd->subsampling_y);
632*77c1e3ccSAndroid Build Coastguard Worker
633*77c1e3ccSAndroid Build Coastguard Worker int est_best_cfl_idx = CFL_INDEX_ZERO;
634*77c1e3ccSAndroid Build Coastguard Worker int fast_mode = 1;
635*77c1e3ccSAndroid Build Coastguard Worker int start_cfl_idx = CFL_INDEX_ZERO;
636*77c1e3ccSAndroid Build Coastguard Worker int64_t best_cfl_cost = cfl_compute_rd(cpi, x, plane, tx_size, plane_bsize,
637*77c1e3ccSAndroid Build Coastguard Worker start_cfl_idx, fast_mode, NULL);
638*77c1e3ccSAndroid Build Coastguard Worker for (int si = 0; si < 2; ++si) {
639*77c1e3ccSAndroid Build Coastguard Worker const int dir = cfl_dir_ls[si];
640*77c1e3ccSAndroid Build Coastguard Worker for (int i = 1; i < CFL_MAGS_SIZE; ++i) {
641*77c1e3ccSAndroid Build Coastguard Worker int cfl_idx = start_cfl_idx + dir * i;
642*77c1e3ccSAndroid Build Coastguard Worker if (cfl_idx < 0 || cfl_idx >= CFL_MAGS_SIZE) break;
643*77c1e3ccSAndroid Build Coastguard Worker int64_t cfl_cost = cfl_compute_rd(cpi, x, plane, tx_size, plane_bsize,
644*77c1e3ccSAndroid Build Coastguard Worker cfl_idx, fast_mode, NULL);
645*77c1e3ccSAndroid Build Coastguard Worker if (cfl_cost < best_cfl_cost) {
646*77c1e3ccSAndroid Build Coastguard Worker best_cfl_cost = cfl_cost;
647*77c1e3ccSAndroid Build Coastguard Worker est_best_cfl_idx = cfl_idx;
648*77c1e3ccSAndroid Build Coastguard Worker } else {
649*77c1e3ccSAndroid Build Coastguard Worker break;
650*77c1e3ccSAndroid Build Coastguard Worker }
651*77c1e3ccSAndroid Build Coastguard Worker }
652*77c1e3ccSAndroid Build Coastguard Worker }
653*77c1e3ccSAndroid Build Coastguard Worker return est_best_cfl_idx;
654*77c1e3ccSAndroid Build Coastguard Worker }
655*77c1e3ccSAndroid Build Coastguard Worker
set_invalid_cfl_parameters(uint8_t * best_cfl_alpha_idx,int8_t * best_cfl_alpha_signs)656*77c1e3ccSAndroid Build Coastguard Worker static inline void set_invalid_cfl_parameters(uint8_t *best_cfl_alpha_idx,
657*77c1e3ccSAndroid Build Coastguard Worker int8_t *best_cfl_alpha_signs) {
658*77c1e3ccSAndroid Build Coastguard Worker *best_cfl_alpha_idx = 0;
659*77c1e3ccSAndroid Build Coastguard Worker *best_cfl_alpha_signs = 0;
660*77c1e3ccSAndroid Build Coastguard Worker }
661*77c1e3ccSAndroid Build Coastguard Worker
cfl_pick_plane_rd(const AV1_COMP * const cpi,MACROBLOCK * x,int plane,TX_SIZE tx_size,int cfl_search_range,RD_STATS cfl_rd_arr[CFL_MAGS_SIZE],int est_best_cfl_idx)662*77c1e3ccSAndroid Build Coastguard Worker static void cfl_pick_plane_rd(const AV1_COMP *const cpi, MACROBLOCK *x,
663*77c1e3ccSAndroid Build Coastguard Worker int plane, TX_SIZE tx_size, int cfl_search_range,
664*77c1e3ccSAndroid Build Coastguard Worker RD_STATS cfl_rd_arr[CFL_MAGS_SIZE],
665*77c1e3ccSAndroid Build Coastguard Worker int est_best_cfl_idx) {
666*77c1e3ccSAndroid Build Coastguard Worker assert(cfl_search_range >= 1 && cfl_search_range <= CFL_MAGS_SIZE);
667*77c1e3ccSAndroid Build Coastguard Worker const MACROBLOCKD *const xd = &x->e_mbd;
668*77c1e3ccSAndroid Build Coastguard Worker const MB_MODE_INFO *const mbmi = xd->mi[0];
669*77c1e3ccSAndroid Build Coastguard Worker assert(mbmi->uv_mode == UV_CFL_PRED);
670*77c1e3ccSAndroid Build Coastguard Worker const MACROBLOCKD_PLANE *pd = &xd->plane[plane];
671*77c1e3ccSAndroid Build Coastguard Worker const BLOCK_SIZE plane_bsize =
672*77c1e3ccSAndroid Build Coastguard Worker get_plane_block_size(mbmi->bsize, pd->subsampling_x, pd->subsampling_y);
673*77c1e3ccSAndroid Build Coastguard Worker
674*77c1e3ccSAndroid Build Coastguard Worker for (int cfl_idx = 0; cfl_idx < CFL_MAGS_SIZE; ++cfl_idx) {
675*77c1e3ccSAndroid Build Coastguard Worker av1_invalid_rd_stats(&cfl_rd_arr[cfl_idx]);
676*77c1e3ccSAndroid Build Coastguard Worker }
677*77c1e3ccSAndroid Build Coastguard Worker
678*77c1e3ccSAndroid Build Coastguard Worker int fast_mode = 0;
679*77c1e3ccSAndroid Build Coastguard Worker int start_cfl_idx = est_best_cfl_idx;
680*77c1e3ccSAndroid Build Coastguard Worker cfl_compute_rd(cpi, x, plane, tx_size, plane_bsize, start_cfl_idx, fast_mode,
681*77c1e3ccSAndroid Build Coastguard Worker &cfl_rd_arr[start_cfl_idx]);
682*77c1e3ccSAndroid Build Coastguard Worker
683*77c1e3ccSAndroid Build Coastguard Worker if (cfl_search_range == 1) return;
684*77c1e3ccSAndroid Build Coastguard Worker
685*77c1e3ccSAndroid Build Coastguard Worker for (int si = 0; si < 2; ++si) {
686*77c1e3ccSAndroid Build Coastguard Worker const int dir = cfl_dir_ls[si];
687*77c1e3ccSAndroid Build Coastguard Worker for (int i = 1; i < cfl_search_range; ++i) {
688*77c1e3ccSAndroid Build Coastguard Worker int cfl_idx = start_cfl_idx + dir * i;
689*77c1e3ccSAndroid Build Coastguard Worker if (cfl_idx < 0 || cfl_idx >= CFL_MAGS_SIZE) break;
690*77c1e3ccSAndroid Build Coastguard Worker cfl_compute_rd(cpi, x, plane, tx_size, plane_bsize, cfl_idx, fast_mode,
691*77c1e3ccSAndroid Build Coastguard Worker &cfl_rd_arr[cfl_idx]);
692*77c1e3ccSAndroid Build Coastguard Worker }
693*77c1e3ccSAndroid Build Coastguard Worker }
694*77c1e3ccSAndroid Build Coastguard Worker }
695*77c1e3ccSAndroid Build Coastguard Worker
696*77c1e3ccSAndroid Build Coastguard Worker /*!\brief Pick the optimal parameters for Chroma to Luma (CFL) component
697*77c1e3ccSAndroid Build Coastguard Worker *
698*77c1e3ccSAndroid Build Coastguard Worker * \ingroup intra_mode_search
699*77c1e3ccSAndroid Build Coastguard Worker * \callergraph
700*77c1e3ccSAndroid Build Coastguard Worker *
701*77c1e3ccSAndroid Build Coastguard Worker * This function will use DCT_DCT followed by computing SATD (sum of absolute
702*77c1e3ccSAndroid Build Coastguard Worker * transformed differences) to estimate the RD score and find the best possible
703*77c1e3ccSAndroid Build Coastguard Worker * CFL parameter.
704*77c1e3ccSAndroid Build Coastguard Worker *
705*77c1e3ccSAndroid Build Coastguard Worker * Then the function will apply a full RD search near the best possible CFL
706*77c1e3ccSAndroid Build Coastguard Worker * parameter to find the best actual CFL parameter.
707*77c1e3ccSAndroid Build Coastguard Worker *
708*77c1e3ccSAndroid Build Coastguard Worker * Side effect:
709*77c1e3ccSAndroid Build Coastguard Worker * We use ths buffers in x->plane[] and xd->plane[] as throw-away buffers for RD
710*77c1e3ccSAndroid Build Coastguard Worker * search.
711*77c1e3ccSAndroid Build Coastguard Worker *
712*77c1e3ccSAndroid Build Coastguard Worker * \param[in] x Encoder prediction block structure.
713*77c1e3ccSAndroid Build Coastguard Worker * \param[in] cpi Top-level encoder instance structure.
714*77c1e3ccSAndroid Build Coastguard Worker * \param[in] tx_size Transform size.
715*77c1e3ccSAndroid Build Coastguard Worker * \param[in] ref_best_rd Reference best RD.
716*77c1e3ccSAndroid Build Coastguard Worker * \param[in] cfl_search_range The search range of full RD search near the
717*77c1e3ccSAndroid Build Coastguard Worker * estimated best CFL parameter.
718*77c1e3ccSAndroid Build Coastguard Worker *
719*77c1e3ccSAndroid Build Coastguard Worker * \param[out] best_rd_stats RD stats of the best CFL parameter
720*77c1e3ccSAndroid Build Coastguard Worker * \param[out] best_cfl_alpha_idx Best CFL alpha index
721*77c1e3ccSAndroid Build Coastguard Worker * \param[out] best_cfl_alpha_signs Best CFL joint signs
722*77c1e3ccSAndroid Build Coastguard Worker *
723*77c1e3ccSAndroid Build Coastguard Worker */
cfl_rd_pick_alpha(MACROBLOCK * const x,const AV1_COMP * const cpi,TX_SIZE tx_size,int64_t ref_best_rd,int cfl_search_range,RD_STATS * best_rd_stats,uint8_t * best_cfl_alpha_idx,int8_t * best_cfl_alpha_signs)724*77c1e3ccSAndroid Build Coastguard Worker static int cfl_rd_pick_alpha(MACROBLOCK *const x, const AV1_COMP *const cpi,
725*77c1e3ccSAndroid Build Coastguard Worker TX_SIZE tx_size, int64_t ref_best_rd,
726*77c1e3ccSAndroid Build Coastguard Worker int cfl_search_range, RD_STATS *best_rd_stats,
727*77c1e3ccSAndroid Build Coastguard Worker uint8_t *best_cfl_alpha_idx,
728*77c1e3ccSAndroid Build Coastguard Worker int8_t *best_cfl_alpha_signs) {
729*77c1e3ccSAndroid Build Coastguard Worker assert(cfl_search_range >= 1 && cfl_search_range <= CFL_MAGS_SIZE);
730*77c1e3ccSAndroid Build Coastguard Worker const ModeCosts *mode_costs = &x->mode_costs;
731*77c1e3ccSAndroid Build Coastguard Worker RD_STATS cfl_rd_arr_u[CFL_MAGS_SIZE];
732*77c1e3ccSAndroid Build Coastguard Worker RD_STATS cfl_rd_arr_v[CFL_MAGS_SIZE];
733*77c1e3ccSAndroid Build Coastguard Worker MACROBLOCKD *const xd = &x->e_mbd;
734*77c1e3ccSAndroid Build Coastguard Worker int est_best_cfl_idx_u, est_best_cfl_idx_v;
735*77c1e3ccSAndroid Build Coastguard Worker
736*77c1e3ccSAndroid Build Coastguard Worker av1_invalid_rd_stats(best_rd_stats);
737*77c1e3ccSAndroid Build Coastguard Worker
738*77c1e3ccSAndroid Build Coastguard Worker // As the dc pred data is same for different values of alpha, enable the
739*77c1e3ccSAndroid Build Coastguard Worker // caching of dc pred data. Call clear_cfl_dc_pred_cache_flags() before
740*77c1e3ccSAndroid Build Coastguard Worker // returning to avoid the unintentional usage of cached dc pred data.
741*77c1e3ccSAndroid Build Coastguard Worker xd->cfl.use_dc_pred_cache = true;
742*77c1e3ccSAndroid Build Coastguard Worker // Evaluate alpha parameter of each chroma plane.
743*77c1e3ccSAndroid Build Coastguard Worker est_best_cfl_idx_u =
744*77c1e3ccSAndroid Build Coastguard Worker cfl_pick_plane_parameter(cpi, x, 1, tx_size, cfl_search_range);
745*77c1e3ccSAndroid Build Coastguard Worker est_best_cfl_idx_v =
746*77c1e3ccSAndroid Build Coastguard Worker cfl_pick_plane_parameter(cpi, x, 2, tx_size, cfl_search_range);
747*77c1e3ccSAndroid Build Coastguard Worker
748*77c1e3ccSAndroid Build Coastguard Worker if (cfl_search_range == 1) {
749*77c1e3ccSAndroid Build Coastguard Worker // For cfl_search_range=1, further refinement of alpha is not enabled. Hence
750*77c1e3ccSAndroid Build Coastguard Worker // CfL index=0 for both the chroma planes implies invalid CfL mode.
751*77c1e3ccSAndroid Build Coastguard Worker if (est_best_cfl_idx_u == CFL_INDEX_ZERO &&
752*77c1e3ccSAndroid Build Coastguard Worker est_best_cfl_idx_v == CFL_INDEX_ZERO) {
753*77c1e3ccSAndroid Build Coastguard Worker set_invalid_cfl_parameters(best_cfl_alpha_idx, best_cfl_alpha_signs);
754*77c1e3ccSAndroid Build Coastguard Worker clear_cfl_dc_pred_cache_flags(&xd->cfl);
755*77c1e3ccSAndroid Build Coastguard Worker return 0;
756*77c1e3ccSAndroid Build Coastguard Worker }
757*77c1e3ccSAndroid Build Coastguard Worker
758*77c1e3ccSAndroid Build Coastguard Worker int cfl_alpha_u, cfl_alpha_v;
759*77c1e3ccSAndroid Build Coastguard Worker CFL_SIGN_TYPE cfl_sign_u, cfl_sign_v;
760*77c1e3ccSAndroid Build Coastguard Worker const MB_MODE_INFO *mbmi = xd->mi[0];
761*77c1e3ccSAndroid Build Coastguard Worker cfl_idx_to_sign_and_alpha(est_best_cfl_idx_u, &cfl_sign_u, &cfl_alpha_u);
762*77c1e3ccSAndroid Build Coastguard Worker cfl_idx_to_sign_and_alpha(est_best_cfl_idx_v, &cfl_sign_v, &cfl_alpha_v);
763*77c1e3ccSAndroid Build Coastguard Worker const int joint_sign = cfl_sign_u * CFL_SIGNS + cfl_sign_v - 1;
764*77c1e3ccSAndroid Build Coastguard Worker // Compute alpha and mode signaling rate.
765*77c1e3ccSAndroid Build Coastguard Worker const int rate_overhead =
766*77c1e3ccSAndroid Build Coastguard Worker mode_costs->cfl_cost[joint_sign][CFL_PRED_U][cfl_alpha_u] +
767*77c1e3ccSAndroid Build Coastguard Worker mode_costs->cfl_cost[joint_sign][CFL_PRED_V][cfl_alpha_v] +
768*77c1e3ccSAndroid Build Coastguard Worker mode_costs
769*77c1e3ccSAndroid Build Coastguard Worker ->intra_uv_mode_cost[is_cfl_allowed(xd)][mbmi->mode][UV_CFL_PRED];
770*77c1e3ccSAndroid Build Coastguard Worker // Skip the CfL mode evaluation if the RD cost derived using the rate needed
771*77c1e3ccSAndroid Build Coastguard Worker // to signal the CfL mode and alpha parameter exceeds the ref_best_rd.
772*77c1e3ccSAndroid Build Coastguard Worker if (RDCOST(x->rdmult, rate_overhead, 0) > ref_best_rd) {
773*77c1e3ccSAndroid Build Coastguard Worker set_invalid_cfl_parameters(best_cfl_alpha_idx, best_cfl_alpha_signs);
774*77c1e3ccSAndroid Build Coastguard Worker clear_cfl_dc_pred_cache_flags(&xd->cfl);
775*77c1e3ccSAndroid Build Coastguard Worker return 0;
776*77c1e3ccSAndroid Build Coastguard Worker }
777*77c1e3ccSAndroid Build Coastguard Worker }
778*77c1e3ccSAndroid Build Coastguard Worker
779*77c1e3ccSAndroid Build Coastguard Worker // Compute the rd cost of each chroma plane using the alpha parameters which
780*77c1e3ccSAndroid Build Coastguard Worker // were already evaluated.
781*77c1e3ccSAndroid Build Coastguard Worker cfl_pick_plane_rd(cpi, x, 1, tx_size, cfl_search_range, cfl_rd_arr_u,
782*77c1e3ccSAndroid Build Coastguard Worker est_best_cfl_idx_u);
783*77c1e3ccSAndroid Build Coastguard Worker cfl_pick_plane_rd(cpi, x, 2, tx_size, cfl_search_range, cfl_rd_arr_v,
784*77c1e3ccSAndroid Build Coastguard Worker est_best_cfl_idx_v);
785*77c1e3ccSAndroid Build Coastguard Worker
786*77c1e3ccSAndroid Build Coastguard Worker clear_cfl_dc_pred_cache_flags(&xd->cfl);
787*77c1e3ccSAndroid Build Coastguard Worker
788*77c1e3ccSAndroid Build Coastguard Worker for (int ui = 0; ui < CFL_MAGS_SIZE; ++ui) {
789*77c1e3ccSAndroid Build Coastguard Worker if (cfl_rd_arr_u[ui].rate == INT_MAX) continue;
790*77c1e3ccSAndroid Build Coastguard Worker int cfl_alpha_u;
791*77c1e3ccSAndroid Build Coastguard Worker CFL_SIGN_TYPE cfl_sign_u;
792*77c1e3ccSAndroid Build Coastguard Worker cfl_idx_to_sign_and_alpha(ui, &cfl_sign_u, &cfl_alpha_u);
793*77c1e3ccSAndroid Build Coastguard Worker for (int vi = 0; vi < CFL_MAGS_SIZE; ++vi) {
794*77c1e3ccSAndroid Build Coastguard Worker if (cfl_rd_arr_v[vi].rate == INT_MAX) continue;
795*77c1e3ccSAndroid Build Coastguard Worker int cfl_alpha_v;
796*77c1e3ccSAndroid Build Coastguard Worker CFL_SIGN_TYPE cfl_sign_v;
797*77c1e3ccSAndroid Build Coastguard Worker cfl_idx_to_sign_and_alpha(vi, &cfl_sign_v, &cfl_alpha_v);
798*77c1e3ccSAndroid Build Coastguard Worker // cfl_sign_u == CFL_SIGN_ZERO && cfl_sign_v == CFL_SIGN_ZERO is not a
799*77c1e3ccSAndroid Build Coastguard Worker // valid parameter for CFL
800*77c1e3ccSAndroid Build Coastguard Worker if (cfl_sign_u == CFL_SIGN_ZERO && cfl_sign_v == CFL_SIGN_ZERO) continue;
801*77c1e3ccSAndroid Build Coastguard Worker int joint_sign = cfl_sign_u * CFL_SIGNS + cfl_sign_v - 1;
802*77c1e3ccSAndroid Build Coastguard Worker RD_STATS rd_stats = cfl_rd_arr_u[ui];
803*77c1e3ccSAndroid Build Coastguard Worker av1_merge_rd_stats(&rd_stats, &cfl_rd_arr_v[vi]);
804*77c1e3ccSAndroid Build Coastguard Worker if (rd_stats.rate != INT_MAX) {
805*77c1e3ccSAndroid Build Coastguard Worker rd_stats.rate +=
806*77c1e3ccSAndroid Build Coastguard Worker mode_costs->cfl_cost[joint_sign][CFL_PRED_U][cfl_alpha_u];
807*77c1e3ccSAndroid Build Coastguard Worker rd_stats.rate +=
808*77c1e3ccSAndroid Build Coastguard Worker mode_costs->cfl_cost[joint_sign][CFL_PRED_V][cfl_alpha_v];
809*77c1e3ccSAndroid Build Coastguard Worker }
810*77c1e3ccSAndroid Build Coastguard Worker av1_rd_cost_update(x->rdmult, &rd_stats);
811*77c1e3ccSAndroid Build Coastguard Worker if (rd_stats.rdcost < best_rd_stats->rdcost) {
812*77c1e3ccSAndroid Build Coastguard Worker *best_rd_stats = rd_stats;
813*77c1e3ccSAndroid Build Coastguard Worker *best_cfl_alpha_idx =
814*77c1e3ccSAndroid Build Coastguard Worker (cfl_alpha_u << CFL_ALPHABET_SIZE_LOG2) + cfl_alpha_v;
815*77c1e3ccSAndroid Build Coastguard Worker *best_cfl_alpha_signs = joint_sign;
816*77c1e3ccSAndroid Build Coastguard Worker }
817*77c1e3ccSAndroid Build Coastguard Worker }
818*77c1e3ccSAndroid Build Coastguard Worker }
819*77c1e3ccSAndroid Build Coastguard Worker if (best_rd_stats->rdcost >= ref_best_rd) {
820*77c1e3ccSAndroid Build Coastguard Worker av1_invalid_rd_stats(best_rd_stats);
821*77c1e3ccSAndroid Build Coastguard Worker // Set invalid CFL parameters here since the rdcost is not better than
822*77c1e3ccSAndroid Build Coastguard Worker // ref_best_rd.
823*77c1e3ccSAndroid Build Coastguard Worker set_invalid_cfl_parameters(best_cfl_alpha_idx, best_cfl_alpha_signs);
824*77c1e3ccSAndroid Build Coastguard Worker return 0;
825*77c1e3ccSAndroid Build Coastguard Worker }
826*77c1e3ccSAndroid Build Coastguard Worker return 1;
827*77c1e3ccSAndroid Build Coastguard Worker }
828*77c1e3ccSAndroid Build Coastguard Worker
should_prune_chroma_smooth_pred_based_on_source_variance(const AV1_COMP * cpi,const MACROBLOCK * x,BLOCK_SIZE bsize)829*77c1e3ccSAndroid Build Coastguard Worker static bool should_prune_chroma_smooth_pred_based_on_source_variance(
830*77c1e3ccSAndroid Build Coastguard Worker const AV1_COMP *cpi, const MACROBLOCK *x, BLOCK_SIZE bsize) {
831*77c1e3ccSAndroid Build Coastguard Worker if (!cpi->sf.intra_sf.prune_smooth_intra_mode_for_chroma) return false;
832*77c1e3ccSAndroid Build Coastguard Worker
833*77c1e3ccSAndroid Build Coastguard Worker // If the source variance of both chroma planes is less than 20 (empirically
834*77c1e3ccSAndroid Build Coastguard Worker // derived), prune UV_SMOOTH_PRED.
835*77c1e3ccSAndroid Build Coastguard Worker for (int i = AOM_PLANE_U; i < av1_num_planes(&cpi->common); i++) {
836*77c1e3ccSAndroid Build Coastguard Worker const unsigned int variance = av1_get_perpixel_variance_facade(
837*77c1e3ccSAndroid Build Coastguard Worker cpi, &x->e_mbd, &x->plane[i].src, bsize, i);
838*77c1e3ccSAndroid Build Coastguard Worker if (variance >= 20) return false;
839*77c1e3ccSAndroid Build Coastguard Worker }
840*77c1e3ccSAndroid Build Coastguard Worker return true;
841*77c1e3ccSAndroid Build Coastguard Worker }
842*77c1e3ccSAndroid Build Coastguard Worker
av1_rd_pick_intra_sbuv_mode(const AV1_COMP * const cpi,MACROBLOCK * x,int * rate,int * rate_tokenonly,int64_t * distortion,uint8_t * skippable,BLOCK_SIZE bsize,TX_SIZE max_tx_size)843*77c1e3ccSAndroid Build Coastguard Worker int64_t av1_rd_pick_intra_sbuv_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
844*77c1e3ccSAndroid Build Coastguard Worker int *rate, int *rate_tokenonly,
845*77c1e3ccSAndroid Build Coastguard Worker int64_t *distortion, uint8_t *skippable,
846*77c1e3ccSAndroid Build Coastguard Worker BLOCK_SIZE bsize, TX_SIZE max_tx_size) {
847*77c1e3ccSAndroid Build Coastguard Worker const AV1_COMMON *const cm = &cpi->common;
848*77c1e3ccSAndroid Build Coastguard Worker MACROBLOCKD *xd = &x->e_mbd;
849*77c1e3ccSAndroid Build Coastguard Worker MB_MODE_INFO *mbmi = xd->mi[0];
850*77c1e3ccSAndroid Build Coastguard Worker assert(!is_inter_block(mbmi));
851*77c1e3ccSAndroid Build Coastguard Worker MB_MODE_INFO best_mbmi = *mbmi;
852*77c1e3ccSAndroid Build Coastguard Worker int64_t best_rd = INT64_MAX, this_rd;
853*77c1e3ccSAndroid Build Coastguard Worker const ModeCosts *mode_costs = &x->mode_costs;
854*77c1e3ccSAndroid Build Coastguard Worker const IntraModeCfg *const intra_mode_cfg = &cpi->oxcf.intra_mode_cfg;
855*77c1e3ccSAndroid Build Coastguard Worker
856*77c1e3ccSAndroid Build Coastguard Worker init_sbuv_mode(mbmi);
857*77c1e3ccSAndroid Build Coastguard Worker
858*77c1e3ccSAndroid Build Coastguard Worker // Return if the current block does not correspond to a chroma block.
859*77c1e3ccSAndroid Build Coastguard Worker if (!xd->is_chroma_ref) {
860*77c1e3ccSAndroid Build Coastguard Worker *rate = 0;
861*77c1e3ccSAndroid Build Coastguard Worker *rate_tokenonly = 0;
862*77c1e3ccSAndroid Build Coastguard Worker *distortion = 0;
863*77c1e3ccSAndroid Build Coastguard Worker *skippable = 1;
864*77c1e3ccSAndroid Build Coastguard Worker return INT64_MAX;
865*77c1e3ccSAndroid Build Coastguard Worker }
866*77c1e3ccSAndroid Build Coastguard Worker
867*77c1e3ccSAndroid Build Coastguard Worker // Only store reconstructed luma when there's chroma RDO. When there's no
868*77c1e3ccSAndroid Build Coastguard Worker // chroma RDO, the reconstructed luma will be stored in encode_superblock().
869*77c1e3ccSAndroid Build Coastguard Worker xd->cfl.store_y = store_cfl_required_rdo(cm, x);
870*77c1e3ccSAndroid Build Coastguard Worker if (xd->cfl.store_y) {
871*77c1e3ccSAndroid Build Coastguard Worker // Restore reconstructed luma values.
872*77c1e3ccSAndroid Build Coastguard Worker // TODO([email protected]): right now we are re-computing the txfm in
873*77c1e3ccSAndroid Build Coastguard Worker // this function everytime we search through uv modes. There is some
874*77c1e3ccSAndroid Build Coastguard Worker // potential speed up here if we cache the result to avoid redundant
875*77c1e3ccSAndroid Build Coastguard Worker // computation.
876*77c1e3ccSAndroid Build Coastguard Worker av1_encode_intra_block_plane(cpi, x, mbmi->bsize, AOM_PLANE_Y,
877*77c1e3ccSAndroid Build Coastguard Worker DRY_RUN_NORMAL,
878*77c1e3ccSAndroid Build Coastguard Worker cpi->optimize_seg_arr[mbmi->segment_id]);
879*77c1e3ccSAndroid Build Coastguard Worker xd->cfl.store_y = 0;
880*77c1e3ccSAndroid Build Coastguard Worker }
881*77c1e3ccSAndroid Build Coastguard Worker IntraModeSearchState intra_search_state;
882*77c1e3ccSAndroid Build Coastguard Worker init_intra_mode_search_state(&intra_search_state);
883*77c1e3ccSAndroid Build Coastguard Worker const CFL_ALLOWED_TYPE cfl_allowed = is_cfl_allowed(xd);
884*77c1e3ccSAndroid Build Coastguard Worker
885*77c1e3ccSAndroid Build Coastguard Worker // Search through all non-palette modes.
886*77c1e3ccSAndroid Build Coastguard Worker for (int mode_idx = 0; mode_idx < UV_INTRA_MODES; ++mode_idx) {
887*77c1e3ccSAndroid Build Coastguard Worker int this_rate;
888*77c1e3ccSAndroid Build Coastguard Worker RD_STATS tokenonly_rd_stats;
889*77c1e3ccSAndroid Build Coastguard Worker UV_PREDICTION_MODE uv_mode = uv_rd_search_mode_order[mode_idx];
890*77c1e3ccSAndroid Build Coastguard Worker
891*77c1e3ccSAndroid Build Coastguard Worker // Skip the current mode evaluation if the RD cost derived using the mode
892*77c1e3ccSAndroid Build Coastguard Worker // signaling rate exceeds the best_rd so far.
893*77c1e3ccSAndroid Build Coastguard Worker const int mode_rate =
894*77c1e3ccSAndroid Build Coastguard Worker mode_costs->intra_uv_mode_cost[cfl_allowed][mbmi->mode][uv_mode];
895*77c1e3ccSAndroid Build Coastguard Worker if (RDCOST(x->rdmult, mode_rate, 0) > best_rd) continue;
896*77c1e3ccSAndroid Build Coastguard Worker
897*77c1e3ccSAndroid Build Coastguard Worker PREDICTION_MODE intra_mode = get_uv_mode(uv_mode);
898*77c1e3ccSAndroid Build Coastguard Worker const int is_diagonal_mode = av1_is_diagonal_mode(intra_mode);
899*77c1e3ccSAndroid Build Coastguard Worker const int is_directional_mode = av1_is_directional_mode(intra_mode);
900*77c1e3ccSAndroid Build Coastguard Worker
901*77c1e3ccSAndroid Build Coastguard Worker if (is_diagonal_mode && !cpi->oxcf.intra_mode_cfg.enable_diagonal_intra)
902*77c1e3ccSAndroid Build Coastguard Worker continue;
903*77c1e3ccSAndroid Build Coastguard Worker if (is_directional_mode &&
904*77c1e3ccSAndroid Build Coastguard Worker !cpi->oxcf.intra_mode_cfg.enable_directional_intra)
905*77c1e3ccSAndroid Build Coastguard Worker continue;
906*77c1e3ccSAndroid Build Coastguard Worker
907*77c1e3ccSAndroid Build Coastguard Worker if (!(cpi->sf.intra_sf.intra_uv_mode_mask[txsize_sqr_up_map[max_tx_size]] &
908*77c1e3ccSAndroid Build Coastguard Worker (1 << uv_mode)))
909*77c1e3ccSAndroid Build Coastguard Worker continue;
910*77c1e3ccSAndroid Build Coastguard Worker if (!intra_mode_cfg->enable_smooth_intra && uv_mode >= UV_SMOOTH_PRED &&
911*77c1e3ccSAndroid Build Coastguard Worker uv_mode <= UV_SMOOTH_H_PRED)
912*77c1e3ccSAndroid Build Coastguard Worker continue;
913*77c1e3ccSAndroid Build Coastguard Worker
914*77c1e3ccSAndroid Build Coastguard Worker if (!intra_mode_cfg->enable_paeth_intra && uv_mode == UV_PAETH_PRED)
915*77c1e3ccSAndroid Build Coastguard Worker continue;
916*77c1e3ccSAndroid Build Coastguard Worker
917*77c1e3ccSAndroid Build Coastguard Worker assert(mbmi->mode < INTRA_MODES);
918*77c1e3ccSAndroid Build Coastguard Worker if (cpi->sf.intra_sf.prune_chroma_modes_using_luma_winner &&
919*77c1e3ccSAndroid Build Coastguard Worker !(av1_derived_chroma_intra_mode_used_flag[mbmi->mode] & (1 << uv_mode)))
920*77c1e3ccSAndroid Build Coastguard Worker continue;
921*77c1e3ccSAndroid Build Coastguard Worker
922*77c1e3ccSAndroid Build Coastguard Worker mbmi->uv_mode = uv_mode;
923*77c1e3ccSAndroid Build Coastguard Worker
924*77c1e3ccSAndroid Build Coastguard Worker // Init variables for cfl and angle delta
925*77c1e3ccSAndroid Build Coastguard Worker const SPEED_FEATURES *sf = &cpi->sf;
926*77c1e3ccSAndroid Build Coastguard Worker mbmi->angle_delta[PLANE_TYPE_UV] = 0;
927*77c1e3ccSAndroid Build Coastguard Worker if (uv_mode == UV_CFL_PRED) {
928*77c1e3ccSAndroid Build Coastguard Worker if (!cfl_allowed || !intra_mode_cfg->enable_cfl_intra) continue;
929*77c1e3ccSAndroid Build Coastguard Worker assert(!is_directional_mode);
930*77c1e3ccSAndroid Build Coastguard Worker const TX_SIZE uv_tx_size = av1_get_tx_size(AOM_PLANE_U, xd);
931*77c1e3ccSAndroid Build Coastguard Worker if (!cfl_rd_pick_alpha(x, cpi, uv_tx_size, best_rd,
932*77c1e3ccSAndroid Build Coastguard Worker sf->intra_sf.cfl_search_range, &tokenonly_rd_stats,
933*77c1e3ccSAndroid Build Coastguard Worker &mbmi->cfl_alpha_idx, &mbmi->cfl_alpha_signs)) {
934*77c1e3ccSAndroid Build Coastguard Worker continue;
935*77c1e3ccSAndroid Build Coastguard Worker }
936*77c1e3ccSAndroid Build Coastguard Worker } else if (is_directional_mode && av1_use_angle_delta(mbmi->bsize) &&
937*77c1e3ccSAndroid Build Coastguard Worker intra_mode_cfg->enable_angle_delta) {
938*77c1e3ccSAndroid Build Coastguard Worker if (sf->intra_sf.chroma_intra_pruning_with_hog &&
939*77c1e3ccSAndroid Build Coastguard Worker !intra_search_state.dir_mode_skip_mask_ready) {
940*77c1e3ccSAndroid Build Coastguard Worker static const float thresh[2][4] = {
941*77c1e3ccSAndroid Build Coastguard Worker { -1.2f, 0.0f, 0.0f, 1.2f }, // Interframe
942*77c1e3ccSAndroid Build Coastguard Worker { -1.2f, -1.2f, -0.6f, 0.4f }, // Intraframe
943*77c1e3ccSAndroid Build Coastguard Worker };
944*77c1e3ccSAndroid Build Coastguard Worker const int is_chroma = 1;
945*77c1e3ccSAndroid Build Coastguard Worker const int is_intra_frame = frame_is_intra_only(cm);
946*77c1e3ccSAndroid Build Coastguard Worker prune_intra_mode_with_hog(
947*77c1e3ccSAndroid Build Coastguard Worker x, bsize, cm->seq_params->sb_size,
948*77c1e3ccSAndroid Build Coastguard Worker thresh[is_intra_frame]
949*77c1e3ccSAndroid Build Coastguard Worker [sf->intra_sf.chroma_intra_pruning_with_hog - 1],
950*77c1e3ccSAndroid Build Coastguard Worker intra_search_state.directional_mode_skip_mask, is_chroma);
951*77c1e3ccSAndroid Build Coastguard Worker intra_search_state.dir_mode_skip_mask_ready = 1;
952*77c1e3ccSAndroid Build Coastguard Worker }
953*77c1e3ccSAndroid Build Coastguard Worker if (intra_search_state.directional_mode_skip_mask[uv_mode]) {
954*77c1e3ccSAndroid Build Coastguard Worker continue;
955*77c1e3ccSAndroid Build Coastguard Worker }
956*77c1e3ccSAndroid Build Coastguard Worker
957*77c1e3ccSAndroid Build Coastguard Worker // Search through angle delta
958*77c1e3ccSAndroid Build Coastguard Worker const int rate_overhead =
959*77c1e3ccSAndroid Build Coastguard Worker mode_costs->intra_uv_mode_cost[cfl_allowed][mbmi->mode][uv_mode];
960*77c1e3ccSAndroid Build Coastguard Worker if (!rd_pick_intra_angle_sbuv(cpi, x, bsize, rate_overhead, best_rd,
961*77c1e3ccSAndroid Build Coastguard Worker &this_rate, &tokenonly_rd_stats))
962*77c1e3ccSAndroid Build Coastguard Worker continue;
963*77c1e3ccSAndroid Build Coastguard Worker } else {
964*77c1e3ccSAndroid Build Coastguard Worker if (uv_mode == UV_SMOOTH_PRED &&
965*77c1e3ccSAndroid Build Coastguard Worker should_prune_chroma_smooth_pred_based_on_source_variance(cpi, x,
966*77c1e3ccSAndroid Build Coastguard Worker bsize))
967*77c1e3ccSAndroid Build Coastguard Worker continue;
968*77c1e3ccSAndroid Build Coastguard Worker
969*77c1e3ccSAndroid Build Coastguard Worker // Predict directly if we don't need to search for angle delta.
970*77c1e3ccSAndroid Build Coastguard Worker if (!av1_txfm_uvrd(cpi, x, &tokenonly_rd_stats, bsize, best_rd)) {
971*77c1e3ccSAndroid Build Coastguard Worker continue;
972*77c1e3ccSAndroid Build Coastguard Worker }
973*77c1e3ccSAndroid Build Coastguard Worker }
974*77c1e3ccSAndroid Build Coastguard Worker const int mode_cost =
975*77c1e3ccSAndroid Build Coastguard Worker mode_costs->intra_uv_mode_cost[cfl_allowed][mbmi->mode][uv_mode];
976*77c1e3ccSAndroid Build Coastguard Worker this_rate = tokenonly_rd_stats.rate +
977*77c1e3ccSAndroid Build Coastguard Worker intra_mode_info_cost_uv(cpi, x, mbmi, bsize, mode_cost);
978*77c1e3ccSAndroid Build Coastguard Worker this_rd = RDCOST(x->rdmult, this_rate, tokenonly_rd_stats.dist);
979*77c1e3ccSAndroid Build Coastguard Worker
980*77c1e3ccSAndroid Build Coastguard Worker if (this_rd < best_rd) {
981*77c1e3ccSAndroid Build Coastguard Worker best_mbmi = *mbmi;
982*77c1e3ccSAndroid Build Coastguard Worker best_rd = this_rd;
983*77c1e3ccSAndroid Build Coastguard Worker *rate = this_rate;
984*77c1e3ccSAndroid Build Coastguard Worker *rate_tokenonly = tokenonly_rd_stats.rate;
985*77c1e3ccSAndroid Build Coastguard Worker *distortion = tokenonly_rd_stats.dist;
986*77c1e3ccSAndroid Build Coastguard Worker *skippable = tokenonly_rd_stats.skip_txfm;
987*77c1e3ccSAndroid Build Coastguard Worker }
988*77c1e3ccSAndroid Build Coastguard Worker }
989*77c1e3ccSAndroid Build Coastguard Worker
990*77c1e3ccSAndroid Build Coastguard Worker // Search palette mode
991*77c1e3ccSAndroid Build Coastguard Worker const int try_palette =
992*77c1e3ccSAndroid Build Coastguard Worker cpi->oxcf.tool_cfg.enable_palette &&
993*77c1e3ccSAndroid Build Coastguard Worker av1_allow_palette(cpi->common.features.allow_screen_content_tools,
994*77c1e3ccSAndroid Build Coastguard Worker mbmi->bsize);
995*77c1e3ccSAndroid Build Coastguard Worker if (try_palette) {
996*77c1e3ccSAndroid Build Coastguard Worker uint8_t *best_palette_color_map = x->palette_buffer->best_palette_color_map;
997*77c1e3ccSAndroid Build Coastguard Worker av1_rd_pick_palette_intra_sbuv(
998*77c1e3ccSAndroid Build Coastguard Worker cpi, x,
999*77c1e3ccSAndroid Build Coastguard Worker mode_costs->intra_uv_mode_cost[cfl_allowed][mbmi->mode][UV_DC_PRED],
1000*77c1e3ccSAndroid Build Coastguard Worker best_palette_color_map, &best_mbmi, &best_rd, rate, rate_tokenonly,
1001*77c1e3ccSAndroid Build Coastguard Worker distortion, skippable);
1002*77c1e3ccSAndroid Build Coastguard Worker }
1003*77c1e3ccSAndroid Build Coastguard Worker
1004*77c1e3ccSAndroid Build Coastguard Worker *mbmi = best_mbmi;
1005*77c1e3ccSAndroid Build Coastguard Worker // Make sure we actually chose a mode
1006*77c1e3ccSAndroid Build Coastguard Worker assert(best_rd < INT64_MAX);
1007*77c1e3ccSAndroid Build Coastguard Worker return best_rd;
1008*77c1e3ccSAndroid Build Coastguard Worker }
1009*77c1e3ccSAndroid Build Coastguard Worker
1010*77c1e3ccSAndroid Build Coastguard Worker // Searches palette mode for luma channel in inter frame.
av1_search_palette_mode(IntraModeSearchState * intra_search_state,const AV1_COMP * cpi,MACROBLOCK * x,BLOCK_SIZE bsize,unsigned int ref_frame_cost,PICK_MODE_CONTEXT * ctx,RD_STATS * this_rd_cost,int64_t best_rd)1011*77c1e3ccSAndroid Build Coastguard Worker int av1_search_palette_mode(IntraModeSearchState *intra_search_state,
1012*77c1e3ccSAndroid Build Coastguard Worker const AV1_COMP *cpi, MACROBLOCK *x,
1013*77c1e3ccSAndroid Build Coastguard Worker BLOCK_SIZE bsize, unsigned int ref_frame_cost,
1014*77c1e3ccSAndroid Build Coastguard Worker PICK_MODE_CONTEXT *ctx, RD_STATS *this_rd_cost,
1015*77c1e3ccSAndroid Build Coastguard Worker int64_t best_rd) {
1016*77c1e3ccSAndroid Build Coastguard Worker const AV1_COMMON *const cm = &cpi->common;
1017*77c1e3ccSAndroid Build Coastguard Worker MB_MODE_INFO *const mbmi = x->e_mbd.mi[0];
1018*77c1e3ccSAndroid Build Coastguard Worker PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
1019*77c1e3ccSAndroid Build Coastguard Worker const int num_planes = av1_num_planes(cm);
1020*77c1e3ccSAndroid Build Coastguard Worker MACROBLOCKD *const xd = &x->e_mbd;
1021*77c1e3ccSAndroid Build Coastguard Worker int rate2 = 0;
1022*77c1e3ccSAndroid Build Coastguard Worker int64_t distortion2 = 0, best_rd_palette = best_rd, this_rd;
1023*77c1e3ccSAndroid Build Coastguard Worker int skippable = 0;
1024*77c1e3ccSAndroid Build Coastguard Worker uint8_t *const best_palette_color_map =
1025*77c1e3ccSAndroid Build Coastguard Worker x->palette_buffer->best_palette_color_map;
1026*77c1e3ccSAndroid Build Coastguard Worker uint8_t *const color_map = xd->plane[0].color_index_map;
1027*77c1e3ccSAndroid Build Coastguard Worker MB_MODE_INFO best_mbmi_palette = *mbmi;
1028*77c1e3ccSAndroid Build Coastguard Worker uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE];
1029*77c1e3ccSAndroid Build Coastguard Worker uint8_t best_tx_type_map[MAX_MIB_SIZE * MAX_MIB_SIZE];
1030*77c1e3ccSAndroid Build Coastguard Worker const ModeCosts *mode_costs = &x->mode_costs;
1031*77c1e3ccSAndroid Build Coastguard Worker const int *const intra_mode_cost =
1032*77c1e3ccSAndroid Build Coastguard Worker mode_costs->mbmode_cost[size_group_lookup[bsize]];
1033*77c1e3ccSAndroid Build Coastguard Worker const int rows = block_size_high[bsize];
1034*77c1e3ccSAndroid Build Coastguard Worker const int cols = block_size_wide[bsize];
1035*77c1e3ccSAndroid Build Coastguard Worker
1036*77c1e3ccSAndroid Build Coastguard Worker mbmi->mode = DC_PRED;
1037*77c1e3ccSAndroid Build Coastguard Worker mbmi->uv_mode = UV_DC_PRED;
1038*77c1e3ccSAndroid Build Coastguard Worker mbmi->ref_frame[0] = INTRA_FRAME;
1039*77c1e3ccSAndroid Build Coastguard Worker mbmi->ref_frame[1] = NONE_FRAME;
1040*77c1e3ccSAndroid Build Coastguard Worker av1_zero(pmi->palette_size);
1041*77c1e3ccSAndroid Build Coastguard Worker
1042*77c1e3ccSAndroid Build Coastguard Worker RD_STATS rd_stats_y;
1043*77c1e3ccSAndroid Build Coastguard Worker av1_invalid_rd_stats(&rd_stats_y);
1044*77c1e3ccSAndroid Build Coastguard Worker av1_rd_pick_palette_intra_sby(cpi, x, bsize, intra_mode_cost[DC_PRED],
1045*77c1e3ccSAndroid Build Coastguard Worker &best_mbmi_palette, best_palette_color_map,
1046*77c1e3ccSAndroid Build Coastguard Worker &best_rd_palette, &rd_stats_y.rate, NULL,
1047*77c1e3ccSAndroid Build Coastguard Worker &rd_stats_y.dist, &rd_stats_y.skip_txfm, NULL,
1048*77c1e3ccSAndroid Build Coastguard Worker ctx, best_blk_skip, best_tx_type_map);
1049*77c1e3ccSAndroid Build Coastguard Worker if (rd_stats_y.rate == INT_MAX || pmi->palette_size[0] == 0) {
1050*77c1e3ccSAndroid Build Coastguard Worker this_rd_cost->rdcost = INT64_MAX;
1051*77c1e3ccSAndroid Build Coastguard Worker return skippable;
1052*77c1e3ccSAndroid Build Coastguard Worker }
1053*77c1e3ccSAndroid Build Coastguard Worker
1054*77c1e3ccSAndroid Build Coastguard Worker memcpy(x->txfm_search_info.blk_skip, best_blk_skip,
1055*77c1e3ccSAndroid Build Coastguard Worker sizeof(best_blk_skip[0]) * bsize_to_num_blk(bsize));
1056*77c1e3ccSAndroid Build Coastguard Worker av1_copy_array(xd->tx_type_map, best_tx_type_map, ctx->num_4x4_blk);
1057*77c1e3ccSAndroid Build Coastguard Worker memcpy(color_map, best_palette_color_map,
1058*77c1e3ccSAndroid Build Coastguard Worker rows * cols * sizeof(best_palette_color_map[0]));
1059*77c1e3ccSAndroid Build Coastguard Worker
1060*77c1e3ccSAndroid Build Coastguard Worker skippable = rd_stats_y.skip_txfm;
1061*77c1e3ccSAndroid Build Coastguard Worker distortion2 = rd_stats_y.dist;
1062*77c1e3ccSAndroid Build Coastguard Worker rate2 = rd_stats_y.rate + ref_frame_cost;
1063*77c1e3ccSAndroid Build Coastguard Worker if (num_planes > 1) {
1064*77c1e3ccSAndroid Build Coastguard Worker if (intra_search_state->rate_uv_intra == INT_MAX) {
1065*77c1e3ccSAndroid Build Coastguard Worker // We have not found any good uv mode yet, so we need to search for it.
1066*77c1e3ccSAndroid Build Coastguard Worker TX_SIZE uv_tx = av1_get_tx_size(AOM_PLANE_U, xd);
1067*77c1e3ccSAndroid Build Coastguard Worker av1_rd_pick_intra_sbuv_mode(cpi, x, &intra_search_state->rate_uv_intra,
1068*77c1e3ccSAndroid Build Coastguard Worker &intra_search_state->rate_uv_tokenonly,
1069*77c1e3ccSAndroid Build Coastguard Worker &intra_search_state->dist_uvs,
1070*77c1e3ccSAndroid Build Coastguard Worker &intra_search_state->skip_uvs, bsize, uv_tx);
1071*77c1e3ccSAndroid Build Coastguard Worker intra_search_state->mode_uv = mbmi->uv_mode;
1072*77c1e3ccSAndroid Build Coastguard Worker intra_search_state->pmi_uv = *pmi;
1073*77c1e3ccSAndroid Build Coastguard Worker intra_search_state->uv_angle_delta = mbmi->angle_delta[PLANE_TYPE_UV];
1074*77c1e3ccSAndroid Build Coastguard Worker }
1075*77c1e3ccSAndroid Build Coastguard Worker
1076*77c1e3ccSAndroid Build Coastguard Worker // We have found at least one good uv mode before, so copy and paste it
1077*77c1e3ccSAndroid Build Coastguard Worker // over.
1078*77c1e3ccSAndroid Build Coastguard Worker mbmi->uv_mode = intra_search_state->mode_uv;
1079*77c1e3ccSAndroid Build Coastguard Worker pmi->palette_size[1] = intra_search_state->pmi_uv.palette_size[1];
1080*77c1e3ccSAndroid Build Coastguard Worker if (pmi->palette_size[1] > 0) {
1081*77c1e3ccSAndroid Build Coastguard Worker memcpy(pmi->palette_colors + PALETTE_MAX_SIZE,
1082*77c1e3ccSAndroid Build Coastguard Worker intra_search_state->pmi_uv.palette_colors + PALETTE_MAX_SIZE,
1083*77c1e3ccSAndroid Build Coastguard Worker 2 * PALETTE_MAX_SIZE * sizeof(pmi->palette_colors[0]));
1084*77c1e3ccSAndroid Build Coastguard Worker }
1085*77c1e3ccSAndroid Build Coastguard Worker mbmi->angle_delta[PLANE_TYPE_UV] = intra_search_state->uv_angle_delta;
1086*77c1e3ccSAndroid Build Coastguard Worker skippable = skippable && intra_search_state->skip_uvs;
1087*77c1e3ccSAndroid Build Coastguard Worker distortion2 += intra_search_state->dist_uvs;
1088*77c1e3ccSAndroid Build Coastguard Worker rate2 += intra_search_state->rate_uv_intra;
1089*77c1e3ccSAndroid Build Coastguard Worker }
1090*77c1e3ccSAndroid Build Coastguard Worker
1091*77c1e3ccSAndroid Build Coastguard Worker if (skippable) {
1092*77c1e3ccSAndroid Build Coastguard Worker rate2 -= rd_stats_y.rate;
1093*77c1e3ccSAndroid Build Coastguard Worker if (num_planes > 1) rate2 -= intra_search_state->rate_uv_tokenonly;
1094*77c1e3ccSAndroid Build Coastguard Worker rate2 += mode_costs->skip_txfm_cost[av1_get_skip_txfm_context(xd)][1];
1095*77c1e3ccSAndroid Build Coastguard Worker } else {
1096*77c1e3ccSAndroid Build Coastguard Worker rate2 += mode_costs->skip_txfm_cost[av1_get_skip_txfm_context(xd)][0];
1097*77c1e3ccSAndroid Build Coastguard Worker }
1098*77c1e3ccSAndroid Build Coastguard Worker this_rd = RDCOST(x->rdmult, rate2, distortion2);
1099*77c1e3ccSAndroid Build Coastguard Worker this_rd_cost->rate = rate2;
1100*77c1e3ccSAndroid Build Coastguard Worker this_rd_cost->dist = distortion2;
1101*77c1e3ccSAndroid Build Coastguard Worker this_rd_cost->rdcost = this_rd;
1102*77c1e3ccSAndroid Build Coastguard Worker return skippable;
1103*77c1e3ccSAndroid Build Coastguard Worker }
1104*77c1e3ccSAndroid Build Coastguard Worker
av1_search_palette_mode_luma(const AV1_COMP * cpi,MACROBLOCK * x,BLOCK_SIZE bsize,unsigned int ref_frame_cost,PICK_MODE_CONTEXT * ctx,RD_STATS * this_rd_cost,int64_t best_rd)1105*77c1e3ccSAndroid Build Coastguard Worker void av1_search_palette_mode_luma(const AV1_COMP *cpi, MACROBLOCK *x,
1106*77c1e3ccSAndroid Build Coastguard Worker BLOCK_SIZE bsize, unsigned int ref_frame_cost,
1107*77c1e3ccSAndroid Build Coastguard Worker PICK_MODE_CONTEXT *ctx,
1108*77c1e3ccSAndroid Build Coastguard Worker RD_STATS *this_rd_cost, int64_t best_rd) {
1109*77c1e3ccSAndroid Build Coastguard Worker MB_MODE_INFO *const mbmi = x->e_mbd.mi[0];
1110*77c1e3ccSAndroid Build Coastguard Worker PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
1111*77c1e3ccSAndroid Build Coastguard Worker MACROBLOCKD *const xd = &x->e_mbd;
1112*77c1e3ccSAndroid Build Coastguard Worker int64_t best_rd_palette = best_rd, this_rd;
1113*77c1e3ccSAndroid Build Coastguard Worker uint8_t *const best_palette_color_map =
1114*77c1e3ccSAndroid Build Coastguard Worker x->palette_buffer->best_palette_color_map;
1115*77c1e3ccSAndroid Build Coastguard Worker uint8_t *const color_map = xd->plane[0].color_index_map;
1116*77c1e3ccSAndroid Build Coastguard Worker MB_MODE_INFO best_mbmi_palette = *mbmi;
1117*77c1e3ccSAndroid Build Coastguard Worker uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE];
1118*77c1e3ccSAndroid Build Coastguard Worker uint8_t best_tx_type_map[MAX_MIB_SIZE * MAX_MIB_SIZE];
1119*77c1e3ccSAndroid Build Coastguard Worker const ModeCosts *mode_costs = &x->mode_costs;
1120*77c1e3ccSAndroid Build Coastguard Worker const int *const intra_mode_cost =
1121*77c1e3ccSAndroid Build Coastguard Worker mode_costs->mbmode_cost[size_group_lookup[bsize]];
1122*77c1e3ccSAndroid Build Coastguard Worker const int rows = block_size_high[bsize];
1123*77c1e3ccSAndroid Build Coastguard Worker const int cols = block_size_wide[bsize];
1124*77c1e3ccSAndroid Build Coastguard Worker
1125*77c1e3ccSAndroid Build Coastguard Worker mbmi->mode = DC_PRED;
1126*77c1e3ccSAndroid Build Coastguard Worker mbmi->uv_mode = UV_DC_PRED;
1127*77c1e3ccSAndroid Build Coastguard Worker mbmi->ref_frame[0] = INTRA_FRAME;
1128*77c1e3ccSAndroid Build Coastguard Worker mbmi->ref_frame[1] = NONE_FRAME;
1129*77c1e3ccSAndroid Build Coastguard Worker av1_zero(pmi->palette_size);
1130*77c1e3ccSAndroid Build Coastguard Worker
1131*77c1e3ccSAndroid Build Coastguard Worker RD_STATS rd_stats_y;
1132*77c1e3ccSAndroid Build Coastguard Worker av1_invalid_rd_stats(&rd_stats_y);
1133*77c1e3ccSAndroid Build Coastguard Worker av1_rd_pick_palette_intra_sby(cpi, x, bsize, intra_mode_cost[DC_PRED],
1134*77c1e3ccSAndroid Build Coastguard Worker &best_mbmi_palette, best_palette_color_map,
1135*77c1e3ccSAndroid Build Coastguard Worker &best_rd_palette, &rd_stats_y.rate, NULL,
1136*77c1e3ccSAndroid Build Coastguard Worker &rd_stats_y.dist, &rd_stats_y.skip_txfm, NULL,
1137*77c1e3ccSAndroid Build Coastguard Worker ctx, best_blk_skip, best_tx_type_map);
1138*77c1e3ccSAndroid Build Coastguard Worker if (rd_stats_y.rate == INT_MAX || pmi->palette_size[0] == 0) {
1139*77c1e3ccSAndroid Build Coastguard Worker this_rd_cost->rdcost = INT64_MAX;
1140*77c1e3ccSAndroid Build Coastguard Worker return;
1141*77c1e3ccSAndroid Build Coastguard Worker }
1142*77c1e3ccSAndroid Build Coastguard Worker
1143*77c1e3ccSAndroid Build Coastguard Worker memcpy(x->txfm_search_info.blk_skip, best_blk_skip,
1144*77c1e3ccSAndroid Build Coastguard Worker sizeof(best_blk_skip[0]) * bsize_to_num_blk(bsize));
1145*77c1e3ccSAndroid Build Coastguard Worker av1_copy_array(xd->tx_type_map, best_tx_type_map, ctx->num_4x4_blk);
1146*77c1e3ccSAndroid Build Coastguard Worker memcpy(color_map, best_palette_color_map,
1147*77c1e3ccSAndroid Build Coastguard Worker rows * cols * sizeof(best_palette_color_map[0]));
1148*77c1e3ccSAndroid Build Coastguard Worker
1149*77c1e3ccSAndroid Build Coastguard Worker rd_stats_y.rate += ref_frame_cost;
1150*77c1e3ccSAndroid Build Coastguard Worker
1151*77c1e3ccSAndroid Build Coastguard Worker if (rd_stats_y.skip_txfm) {
1152*77c1e3ccSAndroid Build Coastguard Worker rd_stats_y.rate =
1153*77c1e3ccSAndroid Build Coastguard Worker ref_frame_cost +
1154*77c1e3ccSAndroid Build Coastguard Worker mode_costs->skip_txfm_cost[av1_get_skip_txfm_context(xd)][1];
1155*77c1e3ccSAndroid Build Coastguard Worker } else {
1156*77c1e3ccSAndroid Build Coastguard Worker rd_stats_y.rate +=
1157*77c1e3ccSAndroid Build Coastguard Worker mode_costs->skip_txfm_cost[av1_get_skip_txfm_context(xd)][0];
1158*77c1e3ccSAndroid Build Coastguard Worker }
1159*77c1e3ccSAndroid Build Coastguard Worker this_rd = RDCOST(x->rdmult, rd_stats_y.rate, rd_stats_y.dist);
1160*77c1e3ccSAndroid Build Coastguard Worker this_rd_cost->rate = rd_stats_y.rate;
1161*77c1e3ccSAndroid Build Coastguard Worker this_rd_cost->dist = rd_stats_y.dist;
1162*77c1e3ccSAndroid Build Coastguard Worker this_rd_cost->rdcost = this_rd;
1163*77c1e3ccSAndroid Build Coastguard Worker this_rd_cost->skip_txfm = rd_stats_y.skip_txfm;
1164*77c1e3ccSAndroid Build Coastguard Worker }
1165*77c1e3ccSAndroid Build Coastguard Worker
1166*77c1e3ccSAndroid Build Coastguard Worker /*!\brief Get the intra prediction by searching through tx_type and tx_size.
1167*77c1e3ccSAndroid Build Coastguard Worker *
1168*77c1e3ccSAndroid Build Coastguard Worker * \ingroup intra_mode_search
1169*77c1e3ccSAndroid Build Coastguard Worker * \callergraph
1170*77c1e3ccSAndroid Build Coastguard Worker * Currently this function is only used in the intra frame code path for
1171*77c1e3ccSAndroid Build Coastguard Worker * winner-mode processing.
1172*77c1e3ccSAndroid Build Coastguard Worker *
1173*77c1e3ccSAndroid Build Coastguard Worker * \return Returns whether the current mode is an improvement over best_rd.
1174*77c1e3ccSAndroid Build Coastguard Worker */
intra_block_yrd(const AV1_COMP * const cpi,MACROBLOCK * x,BLOCK_SIZE bsize,const int * bmode_costs,int64_t * best_rd,int * rate,int * rate_tokenonly,int64_t * distortion,uint8_t * skippable,MB_MODE_INFO * best_mbmi,PICK_MODE_CONTEXT * ctx)1175*77c1e3ccSAndroid Build Coastguard Worker static inline int intra_block_yrd(const AV1_COMP *const cpi, MACROBLOCK *x,
1176*77c1e3ccSAndroid Build Coastguard Worker BLOCK_SIZE bsize, const int *bmode_costs,
1177*77c1e3ccSAndroid Build Coastguard Worker int64_t *best_rd, int *rate,
1178*77c1e3ccSAndroid Build Coastguard Worker int *rate_tokenonly, int64_t *distortion,
1179*77c1e3ccSAndroid Build Coastguard Worker uint8_t *skippable, MB_MODE_INFO *best_mbmi,
1180*77c1e3ccSAndroid Build Coastguard Worker PICK_MODE_CONTEXT *ctx) {
1181*77c1e3ccSAndroid Build Coastguard Worker MACROBLOCKD *const xd = &x->e_mbd;
1182*77c1e3ccSAndroid Build Coastguard Worker MB_MODE_INFO *const mbmi = xd->mi[0];
1183*77c1e3ccSAndroid Build Coastguard Worker RD_STATS rd_stats;
1184*77c1e3ccSAndroid Build Coastguard Worker // In order to improve txfm search, avoid rd based breakouts during winner
1185*77c1e3ccSAndroid Build Coastguard Worker // mode evaluation. Hence passing ref_best_rd as INT64_MAX by default when the
1186*77c1e3ccSAndroid Build Coastguard Worker // speed feature use_rd_based_breakout_for_intra_tx_search is disabled.
1187*77c1e3ccSAndroid Build Coastguard Worker int64_t ref_best_rd = cpi->sf.tx_sf.use_rd_based_breakout_for_intra_tx_search
1188*77c1e3ccSAndroid Build Coastguard Worker ? *best_rd
1189*77c1e3ccSAndroid Build Coastguard Worker : INT64_MAX;
1190*77c1e3ccSAndroid Build Coastguard Worker av1_pick_uniform_tx_size_type_yrd(cpi, x, &rd_stats, bsize, ref_best_rd);
1191*77c1e3ccSAndroid Build Coastguard Worker if (rd_stats.rate == INT_MAX) return 0;
1192*77c1e3ccSAndroid Build Coastguard Worker int this_rate_tokenonly = rd_stats.rate;
1193*77c1e3ccSAndroid Build Coastguard Worker if (!xd->lossless[mbmi->segment_id] && block_signals_txsize(mbmi->bsize)) {
1194*77c1e3ccSAndroid Build Coastguard Worker // av1_pick_uniform_tx_size_type_yrd above includes the cost of the tx_size
1195*77c1e3ccSAndroid Build Coastguard Worker // in the tokenonly rate, but for intra blocks, tx_size is always coded
1196*77c1e3ccSAndroid Build Coastguard Worker // (prediction granularity), so we account for it in the full rate,
1197*77c1e3ccSAndroid Build Coastguard Worker // not the tokenonly rate.
1198*77c1e3ccSAndroid Build Coastguard Worker this_rate_tokenonly -= tx_size_cost(x, bsize, mbmi->tx_size);
1199*77c1e3ccSAndroid Build Coastguard Worker }
1200*77c1e3ccSAndroid Build Coastguard Worker const int this_rate =
1201*77c1e3ccSAndroid Build Coastguard Worker rd_stats.rate +
1202*77c1e3ccSAndroid Build Coastguard Worker intra_mode_info_cost_y(cpi, x, mbmi, bsize, bmode_costs[mbmi->mode], 0);
1203*77c1e3ccSAndroid Build Coastguard Worker const int64_t this_rd = RDCOST(x->rdmult, this_rate, rd_stats.dist);
1204*77c1e3ccSAndroid Build Coastguard Worker if (this_rd < *best_rd) {
1205*77c1e3ccSAndroid Build Coastguard Worker *best_mbmi = *mbmi;
1206*77c1e3ccSAndroid Build Coastguard Worker *best_rd = this_rd;
1207*77c1e3ccSAndroid Build Coastguard Worker *rate = this_rate;
1208*77c1e3ccSAndroid Build Coastguard Worker *rate_tokenonly = this_rate_tokenonly;
1209*77c1e3ccSAndroid Build Coastguard Worker *distortion = rd_stats.dist;
1210*77c1e3ccSAndroid Build Coastguard Worker *skippable = rd_stats.skip_txfm;
1211*77c1e3ccSAndroid Build Coastguard Worker av1_copy_array(ctx->blk_skip, x->txfm_search_info.blk_skip,
1212*77c1e3ccSAndroid Build Coastguard Worker ctx->num_4x4_blk);
1213*77c1e3ccSAndroid Build Coastguard Worker av1_copy_array(ctx->tx_type_map, xd->tx_type_map, ctx->num_4x4_blk);
1214*77c1e3ccSAndroid Build Coastguard Worker return 1;
1215*77c1e3ccSAndroid Build Coastguard Worker }
1216*77c1e3ccSAndroid Build Coastguard Worker return 0;
1217*77c1e3ccSAndroid Build Coastguard Worker }
1218*77c1e3ccSAndroid Build Coastguard Worker
1219*77c1e3ccSAndroid Build Coastguard Worker /*!\brief Search for the best filter_intra mode when coding inter frame.
1220*77c1e3ccSAndroid Build Coastguard Worker *
1221*77c1e3ccSAndroid Build Coastguard Worker * \ingroup intra_mode_search
1222*77c1e3ccSAndroid Build Coastguard Worker * \callergraph
1223*77c1e3ccSAndroid Build Coastguard Worker * This function loops through all filter_intra modes to find the best one.
1224*77c1e3ccSAndroid Build Coastguard Worker *
1225*77c1e3ccSAndroid Build Coastguard Worker * \remark Returns nothing, but updates the mbmi and rd_stats.
1226*77c1e3ccSAndroid Build Coastguard Worker */
handle_filter_intra_mode(const AV1_COMP * cpi,MACROBLOCK * x,BLOCK_SIZE bsize,const PICK_MODE_CONTEXT * ctx,RD_STATS * rd_stats_y,int mode_cost,int64_t best_rd,int64_t best_rd_so_far)1227*77c1e3ccSAndroid Build Coastguard Worker static inline void handle_filter_intra_mode(const AV1_COMP *cpi, MACROBLOCK *x,
1228*77c1e3ccSAndroid Build Coastguard Worker BLOCK_SIZE bsize,
1229*77c1e3ccSAndroid Build Coastguard Worker const PICK_MODE_CONTEXT *ctx,
1230*77c1e3ccSAndroid Build Coastguard Worker RD_STATS *rd_stats_y, int mode_cost,
1231*77c1e3ccSAndroid Build Coastguard Worker int64_t best_rd,
1232*77c1e3ccSAndroid Build Coastguard Worker int64_t best_rd_so_far) {
1233*77c1e3ccSAndroid Build Coastguard Worker MACROBLOCKD *const xd = &x->e_mbd;
1234*77c1e3ccSAndroid Build Coastguard Worker MB_MODE_INFO *const mbmi = xd->mi[0];
1235*77c1e3ccSAndroid Build Coastguard Worker assert(mbmi->mode == DC_PRED &&
1236*77c1e3ccSAndroid Build Coastguard Worker av1_filter_intra_allowed_bsize(&cpi->common, bsize));
1237*77c1e3ccSAndroid Build Coastguard Worker
1238*77c1e3ccSAndroid Build Coastguard Worker RD_STATS rd_stats_y_fi;
1239*77c1e3ccSAndroid Build Coastguard Worker int filter_intra_selected_flag = 0;
1240*77c1e3ccSAndroid Build Coastguard Worker TX_SIZE best_tx_size = mbmi->tx_size;
1241*77c1e3ccSAndroid Build Coastguard Worker FILTER_INTRA_MODE best_fi_mode = FILTER_DC_PRED;
1242*77c1e3ccSAndroid Build Coastguard Worker uint8_t best_blk_skip[MAX_MIB_SIZE * MAX_MIB_SIZE];
1243*77c1e3ccSAndroid Build Coastguard Worker memcpy(best_blk_skip, x->txfm_search_info.blk_skip,
1244*77c1e3ccSAndroid Build Coastguard Worker sizeof(best_blk_skip[0]) * ctx->num_4x4_blk);
1245*77c1e3ccSAndroid Build Coastguard Worker uint8_t best_tx_type_map[MAX_MIB_SIZE * MAX_MIB_SIZE];
1246*77c1e3ccSAndroid Build Coastguard Worker av1_copy_array(best_tx_type_map, xd->tx_type_map, ctx->num_4x4_blk);
1247*77c1e3ccSAndroid Build Coastguard Worker mbmi->filter_intra_mode_info.use_filter_intra = 1;
1248*77c1e3ccSAndroid Build Coastguard Worker for (FILTER_INTRA_MODE fi_mode = FILTER_DC_PRED; fi_mode < FILTER_INTRA_MODES;
1249*77c1e3ccSAndroid Build Coastguard Worker ++fi_mode) {
1250*77c1e3ccSAndroid Build Coastguard Worker mbmi->filter_intra_mode_info.filter_intra_mode = fi_mode;
1251*77c1e3ccSAndroid Build Coastguard Worker av1_pick_uniform_tx_size_type_yrd(cpi, x, &rd_stats_y_fi, bsize, best_rd);
1252*77c1e3ccSAndroid Build Coastguard Worker if (rd_stats_y_fi.rate == INT_MAX) continue;
1253*77c1e3ccSAndroid Build Coastguard Worker const int this_rate_tmp =
1254*77c1e3ccSAndroid Build Coastguard Worker rd_stats_y_fi.rate +
1255*77c1e3ccSAndroid Build Coastguard Worker intra_mode_info_cost_y(cpi, x, mbmi, bsize, mode_cost, 0);
1256*77c1e3ccSAndroid Build Coastguard Worker const int64_t this_rd_tmp =
1257*77c1e3ccSAndroid Build Coastguard Worker RDCOST(x->rdmult, this_rate_tmp, rd_stats_y_fi.dist);
1258*77c1e3ccSAndroid Build Coastguard Worker
1259*77c1e3ccSAndroid Build Coastguard Worker if (this_rd_tmp != INT64_MAX && this_rd_tmp / 2 > best_rd) {
1260*77c1e3ccSAndroid Build Coastguard Worker break;
1261*77c1e3ccSAndroid Build Coastguard Worker }
1262*77c1e3ccSAndroid Build Coastguard Worker if (this_rd_tmp < best_rd_so_far) {
1263*77c1e3ccSAndroid Build Coastguard Worker best_tx_size = mbmi->tx_size;
1264*77c1e3ccSAndroid Build Coastguard Worker av1_copy_array(best_tx_type_map, xd->tx_type_map, ctx->num_4x4_blk);
1265*77c1e3ccSAndroid Build Coastguard Worker memcpy(best_blk_skip, x->txfm_search_info.blk_skip,
1266*77c1e3ccSAndroid Build Coastguard Worker sizeof(best_blk_skip[0]) * ctx->num_4x4_blk);
1267*77c1e3ccSAndroid Build Coastguard Worker best_fi_mode = fi_mode;
1268*77c1e3ccSAndroid Build Coastguard Worker *rd_stats_y = rd_stats_y_fi;
1269*77c1e3ccSAndroid Build Coastguard Worker filter_intra_selected_flag = 1;
1270*77c1e3ccSAndroid Build Coastguard Worker best_rd_so_far = this_rd_tmp;
1271*77c1e3ccSAndroid Build Coastguard Worker }
1272*77c1e3ccSAndroid Build Coastguard Worker }
1273*77c1e3ccSAndroid Build Coastguard Worker
1274*77c1e3ccSAndroid Build Coastguard Worker mbmi->tx_size = best_tx_size;
1275*77c1e3ccSAndroid Build Coastguard Worker av1_copy_array(xd->tx_type_map, best_tx_type_map, ctx->num_4x4_blk);
1276*77c1e3ccSAndroid Build Coastguard Worker memcpy(x->txfm_search_info.blk_skip, best_blk_skip,
1277*77c1e3ccSAndroid Build Coastguard Worker sizeof(x->txfm_search_info.blk_skip[0]) * ctx->num_4x4_blk);
1278*77c1e3ccSAndroid Build Coastguard Worker
1279*77c1e3ccSAndroid Build Coastguard Worker if (filter_intra_selected_flag) {
1280*77c1e3ccSAndroid Build Coastguard Worker mbmi->filter_intra_mode_info.use_filter_intra = 1;
1281*77c1e3ccSAndroid Build Coastguard Worker mbmi->filter_intra_mode_info.filter_intra_mode = best_fi_mode;
1282*77c1e3ccSAndroid Build Coastguard Worker } else {
1283*77c1e3ccSAndroid Build Coastguard Worker mbmi->filter_intra_mode_info.use_filter_intra = 0;
1284*77c1e3ccSAndroid Build Coastguard Worker }
1285*77c1e3ccSAndroid Build Coastguard Worker }
1286*77c1e3ccSAndroid Build Coastguard Worker
1287*77c1e3ccSAndroid Build Coastguard Worker // Evaluate a given luma intra-mode in inter frames.
av1_handle_intra_y_mode(IntraModeSearchState * intra_search_state,const AV1_COMP * cpi,MACROBLOCK * x,BLOCK_SIZE bsize,unsigned int ref_frame_cost,const PICK_MODE_CONTEXT * ctx,RD_STATS * rd_stats_y,int64_t best_rd,int * mode_cost_y,int64_t * rd_y,int64_t * best_model_rd,int64_t top_intra_model_rd[])1288*77c1e3ccSAndroid Build Coastguard Worker int av1_handle_intra_y_mode(IntraModeSearchState *intra_search_state,
1289*77c1e3ccSAndroid Build Coastguard Worker const AV1_COMP *cpi, MACROBLOCK *x,
1290*77c1e3ccSAndroid Build Coastguard Worker BLOCK_SIZE bsize, unsigned int ref_frame_cost,
1291*77c1e3ccSAndroid Build Coastguard Worker const PICK_MODE_CONTEXT *ctx, RD_STATS *rd_stats_y,
1292*77c1e3ccSAndroid Build Coastguard Worker int64_t best_rd, int *mode_cost_y, int64_t *rd_y,
1293*77c1e3ccSAndroid Build Coastguard Worker int64_t *best_model_rd,
1294*77c1e3ccSAndroid Build Coastguard Worker int64_t top_intra_model_rd[]) {
1295*77c1e3ccSAndroid Build Coastguard Worker const AV1_COMMON *cm = &cpi->common;
1296*77c1e3ccSAndroid Build Coastguard Worker const INTRA_MODE_SPEED_FEATURES *const intra_sf = &cpi->sf.intra_sf;
1297*77c1e3ccSAndroid Build Coastguard Worker MACROBLOCKD *const xd = &x->e_mbd;
1298*77c1e3ccSAndroid Build Coastguard Worker MB_MODE_INFO *const mbmi = xd->mi[0];
1299*77c1e3ccSAndroid Build Coastguard Worker assert(mbmi->ref_frame[0] == INTRA_FRAME);
1300*77c1e3ccSAndroid Build Coastguard Worker const PREDICTION_MODE mode = mbmi->mode;
1301*77c1e3ccSAndroid Build Coastguard Worker const ModeCosts *mode_costs = &x->mode_costs;
1302*77c1e3ccSAndroid Build Coastguard Worker const int mode_cost =
1303*77c1e3ccSAndroid Build Coastguard Worker mode_costs->mbmode_cost[size_group_lookup[bsize]][mode] + ref_frame_cost;
1304*77c1e3ccSAndroid Build Coastguard Worker const int skip_ctx = av1_get_skip_txfm_context(xd);
1305*77c1e3ccSAndroid Build Coastguard Worker
1306*77c1e3ccSAndroid Build Coastguard Worker int known_rate = mode_cost;
1307*77c1e3ccSAndroid Build Coastguard Worker const int intra_cost_penalty = av1_get_intra_cost_penalty(
1308*77c1e3ccSAndroid Build Coastguard Worker cm->quant_params.base_qindex, cm->quant_params.y_dc_delta_q,
1309*77c1e3ccSAndroid Build Coastguard Worker cm->seq_params->bit_depth);
1310*77c1e3ccSAndroid Build Coastguard Worker
1311*77c1e3ccSAndroid Build Coastguard Worker if (mode != DC_PRED && mode != PAETH_PRED) known_rate += intra_cost_penalty;
1312*77c1e3ccSAndroid Build Coastguard Worker known_rate += AOMMIN(mode_costs->skip_txfm_cost[skip_ctx][0],
1313*77c1e3ccSAndroid Build Coastguard Worker mode_costs->skip_txfm_cost[skip_ctx][1]);
1314*77c1e3ccSAndroid Build Coastguard Worker const int64_t known_rd = RDCOST(x->rdmult, known_rate, 0);
1315*77c1e3ccSAndroid Build Coastguard Worker if (known_rd > best_rd) {
1316*77c1e3ccSAndroid Build Coastguard Worker intra_search_state->skip_intra_modes = 1;
1317*77c1e3ccSAndroid Build Coastguard Worker return 0;
1318*77c1e3ccSAndroid Build Coastguard Worker }
1319*77c1e3ccSAndroid Build Coastguard Worker
1320*77c1e3ccSAndroid Build Coastguard Worker const int is_directional_mode = av1_is_directional_mode(mode);
1321*77c1e3ccSAndroid Build Coastguard Worker if (is_directional_mode && av1_use_angle_delta(bsize) &&
1322*77c1e3ccSAndroid Build Coastguard Worker cpi->oxcf.intra_mode_cfg.enable_angle_delta) {
1323*77c1e3ccSAndroid Build Coastguard Worker if (intra_sf->intra_pruning_with_hog &&
1324*77c1e3ccSAndroid Build Coastguard Worker !intra_search_state->dir_mode_skip_mask_ready) {
1325*77c1e3ccSAndroid Build Coastguard Worker const float thresh[4] = { -1.2f, 0.0f, 0.0f, 1.2f };
1326*77c1e3ccSAndroid Build Coastguard Worker const int is_chroma = 0;
1327*77c1e3ccSAndroid Build Coastguard Worker prune_intra_mode_with_hog(x, bsize, cm->seq_params->sb_size,
1328*77c1e3ccSAndroid Build Coastguard Worker thresh[intra_sf->intra_pruning_with_hog - 1],
1329*77c1e3ccSAndroid Build Coastguard Worker intra_search_state->directional_mode_skip_mask,
1330*77c1e3ccSAndroid Build Coastguard Worker is_chroma);
1331*77c1e3ccSAndroid Build Coastguard Worker intra_search_state->dir_mode_skip_mask_ready = 1;
1332*77c1e3ccSAndroid Build Coastguard Worker }
1333*77c1e3ccSAndroid Build Coastguard Worker if (intra_search_state->directional_mode_skip_mask[mode]) return 0;
1334*77c1e3ccSAndroid Build Coastguard Worker }
1335*77c1e3ccSAndroid Build Coastguard Worker const TX_SIZE tx_size = AOMMIN(TX_32X32, max_txsize_lookup[bsize]);
1336*77c1e3ccSAndroid Build Coastguard Worker const int64_t this_model_rd =
1337*77c1e3ccSAndroid Build Coastguard Worker intra_model_rd(&cpi->common, x, 0, bsize, tx_size, /*use_hadamard=*/1);
1338*77c1e3ccSAndroid Build Coastguard Worker
1339*77c1e3ccSAndroid Build Coastguard Worker const int model_rd_index_for_pruning =
1340*77c1e3ccSAndroid Build Coastguard Worker get_model_rd_index_for_pruning(x, intra_sf);
1341*77c1e3ccSAndroid Build Coastguard Worker
1342*77c1e3ccSAndroid Build Coastguard Worker if (prune_intra_y_mode(this_model_rd, best_model_rd, top_intra_model_rd,
1343*77c1e3ccSAndroid Build Coastguard Worker intra_sf->top_intra_model_count_allowed,
1344*77c1e3ccSAndroid Build Coastguard Worker model_rd_index_for_pruning))
1345*77c1e3ccSAndroid Build Coastguard Worker return 0;
1346*77c1e3ccSAndroid Build Coastguard Worker av1_init_rd_stats(rd_stats_y);
1347*77c1e3ccSAndroid Build Coastguard Worker av1_pick_uniform_tx_size_type_yrd(cpi, x, rd_stats_y, bsize, best_rd);
1348*77c1e3ccSAndroid Build Coastguard Worker
1349*77c1e3ccSAndroid Build Coastguard Worker // Pick filter intra modes.
1350*77c1e3ccSAndroid Build Coastguard Worker if (mode == DC_PRED && av1_filter_intra_allowed_bsize(cm, bsize)) {
1351*77c1e3ccSAndroid Build Coastguard Worker int try_filter_intra = 1;
1352*77c1e3ccSAndroid Build Coastguard Worker int64_t best_rd_so_far = INT64_MAX;
1353*77c1e3ccSAndroid Build Coastguard Worker if (rd_stats_y->rate != INT_MAX) {
1354*77c1e3ccSAndroid Build Coastguard Worker // best_rd_so_far is the rdcost of DC_PRED without using filter_intra.
1355*77c1e3ccSAndroid Build Coastguard Worker // Later, in filter intra search, best_rd_so_far is used for comparison.
1356*77c1e3ccSAndroid Build Coastguard Worker mbmi->filter_intra_mode_info.use_filter_intra = 0;
1357*77c1e3ccSAndroid Build Coastguard Worker const int tmp_rate =
1358*77c1e3ccSAndroid Build Coastguard Worker rd_stats_y->rate +
1359*77c1e3ccSAndroid Build Coastguard Worker intra_mode_info_cost_y(cpi, x, mbmi, bsize, mode_cost, 0);
1360*77c1e3ccSAndroid Build Coastguard Worker best_rd_so_far = RDCOST(x->rdmult, tmp_rate, rd_stats_y->dist);
1361*77c1e3ccSAndroid Build Coastguard Worker try_filter_intra = (best_rd_so_far / 2) <= best_rd;
1362*77c1e3ccSAndroid Build Coastguard Worker } else if (intra_sf->skip_filter_intra_in_inter_frames >= 1) {
1363*77c1e3ccSAndroid Build Coastguard Worker // As rd cost of luma intra dc mode is more than best_rd (i.e.,
1364*77c1e3ccSAndroid Build Coastguard Worker // rd_stats_y->rate = INT_MAX), skip the evaluation of filter intra modes.
1365*77c1e3ccSAndroid Build Coastguard Worker try_filter_intra = 0;
1366*77c1e3ccSAndroid Build Coastguard Worker }
1367*77c1e3ccSAndroid Build Coastguard Worker
1368*77c1e3ccSAndroid Build Coastguard Worker if (try_filter_intra) {
1369*77c1e3ccSAndroid Build Coastguard Worker handle_filter_intra_mode(cpi, x, bsize, ctx, rd_stats_y, mode_cost,
1370*77c1e3ccSAndroid Build Coastguard Worker best_rd, best_rd_so_far);
1371*77c1e3ccSAndroid Build Coastguard Worker }
1372*77c1e3ccSAndroid Build Coastguard Worker }
1373*77c1e3ccSAndroid Build Coastguard Worker
1374*77c1e3ccSAndroid Build Coastguard Worker if (rd_stats_y->rate == INT_MAX) return 0;
1375*77c1e3ccSAndroid Build Coastguard Worker
1376*77c1e3ccSAndroid Build Coastguard Worker *mode_cost_y = intra_mode_info_cost_y(cpi, x, mbmi, bsize, mode_cost, 0);
1377*77c1e3ccSAndroid Build Coastguard Worker const int rate_y = rd_stats_y->skip_txfm
1378*77c1e3ccSAndroid Build Coastguard Worker ? mode_costs->skip_txfm_cost[skip_ctx][1]
1379*77c1e3ccSAndroid Build Coastguard Worker : rd_stats_y->rate;
1380*77c1e3ccSAndroid Build Coastguard Worker *rd_y = RDCOST(x->rdmult, rate_y + *mode_cost_y, rd_stats_y->dist);
1381*77c1e3ccSAndroid Build Coastguard Worker if (best_rd < (INT64_MAX / 2) && *rd_y > (best_rd + (best_rd >> 2))) {
1382*77c1e3ccSAndroid Build Coastguard Worker intra_search_state->skip_intra_modes = 1;
1383*77c1e3ccSAndroid Build Coastguard Worker return 0;
1384*77c1e3ccSAndroid Build Coastguard Worker }
1385*77c1e3ccSAndroid Build Coastguard Worker
1386*77c1e3ccSAndroid Build Coastguard Worker return 1;
1387*77c1e3ccSAndroid Build Coastguard Worker }
1388*77c1e3ccSAndroid Build Coastguard Worker
av1_search_intra_uv_modes_in_interframe(IntraModeSearchState * intra_search_state,const AV1_COMP * cpi,MACROBLOCK * x,BLOCK_SIZE bsize,RD_STATS * rd_stats,const RD_STATS * rd_stats_y,RD_STATS * rd_stats_uv,int64_t best_rd)1389*77c1e3ccSAndroid Build Coastguard Worker int av1_search_intra_uv_modes_in_interframe(
1390*77c1e3ccSAndroid Build Coastguard Worker IntraModeSearchState *intra_search_state, const AV1_COMP *cpi,
1391*77c1e3ccSAndroid Build Coastguard Worker MACROBLOCK *x, BLOCK_SIZE bsize, RD_STATS *rd_stats,
1392*77c1e3ccSAndroid Build Coastguard Worker const RD_STATS *rd_stats_y, RD_STATS *rd_stats_uv, int64_t best_rd) {
1393*77c1e3ccSAndroid Build Coastguard Worker const AV1_COMMON *cm = &cpi->common;
1394*77c1e3ccSAndroid Build Coastguard Worker MACROBLOCKD *const xd = &x->e_mbd;
1395*77c1e3ccSAndroid Build Coastguard Worker MB_MODE_INFO *const mbmi = xd->mi[0];
1396*77c1e3ccSAndroid Build Coastguard Worker assert(mbmi->ref_frame[0] == INTRA_FRAME);
1397*77c1e3ccSAndroid Build Coastguard Worker
1398*77c1e3ccSAndroid Build Coastguard Worker // TODO([email protected]): Consolidate the chroma search code here with
1399*77c1e3ccSAndroid Build Coastguard Worker // the one in av1_search_palette_mode.
1400*77c1e3ccSAndroid Build Coastguard Worker PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
1401*77c1e3ccSAndroid Build Coastguard Worker const int try_palette =
1402*77c1e3ccSAndroid Build Coastguard Worker cpi->oxcf.tool_cfg.enable_palette &&
1403*77c1e3ccSAndroid Build Coastguard Worker av1_allow_palette(cm->features.allow_screen_content_tools, mbmi->bsize);
1404*77c1e3ccSAndroid Build Coastguard Worker
1405*77c1e3ccSAndroid Build Coastguard Worker assert(intra_search_state->rate_uv_intra == INT_MAX);
1406*77c1e3ccSAndroid Build Coastguard Worker if (intra_search_state->rate_uv_intra == INT_MAX) {
1407*77c1e3ccSAndroid Build Coastguard Worker // If no good uv-predictor had been found, search for it.
1408*77c1e3ccSAndroid Build Coastguard Worker const TX_SIZE uv_tx = av1_get_tx_size(AOM_PLANE_U, xd);
1409*77c1e3ccSAndroid Build Coastguard Worker av1_rd_pick_intra_sbuv_mode(cpi, x, &intra_search_state->rate_uv_intra,
1410*77c1e3ccSAndroid Build Coastguard Worker &intra_search_state->rate_uv_tokenonly,
1411*77c1e3ccSAndroid Build Coastguard Worker &intra_search_state->dist_uvs,
1412*77c1e3ccSAndroid Build Coastguard Worker &intra_search_state->skip_uvs, bsize, uv_tx);
1413*77c1e3ccSAndroid Build Coastguard Worker intra_search_state->mode_uv = mbmi->uv_mode;
1414*77c1e3ccSAndroid Build Coastguard Worker if (try_palette) intra_search_state->pmi_uv = *pmi;
1415*77c1e3ccSAndroid Build Coastguard Worker intra_search_state->uv_angle_delta = mbmi->angle_delta[PLANE_TYPE_UV];
1416*77c1e3ccSAndroid Build Coastguard Worker
1417*77c1e3ccSAndroid Build Coastguard Worker const int uv_rate = intra_search_state->rate_uv_tokenonly;
1418*77c1e3ccSAndroid Build Coastguard Worker const int64_t uv_dist = intra_search_state->dist_uvs;
1419*77c1e3ccSAndroid Build Coastguard Worker const int64_t uv_rd = RDCOST(x->rdmult, uv_rate, uv_dist);
1420*77c1e3ccSAndroid Build Coastguard Worker if (uv_rd > best_rd) {
1421*77c1e3ccSAndroid Build Coastguard Worker // If there is no good intra uv-mode available, we can skip all intra
1422*77c1e3ccSAndroid Build Coastguard Worker // modes.
1423*77c1e3ccSAndroid Build Coastguard Worker intra_search_state->skip_intra_modes = 1;
1424*77c1e3ccSAndroid Build Coastguard Worker return 0;
1425*77c1e3ccSAndroid Build Coastguard Worker }
1426*77c1e3ccSAndroid Build Coastguard Worker }
1427*77c1e3ccSAndroid Build Coastguard Worker
1428*77c1e3ccSAndroid Build Coastguard Worker // If we are here, then the encoder has found at least one good intra uv
1429*77c1e3ccSAndroid Build Coastguard Worker // predictor, so we can directly copy its statistics over.
1430*77c1e3ccSAndroid Build Coastguard Worker // TODO(any): the stats here is not right if the best uv mode is CFL but the
1431*77c1e3ccSAndroid Build Coastguard Worker // best y mode is palette.
1432*77c1e3ccSAndroid Build Coastguard Worker rd_stats_uv->rate = intra_search_state->rate_uv_tokenonly;
1433*77c1e3ccSAndroid Build Coastguard Worker rd_stats_uv->dist = intra_search_state->dist_uvs;
1434*77c1e3ccSAndroid Build Coastguard Worker rd_stats_uv->skip_txfm = intra_search_state->skip_uvs;
1435*77c1e3ccSAndroid Build Coastguard Worker rd_stats->skip_txfm = rd_stats_y->skip_txfm && rd_stats_uv->skip_txfm;
1436*77c1e3ccSAndroid Build Coastguard Worker mbmi->uv_mode = intra_search_state->mode_uv;
1437*77c1e3ccSAndroid Build Coastguard Worker if (try_palette) {
1438*77c1e3ccSAndroid Build Coastguard Worker pmi->palette_size[1] = intra_search_state->pmi_uv.palette_size[1];
1439*77c1e3ccSAndroid Build Coastguard Worker memcpy(pmi->palette_colors + PALETTE_MAX_SIZE,
1440*77c1e3ccSAndroid Build Coastguard Worker intra_search_state->pmi_uv.palette_colors + PALETTE_MAX_SIZE,
1441*77c1e3ccSAndroid Build Coastguard Worker 2 * PALETTE_MAX_SIZE * sizeof(pmi->palette_colors[0]));
1442*77c1e3ccSAndroid Build Coastguard Worker }
1443*77c1e3ccSAndroid Build Coastguard Worker mbmi->angle_delta[PLANE_TYPE_UV] = intra_search_state->uv_angle_delta;
1444*77c1e3ccSAndroid Build Coastguard Worker
1445*77c1e3ccSAndroid Build Coastguard Worker return 1;
1446*77c1e3ccSAndroid Build Coastguard Worker }
1447*77c1e3ccSAndroid Build Coastguard Worker
1448*77c1e3ccSAndroid Build Coastguard Worker // Checks if odd delta angles can be pruned based on rdcosts of even delta
1449*77c1e3ccSAndroid Build Coastguard Worker // angles of the corresponding directional mode.
prune_luma_odd_delta_angles_using_rd_cost(const MB_MODE_INFO * const mbmi,const int64_t * const intra_modes_rd_cost,int64_t best_rd,int prune_luma_odd_delta_angles_in_intra)1450*77c1e3ccSAndroid Build Coastguard Worker static inline int prune_luma_odd_delta_angles_using_rd_cost(
1451*77c1e3ccSAndroid Build Coastguard Worker const MB_MODE_INFO *const mbmi, const int64_t *const intra_modes_rd_cost,
1452*77c1e3ccSAndroid Build Coastguard Worker int64_t best_rd, int prune_luma_odd_delta_angles_in_intra) {
1453*77c1e3ccSAndroid Build Coastguard Worker const int luma_delta_angle = mbmi->angle_delta[PLANE_TYPE_Y];
1454*77c1e3ccSAndroid Build Coastguard Worker if (!prune_luma_odd_delta_angles_in_intra ||
1455*77c1e3ccSAndroid Build Coastguard Worker !av1_is_directional_mode(mbmi->mode) || !(abs(luma_delta_angle) & 1) ||
1456*77c1e3ccSAndroid Build Coastguard Worker best_rd == INT64_MAX)
1457*77c1e3ccSAndroid Build Coastguard Worker return 0;
1458*77c1e3ccSAndroid Build Coastguard Worker
1459*77c1e3ccSAndroid Build Coastguard Worker const int64_t rd_thresh = best_rd + (best_rd >> 3);
1460*77c1e3ccSAndroid Build Coastguard Worker
1461*77c1e3ccSAndroid Build Coastguard Worker // Neighbour rdcosts are considered for pruning of odd delta angles as
1462*77c1e3ccSAndroid Build Coastguard Worker // mentioned below:
1463*77c1e3ccSAndroid Build Coastguard Worker // Delta angle Delta angle rdcost
1464*77c1e3ccSAndroid Build Coastguard Worker // to be pruned to be considered
1465*77c1e3ccSAndroid Build Coastguard Worker // -3 -2
1466*77c1e3ccSAndroid Build Coastguard Worker // -1 -2, 0
1467*77c1e3ccSAndroid Build Coastguard Worker // 1 0, 2
1468*77c1e3ccSAndroid Build Coastguard Worker // 3 2
1469*77c1e3ccSAndroid Build Coastguard Worker return intra_modes_rd_cost[luma_delta_angle + MAX_ANGLE_DELTA] > rd_thresh &&
1470*77c1e3ccSAndroid Build Coastguard Worker intra_modes_rd_cost[luma_delta_angle + MAX_ANGLE_DELTA + 2] >
1471*77c1e3ccSAndroid Build Coastguard Worker rd_thresh;
1472*77c1e3ccSAndroid Build Coastguard Worker }
1473*77c1e3ccSAndroid Build Coastguard Worker
1474*77c1e3ccSAndroid Build Coastguard Worker // Finds the best non-intrabc mode on an intra frame.
av1_rd_pick_intra_sby_mode(const AV1_COMP * const cpi,MACROBLOCK * x,int * rate,int * rate_tokenonly,int64_t * distortion,uint8_t * skippable,BLOCK_SIZE bsize,int64_t best_rd,PICK_MODE_CONTEXT * ctx)1475*77c1e3ccSAndroid Build Coastguard Worker int64_t av1_rd_pick_intra_sby_mode(const AV1_COMP *const cpi, MACROBLOCK *x,
1476*77c1e3ccSAndroid Build Coastguard Worker int *rate, int *rate_tokenonly,
1477*77c1e3ccSAndroid Build Coastguard Worker int64_t *distortion, uint8_t *skippable,
1478*77c1e3ccSAndroid Build Coastguard Worker BLOCK_SIZE bsize, int64_t best_rd,
1479*77c1e3ccSAndroid Build Coastguard Worker PICK_MODE_CONTEXT *ctx) {
1480*77c1e3ccSAndroid Build Coastguard Worker MACROBLOCKD *const xd = &x->e_mbd;
1481*77c1e3ccSAndroid Build Coastguard Worker MB_MODE_INFO *const mbmi = xd->mi[0];
1482*77c1e3ccSAndroid Build Coastguard Worker assert(!is_inter_block(mbmi));
1483*77c1e3ccSAndroid Build Coastguard Worker int64_t best_model_rd = INT64_MAX;
1484*77c1e3ccSAndroid Build Coastguard Worker int is_directional_mode;
1485*77c1e3ccSAndroid Build Coastguard Worker uint8_t directional_mode_skip_mask[INTRA_MODES] = { 0 };
1486*77c1e3ccSAndroid Build Coastguard Worker // Flag to check rd of any intra mode is better than best_rd passed to this
1487*77c1e3ccSAndroid Build Coastguard Worker // function
1488*77c1e3ccSAndroid Build Coastguard Worker int beat_best_rd = 0;
1489*77c1e3ccSAndroid Build Coastguard Worker const int *bmode_costs;
1490*77c1e3ccSAndroid Build Coastguard Worker const IntraModeCfg *const intra_mode_cfg = &cpi->oxcf.intra_mode_cfg;
1491*77c1e3ccSAndroid Build Coastguard Worker PALETTE_MODE_INFO *const pmi = &mbmi->palette_mode_info;
1492*77c1e3ccSAndroid Build Coastguard Worker const int try_palette =
1493*77c1e3ccSAndroid Build Coastguard Worker cpi->oxcf.tool_cfg.enable_palette &&
1494*77c1e3ccSAndroid Build Coastguard Worker av1_allow_palette(cpi->common.features.allow_screen_content_tools,
1495*77c1e3ccSAndroid Build Coastguard Worker mbmi->bsize);
1496*77c1e3ccSAndroid Build Coastguard Worker uint8_t *best_palette_color_map =
1497*77c1e3ccSAndroid Build Coastguard Worker try_palette ? x->palette_buffer->best_palette_color_map : NULL;
1498*77c1e3ccSAndroid Build Coastguard Worker const MB_MODE_INFO *above_mi = xd->above_mbmi;
1499*77c1e3ccSAndroid Build Coastguard Worker const MB_MODE_INFO *left_mi = xd->left_mbmi;
1500*77c1e3ccSAndroid Build Coastguard Worker const PREDICTION_MODE A = av1_above_block_mode(above_mi);
1501*77c1e3ccSAndroid Build Coastguard Worker const PREDICTION_MODE L = av1_left_block_mode(left_mi);
1502*77c1e3ccSAndroid Build Coastguard Worker const int above_ctx = intra_mode_context[A];
1503*77c1e3ccSAndroid Build Coastguard Worker const int left_ctx = intra_mode_context[L];
1504*77c1e3ccSAndroid Build Coastguard Worker bmode_costs = x->mode_costs.y_mode_costs[above_ctx][left_ctx];
1505*77c1e3ccSAndroid Build Coastguard Worker
1506*77c1e3ccSAndroid Build Coastguard Worker mbmi->angle_delta[PLANE_TYPE_Y] = 0;
1507*77c1e3ccSAndroid Build Coastguard Worker const INTRA_MODE_SPEED_FEATURES *const intra_sf = &cpi->sf.intra_sf;
1508*77c1e3ccSAndroid Build Coastguard Worker if (intra_sf->intra_pruning_with_hog) {
1509*77c1e3ccSAndroid Build Coastguard Worker // Less aggressive thresholds are used here than those used in inter frame
1510*77c1e3ccSAndroid Build Coastguard Worker // encoding in av1_handle_intra_y_mode() because we want key frames/intra
1511*77c1e3ccSAndroid Build Coastguard Worker // frames to have higher quality.
1512*77c1e3ccSAndroid Build Coastguard Worker const float thresh[4] = { -1.2f, -1.2f, -0.6f, 0.4f };
1513*77c1e3ccSAndroid Build Coastguard Worker const int is_chroma = 0;
1514*77c1e3ccSAndroid Build Coastguard Worker prune_intra_mode_with_hog(x, bsize, cpi->common.seq_params->sb_size,
1515*77c1e3ccSAndroid Build Coastguard Worker thresh[intra_sf->intra_pruning_with_hog - 1],
1516*77c1e3ccSAndroid Build Coastguard Worker directional_mode_skip_mask, is_chroma);
1517*77c1e3ccSAndroid Build Coastguard Worker }
1518*77c1e3ccSAndroid Build Coastguard Worker mbmi->filter_intra_mode_info.use_filter_intra = 0;
1519*77c1e3ccSAndroid Build Coastguard Worker pmi->palette_size[0] = 0;
1520*77c1e3ccSAndroid Build Coastguard Worker
1521*77c1e3ccSAndroid Build Coastguard Worker // Set params for mode evaluation
1522*77c1e3ccSAndroid Build Coastguard Worker set_mode_eval_params(cpi, x, MODE_EVAL);
1523*77c1e3ccSAndroid Build Coastguard Worker
1524*77c1e3ccSAndroid Build Coastguard Worker MB_MODE_INFO best_mbmi = *mbmi;
1525*77c1e3ccSAndroid Build Coastguard Worker const int max_winner_mode_count =
1526*77c1e3ccSAndroid Build Coastguard Worker winner_mode_count_allowed[cpi->sf.winner_mode_sf.multi_winner_mode_type];
1527*77c1e3ccSAndroid Build Coastguard Worker zero_winner_mode_stats(bsize, max_winner_mode_count, x->winner_mode_stats);
1528*77c1e3ccSAndroid Build Coastguard Worker x->winner_mode_count = 0;
1529*77c1e3ccSAndroid Build Coastguard Worker
1530*77c1e3ccSAndroid Build Coastguard Worker // Searches the intra-modes except for intrabc, palette, and filter_intra.
1531*77c1e3ccSAndroid Build Coastguard Worker int64_t top_intra_model_rd[TOP_INTRA_MODEL_COUNT];
1532*77c1e3ccSAndroid Build Coastguard Worker for (int i = 0; i < TOP_INTRA_MODEL_COUNT; i++) {
1533*77c1e3ccSAndroid Build Coastguard Worker top_intra_model_rd[i] = INT64_MAX;
1534*77c1e3ccSAndroid Build Coastguard Worker }
1535*77c1e3ccSAndroid Build Coastguard Worker
1536*77c1e3ccSAndroid Build Coastguard Worker // Initialize the rdcost corresponding to all the directional and
1537*77c1e3ccSAndroid Build Coastguard Worker // non-directional intra modes.
1538*77c1e3ccSAndroid Build Coastguard Worker // 1. For directional modes, it stores the rdcost values for delta angles -4,
1539*77c1e3ccSAndroid Build Coastguard Worker // -3, ..., 3, 4.
1540*77c1e3ccSAndroid Build Coastguard Worker // 2. The rdcost value for luma_delta_angle is stored at index
1541*77c1e3ccSAndroid Build Coastguard Worker // luma_delta_angle + MAX_ANGLE_DELTA + 1.
1542*77c1e3ccSAndroid Build Coastguard Worker // 3. The rdcost values for fictitious/nonexistent luma_delta_angle -4 and 4
1543*77c1e3ccSAndroid Build Coastguard Worker // (array indices 0 and 8) are always set to INT64_MAX (the initial value).
1544*77c1e3ccSAndroid Build Coastguard Worker int64_t intra_modes_rd_cost[INTRA_MODE_END]
1545*77c1e3ccSAndroid Build Coastguard Worker [SIZE_OF_ANGLE_DELTA_RD_COST_ARRAY];
1546*77c1e3ccSAndroid Build Coastguard Worker for (int i = 0; i < INTRA_MODE_END; i++) {
1547*77c1e3ccSAndroid Build Coastguard Worker for (int j = 0; j < SIZE_OF_ANGLE_DELTA_RD_COST_ARRAY; j++) {
1548*77c1e3ccSAndroid Build Coastguard Worker intra_modes_rd_cost[i][j] = INT64_MAX;
1549*77c1e3ccSAndroid Build Coastguard Worker }
1550*77c1e3ccSAndroid Build Coastguard Worker }
1551*77c1e3ccSAndroid Build Coastguard Worker
1552*77c1e3ccSAndroid Build Coastguard Worker for (int mode_idx = INTRA_MODE_START; mode_idx < LUMA_MODE_COUNT;
1553*77c1e3ccSAndroid Build Coastguard Worker ++mode_idx) {
1554*77c1e3ccSAndroid Build Coastguard Worker set_y_mode_and_delta_angle(mode_idx, mbmi,
1555*77c1e3ccSAndroid Build Coastguard Worker intra_sf->prune_luma_odd_delta_angles_in_intra);
1556*77c1e3ccSAndroid Build Coastguard Worker RD_STATS this_rd_stats;
1557*77c1e3ccSAndroid Build Coastguard Worker int this_rate, this_rate_tokenonly, s;
1558*77c1e3ccSAndroid Build Coastguard Worker int is_diagonal_mode;
1559*77c1e3ccSAndroid Build Coastguard Worker int64_t this_distortion, this_rd;
1560*77c1e3ccSAndroid Build Coastguard Worker const int luma_delta_angle = mbmi->angle_delta[PLANE_TYPE_Y];
1561*77c1e3ccSAndroid Build Coastguard Worker
1562*77c1e3ccSAndroid Build Coastguard Worker is_diagonal_mode = av1_is_diagonal_mode(mbmi->mode);
1563*77c1e3ccSAndroid Build Coastguard Worker if (is_diagonal_mode && !intra_mode_cfg->enable_diagonal_intra) continue;
1564*77c1e3ccSAndroid Build Coastguard Worker if (av1_is_directional_mode(mbmi->mode) &&
1565*77c1e3ccSAndroid Build Coastguard Worker !intra_mode_cfg->enable_directional_intra)
1566*77c1e3ccSAndroid Build Coastguard Worker continue;
1567*77c1e3ccSAndroid Build Coastguard Worker
1568*77c1e3ccSAndroid Build Coastguard Worker // The smooth prediction mode appears to be more frequently picked
1569*77c1e3ccSAndroid Build Coastguard Worker // than horizontal / vertical smooth prediction modes. Hence treat
1570*77c1e3ccSAndroid Build Coastguard Worker // them differently in speed features.
1571*77c1e3ccSAndroid Build Coastguard Worker if ((!intra_mode_cfg->enable_smooth_intra ||
1572*77c1e3ccSAndroid Build Coastguard Worker intra_sf->disable_smooth_intra) &&
1573*77c1e3ccSAndroid Build Coastguard Worker (mbmi->mode == SMOOTH_H_PRED || mbmi->mode == SMOOTH_V_PRED))
1574*77c1e3ccSAndroid Build Coastguard Worker continue;
1575*77c1e3ccSAndroid Build Coastguard Worker if (!intra_mode_cfg->enable_smooth_intra && mbmi->mode == SMOOTH_PRED)
1576*77c1e3ccSAndroid Build Coastguard Worker continue;
1577*77c1e3ccSAndroid Build Coastguard Worker
1578*77c1e3ccSAndroid Build Coastguard Worker // The functionality of filter intra modes and smooth prediction
1579*77c1e3ccSAndroid Build Coastguard Worker // overlap. Hence smooth prediction is pruned only if all the
1580*77c1e3ccSAndroid Build Coastguard Worker // filter intra modes are enabled.
1581*77c1e3ccSAndroid Build Coastguard Worker if (intra_sf->disable_smooth_intra &&
1582*77c1e3ccSAndroid Build Coastguard Worker intra_sf->prune_filter_intra_level == 0 && mbmi->mode == SMOOTH_PRED)
1583*77c1e3ccSAndroid Build Coastguard Worker continue;
1584*77c1e3ccSAndroid Build Coastguard Worker if (!intra_mode_cfg->enable_paeth_intra && mbmi->mode == PAETH_PRED)
1585*77c1e3ccSAndroid Build Coastguard Worker continue;
1586*77c1e3ccSAndroid Build Coastguard Worker
1587*77c1e3ccSAndroid Build Coastguard Worker // Skip the evaluation of modes that do not match with the winner mode in
1588*77c1e3ccSAndroid Build Coastguard Worker // x->mb_mode_cache.
1589*77c1e3ccSAndroid Build Coastguard Worker if (x->use_mb_mode_cache && mbmi->mode != x->mb_mode_cache->mode) continue;
1590*77c1e3ccSAndroid Build Coastguard Worker
1591*77c1e3ccSAndroid Build Coastguard Worker is_directional_mode = av1_is_directional_mode(mbmi->mode);
1592*77c1e3ccSAndroid Build Coastguard Worker if (is_directional_mode && directional_mode_skip_mask[mbmi->mode]) continue;
1593*77c1e3ccSAndroid Build Coastguard Worker if (is_directional_mode &&
1594*77c1e3ccSAndroid Build Coastguard Worker !(av1_use_angle_delta(bsize) && intra_mode_cfg->enable_angle_delta) &&
1595*77c1e3ccSAndroid Build Coastguard Worker luma_delta_angle != 0)
1596*77c1e3ccSAndroid Build Coastguard Worker continue;
1597*77c1e3ccSAndroid Build Coastguard Worker
1598*77c1e3ccSAndroid Build Coastguard Worker // Use intra_y_mode_mask speed feature to skip intra mode evaluation.
1599*77c1e3ccSAndroid Build Coastguard Worker if (!(intra_sf->intra_y_mode_mask[max_txsize_lookup[bsize]] &
1600*77c1e3ccSAndroid Build Coastguard Worker (1 << mbmi->mode)))
1601*77c1e3ccSAndroid Build Coastguard Worker continue;
1602*77c1e3ccSAndroid Build Coastguard Worker
1603*77c1e3ccSAndroid Build Coastguard Worker if (prune_luma_odd_delta_angles_using_rd_cost(
1604*77c1e3ccSAndroid Build Coastguard Worker mbmi, intra_modes_rd_cost[mbmi->mode], best_rd,
1605*77c1e3ccSAndroid Build Coastguard Worker intra_sf->prune_luma_odd_delta_angles_in_intra))
1606*77c1e3ccSAndroid Build Coastguard Worker continue;
1607*77c1e3ccSAndroid Build Coastguard Worker
1608*77c1e3ccSAndroid Build Coastguard Worker const TX_SIZE tx_size = AOMMIN(TX_32X32, max_txsize_lookup[bsize]);
1609*77c1e3ccSAndroid Build Coastguard Worker const int64_t this_model_rd =
1610*77c1e3ccSAndroid Build Coastguard Worker intra_model_rd(&cpi->common, x, 0, bsize, tx_size, /*use_hadamard=*/1);
1611*77c1e3ccSAndroid Build Coastguard Worker
1612*77c1e3ccSAndroid Build Coastguard Worker const int model_rd_index_for_pruning =
1613*77c1e3ccSAndroid Build Coastguard Worker get_model_rd_index_for_pruning(x, intra_sf);
1614*77c1e3ccSAndroid Build Coastguard Worker
1615*77c1e3ccSAndroid Build Coastguard Worker if (prune_intra_y_mode(this_model_rd, &best_model_rd, top_intra_model_rd,
1616*77c1e3ccSAndroid Build Coastguard Worker intra_sf->top_intra_model_count_allowed,
1617*77c1e3ccSAndroid Build Coastguard Worker model_rd_index_for_pruning))
1618*77c1e3ccSAndroid Build Coastguard Worker continue;
1619*77c1e3ccSAndroid Build Coastguard Worker
1620*77c1e3ccSAndroid Build Coastguard Worker // Builds the actual prediction. The prediction from
1621*77c1e3ccSAndroid Build Coastguard Worker // model_intra_yrd_and_prune was just an estimation that did not take into
1622*77c1e3ccSAndroid Build Coastguard Worker // account the effect of txfm pipeline, so we need to redo it for real
1623*77c1e3ccSAndroid Build Coastguard Worker // here.
1624*77c1e3ccSAndroid Build Coastguard Worker av1_pick_uniform_tx_size_type_yrd(cpi, x, &this_rd_stats, bsize, best_rd);
1625*77c1e3ccSAndroid Build Coastguard Worker this_rate_tokenonly = this_rd_stats.rate;
1626*77c1e3ccSAndroid Build Coastguard Worker this_distortion = this_rd_stats.dist;
1627*77c1e3ccSAndroid Build Coastguard Worker s = this_rd_stats.skip_txfm;
1628*77c1e3ccSAndroid Build Coastguard Worker
1629*77c1e3ccSAndroid Build Coastguard Worker if (this_rate_tokenonly == INT_MAX) continue;
1630*77c1e3ccSAndroid Build Coastguard Worker
1631*77c1e3ccSAndroid Build Coastguard Worker if (!xd->lossless[mbmi->segment_id] && block_signals_txsize(mbmi->bsize)) {
1632*77c1e3ccSAndroid Build Coastguard Worker // av1_pick_uniform_tx_size_type_yrd above includes the cost of the
1633*77c1e3ccSAndroid Build Coastguard Worker // tx_size in the tokenonly rate, but for intra blocks, tx_size is always
1634*77c1e3ccSAndroid Build Coastguard Worker // coded (prediction granularity), so we account for it in the full rate,
1635*77c1e3ccSAndroid Build Coastguard Worker // not the tokenonly rate.
1636*77c1e3ccSAndroid Build Coastguard Worker this_rate_tokenonly -= tx_size_cost(x, bsize, mbmi->tx_size);
1637*77c1e3ccSAndroid Build Coastguard Worker }
1638*77c1e3ccSAndroid Build Coastguard Worker this_rate =
1639*77c1e3ccSAndroid Build Coastguard Worker this_rd_stats.rate +
1640*77c1e3ccSAndroid Build Coastguard Worker intra_mode_info_cost_y(cpi, x, mbmi, bsize, bmode_costs[mbmi->mode], 0);
1641*77c1e3ccSAndroid Build Coastguard Worker this_rd = RDCOST(x->rdmult, this_rate, this_distortion);
1642*77c1e3ccSAndroid Build Coastguard Worker
1643*77c1e3ccSAndroid Build Coastguard Worker // Visual quality adjustment based on recon vs source variance.
1644*77c1e3ccSAndroid Build Coastguard Worker if ((cpi->oxcf.mode == ALLINTRA) && (this_rd != INT64_MAX)) {
1645*77c1e3ccSAndroid Build Coastguard Worker this_rd = (int64_t)(this_rd * intra_rd_variance_factor(cpi, x, bsize));
1646*77c1e3ccSAndroid Build Coastguard Worker }
1647*77c1e3ccSAndroid Build Coastguard Worker
1648*77c1e3ccSAndroid Build Coastguard Worker intra_modes_rd_cost[mbmi->mode][luma_delta_angle + MAX_ANGLE_DELTA + 1] =
1649*77c1e3ccSAndroid Build Coastguard Worker this_rd;
1650*77c1e3ccSAndroid Build Coastguard Worker
1651*77c1e3ccSAndroid Build Coastguard Worker // Collect mode stats for multiwinner mode processing
1652*77c1e3ccSAndroid Build Coastguard Worker const int txfm_search_done = 1;
1653*77c1e3ccSAndroid Build Coastguard Worker store_winner_mode_stats(
1654*77c1e3ccSAndroid Build Coastguard Worker &cpi->common, x, mbmi, NULL, NULL, NULL, 0, NULL, bsize, this_rd,
1655*77c1e3ccSAndroid Build Coastguard Worker cpi->sf.winner_mode_sf.multi_winner_mode_type, txfm_search_done);
1656*77c1e3ccSAndroid Build Coastguard Worker if (this_rd < best_rd) {
1657*77c1e3ccSAndroid Build Coastguard Worker best_mbmi = *mbmi;
1658*77c1e3ccSAndroid Build Coastguard Worker best_rd = this_rd;
1659*77c1e3ccSAndroid Build Coastguard Worker // Setting beat_best_rd flag because current mode rd is better than
1660*77c1e3ccSAndroid Build Coastguard Worker // best_rd passed to this function
1661*77c1e3ccSAndroid Build Coastguard Worker beat_best_rd = 1;
1662*77c1e3ccSAndroid Build Coastguard Worker *rate = this_rate;
1663*77c1e3ccSAndroid Build Coastguard Worker *rate_tokenonly = this_rate_tokenonly;
1664*77c1e3ccSAndroid Build Coastguard Worker *distortion = this_distortion;
1665*77c1e3ccSAndroid Build Coastguard Worker *skippable = s;
1666*77c1e3ccSAndroid Build Coastguard Worker memcpy(ctx->blk_skip, x->txfm_search_info.blk_skip,
1667*77c1e3ccSAndroid Build Coastguard Worker sizeof(x->txfm_search_info.blk_skip[0]) * ctx->num_4x4_blk);
1668*77c1e3ccSAndroid Build Coastguard Worker av1_copy_array(ctx->tx_type_map, xd->tx_type_map, ctx->num_4x4_blk);
1669*77c1e3ccSAndroid Build Coastguard Worker }
1670*77c1e3ccSAndroid Build Coastguard Worker }
1671*77c1e3ccSAndroid Build Coastguard Worker
1672*77c1e3ccSAndroid Build Coastguard Worker // Searches palette
1673*77c1e3ccSAndroid Build Coastguard Worker if (try_palette) {
1674*77c1e3ccSAndroid Build Coastguard Worker av1_rd_pick_palette_intra_sby(
1675*77c1e3ccSAndroid Build Coastguard Worker cpi, x, bsize, bmode_costs[DC_PRED], &best_mbmi, best_palette_color_map,
1676*77c1e3ccSAndroid Build Coastguard Worker &best_rd, rate, rate_tokenonly, distortion, skippable, &beat_best_rd,
1677*77c1e3ccSAndroid Build Coastguard Worker ctx, ctx->blk_skip, ctx->tx_type_map);
1678*77c1e3ccSAndroid Build Coastguard Worker }
1679*77c1e3ccSAndroid Build Coastguard Worker
1680*77c1e3ccSAndroid Build Coastguard Worker // Searches filter_intra
1681*77c1e3ccSAndroid Build Coastguard Worker if (beat_best_rd && av1_filter_intra_allowed_bsize(&cpi->common, bsize)) {
1682*77c1e3ccSAndroid Build Coastguard Worker if (rd_pick_filter_intra_sby(cpi, x, rate, rate_tokenonly, distortion,
1683*77c1e3ccSAndroid Build Coastguard Worker skippable, bsize, bmode_costs[DC_PRED],
1684*77c1e3ccSAndroid Build Coastguard Worker best_mbmi.mode, &best_rd, &best_model_rd,
1685*77c1e3ccSAndroid Build Coastguard Worker ctx)) {
1686*77c1e3ccSAndroid Build Coastguard Worker best_mbmi = *mbmi;
1687*77c1e3ccSAndroid Build Coastguard Worker }
1688*77c1e3ccSAndroid Build Coastguard Worker }
1689*77c1e3ccSAndroid Build Coastguard Worker
1690*77c1e3ccSAndroid Build Coastguard Worker // No mode is identified with less rd value than best_rd passed to this
1691*77c1e3ccSAndroid Build Coastguard Worker // function. In such cases winner mode processing is not necessary and return
1692*77c1e3ccSAndroid Build Coastguard Worker // best_rd as INT64_MAX to indicate best mode is not identified
1693*77c1e3ccSAndroid Build Coastguard Worker if (!beat_best_rd) return INT64_MAX;
1694*77c1e3ccSAndroid Build Coastguard Worker
1695*77c1e3ccSAndroid Build Coastguard Worker // In multi-winner mode processing, perform tx search for few best modes
1696*77c1e3ccSAndroid Build Coastguard Worker // identified during mode evaluation. Winner mode processing uses best tx
1697*77c1e3ccSAndroid Build Coastguard Worker // configuration for tx search.
1698*77c1e3ccSAndroid Build Coastguard Worker if (cpi->sf.winner_mode_sf.multi_winner_mode_type) {
1699*77c1e3ccSAndroid Build Coastguard Worker int best_mode_idx = 0;
1700*77c1e3ccSAndroid Build Coastguard Worker int block_width, block_height;
1701*77c1e3ccSAndroid Build Coastguard Worker uint8_t *color_map_dst = xd->plane[PLANE_TYPE_Y].color_index_map;
1702*77c1e3ccSAndroid Build Coastguard Worker av1_get_block_dimensions(bsize, AOM_PLANE_Y, xd, &block_width,
1703*77c1e3ccSAndroid Build Coastguard Worker &block_height, NULL, NULL);
1704*77c1e3ccSAndroid Build Coastguard Worker
1705*77c1e3ccSAndroid Build Coastguard Worker for (int mode_idx = 0; mode_idx < x->winner_mode_count; mode_idx++) {
1706*77c1e3ccSAndroid Build Coastguard Worker *mbmi = x->winner_mode_stats[mode_idx].mbmi;
1707*77c1e3ccSAndroid Build Coastguard Worker if (is_winner_mode_processing_enabled(cpi, x, mbmi, 0)) {
1708*77c1e3ccSAndroid Build Coastguard Worker // Restore color_map of palette mode before winner mode processing
1709*77c1e3ccSAndroid Build Coastguard Worker if (mbmi->palette_mode_info.palette_size[0] > 0) {
1710*77c1e3ccSAndroid Build Coastguard Worker uint8_t *color_map_src =
1711*77c1e3ccSAndroid Build Coastguard Worker x->winner_mode_stats[mode_idx].color_index_map;
1712*77c1e3ccSAndroid Build Coastguard Worker memcpy(color_map_dst, color_map_src,
1713*77c1e3ccSAndroid Build Coastguard Worker block_width * block_height * sizeof(*color_map_src));
1714*77c1e3ccSAndroid Build Coastguard Worker }
1715*77c1e3ccSAndroid Build Coastguard Worker // Set params for winner mode evaluation
1716*77c1e3ccSAndroid Build Coastguard Worker set_mode_eval_params(cpi, x, WINNER_MODE_EVAL);
1717*77c1e3ccSAndroid Build Coastguard Worker
1718*77c1e3ccSAndroid Build Coastguard Worker // Winner mode processing
1719*77c1e3ccSAndroid Build Coastguard Worker // If previous searches use only the default tx type/no R-D optimization
1720*77c1e3ccSAndroid Build Coastguard Worker // of quantized coeffs, do an extra search for the best tx type/better
1721*77c1e3ccSAndroid Build Coastguard Worker // R-D optimization of quantized coeffs
1722*77c1e3ccSAndroid Build Coastguard Worker if (intra_block_yrd(cpi, x, bsize, bmode_costs, &best_rd, rate,
1723*77c1e3ccSAndroid Build Coastguard Worker rate_tokenonly, distortion, skippable, &best_mbmi,
1724*77c1e3ccSAndroid Build Coastguard Worker ctx))
1725*77c1e3ccSAndroid Build Coastguard Worker best_mode_idx = mode_idx;
1726*77c1e3ccSAndroid Build Coastguard Worker }
1727*77c1e3ccSAndroid Build Coastguard Worker }
1728*77c1e3ccSAndroid Build Coastguard Worker // Copy color_map of palette mode for final winner mode
1729*77c1e3ccSAndroid Build Coastguard Worker if (best_mbmi.palette_mode_info.palette_size[0] > 0) {
1730*77c1e3ccSAndroid Build Coastguard Worker uint8_t *color_map_src =
1731*77c1e3ccSAndroid Build Coastguard Worker x->winner_mode_stats[best_mode_idx].color_index_map;
1732*77c1e3ccSAndroid Build Coastguard Worker memcpy(color_map_dst, color_map_src,
1733*77c1e3ccSAndroid Build Coastguard Worker block_width * block_height * sizeof(*color_map_src));
1734*77c1e3ccSAndroid Build Coastguard Worker }
1735*77c1e3ccSAndroid Build Coastguard Worker } else {
1736*77c1e3ccSAndroid Build Coastguard Worker // If previous searches use only the default tx type/no R-D optimization of
1737*77c1e3ccSAndroid Build Coastguard Worker // quantized coeffs, do an extra search for the best tx type/better R-D
1738*77c1e3ccSAndroid Build Coastguard Worker // optimization of quantized coeffs
1739*77c1e3ccSAndroid Build Coastguard Worker if (is_winner_mode_processing_enabled(cpi, x, mbmi, 0)) {
1740*77c1e3ccSAndroid Build Coastguard Worker // Set params for winner mode evaluation
1741*77c1e3ccSAndroid Build Coastguard Worker set_mode_eval_params(cpi, x, WINNER_MODE_EVAL);
1742*77c1e3ccSAndroid Build Coastguard Worker *mbmi = best_mbmi;
1743*77c1e3ccSAndroid Build Coastguard Worker intra_block_yrd(cpi, x, bsize, bmode_costs, &best_rd, rate,
1744*77c1e3ccSAndroid Build Coastguard Worker rate_tokenonly, distortion, skippable, &best_mbmi, ctx);
1745*77c1e3ccSAndroid Build Coastguard Worker }
1746*77c1e3ccSAndroid Build Coastguard Worker }
1747*77c1e3ccSAndroid Build Coastguard Worker *mbmi = best_mbmi;
1748*77c1e3ccSAndroid Build Coastguard Worker av1_copy_array(xd->tx_type_map, ctx->tx_type_map, ctx->num_4x4_blk);
1749*77c1e3ccSAndroid Build Coastguard Worker return best_rd;
1750*77c1e3ccSAndroid Build Coastguard Worker }
1751