1 /*
2 * Copyright (c) 2021, Alliance for Open Media. All rights reserved.
3 *
4 * This source code is subject to the terms of the BSD 2 Clause License and
5 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
6 * was not distributed with this source code in the LICENSE file, you can
7 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
8 * Media Patent License 1.0 was not distributed with this source code in the
9 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
10 */
11
12 #include "av1/ratectrl_rtc.h"
13
14 #include <memory>
15 #include <new>
16
17 #include "aom/aomcx.h"
18 #include "aom/aom_encoder.h"
19 #include "aom_mem/aom_mem.h"
20 #include "aom_dsp/aom_dsp_common.h"
21 #include "av1/encoder/encoder.h"
22 #include "av1/encoder/encoder_utils.h"
23 #include "av1/encoder/pickcdef.h"
24 #include "av1/encoder/picklpf.h"
25 #include "av1/encoder/ratectrl.h"
26 #include "av1/encoder/rc_utils.h"
27 #include "av1/encoder/svc_layercontext.h"
28
29 namespace aom {
30
AV1RateControlRtcConfig()31 AV1RateControlRtcConfig::AV1RateControlRtcConfig() {
32 width = 1280;
33 height = 720;
34 max_quantizer = 63;
35 min_quantizer = 2;
36 target_bandwidth = 1000;
37 buf_initial_sz = 600;
38 buf_optimal_sz = 600;
39 buf_sz = 1000;
40 undershoot_pct = overshoot_pct = 50;
41 max_intra_bitrate_pct = 50;
42 max_inter_bitrate_pct = 0;
43 frame_drop_thresh = 0;
44 max_consec_drop_ms = 0;
45 framerate = 30.0;
46 ss_number_layers = 1;
47 ts_number_layers = 1;
48 aq_mode = 0;
49 layer_target_bitrate[0] = static_cast<int>(target_bandwidth);
50 ts_rate_decimator[0] = 1;
51 av1_zero(max_quantizers);
52 av1_zero(min_quantizers);
53 av1_zero(scaling_factor_den);
54 av1_zero(scaling_factor_num);
55 av1_zero(layer_target_bitrate);
56 av1_zero(ts_rate_decimator);
57 scaling_factor_num[0] = 1;
58 scaling_factor_den[0] = 1;
59 max_quantizers[0] = max_quantizer;
60 min_quantizers[0] = min_quantizer;
61 }
62
Create(const AV1RateControlRtcConfig & cfg)63 std::unique_ptr<AV1RateControlRTC> AV1RateControlRTC::Create(
64 const AV1RateControlRtcConfig &cfg) {
65 std::unique_ptr<AV1RateControlRTC> rc_api(new (std::nothrow)
66 AV1RateControlRTC());
67 if (!rc_api) return nullptr;
68 rc_api->cpi_ = static_cast<AV1_COMP *>(aom_memalign(32, sizeof(*cpi_)));
69 if (!rc_api->cpi_) return nullptr;
70 av1_zero(*rc_api->cpi_);
71 rc_api->cpi_->ppi =
72 static_cast<AV1_PRIMARY *>(aom_memalign(32, sizeof(AV1_PRIMARY)));
73 if (!rc_api->cpi_->ppi) return nullptr;
74 av1_zero(*rc_api->cpi_->ppi);
75 rc_api->cpi_->common.seq_params = &rc_api->cpi_->ppi->seq_params;
76 av1_zero(*rc_api->cpi_->common.seq_params);
77 if (!rc_api->InitRateControl(cfg)) return nullptr;
78 if (cfg.aq_mode) {
79 AV1_COMP *const cpi = rc_api->cpi_;
80 cpi->enc_seg.map = static_cast<uint8_t *>(aom_calloc(
81 cpi->common.mi_params.mi_rows * cpi->common.mi_params.mi_cols,
82 sizeof(*cpi->enc_seg.map)));
83 if (!cpi->enc_seg.map) return nullptr;
84 cpi->cyclic_refresh = av1_cyclic_refresh_alloc(
85 cpi->common.mi_params.mi_rows, cpi->common.mi_params.mi_cols);
86 if (!cpi->cyclic_refresh) return nullptr;
87 }
88 return rc_api;
89 }
90
~AV1RateControlRTC()91 AV1RateControlRTC::~AV1RateControlRTC() {
92 if (cpi_) {
93 if (cpi_->svc.number_spatial_layers > 1 ||
94 cpi_->svc.number_temporal_layers > 1) {
95 for (int sl = 0; sl < cpi_->svc.number_spatial_layers; sl++) {
96 for (int tl = 0; tl < cpi_->svc.number_temporal_layers; tl++) {
97 int layer =
98 LAYER_IDS_TO_IDX(sl, tl, cpi_->svc.number_temporal_layers);
99 LAYER_CONTEXT *const lc = &cpi_->svc.layer_context[layer];
100 aom_free(lc->map);
101 }
102 }
103 }
104 aom_free(cpi_->svc.layer_context);
105 cpi_->svc.layer_context = nullptr;
106
107 if (cpi_->oxcf.q_cfg.aq_mode == CYCLIC_REFRESH_AQ) {
108 aom_free(cpi_->enc_seg.map);
109 cpi_->enc_seg.map = nullptr;
110 av1_cyclic_refresh_free(cpi_->cyclic_refresh);
111 }
112 aom_free(cpi_->ppi);
113 aom_free(cpi_);
114 }
115 }
116
InitRateControl(const AV1RateControlRtcConfig & rc_cfg)117 bool AV1RateControlRTC::InitRateControl(const AV1RateControlRtcConfig &rc_cfg) {
118 AV1_COMMON *cm = &cpi_->common;
119 AV1EncoderConfig *oxcf = &cpi_->oxcf;
120 RATE_CONTROL *const rc = &cpi_->rc;
121 cm->seq_params->profile = PROFILE_0;
122 cm->seq_params->bit_depth = AOM_BITS_8;
123 cm->show_frame = 1;
124 oxcf->profile = cm->seq_params->profile;
125 oxcf->mode = REALTIME;
126 oxcf->rc_cfg.mode = AOM_CBR;
127 oxcf->pass = AOM_RC_ONE_PASS;
128 oxcf->q_cfg.aq_mode = rc_cfg.aq_mode ? CYCLIC_REFRESH_AQ : NO_AQ;
129 oxcf->tune_cfg.content = AOM_CONTENT_DEFAULT;
130 oxcf->rc_cfg.drop_frames_water_mark = rc_cfg.frame_drop_thresh;
131 if (rc_cfg.max_consec_drop_ms > 0) {
132 rc->max_consec_drop = saturate_cast_double_to_int(
133 ceil(cpi_->framerate * rc_cfg.max_consec_drop_ms / 1000));
134 }
135 cpi_->svc.framedrop_mode = AOM_FULL_SUPERFRAME_DROP;
136 oxcf->tool_cfg.bit_depth = AOM_BITS_8;
137 oxcf->tool_cfg.superblock_size = AOM_SUPERBLOCK_SIZE_DYNAMIC;
138 oxcf->algo_cfg.loopfilter_control = LOOPFILTER_ALL;
139 cm->current_frame.frame_number = 0;
140 cpi_->ppi->p_rc.kf_boost = DEFAULT_KF_BOOST_RT;
141 for (auto &lvl_idx : oxcf->target_seq_level_idx) lvl_idx = SEQ_LEVEL_MAX;
142
143 memcpy(cpi_->ppi->level_params.target_seq_level_idx,
144 oxcf->target_seq_level_idx, sizeof(oxcf->target_seq_level_idx));
145 if (!UpdateRateControl(rc_cfg)) return false;
146 set_sb_size(cm->seq_params,
147 av1_select_sb_size(oxcf, cm->width, cm->height,
148 cpi_->svc.number_spatial_layers));
149 cpi_->ppi->use_svc = cpi_->svc.number_spatial_layers > 1 ||
150 cpi_->svc.number_temporal_layers > 1;
151 av1_primary_rc_init(oxcf, &cpi_->ppi->p_rc);
152 rc->rc_1_frame = 0;
153 rc->rc_2_frame = 0;
154 av1_rc_init_minq_luts();
155 av1_rc_init(oxcf, rc);
156 // Enable external rate control.
157 cpi_->rc.rtc_external_ratectrl = 1;
158 cpi_->sf.rt_sf.use_nonrd_pick_mode = 1;
159 return true;
160 }
161
UpdateRateControl(const AV1RateControlRtcConfig & rc_cfg)162 bool AV1RateControlRTC::UpdateRateControl(
163 const AV1RateControlRtcConfig &rc_cfg) {
164 if (rc_cfg.ss_number_layers < 1 ||
165 rc_cfg.ss_number_layers > AOM_MAX_SS_LAYERS ||
166 rc_cfg.ts_number_layers < 1 ||
167 rc_cfg.ts_number_layers > AOM_MAX_TS_LAYERS) {
168 return false;
169 }
170 const int num_layers = rc_cfg.ss_number_layers * rc_cfg.ts_number_layers;
171 if (num_layers > 1 && !av1_alloc_layer_context(cpi_, num_layers)) {
172 return false;
173 }
174 AV1_COMMON *cm = &cpi_->common;
175 AV1EncoderConfig *oxcf = &cpi_->oxcf;
176 RATE_CONTROL *const rc = &cpi_->rc;
177 initial_width_ = rc_cfg.width;
178 initial_height_ = rc_cfg.height;
179 cm->width = rc_cfg.width;
180 cm->height = rc_cfg.height;
181 oxcf->frm_dim_cfg.width = rc_cfg.width;
182 oxcf->frm_dim_cfg.height = rc_cfg.height;
183 oxcf->rc_cfg.worst_allowed_q = av1_quantizer_to_qindex(rc_cfg.max_quantizer);
184 oxcf->rc_cfg.best_allowed_q = av1_quantizer_to_qindex(rc_cfg.min_quantizer);
185 rc->worst_quality = oxcf->rc_cfg.worst_allowed_q;
186 rc->best_quality = oxcf->rc_cfg.best_allowed_q;
187 oxcf->input_cfg.init_framerate = rc_cfg.framerate;
188 oxcf->rc_cfg.target_bandwidth = rc_cfg.target_bandwidth > INT64_MAX / 1000
189 ? INT64_MAX
190 : 1000 * rc_cfg.target_bandwidth;
191 oxcf->rc_cfg.starting_buffer_level_ms = rc_cfg.buf_initial_sz;
192 oxcf->rc_cfg.optimal_buffer_level_ms = rc_cfg.buf_optimal_sz;
193 oxcf->rc_cfg.maximum_buffer_size_ms = rc_cfg.buf_sz;
194 oxcf->rc_cfg.under_shoot_pct = rc_cfg.undershoot_pct;
195 oxcf->rc_cfg.over_shoot_pct = rc_cfg.overshoot_pct;
196 oxcf->rc_cfg.drop_frames_water_mark = rc_cfg.frame_drop_thresh;
197 if (rc_cfg.max_consec_drop_ms > 0) {
198 rc->max_consec_drop = saturate_cast_double_to_int(
199 ceil(cpi_->framerate * rc_cfg.max_consec_drop_ms / 1000));
200 }
201 oxcf->rc_cfg.max_intra_bitrate_pct = rc_cfg.max_intra_bitrate_pct;
202 oxcf->rc_cfg.max_inter_bitrate_pct = rc_cfg.max_inter_bitrate_pct;
203 cpi_->framerate = rc_cfg.framerate;
204 if (rc_cfg.is_screen) {
205 cpi_->oxcf.tune_cfg.content = AOM_CONTENT_SCREEN;
206 cpi_->is_screen_content_type = 1;
207 }
208 cpi_->svc.number_spatial_layers = rc_cfg.ss_number_layers;
209 cpi_->svc.number_temporal_layers = rc_cfg.ts_number_layers;
210 set_primary_rc_buffer_sizes(oxcf, cpi_->ppi);
211 enc_set_mb_mi(&cm->mi_params, cm->width, cm->height, BLOCK_8X8);
212 av1_new_framerate(cpi_, cpi_->framerate);
213 if (cpi_->svc.number_temporal_layers > 1 ||
214 cpi_->svc.number_spatial_layers > 1) {
215 int64_t target_bandwidth_svc = 0;
216 for (int sl = 0; sl < cpi_->svc.number_spatial_layers; ++sl) {
217 for (int tl = 0; tl < cpi_->svc.number_temporal_layers; ++tl) {
218 const int layer =
219 LAYER_IDS_TO_IDX(sl, tl, cpi_->svc.number_temporal_layers);
220 LAYER_CONTEXT *lc = &cpi_->svc.layer_context[layer];
221 RATE_CONTROL *const lrc = &lc->rc;
222 lc->layer_target_bitrate = 1000 * rc_cfg.layer_target_bitrate[layer];
223 lc->max_q = rc_cfg.max_quantizers[layer];
224 lc->min_q = rc_cfg.min_quantizers[layer];
225 lrc->worst_quality =
226 av1_quantizer_to_qindex(rc_cfg.max_quantizers[layer]);
227 lrc->best_quality =
228 av1_quantizer_to_qindex(rc_cfg.min_quantizers[layer]);
229 lc->scaling_factor_num = rc_cfg.scaling_factor_num[sl];
230 lc->scaling_factor_den = rc_cfg.scaling_factor_den[sl];
231 lc->framerate_factor = rc_cfg.ts_rate_decimator[tl];
232 if (tl == cpi_->svc.number_temporal_layers - 1)
233 target_bandwidth_svc += lc->layer_target_bitrate;
234 }
235 }
236
237 if (cm->current_frame.frame_number == 0) av1_init_layer_context(cpi_);
238 // This is needed to initialize external RC flag in layer context structure.
239 cpi_->rc.rtc_external_ratectrl = 1;
240 av1_update_layer_context_change_config(cpi_, target_bandwidth_svc);
241 }
242 check_reset_rc_flag(cpi_);
243 return true;
244 }
245
ComputeQP(const AV1FrameParamsRTC & frame_params)246 FrameDropDecision AV1RateControlRTC::ComputeQP(
247 const AV1FrameParamsRTC &frame_params) {
248 AV1_COMMON *const cm = &cpi_->common;
249 int width, height;
250 GF_GROUP *const gf_group = &cpi_->ppi->gf_group;
251 cpi_->svc.spatial_layer_id = frame_params.spatial_layer_id;
252 cpi_->svc.temporal_layer_id = frame_params.temporal_layer_id;
253 if (cpi_->svc.number_spatial_layers > 1) {
254 const int layer = LAYER_IDS_TO_IDX(cpi_->svc.spatial_layer_id,
255 cpi_->svc.temporal_layer_id,
256 cpi_->svc.number_temporal_layers);
257 LAYER_CONTEXT *lc = &cpi_->svc.layer_context[layer];
258 av1_get_layer_resolution(initial_width_, initial_height_,
259 lc->scaling_factor_num, lc->scaling_factor_den,
260 &width, &height);
261 cm->width = width;
262 cm->height = height;
263 }
264 enc_set_mb_mi(&cm->mi_params, cm->width, cm->height, BLOCK_8X8);
265 cm->current_frame.frame_type = frame_params.frame_type;
266 cpi_->refresh_frame.golden_frame =
267 (cm->current_frame.frame_type == KEY_FRAME) ? 1 : 0;
268 cpi_->sf.rt_sf.use_nonrd_pick_mode = 1;
269
270 if (frame_params.frame_type == kKeyFrame) {
271 gf_group->update_type[cpi_->gf_frame_index] = KF_UPDATE;
272 gf_group->frame_type[cpi_->gf_frame_index] = KEY_FRAME;
273 gf_group->refbuf_state[cpi_->gf_frame_index] = REFBUF_RESET;
274 if (cpi_->ppi->use_svc) {
275 const int layer = LAYER_IDS_TO_IDX(cpi_->svc.spatial_layer_id,
276 cpi_->svc.temporal_layer_id,
277 cpi_->svc.number_temporal_layers);
278 if (cm->current_frame.frame_number > 0)
279 av1_svc_reset_temporal_layers(cpi_, 1);
280 cpi_->svc.layer_context[layer].is_key_frame = 1;
281 }
282 } else {
283 gf_group->update_type[cpi_->gf_frame_index] = LF_UPDATE;
284 gf_group->frame_type[cpi_->gf_frame_index] = INTER_FRAME;
285 gf_group->refbuf_state[cpi_->gf_frame_index] = REFBUF_UPDATE;
286 if (cpi_->ppi->use_svc) {
287 const int layer = LAYER_IDS_TO_IDX(cpi_->svc.spatial_layer_id,
288 cpi_->svc.temporal_layer_id,
289 cpi_->svc.number_temporal_layers);
290 cpi_->svc.layer_context[layer].is_key_frame = 0;
291 }
292 }
293 if (cpi_->svc.number_spatial_layers > 1 ||
294 cpi_->svc.number_temporal_layers > 1) {
295 av1_update_temporal_layer_framerate(cpi_);
296 av1_restore_layer_context(cpi_);
297 }
298 int target = 0;
299 if (cpi_->oxcf.rc_cfg.mode == AOM_CBR) {
300 if (cpi_->oxcf.q_cfg.aq_mode == CYCLIC_REFRESH_AQ)
301 av1_cyclic_refresh_update_parameters(cpi_);
302 if (frame_is_intra_only(cm)) {
303 target = av1_calc_iframe_target_size_one_pass_cbr(cpi_);
304 cpi_->common.current_frame.frame_number = 0;
305 } else {
306 target = av1_calc_pframe_target_size_one_pass_cbr(
307 cpi_, gf_group->update_type[cpi_->gf_frame_index]);
308 }
309 }
310 av1_rc_set_frame_target(cpi_, target, cm->width, cm->height);
311 // Always drop for spatial enhancement layer if layer bandwidth is 0.
312 // Otherwise check for frame-dropping based on buffer level in
313 // av1_rc_drop_frame().
314 if ((cpi_->svc.spatial_layer_id > 0 &&
315 cpi_->oxcf.rc_cfg.target_bandwidth == 0) ||
316 av1_rc_drop_frame(cpi_)) {
317 cpi_->is_dropped_frame = true;
318 av1_rc_postencode_update_drop_frame(cpi_);
319 if (cpi_->svc.spatial_layer_id == cpi_->svc.number_spatial_layers - 1)
320 cpi_->rc.frames_since_key++;
321 cpi_->frame_index_set.show_frame_count++;
322 cpi_->common.current_frame.frame_number++;
323 return FrameDropDecision::kDrop;
324 }
325 int bottom_index = 0, top_index = 0;
326 cpi_->common.quant_params.base_qindex =
327 av1_rc_pick_q_and_bounds(cpi_, cm->width, cm->height,
328 cpi_->gf_frame_index, &bottom_index, &top_index);
329 if (cpi_->oxcf.q_cfg.aq_mode == CYCLIC_REFRESH_AQ)
330 av1_cyclic_refresh_setup(cpi_);
331 return FrameDropDecision::kOk;
332 }
333
GetQP() const334 int AV1RateControlRTC::GetQP() const {
335 return cpi_->common.quant_params.base_qindex;
336 }
337
GetLoopfilterLevel() const338 AV1LoopfilterLevel AV1RateControlRTC::GetLoopfilterLevel() const {
339 av1_pick_filter_level(nullptr, cpi_, LPF_PICK_FROM_Q);
340 AV1LoopfilterLevel lpf_level;
341 lpf_level.filter_level[0] = cpi_->common.lf.filter_level[0];
342 lpf_level.filter_level[1] = cpi_->common.lf.filter_level[1];
343 lpf_level.filter_level_u = cpi_->common.lf.filter_level_u;
344 lpf_level.filter_level_v = cpi_->common.lf.filter_level_v;
345
346 return lpf_level;
347 }
348
GetCdefInfo() const349 AV1CdefInfo AV1RateControlRTC::GetCdefInfo() const {
350 av1_pick_cdef_from_qp(&cpi_->common, 0, 0);
351 AV1CdefInfo cdef_level;
352 cdef_level.cdef_strength_y = cpi_->common.cdef_info.cdef_strengths[0];
353 cdef_level.cdef_strength_uv = cpi_->common.cdef_info.cdef_uv_strengths[0];
354 cdef_level.damping = cpi_->common.cdef_info.cdef_damping;
355
356 return cdef_level;
357 }
358
GetSegmentationData(AV1SegmentationData * segmentation_data) const359 bool AV1RateControlRTC::GetSegmentationData(
360 AV1SegmentationData *segmentation_data) const {
361 if (cpi_->oxcf.q_cfg.aq_mode == 0) {
362 return false;
363 }
364 segmentation_data->segmentation_map = cpi_->enc_seg.map;
365 segmentation_data->segmentation_map_size =
366 cpi_->common.mi_params.mi_rows * cpi_->common.mi_params.mi_cols;
367 segmentation_data->delta_q = cpi_->cyclic_refresh->qindex_delta;
368 segmentation_data->delta_q_size = 3u;
369 return true;
370 }
371
PostEncodeUpdate(uint64_t encoded_frame_size)372 void AV1RateControlRTC::PostEncodeUpdate(uint64_t encoded_frame_size) {
373 cpi_->common.current_frame.frame_number++;
374 av1_rc_postencode_update(cpi_, encoded_frame_size);
375 if (cpi_->svc.spatial_layer_id == cpi_->svc.number_spatial_layers - 1) {
376 cpi_->svc.prev_number_spatial_layers = cpi_->svc.number_spatial_layers;
377 cpi_->rc.frames_since_key++;
378 }
379 if (cpi_->svc.number_spatial_layers > 1 ||
380 cpi_->svc.number_temporal_layers > 1)
381 av1_save_layer_context(cpi_);
382 }
383
384 } // namespace aom
385