1 /* 2 * Copyright 2020 The WebRTC Project Authors. All rights reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_MANAGER_H_ 12 #define VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_MANAGER_H_ 13 14 #include <atomic> 15 #include <map> 16 #include <memory> 17 #include <string> 18 #include <unordered_map> 19 #include <utility> 20 #include <vector> 21 22 #include "absl/types/optional.h" 23 #include "api/adaptation/resource.h" 24 #include "api/field_trials_view.h" 25 #include "api/rtp_parameters.h" 26 #include "api/scoped_refptr.h" 27 #include "api/task_queue/task_queue_base.h" 28 #include "api/video/video_adaptation_counters.h" 29 #include "api/video/video_adaptation_reason.h" 30 #include "api/video/video_frame.h" 31 #include "api/video/video_source_interface.h" 32 #include "api/video_codecs/video_codec.h" 33 #include "api/video_codecs/video_encoder.h" 34 #include "call/adaptation/resource_adaptation_processor_interface.h" 35 #include "call/adaptation/video_stream_adapter.h" 36 #include "call/adaptation/video_stream_input_state_provider.h" 37 #include "rtc_base/experiments/quality_scaler_settings.h" 38 #include "rtc_base/ref_count.h" 39 #include "rtc_base/strings/string_builder.h" 40 #include "rtc_base/synchronization/mutex.h" 41 #include "rtc_base/thread_annotations.h" 42 #include "system_wrappers/include/clock.h" 43 #include "video/adaptation/balanced_constraint.h" 44 #include "video/adaptation/bandwidth_quality_scaler_resource.h" 45 #include "video/adaptation/bitrate_constraint.h" 46 #include "video/adaptation/encode_usage_resource.h" 47 #include "video/adaptation/overuse_frame_detector.h" 48 #include "video/adaptation/pixel_limit_resource.h" 49 #include "video/adaptation/quality_rampup_experiment_helper.h" 50 #include "video/adaptation/quality_scaler_resource.h" 51 #include "video/adaptation/video_stream_encoder_resource.h" 52 #include "video/config/video_encoder_config.h" 53 #include "video/video_stream_encoder_observer.h" 54 55 namespace webrtc { 56 57 // The assumed input frame size if we have not yet received a frame. 58 // TODO(hbos): This is 144p - why are we assuming super low quality? Seems like 59 // a bad heuristic. 60 extern const int kDefaultInputPixelsWidth; 61 extern const int kDefaultInputPixelsHeight; 62 63 // Owns adaptation-related Resources pertaining to a single VideoStreamEncoder 64 // and passes on the relevant input from the encoder to the resources. The 65 // resources provide resource usage states to the ResourceAdaptationProcessor 66 // which is responsible for reconfiguring streams in order not to overuse 67 // resources. 68 // 69 // The manager is also involved with various mitigations not part of the 70 // ResourceAdaptationProcessor code such as the initial frame dropping. 71 class VideoStreamEncoderResourceManager 72 : public VideoSourceRestrictionsListener, 73 public ResourceLimitationsListener, 74 public QualityRampUpExperimentListener { 75 public: 76 VideoStreamEncoderResourceManager( 77 VideoStreamInputStateProvider* input_state_provider, 78 VideoStreamEncoderObserver* encoder_stats_observer, 79 Clock* clock, 80 bool experiment_cpu_load_estimator, 81 std::unique_ptr<OveruseFrameDetector> overuse_detector, 82 DegradationPreferenceProvider* degradation_preference_provider, 83 const FieldTrialsView& field_trials); 84 ~VideoStreamEncoderResourceManager() override; 85 86 void Initialize(TaskQueueBase* encoder_queue); 87 void SetAdaptationProcessor( 88 ResourceAdaptationProcessorInterface* adaptation_processor, 89 VideoStreamAdapter* stream_adapter); 90 91 // TODO(https://crbug.com/webrtc/11563): The degradation preference is a 92 // setting of the Processor, it does not belong to the Manager - can we get 93 // rid of this? 94 void SetDegradationPreferences(DegradationPreference degradation_preference); 95 DegradationPreference degradation_preference() const; 96 97 void ConfigureEncodeUsageResource(); 98 // Initializes the pixel limit resource if the "WebRTC-PixelLimitResource" 99 // field trial is enabled. This can be used for testing. 100 void MaybeInitializePixelLimitResource(); 101 // Stops the encode usage and quality scaler resources if not already stopped. 102 // If the pixel limit resource was created it is also stopped and nulled. 103 void StopManagedResources(); 104 105 // Settings that affect the VideoStreamEncoder-specific resources. 106 void SetEncoderSettings(EncoderSettings encoder_settings); 107 void SetStartBitrate(DataRate start_bitrate); 108 void SetTargetBitrate(DataRate target_bitrate); 109 void SetEncoderRates( 110 const VideoEncoder::RateControlParameters& encoder_rates); 111 // TODO(https://crbug.com/webrtc/11338): This can be made private if we 112 // configure on SetDegredationPreference and SetEncoderSettings. 113 void ConfigureQualityScaler(const VideoEncoder::EncoderInfo& encoder_info); 114 void ConfigureBandwidthQualityScaler( 115 const VideoEncoder::EncoderInfo& encoder_info); 116 117 // Methods corresponding to different points in the encoding pipeline. 118 void OnFrameDroppedDueToSize(); 119 void OnMaybeEncodeFrame(); 120 void OnEncodeStarted(const VideoFrame& cropped_frame, 121 int64_t time_when_first_seen_us); 122 void OnEncodeCompleted(const EncodedImage& encoded_image, 123 int64_t time_sent_in_us, 124 absl::optional<int> encode_duration_us, 125 DataSize frame_size); 126 void OnFrameDropped(EncodedImageCallback::DropReason reason); 127 128 // Resources need to be mapped to an AdaptReason (kCpu or kQuality) in order 129 // to update legacy getStats(). 130 void AddResource(rtc::scoped_refptr<Resource> resource, 131 VideoAdaptationReason reason); 132 void RemoveResource(rtc::scoped_refptr<Resource> resource); 133 std::vector<AdaptationConstraint*> AdaptationConstraints() const; 134 // If true, the VideoStreamEncoder should execute its logic to maybe drop 135 // frames based on size and bitrate. 136 bool DropInitialFrames() const; 137 absl::optional<uint32_t> SingleActiveStreamPixels() const; 138 absl::optional<uint32_t> UseBandwidthAllocationBps() const; 139 140 // VideoSourceRestrictionsListener implementation. 141 // Updates `video_source_restrictions_`. 142 void OnVideoSourceRestrictionsUpdated( 143 VideoSourceRestrictions restrictions, 144 const VideoAdaptationCounters& adaptation_counters, 145 rtc::scoped_refptr<Resource> reason, 146 const VideoSourceRestrictions& unfiltered_restrictions) override; 147 void OnResourceLimitationChanged( 148 rtc::scoped_refptr<Resource> resource, 149 const std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters>& 150 resource_limitations) override; 151 152 // QualityRampUpExperimentListener implementation. 153 void OnQualityRampUp() override; 154 155 static bool IsSimulcastOrMultipleSpatialLayers( 156 const VideoEncoderConfig& encoder_config); 157 158 private: 159 class InitialFrameDropper; 160 161 VideoAdaptationReason GetReasonFromResource( 162 rtc::scoped_refptr<Resource> resource) const; 163 164 CpuOveruseOptions GetCpuOveruseOptions() const; 165 int LastFrameSizeOrDefault() const; 166 167 // Calculates an up-to-date value of the target frame rate and informs the 168 // `encode_usage_resource_` of the new value. 169 void MaybeUpdateTargetFrameRate(); 170 171 // Use nullopt to disable quality scaling. 172 void UpdateQualityScalerSettings( 173 absl::optional<VideoEncoder::QpThresholds> qp_thresholds); 174 175 void UpdateBandwidthQualityScalerSettings( 176 bool bandwidth_quality_scaling_allowed, 177 const std::vector<VideoEncoder::ResolutionBitrateLimits>& 178 resolution_bitrate_limits); 179 180 void UpdateStatsAdaptationSettings() const; 181 182 static std::string ActiveCountsToString( 183 const std::map<VideoAdaptationReason, VideoAdaptationCounters>& 184 active_counts); 185 186 const FieldTrialsView& field_trials_; 187 DegradationPreferenceProvider* const degradation_preference_provider_; 188 std::unique_ptr<BitrateConstraint> bitrate_constraint_ 189 RTC_GUARDED_BY(encoder_queue_); 190 const std::unique_ptr<BalancedConstraint> balanced_constraint_ 191 RTC_GUARDED_BY(encoder_queue_); 192 const rtc::scoped_refptr<EncodeUsageResource> encode_usage_resource_; 193 const rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource_; 194 rtc::scoped_refptr<PixelLimitResource> pixel_limit_resource_; 195 const rtc::scoped_refptr<BandwidthQualityScalerResource> 196 bandwidth_quality_scaler_resource_; 197 198 TaskQueueBase* encoder_queue_; 199 VideoStreamInputStateProvider* const input_state_provider_ 200 RTC_GUARDED_BY(encoder_queue_); 201 ResourceAdaptationProcessorInterface* adaptation_processor_; 202 VideoStreamAdapter* stream_adapter_ RTC_GUARDED_BY(encoder_queue_); 203 // Thread-safe. 204 VideoStreamEncoderObserver* const encoder_stats_observer_; 205 206 DegradationPreference degradation_preference_ RTC_GUARDED_BY(encoder_queue_); 207 VideoSourceRestrictions video_source_restrictions_ 208 RTC_GUARDED_BY(encoder_queue_); 209 210 VideoAdaptationCounters current_adaptation_counters_ 211 RTC_GUARDED_BY(encoder_queue_); 212 213 const BalancedDegradationSettings balanced_settings_; 214 Clock* clock_ RTC_GUARDED_BY(encoder_queue_); 215 const bool experiment_cpu_load_estimator_ RTC_GUARDED_BY(encoder_queue_); 216 const std::unique_ptr<InitialFrameDropper> initial_frame_dropper_ 217 RTC_GUARDED_BY(encoder_queue_); 218 const bool quality_scaling_experiment_enabled_ RTC_GUARDED_BY(encoder_queue_); 219 const bool pixel_limit_resource_experiment_enabled_ 220 RTC_GUARDED_BY(encoder_queue_); 221 absl::optional<uint32_t> encoder_target_bitrate_bps_ 222 RTC_GUARDED_BY(encoder_queue_); 223 absl::optional<VideoEncoder::RateControlParameters> encoder_rates_ 224 RTC_GUARDED_BY(encoder_queue_); 225 std::unique_ptr<QualityRampUpExperimentHelper> quality_rampup_experiment_ 226 RTC_GUARDED_BY(encoder_queue_); 227 absl::optional<EncoderSettings> encoder_settings_ 228 RTC_GUARDED_BY(encoder_queue_); 229 230 // Ties a resource to a reason for statistical reporting. This AdaptReason is 231 // also used by this module to make decisions about how to adapt up/down. 232 std::map<rtc::scoped_refptr<Resource>, VideoAdaptationReason> resources_ 233 RTC_GUARDED_BY(encoder_queue_); 234 }; 235 236 } // namespace webrtc 237 238 #endif // VIDEO_ADAPTATION_VIDEO_STREAM_ENCODER_RESOURCE_MANAGER_H_ 239