1 /* 2 * Copyright (c) 2020 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef VIDEO_ADAPTATION_OVERUSE_FRAME_DETECTOR_H_ 12 #define VIDEO_ADAPTATION_OVERUSE_FRAME_DETECTOR_H_ 13 14 #include <list> 15 #include <memory> 16 17 #include "absl/types/optional.h" 18 #include "api/field_trials_view.h" 19 #include "api/sequence_checker.h" 20 #include "api/task_queue/task_queue_base.h" 21 #include "rtc_base/experiments/field_trial_parser.h" 22 #include "rtc_base/numerics/exp_filter.h" 23 #include "rtc_base/system/no_unique_address.h" 24 #include "rtc_base/task_utils/repeating_task.h" 25 #include "rtc_base/thread_annotations.h" 26 #include "video/video_stream_encoder_observer.h" 27 28 namespace webrtc { 29 30 class VideoFrame; 31 32 struct CpuOveruseOptions { 33 explicit CpuOveruseOptions(const FieldTrialsView& field_trials); 34 35 int low_encode_usage_threshold_percent; // Threshold for triggering underuse. 36 int high_encode_usage_threshold_percent; // Threshold for triggering overuse. 37 // General settings. 38 int frame_timeout_interval_ms; // The maximum allowed interval between two 39 // frames before resetting estimations. 40 int min_frame_samples; // The minimum number of frames required. 41 int min_process_count; // The number of initial process times required before 42 // triggering an overuse/underuse. 43 int high_threshold_consecutive_count; // The number of consecutive checks 44 // above the high threshold before 45 // triggering an overuse. 46 // New estimator enabled if this is set non-zero. 47 int filter_time_ms; // Time constant for averaging 48 }; 49 50 class OveruseFrameDetectorObserverInterface { 51 public: 52 // Called to signal that we can handle larger or more frequent frames. 53 virtual void AdaptUp() = 0; 54 // Called to signal that the source should reduce the resolution or framerate. 55 virtual void AdaptDown() = 0; 56 57 protected: ~OveruseFrameDetectorObserverInterface()58 virtual ~OveruseFrameDetectorObserverInterface() {} 59 }; 60 61 // Use to detect system overuse based on the send-side processing time of 62 // incoming frames. All methods must be called on a single task queue but it can 63 // be created and destroyed on an arbitrary thread. 64 // OveruseFrameDetector::StartCheckForOveruse must be called to periodically 65 // check for overuse. 66 class OveruseFrameDetector { 67 public: 68 explicit OveruseFrameDetector(CpuOveruseMetricsObserver* metrics_observer, 69 const FieldTrialsView& field_trials); 70 virtual ~OveruseFrameDetector(); 71 72 OveruseFrameDetector(const OveruseFrameDetector&) = delete; 73 OveruseFrameDetector& operator=(const OveruseFrameDetector&) = delete; 74 75 // Start to periodically check for overuse. 76 void StartCheckForOveruse( 77 TaskQueueBase* task_queue_base, 78 const CpuOveruseOptions& options, 79 OveruseFrameDetectorObserverInterface* overuse_observer); 80 81 // StopCheckForOveruse must be called before destruction if 82 // StartCheckForOveruse has been called. 83 void StopCheckForOveruse(); 84 85 // Defines the current maximum framerate targeted by the capturer. This is 86 // used to make sure the encode usage percent doesn't drop unduly if the 87 // capturer has quiet periods (for instance caused by screen capturers with 88 // variable capture rate depending on content updates), otherwise we might 89 // experience adaptation toggling. 90 virtual void OnTargetFramerateUpdated(int framerate_fps); 91 92 // Called for each captured frame. 93 void FrameCaptured(const VideoFrame& frame, int64_t time_when_first_seen_us); 94 95 // Called for each sent frame. 96 void FrameSent(uint32_t timestamp, 97 int64_t time_sent_in_us, 98 int64_t capture_time_us, 99 absl::optional<int> encode_duration_us); 100 101 // Interface for cpu load estimation. Intended for internal use only. 102 class ProcessingUsage { 103 public: 104 virtual void Reset() = 0; 105 virtual void SetMaxSampleDiffMs(float diff_ms) = 0; 106 virtual void FrameCaptured(const VideoFrame& frame, 107 int64_t time_when_first_seen_us, 108 int64_t last_capture_time_us) = 0; 109 // Returns encode_time in us, if there's a new measurement. 110 virtual absl::optional<int> FrameSent( 111 // These two argument used by old estimator. 112 uint32_t timestamp, 113 int64_t time_sent_in_us, 114 // And these two by the new estimator. 115 int64_t capture_time_us, 116 absl::optional<int> encode_duration_us) = 0; 117 118 virtual int Value() = 0; 119 virtual ~ProcessingUsage() = default; 120 }; 121 122 protected: 123 // Protected for test purposes. 124 void CheckForOveruse(OveruseFrameDetectorObserverInterface* overuse_observer); 125 void SetOptions(const CpuOveruseOptions& options); 126 127 CpuOveruseOptions options_; 128 129 private: 130 void EncodedFrameTimeMeasured(int encode_duration_ms); 131 bool IsOverusing(int encode_usage_percent); 132 bool IsUnderusing(int encode_usage_percent, int64_t time_now); 133 134 bool FrameTimeoutDetected(int64_t now) const; 135 bool FrameSizeChanged(int num_pixels) const; 136 137 void ResetAll(int num_pixels); 138 139 static std::unique_ptr<ProcessingUsage> CreateProcessingUsage( 140 const CpuOveruseOptions& options); 141 142 RTC_NO_UNIQUE_ADDRESS SequenceChecker task_checker_; 143 // Owned by the task queue from where StartCheckForOveruse is called. 144 RepeatingTaskHandle check_overuse_task_ RTC_GUARDED_BY(task_checker_); 145 146 // Stats metrics. 147 CpuOveruseMetricsObserver* const metrics_observer_; 148 absl::optional<int> encode_usage_percent_ RTC_GUARDED_BY(task_checker_); 149 150 int64_t num_process_times_ RTC_GUARDED_BY(task_checker_); 151 152 int64_t last_capture_time_us_ RTC_GUARDED_BY(task_checker_); 153 154 // Number of pixels of last captured frame. 155 int num_pixels_ RTC_GUARDED_BY(task_checker_); 156 int max_framerate_ RTC_GUARDED_BY(task_checker_); 157 int64_t last_overuse_time_ms_ RTC_GUARDED_BY(task_checker_); 158 int checks_above_threshold_ RTC_GUARDED_BY(task_checker_); 159 int num_overuse_detections_ RTC_GUARDED_BY(task_checker_); 160 int64_t last_rampup_time_ms_ RTC_GUARDED_BY(task_checker_); 161 bool in_quick_rampup_ RTC_GUARDED_BY(task_checker_); 162 int current_rampup_delay_ms_ RTC_GUARDED_BY(task_checker_); 163 164 std::unique_ptr<ProcessingUsage> usage_ RTC_PT_GUARDED_BY(task_checker_); 165 166 // If set by field trial, overrides CpuOveruseOptions::filter_time_ms. 167 FieldTrialOptional<TimeDelta> filter_time_constant_{"tau"}; 168 }; 169 170 } // namespace webrtc 171 172 #endif // VIDEO_ADAPTATION_OVERUSE_FRAME_DETECTOR_H_ 173