1 /* 2 * Copyright (c) 2019 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11 #ifndef TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_ 12 #define TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_ 13 14 #include <list> 15 #include <memory> 16 #include <utility> 17 #include <vector> 18 19 #include "absl/strings/string_view.h" 20 #include "api/test/pclf/media_configuration.h" 21 #include "api/test/video_quality_analyzer_interface.h" 22 #include "api/video/video_frame.h" 23 #include "api/video_codecs/sdp_video_format.h" 24 #include "api/video_codecs/video_codec.h" 25 #include "api/video_codecs/video_encoder.h" 26 #include "api/video_codecs/video_encoder_factory.h" 27 #include "rtc_base/synchronization/mutex.h" 28 #include "test/pc/e2e/analyzer/video/encoded_image_data_injector.h" 29 30 namespace webrtc { 31 namespace webrtc_pc_e2e { 32 33 // QualityAnalyzingVideoEncoder is used to wrap origin video encoder and inject 34 // VideoQualityAnalyzerInterface before and after encoder. 35 // 36 // QualityAnalyzingVideoEncoder propagates all calls to the origin encoder. 37 // It registers its own EncodedImageCallback in the origin encoder and will 38 // store user specified callback inside itself. 39 // 40 // When Encode(...) will be invoked, quality encoder first calls video quality 41 // analyzer with original frame, then encodes frame with original encoder. 42 // 43 // When origin encoder encodes the image it will call quality encoder's special 44 // callback, where video analyzer will be called again and then frame id will be 45 // injected into EncodedImage with passed EncodedImageDataInjector. Then new 46 // EncodedImage will be passed to origin callback, provided by user. 47 // 48 // Quality encoder registers its own callback in origin encoder, at the same 49 // time the user registers their callback in quality encoder. 50 class QualityAnalyzingVideoEncoder : public VideoEncoder, 51 public EncodedImageCallback { 52 public: 53 using EmulatedSFUConfigMap = 54 std::map<std::string, absl::optional<EmulatedSFUConfig>>; 55 56 QualityAnalyzingVideoEncoder(absl::string_view peer_name, 57 std::unique_ptr<VideoEncoder> delegate, 58 double bitrate_multiplier, 59 EmulatedSFUConfigMap stream_to_sfu_config, 60 EncodedImageDataInjector* injector, 61 VideoQualityAnalyzerInterface* analyzer); 62 ~QualityAnalyzingVideoEncoder() override; 63 64 // Methods of VideoEncoder interface. 65 void SetFecControllerOverride( 66 FecControllerOverride* fec_controller_override) override; 67 int32_t InitEncode(const VideoCodec* codec_settings, 68 const Settings& settings) override; 69 int32_t RegisterEncodeCompleteCallback( 70 EncodedImageCallback* callback) override; 71 int32_t Release() override; 72 int32_t Encode(const VideoFrame& frame, 73 const std::vector<VideoFrameType>* frame_types) override; 74 void SetRates(const VideoEncoder::RateControlParameters& parameters) override; 75 EncoderInfo GetEncoderInfo() const override; 76 77 // Methods of EncodedImageCallback interface. 78 EncodedImageCallback::Result OnEncodedImage( 79 const EncodedImage& encoded_image, 80 const CodecSpecificInfo* codec_specific_info) override; 81 void OnDroppedFrame(DropReason reason) override; 82 83 private: 84 enum SimulcastMode { 85 // In this mode encoder assumes not more than 1 encoded image per video 86 // frame 87 kNormal, 88 89 // Next modes are to test video conference behavior. For conference sender 90 // will send multiple spatial layers/simulcast streams for single video 91 // track and there is some Selective Forwarding Unit (SFU), that forwards 92 // only best one, that will pass through downlink to the receiver. 93 // 94 // Here this behavior will be partly emulated. Sender will send all spatial 95 // layers/simulcast streams and then some of them will be filtered out on 96 // the receiver side. During test setup user can specify which spatial 97 // layer/simulcast stream is required, what will simulated which spatial 98 // layer/simulcast stream will be chosen by SFU in the real world. Then 99 // sender will mark encoded images for all spatial layers above required or 100 // all simulcast streams except required as to be discarded and on receiver 101 // side they will be discarded in quality analyzing decoder and won't be 102 // passed into delegate decoder. 103 // 104 // If the sender for some reasons won't send specified spatial layer, then 105 // receiver still will fall back on lower spatial layers. But for simulcast 106 // streams if required one won't be sent, receiver will assume all frames 107 // in that period as dropped and will experience video freeze. 108 // 109 // Test based on this simulation will be used to evaluate video quality 110 // of concrete spatial layers/simulcast streams and also check distribution 111 // of bandwidth between spatial layers/simulcast streams by BWE. 112 113 // In this mode encoder assumes that for each frame simulcast encoded 114 // images will be produced. So all simulcast streams except required will 115 // be marked as to be discarded in decoder and won't reach video quality 116 // analyzer. 117 kSimulcast, 118 // In this mode encoder assumes that for each frame encoded images for 119 // different spatial layers will be produced. So all spatial layers above 120 // required will be marked to be discarded in decoder and won't reach 121 // video quality analyzer. 122 kSVC, 123 // In this mode encoder assumes that for each frame encoded images for 124 // different spatial layers will be produced. Compared to kSVC mode 125 // spatial layers that are above required will be marked to be discarded 126 // only for key frames and for regular frames all except required spatial 127 // layer will be marked as to be discarded in decoder and won't reach video 128 // quality analyzer. 129 kKSVC 130 }; 131 132 bool ShouldDiscard(uint16_t frame_id, const EncodedImage& encoded_image) 133 RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); 134 135 const std::string peer_name_; 136 std::unique_ptr<VideoEncoder> delegate_; 137 const double bitrate_multiplier_; 138 // Contains mapping from stream label to optional spatial index. 139 // If we have stream label "Foo" and mapping contains 140 // 1. `absl::nullopt` means all streams are required 141 // 2. Concrete value means that particular simulcast/SVC stream have to be 142 // analyzed. 143 EmulatedSFUConfigMap stream_to_sfu_config_; 144 EncodedImageDataInjector* const injector_; 145 VideoQualityAnalyzerInterface* const analyzer_; 146 147 // VideoEncoder interface assumes async delivery of encoded images. 148 // This lock is used to protect shared state, that have to be propagated 149 // from received VideoFrame to resulted EncodedImage. 150 Mutex mutex_; 151 152 VideoCodec codec_settings_ RTC_GUARDED_BY(mutex_); 153 SimulcastMode mode_ RTC_GUARDED_BY(mutex_); 154 EncodedImageCallback* delegate_callback_ RTC_GUARDED_BY(mutex_); 155 std::list<std::pair<uint32_t, uint16_t>> timestamp_to_frame_id_list_ 156 RTC_GUARDED_BY(mutex_); 157 VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(mutex_); 158 }; 159 160 // Produces QualityAnalyzingVideoEncoder, which hold decoders, produced by 161 // specified factory as delegates. Forwards all other calls to specified 162 // factory. 163 class QualityAnalyzingVideoEncoderFactory : public VideoEncoderFactory { 164 public: 165 QualityAnalyzingVideoEncoderFactory( 166 absl::string_view peer_name, 167 std::unique_ptr<VideoEncoderFactory> delegate, 168 double bitrate_multiplier, 169 QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap stream_to_sfu_config, 170 EncodedImageDataInjector* injector, 171 VideoQualityAnalyzerInterface* analyzer); 172 ~QualityAnalyzingVideoEncoderFactory() override; 173 174 // Methods of VideoEncoderFactory interface. 175 std::vector<SdpVideoFormat> GetSupportedFormats() const override; 176 std::unique_ptr<VideoEncoder> CreateVideoEncoder( 177 const SdpVideoFormat& format) override; 178 179 private: 180 const std::string peer_name_; 181 std::unique_ptr<VideoEncoderFactory> delegate_; 182 const double bitrate_multiplier_; 183 QualityAnalyzingVideoEncoder::EmulatedSFUConfigMap stream_to_sfu_config_; 184 EncodedImageDataInjector* const injector_; 185 VideoQualityAnalyzerInterface* const analyzer_; 186 }; 187 188 } // namespace webrtc_pc_e2e 189 } // namespace webrtc 190 191 #endif // TEST_PC_E2E_ANALYZER_VIDEO_QUALITY_ANALYZING_VIDEO_ENCODER_H_ 192