1 /*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10 #include "test/frame_generator.h"
11
12 #include <string.h>
13
14 #include <cstdint>
15 #include <cstdio>
16 #include <memory>
17
18 #include "api/video/i010_buffer.h"
19 #include "api/video/nv12_buffer.h"
20 #include "api/video/video_rotation.h"
21 #include "common_video/include/video_frame_buffer.h"
22 #include "common_video/libyuv/include/webrtc_libyuv.h"
23 #include "rtc_base/checks.h"
24 #include "test/frame_utils.h"
25
26 namespace webrtc {
27 namespace test {
28
SquareGenerator(int width,int height,OutputType type,int num_squares)29 SquareGenerator::SquareGenerator(int width,
30 int height,
31 OutputType type,
32 int num_squares)
33 : type_(type) {
34 ChangeResolution(width, height);
35 for (int i = 0; i < num_squares; ++i) {
36 squares_.emplace_back(new Square(width, height, i + 1));
37 }
38 }
39
ChangeResolution(size_t width,size_t height)40 void SquareGenerator::ChangeResolution(size_t width, size_t height) {
41 MutexLock lock(&mutex_);
42 width_ = static_cast<int>(width);
43 height_ = static_cast<int>(height);
44 RTC_CHECK(width_ > 0);
45 RTC_CHECK(height_ > 0);
46 }
47
CreateI420Buffer(int width,int height)48 rtc::scoped_refptr<I420Buffer> SquareGenerator::CreateI420Buffer(int width,
49 int height) {
50 rtc::scoped_refptr<I420Buffer> buffer(I420Buffer::Create(width, height));
51 memset(buffer->MutableDataY(), 127, height * buffer->StrideY());
52 memset(buffer->MutableDataU(), 127,
53 buffer->ChromaHeight() * buffer->StrideU());
54 memset(buffer->MutableDataV(), 127,
55 buffer->ChromaHeight() * buffer->StrideV());
56 return buffer;
57 }
58
NextFrame()59 FrameGeneratorInterface::VideoFrameData SquareGenerator::NextFrame() {
60 MutexLock lock(&mutex_);
61
62 rtc::scoped_refptr<VideoFrameBuffer> buffer = nullptr;
63 switch (type_) {
64 case OutputType::kI420:
65 case OutputType::kI010:
66 case OutputType::kNV12: {
67 buffer = CreateI420Buffer(width_, height_);
68 break;
69 }
70 case OutputType::kI420A: {
71 rtc::scoped_refptr<I420Buffer> yuv_buffer =
72 CreateI420Buffer(width_, height_);
73 rtc::scoped_refptr<I420Buffer> axx_buffer =
74 CreateI420Buffer(width_, height_);
75 buffer = WrapI420ABuffer(yuv_buffer->width(), yuv_buffer->height(),
76 yuv_buffer->DataY(), yuv_buffer->StrideY(),
77 yuv_buffer->DataU(), yuv_buffer->StrideU(),
78 yuv_buffer->DataV(), yuv_buffer->StrideV(),
79 axx_buffer->DataY(), axx_buffer->StrideY(),
80 // To keep references alive.
81 [yuv_buffer, axx_buffer] {});
82 break;
83 }
84 default:
85 RTC_DCHECK_NOTREACHED() << "The given output format is not supported.";
86 }
87
88 for (const auto& square : squares_)
89 square->Draw(buffer);
90
91 if (type_ == OutputType::kI010) {
92 buffer = I010Buffer::Copy(*buffer->ToI420());
93 } else if (type_ == OutputType::kNV12) {
94 buffer = NV12Buffer::Copy(*buffer->ToI420());
95 }
96
97 return VideoFrameData(buffer, absl::nullopt);
98 }
99
Square(int width,int height,int seed)100 SquareGenerator::Square::Square(int width, int height, int seed)
101 : random_generator_(seed),
102 x_(random_generator_.Rand(0, width)),
103 y_(random_generator_.Rand(0, height)),
104 length_(random_generator_.Rand(1, width > 4 ? width / 4 : 1)),
105 yuv_y_(random_generator_.Rand(0, 255)),
106 yuv_u_(random_generator_.Rand(0, 255)),
107 yuv_v_(random_generator_.Rand(0, 255)),
108 yuv_a_(random_generator_.Rand(0, 255)) {}
109
Draw(const rtc::scoped_refptr<VideoFrameBuffer> & frame_buffer)110 void SquareGenerator::Square::Draw(
111 const rtc::scoped_refptr<VideoFrameBuffer>& frame_buffer) {
112 RTC_DCHECK(frame_buffer->type() == VideoFrameBuffer::Type::kI420 ||
113 frame_buffer->type() == VideoFrameBuffer::Type::kI420A);
114 rtc::scoped_refptr<I420BufferInterface> buffer = frame_buffer->ToI420();
115 int length_cap = std::min(buffer->height(), buffer->width()) / 4;
116 int length = std::min(length_, length_cap);
117 x_ = (x_ + random_generator_.Rand(0, 4)) % (buffer->width() - length);
118 y_ = (y_ + random_generator_.Rand(0, 4)) % (buffer->height() - length);
119 for (int y = y_; y < y_ + length; ++y) {
120 uint8_t* pos_y =
121 (const_cast<uint8_t*>(buffer->DataY()) + x_ + y * buffer->StrideY());
122 memset(pos_y, yuv_y_, length);
123 }
124
125 for (int y = y_; y < y_ + length; y = y + 2) {
126 uint8_t* pos_u = (const_cast<uint8_t*>(buffer->DataU()) + x_ / 2 +
127 y / 2 * buffer->StrideU());
128 memset(pos_u, yuv_u_, length / 2);
129 uint8_t* pos_v = (const_cast<uint8_t*>(buffer->DataV()) + x_ / 2 +
130 y / 2 * buffer->StrideV());
131 memset(pos_v, yuv_v_, length / 2);
132 }
133
134 if (frame_buffer->type() == VideoFrameBuffer::Type::kI420)
135 return;
136
137 // Optionally draw on alpha plane if given.
138 const webrtc::I420ABufferInterface* yuva_buffer = frame_buffer->GetI420A();
139 for (int y = y_; y < y_ + length; ++y) {
140 uint8_t* pos_y = (const_cast<uint8_t*>(yuva_buffer->DataA()) + x_ +
141 y * yuva_buffer->StrideA());
142 memset(pos_y, yuv_a_, length);
143 }
144 }
145
YuvFileGenerator(std::vector<FILE * > files,size_t width,size_t height,int frame_repeat_count)146 YuvFileGenerator::YuvFileGenerator(std::vector<FILE*> files,
147 size_t width,
148 size_t height,
149 int frame_repeat_count)
150 : file_index_(0),
151 frame_index_(std::numeric_limits<size_t>::max()),
152 files_(files),
153 width_(width),
154 height_(height),
155 frame_size_(CalcBufferSize(VideoType::kI420,
156 static_cast<int>(width_),
157 static_cast<int>(height_))),
158 frame_buffer_(new uint8_t[frame_size_]),
159 frame_display_count_(frame_repeat_count),
160 current_display_count_(0) {
161 RTC_DCHECK_GT(width, 0);
162 RTC_DCHECK_GT(height, 0);
163 RTC_DCHECK_GT(frame_repeat_count, 0);
164 }
165
~YuvFileGenerator()166 YuvFileGenerator::~YuvFileGenerator() {
167 for (FILE* file : files_)
168 fclose(file);
169 }
170
NextFrame()171 FrameGeneratorInterface::VideoFrameData YuvFileGenerator::NextFrame() {
172 // Empty update by default.
173 VideoFrame::UpdateRect update_rect{0, 0, 0, 0};
174 if (current_display_count_ == 0) {
175 const bool got_new_frame = ReadNextFrame();
176 // Full update on a new frame from file.
177 if (got_new_frame) {
178 update_rect = VideoFrame::UpdateRect{0, 0, static_cast<int>(width_),
179 static_cast<int>(height_)};
180 }
181 }
182 if (++current_display_count_ >= frame_display_count_)
183 current_display_count_ = 0;
184
185 return VideoFrameData(last_read_buffer_, update_rect);
186 }
187
ReadNextFrame()188 bool YuvFileGenerator::ReadNextFrame() {
189 size_t prev_frame_index = frame_index_;
190 size_t prev_file_index = file_index_;
191 last_read_buffer_ = test::ReadI420Buffer(
192 static_cast<int>(width_), static_cast<int>(height_), files_[file_index_]);
193 ++frame_index_;
194 if (!last_read_buffer_) {
195 // No more frames to read in this file, rewind and move to next file.
196 rewind(files_[file_index_]);
197
198 frame_index_ = 0;
199 file_index_ = (file_index_ + 1) % files_.size();
200 last_read_buffer_ =
201 test::ReadI420Buffer(static_cast<int>(width_),
202 static_cast<int>(height_), files_[file_index_]);
203 RTC_CHECK(last_read_buffer_);
204 }
205 return frame_index_ != prev_frame_index || file_index_ != prev_file_index;
206 }
207
NV12FileGenerator(std::vector<FILE * > files,size_t width,size_t height,int frame_repeat_count)208 NV12FileGenerator::NV12FileGenerator(std::vector<FILE*> files,
209 size_t width,
210 size_t height,
211 int frame_repeat_count)
212 : file_index_(0),
213 frame_index_(std::numeric_limits<size_t>::max()),
214 files_(files),
215 width_(width),
216 height_(height),
217 frame_size_(CalcBufferSize(VideoType::kNV12,
218 static_cast<int>(width_),
219 static_cast<int>(height_))),
220 frame_buffer_(new uint8_t[frame_size_]),
221 frame_display_count_(frame_repeat_count),
222 current_display_count_(0) {
223 RTC_DCHECK_GT(width, 0);
224 RTC_DCHECK_GT(height, 0);
225 RTC_DCHECK_GT(frame_repeat_count, 0);
226 }
227
~NV12FileGenerator()228 NV12FileGenerator::~NV12FileGenerator() {
229 for (FILE* file : files_)
230 fclose(file);
231 }
232
NextFrame()233 FrameGeneratorInterface::VideoFrameData NV12FileGenerator::NextFrame() {
234 // Empty update by default.
235 VideoFrame::UpdateRect update_rect{0, 0, 0, 0};
236 if (current_display_count_ == 0) {
237 const bool got_new_frame = ReadNextFrame();
238 // Full update on a new frame from file.
239 if (got_new_frame) {
240 update_rect = VideoFrame::UpdateRect{0, 0, static_cast<int>(width_),
241 static_cast<int>(height_)};
242 }
243 }
244 if (++current_display_count_ >= frame_display_count_)
245 current_display_count_ = 0;
246
247 return VideoFrameData(last_read_buffer_, update_rect);
248 }
249
ReadNextFrame()250 bool NV12FileGenerator::ReadNextFrame() {
251 size_t prev_frame_index = frame_index_;
252 size_t prev_file_index = file_index_;
253 last_read_buffer_ = test::ReadNV12Buffer(
254 static_cast<int>(width_), static_cast<int>(height_), files_[file_index_]);
255 ++frame_index_;
256 if (!last_read_buffer_) {
257 // No more frames to read in this file, rewind and move to next file.
258 rewind(files_[file_index_]);
259
260 frame_index_ = 0;
261 file_index_ = (file_index_ + 1) % files_.size();
262 last_read_buffer_ =
263 test::ReadNV12Buffer(static_cast<int>(width_),
264 static_cast<int>(height_), files_[file_index_]);
265 RTC_CHECK(last_read_buffer_);
266 }
267 return frame_index_ != prev_frame_index || file_index_ != prev_file_index;
268 }
269
SlideGenerator(int width,int height,int frame_repeat_count)270 SlideGenerator::SlideGenerator(int width, int height, int frame_repeat_count)
271 : width_(width),
272 height_(height),
273 frame_display_count_(frame_repeat_count),
274 current_display_count_(0),
275 random_generator_(1234) {
276 RTC_DCHECK_GT(width, 0);
277 RTC_DCHECK_GT(height, 0);
278 RTC_DCHECK_GT(frame_repeat_count, 0);
279 }
280
NextFrame()281 FrameGeneratorInterface::VideoFrameData SlideGenerator::NextFrame() {
282 if (current_display_count_ == 0)
283 GenerateNewFrame();
284 if (++current_display_count_ >= frame_display_count_)
285 current_display_count_ = 0;
286
287 return VideoFrameData(buffer_, absl::nullopt);
288 }
289
GenerateNewFrame()290 void SlideGenerator::GenerateNewFrame() {
291 // The squares should have a varying order of magnitude in order
292 // to simulate variation in the slides' complexity.
293 const int kSquareNum = 1 << (4 + (random_generator_.Rand(0, 3) * 2));
294
295 buffer_ = I420Buffer::Create(width_, height_);
296 memset(buffer_->MutableDataY(), 127, height_ * buffer_->StrideY());
297 memset(buffer_->MutableDataU(), 127,
298 buffer_->ChromaHeight() * buffer_->StrideU());
299 memset(buffer_->MutableDataV(), 127,
300 buffer_->ChromaHeight() * buffer_->StrideV());
301
302 for (int i = 0; i < kSquareNum; ++i) {
303 int length = random_generator_.Rand(1, width_ > 4 ? width_ / 4 : 1);
304 // Limit the length of later squares so that they don't overwrite the
305 // previous ones too much.
306 length = (length * (kSquareNum - i)) / kSquareNum;
307
308 int x = random_generator_.Rand(0, width_ - length);
309 int y = random_generator_.Rand(0, height_ - length);
310 uint8_t yuv_y = random_generator_.Rand(0, 255);
311 uint8_t yuv_u = random_generator_.Rand(0, 255);
312 uint8_t yuv_v = random_generator_.Rand(0, 255);
313
314 for (int yy = y; yy < y + length; ++yy) {
315 uint8_t* pos_y = (buffer_->MutableDataY() + x + yy * buffer_->StrideY());
316 memset(pos_y, yuv_y, length);
317 }
318 for (int yy = y; yy < y + length; yy += 2) {
319 uint8_t* pos_u =
320 (buffer_->MutableDataU() + x / 2 + yy / 2 * buffer_->StrideU());
321 memset(pos_u, yuv_u, length / 2);
322 uint8_t* pos_v =
323 (buffer_->MutableDataV() + x / 2 + yy / 2 * buffer_->StrideV());
324 memset(pos_v, yuv_v, length / 2);
325 }
326 }
327 }
328
ScrollingImageFrameGenerator(Clock * clock,const std::vector<FILE * > & files,size_t source_width,size_t source_height,size_t target_width,size_t target_height,int64_t scroll_time_ms,int64_t pause_time_ms)329 ScrollingImageFrameGenerator::ScrollingImageFrameGenerator(
330 Clock* clock,
331 const std::vector<FILE*>& files,
332 size_t source_width,
333 size_t source_height,
334 size_t target_width,
335 size_t target_height,
336 int64_t scroll_time_ms,
337 int64_t pause_time_ms)
338 : clock_(clock),
339 start_time_(clock->TimeInMilliseconds()),
340 scroll_time_(scroll_time_ms),
341 pause_time_(pause_time_ms),
342 num_frames_(files.size()),
343 target_width_(static_cast<int>(target_width)),
344 target_height_(static_cast<int>(target_height)),
345 current_frame_num_(num_frames_ - 1),
346 prev_frame_not_scrolled_(false),
347 current_source_frame_(nullptr, absl::nullopt),
348 current_frame_(nullptr, absl::nullopt),
349 file_generator_(files, source_width, source_height, 1) {
350 RTC_DCHECK(clock_ != nullptr);
351 RTC_DCHECK_GT(num_frames_, 0);
352 RTC_DCHECK_GE(source_height, target_height);
353 RTC_DCHECK_GE(source_width, target_width);
354 RTC_DCHECK_GE(scroll_time_ms, 0);
355 RTC_DCHECK_GE(pause_time_ms, 0);
356 RTC_DCHECK_GT(scroll_time_ms + pause_time_ms, 0);
357 }
358
359 FrameGeneratorInterface::VideoFrameData
NextFrame()360 ScrollingImageFrameGenerator::NextFrame() {
361 const int64_t kFrameDisplayTime = scroll_time_ + pause_time_;
362 const int64_t now = clock_->TimeInMilliseconds();
363 int64_t ms_since_start = now - start_time_;
364
365 size_t frame_num = (ms_since_start / kFrameDisplayTime) % num_frames_;
366 UpdateSourceFrame(frame_num);
367
368 bool cur_frame_not_scrolled;
369
370 double scroll_factor;
371 int64_t time_into_frame = ms_since_start % kFrameDisplayTime;
372 if (time_into_frame < scroll_time_) {
373 scroll_factor = static_cast<double>(time_into_frame) / scroll_time_;
374 cur_frame_not_scrolled = false;
375 } else {
376 scroll_factor = 1.0;
377 cur_frame_not_scrolled = true;
378 }
379 CropSourceToScrolledImage(scroll_factor);
380
381 bool same_scroll_position =
382 prev_frame_not_scrolled_ && cur_frame_not_scrolled;
383 if (!same_scroll_position) {
384 // If scrolling is not finished yet, force full frame update.
385 current_frame_.update_rect =
386 VideoFrame::UpdateRect{0, 0, target_width_, target_height_};
387 }
388 prev_frame_not_scrolled_ = cur_frame_not_scrolled;
389
390 return current_frame_;
391 }
392
UpdateSourceFrame(size_t frame_num)393 void ScrollingImageFrameGenerator::UpdateSourceFrame(size_t frame_num) {
394 VideoFrame::UpdateRect acc_update{0, 0, 0, 0};
395 while (current_frame_num_ != frame_num) {
396 current_source_frame_ = file_generator_.NextFrame();
397 if (current_source_frame_.update_rect) {
398 acc_update.Union(*current_source_frame_.update_rect);
399 }
400 current_frame_num_ = (current_frame_num_ + 1) % num_frames_;
401 }
402 current_source_frame_.update_rect = acc_update;
403 }
404
CropSourceToScrolledImage(double scroll_factor)405 void ScrollingImageFrameGenerator::CropSourceToScrolledImage(
406 double scroll_factor) {
407 int scroll_margin_x = current_source_frame_.buffer->width() - target_width_;
408 int pixels_scrolled_x =
409 static_cast<int>(scroll_margin_x * scroll_factor + 0.5);
410 int scroll_margin_y = current_source_frame_.buffer->height() - target_height_;
411 int pixels_scrolled_y =
412 static_cast<int>(scroll_margin_y * scroll_factor + 0.5);
413
414 rtc::scoped_refptr<I420BufferInterface> i420_buffer =
415 current_source_frame_.buffer->ToI420();
416 int offset_y =
417 (i420_buffer->StrideY() * pixels_scrolled_y) + pixels_scrolled_x;
418 int offset_u = (i420_buffer->StrideU() * (pixels_scrolled_y / 2)) +
419 (pixels_scrolled_x / 2);
420 int offset_v = (i420_buffer->StrideV() * (pixels_scrolled_y / 2)) +
421 (pixels_scrolled_x / 2);
422
423 VideoFrame::UpdateRect update_rect =
424 current_source_frame_.update_rect->IsEmpty()
425 ? VideoFrame::UpdateRect{0, 0, 0, 0}
426 : VideoFrame::UpdateRect{0, 0, target_width_, target_height_};
427 current_frame_ = VideoFrameData(
428 WrapI420Buffer(target_width_, target_height_,
429 &i420_buffer->DataY()[offset_y], i420_buffer->StrideY(),
430 &i420_buffer->DataU()[offset_u], i420_buffer->StrideU(),
431 &i420_buffer->DataV()[offset_v], i420_buffer->StrideV(),
432 // To keep reference alive.
433 [i420_buffer] {}),
434 update_rect);
435 }
436
437 } // namespace test
438 } // namespace webrtc
439