xref: /aosp_15_r20/external/webrtc/sdk/objc/unittests/RTCCVPixelBuffer_xctest.mm (revision d9f758449e529ab9291ac668be2861e7a55c2422) 
1/*
2 *  Copyright 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 *  Use of this source code is governed by a BSD-style license
5 *  that can be found in the LICENSE file in the root of the source
6 *  tree. An additional intellectual property rights grant can be found
7 *  in the file PATENTS.  All contributing project authors may
8 *  be found in the AUTHORS file in the root of the source tree.
9 */
10
11#import <Foundation/Foundation.h>
12#import <XCTest/XCTest.h>
13
14#import "components/video_frame_buffer/RTCCVPixelBuffer.h"
15
16#import "api/video_frame_buffer/RTCNativeI420Buffer+Private.h"
17#import "base/RTCVideoFrame.h"
18#import "base/RTCVideoFrameBuffer.h"
19#import "frame_buffer_helpers.h"
20
21#include "common_video/libyuv/include/webrtc_libyuv.h"
22#include "third_party/libyuv/include/libyuv.h"
23
24namespace {
25
26struct ToI420WithCropAndScaleSetting {
27  int inputWidth;
28  int inputHeight;
29  int offsetX;
30  int offsetY;
31  int cropWidth;
32  int cropHeight;
33  int scaleWidth;
34  int scaleHeight;
35};
36
37constexpr const ToI420WithCropAndScaleSetting kToI420WithCropAndScaleSettings[] = {
38    ToI420WithCropAndScaleSetting{
39        .inputWidth = 640,
40        .inputHeight = 360,
41        .offsetX = 0,
42        .offsetY = 0,
43        .cropWidth = 640,
44        .cropHeight = 360,
45        .scaleWidth = 320,
46        .scaleHeight = 180,
47    },
48    ToI420WithCropAndScaleSetting{
49        .inputWidth = 640,
50        .inputHeight = 360,
51        .offsetX = 160,
52        .offsetY = 90,
53        .cropWidth = 160,
54        .cropHeight = 90,
55        .scaleWidth = 320,
56        .scaleHeight = 180,
57    },
58};
59
60}  // namespace
61
62@interface RTCCVPixelBufferTests : XCTestCase
63@end
64
65@implementation RTCCVPixelBufferTests {
66}
67
68- (void)testRequiresCroppingNoCrop {
69  CVPixelBufferRef pixelBufferRef = NULL;
70  CVPixelBufferCreate(
71      NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef);
72  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
73      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef];
74
75  XCTAssertFalse([buffer requiresCropping]);
76
77  CVBufferRelease(pixelBufferRef);
78}
79
80- (void)testRequiresCroppingWithCrop {
81  CVPixelBufferRef pixelBufferRef = NULL;
82  CVPixelBufferCreate(
83      NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef);
84  RTC_OBJC_TYPE(RTCCVPixelBuffer) *croppedBuffer =
85      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef
86                                                      adaptedWidth:720
87                                                     adaptedHeight:1280
88                                                         cropWidth:360
89                                                        cropHeight:640
90                                                             cropX:100
91                                                             cropY:100];
92
93  XCTAssertTrue([croppedBuffer requiresCropping]);
94
95  CVBufferRelease(pixelBufferRef);
96}
97
98- (void)testRequiresScalingNoScale {
99  CVPixelBufferRef pixelBufferRef = NULL;
100  CVPixelBufferCreate(
101      NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef);
102
103  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
104      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef];
105  XCTAssertFalse([buffer requiresScalingToWidth:720 height:1280]);
106
107  CVBufferRelease(pixelBufferRef);
108}
109
110- (void)testRequiresScalingWithScale {
111  CVPixelBufferRef pixelBufferRef = NULL;
112  CVPixelBufferCreate(
113      NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef);
114
115  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
116      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef];
117  XCTAssertTrue([buffer requiresScalingToWidth:360 height:640]);
118
119  CVBufferRelease(pixelBufferRef);
120}
121
122- (void)testRequiresScalingWithScaleAndMatchingCrop {
123  CVPixelBufferRef pixelBufferRef = NULL;
124  CVPixelBufferCreate(
125      NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef);
126
127  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
128      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef
129                                                      adaptedWidth:720
130                                                     adaptedHeight:1280
131                                                         cropWidth:360
132                                                        cropHeight:640
133                                                             cropX:100
134                                                             cropY:100];
135  XCTAssertFalse([buffer requiresScalingToWidth:360 height:640]);
136
137  CVBufferRelease(pixelBufferRef);
138}
139
140- (void)testBufferSize_NV12 {
141  CVPixelBufferRef pixelBufferRef = NULL;
142  CVPixelBufferCreate(
143      NULL, 720, 1280, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef);
144
145  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
146      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef];
147  XCTAssertEqual([buffer bufferSizeForCroppingAndScalingToWidth:360 height:640], 576000);
148
149  CVBufferRelease(pixelBufferRef);
150}
151
152- (void)testBufferSize_RGB {
153  CVPixelBufferRef pixelBufferRef = NULL;
154  CVPixelBufferCreate(NULL, 720, 1280, kCVPixelFormatType_32BGRA, NULL, &pixelBufferRef);
155
156  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
157      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef];
158  XCTAssertEqual([buffer bufferSizeForCroppingAndScalingToWidth:360 height:640], 0);
159
160  CVBufferRelease(pixelBufferRef);
161}
162
163- (void)testCropAndScale_NV12 {
164  [self cropAndScaleTestWithNV12];
165}
166
167- (void)testCropAndScaleNoOp_NV12 {
168  [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
169                               outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
170                                 outputSize:CGSizeMake(720, 1280)];
171}
172
173- (void)testCropAndScale_NV12FullToVideo {
174  [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
175                               outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange];
176}
177
178- (void)testCropAndScaleZeroSizeFrame_NV12 {
179  [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
180                               outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
181                                 outputSize:CGSizeMake(0, 0)];
182}
183
184- (void)testCropAndScaleToSmallFormat_NV12 {
185  [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
186                               outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
187                                 outputSize:CGSizeMake(148, 320)];
188}
189
190- (void)testCropAndScaleToOddFormat_NV12 {
191  [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
192                               outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
193                                 outputSize:CGSizeMake(361, 640)];
194}
195
196- (void)testCropAndScale_32BGRA {
197  [self cropAndScaleTestWithRGBPixelFormat:kCVPixelFormatType_32BGRA];
198}
199
200- (void)testCropAndScale_32ARGB {
201  [self cropAndScaleTestWithRGBPixelFormat:kCVPixelFormatType_32ARGB];
202}
203
204- (void)testCropAndScaleWithSmallCropInfo_32ARGB {
205  [self cropAndScaleTestWithRGBPixelFormat:kCVPixelFormatType_32ARGB cropX:2 cropY:3];
206}
207
208- (void)testCropAndScaleWithLargeCropInfo_32ARGB {
209  [self cropAndScaleTestWithRGBPixelFormat:kCVPixelFormatType_32ARGB cropX:200 cropY:300];
210}
211
212- (void)testToI420_NV12 {
213  [self toI420WithPixelFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange];
214}
215
216- (void)testToI420_32BGRA {
217  [self toI420WithPixelFormat:kCVPixelFormatType_32BGRA];
218}
219
220- (void)testToI420_32ARGB {
221  [self toI420WithPixelFormat:kCVPixelFormatType_32ARGB];
222}
223
224- (void)testToI420WithCropAndScale_NV12 {
225  for (const auto &setting : kToI420WithCropAndScaleSettings) {
226    [self toI420WithCropAndScaleWithPixelFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
227                                        setting:setting];
228  }
229}
230
231- (void)testToI420WithCropAndScale_32BGRA {
232  for (const auto &setting : kToI420WithCropAndScaleSettings) {
233    [self toI420WithCropAndScaleWithPixelFormat:kCVPixelFormatType_32BGRA setting:setting];
234  }
235}
236
237- (void)testToI420WithCropAndScale_32ARGB {
238  for (const auto &setting : kToI420WithCropAndScaleSettings) {
239    [self toI420WithCropAndScaleWithPixelFormat:kCVPixelFormatType_32ARGB setting:setting];
240  }
241}
242
243- (void)testScaleBufferTest {
244  CVPixelBufferRef pixelBufferRef = NULL;
245  CVPixelBufferCreate(
246      NULL, 1920, 1080, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, NULL, &pixelBufferRef);
247
248  rtc::scoped_refptr<webrtc::I420Buffer> i420Buffer = CreateI420Gradient(1920, 1080);
249  CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef);
250
251  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
252      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef];
253
254  XCTAssertEqual(buffer.width, 1920);
255  XCTAssertEqual(buffer.height, 1080);
256  XCTAssertEqual(buffer.cropX, 0);
257  XCTAssertEqual(buffer.cropY, 0);
258  XCTAssertEqual(buffer.cropWidth, 1920);
259  XCTAssertEqual(buffer.cropHeight, 1080);
260
261  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer2 =
262      (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)[buffer cropAndScaleWith:320
263                                                          offsetY:180
264                                                        cropWidth:1280
265                                                       cropHeight:720
266                                                       scaleWidth:960
267                                                      scaleHeight:540];
268
269  XCTAssertEqual(buffer2.width, 960);
270  XCTAssertEqual(buffer2.height, 540);
271  XCTAssertEqual(buffer2.cropX, 320);
272  XCTAssertEqual(buffer2.cropY, 180);
273  XCTAssertEqual(buffer2.cropWidth, 1280);
274  XCTAssertEqual(buffer2.cropHeight, 720);
275
276  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer3 =
277      (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)[buffer2 cropAndScaleWith:240
278                                                           offsetY:135
279                                                         cropWidth:480
280                                                        cropHeight:270
281                                                        scaleWidth:320
282                                                       scaleHeight:180];
283
284  XCTAssertEqual(buffer3.width, 320);
285  XCTAssertEqual(buffer3.height, 180);
286  XCTAssertEqual(buffer3.cropX, 640);
287  XCTAssertEqual(buffer3.cropY, 360);
288  XCTAssertEqual(buffer3.cropWidth, 640);
289  XCTAssertEqual(buffer3.cropHeight, 360);
290
291  CVBufferRelease(pixelBufferRef);
292}
293
294#pragma mark - Shared test code
295
296- (void)cropAndScaleTestWithNV12 {
297  [self cropAndScaleTestWithNV12InputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
298                               outputFormat:kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange];
299}
300
301- (void)cropAndScaleTestWithNV12InputFormat:(OSType)inputFormat outputFormat:(OSType)outputFormat {
302  [self cropAndScaleTestWithNV12InputFormat:(OSType)inputFormat
303                               outputFormat:(OSType)outputFormat
304                                 outputSize:CGSizeMake(360, 640)];
305}
306
307- (void)cropAndScaleTestWithNV12InputFormat:(OSType)inputFormat
308                               outputFormat:(OSType)outputFormat
309                                 outputSize:(CGSize)outputSize {
310  CVPixelBufferRef pixelBufferRef = NULL;
311  CVPixelBufferCreate(NULL, 720, 1280, inputFormat, NULL, &pixelBufferRef);
312
313  rtc::scoped_refptr<webrtc::I420Buffer> i420Buffer = CreateI420Gradient(720, 1280);
314  CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef);
315
316  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
317      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef];
318  XCTAssertEqual(buffer.width, 720);
319  XCTAssertEqual(buffer.height, 1280);
320
321  CVPixelBufferRef outputPixelBufferRef = NULL;
322  CVPixelBufferCreate(
323      NULL, outputSize.width, outputSize.height, outputFormat, NULL, &outputPixelBufferRef);
324
325  std::vector<uint8_t> frameScaleBuffer;
326  if ([buffer requiresScalingToWidth:outputSize.width height:outputSize.height]) {
327    int size =
328        [buffer bufferSizeForCroppingAndScalingToWidth:outputSize.width height:outputSize.height];
329    frameScaleBuffer.resize(size);
330  } else {
331    frameScaleBuffer.clear();
332  }
333  frameScaleBuffer.shrink_to_fit();
334
335  [buffer cropAndScaleTo:outputPixelBufferRef withTempBuffer:frameScaleBuffer.data()];
336
337  RTC_OBJC_TYPE(RTCCVPixelBuffer) *scaledBuffer =
338      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:outputPixelBufferRef];
339  XCTAssertEqual(scaledBuffer.width, outputSize.width);
340  XCTAssertEqual(scaledBuffer.height, outputSize.height);
341
342  if (outputSize.width > 0 && outputSize.height > 0) {
343    RTC_OBJC_TYPE(RTCI420Buffer) *originalBufferI420 = [buffer toI420];
344    RTC_OBJC_TYPE(RTCI420Buffer) *scaledBufferI420 = [scaledBuffer toI420];
345    double psnr =
346        I420PSNR(*[originalBufferI420 nativeI420Buffer], *[scaledBufferI420 nativeI420Buffer]);
347    XCTAssertEqual(psnr, webrtc::kPerfectPSNR);
348  }
349
350  CVBufferRelease(pixelBufferRef);
351}
352
353- (void)cropAndScaleTestWithRGBPixelFormat:(OSType)pixelFormat {
354  [self cropAndScaleTestWithRGBPixelFormat:pixelFormat cropX:0 cropY:0];
355}
356
357- (void)cropAndScaleTestWithRGBPixelFormat:(OSType)pixelFormat cropX:(int)cropX cropY:(int)cropY {
358  CVPixelBufferRef pixelBufferRef = NULL;
359  CVPixelBufferCreate(NULL, 720, 1280, pixelFormat, NULL, &pixelBufferRef);
360
361  DrawGradientInRGBPixelBuffer(pixelBufferRef);
362
363  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc]
364      initWithPixelBuffer:pixelBufferRef
365             adaptedWidth:CVPixelBufferGetWidth(pixelBufferRef)
366            adaptedHeight:CVPixelBufferGetHeight(pixelBufferRef)
367                cropWidth:CVPixelBufferGetWidth(pixelBufferRef) - cropX
368               cropHeight:CVPixelBufferGetHeight(pixelBufferRef) - cropY
369                    cropX:cropX
370                    cropY:cropY];
371
372  XCTAssertEqual(buffer.width, 720);
373  XCTAssertEqual(buffer.height, 1280);
374
375  CVPixelBufferRef outputPixelBufferRef = NULL;
376  CVPixelBufferCreate(NULL, 360, 640, pixelFormat, NULL, &outputPixelBufferRef);
377  [buffer cropAndScaleTo:outputPixelBufferRef withTempBuffer:NULL];
378
379  RTC_OBJC_TYPE(RTCCVPixelBuffer) *scaledBuffer =
380      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:outputPixelBufferRef];
381  XCTAssertEqual(scaledBuffer.width, 360);
382  XCTAssertEqual(scaledBuffer.height, 640);
383
384  RTC_OBJC_TYPE(RTCI420Buffer) *originalBufferI420 = [buffer toI420];
385  RTC_OBJC_TYPE(RTCI420Buffer) *scaledBufferI420 = [scaledBuffer toI420];
386  double psnr =
387      I420PSNR(*[originalBufferI420 nativeI420Buffer], *[scaledBufferI420 nativeI420Buffer]);
388  XCTAssertEqual(psnr, webrtc::kPerfectPSNR);
389
390  CVBufferRelease(pixelBufferRef);
391}
392
393- (void)toI420WithPixelFormat:(OSType)pixelFormat {
394  rtc::scoped_refptr<webrtc::I420Buffer> i420Buffer = CreateI420Gradient(360, 640);
395
396  CVPixelBufferRef pixelBufferRef = NULL;
397  CVPixelBufferCreate(NULL, 360, 640, pixelFormat, NULL, &pixelBufferRef);
398
399  CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef);
400
401  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
402      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef];
403  RTC_OBJC_TYPE(RTCI420Buffer) *fromCVPixelBuffer = [buffer toI420];
404
405  double psnr = I420PSNR(*i420Buffer, *[fromCVPixelBuffer nativeI420Buffer]);
406  double target = webrtc::kPerfectPSNR;
407  if (pixelFormat != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) {
408    // libyuv's I420ToRGB functions seem to lose some quality.
409    target = 19.0;
410  }
411  XCTAssertGreaterThanOrEqual(psnr, target);
412
413  CVBufferRelease(pixelBufferRef);
414}
415
416- (void)toI420WithCropAndScaleWithPixelFormat:(OSType)pixelFormat
417                                      setting:(const ToI420WithCropAndScaleSetting &)setting {
418  rtc::scoped_refptr<webrtc::I420Buffer> i420Buffer =
419      CreateI420Gradient(setting.inputWidth, setting.inputHeight);
420
421  CVPixelBufferRef pixelBufferRef = NULL;
422  CVPixelBufferCreate(
423      NULL, setting.inputWidth, setting.inputHeight, pixelFormat, NULL, &pixelBufferRef);
424
425  CopyI420BufferToCVPixelBuffer(i420Buffer, pixelBufferRef);
426
427  RTC_OBJC_TYPE(RTCI420Buffer) *objcI420Buffer =
428      [[RTC_OBJC_TYPE(RTCI420Buffer) alloc] initWithFrameBuffer:i420Buffer];
429  RTC_OBJC_TYPE(RTCI420Buffer) *scaledObjcI420Buffer =
430      (RTC_OBJC_TYPE(RTCI420Buffer) *)[objcI420Buffer cropAndScaleWith:setting.offsetX
431                                                               offsetY:setting.offsetY
432                                                             cropWidth:setting.cropWidth
433                                                            cropHeight:setting.cropHeight
434                                                            scaleWidth:setting.scaleWidth
435                                                           scaleHeight:setting.scaleHeight];
436  RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer =
437      [[RTC_OBJC_TYPE(RTCCVPixelBuffer) alloc] initWithPixelBuffer:pixelBufferRef];
438  id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> scaledBuffer =
439      [buffer cropAndScaleWith:setting.offsetX
440                       offsetY:setting.offsetY
441                     cropWidth:setting.cropWidth
442                    cropHeight:setting.cropHeight
443                    scaleWidth:setting.scaleWidth
444                   scaleHeight:setting.scaleHeight];
445  XCTAssertTrue([scaledBuffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]);
446
447  RTC_OBJC_TYPE(RTCI420Buffer) *fromCVPixelBuffer = [scaledBuffer toI420];
448
449  double psnr =
450      I420PSNR(*[scaledObjcI420Buffer nativeI420Buffer], *[fromCVPixelBuffer nativeI420Buffer]);
451  double target = webrtc::kPerfectPSNR;
452  if (pixelFormat != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) {
453    // libyuv's I420ToRGB functions seem to lose some quality.
454    target = 19.0;
455  }
456  XCTAssertGreaterThanOrEqual(psnr, target);
457
458  CVBufferRelease(pixelBufferRef);
459}
460
461@end
462