1/* 2 * Copyright 2017 The WebRTC project authors. All Rights Reserved. 3 * 4 * Use of this source code is governed by a BSD-style license 5 * that can be found in the LICENSE file in the root of the source 6 * tree. An additional intellectual property rights grant can be found 7 * in the file PATENTS. All contributing project authors may 8 * be found in the AUTHORS file in the root of the source tree. 9 */ 10 11#import "RTCMTLVideoView.h" 12 13#import <Metal/Metal.h> 14#import <MetalKit/MetalKit.h> 15 16#import "base/RTCLogging.h" 17#import "base/RTCVideoFrame.h" 18#import "base/RTCVideoFrameBuffer.h" 19#import "components/video_frame_buffer/RTCCVPixelBuffer.h" 20 21#import "RTCMTLI420Renderer.h" 22#import "RTCMTLNV12Renderer.h" 23#import "RTCMTLRGBRenderer.h" 24 25// To avoid unreconized symbol linker errors, we're taking advantage of the objc runtime. 26// Linking errors occur when compiling for architectures that don't support Metal. 27#define MTKViewClass NSClassFromString(@"MTKView") 28#define RTCMTLNV12RendererClass NSClassFromString(@"RTCMTLNV12Renderer") 29#define RTCMTLI420RendererClass NSClassFromString(@"RTCMTLI420Renderer") 30#define RTCMTLRGBRendererClass NSClassFromString(@"RTCMTLRGBRenderer") 31 32@interface RTC_OBJC_TYPE (RTCMTLVideoView) 33()<MTKViewDelegate> @property(nonatomic) RTCMTLI420Renderer *rendererI420; 34@property(nonatomic) RTCMTLNV12Renderer *rendererNV12; 35@property(nonatomic) RTCMTLRGBRenderer *rendererRGB; 36@property(nonatomic) MTKView *metalView; 37@property(atomic) RTC_OBJC_TYPE(RTCVideoFrame) * videoFrame; 38@property(nonatomic) CGSize videoFrameSize; 39@property(nonatomic) int64_t lastFrameTimeNs; 40@end 41 42@implementation RTC_OBJC_TYPE (RTCMTLVideoView) 43 44@synthesize delegate = _delegate; 45@synthesize rendererI420 = _rendererI420; 46@synthesize rendererNV12 = _rendererNV12; 47@synthesize rendererRGB = _rendererRGB; 48@synthesize metalView = _metalView; 49@synthesize videoFrame = _videoFrame; 50@synthesize videoFrameSize = _videoFrameSize; 51@synthesize lastFrameTimeNs = _lastFrameTimeNs; 52@synthesize rotationOverride = _rotationOverride; 53 54- (instancetype)initWithFrame:(CGRect)frameRect { 55 self = [super initWithFrame:frameRect]; 56 if (self) { 57 [self configure]; 58 } 59 return self; 60} 61 62- (instancetype)initWithCoder:(NSCoder *)aCoder { 63 self = [super initWithCoder:aCoder]; 64 if (self) { 65 [self configure]; 66 } 67 return self; 68} 69 70- (BOOL)isEnabled { 71 return !self.metalView.paused; 72} 73 74- (void)setEnabled:(BOOL)enabled { 75 self.metalView.paused = !enabled; 76} 77 78- (UIViewContentMode)videoContentMode { 79 return self.metalView.contentMode; 80} 81 82- (void)setVideoContentMode:(UIViewContentMode)mode { 83 self.metalView.contentMode = mode; 84} 85 86#pragma mark - Private 87 88+ (BOOL)isMetalAvailable { 89 return MTLCreateSystemDefaultDevice() != nil; 90} 91 92+ (MTKView *)createMetalView:(CGRect)frame { 93 return [[MTKViewClass alloc] initWithFrame:frame]; 94} 95 96+ (RTCMTLNV12Renderer *)createNV12Renderer { 97 return [[RTCMTLNV12RendererClass alloc] init]; 98} 99 100+ (RTCMTLI420Renderer *)createI420Renderer { 101 return [[RTCMTLI420RendererClass alloc] init]; 102} 103 104+ (RTCMTLRGBRenderer *)createRGBRenderer { 105 return [[RTCMTLRGBRenderer alloc] init]; 106} 107 108- (void)configure { 109 NSAssert([RTC_OBJC_TYPE(RTCMTLVideoView) isMetalAvailable], 110 @"Metal not availiable on this device"); 111 112 self.metalView = [RTC_OBJC_TYPE(RTCMTLVideoView) createMetalView:self.bounds]; 113 self.metalView.delegate = self; 114 self.metalView.contentMode = UIViewContentModeScaleAspectFill; 115 [self addSubview:self.metalView]; 116 self.videoFrameSize = CGSizeZero; 117} 118 119- (void)setMultipleTouchEnabled:(BOOL)multipleTouchEnabled { 120 [super setMultipleTouchEnabled:multipleTouchEnabled]; 121 self.metalView.multipleTouchEnabled = multipleTouchEnabled; 122} 123 124- (void)layoutSubviews { 125 [super layoutSubviews]; 126 127 CGRect bounds = self.bounds; 128 self.metalView.frame = bounds; 129 if (!CGSizeEqualToSize(self.videoFrameSize, CGSizeZero)) { 130 self.metalView.drawableSize = [self drawableSize]; 131 } else { 132 self.metalView.drawableSize = bounds.size; 133 } 134} 135 136#pragma mark - MTKViewDelegate methods 137 138- (void)drawInMTKView:(nonnull MTKView *)view { 139 NSAssert(view == self.metalView, @"Receiving draw callbacks from foreign instance."); 140 RTC_OBJC_TYPE(RTCVideoFrame) *videoFrame = self.videoFrame; 141 // Skip rendering if we've already rendered this frame. 142 if (!videoFrame || videoFrame.width <= 0 || videoFrame.height <= 0 || 143 videoFrame.timeStampNs == self.lastFrameTimeNs) { 144 return; 145 } 146 147 if (CGRectIsEmpty(view.bounds)) { 148 return; 149 } 150 151 RTCMTLRenderer *renderer; 152 if ([videoFrame.buffer isKindOfClass:[RTC_OBJC_TYPE(RTCCVPixelBuffer) class]]) { 153 RTC_OBJC_TYPE(RTCCVPixelBuffer) *buffer = (RTC_OBJC_TYPE(RTCCVPixelBuffer) *)videoFrame.buffer; 154 const OSType pixelFormat = CVPixelBufferGetPixelFormatType(buffer.pixelBuffer); 155 if (pixelFormat == kCVPixelFormatType_32BGRA || pixelFormat == kCVPixelFormatType_32ARGB) { 156 if (!self.rendererRGB) { 157 self.rendererRGB = [RTC_OBJC_TYPE(RTCMTLVideoView) createRGBRenderer]; 158 if (![self.rendererRGB addRenderingDestination:self.metalView]) { 159 self.rendererRGB = nil; 160 RTCLogError(@"Failed to create RGB renderer"); 161 return; 162 } 163 } 164 renderer = self.rendererRGB; 165 } else { 166 if (!self.rendererNV12) { 167 self.rendererNV12 = [RTC_OBJC_TYPE(RTCMTLVideoView) createNV12Renderer]; 168 if (![self.rendererNV12 addRenderingDestination:self.metalView]) { 169 self.rendererNV12 = nil; 170 RTCLogError(@"Failed to create NV12 renderer"); 171 return; 172 } 173 } 174 renderer = self.rendererNV12; 175 } 176 } else { 177 if (!self.rendererI420) { 178 self.rendererI420 = [RTC_OBJC_TYPE(RTCMTLVideoView) createI420Renderer]; 179 if (![self.rendererI420 addRenderingDestination:self.metalView]) { 180 self.rendererI420 = nil; 181 RTCLogError(@"Failed to create I420 renderer"); 182 return; 183 } 184 } 185 renderer = self.rendererI420; 186 } 187 188 renderer.rotationOverride = self.rotationOverride; 189 190 [renderer drawFrame:videoFrame]; 191 self.lastFrameTimeNs = videoFrame.timeStampNs; 192} 193 194- (void)mtkView:(MTKView *)view drawableSizeWillChange:(CGSize)size { 195} 196 197#pragma mark - 198 199- (void)setRotationOverride:(NSValue *)rotationOverride { 200 _rotationOverride = rotationOverride; 201 202 self.metalView.drawableSize = [self drawableSize]; 203 [self setNeedsLayout]; 204} 205 206- (RTCVideoRotation)frameRotation { 207 if (self.rotationOverride) { 208 RTCVideoRotation rotation; 209 if (@available(iOS 11, *)) { 210 [self.rotationOverride getValue:&rotation size:sizeof(rotation)]; 211 } else { 212 [self.rotationOverride getValue:&rotation]; 213 } 214 return rotation; 215 } 216 217 return self.videoFrame.rotation; 218} 219 220- (CGSize)drawableSize { 221 // Flip width/height if the rotations are not the same. 222 CGSize videoFrameSize = self.videoFrameSize; 223 RTCVideoRotation frameRotation = [self frameRotation]; 224 225 BOOL useLandscape = 226 (frameRotation == RTCVideoRotation_0) || (frameRotation == RTCVideoRotation_180); 227 BOOL sizeIsLandscape = (self.videoFrame.rotation == RTCVideoRotation_0) || 228 (self.videoFrame.rotation == RTCVideoRotation_180); 229 230 if (useLandscape == sizeIsLandscape) { 231 return videoFrameSize; 232 } else { 233 return CGSizeMake(videoFrameSize.height, videoFrameSize.width); 234 } 235} 236 237#pragma mark - RTC_OBJC_TYPE(RTCVideoRenderer) 238 239- (void)setSize:(CGSize)size { 240 __weak RTC_OBJC_TYPE(RTCMTLVideoView) *weakSelf = self; 241 dispatch_async(dispatch_get_main_queue(), ^{ 242 RTC_OBJC_TYPE(RTCMTLVideoView) *strongSelf = weakSelf; 243 244 strongSelf.videoFrameSize = size; 245 CGSize drawableSize = [strongSelf drawableSize]; 246 247 strongSelf.metalView.drawableSize = drawableSize; 248 [strongSelf setNeedsLayout]; 249 [strongSelf.delegate videoView:self didChangeVideoSize:size]; 250 }); 251} 252 253- (void)renderFrame:(nullable RTC_OBJC_TYPE(RTCVideoFrame) *)frame { 254 if (!self.isEnabled) { 255 return; 256 } 257 258 if (frame == nil) { 259 RTCLogInfo(@"Incoming frame is nil. Exiting render callback."); 260 return; 261 } 262 self.videoFrame = frame; 263} 264 265@end 266