From 9c579c65aaad822136602bb8e39934fe51650654 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 11 Jun 2021 21:06:19 +0200 Subject: [PATCH] try: Improvements from WWDC 2021 1:1 workshop (#197) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf: Automatically determine Pixel Format depending on active format. (More efficient video recording 🚀) * perf: Skip `AVAssetWriter` transform by directly correctly orienting the Video Output connection * feat: Support camera flipping while recording * feat: Run frame processor on separate queue, avoids stutters in video recordigns * feat: Automatically drop late frame processor frames --- example/src/CameraPage.tsx | 4 +- ios/CameraQueues.swift | 9 ++- ios/CameraView+AVCaptureSession.swift | 6 +- ios/CameraView+RecordVideo.swift | 59 +++++++++++++++---- ios/CameraView.swift | 3 + ios/CameraViewManager.swift | 2 +- ...lBufferAdaptor+initWithVideoSettings.swift | 7 ++- .../AVCapturePhotoOutput+mirror.swift | 1 + .../AVCaptureVideoDataOutput+mirror.swift | 25 -------- ...aptureVideoDataOutput+setOrientation.swift | 24 ++++++++ .../FrameProcessorRuntimeManager.mm | 4 +- ios/Frame Processor/FrameProcessorUtils.mm | 13 +--- ios/RecordingSession.swift | 12 ++-- ios/VisionCamera.xcodeproj/project.pbxproj | 8 +-- 14 files changed, 103 insertions(+), 74 deletions(-) delete mode 100644 ios/Extensions/AVCaptureVideoDataOutput+mirror.swift create mode 100644 ios/Extensions/AVCaptureVideoDataOutput+setOrientation.swift diff --git a/example/src/CameraPage.tsx b/example/src/CameraPage.tsx index fe6a9df..167fc1d 100644 --- a/example/src/CameraPage.tsx +++ b/example/src/CameraPage.tsx @@ -151,10 +151,8 @@ export const CameraPage: NavigationFunctionComponent = ({ componentId }) => { //#region Tap Gesture const onDoubleTap = useCallback(() => { - // TODO: (MARC) Allow switching camera (back <-> front) while recording and stich videos together! - if (isPressingButton.value) return; onFlipCameraPressed(); - }, [isPressingButton, onFlipCameraPressed]); + }, [onFlipCameraPressed]); //#endregion //#region Effects diff --git a/ios/CameraQueues.swift b/ios/CameraQueues.swift index 2dc4747..01a7003 100644 --- a/ios/CameraQueues.swift +++ b/ios/CameraQueues.swift @@ -17,13 +17,20 @@ public class CameraQueues: NSObject { autoreleaseFrequency: .inherit, target: nil) - /// The serial execution queue for output processing of videos as well as frame processors. + /// The serial execution queue for output processing of videos for recording. @objc public static let videoQueue = DispatchQueue(label: "mrousavy/VisionCamera.video", qos: .userInteractive, attributes: [], autoreleaseFrequency: .inherit, target: nil) + /// The serial execution queue for output processing of videos for frame processing. + @objc public static let frameProcessorQueue = DispatchQueue(label: "mrousavy/VisionCamera.frame-processor", + qos: .userInteractive, + attributes: [], + autoreleaseFrequency: .inherit, + target: nil) + /// The serial execution queue for output processing of audio buffers. @objc public static let audioQueue = DispatchQueue(label: "mrousavy/VisionCamera.audio", qos: .userInteractive, diff --git a/ios/CameraView+AVCaptureSession.swift b/ios/CameraView+AVCaptureSession.swift index 161485c..3ebc490 100644 --- a/ios/CameraView+AVCaptureSession.swift +++ b/ios/CameraView+AVCaptureSession.swift @@ -133,11 +133,9 @@ extension CameraView { return } videoOutput!.setSampleBufferDelegate(self, queue: videoQueue) - videoOutput!.alwaysDiscardsLateVideoFrames = true + videoOutput!.alwaysDiscardsLateVideoFrames = false captureSession.addOutput(videoOutput!) - if videoDeviceInput!.device.position == .front { - videoOutput!.mirror() - } + videoOutput!.setOrientation(forCameraPosition: videoDeviceInput!.device.position) } invokeOnInitialized() diff --git a/ios/CameraView+RecordVideo.swift b/ios/CameraView+RecordVideo.swift index 987f984..fd5d130 100644 --- a/ios/CameraView+RecordVideo.swift +++ b/ios/CameraView+RecordVideo.swift @@ -8,7 +8,8 @@ import AVFoundation -private var hasLoggedFrameDropWarning = false +private var hasLoggedVideoFrameDropWarning = false +private var hasLoggedFrameProcessorFrameDropWarning = false // MARK: - CameraView + AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAudioDataOutputSampleBufferDelegate @@ -54,6 +55,10 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud return } } + guard let videoInput = self.videoDeviceInput else { + callback.reject(error: .session(.cameraNotReady)) + return + } // TODO: The startRecording() func cannot be async because RN doesn't allow // both a callback and a Promise in a single function. Wait for TurboModules? @@ -108,8 +113,10 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud callback.reject(error: .capture(.createRecorderError(message: "Failed to get video settings!"))) return } + // get pixel format (420f, 420v) + let pixelFormat = CMFormatDescriptionGetMediaSubType(videoInput.device.activeFormat.formatDescription) self.recordingSession!.initializeVideoWriter(withSettings: videoSettings, - isVideoMirrored: self.videoOutput!.isMirrored) + pixelFormat: pixelFormat) // Init Audio (optional, async) if enableAudio { @@ -196,30 +203,60 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud } } - if let frameProcessor = frameProcessorCallback, captureOutput is AVCaptureVideoDataOutput { + // TODO: resize using VideoToolbox (VTPixelTransferSession) + + if let frameProcessor = frameProcessorCallback, + captureOutput is AVCaptureVideoDataOutput { // check if last frame was x nanoseconds ago, effectively throttling FPS let diff = DispatchTime.now().uptimeNanoseconds - lastFrameProcessorCall.uptimeNanoseconds let secondsPerFrame = 1.0 / frameProcessorFps.doubleValue let nanosecondsPerFrame = secondsPerFrame * 1_000_000_000.0 if diff > UInt64(nanosecondsPerFrame) { - let frame = Frame(buffer: sampleBuffer, orientation: bufferOrientation) - frameProcessor(frame) - lastFrameProcessorCall = DispatchTime.now() + if !isRunningFrameProcessor { + // we're not in the middle of executing the Frame Processor, so prepare for next call. + var bufferCopy: CMSampleBuffer? + CMSampleBufferCreateCopy(allocator: kCFAllocatorDefault, + sampleBuffer: sampleBuffer, + sampleBufferOut: &bufferCopy) + if let bufferCopy = bufferCopy { + // successfully copied buffer, dispatch frame processor call. + CameraQueues.frameProcessorQueue.async { + self.isRunningFrameProcessor = true + let frame = Frame(buffer: bufferCopy, orientation: self.bufferOrientation) + frameProcessor(frame) + self.isRunningFrameProcessor = false + } + lastFrameProcessorCall = DispatchTime.now() + } else { + // failed to create a buffer copy. + ReactLogger.log(level: .error, message: "Failed to copy buffer! Frame Processor cannot be called.", alsoLogToJS: true) + } + } else { + // we're still in the middle of executing a Frame Processor for a previous frame, notify user about dropped frame. + if !hasLoggedFrameProcessorFrameDropWarning { + ReactLogger.log(level: .warning, + message: "Your Frame Processor took so long to execute that a frame was dropped. " + + "Either throttle your Frame Processor's frame rate using the `frameProcessorFps` prop, or optimize " + + "it's execution speed. (This warning will only be shown once)", + alsoLogToJS: true) + hasLoggedFrameProcessorFrameDropWarning = true + } + } } } } #if DEBUG public final func captureOutput(_ captureOutput: AVCaptureOutput, didDrop buffer: CMSampleBuffer, from _: AVCaptureConnection) { - if frameProcessorCallback != nil && !hasLoggedFrameDropWarning && captureOutput is AVCaptureVideoDataOutput { + if !hasLoggedVideoFrameDropWarning && captureOutput is AVCaptureVideoDataOutput { let reason = findFrameDropReason(inBuffer: buffer) ReactLogger.log(level: .warning, - message: "Dropped a Frame - This might indicate that your Frame Processor is doing too much work. " + - "Either throttle the frame processor's frame rate using the `frameProcessorFps` prop, or optimize " + - "your frame processor's execution speed. Frame drop reason: \(reason)", + message: "Dropped a Frame - This might indicate that your frame rate is higher than the phone can currently process. " + + "Throttle the Camera frame rate using the `fps` prop and make sure the device stays in optimal condition for recording. " + + "Frame drop reason: \(reason). (This warning will only be shown once)", alsoLogToJS: true) - hasLoggedFrameDropWarning = true + hasLoggedVideoFrameDropWarning = true } } diff --git a/ios/CameraView.swift b/ios/CameraView.swift index b4423f0..926fd27 100644 --- a/ios/CameraView.swift +++ b/ios/CameraView.swift @@ -100,6 +100,9 @@ public final class CameraView: UIView { internal let videoQueue = CameraQueues.videoQueue internal let audioQueue = CameraQueues.audioQueue + /// Specifies whether the frameProcessor() function is currently executing. used to drop late frames. + internal var isRunningFrameProcessor = false + var isRunning: Bool { return captureSession.isRunning } diff --git a/ios/CameraViewManager.swift b/ios/CameraViewManager.swift index 3637dc9..b88f4f3 100644 --- a/ios/CameraViewManager.swift +++ b/ios/CameraViewManager.swift @@ -24,7 +24,7 @@ final class CameraViewManager: RCTViewManager { // Install Frame Processor bindings and setup Runtime if enableFrameProcessors { - CameraQueues.videoQueue.async { + CameraQueues.frameProcessorQueue.async { self.runtimeManager = FrameProcessorRuntimeManager(bridge: self.bridge) self.bridge.runOnJS { self.runtimeManager!.installFrameProcessorBindings() diff --git a/ios/Extensions/AVAssetWriterInputPixelBufferAdaptor+initWithVideoSettings.swift b/ios/Extensions/AVAssetWriterInputPixelBufferAdaptor+initWithVideoSettings.swift index f84d234..34a4940 100644 --- a/ios/Extensions/AVAssetWriterInputPixelBufferAdaptor+initWithVideoSettings.swift +++ b/ios/Extensions/AVAssetWriterInputPixelBufferAdaptor+initWithVideoSettings.swift @@ -13,7 +13,9 @@ extension AVAssetWriterInputPixelBufferAdaptor { /** Convenience initializer to extract correct attributes from the given videoSettings. */ - convenience init(assetWriterInput: AVAssetWriterInput, withVideoSettings videoSettings: [String: Any]) { + convenience init(assetWriterInput: AVAssetWriterInput, + withVideoSettings videoSettings: [String: Any], + pixelFormat: OSType) { var attributes: [String: Any] = [:] if let width = videoSettings[AVVideoWidthKey] as? NSNumber, @@ -22,8 +24,7 @@ extension AVAssetWriterInputPixelBufferAdaptor { attributes[kCVPixelBufferHeightKey as String] = height as CFNumber } - // TODO: Is "Bi-Planar Y'CbCr 8-bit 4:2:0 full-range" the best CVPixelFormatType? How can I find natively supported ones? - attributes[kCVPixelBufferPixelFormatTypeKey as String] = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange + attributes[kCVPixelBufferPixelFormatTypeKey as String] = pixelFormat self.init(assetWriterInput: assetWriterInput, sourcePixelBufferAttributes: attributes) } diff --git a/ios/Extensions/AVCapturePhotoOutput+mirror.swift b/ios/Extensions/AVCapturePhotoOutput+mirror.swift index 4136005..de6dacf 100644 --- a/ios/Extensions/AVCapturePhotoOutput+mirror.swift +++ b/ios/Extensions/AVCapturePhotoOutput+mirror.swift @@ -12,6 +12,7 @@ extension AVCapturePhotoOutput { func mirror() { connections.forEach { connection in if connection.isVideoMirroringSupported { + connection.automaticallyAdjustsVideoMirroring = false connection.isVideoMirrored = true } } diff --git a/ios/Extensions/AVCaptureVideoDataOutput+mirror.swift b/ios/Extensions/AVCaptureVideoDataOutput+mirror.swift deleted file mode 100644 index 8adc4fe..0000000 --- a/ios/Extensions/AVCaptureVideoDataOutput+mirror.swift +++ /dev/null @@ -1,25 +0,0 @@ -// -// AVCaptureVideoDataOutput+mirror.swift -// Cuvent -// -// Created by Marc Rousavy on 18.01.21. -// Copyright © 2021 mrousavy. All rights reserved. -// - -import AVFoundation - -extension AVCaptureVideoDataOutput { - func mirror() { - connections.forEach { connection in - if connection.isVideoMirroringSupported { - connection.isVideoMirrored = true - } - } - } - - var isMirrored: Bool { - return connections.contains { connection in - connection.isVideoMirrored - } - } -} diff --git a/ios/Extensions/AVCaptureVideoDataOutput+setOrientation.swift b/ios/Extensions/AVCaptureVideoDataOutput+setOrientation.swift new file mode 100644 index 0000000..5afc416 --- /dev/null +++ b/ios/Extensions/AVCaptureVideoDataOutput+setOrientation.swift @@ -0,0 +1,24 @@ +// +// AVCaptureVideoDataOutput+setOrientation.swift +// Cuvent +// +// Created by Marc Rousavy on 18.01.21. +// Copyright © 2021 mrousavy. All rights reserved. +// + +import AVFoundation + +extension AVCaptureVideoDataOutput { + func setOrientation(forCameraPosition position: AVCaptureDevice.Position) { + let isMirrored = position == .front + connections.forEach { connection in + if connection.isVideoMirroringSupported { + connection.automaticallyAdjustsVideoMirroring = false + connection.isVideoMirrored = isMirrored + } + if connection.isVideoOrientationSupported { + connection.videoOrientation = .portrait + } + } + } +} diff --git a/ios/Frame Processor/FrameProcessorRuntimeManager.mm b/ios/Frame Processor/FrameProcessorRuntimeManager.mm index fc086b3..dcae577 100644 --- a/ios/Frame Processor/FrameProcessorRuntimeManager.mm +++ b/ios/Frame Processor/FrameProcessorRuntimeManager.mm @@ -43,7 +43,7 @@ // Forward declarations for the Swift classes __attribute__((objc_runtime_name("_TtC12VisionCamera12CameraQueues"))) @interface CameraQueues : NSObject -@property (nonatomic, class, readonly, strong) dispatch_queue_t _Nonnull videoQueue; +@property (nonatomic, class, readonly, strong) dispatch_queue_t _Nonnull frameProcessorQueue; @end __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView"))) @interface CameraView : UIView @@ -153,7 +153,7 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView"))) auto anonymousView = [currentBridge.uiManager viewForReactTag:[NSNumber numberWithDouble:viewTag]]; auto view = static_cast(anonymousView); - dispatch_async(CameraQueues.videoQueue, [worklet, view, self]() { + dispatch_async(CameraQueues.frameProcessorQueue, [worklet, view, self]() { NSLog(@"FrameProcessorBindings: Converting worklet to Objective-C callback..."); auto& rt = *runtimeManager->runtime; auto function = worklet->getValue(rt).asObject(rt).asFunction(rt); diff --git a/ios/Frame Processor/FrameProcessorUtils.mm b/ios/Frame Processor/FrameProcessorUtils.mm index 6690787..21844ef 100644 --- a/ios/Frame Processor/FrameProcessorUtils.mm +++ b/ios/Frame Processor/FrameProcessorUtils.mm @@ -16,10 +16,7 @@ FrameProcessorCallback convertJSIFunctionToFrameProcessorCallback(jsi::Runtime & __block auto cb = value.getFunction(runtime); return ^(Frame* frame) { -#if DEBUG - std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now(); -#endif - + auto frameHostObject = std::make_shared(frame); try { cb.call(runtime, jsi::Object::createFromHostObject(runtime, frameHostObject)); @@ -27,14 +24,6 @@ FrameProcessorCallback convertJSIFunctionToFrameProcessorCallback(jsi::Runtime & NSLog(@"Frame Processor threw an error: %s", jsError.getMessage().c_str()); } -#if DEBUG - std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now(); - auto duration = std::chrono::duration_cast(end - begin).count(); - if (duration > 100) { - NSLog(@"Warning: Frame Processor function took %lld ms to execute. This blocks the video queue from recording, optimize your frame processor!", duration); - } -#endif - // Manually free the buffer because: // 1. we are sure we don't need it anymore, the frame processor worklet has finished executing. // 2. we don't know when the JS runtime garbage collects this object, it might be holding it for a few more frames diff --git a/ios/RecordingSession.swift b/ios/RecordingSession.swift index 319c88e..ede341a 100644 --- a/ios/RecordingSession.swift +++ b/ios/RecordingSession.swift @@ -68,7 +68,7 @@ class RecordingSession { /** Initializes an AssetWriter for video frames (CMSampleBuffers). */ - func initializeVideoWriter(withSettings settings: [String: Any], isVideoMirrored: Bool) { + func initializeVideoWriter(withSettings settings: [String: Any], pixelFormat: OSType) { guard !settings.isEmpty else { ReactLogger.log(level: .error, message: "Tried to initialize Video Writer with empty settings!", alsoLogToJS: true) return @@ -81,14 +81,10 @@ class RecordingSession { let videoWriter = AVAssetWriterInput(mediaType: .video, outputSettings: settings) videoWriter.expectsMediaDataInRealTime = true - if isVideoMirrored { - videoWriter.transform = CGAffineTransform(rotationAngle: -(.pi / 2)) - } else { - videoWriter.transform = CGAffineTransform(rotationAngle: .pi / 2) - } - assetWriter.add(videoWriter) - bufferAdaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: videoWriter, withVideoSettings: settings) + bufferAdaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: videoWriter, + withVideoSettings: settings, + pixelFormat: pixelFormat) ReactLogger.log(level: .info, message: "Initialized Video AssetWriter.") } diff --git a/ios/VisionCamera.xcodeproj/project.pbxproj b/ios/VisionCamera.xcodeproj/project.pbxproj index c2956a8..8fa3e08 100644 --- a/ios/VisionCamera.xcodeproj/project.pbxproj +++ b/ios/VisionCamera.xcodeproj/project.pbxproj @@ -29,7 +29,7 @@ B887518F25E0102000DB86D6 /* AVCapturePhotoOutput+mirror.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887516825E0102000DB86D6 /* AVCapturePhotoOutput+mirror.swift */; }; B887519025E0102000DB86D6 /* AVCaptureDevice.Format+matchesFilter.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887516925E0102000DB86D6 /* AVCaptureDevice.Format+matchesFilter.swift */; }; B887519125E0102000DB86D6 /* AVCaptureDevice.Format+toDictionary.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887516A25E0102000DB86D6 /* AVCaptureDevice.Format+toDictionary.swift */; }; - B887519225E0102000DB86D6 /* AVCaptureVideoDataOutput+mirror.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887516B25E0102000DB86D6 /* AVCaptureVideoDataOutput+mirror.swift */; }; + B887519225E0102000DB86D6 /* AVCaptureVideoDataOutput+setOrientation.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887516B25E0102000DB86D6 /* AVCaptureVideoDataOutput+setOrientation.swift */; }; B887519425E0102000DB86D6 /* MakeReactError.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887516E25E0102000DB86D6 /* MakeReactError.swift */; }; B887519525E0102000DB86D6 /* ReactLogger.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887516F25E0102000DB86D6 /* ReactLogger.swift */; }; B887519625E0102000DB86D6 /* Promise.swift in Sources */ = {isa = PBXBuildFile; fileRef = B887517025E0102000DB86D6 /* Promise.swift */; }; @@ -106,7 +106,7 @@ B887516825E0102000DB86D6 /* AVCapturePhotoOutput+mirror.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "AVCapturePhotoOutput+mirror.swift"; sourceTree = ""; }; B887516925E0102000DB86D6 /* AVCaptureDevice.Format+matchesFilter.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "AVCaptureDevice.Format+matchesFilter.swift"; sourceTree = ""; }; B887516A25E0102000DB86D6 /* AVCaptureDevice.Format+toDictionary.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "AVCaptureDevice.Format+toDictionary.swift"; sourceTree = ""; }; - B887516B25E0102000DB86D6 /* AVCaptureVideoDataOutput+mirror.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "AVCaptureVideoDataOutput+mirror.swift"; sourceTree = ""; }; + B887516B25E0102000DB86D6 /* AVCaptureVideoDataOutput+setOrientation.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "AVCaptureVideoDataOutput+setOrientation.swift"; sourceTree = ""; }; B887516E25E0102000DB86D6 /* MakeReactError.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = MakeReactError.swift; sourceTree = ""; }; B887516F25E0102000DB86D6 /* ReactLogger.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ReactLogger.swift; sourceTree = ""; }; B887517025E0102000DB86D6 /* Promise.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Promise.swift; sourceTree = ""; }; @@ -203,7 +203,7 @@ B887516925E0102000DB86D6 /* AVCaptureDevice.Format+matchesFilter.swift */, B887516A25E0102000DB86D6 /* AVCaptureDevice.Format+toDictionary.swift */, B88B47462667C8E00091F538 /* AVCaptureSession+setVideoStabilizationMode.swift */, - B887516B25E0102000DB86D6 /* AVCaptureVideoDataOutput+mirror.swift */, + B887516B25E0102000DB86D6 /* AVCaptureVideoDataOutput+setOrientation.swift */, B887516225E0102000DB86D6 /* Collection+safe.swift */, ); path = Extensions; @@ -407,7 +407,7 @@ B8A751D82609E4B30011C623 /* FrameProcessorRuntimeManager.mm in Sources */, B887519A25E0102000DB86D6 /* AVVideoCodecType+descriptor.swift in Sources */, B88751A825E0102000DB86D6 /* CameraError.swift in Sources */, - B887519225E0102000DB86D6 /* AVCaptureVideoDataOutput+mirror.swift in Sources */, + B887519225E0102000DB86D6 /* AVCaptureVideoDataOutput+setOrientation.swift in Sources */, B88751A625E0102000DB86D6 /* CameraViewManager.swift in Sources */, B887519F25E0102000DB86D6 /* AVCaptureDevice.DeviceType+descriptor.swift in Sources */, B8D22CDC2642DB4D00234472 /* AVAssetWriterInputPixelBufferAdaptor+initWithVideoSettings.swift in Sources */,