try: Improvements from WWDC 2021 1:1 workshop (#197)

* perf: Automatically determine Pixel Format depending on active format. (More efficient video recording 🚀)
* perf: Skip `AVAssetWriter` transform by directly correctly orienting the Video Output connection
* feat: Support camera flipping while recording
* feat: Run frame processor on separate queue, avoids stutters in video recordigns
* feat: Automatically drop late frame processor frames

ios/CameraView+RecordVideo.swift

@@ -8,7 +8,8 @@
 
 import AVFoundation
 
-private var hasLoggedFrameDropWarning = false
+private var hasLoggedVideoFrameDropWarning = false
+private var hasLoggedFrameProcessorFrameDropWarning = false
 
 // MARK: - CameraView + AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAudioDataOutputSampleBufferDelegate
 
@@ -54,6 +55,10 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud
           return
         }
       }
+      guard let videoInput = self.videoDeviceInput else {
+        callback.reject(error: .session(.cameraNotReady))
+        return
+      }
 
       // TODO: The startRecording() func cannot be async because RN doesn't allow
       //       both a callback and a Promise in a single function. Wait for TurboModules?
@@ -108,8 +113,10 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud
         callback.reject(error: .capture(.createRecorderError(message: "Failed to get video settings!")))
         return
       }
+      // get pixel format (420f, 420v)
+      let pixelFormat = CMFormatDescriptionGetMediaSubType(videoInput.device.activeFormat.formatDescription)
       self.recordingSession!.initializeVideoWriter(withSettings: videoSettings,
-                                                   isVideoMirrored: self.videoOutput!.isMirrored)
+                                                   pixelFormat: pixelFormat)
 
       // Init Audio (optional, async)
       if enableAudio {
@@ -196,30 +203,60 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud
       }
     }
 
-    if let frameProcessor = frameProcessorCallback, captureOutput is AVCaptureVideoDataOutput {
+    // TODO: resize using VideoToolbox (VTPixelTransferSession)
+
+    if let frameProcessor = frameProcessorCallback,
+       captureOutput is AVCaptureVideoDataOutput {
       // check if last frame was x nanoseconds ago, effectively throttling FPS
       let diff = DispatchTime.now().uptimeNanoseconds - lastFrameProcessorCall.uptimeNanoseconds
       let secondsPerFrame = 1.0 / frameProcessorFps.doubleValue
       let nanosecondsPerFrame = secondsPerFrame * 1_000_000_000.0
 
       if diff > UInt64(nanosecondsPerFrame) {
-        let frame = Frame(buffer: sampleBuffer, orientation: bufferOrientation)
-        frameProcessor(frame)
-        lastFrameProcessorCall = DispatchTime.now()
+        if !isRunningFrameProcessor {
+          // we're not in the middle of executing the Frame Processor, so prepare for next call.
+          var bufferCopy: CMSampleBuffer?
+          CMSampleBufferCreateCopy(allocator: kCFAllocatorDefault,
+                                   sampleBuffer: sampleBuffer,
+                                   sampleBufferOut: &bufferCopy)
+          if let bufferCopy = bufferCopy {
+            // successfully copied buffer, dispatch frame processor call.
+            CameraQueues.frameProcessorQueue.async {
+              self.isRunningFrameProcessor = true
+              let frame = Frame(buffer: bufferCopy, orientation: self.bufferOrientation)
+              frameProcessor(frame)
+              self.isRunningFrameProcessor = false
+            }
+            lastFrameProcessorCall = DispatchTime.now()
+          } else {
+            // failed to create a buffer copy.
+            ReactLogger.log(level: .error, message: "Failed to copy buffer! Frame Processor cannot be called.", alsoLogToJS: true)
+          }
+        } else {
+          // we're still in the middle of executing a Frame Processor for a previous frame, notify user about dropped frame.
+          if !hasLoggedFrameProcessorFrameDropWarning {
+            ReactLogger.log(level: .warning,
+                            message: "Your Frame Processor took so long to execute that a frame was dropped. " +
+                              "Either throttle your Frame Processor's frame rate using the `frameProcessorFps` prop, or optimize " +
+                              "it's execution speed. (This warning will only be shown once)",
+                            alsoLogToJS: true)
+            hasLoggedFrameProcessorFrameDropWarning = true
+          }
+        }
       }
     }
   }
 
   #if DEBUG
     public final func captureOutput(_ captureOutput: AVCaptureOutput, didDrop buffer: CMSampleBuffer, from _: AVCaptureConnection) {
-      if frameProcessorCallback != nil && !hasLoggedFrameDropWarning && captureOutput is AVCaptureVideoDataOutput {
+      if !hasLoggedVideoFrameDropWarning && captureOutput is AVCaptureVideoDataOutput {
         let reason = findFrameDropReason(inBuffer: buffer)
         ReactLogger.log(level: .warning,
-                        message: "Dropped a Frame - This might indicate that your Frame Processor is doing too much work. " +
-                          "Either throttle the frame processor's frame rate using the `frameProcessorFps` prop, or optimize " +
-                          "your frame processor's execution speed. Frame drop reason: \(reason)",
+                        message: "Dropped a Frame - This might indicate that your frame rate is higher than the phone can currently process. " +
+                          "Throttle the Camera frame rate using the `fps` prop and make sure the device stays in optimal condition for recording. " +
+                          "Frame drop reason: \(reason). (This warning will only be shown once)",
                         alsoLogToJS: true)
-        hasLoggedFrameDropWarning = true
+        hasLoggedVideoFrameDropWarning = true
       }
     }