perf: Improve pixelFormat and add CameraDevice.sensorOrientation (#1729)

* feat: Orientation

* fix orientation value in manifest

* Update AndroidManifest.xml

* Style

* fix: Set MAX_IMAGES to 3

* Pass `isMirrored` to `VideoPipeline`

* Update docs about Skia FPs

* Options

* Add iPad target

* Remove UIDevice onOrientationChanged listener

* Update CameraView+AVCaptureSession.swift

* Update CameraView+AVCaptureSession.swift

* Update CameraView+AVCaptureSession.swift

* Get available pixelFormats on iOS

* format

* Update CameraSession.kt

* Expose `CameraDevice.sensorOrientation`

* Lock orientation again

ios/CameraError.swift

@@ -79,6 +79,7 @@ enum DeviceError: String {
   case lowLightBoostNotSupported = "low-light-boost-not-supported"
   case focusNotSupported = "focus-not-supported"
   case notAvailableOnSimulator = "camera-not-available-on-simulator"
+  case pixelFormatNotSupported = "pixel-format-not-supported"
 
   var code: String {
     return rawValue
@@ -102,6 +103,8 @@ enum DeviceError: String {
       return "The microphone was unavailable."
     case .notAvailableOnSimulator:
       return "The Camera is not available on the iOS Simulator!"
+    case .pixelFormatNotSupported:
+      return "The given pixelFormat is not supported on the given Camera Device!"
     }
   }
 }

ios/CameraView+AVCaptureSession.swift

@@ -116,13 +116,24 @@ extension CameraView {
       videoOutput!.alwaysDiscardsLateVideoFrames = false
 
       if let pixelFormat = pixelFormat as? String {
-        let defaultFormat = CMFormatDescriptionGetMediaSubType(videoDeviceInput!.device.activeFormat.formatDescription)
+        let supportedPixelFormats = videoOutput!.availableVideoPixelFormatTypes
+        let defaultFormat = supportedPixelFormats.first! // first value is always the most efficient format
         var pixelFormatType: OSType = defaultFormat
         switch pixelFormat {
         case "yuv":
-          pixelFormatType = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
+          if supportedPixelFormats.contains(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
+            pixelFormatType = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange
+          } else if supportedPixelFormats.contains(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange) {
+            pixelFormatType = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange
+          } else {
+            invokeOnError(.device(.pixelFormatNotSupported))
+          }
         case "rgb":
-          pixelFormatType = kCVPixelFormatType_32BGRA
+          if supportedPixelFormats.contains(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
+            pixelFormatType = kCVPixelFormatType_32BGRA
+          } else {
+            invokeOnError(.device(.pixelFormatNotSupported))
+          }
         case "native":
           pixelFormatType = defaultFormat
         default:
@@ -135,7 +146,9 @@ extension CameraView {
       captureSession.addOutput(videoOutput!)
     }
 
-    onOrientationChanged()
+    if outputOrientation != .portrait {
+      updateOrientation()
+    }
 
     invokeOnInitialized()
     isReady = true

ios/CameraView.swift

@@ -107,8 +107,11 @@ public final class CameraView: UIView {
 
   // pragma MARK: Setup
   override public init(frame: CGRect) {
+    previewView = PreviewView(frame: frame, session: captureSession)
     super.init(frame: frame)
 
+    addSubview(previewView)
+
     NotificationCenter.default.addObserver(self,
                                            selector: #selector(sessionRuntimeError),
                                            name: .AVCaptureSessionRuntimeError,
@@ -121,13 +124,6 @@ public final class CameraView: UIView {
                                            selector: #selector(audioSessionInterrupted),
                                            name: AVAudioSession.interruptionNotification,
                                            object: AVAudioSession.sharedInstance)
-    NotificationCenter.default.addObserver(self,
-                                           selector: #selector(onOrientationChanged),
-                                           name: UIDevice.orientationDidChangeNotification,
-                                           object: nil)
-    
-    previewView = PreviewView(frame: frame, session: captureSession)
-    addSubview(previewView)
   }
 
   @available(*, unavailable)
@@ -145,9 +141,6 @@ public final class CameraView: UIView {
     NotificationCenter.default.removeObserver(self,
                                               name: AVAudioSession.interruptionNotification,
                                               object: AVAudioSession.sharedInstance)
-    NotificationCenter.default.removeObserver(self,
-                                              name: UIDevice.orientationDidChangeNotification,
-                                              object: nil)
   }
 
   override public func willMove(toSuperview newSuperview: UIView?) {
@@ -250,7 +243,7 @@ public final class CameraView: UIView {
       }
     }
   }
-  
+
   func setupFpsGraph() {
     #if DEBUG
       if enableFpsGraph {
@@ -265,11 +258,6 @@ public final class CameraView: UIView {
     #endif
   }
 
-  @objc
-  func onOrientationChanged() {
-    updateOrientation()
-  }
-
   // pragma MARK: Event Invokers
   final func invokeOnError(_ error: CameraError, cause: NSError? = nil) {
     ReactLogger.log(level: .error, message: "Invoking onError(): \(error.message)")

ios/CameraViewManager.swift

@@ -102,6 +102,7 @@ final class CameraViewManager: RCTViewManager {
           "supportsLowLightBoost": $0.isLowLightBoostSupported,
           "supportsFocus": $0.isFocusPointOfInterestSupported,
           "hardwareLevel": "full",
+          "sensorOrientation": "portrait", // TODO: Sensor Orientation?
           "formats": $0.formats.map { format -> [String: Any] in
             format.toDictionary()
           },

ios/Extensions/AVCaptureDevice.Format+toDictionary.swift

@@ -36,10 +36,10 @@ extension AVCaptureDevice.Format {
   }
 
   func toDictionary() -> [String: Any] {
-    let mediaSubType = CMFormatDescriptionGetMediaSubType(formatDescription)
-    let pixelFormat = PixelFormat(mediaSubType: mediaSubType)
+    let availablePixelFormats = AVCaptureVideoDataOutput().availableVideoPixelFormatTypes
+    let pixelFormats = availablePixelFormats.map { format in PixelFormat(mediaSubType: format) }
 
-    var dict: [String: Any] = [
+    return [
       "videoStabilizationModes": videoStabilizationModes.map(\.descriptor),
       "autoFocusSystem": autoFocusSystem.descriptor,
       "photoHeight": highResolutionStillImageDimensions.height,
@@ -54,9 +54,7 @@ extension AVCaptureDevice.Format {
       "supportsPhotoHDR": false,
       "minFps": minFrameRate,
       "maxFps": maxFrameRate,
-      "pixelFormats": [pixelFormat.unionValue],
+      "pixelFormats": pixelFormats.map(\.unionValue),
     ]
-
-    return dict
   }
 }