feat: Use correct photo and video format dimensions on iOS (#1929)

* feat: Use new photo dimensions API

* Update AVCaptureDevice.Format+matchesFilter.swift

* fix: Use Pixels instead of Points for video size

* feat: Set `PhotoOutput`'s maximum photo resolution

* fix: Compare dictionaries instead

* chore: Format code

* fix: Try to use hash.... failing atm

* fix: Use rough comparison again

* fix: Also take video HDR into consideration

* chore: Format

* Use contains

* Update AVCaptureDevice.Format+toDictionary.swift

* docs: Add better docs to Camera props

* Update CameraView+AVCaptureSession.swift

* Update CameraView+AVCaptureSession.swift

package/ios/Extensions/AVCaptureDevice.Format+dimensions.swift

@@ -0,0 +1,33 @@
+//
+//  AVCaptureDevice.Format+dimensions.swift
+//  VisionCamera
+//
+//  Created by Marc Rousavy on 03.08.21.
+//  Copyright © 2021 mrousavy. All rights reserved.
+//
+
+import AVFoundation
+import Foundation
+
+extension AVCaptureDevice.Format {
+  /**
+   * Returns the dimensions the video pipeline is streaming at.
+   */
+  var videoDimensions: CMVideoDimensions {
+    return CMVideoFormatDescriptionGetDimensions(formatDescription)
+  }
+
+  /**
+   Returns the maximum available photo resolution this format can use.
+   */
+  var photoDimensions: CMVideoDimensions {
+    if #available(iOS 16.0, *) {
+      if let max = supportedMaxPhotoDimensions.max(by: { left, right in
+        return left.width * left.height < right.width * right.height
+      }) {
+        return max
+      }
+    }
+    return highResolutionStillImageDimensions
+  }
+}

package/ios/Extensions/AVCaptureDevice.Format+isBetterThan.swift

@@ -1,39 +0,0 @@
-//
-//  AVCaptureDevice.Format+isBetterThan.swift
-//  mrousavy
-//
-//  Created by Marc Rousavy on 19.12.20.
-//  Copyright © 2020 mrousavy. All rights reserved.
-//
-
-import AVFoundation
-
-extension AVCaptureDevice.Format {
-  /** Compares the current Format to the given format and returns true if the current format has either:
-   * 1. Higher still image capture dimensions
-   * 2. Higher video format dimensions (iOS 13.0)
-   * 3. Higher FPS
-   */
-  func isBetterThan(_ other: AVCaptureDevice.Format) -> Bool {
-    // compare still image dimensions
-    let leftDimensions = highResolutionStillImageDimensions
-    let rightDimensions = other.highResolutionStillImageDimensions
-    if leftDimensions.height * leftDimensions.width > rightDimensions.height * rightDimensions.width {
-      return true
-    }
-
-    // compare video dimensions
-    let leftVideo = videoDimensions
-    let rightVideo = other.videoDimensions
-    if leftVideo.height * leftVideo.width > rightVideo.height * rightVideo.width {
-      return true
-    }
-
-    // compare max fps
-    if maxFrameRate > other.maxFrameRate {
-      return true
-    }
-
-    return false
-  }
-}

package/ios/Extensions/AVCaptureDevice.Format+matchesFilter.swift

@@ -1,83 +0,0 @@
-//
-//  AVCaptureDevice.Format+matchesFilter.swift
-//  mrousavy
-//
-//  Created by Marc Rousavy on 15.01.21.
-//  Copyright © 2021 mrousavy. All rights reserved.
-//
-
-import AVFoundation
-
-extension AVCaptureDevice.Format {
-  /**
-   * Checks whether the given filter (NSDictionary, JSON Object) matches the given AVCaptureDevice Format.
-   * The `dictionary` dictionary must be of type `CameraDeviceFormat` (from `CameraDevice.d.ts`)
-   */
-  func matchesFilter(_ filter: NSDictionary) -> Bool {
-    if let photoHeight = filter.value(forKey: "photoHeight") as? NSNumber {
-      if highResolutionStillImageDimensions.height != photoHeight.intValue {
-        return false
-      }
-    }
-    if let photoWidth = filter.value(forKey: "photoWidth") as? NSNumber {
-      if highResolutionStillImageDimensions.width != photoWidth.intValue {
-        return false
-      }
-    }
-    if let videoHeight = filter.value(forKey: "videoHeight") as? NSNumber {
-      if videoDimensions.height != CGFloat(videoHeight.doubleValue) {
-        return false
-      }
-    }
-    if let videoWidth = filter.value(forKey: "videoWidth") as? NSNumber {
-      if videoDimensions.width != CGFloat(videoWidth.doubleValue) {
-        return false
-      }
-    }
-    if let maxISO = filter.value(forKey: "maxISO") as? NSNumber {
-      if self.maxISO != maxISO.floatValue {
-        return false
-      }
-    }
-    if let minISO = filter.value(forKey: "minISO") as? NSNumber {
-      if self.minISO != minISO.floatValue {
-        return false
-      }
-    }
-    if let fieldOfView = filter.value(forKey: "fieldOfView") as? NSNumber {
-      if videoFieldOfView != fieldOfView.floatValue {
-        return false
-      }
-    }
-    if let maxZoom = filter.value(forKey: "maxZoom") as? NSNumber {
-      if videoMaxZoomFactor != CGFloat(maxZoom.doubleValue) {
-        return false
-      }
-    }
-    if let minFps = filter.value(forKey: "minFps") as? NSNumber {
-      if minFrameRate != Float64(minFps.doubleValue) {
-        return false
-      }
-    }
-    if let maxFps = filter.value(forKey: "maxFps") as? NSNumber {
-      if maxFrameRate != Float64(maxFps.doubleValue) {
-        return false
-      }
-    }
-    if let autoFocusSystem = filter.value(forKey: "autoFocusSystem") as? String,
-       let avAutoFocusSystem = try? AVCaptureDevice.Format.AutoFocusSystem(withString: autoFocusSystem) {
-      if self.autoFocusSystem != avAutoFocusSystem {
-        return false
-      }
-    }
-    if let videoStabilizationModes = filter.value(forKey: "videoStabilizationModes") as? [String] {
-      let avVideoStabilizationModes = videoStabilizationModes.map { try? AVCaptureVideoStabilizationMode(withString: $0) }
-      let allStabilizationModesIncluded = self.videoStabilizationModes.allSatisfy { avVideoStabilizationModes.contains($0) }
-      if !allStabilizationModesIncluded {
-        return false
-      }
-    }
-
-    return true
-  }
-}

package/ios/Extensions/AVCaptureDevice.Format+toDictionary.swift

@@ -35,22 +35,32 @@ extension AVCaptureDevice.Format {
     return maxRange?.maxFrameRate ?? 0
   }
 
-  func toDictionary() -> [String: Any] {
+  var supportsVideoHDR: Bool {
+    let pixelFormat = CMFormatDescriptionGetMediaSubType(formatDescription)
+    let hdrFormats = [
+      kCVPixelFormatType_420YpCbCr10BiPlanarFullRange,
+      kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange,
+      kCVPixelFormatType_Lossless_420YpCbCr10PackedBiPlanarVideoRange,
+    ]
+    return hdrFormats.contains(pixelFormat)
+  }
+
+  func toDictionary() -> [String: AnyHashable] {
     let availablePixelFormats = AVCaptureVideoDataOutput().availableVideoPixelFormatTypes
     let pixelFormats = availablePixelFormats.map { format in PixelFormat(mediaSubType: format) }
 
     return [
       "videoStabilizationModes": videoStabilizationModes.map(\.descriptor),
       "autoFocusSystem": autoFocusSystem.descriptor,
-      "photoHeight": highResolutionStillImageDimensions.height,
-      "photoWidth": highResolutionStillImageDimensions.width,
+      "photoHeight": photoDimensions.height,
+      "photoWidth": photoDimensions.width,
       "videoHeight": videoDimensions.height,
       "videoWidth": videoDimensions.width,
       "maxISO": maxISO,
       "minISO": minISO,
       "fieldOfView": videoFieldOfView,
       "maxZoom": videoMaxZoomFactor,
-      "supportsVideoHDR": availablePixelFormats.contains(kCVPixelFormatType_420YpCbCr10BiPlanarFullRange),
+      "supportsVideoHDR": supportsVideoHDR,
       "supportsPhotoHDR": false,
       "minFps": minFrameRate,
       "maxFps": maxFrameRate,
@@ -58,4 +68,29 @@ extension AVCaptureDevice.Format {
       "supportsDepthCapture": !supportedDepthDataFormats.isEmpty,
     ]
   }
+
+  /**
+   Compares this format to the given JS `CameraDeviceFormat`.
+   Only the most important properties (such as dimensions and FPS) are taken into consideration,
+   so this is not an exact equals, but more like a "matches filter" comparison.
+   */
+  func isEqualTo(jsFormat dict: NSDictionary) -> Bool {
+    guard dict["photoWidth"] as? Int32 == photoDimensions.width && dict["photoHeight"] as? Int32 == photoDimensions.height else {
+      return false
+    }
+
+    guard dict["videoWidth"] as? Int32 == videoDimensions.width && dict["videoHeight"] as? Int32 == videoDimensions.height else {
+      return false
+    }
+
+    guard dict["minFps"] as? Float64 == minFrameRate && dict["maxFps"] as? Float64 == maxFrameRate else {
+      return false
+    }
+
+    guard dict["supportsVideoHDR"] as? Bool == supportsVideoHDR else {
+      return false
+    }
+
+    return true
+  }
 }

package/ios/Extensions/AVCaptureDevice.Format+videoDimensions.swift

@@ -1,24 +0,0 @@
-//
-//  AVCaptureDevice.Format+videoDimensions.swift
-//  VisionCamera
-//
-//  Created by Marc Rousavy on 03.08.21.
-//  Copyright © 2021 mrousavy. All rights reserved.
-//
-
-import AVFoundation
-import Foundation
-
-extension AVCaptureDevice.Format {
-  /**
-   * Returns the video dimensions, adjusted to take pixel aspect ratio and/or clean
-   * aperture into account.
-   *
-   * Pixel aspect ratio is used to adjust the width, leaving the height alone.
-   */
-  var videoDimensions: CGSize {
-    return CMVideoFormatDescriptionGetPresentationDimensions(formatDescription,
-                                                             usePixelAspectRatio: true,
-                                                             useCleanAperture: true)
-  }
-}

package/ios/Extensions/CMVideoDimensions+toCGSize.swift

@@ -0,0 +1,16 @@
+//
+//  CMVideoDimensions+toCGSize.swift
+//  VisionCamera
+//
+//  Created by Marc Rousavy on 05.10.23.
+//  Copyright © 2023 mrousavy. All rights reserved.
+//
+
+import AVFoundation
+import Foundation
+
+extension CMVideoDimensions {
+  func toCGSize() -> CGSize {
+    return CGSize(width: Int(width), height: Int(height))
+  }
+}