feat: Complete iOS Codebase rewrite (#1647)

* Make Frame Processors an extra subspec * Update VisionCamera.podspec * Make optional * Make VisionCamera compile without Skia * Fix * Add skia again * Update VisionCamera.podspec * Make VisionCamera build without Frame Processors * Rename error to `system/frame-processors-unavailable` * Fix Frame Processor returning early * Remove `preset`, FP partial rewrite * Only warn on frame drop * Fix wrong queue * fix: Run on CameraQueue again * Update CameraView.swift * fix: Activate audio session asynchronously on audio queue * Update CameraView+RecordVideo.swift * Update PreviewView.h * Cleanups * Cleanup * fix cast * feat: Add LiDAR Depth Camera support * Upgrade Ruby * Add vector icons type * Update Gemfile.lock * fix: Stop queues on deinit * Also load `builtInTrueDepthCamera` * Update CameraViewManager.swift * Update SkImageHelpers.mm * Extract FrameProcessorCallback to FrameProcessor Holds more context now :) * Rename to .m * fix: Add `RCTLog` import * Create SkiaFrameProcessor * Update CameraBridge.h * Call Frame Processor * Fix defines * fix: Allow deleting callback funcs * fix Skia build * batch * Just call `setSkiaFrameProcessor` * Rewrite in Swift * Pass `SkiaRenderer` * Fix Import * Move `PreviewView` to Swift * Fix Layer * Set Skia Canvas to Frame Host Object * Make `DrawableFrameHostObject` subclass * Fix TS types * Use same MTLDevice and apply scale * Make getter * Extract `setTorch` and `Preview` * fix: Fix nil metal device * Don't wait for session stop in deinit * Use main pixel ratio * Use unique_ptr for Render Contexts * fix: Fix SkiaPreviewDisplayLink broken after deinit * inline `getTextureCache` * Update CameraPage.tsx * chore: Format iOS * perf: Allow MTLLayer to be optimized for only frame buffers * Add RN Video types * fix: Fix Frame Processors if guard * Find nodeModules recursively * Create `Frame.isDrawable` * Add `cocoapods-check` dependency
2023-07-20 15:30:04 +02:00
parent 5fb594ce6b
commit 375e894038
78 changed files with 1278 additions and 1245 deletions
--- a/src/Camera.tsx
+++ b/src/Camera.tsx
@@ -4,8 +4,7 @@ import type { VideoFileType } from '.';
 import type { CameraDevice } from './CameraDevice';
 import type { ErrorWithCause } from './CameraError';
 import { CameraCaptureError, CameraRuntimeError, tryParseNativeCameraError, isErrorWithCause } from './CameraError';
-import type { CameraProps } from './CameraProps';
-import type { Frame } from './Frame';
+import type { CameraProps, FrameProcessor } from './CameraProps';
 import { assertFrameProcessorsAvailable, assertJSIAvailable } from './JSIHelper';
 import { CameraModule } from './NativeCameraModule';
 import type { PhotoFile, TakePhotoOptions } from './PhotoFile';
@@ -25,6 +24,7 @@ interface OnErrorEvent {
 type NativeCameraViewProps = Omit<CameraProps, 'device' | 'onInitialized' | 'onError' | 'frameProcessor'> & {
  cameraId: string;
  enableFrameProcessor: boolean;
+  previewType: 'native' | 'skia';
  onInitialized?: (event: NativeSyntheticEvent<void>) => void;
  onError?: (event: NativeSyntheticEvent<OnErrorEvent>) => void;
  onViewReady: () => void;
@@ -67,7 +67,7 @@ export class Camera extends React.PureComponent<CameraProps> {
  static displayName = 'Camera';
  /** @internal */
  displayName = Camera.displayName;
-  private lastFrameProcessor: ((frame: Frame) => void) | undefined;
+  private lastFrameProcessor: FrameProcessor | undefined;
  private isNativeViewMounted = false;

  private readonly ref: React.RefObject<RefType>;
@@ -417,7 +417,7 @@ export class Camera extends React.PureComponent<CameraProps> {
  //#endregion

  //#region Lifecycle
-  private setFrameProcessor(frameProcessor: (frame: Frame) => void): void {
+  private setFrameProcessor(frameProcessor: FrameProcessor): void {
    assertFrameProcessorsAvailable();
    // @ts-expect-error JSI functions aren't typed
    global.setFrameProcessor(this.handle, frameProcessor);
@@ -473,6 +473,7 @@ export class Camera extends React.PureComponent<CameraProps> {
        onInitialized={this.onInitialized}
        onError={this.onError}
        enableFrameProcessor={frameProcessor != null}
+        previewType={frameProcessor?.type === 'skia-frame-processor' ? 'skia' : 'native'}
      />
    );
  }
--- a/src/CameraError.ts
+++ b/src/CameraError.ts
@@ -15,14 +15,12 @@ export type DeviceError =
  | 'device/low-light-boost-not-supported'
  | 'device/focus-not-supported'
  | 'device/camera-not-available-on-simulator';
-export type FrameProcessorError = 'frame-processor/unavailable';
 export type FormatError =
  | 'format/invalid-fps'
  | 'format/invalid-hdr'
  | 'format/invalid-low-light-boost'
  | 'format/invalid-format'
-  | 'format/invalid-color-space'
-  | 'format/invalid-preset';
+  | 'format/invalid-color-space';
 export type SessionError =
  | 'session/camera-not-ready'
  | 'session/audio-session-setup-failed'
@@ -50,7 +48,12 @@ export type CaptureError =
  | 'capture/photo-not-enabled'
  | 'capture/aborted'
  | 'capture/unknown';
-export type SystemError = 'system/camera-module-not-found' | 'system/no-camera-manager' | 'system/view-not-found';
+export type SystemError =
+  | 'system/camera-module-not-found'
+  | 'system/no-camera-manager'
+  | 'system/frame-processors-unavailable'
+  | 'system/skia-unavailable'
+  | 'system/view-not-found';
 export type UnknownError = 'unknown/unknown';

 /**
@@ -105,7 +108,6 @@ type CameraErrorCode =
  | PermissionError
  | ParameterError
  | DeviceError
-  | FrameProcessorError
  | FormatError
  | SessionError
  | CaptureError
@@ -162,7 +164,7 @@ export class CameraCaptureError extends CameraError<CaptureError> {}
 * See the ["Camera Errors" documentation](https://mrousavy.github.io/react-native-vision-camera/docs/guides/errors) for more information about Camera Errors.
 */
 export class CameraRuntimeError extends CameraError<
-  PermissionError | ParameterError | DeviceError | FormatError | FrameProcessorError | SessionError | SystemError | UnknownError
+  PermissionError | ParameterError | DeviceError | FormatError | SessionError | SystemError | UnknownError
 > {}

 /**
--- a/src/CameraPreset.ts
+++ b/src/CameraPreset.ts
@@ -1,29 +0,0 @@
-/**
- * Indicates the quality level or bit rate of the output.
- *
- * * `"cif-352x288"`: Specifies capture settings suitable for CIF quality (352 x 288 pixel) video output
- * * `"hd-1280x720"`: Specifies capture settings suitable for 720p quality (1280 x 720 pixel) video output.
- * * `"hd-1920x1080"`: Capture settings suitable for 1080p-quality (1920 x 1080 pixels) video output.
- * * `"hd-3840x2160"`: Capture settings suitable for 2160p-quality (3840 x 2160 pixels, "4k") video output.
- * * `"high"`: Specifies capture settings suitable for high-quality video and audio output.
- * * `"iframe-1280x720"`: Specifies capture settings to achieve 1280 x 720 quality iFrame H.264 video at about 40 Mbits/sec with AAC audio.
- * * `"iframe-960x540"`: Specifies capture settings to achieve 960 x 540 quality iFrame H.264 video at about 30 Mbits/sec with AAC audio.
- * * `"input-priority"`: Specifies that the capture session does not control audio and video output settings.
- * * `"low"`: Specifies capture settings suitable for output video and audio bit rates suitable for sharing over 3G.
- * * `"medium"`: Specifies capture settings suitable for output video and audio bit rates suitable for sharing over WiFi.
- * * `"photo"`: Specifies capture settings suitable for high-resolution photo quality output.
- * * `"vga-640x480"`: Specifies capture settings suitable for VGA quality (640 x 480 pixel) video output.
- */
-export type CameraPreset =
-  | 'cif-352x288'
-  | 'hd-1280x720'
-  | 'hd-1920x1080'
-  | 'hd-3840x2160'
-  | 'high'
-  | 'iframe-1280x720'
-  | 'iframe-960x540'
-  | 'input-priority'
-  | 'low'
-  | 'medium'
-  | 'photo'
-  | 'vga-640x480';
--- a/src/CameraProps.ts
+++ b/src/CameraProps.ts
@@ -1,10 +1,19 @@
 import type { ViewProps } from 'react-native';
 import type { CameraDevice, CameraDeviceFormat, ColorSpace, VideoStabilizationMode } from './CameraDevice';
 import type { CameraRuntimeError } from './CameraError';
-import type { CameraPreset } from './CameraPreset';
-import type { Frame } from './Frame';
+import type { DrawableFrame, Frame } from './Frame';
 import type { Orientation } from './Orientation';

+export type FrameProcessor =
+  | {
+      frameProcessor: (frame: Frame) => void;
+      type: 'frame-processor';
+    }
+  | {
+      frameProcessor: (frame: DrawableFrame) => void;
+      type: 'skia-frame-processor';
+    };
+
 export interface CameraProps extends ViewProps {
  /**
   * The Camera Device to use.
@@ -85,11 +94,7 @@ export interface CameraProps extends ViewProps {

  //#region Format/Preset selection
  /**
-   * Automatically selects a camera format which best matches the given preset. Must be `undefined` when `format` is set!
-   */
-  preset?: CameraPreset;
-  /**
-   * Selects a given format. Must be `undefined` when `preset` is set!
+   * Selects a given format. By default, the best matching format is chosen.
   */
  format?: CameraDeviceFormat;
  /**
@@ -163,15 +168,6 @@ export interface CameraProps extends ViewProps {
   * Represents the orientation of all Camera Outputs (Photo, Video, and Frame Processor). If this value is not set, the device orientation is used.
   */
  orientation?: Orientation;
-  /**
-   * Render type of the Camera Preview Layer.
-   *
-   * * `native`: Uses the default platform native preview Layer. Uses less resources and is more efficient.
-   * * `skia`: Uses a Skia Canvas for rendering Camera frames to the screen. This allows you to draw to the screen using the react-native-skia API inside a Frame Processor.
-   *
-   * @default 'native'
-   */
-  previewType?: 'native' | 'skia';

  //#region Events
  /**
@@ -185,7 +181,7 @@ export interface CameraProps extends ViewProps {
  /**
   * A worklet which will be called for every frame the Camera "sees".
   *
-   * If {@linkcode CameraProps.previewType | previewType} is set to `"skia"`, you can draw content to the `Frame` using the react-native-skia API.
+   * If {@linkcode previewType | previewType} is set to `"skia"`, you can draw content to the `Frame` using the react-native-skia API.
   *
   * Note: If you want to use `video` and `frameProcessor` simultaneously, make sure [`supportsParallelVideoProcessing`](https://mrousavy.github.io/react-native-vision-camera/docs/guides/devices#the-supportsparallelvideoprocessing-prop) is `true`.
   *
@@ -202,6 +198,6 @@ export interface CameraProps extends ViewProps {
   * return <Camera {...cameraProps} frameProcessor={frameProcessor} />
   * ```
   */
-  frameProcessor?: (frame: Frame) => void;
+  frameProcessor?: FrameProcessor;
  //#endregion
 }
--- a/src/Frame.ts
+++ b/src/Frame.ts
@@ -4,7 +4,7 @@ import type { Orientation } from './Orientation';
 /**
 * A single frame, as seen by the camera.
 */
-export interface Frame extends SkCanvas {
+export interface Frame {
  /**
   * Whether the underlying buffer is still valid or not. The buffer will be released after the frame processor returns, or `close()` is called.
   */
@@ -55,6 +55,14 @@ export interface Frame extends SkCanvas {
   * ```
   */
  toString(): string;
+  /**
+   * Whether the Frame can be drawn onto using Skia.
+   * Always false for `useFrameProcessor`. Use `useSkiaFrameProcessor` instead.
+   */
+  isDrawable: boolean;
+}
+
+export interface DrawableFrame extends Frame, SkCanvas {
  /**
   * Renders the Frame to the screen.
   *
@@ -77,7 +85,7 @@ export interface Frame extends SkCanvas {
   * const paint = Skia.Paint()
   * paint.setImageFilter(imageFilter)
   *
-   * const frameProcessor = useFrameProcessor((frame) => {
+   * const frameProcessor = useSkiaFrameProcessor((frame) => {
   *   'worklet'
   *   frame.render(paint) // <-- draws frame with inverted colors now
   * }, [paint])
@@ -86,7 +94,7 @@ export interface Frame extends SkCanvas {
  render: (paint?: SkPaint) => void;
 }

-export interface FrameInternal extends Frame {
+export interface FrameInternal extends Frame, DrawableFrame {
  /**
   * Increment the Frame Buffer ref-count by one.
   *
--- a/src/JSIHelper.ts
+++ b/src/JSIHelper.ts
@@ -5,7 +5,7 @@ export function assertJSIAvailable(): void {
  // @ts-expect-error JSI functions aren't typed
  if (global.nativeCallSyncHook == null) {
    throw new CameraRuntimeError(
-      'frame-processor/unavailable',
+      'system/frame-processors-unavailable',
      'Failed to initialize VisionCamera Frame Processors: React Native is not running on-device. Frame Processors can only be used when synchronous method invocations (JSI) are possible. If you are using a remote debugger (e.g. Chrome), switch to an on-device debugger (e.g. Flipper) instead.',
    );
  }
@@ -17,7 +17,7 @@ export function assertFrameProcessorsAvailable(): void {
  // @ts-expect-error JSI functions aren't typed
  if (global.setFrameProcessor == null || global.unsetFrameProcessor == null) {
    throw new CameraRuntimeError(
-      'frame-processor/unavailable',
+      'system/frame-processors-unavailable',
      'Frame Processors are not enabled. See https://mrousavy.github.io/react-native-vision-camera/docs/guides/troubleshooting',
    );
  }
--- a/src/PhotoFile.ts
+++ b/src/PhotoFile.ts
@@ -1,4 +1,5 @@
 import type { TemporaryFile } from './TemporaryFile';
+import { CameraPhotoCodec } from './VideoFile';

 export interface TakePhotoOptions {
  /**
@@ -36,6 +37,10 @@ export interface TakePhotoOptions {
   * @default false
   */
  enableAutoDistortionCorrection?: boolean;
+  /**
+   * Specifies the photo codec to use for this capture. The provided photo codec has to be supported by the session.
+   */
+  photoCodec?: CameraPhotoCodec;
  /**
   * When set to `true`, metadata reading and mapping will be skipped. ({@linkcode PhotoFile.metadata} will be null)
   *
--- a/src/VideoFile.ts
+++ b/src/VideoFile.ts
@@ -3,16 +3,8 @@ import type { TemporaryFile } from './TemporaryFile';

 export type VideoFileType = 'mov' | 'avci' | 'm4v' | 'mp4';

-export type CameraVideoCodec =
-  | 'h264'
-  | 'hevc'
-  | 'hevc-alpha'
-  | 'jpeg'
-  | 'pro-res-4444'
-  | 'pro-res-422'
-  | 'pro-res-422-hq'
-  | 'pro-res-422-lt'
-  | 'pro-res-422-proxy';
+export type CameraVideoCodec = 'h264' | 'hevc' | 'hevc-alpha';
+export type CameraPhotoCodec = 'jpeg' | 'pro-res-4444' | 'pro-res-422' | 'pro-res-422-hq' | 'pro-res-422-lt' | 'pro-res-422-proxy';

 export interface RecordVideoOptions {
  /**
--- a/src/hooks/useFrameProcessor.ts
+++ b/src/hooks/useFrameProcessor.ts
@@ -1,12 +1,12 @@
-import { DependencyList, useCallback } from 'react';
-import type { Frame, FrameInternal } from '../Frame';
+import { DependencyList, useMemo } from 'react';
+import type { DrawableFrame, Frame, FrameInternal } from '../Frame';
+import { FrameProcessor } from '../CameraProps';
 // Install RN Worklets by importing it
 import 'react-native-worklets/src';

-type FrameProcessor = (frame: Frame) => void;
-
 /**
- * Returns a memoized Frame Processor function wich you can pass to the `<Camera>`. (See ["Frame Processors"](https://mrousavy.github.io/react-native-vision-camera/docs/guides/frame-processors))
+ * Returns a memoized Frame Processor function wich you can pass to the `<Camera>`.
+ * (See ["Frame Processors"](https://mrousavy.github.io/react-native-vision-camera/docs/guides/frame-processors))
 *
 * Make sure to add the `'worklet'` directive to the top of the Frame Processor function, otherwise it will not get compiled into a worklet.
 *
@@ -22,18 +22,66 @@ type FrameProcessor = (frame: Frame) => void;
 * }, [])
 * ```
 */
-export function useFrameProcessor(frameProcessor: FrameProcessor, dependencies: DependencyList): FrameProcessor {
-  return useCallback((frame: Frame) => {
-    'worklet';
-    // Increment ref-count by one
-    (frame as FrameInternal).incrementRefCount();
-    try {
-      // Call sync frame processor
-      frameProcessor(frame);
-    } finally {
-      // Potentially delete Frame if we were the last ref (no runAsync)
-      (frame as FrameInternal).decrementRefCount();
-    }
+export function useFrameProcessor(frameProcessor: (frame: Frame) => void, dependencies: DependencyList): FrameProcessor {
+  return useMemo(
+    () => ({
+      frameProcessor: (frame: Frame) => {
+        'worklet';
+        // Increment ref-count by one
+        (frame as FrameInternal).incrementRefCount();
+        try {
+          // Call sync frame processor
+          frameProcessor(frame);
+        } finally {
+          // Potentially delete Frame if we were the last ref (no runAsync)
+          (frame as FrameInternal).decrementRefCount();
+        }
+      },
+      type: 'frame-processor',
+    }),
    // eslint-disable-next-line react-hooks/exhaustive-deps
-  }, dependencies);
+    dependencies,
+  );
+}
+
+/**
+ * Returns a memoized Skia Frame Processor function wich you can pass to the `<Camera>`.
+ * The Skia Frame Processor allows you to draw anything onto the Frame using react-native-skia.
+ * (See ["Frame Processors"](https://mrousavy.github.io/react-native-vision-camera/docs/guides/frame-processors))
+ *
+ * Make sure to add the `'worklet'` directive to the top of the Frame Processor function, otherwise it will not get compiled into a worklet.
+ *
+ * @param frameProcessor The Frame Processor
+ * @param dependencies The React dependencies which will be copied into the VisionCamera JS-Runtime.
+ * @returns The memoized Frame Processor.
+ * @example
+ * ```ts
+ * const frameProcessor = useSkiaFrameProcessor((frame) => {
+ *   'worklet'
+ *   const qrCodes = scanQRCodes(frame)
+ *   frame.drawRect(...)
+ *   console.log(`QR Codes: ${qrCodes}`)
+ * }, [])
+ * ```
+ */
+export function useSkiaFrameProcessor(frameProcessor: (frame: DrawableFrame) => void, dependencies: DependencyList): FrameProcessor {
+  return useMemo(
+    () => ({
+      frameProcessor: (frame: DrawableFrame) => {
+        'worklet';
+        // Increment ref-count by one
+        (frame as FrameInternal).incrementRefCount();
+        try {
+          // Call sync frame processor
+          frameProcessor(frame);
+        } finally {
+          // Potentially delete Frame if we were the last ref (no runAsync)
+          (frame as FrameInternal).decrementRefCount();
+        }
+      },
+      type: 'skia-frame-processor',
+    }),
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    dependencies,
+  );
 }
--- a/src/index.ts
+++ b/src/index.ts
@@ -2,7 +2,6 @@ export * from './Camera';
 export * from './CameraDevice';
 export * from './CameraError';
 export * from './CameraPosition';
-export * from './CameraPreset';
 export * from './CameraProps';
 export { Frame } from './Frame';
 export * from './FrameProcessorPlugins';