feat: Expose unified VisionCameraProxy object, make FrameProcessorPlugins object-oriented (#1660)

* feat: Replace `FrameProcessorRuntimeManager` with `VisionCameraProxy` (iOS)

* Make `FrameProcessorPlugin` a constructable HostObject

* fix: Fix `name` override

* Simplify `useFrameProcessor

* fix: Fix lint errors

* Remove FrameProcessorPlugin::name

* JSIUtils -> JSINSObjectConversion

src/Camera.tsx

@@ -1,16 +1,16 @@
 import React from 'react';
 import { requireNativeComponent, NativeSyntheticEvent, findNodeHandle, NativeMethods, Platform } from 'react-native';
-import type { VideoFileType } from '.';
 import type { CameraDevice } from './CameraDevice';
 import type { ErrorWithCause } from './CameraError';
 import { CameraCaptureError, CameraRuntimeError, tryParseNativeCameraError, isErrorWithCause } from './CameraError';
 import type { CameraProps, FrameProcessor } from './CameraProps';
-import { assertFrameProcessorsAvailable, assertJSIAvailable } from './JSIHelper';
+import { assertJSIAvailable } from './JSIHelper';
 import { CameraModule } from './NativeCameraModule';
 import type { PhotoFile, TakePhotoOptions } from './PhotoFile';
 import type { Point } from './Point';
 import type { TakeSnapshotOptions } from './Snapshot';
-import type { CameraVideoCodec, RecordVideoOptions, VideoFile } from './VideoFile';
+import type { CameraVideoCodec, RecordVideoOptions, VideoFile, VideoFileType } from './VideoFile';
+import { VisionCameraProxy } from './FrameProcessorPlugins';
 
 //#region Types
 export type CameraPermissionStatus = 'authorized' | 'not-determined' | 'denied' | 'restricted';
@@ -82,7 +82,7 @@ export class Camera extends React.PureComponent<CameraProps> {
     this.lastFrameProcessor = undefined;
   }
 
-  private get handle(): number | null {
+  private get handle(): number {
     const nodeHandle = findNodeHandle(this.ref.current);
     if (nodeHandle == null || nodeHandle === -1) {
       throw new CameraRuntimeError(
@@ -312,7 +312,8 @@ export class Camera extends React.PureComponent<CameraProps> {
   public static installFrameProcessorBindings(): void {
     assertJSIAvailable();
     const result = CameraModule.installFrameProcessorBindings() as unknown;
-    if (result !== true) throw new Error('Failed to install Frame Processor JSI bindings!');
+    if (result !== true)
+      throw new CameraRuntimeError('system/frame-processors-unavailable', 'Failed to install Frame Processor JSI bindings!');
   }
 
   /**
@@ -418,15 +419,11 @@ export class Camera extends React.PureComponent<CameraProps> {
 
   //#region Lifecycle
   private setFrameProcessor(frameProcessor: FrameProcessor): void {
-    assertFrameProcessorsAvailable();
-    // @ts-expect-error JSI functions aren't typed
-    global.setFrameProcessor(this.handle, frameProcessor);
+    VisionCameraProxy.setFrameProcessor(this.handle, frameProcessor);
   }
 
   private unsetFrameProcessor(): void {
-    assertFrameProcessorsAvailable();
-    // @ts-expect-error JSI functions aren't typed
-    global.unsetFrameProcessor(this.handle);
+    VisionCameraProxy.removeFrameProcessor(this.handle);
   }
 
   private onViewReady(): void {

src/FrameProcessorPlugins.ts

@@ -1,26 +1,50 @@
 import type { Frame, FrameInternal } from './Frame';
+import type { FrameProcessor } from './CameraProps';
 import { Camera } from './Camera';
 import { Worklets } from 'react-native-worklets/src';
+import { CameraRuntimeError } from './CameraError';
+
+type BasicParameterType = string | number | boolean | undefined;
+type ParameterType = BasicParameterType | BasicParameterType[] | Record<string, BasicParameterType | undefined>;
+
+interface FrameProcessorPlugin {
+  /**
+   * Call the native Frame Processor Plugin with the given Frame and options.
+   * @param frame The Frame from the Frame Processor.
+   * @param options (optional) Additional options. Options will be converted to a native dictionary
+   * @returns (optional) A value returned from the native Frame Processor Plugin (or undefined)
+   */
+  call: (frame: Frame, options?: Record<string, ParameterType>) => ParameterType;
+}
+
+interface TVisionCameraProxy {
+  setFrameProcessor: (viewTag: number, frameProcessor: FrameProcessor) => void;
+  removeFrameProcessor: (viewTag: number) => void;
+  /**
+   * Creates a new instance of a Frame Processor Plugin.
+   * The Plugin has to be registered on the native side, otherwise this returns `undefined`
+   */
+  getFrameProcessorPlugin: (name: string) => FrameProcessorPlugin | undefined;
+  isSkiaEnabled: boolean;
+}
 
-// Install VisionCamera Frame Processor JSI Bindings and Plugins
 Camera.installFrameProcessorBindings();
 
+// @ts-expect-error global is untyped, it's a C++ host-object
+export const VisionCameraProxy = global.VisionCameraProxy as TVisionCameraProxy;
+// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+if (VisionCameraProxy == null) {
+  throw new CameraRuntimeError(
+    'system/frame-processors-unavailable',
+    'Failed to install VisionCameraProxy. Are Frame Processors properly enabled?',
+  );
+}
+
 declare global {
   // eslint-disable-next-line no-var
   var __frameProcessorRunAtTargetFpsMap: Record<string, number | undefined> | undefined;
 }
 
-type BasicParameterType = string | number | boolean | undefined;
-type ParameterType = BasicParameterType | BasicParameterType[] | Record<string, BasicParameterType | undefined>;
-type FrameProcessor = (frame: Frame, parameters?: Record<string, ParameterType | undefined>) => unknown;
-type TFrameProcessorPlugins = Record<string, FrameProcessor>;
-
-/**
- * All natively installed Frame Processor Plugins.
- */
-// @ts-expect-error The global JSI Proxy object is not typed.
-export const FrameProcessorPlugins = global.FrameProcessorPlugins as TFrameProcessorPlugins;
-
 function getLastFrameProcessorCall(frameProcessorFuncId: string): number {
   'worklet';
   return global.__frameProcessorRunAtTargetFpsMap?.[frameProcessorFuncId] ?? 0;

src/JSIHelper.ts

@@ -10,15 +10,3 @@ export function assertJSIAvailable(): void {
     );
   }
 }
-
-export function assertFrameProcessorsAvailable(): void {
-  assertJSIAvailable();
-
-  // @ts-expect-error JSI functions aren't typed
-  if (global.setFrameProcessor == null || global.unsetFrameProcessor == null) {
-    throw new CameraRuntimeError(
-      'system/frame-processors-unavailable',
-      'Frame Processors are not enabled. See https://mrousavy.github.io/react-native-vision-camera/docs/guides/troubleshooting',
-    );
-  }
-}

src/expo-plugin/withDisableFrameProcessorsIOS.ts

@@ -1,28 +1,13 @@
-/* eslint-disable @typescript-eslint/no-unsafe-assignment */
-import { ConfigPlugin, withXcodeProject, XcodeProject } from '@expo/config-plugins';
+import { ConfigPlugin, withPodfileProperties } from '@expo/config-plugins';
 
 /**
  * Set the `disableFrameProcessors` inside of the XcodeProject.
  * This is used to disable frame processors if you don't need it on iOS. (will save CPU and Memory)
  */
 export const withDisableFrameProcessorsIOS: ConfigPlugin = (c) => {
-  return withXcodeProject(c, (config) => {
-    const xcodeProject: XcodeProject = config.modResults;
-
-    const configurations = xcodeProject.pbxXCBuildConfigurationSection();
-
-    const inheritKey = '"$(inherited)"';
-    const valueKey = '"VISION_CAMERA_DISABLE_FRAME_PROCESSORS=1"';
-
-    for (const key in configurations) {
-      const buildSettings = configurations[key].buildSettings;
-      if (buildSettings == null) continue;
-
-      const preprocessorDefinitions = (buildSettings.GCC_PREPROCESSOR_DEFINITIONS ?? [inheritKey]) as string[];
-
-      if (!preprocessorDefinitions.includes(valueKey)) preprocessorDefinitions.push(valueKey);
-      buildSettings.GCC_PREPROCESSOR_DEFINITIONS = preprocessorDefinitions;
-    }
+  return withPodfileProperties(c, (config) => {
+    // TODO: Implement Podfile writing
+    config.ios = config.ios;
     return config;
   });
 };

src/hooks/useFrameProcessor.ts

@@ -4,6 +4,25 @@ import { FrameProcessor } from '../CameraProps';
 // Install RN Worklets by importing it
 import 'react-native-worklets/src';
 
+export function createFrameProcessor(frameProcessor: FrameProcessor['frameProcessor'], type: FrameProcessor['type']): FrameProcessor {
+  return {
+    frameProcessor: (frame: Frame | DrawableFrame) => {
+      'worklet';
+      // Increment ref-count by one
+      (frame as FrameInternal).incrementRefCount();
+      try {
+        // Call sync frame processor
+        // @ts-expect-error the frame type is ambiguous here
+        frameProcessor(frame);
+      } finally {
+        // Potentially delete Frame if we were the last ref (no runAsync)
+        (frame as FrameInternal).decrementRefCount();
+      }
+    },
+    type: type,
+  };
+}
+
 /**
  * Returns a memoized Frame Processor function wich you can pass to the `<Camera>`.
  * (See ["Frame Processors"](https://mrousavy.github.io/react-native-vision-camera/docs/guides/frame-processors))
@@ -23,25 +42,8 @@ import 'react-native-worklets/src';
  * ```
  */
 export function useFrameProcessor(frameProcessor: (frame: Frame) => void, dependencies: DependencyList): FrameProcessor {
-  return useMemo(
-    () => ({
-      frameProcessor: (frame: Frame) => {
-        'worklet';
-        // Increment ref-count by one
-        (frame as FrameInternal).incrementRefCount();
-        try {
-          // Call sync frame processor
-          frameProcessor(frame);
-        } finally {
-          // Potentially delete Frame if we were the last ref (no runAsync)
-          (frame as FrameInternal).decrementRefCount();
-        }
-      },
-      type: 'frame-processor',
-    }),
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-    dependencies,
-  );
+  // eslint-disable-next-line react-hooks/exhaustive-deps
+  return useMemo(() => createFrameProcessor(frameProcessor, 'frame-processor'), dependencies);
 }
 
 /**
@@ -65,23 +67,6 @@ export function useFrameProcessor(frameProcessor: (frame: Frame) => void, depend
  * ```
  */
 export function useSkiaFrameProcessor(frameProcessor: (frame: DrawableFrame) => void, dependencies: DependencyList): FrameProcessor {
-  return useMemo(
-    () => ({
-      frameProcessor: (frame: DrawableFrame) => {
-        'worklet';
-        // Increment ref-count by one
-        (frame as FrameInternal).incrementRefCount();
-        try {
-          // Call sync frame processor
-          frameProcessor(frame);
-        } finally {
-          // Potentially delete Frame if we were the last ref (no runAsync)
-          (frame as FrameInternal).decrementRefCount();
-        }
-      },
-      type: 'skia-frame-processor',
-    }),
-    // eslint-disable-next-line react-hooks/exhaustive-deps
-    dependencies,
-  );
+  // eslint-disable-next-line react-hooks/exhaustive-deps
+  return useMemo(() => createFrameProcessor(frameProcessor, 'skia-frame-processor'), dependencies);
 }