feat: Draw onto Frame as if it was a Skia Canvas (#1479)

* Create Shaders.ts

* Add `previewType` and `enableFpsGraph`

* Add RN Skia native dependency

* Add Skia Preview View on iOS

* Pass 1

* Update FrameHostObject.mm

* Wrap Canvas

* Lockfiles

* fix: Fix stuff

* chore: Upgrade RNWorklets

* Add `previewType` to set the Preview

* feat: Add Example

* Update project.pbxproj

* `enableFpsGraph`

* Cache the `std::shared_ptr<FrameHostObject>`

* Update CameraView+RecordVideo.swift

* Update SkiaMetalCanvasProvider.mm

* Android: Integrate Skia Dependency

* fix: Use new Prefix

* Add example for rendering shader

* chore: Upgrade CameraX

* Remove KTX

* Enable `viewBinding`

* Revert "Enable `viewBinding`"

This reverts commit f2a603f53b33ea4311a296422ffd1a910ce03f9e.

* Revert "chore: Upgrade CameraX"

This reverts commit 8dc832cf8754490d31a6192e6c1a1f11cdcd94fe.

* Remove unneeded `ProcessCameraProvider.getInstance()` call

* fix: Add REA hotfix patch

* fix: Fix FrameHostObject dead in runAsync

* fix: Make `runAsync` run truly async by dropping new Frames while executing

* chore: Upgrade RN Worklets to latest

* chore: Upgrade RN Skia

* Revert "Remove KTX"

This reverts commit 253f586633f7af2da992d2279fc206dc62597129.

* Make Skia optional in CMake

* Fix import

* Update CMakeLists.txt

* Update build.gradle

* Update CameraView.kt

* Update CameraView.kt

* Update CameraView.kt

* Update Shaders.ts

* Center Blur

* chore: Upgrade RN Worklets

* feat: Add `toByteArray()`, `orientation`, `isMirrored` and `timestamp` to `Frame` (#1487)

* feat: Implement `orientation` and `isMirrored` on Frame

* feat: Add `toArrayBuffer()` func

* perf: Do faster buffer copy

* feat: Implement `toArrayBuffer()` on Android

* feat: Add `orientation` and `isMirrored` to Android

* feat: Add `timestamp` to Frame

* Update Frame.ts

* Update JImageProxy.h

* Update FrameHostObject.cpp

* Update FrameHostObject.cpp

* Update CameraPage.tsx

* fix: Format Swift

src/CameraProps.ts

@@ -3,6 +3,7 @@ import type { CameraDevice, CameraDeviceFormat, ColorSpace, VideoStabilizationMo
 import type { CameraRuntimeError } from './CameraError';
 import type { CameraPreset } from './CameraPreset';
 import type { Frame } from './Frame';
+import type { Orientation } from './Orientation';
 
 export interface CameraProps extends ViewProps {
   /**
@@ -151,10 +152,26 @@ export interface CameraProps extends ViewProps {
    * @default false
    */
   enableHighQualityPhotos?: boolean;
+  /**
+   * If `true`, show a debug view to display the FPS of the Camera session.
+   * This is useful for debugging your Frame Processor's speed.
+   *
+   * @default false
+   */
+  enableFpsGraph?: boolean;
   /**
    * Represents the orientation of all Camera Outputs (Photo, Video, and Frame Processor). If this value is not set, the device orientation is used.
    */
-  orientation?: 'portrait' | 'portraitUpsideDown' | 'landscapeLeft' | 'landscapeRight';
+  orientation?: Orientation;
+  /**
+   * Render type of the Camera Preview Layer.
+   *
+   * * `native`: Uses the default platform native preview Layer. Uses less resources and is more efficient.
+   * * `skia`: Uses a Skia Canvas for rendering Camera frames to the screen. This allows you to draw to the screen using the react-native-skia API inside a Frame Processor.
+   *
+   * @default 'native'
+   */
+  previewType?: 'native' | 'skia';
 
   //#region Events
   /**
@@ -168,10 +185,12 @@ export interface CameraProps extends ViewProps {
   /**
    * A worklet which will be called for every frame the Camera "sees".
    *
-   * > See [the Frame Processors documentation](https://mrousavy.github.io/react-native-vision-camera/docs/guides/frame-processors) for more information
+   * If {@linkcode CameraProps.previewType | previewType} is set to `"skia"`, you can draw content to the `Frame` using the react-native-skia API.
    *
    * Note: If you want to use `video` and `frameProcessor` simultaneously, make sure [`supportsParallelVideoProcessing`](https://mrousavy.github.io/react-native-vision-camera/docs/guides/devices#the-supportsparallelvideoprocessing-prop) is `true`.
    *
+   * > See [the Frame Processors documentation](https://mrousavy.github.io/react-native-vision-camera/docs/guides/frame-processors) for more information
+   *
    * @example
    * ```tsx
    * const frameProcessor = useFrameProcessor((frame) => {

src/Frame.ts

@@ -1,7 +1,10 @@
+import type { SkCanvas, SkPaint } from '@shopify/react-native-skia';
+import type { Orientation } from './Orientation';
+
 /**
  * A single frame, as seen by the camera.
  */
-export interface Frame {
+export interface Frame extends SkCanvas {
   /**
    * Whether the underlying buffer is still valid or not. The buffer will be released after the frame processor returns, or `close()` is called.
    */
@@ -22,7 +25,28 @@ export interface Frame {
    * Returns the number of planes this frame contains.
    */
   planesCount: number;
+  /**
+   * Returns whether the Frame is mirrored (selfie camera) or not.
+   */
+  isMirrored: boolean;
+  /**
+   * Returns the timestamp of the Frame relative to the host sytem's clock.
+   */
+  timestamp: number;
+  /**
+   * Represents the orientation of the Frame.
+   *
+   * Some ML Models are trained for specific orientations, so they need to be taken into
+   * consideration when running a frame processor. See also: `isMirrored`
+   */
+  orientation: Orientation;
 
+  /**
+   * Get the underlying data of the Frame as a uint8 array buffer.
+   *
+   * Note that Frames are allocated on the GPU, so calling `toArrayBuffer()` will copy from the GPU to the CPU.
+   */
+  toArrayBuffer(): Uint8Array;
   /**
    * Returns a string representation of the frame.
    * @example
@@ -31,6 +55,35 @@ export interface Frame {
    * ```
    */
   toString(): string;
+  /**
+   * Renders the Frame to the screen.
+   *
+   * By default a Frame has already been rendered to the screen once, so if you call this method again,
+   * previously drawn content will be overwritten.
+   *
+   * @param paint (Optional) A Paint object to use to draw the Frame with. For example, this can contain a Shader (ImageFilter)
+   * @example
+   * ```ts
+   * const INVERTED_COLORS_SHADER = `
+   *  uniform shader image;
+   *  half4 main(vec2 pos) {
+   *    vec4 color = image.eval(pos);
+   *    return vec4(1.0 - color.rgb, 1.0);
+   * }`
+   * const runtimeEffect = Skia.RuntimeEffect.Make(INVERT_COLORS_SHADER)
+   * if (runtimeEffect == null) throw new Error('Shader failed to compile!')
+   * const shaderBuilder = Skia.RuntimeShaderBuilder(runtimeEffect)
+   * const imageFilter = Skia.ImageFilter.MakeRuntimeShader(shaderBuilder, null, null)
+   * const paint = Skia.Paint()
+   * paint.setImageFilter(imageFilter)
+   *
+   * const frameProcessor = useFrameProcessor((frame) => {
+   *   'worklet'
+   *   frame.render(paint) // <-- draws frame with inverted colors now
+   * }, [paint])
+   * ```
+   */
+  render: (paint?: SkPaint) => void;
 }
 
 export interface FrameInternal extends Frame {

src/FrameProcessorPlugins.ts

@@ -56,6 +56,7 @@ export function runAtTargetFps<T>(fps: number, func: () => T): T | undefined {
   return undefined;
 }
 
+const isAsyncContextBusy = Worklets.createSharedValue(false);
 const asyncContext = Worklets.createContext('VisionCamera.async');
 const runOnAsyncContext = Worklets.createRunInContextFn((frame: Frame, func: () => void) => {
   'worklet';
@@ -65,6 +66,8 @@ const runOnAsyncContext = Worklets.createRunInContextFn((frame: Frame, func: ()
   } finally {
     // Potentially delete Frame if we were the last ref
     (frame as FrameInternal).decrementRefCount();
+
+    isAsyncContextBusy.value = false;
   }
 }, asyncContext);
 
@@ -94,9 +97,18 @@ const runOnAsyncContext = Worklets.createRunInContextFn((frame: Frame, func: ()
  */
 export function runAsync(frame: Frame, func: () => void): void {
   'worklet';
+
+  if (isAsyncContextBusy.value) {
+    // async context is currently busy, we cannot schedule new work in time.
+    // drop this frame/runAsync call.
+    return;
+  }
+
   // Increment ref count by one
   (frame as FrameInternal).incrementRefCount();
 
+  isAsyncContextBusy.value = true;
+
   // Call in separate background context
   runOnAsyncContext(frame, func);
 }

src/Orientation.ts

@@ -0,0 +1 @@
+export type Orientation = 'portrait' | 'portraitUpsideDown' | 'landscapeLeft' | 'landscapeRight';