feat: Draw onto Frame as if it was a Skia Canvas (#1479)

* Create Shaders.ts

* Add `previewType` and `enableFpsGraph`

* Add RN Skia native dependency

* Add Skia Preview View on iOS

* Pass 1

* Update FrameHostObject.mm

* Wrap Canvas

* Lockfiles

* fix: Fix stuff

* chore: Upgrade RNWorklets

* Add `previewType` to set the Preview

* feat: Add Example

* Update project.pbxproj

* `enableFpsGraph`

* Cache the `std::shared_ptr<FrameHostObject>`

* Update CameraView+RecordVideo.swift

* Update SkiaMetalCanvasProvider.mm

* Android: Integrate Skia Dependency

* fix: Use new Prefix

* Add example for rendering shader

* chore: Upgrade CameraX

* Remove KTX

* Enable `viewBinding`

* Revert "Enable `viewBinding`"

This reverts commit f2a603f53b33ea4311a296422ffd1a910ce03f9e.

* Revert "chore: Upgrade CameraX"

This reverts commit 8dc832cf8754490d31a6192e6c1a1f11cdcd94fe.

* Remove unneeded `ProcessCameraProvider.getInstance()` call

* fix: Add REA hotfix patch

* fix: Fix FrameHostObject dead in runAsync

* fix: Make `runAsync` run truly async by dropping new Frames while executing

* chore: Upgrade RN Worklets to latest

* chore: Upgrade RN Skia

* Revert "Remove KTX"

This reverts commit 253f586633f7af2da992d2279fc206dc62597129.

* Make Skia optional in CMake

* Fix import

* Update CMakeLists.txt

* Update build.gradle

* Update CameraView.kt

* Update CameraView.kt

* Update CameraView.kt

* Update Shaders.ts

* Center Blur

* chore: Upgrade RN Worklets

* feat: Add `toByteArray()`, `orientation`, `isMirrored` and `timestamp` to `Frame` (#1487)

* feat: Implement `orientation` and `isMirrored` on Frame

* feat: Add `toArrayBuffer()` func

* perf: Do faster buffer copy

* feat: Implement `toArrayBuffer()` on Android

* feat: Add `orientation` and `isMirrored` to Android

* feat: Add `timestamp` to Frame

* Update Frame.ts

* Update JImageProxy.h

* Update FrameHostObject.cpp

* Update FrameHostObject.cpp

* Update CameraPage.tsx

* fix: Format Swift

android/CMakeLists.txt

@@ -17,10 +17,16 @@ find_package(fbjni REQUIRED CONFIG)
 find_package(react-native-worklets REQUIRED CONFIG)
 find_library(LOG_LIB log)
 
+# Skia is optional, if it's enabled we link it.
+if (ENABLE_SKIA_INTEGRATION)
+    find_package(shopify_react-native-skia REQUIRED CONFIG)
+endif()
+
 # Add react-native-vision-camera sources
 add_library(
         ${PACKAGE_NAME}
         SHARED
+        ../cpp/JSITypedArray.cpp
         src/main/cpp/VisionCamera.cpp
         src/main/cpp/JSIJNIConversion.cpp
         src/main/cpp/FrameHostObject.cpp
@@ -36,6 +42,7 @@ add_library(
 target_include_directories(
         ${PACKAGE_NAME}
         PRIVATE
+        "../cpp"
         "src/main/cpp"
         "${NODE_MODULES_DIR}/react-native/ReactCommon"
         "${NODE_MODULES_DIR}/react-native/ReactCommon/callinvoker"
@@ -53,3 +60,11 @@ target_link_libraries(
         fbjni::fbjni                        # <-- fbjni
         react-native-worklets::rnworklets   # <-- RN Worklets
 )
+
+# Skia is optional. If it's enabled, we link it
+if (ENABLE_SKIA_INTEGRATION)
+    target_link_libraries(
+            ${PACKAGE_NAME}
+            shopify_react-native-skia::rnskia   # <-- RN Skia
+    )
+endif()

android/build.gradle

@@ -66,6 +66,8 @@ static def findNodeModules(baseDir) {
 }
 
 def nodeModules = findNodeModules(projectDir)
+def isSkiaInstalled = findProject(":shopify_react-native-skia") != null
+logger.warn("react-native-vision-camera: Skia integration is ${isSkiaInstalled ? "enabled" : "disabled"}!")
 
 repositories {
   google()
@@ -99,7 +101,8 @@ android {
       cmake {
         cppFlags "-O2 -frtti -fexceptions -Wall -Wno-unused-variable -fstack-protector-all"
         arguments "-DANDROID_STL=c++_shared",
-                "-DNODE_MODULES_DIR=${nodeModules}"
+                "-DNODE_MODULES_DIR=${nodeModules}",
+                "-DENABLE_SKIA_INTEGRATION=${isSkiaInstalled}"
         abiFilters (*reactNativeArchitectures())
       }
     }
@@ -145,6 +148,10 @@ dependencies {
   implementation "androidx.exifinterface:exifinterface:1.3.3"
 
   implementation project(":react-native-worklets")
+
+  if (isSkiaInstalled) {
+    implementation project(":shopify_react-native-skia")
+  }
 }
 
 // Resolves "LOCAL_SRC_FILES points to a missing file, Check that libfb.so exists or that its path is correct".

android/src/main/cpp/FrameHostObject.cpp

@@ -8,7 +8,8 @@
 #include <jni.h>
 #include <vector>
 #include <string>
-#include <JsiHostObject.h>
+#include <WKTJsiHostObject.h>
+#include "JSITypedArray.h"
 
 namespace vision {
 
@@ -30,8 +31,12 @@ std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt)
   result.push_back(jsi::PropNameID::forUtf8(rt, std::string("height")));
   result.push_back(jsi::PropNameID::forUtf8(rt, std::string("bytesPerRow")));
   result.push_back(jsi::PropNameID::forUtf8(rt, std::string("planesCount")));
-  // Debugging
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("orientation")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isMirrored")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("timestamp")));
+  // Conversion
   result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toString")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toArrayBuffer")));
   // Ref Management
   result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isValid")));
   result.push_back(jsi::PropNameID::forUtf8(rt, std::string("incrementRefCount")));
@@ -54,9 +59,38 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
     };
     return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
   }
+  if (name == "toArrayBuffer") {
+    auto toArrayBuffer = JSI_HOST_FUNCTION_LAMBDA {
+      auto buffer = this->frame->toByteArray();
+      auto arraySize = buffer->size();
+
+      static constexpr auto ARRAYBUFFER_CACHE_PROP_NAME = "__frameArrayBufferCache";
+      if (!runtime.global().hasProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME)) {
+        vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray> arrayBuffer(runtime, arraySize);
+        runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer);
+      }
+
+      // Get from global JS cache
+      auto arrayBufferCache = runtime.global().getPropertyAsObject(runtime, ARRAYBUFFER_CACHE_PROP_NAME);
+      auto arrayBuffer = vision::getTypedArray(runtime, arrayBufferCache).get<vision::TypedArrayKind::Uint8ClampedArray>(runtime);
+      if (arrayBuffer.size(runtime) != arraySize) {
+        arrayBuffer = vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray>(runtime, arraySize);
+        runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer);
+      }
+
+      // directly write to C++ JSI ArrayBuffer
+      auto destinationBuffer = arrayBuffer.data(runtime);
+      buffer->getRegion(0,
+                        static_cast<jint>(arraySize),
+                        reinterpret_cast<jbyte*>(destinationBuffer));
+
+      return arrayBuffer;
+    };
+    return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toArrayBuffer"), 0, toArrayBuffer);
+  }
   if (name == "incrementRefCount") {
     auto incrementRefCount = JSI_HOST_FUNCTION_LAMBDA {
-      // Increment retain count by one so ARC doesn't destroy the Frame Buffer.
+      // Increment retain count by one.
       std::lock_guard lock(this->_refCountMutex);
       this->_refCount++;
       return jsi::Value::undefined();
@@ -69,7 +103,7 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
 
   if (name == "decrementRefCount") {
     auto decrementRefCount = JSI_HOST_FUNCTION_LAMBDA {
-      // Decrement retain count by one. If the retain count is zero, ARC will destroy the Frame Buffer.
+      // Decrement retain count by one. If the retain count is zero, we close the Frame.
       std::lock_guard lock(this->_refCountMutex);
       this->_refCount--;
       if (_refCount < 1) {
@@ -92,6 +126,16 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
   if (name == "height") {
     return jsi::Value(this->frame->getHeight());
   }
+  if (name == "isMirrored") {
+    return jsi::Value(this->frame->getIsMirrored());
+  }
+  if (name == "orientation") {
+    auto string = this->frame->getOrientation();
+    return jsi::String::createFromUtf8(runtime, string->toStdString());
+  }
+  if (name == "timestamp") {
+    return jsi::Value(static_cast<double>(this->frame->getTimestamp()));
+  }
   if (name == "bytesPerRow") {
     return jsi::Value(this->frame->getBytesPerRow());
   }

android/src/main/cpp/FrameProcessorRuntimeManager.cpp

@@ -7,8 +7,8 @@
 #include <jni.h>
 #include <utility>
 #include <string>
-#include <JsiWorklet.h>
-#include <JsiHostObject.h>
+#include <WKTJsiWorklet.h>
+#include <WKTJsiHostObject.h>
 
 #include "CameraView.h"
 #include "FrameHostObject.h"

android/src/main/cpp/FrameProcessorRuntimeManager.h

@@ -9,7 +9,7 @@
 #include <ReactCommon/CallInvokerHolder.h>
 #include <memory>
 #include <string>
-#include <JsiWorkletContext.h>
+#include <WKTJsiWorkletContext.h>
 
 #include "CameraView.h"
 #include "VisionCameraScheduler.h"

android/src/main/cpp/java-bindings/JImageProxy.cpp

@@ -33,6 +33,24 @@ bool JImageProxy::getIsValid() const {
   return isImageProxyValidMethod(utilsClass, self());
 }
 
+bool JImageProxy::getIsMirrored() const {
+  auto utilsClass = getUtilsClass();
+  static const auto isImageProxyMirroredMethod = utilsClass->getStaticMethod<jboolean(JImageProxy::javaobject)>("isImageProxyMirrored");
+  return isImageProxyMirroredMethod(utilsClass, self());
+}
+
+jlong JImageProxy::getTimestamp() const {
+    auto utilsClass = getUtilsClass();
+    static const auto getTimestampMethod = utilsClass->getStaticMethod<jlong(JImageProxy::javaobject)>("getTimestamp");
+    return getTimestampMethod(utilsClass, self());
+}
+
+local_ref<JString> JImageProxy::getOrientation() const {
+  auto utilsClass = getUtilsClass();
+  static const auto getOrientationMethod = utilsClass->getStaticMethod<JString(JImageProxy::javaobject)>("getOrientation");
+  return getOrientationMethod(utilsClass, self());
+}
+
 int JImageProxy::getPlanesCount() const {
   auto utilsClass = getUtilsClass();
   static const auto getPlanesCountMethod = utilsClass->getStaticMethod<jint(JImageProxy::javaobject)>("getPlanesCount");
@@ -45,6 +63,13 @@ int JImageProxy::getBytesPerRow() const {
   return getBytesPerRowMethod(utilsClass, self());
 }
 
+local_ref<JArrayByte> JImageProxy::toByteArray() const {
+  auto utilsClass = getUtilsClass();
+
+  static const auto toByteArrayMethod = utilsClass->getStaticMethod<JArrayByte(JImageProxy::javaobject)>("toByteArray");
+  return toByteArrayMethod(utilsClass, self());
+}
+
 void JImageProxy::close() {
   static const auto closeMethod = getClass()->getMethod<void()>("close");
   closeMethod(self());

android/src/main/cpp/java-bindings/JImageProxy.h

@@ -19,8 +19,12 @@ struct JImageProxy : public JavaClass<JImageProxy> {
   int getWidth() const;
   int getHeight() const;
   bool getIsValid() const;
+  bool getIsMirrored() const;
   int getPlanesCount() const;
   int getBytesPerRow() const;
+  jlong getTimestamp() const;
+  local_ref<JString> getOrientation() const;
+  local_ref<JArrayByte> toByteArray() const;
   void close();
 };
 

android/src/main/java/com/mrousavy/camera/CameraView.kt

@@ -123,8 +123,6 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
 
   internal var activeVideoRecording: Recording? = null
 
-  private var lastFrameProcessorCall = System.currentTimeMillis()
-
   private var extensionsManager: ExtensionsManager? = null
 
   private val scaleGestureListener: ScaleGestureDetector.SimpleOnScaleGestureListener
@@ -326,7 +324,7 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
   /**
    * Configures the camera capture session. This should only be called when the camera device changes.
    */
-  @SuppressLint("RestrictedApi")
+  @SuppressLint("RestrictedApi", "UnsafeOptInUsageError")
   private suspend fun configureSession() {
     try {
       val startTime = System.currentTimeMillis()
@@ -461,10 +459,11 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
       if (enableFrameProcessor) {
         Log.i(TAG, "Adding ImageAnalysis use-case...")
         imageAnalysis = imageAnalysisBuilder.build().apply {
-          setAnalyzer(cameraExecutor, { image ->
+          setAnalyzer(cameraExecutor) { image ->
             // Call JS Frame Processor
             frameProcessorCallback(image)
-          })
+            // frame gets closed in FrameHostObject implementation (JS ref counting)
+          }
         }
         useCases.add(imageAnalysis!!)
       }

android/src/main/java/com/mrousavy/camera/CameraViewModule.kt

@@ -172,7 +172,6 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase
       withPromise(promise) {
         val cameraProvider = ProcessCameraProvider.getInstance(reactApplicationContext).await()
         val extensionsManager = ExtensionsManager.getInstanceAsync(reactApplicationContext, cameraProvider).await()
-        ProcessCameraProvider.getInstance(reactApplicationContext).await()
 
         val manager = reactApplicationContext.getSystemService(Context.CAMERA_SERVICE) as? CameraManager
           ?: throw CameraManagerUnavailableError()

android/src/main/java/com/mrousavy/camera/frameprocessor/ImageProxyUtils.java

@@ -1,12 +1,16 @@
 package com.mrousavy.camera.frameprocessor;
 
 import android.annotation.SuppressLint;
+import android.graphics.ImageFormat;
+import android.graphics.Matrix;
 import android.media.Image;
 
 import androidx.annotation.Keep;
 import androidx.camera.core.ImageProxy;
 import com.facebook.proguard.annotations.DoNotStrip;
 
+import java.nio.ByteBuffer;
+
 @SuppressWarnings("unused") // used through JNI
 @DoNotStrip
 @Keep
@@ -28,6 +32,33 @@ public class ImageProxyUtils {
         }
     }
 
+    @DoNotStrip
+    @Keep
+    public static boolean isImageProxyMirrored(ImageProxy imageProxy) {
+        Matrix matrix = imageProxy.getImageInfo().getSensorToBufferTransformMatrix();
+        // TODO: Figure out how to get isMirrored from ImageProxy
+        return false;
+    }
+
+    @DoNotStrip
+    @Keep
+    public static String getOrientation(ImageProxy imageProxy) {
+        int rotation = imageProxy.getImageInfo().getRotationDegrees();
+        if (rotation >= 45 && rotation < 135)
+            return "landscapeRight";
+        if (rotation >= 135 && rotation < 225)
+            return "portraitUpsideDown";
+        if (rotation >= 225 && rotation < 315)
+            return "landscapeLeft";
+        return "portrait";
+    }
+
+    @DoNotStrip
+    @Keep
+    public static long getTimestamp(ImageProxy imageProxy) {
+        return imageProxy.getImageInfo().getTimestamp();
+    }
+
     @DoNotStrip
     @Keep
     public static int getPlanesCount(ImageProxy imageProxy) {
@@ -39,4 +70,29 @@ public class ImageProxyUtils {
     public static int getBytesPerRow(ImageProxy imageProxy) {
         return imageProxy.getPlanes()[0].getRowStride();
     }
+
+    private static byte[] byteArrayCache;
+
+    @DoNotStrip
+    @Keep
+    public static byte[] toByteArray(ImageProxy imageProxy) {
+        switch (imageProxy.getFormat()) {
+        case ImageFormat.YUV_420_888:
+            ByteBuffer yBuffer = imageProxy.getPlanes()[0].getBuffer();
+            ByteBuffer vuBuffer = imageProxy.getPlanes()[2].getBuffer();
+            int ySize = yBuffer.remaining();
+            int vuSize = vuBuffer.remaining();
+
+            if (byteArrayCache == null || byteArrayCache.length != ySize + vuSize) {
+                byteArrayCache = new byte[ySize + vuSize];
+            }
+
+            yBuffer.get(byteArrayCache, 0, ySize);
+            vuBuffer.get(byteArrayCache, ySize, vuSize);
+
+            return byteArrayCache;
+        default:
+            throw new RuntimeException("Cannot convert Frame with Format " + imageProxy.getFormat() + " to byte array!");
+        }
+    }
 }