feat: Add zero-copy SharedArray type to Frame Processor Plugins (#2383)

* feat: Create `TypedArray` class for Frame Processor Plugins

* Type

* feat: Pass `VisionCameraProxy` along (BREAKING)

* feat: Finish implementation

* Log a bit

* feat: Successfully convert JSI <> JNI buffers

* Wrap buffer

* fix: Fix using wrong Runtime

* feat: Add docs

* add zero copy example

* Format C++

* Create iOS base

* feat: Finish iOS implementation

* chore: Format

* fix: Use `NSData` instead of `NSMutableData`

* Format

* fix: Fix build when Frame Processors are disabled

* chore: Rename `TypedArray` to `SharedArray`

* fix: Fix Swift typings for Array

* Remove a few default inits

* fix: Fix Android build

* fix: Use `NSInteger`

* Update SharedArray.mm

* fix: Expose bytes directly on iOS (NSData was immutable)

package/android/src/main/cpp/frameprocessor/JSIJNIConversion.cpp

@@ -15,6 +15,8 @@
 
 #include "FrameHostObject.h"
 #include "JFrame.h"
+#include "JSITypedArray.h"
+#include "JSharedArray.h"
 
 namespace vision {
 
@@ -56,8 +58,24 @@ jni::local_ref<jobject> JSIJNIConversion::convertJSIValueToJNIObject(jsi::Runtim
         arrayList->add(jniItem);
       }
       return arrayList;
+    } else if (valueAsObject.isArrayBuffer(runtime)) {
+      // ArrayBuffer/TypedArray
+
+      TypedArrayBase array = getTypedArray(runtime, valueAsObject);
+      return JSharedArray::create(runtime, std::move(array));
+
     } else if (valueAsObject.isHostObject(runtime)) {
-      throw std::runtime_error("You can't pass HostObjects here.");
+
+      if (valueAsObject.isHostObject<FrameHostObject>(runtime)) {
+        // Frame
+
+        auto frame = valueAsObject.getHostObject<FrameHostObject>(runtime);
+        return jni::make_local(frame->frame);
+
+      } else {
+        throw std::runtime_error("The given HostObject is not supported by a Frame Processor Plugin.");
+      }
+
     } else {
       // Map<String, Object>
 
@@ -75,7 +93,7 @@ jni::local_ref<jobject> JSIJNIConversion::convertJSIValueToJNIObject(jsi::Runtim
     }
   } else {
     auto stringRepresentation = value.toString(runtime).utf8(runtime);
-    throw std::runtime_error("Failed to convert jsi::Value to JNI value - unsupported type!" + stringRepresentation);
+    throw std::runtime_error("Failed to convert jsi::Value to JNI value - unsupported type! " + stringRepresentation);
   }
 }
 
@@ -154,6 +172,12 @@ jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime& runtime, c
     // box into HostObject
     auto hostObject = std::make_shared<FrameHostObject>(frame);
     return jsi::Object::createFromHostObject(runtime, hostObject);
+  } else if (object->isInstanceOf(JSharedArray::javaClassStatic())) {
+    // SharedArray
+    auto sharedArray = static_ref_cast<JSharedArray::javaobject>(object);
+
+    std::shared_ptr<TypedArrayBase> array = sharedArray->cthis()->getTypedArray();
+    return array->getBuffer(runtime);
   }
 
   auto type = object->getClass()->toString();

package/android/src/main/cpp/frameprocessor/java-bindings/JSharedArray.cpp

@@ -0,0 +1,69 @@
+//
+// Created by Marc Rousavy on 12.01.24.
+//
+
+#include "JSharedArray.h"
+#include <android/log.h>
+
+namespace vision {
+
+using namespace facebook;
+
+TypedArrayKind getTypedArrayKind(int unsafeEnumValue) {
+  return static_cast<TypedArrayKind>(unsafeEnumValue);
+}
+
+jni::local_ref<JSharedArray::javaobject> JSharedArray::create(jsi::Runtime& runtime, TypedArrayBase array) {
+  return newObjectCxxArgs(runtime, std::make_shared<TypedArrayBase>(std::move(array)));
+}
+
+jni::global_ref<jni::JByteBuffer> JSharedArray::wrapInByteBuffer(jsi::Runtime& runtime, std::shared_ptr<TypedArrayBase> typedArray) {
+  jsi::ArrayBuffer arrayBuffer = typedArray->getBuffer(runtime);
+  __android_log_print(ANDROID_LOG_INFO, TAG, "Wrapping ArrayBuffer in a JNI ByteBuffer...");
+  auto byteBuffer = jni::JByteBuffer::wrapBytes(arrayBuffer.data(runtime), arrayBuffer.size(runtime));
+  __android_log_print(ANDROID_LOG_INFO, TAG, "Successfully created TypedArray (JNI Size: %i)!", byteBuffer->getDirectSize());
+  return jni::make_global(byteBuffer);
+}
+
+JSharedArray::JSharedArray(jsi::Runtime& runtime, std::shared_ptr<TypedArrayBase> array) {
+  _array = array;
+  _byteBuffer = wrapInByteBuffer(runtime, _array);
+}
+
+JSharedArray::JSharedArray(const jni::alias_ref<JSharedArray::jhybridobject>& javaThis,
+                           const jni::alias_ref<JVisionCameraProxy::javaobject>& proxy, int dataType, int size) {
+  _javaPart = jni::make_global(javaThis);
+
+#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
+  jsi::Runtime& runtime = proxy->cthis()->getWorkletRuntime();
+#else
+  jsi::Runtime& runtime = *proxy->cthis()->getJSRuntime();
+#endif
+  TypedArrayKind kind = getTypedArrayKind(dataType);
+  __android_log_print(ANDROID_LOG_INFO, TAG, "Allocating ArrayBuffer with size %i and type %i...", size, dataType);
+  _array = std::make_shared<TypedArrayBase>(runtime, size, kind);
+  _byteBuffer = wrapInByteBuffer(runtime, _array);
+}
+
+void JSharedArray::registerNatives() {
+  registerHybrid({
+      makeNativeMethod("initHybrid", JSharedArray::initHybrid),
+      makeNativeMethod("getByteBuffer", JSharedArray::getByteBuffer),
+  });
+}
+
+jni::local_ref<jni::JByteBuffer> JSharedArray::getByteBuffer() {
+  return jni::make_local(_byteBuffer);
+}
+
+std::shared_ptr<TypedArrayBase> JSharedArray::getTypedArray() {
+  return _array;
+}
+
+jni::local_ref<JSharedArray::jhybriddata> JSharedArray::initHybrid(jni::alias_ref<jhybridobject> javaThis,
+                                                                   jni::alias_ref<JVisionCameraProxy::javaobject> proxy, jint type,
+                                                                   jint size) {
+  return makeCxxInstance(javaThis, proxy, type, size);
+}
+
+} // namespace vision

package/android/src/main/cpp/frameprocessor/java-bindings/JSharedArray.h

@@ -0,0 +1,47 @@
+//
+// Created by Marc Rousavy on 12.01.24.
+//
+
+#pragma once
+
+#include "JSITypedArray.h"
+#include "JVisionCameraProxy.h"
+#include <fbjni/ByteBuffer.h>
+#include <fbjni/fbjni.h>
+#include <jni.h>
+
+namespace vision {
+
+using namespace facebook;
+
+class JSharedArray : public jni::HybridClass<JSharedArray> {
+public:
+  static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/TypedArray;";
+  static void registerNatives();
+
+public:
+  static jni::local_ref<JSharedArray::javaobject> create(jsi::Runtime& runtime, TypedArrayBase array);
+
+public:
+  jni::local_ref<jni::JByteBuffer> getByteBuffer();
+  std::shared_ptr<TypedArrayBase> getTypedArray();
+
+private:
+  jni::global_ref<jni::JByteBuffer> wrapInByteBuffer(jsi::Runtime& runtime, std::shared_ptr<TypedArrayBase> typedArray);
+
+private:
+  static auto constexpr TAG = "TypedArray";
+  friend HybridBase;
+  jni::global_ref<javaobject> _javaPart;
+  jni::global_ref<jni::JByteBuffer> _byteBuffer;
+  std::shared_ptr<TypedArrayBase> _array;
+
+private:
+  explicit JSharedArray(jsi::Runtime& runtime, std::shared_ptr<TypedArrayBase> array);
+  explicit JSharedArray(const jni::alias_ref<jhybridobject>& javaThis, const jni::alias_ref<JVisionCameraProxy::javaobject>& proxy,
+                        int dataType, int size);
+  static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> javaThis,
+                                                jni::alias_ref<JVisionCameraProxy::javaobject> proxy, jint dataType, jint size);
+};
+
+} // namespace vision

package/android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraProxy.h

@@ -37,6 +37,12 @@ public:
     return _runtime;
   }
 
+#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
+  jsi::Runtime& getWorkletRuntime() {
+    return _workletContext->getWorkletRuntime();
+  }
+#endif
+
 private:
   friend HybridBase;
   jni::global_ref<JVisionCameraProxy::javaobject> _javaPart;