feat: Use JSI's ArrayBuffer instead of TypedArray (#2408)

* feat: Use JSI's `ArrayBuffer` instead of `TypedArray`

* fix: Fix move memory

* feat: Implement iOS

* Format

* Update JSIJNIConversion.cpp

* fix: Fix Android `toArrayBuffer` and other

* Catch FP call errors

* Update return type

* Use `CPU_READ_OFTEN` flag as well

* CPU flag

* Run destructors under `jni::ThreadScope`

* Update FrameProcessorPluginHostObject.cpp

* fix: Fix `toArrayBuffer()` crash

* Update Frame.ts

package/android/CMakeLists.txt

@@ -23,12 +23,15 @@ endif()
 add_library(
         ${PACKAGE_NAME}
         SHARED
-        ../cpp/JSITypedArray.cpp
+        # Shared C++
+        ../cpp/MutableRawBuffer.cpp
+        # Java JNI
         src/main/cpp/VisionCamera.cpp
         src/main/cpp/VideoPipeline.cpp
         src/main/cpp/PassThroughShader.cpp
         src/main/cpp/OpenGLContext.cpp
         src/main/cpp/OpenGLRenderer.cpp
+        src/main/cpp/MutableJByteBuffer.cpp
         # Frame Processor
         src/main/cpp/frameprocessor/FrameHostObject.cpp
         src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.cpp

package/android/src/main/cpp/MutableJByteBuffer.cpp

@@ -0,0 +1,33 @@
+//
+// Created by Marc Rousavy on 17.01.24.
+//
+
+#include "MutableJByteBuffer.h"
+
+#include <fbjni/ByteBuffer.h>
+#include <fbjni/fbjni.h>
+#include <jsi/jsi.h>
+
+namespace vision {
+
+MutableJByteBuffer::MutableJByteBuffer(jni::alias_ref<jni::JByteBuffer> byteBuffer) {
+  _byteBuffer = jni::make_global(byteBuffer);
+}
+
+MutableJByteBuffer::~MutableJByteBuffer() noexcept {
+  jni::ThreadScope::WithClassLoader([&] { _byteBuffer.reset(); });
+}
+
+uint8_t* MutableJByteBuffer::data() {
+  return _byteBuffer->getDirectBytes();
+}
+
+size_t MutableJByteBuffer::size() const {
+  return _byteBuffer->getDirectSize();
+}
+
+jni::global_ref<jni::JByteBuffer> MutableJByteBuffer::getByteBuffer() {
+  return _byteBuffer;
+}
+
+} // namespace vision

package/android/src/main/cpp/MutableJByteBuffer.h

@@ -0,0 +1,33 @@
+//
+// Created by Marc Rousavy on 17.01.24.
+//
+
+#pragma once
+
+#include <fbjni/ByteBuffer.h>
+#include <fbjni/fbjni.h>
+#include <jsi/jsi.h>
+#include <memory>
+
+namespace vision {
+
+using namespace facebook;
+
+class MutableJByteBuffer : public jsi::MutableBuffer {
+public:
+  /**
+   * Wraps the given JByteBuffer in a MutableBuffer for use in JS.
+   */
+  explicit MutableJByteBuffer(jni::alias_ref<jni::JByteBuffer> byteBuffer);
+  ~MutableJByteBuffer();
+
+public:
+  uint8_t* data() override;
+  size_t size() const override;
+  jni::global_ref<jni::JByteBuffer> getByteBuffer();
+
+private:
+  jni::global_ref<jni::JByteBuffer> _byteBuffer;
+};
+
+} // namespace vision

package/android/src/main/cpp/frameprocessor/FrameHostObject.cpp

@@ -7,7 +7,7 @@
 #include <fbjni/fbjni.h>
 #include <jni.h>
 
-#include "JSITypedArray.h"
+#include "MutableRawBuffer.h"
 
 #include <string>
 #include <vector>
@@ -102,15 +102,18 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
 
       static constexpr auto ARRAYBUFFER_CACHE_PROP_NAME = "__frameArrayBufferCache";
       if (!runtime.global().hasProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME)) {
-        vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray> arrayBuffer(runtime, size);
+        auto mutableBuffer = std::make_shared<vision::MutableRawBuffer>(size);
+        jsi::ArrayBuffer arrayBuffer(runtime, mutableBuffer);
         runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer);
       }
 
       // Get from global JS cache
       auto arrayBufferCache = runtime.global().getPropertyAsObject(runtime, ARRAYBUFFER_CACHE_PROP_NAME);
-      auto arrayBuffer = vision::getTypedArray(runtime, arrayBufferCache).get<vision::TypedArrayKind::Uint8ClampedArray>(runtime);
+      auto arrayBuffer = arrayBufferCache.getArrayBuffer(runtime);
+
       if (arrayBuffer.size(runtime) != size) {
-        arrayBuffer = vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray>(runtime, size);
+        auto mutableBuffer = std::make_shared<vision::MutableRawBuffer>(size);
+        arrayBuffer = jsi::ArrayBuffer(runtime, mutableBuffer);
         runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer);
       }
 
@@ -170,4 +173,6 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
   return HostObject::get(runtime, propName);
 }
 
+#undef JSI_FUNC
+
 } // namespace vision

package/android/src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.cpp

@@ -12,6 +12,13 @@ namespace vision {
 
 using namespace facebook;
 
+FrameProcessorPluginHostObject::FrameProcessorPluginHostObject(jni::alias_ref<JFrameProcessorPlugin::javaobject> plugin)
+    : _plugin(make_global(plugin)) {}
+
+FrameProcessorPluginHostObject::~FrameProcessorPluginHostObject() {
+  jni::ThreadScope::WithClassLoader([&] { _plugin.reset(); });
+}
+
 std::vector<jsi::PropNameID> FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime& runtime) {
   std::vector<jsi::PropNameID> result;
   result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("call")));

package/android/src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.h

@@ -16,8 +16,8 @@ using namespace facebook;
 
 class FrameProcessorPluginHostObject : public jsi::HostObject {
 public:
-  explicit FrameProcessorPluginHostObject(jni::alias_ref<JFrameProcessorPlugin::javaobject> plugin) : _plugin(make_global(plugin)) {}
+  explicit FrameProcessorPluginHostObject(jni::alias_ref<JFrameProcessorPlugin::javaobject> plugin);
-  ~FrameProcessorPluginHostObject() {}
+  ~FrameProcessorPluginHostObject();
 
 public:
   std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& runtime) override;

package/android/src/main/cpp/frameprocessor/JSIJNIConversion.cpp

@@ -15,7 +15,6 @@
 
 #include "FrameHostObject.h"
 #include "JFrame.h"
-#include "JSITypedArray.h"
 #include "JSharedArray.h"
 
 namespace vision {
@@ -61,8 +60,8 @@ jni::local_ref<jobject> JSIJNIConversion::convertJSIValueToJNIObject(jsi::Runtim
     } else if (valueAsObject.isArrayBuffer(runtime)) {
       // ArrayBuffer/TypedArray
 
-      TypedArrayBase array = getTypedArray(runtime, valueAsObject);
+      jsi::ArrayBuffer arrayBuffer = valueAsObject.getArrayBuffer(runtime);
-      return JSharedArray::create(runtime, std::move(array));
+      return JSharedArray::create(runtime, std::move(arrayBuffer));
 
     } else if (valueAsObject.isHostObject(runtime)) {
 
@@ -176,8 +175,8 @@ jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime& runtime, c
     // SharedArray
     auto sharedArray = static_ref_cast<JSharedArray::javaobject>(object);
 
-    std::shared_ptr<TypedArrayBase> array = sharedArray->cthis()->getTypedArray();
+    std::shared_ptr<jsi::ArrayBuffer> array = sharedArray->cthis()->getArrayBuffer();
-    return array->getBuffer(runtime);
+    return array->getArrayBuffer(runtime);
   }
 
   auto type = object->getClass()->toString();

package/android/src/main/cpp/frameprocessor/VisionCameraProxy.cpp

@@ -13,7 +13,6 @@
 #include <fbjni/fbjni.h>
 
 #include "FrameProcessorPluginHostObject.h"
-#include "JSITypedArray.h"
 
 #include <memory>
 #include <string>

package/android/src/main/cpp/frameprocessor/java-bindings/JSharedArray.cpp

@@ -9,30 +9,23 @@ namespace vision {
 
 using namespace facebook;
 
-TypedArrayKind getTypedArrayKind(int unsafeEnumValue) {
+jni::local_ref<JSharedArray::javaobject> JSharedArray::create(jsi::Runtime& runtime, jsi::ArrayBuffer arrayBuffer) {
-  return static_cast<TypedArrayKind>(unsafeEnumValue);
+  jni::local_ref<JSharedArray::javaobject> instance = newObjectCxxArgs(runtime, std::make_shared<jsi::ArrayBuffer>(std::move(arrayBuffer)));
+  instance->cthis()->_javaPart = jni::make_global(instance);
+  return instance;
 }
 
-jni::local_ref<JSharedArray::javaobject> JSharedArray::create(jsi::Runtime& runtime, TypedArrayBase array) {
+JSharedArray::JSharedArray(jsi::Runtime& runtime, std::shared_ptr<jsi::ArrayBuffer> arrayBuffer) {
-  return newObjectCxxArgs(runtime, std::make_shared<TypedArrayBase>(std::move(array)));
+  size_t size = arrayBuffer->size(runtime);
+  jni::local_ref<JByteBuffer> byteBuffer = JByteBuffer::allocateDirect(size);
+
+  _arrayBuffer = arrayBuffer;
+  _byteBuffer = jni::make_global(byteBuffer);
+  _size = size;
 }
 
-jni::global_ref<jni::JByteBuffer> JSharedArray::wrapInByteBuffer(jsi::Runtime& runtime, std::shared_ptr<TypedArrayBase> typedArray) {
+JSharedArray::JSharedArray(const jni::alias_ref<jhybridobject>& javaThis, const jni::alias_ref<JVisionCameraProxy::javaobject>& proxy,
-  jsi::ArrayBuffer arrayBuffer = typedArray->getBuffer(runtime);
+                           jni::alias_ref<JByteBuffer> byteBuffer) {
-  __android_log_print(ANDROID_LOG_INFO, TAG, "Wrapping ArrayBuffer in a JNI ByteBuffer...");
-  auto byteBuffer = jni::JByteBuffer::wrapBytes(arrayBuffer.data(runtime), arrayBuffer.size(runtime));
-  __android_log_print(ANDROID_LOG_INFO, TAG, "Successfully created TypedArray (JNI Size: %i)!", byteBuffer->getDirectSize());
-  return jni::make_global(byteBuffer);
-}
-
-JSharedArray::JSharedArray(jsi::Runtime& runtime, std::shared_ptr<TypedArrayBase> array) {
-  _array = array;
-  _byteBuffer = wrapInByteBuffer(runtime, _array);
-  _size = _array->size(runtime);
-}
-
-JSharedArray::JSharedArray(const jni::alias_ref<JSharedArray::jhybridobject>& javaThis,
-                           const jni::alias_ref<JVisionCameraProxy::javaobject>& proxy, int dataType, int size) {
   _javaPart = jni::make_global(javaThis);
 
 #if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
@@ -40,37 +33,48 @@ JSharedArray::JSharedArray(const jni::alias_ref<JSharedArray::jhybridobject>& ja
 #else
   jsi::Runtime& runtime = *proxy->cthis()->getJSRuntime();
 #endif
-  TypedArrayKind kind = getTypedArrayKind(dataType);
+  __android_log_print(ANDROID_LOG_INFO, TAG, "Allocating ArrayBuffer with size %i...", byteBuffer->getDirectSize());
-  __android_log_print(ANDROID_LOG_INFO, TAG, "Allocating ArrayBuffer with size %i and type %i...", size, dataType);
+  _byteBuffer = jni::make_global(byteBuffer);
-  _array = std::make_shared<TypedArrayBase>(runtime, size, kind);
+  _size = _byteBuffer->getDirectSize();
-  _byteBuffer = wrapInByteBuffer(runtime, _array);
+
-  _size = size;
+  auto mutableByteBuffer = std::make_shared<MutableJByteBuffer>(byteBuffer);
+  _arrayBuffer = std::make_shared<jsi::ArrayBuffer>(runtime, std::move(mutableByteBuffer));
 }
 
+JSharedArray::JSharedArray(const jni::alias_ref<JSharedArray::jhybridobject>& javaThis,
+                           const jni::alias_ref<JVisionCameraProxy::javaobject>& proxy, int size)
+    : JSharedArray(javaThis, proxy, JByteBuffer::allocateDirect(size)) {}
+
 void JSharedArray::registerNatives() {
   registerHybrid({
-      makeNativeMethod("initHybrid", JSharedArray::initHybrid),
+      makeNativeMethod("initHybrid", JSharedArray::initHybridAllocate),
+      makeNativeMethod("initHybrid", JSharedArray::initHybridWrap),
       makeNativeMethod("getByteBuffer", JSharedArray::getByteBuffer),
       makeNativeMethod("getSize", JSharedArray::getSize),
   });
 }
 
-jni::local_ref<jni::JByteBuffer> JSharedArray::getByteBuffer() {
+jni::global_ref<jni::JByteBuffer> JSharedArray::getByteBuffer() {
-  return jni::make_local(_byteBuffer);
+  return _byteBuffer;
+}
+
+std::shared_ptr<jsi::ArrayBuffer> JSharedArray::getArrayBuffer() {
+  return _arrayBuffer;
 }
 
 jint JSharedArray::getSize() {
   return _size;
 }
 
-std::shared_ptr<TypedArrayBase> JSharedArray::getTypedArray() {
+jni::local_ref<JSharedArray::jhybriddata>
-  return _array;
+JSharedArray::initHybridAllocate(jni::alias_ref<jhybridobject> javaThis, jni::alias_ref<JVisionCameraProxy::javaobject> proxy, jint size) {
+  return makeCxxInstance(javaThis, proxy, size);
 }
 
-jni::local_ref<JSharedArray::jhybriddata> JSharedArray::initHybrid(jni::alias_ref<jhybridobject> javaThis,
+jni::local_ref<JSharedArray::jhybriddata> JSharedArray::initHybridWrap(jni::alias_ref<jhybridobject> javaThis,
-                                                                   jni::alias_ref<JVisionCameraProxy::javaobject> proxy, jint type,
+                                                                       jni::alias_ref<JVisionCameraProxy::javaobject> proxy,
-                                                                   jint size) {
+                                                                       jni::alias_ref<JByteBuffer> byteBuffer) {
-  return makeCxxInstance(javaThis, proxy, type, size);
+  return makeCxxInstance(javaThis, proxy, byteBuffer);
 }
 
 } // namespace vision

package/android/src/main/cpp/frameprocessor/java-bindings/JSharedArray.h

@@ -4,8 +4,8 @@
 
 #pragma once
 
-#include "JSITypedArray.h"
 #include "JVisionCameraProxy.h"
+#include "MutableJByteBuffer.h"
 #include <fbjni/ByteBuffer.h>
 #include <fbjni/fbjni.h>
 #include <jni.h>
@@ -20,30 +20,32 @@ public:
   static void registerNatives();
 
 public:
-  static jni::local_ref<JSharedArray::javaobject> create(jsi::Runtime& runtime, TypedArrayBase array);
+  static jni::local_ref<JSharedArray::javaobject> create(jsi::Runtime& runtime, jsi::ArrayBuffer arrayBuffer);
 
 public:
   jint getSize();
-  jni::local_ref<jni::JByteBuffer> getByteBuffer();
+  jni::global_ref<jni::JByteBuffer> getByteBuffer();
-  std::shared_ptr<TypedArrayBase> getTypedArray();
+  std::shared_ptr<jsi::ArrayBuffer> getArrayBuffer();
-
-private:
-  jni::global_ref<jni::JByteBuffer> wrapInByteBuffer(jsi::Runtime& runtime, std::shared_ptr<TypedArrayBase> typedArray);
 
 private:
   static auto constexpr TAG = "SharedArray";
   friend HybridBase;
   jni::global_ref<javaobject> _javaPart;
   jni::global_ref<jni::JByteBuffer> _byteBuffer;
-  std::shared_ptr<TypedArrayBase> _array;
+  std::shared_ptr<jsi::ArrayBuffer> _arrayBuffer;
   int _size;
 
 private:
-  explicit JSharedArray(jsi::Runtime& runtime, std::shared_ptr<TypedArrayBase> array);
+  explicit JSharedArray(jsi::Runtime& runtime, std::shared_ptr<jsi::ArrayBuffer> arrayBuffer);
   explicit JSharedArray(const jni::alias_ref<jhybridobject>& javaThis, const jni::alias_ref<JVisionCameraProxy::javaobject>& proxy,
-                        int dataType, int size);
+                        int size);
-  static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> javaThis,
+  explicit JSharedArray(const jni::alias_ref<jhybridobject>& javaThis, const jni::alias_ref<JVisionCameraProxy::javaobject>& proxy,
-                                                jni::alias_ref<JVisionCameraProxy::javaobject> proxy, jint dataType, jint size);
+                        jni::alias_ref<JByteBuffer> byteBuffer);
+  static jni::local_ref<jhybriddata> initHybridAllocate(jni::alias_ref<jhybridobject> javaThis,
+                                                        jni::alias_ref<JVisionCameraProxy::javaobject> proxy, jint size);
+  static jni::local_ref<jhybriddata> initHybridWrap(jni::alias_ref<jhybridobject> javaThis,
+                                                    jni::alias_ref<JVisionCameraProxy::javaobject> proxy,
+                                                    jni::alias_ref<JByteBuffer> byteBuffer);
 };
 
 } // namespace vision

package/android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraProxy.cpp

@@ -11,7 +11,6 @@
 #include <jsi/jsi.h>
 
 #include "FrameProcessorPluginHostObject.h"
-#include "JSITypedArray.h"
 
 #if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
 #include <react-native-worklets-core/WKTJsiWorklet.h>
@@ -51,10 +50,7 @@ JVisionCameraProxy::JVisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::
 
 JVisionCameraProxy::~JVisionCameraProxy() {
 #if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
-  __android_log_write(ANDROID_LOG_INFO, TAG, "Destroying Context...");
+  __android_log_write(ANDROID_LOG_INFO, TAG, "Destroying JVisionCameraProxy...");
-  // Destroy ArrayBuffer cache for both the JS and the Worklet Runtime.
-  invalidateArrayBufferCache(*_workletContext->getJsRuntime());
-  invalidateArrayBufferCache(_workletContext->getWorkletRuntime());
 #endif
 }
 

package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt

@@ -96,8 +96,9 @@ class VideoPipeline(
 
       if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
         Log.i(TAG, "Using API 29 for GPU ImageReader...")
-        // GPU_SAMPLED because we redirect to OpenGL
+        // If we are in PRIVATE, we just pass it to the GPU as efficiently as possible - so use GPU flag.
-        val usage = HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE
+        // If we are in YUV/RGB/..., we probably want to access Frame data - so use CPU flag.
+        val usage = if (format == ImageFormat.PRIVATE) HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE else HardwareBuffer.USAGE_CPU_READ_OFTEN
         imageReader = ImageReader.newInstance(width, height, format, MAX_IMAGES, usage)
         imageWriter = ImageWriter.newInstance(glSurface, MAX_IMAGES, format)
       } else {
@@ -109,6 +110,7 @@ class VideoPipeline(
         Log.i(TAG, "ImageReader::onImageAvailable!")
         val image = reader.acquireNextImage() ?: return@setOnImageAvailableListener
 
+        try {
           // TODO: Get correct orientation and isMirrored
           val frame = Frame(image, image.timestamp, Orientation.PORTRAIT, isMirrored)
           frame.incrementRefCount()
@@ -120,6 +122,9 @@ class VideoPipeline(
           }
 
           frame.decrementRefCount()
+        } catch (e: Throwable) {
+          Log.e(TAG, "Failed to call Frame Processor!", e)
+        }
       }, CameraQueues.videoQueue.handler)
 
       surface = imageReader!!.surface

package/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java

@@ -15,7 +15,6 @@ public class Frame {
     private final long timestamp;
     private final Orientation orientation;
     private int refCount = 0;
-    private HardwareBuffer hardwareBuffer = null;
 
     public Frame(Image image, long timestamp, Orientation orientation, boolean isMirrored) {
         this.image = image;
@@ -114,10 +113,7 @@ public class Frame {
         if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P) {
             throw new HardwareBuffersNotAvailableError();
         }
-        if (hardwareBuffer == null) {
+        return getImage().getHardwareBuffer();
-            hardwareBuffer = getImage().getHardwareBuffer();
-        }
-        return hardwareBuffer;
     }
 
     @SuppressWarnings("unused")
@@ -142,9 +138,6 @@ public class Frame {
 
     private synchronized void close() {
         synchronized (this) {
-            if (hardwareBuffer != null) {
-                hardwareBuffer.close();
-            }
             image.close();
         }
     }

package/android/src/main/java/com/mrousavy/camera/frameprocessor/SharedArray.java

@@ -20,18 +20,26 @@ public final class SharedArray {
 
     @DoNotStrip
     @Keep
-    public SharedArray(HybridData hybridData) {
+    private SharedArray(HybridData hybridData) {
         mHybridData = hybridData;
     }
 
     /**
      * Allocate a new SharedArray. Use `getByteBuffer` to obtain a reference to the direct ByteBuffer for writing.
      * @param proxy The VisionCamera Proxy from the Frame Processor Plugin's initializer.
-     * @param dataType The ArrayBuffer's data type. `Type.Int8Array` = `Int8Array` in JS
      * @param size The size of the ArrayBuffer.
      */
-    public SharedArray(VisionCameraProxy proxy, Type dataType, int size) {
+    public SharedArray(VisionCameraProxy proxy, int size) {
-        mHybridData = initHybrid(proxy, dataType.ordinal(), size);
+        mHybridData = initHybrid(proxy, size);
+    }
+
+    /**
+     * Wraps the given ByteBuffer in a JSI ArrayBuffer. Using `getByteBuffer` will return the same instance which can be used for writing.
+     * @param proxy The VisionCamera Proxy from the Frame Processor Plugin's initializer.
+     * @param byteBuffer The ByteBuffer to wrap.
+     */
+    public SharedArray(VisionCameraProxy proxy, ByteBuffer byteBuffer) {
+        mHybridData = initHybrid(proxy, byteBuffer);
     }
 
     /**
@@ -44,21 +52,6 @@ public final class SharedArray {
      */
     public native int getSize();
 
-    private native HybridData initHybrid(VisionCameraProxy proxy, int dataType, int size);
+    private native HybridData initHybrid(VisionCameraProxy proxy, int size);
-
+    private native HybridData initHybrid(VisionCameraProxy proxy, ByteBuffer byteBuffer);
-    /**
-     * The Type of the SharedArray.
-     */
-    public enum Type {
-        // Values start at 0 and need to match with JSITypedArray.h::TypedArrayKind
-        Int8Array,
-        Int16Array,
-        Int32Array,
-        Uint8Array,
-        Uint8ClampedArray,
-        Uint16Array,
-        Uint32Array,
-        Float32Array,
-        Float64Array,
-    }
 }

package/cpp/JSITypedArray.cpp

@@ -1,314 +0,0 @@
-//
-//  JSITypedArray.cpp
-//  VisionCamera
-//
-//  Created by Marc Rousavy on 21.02.23.
-//  Copyright © 2023 mrousavy. All rights reserved.
-//
-
-// Copied & Adapted from
-// https://github.com/expo/expo/blob/main/packages/expo-gl/common/EXTypedArrayApi.cpp Credits to
-// Expo
-
-#include "JSITypedArray.h"
-
-#include <algorithm>
-#include <memory>
-#include <string>
-#include <unordered_map>
-#include <utility>
-#include <vector>
-
-namespace vision {
-
-template <TypedArrayKind T> using ContentType = typename typedArrayTypeMap<T>::type;
-
-enum class Prop {
-  Buffer,            // "buffer"
-  Constructor,       // "constructor"
-  Name,              // "name"
-  Proto,             // "__proto__"
-  Length,            // "length"
-  ByteLength,        // "byteLength"
-  ByteOffset,        // "offset"
-  IsView,            // "isView"
-  ArrayBuffer,       // "ArrayBuffer"
-  Int8Array,         // "Int8Array"
-  Int16Array,        // "Int16Array"
-  Int32Array,        // "Int32Array"
-  Uint8Array,        // "Uint8Array"
-  Uint8ClampedArray, // "Uint8ClampedArray"
-  Uint16Array,       // "Uint16Array"
-  Uint32Array,       // "Uint32Array"
-  Float32Array,      // "Float32Array"
-  Float64Array,      // "Float64Array"
-};
-
-class PropNameIDCache {
-public:
-  const jsi::PropNameID& get(jsi::Runtime& runtime, Prop prop) {
-    auto key = reinterpret_cast<uintptr_t>(&runtime);
-    if (this->props.find(key) == this->props.end()) {
-      this->props[key] = std::unordered_map<Prop, std::unique_ptr<jsi::PropNameID>>();
-    }
-    if (!this->props[key][prop]) {
-      this->props[key][prop] = std::make_unique<jsi::PropNameID>(createProp(runtime, prop));
-    }
-    return *(this->props[key][prop]);
-  }
-
-  const jsi::PropNameID& getConstructorNameProp(jsi::Runtime& runtime, TypedArrayKind kind);
-
-  void invalidate(uintptr_t key) {
-    if (props.find(key) != props.end()) {
-      props[key].clear();
-    }
-  }
-
-private:
-  std::unordered_map<uintptr_t, std::unordered_map<Prop, std::unique_ptr<jsi::PropNameID>>> props;
-
-  jsi::PropNameID createProp(jsi::Runtime& runtime, Prop prop);
-};
-
-PropNameIDCache propNameIDCache;
-
-void invalidateArrayBufferCache(jsi::Runtime& runtime) {
-  auto key = reinterpret_cast<uintptr_t>(&runtime);
-  propNameIDCache.invalidate(key);
-}
-
-TypedArrayKind getTypedArrayKindForName(const std::string& name);
-
-TypedArrayBase::TypedArrayBase(jsi::Runtime& runtime, size_t size, TypedArrayKind kind)
-    : TypedArrayBase(runtime, runtime.global()
-                                  .getProperty(runtime, propNameIDCache.getConstructorNameProp(runtime, kind))
-                                  .asObject(runtime)
-                                  .asFunction(runtime)
-                                  .callAsConstructor(runtime, {static_cast<double>(size)})
-                                  .asObject(runtime)) {}
-
-TypedArrayBase::TypedArrayBase(jsi::Runtime& runtime, const jsi::Object& obj) : jsi::Object(jsi::Value(runtime, obj).asObject(runtime)) {}
-
-TypedArrayKind TypedArrayBase::getKind(jsi::Runtime& runtime) const {
-  auto constructorName = this->getProperty(runtime, propNameIDCache.get(runtime, Prop::Constructor))
-                             .asObject(runtime)
-                             .getProperty(runtime, propNameIDCache.get(runtime, Prop::Name))
-                             .asString(runtime)
-                             .utf8(runtime);
-  return getTypedArrayKindForName(constructorName);
-}
-
-size_t TypedArrayBase::size(jsi::Runtime& runtime) const {
-  return getProperty(runtime, propNameIDCache.get(runtime, Prop::Length)).asNumber();
-}
-
-size_t TypedArrayBase::length(jsi::Runtime& runtime) const {
-  return getProperty(runtime, propNameIDCache.get(runtime, Prop::Length)).asNumber();
-}
-
-size_t TypedArrayBase::byteLength(jsi::Runtime& runtime) const {
-  return getProperty(runtime, propNameIDCache.get(runtime, Prop::ByteLength)).asNumber();
-}
-
-size_t TypedArrayBase::byteOffset(jsi::Runtime& runtime) const {
-  return getProperty(runtime, propNameIDCache.get(runtime, Prop::ByteOffset)).asNumber();
-}
-
-bool TypedArrayBase::hasBuffer(jsi::Runtime& runtime) const {
-  auto buffer = getProperty(runtime, propNameIDCache.get(runtime, Prop::Buffer));
-  return buffer.isObject() && buffer.asObject(runtime).isArrayBuffer(runtime);
-}
-
-std::vector<uint8_t> TypedArrayBase::toVector(jsi::Runtime& runtime) {
-  auto start = reinterpret_cast<uint8_t*>(getBuffer(runtime).data(runtime) + byteOffset(runtime));
-  auto end = start + byteLength(runtime);
-  return std::vector<uint8_t>(start, end);
-}
-
-jsi::ArrayBuffer TypedArrayBase::getBuffer(jsi::Runtime& runtime) const {
-  auto buffer = getProperty(runtime, propNameIDCache.get(runtime, Prop::Buffer));
-  if (buffer.isObject() && buffer.asObject(runtime).isArrayBuffer(runtime)) {
-    return buffer.asObject(runtime).getArrayBuffer(runtime);
-  } else {
-    throw std::runtime_error("no ArrayBuffer attached");
-  }
-}
-
-bool isTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj) {
-  auto jsVal = runtime.global()
-                   .getProperty(runtime, propNameIDCache.get(runtime, Prop::ArrayBuffer))
-                   .asObject(runtime)
-                   .getProperty(runtime, propNameIDCache.get(runtime, Prop::IsView))
-                   .asObject(runtime)
-                   .asFunction(runtime)
-                   .callWithThis(runtime, runtime.global(), {jsi::Value(runtime, jsObj)});
-  if (jsVal.isBool()) {
-    return jsVal.getBool();
-  } else {
-    throw std::runtime_error("value is not a boolean");
-  }
-}
-
-TypedArrayBase getTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj) {
-  auto jsVal = runtime.global()
-                   .getProperty(runtime, propNameIDCache.get(runtime, Prop::ArrayBuffer))
-                   .asObject(runtime)
-                   .getProperty(runtime, propNameIDCache.get(runtime, Prop::IsView))
-                   .asObject(runtime)
-                   .asFunction(runtime)
-                   .callWithThis(runtime, runtime.global(), {jsi::Value(runtime, jsObj)});
-  if (jsVal.isBool()) {
-    return TypedArrayBase(runtime, jsObj);
-  } else {
-    throw std::runtime_error("value is not a boolean");
-  }
-}
-
-std::vector<uint8_t> arrayBufferToVector(jsi::Runtime& runtime, jsi::Object& jsObj) {
-  if (!jsObj.isArrayBuffer(runtime)) {
-    throw std::runtime_error("Object is not an ArrayBuffer");
-  }
-  auto jsArrayBuffer = jsObj.getArrayBuffer(runtime);
-
-  uint8_t* dataBlock = jsArrayBuffer.data(runtime);
-  size_t blockSize = jsArrayBuffer.getProperty(runtime, propNameIDCache.get(runtime, Prop::ByteLength)).asNumber();
-  return std::vector<uint8_t>(dataBlock, dataBlock + blockSize);
-}
-
-void arrayBufferUpdate(jsi::Runtime& runtime, jsi::ArrayBuffer& buffer, std::vector<uint8_t> data, size_t offset) {
-  uint8_t* dataBlock = buffer.data(runtime);
-  size_t blockSize = buffer.size(runtime);
-  if (data.size() > blockSize) {
-    throw jsi::JSError(runtime, "ArrayBuffer is to small to fit data");
-  }
-  std::copy(data.begin(), data.end(), dataBlock + offset);
-}
-
-template <TypedArrayKind T> TypedArray<T>::TypedArray(jsi::Runtime& runtime, size_t size) : TypedArrayBase(runtime, size, T) {}
-
-template <TypedArrayKind T>
-TypedArray<T>::TypedArray(jsi::Runtime& runtime, std::vector<ContentType<T>> data) : TypedArrayBase(runtime, data.size(), T) {
-  update(runtime, data);
-}
-
-template <TypedArrayKind T> TypedArray<T>::TypedArray(TypedArrayBase&& base) : TypedArrayBase(std::move(base)) {}
-
-template <TypedArrayKind T> std::vector<ContentType<T>> TypedArray<T>::toVector(jsi::Runtime& runtime) {
-  auto start = reinterpret_cast<ContentType<T>*>(getBuffer(runtime).data(runtime) + byteOffset(runtime));
-  auto end = start + size(runtime);
-  return std::vector<ContentType<T>>(start, end);
-}
-
-template <TypedArrayKind T> void TypedArray<T>::update(jsi::Runtime& runtime, const std::vector<ContentType<T>>& data) {
-  if (data.size() != size(runtime)) {
-    throw jsi::JSError(runtime, "TypedArray can only be updated with a vector of the same size");
-  }
-  uint8_t* rawData = getBuffer(runtime).data(runtime) + byteOffset(runtime);
-  std::copy(data.begin(), data.end(), reinterpret_cast<ContentType<T>*>(rawData));
-}
-
-template <TypedArrayKind T> void TypedArray<T>::updateUnsafe(jsi::Runtime& runtime, ContentType<T>* data, size_t length) {
-  if (length != size(runtime)) {
-    throw jsi::JSError(runtime, "TypedArray can only be updated with an array of the same size");
-  }
-  uint8_t* rawData = getBuffer(runtime).data(runtime) + byteOffset(runtime);
-  memcpy(rawData, data, length);
-}
-
-template <TypedArrayKind T> uint8_t* TypedArray<T>::data(jsi::Runtime& runtime) {
-  return getBuffer(runtime).data(runtime) + byteOffset(runtime);
-}
-
-const jsi::PropNameID& PropNameIDCache::getConstructorNameProp(jsi::Runtime& runtime, TypedArrayKind kind) {
-  switch (kind) {
-    case TypedArrayKind::Int8Array:
-      return get(runtime, Prop::Int8Array);
-    case TypedArrayKind::Int16Array:
-      return get(runtime, Prop::Int16Array);
-    case TypedArrayKind::Int32Array:
-      return get(runtime, Prop::Int32Array);
-    case TypedArrayKind::Uint8Array:
-      return get(runtime, Prop::Uint8Array);
-    case TypedArrayKind::Uint8ClampedArray:
-      return get(runtime, Prop::Uint8ClampedArray);
-    case TypedArrayKind::Uint16Array:
-      return get(runtime, Prop::Uint16Array);
-    case TypedArrayKind::Uint32Array:
-      return get(runtime, Prop::Uint32Array);
-    case TypedArrayKind::Float32Array:
-      return get(runtime, Prop::Float32Array);
-    case TypedArrayKind::Float64Array:
-      return get(runtime, Prop::Float64Array);
-  }
-}
-
-jsi::PropNameID PropNameIDCache::createProp(jsi::Runtime& runtime, Prop prop) {
-  auto create = [&](const std::string& propName) { return jsi::PropNameID::forUtf8(runtime, propName); };
-  switch (prop) {
-    case Prop::Buffer:
-      return create("buffer");
-    case Prop::Constructor:
-      return create("constructor");
-    case Prop::Name:
-      return create("name");
-    case Prop::Proto:
-      return create("__proto__");
-    case Prop::Length:
-      return create("length");
-    case Prop::ByteLength:
-      return create("byteLength");
-    case Prop::ByteOffset:
-      return create("byteOffset");
-    case Prop::IsView:
-      return create("isView");
-    case Prop::ArrayBuffer:
-      return create("ArrayBuffer");
-    case Prop::Int8Array:
-      return create("Int8Array");
-    case Prop::Int16Array:
-      return create("Int16Array");
-    case Prop::Int32Array:
-      return create("Int32Array");
-    case Prop::Uint8Array:
-      return create("Uint8Array");
-    case Prop::Uint8ClampedArray:
-      return create("Uint8ClampedArray");
-    case Prop::Uint16Array:
-      return create("Uint16Array");
-    case Prop::Uint32Array:
-      return create("Uint32Array");
-    case Prop::Float32Array:
-      return create("Float32Array");
-    case Prop::Float64Array:
-      return create("Float64Array");
-  }
-}
-
-std::unordered_map<std::string, TypedArrayKind> nameToKindMap = {
-    {"Int8Array", TypedArrayKind::Int8Array},
-    {"Int16Array", TypedArrayKind::Int16Array},
-    {"Int32Array", TypedArrayKind::Int32Array},
-    {"Uint8Array", TypedArrayKind::Uint8Array},
-    {"Uint8ClampedArray", TypedArrayKind::Uint8ClampedArray},
-    {"Uint16Array", TypedArrayKind::Uint16Array},
-    {"Uint32Array", TypedArrayKind::Uint32Array},
-    {"Float32Array", TypedArrayKind::Float32Array},
-    {"Float64Array", TypedArrayKind::Float64Array},
-};
-
-TypedArrayKind getTypedArrayKindForName(const std::string& name) {
-  return nameToKindMap.at(name);
-}
-
-template class TypedArray<TypedArrayKind::Int8Array>;
-template class TypedArray<TypedArrayKind::Int16Array>;
-template class TypedArray<TypedArrayKind::Int32Array>;
-template class TypedArray<TypedArrayKind::Uint8Array>;
-template class TypedArray<TypedArrayKind::Uint8ClampedArray>;
-template class TypedArray<TypedArrayKind::Uint16Array>;
-template class TypedArray<TypedArrayKind::Uint32Array>;
-template class TypedArray<TypedArrayKind::Float32Array>;
-template class TypedArray<TypedArrayKind::Float64Array>;
-
-} // namespace vision

package/cpp/JSITypedArray.h

@@ -1,141 +0,0 @@
-//
-//  JSITypedArray.h
-//  VisionCamera
-//
-//  Created by Marc Rousavy on 21.02.23.
-//  Copyright © 2023 mrousavy. All rights reserved.
-//
-
-// Copied & Adapted from
-// https://github.com/expo/expo/blob/main/packages/expo-gl/common/EXTypedArrayApi.h Credits to Expo
-
-#pragma once
-
-#include <jsi/jsi.h>
-#include <utility>
-#include <vector>
-
-namespace jsi = facebook::jsi;
-
-namespace vision {
-
-enum class TypedArrayKind {
-  Int8Array,
-  Int16Array,
-  Int32Array,
-  Uint8Array,
-  Uint8ClampedArray,
-  Uint16Array,
-  Uint32Array,
-  Float32Array,
-  Float64Array,
-};
-
-template <TypedArrayKind T> class TypedArray;
-
-template <TypedArrayKind T> struct typedArrayTypeMap;
-template <> struct typedArrayTypeMap<TypedArrayKind::Int8Array> {
-  typedef int8_t type;
-};
-template <> struct typedArrayTypeMap<TypedArrayKind::Int16Array> {
-  typedef int16_t type;
-};
-template <> struct typedArrayTypeMap<TypedArrayKind::Int32Array> {
-  typedef int32_t type;
-};
-template <> struct typedArrayTypeMap<TypedArrayKind::Uint8Array> {
-  typedef uint8_t type;
-};
-template <> struct typedArrayTypeMap<TypedArrayKind::Uint8ClampedArray> {
-  typedef uint8_t type;
-};
-template <> struct typedArrayTypeMap<TypedArrayKind::Uint16Array> {
-  typedef uint16_t type;
-};
-template <> struct typedArrayTypeMap<TypedArrayKind::Uint32Array> {
-  typedef uint32_t type;
-};
-template <> struct typedArrayTypeMap<TypedArrayKind::Float32Array> {
-  typedef float type;
-};
-template <> struct typedArrayTypeMap<TypedArrayKind::Float64Array> {
-  typedef double type;
-};
-
-void invalidateArrayBufferCache(jsi::Runtime& runtime);
-
-class TypedArrayBase : public jsi::Object {
-public:
-  template <TypedArrayKind T> using ContentType = typename typedArrayTypeMap<T>::type;
-
-  TypedArrayBase(jsi::Runtime&, size_t, TypedArrayKind);
-  TypedArrayBase(jsi::Runtime&, const jsi::Object&);
-  TypedArrayBase(TypedArrayBase&&) = default;
-  TypedArrayBase& operator=(TypedArrayBase&&) = default;
-
-  TypedArrayKind getKind(jsi::Runtime& runtime) const;
-
-  template <TypedArrayKind T> TypedArray<T> get(jsi::Runtime& runtime) const&;
-  template <TypedArrayKind T> TypedArray<T> get(jsi::Runtime& runtime) &&;
-  template <TypedArrayKind T> TypedArray<T> as(jsi::Runtime& runtime) const&;
-  template <TypedArrayKind T> TypedArray<T> as(jsi::Runtime& runtime) &&;
-
-  size_t size(jsi::Runtime& runtime) const;
-  size_t length(jsi::Runtime& runtime) const;
-  size_t byteLength(jsi::Runtime& runtime) const;
-  size_t byteOffset(jsi::Runtime& runtime) const;
-  bool hasBuffer(jsi::Runtime& runtime) const;
-
-  std::vector<uint8_t> toVector(jsi::Runtime& runtime);
-  jsi::ArrayBuffer getBuffer(jsi::Runtime& runtime) const;
-
-private:
-  template <TypedArrayKind> friend class TypedArray;
-};
-
-bool isTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj);
-TypedArrayBase getTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj);
-
-std::vector<uint8_t> arrayBufferToVector(jsi::Runtime& runtime, jsi::Object& jsObj);
-void arrayBufferUpdate(jsi::Runtime& runtime, jsi::ArrayBuffer& buffer, std::vector<uint8_t> data, size_t offset);
-
-template <TypedArrayKind T> class TypedArray : public TypedArrayBase {
-public:
-  explicit TypedArray(TypedArrayBase&& base);
-  TypedArray(jsi::Runtime& runtime, size_t size);
-  TypedArray(jsi::Runtime& runtime, std::vector<ContentType<T>> data);
-  TypedArray(TypedArray&&) = default;
-  TypedArray& operator=(TypedArray&&) = default;
-
-  std::vector<ContentType<T>> toVector(jsi::Runtime& runtime);
-  void update(jsi::Runtime& runtime, const std::vector<ContentType<T>>& data);
-  void updateUnsafe(jsi::Runtime& runtime, ContentType<T>* data, size_t length);
-  uint8_t* data(jsi::Runtime& runtime);
-};
-
-template <TypedArrayKind T> TypedArray<T> TypedArrayBase::get(jsi::Runtime& runtime) const& {
-  assert(getKind(runtime) == T);
-  (void)runtime; // when assert is disabled we need to mark this as used
-  return TypedArray<T>(jsi::Value(runtime, jsi::Value(runtime, *this).asObject(runtime)));
-}
-
-template <TypedArrayKind T> TypedArray<T> TypedArrayBase::get(jsi::Runtime& runtime) && {
-  assert(getKind(runtime) == T);
-  (void)runtime; // when assert is disabled we need to mark this as used
-  return TypedArray<T>(std::move(*this));
-}
-
-template <TypedArrayKind T> TypedArray<T> TypedArrayBase::as(jsi::Runtime& runtime) const& {
-  if (getKind(runtime) != T) {
-    throw jsi::JSError(runtime, "Object is not a TypedArray");
-  }
-  return get<T>(runtime);
-}
-
-template <TypedArrayKind T> TypedArray<T> TypedArrayBase::as(jsi::Runtime& runtime) && {
-  if (getKind(runtime) != T) {
-    throw jsi::JSError(runtime, "Object is not a TypedArray");
-  }
-  return std::move(*this).get<T>(runtime);
-}
-} // namespace vision

package/cpp/MutableRawBuffer.cpp

@@ -0,0 +1,36 @@
+//
+//  MutableRawBuffer.cpp
+//  VisionCamera
+//
+//  Created by Marc Rousavy on 17.01.24.
+//  Copyright © 2024 mrousavy. All rights reserved.
+//
+
+#include "MutableRawBuffer.h"
+#include <functional>
+#include <memory>
+
+namespace vision {
+
+MutableRawBuffer::MutableRawBuffer(uint8_t* data, size_t size, std::function<void()> cleanup)
+    : _data(data), _size(size), _cleanup(std::move(cleanup)) {}
+
+MutableRawBuffer::MutableRawBuffer(size_t size) {
+  _size = size;
+  _data = (uint8_t*)malloc(size * sizeof(uint8_t));
+  _cleanup = [=]() { free(_data); };
+}
+
+MutableRawBuffer::~MutableRawBuffer() {
+  _cleanup();
+}
+
+size_t MutableRawBuffer::size() const {
+  return _size;
+}
+
+uint8_t* MutableRawBuffer::data() {
+  return _data;
+}
+
+} // namespace vision

package/cpp/MutableRawBuffer.h

@@ -0,0 +1,36 @@
+//
+//  MutableRawBuffer.h
+//  VisionCamera
+//
+//  Created by Marc Rousavy on 17.01.24.
+//  Copyright © 2024 mrousavy. All rights reserved.
+//
+
+#pragma once
+
+#include <functional>
+#include <jsi/jsi.h>
+#include <memory>
+
+namespace vision {
+
+using namespace facebook;
+
+class MutableRawBuffer : public jsi::MutableBuffer {
+
+public:
+  explicit MutableRawBuffer(size_t size);
+  explicit MutableRawBuffer(uint8_t* data, size_t size, std::function<void()> cleanup);
+  ~MutableRawBuffer();
+
+public:
+  uint8_t* data() override;
+  size_t size() const override;
+
+private:
+  uint8_t* _data;
+  size_t _size;
+  std::function<void()> _cleanup;
+};
+
+} // namespace vision

package/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java

@@ -19,17 +19,18 @@ import java.util.Map;
 
 public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin {
     SharedArray _sharedArray;
+    private static final String TAG = "ExamplePlugin";
 
     @Override
     public Object callback(@NotNull Frame frame, @Nullable Map<String, Object> params) {
         if (params == null) return null;
         Image image = frame.getImage();
 
-        Log.d("ExamplePlugin", image.getWidth() + " x " + image.getHeight() + " Image with format #" + image.getFormat() + ". Logging " + params.size() + " parameters:");
+        Log.d(TAG, image.getWidth() + " x " + image.getHeight() + " Image with format #" + image.getFormat() + ". Logging " + params.size() + " parameters:");
 
         for (String key : params.keySet()) {
             Object value = params.get(key);
-            Log.d("ExamplePlugin", "  -> " + (value == null ? "(null)" : value + " (" + value.getClass().getName() + ")"));
+            Log.d(TAG, "  -> " + (value == null ? "(null)" : value + " (" + value.getClass().getName() + ")"));
         }
 
         Map<String, Object> map = new HashMap<>();
@@ -53,7 +54,13 @@ public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin {
 
     ExampleFrameProcessorPlugin(VisionCameraProxy proxy, @Nullable Map<String, Object> options) {
         super();
-        _sharedArray = new SharedArray(proxy, SharedArray.Type.Uint8Array, 5);
+        _sharedArray = new SharedArray(proxy, 5);
-        Log.d("ExamplePlugin", "ExampleFrameProcessorPlugin initialized with options: " + options);
+        Log.d(TAG, "Successfully allocated SharedArray! Size: " + _sharedArray.getSize());
+
+        ByteBuffer buffer = ByteBuffer.allocateDirect(10);
+        SharedArray testArray = new SharedArray(proxy, buffer);
+        Log.d(TAG, "Successfully wrapped SharedArray in ByteBuffer! Size: " + testArray.getSize());
+
+        Log.d(TAG, "ExampleFrameProcessorPlugin initialized with options: " + options);
     }
 }

package/example/ios/Frame Processor Plugins/Example Plugin/ExampleFrameProcessorPlugin.m

@@ -25,7 +25,6 @@
                   withOptions:(NSDictionary* _Nullable)options {
   if (self = [super initWithProxy:proxy withOptions:options]) {
     _sharedArray = [[SharedArray alloc] initWithProxy:proxy
-                                                 type:Int8Array
                                                  size:5];
     NSLog(@"ExampleFrameProcessorPlugin initialized with options: %@", options);
   }

package/ios/Frame Processor/FrameHostObject.mm

@@ -7,13 +7,12 @@
 //
 
 #import "FrameHostObject.h"
+#import "MutableRawBuffer.h"
 #import "UIImageOrientation+descriptor.h"
 #import "WKTJsiHostObject.h"
 #import <Foundation/Foundation.h>
 #import <jsi/jsi.h>
 
-#import "../../cpp/JSITypedArray.h"
-
 std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt) {
   std::vector<jsi::PropNameID> result;
   // Ref Management
@@ -110,21 +109,23 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
 
       static constexpr auto ARRAYBUFFER_CACHE_PROP_NAME = "__frameArrayBufferCache";
       if (!runtime.global().hasProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME)) {
-        vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray> arrayBuffer(runtime, arraySize);
+        auto mutableBuffer = std::make_shared<vision::MutableRawBuffer>(arraySize);
-        runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer);
+        jsi::ArrayBuffer arrayBuffer(runtime, mutableBuffer);
+        runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, std::move(arrayBuffer));
       }
 
       auto arrayBufferCache = runtime.global().getPropertyAsObject(runtime, ARRAYBUFFER_CACHE_PROP_NAME);
-      auto arrayBuffer = vision::getTypedArray(runtime, arrayBufferCache).get<vision::TypedArrayKind::Uint8ClampedArray>(runtime);
+      auto arrayBuffer = arrayBufferCache.getArrayBuffer(runtime);
 
       if (arrayBuffer.size(runtime) != arraySize) {
-        arrayBuffer = vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray>(runtime, arraySize);
+        auto mutableBuffer = std::make_shared<vision::MutableRawBuffer>(arraySize);
+        arrayBuffer = jsi::ArrayBuffer(runtime, mutableBuffer);
         runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer);
       }
 
       CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
       auto buffer = (uint8_t*)CVPixelBufferGetBaseAddress(pixelBuffer);
-      arrayBuffer.updateUnsafe(runtime, buffer, arraySize);
+      memcpy(arrayBuffer.data(runtime), buffer, arraySize);
       CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
 
       return arrayBuffer;
@@ -201,3 +202,5 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
   // fallback to base implementation
   return HostObject::get(runtime, propName);
 }
+
+#undef JSI_FUNC

package/ios/Frame Processor/JSINSObjectConversion.mm

@@ -19,7 +19,6 @@
 #import "../Frame Processor/Frame.h"
 #import "../Frame Processor/FrameHostObject.h"
 #import "../Frame Processor/SharedArray.h"
-#import "JSITypedArray.h"
 #import <Foundation/Foundation.h>
 #import <React/RCTBridge.h>
 #import <ReactCommon/CallInvoker.h>
@@ -61,8 +60,7 @@ jsi::Array convertNSArrayToJSIArray(jsi::Runtime& runtime, NSArray* value) {
 }
 
 jsi::Object convertSharedArrayToJSIArrayBuffer(jsi::Runtime& runtime, SharedArray* sharedArray) {
-  std::shared_ptr<vision::TypedArrayBase> array = sharedArray.typedArray;
+  return sharedArray.arrayBuffer->getArrayBuffer(runtime);
-  return array->getBuffer(runtime);
 }
 
 jsi::Value convertObjCObjectToJSIValue(jsi::Runtime& runtime, id value) {
@@ -171,8 +169,8 @@ id convertJSIValueToObjCObject(jsi::Runtime& runtime, const jsi::Value& value, s
       }
     } else if (o.isArrayBuffer(runtime)) {
       // ArrayBuffer
-      auto typedArray = std::make_shared<vision::TypedArrayBase>(vision::getTypedArray(runtime, o));
+      auto arrayBuffer = std::make_shared<jsi::ArrayBuffer>(o.getArrayBuffer(runtime));
-      return [[SharedArray alloc] initWithRuntime:runtime typedArray:typedArray];
+      return [[SharedArray alloc] initWithRuntime:runtime arrayBuffer:arrayBuffer];
     } else {
       // object
       return convertJSIObjectToNSDictionary(runtime, o, jsInvoker);

package/ios/Frame Processor/SharedArray.h

@@ -12,39 +12,22 @@
 #import <Foundation/Foundation.h>
 
 #ifdef __cplusplus
+#import "../../cpp/MutableRawBuffer.h"
 #import <jsi/jsi.h>
 using namespace facebook;
-
-namespace vision {
-// forward-declaration since we cannot import C++ headers here yet.
-class TypedArrayBase;
-} // namespace vision
 #endif
 
-// Needs to be in sync with JSITypedArray.h as the index is used
-typedef NS_ENUM(NSInteger, SharedArrayType) {
-  Int8Array,
-  Int16Array,
-  Int32Array,
-  Uint8Array,
-  Uint8ClampedArray,
-  Uint16Array,
-  Uint32Array,
-  Float32Array,
-  Float64Array,
-};
-
 NS_ASSUME_NONNULL_BEGIN
 
 @interface SharedArray : NSObject
 
 - (instancetype)init NS_UNAVAILABLE;
-- (instancetype)initWithProxy:(VisionCameraProxyHolder*)proxy type:(SharedArrayType)type size:(NSInteger)size;
+- (instancetype)initWithProxy:(VisionCameraProxyHolder*)proxy size:(NSInteger)size;
 
 #ifdef __cplusplus
-- (instancetype)initWithRuntime:(jsi::Runtime&)runtime typedArray:(std::shared_ptr<vision::TypedArrayBase>)typedArray;
+- (instancetype)initWithRuntime:(jsi::Runtime&)runtime arrayBuffer:(std::shared_ptr<jsi::ArrayBuffer>)arrayBuffer;
 
-- (std::shared_ptr<vision::TypedArrayBase>)typedArray;
+- (std::shared_ptr<jsi::ArrayBuffer>)arrayBuffer;
 #endif
 
 @property(nonatomic, readonly, nonnull) uint8_t* data;

package/ios/Frame Processor/SharedArray.mm

@@ -7,7 +7,7 @@
 //
 
 #import "SharedArray.h"
-#import "JSITypedArray.h"
+#import "../../cpp/MutableRawBuffer.h"
 #import <Foundation/Foundation.h>
 #import <jsi/jsi.h>
 
@@ -16,35 +16,33 @@ using namespace facebook;
 @implementation SharedArray {
   uint8_t* _data;
   NSInteger _size;
-  std::shared_ptr<vision::TypedArrayBase> _array;
+  std::shared_ptr<jsi::ArrayBuffer> _arrayBuffer;
 }
 
-vision::TypedArrayKind getTypedArrayKind(int unsafeEnumValue) {
+- (instancetype)initWithProxy:(VisionCameraProxyHolder*)proxy size:(NSInteger)size {
-  return static_cast<vision::TypedArrayKind>(unsafeEnumValue);
-}
-
-- (instancetype)initWithProxy:(VisionCameraProxyHolder*)proxy type:(SharedArrayType)type size:(NSInteger)size {
   if (self = [super init]) {
     jsi::Runtime& runtime = proxy.proxy->getWorkletRuntime();
-    vision::TypedArrayKind kind = getTypedArrayKind((int)type);
+
-    _array = std::make_shared<vision::TypedArrayBase>(vision::TypedArrayBase(runtime, size, kind));
+    uint8_t* data = (uint8_t*)malloc(size * sizeof(uint8_t));
-    _data = _array->getBuffer(runtime).data(runtime);
+    auto mutableBuffer = std::make_shared<vision::MutableRawBuffer>(data, size, [=]() { free(data); });
+    _arrayBuffer = std::make_shared<jsi::ArrayBuffer>(runtime, mutableBuffer);
+    _data = data;
     _size = size;
   }
   return self;
 }
 
-- (instancetype)initWithRuntime:(jsi::Runtime&)runtime typedArray:(std::shared_ptr<vision::TypedArrayBase>)typedArray {
+- (instancetype)initWithRuntime:(jsi::Runtime&)runtime arrayBuffer:(std::shared_ptr<jsi::ArrayBuffer>)arrayBuffer {
   if (self = [super init]) {
-    _array = typedArray;
+    _arrayBuffer = arrayBuffer;
-    _data = _array->getBuffer(runtime).data(runtime);
+    _data = _arrayBuffer->data(runtime);
-    _size = _array->getBuffer(runtime).size(runtime);
+    _size = _arrayBuffer->size(runtime);
   }
   return self;
 }
 
-- (std::shared_ptr<vision::TypedArrayBase>)typedArray {
+- (std::shared_ptr<jsi::ArrayBuffer>)arrayBuffer {
-  return _array;
+  return _arrayBuffer;
 }
 
 - (uint8_t*)data {

package/ios/Frame Processor/VisionCameraProxy.mm

@@ -10,7 +10,6 @@
 #import <Foundation/Foundation.h>
 #import <jsi/jsi.h>
 
-#import "../../cpp/JSITypedArray.h"
 #import "FrameHostObject.h"
 #import "FrameProcessor.h"
 #import "FrameProcessorPluginHostObject.h"
@@ -55,10 +54,7 @@ VisionCameraProxy::VisionCameraProxy(jsi::Runtime& runtime, std::shared_ptr<reac
 }
 
 VisionCameraProxy::~VisionCameraProxy() {
-  NSLog(@"VisionCameraProxy: Destroying context...");
+  NSLog(@"VisionCameraProxy: Destroying VisionCameraProxy...");
-  // Destroy ArrayBuffer cache for both the JS and the Worklet Runtime.
-  vision::invalidateArrayBufferCache(*_workletContext->getJsRuntime());
-  vision::invalidateArrayBufferCache(_workletContext->getWorkletRuntime());
 }
 
 std::vector<jsi::PropNameID> VisionCameraProxy::getPropertyNames(jsi::Runtime& runtime) {

package/src/Frame.ts

@@ -57,7 +57,9 @@ export interface Frame {
 
   /**
    * Get the underlying data of the Frame as a uint8 array buffer.
+   *
    * The format of the buffer depends on the Frame's {@linkcode pixelFormat}.
+   * This function might fail if the {@linkcode pixelFormat} is `private`.
    *
    * Note that Frames are allocated on the GPU, so calling `toArrayBuffer()` will copy from the GPU to the CPU.
    *
@@ -67,13 +69,14 @@ export interface Frame {
    *   'worklet'
    *
    *   if (frame.pixelFormat === 'rgb') {
-   *     const data = frame.toArrayBuffer()
+   *     const buffer = frame.toArrayBuffer()
+   *     const data = new Uint8Array(buffer)
    *     console.log(`Pixel at 0,0: RGB(${data[0]}, ${data[1]}, ${data[2]})`)
    *   }
    * }, [])
    * ```
    */
-  toArrayBuffer(): Uint8Array
+  toArrayBuffer(): ArrayBuffer
   /**
    * Returns a string representation of the frame.
    * @example