diff --git a/package/android/CMakeLists.txt b/package/android/CMakeLists.txt index dd6228c..5fd3ba1 100644 --- a/package/android/CMakeLists.txt +++ b/package/android/CMakeLists.txt @@ -23,12 +23,15 @@ endif() add_library( ${PACKAGE_NAME} SHARED - ../cpp/JSITypedArray.cpp + # Shared C++ + ../cpp/MutableRawBuffer.cpp + # Java JNI src/main/cpp/VisionCamera.cpp src/main/cpp/VideoPipeline.cpp src/main/cpp/PassThroughShader.cpp src/main/cpp/OpenGLContext.cpp src/main/cpp/OpenGLRenderer.cpp + src/main/cpp/MutableJByteBuffer.cpp # Frame Processor src/main/cpp/frameprocessor/FrameHostObject.cpp src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.cpp diff --git a/package/android/src/main/cpp/MutableJByteBuffer.cpp b/package/android/src/main/cpp/MutableJByteBuffer.cpp new file mode 100644 index 0000000..080f8a5 --- /dev/null +++ b/package/android/src/main/cpp/MutableJByteBuffer.cpp @@ -0,0 +1,33 @@ +// +// Created by Marc Rousavy on 17.01.24. +// + +#include "MutableJByteBuffer.h" + +#include +#include +#include + +namespace vision { + +MutableJByteBuffer::MutableJByteBuffer(jni::alias_ref byteBuffer) { + _byteBuffer = jni::make_global(byteBuffer); +} + +MutableJByteBuffer::~MutableJByteBuffer() noexcept { + jni::ThreadScope::WithClassLoader([&] { _byteBuffer.reset(); }); +} + +uint8_t* MutableJByteBuffer::data() { + return _byteBuffer->getDirectBytes(); +} + +size_t MutableJByteBuffer::size() const { + return _byteBuffer->getDirectSize(); +} + +jni::global_ref MutableJByteBuffer::getByteBuffer() { + return _byteBuffer; +} + +} // namespace vision \ No newline at end of file diff --git a/package/android/src/main/cpp/MutableJByteBuffer.h b/package/android/src/main/cpp/MutableJByteBuffer.h new file mode 100644 index 0000000..421af3e --- /dev/null +++ b/package/android/src/main/cpp/MutableJByteBuffer.h @@ -0,0 +1,33 @@ +// +// Created by Marc Rousavy on 17.01.24. +// + +#pragma once + +#include +#include +#include +#include + +namespace vision { + +using namespace facebook; + +class MutableJByteBuffer : public jsi::MutableBuffer { +public: + /** + * Wraps the given JByteBuffer in a MutableBuffer for use in JS. + */ + explicit MutableJByteBuffer(jni::alias_ref byteBuffer); + ~MutableJByteBuffer(); + +public: + uint8_t* data() override; + size_t size() const override; + jni::global_ref getByteBuffer(); + +private: + jni::global_ref _byteBuffer; +}; + +} // namespace vision diff --git a/package/android/src/main/cpp/frameprocessor/FrameHostObject.cpp b/package/android/src/main/cpp/frameprocessor/FrameHostObject.cpp index 2bd2f2f..27265fd 100644 --- a/package/android/src/main/cpp/frameprocessor/FrameHostObject.cpp +++ b/package/android/src/main/cpp/frameprocessor/FrameHostObject.cpp @@ -7,7 +7,7 @@ #include #include -#include "JSITypedArray.h" +#include "MutableRawBuffer.h" #include #include @@ -102,15 +102,18 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr static constexpr auto ARRAYBUFFER_CACHE_PROP_NAME = "__frameArrayBufferCache"; if (!runtime.global().hasProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME)) { - vision::TypedArray arrayBuffer(runtime, size); + auto mutableBuffer = std::make_shared(size); + jsi::ArrayBuffer arrayBuffer(runtime, mutableBuffer); runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer); } // Get from global JS cache auto arrayBufferCache = runtime.global().getPropertyAsObject(runtime, ARRAYBUFFER_CACHE_PROP_NAME); - auto arrayBuffer = vision::getTypedArray(runtime, arrayBufferCache).get(runtime); + auto arrayBuffer = arrayBufferCache.getArrayBuffer(runtime); + if (arrayBuffer.size(runtime) != size) { - arrayBuffer = vision::TypedArray(runtime, size); + auto mutableBuffer = std::make_shared(size); + arrayBuffer = jsi::ArrayBuffer(runtime, mutableBuffer); runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer); } @@ -170,4 +173,6 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr return HostObject::get(runtime, propName); } +#undef JSI_FUNC + } // namespace vision diff --git a/package/android/src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.cpp b/package/android/src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.cpp index e68b26b..ecd5983 100644 --- a/package/android/src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.cpp +++ b/package/android/src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.cpp @@ -12,6 +12,13 @@ namespace vision { using namespace facebook; +FrameProcessorPluginHostObject::FrameProcessorPluginHostObject(jni::alias_ref plugin) + : _plugin(make_global(plugin)) {} + +FrameProcessorPluginHostObject::~FrameProcessorPluginHostObject() { + jni::ThreadScope::WithClassLoader([&] { _plugin.reset(); }); +} + std::vector FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime& runtime) { std::vector result; result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("call"))); diff --git a/package/android/src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.h b/package/android/src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.h index 88e6889..1f2a94f 100644 --- a/package/android/src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.h +++ b/package/android/src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.h @@ -16,8 +16,8 @@ using namespace facebook; class FrameProcessorPluginHostObject : public jsi::HostObject { public: - explicit FrameProcessorPluginHostObject(jni::alias_ref plugin) : _plugin(make_global(plugin)) {} - ~FrameProcessorPluginHostObject() {} + explicit FrameProcessorPluginHostObject(jni::alias_ref plugin); + ~FrameProcessorPluginHostObject(); public: std::vector getPropertyNames(jsi::Runtime& runtime) override; diff --git a/package/android/src/main/cpp/frameprocessor/JSIJNIConversion.cpp b/package/android/src/main/cpp/frameprocessor/JSIJNIConversion.cpp index d6fe285..139af09 100644 --- a/package/android/src/main/cpp/frameprocessor/JSIJNIConversion.cpp +++ b/package/android/src/main/cpp/frameprocessor/JSIJNIConversion.cpp @@ -15,7 +15,6 @@ #include "FrameHostObject.h" #include "JFrame.h" -#include "JSITypedArray.h" #include "JSharedArray.h" namespace vision { @@ -61,8 +60,8 @@ jni::local_ref JSIJNIConversion::convertJSIValueToJNIObject(jsi::Runtim } else if (valueAsObject.isArrayBuffer(runtime)) { // ArrayBuffer/TypedArray - TypedArrayBase array = getTypedArray(runtime, valueAsObject); - return JSharedArray::create(runtime, std::move(array)); + jsi::ArrayBuffer arrayBuffer = valueAsObject.getArrayBuffer(runtime); + return JSharedArray::create(runtime, std::move(arrayBuffer)); } else if (valueAsObject.isHostObject(runtime)) { @@ -176,8 +175,8 @@ jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime& runtime, c // SharedArray auto sharedArray = static_ref_cast(object); - std::shared_ptr array = sharedArray->cthis()->getTypedArray(); - return array->getBuffer(runtime); + std::shared_ptr array = sharedArray->cthis()->getArrayBuffer(); + return array->getArrayBuffer(runtime); } auto type = object->getClass()->toString(); diff --git a/package/android/src/main/cpp/frameprocessor/VisionCameraProxy.cpp b/package/android/src/main/cpp/frameprocessor/VisionCameraProxy.cpp index 77fdb0e..73dcf85 100644 --- a/package/android/src/main/cpp/frameprocessor/VisionCameraProxy.cpp +++ b/package/android/src/main/cpp/frameprocessor/VisionCameraProxy.cpp @@ -13,7 +13,6 @@ #include #include "FrameProcessorPluginHostObject.h" -#include "JSITypedArray.h" #include #include diff --git a/package/android/src/main/cpp/frameprocessor/java-bindings/JSharedArray.cpp b/package/android/src/main/cpp/frameprocessor/java-bindings/JSharedArray.cpp index ec0f117..089ccf9 100644 --- a/package/android/src/main/cpp/frameprocessor/java-bindings/JSharedArray.cpp +++ b/package/android/src/main/cpp/frameprocessor/java-bindings/JSharedArray.cpp @@ -9,30 +9,23 @@ namespace vision { using namespace facebook; -TypedArrayKind getTypedArrayKind(int unsafeEnumValue) { - return static_cast(unsafeEnumValue); +jni::local_ref JSharedArray::create(jsi::Runtime& runtime, jsi::ArrayBuffer arrayBuffer) { + jni::local_ref instance = newObjectCxxArgs(runtime, std::make_shared(std::move(arrayBuffer))); + instance->cthis()->_javaPart = jni::make_global(instance); + return instance; } -jni::local_ref JSharedArray::create(jsi::Runtime& runtime, TypedArrayBase array) { - return newObjectCxxArgs(runtime, std::make_shared(std::move(array))); +JSharedArray::JSharedArray(jsi::Runtime& runtime, std::shared_ptr arrayBuffer) { + size_t size = arrayBuffer->size(runtime); + jni::local_ref byteBuffer = JByteBuffer::allocateDirect(size); + + _arrayBuffer = arrayBuffer; + _byteBuffer = jni::make_global(byteBuffer); + _size = size; } -jni::global_ref JSharedArray::wrapInByteBuffer(jsi::Runtime& runtime, std::shared_ptr typedArray) { - jsi::ArrayBuffer arrayBuffer = typedArray->getBuffer(runtime); - __android_log_print(ANDROID_LOG_INFO, TAG, "Wrapping ArrayBuffer in a JNI ByteBuffer..."); - auto byteBuffer = jni::JByteBuffer::wrapBytes(arrayBuffer.data(runtime), arrayBuffer.size(runtime)); - __android_log_print(ANDROID_LOG_INFO, TAG, "Successfully created TypedArray (JNI Size: %i)!", byteBuffer->getDirectSize()); - return jni::make_global(byteBuffer); -} - -JSharedArray::JSharedArray(jsi::Runtime& runtime, std::shared_ptr array) { - _array = array; - _byteBuffer = wrapInByteBuffer(runtime, _array); - _size = _array->size(runtime); -} - -JSharedArray::JSharedArray(const jni::alias_ref& javaThis, - const jni::alias_ref& proxy, int dataType, int size) { +JSharedArray::JSharedArray(const jni::alias_ref& javaThis, const jni::alias_ref& proxy, + jni::alias_ref byteBuffer) { _javaPart = jni::make_global(javaThis); #if VISION_CAMERA_ENABLE_FRAME_PROCESSORS @@ -40,37 +33,48 @@ JSharedArray::JSharedArray(const jni::alias_ref& ja #else jsi::Runtime& runtime = *proxy->cthis()->getJSRuntime(); #endif - TypedArrayKind kind = getTypedArrayKind(dataType); - __android_log_print(ANDROID_LOG_INFO, TAG, "Allocating ArrayBuffer with size %i and type %i...", size, dataType); - _array = std::make_shared(runtime, size, kind); - _byteBuffer = wrapInByteBuffer(runtime, _array); - _size = size; + __android_log_print(ANDROID_LOG_INFO, TAG, "Allocating ArrayBuffer with size %i...", byteBuffer->getDirectSize()); + _byteBuffer = jni::make_global(byteBuffer); + _size = _byteBuffer->getDirectSize(); + + auto mutableByteBuffer = std::make_shared(byteBuffer); + _arrayBuffer = std::make_shared(runtime, std::move(mutableByteBuffer)); } +JSharedArray::JSharedArray(const jni::alias_ref& javaThis, + const jni::alias_ref& proxy, int size) + : JSharedArray(javaThis, proxy, JByteBuffer::allocateDirect(size)) {} + void JSharedArray::registerNatives() { registerHybrid({ - makeNativeMethod("initHybrid", JSharedArray::initHybrid), + makeNativeMethod("initHybrid", JSharedArray::initHybridAllocate), + makeNativeMethod("initHybrid", JSharedArray::initHybridWrap), makeNativeMethod("getByteBuffer", JSharedArray::getByteBuffer), makeNativeMethod("getSize", JSharedArray::getSize), }); } -jni::local_ref JSharedArray::getByteBuffer() { - return jni::make_local(_byteBuffer); +jni::global_ref JSharedArray::getByteBuffer() { + return _byteBuffer; +} + +std::shared_ptr JSharedArray::getArrayBuffer() { + return _arrayBuffer; } jint JSharedArray::getSize() { return _size; } -std::shared_ptr JSharedArray::getTypedArray() { - return _array; +jni::local_ref +JSharedArray::initHybridAllocate(jni::alias_ref javaThis, jni::alias_ref proxy, jint size) { + return makeCxxInstance(javaThis, proxy, size); } -jni::local_ref JSharedArray::initHybrid(jni::alias_ref javaThis, - jni::alias_ref proxy, jint type, - jint size) { - return makeCxxInstance(javaThis, proxy, type, size); +jni::local_ref JSharedArray::initHybridWrap(jni::alias_ref javaThis, + jni::alias_ref proxy, + jni::alias_ref byteBuffer) { + return makeCxxInstance(javaThis, proxy, byteBuffer); } } // namespace vision diff --git a/package/android/src/main/cpp/frameprocessor/java-bindings/JSharedArray.h b/package/android/src/main/cpp/frameprocessor/java-bindings/JSharedArray.h index 8a5c3f3..d9f6612 100644 --- a/package/android/src/main/cpp/frameprocessor/java-bindings/JSharedArray.h +++ b/package/android/src/main/cpp/frameprocessor/java-bindings/JSharedArray.h @@ -4,8 +4,8 @@ #pragma once -#include "JSITypedArray.h" #include "JVisionCameraProxy.h" +#include "MutableJByteBuffer.h" #include #include #include @@ -20,30 +20,32 @@ public: static void registerNatives(); public: - static jni::local_ref create(jsi::Runtime& runtime, TypedArrayBase array); + static jni::local_ref create(jsi::Runtime& runtime, jsi::ArrayBuffer arrayBuffer); public: jint getSize(); - jni::local_ref getByteBuffer(); - std::shared_ptr getTypedArray(); - -private: - jni::global_ref wrapInByteBuffer(jsi::Runtime& runtime, std::shared_ptr typedArray); + jni::global_ref getByteBuffer(); + std::shared_ptr getArrayBuffer(); private: static auto constexpr TAG = "SharedArray"; friend HybridBase; jni::global_ref _javaPart; jni::global_ref _byteBuffer; - std::shared_ptr _array; + std::shared_ptr _arrayBuffer; int _size; private: - explicit JSharedArray(jsi::Runtime& runtime, std::shared_ptr array); + explicit JSharedArray(jsi::Runtime& runtime, std::shared_ptr arrayBuffer); explicit JSharedArray(const jni::alias_ref& javaThis, const jni::alias_ref& proxy, - int dataType, int size); - static jni::local_ref initHybrid(jni::alias_ref javaThis, - jni::alias_ref proxy, jint dataType, jint size); + int size); + explicit JSharedArray(const jni::alias_ref& javaThis, const jni::alias_ref& proxy, + jni::alias_ref byteBuffer); + static jni::local_ref initHybridAllocate(jni::alias_ref javaThis, + jni::alias_ref proxy, jint size); + static jni::local_ref initHybridWrap(jni::alias_ref javaThis, + jni::alias_ref proxy, + jni::alias_ref byteBuffer); }; } // namespace vision diff --git a/package/android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraProxy.cpp b/package/android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraProxy.cpp index d235852..f178cb9 100644 --- a/package/android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraProxy.cpp +++ b/package/android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraProxy.cpp @@ -11,7 +11,6 @@ #include #include "FrameProcessorPluginHostObject.h" -#include "JSITypedArray.h" #if VISION_CAMERA_ENABLE_FRAME_PROCESSORS #include @@ -51,10 +50,7 @@ JVisionCameraProxy::JVisionCameraProxy(const jni::alias_refgetJsRuntime()); - invalidateArrayBufferCache(_workletContext->getWorkletRuntime()); + __android_log_write(ANDROID_LOG_INFO, TAG, "Destroying JVisionCameraProxy..."); #endif } diff --git a/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt b/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt index 0b2b69b..343397e 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt @@ -96,8 +96,9 @@ class VideoPipeline( if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { Log.i(TAG, "Using API 29 for GPU ImageReader...") - // GPU_SAMPLED because we redirect to OpenGL - val usage = HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE + // If we are in PRIVATE, we just pass it to the GPU as efficiently as possible - so use GPU flag. + // If we are in YUV/RGB/..., we probably want to access Frame data - so use CPU flag. + val usage = if (format == ImageFormat.PRIVATE) HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE else HardwareBuffer.USAGE_CPU_READ_OFTEN imageReader = ImageReader.newInstance(width, height, format, MAX_IMAGES, usage) imageWriter = ImageWriter.newInstance(glSurface, MAX_IMAGES, format) } else { @@ -109,17 +110,21 @@ class VideoPipeline( Log.i(TAG, "ImageReader::onImageAvailable!") val image = reader.acquireNextImage() ?: return@setOnImageAvailableListener - // TODO: Get correct orientation and isMirrored - val frame = Frame(image, image.timestamp, Orientation.PORTRAIT, isMirrored) - frame.incrementRefCount() - frameProcessor?.call(frame) + try { + // TODO: Get correct orientation and isMirrored + val frame = Frame(image, image.timestamp, Orientation.PORTRAIT, isMirrored) + frame.incrementRefCount() + frameProcessor?.call(frame) - if (hasOutputs) { - // If we have outputs (e.g. a RecordingSession), pass the frame along to the OpenGL pipeline - imageWriter!!.queueInputImage(image) + if (hasOutputs) { + // If we have outputs (e.g. a RecordingSession), pass the frame along to the OpenGL pipeline + imageWriter!!.queueInputImage(image) + } + + frame.decrementRefCount() + } catch (e: Throwable) { + Log.e(TAG, "Failed to call Frame Processor!", e) } - - frame.decrementRefCount() }, CameraQueues.videoQueue.handler) surface = imageReader!!.surface diff --git a/package/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java b/package/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java index 0c6907a..f48006d 100644 --- a/package/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java +++ b/package/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java @@ -15,7 +15,6 @@ public class Frame { private final long timestamp; private final Orientation orientation; private int refCount = 0; - private HardwareBuffer hardwareBuffer = null; public Frame(Image image, long timestamp, Orientation orientation, boolean isMirrored) { this.image = image; @@ -114,10 +113,7 @@ public class Frame { if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P) { throw new HardwareBuffersNotAvailableError(); } - if (hardwareBuffer == null) { - hardwareBuffer = getImage().getHardwareBuffer(); - } - return hardwareBuffer; + return getImage().getHardwareBuffer(); } @SuppressWarnings("unused") @@ -142,9 +138,6 @@ public class Frame { private synchronized void close() { synchronized (this) { - if (hardwareBuffer != null) { - hardwareBuffer.close(); - } image.close(); } } diff --git a/package/android/src/main/java/com/mrousavy/camera/frameprocessor/SharedArray.java b/package/android/src/main/java/com/mrousavy/camera/frameprocessor/SharedArray.java index c01541b..97679d2 100644 --- a/package/android/src/main/java/com/mrousavy/camera/frameprocessor/SharedArray.java +++ b/package/android/src/main/java/com/mrousavy/camera/frameprocessor/SharedArray.java @@ -20,18 +20,26 @@ public final class SharedArray { @DoNotStrip @Keep - public SharedArray(HybridData hybridData) { + private SharedArray(HybridData hybridData) { mHybridData = hybridData; } /** * Allocate a new SharedArray. Use `getByteBuffer` to obtain a reference to the direct ByteBuffer for writing. * @param proxy The VisionCamera Proxy from the Frame Processor Plugin's initializer. - * @param dataType The ArrayBuffer's data type. `Type.Int8Array` = `Int8Array` in JS * @param size The size of the ArrayBuffer. */ - public SharedArray(VisionCameraProxy proxy, Type dataType, int size) { - mHybridData = initHybrid(proxy, dataType.ordinal(), size); + public SharedArray(VisionCameraProxy proxy, int size) { + mHybridData = initHybrid(proxy, size); + } + + /** + * Wraps the given ByteBuffer in a JSI ArrayBuffer. Using `getByteBuffer` will return the same instance which can be used for writing. + * @param proxy The VisionCamera Proxy from the Frame Processor Plugin's initializer. + * @param byteBuffer The ByteBuffer to wrap. + */ + public SharedArray(VisionCameraProxy proxy, ByteBuffer byteBuffer) { + mHybridData = initHybrid(proxy, byteBuffer); } /** @@ -44,21 +52,6 @@ public final class SharedArray { */ public native int getSize(); - private native HybridData initHybrid(VisionCameraProxy proxy, int dataType, int size); - - /** - * The Type of the SharedArray. - */ - public enum Type { - // Values start at 0 and need to match with JSITypedArray.h::TypedArrayKind - Int8Array, - Int16Array, - Int32Array, - Uint8Array, - Uint8ClampedArray, - Uint16Array, - Uint32Array, - Float32Array, - Float64Array, - } + private native HybridData initHybrid(VisionCameraProxy proxy, int size); + private native HybridData initHybrid(VisionCameraProxy proxy, ByteBuffer byteBuffer); } diff --git a/package/cpp/JSITypedArray.cpp b/package/cpp/JSITypedArray.cpp deleted file mode 100644 index 0662a24..0000000 --- a/package/cpp/JSITypedArray.cpp +++ /dev/null @@ -1,314 +0,0 @@ -// -// JSITypedArray.cpp -// VisionCamera -// -// Created by Marc Rousavy on 21.02.23. -// Copyright © 2023 mrousavy. All rights reserved. -// - -// Copied & Adapted from -// https://github.com/expo/expo/blob/main/packages/expo-gl/common/EXTypedArrayApi.cpp Credits to -// Expo - -#include "JSITypedArray.h" - -#include -#include -#include -#include -#include -#include - -namespace vision { - -template using ContentType = typename typedArrayTypeMap::type; - -enum class Prop { - Buffer, // "buffer" - Constructor, // "constructor" - Name, // "name" - Proto, // "__proto__" - Length, // "length" - ByteLength, // "byteLength" - ByteOffset, // "offset" - IsView, // "isView" - ArrayBuffer, // "ArrayBuffer" - Int8Array, // "Int8Array" - Int16Array, // "Int16Array" - Int32Array, // "Int32Array" - Uint8Array, // "Uint8Array" - Uint8ClampedArray, // "Uint8ClampedArray" - Uint16Array, // "Uint16Array" - Uint32Array, // "Uint32Array" - Float32Array, // "Float32Array" - Float64Array, // "Float64Array" -}; - -class PropNameIDCache { -public: - const jsi::PropNameID& get(jsi::Runtime& runtime, Prop prop) { - auto key = reinterpret_cast(&runtime); - if (this->props.find(key) == this->props.end()) { - this->props[key] = std::unordered_map>(); - } - if (!this->props[key][prop]) { - this->props[key][prop] = std::make_unique(createProp(runtime, prop)); - } - return *(this->props[key][prop]); - } - - const jsi::PropNameID& getConstructorNameProp(jsi::Runtime& runtime, TypedArrayKind kind); - - void invalidate(uintptr_t key) { - if (props.find(key) != props.end()) { - props[key].clear(); - } - } - -private: - std::unordered_map>> props; - - jsi::PropNameID createProp(jsi::Runtime& runtime, Prop prop); -}; - -PropNameIDCache propNameIDCache; - -void invalidateArrayBufferCache(jsi::Runtime& runtime) { - auto key = reinterpret_cast(&runtime); - propNameIDCache.invalidate(key); -} - -TypedArrayKind getTypedArrayKindForName(const std::string& name); - -TypedArrayBase::TypedArrayBase(jsi::Runtime& runtime, size_t size, TypedArrayKind kind) - : TypedArrayBase(runtime, runtime.global() - .getProperty(runtime, propNameIDCache.getConstructorNameProp(runtime, kind)) - .asObject(runtime) - .asFunction(runtime) - .callAsConstructor(runtime, {static_cast(size)}) - .asObject(runtime)) {} - -TypedArrayBase::TypedArrayBase(jsi::Runtime& runtime, const jsi::Object& obj) : jsi::Object(jsi::Value(runtime, obj).asObject(runtime)) {} - -TypedArrayKind TypedArrayBase::getKind(jsi::Runtime& runtime) const { - auto constructorName = this->getProperty(runtime, propNameIDCache.get(runtime, Prop::Constructor)) - .asObject(runtime) - .getProperty(runtime, propNameIDCache.get(runtime, Prop::Name)) - .asString(runtime) - .utf8(runtime); - return getTypedArrayKindForName(constructorName); -} - -size_t TypedArrayBase::size(jsi::Runtime& runtime) const { - return getProperty(runtime, propNameIDCache.get(runtime, Prop::Length)).asNumber(); -} - -size_t TypedArrayBase::length(jsi::Runtime& runtime) const { - return getProperty(runtime, propNameIDCache.get(runtime, Prop::Length)).asNumber(); -} - -size_t TypedArrayBase::byteLength(jsi::Runtime& runtime) const { - return getProperty(runtime, propNameIDCache.get(runtime, Prop::ByteLength)).asNumber(); -} - -size_t TypedArrayBase::byteOffset(jsi::Runtime& runtime) const { - return getProperty(runtime, propNameIDCache.get(runtime, Prop::ByteOffset)).asNumber(); -} - -bool TypedArrayBase::hasBuffer(jsi::Runtime& runtime) const { - auto buffer = getProperty(runtime, propNameIDCache.get(runtime, Prop::Buffer)); - return buffer.isObject() && buffer.asObject(runtime).isArrayBuffer(runtime); -} - -std::vector TypedArrayBase::toVector(jsi::Runtime& runtime) { - auto start = reinterpret_cast(getBuffer(runtime).data(runtime) + byteOffset(runtime)); - auto end = start + byteLength(runtime); - return std::vector(start, end); -} - -jsi::ArrayBuffer TypedArrayBase::getBuffer(jsi::Runtime& runtime) const { - auto buffer = getProperty(runtime, propNameIDCache.get(runtime, Prop::Buffer)); - if (buffer.isObject() && buffer.asObject(runtime).isArrayBuffer(runtime)) { - return buffer.asObject(runtime).getArrayBuffer(runtime); - } else { - throw std::runtime_error("no ArrayBuffer attached"); - } -} - -bool isTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj) { - auto jsVal = runtime.global() - .getProperty(runtime, propNameIDCache.get(runtime, Prop::ArrayBuffer)) - .asObject(runtime) - .getProperty(runtime, propNameIDCache.get(runtime, Prop::IsView)) - .asObject(runtime) - .asFunction(runtime) - .callWithThis(runtime, runtime.global(), {jsi::Value(runtime, jsObj)}); - if (jsVal.isBool()) { - return jsVal.getBool(); - } else { - throw std::runtime_error("value is not a boolean"); - } -} - -TypedArrayBase getTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj) { - auto jsVal = runtime.global() - .getProperty(runtime, propNameIDCache.get(runtime, Prop::ArrayBuffer)) - .asObject(runtime) - .getProperty(runtime, propNameIDCache.get(runtime, Prop::IsView)) - .asObject(runtime) - .asFunction(runtime) - .callWithThis(runtime, runtime.global(), {jsi::Value(runtime, jsObj)}); - if (jsVal.isBool()) { - return TypedArrayBase(runtime, jsObj); - } else { - throw std::runtime_error("value is not a boolean"); - } -} - -std::vector arrayBufferToVector(jsi::Runtime& runtime, jsi::Object& jsObj) { - if (!jsObj.isArrayBuffer(runtime)) { - throw std::runtime_error("Object is not an ArrayBuffer"); - } - auto jsArrayBuffer = jsObj.getArrayBuffer(runtime); - - uint8_t* dataBlock = jsArrayBuffer.data(runtime); - size_t blockSize = jsArrayBuffer.getProperty(runtime, propNameIDCache.get(runtime, Prop::ByteLength)).asNumber(); - return std::vector(dataBlock, dataBlock + blockSize); -} - -void arrayBufferUpdate(jsi::Runtime& runtime, jsi::ArrayBuffer& buffer, std::vector data, size_t offset) { - uint8_t* dataBlock = buffer.data(runtime); - size_t blockSize = buffer.size(runtime); - if (data.size() > blockSize) { - throw jsi::JSError(runtime, "ArrayBuffer is to small to fit data"); - } - std::copy(data.begin(), data.end(), dataBlock + offset); -} - -template TypedArray::TypedArray(jsi::Runtime& runtime, size_t size) : TypedArrayBase(runtime, size, T) {} - -template -TypedArray::TypedArray(jsi::Runtime& runtime, std::vector> data) : TypedArrayBase(runtime, data.size(), T) { - update(runtime, data); -} - -template TypedArray::TypedArray(TypedArrayBase&& base) : TypedArrayBase(std::move(base)) {} - -template std::vector> TypedArray::toVector(jsi::Runtime& runtime) { - auto start = reinterpret_cast*>(getBuffer(runtime).data(runtime) + byteOffset(runtime)); - auto end = start + size(runtime); - return std::vector>(start, end); -} - -template void TypedArray::update(jsi::Runtime& runtime, const std::vector>& data) { - if (data.size() != size(runtime)) { - throw jsi::JSError(runtime, "TypedArray can only be updated with a vector of the same size"); - } - uint8_t* rawData = getBuffer(runtime).data(runtime) + byteOffset(runtime); - std::copy(data.begin(), data.end(), reinterpret_cast*>(rawData)); -} - -template void TypedArray::updateUnsafe(jsi::Runtime& runtime, ContentType* data, size_t length) { - if (length != size(runtime)) { - throw jsi::JSError(runtime, "TypedArray can only be updated with an array of the same size"); - } - uint8_t* rawData = getBuffer(runtime).data(runtime) + byteOffset(runtime); - memcpy(rawData, data, length); -} - -template uint8_t* TypedArray::data(jsi::Runtime& runtime) { - return getBuffer(runtime).data(runtime) + byteOffset(runtime); -} - -const jsi::PropNameID& PropNameIDCache::getConstructorNameProp(jsi::Runtime& runtime, TypedArrayKind kind) { - switch (kind) { - case TypedArrayKind::Int8Array: - return get(runtime, Prop::Int8Array); - case TypedArrayKind::Int16Array: - return get(runtime, Prop::Int16Array); - case TypedArrayKind::Int32Array: - return get(runtime, Prop::Int32Array); - case TypedArrayKind::Uint8Array: - return get(runtime, Prop::Uint8Array); - case TypedArrayKind::Uint8ClampedArray: - return get(runtime, Prop::Uint8ClampedArray); - case TypedArrayKind::Uint16Array: - return get(runtime, Prop::Uint16Array); - case TypedArrayKind::Uint32Array: - return get(runtime, Prop::Uint32Array); - case TypedArrayKind::Float32Array: - return get(runtime, Prop::Float32Array); - case TypedArrayKind::Float64Array: - return get(runtime, Prop::Float64Array); - } -} - -jsi::PropNameID PropNameIDCache::createProp(jsi::Runtime& runtime, Prop prop) { - auto create = [&](const std::string& propName) { return jsi::PropNameID::forUtf8(runtime, propName); }; - switch (prop) { - case Prop::Buffer: - return create("buffer"); - case Prop::Constructor: - return create("constructor"); - case Prop::Name: - return create("name"); - case Prop::Proto: - return create("__proto__"); - case Prop::Length: - return create("length"); - case Prop::ByteLength: - return create("byteLength"); - case Prop::ByteOffset: - return create("byteOffset"); - case Prop::IsView: - return create("isView"); - case Prop::ArrayBuffer: - return create("ArrayBuffer"); - case Prop::Int8Array: - return create("Int8Array"); - case Prop::Int16Array: - return create("Int16Array"); - case Prop::Int32Array: - return create("Int32Array"); - case Prop::Uint8Array: - return create("Uint8Array"); - case Prop::Uint8ClampedArray: - return create("Uint8ClampedArray"); - case Prop::Uint16Array: - return create("Uint16Array"); - case Prop::Uint32Array: - return create("Uint32Array"); - case Prop::Float32Array: - return create("Float32Array"); - case Prop::Float64Array: - return create("Float64Array"); - } -} - -std::unordered_map nameToKindMap = { - {"Int8Array", TypedArrayKind::Int8Array}, - {"Int16Array", TypedArrayKind::Int16Array}, - {"Int32Array", TypedArrayKind::Int32Array}, - {"Uint8Array", TypedArrayKind::Uint8Array}, - {"Uint8ClampedArray", TypedArrayKind::Uint8ClampedArray}, - {"Uint16Array", TypedArrayKind::Uint16Array}, - {"Uint32Array", TypedArrayKind::Uint32Array}, - {"Float32Array", TypedArrayKind::Float32Array}, - {"Float64Array", TypedArrayKind::Float64Array}, -}; - -TypedArrayKind getTypedArrayKindForName(const std::string& name) { - return nameToKindMap.at(name); -} - -template class TypedArray; -template class TypedArray; -template class TypedArray; -template class TypedArray; -template class TypedArray; -template class TypedArray; -template class TypedArray; -template class TypedArray; -template class TypedArray; - -} // namespace vision diff --git a/package/cpp/JSITypedArray.h b/package/cpp/JSITypedArray.h deleted file mode 100644 index bed1c53..0000000 --- a/package/cpp/JSITypedArray.h +++ /dev/null @@ -1,141 +0,0 @@ -// -// JSITypedArray.h -// VisionCamera -// -// Created by Marc Rousavy on 21.02.23. -// Copyright © 2023 mrousavy. All rights reserved. -// - -// Copied & Adapted from -// https://github.com/expo/expo/blob/main/packages/expo-gl/common/EXTypedArrayApi.h Credits to Expo - -#pragma once - -#include -#include -#include - -namespace jsi = facebook::jsi; - -namespace vision { - -enum class TypedArrayKind { - Int8Array, - Int16Array, - Int32Array, - Uint8Array, - Uint8ClampedArray, - Uint16Array, - Uint32Array, - Float32Array, - Float64Array, -}; - -template class TypedArray; - -template struct typedArrayTypeMap; -template <> struct typedArrayTypeMap { - typedef int8_t type; -}; -template <> struct typedArrayTypeMap { - typedef int16_t type; -}; -template <> struct typedArrayTypeMap { - typedef int32_t type; -}; -template <> struct typedArrayTypeMap { - typedef uint8_t type; -}; -template <> struct typedArrayTypeMap { - typedef uint8_t type; -}; -template <> struct typedArrayTypeMap { - typedef uint16_t type; -}; -template <> struct typedArrayTypeMap { - typedef uint32_t type; -}; -template <> struct typedArrayTypeMap { - typedef float type; -}; -template <> struct typedArrayTypeMap { - typedef double type; -}; - -void invalidateArrayBufferCache(jsi::Runtime& runtime); - -class TypedArrayBase : public jsi::Object { -public: - template using ContentType = typename typedArrayTypeMap::type; - - TypedArrayBase(jsi::Runtime&, size_t, TypedArrayKind); - TypedArrayBase(jsi::Runtime&, const jsi::Object&); - TypedArrayBase(TypedArrayBase&&) = default; - TypedArrayBase& operator=(TypedArrayBase&&) = default; - - TypedArrayKind getKind(jsi::Runtime& runtime) const; - - template TypedArray get(jsi::Runtime& runtime) const&; - template TypedArray get(jsi::Runtime& runtime) &&; - template TypedArray as(jsi::Runtime& runtime) const&; - template TypedArray as(jsi::Runtime& runtime) &&; - - size_t size(jsi::Runtime& runtime) const; - size_t length(jsi::Runtime& runtime) const; - size_t byteLength(jsi::Runtime& runtime) const; - size_t byteOffset(jsi::Runtime& runtime) const; - bool hasBuffer(jsi::Runtime& runtime) const; - - std::vector toVector(jsi::Runtime& runtime); - jsi::ArrayBuffer getBuffer(jsi::Runtime& runtime) const; - -private: - template friend class TypedArray; -}; - -bool isTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj); -TypedArrayBase getTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj); - -std::vector arrayBufferToVector(jsi::Runtime& runtime, jsi::Object& jsObj); -void arrayBufferUpdate(jsi::Runtime& runtime, jsi::ArrayBuffer& buffer, std::vector data, size_t offset); - -template class TypedArray : public TypedArrayBase { -public: - explicit TypedArray(TypedArrayBase&& base); - TypedArray(jsi::Runtime& runtime, size_t size); - TypedArray(jsi::Runtime& runtime, std::vector> data); - TypedArray(TypedArray&&) = default; - TypedArray& operator=(TypedArray&&) = default; - - std::vector> toVector(jsi::Runtime& runtime); - void update(jsi::Runtime& runtime, const std::vector>& data); - void updateUnsafe(jsi::Runtime& runtime, ContentType* data, size_t length); - uint8_t* data(jsi::Runtime& runtime); -}; - -template TypedArray TypedArrayBase::get(jsi::Runtime& runtime) const& { - assert(getKind(runtime) == T); - (void)runtime; // when assert is disabled we need to mark this as used - return TypedArray(jsi::Value(runtime, jsi::Value(runtime, *this).asObject(runtime))); -} - -template TypedArray TypedArrayBase::get(jsi::Runtime& runtime) && { - assert(getKind(runtime) == T); - (void)runtime; // when assert is disabled we need to mark this as used - return TypedArray(std::move(*this)); -} - -template TypedArray TypedArrayBase::as(jsi::Runtime& runtime) const& { - if (getKind(runtime) != T) { - throw jsi::JSError(runtime, "Object is not a TypedArray"); - } - return get(runtime); -} - -template TypedArray TypedArrayBase::as(jsi::Runtime& runtime) && { - if (getKind(runtime) != T) { - throw jsi::JSError(runtime, "Object is not a TypedArray"); - } - return std::move(*this).get(runtime); -} -} // namespace vision diff --git a/package/cpp/MutableRawBuffer.cpp b/package/cpp/MutableRawBuffer.cpp new file mode 100644 index 0000000..8bdb4b8 --- /dev/null +++ b/package/cpp/MutableRawBuffer.cpp @@ -0,0 +1,36 @@ +// +// MutableRawBuffer.cpp +// VisionCamera +// +// Created by Marc Rousavy on 17.01.24. +// Copyright © 2024 mrousavy. All rights reserved. +// + +#include "MutableRawBuffer.h" +#include +#include + +namespace vision { + +MutableRawBuffer::MutableRawBuffer(uint8_t* data, size_t size, std::function cleanup) + : _data(data), _size(size), _cleanup(std::move(cleanup)) {} + +MutableRawBuffer::MutableRawBuffer(size_t size) { + _size = size; + _data = (uint8_t*)malloc(size * sizeof(uint8_t)); + _cleanup = [=]() { free(_data); }; +} + +MutableRawBuffer::~MutableRawBuffer() { + _cleanup(); +} + +size_t MutableRawBuffer::size() const { + return _size; +} + +uint8_t* MutableRawBuffer::data() { + return _data; +} + +} // namespace vision diff --git a/package/cpp/MutableRawBuffer.h b/package/cpp/MutableRawBuffer.h new file mode 100644 index 0000000..56a8a6d --- /dev/null +++ b/package/cpp/MutableRawBuffer.h @@ -0,0 +1,36 @@ +// +// MutableRawBuffer.h +// VisionCamera +// +// Created by Marc Rousavy on 17.01.24. +// Copyright © 2024 mrousavy. All rights reserved. +// + +#pragma once + +#include +#include +#include + +namespace vision { + +using namespace facebook; + +class MutableRawBuffer : public jsi::MutableBuffer { + +public: + explicit MutableRawBuffer(size_t size); + explicit MutableRawBuffer(uint8_t* data, size_t size, std::function cleanup); + ~MutableRawBuffer(); + +public: + uint8_t* data() override; + size_t size() const override; + +private: + uint8_t* _data; + size_t _size; + std::function _cleanup; +}; + +} // namespace vision diff --git a/package/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java b/package/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java index 9a16065..121e29c 100644 --- a/package/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java +++ b/package/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java @@ -19,17 +19,18 @@ import java.util.Map; public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin { SharedArray _sharedArray; + private static final String TAG = "ExamplePlugin"; @Override public Object callback(@NotNull Frame frame, @Nullable Map params) { if (params == null) return null; Image image = frame.getImage(); - Log.d("ExamplePlugin", image.getWidth() + " x " + image.getHeight() + " Image with format #" + image.getFormat() + ". Logging " + params.size() + " parameters:"); + Log.d(TAG, image.getWidth() + " x " + image.getHeight() + " Image with format #" + image.getFormat() + ". Logging " + params.size() + " parameters:"); for (String key : params.keySet()) { Object value = params.get(key); - Log.d("ExamplePlugin", " -> " + (value == null ? "(null)" : value + " (" + value.getClass().getName() + ")")); + Log.d(TAG, " -> " + (value == null ? "(null)" : value + " (" + value.getClass().getName() + ")")); } Map map = new HashMap<>(); @@ -53,7 +54,13 @@ public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin { ExampleFrameProcessorPlugin(VisionCameraProxy proxy, @Nullable Map options) { super(); - _sharedArray = new SharedArray(proxy, SharedArray.Type.Uint8Array, 5); - Log.d("ExamplePlugin", "ExampleFrameProcessorPlugin initialized with options: " + options); + _sharedArray = new SharedArray(proxy, 5); + Log.d(TAG, "Successfully allocated SharedArray! Size: " + _sharedArray.getSize()); + + ByteBuffer buffer = ByteBuffer.allocateDirect(10); + SharedArray testArray = new SharedArray(proxy, buffer); + Log.d(TAG, "Successfully wrapped SharedArray in ByteBuffer! Size: " + testArray.getSize()); + + Log.d(TAG, "ExampleFrameProcessorPlugin initialized with options: " + options); } } diff --git a/package/example/ios/Frame Processor Plugins/Example Plugin/ExampleFrameProcessorPlugin.m b/package/example/ios/Frame Processor Plugins/Example Plugin/ExampleFrameProcessorPlugin.m index 9e8b0ce..f264355 100644 --- a/package/example/ios/Frame Processor Plugins/Example Plugin/ExampleFrameProcessorPlugin.m +++ b/package/example/ios/Frame Processor Plugins/Example Plugin/ExampleFrameProcessorPlugin.m @@ -25,7 +25,6 @@ withOptions:(NSDictionary* _Nullable)options { if (self = [super initWithProxy:proxy withOptions:options]) { _sharedArray = [[SharedArray alloc] initWithProxy:proxy - type:Int8Array size:5]; NSLog(@"ExampleFrameProcessorPlugin initialized with options: %@", options); } diff --git a/package/ios/Frame Processor/FrameHostObject.mm b/package/ios/Frame Processor/FrameHostObject.mm index 54a2bbc..76436d8 100644 --- a/package/ios/Frame Processor/FrameHostObject.mm +++ b/package/ios/Frame Processor/FrameHostObject.mm @@ -7,13 +7,12 @@ // #import "FrameHostObject.h" +#import "MutableRawBuffer.h" #import "UIImageOrientation+descriptor.h" #import "WKTJsiHostObject.h" #import #import -#import "../../cpp/JSITypedArray.h" - std::vector FrameHostObject::getPropertyNames(jsi::Runtime& rt) { std::vector result; // Ref Management @@ -110,21 +109,23 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr static constexpr auto ARRAYBUFFER_CACHE_PROP_NAME = "__frameArrayBufferCache"; if (!runtime.global().hasProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME)) { - vision::TypedArray arrayBuffer(runtime, arraySize); - runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer); + auto mutableBuffer = std::make_shared(arraySize); + jsi::ArrayBuffer arrayBuffer(runtime, mutableBuffer); + runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, std::move(arrayBuffer)); } auto arrayBufferCache = runtime.global().getPropertyAsObject(runtime, ARRAYBUFFER_CACHE_PROP_NAME); - auto arrayBuffer = vision::getTypedArray(runtime, arrayBufferCache).get(runtime); + auto arrayBuffer = arrayBufferCache.getArrayBuffer(runtime); if (arrayBuffer.size(runtime) != arraySize) { - arrayBuffer = vision::TypedArray(runtime, arraySize); + auto mutableBuffer = std::make_shared(arraySize); + arrayBuffer = jsi::ArrayBuffer(runtime, mutableBuffer); runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer); } CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); auto buffer = (uint8_t*)CVPixelBufferGetBaseAddress(pixelBuffer); - arrayBuffer.updateUnsafe(runtime, buffer, arraySize); + memcpy(arrayBuffer.data(runtime), buffer, arraySize); CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); return arrayBuffer; @@ -201,3 +202,5 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr // fallback to base implementation return HostObject::get(runtime, propName); } + +#undef JSI_FUNC diff --git a/package/ios/Frame Processor/JSINSObjectConversion.mm b/package/ios/Frame Processor/JSINSObjectConversion.mm index 29480a8..5d9a291 100644 --- a/package/ios/Frame Processor/JSINSObjectConversion.mm +++ b/package/ios/Frame Processor/JSINSObjectConversion.mm @@ -19,7 +19,6 @@ #import "../Frame Processor/Frame.h" #import "../Frame Processor/FrameHostObject.h" #import "../Frame Processor/SharedArray.h" -#import "JSITypedArray.h" #import #import #import @@ -61,8 +60,7 @@ jsi::Array convertNSArrayToJSIArray(jsi::Runtime& runtime, NSArray* value) { } jsi::Object convertSharedArrayToJSIArrayBuffer(jsi::Runtime& runtime, SharedArray* sharedArray) { - std::shared_ptr array = sharedArray.typedArray; - return array->getBuffer(runtime); + return sharedArray.arrayBuffer->getArrayBuffer(runtime); } jsi::Value convertObjCObjectToJSIValue(jsi::Runtime& runtime, id value) { @@ -171,8 +169,8 @@ id convertJSIValueToObjCObject(jsi::Runtime& runtime, const jsi::Value& value, s } } else if (o.isArrayBuffer(runtime)) { // ArrayBuffer - auto typedArray = std::make_shared(vision::getTypedArray(runtime, o)); - return [[SharedArray alloc] initWithRuntime:runtime typedArray:typedArray]; + auto arrayBuffer = std::make_shared(o.getArrayBuffer(runtime)); + return [[SharedArray alloc] initWithRuntime:runtime arrayBuffer:arrayBuffer]; } else { // object return convertJSIObjectToNSDictionary(runtime, o, jsInvoker); diff --git a/package/ios/Frame Processor/SharedArray.h b/package/ios/Frame Processor/SharedArray.h index aafe56d..0f00ca9 100644 --- a/package/ios/Frame Processor/SharedArray.h +++ b/package/ios/Frame Processor/SharedArray.h @@ -12,39 +12,22 @@ #import #ifdef __cplusplus +#import "../../cpp/MutableRawBuffer.h" #import using namespace facebook; - -namespace vision { -// forward-declaration since we cannot import C++ headers here yet. -class TypedArrayBase; -} // namespace vision #endif -// Needs to be in sync with JSITypedArray.h as the index is used -typedef NS_ENUM(NSInteger, SharedArrayType) { - Int8Array, - Int16Array, - Int32Array, - Uint8Array, - Uint8ClampedArray, - Uint16Array, - Uint32Array, - Float32Array, - Float64Array, -}; - NS_ASSUME_NONNULL_BEGIN @interface SharedArray : NSObject - (instancetype)init NS_UNAVAILABLE; -- (instancetype)initWithProxy:(VisionCameraProxyHolder*)proxy type:(SharedArrayType)type size:(NSInteger)size; +- (instancetype)initWithProxy:(VisionCameraProxyHolder*)proxy size:(NSInteger)size; #ifdef __cplusplus -- (instancetype)initWithRuntime:(jsi::Runtime&)runtime typedArray:(std::shared_ptr)typedArray; +- (instancetype)initWithRuntime:(jsi::Runtime&)runtime arrayBuffer:(std::shared_ptr)arrayBuffer; -- (std::shared_ptr)typedArray; +- (std::shared_ptr)arrayBuffer; #endif @property(nonatomic, readonly, nonnull) uint8_t* data; diff --git a/package/ios/Frame Processor/SharedArray.mm b/package/ios/Frame Processor/SharedArray.mm index 39cfa1f..27c6cbc 100644 --- a/package/ios/Frame Processor/SharedArray.mm +++ b/package/ios/Frame Processor/SharedArray.mm @@ -7,7 +7,7 @@ // #import "SharedArray.h" -#import "JSITypedArray.h" +#import "../../cpp/MutableRawBuffer.h" #import #import @@ -16,35 +16,33 @@ using namespace facebook; @implementation SharedArray { uint8_t* _data; NSInteger _size; - std::shared_ptr _array; + std::shared_ptr _arrayBuffer; } -vision::TypedArrayKind getTypedArrayKind(int unsafeEnumValue) { - return static_cast(unsafeEnumValue); -} - -- (instancetype)initWithProxy:(VisionCameraProxyHolder*)proxy type:(SharedArrayType)type size:(NSInteger)size { +- (instancetype)initWithProxy:(VisionCameraProxyHolder*)proxy size:(NSInteger)size { if (self = [super init]) { jsi::Runtime& runtime = proxy.proxy->getWorkletRuntime(); - vision::TypedArrayKind kind = getTypedArrayKind((int)type); - _array = std::make_shared(vision::TypedArrayBase(runtime, size, kind)); - _data = _array->getBuffer(runtime).data(runtime); + + uint8_t* data = (uint8_t*)malloc(size * sizeof(uint8_t)); + auto mutableBuffer = std::make_shared(data, size, [=]() { free(data); }); + _arrayBuffer = std::make_shared(runtime, mutableBuffer); + _data = data; _size = size; } return self; } -- (instancetype)initWithRuntime:(jsi::Runtime&)runtime typedArray:(std::shared_ptr)typedArray { +- (instancetype)initWithRuntime:(jsi::Runtime&)runtime arrayBuffer:(std::shared_ptr)arrayBuffer { if (self = [super init]) { - _array = typedArray; - _data = _array->getBuffer(runtime).data(runtime); - _size = _array->getBuffer(runtime).size(runtime); + _arrayBuffer = arrayBuffer; + _data = _arrayBuffer->data(runtime); + _size = _arrayBuffer->size(runtime); } return self; } -- (std::shared_ptr)typedArray { - return _array; +- (std::shared_ptr)arrayBuffer { + return _arrayBuffer; } - (uint8_t*)data { diff --git a/package/ios/Frame Processor/VisionCameraProxy.mm b/package/ios/Frame Processor/VisionCameraProxy.mm index 8f518ab..b94a9bd 100644 --- a/package/ios/Frame Processor/VisionCameraProxy.mm +++ b/package/ios/Frame Processor/VisionCameraProxy.mm @@ -10,7 +10,6 @@ #import #import -#import "../../cpp/JSITypedArray.h" #import "FrameHostObject.h" #import "FrameProcessor.h" #import "FrameProcessorPluginHostObject.h" @@ -55,10 +54,7 @@ VisionCameraProxy::VisionCameraProxy(jsi::Runtime& runtime, std::shared_ptrgetJsRuntime()); - vision::invalidateArrayBufferCache(_workletContext->getWorkletRuntime()); + NSLog(@"VisionCameraProxy: Destroying VisionCameraProxy..."); } std::vector VisionCameraProxy::getPropertyNames(jsi::Runtime& runtime) { diff --git a/package/src/Frame.ts b/package/src/Frame.ts index 61594bd..f3937d0 100644 --- a/package/src/Frame.ts +++ b/package/src/Frame.ts @@ -57,7 +57,9 @@ export interface Frame { /** * Get the underlying data of the Frame as a uint8 array buffer. + * * The format of the buffer depends on the Frame's {@linkcode pixelFormat}. + * This function might fail if the {@linkcode pixelFormat} is `private`. * * Note that Frames are allocated on the GPU, so calling `toArrayBuffer()` will copy from the GPU to the CPU. * @@ -67,13 +69,14 @@ export interface Frame { * 'worklet' * * if (frame.pixelFormat === 'rgb') { - * const data = frame.toArrayBuffer() + * const buffer = frame.toArrayBuffer() + * const data = new Uint8Array(buffer) * console.log(`Pixel at 0,0: RGB(${data[0]}, ${data[1]}, ${data[2]})`) * } * }, []) * ``` */ - toArrayBuffer(): Uint8Array + toArrayBuffer(): ArrayBuffer /** * Returns a string representation of the frame. * @example