railbird/react-native-vision-camera
9
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Use JSI's ArrayBuffer instead of TypedArray (#2408)

Browse Source 
* feat: Use JSI's `ArrayBuffer` instead of `TypedArray`

* fix: Fix move memory

* feat: Implement iOS

* Format

* Update JSIJNIConversion.cpp

* fix: Fix Android `toArrayBuffer` and other

* Catch FP call errors

* Update return type

* Use `CPU_READ_OFTEN` flag as well

* CPU flag

* Run destructors under `jni::ThreadScope`

* Update FrameProcessorPluginHostObject.cpp

* fix: Fix `toArrayBuffer()` crash

* Update Frame.ts
This commit is contained in:
Marc Rousavy 2024-01-17 20:18:46 +01:00 committed by GitHub
parent 2f21609e39
commit ba1d7eec9c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 296 additions and 620 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
package/android/CMakeLists.txt
View File

@ -23,12 +23,15 @@ endif()
add_library(
${PACKAGE_NAME}
SHARED
../cpp/JSITypedArray.cpp
# Shared C++
../cpp/MutableRawBuffer.cpp
# Java JNI
src/main/cpp/VisionCamera.cpp
src/main/cpp/VideoPipeline.cpp
src/main/cpp/PassThroughShader.cpp
src/main/cpp/OpenGLContext.cpp
src/main/cpp/OpenGLRenderer.cpp
src/main/cpp/MutableJByteBuffer.cpp
# Frame Processor
src/main/cpp/frameprocessor/FrameHostObject.cpp
src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.cpp

33
package/android/src/main/cpp/MutableJByteBuffer.cpp Normal file
View File

@ -0,0 +1,33 @@
//
// Created by Marc Rousavy on 17.01.24.
//
#include "MutableJByteBuffer.h"
#include <fbjni/ByteBuffer.h>
#include <fbjni/fbjni.h>
#include <jsi/jsi.h>
namespace vision {
MutableJByteBuffer::MutableJByteBuffer(jni::alias_ref<jni::JByteBuffer> byteBuffer) {
_byteBuffer = jni::make_global(byteBuffer);
}
MutableJByteBuffer::~MutableJByteBuffer() noexcept {
jni::ThreadScope::WithClassLoader([&] { _byteBuffer.reset(); });
}
uint8_t* MutableJByteBuffer::data() {
return _byteBuffer->getDirectBytes();
}
size_t MutableJByteBuffer::size() const {
return _byteBuffer->getDirectSize();
}
jni::global_ref<jni::JByteBuffer> MutableJByteBuffer::getByteBuffer() {
return _byteBuffer;
}
} // namespace vision

33
package/android/src/main/cpp/MutableJByteBuffer.h Normal file
View File

@ -0,0 +1,33 @@
//
// Created by Marc Rousavy on 17.01.24.
//
#pragma once
#include <fbjni/ByteBuffer.h>
#include <fbjni/fbjni.h>
#include <jsi/jsi.h>
#include <memory>
namespace vision {
using namespace facebook;
class MutableJByteBuffer : public jsi::MutableBuffer {
public:
/**
* Wraps the given JByteBuffer in a MutableBuffer for use in JS.
*/
explicit MutableJByteBuffer(jni::alias_ref<jni::JByteBuffer> byteBuffer);
~MutableJByteBuffer();
public:
uint8_t* data() override;
size_t size() const override;
jni::global_ref<jni::JByteBuffer> getByteBuffer();
private:
jni::global_ref<jni::JByteBuffer> _byteBuffer;
};
} // namespace vision

13
package/android/src/main/cpp/frameprocessor/FrameHostObject.cpp
View File

@ -7,7 +7,7 @@
#include <fbjni/fbjni.h>
#include <jni.h>
#include "JSITypedArray.h"
#include "MutableRawBuffer.h"
#include <string>
#include <vector>
@ -102,15 +102,18 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
static constexpr auto ARRAYBUFFER_CACHE_PROP_NAME = "__frameArrayBufferCache";
if (!runtime.global().hasProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME)) {
vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray> arrayBuffer(runtime, size);
auto mutableBuffer = std::make_shared<vision::MutableRawBuffer>(size);
jsi::ArrayBuffer arrayBuffer(runtime, mutableBuffer);
runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer);
}
// Get from global JS cache
auto arrayBufferCache = runtime.global().getPropertyAsObject(runtime, ARRAYBUFFER_CACHE_PROP_NAME);
auto arrayBuffer = vision::getTypedArray(runtime, arrayBufferCache).get<vision::TypedArrayKind::Uint8ClampedArray>(runtime);
auto arrayBuffer = arrayBufferCache.getArrayBuffer(runtime);
if (arrayBuffer.size(runtime) != size) {
arrayBuffer = vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray>(runtime, size);
auto mutableBuffer = std::make_shared<vision::MutableRawBuffer>(size);
arrayBuffer = jsi::ArrayBuffer(runtime, mutableBuffer);
runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer);
}
@ -170,4 +173,6 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
return HostObject::get(runtime, propName);
}
#undef JSI_FUNC
} // namespace vision

7
package/android/src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.cpp
View File

@ -12,6 +12,13 @@ namespace vision {
using namespace facebook;
FrameProcessorPluginHostObject::FrameProcessorPluginHostObject(jni::alias_ref<JFrameProcessorPlugin::javaobject> plugin)
: _plugin(make_global(plugin)) {}
FrameProcessorPluginHostObject::~FrameProcessorPluginHostObject() {
jni::ThreadScope::WithClassLoader([&] { _plugin.reset(); });
}
std::vector<jsi::PropNameID> FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime& runtime) {
std::vector<jsi::PropNameID> result;
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("call")));

4
package/android/src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.h
View File

@ -16,8 +16,8 @@ using namespace facebook;
class FrameProcessorPluginHostObject : public jsi::HostObject {
public:
explicit FrameProcessorPluginHostObject(jni::alias_ref<JFrameProcessorPlugin::javaobject> plugin) : _plugin(make_global(plugin)) {}
~FrameProcessorPluginHostObject() {}
explicit FrameProcessorPluginHostObject(jni::alias_ref<JFrameProcessorPlugin::javaobject> plugin);
~FrameProcessorPluginHostObject();
public:
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& runtime) override;

9
package/android/src/main/cpp/frameprocessor/JSIJNIConversion.cpp
View File

@ -15,7 +15,6 @@
#include "FrameHostObject.h"
#include "JFrame.h"
#include "JSITypedArray.h"
#include "JSharedArray.h"
namespace vision {
@ -61,8 +60,8 @@ jni::local_ref<jobject> JSIJNIConversion::convertJSIValueToJNIObject(jsi::Runtim
} else if (valueAsObject.isArrayBuffer(runtime)) {
// ArrayBuffer/TypedArray
TypedArrayBase array = getTypedArray(runtime, valueAsObject);
return JSharedArray::create(runtime, std::move(array));
jsi::ArrayBuffer arrayBuffer = valueAsObject.getArrayBuffer(runtime);
return JSharedArray::create(runtime, std::move(arrayBuffer));
} else if (valueAsObject.isHostObject(runtime)) {
@ -176,8 +175,8 @@ jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime& runtime, c
// SharedArray
auto sharedArray = static_ref_cast<JSharedArray::javaobject>(object);
std::shared_ptr<TypedArrayBase> array = sharedArray->cthis()->getTypedArray();
return array->getBuffer(runtime);
std::shared_ptr<jsi::ArrayBuffer> array = sharedArray->cthis()->getArrayBuffer();
return array->getArrayBuffer(runtime);
}
auto type = object->getClass()->toString();

1
package/android/src/main/cpp/frameprocessor/VisionCameraProxy.cpp
View File

@ -13,7 +13,6 @@
#include <fbjni/fbjni.h>
#include "FrameProcessorPluginHostObject.h"
#include "JSITypedArray.h"
#include <memory>
#include <string>

72
package/android/src/main/cpp/frameprocessor/java-bindings/JSharedArray.cpp
View File

@ -9,30 +9,23 @@ namespace vision {
using namespace facebook;
TypedArrayKind getTypedArrayKind(int unsafeEnumValue) {
return static_cast<TypedArrayKind>(unsafeEnumValue);
jni::local_ref<JSharedArray::javaobject> JSharedArray::create(jsi::Runtime& runtime, jsi::ArrayBuffer arrayBuffer) {
jni::local_ref<JSharedArray::javaobject> instance = newObjectCxxArgs(runtime, std::make_shared<jsi::ArrayBuffer>(std::move(arrayBuffer)));
instance->cthis()->_javaPart = jni::make_global(instance);
return instance;
}
jni::local_ref<JSharedArray::javaobject> JSharedArray::create(jsi::Runtime& runtime, TypedArrayBase array) {
return newObjectCxxArgs(runtime, std::make_shared<TypedArrayBase>(std::move(array)));
JSharedArray::JSharedArray(jsi::Runtime& runtime, std::shared_ptr<jsi::ArrayBuffer> arrayBuffer) {
size_t size = arrayBuffer->size(runtime);
jni::local_ref<JByteBuffer> byteBuffer = JByteBuffer::allocateDirect(size);
_arrayBuffer = arrayBuffer;
_byteBuffer = jni::make_global(byteBuffer);
_size = size;
}
jni::global_ref<jni::JByteBuffer> JSharedArray::wrapInByteBuffer(jsi::Runtime& runtime, std::shared_ptr<TypedArrayBase> typedArray) {
jsi::ArrayBuffer arrayBuffer = typedArray->getBuffer(runtime);
__android_log_print(ANDROID_LOG_INFO, TAG, "Wrapping ArrayBuffer in a JNI ByteBuffer...");
auto byteBuffer = jni::JByteBuffer::wrapBytes(arrayBuffer.data(runtime), arrayBuffer.size(runtime));
__android_log_print(ANDROID_LOG_INFO, TAG, "Successfully created TypedArray (JNI Size: %i)!", byteBuffer->getDirectSize());
return jni::make_global(byteBuffer);
}
JSharedArray::JSharedArray(jsi::Runtime& runtime, std::shared_ptr<TypedArrayBase> array) {
_array = array;
_byteBuffer = wrapInByteBuffer(runtime, _array);
_size = _array->size(runtime);
}
JSharedArray::JSharedArray(const jni::alias_ref<JSharedArray::jhybridobject>& javaThis,
const jni::alias_ref<JVisionCameraProxy::javaobject>& proxy, int dataType, int size) {
JSharedArray::JSharedArray(const jni::alias_ref<jhybridobject>& javaThis, const jni::alias_ref<JVisionCameraProxy::javaobject>& proxy,
jni::alias_ref<JByteBuffer> byteBuffer) {
_javaPart = jni::make_global(javaThis);
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
@ -40,37 +33,48 @@ JSharedArray::JSharedArray(const jni::alias_ref<JSharedArray::jhybridobject>& ja
#else
jsi::Runtime& runtime = *proxy->cthis()->getJSRuntime();
#endif
TypedArrayKind kind = getTypedArrayKind(dataType);
__android_log_print(ANDROID_LOG_INFO, TAG, "Allocating ArrayBuffer with size %i and type %i...", size, dataType);
_array = std::make_shared<TypedArrayBase>(runtime, size, kind);
_byteBuffer = wrapInByteBuffer(runtime, _array);
_size = size;
__android_log_print(ANDROID_LOG_INFO, TAG, "Allocating ArrayBuffer with size %i...", byteBuffer->getDirectSize());
_byteBuffer = jni::make_global(byteBuffer);
_size = _byteBuffer->getDirectSize();
auto mutableByteBuffer = std::make_shared<MutableJByteBuffer>(byteBuffer);
_arrayBuffer = std::make_shared<jsi::ArrayBuffer>(runtime, std::move(mutableByteBuffer));
}
JSharedArray::JSharedArray(const jni::alias_ref<JSharedArray::jhybridobject>& javaThis,
const jni::alias_ref<JVisionCameraProxy::javaobject>& proxy, int size)
: JSharedArray(javaThis, proxy, JByteBuffer::allocateDirect(size)) {}
void JSharedArray::registerNatives() {
registerHybrid({
makeNativeMethod("initHybrid", JSharedArray::initHybrid),
makeNativeMethod("initHybrid", JSharedArray::initHybridAllocate),
makeNativeMethod("initHybrid", JSharedArray::initHybridWrap),
makeNativeMethod("getByteBuffer", JSharedArray::getByteBuffer),
makeNativeMethod("getSize", JSharedArray::getSize),
});
}
jni::local_ref<jni::JByteBuffer> JSharedArray::getByteBuffer() {
return jni::make_local(_byteBuffer);
jni::global_ref<jni::JByteBuffer> JSharedArray::getByteBuffer() {
return _byteBuffer;
}
std::shared_ptr<jsi::ArrayBuffer> JSharedArray::getArrayBuffer() {
return _arrayBuffer;
}
jint JSharedArray::getSize() {
return _size;
}
std::shared_ptr<TypedArrayBase> JSharedArray::getTypedArray() {
return _array;
jni::local_ref<JSharedArray::jhybriddata>
JSharedArray::initHybridAllocate(jni::alias_ref<jhybridobject> javaThis, jni::alias_ref<JVisionCameraProxy::javaobject> proxy, jint size) {
return makeCxxInstance(javaThis, proxy, size);
}
jni::local_ref<JSharedArray::jhybriddata> JSharedArray::initHybrid(jni::alias_ref<jhybridobject> javaThis,
jni::alias_ref<JVisionCameraProxy::javaobject> proxy, jint type,
jint size) {
return makeCxxInstance(javaThis, proxy, type, size);
jni::local_ref<JSharedArray::jhybriddata> JSharedArray::initHybridWrap(jni::alias_ref<jhybridobject> javaThis,
jni::alias_ref<JVisionCameraProxy::javaobject> proxy,
jni::alias_ref<JByteBuffer> byteBuffer) {
return makeCxxInstance(javaThis, proxy, byteBuffer);
}
} // namespace vision

26
package/android/src/main/cpp/frameprocessor/java-bindings/JSharedArray.h
View File

@ -4,8 +4,8 @@
#pragma once
#include "JSITypedArray.h"
#include "JVisionCameraProxy.h"
#include "MutableJByteBuffer.h"
#include <fbjni/ByteBuffer.h>
#include <fbjni/fbjni.h>
#include <jni.h>
@ -20,30 +20,32 @@ public:
static void registerNatives();
public:
static jni::local_ref<JSharedArray::javaobject> create(jsi::Runtime& runtime, TypedArrayBase array);
static jni::local_ref<JSharedArray::javaobject> create(jsi::Runtime& runtime, jsi::ArrayBuffer arrayBuffer);
public:
jint getSize();
jni::local_ref<jni::JByteBuffer> getByteBuffer();
std::shared_ptr<TypedArrayBase> getTypedArray();
private:
jni::global_ref<jni::JByteBuffer> wrapInByteBuffer(jsi::Runtime& runtime, std::shared_ptr<TypedArrayBase> typedArray);
jni::global_ref<jni::JByteBuffer> getByteBuffer();
std::shared_ptr<jsi::ArrayBuffer> getArrayBuffer();
private:
static auto constexpr TAG = "SharedArray";
friend HybridBase;
jni::global_ref<javaobject> _javaPart;
jni::global_ref<jni::JByteBuffer> _byteBuffer;
std::shared_ptr<TypedArrayBase> _array;
std::shared_ptr<jsi::ArrayBuffer> _arrayBuffer;
int _size;
private:
explicit JSharedArray(jsi::Runtime& runtime, std::shared_ptr<TypedArrayBase> array);
explicit JSharedArray(jsi::Runtime& runtime, std::shared_ptr<jsi::ArrayBuffer> arrayBuffer);
explicit JSharedArray(const jni::alias_ref<jhybridobject>& javaThis, const jni::alias_ref<JVisionCameraProxy::javaobject>& proxy,
int dataType, int size);
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> javaThis,
jni::alias_ref<JVisionCameraProxy::javaobject> proxy, jint dataType, jint size);
int size);
explicit JSharedArray(const jni::alias_ref<jhybridobject>& javaThis, const jni::alias_ref<JVisionCameraProxy::javaobject>& proxy,
jni::alias_ref<JByteBuffer> byteBuffer);
static jni::local_ref<jhybriddata> initHybridAllocate(jni::alias_ref<jhybridobject> javaThis,
jni::alias_ref<JVisionCameraProxy::javaobject> proxy, jint size);
static jni::local_ref<jhybriddata> initHybridWrap(jni::alias_ref<jhybridobject> javaThis,
jni::alias_ref<JVisionCameraProxy::javaobject> proxy,
jni::alias_ref<JByteBuffer> byteBuffer);
};
} // namespace vision

6
package/android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraProxy.cpp
View File

@ -11,7 +11,6 @@
#include <jsi/jsi.h>
#include "FrameProcessorPluginHostObject.h"
#include "JSITypedArray.h"
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
#include <react-native-worklets-core/WKTJsiWorklet.h>
@ -51,10 +50,7 @@ JVisionCameraProxy::JVisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::
JVisionCameraProxy::~JVisionCameraProxy() {
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
__android_log_write(ANDROID_LOG_INFO, TAG, "Destroying Context...");
// Destroy ArrayBuffer cache for both the JS and the Worklet Runtime.
invalidateArrayBufferCache(*_workletContext->getJsRuntime());
invalidateArrayBufferCache(_workletContext->getWorkletRuntime());
__android_log_write(ANDROID_LOG_INFO, TAG, "Destroying JVisionCameraProxy...");
#endif
}

27
package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt
View File

@ -96,8 +96,9 @@ class VideoPipeline(
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
Log.i(TAG, "Using API 29 for GPU ImageReader...")
// GPU_SAMPLED because we redirect to OpenGL
val usage = HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE
// If we are in PRIVATE, we just pass it to the GPU as efficiently as possible - so use GPU flag.
// If we are in YUV/RGB/..., we probably want to access Frame data - so use CPU flag.
val usage = if (format == ImageFormat.PRIVATE) HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE else HardwareBuffer.USAGE_CPU_READ_OFTEN
imageReader = ImageReader.newInstance(width, height, format, MAX_IMAGES, usage)
imageWriter = ImageWriter.newInstance(glSurface, MAX_IMAGES, format)
} else {
@ -109,17 +110,21 @@ class VideoPipeline(
Log.i(TAG, "ImageReader::onImageAvailable!")
val image = reader.acquireNextImage() ?: return@setOnImageAvailableListener
// TODO: Get correct orientation and isMirrored
val frame = Frame(image, image.timestamp, Orientation.PORTRAIT, isMirrored)
frame.incrementRefCount()
frameProcessor?.call(frame)
try {
// TODO: Get correct orientation and isMirrored
val frame = Frame(image, image.timestamp, Orientation.PORTRAIT, isMirrored)
frame.incrementRefCount()
frameProcessor?.call(frame)
if (hasOutputs) {
// If we have outputs (e.g. a RecordingSession), pass the frame along to the OpenGL pipeline
imageWriter!!.queueInputImage(image)
if (hasOutputs) {
// If we have outputs (e.g. a RecordingSession), pass the frame along to the OpenGL pipeline
imageWriter!!.queueInputImage(image)
}
frame.decrementRefCount()
} catch (e: Throwable) {
Log.e(TAG, "Failed to call Frame Processor!", e)
}
frame.decrementRefCount()
}, CameraQueues.videoQueue.handler)
surface = imageReader!!.surface

9
package/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java
View File

@ -15,7 +15,6 @@ public class Frame {
private final long timestamp;
private final Orientation orientation;
private int refCount = 0;
private HardwareBuffer hardwareBuffer = null;
public Frame(Image image, long timestamp, Orientation orientation, boolean isMirrored) {
this.image = image;
@ -114,10 +113,7 @@ public class Frame {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P) {
throw new HardwareBuffersNotAvailableError();
}
if (hardwareBuffer == null) {
hardwareBuffer = getImage().getHardwareBuffer();
}
return hardwareBuffer;
return getImage().getHardwareBuffer();
}
@SuppressWarnings("unused")
@ -142,9 +138,6 @@ public class Frame {
private synchronized void close() {
synchronized (this) {
if (hardwareBuffer != null) {
hardwareBuffer.close();
}
image.close();
}
}

35
package/android/src/main/java/com/mrousavy/camera/frameprocessor/SharedArray.java
View File

@ -20,18 +20,26 @@ public final class SharedArray {
@DoNotStrip
@Keep
public SharedArray(HybridData hybridData) {
private SharedArray(HybridData hybridData) {
mHybridData = hybridData;
}
/**
* Allocate a new SharedArray. Use `getByteBuffer` to obtain a reference to the direct ByteBuffer for writing.
* @param proxy The VisionCamera Proxy from the Frame Processor Plugin's initializer.
* @param dataType The ArrayBuffer's data type. `Type.Int8Array` = `Int8Array` in JS
* @param size The size of the ArrayBuffer.
*/
public SharedArray(VisionCameraProxy proxy, Type dataType, int size) {
mHybridData = initHybrid(proxy, dataType.ordinal(), size);
public SharedArray(VisionCameraProxy proxy, int size) {
mHybridData = initHybrid(proxy, size);
}
/**
* Wraps the given ByteBuffer in a JSI ArrayBuffer. Using `getByteBuffer` will return the same instance which can be used for writing.
* @param proxy The VisionCamera Proxy from the Frame Processor Plugin's initializer.
* @param byteBuffer The ByteBuffer to wrap.
*/
public SharedArray(VisionCameraProxy proxy, ByteBuffer byteBuffer) {
mHybridData = initHybrid(proxy, byteBuffer);
}
/**
@ -44,21 +52,6 @@ public final class SharedArray {
*/
public native int getSize();
private native HybridData initHybrid(VisionCameraProxy proxy, int dataType, int size);
/**
* The Type of the SharedArray.
*/
public enum Type {
// Values start at 0 and need to match with JSITypedArray.h::TypedArrayKind
Int8Array,
Int16Array,
Int32Array,
Uint8Array,
Uint8ClampedArray,
Uint16Array,
Uint32Array,
Float32Array,
Float64Array,
}
private native HybridData initHybrid(VisionCameraProxy proxy, int size);
private native HybridData initHybrid(VisionCameraProxy proxy, ByteBuffer byteBuffer);
}

314
package/cpp/JSITypedArray.cpp
View File

@ -1,314 +0,0 @@
//
// JSITypedArray.cpp
// VisionCamera
//
// Created by Marc Rousavy on 21.02.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
// Copied & Adapted from
// https://github.com/expo/expo/blob/main/packages/expo-gl/common/EXTypedArrayApi.cpp Credits to
// Expo
#include "JSITypedArray.h"
#include <algorithm>
#include <memory>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
namespace vision {
template <TypedArrayKind T> using ContentType = typename typedArrayTypeMap<T>::type;
enum class Prop {
Buffer, // "buffer"
Constructor, // "constructor"
Name, // "name"
Proto, // "__proto__"
Length, // "length"
ByteLength, // "byteLength"
ByteOffset, // "offset"
IsView, // "isView"
ArrayBuffer, // "ArrayBuffer"
Int8Array, // "Int8Array"
Int16Array, // "Int16Array"
Int32Array, // "Int32Array"
Uint8Array, // "Uint8Array"
Uint8ClampedArray, // "Uint8ClampedArray"
Uint16Array, // "Uint16Array"
Uint32Array, // "Uint32Array"
Float32Array, // "Float32Array"
Float64Array, // "Float64Array"
};
class PropNameIDCache {
public:
const jsi::PropNameID& get(jsi::Runtime& runtime, Prop prop) {
auto key = reinterpret_cast<uintptr_t>(&runtime);
if (this->props.find(key) == this->props.end()) {
this->props[key] = std::unordered_map<Prop, std::unique_ptr<jsi::PropNameID>>();
}
if (!this->props[key][prop]) {
this->props[key][prop] = std::make_unique<jsi::PropNameID>(createProp(runtime, prop));
}
return *(this->props[key][prop]);
}
const jsi::PropNameID& getConstructorNameProp(jsi::Runtime& runtime, TypedArrayKind kind);
void invalidate(uintptr_t key) {
if (props.find(key) != props.end()) {
props[key].clear();
}
}
private:
std::unordered_map<uintptr_t, std::unordered_map<Prop, std::unique_ptr<jsi::PropNameID>>> props;
jsi::PropNameID createProp(jsi::Runtime& runtime, Prop prop);
};
PropNameIDCache propNameIDCache;
void invalidateArrayBufferCache(jsi::Runtime& runtime) {
auto key = reinterpret_cast<uintptr_t>(&runtime);
propNameIDCache.invalidate(key);
}
TypedArrayKind getTypedArrayKindForName(const std::string& name);
TypedArrayBase::TypedArrayBase(jsi::Runtime& runtime, size_t size, TypedArrayKind kind)
: TypedArrayBase(runtime, runtime.global()
.getProperty(runtime, propNameIDCache.getConstructorNameProp(runtime, kind))
.asObject(runtime)
.asFunction(runtime)
.callAsConstructor(runtime, {static_cast<double>(size)})
.asObject(runtime)) {}
TypedArrayBase::TypedArrayBase(jsi::Runtime& runtime, const jsi::Object& obj) : jsi::Object(jsi::Value(runtime, obj).asObject(runtime)) {}
TypedArrayKind TypedArrayBase::getKind(jsi::Runtime& runtime) const {
auto constructorName = this->getProperty(runtime, propNameIDCache.get(runtime, Prop::Constructor))
.asObject(runtime)
.getProperty(runtime, propNameIDCache.get(runtime, Prop::Name))
.asString(runtime)
.utf8(runtime);
return getTypedArrayKindForName(constructorName);
}
size_t TypedArrayBase::size(jsi::Runtime& runtime) const {
return getProperty(runtime, propNameIDCache.get(runtime, Prop::Length)).asNumber();
}
size_t TypedArrayBase::length(jsi::Runtime& runtime) const {
return getProperty(runtime, propNameIDCache.get(runtime, Prop::Length)).asNumber();
}
size_t TypedArrayBase::byteLength(jsi::Runtime& runtime) const {
return getProperty(runtime, propNameIDCache.get(runtime, Prop::ByteLength)).asNumber();
}
size_t TypedArrayBase::byteOffset(jsi::Runtime& runtime) const {
return getProperty(runtime, propNameIDCache.get(runtime, Prop::ByteOffset)).asNumber();
}
bool TypedArrayBase::hasBuffer(jsi::Runtime& runtime) const {
auto buffer = getProperty(runtime, propNameIDCache.get(runtime, Prop::Buffer));
return buffer.isObject() && buffer.asObject(runtime).isArrayBuffer(runtime);
}
std::vector<uint8_t> TypedArrayBase::toVector(jsi::Runtime& runtime) {
auto start = reinterpret_cast<uint8_t*>(getBuffer(runtime).data(runtime) + byteOffset(runtime));
auto end = start + byteLength(runtime);
return std::vector<uint8_t>(start, end);
}
jsi::ArrayBuffer TypedArrayBase::getBuffer(jsi::Runtime& runtime) const {
auto buffer = getProperty(runtime, propNameIDCache.get(runtime, Prop::Buffer));
if (buffer.isObject() && buffer.asObject(runtime).isArrayBuffer(runtime)) {
return buffer.asObject(runtime).getArrayBuffer(runtime);
} else {
throw std::runtime_error("no ArrayBuffer attached");
}
}
bool isTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj) {
auto jsVal = runtime.global()
.getProperty(runtime, propNameIDCache.get(runtime, Prop::ArrayBuffer))
.asObject(runtime)
.getProperty(runtime, propNameIDCache.get(runtime, Prop::IsView))
.asObject(runtime)
.asFunction(runtime)
.callWithThis(runtime, runtime.global(), {jsi::Value(runtime, jsObj)});
if (jsVal.isBool()) {
return jsVal.getBool();
} else {
throw std::runtime_error("value is not a boolean");
}
}
TypedArrayBase getTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj) {
auto jsVal = runtime.global()
.getProperty(runtime, propNameIDCache.get(runtime, Prop::ArrayBuffer))
.asObject(runtime)
.getProperty(runtime, propNameIDCache.get(runtime, Prop::IsView))
.asObject(runtime)
.asFunction(runtime)
.callWithThis(runtime, runtime.global(), {jsi::Value(runtime, jsObj)});
if (jsVal.isBool()) {
return TypedArrayBase(runtime, jsObj);
} else {
throw std::runtime_error("value is not a boolean");
}
}
std::vector<uint8_t> arrayBufferToVector(jsi::Runtime& runtime, jsi::Object& jsObj) {
if (!jsObj.isArrayBuffer(runtime)) {
throw std::runtime_error("Object is not an ArrayBuffer");
}
auto jsArrayBuffer = jsObj.getArrayBuffer(runtime);
uint8_t* dataBlock = jsArrayBuffer.data(runtime);
size_t blockSize = jsArrayBuffer.getProperty(runtime, propNameIDCache.get(runtime, Prop::ByteLength)).asNumber();
return std::vector<uint8_t>(dataBlock, dataBlock + blockSize);
}
void arrayBufferUpdate(jsi::Runtime& runtime, jsi::ArrayBuffer& buffer, std::vector<uint8_t> data, size_t offset) {
uint8_t* dataBlock = buffer.data(runtime);
size_t blockSize = buffer.size(runtime);
if (data.size() > blockSize) {
throw jsi::JSError(runtime, "ArrayBuffer is to small to fit data");
}
std::copy(data.begin(), data.end(), dataBlock + offset);
}
template <TypedArrayKind T> TypedArray<T>::TypedArray(jsi::Runtime& runtime, size_t size) : TypedArrayBase(runtime, size, T) {}
template <TypedArrayKind T>
TypedArray<T>::TypedArray(jsi::Runtime& runtime, std::vector<ContentType<T>> data) : TypedArrayBase(runtime, data.size(), T) {
update(runtime, data);
}
template <TypedArrayKind T> TypedArray<T>::TypedArray(TypedArrayBase&& base) : TypedArrayBase(std::move(base)) {}
template <TypedArrayKind T> std::vector<ContentType<T>> TypedArray<T>::toVector(jsi::Runtime& runtime) {
auto start = reinterpret_cast<ContentType<T>*>(getBuffer(runtime).data(runtime) + byteOffset(runtime));
auto end = start + size(runtime);
return std::vector<ContentType<T>>(start, end);
}
template <TypedArrayKind T> void TypedArray<T>::update(jsi::Runtime& runtime, const std::vector<ContentType<T>>& data) {
if (data.size() != size(runtime)) {
throw jsi::JSError(runtime, "TypedArray can only be updated with a vector of the same size");
}
uint8_t* rawData = getBuffer(runtime).data(runtime) + byteOffset(runtime);
std::copy(data.begin(), data.end(), reinterpret_cast<ContentType<T>*>(rawData));
}
template <TypedArrayKind T> void TypedArray<T>::updateUnsafe(jsi::Runtime& runtime, ContentType<T>* data, size_t length) {
if (length != size(runtime)) {
throw jsi::JSError(runtime, "TypedArray can only be updated with an array of the same size");
}
uint8_t* rawData = getBuffer(runtime).data(runtime) + byteOffset(runtime);
memcpy(rawData, data, length);
}
template <TypedArrayKind T> uint8_t* TypedArray<T>::data(jsi::Runtime& runtime) {
return getBuffer(runtime).data(runtime) + byteOffset(runtime);
}
const jsi::PropNameID& PropNameIDCache::getConstructorNameProp(jsi::Runtime& runtime, TypedArrayKind kind) {
switch (kind) {
case TypedArrayKind::Int8Array:
return get(runtime, Prop::Int8Array);
case TypedArrayKind::Int16Array:
return get(runtime, Prop::Int16Array);
case TypedArrayKind::Int32Array:
return get(runtime, Prop::Int32Array);
case TypedArrayKind::Uint8Array:
return get(runtime, Prop::Uint8Array);
case TypedArrayKind::Uint8ClampedArray:
return get(runtime, Prop::Uint8ClampedArray);
case TypedArrayKind::Uint16Array:
return get(runtime, Prop::Uint16Array);
case TypedArrayKind::Uint32Array:
return get(runtime, Prop::Uint32Array);
case TypedArrayKind::Float32Array:
return get(runtime, Prop::Float32Array);
case TypedArrayKind::Float64Array:
return get(runtime, Prop::Float64Array);
}
}
jsi::PropNameID PropNameIDCache::createProp(jsi::Runtime& runtime, Prop prop) {
auto create = [&](const std::string& propName) { return jsi::PropNameID::forUtf8(runtime, propName); };
switch (prop) {
case Prop::Buffer:
return create("buffer");
case Prop::Constructor:
return create("constructor");
case Prop::Name:
return create("name");
case Prop::Proto:
return create("__proto__");
case Prop::Length:
return create("length");
case Prop::ByteLength:
return create("byteLength");
case Prop::ByteOffset:
return create("byteOffset");
case Prop::IsView:
return create("isView");
case Prop::ArrayBuffer:
return create("ArrayBuffer");
case Prop::Int8Array:
return create("Int8Array");
case Prop::Int16Array:
return create("Int16Array");
case Prop::Int32Array:
return create("Int32Array");
case Prop::Uint8Array:
return create("Uint8Array");
case Prop::Uint8ClampedArray:
return create("Uint8ClampedArray");
case Prop::Uint16Array:
return create("Uint16Array");
case Prop::Uint32Array:
return create("Uint32Array");
case Prop::Float32Array:
return create("Float32Array");
case Prop::Float64Array:
return create("Float64Array");
}
}
std::unordered_map<std::string, TypedArrayKind> nameToKindMap = {
{"Int8Array", TypedArrayKind::Int8Array},
{"Int16Array", TypedArrayKind::Int16Array},
{"Int32Array", TypedArrayKind::Int32Array},
{"Uint8Array", TypedArrayKind::Uint8Array},
{"Uint8ClampedArray", TypedArrayKind::Uint8ClampedArray},
{"Uint16Array", TypedArrayKind::Uint16Array},
{"Uint32Array", TypedArrayKind::Uint32Array},
{"Float32Array", TypedArrayKind::Float32Array},
{"Float64Array", TypedArrayKind::Float64Array},
};
TypedArrayKind getTypedArrayKindForName(const std::string& name) {
return nameToKindMap.at(name);
}
template class TypedArray<TypedArrayKind::Int8Array>;
template class TypedArray<TypedArrayKind::Int16Array>;
template class TypedArray<TypedArrayKind::Int32Array>;
template class TypedArray<TypedArrayKind::Uint8Array>;
template class TypedArray<TypedArrayKind::Uint8ClampedArray>;
template class TypedArray<TypedArrayKind::Uint16Array>;
template class TypedArray<TypedArrayKind::Uint32Array>;
template class TypedArray<TypedArrayKind::Float32Array>;
template class TypedArray<TypedArrayKind::Float64Array>;
} // namespace vision

141
package/cpp/JSITypedArray.h
View File

@ -1,141 +0,0 @@
//
// JSITypedArray.h
// VisionCamera
//
// Created by Marc Rousavy on 21.02.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
// Copied & Adapted from
// https://github.com/expo/expo/blob/main/packages/expo-gl/common/EXTypedArrayApi.h Credits to Expo
#pragma once
#include <jsi/jsi.h>
#include <utility>
#include <vector>
namespace jsi = facebook::jsi;
namespace vision {
enum class TypedArrayKind {
Int8Array,
Int16Array,
Int32Array,
Uint8Array,
Uint8ClampedArray,
Uint16Array,
Uint32Array,
Float32Array,
Float64Array,
};
template <TypedArrayKind T> class TypedArray;
template <TypedArrayKind T> struct typedArrayTypeMap;
template <> struct typedArrayTypeMap<TypedArrayKind::Int8Array> {
typedef int8_t type;
};
template <> struct typedArrayTypeMap<TypedArrayKind::Int16Array> {
typedef int16_t type;
};
template <> struct typedArrayTypeMap<TypedArrayKind::Int32Array> {
typedef int32_t type;
};
template <> struct typedArrayTypeMap<TypedArrayKind::Uint8Array> {
typedef uint8_t type;
};
template <> struct typedArrayTypeMap<TypedArrayKind::Uint8ClampedArray> {
typedef uint8_t type;
};
template <> struct typedArrayTypeMap<TypedArrayKind::Uint16Array> {
typedef uint16_t type;
};
template <> struct typedArrayTypeMap<TypedArrayKind::Uint32Array> {
typedef uint32_t type;
};
template <> struct typedArrayTypeMap<TypedArrayKind::Float32Array> {
typedef float type;
};
template <> struct typedArrayTypeMap<TypedArrayKind::Float64Array> {
typedef double type;
};
void invalidateArrayBufferCache(jsi::Runtime& runtime);
class TypedArrayBase : public jsi::Object {
public:
template <TypedArrayKind T> using ContentType = typename typedArrayTypeMap<T>::type;
TypedArrayBase(jsi::Runtime&, size_t, TypedArrayKind);
TypedArrayBase(jsi::Runtime&, const jsi::Object&);
TypedArrayBase(TypedArrayBase&&) = default;
TypedArrayBase& operator=(TypedArrayBase&&) = default;
TypedArrayKind getKind(jsi::Runtime& runtime) const;
template <TypedArrayKind T> TypedArray<T> get(jsi::Runtime& runtime) const&;
template <TypedArrayKind T> TypedArray<T> get(jsi::Runtime& runtime) &&;
template <TypedArrayKind T> TypedArray<T> as(jsi::Runtime& runtime) const&;
template <TypedArrayKind T> TypedArray<T> as(jsi::Runtime& runtime) &&;
size_t size(jsi::Runtime& runtime) const;
size_t length(jsi::Runtime& runtime) const;
size_t byteLength(jsi::Runtime& runtime) const;
size_t byteOffset(jsi::Runtime& runtime) const;
bool hasBuffer(jsi::Runtime& runtime) const;
std::vector<uint8_t> toVector(jsi::Runtime& runtime);
jsi::ArrayBuffer getBuffer(jsi::Runtime& runtime) const;
private:
template <TypedArrayKind> friend class TypedArray;
};
bool isTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj);
TypedArrayBase getTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj);
std::vector<uint8_t> arrayBufferToVector(jsi::Runtime& runtime, jsi::Object& jsObj);
void arrayBufferUpdate(jsi::Runtime& runtime, jsi::ArrayBuffer& buffer, std::vector<uint8_t> data, size_t offset);
template <TypedArrayKind T> class TypedArray : public TypedArrayBase {
public:
explicit TypedArray(TypedArrayBase&& base);
TypedArray(jsi::Runtime& runtime, size_t size);
TypedArray(jsi::Runtime& runtime, std::vector<ContentType<T>> data);
TypedArray(TypedArray&&) = default;
TypedArray& operator=(TypedArray&&) = default;
std::vector<ContentType<T>> toVector(jsi::Runtime& runtime);
void update(jsi::Runtime& runtime, const std::vector<ContentType<T>>& data);
void updateUnsafe(jsi::Runtime& runtime, ContentType<T>* data, size_t length);
uint8_t* data(jsi::Runtime& runtime);
};
template <TypedArrayKind T> TypedArray<T> TypedArrayBase::get(jsi::Runtime& runtime) const& {
assert(getKind(runtime) == T);
(void)runtime; // when assert is disabled we need to mark this as used
return TypedArray<T>(jsi::Value(runtime, jsi::Value(runtime, *this).asObject(runtime)));
}
template <TypedArrayKind T> TypedArray<T> TypedArrayBase::get(jsi::Runtime& runtime) && {
assert(getKind(runtime) == T);
(void)runtime; // when assert is disabled we need to mark this as used
return TypedArray<T>(std::move(*this));
}
template <TypedArrayKind T> TypedArray<T> TypedArrayBase::as(jsi::Runtime& runtime) const& {
if (getKind(runtime) != T) {
throw jsi::JSError(runtime, "Object is not a TypedArray");
}
return get<T>(runtime);
}
template <TypedArrayKind T> TypedArray<T> TypedArrayBase::as(jsi::Runtime& runtime) && {
if (getKind(runtime) != T) {
throw jsi::JSError(runtime, "Object is not a TypedArray");
}
return std::move(*this).get<T>(runtime);
}
} // namespace vision

36
package/cpp/MutableRawBuffer.cpp Normal file
View File

@ -0,0 +1,36 @@
//
// MutableRawBuffer.cpp
// VisionCamera
//
// Created by Marc Rousavy on 17.01.24.
// Copyright © 2024 mrousavy. All rights reserved.
//
#include "MutableRawBuffer.h"
#include <functional>
#include <memory>
namespace vision {
MutableRawBuffer::MutableRawBuffer(uint8_t* data, size_t size, std::function<void()> cleanup)
: _data(data), _size(size), _cleanup(std::move(cleanup)) {}
MutableRawBuffer::MutableRawBuffer(size_t size) {
_size = size;
_data = (uint8_t*)malloc(size * sizeof(uint8_t));
_cleanup = [=]() { free(_data); };
}
MutableRawBuffer::~MutableRawBuffer() {
_cleanup();
}
size_t MutableRawBuffer::size() const {
return _size;
}
uint8_t* MutableRawBuffer::data() {
return _data;
}
} // namespace vision

36
package/cpp/MutableRawBuffer.h Normal file
View File

@ -0,0 +1,36 @@
//
// MutableRawBuffer.h
// VisionCamera
//
// Created by Marc Rousavy on 17.01.24.
// Copyright © 2024 mrousavy. All rights reserved.
//
#pragma once
#include <functional>
#include <jsi/jsi.h>
#include <memory>
namespace vision {
using namespace facebook;
class MutableRawBuffer : public jsi::MutableBuffer {
public:
explicit MutableRawBuffer(size_t size);
explicit MutableRawBuffer(uint8_t* data, size_t size, std::function<void()> cleanup);
~MutableRawBuffer();
public:
uint8_t* data() override;
size_t size() const override;
private:
uint8_t* _data;
size_t _size;
std::function<void()> _cleanup;
};
} // namespace vision

15
package/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java
View File

@ -19,17 +19,18 @@ import java.util.Map;
public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin {
SharedArray _sharedArray;
private static final String TAG = "ExamplePlugin";
@Override
public Object callback(@NotNull Frame frame, @Nullable Map<String, Object> params) {
if (params == null) return null;
Image image = frame.getImage();
Log.d("ExamplePlugin", image.getWidth() + " x " + image.getHeight() + " Image with format #" + image.getFormat() + ". Logging " + params.size() + " parameters:");
Log.d(TAG, image.getWidth() + " x " + image.getHeight() + " Image with format #" + image.getFormat() + ". Logging " + params.size() + " parameters:");
for (String key : params.keySet()) {
Object value = params.get(key);
Log.d("ExamplePlugin", " -> " + (value == null ? "(null)" : value + " (" + value.getClass().getName() + ")"));
Log.d(TAG, " -> " + (value == null ? "(null)" : value + " (" + value.getClass().getName() + ")"));
}
Map<String, Object> map = new HashMap<>();
@ -53,7 +54,13 @@ public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin {
ExampleFrameProcessorPlugin(VisionCameraProxy proxy, @Nullable Map<String, Object> options) {
super();
_sharedArray = new SharedArray(proxy, SharedArray.Type.Uint8Array, 5);
Log.d("ExamplePlugin", "ExampleFrameProcessorPlugin initialized with options: " + options);
_sharedArray = new SharedArray(proxy, 5);
Log.d(TAG, "Successfully allocated SharedArray! Size: " + _sharedArray.getSize());
ByteBuffer buffer = ByteBuffer.allocateDirect(10);
SharedArray testArray = new SharedArray(proxy, buffer);
Log.d(TAG, "Successfully wrapped SharedArray in ByteBuffer! Size: " + testArray.getSize());
Log.d(TAG, "ExampleFrameProcessorPlugin initialized with options: " + options);
}
}

1
package/example/ios/Frame Processor Plugins/Example Plugin/ExampleFrameProcessorPlugin.m
View File

@ -25,7 +25,6 @@
withOptions:(NSDictionary* _Nullable)options {
if (self = [super initWithProxy:proxy withOptions:options]) {
_sharedArray = [[SharedArray alloc] initWithProxy:proxy
type:Int8Array
size:5];
NSLog(@"ExampleFrameProcessorPlugin initialized with options: %@", options);
}

17
package/ios/Frame Processor/FrameHostObject.mm
View File

@ -7,13 +7,12 @@
//
#import "FrameHostObject.h"
#import "MutableRawBuffer.h"
#import "UIImageOrientation+descriptor.h"
#import "WKTJsiHostObject.h"
#import <Foundation/Foundation.h>
#import <jsi/jsi.h>
#import "../../cpp/JSITypedArray.h"
std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt) {
std::vector<jsi::PropNameID> result;
// Ref Management
@ -110,21 +109,23 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
static constexpr auto ARRAYBUFFER_CACHE_PROP_NAME = "__frameArrayBufferCache";
if (!runtime.global().hasProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME)) {
vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray> arrayBuffer(runtime, arraySize);
runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer);
auto mutableBuffer = std::make_shared<vision::MutableRawBuffer>(arraySize);
jsi::ArrayBuffer arrayBuffer(runtime, mutableBuffer);
runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, std::move(arrayBuffer));
}
auto arrayBufferCache = runtime.global().getPropertyAsObject(runtime, ARRAYBUFFER_CACHE_PROP_NAME);
auto arrayBuffer = vision::getTypedArray(runtime, arrayBufferCache).get<vision::TypedArrayKind::Uint8ClampedArray>(runtime);
auto arrayBuffer = arrayBufferCache.getArrayBuffer(runtime);
if (arrayBuffer.size(runtime) != arraySize) {
arrayBuffer = vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray>(runtime, arraySize);
auto mutableBuffer = std::make_shared<vision::MutableRawBuffer>(arraySize);
arrayBuffer = jsi::ArrayBuffer(runtime, mutableBuffer);
runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer);
}
CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
auto buffer = (uint8_t*)CVPixelBufferGetBaseAddress(pixelBuffer);
arrayBuffer.updateUnsafe(runtime, buffer, arraySize);
memcpy(arrayBuffer.data(runtime), buffer, arraySize);
CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
return arrayBuffer;
@ -201,3 +202,5 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
// fallback to base implementation
return HostObject::get(runtime, propName);
}
#undef JSI_FUNC

8
package/ios/Frame Processor/JSINSObjectConversion.mm
View File

@ -19,7 +19,6 @@
#import "../Frame Processor/Frame.h"
#import "../Frame Processor/FrameHostObject.h"
#import "../Frame Processor/SharedArray.h"
#import "JSITypedArray.h"
#import <Foundation/Foundation.h>
#import <React/RCTBridge.h>
#import <ReactCommon/CallInvoker.h>
@ -61,8 +60,7 @@ jsi::Array convertNSArrayToJSIArray(jsi::Runtime& runtime, NSArray* value) {
}
jsi::Object convertSharedArrayToJSIArrayBuffer(jsi::Runtime& runtime, SharedArray* sharedArray) {
std::shared_ptr<vision::TypedArrayBase> array = sharedArray.typedArray;
return array->getBuffer(runtime);
return sharedArray.arrayBuffer->getArrayBuffer(runtime);
}
jsi::Value convertObjCObjectToJSIValue(jsi::Runtime& runtime, id value) {
@ -171,8 +169,8 @@ id convertJSIValueToObjCObject(jsi::Runtime& runtime, const jsi::Value& value, s
}
} else if (o.isArrayBuffer(runtime)) {
// ArrayBuffer
auto typedArray = std::make_shared<vision::TypedArrayBase>(vision::getTypedArray(runtime, o));
return [[SharedArray alloc] initWithRuntime:runtime typedArray:typedArray];
auto arrayBuffer = std::make_shared<jsi::ArrayBuffer>(o.getArrayBuffer(runtime));
return [[SharedArray alloc] initWithRuntime:runtime arrayBuffer:arrayBuffer];
} else {
// object
return convertJSIObjectToNSDictionary(runtime, o, jsInvoker);

25
package/ios/Frame Processor/SharedArray.h
View File

@ -12,39 +12,22 @@
#import <Foundation/Foundation.h>
#ifdef __cplusplus
#import "../../cpp/MutableRawBuffer.h"
#import <jsi/jsi.h>
using namespace facebook;
namespace vision {
// forward-declaration since we cannot import C++ headers here yet.
class TypedArrayBase;
} // namespace vision
#endif
// Needs to be in sync with JSITypedArray.h as the index is used
typedef NS_ENUM(NSInteger, SharedArrayType) {
Int8Array,
Int16Array,
Int32Array,
Uint8Array,
Uint8ClampedArray,
Uint16Array,
Uint32Array,
Float32Array,
Float64Array,
};
NS_ASSUME_NONNULL_BEGIN
@interface SharedArray : NSObject
- (instancetype)init NS_UNAVAILABLE;
- (instancetype)initWithProxy:(VisionCameraProxyHolder*)proxy type:(SharedArrayType)type size:(NSInteger)size;
- (instancetype)initWithProxy:(VisionCameraProxyHolder*)proxy size:(NSInteger)size;
#ifdef __cplusplus
- (instancetype)initWithRuntime:(jsi::Runtime&)runtime typedArray:(std::shared_ptr<vision::TypedArrayBase>)typedArray;
- (instancetype)initWithRuntime:(jsi::Runtime&)runtime arrayBuffer:(std::shared_ptr<jsi::ArrayBuffer>)arrayBuffer;
- (std::shared_ptr<vision::TypedArrayBase>)typedArray;
- (std::shared_ptr<jsi::ArrayBuffer>)arrayBuffer;
#endif
@property(nonatomic, readonly, nonnull) uint8_t* data;

30
package/ios/Frame Processor/SharedArray.mm
View File

@ -7,7 +7,7 @@
//
#import "SharedArray.h"
#import "JSITypedArray.h"
#import "../../cpp/MutableRawBuffer.h"
#import <Foundation/Foundation.h>
#import <jsi/jsi.h>
@ -16,35 +16,33 @@ using namespace facebook;
@implementation SharedArray {
uint8_t* _data;
NSInteger _size;
std::shared_ptr<vision::TypedArrayBase> _array;
std::shared_ptr<jsi::ArrayBuffer> _arrayBuffer;
}
vision::TypedArrayKind getTypedArrayKind(int unsafeEnumValue) {
return static_cast<vision::TypedArrayKind>(unsafeEnumValue);
}
- (instancetype)initWithProxy:(VisionCameraProxyHolder*)proxy type:(SharedArrayType)type size:(NSInteger)size {
- (instancetype)initWithProxy:(VisionCameraProxyHolder*)proxy size:(NSInteger)size {
if (self = [super init]) {
jsi::Runtime& runtime = proxy.proxy->getWorkletRuntime();
vision::TypedArrayKind kind = getTypedArrayKind((int)type);
_array = std::make_shared<vision::TypedArrayBase>(vision::TypedArrayBase(runtime, size, kind));
_data = _array->getBuffer(runtime).data(runtime);
uint8_t* data = (uint8_t*)malloc(size * sizeof(uint8_t));
auto mutableBuffer = std::make_shared<vision::MutableRawBuffer>(data, size, [=]() { free(data); });
_arrayBuffer = std::make_shared<jsi::ArrayBuffer>(runtime, mutableBuffer);
_data = data;
_size = size;
}
return self;
}
- (instancetype)initWithRuntime:(jsi::Runtime&)runtime typedArray:(std::shared_ptr<vision::TypedArrayBase>)typedArray {
- (instancetype)initWithRuntime:(jsi::Runtime&)runtime arrayBuffer:(std::shared_ptr<jsi::ArrayBuffer>)arrayBuffer {
if (self = [super init]) {
_array = typedArray;
_data = _array->getBuffer(runtime).data(runtime);
_size = _array->getBuffer(runtime).size(runtime);
_arrayBuffer = arrayBuffer;
_data = _arrayBuffer->data(runtime);
_size = _arrayBuffer->size(runtime);
}
return self;
}
- (std::shared_ptr<vision::TypedArrayBase>)typedArray {
return _array;
- (std::shared_ptr<jsi::ArrayBuffer>)arrayBuffer {
return _arrayBuffer;
}
- (uint8_t*)data {

6
package/ios/Frame Processor/VisionCameraProxy.mm
View File

@ -10,7 +10,6 @@
#import <Foundation/Foundation.h>
#import <jsi/jsi.h>
#import "../../cpp/JSITypedArray.h"
#import "FrameHostObject.h"
#import "FrameProcessor.h"
#import "FrameProcessorPluginHostObject.h"
@ -55,10 +54,7 @@ VisionCameraProxy::VisionCameraProxy(jsi::Runtime& runtime, std::shared_ptr<reac
}
VisionCameraProxy::~VisionCameraProxy() {
NSLog(@"VisionCameraProxy: Destroying context...");
// Destroy ArrayBuffer cache for both the JS and the Worklet Runtime.
vision::invalidateArrayBufferCache(*_workletContext->getJsRuntime());
vision::invalidateArrayBufferCache(_workletContext->getWorkletRuntime());
NSLog(@"VisionCameraProxy: Destroying VisionCameraProxy...");
}
std::vector<jsi::PropNameID> VisionCameraProxy::getPropertyNames(jsi::Runtime& runtime) {

7
package/src/Frame.ts
View File

@ -57,7 +57,9 @@ export interface Frame {
/**
* Get the underlying data of the Frame as a uint8 array buffer.
*
* The format of the buffer depends on the Frame's {@linkcode pixelFormat}.
* This function might fail if the {@linkcode pixelFormat} is `private`.
*
* Note that Frames are allocated on the GPU, so calling `toArrayBuffer()` will copy from the GPU to the CPU.
*
@ -67,13 +69,14 @@ export interface Frame {
* 'worklet'
*
* if (frame.pixelFormat === 'rgb') {
* const data = frame.toArrayBuffer()
* const buffer = frame.toArrayBuffer()
* const data = new Uint8Array(buffer)
* console.log(`Pixel at 0,0: RGB(${data[0]}, ${data[1]}, ${data[2]})`)
* }
* }, [])
* ```
*/
toArrayBuffer(): Uint8Array
toArrayBuffer(): ArrayBuffer
/**
* Returns a string representation of the frame.
* @example