railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Use JSI's ArrayBuffer instead of TypedArray (#2408)

Browse Source 
* feat: Use JSI's `ArrayBuffer` instead of `TypedArray`

* fix: Fix move memory

* feat: Implement iOS

* Format

* Update JSIJNIConversion.cpp

* fix: Fix Android `toArrayBuffer` and other

* Catch FP call errors

* Update return type

* Use `CPU_READ_OFTEN` flag as well

* CPU flag

* Run destructors under `jni::ThreadScope`

* Update FrameProcessorPluginHostObject.cpp

* fix: Fix `toArrayBuffer()` crash

* Update Frame.ts
This commit is contained in:
Marc Rousavy
2024-01-17 20:18:46 +01:00
committed by GitHub
parent 2f21609e39
commit ba1d7eec9c
26 changed files with 296 additions and 620 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
package/ios/Frame Processor/FrameHostObject.mm
View File

@@ -7,13 +7,12 @@
//
#import "FrameHostObject.h"
#import "MutableRawBuffer.h"
#import "UIImageOrientation+descriptor.h"
#import "WKTJsiHostObject.h"
#import <Foundation/Foundation.h>
#import <jsi/jsi.h>
#import "../../cpp/JSITypedArray.h"
std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt) {
std::vector<jsi::PropNameID> result;
// Ref Management
@@ -110,21 +109,23 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
static constexpr auto ARRAYBUFFER_CACHE_PROP_NAME = "__frameArrayBufferCache";
if (!runtime.global().hasProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME)) {
vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray> arrayBuffer(runtime, arraySize);
runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer);
auto mutableBuffer = std::make_shared<vision::MutableRawBuffer>(arraySize);
jsi::ArrayBuffer arrayBuffer(runtime, mutableBuffer);
runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, std::move(arrayBuffer));
}
auto arrayBufferCache = runtime.global().getPropertyAsObject(runtime, ARRAYBUFFER_CACHE_PROP_NAME);
auto arrayBuffer = vision::getTypedArray(runtime, arrayBufferCache).get<vision::TypedArrayKind::Uint8ClampedArray>(runtime);
auto arrayBuffer = arrayBufferCache.getArrayBuffer(runtime);
if (arrayBuffer.size(runtime) != arraySize) {
arrayBuffer = vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray>(runtime, arraySize);
auto mutableBuffer = std::make_shared<vision::MutableRawBuffer>(arraySize);
arrayBuffer = jsi::ArrayBuffer(runtime, mutableBuffer);
runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer);
}
CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
auto buffer = (uint8_t*)CVPixelBufferGetBaseAddress(pixelBuffer);
arrayBuffer.updateUnsafe(runtime, buffer, arraySize);
memcpy(arrayBuffer.data(runtime), buffer, arraySize);
CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
return arrayBuffer;
@@ -201,3 +202,5 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
// fallback to base implementation
return HostObject::get(runtime, propName);
}
#undef JSI_FUNC

8
package/ios/Frame Processor/JSINSObjectConversion.mm
View File

@@ -19,7 +19,6 @@
#import "../Frame Processor/Frame.h"
#import "../Frame Processor/FrameHostObject.h"
#import "../Frame Processor/SharedArray.h"
#import "JSITypedArray.h"
#import <Foundation/Foundation.h>
#import <React/RCTBridge.h>
#import <ReactCommon/CallInvoker.h>
@@ -61,8 +60,7 @@ jsi::Array convertNSArrayToJSIArray(jsi::Runtime& runtime, NSArray* value) {
}
jsi::Object convertSharedArrayToJSIArrayBuffer(jsi::Runtime& runtime, SharedArray* sharedArray) {
std::shared_ptr<vision::TypedArrayBase> array = sharedArray.typedArray;
return array->getBuffer(runtime);
return sharedArray.arrayBuffer->getArrayBuffer(runtime);
}
jsi::Value convertObjCObjectToJSIValue(jsi::Runtime& runtime, id value) {
@@ -171,8 +169,8 @@ id convertJSIValueToObjCObject(jsi::Runtime& runtime, const jsi::Value& value, s
}
} else if (o.isArrayBuffer(runtime)) {
// ArrayBuffer
auto typedArray = std::make_shared<vision::TypedArrayBase>(vision::getTypedArray(runtime, o));
return [[SharedArray alloc] initWithRuntime:runtime typedArray:typedArray];
auto arrayBuffer = std::make_shared<jsi::ArrayBuffer>(o.getArrayBuffer(runtime));
return [[SharedArray alloc] initWithRuntime:runtime arrayBuffer:arrayBuffer];
} else {
// object
return convertJSIObjectToNSDictionary(runtime, o, jsInvoker);

25
package/ios/Frame Processor/SharedArray.h
View File

@@ -12,39 +12,22 @@
#import <Foundation/Foundation.h>
#ifdef __cplusplus
#import "../../cpp/MutableRawBuffer.h"
#import <jsi/jsi.h>
using namespace facebook;
namespace vision {
// forward-declaration since we cannot import C++ headers here yet.
class TypedArrayBase;
} // namespace vision
#endif
// Needs to be in sync with JSITypedArray.h as the index is used
typedef NS_ENUM(NSInteger, SharedArrayType) {
Int8Array,
Int16Array,
Int32Array,
Uint8Array,
Uint8ClampedArray,
Uint16Array,
Uint32Array,
Float32Array,
Float64Array,
};
NS_ASSUME_NONNULL_BEGIN
@interface SharedArray : NSObject
- (instancetype)init NS_UNAVAILABLE;
- (instancetype)initWithProxy:(VisionCameraProxyHolder*)proxy type:(SharedArrayType)type size:(NSInteger)size;
- (instancetype)initWithProxy:(VisionCameraProxyHolder*)proxy size:(NSInteger)size;
#ifdef __cplusplus
- (instancetype)initWithRuntime:(jsi::Runtime&)runtime typedArray:(std::shared_ptr<vision::TypedArrayBase>)typedArray;
- (instancetype)initWithRuntime:(jsi::Runtime&)runtime arrayBuffer:(std::shared_ptr<jsi::ArrayBuffer>)arrayBuffer;
- (std::shared_ptr<vision::TypedArrayBase>)typedArray;
- (std::shared_ptr<jsi::ArrayBuffer>)arrayBuffer;
#endif
@property(nonatomic, readonly, nonnull) uint8_t* data;

30
package/ios/Frame Processor/SharedArray.mm
View File

@@ -7,7 +7,7 @@
//
#import "SharedArray.h"
#import "JSITypedArray.h"
#import "../../cpp/MutableRawBuffer.h"
#import <Foundation/Foundation.h>
#import <jsi/jsi.h>
@@ -16,35 +16,33 @@ using namespace facebook;
@implementation SharedArray {
uint8_t* _data;
NSInteger _size;
std::shared_ptr<vision::TypedArrayBase> _array;
std::shared_ptr<jsi::ArrayBuffer> _arrayBuffer;
}
vision::TypedArrayKind getTypedArrayKind(int unsafeEnumValue) {
return static_cast<vision::TypedArrayKind>(unsafeEnumValue);
}
- (instancetype)initWithProxy:(VisionCameraProxyHolder*)proxy type:(SharedArrayType)type size:(NSInteger)size {
- (instancetype)initWithProxy:(VisionCameraProxyHolder*)proxy size:(NSInteger)size {
if (self = [super init]) {
jsi::Runtime& runtime = proxy.proxy->getWorkletRuntime();
vision::TypedArrayKind kind = getTypedArrayKind((int)type);
_array = std::make_shared<vision::TypedArrayBase>(vision::TypedArrayBase(runtime, size, kind));
_data = _array->getBuffer(runtime).data(runtime);
uint8_t* data = (uint8_t*)malloc(size * sizeof(uint8_t));
auto mutableBuffer = std::make_shared<vision::MutableRawBuffer>(data, size, [=]() { free(data); });
_arrayBuffer = std::make_shared<jsi::ArrayBuffer>(runtime, mutableBuffer);
_data = data;
_size = size;
}
return self;
}
- (instancetype)initWithRuntime:(jsi::Runtime&)runtime typedArray:(std::shared_ptr<vision::TypedArrayBase>)typedArray {
- (instancetype)initWithRuntime:(jsi::Runtime&)runtime arrayBuffer:(std::shared_ptr<jsi::ArrayBuffer>)arrayBuffer {
if (self = [super init]) {
_array = typedArray;
_data = _array->getBuffer(runtime).data(runtime);
_size = _array->getBuffer(runtime).size(runtime);
_arrayBuffer = arrayBuffer;
_data = _arrayBuffer->data(runtime);
_size = _arrayBuffer->size(runtime);
}
return self;
}
- (std::shared_ptr<vision::TypedArrayBase>)typedArray {
return _array;
- (std::shared_ptr<jsi::ArrayBuffer>)arrayBuffer {
return _arrayBuffer;
}
- (uint8_t*)data {

6
package/ios/Frame Processor/VisionCameraProxy.mm
View File

@@ -10,7 +10,6 @@
#import <Foundation/Foundation.h>
#import <jsi/jsi.h>
#import "../../cpp/JSITypedArray.h"
#import "FrameHostObject.h"
#import "FrameProcessor.h"
#import "FrameProcessorPluginHostObject.h"
@@ -55,10 +54,7 @@ VisionCameraProxy::VisionCameraProxy(jsi::Runtime& runtime, std::shared_ptr<reac
}
VisionCameraProxy::~VisionCameraProxy() {
NSLog(@"VisionCameraProxy: Destroying context...");
// Destroy ArrayBuffer cache for both the JS and the Worklet Runtime.
vision::invalidateArrayBufferCache(*_workletContext->getJsRuntime());
vision::invalidateArrayBufferCache(_workletContext->getWorkletRuntime());
NSLog(@"VisionCameraProxy: Destroying VisionCameraProxy...");
}
std::vector<jsi::PropNameID> VisionCameraProxy::getPropertyNames(jsi::Runtime& runtime) {