railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Replace Reanimated with RN Worklets (#1468)

Browse Source 
* Setup RN Worklets

* Use RN Worklets on iOS

* Fix console

* Add `installFrameProcessorBindings()` function

* Add `FrameProcessorPlugins` proxy (BREAKING CHANGE)

* Clean up docs

* Update FRAME_PROCESSORS.mdx

* Use RN Worklets 0.2.5

* feat: Android build setup

* Rewrite Android Frame Processor Part

* Update CMakeLists.txt

* fix: Add react-native-worklets Gradle dependency

* Update Podfile.lock

* fix build

* gradle:7.4.1

* Init JSI Bindings in method on Android

* Fix Folly flags

* fix: Init `FrameProcessorRuntimeManager` later

* fix: Wrap in `<GestureHandlerRootView>`

* Refactor plugins

* fix: Remove enableFrameProcessors

* Install RN Worklets from current GH master

* Update babel.config.js

* Update CameraViewModule.kt

* Update ImageProxyUtils.java

* feat: Upgrade to Reanimated v3

* fix: Fix crash on Worklet init

* Update RN Worklets to latest master

* fix: Simplify FP Plugins Proxy
This commit is contained in:
Marc Rousavy
2023-02-13 15:22:45 +01:00
committed by GitHub
parent 11d1e7178d
commit a0590dccb5
55 changed files with 469 additions and 861 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
ios/Frame Processor/FrameProcessorPlugin.h
View File

@@ -28,14 +28,13 @@
* * Make sure your frame processor returns a Value that can be converted to JS
* * Make sure to use this Macro in an @implementation, not @interface
*
* The JS function will have the same name as the given Objective-C function, but with a "__" prefix.
* Make sure to add that function to the babel.config.js under reanimated's "globals" option, and add TypeScript type declarations.
* The JS function will have the same name as the given Objective-C function. It can be accessed through the FrameProcessorPlugins object exposed by VisionCamera.
*/
#define VISION_EXPORT_FRAME_PROCESSOR(frame_processor) \
\
+(void)load \
{ \
[FrameProcessorPluginRegistry addFrameProcessorPlugin:@"__" @ #frame_processor callback:^id(Frame* frame, NSArray<id>* args) { \
[FrameProcessorPluginRegistry addFrameProcessorPlugin:@ #frame_processor callback:^id(Frame* frame, NSArray<id>* args) { \
return frame_processor(frame, args); \
}]; \
}
@@ -55,7 +54,7 @@ objc_name : NSObject<FrameProcessorPluginBase>
\
+(void)load \
{ \
[FrameProcessorPluginRegistry addFrameProcessorPlugin:@"__" @ #name callback:^id(Frame* frame, NSArray<id>* args) { \
[FrameProcessorPluginRegistry addFrameProcessorPlugin:@ #name callback:^id(Frame* frame, NSArray<id>* args) { \
return [objc_name callback:frame withArgs:args]; \
}]; \
}

8
ios/Frame Processor/FrameProcessorRuntimeManager.h
View File

@@ -13,13 +13,7 @@
@interface FrameProcessorRuntimeManager : NSObject
- (instancetype)init NS_UNAVAILABLE;
/**
Initializes the Frame Processor Runtime Manager with the given bridge.
This init is not thread safe, so only init this on the Thread you want the runtime to run on.
*/
- (instancetype) initWithBridge:(RCTBridge*)bridge;
- (instancetype) init;
- (void) installFrameProcessorBindings;

181
ios/Frame Processor/FrameProcessorRuntimeManager.mm
View File

@@ -19,25 +19,12 @@
#import <React/RCTUIManager.h>
#import <ReactCommon/RCTTurboModuleManager.h>
#ifndef VISION_CAMERA_DISABLE_FRAME_PROCESSORS
#if __has_include(<RNReanimated/NativeReanimatedModule.h>)
#if __has_include(<RNReanimated/RuntimeManager.h>)
#import <RNReanimated/RuntimeManager.h>
#import <RNReanimated/RuntimeDecorator.h>
#import <RNReanimated/REAIOSErrorHandler.h>
#import "VisionCameraScheduler.h"
#define ENABLE_FRAME_PROCESSORS
#else
#warning Your react-native-reanimated version is not compatible with VisionCamera, Frame Processors are disabled. Make sure you're using reanimated 2.2.0 or above!
#endif
#else
#warning The NativeReanimatedModule.h header could not be found, Frame Processors are disabled. If you want to use Frame Processors, make sure you install react-native-reanimated!
#endif
#endif
#import "JsiWorkletContext.h"
#import "JsiWorkletApi.h"
#import "JsiWorklet.h"
#import "FrameProcessorUtils.h"
#import "FrameProcessorCallback.h"
#import "../React Utils/MakeJSIRuntime.h"
#import "../React Utils/JSIUtils.h"
// Forward declarations for the Swift classes
@@ -51,84 +38,101 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
@end
@implementation FrameProcessorRuntimeManager {
#ifdef ENABLE_FRAME_PROCESSORS
std::unique_ptr<reanimated::RuntimeManager> runtimeManager;
#endif
__weak RCTBridge* weakBridge;
std::shared_ptr<RNWorklet::JsiWorkletContext> workletContext;
}
- (instancetype) initWithBridge:(RCTBridge*)bridge {
self = [super init];
if (self) {
#ifdef ENABLE_FRAME_PROCESSORS
NSLog(@"FrameProcessorBindings: Creating Runtime Manager...");
weakBridge = bridge;
auto runtime = vision::makeJSIRuntime();
reanimated::RuntimeDecorator::decorateRuntime(*runtime, "FRAME_PROCESSOR");
runtime->global().setProperty(*runtime, "_FRAME_PROCESSOR", jsi::Value(true));
auto callInvoker = bridge.jsCallInvoker;
auto scheduler = std::make_shared<vision::VisionCameraScheduler>(callInvoker);
runtimeManager = std::make_unique<reanimated::RuntimeManager>(std::move(runtime),
std::make_shared<reanimated::REAIOSErrorHandler>(scheduler),
scheduler);
NSLog(@"FrameProcessorBindings: Runtime Manager created!");
NSLog(@"FrameProcessorBindings: Installing Frame Processor plugins...");
auto& visionRuntime = *runtimeManager->runtime;
auto visionGlobal = visionRuntime.global();
for (NSString* pluginKey in [FrameProcessorPluginRegistry frameProcessorPlugins]) {
auto pluginName = [pluginKey UTF8String];
NSLog(@"FrameProcessorBindings: Installing Frame Processor plugin \"%s\"...", pluginName);
FrameProcessorPlugin callback = [[FrameProcessorPluginRegistry frameProcessorPlugins] valueForKey:pluginKey];
auto function = [callback, callInvoker](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
size_t count) -> jsi::Value {
auto frameHostObject = arguments[0].asObject(runtime).asHostObject(runtime);
auto frame = static_cast<FrameHostObject*>(frameHostObject.get());
auto args = convertJSICStyleArrayToNSArray(runtime,
arguments + 1, // start at index 1 since first arg = Frame
count - 1, // use smaller count
callInvoker);
id result = callback(frame->frame, args);
return convertObjCObjectToJSIValue(runtime, result);
};
visionGlobal.setProperty(visionRuntime, pluginName, jsi::Function::createFromHostFunction(visionRuntime,
jsi::PropNameID::forAscii(visionRuntime, pluginName),
1, // frame
function));
- (instancetype)init {
if (self = [super init]) {
// Initialize self
}
return self;
}
NSLog(@"FrameProcessorBindings: Frame Processor plugins installed!");
#else
NSLog(@"Reanimated not found, Frame Processors are disabled.");
#endif
- (void) setupWorkletContext:(jsi::Runtime&)runtime {
NSLog(@"FrameProcessorBindings: Creating Worklet Context...");
auto callInvoker = RCTBridge.currentBridge.jsCallInvoker;
auto runOnJS = [callInvoker](std::function<void()>&& f) {
// Run on React JS Runtime
callInvoker->invokeAsync(std::move(f));
};
auto runOnWorklet = [](std::function<void()>&& f) {
// Run on Frame Processor Worklet Runtime
dispatch_async(CameraQueues.frameProcessorQueue, [f = std::move(f)](){
f();
});
};
workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera");
workletContext->initialize("VisionCamera",
&runtime,
runOnJS,
runOnWorklet);
NSLog(@"FrameProcessorBindings: Worklet Context Created!");
NSLog(@"FrameProcessorBindings: Installing Frame Processor plugins...");
jsi::Object frameProcessorPlugins(runtime);
// Iterate through all registered plugins (+init)
for (NSString* pluginKey in [FrameProcessorPluginRegistry frameProcessorPlugins]) {
auto pluginName = [pluginKey UTF8String];
NSLog(@"FrameProcessorBindings: Installing Frame Processor plugin \"%s\"...", pluginName);
// Get the Plugin callback func
FrameProcessorPlugin callback = [[FrameProcessorPluginRegistry frameProcessorPlugins] valueForKey:pluginKey];
// Create the JSI host function
auto function = [callback, callInvoker](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
size_t count) -> jsi::Value {
// Get the first parameter, which is always the native Frame Host Object.
auto frameHostObject = arguments[0].asObject(runtime).asHostObject(runtime);
auto frame = static_cast<FrameHostObject*>(frameHostObject.get());
// Convert any additional parameters to the Frame Processor to ObjC objects
auto args = convertJSICStyleArrayToNSArray(runtime,
arguments + 1, // start at index 1 since first arg = Frame
count - 1, // use smaller count
callInvoker);
// Call the FP Plugin, which might return something.
id result = callback(frame->frame, args);
// Convert the return value (or null) to a JS Value and return it to JS
return convertObjCObjectToJSIValue(runtime, result);
};
// Assign it to the Proxy.
// A FP Plugin called "example_plugin" can be now called from JS using "FrameProcessorPlugins.example_plugin(frame)"
frameProcessorPlugins.setProperty(runtime,
pluginName,
jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forAscii(runtime, pluginName),
1, // frame
function));
}
return self;
// global.FrameProcessorPlugins Proxy
runtime.global().setProperty(runtime, "FrameProcessorPlugins", frameProcessorPlugins);
NSLog(@"FrameProcessorBindings: Frame Processor plugins installed!");
}
- (void) installFrameProcessorBindings {
#ifdef ENABLE_FRAME_PROCESSORS
if (!weakBridge) {
NSLog(@"FrameProcessorBindings: Failed to install Frame Processor Bindings - bridge was null!");
return;
}
NSLog(@"FrameProcessorBindings: Installing Frame Processor Bindings for Bridge...");
RCTCxxBridge *cxxBridge = (RCTCxxBridge *)weakBridge;
RCTCxxBridge *cxxBridge = (RCTCxxBridge *)[RCTBridge currentBridge];
if (!cxxBridge.runtime) {
return;
}
jsi::Runtime& jsiRuntime = *(jsi::Runtime*)cxxBridge.runtime;
// Install the Worklet Runtime in the main React JS Runtime
[self setupWorkletContext:jsiRuntime];
NSLog(@"FrameProcessorBindings: Installing global functions...");
// setFrameProcessor(viewTag: number, frameProcessor: (frame: Frame) => void)
@@ -139,27 +143,21 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
NSLog(@"FrameProcessorBindings: Setting new frame processor...");
if (!arguments[0].isNumber()) throw jsi::JSError(runtime, "Camera::setFrameProcessor: First argument ('viewTag') must be a number!");
if (!arguments[1].isObject()) throw jsi::JSError(runtime, "Camera::setFrameProcessor: Second argument ('frameProcessor') must be a function!");
if (!runtimeManager || !runtimeManager->runtime) throw jsi::JSError(runtime, "Camera::setFrameProcessor: The RuntimeManager is not yet initialized!");
auto viewTag = arguments[0].asNumber();
NSLog(@"FrameProcessorBindings: Adapting Shareable value from function (conversion to worklet)...");
auto worklet = reanimated::ShareableValue::adapt(runtime, arguments[1], runtimeManager.get());
NSLog(@"FrameProcessorBindings: Successfully created worklet!");
NSLog(@"FrameProcessorBindings: Converting JSI Function to Worklet...");
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, arguments[1]);
RCTExecuteOnMainQueue([=]() {
auto currentBridge = [RCTBridge currentBridge];
auto anonymousView = [currentBridge.uiManager viewForReactTag:[NSNumber numberWithDouble:viewTag]];
auto view = static_cast<CameraView*>(anonymousView);
dispatch_async(CameraQueues.frameProcessorQueue, [=]() {
NSLog(@"FrameProcessorBindings: Converting worklet to Objective-C callback...");
NSLog(@"FrameProcessorBindings: Converting worklet to Objective-C callback...");
auto& rt = *runtimeManager->runtime;
auto function = worklet->getValue(rt).asObject(rt).asFunction(rt);
view.frameProcessorCallback = convertWorkletToFrameProcessorCallback(workletContext->getWorkletRuntime(), worklet);
view.frameProcessorCallback = convertJSIFunctionToFrameProcessorCallback(rt, function);
NSLog(@"FrameProcessorBindings: Frame processor set!");
});
NSLog(@"FrameProcessorBindings: Frame processor set!");
});
return jsi::Value::undefined();
@@ -198,7 +196,6 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
unsetFrameProcessor));
NSLog(@"FrameProcessorBindings: Finished installing bindings.");
#endif
}
@end

4
ios/Frame Processor/FrameProcessorUtils.h
View File

@@ -17,7 +17,9 @@
#endif
#import <jsi/jsi.h>
#import "JsiWorklet.h"
#import <memory>
using namespace facebook;
FrameProcessorCallback convertJSIFunctionToFrameProcessorCallback(jsi::Runtime &runtime, const jsi::Function &value);
FrameProcessorCallback convertWorkletToFrameProcessorCallback(jsi::Runtime& runtime, std::shared_ptr<RNWorklet::JsiWorklet> worklet);

15
ios/Frame Processor/FrameProcessorUtils.mm
View File

@@ -19,18 +19,25 @@
#import "JSConsoleHelper.h"
#import <ReactCommon/RCTTurboModule.h>
FrameProcessorCallback convertJSIFunctionToFrameProcessorCallback(jsi::Runtime& runtime, const jsi::Function& value) {
__block auto cb = value.getFunction(runtime);
#import "JsiWorklet.h"
FrameProcessorCallback convertWorkletToFrameProcessorCallback(jsi::Runtime& runtime, std::shared_ptr<RNWorklet::JsiWorklet> worklet) {
auto workletInvoker = std::make_shared<RNWorklet::WorkletInvoker>(worklet);
// Converts a Worklet to a callable Objective-C block function
return ^(Frame* frame) {
auto frameHostObject = std::make_shared<FrameHostObject>(frame);
try {
cb.callWithThis(runtime, cb, jsi::Object::createFromHostObject(runtime, frameHostObject));
// Call JS Frame Processor function with boxed Frame Host Object
auto argument = jsi::Object::createFromHostObject(runtime, frameHostObject);
jsi::Value jsValue(std::move(argument));
workletInvoker->call(runtime, jsi::Value::undefined(), &jsValue, 1);
} catch (jsi::JSError& jsError) {
auto stack = std::regex_replace(jsError.getStack(), std::regex("\n"), "\n ");
auto message = [NSString stringWithFormat:@"Frame Processor threw an error: %s\nIn: %s", jsError.getMessage().c_str(), stack.c_str()];
RCTBridge* bridge = [RCTBridge currentBridge];
if (bridge != nil) {
bridge.jsCallInvoker->invokeAsync([bridge, message]() {

38
ios/Frame Processor/VisionCameraScheduler.h
View File

@@ -1,38 +0,0 @@
//
// VisionCameraScheduler.h
// VisionCamera
//
// Created by Marc Rousavy on 23.07.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
#pragma once
#include <functional>
#import <React-callinvoker/ReactCommon/CallInvoker.h>
#if __has_include(<RNReanimated/RuntimeManager.h>)
#import <RNReanimated/Scheduler.h>
#else
// dummy placeholder
namespace reanimated {
class Scheduler {
public:
virtual void scheduleOnUI(std::function<void()> job);
protected:
std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker_;
};
}
#endif
namespace vision {
class VisionCameraScheduler : public reanimated::Scheduler {
public:
VisionCameraScheduler(std::shared_ptr<facebook::react::CallInvoker> jsInvoker);
virtual ~VisionCameraScheduler();
void scheduleOnUI(std::function<void()> job) override;
};
} // namespace vision

39
ios/Frame Processor/VisionCameraScheduler.mm
View File

@@ -1,39 +0,0 @@
//
// VisionCameraScheduler.mm
// VisionCamera
//
// Created by Marc Rousavy on 23.07.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
#import <Foundation/Foundation.h>
#import "VisionCameraScheduler.h"
#import <React-callinvoker/ReactCommon/CallInvoker.h>
// Forward declarations for the Swift classes
__attribute__((objc_runtime_name("_TtC12VisionCamera12CameraQueues")))
@interface CameraQueues : NSObject
@property (nonatomic, class, readonly, strong) dispatch_queue_t _Nonnull frameProcessorQueue;
@end
namespace vision {
using namespace facebook;
VisionCameraScheduler::VisionCameraScheduler(std::shared_ptr<react::CallInvoker> jsInvoker) {
this->jsCallInvoker_ = jsInvoker;
}
// does not schedule on UI thread but rather on Frame Processor Thread
void VisionCameraScheduler::scheduleOnUI(std::function<void()> job) {
dispatch_async(CameraQueues.frameProcessorQueue, ^{
job();
});
}
VisionCameraScheduler::~VisionCameraScheduler(){
}
} // namespace vision