feat: Replace Reanimated with RN Worklets (#1468)

* Setup RN Worklets

* Use RN Worklets on iOS

* Fix console

* Add `installFrameProcessorBindings()` function

* Add `FrameProcessorPlugins` proxy (BREAKING CHANGE)

* Clean up docs

* Update FRAME_PROCESSORS.mdx

* Use RN Worklets 0.2.5

* feat: Android build setup

* Rewrite Android Frame Processor Part

* Update CMakeLists.txt

* fix: Add react-native-worklets Gradle dependency

* Update Podfile.lock

* fix build

* gradle:7.4.1

* Init JSI Bindings in method on Android

* Fix Folly flags

* fix: Init `FrameProcessorRuntimeManager` later

* fix: Wrap in `<GestureHandlerRootView>`

* Refactor plugins

* fix: Remove enableFrameProcessors

* Install RN Worklets from current GH master

* Update babel.config.js

* Update CameraViewModule.kt

* Update ImageProxyUtils.java

* feat: Upgrade to Reanimated v3

* fix: Fix crash on Worklet init

* Update RN Worklets to latest master

* fix: Simplify FP Plugins Proxy
This commit is contained in:
Marc Rousavy
2023-02-13 15:22:45 +01:00
committed by GitHub
parent 11d1e7178d
commit a0590dccb5
55 changed files with 469 additions and 861 deletions

View File

@@ -19,12 +19,6 @@
#import "RCTBridge+runOnJS.h"
#import "JSConsoleHelper.h"
#ifdef VISION_CAMERA_DISABLE_FRAME_PROCESSORS
static bool VISION_CAMERA_ENABLE_FRAME_PROCESSORS = false;
#else
static bool VISION_CAMERA_ENABLE_FRAME_PROCESSORS = true;
#endif
@interface CameraBridge: RCTViewManager
@end

View File

@@ -29,10 +29,10 @@ extension CameraView {
internal func updateOrientation() {
// Updates the Orientation for all rotable
let isMirrored = self.videoDeviceInput?.device.position == .front
let isMirrored = videoDeviceInput?.device.position == .front
let connectionOrientation = self.outputOrientation
self.captureSession.outputs.forEach { output in
let connectionOrientation = outputOrientation
captureSession.outputs.forEach { output in
output.connections.forEach { connection in
if connection.isVideoMirroringSupported {
connection.automaticallyAdjustsVideoMirroring = false

View File

@@ -60,6 +60,8 @@ RCT_EXTERN_METHOD(stopRecording:(nonnull NSNumber *)node resolve:(RCTPromiseReso
RCT_EXTERN_METHOD(takePhoto:(nonnull NSNumber *)node options:(NSDictionary *)options resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(focus:(nonnull NSNumber *)node point:(NSDictionary *)point resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
// Static Methods
RCT_EXTERN__BLOCKING_SYNCHRONOUS_METHOD(installFrameProcessorBindings);
RCT_EXTERN_METHOD(getAvailableVideoCodecs:(nonnull NSNumber *)node fileType:(NSString *)fileType resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
@end

View File

@@ -15,20 +15,6 @@ final class CameraViewManager: RCTViewManager {
private var runtimeManager: FrameProcessorRuntimeManager?
override var bridge: RCTBridge! {
didSet {
// Install Frame Processor bindings and setup Runtime
if VISION_CAMERA_ENABLE_FRAME_PROCESSORS {
CameraQueues.frameProcessorQueue.async {
self.runtimeManager = FrameProcessorRuntimeManager(bridge: self.bridge)
self.bridge.runOnJS {
self.runtimeManager!.installFrameProcessorBindings()
}
}
}
}
}
override var methodQueue: DispatchQueue! {
return DispatchQueue.main
}
@@ -43,6 +29,14 @@ final class CameraViewManager: RCTViewManager {
// pragma MARK: React Functions
@objc
final func installFrameProcessorBindings() -> NSNumber {
// Runs on JS Thread
runtimeManager = FrameProcessorRuntimeManager()
runtimeManager!.installFrameProcessorBindings()
return NSNumber(booleanLiteral: true)
}
@objc
final func startRecording(_ node: NSNumber, options: NSDictionary, onRecordCallback: @escaping RCTResponseSenderBlock) {
let component = getCameraView(withTag: node)

View File

@@ -28,14 +28,13 @@
* * Make sure your frame processor returns a Value that can be converted to JS
* * Make sure to use this Macro in an @implementation, not @interface
*
* The JS function will have the same name as the given Objective-C function, but with a "__" prefix.
* Make sure to add that function to the babel.config.js under reanimated's "globals" option, and add TypeScript type declarations.
* The JS function will have the same name as the given Objective-C function. It can be accessed through the FrameProcessorPlugins object exposed by VisionCamera.
*/
#define VISION_EXPORT_FRAME_PROCESSOR(frame_processor) \
\
+(void)load \
{ \
[FrameProcessorPluginRegistry addFrameProcessorPlugin:@"__" @ #frame_processor callback:^id(Frame* frame, NSArray<id>* args) { \
[FrameProcessorPluginRegistry addFrameProcessorPlugin:@ #frame_processor callback:^id(Frame* frame, NSArray<id>* args) { \
return frame_processor(frame, args); \
}]; \
}
@@ -55,7 +54,7 @@ objc_name : NSObject<FrameProcessorPluginBase>
\
+(void)load \
{ \
[FrameProcessorPluginRegistry addFrameProcessorPlugin:@"__" @ #name callback:^id(Frame* frame, NSArray<id>* args) { \
[FrameProcessorPluginRegistry addFrameProcessorPlugin:@ #name callback:^id(Frame* frame, NSArray<id>* args) { \
return [objc_name callback:frame withArgs:args]; \
}]; \
}

View File

@@ -13,13 +13,7 @@
@interface FrameProcessorRuntimeManager : NSObject
- (instancetype)init NS_UNAVAILABLE;
/**
Initializes the Frame Processor Runtime Manager with the given bridge.
This init is not thread safe, so only init this on the Thread you want the runtime to run on.
*/
- (instancetype) initWithBridge:(RCTBridge*)bridge;
- (instancetype) init;
- (void) installFrameProcessorBindings;

View File

@@ -19,25 +19,12 @@
#import <React/RCTUIManager.h>
#import <ReactCommon/RCTTurboModuleManager.h>
#ifndef VISION_CAMERA_DISABLE_FRAME_PROCESSORS
#if __has_include(<RNReanimated/NativeReanimatedModule.h>)
#if __has_include(<RNReanimated/RuntimeManager.h>)
#import <RNReanimated/RuntimeManager.h>
#import <RNReanimated/RuntimeDecorator.h>
#import <RNReanimated/REAIOSErrorHandler.h>
#import "VisionCameraScheduler.h"
#define ENABLE_FRAME_PROCESSORS
#else
#warning Your react-native-reanimated version is not compatible with VisionCamera, Frame Processors are disabled. Make sure you're using reanimated 2.2.0 or above!
#endif
#else
#warning The NativeReanimatedModule.h header could not be found, Frame Processors are disabled. If you want to use Frame Processors, make sure you install react-native-reanimated!
#endif
#endif
#import "JsiWorkletContext.h"
#import "JsiWorkletApi.h"
#import "JsiWorklet.h"
#import "FrameProcessorUtils.h"
#import "FrameProcessorCallback.h"
#import "../React Utils/MakeJSIRuntime.h"
#import "../React Utils/JSIUtils.h"
// Forward declarations for the Swift classes
@@ -51,84 +38,101 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
@end
@implementation FrameProcessorRuntimeManager {
#ifdef ENABLE_FRAME_PROCESSORS
std::unique_ptr<reanimated::RuntimeManager> runtimeManager;
#endif
__weak RCTBridge* weakBridge;
std::shared_ptr<RNWorklet::JsiWorkletContext> workletContext;
}
- (instancetype) initWithBridge:(RCTBridge*)bridge {
self = [super init];
if (self) {
#ifdef ENABLE_FRAME_PROCESSORS
NSLog(@"FrameProcessorBindings: Creating Runtime Manager...");
weakBridge = bridge;
auto runtime = vision::makeJSIRuntime();
reanimated::RuntimeDecorator::decorateRuntime(*runtime, "FRAME_PROCESSOR");
runtime->global().setProperty(*runtime, "_FRAME_PROCESSOR", jsi::Value(true));
auto callInvoker = bridge.jsCallInvoker;
auto scheduler = std::make_shared<vision::VisionCameraScheduler>(callInvoker);
runtimeManager = std::make_unique<reanimated::RuntimeManager>(std::move(runtime),
std::make_shared<reanimated::REAIOSErrorHandler>(scheduler),
scheduler);
NSLog(@"FrameProcessorBindings: Runtime Manager created!");
NSLog(@"FrameProcessorBindings: Installing Frame Processor plugins...");
auto& visionRuntime = *runtimeManager->runtime;
auto visionGlobal = visionRuntime.global();
for (NSString* pluginKey in [FrameProcessorPluginRegistry frameProcessorPlugins]) {
auto pluginName = [pluginKey UTF8String];
NSLog(@"FrameProcessorBindings: Installing Frame Processor plugin \"%s\"...", pluginName);
FrameProcessorPlugin callback = [[FrameProcessorPluginRegistry frameProcessorPlugins] valueForKey:pluginKey];
auto function = [callback, callInvoker](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
size_t count) -> jsi::Value {
auto frameHostObject = arguments[0].asObject(runtime).asHostObject(runtime);
auto frame = static_cast<FrameHostObject*>(frameHostObject.get());
auto args = convertJSICStyleArrayToNSArray(runtime,
arguments + 1, // start at index 1 since first arg = Frame
count - 1, // use smaller count
callInvoker);
id result = callback(frame->frame, args);
return convertObjCObjectToJSIValue(runtime, result);
};
visionGlobal.setProperty(visionRuntime, pluginName, jsi::Function::createFromHostFunction(visionRuntime,
jsi::PropNameID::forAscii(visionRuntime, pluginName),
1, // frame
function));
- (instancetype)init {
if (self = [super init]) {
// Initialize self
}
return self;
}
NSLog(@"FrameProcessorBindings: Frame Processor plugins installed!");
#else
NSLog(@"Reanimated not found, Frame Processors are disabled.");
#endif
- (void) setupWorkletContext:(jsi::Runtime&)runtime {
NSLog(@"FrameProcessorBindings: Creating Worklet Context...");
auto callInvoker = RCTBridge.currentBridge.jsCallInvoker;
auto runOnJS = [callInvoker](std::function<void()>&& f) {
// Run on React JS Runtime
callInvoker->invokeAsync(std::move(f));
};
auto runOnWorklet = [](std::function<void()>&& f) {
// Run on Frame Processor Worklet Runtime
dispatch_async(CameraQueues.frameProcessorQueue, [f = std::move(f)](){
f();
});
};
workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera");
workletContext->initialize("VisionCamera",
&runtime,
runOnJS,
runOnWorklet);
NSLog(@"FrameProcessorBindings: Worklet Context Created!");
NSLog(@"FrameProcessorBindings: Installing Frame Processor plugins...");
jsi::Object frameProcessorPlugins(runtime);
// Iterate through all registered plugins (+init)
for (NSString* pluginKey in [FrameProcessorPluginRegistry frameProcessorPlugins]) {
auto pluginName = [pluginKey UTF8String];
NSLog(@"FrameProcessorBindings: Installing Frame Processor plugin \"%s\"...", pluginName);
// Get the Plugin callback func
FrameProcessorPlugin callback = [[FrameProcessorPluginRegistry frameProcessorPlugins] valueForKey:pluginKey];
// Create the JSI host function
auto function = [callback, callInvoker](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
size_t count) -> jsi::Value {
// Get the first parameter, which is always the native Frame Host Object.
auto frameHostObject = arguments[0].asObject(runtime).asHostObject(runtime);
auto frame = static_cast<FrameHostObject*>(frameHostObject.get());
// Convert any additional parameters to the Frame Processor to ObjC objects
auto args = convertJSICStyleArrayToNSArray(runtime,
arguments + 1, // start at index 1 since first arg = Frame
count - 1, // use smaller count
callInvoker);
// Call the FP Plugin, which might return something.
id result = callback(frame->frame, args);
// Convert the return value (or null) to a JS Value and return it to JS
return convertObjCObjectToJSIValue(runtime, result);
};
// Assign it to the Proxy.
// A FP Plugin called "example_plugin" can be now called from JS using "FrameProcessorPlugins.example_plugin(frame)"
frameProcessorPlugins.setProperty(runtime,
pluginName,
jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forAscii(runtime, pluginName),
1, // frame
function));
}
return self;
// global.FrameProcessorPlugins Proxy
runtime.global().setProperty(runtime, "FrameProcessorPlugins", frameProcessorPlugins);
NSLog(@"FrameProcessorBindings: Frame Processor plugins installed!");
}
- (void) installFrameProcessorBindings {
#ifdef ENABLE_FRAME_PROCESSORS
if (!weakBridge) {
NSLog(@"FrameProcessorBindings: Failed to install Frame Processor Bindings - bridge was null!");
return;
}
NSLog(@"FrameProcessorBindings: Installing Frame Processor Bindings for Bridge...");
RCTCxxBridge *cxxBridge = (RCTCxxBridge *)weakBridge;
RCTCxxBridge *cxxBridge = (RCTCxxBridge *)[RCTBridge currentBridge];
if (!cxxBridge.runtime) {
return;
}
jsi::Runtime& jsiRuntime = *(jsi::Runtime*)cxxBridge.runtime;
// Install the Worklet Runtime in the main React JS Runtime
[self setupWorkletContext:jsiRuntime];
NSLog(@"FrameProcessorBindings: Installing global functions...");
// setFrameProcessor(viewTag: number, frameProcessor: (frame: Frame) => void)
@@ -139,27 +143,21 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
NSLog(@"FrameProcessorBindings: Setting new frame processor...");
if (!arguments[0].isNumber()) throw jsi::JSError(runtime, "Camera::setFrameProcessor: First argument ('viewTag') must be a number!");
if (!arguments[1].isObject()) throw jsi::JSError(runtime, "Camera::setFrameProcessor: Second argument ('frameProcessor') must be a function!");
if (!runtimeManager || !runtimeManager->runtime) throw jsi::JSError(runtime, "Camera::setFrameProcessor: The RuntimeManager is not yet initialized!");
auto viewTag = arguments[0].asNumber();
NSLog(@"FrameProcessorBindings: Adapting Shareable value from function (conversion to worklet)...");
auto worklet = reanimated::ShareableValue::adapt(runtime, arguments[1], runtimeManager.get());
NSLog(@"FrameProcessorBindings: Successfully created worklet!");
NSLog(@"FrameProcessorBindings: Converting JSI Function to Worklet...");
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, arguments[1]);
RCTExecuteOnMainQueue([=]() {
auto currentBridge = [RCTBridge currentBridge];
auto anonymousView = [currentBridge.uiManager viewForReactTag:[NSNumber numberWithDouble:viewTag]];
auto view = static_cast<CameraView*>(anonymousView);
dispatch_async(CameraQueues.frameProcessorQueue, [=]() {
NSLog(@"FrameProcessorBindings: Converting worklet to Objective-C callback...");
NSLog(@"FrameProcessorBindings: Converting worklet to Objective-C callback...");
auto& rt = *runtimeManager->runtime;
auto function = worklet->getValue(rt).asObject(rt).asFunction(rt);
view.frameProcessorCallback = convertWorkletToFrameProcessorCallback(workletContext->getWorkletRuntime(), worklet);
view.frameProcessorCallback = convertJSIFunctionToFrameProcessorCallback(rt, function);
NSLog(@"FrameProcessorBindings: Frame processor set!");
});
NSLog(@"FrameProcessorBindings: Frame processor set!");
});
return jsi::Value::undefined();
@@ -198,7 +196,6 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
unsetFrameProcessor));
NSLog(@"FrameProcessorBindings: Finished installing bindings.");
#endif
}
@end

View File

@@ -17,7 +17,9 @@
#endif
#import <jsi/jsi.h>
#import "JsiWorklet.h"
#import <memory>
using namespace facebook;
FrameProcessorCallback convertJSIFunctionToFrameProcessorCallback(jsi::Runtime &runtime, const jsi::Function &value);
FrameProcessorCallback convertWorkletToFrameProcessorCallback(jsi::Runtime& runtime, std::shared_ptr<RNWorklet::JsiWorklet> worklet);

View File

@@ -19,18 +19,25 @@
#import "JSConsoleHelper.h"
#import <ReactCommon/RCTTurboModule.h>
FrameProcessorCallback convertJSIFunctionToFrameProcessorCallback(jsi::Runtime& runtime, const jsi::Function& value) {
__block auto cb = value.getFunction(runtime);
#import "JsiWorklet.h"
FrameProcessorCallback convertWorkletToFrameProcessorCallback(jsi::Runtime& runtime, std::shared_ptr<RNWorklet::JsiWorklet> worklet) {
auto workletInvoker = std::make_shared<RNWorklet::WorkletInvoker>(worklet);
// Converts a Worklet to a callable Objective-C block function
return ^(Frame* frame) {
auto frameHostObject = std::make_shared<FrameHostObject>(frame);
try {
cb.callWithThis(runtime, cb, jsi::Object::createFromHostObject(runtime, frameHostObject));
// Call JS Frame Processor function with boxed Frame Host Object
auto argument = jsi::Object::createFromHostObject(runtime, frameHostObject);
jsi::Value jsValue(std::move(argument));
workletInvoker->call(runtime, jsi::Value::undefined(), &jsValue, 1);
} catch (jsi::JSError& jsError) {
auto stack = std::regex_replace(jsError.getStack(), std::regex("\n"), "\n ");
auto message = [NSString stringWithFormat:@"Frame Processor threw an error: %s\nIn: %s", jsError.getMessage().c_str(), stack.c_str()];
RCTBridge* bridge = [RCTBridge currentBridge];
if (bridge != nil) {
bridge.jsCallInvoker->invokeAsync([bridge, message]() {

View File

@@ -1,38 +0,0 @@
//
// VisionCameraScheduler.h
// VisionCamera
//
// Created by Marc Rousavy on 23.07.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
#pragma once
#include <functional>
#import <React-callinvoker/ReactCommon/CallInvoker.h>
#if __has_include(<RNReanimated/RuntimeManager.h>)
#import <RNReanimated/Scheduler.h>
#else
// dummy placeholder
namespace reanimated {
class Scheduler {
public:
virtual void scheduleOnUI(std::function<void()> job);
protected:
std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker_;
};
}
#endif
namespace vision {
class VisionCameraScheduler : public reanimated::Scheduler {
public:
VisionCameraScheduler(std::shared_ptr<facebook::react::CallInvoker> jsInvoker);
virtual ~VisionCameraScheduler();
void scheduleOnUI(std::function<void()> job) override;
};
} // namespace vision

View File

@@ -1,39 +0,0 @@
//
// VisionCameraScheduler.mm
// VisionCamera
//
// Created by Marc Rousavy on 23.07.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
#import <Foundation/Foundation.h>
#import "VisionCameraScheduler.h"
#import <React-callinvoker/ReactCommon/CallInvoker.h>
// Forward declarations for the Swift classes
__attribute__((objc_runtime_name("_TtC12VisionCamera12CameraQueues")))
@interface CameraQueues : NSObject
@property (nonatomic, class, readonly, strong) dispatch_queue_t _Nonnull frameProcessorQueue;
@end
namespace vision {
using namespace facebook;
VisionCameraScheduler::VisionCameraScheduler(std::shared_ptr<react::CallInvoker> jsInvoker) {
this->jsCallInvoker_ = jsInvoker;
}
// does not schedule on UI thread but rather on Frame Processor Thread
void VisionCameraScheduler::scheduleOnUI(std::function<void()> job) {
dispatch_async(CameraQueues.frameProcessorQueue, ^{
job();
});
}
VisionCameraScheduler::~VisionCameraScheduler(){
}
} // namespace vision

View File

@@ -1,42 +0,0 @@
//
// MakeJSIRuntime.h
// VisionCamera
//
// Created by Marc Rousavy on 06.07.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
#pragma once
#include <jsi/jsi.h>
#include <memory>
#if __has_include(<reacthermes/HermesExecutorFactory.h>)
// Hermes (https://hermesengine.dev) (RN 0.65+)
#include <reacthermes/HermesExecutorFactory.h>
#elif __has_include(<hermes/hermes.h>)
// Hermes (https://hermesengine.dev)
#include <hermes/hermes.h>
#elif __has_include(<v8runtime/V8RuntimeFactory.h>)
// V8 (https://github.com/Kudo/react-native-v8)
#include <v8runtime/V8RuntimeFactory.h>
#else
// JSC
#include <jsi/JSCRuntime.h>
#endif
using namespace facebook;
namespace vision {
static std::unique_ptr<jsi::Runtime> makeJSIRuntime() {
#if __has_include(<hermes/hermes.h>) || __has_include(<reacthermes/HermesExecutorFactory.h>)
return facebook::hermes::makeHermesRuntime();
#elif __has_include(<v8runtime/V8RuntimeFactory.h>)
return facebook::createV8Runtime("");
#else
return facebook::jsc::makeJSCRuntime();
#endif
}
} // namespace vision

View File

@@ -94,7 +94,6 @@
B84760A22608EE38004C3180 /* FrameHostObject.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameHostObject.h; sourceTree = "<group>"; };
B84760A52608EE7C004C3180 /* FrameHostObject.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = FrameHostObject.mm; sourceTree = "<group>"; };
B84760DE2608F57D004C3180 /* CameraQueues.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CameraQueues.swift; sourceTree = "<group>"; };
B84C10592694A182006EFA70 /* MakeJSIRuntime.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MakeJSIRuntime.h; sourceTree = "<group>"; };
B864004F27849A2400E9D2CA /* UIInterfaceOrientation+descriptor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "UIInterfaceOrientation+descriptor.swift"; sourceTree = "<group>"; };
B86400512784A23400E9D2CA /* CameraView+Orientation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CameraView+Orientation.swift"; sourceTree = "<group>"; };
B86DC970260E2D5200FB17B2 /* AVAudioSession+trySetAllowHaptics.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVAudioSession+trySetAllowHaptics.swift"; sourceTree = "<group>"; };
@@ -231,7 +230,6 @@
B8994E6B263F03E100069589 /* JSIUtils.mm */,
B8805065266798AB00EAD7F2 /* JSConsoleHelper.h */,
B8805066266798B600EAD7F2 /* JSConsoleHelper.mm */,
B84C10592694A182006EFA70 /* MakeJSIRuntime.h */,
);
path = "React Utils";
sourceTree = "<group>";