railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

feat: Replace Reanimated with RN Worklets (#1468)

Browse Source 
* Setup RN Worklets

* Use RN Worklets on iOS

* Fix console

* Add `installFrameProcessorBindings()` function

* Add `FrameProcessorPlugins` proxy (BREAKING CHANGE)

* Clean up docs

* Update FRAME_PROCESSORS.mdx

* Use RN Worklets 0.2.5

* feat: Android build setup

* Rewrite Android Frame Processor Part

* Update CMakeLists.txt

* fix: Add react-native-worklets Gradle dependency

* Update Podfile.lock

* fix build

* gradle:7.4.1

* Init JSI Bindings in method on Android

* Fix Folly flags

* fix: Init `FrameProcessorRuntimeManager` later

* fix: Wrap in `<GestureHandlerRootView>`

* Refactor plugins

* fix: Remove enableFrameProcessors

* Install RN Worklets from current GH master

* Update babel.config.js

* Update CameraViewModule.kt

* Update ImageProxyUtils.java

* feat: Upgrade to Reanimated v3

* fix: Fix crash on Worklet init

* Update RN Worklets to latest master

* fix: Simplify FP Plugins Proxy
This commit is contained in:
Marc Rousavy
2023-02-13 15:22:45 +01:00
committed by GitHub
parent 11d1e7178d
commit a0590dccb5
55 changed files with 469 additions and 861 deletions
Download Patch File Download Diff File Expand all files Collapse all files

165
android/src/main/cpp/FrameProcessorRuntimeManager.cpp
View File

@@ -7,17 +7,11 @@
#include <jni.h>
#include <utility>
#include <string>
#include <JsiWorklet.h>
#include "RuntimeDecorator.h"
#include "RuntimeManager.h"
#include "reanimated-headers/AndroidScheduler.h"
#include "reanimated-headers/AndroidErrorHandler.h"
#include "MakeJSIRuntime.h"
#include "CameraView.h"
#include "FrameHostObject.h"
#include "JSIJNIConversion.h"
#include "VisionCameraScheduler.h"
#include "java-bindings/JImageProxy.h"
#include "java-bindings/JFrameProcessorPlugin.h"
@@ -28,6 +22,28 @@ using TSelf = local_ref<HybridClass<vision::FrameProcessorRuntimeManager>::jhybr
using TJSCallInvokerHolder = jni::alias_ref<facebook::react::CallInvokerHolder::javaobject>;
using TAndroidScheduler = jni::alias_ref<VisionCameraScheduler::javaobject>;
FrameProcessorRuntimeManager::FrameProcessorRuntimeManager(jni::alias_ref<FrameProcessorRuntimeManager::jhybridobject> jThis,
jsi::Runtime* jsRuntime,
std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker,
std::shared_ptr<vision::VisionCameraScheduler> scheduler) :
javaPart_(jni::make_global(jThis)),
_jsRuntime(jsRuntime) {
auto runOnJS = [jsCallInvoker](std::function<void()>&& f) {
// Run on React JS Runtime
jsCallInvoker->invokeAsync(std::move(f));
};
auto runOnWorklet = [scheduler](std::function<void()>&& f) {
// Run on Frame Processor Worklet Runtime
scheduler->dispatchAsync(std::move(f));
};
_workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera");
_workletContext->initialize("VisionCamera",
jsRuntime,
runOnJS,
runOnWorklet);
}
// JNI binding
void vision::FrameProcessorRuntimeManager::registerNatives() {
registerHybrid({
@@ -35,8 +51,6 @@ void vision::FrameProcessorRuntimeManager::registerNatives() {
FrameProcessorRuntimeManager::initHybrid),
makeNativeMethod("installJSIBindings",
FrameProcessorRuntimeManager::installJSIBindings),
makeNativeMethod("initializeRuntime",
FrameProcessorRuntimeManager::initializeRuntime),
makeNativeMethod("registerPlugin",
FrameProcessorRuntimeManager::registerPlugin),
});
@@ -52,32 +66,11 @@ TSelf vision::FrameProcessorRuntimeManager::initHybrid(
"Initializing FrameProcessorRuntimeManager...");
// cast from JNI hybrid objects to C++ instances
auto runtime = reinterpret_cast<jsi::Runtime*>(jsRuntimePointer);
auto jsRuntime = reinterpret_cast<jsi::Runtime*>(jsRuntimePointer);
auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker();
auto scheduler = std::shared_ptr<VisionCameraScheduler>(androidScheduler->cthis());
scheduler->setJSCallInvoker(jsCallInvoker);
return makeCxxInstance(jThis, runtime, jsCallInvoker, scheduler);
}
void vision::FrameProcessorRuntimeManager::initializeRuntime() {
__android_log_write(ANDROID_LOG_INFO, TAG,
"Initializing Vision JS-Runtime...");
// create JSI runtime and decorate it
auto runtime = makeJSIRuntime();
reanimated::RuntimeDecorator::decorateRuntime(*runtime, "FRAME_PROCESSOR");
runtime->global().setProperty(*runtime, "_FRAME_PROCESSOR",
jsi::Value(true));
// create REA runtime manager
auto errorHandler = std::make_shared<reanimated::AndroidErrorHandler>(scheduler_);
_runtimeManager = std::make_unique<reanimated::RuntimeManager>(std::move(runtime),
errorHandler,
scheduler_);
__android_log_write(ANDROID_LOG_INFO, TAG,
"Initialized Vision JS-Runtime!");
return makeCxxInstance(jThis, jsRuntime, jsCallInvoker, scheduler);
}
global_ref<CameraView::javaobject> FrameProcessorRuntimeManager::findCameraViewById(int viewId) {
@@ -87,21 +80,16 @@ global_ref<CameraView::javaobject> FrameProcessorRuntimeManager::findCameraViewB
}
void FrameProcessorRuntimeManager::logErrorToJS(const std::string& message) {
if (!this->jsCallInvoker_) {
if (!_workletContext) {
return;
}
this->jsCallInvoker_->invokeAsync([this, message]() {
if (this->runtime_ == nullptr) {
return;
}
auto& runtime = *this->runtime_;
auto consoleError = runtime
.global()
.getPropertyAsObject(runtime, "console")
.getPropertyAsFunction(runtime, "error");
consoleError.call(runtime, jsi::String::createFromUtf8(runtime, message));
// Call console.error() on JS Thread
_workletContext->invokeOnJsThread([message](jsi::Runtime& runtime) {
auto consoleError = runtime
.global()
.getPropertyAsObject(runtime, "console")
.getPropertyAsFunction(runtime, "error");
consoleError.call(runtime, jsi::String::createFromUtf8(runtime, message));
});
}
@@ -111,42 +99,38 @@ void FrameProcessorRuntimeManager::setFrameProcessor(jsi::Runtime& runtime,
__android_log_write(ANDROID_LOG_INFO, TAG,
"Setting new Frame Processor...");
if (!_runtimeManager || !_runtimeManager->runtime) {
if (!_workletContext) {
throw jsi::JSError(runtime,
"setFrameProcessor(..): VisionCamera's RuntimeManager is not yet initialized!");
"setFrameProcessor(..): VisionCamera's Worklet Context is not yet initialized!");
}
// find camera view
auto cameraView = findCameraViewById(viewTag);
__android_log_write(ANDROID_LOG_INFO, TAG, "Found CameraView!");
// convert jsi::Function to a ShareableValue (can be shared across runtimes)
__android_log_write(ANDROID_LOG_INFO, TAG,
"Adapting Shareable value from function (conversion to worklet)...");
auto worklet = reanimated::ShareableValue::adapt(runtime,
frameProcessor,
_runtimeManager.get());
// convert jsi::Function to a Worklet (can be shared across runtimes)
__android_log_write(ANDROID_LOG_INFO, TAG, "Creating Worklet...");
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, frameProcessor);
auto workletInvoker = std::make_shared<RNWorklet::WorkletInvoker>(worklet);
__android_log_write(ANDROID_LOG_INFO, TAG, "Successfully created worklet!");
scheduler_->scheduleOnUI([=]() {
// cast worklet to a jsi::Function for the new runtime
auto& rt = *_runtimeManager->runtime;
auto function = std::make_shared<jsi::Function>(worklet->getValue(rt).asObject(rt).asFunction(rt));
// assign lambda to frame processor
cameraView->cthis()->setFrameProcessor([this, &rt, function](jni::alias_ref<JImageProxy::javaobject> frame) {
try {
// create HostObject which holds the Frame (JImageProxy)
auto hostObject = std::make_shared<FrameHostObject>(frame);
function->callWithThis(rt, *function, jsi::Object::createFromHostObject(rt, hostObject));
} catch (jsi::JSError& jsError) {
auto message = "Frame Processor threw an error: " + jsError.getMessage();
__android_log_write(ANDROID_LOG_ERROR, TAG, message.c_str());
this->logErrorToJS(message);
}
});
__android_log_write(ANDROID_LOG_INFO, TAG, "Frame Processor set!");
_workletContext->invokeOnWorkletThread([=](RNWorklet::JsiWorkletContext*, jsi::Runtime& rt) {
// Set Frame Processor as callable C++ lambda - this will then call the Worklet
cameraView->cthis()->setFrameProcessor([this, workletInvoker, &rt](jni::alias_ref<JImageProxy::javaobject> frame) {
try {
// create HostObject which holds the Frame (JImageProxy)
auto frameHostObject = std::make_shared<FrameHostObject>(frame);
auto argument = jsi::Object::createFromHostObject(rt, frameHostObject);
jsi::Value jsValue(std::move(argument));
// Call the Worklet on the Worklet Runtime
workletInvoker->call(rt, jsi::Value::undefined(), &jsValue, 1);
} catch (jsi::JSError& jsError) {
// Worklet threw a JS Error, catch it and log it to JS.
auto message = "Frame Processor threw an error: " + jsError.getMessage();
__android_log_write(ANDROID_LOG_ERROR, TAG, message.c_str());
this->logErrorToJS(message);
}
});
});
}
@@ -166,13 +150,13 @@ void FrameProcessorRuntimeManager::unsetFrameProcessor(int viewTag) {
void FrameProcessorRuntimeManager::installJSIBindings() {
__android_log_write(ANDROID_LOG_INFO, TAG, "Installing JSI bindings...");
if (runtime_ == nullptr) {
if (_jsRuntime == nullptr) {
__android_log_write(ANDROID_LOG_ERROR, TAG,
"JS-Runtime was null, Frame Processor JSI bindings could not be installed!");
return;
}
auto& jsiRuntime = *runtime_;
auto& jsiRuntime = *_jsRuntime;
auto setFrameProcessor = [this](jsi::Runtime &runtime,
const jsi::Value &thisValue,
@@ -234,27 +218,26 @@ void FrameProcessorRuntimeManager::installJSIBindings() {
}
void FrameProcessorRuntimeManager::registerPlugin(alias_ref<JFrameProcessorPlugin::javaobject> plugin) {
// _runtimeManager might never be null, but we can never be too sure.
if (!_runtimeManager || !_runtimeManager->runtime) {
throw std::runtime_error("Tried to register plugin before initializing JS runtime! Call `initializeRuntime()` first.");
}
auto& runtime = *_runtimeManager->runtime;
auto& runtime = *_jsRuntime;
// we need a strong reference on the plugin, make_global does that.
auto pluginGlobal = make_global(plugin);
// name is always prefixed with two underscores (__)
auto name = "__" + pluginGlobal->getName();
auto pluginName = pluginGlobal->getName();
__android_log_print(ANDROID_LOG_INFO, TAG, "Installing Frame Processor Plugin \"%s\"...", name.c_str());
__android_log_print(ANDROID_LOG_INFO, TAG, "Installing Frame Processor Plugin \"%s\"...", pluginName.c_str());
auto callback = [pluginGlobal](jsi::Runtime& runtime,
if (!runtime.global().hasProperty(runtime, "FrameProcessorPlugins")) {
runtime.global().setProperty(runtime, "FrameProcessorPlugins", jsi::Object(runtime));
}
jsi::Object frameProcessorPlugins = runtime.global().getPropertyAsObject(runtime, "FrameProcessorPlugins");
auto function = [pluginGlobal](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
size_t count) -> jsi::Value {
// Unbox object and get typed HostObject
auto boxedHostObject = arguments[0].asObject(runtime).asHostObject(runtime);
auto frameHostObject = static_cast<FrameHostObject*>(boxedHostObject.get());
auto frameHostObject = dynamic_cast<FrameHostObject*>(boxedHostObject.get());
// parse params - we are offset by `1` because the frame is the first parameter.
auto params = JArrayClass<jobject>::newArray(count - 1);
@@ -269,10 +252,14 @@ void FrameProcessorRuntimeManager::registerPlugin(alias_ref<JFrameProcessorPlugi
return JSIJNIConversion::convertJNIObjectToJSIValue(runtime, result);
};
runtime.global().setProperty(runtime, name.c_str(), jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forAscii(runtime, name),
1, // frame
callback));
// Assign it to the Proxy.
// A FP Plugin called "example_plugin" can be now called from JS using "FrameProcessorPlugins.example_plugin(frame)"
frameProcessorPlugins.setProperty(runtime,
pluginName.c_str(),
jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forAscii(runtime, pluginName),
1, // frame
function));
}
} // namespace vision

20
android/src/main/cpp/FrameProcessorRuntimeManager.h
View File

@@ -9,9 +9,7 @@
#include <ReactCommon/CallInvokerHolder.h>
#include <memory>
#include <string>
#include "RuntimeManager.h"
#include "reanimated-headers/AndroidScheduler.h"
#include <JsiWorkletContext.h>
#include "CameraView.h"
#include "VisionCameraScheduler.h"
@@ -32,25 +30,17 @@ class FrameProcessorRuntimeManager : public jni::HybridClass<FrameProcessorRunti
static void registerNatives();
explicit FrameProcessorRuntimeManager(jni::alias_ref<FrameProcessorRuntimeManager::jhybridobject> jThis,
jsi::Runtime* runtime,
jsi::Runtime* jsRuntime,
std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker,
std::shared_ptr<vision::VisionCameraScheduler> scheduler) :
javaPart_(jni::make_global(jThis)),
runtime_(runtime),
jsCallInvoker_(jsCallInvoker),
scheduler_(scheduler)
{}
std::shared_ptr<vision::VisionCameraScheduler> scheduler);
private:
friend HybridBase;
jni::global_ref<FrameProcessorRuntimeManager::javaobject> javaPart_;
jsi::Runtime* runtime_;
std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker_;
std::shared_ptr<reanimated::RuntimeManager> _runtimeManager;
std::shared_ptr<vision::VisionCameraScheduler> scheduler_;
jsi::Runtime* _jsRuntime;
std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;
jni::global_ref<CameraView::javaobject> findCameraViewById(int viewId);
void initializeRuntime();
void installJSIBindings();
void registerPlugin(alias_ref<JFrameProcessorPlugin::javaobject> plugin);
void logErrorToJS(const std::string& message);

30
android/src/main/cpp/MakeJSIRuntime.h
View File

@@ -1,30 +0,0 @@
//
// Created by Marc Rousavy on 06.07.21.
//
#pragma once
#include <jsi/jsi.h>
#include <memory>
#if FOR_HERMES
// Hermes
#include <hermes/hermes.h>
#else
// JSC
#include <jsi/JSCRuntime.h>
#endif
namespace vision {
using namespace facebook;
static std::unique_ptr<jsi::Runtime> makeJSIRuntime() {
#if FOR_HERMES
return facebook::hermes::makeHermesRuntime();
#else
return facebook::jsc::makeJSCRuntime();
#endif
}
} // namespace vision

12
android/src/main/cpp/VisionCameraScheduler.cpp
View File

@@ -14,9 +14,9 @@ TSelf VisionCameraScheduler::initHybrid(jni::alias_ref<jhybridobject> jThis) {
return makeCxxInstance(jThis);
}
void VisionCameraScheduler::scheduleOnUI(std::function<void()> job) {
void VisionCameraScheduler::dispatchAsync(std::function<void()> job) {
// 1. add job to queue
uiJobs.push(job);
_jobs.push(job);
scheduleTrigger();
}
@@ -26,16 +26,18 @@ void VisionCameraScheduler::scheduleTrigger() {
method(javaPart_.get());
}
void VisionCameraScheduler::triggerUI() {
void VisionCameraScheduler::trigger() {
std::unique_lock<std::mutex> lock(_mutex);
// 3. call job we enqueued in step 1.
auto job = uiJobs.pop();
auto job = _jobs.front();
job();
_jobs.pop();
}
void VisionCameraScheduler::registerNatives() {
registerHybrid({
makeNativeMethod("initHybrid", VisionCameraScheduler::initHybrid),
makeNativeMethod("triggerUI", VisionCameraScheduler::triggerUI),
makeNativeMethod("trigger", VisionCameraScheduler::trigger),
});
}

22
android/src/main/cpp/VisionCameraScheduler.h
View File

@@ -4,35 +4,47 @@
#pragma once
#include "Scheduler.h"
#include <ReactCommon/CallInvokerHolder.h>
#include <jni.h>
#include <fbjni/fbjni.h>
#include <queue>
#include <mutex>
namespace vision {
using namespace facebook;
class VisionCameraScheduler : public reanimated::Scheduler, public jni::HybridClass<VisionCameraScheduler> {
/**
* A Scheduler that runs methods on the Frame Processor Thread (which is a Java Thread).
* In order to call something on the Java Frame Processor Thread, you have to:
*
* 1. Call `dispatchAsync(..)` with the given C++ Method.
* 2. Internally, `scheduleTrigger()` will get called, which is a Java Method.
* 3. The `scheduleTrigger()` Java Method will switch to the Frame Processor Java Thread and call `trigger()` on there
* 4. `trigger()` is a C++ function here that just calls the passed C++ Method from step 1.
*/
class VisionCameraScheduler : public jni::HybridClass<VisionCameraScheduler> {
public:
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraScheduler;";
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis);
static void registerNatives();
// schedules the given job to be run on the VisionCamera FP Thread at some future point in time
void scheduleOnUI(std::function<void()> job) override;
void dispatchAsync(std::function<void()> job);
private:
friend HybridBase;
jni::global_ref<VisionCameraScheduler::javaobject> javaPart_;
std::queue<std::function<void()>> _jobs;
std::mutex _mutex;
explicit VisionCameraScheduler(jni::alias_ref<VisionCameraScheduler::jhybridobject> jThis):
javaPart_(jni::make_global(jThis)) {}
// Schedules a call to `triggerUI` on the VisionCamera FP Thread
// Schedules a call to `trigger` on the VisionCamera FP Thread
void scheduleTrigger();
// Calls the latest job in the job queue
void triggerUI() override;
void trigger();
};
} // namespace vision

30
android/src/main/cpp/reanimated-headers/AndroidErrorHandler.h
View File

@@ -1,30 +0,0 @@
// copied from https://github.com/software-mansion/react-native-reanimated/blob/master/android/src/main/cpp/headers/AndroidErrorHandler.h
#pragma once
#include "ErrorHandler.h"
#include "AndroidScheduler.h"
#include "Scheduler.h"
#include <jni.h>
#include <memory>
#include <fbjni/fbjni.h>
#include "Logger.h"
namespace reanimated
{
class AndroidErrorHandler : public JavaClass<AndroidErrorHandler>, public ErrorHandler {
std::shared_ptr<ErrorWrapper> error;
std::shared_ptr<Scheduler> scheduler;
void raiseSpec() override;
public:
static auto constexpr kJavaDescriptor = "Lcom/swmansion/reanimated/AndroidErrorHandler;";
AndroidErrorHandler(
std::shared_ptr<Scheduler> scheduler);
std::shared_ptr<Scheduler> getScheduler() override;
std::shared_ptr<ErrorWrapper> getError() override;
void setError(std::string message) override;
virtual ~AndroidErrorHandler() {}
};
}

37
android/src/main/cpp/reanimated-headers/AndroidScheduler.h
View File

@@ -1,37 +0,0 @@
// copied from https://github.com/software-mansion/react-native-reanimated/blob/master/android/src/main/cpp/headers/AndroidScheduler.h
#pragma once
#include <jni.h>
#include <fbjni/fbjni.h>
#include <jsi/jsi.h>
#include <react/jni/CxxModuleWrapper.h>
#include <react/jni/JMessageQueueThread.h>
#include "Scheduler.h"
namespace reanimated {
using namespace facebook;
class AndroidScheduler : public jni::HybridClass<AndroidScheduler> {
public:
static auto constexpr kJavaDescriptor = "Lcom/swmansion/reanimated/Scheduler;";
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis);
static void registerNatives();
std::shared_ptr<Scheduler> getScheduler() { return scheduler_; }
void scheduleOnUI();
private:
friend HybridBase;
void triggerUI();
jni::global_ref<AndroidScheduler::javaobject> javaPart_;
std::shared_ptr<Scheduler> scheduler_;
explicit AndroidScheduler(jni::alias_ref<AndroidScheduler::jhybridobject> jThis);
};
}