railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Replace Reanimated with RN Worklets (#1468)

Browse Source 
* Setup RN Worklets

* Use RN Worklets on iOS

* Fix console

* Add `installFrameProcessorBindings()` function

* Add `FrameProcessorPlugins` proxy (BREAKING CHANGE)

* Clean up docs

* Update FRAME_PROCESSORS.mdx

* Use RN Worklets 0.2.5

* feat: Android build setup

* Rewrite Android Frame Processor Part

* Update CMakeLists.txt

* fix: Add react-native-worklets Gradle dependency

* Update Podfile.lock

* fix build

* gradle:7.4.1

* Init JSI Bindings in method on Android

* Fix Folly flags

* fix: Init `FrameProcessorRuntimeManager` later

* fix: Wrap in `<GestureHandlerRootView>`

* Refactor plugins

* fix: Remove enableFrameProcessors

* Install RN Worklets from current GH master

* Update babel.config.js

* Update CameraViewModule.kt

* Update ImageProxyUtils.java

* feat: Upgrade to Reanimated v3

* fix: Fix crash on Worklet init

* Update RN Worklets to latest master

* fix: Simplify FP Plugins Proxy
This commit is contained in:
Marc Rousavy
2023-02-13 15:22:45 +01:00
committed by GitHub
parent 11d1e7178d
commit a0590dccb5
55 changed files with 469 additions and 861 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
android/CMakeLists.txt
View File

@@ -1,20 +1,23 @@
project(VisionCamera)
cmake_minimum_required(VERSION 3.4.1)
cmake_minimum_required(VERSION 3.9.0)
set (CMAKE_VERBOSE_MAKEFILE ON)
set (CMAKE_CXX_STANDARD 14)
set(CMAKE_VERBOSE_MAKEFILE ON)
set(PACKAGE_NAME "VisionCamera")
set(BUILD_DIR ${CMAKE_SOURCE_DIR}/build)
set(CMAKE_VERBOSE_MAKEFILE ON)
set(CMAKE_CXX_STANDARD 17)
# Folly
include("${NODE_MODULES_DIR}/react-native/ReactAndroid/cmake-utils/folly-flags.cmake")
add_compile_options(${folly_FLAGS})
set (PACKAGE_NAME "VisionCamera")
set (BUILD_DIR ${CMAKE_SOURCE_DIR}/build)
# Consume shared libraries and headers from prefabs
find_package(fbjni REQUIRED CONFIG)
# Third party libraries (Prefabs)
find_package(ReactAndroid REQUIRED CONFIG)
# VisionCamera shared
find_package(fbjni REQUIRED CONFIG)
find_package(react-native-worklets REQUIRED CONFIG)
find_library(LOG_LIB log)
# Add react-native-vision-camera sources
add_library(
${PACKAGE_NAME}
SHARED
@@ -29,87 +32,24 @@ add_library(
src/main/cpp/java-bindings/JHashMap.cpp
)
# includes
# Header Search Paths (includes)
target_include_directories(
${PACKAGE_NAME}
PRIVATE
"${NODE_MODULES_DIR}/react-native/ReactAndroid/src/main/jni/react/turbomodule"
"src/main/cpp"
"${NODE_MODULES_DIR}/react-native/ReactCommon"
"${NODE_MODULES_DIR}/react-native/ReactCommon/callinvoker"
"${NODE_MODULES_DIR}/react-native/ReactCommon/jsi"
"${NODE_MODULES_DIR}/react-native/ReactCommon/react/renderer/graphics/platform/cxx"
"${NODE_MODULES_DIR}/react-native/ReactCommon/runtimeexecutor"
"${NODE_MODULES_DIR}/react-native/ReactCommon/yoga"
# --- Reanimated ---
# New
"${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/AnimatedSensor"
"${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/Tools"
"${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/SpecTools"
"${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/SharedItems"
"${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/Registries"
"${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/LayoutAnimations"
"${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/hidden_headers"
"src/main/cpp"
"${NODE_MODULES_DIR}/react-native/ReactAndroid/src/main/jni/react/turbomodule" # <-- CallInvokerHolder JNI wrapper
)
# find libraries
file (GLOB LIBRN_DIR "${BUILD_DIR}/react-native-0*/jni/${ANDROID_ABI}")
if(${FOR_HERMES})
string(APPEND CMAKE_CXX_FLAGS " -DFOR_HERMES=1")
find_package(hermes-engine REQUIRED CONFIG)
target_link_libraries(
${PACKAGE_NAME}
"hermes-engine::libhermes"
)
file (GLOB LIBREANIMATED_DIR "${BUILD_DIR}/react-native-reanimated-*-hermes.aar/jni/${ANDROID_ABI}")
else()
file (GLOB LIBJSC_DIR "${BUILD_DIR}/android-jsc*.aar/jni/${ANDROID_ABI}")
set(JS_ENGINE_LIB ReactAndroid::jscexecutor)
target_link_libraries(
${PACKAGE_NAME}
${JS_ENGINE_LIB}
)
# Use Reanimated JSC
file (GLOB LIBREANIMATED_DIR "${BUILD_DIR}/react-native-reanimated-*-jsc.aar/jni/${ANDROID_ABI}")
endif()
# Link everything together
target_link_libraries(
${PACKAGE_NAME}
ReactAndroid::folly_runtime
ReactAndroid::glog
ReactAndroid::jsi
ReactAndroid::reactnativejni
fbjni::fbjni
)
find_library(
REANIMATED_LIB
reanimated
PATHS ${LIBREANIMATED_DIR}
NO_CMAKE_FIND_ROOT_PATH
)
find_library(
LOG_LIB
log
)
# linking
message(WARNING "VisionCamera linking: FOR_HERMES=${FOR_HERMES}")
target_link_libraries(
${PACKAGE_NAME}
${LOG_LIB}
${JSI_LIB}
${REANIMATED_LIB}
${REACT_NATIVE_JNI_LIB}
${FBJNI_LIB}
${FOLLY_LIB}
android
${LOG_LIB} # <-- Logcat logger
android # <-- Android JNI core
ReactAndroid::jsi # <-- RN: JSI
ReactAndroid::reactnativejni # <-- RN: React Native JNI bindings
ReactAndroid::folly_runtime # <-- RN: For casting JSI <> Java objects
fbjni::fbjni # <-- fbjni
react-native-worklets::rnworklets # <-- RN Worklets
)

43
android/build.gradle
View File

@@ -1,3 +1,5 @@
import java.nio.file.Paths
buildscript {
def kotlin_version = rootProject.ext.has('kotlinVersion') ? rootProject.ext.get('kotlinVersion') : project.properties['VisionCamera_kotlinVersion']
@@ -16,7 +18,6 @@ buildscript {
}
}
def ENABLE_FRAME_PROCESSORS = false // TODO: fix
def kotlin_version = rootProject.ext.has('kotlinVersion') ? rootProject.ext.get('kotlinVersion') : project.properties['VisionCamera_kotlinVersion']
def resolveBuildType() {
@@ -49,6 +50,23 @@ def reactNativeArchitectures() {
return value ? value.split(",") : ["armeabi-v7a", "x86", "x86_64", "arm64-v8a"]
}
static def findNodeModules(baseDir) {
def basePath = baseDir.toPath().normalize()
// Node's module resolution algorithm searches up to the root directory,
// after which the base path will be null
while (basePath) {
def nodeModulesPath = Paths.get(basePath.toString(), "node_modules")
def reactNativePath = Paths.get(nodeModulesPath.toString(), "react-native")
if (nodeModulesPath.toFile().exists() && reactNativePath.toFile().exists()) {
return nodeModulesPath.toString()
}
basePath = basePath.getParent()
}
throw new GradleException("react-native-worklets: Failed to find node_modules/ path!")
}
def nodeModules = findNodeModules(projectDir)
repositories {
google()
mavenCentral()
@@ -77,13 +95,12 @@ android {
versionName "1.0"
buildConfigField "boolean", "IS_NEW_ARCHITECTURE_ENABLED", isNewArchitectureEnabled().toString()
if (ENABLE_FRAME_PROCESSORS) {
externalNativeBuild {
cmake {
cppFlags "-O2 -frtti -fexceptions -Wall -Wno-unused-variable -fstack-protector-all"
arguments "-DANDROID_STL=c++_shared"
abiFilters (*reactNativeArchitectures())
}
externalNativeBuild {
cmake {
cppFlags "-O2 -frtti -fexceptions -Wall -Wno-unused-variable -fstack-protector-all"
arguments "-DANDROID_STL=c++_shared",
"-DNODE_MODULES_DIR=${nodeModules}"
abiFilters (*reactNativeArchitectures())
}
}
}
@@ -93,11 +110,9 @@ android {
targetCompatibility JavaVersion.VERSION_1_8
}
if (ENABLE_FRAME_PROCESSORS) {
externalNativeBuild {
cmake {
path "CMakeLists.txt"
}
externalNativeBuild {
cmake {
path "CMakeLists.txt"
}
}
packagingOptions {
@@ -128,6 +143,8 @@ dependencies {
implementation "androidx.camera:camera-extensions:1.1.0-beta02"
implementation "androidx.exifinterface:exifinterface:1.3.3"
implementation project(":react-native-worklets")
}
// Resolves "LOCAL_SRC_FILES points to a missing file, Check that libfb.so exists or that its path is correct".

3
android/settings.gradle
View File

@@ -1,6 +1,3 @@
rootProject.name = 'VisionCamera'
include ':react-native-reanimated'
project(':react-native-reanimated').projectDir = new File(rootProject.projectDir, '../node_modules/react-native-reanimated/android/')
include ':VisionCamera'

165
android/src/main/cpp/FrameProcessorRuntimeManager.cpp
View File

@@ -7,17 +7,11 @@
#include <jni.h>
#include <utility>
#include <string>
#include <JsiWorklet.h>
#include "RuntimeDecorator.h"
#include "RuntimeManager.h"
#include "reanimated-headers/AndroidScheduler.h"
#include "reanimated-headers/AndroidErrorHandler.h"
#include "MakeJSIRuntime.h"
#include "CameraView.h"
#include "FrameHostObject.h"
#include "JSIJNIConversion.h"
#include "VisionCameraScheduler.h"
#include "java-bindings/JImageProxy.h"
#include "java-bindings/JFrameProcessorPlugin.h"
@@ -28,6 +22,28 @@ using TSelf = local_ref<HybridClass<vision::FrameProcessorRuntimeManager>::jhybr
using TJSCallInvokerHolder = jni::alias_ref<facebook::react::CallInvokerHolder::javaobject>;
using TAndroidScheduler = jni::alias_ref<VisionCameraScheduler::javaobject>;
FrameProcessorRuntimeManager::FrameProcessorRuntimeManager(jni::alias_ref<FrameProcessorRuntimeManager::jhybridobject> jThis,
jsi::Runtime* jsRuntime,
std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker,
std::shared_ptr<vision::VisionCameraScheduler> scheduler) :
javaPart_(jni::make_global(jThis)),
_jsRuntime(jsRuntime) {
auto runOnJS = [jsCallInvoker](std::function<void()>&& f) {
// Run on React JS Runtime
jsCallInvoker->invokeAsync(std::move(f));
};
auto runOnWorklet = [scheduler](std::function<void()>&& f) {
// Run on Frame Processor Worklet Runtime
scheduler->dispatchAsync(std::move(f));
};
_workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera");
_workletContext->initialize("VisionCamera",
jsRuntime,
runOnJS,
runOnWorklet);
}
// JNI binding
void vision::FrameProcessorRuntimeManager::registerNatives() {
registerHybrid({
@@ -35,8 +51,6 @@ void vision::FrameProcessorRuntimeManager::registerNatives() {
FrameProcessorRuntimeManager::initHybrid),
makeNativeMethod("installJSIBindings",
FrameProcessorRuntimeManager::installJSIBindings),
makeNativeMethod("initializeRuntime",
FrameProcessorRuntimeManager::initializeRuntime),
makeNativeMethod("registerPlugin",
FrameProcessorRuntimeManager::registerPlugin),
});
@@ -52,32 +66,11 @@ TSelf vision::FrameProcessorRuntimeManager::initHybrid(
"Initializing FrameProcessorRuntimeManager...");
// cast from JNI hybrid objects to C++ instances
auto runtime = reinterpret_cast<jsi::Runtime*>(jsRuntimePointer);
auto jsRuntime = reinterpret_cast<jsi::Runtime*>(jsRuntimePointer);
auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker();
auto scheduler = std::shared_ptr<VisionCameraScheduler>(androidScheduler->cthis());
scheduler->setJSCallInvoker(jsCallInvoker);
return makeCxxInstance(jThis, runtime, jsCallInvoker, scheduler);
}
void vision::FrameProcessorRuntimeManager::initializeRuntime() {
__android_log_write(ANDROID_LOG_INFO, TAG,
"Initializing Vision JS-Runtime...");
// create JSI runtime and decorate it
auto runtime = makeJSIRuntime();
reanimated::RuntimeDecorator::decorateRuntime(*runtime, "FRAME_PROCESSOR");
runtime->global().setProperty(*runtime, "_FRAME_PROCESSOR",
jsi::Value(true));
// create REA runtime manager
auto errorHandler = std::make_shared<reanimated::AndroidErrorHandler>(scheduler_);
_runtimeManager = std::make_unique<reanimated::RuntimeManager>(std::move(runtime),
errorHandler,
scheduler_);
__android_log_write(ANDROID_LOG_INFO, TAG,
"Initialized Vision JS-Runtime!");
return makeCxxInstance(jThis, jsRuntime, jsCallInvoker, scheduler);
}
global_ref<CameraView::javaobject> FrameProcessorRuntimeManager::findCameraViewById(int viewId) {
@@ -87,21 +80,16 @@ global_ref<CameraView::javaobject> FrameProcessorRuntimeManager::findCameraViewB
}
void FrameProcessorRuntimeManager::logErrorToJS(const std::string& message) {
if (!this->jsCallInvoker_) {
if (!_workletContext) {
return;
}
this->jsCallInvoker_->invokeAsync([this, message]() {
if (this->runtime_ == nullptr) {
return;
}
auto& runtime = *this->runtime_;
auto consoleError = runtime
.global()
.getPropertyAsObject(runtime, "console")
.getPropertyAsFunction(runtime, "error");
consoleError.call(runtime, jsi::String::createFromUtf8(runtime, message));
// Call console.error() on JS Thread
_workletContext->invokeOnJsThread([message](jsi::Runtime& runtime) {
auto consoleError = runtime
.global()
.getPropertyAsObject(runtime, "console")
.getPropertyAsFunction(runtime, "error");
consoleError.call(runtime, jsi::String::createFromUtf8(runtime, message));
});
}
@@ -111,42 +99,38 @@ void FrameProcessorRuntimeManager::setFrameProcessor(jsi::Runtime& runtime,
__android_log_write(ANDROID_LOG_INFO, TAG,
"Setting new Frame Processor...");
if (!_runtimeManager || !_runtimeManager->runtime) {
if (!_workletContext) {
throw jsi::JSError(runtime,
"setFrameProcessor(..): VisionCamera's RuntimeManager is not yet initialized!");
"setFrameProcessor(..): VisionCamera's Worklet Context is not yet initialized!");
}
// find camera view
auto cameraView = findCameraViewById(viewTag);
__android_log_write(ANDROID_LOG_INFO, TAG, "Found CameraView!");
// convert jsi::Function to a ShareableValue (can be shared across runtimes)
__android_log_write(ANDROID_LOG_INFO, TAG,
"Adapting Shareable value from function (conversion to worklet)...");
auto worklet = reanimated::ShareableValue::adapt(runtime,
frameProcessor,
_runtimeManager.get());
// convert jsi::Function to a Worklet (can be shared across runtimes)
__android_log_write(ANDROID_LOG_INFO, TAG, "Creating Worklet...");
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, frameProcessor);
auto workletInvoker = std::make_shared<RNWorklet::WorkletInvoker>(worklet);
__android_log_write(ANDROID_LOG_INFO, TAG, "Successfully created worklet!");
scheduler_->scheduleOnUI([=]() {
// cast worklet to a jsi::Function for the new runtime
auto& rt = *_runtimeManager->runtime;
auto function = std::make_shared<jsi::Function>(worklet->getValue(rt).asObject(rt).asFunction(rt));
// assign lambda to frame processor
cameraView->cthis()->setFrameProcessor([this, &rt, function](jni::alias_ref<JImageProxy::javaobject> frame) {
try {
// create HostObject which holds the Frame (JImageProxy)
auto hostObject = std::make_shared<FrameHostObject>(frame);
function->callWithThis(rt, *function, jsi::Object::createFromHostObject(rt, hostObject));
} catch (jsi::JSError& jsError) {
auto message = "Frame Processor threw an error: " + jsError.getMessage();
__android_log_write(ANDROID_LOG_ERROR, TAG, message.c_str());
this->logErrorToJS(message);
}
});
__android_log_write(ANDROID_LOG_INFO, TAG, "Frame Processor set!");
_workletContext->invokeOnWorkletThread([=](RNWorklet::JsiWorkletContext*, jsi::Runtime& rt) {
// Set Frame Processor as callable C++ lambda - this will then call the Worklet
cameraView->cthis()->setFrameProcessor([this, workletInvoker, &rt](jni::alias_ref<JImageProxy::javaobject> frame) {
try {
// create HostObject which holds the Frame (JImageProxy)
auto frameHostObject = std::make_shared<FrameHostObject>(frame);
auto argument = jsi::Object::createFromHostObject(rt, frameHostObject);
jsi::Value jsValue(std::move(argument));
// Call the Worklet on the Worklet Runtime
workletInvoker->call(rt, jsi::Value::undefined(), &jsValue, 1);
} catch (jsi::JSError& jsError) {
// Worklet threw a JS Error, catch it and log it to JS.
auto message = "Frame Processor threw an error: " + jsError.getMessage();
__android_log_write(ANDROID_LOG_ERROR, TAG, message.c_str());
this->logErrorToJS(message);
}
});
});
}
@@ -166,13 +150,13 @@ void FrameProcessorRuntimeManager::unsetFrameProcessor(int viewTag) {
void FrameProcessorRuntimeManager::installJSIBindings() {
__android_log_write(ANDROID_LOG_INFO, TAG, "Installing JSI bindings...");
if (runtime_ == nullptr) {
if (_jsRuntime == nullptr) {
__android_log_write(ANDROID_LOG_ERROR, TAG,
"JS-Runtime was null, Frame Processor JSI bindings could not be installed!");
return;
}
auto& jsiRuntime = *runtime_;
auto& jsiRuntime = *_jsRuntime;
auto setFrameProcessor = [this](jsi::Runtime &runtime,
const jsi::Value &thisValue,
@@ -234,27 +218,26 @@ void FrameProcessorRuntimeManager::installJSIBindings() {
}
void FrameProcessorRuntimeManager::registerPlugin(alias_ref<JFrameProcessorPlugin::javaobject> plugin) {
// _runtimeManager might never be null, but we can never be too sure.
if (!_runtimeManager || !_runtimeManager->runtime) {
throw std::runtime_error("Tried to register plugin before initializing JS runtime! Call `initializeRuntime()` first.");
}
auto& runtime = *_runtimeManager->runtime;
auto& runtime = *_jsRuntime;
// we need a strong reference on the plugin, make_global does that.
auto pluginGlobal = make_global(plugin);
// name is always prefixed with two underscores (__)
auto name = "__" + pluginGlobal->getName();
auto pluginName = pluginGlobal->getName();
__android_log_print(ANDROID_LOG_INFO, TAG, "Installing Frame Processor Plugin \"%s\"...", name.c_str());
__android_log_print(ANDROID_LOG_INFO, TAG, "Installing Frame Processor Plugin \"%s\"...", pluginName.c_str());
auto callback = [pluginGlobal](jsi::Runtime& runtime,
if (!runtime.global().hasProperty(runtime, "FrameProcessorPlugins")) {
runtime.global().setProperty(runtime, "FrameProcessorPlugins", jsi::Object(runtime));
}
jsi::Object frameProcessorPlugins = runtime.global().getPropertyAsObject(runtime, "FrameProcessorPlugins");
auto function = [pluginGlobal](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
size_t count) -> jsi::Value {
// Unbox object and get typed HostObject
auto boxedHostObject = arguments[0].asObject(runtime).asHostObject(runtime);
auto frameHostObject = static_cast<FrameHostObject*>(boxedHostObject.get());
auto frameHostObject = dynamic_cast<FrameHostObject*>(boxedHostObject.get());
// parse params - we are offset by `1` because the frame is the first parameter.
auto params = JArrayClass<jobject>::newArray(count - 1);
@@ -269,10 +252,14 @@ void FrameProcessorRuntimeManager::registerPlugin(alias_ref<JFrameProcessorPlugi
return JSIJNIConversion::convertJNIObjectToJSIValue(runtime, result);
};
runtime.global().setProperty(runtime, name.c_str(), jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forAscii(runtime, name),
1, // frame
callback));
// Assign it to the Proxy.
// A FP Plugin called "example_plugin" can be now called from JS using "FrameProcessorPlugins.example_plugin(frame)"
frameProcessorPlugins.setProperty(runtime,
pluginName.c_str(),
jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forAscii(runtime, pluginName),
1, // frame
function));
}
} // namespace vision

20
android/src/main/cpp/FrameProcessorRuntimeManager.h
View File

@@ -9,9 +9,7 @@
#include <ReactCommon/CallInvokerHolder.h>
#include <memory>
#include <string>
#include "RuntimeManager.h"
#include "reanimated-headers/AndroidScheduler.h"
#include <JsiWorkletContext.h>
#include "CameraView.h"
#include "VisionCameraScheduler.h"
@@ -32,25 +30,17 @@ class FrameProcessorRuntimeManager : public jni::HybridClass<FrameProcessorRunti
static void registerNatives();
explicit FrameProcessorRuntimeManager(jni::alias_ref<FrameProcessorRuntimeManager::jhybridobject> jThis,
jsi::Runtime* runtime,
jsi::Runtime* jsRuntime,
std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker,
std::shared_ptr<vision::VisionCameraScheduler> scheduler) :
javaPart_(jni::make_global(jThis)),
runtime_(runtime),
jsCallInvoker_(jsCallInvoker),
scheduler_(scheduler)
{}
std::shared_ptr<vision::VisionCameraScheduler> scheduler);
private:
friend HybridBase;
jni::global_ref<FrameProcessorRuntimeManager::javaobject> javaPart_;
jsi::Runtime* runtime_;
std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker_;
std::shared_ptr<reanimated::RuntimeManager> _runtimeManager;
std::shared_ptr<vision::VisionCameraScheduler> scheduler_;
jsi::Runtime* _jsRuntime;
std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;
jni::global_ref<CameraView::javaobject> findCameraViewById(int viewId);
void initializeRuntime();
void installJSIBindings();
void registerPlugin(alias_ref<JFrameProcessorPlugin::javaobject> plugin);
void logErrorToJS(const std::string& message);

30
android/src/main/cpp/MakeJSIRuntime.h
View File

@@ -1,30 +0,0 @@
//
// Created by Marc Rousavy on 06.07.21.
//
#pragma once
#include <jsi/jsi.h>
#include <memory>
#if FOR_HERMES
// Hermes
#include <hermes/hermes.h>
#else
// JSC
#include <jsi/JSCRuntime.h>
#endif
namespace vision {
using namespace facebook;
static std::unique_ptr<jsi::Runtime> makeJSIRuntime() {
#if FOR_HERMES
return facebook::hermes::makeHermesRuntime();
#else
return facebook::jsc::makeJSCRuntime();
#endif
}
} // namespace vision

12
android/src/main/cpp/VisionCameraScheduler.cpp
View File

@@ -14,9 +14,9 @@ TSelf VisionCameraScheduler::initHybrid(jni::alias_ref<jhybridobject> jThis) {
return makeCxxInstance(jThis);
}
void VisionCameraScheduler::scheduleOnUI(std::function<void()> job) {
void VisionCameraScheduler::dispatchAsync(std::function<void()> job) {
// 1. add job to queue
uiJobs.push(job);
_jobs.push(job);
scheduleTrigger();
}
@@ -26,16 +26,18 @@ void VisionCameraScheduler::scheduleTrigger() {
method(javaPart_.get());
}
void VisionCameraScheduler::triggerUI() {
void VisionCameraScheduler::trigger() {
std::unique_lock<std::mutex> lock(_mutex);
// 3. call job we enqueued in step 1.
auto job = uiJobs.pop();
auto job = _jobs.front();
job();
_jobs.pop();
}
void VisionCameraScheduler::registerNatives() {
registerHybrid({
makeNativeMethod("initHybrid", VisionCameraScheduler::initHybrid),
makeNativeMethod("triggerUI", VisionCameraScheduler::triggerUI),
makeNativeMethod("trigger", VisionCameraScheduler::trigger),
});
}

22
android/src/main/cpp/VisionCameraScheduler.h
View File

@@ -4,35 +4,47 @@
#pragma once
#include "Scheduler.h"
#include <ReactCommon/CallInvokerHolder.h>
#include <jni.h>
#include <fbjni/fbjni.h>
#include <queue>
#include <mutex>
namespace vision {
using namespace facebook;
class VisionCameraScheduler : public reanimated::Scheduler, public jni::HybridClass<VisionCameraScheduler> {
/**
* A Scheduler that runs methods on the Frame Processor Thread (which is a Java Thread).
* In order to call something on the Java Frame Processor Thread, you have to:
*
* 1. Call `dispatchAsync(..)` with the given C++ Method.
* 2. Internally, `scheduleTrigger()` will get called, which is a Java Method.
* 3. The `scheduleTrigger()` Java Method will switch to the Frame Processor Java Thread and call `trigger()` on there
* 4. `trigger()` is a C++ function here that just calls the passed C++ Method from step 1.
*/
class VisionCameraScheduler : public jni::HybridClass<VisionCameraScheduler> {
public:
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraScheduler;";
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis);
static void registerNatives();
// schedules the given job to be run on the VisionCamera FP Thread at some future point in time
void scheduleOnUI(std::function<void()> job) override;
void dispatchAsync(std::function<void()> job);
private:
friend HybridBase;
jni::global_ref<VisionCameraScheduler::javaobject> javaPart_;
std::queue<std::function<void()>> _jobs;
std::mutex _mutex;
explicit VisionCameraScheduler(jni::alias_ref<VisionCameraScheduler::jhybridobject> jThis):
javaPart_(jni::make_global(jThis)) {}
// Schedules a call to `triggerUI` on the VisionCamera FP Thread
// Schedules a call to `trigger` on the VisionCamera FP Thread
void scheduleTrigger();
// Calls the latest job in the job queue
void triggerUI() override;
void trigger();
};
} // namespace vision

30
android/src/main/cpp/reanimated-headers/AndroidErrorHandler.h
View File

@@ -1,30 +0,0 @@
// copied from https://github.com/software-mansion/react-native-reanimated/blob/master/android/src/main/cpp/headers/AndroidErrorHandler.h
#pragma once
#include "ErrorHandler.h"
#include "AndroidScheduler.h"
#include "Scheduler.h"
#include <jni.h>
#include <memory>
#include <fbjni/fbjni.h>
#include "Logger.h"
namespace reanimated
{
class AndroidErrorHandler : public JavaClass<AndroidErrorHandler>, public ErrorHandler {
std::shared_ptr<ErrorWrapper> error;
std::shared_ptr<Scheduler> scheduler;
void raiseSpec() override;
public:
static auto constexpr kJavaDescriptor = "Lcom/swmansion/reanimated/AndroidErrorHandler;";
AndroidErrorHandler(
std::shared_ptr<Scheduler> scheduler);
std::shared_ptr<Scheduler> getScheduler() override;
std::shared_ptr<ErrorWrapper> getError() override;
void setError(std::string message) override;
virtual ~AndroidErrorHandler() {}
};
}

37
android/src/main/cpp/reanimated-headers/AndroidScheduler.h
View File

@@ -1,37 +0,0 @@
// copied from https://github.com/software-mansion/react-native-reanimated/blob/master/android/src/main/cpp/headers/AndroidScheduler.h
#pragma once
#include <jni.h>
#include <fbjni/fbjni.h>
#include <jsi/jsi.h>
#include <react/jni/CxxModuleWrapper.h>
#include <react/jni/JMessageQueueThread.h>
#include "Scheduler.h"
namespace reanimated {
using namespace facebook;
class AndroidScheduler : public jni::HybridClass<AndroidScheduler> {
public:
static auto constexpr kJavaDescriptor = "Lcom/swmansion/reanimated/Scheduler;";
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis);
static void registerNatives();
std::shared_ptr<Scheduler> getScheduler() { return scheduler_; }
void scheduleOnUI();
private:
friend HybridBase;
void triggerUI();
jni::global_ref<AndroidScheduler::javaobject> javaPart_;
std::shared_ptr<Scheduler> scheduler_;
explicit AndroidScheduler(jni::alias_ref<AndroidScheduler::jhybridobject> jThis);
};
}

4
android/src/main/java/com/mrousavy/camera/CameraView.kt
View File

@@ -210,9 +210,7 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
}
init {
if (FrameProcessorRuntimeManager.enableFrameProcessors) {
mHybridData = initHybrid()
}
mHybridData = initHybrid()
previewView = PreviewView(context)
previewView.layoutParams = LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT)

26
android/src/main/java/com/mrousavy/camera/CameraViewModule.kt
View File

@@ -20,9 +20,6 @@ import com.facebook.react.modules.core.PermissionAwareActivity
import com.facebook.react.modules.core.PermissionListener
import com.facebook.react.uimanager.UIManagerHelper
import com.facebook.react.bridge.ReactApplicationContext
import com.facebook.react.turbomodule.core.CallInvokerHolderImpl
import com.mrousavy.camera.CameraView
import com.mrousavy.camera.ViewNotFoundError
import java.util.concurrent.ExecutorService
import com.mrousavy.camera.frameprocessor.FrameProcessorRuntimeManager
import com.mrousavy.camera.parsers.*
@@ -55,17 +52,6 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase
if (coroutineScope.isActive) {
coroutineScope.cancel("CameraViewModule has been destroyed.")
}
frameProcessorManager = null
}
override fun initialize() {
super.initialize()
if (frameProcessorManager == null) {
frameProcessorThread.execute {
frameProcessorManager = FrameProcessorRuntimeManager(reactApplicationContext, frameProcessorThread)
}
}
}
override fun onCatalystInstanceDestroy() {
@@ -165,6 +151,18 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase
}
}
@ReactMethod(isBlockingSynchronousMethod = true)
fun installFrameProcessorBindings(): Boolean {
try {
frameProcessorManager = FrameProcessorRuntimeManager(reactApplicationContext, frameProcessorThread)
frameProcessorManager!!.installBindings()
return true
} catch (e: Error) {
Log.e(TAG, "Failed to install Frame Processor JSI Bindings!", e)
return false
}
}
// TODO: This uses the Camera2 API to list all characteristics of a camera device and therefore doesn't work with Camera1. Find a way to use CameraX for this
// https://issuetracker.google.com/issues/179925896
@ReactMethod

2
android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java
View File

@@ -48,6 +48,6 @@ public abstract class FrameProcessorPlugin {
* @param plugin An instance of a plugin.
*/
public static void register(@NonNull FrameProcessorPlugin plugin) {
FrameProcessorRuntimeManager.Companion.getPlugins().add(plugin);
FrameProcessorRuntimeManager.Companion.addPlugin(plugin);
}
}

46
android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager.kt
View File

@@ -16,18 +16,20 @@ import java.util.concurrent.ExecutorService
class FrameProcessorRuntimeManager(context: ReactApplicationContext, frameProcessorThread: ExecutorService) {
companion object {
const val TAG = "FrameProcessorRuntime"
val Plugins: ArrayList<FrameProcessorPlugin> = ArrayList()
var enableFrameProcessors = true
private val Plugins: ArrayList<FrameProcessorPlugin> = ArrayList()
init {
try {
System.loadLibrary("reanimated")
System.loadLibrary("VisionCamera")
} catch (e: UnsatisfiedLinkError) {
Log.w(TAG, "Failed to load Reanimated/VisionCamera C++ library. Frame Processors are disabled!")
enableFrameProcessors = false
Log.e(TAG, "Failed to load VisionCamera C++ library!", e)
throw e
}
}
fun addPlugin(plugin: FrameProcessorPlugin) {
Plugins.add(plugin)
}
}
@DoNotStrip
@@ -36,24 +38,11 @@ class FrameProcessorRuntimeManager(context: ReactApplicationContext, frameProces
private var mScheduler: VisionCameraScheduler? = null
init {
if (enableFrameProcessors) {
val holder = context.catalystInstance.jsCallInvokerHolder as CallInvokerHolderImpl
mScheduler = VisionCameraScheduler(frameProcessorThread)
mContext = WeakReference(context)
mHybridData = initHybrid(context.javaScriptContextHolder.get(), holder, mScheduler!!)
initializeRuntime()
Log.i(TAG, "Installing Frame Processor Plugins...")
Plugins.forEach { plugin ->
registerPlugin(plugin)
}
Log.i(TAG, "Successfully installed ${Plugins.count()} Frame Processor Plugins!")
Log.i(TAG, "Installing JSI Bindings on JS Thread...")
context.runOnJSQueueThread {
installJSIBindings()
}
}
val jsCallInvokerHolder = context.catalystInstance.jsCallInvokerHolder as CallInvokerHolderImpl
val jsRuntimeHolder = context.javaScriptContextHolder.get()
mScheduler = VisionCameraScheduler(frameProcessorThread)
mContext = WeakReference(context)
mHybridData = initHybrid(jsRuntimeHolder, jsCallInvokerHolder, mScheduler!!)
}
@Suppress("unused")
@@ -67,13 +56,22 @@ class FrameProcessorRuntimeManager(context: ReactApplicationContext, frameProces
return view ?: throw ViewNotFoundError(viewId)
}
fun installBindings() {
Log.i(TAG, "Installing JSI Bindings on JS Thread...")
installJSIBindings()
Log.i(TAG, "Installing Frame Processor Plugins...")
Plugins.forEach { plugin ->
registerPlugin(plugin)
}
Log.i(TAG, "Successfully installed ${Plugins.count()} Frame Processor Plugins!")
}
// private C++ funcs
private external fun initHybrid(
jsContext: Long,
jsCallInvokerHolder: CallInvokerHolderImpl,
scheduler: VisionCameraScheduler
): HybridData
private external fun initializeRuntime()
private external fun registerPlugin(plugin: FrameProcessorPlugin)
private external fun installJSIBindings()
}

2
android/src/main/java/com/mrousavy/camera/frameprocessor/ImageProxyUtils.java
View File

@@ -19,7 +19,7 @@ public class ImageProxyUtils {
Image image = imageProxy.getImage();
if (image == null) return false;
// will throw an exception if the image is already closed
imageProxy.getImage().getCropRect();
image.getCropRect();
// no exception thrown, image must still be valid.
return true;
} catch (Exception e) {

4
android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraScheduler.java
View File

@@ -17,11 +17,11 @@ public class VisionCameraScheduler {
}
private native HybridData initHybrid();
private native void triggerUI();
private native void trigger();
@SuppressWarnings("unused")
@DoNotStrip
private void scheduleTrigger() {
frameProcessorThread.submit(this::triggerUI);
frameProcessorThread.submit(this::trigger);
}
}