feat: Add console logging support for Frame Processors (#297)

* Try to log to console via runOnJS

* Call `console.log`

* Create custom `VisionCameraScheduler`

* Fix scheduler call

* Call with this

* Fix console setting

* Move J---- to `java-bindings`

* c++ style

* Android: 1/2 Create custom Scheduler

* Android: 2/2 Use custom Scheduler

* Don't use `runOnJS`, use `__callAsync` directly
This commit is contained in:
Marc Rousavy 2021-07-30 10:27:45 +02:00 committed by GitHub
parent 123d0d9e9c
commit 0f7ee51333
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
31 changed files with 281 additions and 58 deletions

View File

@ -18,9 +18,10 @@ add_library(
src/main/cpp/FrameProcessorRuntimeManager.cpp
src/main/cpp/FrameProcessorPlugin.cpp
src/main/cpp/CameraView.cpp
src/main/cpp/JImageProxy.cpp
src/main/cpp/JImageProxyHostObject.cpp
src/main/cpp/JHashMap.cpp
src/main/cpp/VisionCameraScheduler.cpp
src/main/cpp/java-bindings/JImageProxy.cpp
src/main/cpp/java-bindings/JImageProxyHostObject.cpp
src/main/cpp/java-bindings/JHashMap.cpp
)
# includes

View File

@ -9,7 +9,7 @@
#include <memory>
#include "JImageProxy.h"
#include "java-bindings/JImageProxy.h"
namespace vision {

View File

@ -8,7 +8,7 @@
#include <fbjni/fbjni.h>
#include <string>
#include "JImageProxy.h"
#include "java-bindings/JImageProxy.h"
namespace vision {

View File

@ -15,16 +15,17 @@
#include "MakeJSIRuntime.h"
#include "CameraView.h"
#include "JImageProxy.h"
#include "JImageProxyHostObject.h"
#include "java-bindings/JImageProxy.h"
#include "java-bindings/JImageProxyHostObject.h"
#include "JSIJNIConversion.h"
#include "VisionCameraScheduler.h"
namespace vision {
// type aliases
using TSelf = local_ref<HybridClass<vision::FrameProcessorRuntimeManager>::jhybriddata>;
using JSCallInvokerHolder = jni::alias_ref<facebook::react::CallInvokerHolder::javaobject>;
using AndroidScheduler = jni::alias_ref<reanimated::AndroidScheduler::javaobject>;
using AndroidScheduler = jni::alias_ref<VisionCameraScheduler::javaobject>;
// JNI binding
void vision::FrameProcessorRuntimeManager::registerNatives() {
@ -51,7 +52,7 @@ TSelf vision::FrameProcessorRuntimeManager::initHybrid(
// cast from JNI hybrid objects to C++ instances
auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker();
auto scheduler = androidScheduler->cthis()->getScheduler();
auto scheduler = std::shared_ptr<VisionCameraScheduler>(androidScheduler->cthis());
scheduler->setJSCallInvoker(jsCallInvoker);
return makeCxxInstance(jThis, reinterpret_cast<jsi::Runtime *>(jsContext), jsCallInvoker, scheduler);
@ -147,25 +148,28 @@ void FrameProcessorRuntimeManager::installJSIBindings() {
_runtimeManager.get());
__android_log_write(ANDROID_LOG_INFO, TAG, "Successfully created worklet!");
// cast worklet to a jsi::Function for the new runtime
auto &rt = *this->_runtimeManager->runtime;
auto function = std::make_shared<jsi::Function>(worklet->getValue(rt).asObject(rt).asFunction(rt));
// assign lambda to frame processor
cameraView->setFrameProcessor([this, &rt, function](jni::local_ref<JImageProxy::javaobject> frame) {
try {
// create HostObject which holds the Frame (JImageProxy)
auto hostObject = std::make_shared<JImageProxyHostObject>(frame);
function->call(rt, jsi::Object::createFromHostObject(rt, hostObject));
} catch (jsi::JSError& jsError) {
auto message = "Frame Processor threw an error: " + jsError.getMessage();
__android_log_write(ANDROID_LOG_ERROR, TAG, message.c_str());
this->logErrorToJS(message);
}
scheduler_->scheduleOnUI([=]() {
// cast worklet to a jsi::Function for the new runtime
auto &rt = *_runtimeManager->runtime;
auto function = std::make_shared<jsi::Function>(worklet->getValue(rt).asObject(rt).asFunction(rt));
// assign lambda to frame processor
cameraView->setFrameProcessor([this, &rt, function](jni::local_ref<JImageProxy::javaobject> frame) {
try {
// create HostObject which holds the Frame (JImageProxy)
auto hostObject = std::make_shared<JImageProxyHostObject>(frame);
function->callWithThis(rt, *function, jsi::Object::createFromHostObject(rt, hostObject));
} catch (jsi::JSError& jsError) {
auto message = "Frame Processor threw an error: " + jsError.getMessage();
__android_log_write(ANDROID_LOG_ERROR, TAG, message.c_str());
this->logErrorToJS(message);
}
});
__android_log_write(ANDROID_LOG_INFO, TAG, "Frame Processor set!");
});
__android_log_write(ANDROID_LOG_INFO, TAG, "Frame Processor set!");
return jsi::Value::undefined();
};
jsiRuntime.global().setProperty(jsiRuntime,

View File

@ -10,12 +10,12 @@
#include <memory>
#include <string>
#include "Scheduler.h"
#include "RuntimeManager.h"
#include "reanimated-headers/AndroidScheduler.h"
#include "CameraView.h"
#include "FrameProcessorPlugin.h"
#include "VisionCameraScheduler.h"
namespace vision {
@ -28,14 +28,14 @@ class FrameProcessorRuntimeManager : public jni::HybridClass<FrameProcessorRunti
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis,
jlong jsContext,
jni::alias_ref<facebook::react::CallInvokerHolder::javaobject> jsCallInvokerHolder,
jni::alias_ref<reanimated::AndroidScheduler::javaobject> androidScheduler);
jni::alias_ref<vision::VisionCameraScheduler::javaobject> androidScheduler);
static void registerNatives();
FrameProcessorRuntimeManager() {}
explicit FrameProcessorRuntimeManager(jni::alias_ref<FrameProcessorRuntimeManager::jhybridobject> jThis,
jsi::Runtime* runtime,
std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker,
std::shared_ptr<reanimated::Scheduler> scheduler) :
std::shared_ptr<vision::VisionCameraScheduler> scheduler) :
javaPart_(jni::make_global(jThis)),
runtime_(runtime),
jsCallInvoker_(jsCallInvoker),
@ -48,7 +48,7 @@ class FrameProcessorRuntimeManager : public jni::HybridClass<FrameProcessorRunti
jsi::Runtime* runtime_;
std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker_;
std::shared_ptr<reanimated::RuntimeManager> _runtimeManager;
std::shared_ptr<reanimated::Scheduler> scheduler_;
std::shared_ptr<vision::VisionCameraScheduler> scheduler_;
CameraView* findCameraViewById(int viewId);
void initializeRuntime();

View File

@ -19,12 +19,12 @@
#include <jsi/JSIDynamic.h>
#include <folly/dynamic.h>
#include "JImageProxyHostObject.h"
#include "JImageProxy.h"
#include "JReadableArray.h"
#include "JReadableMap.h"
#include "JArrayList.h"
#include "JHashMap.h"
#include "java-bindings/JImageProxyHostObject.h"
#include "java-bindings/JImageProxy.h"
#include "java-bindings/JReadableArray.h"
#include "java-bindings/JReadableMap.h"
#include "java-bindings/JArrayList.h"
#include "java-bindings/JHashMap.h"
namespace vision {

View File

@ -10,12 +10,14 @@
namespace vision {
namespace JSIJNIConversion {
using namespace facebook;
namespace JSIJNIConversion {
jobject convertJSIValueToJNIObject(jsi::Runtime& runtime, const jsi::Value& value); // NOLINT(runtime/references)
jobject convertJSIValueToJNIObject(jsi::Runtime& runtime, const jsi::Value& value); // NOLINT(runtime/references)
jsi::Value convertJNIObjectToJSIValue(jsi::Runtime& runtime, const jni::local_ref<jobject>& object); // NOLINT(runtime/references)
};
jsi::Value convertJNIObjectToJSIValue(jsi::Runtime& runtime, const jni::local_ref<jobject>& object); // NOLINT(runtime/references)
} // namespace JSIJNIConversion
} // namespace vision

View File

@ -15,10 +15,10 @@
#include <jsi/JSCRuntime.h>
#endif
using namespace facebook;
namespace vision {
using namespace facebook;
static std::unique_ptr<jsi::Runtime> makeJSIRuntime() {
#if FOR_HERMES
return facebook::hermes::makeHermesRuntime();

View File

@ -3,11 +3,13 @@
#include "FrameProcessorRuntimeManager.h"
#include "FrameProcessorPlugin.h"
#include "CameraView.h"
#include "VisionCameraScheduler.h"
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) {
return facebook::jni::initialize(vm, [] {
vision::FrameProcessorRuntimeManager::registerNatives();
vision::FrameProcessorPlugin::registerNatives();
vision::CameraView::registerNatives();
vision::VisionCameraScheduler::registerNatives();
});
}

View File

@ -0,0 +1,42 @@
//
// Created by Marc Rousavy on 25.07.21.
//
#include "VisionCameraScheduler.h"
#include <fbjni/fbjni.h>
namespace vision {
using namespace facebook;
using TSelf = jni::local_ref<VisionCameraScheduler::jhybriddata>;
TSelf VisionCameraScheduler::initHybrid(jni::alias_ref<jhybridobject> jThis) {
return makeCxxInstance(jThis);
}
void VisionCameraScheduler::scheduleOnUI(std::function<void()> job) {
// 1. add job to queue
uiJobs.push(job);
scheduleTrigger();
}
void VisionCameraScheduler::scheduleTrigger() {
// 2. schedule `triggerUI` to be called on the java thread
static auto method = javaPart_->getClass()->getMethod<void()>("scheduleTrigger");
method(javaPart_.get());
}
void VisionCameraScheduler::triggerUI() {
// 3. call job we enqueued in step 1.
auto job = uiJobs.pop();
job();
}
void VisionCameraScheduler::registerNatives() {
registerHybrid({
makeNativeMethod("initHybrid", VisionCameraScheduler::initHybrid),
makeNativeMethod("triggerUI", VisionCameraScheduler::triggerUI),
});
}
} // namespace vision

View File

@ -0,0 +1,38 @@
//
// Created by Marc Rousavy on 25.07.21.
//
#pragma once
#include "Scheduler.h"
#include <ReactCommon/CallInvokerHolder.h>
#include <jni.h>
#include <fbjni/fbjni.h>
namespace vision {
using namespace facebook;
class VisionCameraScheduler : public reanimated::Scheduler, public jni::HybridClass<VisionCameraScheduler> {
public:
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraScheduler;";
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis);
static void registerNatives();
// schedules the given job to be run on the VisionCamera FP Thread at some future point in time
void scheduleOnUI(std::function<void()> job) override;
private:
friend HybridBase;
jni::global_ref<VisionCameraScheduler::javaobject> javaPart_;
explicit VisionCameraScheduler(jni::alias_ref<VisionCameraScheduler::jhybridobject> jThis):
javaPart_(jni::make_global(jThis)) {}
// Schedules a call to `triggerUI` on the VisionCamera FP Thread
void scheduleTrigger();
// Calls the latest job in the job queue
void triggerUI() override;
};
} // namespace vision

View File

@ -26,11 +26,11 @@ class FrameProcessorRuntimeManager(context: ReactApplicationContext) {
@DoNotStrip
private var mHybridData: HybridData
private var mContext: WeakReference<ReactApplicationContext>
private var mScheduler: Scheduler
private var mScheduler: VisionCameraScheduler
init {
val holder = context.catalystInstance.jsCallInvokerHolder as CallInvokerHolderImpl
mScheduler = Scheduler(context)
mScheduler = VisionCameraScheduler()
mContext = WeakReference(context)
mHybridData = initHybrid(context.javaScriptContextHolder.get(), holder, mScheduler)
initializeRuntime()
@ -43,7 +43,6 @@ class FrameProcessorRuntimeManager(context: ReactApplicationContext) {
}
fun destroy() {
mScheduler.deactivate()
mHybridData.resetNative()
}
@ -60,7 +59,7 @@ class FrameProcessorRuntimeManager(context: ReactApplicationContext) {
private external fun initHybrid(
jsContext: Long,
jsCallInvokerHolder: CallInvokerHolderImpl,
scheduler: Scheduler
scheduler: VisionCameraScheduler
): HybridData
private external fun initializeRuntime()
private external fun registerPlugin(plugin: FrameProcessorPlugin)

View File

@ -0,0 +1,29 @@
package com.mrousavy.camera.frameprocessor;
import com.facebook.jni.HybridData;
import com.facebook.proguard.annotations.DoNotStrip;
import com.mrousavy.camera.CameraViewModule;
@SuppressWarnings("JavaJniMissingFunction") // using fbjni here
public class VisionCameraScheduler {
@DoNotStrip
private final HybridData mHybridData;
public VisionCameraScheduler() {
mHybridData = initHybrid();
}
@Override
protected void finalize() throws Throwable {
mHybridData.resetNative();
super.finalize();
}
private native HybridData initHybrid();
private native void triggerUI();
@DoNotStrip
private void scheduleTrigger() {
CameraViewModule.Companion.getFrameProcessorThread().submit(this::triggerUI);
}
}

View File

@ -19,7 +19,7 @@ function App() {
const frameProcessor = useFrameProcessor((frame) => {
'worklet'
const qrCodes = scanQRCodes(frame)
_log(`QR Codes in Frame: ${qrCodes}`)
console.log(`QR Codes in Frame: ${qrCodes}`)
}, [])
return (
@ -62,7 +62,7 @@ Returns a `string` in JS:
export function detectObject(frame: Frame): string {
'worklet'
const result = __detectObject(frame)
_log(result) // <-- "cat"
console.log(result) // <-- "cat"
}
```
@ -86,7 +86,7 @@ const frameProcessor = useFrameProcessor((frame) => {
// by downscaling the frame, the `detectObjects` function runs faster.
const objects = detectObjects(resizedFrame)
_log(objects)
console.log(objects)
}, [])
```
@ -133,7 +133,7 @@ const frameProcessor = useFrameProcessor((frame) => {
try {
const codes = scanCodes(frame, true)
} catch (e) {
_log(`Error: ${e.message}`)
console.log(`Error: ${e.message}`)
}
}, [])
```

View File

@ -48,7 +48,7 @@ function App() {
const frameProcessor = useFrameProcessor((frame) => {
'worklet'
const qrCodes = scanQRCodes(frame)
_log(`QR Codes in Frame: ${qrCodes}`)
console.log(`QR Codes in Frame: ${qrCodes}`)
}, [])
return (

View File

@ -194,7 +194,7 @@ export const CameraPage: NavigationFunctionComponent = ({ componentId }) => {
const frameProcessor = useFrameProcessor((frame) => {
'worklet';
const values = examplePlugin(frame);
_log(`Return Values: ${JSON.stringify(values)}`);
console.log(`Return Values: ${JSON.stringify(values)}`);
}, []);
return (

View File

@ -24,7 +24,6 @@
#if __has_include(<RNReanimated/RuntimeManager.h>)
#import <RNReanimated/RuntimeManager.h>
#import <RNReanimated/RuntimeDecorator.h>
#import <RNReanimated/REAIOSScheduler.h>
#import <RNReanimated/REAIOSErrorHandler.h>
#define ENABLE_FRAME_PROCESSORS
#else
@ -37,6 +36,7 @@
#import "FrameProcessorUtils.h"
#import "FrameProcessorCallback.h"
#import "VisionCameraScheduler.h"
#import "../React Utils/MakeJSIRuntime.h"
#import "../React Utils/JSIUtils.h"
@ -69,7 +69,7 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
runtime->global().setProperty(*runtime, "_FRAME_PROCESSOR", jsi::Value(true));
auto callInvoker = bridge.jsCallInvoker;
auto scheduler = std::make_shared<reanimated::REAIOSScheduler>(callInvoker);
auto scheduler = std::make_shared<vision::VisionCameraScheduler>(callInvoker);
runtimeManager = std::make_unique<reanimated::RuntimeManager>(std::move(runtime),
std::make_shared<reanimated::REAIOSErrorHandler>(scheduler),
scheduler);
@ -148,13 +148,14 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
auto worklet = reanimated::ShareableValue::adapt(runtime, arguments[1], runtimeManager.get());
NSLog(@"FrameProcessorBindings: Successfully created worklet!");
RCTExecuteOnMainQueue([worklet, viewTag, self]() {
RCTExecuteOnMainQueue([=]() {
auto currentBridge = [RCTBridge currentBridge];
auto anonymousView = [currentBridge.uiManager viewForReactTag:[NSNumber numberWithDouble:viewTag]];
auto view = static_cast<CameraView*>(anonymousView);
dispatch_async(CameraQueues.frameProcessorQueue, [worklet, view, self]() {
dispatch_async(CameraQueues.frameProcessorQueue, [=]() {
NSLog(@"FrameProcessorBindings: Converting worklet to Objective-C callback...");
auto& rt = *runtimeManager->runtime;
auto function = worklet->getValue(rt).asObject(rt).asFunction(rt);

View File

@ -25,7 +25,7 @@ FrameProcessorCallback convertJSIFunctionToFrameProcessorCallback(jsi::Runtime &
auto frameHostObject = std::make_shared<FrameHostObject>(frame);
try {
cb.call(runtime, jsi::Object::createFromHostObject(runtime, frameHostObject));
cb.callWithThis(runtime, cb, jsi::Object::createFromHostObject(runtime, frameHostObject));
} catch (jsi::JSError& jsError) {
auto message = jsError.getMessage();
RCTBridge* bridge = [RCTBridge currentBridge];

View File

@ -0,0 +1,26 @@
//
// VisionCameraScheduler.h
// VisionCamera
//
// Created by Marc Rousavy on 23.07.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
#pragma once
#import <RNReanimated/Scheduler.h>
#import <React-callinvoker/ReactCommon/CallInvoker.h>
namespace vision {
using namespace facebook;
class VisionCameraScheduler : public reanimated::Scheduler {
public:
VisionCameraScheduler(std::shared_ptr<react::CallInvoker> jsInvoker);
virtual ~VisionCameraScheduler();
void scheduleOnUI(std::function<void()> job) override;
};
} // namespace vision

View File

@ -0,0 +1,39 @@
//
// VisionCameraScheduler.mm
// VisionCamera
//
// Created by Marc Rousavy on 23.07.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
#import <Foundation/Foundation.h>
#import "VisionCameraScheduler.h"
#import <React-callinvoker/ReactCommon/CallInvoker.h>
// Forward declarations for the Swift classes
__attribute__((objc_runtime_name("_TtC12VisionCamera12CameraQueues")))
@interface CameraQueues : NSObject
@property (nonatomic, class, readonly, strong) dispatch_queue_t _Nonnull frameProcessorQueue;
@end
namespace vision {
using namespace facebook;
VisionCameraScheduler::VisionCameraScheduler(std::shared_ptr<react::CallInvoker> jsInvoker) {
this->jsCallInvoker_ = jsInvoker;
}
// does not schedule on UI thread but rather on Frame Processor Thread
void VisionCameraScheduler::scheduleOnUI(std::function<void()> job) {
dispatch_async(CameraQueues.frameProcessorQueue, ^{
job();
});
}
VisionCameraScheduler::~VisionCameraScheduler(){
}
} // namespace vision

View File

@ -75,6 +75,8 @@
/* Begin PBXFileReference section */
134814201AA4EA6300B7C361 /* libVisionCamera.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libVisionCamera.a; sourceTree = BUILT_PRODUCTS_DIR; };
B80416F026AB16E8000DEB6A /* VisionCameraScheduler.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = VisionCameraScheduler.mm; sourceTree = "<group>"; };
B80416F126AB16F3000DEB6A /* VisionCameraScheduler.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = VisionCameraScheduler.h; sourceTree = "<group>"; };
B80C0DFE260BDD97001699AB /* FrameProcessorPluginRegistry.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorPluginRegistry.h; sourceTree = "<group>"; };
B80C0DFF260BDDF7001699AB /* FrameProcessorPluginRegistry.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = FrameProcessorPluginRegistry.mm; sourceTree = "<group>"; };
B80D67A825FA25380008FE8D /* FrameProcessorCallback.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorCallback.h; sourceTree = "<group>"; };
@ -261,6 +263,8 @@
B80C0DFE260BDD97001699AB /* FrameProcessorPluginRegistry.h */,
B80C0DFF260BDDF7001699AB /* FrameProcessorPluginRegistry.mm */,
B88873E5263D46C7008B1D0E /* FrameProcessorPlugin.h */,
B80416F026AB16E8000DEB6A /* VisionCameraScheduler.mm */,
B80416F126AB16F3000DEB6A /* VisionCameraScheduler.h */,
);
path = "Frame Processor";
sourceTree = "<group>";

View File

@ -1,8 +1,12 @@
/* global _setGlobalConsole */
import { DependencyList, useCallback } from 'react';
import type { Frame } from 'src/Frame';
type FrameProcessor = (frame: Frame) => void;
const capturableConsole = console;
/**
* Returns a memoized Frame Processor function wich you can pass to the `<Camera>`. (See ["Frame Processors"](https://mrousavy.github.io/react-native-vision-camera/docs/guides/frame-processors))
*
@ -16,8 +20,40 @@ type FrameProcessor = (frame: Frame) => void;
* const frameProcessor = useFrameProcessor((frame) => {
* 'worklet'
* const qrCodes = scanQRCodes(frame)
* _log(`QR Codes: ${qrCodes}`)
* console.log(`QR Codes: ${qrCodes}`)
* }, [])
* ```
*/
export const useFrameProcessor: (frameProcessor: FrameProcessor, dependencies: DependencyList) => FrameProcessor = useCallback;
export function useFrameProcessor(frameProcessor: FrameProcessor, dependencies: DependencyList): FrameProcessor {
return useCallback((frame: Frame) => {
'worklet';
// @ts-expect-error
if (global.didSetConsole == null || global.didSetConsole === false) {
const console = {
// @ts-expect-error __callAsync is injected by native REA
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
debug: capturableConsole.debug.__callAsync,
// @ts-expect-error __callAsync is injected by native REA
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
log: capturableConsole.log.__callAsync,
// @ts-expect-error __callAsync is injected by native REA
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
warn: capturableConsole.warn.__callAsync,
// @ts-expect-error __callAsync is injected by native REA
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
error: capturableConsole.error.__callAsync,
// @ts-expect-error __callAsync is injected by native REA
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
info: capturableConsole.info.__callAsync,
};
// @ts-expect-error _setGlobalConsole is set by RuntimeDecorator::decorateRuntime
_setGlobalConsole(console);
// @ts-expect-error
global.didSetConsole = true;
}
frameProcessor(frame);
// eslint-disable-next-line react-hooks/exhaustive-deps
}, dependencies);
}