From 0f7ee51333c47fbfdf432c8608b9785f8eec3c94 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 30 Jul 2021 10:27:45 +0200 Subject: [PATCH] feat: Add `console` logging support for Frame Processors (#297) * Try to log to console via runOnJS * Call `console.log` * Create custom `VisionCameraScheduler` * Fix scheduler call * Call with this * Fix console setting * Move J---- to `java-bindings` * c++ style * Android: 1/2 Create custom Scheduler * Android: 2/2 Use custom Scheduler * Don't use `runOnJS`, use `__callAsync` directly --- android/CMakeLists.txt | 7 +-- android/src/main/cpp/CameraView.h | 2 +- android/src/main/cpp/FrameProcessorPlugin.h | 2 +- .../main/cpp/FrameProcessorRuntimeManager.cpp | 44 ++++++++++--------- .../main/cpp/FrameProcessorRuntimeManager.h | 8 ++-- android/src/main/cpp/JSIJNIConversion.cpp | 12 ++--- android/src/main/cpp/JSIJNIConversion.h | 10 +++-- android/src/main/cpp/MakeJSIRuntime.h | 4 +- android/src/main/cpp/VisionCamera.cpp | 2 + .../src/main/cpp/VisionCameraScheduler.cpp | 42 ++++++++++++++++++ android/src/main/cpp/VisionCameraScheduler.h | 38 ++++++++++++++++ .../main/cpp/{ => java-bindings}/JArrayList.h | 0 .../main/cpp/{ => java-bindings}/JHashMap.cpp | 0 .../main/cpp/{ => java-bindings}/JHashMap.h | 0 .../cpp/{ => java-bindings}/JImageProxy.cpp | 0 .../cpp/{ => java-bindings}/JImageProxy.h | 0 .../JImageProxyHostObject.cpp | 0 .../JImageProxyHostObject.h | 0 .../cpp/{ => java-bindings}/JReadableArray.h | 0 .../cpp/{ => java-bindings}/JReadableMap.h | 0 .../FrameProcessorRuntimeManager.kt | 7 ++- .../frameprocessor/VisionCameraScheduler.java | 29 ++++++++++++ .../FRAME_PROCESSORS_CREATE_OVERVIEW.mdx | 8 ++-- .../guides/FRAME_PROCESSOR_CREATE_FINAL.mdx | 2 +- example/src/CameraPage.tsx | 2 +- .../FrameProcessorRuntimeManager.mm | 9 ++-- ios/Frame Processor/FrameProcessorUtils.mm | 2 +- ios/Frame Processor/VisionCameraScheduler.h | 26 +++++++++++ ios/Frame Processor/VisionCameraScheduler.mm | 39 ++++++++++++++++ ios/VisionCamera.xcodeproj/project.pbxproj | 4 ++ src/hooks/useFrameProcessor.ts | 40 ++++++++++++++++- 31 files changed, 281 insertions(+), 58 deletions(-) create mode 100644 android/src/main/cpp/VisionCameraScheduler.cpp create mode 100644 android/src/main/cpp/VisionCameraScheduler.h rename android/src/main/cpp/{ => java-bindings}/JArrayList.h (100%) rename android/src/main/cpp/{ => java-bindings}/JHashMap.cpp (100%) rename android/src/main/cpp/{ => java-bindings}/JHashMap.h (100%) rename android/src/main/cpp/{ => java-bindings}/JImageProxy.cpp (100%) rename android/src/main/cpp/{ => java-bindings}/JImageProxy.h (100%) rename android/src/main/cpp/{ => java-bindings}/JImageProxyHostObject.cpp (100%) rename android/src/main/cpp/{ => java-bindings}/JImageProxyHostObject.h (100%) rename android/src/main/cpp/{ => java-bindings}/JReadableArray.h (100%) rename android/src/main/cpp/{ => java-bindings}/JReadableMap.h (100%) create mode 100644 android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraScheduler.java create mode 100644 ios/Frame Processor/VisionCameraScheduler.h create mode 100644 ios/Frame Processor/VisionCameraScheduler.mm diff --git a/android/CMakeLists.txt b/android/CMakeLists.txt index b45302b..98e4858 100644 --- a/android/CMakeLists.txt +++ b/android/CMakeLists.txt @@ -18,9 +18,10 @@ add_library( src/main/cpp/FrameProcessorRuntimeManager.cpp src/main/cpp/FrameProcessorPlugin.cpp src/main/cpp/CameraView.cpp - src/main/cpp/JImageProxy.cpp - src/main/cpp/JImageProxyHostObject.cpp - src/main/cpp/JHashMap.cpp + src/main/cpp/VisionCameraScheduler.cpp + src/main/cpp/java-bindings/JImageProxy.cpp + src/main/cpp/java-bindings/JImageProxyHostObject.cpp + src/main/cpp/java-bindings/JHashMap.cpp ) # includes diff --git a/android/src/main/cpp/CameraView.h b/android/src/main/cpp/CameraView.h index 8aed8d5..d97f25d 100644 --- a/android/src/main/cpp/CameraView.h +++ b/android/src/main/cpp/CameraView.h @@ -9,7 +9,7 @@ #include -#include "JImageProxy.h" +#include "java-bindings/JImageProxy.h" namespace vision { diff --git a/android/src/main/cpp/FrameProcessorPlugin.h b/android/src/main/cpp/FrameProcessorPlugin.h index 76a8db7..fe440bc 100644 --- a/android/src/main/cpp/FrameProcessorPlugin.h +++ b/android/src/main/cpp/FrameProcessorPlugin.h @@ -8,7 +8,7 @@ #include #include -#include "JImageProxy.h" +#include "java-bindings/JImageProxy.h" namespace vision { diff --git a/android/src/main/cpp/FrameProcessorRuntimeManager.cpp b/android/src/main/cpp/FrameProcessorRuntimeManager.cpp index 668cb4a..e8a3ddd 100644 --- a/android/src/main/cpp/FrameProcessorRuntimeManager.cpp +++ b/android/src/main/cpp/FrameProcessorRuntimeManager.cpp @@ -15,16 +15,17 @@ #include "MakeJSIRuntime.h" #include "CameraView.h" -#include "JImageProxy.h" -#include "JImageProxyHostObject.h" +#include "java-bindings/JImageProxy.h" +#include "java-bindings/JImageProxyHostObject.h" #include "JSIJNIConversion.h" +#include "VisionCameraScheduler.h" namespace vision { // type aliases using TSelf = local_ref::jhybriddata>; using JSCallInvokerHolder = jni::alias_ref; -using AndroidScheduler = jni::alias_ref; +using AndroidScheduler = jni::alias_ref; // JNI binding void vision::FrameProcessorRuntimeManager::registerNatives() { @@ -51,7 +52,7 @@ TSelf vision::FrameProcessorRuntimeManager::initHybrid( // cast from JNI hybrid objects to C++ instances auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker(); - auto scheduler = androidScheduler->cthis()->getScheduler(); + auto scheduler = std::shared_ptr(androidScheduler->cthis()); scheduler->setJSCallInvoker(jsCallInvoker); return makeCxxInstance(jThis, reinterpret_cast(jsContext), jsCallInvoker, scheduler); @@ -147,25 +148,28 @@ void FrameProcessorRuntimeManager::installJSIBindings() { _runtimeManager.get()); __android_log_write(ANDROID_LOG_INFO, TAG, "Successfully created worklet!"); - // cast worklet to a jsi::Function for the new runtime - auto &rt = *this->_runtimeManager->runtime; - auto function = std::make_shared(worklet->getValue(rt).asObject(rt).asFunction(rt)); - // assign lambda to frame processor - cameraView->setFrameProcessor([this, &rt, function](jni::local_ref frame) { - try { - // create HostObject which holds the Frame (JImageProxy) - auto hostObject = std::make_shared(frame); - function->call(rt, jsi::Object::createFromHostObject(rt, hostObject)); - } catch (jsi::JSError& jsError) { - auto message = "Frame Processor threw an error: " + jsError.getMessage(); - __android_log_write(ANDROID_LOG_ERROR, TAG, message.c_str()); - this->logErrorToJS(message); - } + scheduler_->scheduleOnUI([=]() { + // cast worklet to a jsi::Function for the new runtime + auto &rt = *_runtimeManager->runtime; + auto function = std::make_shared(worklet->getValue(rt).asObject(rt).asFunction(rt)); + + // assign lambda to frame processor + cameraView->setFrameProcessor([this, &rt, function](jni::local_ref frame) { + try { + // create HostObject which holds the Frame (JImageProxy) + auto hostObject = std::make_shared(frame); + function->callWithThis(rt, *function, jsi::Object::createFromHostObject(rt, hostObject)); + } catch (jsi::JSError& jsError) { + auto message = "Frame Processor threw an error: " + jsError.getMessage(); + __android_log_write(ANDROID_LOG_ERROR, TAG, message.c_str()); + this->logErrorToJS(message); + } + }); + + __android_log_write(ANDROID_LOG_INFO, TAG, "Frame Processor set!"); }); - __android_log_write(ANDROID_LOG_INFO, TAG, "Frame Processor set!"); - return jsi::Value::undefined(); }; jsiRuntime.global().setProperty(jsiRuntime, diff --git a/android/src/main/cpp/FrameProcessorRuntimeManager.h b/android/src/main/cpp/FrameProcessorRuntimeManager.h index 0b90eab..d06b852 100644 --- a/android/src/main/cpp/FrameProcessorRuntimeManager.h +++ b/android/src/main/cpp/FrameProcessorRuntimeManager.h @@ -10,12 +10,12 @@ #include #include -#include "Scheduler.h" #include "RuntimeManager.h" #include "reanimated-headers/AndroidScheduler.h" #include "CameraView.h" #include "FrameProcessorPlugin.h" +#include "VisionCameraScheduler.h" namespace vision { @@ -28,14 +28,14 @@ class FrameProcessorRuntimeManager : public jni::HybridClass initHybrid(jni::alias_ref jThis, jlong jsContext, jni::alias_ref jsCallInvokerHolder, - jni::alias_ref androidScheduler); + jni::alias_ref androidScheduler); static void registerNatives(); FrameProcessorRuntimeManager() {} explicit FrameProcessorRuntimeManager(jni::alias_ref jThis, jsi::Runtime* runtime, std::shared_ptr jsCallInvoker, - std::shared_ptr scheduler) : + std::shared_ptr scheduler) : javaPart_(jni::make_global(jThis)), runtime_(runtime), jsCallInvoker_(jsCallInvoker), @@ -48,7 +48,7 @@ class FrameProcessorRuntimeManager : public jni::HybridClass jsCallInvoker_; std::shared_ptr _runtimeManager; - std::shared_ptr scheduler_; + std::shared_ptr scheduler_; CameraView* findCameraViewById(int viewId); void initializeRuntime(); diff --git a/android/src/main/cpp/JSIJNIConversion.cpp b/android/src/main/cpp/JSIJNIConversion.cpp index 796e563..c34054a 100644 --- a/android/src/main/cpp/JSIJNIConversion.cpp +++ b/android/src/main/cpp/JSIJNIConversion.cpp @@ -19,12 +19,12 @@ #include #include -#include "JImageProxyHostObject.h" -#include "JImageProxy.h" -#include "JReadableArray.h" -#include "JReadableMap.h" -#include "JArrayList.h" -#include "JHashMap.h" +#include "java-bindings/JImageProxyHostObject.h" +#include "java-bindings/JImageProxy.h" +#include "java-bindings/JReadableArray.h" +#include "java-bindings/JReadableMap.h" +#include "java-bindings/JArrayList.h" +#include "java-bindings/JHashMap.h" namespace vision { diff --git a/android/src/main/cpp/JSIJNIConversion.h b/android/src/main/cpp/JSIJNIConversion.h index 535e062..4af3126 100644 --- a/android/src/main/cpp/JSIJNIConversion.h +++ b/android/src/main/cpp/JSIJNIConversion.h @@ -10,12 +10,14 @@ namespace vision { +namespace JSIJNIConversion { + using namespace facebook; -namespace JSIJNIConversion { - jobject convertJSIValueToJNIObject(jsi::Runtime& runtime, const jsi::Value& value); // NOLINT(runtime/references) +jobject convertJSIValueToJNIObject(jsi::Runtime& runtime, const jsi::Value& value); // NOLINT(runtime/references) - jsi::Value convertJNIObjectToJSIValue(jsi::Runtime& runtime, const jni::local_ref& object); // NOLINT(runtime/references) -}; +jsi::Value convertJNIObjectToJSIValue(jsi::Runtime& runtime, const jni::local_ref& object); // NOLINT(runtime/references) + +} // namespace JSIJNIConversion } // namespace vision diff --git a/android/src/main/cpp/MakeJSIRuntime.h b/android/src/main/cpp/MakeJSIRuntime.h index fcd3d40..39045bd 100644 --- a/android/src/main/cpp/MakeJSIRuntime.h +++ b/android/src/main/cpp/MakeJSIRuntime.h @@ -15,10 +15,10 @@ #include #endif -using namespace facebook; - namespace vision { +using namespace facebook; + static std::unique_ptr makeJSIRuntime() { #if FOR_HERMES return facebook::hermes::makeHermesRuntime(); diff --git a/android/src/main/cpp/VisionCamera.cpp b/android/src/main/cpp/VisionCamera.cpp index f5e41f8..b841854 100644 --- a/android/src/main/cpp/VisionCamera.cpp +++ b/android/src/main/cpp/VisionCamera.cpp @@ -3,11 +3,13 @@ #include "FrameProcessorRuntimeManager.h" #include "FrameProcessorPlugin.h" #include "CameraView.h" +#include "VisionCameraScheduler.h" JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) { return facebook::jni::initialize(vm, [] { vision::FrameProcessorRuntimeManager::registerNatives(); vision::FrameProcessorPlugin::registerNatives(); vision::CameraView::registerNatives(); + vision::VisionCameraScheduler::registerNatives(); }); } diff --git a/android/src/main/cpp/VisionCameraScheduler.cpp b/android/src/main/cpp/VisionCameraScheduler.cpp new file mode 100644 index 0000000..3f065a5 --- /dev/null +++ b/android/src/main/cpp/VisionCameraScheduler.cpp @@ -0,0 +1,42 @@ +// +// Created by Marc Rousavy on 25.07.21. +// + +#include "VisionCameraScheduler.h" +#include + +namespace vision { + +using namespace facebook; +using TSelf = jni::local_ref; + +TSelf VisionCameraScheduler::initHybrid(jni::alias_ref jThis) { + return makeCxxInstance(jThis); +} + +void VisionCameraScheduler::scheduleOnUI(std::function job) { + // 1. add job to queue + uiJobs.push(job); + scheduleTrigger(); +} + +void VisionCameraScheduler::scheduleTrigger() { + // 2. schedule `triggerUI` to be called on the java thread + static auto method = javaPart_->getClass()->getMethod("scheduleTrigger"); + method(javaPart_.get()); +} + +void VisionCameraScheduler::triggerUI() { + // 3. call job we enqueued in step 1. + auto job = uiJobs.pop(); + job(); +} + +void VisionCameraScheduler::registerNatives() { + registerHybrid({ + makeNativeMethod("initHybrid", VisionCameraScheduler::initHybrid), + makeNativeMethod("triggerUI", VisionCameraScheduler::triggerUI), + }); +} + +} // namespace vision \ No newline at end of file diff --git a/android/src/main/cpp/VisionCameraScheduler.h b/android/src/main/cpp/VisionCameraScheduler.h new file mode 100644 index 0000000..a05c3c9 --- /dev/null +++ b/android/src/main/cpp/VisionCameraScheduler.h @@ -0,0 +1,38 @@ +// +// Created by Marc Rousavy on 25.07.21. +// + +#pragma once + +#include "Scheduler.h" +#include +#include +#include + +namespace vision { + +using namespace facebook; + +class VisionCameraScheduler : public reanimated::Scheduler, public jni::HybridClass { +public: + static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraScheduler;"; + static jni::local_ref initHybrid(jni::alias_ref jThis); + static void registerNatives(); + + // schedules the given job to be run on the VisionCamera FP Thread at some future point in time + void scheduleOnUI(std::function job) override; + +private: + friend HybridBase; + jni::global_ref javaPart_; + + explicit VisionCameraScheduler(jni::alias_ref jThis): + javaPart_(jni::make_global(jThis)) {} + + // Schedules a call to `triggerUI` on the VisionCamera FP Thread + void scheduleTrigger(); + // Calls the latest job in the job queue + void triggerUI() override; +}; + +} // namespace vision \ No newline at end of file diff --git a/android/src/main/cpp/JArrayList.h b/android/src/main/cpp/java-bindings/JArrayList.h similarity index 100% rename from android/src/main/cpp/JArrayList.h rename to android/src/main/cpp/java-bindings/JArrayList.h diff --git a/android/src/main/cpp/JHashMap.cpp b/android/src/main/cpp/java-bindings/JHashMap.cpp similarity index 100% rename from android/src/main/cpp/JHashMap.cpp rename to android/src/main/cpp/java-bindings/JHashMap.cpp diff --git a/android/src/main/cpp/JHashMap.h b/android/src/main/cpp/java-bindings/JHashMap.h similarity index 100% rename from android/src/main/cpp/JHashMap.h rename to android/src/main/cpp/java-bindings/JHashMap.h diff --git a/android/src/main/cpp/JImageProxy.cpp b/android/src/main/cpp/java-bindings/JImageProxy.cpp similarity index 100% rename from android/src/main/cpp/JImageProxy.cpp rename to android/src/main/cpp/java-bindings/JImageProxy.cpp diff --git a/android/src/main/cpp/JImageProxy.h b/android/src/main/cpp/java-bindings/JImageProxy.h similarity index 100% rename from android/src/main/cpp/JImageProxy.h rename to android/src/main/cpp/java-bindings/JImageProxy.h diff --git a/android/src/main/cpp/JImageProxyHostObject.cpp b/android/src/main/cpp/java-bindings/JImageProxyHostObject.cpp similarity index 100% rename from android/src/main/cpp/JImageProxyHostObject.cpp rename to android/src/main/cpp/java-bindings/JImageProxyHostObject.cpp diff --git a/android/src/main/cpp/JImageProxyHostObject.h b/android/src/main/cpp/java-bindings/JImageProxyHostObject.h similarity index 100% rename from android/src/main/cpp/JImageProxyHostObject.h rename to android/src/main/cpp/java-bindings/JImageProxyHostObject.h diff --git a/android/src/main/cpp/JReadableArray.h b/android/src/main/cpp/java-bindings/JReadableArray.h similarity index 100% rename from android/src/main/cpp/JReadableArray.h rename to android/src/main/cpp/java-bindings/JReadableArray.h diff --git a/android/src/main/cpp/JReadableMap.h b/android/src/main/cpp/java-bindings/JReadableMap.h similarity index 100% rename from android/src/main/cpp/JReadableMap.h rename to android/src/main/cpp/java-bindings/JReadableMap.h diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager.kt b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager.kt index 34e0b98..b7d9e30 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager.kt +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager.kt @@ -26,11 +26,11 @@ class FrameProcessorRuntimeManager(context: ReactApplicationContext) { @DoNotStrip private var mHybridData: HybridData private var mContext: WeakReference - private var mScheduler: Scheduler + private var mScheduler: VisionCameraScheduler init { val holder = context.catalystInstance.jsCallInvokerHolder as CallInvokerHolderImpl - mScheduler = Scheduler(context) + mScheduler = VisionCameraScheduler() mContext = WeakReference(context) mHybridData = initHybrid(context.javaScriptContextHolder.get(), holder, mScheduler) initializeRuntime() @@ -43,7 +43,6 @@ class FrameProcessorRuntimeManager(context: ReactApplicationContext) { } fun destroy() { - mScheduler.deactivate() mHybridData.resetNative() } @@ -60,7 +59,7 @@ class FrameProcessorRuntimeManager(context: ReactApplicationContext) { private external fun initHybrid( jsContext: Long, jsCallInvokerHolder: CallInvokerHolderImpl, - scheduler: Scheduler + scheduler: VisionCameraScheduler ): HybridData private external fun initializeRuntime() private external fun registerPlugin(plugin: FrameProcessorPlugin) diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraScheduler.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraScheduler.java new file mode 100644 index 0000000..c88faa2 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraScheduler.java @@ -0,0 +1,29 @@ +package com.mrousavy.camera.frameprocessor; + +import com.facebook.jni.HybridData; +import com.facebook.proguard.annotations.DoNotStrip; +import com.mrousavy.camera.CameraViewModule; + +@SuppressWarnings("JavaJniMissingFunction") // using fbjni here +public class VisionCameraScheduler { + @DoNotStrip + private final HybridData mHybridData; + + public VisionCameraScheduler() { + mHybridData = initHybrid(); + } + + @Override + protected void finalize() throws Throwable { + mHybridData.resetNative(); + super.finalize(); + } + + private native HybridData initHybrid(); + private native void triggerUI(); + + @DoNotStrip + private void scheduleTrigger() { + CameraViewModule.Companion.getFrameProcessorThread().submit(this::triggerUI); + } +} diff --git a/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx b/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx index cf73a4c..ef83388 100644 --- a/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx +++ b/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx @@ -19,7 +19,7 @@ function App() { const frameProcessor = useFrameProcessor((frame) => { 'worklet' const qrCodes = scanQRCodes(frame) - _log(`QR Codes in Frame: ${qrCodes}`) + console.log(`QR Codes in Frame: ${qrCodes}`) }, []) return ( @@ -62,7 +62,7 @@ Returns a `string` in JS: export function detectObject(frame: Frame): string { 'worklet' const result = __detectObject(frame) - _log(result) // <-- "cat" + console.log(result) // <-- "cat" } ``` @@ -86,7 +86,7 @@ const frameProcessor = useFrameProcessor((frame) => { // by downscaling the frame, the `detectObjects` function runs faster. const objects = detectObjects(resizedFrame) - _log(objects) + console.log(objects) }, []) ``` @@ -133,7 +133,7 @@ const frameProcessor = useFrameProcessor((frame) => { try { const codes = scanCodes(frame, true) } catch (e) { - _log(`Error: ${e.message}`) + console.log(`Error: ${e.message}`) } }, []) ``` diff --git a/docs/docs/guides/FRAME_PROCESSOR_CREATE_FINAL.mdx b/docs/docs/guides/FRAME_PROCESSOR_CREATE_FINAL.mdx index ad69ef8..f6a4ec7 100644 --- a/docs/docs/guides/FRAME_PROCESSOR_CREATE_FINAL.mdx +++ b/docs/docs/guides/FRAME_PROCESSOR_CREATE_FINAL.mdx @@ -48,7 +48,7 @@ function App() { const frameProcessor = useFrameProcessor((frame) => { 'worklet' const qrCodes = scanQRCodes(frame) - _log(`QR Codes in Frame: ${qrCodes}`) + console.log(`QR Codes in Frame: ${qrCodes}`) }, []) return ( diff --git a/example/src/CameraPage.tsx b/example/src/CameraPage.tsx index 7fa26b6..840d2af 100644 --- a/example/src/CameraPage.tsx +++ b/example/src/CameraPage.tsx @@ -194,7 +194,7 @@ export const CameraPage: NavigationFunctionComponent = ({ componentId }) => { const frameProcessor = useFrameProcessor((frame) => { 'worklet'; const values = examplePlugin(frame); - _log(`Return Values: ${JSON.stringify(values)}`); + console.log(`Return Values: ${JSON.stringify(values)}`); }, []); return ( diff --git a/ios/Frame Processor/FrameProcessorRuntimeManager.mm b/ios/Frame Processor/FrameProcessorRuntimeManager.mm index db9bcc8..64cd5aa 100644 --- a/ios/Frame Processor/FrameProcessorRuntimeManager.mm +++ b/ios/Frame Processor/FrameProcessorRuntimeManager.mm @@ -24,7 +24,6 @@ #if __has_include() #import #import - #import #import #define ENABLE_FRAME_PROCESSORS #else @@ -37,6 +36,7 @@ #import "FrameProcessorUtils.h" #import "FrameProcessorCallback.h" +#import "VisionCameraScheduler.h" #import "../React Utils/MakeJSIRuntime.h" #import "../React Utils/JSIUtils.h" @@ -69,7 +69,7 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView"))) runtime->global().setProperty(*runtime, "_FRAME_PROCESSOR", jsi::Value(true)); auto callInvoker = bridge.jsCallInvoker; - auto scheduler = std::make_shared(callInvoker); + auto scheduler = std::make_shared(callInvoker); runtimeManager = std::make_unique(std::move(runtime), std::make_shared(scheduler), scheduler); @@ -148,13 +148,14 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView"))) auto worklet = reanimated::ShareableValue::adapt(runtime, arguments[1], runtimeManager.get()); NSLog(@"FrameProcessorBindings: Successfully created worklet!"); - RCTExecuteOnMainQueue([worklet, viewTag, self]() { + RCTExecuteOnMainQueue([=]() { auto currentBridge = [RCTBridge currentBridge]; auto anonymousView = [currentBridge.uiManager viewForReactTag:[NSNumber numberWithDouble:viewTag]]; auto view = static_cast(anonymousView); - dispatch_async(CameraQueues.frameProcessorQueue, [worklet, view, self]() { + dispatch_async(CameraQueues.frameProcessorQueue, [=]() { NSLog(@"FrameProcessorBindings: Converting worklet to Objective-C callback..."); + auto& rt = *runtimeManager->runtime; auto function = worklet->getValue(rt).asObject(rt).asFunction(rt); diff --git a/ios/Frame Processor/FrameProcessorUtils.mm b/ios/Frame Processor/FrameProcessorUtils.mm index 9f5c1ae..6f0fac8 100644 --- a/ios/Frame Processor/FrameProcessorUtils.mm +++ b/ios/Frame Processor/FrameProcessorUtils.mm @@ -25,7 +25,7 @@ FrameProcessorCallback convertJSIFunctionToFrameProcessorCallback(jsi::Runtime & auto frameHostObject = std::make_shared(frame); try { - cb.call(runtime, jsi::Object::createFromHostObject(runtime, frameHostObject)); + cb.callWithThis(runtime, cb, jsi::Object::createFromHostObject(runtime, frameHostObject)); } catch (jsi::JSError& jsError) { auto message = jsError.getMessage(); RCTBridge* bridge = [RCTBridge currentBridge]; diff --git a/ios/Frame Processor/VisionCameraScheduler.h b/ios/Frame Processor/VisionCameraScheduler.h new file mode 100644 index 0000000..c3b5336 --- /dev/null +++ b/ios/Frame Processor/VisionCameraScheduler.h @@ -0,0 +1,26 @@ +// +// VisionCameraScheduler.h +// VisionCamera +// +// Created by Marc Rousavy on 23.07.21. +// Copyright © 2021 mrousavy. All rights reserved. +// + +#pragma once + +#import +#import + +namespace vision { + +using namespace facebook; + +class VisionCameraScheduler : public reanimated::Scheduler { +public: + VisionCameraScheduler(std::shared_ptr jsInvoker); + virtual ~VisionCameraScheduler(); + + void scheduleOnUI(std::function job) override; +}; + +} // namespace vision diff --git a/ios/Frame Processor/VisionCameraScheduler.mm b/ios/Frame Processor/VisionCameraScheduler.mm new file mode 100644 index 0000000..c005477 --- /dev/null +++ b/ios/Frame Processor/VisionCameraScheduler.mm @@ -0,0 +1,39 @@ +// +// VisionCameraScheduler.mm +// VisionCamera +// +// Created by Marc Rousavy on 23.07.21. +// Copyright © 2021 mrousavy. All rights reserved. +// + +#import +#import "VisionCameraScheduler.h" + +#import + +// Forward declarations for the Swift classes +__attribute__((objc_runtime_name("_TtC12VisionCamera12CameraQueues"))) +@interface CameraQueues : NSObject +@property (nonatomic, class, readonly, strong) dispatch_queue_t _Nonnull frameProcessorQueue; +@end + +namespace vision { + +using namespace facebook; + +VisionCameraScheduler::VisionCameraScheduler(std::shared_ptr jsInvoker) { + this->jsCallInvoker_ = jsInvoker; +} + +// does not schedule on UI thread but rather on Frame Processor Thread +void VisionCameraScheduler::scheduleOnUI(std::function job) { + dispatch_async(CameraQueues.frameProcessorQueue, ^{ + job(); + }); +} + +VisionCameraScheduler::~VisionCameraScheduler(){ +} + + +} // namespace vision diff --git a/ios/VisionCamera.xcodeproj/project.pbxproj b/ios/VisionCamera.xcodeproj/project.pbxproj index 7b4d7f9..27dc0da 100644 --- a/ios/VisionCamera.xcodeproj/project.pbxproj +++ b/ios/VisionCamera.xcodeproj/project.pbxproj @@ -75,6 +75,8 @@ /* Begin PBXFileReference section */ 134814201AA4EA6300B7C361 /* libVisionCamera.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libVisionCamera.a; sourceTree = BUILT_PRODUCTS_DIR; }; + B80416F026AB16E8000DEB6A /* VisionCameraScheduler.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = VisionCameraScheduler.mm; sourceTree = ""; }; + B80416F126AB16F3000DEB6A /* VisionCameraScheduler.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = VisionCameraScheduler.h; sourceTree = ""; }; B80C0DFE260BDD97001699AB /* FrameProcessorPluginRegistry.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorPluginRegistry.h; sourceTree = ""; }; B80C0DFF260BDDF7001699AB /* FrameProcessorPluginRegistry.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = FrameProcessorPluginRegistry.mm; sourceTree = ""; }; B80D67A825FA25380008FE8D /* FrameProcessorCallback.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorCallback.h; sourceTree = ""; }; @@ -261,6 +263,8 @@ B80C0DFE260BDD97001699AB /* FrameProcessorPluginRegistry.h */, B80C0DFF260BDDF7001699AB /* FrameProcessorPluginRegistry.mm */, B88873E5263D46C7008B1D0E /* FrameProcessorPlugin.h */, + B80416F026AB16E8000DEB6A /* VisionCameraScheduler.mm */, + B80416F126AB16F3000DEB6A /* VisionCameraScheduler.h */, ); path = "Frame Processor"; sourceTree = ""; diff --git a/src/hooks/useFrameProcessor.ts b/src/hooks/useFrameProcessor.ts index 684e220..4623453 100644 --- a/src/hooks/useFrameProcessor.ts +++ b/src/hooks/useFrameProcessor.ts @@ -1,8 +1,12 @@ +/* global _setGlobalConsole */ + import { DependencyList, useCallback } from 'react'; import type { Frame } from 'src/Frame'; type FrameProcessor = (frame: Frame) => void; +const capturableConsole = console; + /** * Returns a memoized Frame Processor function wich you can pass to the ``. (See ["Frame Processors"](https://mrousavy.github.io/react-native-vision-camera/docs/guides/frame-processors)) * @@ -16,8 +20,40 @@ type FrameProcessor = (frame: Frame) => void; * const frameProcessor = useFrameProcessor((frame) => { * 'worklet' * const qrCodes = scanQRCodes(frame) - * _log(`QR Codes: ${qrCodes}`) + * console.log(`QR Codes: ${qrCodes}`) * }, []) * ``` */ -export const useFrameProcessor: (frameProcessor: FrameProcessor, dependencies: DependencyList) => FrameProcessor = useCallback; +export function useFrameProcessor(frameProcessor: FrameProcessor, dependencies: DependencyList): FrameProcessor { + return useCallback((frame: Frame) => { + 'worklet'; + + // @ts-expect-error + if (global.didSetConsole == null || global.didSetConsole === false) { + const console = { + // @ts-expect-error __callAsync is injected by native REA + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + debug: capturableConsole.debug.__callAsync, + // @ts-expect-error __callAsync is injected by native REA + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + log: capturableConsole.log.__callAsync, + // @ts-expect-error __callAsync is injected by native REA + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + warn: capturableConsole.warn.__callAsync, + // @ts-expect-error __callAsync is injected by native REA + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + error: capturableConsole.error.__callAsync, + // @ts-expect-error __callAsync is injected by native REA + // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment + info: capturableConsole.info.__callAsync, + }; + // @ts-expect-error _setGlobalConsole is set by RuntimeDecorator::decorateRuntime + _setGlobalConsole(console); + // @ts-expect-error + global.didSetConsole = true; + } + + frameProcessor(frame); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, dependencies); +}