feat: Add console logging support for Frame Processors (#297)

* Try to log to console via runOnJS

* Call `console.log`

* Create custom `VisionCameraScheduler`

* Fix scheduler call

* Call with this

* Fix console setting

* Move J---- to `java-bindings`

* c++ style

* Android: 1/2 Create custom Scheduler

* Android: 2/2 Use custom Scheduler

* Don't use `runOnJS`, use `__callAsync` directly

android/CMakeLists.txt

@@ -18,9 +18,10 @@ add_library(
         src/main/cpp/FrameProcessorRuntimeManager.cpp
         src/main/cpp/FrameProcessorPlugin.cpp
         src/main/cpp/CameraView.cpp
-        src/main/cpp/JImageProxy.cpp
-        src/main/cpp/JImageProxyHostObject.cpp
-        src/main/cpp/JHashMap.cpp
+        src/main/cpp/VisionCameraScheduler.cpp
+        src/main/cpp/java-bindings/JImageProxy.cpp
+        src/main/cpp/java-bindings/JImageProxyHostObject.cpp
+        src/main/cpp/java-bindings/JHashMap.cpp
 )
 
 # includes

android/src/main/cpp/CameraView.h

@@ -9,7 +9,7 @@
 
 #include <memory>
 
-#include "JImageProxy.h"
+#include "java-bindings/JImageProxy.h"
 
 namespace vision {
 

android/src/main/cpp/FrameProcessorPlugin.h

@@ -8,7 +8,7 @@
 #include <fbjni/fbjni.h>
 #include <string>
 
-#include "JImageProxy.h"
+#include "java-bindings/JImageProxy.h"
 
 namespace vision {
 

android/src/main/cpp/FrameProcessorRuntimeManager.cpp

@@ -15,16 +15,17 @@
 
 #include "MakeJSIRuntime.h"
 #include "CameraView.h"
-#include "JImageProxy.h"
-#include "JImageProxyHostObject.h"
+#include "java-bindings/JImageProxy.h"
+#include "java-bindings/JImageProxyHostObject.h"
 #include "JSIJNIConversion.h"
+#include "VisionCameraScheduler.h"
 
 namespace vision {
 
 // type aliases
 using TSelf = local_ref<HybridClass<vision::FrameProcessorRuntimeManager>::jhybriddata>;
 using JSCallInvokerHolder = jni::alias_ref<facebook::react::CallInvokerHolder::javaobject>;
-using AndroidScheduler = jni::alias_ref<reanimated::AndroidScheduler::javaobject>;
+using AndroidScheduler = jni::alias_ref<VisionCameraScheduler::javaobject>;
 
 // JNI binding
 void vision::FrameProcessorRuntimeManager::registerNatives() {
@@ -51,7 +52,7 @@ TSelf vision::FrameProcessorRuntimeManager::initHybrid(
 
   // cast from JNI hybrid objects to C++ instances
   auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker();
-  auto scheduler = androidScheduler->cthis()->getScheduler();
+  auto scheduler = std::shared_ptr<VisionCameraScheduler>(androidScheduler->cthis());
   scheduler->setJSCallInvoker(jsCallInvoker);
 
   return makeCxxInstance(jThis, reinterpret_cast<jsi::Runtime *>(jsContext), jsCallInvoker, scheduler);
@@ -147,25 +148,28 @@ void FrameProcessorRuntimeManager::installJSIBindings() {
                                                      _runtimeManager.get());
     __android_log_write(ANDROID_LOG_INFO, TAG, "Successfully created worklet!");
 
-    // cast worklet to a jsi::Function for the new runtime
-    auto &rt = *this->_runtimeManager->runtime;
-    auto function = std::make_shared<jsi::Function>(worklet->getValue(rt).asObject(rt).asFunction(rt));
 
-    // assign lambda to frame processor
-    cameraView->setFrameProcessor([this, &rt, function](jni::local_ref<JImageProxy::javaobject> frame) {
-      try {
-        // create HostObject which holds the Frame (JImageProxy)
-        auto hostObject = std::make_shared<JImageProxyHostObject>(frame);
-        function->call(rt, jsi::Object::createFromHostObject(rt, hostObject));
-      } catch (jsi::JSError& jsError) {
-        auto message = "Frame Processor threw an error: " + jsError.getMessage();
-        __android_log_write(ANDROID_LOG_ERROR, TAG, message.c_str());
-        this->logErrorToJS(message);
-      }
+    scheduler_->scheduleOnUI([=]() {
+      // cast worklet to a jsi::Function for the new runtime
+      auto &rt = *_runtimeManager->runtime;
+      auto function = std::make_shared<jsi::Function>(worklet->getValue(rt).asObject(rt).asFunction(rt));
+
+      // assign lambda to frame processor
+      cameraView->setFrameProcessor([this, &rt, function](jni::local_ref<JImageProxy::javaobject> frame) {
+        try {
+          // create HostObject which holds the Frame (JImageProxy)
+          auto hostObject = std::make_shared<JImageProxyHostObject>(frame);
+          function->callWithThis(rt, *function, jsi::Object::createFromHostObject(rt, hostObject));
+        } catch (jsi::JSError& jsError) {
+          auto message = "Frame Processor threw an error: " + jsError.getMessage();
+          __android_log_write(ANDROID_LOG_ERROR, TAG, message.c_str());
+          this->logErrorToJS(message);
+        }
+      });
+
+      __android_log_write(ANDROID_LOG_INFO, TAG, "Frame Processor set!");
     });
 
-    __android_log_write(ANDROID_LOG_INFO, TAG, "Frame Processor set!");
-
     return jsi::Value::undefined();
   };
   jsiRuntime.global().setProperty(jsiRuntime,

android/src/main/cpp/FrameProcessorRuntimeManager.h

@@ -10,12 +10,12 @@
 #include <memory>
 #include <string>
 
-#include "Scheduler.h"
 #include "RuntimeManager.h"
 #include "reanimated-headers/AndroidScheduler.h"
 
 #include "CameraView.h"
 #include "FrameProcessorPlugin.h"
+#include "VisionCameraScheduler.h"
 
 namespace vision {
 
@@ -28,14 +28,14 @@ class FrameProcessorRuntimeManager : public jni::HybridClass<FrameProcessorRunti
   static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis,
                                                 jlong jsContext,
                                                 jni::alias_ref<facebook::react::CallInvokerHolder::javaobject> jsCallInvokerHolder,
-                                                jni::alias_ref<reanimated::AndroidScheduler::javaobject> androidScheduler);
+                                                jni::alias_ref<vision::VisionCameraScheduler::javaobject> androidScheduler);
   static void registerNatives();
 
   FrameProcessorRuntimeManager() {}
   explicit FrameProcessorRuntimeManager(jni::alias_ref<FrameProcessorRuntimeManager::jhybridobject> jThis,
                                         jsi::Runtime* runtime,
                                         std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker,
-                                        std::shared_ptr<reanimated::Scheduler> scheduler) :
+                                        std::shared_ptr<vision::VisionCameraScheduler> scheduler) :
       javaPart_(jni::make_global(jThis)),
       runtime_(runtime),
       jsCallInvoker_(jsCallInvoker),
@@ -48,7 +48,7 @@ class FrameProcessorRuntimeManager : public jni::HybridClass<FrameProcessorRunti
   jsi::Runtime* runtime_;
   std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker_;
   std::shared_ptr<reanimated::RuntimeManager> _runtimeManager;
-  std::shared_ptr<reanimated::Scheduler> scheduler_;
+  std::shared_ptr<vision::VisionCameraScheduler> scheduler_;
 
   CameraView* findCameraViewById(int viewId);
   void initializeRuntime();

android/src/main/cpp/JSIJNIConversion.cpp

@@ -19,12 +19,12 @@
 #include <jsi/JSIDynamic.h>
 #include <folly/dynamic.h>
 
-#include "JImageProxyHostObject.h"
-#include "JImageProxy.h"
-#include "JReadableArray.h"
-#include "JReadableMap.h"
-#include "JArrayList.h"
-#include "JHashMap.h"
+#include "java-bindings/JImageProxyHostObject.h"
+#include "java-bindings/JImageProxy.h"
+#include "java-bindings/JReadableArray.h"
+#include "java-bindings/JReadableMap.h"
+#include "java-bindings/JArrayList.h"
+#include "java-bindings/JHashMap.h"
 
 namespace vision {
 

android/src/main/cpp/JSIJNIConversion.h

@@ -10,12 +10,14 @@
 
 namespace vision {
 
+namespace JSIJNIConversion {
+
 using namespace facebook;
 
-namespace JSIJNIConversion {
-  jobject convertJSIValueToJNIObject(jsi::Runtime& runtime, const jsi::Value& value); // NOLINT(runtime/references)
+jobject convertJSIValueToJNIObject(jsi::Runtime& runtime, const jsi::Value& value); // NOLINT(runtime/references)
 
-  jsi::Value convertJNIObjectToJSIValue(jsi::Runtime& runtime, const jni::local_ref<jobject>& object); // NOLINT(runtime/references)
-};
+jsi::Value convertJNIObjectToJSIValue(jsi::Runtime& runtime, const jni::local_ref<jobject>& object); // NOLINT(runtime/references)
+
+} // namespace JSIJNIConversion
 
 } // namespace vision

android/src/main/cpp/MakeJSIRuntime.h

@@ -15,10 +15,10 @@
   #include <jsi/JSCRuntime.h>
 #endif
 
-using namespace facebook;
-
 namespace vision {
 
+using namespace facebook;
+
 static std::unique_ptr<jsi::Runtime> makeJSIRuntime() {
   #if FOR_HERMES
   return facebook::hermes::makeHermesRuntime();

android/src/main/cpp/VisionCamera.cpp

@@ -3,11 +3,13 @@
 #include "FrameProcessorRuntimeManager.h"
 #include "FrameProcessorPlugin.h"
 #include "CameraView.h"
+#include "VisionCameraScheduler.h"
 
 JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) {
   return facebook::jni::initialize(vm, [] {
     vision::FrameProcessorRuntimeManager::registerNatives();
     vision::FrameProcessorPlugin::registerNatives();
     vision::CameraView::registerNatives();
+    vision::VisionCameraScheduler::registerNatives();
   });
 }

android/src/main/cpp/VisionCameraScheduler.cpp

@@ -0,0 +1,42 @@
+//
+// Created by Marc Rousavy on 25.07.21.
+//
+
+#include "VisionCameraScheduler.h"
+#include <fbjni/fbjni.h>
+
+namespace vision {
+
+using namespace facebook;
+using TSelf = jni::local_ref<VisionCameraScheduler::jhybriddata>;
+
+TSelf VisionCameraScheduler::initHybrid(jni::alias_ref<jhybridobject> jThis) {
+  return makeCxxInstance(jThis);
+}
+
+void VisionCameraScheduler::scheduleOnUI(std::function<void()> job) {
+  // 1. add job to queue
+  uiJobs.push(job);
+  scheduleTrigger();
+}
+
+void VisionCameraScheduler::scheduleTrigger() {
+  // 2. schedule `triggerUI` to be called on the java thread
+  static auto method = javaPart_->getClass()->getMethod<void()>("scheduleTrigger");
+  method(javaPart_.get());
+}
+
+void VisionCameraScheduler::triggerUI() {
+  // 3. call job we enqueued in step 1.
+  auto job = uiJobs.pop();
+  job();
+}
+
+void VisionCameraScheduler::registerNatives() {
+  registerHybrid({
+    makeNativeMethod("initHybrid", VisionCameraScheduler::initHybrid),
+    makeNativeMethod("triggerUI", VisionCameraScheduler::triggerUI),
+  });
+}
+
+} // namespace vision

android/src/main/cpp/VisionCameraScheduler.h

@@ -0,0 +1,38 @@
+//
+// Created by Marc Rousavy on 25.07.21.
+//
+
+#pragma once
+
+#include "Scheduler.h"
+#include <ReactCommon/CallInvokerHolder.h>
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+namespace vision {
+
+using namespace facebook;
+
+class VisionCameraScheduler : public reanimated::Scheduler, public jni::HybridClass<VisionCameraScheduler> {
+public:
+  static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraScheduler;";
+  static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis);
+  static void registerNatives();
+
+  // schedules the given job to be run on the VisionCamera FP Thread at some future point in time
+  void scheduleOnUI(std::function<void()> job) override;
+
+private:
+  friend HybridBase;
+  jni::global_ref<VisionCameraScheduler::javaobject> javaPart_;
+
+  explicit VisionCameraScheduler(jni::alias_ref<VisionCameraScheduler::jhybridobject> jThis):
+    javaPart_(jni::make_global(jThis)) {}
+
+  // Schedules a call to `triggerUI` on the VisionCamera FP Thread
+  void scheduleTrigger();
+  // Calls the latest job in the job queue
+  void triggerUI() override;
+};
+
+} // namespace vision

android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager.kt

@@ -26,11 +26,11 @@ class FrameProcessorRuntimeManager(context: ReactApplicationContext) {
   @DoNotStrip
   private var mHybridData: HybridData
   private var mContext: WeakReference<ReactApplicationContext>
-  private var mScheduler: Scheduler
+  private var mScheduler: VisionCameraScheduler
 
   init {
     val holder = context.catalystInstance.jsCallInvokerHolder as CallInvokerHolderImpl
-    mScheduler = Scheduler(context)
+    mScheduler = VisionCameraScheduler()
     mContext = WeakReference(context)
     mHybridData = initHybrid(context.javaScriptContextHolder.get(), holder, mScheduler)
     initializeRuntime()
@@ -43,7 +43,6 @@ class FrameProcessorRuntimeManager(context: ReactApplicationContext) {
   }
 
   fun destroy() {
-    mScheduler.deactivate()
     mHybridData.resetNative()
   }
 
@@ -60,7 +59,7 @@ class FrameProcessorRuntimeManager(context: ReactApplicationContext) {
   private external fun initHybrid(
     jsContext: Long,
     jsCallInvokerHolder: CallInvokerHolderImpl,
-    scheduler: Scheduler
+    scheduler: VisionCameraScheduler
   ): HybridData
   private external fun initializeRuntime()
   private external fun registerPlugin(plugin: FrameProcessorPlugin)

android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraScheduler.java

@@ -0,0 +1,29 @@
+package com.mrousavy.camera.frameprocessor;
+
+import com.facebook.jni.HybridData;
+import com.facebook.proguard.annotations.DoNotStrip;
+import com.mrousavy.camera.CameraViewModule;
+
+@SuppressWarnings("JavaJniMissingFunction") // using fbjni here
+public class VisionCameraScheduler {
+    @DoNotStrip
+    private final HybridData mHybridData;
+
+    public VisionCameraScheduler() {
+        mHybridData = initHybrid();
+    }
+
+    @Override
+    protected void finalize() throws Throwable {
+        mHybridData.resetNative();
+        super.finalize();
+    }
+
+    private native HybridData initHybrid();
+    private native void triggerUI();
+
+    @DoNotStrip
+    private void scheduleTrigger() {
+        CameraViewModule.Companion.getFrameProcessorThread().submit(this::triggerUI);
+    }
+}