feat: Rewrite Android C++ part (VisionCameraProxy + JFrame) (#1661)

* First Android rewrite

* Rewrite Android C++ backend

* Pass `ReadableNativeMap`, fix build error

* fix: Fix FrameProcessor init

* Make a bunch of stuff const reference to avoid copies

* Indents

* Cleanup

* indents

* docs: Update Android docs

* Update CameraView.kt

* fix: Format C++ code

android/src/main/cpp/java-bindings/JFrame.cpp

@@ -0,0 +1,65 @@
+//
+// Created by Marc on 21.07.2023.
+//
+
+#include "JFrame.h"
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+int JFrame::getWidth() const {
+  static const auto getWidthMethod = getClass()->getMethod<jint()>("getWidth");
+  return getWidthMethod(self());
+}
+
+int JFrame::getHeight() const {
+  static const auto getWidthMethod = getClass()->getMethod<jint()>("getHeight");
+  return getWidthMethod(self());
+}
+
+bool JFrame::getIsValid() const {
+  static const auto getIsValidMethod = getClass()->getMethod<jboolean()>("getIsValid");
+  return getIsValidMethod(self());
+}
+
+bool JFrame::getIsMirrored() const {
+  static const auto getIsMirroredMethod = getClass()->getMethod<jboolean()>("getIsMirrored");
+  return getIsMirroredMethod(self());
+}
+
+jlong JFrame::getTimestamp() const {
+  static const auto getTimestampMethod = getClass()->getMethod<jlong()>("getTimestamp");
+  return getTimestampMethod(self());
+}
+
+local_ref<JString> JFrame::getOrientation() const {
+  static const auto getOrientationMethod = getClass()->getMethod<JString()>("getOrientation");
+  return getOrientationMethod(self());
+}
+
+int JFrame::getPlanesCount() const {
+  static const auto getPlanesCountMethod = getClass()->getMethod<jint()>("getPlanesCount");
+  return getPlanesCountMethod(self());
+}
+
+int JFrame::getBytesPerRow() const {
+  static const auto getBytesPerRowMethod = getClass()->getMethod<jint()>("getBytesPerRow");
+  return getBytesPerRowMethod(self());
+}
+
+local_ref<JArrayByte> JFrame::toByteArray() const {
+  static const auto toByteArrayMethod = getClass()->getMethod<JArrayByte()>("toByteArray");
+  return toByteArrayMethod(self());
+}
+
+void JFrame::close() {
+  static const auto closeMethod = getClass()->getMethod<void()>("close");
+  closeMethod(self());
+}
+
+} // namespace vision

android/src/main/cpp/java-bindings/JFrame.h

@@ -1,5 +1,5 @@
 //
-// Created by Marc on 19/06/2021.
+// Created by Marc on 21.07.2023.
 //
 
 #pragma once
@@ -12,8 +12,8 @@ namespace vision {
 using namespace facebook;
 using namespace jni;
 
-struct JImageProxy : public JavaClass<JImageProxy> {
-  static constexpr auto kJavaDescriptor = "Landroidx/camera/core/ImageProxy;";
+struct JFrame : public JavaClass<JFrame> {
+  static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/Frame;";
 
  public:
   int getWidth() const;

android/src/main/cpp/java-bindings/JFrameProcessor.cpp

@@ -0,0 +1,65 @@
+//
+// Created by Marc Rousavy on 29.09.21.
+//
+
+#include "JFrameProcessor.h"
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+#include <utility>
+#include "JFrame.h"
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+void JFrameProcessor::registerNatives() {
+  registerHybrid({
+    makeNativeMethod("call", JFrameProcessor::call)
+  });
+}
+
+using TSelf = jni::local_ref<JFrameProcessor::javaobject>;
+
+JFrameProcessor::JFrameProcessor(std::shared_ptr<RNWorklet::JsiWorklet> worklet,
+                                 std::shared_ptr<RNWorklet::JsiWorkletContext> context) {
+  _workletContext = std::move(context);
+  _workletInvoker = std::make_shared<RNWorklet::WorkletInvoker>(worklet);
+}
+
+TSelf JFrameProcessor::create(const std::shared_ptr<RNWorklet::JsiWorklet>& worklet,
+                              const std::shared_ptr<RNWorklet::JsiWorkletContext>& context) {
+  return JFrameProcessor::newObjectCxxArgs(worklet, context);
+}
+
+void JFrameProcessor::callWithFrameHostObject(const std::shared_ptr<FrameHostObject>& frameHostObject) const {
+  // Call the Frame Processor on the Worklet Runtime
+  jsi::Runtime& runtime = _workletContext->getWorkletRuntime();
+
+  try {
+    // Wrap HostObject as JSI Value
+    auto argument = jsi::Object::createFromHostObject(runtime, frameHostObject);
+    jsi::Value jsValue(std::move(argument));
+
+    // Call the Worklet with the Frame JS Host Object as an argument
+    _workletInvoker->call(runtime, jsi::Value::undefined(), &jsValue, 1);
+  } catch (jsi::JSError& jsError) {
+    // JS Error occured, print it to console.
+    const std::string& message = jsError.getMessage();
+
+    _workletContext->invokeOnJsThread([message](jsi::Runtime& jsRuntime) {
+    auto logFn = jsRuntime.global().getPropertyAsObject(jsRuntime, "console").getPropertyAsFunction(jsRuntime, "error");
+    logFn.call(jsRuntime, jsi::String::createFromUtf8(jsRuntime, "Frame Processor threw an error: " + message));
+    });
+  }
+}
+
+void JFrameProcessor::call(jni::alias_ref<JFrame::javaobject> frame) {
+  // Create the Frame Host Object wrapping the internal Frame
+  auto frameHostObject = std::make_shared<FrameHostObject>(frame);
+  callWithFrameHostObject(frameHostObject);
+}
+
+} // namespace vision

android/src/main/cpp/java-bindings/JFrameProcessor.h

@@ -0,0 +1,49 @@
+//
+// Created by Marc Rousavy on 29.09.21
+//
+
+#pragma once
+
+#include <string>
+#include <memory>
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+#include <react-native-worklets/WKTJsiWorklet.h>
+#include <react-native-worklets/WKTJsiHostObject.h>
+
+#include "JFrame.h"
+#include "FrameHostObject.h"
+
+namespace vision {
+
+using namespace facebook;
+
+struct JFrameProcessor : public jni::HybridClass<JFrameProcessor> {
+ public:
+  static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessor;";
+  static void registerNatives();
+  static jni::local_ref<JFrameProcessor::javaobject> create(const std::shared_ptr<RNWorklet::JsiWorklet>& worklet,
+                                                            const std::shared_ptr<RNWorklet::JsiWorkletContext>& context);
+
+ public:
+  /**
+   * Call the JS Frame Processor.
+   */
+  void call(alias_ref<JFrame::javaobject> frame);
+
+ private:
+  // Private constructor. Use `create(..)` to create new instances.
+  explicit JFrameProcessor(std::shared_ptr<RNWorklet::JsiWorklet> worklet,
+                           std::shared_ptr<RNWorklet::JsiWorkletContext> context);
+
+ private:
+  void callWithFrameHostObject(const std::shared_ptr<FrameHostObject>& frameHostObject) const;
+
+ private:
+  friend HybridBase;
+  std::shared_ptr<RNWorklet::WorkletInvoker> _workletInvoker;
+  std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;
+};
+
+} // namespace vision

android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp

@@ -12,19 +12,14 @@ namespace vision {
 using namespace facebook;
 using namespace jni;
 
-using TCallback = jobject(alias_ref<JImageProxy::javaobject>, alias_ref<JArrayClass<jobject>>);
+using TCallback = jobject(alias_ref<JFrame::javaobject>, alias_ref<react::ReadableNativeMap::javaobject> params);
 
-local_ref<jobject> JFrameProcessorPlugin::callback(alias_ref<JImageProxy::javaobject> image,
-                                                   alias_ref<JArrayClass<jobject>> params) const {
+local_ref<jobject> JFrameProcessorPlugin::callback(const alias_ref<JFrame::javaobject>& frame,
+                                                   const alias_ref<react::ReadableNativeMap::javaobject>& params) const {
   auto callbackMethod = getClass()->getMethod<TCallback>("callback");
 
-  auto result = callbackMethod(self(), image, params);
+  auto result = callbackMethod(self(), frame, params);
   return make_local(result);
 }
 
-std::string JFrameProcessorPlugin::getName() const {
-  auto getNameMethod = getClass()->getMethod<jstring()>("getName");
-  return getNameMethod(self())->toStdString();
-}
-
 } // namespace vision

android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h

@@ -7,8 +7,9 @@
 #include <jni.h>
 #include <fbjni/fbjni.h>
 #include <string>
+#include <react/jni/ReadableNativeMap.h>
 
-#include "JImageProxy.h"
+#include "JFrame.h"
 
 namespace vision {
 
@@ -22,12 +23,8 @@ struct JFrameProcessorPlugin : public JavaClass<JFrameProcessorPlugin> {
   /**
    * Call the plugin.
    */
-  local_ref<jobject> callback(alias_ref<JImageProxy::javaobject> image,
-                              alias_ref<JArrayClass<jobject>> params) const;
-  /**
-   * Get the user-defined name of the Frame Processor Plugin
-   */
-  std::string getName() const;
+  local_ref<jobject> callback(const alias_ref<JFrame::javaobject>& frame,
+                              const alias_ref<react::ReadableNativeMap::javaobject>& params) const;
 };
 
 } // namespace vision

android/src/main/cpp/java-bindings/JImageProxy.cpp

@@ -1,78 +0,0 @@
-//
-// Created by Marc Rousavy on 22.06.21.
-//
-
-#include "JImageProxy.h"
-
-#include <jni.h>
-#include <fbjni/fbjni.h>
-
-namespace vision {
-
-using namespace facebook;
-using namespace jni;
-
-int JImageProxy::getWidth() const {
-  static const auto getWidthMethod = getClass()->getMethod<jint()>("getWidth");
-  return getWidthMethod(self());
-}
-
-int JImageProxy::getHeight() const {
-  static const auto getWidthMethod = getClass()->getMethod<jint()>("getHeight");
-  return getWidthMethod(self());
-}
-
-alias_ref<JClass> getUtilsClass() {
-  static const auto ImageProxyUtilsClass = findClassStatic("com/mrousavy/camera/frameprocessor/ImageProxyUtils");
-  return ImageProxyUtilsClass;
-}
-
-bool JImageProxy::getIsValid() const {
-  auto utilsClass = getUtilsClass();
-  static const auto isImageProxyValidMethod = utilsClass->getStaticMethod<jboolean(JImageProxy::javaobject)>("isImageProxyValid");
-  return isImageProxyValidMethod(utilsClass, self());
-}
-
-bool JImageProxy::getIsMirrored() const {
-  auto utilsClass = getUtilsClass();
-  static const auto isImageProxyMirroredMethod = utilsClass->getStaticMethod<jboolean(JImageProxy::javaobject)>("isImageProxyMirrored");
-  return isImageProxyMirroredMethod(utilsClass, self());
-}
-
-jlong JImageProxy::getTimestamp() const {
-    auto utilsClass = getUtilsClass();
-    static const auto getTimestampMethod = utilsClass->getStaticMethod<jlong(JImageProxy::javaobject)>("getTimestamp");
-    return getTimestampMethod(utilsClass, self());
-}
-
-local_ref<JString> JImageProxy::getOrientation() const {
-  auto utilsClass = getUtilsClass();
-  static const auto getOrientationMethod = utilsClass->getStaticMethod<JString(JImageProxy::javaobject)>("getOrientation");
-  return getOrientationMethod(utilsClass, self());
-}
-
-int JImageProxy::getPlanesCount() const {
-  auto utilsClass = getUtilsClass();
-  static const auto getPlanesCountMethod = utilsClass->getStaticMethod<jint(JImageProxy::javaobject)>("getPlanesCount");
-  return getPlanesCountMethod(utilsClass, self());
-}
-
-int JImageProxy::getBytesPerRow() const {
-  auto utilsClass = getUtilsClass();
-  static const auto getBytesPerRowMethod = utilsClass->getStaticMethod<jint(JImageProxy::javaobject)>("getBytesPerRow");
-  return getBytesPerRowMethod(utilsClass, self());
-}
-
-local_ref<JArrayByte> JImageProxy::toByteArray() const {
-  auto utilsClass = getUtilsClass();
-
-  static const auto toByteArrayMethod = utilsClass->getStaticMethod<JArrayByte(JImageProxy::javaobject)>("toByteArray");
-  return toByteArrayMethod(utilsClass, self());
-}
-
-void JImageProxy::close() {
-  static const auto closeMethod = getClass()->getMethod<void()>("close");
-  closeMethod(self());
-}
-
-} // namespace vision

android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp

@@ -0,0 +1,91 @@
+//
+// Created by Marc Rousavy on 21.07.23.
+//
+
+#include "JVisionCameraProxy.h"
+
+#include <memory>
+#include <utility>
+#include <string>
+
+#include <jsi/jsi.h>
+#include <react/jni/ReadableNativeMap.h>
+
+#include <react-native-worklets/WKTJsiWorklet.h>
+#include <react-native-worklets/WKTJsiWorkletContext.h>
+
+#include "FrameProcessorPluginHostObject.h"
+
+namespace vision {
+
+using TSelf = local_ref<HybridClass<JVisionCameraProxy>::jhybriddata>;
+using TJSCallInvokerHolder = jni::alias_ref<facebook::react::CallInvokerHolder::javaobject>;
+using TScheduler = jni::alias_ref<JVisionCameraScheduler::javaobject>;
+using TOptions = jni::local_ref<react::ReadableNativeMap::javaobject>;
+
+JVisionCameraProxy::JVisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::jhybridobject>& javaThis,
+                                       jsi::Runtime* runtime,
+                                       const std::shared_ptr<facebook::react::CallInvoker>& callInvoker,
+                                       const jni::global_ref<JVisionCameraScheduler::javaobject>& scheduler) {
+  _javaPart = make_global(javaThis);
+
+  __android_log_write(ANDROID_LOG_INFO, TAG, "Creating Worklet Context...");
+
+  auto runOnJS = [callInvoker](std::function<void()>&& f) {
+    // Run on React JS Runtime
+    callInvoker->invokeAsync(std::move(f));
+  };
+  auto runOnWorklet = [scheduler](std::function<void()>&& f) {
+    // Run on Frame Processor Worklet Runtime
+    scheduler->cthis()->dispatchAsync([f = std::move(f)](){
+      f();
+    });
+  };
+  _workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera",
+                                                                   runtime,
+                                                                   runOnJS,
+                                                                   runOnWorklet);
+  __android_log_write(ANDROID_LOG_INFO, TAG, "Worklet Context created!");
+}
+
+
+
+void JVisionCameraProxy::setFrameProcessor(int viewTag,
+                                           const alias_ref<JFrameProcessor::javaobject>& frameProcessor) {
+  auto setFrameProcessorMethod = javaClassLocal()->getMethod<void(int, alias_ref<JFrameProcessor::javaobject>)>("setFrameProcessor");
+  setFrameProcessorMethod(_javaPart, viewTag, frameProcessor);
+}
+
+void JVisionCameraProxy::removeFrameProcessor(int viewTag) {
+  auto removeFrameProcessorMethod = javaClassLocal()->getMethod<void(int)>("removeFrameProcessor");
+  removeFrameProcessorMethod(_javaPart, viewTag);
+}
+
+local_ref<JFrameProcessorPlugin::javaobject> JVisionCameraProxy::getFrameProcessorPlugin(const std::string& name,
+                                                                                         TOptions options) {
+  auto getFrameProcessorPluginMethod = javaClassLocal()->getMethod<JFrameProcessorPlugin(local_ref<jstring>, TOptions)>("getFrameProcessorPlugin");
+  return getFrameProcessorPluginMethod(_javaPart, make_jstring(name), std::move(options));
+}
+
+void JVisionCameraProxy::registerNatives() {
+  registerHybrid({
+    makeNativeMethod("initHybrid", JVisionCameraProxy::initHybrid)
+  });
+}
+
+TSelf JVisionCameraProxy::initHybrid(
+    alias_ref<jhybridobject> jThis,
+    jlong jsRuntimePointer,
+    TJSCallInvokerHolder jsCallInvokerHolder,
+    const TScheduler& scheduler) {
+  __android_log_write(ANDROID_LOG_INFO, TAG, "Initializing VisionCameraProxy...");
+
+  // cast from JNI hybrid objects to C++ instances
+  auto jsRuntime = reinterpret_cast<jsi::Runtime*>(jsRuntimePointer);
+  auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker();
+  auto sharedScheduler = make_global(scheduler);
+
+  return makeCxxInstance(jThis, jsRuntime, jsCallInvoker, sharedScheduler);
+}
+
+} // namespace vision

android/src/main/cpp/java-bindings/JVisionCameraProxy.h

@@ -0,0 +1,55 @@
+//
+// Created by Marc Rousavy on 21.07.23.
+//
+
+#pragma once
+
+#include <fbjni/fbjni.h>
+#include <jsi/jsi.h>
+#include <react-native-worklets/WKTJsiWorkletContext.h>
+#include <react/jni/ReadableNativeMap.h>
+
+#include "JFrameProcessorPlugin.h"
+#include "JVisionCameraScheduler.h"
+#include "JFrameProcessor.h"
+
+#include <string>
+#include <memory>
+
+namespace vision {
+
+using namespace facebook;
+
+class JVisionCameraProxy : public jni::HybridClass<JVisionCameraProxy> {
+ public:
+  static void registerNatives();
+
+  void setFrameProcessor(int viewTag,
+                         const jni::alias_ref<JFrameProcessor::javaobject>& frameProcessor);
+  void removeFrameProcessor(int viewTag);
+  jni::local_ref<JFrameProcessorPlugin::javaobject> getFrameProcessorPlugin(const std::string& name,
+                                                                            jni::local_ref<react::ReadableNativeMap::javaobject> options);
+
+ public:
+  std::shared_ptr<RNWorklet::JsiWorkletContext> getWorkletContext() { return _workletContext; }
+
+ private:
+  std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;
+
+ private:
+  friend HybridBase;
+  jni::global_ref<JVisionCameraProxy::javaobject> _javaPart;
+  static auto constexpr TAG = "VisionCameraProxy";
+  static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraProxy;";
+
+  explicit JVisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::jhybridobject>& javaThis,
+                              jsi::Runtime* jsRuntime,
+                              const std::shared_ptr<facebook::react::CallInvoker>& jsCallInvoker,
+                              const jni::global_ref<JVisionCameraScheduler::javaobject>& scheduler);
+  static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> javaThis,
+                                                jlong jsRuntimePointer,
+                                                jni::alias_ref<facebook::react::CallInvokerHolder::javaobject> jsCallInvokerHolder,
+                                                const jni::alias_ref<JVisionCameraScheduler::javaobject>& scheduler);
+};
+
+} // namespace vision

android/src/main/cpp/java-bindings/JVisionCameraScheduler.cpp

@@ -0,0 +1,44 @@
+//
+// Created by Marc Rousavy on 25.07.21.
+//
+
+#include "JVisionCameraScheduler.h"
+#include <fbjni/fbjni.h>
+
+namespace vision {
+
+using namespace facebook;
+using TSelf = jni::local_ref<JVisionCameraScheduler::jhybriddata>;
+
+TSelf JVisionCameraScheduler::initHybrid(jni::alias_ref<jhybridobject> jThis) {
+  return makeCxxInstance(jThis);
+}
+
+void JVisionCameraScheduler::dispatchAsync(const std::function<void()>& job) {
+  // 1. add job to queue
+  _jobs.push(job);
+  scheduleTrigger();
+}
+
+void JVisionCameraScheduler::scheduleTrigger() {
+  // 2. schedule `triggerUI` to be called on the java thread
+  static auto method = javaPart_->getClass()->getMethod<void()>("scheduleTrigger");
+  method(javaPart_.get());
+}
+
+void JVisionCameraScheduler::trigger() {
+  std::unique_lock<std::mutex> lock(_mutex);
+  // 3. call job we enqueued in step 1.
+  auto job = _jobs.front();
+  job();
+  _jobs.pop();
+}
+
+void JVisionCameraScheduler::registerNatives() {
+  registerHybrid({
+    makeNativeMethod("initHybrid", JVisionCameraScheduler::initHybrid),
+    makeNativeMethod("trigger", JVisionCameraScheduler::trigger),
+  });
+}
+
+} // namespace vision

android/src/main/cpp/java-bindings/JVisionCameraScheduler.h

@@ -0,0 +1,50 @@
+//
+// Created by Marc Rousavy on 25.07.21.
+//
+
+#pragma once
+
+#include <ReactCommon/CallInvokerHolder.h>
+#include <jni.h>
+#include <fbjni/fbjni.h>
+#include <queue>
+#include <mutex>
+
+namespace vision {
+
+using namespace facebook;
+
+/**
+ * A Scheduler that runs methods on the Frame Processor Thread (which is a Java Thread).
+ * In order to call something on the Java Frame Processor Thread, you have to:
+ *
+ * 1. Call `dispatchAsync(..)` with the given C++ Method.
+ * 2. Internally, `scheduleTrigger()` will get called, which is a Java Method.
+ * 3. The `scheduleTrigger()` Java Method will switch to the Frame Processor Java Thread and call `trigger()` on there
+ * 4. `trigger()` is a C++ function here that just calls the passed C++ Method from step 1.
+ */
+class JVisionCameraScheduler : public jni::HybridClass<JVisionCameraScheduler> {
+ public:
+  static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraScheduler;";
+  static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis);
+  static void registerNatives();
+
+  // schedules the given job to be run on the VisionCamera FP Thread at some future point in time
+  void dispatchAsync(const std::function<void()>& job);
+
+ private:
+  friend HybridBase;
+  jni::global_ref<JVisionCameraScheduler::javaobject> javaPart_;
+  std::queue<std::function<void()>> _jobs;
+  std::mutex _mutex;
+
+  explicit JVisionCameraScheduler(jni::alias_ref<JVisionCameraScheduler::jhybridobject> jThis):
+    javaPart_(jni::make_global(jThis)) {}
+
+  // Schedules a call to `trigger` on the VisionCamera FP Thread
+  void scheduleTrigger();
+  // Calls the latest job in the job queue
+  void trigger();
+};
+
+} // namespace vision