feat: Frame Processors for Android (#196)

* Create android gradle build setup * Fix `prefab` config * Add `pickFirst **/*.so` to example build.gradle * fix REA path * cache gradle builds * Update validate-android.yml * Create Native Proxy * Copy REA header * implement ctor * Rename CameraViewModule -> FrameProcessorRuntimeManager * init FrameProcessorRuntimeManager * fix name * Update FrameProcessorRuntimeManager.h * format * Create AndroidErrorHandler.h * Initialize runtime and install JSI funcs * Update FrameProcessorRuntimeManager.cpp * Update CameraViewModule.kt * Make CameraView hybrid C++ class to find view & set frame processor * Update FrameProcessorRuntimeManager.cpp * pass function by rvalue * pass by const && * extract hermes and JSC REA * pass `FOR_HERMES` * correctly prepare JSC and Hermes * Update CMakeLists.txt * add missing hermes include * clean up imports * Create JImageProxy.h * pass ImageProxy to JNI as `jobject` * try use `JImageProxy` C++ wrapper type * Use `local_ref<JImageProxy>` * Create `JImageProxyHostObject` for JSI interop * debug call to frame processor * Unset frame processor * Fix CameraView native part not being registered * close image * use `jobject` instead of `JImageProxy` for now :( * fix hermes build error * Set enable FP callback * fix JNI call * Update CameraView.cpp * Get Format * Create plugin abstract * Make `FrameProcessorPlugin` a hybrid object * Register plugin CXX * Call `registerPlugin` * Catch * remove JSI * Create sample QR code plugin * register plugins * Fix missing JNI binding * Add `mHybridData` * prefix name with two underscores (`__`) * Update CameraPage.tsx * wrap `ImageProxy` in host object * Use `jobject` for HO box * Update JImageProxy.h * reinterpret jobject * Try using `JImageProxy` instead of `jobject` * Update JImageProxy.h * get bytes per row and plane count * Update CameraView.cpp * Return base * add some docs and JNI JSI conversion * indent * Convert JSI value to JNI jobject * using namespace facebook * Try using class * Use plain old Object[] * Try convert JNI -> JSI * fix decl * fix bool init * Correctly link folly * Update CMakeLists.txt * Convert Map to Object * Use folly for Map and Array * Return `alias_ref<jobject>` instead of raw `jobject` * fix JNI <-> JSI conversion * Update JSIJNIConversion.cpp * Log parameters * fix params index offset * add more test cases * Update FRAME_PROCESSORS_CREATE_OVERVIEW.mdx * fix types * Rename to example plugin * remove support for hashmap * Try use HashMap iterable fbjni binding * try using JReadableArray/JReadableMap * Fix list return values * Update JSIJNIConversion.cpp * Update JSIJNIConversion.cpp * (iOS) Rename ObjC QR Code Plugin to Example Plugin * Rename Swift plugin QR -> Example * Update ExamplePluginSwift.swift * Fix Map/Dictionary logging format * Update ExampleFrameProcessorPlugin.m * Reconfigure session if frame processor changed * Handle use-cases via `maxUseCasesCount` * Don't crash app on `configureSession` error * Document "use-cases" * Update DEVICES.mdx * fix merge * Make `const &` * iOS: Automatically enable `video` if a `frameProcessor` is set * Update CameraView.cpp * fix docs * Automatically fallback to snapshot capture if `supportsParallelVideoProcessing` is false. * Fix lookup * Update CameraView.kt * Implement `frameProcessorFps` * Finalize Frame Processor Plugin Hybrid * Update CameraViewModule.kt * Support `flash` on `takeSnapshot()` * Update docs * Add docs * Update CameraPage.tsx * Attribute NonNull * remove unused imports * Add Android docs for Frame Processors * Make JNI HashMap <-> JSI Object conversion faster directly access `toHashMap` instead of going through java * add todo * Always run `prepareJSC` and `prepareHermes` * switch jsc and hermes * Specify ndkVersion `21.4.7075529` * Update gradle.properties * Update gradle.properties * Create .aar * Correctly prepare android package * Update package.json * Update package.json * remove `prefab` build feature * split * Add docs for registering the FP plugin * Add step for dep * Update CaptureButton.tsx * Move to `reanimated-headers/` * Exclude reanimated-headers from cpplint * disable `build/include_order` rule * cpplint fixes * perf: Make `JSIJNIConversion` a `namespace` instead of `class` * Ignore runtime/references for `convert` funcs * Build Android .aar in CI * Run android build script only on `prepack` * Update package.json * Update package.json * Update build-android-npm-package.sh * Move to `yarn build` * Also install node_modules in example step * Update validate-android.yml * sort imports * fix torch * Run ImageAnalysis on `FrameProcessorThread` * Update Errors.kt * Add clean android script * Upgrade reanimated to 2.3.0-alpha.1 * Revert "Upgrade reanimated to 2.3.0-alpha.1" This reverts commit c1d3bed5e03728d0b5e335a359524ff4f56f5035. * ⚠️ TEMP FIX: hotfix reanimated build.gradle * Update CameraView+TakeSnapshot.kt * ⚠️ TEMP FIX: Disable ktlint action for now * Update clean.sh * Set max heap size to 4g * rebuild lockfiles * Update Podfile.lock * rename * Build lib .aar before example/
2021-06-27 12:37:54 +02:00
parent a2311c02ac
commit 87e6bb710e
80 changed files with 4115 additions and 1770 deletions
--- a/android/src/main/cpp/CameraView.cpp
+++ b/android/src/main/cpp/CameraView.cpp
@@ -0,0 +1,69 @@
+//
+// Created by Marc Rousavy on 14.06.21.
+//
+
+#include "CameraView.h"
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+#include <memory>
+#include <string>
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+using TSelf = local_ref<HybridClass<vision::CameraView>::jhybriddata>;
+
+TSelf CameraView::initHybrid(alias_ref<HybridClass::jhybridobject> jThis) {
+    return makeCxxInstance(jThis);
+}
+
+void CameraView::registerNatives() {
+    registerHybrid({
+        makeNativeMethod("initHybrid", CameraView::initHybrid),
+        makeNativeMethod("frameProcessorCallback", CameraView::frameProcessorCallback),
+    });
+}
+
+void CameraView::frameProcessorCallback(const alias_ref<JImageProxy::javaobject>& frame) {
+  if (frameProcessor_ == nullptr) {
+    __android_log_write(ANDROID_LOG_WARN, TAG, "Frame Processor is null!");
+    setEnableFrameProcessor(false);
+    return;
+  }
+
+  auto frameStrong = make_local(frame);
+  try {
+    __android_log_write(ANDROID_LOG_INFO, TAG, "Calling Frame Processor...");
+    frameProcessor_(frameStrong);
+  } catch (const std::exception& exception) {
+    // TODO: jsi::JSErrors cannot be caught on Hermes. They crash the entire app.
+    auto message = "Frame Processor threw an error! " + std::string(exception.what());
+    __android_log_write(ANDROID_LOG_ERROR, TAG, message.c_str());
+  }
+}
+
+void CameraView::setEnableFrameProcessor(bool enable) {
+  if (enable) {
+    __android_log_write(ANDROID_LOG_INFO, TAG, "Enabling Frame Processor Callback...");
+  } else {
+    __android_log_write(ANDROID_LOG_INFO, TAG, "Disabling Frame Processor Callback...");
+  }
+  static const auto javaMethod = javaPart_->getClass()->getMethod<void(bool)>("setEnableFrameProcessor");
+  javaMethod(javaPart_.get(), enable);
+}
+
+void CameraView::setFrameProcessor(const FrameProcessor&& frameProcessor) {
+  frameProcessor_ = frameProcessor;
+  setEnableFrameProcessor(true);
+}
+
+void vision::CameraView::unsetFrameProcessor() {
+  frameProcessor_ = nullptr;
+  setEnableFrameProcessor(false);
+}
+
+} // namespace vision
--- a/android/src/main/cpp/CameraView.h
+++ b/android/src/main/cpp/CameraView.h
@@ -0,0 +1,44 @@
+//
+// Created by Marc Rousavy on 14.06.21.
+//
+
+#pragma once
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+#include <memory>
+
+#include "JImageProxy.h"
+
+namespace vision {
+
+using namespace facebook;
+using FrameProcessor = std::function<void(jni::local_ref<JImageProxy::javaobject>)>;
+
+class CameraView : public jni::HybridClass<CameraView> {
+ public:
+  static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/CameraView;";
+  static auto constexpr TAG = "VisionCamera";
+  static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis);
+  static void registerNatives();
+
+  // TODO: Use template<> to avoid heap allocation for std::function<>
+  void setFrameProcessor(const FrameProcessor&& frameProcessor);
+  void unsetFrameProcessor();
+  void setEnableFrameProcessor(bool enable);
+
+ private:
+  friend HybridBase;
+  jni::global_ref<CameraView::javaobject> javaPart_;
+  FrameProcessor frameProcessor_;
+
+  void frameProcessorCallback(const jni::alias_ref<JImageProxy::javaobject>& frame);
+
+  explicit CameraView(jni::alias_ref<CameraView::jhybridobject> jThis) :
+    javaPart_(jni::make_global(jThis)),
+    frameProcessor_(nullptr)
+  {}
+};
+
+} // namespace vision
--- a/android/src/main/cpp/FrameProcessorPlugin.cpp
+++ b/android/src/main/cpp/FrameProcessorPlugin.cpp
@@ -0,0 +1,37 @@
+//
+// Created by Marc Rousavy on 22.06.21.
+//
+
+#include "FrameProcessorPlugin.h"
+#include <string>
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+using TSelf = local_ref<HybridClass<FrameProcessorPlugin>::jhybriddata>;
+using TFrameProcessorPlugin = jobject(alias_ref<JImageProxy::javaobject>, alias_ref<JArrayClass<jobject>>);
+
+TSelf vision::FrameProcessorPlugin::initHybrid(alias_ref<HybridClass::jhybridobject> jThis, const std::string& name) {
+  return makeCxxInstance(jThis, name);
+}
+
+void FrameProcessorPlugin::registerNatives() {
+  registerHybrid({
+    makeNativeMethod("initHybrid",
+                     FrameProcessorPlugin::initHybrid),
+  });
+}
+
+local_ref<jobject> FrameProcessorPlugin::callback(alias_ref<JImageProxy::javaobject> image, alias_ref<JArrayClass<jobject>> params) {
+  static const auto func = javaPart_->getClass()->getMethod<TFrameProcessorPlugin>("callback");
+  auto result = func(javaPart_.get(), image, params);
+  return make_local(result);
+}
+
+std::string FrameProcessorPlugin::getName() {
+  return name;
+}
+
+} // namespace vision
--- a/android/src/main/cpp/FrameProcessorPlugin.h
+++ b/android/src/main/cpp/FrameProcessorPlugin.h
@@ -0,0 +1,39 @@
+//
+// Created by Marc Rousavy on 22.06.21.
+//
+
+#pragma once
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+#include <string>
+
+#include "JImageProxy.h"
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+class FrameProcessorPlugin: public HybridClass<FrameProcessorPlugin> {
+ public:
+  static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessorPlugin;";
+  static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis,
+                                                const std::string& name);
+  static void registerNatives();
+
+  local_ref<jobject> callback(alias_ref<JImageProxy::javaobject> image, alias_ref<JArrayClass<jobject>> params);
+  std::string getName();
+
+ private:
+  friend HybridBase;
+  jni::global_ref<FrameProcessorPlugin::javaobject> javaPart_;
+  std::string name;
+
+  FrameProcessorPlugin(alias_ref<FrameProcessorPlugin::jhybridobject> jThis,
+                       std::string name): javaPart_(make_global(jThis)),
+                       name(name)
+  {}
+};
+
+} // namespace vision
--- a/android/src/main/cpp/FrameProcessorRuntimeManager.cpp
+++ b/android/src/main/cpp/FrameProcessorRuntimeManager.cpp
@@ -0,0 +1,234 @@
+//
+// Created by Marc Rousavy on 11.06.21.
+//
+
+#include "FrameProcessorRuntimeManager.h"
+#include <android/log.h>
+#include <jni.h>
+#include <utility>
+
+#include "RuntimeDecorator.h"
+#include "RuntimeManager.h"
+#include "reanimated-headers/AndroidScheduler.h"
+#include "reanimated-headers/AndroidErrorHandler.h"
+
+#include "MakeJSIRuntime.h"
+#include "CameraView.h"
+#include "JImageProxy.h"
+#include "JImageProxyHostObject.h"
+#include "JSIJNIConversion.h"
+
+namespace vision {
+
+// type aliases
+using TSelf = local_ref<HybridClass<vision::FrameProcessorRuntimeManager>::jhybriddata>;
+using JSCallInvokerHolder = jni::alias_ref<facebook::react::CallInvokerHolder::javaobject>;
+using AndroidScheduler = jni::alias_ref<reanimated::AndroidScheduler::javaobject>;
+
+// JNI binding
+void vision::FrameProcessorRuntimeManager::registerNatives() {
+  registerHybrid({
+    makeNativeMethod("initHybrid",
+                     FrameProcessorRuntimeManager::initHybrid),
+    makeNativeMethod("installJSIBindings",
+                     FrameProcessorRuntimeManager::installJSIBindings),
+    makeNativeMethod("initializeRuntime",
+                     FrameProcessorRuntimeManager::initializeRuntime),
+    makeNativeMethod("registerPlugin",
+                     FrameProcessorRuntimeManager::registerPlugin),
+  });
+}
+
+// JNI init
+TSelf vision::FrameProcessorRuntimeManager::initHybrid(
+    alias_ref<jhybridobject> jThis,
+    jlong jsContext,
+    JSCallInvokerHolder jsCallInvokerHolder,
+    AndroidScheduler androidScheduler) {
+  __android_log_write(ANDROID_LOG_INFO, TAG,
+                      "Initializing FrameProcessorRuntimeManager...");
+
+  // cast from JNI hybrid objects to C++ instances
+  auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker();
+  auto scheduler = androidScheduler->cthis()->getScheduler();
+  scheduler->setJSCallInvoker(jsCallInvoker);
+
+  return makeCxxInstance(jThis, reinterpret_cast<jsi::Runtime *>(jsContext), jsCallInvoker, scheduler);
+}
+
+void vision::FrameProcessorRuntimeManager::initializeRuntime() {
+  __android_log_write(ANDROID_LOG_INFO, TAG,
+                      "Initializing Vision JS-Runtime...");
+
+  // create JSI runtime and decorate it
+  auto runtime = makeJSIRuntime();
+  reanimated::RuntimeDecorator::decorateRuntime(*runtime, "FRAME_PROCESSOR");
+  runtime->global().setProperty(*runtime, "_FRAME_PROCESSOR",
+                                jsi::Value(true));
+
+  // create REA runtime manager
+  auto errorHandler = std::make_shared<reanimated::AndroidErrorHandler>(scheduler_);
+  _runtimeManager = std::make_unique<reanimated::RuntimeManager>(std::move(runtime),
+                                                                 errorHandler,
+                                                                 scheduler_);
+
+  __android_log_write(ANDROID_LOG_INFO, TAG,
+                      "Initialized Vision JS-Runtime!");
+}
+
+CameraView* FrameProcessorRuntimeManager::findCameraViewById(int viewId) {
+  static const auto func = javaPart_->getClass()->getMethod<CameraView*(jint)>("findCameraViewById");
+  auto result = func(javaPart_.get(), viewId);
+  return result->cthis();
+}
+
+// actual JSI installer
+void FrameProcessorRuntimeManager::installJSIBindings() {
+  __android_log_write(ANDROID_LOG_INFO, TAG, "Installing JSI bindings...");
+
+  if (runtime_ == nullptr) {
+    __android_log_write(ANDROID_LOG_ERROR, TAG,
+                        "JS-Runtime was null, Frame Processor JSI bindings could not be installed!");
+    return;
+  }
+
+  auto &jsiRuntime = *runtime_;
+
+  auto setFrameProcessor = [this](jsi::Runtime &runtime,
+                                  const jsi::Value &thisValue,
+                                  const jsi::Value *arguments,
+                                  size_t count) -> jsi::Value {
+    __android_log_write(ANDROID_LOG_INFO, TAG,
+                        "Setting new Frame Processor...");
+
+    if (!arguments[0].isNumber()) {
+      throw jsi::JSError(runtime,
+                         "Camera::setFrameProcessor: First argument ('viewTag') must be a number!");
+    }
+    if (!arguments[1].isObject()) {
+      throw jsi::JSError(runtime,
+                         "Camera::setFrameProcessor: Second argument ('frameProcessor') must be a function!");
+    }
+    if (!_runtimeManager || !_runtimeManager->runtime) {
+      throw jsi::JSError(runtime,
+                         "Camera::setFrameProcessor: The RuntimeManager is not yet initialized!");
+    }
+
+    // find camera view
+    auto viewTag = arguments[0].asNumber();
+    auto cameraView = findCameraViewById(static_cast<int>(viewTag));
+    __android_log_write(ANDROID_LOG_INFO, TAG, "Found CameraView!");
+
+    // TODO: does this have to be called on the separate VisionCamera Frame Processor Thread?
+
+    // convert jsi::Function to a ShareableValue (can be shared across runtimes)
+    __android_log_write(ANDROID_LOG_INFO, TAG, "Adapting Shareable value from function (conversion to worklet)...");
+    auto worklet = reanimated::ShareableValue::adapt(runtime, arguments[1],
+                                                     _runtimeManager.get());
+    __android_log_write(ANDROID_LOG_INFO, TAG, "Successfully created worklet!");
+
+    // cast worklet to a jsi::Function for the new runtime
+    auto &rt = *this->_runtimeManager->runtime;
+    auto function = std::make_shared<jsi::Function>(worklet->getValue(rt).asObject(rt).asFunction(rt));
+
+    // assign lambda to frame processor
+    cameraView->setFrameProcessor([&rt, function](jni::local_ref<JImageProxy::javaobject> frame) {
+      __android_log_write(ANDROID_LOG_INFO, TAG, "Frame Processor called!");
+
+      // create HostObject which holds the Frame (JImageProxy)
+      auto hostObject = std::make_shared<JImageProxyHostObject>(frame);
+      function->call(rt, jsi::Object::createFromHostObject(rt, hostObject));
+    });
+
+    __android_log_write(ANDROID_LOG_INFO, TAG, "Frame Processor set!");
+
+    return jsi::Value::undefined();
+  };
+  jsiRuntime.global().setProperty(jsiRuntime,
+                                  "setFrameProcessor",
+                                  jsi::Function::createFromHostFunction(
+                                      jsiRuntime,
+                                      jsi::PropNameID::forAscii(jsiRuntime,
+                                                                "setFrameProcessor"),
+                                      2,  // viewTag, frameProcessor
+                                      setFrameProcessor));
+
+
+  auto unsetFrameProcessor = [this](jsi::Runtime &runtime,
+                                    const jsi::Value &thisValue,
+                                    const jsi::Value *arguments,
+                                    size_t count) -> jsi::Value {
+    __android_log_write(ANDROID_LOG_INFO, TAG, "Removing Frame Processor...");
+    if (!arguments[0].isNumber()) {
+      throw jsi::JSError(runtime,
+                         "Camera::unsetFrameProcessor: First argument ('viewTag') must be a number!");
+    }
+
+    // find camera view
+    auto viewTag = arguments[0].asNumber();
+    auto cameraView = findCameraViewById(static_cast<int>(viewTag));
+
+    // call Java method to unset frame processor
+    cameraView->unsetFrameProcessor();
+
+    __android_log_write(ANDROID_LOG_INFO, TAG, "Frame Processor removed!");
+
+    return jsi::Value::undefined();
+  };
+  jsiRuntime.global().setProperty(jsiRuntime,
+                                  "unsetFrameProcessor",
+                                  jsi::Function::createFromHostFunction(
+                                      jsiRuntime,
+                                      jsi::PropNameID::forAscii(jsiRuntime,
+                                                                "unsetFrameProcessor"),
+                                      1, // viewTag
+                                      unsetFrameProcessor));
+
+  __android_log_write(ANDROID_LOG_INFO, TAG, "Finished installing JSI bindings!");
+}
+
+void FrameProcessorRuntimeManager::registerPlugin(alias_ref<FrameProcessorPlugin::javaobject> plugin) {
+  // _runtimeManager might never be null, but we can never be too sure.
+  if (!_runtimeManager || !_runtimeManager->runtime) {
+    throw std::runtime_error("Tried to register plugin before initializing JS runtime! Call `initializeRuntime()` first.");
+  }
+
+  auto& runtime = *_runtimeManager->runtime;
+
+  // we need a strong reference on the plugin, make_global does that.
+  auto pluginGlobal = make_global(plugin);
+  auto pluginCxx = pluginGlobal->cthis();
+  // name is always prefixed with two underscores (__)
+  auto name = "__" + pluginCxx->getName();
+
+  auto message = "Installing Frame Processor Plugin \"" + name + "\"...";
+  __android_log_write(ANDROID_LOG_INFO, TAG, message.c_str());
+
+  auto callback = [pluginCxx](jsi::Runtime& runtime,
+                              const jsi::Value& thisValue,
+                              const jsi::Value* arguments,
+                              size_t count) -> jsi::Value {
+    // Unbox object and get typed HostObject
+    auto boxedHostObject = arguments[0].asObject(runtime).asHostObject(runtime);
+    auto frameHostObject = dynamic_cast<JImageProxyHostObject*>(boxedHostObject.get());
+
+    // parse params - we are offset by `1` because the frame is the first parameter.
+    auto params = JArrayClass<jobject>::newArray(count - 1);
+    for (size_t i = 1; i < count; i++) {
+      params->setElement(i - 1, JSIJNIConversion::convertJSIValueToJNIObject(runtime, arguments[i]));
+    }
+
+    // call implemented virtual method
+    auto result = pluginCxx->callback(frameHostObject->frame, params);
+
+    // convert result from JNI to JSI value
+    return JSIJNIConversion::convertJNIObjectToJSIValue(runtime, result);
+  };
+
+  runtime.global().setProperty(runtime, name.c_str(), jsi::Function::createFromHostFunction(runtime,
+                                                                                            jsi::PropNameID::forAscii(runtime, name),
+                                                                                            1, // frame
+                                                                                            callback));
+}
+
+} // namespace vision
--- a/android/src/main/cpp/FrameProcessorRuntimeManager.h
+++ b/android/src/main/cpp/FrameProcessorRuntimeManager.h
@@ -0,0 +1,58 @@
+//
+// Created by Marc Rousavy on 11.06.21.
+//
+
+#pragma once
+
+#include <fbjni/fbjni.h>
+#include <jsi/jsi.h>
+#include <ReactCommon/CallInvokerHolder.h>
+#include <memory>
+
+#include "Scheduler.h"
+#include "RuntimeManager.h"
+#include "reanimated-headers/AndroidScheduler.h"
+
+#include "CameraView.h"
+#include "FrameProcessorPlugin.h"
+
+namespace vision {
+
+using namespace facebook;
+
+class FrameProcessorRuntimeManager : public jni::HybridClass<FrameProcessorRuntimeManager> {
+ public:
+  static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager;";
+  static auto constexpr TAG = "VisionCamera";
+  static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis,
+                                                jlong jsContext,
+                                                jni::alias_ref<facebook::react::CallInvokerHolder::javaobject> jsCallInvokerHolder,
+                                                jni::alias_ref<reanimated::AndroidScheduler::javaobject> androidScheduler);
+  static void registerNatives();
+
+  FrameProcessorRuntimeManager() {}
+  explicit FrameProcessorRuntimeManager(jni::alias_ref<FrameProcessorRuntimeManager::jhybridobject> jThis,
+                                        jsi::Runtime* runtime,
+                                        std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker,
+                                        std::shared_ptr<reanimated::Scheduler> scheduler) :
+      javaPart_(jni::make_global(jThis)),
+      runtime_(runtime),
+      jsCallInvoker_(jsCallInvoker),
+      scheduler_(scheduler)
+  {}
+
+ private:
+  friend HybridBase;
+  jni::global_ref<FrameProcessorRuntimeManager::javaobject> javaPart_;
+  jsi::Runtime* runtime_;
+  std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker_;
+  std::shared_ptr<reanimated::RuntimeManager> _runtimeManager;
+  std::shared_ptr<reanimated::Scheduler> scheduler_;
+
+  CameraView* findCameraViewById(int viewId);
+  void initializeRuntime();
+  void installJSIBindings();
+  void registerPlugin(alias_ref<FrameProcessorPlugin::javaobject> plugin);
+};
+
+} // namespace vision
--- a/android/src/main/cpp/JArrayList.h
+++ b/android/src/main/cpp/JArrayList.h
@@ -0,0 +1,20 @@
+//
+// Created by Marc Rousavy on 24.06.21.
+//
+
+#pragma once
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+// TODO: Remove when fbjni 0.2.3 releases.
+struct JArrayList : public JavaClass<JArrayList> {
+  static constexpr auto kJavaDescriptor = "Ljava/util/ArrayList;";
+};
+
+} // namespace vision
--- a/android/src/main/cpp/JHashMap.cpp
+++ b/android/src/main/cpp/JHashMap.cpp
@@ -0,0 +1,20 @@
+//
+// Created by Marc Rousavy on 25.06.21.
+//
+
+#include "JHashMap.h"
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+
+namespace facebook {
+namespace jni {
+
+template <typename K, typename V>
+local_ref<JHashMap<K, V>> JHashMap<K, V>::create() {
+  return JHashMap<K, V>::newInstance();
+}
+
+} // namespace jni
+} // namespace facebook
--- a/android/src/main/cpp/JHashMap.h
+++ b/android/src/main/cpp/JHashMap.h
@@ -0,0 +1,23 @@
+//
+// Created by Marc Rousavy on 25.06.21.
+//
+
+#pragma once
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+
+namespace facebook {
+namespace jni {
+
+// TODO: Remove when fbjni 0.2.3 releases.
+template <typename K = jobject, typename V = jobject>
+struct JHashMap : JavaClass<JHashMap<K, V>, JMap<K, V>> {
+  constexpr static auto kJavaDescriptor = "Ljava/util/HashMap;";
+
+  static local_ref<JHashMap<K, V>> create();
+};
+
+} // namespace jni
+} // namespace facebook
--- a/android/src/main/cpp/JImageProxy.cpp
+++ b/android/src/main/cpp/JImageProxy.cpp
@@ -0,0 +1,54 @@
+//
+// Created by Marc Rousavy on 22.06.21.
+//
+
+#include "JImageProxy.h"
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+int JImageProxy::getWidth() {
+  static const auto getWidthMethod = getClass()->getMethod<jint()>("getWidth");
+  return getWidthMethod(javaClassLocal());
+}
+
+int JImageProxy::getHeight() {
+  static const auto getWidthMethod = getClass()->getMethod<jint()>("getHeight");
+  return getWidthMethod(javaClassLocal());
+}
+
+bool JImageProxy::getIsValid() {
+  static const auto getImageMethod = getClass()->getMethod<JImageProxy::javaobject()>("getImage");
+  auto image = getImageMethod(javaClassLocal());
+
+  static const auto getHardwareBufferMethod = findClassLocal("android/media/Image")->getMethod<jobject()>("getHardwareBuffer");
+  try {
+    getHardwareBufferMethod(image.get());
+    return true;
+  } catch (...) {
+    // function throws if the image is not active anymore
+    return false;
+  }
+}
+
+int JImageProxy::getPlaneCount() {
+  static const auto getPlanesMethod = getClass()->getMethod<JArrayClass<jobject>()>("getPlanes");
+  auto planes = getPlanesMethod(javaClassLocal());
+  return planes->size();
+}
+
+int JImageProxy::getBytesPerRow() {
+  static const auto getPlanesMethod = getClass()->getMethod<JArrayClass<jobject>()>("getPlanes");
+  auto planes = getPlanesMethod(javaClassLocal());
+  auto firstPlane = planes->getElement(0);
+
+  static const auto getRowStrideMethod = findClassLocal("android/media/Image$PlaneProxy")->getMethod<int()>("getRowStride");
+  return getRowStrideMethod(firstPlane.get());
+}
+
+} // namespace vision
--- a/android/src/main/cpp/JImageProxy.h
+++ b/android/src/main/cpp/JImageProxy.h
@@ -0,0 +1,23 @@
+//
+// Created by Marc on 19/06/2021.
+//
+
+#pragma once
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+namespace vision {
+
+struct JImageProxy : public facebook::jni::JavaClass<JImageProxy> {
+  static constexpr auto kJavaDescriptor = "Landroidx/camera/core/ImageProxy;";
+
+ public:
+  int getWidth();
+  int getHeight();
+  bool getIsValid();
+  int getPlaneCount();
+  int getBytesPerRow();
+};
+
+} // namespace vision
--- a/android/src/main/cpp/JImageProxyHostObject.cpp
+++ b/android/src/main/cpp/JImageProxyHostObject.cpp
@@ -0,0 +1,65 @@
+//
+// Created by Marc on 19/06/2021.
+//
+
+#include "JImageProxyHostObject.h"
+#include <android/log.h>
+#include <vector>
+#include <string>
+
+namespace vision {
+
+std::vector<jsi::PropNameID> JImageProxyHostObject::getPropertyNames(jsi::Runtime& rt) {
+  std::vector<jsi::PropNameID> result;
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toString")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isValid")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isReady")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("width")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("height")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("bytesPerRow")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("planesCount")));
+  return result;
+}
+
+jsi::Value JImageProxyHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propNameId) {
+  auto name = propNameId.utf8(runtime);
+  __android_log_write(ANDROID_LOG_INFO, TAG, ("Getting prop \"" + name + "\"...").c_str());
+
+  if (name == "toString") {
+    auto toString = [this] (jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
+      auto width = this->frame->getWidth();
+      auto height = this->frame->getHeight();
+      auto str = std::to_string(width) + " x " + std::to_string(height) + " Frame";
+      return jsi::String::createFromUtf8(runtime, str);
+    };
+    return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
+  }
+
+  if (name == "isValid") {
+    return jsi::Value(this->frame->getIsValid());
+  }
+  if (name == "isReady") {
+    return jsi::Value(this->frame->getIsValid());
+  }
+  if (name == "width") {
+    return jsi::Value(this->frame->getWidth());
+  }
+  if (name == "height") {
+    return jsi::Value(this->frame->getHeight());
+  }
+  if (name == "bytesPerRow") {
+    return jsi::Value(this->frame->getBytesPerRow());
+  }
+  if (name == "planesCount") {
+    return jsi::Value(this->frame->getPlaneCount());
+  }
+
+  return jsi::Value::undefined();
+}
+
+
+JImageProxyHostObject::~JImageProxyHostObject() {
+  __android_log_write(ANDROID_LOG_INFO, TAG, "Destroying JImageProxyHostObject...");
+}
+
+} // namespace vision
--- a/android/src/main/cpp/JImageProxyHostObject.h
+++ b/android/src/main/cpp/JImageProxyHostObject.h
@@ -0,0 +1,35 @@
+//
+// Created by Marc on 19/06/2021.
+//
+
+#pragma once
+
+#include <jsi/jsi.h>
+#include <jni.h>
+#include <fbjni/fbjni.h>
+#include <vector>
+
+#include "JImageProxy.h"
+
+namespace vision {
+
+using namespace facebook;
+
+class JSI_EXPORT JImageProxyHostObject : public jsi::HostObject {
+ public:
+  explicit JImageProxyHostObject(jni::local_ref<JImageProxy::javaobject> image): frame(image) {}
+  ~JImageProxyHostObject();
+
+ public:
+  jsi::Value get(jsi::Runtime &, const jsi::PropNameID &name) override;
+
+  std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime &rt) override;
+
+ public:
+  jni::local_ref<JImageProxy> frame;
+
+ private:
+  static auto constexpr TAG = "VisionCamera";
+};
+
+} // namespace vision
--- a/android/src/main/cpp/JReadableArray.h
+++ b/android/src/main/cpp/JReadableArray.h
@@ -0,0 +1,19 @@
+//
+// Created by Marc Rousavy on 24.06.21.
+//
+
+#pragma once
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+struct JReadableArray : public JavaClass<JReadableArray> {
+  static constexpr auto kJavaDescriptor = "Lcom/facebook/react/bridge/ReadableArray;";
+};
+
+} // namespace vision
--- a/android/src/main/cpp/JReadableMap.h
+++ b/android/src/main/cpp/JReadableMap.h
@@ -0,0 +1,19 @@
+//
+// Created by Marc Rousavy on 24.06.21.
+//
+
+#pragma once
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+struct JReadableMap : public JavaClass<JReadableMap> {
+  static constexpr auto kJavaDescriptor = "Lcom/facebook/react/bridge/ReadableMap;";
+};
+
+} // namespace vision
--- a/android/src/main/cpp/JSIJNIConversion.cpp
+++ b/android/src/main/cpp/JSIJNIConversion.cpp
@@ -0,0 +1,186 @@
+//
+// Created by Marc Rousavy on 22.06.21.
+//
+
+#include "JSIJNIConversion.h"
+
+#include <jsi/jsi.h>
+#include <jni.h>
+#include <fbjni/fbjni.h>
+#include <android/log.h>
+
+#include <string>
+#include <utility>
+
+#include <react/jni/NativeMap.h>
+#include <react/jni/ReadableNativeMap.h>
+#include <react/jni/WritableNativeMap.h>
+
+#include <jsi/JSIDynamic.h>
+#include <folly/dynamic.h>
+
+#include "JImageProxyHostObject.h"
+#include "JImageProxy.h"
+#include "JReadableArray.h"
+#include "JReadableMap.h"
+#include "JArrayList.h"
+#include "JHashMap.h"
+
+namespace vision {
+
+using namespace facebook;
+
+jobject JSIJNIConversion::convertJSIValueToJNIObject(jsi::Runtime &runtime, const jsi::Value &value) {
+  if (value.isBool()) {
+    // jsi::Bool
+
+    auto boolean = jni::JBoolean::valueOf(value.getBool());
+    return boolean.release();
+
+  } else if (value.isNumber()) {
+    // jsi::Number
+
+    auto number = jni::JDouble::valueOf(value.getNumber());
+    return number.release();
+
+  } else if (value.isNull() || value.isUndefined()) {
+    // jsi::undefined
+
+    return nullptr;
+
+  } else if (value.isString()) {
+    // jsi::String
+
+    auto string = jni::make_jstring(value.getString(runtime).utf8(runtime));
+    return string.release();
+
+  } else if (value.isObject()) {
+    // jsi::Object
+
+    auto object = value.asObject(runtime);
+
+    if (object.isArray(runtime)) {
+      // jsi::Array
+
+      auto dynamic = jsi::dynamicFromValue(runtime, value);
+      auto nativeArray = react::ReadableNativeArray::newObjectCxxArgs(std::move(dynamic));
+      return nativeArray.release();
+
+    } else if (object.isHostObject(runtime)) {
+      // jsi::HostObject
+
+      auto boxedHostObject = object.getHostObject(runtime);
+      auto hostObject = dynamic_cast<JImageProxyHostObject*>(boxedHostObject.get());
+      if (hostObject != nullptr) {
+        // return jni local_ref to the JImageProxy
+        return hostObject->frame.get();
+      } else {
+        // it's different kind of HostObject. We don't support it.
+        return nullptr;
+      }
+
+    } else if (object.isFunction(runtime)) {
+      // jsi::Function
+
+      // TODO: Convert Function to Callback
+      return nullptr;
+
+    } else {
+      // jsi::Object
+
+      auto dynamic = jsi::dynamicFromValue(runtime, value);
+      auto map = react::ReadableNativeMap::createWithContents(std::move(dynamic));
+      return map.release();
+    }
+  } else {
+    // unknown jsi type!
+    auto stringRepresentation = value.toString(runtime).utf8(runtime);
+    auto message = "Received unknown JSI value! (" + stringRepresentation + ") Cannot convert to JNI Object.";
+    __android_log_write(ANDROID_LOG_ERROR, "VisionCamera", message.c_str());
+    return nullptr;
+  }
+}
+
+jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, const jni::local_ref<jobject>& object) {
+  if (object->isInstanceOf(jni::JBoolean::javaClassStatic())) {
+    // Boolean
+
+    static const auto getBooleanFunc = jni::findClassLocal("java/lang/Boolean")->getMethod<jboolean()>("booleanValue");
+    auto boolean = getBooleanFunc(object.get());
+    return jsi::Value(boolean == true);
+
+  } else if (object->isInstanceOf(jni::JDouble::javaClassStatic())) {
+    // Double
+
+    static const auto getDoubleFunc = jni::findClassLocal("java/lang/Double")->getMethod<jdouble()>("doubleValue");
+    auto d = getDoubleFunc(object.get());
+    return jsi::Value(d);
+
+  } else if (object->isInstanceOf(jni::JInteger::javaClassStatic())) {
+    // Integer
+
+    static const auto getIntegerFunc = jni::findClassLocal("java/lang/Integer")->getMethod<jint()>("integerValue");
+    auto i = getIntegerFunc(object.get());
+    return jsi::Value(i);
+
+  } else if (object->isInstanceOf(jni::JString::javaClassStatic())) {
+    // String
+
+    return jsi::String::createFromUtf8(runtime, object->toString());
+
+  } else if (object->isInstanceOf(JReadableArray::javaClassStatic())) {
+    // ReadableArray
+
+    static const auto toArrayListFunc = JReadableArray::javaClassLocal()->getMethod<jni::JArrayClass<jobject>()>("toArrayList");
+
+    auto array = toArrayListFunc(object.get());
+    auto size = array->size();
+
+    auto result = jsi::Array(runtime, size);
+    for (size_t i = 0; i < size; i++) {
+      result.setValueAtIndex(runtime, i, convertJNIObjectToJSIValue(runtime, (*array)[i]));
+    }
+    return result;
+
+  } else if (object->isInstanceOf(JArrayList::javaClassStatic())) {
+    // ArrayList
+
+    static const auto iteratorFunc = JArrayList::javaClassLocal()->getMethod<jni::JIterator<jobject>()>("iterator");
+    static const auto sizeFunc = JArrayList::javaClassLocal()->getMethod<jint()>("size");
+
+    auto iterator = iteratorFunc(object.get());
+    auto size = sizeFunc(object.get());
+
+    auto result = jsi::Array(runtime, size);
+    size_t i = 0;
+    for (auto& item : *iterator) {
+      result.setValueAtIndex(runtime, i, convertJNIObjectToJSIValue(runtime, item));
+      i++;
+    }
+    return result;
+
+  } else if (object->isInstanceOf(JReadableMap::javaClassStatic())) {
+    // ReadableMap
+
+    static const auto toHashMapFunc = JReadableMap::javaClassLocal()->getMethod<jni::JHashMap<jstring, jobject>()>("toHashMap");
+    auto hashMap = toHashMapFunc(object.get());
+
+    auto result = jsi::Object(runtime);
+
+    for (const auto& entry : *hashMap) {
+      auto key = entry.first->toString();
+      auto value = entry.second;
+      auto jsiValue = convertJNIObjectToJSIValue(runtime, value);
+      result.setProperty(runtime, key.c_str(), jsiValue);
+    }
+
+    return result;
+  }
+
+  auto type = object->getClass()->toString();
+  auto message = "Received unknown JNI type \"" + type + "\"! Cannot convert to jsi::Value.";
+  __android_log_write(ANDROID_LOG_ERROR, "VisionCamera", message.c_str());
+  return jsi::Value::undefined();
+}
+
+} // namespace vision
--- a/android/src/main/cpp/JSIJNIConversion.h
+++ b/android/src/main/cpp/JSIJNIConversion.h
@@ -0,0 +1,21 @@
+//
+// Created by Marc Rousavy on 22.06.21.
+//
+
+#pragma once
+
+#include <jsi/jsi.h>
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+namespace vision {
+
+using namespace facebook;
+
+namespace JSIJNIConversion {
+  jobject convertJSIValueToJNIObject(jsi::Runtime& runtime, const jsi::Value& value); // NOLINT(runtime/references)
+
+  jsi::Value convertJNIObjectToJSIValue(jsi::Runtime& runtime, const jni::local_ref<jobject>& object); // NOLINT(runtime/references)
+};
+
+} // namespace vision
--- a/android/src/main/cpp/VisionCamera.cpp
+++ b/android/src/main/cpp/VisionCamera.cpp
@@ -0,0 +1,13 @@
+#include <jni.h>
+#include <fbjni/fbjni.h>
+#include "FrameProcessorRuntimeManager.h"
+#include "FrameProcessorPlugin.h"
+#include "CameraView.h"
+
+JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) {
+  return facebook::jni::initialize(vm, [] {
+    vision::FrameProcessorRuntimeManager::registerNatives();
+    vision::FrameProcessorPlugin::registerNatives();
+    vision::CameraView::registerNatives();
+  });
+}
--- a/android/src/main/cpp/reanimated-headers/AndroidErrorHandler.h
+++ b/android/src/main/cpp/reanimated-headers/AndroidErrorHandler.h
@@ -0,0 +1,30 @@
+// copied from https://github.com/software-mansion/react-native-reanimated/blob/master/android/src/main/cpp/headers/AndroidErrorHandler.h
+
+#pragma once
+
+#include "ErrorHandler.h"
+#include "AndroidScheduler.h"
+#include "Scheduler.h"
+#include <jni.h>
+#include <memory>
+#include <fbjni/fbjni.h>
+#include "Logger.h"
+
+namespace reanimated
+{
+
+  class AndroidErrorHandler : public JavaClass<AndroidErrorHandler>, public ErrorHandler {
+    std::shared_ptr<ErrorWrapper> error;
+    std::shared_ptr<Scheduler> scheduler;
+    void raiseSpec() override;
+  public:
+    static auto constexpr kJavaDescriptor = "Lcom/swmansion/reanimated/AndroidErrorHandler;";
+    AndroidErrorHandler(
+        std::shared_ptr<Scheduler> scheduler);
+    std::shared_ptr<Scheduler> getScheduler() override;
+    std::shared_ptr<ErrorWrapper> getError() override;
+    void setError(std::string message) override;
+    virtual ~AndroidErrorHandler() {}
+  };
+
+}
--- a/android/src/main/cpp/reanimated-headers/AndroidScheduler.h
+++ b/android/src/main/cpp/reanimated-headers/AndroidScheduler.h
@@ -0,0 +1,37 @@
+// copied from https://github.com/software-mansion/react-native-reanimated/blob/master/android/src/main/cpp/headers/AndroidScheduler.h
+
+#pragma once
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+#include <jsi/jsi.h>
+#include <react/jni/CxxModuleWrapper.h>
+#include <react/jni/JMessageQueueThread.h>
+#include "Scheduler.h"
+
+namespace reanimated {
+
+  using namespace facebook;
+
+  class AndroidScheduler : public jni::HybridClass<AndroidScheduler> {
+  public:
+    static auto constexpr kJavaDescriptor = "Lcom/swmansion/reanimated/Scheduler;";
+    static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis);
+    static void registerNatives();
+
+    std::shared_ptr<Scheduler> getScheduler() { return scheduler_; }
+
+    void scheduleOnUI();
+
+  private:
+    friend HybridBase;
+
+    void triggerUI();
+
+    jni::global_ref<AndroidScheduler::javaobject> javaPart_;
+    std::shared_ptr<Scheduler> scheduler_;
+
+    explicit AndroidScheduler(jni::alias_ref<AndroidScheduler::jhybridobject> jThis);
+  };
+
+}