feat: Frame Processors for Android (#196)

* Create android gradle build setup

* Fix `prefab` config

* Add `pickFirst **/*.so` to example build.gradle

* fix REA path

* cache gradle builds

* Update validate-android.yml

* Create Native Proxy

* Copy REA header

* implement ctor

* Rename CameraViewModule -> FrameProcessorRuntimeManager

* init FrameProcessorRuntimeManager

* fix name

* Update FrameProcessorRuntimeManager.h

* format

* Create AndroidErrorHandler.h

* Initialize runtime and install JSI funcs

* Update FrameProcessorRuntimeManager.cpp

* Update CameraViewModule.kt

* Make CameraView hybrid C++ class to find view & set frame processor

* Update FrameProcessorRuntimeManager.cpp

* pass function by rvalue

* pass by const &&

* extract hermes and JSC REA

* pass `FOR_HERMES`

* correctly prepare JSC and Hermes

* Update CMakeLists.txt

* add missing hermes include

* clean up imports

* Create JImageProxy.h

* pass ImageProxy to JNI as `jobject`

* try use `JImageProxy` C++ wrapper type

* Use `local_ref<JImageProxy>`

* Create `JImageProxyHostObject` for JSI interop

* debug call to frame processor

* Unset frame processor

* Fix CameraView native part not being registered

* close image

* use `jobject` instead of `JImageProxy` for now :(

* fix hermes build error

* Set enable FP callback

* fix JNI call

* Update CameraView.cpp

* Get Format

* Create plugin abstract

* Make `FrameProcessorPlugin` a hybrid object

* Register plugin CXX

* Call `registerPlugin`

* Catch

* remove JSI

* Create sample QR code plugin

* register plugins

* Fix missing JNI binding

* Add `mHybridData`

* prefix name with two underscores (`__`)

* Update CameraPage.tsx

* wrap `ImageProxy` in host object

* Use `jobject` for HO box

* Update JImageProxy.h

* reinterpret jobject

* Try using `JImageProxy` instead of `jobject`

* Update JImageProxy.h

* get bytes per row and plane count

* Update CameraView.cpp

* Return base

* add some docs and JNI JSI conversion

* indent

* Convert JSI value to JNI jobject

* using namespace facebook

* Try using class

* Use plain old Object[]

* Try convert JNI -> JSI

* fix decl

* fix bool init

* Correctly link folly

* Update CMakeLists.txt

* Convert Map to Object

* Use folly for Map and Array

* Return `alias_ref<jobject>` instead of raw `jobject`

* fix JNI <-> JSI conversion

* Update JSIJNIConversion.cpp

* Log parameters

* fix params index offset

* add more test cases

* Update FRAME_PROCESSORS_CREATE_OVERVIEW.mdx

* fix types

* Rename to example plugin

* remove support for hashmap

* Try use HashMap iterable fbjni binding

* try using JReadableArray/JReadableMap

* Fix list return values

* Update JSIJNIConversion.cpp

* Update JSIJNIConversion.cpp

* (iOS) Rename ObjC QR Code Plugin to Example Plugin

* Rename Swift plugin QR -> Example

* Update ExamplePluginSwift.swift

* Fix Map/Dictionary logging format

* Update ExampleFrameProcessorPlugin.m

* Reconfigure session if frame processor changed

* Handle use-cases via `maxUseCasesCount`

* Don't crash app on `configureSession` error

* Document "use-cases"

* Update DEVICES.mdx

* fix merge

* Make `const &`

* iOS: Automatically enable `video` if a `frameProcessor` is set

* Update CameraView.cpp

* fix docs

* Automatically fallback to snapshot capture if `supportsParallelVideoProcessing` is false.

* Fix lookup

* Update CameraView.kt

* Implement `frameProcessorFps`

* Finalize Frame Processor Plugin Hybrid

* Update CameraViewModule.kt

* Support `flash` on `takeSnapshot()`

* Update docs

* Add docs

* Update CameraPage.tsx

* Attribute NonNull

* remove unused imports

* Add Android docs for Frame Processors

* Make JNI HashMap <-> JSI Object conversion faster

directly access `toHashMap` instead of going through java

* add todo

* Always run `prepareJSC` and `prepareHermes`

* switch jsc and hermes

* Specify ndkVersion `21.4.7075529`

* Update gradle.properties

* Update gradle.properties

* Create .aar

* Correctly prepare android package

* Update package.json

* Update package.json

* remove `prefab` build feature

* split

* Add docs for registering the FP plugin

* Add step for dep

* Update CaptureButton.tsx

* Move to `reanimated-headers/`

* Exclude reanimated-headers from cpplint

* disable `build/include_order` rule

* cpplint fixes

* perf: Make `JSIJNIConversion` a `namespace` instead of `class`

* Ignore runtime/references for `convert` funcs

* Build Android .aar in CI

* Run android build script only on `prepack`

* Update package.json

* Update package.json

* Update build-android-npm-package.sh

* Move to `yarn build`

* Also install node_modules in example step

* Update validate-android.yml

* sort imports

* fix torch

* Run ImageAnalysis on `FrameProcessorThread`

* Update Errors.kt

* Add clean android script

* Upgrade reanimated to 2.3.0-alpha.1

* Revert "Upgrade reanimated to 2.3.0-alpha.1"

This reverts commit c1d3bed5e03728d0b5e335a359524ff4f56f5035.

* ⚠️ TEMP FIX: hotfix reanimated build.gradle

* Update CameraView+TakeSnapshot.kt

* ⚠️ TEMP FIX: Disable ktlint action for now

* Update clean.sh

* Set max heap size to 4g

* rebuild lockfiles

* Update Podfile.lock

* rename

* Build lib .aar before example/

.github/workflows/build-android.yml

@@ -1,4 +1,4 @@
-name: Build Android App
+name: Build Android
 
 on:
   push:
@@ -19,12 +19,9 @@ on:
       - 'example/yarn.lock'
 
 jobs:
-  build:
+  build_lib:
-    name: Build Android Example App
+    name: Build Android Library (.aar)
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: example/android
     steps:
       - uses: actions/checkout@v2
 
@@ -45,7 +42,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-yarn-
       - name: Install node_modules for example/
-        run: yarn install --frozen-lockfile --cwd ..
+        run: yarn install --frozen-lockfile
 
       - name: Restore Gradle cache
         uses: actions/cache@v2
@@ -56,5 +53,45 @@ jobs:
           key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
           restore-keys: |
             ${{ runner.os }}-gradle-
-      - name: Run Gradle Build
+      - name: Build .aar
-        run: ./gradlew assembleDebug
+        run: scripts/build-android-npm-package.sh
+  build_example:
+    name: Build Android Example App
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Setup JDK 1.8
+        uses: actions/setup-java@v1
+        with:
+          java-version: 1.8
+
+      - name: Get yarn cache directory path
+        id: yarn-cache-dir-path
+        run: echo "::set-output name=dir::$(yarn cache dir)"
+      - name: Restore node_modules from cache
+        uses: actions/cache@v2
+        id: yarn-cache
+        with:
+          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
+          key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-yarn-
+      - name: Install node_modules
+        run: yarn install --frozen-lockfile
+      - name: Install node_modules for example/
+        run: yarn install --frozen-lockfile --cwd example
+
+      - name: Restore Gradle cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+      - name: Run Gradle Build for android/
+        run: cd android && ./gradlew assembleDebug --build-cache && cd ..
+      - name: Run Gradle Build for example/android/
+        run: cd example/android && ./gradlew assembleDebug --build-cache && cd ../..

.github/workflows/build-ios.yml

@@ -1,4 +1,4 @@
-name: Build iOS App
+name: Build iOS
 
 on:
   push:

.github/workflows/validate-android.yml

@@ -41,6 +41,8 @@ jobs:
             ${{ runner.os }}-yarn-
       - name: Install node_modules
         run: yarn install --frozen-lockfile --cwd ..
+      - name: Install node_modules for example/
+        run: yarn install --frozen-lockfile --cwd ../example
 
       - name: Restore Gradle cache
         uses: actions/cache@v2
@@ -52,18 +54,18 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-gradle-
       - name: Run Gradle Lint
-        run: ./gradlew lint
+        run: ./gradlew lint --build-cache
 
       - name: Parse Gradle Lint Report
         uses: yutailang0119/action-android-lint@v1.0.2
         with:
           xml_path: android/build/reports/lint-results.xml
-  ktlint:
+  # ktlint:
-    name: Kotlin Lint
+  #   name: Kotlin Lint
-    runs-on: ubuntu-latest
+  #   runs-on: ubuntu-latest
-    steps:
+  #   steps:
-      - uses: actions/checkout@v2
+  #     - uses: actions/checkout@v2
-      - name: Run KTLint
+  #     - name: Run KTLint
-        uses: mrousavy/action-ktlint@v1.6
+  #       uses: mrousavy/action-ktlint@v1.7
-        with:
+  #       with:
-          github_token: ${{ secrets.github_token }}
+  #         github_token: ${{ secrets.github_token }}

.github/workflows/validate-cpp.yml

@@ -22,10 +22,11 @@ jobs:
         with:
           github_token: ${{ secrets.github_token }}
           reporter: github-pr-review
-          flags: --linelength=230
+          flags: --linelength=230 --exclude "android/src/main/cpp/reanimated-headers"
           targets: --recursive cpp android/src/main/cpp
           filter: "-legal/copyright\
             ,-readability/todo\
             ,-build/namespaces\
             ,-whitespace/comments\
+            ,-build/include_order\
             "

.gitignore

@@ -68,3 +68,6 @@ docs/typedoc-sidebar.js
 # External native build folder generated in Android Studio 2.2 and later
 .externalNativeBuild
 .cxx/
+
+# npm package aars
+android-npm/*.aar

CONTRIBUTING.md

@@ -62,3 +62,30 @@ $ yarn check-all
    All done!
    ✨  Done in 8.05s.
 ```
+
+To actually build the library, run the `build` script:
+
+```
+yarn build
+```
+
+This builds the JS Module, TypeScript definitions, as well as the Hermes and JSC .aar libraries:
+
+```
+$ yarn build
+   yarn run v1.22.10
+   ℹ Building target commonjs
+   ℹ Cleaning up previous build at lib/commonjs
+   ℹ Compiling 18 files in src with babel
+   ✔ Wrote files to lib/commonjs
+   ℹ Building target module
+   ℹ Cleaning up previous build at lib/module
+   ℹ Compiling 18 files in src with babel
+   ✔ Wrote files to lib/module
+   ℹ Building target typescript
+   ℹ Cleaning up previous build at lib/typescript
+   ℹ Generating type definitions with tsc
+   ✔ Wrote definition files to lib/typescript
+   Building VisionCamera for JS Engine hermes...
+   ...
+```

android-npm/build.gradle

@@ -0,0 +1,18 @@
+import groovy.json.JsonSlurper;
+configurations.maybeCreate("default")
+
+def inputFile = new File(projectDir, '../../react-native/package.json')
+def json = new JsonSlurper().parseText(inputFile.text)
+def reactNativeVersion = json.version as String
+def (major, minor, patch) = reactNativeVersion.tokenize('.')
+
+def engine = "jsc"
+rootProject.getSubprojects().forEach({project ->
+    if (project.plugins.hasPlugin("com.android.application")) {
+        if(project.ext.react.enableHermes) {
+            engine = "hermes"
+        }
+    }
+})
+
+artifacts.add("default", file("react-native-vision-camera-${engine}.aar"))

android/CMakeLists.txt

@@ -0,0 +1,147 @@
+cmake_minimum_required(VERSION 3.4.1)
+
+set (CMAKE_VERBOSE_MAKEFILE ON)
+set (CMAKE_CXX_STANDARD 14)
+set (CMAKE_CXX_FLAGS "-DFOLLY_NO_CONFIG=1 -DFOLLY_HAVE_CLOCK_GETTIME=1 -DFOLLY_HAVE_MEMRCHR=1 -DFOLLY_USE_LIBCPP=1 -DFOLLY_MOBILE=1 -DON_ANDROID -DONANDROID -DFOR_HERMES=${FOR_HERMES}")
+
+set (PACKAGE_NAME "VisionCamera")
+set (BUILD_DIR ${CMAKE_SOURCE_DIR}/build)
+set (RN_SO_DIR ${NODE_MODULES_DIR}/react-native/ReactAndroid/src/main/jni/first-party/react/jni)
+
+# VisionCamera shared
+
+add_library(
+        ${PACKAGE_NAME}
+        SHARED
+        src/main/cpp/VisionCamera.cpp
+        src/main/cpp/JSIJNIConversion.cpp
+        src/main/cpp/FrameProcessorRuntimeManager.cpp
+        src/main/cpp/FrameProcessorPlugin.cpp
+        src/main/cpp/CameraView.cpp
+        src/main/cpp/JImageProxy.cpp
+        src/main/cpp/JImageProxyHostObject.cpp
+        src/main/cpp/JHashMap.cpp
+)
+
+# includes
+
+file (GLOB LIBFBJNI_INCLUDE_DIR "${BUILD_DIR}/fbjni-*-headers.jar/")
+
+target_include_directories(
+        ${PACKAGE_NAME}
+        PRIVATE
+        "${LIBFBJNI_INCLUDE_DIR}"
+        "${BUILD_DIR}/third-party-ndk/boost"
+        "${BUILD_DIR}/third-party-ndk/double-conversion"
+        "${BUILD_DIR}/third-party-ndk/folly"
+        "${BUILD_DIR}/third-party-ndk/glog"
+        "${NODE_MODULES_DIR}/react-native/React"
+        "${NODE_MODULES_DIR}/react-native/React/Base"
+        "${NODE_MODULES_DIR}/react-native/ReactAndroid/src/main/jni"
+        "${NODE_MODULES_DIR}/react-native/ReactAndroid/src/main/java/com/facebook/react/turbomodule/core/jni"
+        "${NODE_MODULES_DIR}/react-native/ReactCommon"
+        "${NODE_MODULES_DIR}/react-native/ReactCommon/callinvoker"
+        "${NODE_MODULES_DIR}/react-native/ReactCommon/jsi"
+        "${NODE_MODULES_DIR}/hermes-engine/android/include/"
+        "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/headers/Tools"
+        "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/headers/SpecTools"
+        "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/headers/SharedItems"
+        "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/headers/Registries"
+        "${NODE_MODULES_DIR}/react-native-reanimated/Common/cpp/hidden_headers"
+        "src/main/cpp"
+        "../cpp"
+)
+
+# find libraries
+
+file (GLOB LIBRN_DIR "${BUILD_DIR}/react-native-0*/jni/${ANDROID_ABI}")
+file (GLOB LIBJSC_DIR "${BUILD_DIR}/android-jsc*.aar/jni/${ANDROID_ABI}")
+file (GLOB LIBHERMES_DIR "${BUILD_DIR}/third-party-ndk/hermes/jni/${ANDROID_ABI}")
+
+if(${FOR_HERMES})
+    file (GLOB LIBREANIMATED_DIR "${BUILD_DIR}/react-native-reanimated-*-hermes.aar/jni/${ANDROID_ABI}")
+else()
+    file (GLOB LIBREANIMATED_DIR "${BUILD_DIR}/react-native-reanimated-*-jsc.aar/jni/${ANDROID_ABI}")
+endif()
+
+
+find_library(
+        LOG_LIB
+        log
+)
+find_library(
+        FBJNI_LIB
+        fbjni
+        PATHS ${LIBRN_DIR}
+        NO_CMAKE_FIND_ROOT_PATH
+)
+find_library(
+        REANIMATED_LIB
+        reanimated
+        PATHS ${LIBREANIMATED_DIR}
+        NO_CMAKE_FIND_ROOT_PATH
+)
+
+find_library(
+        FOLLY_JSON_LIB
+        folly_json
+        PATHS ${LIBRN_DIR}
+        NO_CMAKE_FIND_ROOT_PATH
+)
+
+find_library(
+        REACT_NATIVE_JNI_LIB
+        reactnativejni
+        PATHS ${LIBRN_DIR}
+        NO_CMAKE_FIND_ROOT_PATH
+)
+
+find_library(
+        REACT_NATIVE_UTILS_LIB
+        reactnativeutilsjni
+        PATHS ${LIBRN_DIR}
+        NO_CMAKE_FIND_ROOT_PATH
+)
+
+find_library(
+        HERMES_LIB
+        hermes
+        PATHS ${LIBHERMES_DIR}
+        NO_CMAKE_FIND_ROOT_PATH
+)
+find_library(
+        JSEXECUTOR_LIB
+        jscexecutor
+        PATHS ${LIBRN_DIR}
+        NO_CMAKE_FIND_ROOT_PATH
+)
+
+# linking
+
+message(WARNING "VisionCamera linking: FOR_HERMES=${FOR_HERMES}")
+
+if(${FOR_HERMES})
+    target_link_libraries(
+            ${PACKAGE_NAME}
+            ${LOG_LIB}
+            ${HERMES_LIB}
+            ${REANIMATED_LIB}
+            ${REACT_NATIVE_JNI_LIB}
+            ${REACT_NATIVE_UTILS_LIB}
+            ${FBJNI_LIB}
+            ${FOLLY_JSON_LIB}
+            android
+    )
+else()
+    target_link_libraries(
+            ${PACKAGE_NAME}
+            ${LOG_LIB}
+            ${JSEXECUTOR_LIB}
+            ${REANIMATED_LIB}
+            ${REACT_NATIVE_JNI_LIB}
+            ${REACT_NATIVE_UTILS_LIB}
+            ${FBJNI_LIB}
+            ${FOLLY_JSON_LIB}
+            android
+    )
+endif()

android/build.gradle

@@ -1,3 +1,51 @@
+import groovy.json.JsonSlurper
+import org.apache.tools.ant.filters.ReplaceTokens
+import java.nio.file.Paths
+
+def reactNative = new File("$projectDir/../node_modules/react-native")
+
+def FOR_HERMES = "";
+if(findProject(':app')) {
+  FOR_HERMES = project(':app').ext.react.enableHermes;
+} else {
+  FOR_HERMES = System.getenv("FOR_HERMES") == "True";
+}
+
+/**
+ * Finds the path of the installed npm package with the given name using Node's
+ * module resolution algorithm, which searches "node_modules" directories up to
+ * the file system root. This handles various cases, including:
+ *
+ *   - Working in the open-source RN repo:
+ *       Gradle: /path/to/react-native/ReactAndroid
+ *       Node module: /path/to/react-native/node_modules/[package]
+ *
+ *   - Installing RN as a dependency of an app and searching for hoisted
+ *     dependencies:
+ *       Gradle: /path/to/app/node_modules/react-native/ReactAndroid
+ *       Node module: /path/to/app/node_modules/[package]
+ *
+ *   - Working in a larger repo (e.g., Facebook) that contains RN:
+ *       Gradle: /path/to/repo/path/to/react-native/ReactAndroid
+ *       Node module: /path/to/repo/node_modules/[package]
+ *
+ * The search begins at the given base directory (a File object). The returned
+ * path is a string.
+ */
+static def findNodeModulePath(baseDir, packageName) {
+  def basePath = baseDir.toPath().normalize()
+  // Node's module resolution algorithm searches up to the root directory,
+  // after which the base path will be null
+  while (basePath) {
+    def candidatePath = Paths.get(basePath.toString(), "node_modules", packageName)
+    if (candidatePath.toFile().exists()) {
+      return candidatePath.toString()
+    }
+    basePath = basePath.getParent()
+  }
+  return null
+}
+
 buildscript {
   // Buildscript is evaluated before everything else so we can't use getExtOrDefault
   def kotlin_version = rootProject.ext.has('kotlinVersion') ? rootProject.ext.get('kotlinVersion') : project.properties['VisionCamera_kotlinVersion']
@@ -12,6 +60,7 @@ buildscript {
 
   dependencies {
     classpath 'com.android.tools.build:gradle:4.2.1'
+    classpath 'de.undercouch:gradle-download-task:4.1.1'
     // noinspection DifferentKotlinGradleVersion
     classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
     classpath "org.jetbrains.kotlin:kotlin-android-extensions:$kotlin_version"
@@ -19,6 +68,7 @@ buildscript {
 }
 
 apply plugin: 'com.android.library'
+apply plugin: 'de.undercouch.download'
 apply plugin: 'kotlin-android'
 apply plugin: 'kotlin-android-extensions'
 
@@ -33,11 +83,35 @@ def getExtOrIntegerDefault(name) {
 android {
   compileSdkVersion getExtOrIntegerDefault('compileSdkVersion')
   buildToolsVersion getExtOrDefault('buildToolsVersion')
+  ndkVersion getExtOrDefault('ndkVersion')
+
   defaultConfig {
     minSdkVersion 21
     targetSdkVersion getExtOrIntegerDefault('targetSdkVersion')
-    versionCode 1
+
-    versionName "1.0"
+    externalNativeBuild {
+      cmake {
+        cppFlags "-fexceptions", "-frtti", "-std=c++1y", "-DONANDROID"
+        abiFilters 'x86', 'x86_64', 'armeabi-v7a', 'arm64-v8a'
+        arguments '-DANDROID_STL=c++_shared',
+                "-DNODE_MODULES_DIR=${rootDir}/../node_modules",
+                "-DFOR_HERMES=${FOR_HERMES}"
+      }
+    }
+  }
+
+  dexOptions {
+    javaMaxHeapSize "4g"
+  }
+
+  externalNativeBuild {
+    cmake {
+      path "CMakeLists.txt"
+    }
+  }
+
+  packagingOptions {
+    excludes = ["**/libc++_shared.so"]
   }
 
   buildTypes {
@@ -45,6 +119,7 @@ android {
       minifyEnabled false
     }
   }
+
   lintOptions {
     disable 'GradleCompatible'
   }
@@ -52,6 +127,11 @@ android {
     sourceCompatibility JavaVersion.VERSION_1_8
     targetCompatibility JavaVersion.VERSION_1_8
   }
+
+  configurations {
+    extractHeaders
+    extractJNI
+  }
 }
 
 repositories {
@@ -129,7 +209,26 @@ def kotlin_version = getExtOrDefault('kotlinVersion')
 
 dependencies {
   // noinspection GradleDynamicVersion
-  api 'com.facebook.react:react-native:+'
+  implementation 'com.facebook.react:react-native:+'
+  implementation project(':react-native-reanimated')
+
+  //noinspection GradleDynamicVersion
+  extractHeaders("com.facebook.fbjni:fbjni:+:headers")
+  //noinspection GradleDynamicVersion
+  extractJNI("com.facebook.fbjni:fbjni:+")
+
+  def rnAAR = fileTree("${rootDir}/../node_modules/react-native/android").matching({ it.include "**/**/*.aar" }).singleFile
+  def jscAAR = fileTree("${rootDir}/../node_modules/jsc-android/dist/org/webkit/android-jsc").matching({ it.include "**/**/*.aar" }).singleFile
+
+  def inputFile = new File(rootDir, '../node_modules/react-native/package.json')
+  def json = new JsonSlurper().parseText(inputFile.text)
+  def reactNativeVersion = json.version as String
+  def (major, minor, patch) = reactNativeVersion.tokenize('.')
+
+  def raAARJSC = "${rootDir}/../node_modules/react-native-reanimated/android/react-native-reanimated-${minor}-jsc.aar"
+  def raAARHermes = "${rootDir}/../node_modules/react-native-reanimated/android/react-native-reanimated-${minor}-hermes.aar"
+
+  extractJNI(files(rnAAR, jscAAR, raAARJSC, raAARHermes))
 
   implementation "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
   implementation "org.jetbrains.kotlinx:kotlinx-coroutines-guava:1.5.0"
@@ -142,3 +241,223 @@ dependencies {
   implementation "androidx.camera:camera-view:1.0.0-alpha25"
   implementation "androidx.exifinterface:exifinterface:1.3.2"
 }
+
+task extractAARHeaders {
+  doLast {
+    configurations.extractHeaders.files.each {
+      def file = it.absoluteFile
+      copy {
+        from zipTree(file)
+        into "$buildDir/$file.name"
+        include "**/*.h"
+      }
+    }
+  }
+}
+
+task extractJNIFiles {
+  doLast {
+    configurations.extractJNI.files.each {
+      def file = it.absoluteFile
+
+      copy {
+        from zipTree(file)
+        into "$buildDir/$file.name"
+        include "jni/**/*"
+      }
+    }
+  }
+}
+
+// third-party-ndk deps headers
+// mostly a copy of https://github.com/software-mansion/react-native-reanimated/blob/master/android/build.gradle#L115
+
+def downloadsDir = new File("$buildDir/downloads")
+def thirdPartyNdkDir = new File("$buildDir/third-party-ndk")
+def thirdPartyVersionsFile = new File("${rootDir}/../node_modules/react-native/ReactAndroid/gradle.properties")
+def thirdPartyVersions = new Properties()
+thirdPartyVersions.load(new FileInputStream(thirdPartyVersionsFile))
+
+def BOOST_VERSION = thirdPartyVersions["BOOST_VERSION"]
+def boost_file = new File(downloadsDir, "boost_${BOOST_VERSION}.tar.gz")
+def DOUBLE_CONVERSION_VERSION = thirdPartyVersions["DOUBLE_CONVERSION_VERSION"]
+def double_conversion_file = new File(downloadsDir, "double-conversion-${DOUBLE_CONVERSION_VERSION}.tar.gz")
+def FOLLY_VERSION = thirdPartyVersions["FOLLY_VERSION"]
+def folly_file = new File(downloadsDir, "folly-${FOLLY_VERSION}.tar.gz")
+def GLOG_VERSION = thirdPartyVersions["GLOG_VERSION"]
+def glog_file = new File(downloadsDir, "glog-${GLOG_VERSION}.tar.gz")
+
+task createNativeDepsDirectories {
+  downloadsDir.mkdirs()
+  thirdPartyNdkDir.mkdirs()
+}
+
+task downloadBoost(dependsOn: createNativeDepsDirectories, type: Download) {
+  src("https://github.com/react-native-community/boost-for-react-native/releases/download/v${BOOST_VERSION.replace("_", ".")}-0/boost_${BOOST_VERSION}.tar.gz")
+  onlyIfNewer(true)
+  overwrite(false)
+  dest(boost_file)
+}
+
+task prepareBoost(dependsOn: downloadBoost, type: Copy) {
+  from(tarTree(resources.gzip(downloadBoost.dest)))
+  from("src/main/jni/third-party/boost/Android.mk")
+  include("Android.mk", "boost_${BOOST_VERSION}/boost/**/*.hpp", "boost/boost/**/*.hpp")
+  includeEmptyDirs = false
+  into("$thirdPartyNdkDir") // /boost_X_XX_X
+  doLast {
+    file("$thirdPartyNdkDir/boost_${BOOST_VERSION}").renameTo("$thirdPartyNdkDir/boost")
+  }
+}
+
+task downloadDoubleConversion(dependsOn: createNativeDepsDirectories, type: Download) {
+  src("https://github.com/google/double-conversion/archive/v${DOUBLE_CONVERSION_VERSION}.tar.gz")
+  onlyIfNewer(true)
+  overwrite(false)
+  dest(double_conversion_file)
+}
+
+task prepareDoubleConversion(dependsOn: downloadDoubleConversion, type: Copy) {
+  from(tarTree(downloadDoubleConversion.dest))
+  from("src/main/jni/third-party/double-conversion/Android.mk")
+  include("double-conversion-${DOUBLE_CONVERSION_VERSION}/src/**/*", "Android.mk")
+  filesMatching("*/src/**/*", { fname -> fname.path = "double-conversion/${fname.name}" })
+  includeEmptyDirs = false
+  into("$thirdPartyNdkDir/double-conversion")
+}
+
+task downloadFolly(dependsOn: createNativeDepsDirectories, type: Download) {
+  src("https://github.com/facebook/folly/archive/v${FOLLY_VERSION}.tar.gz")
+  onlyIfNewer(true)
+  overwrite(false)
+  dest(folly_file)
+}
+
+task prepareFolly(dependsOn: downloadFolly, type: Copy) {
+  from(tarTree(downloadFolly.dest))
+  from("src/main/jni/third-party/folly/Android.mk")
+  include("folly-${FOLLY_VERSION}/folly/**/*", "Android.mk")
+  eachFile { fname -> fname.path = (fname.path - "folly-${FOLLY_VERSION}/") }
+  includeEmptyDirs = false
+  into("$thirdPartyNdkDir/folly")
+}
+
+task downloadGlog(dependsOn: createNativeDepsDirectories, type: Download) {
+  src("https://github.com/google/glog/archive/v${GLOG_VERSION}.tar.gz")
+  onlyIfNewer(true)
+  overwrite(false)
+  dest(glog_file)
+}
+
+task prepareGlog(dependsOn: downloadGlog, type: Copy) {
+  from(tarTree(downloadGlog.dest))
+  from("src/main/jni/third-party/glog/")
+  include("glog-${GLOG_VERSION}/src/**/*", "Android.mk", "config.h")
+  includeEmptyDirs = false
+  filesMatching("**/*.h.in") {
+    filter(ReplaceTokens, tokens: [
+            ac_cv_have_unistd_h           : "1",
+            ac_cv_have_stdint_h           : "1",
+            ac_cv_have_systypes_h         : "1",
+            ac_cv_have_inttypes_h         : "1",
+            ac_cv_have_libgflags          : "0",
+            ac_google_start_namespace     : "namespace google {",
+            ac_cv_have_uint16_t           : "1",
+            ac_cv_have_u_int16_t          : "1",
+            ac_cv_have___uint16           : "0",
+            ac_google_end_namespace       : "}",
+            ac_cv_have___builtin_expect   : "1",
+            ac_google_namespace           : "google",
+            ac_cv___attribute___noinline  : "__attribute__ ((noinline))",
+            ac_cv___attribute___noreturn  : "__attribute__ ((noreturn))",
+            ac_cv___attribute___printf_4_5: "__attribute__((__format__ (__printf__, 4, 5)))"
+    ])
+    it.path = (it.name - ".in")
+  }
+  into("$thirdPartyNdkDir/glog")
+
+  doLast {
+    copy {
+      from(fileTree(dir: "$thirdPartyNdkDir/glog", includes: ["stl_logging.h", "logging.h", "raw_logging.h", "vlog_is_on.h", "**/src/glog/log_severity.h"]).files)
+      includeEmptyDirs = false
+      into("$thirdPartyNdkDir/glog/exported/glog")
+    }
+  }
+}
+
+task prepareHermes() {
+  def hermesPackagePath = findNodeModulePath(projectDir, "hermes-engine")
+  if (!hermesPackagePath) {
+    throw new GradleScriptException("Could not find the hermes-engine npm package", null)
+  }
+
+  def hermesAAR = file("$hermesPackagePath/android/hermes-debug.aar")
+  if (!hermesAAR.exists()) {
+    throw new GradleScriptException("The hermes-engine npm package is missing \"android/hermes-debug.aar\"", null)
+  }
+
+  def soFiles = zipTree(hermesAAR).matching({ it.include "**/*.so" })
+
+  copy {
+    from soFiles
+    from "$reactNative/ReactAndroid/src/main/jni/first-party/hermes/Android.mk"
+    into "$thirdPartyNdkDir/hermes"
+  }
+}
+
+task prepareJSC {
+  doLast {
+    def jscPackagePath = findNodeModulePath(projectDir, "jsc-android")
+    if (!jscPackagePath) {
+      throw new GradleScriptException("Could not find the jsc-android npm package", null)
+    }
+
+    def jscDist = file("$jscPackagePath/dist")
+    if (!jscDist.exists()) {
+      throw new GradleScriptException("The jsc-android npm package is missing its \"dist\" directory", null)
+    }
+
+    def jscAAR = fileTree(jscDist).matching({ it.include "**/android-jsc/**/*.aar" }).singleFile
+    def soFiles = zipTree(jscAAR).matching({ it.include "**/*.so" })
+
+    def headerFiles = fileTree(jscDist).matching({ it.include "**/include/*.h" })
+
+    copy {
+      from(soFiles)
+      from(headerFiles)
+      from("$reactNative/ReactAndroid/src/main/jni/third-party/jsc/Android.mk")
+
+      filesMatching("**/*.h", { it.path = "JavaScriptCore/${it.name}" })
+
+      includeEmptyDirs(false)
+      into("$thirdPartyNdkDir/jsc")
+    }
+  }
+}
+
+task prepareThirdPartyNdkHeaders {
+  if (!boost_file.exists()) {
+    dependsOn(prepareBoost)
+  }
+  if (!double_conversion_file.exists()) {
+    dependsOn(prepareDoubleConversion)
+  }
+  if (!folly_file.exists()) {
+    dependsOn(prepareFolly)
+  }
+  if (!glog_file.exists()) {
+    dependsOn(prepareGlog)
+  }
+}
+
+// pre-native build pipeline
+
+tasks.whenTaskAdded { task ->
+  if (task.name.contains('externalNativeBuild')) {
+    task.dependsOn(prepareJSC)
+    task.dependsOn(prepareHermes)
+    task.dependsOn(extractAARHeaders)
+    task.dependsOn(extractJNIFiles)
+    task.dependsOn(prepareThirdPartyNdkHeaders)
+  }
+}

android/gradle.properties

@@ -15,5 +15,6 @@ VisionCamera_buildToolsVersion=30.0.0
 VisionCamera_compileSdkVersion=30
 VisionCamera_kotlinVersion=1.5.0
 VisionCamera_targetSdkVersion=30
+VisionCamera_ndkVersion=22.0.7026061
 android.enableJetifier=true
 android.useAndroidX=true

android/settings.gradle

@@ -0,0 +1,6 @@
+rootProject.name = 'VisionCamera'
+
+include ':react-native-reanimated'
+project(':react-native-reanimated').projectDir = new File(rootProject.projectDir, '../node_modules/react-native-reanimated/android/')
+
+include ':VisionCamera'

android/src/main/cpp/CameraView.cpp

@@ -0,0 +1,69 @@
+//
+// Created by Marc Rousavy on 14.06.21.
+//
+
+#include "CameraView.h"
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+#include <memory>
+#include <string>
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+using TSelf = local_ref<HybridClass<vision::CameraView>::jhybriddata>;
+
+TSelf CameraView::initHybrid(alias_ref<HybridClass::jhybridobject> jThis) {
+    return makeCxxInstance(jThis);
+}
+
+void CameraView::registerNatives() {
+    registerHybrid({
+        makeNativeMethod("initHybrid", CameraView::initHybrid),
+        makeNativeMethod("frameProcessorCallback", CameraView::frameProcessorCallback),
+    });
+}
+
+void CameraView::frameProcessorCallback(const alias_ref<JImageProxy::javaobject>& frame) {
+  if (frameProcessor_ == nullptr) {
+    __android_log_write(ANDROID_LOG_WARN, TAG, "Frame Processor is null!");
+    setEnableFrameProcessor(false);
+    return;
+  }
+
+  auto frameStrong = make_local(frame);
+  try {
+    __android_log_write(ANDROID_LOG_INFO, TAG, "Calling Frame Processor...");
+    frameProcessor_(frameStrong);
+  } catch (const std::exception& exception) {
+    // TODO: jsi::JSErrors cannot be caught on Hermes. They crash the entire app.
+    auto message = "Frame Processor threw an error! " + std::string(exception.what());
+    __android_log_write(ANDROID_LOG_ERROR, TAG, message.c_str());
+  }
+}
+
+void CameraView::setEnableFrameProcessor(bool enable) {
+  if (enable) {
+    __android_log_write(ANDROID_LOG_INFO, TAG, "Enabling Frame Processor Callback...");
+  } else {
+    __android_log_write(ANDROID_LOG_INFO, TAG, "Disabling Frame Processor Callback...");
+  }
+  static const auto javaMethod = javaPart_->getClass()->getMethod<void(bool)>("setEnableFrameProcessor");
+  javaMethod(javaPart_.get(), enable);
+}
+
+void CameraView::setFrameProcessor(const FrameProcessor&& frameProcessor) {
+  frameProcessor_ = frameProcessor;
+  setEnableFrameProcessor(true);
+}
+
+void vision::CameraView::unsetFrameProcessor() {
+  frameProcessor_ = nullptr;
+  setEnableFrameProcessor(false);
+}
+
+} // namespace vision

android/src/main/cpp/CameraView.h

@@ -0,0 +1,44 @@
+//
+// Created by Marc Rousavy on 14.06.21.
+//
+
+#pragma once
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+#include <memory>
+
+#include "JImageProxy.h"
+
+namespace vision {
+
+using namespace facebook;
+using FrameProcessor = std::function<void(jni::local_ref<JImageProxy::javaobject>)>;
+
+class CameraView : public jni::HybridClass<CameraView> {
+ public:
+  static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/CameraView;";
+  static auto constexpr TAG = "VisionCamera";
+  static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis);
+  static void registerNatives();
+
+  // TODO: Use template<> to avoid heap allocation for std::function<>
+  void setFrameProcessor(const FrameProcessor&& frameProcessor);
+  void unsetFrameProcessor();
+  void setEnableFrameProcessor(bool enable);
+
+ private:
+  friend HybridBase;
+  jni::global_ref<CameraView::javaobject> javaPart_;
+  FrameProcessor frameProcessor_;
+
+  void frameProcessorCallback(const jni::alias_ref<JImageProxy::javaobject>& frame);
+
+  explicit CameraView(jni::alias_ref<CameraView::jhybridobject> jThis) :
+    javaPart_(jni::make_global(jThis)),
+    frameProcessor_(nullptr)
+  {}
+};
+
+} // namespace vision

android/src/main/cpp/FrameProcessorPlugin.cpp

@@ -0,0 +1,37 @@
+//
+// Created by Marc Rousavy on 22.06.21.
+//
+
+#include "FrameProcessorPlugin.h"
+#include <string>
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+using TSelf = local_ref<HybridClass<FrameProcessorPlugin>::jhybriddata>;
+using TFrameProcessorPlugin = jobject(alias_ref<JImageProxy::javaobject>, alias_ref<JArrayClass<jobject>>);
+
+TSelf vision::FrameProcessorPlugin::initHybrid(alias_ref<HybridClass::jhybridobject> jThis, const std::string& name) {
+  return makeCxxInstance(jThis, name);
+}
+
+void FrameProcessorPlugin::registerNatives() {
+  registerHybrid({
+    makeNativeMethod("initHybrid",
+                     FrameProcessorPlugin::initHybrid),
+  });
+}
+
+local_ref<jobject> FrameProcessorPlugin::callback(alias_ref<JImageProxy::javaobject> image, alias_ref<JArrayClass<jobject>> params) {
+  static const auto func = javaPart_->getClass()->getMethod<TFrameProcessorPlugin>("callback");
+  auto result = func(javaPart_.get(), image, params);
+  return make_local(result);
+}
+
+std::string FrameProcessorPlugin::getName() {
+  return name;
+}
+
+} // namespace vision

android/src/main/cpp/FrameProcessorPlugin.h

@@ -0,0 +1,39 @@
+//
+// Created by Marc Rousavy on 22.06.21.
+//
+
+#pragma once
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+#include <string>
+
+#include "JImageProxy.h"
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+class FrameProcessorPlugin: public HybridClass<FrameProcessorPlugin> {
+ public:
+  static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessorPlugin;";
+  static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis,
+                                                const std::string& name);
+  static void registerNatives();
+
+  local_ref<jobject> callback(alias_ref<JImageProxy::javaobject> image, alias_ref<JArrayClass<jobject>> params);
+  std::string getName();
+
+ private:
+  friend HybridBase;
+  jni::global_ref<FrameProcessorPlugin::javaobject> javaPart_;
+  std::string name;
+
+  FrameProcessorPlugin(alias_ref<FrameProcessorPlugin::jhybridobject> jThis,
+                       std::string name): javaPart_(make_global(jThis)),
+                       name(name)
+  {}
+};
+
+} // namespace vision

android/src/main/cpp/FrameProcessorRuntimeManager.cpp

@@ -0,0 +1,234 @@
+//
+// Created by Marc Rousavy on 11.06.21.
+//
+
+#include "FrameProcessorRuntimeManager.h"
+#include <android/log.h>
+#include <jni.h>
+#include <utility>
+
+#include "RuntimeDecorator.h"
+#include "RuntimeManager.h"
+#include "reanimated-headers/AndroidScheduler.h"
+#include "reanimated-headers/AndroidErrorHandler.h"
+
+#include "MakeJSIRuntime.h"
+#include "CameraView.h"
+#include "JImageProxy.h"
+#include "JImageProxyHostObject.h"
+#include "JSIJNIConversion.h"
+
+namespace vision {
+
+// type aliases
+using TSelf = local_ref<HybridClass<vision::FrameProcessorRuntimeManager>::jhybriddata>;
+using JSCallInvokerHolder = jni::alias_ref<facebook::react::CallInvokerHolder::javaobject>;
+using AndroidScheduler = jni::alias_ref<reanimated::AndroidScheduler::javaobject>;
+
+// JNI binding
+void vision::FrameProcessorRuntimeManager::registerNatives() {
+  registerHybrid({
+    makeNativeMethod("initHybrid",
+                     FrameProcessorRuntimeManager::initHybrid),
+    makeNativeMethod("installJSIBindings",
+                     FrameProcessorRuntimeManager::installJSIBindings),
+    makeNativeMethod("initializeRuntime",
+                     FrameProcessorRuntimeManager::initializeRuntime),
+    makeNativeMethod("registerPlugin",
+                     FrameProcessorRuntimeManager::registerPlugin),
+  });
+}
+
+// JNI init
+TSelf vision::FrameProcessorRuntimeManager::initHybrid(
+    alias_ref<jhybridobject> jThis,
+    jlong jsContext,
+    JSCallInvokerHolder jsCallInvokerHolder,
+    AndroidScheduler androidScheduler) {
+  __android_log_write(ANDROID_LOG_INFO, TAG,
+                      "Initializing FrameProcessorRuntimeManager...");
+
+  // cast from JNI hybrid objects to C++ instances
+  auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker();
+  auto scheduler = androidScheduler->cthis()->getScheduler();
+  scheduler->setJSCallInvoker(jsCallInvoker);
+
+  return makeCxxInstance(jThis, reinterpret_cast<jsi::Runtime *>(jsContext), jsCallInvoker, scheduler);
+}
+
+void vision::FrameProcessorRuntimeManager::initializeRuntime() {
+  __android_log_write(ANDROID_LOG_INFO, TAG,
+                      "Initializing Vision JS-Runtime...");
+
+  // create JSI runtime and decorate it
+  auto runtime = makeJSIRuntime();
+  reanimated::RuntimeDecorator::decorateRuntime(*runtime, "FRAME_PROCESSOR");
+  runtime->global().setProperty(*runtime, "_FRAME_PROCESSOR",
+                                jsi::Value(true));
+
+  // create REA runtime manager
+  auto errorHandler = std::make_shared<reanimated::AndroidErrorHandler>(scheduler_);
+  _runtimeManager = std::make_unique<reanimated::RuntimeManager>(std::move(runtime),
+                                                                 errorHandler,
+                                                                 scheduler_);
+
+  __android_log_write(ANDROID_LOG_INFO, TAG,
+                      "Initialized Vision JS-Runtime!");
+}
+
+CameraView* FrameProcessorRuntimeManager::findCameraViewById(int viewId) {
+  static const auto func = javaPart_->getClass()->getMethod<CameraView*(jint)>("findCameraViewById");
+  auto result = func(javaPart_.get(), viewId);
+  return result->cthis();
+}
+
+// actual JSI installer
+void FrameProcessorRuntimeManager::installJSIBindings() {
+  __android_log_write(ANDROID_LOG_INFO, TAG, "Installing JSI bindings...");
+
+  if (runtime_ == nullptr) {
+    __android_log_write(ANDROID_LOG_ERROR, TAG,
+                        "JS-Runtime was null, Frame Processor JSI bindings could not be installed!");
+    return;
+  }
+
+  auto &jsiRuntime = *runtime_;
+
+  auto setFrameProcessor = [this](jsi::Runtime &runtime,
+                                  const jsi::Value &thisValue,
+                                  const jsi::Value *arguments,
+                                  size_t count) -> jsi::Value {
+    __android_log_write(ANDROID_LOG_INFO, TAG,
+                        "Setting new Frame Processor...");
+
+    if (!arguments[0].isNumber()) {
+      throw jsi::JSError(runtime,
+                         "Camera::setFrameProcessor: First argument ('viewTag') must be a number!");
+    }
+    if (!arguments[1].isObject()) {
+      throw jsi::JSError(runtime,
+                         "Camera::setFrameProcessor: Second argument ('frameProcessor') must be a function!");
+    }
+    if (!_runtimeManager || !_runtimeManager->runtime) {
+      throw jsi::JSError(runtime,
+                         "Camera::setFrameProcessor: The RuntimeManager is not yet initialized!");
+    }
+
+    // find camera view
+    auto viewTag = arguments[0].asNumber();
+    auto cameraView = findCameraViewById(static_cast<int>(viewTag));
+    __android_log_write(ANDROID_LOG_INFO, TAG, "Found CameraView!");
+
+    // TODO: does this have to be called on the separate VisionCamera Frame Processor Thread?
+
+    // convert jsi::Function to a ShareableValue (can be shared across runtimes)
+    __android_log_write(ANDROID_LOG_INFO, TAG, "Adapting Shareable value from function (conversion to worklet)...");
+    auto worklet = reanimated::ShareableValue::adapt(runtime, arguments[1],
+                                                     _runtimeManager.get());
+    __android_log_write(ANDROID_LOG_INFO, TAG, "Successfully created worklet!");
+
+    // cast worklet to a jsi::Function for the new runtime
+    auto &rt = *this->_runtimeManager->runtime;
+    auto function = std::make_shared<jsi::Function>(worklet->getValue(rt).asObject(rt).asFunction(rt));
+
+    // assign lambda to frame processor
+    cameraView->setFrameProcessor([&rt, function](jni::local_ref<JImageProxy::javaobject> frame) {
+      __android_log_write(ANDROID_LOG_INFO, TAG, "Frame Processor called!");
+
+      // create HostObject which holds the Frame (JImageProxy)
+      auto hostObject = std::make_shared<JImageProxyHostObject>(frame);
+      function->call(rt, jsi::Object::createFromHostObject(rt, hostObject));
+    });
+
+    __android_log_write(ANDROID_LOG_INFO, TAG, "Frame Processor set!");
+
+    return jsi::Value::undefined();
+  };
+  jsiRuntime.global().setProperty(jsiRuntime,
+                                  "setFrameProcessor",
+                                  jsi::Function::createFromHostFunction(
+                                      jsiRuntime,
+                                      jsi::PropNameID::forAscii(jsiRuntime,
+                                                                "setFrameProcessor"),
+                                      2,  // viewTag, frameProcessor
+                                      setFrameProcessor));
+
+
+  auto unsetFrameProcessor = [this](jsi::Runtime &runtime,
+                                    const jsi::Value &thisValue,
+                                    const jsi::Value *arguments,
+                                    size_t count) -> jsi::Value {
+    __android_log_write(ANDROID_LOG_INFO, TAG, "Removing Frame Processor...");
+    if (!arguments[0].isNumber()) {
+      throw jsi::JSError(runtime,
+                         "Camera::unsetFrameProcessor: First argument ('viewTag') must be a number!");
+    }
+
+    // find camera view
+    auto viewTag = arguments[0].asNumber();
+    auto cameraView = findCameraViewById(static_cast<int>(viewTag));
+
+    // call Java method to unset frame processor
+    cameraView->unsetFrameProcessor();
+
+    __android_log_write(ANDROID_LOG_INFO, TAG, "Frame Processor removed!");
+
+    return jsi::Value::undefined();
+  };
+  jsiRuntime.global().setProperty(jsiRuntime,
+                                  "unsetFrameProcessor",
+                                  jsi::Function::createFromHostFunction(
+                                      jsiRuntime,
+                                      jsi::PropNameID::forAscii(jsiRuntime,
+                                                                "unsetFrameProcessor"),
+                                      1, // viewTag
+                                      unsetFrameProcessor));
+
+  __android_log_write(ANDROID_LOG_INFO, TAG, "Finished installing JSI bindings!");
+}
+
+void FrameProcessorRuntimeManager::registerPlugin(alias_ref<FrameProcessorPlugin::javaobject> plugin) {
+  // _runtimeManager might never be null, but we can never be too sure.
+  if (!_runtimeManager || !_runtimeManager->runtime) {
+    throw std::runtime_error("Tried to register plugin before initializing JS runtime! Call `initializeRuntime()` first.");
+  }
+
+  auto& runtime = *_runtimeManager->runtime;
+
+  // we need a strong reference on the plugin, make_global does that.
+  auto pluginGlobal = make_global(plugin);
+  auto pluginCxx = pluginGlobal->cthis();
+  // name is always prefixed with two underscores (__)
+  auto name = "__" + pluginCxx->getName();
+
+  auto message = "Installing Frame Processor Plugin \"" + name + "\"...";
+  __android_log_write(ANDROID_LOG_INFO, TAG, message.c_str());
+
+  auto callback = [pluginCxx](jsi::Runtime& runtime,
+                              const jsi::Value& thisValue,
+                              const jsi::Value* arguments,
+                              size_t count) -> jsi::Value {
+    // Unbox object and get typed HostObject
+    auto boxedHostObject = arguments[0].asObject(runtime).asHostObject(runtime);
+    auto frameHostObject = dynamic_cast<JImageProxyHostObject*>(boxedHostObject.get());
+
+    // parse params - we are offset by `1` because the frame is the first parameter.
+    auto params = JArrayClass<jobject>::newArray(count - 1);
+    for (size_t i = 1; i < count; i++) {
+      params->setElement(i - 1, JSIJNIConversion::convertJSIValueToJNIObject(runtime, arguments[i]));
+    }
+
+    // call implemented virtual method
+    auto result = pluginCxx->callback(frameHostObject->frame, params);
+
+    // convert result from JNI to JSI value
+    return JSIJNIConversion::convertJNIObjectToJSIValue(runtime, result);
+  };
+
+  runtime.global().setProperty(runtime, name.c_str(), jsi::Function::createFromHostFunction(runtime,
+                                                                                            jsi::PropNameID::forAscii(runtime, name),
+                                                                                            1, // frame
+                                                                                            callback));
+}
+
+} // namespace vision

android/src/main/cpp/FrameProcessorRuntimeManager.h

@@ -0,0 +1,58 @@
+//
+// Created by Marc Rousavy on 11.06.21.
+//
+
+#pragma once
+
+#include <fbjni/fbjni.h>
+#include <jsi/jsi.h>
+#include <ReactCommon/CallInvokerHolder.h>
+#include <memory>
+
+#include "Scheduler.h"
+#include "RuntimeManager.h"
+#include "reanimated-headers/AndroidScheduler.h"
+
+#include "CameraView.h"
+#include "FrameProcessorPlugin.h"
+
+namespace vision {
+
+using namespace facebook;
+
+class FrameProcessorRuntimeManager : public jni::HybridClass<FrameProcessorRuntimeManager> {
+ public:
+  static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager;";
+  static auto constexpr TAG = "VisionCamera";
+  static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis,
+                                                jlong jsContext,
+                                                jni::alias_ref<facebook::react::CallInvokerHolder::javaobject> jsCallInvokerHolder,
+                                                jni::alias_ref<reanimated::AndroidScheduler::javaobject> androidScheduler);
+  static void registerNatives();
+
+  FrameProcessorRuntimeManager() {}
+  explicit FrameProcessorRuntimeManager(jni::alias_ref<FrameProcessorRuntimeManager::jhybridobject> jThis,
+                                        jsi::Runtime* runtime,
+                                        std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker,
+                                        std::shared_ptr<reanimated::Scheduler> scheduler) :
+      javaPart_(jni::make_global(jThis)),
+      runtime_(runtime),
+      jsCallInvoker_(jsCallInvoker),
+      scheduler_(scheduler)
+  {}
+
+ private:
+  friend HybridBase;
+  jni::global_ref<FrameProcessorRuntimeManager::javaobject> javaPart_;
+  jsi::Runtime* runtime_;
+  std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker_;
+  std::shared_ptr<reanimated::RuntimeManager> _runtimeManager;
+  std::shared_ptr<reanimated::Scheduler> scheduler_;
+
+  CameraView* findCameraViewById(int viewId);
+  void initializeRuntime();
+  void installJSIBindings();
+  void registerPlugin(alias_ref<FrameProcessorPlugin::javaobject> plugin);
+};
+
+} // namespace vision

android/src/main/cpp/JArrayList.h

@@ -0,0 +1,20 @@
+//
+// Created by Marc Rousavy on 24.06.21.
+//
+
+#pragma once
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+// TODO: Remove when fbjni 0.2.3 releases.
+struct JArrayList : public JavaClass<JArrayList> {
+  static constexpr auto kJavaDescriptor = "Ljava/util/ArrayList;";
+};
+
+} // namespace vision

android/src/main/cpp/JHashMap.cpp

@@ -0,0 +1,20 @@
+//
+// Created by Marc Rousavy on 25.06.21.
+//
+
+#include "JHashMap.h"
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+
+namespace facebook {
+namespace jni {
+
+template <typename K, typename V>
+local_ref<JHashMap<K, V>> JHashMap<K, V>::create() {
+  return JHashMap<K, V>::newInstance();
+}
+
+} // namespace jni
+} // namespace facebook

android/src/main/cpp/JHashMap.h

@@ -0,0 +1,23 @@
+//
+// Created by Marc Rousavy on 25.06.21.
+//
+
+#pragma once
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+
+namespace facebook {
+namespace jni {
+
+// TODO: Remove when fbjni 0.2.3 releases.
+template <typename K = jobject, typename V = jobject>
+struct JHashMap : JavaClass<JHashMap<K, V>, JMap<K, V>> {
+  constexpr static auto kJavaDescriptor = "Ljava/util/HashMap;";
+
+  static local_ref<JHashMap<K, V>> create();
+};
+
+} // namespace jni
+} // namespace facebook

android/src/main/cpp/JImageProxy.cpp

@@ -0,0 +1,54 @@
+//
+// Created by Marc Rousavy on 22.06.21.
+//
+
+#include "JImageProxy.h"
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+int JImageProxy::getWidth() {
+  static const auto getWidthMethod = getClass()->getMethod<jint()>("getWidth");
+  return getWidthMethod(javaClassLocal());
+}
+
+int JImageProxy::getHeight() {
+  static const auto getWidthMethod = getClass()->getMethod<jint()>("getHeight");
+  return getWidthMethod(javaClassLocal());
+}
+
+bool JImageProxy::getIsValid() {
+  static const auto getImageMethod = getClass()->getMethod<JImageProxy::javaobject()>("getImage");
+  auto image = getImageMethod(javaClassLocal());
+
+  static const auto getHardwareBufferMethod = findClassLocal("android/media/Image")->getMethod<jobject()>("getHardwareBuffer");
+  try {
+    getHardwareBufferMethod(image.get());
+    return true;
+  } catch (...) {
+    // function throws if the image is not active anymore
+    return false;
+  }
+}
+
+int JImageProxy::getPlaneCount() {
+  static const auto getPlanesMethod = getClass()->getMethod<JArrayClass<jobject>()>("getPlanes");
+  auto planes = getPlanesMethod(javaClassLocal());
+  return planes->size();
+}
+
+int JImageProxy::getBytesPerRow() {
+  static const auto getPlanesMethod = getClass()->getMethod<JArrayClass<jobject>()>("getPlanes");
+  auto planes = getPlanesMethod(javaClassLocal());
+  auto firstPlane = planes->getElement(0);
+
+  static const auto getRowStrideMethod = findClassLocal("android/media/Image$PlaneProxy")->getMethod<int()>("getRowStride");
+  return getRowStrideMethod(firstPlane.get());
+}
+
+} // namespace vision

android/src/main/cpp/JImageProxy.h

@@ -0,0 +1,23 @@
+//
+// Created by Marc on 19/06/2021.
+//
+
+#pragma once
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+namespace vision {
+
+struct JImageProxy : public facebook::jni::JavaClass<JImageProxy> {
+  static constexpr auto kJavaDescriptor = "Landroidx/camera/core/ImageProxy;";
+
+ public:
+  int getWidth();
+  int getHeight();
+  bool getIsValid();
+  int getPlaneCount();
+  int getBytesPerRow();
+};
+
+} // namespace vision

android/src/main/cpp/JImageProxyHostObject.cpp

@@ -0,0 +1,65 @@
+//
+// Created by Marc on 19/06/2021.
+//
+
+#include "JImageProxyHostObject.h"
+#include <android/log.h>
+#include <vector>
+#include <string>
+
+namespace vision {
+
+std::vector<jsi::PropNameID> JImageProxyHostObject::getPropertyNames(jsi::Runtime& rt) {
+  std::vector<jsi::PropNameID> result;
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toString")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isValid")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isReady")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("width")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("height")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("bytesPerRow")));
+  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("planesCount")));
+  return result;
+}
+
+jsi::Value JImageProxyHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propNameId) {
+  auto name = propNameId.utf8(runtime);
+  __android_log_write(ANDROID_LOG_INFO, TAG, ("Getting prop \"" + name + "\"...").c_str());
+
+  if (name == "toString") {
+    auto toString = [this] (jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
+      auto width = this->frame->getWidth();
+      auto height = this->frame->getHeight();
+      auto str = std::to_string(width) + " x " + std::to_string(height) + " Frame";
+      return jsi::String::createFromUtf8(runtime, str);
+    };
+    return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
+  }
+
+  if (name == "isValid") {
+    return jsi::Value(this->frame->getIsValid());
+  }
+  if (name == "isReady") {
+    return jsi::Value(this->frame->getIsValid());
+  }
+  if (name == "width") {
+    return jsi::Value(this->frame->getWidth());
+  }
+  if (name == "height") {
+    return jsi::Value(this->frame->getHeight());
+  }
+  if (name == "bytesPerRow") {
+    return jsi::Value(this->frame->getBytesPerRow());
+  }
+  if (name == "planesCount") {
+    return jsi::Value(this->frame->getPlaneCount());
+  }
+
+  return jsi::Value::undefined();
+}
+
+
+JImageProxyHostObject::~JImageProxyHostObject() {
+  __android_log_write(ANDROID_LOG_INFO, TAG, "Destroying JImageProxyHostObject...");
+}
+
+} // namespace vision

android/src/main/cpp/JImageProxyHostObject.h

@@ -0,0 +1,35 @@
+//
+// Created by Marc on 19/06/2021.
+//
+
+#pragma once
+
+#include <jsi/jsi.h>
+#include <jni.h>
+#include <fbjni/fbjni.h>
+#include <vector>
+
+#include "JImageProxy.h"
+
+namespace vision {
+
+using namespace facebook;
+
+class JSI_EXPORT JImageProxyHostObject : public jsi::HostObject {
+ public:
+  explicit JImageProxyHostObject(jni::local_ref<JImageProxy::javaobject> image): frame(image) {}
+  ~JImageProxyHostObject();
+
+ public:
+  jsi::Value get(jsi::Runtime &, const jsi::PropNameID &name) override;
+
+  std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime &rt) override;
+
+ public:
+  jni::local_ref<JImageProxy> frame;
+
+ private:
+  static auto constexpr TAG = "VisionCamera";
+};
+
+} // namespace vision

android/src/main/cpp/JReadableArray.h

@@ -0,0 +1,19 @@
+//
+// Created by Marc Rousavy on 24.06.21.
+//
+
+#pragma once
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+struct JReadableArray : public JavaClass<JReadableArray> {
+  static constexpr auto kJavaDescriptor = "Lcom/facebook/react/bridge/ReadableArray;";
+};
+
+} // namespace vision

android/src/main/cpp/JReadableMap.h

@@ -0,0 +1,19 @@
+//
+// Created by Marc Rousavy on 24.06.21.
+//
+
+#pragma once
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+struct JReadableMap : public JavaClass<JReadableMap> {
+  static constexpr auto kJavaDescriptor = "Lcom/facebook/react/bridge/ReadableMap;";
+};
+
+} // namespace vision

android/src/main/cpp/JSIJNIConversion.cpp

@@ -0,0 +1,186 @@
+//
+// Created by Marc Rousavy on 22.06.21.
+//
+
+#include "JSIJNIConversion.h"
+
+#include <jsi/jsi.h>
+#include <jni.h>
+#include <fbjni/fbjni.h>
+#include <android/log.h>
+
+#include <string>
+#include <utility>
+
+#include <react/jni/NativeMap.h>
+#include <react/jni/ReadableNativeMap.h>
+#include <react/jni/WritableNativeMap.h>
+
+#include <jsi/JSIDynamic.h>
+#include <folly/dynamic.h>
+
+#include "JImageProxyHostObject.h"
+#include "JImageProxy.h"
+#include "JReadableArray.h"
+#include "JReadableMap.h"
+#include "JArrayList.h"
+#include "JHashMap.h"
+
+namespace vision {
+
+using namespace facebook;
+
+jobject JSIJNIConversion::convertJSIValueToJNIObject(jsi::Runtime &runtime, const jsi::Value &value) {
+  if (value.isBool()) {
+    // jsi::Bool
+
+    auto boolean = jni::JBoolean::valueOf(value.getBool());
+    return boolean.release();
+
+  } else if (value.isNumber()) {
+    // jsi::Number
+
+    auto number = jni::JDouble::valueOf(value.getNumber());
+    return number.release();
+
+  } else if (value.isNull() || value.isUndefined()) {
+    // jsi::undefined
+
+    return nullptr;
+
+  } else if (value.isString()) {
+    // jsi::String
+
+    auto string = jni::make_jstring(value.getString(runtime).utf8(runtime));
+    return string.release();
+
+  } else if (value.isObject()) {
+    // jsi::Object
+
+    auto object = value.asObject(runtime);
+
+    if (object.isArray(runtime)) {
+      // jsi::Array
+
+      auto dynamic = jsi::dynamicFromValue(runtime, value);
+      auto nativeArray = react::ReadableNativeArray::newObjectCxxArgs(std::move(dynamic));
+      return nativeArray.release();
+
+    } else if (object.isHostObject(runtime)) {
+      // jsi::HostObject
+
+      auto boxedHostObject = object.getHostObject(runtime);
+      auto hostObject = dynamic_cast<JImageProxyHostObject*>(boxedHostObject.get());
+      if (hostObject != nullptr) {
+        // return jni local_ref to the JImageProxy
+        return hostObject->frame.get();
+      } else {
+        // it's different kind of HostObject. We don't support it.
+        return nullptr;
+      }
+
+    } else if (object.isFunction(runtime)) {
+      // jsi::Function
+
+      // TODO: Convert Function to Callback
+      return nullptr;
+
+    } else {
+      // jsi::Object
+
+      auto dynamic = jsi::dynamicFromValue(runtime, value);
+      auto map = react::ReadableNativeMap::createWithContents(std::move(dynamic));
+      return map.release();
+    }
+  } else {
+    // unknown jsi type!
+    auto stringRepresentation = value.toString(runtime).utf8(runtime);
+    auto message = "Received unknown JSI value! (" + stringRepresentation + ") Cannot convert to JNI Object.";
+    __android_log_write(ANDROID_LOG_ERROR, "VisionCamera", message.c_str());
+    return nullptr;
+  }
+}
+
+jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, const jni::local_ref<jobject>& object) {
+  if (object->isInstanceOf(jni::JBoolean::javaClassStatic())) {
+    // Boolean
+
+    static const auto getBooleanFunc = jni::findClassLocal("java/lang/Boolean")->getMethod<jboolean()>("booleanValue");
+    auto boolean = getBooleanFunc(object.get());
+    return jsi::Value(boolean == true);
+
+  } else if (object->isInstanceOf(jni::JDouble::javaClassStatic())) {
+    // Double
+
+    static const auto getDoubleFunc = jni::findClassLocal("java/lang/Double")->getMethod<jdouble()>("doubleValue");
+    auto d = getDoubleFunc(object.get());
+    return jsi::Value(d);
+
+  } else if (object->isInstanceOf(jni::JInteger::javaClassStatic())) {
+    // Integer
+
+    static const auto getIntegerFunc = jni::findClassLocal("java/lang/Integer")->getMethod<jint()>("integerValue");
+    auto i = getIntegerFunc(object.get());
+    return jsi::Value(i);
+
+  } else if (object->isInstanceOf(jni::JString::javaClassStatic())) {
+    // String
+
+    return jsi::String::createFromUtf8(runtime, object->toString());
+
+  } else if (object->isInstanceOf(JReadableArray::javaClassStatic())) {
+    // ReadableArray
+
+    static const auto toArrayListFunc = JReadableArray::javaClassLocal()->getMethod<jni::JArrayClass<jobject>()>("toArrayList");
+
+    auto array = toArrayListFunc(object.get());
+    auto size = array->size();
+
+    auto result = jsi::Array(runtime, size);
+    for (size_t i = 0; i < size; i++) {
+      result.setValueAtIndex(runtime, i, convertJNIObjectToJSIValue(runtime, (*array)[i]));
+    }
+    return result;
+
+  } else if (object->isInstanceOf(JArrayList::javaClassStatic())) {
+    // ArrayList
+
+    static const auto iteratorFunc = JArrayList::javaClassLocal()->getMethod<jni::JIterator<jobject>()>("iterator");
+    static const auto sizeFunc = JArrayList::javaClassLocal()->getMethod<jint()>("size");
+
+    auto iterator = iteratorFunc(object.get());
+    auto size = sizeFunc(object.get());
+
+    auto result = jsi::Array(runtime, size);
+    size_t i = 0;
+    for (auto& item : *iterator) {
+      result.setValueAtIndex(runtime, i, convertJNIObjectToJSIValue(runtime, item));
+      i++;
+    }
+    return result;
+
+  } else if (object->isInstanceOf(JReadableMap::javaClassStatic())) {
+    // ReadableMap
+
+    static const auto toHashMapFunc = JReadableMap::javaClassLocal()->getMethod<jni::JHashMap<jstring, jobject>()>("toHashMap");
+    auto hashMap = toHashMapFunc(object.get());
+
+    auto result = jsi::Object(runtime);
+
+    for (const auto& entry : *hashMap) {
+      auto key = entry.first->toString();
+      auto value = entry.second;
+      auto jsiValue = convertJNIObjectToJSIValue(runtime, value);
+      result.setProperty(runtime, key.c_str(), jsiValue);
+    }
+
+    return result;
+  }
+
+  auto type = object->getClass()->toString();
+  auto message = "Received unknown JNI type \"" + type + "\"! Cannot convert to jsi::Value.";
+  __android_log_write(ANDROID_LOG_ERROR, "VisionCamera", message.c_str());
+  return jsi::Value::undefined();
+}
+
+} // namespace vision

android/src/main/cpp/JSIJNIConversion.h

@@ -0,0 +1,21 @@
+//
+// Created by Marc Rousavy on 22.06.21.
+//
+
+#pragma once
+
+#include <jsi/jsi.h>
+#include <jni.h>
+#include <fbjni/fbjni.h>
+
+namespace vision {
+
+using namespace facebook;
+
+namespace JSIJNIConversion {
+  jobject convertJSIValueToJNIObject(jsi::Runtime& runtime, const jsi::Value& value); // NOLINT(runtime/references)
+
+  jsi::Value convertJNIObjectToJSIValue(jsi::Runtime& runtime, const jni::local_ref<jobject>& object); // NOLINT(runtime/references)
+};
+
+} // namespace vision

android/src/main/cpp/VisionCamera.cpp

@@ -0,0 +1,13 @@
+#include <jni.h>
+#include <fbjni/fbjni.h>
+#include "FrameProcessorRuntimeManager.h"
+#include "FrameProcessorPlugin.h"
+#include "CameraView.h"
+
+JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) {
+  return facebook::jni::initialize(vm, [] {
+    vision::FrameProcessorRuntimeManager::registerNatives();
+    vision::FrameProcessorPlugin::registerNatives();
+    vision::CameraView::registerNatives();
+  });
+}

android/src/main/cpp/reanimated-headers/AndroidErrorHandler.h

@@ -0,0 +1,30 @@
+// copied from https://github.com/software-mansion/react-native-reanimated/blob/master/android/src/main/cpp/headers/AndroidErrorHandler.h
+
+#pragma once
+
+#include "ErrorHandler.h"
+#include "AndroidScheduler.h"
+#include "Scheduler.h"
+#include <jni.h>
+#include <memory>
+#include <fbjni/fbjni.h>
+#include "Logger.h"
+
+namespace reanimated
+{
+
+  class AndroidErrorHandler : public JavaClass<AndroidErrorHandler>, public ErrorHandler {
+    std::shared_ptr<ErrorWrapper> error;
+    std::shared_ptr<Scheduler> scheduler;
+    void raiseSpec() override;
+  public:
+    static auto constexpr kJavaDescriptor = "Lcom/swmansion/reanimated/AndroidErrorHandler;";
+    AndroidErrorHandler(
+        std::shared_ptr<Scheduler> scheduler);
+    std::shared_ptr<Scheduler> getScheduler() override;
+    std::shared_ptr<ErrorWrapper> getError() override;
+    void setError(std::string message) override;
+    virtual ~AndroidErrorHandler() {}
+  };
+
+}

android/src/main/cpp/reanimated-headers/AndroidScheduler.h

@@ -0,0 +1,37 @@
+// copied from https://github.com/software-mansion/react-native-reanimated/blob/master/android/src/main/cpp/headers/AndroidScheduler.h
+
+#pragma once
+
+#include <jni.h>
+#include <fbjni/fbjni.h>
+#include <jsi/jsi.h>
+#include <react/jni/CxxModuleWrapper.h>
+#include <react/jni/JMessageQueueThread.h>
+#include "Scheduler.h"
+
+namespace reanimated {
+
+  using namespace facebook;
+
+  class AndroidScheduler : public jni::HybridClass<AndroidScheduler> {
+  public:
+    static auto constexpr kJavaDescriptor = "Lcom/swmansion/reanimated/Scheduler;";
+    static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis);
+    static void registerNatives();
+
+    std::shared_ptr<Scheduler> getScheduler() { return scheduler_; }
+
+    void scheduleOnUI();
+
+  private:
+    friend HybridBase;
+
+    void triggerUI();
+
+    jni::global_ref<AndroidScheduler::javaobject> javaPart_;
+    std::shared_ptr<Scheduler> scheduler_;
+
+    explicit AndroidScheduler(jni::alias_ref<AndroidScheduler::jhybridobject> jThis);
+  };
+
+}

android/src/main/java/com/mrousavy/camera/CameraView+TakePhoto.kt

@@ -17,8 +17,13 @@ import kotlin.system.measureTimeMillis
 
 @SuppressLint("UnsafeOptInUsageError")
 suspend fun CameraView.takePhoto(options: ReadableMap): WritableMap = coroutineScope {
+  if (fallbackToSnapshot) {
+    Log.i(CameraView.TAG, "takePhoto() called, but falling back to Snapshot because 1 use-case is already occupied.")
+    return@coroutineScope takeSnapshot(options)
+  }
+
   val startFunc = System.nanoTime()
-  Log.d(CameraView.TAG, "takePhoto() called")
+  Log.i(CameraView.TAG, "takePhoto() called")
   if (imageCapture == null) {
     if (photo == true) {
       throw CameraNotReadyError()

android/src/main/java/com/mrousavy/camera/CameraView+TakeSnapshot.kt

@@ -11,8 +11,17 @@ import kotlinx.coroutines.coroutineScope
 import kotlinx.coroutines.withContext
 import java.io.File
 import java.io.FileOutputStream
+import kotlinx.coroutines.guava.await
 
 suspend fun CameraView.takeSnapshot(options: ReadableMap): WritableMap = coroutineScope {
+  val camera = camera ?: throw com.mrousavy.camera.CameraNotReadyError()
+  val enableFlash = options.getString("flash") == "on"
+
+  try {
+    if (enableFlash) {
+      camera.cameraControl.enableTorch(true).await()
+    }
+
     val bitmap = this@takeSnapshot.previewView.bitmap ?: throw CameraNotReadyError()
 
     val quality = if (options.hasKey("quality")) options.getInt("quality") else 100
@@ -34,9 +43,16 @@ suspend fun CameraView.takeSnapshot(options: ReadableMap): WritableMap = corouti
     map.putInt("height", bitmap.height)
     map.putBoolean("isRawPhoto", false)
 
-  val skipMetadata = if (options.hasKey("skipMetadata")) options.getBoolean("skipMetadata") else false
+    val skipMetadata =
+      if (options.hasKey("skipMetadata")) options.getBoolean("skipMetadata") else false
     val metadata = if (skipMetadata) null else exif.buildMetadataMap()
     map.putMap("metadata", metadata)
 
     return@coroutineScope map
+  } finally {
+    if (enableFlash) {
+      // reset to `torch` property
+      camera.cameraControl.enableTorch(this@takeSnapshot.torch == "on")
+    }
+  }
 }

android/src/main/java/com/mrousavy/camera/CameraView.kt

@@ -21,6 +21,8 @@ import androidx.camera.lifecycle.ProcessCameraProvider
 import androidx.camera.view.PreviewView
 import androidx.core.content.ContextCompat
 import androidx.lifecycle.*
+import com.facebook.jni.HybridData
+import com.facebook.proguard.annotations.DoNotStrip
 import com.facebook.react.bridge.*
 import com.facebook.react.uimanager.events.RCTEventEmitter
 import com.mrousavy.camera.utils.*
@@ -82,11 +84,14 @@ class CameraView(context: Context) : FrameLayout(context), LifecycleOwner {
   var torch = "off"
   var zoom = 0.0 // in percent
   var enableZoomGesture = false
+  var frameProcessorFps = 1.0
 
   // private properties
   private val reactContext: ReactContext
     get() = context as ReactContext
 
+  private var enableFrameProcessor = false
+
   @Suppress("JoinDeclarationAndAssignment")
   internal val previewView: PreviewView
   private val cameraExecutor = Executors.newSingleThreadExecutor()
@@ -96,6 +101,7 @@ class CameraView(context: Context) : FrameLayout(context), LifecycleOwner {
   internal var camera: Camera? = null
   internal var imageCapture: ImageCapture? = null
   internal var videoCapture: VideoCapture? = null
+  internal var imageAnalysis: ImageAnalysis? = null
 
   private val scaleGestureListener: ScaleGestureDetector.SimpleOnScaleGestureListener
   private val scaleGestureDetector: ScaleGestureDetector
@@ -107,7 +113,42 @@ class CameraView(context: Context) : FrameLayout(context), LifecycleOwner {
   private var minZoom: Float = 1f
   private var maxZoom: Float = 1f
 
+  @DoNotStrip
+  private var mHybridData: HybridData?
+
+  @Suppress("LiftReturnOrAssignment", "RedundantIf")
+  internal val fallbackToSnapshot: Boolean
+    @SuppressLint("UnsafeOptInUsageError")
+    get() {
+      if (video != true && !enableFrameProcessor) {
+        // Both use-cases are disabled, so `photo` is the only use-case anyways. Don't need to fallback here.
+        return false
+      }
+      cameraId?.let { cameraId ->
+        val cameraManger = reactContext.getSystemService(Context.CAMERA_SERVICE) as? CameraManager
+        cameraManger?.let {
+          val characteristics = cameraManger.getCameraCharacteristics(cameraId)
+          val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)
+          if (hardwareLevel == CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY) {
+            // Camera only supports a single use-case at a time
+            return true
+          } else {
+            if (video == true && enableFrameProcessor) {
+              // Camera supports max. 2 use-cases, but both are occupied by `frameProcessor` and `video`
+              return true
+            } else {
+              // Camera supports max. 2 use-cases and only one is occupied (either `frameProcessor` or `video`), so we can add `photo`
+              return false
+            }
+          }
+        }
+      }
+      return false
+    }
+
   init {
+    mHybridData = initHybrid()
+
     previewView = PreviewView(context)
     previewView.layoutParams = LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT)
     previewView.installHierarchyFitter() // If this is not called correctly, view finder will be black/blank
@@ -144,6 +185,28 @@ class CameraView(context: Context) : FrameLayout(context), LifecycleOwner {
     })
   }
 
+  fun finalize() {
+    mHybridData?.resetNative()
+  }
+
+  private external fun initHybrid(): HybridData
+  private external fun frameProcessorCallback(frame: ImageProxy)
+
+  @Suppress("unused")
+  @DoNotStrip
+  fun setEnableFrameProcessor(enable: Boolean) {
+    Log.d(TAG, "Set enable frame processor: $enable")
+    val before = enableFrameProcessor
+    enableFrameProcessor = enable
+
+    if (before != enable) {
+      // reconfigure session if frame processor was added/removed to adjust use-cases.
+      GlobalScope.launch(Dispatchers.Main) {
+        configureSession()
+      }
+    }
+  }
+
   override fun getLifecycle(): Lifecycle {
     return lifecycleRegistry
   }
@@ -245,6 +308,10 @@ class CameraView(context: Context) : FrameLayout(context), LifecycleOwner {
         .setCaptureMode(ImageCapture.CAPTURE_MODE_MINIMIZE_LATENCY)
       val videoCaptureBuilder = VideoCapture.Builder()
         .setTargetRotation(rotation)
+      val imageAnalysisBuilder = ImageAnalysis.Builder()
+        .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
+        .setTargetRotation(rotation)
+        .setBackgroundExecutor(CameraViewModule.FrameProcessorThread)
 
       if (format == null) {
         // let CameraX automatically find best resolution for the target aspect ratio
@@ -311,11 +378,10 @@ class CameraView(context: Context) : FrameLayout(context), LifecycleOwner {
         }
       }
 
-      val preview = previewBuilder.build()
-
       // Unbind use cases before rebinding
       videoCapture = null
       imageCapture = null
+      imageAnalysis = null
       cameraProvider.unbindAll()
 
       // Bind use cases to camera
@@ -325,9 +391,32 @@ class CameraView(context: Context) : FrameLayout(context), LifecycleOwner {
         useCases.add(videoCapture!!)
       }
       if (photo == true) {
+        if (fallbackToSnapshot) {
+          Log.i(TAG, "Tried to add photo use-case (`photo={true}`) but the Camera device only supports " +
+            "a single use-case at a time. Falling back to Snapshot capture.")
+        } else {
           imageCapture = imageCaptureBuilder.build()
           useCases.add(imageCapture!!)
         }
+      }
+      if (enableFrameProcessor) {
+        var lastCall = System.currentTimeMillis() - 1000
+        val intervalMs = (1.0 / frameProcessorFps) * 1000.0
+        imageAnalysis = imageAnalysisBuilder.build().apply {
+          setAnalyzer(cameraExecutor, { image ->
+            val now = System.currentTimeMillis()
+            if (now - lastCall > intervalMs) {
+              lastCall = now
+              Log.d(TAG, "Calling Frame Processor...")
+              frameProcessorCallback(image)
+            }
+            image.close()
+          })
+        }
+        useCases.add(imageAnalysis!!)
+      }
+
+      val preview = previewBuilder.build()
       camera = cameraProvider.bindToLifecycle(this, cameraSelector, preview, *useCases.toTypedArray())
       preview.setSurfaceProvider(previewView.surfaceProvider)
 
@@ -338,11 +427,18 @@ class CameraView(context: Context) : FrameLayout(context), LifecycleOwner {
       Log.i(TAG_PERF, "Session configured in $duration ms! Camera: ${camera!!}")
       invokeOnInitialized()
     } catch (exc: Throwable) {
-      throw when (exc) {
+      val error = when (exc) {
         is CameraError -> exc
-        is IllegalArgumentException -> InvalidCameraDeviceError(exc)
+        is IllegalArgumentException -> {
+          if (exc.message?.contains("too many use cases") == true) {
+            ParallelVideoProcessingNotSupportedError(exc)
+          } else {
+            InvalidCameraDeviceError(exc)
+          }
+        }
         else -> UnknownCameraError(exc)
       }
+      invokeOnError(error)
     }
   }
 
@@ -381,7 +477,7 @@ class CameraView(context: Context) : FrameLayout(context), LifecycleOwner {
     const val TAG = "CameraView"
     const val TAG_PERF = "CameraView.performance"
 
-    private val propsThatRequireSessionReconfiguration = arrayListOf("cameraId", "format", "fps", "hdr", "lowLightBoost", "photo", "video")
+    private val propsThatRequireSessionReconfiguration = arrayListOf("cameraId", "format", "fps", "hdr", "lowLightBoost", "photo", "video", "frameProcessorFps")
 
     private val arrayListOfZoom = arrayListOf("zoom")
   }

android/src/main/java/com/mrousavy/camera/CameraViewManager.kt

@@ -1,7 +1,6 @@
 package com.mrousavy.camera
 
 import android.util.Log
-import com.facebook.react.bridge.ReadableArray
 import com.facebook.react.bridge.ReadableMap
 import com.facebook.react.common.MapBuilder
 import com.facebook.react.uimanager.SimpleViewManager
@@ -81,6 +80,13 @@ class CameraViewManager : SimpleViewManager<CameraView>() {
     view.fps = if (fps > 0) fps else null
   }
 
+  @ReactProp(name = "frameProcessorFps", defaultDouble = 1.0)
+  fun setFrameProcessorFps(view: CameraView, frameProcessorFps: Double) {
+    if (view.frameProcessorFps != frameProcessorFps)
+      addChangedPropToTransaction(view, "frameProcessorFps")
+    view.frameProcessorFps = frameProcessorFps
+  }
+
   @ReactProp(name = "hdr")
   fun setHdr(view: CameraView, hdr: Boolean?) {
     if (view.hdr != hdr)

android/src/main/java/com/mrousavy/camera/CameraViewModule.kt

@@ -17,8 +17,11 @@ import androidx.core.content.ContextCompat
 import com.facebook.react.bridge.*
 import com.facebook.react.modules.core.PermissionAwareActivity
 import com.facebook.react.modules.core.PermissionListener
+import com.mrousavy.camera.frameprocessor.FrameProcessorRuntimeManager
 import com.mrousavy.camera.parsers.*
 import com.mrousavy.camera.utils.*
+import java.util.concurrent.ExecutorService
+import java.util.concurrent.Executors
 import kotlinx.coroutines.*
 import kotlinx.coroutines.guava.await
 
@@ -26,6 +29,7 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase
   companion object {
     const val REACT_CLASS = "CameraView"
     var RequestCode = 10
+    val FrameProcessorThread: ExecutorService = Executors.newSingleThreadExecutor()
 
     fun parsePermissionStatus(status: Int): String {
       return when (status) {
@@ -36,6 +40,23 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase
     }
   }
 
+  private var frameProcessorManager: FrameProcessorRuntimeManager? = null
+
+  override fun initialize() {
+    super.initialize()
+    FrameProcessorThread.execute {
+      frameProcessorManager = FrameProcessorRuntimeManager(reactApplicationContext)
+      reactApplicationContext.runOnJSQueueThread {
+        frameProcessorManager!!.installJSIBindings()
+      }
+    }
+  }
+
+  override fun onCatalystInstanceDestroy() {
+    super.onCatalystInstanceDestroy()
+    frameProcessorManager?.destroy()
+  }
+
   override fun getName(): String {
     return REACT_CLASS
   }
@@ -73,7 +94,7 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase
         val map = makeErrorMap("${error.domain}/${error.id}", error.message, error)
         onRecordCallback(null, map)
       } catch (error: Throwable) {
-        val map = makeErrorMap("capture/unknown", "An unknown error occured while trying to start a video recording!", error)
+        val map = makeErrorMap("capture/unknown", "An unknown error occurred while trying to start a video recording!", error)
         onRecordCallback(null, map)
       }
     }
@@ -149,6 +170,8 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase
           val supportsHdr = hdrExtension.isExtensionAvailable(cameraSelector)
           val nightExtension = NightImageCaptureExtender.create(imageCaptureBuilder)
           val supportsLowLightBoost = nightExtension.isExtensionAvailable(cameraSelector)
+          // see https://developer.android.com/reference/android/hardware/camera2/CameraDevice#regular-capture
+          val supportsParallelVideoProcessing = hardwareLevel != CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY && hardwareLevel != CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LIMITED
 
           val fieldOfView = characteristics.getFieldOfView()
 
@@ -160,7 +183,7 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase
           map.putBoolean("hasFlash", hasFlash)
           map.putBoolean("hasTorch", hasFlash)
           map.putBoolean("isMultiCam", isMultiCam)
-          map.putBoolean("supportsPhotoAndVideoCapture", hardwareLevel != CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY)
+          map.putBoolean("supportsParallelVideoProcessing", supportsParallelVideoProcessing)
           map.putBoolean("supportsRawCapture", supportsRawCapture)
           map.putBoolean("supportsDepthCapture", supportsDepthCapture)
           map.putBoolean("supportsLowLightBoost", supportsLowLightBoost)

android/src/main/java/com/mrousavy/camera/Errors.kt

@@ -37,6 +37,9 @@ class InvalidTypeScriptUnionError(unionName: String, unionValue: String) : Camer
 
 class NoCameraDeviceError : CameraError("device", "no-device", "No device was set! Use `getAvailableCameraDevices()` to select a suitable Camera device.")
 class InvalidCameraDeviceError(cause: Throwable) : CameraError("device", "invalid-device", "The given Camera device could not be found for use-case binding!", cause)
+class ParallelVideoProcessingNotSupportedError(cause: Throwable) : CameraError("device", "parallel-video-processing-not-supported", "The given LEGACY Camera device does not support parallel " +
+  "video processing (`video={true}` + `frameProcessor={...}`). Disable either `video` or `frameProcessor`. To find out if a device supports parallel video processing, check the `supportsParallelVideoProcessing` property on the CameraDevice. " +
+  "See https://mrousavy.github.io/react-native-vision-camera/docs/guides/lifecycle#the-supportsparallelvideoprocessing-prop for more information.", cause)
 
 class FpsNotContainedInFormatError(fps: Int) : CameraError("format", "invalid-fps", "The given FPS were not valid for the currently selected format. Make sure you select a format which `frameRateRanges` includes $fps FPS!")
 class HdrNotContainedInFormatError() : CameraError(

android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java

@@ -0,0 +1,54 @@
+package com.mrousavy.camera.frameprocessor;
+
+import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
+import androidx.camera.core.ImageProxy;
+import com.facebook.jni.HybridData;
+
+/**
+ * Declares a Frame Processor Plugin.
+ */
+public abstract class FrameProcessorPlugin {
+    static {
+        System.loadLibrary("VisionCamera");
+    }
+
+    @SuppressWarnings({"FieldCanBeLocal", "unused"})
+    private final HybridData mHybridData;
+
+    /**
+     * The actual Frame Processor plugin callback. Called for every frame the ImageAnalyzer receives.
+     * @param image The CameraX ImageProxy. Don't call .close() on this, as VisionCamera handles that.
+     * @return You can return any primitive, map or array you want. See the
+     * <a href="https://mrousavy.github.io/react-native-vision-camera/docs/guides/frame-processors-plugins-overview#types">Types</a>
+     * table for a list of supported types.
+     */
+    public abstract @Nullable Object callback(@NonNull ImageProxy image, @NonNull Object[] params);
+
+    /**
+     * Initializes the native plugin part.
+     * @param name Specifies the Frame Processor Plugin's name in the Runtime.
+     *             The actual name in the JS Runtime will be prefixed with two underscores (`__`)
+     */
+    protected FrameProcessorPlugin(@NonNull String name) {
+        mHybridData = initHybrid(name);
+    }
+
+    @Override
+    protected void finalize() throws Throwable {
+        super.finalize();
+        if (mHybridData != null) {
+            mHybridData.resetNative();
+        }
+    }
+
+    private native @NonNull HybridData initHybrid(@NonNull String name);
+
+    /**
+     * Registers the given plugin in the Frame Processor Runtime.
+     * @param plugin An instance of a plugin.
+     */
+    public static void register(@NonNull FrameProcessorPlugin plugin) {
+        FrameProcessorRuntimeManager.Companion.getPlugins().add(plugin);
+    }
+}

android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager.kt

@@ -0,0 +1,75 @@
+package com.mrousavy.camera.frameprocessor
+
+import android.util.Log
+import com.facebook.jni.HybridData
+import com.facebook.proguard.annotations.DoNotStrip
+import com.facebook.react.bridge.ReactApplicationContext
+import com.facebook.react.turbomodule.core.CallInvokerHolderImpl
+import com.mrousavy.camera.CameraView
+import com.mrousavy.camera.ViewNotFoundError
+import com.swmansion.reanimated.Scheduler
+import java.lang.ref.WeakReference
+
+class FrameProcessorRuntimeManager(context: ReactApplicationContext) {
+  companion object {
+    const val TAG = "FrameProcessorRuntime"
+    private var HasRegisteredPlugins = false
+    val Plugins: ArrayList<FrameProcessorPlugin> = ArrayList()
+      get() {
+        if (HasRegisteredPlugins) {
+          throw Error("Tried to access Frame Processor Plugin list, " +
+            "but plugins have already been registered (list is frozen now!).")
+        }
+        return field
+      }
+
+    init {
+      System.loadLibrary("reanimated")
+      System.loadLibrary("VisionCamera")
+    }
+  }
+
+  @DoNotStrip
+  private var mHybridData: HybridData?
+  private var mContext: WeakReference<ReactApplicationContext>?
+  private var mScheduler: Scheduler?
+
+  init {
+    val holder = context.catalystInstance.jsCallInvokerHolder as CallInvokerHolderImpl
+    mScheduler = Scheduler(context)
+    mContext = WeakReference(context)
+    mHybridData = initHybrid(context.javaScriptContextHolder.get(), holder, mScheduler!!)
+    initializeRuntime()
+
+    Log.i(TAG, "Installing Frame Processor Plugins...")
+    Plugins.forEach { plugin ->
+      registerPlugin(plugin)
+    }
+    Log.i(TAG, "Successfully installed ${Plugins.count()} Frame Processor Plugins!")
+    HasRegisteredPlugins = true
+  }
+
+  fun destroy() {
+    mScheduler?.deactivate()
+    mHybridData?.resetNative()
+  }
+
+  fun findCameraViewById(viewId: Int): CameraView {
+    Log.d(TAG, "finding view $viewId...")
+    val view = mContext?.get()?.currentActivity?.findViewById<CameraView>(viewId)
+    Log.d(TAG, "found view $viewId! is null: ${view == null}")
+    return view ?: throw ViewNotFoundError(viewId)
+  }
+
+  // private C++ funcs
+  private external fun initHybrid(
+    jsContext: Long,
+    jsCallInvokerHolder: CallInvokerHolderImpl,
+    scheduler: Scheduler
+  ): HybridData?
+  private external fun initializeRuntime()
+  private external fun registerPlugin(plugin: FrameProcessorPlugin)
+
+  // public C++ funcs
+  external fun installJSIBindings()
+}

android/src/main/java/com/mrousavy/camera/utils/ImageProxy.save.kt

@@ -14,7 +14,6 @@ import java.io.FileOutputStream
 import java.nio.ByteBuffer
 import kotlin.system.measureTimeMillis
 
-
 // TODO: Fix this flip() function (this outputs a black image)
 fun flip(imageBytes: ByteArray, imageWidth: Int): ByteArray {
   // separate out the sub arrays

cpp/MakeJSIRuntime.h

@@ -3,6 +3,22 @@
 #include <jsi/jsi.h>
 #include <memory>
 
+#ifdef ON_ANDROID
+
+// on Android we need to pass FOR_HERMES flag to determine if hermes is used or not since both headers are there.
+
+#if FOR_HERMES
+// Hermes
+#include <hermes/hermes.h>
+#else
+// JSC
+#include <jsi/JSCRuntime.h>
+#endif
+
+#else
+
+// on iOS, we simply check by __has_include. Headers are only available if the sources are there too.
+
 #if __has_include(<hermes/hermes.h>)
 // Hermes (https://hermesengine.dev)
 #include <hermes/hermes.h>
@@ -14,11 +30,23 @@
 #include <jsi/JSCRuntime.h>
 #endif
 
+#endif
+
 using namespace facebook;
 
 namespace vision {
 
 static std::unique_ptr<jsi::Runtime> makeJSIRuntime() {
+#ifdef ON_ANDROID
+
+  #if FOR_HERMES
+  return facebook::hermes::makeHermesRuntime();
+  #else
+  return facebook::jsc::makeJSCRuntime();
+  #endif
+
+#else
+
   #if __has_include(<hermes/hermes.h>)
   return facebook::hermes::makeHermesRuntime();
   #elif __has_include(<v8runtime/V8RuntimeFactory.h>)
@@ -26,6 +54,8 @@ static std::unique_ptr<jsi::Runtime> makeJSIRuntime() {
   #else
   return facebook::jsc::makeJSCRuntime();
   #endif
+
+#endif
 }
 
 } // namespace vision

docs/docs/guides/CAPTURING.mdx

@@ -73,7 +73,7 @@ While taking snapshots is faster than taking photos, the resulting image has way
 :::
 
 :::note
-The `takeSnapshot` function also works with `photo={false}`. For this reason, devices that do not support photo and video capture at the same time (see ["The `supportsPhotoAndVideoCapture` prop"](/docs/guides/devices/#the-supportsphotoandvideocapture-prop)) can use `video={true}` and fall back to snapshot capture for photos. (See ["Taking Snapshots"](/docs/guides/capturing#taking-snapshots))
+The `takeSnapshot` function also works with `photo={false}`. For this reason VisionCamera will automatically fall-back to snapshot capture if you are trying to use more use-cases than the Camera natively supports. (see ["The `supportsParallelVideoProcessing` prop"](/docs/guides/devices#the-supportsparallelvideoprocessing-prop))
 :::
 
 ## Recording Videos

docs/docs/guides/DEVICES.mdx

@@ -98,13 +98,25 @@ function App() {
 }
 ```
 
-### The `supportsPhotoAndVideoCapture` prop
+### The `supportsParallelVideoProcessing` prop
 
-Camera devices provide the [`supportsPhotoAndVideoCapture` property](/docs/api/interfaces/cameradevice.cameradevice-1#supportsphotoandvideocapture) which determines whether the device supports enabling photo- and video-capture at the same time.
+Camera devices provide the [`supportsParallelVideoProcessing` property](/docs/api/interfaces/cameradevice.cameradevice-1#supportsparallelvideoprocessing) which determines whether the device supports using Video Recordings (`video={true}`) and Frame Processors (`frameProcessor={...}`) at the same time.
-While every iOS device supports this feature, there are some older Android devices which only allow enabling one of each - either photo capture or video capture. (Those are `LEGACY` devices, see [this table](https://developer.android.com/reference/android/hardware/camera2/CameraDevice#regular-capture).)
+
+If this property is `false`, you can only enable `video` or add a `frameProcessor`, but not both.
+
+* On iOS this value is always `true`.
+* On newer Android devices this value is always `true`.
+* On older Android devices this value is `false` if the Camera's hardware level is `LEGACY` or `LIMITED`, `true` otherwise. (See [`INFO_SUPPORTED_HARDWARE_LEVEL`](https://developer.android.com/reference/android/hardware/camera2/CameraCharacteristics#INFO_SUPPORTED_HARDWARE_LEVEL) or [the tables at "Regular capture"](https://developer.android.com/reference/android/hardware/camera2/CameraDevice#regular-capture))
+
+#### Examples
+
+* An app that only supports **taking photos** works on every Camera device because this only affects video processing.
+* An app that supports **taking photos** and **videos** works on every Camera device because only a single video processing feature is used (`video`).
+* An app that only uses **Frame Processors** (no taking photos or videos) works on every Camera device because it only uses a single video processing feature (`frameProcessor`).
+* An app that uses **Frame Processors** and supports **taking photos** and **videos** only works on Camera devices where `supportsParallelVideoProcessing` is `true`. (iPhones and newer Android Phones)
 
 :::note
-If `supportsPhotoAndVideoCapture` is `false` but you still need photo- and video-capture at the same time, you can fall back to _snapshot capture_ (see [**"Taking Snapshots"**](/docs/guides/capturing#taking-snapshots)) instead.
+Actually the limitation also affects the `photo` feature, but VisionCamera will automatically fall-back to **Snapshot capture** if you are trying to use multiple features (`photo` + `video` + `frameProcessor`) and they are not natively supported. (See ["Taking Snapshots"](/docs/guides/capturing#taking-snapshots))
 :::
 
 <br />

docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx

@@ -34,11 +34,11 @@ The Frame Processor Plugin Registry API automatically manages type conversion fr
 
 | JS Type              | Objective-C Type              | Java Type                  |
 |----------------------|-------------------------------|----------------------------|
-| `number`             | `NSNumber*` (double)          | `double`                   |
+| `number`             | `NSNumber*` (double)          | `Double`                   |
-| `boolean`            | `NSNumber*` (boolean)         | `boolean`                  |
+| `boolean`            | `NSNumber*` (boolean)         | `Boolean`                  |
 | `string`             | `NSString*`                   | `String`                   |
-| `[]`                 | `NSArray*`                    | `Array<Object>`            |
+| `[]`                 | `NSArray*`                    | `ReadableNativeArray`      |
-| `{}`                 | `NSDictionary*`               | `HashMap<Object>`          |
+| `{}`                 | `NSDictionary*`               | `ReadableNativeMap`        |
 | `undefined` / `null` | `nil`                         | `null`                     |
 | `(any, any) => void` | [`RCTResponseSenderBlock`][4] | `(Object, Object) -> void` |
 | [`Frame`][1]         | [`Frame*`][2]                 | [`ImageProxy`][3]          |

docs/docs/guides/FRAME_PROCESSOR_CREATE_FINAL.mdx

@@ -63,11 +63,12 @@ If you want to distribute your Frame Processor Plugin, simply use npm.
 
 1. Create a blank Native Module using [bob](https://github.com/callstack/react-native-builder-bob) or [create-react-native-module](https://github.com/brodybits/create-react-native-module)
 2. Name it `vision-camera-plugin-xxxxx` where `xxxxx` is the name of your plugin
-3. Remove all the source files for the Example Native Module
+3. Remove the generated template code from the Example Native Module
-4. Implement the Frame Processor Plugin in the iOS, Android and JS/TS Codebase using the guides above
+4. Add VisionCamera to `peerDependencies`: `"react-native-vision-camera": ">= 2"`
-5. Add installation instructions to let users know they have to add your frame processor in the `babel.config.js` configuration.
+5. Implement the Frame Processor Plugin in the iOS, Android and JS/TS Codebase using the guides above
-6. Publish the plugin to npm. Users will only have to install the plugin using `npm i vision-camera-plugin-xxxxx` and add it to their `babel.config.js` file.
+6. Add installation instructions to the `README.md` to let users know they have to add your frame processor in the `babel.config.js` configuration.
-7. [Add the plugin to the **official VisionCamera plugin list**](https://github.com/mrousavy/react-native-vision-camera/edit/main/docs/docs/guides/FRAME_PROCESSOR_PLUGIN_LIST.mdx) for more visibility
+7. Publish the plugin to npm. Users will only have to install the plugin using `npm i vision-camera-plugin-xxxxx` and add it to their `babel.config.js` file.
+8. [Add the plugin to the **official VisionCamera plugin list**](https://github.com/mrousavy/react-native-vision-camera/edit/main/docs/docs/guides/FRAME_PROCESSOR_PLUGIN_LIST.mdx) for more visibility
 
 <br />
 

docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx

@@ -4,14 +4,126 @@ title: Creating Frame Processor Plugins for Android
 sidebar_label: Creating Frame Processor Plugins (Android)
 ---
 
-import useBaseUrl from '@docusaurus/useBaseUrl';
 import Tabs from '@theme/Tabs';
 import TabItem from '@theme/TabItem';
 
-:::warning
+## Creating a Frame Processor
-Frame Processors are not yet available for Android.
+
+The Frame Processor Plugin API is built to be as extensible as possible, which allows you to create custom Frame Processor Plugins.
+In this guide we will create a custom QR Code Scanner Plugin which can be used from JS.
+
+Android Frame Processor Plugins can be written in either **Java**, **Kotlin** or **C++ (JNI)**.
+
+<Tabs
+  defaultValue="java"
+  values={[
+    {label: 'Java', value: 'java'},
+    {label: 'Kotlin', value: 'kotlin'}
+  ]}>
+<TabItem value="java">
+
+1. Open your Project in Android Studio
+2. Create a Java source file, for the QR Code Plugin this will be called `QRCodeFrameProcessorPlugin.java`.
+3. Add the following code:
+
+```java {8}
+import androidx.camera.core.ImageProxy;
+import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin;
+
+public class QRCodeFrameProcessorPlugin extends FrameProcessorPlugin {
+
+  @Override
+  public Object callback(ImageProxy image, Object[] params) {
+    // code goes here
+    return null;
+  }
+
+  QRCodeFrameProcessorPlugin() {
+    super("scanQRCodes");
+  }
+}
+```
+
+:::note
+The JS function name will be equal to the name you pass to the `super(...)` call (with a `__` prefix). Make sure it is unique across other Frame Processor Plugins.
 :::
 
+4. **Implement your Frame Processing.** See the [Example Plugin (Java)](https://github.com/mrousavy/react-native-vision-camera/blob/main/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java) for reference.
+5. Create a new Java file which registers the Frame Processor Plugin in a React Package, for the QR Code Scanner plugin this file will be called `QRCodeFrameProcessorPluginPackage.java`:
+
+```java {12}
+import com.facebook.react.ReactPackage;
+import com.facebook.react.bridge.NativeModule;
+import com.facebook.react.bridge.ReactApplicationContext;
+import com.facebook.react.uimanager.ViewManager;
+import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin;
+import javax.annotation.Nonnull;
+
+public class QRCodeFrameProcessorPluginPackage implements ReactPackage {
+  @NonNull
+  @Override
+  public List<NativeModule> createNativeModules(@NonNull ReactApplicationContext reactContext) {
+    FrameProcessorPlugin.register(new QRCodeFrameProcessorPlugin());
+    return Collections.emptyList();
+  }
+
+  @Nonnull
+  @Override
+  public List<ViewManager> createViewManagers(@Nonnull ReactApplicationContext reactContext) {
+    return Collections.emptyList();
+  }
+}
+```
+
+</TabItem>
+<TabItem value="kotlin">
+
+1. Open your Project in Android Studio
+2. Create a Kotlin source file, for the QR Code Plugin this will be called `QRCodeFrameProcessorPlugin.kt`.
+3. Add the following code:
+
+```kotlin {7}
+import androidx.camera.core.ImageProxy
+import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin
+
+class ExampleFrameProcessorPluginKotlin: FrameProcessorPlugin("scanQRCodes") {
+
+  override fun callback(image: ImageProxy, params: Array<Any>): Any? {
+    // code goes here
+    return null
+  }
+}
+```
+
+:::note
+The JS function name will be equal to the name you pass to the `FrameProcessorPlugin(...)` call (with a `__` prefix). Make sure it is unique across other Frame Processor Plugins.
+:::
+
+4. **Implement your Frame Processing.** See the [Example Plugin (Java)](https://github.com/mrousavy/react-native-vision-camera/blob/main/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java) for reference.
+5. Create a new Kotlin file which registers the Frame Processor Plugin in a React Package, for the QR Code Scanner plugin this file will be called `QRCodeFrameProcessorPluginPackage.kt`:
+
+```kotlin {9}
+import com.facebook.react.ReactPackage
+import com.facebook.react.bridge.NativeModule
+import com.facebook.react.bridge.ReactApplicationContext
+import com.facebook.react.uimanager.ViewManager
+import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin
+
+class QRCodeFrameProcessorPluginPackage : ReactPackage {
+  override fun createNativeModules(reactContext: ReactApplicationContext): List<NativeModule> {
+    FrameProcessorPlugin.register(ExampleFrameProcessorPluginKotlin())
+    return emptyList()
+  }
+
+  override fun createViewManagers(reactContext: ReactApplicationContext): List<ViewManager<*, *>> {
+    return emptyList()
+  }
+}
+```
+
+</TabItem>
+</Tabs>
+
 <br />
 
 #### 🚀 Next section: [Finish creating your Frame Processor Plugin](frame-processors-plugins-final) (or [add iOS support to your Frame Processor Plugin](frame-processors-plugins-ios))

docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx

@@ -48,7 +48,7 @@ VISION_EXPORT_FRAME_PROCESSOR(scanQRCodes)
 @end
 ```
 
-4. **Implement your Frame Processing.** See the [QR Code Plugin (Objective-C)](https://github.com/mrousavy/react-native-vision-camera/blob/main/example/ios/Frame%20Processor%20Plugins/QR%20Code%20Plugin%20%28Objective%2DC%29) for reference.
+4. **Implement your Frame Processing.** See the [Example Plugin (Objective-C)](https://github.com/mrousavy/react-native-vision-camera/blob/main/example/ios/Frame%20Processor%20Plugins/Example%20Plugin%20%28Objective%2DC%29) for reference.
 
 :::note
 The JS function name will be equal to the Objective-C function name you choose (with a `__` prefix). Make sure it is unique across other Frame Processor Plugins.
@@ -57,18 +57,19 @@ The JS function name will be equal to the Objective-C function name you choose (
 </TabItem>
 <TabItem value="swift">
 
-1. Create a Swift file, for the QR Code Plugin this will be `QRCodeFrameProcessorPlugin.swift`. If Xcode asks you to create a Bridging Header, press **create**.
+1. Open your Project in Xcode
+2. Create a Swift file, for the QR Code Plugin this will be `QRCodeFrameProcessorPlugin.swift`. If Xcode asks you to create a Bridging Header, press **create**.
 
 ![Xcode "Create Bridging Header" alert](https://docs-assets.developer.apple.com/published/7ebca7212c/2a065d1a-7e53-4907-a889-b7fa4f2206c9.png)
 
-2. Inside the newly created Bridging Header, add the following code:
+3. Inside the newly created Bridging Header, add the following code:
 
 ```objc
 #import <VisionCamera/FrameProcessorPlugin.h>
 #import <VisionCamera/Frame.h>
 ```
 
-3. Create an Objective-C source file with the same name as the Swift file, for the QR Code Plugin this will be `QRCodeFrameProcessorPlugin.m`. Add the following code:
+4. Create an Objective-C source file with the same name as the Swift file, for the QR Code Plugin this will be `QRCodeFrameProcessorPlugin.m`. Add the following code:
 
 ```objc
 #import <VisionCamera/FrameProcessorPlugin.h>
@@ -81,7 +82,7 @@ The JS function name will be equal to the Objective-C function name you choose (
 The first parameter in the Macro specifies the JS function name. Make sure it is unique across other Frame Processors.
 :::
 
-4. In the Swift file, add the following code:
+5. In the Swift file, add the following code:
 
 ```swift {8}
 @objc(QRCodeFrameProcessorPlugin)
@@ -97,7 +98,7 @@ public class QRCodeFrameProcessorPlugin: NSObject, FrameProcessorPluginBase {
 }
 ```
 
-5. **Implement your frame processing.** See [QR Code Plugin (Swift)](https://github.com/mrousavy/react-native-vision-camera/blob/main/example/ios/Frame%20Processor%20Plugins/QR%20Code%20Plugin%20%28Swift%29) for reference.
+6. **Implement your frame processing.** See [Example Plugin (Swift)](https://github.com/mrousavy/react-native-vision-camera/blob/main/example/ios/Frame%20Processor%20Plugins/Example%20Plugin%20%28Swift%29) for reference.
 
 
 </TabItem>

docs/docusaurus.config.js

@@ -14,7 +14,7 @@ module.exports = {
       indexName: 'react-native-vision-camera',
     },
     prism: {
-      additionalLanguages: ['swift'],
+      additionalLanguages: ['swift', 'java', 'kotlin'],
     },
     navbar: {
       title: 'VisionCamera',

example/android/app/build.gradle

@@ -131,6 +131,10 @@ android {
         targetCompatibility JavaVersion.VERSION_1_8
     }
 
+    dexOptions {
+        javaMaxHeapSize "4g"
+    }
+
     defaultConfig {
         applicationId "com.mrousavy.camera.example"
         minSdkVersion rootProject.ext.minSdkVersion
@@ -179,6 +183,9 @@ android {
 
         }
     }
+    packagingOptions {
+        pickFirst '**/*.so'
+    }
 }
 
 dependencies {
@@ -198,6 +205,7 @@ dependencies {
     }
 
     implementation project(':camera')
+    implementation "androidx.camera:camera-core:1.1.0-alpha05"
 }
 
 // Run this once to be able to run the application with BUCK

example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java

@@ -0,0 +1,36 @@
+package com.mrousavy.camera.example;
+
+import android.util.Log;
+import androidx.camera.core.ImageProxy;
+import com.facebook.react.bridge.WritableNativeArray;
+import com.facebook.react.bridge.WritableNativeMap;
+import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin;
+import org.jetbrains.annotations.NotNull;
+
+public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin {
+    @Override
+    public Object callback(@NotNull ImageProxy image, @NotNull Object[] params) {
+        Log.d("ExamplePlugin", image.getWidth() + " x " + image.getHeight() + " Image with format #" + image.getFormat() + ". Logging " + params.length + " parameters:");
+
+        for (Object param : params) {
+            Log.d("ExamplePlugin", "  -> " + (param == null ? "(null)" : param.toString() + " (" + param.getClass().getName() + ")"));
+        }
+
+        WritableNativeMap map = new WritableNativeMap();
+        map.putString("example_str", "Test");
+        map.putBoolean("example_bool", true);
+        map.putDouble("example_double", 5.3);
+
+        WritableNativeArray array = new WritableNativeArray();
+        array.pushString("Hello!");
+        array.pushBoolean(true);
+        array.pushDouble(17.38);
+
+        map.putArray("example_array", array);
+        return map;
+    }
+
+    ExampleFrameProcessorPlugin() {
+        super("example_plugin");
+    }
+}

example/android/app/src/main/java/com/mrousavy/camera/example/MainApplication.java

@@ -1,13 +1,12 @@
 package com.mrousavy.camera.example;
 
-import android.content.Context;
 import com.facebook.react.PackageList;
+import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin;
 import com.reactnativenavigation.NavigationApplication;
 import com.facebook.react.ReactNativeHost;
 import com.reactnativenavigation.react.NavigationReactNativeHost;
 import com.facebook.react.ReactPackage;
-import com.facebook.react.ReactInstanceManager;
+
-import java.lang.reflect.InvocationTargetException;
 import java.util.List;
 import com.mrousavy.camera.CameraPackage;
 
@@ -52,5 +51,7 @@ public class MainApplication extends NavigationApplication {
   @Override
   public void onCreate() {
     super.onCreate();
+    // register VisionCamera Frame Processor Plugins here.
+    FrameProcessorPlugin.register(new ExampleFrameProcessorPlugin());
   }
 }

example/babel.config.js

@@ -8,7 +8,7 @@ module.exports = {
     [
       'react-native-reanimated/plugin',
       {
-        globals: ['__exampleSwift___scanQRCodes', '__exampleObjC___scanQRCodes'],
+        globals: ['__example_plugin', '__example_plugin_swift'],
       },
     ],
     [

example/clean.sh

@@ -13,6 +13,13 @@ rm -rf ios/Pods
 echo "rm -rf ~/Library/Developer/Xcode/DerivedData/*"
 rm -rf ~/Library/Developer/Xcode/DerivedData/*
 
+echo "rm -rf android/.cxx"
+rm -rf android/.cxx
+echo "rm -rf android/.gradle"
+rm -rf android/.gradle
+echo "rm -rf android/build"
+rm -rf android/build
+
 cd ios
 echo "pod deintegrate"
 pod deintegrate

example/ios/Frame Processor Plugins/Example Plugin (Objective-C)/ExampleFrameProcessorPlugin.m

@@ -0,0 +1,41 @@
+//
+//  ExampleFrameProcessorPlugin.m
+//  VisionCameraExample
+//
+//  Created by Marc Rousavy on 01.05.21.
+//
+
+#import <Foundation/Foundation.h>
+#import <VisionCamera/FrameProcessorPlugin.h>
+#import <VisionCamera/Frame.h>
+
+// Example for an Objective-C Frame Processor plugin
+
+@interface ExampleFrameProcessorPlugin : NSObject
+@end
+
+@implementation ExampleFrameProcessorPlugin
+
+static inline id example_plugin(Frame* frame, NSArray* arguments) {
+  CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
+  NSLog(@"ExamplePlugin: %zu x %zu Image. Logging %lu parameters:", CVPixelBufferGetWidth(imageBuffer), CVPixelBufferGetHeight(imageBuffer), (unsigned long)arguments.count);
+  
+  for (id param in arguments) {
+    NSLog(@"ExamplePlugin:   -> %@ (%@)", param == nil ? @"(nil)" : [param description], NSStringFromClass([param classForCoder]));
+  }
+  
+  return @{
+    @"example_str": @"Test",
+    @"example_bool": @true,
+    @"example_double": @5.3,
+    @"example_array": @[
+      @"Hello",
+      @true,
+      @17.38
+    ]
+  };
+}
+
+VISION_EXPORT_FRAME_PROCESSOR(example_plugin)
+
+@end

example/ios/Frame Processor Plugins/Example Plugin (Swift)/ExamplePluginSwift.m

@@ -1,5 +1,5 @@
 //
-//  QRCodeFrameProcessorPluginSwift.m
+//  ExamplePluginSwift.m
 //  VisionCamera
 //
 //  Created by Marc Rousavy on 01.05.21.
@@ -9,5 +9,5 @@
 #import <Foundation/Foundation.h>
 #import <VisionCamera/FrameProcessorPlugin.h>
 
-@interface VISION_EXPORT_SWIFT_FRAME_PROCESSOR(exampleSwift___scanQRCodes, QRCodeFrameProcessorPluginSwift)
+@interface VISION_EXPORT_SWIFT_FRAME_PROCESSOR(example_plugin_swift, ExamplePluginSwift)
 @end

example/ios/Frame Processor Plugins/Example Plugin (Swift)/ExamplePluginSwift.swift

@@ -0,0 +1,42 @@
+//
+//  ExamplePluginSwift.swift
+//  VisionCamera
+//
+//  Created by Marc Rousavy on 30.04.21.
+//  Copyright © 2021 mrousavy. All rights reserved.
+//
+
+import AVKit
+import Vision
+
+@objc(ExamplePluginSwift)
+public class ExamplePluginSwift: NSObject, FrameProcessorPluginBase {
+    @objc
+    public static func callback(_ frame: Frame!, withArgs args: [Any]!) -> Any! {
+        guard let imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer) else {
+            return nil
+        }
+        NSLog("ExamplePlugin: \(CVPixelBufferGetWidth(imageBuffer)) x \(CVPixelBufferGetHeight(imageBuffer)) Image. Logging \(args.count) parameters:")
+
+        args.forEach { arg in
+            var string = "\(arg)"
+            if let array = arg as? NSArray {
+                string = (array as Array).description
+            } else if let map = arg as? NSDictionary {
+                string = (map as Dictionary).description
+            }
+            NSLog("ExamplePlugin:   -> \(string) (\(type(of: arg)))")
+        }
+
+        return [
+            "example_str": "Test",
+            "example_bool": true,
+            "example_double": 5.3,
+            "example_array": [
+                "Hello",
+                true,
+                17.38,
+            ],
+        ]
+    }
+}

example/ios/Frame Processor Plugins/QR Code Plugin (Objective-C)/QRCodeFrameProcessorPluginObjC.m

@@ -1,26 +0,0 @@
-//
-//  QRCodeFrameProcessorPluginObjC.m
-//  VisionCameraExample
-//
-//  Created by Marc Rousavy on 01.05.21.
-//
-
-#import <Foundation/Foundation.h>
-#import <VisionCamera/FrameProcessorPlugin.h>
-#import <VisionCamera/Frame.h>
-
-// Example for an Objective-C Frame Processor plugin
-
-@interface QRCodeFrameProcessorPluginObjC : NSObject
-@end
-
-@implementation QRCodeFrameProcessorPluginObjC
-
-static inline id exampleObjC___scanQRCodes(Frame* frame, NSArray* arguments) {
-  // TODO: Use some AI to detect QR codes in the frame
-  return @[];
-}
-
-VISION_EXPORT_FRAME_PROCESSOR(exampleObjC___scanQRCodes)
-
-@end

example/ios/Frame Processor Plugins/QR Code Plugin (Swift)/QRCodeFrameProcessorPluginSwift.swift

@@ -1,19 +0,0 @@
-//
-//  QRCodeFrameProcessorPluginSwift.swift
-//  VisionCamera
-//
-//  Created by Marc Rousavy on 30.04.21.
-//  Copyright © 2021 mrousavy. All rights reserved.
-//
-
-import AVKit
-import Vision
-
-@objc(QRCodeFrameProcessorPluginSwift)
-public class QRCodeFrameProcessorPluginSwift: NSObject, FrameProcessorPluginBase {
-    @objc
-    public static func callback(_: Frame!, withArgs _: [Any]!) -> Any! {
-        // TODO: Use some AI to detect QR codes in the CMSampleBufferRef
-        []
-    }
-}

example/ios/Podfile.lock

@@ -322,7 +322,7 @@ PODS:
     - React
   - RNVectorIcons (8.1.0):
     - React-Core
-  - VisionCamera (2.3.0):
+  - VisionCamera (2.4.1):
     - React
     - React-callinvoker
     - React-Core
@@ -490,7 +490,7 @@ SPEC CHECKSUMS:
   RNReanimated: 9c13c86454bfd54dab7505c1a054470bfecd2563
   RNStaticSafeAreaInsets: 6103cf09647fa427186d30f67b0f5163c1ae8252
   RNVectorIcons: 31cebfcf94e8cf8686eb5303ae0357da64d7a5a4
-  VisionCamera: aa5a3c3fd9f32a3d9836872b4aea04b600dddd75
+  VisionCamera: b4cdf9509f6a25b6eff0fbc2ac5fc1d61fc36d54
   Yoga: 575c581c63e0d35c9a83f4b46d01d63abc1100ac
 
 PODFILE CHECKSUM: 4b093c1d474775c2eac3268011e4b0b80929d3a2

example/ios/VisionCameraExample.xcodeproj/project.pbxproj

@@ -12,9 +12,9 @@
 		13B07FC11A68108700A75B9A /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 13B07FB71A68108700A75B9A /* main.m */; };
 		81AB9BB82411601600AC10FF /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 81AB9BB72411601600AC10FF /* LaunchScreen.storyboard */; };
 		B8DB3BD5263DE8B7004C18D7 /* BuildFile in Sources */ = {isa = PBXBuildFile; };
-		B8DB3BDC263DEA31004C18D7 /* QRCodeFrameProcessorPluginObjC.m in Sources */ = {isa = PBXBuildFile; fileRef = B8DB3BD8263DEA31004C18D7 /* QRCodeFrameProcessorPluginObjC.m */; };
+		B8DB3BDC263DEA31004C18D7 /* ExampleFrameProcessorPlugin.m in Sources */ = {isa = PBXBuildFile; fileRef = B8DB3BD8263DEA31004C18D7 /* ExampleFrameProcessorPlugin.m */; };
-		B8DB3BDD263DEA31004C18D7 /* QRCodeFrameProcessorPluginSwift.swift in Sources */ = {isa = PBXBuildFile; fileRef = B8DB3BDA263DEA31004C18D7 /* QRCodeFrameProcessorPluginSwift.swift */; };
+		B8DB3BDD263DEA31004C18D7 /* ExamplePluginSwift.swift in Sources */ = {isa = PBXBuildFile; fileRef = B8DB3BDA263DEA31004C18D7 /* ExamplePluginSwift.swift */; };
-		B8DB3BDE263DEA31004C18D7 /* QRCodeFrameProcessorPluginSwift.m in Sources */ = {isa = PBXBuildFile; fileRef = B8DB3BDB263DEA31004C18D7 /* QRCodeFrameProcessorPluginSwift.m */; };
+		B8DB3BDE263DEA31004C18D7 /* ExamplePluginSwift.m in Sources */ = {isa = PBXBuildFile; fileRef = B8DB3BDB263DEA31004C18D7 /* ExamplePluginSwift.m */; };
 		B8F0E10825E0199F00586F16 /* File.swift in Sources */ = {isa = PBXBuildFile; fileRef = B8F0E10725E0199F00586F16 /* File.swift */; };
 		D27D5196997E7C9532F05776 /* libPods-VisionCameraExample.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 321EF902E53C449CC036AD27 /* libPods-VisionCameraExample.a */; };
 /* End PBXBuildFile section */
@@ -31,9 +31,9 @@
 		321EF902E53C449CC036AD27 /* libPods-VisionCameraExample.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-VisionCameraExample.a"; sourceTree = BUILT_PRODUCTS_DIR; };
 		81AB9BB72411601600AC10FF /* LaunchScreen.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; name = LaunchScreen.storyboard; path = VisionCameraExample/LaunchScreen.storyboard; sourceTree = "<group>"; };
 		9993ED8FD4B3171BB28C87C9 /* Pods-VisionCameraExample.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-VisionCameraExample.debug.xcconfig"; path = "Target Support Files/Pods-VisionCameraExample/Pods-VisionCameraExample.debug.xcconfig"; sourceTree = "<group>"; };
-		B8DB3BD8263DEA31004C18D7 /* QRCodeFrameProcessorPluginObjC.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = QRCodeFrameProcessorPluginObjC.m; sourceTree = "<group>"; };
+		B8DB3BD8263DEA31004C18D7 /* ExampleFrameProcessorPlugin.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ExampleFrameProcessorPlugin.m; sourceTree = "<group>"; };
-		B8DB3BDA263DEA31004C18D7 /* QRCodeFrameProcessorPluginSwift.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = QRCodeFrameProcessorPluginSwift.swift; sourceTree = "<group>"; };
+		B8DB3BDA263DEA31004C18D7 /* ExamplePluginSwift.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ExamplePluginSwift.swift; sourceTree = "<group>"; };
-		B8DB3BDB263DEA31004C18D7 /* QRCodeFrameProcessorPluginSwift.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = QRCodeFrameProcessorPluginSwift.m; sourceTree = "<group>"; };
+		B8DB3BDB263DEA31004C18D7 /* ExamplePluginSwift.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ExamplePluginSwift.m; sourceTree = "<group>"; };
 		B8F0E10625E0199F00586F16 /* VisionCameraExample-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "VisionCameraExample-Bridging-Header.h"; sourceTree = "<group>"; };
 		B8F0E10725E0199F00586F16 /* File.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = File.swift; sourceTree = "<group>"; };
 		ED297162215061F000B7C4FE /* JavaScriptCore.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = JavaScriptCore.framework; path = System/Library/Frameworks/JavaScriptCore.framework; sourceTree = SDKROOT; };
@@ -120,27 +120,27 @@
 		B8DB3BD6263DEA31004C18D7 /* Frame Processor Plugins */ = {
 			isa = PBXGroup;
 			children = (
-				B8DB3BD7263DEA31004C18D7 /* QR Code Plugin (Objective-C) */,
+				B8DB3BD7263DEA31004C18D7 /* Example Plugin (Objective-C) */,
-				B8DB3BD9263DEA31004C18D7 /* QR Code Plugin (Swift) */,
+				B8DB3BD9263DEA31004C18D7 /* Example Plugin (Swift) */,
 			);
 			path = "Frame Processor Plugins";
 			sourceTree = "<group>";
 		};
-		B8DB3BD7263DEA31004C18D7 /* QR Code Plugin (Objective-C) */ = {
+		B8DB3BD7263DEA31004C18D7 /* Example Plugin (Objective-C) */ = {
 			isa = PBXGroup;
 			children = (
-				B8DB3BD8263DEA31004C18D7 /* QRCodeFrameProcessorPluginObjC.m */,
+				B8DB3BD8263DEA31004C18D7 /* ExampleFrameProcessorPlugin.m */,
 			);
-			path = "QR Code Plugin (Objective-C)";
+			path = "Example Plugin (Objective-C)";
 			sourceTree = "<group>";
 		};
-		B8DB3BD9263DEA31004C18D7 /* QR Code Plugin (Swift) */ = {
+		B8DB3BD9263DEA31004C18D7 /* Example Plugin (Swift) */ = {
 			isa = PBXGroup;
 			children = (
-				B8DB3BDA263DEA31004C18D7 /* QRCodeFrameProcessorPluginSwift.swift */,
+				B8DB3BDA263DEA31004C18D7 /* ExamplePluginSwift.swift */,
-				B8DB3BDB263DEA31004C18D7 /* QRCodeFrameProcessorPluginSwift.m */,
+				B8DB3BDB263DEA31004C18D7 /* ExamplePluginSwift.m */,
 			);
-			path = "QR Code Plugin (Swift)";
+			path = "Example Plugin (Swift)";
 			sourceTree = "<group>";
 		};
 /* End PBXGroup section */
@@ -363,12 +363,12 @@
 			buildActionMask = 2147483647;
 			files = (
 				13B07FBC1A68108700A75B9A /* AppDelegate.m in Sources */,
-				B8DB3BDC263DEA31004C18D7 /* QRCodeFrameProcessorPluginObjC.m in Sources */,
+				B8DB3BDC263DEA31004C18D7 /* ExampleFrameProcessorPlugin.m in Sources */,
 				B8DB3BD5263DE8B7004C18D7 /* BuildFile in Sources */,
-				B8DB3BDD263DEA31004C18D7 /* QRCodeFrameProcessorPluginSwift.swift in Sources */,
+				B8DB3BDD263DEA31004C18D7 /* ExamplePluginSwift.swift in Sources */,
 				B8F0E10825E0199F00586F16 /* File.swift in Sources */,
 				13B07FC11A68108700A75B9A /* main.m in Sources */,
-				B8DB3BDE263DEA31004C18D7 /* QRCodeFrameProcessorPluginSwift.m in Sources */,
+				B8DB3BDE263DEA31004C18D7 /* ExamplePluginSwift.m in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};

example/package.json

@@ -8,7 +8,8 @@
     "ios": "react-native run-ios",
     "start": "react-native start",
     "setup": "cd ios && bundle install",
-    "pods": "cd ios && bundle exec pod install"
+    "pods": "cd ios && bundle exec pod install",
+    "postinstall": "patch-package"
   },
   "dependencies": {
     "@react-native-community/blur": "^3.6.0",
@@ -43,6 +44,7 @@
     "eslint-plugin-react-native": "^3.11.0",
     "metro-config": "^0.66.0",
     "metro-react-native-babel-preset": "^0.66.0",
+    "patch-package": "^6.4.7",
     "prettier": "^2.3.1",
     "typescript": "4.3.2"
   }

example/patches/react-native-reanimated+2.2.0.patch

@@ -0,0 +1,20 @@
+diff --git a/node_modules/react-native-reanimated/android/build.gradle b/node_modules/react-native-reanimated/android/build.gradle
+index bb707e7..9186873 100644
+--- a/node_modules/react-native-reanimated/android/build.gradle
++++ b/node_modules/react-native-reanimated/android/build.gradle
+@@ -7,8 +7,12 @@ def reactNativeVersion = json.version as String
+ def (major, minor, patch) = reactNativeVersion.tokenize('.')
+ 
+ def engine = "jsc"
+-if (project(':app').ext.react.enableHermes) {
+-    engine = "hermes"
+-}
++rootProject.getSubprojects().forEach({project ->
++     if (project.plugins.hasPlugin("com.android.application")) {
++         if(project.ext.react.enableHermes) {
++             engine = "hermes"
++         }
++     }
++ })
+ 
+ artifacts.add("default", file("react-native-reanimated-${minor}-${engine}.aar"))

example/src/CameraPage.tsx

@@ -3,7 +3,15 @@ import { useRef, useState, useMemo, useCallback } from 'react';
 import { StyleSheet, Text, View } from 'react-native';
 import { PinchGestureHandler, PinchGestureHandlerGestureEvent, TapGestureHandler } from 'react-native-gesture-handler';
 import { Navigation, NavigationFunctionComponent } from 'react-native-navigation';
-import { CameraDeviceFormat, CameraRuntimeError, PhotoFile, sortFormats, useCameraDevices, VideoFile } from 'react-native-vision-camera';
+import {
+  CameraDeviceFormat,
+  CameraRuntimeError,
+  PhotoFile,
+  sortFormats,
+  useCameraDevices,
+  useFrameProcessor,
+  VideoFile,
+} from 'react-native-vision-camera';
 import { Camera, frameRateIncluded } from 'react-native-vision-camera';
 import { useIsScreenFocussed } from './hooks/useIsScreenFocused';
 import { CONTENT_SPACING, MAX_ZOOM_FACTOR, SAFE_AREA_PADDING } from './Constants';
@@ -15,6 +23,7 @@ import { CaptureButton } from './views/CaptureButton';
 import { PressableOpacity } from 'react-native-pressable-opacity';
 import MaterialIcon from 'react-native-vector-icons/MaterialCommunityIcons';
 import IonIcon from 'react-native-vector-icons/Ionicons';
+import { examplePlugin } from './frame-processors/ExamplePlugin';
 
 const ReanimatedCamera = Reanimated.createAnimatedComponent(Camera);
 Reanimated.addWhitelistedNativeProps({
@@ -187,11 +196,11 @@ export const CameraPage: NavigationFunctionComponent = ({ componentId }) => {
     console.log('re-rendering camera page without active camera');
   }
 
-  // const frameProcessor = useFrameProcessor((frame) => {
+  const frameProcessor = useFrameProcessor((frame) => {
-  //   'worklet';
+    'worklet';
-  //   const codes = scanQRCodesObjC(frame);
+    const values = examplePlugin(frame);
-  //   _log(`Codes: ${JSON.stringify(codes)}`);
+    _log(`Return Values: ${JSON.stringify(values)}`);
-  // }, []);
+  }, []);
 
   return (
     <View style={styles.container}>
@@ -215,8 +224,8 @@ export const CameraPage: NavigationFunctionComponent = ({ componentId }) => {
                 photo={true}
                 video={true}
                 audio={true}
-                // frameProcessor={frameProcessor}
+                frameProcessor={device.supportsParallelVideoProcessing ? frameProcessor : undefined}
-                // frameProcessorFps={1}
+                frameProcessorFps={1}
               />
             </TapGestureHandler>
           </Reanimated.View>

example/src/frame-processors/ExamplePlugin.ts

@@ -0,0 +1,18 @@
+/* global _WORKLET __example_plugin __example_plugin_swift */
+import type { Frame } from 'react-native-vision-camera';
+
+export function examplePluginSwift(frame: Frame): string[] {
+  'worklet';
+  if (!_WORKLET) throw new Error('examplePluginSwift must be called from a frame processor!');
+
+  // @ts-expect-error because this function is dynamically injected by VisionCamera
+  return __example_plugin_swift(frame, 'hello!', 'parameter2', true, 42, { test: 0, second: 'test' }, ['another test', 5]);
+}
+
+export function examplePlugin(frame: Frame): string[] {
+  'worklet';
+  if (!_WORKLET) throw new Error('examplePlugin must be called from a frame processor!');
+
+  // @ts-expect-error because this function is dynamically injected by VisionCamera
+  return __example_plugin(frame, 'hello!', 'parameter2', true, 42, { test: 0, second: 'test' }, ['another test', 5]);
+}

example/src/frame-processors/ScanQRCode.ts

@@ -1,18 +0,0 @@
-/* global _WORKLET __exampleObjC___scanQRCodes __exampleSwift___scanQRCodes */
-import type { Frame } from 'react-native-vision-camera';
-
-export function scanQRCodesSwift(frame: Frame): string[] {
-  'worklet';
-  if (!_WORKLET) throw new Error('scanQRCodesSwift must be called from a frame processor!');
-
-  // @ts-expect-error because this function is dynamically injected by VisionCamera
-  return __exampleSwift___scanQRCodes(frame);
-}
-
-export function scanQRCodesObjC(frame: Frame): string[] {
-  'worklet';
-  if (!_WORKLET) throw new Error('scanQRCodesObjC must be called from a frame processor!');
-
-  // @ts-expect-error because this function is dynamically injected by VisionCamera
-  return __exampleObjC___scanQRCodes(frame, 'hello!', 'parameter2', true, 42);
-}

example/src/views/CaptureButton.tsx

@@ -34,7 +34,7 @@ interface Props extends ViewProps {
 
   cameraZoom: Reanimated.SharedValue<number>;
 
-  flash: 'off' | 'on' | 'auto';
+  flash: 'off' | 'on';
 
   enabled: boolean;
 

ios/CameraViewManager.swift

@@ -100,7 +100,7 @@ final class CameraViewManager: RCTViewManager {
           "neutralZoom": $0.neutralZoomFactor,
           "maxZoom": $0.maxAvailableVideoZoomFactor,
           "isMultiCam": $0.isMultiCam,
-          "supportsPhotoAndVideoCapture": true,
+          "supportsParallelVideoProcessing": true,
           "supportsDepthCapture": false, // TODO: supportsDepthCapture
           "supportsRawCapture": false, // TODO: supportsRawCapture
           "supportsLowLightBoost": $0.isLowLightBoostSupported,

ios/Frame Processor/FrameHostObject.mm

@@ -26,15 +26,6 @@ std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt)
 jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
   auto name = propName.utf8(runtime);
 
-
-  if (name == "Symbol.toPrimitive") {
-    // not implemented
-    return jsi::Value::undefined();
-  }
-  if (name == "valueOf") {
-    // not implemented
-    return jsi::Value::undefined();
-  }
   if (name == "toString") {
     auto toString = [this] (jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
       auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);

package.json

@@ -13,8 +13,7 @@
     "lib/module",
     "lib/typescript",
     "android/build.gradle",
-    "android/gradle.properties",
+    "android/*.aar",
-    "android/src",
     "ios/**/*.h",
     "ios/**/*.m",
     "ios/**/*.mm",
@@ -31,7 +30,7 @@
     "typescript": "tsc --noEmit",
     "lint": "eslint \"**/*.{js,ts,tsx}\"",
     "lint-ci": "yarn lint -f ./node_modules/@firmnav/eslint-github-actions-formatter/dist/formatter.js",
-    "prepare": "bob build",
+    "build": "bob build && scripts/build-android-npm-package.sh",
     "release": "release-it",
     "pods": "cd example && yarn pods",
     "bootstrap": "yarn && cd example && yarn && yarn pods",
@@ -40,7 +39,10 @@
     "check-cpp": "scripts/cpplint.sh",
     "check-all": "scripts/check-all.sh",
     "clean": "scripts/clean.sh",
-    "docs": "cd docs && yarn build"
+    "docs": "cd docs && yarn build",
+    "prepack": "rm -rf android-tmp && mv android android-tmp && mv android-npm android",
+    "postpack": "rm -rf android-npm && mv android android-npm && mv android-tmp android",
+    "postinstall": "patch-package"
   },
   "keywords": [
     "react-native",
@@ -81,6 +83,7 @@
     "eslint-plugin-prettier": "^3.4.0",
     "eslint-plugin-react-hooks": "^4.2.0",
     "eslint-plugin-react-native": "^3.11.0",
+    "patch-package": "^6.4.7",
     "pod-install": "^0.1.23",
     "prettier": "^2.3.0",
     "react": "^17.0.2",

patches/react-native-reanimated+2.2.0.patch

@@ -0,0 +1,20 @@
+diff --git a/node_modules/react-native-reanimated/android/build.gradle b/node_modules/react-native-reanimated/android/build.gradle
+index bb707e7..9186873 100644
+--- a/node_modules/react-native-reanimated/android/build.gradle
++++ b/node_modules/react-native-reanimated/android/build.gradle
+@@ -7,8 +7,12 @@ def reactNativeVersion = json.version as String
+ def (major, minor, patch) = reactNativeVersion.tokenize('.')
+ 
+ def engine = "jsc"
+-if (project(':app').ext.react.enableHermes) {
+-    engine = "hermes"
+-}
++rootProject.getSubprojects().forEach({project ->
++     if (project.plugins.hasPlugin("com.android.application")) {
++         if(project.ext.react.enableHermes) {
++             engine = "hermes"
++         }
++     }
++ })
+ 
+ artifacts.add("default", file("react-native-reanimated-${minor}-${engine}.aar"))

scripts/build-android-npm-package.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e
+
+ROOT=$(pwd)
+
+rm -rf android-npm/*.aar
+
+for for_hermes in "True" "False"
+do
+  engine="jsc"
+  if [ "$for_hermes" == "True" ]; then
+    engine="hermes"
+  fi
+  echo "Building VisionCamera for JS Engine ${engine}..."
+
+  cd android
+  ./gradlew clean
+
+  FOR_HERMES=${for_hermes} ./gradlew assembleDebug
+  cd ..
+
+  cp android/build/outputs/aar/*.aar "android-npm/react-native-vision-camera-${engine}.aar"
+  echo "Built react-native-vision-camera-${engine}.aar!"
+done
+
+echo "Finished building VisionCamera packages!"

scripts/clean.sh

@@ -28,6 +28,13 @@ rm -rf ios/Podfile.lock
 echo "rm -rf ~/Library/Developer/Xcode/DerivedData/*"
 rm -rf ~/Library/Developer/Xcode/DerivedData/*
 
+echo "rm -rf android/.cxx"
+rm -rf android/.cxx
+echo "rm -rf android/.gradle"
+rm -rf android/.gradle
+echo "rm -rf android/build"
+rm -rf android/build
+
 cd ios
 echo "pod deintegrate"
 pod deintegrate

scripts/cpplint.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 if which cpplint >/dev/null; then
-  cpplint --linelength=230 --filter=-legal/copyright,-readability/todo,-build/namespaces,-whitespace/comments --quiet --recursive cpp android/src/main/cpp
+  cpplint --linelength=230 --filter=-legal/copyright,-readability/todo,-build/namespaces,-whitespace/comments,-build/include_order --quiet --recursive --exclude "android/src/main/cpp/reanimated-headers" cpp android/src/main/cpp
 else
   echo "warning: cpplint not installed, download from https://github.com/cpplint/cpplint"
 fi

src/Camera.tsx

@@ -358,13 +358,6 @@ export class Camera extends React.PureComponent<CameraProps> {
       this.assertFrameProcessorsEnabled();
       // frameProcessor argument changed. Update native to reflect the change.
       if (this.props.frameProcessor != null) {
-        if (this.props.video !== true) {
-          throw new CameraCaptureError(
-            'capture/video-not-enabled',
-            'Video capture is disabled! Pass `video={true}` to enable frame processors.',
-          );
-        }
-
         // 1. Spawn threaded JSI Runtime (if not already done)
         // 2. Add video data output to Camera stream (if not already done)
         // 3. Workletize the frameProcessor and prepare it for being called with frames
@@ -386,8 +379,19 @@ export class Camera extends React.PureComponent<CameraProps> {
    */
   public render(): React.ReactNode {
     // We remove the big `device` object from the props because we only need to pass `cameraId` to native.
-    const { device, frameProcessor: _, ...props } = this.props;
+    const { device, video: enableVideo, frameProcessor, ...props } = this.props;
-    return <NativeCameraView {...props} cameraId={device.id} ref={this.ref} onInitialized={this.onInitialized} onError={this.onError} />;
+    // on iOS, enabling a frameProcessor requires `video` to be `true`. On Android, it doesn't.
+    const video = Platform.OS === 'ios' ? frameProcessor != null || enableVideo : enableVideo;
+    return (
+      <NativeCameraView
+        {...props}
+        cameraId={device.id}
+        ref={this.ref}
+        onInitialized={this.onInitialized}
+        onError={this.onError}
+        video={video}
+      />
+    );
   }
 }
 //#endregion

src/CameraDevice.ts

@@ -251,27 +251,15 @@ export interface CameraDevice {
    */
   formats: CameraDeviceFormat[];
   /**
-   * Whether this camera device supports enabling photo and video capture at the same time.
+   * Whether this camera device supports using Video Recordings (`video={true}`) and Frame Processors (`frameProcessor={...}`) at the same time. See ["The `supportsParallelVideoProcessing` prop"](https://mrousavy.github.io/react-native-vision-camera/docs/guides/devices#the-supportsparallelvideoprocessing-prop) for more information.
    *
-   * * On **iOS** devices this value is always `true`.
+   * If this property is `false`, you can only enable `video` or add a `frameProcessor`, but not both.
-   * * On newer **Android** devices this value is always `true`.
-   * * On older **Android** devices this value is `true` if the device's hardware level is `LIMITED` or above, `false` otherwise. (`LEGACY`) (See [this table](https://developer.android.com/reference/android/hardware/camera2/CameraDevice#regular-capture))
    *
-   * If the device does not allow enabling `photo` and `video` capture at the same time, you might want to fall back to **snapshot capture** (See [**"Taking Snapshots"**](https://mrousavy.github.io/react-native-vision-camera/docs/guides/capturing#taking-snapshots)) instead:
+   * * On iOS this value is always `true`.
-   *
+   * * On newer Android devices this value is always `true`.
-   * @example
+   * * On older Android devices this value is `false` if the Camera's hardware level is `LEGACY` or `LIMITED`, `true` otherwise. (See [`INFO_SUPPORTED_HARDWARE_LEVEL`](https://developer.android.com/reference/android/hardware/camera2/CameraCharacteristics#INFO_SUPPORTED_HARDWARE_LEVEL) or [the tables at "Regular capture"](https://developer.android.com/reference/android/hardware/camera2/CameraDevice#regular-capture))
-   * ```tsx
-   * const captureMode = device.supportsPhotoAndVideoCapture ? "photo" : "snapshot"
-   * return (
-   *   <Camera
-   *     photo={captureMode === "photo"}
-   *     video={true}
-   *     audio={true}
-   *   />
-   * )
-   * ```
    */
-  supportsPhotoAndVideoCapture: boolean;
+  supportsParallelVideoProcessing: boolean;
   /**
    * Whether this camera device supports low light boost.
    */

src/CameraError.ts

@@ -9,6 +9,7 @@ export type DeviceError =
   | 'device/configuration-error'
   | 'device/no-device'
   | 'device/invalid-device'
+  | 'device/too-many-use-cases'
   | 'device/torch-unavailable'
   | 'device/microphone-unavailable'
   | 'device/low-light-boost-not-supported'

src/CameraProps.ts

@@ -36,16 +36,19 @@ export interface CameraProps extends ViewProps {
 
   //#region Use-cases
   /**
-   * * Enables **photo capture** with the `takePhoto` function (see ["Taking Photos"](https://mrousavy.github.io/react-native-vision-camera/docs/guides/capturing#taking-photos))
+   * Enables **photo capture** with the `takePhoto` function (see ["Taking Photos"](https://mrousavy.github.io/react-native-vision-camera/docs/guides/capturing#taking-photos))
+   *
+   * Note: This occupies a use-case. (See ["Use-cases"](https://mrousavy.github.io/react-native-vision-camera/docs/guides/devices#use-cases))
    */
   photo?: boolean;
   /**
-   * * Enables **video capture** with the `startRecording` function (see ["Recording Videos"](https://mrousavy.github.io/react-native-vision-camera/docs/guides/capturing/#recording-videos))
+   * Enables **video capture** with the `startRecording` function (see ["Recording Videos"](https://mrousavy.github.io/react-native-vision-camera/docs/guides/capturing/#recording-videos))
-   * * Enables **frame processing** (see ["Frame Processors"](https://mrousavy.github.io/react-native-vision-camera/docs/guides/frame-processors))
+   *
+   * Note: This occupies a use-case. (See ["Use-cases"](https://mrousavy.github.io/react-native-vision-camera/docs/guides/devices#use-cases))
    */
   video?: boolean;
   /**
-   * * Enables **audio capture** for video recordings (see ["Recording Videos"](https://mrousavy.github.io/react-native-vision-camera/docs/guides/capturing/#recording-videos))
+   * Enables **audio capture** for video recordings (see ["Recording Videos"](https://mrousavy.github.io/react-native-vision-camera/docs/guides/capturing/#recording-videos))
    */
   audio?: boolean;
   //#endregion
@@ -161,6 +164,8 @@ export interface CameraProps extends ViewProps {
    *
    * > See [the Frame Processors documentation](https://mrousavy.github.io/react-native-vision-camera/docs/guides/frame-processors) for more information
    *
+   * Note: This occupies a use-case. (See ["Use-cases"](https://mrousavy.github.io/react-native-vision-camera/docs/guides/devices#use-cases))
+   *
    * @example
    * ```tsx
    * const frameProcessor = useFrameProcessor((frame) => {

src/Snapshot.ts

@@ -8,6 +8,13 @@ export interface TakeSnapshotOptions {
    */
   quality?: number;
 
+  /**
+   * Whether the Flash should be enabled or disabled
+   *
+   * @default "off"
+   */
+  flash?: 'on' | 'off';
+
   /**
    * When set to `true`, metadata reading and mapping will be skipped. ({@linkcode PhotoFile.metadata} will be `null`)
    *