feat: Frame Processors for Android (#196)
* Create android gradle build setup * Fix `prefab` config * Add `pickFirst **/*.so` to example build.gradle * fix REA path * cache gradle builds * Update validate-android.yml * Create Native Proxy * Copy REA header * implement ctor * Rename CameraViewModule -> FrameProcessorRuntimeManager * init FrameProcessorRuntimeManager * fix name * Update FrameProcessorRuntimeManager.h * format * Create AndroidErrorHandler.h * Initialize runtime and install JSI funcs * Update FrameProcessorRuntimeManager.cpp * Update CameraViewModule.kt * Make CameraView hybrid C++ class to find view & set frame processor * Update FrameProcessorRuntimeManager.cpp * pass function by rvalue * pass by const && * extract hermes and JSC REA * pass `FOR_HERMES` * correctly prepare JSC and Hermes * Update CMakeLists.txt * add missing hermes include * clean up imports * Create JImageProxy.h * pass ImageProxy to JNI as `jobject` * try use `JImageProxy` C++ wrapper type * Use `local_ref<JImageProxy>` * Create `JImageProxyHostObject` for JSI interop * debug call to frame processor * Unset frame processor * Fix CameraView native part not being registered * close image * use `jobject` instead of `JImageProxy` for now :( * fix hermes build error * Set enable FP callback * fix JNI call * Update CameraView.cpp * Get Format * Create plugin abstract * Make `FrameProcessorPlugin` a hybrid object * Register plugin CXX * Call `registerPlugin` * Catch * remove JSI * Create sample QR code plugin * register plugins * Fix missing JNI binding * Add `mHybridData` * prefix name with two underscores (`__`) * Update CameraPage.tsx * wrap `ImageProxy` in host object * Use `jobject` for HO box * Update JImageProxy.h * reinterpret jobject * Try using `JImageProxy` instead of `jobject` * Update JImageProxy.h * get bytes per row and plane count * Update CameraView.cpp * Return base * add some docs and JNI JSI conversion * indent * Convert JSI value to JNI jobject * using namespace facebook * Try using class * Use plain old Object[] * Try convert JNI -> JSI * fix decl * fix bool init * Correctly link folly * Update CMakeLists.txt * Convert Map to Object * Use folly for Map and Array * Return `alias_ref<jobject>` instead of raw `jobject` * fix JNI <-> JSI conversion * Update JSIJNIConversion.cpp * Log parameters * fix params index offset * add more test cases * Update FRAME_PROCESSORS_CREATE_OVERVIEW.mdx * fix types * Rename to example plugin * remove support for hashmap * Try use HashMap iterable fbjni binding * try using JReadableArray/JReadableMap * Fix list return values * Update JSIJNIConversion.cpp * Update JSIJNIConversion.cpp * (iOS) Rename ObjC QR Code Plugin to Example Plugin * Rename Swift plugin QR -> Example * Update ExamplePluginSwift.swift * Fix Map/Dictionary logging format * Update ExampleFrameProcessorPlugin.m * Reconfigure session if frame processor changed * Handle use-cases via `maxUseCasesCount` * Don't crash app on `configureSession` error * Document "use-cases" * Update DEVICES.mdx * fix merge * Make `const &` * iOS: Automatically enable `video` if a `frameProcessor` is set * Update CameraView.cpp * fix docs * Automatically fallback to snapshot capture if `supportsParallelVideoProcessing` is false. * Fix lookup * Update CameraView.kt * Implement `frameProcessorFps` * Finalize Frame Processor Plugin Hybrid * Update CameraViewModule.kt * Support `flash` on `takeSnapshot()` * Update docs * Add docs * Update CameraPage.tsx * Attribute NonNull * remove unused imports * Add Android docs for Frame Processors * Make JNI HashMap <-> JSI Object conversion faster directly access `toHashMap` instead of going through java * add todo * Always run `prepareJSC` and `prepareHermes` * switch jsc and hermes * Specify ndkVersion `21.4.7075529` * Update gradle.properties * Update gradle.properties * Create .aar * Correctly prepare android package * Update package.json * Update package.json * remove `prefab` build feature * split * Add docs for registering the FP plugin * Add step for dep * Update CaptureButton.tsx * Move to `reanimated-headers/` * Exclude reanimated-headers from cpplint * disable `build/include_order` rule * cpplint fixes * perf: Make `JSIJNIConversion` a `namespace` instead of `class` * Ignore runtime/references for `convert` funcs * Build Android .aar in CI * Run android build script only on `prepack` * Update package.json * Update package.json * Update build-android-npm-package.sh * Move to `yarn build` * Also install node_modules in example step * Update validate-android.yml * sort imports * fix torch * Run ImageAnalysis on `FrameProcessorThread` * Update Errors.kt * Add clean android script * Upgrade reanimated to 2.3.0-alpha.1 * Revert "Upgrade reanimated to 2.3.0-alpha.1" This reverts commit c1d3bed5e03728d0b5e335a359524ff4f56f5035. * ⚠️ TEMP FIX: hotfix reanimated build.gradle * Update CameraView+TakeSnapshot.kt * ⚠️ TEMP FIX: Disable ktlint action for now * Update clean.sh * Set max heap size to 4g * rebuild lockfiles * Update Podfile.lock * rename * Build lib .aar before example/
This commit is contained in:
Marc Rousavy
GitHub
@@ -0,0 +1,54 @@
|
||||
package com.mrousavy.camera.frameprocessor;
|
||||
|
||||
import androidx.annotation.NonNull;
|
||||
import androidx.annotation.Nullable;
|
||||
import androidx.camera.core.ImageProxy;
|
||||
import com.facebook.jni.HybridData;
|
||||
|
||||
/**
|
||||
* Declares a Frame Processor Plugin.
|
||||
*/
|
||||
public abstract class FrameProcessorPlugin {
|
||||
static {
|
||||
System.loadLibrary("VisionCamera");
|
||||
}
|
||||
|
||||
@SuppressWarnings({"FieldCanBeLocal", "unused"})
|
||||
private final HybridData mHybridData;
|
||||
|
||||
/**
|
||||
* The actual Frame Processor plugin callback. Called for every frame the ImageAnalyzer receives.
|
||||
* @param image The CameraX ImageProxy. Don't call .close() on this, as VisionCamera handles that.
|
||||
* @return You can return any primitive, map or array you want. See the
|
||||
* <a href="https://mrousavy.github.io/react-native-vision-camera/docs/guides/frame-processors-plugins-overview#types">Types</a>
|
||||
* table for a list of supported types.
|
||||
*/
|
||||
public abstract @Nullable Object callback(@NonNull ImageProxy image, @NonNull Object[] params);
|
||||
|
||||
/**
|
||||
* Initializes the native plugin part.
|
||||
* @param name Specifies the Frame Processor Plugin's name in the Runtime.
|
||||
* The actual name in the JS Runtime will be prefixed with two underscores (`__`)
|
||||
*/
|
||||
protected FrameProcessorPlugin(@NonNull String name) {
|
||||
mHybridData = initHybrid(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void finalize() throws Throwable {
|
||||
super.finalize();
|
||||
if (mHybridData != null) {
|
||||
mHybridData.resetNative();
|
||||
}
|
||||
}
|
||||
|
||||
private native @NonNull HybridData initHybrid(@NonNull String name);
|
||||
|
||||
/**
|
||||
* Registers the given plugin in the Frame Processor Runtime.
|
||||
* @param plugin An instance of a plugin.
|
||||
*/
|
||||
public static void register(@NonNull FrameProcessorPlugin plugin) {
|
||||
FrameProcessorRuntimeManager.Companion.getPlugins().add(plugin);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package com.mrousavy.camera.frameprocessor
|
||||
|
||||
import android.util.Log
|
||||
import com.facebook.jni.HybridData
|
||||
import com.facebook.proguard.annotations.DoNotStrip
|
||||
import com.facebook.react.bridge.ReactApplicationContext
|
||||
import com.facebook.react.turbomodule.core.CallInvokerHolderImpl
|
||||
import com.mrousavy.camera.CameraView
|
||||
import com.mrousavy.camera.ViewNotFoundError
|
||||
import com.swmansion.reanimated.Scheduler
|
||||
import java.lang.ref.WeakReference
|
||||
|
||||
class FrameProcessorRuntimeManager(context: ReactApplicationContext) {
|
||||
companion object {
|
||||
const val TAG = "FrameProcessorRuntime"
|
||||
private var HasRegisteredPlugins = false
|
||||
val Plugins: ArrayList<FrameProcessorPlugin> = ArrayList()
|
||||
get() {
|
||||
if (HasRegisteredPlugins) {
|
||||
throw Error("Tried to access Frame Processor Plugin list, " +
|
||||
"but plugins have already been registered (list is frozen now!).")
|
||||
}
|
||||
return field
|
||||
}
|
||||
|
||||
init {
|
||||
System.loadLibrary("reanimated")
|
||||
System.loadLibrary("VisionCamera")
|
||||
}
|
||||
}
|
||||
|
||||
@DoNotStrip
|
||||
private var mHybridData: HybridData?
|
||||
private var mContext: WeakReference<ReactApplicationContext>?
|
||||
private var mScheduler: Scheduler?
|
||||
|
||||
init {
|
||||
val holder = context.catalystInstance.jsCallInvokerHolder as CallInvokerHolderImpl
|
||||
mScheduler = Scheduler(context)
|
||||
mContext = WeakReference(context)
|
||||
mHybridData = initHybrid(context.javaScriptContextHolder.get(), holder, mScheduler!!)
|
||||
initializeRuntime()
|
||||
|
||||
Log.i(TAG, "Installing Frame Processor Plugins...")
|
||||
Plugins.forEach { plugin ->
|
||||
registerPlugin(plugin)
|
||||
}
|
||||
Log.i(TAG, "Successfully installed ${Plugins.count()} Frame Processor Plugins!")
|
||||
HasRegisteredPlugins = true
|
||||
}
|
||||
|
||||
fun destroy() {
|
||||
mScheduler?.deactivate()
|
||||
mHybridData?.resetNative()
|
||||
}
|
||||
|
||||
fun findCameraViewById(viewId: Int): CameraView {
|
||||
Log.d(TAG, "finding view $viewId...")
|
||||
val view = mContext?.get()?.currentActivity?.findViewById<CameraView>(viewId)
|
||||
Log.d(TAG, "found view $viewId! is null: ${view == null}")
|
||||
return view ?: throw ViewNotFoundError(viewId)
|
||||
}
|
||||
|
||||
// private C++ funcs
|
||||
private external fun initHybrid(
|
||||
jsContext: Long,
|
||||
jsCallInvokerHolder: CallInvokerHolderImpl,
|
||||
scheduler: Scheduler
|
||||
): HybridData?
|
||||
private external fun initializeRuntime()
|
||||
private external fun registerPlugin(plugin: FrameProcessorPlugin)
|
||||
|
||||
// public C++ funcs
|
||||
external fun installJSIBindings()
|
||||
}
|
||||
Reference in New Issue
Block a user