feat: Frame Processors for Android (#196)
* Create android gradle build setup * Fix `prefab` config * Add `pickFirst **/*.so` to example build.gradle * fix REA path * cache gradle builds * Update validate-android.yml * Create Native Proxy * Copy REA header * implement ctor * Rename CameraViewModule -> FrameProcessorRuntimeManager * init FrameProcessorRuntimeManager * fix name * Update FrameProcessorRuntimeManager.h * format * Create AndroidErrorHandler.h * Initialize runtime and install JSI funcs * Update FrameProcessorRuntimeManager.cpp * Update CameraViewModule.kt * Make CameraView hybrid C++ class to find view & set frame processor * Update FrameProcessorRuntimeManager.cpp * pass function by rvalue * pass by const && * extract hermes and JSC REA * pass `FOR_HERMES` * correctly prepare JSC and Hermes * Update CMakeLists.txt * add missing hermes include * clean up imports * Create JImageProxy.h * pass ImageProxy to JNI as `jobject` * try use `JImageProxy` C++ wrapper type * Use `local_ref<JImageProxy>` * Create `JImageProxyHostObject` for JSI interop * debug call to frame processor * Unset frame processor * Fix CameraView native part not being registered * close image * use `jobject` instead of `JImageProxy` for now :( * fix hermes build error * Set enable FP callback * fix JNI call * Update CameraView.cpp * Get Format * Create plugin abstract * Make `FrameProcessorPlugin` a hybrid object * Register plugin CXX * Call `registerPlugin` * Catch * remove JSI * Create sample QR code plugin * register plugins * Fix missing JNI binding * Add `mHybridData` * prefix name with two underscores (`__`) * Update CameraPage.tsx * wrap `ImageProxy` in host object * Use `jobject` for HO box * Update JImageProxy.h * reinterpret jobject * Try using `JImageProxy` instead of `jobject` * Update JImageProxy.h * get bytes per row and plane count * Update CameraView.cpp * Return base * add some docs and JNI JSI conversion * indent * Convert JSI value to JNI jobject * using namespace facebook * Try using class * Use plain old Object[] * Try convert JNI -> JSI * fix decl * fix bool init * Correctly link folly * Update CMakeLists.txt * Convert Map to Object * Use folly for Map and Array * Return `alias_ref<jobject>` instead of raw `jobject` * fix JNI <-> JSI conversion * Update JSIJNIConversion.cpp * Log parameters * fix params index offset * add more test cases * Update FRAME_PROCESSORS_CREATE_OVERVIEW.mdx * fix types * Rename to example plugin * remove support for hashmap * Try use HashMap iterable fbjni binding * try using JReadableArray/JReadableMap * Fix list return values * Update JSIJNIConversion.cpp * Update JSIJNIConversion.cpp * (iOS) Rename ObjC QR Code Plugin to Example Plugin * Rename Swift plugin QR -> Example * Update ExamplePluginSwift.swift * Fix Map/Dictionary logging format * Update ExampleFrameProcessorPlugin.m * Reconfigure session if frame processor changed * Handle use-cases via `maxUseCasesCount` * Don't crash app on `configureSession` error * Document "use-cases" * Update DEVICES.mdx * fix merge * Make `const &` * iOS: Automatically enable `video` if a `frameProcessor` is set * Update CameraView.cpp * fix docs * Automatically fallback to snapshot capture if `supportsParallelVideoProcessing` is false. * Fix lookup * Update CameraView.kt * Implement `frameProcessorFps` * Finalize Frame Processor Plugin Hybrid * Update CameraViewModule.kt * Support `flash` on `takeSnapshot()` * Update docs * Add docs * Update CameraPage.tsx * Attribute NonNull * remove unused imports * Add Android docs for Frame Processors * Make JNI HashMap <-> JSI Object conversion faster directly access `toHashMap` instead of going through java * add todo * Always run `prepareJSC` and `prepareHermes` * switch jsc and hermes * Specify ndkVersion `21.4.7075529` * Update gradle.properties * Update gradle.properties * Create .aar * Correctly prepare android package * Update package.json * Update package.json * remove `prefab` build feature * split * Add docs for registering the FP plugin * Add step for dep * Update CaptureButton.tsx * Move to `reanimated-headers/` * Exclude reanimated-headers from cpplint * disable `build/include_order` rule * cpplint fixes * perf: Make `JSIJNIConversion` a `namespace` instead of `class` * Ignore runtime/references for `convert` funcs * Build Android .aar in CI * Run android build script only on `prepack` * Update package.json * Update package.json * Update build-android-npm-package.sh * Move to `yarn build` * Also install node_modules in example step * Update validate-android.yml * sort imports * fix torch * Run ImageAnalysis on `FrameProcessorThread` * Update Errors.kt * Add clean android script * Upgrade reanimated to 2.3.0-alpha.1 * Revert "Upgrade reanimated to 2.3.0-alpha.1" This reverts commit c1d3bed5e03728d0b5e335a359524ff4f56f5035. * ⚠️ TEMP FIX: hotfix reanimated build.gradle * Update CameraView+TakeSnapshot.kt * ⚠️ TEMP FIX: Disable ktlint action for now * Update clean.sh * Set max heap size to 4g * rebuild lockfiles * Update Podfile.lock * rename * Build lib .aar before example/
This commit is contained in:
Marc Rousavy
GitHub
@@ -21,6 +21,8 @@ import androidx.camera.lifecycle.ProcessCameraProvider
|
||||
import androidx.camera.view.PreviewView
|
||||
import androidx.core.content.ContextCompat
|
||||
import androidx.lifecycle.*
|
||||
import com.facebook.jni.HybridData
|
||||
import com.facebook.proguard.annotations.DoNotStrip
|
||||
import com.facebook.react.bridge.*
|
||||
import com.facebook.react.uimanager.events.RCTEventEmitter
|
||||
import com.mrousavy.camera.utils.*
|
||||
@@ -82,11 +84,14 @@ class CameraView(context: Context) : FrameLayout(context), LifecycleOwner {
|
||||
var torch = "off"
|
||||
var zoom = 0.0 // in percent
|
||||
var enableZoomGesture = false
|
||||
var frameProcessorFps = 1.0
|
||||
|
||||
// private properties
|
||||
private val reactContext: ReactContext
|
||||
get() = context as ReactContext
|
||||
|
||||
private var enableFrameProcessor = false
|
||||
|
||||
@Suppress("JoinDeclarationAndAssignment")
|
||||
internal val previewView: PreviewView
|
||||
private val cameraExecutor = Executors.newSingleThreadExecutor()
|
||||
@@ -96,6 +101,7 @@ class CameraView(context: Context) : FrameLayout(context), LifecycleOwner {
|
||||
internal var camera: Camera? = null
|
||||
internal var imageCapture: ImageCapture? = null
|
||||
internal var videoCapture: VideoCapture? = null
|
||||
internal var imageAnalysis: ImageAnalysis? = null
|
||||
|
||||
private val scaleGestureListener: ScaleGestureDetector.SimpleOnScaleGestureListener
|
||||
private val scaleGestureDetector: ScaleGestureDetector
|
||||
@@ -107,7 +113,42 @@ class CameraView(context: Context) : FrameLayout(context), LifecycleOwner {
|
||||
private var minZoom: Float = 1f
|
||||
private var maxZoom: Float = 1f
|
||||
|
||||
@DoNotStrip
|
||||
private var mHybridData: HybridData?
|
||||
|
||||
@Suppress("LiftReturnOrAssignment", "RedundantIf")
|
||||
internal val fallbackToSnapshot: Boolean
|
||||
@SuppressLint("UnsafeOptInUsageError")
|
||||
get() {
|
||||
if (video != true && !enableFrameProcessor) {
|
||||
// Both use-cases are disabled, so `photo` is the only use-case anyways. Don't need to fallback here.
|
||||
return false
|
||||
}
|
||||
cameraId?.let { cameraId ->
|
||||
val cameraManger = reactContext.getSystemService(Context.CAMERA_SERVICE) as? CameraManager
|
||||
cameraManger?.let {
|
||||
val characteristics = cameraManger.getCameraCharacteristics(cameraId)
|
||||
val hardwareLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)
|
||||
if (hardwareLevel == CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY) {
|
||||
// Camera only supports a single use-case at a time
|
||||
return true
|
||||
} else {
|
||||
if (video == true && enableFrameProcessor) {
|
||||
// Camera supports max. 2 use-cases, but both are occupied by `frameProcessor` and `video`
|
||||
return true
|
||||
} else {
|
||||
// Camera supports max. 2 use-cases and only one is occupied (either `frameProcessor` or `video`), so we can add `photo`
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
init {
|
||||
mHybridData = initHybrid()
|
||||
|
||||
previewView = PreviewView(context)
|
||||
previewView.layoutParams = LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT)
|
||||
previewView.installHierarchyFitter() // If this is not called correctly, view finder will be black/blank
|
||||
@@ -144,6 +185,28 @@ class CameraView(context: Context) : FrameLayout(context), LifecycleOwner {
|
||||
})
|
||||
}
|
||||
|
||||
fun finalize() {
|
||||
mHybridData?.resetNative()
|
||||
}
|
||||
|
||||
private external fun initHybrid(): HybridData
|
||||
private external fun frameProcessorCallback(frame: ImageProxy)
|
||||
|
||||
@Suppress("unused")
|
||||
@DoNotStrip
|
||||
fun setEnableFrameProcessor(enable: Boolean) {
|
||||
Log.d(TAG, "Set enable frame processor: $enable")
|
||||
val before = enableFrameProcessor
|
||||
enableFrameProcessor = enable
|
||||
|
||||
if (before != enable) {
|
||||
// reconfigure session if frame processor was added/removed to adjust use-cases.
|
||||
GlobalScope.launch(Dispatchers.Main) {
|
||||
configureSession()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
override fun getLifecycle(): Lifecycle {
|
||||
return lifecycleRegistry
|
||||
}
|
||||
@@ -245,6 +308,10 @@ class CameraView(context: Context) : FrameLayout(context), LifecycleOwner {
|
||||
.setCaptureMode(ImageCapture.CAPTURE_MODE_MINIMIZE_LATENCY)
|
||||
val videoCaptureBuilder = VideoCapture.Builder()
|
||||
.setTargetRotation(rotation)
|
||||
val imageAnalysisBuilder = ImageAnalysis.Builder()
|
||||
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
|
||||
.setTargetRotation(rotation)
|
||||
.setBackgroundExecutor(CameraViewModule.FrameProcessorThread)
|
||||
|
||||
if (format == null) {
|
||||
// let CameraX automatically find best resolution for the target aspect ratio
|
||||
@@ -311,11 +378,10 @@ class CameraView(context: Context) : FrameLayout(context), LifecycleOwner {
|
||||
}
|
||||
}
|
||||
|
||||
val preview = previewBuilder.build()
|
||||
|
||||
// Unbind use cases before rebinding
|
||||
videoCapture = null
|
||||
imageCapture = null
|
||||
imageAnalysis = null
|
||||
cameraProvider.unbindAll()
|
||||
|
||||
// Bind use cases to camera
|
||||
@@ -325,9 +391,32 @@ class CameraView(context: Context) : FrameLayout(context), LifecycleOwner {
|
||||
useCases.add(videoCapture!!)
|
||||
}
|
||||
if (photo == true) {
|
||||
imageCapture = imageCaptureBuilder.build()
|
||||
useCases.add(imageCapture!!)
|
||||
if (fallbackToSnapshot) {
|
||||
Log.i(TAG, "Tried to add photo use-case (`photo={true}`) but the Camera device only supports " +
|
||||
"a single use-case at a time. Falling back to Snapshot capture.")
|
||||
} else {
|
||||
imageCapture = imageCaptureBuilder.build()
|
||||
useCases.add(imageCapture!!)
|
||||
}
|
||||
}
|
||||
if (enableFrameProcessor) {
|
||||
var lastCall = System.currentTimeMillis() - 1000
|
||||
val intervalMs = (1.0 / frameProcessorFps) * 1000.0
|
||||
imageAnalysis = imageAnalysisBuilder.build().apply {
|
||||
setAnalyzer(cameraExecutor, { image ->
|
||||
val now = System.currentTimeMillis()
|
||||
if (now - lastCall > intervalMs) {
|
||||
lastCall = now
|
||||
Log.d(TAG, "Calling Frame Processor...")
|
||||
frameProcessorCallback(image)
|
||||
}
|
||||
image.close()
|
||||
})
|
||||
}
|
||||
useCases.add(imageAnalysis!!)
|
||||
}
|
||||
|
||||
val preview = previewBuilder.build()
|
||||
camera = cameraProvider.bindToLifecycle(this, cameraSelector, preview, *useCases.toTypedArray())
|
||||
preview.setSurfaceProvider(previewView.surfaceProvider)
|
||||
|
||||
@@ -338,11 +427,18 @@ class CameraView(context: Context) : FrameLayout(context), LifecycleOwner {
|
||||
Log.i(TAG_PERF, "Session configured in $duration ms! Camera: ${camera!!}")
|
||||
invokeOnInitialized()
|
||||
} catch (exc: Throwable) {
|
||||
throw when (exc) {
|
||||
val error = when (exc) {
|
||||
is CameraError -> exc
|
||||
is IllegalArgumentException -> InvalidCameraDeviceError(exc)
|
||||
is IllegalArgumentException -> {
|
||||
if (exc.message?.contains("too many use cases") == true) {
|
||||
ParallelVideoProcessingNotSupportedError(exc)
|
||||
} else {
|
||||
InvalidCameraDeviceError(exc)
|
||||
}
|
||||
}
|
||||
else -> UnknownCameraError(exc)
|
||||
}
|
||||
invokeOnError(error)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -381,7 +477,7 @@ class CameraView(context: Context) : FrameLayout(context), LifecycleOwner {
|
||||
const val TAG = "CameraView"
|
||||
const val TAG_PERF = "CameraView.performance"
|
||||
|
||||
private val propsThatRequireSessionReconfiguration = arrayListOf("cameraId", "format", "fps", "hdr", "lowLightBoost", "photo", "video")
|
||||
private val propsThatRequireSessionReconfiguration = arrayListOf("cameraId", "format", "fps", "hdr", "lowLightBoost", "photo", "video", "frameProcessorFps")
|
||||
|
||||
private val arrayListOfZoom = arrayListOf("zoom")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user