From af14f912fbec09b88df63d9abb9d570fad1dce1a Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Tue, 30 Jan 2024 14:17:32 +0100 Subject: [PATCH] chore: Move `onFrame` into Callback on Android (#2458) * Separate to onFrame * Restructure FP * Move lib loading into `CameraViewModule` --- .../java/com/mrousavy/camera/CameraView.kt | 11 ++--- .../com/mrousavy/camera/CameraViewModule.kt | 11 +++++ .../com/mrousavy/camera/core/CameraSession.kt | 10 +++-- .../camera/core/CodeScannerPipeline.kt | 2 +- .../com/mrousavy/camera/core/VideoPipeline.kt | 43 +++---------------- .../frameprocessor/VisionCameraProxy.kt | 8 ---- 6 files changed, 30 insertions(+), 55 deletions(-) diff --git a/package/android/src/main/java/com/mrousavy/camera/CameraView.kt b/package/android/src/main/java/com/mrousavy/camera/CameraView.kt index 4d67098..b59af02 100644 --- a/package/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/package/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -14,6 +14,7 @@ import com.mrousavy.camera.core.CameraSession import com.mrousavy.camera.core.CodeScannerFrame import com.mrousavy.camera.core.PreviewView import com.mrousavy.camera.extensions.installHierarchyFitter +import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.FrameProcessor import com.mrousavy.camera.types.CameraDeviceFormat import com.mrousavy.camera.types.CodeScannerOptions @@ -39,7 +40,7 @@ import kotlinx.coroutines.launch class CameraView(context: Context) : FrameLayout(context), CoroutineScope, - CameraSession.CameraSessionCallback { + CameraSession.Callback { companion object { const val TAG = "CameraView" } @@ -96,10 +97,6 @@ class CameraView(context: Context) : private var currentConfigureCall: Long = System.currentTimeMillis() internal var frameProcessor: FrameProcessor? = null - set(value) { - field = value - cameraSession.frameProcessor = frameProcessor - } override val coroutineContext: CoroutineContext = CameraQueues.cameraQueue.coroutineDispatcher @@ -230,6 +227,10 @@ class CameraView(context: Context) : } } + override fun onFrame(frame: Frame) { + frameProcessor?.call(frame) + } + override fun onError(error: Throwable) { invokeOnError(error) } diff --git a/package/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt b/package/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt index 532359f..b422815 100644 --- a/package/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt +++ b/package/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt @@ -26,6 +26,17 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase companion object { const val TAG = "CameraView" var sharedRequestCode = 10 + + init { + try { + // Load the native part of VisionCamera. + // Includes the OpenGL VideoPipeline, as well as Frame Processor JSI bindings + System.loadLibrary("VisionCamera") + } catch (e: UnsatisfiedLinkError) { + Log.e(VisionCameraProxy.TAG, "Failed to load VisionCamera C++ library!", e) + throw e + } + } } private val coroutineScope = CoroutineScope(Dispatchers.Default) // TODO: or Dispatchers.Main? diff --git a/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt b/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt index df14fd2..0fef58e 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt @@ -37,6 +37,7 @@ import com.mrousavy.camera.extensions.getPreviewTargetSize import com.mrousavy.camera.extensions.getVideoSizes import com.mrousavy.camera.extensions.openCamera import com.mrousavy.camera.extensions.setZoom +import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.FrameProcessor import com.mrousavy.camera.types.Flash import com.mrousavy.camera.types.Orientation @@ -55,7 +56,7 @@ import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -class CameraSession(private val context: Context, private val cameraManager: CameraManager, private val callback: CameraSessionCallback) : +class CameraSession(private val context: Context, private val cameraManager: CameraManager, private val callback: Callback) : CameraManager.AvailabilityCallback(), Closeable, CoroutineScope { @@ -383,7 +384,8 @@ class CameraSession(private val context: Context, private val cameraManager: Cam size.height, video.config.pixelFormat, isSelfie, - video.config.enableFrameProcessor + video.config.enableFrameProcessor, + callback ) val output = VideoPipelineOutput(videoPipeline, video.config.enableHdr) outputs.add(output) @@ -615,7 +617,6 @@ class CameraSession(private val context: Context, private val cameraManager: Cam private fun updateVideoOutputs() { val videoOutput = videoOutput ?: return Log.i(TAG, "Updating Video Outputs...") - videoOutput.videoPipeline.setFrameProcessorOutput(frameProcessor) videoOutput.videoPipeline.setRecordingSessionOutput(recording) } @@ -720,8 +721,9 @@ class CameraSession(private val context: Context, private val cameraManager: Cam } } - interface CameraSessionCallback { + interface Callback { fun onError(error: Throwable) + fun onFrame(frame: Frame) fun onInitialized() fun onStarted() fun onStopped() diff --git a/package/android/src/main/java/com/mrousavy/camera/core/CodeScannerPipeline.kt b/package/android/src/main/java/com/mrousavy/camera/core/CodeScannerPipeline.kt index d470245..6a22f59 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/CodeScannerPipeline.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/CodeScannerPipeline.kt @@ -14,7 +14,7 @@ class CodeScannerPipeline( val size: Size, val format: Int, val configuration: CameraConfiguration.CodeScanner, - val callback: CameraSession.CameraSessionCallback + val callback: CameraSession.Callback ) : Closeable { companion object { // We want to have a buffer of 2 images, but we always only acquire one. diff --git a/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt b/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt index dcd9617..57e1d78 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt @@ -31,26 +31,13 @@ class VideoPipeline( val height: Int, val format: PixelFormat = PixelFormat.NATIVE, private val isMirrored: Boolean = false, - enableFrameProcessor: Boolean = false + enableFrameProcessor: Boolean = false, + private val callback: CameraSession.Callback ) : SurfaceTexture.OnFrameAvailableListener, Closeable { companion object { private const val MAX_IMAGES = 3 private const val TAG = "VideoPipeline" - - init { - try { - System.loadLibrary("VisionCamera") - } catch (e: UnsatisfiedLinkError) { - Log.e( - TAG, - "Failed to load VisionCamera C++ library! " + - "OpenGL GPU VideoPipeline cannot be used.", - e - ) - throw e - } - } } @DoNotStrip @@ -60,16 +47,13 @@ class VideoPipeline( private var transformMatrix = FloatArray(16) private var isActive = true - // Output 1 - private var frameProcessor: FrameProcessor? = null - - // Output 2 - private var recordingSession: RecordingSession? = null - // Input private val surfaceTexture: SurfaceTexture val surface: Surface + // Output + private var recordingSession: RecordingSession? = null + // If Frame Processors are enabled, we go through ImageReader first before we go thru OpenGL private var imageReader: ImageReader? = null private var imageWriter: ImageWriter? = null @@ -115,7 +99,7 @@ class VideoPipeline( frame.incrementRefCount() try { - frameProcessor?.call(frame) + callback.onFrame(frame) if (hasOutputs) { // If we have outputs (e.g. a RecordingSession), pass the frame along to the OpenGL pipeline @@ -141,7 +125,6 @@ class VideoPipeline( isActive = false imageWriter?.close() imageReader?.close() - frameProcessor = null recordingSession = null surfaceTexture.release() } @@ -180,20 +163,6 @@ class VideoPipeline( else -> ImageFormat.PRIVATE } - /** - * Configures the Pipeline to also call the given [FrameProcessor] (or null). - */ - fun setFrameProcessorOutput(frameProcessor: FrameProcessor?) { - synchronized(this) { - if (frameProcessor != null) { - Log.i(TAG, "Setting $width x $height FrameProcessor Output...") - } else { - Log.i(TAG, "Removing FrameProcessor Output...") - } - this.frameProcessor = frameProcessor - } - } - /** * Configures the Pipeline to also write Frames to a Surface from a `MediaRecorder` (or null) */ diff --git a/package/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt b/package/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt index 928198f..db41e55 100644 --- a/package/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt +++ b/package/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt @@ -17,14 +17,6 @@ import java.lang.ref.WeakReference class VisionCameraProxy(context: ReactApplicationContext) { companion object { const val TAG = "VisionCameraProxy" - init { - try { - System.loadLibrary("VisionCamera") - } catch (e: UnsatisfiedLinkError) { - Log.e(TAG, "Failed to load VisionCamera C++ library!", e) - throw e - } - } } @DoNotStrip