From 915ef331d3b3f3630f4b8d5a8cdd7f5fcf30c335 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Tue, 10 Oct 2023 18:51:36 +0200 Subject: [PATCH] feat: Extract CodeScannerPipeline & fix stalling (#1971) * chore: Extract CodeScannerPipeline * chore: Format * Update CodeScannerPipeline.kt --- .../camera/core/CodeScannerPipeline.kt | 75 +++++++++++++++++++ .../core/outputs/BarcodeScannerOutput.kt | 12 ++- .../camera/core/outputs/CameraOutputs.kt | 41 +--------- 3 files changed, 84 insertions(+), 44 deletions(-) create mode 100644 package/android/src/main/java/com/mrousavy/camera/core/CodeScannerPipeline.kt diff --git a/package/android/src/main/java/com/mrousavy/camera/core/CodeScannerPipeline.kt b/package/android/src/main/java/com/mrousavy/camera/core/CodeScannerPipeline.kt new file mode 100644 index 0000000..92432a2 --- /dev/null +++ b/package/android/src/main/java/com/mrousavy/camera/core/CodeScannerPipeline.kt @@ -0,0 +1,75 @@ +package com.mrousavy.camera.core + +import android.media.ImageReader +import android.util.Size +import android.view.Surface +import com.google.mlkit.vision.barcode.BarcodeScanner +import com.google.mlkit.vision.barcode.BarcodeScannerOptions +import com.google.mlkit.vision.barcode.BarcodeScanning +import com.google.mlkit.vision.common.InputImage +import com.mrousavy.camera.CameraQueues +import com.mrousavy.camera.core.outputs.CameraOutputs +import com.mrousavy.camera.parsers.Orientation +import java.io.Closeable + +class CodeScannerPipeline(val size: Size, val format: Int, val output: CameraOutputs.CodeScannerOutput) : Closeable { + companion object { + // We want to have a buffer of 2 images, but we always only acquire one. + // That way the pipeline is free to stream frames into the unused buffer, + // while the other buffer is being used for code scanning. + private const val MAX_IMAGES = 2 + } + + private val imageReader: ImageReader + private val scanner: BarcodeScanner + + val surface: Surface + get() = imageReader.surface + + init { + val types = output.codeScanner.codeTypes.map { it.toBarcodeType() } + val barcodeScannerOptions = BarcodeScannerOptions.Builder() + .setBarcodeFormats(types[0], *types.toIntArray()) + .setExecutor(CameraQueues.codeScannerQueue.executor) + .build() + scanner = BarcodeScanning.getClient(barcodeScannerOptions) + + var isBusy = false + imageReader = ImageReader.newInstance(size.width, size.height, format, MAX_IMAGES) + imageReader.setOnImageAvailableListener({ reader -> + if (isBusy) { + // We're currently executing on a previous Frame, so we skip this one. + // We don't try to acquire a new one, so that the Camera is not blocked/stalling. + return@setOnImageAvailableListener + } + val image = reader.acquireNextImage() ?: return@setOnImageAvailableListener + + isBusy = true + // TODO: Get correct orientation + val inputImage = InputImage.fromMediaImage(image, Orientation.PORTRAIT.toDegrees()) + scanner.process(inputImage) + .addOnSuccessListener { barcodes -> + image.close() + isBusy = false + if (barcodes.isNotEmpty()) { + output.onCodeScanned(barcodes) + } + } + .addOnFailureListener { error -> + image.close() + isBusy = false + output.onError(error) + } + }, CameraQueues.videoQueue.handler) + } + + override fun close() { + imageReader.close() + scanner.close() + } + + override fun toString(): String { + val codeTypes = output.codeScanner.codeTypes.joinToString(", ") + return "${size.width} x ${size.height} CodeScanner for [$codeTypes] ($format)" + } +} diff --git a/package/android/src/main/java/com/mrousavy/camera/core/outputs/BarcodeScannerOutput.kt b/package/android/src/main/java/com/mrousavy/camera/core/outputs/BarcodeScannerOutput.kt index 681d45b..7784a2b 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/outputs/BarcodeScannerOutput.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/outputs/BarcodeScannerOutput.kt @@ -1,19 +1,17 @@ package com.mrousavy.camera.core.outputs -import android.media.ImageReader import android.util.Log -import com.google.mlkit.vision.barcode.BarcodeScanner +import com.mrousavy.camera.core.CodeScannerPipeline import java.io.Closeable -class BarcodeScannerOutput(private val imageReader: ImageReader, private val barcodeScanner: BarcodeScanner) : - ImageReaderOutput(imageReader, OutputType.VIDEO), +class BarcodeScannerOutput(private val codeScannerPipeline: CodeScannerPipeline) : + SurfaceOutput(codeScannerPipeline.surface, codeScannerPipeline.size, OutputType.VIDEO), Closeable { override fun close() { Log.i(TAG, "Closing BarcodeScanner..") - barcodeScanner.close() + codeScannerPipeline.close() super.close() } - override fun toString(): String = - "$outputType (${imageReader.width} x ${imageReader.height} ${barcodeScanner.detectorType} BarcodeScanner)" + override fun toString(): String = "$outputType ($codeScannerPipeline)" } diff --git a/package/android/src/main/java/com/mrousavy/camera/core/outputs/CameraOutputs.kt b/package/android/src/main/java/com/mrousavy/camera/core/outputs/CameraOutputs.kt index 55cdc00..2f51646 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/outputs/CameraOutputs.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/outputs/CameraOutputs.kt @@ -8,11 +8,9 @@ import android.media.ImageReader import android.util.Log import android.util.Size import android.view.Surface -import com.google.mlkit.vision.barcode.BarcodeScannerOptions -import com.google.mlkit.vision.barcode.BarcodeScanning import com.google.mlkit.vision.barcode.common.Barcode -import com.google.mlkit.vision.common.InputImage import com.mrousavy.camera.CameraQueues +import com.mrousavy.camera.core.CodeScannerPipeline import com.mrousavy.camera.core.VideoPipeline import com.mrousavy.camera.extensions.bigger import com.mrousavy.camera.extensions.closestToOrMax @@ -21,7 +19,6 @@ import com.mrousavy.camera.extensions.getPreviewTargetSize import com.mrousavy.camera.extensions.getVideoSizes import com.mrousavy.camera.extensions.smaller import com.mrousavy.camera.parsers.CodeScanner -import com.mrousavy.camera.parsers.Orientation import com.mrousavy.camera.parsers.PixelFormat import java.io.Closeable @@ -168,40 +165,10 @@ class CameraOutputs( val format = ImageFormat.YUV_420_888 val targetSize = Size(1280, 720) val size = characteristics.getVideoSizes(cameraId, format).closestToOrMax(targetSize) + val pipeline = CodeScannerPipeline(size, format, codeScanner) - val types = codeScanner.codeScanner.codeTypes.map { it.toBarcodeType() } - val barcodeScannerOptions = BarcodeScannerOptions.Builder() - .setBarcodeFormats(types[0], *types.toIntArray()) - .setExecutor(CameraQueues.codeScannerQueue.executor) - .build() - val scanner = BarcodeScanning.getClient(barcodeScannerOptions) - - var isBusy = false - val imageReader = ImageReader.newInstance(size.width, size.height, format, 1) - imageReader.setOnImageAvailableListener({ reader -> - if (isBusy) return@setOnImageAvailableListener - val image = reader.acquireNextImage() ?: return@setOnImageAvailableListener - - isBusy = true - // TODO: Get correct orientation - val inputImage = InputImage.fromMediaImage(image, Orientation.PORTRAIT.toDegrees()) - scanner.process(inputImage) - .addOnSuccessListener { barcodes -> - image.close() - isBusy = false - if (barcodes.isNotEmpty()) { - codeScanner.onCodeScanned(barcodes) - } - } - .addOnFailureListener { error -> - image.close() - isBusy = false - codeScanner.onError(error) - } - }, CameraQueues.videoQueue.handler) - - Log.i(TAG, "Adding ${size.width}x${size.height} code scanner output. (Code Types: $types)") - codeScannerOutput = BarcodeScannerOutput(imageReader, scanner) + Log.i(TAG, "Adding ${size.width}x${size.height} code scanner output. (Code Types: ${codeScanner.codeScanner.codeTypes})") + codeScannerOutput = BarcodeScannerOutput(pipeline) } Log.i(TAG, "Prepared $size Outputs for Camera $cameraId!")