feat: Extract CodeScannerPipeline & fix stalling (#1971)

* chore: Extract CodeScannerPipeline

* chore: Format

* Update CodeScannerPipeline.kt

package/android/src/main/java/com/mrousavy/camera/core/CodeScannerPipeline.kt

@@ -0,0 +1,75 @@
+package com.mrousavy.camera.core
+
+import android.media.ImageReader
+import android.util.Size
+import android.view.Surface
+import com.google.mlkit.vision.barcode.BarcodeScanner
+import com.google.mlkit.vision.barcode.BarcodeScannerOptions
+import com.google.mlkit.vision.barcode.BarcodeScanning
+import com.google.mlkit.vision.common.InputImage
+import com.mrousavy.camera.CameraQueues
+import com.mrousavy.camera.core.outputs.CameraOutputs
+import com.mrousavy.camera.parsers.Orientation
+import java.io.Closeable
+
+class CodeScannerPipeline(val size: Size, val format: Int, val output: CameraOutputs.CodeScannerOutput) : Closeable {
+  companion object {
+    // We want to have a buffer of 2 images, but we always only acquire one.
+    // That way the pipeline is free to stream frames into the unused buffer,
+    // while the other buffer is being used for code scanning.
+    private const val MAX_IMAGES = 2
+  }
+
+  private val imageReader: ImageReader
+  private val scanner: BarcodeScanner
+
+  val surface: Surface
+    get() = imageReader.surface
+
+  init {
+    val types = output.codeScanner.codeTypes.map { it.toBarcodeType() }
+    val barcodeScannerOptions = BarcodeScannerOptions.Builder()
+      .setBarcodeFormats(types[0], *types.toIntArray())
+      .setExecutor(CameraQueues.codeScannerQueue.executor)
+      .build()
+    scanner = BarcodeScanning.getClient(barcodeScannerOptions)
+
+    var isBusy = false
+    imageReader = ImageReader.newInstance(size.width, size.height, format, MAX_IMAGES)
+    imageReader.setOnImageAvailableListener({ reader ->
+      if (isBusy) {
+        // We're currently executing on a previous Frame, so we skip this one.
+        // We don't try to acquire a new one, so that the Camera is not blocked/stalling.
+        return@setOnImageAvailableListener
+      }
+      val image = reader.acquireNextImage() ?: return@setOnImageAvailableListener
+
+      isBusy = true
+      // TODO: Get correct orientation
+      val inputImage = InputImage.fromMediaImage(image, Orientation.PORTRAIT.toDegrees())
+      scanner.process(inputImage)
+        .addOnSuccessListener { barcodes ->
+          image.close()
+          isBusy = false
+          if (barcodes.isNotEmpty()) {
+            output.onCodeScanned(barcodes)
+          }
+        }
+        .addOnFailureListener { error ->
+          image.close()
+          isBusy = false
+          output.onError(error)
+        }
+    }, CameraQueues.videoQueue.handler)
+  }
+
+  override fun close() {
+    imageReader.close()
+    scanner.close()
+  }
+
+  override fun toString(): String {
+    val codeTypes = output.codeScanner.codeTypes.joinToString(", ")
+    return "${size.width} x ${size.height} CodeScanner for [$codeTypes] ($format)"
+  }
+}

package/android/src/main/java/com/mrousavy/camera/core/outputs/BarcodeScannerOutput.kt

@@ -1,19 +1,17 @@
 package com.mrousavy.camera.core.outputs
 
-import android.media.ImageReader
 import android.util.Log
-import com.google.mlkit.vision.barcode.BarcodeScanner
+import com.mrousavy.camera.core.CodeScannerPipeline
 import java.io.Closeable
 
-class BarcodeScannerOutput(private val imageReader: ImageReader, private val barcodeScanner: BarcodeScanner) :
+class BarcodeScannerOutput(private val codeScannerPipeline: CodeScannerPipeline) :
-  ImageReaderOutput(imageReader, OutputType.VIDEO),
+  SurfaceOutput(codeScannerPipeline.surface, codeScannerPipeline.size, OutputType.VIDEO),
   Closeable {
   override fun close() {
     Log.i(TAG, "Closing BarcodeScanner..")
-    barcodeScanner.close()
+    codeScannerPipeline.close()
     super.close()
   }
 
-  override fun toString(): String =
+  override fun toString(): String = "$outputType ($codeScannerPipeline)"
-    "$outputType (${imageReader.width} x ${imageReader.height} ${barcodeScanner.detectorType} BarcodeScanner)"
 }

package/android/src/main/java/com/mrousavy/camera/core/outputs/CameraOutputs.kt

@@ -8,11 +8,9 @@ import android.media.ImageReader
 import android.util.Log
 import android.util.Size
 import android.view.Surface
-import com.google.mlkit.vision.barcode.BarcodeScannerOptions
-import com.google.mlkit.vision.barcode.BarcodeScanning
 import com.google.mlkit.vision.barcode.common.Barcode
-import com.google.mlkit.vision.common.InputImage
 import com.mrousavy.camera.CameraQueues
+import com.mrousavy.camera.core.CodeScannerPipeline
 import com.mrousavy.camera.core.VideoPipeline
 import com.mrousavy.camera.extensions.bigger
 import com.mrousavy.camera.extensions.closestToOrMax
@@ -21,7 +19,6 @@ import com.mrousavy.camera.extensions.getPreviewTargetSize
 import com.mrousavy.camera.extensions.getVideoSizes
 import com.mrousavy.camera.extensions.smaller
 import com.mrousavy.camera.parsers.CodeScanner
-import com.mrousavy.camera.parsers.Orientation
 import com.mrousavy.camera.parsers.PixelFormat
 import java.io.Closeable
 
@@ -168,40 +165,10 @@ class CameraOutputs(
       val format = ImageFormat.YUV_420_888
       val targetSize = Size(1280, 720)
       val size = characteristics.getVideoSizes(cameraId, format).closestToOrMax(targetSize)
+      val pipeline = CodeScannerPipeline(size, format, codeScanner)
 
-      val types = codeScanner.codeScanner.codeTypes.map { it.toBarcodeType() }
+      Log.i(TAG, "Adding ${size.width}x${size.height} code scanner output. (Code Types: ${codeScanner.codeScanner.codeTypes})")
-      val barcodeScannerOptions = BarcodeScannerOptions.Builder()
+      codeScannerOutput = BarcodeScannerOutput(pipeline)
-        .setBarcodeFormats(types[0], *types.toIntArray())
-        .setExecutor(CameraQueues.codeScannerQueue.executor)
-        .build()
-      val scanner = BarcodeScanning.getClient(barcodeScannerOptions)
-
-      var isBusy = false
-      val imageReader = ImageReader.newInstance(size.width, size.height, format, 1)
-      imageReader.setOnImageAvailableListener({ reader ->
-        if (isBusy) return@setOnImageAvailableListener
-        val image = reader.acquireNextImage() ?: return@setOnImageAvailableListener
-
-        isBusy = true
-        // TODO: Get correct orientation
-        val inputImage = InputImage.fromMediaImage(image, Orientation.PORTRAIT.toDegrees())
-        scanner.process(inputImage)
-          .addOnSuccessListener { barcodes ->
-            image.close()
-            isBusy = false
-            if (barcodes.isNotEmpty()) {
-              codeScanner.onCodeScanned(barcodes)
-            }
-          }
-          .addOnFailureListener { error ->
-            image.close()
-            isBusy = false
-            codeScanner.onError(error)
-          }
-      }, CameraQueues.videoQueue.handler)
-
-      Log.i(TAG, "Adding ${size.width}x${size.height} code scanner output. (Code Types: $types)")
-      codeScannerOutput = BarcodeScannerOutput(imageReader, scanner)
     }
 
     Log.i(TAG, "Prepared $size Outputs for Camera $cameraId!")