feat: Code Scanner API (#1912)

* feat: CodeScanner JS API

* feat: iOS

* Use guard

* Format

* feat: Android base

* fix: Attach Surfaces

* Use isBusy var

* fix: Use separate Queue

* feat: Finish iOS types

* feat: Implement all other code types on Android

* fix: Call JS event

* fix: Pass codetypes on Android

* fix: iOS use Preview coordinate system

* docs: Add comments

* chore: Format code

* Update CameraView+AVCaptureSession.swift

* docs: Add Code Scanner docs

* docs: Update

* feat: Use lazily downloaded model on Android

* Revert changes in CameraPage

* Format

* fix: Fix empty QR codes

* Update README.md
This commit is contained in:
Marc Rousavy
2023-10-04 12:53:52 +02:00
committed by GitHub
parent 2c08e5ae78
commit 6640b72a00
36 changed files with 763 additions and 29 deletions

View File

@@ -79,6 +79,13 @@ class RecordingInProgressError :
"There is already an active video recording in progress! Did you call startRecording() twice?"
)
class CodeTypeNotSupportedError(codeType: String) :
CameraError(
"code-scanner",
"code-type-not-supported",
"The codeType \"$codeType\" is not supported by the Code Scanner!"
)
class ViewNotFoundError(viewId: Int) :
CameraError("system", "view-not-found", "The given view (ID $viewId) was not found in the view manager.")
class FrameProcessorsUnavailableError(reason: String) :

View File

@@ -11,6 +11,7 @@ class CameraQueues {
companion object {
val cameraQueue = CameraQueue("mrousavy/VisionCamera.main")
val videoQueue = CameraQueue("mrousavy/VisionCamera.video")
val codeScannerQueue = CameraQueue("mrousavy/VisionCamera.codeScanner")
}
class CameraQueue(name: String) {

View File

@@ -5,6 +5,8 @@ import com.facebook.react.bridge.Arguments
import com.facebook.react.bridge.ReactContext
import com.facebook.react.bridge.WritableMap
import com.facebook.react.uimanager.events.RCTEventEmitter
import com.google.mlkit.vision.barcode.common.Barcode
import com.mrousavy.camera.parsers.CodeType
fun CameraView.invokeOnInitialized() {
Log.i(CameraView.TAG, "invokeOnInitialized()")
@@ -37,6 +39,31 @@ fun CameraView.invokeOnViewReady() {
reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraViewReady", event)
}
fun CameraView.invokeOnCodeScanned(barcodes: List<Barcode>) {
val codes = Arguments.createArray()
barcodes.forEach { barcode ->
val code = Arguments.createMap()
val type = CodeType.fromBarcodeType(barcode.format)
code.putString("type", type.unionValue)
code.putString("value", barcode.rawValue)
barcode.boundingBox?.let { rect ->
val frame = Arguments.createMap()
frame.putInt("x", rect.left)
frame.putInt("y", rect.top)
frame.putInt("width", rect.right - rect.left)
frame.putInt("height", rect.bottom - rect.top)
code.putMap("frame", frame)
}
codes.pushMap(code)
}
val event = Arguments.createMap()
event.putArray("codes", codes)
val reactContext = context as ReactContext
reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraCodeScanned", event)
}
private fun errorToMap(error: Throwable): WritableMap {
val map = Arguments.createMap()
map.putString("message", error.message)

View File

@@ -22,6 +22,7 @@ import com.mrousavy.camera.extensions.getPreviewTargetSize
import com.mrousavy.camera.extensions.installHierarchyFitter
import com.mrousavy.camera.extensions.smaller
import com.mrousavy.camera.frameprocessor.FrameProcessor
import com.mrousavy.camera.parsers.CodeScanner
import com.mrousavy.camera.parsers.Orientation
import com.mrousavy.camera.parsers.PixelFormat
import com.mrousavy.camera.parsers.ResizeMode
@@ -47,7 +48,7 @@ class CameraView(context: Context) : FrameLayout(context) {
private val propsThatRequirePreviewReconfiguration = arrayListOf("cameraId", "format", "resizeMode")
private val propsThatRequireSessionReconfiguration =
arrayListOf("cameraId", "format", "photo", "video", "enableFrameProcessor", "pixelFormat")
arrayListOf("cameraId", "format", "photo", "video", "enableFrameProcessor", "codeScannerOptions", "pixelFormat")
private val propsThatRequireFormatReconfiguration = arrayListOf("fps", "hdr", "videoStabilizationMode", "lowLightBoost")
}
@@ -80,6 +81,9 @@ class CameraView(context: Context) : FrameLayout(context) {
var orientation: Orientation? = null
var enableZoomGesture: Boolean = false
// code scanner
var codeScannerOptions: CodeScanner? = null
// private properties
private var isMounted = false
internal val cameraManager = context.getSystemService(Context.CAMERA_SERVICE) as CameraManager
@@ -202,6 +206,7 @@ class CameraView(context: Context) : FrameLayout(context) {
val targetPhotoSize = if (format != null) Size(format.getInt("photoWidth"), format.getInt("photoHeight")) else null
// TODO: Allow previewSurface to be null/none
val previewSurface = previewSurface ?: return
val codeScannerOptions = codeScannerOptions
val previewOutput = CameraOutputs.PreviewOutput(previewSurface, previewView?.targetSize)
val photoOutput = if (photo == true) {
@@ -214,8 +219,17 @@ class CameraView(context: Context) : FrameLayout(context) {
} else {
null
}
val codeScanner = if (codeScannerOptions != null) {
CameraOutputs.CodeScannerOutput(
codeScannerOptions,
{ codes -> invokeOnCodeScanned(codes) },
{ error -> invokeOnError(error) }
)
} else {
null
}
cameraSession.configureSession(cameraId, previewOutput, photoOutput, videoOutput)
cameraSession.configureSession(cameraId, previewOutput, photoOutput, videoOutput, codeScanner)
} catch (e: Throwable) {
Log.e(TAG, "Failed to configure session: ${e.message}", e)
invokeOnError(e)

View File

@@ -5,6 +5,7 @@ import com.facebook.react.common.MapBuilder
import com.facebook.react.uimanager.ThemedReactContext
import com.facebook.react.uimanager.ViewGroupManager
import com.facebook.react.uimanager.annotations.ReactProp
import com.mrousavy.camera.parsers.CodeScanner
import com.mrousavy.camera.parsers.Orientation
import com.mrousavy.camera.parsers.PixelFormat
import com.mrousavy.camera.parsers.ResizeMode
@@ -27,6 +28,7 @@ class CameraViewManager : ViewGroupManager<CameraView>() {
.put("cameraViewReady", MapBuilder.of("registrationName", "onViewReady"))
.put("cameraInitialized", MapBuilder.of("registrationName", "onInitialized"))
.put("cameraError", MapBuilder.of("registrationName", "onError"))
.put("cameraCodeScanned", MapBuilder.of("registrationName", "onCodeScanned"))
.build()
override fun getName(): String = TAG
@@ -200,6 +202,15 @@ class CameraViewManager : ViewGroupManager<CameraView>() {
view.orientation = newMode
}
@ReactProp(name = "codeScannerOptions")
fun setCodeScanner(view: CameraView, codeScannerOptions: ReadableMap) {
val newCodeScannerOptions = CodeScanner(codeScannerOptions)
if (view.codeScannerOptions != newCodeScannerOptions) {
addChangedPropToTransaction(view, "codeScannerOptions")
}
view.codeScannerOptions = newCodeScannerOptions
}
companion object {
const val TAG = "CameraView"

View File

@@ -139,7 +139,8 @@ class CameraSession(
cameraId: String,
preview: CameraOutputs.PreviewOutput? = null,
photo: CameraOutputs.PhotoOutput? = null,
video: CameraOutputs.VideoOutput? = null
video: CameraOutputs.VideoOutput? = null,
codeScanner: CameraOutputs.CodeScannerOutput? = null
) {
Log.i(TAG, "Configuring Session for Camera $cameraId...")
val outputs = CameraOutputs(
@@ -148,6 +149,7 @@ class CameraSession(
preview,
photo,
video,
codeScanner,
hdr == true,
this
)
@@ -190,6 +192,7 @@ class CameraSession(
currentOutputs.preview,
currentOutputs.photo,
currentOutputs.video,
currentOutputs.codeScanner,
hdr,
this
)
@@ -534,11 +537,15 @@ class CameraSession(
val template = if (outputs.videoOutput != null) CameraDevice.TEMPLATE_RECORD else CameraDevice.TEMPLATE_PREVIEW
val captureRequest = camera.createCaptureRequest(template)
outputs.previewOutput?.let { output ->
Log.i(TAG, "Adding output surface ${output.outputType}..")
Log.i(TAG, "Adding preview output surface ${output.outputType}..")
captureRequest.addTarget(output.surface)
}
outputs.videoOutput?.let { output ->
Log.i(TAG, "Adding output surface ${output.outputType}..")
Log.i(TAG, "Adding video output surface ${output.outputType}..")
captureRequest.addTarget(output.surface)
}
outputs.codeScannerOutput?.let { output ->
Log.i(TAG, "Adding code scanner output surface ${output.outputType}")
captureRequest.addTarget(output.surface)
}

View File

@@ -0,0 +1,19 @@
package com.mrousavy.camera.core.outputs
import android.media.ImageReader
import android.util.Log
import com.google.mlkit.vision.barcode.BarcodeScanner
import java.io.Closeable
class BarcodeScannerOutput(private val imageReader: ImageReader, private val barcodeScanner: BarcodeScanner) :
ImageReaderOutput(imageReader, OutputType.VIDEO),
Closeable {
override fun close() {
Log.i(TAG, "Closing BarcodeScanner..")
barcodeScanner.close()
super.close()
}
override fun toString(): String =
"$outputType (${imageReader.width} x ${imageReader.height} ${barcodeScanner.detectorType} BarcodeScanner)"
}

View File

@@ -8,6 +8,10 @@ import android.media.ImageReader
import android.util.Log
import android.util.Size
import android.view.Surface
import com.google.mlkit.vision.barcode.BarcodeScannerOptions
import com.google.mlkit.vision.barcode.BarcodeScanning
import com.google.mlkit.vision.barcode.common.Barcode
import com.google.mlkit.vision.common.InputImage
import com.mrousavy.camera.CameraQueues
import com.mrousavy.camera.core.VideoPipeline
import com.mrousavy.camera.extensions.bigger
@@ -16,6 +20,8 @@ import com.mrousavy.camera.extensions.getPhotoSizes
import com.mrousavy.camera.extensions.getPreviewTargetSize
import com.mrousavy.camera.extensions.getVideoSizes
import com.mrousavy.camera.extensions.smaller
import com.mrousavy.camera.parsers.CodeScanner
import com.mrousavy.camera.parsers.Orientation
import com.mrousavy.camera.parsers.PixelFormat
import java.io.Closeable
@@ -25,6 +31,7 @@ class CameraOutputs(
val preview: PreviewOutput? = null,
val photo: PhotoOutput? = null,
val video: VideoOutput? = null,
val codeScanner: CodeScannerOutput? = null,
val enableHdr: Boolean? = false,
val callback: Callback
) : Closeable {
@@ -41,6 +48,11 @@ class CameraOutputs(
val enableFrameProcessor: Boolean? = false,
val format: PixelFormat = PixelFormat.NATIVE
)
data class CodeScannerOutput(
val codeScanner: CodeScanner,
val onCodeScanned: (codes: List<Barcode>) -> Unit,
val onError: (error: Throwable) -> Unit
)
interface Callback {
fun onPhotoCaptured(image: Image)
@@ -52,6 +64,8 @@ class CameraOutputs(
private set
var videoOutput: VideoPipelineOutput? = null
private set
var codeScannerOutput: BarcodeScannerOutput? = null
private set
val size: Int
get() {
@@ -59,6 +73,7 @@ class CameraOutputs(
if (previewOutput != null) size++
if (photoOutput != null) size++
if (videoOutput != null) size++
if (codeScannerOutput != null) size++
return size
}
@@ -72,6 +87,7 @@ class CameraOutputs(
this.video?.enableRecording == other.video?.enableRecording &&
this.video?.targetSize == other.video?.targetSize &&
this.video?.format == other.video?.format &&
this.codeScanner?.codeScanner == other.codeScanner?.codeScanner &&
this.enableHdr == other.enableHdr
}
@@ -80,12 +96,14 @@ class CameraOutputs(
result += (preview?.hashCode() ?: 0)
result += (photo?.hashCode() ?: 0)
result += (video?.hashCode() ?: 0)
result += (codeScanner?.hashCode() ?: 0)
return result
}
override fun close() {
photoOutput?.close()
videoOutput?.close()
codeScannerOutput?.close()
}
override fun toString(): String {
@@ -93,6 +111,7 @@ class CameraOutputs(
previewOutput?.let { strings.add(it.toString()) }
photoOutput?.let { strings.add(it.toString()) }
videoOutput?.let { strings.add(it.toString()) }
codeScannerOutput?.let { strings.add(it.toString()) }
return strings.joinToString(", ", "[", "]")
}
@@ -144,6 +163,47 @@ class CameraOutputs(
videoOutput = VideoPipelineOutput(videoPipeline, SurfaceOutput.OutputType.VIDEO)
}
// Code Scanner
if (codeScanner != null) {
val format = ImageFormat.YUV_420_888
val targetSize = Size(1280, 720)
val size = characteristics.getVideoSizes(cameraId, format).closestToOrMax(targetSize)
val types = codeScanner.codeScanner.codeTypes.map { it.toBarcodeType() }
val barcodeScannerOptions = BarcodeScannerOptions.Builder()
.setBarcodeFormats(types[0], *types.toIntArray())
.setExecutor(CameraQueues.codeScannerQueue.executor)
.build()
val scanner = BarcodeScanning.getClient(barcodeScannerOptions)
var isBusy = false
val imageReader = ImageReader.newInstance(size.width, size.height, format, 1)
imageReader.setOnImageAvailableListener({ reader ->
if (isBusy) return@setOnImageAvailableListener
val image = reader.acquireNextImage() ?: return@setOnImageAvailableListener
isBusy = true
// TODO: Get correct orientation
val inputImage = InputImage.fromMediaImage(image, Orientation.PORTRAIT.toDegrees())
scanner.process(inputImage)
.addOnSuccessListener { barcodes ->
image.close()
isBusy = false
if (barcodes.isNotEmpty()) {
codeScanner.onCodeScanned(barcodes)
}
}
.addOnFailureListener { error ->
image.close()
isBusy = false
codeScanner.onError(error)
}
}, CameraQueues.videoQueue.handler)
Log.i(TAG, "Adding ${size.width}x${size.height} code scanner output. (Code Types: $types)")
codeScannerOutput = BarcodeScannerOutput(imageReader, scanner)
}
Log.i(TAG, "Prepared $size Outputs for Camera $cameraId!")
}
}

View File

@@ -5,7 +5,7 @@ import android.util.Log
import android.util.Size
import java.io.Closeable
class ImageReaderOutput(private val imageReader: ImageReader, outputType: OutputType, dynamicRangeProfile: Long? = null) :
open class ImageReaderOutput(private val imageReader: ImageReader, outputType: OutputType, dynamicRangeProfile: Long? = null) :
SurfaceOutput(
imageReader.surface,
Size(imageReader.width, imageReader.height),
@@ -16,6 +16,7 @@ class ImageReaderOutput(private val imageReader: ImageReader, outputType: Output
override fun close() {
Log.i(TAG, "Closing ${imageReader.width}x${imageReader.height} $outputType ImageReader..")
imageReader.close()
super.close()
}
override fun toString(): String = "$outputType (${imageReader.width} x ${imageReader.height} in format #${imageReader.imageFormat})"

View File

@@ -16,6 +16,7 @@ class VideoPipelineOutput(val videoPipeline: VideoPipeline, outputType: OutputTy
override fun close() {
Log.i(TAG, "Closing ${videoPipeline.width}x${videoPipeline.height} Video Pipeline..")
videoPipeline.close()
super.close()
}
override fun toString(): String = "$outputType (${videoPipeline.width} x ${videoPipeline.height} in format #${videoPipeline.format})"

View File

@@ -62,6 +62,9 @@ suspend fun CameraDevice.createCaptureSession(
outputs.videoOutput?.let { output ->
outputConfigurations.add(output.toOutputConfiguration(characteristics))
}
outputs.codeScannerOutput?.let { output ->
outputConfigurations.add(output.toOutputConfiguration(characteristics))
}
if (outputs.enableHdr == true && Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
val supportedProfiles = characteristics.get(CameraCharacteristics.REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES)
val hdrProfile = supportedProfiles?.bestProfile ?: supportedProfiles?.supportedProfiles?.firstOrNull()

View File

@@ -0,0 +1,22 @@
package com.mrousavy.camera.parsers
import com.facebook.react.bridge.ReadableMap
import com.mrousavy.camera.InvalidTypeScriptUnionError
class CodeScanner(map: ReadableMap) {
val codeTypes: List<CodeType>
init {
val codeTypes = map.getArray("codeTypes")?.toArrayList() ?: throw InvalidTypeScriptUnionError("codeScanner", map.toString())
this.codeTypes = codeTypes.map {
return@map CodeType.fromUnionValue(it as String)
}
}
override fun equals(other: Any?): Boolean {
if (other !is CodeScanner) return false
return codeTypes.size == other.codeTypes.size && codeTypes.containsAll(other.codeTypes)
}
override fun hashCode(): Int = codeTypes.hashCode()
}

View File

@@ -0,0 +1,74 @@
package com.mrousavy.camera.parsers
import com.google.mlkit.vision.barcode.common.Barcode
import com.mrousavy.camera.CodeTypeNotSupportedError
import com.mrousavy.camera.InvalidTypeScriptUnionError
enum class CodeType(override val unionValue: String) : JSUnionValue {
CODE_128("code-128"),
CODE_39("code-39"),
CODE_93("code-93"),
CODABAR("codabar"),
EAN_13("ean-13"),
EAN_8("ean-8"),
ITF("itf"),
UPC_E("upc-e"),
QR("qr"),
PDF_417("pdf-417"),
AZTEC("aztec"),
DATA_MATRIX("data-matrix"),
UNKNOWN("unknown");
fun toBarcodeType(): Int =
when (this) {
CODE_128 -> Barcode.FORMAT_CODE_128
CODE_39 -> Barcode.FORMAT_CODE_39
CODE_93 -> Barcode.FORMAT_CODE_93
CODABAR -> Barcode.FORMAT_CODABAR
EAN_13 -> Barcode.FORMAT_EAN_13
EAN_8 -> Barcode.FORMAT_EAN_8
ITF -> Barcode.FORMAT_ITF
UPC_E -> Barcode.FORMAT_UPC_E
QR -> Barcode.FORMAT_QR_CODE
PDF_417 -> Barcode.FORMAT_PDF417
AZTEC -> Barcode.FORMAT_AZTEC
DATA_MATRIX -> Barcode.FORMAT_DATA_MATRIX
UNKNOWN -> throw CodeTypeNotSupportedError(this.unionValue)
}
companion object : JSUnionValue.Companion<CodeType> {
fun fromBarcodeType(barcodeType: Int): CodeType =
when (barcodeType) {
Barcode.FORMAT_CODE_128 -> CODE_128
Barcode.FORMAT_CODE_39 -> CODE_39
Barcode.FORMAT_CODE_93 -> CODE_93
Barcode.FORMAT_CODABAR -> CODABAR
Barcode.FORMAT_EAN_13 -> EAN_13
Barcode.FORMAT_EAN_8 -> EAN_8
Barcode.FORMAT_ITF -> ITF
Barcode.FORMAT_UPC_E -> UPC_E
Barcode.FORMAT_QR_CODE -> QR
Barcode.FORMAT_PDF417 -> PDF_417
Barcode.FORMAT_AZTEC -> AZTEC
Barcode.FORMAT_DATA_MATRIX -> DATA_MATRIX
else -> UNKNOWN
}
override fun fromUnionValue(unionValue: String?): CodeType =
when (unionValue) {
"code-128" -> CODE_128
"code-39" -> CODE_39
"code-93" -> CODE_93
"codabar" -> CODABAR
"ean-13" -> EAN_13
"ean-8" -> EAN_8
"itf" -> ITF
"upc-e" -> UPC_E
"qr" -> QR
"pdf-417" -> PDF_417
"aztec" -> AZTEC
"data-matrix" -> DATA_MATRIX
else -> throw InvalidTypeScriptUnionError("codeType", unionValue ?: "(null)")
}
}
}

View File

@@ -9,31 +9,28 @@ enum class PixelFormat(override val unionValue: String) : JSUnionValue {
NATIVE("native"),
UNKNOWN("unknown");
fun toImageFormat(): Int {
return when (this) {
fun toImageFormat(): Int =
when (this) {
YUV -> ImageFormat.YUV_420_888
NATIVE -> ImageFormat.PRIVATE
else -> throw PixelFormatNotSupportedError(this.unionValue)
}
}
companion object : JSUnionValue.Companion<PixelFormat> {
fun fromImageFormat(imageFormat: Int): PixelFormat {
return when (imageFormat) {
fun fromImageFormat(imageFormat: Int): PixelFormat =
when (imageFormat) {
ImageFormat.YUV_420_888 -> YUV
ImageFormat.PRIVATE -> NATIVE
else -> UNKNOWN
}
}
override fun fromUnionValue(unionValue: String?): PixelFormat? {
return when (unionValue) {
override fun fromUnionValue(unionValue: String?): PixelFormat? =
when (unionValue) {
"yuv" -> YUV
"rgb" -> RGB
"native" -> NATIVE
"unknown" -> UNKNOWN
else -> null
}
}
}
}