feat: Code Scanner API (#1912)

* feat: CodeScanner JS API

* feat: iOS

* Use guard

* Format

* feat: Android base

* fix: Attach Surfaces

* Use isBusy var

* fix: Use separate Queue

* feat: Finish iOS types

* feat: Implement all other code types on Android

* fix: Call JS event

* fix: Pass codetypes on Android

* fix: iOS use Preview coordinate system

* docs: Add comments

* chore: Format code

* Update CameraView+AVCaptureSession.swift

* docs: Add Code Scanner docs

* docs: Update

* feat: Use lazily downloaded model on Android

* Revert changes in CameraPage

* Format

* fix: Fix empty QR codes

* Update README.md

package/src/Camera.tsx

@@ -11,21 +11,27 @@ import type { RecordVideoOptions, VideoFile } from './VideoFile'
 import { VisionCameraProxy } from './FrameProcessorPlugins'
 import { CameraDevices } from './CameraDevices'
 import type { EmitterSubscription } from 'react-native'
+import { Code, CodeScanner } from './CodeScanner'
 
 //#region Types
 export type CameraPermissionStatus = 'granted' | 'not-determined' | 'denied' | 'restricted'
 export type CameraPermissionRequestResult = 'granted' | 'denied'
 
+interface OnCodeScannedEvent {
+  codes: Code[]
+}
 interface OnErrorEvent {
   code: string
   message: string
   cause?: ErrorWithCause
 }
-type NativeCameraViewProps = Omit<CameraProps, 'device' | 'onInitialized' | 'onError' | 'frameProcessor'> & {
+type NativeCameraViewProps = Omit<CameraProps, 'device' | 'onInitialized' | 'onError' | 'frameProcessor' | 'codeScanner'> & {
   cameraId: string
   enableFrameProcessor: boolean
+  codeScannerOptions?: Omit<CodeScanner, 'onCodeScanned'>
   onInitialized?: (event: NativeSyntheticEvent<void>) => void
   onError?: (event: NativeSyntheticEvent<OnErrorEvent>) => void
+  onCodeScanned?: (event: NativeSyntheticEvent<OnCodeScannedEvent>) => void
   onViewReady: () => void
 }
 type RefType = React.Component<NativeCameraViewProps> & Readonly<NativeMethods>
@@ -76,6 +82,7 @@ export class Camera extends React.PureComponent<CameraProps> {
     this.onViewReady = this.onViewReady.bind(this)
     this.onInitialized = this.onInitialized.bind(this)
     this.onError = this.onError.bind(this)
+    this.onCodeScanned = this.onCodeScanned.bind(this)
     this.ref = React.createRef<RefType>()
     this.lastFrameProcessor = undefined
   }
@@ -387,6 +394,13 @@ export class Camera extends React.PureComponent<CameraProps> {
   }
   //#endregion
 
+  private onCodeScanned(event: NativeSyntheticEvent<OnCodeScannedEvent>): void {
+    const codeScanner = this.props.codeScanner
+    if (codeScanner == null) return
+
+    codeScanner.onCodeScanned(event.nativeEvent.codes)
+  }
+
   //#region Lifecycle
   private setFrameProcessor(frameProcessor: FrameProcessor): void {
     VisionCameraProxy.setFrameProcessor(this.handle, frameProcessor)
@@ -422,7 +436,7 @@ export class Camera extends React.PureComponent<CameraProps> {
   /** @internal */
   public render(): React.ReactNode {
     // We remove the big `device` object from the props because we only need to pass `cameraId` to native.
-    const { device, frameProcessor, ...props } = this.props
+    const { device, frameProcessor, codeScanner, ...props } = this.props
 
     // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
     if (device == null) {
@@ -440,7 +454,9 @@ export class Camera extends React.PureComponent<CameraProps> {
         ref={this.ref}
         onViewReady={this.onViewReady}
         onInitialized={this.onInitialized}
+        onCodeScanned={this.onCodeScanned}
         onError={this.onError}
+        codeScannerOptions={codeScanner}
         enableFrameProcessor={frameProcessor != null}
         enableBufferCompression={props.enableBufferCompression ?? shouldEnableBufferCompression}
       />

package/src/CameraError.ts

@@ -24,6 +24,10 @@ export type SessionError =
   | 'session/camera-has-been-disconnected'
   | 'session/audio-in-use-by-other-app'
   | 'session/audio-session-failed-to-activate'
+export type CodeScannerError =
+  | 'code-scanner/not-compatible-with-outputs'
+  | 'code-scanner/code-type-not-supported'
+  | 'code-scanner/cannot-load-model'
 export type CaptureError =
   | 'capture/recording-in-progress'
   | 'capture/no-recording-in-progress'

package/src/CameraProps.ts

@@ -1,6 +1,7 @@
 import type { ViewProps } from 'react-native'
 import type { CameraDevice, CameraDeviceFormat, VideoStabilizationMode } from './CameraDevice'
 import type { CameraRuntimeError } from './CameraError'
+import { CodeScanner } from './CodeScanner'
 import type { Frame } from './Frame'
 import type { Orientation } from './Orientation'
 
@@ -223,13 +224,17 @@ export interface CameraProps extends ViewProps {
    * ```tsx
    * const frameProcessor = useFrameProcessor((frame) => {
    *   'worklet'
-   *   const qrCodes = scanQRCodes(frame)
-   *   console.log(`Detected QR Codes: ${qrCodes}`)
+   *   const faces = scanFaces(frame)
+   *   console.log(`Faces: ${faces}`)
    * }, [])
    *
    * return <Camera {...cameraProps} frameProcessor={frameProcessor} />
    * ```
    */
   frameProcessor?: FrameProcessor
+  /**
+   * TODO: Desc
+   */
+  codeScanner?: CodeScanner
   //#endregion
 }

package/src/CodeScanner.ts

@@ -0,0 +1,62 @@
+/**
+ * The type of the code to scan.
+ */
+export type CodeType =
+  | 'code-128'
+  | 'code-39'
+  | 'code-93'
+  | 'codabar'
+  | 'ean-13'
+  | 'ean-8'
+  | 'itf'
+  | 'upc-e'
+  | 'qr'
+  | 'pdf-417'
+  | 'aztec'
+  | 'data-matrix'
+
+/**
+ * A scanned code.
+ */
+export interface Code {
+  /**
+   * The type of the code that was scanned.
+   */
+  type: CodeType | 'unknown'
+  /**
+   * The string value, or null if it cannot be decoded.
+   */
+  value?: string
+  /**
+   * The location of the code relative to the Camera Preview (in dp).
+   */
+  frame?: {
+    x: number
+    y: number
+    width: number
+    height: number
+  }
+}
+
+/**
+ * A scanner for detecting codes in a Camera Stream.
+ */
+export interface CodeScanner {
+  /**
+   * The types of codes to configure the code scanner for.
+   */
+  codeTypes: CodeType[]
+  /**
+   * A callback to call whenever the scanned codes change.
+   */
+  onCodeScanned: (codes: Code[]) => void
+  /**
+   * Crops the scanner's view area to the specific region of interest.
+   */
+  regionOfInterest?: {
+    x: number
+    y: number
+    width: number
+    height: number
+  }
+}

package/src/hooks/useCodeScanner.ts

@@ -0,0 +1,23 @@
+import { useCallback, useMemo, useRef } from 'react'
+import { Code, CodeScanner } from '../CodeScanner'
+
+export function useCodeScanner(codeScanner: CodeScanner): CodeScanner {
+  const { onCodeScanned, ...codeScannerOptions } = codeScanner
+
+  // Memoize the  function once and use a ref on any identity changes
+  const ref = useRef(onCodeScanned)
+  ref.current = onCodeScanned
+  const callback = useCallback((codes: Code[]) => {
+    ref.current(codes)
+  }, [])
+
+  // CodeScanner needs to be memoized so it doesn't trigger a Camera Session re-build
+  return useMemo(
+    () => ({
+      ...codeScannerOptions,
+      onCodeScanned: callback,
+    }),
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    [JSON.stringify(codeScannerOptions), callback],
+  )
+}

package/src/hooks/useFrameProcessor.ts

@@ -42,8 +42,8 @@ export function createFrameProcessor(frameProcessor: FrameProcessor['frameProces
  * ```ts
  * const frameProcessor = useFrameProcessor((frame) => {
  *   'worklet'
- *   const qrCodes = scanQRCodes(frame)
- *   console.log(`QR Codes: ${qrCodes}`)
+ *   const faces = scanFaces(frame)
+ *   console.log(`Faces: ${faces}`)
  * }, [])
  * ```
  */

package/src/index.ts

@@ -9,6 +9,7 @@ export * from './PhotoFile'
 export * from './PixelFormat'
 export * from './Point'
 export * from './VideoFile'
+export * from './CodeScanner'
 
 export * from './devices/getCameraFormat'
 export * from './devices/getCameraDevice'
@@ -19,3 +20,4 @@ export * from './hooks/useCameraDevices'
 export * from './hooks/useCameraFormat'
 export * from './hooks/useCameraPermission'
 export * from './hooks/useFrameProcessor'
+export * from './hooks/useCodeScanner'