railbird/react-native-vision-camera
9
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Implement cornerPoints and frame for scanned codes (#2117)

Browse Source 
* Android & TypeScript part of scanned code corner points. Scanned frame dimensions also included in callback. #2076

* TS fix. #2076

* Implement iOS parts of code scanner corner points with additional scanned frame data.

* Add example page for code scanning

* Use Point type from Point.ts

* Update package/src/CodeScanner.ts

Add parameters description to CodeScanner callback.

Co-authored-by: Marc Rousavy <marcrousavy@hotmail.com>

* Update package/src/CodeScanner.ts

More expressive description for CodeScannerFrame.

Co-authored-by: Marc Rousavy <marcrousavy@hotmail.com>

* Update package/src/CodeScanner.ts

Co-authored-by: Marc Rousavy <marcrousavy@hotmail.com>

* Update package/src/CodeScanner.ts

Co-authored-by: Marc Rousavy <marcrousavy@hotmail.com>

* Update package/ios/Core/CameraSession+CodeScanner.swift

Co-authored-by: Marc Rousavy <marcrousavy@hotmail.com>

* Update package/ios/Core/CameraSession+CodeScanner.swift

Co-authored-by: Marc Rousavy <marcrousavy@hotmail.com>

* Remove default values from CodeSCannerFrame

* Linting

* Multiply code corner points in swift

---------

Co-authored-by: stemy <balazs.stemler@metrix.co.hu>
Co-authored-by: Zoli <iamzozo@metrix.co.hu>
Co-authored-by: Marc Rousavy <marcrousavy@hotmail.com>
This commit is contained in:
Metrix Hungary Kft 2023-11-09 11:57:05 +01:00 committed by GitHub
parent 24ddca3409
commit e649aba8e1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 317 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
package/android/src/main/java/com/mrousavy/camera/CameraView+Events.kt
View File

@ -7,6 +7,7 @@ import com.facebook.react.bridge.WritableMap
import com.facebook.react.uimanager.events.RCTEventEmitter
import com.google.mlkit.vision.barcode.common.Barcode
import com.mrousavy.camera.core.CameraError
import com.mrousavy.camera.core.CodeScannerFrame
import com.mrousavy.camera.core.UnknownCameraError
import com.mrousavy.camera.core.code
import com.mrousavy.camera.types.CodeType
@ -42,7 +43,7 @@ fun CameraView.invokeOnViewReady() {
reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraViewReady", event)
}
fun CameraView.invokeOnCodeScanned(barcodes: List<Barcode>) {
fun CameraView.invokeOnCodeScanned(barcodes: List<Barcode>, scannerFrame: CodeScannerFrame) {
val codes = Arguments.createArray()
barcodes.forEach { barcode ->
val code = Arguments.createMap()
@ -58,11 +59,26 @@ fun CameraView.invokeOnCodeScanned(barcodes: List<Barcode>) {
frame.putInt("height", rect.bottom - rect.top)
code.putMap("frame", frame)
}
barcode.cornerPoints?.let { points ->
val corners = Arguments.createArray()
points.forEach { point ->
val pt = Arguments.createMap()
pt.putInt("x", point.x)
pt.putInt("y", point.y)
corners.pushMap(pt)
}
code.putArray("corners", corners)
}
codes.pushMap(code)
}
val event = Arguments.createMap()
event.putArray("codes", codes)
val codeScannerFrame = Arguments.createMap()
codeScannerFrame.putInt("width", scannerFrame.width)
codeScannerFrame.putInt("height", scannerFrame.height)
event.putMap("frame", codeScannerFrame)
val reactContext = context as ReactContext
reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraCodeScanned", event)
}

5
package/android/src/main/java/com/mrousavy/camera/CameraView.kt
View File

@ -11,6 +11,7 @@ import com.google.mlkit.vision.barcode.common.Barcode
import com.mrousavy.camera.core.CameraConfiguration
import com.mrousavy.camera.core.CameraQueues
import com.mrousavy.camera.core.CameraSession
import com.mrousavy.camera.core.CodeScannerFrame
import com.mrousavy.camera.core.PreviewView
import com.mrousavy.camera.extensions.installHierarchyFitter
import com.mrousavy.camera.frameprocessor.FrameProcessor
@ -228,7 +229,7 @@ class CameraView(context: Context) :
invokeOnInitialized()
}
override fun onCodeScanned(codes: List<Barcode>) {
invokeOnCodeScanned(codes)
override fun onCodeScanned(codes: List<Barcode>, scannerFrame: CodeScannerFrame) {
invokeOnCodeScanned(codes, scannerFrame)
}
}

2
package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt
View File

@ -611,6 +611,6 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
interface CameraSessionCallback {
fun onError(error: Throwable)
fun onInitialized()
fun onCodeScanned(codes: List<Barcode>)
fun onCodeScanned(codes: List<Barcode>, scannerFrame: CodeScannerFrame)
}
}

3
package/android/src/main/java/com/mrousavy/camera/core/CodeScannerFrame.kt Normal file
View File

@ -0,0 +1,3 @@
package com.mrousavy.camera.core
data class CodeScannerFrame(val width: Int, val height: Int)

2
package/android/src/main/java/com/mrousavy/camera/core/CodeScannerPipeline.kt
View File

@ -56,7 +56,7 @@ class CodeScannerPipeline(
image.close()
isBusy = false
if (barcodes.isNotEmpty()) {
callback.onCodeScanned(barcodes)
callback.onCodeScanned(barcodes, CodeScannerFrame(inputImage.width, inputImage.height))
}
}
.addOnFailureListener { error ->

4
package/example/ios/Podfile.lock
View File

@ -507,7 +507,7 @@ PODS:
- libwebp (~> 1.0)
- SDWebImage/Core (~> 5.10)
- SocketRocket (0.6.1)
- VisionCamera (3.6.3):
- VisionCamera (3.6.4):
- React
- React-callinvoker
- React-Core
@ -747,7 +747,7 @@ SPEC CHECKSUMS:
SDWebImage: a7f831e1a65eb5e285e3fb046a23fcfbf08e696d
SDWebImageWebPCoder: 908b83b6adda48effe7667cd2b7f78c897e5111d
SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17
VisionCamera: aebb5adef1296732983129c5fe054337b15c3c0b
VisionCamera: db57ca079c4f933f1ddd07f2f8fb2d171a826b85
Yoga: 8796b55dba14d7004f980b54bcc9833ee45b28ce
PODFILE CHECKSUM: 27f53791141a3303d814e09b55770336416ff4eb

6
package/example/ios/VisionCameraExample.xcodeproj/project.pbxproj
View File

@ -178,7 +178,7 @@
LastUpgradeCheck = 1250;
TargetAttributes = {
13B07F861A680F5B00A75B9A = {
DevelopmentTeam = CJW62Q77E7;
DevelopmentTeam = XVBSPUKE9E;
LastSwiftMigration = 1240;
};
};
@ -405,7 +405,7 @@
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CLANG_ENABLE_MODULES = YES;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = CJW62Q77E7;
DEVELOPMENT_TEAM = XVBSPUKE9E;
ENABLE_BITCODE = NO;
INFOPLIST_FILE = VisionCameraExample/Info.plist;
INFOPLIST_KEY_CFBundleDisplayName = "Vision Camera";
@ -437,7 +437,7 @@
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CLANG_ENABLE_MODULES = YES;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = CJW62Q77E7;
DEVELOPMENT_TEAM = XVBSPUKE9E;
INFOPLIST_FILE = VisionCameraExample/Info.plist;
INFOPLIST_KEY_CFBundleDisplayName = "Vision Camera";
INFOPLIST_KEY_LSApplicationCategoryType = "public.app-category.photography";

2
package/example/src/App.tsx
View File

@ -4,6 +4,7 @@ import { createNativeStackNavigator } from '@react-navigation/native-stack'
import { PermissionsPage } from './PermissionsPage'
import { MediaPage } from './MediaPage'
import { CameraPage } from './CameraPage'
import { CodeScannerPage } from './CodeScannerPage'
import type { Routes } from './Routes'
import { Camera, CameraPermissionStatus } from 'react-native-vision-camera'
import { GestureHandlerRootView } from 'react-native-gesture-handler'
@ -41,6 +42,7 @@ export function App(): React.ReactElement | null {
initialRouteName={showPermissionsPage ? 'PermissionsPage' : 'CameraPage'}>
<Stack.Screen name="PermissionsPage" component={PermissionsPage} />
<Stack.Screen name="CameraPage" component={CameraPage} />
<Stack.Screen name="CodeScannerPage" component={CodeScannerPage} />
<Stack.Screen
name="MediaPage"
component={MediaPage}

3
package/example/src/CameraPage.tsx
View File

@ -242,6 +242,9 @@ export function CameraPage({ navigation }: Props): React.ReactElement {
<PressableOpacity style={styles.button} onPress={() => navigation.navigate('Devices')}>
<IonIcon name="settings-outline" color="white" size={24} />
</PressableOpacity>
<PressableOpacity style={styles.button} onPress={() => navigation.navigate('CodeScannerPage')}>
<IonIcon name="qr-code-outline" color="white" size={24} />
</PressableOpacity>
</View>
</View>
)

222
package/example/src/CodeScannerPage.tsx Normal file
View File

@ -0,0 +1,222 @@
import * as React from 'react'
import { useRef, useState, useCallback } from 'react'
import { StyleSheet, View } from 'react-native'
import { PinchGestureHandler, PinchGestureHandlerGestureEvent, TapGestureHandler } from 'react-native-gesture-handler'
import { CameraRuntimeError, useCameraDevice, useCameraFormat, useCodeScanner } from 'react-native-vision-camera'
import { Camera } from 'react-native-vision-camera'
import { CONTENT_SPACING, MAX_ZOOM_FACTOR, SAFE_AREA_PADDING, SCREEN_HEIGHT, SCREEN_WIDTH } from './Constants'
import Reanimated, { Extrapolate, interpolate, useAnimatedGestureHandler, useAnimatedProps, useSharedValue } from 'react-native-reanimated'
import { useEffect } from 'react'
import { useIsForeground } from './hooks/useIsForeground'
import { StatusBarBlurBackground } from './views/StatusBarBlurBackground'
import { PressableOpacity } from 'react-native-pressable-opacity'
import IonIcon from 'react-native-vector-icons/Ionicons'
import type { Routes } from './Routes'
import type { NativeStackScreenProps } from '@react-navigation/native-stack'
import { useIsFocused } from '@react-navigation/core'
import { usePreferredCameraDevice } from './hooks/usePreferredCameraDevice'
const ReanimatedCamera = Reanimated.createAnimatedComponent(Camera)
Reanimated.addWhitelistedNativeProps({
zoom: true,
})
const SCALE_FULL_ZOOM = 3
const BUTTON_SIZE = 40
type Props = NativeStackScreenProps<Routes, 'CodeScannerPage'>
export function CodeScannerPage({ navigation }: Props): React.ReactElement {
const camera = useRef<Camera>(null)
const [isCameraInitialized, setIsCameraInitialized] = useState(false)
const zoom = useSharedValue(0)
const isPressingButton = useSharedValue(false)
// check if camera page is active
const isFocussed = useIsFocused()
const isForeground = useIsForeground()
const isActive = isFocussed && isForeground
const [cameraPosition, setCameraPosition] = useState<'front' | 'back'>('back')
const [enableHdr, setEnableHdr] = useState(false)
const [torch, setTorch] = useState<'off' | 'on'>('off')
const [enableNightMode, setEnableNightMode] = useState(false)
// camera device settings
const [preferredDevice] = usePreferredCameraDevice()
let device = useCameraDevice(cameraPosition)
if (preferredDevice != null && preferredDevice.position === cameraPosition) {
// override default device with the one selected by the user in settings
device = preferredDevice
}
const [targetFps, setTargetFps] = useState(60)
const screenAspectRatio = SCREEN_HEIGHT / SCREEN_WIDTH
const format = useCameraFormat(device, [
{ fps: targetFps },
{ videoAspectRatio: screenAspectRatio },
{ videoResolution: 'max' },
{ photoAspectRatio: screenAspectRatio },
{ photoResolution: 'max' },
])
const fps = Math.min(format?.maxFps ?? 1, targetFps)
const supportsTorch = device?.hasTorch ?? false
//#region Animated Zoom
// This just maps the zoom factor to a percentage value.
// so e.g. for [min, neutr., max] values [1, 2, 128] this would result in [0, 0.0081, 1]
const minZoom = device?.minZoom ?? 1
const maxZoom = Math.min(device?.maxZoom ?? 1, MAX_ZOOM_FACTOR)
const cameraAnimatedProps = useAnimatedProps(() => {
const z = Math.max(Math.min(zoom.value, maxZoom), minZoom)
return {
zoom: z,
}
}, [maxZoom, minZoom, zoom])
//#endregion
//#region Callbacks
// Camera callbacks
const onError = useCallback((error: CameraRuntimeError) => {
console.error(error)
}, [])
const onInitialized = useCallback(() => {
console.log('Camera initialized!')
setIsCameraInitialized(true)
}, [])
const onFlipCameraPressed = useCallback(() => {
setCameraPosition((p) => (p === 'back' ? 'front' : 'back'))
}, [])
const onTorchPressed = useCallback(() => {
setTorch((f) => (f === 'off' ? 'on' : 'off'))
}, [])
//#endregion
//#region Tap Gesture
const onDoubleTap = useCallback(() => {
onFlipCameraPressed()
}, [onFlipCameraPressed])
//#endregion
//#region Effects
const neutralZoom = device?.neutralZoom ?? 1
useEffect(() => {
// Run everytime the neutralZoomScaled value changes. (reset zoom when device changes)
zoom.value = neutralZoom
}, [neutralZoom, zoom])
//#endregion
//#region Pinch to Zoom Gesture
// The gesture handler maps the linear pinch gesture (0 - 1) to an exponential curve since a camera's zoom
// function does not appear linear to the user. (aka zoom 0.1 -> 0.2 does not look equal in difference as 0.8 -> 0.9)
const onPinchGesture = useAnimatedGestureHandler<PinchGestureHandlerGestureEvent, { startZoom?: number }>({
onStart: (_, context) => {
context.startZoom = zoom.value
},
onActive: (event, context) => {
// we're trying to map the scale gesture to a linear zoom here
const startZoom = context.startZoom ?? 0
const scale = interpolate(event.scale, [1 - 1 / SCALE_FULL_ZOOM, 1, SCALE_FULL_ZOOM], [-1, 0, 1], Extrapolate.CLAMP)
zoom.value = interpolate(scale, [-1, 0, 1], [minZoom, startZoom, maxZoom], Extrapolate.CLAMP)
},
})
//#endregion
useEffect(() => {
const f =
format != null
? `(${format.photoWidth}x${format.photoHeight} photo / ${format.videoWidth}x${format.videoHeight}@${format.maxFps} video @ ${fps}fps)`
: undefined
console.log(`Camera: ${device?.name} | Format: ${f}`)
}, [device?.name, format, fps])
const codeScanner = useCodeScanner({
codeTypes: ['qr'],
onCodeScanned: (codes, frame) => {
console.log(codes, frame)
},
})
return (
<View style={styles.container}>
{device != null && (
<PinchGestureHandler onGestureEvent={onPinchGesture} enabled={isActive}>
<Reanimated.View style={StyleSheet.absoluteFill}>
<TapGestureHandler onEnded={onDoubleTap} numberOfTaps={2}>
<ReanimatedCamera
ref={camera}
style={StyleSheet.absoluteFill}
device={device}
format={format}
fps={fps}
hdr={enableHdr}
torch={supportsTorch ? torch : 'off'}
lowLightBoost={device.supportsLowLightBoost && enableNightMode}
isActive={isActive}
onInitialized={onInitialized}
onError={onError}
enableZoomGesture={false}
animatedProps={cameraAnimatedProps}
enableFpsGraph={true}
orientation="portrait"
codeScanner={codeScanner}
/>
</TapGestureHandler>
</Reanimated.View>
</PinchGestureHandler>
)}
<StatusBarBlurBackground />
<View style={styles.rightButtonRow}>
<PressableOpacity style={styles.button} onPress={onFlipCameraPressed} disabledOpacity={0.4}>
<IonIcon name="camera-reverse" color="white" size={24} />
</PressableOpacity>
{torch && (
<PressableOpacity style={styles.button} onPress={onTorchPressed} disabledOpacity={0.4}>
<IonIcon name={torch === 'on' ? 'flash' : 'flash-off'} color="white" size={24} />
</PressableOpacity>
)}
<PressableOpacity style={styles.button} onPress={() => navigation.navigate('CameraPage')}>
<IonIcon name="videocam" color="white" size={24} />
</PressableOpacity>
<PressableOpacity style={styles.button} onPress={() => navigation.navigate('Devices')}>
<IonIcon name="settings-outline" color="white" size={24} />
</PressableOpacity>
</View>
</View>
)
}
const styles = StyleSheet.create({
container: {
flex: 1,
backgroundColor: 'black',
},
button: {
marginBottom: CONTENT_SPACING,
width: BUTTON_SIZE,
height: BUTTON_SIZE,
borderRadius: BUTTON_SIZE / 2,
backgroundColor: 'rgba(140, 140, 140, 0.3)',
justifyContent: 'center',
alignItems: 'center',
},
rightButtonRow: {
position: 'absolute',
right: SAFE_AREA_PADDING.paddingRight,
top: SAFE_AREA_PADDING.paddingTop,
},
text: {
color: 'white',
fontSize: 11,
fontWeight: 'bold',
textAlign: 'center',
},
})

1
package/example/src/Routes.ts
View File

@ -1,6 +1,7 @@
export type Routes = {
PermissionsPage: undefined
CameraPage: undefined
CodeScannerPage: undefined
MediaPage: {
path: string
type: 'video' | 'photo'

3
package/ios/CameraView.swift
View File

@ -291,12 +291,13 @@ public final class CameraView: UIView, CameraSessionDelegate {
#endif
}
func onCodeScanned(codes: [CameraSession.Code]) {
func onCodeScanned(codes: [CameraSession.Code], scannerFrame: CameraSession.CodeScannerFrame) {
guard let onCodeScanned = onCodeScanned else {
return
}
onCodeScanned([
"codes": codes.map { $0.toJSValue() },
"frame": scannerFrame.toJSValue(),
])
}

29
package/ios/Core/CameraSession+CodeScanner.swift
View File

@ -28,8 +28,12 @@ extension CameraSession: AVCaptureMetadataOutputObjectsDelegate {
// Map codes to JS values
let codes = metadataObjects.map { object in
var value: String?
var corners: [CGPoint]?
if let code = object as? AVMetadataMachineReadableCodeObject {
value = code.stringValue
corners = code.corners.map {
CGPoint(x: $0.x * Double(size.width), y: $0.y * Double(size.height))
}
}
let x = object.bounds.origin.x * Double(size.width)
let y = object.bounds.origin.y * Double(size.height)
@ -37,11 +41,11 @@ extension CameraSession: AVCaptureMetadataOutputObjectsDelegate {
let h = object.bounds.height * Double(size.height)
let frame = CGRect(x: x, y: y, width: w, height: h)
return Code(type: object.type, value: value, frame: frame)
return Code(type: object.type, value: value, frame: frame, corners: corners)
}
// Call delegate (JS) event
onCodeScanned(codes)
onCodeScanned(codes, CodeScannerFrame(width: size.width, height: size.height))
}
/**
@ -61,6 +65,11 @@ extension CameraSession: AVCaptureMetadataOutputObjectsDelegate {
*/
let frame: CGRect
/**
Location of the code on-screen, relative to the video output layer
*/
let corners: [CGPoint]?
/**
Converts this Code to a JS Object (Dictionary)
*/
@ -74,6 +83,22 @@ extension CameraSession: AVCaptureMetadataOutputObjectsDelegate {
"width": frame.size.width,
"height": frame.size.height,
],
"corners": corners?.map { [
"x": $0.x,
"y": $0.y,
] } ?? [],
]
}
}
struct CodeScannerFrame {
let width: Int32
let height: Int32
func toJSValue() -> [String: AnyHashable] {
return [
"width": width,
"height": height,
]
}
}

2
package/ios/Core/CameraSessionDelegate.swift
View File

@ -28,5 +28,5 @@ protocol CameraSessionDelegate: AnyObject {
/**
Called whenever a QR/Barcode has been scanned. Only if the CodeScanner Output is enabled
*/
func onCodeScanned(codes: [CameraSession.Code])
func onCodeScanned(codes: [CameraSession.Code], scannerFrame: CameraSession.CodeScannerFrame)
}

5
package/src/Camera.tsx
View File

@ -11,7 +11,7 @@ import type { RecordVideoOptions, VideoFile } from './VideoFile'
import { VisionCameraProxy } from './FrameProcessorPlugins'
import { CameraDevices } from './CameraDevices'
import type { EmitterSubscription } from 'react-native'
import { Code, CodeScanner } from './CodeScanner'
import { Code, CodeScanner, CodeScannerFrame } from './CodeScanner'
//#region Types
export type CameraPermissionStatus = 'granted' | 'not-determined' | 'denied' | 'restricted'
@ -19,6 +19,7 @@ export type CameraPermissionRequestResult = 'granted' | 'denied'
interface OnCodeScannedEvent {
codes: Code[]
frame: CodeScannerFrame
}
interface OnErrorEvent {
code: string
@ -398,7 +399,7 @@ export class Camera extends React.PureComponent<CameraProps> {
const codeScanner = this.props.codeScanner
if (codeScanner == null) return
codeScanner.onCodeScanned(event.nativeEvent.codes)
codeScanner.onCodeScanned(event.nativeEvent.codes, event.nativeEvent.frame)
}
//#region Lifecycle

24
package/src/CodeScanner.ts
View File

@ -1,3 +1,5 @@
import { Point } from './Point'
/**
* The type of the code to scan.
*/
@ -15,6 +17,20 @@ export type CodeType =
| 'aztec'
| 'data-matrix'
/**
* The full area that is used for code scanning. In most cases, this is 1280x720 or 1920x1080.
*/
export interface CodeScannerFrame {
/**
* The width of the frame
*/
width: number
/**
* The height of the frame
*/
height: number
}
/**
* A scanned code.
*/
@ -36,6 +52,10 @@ export interface Code {
width: number
height: number
}
/**
* The location of each corner relative to the Camera Preview (in dp).
*/
corners?: Point[]
}
/**
@ -48,8 +68,10 @@ export interface CodeScanner {
codeTypes: CodeType[]
/**
* A callback to call whenever the scanned codes change.
* @param codes The scanned codes, or an empty array if none.
* @param frame The full area that is used for scanning. Code bounds and corners are relative to this frame.
*/
onCodeScanned: (codes: Code[]) => void
onCodeScanned: (codes: Code[], frame: CodeScannerFrame) => void
/**
* Crops the scanner's view area to the specific region of interest.
*/

6
package/src/hooks/useCodeScanner.ts
View File

@ -1,5 +1,5 @@
import { useCallback, useMemo, useRef } from 'react'
import { Code, CodeScanner } from '../CodeScanner'
import { Code, CodeScanner, CodeScannerFrame } from '../CodeScanner'
export function useCodeScanner(codeScanner: CodeScanner): CodeScanner {
const { onCodeScanned, ...codeScannerOptions } = codeScanner
@ -7,8 +7,8 @@ export function useCodeScanner(codeScanner: CodeScanner): CodeScanner {
// Memoize the function once and use a ref on any identity changes
const ref = useRef(onCodeScanned)
ref.current = onCodeScanned
const callback = useCallback((codes: Code[]) => {
ref.current(codes)
const callback = useCallback((codes: Code[], frame: CodeScannerFrame) => {
ref.current(codes, frame)
}, [])
// CodeScanner needs to be memoized so it doesn't trigger a Camera Session re-build