railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Sync Frame Processors (plus runAsync and runAtTargetFps) (#1472)

Browse Source 
Before, Frame Processors ran on a separate Thread.

After, Frame Processors run fully synchronous and always at the same FPS as the Camera.

Two new functions have been introduced:

* `runAtTargetFps(fps: number, func: () => void)`: Runs the given code as often as the given `fps`, effectively throttling it's calls.
* `runAsync(frame: Frame, func: () => void)`: Runs the given function on a separate Thread for Frame Processing. A strong reference to the Frame is held as long as the function takes to execute.

You can use `runAtTargetFps` to throttle calls to a specific API (e.g. if your Camera is running at 60 FPS, but you only want to run face detection at ~25 FPS, use `runAtTargetFps(25, ...)`.)

You can use `runAsync` to run a heavy algorithm asynchronous, so that the Camera is not blocked while your algorithm runs. This is useful if your main sync processor draws something, and your async processor is doing some image analysis on the side. 

You can also combine both functions.

Examples:

```js
const frameProcessor = useFrameProcessor((frame) => {
  'worklet'
  console.log("I'm running at 60 FPS!")
}, [])
```

```js
const frameProcessor = useFrameProcessor((frame) => {
  'worklet'
  console.log("I'm running at 60 FPS!")

  runAtTargetFps(10, () => {
    'worklet'
    console.log("I'm running at 10 FPS!")
  })
}, [])
```



```js
const frameProcessor = useFrameProcessor((frame) => {
  'worklet'
  console.log("I'm running at 60 FPS!")

  runAsync(frame, () => {
    'worklet'
    console.log("I'm running on another Thread, I can block for longer!")
  })
}, [])
```

```js
const frameProcessor = useFrameProcessor((frame) => {
  'worklet'
  console.log("I'm running at 60 FPS!")

  runAtTargetFps(10, () => {
    'worklet'
    runAsync(frame, () => {
      'worklet'
      console.log("I'm running on another Thread at 10 FPS, I can block for longer!")
    })
  })
}, [])
```
This commit is contained in:
Marc Rousavy
2023-02-15 16:47:09 +01:00
committed by GitHub
parent a0590dccb5
commit 30b56153db
30 changed files with 660 additions and 914 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
src/Camera.tsx
View File

@@ -1,6 +1,6 @@
import React from 'react';
import { requireNativeComponent, NativeModules, NativeSyntheticEvent, findNodeHandle, NativeMethods, Platform } from 'react-native';
import type { FrameProcessorPerformanceSuggestion, VideoFileType } from '.';
import type { VideoFileType } from '.';
import type { CameraDevice } from './CameraDevice';
import type { ErrorWithCause } from './CameraError';
import { CameraCaptureError, CameraRuntimeError, tryParseNativeCameraError, isErrorWithCause } from './CameraError';
@@ -20,16 +20,11 @@ interface OnErrorEvent {
message: string;
cause?: ErrorWithCause;
}
type NativeCameraViewProps = Omit<
CameraProps,
'device' | 'onInitialized' | 'onError' | 'onFrameProcessorPerformanceSuggestionAvailable' | 'frameProcessor' | 'frameProcessorFps'
> & {
type NativeCameraViewProps = Omit<CameraProps, 'device' | 'onInitialized' | 'onError' | 'frameProcessor'> & {
cameraId: string;
frameProcessorFps?: number; // native cannot use number | string, so we use '-1' for 'auto'
enableFrameProcessor: boolean;
onInitialized?: (event: NativeSyntheticEvent<void>) => void;
onError?: (event: NativeSyntheticEvent<OnErrorEvent>) => void;
onFrameProcessorPerformanceSuggestionAvailable?: (event: NativeSyntheticEvent<FrameProcessorPerformanceSuggestion>) => void;
onViewReady: () => void;
};
type RefType = React.Component<NativeCameraViewProps> & Readonly<NativeMethods>;
@@ -86,7 +81,6 @@ export class Camera extends React.PureComponent<CameraProps> {
this.onViewReady = this.onViewReady.bind(this);
this.onInitialized = this.onInitialized.bind(this);
this.onError = this.onError.bind(this);
this.onFrameProcessorPerformanceSuggestionAvailable = this.onFrameProcessorPerformanceSuggestionAvailable.bind(this);
this.ref = React.createRef<RefType>();
this.lastFrameProcessor = undefined;
}
@@ -422,11 +416,6 @@ export class Camera extends React.PureComponent<CameraProps> {
private onInitialized(): void {
this.props.onInitialized?.();
}
private onFrameProcessorPerformanceSuggestionAvailable(event: NativeSyntheticEvent<FrameProcessorPerformanceSuggestion>): void {
if (this.props.onFrameProcessorPerformanceSuggestionAvailable != null)
this.props.onFrameProcessorPerformanceSuggestionAvailable(event.nativeEvent);
}
//#endregion
//#region Lifecycle
@@ -479,17 +468,15 @@ export class Camera extends React.PureComponent<CameraProps> {
/** @internal */
public render(): React.ReactNode {
// We remove the big `device` object from the props because we only need to pass `cameraId` to native.
const { device, frameProcessor, frameProcessorFps, ...props } = this.props;
const { device, frameProcessor, ...props } = this.props;
return (
<NativeCameraView
{...props}
frameProcessorFps={frameProcessorFps === 'auto' ? -1 : frameProcessorFps}
cameraId={device.id}
ref={this.ref}
onViewReady={this.onViewReady}
onInitialized={this.onInitialized}
onError={this.onError}
onFrameProcessorPerformanceSuggestionAvailable={this.onFrameProcessorPerformanceSuggestionAvailable}
enableFrameProcessor={frameProcessor != null}
/>
);

26
src/CameraProps.ts
View File

@@ -4,11 +4,6 @@ import type { CameraRuntimeError } from './CameraError';
import type { CameraPreset } from './CameraPreset';
import type { Frame } from './Frame';
export interface FrameProcessorPerformanceSuggestion {
type: 'can-use-higher-fps' | 'should-use-lower-fps';
suggestedFrameProcessorFps: number;
}
export interface CameraProps extends ViewProps {
/**
* The Camera Device to use.
@@ -171,11 +166,7 @@ export interface CameraProps extends ViewProps {
*/
onInitialized?: () => void;
/**
* Called when a new performance suggestion for a Frame Processor is available - either if your Frame Processor is running too fast and frames are being dropped, or because it is able to run faster. Optionally, you can adjust your `frameProcessorFps` accordingly.
*/
onFrameProcessorPerformanceSuggestionAvailable?: (suggestion: FrameProcessorPerformanceSuggestion) => void;
/**
* A worklet which will be called for every frame the Camera "sees". Throttle the Frame Processor's frame rate with {@linkcode frameProcessorFps}.
* A worklet which will be called for every frame the Camera "sees".
*
* > See [the Frame Processors documentation](https://mrousavy.github.io/react-native-vision-camera/docs/guides/frame-processors) for more information
*
@@ -193,20 +184,5 @@ export interface CameraProps extends ViewProps {
* ```
*/
frameProcessor?: (frame: Frame) => void;
/**
* Specifies the maximum frame rate the frame processor can use, independent of the Camera's frame rate (`fps` property).
*
* * A value of `'auto'` (default) indicates that the frame processor should execute as fast as it can, without dropping frames. This is achieved by collecting historical data for previous frame processor calls and adjusting frame rate accordingly.
* * A value of `1` indicates that the frame processor gets executed once per second, perfect for code scanning.
* * A value of `10` indicates that the frame processor gets executed 10 times per second, perfect for more realtime use-cases.
* * A value of `25` indicates that the frame processor gets executed 25 times per second, perfect for high-speed realtime use-cases.
* * ...and so on
*
* If you're using higher values, always check your Xcode/Android Studio Logs to make sure your frame processors are executing fast enough
* without blocking the video recording queue.
*
* @default 'auto'
*/
frameProcessorFps?: number | 'auto';
//#endregion
}

30
src/Frame.ts
View File

@@ -31,19 +31,19 @@ export interface Frame {
* ```
*/
toString(): string;
/**
* Closes and disposes the Frame.
* Only close frames that you have created yourself, e.g. by copying the frame you receive in a frame processor.
*
* @example
* ```ts
* const frameProcessor = useFrameProcessor((frame) => {
* const smallerCopy = resize(frame, 480, 270)
* // run AI ...
* smallerCopy.close()
* // don't close `frame`!
* })
* ```
*/
close(): void;
}
export interface FrameInternal extends Frame {
/**
* Increment the Frame Buffer ref-count by one.
*
* This is a private API, do not use this.
*/
incrementRefCount(): void;
/**
* Increment the Frame Buffer ref-count by one.
*
* This is a private API, do not use this.
*/
decrementRefCount(): void;
}

92
src/FrameProcessorPlugins.ts
View File

@@ -1,13 +1,101 @@
import type { Frame } from './Frame';
import type { Frame, FrameInternal } from './Frame';
import { Camera } from './Camera';
import { Worklets } from 'react-native-worklets/src';
// Install VisionCamera Frame Processor JSI Bindings and Plugins
Camera.installFrameProcessorBindings();
type FrameProcessor = (frame: Frame, ...args: unknown[]) => unknown;
type BasicParameterType = string | number | boolean | undefined;
type ParameterType = BasicParameterType | BasicParameterType[] | Record<string, BasicParameterType | undefined>;
type FrameProcessor = (frame: Frame, parameters?: Record<string, ParameterType | undefined>) => unknown;
type TFrameProcessorPlugins = Record<string, FrameProcessor>;
/**
* All natively installed Frame Processor Plugins.
*/
export const FrameProcessorPlugins = global.FrameProcessorPlugins as TFrameProcessorPlugins;
const lastFrameProcessorCall = Worklets.createSharedValue(performance.now());
/**
* Runs the given function at the given target FPS rate.
*
* For example, if you want to run a heavy face detection algorithm
* only once per second, you can use `runAtTargetFps(1, ...)` to
* throttle it to 1 FPS.
*
* @param fps The target FPS rate at which the given function should be executed
* @param func The function to execute.
* @returns The result of the function if it was executed, or `undefined` otherwise.
* @example
*
* ```ts
* const frameProcessor = useFrameProcessor((frame) => {
* 'worklet'
* console.log('New Frame')
* const face = runAtTargetFps(5, () => {
* 'worklet'
* const faces = detectFaces(frame)
* return faces[0]
* })
* if (face != null) console.log(`Detected a new face: ${face}`)
* })
* ```
*/
export function runAtTargetFps<T>(fps: number, func: () => T): T | undefined {
'worklet';
const targetIntervalMs = 1000 / fps; // <-- 60 FPS => 16,6667ms interval
const now = performance.now();
const diffToLastCall = now - lastFrameProcessorCall.value;
if (diffToLastCall >= targetIntervalMs) {
lastFrameProcessorCall.value = now;
// Last Frame Processor call is already so long ago that we want to make a new call
return func();
}
return undefined;
}
const asyncContext = Worklets.createContext('VisionCamera.async');
const runOnAsyncContext = Worklets.createRunInContextFn((frame: Frame, func: () => void) => {
'worklet';
try {
// Call long-running function
func();
} finally {
// Potentially delete Frame if we were the last ref
(frame as FrameInternal).decrementRefCount();
}
}, asyncContext);
/**
* Runs the given function asynchronously, while keeping a strong reference to the Frame.
*
* For example, if you want to run a heavy face detection algorithm
* while still drawing to the screen at 60 FPS, you can use `runAsync(...)`
* to offload the face detection algorithm to a separate thread.
*
* @param frame The current Frame of the Frame Processor.
* @param func The function to execute.
* @example
*
* ```ts
* const frameProcessor = useFrameProcessor((frame) => {
* 'worklet'
* console.log('New Frame')
* runAsync(frame, () => {
* 'worklet'
* const faces = detectFaces(frame)
* const face = [faces0]
* console.log(`Detected a new face: ${face}`)
* })
* })
* ```
*/
export function runAsync(frame: Frame, func: () => void): void {
'worklet';
// Increment ref count by one
(frame as FrameInternal).incrementRefCount();
// Call in separate background context
runOnAsyncContext(frame, func);
}

17
src/hooks/useFrameProcessor.ts
View File

@@ -1,5 +1,5 @@
import { DependencyList, useCallback } from 'react';
import type { Frame } from '../Frame';
import type { Frame, FrameInternal } from '../Frame';
// Install RN Worklets by importing it
import 'react-native-worklets/src';
@@ -23,6 +23,17 @@ type FrameProcessor = (frame: Frame) => void;
* ```
*/
export function useFrameProcessor(frameProcessor: FrameProcessor, dependencies: DependencyList): FrameProcessor {
// eslint-disable-next-line react-hooks/exhaustive-deps
return useCallback(frameProcessor, dependencies);
return useCallback((frame: Frame) => {
'worklet';
// Increment ref-count by one
(frame as FrameInternal).incrementRefCount();
try {
// Call sync frame processor
frameProcessor(frame);
} finally {
// Potentially delete Frame if we were the last ref (no runAsync)
(frame as FrameInternal).decrementRefCount();
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, dependencies);
}

2
src/index.ts
View File

@@ -4,7 +4,7 @@ export * from './CameraError';
export * from './CameraPosition';
export * from './CameraPreset';
export * from './CameraProps';
export * from './Frame';
export { Frame } from './Frame';
export * from './FrameProcessorPlugins';
export * from './CameraProps';
export * from './PhotoFile';