2023-02-21 07:00:48 -07:00
|
|
|
import type { Orientation } from './Orientation';
|
2023-08-21 04:50:14 -06:00
|
|
|
import { PixelFormat } from './PixelFormat';
|
2023-02-21 07:00:48 -07:00
|
|
|
|
2021-05-06 06:11:55 -06:00
|
|
|
/**
|
|
|
|
|
* A single frame, as seen by the camera.
|
|
|
|
|
*/
|
2023-07-20 07:30:04 -06:00
|
|
|
export interface Frame {
|
2021-05-06 06:11:55 -06:00
|
|
|
/**
|
2021-07-06 02:08:44 -06:00
|
|
|
* Whether the underlying buffer is still valid or not. The buffer will be released after the frame processor returns, or `close()` is called.
|
2021-05-06 06:11:55 -06:00
|
|
|
*/
|
|
|
|
|
isValid: boolean;
|
|
|
|
|
/**
|
|
|
|
|
* Returns the width of the frame, in pixels.
|
|
|
|
|
*/
|
|
|
|
|
width: number;
|
|
|
|
|
/**
|
|
|
|
|
* Returns the height of the frame, in pixels.
|
|
|
|
|
*/
|
|
|
|
|
height: number;
|
|
|
|
|
/**
|
|
|
|
|
* Returns the amount of bytes per row.
|
|
|
|
|
*/
|
|
|
|
|
bytesPerRow: number;
|
2023-09-01 04:20:17 -06:00
|
|
|
/**
|
|
|
|
|
* Returns the number of planes this frame contains.
|
|
|
|
|
*/
|
|
|
|
|
planesCount: number;
|
2023-02-21 07:00:48 -07:00
|
|
|
/**
|
|
|
|
|
* Returns whether the Frame is mirrored (selfie camera) or not.
|
|
|
|
|
*/
|
|
|
|
|
isMirrored: boolean;
|
|
|
|
|
/**
|
|
|
|
|
* Returns the timestamp of the Frame relative to the host sytem's clock.
|
|
|
|
|
*/
|
|
|
|
|
timestamp: number;
|
|
|
|
|
/**
|
|
|
|
|
* Represents the orientation of the Frame.
|
|
|
|
|
*
|
|
|
|
|
* Some ML Models are trained for specific orientations, so they need to be taken into
|
|
|
|
|
* consideration when running a frame processor. See also: `isMirrored`
|
|
|
|
|
*/
|
|
|
|
|
orientation: Orientation;
|
2023-08-21 04:50:14 -06:00
|
|
|
/**
|
|
|
|
|
* Represents the pixel-format of the Frame.
|
|
|
|
|
*/
|
|
|
|
|
pixelFormat: PixelFormat;
|
2021-05-06 06:11:55 -06:00
|
|
|
|
2023-02-21 07:00:48 -07:00
|
|
|
/**
|
|
|
|
|
* Get the underlying data of the Frame as a uint8 array buffer.
|
|
|
|
|
*
|
|
|
|
|
* Note that Frames are allocated on the GPU, so calling `toArrayBuffer()` will copy from the GPU to the CPU.
|
|
|
|
|
*/
|
|
|
|
|
toArrayBuffer(): Uint8Array;
|
2021-05-06 06:11:55 -06:00
|
|
|
/**
|
|
|
|
|
* Returns a string representation of the frame.
|
|
|
|
|
* @example
|
|
|
|
|
* ```ts
|
|
|
|
|
* console.log(frame.toString()) // -> "3840 x 2160 Frame"
|
|
|
|
|
* ```
|
|
|
|
|
*/
|
|
|
|
|
toString(): string;
|
feat: Sync Frame Processors (plus `runAsync` and `runAtTargetFps`) (#1472)
Before, Frame Processors ran on a separate Thread.
After, Frame Processors run fully synchronous and always at the same FPS as the Camera.
Two new functions have been introduced:
* `runAtTargetFps(fps: number, func: () => void)`: Runs the given code as often as the given `fps`, effectively throttling it's calls.
* `runAsync(frame: Frame, func: () => void)`: Runs the given function on a separate Thread for Frame Processing. A strong reference to the Frame is held as long as the function takes to execute.
You can use `runAtTargetFps` to throttle calls to a specific API (e.g. if your Camera is running at 60 FPS, but you only want to run face detection at ~25 FPS, use `runAtTargetFps(25, ...)`.)
You can use `runAsync` to run a heavy algorithm asynchronous, so that the Camera is not blocked while your algorithm runs. This is useful if your main sync processor draws something, and your async processor is doing some image analysis on the side.
You can also combine both functions.
Examples:
```js
const frameProcessor = useFrameProcessor((frame) => {
'worklet'
console.log("I'm running at 60 FPS!")
}, [])
```
```js
const frameProcessor = useFrameProcessor((frame) => {
'worklet'
console.log("I'm running at 60 FPS!")
runAtTargetFps(10, () => {
'worklet'
console.log("I'm running at 10 FPS!")
})
}, [])
```
```js
const frameProcessor = useFrameProcessor((frame) => {
'worklet'
console.log("I'm running at 60 FPS!")
runAsync(frame, () => {
'worklet'
console.log("I'm running on another Thread, I can block for longer!")
})
}, [])
```
```js
const frameProcessor = useFrameProcessor((frame) => {
'worklet'
console.log("I'm running at 60 FPS!")
runAtTargetFps(10, () => {
'worklet'
runAsync(frame, () => {
'worklet'
console.log("I'm running on another Thread at 10 FPS, I can block for longer!")
})
})
}, [])
```
2023-02-15 08:47:09 -07:00
|
|
|
}
|
|
|
|
|
|
2023-09-01 04:20:17 -06:00
|
|
|
export interface FrameInternal extends Frame {
|
2021-07-06 02:08:44 -06:00
|
|
|
/**
|
feat: Sync Frame Processors (plus `runAsync` and `runAtTargetFps`) (#1472)
Before, Frame Processors ran on a separate Thread.
After, Frame Processors run fully synchronous and always at the same FPS as the Camera.
Two new functions have been introduced:
* `runAtTargetFps(fps: number, func: () => void)`: Runs the given code as often as the given `fps`, effectively throttling it's calls.
* `runAsync(frame: Frame, func: () => void)`: Runs the given function on a separate Thread for Frame Processing. A strong reference to the Frame is held as long as the function takes to execute.
You can use `runAtTargetFps` to throttle calls to a specific API (e.g. if your Camera is running at 60 FPS, but you only want to run face detection at ~25 FPS, use `runAtTargetFps(25, ...)`.)
You can use `runAsync` to run a heavy algorithm asynchronous, so that the Camera is not blocked while your algorithm runs. This is useful if your main sync processor draws something, and your async processor is doing some image analysis on the side.
You can also combine both functions.
Examples:
```js
const frameProcessor = useFrameProcessor((frame) => {
'worklet'
console.log("I'm running at 60 FPS!")
}, [])
```
```js
const frameProcessor = useFrameProcessor((frame) => {
'worklet'
console.log("I'm running at 60 FPS!")
runAtTargetFps(10, () => {
'worklet'
console.log("I'm running at 10 FPS!")
})
}, [])
```
```js
const frameProcessor = useFrameProcessor((frame) => {
'worklet'
console.log("I'm running at 60 FPS!")
runAsync(frame, () => {
'worklet'
console.log("I'm running on another Thread, I can block for longer!")
})
}, [])
```
```js
const frameProcessor = useFrameProcessor((frame) => {
'worklet'
console.log("I'm running at 60 FPS!")
runAtTargetFps(10, () => {
'worklet'
runAsync(frame, () => {
'worklet'
console.log("I'm running on another Thread at 10 FPS, I can block for longer!")
})
})
}, [])
```
2023-02-15 08:47:09 -07:00
|
|
|
* Increment the Frame Buffer ref-count by one.
|
2021-07-06 02:08:44 -06:00
|
|
|
*
|
feat: Sync Frame Processors (plus `runAsync` and `runAtTargetFps`) (#1472)
Before, Frame Processors ran on a separate Thread.
After, Frame Processors run fully synchronous and always at the same FPS as the Camera.
Two new functions have been introduced:
* `runAtTargetFps(fps: number, func: () => void)`: Runs the given code as often as the given `fps`, effectively throttling it's calls.
* `runAsync(frame: Frame, func: () => void)`: Runs the given function on a separate Thread for Frame Processing. A strong reference to the Frame is held as long as the function takes to execute.
You can use `runAtTargetFps` to throttle calls to a specific API (e.g. if your Camera is running at 60 FPS, but you only want to run face detection at ~25 FPS, use `runAtTargetFps(25, ...)`.)
You can use `runAsync` to run a heavy algorithm asynchronous, so that the Camera is not blocked while your algorithm runs. This is useful if your main sync processor draws something, and your async processor is doing some image analysis on the side.
You can also combine both functions.
Examples:
```js
const frameProcessor = useFrameProcessor((frame) => {
'worklet'
console.log("I'm running at 60 FPS!")
}, [])
```
```js
const frameProcessor = useFrameProcessor((frame) => {
'worklet'
console.log("I'm running at 60 FPS!")
runAtTargetFps(10, () => {
'worklet'
console.log("I'm running at 10 FPS!")
})
}, [])
```
```js
const frameProcessor = useFrameProcessor((frame) => {
'worklet'
console.log("I'm running at 60 FPS!")
runAsync(frame, () => {
'worklet'
console.log("I'm running on another Thread, I can block for longer!")
})
}, [])
```
```js
const frameProcessor = useFrameProcessor((frame) => {
'worklet'
console.log("I'm running at 60 FPS!")
runAtTargetFps(10, () => {
'worklet'
runAsync(frame, () => {
'worklet'
console.log("I'm running on another Thread at 10 FPS, I can block for longer!")
})
})
}, [])
```
2023-02-15 08:47:09 -07:00
|
|
|
* This is a private API, do not use this.
|
|
|
|
|
*/
|
|
|
|
|
incrementRefCount(): void;
|
|
|
|
|
/**
|
|
|
|
|
* Increment the Frame Buffer ref-count by one.
|
|
|
|
|
*
|
|
|
|
|
* This is a private API, do not use this.
|
2021-07-06 02:08:44 -06:00
|
|
|
*/
|
feat: Sync Frame Processors (plus `runAsync` and `runAtTargetFps`) (#1472)
Before, Frame Processors ran on a separate Thread.
After, Frame Processors run fully synchronous and always at the same FPS as the Camera.
Two new functions have been introduced:
* `runAtTargetFps(fps: number, func: () => void)`: Runs the given code as often as the given `fps`, effectively throttling it's calls.
* `runAsync(frame: Frame, func: () => void)`: Runs the given function on a separate Thread for Frame Processing. A strong reference to the Frame is held as long as the function takes to execute.
You can use `runAtTargetFps` to throttle calls to a specific API (e.g. if your Camera is running at 60 FPS, but you only want to run face detection at ~25 FPS, use `runAtTargetFps(25, ...)`.)
You can use `runAsync` to run a heavy algorithm asynchronous, so that the Camera is not blocked while your algorithm runs. This is useful if your main sync processor draws something, and your async processor is doing some image analysis on the side.
You can also combine both functions.
Examples:
```js
const frameProcessor = useFrameProcessor((frame) => {
'worklet'
console.log("I'm running at 60 FPS!")
}, [])
```
```js
const frameProcessor = useFrameProcessor((frame) => {
'worklet'
console.log("I'm running at 60 FPS!")
runAtTargetFps(10, () => {
'worklet'
console.log("I'm running at 10 FPS!")
})
}, [])
```
```js
const frameProcessor = useFrameProcessor((frame) => {
'worklet'
console.log("I'm running at 60 FPS!")
runAsync(frame, () => {
'worklet'
console.log("I'm running on another Thread, I can block for longer!")
})
}, [])
```
```js
const frameProcessor = useFrameProcessor((frame) => {
'worklet'
console.log("I'm running at 60 FPS!")
runAtTargetFps(10, () => {
'worklet'
runAsync(frame, () => {
'worklet'
console.log("I'm running on another Thread at 10 FPS, I can block for longer!")
})
})
}, [])
```
2023-02-15 08:47:09 -07:00
|
|
|
decrementRefCount(): void;
|
2021-05-06 06:11:55 -06:00
|
|
|
}
|
