feat: Sync Frame Processors (plus runAsync and runAtTargetFps) (#1472)

Before, Frame Processors ran on a separate Thread.

After, Frame Processors run fully synchronous and always at the same FPS as the Camera.

Two new functions have been introduced:

* `runAtTargetFps(fps: number, func: () => void)`: Runs the given code as often as the given `fps`, effectively throttling it's calls.
* `runAsync(frame: Frame, func: () => void)`: Runs the given function on a separate Thread for Frame Processing. A strong reference to the Frame is held as long as the function takes to execute.

You can use `runAtTargetFps` to throttle calls to a specific API (e.g. if your Camera is running at 60 FPS, but you only want to run face detection at ~25 FPS, use `runAtTargetFps(25, ...)`.)

You can use `runAsync` to run a heavy algorithm asynchronous, so that the Camera is not blocked while your algorithm runs. This is useful if your main sync processor draws something, and your async processor is doing some image analysis on the side. 

You can also combine both functions.

Examples:

```js
const frameProcessor = useFrameProcessor((frame) => {
  'worklet'
  console.log("I'm running at 60 FPS!")
}, [])
```

```js
const frameProcessor = useFrameProcessor((frame) => {
  'worklet'
  console.log("I'm running at 60 FPS!")

  runAtTargetFps(10, () => {
    'worklet'
    console.log("I'm running at 10 FPS!")
  })
}, [])
```



```js
const frameProcessor = useFrameProcessor((frame) => {
  'worklet'
  console.log("I'm running at 60 FPS!")

  runAsync(frame, () => {
    'worklet'
    console.log("I'm running on another Thread, I can block for longer!")
  })
}, [])
```

```js
const frameProcessor = useFrameProcessor((frame) => {
  'worklet'
  console.log("I'm running at 60 FPS!")

  runAtTargetFps(10, () => {
    'worklet'
    runAsync(frame, () => {
      'worklet'
      console.log("I'm running on another Thread at 10 FPS, I can block for longer!")
    })
  })
}, [])
```
This commit is contained in:
Marc Rousavy
2023-02-15 16:47:09 +01:00
committed by GitHub
parent a0590dccb5
commit 30b56153db
30 changed files with 660 additions and 914 deletions

View File

@@ -52,7 +52,6 @@ public final class CameraView: UIView {
// props that require format reconfiguring
@objc var format: NSDictionary?
@objc var fps: NSNumber?
@objc var frameProcessorFps: NSNumber = -1.0 // "auto"
@objc var hdr: NSNumber? // nullable bool
@objc var lowLightBoost: NSNumber? // nullable bool
@objc var colorSpace: NSString?
@@ -65,7 +64,6 @@ public final class CameraView: UIView {
// events
@objc var onInitialized: RCTDirectEventBlock?
@objc var onError: RCTDirectEventBlock?
@objc var onFrameProcessorPerformanceSuggestionAvailable: RCTDirectEventBlock?
@objc var onViewReady: RCTDirectEventBlock?
// zoom
@objc var enableZoomGesture = false {
@@ -105,13 +103,6 @@ public final class CameraView: UIView {
internal let videoQueue = CameraQueues.videoQueue
internal let audioQueue = CameraQueues.audioQueue
/// Specifies whether the frameProcessor() function is currently executing. used to drop late frames.
internal var isRunningFrameProcessor = false
internal let frameProcessorPerformanceDataCollector = FrameProcessorPerformanceDataCollector()
internal var actualFrameProcessorFps = 30.0
internal var lastSuggestedFrameProcessorFps = 0.0
internal var lastFrameProcessorPerformanceEvaluation = DispatchTime.now()
/// Returns whether the AVCaptureSession is currently running (reflected by isActive)
var isRunning: Bool {
return captureSession.isRunning
@@ -259,18 +250,6 @@ public final class CameraView: UIView {
}
}
}
// Frame Processor FPS Configuration
if changedProps.contains("frameProcessorFps") {
if frameProcessorFps.doubleValue == -1 {
// "auto"
actualFrameProcessorFps = 30.0
} else {
actualFrameProcessorFps = frameProcessorFps.doubleValue
}
lastFrameProcessorPerformanceEvaluation = DispatchTime.now()
frameProcessorPerformanceDataCollector.clear()
}
}
internal final func setTorchMode(_ torchMode: String) {
@@ -343,18 +322,4 @@ public final class CameraView: UIView {
guard let onInitialized = onInitialized else { return }
onInitialized([String: Any]())
}
internal final func invokeOnFrameProcessorPerformanceSuggestionAvailable(currentFps: Double, suggestedFps: Double) {
ReactLogger.log(level: .info, message: "Frame Processor Performance Suggestion available!")
guard let onFrameProcessorPerformanceSuggestionAvailable = onFrameProcessorPerformanceSuggestionAvailable else { return }
if lastSuggestedFrameProcessorFps == suggestedFps { return }
if suggestedFps == currentFps { return }
onFrameProcessorPerformanceSuggestionAvailable([
"type": suggestedFps > currentFps ? "can-use-higher-fps" : "should-use-lower-fps",
"suggestedFrameProcessorFps": suggestedFps,
])
lastSuggestedFrameProcessorFps = suggestedFps
}
}