feat: frameProcessorFps="auto" and automatic performance suggestions (throttle or increase FPS) (#393)

* Add `onFrameProcessorPerformanceSuggestionAvailable` and make `frameProcessorFps` support `auto`

* Implement performance suggestion and auto-adjusting

* Fix FPS setting, evaluate correctly

* Floor suggested FPS

* Remove `console.log` for frame drop warnings.

* Swift format

* Use `30` magic number

* only call if FPS is different

* Update CameraView.swift

* Implement Android 1/2

* Cleanup

* Update `frameProcessorFps` if available

* Optimize `FrameProcessorPerformanceDataCollector` initialization

* Cache call

* Set frameProcessorFps directly (Kotlin setter)

* Don't suggest if same value

* Call suggestion every second

* reset time on set

* Always store 15 last samples

* reset counter too

* Update FrameProcessorPerformanceDataCollector.swift

* Update CameraView+RecordVideo.swift

* Update CameraView.kt

* iOS: Redesign evaluation

* Update CameraView+RecordVideo.swift

* Android: Redesign evaluation

* Update CameraView.kt

* Update REA to latest alpha and install RNScreens

* Fix frameProcessorFps updating
This commit is contained in:
Marc Rousavy
2021-09-06 16:27:16 +02:00
committed by GitHub
parent 5b570d611a
commit ad5e131f6a
23 changed files with 335 additions and 120 deletions

View File

@@ -8,9 +8,6 @@
import AVFoundation
private var hasLoggedVideoFrameDropWarning = false
private var hasLoggedFrameProcessorFrameDropWarning = false
// MARK: - CameraView + AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAudioDataOutputSampleBufferDelegate
extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAudioDataOutputSampleBufferDelegate {
@@ -203,63 +200,61 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud
}
}
// TODO: resize using VideoToolbox (VTPixelTransferSession)
if let frameProcessor = frameProcessorCallback,
captureOutput is AVCaptureVideoDataOutput {
if let frameProcessor = frameProcessorCallback, captureOutput is AVCaptureVideoDataOutput {
// check if last frame was x nanoseconds ago, effectively throttling FPS
let diff = DispatchTime.now().uptimeNanoseconds - lastFrameProcessorCall.uptimeNanoseconds
let secondsPerFrame = 1.0 / frameProcessorFps.doubleValue
let lastFrameProcessorCallElapsedTime = DispatchTime.now().uptimeNanoseconds - lastFrameProcessorCall.uptimeNanoseconds
let secondsPerFrame = 1.0 / actualFrameProcessorFps
let nanosecondsPerFrame = secondsPerFrame * 1_000_000_000.0
if diff > UInt64(nanosecondsPerFrame) {
if lastFrameProcessorCallElapsedTime > UInt64(nanosecondsPerFrame) {
if !isRunningFrameProcessor {
// we're not in the middle of executing the Frame Processor, so prepare for next call.
CameraQueues.frameProcessorQueue.async {
self.isRunningFrameProcessor = true
let perfSample = self.frameProcessorPerformanceDataCollector.beginPerformanceSampleCollection()
let frame = Frame(buffer: sampleBuffer, orientation: self.bufferOrientation)
frameProcessor(frame)
perfSample.endPerformanceSampleCollection()
self.isRunningFrameProcessor = false
}
lastFrameProcessorCall = DispatchTime.now()
} else {
// we're still in the middle of executing a Frame Processor for a previous frame, notify user about dropped frame.
if !hasLoggedFrameProcessorFrameDropWarning {
ReactLogger.log(level: .warning,
message: "Your Frame Processor took so long to execute that a frame was dropped. " +
"Either throttle your Frame Processor's frame rate using the `frameProcessorFps` prop, or optimize " +
"it's execution speed. (This warning will only be shown once)",
alsoLogToJS: true)
hasLoggedFrameProcessorFrameDropWarning = true
}
// we're still in the middle of executing a Frame Processor for a previous frame, so a frame was dropped.
ReactLogger.log(level: .warning, message: "The Frame Processor took so long to execute that a frame was dropped.")
}
}
if isReadyForNewEvaluation {
// last evaluation was more than 1sec ago, evaluate again
evaluateNewPerformanceSamples()
}
}
}
#if DEBUG
public final func captureOutput(_ captureOutput: AVCaptureOutput, didDrop buffer: CMSampleBuffer, from _: AVCaptureConnection) {
if !hasLoggedVideoFrameDropWarning && captureOutput is AVCaptureVideoDataOutput {
let reason = findFrameDropReason(inBuffer: buffer)
ReactLogger.log(level: .warning,
message: "Dropped a Frame - This might indicate that your frame rate is higher than the phone can currently process. " +
"Throttle the Camera frame rate using the `fps` prop and make sure the device stays in optimal condition for recording. " +
"Frame drop reason: \(reason). (This warning will only be shown once)",
alsoLogToJS: true)
hasLoggedVideoFrameDropWarning = true
}
}
private func evaluateNewPerformanceSamples() {
lastFrameProcessorPerformanceEvaluation = DispatchTime.now()
guard let videoDevice = videoDeviceInput?.device else { return }
private final func findFrameDropReason(inBuffer buffer: CMSampleBuffer) -> String {
var mode: CMAttachmentMode = 0
guard let reason = CMGetAttachment(buffer,
key: kCMSampleBufferAttachmentKey_DroppedFrameReason,
attachmentModeOut: &mode) else {
return "unknown"
}
return String(describing: reason)
let maxFrameProcessorFps = Double(videoDevice.activeVideoMinFrameDuration.timescale) * Double(videoDevice.activeVideoMinFrameDuration.value)
let averageFps = 1.0 / frameProcessorPerformanceDataCollector.averageExecutionTimeSeconds
let suggestedFrameProcessorFps = floor(min(averageFps, maxFrameProcessorFps))
if frameProcessorFps.intValue == -1 {
// frameProcessorFps="auto"
actualFrameProcessorFps = suggestedFrameProcessorFps
} else {
// frameProcessorFps={someCustomFpsValue}
invokeOnFrameProcessorPerformanceSuggestionAvailable(currentFps: frameProcessorFps.doubleValue,
suggestedFps: suggestedFrameProcessorFps)
}
#endif
}
private var isReadyForNewEvaluation: Bool {
let lastPerformanceEvaluationElapsedTime = DispatchTime.now().uptimeNanoseconds - lastFrameProcessorPerformanceEvaluation.uptimeNanoseconds
return lastPerformanceEvaluationElapsedTime > 1_000_000_000
}
/**
Gets the orientation of the CameraView's images (CMSampleBuffers).