From ad5e131f6a56a96d39693e1d8135cfa062efd9fa Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Mon, 6 Sep 2021 16:27:16 +0200 Subject: [PATCH] feat: `frameProcessorFps="auto"` and automatic performance suggestions (throttle or increase FPS) (#393) * Add `onFrameProcessorPerformanceSuggestionAvailable` and make `frameProcessorFps` support `auto` * Implement performance suggestion and auto-adjusting * Fix FPS setting, evaluate correctly * Floor suggested FPS * Remove `console.log` for frame drop warnings. * Swift format * Use `30` magic number * only call if FPS is different * Update CameraView.swift * Implement Android 1/2 * Cleanup * Update `frameProcessorFps` if available * Optimize `FrameProcessorPerformanceDataCollector` initialization * Cache call * Set frameProcessorFps directly (Kotlin setter) * Don't suggest if same value * Call suggestion every second * reset time on set * Always store 15 last samples * reset counter too * Update FrameProcessorPerformanceDataCollector.swift * Update CameraView+RecordVideo.swift * Update CameraView.kt * iOS: Redesign evaluation * Update CameraView+RecordVideo.swift * Android: Redesign evaluation * Update CameraView.kt * Update REA to latest alpha and install RNScreens * Fix frameProcessorFps updating --- .../com/mrousavy/camera/CameraView+Events.kt | 53 ++++++++++++ .../java/com/mrousavy/camera/CameraView.kt | 80 +++++++++++-------- .../com/mrousavy/camera/CameraViewManager.kt | 1 + .../FrameProcessorPerformanceDataCollector.kt | 38 +++++++++ example/ios/Podfile.lock | 7 ++ example/package.json | 3 +- example/src/CameraPage.tsx | 6 ++ example/yarn.lock | 4 +- ios/CameraView+AVAudioSession.swift | 7 +- ios/CameraView+AVCaptureSession.swift | 2 +- ios/CameraView+RecordVideo.swift | 75 ++++++++--------- ios/CameraView.swift | 33 +++++++- ios/CameraViewManager.m | 1 + ios/CameraViewManager.swift | 5 -- .../AVAudioSession+updateCategory.swift | 3 +- ...ameProcessorPerformanceDataCollector.swift | 63 +++++++++++++++ ios/React Utils/ReactLogger.swift | 9 --- ios/RecordingSession.swift | 25 +++--- ios/VisionCamera.xcodeproj/project.pbxproj | 2 + package.json | 2 +- src/Camera.tsx | 18 ++++- src/CameraProps.ts | 16 +++- yarn.lock | 2 +- 23 files changed, 335 insertions(+), 120 deletions(-) create mode 100644 android/src/main/java/com/mrousavy/camera/CameraView+Events.kt create mode 100644 android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPerformanceDataCollector.kt create mode 100644 ios/Frame Processor/FrameProcessorPerformanceDataCollector.swift diff --git a/android/src/main/java/com/mrousavy/camera/CameraView+Events.kt b/android/src/main/java/com/mrousavy/camera/CameraView+Events.kt new file mode 100644 index 0000000..3536a3c --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/CameraView+Events.kt @@ -0,0 +1,53 @@ +package com.mrousavy.camera + +import android.util.Log +import com.facebook.react.bridge.Arguments +import com.facebook.react.bridge.ReactContext +import com.facebook.react.bridge.WritableMap +import com.facebook.react.uimanager.events.RCTEventEmitter + +fun CameraView.invokeOnInitialized() { + Log.i(CameraView.TAG, "invokeOnInitialized()") + + val reactContext = context as ReactContext + reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraInitialized", null) +} + +fun CameraView.invokeOnError(error: Throwable) { + Log.e(CameraView.TAG, "invokeOnError(...):") + error.printStackTrace() + + val cameraError = when (error) { + is CameraError -> error + else -> UnknownCameraError(error) + } + val event = Arguments.createMap() + event.putString("code", cameraError.code) + event.putString("message", cameraError.message) + cameraError.cause?.let { cause -> + event.putMap("cause", errorToMap(cause)) + } + val reactContext = context as ReactContext + reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraError", event) +} + +fun CameraView.invokeOnFrameProcessorPerformanceSuggestionAvailable(currentFps: Double, suggestedFps: Double) { + Log.e(CameraView.TAG, "invokeOnFrameProcessorPerformanceSuggestionAvailable(suggestedFps: $suggestedFps):") + + val event = Arguments.createMap() + val type = if (suggestedFps > currentFps) "can-use-higher-fps" else "should-use-lower-fps" + event.putString("type", type) + event.putDouble("suggestedFrameProcessorFps", suggestedFps) + val reactContext = context as ReactContext + reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraPerformanceSuggestionAvailable", event) +} + +private fun errorToMap(error: Throwable): WritableMap { + val map = Arguments.createMap() + map.putString("message", error.message) + map.putString("stacktrace", error.stackTraceToString()) + error.cause?.let { cause -> + map.putMap("cause", errorToMap(cause)) + } + return map +} diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index e126a63..4d1032a 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -23,12 +23,14 @@ import com.facebook.jni.HybridData import com.facebook.proguard.annotations.DoNotStrip import com.facebook.react.bridge.* import com.facebook.react.uimanager.events.RCTEventEmitter +import com.mrousavy.camera.frameprocessor.FrameProcessorPerformanceDataCollector import com.mrousavy.camera.utils.* import kotlinx.coroutines.* import kotlinx.coroutines.guava.await import java.lang.IllegalArgumentException import java.util.concurrent.ExecutorService import java.util.concurrent.Executors +import kotlin.math.floor import kotlin.math.max import kotlin.math.min @@ -93,7 +95,17 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer var torch = "off" var zoom: Float = 1f // in "factor" var enableZoomGesture = false + set(value) { + field = value + setOnTouchListener(if (value) touchEventListener else null) + } var frameProcessorFps = 1.0 + set(value) { + field = value + actualFrameProcessorFps = if (value == -1.0) 30.0 else value + lastFrameProcessorPerformanceEvaluation = System.currentTimeMillis() + frameProcessorPerformanceDataCollector.clear() + } // private properties private val reactContext: ReactContext @@ -131,6 +143,16 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer private var minZoom: Float = 1f private var maxZoom: Float = 1f + private var actualFrameProcessorFps = 30.0 + private val frameProcessorPerformanceDataCollector = FrameProcessorPerformanceDataCollector() + private var lastSuggestedFrameProcessorFps = 0.0 + private var lastFrameProcessorPerformanceEvaluation = System.currentTimeMillis() + private val isReadyForNewEvaluation: Boolean + get() { + val lastPerformanceEvaluationElapsedTime = System.currentTimeMillis() - lastFrameProcessorPerformanceEvaluation + return lastPerformanceEvaluationElapsedTime > 1000 + } + @DoNotStrip private var mHybridData: HybridData @@ -277,9 +299,6 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer if (shouldReconfigureTorch) { camera!!.cameraControl.enableTorch(torch == "on") } - if (changedProps.contains("enableZoomGesture")) { - setOnTouchListener(if (enableZoomGesture) touchEventListener else null) - } } catch (e: Throwable) { Log.e(TAG, "update() threw: ${e.message}") invokeOnError(e) @@ -406,12 +425,20 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer imageAnalysis = imageAnalysisBuilder.build().apply { setAnalyzer(cameraExecutor, { image -> val now = System.currentTimeMillis() - val intervalMs = (1.0 / frameProcessorFps) * 1000.0 + val intervalMs = (1.0 / actualFrameProcessorFps) * 1000.0 if (now - lastFrameProcessorCall > intervalMs) { lastFrameProcessorCall = now + + val perfSample = frameProcessorPerformanceDataCollector.beginPerformanceSampleCollection() frameProcessorCallback(image) + perfSample.endPerformanceSampleCollection() } image.close() + + if (isReadyForNewEvaluation) { + // last evaluation was more than a second ago, evaluate again + evaluateNewPerformanceSamples() + } }) } useCases.add(imageAnalysis!!) @@ -444,38 +471,21 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer } } - private fun invokeOnInitialized() { - Log.i(TAG, "invokeOnInitialized()") + private fun evaluateNewPerformanceSamples() { + lastFrameProcessorPerformanceEvaluation = System.currentTimeMillis() + val maxFrameProcessorFps = 30 // TODO: Get maxFrameProcessorFps from ImageAnalyser + val averageFps = 1.0 / frameProcessorPerformanceDataCollector.averageExecutionTimeSeconds + val suggestedFrameProcessorFps = floor(min(averageFps, maxFrameProcessorFps.toDouble())) - val reactContext = context as ReactContext - reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraInitialized", null) - } - - private fun invokeOnError(error: Throwable) { - Log.e(TAG, "invokeOnError(...):") - error.printStackTrace() - - val cameraError = when (error) { - is CameraError -> error - else -> UnknownCameraError(error) + if (frameProcessorFps == -1.0) { + // frameProcessorFps="auto" + actualFrameProcessorFps = suggestedFrameProcessorFps + } else { + // frameProcessorFps={someCustomFpsValue} + if (suggestedFrameProcessorFps != lastSuggestedFrameProcessorFps && suggestedFrameProcessorFps != frameProcessorFps) { + invokeOnFrameProcessorPerformanceSuggestionAvailable(frameProcessorFps, suggestedFrameProcessorFps) + lastSuggestedFrameProcessorFps = suggestedFrameProcessorFps + } } - val event = Arguments.createMap() - event.putString("code", cameraError.code) - event.putString("message", cameraError.message) - cameraError.cause?.let { cause -> - event.putMap("cause", errorToMap(cause)) - } - val reactContext = context as ReactContext - reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraError", event) - } - - private fun errorToMap(error: Throwable): WritableMap { - val map = Arguments.createMap() - map.putString("message", error.message) - map.putString("stacktrace", error.stackTraceToString()) - error.cause?.let { cause -> - map.putMap("cause", errorToMap(cause)) - } - return map } } diff --git a/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt b/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt index 87d6f0f..5b01ffe 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt @@ -57,6 +57,7 @@ class CameraViewManager(reactContext: ReactApplicationContext) : SimpleViewManag return MapBuilder.builder() .put("cameraInitialized", MapBuilder.of("registrationName", "onInitialized")) .put("cameraError", MapBuilder.of("registrationName", "onError")) + .put("cameraPerformanceSuggestionAvailable", MapBuilder.of("registrationName", "onFrameProcessorPerformanceSuggestionAvailable")) .build() } diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPerformanceDataCollector.kt b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPerformanceDataCollector.kt new file mode 100644 index 0000000..80cb428 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPerformanceDataCollector.kt @@ -0,0 +1,38 @@ +package com.mrousavy.camera.frameprocessor + +data class PerformanceSampleCollection(val endPerformanceSampleCollection: () -> Unit) + +// keep a maximum of `maxSampleSize` historical performance data samples cached. +private const val maxSampleSize = 15 + +class FrameProcessorPerformanceDataCollector { + private var counter = 0 + private var performanceSamples: ArrayList = ArrayList() + + val averageExecutionTimeSeconds: Double + get() = performanceSamples.average() + + fun beginPerformanceSampleCollection(): PerformanceSampleCollection { + val begin = System.currentTimeMillis() + + return PerformanceSampleCollection { + val end = System.currentTimeMillis() + val seconds = (end - begin) / 1_000.0 + + val index = counter % maxSampleSize + + if (performanceSamples.size > index) { + performanceSamples[index] = seconds + } else { + performanceSamples.add(seconds) + } + + counter++ + } + } + + fun clear() { + counter = 0 + performanceSamples.clear() + } +} diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 0f762af..25dbb02 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -321,6 +321,9 @@ PODS: - React-RCTVibration - ReactCommon/turbomodule/core - Yoga + - RNScreens (3.6.0): + - React-Core + - React-RCTImage - RNStaticSafeAreaInsets (2.1.1): - React - RNVectorIcons (8.1.0): @@ -368,6 +371,7 @@ DEPENDENCIES: - ReactNativeNavigation (from `../node_modules/react-native-navigation`) - RNGestureHandler (from `../node_modules/react-native-gesture-handler`) - RNReanimated (from `../node_modules/react-native-reanimated`) + - RNScreens (from `../node_modules/react-native-screens`) - RNStaticSafeAreaInsets (from `../node_modules/react-native-static-safe-area-insets`) - RNVectorIcons (from `../node_modules/react-native-vector-icons`) - VisionCamera (from `../..`) @@ -447,6 +451,8 @@ EXTERNAL SOURCES: :path: "../node_modules/react-native-gesture-handler" RNReanimated: :path: "../node_modules/react-native-reanimated" + RNScreens: + :path: "../node_modules/react-native-screens" RNStaticSafeAreaInsets: :path: "../node_modules/react-native-static-safe-area-insets" RNVectorIcons: @@ -493,6 +499,7 @@ SPEC CHECKSUMS: ReactNativeNavigation: 87aa2e3a749a9b338057e8d53af54d865241b843 RNGestureHandler: a479ebd5ed4221a810967000735517df0d2db211 RNReanimated: 48e578538b2fad573d3d5ce2e80ad375e1534d87 + RNScreens: eb0dfb2d6b21d2d7f980ad46b14eb306d2f1062e RNStaticSafeAreaInsets: 6103cf09647fa427186d30f67b0f5163c1ae8252 RNVectorIcons: 31cebfcf94e8cf8686eb5303ae0357da64d7a5a4 VisionCamera: dc481620431e31c1d6bbbc438d8a1d9c56f3d304 diff --git a/example/package.json b/example/package.json index f61c359..c643926 100644 --- a/example/package.json +++ b/example/package.json @@ -23,7 +23,8 @@ "react-native-gesture-handler": "^1.10.3", "react-native-navigation": "^7.19.0", "react-native-pressable-opacity": "^1.0.4", - "react-native-reanimated": "^2.3.0-alpha.2", + "react-native-reanimated": "^2.3.0-alpha.3", + "react-native-screens": "^3.6.0", "react-native-static-safe-area-insets": "^2.1.1", "react-native-vector-icons": "^8.0.0", "react-native-video": "^5.1.1" diff --git a/example/src/CameraPage.tsx b/example/src/CameraPage.tsx index 840d2af..5f3196a 100644 --- a/example/src/CameraPage.tsx +++ b/example/src/CameraPage.tsx @@ -6,6 +6,7 @@ import { Navigation, NavigationFunctionComponent } from 'react-native-navigation import { CameraDeviceFormat, CameraRuntimeError, + FrameProcessorPerformanceSuggestion, PhotoFile, sortFormats, useCameraDevices, @@ -197,6 +198,10 @@ export const CameraPage: NavigationFunctionComponent = ({ componentId }) => { console.log(`Return Values: ${JSON.stringify(values)}`); }, []); + const onFrameProcessorSuggestionAvailable = useCallback((suggestion: FrameProcessorPerformanceSuggestion) => { + console.log(`Suggestion available! ${suggestion.type}: Can do ${suggestion.suggestedFrameProcessorFps} FPS`); + }, []); + return ( {device != null && ( @@ -221,6 +226,7 @@ export const CameraPage: NavigationFunctionComponent = ({ componentId }) => { audio={true} frameProcessor={device.supportsParallelVideoProcessing ? frameProcessor : undefined} frameProcessorFps={1} + onFrameProcessorPerformanceSuggestionAvailable={onFrameProcessorSuggestionAvailable} /> diff --git a/example/yarn.lock b/example/yarn.lock index 4b2ce73..2f5fb47 100644 --- a/example/yarn.lock +++ b/example/yarn.lock @@ -4401,7 +4401,7 @@ react-native-pressable-opacity@^1.0.4: resolved "https://registry.yarnpkg.com/react-native-pressable-opacity/-/react-native-pressable-opacity-1.0.4.tgz#391f33fdc25cb84551f2743a25eced892b9f30f7" integrity sha512-DBIg7UoRiuBYiFEvx+XNMqH0OEx64WrSksXhT6Kq9XuyyKsThMNDqZ9G5QV7vfu7dU2/IctwIz5c0Xwkp4K3tA== -react-native-reanimated@^2.3.0-alpha.2: +react-native-reanimated@^2.3.0-alpha.3: version "2.3.0-alpha.3" resolved "https://registry.yarnpkg.com/react-native-reanimated/-/react-native-reanimated-2.3.0-alpha.3.tgz#022f5b4eaa847304a155942165a77125c86428bc" integrity sha512-Ln+edkTrepKUETKKmGl+GxL8RiB7awBDQFcRJIK39m5ggiLfksOAfqzclpKPLWbAfOvB4cSg+QdowzRN570FXA== @@ -4413,7 +4413,7 @@ react-native-reanimated@^2.3.0-alpha.2: react-native-screens "^3.4.0" string-hash-64 "^1.0.3" -react-native-screens@^3.4.0: +react-native-screens@^3.4.0, react-native-screens@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/react-native-screens/-/react-native-screens-3.6.0.tgz#054af728e50c06bff6b3b4fa7b4b656b70f247cd" integrity sha512-emQmSu+B6cOIjJH2OIgpuxd9zCD6xz7/oo5GCetyjsM5qR3sMnVgOxqtK99xPu9XJH/8k7MplXbtJgtk/PHXwA== diff --git a/ios/CameraView+AVAudioSession.swift b/ios/CameraView+AVAudioSession.swift index 3e47459..46b80e7 100644 --- a/ios/CameraView+AVAudioSession.swift +++ b/ios/CameraView+AVAudioSession.swift @@ -118,10 +118,11 @@ extension CameraView { return } + // TODO: Add JS-Event for Audio Session interruptions? switch type { case .began: // Something interrupted our Audio Session, stop recording audio. - ReactLogger.log(level: .error, message: "The Audio Session was interrupted!", alsoLogToJS: true) + ReactLogger.log(level: .error, message: "The Audio Session was interrupted!") case .ended: ReactLogger.log(level: .info, message: "The Audio Session interruption has ended.") guard let optionsValue = userInfo[AVAudioSessionInterruptionOptionKey] as? UInt else { return } @@ -129,13 +130,13 @@ extension CameraView { if options.contains(.shouldResume) { if isRecording { audioQueue.async { - ReactLogger.log(level: .info, message: "Resuming interrupted Audio Session...", alsoLogToJS: true) + ReactLogger.log(level: .info, message: "Resuming interrupted Audio Session...") // restart audio session because interruption is over self.activateAudioSession() } } } else { - ReactLogger.log(level: .error, message: "Cannot resume interrupted Audio Session!", alsoLogToJS: true) + ReactLogger.log(level: .error, message: "Cannot resume interrupted Audio Session!") } @unknown default: () } diff --git a/ios/CameraView+AVCaptureSession.swift b/ios/CameraView+AVCaptureSession.swift index 2d972a8..a99350f 100644 --- a/ios/CameraView+AVCaptureSession.swift +++ b/ios/CameraView+AVCaptureSession.swift @@ -255,7 +255,7 @@ extension CameraView { @objc func sessionRuntimeError(notification: Notification) { - ReactLogger.log(level: .error, message: "Unexpected Camera Runtime Error occured!", alsoLogToJS: true) + ReactLogger.log(level: .error, message: "Unexpected Camera Runtime Error occured!") guard let error = notification.userInfo?[AVCaptureSessionErrorKey] as? AVError else { return } diff --git a/ios/CameraView+RecordVideo.swift b/ios/CameraView+RecordVideo.swift index a08b65c..12c68ba 100644 --- a/ios/CameraView+RecordVideo.swift +++ b/ios/CameraView+RecordVideo.swift @@ -8,9 +8,6 @@ import AVFoundation -private var hasLoggedVideoFrameDropWarning = false -private var hasLoggedFrameProcessorFrameDropWarning = false - // MARK: - CameraView + AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAudioDataOutputSampleBufferDelegate extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAudioDataOutputSampleBufferDelegate { @@ -203,63 +200,61 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud } } - // TODO: resize using VideoToolbox (VTPixelTransferSession) - - if let frameProcessor = frameProcessorCallback, - captureOutput is AVCaptureVideoDataOutput { + if let frameProcessor = frameProcessorCallback, captureOutput is AVCaptureVideoDataOutput { // check if last frame was x nanoseconds ago, effectively throttling FPS - let diff = DispatchTime.now().uptimeNanoseconds - lastFrameProcessorCall.uptimeNanoseconds - let secondsPerFrame = 1.0 / frameProcessorFps.doubleValue + let lastFrameProcessorCallElapsedTime = DispatchTime.now().uptimeNanoseconds - lastFrameProcessorCall.uptimeNanoseconds + let secondsPerFrame = 1.0 / actualFrameProcessorFps let nanosecondsPerFrame = secondsPerFrame * 1_000_000_000.0 - if diff > UInt64(nanosecondsPerFrame) { + if lastFrameProcessorCallElapsedTime > UInt64(nanosecondsPerFrame) { if !isRunningFrameProcessor { // we're not in the middle of executing the Frame Processor, so prepare for next call. CameraQueues.frameProcessorQueue.async { self.isRunningFrameProcessor = true + + let perfSample = self.frameProcessorPerformanceDataCollector.beginPerformanceSampleCollection() let frame = Frame(buffer: sampleBuffer, orientation: self.bufferOrientation) frameProcessor(frame) + perfSample.endPerformanceSampleCollection() + self.isRunningFrameProcessor = false } lastFrameProcessorCall = DispatchTime.now() } else { - // we're still in the middle of executing a Frame Processor for a previous frame, notify user about dropped frame. - if !hasLoggedFrameProcessorFrameDropWarning { - ReactLogger.log(level: .warning, - message: "Your Frame Processor took so long to execute that a frame was dropped. " + - "Either throttle your Frame Processor's frame rate using the `frameProcessorFps` prop, or optimize " + - "it's execution speed. (This warning will only be shown once)", - alsoLogToJS: true) - hasLoggedFrameProcessorFrameDropWarning = true - } + // we're still in the middle of executing a Frame Processor for a previous frame, so a frame was dropped. + ReactLogger.log(level: .warning, message: "The Frame Processor took so long to execute that a frame was dropped.") } } + + if isReadyForNewEvaluation { + // last evaluation was more than 1sec ago, evaluate again + evaluateNewPerformanceSamples() + } } } - #if DEBUG - public final func captureOutput(_ captureOutput: AVCaptureOutput, didDrop buffer: CMSampleBuffer, from _: AVCaptureConnection) { - if !hasLoggedVideoFrameDropWarning && captureOutput is AVCaptureVideoDataOutput { - let reason = findFrameDropReason(inBuffer: buffer) - ReactLogger.log(level: .warning, - message: "Dropped a Frame - This might indicate that your frame rate is higher than the phone can currently process. " + - "Throttle the Camera frame rate using the `fps` prop and make sure the device stays in optimal condition for recording. " + - "Frame drop reason: \(reason). (This warning will only be shown once)", - alsoLogToJS: true) - hasLoggedVideoFrameDropWarning = true - } - } + private func evaluateNewPerformanceSamples() { + lastFrameProcessorPerformanceEvaluation = DispatchTime.now() + guard let videoDevice = videoDeviceInput?.device else { return } - private final func findFrameDropReason(inBuffer buffer: CMSampleBuffer) -> String { - var mode: CMAttachmentMode = 0 - guard let reason = CMGetAttachment(buffer, - key: kCMSampleBufferAttachmentKey_DroppedFrameReason, - attachmentModeOut: &mode) else { - return "unknown" - } - return String(describing: reason) + let maxFrameProcessorFps = Double(videoDevice.activeVideoMinFrameDuration.timescale) * Double(videoDevice.activeVideoMinFrameDuration.value) + let averageFps = 1.0 / frameProcessorPerformanceDataCollector.averageExecutionTimeSeconds + let suggestedFrameProcessorFps = floor(min(averageFps, maxFrameProcessorFps)) + + if frameProcessorFps.intValue == -1 { + // frameProcessorFps="auto" + actualFrameProcessorFps = suggestedFrameProcessorFps + } else { + // frameProcessorFps={someCustomFpsValue} + invokeOnFrameProcessorPerformanceSuggestionAvailable(currentFps: frameProcessorFps.doubleValue, + suggestedFps: suggestedFrameProcessorFps) } - #endif + } + + private var isReadyForNewEvaluation: Bool { + let lastPerformanceEvaluationElapsedTime = DispatchTime.now().uptimeNanoseconds - lastFrameProcessorPerformanceEvaluation.uptimeNanoseconds + return lastPerformanceEvaluationElapsedTime > 1_000_000_000 + } /** Gets the orientation of the CameraView's images (CMSampleBuffers). diff --git a/ios/CameraView.swift b/ios/CameraView.swift index ba8ceb6..548d7fe 100644 --- a/ios/CameraView.swift +++ b/ios/CameraView.swift @@ -52,7 +52,7 @@ public final class CameraView: UIView { // props that require format reconfiguring @objc var format: NSDictionary? @objc var fps: NSNumber? - @objc var frameProcessorFps: NSNumber = 1.0 + @objc var frameProcessorFps: NSNumber = -1.0 // "auto" @objc var hdr: NSNumber? // nullable bool @objc var lowLightBoost: NSNumber? // nullable bool @objc var colorSpace: NSString? @@ -64,6 +64,7 @@ public final class CameraView: UIView { // events @objc var onInitialized: RCTDirectEventBlock? @objc var onError: RCTDirectEventBlock? + @objc var onFrameProcessorPerformanceSuggestionAvailable: RCTDirectEventBlock? // zoom @objc var enableZoomGesture = false { didSet { @@ -104,6 +105,10 @@ public final class CameraView: UIView { /// Specifies whether the frameProcessor() function is currently executing. used to drop late frames. internal var isRunningFrameProcessor = false + internal let frameProcessorPerformanceDataCollector = FrameProcessorPerformanceDataCollector() + internal var actualFrameProcessorFps = 30.0 + internal var lastSuggestedFrameProcessorFps = 0.0 + internal var lastFrameProcessorPerformanceEvaluation = DispatchTime.now() /// Returns whether the AVCaptureSession is currently running (reflected by isActive) var isRunning: Bool { @@ -244,6 +249,18 @@ public final class CameraView: UIView { } } } + + // Frame Processor FPS Configuration + if changedProps.contains("frameProcessorFps") { + if frameProcessorFps.doubleValue == -1 { + // "auto" + actualFrameProcessorFps = 30.0 + } else { + actualFrameProcessorFps = frameProcessorFps.doubleValue + } + lastFrameProcessorPerformanceEvaluation = DispatchTime.now() + frameProcessorPerformanceDataCollector.clear() + } } internal final func setTorchMode(_ torchMode: String) { @@ -336,4 +353,18 @@ public final class CameraView: UIView { guard let onInitialized = self.onInitialized else { return } onInitialized([String: Any]()) } + + internal final func invokeOnFrameProcessorPerformanceSuggestionAvailable(currentFps: Double, suggestedFps: Double) { + ReactLogger.log(level: .info, message: "Frame Processor Performance Suggestion available!") + guard let onFrameProcessorPerformanceSuggestionAvailable = self.onFrameProcessorPerformanceSuggestionAvailable else { return } + + if lastSuggestedFrameProcessorFps == suggestedFps { return } + if suggestedFps == currentFps { return } + + onFrameProcessorPerformanceSuggestionAvailable([ + "type": suggestedFps > currentFps ? "can-use-higher-fps" : "should-use-lower-fps", + "suggestedFrameProcessorFps": suggestedFps, + ]) + lastSuggestedFrameProcessorFps = suggestedFps + } } diff --git a/ios/CameraViewManager.m b/ios/CameraViewManager.m index 0f86f38..8161908 100644 --- a/ios/CameraViewManager.m +++ b/ios/CameraViewManager.m @@ -48,6 +48,7 @@ RCT_EXPORT_VIEW_PROPERTY(enableZoomGesture, BOOL); // Camera View Properties RCT_EXPORT_VIEW_PROPERTY(onError, RCTDirectEventBlock); RCT_EXPORT_VIEW_PROPERTY(onInitialized, RCTDirectEventBlock); +RCT_EXPORT_VIEW_PROPERTY(onFrameProcessorPerformanceSuggestionAvailable, RCTDirectEventBlock); // Camera View Functions RCT_EXTERN_METHOD(startRecording:(nonnull NSNumber *)node options:(NSDictionary *)options onRecordCallback:(RCTResponseSenderBlock)onRecordCallback); diff --git a/ios/CameraViewManager.swift b/ios/CameraViewManager.swift index 7701e4a..8bd2720 100644 --- a/ios/CameraViewManager.swift +++ b/ios/CameraViewManager.swift @@ -17,11 +17,6 @@ final class CameraViewManager: RCTViewManager { override var bridge: RCTBridge! { didSet { - #if DEBUG - // Install console.log bindings - ReactLogger.ConsoleLogFunction = JSConsoleHelper.getLogFunction(for: bridge) - #endif - // Install Frame Processor bindings and setup Runtime if VISION_CAMERA_ENABLE_FRAME_PROCESSORS { CameraQueues.frameProcessorQueue.async { diff --git a/ios/Extensions/AVAudioSession+updateCategory.swift b/ios/Extensions/AVAudioSession+updateCategory.swift index 7826822..c68c743 100644 --- a/ios/Extensions/AVAudioSession+updateCategory.swift +++ b/ios/Extensions/AVAudioSession+updateCategory.swift @@ -16,8 +16,7 @@ extension AVAudioSession { func updateCategory(_ category: AVAudioSession.Category, options: AVAudioSession.CategoryOptions = []) throws { if self.category != category || categoryOptions.rawValue != options.rawValue { ReactLogger.log(level: .info, - message: "Changing AVAudioSession category from \(self.category.rawValue) -> \(category.rawValue)", - alsoLogToJS: true) + message: "Changing AVAudioSession category from \(self.category.rawValue) -> \(category.rawValue)") try setCategory(category, options: options) } } diff --git a/ios/Frame Processor/FrameProcessorPerformanceDataCollector.swift b/ios/Frame Processor/FrameProcessorPerformanceDataCollector.swift new file mode 100644 index 0000000..5f70ea3 --- /dev/null +++ b/ios/Frame Processor/FrameProcessorPerformanceDataCollector.swift @@ -0,0 +1,63 @@ +// +// FrameProcessorPerformanceDataCollector.swift +// VisionCamera +// +// Created by Marc Rousavy on 30.08.21. +// Copyright © 2021 mrousavy. All rights reserved. +// + +import Foundation + +// keep a maximum of `maxSampleSize` historical performance data samples cached. +private let maxSampleSize = 15 + +// MARK: - PerformanceSampleCollection + +struct PerformanceSampleCollection { + var endPerformanceSampleCollection: () -> Void + + init(end: @escaping () -> Void) { + endPerformanceSampleCollection = end + } +} + +// MARK: - FrameProcessorPerformanceDataCollector + +class FrameProcessorPerformanceDataCollector { + private var performanceSamples: [Double] = [] + private var counter = 0 + private var lastEvaluation = -1 + + var averageExecutionTimeSeconds: Double { + let sum = performanceSamples.reduce(0, +) + let average = sum / Double(performanceSamples.count) + + lastEvaluation = counter + + return average + } + + func beginPerformanceSampleCollection() -> PerformanceSampleCollection { + let begin = DispatchTime.now() + + return PerformanceSampleCollection { + let end = DispatchTime.now() + let seconds = Double(end.uptimeNanoseconds - begin.uptimeNanoseconds) / 1_000_000_000.0 + + let index = self.counter % maxSampleSize + + if self.performanceSamples.count > index { + self.performanceSamples[index] = seconds + } else { + self.performanceSamples.append(seconds) + } + + self.counter += 1 + } + } + + func clear() { + counter = 0 + performanceSamples.removeAll() + } +} diff --git a/ios/React Utils/ReactLogger.swift b/ios/React Utils/ReactLogger.swift index 96324d9..932b104 100644 --- a/ios/React Utils/ReactLogger.swift +++ b/ios/React Utils/ReactLogger.swift @@ -11,11 +11,6 @@ import Foundation // MARK: - ReactLogger enum ReactLogger { - /** - A function that logs to the JavaScript console. - */ - static var ConsoleLogFunction: ConsoleLogFunction? - /** Log a message to the console in the format of `VisionCamera.[caller-function-name]: [message]` @@ -27,14 +22,10 @@ enum ReactLogger { @inlinable static func log(level: RCTLogLevel, message: String, - alsoLogToJS: Bool = false, _ file: String = #file, _ lineNumber: Int = #line, _ function: String = #function) { #if DEBUG - if alsoLogToJS, let log = ConsoleLogFunction { - log(level, "[native] VisionCamera.\(function): \(message)") - } RCTDefaultLogFunction(level, RCTLogSource.native, file, lineNumber as NSNumber, "VisionCamera.\(function): \(message)") #endif } diff --git a/ios/RecordingSession.swift b/ios/RecordingSession.swift index ede341a..e2eb890 100644 --- a/ios/RecordingSession.swift +++ b/ios/RecordingSession.swift @@ -60,7 +60,7 @@ class RecordingSession { deinit { if assetWriter.status == .writing { - ReactLogger.log(level: .info, message: "Cancelling AssetWriter...", alsoLogToJS: true) + ReactLogger.log(level: .info, message: "Cancelling AssetWriter...") assetWriter.cancelWriting() } } @@ -70,11 +70,11 @@ class RecordingSession { */ func initializeVideoWriter(withSettings settings: [String: Any], pixelFormat: OSType) { guard !settings.isEmpty else { - ReactLogger.log(level: .error, message: "Tried to initialize Video Writer with empty settings!", alsoLogToJS: true) + ReactLogger.log(level: .error, message: "Tried to initialize Video Writer with empty settings!") return } guard bufferAdaptor == nil else { - ReactLogger.log(level: .error, message: "Tried to add Video Writer twice!", alsoLogToJS: true) + ReactLogger.log(level: .error, message: "Tried to add Video Writer twice!") return } @@ -93,11 +93,11 @@ class RecordingSession { */ func initializeAudioWriter(withSettings settings: [String: Any]) { guard !settings.isEmpty else { - ReactLogger.log(level: .error, message: "Tried to initialize Audio Writer with empty settings!", alsoLogToJS: true) + ReactLogger.log(level: .error, message: "Tried to initialize Audio Writer with empty settings!") return } guard audioWriter == nil else { - ReactLogger.log(level: .error, message: "Tried to add Audio Writer twice!", alsoLogToJS: true) + ReactLogger.log(level: .error, message: "Tried to add Audio Writer twice!") return } @@ -139,8 +139,7 @@ class RecordingSession { } guard let initialTimestamp = initialTimestamp else { ReactLogger.log(level: .error, - message: "A frame arrived, but initialTimestamp was nil. Is this RecordingSession running?", - alsoLogToJS: true) + message: "A frame arrived, but initialTimestamp was nil. Is this RecordingSession running?") return } @@ -149,17 +148,16 @@ class RecordingSession { switch bufferType { case .video: guard let bufferAdaptor = bufferAdaptor else { - ReactLogger.log(level: .error, message: "Video Frame arrived but VideoWriter was nil!", alsoLogToJS: true) + ReactLogger.log(level: .error, message: "Video Frame arrived but VideoWriter was nil!") return } if !bufferAdaptor.assetWriterInput.isReadyForMoreMediaData { ReactLogger.log(level: .warning, - message: "The Video AVAssetWriterInput was not ready for more data! Is your frame rate too high?", - alsoLogToJS: true) + message: "The Video AVAssetWriterInput was not ready for more data! Is your frame rate too high?") return } guard let imageBuffer = CMSampleBufferGetImageBuffer(buffer) else { - ReactLogger.log(level: .error, message: "Failed to get the CVImageBuffer!", alsoLogToJS: true) + ReactLogger.log(level: .error, message: "Failed to get the CVImageBuffer!") return } bufferAdaptor.append(imageBuffer, withPresentationTime: timestamp) @@ -169,7 +167,7 @@ class RecordingSession { } case .audio: guard let audioWriter = audioWriter else { - ReactLogger.log(level: .error, message: "Audio Frame arrived but AudioWriter was nil!", alsoLogToJS: true) + ReactLogger.log(level: .error, message: "Audio Frame arrived but AudioWriter was nil!") return } if !audioWriter.isReadyForMoreMediaData { @@ -184,8 +182,7 @@ class RecordingSession { if assetWriter.status == .failed { ReactLogger.log(level: .error, - message: "AssetWriter failed to write buffer! Error: \(assetWriter.error?.localizedDescription ?? "none")", - alsoLogToJS: true) + message: "AssetWriter failed to write buffer! Error: \(assetWriter.error?.localizedDescription ?? "none")") finish() } } diff --git a/ios/VisionCamera.xcodeproj/project.pbxproj b/ios/VisionCamera.xcodeproj/project.pbxproj index 6f7655b..499e856 100644 --- a/ios/VisionCamera.xcodeproj/project.pbxproj +++ b/ios/VisionCamera.xcodeproj/project.pbxproj @@ -135,6 +135,7 @@ B887518425E0102000DB86D6 /* CameraView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CameraView.swift; sourceTree = ""; }; B88873E5263D46C7008B1D0E /* FrameProcessorPlugin.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorPlugin.h; sourceTree = ""; }; B88B47462667C8E00091F538 /* AVCaptureSession+setVideoStabilizationMode.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVCaptureSession+setVideoStabilizationMode.swift"; sourceTree = ""; }; + B8948BDF26DCEE2B00B430E2 /* FrameProcessorPerformanceDataCollector.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FrameProcessorPerformanceDataCollector.swift; sourceTree = ""; }; B8994E6B263F03E100069589 /* JSIUtils.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = JSIUtils.mm; sourceTree = ""; }; B8A751D62609E4980011C623 /* FrameProcessorRuntimeManager.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorRuntimeManager.h; sourceTree = ""; }; B8A751D72609E4B30011C623 /* FrameProcessorRuntimeManager.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = FrameProcessorRuntimeManager.mm; sourceTree = ""; }; @@ -268,6 +269,7 @@ B88873E5263D46C7008B1D0E /* FrameProcessorPlugin.h */, B80416F026AB16E8000DEB6A /* VisionCameraScheduler.mm */, B80416F126AB16F3000DEB6A /* VisionCameraScheduler.h */, + B8948BDF26DCEE2B00B430E2 /* FrameProcessorPerformanceDataCollector.swift */, ); path = "Frame Processor"; sourceTree = ""; diff --git a/package.json b/package.json index 9b7202c..b0dbe3e 100644 --- a/package.json +++ b/package.json @@ -84,7 +84,7 @@ "react-native": "^0.65.1", "react-native-builder-bob": "^0.18.1", "react-native-codegen": "^0.0.7", - "react-native-reanimated": "^2.3.0-alpha.2", + "react-native-reanimated": "^2.3.0-alpha.3", "release-it": "^14.11.5", "typescript": "4.3.5" }, diff --git a/src/Camera.tsx b/src/Camera.tsx index 9226622..e99c818 100644 --- a/src/Camera.tsx +++ b/src/Camera.tsx @@ -8,6 +8,7 @@ import { Platform, LayoutChangeEvent, } from 'react-native'; +import type { FrameProcessorPerformanceSuggestion } from '.'; import type { CameraDevice } from './CameraDevice'; import type { ErrorWithCause } from './CameraError'; import { CameraCaptureError, CameraRuntimeError, tryParseNativeCameraError, isErrorWithCause } from './CameraError'; @@ -27,11 +28,16 @@ interface OnErrorEvent { message: string; cause?: ErrorWithCause; } -type NativeCameraViewProps = Omit & { +type NativeCameraViewProps = Omit< + CameraProps, + 'device' | 'onInitialized' | 'onError' | 'onFrameProcessorPerformanceSuggestionAvailable' | 'frameProcessor' | 'frameProcessorFps' +> & { cameraId: string; + frameProcessorFps?: number; // native cannot use number | string, so we use '-1' for 'auto' enableFrameProcessor: boolean; onInitialized?: (event: NativeSyntheticEvent) => void; onError?: (event: NativeSyntheticEvent) => void; + onFrameProcessorPerformanceSuggestionAvailable?: (event: NativeSyntheticEvent) => void; }; type RefType = React.Component & Readonly; //#endregion @@ -86,6 +92,7 @@ export class Camera extends React.PureComponent { super(props); this.onInitialized = this.onInitialized.bind(this); this.onError = this.onError.bind(this); + this.onFrameProcessorPerformanceSuggestionAvailable = this.onFrameProcessorPerformanceSuggestionAvailable.bind(this); this.onLayout = this.onLayout.bind(this); this.ref = React.createRef(); this.lastFrameProcessor = undefined; @@ -333,6 +340,11 @@ export class Camera extends React.PureComponent { private onInitialized(): void { this.props.onInitialized?.(); } + + private onFrameProcessorPerformanceSuggestionAvailable(event: NativeSyntheticEvent): void { + if (this.props.onFrameProcessorPerformanceSuggestionAvailable != null) + this.props.onFrameProcessorPerformanceSuggestionAvailable(event.nativeEvent); + } //#endregion //#region Lifecycle @@ -396,14 +408,16 @@ export class Camera extends React.PureComponent { /** @internal */ public render(): React.ReactNode { // We remove the big `device` object from the props because we only need to pass `cameraId` to native. - const { device, frameProcessor, ...props } = this.props; + const { device, frameProcessor, frameProcessorFps, ...props } = this.props; return ( diff --git a/src/CameraProps.ts b/src/CameraProps.ts index 984a003..b8c60c2 100644 --- a/src/CameraProps.ts +++ b/src/CameraProps.ts @@ -4,6 +4,11 @@ import type { CameraRuntimeError } from './CameraError'; import type { CameraPreset } from './CameraPreset'; import type { Frame } from './Frame'; +export interface FrameProcessorPerformanceSuggestion { + type: 'can-use-higher-fps' | 'should-use-lower-fps'; + suggestedFrameProcessorFps: number; +} + export interface CameraProps extends ViewProps { /** * The Camera Device to use. @@ -161,6 +166,10 @@ export interface CameraProps extends ViewProps { * Called when the camera was successfully initialized. */ onInitialized?: () => void; + /** + * Called when a new performance suggestion for a Frame Processor is available - either if your Frame Processor is running too fast and frames are being dropped, or because it is able to run faster. Optionally, you can adjust your `frameProcessorFps` accordingly. + */ + onFrameProcessorPerformanceSuggestionAvailable?: (suggestion: FrameProcessorPerformanceSuggestion) => void; /** * A worklet which will be called for every frame the Camera "sees". Throttle the Frame Processor's frame rate with {@linkcode frameProcessorFps}. * @@ -183,7 +192,8 @@ export interface CameraProps extends ViewProps { /** * Specifies the maximum frame rate the frame processor can use, independent of the Camera's frame rate (`fps` property). * - * * A value of `1` (default) indicates that the frame processor gets executed once per second, perfect for code scanning. + * * A value of `'auto'` (default) indicates that the frame processor should execute as fast as it can, without dropping frames. This is achieved by collecting historical data for previous frame processor calls and adjusting frame rate accordingly. + * * A value of `1` indicates that the frame processor gets executed once per second, perfect for code scanning. * * A value of `10` indicates that the frame processor gets executed 10 times per second, perfect for more realtime use-cases. * * A value of `25` indicates that the frame processor gets executed 25 times per second, perfect for high-speed realtime use-cases. * * ...and so on @@ -191,8 +201,8 @@ export interface CameraProps extends ViewProps { * If you're using higher values, always check your Xcode/Android Studio Logs to make sure your frame processors are executing fast enough * without blocking the video recording queue. * - * @default 1 + * @default 'auto' */ - frameProcessorFps?: number; + frameProcessorFps?: number | 'auto'; //#endregion } diff --git a/yarn.lock b/yarn.lock index 2597e32..bd9bbf0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6282,7 +6282,7 @@ react-native-codegen@^0.0.7: jscodeshift "^0.11.0" nullthrows "^1.1.1" -react-native-reanimated@^2.3.0-alpha.2: +react-native-reanimated@^2.3.0-alpha.3: version "2.3.0-alpha.3" resolved "https://registry.yarnpkg.com/react-native-reanimated/-/react-native-reanimated-2.3.0-alpha.3.tgz#022f5b4eaa847304a155942165a77125c86428bc" integrity sha512-Ln+edkTrepKUETKKmGl+GxL8RiB7awBDQFcRJIK39m5ggiLfksOAfqzclpKPLWbAfOvB4cSg+QdowzRN570FXA==