feat: frameProcessorFps="auto" and automatic performance suggestions (throttle or increase FPS) (#393)

* Add `onFrameProcessorPerformanceSuggestionAvailable` and make `frameProcessorFps` support `auto`

* Implement performance suggestion and auto-adjusting

* Fix FPS setting, evaluate correctly

* Floor suggested FPS

* Remove `console.log` for frame drop warnings.

* Swift format

* Use `30` magic number

* only call if FPS is different

* Update CameraView.swift

* Implement Android 1/2

* Cleanup

* Update `frameProcessorFps` if available

* Optimize `FrameProcessorPerformanceDataCollector` initialization

* Cache call

* Set frameProcessorFps directly (Kotlin setter)

* Don't suggest if same value

* Call suggestion every second

* reset time on set

* Always store 15 last samples

* reset counter too

* Update FrameProcessorPerformanceDataCollector.swift

* Update CameraView+RecordVideo.swift

* Update CameraView.kt

* iOS: Redesign evaluation

* Update CameraView+RecordVideo.swift

* Android: Redesign evaluation

* Update CameraView.kt

* Update REA to latest alpha and install RNScreens

* Fix frameProcessorFps updating

android/src/main/java/com/mrousavy/camera/CameraView+Events.kt

@@ -0,0 +1,53 @@
+package com.mrousavy.camera
+
+import android.util.Log
+import com.facebook.react.bridge.Arguments
+import com.facebook.react.bridge.ReactContext
+import com.facebook.react.bridge.WritableMap
+import com.facebook.react.uimanager.events.RCTEventEmitter
+
+fun CameraView.invokeOnInitialized() {
+  Log.i(CameraView.TAG, "invokeOnInitialized()")
+
+  val reactContext = context as ReactContext
+  reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraInitialized", null)
+}
+
+fun CameraView.invokeOnError(error: Throwable) {
+  Log.e(CameraView.TAG, "invokeOnError(...):")
+  error.printStackTrace()
+
+  val cameraError = when (error) {
+    is CameraError -> error
+    else -> UnknownCameraError(error)
+  }
+  val event = Arguments.createMap()
+  event.putString("code", cameraError.code)
+  event.putString("message", cameraError.message)
+  cameraError.cause?.let { cause ->
+    event.putMap("cause", errorToMap(cause))
+  }
+  val reactContext = context as ReactContext
+  reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraError", event)
+}
+
+fun CameraView.invokeOnFrameProcessorPerformanceSuggestionAvailable(currentFps: Double, suggestedFps: Double) {
+  Log.e(CameraView.TAG, "invokeOnFrameProcessorPerformanceSuggestionAvailable(suggestedFps: $suggestedFps):")
+
+  val event = Arguments.createMap()
+  val type = if (suggestedFps > currentFps) "can-use-higher-fps" else "should-use-lower-fps"
+  event.putString("type", type)
+  event.putDouble("suggestedFrameProcessorFps", suggestedFps)
+  val reactContext = context as ReactContext
+  reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraPerformanceSuggestionAvailable", event)
+}
+
+private fun errorToMap(error: Throwable): WritableMap {
+  val map = Arguments.createMap()
+  map.putString("message", error.message)
+  map.putString("stacktrace", error.stackTraceToString())
+  error.cause?.let { cause ->
+    map.putMap("cause", errorToMap(cause))
+  }
+  return map
+}

android/src/main/java/com/mrousavy/camera/CameraView.kt

@@ -23,12 +23,14 @@ import com.facebook.jni.HybridData
 import com.facebook.proguard.annotations.DoNotStrip
 import com.facebook.react.bridge.*
 import com.facebook.react.uimanager.events.RCTEventEmitter
+import com.mrousavy.camera.frameprocessor.FrameProcessorPerformanceDataCollector
 import com.mrousavy.camera.utils.*
 import kotlinx.coroutines.*
 import kotlinx.coroutines.guava.await
 import java.lang.IllegalArgumentException
 import java.util.concurrent.ExecutorService
 import java.util.concurrent.Executors
+import kotlin.math.floor
 import kotlin.math.max
 import kotlin.math.min
 
@@ -93,7 +95,17 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
   var torch = "off"
   var zoom: Float = 1f // in "factor"
   var enableZoomGesture = false
+    set(value) {
+      field = value
+      setOnTouchListener(if (value) touchEventListener else null)
+    }
   var frameProcessorFps = 1.0
+    set(value) {
+      field = value
+      actualFrameProcessorFps = if (value == -1.0) 30.0 else value
+      lastFrameProcessorPerformanceEvaluation = System.currentTimeMillis()
+      frameProcessorPerformanceDataCollector.clear()
+    }
 
   // private properties
   private val reactContext: ReactContext
@@ -131,6 +143,16 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
   private var minZoom: Float = 1f
   private var maxZoom: Float = 1f
 
+  private var actualFrameProcessorFps = 30.0
+  private val frameProcessorPerformanceDataCollector = FrameProcessorPerformanceDataCollector()
+  private var lastSuggestedFrameProcessorFps = 0.0
+  private var lastFrameProcessorPerformanceEvaluation = System.currentTimeMillis()
+  private val isReadyForNewEvaluation: Boolean
+    get() {
+      val lastPerformanceEvaluationElapsedTime = System.currentTimeMillis() - lastFrameProcessorPerformanceEvaluation
+      return lastPerformanceEvaluationElapsedTime > 1000
+    }
+
   @DoNotStrip
   private var mHybridData: HybridData
 
@@ -277,9 +299,6 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
         if (shouldReconfigureTorch) {
           camera!!.cameraControl.enableTorch(torch == "on")
         }
-        if (changedProps.contains("enableZoomGesture")) {
-          setOnTouchListener(if (enableZoomGesture) touchEventListener else null)
-        }
       } catch (e: Throwable) {
         Log.e(TAG, "update() threw: ${e.message}")
         invokeOnError(e)
@@ -406,12 +425,20 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
         imageAnalysis = imageAnalysisBuilder.build().apply {
           setAnalyzer(cameraExecutor, { image ->
             val now = System.currentTimeMillis()
-            val intervalMs = (1.0 / frameProcessorFps) * 1000.0
+            val intervalMs = (1.0 / actualFrameProcessorFps) * 1000.0
             if (now - lastFrameProcessorCall > intervalMs) {
               lastFrameProcessorCall = now
+
+              val perfSample = frameProcessorPerformanceDataCollector.beginPerformanceSampleCollection()
               frameProcessorCallback(image)
+              perfSample.endPerformanceSampleCollection()
             }
             image.close()
+
+            if (isReadyForNewEvaluation) {
+              // last evaluation was more than a second ago, evaluate again
+              evaluateNewPerformanceSamples()
+            }
           })
         }
         useCases.add(imageAnalysis!!)
@@ -444,38 +471,21 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
     }
   }
 
-  private fun invokeOnInitialized() {
+  private fun evaluateNewPerformanceSamples() {
-    Log.i(TAG, "invokeOnInitialized()")
+    lastFrameProcessorPerformanceEvaluation = System.currentTimeMillis()
+    val maxFrameProcessorFps = 30 // TODO: Get maxFrameProcessorFps from ImageAnalyser
+    val averageFps = 1.0 / frameProcessorPerformanceDataCollector.averageExecutionTimeSeconds
+    val suggestedFrameProcessorFps = floor(min(averageFps, maxFrameProcessorFps.toDouble()))
 
-    val reactContext = context as ReactContext
+    if (frameProcessorFps == -1.0) {
-    reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraInitialized", null)
+      // frameProcessorFps="auto"
-  }
+      actualFrameProcessorFps = suggestedFrameProcessorFps
-
+    } else {
-  private fun invokeOnError(error: Throwable) {
+      // frameProcessorFps={someCustomFpsValue}
-    Log.e(TAG, "invokeOnError(...):")
+      if (suggestedFrameProcessorFps != lastSuggestedFrameProcessorFps && suggestedFrameProcessorFps != frameProcessorFps) {
-    error.printStackTrace()
+        invokeOnFrameProcessorPerformanceSuggestionAvailable(frameProcessorFps, suggestedFrameProcessorFps)
-
+        lastSuggestedFrameProcessorFps = suggestedFrameProcessorFps
-    val cameraError = when (error) {
+      }
-      is CameraError -> error
-      else -> UnknownCameraError(error)
     }
-    val event = Arguments.createMap()
-    event.putString("code", cameraError.code)
-    event.putString("message", cameraError.message)
-    cameraError.cause?.let { cause ->
-      event.putMap("cause", errorToMap(cause))
-    }
-    val reactContext = context as ReactContext
-    reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraError", event)
-  }
-
-  private fun errorToMap(error: Throwable): WritableMap {
-    val map = Arguments.createMap()
-    map.putString("message", error.message)
-    map.putString("stacktrace", error.stackTraceToString())
-    error.cause?.let { cause ->
-      map.putMap("cause", errorToMap(cause))
-    }
-    return map
   }
 }

android/src/main/java/com/mrousavy/camera/CameraViewManager.kt

@@ -57,6 +57,7 @@ class CameraViewManager(reactContext: ReactApplicationContext) : SimpleViewManag
     return MapBuilder.builder<String, Any>()
       .put("cameraInitialized", MapBuilder.of("registrationName", "onInitialized"))
       .put("cameraError", MapBuilder.of("registrationName", "onError"))
+      .put("cameraPerformanceSuggestionAvailable", MapBuilder.of("registrationName", "onFrameProcessorPerformanceSuggestionAvailable"))
       .build()
   }
 

android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPerformanceDataCollector.kt

@@ -0,0 +1,38 @@
+package com.mrousavy.camera.frameprocessor
+
+data class PerformanceSampleCollection(val endPerformanceSampleCollection: () -> Unit)
+
+// keep a maximum of `maxSampleSize` historical performance data samples cached.
+private const val maxSampleSize = 15
+
+class FrameProcessorPerformanceDataCollector {
+  private var counter = 0
+  private var performanceSamples: ArrayList<Double> = ArrayList()
+
+  val averageExecutionTimeSeconds: Double
+    get() = performanceSamples.average()
+
+  fun beginPerformanceSampleCollection(): PerformanceSampleCollection {
+    val begin = System.currentTimeMillis()
+
+    return PerformanceSampleCollection {
+      val end = System.currentTimeMillis()
+      val seconds = (end - begin) / 1_000.0
+
+      val index = counter % maxSampleSize
+
+      if (performanceSamples.size > index) {
+        performanceSamples[index] = seconds
+      } else {
+        performanceSamples.add(seconds)
+      }
+
+      counter++
+    }
+  }
+
+  fun clear() {
+    counter = 0
+    performanceSamples.clear()
+  }
+}

example/ios/Podfile.lock

@@ -321,6 +321,9 @@ PODS:
     - React-RCTVibration
     - ReactCommon/turbomodule/core
     - Yoga
+  - RNScreens (3.6.0):
+    - React-Core
+    - React-RCTImage
   - RNStaticSafeAreaInsets (2.1.1):
     - React
   - RNVectorIcons (8.1.0):
@@ -368,6 +371,7 @@ DEPENDENCIES:
   - ReactNativeNavigation (from `../node_modules/react-native-navigation`)
   - RNGestureHandler (from `../node_modules/react-native-gesture-handler`)
   - RNReanimated (from `../node_modules/react-native-reanimated`)
+  - RNScreens (from `../node_modules/react-native-screens`)
   - RNStaticSafeAreaInsets (from `../node_modules/react-native-static-safe-area-insets`)
   - RNVectorIcons (from `../node_modules/react-native-vector-icons`)
   - VisionCamera (from `../..`)
@@ -447,6 +451,8 @@ EXTERNAL SOURCES:
     :path: "../node_modules/react-native-gesture-handler"
   RNReanimated:
     :path: "../node_modules/react-native-reanimated"
+  RNScreens:
+    :path: "../node_modules/react-native-screens"
   RNStaticSafeAreaInsets:
     :path: "../node_modules/react-native-static-safe-area-insets"
   RNVectorIcons:
@@ -493,6 +499,7 @@ SPEC CHECKSUMS:
   ReactNativeNavigation: 87aa2e3a749a9b338057e8d53af54d865241b843
   RNGestureHandler: a479ebd5ed4221a810967000735517df0d2db211
   RNReanimated: 48e578538b2fad573d3d5ce2e80ad375e1534d87
+  RNScreens: eb0dfb2d6b21d2d7f980ad46b14eb306d2f1062e
   RNStaticSafeAreaInsets: 6103cf09647fa427186d30f67b0f5163c1ae8252
   RNVectorIcons: 31cebfcf94e8cf8686eb5303ae0357da64d7a5a4
   VisionCamera: dc481620431e31c1d6bbbc438d8a1d9c56f3d304

example/package.json

@@ -23,7 +23,8 @@
     "react-native-gesture-handler": "^1.10.3",
     "react-native-navigation": "^7.19.0",
     "react-native-pressable-opacity": "^1.0.4",
-    "react-native-reanimated": "^2.3.0-alpha.2",
+    "react-native-reanimated": "^2.3.0-alpha.3",
+    "react-native-screens": "^3.6.0",
     "react-native-static-safe-area-insets": "^2.1.1",
     "react-native-vector-icons": "^8.0.0",
     "react-native-video": "^5.1.1"

example/src/CameraPage.tsx

@@ -6,6 +6,7 @@ import { Navigation, NavigationFunctionComponent } from 'react-native-navigation
 import {
   CameraDeviceFormat,
   CameraRuntimeError,
+  FrameProcessorPerformanceSuggestion,
   PhotoFile,
   sortFormats,
   useCameraDevices,
@@ -197,6 +198,10 @@ export const CameraPage: NavigationFunctionComponent = ({ componentId }) => {
     console.log(`Return Values: ${JSON.stringify(values)}`);
   }, []);
 
+  const onFrameProcessorSuggestionAvailable = useCallback((suggestion: FrameProcessorPerformanceSuggestion) => {
+    console.log(`Suggestion available! ${suggestion.type}: Can do ${suggestion.suggestedFrameProcessorFps} FPS`);
+  }, []);
+
   return (
     <View style={styles.container}>
       {device != null && (
@@ -221,6 +226,7 @@ export const CameraPage: NavigationFunctionComponent = ({ componentId }) => {
                 audio={true}
                 frameProcessor={device.supportsParallelVideoProcessing ? frameProcessor : undefined}
                 frameProcessorFps={1}
+                onFrameProcessorPerformanceSuggestionAvailable={onFrameProcessorSuggestionAvailable}
               />
             </TapGestureHandler>
           </Reanimated.View>

example/yarn.lock

@@ -4401,7 +4401,7 @@ react-native-pressable-opacity@^1.0.4:
   resolved "https://registry.yarnpkg.com/react-native-pressable-opacity/-/react-native-pressable-opacity-1.0.4.tgz#391f33fdc25cb84551f2743a25eced892b9f30f7"
   integrity sha512-DBIg7UoRiuBYiFEvx+XNMqH0OEx64WrSksXhT6Kq9XuyyKsThMNDqZ9G5QV7vfu7dU2/IctwIz5c0Xwkp4K3tA==
 
-react-native-reanimated@^2.3.0-alpha.2:
+react-native-reanimated@^2.3.0-alpha.3:
   version "2.3.0-alpha.3"
   resolved "https://registry.yarnpkg.com/react-native-reanimated/-/react-native-reanimated-2.3.0-alpha.3.tgz#022f5b4eaa847304a155942165a77125c86428bc"
   integrity sha512-Ln+edkTrepKUETKKmGl+GxL8RiB7awBDQFcRJIK39m5ggiLfksOAfqzclpKPLWbAfOvB4cSg+QdowzRN570FXA==
@@ -4413,7 +4413,7 @@ react-native-reanimated@^2.3.0-alpha.2:
     react-native-screens "^3.4.0"
     string-hash-64 "^1.0.3"
 
-react-native-screens@^3.4.0:
+react-native-screens@^3.4.0, react-native-screens@^3.6.0:
   version "3.6.0"
   resolved "https://registry.yarnpkg.com/react-native-screens/-/react-native-screens-3.6.0.tgz#054af728e50c06bff6b3b4fa7b4b656b70f247cd"
   integrity sha512-emQmSu+B6cOIjJH2OIgpuxd9zCD6xz7/oo5GCetyjsM5qR3sMnVgOxqtK99xPu9XJH/8k7MplXbtJgtk/PHXwA==

ios/CameraView+AVAudioSession.swift

@@ -118,10 +118,11 @@ extension CameraView {
       return
     }
 
+    // TODO: Add JS-Event for Audio Session interruptions?
     switch type {
     case .began:
       // Something interrupted our Audio Session, stop recording audio.
-      ReactLogger.log(level: .error, message: "The Audio Session was interrupted!", alsoLogToJS: true)
+      ReactLogger.log(level: .error, message: "The Audio Session was interrupted!")
     case .ended:
       ReactLogger.log(level: .info, message: "The Audio Session interruption has ended.")
       guard let optionsValue = userInfo[AVAudioSessionInterruptionOptionKey] as? UInt else { return }
@@ -129,13 +130,13 @@ extension CameraView {
       if options.contains(.shouldResume) {
         if isRecording {
           audioQueue.async {
-            ReactLogger.log(level: .info, message: "Resuming interrupted Audio Session...", alsoLogToJS: true)
+            ReactLogger.log(level: .info, message: "Resuming interrupted Audio Session...")
             // restart audio session because interruption is over
             self.activateAudioSession()
           }
         }
       } else {
-        ReactLogger.log(level: .error, message: "Cannot resume interrupted Audio Session!", alsoLogToJS: true)
+        ReactLogger.log(level: .error, message: "Cannot resume interrupted Audio Session!")
       }
     @unknown default: ()
     }

ios/CameraView+AVCaptureSession.swift

@@ -255,7 +255,7 @@ extension CameraView {
 
   @objc
   func sessionRuntimeError(notification: Notification) {
-    ReactLogger.log(level: .error, message: "Unexpected Camera Runtime Error occured!", alsoLogToJS: true)
+    ReactLogger.log(level: .error, message: "Unexpected Camera Runtime Error occured!")
     guard let error = notification.userInfo?[AVCaptureSessionErrorKey] as? AVError else {
       return
     }

ios/CameraView+RecordVideo.swift

@@ -8,9 +8,6 @@
 
 import AVFoundation
 
-private var hasLoggedVideoFrameDropWarning = false
-private var hasLoggedFrameProcessorFrameDropWarning = false
-
 // MARK: - CameraView + AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAudioDataOutputSampleBufferDelegate
 
 extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAudioDataOutputSampleBufferDelegate {
@@ -203,63 +200,61 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud
       }
     }
 
-    // TODO: resize using VideoToolbox (VTPixelTransferSession)
+    if let frameProcessor = frameProcessorCallback, captureOutput is AVCaptureVideoDataOutput {
-
-    if let frameProcessor = frameProcessorCallback,
-       captureOutput is AVCaptureVideoDataOutput {
       // check if last frame was x nanoseconds ago, effectively throttling FPS
-      let diff = DispatchTime.now().uptimeNanoseconds - lastFrameProcessorCall.uptimeNanoseconds
+      let lastFrameProcessorCallElapsedTime = DispatchTime.now().uptimeNanoseconds - lastFrameProcessorCall.uptimeNanoseconds
-      let secondsPerFrame = 1.0 / frameProcessorFps.doubleValue
+      let secondsPerFrame = 1.0 / actualFrameProcessorFps
       let nanosecondsPerFrame = secondsPerFrame * 1_000_000_000.0
 
-      if diff > UInt64(nanosecondsPerFrame) {
+      if lastFrameProcessorCallElapsedTime > UInt64(nanosecondsPerFrame) {
         if !isRunningFrameProcessor {
           // we're not in the middle of executing the Frame Processor, so prepare for next call.
           CameraQueues.frameProcessorQueue.async {
             self.isRunningFrameProcessor = true
+
+            let perfSample = self.frameProcessorPerformanceDataCollector.beginPerformanceSampleCollection()
             let frame = Frame(buffer: sampleBuffer, orientation: self.bufferOrientation)
             frameProcessor(frame)
+            perfSample.endPerformanceSampleCollection()
+
             self.isRunningFrameProcessor = false
           }
           lastFrameProcessorCall = DispatchTime.now()
         } else {
-          // we're still in the middle of executing a Frame Processor for a previous frame, notify user about dropped frame.
+          // we're still in the middle of executing a Frame Processor for a previous frame, so a frame was dropped.
-          if !hasLoggedFrameProcessorFrameDropWarning {
+          ReactLogger.log(level: .warning, message: "The Frame Processor took so long to execute that a frame was dropped.")
-            ReactLogger.log(level: .warning,
-                            message: "Your Frame Processor took so long to execute that a frame was dropped. " +
-                              "Either throttle your Frame Processor's frame rate using the `frameProcessorFps` prop, or optimize " +
-                              "it's execution speed. (This warning will only be shown once)",
-                            alsoLogToJS: true)
-            hasLoggedFrameProcessorFrameDropWarning = true
-          }
         }
       }
+
+      if isReadyForNewEvaluation {
+        // last evaluation was more than 1sec ago, evaluate again
+        evaluateNewPerformanceSamples()
+      }
     }
   }
 
-  #if DEBUG
+  private func evaluateNewPerformanceSamples() {
-    public final func captureOutput(_ captureOutput: AVCaptureOutput, didDrop buffer: CMSampleBuffer, from _: AVCaptureConnection) {
+    lastFrameProcessorPerformanceEvaluation = DispatchTime.now()
-      if !hasLoggedVideoFrameDropWarning && captureOutput is AVCaptureVideoDataOutput {
+    guard let videoDevice = videoDeviceInput?.device else { return }
-        let reason = findFrameDropReason(inBuffer: buffer)
-        ReactLogger.log(level: .warning,
-                        message: "Dropped a Frame - This might indicate that your frame rate is higher than the phone can currently process. " +
-                          "Throttle the Camera frame rate using the `fps` prop and make sure the device stays in optimal condition for recording. " +
-                          "Frame drop reason: \(reason). (This warning will only be shown once)",
-                        alsoLogToJS: true)
-        hasLoggedVideoFrameDropWarning = true
-      }
-    }
 
-    private final func findFrameDropReason(inBuffer buffer: CMSampleBuffer) -> String {
+    let maxFrameProcessorFps = Double(videoDevice.activeVideoMinFrameDuration.timescale) * Double(videoDevice.activeVideoMinFrameDuration.value)
-      var mode: CMAttachmentMode = 0
+    let averageFps = 1.0 / frameProcessorPerformanceDataCollector.averageExecutionTimeSeconds
-      guard let reason = CMGetAttachment(buffer,
+    let suggestedFrameProcessorFps = floor(min(averageFps, maxFrameProcessorFps))
-                                         key: kCMSampleBufferAttachmentKey_DroppedFrameReason,
+
-                                         attachmentModeOut: &mode) else {
+    if frameProcessorFps.intValue == -1 {
-        return "unknown"
+      // frameProcessorFps="auto"
-      }
+      actualFrameProcessorFps = suggestedFrameProcessorFps
-      return String(describing: reason)
+    } else {
+      // frameProcessorFps={someCustomFpsValue}
+      invokeOnFrameProcessorPerformanceSuggestionAvailable(currentFps: frameProcessorFps.doubleValue,
+                                                           suggestedFps: suggestedFrameProcessorFps)
     }
-  #endif
+  }
+
+  private var isReadyForNewEvaluation: Bool {
+    let lastPerformanceEvaluationElapsedTime = DispatchTime.now().uptimeNanoseconds - lastFrameProcessorPerformanceEvaluation.uptimeNanoseconds
+    return lastPerformanceEvaluationElapsedTime > 1_000_000_000
+  }
 
   /**
    Gets the orientation of the CameraView's images (CMSampleBuffers).

ios/CameraView.swift

@@ -52,7 +52,7 @@ public final class CameraView: UIView {
   // props that require format reconfiguring
   @objc var format: NSDictionary?
   @objc var fps: NSNumber?
-  @objc var frameProcessorFps: NSNumber = 1.0
+  @objc var frameProcessorFps: NSNumber = -1.0 // "auto"
   @objc var hdr: NSNumber? // nullable bool
   @objc var lowLightBoost: NSNumber? // nullable bool
   @objc var colorSpace: NSString?
@@ -64,6 +64,7 @@ public final class CameraView: UIView {
   // events
   @objc var onInitialized: RCTDirectEventBlock?
   @objc var onError: RCTDirectEventBlock?
+  @objc var onFrameProcessorPerformanceSuggestionAvailable: RCTDirectEventBlock?
   // zoom
   @objc var enableZoomGesture = false {
     didSet {
@@ -104,6 +105,10 @@ public final class CameraView: UIView {
 
   /// Specifies whether the frameProcessor() function is currently executing. used to drop late frames.
   internal var isRunningFrameProcessor = false
+  internal let frameProcessorPerformanceDataCollector = FrameProcessorPerformanceDataCollector()
+  internal var actualFrameProcessorFps = 30.0
+  internal var lastSuggestedFrameProcessorFps = 0.0
+  internal var lastFrameProcessorPerformanceEvaluation = DispatchTime.now()
 
   /// Returns whether the AVCaptureSession is currently running (reflected by isActive)
   var isRunning: Bool {
@@ -244,6 +249,18 @@ public final class CameraView: UIView {
         }
       }
     }
+
+    // Frame Processor FPS Configuration
+    if changedProps.contains("frameProcessorFps") {
+      if frameProcessorFps.doubleValue == -1 {
+        // "auto"
+        actualFrameProcessorFps = 30.0
+      } else {
+        actualFrameProcessorFps = frameProcessorFps.doubleValue
+      }
+      lastFrameProcessorPerformanceEvaluation = DispatchTime.now()
+      frameProcessorPerformanceDataCollector.clear()
+    }
   }
 
   internal final func setTorchMode(_ torchMode: String) {
@@ -336,4 +353,18 @@ public final class CameraView: UIView {
     guard let onInitialized = self.onInitialized else { return }
     onInitialized([String: Any]())
   }
+
+  internal final func invokeOnFrameProcessorPerformanceSuggestionAvailable(currentFps: Double, suggestedFps: Double) {
+    ReactLogger.log(level: .info, message: "Frame Processor Performance Suggestion available!")
+    guard let onFrameProcessorPerformanceSuggestionAvailable = self.onFrameProcessorPerformanceSuggestionAvailable else { return }
+
+    if lastSuggestedFrameProcessorFps == suggestedFps { return }
+    if suggestedFps == currentFps { return }
+
+    onFrameProcessorPerformanceSuggestionAvailable([
+      "type": suggestedFps > currentFps ? "can-use-higher-fps" : "should-use-lower-fps",
+      "suggestedFrameProcessorFps": suggestedFps,
+    ])
+    lastSuggestedFrameProcessorFps = suggestedFps
+  }
 }

ios/CameraViewManager.m

@@ -48,6 +48,7 @@ RCT_EXPORT_VIEW_PROPERTY(enableZoomGesture, BOOL);
 // Camera View Properties
 RCT_EXPORT_VIEW_PROPERTY(onError, RCTDirectEventBlock);
 RCT_EXPORT_VIEW_PROPERTY(onInitialized, RCTDirectEventBlock);
+RCT_EXPORT_VIEW_PROPERTY(onFrameProcessorPerformanceSuggestionAvailable, RCTDirectEventBlock);
 
 // Camera View Functions
 RCT_EXTERN_METHOD(startRecording:(nonnull NSNumber *)node options:(NSDictionary *)options onRecordCallback:(RCTResponseSenderBlock)onRecordCallback);

ios/CameraViewManager.swift

@@ -17,11 +17,6 @@ final class CameraViewManager: RCTViewManager {
 
   override var bridge: RCTBridge! {
     didSet {
-      #if DEBUG
-        // Install console.log bindings
-        ReactLogger.ConsoleLogFunction = JSConsoleHelper.getLogFunction(for: bridge)
-      #endif
-
       // Install Frame Processor bindings and setup Runtime
       if VISION_CAMERA_ENABLE_FRAME_PROCESSORS {
         CameraQueues.frameProcessorQueue.async {

ios/Extensions/AVAudioSession+updateCategory.swift

@@ -16,8 +16,7 @@ extension AVAudioSession {
   func updateCategory(_ category: AVAudioSession.Category, options: AVAudioSession.CategoryOptions = []) throws {
     if self.category != category || categoryOptions.rawValue != options.rawValue {
       ReactLogger.log(level: .info,
-                      message: "Changing AVAudioSession category from \(self.category.rawValue) -> \(category.rawValue)",
+                      message: "Changing AVAudioSession category from \(self.category.rawValue) -> \(category.rawValue)")
-                      alsoLogToJS: true)
       try setCategory(category, options: options)
     }
   }

ios/Frame Processor/FrameProcessorPerformanceDataCollector.swift

@@ -0,0 +1,63 @@
+//
+//  FrameProcessorPerformanceDataCollector.swift
+//  VisionCamera
+//
+//  Created by Marc Rousavy on 30.08.21.
+//  Copyright © 2021 mrousavy. All rights reserved.
+//
+
+import Foundation
+
+// keep a maximum of `maxSampleSize` historical performance data samples cached.
+private let maxSampleSize = 15
+
+// MARK: - PerformanceSampleCollection
+
+struct PerformanceSampleCollection {
+  var endPerformanceSampleCollection: () -> Void
+
+  init(end: @escaping () -> Void) {
+    endPerformanceSampleCollection = end
+  }
+}
+
+// MARK: - FrameProcessorPerformanceDataCollector
+
+class FrameProcessorPerformanceDataCollector {
+  private var performanceSamples: [Double] = []
+  private var counter = 0
+  private var lastEvaluation = -1
+
+  var averageExecutionTimeSeconds: Double {
+    let sum = performanceSamples.reduce(0, +)
+    let average = sum / Double(performanceSamples.count)
+
+    lastEvaluation = counter
+
+    return average
+  }
+
+  func beginPerformanceSampleCollection() -> PerformanceSampleCollection {
+    let begin = DispatchTime.now()
+
+    return PerformanceSampleCollection {
+      let end = DispatchTime.now()
+      let seconds = Double(end.uptimeNanoseconds - begin.uptimeNanoseconds) / 1_000_000_000.0
+
+      let index = self.counter % maxSampleSize
+
+      if self.performanceSamples.count > index {
+        self.performanceSamples[index] = seconds
+      } else {
+        self.performanceSamples.append(seconds)
+      }
+
+      self.counter += 1
+    }
+  }
+
+  func clear() {
+    counter = 0
+    performanceSamples.removeAll()
+  }
+}

ios/React Utils/ReactLogger.swift

@@ -11,11 +11,6 @@ import Foundation
 // MARK: - ReactLogger
 
 enum ReactLogger {
-  /**
-   A function that logs to the JavaScript console.
-   */
-  static var ConsoleLogFunction: ConsoleLogFunction?
-
   /**
    Log a message to the console in the format of `VisionCamera.[caller-function-name]: [message]`
 
@@ -27,14 +22,10 @@ enum ReactLogger {
   @inlinable
   static func log(level: RCTLogLevel,
                   message: String,
-                  alsoLogToJS: Bool = false,
                   _ file: String = #file,
                   _ lineNumber: Int = #line,
                   _ function: String = #function) {
     #if DEBUG
-      if alsoLogToJS, let log = ConsoleLogFunction {
-        log(level, "[native] VisionCamera.\(function): \(message)")
-      }
       RCTDefaultLogFunction(level, RCTLogSource.native, file, lineNumber as NSNumber, "VisionCamera.\(function): \(message)")
     #endif
   }

ios/RecordingSession.swift

@@ -60,7 +60,7 @@ class RecordingSession {
 
   deinit {
     if assetWriter.status == .writing {
-      ReactLogger.log(level: .info, message: "Cancelling AssetWriter...", alsoLogToJS: true)
+      ReactLogger.log(level: .info, message: "Cancelling AssetWriter...")
       assetWriter.cancelWriting()
     }
   }
@@ -70,11 +70,11 @@ class RecordingSession {
    */
   func initializeVideoWriter(withSettings settings: [String: Any], pixelFormat: OSType) {
     guard !settings.isEmpty else {
-      ReactLogger.log(level: .error, message: "Tried to initialize Video Writer with empty settings!", alsoLogToJS: true)
+      ReactLogger.log(level: .error, message: "Tried to initialize Video Writer with empty settings!")
       return
     }
     guard bufferAdaptor == nil else {
-      ReactLogger.log(level: .error, message: "Tried to add Video Writer twice!", alsoLogToJS: true)
+      ReactLogger.log(level: .error, message: "Tried to add Video Writer twice!")
       return
     }
 
@@ -93,11 +93,11 @@ class RecordingSession {
    */
   func initializeAudioWriter(withSettings settings: [String: Any]) {
     guard !settings.isEmpty else {
-      ReactLogger.log(level: .error, message: "Tried to initialize Audio Writer with empty settings!", alsoLogToJS: true)
+      ReactLogger.log(level: .error, message: "Tried to initialize Audio Writer with empty settings!")
       return
     }
     guard audioWriter == nil else {
-      ReactLogger.log(level: .error, message: "Tried to add Audio Writer twice!", alsoLogToJS: true)
+      ReactLogger.log(level: .error, message: "Tried to add Audio Writer twice!")
       return
     }
 
@@ -139,8 +139,7 @@ class RecordingSession {
     }
     guard let initialTimestamp = initialTimestamp else {
       ReactLogger.log(level: .error,
-                      message: "A frame arrived, but initialTimestamp was nil. Is this RecordingSession running?",
+                      message: "A frame arrived, but initialTimestamp was nil. Is this RecordingSession running?")
-                      alsoLogToJS: true)
       return
     }
 
@@ -149,17 +148,16 @@ class RecordingSession {
     switch bufferType {
     case .video:
       guard let bufferAdaptor = bufferAdaptor else {
-        ReactLogger.log(level: .error, message: "Video Frame arrived but VideoWriter was nil!", alsoLogToJS: true)
+        ReactLogger.log(level: .error, message: "Video Frame arrived but VideoWriter was nil!")
         return
       }
       if !bufferAdaptor.assetWriterInput.isReadyForMoreMediaData {
         ReactLogger.log(level: .warning,
-                        message: "The Video AVAssetWriterInput was not ready for more data! Is your frame rate too high?",
+                        message: "The Video AVAssetWriterInput was not ready for more data! Is your frame rate too high?")
-                        alsoLogToJS: true)
         return
       }
       guard let imageBuffer = CMSampleBufferGetImageBuffer(buffer) else {
-        ReactLogger.log(level: .error, message: "Failed to get the CVImageBuffer!", alsoLogToJS: true)
+        ReactLogger.log(level: .error, message: "Failed to get the CVImageBuffer!")
         return
       }
       bufferAdaptor.append(imageBuffer, withPresentationTime: timestamp)
@@ -169,7 +167,7 @@ class RecordingSession {
       }
     case .audio:
       guard let audioWriter = audioWriter else {
-        ReactLogger.log(level: .error, message: "Audio Frame arrived but AudioWriter was nil!", alsoLogToJS: true)
+        ReactLogger.log(level: .error, message: "Audio Frame arrived but AudioWriter was nil!")
         return
       }
       if !audioWriter.isReadyForMoreMediaData {
@@ -184,8 +182,7 @@ class RecordingSession {
 
     if assetWriter.status == .failed {
       ReactLogger.log(level: .error,
-                      message: "AssetWriter failed to write buffer! Error: \(assetWriter.error?.localizedDescription ?? "none")",
+                      message: "AssetWriter failed to write buffer! Error: \(assetWriter.error?.localizedDescription ?? "none")")
-                      alsoLogToJS: true)
       finish()
     }
   }

ios/VisionCamera.xcodeproj/project.pbxproj

@@ -135,6 +135,7 @@
 		B887518425E0102000DB86D6 /* CameraView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CameraView.swift; sourceTree = "<group>"; };
 		B88873E5263D46C7008B1D0E /* FrameProcessorPlugin.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorPlugin.h; sourceTree = "<group>"; };
 		B88B47462667C8E00091F538 /* AVCaptureSession+setVideoStabilizationMode.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVCaptureSession+setVideoStabilizationMode.swift"; sourceTree = "<group>"; };
+		B8948BDF26DCEE2B00B430E2 /* FrameProcessorPerformanceDataCollector.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FrameProcessorPerformanceDataCollector.swift; sourceTree = "<group>"; };
 		B8994E6B263F03E100069589 /* JSIUtils.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = JSIUtils.mm; sourceTree = "<group>"; };
 		B8A751D62609E4980011C623 /* FrameProcessorRuntimeManager.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorRuntimeManager.h; sourceTree = "<group>"; };
 		B8A751D72609E4B30011C623 /* FrameProcessorRuntimeManager.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = FrameProcessorRuntimeManager.mm; sourceTree = "<group>"; };
@@ -268,6 +269,7 @@
 				B88873E5263D46C7008B1D0E /* FrameProcessorPlugin.h */,
 				B80416F026AB16E8000DEB6A /* VisionCameraScheduler.mm */,
 				B80416F126AB16F3000DEB6A /* VisionCameraScheduler.h */,
+				B8948BDF26DCEE2B00B430E2 /* FrameProcessorPerformanceDataCollector.swift */,
 			);
 			path = "Frame Processor";
 			sourceTree = "<group>";

package.json

@@ -84,7 +84,7 @@
     "react-native": "^0.65.1",
     "react-native-builder-bob": "^0.18.1",
     "react-native-codegen": "^0.0.7",
-    "react-native-reanimated": "^2.3.0-alpha.2",
+    "react-native-reanimated": "^2.3.0-alpha.3",
     "release-it": "^14.11.5",
     "typescript": "4.3.5"
   },

src/Camera.tsx

@@ -8,6 +8,7 @@ import {
   Platform,
   LayoutChangeEvent,
 } from 'react-native';
+import type { FrameProcessorPerformanceSuggestion } from '.';
 import type { CameraDevice } from './CameraDevice';
 import type { ErrorWithCause } from './CameraError';
 import { CameraCaptureError, CameraRuntimeError, tryParseNativeCameraError, isErrorWithCause } from './CameraError';
@@ -27,11 +28,16 @@ interface OnErrorEvent {
   message: string;
   cause?: ErrorWithCause;
 }
-type NativeCameraViewProps = Omit<CameraProps, 'device' | 'onInitialized' | 'onError' | 'frameProcessor'> & {
+type NativeCameraViewProps = Omit<
+  CameraProps,
+  'device' | 'onInitialized' | 'onError' | 'onFrameProcessorPerformanceSuggestionAvailable' | 'frameProcessor' | 'frameProcessorFps'
+> & {
   cameraId: string;
+  frameProcessorFps?: number; // native cannot use number | string, so we use '-1' for 'auto'
   enableFrameProcessor: boolean;
   onInitialized?: (event: NativeSyntheticEvent<void>) => void;
   onError?: (event: NativeSyntheticEvent<OnErrorEvent>) => void;
+  onFrameProcessorPerformanceSuggestionAvailable?: (event: NativeSyntheticEvent<FrameProcessorPerformanceSuggestion>) => void;
 };
 type RefType = React.Component<NativeCameraViewProps> & Readonly<NativeMethods>;
 //#endregion
@@ -86,6 +92,7 @@ export class Camera extends React.PureComponent<CameraProps> {
     super(props);
     this.onInitialized = this.onInitialized.bind(this);
     this.onError = this.onError.bind(this);
+    this.onFrameProcessorPerformanceSuggestionAvailable = this.onFrameProcessorPerformanceSuggestionAvailable.bind(this);
     this.onLayout = this.onLayout.bind(this);
     this.ref = React.createRef<RefType>();
     this.lastFrameProcessor = undefined;
@@ -333,6 +340,11 @@ export class Camera extends React.PureComponent<CameraProps> {
   private onInitialized(): void {
     this.props.onInitialized?.();
   }
+
+  private onFrameProcessorPerformanceSuggestionAvailable(event: NativeSyntheticEvent<FrameProcessorPerformanceSuggestion>): void {
+    if (this.props.onFrameProcessorPerformanceSuggestionAvailable != null)
+      this.props.onFrameProcessorPerformanceSuggestionAvailable(event.nativeEvent);
+  }
   //#endregion
 
   //#region Lifecycle
@@ -396,14 +408,16 @@ export class Camera extends React.PureComponent<CameraProps> {
   /** @internal */
   public render(): React.ReactNode {
     // We remove the big `device` object from the props because we only need to pass `cameraId` to native.
-    const { device, frameProcessor, ...props } = this.props;
+    const { device, frameProcessor, frameProcessorFps, ...props } = this.props;
     return (
       <NativeCameraView
         {...props}
+        frameProcessorFps={frameProcessorFps === 'auto' ? -1 : frameProcessorFps}
         cameraId={device.id}
         ref={this.ref}
         onInitialized={this.onInitialized}
         onError={this.onError}
+        onFrameProcessorPerformanceSuggestionAvailable={this.onFrameProcessorPerformanceSuggestionAvailable}
         enableFrameProcessor={frameProcessor != null}
         onLayout={this.onLayout}
       />

src/CameraProps.ts

@@ -4,6 +4,11 @@ import type { CameraRuntimeError } from './CameraError';
 import type { CameraPreset } from './CameraPreset';
 import type { Frame } from './Frame';
 
+export interface FrameProcessorPerformanceSuggestion {
+  type: 'can-use-higher-fps' | 'should-use-lower-fps';
+  suggestedFrameProcessorFps: number;
+}
+
 export interface CameraProps extends ViewProps {
   /**
    * The Camera Device to use.
@@ -161,6 +166,10 @@ export interface CameraProps extends ViewProps {
    * Called when the camera was successfully initialized.
    */
   onInitialized?: () => void;
+  /**
+   * Called when a new performance suggestion for a Frame Processor is available - either if your Frame Processor is running too fast and frames are being dropped, or because it is able to run faster. Optionally, you can adjust your `frameProcessorFps` accordingly.
+   */
+  onFrameProcessorPerformanceSuggestionAvailable?: (suggestion: FrameProcessorPerformanceSuggestion) => void;
   /**
    * A worklet which will be called for every frame the Camera "sees". Throttle the Frame Processor's frame rate with {@linkcode frameProcessorFps}.
    *
@@ -183,7 +192,8 @@ export interface CameraProps extends ViewProps {
   /**
    * Specifies the maximum frame rate the frame processor can use, independent of the Camera's frame rate (`fps` property).
    *
-   * * A value of `1` (default) indicates that the frame processor gets executed once per second, perfect for code scanning.
+   * * A value of `'auto'` (default) indicates that the frame processor should execute as fast as it can, without dropping frames. This is achieved by collecting historical data for previous frame processor calls and adjusting frame rate accordingly.
+   * * A value of `1` indicates that the frame processor gets executed once per second, perfect for code scanning.
    * * A value of `10` indicates that the frame processor gets executed 10 times per second, perfect for more realtime use-cases.
    * * A value of `25` indicates that the frame processor gets executed 25 times per second, perfect for high-speed realtime use-cases.
    * * ...and so on
@@ -191,8 +201,8 @@ export interface CameraProps extends ViewProps {
    * If you're using higher values, always check your Xcode/Android Studio Logs to make sure your frame processors are executing fast enough
    * without blocking the video recording queue.
    *
-   * @default 1
+   * @default 'auto'
    */
-  frameProcessorFps?: number;
+  frameProcessorFps?: number | 'auto';
   //#endregion
 }

yarn.lock

@@ -6282,7 +6282,7 @@ react-native-codegen@^0.0.7:
     jscodeshift "^0.11.0"
     nullthrows "^1.1.1"
 
-react-native-reanimated@^2.3.0-alpha.2:
+react-native-reanimated@^2.3.0-alpha.3:
   version "2.3.0-alpha.3"
   resolved "https://registry.yarnpkg.com/react-native-reanimated/-/react-native-reanimated-2.3.0-alpha.3.tgz#022f5b4eaa847304a155942165a77125c86428bc"
   integrity sha512-Ln+edkTrepKUETKKmGl+GxL8RiB7awBDQFcRJIK39m5ggiLfksOAfqzclpKPLWbAfOvB4cSg+QdowzRN570FXA==