feat: frameProcessorFps="auto" and automatic performance suggestions (throttle or increase FPS) (#393)

* Add `onFrameProcessorPerformanceSuggestionAvailable` and make `frameProcessorFps` support `auto`

* Implement performance suggestion and auto-adjusting

* Fix FPS setting, evaluate correctly

* Floor suggested FPS

* Remove `console.log` for frame drop warnings.

* Swift format

* Use `30` magic number

* only call if FPS is different

* Update CameraView.swift

* Implement Android 1/2

* Cleanup

* Update `frameProcessorFps` if available

* Optimize `FrameProcessorPerformanceDataCollector` initialization

* Cache call

* Set frameProcessorFps directly (Kotlin setter)

* Don't suggest if same value

* Call suggestion every second

* reset time on set

* Always store 15 last samples

* reset counter too

* Update FrameProcessorPerformanceDataCollector.swift

* Update CameraView+RecordVideo.swift

* Update CameraView.kt

* iOS: Redesign evaluation

* Update CameraView+RecordVideo.swift

* Android: Redesign evaluation

* Update CameraView.kt

* Update REA to latest alpha and install RNScreens

* Fix frameProcessorFps updating

android/src/main/java/com/mrousavy/camera/CameraView+Events.kt

@@ -0,0 +1,53 @@
+package com.mrousavy.camera
+
+import android.util.Log
+import com.facebook.react.bridge.Arguments
+import com.facebook.react.bridge.ReactContext
+import com.facebook.react.bridge.WritableMap
+import com.facebook.react.uimanager.events.RCTEventEmitter
+
+fun CameraView.invokeOnInitialized() {
+  Log.i(CameraView.TAG, "invokeOnInitialized()")
+
+  val reactContext = context as ReactContext
+  reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraInitialized", null)
+}
+
+fun CameraView.invokeOnError(error: Throwable) {
+  Log.e(CameraView.TAG, "invokeOnError(...):")
+  error.printStackTrace()
+
+  val cameraError = when (error) {
+    is CameraError -> error
+    else -> UnknownCameraError(error)
+  }
+  val event = Arguments.createMap()
+  event.putString("code", cameraError.code)
+  event.putString("message", cameraError.message)
+  cameraError.cause?.let { cause ->
+    event.putMap("cause", errorToMap(cause))
+  }
+  val reactContext = context as ReactContext
+  reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraError", event)
+}
+
+fun CameraView.invokeOnFrameProcessorPerformanceSuggestionAvailable(currentFps: Double, suggestedFps: Double) {
+  Log.e(CameraView.TAG, "invokeOnFrameProcessorPerformanceSuggestionAvailable(suggestedFps: $suggestedFps):")
+
+  val event = Arguments.createMap()
+  val type = if (suggestedFps > currentFps) "can-use-higher-fps" else "should-use-lower-fps"
+  event.putString("type", type)
+  event.putDouble("suggestedFrameProcessorFps", suggestedFps)
+  val reactContext = context as ReactContext
+  reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraPerformanceSuggestionAvailable", event)
+}
+
+private fun errorToMap(error: Throwable): WritableMap {
+  val map = Arguments.createMap()
+  map.putString("message", error.message)
+  map.putString("stacktrace", error.stackTraceToString())
+  error.cause?.let { cause ->
+    map.putMap("cause", errorToMap(cause))
+  }
+  return map
+}

android/src/main/java/com/mrousavy/camera/CameraView.kt

@@ -23,12 +23,14 @@ import com.facebook.jni.HybridData
 import com.facebook.proguard.annotations.DoNotStrip
 import com.facebook.react.bridge.*
 import com.facebook.react.uimanager.events.RCTEventEmitter
+import com.mrousavy.camera.frameprocessor.FrameProcessorPerformanceDataCollector
 import com.mrousavy.camera.utils.*
 import kotlinx.coroutines.*
 import kotlinx.coroutines.guava.await
 import java.lang.IllegalArgumentException
 import java.util.concurrent.ExecutorService
 import java.util.concurrent.Executors
+import kotlin.math.floor
 import kotlin.math.max
 import kotlin.math.min
 
@@ -93,7 +95,17 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
   var torch = "off"
   var zoom: Float = 1f // in "factor"
   var enableZoomGesture = false
+    set(value) {
+      field = value
+      setOnTouchListener(if (value) touchEventListener else null)
+    }
   var frameProcessorFps = 1.0
+    set(value) {
+      field = value
+      actualFrameProcessorFps = if (value == -1.0) 30.0 else value
+      lastFrameProcessorPerformanceEvaluation = System.currentTimeMillis()
+      frameProcessorPerformanceDataCollector.clear()
+    }
 
   // private properties
   private val reactContext: ReactContext
@@ -131,6 +143,16 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
   private var minZoom: Float = 1f
   private var maxZoom: Float = 1f
 
+  private var actualFrameProcessorFps = 30.0
+  private val frameProcessorPerformanceDataCollector = FrameProcessorPerformanceDataCollector()
+  private var lastSuggestedFrameProcessorFps = 0.0
+  private var lastFrameProcessorPerformanceEvaluation = System.currentTimeMillis()
+  private val isReadyForNewEvaluation: Boolean
+    get() {
+      val lastPerformanceEvaluationElapsedTime = System.currentTimeMillis() - lastFrameProcessorPerformanceEvaluation
+      return lastPerformanceEvaluationElapsedTime > 1000
+    }
+
   @DoNotStrip
   private var mHybridData: HybridData
 
@@ -277,9 +299,6 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
         if (shouldReconfigureTorch) {
           camera!!.cameraControl.enableTorch(torch == "on")
         }
-        if (changedProps.contains("enableZoomGesture")) {
-          setOnTouchListener(if (enableZoomGesture) touchEventListener else null)
-        }
       } catch (e: Throwable) {
         Log.e(TAG, "update() threw: ${e.message}")
         invokeOnError(e)
@@ -406,12 +425,20 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
         imageAnalysis = imageAnalysisBuilder.build().apply {
           setAnalyzer(cameraExecutor, { image ->
             val now = System.currentTimeMillis()
-            val intervalMs = (1.0 / frameProcessorFps) * 1000.0
+            val intervalMs = (1.0 / actualFrameProcessorFps) * 1000.0
             if (now - lastFrameProcessorCall > intervalMs) {
               lastFrameProcessorCall = now
+
+              val perfSample = frameProcessorPerformanceDataCollector.beginPerformanceSampleCollection()
               frameProcessorCallback(image)
+              perfSample.endPerformanceSampleCollection()
             }
             image.close()
+
+            if (isReadyForNewEvaluation) {
+              // last evaluation was more than a second ago, evaluate again
+              evaluateNewPerformanceSamples()
+            }
           })
         }
         useCases.add(imageAnalysis!!)
@@ -444,38 +471,21 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
     }
   }
 
-  private fun invokeOnInitialized() {
-    Log.i(TAG, "invokeOnInitialized()")
+  private fun evaluateNewPerformanceSamples() {
+    lastFrameProcessorPerformanceEvaluation = System.currentTimeMillis()
+    val maxFrameProcessorFps = 30 // TODO: Get maxFrameProcessorFps from ImageAnalyser
+    val averageFps = 1.0 / frameProcessorPerformanceDataCollector.averageExecutionTimeSeconds
+    val suggestedFrameProcessorFps = floor(min(averageFps, maxFrameProcessorFps.toDouble()))
 
-    val reactContext = context as ReactContext
-    reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraInitialized", null)
-  }
-
-  private fun invokeOnError(error: Throwable) {
-    Log.e(TAG, "invokeOnError(...):")
-    error.printStackTrace()
-
-    val cameraError = when (error) {
-      is CameraError -> error
-      else -> UnknownCameraError(error)
+    if (frameProcessorFps == -1.0) {
+      // frameProcessorFps="auto"
+      actualFrameProcessorFps = suggestedFrameProcessorFps
+    } else {
+      // frameProcessorFps={someCustomFpsValue}
+      if (suggestedFrameProcessorFps != lastSuggestedFrameProcessorFps && suggestedFrameProcessorFps != frameProcessorFps) {
+        invokeOnFrameProcessorPerformanceSuggestionAvailable(frameProcessorFps, suggestedFrameProcessorFps)
+        lastSuggestedFrameProcessorFps = suggestedFrameProcessorFps
+      }
     }
-    val event = Arguments.createMap()
-    event.putString("code", cameraError.code)
-    event.putString("message", cameraError.message)
-    cameraError.cause?.let { cause ->
-      event.putMap("cause", errorToMap(cause))
-    }
-    val reactContext = context as ReactContext
-    reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraError", event)
-  }
-
-  private fun errorToMap(error: Throwable): WritableMap {
-    val map = Arguments.createMap()
-    map.putString("message", error.message)
-    map.putString("stacktrace", error.stackTraceToString())
-    error.cause?.let { cause ->
-      map.putMap("cause", errorToMap(cause))
-    }
-    return map
   }
 }

android/src/main/java/com/mrousavy/camera/CameraViewManager.kt

@@ -57,6 +57,7 @@ class CameraViewManager(reactContext: ReactApplicationContext) : SimpleViewManag
     return MapBuilder.builder<String, Any>()
       .put("cameraInitialized", MapBuilder.of("registrationName", "onInitialized"))
       .put("cameraError", MapBuilder.of("registrationName", "onError"))
+      .put("cameraPerformanceSuggestionAvailable", MapBuilder.of("registrationName", "onFrameProcessorPerformanceSuggestionAvailable"))
       .build()
   }
 

android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPerformanceDataCollector.kt

@@ -0,0 +1,38 @@
+package com.mrousavy.camera.frameprocessor
+
+data class PerformanceSampleCollection(val endPerformanceSampleCollection: () -> Unit)
+
+// keep a maximum of `maxSampleSize` historical performance data samples cached.
+private const val maxSampleSize = 15
+
+class FrameProcessorPerformanceDataCollector {
+  private var counter = 0
+  private var performanceSamples: ArrayList<Double> = ArrayList()
+
+  val averageExecutionTimeSeconds: Double
+    get() = performanceSamples.average()
+
+  fun beginPerformanceSampleCollection(): PerformanceSampleCollection {
+    val begin = System.currentTimeMillis()
+
+    return PerformanceSampleCollection {
+      val end = System.currentTimeMillis()
+      val seconds = (end - begin) / 1_000.0
+
+      val index = counter % maxSampleSize
+
+      if (performanceSamples.size > index) {
+        performanceSamples[index] = seconds
+      } else {
+        performanceSamples.add(seconds)
+      }
+
+      counter++
+    }
+  }
+
+  fun clear() {
+    counter = 0
+    performanceSamples.clear()
+  }
+}