feat: frameProcessorFps="auto" and automatic performance suggestions (throttle or increase FPS) (#393)

* Add `onFrameProcessorPerformanceSuggestionAvailable` and make `frameProcessorFps` support `auto`

* Implement performance suggestion and auto-adjusting

* Fix FPS setting, evaluate correctly

* Floor suggested FPS

* Remove `console.log` for frame drop warnings.

* Swift format

* Use `30` magic number

* only call if FPS is different

* Update CameraView.swift

* Implement Android 1/2

* Cleanup

* Update `frameProcessorFps` if available

* Optimize `FrameProcessorPerformanceDataCollector` initialization

* Cache call

* Set frameProcessorFps directly (Kotlin setter)

* Don't suggest if same value

* Call suggestion every second

* reset time on set

* Always store 15 last samples

* reset counter too

* Update FrameProcessorPerformanceDataCollector.swift

* Update CameraView+RecordVideo.swift

* Update CameraView.kt

* iOS: Redesign evaluation

* Update CameraView+RecordVideo.swift

* Android: Redesign evaluation

* Update CameraView.kt

* Update REA to latest alpha and install RNScreens

* Fix frameProcessorFps updating
This commit is contained in:
Marc Rousavy 2021-09-06 16:27:16 +02:00 committed by GitHub
parent 5b570d611a
commit ad5e131f6a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 335 additions and 120 deletions

View File

@ -0,0 +1,53 @@
package com.mrousavy.camera
import android.util.Log
import com.facebook.react.bridge.Arguments
import com.facebook.react.bridge.ReactContext
import com.facebook.react.bridge.WritableMap
import com.facebook.react.uimanager.events.RCTEventEmitter
fun CameraView.invokeOnInitialized() {
Log.i(CameraView.TAG, "invokeOnInitialized()")
val reactContext = context as ReactContext
reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraInitialized", null)
}
fun CameraView.invokeOnError(error: Throwable) {
Log.e(CameraView.TAG, "invokeOnError(...):")
error.printStackTrace()
val cameraError = when (error) {
is CameraError -> error
else -> UnknownCameraError(error)
}
val event = Arguments.createMap()
event.putString("code", cameraError.code)
event.putString("message", cameraError.message)
cameraError.cause?.let { cause ->
event.putMap("cause", errorToMap(cause))
}
val reactContext = context as ReactContext
reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraError", event)
}
fun CameraView.invokeOnFrameProcessorPerformanceSuggestionAvailable(currentFps: Double, suggestedFps: Double) {
Log.e(CameraView.TAG, "invokeOnFrameProcessorPerformanceSuggestionAvailable(suggestedFps: $suggestedFps):")
val event = Arguments.createMap()
val type = if (suggestedFps > currentFps) "can-use-higher-fps" else "should-use-lower-fps"
event.putString("type", type)
event.putDouble("suggestedFrameProcessorFps", suggestedFps)
val reactContext = context as ReactContext
reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraPerformanceSuggestionAvailable", event)
}
private fun errorToMap(error: Throwable): WritableMap {
val map = Arguments.createMap()
map.putString("message", error.message)
map.putString("stacktrace", error.stackTraceToString())
error.cause?.let { cause ->
map.putMap("cause", errorToMap(cause))
}
return map
}

View File

@ -23,12 +23,14 @@ import com.facebook.jni.HybridData
import com.facebook.proguard.annotations.DoNotStrip
import com.facebook.react.bridge.*
import com.facebook.react.uimanager.events.RCTEventEmitter
import com.mrousavy.camera.frameprocessor.FrameProcessorPerformanceDataCollector
import com.mrousavy.camera.utils.*
import kotlinx.coroutines.*
import kotlinx.coroutines.guava.await
import java.lang.IllegalArgumentException
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import kotlin.math.floor
import kotlin.math.max
import kotlin.math.min
@ -93,7 +95,17 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
var torch = "off"
var zoom: Float = 1f // in "factor"
var enableZoomGesture = false
set(value) {
field = value
setOnTouchListener(if (value) touchEventListener else null)
}
var frameProcessorFps = 1.0
set(value) {
field = value
actualFrameProcessorFps = if (value == -1.0) 30.0 else value
lastFrameProcessorPerformanceEvaluation = System.currentTimeMillis()
frameProcessorPerformanceDataCollector.clear()
}
// private properties
private val reactContext: ReactContext
@ -131,6 +143,16 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
private var minZoom: Float = 1f
private var maxZoom: Float = 1f
private var actualFrameProcessorFps = 30.0
private val frameProcessorPerformanceDataCollector = FrameProcessorPerformanceDataCollector()
private var lastSuggestedFrameProcessorFps = 0.0
private var lastFrameProcessorPerformanceEvaluation = System.currentTimeMillis()
private val isReadyForNewEvaluation: Boolean
get() {
val lastPerformanceEvaluationElapsedTime = System.currentTimeMillis() - lastFrameProcessorPerformanceEvaluation
return lastPerformanceEvaluationElapsedTime > 1000
}
@DoNotStrip
private var mHybridData: HybridData
@ -277,9 +299,6 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
if (shouldReconfigureTorch) {
camera!!.cameraControl.enableTorch(torch == "on")
}
if (changedProps.contains("enableZoomGesture")) {
setOnTouchListener(if (enableZoomGesture) touchEventListener else null)
}
} catch (e: Throwable) {
Log.e(TAG, "update() threw: ${e.message}")
invokeOnError(e)
@ -406,12 +425,20 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
imageAnalysis = imageAnalysisBuilder.build().apply {
setAnalyzer(cameraExecutor, { image ->
val now = System.currentTimeMillis()
val intervalMs = (1.0 / frameProcessorFps) * 1000.0
val intervalMs = (1.0 / actualFrameProcessorFps) * 1000.0
if (now - lastFrameProcessorCall > intervalMs) {
lastFrameProcessorCall = now
val perfSample = frameProcessorPerformanceDataCollector.beginPerformanceSampleCollection()
frameProcessorCallback(image)
perfSample.endPerformanceSampleCollection()
}
image.close()
if (isReadyForNewEvaluation) {
// last evaluation was more than a second ago, evaluate again
evaluateNewPerformanceSamples()
}
})
}
useCases.add(imageAnalysis!!)
@ -444,38 +471,21 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
}
}
private fun invokeOnInitialized() {
Log.i(TAG, "invokeOnInitialized()")
private fun evaluateNewPerformanceSamples() {
lastFrameProcessorPerformanceEvaluation = System.currentTimeMillis()
val maxFrameProcessorFps = 30 // TODO: Get maxFrameProcessorFps from ImageAnalyser
val averageFps = 1.0 / frameProcessorPerformanceDataCollector.averageExecutionTimeSeconds
val suggestedFrameProcessorFps = floor(min(averageFps, maxFrameProcessorFps.toDouble()))
val reactContext = context as ReactContext
reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraInitialized", null)
}
private fun invokeOnError(error: Throwable) {
Log.e(TAG, "invokeOnError(...):")
error.printStackTrace()
val cameraError = when (error) {
is CameraError -> error
else -> UnknownCameraError(error)
}
val event = Arguments.createMap()
event.putString("code", cameraError.code)
event.putString("message", cameraError.message)
cameraError.cause?.let { cause ->
event.putMap("cause", errorToMap(cause))
}
val reactContext = context as ReactContext
reactContext.getJSModule(RCTEventEmitter::class.java).receiveEvent(id, "cameraError", event)
}
private fun errorToMap(error: Throwable): WritableMap {
val map = Arguments.createMap()
map.putString("message", error.message)
map.putString("stacktrace", error.stackTraceToString())
error.cause?.let { cause ->
map.putMap("cause", errorToMap(cause))
}
return map
if (frameProcessorFps == -1.0) {
// frameProcessorFps="auto"
actualFrameProcessorFps = suggestedFrameProcessorFps
} else {
// frameProcessorFps={someCustomFpsValue}
if (suggestedFrameProcessorFps != lastSuggestedFrameProcessorFps && suggestedFrameProcessorFps != frameProcessorFps) {
invokeOnFrameProcessorPerformanceSuggestionAvailable(frameProcessorFps, suggestedFrameProcessorFps)
lastSuggestedFrameProcessorFps = suggestedFrameProcessorFps
}
}
}
}

View File

@ -57,6 +57,7 @@ class CameraViewManager(reactContext: ReactApplicationContext) : SimpleViewManag
return MapBuilder.builder<String, Any>()
.put("cameraInitialized", MapBuilder.of("registrationName", "onInitialized"))
.put("cameraError", MapBuilder.of("registrationName", "onError"))
.put("cameraPerformanceSuggestionAvailable", MapBuilder.of("registrationName", "onFrameProcessorPerformanceSuggestionAvailable"))
.build()
}

View File

@ -0,0 +1,38 @@
package com.mrousavy.camera.frameprocessor
data class PerformanceSampleCollection(val endPerformanceSampleCollection: () -> Unit)
// keep a maximum of `maxSampleSize` historical performance data samples cached.
private const val maxSampleSize = 15
class FrameProcessorPerformanceDataCollector {
private var counter = 0
private var performanceSamples: ArrayList<Double> = ArrayList()
val averageExecutionTimeSeconds: Double
get() = performanceSamples.average()
fun beginPerformanceSampleCollection(): PerformanceSampleCollection {
val begin = System.currentTimeMillis()
return PerformanceSampleCollection {
val end = System.currentTimeMillis()
val seconds = (end - begin) / 1_000.0
val index = counter % maxSampleSize
if (performanceSamples.size > index) {
performanceSamples[index] = seconds
} else {
performanceSamples.add(seconds)
}
counter++
}
}
fun clear() {
counter = 0
performanceSamples.clear()
}
}

View File

@ -321,6 +321,9 @@ PODS:
- React-RCTVibration
- ReactCommon/turbomodule/core
- Yoga
- RNScreens (3.6.0):
- React-Core
- React-RCTImage
- RNStaticSafeAreaInsets (2.1.1):
- React
- RNVectorIcons (8.1.0):
@ -368,6 +371,7 @@ DEPENDENCIES:
- ReactNativeNavigation (from `../node_modules/react-native-navigation`)
- RNGestureHandler (from `../node_modules/react-native-gesture-handler`)
- RNReanimated (from `../node_modules/react-native-reanimated`)
- RNScreens (from `../node_modules/react-native-screens`)
- RNStaticSafeAreaInsets (from `../node_modules/react-native-static-safe-area-insets`)
- RNVectorIcons (from `../node_modules/react-native-vector-icons`)
- VisionCamera (from `../..`)
@ -447,6 +451,8 @@ EXTERNAL SOURCES:
:path: "../node_modules/react-native-gesture-handler"
RNReanimated:
:path: "../node_modules/react-native-reanimated"
RNScreens:
:path: "../node_modules/react-native-screens"
RNStaticSafeAreaInsets:
:path: "../node_modules/react-native-static-safe-area-insets"
RNVectorIcons:
@ -493,6 +499,7 @@ SPEC CHECKSUMS:
ReactNativeNavigation: 87aa2e3a749a9b338057e8d53af54d865241b843
RNGestureHandler: a479ebd5ed4221a810967000735517df0d2db211
RNReanimated: 48e578538b2fad573d3d5ce2e80ad375e1534d87
RNScreens: eb0dfb2d6b21d2d7f980ad46b14eb306d2f1062e
RNStaticSafeAreaInsets: 6103cf09647fa427186d30f67b0f5163c1ae8252
RNVectorIcons: 31cebfcf94e8cf8686eb5303ae0357da64d7a5a4
VisionCamera: dc481620431e31c1d6bbbc438d8a1d9c56f3d304

View File

@ -23,7 +23,8 @@
"react-native-gesture-handler": "^1.10.3",
"react-native-navigation": "^7.19.0",
"react-native-pressable-opacity": "^1.0.4",
"react-native-reanimated": "^2.3.0-alpha.2",
"react-native-reanimated": "^2.3.0-alpha.3",
"react-native-screens": "^3.6.0",
"react-native-static-safe-area-insets": "^2.1.1",
"react-native-vector-icons": "^8.0.0",
"react-native-video": "^5.1.1"

View File

@ -6,6 +6,7 @@ import { Navigation, NavigationFunctionComponent } from 'react-native-navigation
import {
CameraDeviceFormat,
CameraRuntimeError,
FrameProcessorPerformanceSuggestion,
PhotoFile,
sortFormats,
useCameraDevices,
@ -197,6 +198,10 @@ export const CameraPage: NavigationFunctionComponent = ({ componentId }) => {
console.log(`Return Values: ${JSON.stringify(values)}`);
}, []);
const onFrameProcessorSuggestionAvailable = useCallback((suggestion: FrameProcessorPerformanceSuggestion) => {
console.log(`Suggestion available! ${suggestion.type}: Can do ${suggestion.suggestedFrameProcessorFps} FPS`);
}, []);
return (
<View style={styles.container}>
{device != null && (
@ -221,6 +226,7 @@ export const CameraPage: NavigationFunctionComponent = ({ componentId }) => {
audio={true}
frameProcessor={device.supportsParallelVideoProcessing ? frameProcessor : undefined}
frameProcessorFps={1}
onFrameProcessorPerformanceSuggestionAvailable={onFrameProcessorSuggestionAvailable}
/>
</TapGestureHandler>
</Reanimated.View>

View File

@ -4401,7 +4401,7 @@ react-native-pressable-opacity@^1.0.4:
resolved "https://registry.yarnpkg.com/react-native-pressable-opacity/-/react-native-pressable-opacity-1.0.4.tgz#391f33fdc25cb84551f2743a25eced892b9f30f7"
integrity sha512-DBIg7UoRiuBYiFEvx+XNMqH0OEx64WrSksXhT6Kq9XuyyKsThMNDqZ9G5QV7vfu7dU2/IctwIz5c0Xwkp4K3tA==
react-native-reanimated@^2.3.0-alpha.2:
react-native-reanimated@^2.3.0-alpha.3:
version "2.3.0-alpha.3"
resolved "https://registry.yarnpkg.com/react-native-reanimated/-/react-native-reanimated-2.3.0-alpha.3.tgz#022f5b4eaa847304a155942165a77125c86428bc"
integrity sha512-Ln+edkTrepKUETKKmGl+GxL8RiB7awBDQFcRJIK39m5ggiLfksOAfqzclpKPLWbAfOvB4cSg+QdowzRN570FXA==
@ -4413,7 +4413,7 @@ react-native-reanimated@^2.3.0-alpha.2:
react-native-screens "^3.4.0"
string-hash-64 "^1.0.3"
react-native-screens@^3.4.0:
react-native-screens@^3.4.0, react-native-screens@^3.6.0:
version "3.6.0"
resolved "https://registry.yarnpkg.com/react-native-screens/-/react-native-screens-3.6.0.tgz#054af728e50c06bff6b3b4fa7b4b656b70f247cd"
integrity sha512-emQmSu+B6cOIjJH2OIgpuxd9zCD6xz7/oo5GCetyjsM5qR3sMnVgOxqtK99xPu9XJH/8k7MplXbtJgtk/PHXwA==

View File

@ -118,10 +118,11 @@ extension CameraView {
return
}
// TODO: Add JS-Event for Audio Session interruptions?
switch type {
case .began:
// Something interrupted our Audio Session, stop recording audio.
ReactLogger.log(level: .error, message: "The Audio Session was interrupted!", alsoLogToJS: true)
ReactLogger.log(level: .error, message: "The Audio Session was interrupted!")
case .ended:
ReactLogger.log(level: .info, message: "The Audio Session interruption has ended.")
guard let optionsValue = userInfo[AVAudioSessionInterruptionOptionKey] as? UInt else { return }
@ -129,13 +130,13 @@ extension CameraView {
if options.contains(.shouldResume) {
if isRecording {
audioQueue.async {
ReactLogger.log(level: .info, message: "Resuming interrupted Audio Session...", alsoLogToJS: true)
ReactLogger.log(level: .info, message: "Resuming interrupted Audio Session...")
// restart audio session because interruption is over
self.activateAudioSession()
}
}
} else {
ReactLogger.log(level: .error, message: "Cannot resume interrupted Audio Session!", alsoLogToJS: true)
ReactLogger.log(level: .error, message: "Cannot resume interrupted Audio Session!")
}
@unknown default: ()
}

View File

@ -255,7 +255,7 @@ extension CameraView {
@objc
func sessionRuntimeError(notification: Notification) {
ReactLogger.log(level: .error, message: "Unexpected Camera Runtime Error occured!", alsoLogToJS: true)
ReactLogger.log(level: .error, message: "Unexpected Camera Runtime Error occured!")
guard let error = notification.userInfo?[AVCaptureSessionErrorKey] as? AVError else {
return
}

View File

@ -8,9 +8,6 @@
import AVFoundation
private var hasLoggedVideoFrameDropWarning = false
private var hasLoggedFrameProcessorFrameDropWarning = false
// MARK: - CameraView + AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAudioDataOutputSampleBufferDelegate
extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAudioDataOutputSampleBufferDelegate {
@ -203,63 +200,61 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud
}
}
// TODO: resize using VideoToolbox (VTPixelTransferSession)
if let frameProcessor = frameProcessorCallback,
captureOutput is AVCaptureVideoDataOutput {
if let frameProcessor = frameProcessorCallback, captureOutput is AVCaptureVideoDataOutput {
// check if last frame was x nanoseconds ago, effectively throttling FPS
let diff = DispatchTime.now().uptimeNanoseconds - lastFrameProcessorCall.uptimeNanoseconds
let secondsPerFrame = 1.0 / frameProcessorFps.doubleValue
let lastFrameProcessorCallElapsedTime = DispatchTime.now().uptimeNanoseconds - lastFrameProcessorCall.uptimeNanoseconds
let secondsPerFrame = 1.0 / actualFrameProcessorFps
let nanosecondsPerFrame = secondsPerFrame * 1_000_000_000.0
if diff > UInt64(nanosecondsPerFrame) {
if lastFrameProcessorCallElapsedTime > UInt64(nanosecondsPerFrame) {
if !isRunningFrameProcessor {
// we're not in the middle of executing the Frame Processor, so prepare for next call.
CameraQueues.frameProcessorQueue.async {
self.isRunningFrameProcessor = true
let perfSample = self.frameProcessorPerformanceDataCollector.beginPerformanceSampleCollection()
let frame = Frame(buffer: sampleBuffer, orientation: self.bufferOrientation)
frameProcessor(frame)
perfSample.endPerformanceSampleCollection()
self.isRunningFrameProcessor = false
}
lastFrameProcessorCall = DispatchTime.now()
} else {
// we're still in the middle of executing a Frame Processor for a previous frame, notify user about dropped frame.
if !hasLoggedFrameProcessorFrameDropWarning {
ReactLogger.log(level: .warning,
message: "Your Frame Processor took so long to execute that a frame was dropped. " +
"Either throttle your Frame Processor's frame rate using the `frameProcessorFps` prop, or optimize " +
"it's execution speed. (This warning will only be shown once)",
alsoLogToJS: true)
hasLoggedFrameProcessorFrameDropWarning = true
// we're still in the middle of executing a Frame Processor for a previous frame, so a frame was dropped.
ReactLogger.log(level: .warning, message: "The Frame Processor took so long to execute that a frame was dropped.")
}
}
if isReadyForNewEvaluation {
// last evaluation was more than 1sec ago, evaluate again
evaluateNewPerformanceSamples()
}
}
}
#if DEBUG
public final func captureOutput(_ captureOutput: AVCaptureOutput, didDrop buffer: CMSampleBuffer, from _: AVCaptureConnection) {
if !hasLoggedVideoFrameDropWarning && captureOutput is AVCaptureVideoDataOutput {
let reason = findFrameDropReason(inBuffer: buffer)
ReactLogger.log(level: .warning,
message: "Dropped a Frame - This might indicate that your frame rate is higher than the phone can currently process. " +
"Throttle the Camera frame rate using the `fps` prop and make sure the device stays in optimal condition for recording. " +
"Frame drop reason: \(reason). (This warning will only be shown once)",
alsoLogToJS: true)
hasLoggedVideoFrameDropWarning = true
private func evaluateNewPerformanceSamples() {
lastFrameProcessorPerformanceEvaluation = DispatchTime.now()
guard let videoDevice = videoDeviceInput?.device else { return }
let maxFrameProcessorFps = Double(videoDevice.activeVideoMinFrameDuration.timescale) * Double(videoDevice.activeVideoMinFrameDuration.value)
let averageFps = 1.0 / frameProcessorPerformanceDataCollector.averageExecutionTimeSeconds
let suggestedFrameProcessorFps = floor(min(averageFps, maxFrameProcessorFps))
if frameProcessorFps.intValue == -1 {
// frameProcessorFps="auto"
actualFrameProcessorFps = suggestedFrameProcessorFps
} else {
// frameProcessorFps={someCustomFpsValue}
invokeOnFrameProcessorPerformanceSuggestionAvailable(currentFps: frameProcessorFps.doubleValue,
suggestedFps: suggestedFrameProcessorFps)
}
}
private final func findFrameDropReason(inBuffer buffer: CMSampleBuffer) -> String {
var mode: CMAttachmentMode = 0
guard let reason = CMGetAttachment(buffer,
key: kCMSampleBufferAttachmentKey_DroppedFrameReason,
attachmentModeOut: &mode) else {
return "unknown"
private var isReadyForNewEvaluation: Bool {
let lastPerformanceEvaluationElapsedTime = DispatchTime.now().uptimeNanoseconds - lastFrameProcessorPerformanceEvaluation.uptimeNanoseconds
return lastPerformanceEvaluationElapsedTime > 1_000_000_000
}
return String(describing: reason)
}
#endif
/**
Gets the orientation of the CameraView's images (CMSampleBuffers).

View File

@ -52,7 +52,7 @@ public final class CameraView: UIView {
// props that require format reconfiguring
@objc var format: NSDictionary?
@objc var fps: NSNumber?
@objc var frameProcessorFps: NSNumber = 1.0
@objc var frameProcessorFps: NSNumber = -1.0 // "auto"
@objc var hdr: NSNumber? // nullable bool
@objc var lowLightBoost: NSNumber? // nullable bool
@objc var colorSpace: NSString?
@ -64,6 +64,7 @@ public final class CameraView: UIView {
// events
@objc var onInitialized: RCTDirectEventBlock?
@objc var onError: RCTDirectEventBlock?
@objc var onFrameProcessorPerformanceSuggestionAvailable: RCTDirectEventBlock?
// zoom
@objc var enableZoomGesture = false {
didSet {
@ -104,6 +105,10 @@ public final class CameraView: UIView {
/// Specifies whether the frameProcessor() function is currently executing. used to drop late frames.
internal var isRunningFrameProcessor = false
internal let frameProcessorPerformanceDataCollector = FrameProcessorPerformanceDataCollector()
internal var actualFrameProcessorFps = 30.0
internal var lastSuggestedFrameProcessorFps = 0.0
internal var lastFrameProcessorPerformanceEvaluation = DispatchTime.now()
/// Returns whether the AVCaptureSession is currently running (reflected by isActive)
var isRunning: Bool {
@ -244,6 +249,18 @@ public final class CameraView: UIView {
}
}
}
// Frame Processor FPS Configuration
if changedProps.contains("frameProcessorFps") {
if frameProcessorFps.doubleValue == -1 {
// "auto"
actualFrameProcessorFps = 30.0
} else {
actualFrameProcessorFps = frameProcessorFps.doubleValue
}
lastFrameProcessorPerformanceEvaluation = DispatchTime.now()
frameProcessorPerformanceDataCollector.clear()
}
}
internal final func setTorchMode(_ torchMode: String) {
@ -336,4 +353,18 @@ public final class CameraView: UIView {
guard let onInitialized = self.onInitialized else { return }
onInitialized([String: Any]())
}
internal final func invokeOnFrameProcessorPerformanceSuggestionAvailable(currentFps: Double, suggestedFps: Double) {
ReactLogger.log(level: .info, message: "Frame Processor Performance Suggestion available!")
guard let onFrameProcessorPerformanceSuggestionAvailable = self.onFrameProcessorPerformanceSuggestionAvailable else { return }
if lastSuggestedFrameProcessorFps == suggestedFps { return }
if suggestedFps == currentFps { return }
onFrameProcessorPerformanceSuggestionAvailable([
"type": suggestedFps > currentFps ? "can-use-higher-fps" : "should-use-lower-fps",
"suggestedFrameProcessorFps": suggestedFps,
])
lastSuggestedFrameProcessorFps = suggestedFps
}
}

View File

@ -48,6 +48,7 @@ RCT_EXPORT_VIEW_PROPERTY(enableZoomGesture, BOOL);
// Camera View Properties
RCT_EXPORT_VIEW_PROPERTY(onError, RCTDirectEventBlock);
RCT_EXPORT_VIEW_PROPERTY(onInitialized, RCTDirectEventBlock);
RCT_EXPORT_VIEW_PROPERTY(onFrameProcessorPerformanceSuggestionAvailable, RCTDirectEventBlock);
// Camera View Functions
RCT_EXTERN_METHOD(startRecording:(nonnull NSNumber *)node options:(NSDictionary *)options onRecordCallback:(RCTResponseSenderBlock)onRecordCallback);

View File

@ -17,11 +17,6 @@ final class CameraViewManager: RCTViewManager {
override var bridge: RCTBridge! {
didSet {
#if DEBUG
// Install console.log bindings
ReactLogger.ConsoleLogFunction = JSConsoleHelper.getLogFunction(for: bridge)
#endif
// Install Frame Processor bindings and setup Runtime
if VISION_CAMERA_ENABLE_FRAME_PROCESSORS {
CameraQueues.frameProcessorQueue.async {

View File

@ -16,8 +16,7 @@ extension AVAudioSession {
func updateCategory(_ category: AVAudioSession.Category, options: AVAudioSession.CategoryOptions = []) throws {
if self.category != category || categoryOptions.rawValue != options.rawValue {
ReactLogger.log(level: .info,
message: "Changing AVAudioSession category from \(self.category.rawValue) -> \(category.rawValue)",
alsoLogToJS: true)
message: "Changing AVAudioSession category from \(self.category.rawValue) -> \(category.rawValue)")
try setCategory(category, options: options)
}
}

View File

@ -0,0 +1,63 @@
//
// FrameProcessorPerformanceDataCollector.swift
// VisionCamera
//
// Created by Marc Rousavy on 30.08.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
import Foundation
// keep a maximum of `maxSampleSize` historical performance data samples cached.
private let maxSampleSize = 15
// MARK: - PerformanceSampleCollection
struct PerformanceSampleCollection {
var endPerformanceSampleCollection: () -> Void
init(end: @escaping () -> Void) {
endPerformanceSampleCollection = end
}
}
// MARK: - FrameProcessorPerformanceDataCollector
class FrameProcessorPerformanceDataCollector {
private var performanceSamples: [Double] = []
private var counter = 0
private var lastEvaluation = -1
var averageExecutionTimeSeconds: Double {
let sum = performanceSamples.reduce(0, +)
let average = sum / Double(performanceSamples.count)
lastEvaluation = counter
return average
}
func beginPerformanceSampleCollection() -> PerformanceSampleCollection {
let begin = DispatchTime.now()
return PerformanceSampleCollection {
let end = DispatchTime.now()
let seconds = Double(end.uptimeNanoseconds - begin.uptimeNanoseconds) / 1_000_000_000.0
let index = self.counter % maxSampleSize
if self.performanceSamples.count > index {
self.performanceSamples[index] = seconds
} else {
self.performanceSamples.append(seconds)
}
self.counter += 1
}
}
func clear() {
counter = 0
performanceSamples.removeAll()
}
}

View File

@ -11,11 +11,6 @@ import Foundation
// MARK: - ReactLogger
enum ReactLogger {
/**
A function that logs to the JavaScript console.
*/
static var ConsoleLogFunction: ConsoleLogFunction?
/**
Log a message to the console in the format of `VisionCamera.[caller-function-name]: [message]`
@ -27,14 +22,10 @@ enum ReactLogger {
@inlinable
static func log(level: RCTLogLevel,
message: String,
alsoLogToJS: Bool = false,
_ file: String = #file,
_ lineNumber: Int = #line,
_ function: String = #function) {
#if DEBUG
if alsoLogToJS, let log = ConsoleLogFunction {
log(level, "[native] VisionCamera.\(function): \(message)")
}
RCTDefaultLogFunction(level, RCTLogSource.native, file, lineNumber as NSNumber, "VisionCamera.\(function): \(message)")
#endif
}

View File

@ -60,7 +60,7 @@ class RecordingSession {
deinit {
if assetWriter.status == .writing {
ReactLogger.log(level: .info, message: "Cancelling AssetWriter...", alsoLogToJS: true)
ReactLogger.log(level: .info, message: "Cancelling AssetWriter...")
assetWriter.cancelWriting()
}
}
@ -70,11 +70,11 @@ class RecordingSession {
*/
func initializeVideoWriter(withSettings settings: [String: Any], pixelFormat: OSType) {
guard !settings.isEmpty else {
ReactLogger.log(level: .error, message: "Tried to initialize Video Writer with empty settings!", alsoLogToJS: true)
ReactLogger.log(level: .error, message: "Tried to initialize Video Writer with empty settings!")
return
}
guard bufferAdaptor == nil else {
ReactLogger.log(level: .error, message: "Tried to add Video Writer twice!", alsoLogToJS: true)
ReactLogger.log(level: .error, message: "Tried to add Video Writer twice!")
return
}
@ -93,11 +93,11 @@ class RecordingSession {
*/
func initializeAudioWriter(withSettings settings: [String: Any]) {
guard !settings.isEmpty else {
ReactLogger.log(level: .error, message: "Tried to initialize Audio Writer with empty settings!", alsoLogToJS: true)
ReactLogger.log(level: .error, message: "Tried to initialize Audio Writer with empty settings!")
return
}
guard audioWriter == nil else {
ReactLogger.log(level: .error, message: "Tried to add Audio Writer twice!", alsoLogToJS: true)
ReactLogger.log(level: .error, message: "Tried to add Audio Writer twice!")
return
}
@ -139,8 +139,7 @@ class RecordingSession {
}
guard let initialTimestamp = initialTimestamp else {
ReactLogger.log(level: .error,
message: "A frame arrived, but initialTimestamp was nil. Is this RecordingSession running?",
alsoLogToJS: true)
message: "A frame arrived, but initialTimestamp was nil. Is this RecordingSession running?")
return
}
@ -149,17 +148,16 @@ class RecordingSession {
switch bufferType {
case .video:
guard let bufferAdaptor = bufferAdaptor else {
ReactLogger.log(level: .error, message: "Video Frame arrived but VideoWriter was nil!", alsoLogToJS: true)
ReactLogger.log(level: .error, message: "Video Frame arrived but VideoWriter was nil!")
return
}
if !bufferAdaptor.assetWriterInput.isReadyForMoreMediaData {
ReactLogger.log(level: .warning,
message: "The Video AVAssetWriterInput was not ready for more data! Is your frame rate too high?",
alsoLogToJS: true)
message: "The Video AVAssetWriterInput was not ready for more data! Is your frame rate too high?")
return
}
guard let imageBuffer = CMSampleBufferGetImageBuffer(buffer) else {
ReactLogger.log(level: .error, message: "Failed to get the CVImageBuffer!", alsoLogToJS: true)
ReactLogger.log(level: .error, message: "Failed to get the CVImageBuffer!")
return
}
bufferAdaptor.append(imageBuffer, withPresentationTime: timestamp)
@ -169,7 +167,7 @@ class RecordingSession {
}
case .audio:
guard let audioWriter = audioWriter else {
ReactLogger.log(level: .error, message: "Audio Frame arrived but AudioWriter was nil!", alsoLogToJS: true)
ReactLogger.log(level: .error, message: "Audio Frame arrived but AudioWriter was nil!")
return
}
if !audioWriter.isReadyForMoreMediaData {
@ -184,8 +182,7 @@ class RecordingSession {
if assetWriter.status == .failed {
ReactLogger.log(level: .error,
message: "AssetWriter failed to write buffer! Error: \(assetWriter.error?.localizedDescription ?? "none")",
alsoLogToJS: true)
message: "AssetWriter failed to write buffer! Error: \(assetWriter.error?.localizedDescription ?? "none")")
finish()
}
}

View File

@ -135,6 +135,7 @@
B887518425E0102000DB86D6 /* CameraView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CameraView.swift; sourceTree = "<group>"; };
B88873E5263D46C7008B1D0E /* FrameProcessorPlugin.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorPlugin.h; sourceTree = "<group>"; };
B88B47462667C8E00091F538 /* AVCaptureSession+setVideoStabilizationMode.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVCaptureSession+setVideoStabilizationMode.swift"; sourceTree = "<group>"; };
B8948BDF26DCEE2B00B430E2 /* FrameProcessorPerformanceDataCollector.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FrameProcessorPerformanceDataCollector.swift; sourceTree = "<group>"; };
B8994E6B263F03E100069589 /* JSIUtils.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = JSIUtils.mm; sourceTree = "<group>"; };
B8A751D62609E4980011C623 /* FrameProcessorRuntimeManager.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorRuntimeManager.h; sourceTree = "<group>"; };
B8A751D72609E4B30011C623 /* FrameProcessorRuntimeManager.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = FrameProcessorRuntimeManager.mm; sourceTree = "<group>"; };
@ -268,6 +269,7 @@
B88873E5263D46C7008B1D0E /* FrameProcessorPlugin.h */,
B80416F026AB16E8000DEB6A /* VisionCameraScheduler.mm */,
B80416F126AB16F3000DEB6A /* VisionCameraScheduler.h */,
B8948BDF26DCEE2B00B430E2 /* FrameProcessorPerformanceDataCollector.swift */,
);
path = "Frame Processor";
sourceTree = "<group>";

View File

@ -84,7 +84,7 @@
"react-native": "^0.65.1",
"react-native-builder-bob": "^0.18.1",
"react-native-codegen": "^0.0.7",
"react-native-reanimated": "^2.3.0-alpha.2",
"react-native-reanimated": "^2.3.0-alpha.3",
"release-it": "^14.11.5",
"typescript": "4.3.5"
},

View File

@ -8,6 +8,7 @@ import {
Platform,
LayoutChangeEvent,
} from 'react-native';
import type { FrameProcessorPerformanceSuggestion } from '.';
import type { CameraDevice } from './CameraDevice';
import type { ErrorWithCause } from './CameraError';
import { CameraCaptureError, CameraRuntimeError, tryParseNativeCameraError, isErrorWithCause } from './CameraError';
@ -27,11 +28,16 @@ interface OnErrorEvent {
message: string;
cause?: ErrorWithCause;
}
type NativeCameraViewProps = Omit<CameraProps, 'device' | 'onInitialized' | 'onError' | 'frameProcessor'> & {
type NativeCameraViewProps = Omit<
CameraProps,
'device' | 'onInitialized' | 'onError' | 'onFrameProcessorPerformanceSuggestionAvailable' | 'frameProcessor' | 'frameProcessorFps'
> & {
cameraId: string;
frameProcessorFps?: number; // native cannot use number | string, so we use '-1' for 'auto'
enableFrameProcessor: boolean;
onInitialized?: (event: NativeSyntheticEvent<void>) => void;
onError?: (event: NativeSyntheticEvent<OnErrorEvent>) => void;
onFrameProcessorPerformanceSuggestionAvailable?: (event: NativeSyntheticEvent<FrameProcessorPerformanceSuggestion>) => void;
};
type RefType = React.Component<NativeCameraViewProps> & Readonly<NativeMethods>;
//#endregion
@ -86,6 +92,7 @@ export class Camera extends React.PureComponent<CameraProps> {
super(props);
this.onInitialized = this.onInitialized.bind(this);
this.onError = this.onError.bind(this);
this.onFrameProcessorPerformanceSuggestionAvailable = this.onFrameProcessorPerformanceSuggestionAvailable.bind(this);
this.onLayout = this.onLayout.bind(this);
this.ref = React.createRef<RefType>();
this.lastFrameProcessor = undefined;
@ -333,6 +340,11 @@ export class Camera extends React.PureComponent<CameraProps> {
private onInitialized(): void {
this.props.onInitialized?.();
}
private onFrameProcessorPerformanceSuggestionAvailable(event: NativeSyntheticEvent<FrameProcessorPerformanceSuggestion>): void {
if (this.props.onFrameProcessorPerformanceSuggestionAvailable != null)
this.props.onFrameProcessorPerformanceSuggestionAvailable(event.nativeEvent);
}
//#endregion
//#region Lifecycle
@ -396,14 +408,16 @@ export class Camera extends React.PureComponent<CameraProps> {
/** @internal */
public render(): React.ReactNode {
// We remove the big `device` object from the props because we only need to pass `cameraId` to native.
const { device, frameProcessor, ...props } = this.props;
const { device, frameProcessor, frameProcessorFps, ...props } = this.props;
return (
<NativeCameraView
{...props}
frameProcessorFps={frameProcessorFps === 'auto' ? -1 : frameProcessorFps}
cameraId={device.id}
ref={this.ref}
onInitialized={this.onInitialized}
onError={this.onError}
onFrameProcessorPerformanceSuggestionAvailable={this.onFrameProcessorPerformanceSuggestionAvailable}
enableFrameProcessor={frameProcessor != null}
onLayout={this.onLayout}
/>

View File

@ -4,6 +4,11 @@ import type { CameraRuntimeError } from './CameraError';
import type { CameraPreset } from './CameraPreset';
import type { Frame } from './Frame';
export interface FrameProcessorPerformanceSuggestion {
type: 'can-use-higher-fps' | 'should-use-lower-fps';
suggestedFrameProcessorFps: number;
}
export interface CameraProps extends ViewProps {
/**
* The Camera Device to use.
@ -161,6 +166,10 @@ export interface CameraProps extends ViewProps {
* Called when the camera was successfully initialized.
*/
onInitialized?: () => void;
/**
* Called when a new performance suggestion for a Frame Processor is available - either if your Frame Processor is running too fast and frames are being dropped, or because it is able to run faster. Optionally, you can adjust your `frameProcessorFps` accordingly.
*/
onFrameProcessorPerformanceSuggestionAvailable?: (suggestion: FrameProcessorPerformanceSuggestion) => void;
/**
* A worklet which will be called for every frame the Camera "sees". Throttle the Frame Processor's frame rate with {@linkcode frameProcessorFps}.
*
@ -183,7 +192,8 @@ export interface CameraProps extends ViewProps {
/**
* Specifies the maximum frame rate the frame processor can use, independent of the Camera's frame rate (`fps` property).
*
* * A value of `1` (default) indicates that the frame processor gets executed once per second, perfect for code scanning.
* * A value of `'auto'` (default) indicates that the frame processor should execute as fast as it can, without dropping frames. This is achieved by collecting historical data for previous frame processor calls and adjusting frame rate accordingly.
* * A value of `1` indicates that the frame processor gets executed once per second, perfect for code scanning.
* * A value of `10` indicates that the frame processor gets executed 10 times per second, perfect for more realtime use-cases.
* * A value of `25` indicates that the frame processor gets executed 25 times per second, perfect for high-speed realtime use-cases.
* * ...and so on
@ -191,8 +201,8 @@ export interface CameraProps extends ViewProps {
* If you're using higher values, always check your Xcode/Android Studio Logs to make sure your frame processors are executing fast enough
* without blocking the video recording queue.
*
* @default 1
* @default 'auto'
*/
frameProcessorFps?: number;
frameProcessorFps?: number | 'auto';
//#endregion
}

View File

@ -6282,7 +6282,7 @@ react-native-codegen@^0.0.7:
jscodeshift "^0.11.0"
nullthrows "^1.1.1"
react-native-reanimated@^2.3.0-alpha.2:
react-native-reanimated@^2.3.0-alpha.3:
version "2.3.0-alpha.3"
resolved "https://registry.yarnpkg.com/react-native-reanimated/-/react-native-reanimated-2.3.0-alpha.3.tgz#022f5b4eaa847304a155942165a77125c86428bc"
integrity sha512-Ln+edkTrepKUETKKmGl+GxL8RiB7awBDQFcRJIK39m5ggiLfksOAfqzclpKPLWbAfOvB4cSg+QdowzRN570FXA==