railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

feat: Sync Frame Processors (plus runAsync and runAtTargetFps) (#1472)

Browse Source 
Before, Frame Processors ran on a separate Thread.

After, Frame Processors run fully synchronous and always at the same FPS as the Camera.

Two new functions have been introduced:

* `runAtTargetFps(fps: number, func: () => void)`: Runs the given code as often as the given `fps`, effectively throttling it's calls.
* `runAsync(frame: Frame, func: () => void)`: Runs the given function on a separate Thread for Frame Processing. A strong reference to the Frame is held as long as the function takes to execute.

You can use `runAtTargetFps` to throttle calls to a specific API (e.g. if your Camera is running at 60 FPS, but you only want to run face detection at ~25 FPS, use `runAtTargetFps(25, ...)`.)

You can use `runAsync` to run a heavy algorithm asynchronous, so that the Camera is not blocked while your algorithm runs. This is useful if your main sync processor draws something, and your async processor is doing some image analysis on the side. 

You can also combine both functions.

Examples:

```js
const frameProcessor = useFrameProcessor((frame) => {
  'worklet'
  console.log("I'm running at 60 FPS!")
}, [])
```

```js
const frameProcessor = useFrameProcessor((frame) => {
  'worklet'
  console.log("I'm running at 60 FPS!")

  runAtTargetFps(10, () => {
    'worklet'
    console.log("I'm running at 10 FPS!")
  })
}, [])
```



```js
const frameProcessor = useFrameProcessor((frame) => {
  'worklet'
  console.log("I'm running at 60 FPS!")

  runAsync(frame, () => {
    'worklet'
    console.log("I'm running on another Thread, I can block for longer!")
  })
}, [])
```

```js
const frameProcessor = useFrameProcessor((frame) => {
  'worklet'
  console.log("I'm running at 60 FPS!")

  runAtTargetFps(10, () => {
    'worklet'
    runAsync(frame, () => {
      'worklet'
      console.log("I'm running on another Thread at 10 FPS, I can block for longer!")
    })
  })
}, [])
```
This commit is contained in:
Marc Rousavy
2023-02-15 16:47:09 +01:00
committed by GitHub
parent a0590dccb5
commit 30b56153db
30 changed files with 660 additions and 914 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
ios/Frame Processor/FrameHostObject.h
View File

@@ -16,16 +16,12 @@ using namespace facebook;
class JSI_EXPORT FrameHostObject: public jsi::HostObject {
public:
explicit FrameHostObject(Frame* frame): frame(frame) {}
explicit FrameHostObject(Frame* frame): frame(frame) { }
public:
jsi::Value get(jsi::Runtime&, const jsi::PropNameID& name) override;
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& rt) override;
void close();
public:
Frame* frame;
private:
void assertIsFrameStrong(jsi::Runtime& runtime, const std::string& accessedPropName);
};

64
ios/Frame Processor/FrameHostObject.mm
View File

@@ -9,16 +9,21 @@
#import "FrameHostObject.h"
#import <Foundation/Foundation.h>
#import <jsi/jsi.h>
#import "JsiHostObject.h"
#import "JsiSharedValue.h"
std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt) {
std::vector<jsi::PropNameID> result;
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toString")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isValid")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("width")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("height")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("bytesPerRow")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("planesCount")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("close")));
// Debugging
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toString")));
// Ref Management
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isValid")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("incrementRefCount")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("decrementRefCount")));
return result;
}
@@ -26,7 +31,7 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
auto name = propName.utf8(runtime);
if (name == "toString") {
auto toString = [this] (jsi::Runtime& runtime, const jsi::Value&, const jsi::Value*, size_t) -> jsi::Value {
auto toString = JSI_HOST_FUNCTION_LAMBDA {
if (this->frame == nil) {
return jsi::String::createFromUtf8(runtime, "[closed frame]");
}
@@ -39,60 +44,55 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
};
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
}
if (name == "close") {
auto close = [this] (jsi::Runtime& runtime, const jsi::Value&, const jsi::Value*, size_t) -> jsi::Value {
if (this->frame == nil) {
throw jsi::JSError(runtime, "Trying to close an already closed frame! Did you call frame.close() twice?");
}
this->close();
if (name == "incrementRefCount") {
auto incrementRefCount = JSI_HOST_FUNCTION_LAMBDA {
// Increment retain count by one so ARC doesn't destroy the Frame Buffer.
CFRetain(frame.buffer);
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "close"), 0, close);
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "incrementRefCount"),
0,
incrementRefCount);
}
if (name == "decrementRefCount") {
auto decrementRefCount = JSI_HOST_FUNCTION_LAMBDA {
// Decrement retain count by one. If the retain count is zero, ARC will destroy the Frame Buffer.
CFRelease(frame.buffer);
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "decrementRefCount"),
0,
decrementRefCount);
}
if (name == "isValid") {
auto isValid = frame != nil && CMSampleBufferIsValid(frame.buffer);
auto isValid = frame != nil && frame.buffer != nil && CFGetRetainCount(frame.buffer) > 0 && CMSampleBufferIsValid(frame.buffer);
return jsi::Value(isValid);
}
if (name == "width") {
this->assertIsFrameStrong(runtime, name);
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto width = CVPixelBufferGetWidth(imageBuffer);
return jsi::Value((double) width);
}
if (name == "height") {
this->assertIsFrameStrong(runtime, name);
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto height = CVPixelBufferGetHeight(imageBuffer);
return jsi::Value((double) height);
}
if (name == "bytesPerRow") {
this->assertIsFrameStrong(runtime, name);
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer);
return jsi::Value((double) bytesPerRow);
}
if (name == "planesCount") {
this->assertIsFrameStrong(runtime, name);
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto planesCount = CVPixelBufferGetPlaneCount(imageBuffer);
return jsi::Value((double) planesCount);
}
return jsi::Value::undefined();
}
void FrameHostObject::assertIsFrameStrong(jsi::Runtime &runtime, const std::string &accessedPropName) {
if (frame == nil) {
auto message = "Cannot get `" + accessedPropName + "`, frame is already closed!";
throw jsi::JSError(runtime, message.c_str());
}
}
void FrameHostObject::close() {
if (frame != nil) {
CMSampleBufferInvalidate(frame.buffer);
// ARC will hopefully delete it lol
this->frame = nil;
}
// fallback to base implementation
return HostObject::get(runtime, propName);
}

67
ios/Frame Processor/FrameProcessorPerformanceDataCollector.swift
View File

@@ -1,67 +0,0 @@
//
// FrameProcessorPerformanceDataCollector.swift
// VisionCamera
//
// Created by Marc Rousavy on 30.08.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
import Foundation
// keep a maximum of `maxSampleSize` historical performance data samples cached.
private let maxSampleSize = 15
// MARK: - PerformanceSampleCollection
struct PerformanceSampleCollection {
var endPerformanceSampleCollection: () -> Void
init(end: @escaping () -> Void) {
endPerformanceSampleCollection = end
}
}
// MARK: - FrameProcessorPerformanceDataCollector
class FrameProcessorPerformanceDataCollector {
private var performanceSamples: [Double] = []
private var counter = 0
private var lastEvaluation = -1
var hasEnoughData: Bool {
return !performanceSamples.isEmpty
}
var averageExecutionTimeSeconds: Double {
let sum = performanceSamples.reduce(0, +)
let average = sum / Double(performanceSamples.count)
lastEvaluation = counter
return average
}
func beginPerformanceSampleCollection() -> PerformanceSampleCollection {
let begin = DispatchTime.now()
return PerformanceSampleCollection {
let end = DispatchTime.now()
let seconds = Double(end.uptimeNanoseconds - begin.uptimeNanoseconds) / 1_000_000_000.0
let index = self.counter % maxSampleSize
if self.performanceSamples.count > index {
self.performanceSamples[index] = seconds
} else {
self.performanceSamples.append(seconds)
}
self.counter += 1
}
}
func clear() {
counter = 0
performanceSamples.removeAll()
}
}

41
ios/Frame Processor/FrameProcessorRuntimeManager.mm
View File

@@ -22,6 +22,7 @@
#import "JsiWorkletContext.h"
#import "JsiWorkletApi.h"
#import "JsiWorklet.h"
#import "JsiHostObject.h"
#import "FrameProcessorUtils.h"
#import "FrameProcessorCallback.h"
@@ -30,7 +31,7 @@
// Forward declarations for the Swift classes
__attribute__((objc_runtime_name("_TtC12VisionCamera12CameraQueues")))
@interface CameraQueues : NSObject
@property (nonatomic, class, readonly, strong) dispatch_queue_t _Nonnull frameProcessorQueue;
@property (nonatomic, class, readonly, strong) dispatch_queue_t _Nonnull videoQueue;
@end
__attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
@interface CameraView : UIView
@@ -38,6 +39,7 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
@end
@implementation FrameProcessorRuntimeManager {
// Running Frame Processors on camera's video thread (synchronously)
std::shared_ptr<RNWorklet::JsiWorkletContext> workletContext;
}
@@ -59,16 +61,15 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
};
auto runOnWorklet = [](std::function<void()>&& f) {
// Run on Frame Processor Worklet Runtime
dispatch_async(CameraQueues.frameProcessorQueue, [f = std::move(f)](){
dispatch_async(CameraQueues.videoQueue, [f = std::move(f)](){
f();
});
};
workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera");
workletContext->initialize("VisionCamera",
&runtime,
runOnJS,
runOnWorklet);
workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera",
&runtime,
runOnJS,
runOnWorklet);
NSLog(@"FrameProcessorBindings: Worklet Context Created!");
@@ -136,28 +137,17 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
NSLog(@"FrameProcessorBindings: Installing global functions...");
// setFrameProcessor(viewTag: number, frameProcessor: (frame: Frame) => void)
auto setFrameProcessor = [self](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
size_t count) -> jsi::Value {
auto setFrameProcessor = JSI_HOST_FUNCTION_LAMBDA {
NSLog(@"FrameProcessorBindings: Setting new frame processor...");
if (!arguments[0].isNumber()) throw jsi::JSError(runtime, "Camera::setFrameProcessor: First argument ('viewTag') must be a number!");
if (!arguments[1].isObject()) throw jsi::JSError(runtime, "Camera::setFrameProcessor: Second argument ('frameProcessor') must be a function!");
auto viewTag = arguments[0].asNumber();
NSLog(@"FrameProcessorBindings: Converting JSI Function to Worklet...");
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, arguments[1]);
RCTExecuteOnMainQueue([=]() {
RCTExecuteOnMainQueue(^{
auto currentBridge = [RCTBridge currentBridge];
auto anonymousView = [currentBridge.uiManager viewForReactTag:[NSNumber numberWithDouble:viewTag]];
auto view = static_cast<CameraView*>(anonymousView);
NSLog(@"FrameProcessorBindings: Converting worklet to Objective-C callback...");
view.frameProcessorCallback = convertWorkletToFrameProcessorCallback(workletContext->getWorkletRuntime(), worklet);
NSLog(@"FrameProcessorBindings: Frame processor set!");
auto callback = convertWorkletToFrameProcessorCallback(self->workletContext->getWorkletRuntime(), worklet);
view.frameProcessorCallback = callback;
});
return jsi::Value::undefined();
@@ -168,12 +158,8 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
setFrameProcessor));
// unsetFrameProcessor(viewTag: number)
auto unsetFrameProcessor = [](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
size_t count) -> jsi::Value {
auto unsetFrameProcessor = JSI_HOST_FUNCTION_LAMBDA {
NSLog(@"FrameProcessorBindings: Removing frame processor...");
if (!arguments[0].isNumber()) throw jsi::JSError(runtime, "Camera::unsetFrameProcessor: First argument ('viewTag') must be a number!");
auto viewTag = arguments[0].asNumber();
RCTExecuteOnMainQueue(^{
@@ -185,7 +171,6 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
auto view = static_cast<CameraView*>(anonymousView);
view.frameProcessorCallback = nil;
NSLog(@"FrameProcessorBindings: Frame processor removed!");
});
return jsi::Value::undefined();

12
ios/Frame Processor/FrameProcessorUtils.mm
View File

@@ -28,13 +28,15 @@ FrameProcessorCallback convertWorkletToFrameProcessorCallback(jsi::Runtime& runt
// Converts a Worklet to a callable Objective-C block function
return ^(Frame* frame) {
auto frameHostObject = std::make_shared<FrameHostObject>(frame);
try {
// Call JS Frame Processor function with boxed Frame Host Object
// Box the Frame to a JS Host Object
auto frameHostObject = std::make_shared<FrameHostObject>(frame);
auto argument = jsi::Object::createFromHostObject(runtime, frameHostObject);
jsi::Value jsValue(std::move(argument));
// Call the Worklet with the Frame JS Host Object as an argument
workletInvoker->call(runtime, jsi::Value::undefined(), &jsValue, 1);
} catch (jsi::JSError& jsError) {
// JS Error occured, print it to console.
auto stack = std::regex_replace(jsError.getStack(), std::regex("\n"), "\n ");
auto message = [NSString stringWithFormat:@"Frame Processor threw an error: %s\nIn: %s", jsError.getMessage().c_str(), stack.c_str()];
@@ -48,11 +50,5 @@ FrameProcessorCallback convertWorkletToFrameProcessorCallback(jsi::Runtime& runt
NSLog(@"%@", message);
}
}
// Manually free the buffer because:
// 1. we are sure we don't need it anymore, the frame processor worklet has finished executing.
// 2. we don't know when the JS runtime garbage collects this object, it might be holding it for a few more frames
// which then blocks the camera queue from pushing new frames (memory limit)
frameHostObject->close();
};
}