railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

feat: Sync Frame Processors (plus runAsync and runAtTargetFps) (#1472)

Browse Source 
Before, Frame Processors ran on a separate Thread.

After, Frame Processors run fully synchronous and always at the same FPS as the Camera.

Two new functions have been introduced:

* `runAtTargetFps(fps: number, func: () => void)`: Runs the given code as often as the given `fps`, effectively throttling it's calls.
* `runAsync(frame: Frame, func: () => void)`: Runs the given function on a separate Thread for Frame Processing. A strong reference to the Frame is held as long as the function takes to execute.

You can use `runAtTargetFps` to throttle calls to a specific API (e.g. if your Camera is running at 60 FPS, but you only want to run face detection at ~25 FPS, use `runAtTargetFps(25, ...)`.)

You can use `runAsync` to run a heavy algorithm asynchronous, so that the Camera is not blocked while your algorithm runs. This is useful if your main sync processor draws something, and your async processor is doing some image analysis on the side. 

You can also combine both functions.

Examples:

```js
const frameProcessor = useFrameProcessor((frame) => {
  'worklet'
  console.log("I'm running at 60 FPS!")
}, [])
```

```js
const frameProcessor = useFrameProcessor((frame) => {
  'worklet'
  console.log("I'm running at 60 FPS!")

  runAtTargetFps(10, () => {
    'worklet'
    console.log("I'm running at 10 FPS!")
  })
}, [])
```



```js
const frameProcessor = useFrameProcessor((frame) => {
  'worklet'
  console.log("I'm running at 60 FPS!")

  runAsync(frame, () => {
    'worklet'
    console.log("I'm running on another Thread, I can block for longer!")
  })
}, [])
```

```js
const frameProcessor = useFrameProcessor((frame) => {
  'worklet'
  console.log("I'm running at 60 FPS!")

  runAtTargetFps(10, () => {
    'worklet'
    runAsync(frame, () => {
      'worklet'
      console.log("I'm running on another Thread at 10 FPS, I can block for longer!")
    })
  })
}, [])
```
This commit is contained in:
Marc Rousavy
2023-02-15 16:47:09 +01:00
committed by GitHub
parent a0590dccb5
commit 30b56153db
30 changed files with 660 additions and 914 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
android/src/main/cpp/FrameHostObject.cpp
View File

@@ -8,12 +8,13 @@
#include <jni.h>
#include <vector>
#include <string>
#include <JsiHostObject.h>
namespace vision {
using namespace facebook;
FrameHostObject::FrameHostObject(jni::alias_ref<JImageProxy::javaobject> image): frame(make_global(image)) { }
FrameHostObject::FrameHostObject(jni::alias_ref<JImageProxy::javaobject> image): frame(make_global(image)), _refCount(0) { }
FrameHostObject::~FrameHostObject() {
// Hermes' Garbage Collector (Hades GC) calls destructors on a separate Thread
@@ -25,21 +26,24 @@ FrameHostObject::~FrameHostObject() {
std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt) {
std::vector<jsi::PropNameID> result;
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toString")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isValid")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("width")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("height")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("bytesPerRow")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("planesCount")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("close")));
// Debugging
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toString")));
// Ref Management
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isValid")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("incrementRefCount")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("decrementRefCount")));
return result;
}
jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propNameId) {
auto name = propNameId.utf8(runtime);
jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
auto name = propName.utf8(runtime);
if (name == "toString") {
auto toString = [this] (jsi::Runtime& runtime, const jsi::Value&, const jsi::Value*, size_t) -> jsi::Value {
auto toString = JSI_HOST_FUNCTION_LAMBDA {
if (!this->frame) {
return jsi::String::createFromUtf8(runtime, "[closed frame]");
}
@@ -50,51 +54,53 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
};
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
}
if (name == "close") {
auto close = [this] (jsi::Runtime& runtime, const jsi::Value&, const jsi::Value*, size_t) -> jsi::Value {
if (!this->frame) {
throw jsi::JSError(runtime, "Trying to close an already closed frame! Did you call frame.close() twice?");
}
this->close();
if (name == "incrementRefCount") {
auto incrementRefCount = JSI_HOST_FUNCTION_LAMBDA {
// Increment retain count by one so ARC doesn't destroy the Frame Buffer.
std::lock_guard lock(this->_refCountMutex);
this->_refCount++;
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "close"), 0, close);
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "incrementRefCount"),
0,
incrementRefCount);
}
if (name == "decrementRefCount") {
auto decrementRefCount = JSI_HOST_FUNCTION_LAMBDA {
// Decrement retain count by one. If the retain count is zero, ARC will destroy the Frame Buffer.
std::lock_guard lock(this->_refCountMutex);
this->_refCount--;
if (_refCount < 1) {
this->frame->close();
}
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "decrementRefCount"),
0,
decrementRefCount);
}
if (name == "isValid") {
return jsi::Value(this->frame && this->frame->getIsValid());
}
if (name == "width") {
this->assertIsFrameStrong(runtime, name);
return jsi::Value(this->frame->getWidth());
}
if (name == "height") {
this->assertIsFrameStrong(runtime, name);
return jsi::Value(this->frame->getHeight());
}
if (name == "bytesPerRow") {
this->assertIsFrameStrong(runtime, name);
return jsi::Value(this->frame->getBytesPerRow());
}
if (name == "planesCount") {
this->assertIsFrameStrong(runtime, name);
return jsi::Value(this->frame->getPlanesCount());
}
return jsi::Value::undefined();
}
void FrameHostObject::assertIsFrameStrong(jsi::Runtime& runtime, const std::string& accessedPropName) const {
if (!this->frame) {
auto message = "Cannot get `" + accessedPropName + "`, frame is already closed!";
throw jsi::JSError(runtime, message.c_str());
}
}
void FrameHostObject::close() {
if (this->frame) {
this->frame->close();
}
// fallback to base implementation
return HostObject::get(runtime, propName);
}
} // namespace vision