railbird/react-native-vision-camera
8
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Rewrite Android C++ part (VisionCameraProxy + JFrame) (#1661)

Browse Source 
* First Android rewrite

* Rewrite Android C++ backend

* Pass `ReadableNativeMap`, fix build error

* fix: Fix FrameProcessor init

* Make a bunch of stuff const reference to avoid copies

* Indents

* Cleanup

* indents

* docs: Update Android docs

* Update CameraView.kt

* fix: Format C++ code
This commit is contained in:
Marc Rousavy 2023-07-22 00:15:11 +02:00 committed by GitHub
parent 44ed42d5d6
commit 86dd703c2b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
45 changed files with 985 additions and 859 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
android/CMakeLists.txt
View File

@ -32,15 +32,17 @@ add_library(
${PACKAGE_NAME}
SHARED
../cpp/JSITypedArray.cpp
src/main/cpp/VisionCamera.cpp
src/main/cpp/JSIJNIConversion.cpp
src/main/cpp/FrameHostObject.cpp
src/main/cpp/FrameProcessorRuntimeManager.cpp
src/main/cpp/CameraView.cpp
src/main/cpp/VisionCameraScheduler.cpp
src/main/cpp/FrameProcessorPluginHostObject.cpp
src/main/cpp/JSIJNIConversion.cpp
src/main/cpp/VisionCamera.cpp
src/main/cpp/VisionCameraProxy.cpp
src/main/cpp/java-bindings/JFrame.cpp
src/main/cpp/java-bindings/JFrameProcessor.cpp
src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp
src/main/cpp/java-bindings/JImageProxy.cpp
src/main/cpp/java-bindings/JHashMap.cpp
src/main/cpp/java-bindings/JVisionCameraProxy.cpp
src/main/cpp/java-bindings/JVisionCameraScheduler.cpp
)
# Header Search Paths (includes)

58
android/src/main/cpp/CameraView.cpp
View File

@ -1,58 +0,0 @@
//
// Created by Marc Rousavy on 14.06.21.
//
#include "CameraView.h"
#include <jni.h>
#include <fbjni/fbjni.h>
#include <jsi/jsi.h>
#include <memory>
#include <string>
#include <regex>
namespace vision {
using namespace facebook;
using namespace jni;
using TSelf = local_ref<CameraView::jhybriddata>;
TSelf CameraView::initHybrid(alias_ref<HybridClass::jhybridobject> jThis) {
return makeCxxInstance(jThis);
}
void CameraView::registerNatives() {
registerHybrid({
makeNativeMethod("initHybrid", CameraView::initHybrid),
makeNativeMethod("frameProcessorCallback", CameraView::frameProcessorCallback),
});
}
void CameraView::frameProcessorCallback(const alias_ref<JImageProxy::javaobject>& frame) {
if (frameProcessor_ == nullptr) {
__android_log_write(ANDROID_LOG_WARN, TAG, "Called Frame Processor callback, but `frameProcessor` is null!");
return;
}
try {
frameProcessor_(frame);
} catch (const jsi::JSError& error) {
// TODO: jsi::JSErrors cannot be caught on Hermes. They crash the entire app.
auto stack = std::regex_replace(error.getStack(), std::regex("\n"), "\n ");
__android_log_print(ANDROID_LOG_ERROR, TAG, "Frame Processor threw an error! %s\nIn: %s", error.getMessage().c_str(), stack.c_str());
} catch (const std::exception& exception) {
__android_log_print(ANDROID_LOG_ERROR, TAG, "Frame Processor threw a C++ error! %s", exception.what());
}
}
void CameraView::setFrameProcessor(const TFrameProcessor&& frameProcessor) {
frameProcessor_ = frameProcessor;
}
void vision::CameraView::unsetFrameProcessor() {
frameProcessor_ = nullptr;
}
} // namespace vision

43
android/src/main/cpp/CameraView.h
View File

@ -1,43 +0,0 @@
//
// Created by Marc Rousavy on 14.06.21.
//
#pragma once
#include <jni.h>
#include <fbjni/fbjni.h>
#include <memory>
#include "java-bindings/JImageProxy.h"
namespace vision {
using namespace facebook;
using TFrameProcessor = std::function<void(jni::alias_ref<JImageProxy::javaobject>)>;
class CameraView : public jni::HybridClass<CameraView> {
public:
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/CameraView;";
static auto constexpr TAG = "VisionCamera";
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis);
static void registerNatives();
// TODO: Use template<> to avoid heap allocation for std::function<>
void setFrameProcessor(const TFrameProcessor&& frameProcessor);
void unsetFrameProcessor();
private:
friend HybridBase;
jni::global_ref<CameraView::javaobject> javaPart_;
TFrameProcessor frameProcessor_;
void frameProcessorCallback(const jni::alias_ref<JImageProxy::javaobject>& frame);
explicit CameraView(jni::alias_ref<CameraView::jhybridobject> jThis) :
javaPart_(jni::make_global(jThis)),
frameProcessor_(nullptr)
{}
};
} // namespace vision

2
android/src/main/cpp/FrameHostObject.cpp
View File

@ -18,7 +18,7 @@ namespace vision {
using namespace facebook;
FrameHostObject::FrameHostObject(jni::alias_ref<JImageProxy::javaobject> image): frame(make_global(image)), _refCount(0) { }
FrameHostObject::FrameHostObject(const jni::alias_ref<JFrame::javaobject>& frame): frame(make_global(frame)), _refCount(0) { }
FrameHostObject::~FrameHostObject() {
// Hermes' Garbage Collector (Hades GC) calls destructors on a separate Thread

6
android/src/main/cpp/FrameHostObject.h
View File

@ -11,15 +11,15 @@
#include <string>
#include <mutex>
#include "java-bindings/JImageProxy.h"
#include "java-bindings/JFrame.h"
namespace vision {
using namespace facebook;
class JSI_EXPORT FrameHostObject : public jsi::HostObject {
explicit FrameHostObject(const jni::alias_ref<JFrame::javaobject>& frame);
public:
explicit FrameHostObject(jni::alias_ref<JImageProxy::javaobject> image);
~FrameHostObject();
public:
@ -27,7 +27,7 @@ class JSI_EXPORT FrameHostObject : public jsi::HostObject {
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime &rt) override;
public:
jni::global_ref<JImageProxy> frame;
jni::global_ref<JFrame> frame;
private:
static auto constexpr TAG = "VisionCamera";

53
android/src/main/cpp/FrameProcessorPluginHostObject.cpp Normal file
View File

@ -0,0 +1,53 @@
//
// Created by Marc Rousavy on 21.07.23.
//
#include "FrameProcessorPluginHostObject.h"
#include <vector>
#include "FrameHostObject.h"
#include "JSIJNIConversion.h"
#include <string>
namespace vision {
using namespace facebook;
std::vector<jsi::PropNameID> FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime& runtime) {
std::vector<jsi::PropNameID> result;
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("call")));
return result;
}
jsi::Value FrameProcessorPluginHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
auto name = propName.utf8(runtime);
if (name == "call") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "call"),
2,
[=](jsi::Runtime &runtime,
const jsi::Value &thisValue,
const jsi::Value *arguments,
size_t count) -> jsi::Value {
// Frame is first argument
auto frameHostObject = arguments[0].asObject(runtime).asHostObject<FrameHostObject>(runtime);
auto frame = frameHostObject->frame;
// Options are second argument (possibly undefined)
local_ref<react::ReadableNativeMap::javaobject> options = nullptr;
if (count > 1) {
options = JSIJNIConversion::convertJSIObjectToJNIMap(runtime, arguments[1].asObject(runtime));
}
// Call actual plugin
auto result = _plugin->callback(frame, options);
// Convert result value to jsi::Value (possibly undefined)
return JSIJNIConversion::convertJNIObjectToJSIValue(runtime, result);
});
}
return jsi::Value::undefined();
}
} // namespace vision

32
android/src/main/cpp/FrameProcessorPluginHostObject.h Normal file
View File

@ -0,0 +1,32 @@
//
// Created by Marc Rousavy on 21.07.23.
//
#pragma once
#include <jsi/jsi.h>
#include "java-bindings/JFrameProcessorPlugin.h"
#include <memory>
#include <ReactCommon/CallInvoker.h>
#include <fbjni/fbjni.h>
#include <vector>
namespace vision {
using namespace facebook;
class FrameProcessorPluginHostObject: public jsi::HostObject {
public:
explicit FrameProcessorPluginHostObject(jni::alias_ref<JFrameProcessorPlugin::javaobject> plugin):
_plugin(make_global(plugin)) { }
~FrameProcessorPluginHostObject() { }
public:
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& runtime) override;
jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& name) override;
private:
jni::global_ref<JFrameProcessorPlugin::javaobject> _plugin;
};
} // namespace vision

245
android/src/main/cpp/FrameProcessorRuntimeManager.cpp
View File

@ -1,245 +0,0 @@
//
// Created by Marc Rousavy on 11.06.21.
//
#include "FrameProcessorRuntimeManager.h"
#include <android/log.h>
#include <jni.h>
#include <utility>
#include <string>
#include <react-native-worklets/WKTJsiWorklet.h>
#include <react-native-worklets/WKTJsiHostObject.h>
#include "CameraView.h"
#include "FrameHostObject.h"
#include "JSIJNIConversion.h"
#include "java-bindings/JImageProxy.h"
#include "java-bindings/JFrameProcessorPlugin.h"
#include "JSITypedArray.h"
namespace vision {
// type aliases
using TSelf = local_ref<HybridClass<vision::FrameProcessorRuntimeManager>::jhybriddata>;
using TJSCallInvokerHolder = jni::alias_ref<facebook::react::CallInvokerHolder::javaobject>;
using TAndroidScheduler = jni::alias_ref<VisionCameraScheduler::javaobject>;
FrameProcessorRuntimeManager::FrameProcessorRuntimeManager(jni::alias_ref<FrameProcessorRuntimeManager::jhybridobject> jThis,
jsi::Runtime* jsRuntime,
std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker,
std::shared_ptr<vision::VisionCameraScheduler> scheduler) :
javaPart_(jni::make_global(jThis)),
_jsRuntime(jsRuntime) {
auto runOnJS = [jsCallInvoker](std::function<void()>&& f) {
// Run on React JS Runtime
jsCallInvoker->invokeAsync(std::move(f));
};
auto runOnWorklet = [scheduler](std::function<void()>&& f) {
// Run on Frame Processor Worklet Runtime
scheduler->dispatchAsync(std::move(f));
};
_workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera",
jsRuntime,
runOnJS,
runOnWorklet);
}
// JNI binding
void vision::FrameProcessorRuntimeManager::registerNatives() {
registerHybrid({
makeNativeMethod("initHybrid",
FrameProcessorRuntimeManager::initHybrid),
makeNativeMethod("installJSIBindings",
FrameProcessorRuntimeManager::installJSIBindings),
makeNativeMethod("registerPlugin",
FrameProcessorRuntimeManager::registerPlugin),
});
}
// JNI init
TSelf vision::FrameProcessorRuntimeManager::initHybrid(
alias_ref<jhybridobject> jThis,
jlong jsRuntimePointer,
TJSCallInvokerHolder jsCallInvokerHolder,
TAndroidScheduler androidScheduler) {
__android_log_write(ANDROID_LOG_INFO, TAG,
"Initializing FrameProcessorRuntimeManager...");
// cast from JNI hybrid objects to C++ instances
auto jsRuntime = reinterpret_cast<jsi::Runtime*>(jsRuntimePointer);
auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker();
auto scheduler = std::shared_ptr<VisionCameraScheduler>(androidScheduler->cthis());
return makeCxxInstance(jThis, jsRuntime, jsCallInvoker, scheduler);
}
global_ref<CameraView::javaobject> FrameProcessorRuntimeManager::findCameraViewById(int viewId) {
static const auto findCameraViewByIdMethod = javaPart_->getClass()->getMethod<CameraView(jint)>("findCameraViewById");
auto weakCameraView = findCameraViewByIdMethod(javaPart_.get(), viewId);
return make_global(weakCameraView);
}
void FrameProcessorRuntimeManager::logErrorToJS(const std::string& message) {
if (!_workletContext) {
return;
}
// Call console.error() on JS Thread
_workletContext->invokeOnJsThread([message](jsi::Runtime& runtime) {
auto consoleError = runtime
.global()
.getPropertyAsObject(runtime, "console")
.getPropertyAsFunction(runtime, "error");
consoleError.call(runtime, jsi::String::createFromUtf8(runtime, message));
});
}
void FrameProcessorRuntimeManager::setFrameProcessor(jsi::Runtime& runtime,
int viewTag,
const jsi::Value& frameProcessor) {
__android_log_write(ANDROID_LOG_INFO, TAG, "Setting new Frame Processor...");
if (!_workletContext) {
throw jsi::JSError(runtime,
"setFrameProcessor(..): VisionCamera's Worklet Context is not yet initialized!");
}
// find camera view
auto cameraView = findCameraViewById(viewTag);
// convert jsi::Function to a Worklet (can be shared across runtimes)
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, frameProcessor);
auto workletInvoker = std::make_shared<RNWorklet::WorkletInvoker>(worklet);
_workletContext->invokeOnWorkletThread([=](RNWorklet::JsiWorkletContext*, jsi::Runtime& rt) {
// Set Frame Processor as callable C++ lambda - this will then call the Worklet
cameraView->cthis()->setFrameProcessor([this, workletInvoker, &rt](jni::alias_ref<JImageProxy::javaobject> frame) {
try {
// create HostObject which holds the Frame (JImageProxy)
auto frameHostObject = std::make_shared<FrameHostObject>(frame);
auto argument = jsi::Object::createFromHostObject(rt, frameHostObject);
jsi::Value jsValue(std::move(argument));
// Call the Worklet on the Worklet Runtime
workletInvoker->call(rt, jsi::Value::undefined(), &jsValue, 1);
} catch (jsi::JSError& jsError) {
// Worklet threw a JS Error, catch it and log it to JS.
auto message = "Frame Processor threw an error: " + jsError.getMessage();
__android_log_write(ANDROID_LOG_ERROR, TAG, message.c_str());
this->logErrorToJS(message);
}
});
});
}
void FrameProcessorRuntimeManager::unsetFrameProcessor(int viewTag) {
__android_log_write(ANDROID_LOG_INFO, TAG, "Removing Frame Processor...");
// find camera view
auto cameraView = findCameraViewById(viewTag);
// call Java method to unset frame processor
cameraView->cthis()->unsetFrameProcessor();
}
// actual JSI installer
void FrameProcessorRuntimeManager::installJSIBindings() {
__android_log_write(ANDROID_LOG_INFO, TAG, "Installing JSI bindings...");
if (_jsRuntime == nullptr) {
__android_log_write(ANDROID_LOG_ERROR, TAG,
"JS-Runtime was null, Frame Processor JSI bindings could not be installed!");
return;
}
auto& jsiRuntime = *_jsRuntime;
// HostObject that attaches the cache to the lifecycle of the Runtime. On Runtime destroy, we destroy the cache.
auto propNameCacheObject = std::make_shared<vision::InvalidateCacheOnDestroy>(jsiRuntime);
jsiRuntime.global().setProperty(jsiRuntime,
"__visionCameraPropNameCache",
jsi::Object::createFromHostObject(jsiRuntime, propNameCacheObject));
auto setFrameProcessor = JSI_HOST_FUNCTION_LAMBDA {
__android_log_write(ANDROID_LOG_INFO, TAG, "Setting new Frame Processor...");
double viewTag = arguments[0].asNumber();
const jsi::Value& frameProcessor = arguments[1];
this->setFrameProcessor(runtime, static_cast<int>(viewTag), frameProcessor);
return jsi::Value::undefined();
};
jsiRuntime.global().setProperty(jsiRuntime,
"setFrameProcessor",
jsi::Function::createFromHostFunction(
jsiRuntime,
jsi::PropNameID::forAscii(jsiRuntime,
"setFrameProcessor"),
2, // viewTag, frameProcessor
setFrameProcessor));
auto unsetFrameProcessor = JSI_HOST_FUNCTION_LAMBDA {
__android_log_write(ANDROID_LOG_INFO, TAG, "Removing Frame Processor...");
auto viewTag = arguments[0].asNumber();
this->unsetFrameProcessor(static_cast<int>(viewTag));
return jsi::Value::undefined();
};
jsiRuntime.global().setProperty(jsiRuntime,
"unsetFrameProcessor",
jsi::Function::createFromHostFunction(
jsiRuntime,
jsi::PropNameID::forAscii(jsiRuntime,
"unsetFrameProcessor"),
1, // viewTag
unsetFrameProcessor));
__android_log_write(ANDROID_LOG_INFO, TAG, "Finished installing JSI bindings!");
}
void FrameProcessorRuntimeManager::registerPlugin(alias_ref<JFrameProcessorPlugin::javaobject> plugin) {
auto& runtime = *_jsRuntime;
// we need a strong reference on the plugin, make_global does that.
auto pluginGlobal = make_global(plugin);
auto pluginName = pluginGlobal->getName();
__android_log_print(ANDROID_LOG_INFO, TAG, "Installing Frame Processor Plugin \"%s\"...", pluginName.c_str());
if (!runtime.global().hasProperty(runtime, "FrameProcessorPlugins")) {
runtime.global().setProperty(runtime, "FrameProcessorPlugins", jsi::Object(runtime));
}
jsi::Object frameProcessorPlugins = runtime.global().getPropertyAsObject(runtime, "FrameProcessorPlugins");
auto function = [pluginGlobal](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
size_t count) -> jsi::Value {
// Unbox object and get typed HostObject
auto boxedHostObject = arguments[0].asObject(runtime).asHostObject(runtime);
auto frameHostObject = dynamic_cast<FrameHostObject*>(boxedHostObject.get());
// parse params - we are offset by `1` because the frame is the first parameter.
auto params = JArrayClass<jobject>::newArray(count - 1);
for (size_t i = 1; i < count; i++) {
params->setElement(i - 1, JSIJNIConversion::convertJSIValueToJNIObject(runtime, arguments[i]));
}
// call implemented virtual method
auto result = pluginGlobal->callback(frameHostObject->frame, params);
// convert result from JNI to JSI value
return JSIJNIConversion::convertJNIObjectToJSIValue(runtime, result);
};
// Assign it to the Proxy.
// A FP Plugin called "example_plugin" can be now called from JS using "FrameProcessorPlugins.example_plugin(frame)"
frameProcessorPlugins.setProperty(runtime,
pluginName.c_str(),
jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forAscii(runtime, pluginName),
1, // frame
function));
}
} // namespace vision

54
android/src/main/cpp/FrameProcessorRuntimeManager.h
View File

@ -1,54 +0,0 @@
//
// Created by Marc Rousavy on 11.06.21.
//
#pragma once
#include <fbjni/fbjni.h>
#include <jsi/jsi.h>
#include <memory>
#include <string>
#include <ReactCommon/CallInvokerHolder.h>
#include <react-native-worklets/WKTJsiWorkletContext.h>
#include "CameraView.h"
#include "VisionCameraScheduler.h"
#include "java-bindings/JFrameProcessorPlugin.h"
namespace vision {
using namespace facebook;
class FrameProcessorRuntimeManager : public jni::HybridClass<FrameProcessorRuntimeManager> {
public:
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager;";
static auto constexpr TAG = "VisionCamera";
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis,
jlong jsContext,
jni::alias_ref<facebook::react::CallInvokerHolder::javaobject> jsCallInvokerHolder,
jni::alias_ref<vision::VisionCameraScheduler::javaobject> androidScheduler);
static void registerNatives();
explicit FrameProcessorRuntimeManager(jni::alias_ref<FrameProcessorRuntimeManager::jhybridobject> jThis,
jsi::Runtime* jsRuntime,
std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker,
std::shared_ptr<vision::VisionCameraScheduler> scheduler);
private:
friend HybridBase;
jni::global_ref<FrameProcessorRuntimeManager::javaobject> javaPart_;
jsi::Runtime* _jsRuntime;
std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;
jni::global_ref<CameraView::javaobject> findCameraViewById(int viewId);
void installJSIBindings();
void registerPlugin(alias_ref<JFrameProcessorPlugin::javaobject> plugin);
void logErrorToJS(const std::string& message);
void setFrameProcessor(jsi::Runtime& runtime,
int viewTag,
const jsi::Value& frameProcessor);
void unsetFrameProcessor(int viewTag);
};
} // namespace vision

81
android/src/main/cpp/JSIJNIConversion.cpp
View File

@ -21,7 +21,7 @@
#include <folly/dynamic.h>
#include "FrameHostObject.h"
#include "java-bindings/JImageProxy.h"
#include "java-bindings/JFrame.h"
#include "java-bindings/JArrayList.h"
#include "java-bindings/JHashMap.h"
@ -29,75 +29,9 @@ namespace vision {
using namespace facebook;
jobject JSIJNIConversion::convertJSIValueToJNIObject(jsi::Runtime &runtime, const jsi::Value &value) {
if (value.isBool()) {
// jsi::Bool
auto boolean = jni::JBoolean::valueOf(value.getBool());
return boolean.release();
} else if (value.isNumber()) {
// jsi::Number
auto number = jni::JDouble::valueOf(value.getNumber());
return number.release();
} else if (value.isNull() || value.isUndefined()) {
// jsi::undefined
return nullptr;
} else if (value.isString()) {
// jsi::String
auto string = jni::make_jstring(value.getString(runtime).utf8(runtime));
return string.release();
} else if (value.isObject()) {
// jsi::Object
auto object = value.asObject(runtime);
if (object.isArray(runtime)) {
// jsi::Array
auto dynamic = jsi::dynamicFromValue(runtime, value);
auto nativeArray = react::ReadableNativeArray::newObjectCxxArgs(std::move(dynamic));
return nativeArray.release();
} else if (object.isHostObject(runtime)) {
// jsi::HostObject
auto boxedHostObject = object.getHostObject(runtime);
auto hostObject = dynamic_cast<FrameHostObject*>(boxedHostObject.get());
if (hostObject != nullptr) {
// return jni local_ref to the JImageProxy
return hostObject->frame.get();
} else {
// it's different kind of HostObject. We don't support it.
throw std::runtime_error("Received an unknown HostObject! Cannot convert to a JNI value.");
}
} else if (object.isFunction(runtime)) {
// jsi::Function
// TODO: Convert Function to Callback
throw std::runtime_error("Cannot convert a JS Function to a JNI value (yet)!");
} else {
// jsi::Object
auto dynamic = jsi::dynamicFromValue(runtime, value);
auto map = react::ReadableNativeMap::createWithContents(std::move(dynamic));
return map.release();
}
} else {
// unknown jsi type!
auto stringRepresentation = value.toString(runtime).utf8(runtime);
auto message = "Received unknown JSI value! (" + stringRepresentation + ") Cannot convert to a JNI value.";
throw std::runtime_error(message);
}
jni::local_ref<react::ReadableNativeMap::javaobject> JSIJNIConversion::convertJSIObjectToJNIMap(jsi::Runtime& runtime, const jsi::Object& object) {
auto dynamic = jsi::dynamicFromValue(runtime, jsi::Value(runtime, object));
return react::ReadableNativeMap::createWithContents(std::move(dynamic));
}
jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, const jni::local_ref<jobject>& object) {
@ -178,10 +112,9 @@ jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, c
auto hashMap = toHashMapFunc(object.get());
return convertJNIObjectToJSIValue(runtime, hashMap);
} else if (object->isInstanceOf(JImageProxy::javaClassStatic())) {
// ImageProxy
auto frame = static_ref_cast<JImageProxy>(object);
} else if (object->isInstanceOf(JFrame::javaClassStatic())) {
// Frame
auto frame = static_ref_cast<JFrame>(object);
// box into HostObject
auto hostObject = std::make_shared<FrameHostObject>(frame);

3
android/src/main/cpp/JSIJNIConversion.h
View File

@ -7,6 +7,7 @@
#include <jsi/jsi.h>
#include <jni.h>
#include <fbjni/fbjni.h>
#include <react/jni/ReadableNativeMap.h>
namespace vision {
@ -14,7 +15,7 @@ namespace JSIJNIConversion {
using namespace facebook;
jobject convertJSIValueToJNIObject(jsi::Runtime& runtime, const jsi::Value& value);
jni::local_ref<react::ReadableNativeMap::javaobject> convertJSIObjectToJNIMap(jsi::Runtime& runtime, const jsi::Object& object);
jsi::Value convertJNIObjectToJSIValue(jsi::Runtime& runtime, const jni::local_ref<jobject>& object);

14
android/src/main/cpp/VisionCamera.cpp
View File

@ -1,13 +1,15 @@
#include <jni.h>
#include <fbjni/fbjni.h>
#include "FrameProcessorRuntimeManager.h"
#include "CameraView.h"
#include "VisionCameraScheduler.h"
#include "java-bindings/JVisionCameraScheduler.h"
#include "java-bindings/JFrameProcessor.h"
#include "java-bindings/JVisionCameraProxy.h"
#include "VisionCameraProxy.h"
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) {
return facebook::jni::initialize(vm, [] {
vision::FrameProcessorRuntimeManager::registerNatives();
vision::CameraView::registerNatives();
vision::VisionCameraScheduler::registerNatives();
vision::VisionCameraInstaller::registerNatives();
vision::JFrameProcessor::registerNatives();
vision::JVisionCameraProxy::registerNatives();
vision::JVisionCameraScheduler::registerNatives();
});
}

152
android/src/main/cpp/VisionCameraProxy.cpp Normal file
View File

@ -0,0 +1,152 @@
//
// Created by Marc Rousavy on 21.07.23.
//
#include "VisionCameraProxy.h"
#include <jsi/jsi.h>
#include "java-bindings/JFrameProcessor.h"
#include "java-bindings/JFrameProcessorPlugin.h"
#include "JSIJNIConversion.h"
#include <android/log.h>
#include <fbjni/fbjni.h>
#include "JSITypedArray.h"
#include "FrameProcessorPluginHostObject.h"
#include <vector>
#include <string>
#include <memory>
namespace vision {
using namespace facebook;
VisionCameraProxy::VisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::javaobject>& javaProxy) {
_javaProxy = make_global(javaProxy);
}
VisionCameraProxy::~VisionCameraProxy() {
__android_log_write(ANDROID_LOG_INFO, TAG, "Destroying Context...");
// Destroy ArrayBuffer cache for both the JS and the Worklet Runtime.
auto workletContext = _javaProxy->cthis()->getWorkletContext();
invalidateArrayBufferCache(*workletContext->getJsRuntime());
invalidateArrayBufferCache(workletContext->getWorkletRuntime());
}
std::vector<jsi::PropNameID> VisionCameraProxy::getPropertyNames(jsi::Runtime& runtime) {
std::vector<jsi::PropNameID> result;
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("setFrameProcessor")));
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("removeFrameProcessor")));
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("getFrameProcessorPlugin")));
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("isSkiaEnabled")));
return result;
}
void VisionCameraProxy::setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& object) {
auto frameProcessorType = object.getProperty(runtime, "type").asString(runtime).utf8(runtime);
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, object.getProperty(runtime, "frameProcessor"));
auto workletContext = _javaProxy->cthis()->getWorkletContext();
jni::local_ref<JFrameProcessor::javaobject> frameProcessor;
if (frameProcessorType == "frame-processor") {
frameProcessor = JFrameProcessor::create(worklet, workletContext);
} else if (frameProcessorType == "skia-frame-processor") {
#if VISION_CAMERA_ENABLE_SKIA
throw std::runtime_error("system/skia-unavailable: Skia is not yet implemented on Android!");
#else
throw std::runtime_error("system/skia-unavailable: Skia is not installed!");
#endif
} else {
throw std::runtime_error("Unknown FrameProcessor.type passed! Received: " + frameProcessorType);
}
_javaProxy->cthis()->setFrameProcessor(viewTag, make_global(frameProcessor));
}
void VisionCameraProxy::removeFrameProcessor(int viewTag) {
_javaProxy->cthis()->removeFrameProcessor(viewTag);
}
jsi::Value VisionCameraProxy::getFrameProcessorPlugin(jsi::Runtime& runtime,
const std::string& name,
const jsi::Object& jsOptions) {
auto options = JSIJNIConversion::convertJSIObjectToJNIMap(runtime, jsOptions);
auto plugin = _javaProxy->cthis()->getFrameProcessorPlugin(name, options);
auto pluginHostObject = std::make_shared<FrameProcessorPluginHostObject>(plugin);
return jsi::Object::createFromHostObject(runtime, pluginHostObject);
}
jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
auto name = propName.utf8(runtime);
if (name == "isSkiaEnabled") {
#ifdef VISION_CAMERA_ENABLE_SKIA
return jsi::Value(true);
#else
return jsi::Value(false);
#endif
}
if (name == "setFrameProcessor") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "setFrameProcessor"),
1,
[this](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
size_t count) -> jsi::Value {
auto viewTag = arguments[0].asNumber();
auto object = arguments[1].asObject(runtime);
this->setFrameProcessor(static_cast<int>(viewTag), runtime, object);
return jsi::Value::undefined();
});
}
if (name == "removeFrameProcessor") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "removeFrameProcessor"),
1,
[this](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
size_t count) -> jsi::Value {
auto viewTag = arguments[0].asNumber();
this->removeFrameProcessor(static_cast<int>(viewTag));
return jsi::Value::undefined();
});
}
if (name == "getFrameProcessorPlugin") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "getFrameProcessorPlugin"),
1,
[this](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
size_t count) -> jsi::Value {
if (count < 1 || !arguments[0].isString()) {
throw jsi::JSError(runtime, "First argument needs to be a string (pluginName)!");
}
auto pluginName = arguments[0].asString(runtime).utf8(runtime);
auto options = count > 1 ? arguments[1].asObject(runtime) : jsi::Object(runtime);
return this->getFrameProcessorPlugin(runtime, pluginName, options);
});
}
return jsi::Value::undefined();
}
void VisionCameraInstaller::install(jni::alias_ref<jni::JClass>,
jni::alias_ref<JVisionCameraProxy::javaobject> proxy) {
// global.VisionCameraProxy
auto visionCameraProxy = std::make_shared<VisionCameraProxy>(proxy);
jsi::Runtime& runtime = *proxy->cthis()->getWorkletContext()->getJsRuntime();
runtime.global().setProperty(runtime,
"VisionCameraProxy",
jsi::Object::createFromHostObject(runtime, visionCameraProxy));
}
} // namespace vision

51
android/src/main/cpp/VisionCameraProxy.h Normal file
View File

@ -0,0 +1,51 @@
//
// Created by Marc Rousavy on 21.07.23.
//
#pragma once
#include <jsi/jsi.h>
#include "java-bindings/JVisionCameraScheduler.h"
#include "java-bindings/JVisionCameraProxy.h"
#include <vector>
#include <string>
namespace vision {
using namespace facebook;
class VisionCameraProxy: public jsi::HostObject {
public:
explicit VisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::javaobject>& javaProxy);
~VisionCameraProxy();
public:
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& runtime) override;
jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& name) override;
private:
void setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& frameProcessor);
void removeFrameProcessor(int viewTag);
jsi::Value getFrameProcessorPlugin(jsi::Runtime& runtime, const std::string& name, const jsi::Object& options);
private:
jni::global_ref<JVisionCameraProxy::javaobject> _javaProxy;
static constexpr const char* TAG = "VisionCameraProxy";
};
class VisionCameraInstaller: public jni::JavaClass<VisionCameraInstaller> {
public:
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraInstaller;";
static void registerNatives() {
javaClassStatic()->registerNatives({
makeNativeMethod("install", VisionCameraInstaller::install)
});
}
static void install(jni::alias_ref<jni::JClass> clazz,
jni::alias_ref<JVisionCameraProxy::javaobject> proxy);
};
} // namespace vision

65
android/src/main/cpp/java-bindings/JFrame.cpp Normal file
View File

@ -0,0 +1,65 @@
//
// Created by Marc on 21.07.2023.
//
#include "JFrame.h"
#include <jni.h>
#include <fbjni/fbjni.h>
namespace vision {
using namespace facebook;
using namespace jni;
int JFrame::getWidth() const {
static const auto getWidthMethod = getClass()->getMethod<jint()>("getWidth");
return getWidthMethod(self());
}
int JFrame::getHeight() const {
static const auto getWidthMethod = getClass()->getMethod<jint()>("getHeight");
return getWidthMethod(self());
}
bool JFrame::getIsValid() const {
static const auto getIsValidMethod = getClass()->getMethod<jboolean()>("getIsValid");
return getIsValidMethod(self());
}
bool JFrame::getIsMirrored() const {
static const auto getIsMirroredMethod = getClass()->getMethod<jboolean()>("getIsMirrored");
return getIsMirroredMethod(self());
}
jlong JFrame::getTimestamp() const {
static const auto getTimestampMethod = getClass()->getMethod<jlong()>("getTimestamp");
return getTimestampMethod(self());
}
local_ref<JString> JFrame::getOrientation() const {
static const auto getOrientationMethod = getClass()->getMethod<JString()>("getOrientation");
return getOrientationMethod(self());
}
int JFrame::getPlanesCount() const {
static const auto getPlanesCountMethod = getClass()->getMethod<jint()>("getPlanesCount");
return getPlanesCountMethod(self());
}
int JFrame::getBytesPerRow() const {
static const auto getBytesPerRowMethod = getClass()->getMethod<jint()>("getBytesPerRow");
return getBytesPerRowMethod(self());
}
local_ref<JArrayByte> JFrame::toByteArray() const {
static const auto toByteArrayMethod = getClass()->getMethod<JArrayByte()>("toByteArray");
return toByteArrayMethod(self());
}
void JFrame::close() {
static const auto closeMethod = getClass()->getMethod<void()>("close");
closeMethod(self());
}
} // namespace vision

6
android/src/main/cpp/java-bindings/JImageProxy.h → android/src/main/cpp/java-bindings/JFrame.h
View File

@ -1,5 +1,5 @@
//
// Created by Marc on 19/06/2021.
// Created by Marc on 21.07.2023.
//
#pragma once
@ -12,8 +12,8 @@ namespace vision {
using namespace facebook;
using namespace jni;
struct JImageProxy : public JavaClass<JImageProxy> {
static constexpr auto kJavaDescriptor = "Landroidx/camera/core/ImageProxy;";
struct JFrame : public JavaClass<JFrame> {
static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/Frame;";
public:
int getWidth() const;

65
android/src/main/cpp/java-bindings/JFrameProcessor.cpp Normal file
View File

@ -0,0 +1,65 @@
//
// Created by Marc Rousavy on 29.09.21.
//
#include "JFrameProcessor.h"
#include <jni.h>
#include <fbjni/fbjni.h>
#include <utility>
#include "JFrame.h"
namespace vision {
using namespace facebook;
using namespace jni;
void JFrameProcessor::registerNatives() {
registerHybrid({
makeNativeMethod("call", JFrameProcessor::call)
});
}
using TSelf = jni::local_ref<JFrameProcessor::javaobject>;
JFrameProcessor::JFrameProcessor(std::shared_ptr<RNWorklet::JsiWorklet> worklet,
std::shared_ptr<RNWorklet::JsiWorkletContext> context) {
_workletContext = std::move(context);
_workletInvoker = std::make_shared<RNWorklet::WorkletInvoker>(worklet);
}
TSelf JFrameProcessor::create(const std::shared_ptr<RNWorklet::JsiWorklet>& worklet,
const std::shared_ptr<RNWorklet::JsiWorkletContext>& context) {
return JFrameProcessor::newObjectCxxArgs(worklet, context);
}
void JFrameProcessor::callWithFrameHostObject(const std::shared_ptr<FrameHostObject>& frameHostObject) const {
// Call the Frame Processor on the Worklet Runtime
jsi::Runtime& runtime = _workletContext->getWorkletRuntime();
try {
// Wrap HostObject as JSI Value
auto argument = jsi::Object::createFromHostObject(runtime, frameHostObject);
jsi::Value jsValue(std::move(argument));
// Call the Worklet with the Frame JS Host Object as an argument
_workletInvoker->call(runtime, jsi::Value::undefined(), &jsValue, 1);
} catch (jsi::JSError& jsError) {
// JS Error occured, print it to console.
const std::string& message = jsError.getMessage();
_workletContext->invokeOnJsThread([message](jsi::Runtime& jsRuntime) {
auto logFn = jsRuntime.global().getPropertyAsObject(jsRuntime, "console").getPropertyAsFunction(jsRuntime, "error");
logFn.call(jsRuntime, jsi::String::createFromUtf8(jsRuntime, "Frame Processor threw an error: " + message));
});
}
}
void JFrameProcessor::call(jni::alias_ref<JFrame::javaobject> frame) {
// Create the Frame Host Object wrapping the internal Frame
auto frameHostObject = std::make_shared<FrameHostObject>(frame);
callWithFrameHostObject(frameHostObject);
}
} // namespace vision

49
android/src/main/cpp/java-bindings/JFrameProcessor.h Normal file
View File

@ -0,0 +1,49 @@
//
// Created by Marc Rousavy on 29.09.21
//
#pragma once
#include <string>
#include <memory>
#include <jni.h>
#include <fbjni/fbjni.h>
#include <react-native-worklets/WKTJsiWorklet.h>
#include <react-native-worklets/WKTJsiHostObject.h>
#include "JFrame.h"
#include "FrameHostObject.h"
namespace vision {
using namespace facebook;
struct JFrameProcessor : public jni::HybridClass<JFrameProcessor> {
public:
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessor;";
static void registerNatives();
static jni::local_ref<JFrameProcessor::javaobject> create(const std::shared_ptr<RNWorklet::JsiWorklet>& worklet,
const std::shared_ptr<RNWorklet::JsiWorkletContext>& context);
public:
/**
* Call the JS Frame Processor.
*/
void call(alias_ref<JFrame::javaobject> frame);
private:
// Private constructor. Use `create(..)` to create new instances.
explicit JFrameProcessor(std::shared_ptr<RNWorklet::JsiWorklet> worklet,
std::shared_ptr<RNWorklet::JsiWorkletContext> context);
private:
void callWithFrameHostObject(const std::shared_ptr<FrameHostObject>& frameHostObject) const;
private:
friend HybridBase;
std::shared_ptr<RNWorklet::WorkletInvoker> _workletInvoker;
std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;
};
} // namespace vision

13
android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp
View File

@ -12,19 +12,14 @@ namespace vision {
using namespace facebook;
using namespace jni;
using TCallback = jobject(alias_ref<JImageProxy::javaobject>, alias_ref<JArrayClass<jobject>>);
using TCallback = jobject(alias_ref<JFrame::javaobject>, alias_ref<react::ReadableNativeMap::javaobject> params);
local_ref<jobject> JFrameProcessorPlugin::callback(alias_ref<JImageProxy::javaobject> image,
alias_ref<JArrayClass<jobject>> params) const {
local_ref<jobject> JFrameProcessorPlugin::callback(const alias_ref<JFrame::javaobject>& frame,
const alias_ref<react::ReadableNativeMap::javaobject>& params) const {
auto callbackMethod = getClass()->getMethod<TCallback>("callback");
auto result = callbackMethod(self(), image, params);
auto result = callbackMethod(self(), frame, params);
return make_local(result);
}
std::string JFrameProcessorPlugin::getName() const {
auto getNameMethod = getClass()->getMethod<jstring()>("getName");
return getNameMethod(self())->toStdString();
}
} // namespace vision

11
android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h
View File

@ -7,8 +7,9 @@
#include <jni.h>
#include <fbjni/fbjni.h>
#include <string>
#include <react/jni/ReadableNativeMap.h>
#include "JImageProxy.h"
#include "JFrame.h"
namespace vision {
@ -22,12 +23,8 @@ struct JFrameProcessorPlugin : public JavaClass<JFrameProcessorPlugin> {
/**
* Call the plugin.
*/
local_ref<jobject> callback(alias_ref<JImageProxy::javaobject> image,
alias_ref<JArrayClass<jobject>> params) const;
/**
* Get the user-defined name of the Frame Processor Plugin
*/
std::string getName() const;
local_ref<jobject> callback(const alias_ref<JFrame::javaobject>& frame,
const alias_ref<react::ReadableNativeMap::javaobject>& params) const;
};
} // namespace vision

78
android/src/main/cpp/java-bindings/JImageProxy.cpp
View File

@ -1,78 +0,0 @@
//
// Created by Marc Rousavy on 22.06.21.
//
#include "JImageProxy.h"
#include <jni.h>
#include <fbjni/fbjni.h>
namespace vision {
using namespace facebook;
using namespace jni;
int JImageProxy::getWidth() const {
static const auto getWidthMethod = getClass()->getMethod<jint()>("getWidth");
return getWidthMethod(self());
}
int JImageProxy::getHeight() const {
static const auto getWidthMethod = getClass()->getMethod<jint()>("getHeight");
return getWidthMethod(self());
}
alias_ref<JClass> getUtilsClass() {
static const auto ImageProxyUtilsClass = findClassStatic("com/mrousavy/camera/frameprocessor/ImageProxyUtils");
return ImageProxyUtilsClass;
}
bool JImageProxy::getIsValid() const {
auto utilsClass = getUtilsClass();
static const auto isImageProxyValidMethod = utilsClass->getStaticMethod<jboolean(JImageProxy::javaobject)>("isImageProxyValid");
return isImageProxyValidMethod(utilsClass, self());
}
bool JImageProxy::getIsMirrored() const {
auto utilsClass = getUtilsClass();
static const auto isImageProxyMirroredMethod = utilsClass->getStaticMethod<jboolean(JImageProxy::javaobject)>("isImageProxyMirrored");
return isImageProxyMirroredMethod(utilsClass, self());
}
jlong JImageProxy::getTimestamp() const {
auto utilsClass = getUtilsClass();
static const auto getTimestampMethod = utilsClass->getStaticMethod<jlong(JImageProxy::javaobject)>("getTimestamp");
return getTimestampMethod(utilsClass, self());
}
local_ref<JString> JImageProxy::getOrientation() const {
auto utilsClass = getUtilsClass();
static const auto getOrientationMethod = utilsClass->getStaticMethod<JString(JImageProxy::javaobject)>("getOrientation");
return getOrientationMethod(utilsClass, self());
}
int JImageProxy::getPlanesCount() const {
auto utilsClass = getUtilsClass();
static const auto getPlanesCountMethod = utilsClass->getStaticMethod<jint(JImageProxy::javaobject)>("getPlanesCount");
return getPlanesCountMethod(utilsClass, self());
}
int JImageProxy::getBytesPerRow() const {
auto utilsClass = getUtilsClass();
static const auto getBytesPerRowMethod = utilsClass->getStaticMethod<jint(JImageProxy::javaobject)>("getBytesPerRow");
return getBytesPerRowMethod(utilsClass, self());
}
local_ref<JArrayByte> JImageProxy::toByteArray() const {
auto utilsClass = getUtilsClass();
static const auto toByteArrayMethod = utilsClass->getStaticMethod<JArrayByte(JImageProxy::javaobject)>("toByteArray");
return toByteArrayMethod(utilsClass, self());
}
void JImageProxy::close() {
static const auto closeMethod = getClass()->getMethod<void()>("close");
closeMethod(self());
}
} // namespace vision

91
android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp Normal file
View File

@ -0,0 +1,91 @@
//
// Created by Marc Rousavy on 21.07.23.
//
#include "JVisionCameraProxy.h"
#include <memory>
#include <utility>
#include <string>
#include <jsi/jsi.h>
#include <react/jni/ReadableNativeMap.h>
#include <react-native-worklets/WKTJsiWorklet.h>
#include <react-native-worklets/WKTJsiWorkletContext.h>
#include "FrameProcessorPluginHostObject.h"
namespace vision {
using TSelf = local_ref<HybridClass<JVisionCameraProxy>::jhybriddata>;
using TJSCallInvokerHolder = jni::alias_ref<facebook::react::CallInvokerHolder::javaobject>;
using TScheduler = jni::alias_ref<JVisionCameraScheduler::javaobject>;
using TOptions = jni::local_ref<react::ReadableNativeMap::javaobject>;
JVisionCameraProxy::JVisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::jhybridobject>& javaThis,
jsi::Runtime* runtime,
const std::shared_ptr<facebook::react::CallInvoker>& callInvoker,
const jni::global_ref<JVisionCameraScheduler::javaobject>& scheduler) {
_javaPart = make_global(javaThis);
__android_log_write(ANDROID_LOG_INFO, TAG, "Creating Worklet Context...");
auto runOnJS = [callInvoker](std::function<void()>&& f) {
// Run on React JS Runtime
callInvoker->invokeAsync(std::move(f));
};
auto runOnWorklet = [scheduler](std::function<void()>&& f) {
// Run on Frame Processor Worklet Runtime
scheduler->cthis()->dispatchAsync([f = std::move(f)](){
f();
});
};
_workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera",
runtime,
runOnJS,
runOnWorklet);
__android_log_write(ANDROID_LOG_INFO, TAG, "Worklet Context created!");
}
void JVisionCameraProxy::setFrameProcessor(int viewTag,
const alias_ref<JFrameProcessor::javaobject>& frameProcessor) {
auto setFrameProcessorMethod = javaClassLocal()->getMethod<void(int, alias_ref<JFrameProcessor::javaobject>)>("setFrameProcessor");
setFrameProcessorMethod(_javaPart, viewTag, frameProcessor);
}
void JVisionCameraProxy::removeFrameProcessor(int viewTag) {
auto removeFrameProcessorMethod = javaClassLocal()->getMethod<void(int)>("removeFrameProcessor");
removeFrameProcessorMethod(_javaPart, viewTag);
}
local_ref<JFrameProcessorPlugin::javaobject> JVisionCameraProxy::getFrameProcessorPlugin(const std::string& name,
TOptions options) {
auto getFrameProcessorPluginMethod = javaClassLocal()->getMethod<JFrameProcessorPlugin(local_ref<jstring>, TOptions)>("getFrameProcessorPlugin");
return getFrameProcessorPluginMethod(_javaPart, make_jstring(name), std::move(options));
}
void JVisionCameraProxy::registerNatives() {
registerHybrid({
makeNativeMethod("initHybrid", JVisionCameraProxy::initHybrid)
});
}
TSelf JVisionCameraProxy::initHybrid(
alias_ref<jhybridobject> jThis,
jlong jsRuntimePointer,
TJSCallInvokerHolder jsCallInvokerHolder,
const TScheduler& scheduler) {
__android_log_write(ANDROID_LOG_INFO, TAG, "Initializing VisionCameraProxy...");
// cast from JNI hybrid objects to C++ instances
auto jsRuntime = reinterpret_cast<jsi::Runtime*>(jsRuntimePointer);
auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker();
auto sharedScheduler = make_global(scheduler);
return makeCxxInstance(jThis, jsRuntime, jsCallInvoker, sharedScheduler);
}
} // namespace vision

55
android/src/main/cpp/java-bindings/JVisionCameraProxy.h Normal file
View File

@ -0,0 +1,55 @@
//
// Created by Marc Rousavy on 21.07.23.
//
#pragma once
#include <fbjni/fbjni.h>
#include <jsi/jsi.h>
#include <react-native-worklets/WKTJsiWorkletContext.h>
#include <react/jni/ReadableNativeMap.h>
#include "JFrameProcessorPlugin.h"
#include "JVisionCameraScheduler.h"
#include "JFrameProcessor.h"
#include <string>
#include <memory>
namespace vision {
using namespace facebook;
class JVisionCameraProxy : public jni::HybridClass<JVisionCameraProxy> {
public:
static void registerNatives();
void setFrameProcessor(int viewTag,
const jni::alias_ref<JFrameProcessor::javaobject>& frameProcessor);
void removeFrameProcessor(int viewTag);
jni::local_ref<JFrameProcessorPlugin::javaobject> getFrameProcessorPlugin(const std::string& name,
jni::local_ref<react::ReadableNativeMap::javaobject> options);
public:
std::shared_ptr<RNWorklet::JsiWorkletContext> getWorkletContext() { return _workletContext; }
private:
std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;
private:
friend HybridBase;
jni::global_ref<JVisionCameraProxy::javaobject> _javaPart;
static auto constexpr TAG = "VisionCameraProxy";
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraProxy;";
explicit JVisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::jhybridobject>& javaThis,
jsi::Runtime* jsRuntime,
const std::shared_ptr<facebook::react::CallInvoker>& jsCallInvoker,
const jni::global_ref<JVisionCameraScheduler::javaobject>& scheduler);
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> javaThis,
jlong jsRuntimePointer,
jni::alias_ref<facebook::react::CallInvokerHolder::javaobject> jsCallInvokerHolder,
const jni::alias_ref<JVisionCameraScheduler::javaobject>& scheduler);
};
} // namespace vision

18
android/src/main/cpp/VisionCameraScheduler.cpp → android/src/main/cpp/java-bindings/JVisionCameraScheduler.cpp
View File

@ -2,31 +2,31 @@
// Created by Marc Rousavy on 25.07.21.
//
#include "VisionCameraScheduler.h"
#include "JVisionCameraScheduler.h"
#include <fbjni/fbjni.h>
namespace vision {
using namespace facebook;
using TSelf = jni::local_ref<VisionCameraScheduler::jhybriddata>;
using TSelf = jni::local_ref<JVisionCameraScheduler::jhybriddata>;
TSelf VisionCameraScheduler::initHybrid(jni::alias_ref<jhybridobject> jThis) {
TSelf JVisionCameraScheduler::initHybrid(jni::alias_ref<jhybridobject> jThis) {
return makeCxxInstance(jThis);
}
void VisionCameraScheduler::dispatchAsync(std::function<void()> job) {
void JVisionCameraScheduler::dispatchAsync(const std::function<void()>& job) {
// 1. add job to queue
_jobs.push(job);
scheduleTrigger();
}
void VisionCameraScheduler::scheduleTrigger() {
void JVisionCameraScheduler::scheduleTrigger() {
// 2. schedule `triggerUI` to be called on the java thread
static auto method = javaPart_->getClass()->getMethod<void()>("scheduleTrigger");
method(javaPart_.get());
}
void VisionCameraScheduler::trigger() {
void JVisionCameraScheduler::trigger() {
std::unique_lock<std::mutex> lock(_mutex);
// 3. call job we enqueued in step 1.
auto job = _jobs.front();
@ -34,10 +34,10 @@ void VisionCameraScheduler::trigger() {
_jobs.pop();
}
void VisionCameraScheduler::registerNatives() {
void JVisionCameraScheduler::registerNatives() {
registerHybrid({
makeNativeMethod("initHybrid", VisionCameraScheduler::initHybrid),
makeNativeMethod("trigger", VisionCameraScheduler::trigger),
makeNativeMethod("initHybrid", JVisionCameraScheduler::initHybrid),
makeNativeMethod("trigger", JVisionCameraScheduler::trigger),
});
}

8
android/src/main/cpp/VisionCameraScheduler.h → android/src/main/cpp/java-bindings/JVisionCameraScheduler.h
View File

@ -23,22 +23,22 @@ using namespace facebook;
* 3. The `scheduleTrigger()` Java Method will switch to the Frame Processor Java Thread and call `trigger()` on there
* 4. `trigger()` is a C++ function here that just calls the passed C++ Method from step 1.
*/
class VisionCameraScheduler : public jni::HybridClass<VisionCameraScheduler> {
class JVisionCameraScheduler : public jni::HybridClass<JVisionCameraScheduler> {
public:
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraScheduler;";
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis);
static void registerNatives();
// schedules the given job to be run on the VisionCamera FP Thread at some future point in time
void dispatchAsync(std::function<void()> job);
void dispatchAsync(const std::function<void()>& job);
private:
friend HybridBase;
jni::global_ref<VisionCameraScheduler::javaobject> javaPart_;
jni::global_ref<JVisionCameraScheduler::javaobject> javaPart_;
std::queue<std::function<void()>> _jobs;
std::mutex _mutex;
explicit VisionCameraScheduler(jni::alias_ref<VisionCameraScheduler::jhybridobject> jThis):
explicit JVisionCameraScheduler(jni::alias_ref<JVisionCameraScheduler::jhybridobject> jThis):
javaPart_(jni::make_global(jThis)) {}
// Schedules a call to `trigger` on the VisionCamera FP Thread

2
android/src/main/java/com/mrousavy/camera/CameraView+Focus.kt
View File

@ -18,7 +18,7 @@ suspend fun CameraView.focus(pointMap: ReadableMap) {
// Getting the point from the previewView needs to be run on the UI thread
val point = withContext(coroutineScope.coroutineContext) {
previewView.meteringPointFactory.createPoint(x.toFloat(), y.toFloat());
previewView.meteringPointFactory.createPoint(x.toFloat(), y.toFloat())
}
val action = FocusMeteringAction.Builder(point, FocusMeteringAction.FLAG_AF or FocusMeteringAction.FLAG_AE)

2
android/src/main/java/com/mrousavy/camera/CameraView+TakeSnapshot.kt
View File

@ -14,7 +14,7 @@ import java.io.FileOutputStream
import kotlinx.coroutines.guava.await
suspend fun CameraView.takeSnapshot(options: ReadableMap): WritableMap = coroutineScope {
val camera = camera ?: throw com.mrousavy.camera.CameraNotReadyError()
val camera = camera ?: throw CameraNotReadyError()
val enableFlash = options.getString("flash") == "on"
try {

25
android/src/main/java/com/mrousavy/camera/CameraView.kt
View File

@ -24,15 +24,16 @@ import androidx.lifecycle.*
import com.facebook.jni.HybridData
import com.facebook.proguard.annotations.DoNotStrip
import com.facebook.react.bridge.*
import com.facebook.react.uimanager.events.RCTEventEmitter
import com.mrousavy.camera.frameprocessor.FrameProcessorRuntimeManager
import com.mrousavy.camera.frameprocessor.Frame
import com.mrousavy.camera.frameprocessor.FrameProcessor
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin
import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry
import com.mrousavy.camera.utils.*
import kotlinx.coroutines.*
import kotlinx.coroutines.guava.await
import java.lang.IllegalArgumentException
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import kotlin.math.floor
import kotlin.math.max
import kotlin.math.min
@ -118,8 +119,9 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
internal var camera: Camera? = null
internal var imageCapture: ImageCapture? = null
internal var videoCapture: VideoCapture<Recorder>? = null
private var imageAnalysis: ImageAnalysis? = null
public var frameProcessor: FrameProcessor? = null
private var preview: Preview? = null
private var imageAnalysis: ImageAnalysis? = null
internal var activeVideoRecording: Recording? = null
@ -156,10 +158,7 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
private var minZoom: Float = 1f
private var maxZoom: Float = 1f
@DoNotStrip
private var mHybridData: HybridData? = null
@Suppress("LiftReturnOrAssignment", "RedundantIf")
@Suppress("RedundantIf")
internal val fallbackToSnapshot: Boolean
@SuppressLint("UnsafeOptInUsageError")
get() {
@ -190,8 +189,6 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
}
init {
mHybridData = initHybrid()
previewView = PreviewView(context)
previewView.layoutParams = LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT)
previewView.installHierarchyFitter() // If this is not called correctly, view finder will be black/blank
@ -244,9 +241,6 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
imageAnalysis?.targetRotation = outputRotation
}
private external fun initHybrid(): HybridData
private external fun frameProcessorCallback(frame: ImageProxy)
override fun getLifecycle(): Lifecycle {
return lifecycleRegistry
}
@ -461,8 +455,9 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
imageAnalysis = imageAnalysisBuilder.build().apply {
setAnalyzer(cameraExecutor) { image ->
// Call JS Frame Processor
frameProcessorCallback(image)
// frame gets closed in FrameHostObject implementation (JS ref counting)
val frame = Frame(image)
frameProcessor?.call(frame)
// ...frame gets closed in FrameHostObject implementation via JS ref counting
}
}
useCases.add(imageAnalysis!!)

59
android/src/main/java/com/mrousavy/camera/CameraViewModule.kt
View File

@ -3,16 +3,11 @@ package com.mrousavy.camera
import android.Manifest
import android.content.Context
import android.content.pm.PackageManager
import android.hardware.camera2.CameraCharacteristics
import android.hardware.camera2.CameraManager
import android.os.Build
import android.util.Log
import android.util.Size
import androidx.camera.core.CameraSelector
import androidx.camera.extensions.ExtensionMode
import androidx.camera.extensions.ExtensionsManager
import androidx.camera.lifecycle.ProcessCameraProvider
import androidx.camera.video.QualitySelector
import androidx.core.content.ContextCompat
import com.facebook.react.bridge.*
import com.facebook.react.module.annotations.ReactModule
@ -20,8 +15,9 @@ import com.facebook.react.modules.core.PermissionAwareActivity
import com.facebook.react.modules.core.PermissionListener
import com.facebook.react.uimanager.UIManagerHelper
import com.facebook.react.bridge.ReactApplicationContext
import com.mrousavy.camera.frameprocessor.VisionCameraInstaller
import java.util.concurrent.ExecutorService
import com.mrousavy.camera.frameprocessor.FrameProcessorRuntimeManager
import com.mrousavy.camera.frameprocessor.VisionCameraProxy
import com.mrousavy.camera.parsers.*
import com.mrousavy.camera.utils.*
import kotlinx.coroutines.*
@ -30,38 +26,21 @@ import java.util.concurrent.Executors
@ReactModule(name = CameraViewModule.TAG)
@Suppress("unused")
class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBaseJavaModule(reactContext) {
class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJavaModule(reactContext) {
companion object {
const val TAG = "CameraView"
var RequestCode = 10
fun parsePermissionStatus(status: Int): String {
return when (status) {
PackageManager.PERMISSION_DENIED -> "denied"
PackageManager.PERMISSION_GRANTED -> "authorized"
else -> "not-determined"
}
}
}
var frameProcessorThread: ExecutorService = Executors.newSingleThreadExecutor()
private val coroutineScope = CoroutineScope(Dispatchers.Default) // TODO: or Dispatchers.Main?
private var frameProcessorManager: FrameProcessorRuntimeManager? = null
private fun cleanup() {
if (coroutineScope.isActive) {
coroutineScope.cancel("CameraViewModule has been destroyed.")
}
}
override fun onCatalystInstanceDestroy() {
super.onCatalystInstanceDestroy()
cleanup()
}
override fun invalidate() {
super.invalidate()
cleanup()
frameProcessorThread.shutdown()
if (coroutineScope.isActive) {
coroutineScope.cancel("CameraViewModule has been destroyed.")
}
}
override fun getName(): String {
@ -75,6 +54,18 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase
return view ?: throw ViewNotFoundError(viewId)
}
@ReactMethod(isBlockingSynchronousMethod = true)
fun installFrameProcessorBindings(): Boolean {
return try {
val proxy = VisionCameraProxy(reactApplicationContext, frameProcessorThread)
VisionCameraInstaller.install(proxy)
true
} catch (e: Error) {
Log.e(TAG, "Failed to install Frame Processor JSI Bindings!", e)
false
}
}
@ReactMethod
fun takePhoto(viewTag: Int, options: ReadableMap, promise: Promise) {
coroutineScope.launch {
@ -151,18 +142,6 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase
}
}
@ReactMethod(isBlockingSynchronousMethod = true)
fun installFrameProcessorBindings(): Boolean {
try {
frameProcessorManager = FrameProcessorRuntimeManager(reactApplicationContext, frameProcessorThread)
frameProcessorManager!!.installBindings()
return true
} catch (e: Error) {
Log.e(TAG, "Failed to install Frame Processor JSI Bindings!", e)
return false
}
}
@ReactMethod
fun getAvailableCameraDevices(promise: Promise) {
coroutineScope.launch {

4
android/src/main/java/com/mrousavy/camera/Errors.kt
View File

@ -43,12 +43,12 @@ class ParallelVideoProcessingNotSupportedError(cause: Throwable) : CameraError("
"See https://mrousavy.github.io/react-native-vision-camera/docs/guides/devices#the-supportsparallelvideoprocessing-prop for more information.", cause)
class FpsNotContainedInFormatError(fps: Int) : CameraError("format", "invalid-fps", "The given FPS were not valid for the currently selected format. Make sure you select a format which `frameRateRanges` includes $fps FPS!")
class HdrNotContainedInFormatError() : CameraError(
class HdrNotContainedInFormatError : CameraError(
"format", "invalid-hdr",
"The currently selected format does not support HDR capture! " +
"Make sure you select a format which `frameRateRanges` includes `supportsPhotoHDR`!"
)
class LowLightBoostNotContainedInFormatError() : CameraError(
class LowLightBoostNotContainedInFormatError : CameraError(
"format", "invalid-low-light-boost",
"The currently selected format does not support low-light boost (night mode)! " +
"Make sure you select a format which includes `supportsLowLightBoost`."

120
android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java Normal file
View File

@ -0,0 +1,120 @@
package com.mrousavy.camera.frameprocessor;
import android.annotation.SuppressLint;
import android.graphics.ImageFormat;
import android.graphics.Matrix;
import android.media.Image;
import androidx.camera.core.ImageProxy;
import com.facebook.proguard.annotations.DoNotStrip;
import java.nio.ByteBuffer;
public class Frame {
private final ImageProxy imageProxy;
public Frame(ImageProxy imageProxy) {
this.imageProxy = imageProxy;
}
public ImageProxy getImageProxy() {
return imageProxy;
}
@SuppressWarnings("unused")
@DoNotStrip
public int getWidth() {
return imageProxy.getWidth();
}
@SuppressWarnings("unused")
@DoNotStrip
public int getHeight() {
return imageProxy.getHeight();
}
@SuppressWarnings("unused")
@DoNotStrip
public boolean getIsValid() {
try {
@SuppressLint("UnsafeOptInUsageError")
Image image = imageProxy.getImage();
if (image == null) return false;
// will throw an exception if the image is already closed
image.getCropRect();
// no exception thrown, image must still be valid.
return true;
} catch (Exception e) {
// exception thrown, image has already been closed.
return false;
}
}
@SuppressWarnings("unused")
@DoNotStrip
public boolean getIsMirrored() {
Matrix matrix = imageProxy.getImageInfo().getSensorToBufferTransformMatrix();
// TODO: Figure out how to get isMirrored from ImageProxy
return false;
}
@SuppressWarnings("unused")
@DoNotStrip
public long getTimestamp() {
return imageProxy.getImageInfo().getTimestamp();
}
@SuppressWarnings("unused")
@DoNotStrip
public String getOrientation() {
int rotation = imageProxy.getImageInfo().getRotationDegrees();
if (rotation >= 45 && rotation < 135)
return "landscapeRight";
if (rotation >= 135 && rotation < 225)
return "portraitUpsideDown";
if (rotation >= 225 && rotation < 315)
return "landscapeLeft";
return "portrait";
}
@SuppressWarnings("unused")
@DoNotStrip
public int getPlanesCount() {
return imageProxy.getPlanes().length;
}
@SuppressWarnings("unused")
@DoNotStrip
public int getBytesPerRow() {
return imageProxy.getPlanes()[0].getRowStride();
}
private static byte[] byteArrayCache;
@SuppressWarnings("unused")
@DoNotStrip
public byte[] toByteArray() {
switch (imageProxy.getFormat()) {
case ImageFormat.YUV_420_888:
ByteBuffer yBuffer = imageProxy.getPlanes()[0].getBuffer();
ByteBuffer vuBuffer = imageProxy.getPlanes()[2].getBuffer();
int ySize = yBuffer.remaining();
int vuSize = vuBuffer.remaining();
if (byteArrayCache == null || byteArrayCache.length != ySize + vuSize) {
byteArrayCache = new byte[ySize + vuSize];
}
yBuffer.get(byteArrayCache, 0, ySize);
vuBuffer.get(byteArrayCache, ySize, vuSize);
return byteArrayCache;
default:
throw new RuntimeException("Cannot convert Frame with Format " + imageProxy.getFormat() + " to byte array!");
}
}
@SuppressWarnings("unused")
@DoNotStrip
private void close() {
imageProxy.close();
}
}

27
android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java Normal file
View File

@ -0,0 +1,27 @@
package com.mrousavy.camera.frameprocessor;
import androidx.annotation.Keep;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import com.facebook.jni.HybridData;
import com.facebook.proguard.annotations.DoNotStrip;
/**
* Represents a JS Frame Processor
*/
@SuppressWarnings("JavaJniMissingFunction") // we're using fbjni.
public final class FrameProcessor {
/**
* Call the JS Frame Processor function with the given Frame
*/
public native void call(Frame frame);
@DoNotStrip
@Keep
private final HybridData mHybridData;
public FrameProcessor(HybridData hybridData) {
mHybridData = hybridData;
}
}

34
android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java
View File

@ -3,8 +3,8 @@ package com.mrousavy.camera.frameprocessor;
import androidx.annotation.Keep;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.camera.core.ImageProxy;
import com.facebook.proguard.annotations.DoNotStrip;
import com.facebook.react.bridge.ReadableNativeMap;
/**
* Declares a Frame Processor Plugin.
@ -12,42 +12,14 @@ import com.facebook.proguard.annotations.DoNotStrip;
@DoNotStrip
@Keep
public abstract class FrameProcessorPlugin {
private final @NonNull String mName;
/**
* The actual Frame Processor plugin callback. Called for every frame the ImageAnalyzer receives.
* @param image The CameraX ImageProxy. Don't call .close() on this, as VisionCamera handles that.
* @param frame The Frame from the Camera. Don't call .close() on this, as VisionCamera handles that.
* @return You can return any primitive, map or array you want. See the
* <a href="https://mrousavy.github.io/react-native-vision-camera/docs/guides/frame-processors-plugins-overview#types">Types</a>
* table for a list of supported types.
*/
@DoNotStrip
@Keep
public abstract @Nullable Object callback(@NonNull ImageProxy image, @NonNull Object[] params);
/**
* Initializes the native plugin part.
* @param name Specifies the Frame Processor Plugin's name in the Runtime.
* The actual name in the JS Runtime will be prefixed with two underscores (`__`)
*/
protected FrameProcessorPlugin(@NonNull String name) {
mName = name;
}
/**
* Get the user-defined name of the Frame Processor Plugin.
*/
@DoNotStrip
@Keep
public @NonNull String getName() {
return mName;
}
/**
* Registers the given plugin in the Frame Processor Runtime.
* @param plugin An instance of a plugin.
*/
public static void register(@NonNull FrameProcessorPlugin plugin) {
FrameProcessorRuntimeManager.Companion.addPlugin(plugin);
}
public abstract @Nullable Object callback(@NonNull Frame frame, @Nullable ReadableNativeMap params);
}

38
android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPluginRegistry.java Normal file
View File

@ -0,0 +1,38 @@
package com.mrousavy.camera.frameprocessor;
import androidx.annotation.Keep;
import androidx.annotation.Nullable;
import com.facebook.proguard.annotations.DoNotStrip;
import com.facebook.react.bridge.ReadableNativeMap;
import java.util.HashMap;
import java.util.Map;
@DoNotStrip
@Keep
public class FrameProcessorPluginRegistry {
private static final Map<String, PluginInitializer> Plugins = new HashMap<>();
@DoNotStrip
@Keep
public static void addFrameProcessorPlugin(String name, PluginInitializer pluginInitializer) {
assert !Plugins.containsKey(name) : "Tried to add a Frame Processor Plugin with a name that already exists! " +
"Either choose unique names, or remove the unused plugin. Name: ";
Plugins.put(name, pluginInitializer);
}
@DoNotStrip
@Keep
public static FrameProcessorPlugin getPlugin(String name, ReadableNativeMap options) {
PluginInitializer initializer = Plugins.get(name);
if (initializer == null) {
return null;
}
return initializer.initializePlugin(options);
}
public interface PluginInitializer {
FrameProcessorPlugin initializePlugin(@Nullable ReadableNativeMap options);
}
}

98
android/src/main/java/com/mrousavy/camera/frameprocessor/ImageProxyUtils.java
View File

@ -1,98 +0,0 @@
package com.mrousavy.camera.frameprocessor;
import android.annotation.SuppressLint;
import android.graphics.ImageFormat;
import android.graphics.Matrix;
import android.media.Image;
import androidx.annotation.Keep;
import androidx.camera.core.ImageProxy;
import com.facebook.proguard.annotations.DoNotStrip;
import java.nio.ByteBuffer;
@SuppressWarnings("unused") // used through JNI
@DoNotStrip
@Keep
public class ImageProxyUtils {
@SuppressLint("UnsafeOptInUsageError")
@DoNotStrip
@Keep
public static boolean isImageProxyValid(ImageProxy imageProxy) {
try {
Image image = imageProxy.getImage();
if (image == null) return false;
// will throw an exception if the image is already closed
image.getCropRect();
// no exception thrown, image must still be valid.
return true;
} catch (Exception e) {
// exception thrown, image has already been closed.
return false;
}
}
@DoNotStrip
@Keep
public static boolean isImageProxyMirrored(ImageProxy imageProxy) {
Matrix matrix = imageProxy.getImageInfo().getSensorToBufferTransformMatrix();
// TODO: Figure out how to get isMirrored from ImageProxy
return false;
}
@DoNotStrip
@Keep
public static String getOrientation(ImageProxy imageProxy) {
int rotation = imageProxy.getImageInfo().getRotationDegrees();
if (rotation >= 45 && rotation < 135)
return "landscapeRight";
if (rotation >= 135 && rotation < 225)
return "portraitUpsideDown";
if (rotation >= 225 && rotation < 315)
return "landscapeLeft";
return "portrait";
}
@DoNotStrip
@Keep
public static long getTimestamp(ImageProxy imageProxy) {
return imageProxy.getImageInfo().getTimestamp();
}
@DoNotStrip
@Keep
public static int getPlanesCount(ImageProxy imageProxy) {
return imageProxy.getPlanes().length;
}
@DoNotStrip
@Keep
public static int getBytesPerRow(ImageProxy imageProxy) {
return imageProxy.getPlanes()[0].getRowStride();
}
private static byte[] byteArrayCache;
@DoNotStrip
@Keep
public static byte[] toByteArray(ImageProxy imageProxy) {
switch (imageProxy.getFormat()) {
case ImageFormat.YUV_420_888:
ByteBuffer yBuffer = imageProxy.getPlanes()[0].getBuffer();
ByteBuffer vuBuffer = imageProxy.getPlanes()[2].getBuffer();
int ySize = yBuffer.remaining();
int vuSize = vuBuffer.remaining();
if (byteArrayCache == null || byteArrayCache.length != ySize + vuSize) {
byteArrayCache = new byte[ySize + vuSize];
}
yBuffer.get(byteArrayCache, 0, ySize);
vuBuffer.get(byteArrayCache, ySize, vuSize);
return byteArrayCache;
default:
throw new RuntimeException("Cannot convert Frame with Format " + imageProxy.getFormat() + " to byte array!");
}
}
}

6
android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraInstaller.java Normal file
View File

@ -0,0 +1,6 @@
package com.mrousavy.camera.frameprocessor;
@SuppressWarnings("JavaJniMissingFunction") // we use fbjni
public class VisionCameraInstaller {
public static native void install(VisionCameraProxy proxy);
}

65
android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager.kt → android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt
View File

@ -5,6 +5,7 @@ import androidx.annotation.Keep
import com.facebook.jni.HybridData
import com.facebook.proguard.annotations.DoNotStrip
import com.facebook.react.bridge.ReactApplicationContext
import com.facebook.react.bridge.ReadableNativeMap
import com.facebook.react.turbomodule.core.CallInvokerHolderImpl
import com.facebook.react.uimanager.UIManagerHelper
import com.mrousavy.camera.CameraView
@ -12,12 +13,11 @@ import com.mrousavy.camera.ViewNotFoundError
import java.lang.ref.WeakReference
import java.util.concurrent.ExecutorService
@Suppress("KotlinJniMissingFunction") // I use fbjni, Android Studio is not smart enough to realize that.
class FrameProcessorRuntimeManager(context: ReactApplicationContext, frameProcessorThread: ExecutorService) {
companion object {
const val TAG = "FrameProcessorRuntime"
private val Plugins: ArrayList<FrameProcessorPlugin> = ArrayList()
@Suppress("KotlinJniMissingFunction") // we use fbjni.
class VisionCameraProxy(context: ReactApplicationContext, frameProcessorThread: ExecutorService) {
companion object {
const val TAG = "VisionCameraProxy"
init {
try {
System.loadLibrary("VisionCamera")
@ -26,52 +26,53 @@ class FrameProcessorRuntimeManager(context: ReactApplicationContext, frameProces
throw e
}
}
fun addPlugin(plugin: FrameProcessorPlugin) {
Plugins.add(plugin)
}
}
@DoNotStrip
private var mHybridData: HybridData? = null
private var mContext: WeakReference<ReactApplicationContext>? = null
private var mScheduler: VisionCameraScheduler? = null
@Keep
private var mHybridData: HybridData
private var mContext: WeakReference<ReactApplicationContext>
private var mScheduler: VisionCameraScheduler
init {
val jsCallInvokerHolder = context.catalystInstance.jsCallInvokerHolder as CallInvokerHolderImpl
val jsRuntimeHolder = context.javaScriptContextHolder.get()
mScheduler = VisionCameraScheduler(frameProcessorThread)
mContext = WeakReference(context)
mHybridData = initHybrid(jsRuntimeHolder, jsCallInvokerHolder, mScheduler!!)
mHybridData = initHybrid(jsRuntimeHolder, jsCallInvokerHolder, mScheduler)
}
@Suppress("unused")
@DoNotStrip
@Keep
fun findCameraViewById(viewId: Int): CameraView {
private fun findCameraViewById(viewId: Int): CameraView {
Log.d(TAG, "Finding view $viewId...")
val ctx = mContext?.get()
val ctx = mContext.get()
val view = if (ctx != null) UIManagerHelper.getUIManager(ctx, viewId)?.resolveView(viewId) as CameraView? else null
Log.d(TAG, if (view != null) "Found view $viewId!" else "Couldn't find view $viewId!")
return view ?: throw ViewNotFoundError(viewId)
}
fun installBindings() {
Log.i(TAG, "Installing JSI Bindings on JS Thread...")
installJSIBindings()
Log.i(TAG, "Installing Frame Processor Plugins...")
Plugins.forEach { plugin ->
registerPlugin(plugin)
}
Log.i(TAG, "Successfully installed ${Plugins.count()} Frame Processor Plugins!")
@DoNotStrip
@Keep
fun setFrameProcessor(viewId: Int, frameProcessor: FrameProcessor) {
val view = findCameraViewById(viewId)
view.frameProcessor = frameProcessor
}
@DoNotStrip
@Keep
fun removeFrameProcessor(viewId: Int) {
val view = findCameraViewById(viewId)
view.frameProcessor = null
}
@DoNotStrip
@Keep
fun getFrameProcessorPlugin(name: String, options: ReadableNativeMap): FrameProcessorPlugin {
return FrameProcessorPluginRegistry.getPlugin(name, options)
}
// private C++ funcs
private external fun initHybrid(
jsContext: Long,
jsCallInvokerHolder: CallInvokerHolderImpl,
scheduler: VisionCameraScheduler
jsContext: Long,
jsCallInvokerHolder: CallInvokerHolderImpl,
scheduler: VisionCameraScheduler
): HybridData
private external fun registerPlugin(plugin: FrameProcessorPlugin)
private external fun installJSIBindings()
}

11
android/src/main/java/com/mrousavy/camera/parsers/PermissionStatus+String.kt Normal file
View File

@ -0,0 +1,11 @@
package com.mrousavy.camera.parsers
import android.content.pm.PackageManager
fun parsePermissionStatus(status: Int): String {
return when (status) {
PackageManager.PERMISSION_DENIED -> "denied"
PackageManager.PERMISSION_GRANTED -> "authorized"
else -> "not-determined"
}
}

6
docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx
View File

@ -131,7 +131,7 @@ const frameProcessor = useFrameProcessor((frame) => {
## What's possible?
You can run any native code you want in a Frame Processor Plugin. Just like in the native iOS and Android Camera APIs, you will receive a frame (`CMSampleBuffer` on iOS, `ImageProxy` on Android) which you can use however you want. In other words; **everything is possible**.
You can run any native code you want in a Frame Processor Plugin. Just like in the native iOS and Android Camera APIs, you will receive a frame ([`CMSampleBuffer`][5] on iOS, [`ImageProxy`][6] on Android) which you can use however you want. In other words; **everything is possible**.
## Implementations
@ -194,5 +194,7 @@ Your Frame Processor Plugins have to be fast. Use the FPS Graph (`enableFpsGraph
[1]: https://github.com/mrousavy/react-native-vision-camera/blob/main/src/Frame.ts
[2]: https://github.com/mrousavy/react-native-vision-camera/blob/main/ios/Frame%20Processor/Frame.h
[3]: https://developer.android.com/reference/androidx/camera/core/ImageProxy
[3]: https://github.com/mrousavy/react-native-vision-camera/blob/main/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java
[4]: https://github.com/facebook/react-native/blob/9a43eac7a32a6ba3164a048960101022a92fcd5a/React/Base/RCTBridgeModule.h#L20-L24
[5]: https://developer.apple.com/documentation/coremedia/cmsamplebuffer
[6]: https://developer.android.com/reference/androidx/camera/core/ImageProxy

24
docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx
View File

@ -55,21 +55,16 @@ For reference see the [CLI's docs](https://github.com/mateusz1913/vision-camera-
3. Add the following code:
```java {8}
import androidx.camera.core.ImageProxy;
import com.mrousavy.camera.frameprocessor.Frame;
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin;
public class FaceDetectorFrameProcessorPlugin extends FrameProcessorPlugin {
@Override
public Object callback(ImageProxy image, ReadableNativeMap arguments) {
public Object callback(Frame frame, ReadableNativeMap arguments) {
// code goes here
return null;
}
@Override
public String getName() {
return "detectFaces";
}
}
```
@ -86,13 +81,14 @@ import com.facebook.react.bridge.NativeModule;
import com.facebook.react.bridge.ReactApplicationContext;
import com.facebook.react.uimanager.ViewManager;
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin;
import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry;
import javax.annotation.Nonnull;
public class FaceDetectorFrameProcessorPluginPackage implements ReactPackage {
@NonNull
@Override
public List<NativeModule> createNativeModules(@NonNull ReactApplicationContext reactContext) {
FrameProcessorPlugin.register(new FaceDetectorFrameProcessorPlugin());
FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces", options -> new FaceDetectorFrameProcessorPlugin());
return Collections.emptyList();
}
@ -125,19 +121,15 @@ public class FaceDetectorFrameProcessorPluginPackage implements ReactPackage {
3. Add the following code:
```kotlin {7}
import androidx.camera.core.ImageProxy
import com.mrousavy.camera.frameprocessor.Frame
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin
class FaceDetectorFrameProcessorPlugin: FrameProcessorPlugin() {
override fun callback(image: ImageProxy, arguments: ReadableNativeMap): Any? {
override fun callback(frame: Frame, arguments: ReadableNativeMap): Any? {
// code goes here
return null
}
override fun getName(): String {
return "detectFaces"
}
}
```
@ -157,7 +149,9 @@ import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin
class FaceDetectorFrameProcessorPluginPackage : ReactPackage {
override fun createNativeModules(reactContext: ReactApplicationContext): List<NativeModule> {
FrameProcessorPlugin.register(FaceDetectorFrameProcessorPlugin())
FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces") { options ->
FaceDetectorFrameProcessorPlugin()
}
return emptyList()
}

21
example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java
View File

@ -1,19 +1,30 @@
package com.mrousavy.camera.example;
import android.util.Log;
import androidx.camera.core.ImageProxy;
import com.facebook.react.bridge.ReadableNativeMap;
import com.facebook.react.bridge.WritableNativeArray;
import com.facebook.react.bridge.WritableNativeMap;
import com.mrousavy.camera.frameprocessor.Frame;
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import java.util.HashMap;
public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin {
@Override
public Object callback(@NotNull ImageProxy image, @NotNull Object[] params) {
Log.d("ExamplePlugin", image.getWidth() + " x " + image.getHeight() + " Image with format #" + image.getFormat() + ". Logging " + params.length + " parameters:");
public Object callback(@NotNull Frame frame, @Nullable ReadableNativeMap params) {
HashMap<String, Object> hashMap = params != null ? params.toHashMap() : new HashMap<>();
ImageProxy image = frame.getImageProxy();
for (Object param : params) {
Log.d("ExamplePlugin", " -> " + (param == null ? "(null)" : param.toString() + " (" + param.getClass().getName() + ")"));
Log.d("ExamplePlugin", image.getWidth() + " x " + image.getHeight() + " Image with format #" + image.getFormat() + ". Logging " + hashMap.size() + " parameters:");
for (String key : hashMap.keySet()) {
Object value = hashMap.get(key);
Log.d("ExamplePlugin", " -> " + (value == null ? "(null)" : value.toString() + " (" + value.getClass().getName() + ")"));
}
WritableNativeMap map = new WritableNativeMap();
@ -31,6 +42,6 @@ public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin {
}
ExampleFrameProcessorPlugin() {
super("example_plugin");
}
}

7
example/android/app/src/main/java/com/mrousavy/camera/example/MainApplication.java
View File

@ -1,10 +1,14 @@
package com.mrousavy.camera.example;
import android.app.Application;
import androidx.annotation.Nullable;
import com.facebook.react.PackageList;
import com.facebook.react.ReactApplication;
import com.facebook.react.ReactNativeHost;
import com.facebook.react.ReactPackage;
import com.facebook.react.bridge.ReadableNativeMap;
import com.facebook.soloader.SoLoader;
import java.util.List;
import com.facebook.react.defaults.DefaultNewArchitectureEntryPoint;
@ -12,6 +16,7 @@ import com.facebook.react.defaults.DefaultReactNativeHost;
import com.mrousavy.camera.CameraPackage;
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin;
import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry;
public class MainApplication extends Application implements ReactApplication {
@ -61,6 +66,6 @@ public class MainApplication extends Application implements ReactApplication {
DefaultNewArchitectureEntryPoint.load();
}
FrameProcessorPlugin.register(new ExampleFrameProcessorPlugin());
FrameProcessorPluginRegistry.addFrameProcessorPlugin("example_plugin", options -> new ExampleFrameProcessorPlugin());
}
}

24
example/src/CameraPage.tsx
View File

@ -1,6 +1,6 @@
import * as React from 'react';
import { useRef, useState, useMemo, useCallback } from 'react';
import { Platform, StyleSheet, Text, View } from 'react-native';
import { StyleSheet, Text, View } from 'react-native';
import { PinchGestureHandler, PinchGestureHandlerGestureEvent, TapGestureHandler } from 'react-native-gesture-handler';
import {
CameraDeviceFormat,
@ -8,7 +8,7 @@ import {
PhotoFile,
sortFormats,
useCameraDevices,
useSkiaFrameProcessor,
useFrameProcessor,
VideoFile,
} from 'react-native-vision-camera';
import { Camera, frameRateIncluded } from 'react-native-vision-camera';
@ -26,6 +26,7 @@ import type { NativeStackScreenProps } from '@react-navigation/native-stack';
import { useIsFocused } from '@react-navigation/core';
import { Skia } from '@shopify/react-native-skia';
import { FACE_SHADER } from './Shaders';
import { examplePlugin } from './frame-processors/ExamplePlugin';
const ReanimatedCamera = Reanimated.createAnimatedComponent(Camera);
Reanimated.addWhitelistedNativeProps({
@ -217,16 +218,13 @@ export function CameraPage({ navigation }: Props): React.ReactElement {
const paint = Skia.Paint();
paint.setImageFilter(imageFilter);
const isIOS = Platform.OS === 'ios';
const frameProcessor = useSkiaFrameProcessor(
(frame) => {
'worklet';
console.log(`Width: ${frame.width}`);
const frameProcessor = useFrameProcessor((frame) => {
'worklet';
if (frame.isDrawable) frame.render(paint);
},
[isIOS, paint],
);
console.log(`Width: ${frame.width}`);
const result = examplePlugin(frame);
console.log('Example Plugin: ', result);
}, []);
return (
<View style={styles.container}>
@ -247,12 +245,10 @@ export function CameraPage({ navigation }: Props): React.ReactElement {
onError={onError}
enableZoomGesture={false}
animatedProps={cameraAnimatedProps}
photo={true}
video={true}
audio={hasMicrophonePermission}
enableFpsGraph={true}
orientation="portrait"
frameProcessor={device.supportsParallelVideoProcessing ? frameProcessor : undefined}
frameProcessor={frameProcessor}
/>
</TapGestureHandler>
</Reanimated.View>

2
ios/Frame Processor/FrameProcessorPluginHostObject.mm
View File

@ -12,6 +12,8 @@
#import "FrameHostObject.h"
#import "JSINSObjectConversion.h"
using namespace facebook;
std::vector<jsi::PropNameID> FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime& runtime) {
std::vector<jsi::PropNameID> result;
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("call")));

2
ios/Frame Processor/VisionCameraProxy.mm
View File

@ -177,7 +177,7 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID&
const jsi::Value& thisValue,
const jsi::Value* arguments,
size_t count) -> jsi::Value {
if (count != 1 || !arguments[0].isString()) {
if (count < 1 || !arguments[0].isString()) {
throw jsi::JSError(runtime, "First argument needs to be a string (pluginName)!");
}
auto pluginName = arguments[0].asString(runtime).utf8(runtime);