feat: Rewrite Android C++ part (VisionCameraProxy + JFrame) (#1661)
* First Android rewrite * Rewrite Android C++ backend * Pass `ReadableNativeMap`, fix build error * fix: Fix FrameProcessor init * Make a bunch of stuff const reference to avoid copies * Indents * Cleanup * indents * docs: Update Android docs * Update CameraView.kt * fix: Format C++ code
This commit is contained in:
Marc Rousavy
GitHub
@@ -1,58 +0,0 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 14.06.21.
|
||||
//
|
||||
|
||||
#include "CameraView.h"
|
||||
|
||||
#include <jni.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
#include <jsi/jsi.h>
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <regex>
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
using namespace jni;
|
||||
|
||||
using TSelf = local_ref<CameraView::jhybriddata>;
|
||||
|
||||
TSelf CameraView::initHybrid(alias_ref<HybridClass::jhybridobject> jThis) {
|
||||
return makeCxxInstance(jThis);
|
||||
}
|
||||
|
||||
void CameraView::registerNatives() {
|
||||
registerHybrid({
|
||||
makeNativeMethod("initHybrid", CameraView::initHybrid),
|
||||
makeNativeMethod("frameProcessorCallback", CameraView::frameProcessorCallback),
|
||||
});
|
||||
}
|
||||
|
||||
void CameraView::frameProcessorCallback(const alias_ref<JImageProxy::javaobject>& frame) {
|
||||
if (frameProcessor_ == nullptr) {
|
||||
__android_log_write(ANDROID_LOG_WARN, TAG, "Called Frame Processor callback, but `frameProcessor` is null!");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
frameProcessor_(frame);
|
||||
} catch (const jsi::JSError& error) {
|
||||
// TODO: jsi::JSErrors cannot be caught on Hermes. They crash the entire app.
|
||||
auto stack = std::regex_replace(error.getStack(), std::regex("\n"), "\n ");
|
||||
__android_log_print(ANDROID_LOG_ERROR, TAG, "Frame Processor threw an error! %s\nIn: %s", error.getMessage().c_str(), stack.c_str());
|
||||
} catch (const std::exception& exception) {
|
||||
__android_log_print(ANDROID_LOG_ERROR, TAG, "Frame Processor threw a C++ error! %s", exception.what());
|
||||
}
|
||||
}
|
||||
|
||||
void CameraView::setFrameProcessor(const TFrameProcessor&& frameProcessor) {
|
||||
frameProcessor_ = frameProcessor;
|
||||
}
|
||||
|
||||
void vision::CameraView::unsetFrameProcessor() {
|
||||
frameProcessor_ = nullptr;
|
||||
}
|
||||
|
||||
} // namespace vision
|
||||
@@ -1,43 +0,0 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 14.06.21.
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <jni.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "java-bindings/JImageProxy.h"
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
using TFrameProcessor = std::function<void(jni::alias_ref<JImageProxy::javaobject>)>;
|
||||
|
||||
class CameraView : public jni::HybridClass<CameraView> {
|
||||
public:
|
||||
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/CameraView;";
|
||||
static auto constexpr TAG = "VisionCamera";
|
||||
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis);
|
||||
static void registerNatives();
|
||||
|
||||
// TODO: Use template<> to avoid heap allocation for std::function<>
|
||||
void setFrameProcessor(const TFrameProcessor&& frameProcessor);
|
||||
void unsetFrameProcessor();
|
||||
|
||||
private:
|
||||
friend HybridBase;
|
||||
jni::global_ref<CameraView::javaobject> javaPart_;
|
||||
TFrameProcessor frameProcessor_;
|
||||
|
||||
void frameProcessorCallback(const jni::alias_ref<JImageProxy::javaobject>& frame);
|
||||
|
||||
explicit CameraView(jni::alias_ref<CameraView::jhybridobject> jThis) :
|
||||
javaPart_(jni::make_global(jThis)),
|
||||
frameProcessor_(nullptr)
|
||||
{}
|
||||
};
|
||||
|
||||
} // namespace vision
|
||||
@@ -18,7 +18,7 @@ namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
FrameHostObject::FrameHostObject(jni::alias_ref<JImageProxy::javaobject> image): frame(make_global(image)), _refCount(0) { }
|
||||
FrameHostObject::FrameHostObject(const jni::alias_ref<JFrame::javaobject>& frame): frame(make_global(frame)), _refCount(0) { }
|
||||
|
||||
FrameHostObject::~FrameHostObject() {
|
||||
// Hermes' Garbage Collector (Hades GC) calls destructors on a separate Thread
|
||||
|
||||
@@ -11,15 +11,15 @@
|
||||
#include <string>
|
||||
#include <mutex>
|
||||
|
||||
#include "java-bindings/JImageProxy.h"
|
||||
#include "java-bindings/JFrame.h"
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
class JSI_EXPORT FrameHostObject : public jsi::HostObject {
|
||||
explicit FrameHostObject(const jni::alias_ref<JFrame::javaobject>& frame);
|
||||
public:
|
||||
explicit FrameHostObject(jni::alias_ref<JImageProxy::javaobject> image);
|
||||
~FrameHostObject();
|
||||
|
||||
public:
|
||||
@@ -27,7 +27,7 @@ class JSI_EXPORT FrameHostObject : public jsi::HostObject {
|
||||
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime &rt) override;
|
||||
|
||||
public:
|
||||
jni::global_ref<JImageProxy> frame;
|
||||
jni::global_ref<JFrame> frame;
|
||||
|
||||
private:
|
||||
static auto constexpr TAG = "VisionCamera";
|
||||
|
||||
53
android/src/main/cpp/FrameProcessorPluginHostObject.cpp
Normal file
53
android/src/main/cpp/FrameProcessorPluginHostObject.cpp
Normal file
@@ -0,0 +1,53 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 21.07.23.
|
||||
//
|
||||
|
||||
#include "FrameProcessorPluginHostObject.h"
|
||||
#include <vector>
|
||||
#include "FrameHostObject.h"
|
||||
#include "JSIJNIConversion.h"
|
||||
#include <string>
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
std::vector<jsi::PropNameID> FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime& runtime) {
|
||||
std::vector<jsi::PropNameID> result;
|
||||
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("call")));
|
||||
return result;
|
||||
}
|
||||
|
||||
jsi::Value FrameProcessorPluginHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
|
||||
auto name = propName.utf8(runtime);
|
||||
|
||||
if (name == "call") {
|
||||
return jsi::Function::createFromHostFunction(runtime,
|
||||
jsi::PropNameID::forUtf8(runtime, "call"),
|
||||
2,
|
||||
[=](jsi::Runtime &runtime,
|
||||
const jsi::Value &thisValue,
|
||||
const jsi::Value *arguments,
|
||||
size_t count) -> jsi::Value {
|
||||
// Frame is first argument
|
||||
auto frameHostObject = arguments[0].asObject(runtime).asHostObject<FrameHostObject>(runtime);
|
||||
auto frame = frameHostObject->frame;
|
||||
|
||||
// Options are second argument (possibly undefined)
|
||||
local_ref<react::ReadableNativeMap::javaobject> options = nullptr;
|
||||
if (count > 1) {
|
||||
options = JSIJNIConversion::convertJSIObjectToJNIMap(runtime, arguments[1].asObject(runtime));
|
||||
}
|
||||
|
||||
// Call actual plugin
|
||||
auto result = _plugin->callback(frame, options);
|
||||
|
||||
// Convert result value to jsi::Value (possibly undefined)
|
||||
return JSIJNIConversion::convertJNIObjectToJSIValue(runtime, result);
|
||||
});
|
||||
}
|
||||
|
||||
return jsi::Value::undefined();
|
||||
}
|
||||
|
||||
} // namespace vision
|
||||
32
android/src/main/cpp/FrameProcessorPluginHostObject.h
Normal file
32
android/src/main/cpp/FrameProcessorPluginHostObject.h
Normal file
@@ -0,0 +1,32 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 21.07.23.
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <jsi/jsi.h>
|
||||
#include "java-bindings/JFrameProcessorPlugin.h"
|
||||
#include <memory>
|
||||
#include <ReactCommon/CallInvoker.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
#include <vector>
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
class FrameProcessorPluginHostObject: public jsi::HostObject {
|
||||
public:
|
||||
explicit FrameProcessorPluginHostObject(jni::alias_ref<JFrameProcessorPlugin::javaobject> plugin):
|
||||
_plugin(make_global(plugin)) { }
|
||||
~FrameProcessorPluginHostObject() { }
|
||||
|
||||
public:
|
||||
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& runtime) override;
|
||||
jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& name) override;
|
||||
|
||||
private:
|
||||
jni::global_ref<JFrameProcessorPlugin::javaobject> _plugin;
|
||||
};
|
||||
|
||||
} // namespace vision
|
||||
@@ -1,245 +0,0 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 11.06.21.
|
||||
//
|
||||
|
||||
#include "FrameProcessorRuntimeManager.h"
|
||||
#include <android/log.h>
|
||||
#include <jni.h>
|
||||
#include <utility>
|
||||
#include <string>
|
||||
#include <react-native-worklets/WKTJsiWorklet.h>
|
||||
#include <react-native-worklets/WKTJsiHostObject.h>
|
||||
|
||||
#include "CameraView.h"
|
||||
#include "FrameHostObject.h"
|
||||
#include "JSIJNIConversion.h"
|
||||
#include "java-bindings/JImageProxy.h"
|
||||
#include "java-bindings/JFrameProcessorPlugin.h"
|
||||
#include "JSITypedArray.h"
|
||||
|
||||
namespace vision {
|
||||
|
||||
// type aliases
|
||||
using TSelf = local_ref<HybridClass<vision::FrameProcessorRuntimeManager>::jhybriddata>;
|
||||
using TJSCallInvokerHolder = jni::alias_ref<facebook::react::CallInvokerHolder::javaobject>;
|
||||
using TAndroidScheduler = jni::alias_ref<VisionCameraScheduler::javaobject>;
|
||||
|
||||
FrameProcessorRuntimeManager::FrameProcessorRuntimeManager(jni::alias_ref<FrameProcessorRuntimeManager::jhybridobject> jThis,
|
||||
jsi::Runtime* jsRuntime,
|
||||
std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker,
|
||||
std::shared_ptr<vision::VisionCameraScheduler> scheduler) :
|
||||
javaPart_(jni::make_global(jThis)),
|
||||
_jsRuntime(jsRuntime) {
|
||||
auto runOnJS = [jsCallInvoker](std::function<void()>&& f) {
|
||||
// Run on React JS Runtime
|
||||
jsCallInvoker->invokeAsync(std::move(f));
|
||||
};
|
||||
auto runOnWorklet = [scheduler](std::function<void()>&& f) {
|
||||
// Run on Frame Processor Worklet Runtime
|
||||
scheduler->dispatchAsync(std::move(f));
|
||||
};
|
||||
_workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera",
|
||||
jsRuntime,
|
||||
runOnJS,
|
||||
runOnWorklet);
|
||||
}
|
||||
|
||||
// JNI binding
|
||||
void vision::FrameProcessorRuntimeManager::registerNatives() {
|
||||
registerHybrid({
|
||||
makeNativeMethod("initHybrid",
|
||||
FrameProcessorRuntimeManager::initHybrid),
|
||||
makeNativeMethod("installJSIBindings",
|
||||
FrameProcessorRuntimeManager::installJSIBindings),
|
||||
makeNativeMethod("registerPlugin",
|
||||
FrameProcessorRuntimeManager::registerPlugin),
|
||||
});
|
||||
}
|
||||
|
||||
// JNI init
|
||||
TSelf vision::FrameProcessorRuntimeManager::initHybrid(
|
||||
alias_ref<jhybridobject> jThis,
|
||||
jlong jsRuntimePointer,
|
||||
TJSCallInvokerHolder jsCallInvokerHolder,
|
||||
TAndroidScheduler androidScheduler) {
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG,
|
||||
"Initializing FrameProcessorRuntimeManager...");
|
||||
|
||||
// cast from JNI hybrid objects to C++ instances
|
||||
auto jsRuntime = reinterpret_cast<jsi::Runtime*>(jsRuntimePointer);
|
||||
auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker();
|
||||
auto scheduler = std::shared_ptr<VisionCameraScheduler>(androidScheduler->cthis());
|
||||
|
||||
return makeCxxInstance(jThis, jsRuntime, jsCallInvoker, scheduler);
|
||||
}
|
||||
|
||||
global_ref<CameraView::javaobject> FrameProcessorRuntimeManager::findCameraViewById(int viewId) {
|
||||
static const auto findCameraViewByIdMethod = javaPart_->getClass()->getMethod<CameraView(jint)>("findCameraViewById");
|
||||
auto weakCameraView = findCameraViewByIdMethod(javaPart_.get(), viewId);
|
||||
return make_global(weakCameraView);
|
||||
}
|
||||
|
||||
void FrameProcessorRuntimeManager::logErrorToJS(const std::string& message) {
|
||||
if (!_workletContext) {
|
||||
return;
|
||||
}
|
||||
// Call console.error() on JS Thread
|
||||
_workletContext->invokeOnJsThread([message](jsi::Runtime& runtime) {
|
||||
auto consoleError = runtime
|
||||
.global()
|
||||
.getPropertyAsObject(runtime, "console")
|
||||
.getPropertyAsFunction(runtime, "error");
|
||||
consoleError.call(runtime, jsi::String::createFromUtf8(runtime, message));
|
||||
});
|
||||
}
|
||||
|
||||
void FrameProcessorRuntimeManager::setFrameProcessor(jsi::Runtime& runtime,
|
||||
int viewTag,
|
||||
const jsi::Value& frameProcessor) {
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG, "Setting new Frame Processor...");
|
||||
|
||||
if (!_workletContext) {
|
||||
throw jsi::JSError(runtime,
|
||||
"setFrameProcessor(..): VisionCamera's Worklet Context is not yet initialized!");
|
||||
}
|
||||
|
||||
// find camera view
|
||||
auto cameraView = findCameraViewById(viewTag);
|
||||
|
||||
// convert jsi::Function to a Worklet (can be shared across runtimes)
|
||||
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, frameProcessor);
|
||||
auto workletInvoker = std::make_shared<RNWorklet::WorkletInvoker>(worklet);
|
||||
|
||||
_workletContext->invokeOnWorkletThread([=](RNWorklet::JsiWorkletContext*, jsi::Runtime& rt) {
|
||||
// Set Frame Processor as callable C++ lambda - this will then call the Worklet
|
||||
cameraView->cthis()->setFrameProcessor([this, workletInvoker, &rt](jni::alias_ref<JImageProxy::javaobject> frame) {
|
||||
try {
|
||||
// create HostObject which holds the Frame (JImageProxy)
|
||||
auto frameHostObject = std::make_shared<FrameHostObject>(frame);
|
||||
auto argument = jsi::Object::createFromHostObject(rt, frameHostObject);
|
||||
jsi::Value jsValue(std::move(argument));
|
||||
// Call the Worklet on the Worklet Runtime
|
||||
workletInvoker->call(rt, jsi::Value::undefined(), &jsValue, 1);
|
||||
} catch (jsi::JSError& jsError) {
|
||||
// Worklet threw a JS Error, catch it and log it to JS.
|
||||
auto message = "Frame Processor threw an error: " + jsError.getMessage();
|
||||
__android_log_write(ANDROID_LOG_ERROR, TAG, message.c_str());
|
||||
this->logErrorToJS(message);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
void FrameProcessorRuntimeManager::unsetFrameProcessor(int viewTag) {
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG, "Removing Frame Processor...");
|
||||
|
||||
// find camera view
|
||||
auto cameraView = findCameraViewById(viewTag);
|
||||
|
||||
// call Java method to unset frame processor
|
||||
cameraView->cthis()->unsetFrameProcessor();
|
||||
}
|
||||
|
||||
// actual JSI installer
|
||||
void FrameProcessorRuntimeManager::installJSIBindings() {
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG, "Installing JSI bindings...");
|
||||
|
||||
if (_jsRuntime == nullptr) {
|
||||
__android_log_write(ANDROID_LOG_ERROR, TAG,
|
||||
"JS-Runtime was null, Frame Processor JSI bindings could not be installed!");
|
||||
return;
|
||||
}
|
||||
|
||||
auto& jsiRuntime = *_jsRuntime;
|
||||
|
||||
// HostObject that attaches the cache to the lifecycle of the Runtime. On Runtime destroy, we destroy the cache.
|
||||
auto propNameCacheObject = std::make_shared<vision::InvalidateCacheOnDestroy>(jsiRuntime);
|
||||
jsiRuntime.global().setProperty(jsiRuntime,
|
||||
"__visionCameraPropNameCache",
|
||||
jsi::Object::createFromHostObject(jsiRuntime, propNameCacheObject));
|
||||
|
||||
auto setFrameProcessor = JSI_HOST_FUNCTION_LAMBDA {
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG, "Setting new Frame Processor...");
|
||||
|
||||
double viewTag = arguments[0].asNumber();
|
||||
const jsi::Value& frameProcessor = arguments[1];
|
||||
this->setFrameProcessor(runtime, static_cast<int>(viewTag), frameProcessor);
|
||||
|
||||
return jsi::Value::undefined();
|
||||
};
|
||||
jsiRuntime.global().setProperty(jsiRuntime,
|
||||
"setFrameProcessor",
|
||||
jsi::Function::createFromHostFunction(
|
||||
jsiRuntime,
|
||||
jsi::PropNameID::forAscii(jsiRuntime,
|
||||
"setFrameProcessor"),
|
||||
2, // viewTag, frameProcessor
|
||||
setFrameProcessor));
|
||||
|
||||
|
||||
auto unsetFrameProcessor = JSI_HOST_FUNCTION_LAMBDA {
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG, "Removing Frame Processor...");
|
||||
|
||||
auto viewTag = arguments[0].asNumber();
|
||||
this->unsetFrameProcessor(static_cast<int>(viewTag));
|
||||
|
||||
return jsi::Value::undefined();
|
||||
};
|
||||
jsiRuntime.global().setProperty(jsiRuntime,
|
||||
"unsetFrameProcessor",
|
||||
jsi::Function::createFromHostFunction(
|
||||
jsiRuntime,
|
||||
jsi::PropNameID::forAscii(jsiRuntime,
|
||||
"unsetFrameProcessor"),
|
||||
1, // viewTag
|
||||
unsetFrameProcessor));
|
||||
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG, "Finished installing JSI bindings!");
|
||||
}
|
||||
|
||||
void FrameProcessorRuntimeManager::registerPlugin(alias_ref<JFrameProcessorPlugin::javaobject> plugin) {
|
||||
auto& runtime = *_jsRuntime;
|
||||
|
||||
// we need a strong reference on the plugin, make_global does that.
|
||||
auto pluginGlobal = make_global(plugin);
|
||||
auto pluginName = pluginGlobal->getName();
|
||||
|
||||
__android_log_print(ANDROID_LOG_INFO, TAG, "Installing Frame Processor Plugin \"%s\"...", pluginName.c_str());
|
||||
|
||||
if (!runtime.global().hasProperty(runtime, "FrameProcessorPlugins")) {
|
||||
runtime.global().setProperty(runtime, "FrameProcessorPlugins", jsi::Object(runtime));
|
||||
}
|
||||
jsi::Object frameProcessorPlugins = runtime.global().getPropertyAsObject(runtime, "FrameProcessorPlugins");
|
||||
|
||||
auto function = [pluginGlobal](jsi::Runtime& runtime,
|
||||
const jsi::Value& thisValue,
|
||||
const jsi::Value* arguments,
|
||||
size_t count) -> jsi::Value {
|
||||
// Unbox object and get typed HostObject
|
||||
auto boxedHostObject = arguments[0].asObject(runtime).asHostObject(runtime);
|
||||
auto frameHostObject = dynamic_cast<FrameHostObject*>(boxedHostObject.get());
|
||||
|
||||
// parse params - we are offset by `1` because the frame is the first parameter.
|
||||
auto params = JArrayClass<jobject>::newArray(count - 1);
|
||||
for (size_t i = 1; i < count; i++) {
|
||||
params->setElement(i - 1, JSIJNIConversion::convertJSIValueToJNIObject(runtime, arguments[i]));
|
||||
}
|
||||
|
||||
// call implemented virtual method
|
||||
auto result = pluginGlobal->callback(frameHostObject->frame, params);
|
||||
|
||||
// convert result from JNI to JSI value
|
||||
return JSIJNIConversion::convertJNIObjectToJSIValue(runtime, result);
|
||||
};
|
||||
|
||||
// Assign it to the Proxy.
|
||||
// A FP Plugin called "example_plugin" can be now called from JS using "FrameProcessorPlugins.example_plugin(frame)"
|
||||
frameProcessorPlugins.setProperty(runtime,
|
||||
pluginName.c_str(),
|
||||
jsi::Function::createFromHostFunction(runtime,
|
||||
jsi::PropNameID::forAscii(runtime, pluginName),
|
||||
1, // frame
|
||||
function));
|
||||
}
|
||||
|
||||
} // namespace vision
|
||||
@@ -1,54 +0,0 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 11.06.21.
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <fbjni/fbjni.h>
|
||||
#include <jsi/jsi.h>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <ReactCommon/CallInvokerHolder.h>
|
||||
#include <react-native-worklets/WKTJsiWorkletContext.h>
|
||||
|
||||
#include "CameraView.h"
|
||||
#include "VisionCameraScheduler.h"
|
||||
#include "java-bindings/JFrameProcessorPlugin.h"
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
class FrameProcessorRuntimeManager : public jni::HybridClass<FrameProcessorRuntimeManager> {
|
||||
public:
|
||||
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager;";
|
||||
static auto constexpr TAG = "VisionCamera";
|
||||
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis,
|
||||
jlong jsContext,
|
||||
jni::alias_ref<facebook::react::CallInvokerHolder::javaobject> jsCallInvokerHolder,
|
||||
jni::alias_ref<vision::VisionCameraScheduler::javaobject> androidScheduler);
|
||||
static void registerNatives();
|
||||
|
||||
explicit FrameProcessorRuntimeManager(jni::alias_ref<FrameProcessorRuntimeManager::jhybridobject> jThis,
|
||||
jsi::Runtime* jsRuntime,
|
||||
std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker,
|
||||
std::shared_ptr<vision::VisionCameraScheduler> scheduler);
|
||||
|
||||
private:
|
||||
friend HybridBase;
|
||||
jni::global_ref<FrameProcessorRuntimeManager::javaobject> javaPart_;
|
||||
jsi::Runtime* _jsRuntime;
|
||||
std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;
|
||||
|
||||
jni::global_ref<CameraView::javaobject> findCameraViewById(int viewId);
|
||||
void installJSIBindings();
|
||||
void registerPlugin(alias_ref<JFrameProcessorPlugin::javaobject> plugin);
|
||||
void logErrorToJS(const std::string& message);
|
||||
|
||||
void setFrameProcessor(jsi::Runtime& runtime,
|
||||
int viewTag,
|
||||
const jsi::Value& frameProcessor);
|
||||
void unsetFrameProcessor(int viewTag);
|
||||
};
|
||||
|
||||
} // namespace vision
|
||||
@@ -21,7 +21,7 @@
|
||||
#include <folly/dynamic.h>
|
||||
|
||||
#include "FrameHostObject.h"
|
||||
#include "java-bindings/JImageProxy.h"
|
||||
#include "java-bindings/JFrame.h"
|
||||
#include "java-bindings/JArrayList.h"
|
||||
#include "java-bindings/JHashMap.h"
|
||||
|
||||
@@ -29,75 +29,9 @@ namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
jobject JSIJNIConversion::convertJSIValueToJNIObject(jsi::Runtime &runtime, const jsi::Value &value) {
|
||||
if (value.isBool()) {
|
||||
// jsi::Bool
|
||||
|
||||
auto boolean = jni::JBoolean::valueOf(value.getBool());
|
||||
return boolean.release();
|
||||
|
||||
} else if (value.isNumber()) {
|
||||
// jsi::Number
|
||||
|
||||
auto number = jni::JDouble::valueOf(value.getNumber());
|
||||
return number.release();
|
||||
|
||||
} else if (value.isNull() || value.isUndefined()) {
|
||||
// jsi::undefined
|
||||
|
||||
return nullptr;
|
||||
|
||||
} else if (value.isString()) {
|
||||
// jsi::String
|
||||
|
||||
auto string = jni::make_jstring(value.getString(runtime).utf8(runtime));
|
||||
return string.release();
|
||||
|
||||
} else if (value.isObject()) {
|
||||
// jsi::Object
|
||||
|
||||
auto object = value.asObject(runtime);
|
||||
|
||||
if (object.isArray(runtime)) {
|
||||
// jsi::Array
|
||||
|
||||
auto dynamic = jsi::dynamicFromValue(runtime, value);
|
||||
auto nativeArray = react::ReadableNativeArray::newObjectCxxArgs(std::move(dynamic));
|
||||
return nativeArray.release();
|
||||
|
||||
} else if (object.isHostObject(runtime)) {
|
||||
// jsi::HostObject
|
||||
|
||||
auto boxedHostObject = object.getHostObject(runtime);
|
||||
auto hostObject = dynamic_cast<FrameHostObject*>(boxedHostObject.get());
|
||||
if (hostObject != nullptr) {
|
||||
// return jni local_ref to the JImageProxy
|
||||
return hostObject->frame.get();
|
||||
} else {
|
||||
// it's different kind of HostObject. We don't support it.
|
||||
throw std::runtime_error("Received an unknown HostObject! Cannot convert to a JNI value.");
|
||||
}
|
||||
|
||||
} else if (object.isFunction(runtime)) {
|
||||
// jsi::Function
|
||||
|
||||
// TODO: Convert Function to Callback
|
||||
throw std::runtime_error("Cannot convert a JS Function to a JNI value (yet)!");
|
||||
|
||||
} else {
|
||||
// jsi::Object
|
||||
|
||||
auto dynamic = jsi::dynamicFromValue(runtime, value);
|
||||
auto map = react::ReadableNativeMap::createWithContents(std::move(dynamic));
|
||||
return map.release();
|
||||
}
|
||||
} else {
|
||||
// unknown jsi type!
|
||||
|
||||
auto stringRepresentation = value.toString(runtime).utf8(runtime);
|
||||
auto message = "Received unknown JSI value! (" + stringRepresentation + ") Cannot convert to a JNI value.";
|
||||
throw std::runtime_error(message);
|
||||
}
|
||||
jni::local_ref<react::ReadableNativeMap::javaobject> JSIJNIConversion::convertJSIObjectToJNIMap(jsi::Runtime& runtime, const jsi::Object& object) {
|
||||
auto dynamic = jsi::dynamicFromValue(runtime, jsi::Value(runtime, object));
|
||||
return react::ReadableNativeMap::createWithContents(std::move(dynamic));
|
||||
}
|
||||
|
||||
jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, const jni::local_ref<jobject>& object) {
|
||||
@@ -178,10 +112,9 @@ jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, c
|
||||
auto hashMap = toHashMapFunc(object.get());
|
||||
return convertJNIObjectToJSIValue(runtime, hashMap);
|
||||
|
||||
} else if (object->isInstanceOf(JImageProxy::javaClassStatic())) {
|
||||
// ImageProxy
|
||||
|
||||
auto frame = static_ref_cast<JImageProxy>(object);
|
||||
} else if (object->isInstanceOf(JFrame::javaClassStatic())) {
|
||||
// Frame
|
||||
auto frame = static_ref_cast<JFrame>(object);
|
||||
|
||||
// box into HostObject
|
||||
auto hostObject = std::make_shared<FrameHostObject>(frame);
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
#include <jsi/jsi.h>
|
||||
#include <jni.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
#include <react/jni/ReadableNativeMap.h>
|
||||
|
||||
namespace vision {
|
||||
|
||||
@@ -14,7 +15,7 @@ namespace JSIJNIConversion {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
jobject convertJSIValueToJNIObject(jsi::Runtime& runtime, const jsi::Value& value);
|
||||
jni::local_ref<react::ReadableNativeMap::javaobject> convertJSIObjectToJNIMap(jsi::Runtime& runtime, const jsi::Object& object);
|
||||
|
||||
jsi::Value convertJNIObjectToJSIValue(jsi::Runtime& runtime, const jni::local_ref<jobject>& object);
|
||||
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
#include <jni.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
#include "FrameProcessorRuntimeManager.h"
|
||||
#include "CameraView.h"
|
||||
#include "VisionCameraScheduler.h"
|
||||
#include "java-bindings/JVisionCameraScheduler.h"
|
||||
#include "java-bindings/JFrameProcessor.h"
|
||||
#include "java-bindings/JVisionCameraProxy.h"
|
||||
#include "VisionCameraProxy.h"
|
||||
|
||||
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) {
|
||||
return facebook::jni::initialize(vm, [] {
|
||||
vision::FrameProcessorRuntimeManager::registerNatives();
|
||||
vision::CameraView::registerNatives();
|
||||
vision::VisionCameraScheduler::registerNatives();
|
||||
vision::VisionCameraInstaller::registerNatives();
|
||||
vision::JFrameProcessor::registerNatives();
|
||||
vision::JVisionCameraProxy::registerNatives();
|
||||
vision::JVisionCameraScheduler::registerNatives();
|
||||
});
|
||||
}
|
||||
|
||||
152
android/src/main/cpp/VisionCameraProxy.cpp
Normal file
152
android/src/main/cpp/VisionCameraProxy.cpp
Normal file
@@ -0,0 +1,152 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 21.07.23.
|
||||
//
|
||||
|
||||
#include "VisionCameraProxy.h"
|
||||
#include <jsi/jsi.h>
|
||||
|
||||
#include "java-bindings/JFrameProcessor.h"
|
||||
#include "java-bindings/JFrameProcessorPlugin.h"
|
||||
#include "JSIJNIConversion.h"
|
||||
|
||||
#include <android/log.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
|
||||
#include "JSITypedArray.h"
|
||||
#include "FrameProcessorPluginHostObject.h"
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
VisionCameraProxy::VisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::javaobject>& javaProxy) {
|
||||
_javaProxy = make_global(javaProxy);
|
||||
}
|
||||
|
||||
VisionCameraProxy::~VisionCameraProxy() {
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG, "Destroying Context...");
|
||||
// Destroy ArrayBuffer cache for both the JS and the Worklet Runtime.
|
||||
auto workletContext = _javaProxy->cthis()->getWorkletContext();
|
||||
invalidateArrayBufferCache(*workletContext->getJsRuntime());
|
||||
invalidateArrayBufferCache(workletContext->getWorkletRuntime());
|
||||
}
|
||||
|
||||
std::vector<jsi::PropNameID> VisionCameraProxy::getPropertyNames(jsi::Runtime& runtime) {
|
||||
std::vector<jsi::PropNameID> result;
|
||||
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("setFrameProcessor")));
|
||||
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("removeFrameProcessor")));
|
||||
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("getFrameProcessorPlugin")));
|
||||
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("isSkiaEnabled")));
|
||||
return result;
|
||||
}
|
||||
|
||||
void VisionCameraProxy::setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& object) {
|
||||
auto frameProcessorType = object.getProperty(runtime, "type").asString(runtime).utf8(runtime);
|
||||
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, object.getProperty(runtime, "frameProcessor"));
|
||||
auto workletContext = _javaProxy->cthis()->getWorkletContext();
|
||||
|
||||
jni::local_ref<JFrameProcessor::javaobject> frameProcessor;
|
||||
if (frameProcessorType == "frame-processor") {
|
||||
frameProcessor = JFrameProcessor::create(worklet, workletContext);
|
||||
} else if (frameProcessorType == "skia-frame-processor") {
|
||||
#if VISION_CAMERA_ENABLE_SKIA
|
||||
throw std::runtime_error("system/skia-unavailable: Skia is not yet implemented on Android!");
|
||||
#else
|
||||
throw std::runtime_error("system/skia-unavailable: Skia is not installed!");
|
||||
#endif
|
||||
} else {
|
||||
throw std::runtime_error("Unknown FrameProcessor.type passed! Received: " + frameProcessorType);
|
||||
}
|
||||
|
||||
_javaProxy->cthis()->setFrameProcessor(viewTag, make_global(frameProcessor));
|
||||
}
|
||||
|
||||
void VisionCameraProxy::removeFrameProcessor(int viewTag) {
|
||||
_javaProxy->cthis()->removeFrameProcessor(viewTag);
|
||||
}
|
||||
|
||||
jsi::Value VisionCameraProxy::getFrameProcessorPlugin(jsi::Runtime& runtime,
|
||||
const std::string& name,
|
||||
const jsi::Object& jsOptions) {
|
||||
auto options = JSIJNIConversion::convertJSIObjectToJNIMap(runtime, jsOptions);
|
||||
|
||||
auto plugin = _javaProxy->cthis()->getFrameProcessorPlugin(name, options);
|
||||
|
||||
auto pluginHostObject = std::make_shared<FrameProcessorPluginHostObject>(plugin);
|
||||
return jsi::Object::createFromHostObject(runtime, pluginHostObject);
|
||||
}
|
||||
|
||||
jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
|
||||
auto name = propName.utf8(runtime);
|
||||
|
||||
if (name == "isSkiaEnabled") {
|
||||
#ifdef VISION_CAMERA_ENABLE_SKIA
|
||||
return jsi::Value(true);
|
||||
#else
|
||||
return jsi::Value(false);
|
||||
#endif
|
||||
}
|
||||
if (name == "setFrameProcessor") {
|
||||
return jsi::Function::createFromHostFunction(runtime,
|
||||
jsi::PropNameID::forUtf8(runtime, "setFrameProcessor"),
|
||||
1,
|
||||
[this](jsi::Runtime& runtime,
|
||||
const jsi::Value& thisValue,
|
||||
const jsi::Value* arguments,
|
||||
size_t count) -> jsi::Value {
|
||||
auto viewTag = arguments[0].asNumber();
|
||||
auto object = arguments[1].asObject(runtime);
|
||||
this->setFrameProcessor(static_cast<int>(viewTag), runtime, object);
|
||||
return jsi::Value::undefined();
|
||||
});
|
||||
}
|
||||
if (name == "removeFrameProcessor") {
|
||||
return jsi::Function::createFromHostFunction(runtime,
|
||||
jsi::PropNameID::forUtf8(runtime, "removeFrameProcessor"),
|
||||
1,
|
||||
[this](jsi::Runtime& runtime,
|
||||
const jsi::Value& thisValue,
|
||||
const jsi::Value* arguments,
|
||||
size_t count) -> jsi::Value {
|
||||
auto viewTag = arguments[0].asNumber();
|
||||
this->removeFrameProcessor(static_cast<int>(viewTag));
|
||||
return jsi::Value::undefined();
|
||||
});
|
||||
}
|
||||
if (name == "getFrameProcessorPlugin") {
|
||||
return jsi::Function::createFromHostFunction(runtime,
|
||||
jsi::PropNameID::forUtf8(runtime, "getFrameProcessorPlugin"),
|
||||
1,
|
||||
[this](jsi::Runtime& runtime,
|
||||
const jsi::Value& thisValue,
|
||||
const jsi::Value* arguments,
|
||||
size_t count) -> jsi::Value {
|
||||
if (count < 1 || !arguments[0].isString()) {
|
||||
throw jsi::JSError(runtime, "First argument needs to be a string (pluginName)!");
|
||||
}
|
||||
auto pluginName = arguments[0].asString(runtime).utf8(runtime);
|
||||
auto options = count > 1 ? arguments[1].asObject(runtime) : jsi::Object(runtime);
|
||||
|
||||
return this->getFrameProcessorPlugin(runtime, pluginName, options);
|
||||
});
|
||||
}
|
||||
|
||||
return jsi::Value::undefined();
|
||||
}
|
||||
|
||||
|
||||
void VisionCameraInstaller::install(jni::alias_ref<jni::JClass>,
|
||||
jni::alias_ref<JVisionCameraProxy::javaobject> proxy) {
|
||||
// global.VisionCameraProxy
|
||||
auto visionCameraProxy = std::make_shared<VisionCameraProxy>(proxy);
|
||||
jsi::Runtime& runtime = *proxy->cthis()->getWorkletContext()->getJsRuntime();
|
||||
runtime.global().setProperty(runtime,
|
||||
"VisionCameraProxy",
|
||||
jsi::Object::createFromHostObject(runtime, visionCameraProxy));
|
||||
}
|
||||
|
||||
} // namespace vision
|
||||
51
android/src/main/cpp/VisionCameraProxy.h
Normal file
51
android/src/main/cpp/VisionCameraProxy.h
Normal file
@@ -0,0 +1,51 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 21.07.23.
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <jsi/jsi.h>
|
||||
|
||||
#include "java-bindings/JVisionCameraScheduler.h"
|
||||
#include "java-bindings/JVisionCameraProxy.h"
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
class VisionCameraProxy: public jsi::HostObject {
|
||||
public:
|
||||
explicit VisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::javaobject>& javaProxy);
|
||||
~VisionCameraProxy();
|
||||
|
||||
public:
|
||||
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& runtime) override;
|
||||
jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& name) override;
|
||||
|
||||
private:
|
||||
void setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& frameProcessor);
|
||||
void removeFrameProcessor(int viewTag);
|
||||
jsi::Value getFrameProcessorPlugin(jsi::Runtime& runtime, const std::string& name, const jsi::Object& options);
|
||||
|
||||
private:
|
||||
jni::global_ref<JVisionCameraProxy::javaobject> _javaProxy;
|
||||
static constexpr const char* TAG = "VisionCameraProxy";
|
||||
};
|
||||
|
||||
|
||||
class VisionCameraInstaller: public jni::JavaClass<VisionCameraInstaller> {
|
||||
public:
|
||||
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraInstaller;";
|
||||
static void registerNatives() {
|
||||
javaClassStatic()->registerNatives({
|
||||
makeNativeMethod("install", VisionCameraInstaller::install)
|
||||
});
|
||||
}
|
||||
static void install(jni::alias_ref<jni::JClass> clazz,
|
||||
jni::alias_ref<JVisionCameraProxy::javaobject> proxy);
|
||||
};
|
||||
|
||||
} // namespace vision
|
||||
65
android/src/main/cpp/java-bindings/JFrame.cpp
Normal file
65
android/src/main/cpp/java-bindings/JFrame.cpp
Normal file
@@ -0,0 +1,65 @@
|
||||
//
|
||||
// Created by Marc on 21.07.2023.
|
||||
//
|
||||
|
||||
#include "JFrame.h"
|
||||
|
||||
#include <jni.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
using namespace jni;
|
||||
|
||||
int JFrame::getWidth() const {
|
||||
static const auto getWidthMethod = getClass()->getMethod<jint()>("getWidth");
|
||||
return getWidthMethod(self());
|
||||
}
|
||||
|
||||
int JFrame::getHeight() const {
|
||||
static const auto getWidthMethod = getClass()->getMethod<jint()>("getHeight");
|
||||
return getWidthMethod(self());
|
||||
}
|
||||
|
||||
bool JFrame::getIsValid() const {
|
||||
static const auto getIsValidMethod = getClass()->getMethod<jboolean()>("getIsValid");
|
||||
return getIsValidMethod(self());
|
||||
}
|
||||
|
||||
bool JFrame::getIsMirrored() const {
|
||||
static const auto getIsMirroredMethod = getClass()->getMethod<jboolean()>("getIsMirrored");
|
||||
return getIsMirroredMethod(self());
|
||||
}
|
||||
|
||||
jlong JFrame::getTimestamp() const {
|
||||
static const auto getTimestampMethod = getClass()->getMethod<jlong()>("getTimestamp");
|
||||
return getTimestampMethod(self());
|
||||
}
|
||||
|
||||
local_ref<JString> JFrame::getOrientation() const {
|
||||
static const auto getOrientationMethod = getClass()->getMethod<JString()>("getOrientation");
|
||||
return getOrientationMethod(self());
|
||||
}
|
||||
|
||||
int JFrame::getPlanesCount() const {
|
||||
static const auto getPlanesCountMethod = getClass()->getMethod<jint()>("getPlanesCount");
|
||||
return getPlanesCountMethod(self());
|
||||
}
|
||||
|
||||
int JFrame::getBytesPerRow() const {
|
||||
static const auto getBytesPerRowMethod = getClass()->getMethod<jint()>("getBytesPerRow");
|
||||
return getBytesPerRowMethod(self());
|
||||
}
|
||||
|
||||
local_ref<JArrayByte> JFrame::toByteArray() const {
|
||||
static const auto toByteArrayMethod = getClass()->getMethod<JArrayByte()>("toByteArray");
|
||||
return toByteArrayMethod(self());
|
||||
}
|
||||
|
||||
void JFrame::close() {
|
||||
static const auto closeMethod = getClass()->getMethod<void()>("close");
|
||||
closeMethod(self());
|
||||
}
|
||||
|
||||
} // namespace vision
|
||||
@@ -1,5 +1,5 @@
|
||||
//
|
||||
// Created by Marc on 19/06/2021.
|
||||
// Created by Marc on 21.07.2023.
|
||||
//
|
||||
|
||||
#pragma once
|
||||
@@ -12,8 +12,8 @@ namespace vision {
|
||||
using namespace facebook;
|
||||
using namespace jni;
|
||||
|
||||
struct JImageProxy : public JavaClass<JImageProxy> {
|
||||
static constexpr auto kJavaDescriptor = "Landroidx/camera/core/ImageProxy;";
|
||||
struct JFrame : public JavaClass<JFrame> {
|
||||
static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/Frame;";
|
||||
|
||||
public:
|
||||
int getWidth() const;
|
||||
65
android/src/main/cpp/java-bindings/JFrameProcessor.cpp
Normal file
65
android/src/main/cpp/java-bindings/JFrameProcessor.cpp
Normal file
@@ -0,0 +1,65 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 29.09.21.
|
||||
//
|
||||
|
||||
#include "JFrameProcessor.h"
|
||||
|
||||
#include <jni.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
|
||||
#include <utility>
|
||||
#include "JFrame.h"
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
using namespace jni;
|
||||
|
||||
void JFrameProcessor::registerNatives() {
|
||||
registerHybrid({
|
||||
makeNativeMethod("call", JFrameProcessor::call)
|
||||
});
|
||||
}
|
||||
|
||||
using TSelf = jni::local_ref<JFrameProcessor::javaobject>;
|
||||
|
||||
JFrameProcessor::JFrameProcessor(std::shared_ptr<RNWorklet::JsiWorklet> worklet,
|
||||
std::shared_ptr<RNWorklet::JsiWorkletContext> context) {
|
||||
_workletContext = std::move(context);
|
||||
_workletInvoker = std::make_shared<RNWorklet::WorkletInvoker>(worklet);
|
||||
}
|
||||
|
||||
TSelf JFrameProcessor::create(const std::shared_ptr<RNWorklet::JsiWorklet>& worklet,
|
||||
const std::shared_ptr<RNWorklet::JsiWorkletContext>& context) {
|
||||
return JFrameProcessor::newObjectCxxArgs(worklet, context);
|
||||
}
|
||||
|
||||
void JFrameProcessor::callWithFrameHostObject(const std::shared_ptr<FrameHostObject>& frameHostObject) const {
|
||||
// Call the Frame Processor on the Worklet Runtime
|
||||
jsi::Runtime& runtime = _workletContext->getWorkletRuntime();
|
||||
|
||||
try {
|
||||
// Wrap HostObject as JSI Value
|
||||
auto argument = jsi::Object::createFromHostObject(runtime, frameHostObject);
|
||||
jsi::Value jsValue(std::move(argument));
|
||||
|
||||
// Call the Worklet with the Frame JS Host Object as an argument
|
||||
_workletInvoker->call(runtime, jsi::Value::undefined(), &jsValue, 1);
|
||||
} catch (jsi::JSError& jsError) {
|
||||
// JS Error occured, print it to console.
|
||||
const std::string& message = jsError.getMessage();
|
||||
|
||||
_workletContext->invokeOnJsThread([message](jsi::Runtime& jsRuntime) {
|
||||
auto logFn = jsRuntime.global().getPropertyAsObject(jsRuntime, "console").getPropertyAsFunction(jsRuntime, "error");
|
||||
logFn.call(jsRuntime, jsi::String::createFromUtf8(jsRuntime, "Frame Processor threw an error: " + message));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void JFrameProcessor::call(jni::alias_ref<JFrame::javaobject> frame) {
|
||||
// Create the Frame Host Object wrapping the internal Frame
|
||||
auto frameHostObject = std::make_shared<FrameHostObject>(frame);
|
||||
callWithFrameHostObject(frameHostObject);
|
||||
}
|
||||
|
||||
} // namespace vision
|
||||
49
android/src/main/cpp/java-bindings/JFrameProcessor.h
Normal file
49
android/src/main/cpp/java-bindings/JFrameProcessor.h
Normal file
@@ -0,0 +1,49 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 29.09.21
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <jni.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
|
||||
#include <react-native-worklets/WKTJsiWorklet.h>
|
||||
#include <react-native-worklets/WKTJsiHostObject.h>
|
||||
|
||||
#include "JFrame.h"
|
||||
#include "FrameHostObject.h"
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
struct JFrameProcessor : public jni::HybridClass<JFrameProcessor> {
|
||||
public:
|
||||
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessor;";
|
||||
static void registerNatives();
|
||||
static jni::local_ref<JFrameProcessor::javaobject> create(const std::shared_ptr<RNWorklet::JsiWorklet>& worklet,
|
||||
const std::shared_ptr<RNWorklet::JsiWorkletContext>& context);
|
||||
|
||||
public:
|
||||
/**
|
||||
* Call the JS Frame Processor.
|
||||
*/
|
||||
void call(alias_ref<JFrame::javaobject> frame);
|
||||
|
||||
private:
|
||||
// Private constructor. Use `create(..)` to create new instances.
|
||||
explicit JFrameProcessor(std::shared_ptr<RNWorklet::JsiWorklet> worklet,
|
||||
std::shared_ptr<RNWorklet::JsiWorkletContext> context);
|
||||
|
||||
private:
|
||||
void callWithFrameHostObject(const std::shared_ptr<FrameHostObject>& frameHostObject) const;
|
||||
|
||||
private:
|
||||
friend HybridBase;
|
||||
std::shared_ptr<RNWorklet::WorkletInvoker> _workletInvoker;
|
||||
std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;
|
||||
};
|
||||
|
||||
} // namespace vision
|
||||
@@ -12,19 +12,14 @@ namespace vision {
|
||||
using namespace facebook;
|
||||
using namespace jni;
|
||||
|
||||
using TCallback = jobject(alias_ref<JImageProxy::javaobject>, alias_ref<JArrayClass<jobject>>);
|
||||
using TCallback = jobject(alias_ref<JFrame::javaobject>, alias_ref<react::ReadableNativeMap::javaobject> params);
|
||||
|
||||
local_ref<jobject> JFrameProcessorPlugin::callback(alias_ref<JImageProxy::javaobject> image,
|
||||
alias_ref<JArrayClass<jobject>> params) const {
|
||||
local_ref<jobject> JFrameProcessorPlugin::callback(const alias_ref<JFrame::javaobject>& frame,
|
||||
const alias_ref<react::ReadableNativeMap::javaobject>& params) const {
|
||||
auto callbackMethod = getClass()->getMethod<TCallback>("callback");
|
||||
|
||||
auto result = callbackMethod(self(), image, params);
|
||||
auto result = callbackMethod(self(), frame, params);
|
||||
return make_local(result);
|
||||
}
|
||||
|
||||
std::string JFrameProcessorPlugin::getName() const {
|
||||
auto getNameMethod = getClass()->getMethod<jstring()>("getName");
|
||||
return getNameMethod(self())->toStdString();
|
||||
}
|
||||
|
||||
} // namespace vision
|
||||
|
||||
@@ -7,8 +7,9 @@
|
||||
#include <jni.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
#include <string>
|
||||
#include <react/jni/ReadableNativeMap.h>
|
||||
|
||||
#include "JImageProxy.h"
|
||||
#include "JFrame.h"
|
||||
|
||||
namespace vision {
|
||||
|
||||
@@ -22,12 +23,8 @@ struct JFrameProcessorPlugin : public JavaClass<JFrameProcessorPlugin> {
|
||||
/**
|
||||
* Call the plugin.
|
||||
*/
|
||||
local_ref<jobject> callback(alias_ref<JImageProxy::javaobject> image,
|
||||
alias_ref<JArrayClass<jobject>> params) const;
|
||||
/**
|
||||
* Get the user-defined name of the Frame Processor Plugin
|
||||
*/
|
||||
std::string getName() const;
|
||||
local_ref<jobject> callback(const alias_ref<JFrame::javaobject>& frame,
|
||||
const alias_ref<react::ReadableNativeMap::javaobject>& params) const;
|
||||
};
|
||||
|
||||
} // namespace vision
|
||||
|
||||
@@ -1,78 +0,0 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 22.06.21.
|
||||
//
|
||||
|
||||
#include "JImageProxy.h"
|
||||
|
||||
#include <jni.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
using namespace jni;
|
||||
|
||||
int JImageProxy::getWidth() const {
|
||||
static const auto getWidthMethod = getClass()->getMethod<jint()>("getWidth");
|
||||
return getWidthMethod(self());
|
||||
}
|
||||
|
||||
int JImageProxy::getHeight() const {
|
||||
static const auto getWidthMethod = getClass()->getMethod<jint()>("getHeight");
|
||||
return getWidthMethod(self());
|
||||
}
|
||||
|
||||
alias_ref<JClass> getUtilsClass() {
|
||||
static const auto ImageProxyUtilsClass = findClassStatic("com/mrousavy/camera/frameprocessor/ImageProxyUtils");
|
||||
return ImageProxyUtilsClass;
|
||||
}
|
||||
|
||||
bool JImageProxy::getIsValid() const {
|
||||
auto utilsClass = getUtilsClass();
|
||||
static const auto isImageProxyValidMethod = utilsClass->getStaticMethod<jboolean(JImageProxy::javaobject)>("isImageProxyValid");
|
||||
return isImageProxyValidMethod(utilsClass, self());
|
||||
}
|
||||
|
||||
bool JImageProxy::getIsMirrored() const {
|
||||
auto utilsClass = getUtilsClass();
|
||||
static const auto isImageProxyMirroredMethod = utilsClass->getStaticMethod<jboolean(JImageProxy::javaobject)>("isImageProxyMirrored");
|
||||
return isImageProxyMirroredMethod(utilsClass, self());
|
||||
}
|
||||
|
||||
jlong JImageProxy::getTimestamp() const {
|
||||
auto utilsClass = getUtilsClass();
|
||||
static const auto getTimestampMethod = utilsClass->getStaticMethod<jlong(JImageProxy::javaobject)>("getTimestamp");
|
||||
return getTimestampMethod(utilsClass, self());
|
||||
}
|
||||
|
||||
local_ref<JString> JImageProxy::getOrientation() const {
|
||||
auto utilsClass = getUtilsClass();
|
||||
static const auto getOrientationMethod = utilsClass->getStaticMethod<JString(JImageProxy::javaobject)>("getOrientation");
|
||||
return getOrientationMethod(utilsClass, self());
|
||||
}
|
||||
|
||||
int JImageProxy::getPlanesCount() const {
|
||||
auto utilsClass = getUtilsClass();
|
||||
static const auto getPlanesCountMethod = utilsClass->getStaticMethod<jint(JImageProxy::javaobject)>("getPlanesCount");
|
||||
return getPlanesCountMethod(utilsClass, self());
|
||||
}
|
||||
|
||||
int JImageProxy::getBytesPerRow() const {
|
||||
auto utilsClass = getUtilsClass();
|
||||
static const auto getBytesPerRowMethod = utilsClass->getStaticMethod<jint(JImageProxy::javaobject)>("getBytesPerRow");
|
||||
return getBytesPerRowMethod(utilsClass, self());
|
||||
}
|
||||
|
||||
local_ref<JArrayByte> JImageProxy::toByteArray() const {
|
||||
auto utilsClass = getUtilsClass();
|
||||
|
||||
static const auto toByteArrayMethod = utilsClass->getStaticMethod<JArrayByte(JImageProxy::javaobject)>("toByteArray");
|
||||
return toByteArrayMethod(utilsClass, self());
|
||||
}
|
||||
|
||||
void JImageProxy::close() {
|
||||
static const auto closeMethod = getClass()->getMethod<void()>("close");
|
||||
closeMethod(self());
|
||||
}
|
||||
|
||||
} // namespace vision
|
||||
91
android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp
Normal file
91
android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp
Normal file
@@ -0,0 +1,91 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 21.07.23.
|
||||
//
|
||||
|
||||
#include "JVisionCameraProxy.h"
|
||||
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
#include <string>
|
||||
|
||||
#include <jsi/jsi.h>
|
||||
#include <react/jni/ReadableNativeMap.h>
|
||||
|
||||
#include <react-native-worklets/WKTJsiWorklet.h>
|
||||
#include <react-native-worklets/WKTJsiWorkletContext.h>
|
||||
|
||||
#include "FrameProcessorPluginHostObject.h"
|
||||
|
||||
namespace vision {
|
||||
|
||||
using TSelf = local_ref<HybridClass<JVisionCameraProxy>::jhybriddata>;
|
||||
using TJSCallInvokerHolder = jni::alias_ref<facebook::react::CallInvokerHolder::javaobject>;
|
||||
using TScheduler = jni::alias_ref<JVisionCameraScheduler::javaobject>;
|
||||
using TOptions = jni::local_ref<react::ReadableNativeMap::javaobject>;
|
||||
|
||||
JVisionCameraProxy::JVisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::jhybridobject>& javaThis,
|
||||
jsi::Runtime* runtime,
|
||||
const std::shared_ptr<facebook::react::CallInvoker>& callInvoker,
|
||||
const jni::global_ref<JVisionCameraScheduler::javaobject>& scheduler) {
|
||||
_javaPart = make_global(javaThis);
|
||||
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG, "Creating Worklet Context...");
|
||||
|
||||
auto runOnJS = [callInvoker](std::function<void()>&& f) {
|
||||
// Run on React JS Runtime
|
||||
callInvoker->invokeAsync(std::move(f));
|
||||
};
|
||||
auto runOnWorklet = [scheduler](std::function<void()>&& f) {
|
||||
// Run on Frame Processor Worklet Runtime
|
||||
scheduler->cthis()->dispatchAsync([f = std::move(f)](){
|
||||
f();
|
||||
});
|
||||
};
|
||||
_workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera",
|
||||
runtime,
|
||||
runOnJS,
|
||||
runOnWorklet);
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG, "Worklet Context created!");
|
||||
}
|
||||
|
||||
|
||||
|
||||
void JVisionCameraProxy::setFrameProcessor(int viewTag,
|
||||
const alias_ref<JFrameProcessor::javaobject>& frameProcessor) {
|
||||
auto setFrameProcessorMethod = javaClassLocal()->getMethod<void(int, alias_ref<JFrameProcessor::javaobject>)>("setFrameProcessor");
|
||||
setFrameProcessorMethod(_javaPart, viewTag, frameProcessor);
|
||||
}
|
||||
|
||||
void JVisionCameraProxy::removeFrameProcessor(int viewTag) {
|
||||
auto removeFrameProcessorMethod = javaClassLocal()->getMethod<void(int)>("removeFrameProcessor");
|
||||
removeFrameProcessorMethod(_javaPart, viewTag);
|
||||
}
|
||||
|
||||
local_ref<JFrameProcessorPlugin::javaobject> JVisionCameraProxy::getFrameProcessorPlugin(const std::string& name,
|
||||
TOptions options) {
|
||||
auto getFrameProcessorPluginMethod = javaClassLocal()->getMethod<JFrameProcessorPlugin(local_ref<jstring>, TOptions)>("getFrameProcessorPlugin");
|
||||
return getFrameProcessorPluginMethod(_javaPart, make_jstring(name), std::move(options));
|
||||
}
|
||||
|
||||
void JVisionCameraProxy::registerNatives() {
|
||||
registerHybrid({
|
||||
makeNativeMethod("initHybrid", JVisionCameraProxy::initHybrid)
|
||||
});
|
||||
}
|
||||
|
||||
TSelf JVisionCameraProxy::initHybrid(
|
||||
alias_ref<jhybridobject> jThis,
|
||||
jlong jsRuntimePointer,
|
||||
TJSCallInvokerHolder jsCallInvokerHolder,
|
||||
const TScheduler& scheduler) {
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG, "Initializing VisionCameraProxy...");
|
||||
|
||||
// cast from JNI hybrid objects to C++ instances
|
||||
auto jsRuntime = reinterpret_cast<jsi::Runtime*>(jsRuntimePointer);
|
||||
auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker();
|
||||
auto sharedScheduler = make_global(scheduler);
|
||||
|
||||
return makeCxxInstance(jThis, jsRuntime, jsCallInvoker, sharedScheduler);
|
||||
}
|
||||
|
||||
} // namespace vision
|
||||
55
android/src/main/cpp/java-bindings/JVisionCameraProxy.h
Normal file
55
android/src/main/cpp/java-bindings/JVisionCameraProxy.h
Normal file
@@ -0,0 +1,55 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 21.07.23.
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <fbjni/fbjni.h>
|
||||
#include <jsi/jsi.h>
|
||||
#include <react-native-worklets/WKTJsiWorkletContext.h>
|
||||
#include <react/jni/ReadableNativeMap.h>
|
||||
|
||||
#include "JFrameProcessorPlugin.h"
|
||||
#include "JVisionCameraScheduler.h"
|
||||
#include "JFrameProcessor.h"
|
||||
|
||||
#include <string>
|
||||
#include <memory>
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
class JVisionCameraProxy : public jni::HybridClass<JVisionCameraProxy> {
|
||||
public:
|
||||
static void registerNatives();
|
||||
|
||||
void setFrameProcessor(int viewTag,
|
||||
const jni::alias_ref<JFrameProcessor::javaobject>& frameProcessor);
|
||||
void removeFrameProcessor(int viewTag);
|
||||
jni::local_ref<JFrameProcessorPlugin::javaobject> getFrameProcessorPlugin(const std::string& name,
|
||||
jni::local_ref<react::ReadableNativeMap::javaobject> options);
|
||||
|
||||
public:
|
||||
std::shared_ptr<RNWorklet::JsiWorkletContext> getWorkletContext() { return _workletContext; }
|
||||
|
||||
private:
|
||||
std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;
|
||||
|
||||
private:
|
||||
friend HybridBase;
|
||||
jni::global_ref<JVisionCameraProxy::javaobject> _javaPart;
|
||||
static auto constexpr TAG = "VisionCameraProxy";
|
||||
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraProxy;";
|
||||
|
||||
explicit JVisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::jhybridobject>& javaThis,
|
||||
jsi::Runtime* jsRuntime,
|
||||
const std::shared_ptr<facebook::react::CallInvoker>& jsCallInvoker,
|
||||
const jni::global_ref<JVisionCameraScheduler::javaobject>& scheduler);
|
||||
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> javaThis,
|
||||
jlong jsRuntimePointer,
|
||||
jni::alias_ref<facebook::react::CallInvokerHolder::javaobject> jsCallInvokerHolder,
|
||||
const jni::alias_ref<JVisionCameraScheduler::javaobject>& scheduler);
|
||||
};
|
||||
|
||||
} // namespace vision
|
||||
@@ -2,31 +2,31 @@
|
||||
// Created by Marc Rousavy on 25.07.21.
|
||||
//
|
||||
|
||||
#include "VisionCameraScheduler.h"
|
||||
#include "JVisionCameraScheduler.h"
|
||||
#include <fbjni/fbjni.h>
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
using TSelf = jni::local_ref<VisionCameraScheduler::jhybriddata>;
|
||||
using TSelf = jni::local_ref<JVisionCameraScheduler::jhybriddata>;
|
||||
|
||||
TSelf VisionCameraScheduler::initHybrid(jni::alias_ref<jhybridobject> jThis) {
|
||||
TSelf JVisionCameraScheduler::initHybrid(jni::alias_ref<jhybridobject> jThis) {
|
||||
return makeCxxInstance(jThis);
|
||||
}
|
||||
|
||||
void VisionCameraScheduler::dispatchAsync(std::function<void()> job) {
|
||||
void JVisionCameraScheduler::dispatchAsync(const std::function<void()>& job) {
|
||||
// 1. add job to queue
|
||||
_jobs.push(job);
|
||||
scheduleTrigger();
|
||||
}
|
||||
|
||||
void VisionCameraScheduler::scheduleTrigger() {
|
||||
void JVisionCameraScheduler::scheduleTrigger() {
|
||||
// 2. schedule `triggerUI` to be called on the java thread
|
||||
static auto method = javaPart_->getClass()->getMethod<void()>("scheduleTrigger");
|
||||
method(javaPart_.get());
|
||||
}
|
||||
|
||||
void VisionCameraScheduler::trigger() {
|
||||
void JVisionCameraScheduler::trigger() {
|
||||
std::unique_lock<std::mutex> lock(_mutex);
|
||||
// 3. call job we enqueued in step 1.
|
||||
auto job = _jobs.front();
|
||||
@@ -34,10 +34,10 @@ void VisionCameraScheduler::trigger() {
|
||||
_jobs.pop();
|
||||
}
|
||||
|
||||
void VisionCameraScheduler::registerNatives() {
|
||||
void JVisionCameraScheduler::registerNatives() {
|
||||
registerHybrid({
|
||||
makeNativeMethod("initHybrid", VisionCameraScheduler::initHybrid),
|
||||
makeNativeMethod("trigger", VisionCameraScheduler::trigger),
|
||||
makeNativeMethod("initHybrid", JVisionCameraScheduler::initHybrid),
|
||||
makeNativeMethod("trigger", JVisionCameraScheduler::trigger),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -23,22 +23,22 @@ using namespace facebook;
|
||||
* 3. The `scheduleTrigger()` Java Method will switch to the Frame Processor Java Thread and call `trigger()` on there
|
||||
* 4. `trigger()` is a C++ function here that just calls the passed C++ Method from step 1.
|
||||
*/
|
||||
class VisionCameraScheduler : public jni::HybridClass<VisionCameraScheduler> {
|
||||
class JVisionCameraScheduler : public jni::HybridClass<JVisionCameraScheduler> {
|
||||
public:
|
||||
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraScheduler;";
|
||||
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis);
|
||||
static void registerNatives();
|
||||
|
||||
// schedules the given job to be run on the VisionCamera FP Thread at some future point in time
|
||||
void dispatchAsync(std::function<void()> job);
|
||||
void dispatchAsync(const std::function<void()>& job);
|
||||
|
||||
private:
|
||||
friend HybridBase;
|
||||
jni::global_ref<VisionCameraScheduler::javaobject> javaPart_;
|
||||
jni::global_ref<JVisionCameraScheduler::javaobject> javaPart_;
|
||||
std::queue<std::function<void()>> _jobs;
|
||||
std::mutex _mutex;
|
||||
|
||||
explicit VisionCameraScheduler(jni::alias_ref<VisionCameraScheduler::jhybridobject> jThis):
|
||||
explicit JVisionCameraScheduler(jni::alias_ref<JVisionCameraScheduler::jhybridobject> jThis):
|
||||
javaPart_(jni::make_global(jThis)) {}
|
||||
|
||||
// Schedules a call to `trigger` on the VisionCamera FP Thread
|
||||
Reference in New Issue
Block a user