feat: Rewrite Android C++ part (VisionCameraProxy
+ JFrame
) (#1661)
* First Android rewrite * Rewrite Android C++ backend * Pass `ReadableNativeMap`, fix build error * fix: Fix FrameProcessor init * Make a bunch of stuff const reference to avoid copies * Indents * Cleanup * indents * docs: Update Android docs * Update CameraView.kt * fix: Format C++ code
This commit is contained in:
parent
44ed42d5d6
commit
86dd703c2b
@ -32,15 +32,17 @@ add_library(
|
||||
${PACKAGE_NAME}
|
||||
SHARED
|
||||
../cpp/JSITypedArray.cpp
|
||||
src/main/cpp/VisionCamera.cpp
|
||||
src/main/cpp/JSIJNIConversion.cpp
|
||||
src/main/cpp/FrameHostObject.cpp
|
||||
src/main/cpp/FrameProcessorRuntimeManager.cpp
|
||||
src/main/cpp/CameraView.cpp
|
||||
src/main/cpp/VisionCameraScheduler.cpp
|
||||
src/main/cpp/FrameProcessorPluginHostObject.cpp
|
||||
src/main/cpp/JSIJNIConversion.cpp
|
||||
src/main/cpp/VisionCamera.cpp
|
||||
src/main/cpp/VisionCameraProxy.cpp
|
||||
src/main/cpp/java-bindings/JFrame.cpp
|
||||
src/main/cpp/java-bindings/JFrameProcessor.cpp
|
||||
src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp
|
||||
src/main/cpp/java-bindings/JImageProxy.cpp
|
||||
src/main/cpp/java-bindings/JHashMap.cpp
|
||||
src/main/cpp/java-bindings/JVisionCameraProxy.cpp
|
||||
src/main/cpp/java-bindings/JVisionCameraScheduler.cpp
|
||||
)
|
||||
|
||||
# Header Search Paths (includes)
|
||||
|
@ -1,58 +0,0 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 14.06.21.
|
||||
//
|
||||
|
||||
#include "CameraView.h"
|
||||
|
||||
#include <jni.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
#include <jsi/jsi.h>
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <regex>
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
using namespace jni;
|
||||
|
||||
using TSelf = local_ref<CameraView::jhybriddata>;
|
||||
|
||||
TSelf CameraView::initHybrid(alias_ref<HybridClass::jhybridobject> jThis) {
|
||||
return makeCxxInstance(jThis);
|
||||
}
|
||||
|
||||
void CameraView::registerNatives() {
|
||||
registerHybrid({
|
||||
makeNativeMethod("initHybrid", CameraView::initHybrid),
|
||||
makeNativeMethod("frameProcessorCallback", CameraView::frameProcessorCallback),
|
||||
});
|
||||
}
|
||||
|
||||
void CameraView::frameProcessorCallback(const alias_ref<JImageProxy::javaobject>& frame) {
|
||||
if (frameProcessor_ == nullptr) {
|
||||
__android_log_write(ANDROID_LOG_WARN, TAG, "Called Frame Processor callback, but `frameProcessor` is null!");
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
frameProcessor_(frame);
|
||||
} catch (const jsi::JSError& error) {
|
||||
// TODO: jsi::JSErrors cannot be caught on Hermes. They crash the entire app.
|
||||
auto stack = std::regex_replace(error.getStack(), std::regex("\n"), "\n ");
|
||||
__android_log_print(ANDROID_LOG_ERROR, TAG, "Frame Processor threw an error! %s\nIn: %s", error.getMessage().c_str(), stack.c_str());
|
||||
} catch (const std::exception& exception) {
|
||||
__android_log_print(ANDROID_LOG_ERROR, TAG, "Frame Processor threw a C++ error! %s", exception.what());
|
||||
}
|
||||
}
|
||||
|
||||
void CameraView::setFrameProcessor(const TFrameProcessor&& frameProcessor) {
|
||||
frameProcessor_ = frameProcessor;
|
||||
}
|
||||
|
||||
void vision::CameraView::unsetFrameProcessor() {
|
||||
frameProcessor_ = nullptr;
|
||||
}
|
||||
|
||||
} // namespace vision
|
@ -1,43 +0,0 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 14.06.21.
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <jni.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
|
||||
#include <memory>
|
||||
|
||||
#include "java-bindings/JImageProxy.h"
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
using TFrameProcessor = std::function<void(jni::alias_ref<JImageProxy::javaobject>)>;
|
||||
|
||||
class CameraView : public jni::HybridClass<CameraView> {
|
||||
public:
|
||||
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/CameraView;";
|
||||
static auto constexpr TAG = "VisionCamera";
|
||||
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis);
|
||||
static void registerNatives();
|
||||
|
||||
// TODO: Use template<> to avoid heap allocation for std::function<>
|
||||
void setFrameProcessor(const TFrameProcessor&& frameProcessor);
|
||||
void unsetFrameProcessor();
|
||||
|
||||
private:
|
||||
friend HybridBase;
|
||||
jni::global_ref<CameraView::javaobject> javaPart_;
|
||||
TFrameProcessor frameProcessor_;
|
||||
|
||||
void frameProcessorCallback(const jni::alias_ref<JImageProxy::javaobject>& frame);
|
||||
|
||||
explicit CameraView(jni::alias_ref<CameraView::jhybridobject> jThis) :
|
||||
javaPart_(jni::make_global(jThis)),
|
||||
frameProcessor_(nullptr)
|
||||
{}
|
||||
};
|
||||
|
||||
} // namespace vision
|
@ -18,7 +18,7 @@ namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
FrameHostObject::FrameHostObject(jni::alias_ref<JImageProxy::javaobject> image): frame(make_global(image)), _refCount(0) { }
|
||||
FrameHostObject::FrameHostObject(const jni::alias_ref<JFrame::javaobject>& frame): frame(make_global(frame)), _refCount(0) { }
|
||||
|
||||
FrameHostObject::~FrameHostObject() {
|
||||
// Hermes' Garbage Collector (Hades GC) calls destructors on a separate Thread
|
||||
|
@ -11,15 +11,15 @@
|
||||
#include <string>
|
||||
#include <mutex>
|
||||
|
||||
#include "java-bindings/JImageProxy.h"
|
||||
#include "java-bindings/JFrame.h"
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
class JSI_EXPORT FrameHostObject : public jsi::HostObject {
|
||||
explicit FrameHostObject(const jni::alias_ref<JFrame::javaobject>& frame);
|
||||
public:
|
||||
explicit FrameHostObject(jni::alias_ref<JImageProxy::javaobject> image);
|
||||
~FrameHostObject();
|
||||
|
||||
public:
|
||||
@ -27,7 +27,7 @@ class JSI_EXPORT FrameHostObject : public jsi::HostObject {
|
||||
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime &rt) override;
|
||||
|
||||
public:
|
||||
jni::global_ref<JImageProxy> frame;
|
||||
jni::global_ref<JFrame> frame;
|
||||
|
||||
private:
|
||||
static auto constexpr TAG = "VisionCamera";
|
||||
|
53
android/src/main/cpp/FrameProcessorPluginHostObject.cpp
Normal file
53
android/src/main/cpp/FrameProcessorPluginHostObject.cpp
Normal file
@ -0,0 +1,53 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 21.07.23.
|
||||
//
|
||||
|
||||
#include "FrameProcessorPluginHostObject.h"
|
||||
#include <vector>
|
||||
#include "FrameHostObject.h"
|
||||
#include "JSIJNIConversion.h"
|
||||
#include <string>
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
std::vector<jsi::PropNameID> FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime& runtime) {
|
||||
std::vector<jsi::PropNameID> result;
|
||||
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("call")));
|
||||
return result;
|
||||
}
|
||||
|
||||
jsi::Value FrameProcessorPluginHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
|
||||
auto name = propName.utf8(runtime);
|
||||
|
||||
if (name == "call") {
|
||||
return jsi::Function::createFromHostFunction(runtime,
|
||||
jsi::PropNameID::forUtf8(runtime, "call"),
|
||||
2,
|
||||
[=](jsi::Runtime &runtime,
|
||||
const jsi::Value &thisValue,
|
||||
const jsi::Value *arguments,
|
||||
size_t count) -> jsi::Value {
|
||||
// Frame is first argument
|
||||
auto frameHostObject = arguments[0].asObject(runtime).asHostObject<FrameHostObject>(runtime);
|
||||
auto frame = frameHostObject->frame;
|
||||
|
||||
// Options are second argument (possibly undefined)
|
||||
local_ref<react::ReadableNativeMap::javaobject> options = nullptr;
|
||||
if (count > 1) {
|
||||
options = JSIJNIConversion::convertJSIObjectToJNIMap(runtime, arguments[1].asObject(runtime));
|
||||
}
|
||||
|
||||
// Call actual plugin
|
||||
auto result = _plugin->callback(frame, options);
|
||||
|
||||
// Convert result value to jsi::Value (possibly undefined)
|
||||
return JSIJNIConversion::convertJNIObjectToJSIValue(runtime, result);
|
||||
});
|
||||
}
|
||||
|
||||
return jsi::Value::undefined();
|
||||
}
|
||||
|
||||
} // namespace vision
|
32
android/src/main/cpp/FrameProcessorPluginHostObject.h
Normal file
32
android/src/main/cpp/FrameProcessorPluginHostObject.h
Normal file
@ -0,0 +1,32 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 21.07.23.
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <jsi/jsi.h>
|
||||
#include "java-bindings/JFrameProcessorPlugin.h"
|
||||
#include <memory>
|
||||
#include <ReactCommon/CallInvoker.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
#include <vector>
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
class FrameProcessorPluginHostObject: public jsi::HostObject {
|
||||
public:
|
||||
explicit FrameProcessorPluginHostObject(jni::alias_ref<JFrameProcessorPlugin::javaobject> plugin):
|
||||
_plugin(make_global(plugin)) { }
|
||||
~FrameProcessorPluginHostObject() { }
|
||||
|
||||
public:
|
||||
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& runtime) override;
|
||||
jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& name) override;
|
||||
|
||||
private:
|
||||
jni::global_ref<JFrameProcessorPlugin::javaobject> _plugin;
|
||||
};
|
||||
|
||||
} // namespace vision
|
@ -1,245 +0,0 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 11.06.21.
|
||||
//
|
||||
|
||||
#include "FrameProcessorRuntimeManager.h"
|
||||
#include <android/log.h>
|
||||
#include <jni.h>
|
||||
#include <utility>
|
||||
#include <string>
|
||||
#include <react-native-worklets/WKTJsiWorklet.h>
|
||||
#include <react-native-worklets/WKTJsiHostObject.h>
|
||||
|
||||
#include "CameraView.h"
|
||||
#include "FrameHostObject.h"
|
||||
#include "JSIJNIConversion.h"
|
||||
#include "java-bindings/JImageProxy.h"
|
||||
#include "java-bindings/JFrameProcessorPlugin.h"
|
||||
#include "JSITypedArray.h"
|
||||
|
||||
namespace vision {
|
||||
|
||||
// type aliases
|
||||
using TSelf = local_ref<HybridClass<vision::FrameProcessorRuntimeManager>::jhybriddata>;
|
||||
using TJSCallInvokerHolder = jni::alias_ref<facebook::react::CallInvokerHolder::javaobject>;
|
||||
using TAndroidScheduler = jni::alias_ref<VisionCameraScheduler::javaobject>;
|
||||
|
||||
FrameProcessorRuntimeManager::FrameProcessorRuntimeManager(jni::alias_ref<FrameProcessorRuntimeManager::jhybridobject> jThis,
|
||||
jsi::Runtime* jsRuntime,
|
||||
std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker,
|
||||
std::shared_ptr<vision::VisionCameraScheduler> scheduler) :
|
||||
javaPart_(jni::make_global(jThis)),
|
||||
_jsRuntime(jsRuntime) {
|
||||
auto runOnJS = [jsCallInvoker](std::function<void()>&& f) {
|
||||
// Run on React JS Runtime
|
||||
jsCallInvoker->invokeAsync(std::move(f));
|
||||
};
|
||||
auto runOnWorklet = [scheduler](std::function<void()>&& f) {
|
||||
// Run on Frame Processor Worklet Runtime
|
||||
scheduler->dispatchAsync(std::move(f));
|
||||
};
|
||||
_workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera",
|
||||
jsRuntime,
|
||||
runOnJS,
|
||||
runOnWorklet);
|
||||
}
|
||||
|
||||
// JNI binding
|
||||
void vision::FrameProcessorRuntimeManager::registerNatives() {
|
||||
registerHybrid({
|
||||
makeNativeMethod("initHybrid",
|
||||
FrameProcessorRuntimeManager::initHybrid),
|
||||
makeNativeMethod("installJSIBindings",
|
||||
FrameProcessorRuntimeManager::installJSIBindings),
|
||||
makeNativeMethod("registerPlugin",
|
||||
FrameProcessorRuntimeManager::registerPlugin),
|
||||
});
|
||||
}
|
||||
|
||||
// JNI init
|
||||
TSelf vision::FrameProcessorRuntimeManager::initHybrid(
|
||||
alias_ref<jhybridobject> jThis,
|
||||
jlong jsRuntimePointer,
|
||||
TJSCallInvokerHolder jsCallInvokerHolder,
|
||||
TAndroidScheduler androidScheduler) {
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG,
|
||||
"Initializing FrameProcessorRuntimeManager...");
|
||||
|
||||
// cast from JNI hybrid objects to C++ instances
|
||||
auto jsRuntime = reinterpret_cast<jsi::Runtime*>(jsRuntimePointer);
|
||||
auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker();
|
||||
auto scheduler = std::shared_ptr<VisionCameraScheduler>(androidScheduler->cthis());
|
||||
|
||||
return makeCxxInstance(jThis, jsRuntime, jsCallInvoker, scheduler);
|
||||
}
|
||||
|
||||
global_ref<CameraView::javaobject> FrameProcessorRuntimeManager::findCameraViewById(int viewId) {
|
||||
static const auto findCameraViewByIdMethod = javaPart_->getClass()->getMethod<CameraView(jint)>("findCameraViewById");
|
||||
auto weakCameraView = findCameraViewByIdMethod(javaPart_.get(), viewId);
|
||||
return make_global(weakCameraView);
|
||||
}
|
||||
|
||||
void FrameProcessorRuntimeManager::logErrorToJS(const std::string& message) {
|
||||
if (!_workletContext) {
|
||||
return;
|
||||
}
|
||||
// Call console.error() on JS Thread
|
||||
_workletContext->invokeOnJsThread([message](jsi::Runtime& runtime) {
|
||||
auto consoleError = runtime
|
||||
.global()
|
||||
.getPropertyAsObject(runtime, "console")
|
||||
.getPropertyAsFunction(runtime, "error");
|
||||
consoleError.call(runtime, jsi::String::createFromUtf8(runtime, message));
|
||||
});
|
||||
}
|
||||
|
||||
void FrameProcessorRuntimeManager::setFrameProcessor(jsi::Runtime& runtime,
|
||||
int viewTag,
|
||||
const jsi::Value& frameProcessor) {
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG, "Setting new Frame Processor...");
|
||||
|
||||
if (!_workletContext) {
|
||||
throw jsi::JSError(runtime,
|
||||
"setFrameProcessor(..): VisionCamera's Worklet Context is not yet initialized!");
|
||||
}
|
||||
|
||||
// find camera view
|
||||
auto cameraView = findCameraViewById(viewTag);
|
||||
|
||||
// convert jsi::Function to a Worklet (can be shared across runtimes)
|
||||
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, frameProcessor);
|
||||
auto workletInvoker = std::make_shared<RNWorklet::WorkletInvoker>(worklet);
|
||||
|
||||
_workletContext->invokeOnWorkletThread([=](RNWorklet::JsiWorkletContext*, jsi::Runtime& rt) {
|
||||
// Set Frame Processor as callable C++ lambda - this will then call the Worklet
|
||||
cameraView->cthis()->setFrameProcessor([this, workletInvoker, &rt](jni::alias_ref<JImageProxy::javaobject> frame) {
|
||||
try {
|
||||
// create HostObject which holds the Frame (JImageProxy)
|
||||
auto frameHostObject = std::make_shared<FrameHostObject>(frame);
|
||||
auto argument = jsi::Object::createFromHostObject(rt, frameHostObject);
|
||||
jsi::Value jsValue(std::move(argument));
|
||||
// Call the Worklet on the Worklet Runtime
|
||||
workletInvoker->call(rt, jsi::Value::undefined(), &jsValue, 1);
|
||||
} catch (jsi::JSError& jsError) {
|
||||
// Worklet threw a JS Error, catch it and log it to JS.
|
||||
auto message = "Frame Processor threw an error: " + jsError.getMessage();
|
||||
__android_log_write(ANDROID_LOG_ERROR, TAG, message.c_str());
|
||||
this->logErrorToJS(message);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
void FrameProcessorRuntimeManager::unsetFrameProcessor(int viewTag) {
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG, "Removing Frame Processor...");
|
||||
|
||||
// find camera view
|
||||
auto cameraView = findCameraViewById(viewTag);
|
||||
|
||||
// call Java method to unset frame processor
|
||||
cameraView->cthis()->unsetFrameProcessor();
|
||||
}
|
||||
|
||||
// actual JSI installer
|
||||
void FrameProcessorRuntimeManager::installJSIBindings() {
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG, "Installing JSI bindings...");
|
||||
|
||||
if (_jsRuntime == nullptr) {
|
||||
__android_log_write(ANDROID_LOG_ERROR, TAG,
|
||||
"JS-Runtime was null, Frame Processor JSI bindings could not be installed!");
|
||||
return;
|
||||
}
|
||||
|
||||
auto& jsiRuntime = *_jsRuntime;
|
||||
|
||||
// HostObject that attaches the cache to the lifecycle of the Runtime. On Runtime destroy, we destroy the cache.
|
||||
auto propNameCacheObject = std::make_shared<vision::InvalidateCacheOnDestroy>(jsiRuntime);
|
||||
jsiRuntime.global().setProperty(jsiRuntime,
|
||||
"__visionCameraPropNameCache",
|
||||
jsi::Object::createFromHostObject(jsiRuntime, propNameCacheObject));
|
||||
|
||||
auto setFrameProcessor = JSI_HOST_FUNCTION_LAMBDA {
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG, "Setting new Frame Processor...");
|
||||
|
||||
double viewTag = arguments[0].asNumber();
|
||||
const jsi::Value& frameProcessor = arguments[1];
|
||||
this->setFrameProcessor(runtime, static_cast<int>(viewTag), frameProcessor);
|
||||
|
||||
return jsi::Value::undefined();
|
||||
};
|
||||
jsiRuntime.global().setProperty(jsiRuntime,
|
||||
"setFrameProcessor",
|
||||
jsi::Function::createFromHostFunction(
|
||||
jsiRuntime,
|
||||
jsi::PropNameID::forAscii(jsiRuntime,
|
||||
"setFrameProcessor"),
|
||||
2, // viewTag, frameProcessor
|
||||
setFrameProcessor));
|
||||
|
||||
|
||||
auto unsetFrameProcessor = JSI_HOST_FUNCTION_LAMBDA {
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG, "Removing Frame Processor...");
|
||||
|
||||
auto viewTag = arguments[0].asNumber();
|
||||
this->unsetFrameProcessor(static_cast<int>(viewTag));
|
||||
|
||||
return jsi::Value::undefined();
|
||||
};
|
||||
jsiRuntime.global().setProperty(jsiRuntime,
|
||||
"unsetFrameProcessor",
|
||||
jsi::Function::createFromHostFunction(
|
||||
jsiRuntime,
|
||||
jsi::PropNameID::forAscii(jsiRuntime,
|
||||
"unsetFrameProcessor"),
|
||||
1, // viewTag
|
||||
unsetFrameProcessor));
|
||||
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG, "Finished installing JSI bindings!");
|
||||
}
|
||||
|
||||
void FrameProcessorRuntimeManager::registerPlugin(alias_ref<JFrameProcessorPlugin::javaobject> plugin) {
|
||||
auto& runtime = *_jsRuntime;
|
||||
|
||||
// we need a strong reference on the plugin, make_global does that.
|
||||
auto pluginGlobal = make_global(plugin);
|
||||
auto pluginName = pluginGlobal->getName();
|
||||
|
||||
__android_log_print(ANDROID_LOG_INFO, TAG, "Installing Frame Processor Plugin \"%s\"...", pluginName.c_str());
|
||||
|
||||
if (!runtime.global().hasProperty(runtime, "FrameProcessorPlugins")) {
|
||||
runtime.global().setProperty(runtime, "FrameProcessorPlugins", jsi::Object(runtime));
|
||||
}
|
||||
jsi::Object frameProcessorPlugins = runtime.global().getPropertyAsObject(runtime, "FrameProcessorPlugins");
|
||||
|
||||
auto function = [pluginGlobal](jsi::Runtime& runtime,
|
||||
const jsi::Value& thisValue,
|
||||
const jsi::Value* arguments,
|
||||
size_t count) -> jsi::Value {
|
||||
// Unbox object and get typed HostObject
|
||||
auto boxedHostObject = arguments[0].asObject(runtime).asHostObject(runtime);
|
||||
auto frameHostObject = dynamic_cast<FrameHostObject*>(boxedHostObject.get());
|
||||
|
||||
// parse params - we are offset by `1` because the frame is the first parameter.
|
||||
auto params = JArrayClass<jobject>::newArray(count - 1);
|
||||
for (size_t i = 1; i < count; i++) {
|
||||
params->setElement(i - 1, JSIJNIConversion::convertJSIValueToJNIObject(runtime, arguments[i]));
|
||||
}
|
||||
|
||||
// call implemented virtual method
|
||||
auto result = pluginGlobal->callback(frameHostObject->frame, params);
|
||||
|
||||
// convert result from JNI to JSI value
|
||||
return JSIJNIConversion::convertJNIObjectToJSIValue(runtime, result);
|
||||
};
|
||||
|
||||
// Assign it to the Proxy.
|
||||
// A FP Plugin called "example_plugin" can be now called from JS using "FrameProcessorPlugins.example_plugin(frame)"
|
||||
frameProcessorPlugins.setProperty(runtime,
|
||||
pluginName.c_str(),
|
||||
jsi::Function::createFromHostFunction(runtime,
|
||||
jsi::PropNameID::forAscii(runtime, pluginName),
|
||||
1, // frame
|
||||
function));
|
||||
}
|
||||
|
||||
} // namespace vision
|
@ -1,54 +0,0 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 11.06.21.
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <fbjni/fbjni.h>
|
||||
#include <jsi/jsi.h>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <ReactCommon/CallInvokerHolder.h>
|
||||
#include <react-native-worklets/WKTJsiWorkletContext.h>
|
||||
|
||||
#include "CameraView.h"
|
||||
#include "VisionCameraScheduler.h"
|
||||
#include "java-bindings/JFrameProcessorPlugin.h"
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
class FrameProcessorRuntimeManager : public jni::HybridClass<FrameProcessorRuntimeManager> {
|
||||
public:
|
||||
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager;";
|
||||
static auto constexpr TAG = "VisionCamera";
|
||||
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis,
|
||||
jlong jsContext,
|
||||
jni::alias_ref<facebook::react::CallInvokerHolder::javaobject> jsCallInvokerHolder,
|
||||
jni::alias_ref<vision::VisionCameraScheduler::javaobject> androidScheduler);
|
||||
static void registerNatives();
|
||||
|
||||
explicit FrameProcessorRuntimeManager(jni::alias_ref<FrameProcessorRuntimeManager::jhybridobject> jThis,
|
||||
jsi::Runtime* jsRuntime,
|
||||
std::shared_ptr<facebook::react::CallInvoker> jsCallInvoker,
|
||||
std::shared_ptr<vision::VisionCameraScheduler> scheduler);
|
||||
|
||||
private:
|
||||
friend HybridBase;
|
||||
jni::global_ref<FrameProcessorRuntimeManager::javaobject> javaPart_;
|
||||
jsi::Runtime* _jsRuntime;
|
||||
std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;
|
||||
|
||||
jni::global_ref<CameraView::javaobject> findCameraViewById(int viewId);
|
||||
void installJSIBindings();
|
||||
void registerPlugin(alias_ref<JFrameProcessorPlugin::javaobject> plugin);
|
||||
void logErrorToJS(const std::string& message);
|
||||
|
||||
void setFrameProcessor(jsi::Runtime& runtime,
|
||||
int viewTag,
|
||||
const jsi::Value& frameProcessor);
|
||||
void unsetFrameProcessor(int viewTag);
|
||||
};
|
||||
|
||||
} // namespace vision
|
@ -21,7 +21,7 @@
|
||||
#include <folly/dynamic.h>
|
||||
|
||||
#include "FrameHostObject.h"
|
||||
#include "java-bindings/JImageProxy.h"
|
||||
#include "java-bindings/JFrame.h"
|
||||
#include "java-bindings/JArrayList.h"
|
||||
#include "java-bindings/JHashMap.h"
|
||||
|
||||
@ -29,75 +29,9 @@ namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
jobject JSIJNIConversion::convertJSIValueToJNIObject(jsi::Runtime &runtime, const jsi::Value &value) {
|
||||
if (value.isBool()) {
|
||||
// jsi::Bool
|
||||
|
||||
auto boolean = jni::JBoolean::valueOf(value.getBool());
|
||||
return boolean.release();
|
||||
|
||||
} else if (value.isNumber()) {
|
||||
// jsi::Number
|
||||
|
||||
auto number = jni::JDouble::valueOf(value.getNumber());
|
||||
return number.release();
|
||||
|
||||
} else if (value.isNull() || value.isUndefined()) {
|
||||
// jsi::undefined
|
||||
|
||||
return nullptr;
|
||||
|
||||
} else if (value.isString()) {
|
||||
// jsi::String
|
||||
|
||||
auto string = jni::make_jstring(value.getString(runtime).utf8(runtime));
|
||||
return string.release();
|
||||
|
||||
} else if (value.isObject()) {
|
||||
// jsi::Object
|
||||
|
||||
auto object = value.asObject(runtime);
|
||||
|
||||
if (object.isArray(runtime)) {
|
||||
// jsi::Array
|
||||
|
||||
auto dynamic = jsi::dynamicFromValue(runtime, value);
|
||||
auto nativeArray = react::ReadableNativeArray::newObjectCxxArgs(std::move(dynamic));
|
||||
return nativeArray.release();
|
||||
|
||||
} else if (object.isHostObject(runtime)) {
|
||||
// jsi::HostObject
|
||||
|
||||
auto boxedHostObject = object.getHostObject(runtime);
|
||||
auto hostObject = dynamic_cast<FrameHostObject*>(boxedHostObject.get());
|
||||
if (hostObject != nullptr) {
|
||||
// return jni local_ref to the JImageProxy
|
||||
return hostObject->frame.get();
|
||||
} else {
|
||||
// it's different kind of HostObject. We don't support it.
|
||||
throw std::runtime_error("Received an unknown HostObject! Cannot convert to a JNI value.");
|
||||
}
|
||||
|
||||
} else if (object.isFunction(runtime)) {
|
||||
// jsi::Function
|
||||
|
||||
// TODO: Convert Function to Callback
|
||||
throw std::runtime_error("Cannot convert a JS Function to a JNI value (yet)!");
|
||||
|
||||
} else {
|
||||
// jsi::Object
|
||||
|
||||
auto dynamic = jsi::dynamicFromValue(runtime, value);
|
||||
auto map = react::ReadableNativeMap::createWithContents(std::move(dynamic));
|
||||
return map.release();
|
||||
}
|
||||
} else {
|
||||
// unknown jsi type!
|
||||
|
||||
auto stringRepresentation = value.toString(runtime).utf8(runtime);
|
||||
auto message = "Received unknown JSI value! (" + stringRepresentation + ") Cannot convert to a JNI value.";
|
||||
throw std::runtime_error(message);
|
||||
}
|
||||
jni::local_ref<react::ReadableNativeMap::javaobject> JSIJNIConversion::convertJSIObjectToJNIMap(jsi::Runtime& runtime, const jsi::Object& object) {
|
||||
auto dynamic = jsi::dynamicFromValue(runtime, jsi::Value(runtime, object));
|
||||
return react::ReadableNativeMap::createWithContents(std::move(dynamic));
|
||||
}
|
||||
|
||||
jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, const jni::local_ref<jobject>& object) {
|
||||
@ -178,10 +112,9 @@ jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, c
|
||||
auto hashMap = toHashMapFunc(object.get());
|
||||
return convertJNIObjectToJSIValue(runtime, hashMap);
|
||||
|
||||
} else if (object->isInstanceOf(JImageProxy::javaClassStatic())) {
|
||||
// ImageProxy
|
||||
|
||||
auto frame = static_ref_cast<JImageProxy>(object);
|
||||
} else if (object->isInstanceOf(JFrame::javaClassStatic())) {
|
||||
// Frame
|
||||
auto frame = static_ref_cast<JFrame>(object);
|
||||
|
||||
// box into HostObject
|
||||
auto hostObject = std::make_shared<FrameHostObject>(frame);
|
||||
|
@ -7,6 +7,7 @@
|
||||
#include <jsi/jsi.h>
|
||||
#include <jni.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
#include <react/jni/ReadableNativeMap.h>
|
||||
|
||||
namespace vision {
|
||||
|
||||
@ -14,7 +15,7 @@ namespace JSIJNIConversion {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
jobject convertJSIValueToJNIObject(jsi::Runtime& runtime, const jsi::Value& value);
|
||||
jni::local_ref<react::ReadableNativeMap::javaobject> convertJSIObjectToJNIMap(jsi::Runtime& runtime, const jsi::Object& object);
|
||||
|
||||
jsi::Value convertJNIObjectToJSIValue(jsi::Runtime& runtime, const jni::local_ref<jobject>& object);
|
||||
|
||||
|
@ -1,13 +1,15 @@
|
||||
#include <jni.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
#include "FrameProcessorRuntimeManager.h"
|
||||
#include "CameraView.h"
|
||||
#include "VisionCameraScheduler.h"
|
||||
#include "java-bindings/JVisionCameraScheduler.h"
|
||||
#include "java-bindings/JFrameProcessor.h"
|
||||
#include "java-bindings/JVisionCameraProxy.h"
|
||||
#include "VisionCameraProxy.h"
|
||||
|
||||
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) {
|
||||
return facebook::jni::initialize(vm, [] {
|
||||
vision::FrameProcessorRuntimeManager::registerNatives();
|
||||
vision::CameraView::registerNatives();
|
||||
vision::VisionCameraScheduler::registerNatives();
|
||||
vision::VisionCameraInstaller::registerNatives();
|
||||
vision::JFrameProcessor::registerNatives();
|
||||
vision::JVisionCameraProxy::registerNatives();
|
||||
vision::JVisionCameraScheduler::registerNatives();
|
||||
});
|
||||
}
|
||||
|
152
android/src/main/cpp/VisionCameraProxy.cpp
Normal file
152
android/src/main/cpp/VisionCameraProxy.cpp
Normal file
@ -0,0 +1,152 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 21.07.23.
|
||||
//
|
||||
|
||||
#include "VisionCameraProxy.h"
|
||||
#include <jsi/jsi.h>
|
||||
|
||||
#include "java-bindings/JFrameProcessor.h"
|
||||
#include "java-bindings/JFrameProcessorPlugin.h"
|
||||
#include "JSIJNIConversion.h"
|
||||
|
||||
#include <android/log.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
|
||||
#include "JSITypedArray.h"
|
||||
#include "FrameProcessorPluginHostObject.h"
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
VisionCameraProxy::VisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::javaobject>& javaProxy) {
|
||||
_javaProxy = make_global(javaProxy);
|
||||
}
|
||||
|
||||
VisionCameraProxy::~VisionCameraProxy() {
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG, "Destroying Context...");
|
||||
// Destroy ArrayBuffer cache for both the JS and the Worklet Runtime.
|
||||
auto workletContext = _javaProxy->cthis()->getWorkletContext();
|
||||
invalidateArrayBufferCache(*workletContext->getJsRuntime());
|
||||
invalidateArrayBufferCache(workletContext->getWorkletRuntime());
|
||||
}
|
||||
|
||||
std::vector<jsi::PropNameID> VisionCameraProxy::getPropertyNames(jsi::Runtime& runtime) {
|
||||
std::vector<jsi::PropNameID> result;
|
||||
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("setFrameProcessor")));
|
||||
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("removeFrameProcessor")));
|
||||
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("getFrameProcessorPlugin")));
|
||||
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("isSkiaEnabled")));
|
||||
return result;
|
||||
}
|
||||
|
||||
void VisionCameraProxy::setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& object) {
|
||||
auto frameProcessorType = object.getProperty(runtime, "type").asString(runtime).utf8(runtime);
|
||||
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, object.getProperty(runtime, "frameProcessor"));
|
||||
auto workletContext = _javaProxy->cthis()->getWorkletContext();
|
||||
|
||||
jni::local_ref<JFrameProcessor::javaobject> frameProcessor;
|
||||
if (frameProcessorType == "frame-processor") {
|
||||
frameProcessor = JFrameProcessor::create(worklet, workletContext);
|
||||
} else if (frameProcessorType == "skia-frame-processor") {
|
||||
#if VISION_CAMERA_ENABLE_SKIA
|
||||
throw std::runtime_error("system/skia-unavailable: Skia is not yet implemented on Android!");
|
||||
#else
|
||||
throw std::runtime_error("system/skia-unavailable: Skia is not installed!");
|
||||
#endif
|
||||
} else {
|
||||
throw std::runtime_error("Unknown FrameProcessor.type passed! Received: " + frameProcessorType);
|
||||
}
|
||||
|
||||
_javaProxy->cthis()->setFrameProcessor(viewTag, make_global(frameProcessor));
|
||||
}
|
||||
|
||||
void VisionCameraProxy::removeFrameProcessor(int viewTag) {
|
||||
_javaProxy->cthis()->removeFrameProcessor(viewTag);
|
||||
}
|
||||
|
||||
jsi::Value VisionCameraProxy::getFrameProcessorPlugin(jsi::Runtime& runtime,
|
||||
const std::string& name,
|
||||
const jsi::Object& jsOptions) {
|
||||
auto options = JSIJNIConversion::convertJSIObjectToJNIMap(runtime, jsOptions);
|
||||
|
||||
auto plugin = _javaProxy->cthis()->getFrameProcessorPlugin(name, options);
|
||||
|
||||
auto pluginHostObject = std::make_shared<FrameProcessorPluginHostObject>(plugin);
|
||||
return jsi::Object::createFromHostObject(runtime, pluginHostObject);
|
||||
}
|
||||
|
||||
jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
|
||||
auto name = propName.utf8(runtime);
|
||||
|
||||
if (name == "isSkiaEnabled") {
|
||||
#ifdef VISION_CAMERA_ENABLE_SKIA
|
||||
return jsi::Value(true);
|
||||
#else
|
||||
return jsi::Value(false);
|
||||
#endif
|
||||
}
|
||||
if (name == "setFrameProcessor") {
|
||||
return jsi::Function::createFromHostFunction(runtime,
|
||||
jsi::PropNameID::forUtf8(runtime, "setFrameProcessor"),
|
||||
1,
|
||||
[this](jsi::Runtime& runtime,
|
||||
const jsi::Value& thisValue,
|
||||
const jsi::Value* arguments,
|
||||
size_t count) -> jsi::Value {
|
||||
auto viewTag = arguments[0].asNumber();
|
||||
auto object = arguments[1].asObject(runtime);
|
||||
this->setFrameProcessor(static_cast<int>(viewTag), runtime, object);
|
||||
return jsi::Value::undefined();
|
||||
});
|
||||
}
|
||||
if (name == "removeFrameProcessor") {
|
||||
return jsi::Function::createFromHostFunction(runtime,
|
||||
jsi::PropNameID::forUtf8(runtime, "removeFrameProcessor"),
|
||||
1,
|
||||
[this](jsi::Runtime& runtime,
|
||||
const jsi::Value& thisValue,
|
||||
const jsi::Value* arguments,
|
||||
size_t count) -> jsi::Value {
|
||||
auto viewTag = arguments[0].asNumber();
|
||||
this->removeFrameProcessor(static_cast<int>(viewTag));
|
||||
return jsi::Value::undefined();
|
||||
});
|
||||
}
|
||||
if (name == "getFrameProcessorPlugin") {
|
||||
return jsi::Function::createFromHostFunction(runtime,
|
||||
jsi::PropNameID::forUtf8(runtime, "getFrameProcessorPlugin"),
|
||||
1,
|
||||
[this](jsi::Runtime& runtime,
|
||||
const jsi::Value& thisValue,
|
||||
const jsi::Value* arguments,
|
||||
size_t count) -> jsi::Value {
|
||||
if (count < 1 || !arguments[0].isString()) {
|
||||
throw jsi::JSError(runtime, "First argument needs to be a string (pluginName)!");
|
||||
}
|
||||
auto pluginName = arguments[0].asString(runtime).utf8(runtime);
|
||||
auto options = count > 1 ? arguments[1].asObject(runtime) : jsi::Object(runtime);
|
||||
|
||||
return this->getFrameProcessorPlugin(runtime, pluginName, options);
|
||||
});
|
||||
}
|
||||
|
||||
return jsi::Value::undefined();
|
||||
}
|
||||
|
||||
|
||||
void VisionCameraInstaller::install(jni::alias_ref<jni::JClass>,
|
||||
jni::alias_ref<JVisionCameraProxy::javaobject> proxy) {
|
||||
// global.VisionCameraProxy
|
||||
auto visionCameraProxy = std::make_shared<VisionCameraProxy>(proxy);
|
||||
jsi::Runtime& runtime = *proxy->cthis()->getWorkletContext()->getJsRuntime();
|
||||
runtime.global().setProperty(runtime,
|
||||
"VisionCameraProxy",
|
||||
jsi::Object::createFromHostObject(runtime, visionCameraProxy));
|
||||
}
|
||||
|
||||
} // namespace vision
|
51
android/src/main/cpp/VisionCameraProxy.h
Normal file
51
android/src/main/cpp/VisionCameraProxy.h
Normal file
@ -0,0 +1,51 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 21.07.23.
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <jsi/jsi.h>
|
||||
|
||||
#include "java-bindings/JVisionCameraScheduler.h"
|
||||
#include "java-bindings/JVisionCameraProxy.h"
|
||||
|
||||
#include <vector>
|
||||
#include <string>
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
class VisionCameraProxy: public jsi::HostObject {
|
||||
public:
|
||||
explicit VisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::javaobject>& javaProxy);
|
||||
~VisionCameraProxy();
|
||||
|
||||
public:
|
||||
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& runtime) override;
|
||||
jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& name) override;
|
||||
|
||||
private:
|
||||
void setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& frameProcessor);
|
||||
void removeFrameProcessor(int viewTag);
|
||||
jsi::Value getFrameProcessorPlugin(jsi::Runtime& runtime, const std::string& name, const jsi::Object& options);
|
||||
|
||||
private:
|
||||
jni::global_ref<JVisionCameraProxy::javaobject> _javaProxy;
|
||||
static constexpr const char* TAG = "VisionCameraProxy";
|
||||
};
|
||||
|
||||
|
||||
class VisionCameraInstaller: public jni::JavaClass<VisionCameraInstaller> {
|
||||
public:
|
||||
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraInstaller;";
|
||||
static void registerNatives() {
|
||||
javaClassStatic()->registerNatives({
|
||||
makeNativeMethod("install", VisionCameraInstaller::install)
|
||||
});
|
||||
}
|
||||
static void install(jni::alias_ref<jni::JClass> clazz,
|
||||
jni::alias_ref<JVisionCameraProxy::javaobject> proxy);
|
||||
};
|
||||
|
||||
} // namespace vision
|
65
android/src/main/cpp/java-bindings/JFrame.cpp
Normal file
65
android/src/main/cpp/java-bindings/JFrame.cpp
Normal file
@ -0,0 +1,65 @@
|
||||
//
|
||||
// Created by Marc on 21.07.2023.
|
||||
//
|
||||
|
||||
#include "JFrame.h"
|
||||
|
||||
#include <jni.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
using namespace jni;
|
||||
|
||||
int JFrame::getWidth() const {
|
||||
static const auto getWidthMethod = getClass()->getMethod<jint()>("getWidth");
|
||||
return getWidthMethod(self());
|
||||
}
|
||||
|
||||
int JFrame::getHeight() const {
|
||||
static const auto getWidthMethod = getClass()->getMethod<jint()>("getHeight");
|
||||
return getWidthMethod(self());
|
||||
}
|
||||
|
||||
bool JFrame::getIsValid() const {
|
||||
static const auto getIsValidMethod = getClass()->getMethod<jboolean()>("getIsValid");
|
||||
return getIsValidMethod(self());
|
||||
}
|
||||
|
||||
bool JFrame::getIsMirrored() const {
|
||||
static const auto getIsMirroredMethod = getClass()->getMethod<jboolean()>("getIsMirrored");
|
||||
return getIsMirroredMethod(self());
|
||||
}
|
||||
|
||||
jlong JFrame::getTimestamp() const {
|
||||
static const auto getTimestampMethod = getClass()->getMethod<jlong()>("getTimestamp");
|
||||
return getTimestampMethod(self());
|
||||
}
|
||||
|
||||
local_ref<JString> JFrame::getOrientation() const {
|
||||
static const auto getOrientationMethod = getClass()->getMethod<JString()>("getOrientation");
|
||||
return getOrientationMethod(self());
|
||||
}
|
||||
|
||||
int JFrame::getPlanesCount() const {
|
||||
static const auto getPlanesCountMethod = getClass()->getMethod<jint()>("getPlanesCount");
|
||||
return getPlanesCountMethod(self());
|
||||
}
|
||||
|
||||
int JFrame::getBytesPerRow() const {
|
||||
static const auto getBytesPerRowMethod = getClass()->getMethod<jint()>("getBytesPerRow");
|
||||
return getBytesPerRowMethod(self());
|
||||
}
|
||||
|
||||
local_ref<JArrayByte> JFrame::toByteArray() const {
|
||||
static const auto toByteArrayMethod = getClass()->getMethod<JArrayByte()>("toByteArray");
|
||||
return toByteArrayMethod(self());
|
||||
}
|
||||
|
||||
void JFrame::close() {
|
||||
static const auto closeMethod = getClass()->getMethod<void()>("close");
|
||||
closeMethod(self());
|
||||
}
|
||||
|
||||
} // namespace vision
|
@ -1,5 +1,5 @@
|
||||
//
|
||||
// Created by Marc on 19/06/2021.
|
||||
// Created by Marc on 21.07.2023.
|
||||
//
|
||||
|
||||
#pragma once
|
||||
@ -12,8 +12,8 @@ namespace vision {
|
||||
using namespace facebook;
|
||||
using namespace jni;
|
||||
|
||||
struct JImageProxy : public JavaClass<JImageProxy> {
|
||||
static constexpr auto kJavaDescriptor = "Landroidx/camera/core/ImageProxy;";
|
||||
struct JFrame : public JavaClass<JFrame> {
|
||||
static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/Frame;";
|
||||
|
||||
public:
|
||||
int getWidth() const;
|
65
android/src/main/cpp/java-bindings/JFrameProcessor.cpp
Normal file
65
android/src/main/cpp/java-bindings/JFrameProcessor.cpp
Normal file
@ -0,0 +1,65 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 29.09.21.
|
||||
//
|
||||
|
||||
#include "JFrameProcessor.h"
|
||||
|
||||
#include <jni.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
|
||||
#include <utility>
|
||||
#include "JFrame.h"
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
using namespace jni;
|
||||
|
||||
void JFrameProcessor::registerNatives() {
|
||||
registerHybrid({
|
||||
makeNativeMethod("call", JFrameProcessor::call)
|
||||
});
|
||||
}
|
||||
|
||||
using TSelf = jni::local_ref<JFrameProcessor::javaobject>;
|
||||
|
||||
JFrameProcessor::JFrameProcessor(std::shared_ptr<RNWorklet::JsiWorklet> worklet,
|
||||
std::shared_ptr<RNWorklet::JsiWorkletContext> context) {
|
||||
_workletContext = std::move(context);
|
||||
_workletInvoker = std::make_shared<RNWorklet::WorkletInvoker>(worklet);
|
||||
}
|
||||
|
||||
TSelf JFrameProcessor::create(const std::shared_ptr<RNWorklet::JsiWorklet>& worklet,
|
||||
const std::shared_ptr<RNWorklet::JsiWorkletContext>& context) {
|
||||
return JFrameProcessor::newObjectCxxArgs(worklet, context);
|
||||
}
|
||||
|
||||
void JFrameProcessor::callWithFrameHostObject(const std::shared_ptr<FrameHostObject>& frameHostObject) const {
|
||||
// Call the Frame Processor on the Worklet Runtime
|
||||
jsi::Runtime& runtime = _workletContext->getWorkletRuntime();
|
||||
|
||||
try {
|
||||
// Wrap HostObject as JSI Value
|
||||
auto argument = jsi::Object::createFromHostObject(runtime, frameHostObject);
|
||||
jsi::Value jsValue(std::move(argument));
|
||||
|
||||
// Call the Worklet with the Frame JS Host Object as an argument
|
||||
_workletInvoker->call(runtime, jsi::Value::undefined(), &jsValue, 1);
|
||||
} catch (jsi::JSError& jsError) {
|
||||
// JS Error occured, print it to console.
|
||||
const std::string& message = jsError.getMessage();
|
||||
|
||||
_workletContext->invokeOnJsThread([message](jsi::Runtime& jsRuntime) {
|
||||
auto logFn = jsRuntime.global().getPropertyAsObject(jsRuntime, "console").getPropertyAsFunction(jsRuntime, "error");
|
||||
logFn.call(jsRuntime, jsi::String::createFromUtf8(jsRuntime, "Frame Processor threw an error: " + message));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
void JFrameProcessor::call(jni::alias_ref<JFrame::javaobject> frame) {
|
||||
// Create the Frame Host Object wrapping the internal Frame
|
||||
auto frameHostObject = std::make_shared<FrameHostObject>(frame);
|
||||
callWithFrameHostObject(frameHostObject);
|
||||
}
|
||||
|
||||
} // namespace vision
|
49
android/src/main/cpp/java-bindings/JFrameProcessor.h
Normal file
49
android/src/main/cpp/java-bindings/JFrameProcessor.h
Normal file
@ -0,0 +1,49 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 29.09.21
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <jni.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
|
||||
#include <react-native-worklets/WKTJsiWorklet.h>
|
||||
#include <react-native-worklets/WKTJsiHostObject.h>
|
||||
|
||||
#include "JFrame.h"
|
||||
#include "FrameHostObject.h"
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
struct JFrameProcessor : public jni::HybridClass<JFrameProcessor> {
|
||||
public:
|
||||
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessor;";
|
||||
static void registerNatives();
|
||||
static jni::local_ref<JFrameProcessor::javaobject> create(const std::shared_ptr<RNWorklet::JsiWorklet>& worklet,
|
||||
const std::shared_ptr<RNWorklet::JsiWorkletContext>& context);
|
||||
|
||||
public:
|
||||
/**
|
||||
* Call the JS Frame Processor.
|
||||
*/
|
||||
void call(alias_ref<JFrame::javaobject> frame);
|
||||
|
||||
private:
|
||||
// Private constructor. Use `create(..)` to create new instances.
|
||||
explicit JFrameProcessor(std::shared_ptr<RNWorklet::JsiWorklet> worklet,
|
||||
std::shared_ptr<RNWorklet::JsiWorkletContext> context);
|
||||
|
||||
private:
|
||||
void callWithFrameHostObject(const std::shared_ptr<FrameHostObject>& frameHostObject) const;
|
||||
|
||||
private:
|
||||
friend HybridBase;
|
||||
std::shared_ptr<RNWorklet::WorkletInvoker> _workletInvoker;
|
||||
std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;
|
||||
};
|
||||
|
||||
} // namespace vision
|
@ -12,19 +12,14 @@ namespace vision {
|
||||
using namespace facebook;
|
||||
using namespace jni;
|
||||
|
||||
using TCallback = jobject(alias_ref<JImageProxy::javaobject>, alias_ref<JArrayClass<jobject>>);
|
||||
using TCallback = jobject(alias_ref<JFrame::javaobject>, alias_ref<react::ReadableNativeMap::javaobject> params);
|
||||
|
||||
local_ref<jobject> JFrameProcessorPlugin::callback(alias_ref<JImageProxy::javaobject> image,
|
||||
alias_ref<JArrayClass<jobject>> params) const {
|
||||
local_ref<jobject> JFrameProcessorPlugin::callback(const alias_ref<JFrame::javaobject>& frame,
|
||||
const alias_ref<react::ReadableNativeMap::javaobject>& params) const {
|
||||
auto callbackMethod = getClass()->getMethod<TCallback>("callback");
|
||||
|
||||
auto result = callbackMethod(self(), image, params);
|
||||
auto result = callbackMethod(self(), frame, params);
|
||||
return make_local(result);
|
||||
}
|
||||
|
||||
std::string JFrameProcessorPlugin::getName() const {
|
||||
auto getNameMethod = getClass()->getMethod<jstring()>("getName");
|
||||
return getNameMethod(self())->toStdString();
|
||||
}
|
||||
|
||||
} // namespace vision
|
||||
|
@ -7,8 +7,9 @@
|
||||
#include <jni.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
#include <string>
|
||||
#include <react/jni/ReadableNativeMap.h>
|
||||
|
||||
#include "JImageProxy.h"
|
||||
#include "JFrame.h"
|
||||
|
||||
namespace vision {
|
||||
|
||||
@ -22,12 +23,8 @@ struct JFrameProcessorPlugin : public JavaClass<JFrameProcessorPlugin> {
|
||||
/**
|
||||
* Call the plugin.
|
||||
*/
|
||||
local_ref<jobject> callback(alias_ref<JImageProxy::javaobject> image,
|
||||
alias_ref<JArrayClass<jobject>> params) const;
|
||||
/**
|
||||
* Get the user-defined name of the Frame Processor Plugin
|
||||
*/
|
||||
std::string getName() const;
|
||||
local_ref<jobject> callback(const alias_ref<JFrame::javaobject>& frame,
|
||||
const alias_ref<react::ReadableNativeMap::javaobject>& params) const;
|
||||
};
|
||||
|
||||
} // namespace vision
|
||||
|
@ -1,78 +0,0 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 22.06.21.
|
||||
//
|
||||
|
||||
#include "JImageProxy.h"
|
||||
|
||||
#include <jni.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
using namespace jni;
|
||||
|
||||
int JImageProxy::getWidth() const {
|
||||
static const auto getWidthMethod = getClass()->getMethod<jint()>("getWidth");
|
||||
return getWidthMethod(self());
|
||||
}
|
||||
|
||||
int JImageProxy::getHeight() const {
|
||||
static const auto getWidthMethod = getClass()->getMethod<jint()>("getHeight");
|
||||
return getWidthMethod(self());
|
||||
}
|
||||
|
||||
alias_ref<JClass> getUtilsClass() {
|
||||
static const auto ImageProxyUtilsClass = findClassStatic("com/mrousavy/camera/frameprocessor/ImageProxyUtils");
|
||||
return ImageProxyUtilsClass;
|
||||
}
|
||||
|
||||
bool JImageProxy::getIsValid() const {
|
||||
auto utilsClass = getUtilsClass();
|
||||
static const auto isImageProxyValidMethod = utilsClass->getStaticMethod<jboolean(JImageProxy::javaobject)>("isImageProxyValid");
|
||||
return isImageProxyValidMethod(utilsClass, self());
|
||||
}
|
||||
|
||||
bool JImageProxy::getIsMirrored() const {
|
||||
auto utilsClass = getUtilsClass();
|
||||
static const auto isImageProxyMirroredMethod = utilsClass->getStaticMethod<jboolean(JImageProxy::javaobject)>("isImageProxyMirrored");
|
||||
return isImageProxyMirroredMethod(utilsClass, self());
|
||||
}
|
||||
|
||||
jlong JImageProxy::getTimestamp() const {
|
||||
auto utilsClass = getUtilsClass();
|
||||
static const auto getTimestampMethod = utilsClass->getStaticMethod<jlong(JImageProxy::javaobject)>("getTimestamp");
|
||||
return getTimestampMethod(utilsClass, self());
|
||||
}
|
||||
|
||||
local_ref<JString> JImageProxy::getOrientation() const {
|
||||
auto utilsClass = getUtilsClass();
|
||||
static const auto getOrientationMethod = utilsClass->getStaticMethod<JString(JImageProxy::javaobject)>("getOrientation");
|
||||
return getOrientationMethod(utilsClass, self());
|
||||
}
|
||||
|
||||
int JImageProxy::getPlanesCount() const {
|
||||
auto utilsClass = getUtilsClass();
|
||||
static const auto getPlanesCountMethod = utilsClass->getStaticMethod<jint(JImageProxy::javaobject)>("getPlanesCount");
|
||||
return getPlanesCountMethod(utilsClass, self());
|
||||
}
|
||||
|
||||
int JImageProxy::getBytesPerRow() const {
|
||||
auto utilsClass = getUtilsClass();
|
||||
static const auto getBytesPerRowMethod = utilsClass->getStaticMethod<jint(JImageProxy::javaobject)>("getBytesPerRow");
|
||||
return getBytesPerRowMethod(utilsClass, self());
|
||||
}
|
||||
|
||||
local_ref<JArrayByte> JImageProxy::toByteArray() const {
|
||||
auto utilsClass = getUtilsClass();
|
||||
|
||||
static const auto toByteArrayMethod = utilsClass->getStaticMethod<JArrayByte(JImageProxy::javaobject)>("toByteArray");
|
||||
return toByteArrayMethod(utilsClass, self());
|
||||
}
|
||||
|
||||
void JImageProxy::close() {
|
||||
static const auto closeMethod = getClass()->getMethod<void()>("close");
|
||||
closeMethod(self());
|
||||
}
|
||||
|
||||
} // namespace vision
|
91
android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp
Normal file
91
android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp
Normal file
@ -0,0 +1,91 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 21.07.23.
|
||||
//
|
||||
|
||||
#include "JVisionCameraProxy.h"
|
||||
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
#include <string>
|
||||
|
||||
#include <jsi/jsi.h>
|
||||
#include <react/jni/ReadableNativeMap.h>
|
||||
|
||||
#include <react-native-worklets/WKTJsiWorklet.h>
|
||||
#include <react-native-worklets/WKTJsiWorkletContext.h>
|
||||
|
||||
#include "FrameProcessorPluginHostObject.h"
|
||||
|
||||
namespace vision {
|
||||
|
||||
using TSelf = local_ref<HybridClass<JVisionCameraProxy>::jhybriddata>;
|
||||
using TJSCallInvokerHolder = jni::alias_ref<facebook::react::CallInvokerHolder::javaobject>;
|
||||
using TScheduler = jni::alias_ref<JVisionCameraScheduler::javaobject>;
|
||||
using TOptions = jni::local_ref<react::ReadableNativeMap::javaobject>;
|
||||
|
||||
JVisionCameraProxy::JVisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::jhybridobject>& javaThis,
|
||||
jsi::Runtime* runtime,
|
||||
const std::shared_ptr<facebook::react::CallInvoker>& callInvoker,
|
||||
const jni::global_ref<JVisionCameraScheduler::javaobject>& scheduler) {
|
||||
_javaPart = make_global(javaThis);
|
||||
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG, "Creating Worklet Context...");
|
||||
|
||||
auto runOnJS = [callInvoker](std::function<void()>&& f) {
|
||||
// Run on React JS Runtime
|
||||
callInvoker->invokeAsync(std::move(f));
|
||||
};
|
||||
auto runOnWorklet = [scheduler](std::function<void()>&& f) {
|
||||
// Run on Frame Processor Worklet Runtime
|
||||
scheduler->cthis()->dispatchAsync([f = std::move(f)](){
|
||||
f();
|
||||
});
|
||||
};
|
||||
_workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera",
|
||||
runtime,
|
||||
runOnJS,
|
||||
runOnWorklet);
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG, "Worklet Context created!");
|
||||
}
|
||||
|
||||
|
||||
|
||||
void JVisionCameraProxy::setFrameProcessor(int viewTag,
|
||||
const alias_ref<JFrameProcessor::javaobject>& frameProcessor) {
|
||||
auto setFrameProcessorMethod = javaClassLocal()->getMethod<void(int, alias_ref<JFrameProcessor::javaobject>)>("setFrameProcessor");
|
||||
setFrameProcessorMethod(_javaPart, viewTag, frameProcessor);
|
||||
}
|
||||
|
||||
void JVisionCameraProxy::removeFrameProcessor(int viewTag) {
|
||||
auto removeFrameProcessorMethod = javaClassLocal()->getMethod<void(int)>("removeFrameProcessor");
|
||||
removeFrameProcessorMethod(_javaPart, viewTag);
|
||||
}
|
||||
|
||||
local_ref<JFrameProcessorPlugin::javaobject> JVisionCameraProxy::getFrameProcessorPlugin(const std::string& name,
|
||||
TOptions options) {
|
||||
auto getFrameProcessorPluginMethod = javaClassLocal()->getMethod<JFrameProcessorPlugin(local_ref<jstring>, TOptions)>("getFrameProcessorPlugin");
|
||||
return getFrameProcessorPluginMethod(_javaPart, make_jstring(name), std::move(options));
|
||||
}
|
||||
|
||||
void JVisionCameraProxy::registerNatives() {
|
||||
registerHybrid({
|
||||
makeNativeMethod("initHybrid", JVisionCameraProxy::initHybrid)
|
||||
});
|
||||
}
|
||||
|
||||
TSelf JVisionCameraProxy::initHybrid(
|
||||
alias_ref<jhybridobject> jThis,
|
||||
jlong jsRuntimePointer,
|
||||
TJSCallInvokerHolder jsCallInvokerHolder,
|
||||
const TScheduler& scheduler) {
|
||||
__android_log_write(ANDROID_LOG_INFO, TAG, "Initializing VisionCameraProxy...");
|
||||
|
||||
// cast from JNI hybrid objects to C++ instances
|
||||
auto jsRuntime = reinterpret_cast<jsi::Runtime*>(jsRuntimePointer);
|
||||
auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker();
|
||||
auto sharedScheduler = make_global(scheduler);
|
||||
|
||||
return makeCxxInstance(jThis, jsRuntime, jsCallInvoker, sharedScheduler);
|
||||
}
|
||||
|
||||
} // namespace vision
|
55
android/src/main/cpp/java-bindings/JVisionCameraProxy.h
Normal file
55
android/src/main/cpp/java-bindings/JVisionCameraProxy.h
Normal file
@ -0,0 +1,55 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 21.07.23.
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <fbjni/fbjni.h>
|
||||
#include <jsi/jsi.h>
|
||||
#include <react-native-worklets/WKTJsiWorkletContext.h>
|
||||
#include <react/jni/ReadableNativeMap.h>
|
||||
|
||||
#include "JFrameProcessorPlugin.h"
|
||||
#include "JVisionCameraScheduler.h"
|
||||
#include "JFrameProcessor.h"
|
||||
|
||||
#include <string>
|
||||
#include <memory>
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
class JVisionCameraProxy : public jni::HybridClass<JVisionCameraProxy> {
|
||||
public:
|
||||
static void registerNatives();
|
||||
|
||||
void setFrameProcessor(int viewTag,
|
||||
const jni::alias_ref<JFrameProcessor::javaobject>& frameProcessor);
|
||||
void removeFrameProcessor(int viewTag);
|
||||
jni::local_ref<JFrameProcessorPlugin::javaobject> getFrameProcessorPlugin(const std::string& name,
|
||||
jni::local_ref<react::ReadableNativeMap::javaobject> options);
|
||||
|
||||
public:
|
||||
std::shared_ptr<RNWorklet::JsiWorkletContext> getWorkletContext() { return _workletContext; }
|
||||
|
||||
private:
|
||||
std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;
|
||||
|
||||
private:
|
||||
friend HybridBase;
|
||||
jni::global_ref<JVisionCameraProxy::javaobject> _javaPart;
|
||||
static auto constexpr TAG = "VisionCameraProxy";
|
||||
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraProxy;";
|
||||
|
||||
explicit JVisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::jhybridobject>& javaThis,
|
||||
jsi::Runtime* jsRuntime,
|
||||
const std::shared_ptr<facebook::react::CallInvoker>& jsCallInvoker,
|
||||
const jni::global_ref<JVisionCameraScheduler::javaobject>& scheduler);
|
||||
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> javaThis,
|
||||
jlong jsRuntimePointer,
|
||||
jni::alias_ref<facebook::react::CallInvokerHolder::javaobject> jsCallInvokerHolder,
|
||||
const jni::alias_ref<JVisionCameraScheduler::javaobject>& scheduler);
|
||||
};
|
||||
|
||||
} // namespace vision
|
@ -2,31 +2,31 @@
|
||||
// Created by Marc Rousavy on 25.07.21.
|
||||
//
|
||||
|
||||
#include "VisionCameraScheduler.h"
|
||||
#include "JVisionCameraScheduler.h"
|
||||
#include <fbjni/fbjni.h>
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
using TSelf = jni::local_ref<VisionCameraScheduler::jhybriddata>;
|
||||
using TSelf = jni::local_ref<JVisionCameraScheduler::jhybriddata>;
|
||||
|
||||
TSelf VisionCameraScheduler::initHybrid(jni::alias_ref<jhybridobject> jThis) {
|
||||
TSelf JVisionCameraScheduler::initHybrid(jni::alias_ref<jhybridobject> jThis) {
|
||||
return makeCxxInstance(jThis);
|
||||
}
|
||||
|
||||
void VisionCameraScheduler::dispatchAsync(std::function<void()> job) {
|
||||
void JVisionCameraScheduler::dispatchAsync(const std::function<void()>& job) {
|
||||
// 1. add job to queue
|
||||
_jobs.push(job);
|
||||
scheduleTrigger();
|
||||
}
|
||||
|
||||
void VisionCameraScheduler::scheduleTrigger() {
|
||||
void JVisionCameraScheduler::scheduleTrigger() {
|
||||
// 2. schedule `triggerUI` to be called on the java thread
|
||||
static auto method = javaPart_->getClass()->getMethod<void()>("scheduleTrigger");
|
||||
method(javaPart_.get());
|
||||
}
|
||||
|
||||
void VisionCameraScheduler::trigger() {
|
||||
void JVisionCameraScheduler::trigger() {
|
||||
std::unique_lock<std::mutex> lock(_mutex);
|
||||
// 3. call job we enqueued in step 1.
|
||||
auto job = _jobs.front();
|
||||
@ -34,10 +34,10 @@ void VisionCameraScheduler::trigger() {
|
||||
_jobs.pop();
|
||||
}
|
||||
|
||||
void VisionCameraScheduler::registerNatives() {
|
||||
void JVisionCameraScheduler::registerNatives() {
|
||||
registerHybrid({
|
||||
makeNativeMethod("initHybrid", VisionCameraScheduler::initHybrid),
|
||||
makeNativeMethod("trigger", VisionCameraScheduler::trigger),
|
||||
makeNativeMethod("initHybrid", JVisionCameraScheduler::initHybrid),
|
||||
makeNativeMethod("trigger", JVisionCameraScheduler::trigger),
|
||||
});
|
||||
}
|
||||
|
@ -23,22 +23,22 @@ using namespace facebook;
|
||||
* 3. The `scheduleTrigger()` Java Method will switch to the Frame Processor Java Thread and call `trigger()` on there
|
||||
* 4. `trigger()` is a C++ function here that just calls the passed C++ Method from step 1.
|
||||
*/
|
||||
class VisionCameraScheduler : public jni::HybridClass<VisionCameraScheduler> {
|
||||
class JVisionCameraScheduler : public jni::HybridClass<JVisionCameraScheduler> {
|
||||
public:
|
||||
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraScheduler;";
|
||||
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis);
|
||||
static void registerNatives();
|
||||
|
||||
// schedules the given job to be run on the VisionCamera FP Thread at some future point in time
|
||||
void dispatchAsync(std::function<void()> job);
|
||||
void dispatchAsync(const std::function<void()>& job);
|
||||
|
||||
private:
|
||||
friend HybridBase;
|
||||
jni::global_ref<VisionCameraScheduler::javaobject> javaPart_;
|
||||
jni::global_ref<JVisionCameraScheduler::javaobject> javaPart_;
|
||||
std::queue<std::function<void()>> _jobs;
|
||||
std::mutex _mutex;
|
||||
|
||||
explicit VisionCameraScheduler(jni::alias_ref<VisionCameraScheduler::jhybridobject> jThis):
|
||||
explicit JVisionCameraScheduler(jni::alias_ref<JVisionCameraScheduler::jhybridobject> jThis):
|
||||
javaPart_(jni::make_global(jThis)) {}
|
||||
|
||||
// Schedules a call to `trigger` on the VisionCamera FP Thread
|
@ -18,7 +18,7 @@ suspend fun CameraView.focus(pointMap: ReadableMap) {
|
||||
|
||||
// Getting the point from the previewView needs to be run on the UI thread
|
||||
val point = withContext(coroutineScope.coroutineContext) {
|
||||
previewView.meteringPointFactory.createPoint(x.toFloat(), y.toFloat());
|
||||
previewView.meteringPointFactory.createPoint(x.toFloat(), y.toFloat())
|
||||
}
|
||||
|
||||
val action = FocusMeteringAction.Builder(point, FocusMeteringAction.FLAG_AF or FocusMeteringAction.FLAG_AE)
|
||||
|
@ -14,7 +14,7 @@ import java.io.FileOutputStream
|
||||
import kotlinx.coroutines.guava.await
|
||||
|
||||
suspend fun CameraView.takeSnapshot(options: ReadableMap): WritableMap = coroutineScope {
|
||||
val camera = camera ?: throw com.mrousavy.camera.CameraNotReadyError()
|
||||
val camera = camera ?: throw CameraNotReadyError()
|
||||
val enableFlash = options.getString("flash") == "on"
|
||||
|
||||
try {
|
||||
|
@ -24,15 +24,16 @@ import androidx.lifecycle.*
|
||||
import com.facebook.jni.HybridData
|
||||
import com.facebook.proguard.annotations.DoNotStrip
|
||||
import com.facebook.react.bridge.*
|
||||
import com.facebook.react.uimanager.events.RCTEventEmitter
|
||||
import com.mrousavy.camera.frameprocessor.FrameProcessorRuntimeManager
|
||||
import com.mrousavy.camera.frameprocessor.Frame
|
||||
import com.mrousavy.camera.frameprocessor.FrameProcessor
|
||||
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin
|
||||
import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry
|
||||
import com.mrousavy.camera.utils.*
|
||||
import kotlinx.coroutines.*
|
||||
import kotlinx.coroutines.guava.await
|
||||
import java.lang.IllegalArgumentException
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import kotlin.math.floor
|
||||
import kotlin.math.max
|
||||
import kotlin.math.min
|
||||
|
||||
@ -118,8 +119,9 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
|
||||
internal var camera: Camera? = null
|
||||
internal var imageCapture: ImageCapture? = null
|
||||
internal var videoCapture: VideoCapture<Recorder>? = null
|
||||
private var imageAnalysis: ImageAnalysis? = null
|
||||
public var frameProcessor: FrameProcessor? = null
|
||||
private var preview: Preview? = null
|
||||
private var imageAnalysis: ImageAnalysis? = null
|
||||
|
||||
internal var activeVideoRecording: Recording? = null
|
||||
|
||||
@ -156,10 +158,7 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
|
||||
private var minZoom: Float = 1f
|
||||
private var maxZoom: Float = 1f
|
||||
|
||||
@DoNotStrip
|
||||
private var mHybridData: HybridData? = null
|
||||
|
||||
@Suppress("LiftReturnOrAssignment", "RedundantIf")
|
||||
@Suppress("RedundantIf")
|
||||
internal val fallbackToSnapshot: Boolean
|
||||
@SuppressLint("UnsafeOptInUsageError")
|
||||
get() {
|
||||
@ -190,8 +189,6 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
|
||||
}
|
||||
|
||||
init {
|
||||
mHybridData = initHybrid()
|
||||
|
||||
previewView = PreviewView(context)
|
||||
previewView.layoutParams = LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT)
|
||||
previewView.installHierarchyFitter() // If this is not called correctly, view finder will be black/blank
|
||||
@ -244,9 +241,6 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
|
||||
imageAnalysis?.targetRotation = outputRotation
|
||||
}
|
||||
|
||||
private external fun initHybrid(): HybridData
|
||||
private external fun frameProcessorCallback(frame: ImageProxy)
|
||||
|
||||
override fun getLifecycle(): Lifecycle {
|
||||
return lifecycleRegistry
|
||||
}
|
||||
@ -461,8 +455,9 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
|
||||
imageAnalysis = imageAnalysisBuilder.build().apply {
|
||||
setAnalyzer(cameraExecutor) { image ->
|
||||
// Call JS Frame Processor
|
||||
frameProcessorCallback(image)
|
||||
// frame gets closed in FrameHostObject implementation (JS ref counting)
|
||||
val frame = Frame(image)
|
||||
frameProcessor?.call(frame)
|
||||
// ...frame gets closed in FrameHostObject implementation via JS ref counting
|
||||
}
|
||||
}
|
||||
useCases.add(imageAnalysis!!)
|
||||
|
@ -3,16 +3,11 @@ package com.mrousavy.camera
|
||||
import android.Manifest
|
||||
import android.content.Context
|
||||
import android.content.pm.PackageManager
|
||||
import android.hardware.camera2.CameraCharacteristics
|
||||
import android.hardware.camera2.CameraManager
|
||||
import android.os.Build
|
||||
import android.util.Log
|
||||
import android.util.Size
|
||||
import androidx.camera.core.CameraSelector
|
||||
import androidx.camera.extensions.ExtensionMode
|
||||
import androidx.camera.extensions.ExtensionsManager
|
||||
import androidx.camera.lifecycle.ProcessCameraProvider
|
||||
import androidx.camera.video.QualitySelector
|
||||
import androidx.core.content.ContextCompat
|
||||
import com.facebook.react.bridge.*
|
||||
import com.facebook.react.module.annotations.ReactModule
|
||||
@ -20,8 +15,9 @@ import com.facebook.react.modules.core.PermissionAwareActivity
|
||||
import com.facebook.react.modules.core.PermissionListener
|
||||
import com.facebook.react.uimanager.UIManagerHelper
|
||||
import com.facebook.react.bridge.ReactApplicationContext
|
||||
import com.mrousavy.camera.frameprocessor.VisionCameraInstaller
|
||||
import java.util.concurrent.ExecutorService
|
||||
import com.mrousavy.camera.frameprocessor.FrameProcessorRuntimeManager
|
||||
import com.mrousavy.camera.frameprocessor.VisionCameraProxy
|
||||
import com.mrousavy.camera.parsers.*
|
||||
import com.mrousavy.camera.utils.*
|
||||
import kotlinx.coroutines.*
|
||||
@ -30,38 +26,21 @@ import java.util.concurrent.Executors
|
||||
|
||||
@ReactModule(name = CameraViewModule.TAG)
|
||||
@Suppress("unused")
|
||||
class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBaseJavaModule(reactContext) {
|
||||
class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJavaModule(reactContext) {
|
||||
companion object {
|
||||
const val TAG = "CameraView"
|
||||
var RequestCode = 10
|
||||
|
||||
fun parsePermissionStatus(status: Int): String {
|
||||
return when (status) {
|
||||
PackageManager.PERMISSION_DENIED -> "denied"
|
||||
PackageManager.PERMISSION_GRANTED -> "authorized"
|
||||
else -> "not-determined"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var frameProcessorThread: ExecutorService = Executors.newSingleThreadExecutor()
|
||||
private val coroutineScope = CoroutineScope(Dispatchers.Default) // TODO: or Dispatchers.Main?
|
||||
private var frameProcessorManager: FrameProcessorRuntimeManager? = null
|
||||
|
||||
private fun cleanup() {
|
||||
if (coroutineScope.isActive) {
|
||||
coroutineScope.cancel("CameraViewModule has been destroyed.")
|
||||
}
|
||||
}
|
||||
|
||||
override fun onCatalystInstanceDestroy() {
|
||||
super.onCatalystInstanceDestroy()
|
||||
cleanup()
|
||||
}
|
||||
|
||||
override fun invalidate() {
|
||||
super.invalidate()
|
||||
cleanup()
|
||||
frameProcessorThread.shutdown()
|
||||
if (coroutineScope.isActive) {
|
||||
coroutineScope.cancel("CameraViewModule has been destroyed.")
|
||||
}
|
||||
}
|
||||
|
||||
override fun getName(): String {
|
||||
@ -75,6 +54,18 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase
|
||||
return view ?: throw ViewNotFoundError(viewId)
|
||||
}
|
||||
|
||||
@ReactMethod(isBlockingSynchronousMethod = true)
|
||||
fun installFrameProcessorBindings(): Boolean {
|
||||
return try {
|
||||
val proxy = VisionCameraProxy(reactApplicationContext, frameProcessorThread)
|
||||
VisionCameraInstaller.install(proxy)
|
||||
true
|
||||
} catch (e: Error) {
|
||||
Log.e(TAG, "Failed to install Frame Processor JSI Bindings!", e)
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
@ReactMethod
|
||||
fun takePhoto(viewTag: Int, options: ReadableMap, promise: Promise) {
|
||||
coroutineScope.launch {
|
||||
@ -151,18 +142,6 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase
|
||||
}
|
||||
}
|
||||
|
||||
@ReactMethod(isBlockingSynchronousMethod = true)
|
||||
fun installFrameProcessorBindings(): Boolean {
|
||||
try {
|
||||
frameProcessorManager = FrameProcessorRuntimeManager(reactApplicationContext, frameProcessorThread)
|
||||
frameProcessorManager!!.installBindings()
|
||||
return true
|
||||
} catch (e: Error) {
|
||||
Log.e(TAG, "Failed to install Frame Processor JSI Bindings!", e)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
@ReactMethod
|
||||
fun getAvailableCameraDevices(promise: Promise) {
|
||||
coroutineScope.launch {
|
||||
|
@ -43,12 +43,12 @@ class ParallelVideoProcessingNotSupportedError(cause: Throwable) : CameraError("
|
||||
"See https://mrousavy.github.io/react-native-vision-camera/docs/guides/devices#the-supportsparallelvideoprocessing-prop for more information.", cause)
|
||||
|
||||
class FpsNotContainedInFormatError(fps: Int) : CameraError("format", "invalid-fps", "The given FPS were not valid for the currently selected format. Make sure you select a format which `frameRateRanges` includes $fps FPS!")
|
||||
class HdrNotContainedInFormatError() : CameraError(
|
||||
class HdrNotContainedInFormatError : CameraError(
|
||||
"format", "invalid-hdr",
|
||||
"The currently selected format does not support HDR capture! " +
|
||||
"Make sure you select a format which `frameRateRanges` includes `supportsPhotoHDR`!"
|
||||
)
|
||||
class LowLightBoostNotContainedInFormatError() : CameraError(
|
||||
class LowLightBoostNotContainedInFormatError : CameraError(
|
||||
"format", "invalid-low-light-boost",
|
||||
"The currently selected format does not support low-light boost (night mode)! " +
|
||||
"Make sure you select a format which includes `supportsLowLightBoost`."
|
||||
|
@ -0,0 +1,120 @@
|
||||
package com.mrousavy.camera.frameprocessor;
|
||||
|
||||
import android.annotation.SuppressLint;
|
||||
import android.graphics.ImageFormat;
|
||||
import android.graphics.Matrix;
|
||||
import android.media.Image;
|
||||
import androidx.camera.core.ImageProxy;
|
||||
import com.facebook.proguard.annotations.DoNotStrip;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
public class Frame {
|
||||
private final ImageProxy imageProxy;
|
||||
|
||||
public Frame(ImageProxy imageProxy) {
|
||||
this.imageProxy = imageProxy;
|
||||
}
|
||||
|
||||
public ImageProxy getImageProxy() {
|
||||
return imageProxy;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@DoNotStrip
|
||||
public int getWidth() {
|
||||
return imageProxy.getWidth();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@DoNotStrip
|
||||
public int getHeight() {
|
||||
return imageProxy.getHeight();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@DoNotStrip
|
||||
public boolean getIsValid() {
|
||||
try {
|
||||
@SuppressLint("UnsafeOptInUsageError")
|
||||
Image image = imageProxy.getImage();
|
||||
if (image == null) return false;
|
||||
// will throw an exception if the image is already closed
|
||||
image.getCropRect();
|
||||
// no exception thrown, image must still be valid.
|
||||
return true;
|
||||
} catch (Exception e) {
|
||||
// exception thrown, image has already been closed.
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@DoNotStrip
|
||||
public boolean getIsMirrored() {
|
||||
Matrix matrix = imageProxy.getImageInfo().getSensorToBufferTransformMatrix();
|
||||
// TODO: Figure out how to get isMirrored from ImageProxy
|
||||
return false;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@DoNotStrip
|
||||
public long getTimestamp() {
|
||||
return imageProxy.getImageInfo().getTimestamp();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@DoNotStrip
|
||||
public String getOrientation() {
|
||||
int rotation = imageProxy.getImageInfo().getRotationDegrees();
|
||||
if (rotation >= 45 && rotation < 135)
|
||||
return "landscapeRight";
|
||||
if (rotation >= 135 && rotation < 225)
|
||||
return "portraitUpsideDown";
|
||||
if (rotation >= 225 && rotation < 315)
|
||||
return "landscapeLeft";
|
||||
return "portrait";
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@DoNotStrip
|
||||
public int getPlanesCount() {
|
||||
return imageProxy.getPlanes().length;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@DoNotStrip
|
||||
public int getBytesPerRow() {
|
||||
return imageProxy.getPlanes()[0].getRowStride();
|
||||
}
|
||||
|
||||
private static byte[] byteArrayCache;
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@DoNotStrip
|
||||
public byte[] toByteArray() {
|
||||
switch (imageProxy.getFormat()) {
|
||||
case ImageFormat.YUV_420_888:
|
||||
ByteBuffer yBuffer = imageProxy.getPlanes()[0].getBuffer();
|
||||
ByteBuffer vuBuffer = imageProxy.getPlanes()[2].getBuffer();
|
||||
int ySize = yBuffer.remaining();
|
||||
int vuSize = vuBuffer.remaining();
|
||||
|
||||
if (byteArrayCache == null || byteArrayCache.length != ySize + vuSize) {
|
||||
byteArrayCache = new byte[ySize + vuSize];
|
||||
}
|
||||
|
||||
yBuffer.get(byteArrayCache, 0, ySize);
|
||||
vuBuffer.get(byteArrayCache, ySize, vuSize);
|
||||
|
||||
return byteArrayCache;
|
||||
default:
|
||||
throw new RuntimeException("Cannot convert Frame with Format " + imageProxy.getFormat() + " to byte array!");
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@DoNotStrip
|
||||
private void close() {
|
||||
imageProxy.close();
|
||||
}
|
||||
}
|
@ -0,0 +1,27 @@
|
||||
package com.mrousavy.camera.frameprocessor;
|
||||
|
||||
import androidx.annotation.Keep;
|
||||
import androidx.annotation.NonNull;
|
||||
import androidx.annotation.Nullable;
|
||||
|
||||
import com.facebook.jni.HybridData;
|
||||
import com.facebook.proguard.annotations.DoNotStrip;
|
||||
|
||||
/**
|
||||
* Represents a JS Frame Processor
|
||||
*/
|
||||
@SuppressWarnings("JavaJniMissingFunction") // we're using fbjni.
|
||||
public final class FrameProcessor {
|
||||
/**
|
||||
* Call the JS Frame Processor function with the given Frame
|
||||
*/
|
||||
public native void call(Frame frame);
|
||||
|
||||
@DoNotStrip
|
||||
@Keep
|
||||
private final HybridData mHybridData;
|
||||
|
||||
public FrameProcessor(HybridData hybridData) {
|
||||
mHybridData = hybridData;
|
||||
}
|
||||
}
|
@ -3,8 +3,8 @@ package com.mrousavy.camera.frameprocessor;
|
||||
import androidx.annotation.Keep;
|
||||
import androidx.annotation.NonNull;
|
||||
import androidx.annotation.Nullable;
|
||||
import androidx.camera.core.ImageProxy;
|
||||
import com.facebook.proguard.annotations.DoNotStrip;
|
||||
import com.facebook.react.bridge.ReadableNativeMap;
|
||||
|
||||
/**
|
||||
* Declares a Frame Processor Plugin.
|
||||
@ -12,42 +12,14 @@ import com.facebook.proguard.annotations.DoNotStrip;
|
||||
@DoNotStrip
|
||||
@Keep
|
||||
public abstract class FrameProcessorPlugin {
|
||||
private final @NonNull String mName;
|
||||
|
||||
/**
|
||||
* The actual Frame Processor plugin callback. Called for every frame the ImageAnalyzer receives.
|
||||
* @param image The CameraX ImageProxy. Don't call .close() on this, as VisionCamera handles that.
|
||||
* @param frame The Frame from the Camera. Don't call .close() on this, as VisionCamera handles that.
|
||||
* @return You can return any primitive, map or array you want. See the
|
||||
* <a href="https://mrousavy.github.io/react-native-vision-camera/docs/guides/frame-processors-plugins-overview#types">Types</a>
|
||||
* table for a list of supported types.
|
||||
*/
|
||||
@DoNotStrip
|
||||
@Keep
|
||||
public abstract @Nullable Object callback(@NonNull ImageProxy image, @NonNull Object[] params);
|
||||
|
||||
/**
|
||||
* Initializes the native plugin part.
|
||||
* @param name Specifies the Frame Processor Plugin's name in the Runtime.
|
||||
* The actual name in the JS Runtime will be prefixed with two underscores (`__`)
|
||||
*/
|
||||
protected FrameProcessorPlugin(@NonNull String name) {
|
||||
mName = name;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the user-defined name of the Frame Processor Plugin.
|
||||
*/
|
||||
@DoNotStrip
|
||||
@Keep
|
||||
public @NonNull String getName() {
|
||||
return mName;
|
||||
}
|
||||
|
||||
/**
|
||||
* Registers the given plugin in the Frame Processor Runtime.
|
||||
* @param plugin An instance of a plugin.
|
||||
*/
|
||||
public static void register(@NonNull FrameProcessorPlugin plugin) {
|
||||
FrameProcessorRuntimeManager.Companion.addPlugin(plugin);
|
||||
}
|
||||
public abstract @Nullable Object callback(@NonNull Frame frame, @Nullable ReadableNativeMap params);
|
||||
}
|
||||
|
@ -0,0 +1,38 @@
|
||||
package com.mrousavy.camera.frameprocessor;
|
||||
|
||||
import androidx.annotation.Keep;
|
||||
import androidx.annotation.Nullable;
|
||||
|
||||
import com.facebook.proguard.annotations.DoNotStrip;
|
||||
import com.facebook.react.bridge.ReadableNativeMap;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
|
||||
@DoNotStrip
|
||||
@Keep
|
||||
public class FrameProcessorPluginRegistry {
|
||||
private static final Map<String, PluginInitializer> Plugins = new HashMap<>();
|
||||
|
||||
@DoNotStrip
|
||||
@Keep
|
||||
public static void addFrameProcessorPlugin(String name, PluginInitializer pluginInitializer) {
|
||||
assert !Plugins.containsKey(name) : "Tried to add a Frame Processor Plugin with a name that already exists! " +
|
||||
"Either choose unique names, or remove the unused plugin. Name: ";
|
||||
Plugins.put(name, pluginInitializer);
|
||||
}
|
||||
|
||||
@DoNotStrip
|
||||
@Keep
|
||||
public static FrameProcessorPlugin getPlugin(String name, ReadableNativeMap options) {
|
||||
PluginInitializer initializer = Plugins.get(name);
|
||||
if (initializer == null) {
|
||||
return null;
|
||||
}
|
||||
return initializer.initializePlugin(options);
|
||||
}
|
||||
|
||||
public interface PluginInitializer {
|
||||
FrameProcessorPlugin initializePlugin(@Nullable ReadableNativeMap options);
|
||||
}
|
||||
}
|
@ -1,98 +0,0 @@
|
||||
package com.mrousavy.camera.frameprocessor;
|
||||
|
||||
import android.annotation.SuppressLint;
|
||||
import android.graphics.ImageFormat;
|
||||
import android.graphics.Matrix;
|
||||
import android.media.Image;
|
||||
|
||||
import androidx.annotation.Keep;
|
||||
import androidx.camera.core.ImageProxy;
|
||||
import com.facebook.proguard.annotations.DoNotStrip;
|
||||
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
@SuppressWarnings("unused") // used through JNI
|
||||
@DoNotStrip
|
||||
@Keep
|
||||
public class ImageProxyUtils {
|
||||
@SuppressLint("UnsafeOptInUsageError")
|
||||
@DoNotStrip
|
||||
@Keep
|
||||
public static boolean isImageProxyValid(ImageProxy imageProxy) {
|
||||
try {
|
||||
Image image = imageProxy.getImage();
|
||||
if (image == null) return false;
|
||||
// will throw an exception if the image is already closed
|
||||
image.getCropRect();
|
||||
// no exception thrown, image must still be valid.
|
||||
return true;
|
||||
} catch (Exception e) {
|
||||
// exception thrown, image has already been closed.
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@DoNotStrip
|
||||
@Keep
|
||||
public static boolean isImageProxyMirrored(ImageProxy imageProxy) {
|
||||
Matrix matrix = imageProxy.getImageInfo().getSensorToBufferTransformMatrix();
|
||||
// TODO: Figure out how to get isMirrored from ImageProxy
|
||||
return false;
|
||||
}
|
||||
|
||||
@DoNotStrip
|
||||
@Keep
|
||||
public static String getOrientation(ImageProxy imageProxy) {
|
||||
int rotation = imageProxy.getImageInfo().getRotationDegrees();
|
||||
if (rotation >= 45 && rotation < 135)
|
||||
return "landscapeRight";
|
||||
if (rotation >= 135 && rotation < 225)
|
||||
return "portraitUpsideDown";
|
||||
if (rotation >= 225 && rotation < 315)
|
||||
return "landscapeLeft";
|
||||
return "portrait";
|
||||
}
|
||||
|
||||
@DoNotStrip
|
||||
@Keep
|
||||
public static long getTimestamp(ImageProxy imageProxy) {
|
||||
return imageProxy.getImageInfo().getTimestamp();
|
||||
}
|
||||
|
||||
@DoNotStrip
|
||||
@Keep
|
||||
public static int getPlanesCount(ImageProxy imageProxy) {
|
||||
return imageProxy.getPlanes().length;
|
||||
}
|
||||
|
||||
@DoNotStrip
|
||||
@Keep
|
||||
public static int getBytesPerRow(ImageProxy imageProxy) {
|
||||
return imageProxy.getPlanes()[0].getRowStride();
|
||||
}
|
||||
|
||||
private static byte[] byteArrayCache;
|
||||
|
||||
@DoNotStrip
|
||||
@Keep
|
||||
public static byte[] toByteArray(ImageProxy imageProxy) {
|
||||
switch (imageProxy.getFormat()) {
|
||||
case ImageFormat.YUV_420_888:
|
||||
ByteBuffer yBuffer = imageProxy.getPlanes()[0].getBuffer();
|
||||
ByteBuffer vuBuffer = imageProxy.getPlanes()[2].getBuffer();
|
||||
int ySize = yBuffer.remaining();
|
||||
int vuSize = vuBuffer.remaining();
|
||||
|
||||
if (byteArrayCache == null || byteArrayCache.length != ySize + vuSize) {
|
||||
byteArrayCache = new byte[ySize + vuSize];
|
||||
}
|
||||
|
||||
yBuffer.get(byteArrayCache, 0, ySize);
|
||||
vuBuffer.get(byteArrayCache, ySize, vuSize);
|
||||
|
||||
return byteArrayCache;
|
||||
default:
|
||||
throw new RuntimeException("Cannot convert Frame with Format " + imageProxy.getFormat() + " to byte array!");
|
||||
}
|
||||
}
|
||||
}
|
@ -0,0 +1,6 @@
|
||||
package com.mrousavy.camera.frameprocessor;
|
||||
|
||||
@SuppressWarnings("JavaJniMissingFunction") // we use fbjni
|
||||
public class VisionCameraInstaller {
|
||||
public static native void install(VisionCameraProxy proxy);
|
||||
}
|
@ -5,6 +5,7 @@ import androidx.annotation.Keep
|
||||
import com.facebook.jni.HybridData
|
||||
import com.facebook.proguard.annotations.DoNotStrip
|
||||
import com.facebook.react.bridge.ReactApplicationContext
|
||||
import com.facebook.react.bridge.ReadableNativeMap
|
||||
import com.facebook.react.turbomodule.core.CallInvokerHolderImpl
|
||||
import com.facebook.react.uimanager.UIManagerHelper
|
||||
import com.mrousavy.camera.CameraView
|
||||
@ -12,12 +13,11 @@ import com.mrousavy.camera.ViewNotFoundError
|
||||
import java.lang.ref.WeakReference
|
||||
import java.util.concurrent.ExecutorService
|
||||
|
||||
@Suppress("KotlinJniMissingFunction") // I use fbjni, Android Studio is not smart enough to realize that.
|
||||
class FrameProcessorRuntimeManager(context: ReactApplicationContext, frameProcessorThread: ExecutorService) {
|
||||
companion object {
|
||||
const val TAG = "FrameProcessorRuntime"
|
||||
private val Plugins: ArrayList<FrameProcessorPlugin> = ArrayList()
|
||||
|
||||
@Suppress("KotlinJniMissingFunction") // we use fbjni.
|
||||
class VisionCameraProxy(context: ReactApplicationContext, frameProcessorThread: ExecutorService) {
|
||||
companion object {
|
||||
const val TAG = "VisionCameraProxy"
|
||||
init {
|
||||
try {
|
||||
System.loadLibrary("VisionCamera")
|
||||
@ -26,52 +26,53 @@ class FrameProcessorRuntimeManager(context: ReactApplicationContext, frameProces
|
||||
throw e
|
||||
}
|
||||
}
|
||||
|
||||
fun addPlugin(plugin: FrameProcessorPlugin) {
|
||||
Plugins.add(plugin)
|
||||
}
|
||||
}
|
||||
|
||||
@DoNotStrip
|
||||
private var mHybridData: HybridData? = null
|
||||
private var mContext: WeakReference<ReactApplicationContext>? = null
|
||||
private var mScheduler: VisionCameraScheduler? = null
|
||||
@Keep
|
||||
private var mHybridData: HybridData
|
||||
private var mContext: WeakReference<ReactApplicationContext>
|
||||
private var mScheduler: VisionCameraScheduler
|
||||
|
||||
init {
|
||||
val jsCallInvokerHolder = context.catalystInstance.jsCallInvokerHolder as CallInvokerHolderImpl
|
||||
val jsRuntimeHolder = context.javaScriptContextHolder.get()
|
||||
mScheduler = VisionCameraScheduler(frameProcessorThread)
|
||||
mContext = WeakReference(context)
|
||||
mHybridData = initHybrid(jsRuntimeHolder, jsCallInvokerHolder, mScheduler!!)
|
||||
mHybridData = initHybrid(jsRuntimeHolder, jsCallInvokerHolder, mScheduler)
|
||||
}
|
||||
|
||||
@Suppress("unused")
|
||||
@DoNotStrip
|
||||
@Keep
|
||||
fun findCameraViewById(viewId: Int): CameraView {
|
||||
private fun findCameraViewById(viewId: Int): CameraView {
|
||||
Log.d(TAG, "Finding view $viewId...")
|
||||
val ctx = mContext?.get()
|
||||
val ctx = mContext.get()
|
||||
val view = if (ctx != null) UIManagerHelper.getUIManager(ctx, viewId)?.resolveView(viewId) as CameraView? else null
|
||||
Log.d(TAG, if (view != null) "Found view $viewId!" else "Couldn't find view $viewId!")
|
||||
return view ?: throw ViewNotFoundError(viewId)
|
||||
}
|
||||
|
||||
fun installBindings() {
|
||||
Log.i(TAG, "Installing JSI Bindings on JS Thread...")
|
||||
installJSIBindings()
|
||||
Log.i(TAG, "Installing Frame Processor Plugins...")
|
||||
Plugins.forEach { plugin ->
|
||||
registerPlugin(plugin)
|
||||
}
|
||||
Log.i(TAG, "Successfully installed ${Plugins.count()} Frame Processor Plugins!")
|
||||
@DoNotStrip
|
||||
@Keep
|
||||
fun setFrameProcessor(viewId: Int, frameProcessor: FrameProcessor) {
|
||||
val view = findCameraViewById(viewId)
|
||||
view.frameProcessor = frameProcessor
|
||||
}
|
||||
|
||||
@DoNotStrip
|
||||
@Keep
|
||||
fun removeFrameProcessor(viewId: Int) {
|
||||
val view = findCameraViewById(viewId)
|
||||
view.frameProcessor = null
|
||||
}
|
||||
|
||||
@DoNotStrip
|
||||
@Keep
|
||||
fun getFrameProcessorPlugin(name: String, options: ReadableNativeMap): FrameProcessorPlugin {
|
||||
return FrameProcessorPluginRegistry.getPlugin(name, options)
|
||||
}
|
||||
|
||||
// private C++ funcs
|
||||
private external fun initHybrid(
|
||||
jsContext: Long,
|
||||
jsCallInvokerHolder: CallInvokerHolderImpl,
|
||||
scheduler: VisionCameraScheduler
|
||||
jsContext: Long,
|
||||
jsCallInvokerHolder: CallInvokerHolderImpl,
|
||||
scheduler: VisionCameraScheduler
|
||||
): HybridData
|
||||
private external fun registerPlugin(plugin: FrameProcessorPlugin)
|
||||
private external fun installJSIBindings()
|
||||
}
|
@ -0,0 +1,11 @@
|
||||
package com.mrousavy.camera.parsers
|
||||
|
||||
import android.content.pm.PackageManager
|
||||
|
||||
fun parsePermissionStatus(status: Int): String {
|
||||
return when (status) {
|
||||
PackageManager.PERMISSION_DENIED -> "denied"
|
||||
PackageManager.PERMISSION_GRANTED -> "authorized"
|
||||
else -> "not-determined"
|
||||
}
|
||||
}
|
@ -131,7 +131,7 @@ const frameProcessor = useFrameProcessor((frame) => {
|
||||
|
||||
## What's possible?
|
||||
|
||||
You can run any native code you want in a Frame Processor Plugin. Just like in the native iOS and Android Camera APIs, you will receive a frame (`CMSampleBuffer` on iOS, `ImageProxy` on Android) which you can use however you want. In other words; **everything is possible**.
|
||||
You can run any native code you want in a Frame Processor Plugin. Just like in the native iOS and Android Camera APIs, you will receive a frame ([`CMSampleBuffer`][5] on iOS, [`ImageProxy`][6] on Android) which you can use however you want. In other words; **everything is possible**.
|
||||
|
||||
## Implementations
|
||||
|
||||
@ -194,5 +194,7 @@ Your Frame Processor Plugins have to be fast. Use the FPS Graph (`enableFpsGraph
|
||||
|
||||
[1]: https://github.com/mrousavy/react-native-vision-camera/blob/main/src/Frame.ts
|
||||
[2]: https://github.com/mrousavy/react-native-vision-camera/blob/main/ios/Frame%20Processor/Frame.h
|
||||
[3]: https://developer.android.com/reference/androidx/camera/core/ImageProxy
|
||||
[3]: https://github.com/mrousavy/react-native-vision-camera/blob/main/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java
|
||||
[4]: https://github.com/facebook/react-native/blob/9a43eac7a32a6ba3164a048960101022a92fcd5a/React/Base/RCTBridgeModule.h#L20-L24
|
||||
[5]: https://developer.apple.com/documentation/coremedia/cmsamplebuffer
|
||||
[6]: https://developer.android.com/reference/androidx/camera/core/ImageProxy
|
||||
|
@ -55,21 +55,16 @@ For reference see the [CLI's docs](https://github.com/mateusz1913/vision-camera-
|
||||
3. Add the following code:
|
||||
|
||||
```java {8}
|
||||
import androidx.camera.core.ImageProxy;
|
||||
import com.mrousavy.camera.frameprocessor.Frame;
|
||||
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin;
|
||||
|
||||
public class FaceDetectorFrameProcessorPlugin extends FrameProcessorPlugin {
|
||||
|
||||
@Override
|
||||
public Object callback(ImageProxy image, ReadableNativeMap arguments) {
|
||||
public Object callback(Frame frame, ReadableNativeMap arguments) {
|
||||
// code goes here
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getName() {
|
||||
return "detectFaces";
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@ -86,13 +81,14 @@ import com.facebook.react.bridge.NativeModule;
|
||||
import com.facebook.react.bridge.ReactApplicationContext;
|
||||
import com.facebook.react.uimanager.ViewManager;
|
||||
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin;
|
||||
import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry;
|
||||
import javax.annotation.Nonnull;
|
||||
|
||||
public class FaceDetectorFrameProcessorPluginPackage implements ReactPackage {
|
||||
@NonNull
|
||||
@Override
|
||||
public List<NativeModule> createNativeModules(@NonNull ReactApplicationContext reactContext) {
|
||||
FrameProcessorPlugin.register(new FaceDetectorFrameProcessorPlugin());
|
||||
FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces", options -> new FaceDetectorFrameProcessorPlugin());
|
||||
return Collections.emptyList();
|
||||
}
|
||||
|
||||
@ -125,19 +121,15 @@ public class FaceDetectorFrameProcessorPluginPackage implements ReactPackage {
|
||||
3. Add the following code:
|
||||
|
||||
```kotlin {7}
|
||||
import androidx.camera.core.ImageProxy
|
||||
import com.mrousavy.camera.frameprocessor.Frame
|
||||
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin
|
||||
|
||||
class FaceDetectorFrameProcessorPlugin: FrameProcessorPlugin() {
|
||||
|
||||
override fun callback(image: ImageProxy, arguments: ReadableNativeMap): Any? {
|
||||
override fun callback(frame: Frame, arguments: ReadableNativeMap): Any? {
|
||||
// code goes here
|
||||
return null
|
||||
}
|
||||
|
||||
override fun getName(): String {
|
||||
return "detectFaces"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
@ -157,7 +149,9 @@ import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin
|
||||
|
||||
class FaceDetectorFrameProcessorPluginPackage : ReactPackage {
|
||||
override fun createNativeModules(reactContext: ReactApplicationContext): List<NativeModule> {
|
||||
FrameProcessorPlugin.register(FaceDetectorFrameProcessorPlugin())
|
||||
FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces") { options ->
|
||||
FaceDetectorFrameProcessorPlugin()
|
||||
}
|
||||
return emptyList()
|
||||
}
|
||||
|
||||
|
@ -1,19 +1,30 @@
|
||||
package com.mrousavy.camera.example;
|
||||
|
||||
import android.util.Log;
|
||||
|
||||
import androidx.camera.core.ImageProxy;
|
||||
|
||||
import com.facebook.react.bridge.ReadableNativeMap;
|
||||
import com.facebook.react.bridge.WritableNativeArray;
|
||||
import com.facebook.react.bridge.WritableNativeMap;
|
||||
import com.mrousavy.camera.frameprocessor.Frame;
|
||||
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
|
||||
import java.util.HashMap;
|
||||
|
||||
public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin {
|
||||
@Override
|
||||
public Object callback(@NotNull ImageProxy image, @NotNull Object[] params) {
|
||||
Log.d("ExamplePlugin", image.getWidth() + " x " + image.getHeight() + " Image with format #" + image.getFormat() + ". Logging " + params.length + " parameters:");
|
||||
public Object callback(@NotNull Frame frame, @Nullable ReadableNativeMap params) {
|
||||
HashMap<String, Object> hashMap = params != null ? params.toHashMap() : new HashMap<>();
|
||||
ImageProxy image = frame.getImageProxy();
|
||||
|
||||
for (Object param : params) {
|
||||
Log.d("ExamplePlugin", " -> " + (param == null ? "(null)" : param.toString() + " (" + param.getClass().getName() + ")"));
|
||||
Log.d("ExamplePlugin", image.getWidth() + " x " + image.getHeight() + " Image with format #" + image.getFormat() + ". Logging " + hashMap.size() + " parameters:");
|
||||
|
||||
for (String key : hashMap.keySet()) {
|
||||
Object value = hashMap.get(key);
|
||||
Log.d("ExamplePlugin", " -> " + (value == null ? "(null)" : value.toString() + " (" + value.getClass().getName() + ")"));
|
||||
}
|
||||
|
||||
WritableNativeMap map = new WritableNativeMap();
|
||||
@ -31,6 +42,6 @@ public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin {
|
||||
}
|
||||
|
||||
ExampleFrameProcessorPlugin() {
|
||||
super("example_plugin");
|
||||
|
||||
}
|
||||
}
|
||||
|
@ -1,10 +1,14 @@
|
||||
package com.mrousavy.camera.example;
|
||||
|
||||
import android.app.Application;
|
||||
|
||||
import androidx.annotation.Nullable;
|
||||
|
||||
import com.facebook.react.PackageList;
|
||||
import com.facebook.react.ReactApplication;
|
||||
import com.facebook.react.ReactNativeHost;
|
||||
import com.facebook.react.ReactPackage;
|
||||
import com.facebook.react.bridge.ReadableNativeMap;
|
||||
import com.facebook.soloader.SoLoader;
|
||||
import java.util.List;
|
||||
import com.facebook.react.defaults.DefaultNewArchitectureEntryPoint;
|
||||
@ -12,6 +16,7 @@ import com.facebook.react.defaults.DefaultReactNativeHost;
|
||||
|
||||
import com.mrousavy.camera.CameraPackage;
|
||||
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin;
|
||||
import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry;
|
||||
|
||||
public class MainApplication extends Application implements ReactApplication {
|
||||
|
||||
@ -61,6 +66,6 @@ public class MainApplication extends Application implements ReactApplication {
|
||||
DefaultNewArchitectureEntryPoint.load();
|
||||
}
|
||||
|
||||
FrameProcessorPlugin.register(new ExampleFrameProcessorPlugin());
|
||||
FrameProcessorPluginRegistry.addFrameProcessorPlugin("example_plugin", options -> new ExampleFrameProcessorPlugin());
|
||||
}
|
||||
}
|
||||
|
@ -1,6 +1,6 @@
|
||||
import * as React from 'react';
|
||||
import { useRef, useState, useMemo, useCallback } from 'react';
|
||||
import { Platform, StyleSheet, Text, View } from 'react-native';
|
||||
import { StyleSheet, Text, View } from 'react-native';
|
||||
import { PinchGestureHandler, PinchGestureHandlerGestureEvent, TapGestureHandler } from 'react-native-gesture-handler';
|
||||
import {
|
||||
CameraDeviceFormat,
|
||||
@ -8,7 +8,7 @@ import {
|
||||
PhotoFile,
|
||||
sortFormats,
|
||||
useCameraDevices,
|
||||
useSkiaFrameProcessor,
|
||||
useFrameProcessor,
|
||||
VideoFile,
|
||||
} from 'react-native-vision-camera';
|
||||
import { Camera, frameRateIncluded } from 'react-native-vision-camera';
|
||||
@ -26,6 +26,7 @@ import type { NativeStackScreenProps } from '@react-navigation/native-stack';
|
||||
import { useIsFocused } from '@react-navigation/core';
|
||||
import { Skia } from '@shopify/react-native-skia';
|
||||
import { FACE_SHADER } from './Shaders';
|
||||
import { examplePlugin } from './frame-processors/ExamplePlugin';
|
||||
|
||||
const ReanimatedCamera = Reanimated.createAnimatedComponent(Camera);
|
||||
Reanimated.addWhitelistedNativeProps({
|
||||
@ -217,16 +218,13 @@ export function CameraPage({ navigation }: Props): React.ReactElement {
|
||||
const paint = Skia.Paint();
|
||||
paint.setImageFilter(imageFilter);
|
||||
|
||||
const isIOS = Platform.OS === 'ios';
|
||||
const frameProcessor = useSkiaFrameProcessor(
|
||||
(frame) => {
|
||||
'worklet';
|
||||
console.log(`Width: ${frame.width}`);
|
||||
const frameProcessor = useFrameProcessor((frame) => {
|
||||
'worklet';
|
||||
|
||||
if (frame.isDrawable) frame.render(paint);
|
||||
},
|
||||
[isIOS, paint],
|
||||
);
|
||||
console.log(`Width: ${frame.width}`);
|
||||
const result = examplePlugin(frame);
|
||||
console.log('Example Plugin: ', result);
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<View style={styles.container}>
|
||||
@ -247,12 +245,10 @@ export function CameraPage({ navigation }: Props): React.ReactElement {
|
||||
onError={onError}
|
||||
enableZoomGesture={false}
|
||||
animatedProps={cameraAnimatedProps}
|
||||
photo={true}
|
||||
video={true}
|
||||
audio={hasMicrophonePermission}
|
||||
enableFpsGraph={true}
|
||||
orientation="portrait"
|
||||
frameProcessor={device.supportsParallelVideoProcessing ? frameProcessor : undefined}
|
||||
frameProcessor={frameProcessor}
|
||||
/>
|
||||
</TapGestureHandler>
|
||||
</Reanimated.View>
|
||||
|
@ -12,6 +12,8 @@
|
||||
#import "FrameHostObject.h"
|
||||
#import "JSINSObjectConversion.h"
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
std::vector<jsi::PropNameID> FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime& runtime) {
|
||||
std::vector<jsi::PropNameID> result;
|
||||
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("call")));
|
||||
|
@ -177,7 +177,7 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID&
|
||||
const jsi::Value& thisValue,
|
||||
const jsi::Value* arguments,
|
||||
size_t count) -> jsi::Value {
|
||||
if (count != 1 || !arguments[0].isString()) {
|
||||
if (count < 1 || !arguments[0].isString()) {
|
||||
throw jsi::JSError(runtime, "First argument needs to be a string (pluginName)!");
|
||||
}
|
||||
auto pluginName = arguments[0].asString(runtime).utf8(runtime);
|
||||
|
Loading…
Reference in New Issue
Block a user