diff --git a/android/CMakeLists.txt b/android/CMakeLists.txt index 088f664..7b19de1 100644 --- a/android/CMakeLists.txt +++ b/android/CMakeLists.txt @@ -32,15 +32,17 @@ add_library( ${PACKAGE_NAME} SHARED ../cpp/JSITypedArray.cpp - src/main/cpp/VisionCamera.cpp - src/main/cpp/JSIJNIConversion.cpp src/main/cpp/FrameHostObject.cpp - src/main/cpp/FrameProcessorRuntimeManager.cpp - src/main/cpp/CameraView.cpp - src/main/cpp/VisionCameraScheduler.cpp + src/main/cpp/FrameProcessorPluginHostObject.cpp + src/main/cpp/JSIJNIConversion.cpp + src/main/cpp/VisionCamera.cpp + src/main/cpp/VisionCameraProxy.cpp + src/main/cpp/java-bindings/JFrame.cpp + src/main/cpp/java-bindings/JFrameProcessor.cpp src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp - src/main/cpp/java-bindings/JImageProxy.cpp src/main/cpp/java-bindings/JHashMap.cpp + src/main/cpp/java-bindings/JVisionCameraProxy.cpp + src/main/cpp/java-bindings/JVisionCameraScheduler.cpp ) # Header Search Paths (includes) diff --git a/android/src/main/cpp/CameraView.cpp b/android/src/main/cpp/CameraView.cpp deleted file mode 100644 index 665bd31..0000000 --- a/android/src/main/cpp/CameraView.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// -// Created by Marc Rousavy on 14.06.21. -// - -#include "CameraView.h" - -#include -#include -#include - -#include -#include -#include - -namespace vision { - -using namespace facebook; -using namespace jni; - -using TSelf = local_ref; - -TSelf CameraView::initHybrid(alias_ref jThis) { - return makeCxxInstance(jThis); -} - -void CameraView::registerNatives() { - registerHybrid({ - makeNativeMethod("initHybrid", CameraView::initHybrid), - makeNativeMethod("frameProcessorCallback", CameraView::frameProcessorCallback), - }); -} - -void CameraView::frameProcessorCallback(const alias_ref& frame) { - if (frameProcessor_ == nullptr) { - __android_log_write(ANDROID_LOG_WARN, TAG, "Called Frame Processor callback, but `frameProcessor` is null!"); - return; - } - - try { - frameProcessor_(frame); - } catch (const jsi::JSError& error) { - // TODO: jsi::JSErrors cannot be caught on Hermes. They crash the entire app. - auto stack = std::regex_replace(error.getStack(), std::regex("\n"), "\n "); - __android_log_print(ANDROID_LOG_ERROR, TAG, "Frame Processor threw an error! %s\nIn: %s", error.getMessage().c_str(), stack.c_str()); - } catch (const std::exception& exception) { - __android_log_print(ANDROID_LOG_ERROR, TAG, "Frame Processor threw a C++ error! %s", exception.what()); - } -} - -void CameraView::setFrameProcessor(const TFrameProcessor&& frameProcessor) { - frameProcessor_ = frameProcessor; -} - -void vision::CameraView::unsetFrameProcessor() { - frameProcessor_ = nullptr; -} - -} // namespace vision diff --git a/android/src/main/cpp/CameraView.h b/android/src/main/cpp/CameraView.h deleted file mode 100644 index a1ca070..0000000 --- a/android/src/main/cpp/CameraView.h +++ /dev/null @@ -1,43 +0,0 @@ -// -// Created by Marc Rousavy on 14.06.21. -// - -#pragma once - -#include -#include - -#include - -#include "java-bindings/JImageProxy.h" - -namespace vision { - -using namespace facebook; -using TFrameProcessor = std::function)>; - -class CameraView : public jni::HybridClass { - public: - static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/CameraView;"; - static auto constexpr TAG = "VisionCamera"; - static jni::local_ref initHybrid(jni::alias_ref jThis); - static void registerNatives(); - - // TODO: Use template<> to avoid heap allocation for std::function<> - void setFrameProcessor(const TFrameProcessor&& frameProcessor); - void unsetFrameProcessor(); - - private: - friend HybridBase; - jni::global_ref javaPart_; - TFrameProcessor frameProcessor_; - - void frameProcessorCallback(const jni::alias_ref& frame); - - explicit CameraView(jni::alias_ref jThis) : - javaPart_(jni::make_global(jThis)), - frameProcessor_(nullptr) - {} -}; - -} // namespace vision diff --git a/android/src/main/cpp/FrameHostObject.cpp b/android/src/main/cpp/FrameHostObject.cpp index 60b367a..b20ccac 100644 --- a/android/src/main/cpp/FrameHostObject.cpp +++ b/android/src/main/cpp/FrameHostObject.cpp @@ -18,7 +18,7 @@ namespace vision { using namespace facebook; -FrameHostObject::FrameHostObject(jni::alias_ref image): frame(make_global(image)), _refCount(0) { } +FrameHostObject::FrameHostObject(const jni::alias_ref& frame): frame(make_global(frame)), _refCount(0) { } FrameHostObject::~FrameHostObject() { // Hermes' Garbage Collector (Hades GC) calls destructors on a separate Thread diff --git a/android/src/main/cpp/FrameHostObject.h b/android/src/main/cpp/FrameHostObject.h index eb22db5..92e5ff3 100644 --- a/android/src/main/cpp/FrameHostObject.h +++ b/android/src/main/cpp/FrameHostObject.h @@ -11,15 +11,15 @@ #include #include -#include "java-bindings/JImageProxy.h" +#include "java-bindings/JFrame.h" namespace vision { using namespace facebook; class JSI_EXPORT FrameHostObject : public jsi::HostObject { + explicit FrameHostObject(const jni::alias_ref& frame); public: - explicit FrameHostObject(jni::alias_ref image); ~FrameHostObject(); public: @@ -27,7 +27,7 @@ class JSI_EXPORT FrameHostObject : public jsi::HostObject { std::vector getPropertyNames(jsi::Runtime &rt) override; public: - jni::global_ref frame; + jni::global_ref frame; private: static auto constexpr TAG = "VisionCamera"; diff --git a/android/src/main/cpp/FrameProcessorPluginHostObject.cpp b/android/src/main/cpp/FrameProcessorPluginHostObject.cpp new file mode 100644 index 0000000..43312d3 --- /dev/null +++ b/android/src/main/cpp/FrameProcessorPluginHostObject.cpp @@ -0,0 +1,53 @@ +// +// Created by Marc Rousavy on 21.07.23. +// + +#include "FrameProcessorPluginHostObject.h" +#include +#include "FrameHostObject.h" +#include "JSIJNIConversion.h" +#include + +namespace vision { + +using namespace facebook; + +std::vector FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime& runtime) { + std::vector result; + result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("call"))); + return result; +} + +jsi::Value FrameProcessorPluginHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) { + auto name = propName.utf8(runtime); + + if (name == "call") { + return jsi::Function::createFromHostFunction(runtime, + jsi::PropNameID::forUtf8(runtime, "call"), + 2, + [=](jsi::Runtime &runtime, + const jsi::Value &thisValue, + const jsi::Value *arguments, + size_t count) -> jsi::Value { + // Frame is first argument + auto frameHostObject = arguments[0].asObject(runtime).asHostObject(runtime); + auto frame = frameHostObject->frame; + + // Options are second argument (possibly undefined) + local_ref options = nullptr; + if (count > 1) { + options = JSIJNIConversion::convertJSIObjectToJNIMap(runtime, arguments[1].asObject(runtime)); + } + + // Call actual plugin + auto result = _plugin->callback(frame, options); + + // Convert result value to jsi::Value (possibly undefined) + return JSIJNIConversion::convertJNIObjectToJSIValue(runtime, result); + }); + } + + return jsi::Value::undefined(); +} + +} // namespace vision diff --git a/android/src/main/cpp/FrameProcessorPluginHostObject.h b/android/src/main/cpp/FrameProcessorPluginHostObject.h new file mode 100644 index 0000000..20a516e --- /dev/null +++ b/android/src/main/cpp/FrameProcessorPluginHostObject.h @@ -0,0 +1,32 @@ +// +// Created by Marc Rousavy on 21.07.23. +// + +#pragma once + +#include +#include "java-bindings/JFrameProcessorPlugin.h" +#include +#include +#include +#include + +namespace vision { + +using namespace facebook; + +class FrameProcessorPluginHostObject: public jsi::HostObject { + public: + explicit FrameProcessorPluginHostObject(jni::alias_ref plugin): + _plugin(make_global(plugin)) { } + ~FrameProcessorPluginHostObject() { } + + public: + std::vector getPropertyNames(jsi::Runtime& runtime) override; + jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& name) override; + + private: + jni::global_ref _plugin; +}; + +} // namespace vision diff --git a/android/src/main/cpp/FrameProcessorRuntimeManager.cpp b/android/src/main/cpp/FrameProcessorRuntimeManager.cpp deleted file mode 100644 index 412006e..0000000 --- a/android/src/main/cpp/FrameProcessorRuntimeManager.cpp +++ /dev/null @@ -1,245 +0,0 @@ -// -// Created by Marc Rousavy on 11.06.21. -// - -#include "FrameProcessorRuntimeManager.h" -#include -#include -#include -#include -#include -#include - -#include "CameraView.h" -#include "FrameHostObject.h" -#include "JSIJNIConversion.h" -#include "java-bindings/JImageProxy.h" -#include "java-bindings/JFrameProcessorPlugin.h" -#include "JSITypedArray.h" - -namespace vision { - -// type aliases -using TSelf = local_ref::jhybriddata>; -using TJSCallInvokerHolder = jni::alias_ref; -using TAndroidScheduler = jni::alias_ref; - -FrameProcessorRuntimeManager::FrameProcessorRuntimeManager(jni::alias_ref jThis, - jsi::Runtime* jsRuntime, - std::shared_ptr jsCallInvoker, - std::shared_ptr scheduler) : - javaPart_(jni::make_global(jThis)), - _jsRuntime(jsRuntime) { - auto runOnJS = [jsCallInvoker](std::function&& f) { - // Run on React JS Runtime - jsCallInvoker->invokeAsync(std::move(f)); - }; - auto runOnWorklet = [scheduler](std::function&& f) { - // Run on Frame Processor Worklet Runtime - scheduler->dispatchAsync(std::move(f)); - }; - _workletContext = std::make_shared("VisionCamera", - jsRuntime, - runOnJS, - runOnWorklet); -} - -// JNI binding -void vision::FrameProcessorRuntimeManager::registerNatives() { - registerHybrid({ - makeNativeMethod("initHybrid", - FrameProcessorRuntimeManager::initHybrid), - makeNativeMethod("installJSIBindings", - FrameProcessorRuntimeManager::installJSIBindings), - makeNativeMethod("registerPlugin", - FrameProcessorRuntimeManager::registerPlugin), - }); -} - -// JNI init -TSelf vision::FrameProcessorRuntimeManager::initHybrid( - alias_ref jThis, - jlong jsRuntimePointer, - TJSCallInvokerHolder jsCallInvokerHolder, - TAndroidScheduler androidScheduler) { - __android_log_write(ANDROID_LOG_INFO, TAG, - "Initializing FrameProcessorRuntimeManager..."); - - // cast from JNI hybrid objects to C++ instances - auto jsRuntime = reinterpret_cast(jsRuntimePointer); - auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker(); - auto scheduler = std::shared_ptr(androidScheduler->cthis()); - - return makeCxxInstance(jThis, jsRuntime, jsCallInvoker, scheduler); -} - -global_ref FrameProcessorRuntimeManager::findCameraViewById(int viewId) { - static const auto findCameraViewByIdMethod = javaPart_->getClass()->getMethod("findCameraViewById"); - auto weakCameraView = findCameraViewByIdMethod(javaPart_.get(), viewId); - return make_global(weakCameraView); -} - -void FrameProcessorRuntimeManager::logErrorToJS(const std::string& message) { - if (!_workletContext) { - return; - } - // Call console.error() on JS Thread - _workletContext->invokeOnJsThread([message](jsi::Runtime& runtime) { - auto consoleError = runtime - .global() - .getPropertyAsObject(runtime, "console") - .getPropertyAsFunction(runtime, "error"); - consoleError.call(runtime, jsi::String::createFromUtf8(runtime, message)); - }); -} - -void FrameProcessorRuntimeManager::setFrameProcessor(jsi::Runtime& runtime, - int viewTag, - const jsi::Value& frameProcessor) { - __android_log_write(ANDROID_LOG_INFO, TAG, "Setting new Frame Processor..."); - - if (!_workletContext) { - throw jsi::JSError(runtime, - "setFrameProcessor(..): VisionCamera's Worklet Context is not yet initialized!"); - } - - // find camera view - auto cameraView = findCameraViewById(viewTag); - - // convert jsi::Function to a Worklet (can be shared across runtimes) - auto worklet = std::make_shared(runtime, frameProcessor); - auto workletInvoker = std::make_shared(worklet); - - _workletContext->invokeOnWorkletThread([=](RNWorklet::JsiWorkletContext*, jsi::Runtime& rt) { - // Set Frame Processor as callable C++ lambda - this will then call the Worklet - cameraView->cthis()->setFrameProcessor([this, workletInvoker, &rt](jni::alias_ref frame) { - try { - // create HostObject which holds the Frame (JImageProxy) - auto frameHostObject = std::make_shared(frame); - auto argument = jsi::Object::createFromHostObject(rt, frameHostObject); - jsi::Value jsValue(std::move(argument)); - // Call the Worklet on the Worklet Runtime - workletInvoker->call(rt, jsi::Value::undefined(), &jsValue, 1); - } catch (jsi::JSError& jsError) { - // Worklet threw a JS Error, catch it and log it to JS. - auto message = "Frame Processor threw an error: " + jsError.getMessage(); - __android_log_write(ANDROID_LOG_ERROR, TAG, message.c_str()); - this->logErrorToJS(message); - } - }); - }); -} - -void FrameProcessorRuntimeManager::unsetFrameProcessor(int viewTag) { - __android_log_write(ANDROID_LOG_INFO, TAG, "Removing Frame Processor..."); - - // find camera view - auto cameraView = findCameraViewById(viewTag); - - // call Java method to unset frame processor - cameraView->cthis()->unsetFrameProcessor(); -} - -// actual JSI installer -void FrameProcessorRuntimeManager::installJSIBindings() { - __android_log_write(ANDROID_LOG_INFO, TAG, "Installing JSI bindings..."); - - if (_jsRuntime == nullptr) { - __android_log_write(ANDROID_LOG_ERROR, TAG, - "JS-Runtime was null, Frame Processor JSI bindings could not be installed!"); - return; - } - - auto& jsiRuntime = *_jsRuntime; - - // HostObject that attaches the cache to the lifecycle of the Runtime. On Runtime destroy, we destroy the cache. - auto propNameCacheObject = std::make_shared(jsiRuntime); - jsiRuntime.global().setProperty(jsiRuntime, - "__visionCameraPropNameCache", - jsi::Object::createFromHostObject(jsiRuntime, propNameCacheObject)); - - auto setFrameProcessor = JSI_HOST_FUNCTION_LAMBDA { - __android_log_write(ANDROID_LOG_INFO, TAG, "Setting new Frame Processor..."); - - double viewTag = arguments[0].asNumber(); - const jsi::Value& frameProcessor = arguments[1]; - this->setFrameProcessor(runtime, static_cast(viewTag), frameProcessor); - - return jsi::Value::undefined(); - }; - jsiRuntime.global().setProperty(jsiRuntime, - "setFrameProcessor", - jsi::Function::createFromHostFunction( - jsiRuntime, - jsi::PropNameID::forAscii(jsiRuntime, - "setFrameProcessor"), - 2, // viewTag, frameProcessor - setFrameProcessor)); - - - auto unsetFrameProcessor = JSI_HOST_FUNCTION_LAMBDA { - __android_log_write(ANDROID_LOG_INFO, TAG, "Removing Frame Processor..."); - - auto viewTag = arguments[0].asNumber(); - this->unsetFrameProcessor(static_cast(viewTag)); - - return jsi::Value::undefined(); - }; - jsiRuntime.global().setProperty(jsiRuntime, - "unsetFrameProcessor", - jsi::Function::createFromHostFunction( - jsiRuntime, - jsi::PropNameID::forAscii(jsiRuntime, - "unsetFrameProcessor"), - 1, // viewTag - unsetFrameProcessor)); - - __android_log_write(ANDROID_LOG_INFO, TAG, "Finished installing JSI bindings!"); -} - -void FrameProcessorRuntimeManager::registerPlugin(alias_ref plugin) { - auto& runtime = *_jsRuntime; - - // we need a strong reference on the plugin, make_global does that. - auto pluginGlobal = make_global(plugin); - auto pluginName = pluginGlobal->getName(); - - __android_log_print(ANDROID_LOG_INFO, TAG, "Installing Frame Processor Plugin \"%s\"...", pluginName.c_str()); - - if (!runtime.global().hasProperty(runtime, "FrameProcessorPlugins")) { - runtime.global().setProperty(runtime, "FrameProcessorPlugins", jsi::Object(runtime)); - } - jsi::Object frameProcessorPlugins = runtime.global().getPropertyAsObject(runtime, "FrameProcessorPlugins"); - - auto function = [pluginGlobal](jsi::Runtime& runtime, - const jsi::Value& thisValue, - const jsi::Value* arguments, - size_t count) -> jsi::Value { - // Unbox object and get typed HostObject - auto boxedHostObject = arguments[0].asObject(runtime).asHostObject(runtime); - auto frameHostObject = dynamic_cast(boxedHostObject.get()); - - // parse params - we are offset by `1` because the frame is the first parameter. - auto params = JArrayClass::newArray(count - 1); - for (size_t i = 1; i < count; i++) { - params->setElement(i - 1, JSIJNIConversion::convertJSIValueToJNIObject(runtime, arguments[i])); - } - - // call implemented virtual method - auto result = pluginGlobal->callback(frameHostObject->frame, params); - - // convert result from JNI to JSI value - return JSIJNIConversion::convertJNIObjectToJSIValue(runtime, result); - }; - - // Assign it to the Proxy. - // A FP Plugin called "example_plugin" can be now called from JS using "FrameProcessorPlugins.example_plugin(frame)" - frameProcessorPlugins.setProperty(runtime, - pluginName.c_str(), - jsi::Function::createFromHostFunction(runtime, - jsi::PropNameID::forAscii(runtime, pluginName), - 1, // frame - function)); -} - -} // namespace vision diff --git a/android/src/main/cpp/FrameProcessorRuntimeManager.h b/android/src/main/cpp/FrameProcessorRuntimeManager.h deleted file mode 100644 index 450670e..0000000 --- a/android/src/main/cpp/FrameProcessorRuntimeManager.h +++ /dev/null @@ -1,54 +0,0 @@ -// -// Created by Marc Rousavy on 11.06.21. -// - -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "CameraView.h" -#include "VisionCameraScheduler.h" -#include "java-bindings/JFrameProcessorPlugin.h" - -namespace vision { - -using namespace facebook; - -class FrameProcessorRuntimeManager : public jni::HybridClass { - public: - static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager;"; - static auto constexpr TAG = "VisionCamera"; - static jni::local_ref initHybrid(jni::alias_ref jThis, - jlong jsContext, - jni::alias_ref jsCallInvokerHolder, - jni::alias_ref androidScheduler); - static void registerNatives(); - - explicit FrameProcessorRuntimeManager(jni::alias_ref jThis, - jsi::Runtime* jsRuntime, - std::shared_ptr jsCallInvoker, - std::shared_ptr scheduler); - - private: - friend HybridBase; - jni::global_ref javaPart_; - jsi::Runtime* _jsRuntime; - std::shared_ptr _workletContext; - - jni::global_ref findCameraViewById(int viewId); - void installJSIBindings(); - void registerPlugin(alias_ref plugin); - void logErrorToJS(const std::string& message); - - void setFrameProcessor(jsi::Runtime& runtime, - int viewTag, - const jsi::Value& frameProcessor); - void unsetFrameProcessor(int viewTag); -}; - -} // namespace vision diff --git a/android/src/main/cpp/JSIJNIConversion.cpp b/android/src/main/cpp/JSIJNIConversion.cpp index d34308f..5655a1b 100644 --- a/android/src/main/cpp/JSIJNIConversion.cpp +++ b/android/src/main/cpp/JSIJNIConversion.cpp @@ -21,7 +21,7 @@ #include #include "FrameHostObject.h" -#include "java-bindings/JImageProxy.h" +#include "java-bindings/JFrame.h" #include "java-bindings/JArrayList.h" #include "java-bindings/JHashMap.h" @@ -29,75 +29,9 @@ namespace vision { using namespace facebook; -jobject JSIJNIConversion::convertJSIValueToJNIObject(jsi::Runtime &runtime, const jsi::Value &value) { - if (value.isBool()) { - // jsi::Bool - - auto boolean = jni::JBoolean::valueOf(value.getBool()); - return boolean.release(); - - } else if (value.isNumber()) { - // jsi::Number - - auto number = jni::JDouble::valueOf(value.getNumber()); - return number.release(); - - } else if (value.isNull() || value.isUndefined()) { - // jsi::undefined - - return nullptr; - - } else if (value.isString()) { - // jsi::String - - auto string = jni::make_jstring(value.getString(runtime).utf8(runtime)); - return string.release(); - - } else if (value.isObject()) { - // jsi::Object - - auto object = value.asObject(runtime); - - if (object.isArray(runtime)) { - // jsi::Array - - auto dynamic = jsi::dynamicFromValue(runtime, value); - auto nativeArray = react::ReadableNativeArray::newObjectCxxArgs(std::move(dynamic)); - return nativeArray.release(); - - } else if (object.isHostObject(runtime)) { - // jsi::HostObject - - auto boxedHostObject = object.getHostObject(runtime); - auto hostObject = dynamic_cast(boxedHostObject.get()); - if (hostObject != nullptr) { - // return jni local_ref to the JImageProxy - return hostObject->frame.get(); - } else { - // it's different kind of HostObject. We don't support it. - throw std::runtime_error("Received an unknown HostObject! Cannot convert to a JNI value."); - } - - } else if (object.isFunction(runtime)) { - // jsi::Function - - // TODO: Convert Function to Callback - throw std::runtime_error("Cannot convert a JS Function to a JNI value (yet)!"); - - } else { - // jsi::Object - - auto dynamic = jsi::dynamicFromValue(runtime, value); - auto map = react::ReadableNativeMap::createWithContents(std::move(dynamic)); - return map.release(); - } - } else { - // unknown jsi type! - - auto stringRepresentation = value.toString(runtime).utf8(runtime); - auto message = "Received unknown JSI value! (" + stringRepresentation + ") Cannot convert to a JNI value."; - throw std::runtime_error(message); - } +jni::local_ref JSIJNIConversion::convertJSIObjectToJNIMap(jsi::Runtime& runtime, const jsi::Object& object) { + auto dynamic = jsi::dynamicFromValue(runtime, jsi::Value(runtime, object)); + return react::ReadableNativeMap::createWithContents(std::move(dynamic)); } jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, const jni::local_ref& object) { @@ -178,10 +112,9 @@ jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, c auto hashMap = toHashMapFunc(object.get()); return convertJNIObjectToJSIValue(runtime, hashMap); - } else if (object->isInstanceOf(JImageProxy::javaClassStatic())) { - // ImageProxy - - auto frame = static_ref_cast(object); + } else if (object->isInstanceOf(JFrame::javaClassStatic())) { + // Frame + auto frame = static_ref_cast(object); // box into HostObject auto hostObject = std::make_shared(frame); diff --git a/android/src/main/cpp/JSIJNIConversion.h b/android/src/main/cpp/JSIJNIConversion.h index 5f031ec..3e55848 100644 --- a/android/src/main/cpp/JSIJNIConversion.h +++ b/android/src/main/cpp/JSIJNIConversion.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace vision { @@ -14,7 +15,7 @@ namespace JSIJNIConversion { using namespace facebook; -jobject convertJSIValueToJNIObject(jsi::Runtime& runtime, const jsi::Value& value); +jni::local_ref convertJSIObjectToJNIMap(jsi::Runtime& runtime, const jsi::Object& object); jsi::Value convertJNIObjectToJSIValue(jsi::Runtime& runtime, const jni::local_ref& object); diff --git a/android/src/main/cpp/VisionCamera.cpp b/android/src/main/cpp/VisionCamera.cpp index 6b91b8e..1da9d50 100644 --- a/android/src/main/cpp/VisionCamera.cpp +++ b/android/src/main/cpp/VisionCamera.cpp @@ -1,13 +1,15 @@ #include #include -#include "FrameProcessorRuntimeManager.h" -#include "CameraView.h" -#include "VisionCameraScheduler.h" +#include "java-bindings/JVisionCameraScheduler.h" +#include "java-bindings/JFrameProcessor.h" +#include "java-bindings/JVisionCameraProxy.h" +#include "VisionCameraProxy.h" JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) { return facebook::jni::initialize(vm, [] { - vision::FrameProcessorRuntimeManager::registerNatives(); - vision::CameraView::registerNatives(); - vision::VisionCameraScheduler::registerNatives(); + vision::VisionCameraInstaller::registerNatives(); + vision::JFrameProcessor::registerNatives(); + vision::JVisionCameraProxy::registerNatives(); + vision::JVisionCameraScheduler::registerNatives(); }); } diff --git a/android/src/main/cpp/VisionCameraProxy.cpp b/android/src/main/cpp/VisionCameraProxy.cpp new file mode 100644 index 0000000..ba302cd --- /dev/null +++ b/android/src/main/cpp/VisionCameraProxy.cpp @@ -0,0 +1,152 @@ +// +// Created by Marc Rousavy on 21.07.23. +// + +#include "VisionCameraProxy.h" +#include + +#include "java-bindings/JFrameProcessor.h" +#include "java-bindings/JFrameProcessorPlugin.h" +#include "JSIJNIConversion.h" + +#include +#include + +#include "JSITypedArray.h" +#include "FrameProcessorPluginHostObject.h" + +#include +#include +#include + +namespace vision { + +using namespace facebook; + +VisionCameraProxy::VisionCameraProxy(const jni::alias_ref& javaProxy) { + _javaProxy = make_global(javaProxy); +} + +VisionCameraProxy::~VisionCameraProxy() { + __android_log_write(ANDROID_LOG_INFO, TAG, "Destroying Context..."); + // Destroy ArrayBuffer cache for both the JS and the Worklet Runtime. + auto workletContext = _javaProxy->cthis()->getWorkletContext(); + invalidateArrayBufferCache(*workletContext->getJsRuntime()); + invalidateArrayBufferCache(workletContext->getWorkletRuntime()); +} + +std::vector VisionCameraProxy::getPropertyNames(jsi::Runtime& runtime) { + std::vector result; + result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("setFrameProcessor"))); + result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("removeFrameProcessor"))); + result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("getFrameProcessorPlugin"))); + result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("isSkiaEnabled"))); + return result; +} + +void VisionCameraProxy::setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& object) { + auto frameProcessorType = object.getProperty(runtime, "type").asString(runtime).utf8(runtime); + auto worklet = std::make_shared(runtime, object.getProperty(runtime, "frameProcessor")); + auto workletContext = _javaProxy->cthis()->getWorkletContext(); + + jni::local_ref frameProcessor; + if (frameProcessorType == "frame-processor") { + frameProcessor = JFrameProcessor::create(worklet, workletContext); + } else if (frameProcessorType == "skia-frame-processor") { +#if VISION_CAMERA_ENABLE_SKIA + throw std::runtime_error("system/skia-unavailable: Skia is not yet implemented on Android!"); +#else + throw std::runtime_error("system/skia-unavailable: Skia is not installed!"); +#endif + } else { + throw std::runtime_error("Unknown FrameProcessor.type passed! Received: " + frameProcessorType); + } + + _javaProxy->cthis()->setFrameProcessor(viewTag, make_global(frameProcessor)); +} + +void VisionCameraProxy::removeFrameProcessor(int viewTag) { + _javaProxy->cthis()->removeFrameProcessor(viewTag); +} + +jsi::Value VisionCameraProxy::getFrameProcessorPlugin(jsi::Runtime& runtime, + const std::string& name, + const jsi::Object& jsOptions) { + auto options = JSIJNIConversion::convertJSIObjectToJNIMap(runtime, jsOptions); + + auto plugin = _javaProxy->cthis()->getFrameProcessorPlugin(name, options); + + auto pluginHostObject = std::make_shared(plugin); + return jsi::Object::createFromHostObject(runtime, pluginHostObject); +} + +jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) { + auto name = propName.utf8(runtime); + + if (name == "isSkiaEnabled") { +#ifdef VISION_CAMERA_ENABLE_SKIA + return jsi::Value(true); +#else + return jsi::Value(false); +#endif + } + if (name == "setFrameProcessor") { + return jsi::Function::createFromHostFunction(runtime, + jsi::PropNameID::forUtf8(runtime, "setFrameProcessor"), + 1, + [this](jsi::Runtime& runtime, + const jsi::Value& thisValue, + const jsi::Value* arguments, + size_t count) -> jsi::Value { + auto viewTag = arguments[0].asNumber(); + auto object = arguments[1].asObject(runtime); + this->setFrameProcessor(static_cast(viewTag), runtime, object); + return jsi::Value::undefined(); + }); + } + if (name == "removeFrameProcessor") { + return jsi::Function::createFromHostFunction(runtime, + jsi::PropNameID::forUtf8(runtime, "removeFrameProcessor"), + 1, + [this](jsi::Runtime& runtime, + const jsi::Value& thisValue, + const jsi::Value* arguments, + size_t count) -> jsi::Value { + auto viewTag = arguments[0].asNumber(); + this->removeFrameProcessor(static_cast(viewTag)); + return jsi::Value::undefined(); + }); + } + if (name == "getFrameProcessorPlugin") { + return jsi::Function::createFromHostFunction(runtime, + jsi::PropNameID::forUtf8(runtime, "getFrameProcessorPlugin"), + 1, + [this](jsi::Runtime& runtime, + const jsi::Value& thisValue, + const jsi::Value* arguments, + size_t count) -> jsi::Value { + if (count < 1 || !arguments[0].isString()) { + throw jsi::JSError(runtime, "First argument needs to be a string (pluginName)!"); + } + auto pluginName = arguments[0].asString(runtime).utf8(runtime); + auto options = count > 1 ? arguments[1].asObject(runtime) : jsi::Object(runtime); + + return this->getFrameProcessorPlugin(runtime, pluginName, options); + }); + } + + return jsi::Value::undefined(); +} + + +void VisionCameraInstaller::install(jni::alias_ref, + jni::alias_ref proxy) { + // global.VisionCameraProxy + auto visionCameraProxy = std::make_shared(proxy); + jsi::Runtime& runtime = *proxy->cthis()->getWorkletContext()->getJsRuntime(); + runtime.global().setProperty(runtime, + "VisionCameraProxy", + jsi::Object::createFromHostObject(runtime, visionCameraProxy)); +} + +} // namespace vision diff --git a/android/src/main/cpp/VisionCameraProxy.h b/android/src/main/cpp/VisionCameraProxy.h new file mode 100644 index 0000000..65830b1 --- /dev/null +++ b/android/src/main/cpp/VisionCameraProxy.h @@ -0,0 +1,51 @@ +// +// Created by Marc Rousavy on 21.07.23. +// + +#pragma once + +#include + +#include "java-bindings/JVisionCameraScheduler.h" +#include "java-bindings/JVisionCameraProxy.h" + +#include +#include + +namespace vision { + +using namespace facebook; + +class VisionCameraProxy: public jsi::HostObject { + public: + explicit VisionCameraProxy(const jni::alias_ref& javaProxy); + ~VisionCameraProxy(); + + public: + std::vector getPropertyNames(jsi::Runtime& runtime) override; + jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& name) override; + + private: + void setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& frameProcessor); + void removeFrameProcessor(int viewTag); + jsi::Value getFrameProcessorPlugin(jsi::Runtime& runtime, const std::string& name, const jsi::Object& options); + + private: + jni::global_ref _javaProxy; + static constexpr const char* TAG = "VisionCameraProxy"; +}; + + +class VisionCameraInstaller: public jni::JavaClass { + public: + static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraInstaller;"; + static void registerNatives() { + javaClassStatic()->registerNatives({ + makeNativeMethod("install", VisionCameraInstaller::install) + }); + } + static void install(jni::alias_ref clazz, + jni::alias_ref proxy); +}; + +} // namespace vision diff --git a/android/src/main/cpp/java-bindings/JFrame.cpp b/android/src/main/cpp/java-bindings/JFrame.cpp new file mode 100644 index 0000000..d79658d --- /dev/null +++ b/android/src/main/cpp/java-bindings/JFrame.cpp @@ -0,0 +1,65 @@ +// +// Created by Marc on 21.07.2023. +// + +#include "JFrame.h" + +#include +#include + +namespace vision { + +using namespace facebook; +using namespace jni; + +int JFrame::getWidth() const { + static const auto getWidthMethod = getClass()->getMethod("getWidth"); + return getWidthMethod(self()); +} + +int JFrame::getHeight() const { + static const auto getWidthMethod = getClass()->getMethod("getHeight"); + return getWidthMethod(self()); +} + +bool JFrame::getIsValid() const { + static const auto getIsValidMethod = getClass()->getMethod("getIsValid"); + return getIsValidMethod(self()); +} + +bool JFrame::getIsMirrored() const { + static const auto getIsMirroredMethod = getClass()->getMethod("getIsMirrored"); + return getIsMirroredMethod(self()); +} + +jlong JFrame::getTimestamp() const { + static const auto getTimestampMethod = getClass()->getMethod("getTimestamp"); + return getTimestampMethod(self()); +} + +local_ref JFrame::getOrientation() const { + static const auto getOrientationMethod = getClass()->getMethod("getOrientation"); + return getOrientationMethod(self()); +} + +int JFrame::getPlanesCount() const { + static const auto getPlanesCountMethod = getClass()->getMethod("getPlanesCount"); + return getPlanesCountMethod(self()); +} + +int JFrame::getBytesPerRow() const { + static const auto getBytesPerRowMethod = getClass()->getMethod("getBytesPerRow"); + return getBytesPerRowMethod(self()); +} + +local_ref JFrame::toByteArray() const { + static const auto toByteArrayMethod = getClass()->getMethod("toByteArray"); + return toByteArrayMethod(self()); +} + +void JFrame::close() { + static const auto closeMethod = getClass()->getMethod("close"); + closeMethod(self()); +} + +} // namespace vision diff --git a/android/src/main/cpp/java-bindings/JImageProxy.h b/android/src/main/cpp/java-bindings/JFrame.h similarity index 73% rename from android/src/main/cpp/java-bindings/JImageProxy.h rename to android/src/main/cpp/java-bindings/JFrame.h index 81cb5f2..8d9949b 100644 --- a/android/src/main/cpp/java-bindings/JImageProxy.h +++ b/android/src/main/cpp/java-bindings/JFrame.h @@ -1,5 +1,5 @@ // -// Created by Marc on 19/06/2021. +// Created by Marc on 21.07.2023. // #pragma once @@ -12,8 +12,8 @@ namespace vision { using namespace facebook; using namespace jni; -struct JImageProxy : public JavaClass { - static constexpr auto kJavaDescriptor = "Landroidx/camera/core/ImageProxy;"; +struct JFrame : public JavaClass { + static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/Frame;"; public: int getWidth() const; diff --git a/android/src/main/cpp/java-bindings/JFrameProcessor.cpp b/android/src/main/cpp/java-bindings/JFrameProcessor.cpp new file mode 100644 index 0000000..9c6aa2c --- /dev/null +++ b/android/src/main/cpp/java-bindings/JFrameProcessor.cpp @@ -0,0 +1,65 @@ +// +// Created by Marc Rousavy on 29.09.21. +// + +#include "JFrameProcessor.h" + +#include +#include + +#include +#include "JFrame.h" + +namespace vision { + +using namespace facebook; +using namespace jni; + +void JFrameProcessor::registerNatives() { + registerHybrid({ + makeNativeMethod("call", JFrameProcessor::call) + }); +} + +using TSelf = jni::local_ref; + +JFrameProcessor::JFrameProcessor(std::shared_ptr worklet, + std::shared_ptr context) { + _workletContext = std::move(context); + _workletInvoker = std::make_shared(worklet); +} + +TSelf JFrameProcessor::create(const std::shared_ptr& worklet, + const std::shared_ptr& context) { + return JFrameProcessor::newObjectCxxArgs(worklet, context); +} + +void JFrameProcessor::callWithFrameHostObject(const std::shared_ptr& frameHostObject) const { + // Call the Frame Processor on the Worklet Runtime + jsi::Runtime& runtime = _workletContext->getWorkletRuntime(); + + try { + // Wrap HostObject as JSI Value + auto argument = jsi::Object::createFromHostObject(runtime, frameHostObject); + jsi::Value jsValue(std::move(argument)); + + // Call the Worklet with the Frame JS Host Object as an argument + _workletInvoker->call(runtime, jsi::Value::undefined(), &jsValue, 1); + } catch (jsi::JSError& jsError) { + // JS Error occured, print it to console. + const std::string& message = jsError.getMessage(); + + _workletContext->invokeOnJsThread([message](jsi::Runtime& jsRuntime) { + auto logFn = jsRuntime.global().getPropertyAsObject(jsRuntime, "console").getPropertyAsFunction(jsRuntime, "error"); + logFn.call(jsRuntime, jsi::String::createFromUtf8(jsRuntime, "Frame Processor threw an error: " + message)); + }); + } +} + +void JFrameProcessor::call(jni::alias_ref frame) { + // Create the Frame Host Object wrapping the internal Frame + auto frameHostObject = std::make_shared(frame); + callWithFrameHostObject(frameHostObject); +} + +} // namespace vision diff --git a/android/src/main/cpp/java-bindings/JFrameProcessor.h b/android/src/main/cpp/java-bindings/JFrameProcessor.h new file mode 100644 index 0000000..063b9fd --- /dev/null +++ b/android/src/main/cpp/java-bindings/JFrameProcessor.h @@ -0,0 +1,49 @@ +// +// Created by Marc Rousavy on 29.09.21 +// + +#pragma once + +#include +#include +#include +#include + +#include +#include + +#include "JFrame.h" +#include "FrameHostObject.h" + +namespace vision { + +using namespace facebook; + +struct JFrameProcessor : public jni::HybridClass { + public: + static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessor;"; + static void registerNatives(); + static jni::local_ref create(const std::shared_ptr& worklet, + const std::shared_ptr& context); + + public: + /** + * Call the JS Frame Processor. + */ + void call(alias_ref frame); + + private: + // Private constructor. Use `create(..)` to create new instances. + explicit JFrameProcessor(std::shared_ptr worklet, + std::shared_ptr context); + + private: + void callWithFrameHostObject(const std::shared_ptr& frameHostObject) const; + + private: + friend HybridBase; + std::shared_ptr _workletInvoker; + std::shared_ptr _workletContext; +}; + +} // namespace vision diff --git a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp index 54a162b..9fa1d45 100644 --- a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp +++ b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.cpp @@ -12,19 +12,14 @@ namespace vision { using namespace facebook; using namespace jni; -using TCallback = jobject(alias_ref, alias_ref>); +using TCallback = jobject(alias_ref, alias_ref params); -local_ref JFrameProcessorPlugin::callback(alias_ref image, - alias_ref> params) const { +local_ref JFrameProcessorPlugin::callback(const alias_ref& frame, + const alias_ref& params) const { auto callbackMethod = getClass()->getMethod("callback"); - auto result = callbackMethod(self(), image, params); + auto result = callbackMethod(self(), frame, params); return make_local(result); } -std::string JFrameProcessorPlugin::getName() const { - auto getNameMethod = getClass()->getMethod("getName"); - return getNameMethod(self())->toStdString(); -} - } // namespace vision diff --git a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h index ad66c5d..f6f7090 100644 --- a/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h +++ b/android/src/main/cpp/java-bindings/JFrameProcessorPlugin.h @@ -7,8 +7,9 @@ #include #include #include +#include -#include "JImageProxy.h" +#include "JFrame.h" namespace vision { @@ -22,12 +23,8 @@ struct JFrameProcessorPlugin : public JavaClass { /** * Call the plugin. */ - local_ref callback(alias_ref image, - alias_ref> params) const; - /** - * Get the user-defined name of the Frame Processor Plugin - */ - std::string getName() const; + local_ref callback(const alias_ref& frame, + const alias_ref& params) const; }; } // namespace vision diff --git a/android/src/main/cpp/java-bindings/JImageProxy.cpp b/android/src/main/cpp/java-bindings/JImageProxy.cpp deleted file mode 100644 index 4c7b210..0000000 --- a/android/src/main/cpp/java-bindings/JImageProxy.cpp +++ /dev/null @@ -1,78 +0,0 @@ -// -// Created by Marc Rousavy on 22.06.21. -// - -#include "JImageProxy.h" - -#include -#include - -namespace vision { - -using namespace facebook; -using namespace jni; - -int JImageProxy::getWidth() const { - static const auto getWidthMethod = getClass()->getMethod("getWidth"); - return getWidthMethod(self()); -} - -int JImageProxy::getHeight() const { - static const auto getWidthMethod = getClass()->getMethod("getHeight"); - return getWidthMethod(self()); -} - -alias_ref getUtilsClass() { - static const auto ImageProxyUtilsClass = findClassStatic("com/mrousavy/camera/frameprocessor/ImageProxyUtils"); - return ImageProxyUtilsClass; -} - -bool JImageProxy::getIsValid() const { - auto utilsClass = getUtilsClass(); - static const auto isImageProxyValidMethod = utilsClass->getStaticMethod("isImageProxyValid"); - return isImageProxyValidMethod(utilsClass, self()); -} - -bool JImageProxy::getIsMirrored() const { - auto utilsClass = getUtilsClass(); - static const auto isImageProxyMirroredMethod = utilsClass->getStaticMethod("isImageProxyMirrored"); - return isImageProxyMirroredMethod(utilsClass, self()); -} - -jlong JImageProxy::getTimestamp() const { - auto utilsClass = getUtilsClass(); - static const auto getTimestampMethod = utilsClass->getStaticMethod("getTimestamp"); - return getTimestampMethod(utilsClass, self()); -} - -local_ref JImageProxy::getOrientation() const { - auto utilsClass = getUtilsClass(); - static const auto getOrientationMethod = utilsClass->getStaticMethod("getOrientation"); - return getOrientationMethod(utilsClass, self()); -} - -int JImageProxy::getPlanesCount() const { - auto utilsClass = getUtilsClass(); - static const auto getPlanesCountMethod = utilsClass->getStaticMethod("getPlanesCount"); - return getPlanesCountMethod(utilsClass, self()); -} - -int JImageProxy::getBytesPerRow() const { - auto utilsClass = getUtilsClass(); - static const auto getBytesPerRowMethod = utilsClass->getStaticMethod("getBytesPerRow"); - return getBytesPerRowMethod(utilsClass, self()); -} - -local_ref JImageProxy::toByteArray() const { - auto utilsClass = getUtilsClass(); - - static const auto toByteArrayMethod = utilsClass->getStaticMethod("toByteArray"); - return toByteArrayMethod(utilsClass, self()); -} - -void JImageProxy::close() { - static const auto closeMethod = getClass()->getMethod("close"); - closeMethod(self()); -} - -} // namespace vision diff --git a/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp b/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp new file mode 100644 index 0000000..73c1cd0 --- /dev/null +++ b/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp @@ -0,0 +1,91 @@ +// +// Created by Marc Rousavy on 21.07.23. +// + +#include "JVisionCameraProxy.h" + +#include +#include +#include + +#include +#include + +#include +#include + +#include "FrameProcessorPluginHostObject.h" + +namespace vision { + +using TSelf = local_ref::jhybriddata>; +using TJSCallInvokerHolder = jni::alias_ref; +using TScheduler = jni::alias_ref; +using TOptions = jni::local_ref; + +JVisionCameraProxy::JVisionCameraProxy(const jni::alias_ref& javaThis, + jsi::Runtime* runtime, + const std::shared_ptr& callInvoker, + const jni::global_ref& scheduler) { + _javaPart = make_global(javaThis); + + __android_log_write(ANDROID_LOG_INFO, TAG, "Creating Worklet Context..."); + + auto runOnJS = [callInvoker](std::function&& f) { + // Run on React JS Runtime + callInvoker->invokeAsync(std::move(f)); + }; + auto runOnWorklet = [scheduler](std::function&& f) { + // Run on Frame Processor Worklet Runtime + scheduler->cthis()->dispatchAsync([f = std::move(f)](){ + f(); + }); + }; + _workletContext = std::make_shared("VisionCamera", + runtime, + runOnJS, + runOnWorklet); + __android_log_write(ANDROID_LOG_INFO, TAG, "Worklet Context created!"); +} + + + +void JVisionCameraProxy::setFrameProcessor(int viewTag, + const alias_ref& frameProcessor) { + auto setFrameProcessorMethod = javaClassLocal()->getMethod)>("setFrameProcessor"); + setFrameProcessorMethod(_javaPart, viewTag, frameProcessor); +} + +void JVisionCameraProxy::removeFrameProcessor(int viewTag) { + auto removeFrameProcessorMethod = javaClassLocal()->getMethod("removeFrameProcessor"); + removeFrameProcessorMethod(_javaPart, viewTag); +} + +local_ref JVisionCameraProxy::getFrameProcessorPlugin(const std::string& name, + TOptions options) { + auto getFrameProcessorPluginMethod = javaClassLocal()->getMethod, TOptions)>("getFrameProcessorPlugin"); + return getFrameProcessorPluginMethod(_javaPart, make_jstring(name), std::move(options)); +} + +void JVisionCameraProxy::registerNatives() { + registerHybrid({ + makeNativeMethod("initHybrid", JVisionCameraProxy::initHybrid) + }); +} + +TSelf JVisionCameraProxy::initHybrid( + alias_ref jThis, + jlong jsRuntimePointer, + TJSCallInvokerHolder jsCallInvokerHolder, + const TScheduler& scheduler) { + __android_log_write(ANDROID_LOG_INFO, TAG, "Initializing VisionCameraProxy..."); + + // cast from JNI hybrid objects to C++ instances + auto jsRuntime = reinterpret_cast(jsRuntimePointer); + auto jsCallInvoker = jsCallInvokerHolder->cthis()->getCallInvoker(); + auto sharedScheduler = make_global(scheduler); + + return makeCxxInstance(jThis, jsRuntime, jsCallInvoker, sharedScheduler); +} + +} // namespace vision diff --git a/android/src/main/cpp/java-bindings/JVisionCameraProxy.h b/android/src/main/cpp/java-bindings/JVisionCameraProxy.h new file mode 100644 index 0000000..29c0ab5 --- /dev/null +++ b/android/src/main/cpp/java-bindings/JVisionCameraProxy.h @@ -0,0 +1,55 @@ +// +// Created by Marc Rousavy on 21.07.23. +// + +#pragma once + +#include +#include +#include +#include + +#include "JFrameProcessorPlugin.h" +#include "JVisionCameraScheduler.h" +#include "JFrameProcessor.h" + +#include +#include + +namespace vision { + +using namespace facebook; + +class JVisionCameraProxy : public jni::HybridClass { + public: + static void registerNatives(); + + void setFrameProcessor(int viewTag, + const jni::alias_ref& frameProcessor); + void removeFrameProcessor(int viewTag); + jni::local_ref getFrameProcessorPlugin(const std::string& name, + jni::local_ref options); + + public: + std::shared_ptr getWorkletContext() { return _workletContext; } + + private: + std::shared_ptr _workletContext; + + private: + friend HybridBase; + jni::global_ref _javaPart; + static auto constexpr TAG = "VisionCameraProxy"; + static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraProxy;"; + + explicit JVisionCameraProxy(const jni::alias_ref& javaThis, + jsi::Runtime* jsRuntime, + const std::shared_ptr& jsCallInvoker, + const jni::global_ref& scheduler); + static jni::local_ref initHybrid(jni::alias_ref javaThis, + jlong jsRuntimePointer, + jni::alias_ref jsCallInvokerHolder, + const jni::alias_ref& scheduler); +}; + +} // namespace vision diff --git a/android/src/main/cpp/VisionCameraScheduler.cpp b/android/src/main/cpp/java-bindings/JVisionCameraScheduler.cpp similarity index 52% rename from android/src/main/cpp/VisionCameraScheduler.cpp rename to android/src/main/cpp/java-bindings/JVisionCameraScheduler.cpp index 790ef07..7490643 100644 --- a/android/src/main/cpp/VisionCameraScheduler.cpp +++ b/android/src/main/cpp/java-bindings/JVisionCameraScheduler.cpp @@ -2,31 +2,31 @@ // Created by Marc Rousavy on 25.07.21. // -#include "VisionCameraScheduler.h" +#include "JVisionCameraScheduler.h" #include namespace vision { using namespace facebook; -using TSelf = jni::local_ref; +using TSelf = jni::local_ref; -TSelf VisionCameraScheduler::initHybrid(jni::alias_ref jThis) { +TSelf JVisionCameraScheduler::initHybrid(jni::alias_ref jThis) { return makeCxxInstance(jThis); } -void VisionCameraScheduler::dispatchAsync(std::function job) { +void JVisionCameraScheduler::dispatchAsync(const std::function& job) { // 1. add job to queue _jobs.push(job); scheduleTrigger(); } -void VisionCameraScheduler::scheduleTrigger() { +void JVisionCameraScheduler::scheduleTrigger() { // 2. schedule `triggerUI` to be called on the java thread static auto method = javaPart_->getClass()->getMethod("scheduleTrigger"); method(javaPart_.get()); } -void VisionCameraScheduler::trigger() { +void JVisionCameraScheduler::trigger() { std::unique_lock lock(_mutex); // 3. call job we enqueued in step 1. auto job = _jobs.front(); @@ -34,10 +34,10 @@ void VisionCameraScheduler::trigger() { _jobs.pop(); } -void VisionCameraScheduler::registerNatives() { +void JVisionCameraScheduler::registerNatives() { registerHybrid({ - makeNativeMethod("initHybrid", VisionCameraScheduler::initHybrid), - makeNativeMethod("trigger", VisionCameraScheduler::trigger), + makeNativeMethod("initHybrid", JVisionCameraScheduler::initHybrid), + makeNativeMethod("trigger", JVisionCameraScheduler::trigger), }); } diff --git a/android/src/main/cpp/VisionCameraScheduler.h b/android/src/main/cpp/java-bindings/JVisionCameraScheduler.h similarity index 82% rename from android/src/main/cpp/VisionCameraScheduler.h rename to android/src/main/cpp/java-bindings/JVisionCameraScheduler.h index 59f48e8..34aaae8 100644 --- a/android/src/main/cpp/VisionCameraScheduler.h +++ b/android/src/main/cpp/java-bindings/JVisionCameraScheduler.h @@ -23,22 +23,22 @@ using namespace facebook; * 3. The `scheduleTrigger()` Java Method will switch to the Frame Processor Java Thread and call `trigger()` on there * 4. `trigger()` is a C++ function here that just calls the passed C++ Method from step 1. */ -class VisionCameraScheduler : public jni::HybridClass { +class JVisionCameraScheduler : public jni::HybridClass { public: static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraScheduler;"; static jni::local_ref initHybrid(jni::alias_ref jThis); static void registerNatives(); // schedules the given job to be run on the VisionCamera FP Thread at some future point in time - void dispatchAsync(std::function job); + void dispatchAsync(const std::function& job); private: friend HybridBase; - jni::global_ref javaPart_; + jni::global_ref javaPart_; std::queue> _jobs; std::mutex _mutex; - explicit VisionCameraScheduler(jni::alias_ref jThis): + explicit JVisionCameraScheduler(jni::alias_ref jThis): javaPart_(jni::make_global(jThis)) {} // Schedules a call to `trigger` on the VisionCamera FP Thread diff --git a/android/src/main/java/com/mrousavy/camera/CameraView+Focus.kt b/android/src/main/java/com/mrousavy/camera/CameraView+Focus.kt index c945c0b..0a13fa3 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView+Focus.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView+Focus.kt @@ -18,7 +18,7 @@ suspend fun CameraView.focus(pointMap: ReadableMap) { // Getting the point from the previewView needs to be run on the UI thread val point = withContext(coroutineScope.coroutineContext) { - previewView.meteringPointFactory.createPoint(x.toFloat(), y.toFloat()); + previewView.meteringPointFactory.createPoint(x.toFloat(), y.toFloat()) } val action = FocusMeteringAction.Builder(point, FocusMeteringAction.FLAG_AF or FocusMeteringAction.FLAG_AE) diff --git a/android/src/main/java/com/mrousavy/camera/CameraView+TakeSnapshot.kt b/android/src/main/java/com/mrousavy/camera/CameraView+TakeSnapshot.kt index 3f3299b..bb40925 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView+TakeSnapshot.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView+TakeSnapshot.kt @@ -14,7 +14,7 @@ import java.io.FileOutputStream import kotlinx.coroutines.guava.await suspend fun CameraView.takeSnapshot(options: ReadableMap): WritableMap = coroutineScope { - val camera = camera ?: throw com.mrousavy.camera.CameraNotReadyError() + val camera = camera ?: throw CameraNotReadyError() val enableFlash = options.getString("flash") == "on" try { diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index fcd1b82..5cc7654 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -24,15 +24,16 @@ import androidx.lifecycle.* import com.facebook.jni.HybridData import com.facebook.proguard.annotations.DoNotStrip import com.facebook.react.bridge.* -import com.facebook.react.uimanager.events.RCTEventEmitter -import com.mrousavy.camera.frameprocessor.FrameProcessorRuntimeManager +import com.mrousavy.camera.frameprocessor.Frame +import com.mrousavy.camera.frameprocessor.FrameProcessor +import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin +import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry import com.mrousavy.camera.utils.* import kotlinx.coroutines.* import kotlinx.coroutines.guava.await import java.lang.IllegalArgumentException import java.util.concurrent.ExecutorService import java.util.concurrent.Executors -import kotlin.math.floor import kotlin.math.max import kotlin.math.min @@ -118,8 +119,9 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer internal var camera: Camera? = null internal var imageCapture: ImageCapture? = null internal var videoCapture: VideoCapture? = null - private var imageAnalysis: ImageAnalysis? = null + public var frameProcessor: FrameProcessor? = null private var preview: Preview? = null + private var imageAnalysis: ImageAnalysis? = null internal var activeVideoRecording: Recording? = null @@ -156,10 +158,7 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer private var minZoom: Float = 1f private var maxZoom: Float = 1f - @DoNotStrip - private var mHybridData: HybridData? = null - - @Suppress("LiftReturnOrAssignment", "RedundantIf") + @Suppress("RedundantIf") internal val fallbackToSnapshot: Boolean @SuppressLint("UnsafeOptInUsageError") get() { @@ -190,8 +189,6 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer } init { - mHybridData = initHybrid() - previewView = PreviewView(context) previewView.layoutParams = LayoutParams(LayoutParams.MATCH_PARENT, LayoutParams.MATCH_PARENT) previewView.installHierarchyFitter() // If this is not called correctly, view finder will be black/blank @@ -244,9 +241,6 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer imageAnalysis?.targetRotation = outputRotation } - private external fun initHybrid(): HybridData - private external fun frameProcessorCallback(frame: ImageProxy) - override fun getLifecycle(): Lifecycle { return lifecycleRegistry } @@ -461,8 +455,9 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer imageAnalysis = imageAnalysisBuilder.build().apply { setAnalyzer(cameraExecutor) { image -> // Call JS Frame Processor - frameProcessorCallback(image) - // frame gets closed in FrameHostObject implementation (JS ref counting) + val frame = Frame(image) + frameProcessor?.call(frame) + // ...frame gets closed in FrameHostObject implementation via JS ref counting } } useCases.add(imageAnalysis!!) diff --git a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt index a229a1c..5a7a875 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt @@ -3,16 +3,11 @@ package com.mrousavy.camera import android.Manifest import android.content.Context import android.content.pm.PackageManager -import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraManager import android.os.Build import android.util.Log -import android.util.Size -import androidx.camera.core.CameraSelector -import androidx.camera.extensions.ExtensionMode import androidx.camera.extensions.ExtensionsManager import androidx.camera.lifecycle.ProcessCameraProvider -import androidx.camera.video.QualitySelector import androidx.core.content.ContextCompat import com.facebook.react.bridge.* import com.facebook.react.module.annotations.ReactModule @@ -20,8 +15,9 @@ import com.facebook.react.modules.core.PermissionAwareActivity import com.facebook.react.modules.core.PermissionListener import com.facebook.react.uimanager.UIManagerHelper import com.facebook.react.bridge.ReactApplicationContext +import com.mrousavy.camera.frameprocessor.VisionCameraInstaller import java.util.concurrent.ExecutorService -import com.mrousavy.camera.frameprocessor.FrameProcessorRuntimeManager +import com.mrousavy.camera.frameprocessor.VisionCameraProxy import com.mrousavy.camera.parsers.* import com.mrousavy.camera.utils.* import kotlinx.coroutines.* @@ -30,38 +26,21 @@ import java.util.concurrent.Executors @ReactModule(name = CameraViewModule.TAG) @Suppress("unused") -class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBaseJavaModule(reactContext) { +class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJavaModule(reactContext) { companion object { const val TAG = "CameraView" var RequestCode = 10 - - fun parsePermissionStatus(status: Int): String { - return when (status) { - PackageManager.PERMISSION_DENIED -> "denied" - PackageManager.PERMISSION_GRANTED -> "authorized" - else -> "not-determined" - } - } } var frameProcessorThread: ExecutorService = Executors.newSingleThreadExecutor() private val coroutineScope = CoroutineScope(Dispatchers.Default) // TODO: or Dispatchers.Main? - private var frameProcessorManager: FrameProcessorRuntimeManager? = null - - private fun cleanup() { - if (coroutineScope.isActive) { - coroutineScope.cancel("CameraViewModule has been destroyed.") - } - } - - override fun onCatalystInstanceDestroy() { - super.onCatalystInstanceDestroy() - cleanup() - } override fun invalidate() { super.invalidate() - cleanup() + frameProcessorThread.shutdown() + if (coroutineScope.isActive) { + coroutineScope.cancel("CameraViewModule has been destroyed.") + } } override fun getName(): String { @@ -75,6 +54,18 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase return view ?: throw ViewNotFoundError(viewId) } + @ReactMethod(isBlockingSynchronousMethod = true) + fun installFrameProcessorBindings(): Boolean { + return try { + val proxy = VisionCameraProxy(reactApplicationContext, frameProcessorThread) + VisionCameraInstaller.install(proxy) + true + } catch (e: Error) { + Log.e(TAG, "Failed to install Frame Processor JSI Bindings!", e) + false + } + } + @ReactMethod fun takePhoto(viewTag: Int, options: ReadableMap, promise: Promise) { coroutineScope.launch { @@ -151,18 +142,6 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase } } - @ReactMethod(isBlockingSynchronousMethod = true) - fun installFrameProcessorBindings(): Boolean { - try { - frameProcessorManager = FrameProcessorRuntimeManager(reactApplicationContext, frameProcessorThread) - frameProcessorManager!!.installBindings() - return true - } catch (e: Error) { - Log.e(TAG, "Failed to install Frame Processor JSI Bindings!", e) - return false - } - } - @ReactMethod fun getAvailableCameraDevices(promise: Promise) { coroutineScope.launch { diff --git a/android/src/main/java/com/mrousavy/camera/Errors.kt b/android/src/main/java/com/mrousavy/camera/Errors.kt index 5281222..2311bfb 100644 --- a/android/src/main/java/com/mrousavy/camera/Errors.kt +++ b/android/src/main/java/com/mrousavy/camera/Errors.kt @@ -43,12 +43,12 @@ class ParallelVideoProcessingNotSupportedError(cause: Throwable) : CameraError(" "See https://mrousavy.github.io/react-native-vision-camera/docs/guides/devices#the-supportsparallelvideoprocessing-prop for more information.", cause) class FpsNotContainedInFormatError(fps: Int) : CameraError("format", "invalid-fps", "The given FPS were not valid for the currently selected format. Make sure you select a format which `frameRateRanges` includes $fps FPS!") -class HdrNotContainedInFormatError() : CameraError( +class HdrNotContainedInFormatError : CameraError( "format", "invalid-hdr", "The currently selected format does not support HDR capture! " + "Make sure you select a format which `frameRateRanges` includes `supportsPhotoHDR`!" ) -class LowLightBoostNotContainedInFormatError() : CameraError( +class LowLightBoostNotContainedInFormatError : CameraError( "format", "invalid-low-light-boost", "The currently selected format does not support low-light boost (night mode)! " + "Make sure you select a format which includes `supportsLowLightBoost`." diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java new file mode 100644 index 0000000..d5ccb2e --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java @@ -0,0 +1,120 @@ +package com.mrousavy.camera.frameprocessor; + +import android.annotation.SuppressLint; +import android.graphics.ImageFormat; +import android.graphics.Matrix; +import android.media.Image; +import androidx.camera.core.ImageProxy; +import com.facebook.proguard.annotations.DoNotStrip; +import java.nio.ByteBuffer; + +public class Frame { + private final ImageProxy imageProxy; + + public Frame(ImageProxy imageProxy) { + this.imageProxy = imageProxy; + } + + public ImageProxy getImageProxy() { + return imageProxy; + } + + @SuppressWarnings("unused") + @DoNotStrip + public int getWidth() { + return imageProxy.getWidth(); + } + + @SuppressWarnings("unused") + @DoNotStrip + public int getHeight() { + return imageProxy.getHeight(); + } + + @SuppressWarnings("unused") + @DoNotStrip + public boolean getIsValid() { + try { + @SuppressLint("UnsafeOptInUsageError") + Image image = imageProxy.getImage(); + if (image == null) return false; + // will throw an exception if the image is already closed + image.getCropRect(); + // no exception thrown, image must still be valid. + return true; + } catch (Exception e) { + // exception thrown, image has already been closed. + return false; + } + } + + @SuppressWarnings("unused") + @DoNotStrip + public boolean getIsMirrored() { + Matrix matrix = imageProxy.getImageInfo().getSensorToBufferTransformMatrix(); + // TODO: Figure out how to get isMirrored from ImageProxy + return false; + } + + @SuppressWarnings("unused") + @DoNotStrip + public long getTimestamp() { + return imageProxy.getImageInfo().getTimestamp(); + } + + @SuppressWarnings("unused") + @DoNotStrip + public String getOrientation() { + int rotation = imageProxy.getImageInfo().getRotationDegrees(); + if (rotation >= 45 && rotation < 135) + return "landscapeRight"; + if (rotation >= 135 && rotation < 225) + return "portraitUpsideDown"; + if (rotation >= 225 && rotation < 315) + return "landscapeLeft"; + return "portrait"; + } + + @SuppressWarnings("unused") + @DoNotStrip + public int getPlanesCount() { + return imageProxy.getPlanes().length; + } + + @SuppressWarnings("unused") + @DoNotStrip + public int getBytesPerRow() { + return imageProxy.getPlanes()[0].getRowStride(); + } + + private static byte[] byteArrayCache; + + @SuppressWarnings("unused") + @DoNotStrip + public byte[] toByteArray() { + switch (imageProxy.getFormat()) { + case ImageFormat.YUV_420_888: + ByteBuffer yBuffer = imageProxy.getPlanes()[0].getBuffer(); + ByteBuffer vuBuffer = imageProxy.getPlanes()[2].getBuffer(); + int ySize = yBuffer.remaining(); + int vuSize = vuBuffer.remaining(); + + if (byteArrayCache == null || byteArrayCache.length != ySize + vuSize) { + byteArrayCache = new byte[ySize + vuSize]; + } + + yBuffer.get(byteArrayCache, 0, ySize); + vuBuffer.get(byteArrayCache, ySize, vuSize); + + return byteArrayCache; + default: + throw new RuntimeException("Cannot convert Frame with Format " + imageProxy.getFormat() + " to byte array!"); + } + } + + @SuppressWarnings("unused") + @DoNotStrip + private void close() { + imageProxy.close(); + } +} diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java new file mode 100644 index 0000000..ec9501a --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java @@ -0,0 +1,27 @@ +package com.mrousavy.camera.frameprocessor; + +import androidx.annotation.Keep; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + +import com.facebook.jni.HybridData; +import com.facebook.proguard.annotations.DoNotStrip; + +/** + * Represents a JS Frame Processor + */ +@SuppressWarnings("JavaJniMissingFunction") // we're using fbjni. +public final class FrameProcessor { + /** + * Call the JS Frame Processor function with the given Frame + */ + public native void call(Frame frame); + + @DoNotStrip + @Keep + private final HybridData mHybridData; + + public FrameProcessor(HybridData hybridData) { + mHybridData = hybridData; + } +} diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java index 98905d5..d3b631f 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java @@ -3,8 +3,8 @@ package com.mrousavy.camera.frameprocessor; import androidx.annotation.Keep; import androidx.annotation.NonNull; import androidx.annotation.Nullable; -import androidx.camera.core.ImageProxy; import com.facebook.proguard.annotations.DoNotStrip; +import com.facebook.react.bridge.ReadableNativeMap; /** * Declares a Frame Processor Plugin. @@ -12,42 +12,14 @@ import com.facebook.proguard.annotations.DoNotStrip; @DoNotStrip @Keep public abstract class FrameProcessorPlugin { - private final @NonNull String mName; - /** * The actual Frame Processor plugin callback. Called for every frame the ImageAnalyzer receives. - * @param image The CameraX ImageProxy. Don't call .close() on this, as VisionCamera handles that. + * @param frame The Frame from the Camera. Don't call .close() on this, as VisionCamera handles that. * @return You can return any primitive, map or array you want. See the * Types * table for a list of supported types. */ @DoNotStrip @Keep - public abstract @Nullable Object callback(@NonNull ImageProxy image, @NonNull Object[] params); - - /** - * Initializes the native plugin part. - * @param name Specifies the Frame Processor Plugin's name in the Runtime. - * The actual name in the JS Runtime will be prefixed with two underscores (`__`) - */ - protected FrameProcessorPlugin(@NonNull String name) { - mName = name; - } - - /** - * Get the user-defined name of the Frame Processor Plugin. - */ - @DoNotStrip - @Keep - public @NonNull String getName() { - return mName; - } - - /** - * Registers the given plugin in the Frame Processor Runtime. - * @param plugin An instance of a plugin. - */ - public static void register(@NonNull FrameProcessorPlugin plugin) { - FrameProcessorRuntimeManager.Companion.addPlugin(plugin); - } + public abstract @Nullable Object callback(@NonNull Frame frame, @Nullable ReadableNativeMap params); } diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPluginRegistry.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPluginRegistry.java new file mode 100644 index 0000000..ecbd2a2 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPluginRegistry.java @@ -0,0 +1,38 @@ +package com.mrousavy.camera.frameprocessor; + +import androidx.annotation.Keep; +import androidx.annotation.Nullable; + +import com.facebook.proguard.annotations.DoNotStrip; +import com.facebook.react.bridge.ReadableNativeMap; + +import java.util.HashMap; +import java.util.Map; + +@DoNotStrip +@Keep +public class FrameProcessorPluginRegistry { + private static final Map Plugins = new HashMap<>(); + + @DoNotStrip + @Keep + public static void addFrameProcessorPlugin(String name, PluginInitializer pluginInitializer) { + assert !Plugins.containsKey(name) : "Tried to add a Frame Processor Plugin with a name that already exists! " + + "Either choose unique names, or remove the unused plugin. Name: "; + Plugins.put(name, pluginInitializer); + } + + @DoNotStrip + @Keep + public static FrameProcessorPlugin getPlugin(String name, ReadableNativeMap options) { + PluginInitializer initializer = Plugins.get(name); + if (initializer == null) { + return null; + } + return initializer.initializePlugin(options); + } + + public interface PluginInitializer { + FrameProcessorPlugin initializePlugin(@Nullable ReadableNativeMap options); + } +} diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/ImageProxyUtils.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/ImageProxyUtils.java deleted file mode 100644 index f1cf0a1..0000000 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/ImageProxyUtils.java +++ /dev/null @@ -1,98 +0,0 @@ -package com.mrousavy.camera.frameprocessor; - -import android.annotation.SuppressLint; -import android.graphics.ImageFormat; -import android.graphics.Matrix; -import android.media.Image; - -import androidx.annotation.Keep; -import androidx.camera.core.ImageProxy; -import com.facebook.proguard.annotations.DoNotStrip; - -import java.nio.ByteBuffer; - -@SuppressWarnings("unused") // used through JNI -@DoNotStrip -@Keep -public class ImageProxyUtils { - @SuppressLint("UnsafeOptInUsageError") - @DoNotStrip - @Keep - public static boolean isImageProxyValid(ImageProxy imageProxy) { - try { - Image image = imageProxy.getImage(); - if (image == null) return false; - // will throw an exception if the image is already closed - image.getCropRect(); - // no exception thrown, image must still be valid. - return true; - } catch (Exception e) { - // exception thrown, image has already been closed. - return false; - } - } - - @DoNotStrip - @Keep - public static boolean isImageProxyMirrored(ImageProxy imageProxy) { - Matrix matrix = imageProxy.getImageInfo().getSensorToBufferTransformMatrix(); - // TODO: Figure out how to get isMirrored from ImageProxy - return false; - } - - @DoNotStrip - @Keep - public static String getOrientation(ImageProxy imageProxy) { - int rotation = imageProxy.getImageInfo().getRotationDegrees(); - if (rotation >= 45 && rotation < 135) - return "landscapeRight"; - if (rotation >= 135 && rotation < 225) - return "portraitUpsideDown"; - if (rotation >= 225 && rotation < 315) - return "landscapeLeft"; - return "portrait"; - } - - @DoNotStrip - @Keep - public static long getTimestamp(ImageProxy imageProxy) { - return imageProxy.getImageInfo().getTimestamp(); - } - - @DoNotStrip - @Keep - public static int getPlanesCount(ImageProxy imageProxy) { - return imageProxy.getPlanes().length; - } - - @DoNotStrip - @Keep - public static int getBytesPerRow(ImageProxy imageProxy) { - return imageProxy.getPlanes()[0].getRowStride(); - } - - private static byte[] byteArrayCache; - - @DoNotStrip - @Keep - public static byte[] toByteArray(ImageProxy imageProxy) { - switch (imageProxy.getFormat()) { - case ImageFormat.YUV_420_888: - ByteBuffer yBuffer = imageProxy.getPlanes()[0].getBuffer(); - ByteBuffer vuBuffer = imageProxy.getPlanes()[2].getBuffer(); - int ySize = yBuffer.remaining(); - int vuSize = vuBuffer.remaining(); - - if (byteArrayCache == null || byteArrayCache.length != ySize + vuSize) { - byteArrayCache = new byte[ySize + vuSize]; - } - - yBuffer.get(byteArrayCache, 0, ySize); - vuBuffer.get(byteArrayCache, ySize, vuSize); - - return byteArrayCache; - default: - throw new RuntimeException("Cannot convert Frame with Format " + imageProxy.getFormat() + " to byte array!"); - } - } -} diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraInstaller.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraInstaller.java new file mode 100644 index 0000000..23833c3 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraInstaller.java @@ -0,0 +1,6 @@ +package com.mrousavy.camera.frameprocessor; + +@SuppressWarnings("JavaJniMissingFunction") // we use fbjni +public class VisionCameraInstaller { + public static native void install(VisionCameraProxy proxy); +} diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager.kt b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt similarity index 55% rename from android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager.kt rename to android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt index b6abf7e..246a745 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorRuntimeManager.kt +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/VisionCameraProxy.kt @@ -5,6 +5,7 @@ import androidx.annotation.Keep import com.facebook.jni.HybridData import com.facebook.proguard.annotations.DoNotStrip import com.facebook.react.bridge.ReactApplicationContext +import com.facebook.react.bridge.ReadableNativeMap import com.facebook.react.turbomodule.core.CallInvokerHolderImpl import com.facebook.react.uimanager.UIManagerHelper import com.mrousavy.camera.CameraView @@ -12,12 +13,11 @@ import com.mrousavy.camera.ViewNotFoundError import java.lang.ref.WeakReference import java.util.concurrent.ExecutorService -@Suppress("KotlinJniMissingFunction") // I use fbjni, Android Studio is not smart enough to realize that. -class FrameProcessorRuntimeManager(context: ReactApplicationContext, frameProcessorThread: ExecutorService) { - companion object { - const val TAG = "FrameProcessorRuntime" - private val Plugins: ArrayList = ArrayList() +@Suppress("KotlinJniMissingFunction") // we use fbjni. +class VisionCameraProxy(context: ReactApplicationContext, frameProcessorThread: ExecutorService) { + companion object { + const val TAG = "VisionCameraProxy" init { try { System.loadLibrary("VisionCamera") @@ -26,52 +26,53 @@ class FrameProcessorRuntimeManager(context: ReactApplicationContext, frameProces throw e } } - - fun addPlugin(plugin: FrameProcessorPlugin) { - Plugins.add(plugin) - } } - @DoNotStrip - private var mHybridData: HybridData? = null - private var mContext: WeakReference? = null - private var mScheduler: VisionCameraScheduler? = null + @Keep + private var mHybridData: HybridData + private var mContext: WeakReference + private var mScheduler: VisionCameraScheduler init { val jsCallInvokerHolder = context.catalystInstance.jsCallInvokerHolder as CallInvokerHolderImpl val jsRuntimeHolder = context.javaScriptContextHolder.get() mScheduler = VisionCameraScheduler(frameProcessorThread) mContext = WeakReference(context) - mHybridData = initHybrid(jsRuntimeHolder, jsCallInvokerHolder, mScheduler!!) + mHybridData = initHybrid(jsRuntimeHolder, jsCallInvokerHolder, mScheduler) } - @Suppress("unused") - @DoNotStrip - @Keep - fun findCameraViewById(viewId: Int): CameraView { + private fun findCameraViewById(viewId: Int): CameraView { Log.d(TAG, "Finding view $viewId...") - val ctx = mContext?.get() + val ctx = mContext.get() val view = if (ctx != null) UIManagerHelper.getUIManager(ctx, viewId)?.resolveView(viewId) as CameraView? else null Log.d(TAG, if (view != null) "Found view $viewId!" else "Couldn't find view $viewId!") return view ?: throw ViewNotFoundError(viewId) } - fun installBindings() { - Log.i(TAG, "Installing JSI Bindings on JS Thread...") - installJSIBindings() - Log.i(TAG, "Installing Frame Processor Plugins...") - Plugins.forEach { plugin -> - registerPlugin(plugin) - } - Log.i(TAG, "Successfully installed ${Plugins.count()} Frame Processor Plugins!") + @DoNotStrip + @Keep + fun setFrameProcessor(viewId: Int, frameProcessor: FrameProcessor) { + val view = findCameraViewById(viewId) + view.frameProcessor = frameProcessor + } + + @DoNotStrip + @Keep + fun removeFrameProcessor(viewId: Int) { + val view = findCameraViewById(viewId) + view.frameProcessor = null + } + + @DoNotStrip + @Keep + fun getFrameProcessorPlugin(name: String, options: ReadableNativeMap): FrameProcessorPlugin { + return FrameProcessorPluginRegistry.getPlugin(name, options) } // private C++ funcs private external fun initHybrid( - jsContext: Long, - jsCallInvokerHolder: CallInvokerHolderImpl, - scheduler: VisionCameraScheduler + jsContext: Long, + jsCallInvokerHolder: CallInvokerHolderImpl, + scheduler: VisionCameraScheduler ): HybridData - private external fun registerPlugin(plugin: FrameProcessorPlugin) - private external fun installJSIBindings() } diff --git a/android/src/main/java/com/mrousavy/camera/parsers/PermissionStatus+String.kt b/android/src/main/java/com/mrousavy/camera/parsers/PermissionStatus+String.kt new file mode 100644 index 0000000..a85a7bc --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/parsers/PermissionStatus+String.kt @@ -0,0 +1,11 @@ +package com.mrousavy.camera.parsers + +import android.content.pm.PackageManager + +fun parsePermissionStatus(status: Int): String { + return when (status) { + PackageManager.PERMISSION_DENIED -> "denied" + PackageManager.PERMISSION_GRANTED -> "authorized" + else -> "not-determined" + } +} diff --git a/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx b/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx index e94f8b7..13fe39d 100644 --- a/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx +++ b/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx @@ -131,7 +131,7 @@ const frameProcessor = useFrameProcessor((frame) => { ## What's possible? -You can run any native code you want in a Frame Processor Plugin. Just like in the native iOS and Android Camera APIs, you will receive a frame (`CMSampleBuffer` on iOS, `ImageProxy` on Android) which you can use however you want. In other words; **everything is possible**. +You can run any native code you want in a Frame Processor Plugin. Just like in the native iOS and Android Camera APIs, you will receive a frame ([`CMSampleBuffer`][5] on iOS, [`ImageProxy`][6] on Android) which you can use however you want. In other words; **everything is possible**. ## Implementations @@ -194,5 +194,7 @@ Your Frame Processor Plugins have to be fast. Use the FPS Graph (`enableFpsGraph [1]: https://github.com/mrousavy/react-native-vision-camera/blob/main/src/Frame.ts [2]: https://github.com/mrousavy/react-native-vision-camera/blob/main/ios/Frame%20Processor/Frame.h -[3]: https://developer.android.com/reference/androidx/camera/core/ImageProxy +[3]: https://github.com/mrousavy/react-native-vision-camera/blob/main/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java [4]: https://github.com/facebook/react-native/blob/9a43eac7a32a6ba3164a048960101022a92fcd5a/React/Base/RCTBridgeModule.h#L20-L24 +[5]: https://developer.apple.com/documentation/coremedia/cmsamplebuffer +[6]: https://developer.android.com/reference/androidx/camera/core/ImageProxy diff --git a/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx b/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx index d6667bf..6de0deb 100644 --- a/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx +++ b/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx @@ -55,21 +55,16 @@ For reference see the [CLI's docs](https://github.com/mateusz1913/vision-camera- 3. Add the following code: ```java {8} -import androidx.camera.core.ImageProxy; +import com.mrousavy.camera.frameprocessor.Frame; import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin; public class FaceDetectorFrameProcessorPlugin extends FrameProcessorPlugin { @Override - public Object callback(ImageProxy image, ReadableNativeMap arguments) { + public Object callback(Frame frame, ReadableNativeMap arguments) { // code goes here return null; } - - @Override - public String getName() { - return "detectFaces"; - } } ``` @@ -86,13 +81,14 @@ import com.facebook.react.bridge.NativeModule; import com.facebook.react.bridge.ReactApplicationContext; import com.facebook.react.uimanager.ViewManager; import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin; +import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry; import javax.annotation.Nonnull; public class FaceDetectorFrameProcessorPluginPackage implements ReactPackage { @NonNull @Override public List createNativeModules(@NonNull ReactApplicationContext reactContext) { - FrameProcessorPlugin.register(new FaceDetectorFrameProcessorPlugin()); + FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces", options -> new FaceDetectorFrameProcessorPlugin()); return Collections.emptyList(); } @@ -125,19 +121,15 @@ public class FaceDetectorFrameProcessorPluginPackage implements ReactPackage { 3. Add the following code: ```kotlin {7} -import androidx.camera.core.ImageProxy +import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin class FaceDetectorFrameProcessorPlugin: FrameProcessorPlugin() { - override fun callback(image: ImageProxy, arguments: ReadableNativeMap): Any? { + override fun callback(frame: Frame, arguments: ReadableNativeMap): Any? { // code goes here return null } - - override fun getName(): String { - return "detectFaces" - } } ``` @@ -157,7 +149,9 @@ import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin class FaceDetectorFrameProcessorPluginPackage : ReactPackage { override fun createNativeModules(reactContext: ReactApplicationContext): List { - FrameProcessorPlugin.register(FaceDetectorFrameProcessorPlugin()) + FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces") { options -> + FaceDetectorFrameProcessorPlugin() + } return emptyList() } diff --git a/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java b/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java index f5b8c30..ff984a5 100644 --- a/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java +++ b/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java @@ -1,19 +1,30 @@ package com.mrousavy.camera.example; import android.util.Log; + import androidx.camera.core.ImageProxy; + +import com.facebook.react.bridge.ReadableNativeMap; import com.facebook.react.bridge.WritableNativeArray; import com.facebook.react.bridge.WritableNativeMap; +import com.mrousavy.camera.frameprocessor.Frame; import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.HashMap; public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin { @Override - public Object callback(@NotNull ImageProxy image, @NotNull Object[] params) { - Log.d("ExamplePlugin", image.getWidth() + " x " + image.getHeight() + " Image with format #" + image.getFormat() + ". Logging " + params.length + " parameters:"); + public Object callback(@NotNull Frame frame, @Nullable ReadableNativeMap params) { + HashMap hashMap = params != null ? params.toHashMap() : new HashMap<>(); + ImageProxy image = frame.getImageProxy(); - for (Object param : params) { - Log.d("ExamplePlugin", " -> " + (param == null ? "(null)" : param.toString() + " (" + param.getClass().getName() + ")")); + Log.d("ExamplePlugin", image.getWidth() + " x " + image.getHeight() + " Image with format #" + image.getFormat() + ". Logging " + hashMap.size() + " parameters:"); + + for (String key : hashMap.keySet()) { + Object value = hashMap.get(key); + Log.d("ExamplePlugin", " -> " + (value == null ? "(null)" : value.toString() + " (" + value.getClass().getName() + ")")); } WritableNativeMap map = new WritableNativeMap(); @@ -31,6 +42,6 @@ public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin { } ExampleFrameProcessorPlugin() { - super("example_plugin"); + } } diff --git a/example/android/app/src/main/java/com/mrousavy/camera/example/MainApplication.java b/example/android/app/src/main/java/com/mrousavy/camera/example/MainApplication.java index 14f79ce..ba7e6fb 100644 --- a/example/android/app/src/main/java/com/mrousavy/camera/example/MainApplication.java +++ b/example/android/app/src/main/java/com/mrousavy/camera/example/MainApplication.java @@ -1,10 +1,14 @@ package com.mrousavy.camera.example; import android.app.Application; + +import androidx.annotation.Nullable; + import com.facebook.react.PackageList; import com.facebook.react.ReactApplication; import com.facebook.react.ReactNativeHost; import com.facebook.react.ReactPackage; +import com.facebook.react.bridge.ReadableNativeMap; import com.facebook.soloader.SoLoader; import java.util.List; import com.facebook.react.defaults.DefaultNewArchitectureEntryPoint; @@ -12,6 +16,7 @@ import com.facebook.react.defaults.DefaultReactNativeHost; import com.mrousavy.camera.CameraPackage; import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin; +import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry; public class MainApplication extends Application implements ReactApplication { @@ -61,6 +66,6 @@ public class MainApplication extends Application implements ReactApplication { DefaultNewArchitectureEntryPoint.load(); } - FrameProcessorPlugin.register(new ExampleFrameProcessorPlugin()); + FrameProcessorPluginRegistry.addFrameProcessorPlugin("example_plugin", options -> new ExampleFrameProcessorPlugin()); } } diff --git a/example/src/CameraPage.tsx b/example/src/CameraPage.tsx index fbc6be3..29cbbb2 100644 --- a/example/src/CameraPage.tsx +++ b/example/src/CameraPage.tsx @@ -1,6 +1,6 @@ import * as React from 'react'; import { useRef, useState, useMemo, useCallback } from 'react'; -import { Platform, StyleSheet, Text, View } from 'react-native'; +import { StyleSheet, Text, View } from 'react-native'; import { PinchGestureHandler, PinchGestureHandlerGestureEvent, TapGestureHandler } from 'react-native-gesture-handler'; import { CameraDeviceFormat, @@ -8,7 +8,7 @@ import { PhotoFile, sortFormats, useCameraDevices, - useSkiaFrameProcessor, + useFrameProcessor, VideoFile, } from 'react-native-vision-camera'; import { Camera, frameRateIncluded } from 'react-native-vision-camera'; @@ -26,6 +26,7 @@ import type { NativeStackScreenProps } from '@react-navigation/native-stack'; import { useIsFocused } from '@react-navigation/core'; import { Skia } from '@shopify/react-native-skia'; import { FACE_SHADER } from './Shaders'; +import { examplePlugin } from './frame-processors/ExamplePlugin'; const ReanimatedCamera = Reanimated.createAnimatedComponent(Camera); Reanimated.addWhitelistedNativeProps({ @@ -217,16 +218,13 @@ export function CameraPage({ navigation }: Props): React.ReactElement { const paint = Skia.Paint(); paint.setImageFilter(imageFilter); - const isIOS = Platform.OS === 'ios'; - const frameProcessor = useSkiaFrameProcessor( - (frame) => { - 'worklet'; - console.log(`Width: ${frame.width}`); + const frameProcessor = useFrameProcessor((frame) => { + 'worklet'; - if (frame.isDrawable) frame.render(paint); - }, - [isIOS, paint], - ); + console.log(`Width: ${frame.width}`); + const result = examplePlugin(frame); + console.log('Example Plugin: ', result); + }, []); return ( @@ -247,12 +245,10 @@ export function CameraPage({ navigation }: Props): React.ReactElement { onError={onError} enableZoomGesture={false} animatedProps={cameraAnimatedProps} - photo={true} - video={true} audio={hasMicrophonePermission} enableFpsGraph={true} orientation="portrait" - frameProcessor={device.supportsParallelVideoProcessing ? frameProcessor : undefined} + frameProcessor={frameProcessor} /> diff --git a/ios/Frame Processor/FrameProcessorPluginHostObject.mm b/ios/Frame Processor/FrameProcessorPluginHostObject.mm index a91b5f8..ce631ce 100644 --- a/ios/Frame Processor/FrameProcessorPluginHostObject.mm +++ b/ios/Frame Processor/FrameProcessorPluginHostObject.mm @@ -12,6 +12,8 @@ #import "FrameHostObject.h" #import "JSINSObjectConversion.h" +using namespace facebook; + std::vector FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime& runtime) { std::vector result; result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("call"))); diff --git a/ios/Frame Processor/VisionCameraProxy.mm b/ios/Frame Processor/VisionCameraProxy.mm index 7845707..81cecfd 100644 --- a/ios/Frame Processor/VisionCameraProxy.mm +++ b/ios/Frame Processor/VisionCameraProxy.mm @@ -177,7 +177,7 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID& const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value { - if (count != 1 || !arguments[0].isString()) { + if (count < 1 || !arguments[0].isString()) { throw jsi::JSError(runtime, "First argument needs to be a string (pluginName)!"); } auto pluginName = arguments[0].asString(runtime).utf8(runtime);