From 862e05b64ff6e7159776c1b310e38277a0b17ea0 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 23 Aug 2023 12:42:38 +0200 Subject: [PATCH] feat: Allow build without Skia or Frame Processors (#1710) * feat: Make Frame Processors optional in JS * Allow Android build without Frame Processors * fix: Fix `EncoderProfiles.width` null-error * Update gradle.properties * Update gradle.properties * fix: Use `#ifdef` instead of `#if` * Update JVisionCameraProxy.cpp * fix: Fix definitions * Revert "fix: Use `#ifdef` instead of `#if`" This reverts commit b19f32e5ce7df558cadcc8c4b5006c9cdf2cbe66. * fix: Fix build * chore: Codestyle * Update JFrameProcessor.cpp --- android/CMakeLists.txt | 105 +++++++++++------- android/build.gradle | 22 +++- android/src/main/cpp/FrameHostObject.cpp | 62 ++++++----- android/src/main/cpp/VisionCamera.cpp | 6 +- android/src/main/cpp/VisionCameraProxy.cpp | 30 +---- .../cpp/java-bindings/JFrameProcessor.cpp | 9 +- .../main/cpp/java-bindings/JFrameProcessor.h | 4 + .../cpp/java-bindings/JVisionCameraProxy.cpp | 50 ++++++++- .../cpp/java-bindings/JVisionCameraProxy.h | 21 ++-- android/src/main/cpp/skia/OpenGLError.h | 4 + android/src/main/cpp/skia/SkiaRenderer.cpp | 4 + android/src/main/cpp/skia/SkiaRenderer.h | 4 + .../CameraCharacteristics+getOutputSizes.kt | 6 +- docs/docs/guides/FRAME_PROCESSORS.mdx | 2 +- docs/docs/guides/FRAME_PROCESSORS_SKIA.mdx | 2 +- docs/docs/guides/TROUBLESHOOTING.mdx | 4 +- example/android/gradle.properties | 4 + ios/Skia Render Layer/SkiaRenderer.h | 2 +- src/FrameProcessorPlugins.ts | 66 +++++++---- .../withDisableFrameProcessorsAndroid.ts | 4 +- src/hooks/useFrameProcessor.ts | 2 - 21 files changed, 266 insertions(+), 147 deletions(-) diff --git a/android/CMakeLists.txt b/android/CMakeLists.txt index 7c3b6e5..00785bb 100644 --- a/android/CMakeLists.txt +++ b/android/CMakeLists.txt @@ -6,7 +6,6 @@ set(PACKAGE_NAME "VisionCamera") set(BUILD_DIR ${CMAKE_SOURCE_DIR}/build) set(CMAKE_VERBOSE_MAKEFILE ON) set(CMAKE_CXX_STANDARD 17) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSK_GL -DSK_GANESH -DSK_BUILD_FOR_ANDROID") # Folly include("${NODE_MODULES_DIR}/react-native/ReactAndroid/cmake-utils/folly-flags.cmake") @@ -15,25 +14,10 @@ add_compile_options(${folly_FLAGS}) # Third party libraries (Prefabs) find_package(ReactAndroid REQUIRED CONFIG) find_package(fbjni REQUIRED CONFIG) -find_package(react-native-worklets-core REQUIRED CONFIG) find_library(LOG_LIB log) -set(RNSKIA_PATH ${NODE_MODULES_DIR}/@shopify/react-native-skia) -if(EXISTS ${RNSKIA_PATH}) - find_package(shopify_react-native-skia REQUIRED CONFIG) - set(SKIA_PACKAGE shopify_react-native-skia::rnskia) - message("VisionCamera: Skia integration enabled!") -else() - message("VisionCamera: Skia integration disabled!") -ENDIF() +add_definitions(-DVISION_CAMERA_ENABLE_FRAME_PROCESSORS=${ENABLE_FRAME_PROCESSORS} -DVISION_CAMERA_ENABLE_SKIA=${ENABLE_SKIA}) -set (SKIA_LIBS_PATH "${RNSKIA_PATH}/libs/android/${ANDROID_ABI}") -add_library(skia STATIC IMPORTED) -set_property(TARGET skia PROPERTY IMPORTED_LOCATION "${SKIA_LIBS_PATH}/libskia.a") -add_library(svg STATIC IMPORTED) -set_property(TARGET svg PROPERTY IMPORTED_LOCATION "${SKIA_LIBS_PATH}/libsvg.a") -add_library(skshaper STATIC IMPORTED) -set_property(TARGET skshaper PROPERTY IMPORTED_LOCATION "${SKIA_LIBS_PATH}/libskshaper.a") # Add react-native-vision-camera sources add_library( @@ -63,24 +47,6 @@ target_include_directories( "${NODE_MODULES_DIR}/react-native/ReactCommon" "${NODE_MODULES_DIR}/react-native/ReactCommon/callinvoker" "${NODE_MODULES_DIR}/react-native/ReactAndroid/src/main/jni/react/turbomodule" # <-- CallInvokerHolder JNI wrapper - - # We need to include the headers from skia - # (Note: rnskia includes all their files without any relative path - # so for example "include/core/SkImage.h" becomes #include "SkImage.h". - # That's why for the prefab of rnskia, we flatten all cpp files into - # just one directory. HOWEVER, skia itself uses relative paths in - # their include statements, and so we have to include the path to skia) - "${RNSKIA_PATH}/cpp/skia" - - "${RNSKIA_PATH}/cpp/skia/include/config/" - "${RNSKIA_PATH}/cpp/skia/include/core/" - "${RNSKIA_PATH}/cpp/skia/include/effects/" - "${RNSKIA_PATH}/cpp/skia/include/utils/" - "${RNSKIA_PATH}/cpp/skia/include/pathops/" - "${RNSKIA_PATH}/cpp/skia/modules/" - # "${RNSKIA_PATH}/cpp/skia/modules/skparagraph/include/" - "${RNSKIA_PATH}/cpp/skia/include/" - "${RNSKIA_PATH}/cpp/skia" ) # Link everything together @@ -92,12 +58,65 @@ target_link_libraries( ReactAndroid::reactnativejni # <-- RN: React Native JNI bindings ReactAndroid::folly_runtime # <-- RN: For casting JSI <> Java objects fbjni::fbjni # <-- fbjni - react-native-worklets-core::rnworklets # <-- RN Worklets - GLESv2 # <-- Optional: OpenGL (for Skia) - EGL # <-- Optional: OpenGL (EGL) (for Skia) - ${SKIA_PACKAGE} # <-- Optional: RN Skia - jnigraphics - skia - svg - skshaper ) + +# Optionally also add Frame Processors here +if(ENABLE_FRAME_PROCESSORS) + find_package(react-native-worklets-core REQUIRED CONFIG) + target_link_libraries( + ${PACKAGE_NAME} + react-native-worklets-core::rnworklets + ) + message("VisionCamera: Frame Processors enabled!") + + # Optionally also add Skia Integration here + if(ENABLE_SKIA) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSK_GL -DSK_GANESH -DSK_BUILD_FOR_ANDROID") + + find_package(shopify_react-native-skia REQUIRED CONFIG) + + set(SKIA_PACKAGE shopify_react-native-skia::rnskia) + set(RNSKIA_PATH ${NODE_MODULES_DIR}/@shopify/react-native-skia) + set (SKIA_LIBS_PATH "${RNSKIA_PATH}/libs/android/${ANDROID_ABI}") + add_library(skia STATIC IMPORTED) + set_property(TARGET skia PROPERTY IMPORTED_LOCATION "${SKIA_LIBS_PATH}/libskia.a") + add_library(svg STATIC IMPORTED) + set_property(TARGET svg PROPERTY IMPORTED_LOCATION "${SKIA_LIBS_PATH}/libsvg.a") + add_library(skshaper STATIC IMPORTED) + set_property(TARGET skshaper PROPERTY IMPORTED_LOCATION "${SKIA_LIBS_PATH}/libskshaper.a") + + # We need to include the headers from skia + # (Note: rnskia includes all their files without any relative path + # so for example "include/core/SkImage.h" becomes #include "SkImage.h". + # That's why for the prefab of rnskia, we flatten all cpp files into + # just one directory. HOWEVER, skia itself uses relative paths in + # their include statements, and so we have to include the path to skia) + target_include_directories( + ${PACKAGE_NAME} + PRIVATE + "${RNSKIA_PATH}/cpp/skia" + "${RNSKIA_PATH}/cpp/skia/include/config/" + "${RNSKIA_PATH}/cpp/skia/include/core/" + "${RNSKIA_PATH}/cpp/skia/include/effects/" + "${RNSKIA_PATH}/cpp/skia/include/utils/" + "${RNSKIA_PATH}/cpp/skia/include/pathops/" + "${RNSKIA_PATH}/cpp/skia/modules/" + # "${RNSKIA_PATH}/cpp/skia/modules/skparagraph/include/" + "${RNSKIA_PATH}/cpp/skia/include/" + "${RNSKIA_PATH}/cpp/skia" + ) + + target_link_libraries( + ${PACKAGE_NAME} + GLESv2 # <-- Optional: OpenGL (for Skia) + EGL # <-- Optional: OpenGL (EGL) (for Skia) + ${SKIA_PACKAGE} # <-- Optional: RN Skia + jnigraphics + skia + svg + skshaper + ) + + message("VisionCamera: Skia enabled!") + endif() +endif() diff --git a/android/build.gradle b/android/build.gradle index c9e665d..0e6cc57 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -65,6 +65,12 @@ static def findNodeModules(baseDir) { def nodeModules = findNodeModules(projectDir) +def hasWorklets = !safeExtGet("VisionCamera_disableFrameProcessors", false) && findProject(":react-native-worklets-core") != null +def hasSkia = !safeExtGet("VisionCamera_disableSkia", false) && findProject(":shopify_react-native-skia") != null + +logger.warn("[VisionCamera] react-native-worklets-core ${hasWorklets ? "found" : "not found"}, Frame Processors ${hasWorklets ? "enabled" : "disabled"}!") +logger.warn("[VisionCamera] react-native-skia ${hasSkia ? "found" : "not found"}, Skia Frame Processors ${hasSkia ? "enabled" : "disabled"}!") + repositories { google() mavenCentral() @@ -98,7 +104,9 @@ android { cmake { cppFlags "-O2 -frtti -fexceptions -Wall -Wno-unused-variable -fstack-protector-all" arguments "-DANDROID_STL=c++_shared", - "-DNODE_MODULES_DIR=${nodeModules}" + "-DNODE_MODULES_DIR=${nodeModules}", + "-DENABLE_FRAME_PROCESSORS=${hasWorklets}", + "-DENABLE_SKIA=${hasWorklets && hasSkia}" abiFilters (*reactNativeArchitectures()) } } @@ -141,11 +149,15 @@ dependencies { implementation 'com.facebook.react:react-android:+' implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:1.5.2" - // Frame Processor integration (optional) - implementation project(":react-native-worklets-core") + if (hasWorklets) { + // Frame Processor integration (optional) + implementation project(":react-native-worklets-core") - // Skia Frame Processor integration (optional) - implementation project(":shopify_react-native-skia") + if (hasSkia) { + // Skia Frame Processor integration (optional) + implementation project(":shopify_react-native-skia") + } + } } // Resolves "LOCAL_SRC_FILES points to a missing file, Check that libfb.so exists or that its path is correct". diff --git a/android/src/main/cpp/FrameHostObject.cpp b/android/src/main/cpp/FrameHostObject.cpp index ded7657..81de379 100644 --- a/android/src/main/cpp/FrameHostObject.cpp +++ b/android/src/main/cpp/FrameHostObject.cpp @@ -8,7 +8,6 @@ #include #include -#include #include "JSITypedArray.h" #include @@ -51,8 +50,39 @@ std::vector FrameHostObject::getPropertyNames(jsi::Runtime& rt) jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) { auto name = propName.utf8(runtime); + if (name == "incrementRefCount") { + jsi::HostFunctionType incrementRefCount = [=](jsi::Runtime& runtime, + const jsi::Value& thisArg, + const jsi::Value* args, + size_t count) -> jsi::Value { + // Increment retain count by one. + this->frame->incrementRefCount(); + return jsi::Value::undefined(); + }; + return jsi::Function::createFromHostFunction(runtime, + jsi::PropNameID::forUtf8(runtime, "incrementRefCount"), + 0, + incrementRefCount); + } + if (name == "decrementRefCount") { + auto decrementRefCount = [=](jsi::Runtime& runtime, + const jsi::Value& thisArg, + const jsi::Value* args, + size_t count) -> jsi::Value { + // Decrement retain count by one. If the retain count is zero, the Frame gets closed. + this->frame->decrementRefCount(); + return jsi::Value::undefined(); + }; + return jsi::Function::createFromHostFunction(runtime, + jsi::PropNameID::forUtf8(runtime, "decrementRefCount"), + 0, + decrementRefCount); + } if (name == "toString") { - auto toString = JSI_HOST_FUNCTION_LAMBDA { + jsi::HostFunctionType toString = [=](jsi::Runtime& runtime, + const jsi::Value& thisArg, + const jsi::Value* args, + size_t count) -> jsi::Value { if (!this->frame) { return jsi::String::createFromUtf8(runtime, "[closed frame]"); } @@ -64,7 +94,10 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString); } if (name == "toArrayBuffer") { - auto toArrayBuffer = JSI_HOST_FUNCTION_LAMBDA { + jsi::HostFunctionType toArrayBuffer = [=](jsi::Runtime& runtime, + const jsi::Value& thisArg, + const jsi::Value* args, + size_t count) -> jsi::Value { auto buffer = this->frame->toByteArray(); auto arraySize = buffer->size(); @@ -92,29 +125,6 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr }; return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toArrayBuffer"), 0, toArrayBuffer); } - if (name == "incrementRefCount") { - auto incrementRefCount = JSI_HOST_FUNCTION_LAMBDA { - // Increment retain count by one. - this->frame->incrementRefCount(); - return jsi::Value::undefined(); - }; - return jsi::Function::createFromHostFunction(runtime, - jsi::PropNameID::forUtf8(runtime, "incrementRefCount"), - 0, - incrementRefCount); - } - - if (name == "decrementRefCount") { - auto decrementRefCount = JSI_HOST_FUNCTION_LAMBDA { - // Decrement retain count by one. If the retain count is zero, the Frame gets closed. - this->frame->decrementRefCount(); - return jsi::Value::undefined(); - }; - return jsi::Function::createFromHostFunction(runtime, - jsi::PropNameID::forUtf8(runtime, "decrementRefCount"), - 0, - decrementRefCount); - } if (name == "isValid") { return jsi::Value(this->frame && this->frame->getIsValid()); diff --git a/android/src/main/cpp/VisionCamera.cpp b/android/src/main/cpp/VisionCamera.cpp index ea4c43d..f58de85 100644 --- a/android/src/main/cpp/VisionCamera.cpp +++ b/android/src/main/cpp/VisionCamera.cpp @@ -9,9 +9,13 @@ JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) { return facebook::jni::initialize(vm, [] { vision::VisionCameraInstaller::registerNatives(); - vision::JFrameProcessor::registerNatives(); vision::JVisionCameraProxy::registerNatives(); vision::JVisionCameraScheduler::registerNatives(); +#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS + vision::JFrameProcessor::registerNatives(); +#endif +#if VISION_CAMERA_ENABLE_SKIA vision::SkiaRenderer::registerNatives(); +#endif }); } diff --git a/android/src/main/cpp/VisionCameraProxy.cpp b/android/src/main/cpp/VisionCameraProxy.cpp index ba302cd..b41d1e2 100644 --- a/android/src/main/cpp/VisionCameraProxy.cpp +++ b/android/src/main/cpp/VisionCameraProxy.cpp @@ -19,6 +19,10 @@ #include #include +#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS +#include +#endif + namespace vision { using namespace facebook; @@ -28,11 +32,6 @@ VisionCameraProxy::VisionCameraProxy(const jni::alias_refcthis()->getWorkletContext(); - invalidateArrayBufferCache(*workletContext->getJsRuntime()); - invalidateArrayBufferCache(workletContext->getWorkletRuntime()); } std::vector VisionCameraProxy::getPropertyNames(jsi::Runtime& runtime) { @@ -45,24 +44,7 @@ std::vector VisionCameraProxy::getPropertyNames(jsi::Runtime& r } void VisionCameraProxy::setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& object) { - auto frameProcessorType = object.getProperty(runtime, "type").asString(runtime).utf8(runtime); - auto worklet = std::make_shared(runtime, object.getProperty(runtime, "frameProcessor")); - auto workletContext = _javaProxy->cthis()->getWorkletContext(); - - jni::local_ref frameProcessor; - if (frameProcessorType == "frame-processor") { - frameProcessor = JFrameProcessor::create(worklet, workletContext); - } else if (frameProcessorType == "skia-frame-processor") { -#if VISION_CAMERA_ENABLE_SKIA - throw std::runtime_error("system/skia-unavailable: Skia is not yet implemented on Android!"); -#else - throw std::runtime_error("system/skia-unavailable: Skia is not installed!"); -#endif - } else { - throw std::runtime_error("Unknown FrameProcessor.type passed! Received: " + frameProcessorType); - } - - _javaProxy->cthis()->setFrameProcessor(viewTag, make_global(frameProcessor)); + _javaProxy->cthis()->setFrameProcessor(viewTag, runtime, object); } void VisionCameraProxy::removeFrameProcessor(int viewTag) { @@ -143,7 +125,7 @@ void VisionCameraInstaller::install(jni::alias_ref, jni::alias_ref proxy) { // global.VisionCameraProxy auto visionCameraProxy = std::make_shared(proxy); - jsi::Runtime& runtime = *proxy->cthis()->getWorkletContext()->getJsRuntime(); + jsi::Runtime& runtime = *proxy->cthis()->getJSRuntime(); runtime.global().setProperty(runtime, "VisionCameraProxy", jsi::Object::createFromHostObject(runtime, visionCameraProxy)); diff --git a/android/src/main/cpp/java-bindings/JFrameProcessor.cpp b/android/src/main/cpp/java-bindings/JFrameProcessor.cpp index 9c6aa2c..ab3846e 100644 --- a/android/src/main/cpp/java-bindings/JFrameProcessor.cpp +++ b/android/src/main/cpp/java-bindings/JFrameProcessor.cpp @@ -2,8 +2,9 @@ // Created by Marc Rousavy on 29.09.21. // -#include "JFrameProcessor.h" +#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS +#include "JFrameProcessor.h" #include #include @@ -50,8 +51,8 @@ void JFrameProcessor::callWithFrameHostObject(const std::shared_ptrinvokeOnJsThread([message](jsi::Runtime& jsRuntime) { - auto logFn = jsRuntime.global().getPropertyAsObject(jsRuntime, "console").getPropertyAsFunction(jsRuntime, "error"); - logFn.call(jsRuntime, jsi::String::createFromUtf8(jsRuntime, "Frame Processor threw an error: " + message)); + auto logFn = jsRuntime.global().getPropertyAsObject(jsRuntime, "console").getPropertyAsFunction(jsRuntime, "error"); + logFn.call(jsRuntime, jsi::String::createFromUtf8(jsRuntime, "Frame Processor threw an error: " + message)); }); } } @@ -63,3 +64,5 @@ void JFrameProcessor::call(jni::alias_ref frame) { } } // namespace vision + +#endif diff --git a/android/src/main/cpp/java-bindings/JFrameProcessor.h b/android/src/main/cpp/java-bindings/JFrameProcessor.h index 8b9229d..940839e 100644 --- a/android/src/main/cpp/java-bindings/JFrameProcessor.h +++ b/android/src/main/cpp/java-bindings/JFrameProcessor.h @@ -4,6 +4,8 @@ #pragma once +#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS + #include #include #include @@ -47,3 +49,5 @@ struct JFrameProcessor : public jni::HybridClass { }; } // namespace vision + +#endif diff --git a/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp b/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp index 6f981cd..cf48a32 100644 --- a/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp +++ b/android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp @@ -11,10 +11,13 @@ #include #include +#include "FrameProcessorPluginHostObject.h" +#include "JSITypedArray.h" + +#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS #include #include - -#include "FrameProcessorPluginHostObject.h" +#endif namespace vision { @@ -28,7 +31,9 @@ JVisionCameraProxy::JVisionCameraProxy(const jni::alias_ref& callInvoker, const jni::global_ref& scheduler) { _javaPart = make_global(javaThis); + _runtime = runtime; +#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS __android_log_write(ANDROID_LOG_INFO, TAG, "Creating Worklet Context..."); auto runOnJS = [callInvoker](std::function&& f) { @@ -46,14 +51,53 @@ JVisionCameraProxy::JVisionCameraProxy(const jni::alias_refgetJsRuntime()); + invalidateArrayBufferCache(_workletContext->getWorkletRuntime()); +#endif } void JVisionCameraProxy::setFrameProcessor(int viewTag, - const alias_ref& frameProcessor) { + jsi::Runtime& runtime, + const jsi::Object& frameProcessorObject) { +#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS + auto frameProcessorType = frameProcessorObject.getProperty(runtime, "type").asString(runtime).utf8(runtime); + auto worklet = std::make_shared(runtime, frameProcessorObject.getProperty(runtime, "frameProcessor")); + + jni::local_ref frameProcessor; + if (frameProcessorType == "frame-processor") { + frameProcessor = JFrameProcessor::create(worklet, _workletContext); + } else if (frameProcessorType == "skia-frame-processor") { +#if VISION_CAMERA_ENABLE_SKIA + throw std::runtime_error("system/skia-unavailable: Skia is not yet implemented on Android!"); +#else + throw std::runtime_error("system/skia-unavailable: Skia is not installed!"); +#endif + } else { + throw std::runtime_error("Unknown FrameProcessor.type passed! Received: " + frameProcessorType); + } + auto setFrameProcessorMethod = javaClassLocal()->getMethod)>("setFrameProcessor"); setFrameProcessorMethod(_javaPart, viewTag, frameProcessor); +#else + throw std::runtime_error("system/frame-processors-unavailable: Frame Processors are disabled!"); +#endif } void JVisionCameraProxy::removeFrameProcessor(int viewTag) { diff --git a/android/src/main/cpp/java-bindings/JVisionCameraProxy.h b/android/src/main/cpp/java-bindings/JVisionCameraProxy.h index 84c42bf..9840e4f 100644 --- a/android/src/main/cpp/java-bindings/JVisionCameraProxy.h +++ b/android/src/main/cpp/java-bindings/JVisionCameraProxy.h @@ -6,8 +6,8 @@ #include #include -#include #include +#include #include "JFrameProcessorPlugin.h" #include "JVisionCameraScheduler.h" @@ -16,29 +16,36 @@ #include #include +#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS +#include +#endif + namespace vision { using namespace facebook; class JVisionCameraProxy : public jni::HybridClass { public: + ~JVisionCameraProxy(); static void registerNatives(); void setFrameProcessor(int viewTag, - const jni::alias_ref& frameProcessor); + jsi::Runtime& runtime, + const jsi::Object& frameProcessor); void removeFrameProcessor(int viewTag); jni::local_ref getFrameProcessorPlugin(const std::string& name, jni::local_ref options); - public: - std::shared_ptr getWorkletContext() { return _workletContext; } - - private: - std::shared_ptr _workletContext; + jsi::Runtime* getJSRuntime() { return _runtime; } private: friend HybridBase; jni::global_ref _javaPart; + jsi::Runtime* _runtime; +#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS + std::shared_ptr _workletContext; +#endif + static auto constexpr TAG = "VisionCameraProxy"; static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraProxy;"; diff --git a/android/src/main/cpp/skia/OpenGLError.h b/android/src/main/cpp/skia/OpenGLError.h index 335a077..e255184 100644 --- a/android/src/main/cpp/skia/OpenGLError.h +++ b/android/src/main/cpp/skia/OpenGLError.h @@ -4,6 +4,8 @@ #pragma once +#if VISION_CAMERA_ENABLE_SKIA + #include #include #include @@ -24,3 +26,5 @@ class OpenGLError: public std::runtime_error { }; } // namespace vision + +#endif diff --git a/android/src/main/cpp/skia/SkiaRenderer.cpp b/android/src/main/cpp/skia/SkiaRenderer.cpp index 47d53ab..a25c8a6 100644 --- a/android/src/main/cpp/skia/SkiaRenderer.cpp +++ b/android/src/main/cpp/skia/SkiaRenderer.cpp @@ -2,6 +2,8 @@ // Created by Marc Rousavy on 10.08.23. // +#if VISION_CAMERA_ENABLE_SKIA + #include "SkiaRenderer.h" #include #include "OpenGLError.h" @@ -325,3 +327,5 @@ void SkiaRenderer::registerNatives() { } } // namespace vision + +#endif diff --git a/android/src/main/cpp/skia/SkiaRenderer.h b/android/src/main/cpp/skia/SkiaRenderer.h index 7a9a97e..a93678f 100644 --- a/android/src/main/cpp/skia/SkiaRenderer.h +++ b/android/src/main/cpp/skia/SkiaRenderer.h @@ -4,6 +4,8 @@ #pragma once +#if VISION_CAMERA_ENABLE_SKIA + #include #include #include @@ -75,3 +77,5 @@ class SkiaRenderer: public jni::HybridClass { }; } // namespace vision + +#endif diff --git a/android/src/main/java/com/mrousavy/camera/extensions/CameraCharacteristics+getOutputSizes.kt b/android/src/main/java/com/mrousavy/camera/extensions/CameraCharacteristics+getOutputSizes.kt index 13021f8..64523a3 100644 --- a/android/src/main/java/com/mrousavy/camera/extensions/CameraCharacteristics+getOutputSizes.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/CameraCharacteristics+getOutputSizes.kt @@ -34,8 +34,10 @@ private fun getMaximumVideoSize(cameraId: String): Size? { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) { val profiles = CamcorderProfile.getAll(cameraId, CamcorderProfile.QUALITY_HIGH) if (profiles != null) { - val largestProfile = profiles.videoProfiles.maxBy { it.width * it.height } - return Size(largestProfile.width, largestProfile.height) + val largestProfile = profiles.videoProfiles.filterNotNull().maxByOrNull { it.width * it.height } + if (largestProfile != null) { + return Size(largestProfile.width, largestProfile.height) + } } } diff --git a/docs/docs/guides/FRAME_PROCESSORS.mdx b/docs/docs/guides/FRAME_PROCESSORS.mdx index 956ae3d..813466e 100644 --- a/docs/docs/guides/FRAME_PROCESSORS.mdx +++ b/docs/docs/guides/FRAME_PROCESSORS.mdx @@ -235,7 +235,7 @@ The Frame Processor API spawns a secondary JavaScript Runtime which consumes a s Inside your `gradle.properties` file, add the `disableFrameProcessors` flag: ```groovy -disableFrameProcessors=true +VisionCamera_disableFrameProcessors=true ``` Then, clean and rebuild your project. diff --git a/docs/docs/guides/FRAME_PROCESSORS_SKIA.mdx b/docs/docs/guides/FRAME_PROCESSORS_SKIA.mdx index 6addd34..619989e 100644 --- a/docs/docs/guides/FRAME_PROCESSORS_SKIA.mdx +++ b/docs/docs/guides/FRAME_PROCESSORS_SKIA.mdx @@ -91,7 +91,7 @@ Skia Frame Processors ship with additional C++ files which might slightly increa Inside your `gradle.properties` file, add the `disableSkia` flag: ```groovy -disableSkia=true +VisionCamera_disableSkia=true ``` Then, clean and rebuild your project. diff --git a/docs/docs/guides/TROUBLESHOOTING.mdx b/docs/docs/guides/TROUBLESHOOTING.mdx index d98515e..802ef70 100644 --- a/docs/docs/guides/TROUBLESHOOTING.mdx +++ b/docs/docs/guides/TROUBLESHOOTING.mdx @@ -85,8 +85,8 @@ Before opening an issue, make sure you try the following: ``` distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-all.zip ``` -7. Try building without Skia. Set `disableSkia = true` in your `gradle.properties`, and try rebuilding. -8. Try building without Frame Processors. Set `disableFrameProcessors = true` in your `gradle.properties`, and try rebuilding. +7. Try building without Skia. Set `VisionCamera_disableSkia = true` in your `gradle.properties`, and try rebuilding. +8. Try building without Frame Processors. Set `VisionCamera_disableFrameProcessors = true` in your `gradle.properties`, and try rebuilding. ### Runtime Issues diff --git a/example/android/gradle.properties b/example/android/gradle.properties index 42cc94c..5295f11 100644 --- a/example/android/gradle.properties +++ b/example/android/gradle.properties @@ -38,3 +38,7 @@ newArchEnabled=false # Use this property to enable or disable the Hermes JS engine. # If set to false, you will be using JSC instead. hermesEnabled=true + +# Can be set to true to disable the build setup +#VisionCamera_disableFrameProcessors=true +#VisionCamera_disableSkia=true diff --git a/ios/Skia Render Layer/SkiaRenderer.h b/ios/Skia Render Layer/SkiaRenderer.h index 043f47a..5bbc133 100644 --- a/ios/Skia Render Layer/SkiaRenderer.h +++ b/ios/Skia Render Layer/SkiaRenderer.h @@ -20,7 +20,7 @@ typedef void(^draw_callback_t)(SkiaCanvas _Nonnull); It provides two Contexts, one offscreen and one onscreen. - Offscreen Context: Allows you to render a Frame into a Skia Canvas and draw onto it using Skia commands - Onscreen Context: Allows you to render a Frame from the offscreen context onto a Layer allowing it to be displayed for Preview. - + The two contexts may run at different Frame Rates. */ @interface SkiaRenderer : NSObject diff --git a/src/FrameProcessorPlugins.ts b/src/FrameProcessorPlugins.ts index 6b0d447..29e5352 100644 --- a/src/FrameProcessorPlugins.ts +++ b/src/FrameProcessorPlugins.ts @@ -1,9 +1,11 @@ import type { Frame, FrameInternal } from './Frame'; import type { FrameProcessor } from './CameraProps'; import { Camera } from './Camera'; -import { Worklets } from 'react-native-worklets-core'; import { CameraRuntimeError } from './CameraError'; +// only import typescript types +import type TWorklets from 'react-native-worklets-core'; + type BasicParameterType = string | number | boolean | undefined; type ParameterType = BasicParameterType | BasicParameterType[] | Record; @@ -28,17 +30,48 @@ interface TVisionCameraProxy { isSkiaEnabled: boolean; } -Camera.installFrameProcessorBindings(); +let isAsyncContextBusy = { value: false }; +let runOnAsyncContext = (_frame: Frame, _func: () => void): void => { + throw new CameraRuntimeError( + 'system/frame-processors-unavailable', + 'Frame Processors are not available, react-native-worklets-core is not installed!', + ); +}; + +try { + // eslint-disable-next-line @typescript-eslint/no-var-requires + const { Worklets } = require('react-native-worklets-core') as typeof TWorklets; + + // Install native Frame Processor Runtime Manager + Camera.installFrameProcessorBindings(); + // @ts-expect-error global is untyped, it's a C++ host-object + if (global.VisionCameraProxy == null) { + throw new CameraRuntimeError( + 'system/frame-processors-unavailable', + 'Failed to install VisionCameraProxy. Are Frame Processors properly enabled?', + ); + } + + isAsyncContextBusy = Worklets.createSharedValue(false); + const asyncContext = Worklets.createContext('VisionCamera.async'); + runOnAsyncContext = Worklets.createRunInContextFn((frame: Frame, func: () => void) => { + 'worklet'; + try { + // Call long-running function + func(); + } finally { + // Potentially delete Frame if we were the last ref + (frame as FrameInternal).decrementRefCount(); + + isAsyncContextBusy.value = false; + } + }, asyncContext); +} catch (e) { + // Worklets are not installed, so Frame Processors are disabled. +} // @ts-expect-error global is untyped, it's a C++ host-object export const VisionCameraProxy = global.VisionCameraProxy as TVisionCameraProxy; -// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -if (VisionCameraProxy == null) { - throw new CameraRuntimeError( - 'system/frame-processors-unavailable', - 'Failed to install VisionCameraProxy. Are Frame Processors properly enabled?', - ); -} declare global { // eslint-disable-next-line no-var @@ -96,21 +129,6 @@ export function runAtTargetFps(fps: number, func: () => T): T | undefined { return undefined; } -const isAsyncContextBusy = Worklets.createSharedValue(false); -const asyncContext = Worklets.createContext('VisionCamera.async'); -const runOnAsyncContext = Worklets.createRunInContextFn((frame: Frame, func: () => void) => { - 'worklet'; - try { - // Call long-running function - func(); - } finally { - // Potentially delete Frame if we were the last ref - (frame as FrameInternal).decrementRefCount(); - - isAsyncContextBusy.value = false; - } -}, asyncContext); - /** * Runs the given function asynchronously, while keeping a strong reference to the Frame. * diff --git a/src/expo-plugin/withDisableFrameProcessorsAndroid.ts b/src/expo-plugin/withDisableFrameProcessorsAndroid.ts index caed43d..c530e03 100644 --- a/src/expo-plugin/withDisableFrameProcessorsAndroid.ts +++ b/src/expo-plugin/withDisableFrameProcessorsAndroid.ts @@ -1,11 +1,11 @@ import { ConfigPlugin, withGradleProperties } from '@expo/config-plugins'; /** - * Set the `disableFrameProcessors` value in the static `gradle.properties` file. + * Set the `VisionCamera_disableFrameProcessors` value in the static `gradle.properties` file. * This is used to disable frame processors if you don't need it for android. */ export const withDisableFrameProcessorsAndroid: ConfigPlugin = (c) => { - const disableFrameProcessorsKey = 'disableFrameProcessors'; + const disableFrameProcessorsKey = 'VisionCamera_disableFrameProcessors'; return withGradleProperties(c, (config) => { config.modResults = config.modResults.filter((item) => { if (item.type === 'property' && item.key === disableFrameProcessorsKey) return false; diff --git a/src/hooks/useFrameProcessor.ts b/src/hooks/useFrameProcessor.ts index 3674917..227dab6 100644 --- a/src/hooks/useFrameProcessor.ts +++ b/src/hooks/useFrameProcessor.ts @@ -1,8 +1,6 @@ import { DependencyList, useMemo } from 'react'; import type { DrawableFrame, Frame, FrameInternal } from '../Frame'; import { FrameProcessor } from '../CameraProps'; -// Install RN Worklets by importing it -import 'react-native-worklets-core'; export function createFrameProcessor(frameProcessor: FrameProcessor['frameProcessor'], type: FrameProcessor['type']): FrameProcessor { return {