railbird/react-native-vision-camera
9
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Allow build without Skia or Frame Processors (#1710)

Browse Source 
* feat: Make Frame Processors optional in JS

* Allow Android build without Frame Processors

* fix: Fix `EncoderProfiles.width` null-error

* Update gradle.properties

* Update gradle.properties

* fix: Use `#ifdef` instead of `#if`

* Update JVisionCameraProxy.cpp

* fix: Fix definitions

* Revert "fix: Use `#ifdef` instead of `#if`"

This reverts commit b19f32e5ce7df558cadcc8c4b5006c9cdf2cbe66.

* fix: Fix build

* chore: Codestyle

* Update JFrameProcessor.cpp
This commit is contained in:
Marc Rousavy 2023-08-23 12:42:38 +02:00 committed by GitHub
parent 617c5607d4
commit 862e05b64f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
21 changed files with 266 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

105
android/CMakeLists.txt
View File

@ -6,7 +6,6 @@ set(PACKAGE_NAME "VisionCamera")
set(BUILD_DIR ${CMAKE_SOURCE_DIR}/build)
set(CMAKE_VERBOSE_MAKEFILE ON)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSK_GL -DSK_GANESH -DSK_BUILD_FOR_ANDROID")
# Folly
include("${NODE_MODULES_DIR}/react-native/ReactAndroid/cmake-utils/folly-flags.cmake")
@ -15,25 +14,10 @@ add_compile_options(${folly_FLAGS})
# Third party libraries (Prefabs)
find_package(ReactAndroid REQUIRED CONFIG)
find_package(fbjni REQUIRED CONFIG)
find_package(react-native-worklets-core REQUIRED CONFIG)
find_library(LOG_LIB log)
set(RNSKIA_PATH ${NODE_MODULES_DIR}/@shopify/react-native-skia)
if(EXISTS ${RNSKIA_PATH})
find_package(shopify_react-native-skia REQUIRED CONFIG)
set(SKIA_PACKAGE shopify_react-native-skia::rnskia)
message("VisionCamera: Skia integration enabled!")
else()
message("VisionCamera: Skia integration disabled!")
ENDIF()
add_definitions(-DVISION_CAMERA_ENABLE_FRAME_PROCESSORS=${ENABLE_FRAME_PROCESSORS} -DVISION_CAMERA_ENABLE_SKIA=${ENABLE_SKIA})
set (SKIA_LIBS_PATH "${RNSKIA_PATH}/libs/android/${ANDROID_ABI}")
add_library(skia STATIC IMPORTED)
set_property(TARGET skia PROPERTY IMPORTED_LOCATION "${SKIA_LIBS_PATH}/libskia.a")
add_library(svg STATIC IMPORTED)
set_property(TARGET svg PROPERTY IMPORTED_LOCATION "${SKIA_LIBS_PATH}/libsvg.a")
add_library(skshaper STATIC IMPORTED)
set_property(TARGET skshaper PROPERTY IMPORTED_LOCATION "${SKIA_LIBS_PATH}/libskshaper.a")
# Add react-native-vision-camera sources
add_library(
@ -63,24 +47,6 @@ target_include_directories(
"${NODE_MODULES_DIR}/react-native/ReactCommon"
"${NODE_MODULES_DIR}/react-native/ReactCommon/callinvoker"
"${NODE_MODULES_DIR}/react-native/ReactAndroid/src/main/jni/react/turbomodule" # <-- CallInvokerHolder JNI wrapper
# We need to include the headers from skia
# (Note: rnskia includes all their files without any relative path
# so for example "include/core/SkImage.h" becomes #include "SkImage.h".
# That's why for the prefab of rnskia, we flatten all cpp files into
# just one directory. HOWEVER, skia itself uses relative paths in
# their include statements, and so we have to include the path to skia)
"${RNSKIA_PATH}/cpp/skia"
"${RNSKIA_PATH}/cpp/skia/include/config/"
"${RNSKIA_PATH}/cpp/skia/include/core/"
"${RNSKIA_PATH}/cpp/skia/include/effects/"
"${RNSKIA_PATH}/cpp/skia/include/utils/"
"${RNSKIA_PATH}/cpp/skia/include/pathops/"
"${RNSKIA_PATH}/cpp/skia/modules/"
# "${RNSKIA_PATH}/cpp/skia/modules/skparagraph/include/"
"${RNSKIA_PATH}/cpp/skia/include/"
"${RNSKIA_PATH}/cpp/skia"
)
# Link everything together
@ -92,12 +58,65 @@ target_link_libraries(
ReactAndroid::reactnativejni # <-- RN: React Native JNI bindings
ReactAndroid::folly_runtime # <-- RN: For casting JSI <> Java objects
fbjni::fbjni # <-- fbjni
react-native-worklets-core::rnworklets # <-- RN Worklets
GLESv2 # <-- Optional: OpenGL (for Skia)
EGL # <-- Optional: OpenGL (EGL) (for Skia)
${SKIA_PACKAGE} # <-- Optional: RN Skia
jnigraphics
skia
svg
skshaper
)
# Optionally also add Frame Processors here
if(ENABLE_FRAME_PROCESSORS)
find_package(react-native-worklets-core REQUIRED CONFIG)
target_link_libraries(
${PACKAGE_NAME}
react-native-worklets-core::rnworklets
)
message("VisionCamera: Frame Processors enabled!")
# Optionally also add Skia Integration here
if(ENABLE_SKIA)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSK_GL -DSK_GANESH -DSK_BUILD_FOR_ANDROID")
find_package(shopify_react-native-skia REQUIRED CONFIG)
set(SKIA_PACKAGE shopify_react-native-skia::rnskia)
set(RNSKIA_PATH ${NODE_MODULES_DIR}/@shopify/react-native-skia)
set (SKIA_LIBS_PATH "${RNSKIA_PATH}/libs/android/${ANDROID_ABI}")
add_library(skia STATIC IMPORTED)
set_property(TARGET skia PROPERTY IMPORTED_LOCATION "${SKIA_LIBS_PATH}/libskia.a")
add_library(svg STATIC IMPORTED)
set_property(TARGET svg PROPERTY IMPORTED_LOCATION "${SKIA_LIBS_PATH}/libsvg.a")
add_library(skshaper STATIC IMPORTED)
set_property(TARGET skshaper PROPERTY IMPORTED_LOCATION "${SKIA_LIBS_PATH}/libskshaper.a")
# We need to include the headers from skia
# (Note: rnskia includes all their files without any relative path
# so for example "include/core/SkImage.h" becomes #include "SkImage.h".
# That's why for the prefab of rnskia, we flatten all cpp files into
# just one directory. HOWEVER, skia itself uses relative paths in
# their include statements, and so we have to include the path to skia)
target_include_directories(
${PACKAGE_NAME}
PRIVATE
"${RNSKIA_PATH}/cpp/skia"
"${RNSKIA_PATH}/cpp/skia/include/config/"
"${RNSKIA_PATH}/cpp/skia/include/core/"
"${RNSKIA_PATH}/cpp/skia/include/effects/"
"${RNSKIA_PATH}/cpp/skia/include/utils/"
"${RNSKIA_PATH}/cpp/skia/include/pathops/"
"${RNSKIA_PATH}/cpp/skia/modules/"
# "${RNSKIA_PATH}/cpp/skia/modules/skparagraph/include/"
"${RNSKIA_PATH}/cpp/skia/include/"
"${RNSKIA_PATH}/cpp/skia"
)
target_link_libraries(
${PACKAGE_NAME}
GLESv2 # <-- Optional: OpenGL (for Skia)
EGL # <-- Optional: OpenGL (EGL) (for Skia)
${SKIA_PACKAGE} # <-- Optional: RN Skia
jnigraphics
skia
svg
skshaper
)
message("VisionCamera: Skia enabled!")
endif()
endif()

22
android/build.gradle
View File

@ -65,6 +65,12 @@ static def findNodeModules(baseDir) {
def nodeModules = findNodeModules(projectDir)
def hasWorklets = !safeExtGet("VisionCamera_disableFrameProcessors", false) && findProject(":react-native-worklets-core") != null
def hasSkia = !safeExtGet("VisionCamera_disableSkia", false) && findProject(":shopify_react-native-skia") != null
logger.warn("[VisionCamera] react-native-worklets-core ${hasWorklets ? "found" : "not found"}, Frame Processors ${hasWorklets ? "enabled" : "disabled"}!")
logger.warn("[VisionCamera] react-native-skia ${hasSkia ? "found" : "not found"}, Skia Frame Processors ${hasSkia ? "enabled" : "disabled"}!")
repositories {
google()
mavenCentral()
@ -98,7 +104,9 @@ android {
cmake {
cppFlags "-O2 -frtti -fexceptions -Wall -Wno-unused-variable -fstack-protector-all"
arguments "-DANDROID_STL=c++_shared",
"-DNODE_MODULES_DIR=${nodeModules}"
"-DNODE_MODULES_DIR=${nodeModules}",
"-DENABLE_FRAME_PROCESSORS=${hasWorklets}",
"-DENABLE_SKIA=${hasWorklets && hasSkia}"
abiFilters (*reactNativeArchitectures())
}
}
@ -141,11 +149,15 @@ dependencies {
implementation 'com.facebook.react:react-android:+'
implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:1.5.2"
// Frame Processor integration (optional)
implementation project(":react-native-worklets-core")
if (hasWorklets) {
// Frame Processor integration (optional)
implementation project(":react-native-worklets-core")
// Skia Frame Processor integration (optional)
implementation project(":shopify_react-native-skia")
if (hasSkia) {
// Skia Frame Processor integration (optional)
implementation project(":shopify_react-native-skia")
}
}
}
// Resolves "LOCAL_SRC_FILES points to a missing file, Check that libfb.so exists or that its path is correct".

62
android/src/main/cpp/FrameHostObject.cpp
View File

@ -8,7 +8,6 @@
#include <fbjni/fbjni.h>
#include <jni.h>
#include <react-native-worklets-core/WKTJsiHostObject.h>
#include "JSITypedArray.h"
#include <vector>
@ -51,8 +50,39 @@ std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt)
jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
auto name = propName.utf8(runtime);
if (name == "incrementRefCount") {
jsi::HostFunctionType incrementRefCount = [=](jsi::Runtime& runtime,
const jsi::Value& thisArg,
const jsi::Value* args,
size_t count) -> jsi::Value {
// Increment retain count by one.
this->frame->incrementRefCount();
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "incrementRefCount"),
0,
incrementRefCount);
}
if (name == "decrementRefCount") {
auto decrementRefCount = [=](jsi::Runtime& runtime,
const jsi::Value& thisArg,
const jsi::Value* args,
size_t count) -> jsi::Value {
// Decrement retain count by one. If the retain count is zero, the Frame gets closed.
this->frame->decrementRefCount();
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "decrementRefCount"),
0,
decrementRefCount);
}
if (name == "toString") {
auto toString = JSI_HOST_FUNCTION_LAMBDA {
jsi::HostFunctionType toString = [=](jsi::Runtime& runtime,
const jsi::Value& thisArg,
const jsi::Value* args,
size_t count) -> jsi::Value {
if (!this->frame) {
return jsi::String::createFromUtf8(runtime, "[closed frame]");
}
@ -64,7 +94,10 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
}
if (name == "toArrayBuffer") {
auto toArrayBuffer = JSI_HOST_FUNCTION_LAMBDA {
jsi::HostFunctionType toArrayBuffer = [=](jsi::Runtime& runtime,
const jsi::Value& thisArg,
const jsi::Value* args,
size_t count) -> jsi::Value {
auto buffer = this->frame->toByteArray();
auto arraySize = buffer->size();
@ -92,29 +125,6 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
};
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toArrayBuffer"), 0, toArrayBuffer);
}
if (name == "incrementRefCount") {
auto incrementRefCount = JSI_HOST_FUNCTION_LAMBDA {
// Increment retain count by one.
this->frame->incrementRefCount();
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "incrementRefCount"),
0,
incrementRefCount);
}
if (name == "decrementRefCount") {
auto decrementRefCount = JSI_HOST_FUNCTION_LAMBDA {
// Decrement retain count by one. If the retain count is zero, the Frame gets closed.
this->frame->decrementRefCount();
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "decrementRefCount"),
0,
decrementRefCount);
}
if (name == "isValid") {
return jsi::Value(this->frame && this->frame->getIsValid());

6
android/src/main/cpp/VisionCamera.cpp
View File

@ -9,9 +9,13 @@
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) {
return facebook::jni::initialize(vm, [] {
vision::VisionCameraInstaller::registerNatives();
vision::JFrameProcessor::registerNatives();
vision::JVisionCameraProxy::registerNatives();
vision::JVisionCameraScheduler::registerNatives();
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
vision::JFrameProcessor::registerNatives();
#endif
#if VISION_CAMERA_ENABLE_SKIA
vision::SkiaRenderer::registerNatives();
#endif
});
}

30
android/src/main/cpp/VisionCameraProxy.cpp
View File

@ -19,6 +19,10 @@
#include <string>
#include <memory>
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
#include <react-native-worklets-core/WKTJsiWorkletContext.h>
#endif
namespace vision {
using namespace facebook;
@ -28,11 +32,6 @@ VisionCameraProxy::VisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::ja
}
VisionCameraProxy::~VisionCameraProxy() {
__android_log_write(ANDROID_LOG_INFO, TAG, "Destroying Context...");
// Destroy ArrayBuffer cache for both the JS and the Worklet Runtime.
auto workletContext = _javaProxy->cthis()->getWorkletContext();
invalidateArrayBufferCache(*workletContext->getJsRuntime());
invalidateArrayBufferCache(workletContext->getWorkletRuntime());
}
std::vector<jsi::PropNameID> VisionCameraProxy::getPropertyNames(jsi::Runtime& runtime) {
@ -45,24 +44,7 @@ std::vector<jsi::PropNameID> VisionCameraProxy::getPropertyNames(jsi::Runtime& r
}
void VisionCameraProxy::setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& object) {
auto frameProcessorType = object.getProperty(runtime, "type").asString(runtime).utf8(runtime);
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, object.getProperty(runtime, "frameProcessor"));
auto workletContext = _javaProxy->cthis()->getWorkletContext();
jni::local_ref<JFrameProcessor::javaobject> frameProcessor;
if (frameProcessorType == "frame-processor") {
frameProcessor = JFrameProcessor::create(worklet, workletContext);
} else if (frameProcessorType == "skia-frame-processor") {
#if VISION_CAMERA_ENABLE_SKIA
throw std::runtime_error("system/skia-unavailable: Skia is not yet implemented on Android!");
#else
throw std::runtime_error("system/skia-unavailable: Skia is not installed!");
#endif
} else {
throw std::runtime_error("Unknown FrameProcessor.type passed! Received: " + frameProcessorType);
}
_javaProxy->cthis()->setFrameProcessor(viewTag, make_global(frameProcessor));
_javaProxy->cthis()->setFrameProcessor(viewTag, runtime, object);
}
void VisionCameraProxy::removeFrameProcessor(int viewTag) {
@ -143,7 +125,7 @@ void VisionCameraInstaller::install(jni::alias_ref<jni::JClass>,
jni::alias_ref<JVisionCameraProxy::javaobject> proxy) {
// global.VisionCameraProxy
auto visionCameraProxy = std::make_shared<VisionCameraProxy>(proxy);
jsi::Runtime& runtime = *proxy->cthis()->getWorkletContext()->getJsRuntime();
jsi::Runtime& runtime = *proxy->cthis()->getJSRuntime();
runtime.global().setProperty(runtime,
"VisionCameraProxy",
jsi::Object::createFromHostObject(runtime, visionCameraProxy));

9
android/src/main/cpp/java-bindings/JFrameProcessor.cpp
View File

@ -2,8 +2,9 @@
// Created by Marc Rousavy on 29.09.21.
//
#include "JFrameProcessor.h"
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
#include "JFrameProcessor.h"
#include <jni.h>
#include <fbjni/fbjni.h>
@ -50,8 +51,8 @@ void JFrameProcessor::callWithFrameHostObject(const std::shared_ptr<FrameHostObj
const std::string& message = jsError.getMessage();
_workletContext->invokeOnJsThread([message](jsi::Runtime& jsRuntime) {
auto logFn = jsRuntime.global().getPropertyAsObject(jsRuntime, "console").getPropertyAsFunction(jsRuntime, "error");
logFn.call(jsRuntime, jsi::String::createFromUtf8(jsRuntime, "Frame Processor threw an error: " + message));
auto logFn = jsRuntime.global().getPropertyAsObject(jsRuntime, "console").getPropertyAsFunction(jsRuntime, "error");
logFn.call(jsRuntime, jsi::String::createFromUtf8(jsRuntime, "Frame Processor threw an error: " + message));
});
}
}
@ -63,3 +64,5 @@ void JFrameProcessor::call(jni::alias_ref<JFrame::javaobject> frame) {
}
} // namespace vision
#endif

4
android/src/main/cpp/java-bindings/JFrameProcessor.h
View File

@ -4,6 +4,8 @@
#pragma once
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
#include <string>
#include <memory>
#include <jni.h>
@ -47,3 +49,5 @@ struct JFrameProcessor : public jni::HybridClass<JFrameProcessor> {
};
} // namespace vision
#endif

50
android/src/main/cpp/java-bindings/JVisionCameraProxy.cpp
View File

@ -11,10 +11,13 @@
#include <jsi/jsi.h>
#include <react/jni/ReadableNativeMap.h>
#include "FrameProcessorPluginHostObject.h"
#include "JSITypedArray.h"
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
#include <react-native-worklets-core/WKTJsiWorklet.h>
#include <react-native-worklets-core/WKTJsiWorkletContext.h>
#include "FrameProcessorPluginHostObject.h"
#endif
namespace vision {
@ -28,7 +31,9 @@ JVisionCameraProxy::JVisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::
const std::shared_ptr<facebook::react::CallInvoker>& callInvoker,
const jni::global_ref<JVisionCameraScheduler::javaobject>& scheduler) {
_javaPart = make_global(javaThis);
_runtime = runtime;
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
__android_log_write(ANDROID_LOG_INFO, TAG, "Creating Worklet Context...");
auto runOnJS = [callInvoker](std::function<void()>&& f) {
@ -46,14 +51,53 @@ JVisionCameraProxy::JVisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::
runOnJS,
runOnWorklet);
__android_log_write(ANDROID_LOG_INFO, TAG, "Worklet Context created!");
#else
__android_log_write(ANDROID_LOG_INFO, TAG, "Frame Processors are disabled!");
#endif
#ifdef VISION_CAMERA_ENABLE_SKIA
__android_log_write(ANDROID_LOG_INFO, TAG, "Skia is enabled!");
#else
__android_log_write(ANDROID_LOG_INFO, TAG, "Skia is disabled!");
#endif
}
JVisionCameraProxy::~JVisionCameraProxy() {
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
__android_log_write(ANDROID_LOG_INFO, TAG, "Destroying Context...");
// Destroy ArrayBuffer cache for both the JS and the Worklet Runtime.
invalidateArrayBufferCache(*_workletContext->getJsRuntime());
invalidateArrayBufferCache(_workletContext->getWorkletRuntime());
#endif
}
void JVisionCameraProxy::setFrameProcessor(int viewTag,
const alias_ref<JFrameProcessor::javaobject>& frameProcessor) {
jsi::Runtime& runtime,
const jsi::Object& frameProcessorObject) {
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
auto frameProcessorType = frameProcessorObject.getProperty(runtime, "type").asString(runtime).utf8(runtime);
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, frameProcessorObject.getProperty(runtime, "frameProcessor"));
jni::local_ref<JFrameProcessor::javaobject> frameProcessor;
if (frameProcessorType == "frame-processor") {
frameProcessor = JFrameProcessor::create(worklet, _workletContext);
} else if (frameProcessorType == "skia-frame-processor") {
#if VISION_CAMERA_ENABLE_SKIA
throw std::runtime_error("system/skia-unavailable: Skia is not yet implemented on Android!");
#else
throw std::runtime_error("system/skia-unavailable: Skia is not installed!");
#endif
} else {
throw std::runtime_error("Unknown FrameProcessor.type passed! Received: " + frameProcessorType);
}
auto setFrameProcessorMethod = javaClassLocal()->getMethod<void(int, alias_ref<JFrameProcessor::javaobject>)>("setFrameProcessor");
setFrameProcessorMethod(_javaPart, viewTag, frameProcessor);
#else
throw std::runtime_error("system/frame-processors-unavailable: Frame Processors are disabled!");
#endif
}
void JVisionCameraProxy::removeFrameProcessor(int viewTag) {

21
android/src/main/cpp/java-bindings/JVisionCameraProxy.h
View File

@ -6,8 +6,8 @@
#include <fbjni/fbjni.h>
#include <jsi/jsi.h>
#include <react-native-worklets-core/WKTJsiWorkletContext.h>
#include <react/jni/ReadableNativeMap.h>
#include <ReactCommon/CallInvokerHolder.h>
#include "JFrameProcessorPlugin.h"
#include "JVisionCameraScheduler.h"
@ -16,29 +16,36 @@
#include <string>
#include <memory>
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
#include <react-native-worklets-core/WKTJsiWorkletContext.h>
#endif
namespace vision {
using namespace facebook;
class JVisionCameraProxy : public jni::HybridClass<JVisionCameraProxy> {
public:
~JVisionCameraProxy();
static void registerNatives();
void setFrameProcessor(int viewTag,
const jni::alias_ref<JFrameProcessor::javaobject>& frameProcessor);
jsi::Runtime& runtime,
const jsi::Object& frameProcessor);
void removeFrameProcessor(int viewTag);
jni::local_ref<JFrameProcessorPlugin::javaobject> getFrameProcessorPlugin(const std::string& name,
jni::local_ref<react::ReadableNativeMap::javaobject> options);
public:
std::shared_ptr<RNWorklet::JsiWorkletContext> getWorkletContext() { return _workletContext; }
private:
std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;
jsi::Runtime* getJSRuntime() { return _runtime; }
private:
friend HybridBase;
jni::global_ref<JVisionCameraProxy::javaobject> _javaPart;
jsi::Runtime* _runtime;
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;
#endif
static auto constexpr TAG = "VisionCameraProxy";
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraProxy;";

4
android/src/main/cpp/skia/OpenGLError.h
View File

@ -4,6 +4,8 @@
#pragma once
#if VISION_CAMERA_ENABLE_SKIA
#include <string>
#include <stdexcept>
#include <GLES2/gl2.h>
@ -24,3 +26,5 @@ class OpenGLError: public std::runtime_error {
};
} // namespace vision
#endif

4
android/src/main/cpp/skia/SkiaRenderer.cpp
View File

@ -2,6 +2,8 @@
// Created by Marc Rousavy on 10.08.23.
//
#if VISION_CAMERA_ENABLE_SKIA
#include "SkiaRenderer.h"
#include <android/log.h>
#include "OpenGLError.h"
@ -325,3 +327,5 @@ void SkiaRenderer::registerNatives() {
}
} // namespace vision
#endif

4
android/src/main/cpp/skia/SkiaRenderer.h
View File

@ -4,6 +4,8 @@
#pragma once
#if VISION_CAMERA_ENABLE_SKIA
#include <jni.h>
#include <fbjni/fbjni.h>
#include <fbjni/ByteBuffer.h>
@ -75,3 +77,5 @@ class SkiaRenderer: public jni::HybridClass<SkiaRenderer> {
};
} // namespace vision
#endif

6
android/src/main/java/com/mrousavy/camera/extensions/CameraCharacteristics+getOutputSizes.kt
View File

@ -34,8 +34,10 @@ private fun getMaximumVideoSize(cameraId: String): Size? {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
val profiles = CamcorderProfile.getAll(cameraId, CamcorderProfile.QUALITY_HIGH)
if (profiles != null) {
val largestProfile = profiles.videoProfiles.maxBy { it.width * it.height }
return Size(largestProfile.width, largestProfile.height)
val largestProfile = profiles.videoProfiles.filterNotNull().maxByOrNull { it.width * it.height }
if (largestProfile != null) {
return Size(largestProfile.width, largestProfile.height)
}
}
}

2
docs/docs/guides/FRAME_PROCESSORS.mdx
View File

@ -235,7 +235,7 @@ The Frame Processor API spawns a secondary JavaScript Runtime which consumes a s
Inside your `gradle.properties` file, add the `disableFrameProcessors` flag:
```groovy
disableFrameProcessors=true
VisionCamera_disableFrameProcessors=true
```
Then, clean and rebuild your project.

2
docs/docs/guides/FRAME_PROCESSORS_SKIA.mdx
View File

@ -91,7 +91,7 @@ Skia Frame Processors ship with additional C++ files which might slightly increa
Inside your `gradle.properties` file, add the `disableSkia` flag:
```groovy
disableSkia=true
VisionCamera_disableSkia=true
```
Then, clean and rebuild your project.

4
docs/docs/guides/TROUBLESHOOTING.mdx
View File

@ -85,8 +85,8 @@ Before opening an issue, make sure you try the following:
```
distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-all.zip
```
7. Try building without Skia. Set `disableSkia = true` in your `gradle.properties`, and try rebuilding.
8. Try building without Frame Processors. Set `disableFrameProcessors = true` in your `gradle.properties`, and try rebuilding.
7. Try building without Skia. Set `VisionCamera_disableSkia = true` in your `gradle.properties`, and try rebuilding.
8. Try building without Frame Processors. Set `VisionCamera_disableFrameProcessors = true` in your `gradle.properties`, and try rebuilding.
### Runtime Issues

4
example/android/gradle.properties
View File

@ -38,3 +38,7 @@ newArchEnabled=false
# Use this property to enable or disable the Hermes JS engine.
# If set to false, you will be using JSC instead.
hermesEnabled=true
# Can be set to true to disable the build setup
#VisionCamera_disableFrameProcessors=true
#VisionCamera_disableSkia=true

2
ios/Skia Render Layer/SkiaRenderer.h
View File

@ -20,7 +20,7 @@ typedef void(^draw_callback_t)(SkiaCanvas _Nonnull);
It provides two Contexts, one offscreen and one onscreen.
- Offscreen Context: Allows you to render a Frame into a Skia Canvas and draw onto it using Skia commands
- Onscreen Context: Allows you to render a Frame from the offscreen context onto a Layer allowing it to be displayed for Preview.
The two contexts may run at different Frame Rates.
*/
@interface SkiaRenderer : NSObject

66
src/FrameProcessorPlugins.ts
View File

@ -1,9 +1,11 @@
import type { Frame, FrameInternal } from './Frame';
import type { FrameProcessor } from './CameraProps';
import { Camera } from './Camera';
import { Worklets } from 'react-native-worklets-core';
import { CameraRuntimeError } from './CameraError';
// only import typescript types
import type TWorklets from 'react-native-worklets-core';
type BasicParameterType = string | number | boolean | undefined;
type ParameterType = BasicParameterType | BasicParameterType[] | Record<string, BasicParameterType | undefined>;
@ -28,17 +30,48 @@ interface TVisionCameraProxy {
isSkiaEnabled: boolean;
}
Camera.installFrameProcessorBindings();
let isAsyncContextBusy = { value: false };
let runOnAsyncContext = (_frame: Frame, _func: () => void): void => {
throw new CameraRuntimeError(
'system/frame-processors-unavailable',
'Frame Processors are not available, react-native-worklets-core is not installed!',
);
};
try {
// eslint-disable-next-line @typescript-eslint/no-var-requires
const { Worklets } = require('react-native-worklets-core') as typeof TWorklets;
// Install native Frame Processor Runtime Manager
Camera.installFrameProcessorBindings();
// @ts-expect-error global is untyped, it's a C++ host-object
if (global.VisionCameraProxy == null) {
throw new CameraRuntimeError(
'system/frame-processors-unavailable',
'Failed to install VisionCameraProxy. Are Frame Processors properly enabled?',
);
}
isAsyncContextBusy = Worklets.createSharedValue(false);
const asyncContext = Worklets.createContext('VisionCamera.async');
runOnAsyncContext = Worklets.createRunInContextFn((frame: Frame, func: () => void) => {
'worklet';
try {
// Call long-running function
func();
} finally {
// Potentially delete Frame if we were the last ref
(frame as FrameInternal).decrementRefCount();
isAsyncContextBusy.value = false;
}
}, asyncContext);
} catch (e) {
// Worklets are not installed, so Frame Processors are disabled.
}
// @ts-expect-error global is untyped, it's a C++ host-object
export const VisionCameraProxy = global.VisionCameraProxy as TVisionCameraProxy;
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
if (VisionCameraProxy == null) {
throw new CameraRuntimeError(
'system/frame-processors-unavailable',
'Failed to install VisionCameraProxy. Are Frame Processors properly enabled?',
);
}
declare global {
// eslint-disable-next-line no-var
@ -96,21 +129,6 @@ export function runAtTargetFps<T>(fps: number, func: () => T): T | undefined {
return undefined;
}
const isAsyncContextBusy = Worklets.createSharedValue(false);
const asyncContext = Worklets.createContext('VisionCamera.async');
const runOnAsyncContext = Worklets.createRunInContextFn((frame: Frame, func: () => void) => {
'worklet';
try {
// Call long-running function
func();
} finally {
// Potentially delete Frame if we were the last ref
(frame as FrameInternal).decrementRefCount();
isAsyncContextBusy.value = false;
}
}, asyncContext);
/**
* Runs the given function asynchronously, while keeping a strong reference to the Frame.
*

4
src/expo-plugin/withDisableFrameProcessorsAndroid.ts
View File

@ -1,11 +1,11 @@
import { ConfigPlugin, withGradleProperties } from '@expo/config-plugins';
/**
* Set the `disableFrameProcessors` value in the static `gradle.properties` file.
* Set the `VisionCamera_disableFrameProcessors` value in the static `gradle.properties` file.
* This is used to disable frame processors if you don't need it for android.
*/
export const withDisableFrameProcessorsAndroid: ConfigPlugin = (c) => {
const disableFrameProcessorsKey = 'disableFrameProcessors';
const disableFrameProcessorsKey = 'VisionCamera_disableFrameProcessors';
return withGradleProperties(c, (config) => {
config.modResults = config.modResults.filter((item) => {
if (item.type === 'property' && item.key === disableFrameProcessorsKey) return false;

2
src/hooks/useFrameProcessor.ts
View File

@ -1,8 +1,6 @@
import { DependencyList, useMemo } from 'react';
import type { DrawableFrame, Frame, FrameInternal } from '../Frame';
import { FrameProcessor } from '../CameraProps';
// Install RN Worklets by importing it
import 'react-native-worklets-core';
export function createFrameProcessor(frameProcessor: FrameProcessor['frameProcessor'], type: FrameProcessor['type']): FrameProcessor {
return {