railbird/react-native-vision-camera
9
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Use clang-format to keep the C++ codebase clean (#1741)

Browse Source 
* Use clang-format

* Create .clang-format

* Update .clang-format

* Update .clang-format

* Update .clang-format

* Only search in cpp dirs

* Update clang-format.sh

* Update .clang-format

* Update .clang-format

* Update .clang-format

* Format C++ code!

* Use version 16

* Update clang-format.sh

* Remove Shaders.ts

* fix: Lint Swift
This commit is contained in:
Marc Rousavy 2023-09-01 12:58:32 +02:00 committed by GitHub
parent 0a28454579
commit 03b57a7d27
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
61 changed files with 851 additions and 886 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
.clang-format Normal file
View File

@ -0,0 +1,26 @@
# Config for clang-format version 16
# Standard
BasedOnStyle: llvm
Standard: c++14
# Indentation
IndentWidth: 2
ColumnLimit: 100
# Includes
SortIncludes: true
SortUsingDeclarations: true
# Pointer and reference alignment
PointerAlignment: Left
ReferenceAlignment: Left
ReflowComments: true
# Line breaking options
BreakBeforeBraces: Attach
BreakConstructorInitializers: BeforeColon
AllowShortFunctionsOnASingleLine: Empty
IndentCaseLabels: true
NamespaceIndentation: Inner

28
.github/workflows/validate-cpp.yml vendored
View File

@ -8,29 +8,29 @@ on:
- '.github/workflows/validate-cpp.yml'
- 'cpp/**'
- 'android/src/main/cpp/**'
- 'ios/**'
pull_request:
paths:
- '.github/workflows/validate-cpp.yml'
- 'cpp/**'
- 'android/src/main/cpp/**'
- 'ios/**'
jobs:
lint:
name: cpplint
name: Check clang-format
runs-on: ubuntu-latest
strategy:
matrix:
path:
- 'cpp'
- 'android/src/main/cpp'
- 'ios'
steps:
- uses: actions/checkout@v2
- uses: reviewdog/action-cpplint@master
- name: Run clang-format style check
uses: jidicula/clang-format-action@v4.11.0
with:
github_token: ${{ secrets.github_token }}
reporter: github-pr-review
flags: --linelength=230
targets: --recursive cpp android/src/main/cpp
filter: "-legal/copyright\
,-readability/todo\
,-build/namespaces\
,-whitespace/comments\
,-runtime/references\
,-build/include_order\
,-build/c++11\
"
clang-format-version: '16'
check-path: ${{ matrix.path }}

64
android/src/main/cpp/OpenGLContext.cpp
View File

@ -8,8 +8,8 @@
#include <GLES2/gl2.h>
#include <GLES2/gl2ext.h>
#include <android/native_window.h>
#include <android/log.h>
#include <android/native_window.h>
#include "OpenGLError.h"
@ -49,29 +49,40 @@ void OpenGLContext::ensureOpenGL() {
if (display == EGL_NO_DISPLAY) {
__android_log_print(ANDROID_LOG_INFO, TAG, "Initializing EGLDisplay..");
display = eglGetDisplay(EGL_DEFAULT_DISPLAY);
if (display == EGL_NO_DISPLAY) throw OpenGLError("Failed to get default OpenGL Display!");
if (display == EGL_NO_DISPLAY)
throw OpenGLError("Failed to get default OpenGL Display!");
EGLint major;
EGLint minor;
successful = eglInitialize(display, &major, &minor);
if (!successful) throw OpenGLError("Failed to initialize OpenGL!");
if (!successful)
throw OpenGLError("Failed to initialize OpenGL!");
}
// EGLConfig
if (config == nullptr) {
__android_log_print(ANDROID_LOG_INFO, TAG, "Initializing EGLConfig..");
EGLint attributes[] = {EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
EGL_SURFACE_TYPE, EGL_WINDOW_BIT,
EGL_RED_SIZE, 8,
EGL_GREEN_SIZE, 8,
EGL_BLUE_SIZE, 8,
EGL_ALPHA_SIZE, 8,
EGL_DEPTH_SIZE, 0,
EGL_STENCIL_SIZE, 0,
EGLint attributes[] = {EGL_RENDERABLE_TYPE,
EGL_OPENGL_ES2_BIT,
EGL_SURFACE_TYPE,
EGL_WINDOW_BIT,
EGL_RED_SIZE,
8,
EGL_GREEN_SIZE,
8,
EGL_BLUE_SIZE,
8,
EGL_ALPHA_SIZE,
8,
EGL_DEPTH_SIZE,
0,
EGL_STENCIL_SIZE,
0,
EGL_NONE};
EGLint numConfigs;
successful = eglChooseConfig(display, attributes, &config, 1, &numConfigs);
if (!successful || numConfigs == 0) throw OpenGLError("Failed to choose OpenGL config!");
if (!successful || numConfigs == 0)
throw OpenGLError("Failed to choose OpenGL config!");
}
// EGLContext
@ -79,18 +90,18 @@ void OpenGLContext::ensureOpenGL() {
__android_log_print(ANDROID_LOG_INFO, TAG, "Initializing EGLContext..");
EGLint contextAttributes[] = {EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE};
context = eglCreateContext(display, config, nullptr, contextAttributes);
if (context == EGL_NO_CONTEXT) throw OpenGLError("Failed to create OpenGL context!");
if (context == EGL_NO_CONTEXT)
throw OpenGLError("Failed to create OpenGL context!");
}
// EGLSurface
if (offscreenSurface == EGL_NO_SURFACE) {
// If we don't have a surface at all
__android_log_print(ANDROID_LOG_INFO, TAG, "Initializing 1x1 offscreen pbuffer EGLSurface..");
EGLint attributes[] = {EGL_WIDTH, 1,
EGL_HEIGHT, 1,
EGL_NONE};
EGLint attributes[] = {EGL_WIDTH, 1, EGL_HEIGHT, 1, EGL_NONE};
offscreenSurface = eglCreatePbufferSurface(display, config, attributes);
if (offscreenSurface == EGL_NO_SURFACE) throw OpenGLError("Failed to create OpenGL Surface!");
if (offscreenSurface == EGL_NO_SURFACE)
throw OpenGLError("Failed to create OpenGL Surface!");
}
}
@ -99,21 +110,24 @@ void OpenGLContext::use() {
}
void OpenGLContext::use(EGLSurface surface) {
if (surface == EGL_NO_SURFACE) throw OpenGLError("Cannot render to a null Surface!");
if (surface == EGL_NO_SURFACE)
throw OpenGLError("Cannot render to a null Surface!");
// 1. Make sure the OpenGL context is initialized
this->ensureOpenGL();
// 2. Make the OpenGL context current
bool successful = eglMakeCurrent(display, surface, surface, context);
if (!successful || eglGetError() != EGL_SUCCESS) throw OpenGLError("Failed to use current OpenGL context!");
if (!successful || eglGetError() != EGL_SUCCESS)
throw OpenGLError("Failed to use current OpenGL context!");
// 3. Caller can now render to this surface
}
void OpenGLContext::flush() const {
bool successful = eglSwapBuffers(display, eglGetCurrentSurface(EGL_DRAW));
if (!successful || eglGetError() != EGL_SUCCESS) throw OpenGLError("Failed to swap OpenGL buffers!");
if (!successful || eglGetError() != EGL_SUCCESS)
throw OpenGLError("Failed to swap OpenGL buffers!");
}
OpenGLTexture OpenGLContext::createTexture(OpenGLTexture::Type type, int width, int height) {
@ -122,7 +136,8 @@ OpenGLTexture OpenGLContext::createTexture(OpenGLTexture::Type type, int width,
// 2. Make the OpenGL context current
bool successful = eglMakeCurrent(display, offscreenSurface, offscreenSurface, context);
if (!successful || eglGetError() != EGL_SUCCESS) throw OpenGLError("Failed to use current OpenGL context!");
if (!successful || eglGetError() != EGL_SUCCESS)
throw OpenGLError("Failed to use current OpenGL context!");
GLuint textureId;
glGenTextures(1, &textureId);
@ -142,12 +157,7 @@ OpenGLTexture OpenGLContext::createTexture(OpenGLTexture::Type type, int width,
glTexParameteri(target, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(target, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
return {
.id = textureId,
.target = target,
.width = width,
.height = height
};
return {.id = textureId, .target = target, .width = width, .height = height};
}
} // namespace vision

4
android/src/main/cpp/OpenGLContext.h
View File

@ -7,11 +7,11 @@
#include <EGL/egl.h>
#include <GLES2/gl2.h>
#include <memory>
#include <functional>
#include <memory>
#include "PassThroughShader.h"
#include "OpenGLTexture.h"
#include "PassThroughShader.h"
namespace vision {

16
android/src/main/cpp/OpenGLError.h
View File

@ -4,27 +4,31 @@
#pragma once
#include <string>
#include <stdexcept>
#include <GLES2/gl2.h>
#include <stdexcept>
#include <string>
namespace vision {
inline std::string getEglErrorIfAny() {
EGLint error = glGetError();
if (error != GL_NO_ERROR) return " Error: " + std::to_string(error);
if (error != GL_NO_ERROR)
return " Error: " + std::to_string(error);
error = eglGetError();
if (error != EGL_SUCCESS) return " Error: " + std::to_string(error);
if (error != EGL_SUCCESS)
return " Error: " + std::to_string(error);
return "";
}
class OpenGLError : public std::runtime_error {
public:
explicit OpenGLError(const std::string&& message): std::runtime_error(message + getEglErrorIfAny()) {}
explicit OpenGLError(const std::string&& message)
: std::runtime_error(message + getEglErrorIfAny()) {}
static inline void checkIfError(const std::string&& message) {
auto error = getEglErrorIfAny();
if (error.length() > 0) throw std::runtime_error(message + error);
if (error.length() > 0)
throw std::runtime_error(message + error);
}
};

6
android/src/main/cpp/OpenGLRenderer.cpp
View File

@ -8,8 +8,8 @@
#include <GLES2/gl2.h>
#include <GLES2/gl2ext.h>
#include <android/native_window.h>
#include <android/log.h>
#include <android/native_window.h>
#include <utility>
@ -17,7 +17,9 @@
namespace vision {
std::unique_ptr<OpenGLRenderer> OpenGLRenderer::CreateWithWindowSurface(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface) {
std::unique_ptr<OpenGLRenderer>
OpenGLRenderer::CreateWithWindowSurface(std::shared_ptr<OpenGLContext> context,
ANativeWindow* surface) {
return std::unique_ptr<OpenGLRenderer>(new OpenGLRenderer(std::move(context), surface));
}

13
android/src/main/cpp/OpenGLRenderer.h
View File

@ -4,11 +4,11 @@
#pragma once
#include "PassThroughShader.h"
#include <EGL/egl.h>
#include <GLES2/gl2.h>
#include <android/native_window.h>
#include <memory>
#include "PassThroughShader.h"
#include "OpenGLContext.h"
#include "OpenGLTexture.h"
@ -21,9 +21,11 @@ class OpenGLRenderer {
* Create a new instance of the OpenGLRenderer that draws to an on-screen window surface.
* This will not perform any OpenGL operations yet, and is therefore safe to call from any Thread.
*
* Note: The `surface` is considered moved, and the OpenGL context will release it when it is being deleted.
* Note: The `surface` is considered moved, and the OpenGL context will release it when it is
* being deleted.
*/
static std::unique_ptr<OpenGLRenderer> CreateWithWindowSurface(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface);
static std::unique_ptr<OpenGLRenderer>
CreateWithWindowSurface(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface);
/**
* Destroy the OpenGL Context. This needs to be called on the same thread that `use()` was called.
*/
@ -35,8 +37,9 @@ class OpenGLRenderer {
void renderTextureToSurface(const OpenGLTexture& texture, float* transformMatrix);
/**
* Destroys the OpenGL context. This needs to be called on the same thread that `use()` was called.
* After calling `destroy()`, it is legal to call `use()` again, which will re-construct everything.
* Destroys the OpenGL context. This needs to be called on the same thread that `use()` was
* called. After calling `destroy()`, it is legal to call `use()` again, which will re-construct
* everything.
*/
void destroy();

20
android/src/main/cpp/PassThroughShader.cpp
View File

@ -3,11 +3,11 @@
//
#include "PassThroughShader.h"
#include "OpenGLError.h"
#include <EGL/egl.h>
#include <GLES2/gl2.h>
#include <GLES2/gl2ext.h>
#include <memory>
#include "OpenGLError.h"
#include <string>
namespace vision {
@ -50,19 +50,11 @@ void PassThroughShader::draw(const OpenGLTexture& texture, float* transformMatri
// 3. Pass all uniforms/attributes for vertex shader
glEnableVertexAttribArray(_vertexParameters.aPosition);
glVertexAttribPointer(_vertexParameters.aPosition,
2,
GL_FLOAT,
GL_FALSE,
sizeof(Vertex),
glVertexAttribPointer(_vertexParameters.aPosition, 2, GL_FLOAT, GL_FALSE, sizeof(Vertex),
reinterpret_cast<void*>(offsetof(Vertex, position)));
glEnableVertexAttribArray(_vertexParameters.aTexCoord);
glVertexAttribPointer(_vertexParameters.aTexCoord,
2,
GL_FLOAT,
GL_FALSE,
sizeof(Vertex),
glVertexAttribPointer(_vertexParameters.aTexCoord, 2, GL_FLOAT, GL_FALSE, sizeof(Vertex),
reinterpret_cast<void*>(offsetof(Vertex, texCoord)));
glUniformMatrix4fv(_vertexParameters.uTransformMatrix, 1, GL_FALSE, transformMatrix);
@ -78,7 +70,8 @@ void PassThroughShader::draw(const OpenGLTexture& texture, float* transformMatri
GLuint PassThroughShader::loadShader(GLenum shaderType, const char* shaderCode) {
GLuint shader = glCreateShader(shaderType);
if (shader == 0) throw OpenGLError("Failed to load shader!");
if (shader == 0)
throw OpenGLError("Failed to load shader!");
glShaderSource(shader, 1, &shaderCode, nullptr);
glCompileShader(shader);
@ -96,7 +89,8 @@ GLuint PassThroughShader::createProgram() {
GLuint fragmentShader = loadShader(GL_FRAGMENT_SHADER, FRAGMENT_SHADER);
GLuint program = glCreateProgram();
if (program == 0) throw OpenGLError("Failed to create pass-through program!");
if (program == 0)
throw OpenGLError("Failed to create pass-through program!");
glAttachShader(program, vertexShader);
OpenGLError::checkIfError("Failed to attach Vertex Shader!");

32
android/src/main/cpp/VideoPipeline.cpp
View File

@ -5,25 +5,27 @@
#include "VideoPipeline.h"
#include "OpenGLError.h"
#include <android/native_window_jni.h>
#include <GLES2/gl2.h>
#include <GLES2/gl2ext.h>
#include <EGL/egl.h>
#include <EGL/eglext.h>
#include <GLES/gl.h>
#include <GLES2/gl2.h>
#include <GLES2/gl2ext.h>
#include <android/native_window_jni.h>
#include <chrono>
#include "OpenGLTexture.h"
#include "JFrameProcessor.h"
#include "OpenGLTexture.h"
namespace vision {
jni::local_ref<VideoPipeline::jhybriddata> VideoPipeline::initHybrid(jni::alias_ref<jhybridobject> jThis, int width, int height) {
jni::local_ref<VideoPipeline::jhybriddata>
VideoPipeline::initHybrid(jni::alias_ref<jhybridobject> jThis, int width, int height) {
return makeCxxInstance(jThis, width, height);
}
VideoPipeline::VideoPipeline(jni::alias_ref<jhybridobject> jThis, int width, int height): _javaPart(jni::make_global(jThis)) {
VideoPipeline::VideoPipeline(jni::alias_ref<jhybridobject> jThis, int width, int height)
: _javaPart(jni::make_global(jThis)) {
_width = width;
_height = height;
_context = OpenGLContext::CreateWithOffscreenSurface();
@ -43,7 +45,8 @@ VideoPipeline::~VideoPipeline() {
}
void VideoPipeline::removeFrameProcessorOutputSurface() {
if (_frameProcessorOutput) _frameProcessorOutput->destroy();
if (_frameProcessorOutput)
_frameProcessorOutput->destroy();
_frameProcessorOutput = nullptr;
}
@ -57,7 +60,8 @@ void VideoPipeline::setFrameProcessorOutputSurface(jobject surface) {
}
void VideoPipeline::removeRecordingSessionOutputSurface() {
if (_recordingSessionOutput) _recordingSessionOutput->destroy();
if (_recordingSessionOutput)
_recordingSessionOutput->destroy();
_recordingSessionOutput = nullptr;
}
@ -104,10 +108,14 @@ void VideoPipeline::onFrame(jni::alias_ref<jni::JArrayFloat> transformMatrixPara
void VideoPipeline::registerNatives() {
registerHybrid({
makeNativeMethod("initHybrid", VideoPipeline::initHybrid),
makeNativeMethod("setFrameProcessorOutputSurface", VideoPipeline::setFrameProcessorOutputSurface),
makeNativeMethod("removeFrameProcessorOutputSurface", VideoPipeline::removeFrameProcessorOutputSurface),
makeNativeMethod("setRecordingSessionOutputSurface", VideoPipeline::setRecordingSessionOutputSurface),
makeNativeMethod("removeRecordingSessionOutputSurface", VideoPipeline::removeRecordingSessionOutputSurface),
makeNativeMethod("setFrameProcessorOutputSurface",
VideoPipeline::setFrameProcessorOutputSurface),
makeNativeMethod("removeFrameProcessorOutputSurface",
VideoPipeline::removeFrameProcessorOutputSurface),
makeNativeMethod("setRecordingSessionOutputSurface",
VideoPipeline::setRecordingSessionOutputSurface),
makeNativeMethod("removeRecordingSessionOutputSurface",
VideoPipeline::removeRecordingSessionOutputSurface),
makeNativeMethod("getInputTextureId", VideoPipeline::getInputTextureId),
makeNativeMethod("onBeforeFrame", VideoPipeline::onBeforeFrame),
makeNativeMethod("onFrame", VideoPipeline::onFrame),

13
android/src/main/cpp/VideoPipeline.h
View File

@ -4,13 +4,13 @@
#pragma once
#include <jni.h>
#include <fbjni/fbjni.h>
#include "OpenGLContext.h"
#include "OpenGLRenderer.h"
#include "PassThroughShader.h"
#include <EGL/egl.h>
#include <android/native_window.h>
#include "PassThroughShader.h"
#include "OpenGLRenderer.h"
#include "OpenGLContext.h"
#include <fbjni/fbjni.h>
#include <jni.h>
#include <memory>
#include <optional>
@ -21,7 +21,8 @@ using namespace facebook;
class VideoPipeline : public jni::HybridClass<VideoPipeline> {
public:
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/utils/VideoPipeline;";
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis, int width, int height);
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis, int width,
int height);
static void registerNatives();
public:

8
android/src/main/cpp/VisionCamera.cpp
View File

@ -1,10 +1,10 @@
#include <jni.h>
#include <fbjni/fbjni.h>
#include "JVisionCameraScheduler.h"
#include "JFrameProcessor.h"
#include "JVisionCameraProxy.h"
#include "VisionCameraProxy.h"
#include "JVisionCameraScheduler.h"
#include "VideoPipeline.h"
#include "VisionCameraProxy.h"
#include <fbjni/fbjni.h>
#include <jni.h>
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void*) {
return facebook::jni::initialize(vm, [] {

53
android/src/main/cpp/frameprocessor/FrameHostObject.cpp
View File

@ -9,14 +9,15 @@
#include "JSITypedArray.h"
#include <vector>
#include <string>
#include <vector>
namespace vision {
using namespace facebook;
FrameHostObject::FrameHostObject(const jni::alias_ref<JFrame::javaobject>& frame): frame(make_global(frame)) { }
FrameHostObject::FrameHostObject(const jni::alias_ref<JFrame::javaobject>& frame)
: frame(make_global(frame)) {}
FrameHostObject::~FrameHostObject() {
// Hermes' Garbage Collector (Hades GC) calls destructors on a separate Thread
@ -50,38 +51,29 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
auto name = propName.utf8(runtime);
if (name == "incrementRefCount") {
jsi::HostFunctionType incrementRefCount = [=](jsi::Runtime& runtime,
const jsi::Value& thisArg,
jsi::HostFunctionType incrementRefCount = [=](jsi::Runtime& runtime, const jsi::Value& thisArg,
const jsi::Value* args,
size_t count) -> jsi::Value {
// Increment retain count by one.
this->frame->incrementRefCount();
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "incrementRefCount"),
0,
incrementRefCount);
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "incrementRefCount"), 0, incrementRefCount);
}
if (name == "decrementRefCount") {
auto decrementRefCount = [=](jsi::Runtime& runtime,
const jsi::Value& thisArg,
const jsi::Value* args,
size_t count) -> jsi::Value {
auto decrementRefCount = [=](jsi::Runtime& runtime, const jsi::Value& thisArg,
const jsi::Value* args, size_t count) -> jsi::Value {
// Decrement retain count by one. If the retain count is zero, the Frame gets closed.
this->frame->decrementRefCount();
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "decrementRefCount"),
0,
decrementRefCount);
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "decrementRefCount"), 0, decrementRefCount);
}
if (name == "toString") {
jsi::HostFunctionType toString = [=](jsi::Runtime& runtime,
const jsi::Value& thisArg,
const jsi::Value* args,
size_t count) -> jsi::Value {
jsi::HostFunctionType toString = [=](jsi::Runtime& runtime, const jsi::Value& thisArg,
const jsi::Value* args, size_t count) -> jsi::Value {
if (!this->frame) {
return jsi::String::createFromUtf8(runtime, "[closed frame]");
}
@ -90,16 +82,16 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
auto str = std::to_string(width) + " x " + std::to_string(height) + " Frame";
return jsi::String::createFromUtf8(runtime, str);
};
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
}
if (name == "toArrayBuffer") {
jsi::HostFunctionType toArrayBuffer = [=](jsi::Runtime& runtime,
const jsi::Value& thisArg,
const jsi::Value* args,
size_t count) -> jsi::Value {
jsi::HostFunctionType toArrayBuffer = [=](jsi::Runtime& runtime, const jsi::Value& thisArg,
const jsi::Value* args, size_t count) -> jsi::Value {
auto buffer = this->frame->toByteBuffer();
if (!buffer->isDirect()) {
throw std::runtime_error("Failed to get byte content of Frame - array is not direct ByteBuffer!");
throw std::runtime_error(
"Failed to get byte content of Frame - array is not direct ByteBuffer!");
}
auto size = buffer->getDirectSize();
@ -110,8 +102,10 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
}
// Get from global JS cache
auto arrayBufferCache = runtime.global().getPropertyAsObject(runtime, ARRAYBUFFER_CACHE_PROP_NAME);
auto arrayBuffer = vision::getTypedArray(runtime, arrayBufferCache).get<vision::TypedArrayKind::Uint8ClampedArray>(runtime);
auto arrayBufferCache =
runtime.global().getPropertyAsObject(runtime, ARRAYBUFFER_CACHE_PROP_NAME);
auto arrayBuffer = vision::getTypedArray(runtime, arrayBufferCache)
.get<vision::TypedArrayKind::Uint8ClampedArray>(runtime);
if (arrayBuffer.size(runtime) != size) {
arrayBuffer = vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray>(runtime, size);
runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer);
@ -123,7 +117,8 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
return arrayBuffer;
};
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toArrayBuffer"), 0, toArrayBuffer);
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "toArrayBuffer"), 0, toArrayBuffer);
}
if (name == "isValid") {

6
android/src/main/cpp/frameprocessor/FrameHostObject.h
View File

@ -4,11 +4,11 @@
#pragma once
#include <jsi/jsi.h>
#include <jni.h>
#include <fbjni/fbjni.h>
#include <vector>
#include <jni.h>
#include <jsi/jsi.h>
#include <string>
#include <vector>
#include "JFrame.h"

23
android/src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.cpp
View File

@ -3,40 +3,41 @@
//
#include "FrameProcessorPluginHostObject.h"
#include <vector>
#include "FrameHostObject.h"
#include "JSIJNIConversion.h"
#include <string>
#include <vector>
namespace vision {
using namespace facebook;
std::vector<jsi::PropNameID> FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime& runtime) {
std::vector<jsi::PropNameID>
FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime& runtime) {
std::vector<jsi::PropNameID> result;
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("call")));
return result;
}
jsi::Value FrameProcessorPluginHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
jsi::Value FrameProcessorPluginHostObject::get(jsi::Runtime& runtime,
const jsi::PropNameID& propName) {
auto name = propName.utf8(runtime);
if (name == "call") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "call"),
2,
[=](jsi::Runtime &runtime,
const jsi::Value &thisValue,
const jsi::Value *arguments,
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "call"), 2,
[=](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
size_t count) -> jsi::Value {
// Frame is first argument
auto frameHostObject = arguments[0].asObject(runtime).asHostObject<FrameHostObject>(runtime);
auto frameHostObject =
arguments[0].asObject(runtime).asHostObject<FrameHostObject>(runtime);
auto frame = frameHostObject->frame;
// Options are second argument (possibly undefined)
local_ref<JMap<jstring, jobject>> options = nullptr;
if (count > 1) {
options = JSIJNIConversion::convertJSIObjectToJNIMap(runtime, arguments[1].asObject(runtime));
options =
JSIJNIConversion::convertJSIObjectToJNIMap(runtime, arguments[1].asObject(runtime));
}
// Call actual plugin

8
android/src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.h
View File

@ -4,10 +4,10 @@
#pragma once
#include <jsi/jsi.h>
#include "JFrameProcessorPlugin.h"
#include <memory>
#include <fbjni/fbjni.h>
#include <jsi/jsi.h>
#include <memory>
#include <vector>
namespace vision {
@ -16,8 +16,8 @@ using namespace facebook;
class FrameProcessorPluginHostObject : public jsi::HostObject {
public:
explicit FrameProcessorPluginHostObject(jni::alias_ref<JFrameProcessorPlugin::javaobject> plugin):
_plugin(make_global(plugin)) { }
explicit FrameProcessorPluginHostObject(jni::alias_ref<JFrameProcessorPlugin::javaobject> plugin)
: _plugin(make_global(plugin)) {}
~FrameProcessorPluginHostObject() {}
public:

38
android/src/main/cpp/frameprocessor/JSIJNIConversion.cpp
View File

@ -4,14 +4,14 @@
#include "JSIJNIConversion.h"
#include <jsi/jsi.h>
#include <jni.h>
#include <fbjni/fbjni.h>
#include <android/log.h>
#include <fbjni/fbjni.h>
#include <jni.h>
#include <jsi/jsi.h>
#include <memory>
#include <string>
#include <utility>
#include <memory>
#include "FrameHostObject.h"
#include "JFrame.h"
@ -20,7 +20,8 @@ namespace vision {
using namespace facebook;
jni::local_ref<jni::JMap<jstring, jobject>> JSIJNIConversion::convertJSIObjectToJNIMap(jsi::Runtime& runtime, const jsi::Object& object) {
jni::local_ref<jni::JMap<jstring, jobject>>
JSIJNIConversion::convertJSIObjectToJNIMap(jsi::Runtime& runtime, const jsi::Object& object) {
auto propertyNames = object.getPropertyNames(runtime);
auto size = propertyNames.size(runtime);
auto hashMap = jni::JHashMap<jstring, jobject>::create();
@ -34,25 +35,21 @@ jni::local_ref<jni::JMap<jstring, jobject>> JSIJNIConversion::convertJSIObjectTo
// null
hashMap->put(key, nullptr);
} else if (value.isBool()) {
// Boolean
auto boolean = value.getBool();
hashMap->put(key, jni::JBoolean::valueOf(boolean));
} else if (value.isNumber()) {
// Double
auto number = value.getNumber();
hashMap->put(key, jni::JDouble::valueOf(number));
} else if (value.isString()) {
// String
auto str = value.getString(runtime).utf8(runtime);
hashMap->put(key, jni::make_jstring(str));
} else if (value.isObject()) {
// Object
@ -60,7 +57,6 @@ jni::local_ref<jni::JMap<jstring, jobject>> JSIJNIConversion::convertJSIObjectTo
if (valueAsObject.isArray(runtime)) {
// List<Object>
} else if (valueAsObject.isHostObject(runtime)) {
throw std::runtime_error("You can't pass HostObjects here.");
} else {
@ -69,48 +65,47 @@ jni::local_ref<jni::JMap<jstring, jobject>> JSIJNIConversion::convertJSIObjectTo
auto map = convertJSIObjectToJNIMap(runtime, valueAsObject);
hashMap->put(key, map);
}
} else {
auto stringRepresentation = value.toString(runtime).utf8(runtime);
throw std::runtime_error("Failed to convert jsi::Value to JNI value - unsupported type!" + stringRepresentation);
throw std::runtime_error("Failed to convert jsi::Value to JNI value - unsupported type!" +
stringRepresentation);
}
}
return hashMap;
}
jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, const jni::local_ref<jobject>& object) {
jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime& runtime,
const jni::local_ref<jobject>& object) {
if (object == nullptr) {
// null
return jsi::Value::undefined();
} else if (object->isInstanceOf(jni::JBoolean::javaClassStatic())) {
// Boolean
static const auto getBooleanFunc = jni::findClassLocal("java/lang/Boolean")->getMethod<jboolean()>("booleanValue");
static const auto getBooleanFunc =
jni::findClassLocal("java/lang/Boolean")->getMethod<jboolean()>("booleanValue");
auto boolean = getBooleanFunc(object.get());
return jsi::Value(boolean == true);
} else if (object->isInstanceOf(jni::JDouble::javaClassStatic())) {
// Double
static const auto getDoubleFunc = jni::findClassLocal("java/lang/Double")->getMethod<jdouble()>("doubleValue");
static const auto getDoubleFunc =
jni::findClassLocal("java/lang/Double")->getMethod<jdouble()>("doubleValue");
auto d = getDoubleFunc(object.get());
return jsi::Value(d);
} else if (object->isInstanceOf(jni::JInteger::javaClassStatic())) {
// Integer
static const auto getIntegerFunc = jni::findClassLocal("java/lang/Integer")->getMethod<jint()>("intValue");
static const auto getIntegerFunc =
jni::findClassLocal("java/lang/Integer")->getMethod<jint()>("intValue");
auto i = getIntegerFunc(object.get());
return jsi::Value(i);
} else if (object->isInstanceOf(jni::JString::javaClassStatic())) {
// String
return jsi::String::createFromUtf8(runtime, object->toString());
} else if (object->isInstanceOf(JList<jobject>::javaClassStatic())) {
// List<E>
@ -124,7 +119,6 @@ jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, c
i++;
}
return result;
} else if (object->isInstanceOf(JMap<jstring, jobject>::javaClassStatic())) {
// Map<K, V>

10
android/src/main/cpp/frameprocessor/JSIJNIConversion.h
View File

@ -4,9 +4,9 @@
#pragma once
#include <jsi/jsi.h>
#include <jni.h>
#include <fbjni/fbjni.h>
#include <jni.h>
#include <jsi/jsi.h>
namespace vision {
@ -14,9 +14,11 @@ namespace JSIJNIConversion {
using namespace facebook;
jni::local_ref<jni::JMap<jstring, jobject>> convertJSIObjectToJNIMap(jsi::Runtime& runtime, const jsi::Object& object);
jni::local_ref<jni::JMap<jstring, jobject>> convertJSIObjectToJNIMap(jsi::Runtime& runtime,
const jsi::Object& object);
jsi::Value convertJNIObjectToJSIValue(jsi::Runtime& runtime, const jni::local_ref<jobject>& object);
jsi::Value convertJNIObjectToJSIValue(jsi::Runtime& runtime,
const jni::local_ref<jobject>& object);
} // namespace JSIJNIConversion

46
android/src/main/cpp/frameprocessor/VisionCameraProxy.cpp
View File

@ -12,12 +12,12 @@
#include <android/log.h>
#include <fbjni/fbjni.h>
#include "JSITypedArray.h"
#include "FrameProcessorPluginHostObject.h"
#include "JSITypedArray.h"
#include <vector>
#include <string>
#include <memory>
#include <string>
#include <vector>
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
#include <react-native-worklets-core/WKTJsiWorkletContext.h>
@ -27,12 +27,12 @@ namespace vision {
using namespace facebook;
VisionCameraProxy::VisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::javaobject>& javaProxy) {
VisionCameraProxy::VisionCameraProxy(
const jni::alias_ref<JVisionCameraProxy::javaobject>& javaProxy) {
_javaProxy = make_global(javaProxy);
}
VisionCameraProxy::~VisionCameraProxy() {
}
VisionCameraProxy::~VisionCameraProxy() {}
std::vector<jsi::PropNameID> VisionCameraProxy::getPropertyNames(jsi::Runtime& runtime) {
std::vector<jsi::PropNameID> result;
@ -42,7 +42,8 @@ std::vector<jsi::PropNameID> VisionCameraProxy::getPropertyNames(jsi::Runtime& r
return result;
}
void VisionCameraProxy::setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& object) {
void VisionCameraProxy::setFrameProcessor(int viewTag, jsi::Runtime& runtime,
const jsi::Object& object) {
_javaProxy->cthis()->setFrameProcessor(viewTag, runtime, object);
}
@ -65,12 +66,9 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID&
auto name = propName.utf8(runtime);
if (name == "setFrameProcessor") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "setFrameProcessor"),
1,
[this](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "setFrameProcessor"), 1,
[this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
size_t count) -> jsi::Value {
auto viewTag = arguments[0].asNumber();
auto object = arguments[1].asObject(runtime);
@ -79,12 +77,9 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID&
});
}
if (name == "removeFrameProcessor") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "removeFrameProcessor"),
1,
[this](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "removeFrameProcessor"), 1,
[this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
size_t count) -> jsi::Value {
auto viewTag = arguments[0].asNumber();
this->removeFrameProcessor(static_cast<int>(viewTag));
@ -92,12 +87,9 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID&
});
}
if (name == "getFrameProcessorPlugin") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "getFrameProcessorPlugin"),
1,
[this](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "getFrameProcessorPlugin"), 1,
[this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
size_t count) -> jsi::Value {
if (count < 1 || !arguments[0].isString()) {
throw jsi::JSError(runtime, "First argument needs to be a string (pluginName)!");
@ -112,14 +104,12 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID&
return jsi::Value::undefined();
}
void VisionCameraInstaller::install(jni::alias_ref<jni::JClass>,
jni::alias_ref<JVisionCameraProxy::javaobject> proxy) {
// global.VisionCameraProxy
auto visionCameraProxy = std::make_shared<VisionCameraProxy>(proxy);
jsi::Runtime& runtime = *proxy->cthis()->getJSRuntime();
runtime.global().setProperty(runtime,
"VisionCameraProxy",
runtime.global().setProperty(runtime, "VisionCameraProxy",
jsi::Object::createFromHostObject(runtime, visionCameraProxy));
}

16
android/src/main/cpp/frameprocessor/VisionCameraProxy.h
View File

@ -6,11 +6,11 @@
#include <jsi/jsi.h>
#include "JVisionCameraScheduler.h"
#include "JVisionCameraProxy.h"
#include "JVisionCameraScheduler.h"
#include <vector>
#include <string>
#include <vector>
namespace vision {
@ -28,21 +28,21 @@ class VisionCameraProxy: public jsi::HostObject {
private:
void setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& frameProcessor);
void removeFrameProcessor(int viewTag);
jsi::Value getFrameProcessorPlugin(jsi::Runtime& runtime, const std::string& name, const jsi::Object& options);
jsi::Value getFrameProcessorPlugin(jsi::Runtime& runtime, const std::string& name,
const jsi::Object& options);
private:
jni::global_ref<JVisionCameraProxy::javaobject> _javaProxy;
static constexpr const char* TAG = "VisionCameraProxy";
};
class VisionCameraInstaller : public jni::JavaClass<VisionCameraInstaller> {
public:
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraInstaller;";
static auto constexpr kJavaDescriptor =
"Lcom/mrousavy/camera/frameprocessor/VisionCameraInstaller;";
static void registerNatives() {
javaClassStatic()->registerNatives({
makeNativeMethod("install", VisionCameraInstaller::install)
});
javaClassStatic()->registerNatives(
{makeNativeMethod("install", VisionCameraInstaller::install)});
}
static void install(jni::alias_ref<jni::JClass> clazz,
jni::alias_ref<JVisionCameraProxy::javaobject> proxy);

4
android/src/main/cpp/frameprocessor/java-bindings/JFrame.cpp
View File

@ -4,9 +4,9 @@
#include "JFrame.h"
#include <jni.h>
#include <fbjni/fbjni.h>
#include <fbjni/ByteBuffer.h>
#include <fbjni/fbjni.h>
#include <jni.h>
namespace vision {

4
android/src/main/cpp/frameprocessor/java-bindings/JFrame.h
View File

@ -4,9 +4,9 @@
#pragma once
#include <jni.h>
#include <fbjni/fbjni.h>
#include <fbjni/ByteBuffer.h>
#include <fbjni/fbjni.h>
#include <jni.h>
namespace vision {

18
android/src/main/cpp/frameprocessor/java-bindings/JFrameProcessor.cpp
View File

@ -5,11 +5,11 @@
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
#include "JFrameProcessor.h"
#include <jni.h>
#include <fbjni/fbjni.h>
#include <jni.h>
#include <utility>
#include "JFrame.h"
#include <utility>
namespace vision {
@ -17,9 +17,7 @@ using namespace facebook;
using namespace jni;
void JFrameProcessor::registerNatives() {
registerHybrid({
makeNativeMethod("call", JFrameProcessor::call)
});
registerHybrid({makeNativeMethod("call", JFrameProcessor::call)});
}
using TSelf = jni::local_ref<JFrameProcessor::javaobject>;
@ -35,7 +33,8 @@ TSelf JFrameProcessor::create(const std::shared_ptr<RNWorklet::JsiWorklet>& work
return JFrameProcessor::newObjectCxxArgs(worklet, context);
}
void JFrameProcessor::callWithFrameHostObject(const std::shared_ptr<FrameHostObject>& frameHostObject) const {
void JFrameProcessor::callWithFrameHostObject(
const std::shared_ptr<FrameHostObject>& frameHostObject) const {
// Call the Frame Processor on the Worklet Runtime
jsi::Runtime& runtime = _workletContext->getWorkletRuntime();
@ -51,8 +50,11 @@ void JFrameProcessor::callWithFrameHostObject(const std::shared_ptr<FrameHostObj
const std::string& message = jsError.getMessage();
_workletContext->invokeOnJsThread([message](jsi::Runtime& jsRuntime) {
auto logFn = jsRuntime.global().getPropertyAsObject(jsRuntime, "console").getPropertyAsFunction(jsRuntime, "error");
logFn.call(jsRuntime, jsi::String::createFromUtf8(jsRuntime, "Frame Processor threw an error: " + message));
auto logFn = jsRuntime.global()
.getPropertyAsObject(jsRuntime, "console")
.getPropertyAsFunction(jsRuntime, "error");
logFn.call(jsRuntime, jsi::String::createFromUtf8(
jsRuntime, "Frame Processor threw an error: " + message));
});
}
}

13
android/src/main/cpp/frameprocessor/java-bindings/JFrameProcessor.h
View File

@ -6,16 +6,16 @@
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
#include <string>
#include <memory>
#include <jni.h>
#include <fbjni/fbjni.h>
#include <jni.h>
#include <memory>
#include <string>
#include <react-native-worklets-core/WKTJsiWorklet.h>
#include <react-native-worklets-core/WKTJsiHostObject.h>
#include <react-native-worklets-core/WKTJsiWorklet.h>
#include "JFrame.h"
#include "FrameHostObject.h"
#include "JFrame.h"
namespace vision {
@ -25,7 +25,8 @@ struct JFrameProcessor : public jni::HybridClass<JFrameProcessor> {
public:
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessor;";
static void registerNatives();
static jni::local_ref<JFrameProcessor::javaobject> create(const std::shared_ptr<RNWorklet::JsiWorklet>& worklet,
static jni::local_ref<JFrameProcessor::javaobject>
create(const std::shared_ptr<RNWorklet::JsiWorklet>& worklet,
const std::shared_ptr<RNWorklet::JsiWorkletContext>& context);
public:

5
android/src/main/cpp/frameprocessor/java-bindings/JFrameProcessorPlugin.cpp
View File

@ -4,8 +4,8 @@
#include "JFrameProcessorPlugin.h"
#include <jni.h>
#include <fbjni/fbjni.h>
#include <jni.h>
namespace vision {
@ -14,7 +14,8 @@ using namespace jni;
using TCallback = jobject(alias_ref<JFrame::javaobject>, alias_ref<JMap<jstring, jobject>> params);
local_ref<jobject> JFrameProcessorPlugin::callback(const alias_ref<JFrame::javaobject>& frame,
local_ref<jobject>
JFrameProcessorPlugin::callback(const alias_ref<JFrame::javaobject>& frame,
const alias_ref<JMap<jstring, jobject>>& params) const {
auto callbackMethod = getClass()->getMethod<TCallback>("callback");

5
android/src/main/cpp/frameprocessor/java-bindings/JFrameProcessorPlugin.h
View File

@ -4,8 +4,8 @@
#pragma once
#include <jni.h>
#include <fbjni/fbjni.h>
#include <jni.h>
#include <string>
#include "JFrame.h"
@ -16,7 +16,8 @@ using namespace facebook;
using namespace jni;
struct JFrameProcessorPlugin : public JavaClass<JFrameProcessorPlugin> {
static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessorPlugin;";
static constexpr auto kJavaDescriptor =
"Lcom/mrousavy/camera/frameprocessor/FrameProcessorPlugin;";
public:
/**

45
android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraProxy.cpp
View File

@ -5,8 +5,8 @@
#include "JVisionCameraProxy.h"
#include <memory>
#include <utility>
#include <string>
#include <utility>
#include <jsi/jsi.h>
@ -25,8 +25,8 @@ using TJSCallInvokerHolder = jni::alias_ref<facebook::react::CallInvokerHolder::
using TScheduler = jni::alias_ref<JVisionCameraScheduler::javaobject>;
using TOptions = jni::local_ref<JMap<jstring, jobject>>;
JVisionCameraProxy::JVisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::jhybridobject>& javaThis,
jsi::Runtime* runtime,
JVisionCameraProxy::JVisionCameraProxy(
const jni::alias_ref<JVisionCameraProxy::jhybridobject>& javaThis, jsi::Runtime* runtime,
const std::shared_ptr<facebook::react::CallInvoker>& callInvoker,
const jni::global_ref<JVisionCameraScheduler::javaobject>& scheduler) {
_javaPart = make_global(javaThis);
@ -41,13 +41,9 @@ JVisionCameraProxy::JVisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::
};
auto runOnWorklet = [scheduler](std::function<void()>&& f) {
// Run on Frame Processor Worklet Runtime
scheduler->cthis()->dispatchAsync([f = std::move(f)](){
f();
});
scheduler->cthis()->dispatchAsync([f = std::move(f)]() { f(); });
};
_workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera",
runtime,
runOnJS,
_workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera", runtime, runOnJS,
runOnWorklet);
__android_log_write(ANDROID_LOG_INFO, TAG, "Worklet Context created!");
#else
@ -64,14 +60,13 @@ JVisionCameraProxy::~JVisionCameraProxy() {
#endif
}
void JVisionCameraProxy::setFrameProcessor(int viewTag,
jsi::Runtime& runtime,
void JVisionCameraProxy::setFrameProcessor(int viewTag, jsi::Runtime& runtime,
const jsi::Object& frameProcessorObject) {
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
auto frameProcessorType = frameProcessorObject.getProperty(runtime, "type").asString(runtime).utf8(runtime);
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, frameProcessorObject.getProperty(runtime, "frameProcessor"));
auto frameProcessorType =
frameProcessorObject.getProperty(runtime, "type").asString(runtime).utf8(runtime);
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(
runtime, frameProcessorObject.getProperty(runtime, "frameProcessor"));
jni::local_ref<JFrameProcessor::javaobject> frameProcessor;
if (frameProcessorType == "frame-processor") {
@ -80,7 +75,9 @@ void JVisionCameraProxy::setFrameProcessor(int viewTag,
throw std::runtime_error("Unknown FrameProcessor.type passed! Received: " + frameProcessorType);
}
auto setFrameProcessorMethod = javaClassLocal()->getMethod<void(int, alias_ref<JFrameProcessor::javaobject>)>("setFrameProcessor");
auto setFrameProcessorMethod =
javaClassLocal()->getMethod<void(int, alias_ref<JFrameProcessor::javaobject>)>(
"setFrameProcessor");
setFrameProcessorMethod(_javaPart, viewTag, frameProcessor);
#else
throw std::runtime_error("system/frame-processors-unavailable: Frame Processors are disabled!");
@ -92,21 +89,19 @@ void JVisionCameraProxy::removeFrameProcessor(int viewTag) {
removeFrameProcessorMethod(_javaPart, viewTag);
}
local_ref<JFrameProcessorPlugin::javaobject> JVisionCameraProxy::getFrameProcessorPlugin(const std::string& name,
TOptions options) {
auto getFrameProcessorPluginMethod = javaClassLocal()->getMethod<JFrameProcessorPlugin(local_ref<jstring>, TOptions)>("getFrameProcessorPlugin");
local_ref<JFrameProcessorPlugin::javaobject>
JVisionCameraProxy::getFrameProcessorPlugin(const std::string& name, TOptions options) {
auto getFrameProcessorPluginMethod =
javaClassLocal()->getMethod<JFrameProcessorPlugin(local_ref<jstring>, TOptions)>(
"getFrameProcessorPlugin");
return getFrameProcessorPluginMethod(_javaPart, make_jstring(name), std::move(options));
}
void JVisionCameraProxy::registerNatives() {
registerHybrid({
makeNativeMethod("initHybrid", JVisionCameraProxy::initHybrid)
});
registerHybrid({makeNativeMethod("initHybrid", JVisionCameraProxy::initHybrid)});
}
TSelf JVisionCameraProxy::initHybrid(
alias_ref<jhybridobject> jThis,
jlong jsRuntimePointer,
TSelf JVisionCameraProxy::initHybrid(alias_ref<jhybridobject> jThis, jlong jsRuntimePointer,
TJSCallInvokerHolder jsCallInvokerHolder,
const TScheduler& scheduler) {
__android_log_write(ANDROID_LOG_INFO, TAG, "Initializing VisionCameraProxy...");

22
android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraProxy.h
View File

@ -4,16 +4,16 @@
#pragma once
#include <ReactCommon/CallInvokerHolder.h>
#include <fbjni/fbjni.h>
#include <jsi/jsi.h>
#include <ReactCommon/CallInvokerHolder.h>
#include "JFrameProcessor.h"
#include "JFrameProcessorPlugin.h"
#include "JVisionCameraScheduler.h"
#include "JFrameProcessor.h"
#include <string>
#include <memory>
#include <string>
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
#include <react-native-worklets-core/WKTJsiWorkletContext.h>
@ -28,14 +28,14 @@ class JVisionCameraProxy : public jni::HybridClass<JVisionCameraProxy> {
~JVisionCameraProxy();
static void registerNatives();
void setFrameProcessor(int viewTag,
jsi::Runtime& runtime,
const jsi::Object& frameProcessor);
void setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& frameProcessor);
void removeFrameProcessor(int viewTag);
jni::local_ref<JFrameProcessorPlugin::javaobject> getFrameProcessorPlugin(const std::string& name,
jni::local_ref<JMap<jstring, jobject>> options);
jni::local_ref<JFrameProcessorPlugin::javaobject>
getFrameProcessorPlugin(const std::string& name, jni::local_ref<JMap<jstring, jobject>> options);
jsi::Runtime* getJSRuntime() { return _runtime; }
jsi::Runtime* getJSRuntime() {
return _runtime;
}
private:
friend HybridBase;
@ -52,8 +52,8 @@ class JVisionCameraProxy : public jni::HybridClass<JVisionCameraProxy> {
jsi::Runtime* jsRuntime,
const std::shared_ptr<facebook::react::CallInvoker>& jsCallInvoker,
const jni::global_ref<JVisionCameraScheduler::javaobject>& scheduler);
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> javaThis,
jlong jsRuntimePointer,
static jni::local_ref<jhybriddata>
initHybrid(jni::alias_ref<jhybridobject> javaThis, jlong jsRuntimePointer,
jni::alias_ref<facebook::react::CallInvokerHolder::javaobject> jsCallInvokerHolder,
const jni::alias_ref<JVisionCameraScheduler::javaobject>& scheduler);
};

14
android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraScheduler.h
View File

@ -4,10 +4,10 @@
#pragma once
#include <jni.h>
#include <fbjni/fbjni.h>
#include <queue>
#include <jni.h>
#include <mutex>
#include <queue>
namespace vision {
@ -19,12 +19,14 @@ using namespace facebook;
*
* 1. Call `dispatchAsync(..)` with the given C++ Method.
* 2. Internally, `scheduleTrigger()` will get called, which is a Java Method.
* 3. The `scheduleTrigger()` Java Method will switch to the Frame Processor Java Thread and call `trigger()` on there
* 3. The `scheduleTrigger()` Java Method will switch to the Frame Processor Java Thread and call
* `trigger()` on there
* 4. `trigger()` is a C++ function here that just calls the passed C++ Method from step 1.
*/
class JVisionCameraScheduler : public jni::HybridClass<JVisionCameraScheduler> {
public:
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraScheduler;";
static auto constexpr kJavaDescriptor =
"Lcom/mrousavy/camera/frameprocessor/VisionCameraScheduler;";
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis);
static void registerNatives();
@ -37,8 +39,8 @@ class JVisionCameraScheduler : public jni::HybridClass<JVisionCameraScheduler> {
std::queue<std::function<void()>> _jobs;
std::mutex _mutex;
explicit JVisionCameraScheduler(jni::alias_ref<JVisionCameraScheduler::jhybridobject> jThis):
_javaPart(jni::make_global(jThis)) {}
explicit JVisionCameraScheduler(jni::alias_ref<JVisionCameraScheduler::jhybridobject> jThis)
: _javaPart(jni::make_global(jThis)) {}
// Schedules a call to `trigger` on the VisionCamera FP Thread
void scheduleTrigger();

28
cpp/JSITypedArray.cpp
View File

@ -6,22 +6,22 @@
// Copyright © 2023 mrousavy. All rights reserved.
//
// Copied & Adapted from https://github.com/expo/expo/blob/main/packages/expo-gl/common/EXTypedArrayApi.cpp
// Credits to Expo
// Copied & Adapted from
// https://github.com/expo/expo/blob/main/packages/expo-gl/common/EXTypedArrayApi.cpp Credits to
// Expo
#include "JSITypedArray.h"
#include <unordered_map>
#include <algorithm>
#include <memory>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
#include <algorithm>
#include <string>
namespace vision {
template <TypedArrayKind T>
using ContentType = typename typedArrayTypeMap<T>::type;
template <TypedArrayKind T> using ContentType = typename typedArrayTypeMap<T>::type;
enum class Prop {
Buffer, // "buffer"
@ -82,8 +82,7 @@ TypedArrayKind getTypedArrayKindForName(const std::string &name);
TypedArrayBase::TypedArrayBase(jsi::Runtime& runtime, size_t size, TypedArrayKind kind)
: TypedArrayBase(
runtime,
runtime.global()
runtime, runtime.global()
.getProperty(runtime, propNameIDCache.getConstructorNameProp(runtime, kind))
.asObject(runtime)
.asFunction(runtime)
@ -180,10 +179,7 @@ std::vector<uint8_t> arrayBufferToVector(jsi::Runtime &runtime, jsi::Object &jsO
return std::vector<uint8_t>(dataBlock, dataBlock + blockSize);
}
void arrayBufferUpdate(
jsi::Runtime &runtime,
jsi::ArrayBuffer &buffer,
std::vector<uint8_t> data,
void arrayBufferUpdate(jsi::Runtime& runtime, jsi::ArrayBuffer& buffer, std::vector<uint8_t> data,
size_t offset) {
uint8_t* dataBlock = buffer.data(runtime);
size_t blockSize = buffer.size(runtime);
@ -231,13 +227,11 @@ void TypedArray<T>::updateUnsafe(jsi::Runtime &runtime, ContentType<T> *data, si
memcpy(rawData, data, length);
}
template <TypedArrayKind T>
uint8_t* TypedArray<T>::data(jsi::Runtime &runtime) {
template <TypedArrayKind T> uint8_t* TypedArray<T>::data(jsi::Runtime& runtime) {
return getBuffer(runtime).data(runtime) + byteOffset(runtime);
}
const jsi::PropNameID &PropNameIDCache::getConstructorNameProp(
jsi::Runtime &runtime,
const jsi::PropNameID& PropNameIDCache::getConstructorNameProp(jsi::Runtime& runtime,
TypedArrayKind kind) {
switch (kind) {
case TypedArrayKind::Int8Array:

76
cpp/JSITypedArray.h
View File

@ -6,9 +6,8 @@
// Copyright © 2023 mrousavy. All rights reserved.
//
// Copied & Adapted from https://github.com/expo/expo/blob/main/packages/expo-gl/common/EXTypedArrayApi.h
// Credits to Expo
// Copied & Adapted from
// https://github.com/expo/expo/blob/main/packages/expo-gl/common/EXTypedArrayApi.h Credits to Expo
#pragma once
@ -32,45 +31,34 @@ enum class TypedArrayKind {
Float64Array,
};
template <TypedArrayKind T>
class TypedArray;
template <TypedArrayKind T> class TypedArray;
template <TypedArrayKind T>
struct typedArrayTypeMap;
template <>
struct typedArrayTypeMap<TypedArrayKind::Int8Array> {
template <TypedArrayKind T> struct typedArrayTypeMap;
template <> struct typedArrayTypeMap<TypedArrayKind::Int8Array> {
typedef int8_t type;
};
template <>
struct typedArrayTypeMap<TypedArrayKind::Int16Array> {
template <> struct typedArrayTypeMap<TypedArrayKind::Int16Array> {
typedef int16_t type;
};
template <>
struct typedArrayTypeMap<TypedArrayKind::Int32Array> {
template <> struct typedArrayTypeMap<TypedArrayKind::Int32Array> {
typedef int32_t type;
};
template <>
struct typedArrayTypeMap<TypedArrayKind::Uint8Array> {
template <> struct typedArrayTypeMap<TypedArrayKind::Uint8Array> {
typedef uint8_t type;
};
template <>
struct typedArrayTypeMap<TypedArrayKind::Uint8ClampedArray> {
template <> struct typedArrayTypeMap<TypedArrayKind::Uint8ClampedArray> {
typedef uint8_t type;
};
template <>
struct typedArrayTypeMap<TypedArrayKind::Uint16Array> {
template <> struct typedArrayTypeMap<TypedArrayKind::Uint16Array> {
typedef uint16_t type;
};
template <>
struct typedArrayTypeMap<TypedArrayKind::Uint32Array> {
template <> struct typedArrayTypeMap<TypedArrayKind::Uint32Array> {
typedef uint32_t type;
};
template <>
struct typedArrayTypeMap<TypedArrayKind::Float32Array> {
template <> struct typedArrayTypeMap<TypedArrayKind::Float32Array> {
typedef float type;
};
template <>
struct typedArrayTypeMap<TypedArrayKind::Float64Array> {
template <> struct typedArrayTypeMap<TypedArrayKind::Float64Array> {
typedef double type;
};
@ -78,8 +66,7 @@ void invalidateArrayBufferCache(jsi::Runtime& runtime);
class TypedArrayBase : public jsi::Object {
public:
template <TypedArrayKind T>
using ContentType = typename typedArrayTypeMap<T>::type;
template <TypedArrayKind T> using ContentType = typename typedArrayTypeMap<T>::type;
TypedArrayBase(jsi::Runtime&, size_t, TypedArrayKind);
TypedArrayBase(jsi::Runtime&, const jsi::Object&);
@ -88,14 +75,10 @@ class TypedArrayBase : public jsi::Object {
TypedArrayKind getKind(jsi::Runtime& runtime) const;
template <TypedArrayKind T>
TypedArray<T> get(jsi::Runtime &runtime) const &;
template <TypedArrayKind T>
TypedArray<T> get(jsi::Runtime &runtime) &&;
template <TypedArrayKind T>
TypedArray<T> as(jsi::Runtime &runtime) const &;
template <TypedArrayKind T>
TypedArray<T> as(jsi::Runtime &runtime) &&;
template <TypedArrayKind T> TypedArray<T> get(jsi::Runtime& runtime) const&;
template <TypedArrayKind T> TypedArray<T> get(jsi::Runtime& runtime) &&;
template <TypedArrayKind T> TypedArray<T> as(jsi::Runtime& runtime) const&;
template <TypedArrayKind T> TypedArray<T> as(jsi::Runtime& runtime) &&;
size_t size(jsi::Runtime& runtime) const;
size_t length(jsi::Runtime& runtime) const;
@ -107,22 +90,17 @@ class TypedArrayBase : public jsi::Object {
jsi::ArrayBuffer getBuffer(jsi::Runtime& runtime) const;
private:
template <TypedArrayKind>
friend class TypedArray;
template <TypedArrayKind> friend class TypedArray;
};
bool isTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj);
TypedArrayBase getTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj);
std::vector<uint8_t> arrayBufferToVector(jsi::Runtime& runtime, jsi::Object& jsObj);
void arrayBufferUpdate(
jsi::Runtime &runtime,
jsi::ArrayBuffer &buffer,
std::vector<uint8_t> data,
void arrayBufferUpdate(jsi::Runtime& runtime, jsi::ArrayBuffer& buffer, std::vector<uint8_t> data,
size_t offset);
template <TypedArrayKind T>
class TypedArray : public TypedArrayBase {
template <TypedArrayKind T> class TypedArray : public TypedArrayBase {
public:
explicit TypedArray(TypedArrayBase&& base);
TypedArray(jsi::Runtime& runtime, size_t size);
@ -136,30 +114,26 @@ class TypedArray : public TypedArrayBase {
uint8_t* data(jsi::Runtime& runtime);
};
template <TypedArrayKind T>
TypedArray<T> TypedArrayBase::get(jsi::Runtime &runtime) const & {
template <TypedArrayKind T> TypedArray<T> TypedArrayBase::get(jsi::Runtime& runtime) const& {
assert(getKind(runtime) == T);
(void)runtime; // when assert is disabled we need to mark this as used
return TypedArray<T>(jsi::Value(runtime, jsi::Value(runtime, *this).asObject(runtime)));
}
template <TypedArrayKind T>
TypedArray<T> TypedArrayBase::get(jsi::Runtime &runtime) && {
template <TypedArrayKind T> TypedArray<T> TypedArrayBase::get(jsi::Runtime& runtime) && {
assert(getKind(runtime) == T);
(void)runtime; // when assert is disabled we need to mark this as used
return TypedArray<T>(std::move(*this));
}
template <TypedArrayKind T>
TypedArray<T> TypedArrayBase::as(jsi::Runtime &runtime) const & {
template <TypedArrayKind T> TypedArray<T> TypedArrayBase::as(jsi::Runtime& runtime) const& {
if (getKind(runtime) != T) {
throw jsi::JSError(runtime, "Object is not a TypedArray");
}
return get<T>(runtime);
}
template <TypedArrayKind T>
TypedArray<T> TypedArrayBase::as(jsi::Runtime &runtime) && {
template <TypedArrayKind T> TypedArray<T> TypedArrayBase::as(jsi::Runtime& runtime) && {
if (getKind(runtime) != T) {
throw jsi::JSError(runtime, "Object is not a TypedArray");
}

1
example/src/CameraPage.tsx
View File

@ -24,7 +24,6 @@ import IonIcon from 'react-native-vector-icons/Ionicons';
import type { Routes } from './Routes';
import type { NativeStackScreenProps } from '@react-navigation/native-stack';
import { useIsFocused } from '@react-navigation/core';
import { FACE_SHADER } from './Shaders';
import { examplePlugin } from './frame-processors/ExamplePlugin';
const ReanimatedCamera = Reanimated.createAnimatedComponent(Camera);

89
example/src/Shaders.ts
View File

@ -1,89 +0,0 @@
export const INVERTED_COLORS_SHADER = `
uniform shader image;
half4 main(vec2 pos) {
vec4 color = image.eval(pos);
return vec4(1.0 - color.rgb, 1.0);
}
`;
export const CHROMATIC_ABERRATION_SHADER = `
uniform shader image;
vec4 chromatic(vec2 pos, float offset) {
float r = image.eval(pos).r;
float g = image.eval(vec2(pos.x + offset, pos.y)).g;
float b = image.eval(vec2(pos.x + offset * 2.0, pos.y)).b;
return vec4(r, g, b, 1.0);
}
half4 main(vec2 pos) {
float offset = 50.0;
return chromatic(pos, offset);
}
`;
export const NO_SHADER = `
half4 main(vec2 pos) {
return vec4(1.0);
}
`;
export const BLUR_SHADER = `
const int samples = 35,
LOD = 2, // gaussian done on MIPmap at scale LOD
sLOD = 1 << LOD; // tile size = 2^LOD
const float sigma = float(samples) * .25;
float gaussian(vec2 i) {
return exp( -.5* dot(i/=sigma,i) ) / ( 6.28 * sigma*sigma );
}
vec4 blur(sampler2D sp, vec2 U, vec2 scale) {
vec4 O = vec4(0);
int s = samples/sLOD;
for ( int i = 0; i < s*s; i++ ) {
vec2 d = vec2(i%s, i/s)*float(sLOD) - float(samples)/2.;
O += gaussian(d) * textureLod( sp, U + scale * d , float(LOD) );
}
return O / O.a;
}
void mainImage(out vec4 O, vec2 U) {
O = blur( iChannel0, U/iResolution.xy, 1./iChannelResolution[0].xy );
}
`;
export const FACE_SHADER = `
uniform shader image;
uniform float x;
uniform float y;
uniform float r;
uniform vec2 resolution;
const float samples = 3.0;
const float radius = 40.0;
const float weight = 1.0;
half4 main(vec2 pos) {
float delta = pow((pow(pos.x - x, 2) + pow(pos.y - y, 2)), 0.5);
if (delta < r) {
vec3 sum = vec3(0.0);
vec3 accumulation = vec3(0);
vec3 weightedsum = vec3(0);
for (float deltaX = -samples * radius; deltaX <= samples * radius; deltaX += radius / samples) {
for (float deltaY = -samples * radius; deltaY <= samples * radius; deltaY += radius / samples) {
accumulation += image.eval(vec2(pos.x + deltaX, pos.y + deltaY)).rgb;
weightedsum += weight;
}
}
sum = accumulation / weightedsum;
return vec4(sum, 1.0);
}
else {
return image.eval(pos);
}
}
`;

6
ios/CameraBridge.h
View File

@ -10,13 +10,13 @@
#import <Foundation/Foundation.h>
#import <React/RCTViewManager.h>
#import <React/RCTUIManager.h>
#import <React/RCTFPSGraph.h>
#import <React/RCTLog.h>
#import <React/RCTUIManager.h>
#import <React/RCTViewManager.h>
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
#import "FrameProcessor.h"
#import "Frame.h"
#import "FrameProcessor.h"
#import "VisionCameraProxy.h"
#endif

2
ios/CameraView+Focus.swift
View File

@ -1,5 +1,5 @@
//
// CameraView+focus.swift
// CameraView+Focus.swift
// mrousavy
//
// Created by Marc Rousavy on 19.02.21.

2
ios/CameraView+Preview.swift
View File

@ -16,7 +16,7 @@ extension CameraView {
addSubview(previewView!)
}
internal func setupFpsGraph() {
func setupFpsGraph() {
#if DEBUG
if enableFpsGraph {
if fpsGraph != nil { return }

34
ios/CameraView.swift
View File

@ -74,30 +74,30 @@ public final class CameraView: UIView {
}
// pragma MARK: Internal Properties
internal var isMounted = false
internal var isReady = false
var isMounted = false
var isReady = false
// Capture Session
internal let captureSession = AVCaptureSession()
internal let audioCaptureSession = AVCaptureSession()
let captureSession = AVCaptureSession()
let audioCaptureSession = AVCaptureSession()
// Inputs & Outputs
internal var videoDeviceInput: AVCaptureDeviceInput?
internal var audioDeviceInput: AVCaptureDeviceInput?
internal var photoOutput: AVCapturePhotoOutput?
internal var videoOutput: AVCaptureVideoDataOutput?
internal var audioOutput: AVCaptureAudioDataOutput?
var videoDeviceInput: AVCaptureDeviceInput?
var audioDeviceInput: AVCaptureDeviceInput?
var photoOutput: AVCapturePhotoOutput?
var videoOutput: AVCaptureVideoDataOutput?
var audioOutput: AVCaptureAudioDataOutput?
// CameraView+RecordView (+ Frame Processor)
internal var isRecording = false
internal var recordingSession: RecordingSession?
var isRecording = false
var recordingSession: RecordingSession?
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
@objc public var frameProcessor: FrameProcessor?
#endif
// CameraView+Zoom
internal var pinchGestureRecognizer: UIPinchGestureRecognizer?
internal var pinchScaleOffset: CGFloat = 1.0
var pinchGestureRecognizer: UIPinchGestureRecognizer?
var pinchScaleOffset: CGFloat = 1.0
internal var previewView: PreviewView?
var previewView: PreviewView?
#if DEBUG
internal var fpsGraph: RCTFPSGraph?
var fpsGraph: RCTFPSGraph?
#endif
/// Returns whether the AVCaptureSession is currently running (reflected by isActive)
@ -258,7 +258,7 @@ public final class CameraView: UIView {
}
// pragma MARK: Event Invokers
internal final func invokeOnError(_ error: CameraError, cause: NSError? = nil) {
final func invokeOnError(_ error: CameraError, cause: NSError? = nil) {
ReactLogger.log(level: .error, message: "Invoking onError(): \(error.message)")
guard let onError = onError else { return }
@ -278,7 +278,7 @@ public final class CameraView: UIView {
])
}
internal final func invokeOnInitialized() {
final func invokeOnInitialized() {
ReactLogger.log(level: .info, message: "Camera initialized!")
guard let onInitialized = onInitialized else { return }
onInitialized([String: Any]())

54
ios/CameraViewManager.m
View File

@ -8,18 +8,28 @@
#import <Foundation/Foundation.h>
#import <React/RCTViewManager.h>
#import <React/RCTUtils.h>
#import <React/RCTViewManager.h>
@interface RCT_EXTERN_REMAP_MODULE (CameraView, CameraViewManager, RCTViewManager)
// Module Functions
RCT_EXTERN_METHOD(getCameraPermissionStatus:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(getMicrophonePermissionStatus:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(requestCameraPermission:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(requestMicrophonePermission:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(getCameraPermissionStatus
: (RCTPromiseResolveBlock)resolve reject
: (RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(getMicrophonePermissionStatus
: (RCTPromiseResolveBlock)resolve reject
: (RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(requestCameraPermission
: (RCTPromiseResolveBlock)resolve reject
: (RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(requestMicrophonePermission
: (RCTPromiseResolveBlock)resolve reject
: (RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(getAvailableCameraDevices:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(getAvailableCameraDevices
: (RCTPromiseResolveBlock)resolve reject
: (RCTPromiseRejectBlock)reject);
// Camera View Properties
RCT_EXPORT_VIEW_PROPERTY(isActive, BOOL);
@ -51,12 +61,32 @@ RCT_EXPORT_VIEW_PROPERTY(onInitialized, RCTDirectEventBlock);
RCT_EXPORT_VIEW_PROPERTY(onViewReady, RCTDirectEventBlock);
// Camera View Functions
RCT_EXTERN_METHOD(startRecording:(nonnull NSNumber *)node options:(NSDictionary *)options onRecordCallback:(RCTResponseSenderBlock)onRecordCallback);
RCT_EXTERN_METHOD(pauseRecording:(nonnull NSNumber *)node resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(resumeRecording:(nonnull NSNumber *)node resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(stopRecording:(nonnull NSNumber *)node resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(takePhoto:(nonnull NSNumber *)node options:(NSDictionary *)options resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(focus:(nonnull NSNumber *)node point:(NSDictionary *)point resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(startRecording
: (nonnull NSNumber*)node options
: (NSDictionary*)options onRecordCallback
: (RCTResponseSenderBlock)onRecordCallback);
RCT_EXTERN_METHOD(pauseRecording
: (nonnull NSNumber*)node resolve
: (RCTPromiseResolveBlock)resolve reject
: (RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(resumeRecording
: (nonnull NSNumber*)node resolve
: (RCTPromiseResolveBlock)resolve reject
: (RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(stopRecording
: (nonnull NSNumber*)node resolve
: (RCTPromiseResolveBlock)resolve reject
: (RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(takePhoto
: (nonnull NSNumber*)node options
: (NSDictionary*)options resolve
: (RCTPromiseResolveBlock)resolve reject
: (RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(focus
: (nonnull NSNumber*)node point
: (NSDictionary*)point resolve
: (RCTPromiseResolveBlock)resolve reject
: (RCTPromiseRejectBlock)reject);
// Static Methods
RCT_EXTERN__BLOCKING_SYNCHRONOUS_METHOD(installFrameProcessorBindings);

5
ios/Frame Processor/Frame.h
View File

@ -8,13 +8,14 @@
#pragma once
#import <Foundation/Foundation.h>
#import <CoreMedia/CMSampleBuffer.h>
#import <Foundation/Foundation.h>
#import <UIKit/UIImage.h>
@interface Frame : NSObject
- (instancetype _Nonnull) initWithBuffer:(CMSampleBufferRef _Nonnull)buffer orientation:(UIImageOrientation)orientation;
- (instancetype _Nonnull)initWithBuffer:(CMSampleBufferRef _Nonnull)buffer
orientation:(UIImageOrientation)orientation;
@property(nonatomic, readonly) CMSampleBufferRef _Nonnull buffer;
@property(nonatomic, readonly) UIImageOrientation orientation;

5
ios/Frame Processor/Frame.m
View File

@ -7,15 +7,16 @@
//
#import "Frame.h"
#import <Foundation/Foundation.h>
#import <CoreMedia/CMSampleBuffer.h>
#import <Foundation/Foundation.h>
@implementation Frame {
CMSampleBufferRef _Nonnull buffer;
UIImageOrientation orientation;
}
- (instancetype) initWithBuffer:(CMSampleBufferRef _Nonnull)buffer orientation:(UIImageOrientation)orientation {
- (instancetype)initWithBuffer:(CMSampleBufferRef _Nonnull)buffer
orientation:(UIImageOrientation)orientation {
self = [super init];
if (self) {
_buffer = buffer;

2
ios/Frame Processor/FrameHostObject.h
View File

@ -8,8 +8,8 @@
#pragma once
#import <jsi/jsi.h>
#import <CoreMedia/CMSampleBuffer.h>
#import <jsi/jsi.h>
#import "Frame.h"

41
ios/Frame Processor/FrameHostObject.mm
View File

@ -7,9 +7,9 @@
//
#import "FrameHostObject.h"
#import "WKTJsiHostObject.h"
#import <Foundation/Foundation.h>
#import <jsi/jsi.h>
#import "WKTJsiHostObject.h"
#import "../../cpp/JSITypedArray.h"
@ -46,10 +46,12 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
auto width = CVPixelBufferGetWidth(imageBuffer);
auto height = CVPixelBufferGetHeight(imageBuffer);
NSMutableString* string = [NSMutableString stringWithFormat:@"%lu x %lu Frame", width, height];
NSMutableString* string =
[NSMutableString stringWithFormat:@"%lu x %lu Frame", width, height];
return jsi::String::createFromUtf8(runtime, string.UTF8String);
};
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
}
if (name == "incrementRefCount") {
auto incrementRefCount = JSI_HOST_FUNCTION_LAMBDA {
@ -57,21 +59,18 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
CFRetain(frame.buffer);
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "incrementRefCount"),
0,
incrementRefCount);
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "incrementRefCount"), 0, incrementRefCount);
}
if (name == "decrementRefCount") {
auto decrementRefCount = JSI_HOST_FUNCTION_LAMBDA {
// Decrement retain count by one. If the retain count is zero, ARC will destroy the Frame Buffer.
// Decrement retain count by one. If the retain count is zero, ARC will destroy the Frame
// Buffer.
CFRelease(frame.buffer);
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "decrementRefCount"),
0,
decrementRefCount);
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "decrementRefCount"), 0, decrementRefCount);
}
if (name == "toArrayBuffer") {
auto toArrayBuffer = JSI_HOST_FUNCTION_LAMBDA {
@ -83,15 +82,19 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
static constexpr auto ARRAYBUFFER_CACHE_PROP_NAME = "__frameArrayBufferCache";
if (!runtime.global().hasProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME)) {
vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray> arrayBuffer(runtime, arraySize);
vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray> arrayBuffer(runtime,
arraySize);
runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer);
}
auto arrayBufferCache = runtime.global().getPropertyAsObject(runtime, ARRAYBUFFER_CACHE_PROP_NAME);
auto arrayBuffer = vision::getTypedArray(runtime, arrayBufferCache).get<vision::TypedArrayKind::Uint8ClampedArray>(runtime);
auto arrayBufferCache =
runtime.global().getPropertyAsObject(runtime, ARRAYBUFFER_CACHE_PROP_NAME);
auto arrayBuffer = vision::getTypedArray(runtime, arrayBufferCache)
.get<vision::TypedArrayKind::Uint8ClampedArray>(runtime);
if (arrayBuffer.size(runtime) != arraySize) {
arrayBuffer = vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray>(runtime, arraySize);
arrayBuffer =
vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray>(runtime, arraySize);
runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer);
}
@ -102,11 +105,13 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
return arrayBuffer;
};
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toArrayBuffer"), 0, toArrayBuffer);
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "toArrayBuffer"), 0, toArrayBuffer);
}
if (name == "isValid") {
auto isValid = frame != nil && frame.buffer != nil && CFGetRetainCount(frame.buffer) > 0 && CMSampleBufferIsValid(frame.buffer);
auto isValid = frame != nil && frame.buffer != nil && CFGetRetainCount(frame.buffer) > 0 &&
CMSampleBufferIsValid(frame.buffer);
return jsi::Value(isValid);
}
if (name == "width") {

6
ios/Frame Processor/FrameProcessor.h
View File

@ -8,14 +8,14 @@
#pragma once
#import <Foundation/Foundation.h>
#import <AVFoundation/AVFoundation.h>
#import "Frame.h"
#import <AVFoundation/AVFoundation.h>
#import <Foundation/Foundation.h>
#ifdef __cplusplus
#import "FrameHostObject.h"
#import "WKTJsiWorklet.h"
#import <jsi/jsi.h>
#import "FrameHostObject.h"
#import <memory.h>
#endif

15
ios/Frame Processor/FrameProcessor.mm
View File

@ -6,13 +6,13 @@
// Copyright © 2023 mrousavy. All rights reserved.
//
#import <Foundation/Foundation.h>
#import "FrameProcessor.h"
#import <Foundation/Foundation.h>
#import <memory>
#import <jsi/jsi.h>
#import "WKTJsiWorklet.h"
#import "FrameHostObject.h"
#import "WKTJsiWorklet.h"
#import <jsi/jsi.h>
#import <memory>
using namespace facebook;
@ -46,8 +46,11 @@ using namespace facebook;
auto message = jsError.getMessage();
_workletContext->invokeOnJsThread([message](jsi::Runtime& jsRuntime) {
auto logFn = jsRuntime.global().getPropertyAsObject(jsRuntime, "console").getPropertyAsFunction(jsRuntime, "error");
logFn.call(jsRuntime, jsi::String::createFromUtf8(jsRuntime, "Frame Processor threw an error: " + message));
auto logFn = jsRuntime.global()
.getPropertyAsObject(jsRuntime, "console")
.getPropertyAsFunction(jsRuntime, "error");
logFn.call(jsRuntime, jsi::String::createFromUtf8(
jsRuntime, "Frame Processor threw an error: " + message));
});
}
}

12
ios/Frame Processor/FrameProcessorPlugin.h
View File

@ -8,15 +8,17 @@
#pragma once
#import <Foundation/Foundation.h>
#import "Frame.h"
#import <Foundation/Foundation.h>
/// The base class for a Frame Processor Plugin which can be called synchronously from a JS Frame Processor.
/// The base class for a Frame Processor Plugin which can be called synchronously from a JS Frame
/// Processor.
///
/// Subclass this class in a Swift or Objective-C class and override the `callback:withArguments:` method, and
/// implement your Frame Processing there.
/// Subclass this class in a Swift or Objective-C class and override the `callback:withArguments:`
/// method, and implement your Frame Processing there.
///
/// Use `[FrameProcessorPluginRegistry addFrameProcessorPlugin:]` to register the Plugin to the VisionCamera Runtime.
/// Use `[FrameProcessorPluginRegistry addFrameProcessorPlugin:]` to register the Plugin to the
/// VisionCamera Runtime.
@interface FrameProcessorPlugin : NSObject
/// The actual callback when calling this plugin. Any Frame Processing should be handled there.

6
ios/Frame Processor/FrameProcessorPlugin.m
View File

@ -12,8 +12,10 @@
@implementation FrameProcessorPlugin
- (id _Nullable)callback:(Frame* _Nonnull)frame withArguments:(NSDictionary* _Nullable)arguments {
[NSException raise:NSInternalInconsistencyException
format:@"Frame Processor Plugin does not override the `callback(frame:withArguments:)` method!"];
[NSException
raise:NSInternalInconsistencyException
format:
@"Frame Processor Plugin does not override the `callback(frame:withArguments:)` method!"];
return nil;
}

8
ios/Frame Processor/FrameProcessorPluginHostObject.h
View File

@ -8,18 +8,18 @@
#pragma once
#import <jsi/jsi.h>
#import "FrameProcessorPlugin.h"
#import <memory>
#import <ReactCommon/CallInvoker.h>
#import <jsi/jsi.h>
#import <memory>
using namespace facebook;
class FrameProcessorPluginHostObject : public jsi::HostObject {
public:
explicit FrameProcessorPluginHostObject(FrameProcessorPlugin* plugin,
std::shared_ptr<react::CallInvoker> callInvoker):
_plugin(plugin), _callInvoker(callInvoker) { }
std::shared_ptr<react::CallInvoker> callInvoker)
: _plugin(plugin), _callInvoker(callInvoker) {}
~FrameProcessorPluginHostObject() {}
public:

25
ios/Frame Processor/FrameProcessorPluginHostObject.mm
View File

@ -7,39 +7,40 @@
//
#import "FrameProcessorPluginHostObject.h"
#import <Foundation/Foundation.h>
#import <vector>
#import "FrameHostObject.h"
#import "JSINSObjectConversion.h"
#import <Foundation/Foundation.h>
#import <vector>
using namespace facebook;
std::vector<jsi::PropNameID> FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime& runtime) {
std::vector<jsi::PropNameID>
FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime& runtime) {
std::vector<jsi::PropNameID> result;
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("call")));
return result;
}
jsi::Value FrameProcessorPluginHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
jsi::Value FrameProcessorPluginHostObject::get(jsi::Runtime& runtime,
const jsi::PropNameID& propName) {
auto name = propName.utf8(runtime);
if (name == "call") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "call"),
2,
[=](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "call"), 2,
[=](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
size_t count) -> jsi::Value {
// Frame is first argument
auto frameHostObject = arguments[0].asObject(runtime).asHostObject<FrameHostObject>(runtime);
auto frameHostObject =
arguments[0].asObject(runtime).asHostObject<FrameHostObject>(runtime);
Frame* frame = frameHostObject->frame;
// Options are second argument (possibly undefined)
NSDictionary* options = nil;
if (count > 1) {
auto optionsObject = arguments[1].asObject(runtime);
options = JSINSObjectConversion::convertJSIObjectToNSDictionary(runtime, optionsObject, _callInvoker);
options = JSINSObjectConversion::convertJSIObjectToNSDictionary(runtime, optionsObject,
_callInvoker);
}
// Call actual Frame Processor Plugin

5
ios/Frame Processor/FrameProcessorPluginRegistry.h
View File

@ -8,13 +8,14 @@
#pragma once
#import <Foundation/Foundation.h>
#import "Frame.h"
#import "FrameProcessorPlugin.h"
#import <Foundation/Foundation.h>
@interface FrameProcessorPluginRegistry : NSObject
typedef FrameProcessorPlugin* _Nonnull (^PluginInitializerFunction)(NSDictionary* _Nullable options);
typedef FrameProcessorPlugin* _Nonnull (^PluginInitializerFunction)(
NSDictionary* _Nullable options);
+ (void)addFrameProcessorPlugin:(NSString* _Nonnull)name
withInitializer:(PluginInitializerFunction _Nonnull)pluginInitializer;

18
ios/Frame Processor/FrameProcessorPluginRegistry.m
View File

@ -19,15 +19,23 @@
return plugins;
}
+ (void) addFrameProcessorPlugin:(NSString *)name withInitializer:(PluginInitializerFunction)pluginInitializer {
BOOL alreadyExists = [[FrameProcessorPluginRegistry frameProcessorPlugins] valueForKey:name] != nil;
NSAssert(!alreadyExists, @"Tried to add a Frame Processor Plugin with a name that already exists! Either choose unique names, or remove the unused plugin. Name: %@", name);
+ (void)addFrameProcessorPlugin:(NSString*)name
withInitializer:(PluginInitializerFunction)pluginInitializer {
BOOL alreadyExists =
[[FrameProcessorPluginRegistry frameProcessorPlugins] valueForKey:name] != nil;
NSAssert(!alreadyExists,
@"Tried to add a Frame Processor Plugin with a name that already exists! Either choose "
@"unique names, or "
@"remove the unused plugin. Name: %@",
name);
[[FrameProcessorPluginRegistry frameProcessorPlugins] setValue:pluginInitializer forKey:name];
}
+ (FrameProcessorPlugin*)getPlugin:(NSString* _Nonnull)name withOptions:(NSDictionary* _Nullable)options {
PluginInitializerFunction initializer = [[FrameProcessorPluginRegistry frameProcessorPlugins] objectForKey:name];
+ (FrameProcessorPlugin*)getPlugin:(NSString* _Nonnull)name
withOptions:(NSDictionary* _Nullable)options {
PluginInitializerFunction initializer =
[[FrameProcessorPluginRegistry frameProcessorPlugins] objectForKey:name];
if (initializer == nil) {
return nil;
}

20
ios/Frame Processor/JSINSObjectConversion.h
View File

@ -8,9 +8,9 @@
#pragma once
#import <jsi/jsi.h>
#import <ReactCommon/CallInvoker.h>
#import <React/RCTBridgeModule.h>
#import <ReactCommon/CallInvoker.h>
#import <jsi/jsi.h>
namespace JSINSObjectConversion {
@ -39,21 +39,27 @@ jsi::Value convertObjCObjectToJSIValue(jsi::Runtime& runtime, id value);
NSString* convertJSIStringToNSString(jsi::Runtime& runtime, const jsi::String& value);
// any... -> NSArray
NSArray* convertJSICStyleArrayToNSArray(jsi::Runtime& runtime, const jsi::Value* array, size_t length, std::shared_ptr<CallInvoker> jsInvoker);
NSArray* convertJSICStyleArrayToNSArray(jsi::Runtime& runtime, const jsi::Value* array,
size_t length, std::shared_ptr<CallInvoker> jsInvoker);
// NSArray -> any...
jsi::Value* convertNSArrayToJSICStyleArray(jsi::Runtime& runtime, NSArray* array);
// [] -> NSArray
NSArray* convertJSIArrayToNSArray(jsi::Runtime& runtime, const jsi::Array& value, std::shared_ptr<CallInvoker> jsInvoker);
NSArray* convertJSIArrayToNSArray(jsi::Runtime& runtime, const jsi::Array& value,
std::shared_ptr<CallInvoker> jsInvoker);
// {} -> NSDictionary
NSDictionary* convertJSIObjectToNSDictionary(jsi::Runtime& runtime, const jsi::Object& value, std::shared_ptr<CallInvoker> jsInvoker);
NSDictionary* convertJSIObjectToNSDictionary(jsi::Runtime& runtime, const jsi::Object& value,
std::shared_ptr<CallInvoker> jsInvoker);
// any -> id
id convertJSIValueToObjCObject(jsi::Runtime& runtime, const jsi::Value& value, std::shared_ptr<CallInvoker> jsInvoker);
id convertJSIValueToObjCObject(jsi::Runtime& runtime, const jsi::Value& value,
std::shared_ptr<CallInvoker> jsInvoker);
// (any...) => any -> (void)(id, id)
RCTResponseSenderBlock convertJSIFunctionToCallback(jsi::Runtime& runtime, const jsi::Function& value, std::shared_ptr<CallInvoker> jsInvoker);
RCTResponseSenderBlock convertJSIFunctionToCallback(jsi::Runtime& runtime,
const jsi::Function& value,
std::shared_ptr<CallInvoker> jsInvoker);
} // namespace JSINSObjectConversion

70
ios/Frame Processor/JSINSObjectConversion.mm
View File

@ -5,7 +5,8 @@
// Forked and Adjusted by Marc Rousavy on 02.05.21.
// Copyright © 2021 mrousavy & Facebook. All rights reserved.
//
// Forked and adjusted from: https://github.com/facebook/react-native/blob/900210cacc4abca0079e3903781bc223c80c8ac7/ReactCommon/react/nativemodule/core/platform/ios/RCTTurboModule.mm
// Forked and adjusted from:
// https://github.com/facebook/react-native/blob/900210cacc4abca0079e3903781bc223c80c8ac7/ReactCommon/react/nativemodule/core/platform/ios/RCTTurboModule.mm
// Original Copyright Notice:
//
// Copyright (c) Facebook, Inc. and its affiliates.
@ -15,37 +16,33 @@
//
#import "JSINSObjectConversion.h"
#import <Foundation/Foundation.h>
#import <jsi/jsi.h>
#import <ReactCommon/CallInvoker.h>
#import <React/RCTBridge.h>
#import <ReactCommon/TurboModuleUtils.h>
#import <ReactCommon/RCTBlockGuard.h>
#import "../Frame Processor/Frame.h"
#import "../Frame Processor/FrameHostObject.h"
#import <Foundation/Foundation.h>
#import <React/RCTBridge.h>
#import <ReactCommon/CallInvoker.h>
#import <ReactCommon/RCTBlockGuard.h>
#import <ReactCommon/TurboModuleUtils.h>
#import <jsi/jsi.h>
using namespace facebook;
using namespace facebook::react;
namespace JSINSObjectConversion {
jsi::Value convertNSNumberToJSIBoolean(jsi::Runtime &runtime, NSNumber *value)
{
jsi::Value convertNSNumberToJSIBoolean(jsi::Runtime& runtime, NSNumber* value) {
return jsi::Value((bool)[value boolValue]);
}
jsi::Value convertNSNumberToJSINumber(jsi::Runtime &runtime, NSNumber *value)
{
jsi::Value convertNSNumberToJSINumber(jsi::Runtime& runtime, NSNumber* value) {
return jsi::Value([value doubleValue]);
}
jsi::String convertNSStringToJSIString(jsi::Runtime &runtime, NSString *value)
{
jsi::String convertNSStringToJSIString(jsi::Runtime& runtime, NSString* value) {
return jsi::String::createFromUtf8(runtime, [value UTF8String] ?: "");
}
jsi::Object convertNSDictionaryToJSIObject(jsi::Runtime &runtime, NSDictionary *value)
{
jsi::Object convertNSDictionaryToJSIObject(jsi::Runtime& runtime, NSDictionary* value) {
jsi::Object result = jsi::Object(runtime);
for (NSString* k in value) {
result.setProperty(runtime, [k UTF8String], convertObjCObjectToJSIValue(runtime, value[k]));
@ -53,8 +50,7 @@ jsi::Object convertNSDictionaryToJSIObject(jsi::Runtime &runtime, NSDictionary *
return result;
}
jsi::Array convertNSArrayToJSIArray(jsi::Runtime &runtime, NSArray *value)
{
jsi::Array convertNSArrayToJSIArray(jsi::Runtime& runtime, NSArray* value) {
jsi::Array result = jsi::Array(runtime, value.count);
for (size_t i = 0; i < value.count; i++) {
result.setValueAtIndex(runtime, i, convertObjCObjectToJSIValue(runtime, value[i]));
@ -62,8 +58,7 @@ jsi::Array convertNSArrayToJSIArray(jsi::Runtime &runtime, NSArray *value)
return result;
}
jsi::Value convertObjCObjectToJSIValue(jsi::Runtime &runtime, id value)
{
jsi::Value convertObjCObjectToJSIValue(jsi::Runtime& runtime, id value) {
if (value == nil) {
return jsi::Value::undefined();
} else if ([value isKindOfClass:[NSString class]]) {
@ -86,18 +81,18 @@ jsi::Value convertObjCObjectToJSIValue(jsi::Runtime &runtime, id value)
return jsi::Value::undefined();
}
NSString *convertJSIStringToNSString(jsi::Runtime &runtime, const jsi::String &value)
{
NSString* convertJSIStringToNSString(jsi::Runtime& runtime, const jsi::String& value) {
return [NSString stringWithUTF8String:value.utf8(runtime).c_str()];
}
NSArray* convertJSICStyleArrayToNSArray(jsi::Runtime &runtime, const jsi::Value* array, size_t length, std::shared_ptr<CallInvoker> jsInvoker) {
if (length < 1) return @[];
NSArray* convertJSICStyleArrayToNSArray(jsi::Runtime& runtime, const jsi::Value* array,
size_t length, std::shared_ptr<CallInvoker> jsInvoker) {
if (length < 1)
return @[];
NSMutableArray* result = [NSMutableArray new];
for (size_t i = 0; i < length; i++) {
// Insert kCFNull when it's `undefined` value to preserve the indices.
[result
addObject:convertJSIValueToObjCObject(runtime, array[i], jsInvoker) ?: (id)kCFNull];
[result addObject:convertJSIValueToObjCObject(runtime, array[i], jsInvoker) ?: (id)kCFNull];
}
return [result copy];
}
@ -110,20 +105,21 @@ jsi::Value* convertNSArrayToJSICStyleArray(jsi::Runtime &runtime, NSArray* array
return result;
}
NSArray* convertJSIArrayToNSArray(jsi::Runtime &runtime, const jsi::Array &value, std::shared_ptr<CallInvoker> jsInvoker)
{
NSArray* convertJSIArrayToNSArray(jsi::Runtime& runtime, const jsi::Array& value,
std::shared_ptr<CallInvoker> jsInvoker) {
size_t size = value.size(runtime);
NSMutableArray* result = [NSMutableArray new];
for (size_t i = 0; i < size; i++) {
// Insert kCFNull when it's `undefined` value to preserve the indices.
[result
addObject:convertJSIValueToObjCObject(runtime, value.getValueAtIndex(runtime, i), jsInvoker) ?: (id)kCFNull];
addObject:convertJSIValueToObjCObject(runtime, value.getValueAtIndex(runtime, i), jsInvoker)
?: (id)kCFNull];
}
return [result copy];
}
NSDictionary* convertJSIObjectToNSDictionary(jsi::Runtime &runtime, const jsi::Object &value, std::shared_ptr<CallInvoker> jsInvoker)
{
NSDictionary* convertJSIObjectToNSDictionary(jsi::Runtime& runtime, const jsi::Object& value,
std::shared_ptr<CallInvoker> jsInvoker) {
jsi::Array propertyNames = value.getPropertyNames(runtime);
size_t size = propertyNames.size(runtime);
NSMutableDictionary* result = [NSMutableDictionary new];
@ -138,8 +134,8 @@ NSDictionary* convertJSIObjectToNSDictionary(jsi::Runtime &runtime, const jsi::O
return [result copy];
}
id convertJSIValueToObjCObject(jsi::Runtime &runtime, const jsi::Value &value, std::shared_ptr<CallInvoker> jsInvoker)
{
id convertJSIValueToObjCObject(jsi::Runtime& runtime, const jsi::Value& value,
std::shared_ptr<CallInvoker> jsInvoker) {
if (value.isUndefined() || value.isNull()) {
return nil;
}
@ -173,8 +169,9 @@ id convertJSIValueToObjCObject(jsi::Runtime &runtime, const jsi::Value &value, s
throw std::runtime_error("Unsupported jsi::jsi::Value kind");
}
RCTResponseSenderBlock convertJSIFunctionToCallback(jsi::Runtime &runtime, const jsi::Function &value, std::shared_ptr<CallInvoker> jsInvoker)
{
RCTResponseSenderBlock convertJSIFunctionToCallback(jsi::Runtime& runtime,
const jsi::Function& value,
std::shared_ptr<CallInvoker> jsInvoker) {
auto weakWrapper = CallbackWrapper::createWeak(value.getFunction(runtime), runtime, jsInvoker);
RCTBlockGuard* blockGuard = [[RCTBlockGuard alloc] initWithCleanup:^() {
auto strongWrapper = weakWrapper.lock();
@ -201,7 +198,8 @@ RCTResponseSenderBlock convertJSIFunctionToCallback(jsi::Runtime &runtime, const
}
const jsi::Value* args = convertNSArrayToJSICStyleArray(strongWrapper2->runtime(), responses);
strongWrapper2->callback().call(strongWrapper2->runtime(), args, static_cast<size_t>(responses.count));
strongWrapper2->callback().call(strongWrapper2->runtime(), args,
static_cast<size_t>(responses.count));
strongWrapper2->destroy();
delete[] args;
@ -215,4 +213,4 @@ RCTResponseSenderBlock convertJSIFunctionToCallback(jsi::Runtime &runtime, const
return [callback copy];
}
} // namespace JSIJNIObjectConversion
} // namespace JSINSObjectConversion

5
ios/Frame Processor/VisionCameraProxy.h
View File

@ -12,9 +12,9 @@
#import <React/RCTBridge.h>
#ifdef __cplusplus
#import <jsi/jsi.h>
#import "WKTJsiWorkletContext.h"
#import <ReactCommon/CallInvoker.h>
#import <jsi/jsi.h>
using namespace facebook;
@ -31,7 +31,8 @@ public:
private:
void setFrameProcessor(jsi::Runtime& runtime, int viewTag, const jsi::Object& frameProcessor);
void removeFrameProcessor(jsi::Runtime& runtime, int viewTag);
jsi::Value getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name, const jsi::Object& options);
jsi::Value getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name,
const jsi::Object& options);
private:
std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;

80
ios/Frame Processor/VisionCameraProxy.mm
View File

@ -10,18 +10,18 @@
#import <Foundation/Foundation.h>
#import <jsi/jsi.h>
#import "FrameProcessorPluginRegistry.h"
#import "FrameProcessorPluginHostObject.h"
#import "FrameProcessor.h"
#import "FrameHostObject.h"
#import "JSINSObjectConversion.h"
#import "../../cpp/JSITypedArray.h"
#import "FrameHostObject.h"
#import "FrameProcessor.h"
#import "FrameProcessorPluginHostObject.h"
#import "FrameProcessorPluginRegistry.h"
#import "JSINSObjectConversion.h"
#import "WKTJsiWorklet.h"
#import <React/RCTUtils.h>
#import <React/RCTBridge.h>
#import <React/RCTBridge+Private.h>
#import <React/RCTBridge.h>
#import <React/RCTUIManager.h>
#import <React/RCTUtils.h>
#import <ReactCommon/RCTTurboModuleManager.h>
// Swift forward-declarations
@ -48,15 +48,11 @@ VisionCameraProxy::VisionCameraProxy(jsi::Runtime& runtime,
};
auto runOnWorklet = [](std::function<void()>&& f) {
// Run on Frame Processor Worklet Runtime
dispatch_async(CameraQueues.videoQueue, [f = std::move(f)](){
f();
});
dispatch_async(CameraQueues.videoQueue, [f = std::move(f)]() { f(); });
};
_workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera",
&runtime,
runOnJS,
runOnWorklet);
_workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera", &runtime,
runOnJS, runOnWorklet);
NSLog(@"VisionCameraProxy: Worklet Context Created!");
}
@ -75,20 +71,23 @@ std::vector<jsi::PropNameID> VisionCameraProxy::getPropertyNames(jsi::Runtime& r
return result;
}
void VisionCameraProxy::setFrameProcessor(jsi::Runtime& runtime, int viewTag, const jsi::Object& object) {
void VisionCameraProxy::setFrameProcessor(jsi::Runtime& runtime, int viewTag,
const jsi::Object& object) {
auto frameProcessorType = object.getProperty(runtime, "type").asString(runtime).utf8(runtime);
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, object.getProperty(runtime, "frameProcessor"));
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(
runtime, object.getProperty(runtime, "frameProcessor"));
RCTExecuteOnMainQueue(^{
auto currentBridge = [RCTBridge currentBridge];
auto anonymousView = [currentBridge.uiManager viewForReactTag:[NSNumber numberWithDouble:viewTag]];
auto anonymousView =
[currentBridge.uiManager viewForReactTag:[NSNumber numberWithDouble:viewTag]];
auto view = static_cast<CameraView*>(anonymousView);
if (frameProcessorType == "frame-processor") {
view.frameProcessor = [[FrameProcessor alloc] initWithWorklet:worklet
context:_workletContext];
} else {
throw std::runtime_error("Unknown FrameProcessor.type passed! Received: " + frameProcessorType);
throw std::runtime_error("Unknown FrameProcessor.type passed! Received: " +
frameProcessorType);
}
});
}
@ -96,16 +95,20 @@ void VisionCameraProxy::setFrameProcessor(jsi::Runtime& runtime, int viewTag, co
void VisionCameraProxy::removeFrameProcessor(jsi::Runtime& runtime, int viewTag) {
RCTExecuteOnMainQueue(^{
auto currentBridge = [RCTBridge currentBridge];
auto anonymousView = [currentBridge.uiManager viewForReactTag:[NSNumber numberWithDouble:viewTag]];
auto anonymousView =
[currentBridge.uiManager viewForReactTag:[NSNumber numberWithDouble:viewTag]];
auto view = static_cast<CameraView*>(anonymousView);
view.frameProcessor = nil;
});
}
jsi::Value VisionCameraProxy::getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name, const jsi::Object& options) {
jsi::Value VisionCameraProxy::getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name,
const jsi::Object& options) {
NSString* key = [NSString stringWithUTF8String:name.c_str()];
NSDictionary* optionsObjc = JSINSObjectConversion::convertJSIObjectToNSDictionary(runtime, options, _callInvoker);
FrameProcessorPlugin* plugin = [FrameProcessorPluginRegistry getPlugin:key withOptions:optionsObjc];
NSDictionary* optionsObjc =
JSINSObjectConversion::convertJSIObjectToNSDictionary(runtime, options, _callInvoker);
FrameProcessorPlugin* plugin = [FrameProcessorPluginRegistry getPlugin:key
withOptions:optionsObjc];
if (plugin == nil) {
return jsi::Value::undefined();
}
@ -118,12 +121,9 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID&
auto name = propName.utf8(runtime);
if (name == "setFrameProcessor") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "setFrameProcessor"),
1,
[this](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "setFrameProcessor"), 1,
[this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
size_t count) -> jsi::Value {
auto viewTag = arguments[0].asNumber();
auto object = arguments[1].asObject(runtime);
@ -132,12 +132,9 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID&
});
}
if (name == "removeFrameProcessor") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "removeFrameProcessor"),
1,
[this](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "removeFrameProcessor"), 1,
[this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
size_t count) -> jsi::Value {
auto viewTag = arguments[0].asNumber();
this->removeFrameProcessor(runtime, static_cast<int>(viewTag));
@ -145,12 +142,9 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID&
});
}
if (name == "getFrameProcessorPlugin") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "getFrameProcessorPlugin"),
1,
[this](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "getFrameProcessorPlugin"), 1,
[this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
size_t count) -> jsi::Value {
if (count < 1 || !arguments[0].isString()) {
throw jsi::JSError(runtime, "First argument needs to be a string (pluginName)!");
@ -165,7 +159,6 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID&
return jsi::Value::undefined();
}
@implementation VisionCameraInstaller
+ (BOOL)installToBridge:(RCTBridge* _Nonnull)bridge {
RCTCxxBridge* cxxBridge = (RCTCxxBridge*)[RCTBridge currentBridge];
@ -177,8 +170,7 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID&
// global.VisionCameraProxy
auto visionCameraProxy = std::make_shared<VisionCameraProxy>(runtime, bridge.jsCallInvoker);
runtime.global().setProperty(runtime,
"VisionCameraProxy",
runtime.global().setProperty(runtime, "VisionCameraProxy",
jsi::Object::createFromHostObject(runtime, visionCameraProxy));
return YES;

2
ios/NativePreviewView.swift
View File

@ -1,5 +1,5 @@
//
// PreviewView.swift
// NativePreviewView.swift
// VisionCamera
//
// Created by Marc Rousavy on 30.11.22.

2
ios/Parsers/AVCaptureDevice.Position+descriptor.swift
View File

@ -1,5 +1,5 @@
//
// AVCaptureDevice.Position+String.swift
// AVCaptureDevice.Position+descriptor.swift
// mrousavy
//
// Created by Marc Rousavy on 15.12.20.

5
package.json
View File

@ -36,9 +36,10 @@
"release": "release-it",
"pods": "cd example && yarn pods",
"bootstrap": "yarn && cd example && yarn && yarn setup && yarn pods",
"check-android": "scripts/ktlint.sh && scripts/cpplint.sh",
"check-ios": "scripts/swiftformat.sh && scripts/swiftlint.sh",
"check-android": "scripts/ktlint.sh && scripts/clang-format.sh",
"check-ios": "scripts/swiftlint.sh && scripts/swiftformat.sh && scripts/clang-format.sh",
"check-js": "yarn lint --fix && yarn typescript",
"check-cpp": "scripts/clang-format.sh",
"check-all": "scripts/check-all.sh",
"clean-ios": "scripts/clean-ios.sh",
"clean-android": "scripts/clean-android.sh",

4
scripts/check-all.sh
View File

@ -9,8 +9,8 @@ echo "Linting Swift code.."
echo "Linting Kotlin code.."
./scripts/ktlint.sh
echo "Linting C++ code.."
./scripts/cpplint.sh
echo "Formatting C++ code.."
./scripts/clang-format.sh
echo "Linting JS/TS code.."
yarn lint --fix

10
scripts/clang-format.sh Executable file
View File

@ -0,0 +1,10 @@
#!/bin/bash
if which clang-format >/dev/null; then
find cpp ios android/src/main/cpp -type f \( -name "*.h" -o -name "*.cpp" -o -name "*.m" -o -name "*.mm" \) -print0 | while read -d $'\0' file; do
echo "-> cpp-lint $file"
clang-format -i "$file"
done
else
echo "warning: clang-format not installed, download from https://clang.llvm.org/docs/ClangFormat.html (or run brew install clang-format)"
fi

7
scripts/cpplint.sh
View File

@ -1,7 +0,0 @@
#!/bin/bash
if which cpplint >/dev/null; then
cpplint --linelength=230 --filter=-legal/copyright,-readability/todo,-build/namespaces,-runtime/references,-whitespace/comments,-build/include_order,-build/c++11 --quiet --recursive cpp android/src/main/cpp
else
echo "warning: cpplint not installed, download from https://github.com/cpplint/cpplint"
fi