railbird/react-native-vision-camera
9
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Use clang-format to keep the C++ codebase clean (#1741)

Browse Source 
* Use clang-format

* Create .clang-format

* Update .clang-format

* Update .clang-format

* Update .clang-format

* Only search in cpp dirs

* Update clang-format.sh

* Update .clang-format

* Update .clang-format

* Update .clang-format

* Format C++ code!

* Use version 16

* Update clang-format.sh

* Remove Shaders.ts

* fix: Lint Swift
This commit is contained in:
Marc Rousavy 2023-09-01 12:58:32 +02:00 committed by GitHub
parent 0a28454579
commit 03b57a7d27
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
61 changed files with 851 additions and 886 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
.clang-format Normal file
View File

@ -0,0 +1,26 @@
# Config for clang-format version 16
# Standard
BasedOnStyle: llvm
Standard: c++14
# Indentation
IndentWidth: 2
ColumnLimit: 100
# Includes
SortIncludes: true
SortUsingDeclarations: true
# Pointer and reference alignment
PointerAlignment: Left
ReferenceAlignment: Left
ReflowComments: true
# Line breaking options
BreakBeforeBraces: Attach
BreakConstructorInitializers: BeforeColon
AllowShortFunctionsOnASingleLine: Empty
IndentCaseLabels: true
NamespaceIndentation: Inner

28
.github/workflows/validate-cpp.yml vendored
View File

@ -8,29 +8,29 @@ on:
- '.github/workflows/validate-cpp.yml'
- 'cpp/**'
- 'android/src/main/cpp/**'
- 'ios/**'
pull_request:
paths:
- '.github/workflows/validate-cpp.yml'
- 'cpp/**'
- 'android/src/main/cpp/**'
- 'ios/**'
jobs:
lint:
name: cpplint
name: Check clang-format
runs-on: ubuntu-latest
strategy:
matrix:
path:
- 'cpp'
- 'android/src/main/cpp'
- 'ios'
steps:
- uses: actions/checkout@v2
- uses: reviewdog/action-cpplint@master
- name: Run clang-format style check
uses: jidicula/clang-format-action@v4.11.0
with:
github_token: ${{ secrets.github_token }}
reporter: github-pr-review
flags: --linelength=230
targets: --recursive cpp android/src/main/cpp
filter: "-legal/copyright\
,-readability/todo\
,-build/namespaces\
,-whitespace/comments\
,-runtime/references\
,-build/include_order\
,-build/c++11\
"
clang-format-version: '16'
check-path: ${{ matrix.path }}

64
android/src/main/cpp/OpenGLContext.cpp
View File

@ -8,8 +8,8 @@
#include <GLES2/gl2.h>
#include <GLES2/gl2ext.h>
#include <android/native_window.h>
#include <android/log.h>
#include <android/native_window.h>
#include "OpenGLError.h"
@ -49,29 +49,40 @@ void OpenGLContext::ensureOpenGL() {
if (display == EGL_NO_DISPLAY) {
__android_log_print(ANDROID_LOG_INFO, TAG, "Initializing EGLDisplay..");
display = eglGetDisplay(EGL_DEFAULT_DISPLAY);
if (display == EGL_NO_DISPLAY) throw OpenGLError("Failed to get default OpenGL Display!");
if (display == EGL_NO_DISPLAY)
throw OpenGLError("Failed to get default OpenGL Display!");
EGLint major;
EGLint minor;
successful = eglInitialize(display, &major, &minor);
if (!successful) throw OpenGLError("Failed to initialize OpenGL!");
if (!successful)
throw OpenGLError("Failed to initialize OpenGL!");
}
// EGLConfig
if (config == nullptr) {
__android_log_print(ANDROID_LOG_INFO, TAG, "Initializing EGLConfig..");
EGLint attributes[] = {EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT,
EGL_SURFACE_TYPE, EGL_WINDOW_BIT,
EGL_RED_SIZE, 8,
EGL_GREEN_SIZE, 8,
EGL_BLUE_SIZE, 8,
EGL_ALPHA_SIZE, 8,
EGL_DEPTH_SIZE, 0,
EGL_STENCIL_SIZE, 0,
EGLint attributes[] = {EGL_RENDERABLE_TYPE,
EGL_OPENGL_ES2_BIT,
EGL_SURFACE_TYPE,
EGL_WINDOW_BIT,
EGL_RED_SIZE,
8,
EGL_GREEN_SIZE,
8,
EGL_BLUE_SIZE,
8,
EGL_ALPHA_SIZE,
8,
EGL_DEPTH_SIZE,
0,
EGL_STENCIL_SIZE,
0,
EGL_NONE};
EGLint numConfigs;
successful = eglChooseConfig(display, attributes, &config, 1, &numConfigs);
if (!successful || numConfigs == 0) throw OpenGLError("Failed to choose OpenGL config!");
if (!successful || numConfigs == 0)
throw OpenGLError("Failed to choose OpenGL config!");
}
// EGLContext
@ -79,18 +90,18 @@ void OpenGLContext::ensureOpenGL() {
__android_log_print(ANDROID_LOG_INFO, TAG, "Initializing EGLContext..");
EGLint contextAttributes[] = {EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE};
context = eglCreateContext(display, config, nullptr, contextAttributes);
if (context == EGL_NO_CONTEXT) throw OpenGLError("Failed to create OpenGL context!");
if (context == EGL_NO_CONTEXT)
throw OpenGLError("Failed to create OpenGL context!");
}
// EGLSurface
if (offscreenSurface == EGL_NO_SURFACE) {
// If we don't have a surface at all
__android_log_print(ANDROID_LOG_INFO, TAG, "Initializing 1x1 offscreen pbuffer EGLSurface..");
EGLint attributes[] = {EGL_WIDTH, 1,
EGL_HEIGHT, 1,
EGL_NONE};
EGLint attributes[] = {EGL_WIDTH, 1, EGL_HEIGHT, 1, EGL_NONE};
offscreenSurface = eglCreatePbufferSurface(display, config, attributes);
if (offscreenSurface == EGL_NO_SURFACE) throw OpenGLError("Failed to create OpenGL Surface!");
if (offscreenSurface == EGL_NO_SURFACE)
throw OpenGLError("Failed to create OpenGL Surface!");
}
}
@ -99,21 +110,24 @@ void OpenGLContext::use() {
}
void OpenGLContext::use(EGLSurface surface) {
if (surface == EGL_NO_SURFACE) throw OpenGLError("Cannot render to a null Surface!");
if (surface == EGL_NO_SURFACE)
throw OpenGLError("Cannot render to a null Surface!");
// 1. Make sure the OpenGL context is initialized
this->ensureOpenGL();
// 2. Make the OpenGL context current
bool successful = eglMakeCurrent(display, surface, surface, context);
if (!successful || eglGetError() != EGL_SUCCESS) throw OpenGLError("Failed to use current OpenGL context!");
if (!successful || eglGetError() != EGL_SUCCESS)
throw OpenGLError("Failed to use current OpenGL context!");
// 3. Caller can now render to this surface
}
void OpenGLContext::flush() const {
bool successful = eglSwapBuffers(display, eglGetCurrentSurface(EGL_DRAW));
if (!successful || eglGetError() != EGL_SUCCESS) throw OpenGLError("Failed to swap OpenGL buffers!");
if (!successful || eglGetError() != EGL_SUCCESS)
throw OpenGLError("Failed to swap OpenGL buffers!");
}
OpenGLTexture OpenGLContext::createTexture(OpenGLTexture::Type type, int width, int height) {
@ -122,7 +136,8 @@ OpenGLTexture OpenGLContext::createTexture(OpenGLTexture::Type type, int width,
// 2. Make the OpenGL context current
bool successful = eglMakeCurrent(display, offscreenSurface, offscreenSurface, context);
if (!successful || eglGetError() != EGL_SUCCESS) throw OpenGLError("Failed to use current OpenGL context!");
if (!successful || eglGetError() != EGL_SUCCESS)
throw OpenGLError("Failed to use current OpenGL context!");
GLuint textureId;
glGenTextures(1, &textureId);
@ -142,12 +157,7 @@ OpenGLTexture OpenGLContext::createTexture(OpenGLTexture::Type type, int width,
glTexParameteri(target, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(target, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
return {
.id = textureId,
.target = target,
.width = width,
.height = height
};
return {.id = textureId, .target = target, .width = width, .height = height};
}
} // namespace vision

14
android/src/main/cpp/OpenGLContext.h
View File

@ -7,11 +7,11 @@
#include <EGL/egl.h>
#include <GLES2/gl2.h>
#include <memory>
#include <functional>
#include <memory>
#include "PassThroughShader.h"
#include "OpenGLTexture.h"
#include "PassThroughShader.h"
namespace vision {
@ -20,7 +20,7 @@ namespace vision {
* By default, it creates an off-screen PixelBuffer surface.
*/
class OpenGLContext {
public:
public:
/**
* Create a new instance of the OpenGLContext that draws to an off-screen PixelBuffer surface.
* This will not perform any OpenGL operations yet, and is therefore safe to call from any Thread.
@ -52,21 +52,21 @@ class OpenGLContext {
*/
OpenGLTexture createTexture(OpenGLTexture::Type type, int width, int height);
public:
public:
EGLDisplay display = EGL_NO_DISPLAY;
EGLContext context = EGL_NO_CONTEXT;
EGLSurface offscreenSurface = EGL_NO_SURFACE;
EGLConfig config = nullptr;
private:
private:
OpenGLContext() = default;
void destroy();
void ensureOpenGL();
private:
private:
PassThroughShader _passThroughShader;
private:
private:
static constexpr auto TAG = "OpenGLContext";
};

20
android/src/main/cpp/OpenGLError.h
View File

@ -4,27 +4,31 @@
#pragma once
#include <string>
#include <stdexcept>
#include <GLES2/gl2.h>
#include <stdexcept>
#include <string>
namespace vision {
inline std::string getEglErrorIfAny() {
EGLint error = glGetError();
if (error != GL_NO_ERROR) return " Error: " + std::to_string(error);
if (error != GL_NO_ERROR)
return " Error: " + std::to_string(error);
error = eglGetError();
if (error != EGL_SUCCESS) return " Error: " + std::to_string(error);
if (error != EGL_SUCCESS)
return " Error: " + std::to_string(error);
return "";
}
class OpenGLError: public std::runtime_error {
public:
explicit OpenGLError(const std::string&& message): std::runtime_error(message + getEglErrorIfAny()) {}
class OpenGLError : public std::runtime_error {
public:
explicit OpenGLError(const std::string&& message)
: std::runtime_error(message + getEglErrorIfAny()) {}
static inline void checkIfError(const std::string&& message) {
auto error = getEglErrorIfAny();
if (error.length() > 0) throw std::runtime_error(message + error);
if (error.length() > 0)
throw std::runtime_error(message + error);
}
};

6
android/src/main/cpp/OpenGLRenderer.cpp
View File

@ -8,8 +8,8 @@
#include <GLES2/gl2.h>
#include <GLES2/gl2ext.h>
#include <android/native_window.h>
#include <android/log.h>
#include <android/native_window.h>
#include <utility>
@ -17,7 +17,9 @@
namespace vision {
std::unique_ptr<OpenGLRenderer> OpenGLRenderer::CreateWithWindowSurface(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface) {
std::unique_ptr<OpenGLRenderer>
OpenGLRenderer::CreateWithWindowSurface(std::shared_ptr<OpenGLContext> context,
ANativeWindow* surface) {
return std::unique_ptr<OpenGLRenderer>(new OpenGLRenderer(std::move(context), surface));
}

23
android/src/main/cpp/OpenGLRenderer.h
View File

@ -4,11 +4,11 @@
#pragma once
#include "PassThroughShader.h"
#include <EGL/egl.h>
#include <GLES2/gl2.h>
#include <android/native_window.h>
#include <memory>
#include "PassThroughShader.h"
#include "OpenGLContext.h"
#include "OpenGLTexture.h"
@ -16,14 +16,16 @@
namespace vision {
class OpenGLRenderer {
public:
public:
/**
* Create a new instance of the OpenGLRenderer that draws to an on-screen window surface.
* This will not perform any OpenGL operations yet, and is therefore safe to call from any Thread.
*
* Note: The `surface` is considered moved, and the OpenGL context will release it when it is being deleted.
* Note: The `surface` is considered moved, and the OpenGL context will release it when it is
* being deleted.
*/
static std::unique_ptr<OpenGLRenderer> CreateWithWindowSurface(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface);
static std::unique_ptr<OpenGLRenderer>
CreateWithWindowSurface(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface);
/**
* Destroy the OpenGL Context. This needs to be called on the same thread that `use()` was called.
*/
@ -35,24 +37,25 @@ class OpenGLRenderer {
void renderTextureToSurface(const OpenGLTexture& texture, float* transformMatrix);
/**
* Destroys the OpenGL context. This needs to be called on the same thread that `use()` was called.
* After calling `destroy()`, it is legal to call `use()` again, which will re-construct everything.
* Destroys the OpenGL context. This needs to be called on the same thread that `use()` was
* called. After calling `destroy()`, it is legal to call `use()` again, which will re-construct
* everything.
*/
void destroy();
private:
private:
explicit OpenGLRenderer(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface);
private:
private:
int _width = 0, _height = 0;
std::shared_ptr<OpenGLContext> _context;
ANativeWindow* _outputSurface;
EGLSurface _surface = EGL_NO_SURFACE;
private:
private:
PassThroughShader _passThroughShader;
private:
private:
static constexpr auto TAG = "OpenGLRenderer";
};

20
android/src/main/cpp/PassThroughShader.cpp
View File

@ -3,11 +3,11 @@
//
#include "PassThroughShader.h"
#include "OpenGLError.h"
#include <EGL/egl.h>
#include <GLES2/gl2.h>
#include <GLES2/gl2ext.h>
#include <memory>
#include "OpenGLError.h"
#include <string>
namespace vision {
@ -50,19 +50,11 @@ void PassThroughShader::draw(const OpenGLTexture& texture, float* transformMatri
// 3. Pass all uniforms/attributes for vertex shader
glEnableVertexAttribArray(_vertexParameters.aPosition);
glVertexAttribPointer(_vertexParameters.aPosition,
2,
GL_FLOAT,
GL_FALSE,
sizeof(Vertex),
glVertexAttribPointer(_vertexParameters.aPosition, 2, GL_FLOAT, GL_FALSE, sizeof(Vertex),
reinterpret_cast<void*>(offsetof(Vertex, position)));
glEnableVertexAttribArray(_vertexParameters.aTexCoord);
glVertexAttribPointer(_vertexParameters.aTexCoord,
2,
GL_FLOAT,
GL_FALSE,
sizeof(Vertex),
glVertexAttribPointer(_vertexParameters.aTexCoord, 2, GL_FLOAT, GL_FALSE, sizeof(Vertex),
reinterpret_cast<void*>(offsetof(Vertex, texCoord)));
glUniformMatrix4fv(_vertexParameters.uTransformMatrix, 1, GL_FALSE, transformMatrix);
@ -78,7 +70,8 @@ void PassThroughShader::draw(const OpenGLTexture& texture, float* transformMatri
GLuint PassThroughShader::loadShader(GLenum shaderType, const char* shaderCode) {
GLuint shader = glCreateShader(shaderType);
if (shader == 0) throw OpenGLError("Failed to load shader!");
if (shader == 0)
throw OpenGLError("Failed to load shader!");
glShaderSource(shader, 1, &shaderCode, nullptr);
glCompileShader(shader);
@ -96,7 +89,8 @@ GLuint PassThroughShader::createProgram() {
GLuint fragmentShader = loadShader(GL_FRAGMENT_SHADER, FRAGMENT_SHADER);
GLuint program = glCreateProgram();
if (program == 0) throw OpenGLError("Failed to create pass-through program!");
if (program == 0)
throw OpenGLError("Failed to create pass-through program!");
glAttachShader(program, vertexShader);
OpenGLError::checkIfError("Failed to attach Vertex Shader!");

8
android/src/main/cpp/PassThroughShader.h
View File

@ -21,7 +21,7 @@ struct Vertex {
};
class PassThroughShader {
public:
public:
PassThroughShader() = default;
~PassThroughShader();
@ -31,12 +31,12 @@ class PassThroughShader {
*/
void draw(const OpenGLTexture& texture, float* transformMatrix);
private:
private:
// Loading
static GLuint loadShader(GLenum shaderType, const char* shaderCode);
static GLuint createProgram();
private:
private:
// Parameters
GLuint _programId = NO_SHADER;
GLuint _vertexBuffer = NO_BUFFER;
@ -49,7 +49,7 @@ class PassThroughShader {
GLint uTexture = NO_POSITION;
} _fragmentParameters;
private:
private:
// Statics
static constexpr Vertex VERTICES[] = {
{{-1.0f, -1.0f}, {0.0f, 0.0f}}, // bottom-left

32
android/src/main/cpp/VideoPipeline.cpp
View File

@ -5,25 +5,27 @@
#include "VideoPipeline.h"
#include "OpenGLError.h"
#include <android/native_window_jni.h>
#include <GLES2/gl2.h>
#include <GLES2/gl2ext.h>
#include <EGL/egl.h>
#include <EGL/eglext.h>
#include <GLES/gl.h>
#include <GLES2/gl2.h>
#include <GLES2/gl2ext.h>
#include <android/native_window_jni.h>
#include <chrono>
#include "OpenGLTexture.h"
#include "JFrameProcessor.h"
#include "OpenGLTexture.h"
namespace vision {
jni::local_ref<VideoPipeline::jhybriddata> VideoPipeline::initHybrid(jni::alias_ref<jhybridobject> jThis, int width, int height) {
jni::local_ref<VideoPipeline::jhybriddata>
VideoPipeline::initHybrid(jni::alias_ref<jhybridobject> jThis, int width, int height) {
return makeCxxInstance(jThis, width, height);
}
VideoPipeline::VideoPipeline(jni::alias_ref<jhybridobject> jThis, int width, int height): _javaPart(jni::make_global(jThis)) {
VideoPipeline::VideoPipeline(jni::alias_ref<jhybridobject> jThis, int width, int height)
: _javaPart(jni::make_global(jThis)) {
_width = width;
_height = height;
_context = OpenGLContext::CreateWithOffscreenSurface();
@ -43,7 +45,8 @@ VideoPipeline::~VideoPipeline() {
}
void VideoPipeline::removeFrameProcessorOutputSurface() {
if (_frameProcessorOutput) _frameProcessorOutput->destroy();
if (_frameProcessorOutput)
_frameProcessorOutput->destroy();
_frameProcessorOutput = nullptr;
}
@ -57,7 +60,8 @@ void VideoPipeline::setFrameProcessorOutputSurface(jobject surface) {
}
void VideoPipeline::removeRecordingSessionOutputSurface() {
if (_recordingSessionOutput) _recordingSessionOutput->destroy();
if (_recordingSessionOutput)
_recordingSessionOutput->destroy();
_recordingSessionOutput = nullptr;
}
@ -104,10 +108,14 @@ void VideoPipeline::onFrame(jni::alias_ref<jni::JArrayFloat> transformMatrixPara
void VideoPipeline::registerNatives() {
registerHybrid({
makeNativeMethod("initHybrid", VideoPipeline::initHybrid),
makeNativeMethod("setFrameProcessorOutputSurface", VideoPipeline::setFrameProcessorOutputSurface),
makeNativeMethod("removeFrameProcessorOutputSurface", VideoPipeline::removeFrameProcessorOutputSurface),
makeNativeMethod("setRecordingSessionOutputSurface", VideoPipeline::setRecordingSessionOutputSurface),
makeNativeMethod("removeRecordingSessionOutputSurface", VideoPipeline::removeRecordingSessionOutputSurface),
makeNativeMethod("setFrameProcessorOutputSurface",
VideoPipeline::setFrameProcessorOutputSurface),
makeNativeMethod("removeFrameProcessorOutputSurface",
VideoPipeline::removeFrameProcessorOutputSurface),
makeNativeMethod("setRecordingSessionOutputSurface",
VideoPipeline::setRecordingSessionOutputSurface),
makeNativeMethod("removeRecordingSessionOutputSurface",
VideoPipeline::removeRecordingSessionOutputSurface),
makeNativeMethod("getInputTextureId", VideoPipeline::getInputTextureId),
makeNativeMethod("onBeforeFrame", VideoPipeline::onBeforeFrame),
makeNativeMethod("onFrame", VideoPipeline::onFrame),

25
android/src/main/cpp/VideoPipeline.h
View File

@ -4,13 +4,13 @@
#pragma once
#include <jni.h>
#include <fbjni/fbjni.h>
#include "OpenGLContext.h"
#include "OpenGLRenderer.h"
#include "PassThroughShader.h"
#include <EGL/egl.h>
#include <android/native_window.h>
#include "PassThroughShader.h"
#include "OpenGLRenderer.h"
#include "OpenGLContext.h"
#include <fbjni/fbjni.h>
#include <jni.h>
#include <memory>
#include <optional>
@ -18,13 +18,14 @@ namespace vision {
using namespace facebook;
class VideoPipeline: public jni::HybridClass<VideoPipeline> {
public:
class VideoPipeline : public jni::HybridClass<VideoPipeline> {
public:
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/utils/VideoPipeline;";
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis, int width, int height);
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis, int width,
int height);
static void registerNatives();
public:
public:
~VideoPipeline();
// -> SurfaceTexture input
@ -42,11 +43,11 @@ class VideoPipeline: public jni::HybridClass<VideoPipeline> {
void onBeforeFrame();
void onFrame(jni::alias_ref<jni::JArrayFloat> transformMatrix);
private:
private:
// Private constructor. Use `create(..)` to create new instances.
explicit VideoPipeline(jni::alias_ref<jhybridobject> jThis, int width, int height);
private:
private:
// Input Surface Texture
std::optional<OpenGLTexture> _inputTexture = std::nullopt;
int _width = 0;
@ -57,7 +58,7 @@ class VideoPipeline: public jni::HybridClass<VideoPipeline> {
std::unique_ptr<OpenGLRenderer> _frameProcessorOutput = nullptr;
std::unique_ptr<OpenGLRenderer> _recordingSessionOutput = nullptr;
private:
private:
friend HybridBase;
jni::global_ref<javaobject> _javaPart;
static constexpr auto TAG = "VideoPipeline";

10
android/src/main/cpp/VisionCamera.cpp
View File

@ -1,12 +1,12 @@
#include <jni.h>
#include <fbjni/fbjni.h>
#include "JVisionCameraScheduler.h"
#include "JFrameProcessor.h"
#include "JVisionCameraProxy.h"
#include "VisionCameraProxy.h"
#include "JVisionCameraScheduler.h"
#include "VideoPipeline.h"
#include "VisionCameraProxy.h"
#include <fbjni/fbjni.h>
#include <jni.h>
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) {
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void*) {
return facebook::jni::initialize(vm, [] {
vision::VisionCameraInstaller::registerNatives();
vision::JVisionCameraProxy::registerNatives();

53
android/src/main/cpp/frameprocessor/FrameHostObject.cpp
View File

@ -9,14 +9,15 @@
#include "JSITypedArray.h"
#include <vector>
#include <string>
#include <vector>
namespace vision {
using namespace facebook;
FrameHostObject::FrameHostObject(const jni::alias_ref<JFrame::javaobject>& frame): frame(make_global(frame)) { }
FrameHostObject::FrameHostObject(const jni::alias_ref<JFrame::javaobject>& frame)
: frame(make_global(frame)) {}
FrameHostObject::~FrameHostObject() {
// Hermes' Garbage Collector (Hades GC) calls destructors on a separate Thread
@ -50,38 +51,29 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
auto name = propName.utf8(runtime);
if (name == "incrementRefCount") {
jsi::HostFunctionType incrementRefCount = [=](jsi::Runtime& runtime,
const jsi::Value& thisArg,
jsi::HostFunctionType incrementRefCount = [=](jsi::Runtime& runtime, const jsi::Value& thisArg,
const jsi::Value* args,
size_t count) -> jsi::Value {
// Increment retain count by one.
this->frame->incrementRefCount();
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "incrementRefCount"),
0,
incrementRefCount);
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "incrementRefCount"), 0, incrementRefCount);
}
if (name == "decrementRefCount") {
auto decrementRefCount = [=](jsi::Runtime& runtime,
const jsi::Value& thisArg,
const jsi::Value* args,
size_t count) -> jsi::Value {
auto decrementRefCount = [=](jsi::Runtime& runtime, const jsi::Value& thisArg,
const jsi::Value* args, size_t count) -> jsi::Value {
// Decrement retain count by one. If the retain count is zero, the Frame gets closed.
this->frame->decrementRefCount();
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "decrementRefCount"),
0,
decrementRefCount);
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "decrementRefCount"), 0, decrementRefCount);
}
if (name == "toString") {
jsi::HostFunctionType toString = [=](jsi::Runtime& runtime,
const jsi::Value& thisArg,
const jsi::Value* args,
size_t count) -> jsi::Value {
jsi::HostFunctionType toString = [=](jsi::Runtime& runtime, const jsi::Value& thisArg,
const jsi::Value* args, size_t count) -> jsi::Value {
if (!this->frame) {
return jsi::String::createFromUtf8(runtime, "[closed frame]");
}
@ -90,16 +82,16 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
auto str = std::to_string(width) + " x " + std::to_string(height) + " Frame";
return jsi::String::createFromUtf8(runtime, str);
};
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
}
if (name == "toArrayBuffer") {
jsi::HostFunctionType toArrayBuffer = [=](jsi::Runtime& runtime,
const jsi::Value& thisArg,
const jsi::Value* args,
size_t count) -> jsi::Value {
jsi::HostFunctionType toArrayBuffer = [=](jsi::Runtime& runtime, const jsi::Value& thisArg,
const jsi::Value* args, size_t count) -> jsi::Value {
auto buffer = this->frame->toByteBuffer();
if (!buffer->isDirect()) {
throw std::runtime_error("Failed to get byte content of Frame - array is not direct ByteBuffer!");
throw std::runtime_error(
"Failed to get byte content of Frame - array is not direct ByteBuffer!");
}
auto size = buffer->getDirectSize();
@ -110,8 +102,10 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
}
// Get from global JS cache
auto arrayBufferCache = runtime.global().getPropertyAsObject(runtime, ARRAYBUFFER_CACHE_PROP_NAME);
auto arrayBuffer = vision::getTypedArray(runtime, arrayBufferCache).get<vision::TypedArrayKind::Uint8ClampedArray>(runtime);
auto arrayBufferCache =
runtime.global().getPropertyAsObject(runtime, ARRAYBUFFER_CACHE_PROP_NAME);
auto arrayBuffer = vision::getTypedArray(runtime, arrayBufferCache)
.get<vision::TypedArrayKind::Uint8ClampedArray>(runtime);
if (arrayBuffer.size(runtime) != size) {
arrayBuffer = vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray>(runtime, size);
runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer);
@ -123,7 +117,8 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
return arrayBuffer;
};
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toArrayBuffer"), 0, toArrayBuffer);
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "toArrayBuffer"), 0, toArrayBuffer);
}
if (name == "isValid") {

16
android/src/main/cpp/frameprocessor/FrameHostObject.h
View File

@ -4,11 +4,11 @@
#pragma once
#include <jsi/jsi.h>
#include <jni.h>
#include <fbjni/fbjni.h>
#include <vector>
#include <jni.h>
#include <jsi/jsi.h>
#include <string>
#include <vector>
#include "JFrame.h"
@ -17,15 +17,15 @@ namespace vision {
using namespace facebook;
class JSI_EXPORT FrameHostObject : public jsi::HostObject {
public:
public:
explicit FrameHostObject(const jni::alias_ref<JFrame::javaobject>& frame);
~FrameHostObject();
public:
jsi::Value get(jsi::Runtime &, const jsi::PropNameID &name) override;
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime &rt) override;
public:
jsi::Value get(jsi::Runtime&, const jsi::PropNameID& name) override;
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& rt) override;
public:
public:
jni::global_ref<JFrame> frame;
};

23
android/src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.cpp
View File

@ -3,40 +3,41 @@
//
#include "FrameProcessorPluginHostObject.h"
#include <vector>
#include "FrameHostObject.h"
#include "JSIJNIConversion.h"
#include <string>
#include <vector>
namespace vision {
using namespace facebook;
std::vector<jsi::PropNameID> FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime& runtime) {
std::vector<jsi::PropNameID>
FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime& runtime) {
std::vector<jsi::PropNameID> result;
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("call")));
return result;
}
jsi::Value FrameProcessorPluginHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
jsi::Value FrameProcessorPluginHostObject::get(jsi::Runtime& runtime,
const jsi::PropNameID& propName) {
auto name = propName.utf8(runtime);
if (name == "call") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "call"),
2,
[=](jsi::Runtime &runtime,
const jsi::Value &thisValue,
const jsi::Value *arguments,
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "call"), 2,
[=](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
size_t count) -> jsi::Value {
// Frame is first argument
auto frameHostObject = arguments[0].asObject(runtime).asHostObject<FrameHostObject>(runtime);
auto frameHostObject =
arguments[0].asObject(runtime).asHostObject<FrameHostObject>(runtime);
auto frame = frameHostObject->frame;
// Options are second argument (possibly undefined)
local_ref<JMap<jstring, jobject>> options = nullptr;
if (count > 1) {
options = JSIJNIConversion::convertJSIObjectToJNIMap(runtime, arguments[1].asObject(runtime));
options =
JSIJNIConversion::convertJSIObjectToJNIMap(runtime, arguments[1].asObject(runtime));
}
// Call actual plugin

18
android/src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.h
View File

@ -4,27 +4,27 @@
#pragma once
#include <jsi/jsi.h>
#include "JFrameProcessorPlugin.h"
#include <memory>
#include <fbjni/fbjni.h>
#include <jsi/jsi.h>
#include <memory>
#include <vector>
namespace vision {
using namespace facebook;
class FrameProcessorPluginHostObject: public jsi::HostObject {
public:
explicit FrameProcessorPluginHostObject(jni::alias_ref<JFrameProcessorPlugin::javaobject> plugin):
_plugin(make_global(plugin)) { }
~FrameProcessorPluginHostObject() { }
class FrameProcessorPluginHostObject : public jsi::HostObject {
public:
explicit FrameProcessorPluginHostObject(jni::alias_ref<JFrameProcessorPlugin::javaobject> plugin)
: _plugin(make_global(plugin)) {}
~FrameProcessorPluginHostObject() {}
public:
public:
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& runtime) override;
jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& name) override;
private:
private:
jni::global_ref<JFrameProcessorPlugin::javaobject> _plugin;
};

38
android/src/main/cpp/frameprocessor/JSIJNIConversion.cpp
View File

@ -4,14 +4,14 @@
#include "JSIJNIConversion.h"
#include <jsi/jsi.h>
#include <jni.h>
#include <fbjni/fbjni.h>
#include <android/log.h>
#include <fbjni/fbjni.h>
#include <jni.h>
#include <jsi/jsi.h>
#include <memory>
#include <string>
#include <utility>
#include <memory>
#include "FrameHostObject.h"
#include "JFrame.h"
@ -20,7 +20,8 @@ namespace vision {
using namespace facebook;
jni::local_ref<jni::JMap<jstring, jobject>> JSIJNIConversion::convertJSIObjectToJNIMap(jsi::Runtime& runtime, const jsi::Object& object) {
jni::local_ref<jni::JMap<jstring, jobject>>
JSIJNIConversion::convertJSIObjectToJNIMap(jsi::Runtime& runtime, const jsi::Object& object) {
auto propertyNames = object.getPropertyNames(runtime);
auto size = propertyNames.size(runtime);
auto hashMap = jni::JHashMap<jstring, jobject>::create();
@ -34,25 +35,21 @@ jni::local_ref<jni::JMap<jstring, jobject>> JSIJNIConversion::convertJSIObjectTo
// null
hashMap->put(key, nullptr);
} else if (value.isBool()) {
// Boolean
auto boolean = value.getBool();
hashMap->put(key, jni::JBoolean::valueOf(boolean));
} else if (value.isNumber()) {
// Double
auto number = value.getNumber();
hashMap->put(key, jni::JDouble::valueOf(number));
} else if (value.isString()) {
// String
auto str = value.getString(runtime).utf8(runtime);
hashMap->put(key, jni::make_jstring(str));
} else if (value.isObject()) {
// Object
@ -60,7 +57,6 @@ jni::local_ref<jni::JMap<jstring, jobject>> JSIJNIConversion::convertJSIObjectTo
if (valueAsObject.isArray(runtime)) {
// List<Object>
} else if (valueAsObject.isHostObject(runtime)) {
throw std::runtime_error("You can't pass HostObjects here.");
} else {
@ -69,48 +65,47 @@ jni::local_ref<jni::JMap<jstring, jobject>> JSIJNIConversion::convertJSIObjectTo
auto map = convertJSIObjectToJNIMap(runtime, valueAsObject);
hashMap->put(key, map);
}
} else {
auto stringRepresentation = value.toString(runtime).utf8(runtime);
throw std::runtime_error("Failed to convert jsi::Value to JNI value - unsupported type!" + stringRepresentation);
throw std::runtime_error("Failed to convert jsi::Value to JNI value - unsupported type!" +
stringRepresentation);
}
}
return hashMap;
}
jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, const jni::local_ref<jobject>& object) {
jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime& runtime,
const jni::local_ref<jobject>& object) {
if (object == nullptr) {
// null
return jsi::Value::undefined();
} else if (object->isInstanceOf(jni::JBoolean::javaClassStatic())) {
// Boolean
static const auto getBooleanFunc = jni::findClassLocal("java/lang/Boolean")->getMethod<jboolean()>("booleanValue");
static const auto getBooleanFunc =
jni::findClassLocal("java/lang/Boolean")->getMethod<jboolean()>("booleanValue");
auto boolean = getBooleanFunc(object.get());
return jsi::Value(boolean == true);
} else if (object->isInstanceOf(jni::JDouble::javaClassStatic())) {
// Double
static const auto getDoubleFunc = jni::findClassLocal("java/lang/Double")->getMethod<jdouble()>("doubleValue");
static const auto getDoubleFunc =
jni::findClassLocal("java/lang/Double")->getMethod<jdouble()>("doubleValue");
auto d = getDoubleFunc(object.get());
return jsi::Value(d);
} else if (object->isInstanceOf(jni::JInteger::javaClassStatic())) {
// Integer
static const auto getIntegerFunc = jni::findClassLocal("java/lang/Integer")->getMethod<jint()>("intValue");
static const auto getIntegerFunc =
jni::findClassLocal("java/lang/Integer")->getMethod<jint()>("intValue");
auto i = getIntegerFunc(object.get());
return jsi::Value(i);
} else if (object->isInstanceOf(jni::JString::javaClassStatic())) {
// String
return jsi::String::createFromUtf8(runtime, object->toString());
} else if (object->isInstanceOf(JList<jobject>::javaClassStatic())) {
// List<E>
@ -124,7 +119,6 @@ jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, c
i++;
}
return result;
} else if (object->isInstanceOf(JMap<jstring, jobject>::javaClassStatic())) {
// Map<K, V>

12
android/src/main/cpp/frameprocessor/JSIJNIConversion.h
View File

@ -4,19 +4,21 @@
#pragma once
#include <jsi/jsi.h>
#include <jni.h>
#include <fbjni/fbjni.h>
#include <jni.h>
#include <jsi/jsi.h>
namespace vision {
namespace JSIJNIConversion {
using namespace facebook;
using namespace facebook;
jni::local_ref<jni::JMap<jstring, jobject>> convertJSIObjectToJNIMap(jsi::Runtime& runtime, const jsi::Object& object);
jni::local_ref<jni::JMap<jstring, jobject>> convertJSIObjectToJNIMap(jsi::Runtime& runtime,
const jsi::Object& object);
jsi::Value convertJNIObjectToJSIValue(jsi::Runtime& runtime, const jni::local_ref<jobject>& object);
jsi::Value convertJNIObjectToJSIValue(jsi::Runtime& runtime,
const jni::local_ref<jobject>& object);
} // namespace JSIJNIConversion

46
android/src/main/cpp/frameprocessor/VisionCameraProxy.cpp
View File

@ -12,12 +12,12 @@
#include <android/log.h>
#include <fbjni/fbjni.h>
#include "JSITypedArray.h"
#include "FrameProcessorPluginHostObject.h"
#include "JSITypedArray.h"
#include <vector>
#include <string>
#include <memory>
#include <string>
#include <vector>
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
#include <react-native-worklets-core/WKTJsiWorkletContext.h>
@ -27,12 +27,12 @@ namespace vision {
using namespace facebook;
VisionCameraProxy::VisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::javaobject>& javaProxy) {
VisionCameraProxy::VisionCameraProxy(
const jni::alias_ref<JVisionCameraProxy::javaobject>& javaProxy) {
_javaProxy = make_global(javaProxy);
}
VisionCameraProxy::~VisionCameraProxy() {
}
VisionCameraProxy::~VisionCameraProxy() {}
std::vector<jsi::PropNameID> VisionCameraProxy::getPropertyNames(jsi::Runtime& runtime) {
std::vector<jsi::PropNameID> result;
@ -42,7 +42,8 @@ std::vector<jsi::PropNameID> VisionCameraProxy::getPropertyNames(jsi::Runtime& r
return result;
}
void VisionCameraProxy::setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& object) {
void VisionCameraProxy::setFrameProcessor(int viewTag, jsi::Runtime& runtime,
const jsi::Object& object) {
_javaProxy->cthis()->setFrameProcessor(viewTag, runtime, object);
}
@ -65,12 +66,9 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID&
auto name = propName.utf8(runtime);
if (name == "setFrameProcessor") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "setFrameProcessor"),
1,
[this](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "setFrameProcessor"), 1,
[this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
size_t count) -> jsi::Value {
auto viewTag = arguments[0].asNumber();
auto object = arguments[1].asObject(runtime);
@ -79,12 +77,9 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID&
});
}
if (name == "removeFrameProcessor") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "removeFrameProcessor"),
1,
[this](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "removeFrameProcessor"), 1,
[this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
size_t count) -> jsi::Value {
auto viewTag = arguments[0].asNumber();
this->removeFrameProcessor(static_cast<int>(viewTag));
@ -92,12 +87,9 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID&
});
}
if (name == "getFrameProcessorPlugin") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "getFrameProcessorPlugin"),
1,
[this](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "getFrameProcessorPlugin"), 1,
[this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
size_t count) -> jsi::Value {
if (count < 1 || !arguments[0].isString()) {
throw jsi::JSError(runtime, "First argument needs to be a string (pluginName)!");
@ -112,14 +104,12 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID&
return jsi::Value::undefined();
}
void VisionCameraInstaller::install(jni::alias_ref<jni::JClass>,
jni::alias_ref<JVisionCameraProxy::javaobject> proxy) {
// global.VisionCameraProxy
auto visionCameraProxy = std::make_shared<VisionCameraProxy>(proxy);
jsi::Runtime& runtime = *proxy->cthis()->getJSRuntime();
runtime.global().setProperty(runtime,
"VisionCameraProxy",
runtime.global().setProperty(runtime, "VisionCameraProxy",
jsi::Object::createFromHostObject(runtime, visionCameraProxy));
}

30
android/src/main/cpp/frameprocessor/VisionCameraProxy.h
View File

@ -6,43 +6,43 @@
#include <jsi/jsi.h>
#include "JVisionCameraScheduler.h"
#include "JVisionCameraProxy.h"
#include "JVisionCameraScheduler.h"
#include <vector>
#include <string>
#include <vector>
namespace vision {
using namespace facebook;
class VisionCameraProxy: public jsi::HostObject {
public:
class VisionCameraProxy : public jsi::HostObject {
public:
explicit VisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::javaobject>& javaProxy);
~VisionCameraProxy();
public:
public:
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& runtime) override;
jsi::Value get(jsi::Runtime& runtime, const jsi::PropNameID& name) override;
private:
private:
void setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& frameProcessor);
void removeFrameProcessor(int viewTag);
jsi::Value getFrameProcessorPlugin(jsi::Runtime& runtime, const std::string& name, const jsi::Object& options);
jsi::Value getFrameProcessorPlugin(jsi::Runtime& runtime, const std::string& name,
const jsi::Object& options);
private:
private:
jni::global_ref<JVisionCameraProxy::javaobject> _javaProxy;
static constexpr const char* TAG = "VisionCameraProxy";
};
class VisionCameraInstaller: public jni::JavaClass<VisionCameraInstaller> {
public:
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraInstaller;";
class VisionCameraInstaller : public jni::JavaClass<VisionCameraInstaller> {
public:
static auto constexpr kJavaDescriptor =
"Lcom/mrousavy/camera/frameprocessor/VisionCameraInstaller;";
static void registerNatives() {
javaClassStatic()->registerNatives({
makeNativeMethod("install", VisionCameraInstaller::install)
});
javaClassStatic()->registerNatives(
{makeNativeMethod("install", VisionCameraInstaller::install)});
}
static void install(jni::alias_ref<jni::JClass> clazz,
jni::alias_ref<JVisionCameraProxy::javaobject> proxy);

4
android/src/main/cpp/frameprocessor/java-bindings/JFrame.cpp
View File

@ -4,9 +4,9 @@
#include "JFrame.h"
#include <jni.h>
#include <fbjni/fbjni.h>
#include <fbjni/ByteBuffer.h>
#include <fbjni/fbjni.h>
#include <jni.h>
namespace vision {

6
android/src/main/cpp/frameprocessor/java-bindings/JFrame.h
View File

@ -4,9 +4,9 @@
#pragma once
#include <jni.h>
#include <fbjni/fbjni.h>
#include <fbjni/ByteBuffer.h>
#include <fbjni/fbjni.h>
#include <jni.h>
namespace vision {
@ -16,7 +16,7 @@ using namespace jni;
struct JFrame : public JavaClass<JFrame> {
static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/Frame;";
public:
public:
int getWidth() const;
int getHeight() const;
bool getIsValid() const;

18
android/src/main/cpp/frameprocessor/java-bindings/JFrameProcessor.cpp
View File

@ -5,11 +5,11 @@
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
#include "JFrameProcessor.h"
#include <jni.h>
#include <fbjni/fbjni.h>
#include <jni.h>
#include <utility>
#include "JFrame.h"
#include <utility>
namespace vision {
@ -17,9 +17,7 @@ using namespace facebook;
using namespace jni;
void JFrameProcessor::registerNatives() {
registerHybrid({
makeNativeMethod("call", JFrameProcessor::call)
});
registerHybrid({makeNativeMethod("call", JFrameProcessor::call)});
}
using TSelf = jni::local_ref<JFrameProcessor::javaobject>;
@ -35,7 +33,8 @@ TSelf JFrameProcessor::create(const std::shared_ptr<RNWorklet::JsiWorklet>& work
return JFrameProcessor::newObjectCxxArgs(worklet, context);
}
void JFrameProcessor::callWithFrameHostObject(const std::shared_ptr<FrameHostObject>& frameHostObject) const {
void JFrameProcessor::callWithFrameHostObject(
const std::shared_ptr<FrameHostObject>& frameHostObject) const {
// Call the Frame Processor on the Worklet Runtime
jsi::Runtime& runtime = _workletContext->getWorkletRuntime();
@ -51,8 +50,11 @@ void JFrameProcessor::callWithFrameHostObject(const std::shared_ptr<FrameHostObj
const std::string& message = jsError.getMessage();
_workletContext->invokeOnJsThread([message](jsi::Runtime& jsRuntime) {
auto logFn = jsRuntime.global().getPropertyAsObject(jsRuntime, "console").getPropertyAsFunction(jsRuntime, "error");
logFn.call(jsRuntime, jsi::String::createFromUtf8(jsRuntime, "Frame Processor threw an error: " + message));
auto logFn = jsRuntime.global()
.getPropertyAsObject(jsRuntime, "console")
.getPropertyAsFunction(jsRuntime, "error");
logFn.call(jsRuntime, jsi::String::createFromUtf8(
jsRuntime, "Frame Processor threw an error: " + message));
});
}
}

23
android/src/main/cpp/frameprocessor/java-bindings/JFrameProcessor.h
View File

@ -6,43 +6,44 @@
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
#include <string>
#include <memory>
#include <jni.h>
#include <fbjni/fbjni.h>
#include <jni.h>
#include <memory>
#include <string>
#include <react-native-worklets-core/WKTJsiWorklet.h>
#include <react-native-worklets-core/WKTJsiHostObject.h>
#include <react-native-worklets-core/WKTJsiWorklet.h>
#include "JFrame.h"
#include "FrameHostObject.h"
#include "JFrame.h"
namespace vision {
using namespace facebook;
struct JFrameProcessor : public jni::HybridClass<JFrameProcessor> {
public:
public:
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessor;";
static void registerNatives();
static jni::local_ref<JFrameProcessor::javaobject> create(const std::shared_ptr<RNWorklet::JsiWorklet>& worklet,
static jni::local_ref<JFrameProcessor::javaobject>
create(const std::shared_ptr<RNWorklet::JsiWorklet>& worklet,
const std::shared_ptr<RNWorklet::JsiWorkletContext>& context);
public:
public:
/**
* Call the JS Frame Processor.
*/
void call(alias_ref<JFrame::javaobject> frame);
private:
private:
// Private constructor. Use `create(..)` to create new instances.
explicit JFrameProcessor(std::shared_ptr<RNWorklet::JsiWorklet> worklet,
std::shared_ptr<RNWorklet::JsiWorkletContext> context);
private:
private:
void callWithFrameHostObject(const std::shared_ptr<FrameHostObject>& frameHostObject) const;
private:
private:
friend HybridBase;
std::shared_ptr<RNWorklet::WorkletInvoker> _workletInvoker;
std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;

5
android/src/main/cpp/frameprocessor/java-bindings/JFrameProcessorPlugin.cpp
View File

@ -4,8 +4,8 @@
#include "JFrameProcessorPlugin.h"
#include <jni.h>
#include <fbjni/fbjni.h>
#include <jni.h>
namespace vision {
@ -14,7 +14,8 @@ using namespace jni;
using TCallback = jobject(alias_ref<JFrame::javaobject>, alias_ref<JMap<jstring, jobject>> params);
local_ref<jobject> JFrameProcessorPlugin::callback(const alias_ref<JFrame::javaobject>& frame,
local_ref<jobject>
JFrameProcessorPlugin::callback(const alias_ref<JFrame::javaobject>& frame,
const alias_ref<JMap<jstring, jobject>>& params) const {
auto callbackMethod = getClass()->getMethod<TCallback>("callback");

7
android/src/main/cpp/frameprocessor/java-bindings/JFrameProcessorPlugin.h
View File

@ -4,8 +4,8 @@
#pragma once
#include <jni.h>
#include <fbjni/fbjni.h>
#include <jni.h>
#include <string>
#include "JFrame.h"
@ -16,9 +16,10 @@ using namespace facebook;
using namespace jni;
struct JFrameProcessorPlugin : public JavaClass<JFrameProcessorPlugin> {
static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessorPlugin;";
static constexpr auto kJavaDescriptor =
"Lcom/mrousavy/camera/frameprocessor/FrameProcessorPlugin;";
public:
public:
/**
* Call the plugin.
*/

45
android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraProxy.cpp
View File

@ -5,8 +5,8 @@
#include "JVisionCameraProxy.h"
#include <memory>
#include <utility>
#include <string>
#include <utility>
#include <jsi/jsi.h>
@ -25,8 +25,8 @@ using TJSCallInvokerHolder = jni::alias_ref<facebook::react::CallInvokerHolder::
using TScheduler = jni::alias_ref<JVisionCameraScheduler::javaobject>;
using TOptions = jni::local_ref<JMap<jstring, jobject>>;
JVisionCameraProxy::JVisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::jhybridobject>& javaThis,
jsi::Runtime* runtime,
JVisionCameraProxy::JVisionCameraProxy(
const jni::alias_ref<JVisionCameraProxy::jhybridobject>& javaThis, jsi::Runtime* runtime,
const std::shared_ptr<facebook::react::CallInvoker>& callInvoker,
const jni::global_ref<JVisionCameraScheduler::javaobject>& scheduler) {
_javaPart = make_global(javaThis);
@ -41,13 +41,9 @@ JVisionCameraProxy::JVisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::
};
auto runOnWorklet = [scheduler](std::function<void()>&& f) {
// Run on Frame Processor Worklet Runtime
scheduler->cthis()->dispatchAsync([f = std::move(f)](){
f();
});
scheduler->cthis()->dispatchAsync([f = std::move(f)]() { f(); });
};
_workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera",
runtime,
runOnJS,
_workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera", runtime, runOnJS,
runOnWorklet);
__android_log_write(ANDROID_LOG_INFO, TAG, "Worklet Context created!");
#else
@ -64,14 +60,13 @@ JVisionCameraProxy::~JVisionCameraProxy() {
#endif
}
void JVisionCameraProxy::setFrameProcessor(int viewTag,
jsi::Runtime& runtime,
void JVisionCameraProxy::setFrameProcessor(int viewTag, jsi::Runtime& runtime,
const jsi::Object& frameProcessorObject) {
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
auto frameProcessorType = frameProcessorObject.getProperty(runtime, "type").asString(runtime).utf8(runtime);
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, frameProcessorObject.getProperty(runtime, "frameProcessor"));
auto frameProcessorType =
frameProcessorObject.getProperty(runtime, "type").asString(runtime).utf8(runtime);
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(
runtime, frameProcessorObject.getProperty(runtime, "frameProcessor"));
jni::local_ref<JFrameProcessor::javaobject> frameProcessor;
if (frameProcessorType == "frame-processor") {
@ -80,7 +75,9 @@ void JVisionCameraProxy::setFrameProcessor(int viewTag,
throw std::runtime_error("Unknown FrameProcessor.type passed! Received: " + frameProcessorType);
}
auto setFrameProcessorMethod = javaClassLocal()->getMethod<void(int, alias_ref<JFrameProcessor::javaobject>)>("setFrameProcessor");
auto setFrameProcessorMethod =
javaClassLocal()->getMethod<void(int, alias_ref<JFrameProcessor::javaobject>)>(
"setFrameProcessor");
setFrameProcessorMethod(_javaPart, viewTag, frameProcessor);
#else
throw std::runtime_error("system/frame-processors-unavailable: Frame Processors are disabled!");
@ -92,21 +89,19 @@ void JVisionCameraProxy::removeFrameProcessor(int viewTag) {
removeFrameProcessorMethod(_javaPart, viewTag);
}
local_ref<JFrameProcessorPlugin::javaobject> JVisionCameraProxy::getFrameProcessorPlugin(const std::string& name,
TOptions options) {
auto getFrameProcessorPluginMethod = javaClassLocal()->getMethod<JFrameProcessorPlugin(local_ref<jstring>, TOptions)>("getFrameProcessorPlugin");
local_ref<JFrameProcessorPlugin::javaobject>
JVisionCameraProxy::getFrameProcessorPlugin(const std::string& name, TOptions options) {
auto getFrameProcessorPluginMethod =
javaClassLocal()->getMethod<JFrameProcessorPlugin(local_ref<jstring>, TOptions)>(
"getFrameProcessorPlugin");
return getFrameProcessorPluginMethod(_javaPart, make_jstring(name), std::move(options));
}
void JVisionCameraProxy::registerNatives() {
registerHybrid({
makeNativeMethod("initHybrid", JVisionCameraProxy::initHybrid)
});
registerHybrid({makeNativeMethod("initHybrid", JVisionCameraProxy::initHybrid)});
}
TSelf JVisionCameraProxy::initHybrid(
alias_ref<jhybridobject> jThis,
jlong jsRuntimePointer,
TSelf JVisionCameraProxy::initHybrid(alias_ref<jhybridobject> jThis, jlong jsRuntimePointer,
TJSCallInvokerHolder jsCallInvokerHolder,
const TScheduler& scheduler) {
__android_log_write(ANDROID_LOG_INFO, TAG, "Initializing VisionCameraProxy...");

26
android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraProxy.h
View File

@ -4,16 +4,16 @@
#pragma once
#include <ReactCommon/CallInvokerHolder.h>
#include <fbjni/fbjni.h>
#include <jsi/jsi.h>
#include <ReactCommon/CallInvokerHolder.h>
#include "JFrameProcessor.h"
#include "JFrameProcessorPlugin.h"
#include "JVisionCameraScheduler.h"
#include "JFrameProcessor.h"
#include <string>
#include <memory>
#include <string>
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
#include <react-native-worklets-core/WKTJsiWorkletContext.h>
@ -24,20 +24,20 @@ namespace vision {
using namespace facebook;
class JVisionCameraProxy : public jni::HybridClass<JVisionCameraProxy> {
public:
public:
~JVisionCameraProxy();
static void registerNatives();
void setFrameProcessor(int viewTag,
jsi::Runtime& runtime,
const jsi::Object& frameProcessor);
void setFrameProcessor(int viewTag, jsi::Runtime& runtime, const jsi::Object& frameProcessor);
void removeFrameProcessor(int viewTag);
jni::local_ref<JFrameProcessorPlugin::javaobject> getFrameProcessorPlugin(const std::string& name,
jni::local_ref<JMap<jstring, jobject>> options);
jni::local_ref<JFrameProcessorPlugin::javaobject>
getFrameProcessorPlugin(const std::string& name, jni::local_ref<JMap<jstring, jobject>> options);
jsi::Runtime* getJSRuntime() { return _runtime; }
jsi::Runtime* getJSRuntime() {
return _runtime;
}
private:
private:
friend HybridBase;
jni::global_ref<JVisionCameraProxy::javaobject> _javaPart;
jsi::Runtime* _runtime;
@ -52,8 +52,8 @@ class JVisionCameraProxy : public jni::HybridClass<JVisionCameraProxy> {
jsi::Runtime* jsRuntime,
const std::shared_ptr<facebook::react::CallInvoker>& jsCallInvoker,
const jni::global_ref<JVisionCameraScheduler::javaobject>& scheduler);
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> javaThis,
jlong jsRuntimePointer,
static jni::local_ref<jhybriddata>
initHybrid(jni::alias_ref<jhybridobject> javaThis, jlong jsRuntimePointer,
jni::alias_ref<facebook::react::CallInvokerHolder::javaobject> jsCallInvokerHolder,
const jni::alias_ref<JVisionCameraScheduler::javaobject>& scheduler);
};

18
android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraScheduler.h
View File

@ -4,10 +4,10 @@
#pragma once
#include <jni.h>
#include <fbjni/fbjni.h>
#include <queue>
#include <jni.h>
#include <mutex>
#include <queue>
namespace vision {
@ -19,26 +19,28 @@ using namespace facebook;
*
* 1. Call `dispatchAsync(..)` with the given C++ Method.
* 2. Internally, `scheduleTrigger()` will get called, which is a Java Method.
* 3. The `scheduleTrigger()` Java Method will switch to the Frame Processor Java Thread and call `trigger()` on there
* 3. The `scheduleTrigger()` Java Method will switch to the Frame Processor Java Thread and call
* `trigger()` on there
* 4. `trigger()` is a C++ function here that just calls the passed C++ Method from step 1.
*/
class JVisionCameraScheduler : public jni::HybridClass<JVisionCameraScheduler> {
public:
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/VisionCameraScheduler;";
public:
static auto constexpr kJavaDescriptor =
"Lcom/mrousavy/camera/frameprocessor/VisionCameraScheduler;";
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis);
static void registerNatives();
// schedules the given job to be run on the VisionCamera FP Thread at some future point in time
void dispatchAsync(const std::function<void()>& job);
private:
private:
friend HybridBase;
jni::global_ref<JVisionCameraScheduler::javaobject> _javaPart;
std::queue<std::function<void()>> _jobs;
std::mutex _mutex;
explicit JVisionCameraScheduler(jni::alias_ref<JVisionCameraScheduler::jhybridobject> jThis):
_javaPart(jni::make_global(jThis)) {}
explicit JVisionCameraScheduler(jni::alias_ref<JVisionCameraScheduler::jhybridobject> jThis)
: _javaPart(jni::make_global(jThis)) {}
// Schedules a call to `trigger` on the VisionCamera FP Thread
void scheduleTrigger();

98
cpp/JSITypedArray.cpp
View File

@ -6,22 +6,22 @@
// Copyright © 2023 mrousavy. All rights reserved.
//
// Copied & Adapted from https://github.com/expo/expo/blob/main/packages/expo-gl/common/EXTypedArrayApi.cpp
// Credits to Expo
// Copied & Adapted from
// https://github.com/expo/expo/blob/main/packages/expo-gl/common/EXTypedArrayApi.cpp Credits to
// Expo
#include "JSITypedArray.h"
#include <unordered_map>
#include <algorithm>
#include <memory>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
#include <algorithm>
#include <string>
namespace vision {
template <TypedArrayKind T>
using ContentType = typename typedArrayTypeMap<T>::type;
template <TypedArrayKind T> using ContentType = typename typedArrayTypeMap<T>::type;
enum class Prop {
Buffer, // "buffer"
@ -45,8 +45,8 @@ enum class Prop {
};
class PropNameIDCache {
public:
const jsi::PropNameID &get(jsi::Runtime &runtime, Prop prop) {
public:
const jsi::PropNameID& get(jsi::Runtime& runtime, Prop prop) {
auto key = reinterpret_cast<uintptr_t>(&runtime);
if (this->props.find(key) == this->props.end()) {
this->props[key] = std::unordered_map<Prop, std::unique_ptr<jsi::PropNameID>>();
@ -57,7 +57,7 @@ class PropNameIDCache {
return *(this->props[key][prop]);
}
const jsi::PropNameID &getConstructorNameProp(jsi::Runtime &runtime, TypedArrayKind kind);
const jsi::PropNameID& getConstructorNameProp(jsi::Runtime& runtime, TypedArrayKind kind);
void invalidate(uintptr_t key) {
if (props.find(key) != props.end()) {
@ -65,10 +65,10 @@ class PropNameIDCache {
}
}
private:
private:
std::unordered_map<uintptr_t, std::unordered_map<Prop, std::unique_ptr<jsi::PropNameID>>> props;
jsi::PropNameID createProp(jsi::Runtime &runtime, Prop prop);
jsi::PropNameID createProp(jsi::Runtime& runtime, Prop prop);
};
PropNameIDCache propNameIDCache;
@ -78,22 +78,21 @@ void invalidateArrayBufferCache(jsi::Runtime& runtime) {
propNameIDCache.invalidate(key);
}
TypedArrayKind getTypedArrayKindForName(const std::string &name);
TypedArrayKind getTypedArrayKindForName(const std::string& name);
TypedArrayBase::TypedArrayBase(jsi::Runtime &runtime, size_t size, TypedArrayKind kind)
TypedArrayBase::TypedArrayBase(jsi::Runtime& runtime, size_t size, TypedArrayKind kind)
: TypedArrayBase(
runtime,
runtime.global()
runtime, runtime.global()
.getProperty(runtime, propNameIDCache.getConstructorNameProp(runtime, kind))
.asObject(runtime)
.asFunction(runtime)
.callAsConstructor(runtime, {static_cast<double>(size)})
.asObject(runtime)) {}
TypedArrayBase::TypedArrayBase(jsi::Runtime &runtime, const jsi::Object &obj)
TypedArrayBase::TypedArrayBase(jsi::Runtime& runtime, const jsi::Object& obj)
: jsi::Object(jsi::Value(runtime, obj).asObject(runtime)) {}
TypedArrayKind TypedArrayBase::getKind(jsi::Runtime &runtime) const {
TypedArrayKind TypedArrayBase::getKind(jsi::Runtime& runtime) const {
auto constructorName = this->getProperty(runtime, propNameIDCache.get(runtime, Prop::Constructor))
.asObject(runtime)
.getProperty(runtime, propNameIDCache.get(runtime, Prop::Name))
@ -102,34 +101,34 @@ TypedArrayKind TypedArrayBase::getKind(jsi::Runtime &runtime) const {
return getTypedArrayKindForName(constructorName);
}
size_t TypedArrayBase::size(jsi::Runtime &runtime) const {
size_t TypedArrayBase::size(jsi::Runtime& runtime) const {
return getProperty(runtime, propNameIDCache.get(runtime, Prop::Length)).asNumber();
}
size_t TypedArrayBase::length(jsi::Runtime &runtime) const {
size_t TypedArrayBase::length(jsi::Runtime& runtime) const {
return getProperty(runtime, propNameIDCache.get(runtime, Prop::Length)).asNumber();
}
size_t TypedArrayBase::byteLength(jsi::Runtime &runtime) const {
size_t TypedArrayBase::byteLength(jsi::Runtime& runtime) const {
return getProperty(runtime, propNameIDCache.get(runtime, Prop::ByteLength)).asNumber();
}
size_t TypedArrayBase::byteOffset(jsi::Runtime &runtime) const {
size_t TypedArrayBase::byteOffset(jsi::Runtime& runtime) const {
return getProperty(runtime, propNameIDCache.get(runtime, Prop::ByteOffset)).asNumber();
}
bool TypedArrayBase::hasBuffer(jsi::Runtime &runtime) const {
bool TypedArrayBase::hasBuffer(jsi::Runtime& runtime) const {
auto buffer = getProperty(runtime, propNameIDCache.get(runtime, Prop::Buffer));
return buffer.isObject() && buffer.asObject(runtime).isArrayBuffer(runtime);
}
std::vector<uint8_t> TypedArrayBase::toVector(jsi::Runtime &runtime) {
auto start = reinterpret_cast<uint8_t *>(getBuffer(runtime).data(runtime) + byteOffset(runtime));
std::vector<uint8_t> TypedArrayBase::toVector(jsi::Runtime& runtime) {
auto start = reinterpret_cast<uint8_t*>(getBuffer(runtime).data(runtime) + byteOffset(runtime));
auto end = start + byteLength(runtime);
return std::vector<uint8_t>(start, end);
}
jsi::ArrayBuffer TypedArrayBase::getBuffer(jsi::Runtime &runtime) const {
jsi::ArrayBuffer TypedArrayBase::getBuffer(jsi::Runtime& runtime) const {
auto buffer = getProperty(runtime, propNameIDCache.get(runtime, Prop::Buffer));
if (buffer.isObject() && buffer.asObject(runtime).isArrayBuffer(runtime)) {
return buffer.asObject(runtime).getArrayBuffer(runtime);
@ -138,7 +137,7 @@ jsi::ArrayBuffer TypedArrayBase::getBuffer(jsi::Runtime &runtime) const {
}
}
bool isTypedArray(jsi::Runtime &runtime, const jsi::Object &jsObj) {
bool isTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj) {
auto jsVal = runtime.global()
.getProperty(runtime, propNameIDCache.get(runtime, Prop::ArrayBuffer))
.asObject(runtime)
@ -153,7 +152,7 @@ bool isTypedArray(jsi::Runtime &runtime, const jsi::Object &jsObj) {
}
}
TypedArrayBase getTypedArray(jsi::Runtime &runtime, const jsi::Object &jsObj) {
TypedArrayBase getTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj) {
auto jsVal = runtime.global()
.getProperty(runtime, propNameIDCache.get(runtime, Prop::ArrayBuffer))
.asObject(runtime)
@ -168,24 +167,21 @@ TypedArrayBase getTypedArray(jsi::Runtime &runtime, const jsi::Object &jsObj) {
}
}
std::vector<uint8_t> arrayBufferToVector(jsi::Runtime &runtime, jsi::Object &jsObj) {
std::vector<uint8_t> arrayBufferToVector(jsi::Runtime& runtime, jsi::Object& jsObj) {
if (!jsObj.isArrayBuffer(runtime)) {
throw std::runtime_error("Object is not an ArrayBuffer");
}
auto jsArrayBuffer = jsObj.getArrayBuffer(runtime);
uint8_t *dataBlock = jsArrayBuffer.data(runtime);
uint8_t* dataBlock = jsArrayBuffer.data(runtime);
size_t blockSize =
jsArrayBuffer.getProperty(runtime, propNameIDCache.get(runtime, Prop::ByteLength)).asNumber();
return std::vector<uint8_t>(dataBlock, dataBlock + blockSize);
}
void arrayBufferUpdate(
jsi::Runtime &runtime,
jsi::ArrayBuffer &buffer,
std::vector<uint8_t> data,
void arrayBufferUpdate(jsi::Runtime& runtime, jsi::ArrayBuffer& buffer, std::vector<uint8_t> data,
size_t offset) {
uint8_t *dataBlock = buffer.data(runtime);
uint8_t* dataBlock = buffer.data(runtime);
size_t blockSize = buffer.size(runtime);
if (data.size() > blockSize) {
throw jsi::JSError(runtime, "ArrayBuffer is to small to fit data");
@ -194,50 +190,48 @@ void arrayBufferUpdate(
}
template <TypedArrayKind T>
TypedArray<T>::TypedArray(jsi::Runtime &runtime, size_t size) : TypedArrayBase(runtime, size, T) {}
TypedArray<T>::TypedArray(jsi::Runtime& runtime, size_t size) : TypedArrayBase(runtime, size, T) {}
template <TypedArrayKind T>
TypedArray<T>::TypedArray(jsi::Runtime &runtime, std::vector<ContentType<T>> data)
TypedArray<T>::TypedArray(jsi::Runtime& runtime, std::vector<ContentType<T>> data)
: TypedArrayBase(runtime, data.size(), T) {
update(runtime, data);
}
template <TypedArrayKind T>
TypedArray<T>::TypedArray(TypedArrayBase &&base) : TypedArrayBase(std::move(base)) {}
TypedArray<T>::TypedArray(TypedArrayBase&& base) : TypedArrayBase(std::move(base)) {}
template <TypedArrayKind T>
std::vector<ContentType<T>> TypedArray<T>::toVector(jsi::Runtime &runtime) {
std::vector<ContentType<T>> TypedArray<T>::toVector(jsi::Runtime& runtime) {
auto start =
reinterpret_cast<ContentType<T> *>(getBuffer(runtime).data(runtime) + byteOffset(runtime));
reinterpret_cast<ContentType<T>*>(getBuffer(runtime).data(runtime) + byteOffset(runtime));
auto end = start + size(runtime);
return std::vector<ContentType<T>>(start, end);
}
template <TypedArrayKind T>
void TypedArray<T>::update(jsi::Runtime &runtime, const std::vector<ContentType<T>> &data) {
void TypedArray<T>::update(jsi::Runtime& runtime, const std::vector<ContentType<T>>& data) {
if (data.size() != size(runtime)) {
throw jsi::JSError(runtime, "TypedArray can only be updated with a vector of the same size");
}
uint8_t *rawData = getBuffer(runtime).data(runtime) + byteOffset(runtime);
std::copy(data.begin(), data.end(), reinterpret_cast<ContentType<T> *>(rawData));
uint8_t* rawData = getBuffer(runtime).data(runtime) + byteOffset(runtime);
std::copy(data.begin(), data.end(), reinterpret_cast<ContentType<T>*>(rawData));
}
template <TypedArrayKind T>
void TypedArray<T>::updateUnsafe(jsi::Runtime &runtime, ContentType<T> *data, size_t length) {
void TypedArray<T>::updateUnsafe(jsi::Runtime& runtime, ContentType<T>* data, size_t length) {
if (length != size(runtime)) {
throw jsi::JSError(runtime, "TypedArray can only be updated with an array of the same size");
}
uint8_t *rawData = getBuffer(runtime).data(runtime) + byteOffset(runtime);
uint8_t* rawData = getBuffer(runtime).data(runtime) + byteOffset(runtime);
memcpy(rawData, data, length);
}
template <TypedArrayKind T>
uint8_t* TypedArray<T>::data(jsi::Runtime &runtime) {
template <TypedArrayKind T> uint8_t* TypedArray<T>::data(jsi::Runtime& runtime) {
return getBuffer(runtime).data(runtime) + byteOffset(runtime);
}
const jsi::PropNameID &PropNameIDCache::getConstructorNameProp(
jsi::Runtime &runtime,
const jsi::PropNameID& PropNameIDCache::getConstructorNameProp(jsi::Runtime& runtime,
TypedArrayKind kind) {
switch (kind) {
case TypedArrayKind::Int8Array:
@ -261,8 +255,8 @@ const jsi::PropNameID &PropNameIDCache::getConstructorNameProp(
}
}
jsi::PropNameID PropNameIDCache::createProp(jsi::Runtime &runtime, Prop prop) {
auto create = [&](const std::string &propName) {
jsi::PropNameID PropNameIDCache::createProp(jsi::Runtime& runtime, Prop prop) {
auto create = [&](const std::string& propName) {
return jsi::PropNameID::forUtf8(runtime, propName);
};
switch (prop) {
@ -317,7 +311,7 @@ std::unordered_map<std::string, TypedArrayKind> nameToKindMap = {
{"Float64Array", TypedArrayKind::Float64Array},
};
TypedArrayKind getTypedArrayKindForName(const std::string &name) {
TypedArrayKind getTypedArrayKindForName(const std::string& name) {
return nameToKindMap.at(name);
}

130
cpp/JSITypedArray.h
View File

@ -6,9 +6,8 @@
// Copyright © 2023 mrousavy. All rights reserved.
//
// Copied & Adapted from https://github.com/expo/expo/blob/main/packages/expo-gl/common/EXTypedArrayApi.h
// Credits to Expo
// Copied & Adapted from
// https://github.com/expo/expo/blob/main/packages/expo-gl/common/EXTypedArrayApi.h Credits to Expo
#pragma once
@ -32,134 +31,109 @@ enum class TypedArrayKind {
Float64Array,
};
template <TypedArrayKind T>
class TypedArray;
template <TypedArrayKind T> class TypedArray;
template <TypedArrayKind T>
struct typedArrayTypeMap;
template <>
struct typedArrayTypeMap<TypedArrayKind::Int8Array> {
template <TypedArrayKind T> struct typedArrayTypeMap;
template <> struct typedArrayTypeMap<TypedArrayKind::Int8Array> {
typedef int8_t type;
};
template <>
struct typedArrayTypeMap<TypedArrayKind::Int16Array> {
template <> struct typedArrayTypeMap<TypedArrayKind::Int16Array> {
typedef int16_t type;
};
template <>
struct typedArrayTypeMap<TypedArrayKind::Int32Array> {
template <> struct typedArrayTypeMap<TypedArrayKind::Int32Array> {
typedef int32_t type;
};
template <>
struct typedArrayTypeMap<TypedArrayKind::Uint8Array> {
template <> struct typedArrayTypeMap<TypedArrayKind::Uint8Array> {
typedef uint8_t type;
};
template <>
struct typedArrayTypeMap<TypedArrayKind::Uint8ClampedArray> {
template <> struct typedArrayTypeMap<TypedArrayKind::Uint8ClampedArray> {
typedef uint8_t type;
};
template <>
struct typedArrayTypeMap<TypedArrayKind::Uint16Array> {
template <> struct typedArrayTypeMap<TypedArrayKind::Uint16Array> {
typedef uint16_t type;
};
template <>
struct typedArrayTypeMap<TypedArrayKind::Uint32Array> {
template <> struct typedArrayTypeMap<TypedArrayKind::Uint32Array> {
typedef uint32_t type;
};
template <>
struct typedArrayTypeMap<TypedArrayKind::Float32Array> {
template <> struct typedArrayTypeMap<TypedArrayKind::Float32Array> {
typedef float type;
};
template <>
struct typedArrayTypeMap<TypedArrayKind::Float64Array> {
template <> struct typedArrayTypeMap<TypedArrayKind::Float64Array> {
typedef double type;
};
void invalidateArrayBufferCache(jsi::Runtime& runtime);
class TypedArrayBase : public jsi::Object {
public:
template <TypedArrayKind T>
using ContentType = typename typedArrayTypeMap<T>::type;
public:
template <TypedArrayKind T> using ContentType = typename typedArrayTypeMap<T>::type;
TypedArrayBase(jsi::Runtime &, size_t, TypedArrayKind);
TypedArrayBase(jsi::Runtime &, const jsi::Object &);
TypedArrayBase(TypedArrayBase &&) = default;
TypedArrayBase &operator=(TypedArrayBase &&) = default;
TypedArrayBase(jsi::Runtime&, size_t, TypedArrayKind);
TypedArrayBase(jsi::Runtime&, const jsi::Object&);
TypedArrayBase(TypedArrayBase&&) = default;
TypedArrayBase& operator=(TypedArrayBase&&) = default;
TypedArrayKind getKind(jsi::Runtime &runtime) const;
TypedArrayKind getKind(jsi::Runtime& runtime) const;
template <TypedArrayKind T>
TypedArray<T> get(jsi::Runtime &runtime) const &;
template <TypedArrayKind T>
TypedArray<T> get(jsi::Runtime &runtime) &&;
template <TypedArrayKind T>
TypedArray<T> as(jsi::Runtime &runtime) const &;
template <TypedArrayKind T>
TypedArray<T> as(jsi::Runtime &runtime) &&;
template <TypedArrayKind T> TypedArray<T> get(jsi::Runtime& runtime) const&;
template <TypedArrayKind T> TypedArray<T> get(jsi::Runtime& runtime) &&;
template <TypedArrayKind T> TypedArray<T> as(jsi::Runtime& runtime) const&;
template <TypedArrayKind T> TypedArray<T> as(jsi::Runtime& runtime) &&;
size_t size(jsi::Runtime &runtime) const;
size_t length(jsi::Runtime &runtime) const;
size_t byteLength(jsi::Runtime &runtime) const;
size_t byteOffset(jsi::Runtime &runtime) const;
bool hasBuffer(jsi::Runtime &runtime) const;
size_t size(jsi::Runtime& runtime) const;
size_t length(jsi::Runtime& runtime) const;
size_t byteLength(jsi::Runtime& runtime) const;
size_t byteOffset(jsi::Runtime& runtime) const;
bool hasBuffer(jsi::Runtime& runtime) const;
std::vector<uint8_t> toVector(jsi::Runtime &runtime);
jsi::ArrayBuffer getBuffer(jsi::Runtime &runtime) const;
std::vector<uint8_t> toVector(jsi::Runtime& runtime);
jsi::ArrayBuffer getBuffer(jsi::Runtime& runtime) const;
private:
template <TypedArrayKind>
friend class TypedArray;
private:
template <TypedArrayKind> friend class TypedArray;
};
bool isTypedArray(jsi::Runtime &runtime, const jsi::Object &jsObj);
TypedArrayBase getTypedArray(jsi::Runtime &runtime, const jsi::Object &jsObj);
bool isTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj);
TypedArrayBase getTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj);
std::vector<uint8_t> arrayBufferToVector(jsi::Runtime &runtime, jsi::Object &jsObj);
void arrayBufferUpdate(
jsi::Runtime &runtime,
jsi::ArrayBuffer &buffer,
std::vector<uint8_t> data,
std::vector<uint8_t> arrayBufferToVector(jsi::Runtime& runtime, jsi::Object& jsObj);
void arrayBufferUpdate(jsi::Runtime& runtime, jsi::ArrayBuffer& buffer, std::vector<uint8_t> data,
size_t offset);
template <TypedArrayKind T>
class TypedArray : public TypedArrayBase {
public:
explicit TypedArray(TypedArrayBase &&base);
TypedArray(jsi::Runtime &runtime, size_t size);
TypedArray(jsi::Runtime &runtime, std::vector<ContentType<T>> data);
TypedArray(TypedArray &&) = default;
TypedArray &operator=(TypedArray &&) = default;
template <TypedArrayKind T> class TypedArray : public TypedArrayBase {
public:
explicit TypedArray(TypedArrayBase&& base);
TypedArray(jsi::Runtime& runtime, size_t size);
TypedArray(jsi::Runtime& runtime, std::vector<ContentType<T>> data);
TypedArray(TypedArray&&) = default;
TypedArray& operator=(TypedArray&&) = default;
std::vector<ContentType<T>> toVector(jsi::Runtime &runtime);
void update(jsi::Runtime &runtime, const std::vector<ContentType<T>> &data);
void updateUnsafe(jsi::Runtime &runtime, ContentType<T> *data, size_t length);
uint8_t* data(jsi::Runtime &runtime);
std::vector<ContentType<T>> toVector(jsi::Runtime& runtime);
void update(jsi::Runtime& runtime, const std::vector<ContentType<T>>& data);
void updateUnsafe(jsi::Runtime& runtime, ContentType<T>* data, size_t length);
uint8_t* data(jsi::Runtime& runtime);
};
template <TypedArrayKind T>
TypedArray<T> TypedArrayBase::get(jsi::Runtime &runtime) const & {
template <TypedArrayKind T> TypedArray<T> TypedArrayBase::get(jsi::Runtime& runtime) const& {
assert(getKind(runtime) == T);
(void)runtime; // when assert is disabled we need to mark this as used
return TypedArray<T>(jsi::Value(runtime, jsi::Value(runtime, *this).asObject(runtime)));
}
template <TypedArrayKind T>
TypedArray<T> TypedArrayBase::get(jsi::Runtime &runtime) && {
template <TypedArrayKind T> TypedArray<T> TypedArrayBase::get(jsi::Runtime& runtime) && {
assert(getKind(runtime) == T);
(void)runtime; // when assert is disabled we need to mark this as used
return TypedArray<T>(std::move(*this));
}
template <TypedArrayKind T>
TypedArray<T> TypedArrayBase::as(jsi::Runtime &runtime) const & {
template <TypedArrayKind T> TypedArray<T> TypedArrayBase::as(jsi::Runtime& runtime) const& {
if (getKind(runtime) != T) {
throw jsi::JSError(runtime, "Object is not a TypedArray");
}
return get<T>(runtime);
}
template <TypedArrayKind T>
TypedArray<T> TypedArrayBase::as(jsi::Runtime &runtime) && {
template <TypedArrayKind T> TypedArray<T> TypedArrayBase::as(jsi::Runtime& runtime) && {
if (getKind(runtime) != T) {
throw jsi::JSError(runtime, "Object is not a TypedArray");
}

1
example/src/CameraPage.tsx
View File

@ -24,7 +24,6 @@ import IonIcon from 'react-native-vector-icons/Ionicons';
import type { Routes } from './Routes';
import type { NativeStackScreenProps } from '@react-navigation/native-stack';
import { useIsFocused } from '@react-navigation/core';
import { FACE_SHADER } from './Shaders';
import { examplePlugin } from './frame-processors/ExamplePlugin';
const ReanimatedCamera = Reanimated.createAnimatedComponent(Camera);

89
example/src/Shaders.ts
View File

@ -1,89 +0,0 @@
export const INVERTED_COLORS_SHADER = `
uniform shader image;
half4 main(vec2 pos) {
vec4 color = image.eval(pos);
return vec4(1.0 - color.rgb, 1.0);
}
`;
export const CHROMATIC_ABERRATION_SHADER = `
uniform shader image;
vec4 chromatic(vec2 pos, float offset) {
float r = image.eval(pos).r;
float g = image.eval(vec2(pos.x + offset, pos.y)).g;
float b = image.eval(vec2(pos.x + offset * 2.0, pos.y)).b;
return vec4(r, g, b, 1.0);
}
half4 main(vec2 pos) {
float offset = 50.0;
return chromatic(pos, offset);
}
`;
export const NO_SHADER = `
half4 main(vec2 pos) {
return vec4(1.0);
}
`;
export const BLUR_SHADER = `
const int samples = 35,
LOD = 2, // gaussian done on MIPmap at scale LOD
sLOD = 1 << LOD; // tile size = 2^LOD
const float sigma = float(samples) * .25;
float gaussian(vec2 i) {
return exp( -.5* dot(i/=sigma,i) ) / ( 6.28 * sigma*sigma );
}
vec4 blur(sampler2D sp, vec2 U, vec2 scale) {
vec4 O = vec4(0);
int s = samples/sLOD;
for ( int i = 0; i < s*s; i++ ) {
vec2 d = vec2(i%s, i/s)*float(sLOD) - float(samples)/2.;
O += gaussian(d) * textureLod( sp, U + scale * d , float(LOD) );
}
return O / O.a;
}
void mainImage(out vec4 O, vec2 U) {
O = blur( iChannel0, U/iResolution.xy, 1./iChannelResolution[0].xy );
}
`;
export const FACE_SHADER = `
uniform shader image;
uniform float x;
uniform float y;
uniform float r;
uniform vec2 resolution;
const float samples = 3.0;
const float radius = 40.0;
const float weight = 1.0;
half4 main(vec2 pos) {
float delta = pow((pow(pos.x - x, 2) + pow(pos.y - y, 2)), 0.5);
if (delta < r) {
vec3 sum = vec3(0.0);
vec3 accumulation = vec3(0);
vec3 weightedsum = vec3(0);
for (float deltaX = -samples * radius; deltaX <= samples * radius; deltaX += radius / samples) {
for (float deltaY = -samples * radius; deltaY <= samples * radius; deltaY += radius / samples) {
accumulation += image.eval(vec2(pos.x + deltaX, pos.y + deltaY)).rgb;
weightedsum += weight;
}
}
sum = accumulation / weightedsum;
return vec4(sum, 1.0);
}
else {
return image.eval(pos);
}
}
`;

6
ios/CameraBridge.h
View File

@ -10,13 +10,13 @@
#import <Foundation/Foundation.h>
#import <React/RCTViewManager.h>
#import <React/RCTUIManager.h>
#import <React/RCTFPSGraph.h>
#import <React/RCTLog.h>
#import <React/RCTUIManager.h>
#import <React/RCTViewManager.h>
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
#import "FrameProcessor.h"
#import "Frame.h"
#import "FrameProcessor.h"
#import "VisionCameraProxy.h"
#endif

2
ios/CameraView+Focus.swift
View File

@ -1,5 +1,5 @@
//
// CameraView+focus.swift
// CameraView+Focus.swift
// mrousavy
//
// Created by Marc Rousavy on 19.02.21.

2
ios/CameraView+Preview.swift
View File

@ -16,7 +16,7 @@ extension CameraView {
addSubview(previewView!)
}
internal func setupFpsGraph() {
func setupFpsGraph() {
#if DEBUG
if enableFpsGraph {
if fpsGraph != nil { return }

34
ios/CameraView.swift
View File

@ -74,30 +74,30 @@ public final class CameraView: UIView {
}
// pragma MARK: Internal Properties
internal var isMounted = false
internal var isReady = false
var isMounted = false
var isReady = false
// Capture Session
internal let captureSession = AVCaptureSession()
internal let audioCaptureSession = AVCaptureSession()
let captureSession = AVCaptureSession()
let audioCaptureSession = AVCaptureSession()
// Inputs & Outputs
internal var videoDeviceInput: AVCaptureDeviceInput?
internal var audioDeviceInput: AVCaptureDeviceInput?
internal var photoOutput: AVCapturePhotoOutput?
internal var videoOutput: AVCaptureVideoDataOutput?
internal var audioOutput: AVCaptureAudioDataOutput?
var videoDeviceInput: AVCaptureDeviceInput?
var audioDeviceInput: AVCaptureDeviceInput?
var photoOutput: AVCapturePhotoOutput?
var videoOutput: AVCaptureVideoDataOutput?
var audioOutput: AVCaptureAudioDataOutput?
// CameraView+RecordView (+ Frame Processor)
internal var isRecording = false
internal var recordingSession: RecordingSession?
var isRecording = false
var recordingSession: RecordingSession?
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
@objc public var frameProcessor: FrameProcessor?
#endif
// CameraView+Zoom
internal var pinchGestureRecognizer: UIPinchGestureRecognizer?
internal var pinchScaleOffset: CGFloat = 1.0
var pinchGestureRecognizer: UIPinchGestureRecognizer?
var pinchScaleOffset: CGFloat = 1.0
internal var previewView: PreviewView?
var previewView: PreviewView?
#if DEBUG
internal var fpsGraph: RCTFPSGraph?
var fpsGraph: RCTFPSGraph?
#endif
/// Returns whether the AVCaptureSession is currently running (reflected by isActive)
@ -258,7 +258,7 @@ public final class CameraView: UIView {
}
// pragma MARK: Event Invokers
internal final func invokeOnError(_ error: CameraError, cause: NSError? = nil) {
final func invokeOnError(_ error: CameraError, cause: NSError? = nil) {
ReactLogger.log(level: .error, message: "Invoking onError(): \(error.message)")
guard let onError = onError else { return }
@ -278,7 +278,7 @@ public final class CameraView: UIView {
])
}
internal final func invokeOnInitialized() {
final func invokeOnInitialized() {
ReactLogger.log(level: .info, message: "Camera initialized!")
guard let onInitialized = onInitialized else { return }
onInitialized([String: Any]())

56
ios/CameraViewManager.m
View File

@ -8,18 +8,28 @@
#import <Foundation/Foundation.h>
#import <React/RCTViewManager.h>
#import <React/RCTUtils.h>
#import <React/RCTViewManager.h>
@interface RCT_EXTERN_REMAP_MODULE(CameraView, CameraViewManager, RCTViewManager)
@interface RCT_EXTERN_REMAP_MODULE (CameraView, CameraViewManager, RCTViewManager)
// Module Functions
RCT_EXTERN_METHOD(getCameraPermissionStatus:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(getMicrophonePermissionStatus:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(requestCameraPermission:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(requestMicrophonePermission:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(getCameraPermissionStatus
: (RCTPromiseResolveBlock)resolve reject
: (RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(getMicrophonePermissionStatus
: (RCTPromiseResolveBlock)resolve reject
: (RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(requestCameraPermission
: (RCTPromiseResolveBlock)resolve reject
: (RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(requestMicrophonePermission
: (RCTPromiseResolveBlock)resolve reject
: (RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(getAvailableCameraDevices:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(getAvailableCameraDevices
: (RCTPromiseResolveBlock)resolve reject
: (RCTPromiseRejectBlock)reject);
// Camera View Properties
RCT_EXPORT_VIEW_PROPERTY(isActive, BOOL);
@ -51,12 +61,32 @@ RCT_EXPORT_VIEW_PROPERTY(onInitialized, RCTDirectEventBlock);
RCT_EXPORT_VIEW_PROPERTY(onViewReady, RCTDirectEventBlock);
// Camera View Functions
RCT_EXTERN_METHOD(startRecording:(nonnull NSNumber *)node options:(NSDictionary *)options onRecordCallback:(RCTResponseSenderBlock)onRecordCallback);
RCT_EXTERN_METHOD(pauseRecording:(nonnull NSNumber *)node resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(resumeRecording:(nonnull NSNumber *)node resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(stopRecording:(nonnull NSNumber *)node resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(takePhoto:(nonnull NSNumber *)node options:(NSDictionary *)options resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(focus:(nonnull NSNumber *)node point:(NSDictionary *)point resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(startRecording
: (nonnull NSNumber*)node options
: (NSDictionary*)options onRecordCallback
: (RCTResponseSenderBlock)onRecordCallback);
RCT_EXTERN_METHOD(pauseRecording
: (nonnull NSNumber*)node resolve
: (RCTPromiseResolveBlock)resolve reject
: (RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(resumeRecording
: (nonnull NSNumber*)node resolve
: (RCTPromiseResolveBlock)resolve reject
: (RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(stopRecording
: (nonnull NSNumber*)node resolve
: (RCTPromiseResolveBlock)resolve reject
: (RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(takePhoto
: (nonnull NSNumber*)node options
: (NSDictionary*)options resolve
: (RCTPromiseResolveBlock)resolve reject
: (RCTPromiseRejectBlock)reject);
RCT_EXTERN_METHOD(focus
: (nonnull NSNumber*)node point
: (NSDictionary*)point resolve
: (RCTPromiseResolveBlock)resolve reject
: (RCTPromiseRejectBlock)reject);
// Static Methods
RCT_EXTERN__BLOCKING_SYNCHRONOUS_METHOD(installFrameProcessorBindings);

9
ios/Frame Processor/Frame.h
View File

@ -8,15 +8,16 @@
#pragma once
#import <Foundation/Foundation.h>
#import <CoreMedia/CMSampleBuffer.h>
#import <Foundation/Foundation.h>
#import <UIKit/UIImage.h>
@interface Frame : NSObject
- (instancetype _Nonnull) initWithBuffer:(CMSampleBufferRef _Nonnull)buffer orientation:(UIImageOrientation)orientation;
- (instancetype _Nonnull)initWithBuffer:(CMSampleBufferRef _Nonnull)buffer
orientation:(UIImageOrientation)orientation;
@property (nonatomic, readonly) CMSampleBufferRef _Nonnull buffer;
@property (nonatomic, readonly) UIImageOrientation orientation;
@property(nonatomic, readonly) CMSampleBufferRef _Nonnull buffer;
@property(nonatomic, readonly) UIImageOrientation orientation;
@end

5
ios/Frame Processor/Frame.m
View File

@ -7,15 +7,16 @@
//
#import "Frame.h"
#import <Foundation/Foundation.h>
#import <CoreMedia/CMSampleBuffer.h>
#import <Foundation/Foundation.h>
@implementation Frame {
CMSampleBufferRef _Nonnull buffer;
UIImageOrientation orientation;
}
- (instancetype) initWithBuffer:(CMSampleBufferRef _Nonnull)buffer orientation:(UIImageOrientation)orientation {
- (instancetype)initWithBuffer:(CMSampleBufferRef _Nonnull)buffer
orientation:(UIImageOrientation)orientation {
self = [super init];
if (self) {
_buffer = buffer;

6
ios/Frame Processor/FrameHostObject.h
View File

@ -8,16 +8,16 @@
#pragma once
#import <jsi/jsi.h>
#import <CoreMedia/CMSampleBuffer.h>
#import <jsi/jsi.h>
#import "Frame.h"
using namespace facebook;
class JSI_EXPORT FrameHostObject: public jsi::HostObject {
class JSI_EXPORT FrameHostObject : public jsi::HostObject {
public:
explicit FrameHostObject(Frame* frame): frame(frame) {}
explicit FrameHostObject(Frame* frame) : frame(frame) {}
public:
jsi::Value get(jsi::Runtime&, const jsi::PropNameID& name) override;

51
ios/Frame Processor/FrameHostObject.mm
View File

@ -7,9 +7,9 @@
//
#import "FrameHostObject.h"
#import "WKTJsiHostObject.h"
#import <Foundation/Foundation.h>
#import <jsi/jsi.h>
#import "WKTJsiHostObject.h"
#import "../../cpp/JSITypedArray.h"
@ -46,10 +46,12 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
auto width = CVPixelBufferGetWidth(imageBuffer);
auto height = CVPixelBufferGetHeight(imageBuffer);
NSMutableString* string = [NSMutableString stringWithFormat:@"%lu x %lu Frame", width, height];
NSMutableString* string =
[NSMutableString stringWithFormat:@"%lu x %lu Frame", width, height];
return jsi::String::createFromUtf8(runtime, string.UTF8String);
};
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
}
if (name == "incrementRefCount") {
auto incrementRefCount = JSI_HOST_FUNCTION_LAMBDA {
@ -57,21 +59,18 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
CFRetain(frame.buffer);
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "incrementRefCount"),
0,
incrementRefCount);
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "incrementRefCount"), 0, incrementRefCount);
}
if (name == "decrementRefCount") {
auto decrementRefCount = JSI_HOST_FUNCTION_LAMBDA {
// Decrement retain count by one. If the retain count is zero, ARC will destroy the Frame Buffer.
// Decrement retain count by one. If the retain count is zero, ARC will destroy the Frame
// Buffer.
CFRelease(frame.buffer);
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "decrementRefCount"),
0,
decrementRefCount);
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "decrementRefCount"), 0, decrementRefCount);
}
if (name == "toArrayBuffer") {
auto toArrayBuffer = JSI_HOST_FUNCTION_LAMBDA {
@ -83,41 +82,47 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
static constexpr auto ARRAYBUFFER_CACHE_PROP_NAME = "__frameArrayBufferCache";
if (!runtime.global().hasProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME)) {
vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray> arrayBuffer(runtime, arraySize);
vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray> arrayBuffer(runtime,
arraySize);
runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer);
}
auto arrayBufferCache = runtime.global().getPropertyAsObject(runtime, ARRAYBUFFER_CACHE_PROP_NAME);
auto arrayBuffer = vision::getTypedArray(runtime, arrayBufferCache).get<vision::TypedArrayKind::Uint8ClampedArray>(runtime);
auto arrayBufferCache =
runtime.global().getPropertyAsObject(runtime, ARRAYBUFFER_CACHE_PROP_NAME);
auto arrayBuffer = vision::getTypedArray(runtime, arrayBufferCache)
.get<vision::TypedArrayKind::Uint8ClampedArray>(runtime);
if (arrayBuffer.size(runtime) != arraySize) {
arrayBuffer = vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray>(runtime, arraySize);
arrayBuffer =
vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray>(runtime, arraySize);
runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer);
}
CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
auto buffer = (uint8_t*) CVPixelBufferGetBaseAddress(pixelBuffer);
auto buffer = (uint8_t*)CVPixelBufferGetBaseAddress(pixelBuffer);
arrayBuffer.updateUnsafe(runtime, buffer, arraySize);
CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
return arrayBuffer;
};
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toArrayBuffer"), 0, toArrayBuffer);
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "toArrayBuffer"), 0, toArrayBuffer);
}
if (name == "isValid") {
auto isValid = frame != nil && frame.buffer != nil && CFGetRetainCount(frame.buffer) > 0 && CMSampleBufferIsValid(frame.buffer);
auto isValid = frame != nil && frame.buffer != nil && CFGetRetainCount(frame.buffer) > 0 &&
CMSampleBufferIsValid(frame.buffer);
return jsi::Value(isValid);
}
if (name == "width") {
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto width = CVPixelBufferGetWidth(imageBuffer);
return jsi::Value((double) width);
return jsi::Value((double)width);
}
if (name == "height") {
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto height = CVPixelBufferGetHeight(imageBuffer);
return jsi::Value((double) height);
return jsi::Value((double)height);
}
if (name == "orientation") {
switch (frame.orientation) {
@ -170,12 +175,12 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
if (name == "bytesPerRow") {
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer);
return jsi::Value((double) bytesPerRow);
return jsi::Value((double)bytesPerRow);
}
if (name == "planesCount") {
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto planesCount = CVPixelBufferGetPlaneCount(imageBuffer);
return jsi::Value((double) planesCount);
return jsi::Value((double)planesCount);
}
// fallback to base implementation

6
ios/Frame Processor/FrameProcessor.h
View File

@ -8,14 +8,14 @@
#pragma once
#import <Foundation/Foundation.h>
#import <AVFoundation/AVFoundation.h>
#import "Frame.h"
#import <AVFoundation/AVFoundation.h>
#import <Foundation/Foundation.h>
#ifdef __cplusplus
#import "FrameHostObject.h"
#import "WKTJsiWorklet.h"
#import <jsi/jsi.h>
#import "FrameHostObject.h"
#import <memory.h>
#endif

15
ios/Frame Processor/FrameProcessor.mm
View File

@ -6,13 +6,13 @@
// Copyright © 2023 mrousavy. All rights reserved.
//
#import <Foundation/Foundation.h>
#import "FrameProcessor.h"
#import <Foundation/Foundation.h>
#import <memory>
#import <jsi/jsi.h>
#import "WKTJsiWorklet.h"
#import "FrameHostObject.h"
#import "WKTJsiWorklet.h"
#import <jsi/jsi.h>
#import <memory>
using namespace facebook;
@ -46,8 +46,11 @@ using namespace facebook;
auto message = jsError.getMessage();
_workletContext->invokeOnJsThread([message](jsi::Runtime& jsRuntime) {
auto logFn = jsRuntime.global().getPropertyAsObject(jsRuntime, "console").getPropertyAsFunction(jsRuntime, "error");
logFn.call(jsRuntime, jsi::String::createFromUtf8(jsRuntime, "Frame Processor threw an error: " + message));
auto logFn = jsRuntime.global()
.getPropertyAsObject(jsRuntime, "console")
.getPropertyAsFunction(jsRuntime, "error");
logFn.call(jsRuntime, jsi::String::createFromUtf8(
jsRuntime, "Frame Processor threw an error: " + message));
});
}
}

14
ios/Frame Processor/FrameProcessorPlugin.h
View File

@ -8,19 +8,21 @@
#pragma once
#import <Foundation/Foundation.h>
#import "Frame.h"
#import <Foundation/Foundation.h>
/// The base class for a Frame Processor Plugin which can be called synchronously from a JS Frame Processor.
/// The base class for a Frame Processor Plugin which can be called synchronously from a JS Frame
/// Processor.
///
/// Subclass this class in a Swift or Objective-C class and override the `callback:withArguments:` method, and
/// implement your Frame Processing there.
/// Subclass this class in a Swift or Objective-C class and override the `callback:withArguments:`
/// method, and implement your Frame Processing there.
///
/// Use `[FrameProcessorPluginRegistry addFrameProcessorPlugin:]` to register the Plugin to the VisionCamera Runtime.
/// Use `[FrameProcessorPluginRegistry addFrameProcessorPlugin:]` to register the Plugin to the
/// VisionCamera Runtime.
@interface FrameProcessorPlugin : NSObject
/// The actual callback when calling this plugin. Any Frame Processing should be handled there.
/// Make sure your code is optimized, as this is a hot path.
- (id _Nullable) callback:(Frame* _Nonnull)frame withArguments:(NSDictionary* _Nullable)arguments;
- (id _Nullable)callback:(Frame* _Nonnull)frame withArguments:(NSDictionary* _Nullable)arguments;
@end

6
ios/Frame Processor/FrameProcessorPlugin.m
View File

@ -12,8 +12,10 @@
@implementation FrameProcessorPlugin
- (id _Nullable)callback:(Frame* _Nonnull)frame withArguments:(NSDictionary* _Nullable)arguments {
[NSException raise:NSInternalInconsistencyException
format:@"Frame Processor Plugin does not override the `callback(frame:withArguments:)` method!"];
[NSException
raise:NSInternalInconsistencyException
format:
@"Frame Processor Plugin does not override the `callback(frame:withArguments:)` method!"];
return nil;
}

12
ios/Frame Processor/FrameProcessorPluginHostObject.h
View File

@ -8,19 +8,19 @@
#pragma once
#import <jsi/jsi.h>
#import "FrameProcessorPlugin.h"
#import <memory>
#import <ReactCommon/CallInvoker.h>
#import <jsi/jsi.h>
#import <memory>
using namespace facebook;
class FrameProcessorPluginHostObject: public jsi::HostObject {
class FrameProcessorPluginHostObject : public jsi::HostObject {
public:
explicit FrameProcessorPluginHostObject(FrameProcessorPlugin* plugin,
std::shared_ptr<react::CallInvoker> callInvoker):
_plugin(plugin), _callInvoker(callInvoker) { }
~FrameProcessorPluginHostObject() { }
std::shared_ptr<react::CallInvoker> callInvoker)
: _plugin(plugin), _callInvoker(callInvoker) {}
~FrameProcessorPluginHostObject() {}
public:
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& runtime) override;

25
ios/Frame Processor/FrameProcessorPluginHostObject.mm
View File

@ -7,39 +7,40 @@
//
#import "FrameProcessorPluginHostObject.h"
#import <Foundation/Foundation.h>
#import <vector>
#import "FrameHostObject.h"
#import "JSINSObjectConversion.h"
#import <Foundation/Foundation.h>
#import <vector>
using namespace facebook;
std::vector<jsi::PropNameID> FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime& runtime) {
std::vector<jsi::PropNameID>
FrameProcessorPluginHostObject::getPropertyNames(jsi::Runtime& runtime) {
std::vector<jsi::PropNameID> result;
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("call")));
return result;
}
jsi::Value FrameProcessorPluginHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
jsi::Value FrameProcessorPluginHostObject::get(jsi::Runtime& runtime,
const jsi::PropNameID& propName) {
auto name = propName.utf8(runtime);
if (name == "call") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "call"),
2,
[=](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "call"), 2,
[=](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
size_t count) -> jsi::Value {
// Frame is first argument
auto frameHostObject = arguments[0].asObject(runtime).asHostObject<FrameHostObject>(runtime);
auto frameHostObject =
arguments[0].asObject(runtime).asHostObject<FrameHostObject>(runtime);
Frame* frame = frameHostObject->frame;
// Options are second argument (possibly undefined)
NSDictionary* options = nil;
if (count > 1) {
auto optionsObject = arguments[1].asObject(runtime);
options = JSINSObjectConversion::convertJSIObjectToNSDictionary(runtime, optionsObject, _callInvoker);
options = JSINSObjectConversion::convertJSIObjectToNSDictionary(runtime, optionsObject,
_callInvoker);
}
// Call actual Frame Processor Plugin

5
ios/Frame Processor/FrameProcessorPluginRegistry.h
View File

@ -8,13 +8,14 @@
#pragma once
#import <Foundation/Foundation.h>
#import "Frame.h"
#import "FrameProcessorPlugin.h"
#import <Foundation/Foundation.h>
@interface FrameProcessorPluginRegistry : NSObject
typedef FrameProcessorPlugin* _Nonnull (^PluginInitializerFunction)(NSDictionary* _Nullable options);
typedef FrameProcessorPlugin* _Nonnull (^PluginInitializerFunction)(
NSDictionary* _Nullable options);
+ (void)addFrameProcessorPlugin:(NSString* _Nonnull)name
withInitializer:(PluginInitializerFunction _Nonnull)pluginInitializer;

18
ios/Frame Processor/FrameProcessorPluginRegistry.m
View File

@ -19,15 +19,23 @@
return plugins;
}
+ (void) addFrameProcessorPlugin:(NSString *)name withInitializer:(PluginInitializerFunction)pluginInitializer {
BOOL alreadyExists = [[FrameProcessorPluginRegistry frameProcessorPlugins] valueForKey:name] != nil;
NSAssert(!alreadyExists, @"Tried to add a Frame Processor Plugin with a name that already exists! Either choose unique names, or remove the unused plugin. Name: %@", name);
+ (void)addFrameProcessorPlugin:(NSString*)name
withInitializer:(PluginInitializerFunction)pluginInitializer {
BOOL alreadyExists =
[[FrameProcessorPluginRegistry frameProcessorPlugins] valueForKey:name] != nil;
NSAssert(!alreadyExists,
@"Tried to add a Frame Processor Plugin with a name that already exists! Either choose "
@"unique names, or "
@"remove the unused plugin. Name: %@",
name);
[[FrameProcessorPluginRegistry frameProcessorPlugins] setValue:pluginInitializer forKey:name];
}
+ (FrameProcessorPlugin*)getPlugin:(NSString* _Nonnull)name withOptions:(NSDictionary* _Nullable)options {
PluginInitializerFunction initializer = [[FrameProcessorPluginRegistry frameProcessorPlugins] objectForKey:name];
+ (FrameProcessorPlugin*)getPlugin:(NSString* _Nonnull)name
withOptions:(NSDictionary* _Nullable)options {
PluginInitializerFunction initializer =
[[FrameProcessorPluginRegistry frameProcessorPlugins] objectForKey:name];
if (initializer == nil) {
return nil;
}

20
ios/Frame Processor/JSINSObjectConversion.h
View File

@ -8,9 +8,9 @@
#pragma once
#import <jsi/jsi.h>
#import <ReactCommon/CallInvoker.h>
#import <React/RCTBridgeModule.h>
#import <ReactCommon/CallInvoker.h>
#import <jsi/jsi.h>
namespace JSINSObjectConversion {
@ -39,21 +39,27 @@ jsi::Value convertObjCObjectToJSIValue(jsi::Runtime& runtime, id value);
NSString* convertJSIStringToNSString(jsi::Runtime& runtime, const jsi::String& value);
// any... -> NSArray
NSArray* convertJSICStyleArrayToNSArray(jsi::Runtime& runtime, const jsi::Value* array, size_t length, std::shared_ptr<CallInvoker> jsInvoker);
NSArray* convertJSICStyleArrayToNSArray(jsi::Runtime& runtime, const jsi::Value* array,
size_t length, std::shared_ptr<CallInvoker> jsInvoker);
// NSArray -> any...
jsi::Value* convertNSArrayToJSICStyleArray(jsi::Runtime& runtime, NSArray* array);
// [] -> NSArray
NSArray* convertJSIArrayToNSArray(jsi::Runtime& runtime, const jsi::Array& value, std::shared_ptr<CallInvoker> jsInvoker);
NSArray* convertJSIArrayToNSArray(jsi::Runtime& runtime, const jsi::Array& value,
std::shared_ptr<CallInvoker> jsInvoker);
// {} -> NSDictionary
NSDictionary* convertJSIObjectToNSDictionary(jsi::Runtime& runtime, const jsi::Object& value, std::shared_ptr<CallInvoker> jsInvoker);
NSDictionary* convertJSIObjectToNSDictionary(jsi::Runtime& runtime, const jsi::Object& value,
std::shared_ptr<CallInvoker> jsInvoker);
// any -> id
id convertJSIValueToObjCObject(jsi::Runtime& runtime, const jsi::Value& value, std::shared_ptr<CallInvoker> jsInvoker);
id convertJSIValueToObjCObject(jsi::Runtime& runtime, const jsi::Value& value,
std::shared_ptr<CallInvoker> jsInvoker);
// (any...) => any -> (void)(id, id)
RCTResponseSenderBlock convertJSIFunctionToCallback(jsi::Runtime& runtime, const jsi::Function& value, std::shared_ptr<CallInvoker> jsInvoker);
RCTResponseSenderBlock convertJSIFunctionToCallback(jsi::Runtime& runtime,
const jsi::Function& value,
std::shared_ptr<CallInvoker> jsInvoker);
} // namespace JSINSObjectConversion

96
ios/Frame Processor/JSINSObjectConversion.mm
View File

@ -5,7 +5,8 @@
// Forked and Adjusted by Marc Rousavy on 02.05.21.
// Copyright © 2021 mrousavy & Facebook. All rights reserved.
//
// Forked and adjusted from: https://github.com/facebook/react-native/blob/900210cacc4abca0079e3903781bc223c80c8ac7/ReactCommon/react/nativemodule/core/platform/ios/RCTTurboModule.mm
// Forked and adjusted from:
// https://github.com/facebook/react-native/blob/900210cacc4abca0079e3903781bc223c80c8ac7/ReactCommon/react/nativemodule/core/platform/ios/RCTTurboModule.mm
// Original Copyright Notice:
//
// Copyright (c) Facebook, Inc. and its affiliates.
@ -15,46 +16,41 @@
//
#import "JSINSObjectConversion.h"
#import <Foundation/Foundation.h>
#import <jsi/jsi.h>
#import <ReactCommon/CallInvoker.h>
#import <React/RCTBridge.h>
#import <ReactCommon/TurboModuleUtils.h>
#import <ReactCommon/RCTBlockGuard.h>
#import "../Frame Processor/Frame.h"
#import "../Frame Processor/FrameHostObject.h"
#import <Foundation/Foundation.h>
#import <React/RCTBridge.h>
#import <ReactCommon/CallInvoker.h>
#import <ReactCommon/RCTBlockGuard.h>
#import <ReactCommon/TurboModuleUtils.h>
#import <jsi/jsi.h>
using namespace facebook;
using namespace facebook::react;
namespace JSINSObjectConversion {
jsi::Value convertNSNumberToJSIBoolean(jsi::Runtime &runtime, NSNumber *value)
{
jsi::Value convertNSNumberToJSIBoolean(jsi::Runtime& runtime, NSNumber* value) {
return jsi::Value((bool)[value boolValue]);
}
jsi::Value convertNSNumberToJSINumber(jsi::Runtime &runtime, NSNumber *value)
{
jsi::Value convertNSNumberToJSINumber(jsi::Runtime& runtime, NSNumber* value) {
return jsi::Value([value doubleValue]);
}
jsi::String convertNSStringToJSIString(jsi::Runtime &runtime, NSString *value)
{
jsi::String convertNSStringToJSIString(jsi::Runtime& runtime, NSString* value) {
return jsi::String::createFromUtf8(runtime, [value UTF8String] ?: "");
}
jsi::Object convertNSDictionaryToJSIObject(jsi::Runtime &runtime, NSDictionary *value)
{
jsi::Object convertNSDictionaryToJSIObject(jsi::Runtime& runtime, NSDictionary* value) {
jsi::Object result = jsi::Object(runtime);
for (NSString *k in value) {
for (NSString* k in value) {
result.setProperty(runtime, [k UTF8String], convertObjCObjectToJSIValue(runtime, value[k]));
}
return result;
}
jsi::Array convertNSArrayToJSIArray(jsi::Runtime &runtime, NSArray *value)
{
jsi::Array convertNSArrayToJSIArray(jsi::Runtime& runtime, NSArray* value) {
jsi::Array result = jsi::Array(runtime, value.count);
for (size_t i = 0; i < value.count; i++) {
result.setValueAtIndex(runtime, i, convertObjCObjectToJSIValue(runtime, value[i]));
@ -62,21 +58,20 @@ jsi::Array convertNSArrayToJSIArray(jsi::Runtime &runtime, NSArray *value)
return result;
}
jsi::Value convertObjCObjectToJSIValue(jsi::Runtime &runtime, id value)
{
jsi::Value convertObjCObjectToJSIValue(jsi::Runtime& runtime, id value) {
if (value == nil) {
return jsi::Value::undefined();
} else if ([value isKindOfClass:[NSString class]]) {
return convertNSStringToJSIString(runtime, (NSString *)value);
return convertNSStringToJSIString(runtime, (NSString*)value);
} else if ([value isKindOfClass:[NSNumber class]]) {
if ([value isKindOfClass:[@YES class]]) {
return convertNSNumberToJSIBoolean(runtime, (NSNumber *)value);
return convertNSNumberToJSIBoolean(runtime, (NSNumber*)value);
}
return convertNSNumberToJSINumber(runtime, (NSNumber *)value);
return convertNSNumberToJSINumber(runtime, (NSNumber*)value);
} else if ([value isKindOfClass:[NSDictionary class]]) {
return convertNSDictionaryToJSIObject(runtime, (NSDictionary *)value);
return convertNSDictionaryToJSIObject(runtime, (NSDictionary*)value);
} else if ([value isKindOfClass:[NSArray class]]) {
return convertNSArrayToJSIArray(runtime, (NSArray *)value);
return convertNSArrayToJSIArray(runtime, (NSArray*)value);
} else if (value == (id)kCFNull) {
return jsi::Value::null();
} else if ([value isKindOfClass:[Frame class]]) {
@ -86,23 +81,23 @@ jsi::Value convertObjCObjectToJSIValue(jsi::Runtime &runtime, id value)
return jsi::Value::undefined();
}
NSString *convertJSIStringToNSString(jsi::Runtime &runtime, const jsi::String &value)
{
NSString* convertJSIStringToNSString(jsi::Runtime& runtime, const jsi::String& value) {
return [NSString stringWithUTF8String:value.utf8(runtime).c_str()];
}
NSArray* convertJSICStyleArrayToNSArray(jsi::Runtime &runtime, const jsi::Value* array, size_t length, std::shared_ptr<CallInvoker> jsInvoker) {
if (length < 1) return @[];
NSMutableArray *result = [NSMutableArray new];
NSArray* convertJSICStyleArrayToNSArray(jsi::Runtime& runtime, const jsi::Value* array,
size_t length, std::shared_ptr<CallInvoker> jsInvoker) {
if (length < 1)
return @[];
NSMutableArray* result = [NSMutableArray new];
for (size_t i = 0; i < length; i++) {
// Insert kCFNull when it's `undefined` value to preserve the indices.
[result
addObject:convertJSIValueToObjCObject(runtime, array[i], jsInvoker) ?: (id)kCFNull];
[result addObject:convertJSIValueToObjCObject(runtime, array[i], jsInvoker) ?: (id)kCFNull];
}
return [result copy];
}
jsi::Value* convertNSArrayToJSICStyleArray(jsi::Runtime &runtime, NSArray* array) {
jsi::Value* convertNSArrayToJSICStyleArray(jsi::Runtime& runtime, NSArray* array) {
auto result = new jsi::Value[array.count];
for (size_t i = 0; i < array.count; i++) {
result[i] = convertObjCObjectToJSIValue(runtime, array[i]);
@ -110,26 +105,27 @@ jsi::Value* convertNSArrayToJSICStyleArray(jsi::Runtime &runtime, NSArray* array
return result;
}
NSArray* convertJSIArrayToNSArray(jsi::Runtime &runtime, const jsi::Array &value, std::shared_ptr<CallInvoker> jsInvoker)
{
NSArray* convertJSIArrayToNSArray(jsi::Runtime& runtime, const jsi::Array& value,
std::shared_ptr<CallInvoker> jsInvoker) {
size_t size = value.size(runtime);
NSMutableArray *result = [NSMutableArray new];
NSMutableArray* result = [NSMutableArray new];
for (size_t i = 0; i < size; i++) {
// Insert kCFNull when it's `undefined` value to preserve the indices.
[result
addObject:convertJSIValueToObjCObject(runtime, value.getValueAtIndex(runtime, i), jsInvoker) ?: (id)kCFNull];
addObject:convertJSIValueToObjCObject(runtime, value.getValueAtIndex(runtime, i), jsInvoker)
?: (id)kCFNull];
}
return [result copy];
}
NSDictionary* convertJSIObjectToNSDictionary(jsi::Runtime &runtime, const jsi::Object &value, std::shared_ptr<CallInvoker> jsInvoker)
{
NSDictionary* convertJSIObjectToNSDictionary(jsi::Runtime& runtime, const jsi::Object& value,
std::shared_ptr<CallInvoker> jsInvoker) {
jsi::Array propertyNames = value.getPropertyNames(runtime);
size_t size = propertyNames.size(runtime);
NSMutableDictionary *result = [NSMutableDictionary new];
NSMutableDictionary* result = [NSMutableDictionary new];
for (size_t i = 0; i < size; i++) {
jsi::String name = propertyNames.getValueAtIndex(runtime, i).getString(runtime);
NSString *k = convertJSIStringToNSString(runtime, name);
NSString* k = convertJSIStringToNSString(runtime, name);
id v = convertJSIValueToObjCObject(runtime, value.getProperty(runtime, name), jsInvoker);
if (v) {
result[k] = v;
@ -138,8 +134,8 @@ NSDictionary* convertJSIObjectToNSDictionary(jsi::Runtime &runtime, const jsi::O
return [result copy];
}
id convertJSIValueToObjCObject(jsi::Runtime &runtime, const jsi::Value &value, std::shared_ptr<CallInvoker> jsInvoker)
{
id convertJSIValueToObjCObject(jsi::Runtime& runtime, const jsi::Value& value,
std::shared_ptr<CallInvoker> jsInvoker) {
if (value.isUndefined() || value.isNull()) {
return nil;
}
@ -173,10 +169,11 @@ id convertJSIValueToObjCObject(jsi::Runtime &runtime, const jsi::Value &value, s
throw std::runtime_error("Unsupported jsi::jsi::Value kind");
}
RCTResponseSenderBlock convertJSIFunctionToCallback(jsi::Runtime &runtime, const jsi::Function &value, std::shared_ptr<CallInvoker> jsInvoker)
{
RCTResponseSenderBlock convertJSIFunctionToCallback(jsi::Runtime& runtime,
const jsi::Function& value,
std::shared_ptr<CallInvoker> jsInvoker) {
auto weakWrapper = CallbackWrapper::createWeak(value.getFunction(runtime), runtime, jsInvoker);
RCTBlockGuard *blockGuard = [[RCTBlockGuard alloc] initWithCleanup:^() {
RCTBlockGuard* blockGuard = [[RCTBlockGuard alloc] initWithCleanup:^() {
auto strongWrapper = weakWrapper.lock();
if (strongWrapper) {
strongWrapper->destroy();
@ -184,7 +181,7 @@ RCTResponseSenderBlock convertJSIFunctionToCallback(jsi::Runtime &runtime, const
}];
BOOL __block wrapperWasCalled = NO;
RCTResponseSenderBlock callback = ^(NSArray *responses) {
RCTResponseSenderBlock callback = ^(NSArray* responses) {
if (wrapperWasCalled) {
throw std::runtime_error("callback arg cannot be called more than once");
}
@ -201,7 +198,8 @@ RCTResponseSenderBlock convertJSIFunctionToCallback(jsi::Runtime &runtime, const
}
const jsi::Value* args = convertNSArrayToJSICStyleArray(strongWrapper2->runtime(), responses);
strongWrapper2->callback().call(strongWrapper2->runtime(), args, static_cast<size_t>(responses.count));
strongWrapper2->callback().call(strongWrapper2->runtime(), args,
static_cast<size_t>(responses.count));
strongWrapper2->destroy();
delete[] args;
@ -215,4 +213,4 @@ RCTResponseSenderBlock convertJSIFunctionToCallback(jsi::Runtime &runtime, const
return [callback copy];
}
} // namespace JSIJNIObjectConversion
} // namespace JSINSObjectConversion

7
ios/Frame Processor/VisionCameraProxy.h
View File

@ -12,13 +12,13 @@
#import <React/RCTBridge.h>
#ifdef __cplusplus
#import <jsi/jsi.h>
#import "WKTJsiWorkletContext.h"
#import <ReactCommon/CallInvoker.h>
#import <jsi/jsi.h>
using namespace facebook;
class VisionCameraProxy: public jsi::HostObject {
class VisionCameraProxy : public jsi::HostObject {
public:
explicit VisionCameraProxy(jsi::Runtime& runtime,
std::shared_ptr<react::CallInvoker> callInvoker);
@ -31,7 +31,8 @@ public:
private:
void setFrameProcessor(jsi::Runtime& runtime, int viewTag, const jsi::Object& frameProcessor);
void removeFrameProcessor(jsi::Runtime& runtime, int viewTag);
jsi::Value getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name, const jsi::Object& options);
jsi::Value getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name,
const jsi::Object& options);
private:
std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;

88
ios/Frame Processor/VisionCameraProxy.mm
View File

@ -10,29 +10,29 @@
#import <Foundation/Foundation.h>
#import <jsi/jsi.h>
#import "FrameProcessorPluginRegistry.h"
#import "FrameProcessorPluginHostObject.h"
#import "FrameProcessor.h"
#import "FrameHostObject.h"
#import "JSINSObjectConversion.h"
#import "../../cpp/JSITypedArray.h"
#import "FrameHostObject.h"
#import "FrameProcessor.h"
#import "FrameProcessorPluginHostObject.h"
#import "FrameProcessorPluginRegistry.h"
#import "JSINSObjectConversion.h"
#import "WKTJsiWorklet.h"
#import <React/RCTUtils.h>
#import <React/RCTBridge.h>
#import <React/RCTBridge+Private.h>
#import <React/RCTBridge.h>
#import <React/RCTUIManager.h>
#import <React/RCTUtils.h>
#import <ReactCommon/RCTTurboModuleManager.h>
// Swift forward-declarations
__attribute__((objc_runtime_name("_TtC12VisionCamera12CameraQueues")))
@interface CameraQueues: NSObject
@property (nonatomic, class, readonly, strong) dispatch_queue_t _Nonnull videoQueue;
@interface CameraQueues : NSObject
@property(nonatomic, class, readonly, strong) dispatch_queue_t _Nonnull videoQueue;
@end
__attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
@interface CameraView: UIView
@property (nonatomic, copy) FrameProcessor* _Nullable frameProcessor;
@interface CameraView : UIView
@property(nonatomic, copy) FrameProcessor* _Nullable frameProcessor;
@end
using namespace facebook;
@ -48,15 +48,11 @@ VisionCameraProxy::VisionCameraProxy(jsi::Runtime& runtime,
};
auto runOnWorklet = [](std::function<void()>&& f) {
// Run on Frame Processor Worklet Runtime
dispatch_async(CameraQueues.videoQueue, [f = std::move(f)](){
f();
});
dispatch_async(CameraQueues.videoQueue, [f = std::move(f)]() { f(); });
};
_workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera",
&runtime,
runOnJS,
runOnWorklet);
_workletContext = std::make_shared<RNWorklet::JsiWorkletContext>("VisionCamera", &runtime,
runOnJS, runOnWorklet);
NSLog(@"VisionCameraProxy: Worklet Context Created!");
}
@ -75,20 +71,23 @@ std::vector<jsi::PropNameID> VisionCameraProxy::getPropertyNames(jsi::Runtime& r
return result;
}
void VisionCameraProxy::setFrameProcessor(jsi::Runtime& runtime, int viewTag, const jsi::Object& object) {
void VisionCameraProxy::setFrameProcessor(jsi::Runtime& runtime, int viewTag,
const jsi::Object& object) {
auto frameProcessorType = object.getProperty(runtime, "type").asString(runtime).utf8(runtime);
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, object.getProperty(runtime, "frameProcessor"));
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(
runtime, object.getProperty(runtime, "frameProcessor"));
RCTExecuteOnMainQueue(^{
auto currentBridge = [RCTBridge currentBridge];
auto anonymousView = [currentBridge.uiManager viewForReactTag:[NSNumber numberWithDouble:viewTag]];
auto anonymousView =
[currentBridge.uiManager viewForReactTag:[NSNumber numberWithDouble:viewTag]];
auto view = static_cast<CameraView*>(anonymousView);
if (frameProcessorType == "frame-processor") {
view.frameProcessor = [[FrameProcessor alloc] initWithWorklet:worklet
context:_workletContext];
} else {
throw std::runtime_error("Unknown FrameProcessor.type passed! Received: " + frameProcessorType);
throw std::runtime_error("Unknown FrameProcessor.type passed! Received: " +
frameProcessorType);
}
});
}
@ -96,16 +95,20 @@ void VisionCameraProxy::setFrameProcessor(jsi::Runtime& runtime, int viewTag, co
void VisionCameraProxy::removeFrameProcessor(jsi::Runtime& runtime, int viewTag) {
RCTExecuteOnMainQueue(^{
auto currentBridge = [RCTBridge currentBridge];
auto anonymousView = [currentBridge.uiManager viewForReactTag:[NSNumber numberWithDouble:viewTag]];
auto anonymousView =
[currentBridge.uiManager viewForReactTag:[NSNumber numberWithDouble:viewTag]];
auto view = static_cast<CameraView*>(anonymousView);
view.frameProcessor = nil;
});
}
jsi::Value VisionCameraProxy::getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name, const jsi::Object& options) {
jsi::Value VisionCameraProxy::getFrameProcessorPlugin(jsi::Runtime& runtime, std::string name,
const jsi::Object& options) {
NSString* key = [NSString stringWithUTF8String:name.c_str()];
NSDictionary* optionsObjc = JSINSObjectConversion::convertJSIObjectToNSDictionary(runtime, options, _callInvoker);
FrameProcessorPlugin* plugin = [FrameProcessorPluginRegistry getPlugin:key withOptions:optionsObjc];
NSDictionary* optionsObjc =
JSINSObjectConversion::convertJSIObjectToNSDictionary(runtime, options, _callInvoker);
FrameProcessorPlugin* plugin = [FrameProcessorPluginRegistry getPlugin:key
withOptions:optionsObjc];
if (plugin == nil) {
return jsi::Value::undefined();
}
@ -118,12 +121,9 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID&
auto name = propName.utf8(runtime);
if (name == "setFrameProcessor") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "setFrameProcessor"),
1,
[this](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "setFrameProcessor"), 1,
[this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
size_t count) -> jsi::Value {
auto viewTag = arguments[0].asNumber();
auto object = arguments[1].asObject(runtime);
@ -132,12 +132,9 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID&
});
}
if (name == "removeFrameProcessor") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "removeFrameProcessor"),
1,
[this](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "removeFrameProcessor"), 1,
[this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
size_t count) -> jsi::Value {
auto viewTag = arguments[0].asNumber();
this->removeFrameProcessor(runtime, static_cast<int>(viewTag));
@ -145,12 +142,9 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID&
});
}
if (name == "getFrameProcessorPlugin") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "getFrameProcessorPlugin"),
1,
[this](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
return jsi::Function::createFromHostFunction(
runtime, jsi::PropNameID::forUtf8(runtime, "getFrameProcessorPlugin"), 1,
[this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
size_t count) -> jsi::Value {
if (count < 1 || !arguments[0].isString()) {
throw jsi::JSError(runtime, "First argument needs to be a string (pluginName)!");
@ -165,7 +159,6 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID&
return jsi::Value::undefined();
}
@implementation VisionCameraInstaller
+ (BOOL)installToBridge:(RCTBridge* _Nonnull)bridge {
RCTCxxBridge* cxxBridge = (RCTCxxBridge*)[RCTBridge currentBridge];
@ -177,8 +170,7 @@ jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID&
// global.VisionCameraProxy
auto visionCameraProxy = std::make_shared<VisionCameraProxy>(runtime, bridge.jsCallInvoker);
runtime.global().setProperty(runtime,
"VisionCameraProxy",
runtime.global().setProperty(runtime, "VisionCameraProxy",
jsi::Object::createFromHostObject(runtime, visionCameraProxy));
return YES;

2
ios/NativePreviewView.swift
View File

@ -1,5 +1,5 @@
//
// PreviewView.swift
// NativePreviewView.swift
// VisionCamera
//
// Created by Marc Rousavy on 30.11.22.

2
ios/Parsers/AVCaptureDevice.Position+descriptor.swift
View File

@ -1,5 +1,5 @@
//
// AVCaptureDevice.Position+String.swift
// AVCaptureDevice.Position+descriptor.swift
// mrousavy
//
// Created by Marc Rousavy on 15.12.20.

5
package.json
View File

@ -36,9 +36,10 @@
"release": "release-it",
"pods": "cd example && yarn pods",
"bootstrap": "yarn && cd example && yarn && yarn setup && yarn pods",
"check-android": "scripts/ktlint.sh && scripts/cpplint.sh",
"check-ios": "scripts/swiftformat.sh && scripts/swiftlint.sh",
"check-android": "scripts/ktlint.sh && scripts/clang-format.sh",
"check-ios": "scripts/swiftlint.sh && scripts/swiftformat.sh && scripts/clang-format.sh",
"check-js": "yarn lint --fix && yarn typescript",
"check-cpp": "scripts/clang-format.sh",
"check-all": "scripts/check-all.sh",
"clean-ios": "scripts/clean-ios.sh",
"clean-android": "scripts/clean-android.sh",

4
scripts/check-all.sh
View File

@ -9,8 +9,8 @@ echo "Linting Swift code.."
echo "Linting Kotlin code.."
./scripts/ktlint.sh
echo "Linting C++ code.."
./scripts/cpplint.sh
echo "Formatting C++ code.."
./scripts/clang-format.sh
echo "Linting JS/TS code.."
yarn lint --fix

10
scripts/clang-format.sh Executable file
View File

@ -0,0 +1,10 @@
#!/bin/bash
if which clang-format >/dev/null; then
find cpp ios android/src/main/cpp -type f \( -name "*.h" -o -name "*.cpp" -o -name "*.m" -o -name "*.mm" \) -print0 | while read -d $'\0' file; do
echo "-> cpp-lint $file"
clang-format -i "$file"
done
else
echo "warning: clang-format not installed, download from https://clang.llvm.org/docs/ClangFormat.html (or run brew install clang-format)"
fi

7
scripts/cpplint.sh
View File

@ -1,7 +0,0 @@
#!/bin/bash
if which cpplint >/dev/null; then
cpplint --linelength=230 --filter=-legal/copyright,-readability/todo,-build/namespaces,-runtime/references,-whitespace/comments,-build/include_order,-build/c++11 --quiet --recursive cpp android/src/main/cpp
else
echo "warning: cpplint not installed, download from https://github.com/cpplint/cpplint"
fi