railbird/react-native-vision-camera
8
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: Remove Skia ❌🎨 (#1740)

Browse Source 
* Revert "feat: Skia for Android (#1731)"

This reverts commit a7c137da07.

* Remove some skia

* Remove all the Skia stuff.

* Update useFrameProcessor.ts

* Update lockfiles

* fix: Use native Preview again

* Use `OpenGLTexture&` again

* Remove `PreviewOutput` (we use `SurfaceView` in parallel)

* fix: Log photo widths

* fix: Fix cpplint
This commit is contained in:
Marc Rousavy 2023-09-01 12:20:17 +02:00 committed by GitHub
parent 22829083cd
commit 0a28454579
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
75 changed files with 404 additions and 2415 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
.github/workflows/build-android.yml vendored
View File

@ -60,47 +60,6 @@ jobs:
- name: Run Gradle Build for example/android/
run: cd example/android && ./gradlew assembleDebug --build-cache && cd ../..
build-no-skia:
name: Build Android Example App (without Skia)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Setup JDK 11
uses: actions/setup-java@v1
with:
java-version: 11
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "::set-output name=dir::$(yarn cache dir)"
- name: Restore node_modules from cache
uses: actions/cache@v2
id: yarn-cache
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Install node_modules
run: yarn install --frozen-lockfile
- name: Install node_modules for example/
run: yarn install --frozen-lockfile --cwd example
- name: Remove react-native-skia
run: yarn remove @shopify/react-native-skia --cwd example
- name: Restore Gradle cache
uses: actions/cache@v2
with:
path: |
~/.gradle/caches
~/.gradle/wrapper
key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
restore-keys: |
${{ runner.os }}-gradle-
- name: Run Gradle Build for example/android/
run: cd example/android && ./gradlew assembleDebug --build-cache && cd ../..
build-no-frame-processors:
name: Build Android Example App (without Frame Processors)
runs-on: ubuntu-latest

62
.github/workflows/build-ios.yml vendored
View File

@ -79,68 +79,6 @@ jobs:
build \
CODE_SIGNING_ALLOWED=NO | xcpretty"
build-no-skia:
name: Build iOS Example App without Skia
runs-on: macOS-latest
defaults:
run:
working-directory: example/ios
steps:
- uses: actions/checkout@v2
- name: Get yarn cache directory path
id: yarn-cache-dir-path
run: echo "::set-output name=dir::$(yarn cache dir)"
- name: Restore node_modules from cache
uses: actions/cache@v2
id: yarn-cache
with:
path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
restore-keys: |
${{ runner.os }}-yarn-
- name: Install node_modules for example/
run: yarn install --frozen-lockfile --cwd ..
- name: Remove react-native-skia
run: yarn remove @shopify/react-native-skia --cwd ..
- name: Restore buildcache
uses: mikehardy/buildcache-action@v1
continue-on-error: true
- name: Setup Ruby (bundle)
uses: ruby/setup-ruby@v1
with:
ruby-version: 2.6.10
bundler-cache: true
working-directory: example/ios
- name: Restore Pods cache
uses: actions/cache@v2
with:
path: |
example/ios/Pods
~/Library/Caches/CocoaPods
~/.cocoapods
key: ${{ runner.os }}-pods-${{ hashFiles('**/Podfile.lock') }}
restore-keys: |
${{ runner.os }}-pods-
- name: Install Pods
run: bundle exec pod check || bundle exec pod install
- name: Install xcpretty
run: gem install xcpretty
- name: Build App
run: "set -o pipefail && xcodebuild \
CC=clang CPLUSPLUS=clang++ LD=clang LDPLUSPLUS=clang++ \
-derivedDataPath build -UseModernBuildSystem=YES \
-workspace VisionCameraExample.xcworkspace \
-scheme VisionCameraExample \
-sdk iphonesimulator \
-configuration Debug \
-destination 'platform=iOS Simulator,name=iPhone 11 Pro' \
build \
CODE_SIGNING_ALLOWED=NO | xcpretty"
build-no-frame-processors:
name: Build iOS Example App without Frame Processors
runs-on: macOS-latest

22
VisionCamera.podspec
View File

@ -9,21 +9,12 @@ if defined?($VCDisableFrameProcessors)
Pod::UI.puts "[VisionCamera] $VCDisableFrameProcesors is set to #{$VCDisableFrameProcessors}!"
forceDisableFrameProcessors = $VCDisableFrameProcessors
end
forceDisableSkia = false
if defined?($VCDisableSkia)
Pod::UI.puts "[VisionCamera] $VCDisableSkia is set to #{$VCDisableSkia}!"
forceDisableSkia = $VCDisableSkia
end
Pod::UI.puts("[VisionCamera] node modules #{Dir.exist?(nodeModules) ? "found at #{nodeModules}" : "not found!"}")
workletsPath = File.join(nodeModules, "react-native-worklets-core")
hasWorklets = File.exist?(workletsPath) && !forceDisableFrameProcessors
Pod::UI.puts("[VisionCamera] react-native-worklets-core #{hasWorklets ? "found" : "not found"}, Frame Processors #{hasWorklets ? "enabled" : "disabled"}!")
skiaPath = File.join(nodeModules, "@shopify", "react-native-skia")
hasSkia = hasWorklets && File.exist?(skiaPath) && !forceDisableSkia
Pod::UI.puts("[VisionCamera] react-native-skia #{hasSkia ? "found" : "not found"}, Skia Frame Processors #{hasSkia ? "enabled" : "disabled"}!")
Pod::Spec.new do |s|
s.name = "VisionCamera"
s.version = package["version"]
@ -37,10 +28,10 @@ Pod::Spec.new do |s|
s.source = { :git => "https://github.com/mrousavy/react-native-vision-camera.git", :tag => "#{s.version}" }
s.pod_target_xcconfig = {
"GCC_PREPROCESSOR_DEFINITIONS" => "$(inherited) SK_METAL=1 SK_GANESH=1 VISION_CAMERA_ENABLE_FRAME_PROCESSORS=#{hasWorklets} VISION_CAMERA_ENABLE_SKIA=#{hasSkia}",
"OTHER_SWIFT_FLAGS" => "$(inherited) #{hasWorklets ? "-D VISION_CAMERA_ENABLE_FRAME_PROCESSORS" : ""} #{hasSkia ? "-D VISION_CAMERA_ENABLE_SKIA" : ""}",
"GCC_PREPROCESSOR_DEFINITIONS" => "$(inherited) SK_METAL=1 SK_GANESH=1 VISION_CAMERA_ENABLE_FRAME_PROCESSORS=#{hasWorklets}",
"OTHER_SWIFT_FLAGS" => "$(inherited) #{hasWorklets ? "-D VISION_CAMERA_ENABLE_FRAME_PROCESSORS" : ""}",
"CLANG_CXX_LANGUAGE_STANDARD" => "c++17",
"HEADER_SEARCH_PATHS" => "\"$(PODS_TARGET_SRCROOT)/cpp/\"/** \"#{skiaPath}/cpp/skia/**\" "
"HEADER_SEARCH_PATHS" => "\"$(PODS_TARGET_SRCROOT)/cpp/\"/** "
}
s.requires_arc = true
@ -63,10 +54,6 @@ Pod::Spec.new do |s|
hasWorklets ? "ios/Frame Processor/FrameProcessorPluginRegistry.h" : "",
hasWorklets ? "ios/Frame Processor/VisionCameraProxy.h" : "",
hasWorklets ? "cpp/**/*.{cpp}" : "",
# Skia Frame Processors
hasSkia ? "ios/Skia Render Layer/*.{m,mm,swift}" : "",
hasSkia ? "ios/Skia Render Layer/SkiaRenderer.h" : "",
]
# Any private headers that are not globally unique should be mentioned here.
# Otherwise there will be a nameclash, since CocoaPods flattens out any header directories
@ -82,8 +69,5 @@ Pod::Spec.new do |s|
if hasWorklets
s.dependency "react-native-worklets-core"
if hasSkia
s.dependency "react-native-skia"
end
end
end

64
android/CMakeLists.txt
View File

@ -12,7 +12,7 @@ find_package(ReactAndroid REQUIRED CONFIG)
find_package(fbjni REQUIRED CONFIG)
find_library(LOG_LIB log)
add_definitions(-DVISION_CAMERA_ENABLE_FRAME_PROCESSORS=${ENABLE_FRAME_PROCESSORS} -DVISION_CAMERA_ENABLE_SKIA=${ENABLE_SKIA} -DEGL_EGLEXT_PROTOTYPES=1)
add_definitions(-DVISION_CAMERA_ENABLE_FRAME_PROCESSORS=${ENABLE_FRAME_PROCESSORS})
# Add react-native-vision-camera sources
@ -35,11 +35,6 @@ add_library(
src/main/cpp/frameprocessor/java-bindings/JFrameProcessorPlugin.cpp
src/main/cpp/frameprocessor/java-bindings/JVisionCameraProxy.cpp
src/main/cpp/frameprocessor/java-bindings/JVisionCameraScheduler.cpp
# Skia Frame Processor
src/main/cpp/skia/SkiaRenderer.cpp
src/main/cpp/skia/JSkiaFrameProcessor.cpp
src/main/cpp/skia/DrawableFrameHostObject.cpp
src/main/cpp/skia/VisionCameraSkiaContext.cpp
)
# Header Search Paths (includes)
@ -50,8 +45,6 @@ target_include_directories(
"src/main/cpp"
"src/main/cpp/frameprocessor"
"src/main/cpp/frameprocessor/java-bindings"
"src/main/cpp/skia"
"src/main/cpp/skia/java-bindings"
"${NODE_MODULES_DIR}/react-native/ReactCommon"
"${NODE_MODULES_DIR}/react-native/ReactCommon/callinvoker"
"${NODE_MODULES_DIR}/react-native/ReactAndroid/src/main/jni/react/turbomodule" # <-- CallInvokerHolder JNI wrapper
@ -65,6 +58,8 @@ target_link_libraries(
ReactAndroid::jsi # <-- RN: JSI
ReactAndroid::reactnativejni # <-- RN: React Native JNI bindings
fbjni::fbjni # <-- fbjni
GLESv2 # <-- OpenGL (for VideoPipeline)
EGL # <-- OpenGL (EGL) (for VideoPipeline)
)
# Optionally also add Frame Processors here
@ -75,57 +70,4 @@ if(ENABLE_FRAME_PROCESSORS)
react-native-worklets-core::rnworklets
)
message("VisionCamera: Frame Processors enabled!")
# Optionally also add Skia Integration here
if(ENABLE_SKIA)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DSK_GL -DSK_GANESH -DSK_BUILD_FOR_ANDROID")
find_package(shopify_react-native-skia REQUIRED CONFIG)
set(SKIA_PACKAGE shopify_react-native-skia::rnskia)
set(RNSKIA_PATH ${NODE_MODULES_DIR}/@shopify/react-native-skia)
set (SKIA_LIBS_PATH "${RNSKIA_PATH}/libs/android/${ANDROID_ABI}")
add_library(skia STATIC IMPORTED)
set_property(TARGET skia PROPERTY IMPORTED_LOCATION "${SKIA_LIBS_PATH}/libskia.a")
add_library(svg STATIC IMPORTED)
set_property(TARGET svg PROPERTY IMPORTED_LOCATION "${SKIA_LIBS_PATH}/libsvg.a")
add_library(skshaper STATIC IMPORTED)
set_property(TARGET skshaper PROPERTY IMPORTED_LOCATION "${SKIA_LIBS_PATH}/libskshaper.a")
# We need to include the headers from skia
# (Note: rnskia includes all their files without any relative path
# so for example "include/core/SkImage.h" becomes #include "SkImage.h".
# That's why for the prefab of rnskia, we flatten all cpp files into
# just one directory. HOWEVER, skia itself uses relative paths in
# their include statements, and so we have to include the path to skia)
target_include_directories(
${PACKAGE_NAME}
PRIVATE
"${RNSKIA_PATH}/cpp/api/"
"${RNSKIA_PATH}/cpp/jsi/"
"${RNSKIA_PATH}/cpp/rnskia/"
"${RNSKIA_PATH}/cpp/skia"
"${RNSKIA_PATH}/cpp/skia/include/"
"${RNSKIA_PATH}/cpp/skia/include/config/"
"${RNSKIA_PATH}/cpp/skia/include/core/"
"${RNSKIA_PATH}/cpp/skia/include/effects/"
"${RNSKIA_PATH}/cpp/skia/include/utils/"
"${RNSKIA_PATH}/cpp/skia/include/pathops/"
"${RNSKIA_PATH}/cpp/skia/modules/"
"${RNSKIA_PATH}/cpp/utils/"
)
target_link_libraries(
${PACKAGE_NAME}
GLESv2 # <-- Optional: OpenGL (for Skia)
EGL # <-- Optional: OpenGL (EGL) (for Skia)
${SKIA_PACKAGE} # <-- Optional: RN Skia
jnigraphics
skia
svg
skshaper
)
message("VisionCamera: Skia enabled!")
endif()
endif()

11
android/build.gradle
View File

@ -66,10 +66,7 @@ static def findNodeModules(baseDir) {
def nodeModules = findNodeModules(projectDir)
def hasWorklets = !safeExtGet("VisionCamera_disableFrameProcessors", false) && findProject(":react-native-worklets-core") != null
def hasSkia = !safeExtGet("VisionCamera_disableSkia", false) && findProject(":shopify_react-native-skia") != null
logger.warn("[VisionCamera] react-native-worklets-core ${hasWorklets ? "found" : "not found"}, Frame Processors ${hasWorklets ? "enabled" : "disabled"}!")
logger.warn("[VisionCamera] react-native-skia ${hasSkia ? "found" : "not found"}, Skia Frame Processors ${hasSkia ? "enabled" : "disabled"}!")
repositories {
google()
@ -105,8 +102,7 @@ android {
cppFlags "-O2 -frtti -fexceptions -Wall -Wno-unused-variable -fstack-protector-all"
arguments "-DANDROID_STL=c++_shared",
"-DNODE_MODULES_DIR=${nodeModules}",
"-DENABLE_FRAME_PROCESSORS=${hasWorklets}",
"-DENABLE_SKIA=${hasWorklets && hasSkia}"
"-DENABLE_FRAME_PROCESSORS=${hasWorklets}"
abiFilters (*reactNativeArchitectures())
}
}
@ -152,11 +148,6 @@ dependencies {
if (hasWorklets) {
// Frame Processor integration (optional)
implementation project(":react-native-worklets-core")
if (hasSkia) {
// Skia Frame Processor integration (optional)
implementation project(":shopify_react-native-skia")
}
}
}

16
android/src/main/cpp/OpenGLContext.cpp
View File

@ -10,7 +10,6 @@
#include <android/native_window.h>
#include <android/log.h>
#include <chrono>
#include "OpenGLError.h"
@ -151,19 +150,4 @@ OpenGLTexture OpenGLContext::createTexture(OpenGLTexture::Type type, int width,
};
}
void OpenGLContext::getPixelsOfTexture(const OpenGLTexture& texture, size_t* outSize, uint8_t** outPixels) {
glActiveTexture(GL_TEXTURE0);
glBindTexture(texture.target, texture.id);
glReadPixels(0, 0, texture.width, texture.height, GL_RGBA, GL_UNSIGNED_BYTE, *outPixels);
// height * width * components per pixel (4 for RGBA) * size of one number (byte)
*outSize = texture.height * texture.width * 4 * sizeof(uint8_t);
}
long OpenGLContext::getCurrentPresentationTime() {
auto now = std::chrono::steady_clock::now();
auto duration = now.time_since_epoch();
long long milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(duration).count();
return static_cast<long>(milliseconds);
}
} // namespace vision

16
android/src/main/cpp/OpenGLContext.h
View File

@ -6,13 +6,12 @@
#include <EGL/egl.h>
#include <GLES2/gl2.h>
#include "OpenGLTexture.h"
#include <memory>
#include <functional>
#include <chrono>
#include "PassThroughShader.h"
#include "OpenGLTexture.h"
namespace vision {
@ -53,16 +52,6 @@ class OpenGLContext {
*/
OpenGLTexture createTexture(OpenGLTexture::Type type, int width, int height);
/**
* Gets the pixels as CPU accessible memory of the given input texture
*/
void getPixelsOfTexture(const OpenGLTexture& texture, size_t* outSize, uint8_t** outPixels);
/**
* Gets the current presentation time for this OpenGL surface.
*/
long getCurrentPresentationTime();
public:
EGLDisplay display = EGL_NO_DISPLAY;
EGLContext context = EGL_NO_CONTEXT;
@ -70,13 +59,12 @@ class OpenGLContext {
EGLConfig config = nullptr;
private:
explicit OpenGLContext() = default;
OpenGLContext() = default;
void destroy();
void ensureOpenGL();
private:
PassThroughShader _passThroughShader;
std::chrono::time_point<std::chrono::system_clock> _startTime;
private:
static constexpr auto TAG = "OpenGLContext";

31
android/src/main/cpp/OpenGLRenderer.cpp
View File

@ -43,35 +43,32 @@ void OpenGLRenderer::destroy() {
}
}
EGLSurface OpenGLRenderer::getEGLSurface() {
void OpenGLRenderer::renderTextureToSurface(const OpenGLTexture& texture, float* transformMatrix) {
if (_surface == EGL_NO_SURFACE) {
__android_log_print(ANDROID_LOG_INFO, TAG, "Creating Window Surface...");
_context->use();
_surface = eglCreateWindowSurface(_context->display, _context->config, _outputSurface, nullptr);
}
return _surface;
}
void OpenGLRenderer::renderTextureToSurface(const OpenGLTexture& texture, float* transformMatrix) {
// 1. Get (or create) the OpenGL EGLSurface which is the window render target (Android Surface)
EGLSurface surface = getEGLSurface();
// 1. Activate the OpenGL context for this surface
_context->use(_surface);
// 2. Activate the OpenGL context for this surface
_context->use(surface);
OpenGLError::checkIfError("Failed to use context!");
// 3. Set the viewport for rendering
// 2. Set the viewport for rendering
glViewport(0, 0, _width, _height);
glDisable(GL_BLEND);
glClearColor(0.0f, 0.0f, 0.0f, 1.0f);
glClear(GL_COLOR_BUFFER_BIT);
// 4. Draw it using the pass-through shader which binds the texture and applies transforms
// 3. Bind the input texture
glBindTexture(texture.target, texture.id);
glTexParameteri(texture.target, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(texture.target, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(texture.target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(texture.target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
// 4. Draw it using the pass-through shader which also applies transforms
_passThroughShader.draw(texture, transformMatrix);
// 5 Swap buffers to pass it to the window surface
_context->flush();
OpenGLError::checkIfError("Failed to render Frame to Surface!");
// 5. Swap buffers to pass it to the window surface
eglSwapBuffers(_context->display, _surface);
}
} // namespace vision

6
android/src/main/cpp/OpenGLRenderer.h
View File

@ -11,6 +11,7 @@
#include "PassThroughShader.h"
#include "OpenGLContext.h"
#include "OpenGLTexture.h"
namespace vision {
@ -39,11 +40,6 @@ class OpenGLRenderer {
*/
void destroy();
/**
* Gets the EGLSurface (window surface) that this OpenGL renderer is configured to render to.
*/
EGLSurface getEGLSurface();
private:
explicit OpenGLRenderer(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface);

37
android/src/main/cpp/PassThroughShader.cpp
View File

@ -10,38 +10,33 @@
#include "OpenGLError.h"
#include <string>
#include <android/log.h>
namespace vision {
PassThroughShader::~PassThroughShader() {
if (_vertexBuffer != NO_BUFFER) {
glDeleteBuffers(1, &_vertexBuffer);
_vertexBuffer = NO_BUFFER;
}
if (_programId != NO_SHADER) {
glDeleteProgram(_programId);
_programId = NO_SHADER;
}
if (_vertexBuffer != NO_BUFFER) {
glDeleteBuffers(1, &_vertexBuffer);
_vertexBuffer = NO_BUFFER;
}
}
void PassThroughShader::draw(const OpenGLTexture& texture, float* transformMatrix) {
// 1. Set up Shader Program
if (_programId == NO_SHADER || _shaderTarget != texture.target) {
if (_programId != NO_SHADER) {
glDeleteProgram(_programId);
}
_programId = createProgram(texture.target);
if (_programId == NO_SHADER) {
_programId = createProgram();
glUseProgram(_programId);
_vertexParameters = {
.aPosition = glGetAttribLocation(_programId, "aPosition"),
.aTexCoord = glGetAttribLocation(_programId, "aTexCoord"),
.uTransformMatrix = glGetUniformLocation(_programId, "uTransformMatrix"),
.aPosition = glGetAttribLocation(_programId, "aPosition"),
.aTexCoord = glGetAttribLocation(_programId, "aTexCoord"),
.uTransformMatrix = glGetUniformLocation(_programId, "uTransformMatrix"),
};
_fragmentParameters = {
.uTexture = glGetUniformLocation(_programId, "uTexture"),
.uTexture = glGetUniformLocation(_programId, "uTexture"),
};
_shaderTarget = texture.target;
}
glUseProgram(_programId);
@ -49,11 +44,10 @@ void PassThroughShader::draw(const OpenGLTexture& texture, float* transformMatri
// 2. Set up Vertices Buffer
if (_vertexBuffer == NO_BUFFER) {
glGenBuffers(1, &_vertexBuffer);
glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer);
glBufferData(GL_ARRAY_BUFFER, sizeof(VERTICES), VERTICES, GL_STATIC_DRAW);
}
glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer);
glBufferData(GL_ARRAY_BUFFER, sizeof(VERTICES), VERTICES, GL_STATIC_DRAW);
// 3. Pass all uniforms/attributes for vertex shader
glEnableVertexAttribArray(_vertexParameters.aPosition);
glVertexAttribPointer(_vertexParameters.aPosition,
@ -97,10 +91,9 @@ GLuint PassThroughShader::loadShader(GLenum shaderType, const char* shaderCode)
return shader;
}
GLuint PassThroughShader::createProgram(GLenum textureTarget) {
GLuint PassThroughShader::createProgram() {
GLuint vertexShader = loadShader(GL_VERTEX_SHADER, VERTEX_SHADER);
auto fragmentShaderCode = textureTarget == GL_TEXTURE_EXTERNAL_OES ? FRAGMENT_SHADER_EXTERNAL_TEXTURE : FRAGMENT_SHADER;
GLuint fragmentShader = loadShader(GL_FRAGMENT_SHADER, fragmentShaderCode);
GLuint fragmentShader = loadShader(GL_FRAGMENT_SHADER, FRAGMENT_SHADER);
GLuint program = glCreateProgram();
if (program == 0) throw OpenGLError("Failed to create pass-through program!");

17
android/src/main/cpp/PassThroughShader.h
View File

@ -14,7 +14,6 @@ namespace vision {
#define NO_SHADER 0
#define NO_POSITION 0
#define NO_BUFFER 0
#define NO_SHADER_TARGET 0
struct Vertex {
GLfloat position[2];
@ -28,17 +27,17 @@ class PassThroughShader {
/**
* Draw the texture using this shader.
* Note: At the moment, only EXTERNAL textures are supported by the Shader.
*/
void draw(const OpenGLTexture& texture, float* transformMatrix);
private:
// Loading
static GLuint loadShader(GLenum shaderType, const char* shaderCode);
static GLuint createProgram(GLenum textureTarget);
static GLuint createProgram();
private:
// Shader program in memory
GLenum _shaderTarget = NO_SHADER_TARGET;
// Parameters
GLuint _programId = NO_SHADER;
GLuint _vertexBuffer = NO_BUFFER;
struct VertexParameters {
@ -71,17 +70,7 @@ class PassThroughShader {
}
)";
static constexpr char FRAGMENT_SHADER[] = R"(
precision mediump float;
varying vec2 vTexCoord;
uniform sampler2D uTexture;
void main() {
gl_FragColor = texture2D(uTexture, vTexCoord);
}
)";
static constexpr char FRAGMENT_SHADER_EXTERNAL_TEXTURE[] = R"(
#extension GL_OES_EGL_image_external : require
precision mediump float;
varying vec2 vTexCoord;
uniform samplerExternalOES uTexture;

139
android/src/main/cpp/VideoPipeline.cpp
View File

@ -16,7 +16,6 @@
#include "OpenGLTexture.h"
#include "JFrameProcessor.h"
#include "JSkiaFrameProcessor.h"
namespace vision {
@ -32,23 +31,29 @@ VideoPipeline::VideoPipeline(jni::alias_ref<jhybridobject> jThis, int width, int
VideoPipeline::~VideoPipeline() {
// 1. Remove output surfaces
removeFrameProcessor();
removeFrameProcessorOutputSurface();
removeRecordingSessionOutputSurface();
removePreviewOutputSurface();
// 2. Delete the input textures
if (_inputTexture != std::nullopt) {
glDeleteTextures(1, &_inputTexture->id);
_inputTexture = std::nullopt;
}
// 3. Destroy the OpenGL context
_context = nullptr;
}
void VideoPipeline::removeFrameProcessor() {
_frameProcessor = nullptr;
void VideoPipeline::removeFrameProcessorOutputSurface() {
if (_frameProcessorOutput) _frameProcessorOutput->destroy();
_frameProcessorOutput = nullptr;
}
void VideoPipeline::setFrameProcessor(jni::alias_ref<JFrameProcessor::javaobject> frameProcessor) {
_frameProcessor = jni::make_global(frameProcessor);
void VideoPipeline::setFrameProcessorOutputSurface(jobject surface) {
// 1. Delete existing output surface
removeFrameProcessorOutputSurface();
// 2. Set new output surface if it is not null
ANativeWindow* window = ANativeWindow_fromSurface(jni::Environment::current(), surface);
_frameProcessorOutput = OpenGLRenderer::CreateWithWindowSurface(_context, window);
}
void VideoPipeline::removeRecordingSessionOutputSurface() {
@ -65,139 +70,45 @@ void VideoPipeline::setRecordingSessionOutputSurface(jobject surface) {
_recordingSessionOutput = OpenGLRenderer::CreateWithWindowSurface(_context, window);
}
void VideoPipeline::removePreviewOutputSurface() {
if (_previewOutput) _previewOutput->destroy();
_previewOutput = nullptr;
}
jni::local_ref<JFrame> VideoPipeline::createFrame() {
static const auto createFrameMethod = javaClassLocal()->getMethod<JFrame()>("createFrame");
return createFrameMethod(_javaPart);
}
void VideoPipeline::setPreviewOutputSurface(jobject surface) {
// 1. Delete existing output surface
removePreviewOutputSurface();
// 2. Set new output surface if it is not null
ANativeWindow* window = ANativeWindow_fromSurface(jni::Environment::current(), surface);
_previewOutput = OpenGLRenderer::CreateWithWindowSurface(_context, window);
}
int VideoPipeline::getInputTextureId() {
if (_inputTexture == std::nullopt) {
_inputTexture = _context->createTexture(OpenGLTexture::Type::ExternalOES, _width, _height);
}
return static_cast<int>(_inputTexture->id);
}
void VideoPipeline::onBeforeFrame() {
// 1. Activate the offscreen context
_context->use();
// 2. Prepare the external texture so the Camera can render into it
OpenGLTexture& texture = _inputTexture.value();
glBindTexture(texture.target, texture.id);
glBindTexture(_inputTexture->target, _inputTexture->id);
}
void VideoPipeline::onFrame(jni::alias_ref<jni::JArrayFloat> transformMatrixParam) {
// 1. Activate the offscreen context
_context->use();
// 2. Get the OpenGL transform Matrix (transforms, scales, rotations)
// Get the OpenGL transform Matrix (transforms, scales, rotations)
float transformMatrix[16];
transformMatrixParam->getRegion(0, 16, transformMatrix);
// 3. Prepare the texture we are going to render
OpenGLTexture& texture = _inputTexture.value();
// 4. Render to all outputs!
auto isSkiaFrameProcessor = _frameProcessor != nullptr && _frameProcessor->isInstanceOf(JSkiaFrameProcessor::javaClassStatic());
if (isSkiaFrameProcessor) {
// 4.1. If we have a Skia Frame Processor, prepare to render to an offscreen surface using Skia
jni::global_ref<JSkiaFrameProcessor::javaobject> skiaFrameProcessor = jni::static_ref_cast<JSkiaFrameProcessor::javaobject>(_frameProcessor);
SkiaRenderer& skiaRenderer = skiaFrameProcessor->cthis()->getSkiaRenderer();
auto drawCallback = [=](SkCanvas* canvas) {
// Create a JFrame instance (this uses queues/recycling)
auto frame = JFrame::create(texture.width,
texture.height,
texture.width * 4,
_context->getCurrentPresentationTime(),
"portrait",
false);
// Fill the Frame with the contents of the GL surface
_context->getPixelsOfTexture(texture,
&frame->cthis()->pixelsSize,
&frame->cthis()->pixels);
// Call the Frame processor with the Frame
frame->cthis()->incrementRefCount();
skiaFrameProcessor->cthis()->call(frame, canvas);
frame->cthis()->decrementRefCount();
};
// 4.2. Render to the offscreen surface using Skia
__android_log_print(ANDROID_LOG_INFO, TAG, "Rendering using Skia..");
OpenGLTexture offscreenTexture = skiaRenderer.renderTextureToOffscreenSurface(*_context,
texture,
transformMatrix,
drawCallback);
// 4.3. Now render the result of the offscreen surface to all output surfaces!
if (_previewOutput) {
__android_log_print(ANDROID_LOG_INFO, TAG, "Rendering to Preview..");
skiaRenderer.renderTextureToSurface(*_context, offscreenTexture, _previewOutput->getEGLSurface());
}
if (_recordingSessionOutput) {
__android_log_print(ANDROID_LOG_INFO, TAG, "Rendering to RecordingSession..");
skiaRenderer.renderTextureToSurface(*_context, offscreenTexture, _recordingSessionOutput->getEGLSurface());
}
} else {
// 4.1. If we have a Frame Processor, call it
if (_frameProcessor != nullptr) {
// Create a JFrame instance (this uses queues/recycling)
auto frame = JFrame::create(texture.width,
texture.height,
texture.width * 4,
_context->getCurrentPresentationTime(),
"portrait",
false);
// Fill the Frame with the contents of the GL surface
_context->getPixelsOfTexture(texture,
&frame->cthis()->pixelsSize,
&frame->cthis()->pixels);
// Call the Frame processor with the Frame
frame->cthis()->incrementRefCount();
_frameProcessor->cthis()->call(frame);
frame->cthis()->decrementRefCount();
}
// 4.2. Simply pass-through shader to render the texture to all output EGLSurfaces
__android_log_print(ANDROID_LOG_INFO, TAG, "Rendering using pass-through OpenGL Shader..");
if (_previewOutput) {
__android_log_print(ANDROID_LOG_INFO, TAG, "Rendering to Preview..");
_previewOutput->renderTextureToSurface(texture, transformMatrix);
}
if (_recordingSessionOutput) {
__android_log_print(ANDROID_LOG_INFO, TAG, "Rendering to RecordingSession..");
_recordingSessionOutput->renderTextureToSurface(texture, transformMatrix);
}
if (_frameProcessorOutput) {
__android_log_print(ANDROID_LOG_INFO, TAG, "Rendering to FrameProcessor..");
_frameProcessorOutput->renderTextureToSurface(texture, transformMatrix);
}
if (_recordingSessionOutput) {
__android_log_print(ANDROID_LOG_INFO, TAG, "Rendering to RecordingSession..");
_recordingSessionOutput->renderTextureToSurface(texture, transformMatrix);
}
}
void VideoPipeline::registerNatives() {
registerHybrid({
makeNativeMethod("initHybrid", VideoPipeline::initHybrid),
makeNativeMethod("getInputTextureId", VideoPipeline::getInputTextureId),
makeNativeMethod("setFrameProcessor", VideoPipeline::setFrameProcessor),
makeNativeMethod("removeFrameProcessor", VideoPipeline::removeFrameProcessor),
makeNativeMethod("setPreviewOutputSurface", VideoPipeline::setPreviewOutputSurface),
makeNativeMethod("removePreviewOutputSurface", VideoPipeline::removePreviewOutputSurface),
makeNativeMethod("setFrameProcessorOutputSurface", VideoPipeline::setFrameProcessorOutputSurface),
makeNativeMethod("removeFrameProcessorOutputSurface", VideoPipeline::removeFrameProcessorOutputSurface),
makeNativeMethod("setRecordingSessionOutputSurface", VideoPipeline::setRecordingSessionOutputSurface),
makeNativeMethod("removeRecordingSessionOutputSurface", VideoPipeline::removeRecordingSessionOutputSurface),
makeNativeMethod("getInputTextureId", VideoPipeline::getInputTextureId),
makeNativeMethod("onBeforeFrame", VideoPipeline::onBeforeFrame),
makeNativeMethod("onFrame", VideoPipeline::onFrame),
});

25
android/src/main/cpp/VideoPipeline.h
View File

@ -8,13 +8,11 @@
#include <fbjni/fbjni.h>
#include <EGL/egl.h>
#include <android/native_window.h>
#include <memory>
#include "PassThroughShader.h"
#include "OpenGLRenderer.h"
#include "OpenGLContext.h"
#include "OpenGLTexture.h"
#include "JFrameProcessor.h"
#include <memory>
#include <optional>
namespace vision {
@ -33,17 +31,13 @@ class VideoPipeline: public jni::HybridClass<VideoPipeline> {
int getInputTextureId();
// <- Frame Processor output
void setFrameProcessor(jni::alias_ref<JFrameProcessor::javaobject> frameProcessor);
void removeFrameProcessor();
void setFrameProcessorOutputSurface(jobject surface);
void removeFrameProcessorOutputSurface();
// <- MediaRecorder output
void setRecordingSessionOutputSurface(jobject surface);
void removeRecordingSessionOutputSurface();
// <- Preview output
void setPreviewOutputSurface(jobject surface);
void removePreviewOutputSurface();
// Frame callbacks
void onBeforeFrame();
void onFrame(jni::alias_ref<jni::JArrayFloat> transformMatrix);
@ -51,22 +45,17 @@ class VideoPipeline: public jni::HybridClass<VideoPipeline> {
private:
// Private constructor. Use `create(..)` to create new instances.
explicit VideoPipeline(jni::alias_ref<jhybridobject> jThis, int width, int height);
// Creates a new Frame instance which should be filled with data.
jni::local_ref<JFrame> createFrame();
private:
// Input Surface Texture
std::optional<OpenGLTexture> _inputTexture;
std::optional<OpenGLTexture> _inputTexture = std::nullopt;
int _width = 0;
int _height = 0;
// (Optional) Frame Processor that processes frames before they go into output
jni::global_ref<JFrameProcessor::javaobject> _frameProcessor = nullptr;
// Output Contexts
std::shared_ptr<OpenGLContext> _context = nullptr;
std::unique_ptr<OpenGLRenderer> _frameProcessorOutput = nullptr;
std::unique_ptr<OpenGLRenderer> _recordingSessionOutput = nullptr;
std::unique_ptr<OpenGLRenderer> _previewOutput = nullptr;
private:
friend HybridBase;

4
android/src/main/cpp/VisionCamera.cpp
View File

@ -4,7 +4,6 @@
#include "JFrameProcessor.h"
#include "JVisionCameraProxy.h"
#include "VisionCameraProxy.h"
#include "JSkiaFrameProcessor.h"
#include "VideoPipeline.h"
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) {
@ -15,9 +14,6 @@ JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) {
vision::VideoPipeline::registerNatives();
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
vision::JFrameProcessor::registerNatives();
#endif
#if VISION_CAMERA_ENABLE_SKIA
vision::JSkiaFrameProcessor::registerNatives();
#endif
});
}

37
android/src/main/cpp/frameprocessor/FrameHostObject.cpp
View File

@ -31,6 +31,7 @@ std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt)
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("width")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("height")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("bytesPerRow")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("planesCount")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("orientation")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isMirrored")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("timestamp")));
@ -54,7 +55,7 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
const jsi::Value* args,
size_t count) -> jsi::Value {
// Increment retain count by one.
this->frame->cthis()->incrementRefCount();
this->frame->incrementRefCount();
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime,
@ -68,7 +69,7 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
const jsi::Value* args,
size_t count) -> jsi::Value {
// Decrement retain count by one. If the retain count is zero, the Frame gets closed.
this->frame->cthis()->decrementRefCount();
this->frame->decrementRefCount();
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime,
@ -84,8 +85,8 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
if (!this->frame) {
return jsi::String::createFromUtf8(runtime, "[closed frame]");
}
auto width = this->frame->cthis()->getWidth();
auto height = this->frame->cthis()->getHeight();
auto width = this->frame->getWidth();
auto height = this->frame->getHeight();
auto str = std::to_string(width) + " x " + std::to_string(height) + " Frame";
return jsi::String::createFromUtf8(runtime, str);
};
@ -96,8 +97,11 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
const jsi::Value& thisArg,
const jsi::Value* args,
size_t count) -> jsi::Value {
size_t size = frame->cthis()->pixelsSize;
uint8_t* pixels = frame->cthis()->pixels;
auto buffer = this->frame->toByteBuffer();
if (!buffer->isDirect()) {
throw std::runtime_error("Failed to get byte content of Frame - array is not direct ByteBuffer!");
}
auto size = buffer->getDirectSize();
static constexpr auto ARRAYBUFFER_CACHE_PROP_NAME = "__frameArrayBufferCache";
if (!runtime.global().hasProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME)) {
@ -115,7 +119,7 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
// directly write to C++ JSI ArrayBuffer
auto destinationBuffer = arrayBuffer.data(runtime);
memcpy(destinationBuffer, pixels, sizeof(uint8_t) * size);
memcpy(destinationBuffer, buffer->getDirectAddress(), sizeof(uint8_t) * size);
return arrayBuffer;
};
@ -123,30 +127,33 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
}
if (name == "isValid") {
return jsi::Value(this->frame && this->frame->cthis()->getIsValid());
return jsi::Value(this->frame && this->frame->getIsValid());
}
if (name == "width") {
return jsi::Value(this->frame->cthis()->getWidth());
return jsi::Value(this->frame->getWidth());
}
if (name == "height") {
return jsi::Value(this->frame->cthis()->getHeight());
return jsi::Value(this->frame->getHeight());
}
if (name == "isMirrored") {
return jsi::Value(this->frame->cthis()->getIsMirrored());
return jsi::Value(this->frame->getIsMirrored());
}
if (name == "orientation") {
auto string = this->frame->cthis()->getOrientation();
auto string = this->frame->getOrientation();
return jsi::String::createFromUtf8(runtime, string->toStdString());
}
if (name == "pixelFormat") {
auto string = this->frame->cthis()->getPixelFormat();
auto string = this->frame->getPixelFormat();
return jsi::String::createFromUtf8(runtime, string->toStdString());
}
if (name == "timestamp") {
return jsi::Value(static_cast<double>(this->frame->cthis()->getTimestamp()));
return jsi::Value(static_cast<double>(this->frame->getTimestamp()));
}
if (name == "bytesPerRow") {
return jsi::Value(this->frame->cthis()->getBytesPerRow());
return jsi::Value(this->frame->getBytesPerRow());
}
if (name == "planesCount") {
return jsi::Value(this->frame->getPlanesCount());
}
// fallback to base implementation

2
android/src/main/cpp/frameprocessor/FrameHostObject.h
View File

@ -26,7 +26,7 @@ class JSI_EXPORT FrameHostObject : public jsi::HostObject {
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime &rt) override;
public:
jni::global_ref<JFrame::javaobject> frame;
jni::global_ref<JFrame> frame;
};
} // namespace vision

10
android/src/main/cpp/frameprocessor/JSIJNIConversion.cpp
View File

@ -111,10 +111,10 @@ jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, c
return jsi::String::createFromUtf8(runtime, object->toString());
} else if (object->isInstanceOf(jni::JList<jobject>::javaClassStatic())) {
} else if (object->isInstanceOf(JList<jobject>::javaClassStatic())) {
// List<E>
auto arrayList = jni::static_ref_cast<jni::JList<jobject>>(object);
auto arrayList = static_ref_cast<JList<jobject>>(object);
auto size = arrayList->size();
auto result = jsi::Array(runtime, size);
@ -125,10 +125,10 @@ jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, c
}
return result;
} else if (object->isInstanceOf(jni::JMap<jstring, jobject>::javaClassStatic())) {
} else if (object->isInstanceOf(JMap<jstring, jobject>::javaClassStatic())) {
// Map<K, V>
auto map = jni::static_ref_cast<jni::JMap<jstring, jobject>>(object);
auto map = static_ref_cast<JMap<jstring, jobject>>(object);
auto result = jsi::Object(runtime);
for (const auto& entry : *map) {
@ -140,7 +140,7 @@ jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, c
return result;
} else if (object->isInstanceOf(JFrame::javaClassStatic())) {
// Frame
auto frame = jni::static_ref_cast<JFrame::javaobject>(object);
auto frame = static_ref_cast<JFrame>(object);
// box into HostObject
auto hostObject = std::make_shared<FrameHostObject>(frame);

8
android/src/main/cpp/frameprocessor/VisionCameraProxy.cpp
View File

@ -39,7 +39,6 @@ std::vector<jsi::PropNameID> VisionCameraProxy::getPropertyNames(jsi::Runtime& r
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("setFrameProcessor")));
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("removeFrameProcessor")));
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("getFrameProcessorPlugin")));
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("isSkiaEnabled")));
return result;
}
@ -65,13 +64,6 @@ jsi::Value VisionCameraProxy::getFrameProcessorPlugin(jsi::Runtime& runtime,
jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
auto name = propName.utf8(runtime);
if (name == "isSkiaEnabled") {
#ifdef VISION_CAMERA_ENABLE_SKIA
return jsi::Value(true);
#else
return jsi::Value(false);
#endif
}
if (name == "setFrameProcessor") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "setFrameProcessor"),

104
android/src/main/cpp/frameprocessor/java-bindings/JFrame.cpp
View File

@ -11,85 +11,71 @@
namespace vision {
using namespace facebook;
using namespace jni;
void JFrame::registerNatives() {
registerHybrid({
makeNativeMethod("getWidth", JFrame::getWidth),
makeNativeMethod("getHeight", JFrame::getHeight),
makeNativeMethod("getBytesPerRow", JFrame::getBytesPerRow),
makeNativeMethod("getTimestamp", JFrame::getTimestamp),
makeNativeMethod("getOrientation", JFrame::getOrientation),
makeNativeMethod("getIsMirrored", JFrame::getIsMirrored),
makeNativeMethod("getPixelFormat", JFrame::getPixelFormat),
makeNativeMethod("getByteBuffer", JFrame::getByteBuffer),
makeNativeMethod("getIsValid", JFrame::getIsValid),
});
int JFrame::getWidth() const {
static const auto getWidthMethod = getClass()->getMethod<jint()>("getWidth");
return getWidthMethod(self());
}
jni::local_ref<JFrame::javaobject> JFrame::create(int width,
int height,
int bytesPerRow,
long timestamp,
const std::string& orientation,
bool isMirrored) {
return newObjectCxxArgs(width,
height,
bytesPerRow,
timestamp,
orientation,
isMirrored);
int JFrame::getHeight() const {
static const auto getWidthMethod = getClass()->getMethod<jint()>("getHeight");
return getWidthMethod(self());
}
JFrame::JFrame(int width,
int height,
int bytesPerRow,
long timestamp,
const std::string& orientation,
bool isMirrored) {
_width = width;
_height = height;
_bytesPerRow = bytesPerRow;
_timestamp = timestamp;
_orientation = orientation;
_isMirrored = isMirrored;
_refCount = 0;
pixelsSize = height * bytesPerRow;
pixels = (uint8_t*) malloc(pixelsSize);
bool JFrame::getIsValid() const {
static const auto getIsValidMethod = getClass()->getMethod<jboolean()>("getIsValid");
return getIsValidMethod(self());
}
JFrame::~JFrame() noexcept {
close();
bool JFrame::getIsMirrored() const {
static const auto getIsMirroredMethod = getClass()->getMethod<jboolean()>("getIsMirrored");
return getIsMirroredMethod(self());
}
bool JFrame::getIsValid() {
return _refCount > 0 && !_isClosed;
jlong JFrame::getTimestamp() const {
static const auto getTimestampMethod = getClass()->getMethod<jlong()>("getTimestamp");
return getTimestampMethod(self());
}
jni::local_ref<jni::JByteBuffer> JFrame::getByteBuffer() {
if (!getIsValid()) {
[[unlikely]]
throw std::runtime_error("Frame is no longer valid, cannot access getByteBuffer!");
}
return jni::JByteBuffer::wrapBytes(pixels, pixelsSize);
local_ref<JString> JFrame::getOrientation() const {
static const auto getOrientationMethod = getClass()->getMethod<JString()>("getOrientation");
return getOrientationMethod(self());
}
local_ref<JString> JFrame::getPixelFormat() const {
static const auto getPixelFormatMethod = getClass()->getMethod<JString()>("getPixelFormat");
return getPixelFormatMethod(self());
}
int JFrame::getPlanesCount() const {
static const auto getPlanesCountMethod = getClass()->getMethod<jint()>("getPlanesCount");
return getPlanesCountMethod(self());
}
int JFrame::getBytesPerRow() const {
static const auto getBytesPerRowMethod = getClass()->getMethod<jint()>("getBytesPerRow");
return getBytesPerRowMethod(self());
}
local_ref<JByteBuffer> JFrame::toByteBuffer() const {
static const auto toByteBufferMethod = getClass()->getMethod<JByteBuffer()>("toByteBuffer");
return toByteBufferMethod(self());
}
void JFrame::incrementRefCount() {
std::unique_lock lock(_mutex);
_refCount++;
static const auto incrementRefCountMethod = getClass()->getMethod<void()>("incrementRefCount");
incrementRefCountMethod(self());
}
void JFrame::decrementRefCount() {
std::unique_lock lock(_mutex);
_refCount--;
if (_refCount <= 0) {
this->close();
}
static const auto decrementRefCountMethod = getClass()->getMethod<void()>("decrementRefCount");
decrementRefCountMethod(self());
}
void JFrame::close() {
_isClosed = true;
free(pixels);
pixels = nullptr;
static const auto closeMethod = getClass()->getMethod<void()>("close");
closeMethod(self());
}
} // namespace vision

65
android/src/main/cpp/frameprocessor/java-bindings/JFrame.h
View File

@ -7,70 +7,29 @@
#include <jni.h>
#include <fbjni/fbjni.h>
#include <fbjni/ByteBuffer.h>
#include <android/hardware_buffer.h>
#include <android/hardware_buffer_jni.h>
#include <mutex>
namespace vision {
using namespace facebook;
using namespace jni;
class JFrame : public jni::HybridClass<JFrame> {
public:
struct JFrame : public JavaClass<JFrame> {
static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/Frame;";
static void registerNatives();
static jni::local_ref<JFrame::javaobject> create(int width,
int height,
int bytesPerRow,
long timestamp,
const std::string& orientation,
bool isMirrored);
~JFrame() noexcept;
protected:
friend HybridBase;
explicit JFrame(int width,
int height,
int bytesPerRow,
long timestamp,
const std::string& orientation,
bool isMirrored);
public:
int getWidth() { return _width; }
int getHeight() { return _height; }
int getBytesPerRow() { return _bytesPerRow; }
jlong getTimestamp() { return _timestamp; }
jni::local_ref<jni::JString> getOrientation() { return jni::make_jstring(_orientation); }
bool getIsMirrored() { return _isMirrored; }
// TODO: Can this be something other than RGB?
jni::local_ref<jni::JString> getPixelFormat() { return jni::make_jstring("rgb"); }
bool getIsValid();
jni::local_ref<jni::JByteBuffer> getByteBuffer();
int getWidth() const;
int getHeight() const;
bool getIsValid() const;
bool getIsMirrored() const;
int getPlanesCount() const;
int getBytesPerRow() const;
jlong getTimestamp() const;
local_ref<JString> getOrientation() const;
local_ref<JString> getPixelFormat() const;
local_ref<JByteBuffer> toByteBuffer() const;
void incrementRefCount();
void decrementRefCount();
void close();
// Backing byte data
uint8_t* pixels = nullptr;
size_t pixelsSize = 0;
private:
// Frame info
int _width = 0;
int _height = 0;
int _bytesPerRow = 0;
long _timestamp = 0;
std::string _orientation = {};
bool _isMirrored = false;
// Ref-counting
int _refCount = 0;
bool _isClosed = false;
std::mutex _mutex;
};
} // namespace vision

3
android/src/main/cpp/frameprocessor/java-bindings/JFrameProcessor.cpp
View File

@ -17,6 +17,9 @@ using namespace facebook;
using namespace jni;
void JFrameProcessor::registerNatives() {
registerHybrid({
makeNativeMethod("call", JFrameProcessor::call)
});
}
using TSelf = jni::local_ref<JFrameProcessor::javaobject>;

19
android/src/main/cpp/frameprocessor/java-bindings/JFrameProcessor.h
View File

@ -21,7 +21,7 @@ namespace vision {
using namespace facebook;
class JFrameProcessor : public jni::HybridClass<JFrameProcessor> {
struct JFrameProcessor : public jni::HybridClass<JFrameProcessor> {
public:
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessor;";
static void registerNatives();
@ -30,25 +30,20 @@ class JFrameProcessor : public jni::HybridClass<JFrameProcessor> {
public:
/**
* Wrap the Frame in a HostObject and call the Frame Processor.
* Call the JS Frame Processor.
*/
void call(jni::alias_ref<JFrame::javaobject> frame);
void call(alias_ref<JFrame::javaobject> frame);
protected:
friend HybridBase;
// C++ only constructor. Use `create(..)` to create new instances.
private:
// Private constructor. Use `create(..)` to create new instances.
explicit JFrameProcessor(std::shared_ptr<RNWorklet::JsiWorklet> worklet,
std::shared_ptr<RNWorklet::JsiWorkletContext> context);
JFrameProcessor(const JFrameProcessor &) = delete;
JFrameProcessor &operator=(const JFrameProcessor &) = delete;
protected:
/**
* Call the JS Frame Processor with the given Frame Host Object.
*/
private:
void callWithFrameHostObject(const std::shared_ptr<FrameHostObject>& frameHostObject) const;
private:
friend HybridBase;
std::shared_ptr<RNWorklet::WorkletInvoker> _workletInvoker;
std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;
};

17
android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraProxy.cpp
View File

@ -18,10 +18,6 @@
#include <react-native-worklets-core/WKTJsiWorkletContext.h>
#endif
#if VISION_CAMERA_ENABLE_SKIA
#include "JSkiaFrameProcessor.h"
#endif
namespace vision {
using TSelf = local_ref<HybridClass<JVisionCameraProxy>::jhybriddata>;
@ -35,7 +31,6 @@ JVisionCameraProxy::JVisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::
const jni::global_ref<JVisionCameraScheduler::javaobject>& scheduler) {
_javaPart = make_global(javaThis);
_runtime = runtime;
_callInvoker = callInvoker;
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
__android_log_write(ANDROID_LOG_INFO, TAG, "Creating Worklet Context...");
@ -58,12 +53,6 @@ JVisionCameraProxy::JVisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::
#else
__android_log_write(ANDROID_LOG_INFO, TAG, "Frame Processors are disabled!");
#endif
#ifdef VISION_CAMERA_ENABLE_SKIA
__android_log_write(ANDROID_LOG_INFO, TAG, "Skia is enabled!");
#else
__android_log_write(ANDROID_LOG_INFO, TAG, "Skia is disabled!");
#endif
}
JVisionCameraProxy::~JVisionCameraProxy() {
@ -87,12 +76,6 @@ void JVisionCameraProxy::setFrameProcessor(int viewTag,
jni::local_ref<JFrameProcessor::javaobject> frameProcessor;
if (frameProcessorType == "frame-processor") {
frameProcessor = JFrameProcessor::create(worklet, _workletContext);
} else if (frameProcessorType == "skia-frame-processor") {
#if VISION_CAMERA_ENABLE_SKIA
frameProcessor = JSkiaFrameProcessor::create(worklet, _workletContext, _callInvoker);
#else
throw std::runtime_error("system/skia-unavailable: Skia is not installed!");
#endif
} else {
throw std::runtime_error("Unknown FrameProcessor.type passed! Received: " + frameProcessorType);
}

2
android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraProxy.h
View File

@ -36,13 +36,11 @@ class JVisionCameraProxy : public jni::HybridClass<JVisionCameraProxy> {
jni::local_ref<JMap<jstring, jobject>> options);
jsi::Runtime* getJSRuntime() { return _runtime; }
std::shared_ptr<react::CallInvoker> getCallInvoker() { return _callInvoker; }
private:
friend HybridBase;
jni::global_ref<JVisionCameraProxy::javaobject> _javaPart;
jsi::Runtime* _runtime;
std::shared_ptr<react::CallInvoker> _callInvoker;
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;
#endif

72
android/src/main/cpp/skia/DrawableFrameHostObject.cpp
View File

@ -1,72 +0,0 @@
//
// Created by Marc Rousavy on 31.08.23.
//
#include "DrawableFrameHostObject.h"
#include <SkCanvas.h>
#include "FrameHostObject.h"
namespace vision {
std::vector<jsi::PropNameID> DrawableFrameHostObject::getPropertyNames(jsi::Runtime& rt) {
auto result = FrameHostObject::getPropertyNames(rt);
// Skia - Render Frame
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("render")));
if (_canvas != nullptr) {
auto canvasPropNames = _canvas->getPropertyNames(rt);
for (auto& prop : canvasPropNames) {
result.push_back(std::move(prop));
}
}
return result;
}
SkRect inscribe(SkSize size, SkRect rect) {
auto halfWidthDelta = (rect.width() - size.width()) / 2.0;
auto halfHeightDelta = (rect.height() - size.height()) / 2.0;
return SkRect::MakeXYWH(rect.x() + halfWidthDelta,
rect.y() + halfHeightDelta, size.width(),
size.height());
}
jsi::Value DrawableFrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
auto name = propName.utf8(runtime);
if (name == "render") {
auto render = JSI_HOST_FUNCTION_LAMBDA {
if (_canvas == nullptr) {
throw jsi::JSError(runtime, "Trying to render a Frame without a Skia Canvas! Did you install Skia?");
}
throw std::runtime_error("render() is not yet implemented!");
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "render"), 1, render);
}
if (name == "isDrawable") {
return jsi::Value(_canvas != nullptr);
}
if (_canvas != nullptr) {
// If we have a Canvas, try to access the property on there.
auto result = _canvas->get(runtime, propName);
if (!result.isUndefined()) {
return result;
}
}
// fallback to base implementation
return FrameHostObject::get(runtime, propName);
}
void DrawableFrameHostObject::invalidateCanvas() {
_canvas = nullptr;
}
} // namespace vision

33
android/src/main/cpp/skia/DrawableFrameHostObject.h
View File

@ -1,33 +0,0 @@
//
// Created by Marc Rousavy on 31.08.23.
//
#pragma once
#include <jsi/jsi.h>
#include "FrameHostObject.h"
#include "JFrame.h"
#include <SkCanvas.h>
#include <JsiSkCanvas.h>
namespace vision {
using namespace facebook;
class JSI_EXPORT DrawableFrameHostObject: public FrameHostObject {
public:
explicit DrawableFrameHostObject(const jni::alias_ref<JFrame::javaobject>& frame,
std::shared_ptr<RNSkia::JsiSkCanvas> canvas): FrameHostObject(frame), _canvas(canvas) {}
public:
jsi::Value get(jsi::Runtime&, const jsi::PropNameID& name) override;
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& rt) override;
void invalidateCanvas();
private:
std::shared_ptr<RNSkia::JsiSkCanvas> _canvas;
};
} // namespace vision

61
android/src/main/cpp/skia/JSkiaFrameProcessor.cpp
View File

@ -1,61 +0,0 @@
//
// Created by Marc Rousavy on 31.08.23.
//
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS && VISION_CAMERA_ENABLE_SKIA
#include "JSkiaFrameProcessor.h"
#include <jni.h>
#include <fbjni/fbjni.h>
#include <utility>
#include "JFrame.h"
#include "DrawableFrameHostObject.h"
#include <RNSkPlatformContext.h>
#include "VisionCameraSkiaContext.h"
namespace vision {
using namespace facebook;
using namespace jni;
void JSkiaFrameProcessor::registerNatives() {
}
using TSelf = jni::local_ref<JSkiaFrameProcessor::javaobject>;
JSkiaFrameProcessor::JSkiaFrameProcessor(const std::shared_ptr<RNWorklet::JsiWorklet>& worklet,
const std::shared_ptr<RNWorklet::JsiWorkletContext>& context,
const std::shared_ptr<react::CallInvoker>& callInvoker)
: JSkiaFrameProcessor::HybridBase(worklet, context) {
// TODO: Can I use the Android Platform Context from react-native-skia here?
auto skiaPlatformContext = std::make_shared<VisionCameraSkiaContext>(context->getJsRuntime(),
callInvoker,
1.0f);
_jsiCanvas = std::make_shared<RNSkia::JsiSkCanvas>(skiaPlatformContext);
_skiaRenderer = std::make_shared<SkiaRenderer>();
}
TSelf JSkiaFrameProcessor::create(const std::shared_ptr<RNWorklet::JsiWorklet>& worklet,
const std::shared_ptr<RNWorklet::JsiWorkletContext>& context,
const std::shared_ptr<react::CallInvoker>& callInvoker) {
return JSkiaFrameProcessor::newObjectCxxArgs(worklet, context, callInvoker);
}
void JSkiaFrameProcessor::call(alias_ref<JFrame::javaobject> frame,
SkCanvas* canvas) {
// Create the Frame Host Object wrapping the internal Frame and Skia Canvas
_jsiCanvas->setCanvas(canvas);
auto frameHostObject = std::make_shared<DrawableFrameHostObject>(frame, _jsiCanvas);
// Call the base function in JFrameProcessor
callWithFrameHostObject(frameHostObject);
// Remove Skia Canvas from Host Object because it is no longer valid
frameHostObject->invalidateCanvas();
}
} // namespace vision
#endif

59
android/src/main/cpp/skia/JSkiaFrameProcessor.h
View File

@ -1,59 +0,0 @@
//
// Created by Marc Rousavy on 31.08.23.
//
#pragma once
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS && VISION_CAMERA_ENABLE_SKIA
#include <string>
#include <memory>
#include <jni.h>
#include <fbjni/fbjni.h>
#include <react-native-worklets-core/WKTJsiWorklet.h>
#include <react-native-worklets-core/WKTJsiHostObject.h>
#include "JFrame.h"
#include "FrameHostObject.h"
#include "SkiaRenderer.h"
#include "JFrameProcessor.h"
#include <JsiSkCanvas.h>
#include <RNSkPlatformContext.h>
namespace vision {
using namespace facebook;
class JSkiaFrameProcessor : public jni::HybridClass<JSkiaFrameProcessor, JFrameProcessor> {
public:
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/skia/SkiaFrameProcessor;";
static void registerNatives();
static jni::local_ref<JSkiaFrameProcessor::javaobject> create(const std::shared_ptr<RNWorklet::JsiWorklet>& worklet,
const std::shared_ptr<RNWorklet::JsiWorkletContext>& context,
const std::shared_ptr<react::CallInvoker>& callInvoker);
public:
/**
* Call the JS Frame Processor with the given valid Canvas to draw on.
*/
void call(jni::alias_ref<JFrame::javaobject> frame,
SkCanvas* canvas);
SkiaRenderer& getSkiaRenderer() { return *_skiaRenderer; }
protected:
friend HybridBase;
// Private constructor. Use `create(..)` to create new instances.
explicit JSkiaFrameProcessor(const std::shared_ptr<RNWorklet::JsiWorklet>& worklet,
const std::shared_ptr<RNWorklet::JsiWorkletContext>& context,
const std::shared_ptr<react::CallInvoker>& callInvoker);
private:
std::shared_ptr<RNSkia::JsiSkCanvas> _jsiCanvas;
std::shared_ptr<SkiaRenderer> _skiaRenderer;
};
} // namespace vision
#endif

234
android/src/main/cpp/skia/SkiaRenderer.cpp
View File

@ -1,234 +0,0 @@
//
// Created by Marc Rousavy on 10.08.23.
//
#if VISION_CAMERA_ENABLE_SKIA
#include "SkiaRenderer.h"
#include <android/log.h>
#include "OpenGLError.h"
#include <GLES2/gl2ext.h>
#include <core/SkColorSpace.h>
#include <core/SkCanvas.h>
#include <core/SkYUVAPixmaps.h>
#include <gpu/gl/GrGLInterface.h>
#include <gpu/GrDirectContext.h>
#include <gpu/GrBackendSurface.h>
#include <gpu/ganesh/SkSurfaceGanesh.h>
#include <gpu/ganesh/SkImageGanesh.h>
#include <android/native_window_jni.h>
#include <android/surface_texture_jni.h>
// from <gpu/ganesh/gl/GrGLDefines.h>
#define GR_GL_RGBA8 0x8058
#define DEFAULT_FBO 0
namespace vision {
SkiaRenderer::~SkiaRenderer() {
_offscreenSurface = nullptr;
_offscreenSurfaceTextureId = NO_TEXTURE;
// 3. Delete the Skia context
if (_skiaContext != nullptr) {
_skiaContext->abandonContext();
_skiaContext = nullptr;
}
}
sk_sp<GrDirectContext> SkiaRenderer::getSkiaContext() {
if (_skiaContext == nullptr) {
_skiaContext = GrDirectContext::MakeGL();
}
return _skiaContext;
}
sk_sp<SkImage> SkiaRenderer::wrapTextureAsImage(OpenGLTexture &texture) {
GrGLTextureInfo textureInfo {
// OpenGL will automatically convert YUV -> RGB - if it's an EXTERNAL texture
.fTarget = texture.target,
.fID = texture.id,
.fFormat = GR_GL_RGBA8,
};
GrBackendTexture skiaTexture(texture.width,
texture.height,
GrMipMapped::kNo,
textureInfo);
sk_sp<SkImage> image = SkImages::BorrowTextureFrom(_skiaContext.get(),
skiaTexture,
kBottomLeft_GrSurfaceOrigin,
kN32_SkColorType,
kOpaque_SkAlphaType,
nullptr,
nullptr);
if (image == nullptr) {
[[unlikely]];
throw std::runtime_error("Failed to create Skia Image! Cannot wrap input texture (frame) using Skia.");
}
return image;
}
sk_sp<SkSurface> SkiaRenderer::wrapEglSurfaceAsSurface(EGLSurface eglSurface) {
GLint sampleCnt;
glGetIntegerv(GL_SAMPLES, &sampleCnt);
GLint stencilBits;
glGetIntegerv(GL_STENCIL_BITS, &stencilBits);
GrGLFramebufferInfo fboInfo {
// DEFAULT_FBO is FBO0, meaning the default on-screen FBO for that given surface
.fFBOID = DEFAULT_FBO,
.fFormat = GR_GL_RGBA8
};
EGLint width = 0, height = 0;
eglQuerySurface(eglGetCurrentDisplay(), eglSurface, EGL_WIDTH, &width);
eglQuerySurface(eglGetCurrentDisplay(), eglSurface, EGL_HEIGHT, &height);
GrBackendRenderTarget renderTarget(width,
height,
sampleCnt,
stencilBits,
fboInfo);
SkSurfaceProps props(0, kUnknown_SkPixelGeometry);
sk_sp<SkSurface> surface = SkSurfaces::WrapBackendRenderTarget(_skiaContext.get(),
renderTarget,
kBottomLeft_GrSurfaceOrigin,
kN32_SkColorType,
nullptr,
&props,
nullptr,
nullptr);
if (surface == nullptr) {
[[unlikely]];
throw std::runtime_error("Failed to create Skia Surface! Cannot wrap EGLSurface/FrameBuffer using Skia.");
}
return surface;
}
sk_sp<SkSurface> SkiaRenderer::getOffscreenSurface(int width, int height) {
if (_offscreenSurface == nullptr || _offscreenSurface->width() != width || _offscreenSurface->height() != height) {
// 1. Get Skia Context
sk_sp<GrDirectContext> skiaContext = getSkiaContext();
// 2. Create a backend texture (TEXTURE_2D + Frame Buffer)
GrBackendTexture backendTexture = skiaContext->createBackendTexture(width,
height,
SkColorType::kN32_SkColorType,
GrMipMapped::kNo,
GrRenderable::kYes);
// 3. Get it's Texture ID
GrGLTextureInfo info;
backendTexture.getGLTextureInfo(&info);
_offscreenSurfaceTextureId = info.fID;
struct ReleaseContext {
GrDirectContext* context;
GrBackendTexture texture;
};
auto releaseCtx = new ReleaseContext(
{skiaContext.get(), backendTexture});
SkSurfaces::TextureReleaseProc releaseProc = [] (void* address) {
// 5. Once done using, delete the backend OpenGL texture.
auto releaseCtx = reinterpret_cast<ReleaseContext*>(address);
releaseCtx->context->deleteBackendTexture(releaseCtx->texture);
};
// 4. Wrap the newly created texture as an SkSurface
SkSurfaceProps props(0, kUnknown_SkPixelGeometry);
_offscreenSurface = SkSurfaces::WrapBackendTexture(skiaContext.get(),
backendTexture,
kBottomLeft_GrSurfaceOrigin,
0,
SkColorType::kN32_SkColorType,
nullptr,
&props,
releaseProc,
releaseCtx);
if (_offscreenSurface == nullptr) {
[[unlikely]];
throw std::runtime_error("Failed to create offscreen Skia Surface!");
}
}
return _offscreenSurface;
}
OpenGLTexture SkiaRenderer::renderTextureToOffscreenSurface(OpenGLContext& glContext,
OpenGLTexture& texture,
float* transformMatrix,
const DrawCallback& drawCallback) {
// 1. Activate the OpenGL context (eglMakeCurrent)
glContext.use();
// 2. Initialize Skia
sk_sp<GrDirectContext> skiaContext = getSkiaContext();
// 3. Create the offscreen Skia Surface
sk_sp<SkSurface> surface = getOffscreenSurface(texture.width, texture.height);
// 4. Wrap the input texture as an image so we can draw it to the surface
sk_sp<SkImage> frame = wrapTextureAsImage(texture);
// 5. Prepare the Canvas
SkCanvas* canvas = _offscreenSurface->getCanvas();
if (canvas == nullptr) {
[[unlikely]];
throw std::runtime_error("Failed to get Skia Canvas!");
}
// TODO: Apply Matrix. No idea how though.
SkM44 matrix = SkM44::ColMajor(transformMatrix);
// 6. Render it!
canvas->clear(SkColors::kBlack);
canvas->drawImage(frame, 0, 0);
drawCallback(canvas);
// 8. Flush all Skia operations to OpenGL
_offscreenSurface->flushAndSubmit();
return OpenGLTexture {
.id = _offscreenSurfaceTextureId,
.target = GL_TEXTURE_2D,
.width = texture.width,
.height = texture.height,
};
}
void SkiaRenderer::renderTextureToSurface(OpenGLContext &glContext, OpenGLTexture &texture, EGLSurface surface) {
// 1. Activate the OpenGL context (eglMakeCurrent)
glContext.use(surface);
// 2. Initialize Skia
sk_sp<GrDirectContext> skiaContext = getSkiaContext();
// 3. Wrap the output EGLSurface in a Skia SkSurface
sk_sp<SkSurface> skSurface = wrapEglSurfaceAsSurface(surface);
// 4. Wrap the input texture in a Skia SkImage
sk_sp<SkImage> frame = wrapTextureAsImage(texture);
// 5. Prepare the Canvas!
SkCanvas* canvas = skSurface->getCanvas();
if (canvas == nullptr) {
[[unlikely]];
throw std::runtime_error("Failed to get Skia Canvas!");
}
// 6. Render it!
canvas->clear(SkColors::kBlack);
canvas->drawImage(frame, 0, 0);
// 7. Flush all Skia operations to OpenGL
skSurface->flushAndSubmit();
// 8. Swap the buffers so the onscreen surface gets updated.
glContext.flush();
}
} // namespace vision
#endif

72
android/src/main/cpp/skia/SkiaRenderer.h
View File

@ -1,72 +0,0 @@
//
// Created by Marc Rousavy on 10.08.23.
//
#pragma once
#if VISION_CAMERA_ENABLE_SKIA
#include <GLES2/gl2.h>
#include <EGL/egl.h>
#include <android/native_window.h>
#include <include/core/SkSurface.h>
#include <include/gpu/GrDirectContext.h>
#include "OpenGLContext.h"
#include "OpenGLTexture.h"
namespace vision {
#define NO_TEXTURE 0
using DrawCallback = std::function<void(SkCanvas*)>;
class SkiaRenderer {
public:
/**
* Create a new Skia renderer. You need to use OpenGL outside of this context to make sure the
* Skia renderer can use the global OpenGL context.
*/
explicit SkiaRenderer() {};
~SkiaRenderer();
/**
* Renders the given Texture (might be a Camera Frame) to a cached offscreen Texture using Skia.
*
* @returns The texture that was rendered to.
*/
OpenGLTexture renderTextureToOffscreenSurface(OpenGLContext& glContext,
OpenGLTexture& texture,
float* transformMatrix,
const DrawCallback& drawCallback);
/**
* Renders the given texture to the target output surface using Skia.
*/
void renderTextureToSurface(OpenGLContext& glContext,
OpenGLTexture& texture,
EGLSurface surface);
private:
// Gets or creates the Skia context.
sk_sp<GrDirectContext> getSkiaContext();
// Wraps a Texture as an SkImage allowing you to draw it
sk_sp<SkImage> wrapTextureAsImage(OpenGLTexture& texture);
// Wraps an EGLSurface as an SkSurface allowing you to draw into it
sk_sp<SkSurface> wrapEglSurfaceAsSurface(EGLSurface eglSurface);
// Gets or creates an off-screen surface that you can draw into
sk_sp<SkSurface> getOffscreenSurface(int width, int height);
private:
// Skia Context
sk_sp<GrDirectContext> _skiaContext = nullptr;
sk_sp<SkSurface> _offscreenSurface = nullptr;
GLuint _offscreenSurfaceTextureId = NO_TEXTURE;
static auto constexpr TAG = "SkiaRenderer";
};
} // namespace vision
#endif

8
android/src/main/cpp/skia/VisionCameraSkiaContext.cpp
View File

@ -1,8 +0,0 @@
//
// Created by Marc Rousavy on 31.08.23.
//
#include "VisionCameraSkiaContext.h"
namespace vision {
} // vision

52
android/src/main/cpp/skia/VisionCameraSkiaContext.h
View File

@ -1,52 +0,0 @@
//
// Created by Marc Rousavy on 31.08.23.
//
#pragma once
#include <jsi/jsi.h>
#include <RNSkPlatformContext.h>
namespace vision {
using namespace facebook;
class VisionCameraSkiaContext: public RNSkia::RNSkPlatformContext {
public:
VisionCameraSkiaContext(jsi::Runtime* runtime,
std::shared_ptr<react::CallInvoker> callInvoker,
float pixelDensity)
: RNSkia::RNSkPlatformContext(runtime, callInvoker, pixelDensity) { }
void raiseError(const std::exception &err) override {
throw std::runtime_error("VisionCameraSkiaContext Error: " + std::string(err.what()));
}
void performStreamOperation(
const std::string &sourceUri,
const std::function<void(std::unique_ptr<SkStreamAsset>)> &op) override {
throw std::runtime_error("VisionCameraSkiaContext::performStreamOperation is not yet implemented!");
}
sk_sp<SkSurface> makeOffscreenSurface(int width, int height) override {
throw std::runtime_error("VisionCameraSkiaContext::makeOffscreenSurface is not yet implemented!");
}
void runOnMainThread(std::function<void()> task) override {
throw std::runtime_error("VisionCameraSkiaContext::runOnMainThread is not yet implemented!");
}
sk_sp<SkImage> takeScreenshotFromViewTag(size_t tag) override {
throw std::runtime_error("VisionCameraSkiaContext::takeScreenshotFromViewTag is not yet implemented!");
}
void startDrawLoop() override {
throw std::runtime_error("VisionCameraSkiaContext::startDrawLoop is not yet implemented!");
}
void stopDrawLoop() override {
throw std::runtime_error("VisionCameraSkiaContext::stopDrawLoop is not yet implemented!");
}
};
} // namespace vision

16
android/src/main/java/com/mrousavy/camera/CameraSession.kt
View File

@ -200,9 +200,8 @@ class CameraSession(private val context: Context,
private fun updateVideoOutputs() {
val videoPipeline = outputs?.videoOutput?.videoPipeline ?: return
val previewOutput = outputs?.previewOutput
videoPipeline.setRecordingSessionOutput(recording)
videoPipeline.setFrameProcessorOutput(frameProcessor)
videoPipeline.setPreviewOutput(previewOutput?.surface)
videoPipeline.setRecordingSessionOutput(this.recording)
videoPipeline.setFrameProcessorOutput(this.frameProcessor)
}
suspend fun takePhoto(qualityPrioritization: QualityPrioritization,
@ -216,6 +215,8 @@ class CameraSession(private val context: Context,
val photoOutput = outputs.photoOutput ?: throw PhotoNotEnabledError()
Log.i(TAG, "Photo capture 0/3 - preparing capture request (${photoOutput.size.width}x${photoOutput.size.height})...")
val cameraCharacteristics = cameraManager.getCameraCharacteristics(captureSession.device.id)
val orientation = outputOrientation.toSensorRelativeOrientation(cameraCharacteristics)
val captureRequest = captureSession.device.createPhotoCaptureRequest(cameraManager,
@ -226,16 +227,16 @@ class CameraSession(private val context: Context,
enableRedEyeReduction,
enableAutoStabilization,
orientation)
Log.i(TAG, "Photo capture 0/2 - starting capture...")
Log.i(TAG, "Photo capture 1/3 - starting capture...")
val result = captureSession.capture(captureRequest, enableShutterSound)
val timestamp = result[CaptureResult.SENSOR_TIMESTAMP]!!
Log.i(TAG, "Photo capture 1/2 complete - received metadata with timestamp $timestamp")
Log.i(TAG, "Photo capture 2/3 complete - received metadata with timestamp $timestamp")
try {
val image = photoOutputSynchronizer.await(timestamp)
val isMirrored = cameraCharacteristics.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT
Log.i(TAG, "Photo capture 2/2 complete - received ${image.width} x ${image.height} image.")
Log.i(TAG, "Photo capture 3/3 complete - received ${image.width} x ${image.height} image.")
return CapturedPhoto(image, result, orientation, isMirrored, image.format)
} catch (e: CancellationException) {
throw CaptureAbortedError(false)
@ -501,8 +502,7 @@ class CameraSession(private val context: Context,
val captureRequest = camera.createCaptureRequest(template)
outputs.previewOutput?.let { output ->
Log.i(TAG, "Adding output surface ${output.outputType}..")
// TODO: Add here again?
// captureRequest.addTarget(output.surface)
captureRequest.addTarget(output.surface)
}
outputs.videoOutput?.let { output ->
Log.i(TAG, "Adding output surface ${output.outputType}..")

3
android/src/main/java/com/mrousavy/camera/CameraView.kt
View File

@ -25,6 +25,7 @@ import com.mrousavy.camera.utils.outputs.CameraOutputs
import kotlinx.coroutines.CoroutineScope
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.launch
import java.io.Closeable
//
// TODOs for the CameraView which are currently too hard to implement either because of CameraX' limitations, or my brain capacity.
@ -90,7 +91,7 @@ class CameraView(context: Context) : FrameLayout(context) {
internal var frameProcessor: FrameProcessor? = null
set(value) {
field = value
cameraSession.frameProcessor = value
cameraSession.frameProcessor = frameProcessor
}
private val inputOrientation: Orientation

3
android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt
View File

@ -64,8 +64,7 @@ suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager,
val outputConfigurations = arrayListOf<OutputConfiguration>()
outputs.previewOutput?.let { output ->
// TODO: add here again?
// outputConfigurations.add(output.toOutputConfiguration(characteristics))
outputConfigurations.add(output.toOutputConfiguration(characteristics))
}
outputs.photoOutput?.let { output ->
outputConfigurations.add(output.toOutputConfiguration(characteristics))

187
android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java
View File

@ -1,66 +1,147 @@
package com.mrousavy.camera.frameprocessor;
import com.facebook.jni.HybridData;
import android.graphics.ImageFormat;
import android.media.Image;
import com.facebook.proguard.annotations.DoNotStrip;
import com.mrousavy.camera.parsers.PixelFormat;
import com.mrousavy.camera.parsers.Orientation;
import java.nio.ByteBuffer;
/** @noinspection JavaJniMissingFunction*/
public class Frame {
private final HybridData mHybridData;
private final Image image;
private final boolean isMirrored;
private final long timestamp;
private final Orientation orientation;
private int refCount = 0;
private Frame(HybridData hybridData) {
mHybridData = hybridData;
public Frame(Image image, long timestamp, Orientation orientation, boolean isMirrored) {
this.image = image;
this.timestamp = timestamp;
this.orientation = orientation;
this.isMirrored = isMirrored;
}
@Override
protected void finalize() throws Throwable {
super.finalize();
mHybridData.resetNative();
public Image getImage() {
return image;
}
/**
* Get the width of the Frame, in it's sensor orientation. (in pixels)
*/
public native int getWidth();
/**
* Get the height of the Frame, in it's sensor orientation. (in pixels)
*/
public native int getHeight();
/**
* Get the number of bytes per row.
* * To get the number of components per pixel you can divide this with the Frame's width.
* * To get the total size of the byte buffer you can multiply this with the Frame's height.
*/
public native int getBytesPerRow();
/**
* Get the local timestamp of this Frame. This is always monotonically increasing for each Frame.
*/
public native long getTimestamp();
/**
* Get the Orientation of this Frame. The return value is the result of `Orientation.toUnionValue()`.
*/
public native String getOrientation();
/**
* Return whether this Frame is mirrored or not. Frames from the front-facing Camera are often mirrored.
*/
public native boolean getIsMirrored();
/**
* Get the pixel-format of this Frame. The return value is the result of `PixelFormat.toUnionValue()`.
*/
public native String getPixelFormat();
/**
* Get the actual backing pixel data of this Frame using a zero-copy C++ ByteBuffer.
*/
public native ByteBuffer getByteBuffer();
/**
* Get whether this Frame is still valid.
* A Frame is valid as long as it hasn't been closed by the Frame Processor Runtime Manager
* (either because it ran out of Frames in it's queue and needs to close old ones, or because
* a Frame Processor finished executing and you're still trying to hold onto this Frame in native)
*/
public native boolean getIsValid();
@SuppressWarnings("unused")
@DoNotStrip
public int getWidth() {
return image.getWidth();
}
private native void incrementRefCount();
private native void decrementRefCount();
private native void close();
@SuppressWarnings("unused")
@DoNotStrip
public int getHeight() {
return image.getHeight();
}
@SuppressWarnings("unused")
@DoNotStrip
public boolean getIsValid() {
try {
// will throw an exception if the image is already closed
image.getCropRect();
// no exception thrown, image must still be valid.
return true;
} catch (Exception e) {
// exception thrown, image has already been closed.
return false;
}
}
@SuppressWarnings("unused")
@DoNotStrip
public boolean getIsMirrored() {
return isMirrored;
}
@SuppressWarnings("unused")
@DoNotStrip
public long getTimestamp() {
return timestamp;
}
@SuppressWarnings("unused")
@DoNotStrip
public String getOrientation() {
return orientation.getUnionValue();
}
@SuppressWarnings("unused")
@DoNotStrip
public String getPixelFormat() {
PixelFormat format = PixelFormat.Companion.fromImageFormat(image.getFormat());
return format.getUnionValue();
}
@SuppressWarnings("unused")
@DoNotStrip
public int getPlanesCount() {
return image.getPlanes().length;
}
@SuppressWarnings("unused")
@DoNotStrip
public int getBytesPerRow() {
return image.getPlanes()[0].getRowStride();
}
private static ByteBuffer byteArrayCache;
@SuppressWarnings("unused")
@DoNotStrip
public ByteBuffer toByteBuffer() {
switch (image.getFormat()) {
case ImageFormat.YUV_420_888:
ByteBuffer yBuffer = image.getPlanes()[0].getBuffer();
ByteBuffer uBuffer = image.getPlanes()[1].getBuffer();
ByteBuffer vBuffer = image.getPlanes()[2].getBuffer();
int ySize = yBuffer.remaining();
int uSize = uBuffer.remaining();
int vSize = vBuffer.remaining();
int totalSize = ySize + uSize + vSize;
if (byteArrayCache != null) byteArrayCache.rewind();
if (byteArrayCache == null || byteArrayCache.remaining() != totalSize) {
byteArrayCache = ByteBuffer.allocateDirect(totalSize);
}
byteArrayCache.put(yBuffer).put(uBuffer).put(vBuffer);
return byteArrayCache;
case ImageFormat.JPEG:
return image.getPlanes()[0].getBuffer();
default:
throw new RuntimeException("Cannot convert Frame with Format " + image.getFormat() + " to byte array!");
}
}
@SuppressWarnings("unused")
@DoNotStrip
public void incrementRefCount() {
synchronized (this) {
refCount++;
}
}
@SuppressWarnings("unused")
@DoNotStrip
public void decrementRefCount() {
synchronized (this) {
refCount--;
if (refCount <= 0) {
// If no reference is held on this Image, close it.
image.close();
}
}
}
@SuppressWarnings("unused")
@DoNotStrip
private void close() {
image.close();
}
}

10
android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessor.java
View File

@ -8,9 +8,15 @@ import com.facebook.jni.HybridData;
import com.facebook.proguard.annotations.DoNotStrip;
/**
* Represents a JS Frame Processor. It's actual implementation is in NDK/C++.
* Represents a JS Frame Processor
*/
public class FrameProcessor {
@SuppressWarnings("JavaJniMissingFunction") // we're using fbjni.
public final class FrameProcessor {
/**
* Call the JS Frame Processor function with the given Frame
*/
public native void call(Frame frame);
@DoNotStrip
@Keep
private final HybridData mHybridData;

18
android/src/main/java/com/mrousavy/camera/parsers/PreviewType.kt
View File

@ -1,18 +0,0 @@
package com.mrousavy.camera.parsers
enum class PreviewType(override val unionValue: String): JSUnionValue {
NONE("none"),
NATIVE("native"),
SKIA("skia");
companion object: JSUnionValue.Companion<PreviewType> {
override fun fromUnionValue(unionValue: String?): PreviewType {
return when (unionValue) {
"none" -> NONE
"native" -> NATIVE
"skia" -> SKIA
else -> NONE
}
}
}
}

11
android/src/main/java/com/mrousavy/camera/skia/SkiaFrameProcessor.java
View File

@ -1,11 +0,0 @@
package com.mrousavy.camera.skia;
import com.facebook.jni.HybridData;
import com.mrousavy.camera.frameprocessor.FrameProcessor;
public class SkiaFrameProcessor extends FrameProcessor {
// Implementation is in JSkiaFrameProcessor.cpp
public SkiaFrameProcessor(HybridData hybridData) {
super(hybridData);
}
}

32
android/src/main/java/com/mrousavy/camera/utils/ImageCreator.kt
View File

@ -1,32 +0,0 @@
package com.mrousavy.camera.utils
import android.graphics.ImageFormat
import android.media.Image
import android.media.ImageReader
import android.media.ImageWriter
import java.io.Closeable
class ImageCreator(private val width: Int,
private val height: Int,
private val format: Int = ImageFormat.PRIVATE,
private val maxImages: Int = 3): Closeable {
private var imageReader: ImageReader? = null
private var imageWriter: ImageWriter? = null
override fun close() {
imageWriter?.close()
imageReader?.close()
}
fun createImage(): Image {
if (imageReader == null || imageWriter == null) {
imageWriter?.close()
imageReader?.close()
imageReader = ImageReader.newInstance(width, height, format, maxImages)
imageWriter = ImageWriter.newInstance(imageReader!!.surface, maxImages)
}
return imageWriter!!.dequeueInputImage()
}
}

5
android/src/main/java/com/mrousavy/camera/utils/RecordingSession.kt
View File

@ -41,6 +41,7 @@ class RecordingSession(context: Context,
val surface: Surface = MediaCodec.createPersistentInputSurface()
init {
outputFile = File.createTempFile("mrousavy", fileType.toExtension(), context.cacheDir)
Log.i(TAG, "Creating RecordingSession for ${outputFile.absolutePath}")
@ -53,7 +54,7 @@ class RecordingSession(context: Context,
recorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4)
recorder.setOutputFile(outputFile.absolutePath)
recorder.setVideoEncodingBitRate(VIDEO_BIT_RATE)
recorder.setVideoSize(size.width, size.height)
recorder.setVideoSize(size.height, size.width)
if (fps != null) recorder.setVideoFrameRate(fps)
Log.i(TAG, "Using $codec Video Codec..")
@ -66,7 +67,7 @@ class RecordingSession(context: Context,
recorder.setAudioChannels(AUDIO_CHANNELS)
}
recorder.setInputSurface(surface)
recorder.setOrientationHint(orientation.toDegrees())
//recorder.setOrientationHint(orientation.toDegrees())
recorder.setOnErrorListener { _, what, extra ->
Log.e(TAG, "MediaRecorder Error: $what ($extra)")

70
android/src/main/java/com/mrousavy/camera/utils/VideoPipeline.kt
View File

@ -2,12 +2,15 @@ package com.mrousavy.camera.utils
import android.graphics.ImageFormat
import android.graphics.SurfaceTexture
import android.media.ImageReader
import android.media.ImageWriter
import android.media.MediaRecorder
import android.util.Log
import android.view.Surface
import com.facebook.jni.HybridData
import com.mrousavy.camera.frameprocessor.Frame
import com.mrousavy.camera.frameprocessor.FrameProcessor
import com.mrousavy.camera.parsers.Orientation
import java.io.Closeable
/**
@ -23,25 +26,21 @@ class VideoPipeline(val width: Int,
val height: Int,
val format: Int = ImageFormat.PRIVATE): SurfaceTexture.OnFrameAvailableListener, Closeable {
companion object {
private const val MAX_IMAGES = 3
private const val MAX_IMAGES = 5
private const val TAG = "VideoPipeline"
}
private val mHybridData: HybridData
private var isActive = true
// Input Texture
private var openGLTextureId: Int? = null
private var transformMatrix = FloatArray(16)
// Processing input texture
private var frameProcessor: FrameProcessor? = null
private var isActive = true
// Output 1
private var recordingSession: RecordingSession? = null
private var frameProcessor: FrameProcessor? = null
private var imageReader: ImageReader? = null
// Output 2
private var previewSurface: Surface? = null
private var recordingSession: RecordingSession? = null
// Input
private val surfaceTexture: SurfaceTexture
@ -58,6 +57,8 @@ class VideoPipeline(val width: Int,
override fun close() {
synchronized(this) {
isActive = false
imageReader?.close()
imageReader = null
frameProcessor = null
recordingSession = null
surfaceTexture.release()
@ -90,6 +91,21 @@ class VideoPipeline(val width: Int,
}
}
private fun getImageReader(): ImageReader {
val imageReader = ImageReader.newInstance(width, height, format, MAX_IMAGES)
imageReader.setOnImageAvailableListener({ reader ->
Log.i("VideoPipeline", "ImageReader::onImageAvailable!")
val image = reader.acquireLatestImage() ?: return@setOnImageAvailableListener
// TODO: Get correct orientation and isMirrored
val frame = Frame(image, image.timestamp, Orientation.PORTRAIT, false)
frame.incrementRefCount()
frameProcessor?.call(frame)
frame.decrementRefCount()
}, null)
return imageReader
}
/**
* Configures the Pipeline to also call the given [FrameProcessor].
* * If the [frameProcessor] is `null`, this output channel will be removed.
@ -102,11 +118,20 @@ class VideoPipeline(val width: Int,
this.frameProcessor = frameProcessor
if (frameProcessor != null) {
// Configure OpenGL pipeline to stream Frames into the Frame Processor (CPU pixel access)
setFrameProcessor(frameProcessor)
if (this.imageReader == null) {
// 1. Create new ImageReader that just calls the Frame Processor
this.imageReader = getImageReader()
}
// 2. Configure OpenGL pipeline to stream Frames into the ImageReader's surface
setFrameProcessorOutputSurface(imageReader!!.surface)
} else {
// Configure OpenGL pipeline to stop streaming Frames into a Frame Processor
removeFrameProcessor()
// 1. Configure OpenGL pipeline to stop streaming Frames into the ImageReader's surface
removeFrameProcessorOutputSurface()
// 2. Close the ImageReader
this.imageReader?.close()
this.imageReader = null
}
}
}
@ -131,27 +156,12 @@ class VideoPipeline(val width: Int,
}
}
fun setPreviewOutput(surface: Surface?) {
synchronized(this) {
Log.i(TAG, "Setting Preview Output...")
if (surface != null) {
setPreviewOutputSurface(surface)
this.previewSurface = surface
} else {
removePreviewOutputSurface()
this.previewSurface = null
}
}
}
private external fun getInputTextureId(): Int
private external fun onBeforeFrame()
private external fun onFrame(transformMatrix: FloatArray)
private external fun setFrameProcessor(frameProcessor: FrameProcessor)
private external fun removeFrameProcessor()
private external fun setFrameProcessorOutputSurface(surface: Any)
private external fun removeFrameProcessorOutputSurface()
private external fun setRecordingSessionOutputSurface(surface: Any)
private external fun removeRecordingSessionOutputSurface()
private external fun setPreviewOutputSurface(surface: Any)
private external fun removePreviewOutputSurface()
private external fun initHybrid(width: Int, height: Int): HybridData
}

6
docs/docs/guides/TROUBLESHOOTING.mdx
View File

@ -38,8 +38,7 @@ Before opening an issue, make sure you try the following:
3. Select **Swift File** and press **Next**
4. Choose whatever name you want, e.g. `File.swift` and press **Create**
5. Press **Create Bridging Header** when promted.
6. Try building without Skia. Set `$VCDisableSkia = true` in the top of your Podfile, and try rebuilding.
7. Try building without Frame Processors. Set `$VCDisableFrameProcessors = true` in the top of your Podfile, and try rebuilding.
6. Try building without Frame Processors. Set `$VCDisableFrameProcessors = true` in the top of your Podfile, and try rebuilding.
### Runtime Issues
@ -85,8 +84,7 @@ Before opening an issue, make sure you try the following:
```
distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-all.zip
```
7. Try building without Skia. Set `VisionCamera_disableSkia = true` in your `gradle.properties`, and try rebuilding.
8. Try building without Frame Processors. Set `VisionCamera_disableFrameProcessors = true` in your `gradle.properties`, and try rebuilding.
7. Try building without Frame Processors. Set `VisionCamera_disableFrameProcessors = true` in your `gradle.properties`, and try rebuilding.
### Runtime Issues

3
example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java
View File

@ -17,8 +17,9 @@ public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin {
@Override
public Object callback(@NotNull Frame frame, @Nullable Map<String, Object> params) {
if (params == null) return null;
Image image = frame.getImage();
Log.d("ExamplePlugin", frame.getWidth() + " x " + frame.getHeight() + " Image with format #" + frame.getPixelFormat() + ". Logging " + params.size() + " parameters:");
Log.d("ExamplePlugin", image.getWidth() + " x " + image.getHeight() + " Image with format #" + image.getFormat() + ". Logging " + params.size() + " parameters:");
for (String key : params.keySet()) {
Object value = params.get(key);

1
example/android/gradle.properties
View File

@ -41,4 +41,3 @@ hermesEnabled=true
# Can be set to true to disable the build setup
#VisionCamera_disableFrameProcessors=true
#VisionCamera_disableSkia=true

13
example/ios/Podfile.lock
View File

@ -333,10 +333,6 @@ PODS:
- React-Core
- react-native-safe-area-context (4.7.1):
- React-Core
- react-native-skia (0.1.200):
- React
- React-callinvoker
- React-Core
- react-native-video (5.2.1):
- React-Core
- react-native-video/Video (= 5.2.1)
@ -505,11 +501,10 @@ PODS:
- libwebp (~> 1.0)
- SDWebImage/Core (~> 5.10)
- SocketRocket (0.6.1)
- VisionCamera (3.0.0-rc.6):
- VisionCamera (3.0.0-rc.8):
- React
- React-callinvoker
- React-Core
- react-native-skia
- react-native-worklets-core
- Yoga (1.14.0)
@ -540,7 +535,6 @@ DEPENDENCIES:
- "react-native-blur (from `../node_modules/@react-native-community/blur`)"
- "react-native-cameraroll (from `../node_modules/@react-native-camera-roll/camera-roll`)"
- react-native-safe-area-context (from `../node_modules/react-native-safe-area-context`)
- "react-native-skia (from `../node_modules/@shopify/react-native-skia`)"
- react-native-video (from `../node_modules/react-native-video`)
- react-native-worklets-core (from `../node_modules/react-native-worklets-core`)
- React-NativeModulesApple (from `../node_modules/react-native/ReactCommon/react/nativemodule/core/platform/ios`)
@ -628,8 +622,6 @@ EXTERNAL SOURCES:
:path: "../node_modules/@react-native-camera-roll/camera-roll"
react-native-safe-area-context:
:path: "../node_modules/react-native-safe-area-context"
react-native-skia:
:path: "../node_modules/@shopify/react-native-skia"
react-native-video:
:path: "../node_modules/react-native-video"
react-native-worklets-core:
@ -713,7 +705,6 @@ SPEC CHECKSUMS:
react-native-blur: cfdad7b3c01d725ab62a8a729f42ea463998afa2
react-native-cameraroll: 134805127580aed23403b8c2cb1548920dd77b3a
react-native-safe-area-context: 9697629f7b2cda43cf52169bb7e0767d330648c2
react-native-skia: d0b0aab6bb1f146eb6f379fb671b719deabd20fb
react-native-video: c26780b224543c62d5e1b2a7244a5cd1b50e8253
react-native-worklets-core: 7ad416a8965086b98b07964f7f6932560a54a14c
React-NativeModulesApple: c57f3efe0df288a6532b726ad2d0322a9bf38472
@ -742,7 +733,7 @@ SPEC CHECKSUMS:
SDWebImage: a7f831e1a65eb5e285e3fb046a23fcfbf08e696d
SDWebImageWebPCoder: 908b83b6adda48effe7667cd2b7f78c897e5111d
SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17
VisionCamera: 35a762f77816462a4d59a580ca197ffa29954112
VisionCamera: 5bd7961602a7db4de21fdc3588df6ce01d693d37
Yoga: 8796b55dba14d7004f980b54bcc9833ee45b28ce
PODFILE CHECKSUM: ab9c06b18c63e741c04349c0fd630c6d3145081c

1
example/package.json
View File

@ -18,7 +18,6 @@
"@react-native-community/blur": "^4.3.2",
"@react-navigation/native": "^6.1.7",
"@react-navigation/native-stack": "^6.9.13",
"@shopify/react-native-skia": "^0.1.200",
"react": "^18.2.0",
"react-native": "^0.72.3",
"react-native-fast-image": "^8.6.3",

30
example/src/CameraPage.tsx
View File

@ -9,7 +9,6 @@ import {
sortFormats,
useCameraDevices,
useFrameProcessor,
useSkiaFrameProcessor,
VideoFile,
} from 'react-native-vision-camera';
import { Camera } from 'react-native-vision-camera';
@ -25,7 +24,6 @@ import IonIcon from 'react-native-vector-icons/Ionicons';
import type { Routes } from './Routes';
import type { NativeStackScreenProps } from '@react-navigation/native-stack';
import { useIsFocused } from '@react-navigation/core';
import { Skia } from '@shopify/react-native-skia';
import { FACE_SHADER } from './Shaders';
import { examplePlugin } from './frame-processors/ExamplePlugin';
@ -199,35 +197,11 @@ export function CameraPage({ navigation }: Props): React.ReactElement {
console.log('re-rendering camera page without active camera');
}
const radius = (format?.videoHeight ?? 1080) * 0.1;
const width = radius;
const height = radius;
const x = (format?.videoHeight ?? 1080) / 2 - radius / 2;
const y = (format?.videoWidth ?? 1920) / 2 - radius / 2;
const centerX = x + width / 2;
const centerY = y + height / 2;
const runtimeEffect = Skia.RuntimeEffect.Make(FACE_SHADER);
if (runtimeEffect == null) throw new Error('Shader failed to compile!');
const shaderBuilder = Skia.RuntimeShaderBuilder(runtimeEffect);
shaderBuilder.setUniform('r', [width]);
shaderBuilder.setUniform('x', [centerX]);
shaderBuilder.setUniform('y', [centerY]);
shaderBuilder.setUniform('resolution', [1920, 1080]);
const imageFilter = Skia.ImageFilter.MakeRuntimeShader(shaderBuilder, null, null);
const paint = Skia.Paint();
paint.setImageFilter(imageFilter);
const frameProcessor = useSkiaFrameProcessor((frame) => {
const frameProcessor = useFrameProcessor((frame) => {
'worklet';
const rect = Skia.XYWHRect(150, 150, 300, 300);
const paint = Skia.Paint();
paint.setColor(Skia.Color('red'));
frame.drawRect(rect, paint);
console.log(frame.timestamp, frame.toString(), frame.pixelFormat);
examplePlugin(frame);
}, []);
return (

28
example/yarn.lock
View File

@ -2319,14 +2319,6 @@
dependencies:
nanoid "^3.1.23"
"@shopify/react-native-skia@^0.1.200":
version "0.1.200"
resolved "https://registry.yarnpkg.com/@shopify/react-native-skia/-/react-native-skia-0.1.200.tgz#3ef86750106a3b7e02496133173b449bfce6abc2"
integrity sha512-wAauKsLgScLspJY4KzoV0lWoXFCbzsUDJ3uso0o81HQMKBjDvXG9aOq/xE0KFLQsrQVICRdbfvvoYLQvSh/Xmw==
dependencies:
canvaskit-wasm "0.38.0"
react-reconciler "^0.27.0"
"@sideway/address@^4.1.3":
version "4.1.4"
resolved "https://registry.yarnpkg.com/@sideway/address/-/address-4.1.4.tgz#03dccebc6ea47fdc226f7d3d1ad512955d4783f0"
@ -3019,11 +3011,6 @@ caniuse-lite@^1.0.30001503:
resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001517.tgz#90fabae294215c3495807eb24fc809e11dc2f0a8"
integrity sha512-Vdhm5S11DaFVLlyiKu4hiUTkpZu+y1KA/rZZqVQfOD5YdDT/eQKlkt7NaE0WGOFgX32diqt9MiP9CAiFeRklaA==
canvaskit-wasm@0.38.0:
version "0.38.0"
resolved "https://registry.yarnpkg.com/canvaskit-wasm/-/canvaskit-wasm-0.38.0.tgz#83e6c46f3015c2ff3f6503157f47453af76a7be7"
integrity sha512-ZEG6lucpbQ4Ld+mY8C1Ng+PMLVP+/AX02jS0Sdl28NyMxuKSa9uKB8oGd1BYp1XWPyO2Jgr7U8pdyjJ/F3xR5Q==
chalk@^2.0.0, chalk@^2.4.2:
version "2.4.2"
resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424"
@ -5857,14 +5844,6 @@ react-native@^0.72.3:
ws "^6.2.2"
yargs "^17.6.2"
react-reconciler@^0.27.0:
version "0.27.0"
resolved "https://registry.yarnpkg.com/react-reconciler/-/react-reconciler-0.27.0.tgz#360124fdf2d76447c7491ee5f0e04503ed9acf5b"
integrity sha512-HmMDKciQjYmBRGuuhIaKA1ba/7a+UsM5FzOZsMO2JYHt9Jh8reCb7j1eDC95NOyUlKM9KRyvdx0flBuDvYSBoA==
dependencies:
loose-envify "^1.1.0"
scheduler "^0.21.0"
react-refresh@^0.4.0:
version "0.4.3"
resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.4.3.tgz#966f1750c191672e76e16c2efa569150cc73ab53"
@ -6115,13 +6094,6 @@ scheduler@0.24.0-canary-efb381bbf-20230505:
dependencies:
loose-envify "^1.1.0"
scheduler@^0.21.0:
version "0.21.0"
resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.21.0.tgz#6fd2532ff5a6d877b6edb12f00d8ab7e8f308820"
integrity sha512-1r87x5fz9MXqswA2ERLo0EbOAU74DpIUO090gIasYTqlVoJeMcl+Z1Rg7WHz+qtPujhS/hGIt9kxZOYBV3faRQ==
dependencies:
loose-envify "^1.1.0"
semver@^5.6.0:
version "5.7.1"
resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"

3
ios/CameraError.swift
View File

@ -245,7 +245,6 @@ enum CaptureError {
enum SystemError: String {
case noManager = "no-camera-manager"
case skiaUnavailable = "skia-unavailable"
case frameProcessorsUnavailable = "frame-processors-unavailable"
var code: String {
@ -256,8 +255,6 @@ enum SystemError: String {
switch self {
case .noManager:
return "No Camera Manager was found."
case .skiaUnavailable:
return "Skia Integration is unavailable - is @shopify/react-native-skia installed?"
case .frameProcessorsUnavailable:
return "Frame Processors are unavailable - is react-native-worklets-core installed?"
}

37
ios/CameraView+Preview.swift
View File

@ -10,41 +10,10 @@ import AVFoundation
import Foundation
extension CameraView {
#if VISION_CAMERA_ENABLE_SKIA
@objc
func getSkiaRenderer() -> SkiaRenderer {
if skiaRenderer == nil {
skiaRenderer = SkiaRenderer()
}
return skiaRenderer!
}
#endif
public func setupPreviewView() {
switch previewType {
case "none":
previewView?.removeFromSuperview()
previewView = nil
case "native":
// Normal iOS PreviewView is lighter and more performant (YUV Format, GPU only)
if previewView is NativePreviewView { return }
previewView?.removeFromSuperview()
previewView = NativePreviewView(frame: frame, session: captureSession)
addSubview(previewView!)
case "skia":
// Skia Preview View allows user to draw onto a Frame in a Frame Processor
#if VISION_CAMERA_ENABLE_SKIA
if previewView is SkiaPreviewView { return }
previewView?.removeFromSuperview()
previewView = SkiaPreviewView(frame: frame, skiaRenderer: getSkiaRenderer())
addSubview(previewView!)
#else
invokeOnError(.system(.skiaUnavailable))
return
#endif
default:
invokeOnError(.parameter(.invalid(unionName: "previewType", receivedValue: previewType as String)))
}
previewView?.removeFromSuperview()
previewView = NativePreviewView(frame: frame, session: captureSession)
addSubview(previewView!)
}
internal func setupFpsGraph() {

12
ios/CameraView.swift
View File

@ -26,8 +26,7 @@ private let propsThatRequireReconfiguration = ["cameraId",
"photo",
"video",
"enableFrameProcessor",
"pixelFormat",
"previewType"]
"pixelFormat"]
private let propsThatRequireDeviceReconfiguration = ["fps",
"hdr",
"lowLightBoost"]
@ -59,7 +58,6 @@ public final class CameraView: UIView {
@objc var zoom: NSNumber = 1.0 // in "factor"
@objc var enableFpsGraph = false
@objc var videoStabilizationMode: NSString?
@objc var previewType: NSString = "none"
// events
@objc var onInitialized: RCTDirectEventBlock?
@objc var onError: RCTDirectEventBlock?
@ -93,9 +91,6 @@ public final class CameraView: UIView {
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
@objc public var frameProcessor: FrameProcessor?
#endif
#if VISION_CAMERA_ENABLE_SKIA
internal var skiaRenderer: SkiaRenderer?
#endif
// CameraView+Zoom
internal var pinchGestureRecognizer: UIPinchGestureRecognizer?
internal var pinchScaleOffset: CGFloat = 1.0
@ -188,11 +183,6 @@ public final class CameraView: UIView {
let shouldUpdateVideoStabilization = willReconfigure || changedProps.contains("videoStabilizationMode")
let shouldUpdateOrientation = willReconfigure || changedProps.contains("orientation")
if changedProps.contains("previewType") {
DispatchQueue.main.async {
self.setupPreviewView()
}
}
if changedProps.contains("enableFpsGraph") {
DispatchQueue.main.async {
self.setupFpsGraph()

1
ios/CameraViewManager.m
View File

@ -41,7 +41,6 @@ RCT_EXPORT_VIEW_PROPERTY(videoStabilizationMode, NSString);
RCT_EXPORT_VIEW_PROPERTY(pixelFormat, NSString);
// other props
RCT_EXPORT_VIEW_PROPERTY(torch, NSString);
RCT_EXPORT_VIEW_PROPERTY(previewType, NSString);
RCT_EXPORT_VIEW_PROPERTY(zoom, NSNumber);
RCT_EXPORT_VIEW_PROPERTY(enableZoomGesture, BOOL);
RCT_EXPORT_VIEW_PROPERTY(enableFpsGraph, BOOL);

10
ios/Frame Processor/FrameHostObject.mm
View File

@ -18,10 +18,10 @@ std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt)
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("width")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("height")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("bytesPerRow")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("planesCount")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("orientation")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isMirrored")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("timestamp")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isDrawable")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("pixelFormat")));
// Conversion
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toString")));
@ -105,9 +105,6 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toArrayBuffer"), 0, toArrayBuffer);
}
if (name == "isDrawable") {
return jsi::Value(false);
}
if (name == "isValid") {
auto isValid = frame != nil && frame.buffer != nil && CFGetRetainCount(frame.buffer) > 0 && CMSampleBufferIsValid(frame.buffer);
return jsi::Value(isValid);
@ -175,6 +172,11 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
auto bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer);
return jsi::Value((double) bytesPerRow);
}
if (name == "planesCount") {
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto planesCount = CVPixelBufferGetPlaneCount(imageBuffer);
return jsi::Value((double) planesCount);
}
// fallback to base implementation
return HostObject::get(runtime, propName);

25
ios/Frame Processor/VisionCameraProxy.mm
View File

@ -24,11 +24,6 @@
#import <React/RCTUIManager.h>
#import <ReactCommon/RCTTurboModuleManager.h>
#if VISION_CAMERA_ENABLE_SKIA
#import "SkiaRenderer.h"
#import "../Skia Render Layer/SkiaFrameProcessor.h"
#endif
// Swift forward-declarations
__attribute__((objc_runtime_name("_TtC12VisionCamera12CameraQueues")))
@interface CameraQueues: NSObject
@ -38,9 +33,6 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera12CameraQueues")))
__attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
@interface CameraView: UIView
@property (nonatomic, copy) FrameProcessor* _Nullable frameProcessor;
#if VISION_CAMERA_ENABLE_SKIA
- (SkiaRenderer* _Nonnull)getSkiaRenderer;
#endif
@end
using namespace facebook;
@ -80,7 +72,6 @@ std::vector<jsi::PropNameID> VisionCameraProxy::getPropertyNames(jsi::Runtime& r
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("setFrameProcessor")));
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("removeFrameProcessor")));
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("getFrameProcessorPlugin")));
result.push_back(jsi::PropNameID::forUtf8(runtime, std::string("isSkiaEnabled")));
return result;
}
@ -96,15 +87,6 @@ void VisionCameraProxy::setFrameProcessor(jsi::Runtime& runtime, int viewTag, co
view.frameProcessor = [[FrameProcessor alloc] initWithWorklet:worklet
context:_workletContext];
} else if (frameProcessorType == "skia-frame-processor") {
#if VISION_CAMERA_ENABLE_SKIA
SkiaRenderer* skiaRenderer = [view getSkiaRenderer];
view.frameProcessor = [[SkiaFrameProcessor alloc] initWithWorklet:worklet
context:_workletContext
skiaRenderer:skiaRenderer];
#else
throw std::runtime_error("system/skia-unavailable: Skia is not installed!");
#endif
} else {
throw std::runtime_error("Unknown FrameProcessor.type passed! Received: " + frameProcessorType);
}
@ -135,13 +117,6 @@ jsi::Value VisionCameraProxy::getFrameProcessorPlugin(jsi::Runtime& runtime, std
jsi::Value VisionCameraProxy::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
auto name = propName.utf8(runtime);
if (name == "isSkiaEnabled") {
#ifdef VISION_CAMERA_ENABLE_SKIA
return jsi::Value(true);
#else
return jsi::Value(false);
#endif
}
if (name == "setFrameProcessor") {
return jsi::Function::createFromHostFunction(runtime,
jsi::PropNameID::forUtf8(runtime, "setFrameProcessor"),

35
ios/Skia Render Layer/DrawableFrameHostObject.h
View File

@ -1,35 +0,0 @@
//
// DrawableFrameHostObject.h
// VisionCamera
//
// Created by Marc Rousavy on 20.07.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
#pragma once
#import <jsi/jsi.h>
#import "../Frame Processor/FrameHostObject.h"
#import "../Frame Processor/Frame.h"
#import <CoreMedia/CMSampleBuffer.h>
#import "SkCanvas.h"
#import "JsiSkCanvas.h"
using namespace facebook;
class JSI_EXPORT DrawableFrameHostObject: public FrameHostObject {
public:
explicit DrawableFrameHostObject(Frame* frame,
std::shared_ptr<RNSkia::JsiSkCanvas> canvas):
FrameHostObject(frame), _canvas(canvas) {}
public:
jsi::Value get(jsi::Runtime&, const jsi::PropNameID& name) override;
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& rt) override;
void invalidateCanvas();
private:
std::shared_ptr<RNSkia::JsiSkCanvas> _canvas;
};

83
ios/Skia Render Layer/DrawableFrameHostObject.mm
View File

@ -1,83 +0,0 @@
//
// DrawableFrameHostObject.mm
// VisionCamera
//
// Created by Marc Rousavy on 20.07.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
#import "DrawableFrameHostObject.h"
#import "SkCanvas.h"
#import "SkImageHelpers.h"
std::vector<jsi::PropNameID> DrawableFrameHostObject::getPropertyNames(jsi::Runtime& rt) {
auto result = FrameHostObject::getPropertyNames(rt);
// Skia - Render Frame
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("render")));
if (_canvas != nullptr) {
auto canvasPropNames = _canvas->getPropertyNames(rt);
for (auto& prop : canvasPropNames) {
result.push_back(std::move(prop));
}
}
return result;
}
SkRect inscribe(SkSize size, SkRect rect) {
auto halfWidthDelta = (rect.width() - size.width()) / 2.0;
auto halfHeightDelta = (rect.height() - size.height()) / 2.0;
return SkRect::MakeXYWH(rect.x() + halfWidthDelta,
rect.y() + halfHeightDelta, size.width(),
size.height());
}
jsi::Value DrawableFrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
auto name = propName.utf8(runtime);
if (name == "render") {
auto render = JSI_HOST_FUNCTION_LAMBDA {
if (_canvas == nullptr) {
throw jsi::JSError(runtime, "Trying to render a Frame without a Skia Canvas! Did you install Skia?");
}
// convert CMSampleBuffer to SkImage
auto context = _canvas->getCanvas()->recordingContext();
auto image = SkImageHelpers::convertCMSampleBufferToSkImage(context, frame.buffer);
// draw SkImage
if (count > 0) {
// ..with paint/shader
auto paintHostObject = arguments[0].asObject(runtime).asHostObject<RNSkia::JsiSkPaint>(runtime);
auto paint = paintHostObject->getObject();
_canvas->getCanvas()->drawImage(image, 0, 0, SkSamplingOptions(), paint.get());
} else {
// ..without paint/shader
_canvas->getCanvas()->drawImage(image, 0, 0);
}
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "render"), 1, render);
}
if (name == "isDrawable") {
return jsi::Value(_canvas != nullptr);
}
if (_canvas != nullptr) {
// If we have a Canvas, try to access the property on there.
auto result = _canvas->get(runtime, propName);
if (!result.isUndefined()) {
return result;
}
}
// fallback to base implementation
return FrameHostObject::get(runtime, propName);
}
void DrawableFrameHostObject::invalidateCanvas() {
_canvas = nullptr;
}

42
ios/Skia Render Layer/SkImageHelpers.h
View File

@ -1,42 +0,0 @@
//
// SkImageHelpers.h
// VisionCamera
//
// Created by Marc Rousavy on 23.11.22.
// Copyright © 2022 mrousavy. All rights reserved.
//
#pragma once
#import <Foundation/Foundation.h>
#import <AVFoundation/AVFoundation.h>
#import <MetalKit/MetalKit.h>
#import <include/gpu/GrRecordingContext.h>
#import "SkImage.h"
#import "SkSize.h"
#import "SkRect.h"
class SkImageHelpers {
public:
SkImageHelpers() = delete;
public:
/**
Convert a CMSampleBuffer to an SkImage. Format has to be RGB.
*/
static sk_sp<SkImage> convertCMSampleBufferToSkImage(GrRecordingContext* context, CMSampleBufferRef sampleBuffer);
/**
Convert a MTLTexture to an SkImage. Format has to be RGB.
*/
static sk_sp<SkImage> convertMTLTextureToSkImage(GrRecordingContext* context, id<MTLTexture> mtlTexture);
/**
Creates a Center Crop Transformation Rect so that the source rect fills (aspectRatio: cover) the destination rect.
The return value should be passed as a sourceRect to a canvas->draw...Rect(..) function, destinationRect should stay the same.
*/
static SkRect createCenterCropRect(SkRect source, SkRect destination);
private:
static SkRect inscribe(SkSize size, SkRect rect);
};

116
ios/Skia Render Layer/SkImageHelpers.mm
View File

@ -1,116 +0,0 @@
//
// CMSampleBuffer+toSkImage.m
// VisionCamera
//
// Created by Marc Rousavy on 23.11.22.
// Copyright © 2022 mrousavy. All rights reserved.
//
#import "SkImageHelpers.h"
#import <AVFoundation/AVFoundation.h>
#import <Metal/Metal.h>
#import <include/core/SkColorSpace.h>
#import <include/core/SkSurface.h>
#import <include/core/SkCanvas.h>
#import <include/core/SkImage.h>
#import <include/gpu/ganesh/SkImageGanesh.h>
#import <include/gpu/mtl/GrMtlTypes.h>
#import <include/gpu/GrBackendSurface.h>
#include <TargetConditionals.h>
#if TARGET_RT_BIG_ENDIAN
# define FourCC2Str(fourcc) (const char[]){*((char*)&fourcc), *(((char*)&fourcc)+1), *(((char*)&fourcc)+2), *(((char*)&fourcc)+3),0}
#else
# define FourCC2Str(fourcc) (const char[]){*(((char*)&fourcc)+3), *(((char*)&fourcc)+2), *(((char*)&fourcc)+1), *(((char*)&fourcc)+0),0}
#endif
inline CVMetalTextureCacheRef getTextureCache() {
static CVMetalTextureCacheRef textureCache = nil;
if (textureCache == nil) {
// Create a new Texture Cache
auto result = CVMetalTextureCacheCreate(kCFAllocatorDefault,
nil,
MTLCreateSystemDefaultDevice(),
nil,
&textureCache);
if (result != kCVReturnSuccess || textureCache == nil) {
throw std::runtime_error("Failed to create Metal Texture Cache!");
}
}
return textureCache;
}
sk_sp<SkImage> SkImageHelpers::convertCMSampleBufferToSkImage(GrRecordingContext* context, CMSampleBufferRef sampleBuffer) {
auto pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
double width = CVPixelBufferGetWidth(pixelBuffer);
double height = CVPixelBufferGetHeight(pixelBuffer);
// Make sure the format is RGB (BGRA_8888)
auto format = CVPixelBufferGetPixelFormatType(pixelBuffer);
if (format != kCVPixelFormatType_32BGRA) {
auto error = std::string("VisionCamera: Frame has unknown Pixel Format (") + FourCC2Str(format) + std::string(") - cannot convert to SkImage!");
throw std::runtime_error(error);
}
auto textureCache = getTextureCache();
// Convert CMSampleBuffer* -> CVMetalTexture*
CVMetalTextureRef cvTexture;
CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
textureCache,
pixelBuffer,
nil,
MTLPixelFormatBGRA8Unorm,
width,
height,
0, // plane index
&cvTexture);
auto mtlTexture = CVMetalTextureGetTexture(cvTexture);
auto image = convertMTLTextureToSkImage(context, mtlTexture);
// Release the Texture wrapper (it will still be strong)
CFRelease(cvTexture);
return image;
}
sk_sp<SkImage> SkImageHelpers::convertMTLTextureToSkImage(GrRecordingContext* context, id<MTLTexture> texture) {
// Convert the rendered MTLTexture to an SkImage
GrMtlTextureInfo textureInfo;
textureInfo.fTexture.retain((__bridge void*)texture);
GrBackendTexture backendTexture((int)texture.width,
(int)texture.height,
GrMipmapped::kNo,
textureInfo);
// TODO: Adopt or Borrow?
auto image = SkImages::AdoptTextureFrom(context,
backendTexture,
kTopLeft_GrSurfaceOrigin,
kBGRA_8888_SkColorType,
kOpaque_SkAlphaType,
SkColorSpace::MakeSRGB());
return image;
}
SkRect SkImageHelpers::createCenterCropRect(SkRect sourceRect, SkRect destinationRect) {
SkSize src;
if (destinationRect.width() / destinationRect.height() > sourceRect.width() / sourceRect.height()) {
src = SkSize::Make(sourceRect.width(), (sourceRect.width() * destinationRect.height()) / destinationRect.width());
} else {
src = SkSize::Make((sourceRect.height() * destinationRect.width()) / destinationRect.height(), sourceRect.height());
}
return inscribe(src, sourceRect);
}
SkRect SkImageHelpers::inscribe(SkSize size, SkRect rect) {
auto halfWidthDelta = (rect.width() - size.width()) / 2.0;
auto halfHeightDelta = (rect.height() - size.height()) / 2.0;
return SkRect::MakeXYWH(rect.x() + halfWidthDelta,
rect.y() + halfHeightDelta,
size.width(),
size.height());
}

27
ios/Skia Render Layer/SkiaFrameProcessor.h
View File

@ -1,27 +0,0 @@
//
// SkiaFrameProcessor.h
// VisionCamera
//
// Created by Marc Rousavy on 14.07.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
#pragma once
#import <Foundation/Foundation.h>
#import "FrameProcessor.h"
#import "SkiaRenderer.h"
#ifdef __cplusplus
#import "WKTJsiWorklet.h"
#endif
@interface SkiaFrameProcessor: FrameProcessor
#ifdef __cplusplus
- (instancetype _Nonnull) initWithWorklet:(std::shared_ptr<RNWorklet::JsiWorklet>)worklet
context:(std::shared_ptr<RNWorklet::JsiWorkletContext>)context
skiaRenderer:(SkiaRenderer* _Nonnull)skiaRenderer;
#endif
@end

56
ios/Skia Render Layer/SkiaFrameProcessor.mm
View File

@ -1,56 +0,0 @@
//
// SkiaFrameProcessor.mm
// VisionCamera
//
// Created by Marc Rousavy on 14.07.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
#import <Foundation/Foundation.h>
#import "SkiaFrameProcessor.h"
#import "SkiaRenderer.h"
#import <memory>
#import <jsi/jsi.h>
#import "DrawableFrameHostObject.h"
#import <react-native-skia/JsiSkCanvas.h>
#import <react-native-skia/RNSkiOSPlatformContext.h>
using namespace facebook;
@implementation SkiaFrameProcessor {
SkiaRenderer* _skiaRenderer;
std::shared_ptr<RNSkia::JsiSkCanvas> _skiaCanvas;
}
- (instancetype _Nonnull)initWithWorklet:(std::shared_ptr<RNWorklet::JsiWorklet>)worklet
context:(std::shared_ptr<RNWorklet::JsiWorkletContext>)context
skiaRenderer:(SkiaRenderer* _Nonnull)skiaRenderer {
if (self = [super initWithWorklet:worklet
context:context]) {
_skiaRenderer = skiaRenderer;
auto platformContext = std::make_shared<RNSkia::RNSkiOSPlatformContext>(context->getJsRuntime(),
RCTBridge.currentBridge);
_skiaCanvas = std::make_shared<RNSkia::JsiSkCanvas>(platformContext);
}
return self;
}
- (void)call:(Frame*)frame {
[_skiaRenderer renderCameraFrameToOffscreenSurface:frame.buffer
withDrawCallback:^(SkiaCanvas _Nonnull canvas) {
// Create the Frame Host Object wrapping the internal Frame and Skia Canvas
self->_skiaCanvas->setCanvas(static_cast<SkCanvas*>(canvas));
auto frameHostObject = std::make_shared<DrawableFrameHostObject>(frame, self->_skiaCanvas);
// Call JS Frame Processor
[self callWithFrameHostObject:frameHostObject];
// Remove Skia Canvas from Host Object because it is no longer valid
frameHostObject->invalidateCanvas();
}];
}
@end

51
ios/Skia Render Layer/SkiaPreviewDisplayLink.swift
View File

@ -1,51 +0,0 @@
//
// SkiaPreviewDisplayLink.swift
// VisionCamera
//
// Created by Marc Rousavy on 19.07.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
import Foundation
class SkiaPreviewDisplayLink {
private var displayLink: CADisplayLink?
private let callback: (_ timestamp: Double) -> Void
init(callback: @escaping (_ timestamp: Double) -> Void) {
self.callback = callback
}
deinit {
stop()
}
@objc
func update(_ displayLink: CADisplayLink) {
callback(displayLink.timestamp)
}
func start() {
if displayLink == nil {
let displayLink = CADisplayLink(target: self, selector: #selector(update))
let queue = DispatchQueue(label: "mrousavy/VisionCamera.preview",
qos: .userInteractive,
attributes: [],
autoreleaseFrequency: .inherit,
target: nil)
queue.async {
displayLink.add(to: .current, forMode: .common)
self.displayLink = displayLink
ReactLogger.log(level: .info, message: "Starting Skia Preview Display Link...")
RunLoop.current.run()
ReactLogger.log(level: .info, message: "Skia Preview Display Link stopped.")
}
}
}
func stop() {
displayLink?.invalidate()
displayLink = nil
}
}

81
ios/Skia Render Layer/SkiaPreviewView.swift
View File

@ -1,81 +0,0 @@
//
// SkiaPreviewView.swift
// VisionCamera
//
// Created by Marc Rousavy on 19.07.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
import Foundation
// MARK: - SkiaPreviewLayer
class SkiaPreviewLayer: CAMetalLayer {
private var pixelRatio: CGFloat {
return UIScreen.main.scale
}
init(device: MTLDevice) {
super.init()
framebufferOnly = true
self.device = device
isOpaque = false
pixelFormat = .bgra8Unorm
contentsScale = pixelRatio
}
@available(*, unavailable)
required init?(coder _: NSCoder) {
fatalError("init(coder:) has not been implemented")
}
func setSize(width: CGFloat, height: CGFloat) {
frame = CGRect(x: 0, y: 0, width: width, height: height)
drawableSize = CGSize(width: width * pixelRatio,
height: height * pixelRatio)
}
}
// MARK: - SkiaPreviewView
class SkiaPreviewView: PreviewView {
private let skiaRenderer: SkiaRenderer
private let previewLayer: SkiaPreviewLayer
private lazy var displayLink = SkiaPreviewDisplayLink(callback: { [weak self] _ in
// Called everytime to render the screen - e.g. 60 FPS
if let self = self {
self.skiaRenderer.renderLatestFrame(to: self.previewLayer)
}
})
init(frame: CGRect, skiaRenderer: SkiaRenderer) {
self.skiaRenderer = skiaRenderer
previewLayer = SkiaPreviewLayer(device: skiaRenderer.metalDevice)
super.init(frame: frame)
}
deinit {
self.displayLink.stop()
}
@available(*, unavailable)
required init?(coder _: NSCoder) {
fatalError("init(coder:) has not been implemented")
}
override func willMove(toSuperview newSuperview: UIView?) {
if newSuperview != nil {
layer.addSublayer(previewLayer)
displayLink.start()
} else {
previewLayer.removeFromSuperlayer()
displayLink.stop()
}
}
override func layoutSubviews() {
previewLayer.setSize(width: bounds.size.width,
height: bounds.size.height)
}
}

26
ios/Skia Render Layer/SkiaRenderContext.h
View File

@ -1,26 +0,0 @@
//
// SkiaRenderContext.h
// VisionCamera
//
// Created by Marc Rousavy on 02.12.22.
// Copyright © 2022 mrousavy. All rights reserved.
//
#pragma once
#import <MetalKit/MetalKit.h>
#import <include/gpu/GrDirectContext.h>
#import <include/gpu/mtl/GrMtlTypes.h>
struct RenderContext {
id<MTLDevice> device;
id<MTLCommandQueue> commandQueue;
sk_sp<GrDirectContext> skiaContext;
RenderContext() {
device = MTLCreateSystemDefaultDevice();
commandQueue = id<MTLCommandQueue>(CFRetain((GrMTLHandle)[device newCommandQueue]));
skiaContext = GrDirectContext::MakeMetal((__bridge void*)device,
(__bridge void*)commandQueue);
}
};

45
ios/Skia Render Layer/SkiaRenderer.h
View File

@ -1,45 +0,0 @@
//
// SkiaRenderer.h
// VisionCamera
//
// Created by Marc Rousavy on 19.07.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
#pragma once
#import <Foundation/Foundation.h>
#import <AVFoundation/AVFoundation.h>
#import <Metal/Metal.h>
typedef void* SkiaCanvas;
typedef void(^draw_callback_t)(SkiaCanvas _Nonnull);
/**
A Camera Frame Renderer powered by Skia.
It provides two Contexts, one offscreen and one onscreen.
- Offscreen Context: Allows you to render a Frame into a Skia Canvas and draw onto it using Skia commands
- Onscreen Context: Allows you to render a Frame from the offscreen context onto a Layer allowing it to be displayed for Preview.
The two contexts may run at different Frame Rates.
*/
@interface SkiaRenderer : NSObject
/**
Renders the given Camera Frame to the offscreen Skia Canvas.
The given callback will be executed with a reference to the Skia Canvas
for the user to perform draw operations on (in this case, through a JS Frame Processor)
*/
- (void)renderCameraFrameToOffscreenSurface:(CMSampleBufferRef _Nonnull)sampleBuffer withDrawCallback:(draw_callback_t _Nonnull)callback;
/**
Renders the latest Frame to the onscreen Layer.
This should be called everytime you want the UI to update, e.g. for 60 FPS; every 16.66ms.
*/
- (void)renderLatestFrameToLayer:(CALayer* _Nonnull)layer;
/**
The Metal Device used for Rendering to the Layer
*/
@property (nonatomic, readonly) id<MTLDevice> _Nonnull metalDevice;
@end

212
ios/Skia Render Layer/SkiaRenderer.mm
View File

@ -1,212 +0,0 @@
//
// SkiaRenderer.mm
// VisionCamera
//
// Created by Marc Rousavy on 19.07.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
#import <Foundation/Foundation.h>
#import "SkiaRenderer.h"
#import <AVFoundation/AVFoundation.h>
#import <Metal/Metal.h>
#import "SkiaRenderContext.h"
#import <include/core/SkSurface.h>
#import <include/core/SkCanvas.h>
#import <include/core/SkColorSpace.h>
#import <include/gpu/mtl/GrMtlTypes.h>
#import <include/gpu/GrBackendSurface.h>
#import <include/gpu/ganesh/SkSurfaceGanesh.h>
#import <include/gpu/ganesh/mtl/SkSurfaceMetal.h>
#import "SkImageHelpers.h"
#import <system_error>
#import <memory>
#import <mutex>
@implementation SkiaRenderer {
// The context we draw each Frame on
std::unique_ptr<RenderContext> _offscreenContext;
// The context the preview runs on
std::unique_ptr<RenderContext> _layerContext;
// The texture holding the drawn-to Frame
id<MTLTexture> _texture;
// For synchronization between the two Threads/Contexts
std::mutex _textureMutex;
std::atomic<bool> _hasNewFrame;
}
- (instancetype)init {
if (self = [super init]) {
_offscreenContext = std::make_unique<RenderContext>();
_layerContext = std::make_unique<RenderContext>();
_texture = nil;
_hasNewFrame = false;
}
return self;
}
- (id<MTLDevice>)metalDevice {
return _layerContext->device;
}
- (id<MTLTexture>)getTexture:(NSUInteger)width height:(NSUInteger)height {
if (_texture == nil
|| _texture.width != width
|| _texture.height != height) {
// Create new texture with the given width and height
MTLTextureDescriptor* textureDescriptor = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm
width:width
height:height
mipmapped:NO];
textureDescriptor.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead;
_texture = [_offscreenContext->device newTextureWithDescriptor:textureDescriptor];
}
return _texture;
}
- (void)renderCameraFrameToOffscreenSurface:(CMSampleBufferRef)sampleBuffer withDrawCallback:(draw_callback_t)callback {
// Wrap in auto release pool since we want the system to clean up after rendering
@autoreleasepool {
// Get the Frame's PixelBuffer
CVImageBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
if (pixelBuffer == nil) {
throw std::runtime_error("SkiaRenderer: Pixel Buffer is corrupt/empty.");
}
// Lock Mutex to block the runLoop from overwriting the _currentDrawable
std::unique_lock lock(_textureMutex);
// Get the Metal Texture we use for in-memory drawing
auto texture = [self getTexture:CVPixelBufferGetWidth(pixelBuffer)
height:CVPixelBufferGetHeight(pixelBuffer)];
// Get & Lock the writeable Texture from the Metal Drawable
GrMtlTextureInfo textureInfo;
textureInfo.fTexture.retain((__bridge void*)texture);
GrBackendRenderTarget backendRenderTarget((int)texture.width,
(int)texture.height,
1,
textureInfo);
auto context = _offscreenContext->skiaContext.get();
// Create a Skia Surface from the writable Texture
auto surface = SkSurfaces::WrapBackendRenderTarget(context,
backendRenderTarget,
kTopLeft_GrSurfaceOrigin,
kBGRA_8888_SkColorType,
SkColorSpace::MakeSRGB(),
nullptr);
if (surface == nullptr || surface->getCanvas() == nullptr) {
throw std::runtime_error("Skia surface could not be created from parameters.");
}
// Converts the CMSampleBuffer to an SkImage - RGB.
CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
auto image = SkImageHelpers::convertCMSampleBufferToSkImage(context, sampleBuffer);
auto canvas = surface->getCanvas();
// Clear everything so we keep it at a clean state
canvas->clear(SkColors::kBlack);
// Draw the Image into the Frame (aspectRatio: cover)
// The Frame Processor might draw the Frame again (through render()) to pass a custom paint/shader,
// but that'll just overwrite the existing one - no need to worry.
canvas->drawImage(image, 0, 0);
// Call the draw callback - probably a JS Frame Processor.
callback(static_cast<void*>(canvas));
// Flush all appended operations on the canvas and commit it to the SkSurface
surface->flushAndSubmit();
// Set dirty & free locks
_hasNewFrame = true;
lock.unlock();
CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
}
}
- (void)renderLatestFrameToLayer:(CALayer* _Nonnull)layer {
if (!_hasNewFrame) {
// No new Frame has arrived in the meantime.
// We don't need to re-draw the texture to the screen if nothing has changed, abort.
return;
}
@autoreleasepool {
auto context = _layerContext->skiaContext.get();
// Create a Skia Surface from the CAMetalLayer (use to draw to the View)
GrMTLHandle drawableHandle;
auto surface = SkSurfaces::WrapCAMetalLayer(context,
(__bridge GrMTLHandle)layer,
kTopLeft_GrSurfaceOrigin,
1,
kBGRA_8888_SkColorType,
nullptr,
nullptr,
&drawableHandle);
if (surface == nullptr || surface->getCanvas() == nullptr) {
throw std::runtime_error("Skia surface could not be created from parameters.");
}
auto canvas = surface->getCanvas();
// Lock the Mutex so we can operate on the Texture atomically without
// renderFrameToCanvas() overwriting in between from a different thread
std::unique_lock lock(_textureMutex);
auto texture = _texture;
if (texture == nil) return;
// Calculate Center Crop (aspectRatio: cover) transform
auto sourceRect = SkRect::MakeXYWH(0, 0, texture.width, texture.height);
auto destinationRect = SkRect::MakeXYWH(0, 0, surface->width(), surface->height());
sourceRect = SkImageHelpers::createCenterCropRect(sourceRect, destinationRect);
auto offsetX = -sourceRect.left();
auto offsetY = -sourceRect.top();
// The Canvas is equal to the View size, where-as the Frame has a different size (e.g. 4k)
// We scale the Canvas to the exact dimensions of the Frame so that the user can use the Frame as a coordinate system
canvas->save();
auto scaleW = static_cast<double>(surface->width()) / texture.width;
auto scaleH = static_cast<double>(surface->height()) / texture.height;
auto scale = MAX(scaleW, scaleH);
canvas->scale(scale, scale);
canvas->translate(offsetX, offsetY);
// Convert the rendered MTLTexture to an SkImage
auto image = SkImageHelpers::convertMTLTextureToSkImage(context, texture);
// Draw the Texture (Frame) to the Canvas
canvas->drawImage(image, 0, 0);
// Restore the scale & transform
canvas->restore();
surface->flushAndSubmit();
// Pass the drawable into the Metal Command Buffer and submit it to the GPU
id<CAMetalDrawable> drawable = (__bridge id<CAMetalDrawable>)drawableHandle;
id<MTLCommandBuffer> commandBuffer([_layerContext->commandQueue commandBuffer]);
[commandBuffer presentDrawable:drawable];
[commandBuffer commit];
// Set flag back to false
_hasNewFrame = false;
lock.unlock();
}
}
@end

32
ios/VisionCamera.xcodeproj/project.pbxproj
View File

@ -12,7 +12,6 @@
B81BE1BF26B936FF002696CC /* AVCaptureDevice.Format+videoDimensions.swift in Sources */ = {isa = PBXBuildFile; fileRef = B81BE1BE26B936FF002696CC /* AVCaptureDevice.Format+videoDimensions.swift */; };
B82F3A0B2A6896E3002BB804 /* PreviewView.swift in Sources */ = {isa = PBXBuildFile; fileRef = B82F3A0A2A6896E3002BB804 /* PreviewView.swift */; };
B83D5EE729377117000AFD2F /* NativePreviewView.swift in Sources */ = {isa = PBXBuildFile; fileRef = B83D5EE629377117000AFD2F /* NativePreviewView.swift */; };
B841262F292E41A1001AB448 /* SkImageHelpers.mm in Sources */ = {isa = PBXBuildFile; fileRef = B841262E292E41A1001AB448 /* SkImageHelpers.mm */; };
B84760A62608EE7C004C3180 /* FrameHostObject.mm in Sources */ = {isa = PBXBuildFile; fileRef = B84760A52608EE7C004C3180 /* FrameHostObject.mm */; };
B84760DF2608F57D004C3180 /* CameraQueues.swift in Sources */ = {isa = PBXBuildFile; fileRef = B84760DE2608F57D004C3180 /* CameraQueues.swift */; };
B85F7AE92A77BB680089C539 /* FrameProcessorPlugin.m in Sources */ = {isa = PBXBuildFile; fileRef = B85F7AE82A77BB680089C539 /* FrameProcessorPlugin.m */; };
@ -79,27 +78,22 @@
/* Begin PBXFileReference section */
134814201AA4EA6300B7C361 /* libVisionCamera.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libVisionCamera.a; sourceTree = BUILT_PRODUCTS_DIR; };
B80A319E293A5C10003EE681 /* SkiaRenderContext.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SkiaRenderContext.h; sourceTree = "<group>"; };
B80C02EB2A6A954D001975E2 /* FrameProcessorPluginHostObject.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = FrameProcessorPluginHostObject.mm; sourceTree = "<group>"; };
B80C02EC2A6A9552001975E2 /* FrameProcessorPluginHostObject.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorPluginHostObject.h; sourceTree = "<group>"; };
B80C0DFE260BDD97001699AB /* FrameProcessorPluginRegistry.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorPluginRegistry.h; sourceTree = "<group>"; };
B80C0DFF260BDDF7001699AB /* FrameProcessorPluginRegistry.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = FrameProcessorPluginRegistry.m; sourceTree = "<group>"; };
B80E069F266632F000728644 /* AVAudioSession+updateCategory.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVAudioSession+updateCategory.swift"; sourceTree = "<group>"; };
B8103E5725FF56F0007A1684 /* Frame.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Frame.h; sourceTree = "<group>"; };
B8127E382A68871C00B06972 /* SkiaPreviewView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SkiaPreviewView.swift; sourceTree = "<group>"; };
B81BE1BE26B936FF002696CC /* AVCaptureDevice.Format+videoDimensions.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVCaptureDevice.Format+videoDimensions.swift"; sourceTree = "<group>"; };
B81D41EF263C86F900B041FD /* JSINSObjectConversion.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = JSINSObjectConversion.h; sourceTree = "<group>"; };
B82F3A0A2A6896E3002BB804 /* PreviewView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PreviewView.swift; sourceTree = "<group>"; };
B83D5EE629377117000AFD2F /* NativePreviewView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NativePreviewView.swift; sourceTree = "<group>"; };
B841262E292E41A1001AB448 /* SkImageHelpers.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = SkImageHelpers.mm; sourceTree = "<group>"; };
B8412630292E41AD001AB448 /* SkImageHelpers.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SkImageHelpers.h; sourceTree = "<group>"; };
B84760A22608EE38004C3180 /* FrameHostObject.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameHostObject.h; sourceTree = "<group>"; };
B84760A52608EE7C004C3180 /* FrameHostObject.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = FrameHostObject.mm; sourceTree = "<group>"; };
B84760DE2608F57D004C3180 /* CameraQueues.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CameraQueues.swift; sourceTree = "<group>"; };
B85F7AE82A77BB680089C539 /* FrameProcessorPlugin.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = FrameProcessorPlugin.m; sourceTree = "<group>"; };
B864004F27849A2400E9D2CA /* UIInterfaceOrientation+descriptor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "UIInterfaceOrientation+descriptor.swift"; sourceTree = "<group>"; };
B86400512784A23400E9D2CA /* CameraView+Orientation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CameraView+Orientation.swift"; sourceTree = "<group>"; };
B865BC5F2A6888DA0093DF1A /* SkiaPreviewDisplayLink.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SkiaPreviewDisplayLink.swift; sourceTree = "<group>"; };
B86DC970260E2D5200FB17B2 /* AVAudioSession+trySetAllowHaptics.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVAudioSession+trySetAllowHaptics.swift"; sourceTree = "<group>"; };
B86DC973260E310600FB17B2 /* CameraView+AVAudioSession.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CameraView+AVAudioSession.swift"; sourceTree = "<group>"; };
B86DC976260E315100FB17B2 /* CameraView+AVCaptureSession.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CameraView+AVCaptureSession.swift"; sourceTree = "<group>"; };
@ -140,17 +134,11 @@
B88873E5263D46C7008B1D0E /* FrameProcessorPlugin.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorPlugin.h; sourceTree = "<group>"; };
B88B47462667C8E00091F538 /* AVCaptureSession+setVideoStabilizationMode.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVCaptureSession+setVideoStabilizationMode.swift"; sourceTree = "<group>"; };
B8994E6B263F03E100069589 /* JSINSObjectConversion.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = JSINSObjectConversion.mm; sourceTree = "<group>"; };
B89A28742A68795E0092207F /* SkiaRenderer.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = SkiaRenderer.mm; sourceTree = "<group>"; };
B89A28752A68796A0092207F /* SkiaRenderer.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SkiaRenderer.h; sourceTree = "<group>"; };
B8BD3BA1266E22D2006C80A2 /* Callback.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Callback.swift; sourceTree = "<group>"; };
B8C1FD222A613607007A06D6 /* SkiaFrameProcessor.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SkiaFrameProcessor.h; sourceTree = "<group>"; };
B8C1FD232A613612007A06D6 /* SkiaFrameProcessor.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = SkiaFrameProcessor.mm; sourceTree = "<group>"; };
B8D22CDB2642DB4D00234472 /* AVAssetWriterInputPixelBufferAdaptor+initWithVideoSettings.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVAssetWriterInputPixelBufferAdaptor+initWithVideoSettings.swift"; sourceTree = "<group>"; };
B8DB3BC7263DC28C004C18D7 /* AVAssetWriter.Status+descriptor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVAssetWriter.Status+descriptor.swift"; sourceTree = "<group>"; };
B8DB3BC9263DC4D8004C18D7 /* RecordingSession.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RecordingSession.swift; sourceTree = "<group>"; };
B8DB3BCB263DC97E004C18D7 /* AVFileType+descriptor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVFileType+descriptor.swift"; sourceTree = "<group>"; };
B8DFBA362A68A17E00941736 /* DrawableFrameHostObject.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = DrawableFrameHostObject.mm; sourceTree = "<group>"; };
B8DFBA372A68A17E00941736 /* DrawableFrameHostObject.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = DrawableFrameHostObject.h; sourceTree = "<group>"; };
B8E8467D2A696F44000D6A11 /* VisionCameraProxy.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = VisionCameraProxy.h; sourceTree = "<group>"; };
B8E8467E2A696F4D000D6A11 /* VisionCameraProxy.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = VisionCameraProxy.mm; sourceTree = "<group>"; };
B8E957CD2A6939A6008F5480 /* CameraView+Preview.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CameraView+Preview.swift"; sourceTree = "<group>"; };
@ -202,7 +190,6 @@
B82F3A0A2A6896E3002BB804 /* PreviewView.swift */,
B83D5EE629377117000AFD2F /* NativePreviewView.swift */,
B887515C25E0102000DB86D6 /* PhotoCaptureDelegate.swift */,
B8FCA20C292669B800F1AC82 /* Skia Render Layer */,
B887516125E0102000DB86D6 /* Extensions */,
B887517225E0102000DB86D6 /* Parsers */,
B887516D25E0102000DB86D6 /* React Utils */,
@ -287,24 +274,6 @@
path = "Frame Processor";
sourceTree = "<group>";
};
B8FCA20C292669B800F1AC82 /* Skia Render Layer */ = {
isa = PBXGroup;
children = (
B8C1FD222A613607007A06D6 /* SkiaFrameProcessor.h */,
B8C1FD232A613612007A06D6 /* SkiaFrameProcessor.mm */,
B8412630292E41AD001AB448 /* SkImageHelpers.h */,
B841262E292E41A1001AB448 /* SkImageHelpers.mm */,
B80A319E293A5C10003EE681 /* SkiaRenderContext.h */,
B89A28752A68796A0092207F /* SkiaRenderer.h */,
B89A28742A68795E0092207F /* SkiaRenderer.mm */,
B8127E382A68871C00B06972 /* SkiaPreviewView.swift */,
B865BC5F2A6888DA0093DF1A /* SkiaPreviewDisplayLink.swift */,
B8DFBA372A68A17E00941736 /* DrawableFrameHostObject.h */,
B8DFBA362A68A17E00941736 /* DrawableFrameHostObject.mm */,
);
path = "Skia Render Layer";
sourceTree = "<group>";
};
/* End PBXGroup section */
/* Begin PBXNativeTarget section */
@ -417,7 +386,6 @@
B887518D25E0102000DB86D6 /* AVCaptureDevice+physicalDevices.swift in Sources */,
B887519625E0102000DB86D6 /* Promise.swift in Sources */,
B8DB3BC8263DC28C004C18D7 /* AVAssetWriter.Status+descriptor.swift in Sources */,
B841262F292E41A1001AB448 /* SkImageHelpers.mm in Sources */,
B887518725E0102000DB86D6 /* CameraViewManager.m in Sources */,
B88751A925E0102000DB86D6 /* CameraView.swift in Sources */,
B887519925E0102000DB86D6 /* AVCaptureVideoStabilizationMode+descriptor.swift in Sources */,

5
package.json
View File

@ -76,7 +76,6 @@
"@react-native/eslint-config": "^0.72.2",
"@react-native/typescript-config": "^0.73.0",
"@release-it/conventional-changelog": "^7.0.0",
"@shopify/react-native-skia": "^0.1.200",
"@types/react": "^18.2.19",
"@types/react-native": "^0.72.2",
"eslint": "^8.46.0",
@ -91,15 +90,11 @@
"typescript": "^5.1.6"
},
"peerDependencies": {
"@shopify/react-native-skia": "*",
"react": "*",
"react-native": "*",
"react-native-worklets-core": "*"
},
"peerDependenciesMeta": {
"@shopify/react-native-skia": {
"optional": true
},
"react-native-worklets-core": {
"optional": true
}

2
src/Camera.tsx
View File

@ -22,7 +22,6 @@ interface OnErrorEvent {
type NativeCameraViewProps = Omit<CameraProps, 'device' | 'onInitialized' | 'onError' | 'frameProcessor'> & {
cameraId: string;
enableFrameProcessor: boolean;
previewType: 'native' | 'skia' | 'none';
onInitialized?: (event: NativeSyntheticEvent<void>) => void;
onError?: (event: NativeSyntheticEvent<OnErrorEvent>) => void;
onViewReady: () => void;
@ -413,7 +412,6 @@ export class Camera extends React.PureComponent<CameraProps> {
onInitialized={this.onInitialized}
onError={this.onError}
enableFrameProcessor={frameProcessor != null}
previewType={frameProcessor?.type === 'skia-frame-processor' ? 'skia' : 'native'}
/>
);
}

1
src/CameraError.ts
View File

@ -54,7 +54,6 @@ export type SystemError =
| 'system/camera-module-not-found'
| 'system/no-camera-manager'
| 'system/frame-processors-unavailable'
| 'system/skia-unavailable'
| 'system/view-not-found';
export type UnknownError = 'unknown/unknown';

19
src/CameraProps.ts
View File

@ -1,21 +1,16 @@
import type { ViewProps } from 'react-native';
import type { CameraDevice, CameraDeviceFormat, VideoStabilizationMode } from './CameraDevice';
import type { CameraRuntimeError } from './CameraError';
import type { DrawableFrame, Frame } from './Frame';
import type { Frame } from './Frame';
import type { Orientation } from './Orientation';
export type FrameProcessor =
| {
frameProcessor: (frame: Frame) => void;
type: 'frame-processor';
}
| {
frameProcessor: (frame: DrawableFrame) => void;
type: 'skia-frame-processor';
};
export type FrameProcessor = {
frameProcessor: (frame: Frame) => void;
type: 'frame-processor';
};
// TODO: Replace `enableHighQualityPhotos: boolean` in favor of `priorization: 'photo' | 'video'`
// TODO: Use RCT_ENUM_PARSER for stuff like previewType, torch, videoStabilizationMode, and orientation
// TODO: Use RCT_ENUM_PARSER for stuff like torch, videoStabilizationMode, and orientation
// TODO: Use Photo HostObject for stuff like depthData, portraitEffects, etc.
// TODO: Add RAW capture support
@ -193,8 +188,6 @@ export interface CameraProps extends ViewProps {
/**
* A worklet which will be called for every frame the Camera "sees".
*
* If {@linkcode previewType | previewType} is set to `"skia"`, you can draw content to the `Frame` using the react-native-skia API.
*
* > See [the Frame Processors documentation](https://mrousavy.github.io/react-native-vision-camera/docs/guides/frame-processors) for more information
*
* @example

44
src/Frame.ts
View File

@ -1,4 +1,3 @@
import type { SkCanvas, SkPaint } from '@shopify/react-native-skia';
import type { Orientation } from './Orientation';
import { PixelFormat } from './PixelFormat';
@ -22,6 +21,10 @@ export interface Frame {
* Returns the amount of bytes per row.
*/
bytesPerRow: number;
/**
* Returns the number of planes this frame contains.
*/
planesCount: number;
/**
* Returns whether the Frame is mirrored (selfie camera) or not.
*/
@ -56,46 +59,9 @@ export interface Frame {
* ```
*/
toString(): string;
/**
* Whether the Frame can be drawn onto using Skia.
* Always false for `useFrameProcessor`. Use `useSkiaFrameProcessor` instead.
*/
isDrawable: boolean;
}
export interface DrawableFrame extends Frame, SkCanvas {
/**
* Renders the Frame to the screen.
*
* By default a Frame has already been rendered to the screen once, so if you call this method again,
* previously drawn content will be overwritten.
*
* @param paint (Optional) A Paint object to use to draw the Frame with. For example, this can contain a Shader (ImageFilter)
* @example
* ```ts
* const INVERTED_COLORS_SHADER = `
* uniform shader image;
* half4 main(vec2 pos) {
* vec4 color = image.eval(pos);
* return vec4(1.0 - color.rgb, 1.0);
* }`
* const runtimeEffect = Skia.RuntimeEffect.Make(INVERT_COLORS_SHADER)
* if (runtimeEffect == null) throw new Error('Shader failed to compile!')
* const shaderBuilder = Skia.RuntimeShaderBuilder(runtimeEffect)
* const imageFilter = Skia.ImageFilter.MakeRuntimeShader(shaderBuilder, null, null)
* const paint = Skia.Paint()
* paint.setImageFilter(imageFilter)
*
* const frameProcessor = useSkiaFrameProcessor((frame) => {
* 'worklet'
* frame.render(paint) // <-- draws frame with inverted colors now
* }, [paint])
* ```
*/
render: (paint?: SkPaint) => void;
}
export interface FrameInternal extends Frame, DrawableFrame {
export interface FrameInternal extends Frame {
/**
* Increment the Frame Buffer ref-count by one.
*

2
src/FrameProcessorPlugins.ts
View File

@ -28,7 +28,6 @@ interface TVisionCameraProxy {
* The Plugin has to be registered on the native side, otherwise this returns `undefined`
*/
getFrameProcessorPlugin: (name: string) => FrameProcessorPlugin | undefined;
isSkiaEnabled: boolean;
}
let hasWorklets = false;
@ -66,7 +65,6 @@ try {
}
let proxy: TVisionCameraProxy = {
isSkiaEnabled: false,
getFrameProcessorPlugin: () => {
throw new CameraRuntimeError('system/frame-processors-unavailable', 'Frame Processors are not enabled!');
},

38
src/hooks/useFrameProcessor.ts
View File

@ -1,16 +1,23 @@
import { DependencyList, useMemo } from 'react';
import type { DrawableFrame, Frame, FrameInternal } from '../Frame';
import type { Frame, FrameInternal } from '../Frame';
import { FrameProcessor } from '../CameraProps';
/**
* Create a new Frame Processor function which you can pass to the `<Camera>`.
* (See ["Frame Processors"](https://mrousavy.github.io/react-native-vision-camera/docs/guides/frame-processors))
*
* Make sure to add the `'worklet'` directive to the top of the Frame Processor function, otherwise it will not get compiled into a worklet.
*
* Also make sure to memoize the returned object, so that the Camera doesn't reset the Frame Processor Context each time.
*/
export function createFrameProcessor(frameProcessor: FrameProcessor['frameProcessor'], type: FrameProcessor['type']): FrameProcessor {
return {
frameProcessor: (frame: Frame | DrawableFrame) => {
frameProcessor: (frame: Frame) => {
'worklet';
// Increment ref-count by one
(frame as FrameInternal).incrementRefCount();
try {
// Call sync frame processor
// @ts-expect-error the frame type is ambiguous here
frameProcessor(frame);
} finally {
// Potentially delete Frame if we were the last ref (no runAsync)
@ -43,28 +50,3 @@ export function useFrameProcessor(frameProcessor: (frame: Frame) => void, depend
// eslint-disable-next-line react-hooks/exhaustive-deps
return useMemo(() => createFrameProcessor(frameProcessor, 'frame-processor'), dependencies);
}
/**
* Returns a memoized Skia Frame Processor function wich you can pass to the `<Camera>`.
* The Skia Frame Processor allows you to draw anything onto the Frame using react-native-skia.
* (See ["Frame Processors"](https://mrousavy.github.io/react-native-vision-camera/docs/guides/frame-processors))
*
* Make sure to add the `'worklet'` directive to the top of the Frame Processor function, otherwise it will not get compiled into a worklet.
*
* @param frameProcessor The Frame Processor
* @param dependencies The React dependencies which will be copied into the VisionCamera JS-Runtime.
* @returns The memoized Frame Processor.
* @example
* ```ts
* const frameProcessor = useSkiaFrameProcessor((frame) => {
* 'worklet'
* const qrCodes = scanQRCodes(frame)
* frame.drawRect(...)
* console.log(`QR Codes: ${qrCodes}`)
* }, [])
* ```
*/
export function useSkiaFrameProcessor(frameProcessor: (frame: DrawableFrame) => void, dependencies: DependencyList): FrameProcessor {
// eslint-disable-next-line react-hooks/exhaustive-deps
return useMemo(() => createFrameProcessor(frameProcessor, 'skia-frame-processor'), dependencies);
}

28
yarn.lock
View File

@ -1984,14 +1984,6 @@
conventional-recommended-bump "^7.0.1"
semver "7.5.1"
"@shopify/react-native-skia@^0.1.200":
version "0.1.200"
resolved "https://registry.yarnpkg.com/@shopify/react-native-skia/-/react-native-skia-0.1.200.tgz#3ef86750106a3b7e02496133173b449bfce6abc2"
integrity sha512-wAauKsLgScLspJY4KzoV0lWoXFCbzsUDJ3uso0o81HQMKBjDvXG9aOq/xE0KFLQsrQVICRdbfvvoYLQvSh/Xmw==
dependencies:
canvaskit-wasm "0.38.0"
react-reconciler "^0.27.0"
"@sideway/address@^4.1.3":
version "4.1.4"
resolved "https://registry.yarnpkg.com/@sideway/address/-/address-4.1.4.tgz#03dccebc6ea47fdc226f7d3d1ad512955d4783f0"
@ -2835,11 +2827,6 @@ caniuse-lite@^1.0.30001449:
resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001451.tgz#2e197c698fc1373d63e1406d6607ea4617c613f1"
integrity sha512-XY7UbUpGRatZzoRft//5xOa69/1iGJRBlrieH6QYrkKLIFn3m7OVEJ81dSrKoy2BnKsdbX5cLrOispZNYo9v2w==
canvaskit-wasm@0.38.0:
version "0.38.0"
resolved "https://registry.yarnpkg.com/canvaskit-wasm/-/canvaskit-wasm-0.38.0.tgz#83e6c46f3015c2ff3f6503157f47453af76a7be7"
integrity sha512-ZEG6lucpbQ4Ld+mY8C1Ng+PMLVP+/AX02jS0Sdl28NyMxuKSa9uKB8oGd1BYp1XWPyO2Jgr7U8pdyjJ/F3xR5Q==
chalk@5.3.0, chalk@^5.3.0:
version "5.3.0"
resolved "https://registry.yarnpkg.com/chalk/-/chalk-5.3.0.tgz#67c20a7ebef70e7f3970a01f90fa210cb6860385"
@ -6838,14 +6825,6 @@ react-native@^0.72.3:
ws "^6.2.2"
yargs "^17.6.2"
react-reconciler@^0.27.0:
version "0.27.0"
resolved "https://registry.yarnpkg.com/react-reconciler/-/react-reconciler-0.27.0.tgz#360124fdf2d76447c7491ee5f0e04503ed9acf5b"
integrity sha512-HmMDKciQjYmBRGuuhIaKA1ba/7a+UsM5FzOZsMO2JYHt9Jh8reCb7j1eDC95NOyUlKM9KRyvdx0flBuDvYSBoA==
dependencies:
loose-envify "^1.1.0"
scheduler "^0.21.0"
react-refresh@^0.4.0:
version "0.4.3"
resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.4.3.tgz#966f1750c191672e76e16c2efa569150cc73ab53"
@ -7215,13 +7194,6 @@ scheduler@0.24.0-canary-efb381bbf-20230505:
dependencies:
loose-envify "^1.1.0"
scheduler@^0.21.0:
version "0.21.0"
resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.21.0.tgz#6fd2532ff5a6d877b6edb12f00d8ab7e8f308820"
integrity sha512-1r87x5fz9MXqswA2ERLo0EbOAU74DpIUO090gIasYTqlVoJeMcl+Z1Rg7WHz+qtPujhS/hGIt9kxZOYBV3faRQ==
dependencies:
loose-envify "^1.1.0"
semver-diff@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/semver-diff/-/semver-diff-4.0.0.tgz#3afcf5ed6d62259f5c72d0d5d50dffbdc9680df5"