diff --git a/README.md b/README.md index 8348650..e6acb2d 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ VisionCamera is a powerful and fast Camera component for React Native. It featur * 🔍 Smooth zooming (Reanimated) * ⏯️ Fast pause and resume * 🌓 HDR & Night modes -* ⚡ Highly efficient C++/GPU buffers +* ⚡ Custom C++/GPU accelerated video pipeline (OpenGL) Install VisionCamera from npm: diff --git a/package/android/CMakeLists.txt b/package/android/CMakeLists.txt index 09c335a..bf660ce 100644 --- a/package/android/CMakeLists.txt +++ b/package/android/CMakeLists.txt @@ -21,6 +21,10 @@ add_library( SHARED ../cpp/JSITypedArray.cpp src/main/cpp/VisionCamera.cpp + src/main/cpp/VideoPipeline.cpp + src/main/cpp/PassThroughShader.cpp + src/main/cpp/OpenGLContext.cpp + src/main/cpp/OpenGLRenderer.cpp # Frame Processor src/main/cpp/frameprocessor/FrameHostObject.cpp src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.cpp @@ -54,6 +58,8 @@ target_link_libraries( ReactAndroid::jsi # <-- RN: JSI ReactAndroid::reactnativejni # <-- RN: React Native JNI bindings fbjni::fbjni # <-- fbjni + GLESv2 # <-- OpenGL (for VideoPipeline) + EGL # <-- OpenGL (EGL) (for VideoPipeline) ) # Optionally also add Frame Processors here diff --git a/package/android/src/main/cpp/OpenGLContext.cpp b/package/android/src/main/cpp/OpenGLContext.cpp new file mode 100644 index 0000000..307754c --- /dev/null +++ b/package/android/src/main/cpp/OpenGLContext.cpp @@ -0,0 +1,163 @@ +// +// Created by Marc Rousavy on 29.08.23. +// + +#include "OpenGLContext.h" + +#include +#include +#include + +#include +#include + +#include "OpenGLError.h" + +namespace vision { + +std::shared_ptr OpenGLContext::CreateWithOffscreenSurface() { + return std::unique_ptr(new OpenGLContext()); +} + +OpenGLContext::~OpenGLContext() { + destroy(); +} + +void OpenGLContext::destroy() { + if (display != EGL_NO_DISPLAY) { + eglMakeCurrent(display, offscreenSurface, offscreenSurface, context); + if (offscreenSurface != EGL_NO_SURFACE) { + __android_log_print(ANDROID_LOG_INFO, TAG, "Destroying OpenGL Surface..."); + eglDestroySurface(display, offscreenSurface); + offscreenSurface = EGL_NO_SURFACE; + } + if (context != EGL_NO_CONTEXT) { + __android_log_print(ANDROID_LOG_INFO, TAG, "Destroying OpenGL Context..."); + eglDestroyContext(display, context); + context = EGL_NO_CONTEXT; + } + __android_log_print(ANDROID_LOG_INFO, TAG, "Destroying OpenGL Display..."); + eglTerminate(display); + display = EGL_NO_DISPLAY; + config = nullptr; + } +} + +void OpenGLContext::ensureOpenGL() { + bool successful; + // EGLDisplay + if (display == EGL_NO_DISPLAY) { + __android_log_print(ANDROID_LOG_INFO, TAG, "Initializing EGLDisplay.."); + display = eglGetDisplay(EGL_DEFAULT_DISPLAY); + if (display == EGL_NO_DISPLAY) + throw OpenGLError("Failed to get default OpenGL Display!"); + + EGLint major; + EGLint minor; + successful = eglInitialize(display, &major, &minor); + if (!successful) + throw OpenGLError("Failed to initialize OpenGL!"); + } + + // EGLConfig + if (config == nullptr) { + __android_log_print(ANDROID_LOG_INFO, TAG, "Initializing EGLConfig.."); + EGLint attributes[] = {EGL_RENDERABLE_TYPE, + EGL_OPENGL_ES2_BIT, + EGL_SURFACE_TYPE, + EGL_WINDOW_BIT, + EGL_RED_SIZE, + 8, + EGL_GREEN_SIZE, + 8, + EGL_BLUE_SIZE, + 8, + EGL_ALPHA_SIZE, + 8, + EGL_DEPTH_SIZE, + 0, + EGL_STENCIL_SIZE, + 0, + EGL_NONE}; + EGLint numConfigs; + successful = eglChooseConfig(display, attributes, &config, 1, &numConfigs); + if (!successful || numConfigs == 0) + throw OpenGLError("Failed to choose OpenGL config!"); + } + + // EGLContext + if (context == EGL_NO_CONTEXT) { + __android_log_print(ANDROID_LOG_INFO, TAG, "Initializing EGLContext.."); + EGLint contextAttributes[] = {EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE}; + context = eglCreateContext(display, config, nullptr, contextAttributes); + if (context == EGL_NO_CONTEXT) + throw OpenGLError("Failed to create OpenGL context!"); + } + + // EGLSurface + if (offscreenSurface == EGL_NO_SURFACE) { + // If we don't have a surface at all + __android_log_print(ANDROID_LOG_INFO, TAG, "Initializing 1x1 offscreen pbuffer EGLSurface.."); + EGLint attributes[] = {EGL_WIDTH, 1, EGL_HEIGHT, 1, EGL_NONE}; + offscreenSurface = eglCreatePbufferSurface(display, config, attributes); + if (offscreenSurface == EGL_NO_SURFACE) + throw OpenGLError("Failed to create OpenGL Surface!"); + } +} + +void OpenGLContext::use() { + this->use(offscreenSurface); +} + +void OpenGLContext::use(EGLSurface surface) { + if (surface == EGL_NO_SURFACE) + throw OpenGLError("Cannot render to a null Surface!"); + + // 1. Make sure the OpenGL context is initialized + this->ensureOpenGL(); + + // 2. Make the OpenGL context current + bool successful = eglMakeCurrent(display, surface, surface, context); + if (!successful || eglGetError() != EGL_SUCCESS) + throw OpenGLError("Failed to use current OpenGL context!"); + + // 3. Caller can now render to this surface +} + +void OpenGLContext::flush() const { + bool successful = eglSwapBuffers(display, eglGetCurrentSurface(EGL_DRAW)); + if (!successful || eglGetError() != EGL_SUCCESS) + throw OpenGLError("Failed to swap OpenGL buffers!"); +} + +OpenGLTexture OpenGLContext::createTexture(OpenGLTexture::Type type, int width, int height) { + // 1. Make sure the OpenGL context is initialized + this->ensureOpenGL(); + + // 2. Make the OpenGL context current + bool successful = eglMakeCurrent(display, offscreenSurface, offscreenSurface, context); + if (!successful || eglGetError() != EGL_SUCCESS) + throw OpenGLError("Failed to use current OpenGL context!"); + + GLuint textureId; + glGenTextures(1, &textureId); + + GLenum target; + switch (type) { + case OpenGLTexture::Type::ExternalOES: + target = GL_TEXTURE_EXTERNAL_OES; + break; + case OpenGLTexture::Type::Texture2D: + target = GL_TEXTURE_2D; + break; + default: + throw std::runtime_error("Invalid OpenGL Texture Type!"); + } + glBindTexture(target, textureId); + glTexParameteri(target, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(target, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + + return {.id = textureId, .target = target, .width = width, .height = height}; +} + +} // namespace vision diff --git a/package/android/src/main/cpp/OpenGLContext.h b/package/android/src/main/cpp/OpenGLContext.h new file mode 100644 index 0000000..886bf57 --- /dev/null +++ b/package/android/src/main/cpp/OpenGLContext.h @@ -0,0 +1,73 @@ +// +// Created by Marc Rousavy on 29.08.23. +// + +#pragma once + +#include +#include + +#include +#include + +#include "OpenGLTexture.h" +#include "PassThroughShader.h" + +namespace vision { + +/** + * An OpenGL Context that can be used to render to different surfaces. + * By default, it creates an off-screen PixelBuffer surface. + */ +class OpenGLContext { +public: + /** + * Create a new instance of the OpenGLContext that draws to an off-screen PixelBuffer surface. + * This will not perform any OpenGL operations yet, and is therefore safe to call from any Thread. + */ + static std::shared_ptr CreateWithOffscreenSurface(); + /** + * Destroy the OpenGL Context. This needs to be called on the same thread that `use()` was called. + */ + ~OpenGLContext(); + + /** + * Use this OpenGL Context to render to the given EGLSurface. + * After the `renderFunc` returns, the default offscreen PixelBuffer surface becomes active again. + */ + void use(EGLSurface surface); + + /** + * Use this OpenGL Context to render to the offscreen PixelBuffer surface. + */ + void use(); + + /** + * Flushes all drawing operations by swapping the buffers and submitting the Frame to the GPU + */ + void flush() const; + + /** + * Create a new texture on this context + */ + OpenGLTexture createTexture(OpenGLTexture::Type type, int width, int height); + +public: + EGLDisplay display = EGL_NO_DISPLAY; + EGLContext context = EGL_NO_CONTEXT; + EGLSurface offscreenSurface = EGL_NO_SURFACE; + EGLConfig config = nullptr; + +private: + OpenGLContext() = default; + void destroy(); + void ensureOpenGL(); + +private: + PassThroughShader _passThroughShader; + +private: + static constexpr auto TAG = "OpenGLContext"; +}; + +} // namespace vision diff --git a/package/android/src/main/cpp/OpenGLError.h b/package/android/src/main/cpp/OpenGLError.h new file mode 100644 index 0000000..72e97d0 --- /dev/null +++ b/package/android/src/main/cpp/OpenGLError.h @@ -0,0 +1,34 @@ +// +// Created by Marc Rousavy on 09.08.23. +// + +#pragma once + +#include +#include +#include + +namespace vision { + +inline std::string getEglErrorIfAny() { + EGLint error = glGetError(); + if (error != GL_NO_ERROR) + return " Error: " + std::to_string(error); + error = eglGetError(); + if (error != EGL_SUCCESS) + return " Error: " + std::to_string(error); + return ""; +} + +class OpenGLError : public std::runtime_error { +public: + explicit OpenGLError(const std::string&& message) : std::runtime_error(message + getEglErrorIfAny()) {} + + static inline void checkIfError(const std::string&& message) { + auto error = getEglErrorIfAny(); + if (error.length() > 0) + throw std::runtime_error(message + error); + } +}; + +} // namespace vision diff --git a/package/android/src/main/cpp/OpenGLRenderer.cpp b/package/android/src/main/cpp/OpenGLRenderer.cpp new file mode 100644 index 0000000..3768399 --- /dev/null +++ b/package/android/src/main/cpp/OpenGLRenderer.cpp @@ -0,0 +1,74 @@ +// +// Created by Marc Rousavy on 29.08.23. +// + +#include "OpenGLRenderer.h" + +#include +#include +#include + +#include +#include + +#include + +#include "OpenGLError.h" + +namespace vision { + +std::unique_ptr OpenGLRenderer::CreateWithWindowSurface(std::shared_ptr context, ANativeWindow* surface) { + return std::unique_ptr(new OpenGLRenderer(std::move(context), surface)); +} + +OpenGLRenderer::OpenGLRenderer(std::shared_ptr context, ANativeWindow* surface) { + _context = std::move(context); + _outputSurface = surface; + _width = ANativeWindow_getWidth(surface); + _height = ANativeWindow_getHeight(surface); +} + +OpenGLRenderer::~OpenGLRenderer() { + if (_outputSurface != nullptr) { + ANativeWindow_release(_outputSurface); + } + destroy(); +} + +void OpenGLRenderer::destroy() { + if (_context != nullptr && _surface != EGL_NO_DISPLAY) { + __android_log_print(ANDROID_LOG_INFO, TAG, "Destroying OpenGL Surface..."); + eglDestroySurface(_context->display, _surface); + _surface = EGL_NO_SURFACE; + } +} + +void OpenGLRenderer::renderTextureToSurface(const OpenGLTexture& texture, float* transformMatrix) { + if (_surface == EGL_NO_SURFACE) { + __android_log_print(ANDROID_LOG_INFO, TAG, "Creating Window Surface..."); + _context->use(); + _surface = eglCreateWindowSurface(_context->display, _context->config, _outputSurface, nullptr); + } + + // 1. Activate the OpenGL context for this surface + _context->use(_surface); + + // 2. Set the viewport for rendering + glViewport(0, 0, _width, _height); + glDisable(GL_BLEND); + + // 3. Bind the input texture + glBindTexture(texture.target, texture.id); + glTexParameteri(texture.target, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(texture.target, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(texture.target, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(texture.target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + + // 4. Draw it using the pass-through shader which also applies transforms + _passThroughShader.draw(texture, transformMatrix); + + // 5. Swap buffers to pass it to the window surface + eglSwapBuffers(_context->display, _surface); +} + +} // namespace vision diff --git a/package/android/src/main/cpp/OpenGLRenderer.h b/package/android/src/main/cpp/OpenGLRenderer.h new file mode 100644 index 0000000..b2b96cb --- /dev/null +++ b/package/android/src/main/cpp/OpenGLRenderer.h @@ -0,0 +1,61 @@ +// +// Created by Marc Rousavy on 29.08.23. +// + +#pragma once + +#include "PassThroughShader.h" +#include +#include +#include +#include + +#include "OpenGLContext.h" +#include "OpenGLTexture.h" + +namespace vision { + +class OpenGLRenderer { +public: + /** + * Create a new instance of the OpenGLRenderer that draws to an on-screen window surface. + * This will not perform any OpenGL operations yet, and is therefore safe to call from any Thread. + * + * Note: The `surface` is considered moved, and the OpenGL context will release it when it is + * being deleted. + */ + static std::unique_ptr CreateWithWindowSurface(std::shared_ptr context, ANativeWindow* surface); + /** + * Destroy the OpenGL Context. This needs to be called on the same thread that `use()` was called. + */ + ~OpenGLRenderer(); + + /** + * Renders the given Texture to the Surface + */ + void renderTextureToSurface(const OpenGLTexture& texture, float* transformMatrix); + + /** + * Destroys the OpenGL context. This needs to be called on the same thread that `use()` was + * called. After calling `destroy()`, it is legal to call `use()` again, which will re-construct + * everything. + */ + void destroy(); + +private: + explicit OpenGLRenderer(std::shared_ptr context, ANativeWindow* surface); + +private: + int _width = 0, _height = 0; + std::shared_ptr _context; + ANativeWindow* _outputSurface; + EGLSurface _surface = EGL_NO_SURFACE; + +private: + PassThroughShader _passThroughShader; + +private: + static constexpr auto TAG = "OpenGLRenderer"; +}; + +} // namespace vision diff --git a/package/android/src/main/cpp/OpenGLTexture.h b/package/android/src/main/cpp/OpenGLTexture.h new file mode 100644 index 0000000..ec93c2a --- /dev/null +++ b/package/android/src/main/cpp/OpenGLTexture.h @@ -0,0 +1,22 @@ +// +// Created by Marc Rousavy on 30.08.23. +// + +#pragma once + +#include +#include +#include + +struct OpenGLTexture { + enum Type { Texture2D, ExternalOES }; + + // The ID of the texture as returned in glGenTextures(..) + GLuint id; + // GL_TEXTURE_2D or GL_TEXTURE_EXTERNAL_OES + GLenum target; + + // Width and height of the texture + int width = 0; + int height = 0; +}; diff --git a/package/android/src/main/cpp/PassThroughShader.cpp b/package/android/src/main/cpp/PassThroughShader.cpp new file mode 100644 index 0000000..ad51fdf --- /dev/null +++ b/package/android/src/main/cpp/PassThroughShader.cpp @@ -0,0 +1,111 @@ +// +// Created by Marc Rousavy on 28.08.23. +// + +#include "PassThroughShader.h" +#include "OpenGLError.h" +#include +#include +#include +#include +#include + +namespace vision { + +PassThroughShader::~PassThroughShader() { + if (_programId != NO_SHADER) { + glDeleteProgram(_programId); + _programId = NO_SHADER; + } + + if (_vertexBuffer != NO_BUFFER) { + glDeleteBuffers(1, &_vertexBuffer); + _vertexBuffer = NO_BUFFER; + } +} + +void PassThroughShader::draw(const OpenGLTexture& texture, float* transformMatrix) { + // 1. Set up Shader Program + if (_programId == NO_SHADER) { + _programId = createProgram(); + glUseProgram(_programId); + _vertexParameters = { + .aPosition = glGetAttribLocation(_programId, "aPosition"), + .aTexCoord = glGetAttribLocation(_programId, "aTexCoord"), + .uTransformMatrix = glGetUniformLocation(_programId, "uTransformMatrix"), + }; + _fragmentParameters = { + .uTexture = glGetUniformLocation(_programId, "uTexture"), + }; + } + + glUseProgram(_programId); + + // 2. Set up Vertices Buffer + if (_vertexBuffer == NO_BUFFER) { + glGenBuffers(1, &_vertexBuffer); + glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); + glBufferData(GL_ARRAY_BUFFER, sizeof(VERTICES), VERTICES, GL_STATIC_DRAW); + } + + // 3. Pass all uniforms/attributes for vertex shader + glEnableVertexAttribArray(_vertexParameters.aPosition); + glVertexAttribPointer(_vertexParameters.aPosition, 2, GL_FLOAT, GL_FALSE, sizeof(Vertex), + reinterpret_cast(offsetof(Vertex, position))); + + glEnableVertexAttribArray(_vertexParameters.aTexCoord); + glVertexAttribPointer(_vertexParameters.aTexCoord, 2, GL_FLOAT, GL_FALSE, sizeof(Vertex), + reinterpret_cast(offsetof(Vertex, texCoord))); + + glUniformMatrix4fv(_vertexParameters.uTransformMatrix, 1, GL_FALSE, transformMatrix); + + // 4. Pass texture to fragment shader + glActiveTexture(GL_TEXTURE0); + glBindTexture(texture.target, texture.id); + glUniform1i(_fragmentParameters.uTexture, 0); + + // 5. Draw! + glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); +} + +GLuint PassThroughShader::loadShader(GLenum shaderType, const char* shaderCode) { + GLuint shader = glCreateShader(shaderType); + if (shader == 0) + throw OpenGLError("Failed to load shader!"); + + glShaderSource(shader, 1, &shaderCode, nullptr); + glCompileShader(shader); + GLint compileStatus = GL_FALSE; + glGetShaderiv(shader, GL_COMPILE_STATUS, &compileStatus); + if (compileStatus == GL_FALSE) { + glDeleteShader(shader); + throw OpenGLError("Failed to compile shader!"); + } + return shader; +} + +GLuint PassThroughShader::createProgram() { + GLuint vertexShader = loadShader(GL_VERTEX_SHADER, VERTEX_SHADER); + GLuint fragmentShader = loadShader(GL_FRAGMENT_SHADER, FRAGMENT_SHADER); + + GLuint program = glCreateProgram(); + if (program == 0) + throw OpenGLError("Failed to create pass-through program!"); + + glAttachShader(program, vertexShader); + OpenGLError::checkIfError("Failed to attach Vertex Shader!"); + + glAttachShader(program, fragmentShader); + OpenGLError::checkIfError("Failed to attach Fragment Shader!"); + + glLinkProgram(program); + GLint linkStatus = GL_FALSE; + glGetProgramiv(program, GL_LINK_STATUS, &linkStatus); + if (!linkStatus) { + glDeleteProgram(program); + throw OpenGLError("Failed to load pass-through program!"); + } + return program; +} + +} // namespace vision diff --git a/package/android/src/main/cpp/PassThroughShader.h b/package/android/src/main/cpp/PassThroughShader.h new file mode 100644 index 0000000..b199b3d --- /dev/null +++ b/package/android/src/main/cpp/PassThroughShader.h @@ -0,0 +1,84 @@ +// +// Created by Marc Rousavy on 28.08.23. +// + +#pragma once + +#include +#include + +#include "OpenGLTexture.h" + +namespace vision { + +#define NO_SHADER 0 +#define NO_POSITION 0 +#define NO_BUFFER 0 + +struct Vertex { + GLfloat position[2]; + GLfloat texCoord[2]; +}; + +class PassThroughShader { +public: + PassThroughShader() = default; + ~PassThroughShader(); + + /** + * Draw the texture using this shader. + * Note: At the moment, only EXTERNAL textures are supported by the Shader. + */ + void draw(const OpenGLTexture& texture, float* transformMatrix); + +private: + // Loading + static GLuint loadShader(GLenum shaderType, const char* shaderCode); + static GLuint createProgram(); + +private: + // Parameters + GLuint _programId = NO_SHADER; + GLuint _vertexBuffer = NO_BUFFER; + struct VertexParameters { + GLint aPosition = NO_POSITION; + GLint aTexCoord = NO_POSITION; + GLint uTransformMatrix = NO_POSITION; + } _vertexParameters; + struct FragmentParameters { + GLint uTexture = NO_POSITION; + } _fragmentParameters; + +private: + // Statics + static constexpr Vertex VERTICES[] = { + {{-1.0f, -1.0f}, {0.0f, 0.0f}}, // bottom-left + {{1.0f, -1.0f}, {1.0f, 0.0f}}, // bottom-right + {{-1.0f, 1.0f}, {0.0f, 1.0f}}, // top-left + {{1.0f, 1.0f}, {1.0f, 1.0f}} // top-right + }; + + static constexpr char VERTEX_SHADER[] = R"( + attribute vec4 aPosition; + attribute vec2 aTexCoord; + uniform mat4 uTransformMatrix; + varying vec2 vTexCoord; + + void main() { + gl_Position = aPosition; + vTexCoord = (uTransformMatrix * vec4(aTexCoord, 0.0, 1.0)).xy; + } + )"; + static constexpr char FRAGMENT_SHADER[] = R"( + #extension GL_OES_EGL_image_external : require + precision mediump float; + varying vec2 vTexCoord; + uniform samplerExternalOES uTexture; + + void main() { + gl_FragColor = texture2D(uTexture, vTexCoord); + } + )"; +}; + +} // namespace vision diff --git a/package/android/src/main/cpp/VideoPipeline.cpp b/package/android/src/main/cpp/VideoPipeline.cpp new file mode 100644 index 0000000..387e2fc --- /dev/null +++ b/package/android/src/main/cpp/VideoPipeline.cpp @@ -0,0 +1,119 @@ +// +// Created by Marc Rousavy on 25.08.23. +// + +#include "VideoPipeline.h" +#include "OpenGLError.h" + +#include +#include +#include +#include +#include +#include + +#include + +#include "JFrameProcessor.h" +#include "OpenGLTexture.h" + +namespace vision { + +jni::local_ref VideoPipeline::initHybrid(jni::alias_ref jThis, int width, int height) { + return makeCxxInstance(jThis, width, height); +} + +VideoPipeline::VideoPipeline(jni::alias_ref jThis, int width, int height) : _javaPart(jni::make_global(jThis)) { + _width = width; + _height = height; + _context = OpenGLContext::CreateWithOffscreenSurface(); +} + +VideoPipeline::~VideoPipeline() { + // 1. Remove output surfaces + removeFrameProcessorOutputSurface(); + removeRecordingSessionOutputSurface(); + // 2. Delete the input textures + if (_inputTexture != std::nullopt) { + glDeleteTextures(1, &_inputTexture->id); + _inputTexture = std::nullopt; + } + // 3. Destroy the OpenGL context + _context = nullptr; +} + +void VideoPipeline::removeFrameProcessorOutputSurface() { + if (_frameProcessorOutput) + _frameProcessorOutput->destroy(); + _frameProcessorOutput = nullptr; +} + +void VideoPipeline::setFrameProcessorOutputSurface(jobject surface) { + // 1. Delete existing output surface + removeFrameProcessorOutputSurface(); + + // 2. Set new output surface if it is not null + ANativeWindow* window = ANativeWindow_fromSurface(jni::Environment::current(), surface); + _frameProcessorOutput = OpenGLRenderer::CreateWithWindowSurface(_context, window); +} + +void VideoPipeline::removeRecordingSessionOutputSurface() { + if (_recordingSessionOutput) + _recordingSessionOutput->destroy(); + _recordingSessionOutput = nullptr; +} + +void VideoPipeline::setRecordingSessionOutputSurface(jobject surface) { + // 1. Delete existing output surface + removeRecordingSessionOutputSurface(); + + // 2. Set new output surface if it is not null + ANativeWindow* window = ANativeWindow_fromSurface(jni::Environment::current(), surface); + _recordingSessionOutput = OpenGLRenderer::CreateWithWindowSurface(_context, window); +} + +int VideoPipeline::getInputTextureId() { + if (_inputTexture == std::nullopt) { + _inputTexture = _context->createTexture(OpenGLTexture::Type::ExternalOES, _width, _height); + } + + return static_cast(_inputTexture->id); +} + +void VideoPipeline::onBeforeFrame() { + _context->use(); + + glBindTexture(_inputTexture->target, _inputTexture->id); +} + +void VideoPipeline::onFrame(jni::alias_ref transformMatrixParam) { + // Get the OpenGL transform Matrix (transforms, scales, rotations) + float transformMatrix[16]; + transformMatrixParam->getRegion(0, 16, transformMatrix); + + OpenGLTexture& texture = _inputTexture.value(); + + if (_frameProcessorOutput) { + __android_log_print(ANDROID_LOG_INFO, TAG, "Rendering to FrameProcessor.."); + _frameProcessorOutput->renderTextureToSurface(texture, transformMatrix); + } + if (_recordingSessionOutput) { + __android_log_print(ANDROID_LOG_INFO, TAG, "Rendering to RecordingSession.."); + _recordingSessionOutput->renderTextureToSurface(texture, transformMatrix); + } +} + +void VideoPipeline::registerNatives() { + registerHybrid({ + makeNativeMethod("initHybrid", VideoPipeline::initHybrid), + makeNativeMethod("setFrameProcessorOutputSurface", VideoPipeline::setFrameProcessorOutputSurface), + makeNativeMethod("removeFrameProcessorOutputSurface", VideoPipeline::removeFrameProcessorOutputSurface), + makeNativeMethod("setRecordingSessionOutputSurface", VideoPipeline::setRecordingSessionOutputSurface), + makeNativeMethod("removeRecordingSessionOutputSurface", VideoPipeline::removeRecordingSessionOutputSurface), + makeNativeMethod("getInputTextureId", VideoPipeline::getInputTextureId), + makeNativeMethod("onBeforeFrame", VideoPipeline::onBeforeFrame), + makeNativeMethod("onFrame", VideoPipeline::onFrame), + }); +} + +} // namespace vision diff --git a/package/android/src/main/cpp/VideoPipeline.h b/package/android/src/main/cpp/VideoPipeline.h new file mode 100644 index 0000000..ca2551f --- /dev/null +++ b/package/android/src/main/cpp/VideoPipeline.h @@ -0,0 +1,66 @@ +// +// Created by Marc Rousavy on 25.08.23. +// + +#pragma once + +#include "OpenGLContext.h" +#include "OpenGLRenderer.h" +#include "PassThroughShader.h" +#include +#include +#include +#include +#include +#include + +namespace vision { + +using namespace facebook; + +class VideoPipeline : public jni::HybridClass { +public: + static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/core/VideoPipeline;"; + static jni::local_ref initHybrid(jni::alias_ref jThis, int width, int height); + static void registerNatives(); + +public: + ~VideoPipeline(); + + // -> SurfaceTexture input + int getInputTextureId(); + + // <- Frame Processor output + void setFrameProcessorOutputSurface(jobject surface); + void removeFrameProcessorOutputSurface(); + + // <- MediaRecorder output + void setRecordingSessionOutputSurface(jobject surface); + void removeRecordingSessionOutputSurface(); + + // Frame callbacks + void onBeforeFrame(); + void onFrame(jni::alias_ref transformMatrix); + +private: + // Private constructor. Use `create(..)` to create new instances. + explicit VideoPipeline(jni::alias_ref jThis, int width, int height); + +private: + // Input Surface Texture + std::optional _inputTexture = std::nullopt; + int _width = 0; + int _height = 0; + + // Output Contexts + std::shared_ptr _context = nullptr; + std::unique_ptr _frameProcessorOutput = nullptr; + std::unique_ptr _recordingSessionOutput = nullptr; + +private: + friend HybridBase; + jni::global_ref _javaPart; + static constexpr auto TAG = "VideoPipeline"; +}; + +} // namespace vision diff --git a/package/android/src/main/cpp/VisionCamera.cpp b/package/android/src/main/cpp/VisionCamera.cpp index bab71f7..3637e7e 100644 --- a/package/android/src/main/cpp/VisionCamera.cpp +++ b/package/android/src/main/cpp/VisionCamera.cpp @@ -1,6 +1,7 @@ #include "JFrameProcessor.h" #include "JVisionCameraProxy.h" #include "JVisionCameraScheduler.h" +#include "VideoPipeline.h" #include "VisionCameraProxy.h" #include #include @@ -10,6 +11,7 @@ JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void*) { vision::VisionCameraInstaller::registerNatives(); vision::JVisionCameraProxy::registerNatives(); vision::JVisionCameraScheduler::registerNatives(); + vision::VideoPipeline::registerNatives(); #if VISION_CAMERA_ENABLE_FRAME_PROCESSORS vision::JFrameProcessor::registerNatives(); #endif diff --git a/package/android/src/main/java/com/mrousavy/camera/Errors.kt b/package/android/src/main/java/com/mrousavy/camera/Errors.kt index 87a8410..ab52e68 100644 --- a/package/android/src/main/java/com/mrousavy/camera/Errors.kt +++ b/package/android/src/main/java/com/mrousavy/camera/Errors.kt @@ -4,12 +4,27 @@ import com.mrousavy.camera.core.outputs.CameraOutputs import com.mrousavy.camera.parsers.CameraDeviceError abstract class CameraError( - // example: "permission" + /** + * The domain of the error. Error domains are used to group errors. + * + * Example: "permission" + */ val domain: String, - // example: "microphone-permission-denied" + /** + * The id of the error. Errors are uniquely identified under a given domain. + * + * Example: "microphone-permission-denied" + */ val id: String, - // example: "The microphone permission was denied!" + /** + * A detailed error description of "what went wrong". + * + * Example: "The microphone permission was denied!" + */ message: String, + /** + * A throwable that caused this error. + */ cause: Throwable? = null ) : Throwable("[$domain/$id] $message", cause) @@ -31,6 +46,13 @@ class NoCameraDeviceError : CameraError("device", "no-device", "No device was set! Use `getAvailableCameraDevices()` to select a suitable Camera device.") class PixelFormatNotSupportedError(format: String) : CameraError("device", "pixel-format-not-supported", "The pixelFormat $format is not supported on the given Camera Device!") +class PixelFormatNotSupportedInVideoPipelineError(format: String) : + CameraError( + "device", + "pixel-format-not-supported", + "The pixelFormat $format is currently not supported in the VideoPipeline! " + + "See this issue for more details ($4.000 bounty!): https://github.com/mrousavy/react-native-vision-camera/issues/1837" + ) class CameraNotReadyError : CameraError("session", "camera-not-ready", "The Camera is not ready yet! Wait for the onInitialized() callback!") diff --git a/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt b/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt index dfce4d7..16cca60 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt @@ -1,84 +1,203 @@ package com.mrousavy.camera.core import android.graphics.ImageFormat +import android.graphics.SurfaceTexture import android.media.ImageReader -import android.media.ImageWriter import android.util.Log import android.view.Surface +import com.facebook.jni.HybridData import com.mrousavy.camera.CameraQueues +import com.mrousavy.camera.PixelFormatNotSupportedInVideoPipelineError import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.FrameProcessor import com.mrousavy.camera.parsers.Orientation +import com.mrousavy.camera.parsers.PixelFormat import java.io.Closeable -@Suppress("JoinDeclarationAndAssignment") +/** + * An OpenGL pipeline for streaming Camera Frames to one or more outputs. + * Currently, [VideoPipeline] can stream to a [FrameProcessor] and a [MediaRecorder]. + * + * @param [width] The width of the Frames to stream (> 0) + * @param [height] The height of the Frames to stream (> 0) + * @param [format] The format of the Frames to stream. ([ImageFormat.PRIVATE], [ImageFormat.YUV_420_888] or [ImageFormat.JPEG]) + */ +@Suppress("KotlinJniMissingFunction") class VideoPipeline(val width: Int, val height: Int, val format: Int = ImageFormat.PRIVATE, private val isMirrored: Boolean = false) : - ImageReader.OnImageAvailableListener, + SurfaceTexture.OnFrameAvailableListener, Closeable { companion object { private const val MAX_IMAGES = 3 private const val TAG = "VideoPipeline" - } - // Output 1 - private var frameProcessor: FrameProcessor? = null - - // Output 2 - private var recordingSession: RecordingSession? = null - private var recordingSessionImageWriter: ImageWriter? = null - - // Input - private val imageReader: ImageReader - val surface: Surface - - init { - imageReader = ImageReader.newInstance(width, height, format, MAX_IMAGES) - imageReader.setOnImageAvailableListener(this, CameraQueues.videoQueue.handler) - surface = imageReader.surface - } - - override fun close() { - synchronized(this) { - imageReader.close() - frameProcessor = null - recordingSessionImageWriter?.close() - recordingSessionImageWriter = null - recordingSession = null - } - } - - fun setFrameProcessorOutput(frameProcessor: FrameProcessor?) { - this.frameProcessor = frameProcessor - } - - fun setRecordingSessionOutput(recordingSession: RecordingSession?) { - synchronized(this) { - this.recordingSessionImageWriter?.close() - this.recordingSessionImageWriter = null - this.recordingSession = recordingSession - - if (recordingSession != null) { - this.recordingSessionImageWriter = ImageWriter.newInstance(recordingSession.surface, MAX_IMAGES) + init { + try { + System.loadLibrary("VisionCamera") + } catch (e: UnsatisfiedLinkError) { + Log.e( + TAG, + "Failed to load VisionCamera C++ library! " + + "OpenGL GPU VideoPipeline cannot be used.", + e + ) + throw e } } } - override fun onImageAvailable(reader: ImageReader) { - val image = reader.acquireLatestImage() - if (image == null) { - Log.w(TAG, "ImageReader failed to acquire a new image!") - return - } + private val mHybridData: HybridData + private var openGLTextureId: Int? = null + private var transformMatrix = FloatArray(16) + private var isActive = true - // If we have a Frame Processor, call it - frameProcessor?.let { fp -> + // Output 1 + private var frameProcessor: FrameProcessor? = null + private var imageReader: ImageReader? = null + + // Output 2 + private var recordingSession: RecordingSession? = null + + // Input + private val surfaceTexture: SurfaceTexture + val surface: Surface + + init { + Log.i( + TAG, + "Initializing $width x $height Video Pipeline " + + "(format: ${PixelFormat.fromImageFormat(format)} #$format)" + ) + // TODO: We currently use OpenGL for the Video Pipeline. + // OpenGL only works in the RGB (RGBA_8888; 0x23) pixel-format, so we cannot + // override the pixel-format to something like YUV or PRIVATE. + // This absolutely sucks and I would prefer to replace the OpenGL pipeline with + // something similar to how iOS works where we just pass GPU buffers around, + // but android.media APIs are just not as advanced yet. + // For example, ImageReader/ImageWriter is way too buggy and does not work with MediaRecorder. + // See this issue ($4.000 bounty!) for more details: + // https://github.com/mrousavy/react-native-vision-camera/issues/1837 + if (format != ImageFormat.PRIVATE && format != 0x23) { + throw PixelFormatNotSupportedInVideoPipelineError(PixelFormat.fromImageFormat(format).unionValue) + } + mHybridData = initHybrid(width, height) + surfaceTexture = SurfaceTexture(false) + surfaceTexture.setDefaultBufferSize(width, height) + surfaceTexture.setOnFrameAvailableListener(this) + surface = Surface(surfaceTexture) + } + + override fun close() { + synchronized(this) { + isActive = false + imageReader?.close() + imageReader = null + frameProcessor = null + recordingSession = null + surfaceTexture.release() + mHybridData.resetNative() + } + } + + override fun onFrameAvailable(surfaceTexture: SurfaceTexture) { + synchronized(this) { + if (!isActive) return@synchronized + + // 1. Attach Surface to OpenGL context + if (openGLTextureId == null) { + openGLTextureId = getInputTextureId() + surfaceTexture.attachToGLContext(openGLTextureId!!) + Log.i(TAG, "Attached Texture to Context $openGLTextureId") + } + + // 2. Prepare the OpenGL context (eglMakeCurrent) + onBeforeFrame() + + // 3. Update the OpenGL texture + surfaceTexture.updateTexImage() + + // 4. Get the transform matrix from the SurfaceTexture (rotations/scales applied by Camera) + surfaceTexture.getTransformMatrix(transformMatrix) + + // 5. Draw it with applied rotation/mirroring + onFrame(transformMatrix) + } + } + + private fun getImageReader(): ImageReader { + if (format != ImageFormat.PRIVATE) { + Log.w( + TAG, + "Warning: pixelFormat \"${PixelFormat.fromImageFormat(format).unionValue}\" might " + + "not be supported on this device because the C++ OpenGL GPU Video Pipeline operates in RGBA_8888. " + + "I wanted to use an ImageReader -> ImageWriter setup for this, but I couldn't get it to work. " + + "See this PR for more details: https://github.com/mrousavy/react-native-vision-camera/pull/1836" + ) + } + val imageReader = ImageReader.newInstance(width, height, format, MAX_IMAGES) + imageReader.setOnImageAvailableListener({ reader -> + Log.i("VideoPipeline", "ImageReader::onImageAvailable!") + val image = reader.acquireNextImage() ?: return@setOnImageAvailableListener + + // TODO: Get correct orientation and isMirrored val frame = Frame(image, image.timestamp, Orientation.PORTRAIT, isMirrored) frame.incrementRefCount() - fp.call(frame) + frameProcessor?.call(frame) frame.decrementRefCount() - } - - // If we have a RecordingSession, pass the image through - recordingSessionImageWriter?.queueInputImage(image) + }, CameraQueues.videoQueue.handler) + return imageReader } + + /** + * Configures the Pipeline to also call the given [FrameProcessor] (or null). + */ + fun setFrameProcessorOutput(frameProcessor: FrameProcessor?) { + synchronized(this) { + Log.i(TAG, "Setting $width x $height FrameProcessor Output...") + this.frameProcessor = frameProcessor + + if (frameProcessor != null) { + if (this.imageReader == null) { + // 1. Create new ImageReader that just calls the Frame Processor + this.imageReader = getImageReader() + } + + // 2. Configure OpenGL pipeline to stream Frames into the ImageReader's surface + setFrameProcessorOutputSurface(imageReader!!.surface) + } else { + // 1. Configure OpenGL pipeline to stop streaming Frames into the ImageReader's surface + removeFrameProcessorOutputSurface() + + // 2. Close the ImageReader + this.imageReader?.close() + this.imageReader = null + } + } + } + + /** + * Configures the Pipeline to also write Frames to a Surface from a [MediaRecorder] (or null) + */ + fun setRecordingSessionOutput(recordingSession: RecordingSession?) { + synchronized(this) { + Log.i(TAG, "Setting $width x $height RecordingSession Output...") + if (recordingSession != null) { + // Configure OpenGL pipeline to stream Frames into the Recording Session's surface + setRecordingSessionOutputSurface(recordingSession.surface) + this.recordingSession = recordingSession + } else { + // Configure OpenGL pipeline to stop streaming Frames into the Recording Session's surface + removeRecordingSessionOutputSurface() + this.recordingSession = null + } + } + } + + private external fun getInputTextureId(): Int + private external fun onBeforeFrame() + private external fun onFrame(transformMatrix: FloatArray) + private external fun setFrameProcessorOutputSurface(surface: Any) + private external fun removeFrameProcessorOutputSurface() + private external fun setRecordingSessionOutputSurface(surface: Any) + private external fun removeRecordingSessionOutputSurface() + private external fun initHybrid(width: Int, height: Int): HybridData }