From ea3686cb9ad067ae06d3ba9f259e7d9d2ed6e32f Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Tue, 29 Aug 2023 17:52:03 +0200 Subject: [PATCH] feat: Create C++/OpenGL-based Video Pipeline for more efficient Recording and Frame Processing (#1721) * Create `VideoPipeline` c++ * Remove folly C++ dependency * Create `VideoPipeline` HybridClass * Set up OpenGL * Add outputs * Update VideoPipeline.kt * Bum `minSdkVersion` to `26` * Create `VideoPipelineOutput` * Create output funcs * Set output pipelines * Add FP/Recording on Output change * Update VideoPipeline.cpp * Create `PassThroughShader` * Try to draw? I have honestly no idea * fix: Fix `setFrameProcessor` nameclash * fix: Fix `high-res-sizes` being null * Add preview output * Create `OpenGLContext.cpp` * Make screen red * This _should_ work (MESSY) * FINALLY RENDER TEXTURE * Rotate * Mirror * Clean up a bit * Add `getWidth()`/`getHeight()` * Cleanup * fix: Use uniforms instead of attributes * Draw with passed rotation/mirror mode * feat: Use SurfaceTexture's transformMatrix in OpenGL pipeline (#1727) * feat: Use Transform Matrix from SurfaceTexture * Renam * feat: Fix OpenGL Shader * Update VideoPipeline.kt * Measure elapsed time * fix: Fix low resolution * Render to offscreen * Render to every context * Release `SurfaceTexture` on close * Use one OpenGL context to render to multiple EGLSurfaces * Clean up a bit * fix: Fix recording pipeline not triggering * fix: Synchronize close to prevent nulls * Update OpenGLRenderer.cpp * fix: Hardcode Android recorder size --- android/CMakeLists.txt | 9 +- android/build.gradle | 2 +- android/gradle.properties | 4 - android/src/main/cpp/OpenGLContext.cpp | 133 +++++++++++++ android/src/main/cpp/OpenGLContext.h | 68 +++++++ android/src/main/cpp/{skia => }/OpenGLError.h | 0 android/src/main/cpp/OpenGLRenderer.cpp | 74 +++++++ android/src/main/cpp/OpenGLRenderer.h | 58 ++++++ android/src/main/cpp/PassThroughShader.cpp | 119 +++++++++++ android/src/main/cpp/PassThroughShader.h | 81 ++++++++ android/src/main/cpp/VideoPipeline.cpp | 133 +++++++++++++ android/src/main/cpp/VideoPipeline.h | 72 +++++++ android/src/main/cpp/VisionCamera.cpp | 2 + .../cpp/frameprocessor/FrameHostObject.cpp | 1 - .../main/cpp/frameprocessor/FrameHostObject.h | 3 - .../cpp/frameprocessor/JSIJNIConversion.cpp | 4 +- .../java-bindings/JVisionCameraScheduler.cpp | 4 +- .../java-bindings/JVisionCameraScheduler.h | 4 +- .../java/com/mrousavy/camera/CameraSession.kt | 47 +++-- .../java/com/mrousavy/camera/CameraView.kt | 2 +- .../com/mrousavy/camera/CameraViewModule.kt | 10 - .../CameraCharacteristics+getOutputSizes.kt | 6 +- .../CameraDevice+createCaptureSession.kt | 65 +++--- .../mrousavy/camera/utils/RecordingSession.kt | 45 +---- .../mrousavy/camera/utils/VideoPipeline.kt | 185 ++++++++++++++++++ .../camera/utils/outputs/CameraOutputs.kt | 23 +-- .../camera/utils/outputs/SurfaceOutput.kt | 1 - .../utils/outputs/VideoPipelineOutput.kt | 22 +++ docs/docs/guides/TROUBLESHOOTING.mdx | 2 +- example/android/build.gradle | 2 +- 30 files changed, 1024 insertions(+), 157 deletions(-) create mode 100644 android/src/main/cpp/OpenGLContext.cpp create mode 100644 android/src/main/cpp/OpenGLContext.h rename android/src/main/cpp/{skia => }/OpenGLError.h (100%) create mode 100644 android/src/main/cpp/OpenGLRenderer.cpp create mode 100644 android/src/main/cpp/OpenGLRenderer.h create mode 100644 android/src/main/cpp/PassThroughShader.cpp create mode 100644 android/src/main/cpp/PassThroughShader.h create mode 100644 android/src/main/cpp/VideoPipeline.cpp create mode 100644 android/src/main/cpp/VideoPipeline.h create mode 100644 android/src/main/java/com/mrousavy/camera/utils/VideoPipeline.kt create mode 100644 android/src/main/java/com/mrousavy/camera/utils/outputs/VideoPipelineOutput.kt diff --git a/android/CMakeLists.txt b/android/CMakeLists.txt index 97406ef..943d5b5 100644 --- a/android/CMakeLists.txt +++ b/android/CMakeLists.txt @@ -7,10 +7,6 @@ set(BUILD_DIR ${CMAKE_SOURCE_DIR}/build) set(CMAKE_VERBOSE_MAKEFILE ON) set(CMAKE_CXX_STANDARD 17) -# Folly -include("${NODE_MODULES_DIR}/react-native/ReactAndroid/cmake-utils/folly-flags.cmake") -add_compile_options(${folly_FLAGS}) - # Third party libraries (Prefabs) find_package(ReactAndroid REQUIRED CONFIG) find_package(fbjni REQUIRED CONFIG) @@ -25,6 +21,10 @@ add_library( SHARED ../cpp/JSITypedArray.cpp src/main/cpp/VisionCamera.cpp + src/main/cpp/VideoPipeline.cpp + src/main/cpp/PassThroughShader.cpp + src/main/cpp/OpenGLContext.cpp + src/main/cpp/OpenGLRenderer.cpp # Frame Processor src/main/cpp/frameprocessor/FrameHostObject.cpp src/main/cpp/frameprocessor/FrameProcessorPluginHostObject.cpp @@ -60,7 +60,6 @@ target_link_libraries( android # <-- Android JNI core ReactAndroid::jsi # <-- RN: JSI ReactAndroid::reactnativejni # <-- RN: React Native JNI bindings - ReactAndroid::folly_runtime # <-- RN: For casting JSI <> Java objects fbjni::fbjni # <-- fbjni ) diff --git a/android/build.gradle b/android/build.gradle index 0e6cc57..0e1387b 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -93,7 +93,7 @@ android { } defaultConfig { - minSdkVersion safeExtGet('minSdkVersion', 21) + minSdkVersion safeExtGet('minSdkVersion', 26) compileSdkVersion safeExtGet('compileSdkVersion', 33) targetSdkVersion safeExtGet('targetSdkVersion', 33) versionCode 1 diff --git a/android/gradle.properties b/android/gradle.properties index e29191a..4a166b7 100644 --- a/android/gradle.properties +++ b/android/gradle.properties @@ -14,10 +14,6 @@ org.gradle.configureondemand=true # http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects # org.gradle.parallel=true #Fri Feb 19 20:46:14 CET 2021 -VisionCamera_buildToolsVersion=30.0.0 -VisionCamera_compileSdkVersion=31 VisionCamera_kotlinVersion=1.7.20 -VisionCamera_targetSdkVersion=31 -VisionCamera_ndkVersion=21.4.7075529 android.enableJetifier=true android.useAndroidX=true diff --git a/android/src/main/cpp/OpenGLContext.cpp b/android/src/main/cpp/OpenGLContext.cpp new file mode 100644 index 0000000..89a8b96 --- /dev/null +++ b/android/src/main/cpp/OpenGLContext.cpp @@ -0,0 +1,133 @@ +// +// Created by Marc Rousavy on 29.08.23. +// + +#include "OpenGLContext.h" + +#include +#include +#include + +#include +#include + +#include "OpenGLError.h" + +namespace vision { + +std::shared_ptr OpenGLContext::CreateWithOffscreenSurface(int width, int height) { + return std::unique_ptr(new OpenGLContext(width, height)); +} + +OpenGLContext::OpenGLContext(int width, int height) { + _width = width; + _height = height; +} + +OpenGLContext::~OpenGLContext() { + destroy(); +} + +void OpenGLContext::destroy() { + if (display != EGL_NO_DISPLAY) { + eglMakeCurrent(display, offscreenSurface, offscreenSurface, context); + if (offscreenSurface != EGL_NO_SURFACE) { + __android_log_print(ANDROID_LOG_INFO, TAG, "Destroying OpenGL Surface..."); + eglDestroySurface(display, offscreenSurface); + offscreenSurface = EGL_NO_SURFACE; + } + if (context != EGL_NO_CONTEXT) { + __android_log_print(ANDROID_LOG_INFO, TAG, "Destroying OpenGL Context..."); + eglDestroyContext(display, context); + context = EGL_NO_CONTEXT; + } + __android_log_print(ANDROID_LOG_INFO, TAG, "Destroying OpenGL Display..."); + eglTerminate(display); + display = EGL_NO_DISPLAY; + config = nullptr; + } +} + +void OpenGLContext::ensureOpenGL() { + bool successful; + // EGLDisplay + if (display == EGL_NO_DISPLAY) { + __android_log_print(ANDROID_LOG_INFO, TAG, "Initializing EGLDisplay.."); + display = eglGetDisplay(EGL_DEFAULT_DISPLAY); + if (display == EGL_NO_DISPLAY) throw OpenGLError("Failed to get default OpenGL Display!"); + + EGLint major; + EGLint minor; + successful = eglInitialize(display, &major, &minor); + if (!successful) throw OpenGLError("Failed to initialize OpenGL!"); + } + + // EGLConfig + if (config == nullptr) { + __android_log_print(ANDROID_LOG_INFO, TAG, "Initializing EGLConfig.."); + EGLint attributes[] = {EGL_RENDERABLE_TYPE, EGL_OPENGL_ES2_BIT, + EGL_SURFACE_TYPE, EGL_WINDOW_BIT, + EGL_ALPHA_SIZE, 8, + EGL_BLUE_SIZE, 8, + EGL_GREEN_SIZE, 8, + EGL_RED_SIZE, 8, + EGL_DEPTH_SIZE, 0, + EGL_STENCIL_SIZE, 0, + EGL_NONE}; + EGLint numConfigs; + successful = eglChooseConfig(display, attributes, &config, 1, &numConfigs); + if (!successful || numConfigs == 0) throw OpenGLError("Failed to choose OpenGL config!"); + } + + // EGLContext + if (context == EGL_NO_CONTEXT) { + __android_log_print(ANDROID_LOG_INFO, TAG, "Initializing EGLContext.."); + EGLint contextAttributes[] = {EGL_CONTEXT_CLIENT_VERSION, 2, EGL_NONE}; + context = eglCreateContext(display, config, nullptr, contextAttributes); + if (context == EGL_NO_CONTEXT) throw OpenGLError("Failed to create OpenGL context!"); + } + + // EGLSurface + if (offscreenSurface == EGL_NO_SURFACE) { + // If we don't have a surface at all + __android_log_print(ANDROID_LOG_INFO, TAG, "Initializing %i x %i offscreen pbuffer EGLSurface..", _width, _height); + EGLint attributes[] = {EGL_WIDTH, _width, + EGL_HEIGHT, _height, + EGL_NONE}; + offscreenSurface = eglCreatePbufferSurface(display, config, attributes); + if (offscreenSurface == EGL_NO_SURFACE) throw OpenGLError("Failed to create OpenGL Surface!"); + } +} + +void OpenGLContext::use() { + this->use(offscreenSurface); +} + +void OpenGLContext::use(EGLSurface surface) { + if (surface == EGL_NO_SURFACE) throw OpenGLError("Cannot render to a null Surface!"); + + // 1. Make sure the OpenGL context is initialized + this->ensureOpenGL(); + + // 2. Make the OpenGL context current + bool successful = eglMakeCurrent(display, surface, surface, context); + if (!successful || eglGetError() != EGL_SUCCESS) throw OpenGLError("Failed to use current OpenGL context!"); + + // 3. Caller can now render to this surface +} + +GLuint OpenGLContext::createTexture() { + // 1. Make sure the OpenGL context is initialized + this->ensureOpenGL(); + + // 2. Make the OpenGL context current + bool successful = eglMakeCurrent(display, offscreenSurface, offscreenSurface, context); + if (!successful || eglGetError() != EGL_SUCCESS) throw OpenGLError("Failed to use current OpenGL context!"); + + GLuint textureId; + glGenTextures(1, &textureId); + + return textureId; +} + +} // namespace vision diff --git a/android/src/main/cpp/OpenGLContext.h b/android/src/main/cpp/OpenGLContext.h new file mode 100644 index 0000000..770fa54 --- /dev/null +++ b/android/src/main/cpp/OpenGLContext.h @@ -0,0 +1,68 @@ +// +// Created by Marc Rousavy on 29.08.23. +// + +#pragma once + +#include +#include + +#include +#include + +#include "PassThroughShader.h" + +namespace vision { + +/** + * An OpenGL Context that can be used to render to different surfaces. + * By default, it creates an off-screen PixelBuffer surface. + */ +class OpenGLContext { + public: + /** + * Create a new instance of the OpenGLContext that draws to an off-screen PixelBuffer surface. + * This will not perform any OpenGL operations yet, and is therefore safe to call from any Thread. + */ + static std::shared_ptr CreateWithOffscreenSurface(int width, int height); + /** + * Destroy the OpenGL Context. This needs to be called on the same thread that `use()` was called. + */ + ~OpenGLContext(); + + /** + * Use this OpenGL Context to render to the given EGLSurface. + * After the `renderFunc` returns, the default offscreen PixelBuffer surface becomes active again. + */ + void use(EGLSurface surface); + + /** + * Use this OpenGL Context to render to the offscreen PixelBuffer surface. + */ + void use(); + + /** + * Create a new texture on this context + */ + GLuint createTexture(); + + public: + EGLDisplay display = EGL_NO_DISPLAY; + EGLContext context = EGL_NO_CONTEXT; + EGLSurface offscreenSurface = EGL_NO_SURFACE; + EGLConfig config = nullptr; + + private: + int _width = 0, _height = 0; + explicit OpenGLContext(int width, int height); + void destroy(); + void ensureOpenGL(); + + private: + PassThroughShader _passThroughShader; + + private: + static constexpr auto TAG = "OpenGLContext"; +}; + +} // namespace vision diff --git a/android/src/main/cpp/skia/OpenGLError.h b/android/src/main/cpp/OpenGLError.h similarity index 100% rename from android/src/main/cpp/skia/OpenGLError.h rename to android/src/main/cpp/OpenGLError.h diff --git a/android/src/main/cpp/OpenGLRenderer.cpp b/android/src/main/cpp/OpenGLRenderer.cpp new file mode 100644 index 0000000..b8b98ee --- /dev/null +++ b/android/src/main/cpp/OpenGLRenderer.cpp @@ -0,0 +1,74 @@ +// +// Created by Marc Rousavy on 29.08.23. +// + +#include "OpenGLRenderer.h" + +#include +#include +#include + +#include +#include + +#include + +#include "OpenGLError.h" + +namespace vision { + +std::unique_ptr OpenGLRenderer::CreateWithWindowSurface(std::shared_ptr context, ANativeWindow* surface) { + return std::unique_ptr(new OpenGLRenderer(std::move(context), surface)); +} + +OpenGLRenderer::OpenGLRenderer(std::shared_ptr context, ANativeWindow* surface) { + _context = std::move(context); + _outputSurface = surface; + _width = ANativeWindow_getWidth(surface); + _height = ANativeWindow_getHeight(surface); +} + +OpenGLRenderer::~OpenGLRenderer() { + if (_outputSurface != nullptr) { + ANativeWindow_release(_outputSurface); + } + destroy(); +} + +void OpenGLRenderer::destroy() { + if (_context != nullptr && _surface != EGL_NO_DISPLAY) { + __android_log_print(ANDROID_LOG_INFO, TAG, "Destroying OpenGL Surface..."); + eglDestroySurface(_context->display, _surface); + _surface = EGL_NO_SURFACE; + } +} + +void OpenGLRenderer::renderTextureToSurface(GLuint textureId, float* transformMatrix) { + if (_surface == EGL_NO_SURFACE) { + __android_log_print(ANDROID_LOG_INFO, TAG, "Creating Window Surface..."); + _context->use(); + _surface = eglCreateWindowSurface(_context->display, _context->config, _outputSurface, nullptr); + } + + // 1. Activate the OpenGL context for this surface + _context->use(_surface); + + // 2. Set the viewport for rendering + glViewport(0, 0, _width, _height); + glDisable(GL_BLEND); + + // 3. Bind the input texture + glBindTexture(GL_TEXTURE_EXTERNAL_OES, textureId); + glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + + // 4. Draw it using the pass-through shader which also applies transforms + _passThroughShader.draw(textureId, transformMatrix); + + // 5. Swap buffers to pass it to the window surface + eglSwapBuffers(_context->display, _surface); +} + +} // namespace vision diff --git a/android/src/main/cpp/OpenGLRenderer.h b/android/src/main/cpp/OpenGLRenderer.h new file mode 100644 index 0000000..508f897 --- /dev/null +++ b/android/src/main/cpp/OpenGLRenderer.h @@ -0,0 +1,58 @@ +// +// Created by Marc Rousavy on 29.08.23. +// + +#pragma once + +#include +#include +#include +#include +#include "PassThroughShader.h" + +#include "OpenGLContext.h" + +namespace vision { + +class OpenGLRenderer { + public: + /** + * Create a new instance of the OpenGLRenderer that draws to an on-screen window surface. + * This will not perform any OpenGL operations yet, and is therefore safe to call from any Thread. + * + * Note: The `surface` is considered moved, and the OpenGL context will release it when it is being deleted. + */ + static std::unique_ptr CreateWithWindowSurface(std::shared_ptr context, ANativeWindow* surface); + /** + * Destroy the OpenGL Context. This needs to be called on the same thread that `use()` was called. + */ + ~OpenGLRenderer(); + + /** + * Renders the given Texture ID to the Surface + */ + void renderTextureToSurface(GLuint textureId, float* transformMatrix); + + /** + * Destroys the OpenGL context. This needs to be called on the same thread that `use()` was called. + * After calling `destroy()`, it is legal to call `use()` again, which will re-construct everything. + */ + void destroy(); + + private: + explicit OpenGLRenderer(std::shared_ptr context, ANativeWindow* surface); + + private: + int _width = 0, _height = 0; + std::shared_ptr _context; + ANativeWindow* _outputSurface; + EGLSurface _surface = EGL_NO_SURFACE; + + private: + PassThroughShader _passThroughShader; + + private: + static constexpr auto TAG = "OpenGLRenderer"; +}; + +} // namespace vision diff --git a/android/src/main/cpp/PassThroughShader.cpp b/android/src/main/cpp/PassThroughShader.cpp new file mode 100644 index 0000000..1851dac --- /dev/null +++ b/android/src/main/cpp/PassThroughShader.cpp @@ -0,0 +1,119 @@ +// +// Created by Marc Rousavy on 28.08.23. +// + +#include "PassThroughShader.h" +#include +#include +#include +#include +#include "OpenGLError.h" +#include + +namespace vision { + +PassThroughShader::~PassThroughShader() { + if (_programId != NO_SHADER) { + glDeleteProgram(_programId); + _programId = NO_SHADER; + } + + if (_vertexBuffer != NO_BUFFER) { + glDeleteBuffers(1, &_vertexBuffer); + _vertexBuffer = NO_BUFFER; + } +} + +void PassThroughShader::draw(GLuint textureId, float* transformMatrix) { + // 1. Set up Shader Program + if (_programId == NO_SHADER) { + _programId = createProgram(); + } + + glUseProgram(_programId); + + if (_vertexParameters.aPosition == NO_POSITION) { + _vertexParameters = { + .aPosition = glGetAttribLocation(_programId, "aPosition"), + .aTexCoord = glGetAttribLocation(_programId, "aTexCoord"), + .uTransformMatrix = glGetUniformLocation(_programId, "uTransformMatrix"), + }; + _fragmentParameters = { + .uTexture = glGetUniformLocation(_programId, "uTexture"), + }; + } + + // 2. Set up Vertices Buffer + if (_vertexBuffer == NO_BUFFER) { + glGenBuffers(1, &_vertexBuffer); + glBindBuffer(GL_ARRAY_BUFFER, _vertexBuffer); + glBufferData(GL_ARRAY_BUFFER, sizeof(VERTICES), VERTICES, GL_STATIC_DRAW); + } + + // 3. Pass all uniforms/attributes for vertex shader + glEnableVertexAttribArray(_vertexParameters.aPosition); + glVertexAttribPointer(_vertexParameters.aPosition, + 2, + GL_FLOAT, + GL_FALSE, + sizeof(Vertex), + reinterpret_cast(offsetof(Vertex, position))); + + glEnableVertexAttribArray(_vertexParameters.aTexCoord); + glVertexAttribPointer(_vertexParameters.aTexCoord, + 2, + GL_FLOAT, + GL_FALSE, + sizeof(Vertex), + reinterpret_cast(offsetof(Vertex, texCoord))); + + glUniformMatrix4fv(_vertexParameters.uTransformMatrix, 1, GL_FALSE, transformMatrix); + + // 4. Pass texture to fragment shader + glActiveTexture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_EXTERNAL_OES, textureId); + glUniform1i(_fragmentParameters.uTexture, 0); + + // 5. Draw! + glDrawArrays(GL_TRIANGLE_STRIP, 0, 4); +} + +GLuint PassThroughShader::loadShader(GLenum shaderType, const char* shaderCode) { + GLuint shader = glCreateShader(shaderType); + if (shader == 0) throw OpenGLError("Failed to load shader!"); + + glShaderSource(shader, 1, &shaderCode, nullptr); + glCompileShader(shader); + GLint compileStatus = GL_FALSE; + glGetShaderiv(shader, GL_COMPILE_STATUS, &compileStatus); + if (compileStatus == GL_FALSE) { + glDeleteShader(shader); + throw OpenGLError("Failed to compile shader!"); + } + return shader; +} + +GLuint PassThroughShader::createProgram() { + GLuint vertexShader = loadShader(GL_VERTEX_SHADER, VERTEX_SHADER); + GLuint fragmentShader = loadShader(GL_FRAGMENT_SHADER, FRAGMENT_SHADER); + + GLuint program = glCreateProgram(); + if (program == 0) throw OpenGLError("Failed to create pass-through program!"); + + glAttachShader(program, vertexShader); + if (glGetError() != GL_NO_ERROR) throw OpenGLError("Failed to attach Vertex Shader!"); + + glAttachShader(program, fragmentShader); + if (glGetError() != GL_NO_ERROR) throw OpenGLError("Failed to attach Fragment Shader!"); + + glLinkProgram(program); + GLint linkStatus = GL_FALSE; + glGetProgramiv(program, GL_LINK_STATUS, &linkStatus); + if (!linkStatus) { + glDeleteProgram(program); + throw OpenGLError("Failed to load pass-through program!"); + } + return program; +} + +} // namespace vision diff --git a/android/src/main/cpp/PassThroughShader.h b/android/src/main/cpp/PassThroughShader.h new file mode 100644 index 0000000..9e151aa --- /dev/null +++ b/android/src/main/cpp/PassThroughShader.h @@ -0,0 +1,81 @@ +// +// Created by Marc Rousavy on 28.08.23. +// + +#pragma once + +#include +#include + +namespace vision { + +#define NO_SHADER 0 +#define NO_POSITION 0 +#define NO_BUFFER 0 + +struct Vertex { + GLfloat position[2]; + GLfloat texCoord[2]; +}; + +class PassThroughShader { + public: + PassThroughShader() = default; + ~PassThroughShader(); + + /** + * Draw the texture using this shader. + */ + void draw(GLuint textureId, float* transformMatrix); + + private: + // Loading + static GLuint loadShader(GLenum shaderType, const char* shaderCode); + static GLuint createProgram(); + + private: + // Parameters + GLuint _programId = NO_SHADER; + GLuint _vertexBuffer = NO_BUFFER; + struct VertexParameters { + GLint aPosition = NO_POSITION; + GLint aTexCoord = NO_POSITION; + GLint uTransformMatrix = NO_POSITION; + } _vertexParameters; + struct FragmentParameters { + GLint uTexture = NO_POSITION; + } _fragmentParameters; + + private: + // Statics + static constexpr Vertex VERTICES[] = { + {{-1.0f, -1.0f}, {0.0f, 0.0f}}, // bottom-left + {{1.0f, -1.0f}, {1.0f, 0.0f}}, // bottom-right + {{-1.0f, 1.0f}, {0.0f, 1.0f}}, // top-left + {{1.0f, 1.0f}, {1.0f, 1.0f}} // top-right + }; + + static constexpr char VERTEX_SHADER[] = R"( + attribute vec4 aPosition; + attribute vec2 aTexCoord; + uniform mat4 uTransformMatrix; + varying vec2 vTexCoord; + + void main() { + gl_Position = aPosition; + vTexCoord = (uTransformMatrix * vec4(aTexCoord, 0.0, 1.0)).xy; + } + )"; + static constexpr char FRAGMENT_SHADER[] = R"( + #extension GL_OES_EGL_image_external : require + precision mediump float; + varying vec2 vTexCoord; + uniform samplerExternalOES uTexture; + + void main() { + gl_FragColor = texture2D(uTexture, vTexCoord); + } + )"; +}; + +} // namespace vision diff --git a/android/src/main/cpp/VideoPipeline.cpp b/android/src/main/cpp/VideoPipeline.cpp new file mode 100644 index 0000000..3242210 --- /dev/null +++ b/android/src/main/cpp/VideoPipeline.cpp @@ -0,0 +1,133 @@ +// +// Created by Marc Rousavy on 25.08.23. +// + +#include "VideoPipeline.h" +#include "OpenGLError.h" + +#include +#include +#include +#include + +namespace vision { + +jni::local_ref VideoPipeline::initHybrid(jni::alias_ref jThis, int width, int height) { + return makeCxxInstance(jThis, width, height); +} + +VideoPipeline::VideoPipeline(jni::alias_ref jThis, int width, int height): _javaPart(jni::make_global(jThis)) { + _width = width; + _height = height; + _context = OpenGLContext::CreateWithOffscreenSurface(width, height); +} + +VideoPipeline::~VideoPipeline() { + // 1. Remove output surfaces + removeFrameProcessorOutputSurface(); + removeRecordingSessionOutputSurface(); + removePreviewOutputSurface(); + // 2. Delete the input textures + if (_inputTextureId != NO_TEXTURE) { + glDeleteTextures(1, &_inputTextureId); + _inputTextureId = NO_TEXTURE; + } + // 4. Destroy all surfaces + _previewOutput = nullptr; + _frameProcessorOutput = nullptr; + _recordingSessionOutput = nullptr; + // 5. Destroy the OpenGL context + _context = nullptr; +} + +void VideoPipeline::removeFrameProcessorOutputSurface() { + if (_frameProcessorOutput) _frameProcessorOutput->destroy(); + _frameProcessorOutput = nullptr; +} + +void VideoPipeline::setFrameProcessorOutputSurface(jobject surface) { + // 1. Delete existing output surface + removeFrameProcessorOutputSurface(); + + // 2. Set new output surface if it is not null + ANativeWindow* window = ANativeWindow_fromSurface(jni::Environment::current(), surface); + _frameProcessorOutput = OpenGLRenderer::CreateWithWindowSurface(_context, window); +} + +void VideoPipeline::removeRecordingSessionOutputSurface() { + if (_recordingSessionOutput) _recordingSessionOutput->destroy(); + _recordingSessionOutput = nullptr; +} + +void VideoPipeline::setRecordingSessionOutputSurface(jobject surface) { + // 1. Delete existing output surface + removeRecordingSessionOutputSurface(); + + // 2. Set new output surface if it is not null + ANativeWindow* window = ANativeWindow_fromSurface(jni::Environment::current(), surface); + _recordingSessionOutput = OpenGLRenderer::CreateWithWindowSurface(_context, window); +} + +void VideoPipeline::removePreviewOutputSurface() { + if (_previewOutput) _previewOutput->destroy(); + _previewOutput = nullptr; +} + +void VideoPipeline::setPreviewOutputSurface(jobject surface) { + // 1. Delete existing output surface + removePreviewOutputSurface(); + + // 2. Set new output surface if it is not null + ANativeWindow* window = ANativeWindow_fromSurface(jni::Environment::current(), surface); + _previewOutput = OpenGLRenderer::CreateWithWindowSurface(_context, window); +} + +int VideoPipeline::getInputTextureId() { + if (_inputTextureId != NO_TEXTURE) return static_cast(_inputTextureId); + + _inputTextureId = _context->createTexture(); + + return static_cast(_inputTextureId); +} + +void VideoPipeline::onBeforeFrame() { + _context->use(); + + glBindTexture(GL_TEXTURE_EXTERNAL_OES, _inputTextureId); +} + +void VideoPipeline::onFrame(jni::alias_ref transformMatrixParam) { + // Get the OpenGL transform Matrix (transforms, scales, rotations) + float transformMatrix[16]; + transformMatrixParam->getRegion(0, 16, transformMatrix); + + if (_previewOutput) { + __android_log_print(ANDROID_LOG_INFO, TAG, "Rendering to Preview.."); + _previewOutput->renderTextureToSurface(_inputTextureId, transformMatrix); + } + if (_frameProcessorOutput) { + __android_log_print(ANDROID_LOG_INFO, TAG, "Rendering to FrameProcessor.."); + _frameProcessorOutput->renderTextureToSurface(_inputTextureId, transformMatrix); + } + if (_recordingSessionOutput) { + __android_log_print(ANDROID_LOG_INFO, TAG, "Rendering to RecordingSession.."); + _recordingSessionOutput->renderTextureToSurface(_inputTextureId, transformMatrix); + } +} + +void VideoPipeline::registerNatives() { + registerHybrid({ + makeNativeMethod("initHybrid", VideoPipeline::initHybrid), + makeNativeMethod("setFrameProcessorOutputSurface", VideoPipeline::setFrameProcessorOutputSurface), + makeNativeMethod("removeFrameProcessorOutputSurface", VideoPipeline::removeFrameProcessorOutputSurface), + makeNativeMethod("setRecordingSessionOutputSurface", VideoPipeline::setRecordingSessionOutputSurface), + makeNativeMethod("removeRecordingSessionOutputSurface", VideoPipeline::removeRecordingSessionOutputSurface), + makeNativeMethod("setPreviewOutputSurface", VideoPipeline::setPreviewOutputSurface), + makeNativeMethod("removePreviewOutputSurface", VideoPipeline::removePreviewOutputSurface), + makeNativeMethod("getInputTextureId", VideoPipeline::getInputTextureId), + makeNativeMethod("onBeforeFrame", VideoPipeline::onBeforeFrame), + makeNativeMethod("onFrame", VideoPipeline::onFrame), + }); +} + +} // namespace vision diff --git a/android/src/main/cpp/VideoPipeline.h b/android/src/main/cpp/VideoPipeline.h new file mode 100644 index 0000000..49467ce --- /dev/null +++ b/android/src/main/cpp/VideoPipeline.h @@ -0,0 +1,72 @@ +// +// Created by Marc Rousavy on 25.08.23. +// + +#pragma once + +#include +#include +#include +#include +#include "PassThroughShader.h" +#include "OpenGLRenderer.h" +#include "OpenGLContext.h" +#include + +namespace vision { + +#define NO_TEXTURE 0 + +using namespace facebook; + +class VideoPipeline: public jni::HybridClass { + public: + static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/utils/VideoPipeline;"; + static jni::local_ref initHybrid(jni::alias_ref jThis, int width, int height); + static void registerNatives(); + + public: + ~VideoPipeline(); + + // -> SurfaceTexture input + int getInputTextureId(); + + // <- Frame Processor output + void setFrameProcessorOutputSurface(jobject surface); + void removeFrameProcessorOutputSurface(); + + // <- MediaRecorder output + void setRecordingSessionOutputSurface(jobject surface); + void removeRecordingSessionOutputSurface(); + + // <- Preview output + void setPreviewOutputSurface(jobject surface); + void removePreviewOutputSurface(); + + // Frame callbacks + void onBeforeFrame(); + void onFrame(jni::alias_ref transformMatrix); + + private: + // Private constructor. Use `create(..)` to create new instances. + explicit VideoPipeline(jni::alias_ref jThis, int width, int height); + + private: + // Input Surface Texture + GLuint _inputTextureId = NO_TEXTURE; + int _width = 0; + int _height = 0; + + // Output Contexts + std::shared_ptr _context = nullptr; + std::unique_ptr _frameProcessorOutput = nullptr; + std::unique_ptr _recordingSessionOutput = nullptr; + std::unique_ptr _previewOutput = nullptr; + + private: + friend HybridBase; + jni::global_ref _javaPart; + static constexpr auto TAG = "VideoPipeline"; +}; + +} // namespace vision diff --git a/android/src/main/cpp/VisionCamera.cpp b/android/src/main/cpp/VisionCamera.cpp index 275df90..4ab9082 100644 --- a/android/src/main/cpp/VisionCamera.cpp +++ b/android/src/main/cpp/VisionCamera.cpp @@ -5,12 +5,14 @@ #include "JVisionCameraProxy.h" #include "VisionCameraProxy.h" #include "SkiaRenderer.h" +#include "VideoPipeline.h" JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) { return facebook::jni::initialize(vm, [] { vision::VisionCameraInstaller::registerNatives(); vision::JVisionCameraProxy::registerNatives(); vision::JVisionCameraScheduler::registerNatives(); + vision::VideoPipeline::registerNatives(); #if VISION_CAMERA_ENABLE_FRAME_PROCESSORS vision::JFrameProcessor::registerNatives(); #endif diff --git a/android/src/main/cpp/frameprocessor/FrameHostObject.cpp b/android/src/main/cpp/frameprocessor/FrameHostObject.cpp index b0bf850..c72913a 100644 --- a/android/src/main/cpp/frameprocessor/FrameHostObject.cpp +++ b/android/src/main/cpp/frameprocessor/FrameHostObject.cpp @@ -4,7 +4,6 @@ #include "FrameHostObject.h" -#include #include #include diff --git a/android/src/main/cpp/frameprocessor/FrameHostObject.h b/android/src/main/cpp/frameprocessor/FrameHostObject.h index 297ee24..09b076e 100644 --- a/android/src/main/cpp/frameprocessor/FrameHostObject.h +++ b/android/src/main/cpp/frameprocessor/FrameHostObject.h @@ -27,9 +27,6 @@ class JSI_EXPORT FrameHostObject : public jsi::HostObject { public: jni::global_ref frame; - - private: - static auto constexpr TAG = "VisionCamera"; }; } // namespace vision diff --git a/android/src/main/cpp/frameprocessor/JSIJNIConversion.cpp b/android/src/main/cpp/frameprocessor/JSIJNIConversion.cpp index 558b1b9..8cddc95 100644 --- a/android/src/main/cpp/frameprocessor/JSIJNIConversion.cpp +++ b/android/src/main/cpp/frameprocessor/JSIJNIConversion.cpp @@ -68,7 +68,6 @@ jni::local_ref> JSIJNIConversion::convertJSIObjectTo auto map = convertJSIObjectToJNIMap(runtime, valueAsObject); hashMap->put(key, map); - } } else { @@ -139,8 +138,7 @@ jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, c result.setProperty(runtime, key.c_str(), jsiValue); } return result; - - } if (object->isInstanceOf(JFrame::javaClassStatic())) { + } else if (object->isInstanceOf(JFrame::javaClassStatic())) { // Frame auto frame = static_ref_cast(object); diff --git a/android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraScheduler.cpp b/android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraScheduler.cpp index 7490643..1ef6136 100644 --- a/android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraScheduler.cpp +++ b/android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraScheduler.cpp @@ -22,8 +22,8 @@ void JVisionCameraScheduler::dispatchAsync(const std::function& job) { void JVisionCameraScheduler::scheduleTrigger() { // 2. schedule `triggerUI` to be called on the java thread - static auto method = javaPart_->getClass()->getMethod("scheduleTrigger"); - method(javaPart_.get()); + static auto method = _javaPart->getClass()->getMethod("scheduleTrigger"); + method(_javaPart.get()); } void JVisionCameraScheduler::trigger() { diff --git a/android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraScheduler.h b/android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraScheduler.h index 567815b..b9c6935 100644 --- a/android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraScheduler.h +++ b/android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraScheduler.h @@ -33,12 +33,12 @@ class JVisionCameraScheduler : public jni::HybridClass { private: friend HybridBase; - jni::global_ref javaPart_; + jni::global_ref _javaPart; std::queue> _jobs; std::mutex _mutex; explicit JVisionCameraScheduler(jni::alias_ref jThis): - javaPart_(jni::make_global(jThis)) {} + _javaPart(jni::make_global(jThis)) {} // Schedules a call to `trigger` on the VisionCamera FP Thread void scheduleTrigger(); diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index 8a58643..864acbd 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -23,7 +23,6 @@ import com.mrousavy.camera.extensions.createPhotoCaptureRequest import com.mrousavy.camera.extensions.openCamera import com.mrousavy.camera.extensions.tryClose import com.mrousavy.camera.extensions.zoomed -import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.FrameProcessor import com.mrousavy.camera.parsers.Flash import com.mrousavy.camera.parsers.Orientation @@ -88,8 +87,17 @@ class CameraSession(private val context: Context, private val mutex = Mutex() private var isRunning = false private var enableTorch = false + // Video Outputs private var recording: RecordingSession? = null - private var frameProcessor: FrameProcessor? = null + set(value) { + field = value + updateVideoOutputs() + } + var frameProcessor: FrameProcessor? = null + set(value) { + field = value + updateVideoOutputs() + } override val coroutineContext: CoroutineContext = CameraQueues.cameraQueue.coroutineDispatcher @@ -130,8 +138,14 @@ class CameraSession(private val context: Context, Log.i(TAG, "Nothing changed in configuration, canceling..") } - this.cameraId = cameraId + // 1. Close previous outputs + this.outputs?.close() + // 2. Assign new outputs this.outputs = outputs + // 3. Update with existing render targets (surfaces) + updateVideoOutputs() + + this.cameraId = cameraId launch { startRunning() } @@ -183,8 +197,12 @@ class CameraSession(private val context: Context, } } - fun setFrameProcessor(frameProcessor: FrameProcessor?) { - this.frameProcessor = frameProcessor + private fun updateVideoOutputs() { + val videoPipeline = outputs?.videoOutput?.videoPipeline ?: return + val previewOutput = outputs?.previewOutput + videoPipeline.setRecordingSessionOutput(this.recording) + videoPipeline.setFrameProcessorOutput(this.frameProcessor) + videoPipeline.setPreviewOutput(previewOutput?.surface) } suspend fun takePhoto(qualityPrioritization: QualityPrioritization, @@ -229,20 +247,6 @@ class CameraSession(private val context: Context, photoOutputSynchronizer.set(image.timestamp, image) } - override fun onVideoFrameCaptured(image: Image) { - // TODO: Correctly get orientation and everything - val frame = Frame(image, System.currentTimeMillis(), Orientation.PORTRAIT, false) - frame.incrementRefCount() - - // Call (Skia-) Frame Processor - frameProcessor?.call(frame) - - // Write Image to the Recording - recording?.appendImage(image) - - frame.decrementRefCount() - } - suspend fun startRecording(enableAudio: Boolean, codec: VideoCodec, fileType: VideoFileType, @@ -253,7 +257,7 @@ class CameraSession(private val context: Context, val outputs = outputs ?: throw CameraNotReadyError() val videoOutput = outputs.videoOutput ?: throw VideoNotEnabledError() - val recording = RecordingSession(context, enableAudio, videoOutput.size, fps, codec, orientation, fileType, callback, onError) + val recording = RecordingSession(context, videoOutput.size, enableAudio, fps, codec, orientation, fileType, callback, onError) recording.start() this.recording = recording } @@ -497,7 +501,8 @@ class CameraSession(private val context: Context, val captureRequest = camera.createCaptureRequest(template) outputs.previewOutput?.let { output -> Log.i(TAG, "Adding output surface ${output.outputType}..") - captureRequest.addTarget(output.surface) + // TODO: Add here again? + // captureRequest.addTarget(output.surface) } outputs.videoOutput?.let { output -> Log.i(TAG, "Adding output surface ${output.outputType}..") diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index 3d3c574..7f7c299 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -96,7 +96,7 @@ class CameraView(context: Context) : FrameLayout(context) { internal var frameProcessor: FrameProcessor? = null set(value) { field = value - cameraSession.setFrameProcessor(frameProcessor) + cameraSession.frameProcessor = frameProcessor } private val inputOrientation: Orientation diff --git a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt index bcdfdab..60d9f19 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt @@ -180,11 +180,6 @@ class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJ @ReactMethod fun requestCameraPermission(promise: Promise) { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) { - // API 21 and below always grants permission on app install - return promise.resolve(PermissionStatus.GRANTED.unionValue) - } - val activity = reactApplicationContext.currentActivity if (activity is PermissionAwareActivity) { val currentRequestCode = RequestCode++ @@ -205,11 +200,6 @@ class CameraViewModule(reactContext: ReactApplicationContext): ReactContextBaseJ @ReactMethod fun requestMicrophonePermission(promise: Promise) { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) { - // API 21 and below always grants permission on app install - return promise.resolve(PermissionStatus.GRANTED.unionValue) - } - val activity = reactApplicationContext.currentActivity if (activity is PermissionAwareActivity) { val currentRequestCode = RequestCode++ diff --git a/android/src/main/java/com/mrousavy/camera/extensions/CameraCharacteristics+getOutputSizes.kt b/android/src/main/java/com/mrousavy/camera/extensions/CameraCharacteristics+getOutputSizes.kt index 64523a3..c0db665 100644 --- a/android/src/main/java/com/mrousavy/camera/extensions/CameraCharacteristics+getOutputSizes.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/CameraCharacteristics+getOutputSizes.kt @@ -63,10 +63,6 @@ fun CameraCharacteristics.getVideoSizes(cameraId: String, format: Int): List { val config = this.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! val sizes = config.getOutputSizes(format) ?: emptyArray() - val highResSizes = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - config.getHighResolutionOutputSizes(format) - } else { - null - } ?: emptyArray() + val highResSizes = config.getHighResolutionOutputSizes(format) ?: emptyArray() return sizes.plus(highResSizes).toList() } diff --git a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt index aaccebc..957b5d3 100644 --- a/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt +++ b/android/src/main/java/com/mrousavy/camera/extensions/CameraDevice+createCaptureSession.kt @@ -8,7 +8,6 @@ import android.hardware.camera2.params.OutputConfiguration import android.hardware.camera2.params.SessionConfiguration import android.os.Build import android.util.Log -import android.view.Surface import androidx.annotation.RequiresApi import com.mrousavy.camera.CameraQueues import com.mrousavy.camera.CameraSessionCannotBeConfiguredError @@ -63,47 +62,35 @@ suspend fun CameraDevice.createCaptureSession(cameraManager: CameraManager, } } - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { - // API >= 24 - val outputConfigurations = arrayListOf() - outputs.previewOutput?.let { output -> - outputConfigurations.add(output.toOutputConfiguration(characteristics)) - } - outputs.photoOutput?.let { output -> - outputConfigurations.add(output.toOutputConfiguration(characteristics)) - } - outputs.videoOutput?.let { output -> - outputConfigurations.add(output.toOutputConfiguration(characteristics)) - } - if (outputs.enableHdr == true && Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { - val supportedProfiles = characteristics.get(CameraCharacteristics.REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES) - val hdrProfile = supportedProfiles?.bestProfile ?: supportedProfiles?.supportedProfiles?.firstOrNull() - if (hdrProfile != null) { - Log.i(TAG, "Camera $id: Using HDR Profile $hdrProfile...") - outputConfigurations.forEach { it.dynamicRangeProfile = hdrProfile } - } else { - Log.w(TAG, "Camera $id: HDR was enabled, but the device does not support any matching HDR profile!") - } - } - - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { - // API >=28 - Log.i(TAG, "Using new API (>=28)") - val config = SessionConfiguration(sessionType.toSessionType(), outputConfigurations, queue.executor, callback) - this.createCaptureSession(config) + val outputConfigurations = arrayListOf() + outputs.previewOutput?.let { output -> + // TODO: add here again? + // outputConfigurations.add(output.toOutputConfiguration(characteristics)) + } + outputs.photoOutput?.let { output -> + outputConfigurations.add(output.toOutputConfiguration(characteristics)) + } + outputs.videoOutput?.let { output -> + outputConfigurations.add(output.toOutputConfiguration(characteristics)) + } + if (outputs.enableHdr == true && Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + val supportedProfiles = characteristics.get(CameraCharacteristics.REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES) + val hdrProfile = supportedProfiles?.bestProfile ?: supportedProfiles?.supportedProfiles?.firstOrNull() + if (hdrProfile != null) { + Log.i(TAG, "Camera $id: Using HDR Profile $hdrProfile...") + outputConfigurations.forEach { it.dynamicRangeProfile = hdrProfile } } else { - // API >=24 - Log.i(TAG, "Using legacy API (<28)") - this.createCaptureSessionByOutputConfigurations(outputConfigurations, callback, queue.handler) + Log.w(TAG, "Camera $id: HDR was enabled, but the device does not support any matching HDR profile!") } + } + + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + Log.i(TAG, "Using new API (>=28)") + val config = SessionConfiguration(sessionType.toSessionType(), outputConfigurations, queue.executor, callback) + this.createCaptureSession(config) } else { - // API <24 - Log.i(TAG, "Using legacy API (<24)") - val surfaces = arrayListOf() - outputs.previewOutput?.let { surfaces.add(it.surface) } - outputs.photoOutput?.let { surfaces.add(it.surface) } - outputs.videoOutput?.let { surfaces.add(it.surface) } - this.createCaptureSession(surfaces, callback, queue.handler) + Log.i(TAG, "Using legacy API (<28)") + this.createCaptureSessionByOutputConfigurations(outputConfigurations, callback, queue.handler) } } } diff --git a/android/src/main/java/com/mrousavy/camera/utils/RecordingSession.kt b/android/src/main/java/com/mrousavy/camera/utils/RecordingSession.kt index ccbb170..717138c 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/RecordingSession.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/RecordingSession.kt @@ -1,7 +1,6 @@ package com.mrousavy.camera.utils import android.content.Context -import android.media.Image import android.media.ImageWriter import android.media.MediaCodec import android.media.MediaRecorder @@ -13,12 +12,11 @@ import com.mrousavy.camera.RecorderError import com.mrousavy.camera.parsers.Orientation import com.mrousavy.camera.parsers.VideoCodec import com.mrousavy.camera.parsers.VideoFileType -import com.mrousavy.camera.utils.outputs.CameraOutputs import java.io.File class RecordingSession(context: Context, + val size: Size, private val enableAudio: Boolean, - private val videoSize: Size, private val fps: Int? = null, private val codec: VideoCodec = VideoCodec.H264, private val orientation: Orientation, @@ -40,14 +38,9 @@ class RecordingSession(context: Context, private val outputFile: File private var startTime: Long? = null private var imageWriter: ImageWriter? = null - val surface: Surface + val surface: Surface = MediaCodec.createPersistentInputSurface() init { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) { - throw Error("Video Recording is only supported on Devices running Android version 23 (M) or newer.") - } - - surface = MediaCodec.createPersistentInputSurface() outputFile = File.createTempFile("mrousavy", fileType.toExtension(), context.cacheDir) @@ -61,7 +54,7 @@ class RecordingSession(context: Context, recorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4) recorder.setOutputFile(outputFile.absolutePath) recorder.setVideoEncodingBitRate(VIDEO_BIT_RATE) - recorder.setVideoSize(videoSize.width, videoSize.height) + recorder.setVideoSize(size.height, size.width) if (fps != null) recorder.setVideoFrameRate(fps) Log.i(TAG, "Using $codec Video Codec..") @@ -74,7 +67,7 @@ class RecordingSession(context: Context, recorder.setAudioChannels(AUDIO_CHANNELS) } recorder.setInputSurface(surface) - recorder.setOrientationHint(orientation.toDegrees()) + //recorder.setOrientationHint(orientation.toDegrees()) recorder.setOnErrorListener { _, what, extra -> Log.e(TAG, "MediaRecorder Error: $what ($extra)") @@ -109,10 +102,8 @@ class RecordingSession(context: Context, recorder.stop() recorder.release() - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { - imageWriter?.close() - imageWriter = null - } + imageWriter?.close() + imageWriter = null } catch (e: Error) { Log.e(TAG, "Failed to stop MediaRecorder!", e) } @@ -125,9 +116,6 @@ class RecordingSession(context: Context, fun pause() { synchronized(this) { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) { - throw Error("Pausing a recording is only supported on Devices running Android version 24 (N) or newer.") - } Log.i(TAG, "Pausing Recording Session..") recorder.pause() } @@ -135,32 +123,13 @@ class RecordingSession(context: Context, fun resume() { synchronized(this) { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N) { - throw Error("Resuming a recording is only supported on Devices running Android version 24 (N) or newer.") - } Log.i(TAG, "Resuming Recording Session..") recorder.resume() } } - fun appendImage(image: Image) { - synchronized(this) { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.M) { - throw Error("Video Recording is only supported on Devices running Android version 23 (M) or newer.") - } - - // TODO: Correctly mirror/flip Image in OpenGL pipeline, otherwise flipping camera while recording results in inverted frames - - if (imageWriter == null) { - imageWriter = ImageWriter.newInstance(surface, CameraOutputs.VIDEO_OUTPUT_BUFFER_SIZE) - } - image.timestamp = System.nanoTime() - imageWriter!!.queueInputImage(image) - } - } - override fun toString(): String { val audio = if (enableAudio) "with audio" else "without audio" - return "${videoSize.width} x ${videoSize.height} @ $fps FPS $codec $fileType $orientation RecordingSession ($audio)" + return "${size.width} x ${size.height} @ $fps FPS $codec $fileType $orientation RecordingSession ($audio)" } } diff --git a/android/src/main/java/com/mrousavy/camera/utils/VideoPipeline.kt b/android/src/main/java/com/mrousavy/camera/utils/VideoPipeline.kt new file mode 100644 index 0000000..8d3cab3 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/utils/VideoPipeline.kt @@ -0,0 +1,185 @@ +package com.mrousavy.camera.utils + +import android.graphics.ImageFormat +import android.graphics.SurfaceTexture +import android.media.ImageReader +import android.media.ImageWriter +import android.media.MediaRecorder +import android.util.Log +import android.view.Surface +import com.facebook.jni.HybridData +import com.mrousavy.camera.frameprocessor.Frame +import com.mrousavy.camera.frameprocessor.FrameProcessor +import com.mrousavy.camera.parsers.Orientation +import java.io.Closeable + +/** + * An OpenGL pipeline for streaming Camera Frames to one or more outputs. + * Currently, [VideoPipeline] can stream to a [FrameProcessor] and a [MediaRecorder]. + * + * @param [width] The width of the Frames to stream (> 0) + * @param [height] The height of the Frames to stream (> 0) + * @param [format] The format of the Frames to stream. ([ImageFormat.PRIVATE], [ImageFormat.YUV_420_888] or [ImageFormat.JPEG]) + */ +@Suppress("KotlinJniMissingFunction") +class VideoPipeline(val width: Int, + val height: Int, + val format: Int = ImageFormat.PRIVATE): SurfaceTexture.OnFrameAvailableListener, Closeable { + companion object { + private const val MAX_IMAGES = 5 + private const val TAG = "VideoPipeline" + } + + private val mHybridData: HybridData + private var openGLTextureId: Int? = null + private var transformMatrix = FloatArray(16) + private var isActive = true + + // Output 1 + private var frameProcessor: FrameProcessor? = null + private var imageReader: ImageReader? = null + + // Output 2 + private var recordingSession: RecordingSession? = null + + // Output 3 + private var previewSurface: Surface? = null + + // Input + private val surfaceTexture: SurfaceTexture + val surface: Surface + + init { + mHybridData = initHybrid(width, height) + surfaceTexture = SurfaceTexture(false) + surfaceTexture.setDefaultBufferSize(width, height) + surfaceTexture.setOnFrameAvailableListener(this) + surface = Surface(surfaceTexture) + } + + override fun close() { + synchronized(this) { + isActive = false + imageReader?.close() + imageReader = null + frameProcessor = null + recordingSession = null + surfaceTexture.release() + mHybridData.resetNative() + } + } + + override fun onFrameAvailable(surfaceTexture: SurfaceTexture) { + synchronized(this) { + if (!isActive) return@synchronized + + // 1. Attach Surface to OpenGL context + if (openGLTextureId == null) { + openGLTextureId = getInputTextureId() + surfaceTexture.attachToGLContext(openGLTextureId!!) + Log.i(TAG, "Attached Texture to Context $openGLTextureId") + } + + // 2. Prepare the OpenGL context (eglMakeCurrent) + onBeforeFrame() + + // 3. Update the OpenGL texture + surfaceTexture.updateTexImage() + + // 4. Get the transform matrix from the SurfaceTexture (rotations/scales applied by Camera) + surfaceTexture.getTransformMatrix(transformMatrix) + + // 5. Draw it with applied rotation/mirroring + onFrame(transformMatrix) + } + } + + private fun getImageReader(): ImageReader { + val imageReader = ImageReader.newInstance(width, height, format, MAX_IMAGES) + imageReader.setOnImageAvailableListener({ reader -> + Log.i("VideoPipeline", "ImageReader::onImageAvailable!") + val image = reader.acquireLatestImage() ?: return@setOnImageAvailableListener + + // TODO: Get correct orientation and isMirrored + val frame = Frame(image, image.timestamp, Orientation.PORTRAIT, false) + frame.incrementRefCount() + frameProcessor?.call(frame) + frame.decrementRefCount() + }, null) + return imageReader + } + + /** + * Configures the Pipeline to also call the given [FrameProcessor]. + * * If the [frameProcessor] is `null`, this output channel will be removed. + * * If the [frameProcessor] is not `null`, the [VideoPipeline] will create Frames + * using an [ImageWriter] and call the [FrameProcessor] with those Frames. + */ + fun setFrameProcessorOutput(frameProcessor: FrameProcessor?) { + synchronized(this) { + Log.i(TAG, "Setting $width x $height FrameProcessor Output...") + this.frameProcessor = frameProcessor + + if (frameProcessor != null) { + if (this.imageReader == null) { + // 1. Create new ImageReader that just calls the Frame Processor + this.imageReader = getImageReader() + } + + // 2. Configure OpenGL pipeline to stream Frames into the ImageReader's surface + setFrameProcessorOutputSurface(imageReader!!.surface) + } else { + // 1. Configure OpenGL pipeline to stop streaming Frames into the ImageReader's surface + removeFrameProcessorOutputSurface() + + // 2. Close the ImageReader + this.imageReader?.close() + this.imageReader = null + } + } + } + + /** + * Configures the Pipeline to also write Frames to a Surface from a [MediaRecorder]. + * * If the [surface] is `null`, this output channel will be removed. + * * If the [surface] is not `null`, the [VideoPipeline] will write Frames to this Surface. + */ + fun setRecordingSessionOutput(recordingSession: RecordingSession?) { + synchronized(this) { + Log.i(TAG, "Setting $width x $height RecordingSession Output...") + if (recordingSession != null) { + // Configure OpenGL pipeline to stream Frames into the Recording Session's surface + setRecordingSessionOutputSurface(recordingSession.surface) + this.recordingSession = recordingSession + } else { + // Configure OpenGL pipeline to stop streaming Frames into the Recording Session's surface + removeRecordingSessionOutputSurface() + this.recordingSession = null + } + } + } + + fun setPreviewOutput(surface: Surface?) { + synchronized(this) { + Log.i(TAG, "Setting Preview Output...") + if (surface != null) { + setPreviewOutputSurface(surface) + this.previewSurface = surface + } else { + removePreviewOutputSurface() + this.previewSurface = null + } + } + } + + private external fun getInputTextureId(): Int + private external fun onBeforeFrame() + private external fun onFrame(transformMatrix: FloatArray) + private external fun setFrameProcessorOutputSurface(surface: Any) + private external fun removeFrameProcessorOutputSurface() + private external fun setRecordingSessionOutputSurface(surface: Any) + private external fun removeRecordingSessionOutputSurface() + private external fun setPreviewOutputSurface(surface: Any) + private external fun removePreviewOutputSurface() + private external fun initHybrid(width: Int, height: Int): HybridData +} diff --git a/android/src/main/java/com/mrousavy/camera/utils/outputs/CameraOutputs.kt b/android/src/main/java/com/mrousavy/camera/utils/outputs/CameraOutputs.kt index dfe1dd6..9cdf4c6 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/outputs/CameraOutputs.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/outputs/CameraOutputs.kt @@ -1,11 +1,9 @@ package com.mrousavy.camera.utils.outputs import android.graphics.ImageFormat -import android.hardware.HardwareBuffer import android.hardware.camera2.CameraManager import android.media.Image import android.media.ImageReader -import android.os.Build import android.util.Log import android.util.Size import android.view.Surface @@ -14,6 +12,7 @@ import com.mrousavy.camera.extensions.closestToOrMax import com.mrousavy.camera.extensions.getPhotoSizes import com.mrousavy.camera.extensions.getPreviewSize import com.mrousavy.camera.extensions.getVideoSizes +import com.mrousavy.camera.utils.VideoPipeline import java.io.Closeable class CameraOutputs(val cameraId: String, @@ -25,7 +24,6 @@ class CameraOutputs(val cameraId: String, val callback: Callback): Closeable { companion object { private const val TAG = "CameraOutputs" - const val VIDEO_OUTPUT_BUFFER_SIZE = 3 const val PHOTO_OUTPUT_BUFFER_SIZE = 3 } @@ -39,14 +37,13 @@ class CameraOutputs(val cameraId: String, interface Callback { fun onPhotoCaptured(image: Image) - fun onVideoFrameCaptured(image: Image) } var previewOutput: SurfaceOutput? = null private set var photoOutput: ImageReaderOutput? = null private set - var videoOutput: SurfaceOutput? = null + var videoOutput: VideoPipelineOutput? = null private set val size: Int @@ -118,23 +115,11 @@ class CameraOutputs(val cameraId: String, // Video output: High resolution repeating images (startRecording() or useFrameProcessor()) if (video != null) { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) throw Error("Video Recordings and/or Frame Processors are only available on API 29 and above!") - val size = characteristics.getVideoSizes(cameraId, video.format).closestToOrMax(video.targetSize) - - val flags = HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE or HardwareBuffer.USAGE_VIDEO_ENCODE - val imageReader = ImageReader.newInstance(size.width, size.height, video.format, VIDEO_OUTPUT_BUFFER_SIZE, flags) - imageReader.setOnImageAvailableListener({ reader -> - try { - val image = reader.acquireNextImage() ?: return@setOnImageAvailableListener - callback.onVideoFrameCaptured(image) - } catch (e: IllegalStateException) { - Log.e(TAG, "Failed to acquire a new Image, dropping a Frame.. The Frame Processor cannot keep up with the Camera's FPS!", e) - } - }, CameraQueues.videoQueue.handler) + val videoPipeline = VideoPipeline(size.width, size.height, video.format) Log.i(TAG, "Adding ${size.width}x${size.height} video output. (Format: ${video.format})") - videoOutput = ImageReaderOutput(imageReader, SurfaceOutput.OutputType.VIDEO) + videoOutput = VideoPipelineOutput(videoPipeline, SurfaceOutput.OutputType.VIDEO) } Log.i(TAG, "Prepared $size Outputs for Camera $cameraId!") diff --git a/android/src/main/java/com/mrousavy/camera/utils/outputs/SurfaceOutput.kt b/android/src/main/java/com/mrousavy/camera/utils/outputs/SurfaceOutput.kt index 593a55e..55cefee 100644 --- a/android/src/main/java/com/mrousavy/camera/utils/outputs/SurfaceOutput.kt +++ b/android/src/main/java/com/mrousavy/camera/utils/outputs/SurfaceOutput.kt @@ -35,7 +35,6 @@ open class SurfaceOutput(val surface: Surface, } } - @RequiresApi(Build.VERSION_CODES.N) fun toOutputConfiguration(characteristics: CameraCharacteristics): OutputConfiguration { val result = OutputConfiguration(surface) if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { diff --git a/android/src/main/java/com/mrousavy/camera/utils/outputs/VideoPipelineOutput.kt b/android/src/main/java/com/mrousavy/camera/utils/outputs/VideoPipelineOutput.kt new file mode 100644 index 0000000..6b590e9 --- /dev/null +++ b/android/src/main/java/com/mrousavy/camera/utils/outputs/VideoPipelineOutput.kt @@ -0,0 +1,22 @@ +package com.mrousavy.camera.utils.outputs + +import android.util.Log +import android.util.Size +import com.mrousavy.camera.utils.VideoPipeline +import java.io.Closeable + +/** + * A [SurfaceOutput] that uses a [VideoPipeline] as it's surface. + */ +class VideoPipelineOutput(val videoPipeline: VideoPipeline, + outputType: OutputType, + dynamicRangeProfile: Long? = null): Closeable, SurfaceOutput(videoPipeline.surface, Size(videoPipeline.width, videoPipeline.height), outputType, dynamicRangeProfile) { + override fun close() { + Log.i(TAG, "Closing ${videoPipeline.width}x${videoPipeline.height} Video Pipeline..") + videoPipeline.close() + } + + override fun toString(): String { + return "$outputType (${videoPipeline.width} x ${videoPipeline.height} in format #${videoPipeline.format})" + } +} diff --git a/docs/docs/guides/TROUBLESHOOTING.mdx b/docs/docs/guides/TROUBLESHOOTING.mdx index 802ef70..9515b3f 100644 --- a/docs/docs/guides/TROUBLESHOOTING.mdx +++ b/docs/docs/guides/TROUBLESHOOTING.mdx @@ -75,7 +75,7 @@ Before opening an issue, make sure you try the following: 2. Set `buildToolsVersion` to `33.0.0` or higher 3. Set `compileSdkVersion` to `33` or higher 4. Set `targetSdkVersion` to `33` or higher - 5. Set `minSdkVersion` to `21` or higher + 5. Set `minSdkVersion` to `26` or higher 6. Set `ndkVersion` to `"23.1.7779620"` or higher 7. Update the Gradle Build-Tools version to `7.3.1` or higher: ``` diff --git a/example/android/build.gradle b/example/android/build.gradle index e74c245..9babb79 100644 --- a/example/android/build.gradle +++ b/example/android/build.gradle @@ -3,7 +3,7 @@ buildscript { ext { buildToolsVersion = "33.0.0" - minSdkVersion = 21 + minSdkVersion = 26 compileSdkVersion = 33 targetSdkVersion = 33 ndkVersion = "23.1.7779620"