9add0eb571
1. Reverts 4e96eb77e0 (PR #1789) to bring the C++ OpenGL GPU Pipeline back.
2. Fixes the "initHybrid JNI not found" error by loading the native JNI/C++ library in `VideoPipeline.kt`.
This PR has two downsides:
1. `pixelFormat="yuv"` does not work on Android. OpenGL only works in RGB
2. OpenGL rendering is fast, but it has an overhead. I think for Camera -> Video Recording we shouldn't be using an entire OpenGL rendering pipeline.
The original plan was to use something similar to how it works on iOS by just passing GPU buffers around, but the android.media APIs just aren't as advanced yet. `ImageReader`/`ImageWriter` is way too buggy and doesn't really work with `MediaRecorder`/`MediaCodec`.
This sucks, I hope in the future we can use something like `AHardwareBuffer`s.
120 lines
4.0 KiB
C++
120 lines
4.0 KiB
C++
//
|
|
// Created by Marc Rousavy on 25.08.23.
|
|
//
|
|
|
|
#include "VideoPipeline.h"
|
|
#include "OpenGLError.h"
|
|
|
|
#include <EGL/egl.h>
|
|
#include <EGL/eglext.h>
|
|
#include <GLES/gl.h>
|
|
#include <GLES2/gl2.h>
|
|
#include <GLES2/gl2ext.h>
|
|
#include <android/native_window_jni.h>
|
|
|
|
#include <chrono>
|
|
|
|
#include "JFrameProcessor.h"
|
|
#include "OpenGLTexture.h"
|
|
|
|
namespace vision {
|
|
|
|
jni::local_ref<VideoPipeline::jhybriddata> VideoPipeline::initHybrid(jni::alias_ref<jhybridobject> jThis, int width, int height) {
|
|
return makeCxxInstance(jThis, width, height);
|
|
}
|
|
|
|
VideoPipeline::VideoPipeline(jni::alias_ref<jhybridobject> jThis, int width, int height) : _javaPart(jni::make_global(jThis)) {
|
|
_width = width;
|
|
_height = height;
|
|
_context = OpenGLContext::CreateWithOffscreenSurface();
|
|
}
|
|
|
|
VideoPipeline::~VideoPipeline() {
|
|
// 1. Remove output surfaces
|
|
removeFrameProcessorOutputSurface();
|
|
removeRecordingSessionOutputSurface();
|
|
// 2. Delete the input textures
|
|
if (_inputTexture != std::nullopt) {
|
|
glDeleteTextures(1, &_inputTexture->id);
|
|
_inputTexture = std::nullopt;
|
|
}
|
|
// 3. Destroy the OpenGL context
|
|
_context = nullptr;
|
|
}
|
|
|
|
void VideoPipeline::removeFrameProcessorOutputSurface() {
|
|
if (_frameProcessorOutput)
|
|
_frameProcessorOutput->destroy();
|
|
_frameProcessorOutput = nullptr;
|
|
}
|
|
|
|
void VideoPipeline::setFrameProcessorOutputSurface(jobject surface) {
|
|
// 1. Delete existing output surface
|
|
removeFrameProcessorOutputSurface();
|
|
|
|
// 2. Set new output surface if it is not null
|
|
ANativeWindow* window = ANativeWindow_fromSurface(jni::Environment::current(), surface);
|
|
_frameProcessorOutput = OpenGLRenderer::CreateWithWindowSurface(_context, window);
|
|
}
|
|
|
|
void VideoPipeline::removeRecordingSessionOutputSurface() {
|
|
if (_recordingSessionOutput)
|
|
_recordingSessionOutput->destroy();
|
|
_recordingSessionOutput = nullptr;
|
|
}
|
|
|
|
void VideoPipeline::setRecordingSessionOutputSurface(jobject surface) {
|
|
// 1. Delete existing output surface
|
|
removeRecordingSessionOutputSurface();
|
|
|
|
// 2. Set new output surface if it is not null
|
|
ANativeWindow* window = ANativeWindow_fromSurface(jni::Environment::current(), surface);
|
|
_recordingSessionOutput = OpenGLRenderer::CreateWithWindowSurface(_context, window);
|
|
}
|
|
|
|
int VideoPipeline::getInputTextureId() {
|
|
if (_inputTexture == std::nullopt) {
|
|
_inputTexture = _context->createTexture(OpenGLTexture::Type::ExternalOES, _width, _height);
|
|
}
|
|
|
|
return static_cast<int>(_inputTexture->id);
|
|
}
|
|
|
|
void VideoPipeline::onBeforeFrame() {
|
|
_context->use();
|
|
|
|
glBindTexture(_inputTexture->target, _inputTexture->id);
|
|
}
|
|
|
|
void VideoPipeline::onFrame(jni::alias_ref<jni::JArrayFloat> transformMatrixParam) {
|
|
// Get the OpenGL transform Matrix (transforms, scales, rotations)
|
|
float transformMatrix[16];
|
|
transformMatrixParam->getRegion(0, 16, transformMatrix);
|
|
|
|
OpenGLTexture& texture = _inputTexture.value();
|
|
|
|
if (_frameProcessorOutput) {
|
|
__android_log_print(ANDROID_LOG_INFO, TAG, "Rendering to FrameProcessor..");
|
|
_frameProcessorOutput->renderTextureToSurface(texture, transformMatrix);
|
|
}
|
|
if (_recordingSessionOutput) {
|
|
__android_log_print(ANDROID_LOG_INFO, TAG, "Rendering to RecordingSession..");
|
|
_recordingSessionOutput->renderTextureToSurface(texture, transformMatrix);
|
|
}
|
|
}
|
|
|
|
void VideoPipeline::registerNatives() {
|
|
registerHybrid({
|
|
makeNativeMethod("initHybrid", VideoPipeline::initHybrid),
|
|
makeNativeMethod("setFrameProcessorOutputSurface", VideoPipeline::setFrameProcessorOutputSurface),
|
|
makeNativeMethod("removeFrameProcessorOutputSurface", VideoPipeline::removeFrameProcessorOutputSurface),
|
|
makeNativeMethod("setRecordingSessionOutputSurface", VideoPipeline::setRecordingSessionOutputSurface),
|
|
makeNativeMethod("removeRecordingSessionOutputSurface", VideoPipeline::removeRecordingSessionOutputSurface),
|
|
makeNativeMethod("getInputTextureId", VideoPipeline::getInputTextureId),
|
|
makeNativeMethod("onBeforeFrame", VideoPipeline::onBeforeFrame),
|
|
makeNativeMethod("onFrame", VideoPipeline::onFrame),
|
|
});
|
|
}
|
|
|
|
} // namespace vision
|