feat: Use C++ OpenGL GPU VideoPipeline again (#1836)
1. Reverts 4e96eb77e0 (PR #1789) to bring the C++ OpenGL GPU Pipeline back.
2. Fixes the "initHybrid JNI not found" error by loading the native JNI/C++ library in `VideoPipeline.kt`.
This PR has two downsides:
1. `pixelFormat="yuv"` does not work on Android. OpenGL only works in RGB
2. OpenGL rendering is fast, but it has an overhead. I think for Camera -> Video Recording we shouldn't be using an entire OpenGL rendering pipeline.
The original plan was to use something similar to how it works on iOS by just passing GPU buffers around, but the android.media APIs just aren't as advanced yet. `ImageReader`/`ImageWriter` is way too buggy and doesn't really work with `MediaRecorder`/`MediaCodec`.
This sucks, I hope in the future we can use something like `AHardwareBuffer`s.
This commit is contained in:
66
package/android/src/main/cpp/VideoPipeline.h
Normal file
66
package/android/src/main/cpp/VideoPipeline.h
Normal file
@@ -0,0 +1,66 @@
|
||||
//
|
||||
// Created by Marc Rousavy on 25.08.23.
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "OpenGLContext.h"
|
||||
#include "OpenGLRenderer.h"
|
||||
#include "PassThroughShader.h"
|
||||
#include <EGL/egl.h>
|
||||
#include <android/native_window.h>
|
||||
#include <fbjni/fbjni.h>
|
||||
#include <jni.h>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
|
||||
namespace vision {
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
class VideoPipeline : public jni::HybridClass<VideoPipeline> {
|
||||
public:
|
||||
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/core/VideoPipeline;";
|
||||
static jni::local_ref<jhybriddata> initHybrid(jni::alias_ref<jhybridobject> jThis, int width, int height);
|
||||
static void registerNatives();
|
||||
|
||||
public:
|
||||
~VideoPipeline();
|
||||
|
||||
// -> SurfaceTexture input
|
||||
int getInputTextureId();
|
||||
|
||||
// <- Frame Processor output
|
||||
void setFrameProcessorOutputSurface(jobject surface);
|
||||
void removeFrameProcessorOutputSurface();
|
||||
|
||||
// <- MediaRecorder output
|
||||
void setRecordingSessionOutputSurface(jobject surface);
|
||||
void removeRecordingSessionOutputSurface();
|
||||
|
||||
// Frame callbacks
|
||||
void onBeforeFrame();
|
||||
void onFrame(jni::alias_ref<jni::JArrayFloat> transformMatrix);
|
||||
|
||||
private:
|
||||
// Private constructor. Use `create(..)` to create new instances.
|
||||
explicit VideoPipeline(jni::alias_ref<jhybridobject> jThis, int width, int height);
|
||||
|
||||
private:
|
||||
// Input Surface Texture
|
||||
std::optional<OpenGLTexture> _inputTexture = std::nullopt;
|
||||
int _width = 0;
|
||||
int _height = 0;
|
||||
|
||||
// Output Contexts
|
||||
std::shared_ptr<OpenGLContext> _context = nullptr;
|
||||
std::unique_ptr<OpenGLRenderer> _frameProcessorOutput = nullptr;
|
||||
std::unique_ptr<OpenGLRenderer> _recordingSessionOutput = nullptr;
|
||||
|
||||
private:
|
||||
friend HybridBase;
|
||||
jni::global_ref<javaobject> _javaPart;
|
||||
static constexpr auto TAG = "VideoPipeline";
|
||||
};
|
||||
|
||||
} // namespace vision
|
||||
Reference in New Issue
Block a user