railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

feat: Create C++/OpenGL-based Video Pipeline for more efficient Recording and Frame Processing (#1721)

Browse Source 
* Create `VideoPipeline` c++

* Remove folly C++ dependency

* Create `VideoPipeline` HybridClass

* Set up OpenGL

* Add outputs

* Update VideoPipeline.kt

* Bum `minSdkVersion` to `26`

* Create `VideoPipelineOutput`

* Create output funcs

* Set output pipelines

* Add FP/Recording on Output change

* Update VideoPipeline.cpp

* Create `PassThroughShader`

* Try to draw? I have honestly no idea

* fix: Fix `setFrameProcessor` nameclash

* fix: Fix `high-res-sizes` being null

* Add preview output

* Create `OpenGLContext.cpp`

* Make screen red

* This _should_ work (MESSY)

* FINALLY RENDER TEXTURE

* Rotate

* Mirror

* Clean up a bit

* Add `getWidth()`/`getHeight()`

* Cleanup

* fix: Use uniforms instead of attributes

* Draw with passed rotation/mirror mode

* feat: Use SurfaceTexture's transformMatrix in OpenGL pipeline (#1727)

* feat: Use Transform Matrix from SurfaceTexture

* Renam

* feat: Fix OpenGL Shader

* Update VideoPipeline.kt

* Measure elapsed time

* fix: Fix low resolution

* Render to offscreen

* Render to every context

* Release `SurfaceTexture` on close

* Use one OpenGL context to render to multiple EGLSurfaces

* Clean up a bit

* fix: Fix recording pipeline not triggering

* fix: Synchronize close to prevent nulls

* Update OpenGLRenderer.cpp

* fix: Hardcode Android recorder size
This commit is contained in:
Marc Rousavy
2023-08-29 17:52:03 +02:00
committed by GitHub
parent dfb86e174b
commit ea3686cb9a
30 changed files with 1024 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
android/src/main/cpp/frameprocessor/FrameHostObject.cpp
View File

@@ -4,7 +4,6 @@
#include "FrameHostObject.h"
#include <android/log.h>
#include <fbjni/fbjni.h>
#include <jni.h>

3
android/src/main/cpp/frameprocessor/FrameHostObject.h
View File

@@ -27,9 +27,6 @@ class JSI_EXPORT FrameHostObject : public jsi::HostObject {
public:
jni::global_ref<JFrame> frame;
private:
static auto constexpr TAG = "VisionCamera";
};
} // namespace vision

4
android/src/main/cpp/frameprocessor/JSIJNIConversion.cpp
View File

@@ -68,7 +68,6 @@ jni::local_ref<jni::JMap<jstring, jobject>> JSIJNIConversion::convertJSIObjectTo
auto map = convertJSIObjectToJNIMap(runtime, valueAsObject);
hashMap->put(key, map);
}
} else {
@@ -139,8 +138,7 @@ jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, c
result.setProperty(runtime, key.c_str(), jsiValue);
}
return result;
} if (object->isInstanceOf(JFrame::javaClassStatic())) {
} else if (object->isInstanceOf(JFrame::javaClassStatic())) {
// Frame
auto frame = static_ref_cast<JFrame>(object);

4
android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraScheduler.cpp
View File

@@ -22,8 +22,8 @@ void JVisionCameraScheduler::dispatchAsync(const std::function<void()>& job) {
void JVisionCameraScheduler::scheduleTrigger() {
// 2. schedule `triggerUI` to be called on the java thread
static auto method = javaPart_->getClass()->getMethod<void()>("scheduleTrigger");
method(javaPart_.get());
static auto method = _javaPart->getClass()->getMethod<void()>("scheduleTrigger");
method(_javaPart.get());
}
void JVisionCameraScheduler::trigger() {

4
android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraScheduler.h
View File

@@ -33,12 +33,12 @@ class JVisionCameraScheduler : public jni::HybridClass<JVisionCameraScheduler> {
private:
friend HybridBase;
jni::global_ref<JVisionCameraScheduler::javaobject> javaPart_;
jni::global_ref<JVisionCameraScheduler::javaobject> _javaPart;
std::queue<std::function<void()>> _jobs;
std::mutex _mutex;
explicit JVisionCameraScheduler(jni::alias_ref<JVisionCameraScheduler::jhybridobject> jThis):
javaPart_(jni::make_global(jThis)) {}
_javaPart(jni::make_global(jThis)) {}
// Schedules a call to `trigger` on the VisionCamera FP Thread
void scheduleTrigger();