railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Use AHardwareBuffer* for frame.toArrayBuffer() (#1888)

Browse Source 
* feat: Route images through `ImageWriter` into OpenGL pipeline

* fix: Use RGB format

* fix: Every device supports YUV, RGB and NATIVE

* Update VideoPipeline.kt

* log format

* Plug ImageReader between OpenGL pipeline

* Call Frame Processor

* Format

* Remove logs

* feat: Use `HardwareBuffer` for `toArrayBuffer()`

* Format
This commit is contained in:
Marc Rousavy
2023-09-29 21:54:04 +02:00
committed by GitHub
parent 954b44810b
commit cf1952d8f7
5 changed files with 47 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
package/android/src/main/cpp/frameprocessor/FrameHostObject.cpp
View File

@@ -12,6 +12,9 @@
#include <string>
#include <vector>
#include <android/hardware_buffer.h>
#include <android/hardware_buffer_jni.h>
namespace vision {
using namespace facebook;
@@ -82,11 +85,13 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
if (name == "toArrayBuffer") {
jsi::HostFunctionType toArrayBuffer = [=](jsi::Runtime& runtime, const jsi::Value& thisArg, const jsi::Value* args,
size_t count) -> jsi::Value {
auto buffer = this->frame->toByteBuffer();
if (!buffer->isDirect()) {
throw std::runtime_error("Failed to get byte content of Frame - array is not direct ByteBuffer!");
}
auto size = buffer->getDirectSize();
AHardwareBuffer* hardwareBuffer = this->frame->getHardwareBuffer();
AHardwareBuffer_Desc bufferDescription;
AHardwareBuffer_describe(hardwareBuffer, &bufferDescription);
__android_log_print(ANDROID_LOG_INFO, "Frame", "Buffer %i x %i @ %i", bufferDescription.width, bufferDescription.height,
bufferDescription.stride);
size_t size = bufferDescription.height * bufferDescription.stride;
static constexpr auto ARRAYBUFFER_CACHE_PROP_NAME = "__frameArrayBufferCache";
if (!runtime.global().hasProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME)) {
@@ -102,9 +107,17 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer);
}
// Get CPU access to the HardwareBuffer (&buffer is a virtual temporary address)
void* buffer;
AHardwareBuffer_lock(hardwareBuffer, AHARDWAREBUFFER_USAGE_CPU_READ_MASK, -1, nullptr, &buffer);
// directly write to C++ JSI ArrayBuffer
auto destinationBuffer = arrayBuffer.data(runtime);
memcpy(destinationBuffer, buffer->getDirectAddress(), sizeof(uint8_t) * size);
memcpy(destinationBuffer, buffer, sizeof(uint8_t) * size);
// Release HardwareBuffer again
AHardwareBuffer_unlock(hardwareBuffer, nullptr);
AHardwareBuffer_release(hardwareBuffer);
return arrayBuffer;
};

8
package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.cpp
View File

@@ -4,6 +4,7 @@
#include "JFrame.h"
#include <android/hardware_buffer_jni.h>
#include <fbjni/ByteBuffer.h>
#include <fbjni/fbjni.h>
#include <jni.h>
@@ -58,9 +59,10 @@ int JFrame::getBytesPerRow() const {
return getBytesPerRowMethod(self());
}
local_ref<JByteBuffer> JFrame::toByteBuffer() const {
static const auto toByteBufferMethod = getClass()->getMethod<JByteBuffer()>("toByteBuffer");
return toByteBufferMethod(self());
AHardwareBuffer* JFrame::getHardwareBuffer() const {
static const auto getHardwareBufferMethod = getClass()->getMethod<jobject()>("getHardwareBufferBoxed");
auto hardwareBuffer = getHardwareBufferMethod(self());
return AHardwareBuffer_fromHardwareBuffer(jni::Environment::current(), hardwareBuffer.get());
}
void JFrame::incrementRefCount() {

3
package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.h
View File

@@ -4,6 +4,7 @@
#pragma once
#include <android/hardware_buffer.h>
#include <fbjni/ByteBuffer.h>
#include <fbjni/fbjni.h>
#include <jni.h>
@@ -26,7 +27,7 @@ public:
jlong getTimestamp() const;
local_ref<JString> getOrientation() const;
local_ref<JString> getPixelFormat() const;
local_ref<JByteBuffer> toByteBuffer() const;
AHardwareBuffer* getHardwareBuffer() const;
void incrementRefCount();
void decrementRefCount();
void close();