feat: Use JSI's ArrayBuffer instead of TypedArray (#2408)

* feat: Use JSI's `ArrayBuffer` instead of `TypedArray`

* fix: Fix move memory

* feat: Implement iOS

* Format

* Update JSIJNIConversion.cpp

* fix: Fix Android `toArrayBuffer` and other

* Catch FP call errors

* Update return type

* Use `CPU_READ_OFTEN` flag as well

* CPU flag

* Run destructors under `jni::ThreadScope`

* Update FrameProcessorPluginHostObject.cpp

* fix: Fix `toArrayBuffer()` crash

* Update Frame.ts
This commit is contained in:
Marc Rousavy
2024-01-17 20:18:46 +01:00
committed by GitHub
parent 2f21609e39
commit ba1d7eec9c
26 changed files with 296 additions and 620 deletions

View File

@@ -96,8 +96,9 @@ class VideoPipeline(
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
Log.i(TAG, "Using API 29 for GPU ImageReader...")
// GPU_SAMPLED because we redirect to OpenGL
val usage = HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE
// If we are in PRIVATE, we just pass it to the GPU as efficiently as possible - so use GPU flag.
// If we are in YUV/RGB/..., we probably want to access Frame data - so use CPU flag.
val usage = if (format == ImageFormat.PRIVATE) HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE else HardwareBuffer.USAGE_CPU_READ_OFTEN
imageReader = ImageReader.newInstance(width, height, format, MAX_IMAGES, usage)
imageWriter = ImageWriter.newInstance(glSurface, MAX_IMAGES, format)
} else {
@@ -109,17 +110,21 @@ class VideoPipeline(
Log.i(TAG, "ImageReader::onImageAvailable!")
val image = reader.acquireNextImage() ?: return@setOnImageAvailableListener
// TODO: Get correct orientation and isMirrored
val frame = Frame(image, image.timestamp, Orientation.PORTRAIT, isMirrored)
frame.incrementRefCount()
frameProcessor?.call(frame)
try {
// TODO: Get correct orientation and isMirrored
val frame = Frame(image, image.timestamp, Orientation.PORTRAIT, isMirrored)
frame.incrementRefCount()
frameProcessor?.call(frame)
if (hasOutputs) {
// If we have outputs (e.g. a RecordingSession), pass the frame along to the OpenGL pipeline
imageWriter!!.queueInputImage(image)
if (hasOutputs) {
// If we have outputs (e.g. a RecordingSession), pass the frame along to the OpenGL pipeline
imageWriter!!.queueInputImage(image)
}
frame.decrementRefCount()
} catch (e: Throwable) {
Log.e(TAG, "Failed to call Frame Processor!", e)
}
frame.decrementRefCount()
}, CameraQueues.videoQueue.handler)
surface = imageReader!!.surface

View File

@@ -15,7 +15,6 @@ public class Frame {
private final long timestamp;
private final Orientation orientation;
private int refCount = 0;
private HardwareBuffer hardwareBuffer = null;
public Frame(Image image, long timestamp, Orientation orientation, boolean isMirrored) {
this.image = image;
@@ -114,10 +113,7 @@ public class Frame {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P) {
throw new HardwareBuffersNotAvailableError();
}
if (hardwareBuffer == null) {
hardwareBuffer = getImage().getHardwareBuffer();
}
return hardwareBuffer;
return getImage().getHardwareBuffer();
}
@SuppressWarnings("unused")
@@ -142,9 +138,6 @@ public class Frame {
private synchronized void close() {
synchronized (this) {
if (hardwareBuffer != null) {
hardwareBuffer.close();
}
image.close();
}
}

View File

@@ -20,18 +20,26 @@ public final class SharedArray {
@DoNotStrip
@Keep
public SharedArray(HybridData hybridData) {
private SharedArray(HybridData hybridData) {
mHybridData = hybridData;
}
/**
* Allocate a new SharedArray. Use `getByteBuffer` to obtain a reference to the direct ByteBuffer for writing.
* @param proxy The VisionCamera Proxy from the Frame Processor Plugin's initializer.
* @param dataType The ArrayBuffer's data type. `Type.Int8Array` = `Int8Array` in JS
* @param size The size of the ArrayBuffer.
*/
public SharedArray(VisionCameraProxy proxy, Type dataType, int size) {
mHybridData = initHybrid(proxy, dataType.ordinal(), size);
public SharedArray(VisionCameraProxy proxy, int size) {
mHybridData = initHybrid(proxy, size);
}
/**
* Wraps the given ByteBuffer in a JSI ArrayBuffer. Using `getByteBuffer` will return the same instance which can be used for writing.
* @param proxy The VisionCamera Proxy from the Frame Processor Plugin's initializer.
* @param byteBuffer The ByteBuffer to wrap.
*/
public SharedArray(VisionCameraProxy proxy, ByteBuffer byteBuffer) {
mHybridData = initHybrid(proxy, byteBuffer);
}
/**
@@ -44,21 +52,6 @@ public final class SharedArray {
*/
public native int getSize();
private native HybridData initHybrid(VisionCameraProxy proxy, int dataType, int size);
/**
* The Type of the SharedArray.
*/
public enum Type {
// Values start at 0 and need to match with JSITypedArray.h::TypedArrayKind
Int8Array,
Int16Array,
Int32Array,
Uint8Array,
Uint8ClampedArray,
Uint16Array,
Uint32Array,
Float32Array,
Float64Array,
}
private native HybridData initHybrid(VisionCameraProxy proxy, int size);
private native HybridData initHybrid(VisionCameraProxy proxy, ByteBuffer byteBuffer);
}