feat: Use JSI's ArrayBuffer
instead of TypedArray
(#2408)
* feat: Use JSI's `ArrayBuffer` instead of `TypedArray` * fix: Fix move memory * feat: Implement iOS * Format * Update JSIJNIConversion.cpp * fix: Fix Android `toArrayBuffer` and other * Catch FP call errors * Update return type * Use `CPU_READ_OFTEN` flag as well * CPU flag * Run destructors under `jni::ThreadScope` * Update FrameProcessorPluginHostObject.cpp * fix: Fix `toArrayBuffer()` crash * Update Frame.ts
This commit is contained in:
@@ -96,8 +96,9 @@ class VideoPipeline(
|
||||
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
|
||||
Log.i(TAG, "Using API 29 for GPU ImageReader...")
|
||||
// GPU_SAMPLED because we redirect to OpenGL
|
||||
val usage = HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE
|
||||
// If we are in PRIVATE, we just pass it to the GPU as efficiently as possible - so use GPU flag.
|
||||
// If we are in YUV/RGB/..., we probably want to access Frame data - so use CPU flag.
|
||||
val usage = if (format == ImageFormat.PRIVATE) HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE else HardwareBuffer.USAGE_CPU_READ_OFTEN
|
||||
imageReader = ImageReader.newInstance(width, height, format, MAX_IMAGES, usage)
|
||||
imageWriter = ImageWriter.newInstance(glSurface, MAX_IMAGES, format)
|
||||
} else {
|
||||
@@ -109,17 +110,21 @@ class VideoPipeline(
|
||||
Log.i(TAG, "ImageReader::onImageAvailable!")
|
||||
val image = reader.acquireNextImage() ?: return@setOnImageAvailableListener
|
||||
|
||||
// TODO: Get correct orientation and isMirrored
|
||||
val frame = Frame(image, image.timestamp, Orientation.PORTRAIT, isMirrored)
|
||||
frame.incrementRefCount()
|
||||
frameProcessor?.call(frame)
|
||||
try {
|
||||
// TODO: Get correct orientation and isMirrored
|
||||
val frame = Frame(image, image.timestamp, Orientation.PORTRAIT, isMirrored)
|
||||
frame.incrementRefCount()
|
||||
frameProcessor?.call(frame)
|
||||
|
||||
if (hasOutputs) {
|
||||
// If we have outputs (e.g. a RecordingSession), pass the frame along to the OpenGL pipeline
|
||||
imageWriter!!.queueInputImage(image)
|
||||
if (hasOutputs) {
|
||||
// If we have outputs (e.g. a RecordingSession), pass the frame along to the OpenGL pipeline
|
||||
imageWriter!!.queueInputImage(image)
|
||||
}
|
||||
|
||||
frame.decrementRefCount()
|
||||
} catch (e: Throwable) {
|
||||
Log.e(TAG, "Failed to call Frame Processor!", e)
|
||||
}
|
||||
|
||||
frame.decrementRefCount()
|
||||
}, CameraQueues.videoQueue.handler)
|
||||
|
||||
surface = imageReader!!.surface
|
||||
|
@@ -15,7 +15,6 @@ public class Frame {
|
||||
private final long timestamp;
|
||||
private final Orientation orientation;
|
||||
private int refCount = 0;
|
||||
private HardwareBuffer hardwareBuffer = null;
|
||||
|
||||
public Frame(Image image, long timestamp, Orientation orientation, boolean isMirrored) {
|
||||
this.image = image;
|
||||
@@ -114,10 +113,7 @@ public class Frame {
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P) {
|
||||
throw new HardwareBuffersNotAvailableError();
|
||||
}
|
||||
if (hardwareBuffer == null) {
|
||||
hardwareBuffer = getImage().getHardwareBuffer();
|
||||
}
|
||||
return hardwareBuffer;
|
||||
return getImage().getHardwareBuffer();
|
||||
}
|
||||
|
||||
@SuppressWarnings("unused")
|
||||
@@ -142,9 +138,6 @@ public class Frame {
|
||||
|
||||
private synchronized void close() {
|
||||
synchronized (this) {
|
||||
if (hardwareBuffer != null) {
|
||||
hardwareBuffer.close();
|
||||
}
|
||||
image.close();
|
||||
}
|
||||
}
|
||||
|
@@ -20,18 +20,26 @@ public final class SharedArray {
|
||||
|
||||
@DoNotStrip
|
||||
@Keep
|
||||
public SharedArray(HybridData hybridData) {
|
||||
private SharedArray(HybridData hybridData) {
|
||||
mHybridData = hybridData;
|
||||
}
|
||||
|
||||
/**
|
||||
* Allocate a new SharedArray. Use `getByteBuffer` to obtain a reference to the direct ByteBuffer for writing.
|
||||
* @param proxy The VisionCamera Proxy from the Frame Processor Plugin's initializer.
|
||||
* @param dataType The ArrayBuffer's data type. `Type.Int8Array` = `Int8Array` in JS
|
||||
* @param size The size of the ArrayBuffer.
|
||||
*/
|
||||
public SharedArray(VisionCameraProxy proxy, Type dataType, int size) {
|
||||
mHybridData = initHybrid(proxy, dataType.ordinal(), size);
|
||||
public SharedArray(VisionCameraProxy proxy, int size) {
|
||||
mHybridData = initHybrid(proxy, size);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps the given ByteBuffer in a JSI ArrayBuffer. Using `getByteBuffer` will return the same instance which can be used for writing.
|
||||
* @param proxy The VisionCamera Proxy from the Frame Processor Plugin's initializer.
|
||||
* @param byteBuffer The ByteBuffer to wrap.
|
||||
*/
|
||||
public SharedArray(VisionCameraProxy proxy, ByteBuffer byteBuffer) {
|
||||
mHybridData = initHybrid(proxy, byteBuffer);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -44,21 +52,6 @@ public final class SharedArray {
|
||||
*/
|
||||
public native int getSize();
|
||||
|
||||
private native HybridData initHybrid(VisionCameraProxy proxy, int dataType, int size);
|
||||
|
||||
/**
|
||||
* The Type of the SharedArray.
|
||||
*/
|
||||
public enum Type {
|
||||
// Values start at 0 and need to match with JSITypedArray.h::TypedArrayKind
|
||||
Int8Array,
|
||||
Int16Array,
|
||||
Int32Array,
|
||||
Uint8Array,
|
||||
Uint8ClampedArray,
|
||||
Uint16Array,
|
||||
Uint32Array,
|
||||
Float32Array,
|
||||
Float64Array,
|
||||
}
|
||||
private native HybridData initHybrid(VisionCameraProxy proxy, int size);
|
||||
private native HybridData initHybrid(VisionCameraProxy proxy, ByteBuffer byteBuffer);
|
||||
}
|
||||
|
Reference in New Issue
Block a user