diff --git a/package/android/src/main/cpp/frameprocessor/FrameHostObject.cpp b/package/android/src/main/cpp/frameprocessor/FrameHostObject.cpp index c528c25..66a274b 100644 --- a/package/android/src/main/cpp/frameprocessor/FrameHostObject.cpp +++ b/package/android/src/main/cpp/frameprocessor/FrameHostObject.cpp @@ -12,6 +12,9 @@ #include #include +#include +#include + namespace vision { using namespace facebook; @@ -82,11 +85,13 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr if (name == "toArrayBuffer") { jsi::HostFunctionType toArrayBuffer = [=](jsi::Runtime& runtime, const jsi::Value& thisArg, const jsi::Value* args, size_t count) -> jsi::Value { - auto buffer = this->frame->toByteBuffer(); - if (!buffer->isDirect()) { - throw std::runtime_error("Failed to get byte content of Frame - array is not direct ByteBuffer!"); - } - auto size = buffer->getDirectSize(); + AHardwareBuffer* hardwareBuffer = this->frame->getHardwareBuffer(); + + AHardwareBuffer_Desc bufferDescription; + AHardwareBuffer_describe(hardwareBuffer, &bufferDescription); + __android_log_print(ANDROID_LOG_INFO, "Frame", "Buffer %i x %i @ %i", bufferDescription.width, bufferDescription.height, + bufferDescription.stride); + size_t size = bufferDescription.height * bufferDescription.stride; static constexpr auto ARRAYBUFFER_CACHE_PROP_NAME = "__frameArrayBufferCache"; if (!runtime.global().hasProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME)) { @@ -102,9 +107,17 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer); } + // Get CPU access to the HardwareBuffer (&buffer is a virtual temporary address) + void* buffer; + AHardwareBuffer_lock(hardwareBuffer, AHARDWAREBUFFER_USAGE_CPU_READ_MASK, -1, nullptr, &buffer); + // directly write to C++ JSI ArrayBuffer auto destinationBuffer = arrayBuffer.data(runtime); - memcpy(destinationBuffer, buffer->getDirectAddress(), sizeof(uint8_t) * size); + memcpy(destinationBuffer, buffer, sizeof(uint8_t) * size); + + // Release HardwareBuffer again + AHardwareBuffer_unlock(hardwareBuffer, nullptr); + AHardwareBuffer_release(hardwareBuffer); return arrayBuffer; }; diff --git a/package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.cpp b/package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.cpp index bef6ad1..8146d8c 100644 --- a/package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.cpp +++ b/package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.cpp @@ -4,6 +4,7 @@ #include "JFrame.h" +#include #include #include #include @@ -58,9 +59,10 @@ int JFrame::getBytesPerRow() const { return getBytesPerRowMethod(self()); } -local_ref JFrame::toByteBuffer() const { - static const auto toByteBufferMethod = getClass()->getMethod("toByteBuffer"); - return toByteBufferMethod(self()); +AHardwareBuffer* JFrame::getHardwareBuffer() const { + static const auto getHardwareBufferMethod = getClass()->getMethod("getHardwareBufferBoxed"); + auto hardwareBuffer = getHardwareBufferMethod(self()); + return AHardwareBuffer_fromHardwareBuffer(jni::Environment::current(), hardwareBuffer.get()); } void JFrame::incrementRefCount() { diff --git a/package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.h b/package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.h index 92b23d2..0311da7 100644 --- a/package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.h +++ b/package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.h @@ -4,6 +4,7 @@ #pragma once +#include #include #include #include @@ -26,7 +27,7 @@ public: jlong getTimestamp() const; local_ref getOrientation() const; local_ref getPixelFormat() const; - local_ref toByteBuffer() const; + AHardwareBuffer* getHardwareBuffer() const; void incrementRefCount(); void decrementRefCount(); void close(); diff --git a/package/android/src/main/java/com/mrousavy/camera/Errors.kt b/package/android/src/main/java/com/mrousavy/camera/Errors.kt index 0cdc043..faa23ba 100644 --- a/package/android/src/main/java/com/mrousavy/camera/Errors.kt +++ b/package/android/src/main/java/com/mrousavy/camera/Errors.kt @@ -50,13 +50,6 @@ class NoCameraDeviceError : ) class PixelFormatNotSupportedError(format: String) : CameraError("device", "pixel-format-not-supported", "The pixelFormat $format is not supported on the given Camera Device!") -class PixelFormatNotSupportedInVideoPipelineError(format: String) : - CameraError( - "device", - "pixel-format-not-supported", - "The pixelFormat $format is currently not supported in the VideoPipeline! " + - "See this issue for more details ($4.000 bounty!): https://github.com/mrousavy/react-native-vision-camera/issues/1837" - ) class CameraNotReadyError : CameraError("session", "camera-not-ready", "The Camera is not ready yet! Wait for the onInitialized() callback!") @@ -77,10 +70,8 @@ class CaptureAbortedError(wasImageCaptured: Boolean) : CameraError("capture", "aborted", "The image capture was aborted! Was Image captured: $wasImageCaptured") class UnknownCaptureError(wasImageCaptured: Boolean) : CameraError("capture", "unknown", "An unknown error occurred while trying to capture an Image! Was Image captured: $wasImageCaptured") - class RecorderError(name: String, extra: Int) : CameraError("capture", "recorder-error", "An error occured while recording a video! $name $extra") - class NoRecordingInProgressError : CameraError("capture", "no-recording-in-progress", "There was no active video recording in progress! Did you call stopRecording() twice?") class RecordingInProgressError : @@ -92,5 +83,9 @@ class RecordingInProgressError : class ViewNotFoundError(viewId: Int) : CameraError("system", "view-not-found", "The given view (ID $viewId) was not found in the view manager.") +class FrameProcessorsUnavailableError(reason: String) : + CameraError("system", "frame-processors-unavailable", "Frame Processors are unavailable! Reason: $reason") +class HardwareBuffersNotAvailableError : + CameraError("system", "hardware-buffers-unavailable", "HardwareBuffers are only available on API 28 or higher!") class UnknownCameraError(cause: Throwable?) : CameraError("unknown", "unknown", cause?.message ?: "An unknown camera error occured.", cause) diff --git a/package/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java b/package/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java index 0171f80..f92f2be 100644 --- a/package/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java +++ b/package/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java @@ -1,8 +1,10 @@ package com.mrousavy.camera.frameprocessor; -import android.graphics.ImageFormat; +import android.hardware.HardwareBuffer; import android.media.Image; +import android.os.Build; import com.facebook.proguard.annotations.DoNotStrip; +import com.mrousavy.camera.HardwareBuffersNotAvailableError; import com.mrousavy.camera.parsers.PixelFormat; import com.mrousavy.camera.parsers.Orientation; @@ -14,6 +16,7 @@ public class Frame { private final long timestamp; private final Orientation orientation; private int refCount = 0; + private HardwareBuffer hardwareBuffer = null; public Frame(Image image, long timestamp, Orientation orientation, boolean isMirrored) { this.image = image; @@ -89,34 +92,20 @@ public class Frame { return image.getPlanes()[0].getRowStride(); } - private static ByteBuffer byteArrayCache; - @SuppressWarnings("unused") @DoNotStrip - public ByteBuffer toByteBuffer() { - switch (image.getFormat()) { - case ImageFormat.YUV_420_888: - ByteBuffer yBuffer = image.getPlanes()[0].getBuffer(); - ByteBuffer uBuffer = image.getPlanes()[1].getBuffer(); - ByteBuffer vBuffer = image.getPlanes()[2].getBuffer(); - int ySize = yBuffer.remaining(); - int uSize = uBuffer.remaining(); - int vSize = vBuffer.remaining(); - int totalSize = ySize + uSize + vSize; + public Object getHardwareBufferBoxed() throws HardwareBuffersNotAvailableError { + return getHardwareBuffer(); + } - if (byteArrayCache != null) byteArrayCache.rewind(); - if (byteArrayCache == null || byteArrayCache.remaining() != totalSize) { - byteArrayCache = ByteBuffer.allocateDirect(totalSize); - } - - byteArrayCache.put(yBuffer).put(uBuffer).put(vBuffer); - - return byteArrayCache; - case ImageFormat.JPEG: - return image.getPlanes()[0].getBuffer(); - default: - throw new RuntimeException("Cannot convert Frame with Format " + image.getFormat() + " to byte array!"); + public HardwareBuffer getHardwareBuffer() throws HardwareBuffersNotAvailableError { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P) { + throw new HardwareBuffersNotAvailableError(); } + if (hardwareBuffer == null) { + hardwareBuffer = image.getHardwareBuffer(); + } + return hardwareBuffer; } @SuppressWarnings("unused") @@ -142,6 +131,9 @@ public class Frame { @SuppressWarnings("unused") @DoNotStrip private void close() { + if (hardwareBuffer != null) { + hardwareBuffer.close(); + } image.close(); } }