From 1a0bd8f7c287c878f0c50fe9ba74129cbb2b3da7 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 14 Feb 2024 12:47:40 +0100 Subject: [PATCH] feat: Add `enableGpuBuffers` property (#2557) * Revert "fix: Fix VideoPipeline crash on Samsung (Disable `USAGE_GPU_SAMPLED_IMAGE` ImageReader) (#2555)" This reverts commit ad33dd91b1c5d13f9c6bab6589417513810409c0. * feat: Add `enableGpuBuffers` prop * Create ImageWriter separately --- docs/docs/guides/PERFORMANCE.mdx | 7 ++ docs/docs/guides/TROUBLESHOOTING.mdx | 2 +- .../java/com/mrousavy/camera/CameraView.kt | 4 +- .../com/mrousavy/camera/CameraViewManager.kt | 5 ++ .../camera/core/CameraConfiguration.kt | 2 +- .../com/mrousavy/camera/core/CameraSession.kt | 1 + .../com/mrousavy/camera/core/VideoPipeline.kt | 69 ++++++++++++++++++- .../com/mrousavy/camera/types/PixelFormat.kt | 6 ++ .../camera/utils/HardwareBufferUtils.kt | 36 ++++++++++ package/src/CameraProps.ts | 21 +++++- 10 files changed, 146 insertions(+), 7 deletions(-) create mode 100644 package/android/src/main/java/com/mrousavy/camera/utils/HardwareBufferUtils.kt diff --git a/docs/docs/guides/PERFORMANCE.mdx b/docs/docs/guides/PERFORMANCE.mdx index c943f97..8e2434f 100644 --- a/docs/docs/guides/PERFORMANCE.mdx +++ b/docs/docs/guides/PERFORMANCE.mdx @@ -70,6 +70,13 @@ Enable Buffer Compression ([`enableBufferCompression`](/docs/api/interfaces/Came Note: When not using a `frameProcessor`, buffer compression is automatically enabled. +### GPU buffers + +Enable GPU Buffer flags ([`enableGpuBuffers`](/docs/api/interfaces/CameraProps#enablegpubuffers)) to optimize the Video Pipeline for zero-copy buffer forwarding. +If this is enabled, the Video Pipeline can avoid an additional CPU -> GPU copy, resulting in better performance and more efficiency. + +Note: This only takes effect when using a `frameProcessor`. + ### Video Stabilization Video Stabilization requires additional overhead to start the algorithm, so disabling [`videoStabilizationMode`](/docs/api/interfaces/CameraProps#videostabilizationmode) can significantly speed up the Camera initialization time. diff --git a/docs/docs/guides/TROUBLESHOOTING.mdx b/docs/docs/guides/TROUBLESHOOTING.mdx index 18398c5..c068c3f 100644 --- a/docs/docs/guides/TROUBLESHOOTING.mdx +++ b/docs/docs/guides/TROUBLESHOOTING.mdx @@ -112,7 +112,7 @@ If you're experiencing build issues or runtime issues in VisionCamera, make sure 2. If a camera device is not being returned by [`Camera.getAvailableCameraDevices()`](/docs/api/classes/Camera#getavailablecameradevices), make sure it is a Camera2 compatible device. See [this section in the Android docs](https://developer.android.com/reference/android/hardware/camera2/CameraDevice#reprocessing) for more information. 3. If your Frame Processor is not running, make sure you check the native Android Studio/Logcat logs. There is useful information about the Frame Processor Runtime that will tell you if something goes wrong. 4. If your Frame Processor is not running, make sure you are not using a remote JS debugger such as Google Chrome, since those don't work with JSI. -5. If you are experiencing black-screens, try removing all properties such as `fps`, `videoHdr` or `format` on the `` component except for the required ones: +5. If you are experiencing black-screens, try removing all properties such as `fps`, `videoHdr`, `enableGpuBuffers` or `format` on the `` component except for the required ones: ```tsx ``` diff --git a/package/android/src/main/java/com/mrousavy/camera/CameraView.kt b/package/android/src/main/java/com/mrousavy/camera/CameraView.kt index 1f24448..7f0f7b4 100644 --- a/package/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/package/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -64,6 +64,7 @@ class CameraView(context: Context) : var videoHdr = false var photoHdr = false var lowLightBoost: Boolean? = null // nullable bool + var enableGpuBuffers: Boolean = false // other props var isActive = false @@ -160,7 +161,8 @@ class CameraView(context: Context) : CameraConfiguration.Video( videoHdr, pixelFormat, - enableFrameProcessor + enableFrameProcessor, + enableGpuBuffers ) ) } else { diff --git a/package/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt b/package/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt index 54f7fce..ab047d7 100644 --- a/package/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt +++ b/package/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt @@ -84,6 +84,11 @@ class CameraViewManager : ViewGroupManager() { view.enableFpsGraph = enableFpsGraph } + @ReactProp(name = "enableGpuBuffers") + fun setEnableGpuBuffers(view: CameraView, enableGpuBuffers: Boolean) { + view.enableGpuBuffers = enableGpuBuffers + } + @ReactProp(name = "videoStabilizationMode") fun setVideoStabilizationMode(view: CameraView, videoStabilizationMode: String?) { val newMode = VideoStabilizationMode.fromUnionValue(videoStabilizationMode) diff --git a/package/android/src/main/java/com/mrousavy/camera/core/CameraConfiguration.kt b/package/android/src/main/java/com/mrousavy/camera/core/CameraConfiguration.kt index f1dca59..593d40a 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/CameraConfiguration.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/CameraConfiguration.kt @@ -44,7 +44,7 @@ data class CameraConfiguration( // Output types, those need to be comparable data class CodeScanner(val codeTypes: List) data class Photo(val enableHdr: Boolean) - data class Video(val enableHdr: Boolean, val pixelFormat: PixelFormat, val enableFrameProcessor: Boolean) + data class Video(val enableHdr: Boolean, val pixelFormat: PixelFormat, val enableFrameProcessor: Boolean, val enableGpuBuffers: Boolean) data class Audio(val nothing: Unit) data class Preview(val surface: Surface) diff --git a/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt b/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt index fef6998..4d52117 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt @@ -288,6 +288,7 @@ class CameraSession(private val context: Context, private val cameraManager: Cam video.config.pixelFormat, isSelfie, video.config.enableFrameProcessor, + video.config.enableGpuBuffers, callback ) val output = VideoPipelineOutput(videoPipeline, video.config.enableHdr) diff --git a/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt b/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt index f45520c..ae2c107 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt @@ -9,6 +9,7 @@ import android.os.Build import android.util.Log import android.view.Surface import androidx.annotation.Keep +import androidx.annotation.RequiresApi import com.facebook.jni.HybridData import com.facebook.proguard.annotations.DoNotStrip import com.mrousavy.camera.frameprocessor.Frame @@ -32,6 +33,7 @@ class VideoPipeline( val format: PixelFormat = PixelFormat.NATIVE, private val isMirrored: Boolean = false, private val enableFrameProcessor: Boolean = false, + private val enableGpuBuffers: Boolean = false, private val callback: CameraSession.Callback ) : SurfaceTexture.OnFrameAvailableListener, Closeable { @@ -78,14 +80,25 @@ class VideoPipeline( val format = getImageReaderFormat() Log.i(TAG, "Using ImageReader round-trip (format: #$format)") - imageReader = ImageReader.newInstance(width, height, format, MAX_IMAGES) + // Create ImageReader + if (enableGpuBuffers && Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { + val usageFlags = getRecommendedHardwareBufferFlags() + Log.i(TAG, "Creating ImageReader with GPU-optimized usage flags: $usageFlags") + imageReader = ImageReader.newInstance(width, height, format, MAX_IMAGES, usageFlags) + } else { + Log.i(TAG, "Creating ImageReader with default usage flags...") + imageReader = ImageReader.newInstance(width, height, format, MAX_IMAGES) + } + + // Create ImageWriter if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) { - Log.i(TAG, "Using ImageWriter with custom format (#$format)...") + Log.i(TAG, "Creating ImageWriter with format #$format...") imageWriter = ImageWriter.newInstance(glSurface, MAX_IMAGES, format) } else { - Log.i(TAG, "Using ImageWriter with default format...") + Log.i(TAG, "Creating ImageWriter with default format...") imageWriter = ImageWriter.newInstance(glSurface, MAX_IMAGES) } + imageReader!!.setOnImageAvailableListener({ reader -> Log.i(TAG, "ImageReader::onImageAvailable!") val image = reader.acquireNextImage() ?: return@setOnImageAvailableListener @@ -181,6 +194,56 @@ class VideoPipeline( } } + /** + * Get the recommended HardwareBuffer flags for creating ImageReader instances with. + * + * Tries to use [HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE] if possible, [HardwareBuffer.USAGE_CPU_READ_OFTEN] + * or a combination of both flags if CPU access is needed ([enableFrameProcessor]), and [0] otherwise. + */ + @RequiresApi(Build.VERSION_CODES.Q) + @Suppress("LiftReturnOrAssignment") + private fun getRecommendedHardwareBufferFlags(): Long { + val cpuFlag = HardwareBuffer.USAGE_CPU_READ_OFTEN + val gpuFlag = HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE + val bothFlags = gpuFlag or cpuFlag + + if (format == PixelFormat.NATIVE) { + // We don't need CPU access, so we can use GPU optimized buffers + if (supportsHardwareBufferFlags(gpuFlag)) { + // We support GPU Buffers directly and + Log.i(TAG, "GPU HardwareBuffers are supported!") + return gpuFlag + } else { + // no flags are supported - fall back to default + return 0 + } + } else { + // We are using YUV or RGB formats, so we need CPU access on the Frame + if (supportsHardwareBufferFlags(bothFlags)) { + // We support both CPU and GPU flags! + Log.i(TAG, "GPU + CPU HardwareBuffers are supported!") + return bothFlags + } else if (supportsHardwareBufferFlags(cpuFlag)) { + // We only support a CPU read flag, that's fine + Log.i(TAG, "CPU HardwareBuffers are supported!") + return cpuFlag + } else { + // no flags are supported - fall back to default + return 0 + } + } + } + + @RequiresApi(Build.VERSION_CODES.Q) + private fun supportsHardwareBufferFlags(flags: Long): Boolean { + val hardwareBufferFormat = format.toHardwareBufferFormat() + try { + return HardwareBuffer.isSupported(width, height, hardwareBufferFormat, 1, flags) + } catch (_: Throwable) { + return false + } + } + private external fun getInputTextureId(): Int private external fun onBeforeFrame() private external fun onFrame(transformMatrix: FloatArray) diff --git a/package/android/src/main/java/com/mrousavy/camera/types/PixelFormat.kt b/package/android/src/main/java/com/mrousavy/camera/types/PixelFormat.kt index c4a7a3c..697c53b 100644 --- a/package/android/src/main/java/com/mrousavy/camera/types/PixelFormat.kt +++ b/package/android/src/main/java/com/mrousavy/camera/types/PixelFormat.kt @@ -4,6 +4,7 @@ import android.graphics.ImageFormat import android.util.Log import com.mrousavy.camera.core.InvalidTypeScriptUnionError import com.mrousavy.camera.core.PixelFormatNotSupportedError +import com.mrousavy.camera.utils.HardwareBufferUtils import com.mrousavy.camera.utils.ImageFormatUtils enum class PixelFormat(override val unionValue: String) : JSUnionValue { @@ -19,6 +20,11 @@ enum class PixelFormat(override val unionValue: String) : JSUnionValue { else -> throw PixelFormatNotSupportedError(this.unionValue) } + fun toHardwareBufferFormat(): Int { + val imageFormat = toImageFormat() + return HardwareBufferUtils.getHardwareBufferFormat(imageFormat) + } + companion object : JSUnionValue.Companion { private const val TAG = "PixelFormat" fun fromImageFormat(imageFormat: Int): PixelFormat = diff --git a/package/android/src/main/java/com/mrousavy/camera/utils/HardwareBufferUtils.kt b/package/android/src/main/java/com/mrousavy/camera/utils/HardwareBufferUtils.kt new file mode 100644 index 0000000..c1d5106 --- /dev/null +++ b/package/android/src/main/java/com/mrousavy/camera/utils/HardwareBufferUtils.kt @@ -0,0 +1,36 @@ +package com.mrousavy.camera.utils + +import android.graphics.ImageFormat +import android.hardware.HardwareBuffer +import android.media.ImageReader +import android.os.Build + +class HardwareBufferUtils { + companion object { + fun getHardwareBufferFormat(imageFormat: Int): Int { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + // Dynamically create an ImageReader with the target ImageFormat, and then + // get it's HardwareBuffer format to see what it uses underneath. + val imageReader = ImageReader.newInstance(1, 1, imageFormat, 1, HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE) + val format = imageReader.hardwareBufferFormat + imageReader.close() + return format + } + + if (imageFormat == ImageFormat.PRIVATE) { + // PRIVATE formats are opaque, their actual equivalent HardwareBuffer format is unknown. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { + // We can assume that YUV 4:2:0 or RGB is used. + return HardwareBuffer.YCBCR_420_888 + } else { + // Maybe assume we are on RGB if we're not on API R or above... + return HardwareBuffer.RGB_888 + } + } + + // According to PublicFormat.cpp in Android's codebase, the formats map 1:1 anyways.. + // https://cs.android.com/android/platform/superproject/main/+/main:frameworks/native/libs/ui/PublicFormat.cpp + return imageFormat + } + } +} diff --git a/package/src/CameraProps.ts b/package/src/CameraProps.ts index 83ed1f6..e78e26e 100644 --- a/package/src/CameraProps.ts +++ b/package/src/CameraProps.ts @@ -183,10 +183,29 @@ export interface CameraProps extends ViewProps { * * @platform iOS * @default - * - true // if video={true} and frameProcessor={undefined} + * - true // if frameProcessor={undefined} * - false // otherwise */ enableBufferCompression?: boolean + /** + * Enables or disables GPU-sampled buffers for the video stream. This only takes effect when using a {@linkcode frameProcessor}. + * + * When recording a Video ({@linkcode video}) while a Frame Processor is running ({@linkcode frameProcessor}), + * the {@linkcode Frame | Frames} will need to be forwarded to the Media Encoder. + * + * - When `enableGpuBuffers` is `false`, the Video Pipeline will use CPU buffers causing an additional copy + * from the Frame Processor to the Media Encoder, which potentially results in increased latency. + * - When `enableGpuBuffers` is `true`, the Video Pipeline will use shared GPU buffers which greatly increases + * it's efficiency as an additional buffer copy is avoided. + * (See [`USAGE_GPU_SAMPLED_IMAGE`](https://developer.android.com/reference/android/hardware/HardwareBuffer#USAGE_GPU_SAMPLED_IMAGE)) + * + * In general, it is recommended to set this to `true` if possible, as this can increase performance and efficiency of the Video Pipeline. + * This is an experimental feature flag however, so use at your own risk. + * + * @platform Android + * @default false + */ + enableGpuBuffers?: boolean /** * Enables or disables low-light boost on this camera device. *