From 23af74aaf142bea6c04d8315cbfc20eddd5cf522 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 24 Aug 2023 11:49:27 +0200 Subject: [PATCH] feat: Support `focus()` on Android (#1713) * feat: Support `focus()` on Android * perf: Cache RequestBuilder * Use 30 FPS as default * feat: Implement focus(x, y) * fix: Fix ByteBuffer resizing --- .../java/com/mrousavy/camera/CameraSession.kt | 127 ++++++++++++------ .../com/mrousavy/camera/CameraView+Focus.kt | 4 +- .../mrousavy/camera/frameprocessor/Frame.java | 4 +- 3 files changed, 93 insertions(+), 42 deletions(-) diff --git a/android/src/main/java/com/mrousavy/camera/CameraSession.kt b/android/src/main/java/com/mrousavy/camera/CameraSession.kt index ab80999..8a58643 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraSession.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraSession.kt @@ -1,17 +1,21 @@ package com.mrousavy.camera import android.content.Context +import android.graphics.Point import android.hardware.camera2.CameraCaptureSession import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraDevice import android.hardware.camera2.CameraManager +import android.hardware.camera2.CameraMetadata import android.hardware.camera2.CaptureRequest import android.hardware.camera2.CaptureResult import android.hardware.camera2.TotalCaptureResult +import android.hardware.camera2.params.MeteringRectangle import android.media.Image import android.os.Build import android.util.Log import android.util.Range +import android.util.Size import com.mrousavy.camera.extensions.SessionType import com.mrousavy.camera.extensions.capture import com.mrousavy.camera.extensions.createCaptureSession @@ -44,6 +48,9 @@ class CameraSession(private val context: Context, private val onError: (e: Throwable) -> Unit): CoroutineScope, Closeable, CameraOutputs.Callback, CameraManager.AvailabilityCallback() { companion object { private const val TAG = "CameraSession" + + // TODO: Samsung advertises 60 FPS but only allows 30 FPS for some reason. + private val CAN_SET_FPS = Build.MANUFACTURER != "samsung" } data class CapturedPhoto(val image: Image, @@ -76,6 +83,7 @@ class CameraSession(private val context: Context, private var captureSession: CameraCaptureSession? = null private var cameraDevice: CameraDevice? = null + private var previewRequest: CaptureRequest.Builder? = null private val photoOutputSynchronizer = PhotoOutputSynchronizer() private val mutex = Mutex() private var isRunning = false @@ -290,6 +298,20 @@ class CameraSession(private val context: Context, } } + suspend fun focus(x: Int, y: Int) { + val captureSession = captureSession ?: throw CameraNotReadyError() + val previewOutput = outputs?.previewOutput ?: throw CameraNotReadyError() + val characteristics = cameraManager.getCameraCharacteristics(captureSession.device.id) + val sensorSize = characteristics.get(CameraCharacteristics.SENSOR_INFO_ACTIVE_ARRAY_SIZE)!! + val previewSize = previewOutput.size + val pX = x.toDouble() / previewSize.width * sensorSize.height() + val pY = y.toDouble() / previewSize.height * sensorSize.width() + val point = Point(pX.toInt(), pY.toInt()) + + Log.i(TAG, "Focusing (${point.x}, ${point.y})...") + focus(point) + } + override fun onCameraAvailable(cameraId: String) { super.onCameraAvailable(cameraId) Log.i(TAG, "Camera became available: $cameraId") @@ -300,6 +322,39 @@ class CameraSession(private val context: Context, Log.i(TAG, "Camera became un-available: $cameraId") } + private suspend fun focus(point: Point) { + mutex.withLock { + val captureSession = captureSession ?: throw CameraNotReadyError() + val request = previewRequest ?: throw CameraNotReadyError() + + val weight = MeteringRectangle.METERING_WEIGHT_MAX - 1 + val focusAreaTouch = MeteringRectangle(point, Size(150, 150), weight) + + // Quickly pause preview + captureSession.stopRepeating() + + request.set(CaptureRequest.CONTROL_AF_TRIGGER, CameraMetadata.CONTROL_AF_TRIGGER_CANCEL) + request.set(CaptureRequest.CONTROL_AF_MODE, CaptureRequest.CONTROL_AF_MODE_OFF) + captureSession.capture(request.build(), null, null) + + // Add AF trigger with focus region + val characteristics = cameraManager.getCameraCharacteristics(captureSession.device.id) + val maxSupportedFocusRegions = characteristics.get(CameraCharacteristics.CONTROL_MAX_REGIONS_AE) ?: 0 + if (maxSupportedFocusRegions >= 1) { + request.set(CaptureRequest.CONTROL_AF_REGIONS, arrayOf(focusAreaTouch)) + } + request.set(CaptureRequest.CONTROL_MODE, CameraMetadata.CONTROL_MODE_AUTO) + request.set(CaptureRequest.CONTROL_AF_MODE, CaptureRequest.CONTROL_AF_MODE_AUTO) + request.set(CaptureRequest.CONTROL_AF_TRIGGER, CameraMetadata.CONTROL_AF_TRIGGER_START) + + captureSession.capture(request.build(), false) + + // Resume preview + request.set(CaptureRequest.CONTROL_AF_TRIGGER, CaptureRequest.CONTROL_AF_TRIGGER_IDLE) + captureSession.setRepeatingRequest(request.build(), null, null) + } + } + /** * Opens a [CameraDevice]. If there already is an open Camera for the given [cameraId], use that. */ @@ -362,43 +417,27 @@ class CameraSession(private val context: Context, return session } - private fun getPreviewCaptureRequest(captureSession: CameraCaptureSession, - outputs: CameraOutputs, - fps: Int? = null, + private fun getPreviewCaptureRequest(fps: Int? = null, videoStabilizationMode: VideoStabilizationMode? = null, lowLightBoost: Boolean? = null, hdr: Boolean? = null, torch: Boolean? = null): CaptureRequest { - val template = if (outputs.videoOutput != null) CameraDevice.TEMPLATE_RECORD else CameraDevice.TEMPLATE_PREVIEW - val captureRequest = captureSession.device.createCaptureRequest(template) - outputs.previewOutput?.let { output -> - Log.i(TAG, "Adding output surface ${output.outputType}..") - captureRequest.addTarget(output.surface) - } - outputs.videoOutput?.let { output -> - Log.i(TAG, "Adding output surface ${output.outputType}..") - captureRequest.addTarget(output.surface) - } + val captureRequest = previewRequest ?: throw CameraNotReadyError() - if (fps != null) { - // TODO: Samsung advertises 60 FPS but only allows 30 FPS for some reason. - val isSamsung = Build.MANUFACTURER == "samsung" - val targetFps = if (isSamsung) 30 else fps + // FPS + val fpsRange = if (fps != null && CAN_SET_FPS) Range(fps, fps) else Range(30, 30) + captureRequest.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, fpsRange) - captureRequest.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(targetFps, targetFps)) - } - if (videoStabilizationMode != null) { - captureRequest.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, videoStabilizationMode.toDigitalStabilizationMode()) - captureRequest.set(CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE, videoStabilizationMode.toOpticalStabilizationMode()) - } - if (lowLightBoost == true) { - captureRequest.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_NIGHT) - } - if (hdr == true) { - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP_MR1) { - captureRequest.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_HDR) - } - } + // Video Stabilization + captureRequest.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, videoStabilizationMode?.toDigitalStabilizationMode()) + captureRequest.set(CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE, videoStabilizationMode?.toOpticalStabilizationMode()) + + // Night/HDR Mode + val sceneMode = if (hdr == true) CaptureRequest.CONTROL_SCENE_MODE_HDR else if (lowLightBoost == true) CaptureRequest.CONTROL_SCENE_MODE_NIGHT else null + captureRequest.set(CaptureRequest.CONTROL_SCENE_MODE, sceneMode) + captureRequest.set(CaptureRequest.CONTROL_MODE, if (sceneMode != null) CaptureRequest.CONTROL_MODE_USE_SCENE_MODE else null) + + // Zoom if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { captureRequest.set(CaptureRequest.CONTROL_ZOOM_RATIO, zoom) } else { @@ -407,6 +446,7 @@ class CameraSession(private val context: Context, captureRequest.set(CaptureRequest.SCALER_CROP_REGION, size.zoomed(zoom)) } + // Torch Mode val torchMode = if (torch == true) CaptureRequest.FLASH_MODE_TORCH else CaptureRequest.FLASH_MODE_OFF captureRequest.set(CaptureRequest.FLASH_MODE, torchMode) @@ -441,19 +481,32 @@ class CameraSession(private val context: Context, return@withLock } - // 2. Open Camera Device + // 1. Open Camera Device val camera = getCameraDevice(cameraId) { reason -> isRunning = false onError(reason) } - // 3. Create capture session with outputs + // 2. Create capture session with outputs val session = getCaptureSession(camera, outputs) { isRunning = false } + // 3. Create request template + val template = if (outputs.videoOutput != null) CameraDevice.TEMPLATE_RECORD else CameraDevice.TEMPLATE_PREVIEW + val captureRequest = camera.createCaptureRequest(template) + outputs.previewOutput?.let { output -> + Log.i(TAG, "Adding output surface ${output.outputType}..") + captureRequest.addTarget(output.surface) + } + outputs.videoOutput?.let { output -> + Log.i(TAG, "Adding output surface ${output.outputType}..") + captureRequest.addTarget(output.surface) + } + Log.i(TAG, "Camera Session initialized! Starting repeating request..") isRunning = true + this.previewRequest = captureRequest this.captureSession = session this.cameraDevice = camera } @@ -477,13 +530,9 @@ class CameraSession(private val context: Context, val videoStabilizationMode = videoStabilizationMode val lowLightBoost = lowLightBoost val hdr = hdr - val outputs = outputs - if (outputs == null || outputs.size == 0) { - Log.i(TAG, "CameraSession doesn't have any Outputs, canceling..") - return - } - val repeatingRequest = getPreviewCaptureRequest(session, outputs, fps, videoStabilizationMode, lowLightBoost, hdr) + val repeatingRequest = getPreviewCaptureRequest(fps, videoStabilizationMode, lowLightBoost, hdr) + Log.d(TAG, "Setting Repeating Request..") session.setRepeatingRequest(repeatingRequest, null, null) } } diff --git a/android/src/main/java/com/mrousavy/camera/CameraView+Focus.kt b/android/src/main/java/com/mrousavy/camera/CameraView+Focus.kt index 0c17dad..30e465c 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView+Focus.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView+Focus.kt @@ -3,5 +3,7 @@ package com.mrousavy.camera import com.facebook.react.bridge.ReadableMap suspend fun CameraView.focus(pointMap: ReadableMap) { - // TODO: CameraView.focus!! + val x = pointMap.getInt("x") + val y = pointMap.getInt("y") + cameraSession.focus(x, y) } diff --git a/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java b/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java index b9ea221..0171f80 100644 --- a/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java +++ b/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java @@ -104,11 +104,11 @@ public class Frame { int vSize = vBuffer.remaining(); int totalSize = ySize + uSize + vSize; - if (byteArrayCache == null) { + if (byteArrayCache != null) byteArrayCache.rewind(); + if (byteArrayCache == null || byteArrayCache.remaining() != totalSize) { byteArrayCache = ByteBuffer.allocateDirect(totalSize); } - byteArrayCache.rewind(); byteArrayCache.put(yBuffer).put(uBuffer).put(vBuffer); return byteArrayCache;