From 37398cc909aa4d7754279c96e0b21d4ebe746b3d Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 14 Feb 2024 16:43:30 +0100 Subject: [PATCH] feat: Flash with AE Pre-capture trigger for Android (#2558) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Flash (`flash`) Adds `flash` functionality with a fully custom pre-capture AE/AF/AWB trigger sequence for Android. 🎉 ```ts camera.current.takePhoto({ flash: 'on' // or 'auto' }) ``` ### Better photos (`qualityPrioritization`) We now also run the AE/AF/AWB precapture sequence on every photo (unless `qualityPrioritization` is `speed`), meaning photos are now less blurry, properly exposed, and properly white-balanced - so in short: **photo quality is now better!**. The fast path still exists when using `qualityPrioritization: speed`, as that will skip the precapture sequence and metering actions and just grab an Image from the Camera as quickly as possible. Additionally, `qualityPrioritization` now controls these options: - [COLOR_CORRECTION_MODE](https://developer.android.com/reference/android/hardware/camera2/CaptureRequest#COLOR_CORRECTION_MODE) - [EDGE_MODE](https://developer.android.com/reference/android/hardware/camera2/CaptureRequest#EDGE_MODE) - [COLOR_CORRECTION_ABERRATION_MODE](https://developer.android.com/reference/android/hardware/camera2/CaptureRequest#COLOR_CORRECTION_ABERRATION_MODE) - [HOT_PIXEL_MODE](https://developer.android.com/reference/android/hardware/camera2/CaptureRequest#HOT_PIXEL_MODE) - [DISTORTION_CORRECTION_MODE](https://developer.android.com/reference/android/hardware/camera2/CaptureRequest#DISTORTION_CORRECTION_MODE) - [NOISE_REDUCTION_MODE](https://developer.android.com/reference/android/hardware/camera2/CaptureRequest#NOISE_REDUCTION_MODE) - [SHADING_MODE](https://developer.android.com/reference/android/hardware/camera2/CaptureRequest#SHADING_MODE) - [TONEMAP_MODE](https://developer.android.com/reference/android/hardware/camera2/CaptureRequest#TONEMAP_MODE) ..by setting them to `_FAST` or `_HIGH_QUALITY`, which was previously left untouched. This now means: - `takePhoto({ qualityPrioritization: 'speed' })` got FASTER 🚀 - `takePhoto({ qualityPrioritization: 'quality' })` got BETTER QUALITY 📸 - `takePhoto({ qualityPrioritization: 'balanced' })` is left unchanged ✅ --- .../camera/core/CameraDeviceDetails.kt | 31 ++- .../com/mrousavy/camera/core/CameraError.kt | 1 + .../core/PersistentCameraCaptureSession.kt | 96 ++++------ .../core/capture/PhotoCaptureRequest.kt | 71 ++++++- .../CameraCaptureSession+capture.kt | 21 ++- .../CameraCaptureSession+precapture.kt | 99 ++++++++++ ...Session+setRepeatingRequestAndWaitForAF.kt | 47 ----- ...setRepeatingRequestAndWaitForPrecapture.kt | 178 ++++++++++++++++++ .../mrousavy/camera/types/HardwareLevel.kt | 13 ++ package/example/src/views/CaptureButton.tsx | 21 +-- package/src/CameraError.ts | 1 + 11 files changed, 448 insertions(+), 131 deletions(-) create mode 100644 package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+precapture.kt delete mode 100644 package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+setRepeatingRequestAndWaitForAF.kt create mode 100644 package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+setRepeatingRequestAndWaitForPrecapture.kt diff --git a/package/android/src/main/java/com/mrousavy/camera/core/CameraDeviceDetails.kt b/package/android/src/main/java/com/mrousavy/camera/core/CameraDeviceDetails.kt index 43a09db..5dbae98 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/CameraDeviceDetails.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/CameraDeviceDetails.kt @@ -50,7 +50,7 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, val cameraId val sensorSize by lazy { characteristics.get(CameraCharacteristics.SENSOR_INFO_PHYSICAL_SIZE)!! } val activeSize get() = characteristics.get(CameraCharacteristics.SENSOR_INFO_ACTIVE_ARRAY_SIZE)!! - val sensorOrientation by lazy { characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0 } + val sensorOrientation = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0 val minFocusDistance by lazy { getMinFocusDistanceCm() } val name by lazy { val info = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) characteristics.get(CameraCharacteristics.INFO_VERSION) else null @@ -97,14 +97,28 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, val cameraId val isBackwardsCompatible by lazy { capabilities.contains(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_BACKWARD_COMPATIBLE) } val supportsSnapshotCapture by lazy { supportsSnapshotCapture() } - val supportsTapToFocus by lazy { (characteristics.get(CameraCharacteristics.CONTROL_MAX_REGIONS_AF) ?: 0) > 0 } - val supportsTapToExposure by lazy { (characteristics.get(CameraCharacteristics.CONTROL_MAX_REGIONS_AE) ?: 0) > 0 } - val supportsTapToWhiteBalance by lazy { (characteristics.get(CameraCharacteristics.CONTROL_MAX_REGIONS_AWB) ?: 0) > 0 } + val supportsFocusRegions by lazy { (characteristics.get(CameraCharacteristics.CONTROL_MAX_REGIONS_AF) ?: 0) > 0 } + val supportsExposureRegions by lazy { (characteristics.get(CameraCharacteristics.CONTROL_MAX_REGIONS_AE) ?: 0) > 0 } + val supportsWhiteBalanceRegions by lazy { (characteristics.get(CameraCharacteristics.CONTROL_MAX_REGIONS_AWB) ?: 0) > 0 } val afModes by lazy { characteristics.get(CameraCharacteristics.CONTROL_AF_AVAILABLE_MODES)?.toList() ?: emptyList() } val aeModes by lazy { characteristics.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_MODES)?.toList() ?: emptyList() } val awbModes by lazy { characteristics.get(CameraCharacteristics.CONTROL_AWB_AVAILABLE_MODES)?.toList() ?: emptyList() } + val availableAberrationModes by lazy { + characteristics.get(CameraCharacteristics.COLOR_CORRECTION_AVAILABLE_ABERRATION_MODES) + ?: intArrayOf() + } + val availableHotPixelModes by lazy { characteristics.get(CameraCharacteristics.HOT_PIXEL_AVAILABLE_HOT_PIXEL_MODES) ?: intArrayOf() } + val availableEdgeModes by lazy { characteristics.get(CameraCharacteristics.EDGE_AVAILABLE_EDGE_MODES) ?: intArrayOf() } + val availableDistortionCorrectionModes by lazy { getAvailableDistortionCorrectionModesOrEmptyArray() } + val availableShadingModes by lazy { characteristics.get(CameraCharacteristics.SHADING_AVAILABLE_MODES) ?: intArrayOf() } + val availableToneMapModes by lazy { characteristics.get(CameraCharacteristics.TONEMAP_AVAILABLE_TONE_MAP_MODES) ?: intArrayOf() } + val availableNoiseReductionModes by lazy { + characteristics.get(CameraCharacteristics.NOISE_REDUCTION_AVAILABLE_NOISE_REDUCTION_MODES) + ?: intArrayOf() + } + // TODO: Also add 10-bit YUV here? val videoFormat = ImageFormat.YUV_420_888 @@ -117,6 +131,13 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, val cameraId emptyList() } + private fun getAvailableDistortionCorrectionModesOrEmptyArray(): IntArray = + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { + characteristics.get(CameraCharacteristics.DISTORTION_CORRECTION_AVAILABLE_MODES) ?: intArrayOf() + } else { + intArrayOf() + } + private fun getHasVideoHdr(): Boolean { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { if (capabilities.contains(CameraMetadata.REQUEST_AVAILABLE_CAPABILITIES_DYNAMIC_RANGE_TEN_BIT)) { @@ -266,7 +287,7 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, val cameraId map.putBoolean("isMultiCam", isMultiCam) map.putBoolean("supportsRawCapture", supportsRawCapture) map.putBoolean("supportsLowLightBoost", supportsLowLightBoost) - map.putBoolean("supportsFocus", supportsTapToFocus) + map.putBoolean("supportsFocus", supportsFocusRegions) map.putDouble("minZoom", minZoom) map.putDouble("maxZoom", maxZoom) map.putDouble("neutralZoom", 1.0) // Zoom is always relative to 1.0 on Android diff --git a/package/android/src/main/java/com/mrousavy/camera/core/CameraError.kt b/package/android/src/main/java/com/mrousavy/camera/core/CameraError.kt index 838ee7b..6e7c8c6 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/CameraError.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/CameraError.kt @@ -104,6 +104,7 @@ class PhotoNotEnabledError : CameraError("capture", "photo-not-enabled", "Photo capture is disabled! Pass `photo={true}` to enable photo capture.") class CaptureAbortedError(wasImageCaptured: Boolean) : CameraError("capture", "aborted", "The image capture was aborted! Was Image captured: $wasImageCaptured") +class CaptureTimedOutError : CameraError("capture", "timed-out", "The image capture was aborted because it timed out.") class UnknownCaptureError(wasImageCaptured: Boolean) : CameraError("capture", "unknown", "An unknown error occurred while trying to capture an Image! Was Image captured: $wasImageCaptured") class RecorderError(name: String, extra: Int) : diff --git a/package/android/src/main/java/com/mrousavy/camera/core/PersistentCameraCaptureSession.kt b/package/android/src/main/java/com/mrousavy/camera/core/PersistentCameraCaptureSession.kt index 3b6e056..f953b23 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/PersistentCameraCaptureSession.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/PersistentCameraCaptureSession.kt @@ -7,17 +7,17 @@ import android.hardware.camera2.CameraDevice import android.hardware.camera2.CameraManager import android.hardware.camera2.CaptureRequest import android.hardware.camera2.TotalCaptureResult -import android.hardware.camera2.params.MeteringRectangle import android.util.Log -import android.util.Size import com.mrousavy.camera.core.capture.PhotoCaptureRequest import com.mrousavy.camera.core.capture.RepeatingCaptureRequest import com.mrousavy.camera.core.outputs.SurfaceOutput +import com.mrousavy.camera.extensions.PrecaptureOptions +import com.mrousavy.camera.extensions.PrecaptureTrigger import com.mrousavy.camera.extensions.capture import com.mrousavy.camera.extensions.createCaptureSession import com.mrousavy.camera.extensions.isValid import com.mrousavy.camera.extensions.openCamera -import com.mrousavy.camera.extensions.setRepeatingRequestAndWaitForAF +import com.mrousavy.camera.extensions.precapture import com.mrousavy.camera.extensions.tryAbortCaptures import com.mrousavy.camera.types.Flash import com.mrousavy.camera.types.Orientation @@ -40,7 +40,6 @@ import kotlinx.coroutines.sync.withLock class PersistentCameraCaptureSession(private val cameraManager: CameraManager, private val callback: Callback) : Closeable { companion object { private const val TAG = "PersistentCameraCaptureSession" - private val DEFAULT_METERING_SIZE = Size(100, 100) private const val FOCUS_RESET_TIMEOUT = 3000L } @@ -160,8 +159,34 @@ class PersistentCameraCaptureSession(private val cameraManager: CameraManager, p // Submit a single high-res capture to photo output as well as all preview outputs val outputs = outputs - val request = photoRequest.createCaptureRequest(device, deviceDetails, outputs) - return session.capture(request.build(), enableShutterSound) + val repeatingOutputs = outputs.filter { it.isRepeating } + + if (qualityPrioritization == QualityPrioritization.SPEED && flash == Flash.OFF) { + // 0. We want to take a picture as fast as possible, so skip any precapture sequence and just capture one Frame. + Log.i(TAG, "Using fast capture path without pre-capture sequence...") + val singleRequest = photoRequest.createCaptureRequest(device, deviceDetails, outputs) + return session.capture(singleRequest.build(), enableShutterSound) + } + + Log.i(TAG, "Locking AF/AE/AWB...") + + // 1. Run precapture sequence + val precaptureRequest = repeatingRequest.createCaptureRequest(device, deviceDetails, repeatingOutputs) + val options = PrecaptureOptions(listOf(PrecaptureTrigger.AF, PrecaptureTrigger.AE, PrecaptureTrigger.AWB), flash, emptyList()) + val result = session.precapture(precaptureRequest, deviceDetails, options) + + try { + // 2. Once precapture AF/AE/AWB successfully locked, capture the actual photo + val singleRequest = photoRequest.createCaptureRequest(device, deviceDetails, outputs) + if (result.needsFlash) { + singleRequest.set(CaptureRequest.FLASH_MODE, CaptureRequest.FLASH_MODE_SINGLE) + } + return session.capture(singleRequest.build(), enableShutterSound) + } finally { + // 5. After taking a photo we set the repeating request back to idle to remove the AE/AF/AWB locks again + val idleRequest = repeatingRequest.createCaptureRequest(device, deviceDetails, repeatingOutputs) + session.setRepeatingRequest(idleRequest.build(), null, null) + } } } @@ -172,66 +197,21 @@ class PersistentCameraCaptureSession(private val cameraManager: CameraManager, p val repeatingRequest = repeatingRequest ?: throw CameraNotReadyError() val device = session.device val deviceDetails = getOrCreateCameraDeviceDetails(device) - if (!deviceDetails.supportsTapToFocus) { + if (!deviceDetails.supportsFocusRegions) { throw FocusNotSupportedError() } val outputs = outputs.filter { it.isRepeating } - val meteringRectangle = MeteringRectangle(point, DEFAULT_METERING_SIZE, MeteringRectangle.METERING_WEIGHT_MAX - 1) // 0. Cancel the 3 second focus reset task focusResetJob?.cancelAndJoin() focusResetJob = null - // 1. Cancel any ongoing AF/AE/AWB request - repeatingRequest.createCaptureRequest(device, deviceDetails, outputs).also { request -> - if (deviceDetails.supportsTapToFocus) { - request.set(CaptureRequest.CONTROL_AF_TRIGGER, CaptureRequest.CONTROL_AF_TRIGGER_CANCEL) - } - if (deviceDetails.supportsTapToExposure) { - request.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER_CANCEL) - } - session.capture(request.build(), null, null) - } + // 1. Run a precapture sequence for AF, AE and AWB. + val request = repeatingRequest.createCaptureRequest(device, deviceDetails, outputs) + val options = PrecaptureOptions(listOf(PrecaptureTrigger.AF, PrecaptureTrigger.AE), Flash.OFF, listOf(point)) + session.precapture(request, deviceDetails, options) - // 2. After previous AF/AE/AWB requests have been canceled, start a new AF/AE/AWB request - repeatingRequest.createCaptureRequest(device, deviceDetails, outputs).also { request -> - request.set(CaptureRequest.CONTROL_MODE, CaptureRequest.CONTROL_MODE_AUTO) - if (deviceDetails.supportsTapToFocus) { - request.set(CaptureRequest.CONTROL_AF_MODE, CaptureRequest.CONTROL_AF_MODE_AUTO) - request.set(CaptureRequest.CONTROL_AF_REGIONS, arrayOf(meteringRectangle)) - request.set(CaptureRequest.CONTROL_AF_TRIGGER, CaptureRequest.CONTROL_AF_TRIGGER_START) - } - if (deviceDetails.supportsTapToExposure) { - request.set(CaptureRequest.CONTROL_AE_REGIONS, arrayOf(meteringRectangle)) - request.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER_START) - } - if (deviceDetails.supportsTapToWhiteBalance) { - request.set(CaptureRequest.CONTROL_AWB_REGIONS, arrayOf(meteringRectangle)) - } - session.capture(request.build(), null, null) - - // 3. Start a repeating request without the trigger and wait until AF/AE/AWB locks - request.set(CaptureRequest.CONTROL_AF_TRIGGER, null) - request.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, null) - session.setRepeatingRequestAndWaitForAF(request.build()) - } - - // 4. After the Camera has successfully found the AF/AE/AWB lock-point, we set it to idle and keep the point metered - repeatingRequest.createCaptureRequest(device, deviceDetails, outputs).also { request -> - request.set(CaptureRequest.CONTROL_MODE, CaptureRequest.CONTROL_MODE_AUTO) - if (deviceDetails.supportsTapToFocus) { - request.set(CaptureRequest.CONTROL_AF_MODE, CaptureRequest.CONTROL_AF_MODE_AUTO) - request.set(CaptureRequest.CONTROL_AF_REGIONS, arrayOf(meteringRectangle)) - request.set(CaptureRequest.CONTROL_AF_TRIGGER, CaptureRequest.CONTROL_AF_TRIGGER_IDLE) - } - if (deviceDetails.supportsTapToExposure) { - request.set(CaptureRequest.CONTROL_AE_REGIONS, arrayOf(meteringRectangle)) - request.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER_IDLE) - } - session.setRepeatingRequest(request.build(), null, null) - } - - // 5. Wait 3 seconds + // 2. Wait 3 seconds focusResetJob = coroutineScope.launch { delay(FOCUS_RESET_TIMEOUT) if (!this.isActive) { @@ -243,7 +223,7 @@ class PersistentCameraCaptureSession(private val cameraManager: CameraManager, p return@launch } Log.i(TAG, "Resetting focus to auto-focus...") - // 6. Reset AF/AE/AWB to continuous auto-focus again, which is the default here. + // 3. Reset AF/AE/AWB to continuous auto-focus again, which is the default here. repeatingRequest.createCaptureRequest(device, deviceDetails, outputs).also { request -> session.setRepeatingRequest(request.build(), null, null) } diff --git a/package/android/src/main/java/com/mrousavy/camera/core/capture/PhotoCaptureRequest.kt b/package/android/src/main/java/com/mrousavy/camera/core/capture/PhotoCaptureRequest.kt index 82b8dc0..cfb5ae0 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/capture/PhotoCaptureRequest.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/capture/PhotoCaptureRequest.kt @@ -3,10 +3,12 @@ package com.mrousavy.camera.core.capture import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraDevice import android.hardware.camera2.CaptureRequest +import android.os.Build import android.util.Log import com.mrousavy.camera.core.CameraDeviceDetails import com.mrousavy.camera.core.outputs.SurfaceOutput import com.mrousavy.camera.types.Flash +import com.mrousavy.camera.types.HardwareLevel import com.mrousavy.camera.types.Orientation import com.mrousavy.camera.types.QualityPrioritization import com.mrousavy.camera.types.Torch @@ -67,13 +69,70 @@ class PhotoCaptureRequest( ): CaptureRequest.Builder { val builder = super.createCaptureRequest(template, device, deviceDetails, outputs) - // Set JPEG quality - val jpegQuality = when (qualityPrioritization) { - QualityPrioritization.SPEED -> 85 - QualityPrioritization.BALANCED -> 92 - QualityPrioritization.QUALITY -> 100 + // Set various speed vs quality optimization flags + when (qualityPrioritization) { + QualityPrioritization.SPEED -> { + if (deviceDetails.hardwareLevel.isAtLeast(HardwareLevel.FULL)) { + builder.set(CaptureRequest.COLOR_CORRECTION_MODE, CaptureRequest.COLOR_CORRECTION_MODE_FAST) + if (deviceDetails.availableEdgeModes.contains(CaptureRequest.EDGE_MODE_FAST)) { + builder.set(CaptureRequest.EDGE_MODE, CaptureRequest.EDGE_MODE_FAST) + } + } + if (deviceDetails.availableAberrationModes.contains(CaptureRequest.COLOR_CORRECTION_ABERRATION_MODE_FAST)) { + builder.set(CaptureRequest.COLOR_CORRECTION_ABERRATION_MODE, CaptureRequest.COLOR_CORRECTION_ABERRATION_MODE_FAST) + } + if (deviceDetails.availableHotPixelModes.contains(CaptureRequest.HOT_PIXEL_MODE_FAST)) { + builder.set(CaptureRequest.HOT_PIXEL_MODE, CaptureRequest.HOT_PIXEL_MODE_FAST) + } + if (deviceDetails.availableDistortionCorrectionModes.contains(CaptureRequest.DISTORTION_CORRECTION_MODE_FAST) && + Build.VERSION.SDK_INT >= Build.VERSION_CODES.P + ) { + builder.set(CaptureRequest.DISTORTION_CORRECTION_MODE, CaptureRequest.DISTORTION_CORRECTION_MODE_FAST) + } + if (deviceDetails.availableNoiseReductionModes.contains(CaptureRequest.NOISE_REDUCTION_MODE_FAST)) { + builder.set(CaptureRequest.NOISE_REDUCTION_MODE, CaptureRequest.NOISE_REDUCTION_MODE_FAST) + } + if (deviceDetails.availableShadingModes.contains(CaptureRequest.SHADING_MODE_FAST)) { + builder.set(CaptureRequest.SHADING_MODE, CaptureRequest.SHADING_MODE_FAST) + } + if (deviceDetails.availableToneMapModes.contains(CaptureRequest.TONEMAP_MODE_FAST)) { + builder.set(CaptureRequest.TONEMAP_MODE, CaptureRequest.TONEMAP_MODE_FAST) + } + builder.set(CaptureRequest.JPEG_QUALITY, 85) + } + QualityPrioritization.BALANCED -> { + builder.set(CaptureRequest.JPEG_QUALITY, 92) + } + QualityPrioritization.QUALITY -> { + if (deviceDetails.hardwareLevel.isAtLeast(HardwareLevel.FULL)) { + builder.set(CaptureRequest.COLOR_CORRECTION_MODE, CaptureRequest.COLOR_CORRECTION_MODE_HIGH_QUALITY) + if (deviceDetails.availableEdgeModes.contains(CaptureRequest.EDGE_MODE_HIGH_QUALITY)) { + builder.set(CaptureRequest.EDGE_MODE, CaptureRequest.EDGE_MODE_HIGH_QUALITY) + } + } + if (deviceDetails.availableAberrationModes.contains(CaptureRequest.COLOR_CORRECTION_ABERRATION_MODE_HIGH_QUALITY)) { + builder.set(CaptureRequest.COLOR_CORRECTION_ABERRATION_MODE, CaptureRequest.COLOR_CORRECTION_ABERRATION_MODE_HIGH_QUALITY) + } + if (deviceDetails.availableHotPixelModes.contains(CaptureRequest.HOT_PIXEL_MODE_HIGH_QUALITY)) { + builder.set(CaptureRequest.HOT_PIXEL_MODE, CaptureRequest.HOT_PIXEL_MODE_HIGH_QUALITY) + } + if (deviceDetails.availableDistortionCorrectionModes.contains(CaptureRequest.DISTORTION_CORRECTION_MODE_HIGH_QUALITY) && + Build.VERSION.SDK_INT >= Build.VERSION_CODES.P + ) { + builder.set(CaptureRequest.DISTORTION_CORRECTION_MODE, CaptureRequest.DISTORTION_CORRECTION_MODE_HIGH_QUALITY) + } + if (deviceDetails.availableNoiseReductionModes.contains(CaptureRequest.NOISE_REDUCTION_MODE_HIGH_QUALITY)) { + builder.set(CaptureRequest.NOISE_REDUCTION_MODE, CaptureRequest.NOISE_REDUCTION_MODE_HIGH_QUALITY) + } + if (deviceDetails.availableShadingModes.contains(CaptureRequest.SHADING_MODE_HIGH_QUALITY)) { + builder.set(CaptureRequest.SHADING_MODE, CaptureRequest.SHADING_MODE_HIGH_QUALITY) + } + if (deviceDetails.availableToneMapModes.contains(CaptureRequest.TONEMAP_MODE_HIGH_QUALITY)) { + builder.set(CaptureRequest.TONEMAP_MODE, CaptureRequest.TONEMAP_MODE_HIGH_QUALITY) + } + builder.set(CaptureRequest.JPEG_QUALITY, 100) + } } - builder.set(CaptureRequest.JPEG_QUALITY, jpegQuality.toByte()) // Set JPEG Orientation val targetOrientation = outputOrientation.toSensorRelativeOrientation(deviceDetails) diff --git a/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+capture.kt b/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+capture.kt index a3dce45..4ff3e3f 100644 --- a/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+capture.kt +++ b/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+capture.kt @@ -5,17 +5,34 @@ import android.hardware.camera2.CaptureFailure import android.hardware.camera2.CaptureRequest import android.hardware.camera2.TotalCaptureResult import android.media.MediaActionSound +import android.util.Log import com.mrousavy.camera.core.CaptureAbortedError +import com.mrousavy.camera.core.CaptureTimedOutError import com.mrousavy.camera.core.UnknownCaptureError import kotlin.coroutines.resume import kotlin.coroutines.resumeWithException -import kotlin.coroutines.suspendCoroutine +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.coroutines.suspendCancellableCoroutine + +private const val TAG = "CameraCaptureSession" suspend fun CameraCaptureSession.capture(captureRequest: CaptureRequest, enableShutterSound: Boolean): TotalCaptureResult = - suspendCoroutine { continuation -> + suspendCancellableCoroutine { continuation -> val shutterSound = if (enableShutterSound) MediaActionSound() else null shutterSound?.load(MediaActionSound.SHUTTER_CLICK) + CoroutineScope(Dispatchers.Default).launch { + delay(5000) // after 5s, cancel capture + if (continuation.isActive) { + Log.e(TAG, "Capture timed out after 5 seconds!") + continuation.resumeWithException(CaptureTimedOutError()) + tryAbortCaptures() + } + } + this.capture( captureRequest, object : CameraCaptureSession.CaptureCallback() { diff --git a/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+precapture.kt b/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+precapture.kt new file mode 100644 index 0000000..0372e05 --- /dev/null +++ b/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+precapture.kt @@ -0,0 +1,99 @@ +package com.mrousavy.camera.extensions + +import android.graphics.Point +import android.hardware.camera2.CameraCaptureSession +import android.hardware.camera2.CaptureRequest +import android.hardware.camera2.CaptureResult +import android.hardware.camera2.params.MeteringRectangle +import android.util.Log +import android.util.Size +import com.mrousavy.camera.core.CameraDeviceDetails +import com.mrousavy.camera.types.Flash +import com.mrousavy.camera.types.HardwareLevel + +data class PrecaptureOptions(val modes: List, val flash: Flash = Flash.OFF, val pointsOfInterest: List) + +data class PrecaptureResult(val needsFlash: Boolean) + +private const val TAG = "Precapture" +private val DEFAULT_METERING_SIZE = Size(100, 100) + +/** + * Run a precapture sequence to trigger an AF, AE or AWB scan and lock to the optimal values. + * After this function completes, you can capture high quality photos as AF/AE/AWB are in focused state. + * + * To reset to auto-focus again, create a new `RepeatingRequest` with a fresh set of CONTROL_MODEs set. + */ +suspend fun CameraCaptureSession.precapture( + request: CaptureRequest.Builder, + deviceDetails: CameraDeviceDetails, + options: PrecaptureOptions +): PrecaptureResult { + Log.i(TAG, "Running precapture sequence... ($options)") + request.set(CaptureRequest.CONTROL_MODE, CaptureRequest.CONTROL_MODE_AUTO) + + var enableFlash = options.flash == Flash.ON + + // 1. Cancel any ongoing precapture sequences + request.set(CaptureRequest.CONTROL_AF_TRIGGER, CaptureRequest.CONTROL_AF_TRIGGER_CANCEL) + request.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER_CANCEL) + if (options.flash == Flash.AUTO) { + // we want Auto-Flash, so check the current lighting conditions if we need it. + val result = this.capture(request.build(), false) + val aeState = result.get(CaptureResult.CONTROL_AE_STATE) + if (aeState == CaptureResult.CONTROL_AE_STATE_FLASH_REQUIRED) { + Log.i(TAG, "Auto-Flash: Flash is required for photo capture, enabling flash...") + enableFlash = true + } else { + Log.i(TAG, "Auto-Flash: Flash is not required for photo capture.") + } + } else { + // we either want Flash ON or OFF, so we don't care about lighting conditions - do a fast capture. + this.capture(request.build(), null, null) + } + + val meteringWeight = MeteringRectangle.METERING_WEIGHT_MAX - 1 + val meteringRectangles = options.pointsOfInterest.map { point -> + MeteringRectangle(point, DEFAULT_METERING_SIZE, meteringWeight) + }.toTypedArray() + + // 2. Submit a precapture start sequence + if (enableFlash && deviceDetails.hasFlash) { + request.set(CaptureRequest.FLASH_MODE, CaptureRequest.FLASH_MODE_TORCH) + } + if (options.modes.contains(PrecaptureTrigger.AF)) { + // AF Precapture + if (deviceDetails.afModes.contains(CaptureRequest.CONTROL_AF_MODE_AUTO)) { + request.set(CaptureRequest.CONTROL_AF_MODE, CaptureRequest.CONTROL_AF_MODE_AUTO) + } + if (meteringRectangles.isNotEmpty() && deviceDetails.supportsFocusRegions) { + request.set(CaptureRequest.CONTROL_AF_REGIONS, meteringRectangles) + } + request.set(CaptureRequest.CONTROL_AF_TRIGGER, CaptureRequest.CONTROL_AF_TRIGGER_START) + } + if (options.modes.contains(PrecaptureTrigger.AE) && deviceDetails.hardwareLevel.isAtLeast(HardwareLevel.LIMITED)) { + // AE Precapture + if (meteringRectangles.isNotEmpty() && deviceDetails.supportsExposureRegions) { + request.set(CaptureRequest.CONTROL_AE_REGIONS, meteringRectangles) + } + request.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER_START) + } + if (options.modes.contains(PrecaptureTrigger.AWB)) { + // AWB Precapture + if (meteringRectangles.isNotEmpty() && deviceDetails.supportsWhiteBalanceRegions) { + request.set(CaptureRequest.CONTROL_AWB_REGIONS, meteringRectangles) + } + } + this.capture(request.build(), null, null) + + // 3. Start a repeating request without the trigger and wait until AF/AE/AWB locks + request.set(CaptureRequest.CONTROL_AF_TRIGGER, null) + request.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, null) + val result = this.setRepeatingRequestAndWaitForPrecapture(request.build(), *options.modes.toTypedArray()) + + Log.i(TAG, "AF/AE/AWB successfully locked!") + // TODO: Set to idle again? + + val needsFlash = result.exposureState == ExposureState.FlashRequired + return PrecaptureResult(needsFlash) +} diff --git a/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+setRepeatingRequestAndWaitForAF.kt b/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+setRepeatingRequestAndWaitForAF.kt deleted file mode 100644 index e085a02..0000000 --- a/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+setRepeatingRequestAndWaitForAF.kt +++ /dev/null @@ -1,47 +0,0 @@ -package com.mrousavy.camera.extensions - -import android.hardware.camera2.CameraCaptureSession -import android.hardware.camera2.CaptureFailure -import android.hardware.camera2.CaptureRequest -import android.hardware.camera2.CaptureResult -import android.hardware.camera2.TotalCaptureResult -import android.util.Log -import com.mrousavy.camera.core.CaptureAbortedError -import kotlin.coroutines.resume -import kotlin.coroutines.resumeWithException -import kotlinx.coroutines.suspendCancellableCoroutine - -private const val TAG = "CameraCaptureSession" - -/** - * Set a new repeating request for the [CameraCaptureSession] that contains an AF trigger, and wait until AF has locked. - */ -suspend fun CameraCaptureSession.setRepeatingRequestAndWaitForAF(request: CaptureRequest) = - suspendCancellableCoroutine { continuation -> - this.setRepeatingRequest( - request, - object : CameraCaptureSession.CaptureCallback() { - override fun onCaptureCompleted(session: CameraCaptureSession, request: CaptureRequest, result: TotalCaptureResult) { - super.onCaptureCompleted(session, request, result) - - if (continuation.isActive) { - val afState = result.get(CaptureResult.CONTROL_AF_STATE) - Log.i(TAG, "AF State: $afState") - if (afState == CaptureResult.CONTROL_AF_STATE_FOCUSED_LOCKED || afState == CaptureResult.CONTROL_AF_STATE_NOT_FOCUSED_LOCKED) { - continuation.resume(Unit) - session.setRepeatingRequest(request, null, null) - } - } - } - override fun onCaptureFailed(session: CameraCaptureSession, request: CaptureRequest, failure: CaptureFailure) { - super.onCaptureFailed(session, request, failure) - - if (continuation.isActive) { - continuation.resumeWithException(CaptureAbortedError(failure.wasImageCaptured())) - session.setRepeatingRequest(request, null, null) - } - } - }, - null - ) - } diff --git a/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+setRepeatingRequestAndWaitForPrecapture.kt b/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+setRepeatingRequestAndWaitForPrecapture.kt new file mode 100644 index 0000000..a655657 --- /dev/null +++ b/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+setRepeatingRequestAndWaitForPrecapture.kt @@ -0,0 +1,178 @@ +package com.mrousavy.camera.extensions + +import android.hardware.camera2.CameraCaptureSession +import android.hardware.camera2.CaptureFailure +import android.hardware.camera2.CaptureRequest +import android.hardware.camera2.CaptureResult +import android.hardware.camera2.TotalCaptureResult +import android.util.Log +import com.mrousavy.camera.core.CaptureAbortedError +import com.mrousavy.camera.core.CaptureTimedOutError +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.launch +import kotlinx.coroutines.suspendCancellableCoroutine + +private const val TAG = "CameraCaptureSession" + +enum class PrecaptureTrigger { + AE, + AF, + AWB +} + +enum class FocusState { + Inactive, + Scanning, + Focused, + Unfocused, + PassiveScanning, + PassiveFocused, + PassiveUnfocused; + + val isCompleted: Boolean + get() = this == Focused || this == Unfocused + + companion object { + fun fromAFState(afState: Int): FocusState = + when (afState) { + CaptureResult.CONTROL_AF_STATE_INACTIVE -> Inactive + CaptureResult.CONTROL_AF_STATE_ACTIVE_SCAN -> Scanning + CaptureResult.CONTROL_AF_STATE_FOCUSED_LOCKED -> Focused + CaptureResult.CONTROL_AF_STATE_NOT_FOCUSED_LOCKED -> Unfocused + CaptureResult.CONTROL_AF_STATE_PASSIVE_SCAN -> PassiveScanning + CaptureResult.CONTROL_AF_STATE_PASSIVE_FOCUSED -> PassiveFocused + CaptureResult.CONTROL_AF_STATE_PASSIVE_UNFOCUSED -> PassiveUnfocused + else -> throw Error("Invalid CONTROL_AF_STATE! $afState") + } + } +} +enum class ExposureState { + Locked, + Inactive, + Precapture, + Searching, + Converged, + FlashRequired; + + val isCompleted: Boolean + get() = this == Converged || this == FlashRequired + + companion object { + fun fromAEState(aeState: Int): ExposureState = + when (aeState) { + CaptureResult.CONTROL_AE_STATE_INACTIVE -> Inactive + CaptureResult.CONTROL_AE_STATE_SEARCHING -> Searching + CaptureResult.CONTROL_AE_STATE_PRECAPTURE -> Precapture + CaptureResult.CONTROL_AE_STATE_CONVERGED -> Converged + CaptureResult.CONTROL_AE_STATE_FLASH_REQUIRED -> FlashRequired + CaptureResult.CONTROL_AE_STATE_LOCKED -> Locked + else -> throw Error("Invalid CONTROL_AE_STATE! $aeState") + } + } +} + +enum class WhiteBalanceState { + Inactive, + Locked, + Searching, + Converged; + + val isCompleted: Boolean + get() = this == Converged + + companion object { + fun fromAWBState(awbState: Int): WhiteBalanceState = + when (awbState) { + CaptureResult.CONTROL_AWB_STATE_INACTIVE -> Inactive + CaptureResult.CONTROL_AWB_STATE_SEARCHING -> Searching + CaptureResult.CONTROL_AWB_STATE_CONVERGED -> Converged + CaptureResult.CONTROL_AWB_STATE_LOCKED -> Locked + else -> throw Error("Invalid CONTROL_AWB_STATE! $awbState") + } + } +} + +data class ResultState(val focusState: FocusState, val exposureState: ExposureState, val whiteBalanceState: WhiteBalanceState) + +/** + * Set a new repeating request for the [CameraCaptureSession] that contains a precapture trigger, and wait until the given precaptures have locked. + */ +suspend fun CameraCaptureSession.setRepeatingRequestAndWaitForPrecapture( + request: CaptureRequest, + vararg precaptureTriggers: PrecaptureTrigger +): ResultState = + suspendCancellableCoroutine { continuation -> + // Map of all completed precaptures + val completed = precaptureTriggers.associateWith { false }.toMutableMap() + + CoroutineScope(Dispatchers.Default).launch { + delay(5000) // after 5s, cancel capture + if (continuation.isActive) { + Log.e(TAG, "Precapture timed out after 5 seconds!") + continuation.resumeWithException(CaptureTimedOutError()) + try { + setRepeatingRequest(request, null, null) + } catch (e: Throwable) { + // session might have already been closed + Log.e(TAG, "Error resetting session repeating request..", e) + } + } + } + + this.setRepeatingRequest( + request, + object : CameraCaptureSession.CaptureCallback() { + override fun onCaptureCompleted(session: CameraCaptureSession, request: CaptureRequest, result: TotalCaptureResult) { + super.onCaptureCompleted(session, request, result) + + if (continuation.isActive) { + // AF Precapture + val afState = FocusState.fromAFState(result.get(CaptureResult.CONTROL_AF_STATE) ?: CaptureResult.CONTROL_AF_STATE_INACTIVE) + val aeState = ExposureState.fromAEState(result.get(CaptureResult.CONTROL_AE_STATE) ?: CaptureResult.CONTROL_AE_STATE_INACTIVE) + val awbState = WhiteBalanceState.fromAWBState( + result.get(CaptureResult.CONTROL_AWB_STATE) ?: CaptureResult.CONTROL_AWB_STATE_INACTIVE + ) + + if (precaptureTriggers.contains(PrecaptureTrigger.AF)) { + Log.i(TAG, "AF State: $afState (isCompleted: ${afState.isCompleted})") + completed[PrecaptureTrigger.AF] = afState.isCompleted + } + // AE Precapture + if (precaptureTriggers.contains(PrecaptureTrigger.AE)) { + Log.i(TAG, "AE State: $aeState (isCompleted: ${aeState.isCompleted})") + completed[PrecaptureTrigger.AE] = aeState.isCompleted + } + // AWB Precapture + if (precaptureTriggers.contains(PrecaptureTrigger.AWB)) { + Log.i(TAG, "AWB State: $awbState (isCompleted: ${awbState.isCompleted})") + completed[PrecaptureTrigger.AWB] = awbState.isCompleted + } + + if (completed.values.all { it == true }) { + // All precaptures did complete! + continuation.resume(ResultState(afState, aeState, awbState)) + session.setRepeatingRequest(request, null, null) + } + } + } + override fun onCaptureFailed(session: CameraCaptureSession, request: CaptureRequest, failure: CaptureFailure) { + super.onCaptureFailed(session, request, failure) + + if (continuation.isActive) { + // Capture failed or session closed. + continuation.resumeWithException(CaptureAbortedError(failure.wasImageCaptured())) + try { + session.setRepeatingRequest(request, null, null) + } catch (e: Throwable) { + Log.e(TAG, "Failed to continue repeating request!", e) + } + } + } + }, + null + ) + } diff --git a/package/android/src/main/java/com/mrousavy/camera/types/HardwareLevel.kt b/package/android/src/main/java/com/mrousavy/camera/types/HardwareLevel.kt index 2df2049..ca34ea6 100644 --- a/package/android/src/main/java/com/mrousavy/camera/types/HardwareLevel.kt +++ b/package/android/src/main/java/com/mrousavy/camera/types/HardwareLevel.kt @@ -9,6 +9,19 @@ enum class HardwareLevel(override val unionValue: String) : JSUnionValue { FULL("full"), LEVEL_3("full"); + private val rank: Int + get() { + return when (this) { + LEGACY -> 0 + LIMITED -> 1 + EXTERNAL -> 1 + FULL -> 2 + LEVEL_3 -> 3 + } + } + + fun isAtLeast(level: HardwareLevel): Boolean = this.rank >= level.rank + companion object { fun fromCameraCharacteristics(cameraCharacteristics: CameraCharacteristics): HardwareLevel = when (cameraCharacteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL)) { diff --git a/package/example/src/views/CaptureButton.tsx b/package/example/src/views/CaptureButton.tsx index 16da7bf..e4a91c8 100644 --- a/package/example/src/views/CaptureButton.tsx +++ b/package/example/src/views/CaptureButton.tsx @@ -1,4 +1,4 @@ -import React, { useCallback, useMemo, useRef } from 'react' +import React, { useCallback, useRef } from 'react' import { StyleSheet, View, ViewProps } from 'react-native' import { PanGestureHandler, @@ -19,7 +19,7 @@ import Reanimated, { useSharedValue, withRepeat, } from 'react-native-reanimated' -import type { Camera, PhotoFile, TakePhotoOptions, VideoFile } from 'react-native-vision-camera' +import type { Camera, PhotoFile, VideoFile } from 'react-native-vision-camera' import { CAPTURE_BUTTON_SIZE, SCREEN_HEIGHT, SCREEN_WIDTH } from './../Constants' const PAN_GESTURE_HANDLER_FAIL_X = [-SCREEN_WIDTH, SCREEN_WIDTH] @@ -58,15 +58,6 @@ const _CaptureButton: React.FC = ({ const pressDownDate = useRef(undefined) const isRecording = useRef(false) const recordingProgress = useSharedValue(0) - const takePhotoOptions = useMemo( - () => ({ - qualityPrioritization: 'speed', - flash: flash, - quality: 90, - enableShutterSound: false, - }), - [flash], - ) const isPressingButton = useSharedValue(false) //#region Camera Capture @@ -75,12 +66,16 @@ const _CaptureButton: React.FC = ({ if (camera.current == null) throw new Error('Camera ref is null!') console.log('Taking photo...') - const photo = await camera.current.takePhoto(takePhotoOptions) + const photo = await camera.current.takePhoto({ + qualityPrioritization: 'quality', + flash: flash, + enableShutterSound: false, + }) onMediaCaptured(photo, 'photo') } catch (e) { console.error('Failed to take photo!', e) } - }, [camera, onMediaCaptured, takePhotoOptions]) + }, [camera, flash, onMediaCaptured]) const onStoppedRecording = useCallback(() => { isRecording.current = false diff --git a/package/src/CameraError.ts b/package/src/CameraError.ts index e39beb5..f47f61f 100644 --- a/package/src/CameraError.ts +++ b/package/src/CameraError.ts @@ -43,6 +43,7 @@ export type CaptureError = | 'capture/photo-not-enabled' | 'capture/frame-invalid' | 'capture/aborted' + | 'capture/timed-out' | 'capture/unknown' export type SystemError = | 'system/camera-module-not-found'