feat: Flash with AE Pre-capture trigger for Android (#2558)

### Flash (`flash`) Adds `flash` functionality with a fully custom pre-capture AE/AF/AWB trigger sequence for Android. 🎉 ```ts camera.current.takePhoto({ flash: 'on' // or 'auto' }) ``` ### Better photos (`qualityPrioritization`) We now also run the AE/AF/AWB precapture sequence on every photo (unless `qualityPrioritization` is `speed`), meaning photos are now less blurry, properly exposed, and properly white-balanced - so in short: **photo quality is now better!**. The fast path still exists when using `qualityPrioritization: speed`, as that will skip the precapture sequence and metering actions and just grab an Image from the Camera as quickly as possible. Additionally, `qualityPrioritization` now controls these options: - [COLOR_CORRECTION_MODE](https://developer.android.com/reference/android/hardware/camera2/CaptureRequest#COLOR_CORRECTION_MODE) - [EDGE_MODE](https://developer.android.com/reference/android/hardware/camera2/CaptureRequest#EDGE_MODE) - [COLOR_CORRECTION_ABERRATION_MODE](https://developer.android.com/reference/android/hardware/camera2/CaptureRequest#COLOR_CORRECTION_ABERRATION_MODE) - [HOT_PIXEL_MODE](https://developer.android.com/reference/android/hardware/camera2/CaptureRequest#HOT_PIXEL_MODE) - [DISTORTION_CORRECTION_MODE](https://developer.android.com/reference/android/hardware/camera2/CaptureRequest#DISTORTION_CORRECTION_MODE) - [NOISE_REDUCTION_MODE](https://developer.android.com/reference/android/hardware/camera2/CaptureRequest#NOISE_REDUCTION_MODE) - [SHADING_MODE](https://developer.android.com/reference/android/hardware/camera2/CaptureRequest#SHADING_MODE) - [TONEMAP_MODE](https://developer.android.com/reference/android/hardware/camera2/CaptureRequest#TONEMAP_MODE) ..by setting them to `_FAST` or `_HIGH_QUALITY`, which was previously left untouched. This now means: - `takePhoto({ qualityPrioritization: 'speed' })` got FASTER 🚀 - `takePhoto({ qualityPrioritization: 'quality' })` got BETTER QUALITY 📸 - `takePhoto({ qualityPrioritization: 'balanced' })` is left unchanged ✅
2024-02-14 16:43:30 +01:00
parent 61b2f7dd4a
commit 37398cc909
11 changed files with 448 additions and 131 deletions
--- a/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+capture.kt
+++ b/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+capture.kt
@@ -5,17 +5,34 @@ import android.hardware.camera2.CaptureFailure
 import android.hardware.camera2.CaptureRequest
 import android.hardware.camera2.TotalCaptureResult
 import android.media.MediaActionSound
+import android.util.Log
 import com.mrousavy.camera.core.CaptureAbortedError
+import com.mrousavy.camera.core.CaptureTimedOutError
 import com.mrousavy.camera.core.UnknownCaptureError
 import kotlin.coroutines.resume
 import kotlin.coroutines.resumeWithException
-import kotlin.coroutines.suspendCoroutine
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.suspendCancellableCoroutine
+
+private const val TAG = "CameraCaptureSession"

 suspend fun CameraCaptureSession.capture(captureRequest: CaptureRequest, enableShutterSound: Boolean): TotalCaptureResult =
-  suspendCoroutine { continuation ->
+  suspendCancellableCoroutine { continuation ->
    val shutterSound = if (enableShutterSound) MediaActionSound() else null
    shutterSound?.load(MediaActionSound.SHUTTER_CLICK)

+    CoroutineScope(Dispatchers.Default).launch {
+      delay(5000) // after 5s, cancel capture
+      if (continuation.isActive) {
+        Log.e(TAG, "Capture timed out after 5 seconds!")
+        continuation.resumeWithException(CaptureTimedOutError())
+        tryAbortCaptures()
+      }
+    }
+
    this.capture(
      captureRequest,
      object : CameraCaptureSession.CaptureCallback() {
--- a/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+precapture.kt
+++ b/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+precapture.kt
@@ -0,0 +1,99 @@
+package com.mrousavy.camera.extensions
+
+import android.graphics.Point
+import android.hardware.camera2.CameraCaptureSession
+import android.hardware.camera2.CaptureRequest
+import android.hardware.camera2.CaptureResult
+import android.hardware.camera2.params.MeteringRectangle
+import android.util.Log
+import android.util.Size
+import com.mrousavy.camera.core.CameraDeviceDetails
+import com.mrousavy.camera.types.Flash
+import com.mrousavy.camera.types.HardwareLevel
+
+data class PrecaptureOptions(val modes: List<PrecaptureTrigger>, val flash: Flash = Flash.OFF, val pointsOfInterest: List<Point>)
+
+data class PrecaptureResult(val needsFlash: Boolean)
+
+private const val TAG = "Precapture"
+private val DEFAULT_METERING_SIZE = Size(100, 100)
+
+/**
+ * Run a precapture sequence to trigger an AF, AE or AWB scan and lock to the optimal values.
+ * After this function completes, you can capture high quality photos as AF/AE/AWB are in focused state.
+ *
+ * To reset to auto-focus again, create a new `RepeatingRequest` with a fresh set of CONTROL_MODEs set.
+ */
+suspend fun CameraCaptureSession.precapture(
+  request: CaptureRequest.Builder,
+  deviceDetails: CameraDeviceDetails,
+  options: PrecaptureOptions
+): PrecaptureResult {
+  Log.i(TAG, "Running precapture sequence... ($options)")
+  request.set(CaptureRequest.CONTROL_MODE, CaptureRequest.CONTROL_MODE_AUTO)
+
+  var enableFlash = options.flash == Flash.ON
+
+  // 1. Cancel any ongoing precapture sequences
+  request.set(CaptureRequest.CONTROL_AF_TRIGGER, CaptureRequest.CONTROL_AF_TRIGGER_CANCEL)
+  request.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER_CANCEL)
+  if (options.flash == Flash.AUTO) {
+    // we want Auto-Flash, so check the current lighting conditions if we need it.
+    val result = this.capture(request.build(), false)
+    val aeState = result.get(CaptureResult.CONTROL_AE_STATE)
+    if (aeState == CaptureResult.CONTROL_AE_STATE_FLASH_REQUIRED) {
+      Log.i(TAG, "Auto-Flash: Flash is required for photo capture, enabling flash...")
+      enableFlash = true
+    } else {
+      Log.i(TAG, "Auto-Flash: Flash is not required for photo capture.")
+    }
+  } else {
+    // we either want Flash ON or OFF, so we don't care about lighting conditions - do a fast capture.
+    this.capture(request.build(), null, null)
+  }
+
+  val meteringWeight = MeteringRectangle.METERING_WEIGHT_MAX - 1
+  val meteringRectangles = options.pointsOfInterest.map { point ->
+    MeteringRectangle(point, DEFAULT_METERING_SIZE, meteringWeight)
+  }.toTypedArray()
+
+  // 2. Submit a precapture start sequence
+  if (enableFlash && deviceDetails.hasFlash) {
+    request.set(CaptureRequest.FLASH_MODE, CaptureRequest.FLASH_MODE_TORCH)
+  }
+  if (options.modes.contains(PrecaptureTrigger.AF)) {
+    // AF Precapture
+    if (deviceDetails.afModes.contains(CaptureRequest.CONTROL_AF_MODE_AUTO)) {
+      request.set(CaptureRequest.CONTROL_AF_MODE, CaptureRequest.CONTROL_AF_MODE_AUTO)
+    }
+    if (meteringRectangles.isNotEmpty() && deviceDetails.supportsFocusRegions) {
+      request.set(CaptureRequest.CONTROL_AF_REGIONS, meteringRectangles)
+    }
+    request.set(CaptureRequest.CONTROL_AF_TRIGGER, CaptureRequest.CONTROL_AF_TRIGGER_START)
+  }
+  if (options.modes.contains(PrecaptureTrigger.AE) && deviceDetails.hardwareLevel.isAtLeast(HardwareLevel.LIMITED)) {
+    // AE Precapture
+    if (meteringRectangles.isNotEmpty() && deviceDetails.supportsExposureRegions) {
+      request.set(CaptureRequest.CONTROL_AE_REGIONS, meteringRectangles)
+    }
+    request.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER_START)
+  }
+  if (options.modes.contains(PrecaptureTrigger.AWB)) {
+    // AWB Precapture
+    if (meteringRectangles.isNotEmpty() && deviceDetails.supportsWhiteBalanceRegions) {
+      request.set(CaptureRequest.CONTROL_AWB_REGIONS, meteringRectangles)
+    }
+  }
+  this.capture(request.build(), null, null)
+
+  // 3. Start a repeating request without the trigger and wait until AF/AE/AWB locks
+  request.set(CaptureRequest.CONTROL_AF_TRIGGER, null)
+  request.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, null)
+  val result = this.setRepeatingRequestAndWaitForPrecapture(request.build(), *options.modes.toTypedArray())
+
+  Log.i(TAG, "AF/AE/AWB successfully locked!")
+  // TODO: Set to idle again?
+
+  val needsFlash = result.exposureState == ExposureState.FlashRequired
+  return PrecaptureResult(needsFlash)
+}
--- a/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+setRepeatingRequestAndWaitForAF.kt
+++ b/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+setRepeatingRequestAndWaitForAF.kt
@@ -1,47 +0,0 @@
-package com.mrousavy.camera.extensions
-
-import android.hardware.camera2.CameraCaptureSession
-import android.hardware.camera2.CaptureFailure
-import android.hardware.camera2.CaptureRequest
-import android.hardware.camera2.CaptureResult
-import android.hardware.camera2.TotalCaptureResult
-import android.util.Log
-import com.mrousavy.camera.core.CaptureAbortedError
-import kotlin.coroutines.resume
-import kotlin.coroutines.resumeWithException
-import kotlinx.coroutines.suspendCancellableCoroutine
-
-private const val TAG = "CameraCaptureSession"
-
-/**
- * Set a new repeating request for the [CameraCaptureSession] that contains an AF trigger, and wait until AF has locked.
- */
-suspend fun CameraCaptureSession.setRepeatingRequestAndWaitForAF(request: CaptureRequest) =
-  suspendCancellableCoroutine { continuation ->
-    this.setRepeatingRequest(
-      request,
-      object : CameraCaptureSession.CaptureCallback() {
-        override fun onCaptureCompleted(session: CameraCaptureSession, request: CaptureRequest, result: TotalCaptureResult) {
-          super.onCaptureCompleted(session, request, result)
-
-          if (continuation.isActive) {
-            val afState = result.get(CaptureResult.CONTROL_AF_STATE)
-            Log.i(TAG, "AF State: $afState")
-            if (afState == CaptureResult.CONTROL_AF_STATE_FOCUSED_LOCKED || afState == CaptureResult.CONTROL_AF_STATE_NOT_FOCUSED_LOCKED) {
-              continuation.resume(Unit)
-              session.setRepeatingRequest(request, null, null)
-            }
-          }
-        }
-        override fun onCaptureFailed(session: CameraCaptureSession, request: CaptureRequest, failure: CaptureFailure) {
-          super.onCaptureFailed(session, request, failure)
-
-          if (continuation.isActive) {
-            continuation.resumeWithException(CaptureAbortedError(failure.wasImageCaptured()))
-            session.setRepeatingRequest(request, null, null)
-          }
-        }
-      },
-      null
-    )
-  }
--- a/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+setRepeatingRequestAndWaitForPrecapture.kt
+++ b/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCaptureSession+setRepeatingRequestAndWaitForPrecapture.kt
@@ -0,0 +1,178 @@
+package com.mrousavy.camera.extensions
+
+import android.hardware.camera2.CameraCaptureSession
+import android.hardware.camera2.CaptureFailure
+import android.hardware.camera2.CaptureRequest
+import android.hardware.camera2.CaptureResult
+import android.hardware.camera2.TotalCaptureResult
+import android.util.Log
+import com.mrousavy.camera.core.CaptureAbortedError
+import com.mrousavy.camera.core.CaptureTimedOutError
+import kotlin.coroutines.resume
+import kotlin.coroutines.resumeWithException
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.delay
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.suspendCancellableCoroutine
+
+private const val TAG = "CameraCaptureSession"
+
+enum class PrecaptureTrigger {
+  AE,
+  AF,
+  AWB
+}
+
+enum class FocusState {
+  Inactive,
+  Scanning,
+  Focused,
+  Unfocused,
+  PassiveScanning,
+  PassiveFocused,
+  PassiveUnfocused;
+
+  val isCompleted: Boolean
+    get() = this == Focused || this == Unfocused
+
+  companion object {
+    fun fromAFState(afState: Int): FocusState =
+      when (afState) {
+        CaptureResult.CONTROL_AF_STATE_INACTIVE -> Inactive
+        CaptureResult.CONTROL_AF_STATE_ACTIVE_SCAN -> Scanning
+        CaptureResult.CONTROL_AF_STATE_FOCUSED_LOCKED -> Focused
+        CaptureResult.CONTROL_AF_STATE_NOT_FOCUSED_LOCKED -> Unfocused
+        CaptureResult.CONTROL_AF_STATE_PASSIVE_SCAN -> PassiveScanning
+        CaptureResult.CONTROL_AF_STATE_PASSIVE_FOCUSED -> PassiveFocused
+        CaptureResult.CONTROL_AF_STATE_PASSIVE_UNFOCUSED -> PassiveUnfocused
+        else -> throw Error("Invalid CONTROL_AF_STATE! $afState")
+      }
+  }
+}
+enum class ExposureState {
+  Locked,
+  Inactive,
+  Precapture,
+  Searching,
+  Converged,
+  FlashRequired;
+
+  val isCompleted: Boolean
+    get() = this == Converged || this == FlashRequired
+
+  companion object {
+    fun fromAEState(aeState: Int): ExposureState =
+      when (aeState) {
+        CaptureResult.CONTROL_AE_STATE_INACTIVE -> Inactive
+        CaptureResult.CONTROL_AE_STATE_SEARCHING -> Searching
+        CaptureResult.CONTROL_AE_STATE_PRECAPTURE -> Precapture
+        CaptureResult.CONTROL_AE_STATE_CONVERGED -> Converged
+        CaptureResult.CONTROL_AE_STATE_FLASH_REQUIRED -> FlashRequired
+        CaptureResult.CONTROL_AE_STATE_LOCKED -> Locked
+        else -> throw Error("Invalid CONTROL_AE_STATE! $aeState")
+      }
+  }
+}
+
+enum class WhiteBalanceState {
+  Inactive,
+  Locked,
+  Searching,
+  Converged;
+
+  val isCompleted: Boolean
+    get() = this == Converged
+
+  companion object {
+    fun fromAWBState(awbState: Int): WhiteBalanceState =
+      when (awbState) {
+        CaptureResult.CONTROL_AWB_STATE_INACTIVE -> Inactive
+        CaptureResult.CONTROL_AWB_STATE_SEARCHING -> Searching
+        CaptureResult.CONTROL_AWB_STATE_CONVERGED -> Converged
+        CaptureResult.CONTROL_AWB_STATE_LOCKED -> Locked
+        else -> throw Error("Invalid CONTROL_AWB_STATE! $awbState")
+      }
+  }
+}
+
+data class ResultState(val focusState: FocusState, val exposureState: ExposureState, val whiteBalanceState: WhiteBalanceState)
+
+/**
+ * Set a new repeating request for the [CameraCaptureSession] that contains a precapture trigger, and wait until the given precaptures have locked.
+ */
+suspend fun CameraCaptureSession.setRepeatingRequestAndWaitForPrecapture(
+  request: CaptureRequest,
+  vararg precaptureTriggers: PrecaptureTrigger
+): ResultState =
+  suspendCancellableCoroutine { continuation ->
+    // Map<PrecaptureTrigger, Boolean> of all completed precaptures
+    val completed = precaptureTriggers.associateWith { false }.toMutableMap()
+
+    CoroutineScope(Dispatchers.Default).launch {
+      delay(5000) // after 5s, cancel capture
+      if (continuation.isActive) {
+        Log.e(TAG, "Precapture timed out after 5 seconds!")
+        continuation.resumeWithException(CaptureTimedOutError())
+        try {
+          setRepeatingRequest(request, null, null)
+        } catch (e: Throwable) {
+          // session might have already been closed
+          Log.e(TAG, "Error resetting session repeating request..", e)
+        }
+      }
+    }
+
+    this.setRepeatingRequest(
+      request,
+      object : CameraCaptureSession.CaptureCallback() {
+        override fun onCaptureCompleted(session: CameraCaptureSession, request: CaptureRequest, result: TotalCaptureResult) {
+          super.onCaptureCompleted(session, request, result)
+
+          if (continuation.isActive) {
+            // AF Precapture
+            val afState = FocusState.fromAFState(result.get(CaptureResult.CONTROL_AF_STATE) ?: CaptureResult.CONTROL_AF_STATE_INACTIVE)
+            val aeState = ExposureState.fromAEState(result.get(CaptureResult.CONTROL_AE_STATE) ?: CaptureResult.CONTROL_AE_STATE_INACTIVE)
+            val awbState = WhiteBalanceState.fromAWBState(
+              result.get(CaptureResult.CONTROL_AWB_STATE) ?: CaptureResult.CONTROL_AWB_STATE_INACTIVE
+            )
+
+            if (precaptureTriggers.contains(PrecaptureTrigger.AF)) {
+              Log.i(TAG, "AF State: $afState (isCompleted: ${afState.isCompleted})")
+              completed[PrecaptureTrigger.AF] = afState.isCompleted
+            }
+            // AE Precapture
+            if (precaptureTriggers.contains(PrecaptureTrigger.AE)) {
+              Log.i(TAG, "AE State: $aeState (isCompleted: ${aeState.isCompleted})")
+              completed[PrecaptureTrigger.AE] = aeState.isCompleted
+            }
+            // AWB Precapture
+            if (precaptureTriggers.contains(PrecaptureTrigger.AWB)) {
+              Log.i(TAG, "AWB State: $awbState (isCompleted: ${awbState.isCompleted})")
+              completed[PrecaptureTrigger.AWB] = awbState.isCompleted
+            }
+
+            if (completed.values.all { it == true }) {
+              // All precaptures did complete!
+              continuation.resume(ResultState(afState, aeState, awbState))
+              session.setRepeatingRequest(request, null, null)
+            }
+          }
+        }
+        override fun onCaptureFailed(session: CameraCaptureSession, request: CaptureRequest, failure: CaptureFailure) {
+          super.onCaptureFailed(session, request, failure)
+
+          if (continuation.isActive) {
+            // Capture failed or session closed.
+            continuation.resumeWithException(CaptureAbortedError(failure.wasImageCaptured()))
+            try {
+              session.setRepeatingRequest(request, null, null)
+            } catch (e: Throwable) {
+              Log.e(TAG, "Failed to continue repeating request!", e)
+            }
+          }
+        }
+      },
+      null
+    )
+  }