From cc60ad296af30a7ce9ab2ea6450110b2976178c8 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Mon, 8 Jan 2024 12:13:05 +0100 Subject: [PATCH] fix: Validate input props (`fps`, `hdr`, `torch`, ...) instead of silently crashing (#2354) * fix: Fix Blackscreen by deterministically destroying session if `isActive=false` * Re-open Camera if session died * Simplify Camera * Disconnect is optional, block when resetting state * fix: Log in `configure { ... }` * fix: Make concurrent configure safe * fix: Don't resize preview * fix: Use current `CameraConfiguration` * Don't start if no outputs are available * Only mount with preview outputs * Update CameraSession.kt * Update PreviewView.kt * Better logging * Update CameraSession.kt * Extract * fix: Rebuild entire session if `isActive` changed * isActive safe * Start session at 1 * Create ActiveCameraDevice.kt * interrupts * chore: Freeze `frame` in `useFrameProcessor` * Revert "chore: Freeze `frame` in `useFrameProcessor`" This reverts commit dff93d506e29a791d8dea8842b880ab5c892211e. * chore: Better logging * fix: Move HDR to `video`/`photo` config * fix: Fix hdr usage * fix: Ignore any updates after destroying Camera * fix: Fix video HDR * chore: Format code * fix: Check Camera permission * Remove unneeded error * Update CameraSession.kt * Update CameraPage.tsx * Delete OutputConfiguration.toDebugString.kt * Update CameraSession.kt * fix: Perform sanity checks to make sure props are valid * format --- .../camera/core/CameraDeviceDetails.kt | 54 +++++++++---------- .../com/mrousavy/camera/core/CameraError.kt | 35 ++++++++++++ .../com/mrousavy/camera/core/CameraSession.kt | 38 +++++++++---- package/ios/Core/CameraError.swift | 4 +- package/src/CameraError.ts | 2 + 5 files changed, 94 insertions(+), 39 deletions(-) diff --git a/package/android/src/main/java/com/mrousavy/camera/core/CameraDeviceDetails.kt b/package/android/src/main/java/com/mrousavy/camera/core/CameraDeviceDetails.kt index c5f5921..7738a72 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/CameraDeviceDetails.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/CameraDeviceDetails.kt @@ -23,26 +23,26 @@ import com.mrousavy.camera.types.VideoStabilizationMode import kotlin.math.atan2 import kotlin.math.sqrt -class CameraDeviceDetails(private val cameraManager: CameraManager, private val cameraId: String) { - private val characteristics = cameraManager.getCameraCharacteristics(cameraId) - private val hardwareLevel = HardwareLevel.fromCameraCharacteristics(characteristics) - private val capabilities = characteristics.get(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES) ?: IntArray(0) - private val extensions = getSupportedExtensions() +class CameraDeviceDetails(val cameraManager: CameraManager, val cameraId: String) { + val characteristics = cameraManager.getCameraCharacteristics(cameraId) + val hardwareLevel = HardwareLevel.fromCameraCharacteristics(characteristics) + val capabilities = characteristics.get(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES) ?: IntArray(0) + val extensions = getSupportedExtensions() // device characteristics - private val isMultiCam = capabilities.contains(11) // TODO: CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_LOGICAL_MULTI_CAMERA - private val supportsDepthCapture = capabilities.contains(8) // TODO: CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_DEPTH_OUTPUT - private val supportsRawCapture = capabilities.contains(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_RAW) - private val supportsLowLightBoost = extensions.contains(4) // TODO: CameraExtensionCharacteristics.EXTENSION_NIGHT - private val lensFacing = LensFacing.fromCameraCharacteristics(characteristics) - private val hasFlash = characteristics.get(CameraCharacteristics.FLASH_INFO_AVAILABLE) ?: false - private val focalLengths = + val isMultiCam = capabilities.contains(11) // TODO: CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_LOGICAL_MULTI_CAMERA + val supportsDepthCapture = capabilities.contains(8) // TODO: CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_DEPTH_OUTPUT + val supportsRawCapture = capabilities.contains(CameraCharacteristics.REQUEST_AVAILABLE_CAPABILITIES_RAW) + val supportsLowLightBoost = extensions.contains(4) // TODO: CameraExtensionCharacteristics.EXTENSION_NIGHT + val lensFacing = LensFacing.fromCameraCharacteristics(characteristics) + val hasFlash = characteristics.get(CameraCharacteristics.FLASH_INFO_AVAILABLE) ?: false + val focalLengths = characteristics.get(CameraCharacteristics.LENS_INFO_AVAILABLE_FOCAL_LENGTHS) // 35mm is the film standard sensor size ?: floatArrayOf(35f) - private val sensorSize = characteristics.get(CameraCharacteristics.SENSOR_INFO_PHYSICAL_SIZE)!! - private val sensorOrientation = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION)!! - private val name = ( + val sensorSize = characteristics.get(CameraCharacteristics.SENSOR_INFO_PHYSICAL_SIZE)!! + val sensorOrientation = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION)!! + val name = ( if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) { characteristics.get(CameraCharacteristics.INFO_VERSION) } else { @@ -51,32 +51,32 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val ) ?: "$lensFacing ($cameraId)" // "formats" (all possible configurations for this device) - private val zoomRange = ( + val zoomRange = ( if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) { characteristics.get(CameraCharacteristics.CONTROL_ZOOM_RATIO_RANGE) } else { null } ) ?: Range(1f, characteristics.get(CameraCharacteristics.SCALER_AVAILABLE_MAX_DIGITAL_ZOOM) ?: 1f) - private val physicalDevices = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P && characteristics.physicalCameraIds.isNotEmpty()) { + val physicalDevices = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P && characteristics.physicalCameraIds.isNotEmpty()) { characteristics.physicalCameraIds } else { setOf(cameraId) } - private val minZoom = zoomRange.lower.toDouble() - private val maxZoom = zoomRange.upper.toDouble() + val minZoom = zoomRange.lower.toDouble() + val maxZoom = zoomRange.upper.toDouble() - private val cameraConfig = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! - private val isoRange = characteristics.get(CameraCharacteristics.SENSOR_INFO_SENSITIVITY_RANGE) ?: Range(0, 0) - private val exposureRange = characteristics.get(CameraCharacteristics.CONTROL_AE_COMPENSATION_RANGE) ?: Range(0, 0) - private val digitalStabilizationModes = + val cameraConfig = characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!! + val isoRange = characteristics.get(CameraCharacteristics.SENSOR_INFO_SENSITIVITY_RANGE) ?: Range(0, 0) + val exposureRange = characteristics.get(CameraCharacteristics.CONTROL_AE_COMPENSATION_RANGE) ?: Range(0, 0) + val digitalStabilizationModes = characteristics.get(CameraCharacteristics.CONTROL_AVAILABLE_VIDEO_STABILIZATION_MODES) ?: IntArray(0) - private val opticalStabilizationModes = + val opticalStabilizationModes = characteristics.get(CameraCharacteristics.LENS_INFO_AVAILABLE_OPTICAL_STABILIZATION) ?: IntArray(0) - private val supportsPhotoHdr = extensions.contains(3) // TODO: CameraExtensionCharacteristics.EXTENSION_HDR - private val supportsVideoHdr = getHasVideoHdr() + val supportsPhotoHdr = extensions.contains(3) // TODO: CameraExtensionCharacteristics.EXTENSION_HDR + val supportsVideoHdr = getHasVideoHdr() - private val videoFormat = ImageFormat.YUV_420_888 + val videoFormat = ImageFormat.YUV_420_888 // get extensions (HDR, Night Mode, ..) private fun getSupportedExtensions(): List = diff --git a/package/android/src/main/java/com/mrousavy/camera/core/CameraError.kt b/package/android/src/main/java/com/mrousavy/camera/core/CameraError.kt index 8ce8968..86f2eee 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/CameraError.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/CameraError.kt @@ -1,6 +1,7 @@ package com.mrousavy.camera.core import com.mrousavy.camera.types.CameraDeviceError +import com.mrousavy.camera.types.VideoStabilizationMode abstract class CameraError( /** @@ -49,6 +50,18 @@ class NoCameraDeviceError : ) class PixelFormatNotSupportedError(format: String) : CameraError("device", "pixel-format-not-supported", "The pixelFormat $format is not supported on the given Camera Device!") +class LowLightBoostNotSupportedError : + CameraError( + "device", + "low-light-boost-not-supported", + "The currently selected camera device does not support low-light boost! Select a device where `device.supportsLowLightBoost` is true." + ) +class FlashUnavailableError : + CameraError( + "device", + "flash-unavailable", + "The Camera Device does not have a flash unit! Make sure you select a device where `device.hasFlash`/`device.hasTorch` is true." + ) class CameraNotReadyError : CameraError("session", "camera-not-ready", "The Camera is not ready yet! Wait for the onInitialized() callback!") @@ -59,6 +72,28 @@ class CameraSessionCannotBeConfiguredError(cameraId: String) : class CameraDisconnectedError(cameraId: String, error: CameraDeviceError) : CameraError("session", "camera-has-been-disconnected", "The given Camera device (id: $cameraId) has been disconnected! Error: $error") +class PropRequiresFormatToBeNonNullError(propName: String) : + CameraError("format", "format-required", "The prop \"$propName\" requires a format to be set, but format was null!") +class InvalidFpsError(fps: Int) : + CameraError( + "format", + "invalid-fps", + "The given format cannot run at $fps FPS! Make sure your FPS is lower than `format.maxFps` but higher than `format.minFps`." + ) +class InvalidVideoStabilizationMode(mode: VideoStabilizationMode) : + CameraError( + "format", + "invalid-video-stabilization-mode", + "The given format does not support the videoStabilizationMode \"${mode.unionValue}\"! " + + "Select a format that contains ${mode.unionValue} in `format.supportedVideoStabilizationModes`." + ) +class InvalidVideoHdrError : + CameraError( + "format", + "invalid-video-hdr", + "The given format does not support videoHdr! Select a format where `format.supportsVideoHdr` is true." + ) + class VideoNotEnabledError : CameraError("capture", "video-not-enabled", "Video capture is disabled! Pass `video={true}` to enable video recordings.") class PhotoNotEnabledError : diff --git a/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt b/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt index 0078d22..d238f64 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt @@ -68,8 +68,13 @@ class CameraSession(private val context: Context, private val cameraManager: Cam private var configuration: CameraConfiguration? = null // Camera State - private var captureSession: CameraCaptureSession? = null private var cameraDevice: CameraDevice? = null + set(value) { + field = value + cameraDeviceDetails = if (value != null) CameraDeviceDetails(cameraManager, value.id) else null + } + private var cameraDeviceDetails: CameraDeviceDetails? = null + private var captureSession: CameraCaptureSession? = null private var previewRequest: CaptureRequest.Builder? = null private var photoOutput: PhotoOutput? = null private var videoOutput: VideoPipelineOutput? = null @@ -430,27 +435,38 @@ class CameraSession(private val context: Context, private val cameraManager: Cam } private fun createRepeatingRequest(device: CameraDevice, targets: List, config: CameraConfiguration): CaptureRequest { - val cameraCharacteristics = cameraManager.getCameraCharacteristics(device.id) + val deviceDetails = cameraDeviceDetails ?: CameraDeviceDetails(cameraManager, device.id) val template = if (config.video.isEnabled) CameraDevice.TEMPLATE_RECORD else CameraDevice.TEMPLATE_PREVIEW val captureRequest = device.createCaptureRequest(template) targets.forEach { t -> captureRequest.addTarget(t) } + val format = config.format + // Set FPS - // TODO: Check if the FPS range is actually supported in the current configuration. val fps = config.fps if (fps != null) { + if (format == null) throw PropRequiresFormatToBeNonNullError("fps") + if (format.maxFps < fps) throw InvalidFpsError(fps) captureRequest.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(fps, fps)) } // Set Video Stabilization + if (config.videoStabilizationMode != VideoStabilizationMode.OFF) { + if (format == null) throw PropRequiresFormatToBeNonNullError("videoStabilizationMode") + if (!format.videoStabilizationModes.contains( + config.videoStabilizationMode + ) + ) { + throw InvalidVideoStabilizationMode(config.videoStabilizationMode) + } + } when (config.videoStabilizationMode) { VideoStabilizationMode.OFF -> { // do nothing } VideoStabilizationMode.STANDARD -> { - // TODO: Check if that stabilization mode is even supported val mode = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU ) { @@ -461,35 +477,37 @@ class CameraSession(private val context: Context, private val cameraManager: Cam captureRequest.set(CaptureRequest.CONTROL_VIDEO_STABILIZATION_MODE, mode) } VideoStabilizationMode.CINEMATIC, VideoStabilizationMode.CINEMATIC_EXTENDED -> { - // TODO: Check if that stabilization mode is even supported captureRequest.set(CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE, CaptureRequest.LENS_OPTICAL_STABILIZATION_MODE_ON) } } // Set HDR - // TODO: Check if that value is even supported val video = config.video as? CameraConfiguration.Output.Enabled val videoHdr = video?.config?.enableHdr if (videoHdr == true) { + if (format == null) throw PropRequiresFormatToBeNonNullError("videoHdr") + if (!format.supportsVideoHdr) throw InvalidVideoHdrError() captureRequest.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_HDR) } else if (config.enableLowLightBoost) { + if (!deviceDetails.supportsLowLightBoost) throw LowLightBoostNotSupportedError() captureRequest.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_NIGHT) } // Set Exposure Bias - // TODO: Check if that exposure value is even supported val exposure = config.exposure?.toInt() if (exposure != null) { - captureRequest.set(CaptureRequest.CONTROL_AE_EXPOSURE_COMPENSATION, exposure) + val clamped = deviceDetails.exposureRange.clamp(exposure) + captureRequest.set(CaptureRequest.CONTROL_AE_EXPOSURE_COMPENSATION, clamped) } // Set Zoom - // TODO: Check if that zoom value is even supported + // TODO: Cache camera characteristics? Check perf. + val cameraCharacteristics = cameraManager.getCameraCharacteristics(device.id) captureRequest.setZoom(config.zoom, cameraCharacteristics) // Set Torch - // TODO: Check if torch is even supported if (config.torch == Torch.ON) { + if (!deviceDetails.hasFlash) throw FlashUnavailableError() captureRequest.set(CaptureRequest.FLASH_MODE, CaptureRequest.FLASH_MODE_TORCH) } diff --git a/package/ios/Core/CameraError.swift b/package/ios/Core/CameraError.swift index 3081ece..46235de 100644 --- a/package/ios/Core/CameraError.swift +++ b/package/ios/Core/CameraError.swift @@ -89,9 +89,9 @@ enum DeviceError: String { case .invalid: return "The given Camera device was invalid. Use `useCameraDevice(..)` or `Camera.getAvailableCameraDevices()` to select a suitable Camera device." case .flashUnavailable: - return "The Camera Device does not have a flash unit! Make sure you select a device where `hasFlash`/`hasTorch` is true!" + return "The Camera Device does not have a flash unit! Select a device where `device.hasFlash`/`device.hasTorch` is true." case .lowLightBoostNotSupported: - return "The currently selected camera device does not support low-light boost! Make sure you select a device where `supportsLowLightBoost` is true!" + return "The currently selected camera device does not support low-light boost! Select a device where `device.supportsLowLightBoost` is true." case .focusNotSupported: return "The currently selected camera device does not support focussing!" case .microphoneUnavailable: diff --git a/package/src/CameraError.ts b/package/src/CameraError.ts index ebac552..1f34280 100644 --- a/package/src/CameraError.ts +++ b/package/src/CameraError.ts @@ -16,8 +16,10 @@ export type DeviceError = export type FormatError = | 'format/invalid-fps' | 'format/invalid-video-hdr' + | 'format/invalid-video-stabilization-mode' | 'format/incompatible-pixel-format-with-hdr-setting' | 'format/invalid-format' + | 'format/format-required' export type SessionError = | 'session/camera-not-ready' | 'session/camera-cannot-be-opened'