fix: Fix blackscreen issues and lifecycle when closing Camera (#2339)

* fix: Fix Blackscreen by deterministically destroying session if `isActive=false`

* Re-open Camera if session died

* Simplify Camera

* Disconnect is optional, block when resetting state

* fix: Log in `configure { ... }`

* fix: Make concurrent configure safe

* fix: Don't resize preview

* fix: Use current `CameraConfiguration`

* Don't start if no outputs are available

* Only mount with preview outputs

* Update CameraSession.kt

* Update PreviewView.kt

* Better logging

* Update CameraSession.kt

* Extract

* fix: Rebuild entire session if `isActive` changed

* isActive safe

* Start session at 1

* Create ActiveCameraDevice.kt

* interrupts

* chore: Freeze `frame` in `useFrameProcessor`

* Revert "chore: Freeze `frame` in `useFrameProcessor`"

This reverts commit dff93d506e29a791d8dea8842b880ab5c892211e.

* chore: Better logging

* fix: Move HDR to `video`/`photo` config

* fix: Fix hdr usage

* fix: Ignore any updates after destroying Camera

* fix: Fix video HDR

* chore: Format code

* fix: Check Camera permission

* Remove unneeded error

* Update CameraSession.kt

* Update CameraPage.tsx

* Delete OutputConfiguration.toDebugString.kt

* Update CameraSession.kt
This commit is contained in:
Marc Rousavy 2024-01-08 11:41:57 +01:00 committed by GitHub
parent 2cd22ad236
commit 0d21bc3a57
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 297 additions and 239 deletions

View File

@ -30,7 +30,7 @@ class CameraDevicesManager(private val reactContext: ReactApplicationContext) :
}
override fun onCameraAvailable(cameraId: String) {
Log.i(TAG, "Camera #$cameraId: Available!")
Log.i(TAG, "Camera #$cameraId is now available.")
if (!devices.contains(cameraId)) {
devices.add(cameraId)
sendAvailableDevicesChangedEvent()
@ -38,7 +38,7 @@ class CameraDevicesManager(private val reactContext: ReactApplicationContext) :
}
override fun onCameraUnavailable(cameraId: String) {
Log.i(TAG, "Camera #$cameraId: Unavailable!")
Log.i(TAG, "Camera #$cameraId is now unavailable.")
if (devices.contains(cameraId) && !isDeviceConnected(cameraId)) {
devices.remove(cameraId)
sendAvailableDevicesChangedEvent()

View File

@ -47,14 +47,6 @@ class CameraView(context: Context) :
// react properties
// props that require reconfiguring
var cameraId: String? = null
set(value) {
if (value != null) {
// TODO: Move this into CameraSession
val f = if (format != null) CameraDeviceFormat.fromJSValue(format!!) else null
previewView.resizeToInputCamera(value, cameraManager, f)
}
field = value
}
var enableDepthData = false
var enableHighQualityPhotos: Boolean? = null
var enablePortraitEffectsMatteDelivery = false
@ -101,6 +93,7 @@ class CameraView(context: Context) :
// session
internal val cameraSession: CameraSession
private val previewView: PreviewView
private var currentConfigureCall: Long = System.currentTimeMillis()
internal var frameProcessor: FrameProcessor? = null
set(value) {
@ -138,15 +131,24 @@ class CameraView(context: Context) :
fun update() {
Log.i(TAG, "Updating CameraSession...")
val now = System.currentTimeMillis()
currentConfigureCall = now
launch {
cameraSession.configure { config ->
if (currentConfigureCall != now) {
// configure waits for a lock, and if a new call to update() happens in the meantime we can drop this one.
// this works similar to how React implemented concurrent rendering, the newer call to update() has higher priority.
Log.i(TAG, "A new configure { ... } call arrived, aborting this one...")
return@configure
}
// Input Camera Device
config.cameraId = cameraId
// Photo
if (photo == true) {
config.photo = CameraConfiguration.Output.Enabled.create(CameraConfiguration.Photo(Unit))
config.photo = CameraConfiguration.Output.Enabled.create(CameraConfiguration.Photo(photoHdr))
} else {
config.photo = CameraConfiguration.Output.Disabled.create()
}
@ -155,6 +157,7 @@ class CameraView(context: Context) :
if (video == true || enableFrameProcessor) {
config.video = CameraConfiguration.Output.Enabled.create(
CameraConfiguration.Video(
videoHdr,
pixelFormat,
enableFrameProcessor
)
@ -183,10 +186,6 @@ class CameraView(context: Context) :
// Orientation
config.orientation = orientation
// HDR
config.videoHdr = videoHdr
config.photoHdr = photoHdr
// Format
val format = format
if (format != null) {

View File

@ -18,10 +18,6 @@ data class CameraConfiguration(
var video: Output<Video> = Output.Disabled.create(),
var codeScanner: Output<CodeScanner> = Output.Disabled.create(),
// HDR
var videoHdr: Boolean = false,
var photoHdr: Boolean = false,
// Orientation
var orientation: Orientation = Orientation.PORTRAIT,
@ -47,8 +43,8 @@ data class CameraConfiguration(
// Output<T> types, those need to be comparable
data class CodeScanner(val codeTypes: List<CodeType>)
data class Photo(val nothing: Unit)
data class Video(val pixelFormat: PixelFormat, val enableFrameProcessor: Boolean)
data class Photo(val enableHdr: Boolean)
data class Video(val enableHdr: Boolean, val pixelFormat: PixelFormat, val enableFrameProcessor: Boolean)
data class Audio(val nothing: Unit)
data class Preview(val surface: Surface)
@ -71,7 +67,7 @@ data class CameraConfiguration(
}
data class Difference(
// Input Camera (cameraId and isActive)
// Input Camera (cameraId, isActive)
val deviceChanged: Boolean,
// Outputs & Session (Photo, Video, CodeScanner, HDR, Format)
val outputsChanged: Boolean,
@ -79,36 +75,30 @@ data class CameraConfiguration(
val sidePropsChanged: Boolean,
// (isActive) changed
val isActiveChanged: Boolean
) {
val hasAnyDifference: Boolean
get() = sidePropsChanged || outputsChanged || deviceChanged
}
)
companion object {
fun copyOf(other: CameraConfiguration?): CameraConfiguration = other?.copy() ?: CameraConfiguration()
fun difference(left: CameraConfiguration?, right: CameraConfiguration): Difference {
val deviceChanged = left?.cameraId != right.cameraId
val outputsChanged = deviceChanged ||
// input device
val deviceChanged = left?.cameraId != right.cameraId || left?.isActive != right.isActive
// outputs
val outputsChanged = deviceChanged ||
left?.photo != right.photo ||
left.video != right.video ||
left.codeScanner != right.codeScanner ||
left.preview != right.preview ||
// outputs
left.videoHdr != right.videoHdr ||
left.photoHdr != right.photoHdr ||
left.format != right.format // props that affect the outputs
left.format != right.format
// repeating request
val sidePropsChanged = outputsChanged ||
// depend on outputs
left?.torch != right.torch ||
left.enableLowLightBoost != right.enableLowLightBoost ||
left.fps != right.fps ||
left.zoom != right.zoom ||
left.videoStabilizationMode != right.videoStabilizationMode ||
left.isActive != right.isActive ||
left.exposure != right.exposure
val isActiveChanged = left?.isActive != right.isActive

View File

@ -55,7 +55,7 @@ class CameraNotReadyError :
class CameraCannotBeOpenedError(cameraId: String, error: CameraDeviceError) :
CameraError("session", "camera-cannot-be-opened", "The given Camera device (id: $cameraId) could not be opened! Error: $error")
class CameraSessionCannotBeConfiguredError(cameraId: String) :
CameraError("session", "cannot-create-session", "Failed to create a Camera Session for Camera $cameraId!")
CameraError("session", "cannot-create-session", "Failed to create a Camera Session for Camera #$cameraId!")
class CameraDisconnectedError(cameraId: String, error: CameraDeviceError) :
CameraError("session", "camera-has-been-disconnected", "The given Camera device (id: $cameraId) has been disconnected! Error: $error")
@ -87,8 +87,6 @@ class CodeTypeNotSupportedError(codeType: String) :
class ViewNotFoundError(viewId: Int) :
CameraError("system", "view-not-found", "The given view (ID $viewId) was not found in the view manager.")
class FrameProcessorsUnavailableError(reason: String) :
CameraError("system", "frame-processors-unavailable", "Frame Processors are unavailable! Reason: $reason")
class HardwareBuffersNotAvailableError :
CameraError("system", "hardware-buffers-unavailable", "HardwareBuffers are only available on API 28 or higher!")

View File

@ -1,6 +1,8 @@
package com.mrousavy.camera.core
import android.Manifest
import android.content.Context
import android.content.pm.PackageManager
import android.graphics.ImageFormat
import android.graphics.Point
import android.hardware.camera2.CameraCaptureSession
@ -12,7 +14,6 @@ import android.hardware.camera2.CaptureRequest
import android.hardware.camera2.CaptureResult
import android.hardware.camera2.TotalCaptureResult
import android.hardware.camera2.params.MeteringRectangle
import android.hardware.camera2.params.OutputConfiguration
import android.media.Image
import android.media.ImageReader
import android.os.Build
@ -21,6 +22,7 @@ import android.util.Range
import android.util.Size
import android.view.Surface
import android.view.SurfaceHolder
import androidx.core.content.ContextCompat
import com.google.mlkit.vision.barcode.common.Barcode
import com.mrousavy.camera.core.outputs.BarcodeScannerOutput
import com.mrousavy.camera.core.outputs.PhotoOutput
@ -44,6 +46,7 @@ import com.mrousavy.camera.types.QualityPrioritization
import com.mrousavy.camera.types.RecordVideoOptions
import com.mrousavy.camera.types.Torch
import com.mrousavy.camera.types.VideoStabilizationMode
import com.mrousavy.camera.utils.ImageFormatUtils
import java.io.Closeable
import java.util.concurrent.CancellationException
import kotlin.coroutines.CoroutineContext
@ -54,18 +57,15 @@ import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
class CameraSession(private val context: Context, private val cameraManager: CameraManager, private val callback: CameraSessionCallback) :
CameraManager.AvailabilityCallback(),
Closeable,
CoroutineScope {
companion object {
private const val TAG = "CameraSession"
// TODO: Samsung advertises 60 FPS but only allows 30 FPS for some reason.
private val CAN_DO_60_FPS = !Build.MANUFACTURER.equals("samsung", true)
}
// Camera Configuration
private var configuration: CameraConfiguration? = null
private var currentConfigureCall: Long = System.currentTimeMillis()
// Camera State
private var captureSession: CameraCaptureSession? = null
@ -73,12 +73,22 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
private var previewRequest: CaptureRequest.Builder? = null
private var photoOutput: PhotoOutput? = null
private var videoOutput: VideoPipelineOutput? = null
private var previewOutput: SurfaceOutput? = null
private var codeScannerOutput: BarcodeScannerOutput? = null
private var previewView: PreviewView? = null
private val photoOutputSynchronizer = PhotoOutputSynchronizer()
private val mutex = Mutex()
private var isDestroyed = false
private var isRunning = false
set(value) {
if (field != value) {
if (value) {
callback.onStarted()
} else {
callback.onStopped()
}
}
field = value
}
override val coroutineContext: CoroutineContext
get() = CameraQueues.cameraQueue.coroutineDispatcher
@ -95,15 +105,6 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
updateVideoOutputs()
}
override fun close() {
runBlocking {
mutex.withLock {
destroy()
photoOutputSynchronizer.clear()
}
}
}
val orientation: Orientation
get() {
val cameraId = configuration?.cameraId ?: return Orientation.PORTRAIT
@ -112,41 +113,67 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
return Orientation.fromRotationDegrees(sensorRotation)
}
init {
cameraManager.registerAvailabilityCallback(this, CameraQueues.cameraQueue.handler)
}
override fun close() {
Log.i(TAG, "Closing CameraSession...")
isDestroyed = true
cameraManager.unregisterAvailabilityCallback(this)
runBlocking {
mutex.withLock {
destroy()
photoOutputSynchronizer.clear()
}
}
Log.i(TAG, "CameraSession closed!")
}
override fun onCameraAvailable(cameraId: String) {
super.onCameraAvailable(cameraId)
if (this.configuration?.cameraId == cameraId && cameraDevice == null && configuration?.isActive == true) {
Log.i(TAG, "Camera #$cameraId is now available again, trying to re-open it now...")
launch {
configure {
// re-open CameraDevice if needed
}
}
}
}
suspend fun configure(lambda: (configuration: CameraConfiguration) -> Unit) {
// This is the latest call to configure()
val now = System.currentTimeMillis()
currentConfigureCall = now
Log.i(TAG, "Updating CameraSession Configuration...")
Log.i(TAG, "configure { ... }: Waiting for lock...")
mutex.withLock {
// Let caller configure a new configuration for the Camera.
val config = CameraConfiguration.copyOf(this.configuration)
lambda(config)
val diff = CameraConfiguration.difference(this.configuration, config)
if (!diff.hasAnyDifference) {
Log.w(TAG, "Called configure(...) but nothing changed...")
return
if (isDestroyed) {
Log.i(TAG, "CameraSession is already destroyed. Skipping configure { ... }")
return@withLock
}
mutex.withLock {
// Cancel configuration if there has already been a new config
if (currentConfigureCall != now) {
// configure() has been called again just now, skip this one so the new call takes over.
return
}
Log.i(TAG, "configure { ... }: Updating CameraSession Configuration... $diff")
try {
val needsRebuild = config.isActive && (cameraDevice == null || captureSession == null)
if (needsRebuild) {
Log.i(TAG, "Need to rebuild CameraDevice and CameraCaptureSession...")
}
// Build up session or update any props
if (diff.deviceChanged) {
if (diff.deviceChanged || needsRebuild) {
// 1. cameraId changed, open device
configureCameraDevice(config)
}
if (diff.outputsChanged) {
if (diff.outputsChanged || needsRebuild) {
// 2. outputs changed, build new session
configureOutputs(config)
}
if (diff.sidePropsChanged) {
if (diff.sidePropsChanged || needsRebuild) {
// 3. zoom etc changed, update repeating request
configureCaptureRequest(config)
}
@ -155,21 +182,11 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
this.configuration = config
// Notify about Camera initialization
if (diff.deviceChanged) {
if (diff.deviceChanged && config.isActive) {
callback.onInitialized()
}
// Notify about Camera start/stop
if (diff.isActiveChanged) {
// TODO: Move that into the CaptureRequest callback to get actual first-frame arrive time?
if (config.isActive) {
callback.onStarted()
} else {
callback.onStopped()
}
}
} catch (error: Throwable) {
Log.e(TAG, "Failed to configure CameraSession! Error: ${error.message}, Config-Diff: $diff", error)
Log.e(TAG, "Failed to configure CameraSession! Error: ${error.message}, isRunning: $isRunning, Config-Diff: $diff", error)
callback.onError(error)
}
}
@ -177,15 +194,9 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
private fun destroy() {
Log.i(TAG, "Destroying session..")
captureSession?.stopRepeating()
captureSession?.close()
captureSession = null
cameraDevice?.close()
cameraDevice = null
previewOutput?.close()
previewOutput = null
photoOutput?.close()
photoOutput = null
videoOutput?.close()
@ -193,7 +204,6 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
codeScannerOutput?.close()
codeScannerOutput = null
configuration = null
isRunning = false
}
@ -236,31 +246,53 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
config.preview = CameraConfiguration.Output.Disabled.create()
}
}
Log.i(TAG, "Preview Output destroyed!")
}
/**
* Set up the `CameraDevice` (`cameraId`)
*/
private suspend fun configureCameraDevice(configuration: CameraConfiguration) {
val cameraId = configuration.cameraId ?: throw NoCameraDeviceError()
Log.i(TAG, "Configuring Camera #$cameraId...")
if (!configuration.isActive) {
// If isActive=false, we don't care if the device is opened or closed.
// Android OS can close the CameraDevice if it needs it, otherwise we keep it warm.
Log.i(TAG, "isActive is false, skipping CameraDevice configuration.")
return
}
if (cameraDevice != null) {
// Close existing device
Log.i(TAG, "Closing previous Camera #${cameraDevice?.id}...")
cameraDevice?.close()
cameraDevice = cameraManager.openCamera(cameraId, { device, error ->
if (this.cameraDevice == device) {
Log.e(TAG, "Camera Device $device has been disconnected!", error)
cameraDevice = null
}
isRunning = false
callback.onError(error)
} else {
// Check Camera Permission
val cameraPermission = ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA)
if (cameraPermission != PackageManager.PERMISSION_GRANTED) throw CameraPermissionError()
// Open new device
val cameraId = configuration.cameraId ?: throw NoCameraDeviceError()
Log.i(TAG, "Configuring Camera #$cameraId...")
cameraDevice = cameraManager.openCamera(cameraId, { device, error ->
if (cameraDevice != device) {
// a previous device has been disconnected, but we already have a new one.
// this is just normal behavior
return@openCamera
}
this.cameraDevice = null
isRunning = false
if (error != null) {
Log.e(TAG, "Camera #${device.id} has been unexpectedly disconnected!", error)
callback.onError(error)
} else {
Log.i(TAG, "Camera #${device.id} has been gracefully disconnected!")
}
}, CameraQueues.cameraQueue)
// Update PreviewView's Surface Size to a supported value from this Capture Device
previewView?.resizeToInputCamera(cameraId, cameraManager, configuration.format)
Log.i(TAG, "Successfully configured Camera #$cameraId!")
}
@ -268,29 +300,34 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
* Set up the `CaptureSession` with all outputs (preview, photo, video, codeScanner) and their HDR/Format settings.
*/
private suspend fun configureOutputs(configuration: CameraConfiguration) {
val cameraDevice = cameraDevice ?: throw NoCameraDeviceError()
val characteristics = cameraManager.getCameraCharacteristics(cameraDevice.id)
val format = configuration.format
if (!configuration.isActive) {
Log.i(TAG, "isActive is false, skipping CameraCaptureSession configuration.")
return
}
val cameraDevice = cameraDevice
if (cameraDevice == null) {
Log.i(TAG, "CameraSession hasn't configured a CameraDevice, skipping session configuration...")
return
}
Log.i(TAG, "Configuring Session for Camera #${cameraDevice.id}...")
// TODO: Do we want to skip this is this.cameraSession already contains all outputs?
// Destroy previous CaptureSession
captureSession?.close()
captureSession = null
// Destroy previous outputs
Log.i(TAG, "Destroying previous outputs...")
photoOutput?.close()
photoOutput = null
videoOutput?.close()
videoOutput = null
previewOutput?.close()
previewOutput = null
codeScannerOutput?.close()
codeScannerOutput = null
isRunning = false
val characteristics = cameraManager.getCameraCharacteristics(cameraDevice.id)
val format = configuration.format
Log.i(TAG, "Creating outputs for Camera #${cameraDevice.id}...")
val isSelfie = characteristics.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT
val outputs = mutableListOf<OutputConfiguration>()
val outputs = mutableListOf<SurfaceOutput>()
// Photo Output
val photo = configuration.photo as? CameraConfiguration.Output.Enabled<CameraConfiguration.Photo>
@ -300,15 +337,15 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
val size = sizes.closestToOrMax(format?.photoSize)
val maxImages = 10
Log.i(TAG, "Adding ${size.width} x ${size.height} Photo Output in Format #$imageFormat...")
Log.i(TAG, "Adding ${size.width}x${size.height} Photo Output in ${ImageFormatUtils.imageFormatToString(imageFormat)}...")
val imageReader = ImageReader.newInstance(size.width, size.height, imageFormat, maxImages)
imageReader.setOnImageAvailableListener({ reader ->
Log.i(TAG, "Photo Captured!")
val image = reader.acquireLatestImage()
onPhotoCaptured(image)
}, CameraQueues.cameraQueue.handler)
val output = PhotoOutput(imageReader, configuration.photoHdr)
outputs.add(output.toOutputConfiguration(characteristics))
val output = PhotoOutput(imageReader, photo.config.enableHdr)
outputs.add(output)
photoOutput = output
}
@ -319,7 +356,7 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
val sizes = characteristics.getVideoSizes(cameraDevice.id, imageFormat)
val size = sizes.closestToOrMax(format?.videoSize)
Log.i(TAG, "Adding ${size.width} x ${size.height} Video Output in Format #$imageFormat...")
Log.i(TAG, "Adding ${size.width}x${size.height} Video Output in ${ImageFormatUtils.imageFormatToString(imageFormat)}...")
val videoPipeline = VideoPipeline(
size.width,
size.height,
@ -327,8 +364,8 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
isSelfie,
video.config.enableFrameProcessor
)
val output = VideoPipelineOutput(videoPipeline, configuration.videoHdr)
outputs.add(output.toOutputConfiguration(characteristics))
val output = VideoPipelineOutput(videoPipeline, video.config.enableHdr)
outputs.add(output)
videoOutput = output
}
@ -344,15 +381,16 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
characteristics.getPreviewTargetSize(null)
}
Log.i(TAG, "Adding ${size.width} x ${size.height} Preview Output...")
val enableHdr = video?.config?.enableHdr ?: false
Log.i(TAG, "Adding ${size.width}x${size.height} Preview Output...")
val output = SurfaceOutput(
preview.config.surface,
size,
SurfaceOutput.OutputType.PREVIEW,
configuration.videoHdr
enableHdr
)
outputs.add(output.toOutputConfiguration(characteristics))
previewOutput = output
outputs.add(output)
previewView?.size = size
}
@ -363,19 +401,26 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
val sizes = characteristics.getVideoSizes(cameraDevice.id, imageFormat)
val size = sizes.closestToOrMax(Size(1280, 720))
Log.i(TAG, "Adding ${size.width} x ${size.height} CodeScanner Output in Format #$imageFormat...")
Log.i(TAG, "Adding ${size.width}x${size.height} CodeScanner Output in ${ImageFormatUtils.imageFormatToString(imageFormat)}...")
val pipeline = CodeScannerPipeline(size, imageFormat, codeScanner.config, callback)
val output = BarcodeScannerOutput(pipeline)
outputs.add(output.toOutputConfiguration(characteristics))
outputs.add(output)
codeScannerOutput = output
}
// Create new session
// Create session
captureSession = cameraDevice.createCaptureSession(cameraManager, outputs, { session ->
if (this.captureSession == session) {
Log.i(TAG, "Camera Session $session has been closed!")
isRunning = false
if (this.captureSession != session) {
// a previous session has been closed, but we already have a new one.
// this is just normal behavior
return@createCaptureSession
}
// onClosed
this.captureSession = null
isRunning = false
Log.i(TAG, "Camera Session $session has been closed.")
}, CameraQueues.cameraQueue)
Log.i(TAG, "Successfully configured Session with ${outputs.size} outputs for Camera #${cameraDevice.id}!")
@ -384,45 +429,18 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
updateVideoOutputs()
}
private fun configureCaptureRequest(config: CameraConfiguration) {
if (!config.isActive) {
// TODO: Do we want to do stopRepeating() or entirely destroy the session?
// If the Camera is not active, we don't do anything.
captureSession?.stopRepeating()
isRunning = false
return
}
val device = cameraDevice ?: throw NoCameraDeviceError()
val captureSession = captureSession ?: throw CameraNotReadyError()
val previewOutput = previewOutput
if (previewOutput == null) {
Log.w(TAG, "Preview Output is null, aborting...")
return
}
private fun createRepeatingRequest(device: CameraDevice, targets: List<Surface>, config: CameraConfiguration): CaptureRequest {
val cameraCharacteristics = cameraManager.getCameraCharacteristics(device.id)
val template = if (config.video.isEnabled) CameraDevice.TEMPLATE_RECORD else CameraDevice.TEMPLATE_PREVIEW
val captureRequest = device.createCaptureRequest(template)
captureRequest.addTarget(previewOutput.surface)
videoOutput?.let { output ->
captureRequest.addTarget(output.surface)
}
codeScannerOutput?.let { output ->
captureRequest.addTarget(output.surface)
}
targets.forEach { t -> captureRequest.addTarget(t) }
// Set FPS
// TODO: Check if the FPS range is actually supported in the current configuration.
var fps = config.fps
val fps = config.fps
if (fps != null) {
if (!CAN_DO_60_FPS) {
// If we can't do 60 FPS, we clamp it to 30 FPS - that's always supported.
fps = 30.coerceAtMost(fps)
}
captureRequest.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(fps, fps))
}
@ -450,7 +468,9 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
// Set HDR
// TODO: Check if that value is even supported
if (config.videoHdr) {
val video = config.video as? CameraConfiguration.Output.Enabled<CameraConfiguration.Video>
val videoHdr = video?.config?.enableHdr
if (videoHdr == true) {
captureRequest.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_HDR)
} else if (config.enableLowLightBoost) {
captureRequest.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_NIGHT)
@ -474,7 +494,30 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
}
// Start repeating request if the Camera is active
val request = captureRequest.build()
return captureRequest.build()
}
private fun configureCaptureRequest(config: CameraConfiguration) {
val captureSession = captureSession
if (!config.isActive) {
isRunning = false
return
}
if (captureSession == null) {
Log.i(TAG, "CameraSession hasn't configured the capture session, skipping CaptureRequest...")
return
}
val preview = config.preview as? CameraConfiguration.Output.Enabled<CameraConfiguration.Preview>
val previewSurface = preview?.config?.surface
val targets = listOfNotNull(previewSurface, videoOutput?.surface, codeScannerOutput?.surface)
if (targets.isEmpty()) {
Log.i(TAG, "CameraSession has no repeating outputs (Preview, Video, CodeScanner), skipping CaptureRequest...")
return
}
val request = createRepeatingRequest(captureSession.device, targets, config)
captureSession.setRepeatingRequest(request, null, null)
isRunning = true
}
@ -496,7 +539,6 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
val cameraCharacteristics = cameraManager.getCameraCharacteristics(captureSession.device.id)
val orientation = outputOrientation.toSensorRelativeOrientation(cameraCharacteristics)
val enableHdr = configuration?.photoHdr ?: false
val captureRequest = captureSession.device.createPhotoCaptureRequest(
cameraManager,
photoOutput.surface,
@ -505,7 +547,7 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
flashMode,
enableRedEyeReduction,
enableAutoStabilization,
enableHdr,
photoOutput.enableHdr,
orientation
)
Log.i(TAG, "Photo capture 1/3 - starting capture...")
@ -547,12 +589,20 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
val videoOutput = videoOutput ?: throw VideoNotEnabledError()
val cameraDevice = cameraDevice ?: throw CameraNotReadyError()
// TODO: Implement HDR
val hdr = configuration?.videoHdr ?: false
val fps = configuration?.fps ?: 30
val recording =
RecordingSession(context, cameraDevice.id, videoOutput.size, enableAudio, fps, hdr, orientation, options, callback, onError)
val recording = RecordingSession(
context,
cameraDevice.id,
videoOutput.size,
enableAudio,
fps,
videoOutput.enableHdr,
orientation,
options,
callback,
onError
)
recording.start()
this.recording = recording
}
@ -581,19 +631,7 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
}
}
suspend fun focus(x: Int, y: Int) {
val captureSession = captureSession ?: throw CameraNotReadyError()
val previewOutput = previewOutput ?: throw CameraNotReadyError()
val characteristics = cameraManager.getCameraCharacteristics(captureSession.device.id)
val sensorSize = characteristics.get(CameraCharacteristics.SENSOR_INFO_ACTIVE_ARRAY_SIZE)!!
val previewSize = previewOutput.size
val pX = x.toDouble() / previewSize.width * sensorSize.height()
val pY = y.toDouble() / previewSize.height * sensorSize.width()
val point = Point(pX.toInt(), pY.toInt())
Log.i(TAG, "Focusing (${point.x}, ${point.y})...")
focus(point)
}
suspend fun focus(x: Int, y: Int): Unit = throw NotImplementedError("focus() is not yet implemented!")
private suspend fun focus(point: Point) {
mutex.withLock {

View File

@ -2,7 +2,6 @@ package com.mrousavy.camera.core
import android.annotation.SuppressLint
import android.content.Context
import android.hardware.camera2.CameraManager
import android.util.Log
import android.util.Size
import android.view.Gravity
@ -10,11 +9,7 @@ import android.view.SurfaceHolder
import android.view.SurfaceView
import android.widget.FrameLayout
import com.facebook.react.bridge.UiThreadUtil
import com.mrousavy.camera.extensions.bigger
import com.mrousavy.camera.extensions.getMaximumPreviewSize
import com.mrousavy.camera.extensions.getPreviewTargetSize
import com.mrousavy.camera.extensions.smaller
import com.mrousavy.camera.types.CameraDeviceFormat
import com.mrousavy.camera.types.ResizeMode
import kotlin.math.roundToInt
@ -49,19 +44,19 @@ class PreviewView(context: Context, callback: SurfaceHolder.Callback) : SurfaceV
holder.addCallback(callback)
}
fun resizeToInputCamera(cameraId: String, cameraManager: CameraManager, format: CameraDeviceFormat?) {
/*fun resizeToInputCamera(cameraId: String, cameraManager: CameraManager, format: CameraDeviceFormat?) {
val characteristics = cameraManager.getCameraCharacteristics(cameraId)
val targetPreviewSize = format?.videoSize
val formatAspectRatio = if (targetPreviewSize != null) targetPreviewSize.bigger.toDouble() / targetPreviewSize.smaller else null
size = characteristics.getPreviewTargetSize(formatAspectRatio)
}
}*/
private fun getSize(contentSize: Size, containerSize: Size, resizeMode: ResizeMode): Size {
val contentAspectRatio = contentSize.height.toDouble() / contentSize.width
val containerAspectRatio = containerSize.width.toDouble() / containerSize.height
Log.d(TAG, "coverSize :: $contentSize ($contentAspectRatio), ${containerSize.width}x${containerSize.height} ($containerAspectRatio)")
Log.i(TAG, "Content Size: $contentSize ($contentAspectRatio) | Container Size: $containerSize ($containerAspectRatio)")
val widthOverHeight = when (resizeMode) {
ResizeMode.COVER -> contentAspectRatio > containerAspectRatio
@ -82,14 +77,11 @@ class PreviewView(context: Context, callback: SurfaceHolder.Callback) : SurfaceV
@SuppressLint("DrawAllocation")
override fun onMeasure(widthMeasureSpec: Int, heightMeasureSpec: Int) {
super.onMeasure(widthMeasureSpec, heightMeasureSpec)
val viewWidth = MeasureSpec.getSize(widthMeasureSpec)
val viewHeight = MeasureSpec.getSize(heightMeasureSpec)
Log.i(TAG, "PreviewView onMeasure($viewWidth, $viewHeight)")
val viewSize = Size(MeasureSpec.getSize(widthMeasureSpec), MeasureSpec.getSize(heightMeasureSpec))
val fittedSize = getSize(size, viewSize, resizeMode)
val fittedSize = getSize(size, Size(viewWidth, viewHeight), resizeMode)
Log.d(TAG, "Fitted dimensions set: $fittedSize")
Log.i(TAG, "PreviewView is $viewSize, rendering $size content. Resizing to: $fittedSize ($resizeMode)")
setMeasuredDimension(fittedSize.width, fittedSize.height)
}

View File

@ -40,6 +40,7 @@ class RecordingSession(
private var startTime: Long? = null
val surface: Surface = MediaCodec.createPersistentInputSurface()
// TODO: Implement HDR
init {
outputFile = File.createTempFile("mrousavy", options.fileType.toExtension(), context.cacheDir)

View File

@ -3,7 +3,7 @@ package com.mrousavy.camera.core.outputs
import android.media.ImageReader
import android.util.Log
import android.util.Size
import java.io.Closeable
import com.mrousavy.camera.utils.ImageFormatUtils
open class PhotoOutput(private val imageReader: ImageReader, enableHdr: Boolean = false) :
SurfaceOutput(
@ -11,13 +11,15 @@ open class PhotoOutput(private val imageReader: ImageReader, enableHdr: Boolean
Size(imageReader.width, imageReader.height),
OutputType.PHOTO,
enableHdr
),
Closeable {
) {
override fun close() {
Log.i(TAG, "Closing ${imageReader.width}x${imageReader.height} $outputType ImageReader..")
imageReader.close()
super.close()
}
override fun toString(): String = "$outputType (${imageReader.width} x ${imageReader.height} in format #${imageReader.imageFormat})"
override fun toString(): String =
"$outputType (${size.width}x${size.height} in ${ImageFormatUtils.imageFormatToString(
imageReader.imageFormat
)})"
}

View File

@ -14,7 +14,7 @@ open class SurfaceOutput(
val surface: Surface,
val size: Size,
val outputType: OutputType,
private val enableHdr: Boolean = false,
val enableHdr: Boolean = false,
private val closeSurfaceOnEnd: Boolean = false
) : Closeable {
companion object {

View File

@ -19,5 +19,5 @@ class VideoPipelineOutput(val videoPipeline: VideoPipeline, enableHdr: Boolean =
super.close()
}
override fun toString(): String = "$outputType (${videoPipeline.width} x ${videoPipeline.height} in format #${videoPipeline.format})"
override fun toString(): String = "$outputType (${size.width}x${size.height} in ${videoPipeline.format})"
}

View File

@ -4,22 +4,22 @@ import android.hardware.camera2.CameraCaptureSession
import android.hardware.camera2.CameraCharacteristics
import android.hardware.camera2.CameraDevice
import android.hardware.camera2.CameraManager
import android.hardware.camera2.params.OutputConfiguration
import android.hardware.camera2.params.SessionConfiguration
import android.os.Build
import android.util.Log
import com.mrousavy.camera.core.CameraQueues
import com.mrousavy.camera.core.CameraSessionCannotBeConfiguredError
import com.mrousavy.camera.core.outputs.SurfaceOutput
import kotlin.coroutines.resume
import kotlin.coroutines.resumeWithException
import kotlinx.coroutines.suspendCancellableCoroutine
private const val TAG = "CreateCaptureSession"
private var sessionId = 1000
private var sessionId = 1
suspend fun CameraDevice.createCaptureSession(
cameraManager: CameraManager,
outputs: List<OutputConfiguration>,
outputs: List<SurfaceOutput>,
onClosed: (session: CameraCaptureSession) -> Unit,
queue: CameraQueues.CameraQueue
): CameraCaptureSession =
@ -29,34 +29,35 @@ suspend fun CameraDevice.createCaptureSession(
val sessionId = sessionId++
Log.i(
TAG,
"Camera $id: Creating Capture Session #$sessionId... " +
"Hardware Level: $hardwareLevel} | Outputs: $outputs"
"Camera #$id: Creating Capture Session #$sessionId... " +
"(Hardware Level: $hardwareLevel | Outputs: [${outputs.joinToString()}])"
)
val callback = object : CameraCaptureSession.StateCallback() {
override fun onConfigured(session: CameraCaptureSession) {
Log.i(TAG, "Camera $id: Capture Session #$sessionId configured!")
Log.i(TAG, "Camera #$id: Successfully created CameraCaptureSession #$sessionId!")
continuation.resume(session)
}
override fun onConfigureFailed(session: CameraCaptureSession) {
Log.e(TAG, "Camera $id: Failed to configure Capture Session #$sessionId!")
Log.e(TAG, "Camera #$id: Failed to create CameraCaptureSession #$sessionId!")
continuation.resumeWithException(CameraSessionCannotBeConfiguredError(id))
}
override fun onClosed(session: CameraCaptureSession) {
Log.i(TAG, "Camera #$id: CameraCaptureSession #$sessionId has been closed.")
super.onClosed(session)
Log.i(TAG, "Camera $id: Capture Session #$sessionId closed!")
onClosed(session)
}
}
val configurations = outputs.map { it.toOutputConfiguration(characteristics) }
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
Log.i(TAG, "Using new API (>=28)")
val config = SessionConfiguration(SessionConfiguration.SESSION_REGULAR, outputs, queue.executor, callback)
val config = SessionConfiguration(SessionConfiguration.SESSION_REGULAR, configurations, queue.executor, callback)
this.createCaptureSession(config)
} else {
Log.i(TAG, "Using legacy API (<28)")
this.createCaptureSessionByOutputConfigurations(outputs, callback, queue.handler)
this.createCaptureSessionByOutputConfigurations(configurations, callback, queue.handler)
}
}

View File

@ -18,30 +18,30 @@ private const val TAG = "CameraManager"
@SuppressLint("MissingPermission")
suspend fun CameraManager.openCamera(
cameraId: String,
onDisconnected: (camera: CameraDevice, reason: Throwable) -> Unit,
onDisconnected: (camera: CameraDevice, error: Throwable?) -> Unit,
queue: CameraQueues.CameraQueue
): CameraDevice =
suspendCancellableCoroutine { continuation ->
Log.i(TAG, "Camera $cameraId: Opening...")
Log.i(TAG, "Camera #$cameraId: Opening...")
val callback = object : CameraDevice.StateCallback() {
override fun onOpened(camera: CameraDevice) {
Log.i(TAG, "Camera $cameraId: Opened!")
Log.i(TAG, "Camera #$cameraId: Opened!")
continuation.resume(camera)
}
override fun onDisconnected(camera: CameraDevice) {
Log.i(TAG, "Camera $cameraId: Disconnected!")
Log.i(TAG, "Camera #$cameraId: Disconnected!")
if (continuation.isActive) {
continuation.resumeWithException(CameraCannotBeOpenedError(cameraId, CameraDeviceError.DISCONNECTED))
} else {
onDisconnected(camera, CameraDisconnectedError(cameraId, CameraDeviceError.DISCONNECTED))
onDisconnected(camera, null)
}
camera.close()
}
override fun onError(camera: CameraDevice, errorCode: Int) {
Log.e(TAG, "Camera $cameraId: Error! $errorCode")
Log.e(TAG, "Camera #$cameraId: Error! $errorCode")
val error = CameraDeviceError.fromCameraDeviceError(errorCode)
if (continuation.isActive) {
continuation.resumeWithException(CameraCannotBeOpenedError(cameraId, error))

View File

@ -1,8 +1,10 @@
package com.mrousavy.camera.types
import android.graphics.ImageFormat
import android.util.Log
import com.mrousavy.camera.core.InvalidTypeScriptUnionError
import com.mrousavy.camera.core.PixelFormatNotSupportedError
import com.mrousavy.camera.utils.ImageFormatUtils
enum class PixelFormat(override val unionValue: String) : JSUnionValue {
YUV("yuv"),
@ -18,11 +20,15 @@ enum class PixelFormat(override val unionValue: String) : JSUnionValue {
}
companion object : JSUnionValue.Companion<PixelFormat> {
private const val TAG = "PixelFormat"
fun fromImageFormat(imageFormat: Int): PixelFormat =
when (imageFormat) {
ImageFormat.YUV_420_888 -> YUV
ImageFormat.PRIVATE -> NATIVE
else -> UNKNOWN
else -> {
Log.w(TAG, "Unknown PixelFormat! ${ImageFormatUtils.imageFormatToString(imageFormat)}")
UNKNOWN
}
}
override fun fromUnionValue(unionValue: String?): PixelFormat =

View File

@ -0,0 +1,28 @@
package com.mrousavy.camera.utils
import android.graphics.ImageFormat
import android.graphics.PixelFormat
class ImageFormatUtils {
companion object {
fun imageFormatToString(format: Int): String =
when (format) {
ImageFormat.YUV_420_888 -> "YUV_420_888"
ImageFormat.NV21 -> "NV21"
ImageFormat.NV16 -> "NV16"
ImageFormat.YV12 -> "YV12"
ImageFormat.YUV_422_888 -> "YUV_422_888"
ImageFormat.YCBCR_P010 -> "YCBCR_P010"
ImageFormat.YUV_444_888 -> "YUV_444_888"
ImageFormat.YUY2 -> "YUY2"
ImageFormat.Y8 -> "Y8"
ImageFormat.JPEG -> "JPEG"
ImageFormat.RGB_565 -> "RGB_565"
ImageFormat.FLEX_RGB_888 -> "FLEX_RGB_888"
ImageFormat.FLEX_RGBA_8888 -> "FLEX_RGBA_8888"
PixelFormat.RGB_888 -> "RGB_888"
ImageFormat.PRIVATE -> "PRIVATE"
else -> "UNKNOWN ($format)"
}
}
}

View File

@ -84,6 +84,7 @@ public final class CameraView: UIView, CameraSessionDelegate {
// CameraView+Zoom
var pinchGestureRecognizer: UIPinchGestureRecognizer?
var pinchScaleOffset: CGFloat = 1.0
private var currentConfigureCall: DispatchTime?
var previewView: PreviewView
#if DEBUG
@ -150,8 +151,18 @@ public final class CameraView: UIView, CameraSessionDelegate {
// pragma MARK: Props updating
override public final func didSetProps(_ changedProps: [String]!) {
ReactLogger.log(level: .info, message: "Updating \(changedProps.count) props: [\(changedProps.joined(separator: ", "))]")
let now = DispatchTime.now()
currentConfigureCall = now
cameraSession.configure { [self] config in
// Check if we're still the latest call to configure { ... }
guard currentConfigureCall == now else {
// configure waits for a lock, and if a new call to update() happens in the meantime we can drop this one.
// this works similar to how React implemented concurrent rendering, the newer call to update() has higher priority.
ReactLogger.log(level: .info, message: "A new configure { ... } call arrived, aborting this one...")
return
}
cameraSession.configure { config in
// Input Camera Device
config.cameraId = cameraId as? String

View File

@ -98,29 +98,21 @@ class CameraSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate, AVC
Any changes in here will be re-configured only if required, and under a lock.
The `configuration` object is a copy of the currently active configuration that can be modified by the caller in the lambda.
*/
func configure(_ lambda: (_ configuration: CameraConfiguration) throws -> Void) {
// This is the latest call to configure()
let time = DispatchTime.now()
currentConfigureCall = time
ReactLogger.log(level: .info, message: "Updating Session Configuration...")
// Let caller configure a new configuration for the Camera.
let config = CameraConfiguration(copyOf: configuration)
do {
try lambda(config)
} catch {
onConfigureError(error)
}
let difference = CameraConfiguration.Difference(between: configuration, and: config)
func configure(_ lambda: @escaping (_ configuration: CameraConfiguration) throws -> Void) {
ReactLogger.log(level: .info, message: "configure { ... }: Waiting for lock...")
// Set up Camera (Video) Capture Session (on camera queue, acts like a lock)
CameraQueues.cameraQueue.async {
// Cancel configuration if there has already been a new config
guard self.currentConfigureCall == time else {
// configure() has been called again just now, skip this one so the new call takes over.
return
// Let caller configure a new configuration for the Camera.
let config = CameraConfiguration(copyOf: self.configuration)
do {
try lambda(config)
} catch {
self.onConfigureError(error)
}
let difference = CameraConfiguration.Difference(between: self.configuration, and: config)
ReactLogger.log(level: .info, message: "configure { ... }: Updating CameraSession Configuration... \(difference)")
do {
// If needed, configure the AVCaptureSession (inputs, outputs)