From 68707322feee0ae31196392880cde167f6fb7517 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Thu, 30 Dec 2021 11:39:17 +0100 Subject: [PATCH] feat: Use new CameraX Recorder API (Upgrade CameraX to alpha12/32) (#543) * chore(deps): Upgrade CameraX to alpha10/30 * chore(deps): Add first stable CameraX Video library * feat: Use new CameraX Video API * chore(deps): Upgrade CameraX from 10 -> 12 * fix: Replace deprecated APIs * Update CameraViewModule.kt * fix: Fix file creation --- android/build.gradle | 13 +-- .../mrousavy/camera/CameraView+RecordVideo.kt | 81 +++++++++++-------- .../java/com/mrousavy/camera/CameraView.kt | 44 +++++++--- .../com/mrousavy/camera/CameraViewModule.kt | 7 +- .../main/java/com/mrousavy/camera/Errors.kt | 53 ++++++++++-- 5 files changed, 136 insertions(+), 62 deletions(-) diff --git a/android/build.gradle b/android/build.gradle index 2f34c11..f98216e 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -233,11 +233,14 @@ dependencies { implementation "org.jetbrains.kotlinx:kotlinx-coroutines-guava:1.5.2" implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:1.5.2" - implementation "androidx.camera:camera-core:1.1.0-alpha09" - implementation "androidx.camera:camera-camera2:1.1.0-alpha09" - implementation "androidx.camera:camera-lifecycle:1.1.0-alpha09" - implementation "androidx.camera:camera-extensions:1.0.0-alpha29" - implementation "androidx.camera:camera-view:1.0.0-alpha29" + implementation "androidx.camera:camera-core:1.1.0-alpha12" + implementation "androidx.camera:camera-camera2:1.1.0-alpha12" + implementation "androidx.camera:camera-lifecycle:1.1.0-alpha12" + implementation "androidx.camera:camera-video:1.1.0-alpha12" + + implementation "androidx.camera:camera-view:1.0.0-alpha32" + implementation "androidx.camera:camera-extensions:1.0.0-alpha32" + implementation "androidx.exifinterface:exifinterface:1.3.3" } diff --git a/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt b/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt index 03efc8e..d72a07c 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt @@ -3,17 +3,19 @@ package com.mrousavy.camera import android.Manifest import android.annotation.SuppressLint import android.content.pm.PackageManager -import androidx.camera.core.VideoCapture +import androidx.camera.video.FileOutputOptions +import androidx.camera.video.VideoRecordEvent import androidx.core.content.ContextCompat +import androidx.core.util.Consumer import com.facebook.react.bridge.* import com.mrousavy.camera.utils.makeErrorMap -import kotlinx.coroutines.* import java.io.File +import java.text.SimpleDateFormat +import java.util.* data class TemporaryFile(val path: String) -@SuppressLint("RestrictedApi", "MissingPermission") -suspend fun CameraView.startRecording(options: ReadableMap, onRecordCallback: Callback): TemporaryFile { +fun CameraView.startRecording(options: ReadableMap, onRecordCallback: Callback) { if (videoCapture == null) { if (video == true) { throw CameraNotReadyError() @@ -35,44 +37,49 @@ suspend fun CameraView.startRecording(options: ReadableMap, onRecordCallback: Ca camera!!.cameraControl.enableTorch(enableFlash) } - @Suppress("BlockingMethodInNonBlockingContext") // in withContext we are not blocking. False positive. - val videoFile = withContext(Dispatchers.IO) { - File.createTempFile("video", ".mp4", context.cacheDir).apply { deleteOnExit() } + val id = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(Date()) + val file = File.createTempFile("VisionCamera-${id}", ".mp4") + val fileOptions = FileOutputOptions.Builder(file).build() + + var recording = videoCapture!! + .prepareRecording(context, fileOptions) + + if (audio == true) { + @SuppressLint("MissingPermission") + recording = recording.withAudioEnabled() } - val videoFileOptions = VideoCapture.OutputFileOptions.Builder(videoFile) - videoCapture!!.startRecording( - videoFileOptions.build(), recordVideoExecutor, - object : VideoCapture.OnVideoSavedCallback { - override fun onVideoSaved(outputFileResults: VideoCapture.OutputFileResults) { - val map = Arguments.createMap() - map.putString("path", videoFile.absolutePath) - // TODO: duration and size - see https://github.com/mrousavy/react-native-vision-camera/issues/77 - onRecordCallback(map, null) - - // reset the torch mode - camera!!.cameraControl.enableTorch(torch == "on") - } - - override fun onError(videoCaptureError: Int, message: String, cause: Throwable?) { - val error = when (videoCaptureError) { - VideoCapture.ERROR_ENCODER -> VideoEncoderError(message, cause) - VideoCapture.ERROR_FILE_IO -> FileIOError(message, cause) - VideoCapture.ERROR_INVALID_CAMERA -> InvalidCameraError(message, cause) - VideoCapture.ERROR_MUXER -> VideoMuxerError(message, cause) - VideoCapture.ERROR_RECORDING_IN_PROGRESS -> RecordingInProgressError(message, cause) - else -> UnknownCameraError(Error(message, cause)) + activeVideoRecording = recording.start(ContextCompat.getMainExecutor(context), object : Consumer { + override fun accept(event: VideoRecordEvent?) { + if (event is VideoRecordEvent.Finalize) { + if (event.hasError()) { + // error occured! + val error = when (event.error) { + VideoRecordEvent.Finalize.ERROR_ENCODING_FAILED -> VideoEncoderError(event.cause) + VideoRecordEvent.Finalize.ERROR_FILE_SIZE_LIMIT_REACHED -> FileSizeLimitReachedError(event.cause) + VideoRecordEvent.Finalize.ERROR_INSUFFICIENT_STORAGE -> InsufficientStorageError(event.cause) + VideoRecordEvent.Finalize.ERROR_INVALID_OUTPUT_OPTIONS -> InvalidVideoOutputOptionsError(event.cause) + VideoRecordEvent.Finalize.ERROR_NO_VALID_DATA -> NoValidDataError(event.cause) + VideoRecordEvent.Finalize.ERROR_RECORDER_ERROR -> RecorderError(event.cause) + VideoRecordEvent.Finalize.ERROR_SOURCE_INACTIVE -> InactiveSourceError(event.cause) + else -> UnknownCameraError(event.cause) + } + val map = makeErrorMap("${error.domain}/${error.id}", error.message, error) + onRecordCallback(null, map) + } else { + // recording saved successfully! + val map = Arguments.createMap() + map.putString("path", event.outputResults.outputUri.toString()) + map.putDouble("duration", /* seconds */ event.recordingStats.recordedDurationNanos.toDouble() / 1000000.0 / 1000.0) + map.putDouble("size", /* kB */ event.recordingStats.numBytesRecorded.toDouble() / 1000.0) + onRecordCallback(map, null) } - val map = makeErrorMap("${error.domain}/${error.id}", error.message, error) - onRecordCallback(null, map) // reset the torch mode camera!!.cameraControl.enableTorch(torch == "on") } } - ) - - return TemporaryFile(videoFile.absolutePath) + }) } @SuppressLint("RestrictedApi") @@ -80,8 +87,12 @@ fun CameraView.stopRecording() { if (videoCapture == null) { throw CameraNotReadyError() } + if (activeVideoRecording == null) { + throw NoRecordingInProgressError() + } + + activeVideoRecording!!.stop() - videoCapture!!.stopRecording() // reset torch mode to original value camera!!.cameraControl.enableTorch(torch == "on") } diff --git a/android/src/main/java/com/mrousavy/camera/CameraView.kt b/android/src/main/java/com/mrousavy/camera/CameraView.kt index a6c9c5a..171c67d 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraView.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraView.kt @@ -16,6 +16,8 @@ import androidx.camera.core.* import androidx.camera.core.impl.* import androidx.camera.extensions.* import androidx.camera.lifecycle.ProcessCameraProvider +import androidx.camera.video.* +import androidx.camera.video.VideoCapture import androidx.camera.view.PreviewView import androidx.core.content.ContextCompat import androidx.lifecycle.* @@ -121,10 +123,12 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer internal var camera: Camera? = null internal var imageCapture: ImageCapture? = null - internal var videoCapture: VideoCapture? = null + internal var videoCapture: Recorder? = null private var imageAnalysis: ImageAnalysis? = null private var preview: Preview? = null + internal var activeVideoRecording: Recording? = null + private var lastFrameProcessorCall = System.currentTimeMillis() private var extensionsManager: ExtensionsManager? = null @@ -234,7 +238,7 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer preview?.targetRotation = rotation imageCapture?.targetRotation = rotation imageAnalysis?.targetRotation = rotation - videoCapture?.setTargetRotation(rotation) + // TODO: videoCapture?.setTargetRotation(rotation) } } @@ -338,11 +342,11 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer val tryEnableExtension: (suspend (extension: Int) -> Unit) = lambda@ { extension -> if (extensionsManager == null) { Log.i(TAG, "Initializing ExtensionsManager...") - extensionsManager = ExtensionsManager.getInstance(context).await() + extensionsManager = ExtensionsManager.getInstanceAsync(context, cameraProvider).await() } - if (extensionsManager!!.isExtensionAvailable(cameraProvider, cameraSelector, extension)) { + if (extensionsManager!!.isExtensionAvailable(cameraSelector, extension)) { Log.i(TAG, "Enabling extension $extension...") - cameraSelector = extensionsManager!!.getExtensionEnabledCameraSelector(cameraProvider, cameraSelector, extension) + cameraSelector = extensionsManager!!.getExtensionEnabledCameraSelector(cameraSelector, extension) } else { Log.e(TAG, "Extension $extension is not available for the given Camera!") throw when (extension) { @@ -355,11 +359,14 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer val previewBuilder = Preview.Builder() .setTargetRotation(rotation) + val imageCaptureBuilder = ImageCapture.Builder() .setTargetRotation(rotation) .setCaptureMode(ImageCapture.CAPTURE_MODE_MINIMIZE_LATENCY) - val videoCaptureBuilder = VideoCapture.Builder() - .setTargetRotation(rotation) + + val videoRecorderBuilder = Recorder.Builder() + .setExecutor(cameraExecutor) + val imageAnalysisBuilder = ImageAnalysis.Builder() .setTargetRotation(rotation) .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) @@ -371,7 +378,7 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer val aspectRatio = aspectRatio(previewView.height, previewView.width) // flipped because it's in sensor orientation. previewBuilder.setTargetAspectRatio(aspectRatio) imageCaptureBuilder.setTargetAspectRatio(aspectRatio) - videoCaptureBuilder.setTargetAspectRatio(aspectRatio) + // TODO: Aspect Ratio for Video Recorder? imageAnalysisBuilder.setTargetAspectRatio(aspectRatio) } else { // User has selected a custom format={}. Use that @@ -379,9 +386,16 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer Log.i(TAG, "Using custom format - photo: ${format.photoSize}, video: ${format.videoSize} @ $fps FPS") previewBuilder.setTargetResolution(format.videoSize) imageCaptureBuilder.setTargetResolution(format.photoSize) - videoCaptureBuilder.setTargetResolution(format.videoSize) imageAnalysisBuilder.setTargetResolution(format.videoSize) + // TODO: Ability to select resolution exactly depending on format? Just like on iOS... + when (min(format.videoSize.height, format.videoSize.width)) { + in 0..480 -> videoRecorderBuilder.setQualitySelector(QualitySelector.from(Quality.SD)) + in 480..720 -> videoRecorderBuilder.setQualitySelector(QualitySelector.from(Quality.HD)) + in 720..1080 -> videoRecorderBuilder.setQualitySelector(QualitySelector.from(Quality.FHD)) + in 1080..2160 -> videoRecorderBuilder.setQualitySelector(QualitySelector.from(Quality.UHD)) + } + fps?.let { fps -> if (format.frameRateRanges.any { it.contains(fps) }) { // Camera supports the given FPS (frame rate range) @@ -391,7 +405,7 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer Camera2Interop.Extender(previewBuilder) .setCaptureRequestOption(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(fps, fps)) .setCaptureRequestOption(CaptureRequest.SENSOR_FRAME_DURATION, frameDuration) - videoCaptureBuilder.setVideoFrameRate(fps) + // TODO: Frame Rate/FPS for Video Recorder? } else { throw FpsNotContainedInFormatError(fps) } @@ -404,8 +418,12 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer } } + val videoRecorder = videoRecorderBuilder.build() + val videoCapture = VideoCapture.withOutput(videoRecorder) + videoCapture.targetRotation = rotation + // Unbind use cases before rebinding - videoCapture = null + this.videoCapture = null imageCapture = null imageAnalysis = null cameraProvider.unbindAll() @@ -414,8 +432,8 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer val useCases = ArrayList() if (video == true) { Log.i(TAG, "Adding VideoCapture use-case...") - videoCapture = videoCaptureBuilder.build() - useCases.add(videoCapture!!) + this.videoCapture = videoRecorder + useCases.add(videoCapture) } if (photo == true) { if (fallbackToSnapshot) { diff --git a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt index be04151..26da75c 100644 --- a/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt +++ b/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt @@ -124,8 +124,8 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase val startTime = System.currentTimeMillis() coroutineScope.launch { withPromise(promise) { - val extensionsManager = ExtensionsManager.getInstance(reactApplicationContext).await() val cameraProvider = ProcessCameraProvider.getInstance(reactApplicationContext).await() + val extensionsManager = ExtensionsManager.getInstanceAsync(reactApplicationContext, cameraProvider).await() ProcessCameraProvider.getInstance(reactApplicationContext).await() val manager = reactApplicationContext.getSystemService(Context.CAMERA_SERVICE) as? CameraManager @@ -162,8 +162,8 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase else null val fpsRanges = characteristics.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES)!! - val supportsHdr = extensionsManager.isExtensionAvailable(cameraProvider, cameraSelector, ExtensionMode.HDR) - val supportsLowLightBoost = extensionsManager.isExtensionAvailable(cameraProvider, cameraSelector, ExtensionMode.NIGHT) + val supportsHdr = extensionsManager.isExtensionAvailable(cameraSelector, ExtensionMode.HDR) + val supportsLowLightBoost = extensionsManager.isExtensionAvailable(cameraSelector, ExtensionMode.NIGHT) // see https://developer.android.com/reference/android/hardware/camera2/CameraDevice#regular-capture val supportsParallelVideoProcessing = hardwareLevel != CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY && hardwareLevel != CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LIMITED @@ -198,6 +198,7 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase val formats = Arguments.createArray() + // TODO: Get supported video qualities with QualitySelector.getSupportedQualities(...) cameraConfig.outputFormats.forEach { formatId -> val formatName = parseImageFormat(formatId) diff --git a/android/src/main/java/com/mrousavy/camera/Errors.kt b/android/src/main/java/com/mrousavy/camera/Errors.kt index 791891d..b81fe17 100644 --- a/android/src/main/java/com/mrousavy/camera/Errors.kt +++ b/android/src/main/java/com/mrousavy/camera/Errors.kt @@ -1,6 +1,7 @@ package com.mrousavy.camera import android.graphics.ImageFormat +import androidx.camera.video.VideoRecordEvent.Finalize.VideoRecordError abstract class CameraError( /** @@ -59,13 +60,53 @@ class VideoNotEnabledError : CameraError("capture", "video-not-enabled", "Video class PhotoNotEnabledError : CameraError("capture", "photo-not-enabled", "Photo capture is disabled! Pass `photo={true}` to enable photo capture.") class InvalidFormatError(format: Int) : CameraError("capture", "invalid-photo-format", "The Photo has an invalid format! Expected ${ImageFormat.YUV_420_888}, actual: $format") -class VideoEncoderError(message: String, cause: Throwable? = null) : CameraError("capture", "encoder-error", message, cause) -class VideoMuxerError(message: String, cause: Throwable? = null) : CameraError("capture", "muxer-error", message, cause) -class RecordingInProgressError(message: String, cause: Throwable? = null) : CameraError("capture", "recording-in-progress", message, cause) -class FileIOError(message: String, cause: Throwable? = null) : CameraError("capture", "file-io-error", message, cause) -class InvalidCameraError(message: String, cause: Throwable? = null) : CameraError("capture", "not-bound-error", message, cause) + +class VideoEncoderError(cause: Throwable?) : CameraError("capture", "encoder-error", "The recording failed while encoding.\n" + + "This error may be generated when the video or audio codec encounters an error during encoding. " + + "When this happens and the output file is generated, the output file is not properly constructed. " + + "The application will need to clean up the output file, such as deleting the file.", + cause) + +class InvalidVideoOutputOptionsError(cause: Throwable?) : CameraError("capture", "invalid-video-options", + "The recording failed due to invalid output options.\n" + + "This error is generated when invalid output options have been used while preparing a recording", + cause) + +class RecorderError(cause: Throwable?) : CameraError("capture", "recorder-error", + "The recording failed because the Recorder is in an unrecoverable error state.\n" + + "When this happens and the output file is generated, the output file is not properly constructed. " + + "The application will need to clean up the output file, such as deleting the file. " + + "Such an error will usually require creating a new Recorder object to start a new recording.", + cause) + +class NoValidDataError(cause: Throwable?) : CameraError("capture", "no-valid-data", + "The recording failed because no valid data was produced to be recorded.\n" + + "This error is generated when the essential data for a recording to be played correctly is missing, for example, " + + "a recording must contain at least one key frame. The application will need to clean up the output file, such as deleting the file.", + cause) + +class InactiveSourceError(cause: Throwable?) : CameraError("capture", "inactive-source", + "The recording failed because the source becomes inactive and stops sending frames.\n" + + "One case is that if camera is closed due to lifecycle stopped, the active recording will be finalized with this error, " + + "and the output will be generated, containing the frames produced before camera closing. " + + "Attempting to start a new recording will be finalized immediately if the source remains inactive and no output will be generated.", + cause) + +class InsufficientStorageError(cause: Throwable?) : CameraError("capture", "insufficient-storage", + "The recording failed due to insufficient storage space.\n" + + "There are two possible cases that will cause this error.\n" + + "1. The storage is already full before the recording starts, so no output file will be generated.\n" + + "2. The storage becomes full during recording, so the output file will be generated.", + cause) + +class FileSizeLimitReachedError(cause: Throwable?) : CameraError("capture", "file-size-limit-reached", + "The recording failed due to file size limitation.\n" + + "The file size limitation will refer to OutputOptions.getFileSizeLimit(). The output file will still be generated with this error.", + cause) + +class NoRecordingInProgressError : CameraError("capture", "no-recording-in-progress", "No active recording in progress!") class CameraManagerUnavailableError : CameraError("system", "no-camera-manager", "The Camera manager instance was unavailable for the current Application!") class ViewNotFoundError(viewId: Int) : CameraError("system", "view-not-found", "The given view (ID $viewId) was not found in the view manager.") -class UnknownCameraError(cause: Throwable) : CameraError("unknown", "unknown", cause.message ?: "An unknown camera error occured.", cause) +class UnknownCameraError(cause: Throwable?) : CameraError("unknown", "unknown", cause?.message ?: "An unknown camera error occured.", cause)