feat: Use new CameraX Recorder API (Upgrade CameraX to alpha12/32) (#543)

* chore(deps): Upgrade CameraX to alpha10/30

* chore(deps): Add first stable CameraX Video library

* feat: Use new CameraX Video API

* chore(deps): Upgrade CameraX from 10 -> 12

* fix: Replace deprecated APIs

* Update CameraViewModule.kt

* fix: Fix file creation

android/build.gradle

@@ -233,11 +233,14 @@ dependencies {
   implementation "org.jetbrains.kotlinx:kotlinx-coroutines-guava:1.5.2"
   implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:1.5.2"
 
-  implementation "androidx.camera:camera-core:1.1.0-alpha09"
+  implementation "androidx.camera:camera-core:1.1.0-alpha12"
-  implementation "androidx.camera:camera-camera2:1.1.0-alpha09"
+  implementation "androidx.camera:camera-camera2:1.1.0-alpha12"
-  implementation "androidx.camera:camera-lifecycle:1.1.0-alpha09"
+  implementation "androidx.camera:camera-lifecycle:1.1.0-alpha12"
-  implementation "androidx.camera:camera-extensions:1.0.0-alpha29"
+  implementation "androidx.camera:camera-video:1.1.0-alpha12"
-  implementation "androidx.camera:camera-view:1.0.0-alpha29"
+
+  implementation "androidx.camera:camera-view:1.0.0-alpha32"
+  implementation "androidx.camera:camera-extensions:1.0.0-alpha32"
+
   implementation "androidx.exifinterface:exifinterface:1.3.3"
 }
 

android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt

@@ -3,17 +3,19 @@ package com.mrousavy.camera
 import android.Manifest
 import android.annotation.SuppressLint
 import android.content.pm.PackageManager
-import androidx.camera.core.VideoCapture
+import androidx.camera.video.FileOutputOptions
+import androidx.camera.video.VideoRecordEvent
 import androidx.core.content.ContextCompat
+import androidx.core.util.Consumer
 import com.facebook.react.bridge.*
 import com.mrousavy.camera.utils.makeErrorMap
-import kotlinx.coroutines.*
 import java.io.File
+import java.text.SimpleDateFormat
+import java.util.*
 
 data class TemporaryFile(val path: String)
 
-@SuppressLint("RestrictedApi", "MissingPermission")
+fun CameraView.startRecording(options: ReadableMap, onRecordCallback: Callback) {
-suspend fun CameraView.startRecording(options: ReadableMap, onRecordCallback: Callback): TemporaryFile {
   if (videoCapture == null) {
     if (video == true) {
       throw CameraNotReadyError()
@@ -35,44 +37,49 @@ suspend fun CameraView.startRecording(options: ReadableMap, onRecordCallback: Ca
     camera!!.cameraControl.enableTorch(enableFlash)
   }
 
-  @Suppress("BlockingMethodInNonBlockingContext") // in withContext we are not blocking. False positive.
+  val id = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(Date())
-  val videoFile = withContext(Dispatchers.IO) {
+  val file = File.createTempFile("VisionCamera-${id}", ".mp4")
-    File.createTempFile("video", ".mp4", context.cacheDir).apply { deleteOnExit() }
+  val fileOptions = FileOutputOptions.Builder(file).build()
-  }
-  val videoFileOptions = VideoCapture.OutputFileOptions.Builder(videoFile)
 
-  videoCapture!!.startRecording(
+  var recording = videoCapture!!
-    videoFileOptions.build(), recordVideoExecutor,
+    .prepareRecording(context, fileOptions)
-    object : VideoCapture.OnVideoSavedCallback {
-      override fun onVideoSaved(outputFileResults: VideoCapture.OutputFileResults) {
-        val map = Arguments.createMap()
-        map.putString("path", videoFile.absolutePath)
-        // TODO: duration and size - see https://github.com/mrousavy/react-native-vision-camera/issues/77
-        onRecordCallback(map, null)
 
-        // reset the torch mode
+  if (audio == true) {
-        camera!!.cameraControl.enableTorch(torch == "on")
+    @SuppressLint("MissingPermission")
+    recording = recording.withAudioEnabled()
   }
 
-      override fun onError(videoCaptureError: Int, message: String, cause: Throwable?) {
+  activeVideoRecording = recording.start(ContextCompat.getMainExecutor(context), object : Consumer<VideoRecordEvent> {
-        val error = when (videoCaptureError) {
+    override fun accept(event: VideoRecordEvent?) {
-          VideoCapture.ERROR_ENCODER -> VideoEncoderError(message, cause)
+      if (event is VideoRecordEvent.Finalize) {
-          VideoCapture.ERROR_FILE_IO -> FileIOError(message, cause)
+        if (event.hasError()) {
-          VideoCapture.ERROR_INVALID_CAMERA -> InvalidCameraError(message, cause)
+          // error occured!
-          VideoCapture.ERROR_MUXER -> VideoMuxerError(message, cause)
+          val error = when (event.error) {
-          VideoCapture.ERROR_RECORDING_IN_PROGRESS -> RecordingInProgressError(message, cause)
+            VideoRecordEvent.Finalize.ERROR_ENCODING_FAILED -> VideoEncoderError(event.cause)
-          else -> UnknownCameraError(Error(message, cause))
+            VideoRecordEvent.Finalize.ERROR_FILE_SIZE_LIMIT_REACHED -> FileSizeLimitReachedError(event.cause)
+            VideoRecordEvent.Finalize.ERROR_INSUFFICIENT_STORAGE -> InsufficientStorageError(event.cause)
+            VideoRecordEvent.Finalize.ERROR_INVALID_OUTPUT_OPTIONS -> InvalidVideoOutputOptionsError(event.cause)
+            VideoRecordEvent.Finalize.ERROR_NO_VALID_DATA -> NoValidDataError(event.cause)
+            VideoRecordEvent.Finalize.ERROR_RECORDER_ERROR -> RecorderError(event.cause)
+            VideoRecordEvent.Finalize.ERROR_SOURCE_INACTIVE -> InactiveSourceError(event.cause)
+            else -> UnknownCameraError(event.cause)
           }
           val map = makeErrorMap("${error.domain}/${error.id}", error.message, error)
           onRecordCallback(null, map)
+        } else {
+          // recording saved successfully!
+          val map = Arguments.createMap()
+          map.putString("path", event.outputResults.outputUri.toString())
+          map.putDouble("duration", /* seconds */ event.recordingStats.recordedDurationNanos.toDouble() / 1000000.0 / 1000.0)
+          map.putDouble("size", /* kB */ event.recordingStats.numBytesRecorded.toDouble() / 1000.0)
+          onRecordCallback(map, null)
+        }
 
         // reset the torch mode
         camera!!.cameraControl.enableTorch(torch == "on")
       }
     }
-  )
+  })
-
-  return TemporaryFile(videoFile.absolutePath)
 }
 
 @SuppressLint("RestrictedApi")
@@ -80,8 +87,12 @@ fun CameraView.stopRecording() {
   if (videoCapture == null) {
     throw CameraNotReadyError()
   }
+  if (activeVideoRecording == null) {
+    throw NoRecordingInProgressError()
+  }
+
+  activeVideoRecording!!.stop()
 
-  videoCapture!!.stopRecording()
   // reset torch mode to original value
   camera!!.cameraControl.enableTorch(torch == "on")
 }

android/src/main/java/com/mrousavy/camera/CameraView.kt

@@ -16,6 +16,8 @@ import androidx.camera.core.*
 import androidx.camera.core.impl.*
 import androidx.camera.extensions.*
 import androidx.camera.lifecycle.ProcessCameraProvider
+import androidx.camera.video.*
+import androidx.camera.video.VideoCapture
 import androidx.camera.view.PreviewView
 import androidx.core.content.ContextCompat
 import androidx.lifecycle.*
@@ -121,10 +123,12 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
 
   internal var camera: Camera? = null
   internal var imageCapture: ImageCapture? = null
-  internal var videoCapture: VideoCapture? = null
+  internal var videoCapture: Recorder? = null
   private var imageAnalysis: ImageAnalysis? = null
   private var preview: Preview? = null
 
+  internal var activeVideoRecording: Recording? = null
+
   private var lastFrameProcessorCall = System.currentTimeMillis()
 
   private var extensionsManager: ExtensionsManager? = null
@@ -234,7 +238,7 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
       preview?.targetRotation = rotation
       imageCapture?.targetRotation = rotation
       imageAnalysis?.targetRotation = rotation
-      videoCapture?.setTargetRotation(rotation)
+      // TODO: videoCapture?.setTargetRotation(rotation)
     }
   }
 
@@ -338,11 +342,11 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
       val tryEnableExtension: (suspend (extension: Int) -> Unit) = lambda@ { extension ->
         if (extensionsManager == null) {
           Log.i(TAG, "Initializing ExtensionsManager...")
-          extensionsManager = ExtensionsManager.getInstance(context).await()
+          extensionsManager = ExtensionsManager.getInstanceAsync(context, cameraProvider).await()
         }
-        if (extensionsManager!!.isExtensionAvailable(cameraProvider, cameraSelector, extension)) {
+        if (extensionsManager!!.isExtensionAvailable(cameraSelector, extension)) {
           Log.i(TAG, "Enabling extension $extension...")
-          cameraSelector = extensionsManager!!.getExtensionEnabledCameraSelector(cameraProvider, cameraSelector, extension)
+          cameraSelector = extensionsManager!!.getExtensionEnabledCameraSelector(cameraSelector, extension)
         } else {
           Log.e(TAG, "Extension $extension is not available for the given Camera!")
           throw when (extension) {
@@ -355,11 +359,14 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
 
       val previewBuilder = Preview.Builder()
         .setTargetRotation(rotation)
+
       val imageCaptureBuilder = ImageCapture.Builder()
         .setTargetRotation(rotation)
         .setCaptureMode(ImageCapture.CAPTURE_MODE_MINIMIZE_LATENCY)
-      val videoCaptureBuilder = VideoCapture.Builder()
+
-        .setTargetRotation(rotation)
+      val videoRecorderBuilder = Recorder.Builder()
+        .setExecutor(cameraExecutor)
+
       val imageAnalysisBuilder = ImageAnalysis.Builder()
         .setTargetRotation(rotation)
         .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
@@ -371,7 +378,7 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
         val aspectRatio = aspectRatio(previewView.height, previewView.width) // flipped because it's in sensor orientation.
         previewBuilder.setTargetAspectRatio(aspectRatio)
         imageCaptureBuilder.setTargetAspectRatio(aspectRatio)
-        videoCaptureBuilder.setTargetAspectRatio(aspectRatio)
+        // TODO: Aspect Ratio for Video Recorder?
         imageAnalysisBuilder.setTargetAspectRatio(aspectRatio)
       } else {
         // User has selected a custom format={}. Use that
@@ -379,9 +386,16 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
         Log.i(TAG, "Using custom format - photo: ${format.photoSize}, video: ${format.videoSize} @ $fps FPS")
         previewBuilder.setTargetResolution(format.videoSize)
         imageCaptureBuilder.setTargetResolution(format.photoSize)
-        videoCaptureBuilder.setTargetResolution(format.videoSize)
         imageAnalysisBuilder.setTargetResolution(format.videoSize)
 
+        // TODO: Ability to select resolution exactly depending on format? Just like on iOS...
+        when (min(format.videoSize.height, format.videoSize.width)) {
+          in 0..480 -> videoRecorderBuilder.setQualitySelector(QualitySelector.from(Quality.SD))
+          in 480..720 -> videoRecorderBuilder.setQualitySelector(QualitySelector.from(Quality.HD))
+          in 720..1080 -> videoRecorderBuilder.setQualitySelector(QualitySelector.from(Quality.FHD))
+          in 1080..2160 -> videoRecorderBuilder.setQualitySelector(QualitySelector.from(Quality.UHD))
+        }
+
         fps?.let { fps ->
           if (format.frameRateRanges.any { it.contains(fps) }) {
             // Camera supports the given FPS (frame rate range)
@@ -391,7 +405,7 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
             Camera2Interop.Extender(previewBuilder)
               .setCaptureRequestOption(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(fps, fps))
               .setCaptureRequestOption(CaptureRequest.SENSOR_FRAME_DURATION, frameDuration)
-            videoCaptureBuilder.setVideoFrameRate(fps)
+            // TODO: Frame Rate/FPS for Video Recorder?
           } else {
             throw FpsNotContainedInFormatError(fps)
           }
@@ -404,8 +418,12 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
         }
       }
 
+      val videoRecorder = videoRecorderBuilder.build()
+      val videoCapture = VideoCapture.withOutput(videoRecorder)
+      videoCapture.targetRotation = rotation
+
       // Unbind use cases before rebinding
-      videoCapture = null
+      this.videoCapture = null
       imageCapture = null
       imageAnalysis = null
       cameraProvider.unbindAll()
@@ -414,8 +432,8 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
       val useCases = ArrayList<UseCase>()
       if (video == true) {
         Log.i(TAG, "Adding VideoCapture use-case...")
-        videoCapture = videoCaptureBuilder.build()
+        this.videoCapture = videoRecorder
-        useCases.add(videoCapture!!)
+        useCases.add(videoCapture)
       }
       if (photo == true) {
         if (fallbackToSnapshot) {

android/src/main/java/com/mrousavy/camera/CameraViewModule.kt

@@ -124,8 +124,8 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase
     val startTime = System.currentTimeMillis()
     coroutineScope.launch {
       withPromise(promise) {
-        val extensionsManager = ExtensionsManager.getInstance(reactApplicationContext).await()
         val cameraProvider = ProcessCameraProvider.getInstance(reactApplicationContext).await()
+        val extensionsManager = ExtensionsManager.getInstanceAsync(reactApplicationContext, cameraProvider).await()
         ProcessCameraProvider.getInstance(reactApplicationContext).await()
 
         val manager = reactApplicationContext.getSystemService(Context.CAMERA_SERVICE) as? CameraManager
@@ -162,8 +162,8 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase
           else null
           val fpsRanges = characteristics.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES)!!
 
-          val supportsHdr = extensionsManager.isExtensionAvailable(cameraProvider, cameraSelector, ExtensionMode.HDR)
+          val supportsHdr = extensionsManager.isExtensionAvailable(cameraSelector, ExtensionMode.HDR)
-          val supportsLowLightBoost = extensionsManager.isExtensionAvailable(cameraProvider, cameraSelector, ExtensionMode.NIGHT)
+          val supportsLowLightBoost = extensionsManager.isExtensionAvailable(cameraSelector, ExtensionMode.NIGHT)
           // see https://developer.android.com/reference/android/hardware/camera2/CameraDevice#regular-capture
           val supportsParallelVideoProcessing = hardwareLevel != CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY && hardwareLevel != CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LIMITED
 
@@ -198,6 +198,7 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase
 
           val formats = Arguments.createArray()
 
+          // TODO: Get supported video qualities with QualitySelector.getSupportedQualities(...)
           cameraConfig.outputFormats.forEach { formatId ->
             val formatName = parseImageFormat(formatId)
 

android/src/main/java/com/mrousavy/camera/Errors.kt

@@ -1,6 +1,7 @@
 package com.mrousavy.camera
 
 import android.graphics.ImageFormat
+import androidx.camera.video.VideoRecordEvent.Finalize.VideoRecordError
 
 abstract class CameraError(
   /**
@@ -59,13 +60,53 @@ class VideoNotEnabledError : CameraError("capture", "video-not-enabled", "Video
 class PhotoNotEnabledError : CameraError("capture", "photo-not-enabled", "Photo capture is disabled! Pass `photo={true}` to enable photo capture.")
 
 class InvalidFormatError(format: Int) : CameraError("capture", "invalid-photo-format", "The Photo has an invalid format! Expected ${ImageFormat.YUV_420_888}, actual: $format")
-class VideoEncoderError(message: String, cause: Throwable? = null) : CameraError("capture", "encoder-error", message, cause)
+
-class VideoMuxerError(message: String, cause: Throwable? = null) : CameraError("capture", "muxer-error", message, cause)
+class VideoEncoderError(cause: Throwable?) : CameraError("capture", "encoder-error", "The recording failed while encoding.\n" +
-class RecordingInProgressError(message: String, cause: Throwable? = null) : CameraError("capture", "recording-in-progress", message, cause)
+  "This error may be generated when the video or audio codec encounters an error during encoding. " +
-class FileIOError(message: String, cause: Throwable? = null) : CameraError("capture", "file-io-error", message, cause)
+  "When this happens and the output file is generated, the output file is not properly constructed. " +
-class InvalidCameraError(message: String, cause: Throwable? = null) : CameraError("capture", "not-bound-error", message, cause)
+  "The application will need to clean up the output file, such as deleting the file.",
+  cause)
+
+class InvalidVideoOutputOptionsError(cause: Throwable?) : CameraError("capture", "invalid-video-options",
+  "The recording failed due to invalid output options.\n" +
+  "This error is generated when invalid output options have been used while preparing a recording",
+  cause)
+
+class RecorderError(cause: Throwable?) : CameraError("capture", "recorder-error",
+  "The recording failed because the Recorder is in an unrecoverable error state.\n" +
+  "When this happens and the output file is generated, the output file is not properly constructed. " +
+  "The application will need to clean up the output file, such as deleting the file. " +
+  "Such an error will usually require creating a new Recorder object to start a new recording.",
+  cause)
+
+class NoValidDataError(cause: Throwable?) : CameraError("capture", "no-valid-data",
+  "The recording failed because no valid data was produced to be recorded.\n" +
+  "This error is generated when the essential data for a recording to be played correctly is missing, for example, " +
+  "a recording must contain at least one key frame. The application will need to clean up the output file, such as deleting the file.",
+  cause)
+
+class InactiveSourceError(cause: Throwable?) : CameraError("capture", "inactive-source",
+  "The recording failed because the source becomes inactive and stops sending frames.\n" +
+  "One case is that if camera is closed due to lifecycle stopped, the active recording will be finalized with this error, " +
+  "and the output will be generated, containing the frames produced before camera closing. " +
+  "Attempting to start a new recording will be finalized immediately if the source remains inactive and no output will be generated.",
+  cause)
+
+class InsufficientStorageError(cause: Throwable?) : CameraError("capture", "insufficient-storage",
+  "The recording failed due to insufficient storage space.\n" +
+  "There are two possible cases that will cause this error.\n" +
+  "1. The storage is already full before the recording starts, so no output file will be generated.\n" +
+  "2. The storage becomes full during recording, so the output file will be generated.",
+  cause)
+
+class FileSizeLimitReachedError(cause: Throwable?) : CameraError("capture", "file-size-limit-reached",
+  "The recording failed due to file size limitation.\n" +
+  "The file size limitation will refer to OutputOptions.getFileSizeLimit(). The output file will still be generated with this error.",
+  cause)
+
+class NoRecordingInProgressError : CameraError("capture", "no-recording-in-progress", "No active recording in progress!")
 
 class CameraManagerUnavailableError : CameraError("system", "no-camera-manager", "The Camera manager instance was unavailable for the current Application!")
 class ViewNotFoundError(viewId: Int) : CameraError("system", "view-not-found", "The given view (ID $viewId) was not found in the view manager.")
 
-class UnknownCameraError(cause: Throwable) : CameraError("unknown", "unknown", cause.message ?: "An unknown camera error occured.", cause)
+class UnknownCameraError(cause: Throwable?) : CameraError("unknown", "unknown", cause?.message ?: "An unknown camera error occured.", cause)