railbird/react-native-vision-camera
8
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Use new CameraX Recorder API (Upgrade CameraX to alpha12/32) (#543)

Browse Source 
* chore(deps): Upgrade CameraX to alpha10/30

* chore(deps): Add first stable CameraX Video library

* feat: Use new CameraX Video API

* chore(deps): Upgrade CameraX from 10 -> 12

* fix: Replace deprecated APIs

* Update CameraViewModule.kt

* fix: Fix file creation
This commit is contained in:
Marc Rousavy 2021-12-30 11:39:17 +01:00 committed by GitHub
parent b72176fae9
commit 68707322fe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 136 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
android/build.gradle
View File

@ -233,11 +233,14 @@ dependencies {
implementation "org.jetbrains.kotlinx:kotlinx-coroutines-guava:1.5.2"
implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:1.5.2"
implementation "androidx.camera:camera-core:1.1.0-alpha09"
implementation "androidx.camera:camera-camera2:1.1.0-alpha09"
implementation "androidx.camera:camera-lifecycle:1.1.0-alpha09"
implementation "androidx.camera:camera-extensions:1.0.0-alpha29"
implementation "androidx.camera:camera-view:1.0.0-alpha29"
implementation "androidx.camera:camera-core:1.1.0-alpha12"
implementation "androidx.camera:camera-camera2:1.1.0-alpha12"
implementation "androidx.camera:camera-lifecycle:1.1.0-alpha12"
implementation "androidx.camera:camera-video:1.1.0-alpha12"
implementation "androidx.camera:camera-view:1.0.0-alpha32"
implementation "androidx.camera:camera-extensions:1.0.0-alpha32"
implementation "androidx.exifinterface:exifinterface:1.3.3"
}

81
android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt
View File

@ -3,17 +3,19 @@ package com.mrousavy.camera
import android.Manifest
import android.annotation.SuppressLint
import android.content.pm.PackageManager
import androidx.camera.core.VideoCapture
import androidx.camera.video.FileOutputOptions
import androidx.camera.video.VideoRecordEvent
import androidx.core.content.ContextCompat
import androidx.core.util.Consumer
import com.facebook.react.bridge.*
import com.mrousavy.camera.utils.makeErrorMap
import kotlinx.coroutines.*
import java.io.File
import java.text.SimpleDateFormat
import java.util.*
data class TemporaryFile(val path: String)
@SuppressLint("RestrictedApi", "MissingPermission")
suspend fun CameraView.startRecording(options: ReadableMap, onRecordCallback: Callback): TemporaryFile {
fun CameraView.startRecording(options: ReadableMap, onRecordCallback: Callback) {
if (videoCapture == null) {
if (video == true) {
throw CameraNotReadyError()
@ -35,44 +37,49 @@ suspend fun CameraView.startRecording(options: ReadableMap, onRecordCallback: Ca
camera!!.cameraControl.enableTorch(enableFlash)
}
@Suppress("BlockingMethodInNonBlockingContext") // in withContext we are not blocking. False positive.
val videoFile = withContext(Dispatchers.IO) {
File.createTempFile("video", ".mp4", context.cacheDir).apply { deleteOnExit() }
val id = SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(Date())
val file = File.createTempFile("VisionCamera-${id}", ".mp4")
val fileOptions = FileOutputOptions.Builder(file).build()
var recording = videoCapture!!
.prepareRecording(context, fileOptions)
if (audio == true) {
@SuppressLint("MissingPermission")
recording = recording.withAudioEnabled()
}
val videoFileOptions = VideoCapture.OutputFileOptions.Builder(videoFile)
videoCapture!!.startRecording(
videoFileOptions.build(), recordVideoExecutor,
object : VideoCapture.OnVideoSavedCallback {
override fun onVideoSaved(outputFileResults: VideoCapture.OutputFileResults) {
val map = Arguments.createMap()
map.putString("path", videoFile.absolutePath)
// TODO: duration and size - see https://github.com/mrousavy/react-native-vision-camera/issues/77
onRecordCallback(map, null)
// reset the torch mode
camera!!.cameraControl.enableTorch(torch == "on")
}
override fun onError(videoCaptureError: Int, message: String, cause: Throwable?) {
val error = when (videoCaptureError) {
VideoCapture.ERROR_ENCODER -> VideoEncoderError(message, cause)
VideoCapture.ERROR_FILE_IO -> FileIOError(message, cause)
VideoCapture.ERROR_INVALID_CAMERA -> InvalidCameraError(message, cause)
VideoCapture.ERROR_MUXER -> VideoMuxerError(message, cause)
VideoCapture.ERROR_RECORDING_IN_PROGRESS -> RecordingInProgressError(message, cause)
else -> UnknownCameraError(Error(message, cause))
activeVideoRecording = recording.start(ContextCompat.getMainExecutor(context), object : Consumer<VideoRecordEvent> {
override fun accept(event: VideoRecordEvent?) {
if (event is VideoRecordEvent.Finalize) {
if (event.hasError()) {
// error occured!
val error = when (event.error) {
VideoRecordEvent.Finalize.ERROR_ENCODING_FAILED -> VideoEncoderError(event.cause)
VideoRecordEvent.Finalize.ERROR_FILE_SIZE_LIMIT_REACHED -> FileSizeLimitReachedError(event.cause)
VideoRecordEvent.Finalize.ERROR_INSUFFICIENT_STORAGE -> InsufficientStorageError(event.cause)
VideoRecordEvent.Finalize.ERROR_INVALID_OUTPUT_OPTIONS -> InvalidVideoOutputOptionsError(event.cause)
VideoRecordEvent.Finalize.ERROR_NO_VALID_DATA -> NoValidDataError(event.cause)
VideoRecordEvent.Finalize.ERROR_RECORDER_ERROR -> RecorderError(event.cause)
VideoRecordEvent.Finalize.ERROR_SOURCE_INACTIVE -> InactiveSourceError(event.cause)
else -> UnknownCameraError(event.cause)
}
val map = makeErrorMap("${error.domain}/${error.id}", error.message, error)
onRecordCallback(null, map)
} else {
// recording saved successfully!
val map = Arguments.createMap()
map.putString("path", event.outputResults.outputUri.toString())
map.putDouble("duration", /* seconds */ event.recordingStats.recordedDurationNanos.toDouble() / 1000000.0 / 1000.0)
map.putDouble("size", /* kB */ event.recordingStats.numBytesRecorded.toDouble() / 1000.0)
onRecordCallback(map, null)
}
val map = makeErrorMap("${error.domain}/${error.id}", error.message, error)
onRecordCallback(null, map)
// reset the torch mode
camera!!.cameraControl.enableTorch(torch == "on")
}
}
)
return TemporaryFile(videoFile.absolutePath)
})
}
@SuppressLint("RestrictedApi")
@ -80,8 +87,12 @@ fun CameraView.stopRecording() {
if (videoCapture == null) {
throw CameraNotReadyError()
}
if (activeVideoRecording == null) {
throw NoRecordingInProgressError()
}
activeVideoRecording!!.stop()
videoCapture!!.stopRecording()
// reset torch mode to original value
camera!!.cameraControl.enableTorch(torch == "on")
}

44
android/src/main/java/com/mrousavy/camera/CameraView.kt
View File

@ -16,6 +16,8 @@ import androidx.camera.core.*
import androidx.camera.core.impl.*
import androidx.camera.extensions.*
import androidx.camera.lifecycle.ProcessCameraProvider
import androidx.camera.video.*
import androidx.camera.video.VideoCapture
import androidx.camera.view.PreviewView
import androidx.core.content.ContextCompat
import androidx.lifecycle.*
@ -121,10 +123,12 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
internal var camera: Camera? = null
internal var imageCapture: ImageCapture? = null
internal var videoCapture: VideoCapture? = null
internal var videoCapture: Recorder? = null
private var imageAnalysis: ImageAnalysis? = null
private var preview: Preview? = null
internal var activeVideoRecording: Recording? = null
private var lastFrameProcessorCall = System.currentTimeMillis()
private var extensionsManager: ExtensionsManager? = null
@ -234,7 +238,7 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
preview?.targetRotation = rotation
imageCapture?.targetRotation = rotation
imageAnalysis?.targetRotation = rotation
videoCapture?.setTargetRotation(rotation)
// TODO: videoCapture?.setTargetRotation(rotation)
}
}
@ -338,11 +342,11 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
val tryEnableExtension: (suspend (extension: Int) -> Unit) = lambda@ { extension ->
if (extensionsManager == null) {
Log.i(TAG, "Initializing ExtensionsManager...")
extensionsManager = ExtensionsManager.getInstance(context).await()
extensionsManager = ExtensionsManager.getInstanceAsync(context, cameraProvider).await()
}
if (extensionsManager!!.isExtensionAvailable(cameraProvider, cameraSelector, extension)) {
if (extensionsManager!!.isExtensionAvailable(cameraSelector, extension)) {
Log.i(TAG, "Enabling extension $extension...")
cameraSelector = extensionsManager!!.getExtensionEnabledCameraSelector(cameraProvider, cameraSelector, extension)
cameraSelector = extensionsManager!!.getExtensionEnabledCameraSelector(cameraSelector, extension)
} else {
Log.e(TAG, "Extension $extension is not available for the given Camera!")
throw when (extension) {
@ -355,11 +359,14 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
val previewBuilder = Preview.Builder()
.setTargetRotation(rotation)
val imageCaptureBuilder = ImageCapture.Builder()
.setTargetRotation(rotation)
.setCaptureMode(ImageCapture.CAPTURE_MODE_MINIMIZE_LATENCY)
val videoCaptureBuilder = VideoCapture.Builder()
.setTargetRotation(rotation)
val videoRecorderBuilder = Recorder.Builder()
.setExecutor(cameraExecutor)
val imageAnalysisBuilder = ImageAnalysis.Builder()
.setTargetRotation(rotation)
.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST)
@ -371,7 +378,7 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
val aspectRatio = aspectRatio(previewView.height, previewView.width) // flipped because it's in sensor orientation.
previewBuilder.setTargetAspectRatio(aspectRatio)
imageCaptureBuilder.setTargetAspectRatio(aspectRatio)
videoCaptureBuilder.setTargetAspectRatio(aspectRatio)
// TODO: Aspect Ratio for Video Recorder?
imageAnalysisBuilder.setTargetAspectRatio(aspectRatio)
} else {
// User has selected a custom format={}. Use that
@ -379,9 +386,16 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
Log.i(TAG, "Using custom format - photo: ${format.photoSize}, video: ${format.videoSize} @ $fps FPS")
previewBuilder.setTargetResolution(format.videoSize)
imageCaptureBuilder.setTargetResolution(format.photoSize)
videoCaptureBuilder.setTargetResolution(format.videoSize)
imageAnalysisBuilder.setTargetResolution(format.videoSize)
// TODO: Ability to select resolution exactly depending on format? Just like on iOS...
when (min(format.videoSize.height, format.videoSize.width)) {
in 0..480 -> videoRecorderBuilder.setQualitySelector(QualitySelector.from(Quality.SD))
in 480..720 -> videoRecorderBuilder.setQualitySelector(QualitySelector.from(Quality.HD))
in 720..1080 -> videoRecorderBuilder.setQualitySelector(QualitySelector.from(Quality.FHD))
in 1080..2160 -> videoRecorderBuilder.setQualitySelector(QualitySelector.from(Quality.UHD))
}
fps?.let { fps ->
if (format.frameRateRanges.any { it.contains(fps) }) {
// Camera supports the given FPS (frame rate range)
@ -391,7 +405,7 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
Camera2Interop.Extender(previewBuilder)
.setCaptureRequestOption(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(fps, fps))
.setCaptureRequestOption(CaptureRequest.SENSOR_FRAME_DURATION, frameDuration)
videoCaptureBuilder.setVideoFrameRate(fps)
// TODO: Frame Rate/FPS for Video Recorder?
} else {
throw FpsNotContainedInFormatError(fps)
}
@ -404,8 +418,12 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
}
}
val videoRecorder = videoRecorderBuilder.build()
val videoCapture = VideoCapture.withOutput(videoRecorder)
videoCapture.targetRotation = rotation
// Unbind use cases before rebinding
videoCapture = null
this.videoCapture = null
imageCapture = null
imageAnalysis = null
cameraProvider.unbindAll()
@ -414,8 +432,8 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
val useCases = ArrayList<UseCase>()
if (video == true) {
Log.i(TAG, "Adding VideoCapture use-case...")
videoCapture = videoCaptureBuilder.build()
useCases.add(videoCapture!!)
this.videoCapture = videoRecorder
useCases.add(videoCapture)
}
if (photo == true) {
if (fallbackToSnapshot) {

7
android/src/main/java/com/mrousavy/camera/CameraViewModule.kt
View File

@ -124,8 +124,8 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase
val startTime = System.currentTimeMillis()
coroutineScope.launch {
withPromise(promise) {
val extensionsManager = ExtensionsManager.getInstance(reactApplicationContext).await()
val cameraProvider = ProcessCameraProvider.getInstance(reactApplicationContext).await()
val extensionsManager = ExtensionsManager.getInstanceAsync(reactApplicationContext, cameraProvider).await()
ProcessCameraProvider.getInstance(reactApplicationContext).await()
val manager = reactApplicationContext.getSystemService(Context.CAMERA_SERVICE) as? CameraManager
@ -162,8 +162,8 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase
else null
val fpsRanges = characteristics.get(CameraCharacteristics.CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES)!!
val supportsHdr = extensionsManager.isExtensionAvailable(cameraProvider, cameraSelector, ExtensionMode.HDR)
val supportsLowLightBoost = extensionsManager.isExtensionAvailable(cameraProvider, cameraSelector, ExtensionMode.NIGHT)
val supportsHdr = extensionsManager.isExtensionAvailable(cameraSelector, ExtensionMode.HDR)
val supportsLowLightBoost = extensionsManager.isExtensionAvailable(cameraSelector, ExtensionMode.NIGHT)
// see https://developer.android.com/reference/android/hardware/camera2/CameraDevice#regular-capture
val supportsParallelVideoProcessing = hardwareLevel != CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY && hardwareLevel != CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LIMITED
@ -198,6 +198,7 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase
val formats = Arguments.createArray()
// TODO: Get supported video qualities with QualitySelector.getSupportedQualities(...)
cameraConfig.outputFormats.forEach { formatId ->
val formatName = parseImageFormat(formatId)

53
android/src/main/java/com/mrousavy/camera/Errors.kt
View File

@ -1,6 +1,7 @@
package com.mrousavy.camera
import android.graphics.ImageFormat
import androidx.camera.video.VideoRecordEvent.Finalize.VideoRecordError
abstract class CameraError(
/**
@ -59,13 +60,53 @@ class VideoNotEnabledError : CameraError("capture", "video-not-enabled", "Video
class PhotoNotEnabledError : CameraError("capture", "photo-not-enabled", "Photo capture is disabled! Pass `photo={true}` to enable photo capture.")
class InvalidFormatError(format: Int) : CameraError("capture", "invalid-photo-format", "The Photo has an invalid format! Expected ${ImageFormat.YUV_420_888}, actual: $format")
class VideoEncoderError(message: String, cause: Throwable? = null) : CameraError("capture", "encoder-error", message, cause)
class VideoMuxerError(message: String, cause: Throwable? = null) : CameraError("capture", "muxer-error", message, cause)
class RecordingInProgressError(message: String, cause: Throwable? = null) : CameraError("capture", "recording-in-progress", message, cause)
class FileIOError(message: String, cause: Throwable? = null) : CameraError("capture", "file-io-error", message, cause)
class InvalidCameraError(message: String, cause: Throwable? = null) : CameraError("capture", "not-bound-error", message, cause)
class VideoEncoderError(cause: Throwable?) : CameraError("capture", "encoder-error", "The recording failed while encoding.\n" +
"This error may be generated when the video or audio codec encounters an error during encoding. " +
"When this happens and the output file is generated, the output file is not properly constructed. " +
"The application will need to clean up the output file, such as deleting the file.",
cause)
class InvalidVideoOutputOptionsError(cause: Throwable?) : CameraError("capture", "invalid-video-options",
"The recording failed due to invalid output options.\n" +
"This error is generated when invalid output options have been used while preparing a recording",
cause)
class RecorderError(cause: Throwable?) : CameraError("capture", "recorder-error",
"The recording failed because the Recorder is in an unrecoverable error state.\n" +
"When this happens and the output file is generated, the output file is not properly constructed. " +
"The application will need to clean up the output file, such as deleting the file. " +
"Such an error will usually require creating a new Recorder object to start a new recording.",
cause)
class NoValidDataError(cause: Throwable?) : CameraError("capture", "no-valid-data",
"The recording failed because no valid data was produced to be recorded.\n" +
"This error is generated when the essential data for a recording to be played correctly is missing, for example, " +
"a recording must contain at least one key frame. The application will need to clean up the output file, such as deleting the file.",
cause)
class InactiveSourceError(cause: Throwable?) : CameraError("capture", "inactive-source",
"The recording failed because the source becomes inactive and stops sending frames.\n" +
"One case is that if camera is closed due to lifecycle stopped, the active recording will be finalized with this error, " +
"and the output will be generated, containing the frames produced before camera closing. " +
"Attempting to start a new recording will be finalized immediately if the source remains inactive and no output will be generated.",
cause)
class InsufficientStorageError(cause: Throwable?) : CameraError("capture", "insufficient-storage",
"The recording failed due to insufficient storage space.\n" +
"There are two possible cases that will cause this error.\n" +
"1. The storage is already full before the recording starts, so no output file will be generated.\n" +
"2. The storage becomes full during recording, so the output file will be generated.",
cause)
class FileSizeLimitReachedError(cause: Throwable?) : CameraError("capture", "file-size-limit-reached",
"The recording failed due to file size limitation.\n" +
"The file size limitation will refer to OutputOptions.getFileSizeLimit(). The output file will still be generated with this error.",
cause)
class NoRecordingInProgressError : CameraError("capture", "no-recording-in-progress", "No active recording in progress!")
class CameraManagerUnavailableError : CameraError("system", "no-camera-manager", "The Camera manager instance was unavailable for the current Application!")
class ViewNotFoundError(viewId: Int) : CameraError("system", "view-not-found", "The given view (ID $viewId) was not found in the view manager.")
class UnknownCameraError(cause: Throwable) : CameraError("unknown", "unknown", cause.message ?: "An unknown camera error occured.", cause)
class UnknownCameraError(cause: Throwable?) : CameraError("unknown", "unknown", cause?.message ?: "An unknown camera error occured.", cause)