fix: Use bitRate multiplier instead of setting it to an absolute value (#2216)

* fix: Use `bitRate` multiplier instead of setting it to an absolute value * Pass override * Format * Rename * feat: Also implement Android * fix: Log Mbps properly * fix: Up-/Down-scale bit-rate if different options * fix: Parse in Manager * Update RecordingSession+getRecommendedBitRate.kt
2023-11-27 17:20:26 +01:00
parent d78798ff84
commit d7f7095d1a
10 changed files with 231 additions and 84 deletions
--- a/package/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt
+++ b/package/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt
@@ -9,13 +9,13 @@ import com.mrousavy.camera.core.MicrophonePermissionError
 import com.mrousavy.camera.core.RecorderError
 import com.mrousavy.camera.core.RecordingSession
 import com.mrousavy.camera.core.code
+import com.mrousavy.camera.types.Flash
+import com.mrousavy.camera.types.RecordVideoOptions
 import com.mrousavy.camera.types.Torch
-import com.mrousavy.camera.types.VideoCodec
-import com.mrousavy.camera.types.VideoFileType
 import com.mrousavy.camera.utils.makeErrorMap
 import java.util.*

-suspend fun CameraView.startRecording(options: ReadableMap, onRecordCallback: Callback) {
+suspend fun CameraView.startRecording(options: RecordVideoOptions, onRecordCallback: Callback) {
  // check audio permission
  if (audio == true) {
    if (ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) {
@@ -23,25 +23,13 @@ suspend fun CameraView.startRecording(options: ReadableMap, onRecordCallback: Ca
    }
  }

-  val enableFlash = options.getString("flash") == "on"
+  val enableFlash = options.flash == Flash.ON
  if (enableFlash) {
    // overrides current torch mode value to enable flash while recording
    cameraSession.configure { config ->
      config.torch = Torch.ON
    }
  }
-  var codec = VideoCodec.H264
-  if (options.hasKey("videoCodec")) {
-    codec = VideoCodec.fromUnionValue(options.getString("videoCodec"))
-  }
-  var fileType = VideoFileType.MP4
-  if (options.hasKey("fileType")) {
-    fileType = VideoFileType.fromUnionValue(options.getString("fileType"))
-  }
-  var bitRate: Double? = null
-  if (options.hasKey("videoBitRate")) {
-    bitRate = options.getDouble("videoBitRate")
-  }

  val callback = { video: RecordingSession.Video ->
    val map = Arguments.createMap()
@@ -53,7 +41,7 @@ suspend fun CameraView.startRecording(options: ReadableMap, onRecordCallback: Ca
    val errorMap = makeErrorMap(error.code, error.message)
    onRecordCallback(null, errorMap)
  }
-  cameraSession.startRecording(audio == true, codec, fileType, bitRate, callback, onError)
+  cameraSession.startRecording(audio == true, options, callback, onError)
 }

@SuppressLint("RestrictedApi")
--- a/package/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt
+++ b/package/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt
@@ -83,10 +83,11 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase

  // TODO: startRecording() cannot be awaited, because I can't have a Promise and a onRecordedCallback in the same function. Hopefully TurboModules allows that
  @ReactMethod
-  fun startRecording(viewTag: Int, options: ReadableMap, onRecordCallback: Callback) {
+  fun startRecording(viewTag: Int, jsOptions: ReadableMap, onRecordCallback: Callback) {
    coroutineScope.launch {
      val view = findCameraView(viewTag)
      try {
+        val options = RecordVideoOptions(jsOptions)
        view.startRecording(options, onRecordCallback)
      } catch (error: CameraError) {
        val map = makeErrorMap("${error.domain}/${error.id}", error.message, error)
--- a/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt
+++ b/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt
@@ -41,9 +41,8 @@ import com.mrousavy.camera.frameprocessor.FrameProcessor
 import com.mrousavy.camera.types.Flash
 import com.mrousavy.camera.types.Orientation
 import com.mrousavy.camera.types.QualityPrioritization
+import com.mrousavy.camera.types.RecordVideoOptions
 import com.mrousavy.camera.types.Torch
-import com.mrousavy.camera.types.VideoCodec
-import com.mrousavy.camera.types.VideoFileType
 import com.mrousavy.camera.types.VideoStabilizationMode
 import java.io.Closeable
 import java.util.concurrent.CancellationException
@@ -516,20 +515,21 @@ class CameraSession(private val context: Context, private val cameraManager: Cam

  suspend fun startRecording(
    enableAudio: Boolean,
-    codec: VideoCodec,
-    fileType: VideoFileType,
-    bitRate: Double?,
+    options: RecordVideoOptions,
    callback: (video: RecordingSession.Video) -> Unit,
    onError: (error: RecorderError) -> Unit
  ) {
    mutex.withLock {
      if (recording != null) throw RecordingInProgressError()
      val videoOutput = videoOutput ?: throw VideoNotEnabledError()
+      val cameraDevice = cameraDevice ?: throw CameraNotReadyError()

+      // TODO: Implement HDR
+      val hdr = configuration?.videoHdr ?: false
      val fps = configuration?.fps ?: 30

      val recording =
-        RecordingSession(context, videoOutput.size, enableAudio, fps, codec, orientation, fileType, bitRate, callback, onError)
+        RecordingSession(context, cameraDevice.id, videoOutput.size, enableAudio, fps, hdr, orientation, options, callback, onError)
      recording.start()
      this.recording = recording
    }
--- a/package/android/src/main/java/com/mrousavy/camera/core/RecordingSession.kt
+++ b/package/android/src/main/java/com/mrousavy/camera/core/RecordingSession.kt
@@ -7,20 +7,20 @@ import android.os.Build
 import android.util.Log
 import android.util.Size
 import android.view.Surface
+import com.mrousavy.camera.extensions.getRecommendedBitRate
 import com.mrousavy.camera.types.Orientation
-import com.mrousavy.camera.types.VideoCodec
-import com.mrousavy.camera.types.VideoFileType
+import com.mrousavy.camera.types.RecordVideoOptions
 import java.io.File

 class RecordingSession(
  context: Context,
+  val cameraId: String,
  val size: Size,
  private val enableAudio: Boolean,
  private val fps: Int? = null,
-  private val codec: VideoCodec = VideoCodec.H264,
+  private val hdr: Boolean = false,
  private val orientation: Orientation,
-  private val fileType: VideoFileType = VideoFileType.MP4,
-  videoBitRate: Double? = null,
+  private val options: RecordVideoOptions,
  private val callback: (video: Video) -> Unit,
  private val onError: (error: RecorderError) -> Unit
 ) {
@@ -34,14 +34,14 @@ class RecordingSession(

  data class Video(val path: String, val durationMs: Long)

-  private val bitRate = videoBitRate ?: getDefaultBitRate()
+  private val bitRate = getBitRate()
  private val recorder: MediaRecorder
  private val outputFile: File
  private var startTime: Long? = null
  val surface: Surface = MediaCodec.createPersistentInputSurface()

  init {
-    outputFile = File.createTempFile("mrousavy", fileType.toExtension(), context.cacheDir)
+    outputFile = File.createTempFile("mrousavy", options.fileType.toExtension(), context.cacheDir)

    Log.i(TAG, "Creating RecordingSession for ${outputFile.absolutePath}")

@@ -52,12 +52,12 @@ class RecordingSession(

    recorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4)
    recorder.setOutputFile(outputFile.absolutePath)
-    recorder.setVideoEncodingBitRate((bitRate * 1_000_000).toInt())
+    recorder.setVideoEncodingBitRate(bitRate)
    recorder.setVideoSize(size.height, size.width)
    if (fps != null) recorder.setVideoFrameRate(fps)

-    Log.i(TAG, "Using $codec Video Codec at $bitRate Mbps..")
-    recorder.setVideoEncoder(codec.toVideoCodec())
+    Log.i(TAG, "Using ${options.videoCodec} Video Codec at ${bitRate / 1_000_000.0} Mbps..")
+    recorder.setVideoEncoder(options.videoCodec.toVideoEncoder())
    if (enableAudio) {
      Log.i(TAG, "Adding Audio Channel..")
      recorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC)
@@ -124,22 +124,26 @@ class RecordingSession(
    }
  }

-  private fun getDefaultBitRate(): Double {
-    var baseBitRate = when (size.width * size.height) {
-      in 0..640 * 480 -> 2.0
-      in 640 * 480..1280 * 720 -> 5.0
-      in 1280 * 720..1920 * 1080 -> 10.0
-      in 1920 * 1080..3840 * 2160 -> 30.0
-      in 3840 * 2160..7680 * 4320 -> 100.0
-      else -> 100.0
+  /**
+   * Get the bit-rate to use, in bits per seconds.
+   * This can either be overridden, multiplied, or just left at the recommended value.
+   */
+  private fun getBitRate(): Int {
+    var bitRate = getRecommendedBitRate(fps ?: 30, options.videoCodec, hdr)
+    options.videoBitRateOverride?.let { override ->
+      // Mbps -> bps
+      bitRate = (override * 1_000_000).toInt()
    }
-    baseBitRate = baseBitRate / 30.0 * (fps ?: 30).toDouble()
-    if (this.codec == VideoCodec.H265) baseBitRate *= 0.8
-    return baseBitRate
+    options.videoBitRateMultiplier?.let { multiplier ->
+      // multiply by 1.2, 0.8, ...
+      bitRate = (bitRate * multiplier).toInt()
+    }
+    return bitRate
  }

  override fun toString(): String {
    val audio = if (enableAudio) "with audio" else "without audio"
-    return "${size.width} x ${size.height} @ $fps FPS $codec $fileType $orientation $bitRate Mbps RecordingSession ($audio)"
+    return "${size.width} x ${size.height} @ $fps FPS ${options.videoCodec} ${options.fileType} " +
+      "$orientation ${bitRate / 1_000_000.0} Mbps RecordingSession ($audio)"
  }
 }
--- a/package/android/src/main/java/com/mrousavy/camera/extensions/RecordingSession+getRecommendedBitRate.kt
+++ b/package/android/src/main/java/com/mrousavy/camera/extensions/RecordingSession+getRecommendedBitRate.kt
@@ -0,0 +1,100 @@
+package com.mrousavy.camera.extensions
+
+import android.media.CamcorderProfile
+import android.media.MediaRecorder.VideoEncoder
+import android.os.Build
+import android.util.Log
+import android.util.Size
+import com.mrousavy.camera.core.RecordingSession
+import com.mrousavy.camera.types.VideoCodec
+import kotlin.math.abs
+
+data class RecommendedProfile(
+  val bitRate: Int,
+  // VideoEncoder.H264 or VideoEncoder.HEVC
+  val codec: Int,
+  // 8-bit or 10-bit
+  val bitDepth: Int,
+  // 30 or 60 FPS
+  val fps: Int
+)
+
+fun RecordingSession.getRecommendedBitRate(fps: Int, codec: VideoCodec, hdr: Boolean): Int {
+  val targetResolution = size
+  val encoder = codec.toVideoEncoder()
+  val bitDepth = if (hdr) 10 else 8
+  val quality = findClosestCamcorderProfileQuality(targetResolution)
+  Log.i("CamcorderProfile", "Closest matching CamcorderProfile: $quality")
+
+  var recommendedProfile: RecommendedProfile? = null
+
+  if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
+    val profiles = CamcorderProfile.getAll(cameraId, quality)
+    if (profiles != null) {
+      val best = profiles.videoProfiles.minBy { abs(it.width * it.height - targetResolution.width * targetResolution.height) }
+
+      recommendedProfile = RecommendedProfile(
+        best.bitrate,
+        best.codec,
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) best.bitDepth else 8,
+        best.frameRate
+      )
+    }
+  }
+
+  if (recommendedProfile == null) {
+    val cameraIdInt = cameraId.toIntOrNull()
+    val profile = if (cameraIdInt != null) {
+      CamcorderProfile.get(cameraIdInt, quality)
+    } else {
+      CamcorderProfile.get(quality)
+    }
+    recommendedProfile = RecommendedProfile(
+      profile.videoBitRate,
+      profile.videoCodec,
+      8,
+      profile.videoFrameRate
+    )
+  }
+
+  var bitRate = recommendedProfile.bitRate.toDouble()
+  // the target bit-rate is for e.g. 30 FPS, but we use 60 FPS. up-scale it
+  bitRate = bitRate / recommendedProfile.fps * fps
+  // the target bit-rate might be in 8-bit SDR, but we record in 10-bit HDR. up-scale it
+  bitRate = bitRate / recommendedProfile.bitDepth * bitDepth
+  if (recommendedProfile.codec == VideoEncoder.H264 && encoder == VideoEncoder.HEVC) {
+    // the target bit-rate is for H.264, but we use H.265, which is 20% smaller
+    bitRate *= 0.8
+  } else if (recommendedProfile.codec == VideoEncoder.HEVC && encoder == VideoEncoder.H264) {
+    // the target bit-rate is for H.265, but we use H.264, which is 20% larger
+    bitRate *= 1.2
+  }
+  return bitRate.toInt()
+}
+
+private fun getResolutionForCamcorderProfileQuality(camcorderProfile: Int): Int =
+  when (camcorderProfile) {
+    CamcorderProfile.QUALITY_QCIF -> 176 * 144
+    CamcorderProfile.QUALITY_QVGA -> 320 * 240
+    CamcorderProfile.QUALITY_CIF -> 352 * 288
+    CamcorderProfile.QUALITY_VGA -> 640 * 480
+    CamcorderProfile.QUALITY_480P -> 720 * 480
+    CamcorderProfile.QUALITY_720P -> 1280 * 720
+    CamcorderProfile.QUALITY_1080P -> 1920 * 1080
+    CamcorderProfile.QUALITY_2K -> 2048 * 1080
+    CamcorderProfile.QUALITY_QHD -> 2560 * 1440
+    CamcorderProfile.QUALITY_2160P -> 3840 * 2160
+    CamcorderProfile.QUALITY_4KDCI -> 4096 * 2160
+    CamcorderProfile.QUALITY_8KUHD -> 7680 * 4320
+    else -> throw Error("Invalid CamcorderProfile \"$camcorderProfile\"!")
+  }
+
+private fun findClosestCamcorderProfileQuality(resolution: Size): Int {
+  // Iterate through all available CamcorderProfiles and find the one that matches the closest
+  val targetResolution = resolution.width * resolution.height
+  val closestProfile = (CamcorderProfile.QUALITY_QCIF..CamcorderProfile.QUALITY_8KUHD).minBy { profile ->
+    val currentResolution = getResolutionForCamcorderProfileQuality(profile)
+    return@minBy abs(currentResolution - targetResolution)
+  }
+  return closestProfile
+}
--- a/package/android/src/main/java/com/mrousavy/camera/types/RecordVideoOptions.kt
+++ b/package/android/src/main/java/com/mrousavy/camera/types/RecordVideoOptions.kt
@@ -0,0 +1,29 @@
+package com.mrousavy.camera.types
+
+import com.facebook.react.bridge.ReadableMap
+
+class RecordVideoOptions(map: ReadableMap) {
+  var fileType: VideoFileType = VideoFileType.MOV
+  var flash: Flash = Flash.OFF
+  var videoCodec = VideoCodec.H264
+  var videoBitRateOverride: Double? = null
+  var videoBitRateMultiplier: Double? = null
+
+  init {
+    if (map.hasKey("fileType")) {
+      fileType = VideoFileType.fromUnionValue(map.getString("fileType"))
+    }
+    if (map.hasKey("flash")) {
+      flash = Flash.fromUnionValue(map.getString("flash"))
+    }
+    if (map.hasKey("videoCodec")) {
+      videoCodec = VideoCodec.fromUnionValue(map.getString("fileType"))
+    }
+    if (map.hasKey("videoBitRateOverride")) {
+      videoBitRateOverride = map.getDouble("videoBitRateOverride")
+    }
+    if (map.hasKey("videoBitRateMultiplier")) {
+      videoBitRateMultiplier = map.getDouble("videoBitRateMultiplier")
+    }
+  }
+}
--- a/package/android/src/main/java/com/mrousavy/camera/types/VideoCodec.kt
+++ b/package/android/src/main/java/com/mrousavy/camera/types/VideoCodec.kt
@@ -6,7 +6,7 @@ enum class VideoCodec(override val unionValue: String) : JSUnionValue {
  H264("h264"),
  H265("h265");

-  fun toVideoCodec(): Int =
+  fun toVideoEncoder(): Int =
    when (this) {
      H264 -> MediaRecorder.VideoEncoder.H264
      H265 -> MediaRecorder.VideoEncoder.HEVC