railbird/react-native-vision-camera
9
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: Use bitRate multiplier instead of setting it to an absolute value (#2216)

Browse Source 
* fix: Use `bitRate` multiplier instead of setting it to an absolute value

* Pass override

* Format

* Rename

* feat: Also implement Android

* fix: Log Mbps properly

* fix: Up-/Down-scale bit-rate if different options

* fix: Parse in Manager

* Update RecordingSession+getRecommendedBitRate.kt
This commit is contained in:
Marc Rousavy 2023-11-27 17:20:26 +01:00 committed by GitHub
parent d78798ff84
commit d7f7095d1a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 231 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
package/android/src/main/java/com/mrousavy/camera/CameraView+RecordVideo.kt
View File

@ -9,13 +9,13 @@ import com.mrousavy.camera.core.MicrophonePermissionError
import com.mrousavy.camera.core.RecorderError
import com.mrousavy.camera.core.RecordingSession
import com.mrousavy.camera.core.code
import com.mrousavy.camera.types.Flash
import com.mrousavy.camera.types.RecordVideoOptions
import com.mrousavy.camera.types.Torch
import com.mrousavy.camera.types.VideoCodec
import com.mrousavy.camera.types.VideoFileType
import com.mrousavy.camera.utils.makeErrorMap
import java.util.*
suspend fun CameraView.startRecording(options: ReadableMap, onRecordCallback: Callback) {
suspend fun CameraView.startRecording(options: RecordVideoOptions, onRecordCallback: Callback) {
// check audio permission
if (audio == true) {
if (ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) {
@ -23,25 +23,13 @@ suspend fun CameraView.startRecording(options: ReadableMap, onRecordCallback: Ca
}
}
val enableFlash = options.getString("flash") == "on"
val enableFlash = options.flash == Flash.ON
if (enableFlash) {
// overrides current torch mode value to enable flash while recording
cameraSession.configure { config ->
config.torch = Torch.ON
}
}
var codec = VideoCodec.H264
if (options.hasKey("videoCodec")) {
codec = VideoCodec.fromUnionValue(options.getString("videoCodec"))
}
var fileType = VideoFileType.MP4
if (options.hasKey("fileType")) {
fileType = VideoFileType.fromUnionValue(options.getString("fileType"))
}
var bitRate: Double? = null
if (options.hasKey("videoBitRate")) {
bitRate = options.getDouble("videoBitRate")
}
val callback = { video: RecordingSession.Video ->
val map = Arguments.createMap()
@ -53,7 +41,7 @@ suspend fun CameraView.startRecording(options: ReadableMap, onRecordCallback: Ca
val errorMap = makeErrorMap(error.code, error.message)
onRecordCallback(null, errorMap)
}
cameraSession.startRecording(audio == true, codec, fileType, bitRate, callback, onError)
cameraSession.startRecording(audio == true, options, callback, onError)
}
@SuppressLint("RestrictedApi")

3
package/android/src/main/java/com/mrousavy/camera/CameraViewModule.kt
View File

@ -83,10 +83,11 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase
// TODO: startRecording() cannot be awaited, because I can't have a Promise and a onRecordedCallback in the same function. Hopefully TurboModules allows that
@ReactMethod
fun startRecording(viewTag: Int, options: ReadableMap, onRecordCallback: Callback) {
fun startRecording(viewTag: Int, jsOptions: ReadableMap, onRecordCallback: Callback) {
coroutineScope.launch {
val view = findCameraView(viewTag)
try {
val options = RecordVideoOptions(jsOptions)
view.startRecording(options, onRecordCallback)
} catch (error: CameraError) {
val map = makeErrorMap("${error.domain}/${error.id}", error.message, error)

12
package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt
View File

@ -41,9 +41,8 @@ import com.mrousavy.camera.frameprocessor.FrameProcessor
import com.mrousavy.camera.types.Flash
import com.mrousavy.camera.types.Orientation
import com.mrousavy.camera.types.QualityPrioritization
import com.mrousavy.camera.types.RecordVideoOptions
import com.mrousavy.camera.types.Torch
import com.mrousavy.camera.types.VideoCodec
import com.mrousavy.camera.types.VideoFileType
import com.mrousavy.camera.types.VideoStabilizationMode
import java.io.Closeable
import java.util.concurrent.CancellationException
@ -516,20 +515,21 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
suspend fun startRecording(
enableAudio: Boolean,
codec: VideoCodec,
fileType: VideoFileType,
bitRate: Double?,
options: RecordVideoOptions,
callback: (video: RecordingSession.Video) -> Unit,
onError: (error: RecorderError) -> Unit
) {
mutex.withLock {
if (recording != null) throw RecordingInProgressError()
val videoOutput = videoOutput ?: throw VideoNotEnabledError()
val cameraDevice = cameraDevice ?: throw CameraNotReadyError()
// TODO: Implement HDR
val hdr = configuration?.videoHdr ?: false
val fps = configuration?.fps ?: 30
val recording =
RecordingSession(context, videoOutput.size, enableAudio, fps, codec, orientation, fileType, bitRate, callback, onError)
RecordingSession(context, cameraDevice.id, videoOutput.size, enableAudio, fps, hdr, orientation, options, callback, onError)
recording.start()
this.recording = recording
}

48
package/android/src/main/java/com/mrousavy/camera/core/RecordingSession.kt
View File

@ -7,20 +7,20 @@ import android.os.Build
import android.util.Log
import android.util.Size
import android.view.Surface
import com.mrousavy.camera.extensions.getRecommendedBitRate
import com.mrousavy.camera.types.Orientation
import com.mrousavy.camera.types.VideoCodec
import com.mrousavy.camera.types.VideoFileType
import com.mrousavy.camera.types.RecordVideoOptions
import java.io.File
class RecordingSession(
context: Context,
val cameraId: String,
val size: Size,
private val enableAudio: Boolean,
private val fps: Int? = null,
private val codec: VideoCodec = VideoCodec.H264,
private val hdr: Boolean = false,
private val orientation: Orientation,
private val fileType: VideoFileType = VideoFileType.MP4,
videoBitRate: Double? = null,
private val options: RecordVideoOptions,
private val callback: (video: Video) -> Unit,
private val onError: (error: RecorderError) -> Unit
) {
@ -34,14 +34,14 @@ class RecordingSession(
data class Video(val path: String, val durationMs: Long)
private val bitRate = videoBitRate ?: getDefaultBitRate()
private val bitRate = getBitRate()
private val recorder: MediaRecorder
private val outputFile: File
private var startTime: Long? = null
val surface: Surface = MediaCodec.createPersistentInputSurface()
init {
outputFile = File.createTempFile("mrousavy", fileType.toExtension(), context.cacheDir)
outputFile = File.createTempFile("mrousavy", options.fileType.toExtension(), context.cacheDir)
Log.i(TAG, "Creating RecordingSession for ${outputFile.absolutePath}")
@ -52,12 +52,12 @@ class RecordingSession(
recorder.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4)
recorder.setOutputFile(outputFile.absolutePath)
recorder.setVideoEncodingBitRate((bitRate * 1_000_000).toInt())
recorder.setVideoEncodingBitRate(bitRate)
recorder.setVideoSize(size.height, size.width)
if (fps != null) recorder.setVideoFrameRate(fps)
Log.i(TAG, "Using $codec Video Codec at $bitRate Mbps..")
recorder.setVideoEncoder(codec.toVideoCodec())
Log.i(TAG, "Using ${options.videoCodec} Video Codec at ${bitRate / 1_000_000.0} Mbps..")
recorder.setVideoEncoder(options.videoCodec.toVideoEncoder())
if (enableAudio) {
Log.i(TAG, "Adding Audio Channel..")
recorder.setAudioEncoder(MediaRecorder.AudioEncoder.AAC)
@ -124,22 +124,26 @@ class RecordingSession(
}
}
private fun getDefaultBitRate(): Double {
var baseBitRate = when (size.width * size.height) {
in 0..640 * 480 -> 2.0
in 640 * 480..1280 * 720 -> 5.0
in 1280 * 720..1920 * 1080 -> 10.0
in 1920 * 1080..3840 * 2160 -> 30.0
in 3840 * 2160..7680 * 4320 -> 100.0
else -> 100.0
/**
* Get the bit-rate to use, in bits per seconds.
* This can either be overridden, multiplied, or just left at the recommended value.
*/
private fun getBitRate(): Int {
var bitRate = getRecommendedBitRate(fps ?: 30, options.videoCodec, hdr)
options.videoBitRateOverride?.let { override ->
// Mbps -> bps
bitRate = (override * 1_000_000).toInt()
}
baseBitRate = baseBitRate / 30.0 * (fps ?: 30).toDouble()
if (this.codec == VideoCodec.H265) baseBitRate *= 0.8
return baseBitRate
options.videoBitRateMultiplier?.let { multiplier ->
// multiply by 1.2, 0.8, ...
bitRate = (bitRate * multiplier).toInt()
}
return bitRate
}
override fun toString(): String {
val audio = if (enableAudio) "with audio" else "without audio"
return "${size.width} x ${size.height} @ $fps FPS $codec $fileType $orientation $bitRate Mbps RecordingSession ($audio)"
return "${size.width} x ${size.height} @ $fps FPS ${options.videoCodec} ${options.fileType} " +
"$orientation ${bitRate / 1_000_000.0} Mbps RecordingSession ($audio)"
}
}

100
package/android/src/main/java/com/mrousavy/camera/extensions/RecordingSession+getRecommendedBitRate.kt Normal file
View File

@ -0,0 +1,100 @@
package com.mrousavy.camera.extensions
import android.media.CamcorderProfile
import android.media.MediaRecorder.VideoEncoder
import android.os.Build
import android.util.Log
import android.util.Size
import com.mrousavy.camera.core.RecordingSession
import com.mrousavy.camera.types.VideoCodec
import kotlin.math.abs
data class RecommendedProfile(
val bitRate: Int,
// VideoEncoder.H264 or VideoEncoder.HEVC
val codec: Int,
// 8-bit or 10-bit
val bitDepth: Int,
// 30 or 60 FPS
val fps: Int
)
fun RecordingSession.getRecommendedBitRate(fps: Int, codec: VideoCodec, hdr: Boolean): Int {
val targetResolution = size
val encoder = codec.toVideoEncoder()
val bitDepth = if (hdr) 10 else 8
val quality = findClosestCamcorderProfileQuality(targetResolution)
Log.i("CamcorderProfile", "Closest matching CamcorderProfile: $quality")
var recommendedProfile: RecommendedProfile? = null
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
val profiles = CamcorderProfile.getAll(cameraId, quality)
if (profiles != null) {
val best = profiles.videoProfiles.minBy { abs(it.width * it.height - targetResolution.width * targetResolution.height) }
recommendedProfile = RecommendedProfile(
best.bitrate,
best.codec,
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) best.bitDepth else 8,
best.frameRate
)
}
}
if (recommendedProfile == null) {
val cameraIdInt = cameraId.toIntOrNull()
val profile = if (cameraIdInt != null) {
CamcorderProfile.get(cameraIdInt, quality)
} else {
CamcorderProfile.get(quality)
}
recommendedProfile = RecommendedProfile(
profile.videoBitRate,
profile.videoCodec,
8,
profile.videoFrameRate
)
}
var bitRate = recommendedProfile.bitRate.toDouble()
// the target bit-rate is for e.g. 30 FPS, but we use 60 FPS. up-scale it
bitRate = bitRate / recommendedProfile.fps * fps
// the target bit-rate might be in 8-bit SDR, but we record in 10-bit HDR. up-scale it
bitRate = bitRate / recommendedProfile.bitDepth * bitDepth
if (recommendedProfile.codec == VideoEncoder.H264 && encoder == VideoEncoder.HEVC) {
// the target bit-rate is for H.264, but we use H.265, which is 20% smaller
bitRate *= 0.8
} else if (recommendedProfile.codec == VideoEncoder.HEVC && encoder == VideoEncoder.H264) {
// the target bit-rate is for H.265, but we use H.264, which is 20% larger
bitRate *= 1.2
}
return bitRate.toInt()
}
private fun getResolutionForCamcorderProfileQuality(camcorderProfile: Int): Int =
when (camcorderProfile) {
CamcorderProfile.QUALITY_QCIF -> 176 * 144
CamcorderProfile.QUALITY_QVGA -> 320 * 240
CamcorderProfile.QUALITY_CIF -> 352 * 288
CamcorderProfile.QUALITY_VGA -> 640 * 480
CamcorderProfile.QUALITY_480P -> 720 * 480
CamcorderProfile.QUALITY_720P -> 1280 * 720
CamcorderProfile.QUALITY_1080P -> 1920 * 1080
CamcorderProfile.QUALITY_2K -> 2048 * 1080
CamcorderProfile.QUALITY_QHD -> 2560 * 1440
CamcorderProfile.QUALITY_2160P -> 3840 * 2160
CamcorderProfile.QUALITY_4KDCI -> 4096 * 2160
CamcorderProfile.QUALITY_8KUHD -> 7680 * 4320
else -> throw Error("Invalid CamcorderProfile \"$camcorderProfile\"!")
}
private fun findClosestCamcorderProfileQuality(resolution: Size): Int {
// Iterate through all available CamcorderProfiles and find the one that matches the closest
val targetResolution = resolution.width * resolution.height
val closestProfile = (CamcorderProfile.QUALITY_QCIF..CamcorderProfile.QUALITY_8KUHD).minBy { profile ->
val currentResolution = getResolutionForCamcorderProfileQuality(profile)
return@minBy abs(currentResolution - targetResolution)
}
return closestProfile
}

29
package/android/src/main/java/com/mrousavy/camera/types/RecordVideoOptions.kt Normal file
View File

@ -0,0 +1,29 @@
package com.mrousavy.camera.types
import com.facebook.react.bridge.ReadableMap
class RecordVideoOptions(map: ReadableMap) {
var fileType: VideoFileType = VideoFileType.MOV
var flash: Flash = Flash.OFF
var videoCodec = VideoCodec.H264
var videoBitRateOverride: Double? = null
var videoBitRateMultiplier: Double? = null
init {
if (map.hasKey("fileType")) {
fileType = VideoFileType.fromUnionValue(map.getString("fileType"))
}
if (map.hasKey("flash")) {
flash = Flash.fromUnionValue(map.getString("flash"))
}
if (map.hasKey("videoCodec")) {
videoCodec = VideoCodec.fromUnionValue(map.getString("fileType"))
}
if (map.hasKey("videoBitRateOverride")) {
videoBitRateOverride = map.getDouble("videoBitRateOverride")
}
if (map.hasKey("videoBitRateMultiplier")) {
videoBitRateMultiplier = map.getDouble("videoBitRateMultiplier")
}
}
}

2
package/android/src/main/java/com/mrousavy/camera/types/VideoCodec.kt
View File

@ -6,7 +6,7 @@ enum class VideoCodec(override val unionValue: String) : JSUnionValue {
H264("h264"),
H265("h265");
fun toVideoCodec(): Int =
fun toVideoEncoder(): Int =
when (this) {
H264 -> MediaRecorder.VideoEncoder.H264
H265 -> MediaRecorder.VideoEncoder.HEVC

29
package/ios/Extensions/AVCaptureVideoDataOutput+recommendedVideoSettings.swift
View File

@ -24,12 +24,31 @@ extension AVCaptureVideoDataOutput {
throw CameraError.capture(.createRecorderError(message: "Failed to get video settings!"))
}
if let bitRate = options.bitRate {
if let bitRateOverride = options.bitRateOverride {
// Convert from Mbps -> bps
let bitsPerSecond = bitRate * 1_000_000
settings[AVVideoCompressionPropertiesKey] = [
AVVideoAverageBitRateKey: NSNumber(value: bitsPerSecond),
]
let bitsPerSecond = bitRateOverride * 1_000_000
if settings[AVVideoCompressionPropertiesKey] == nil {
settings[AVVideoCompressionPropertiesKey] = [:]
}
var compressionSettings = settings[AVVideoCompressionPropertiesKey] as? [String: Any] ?? [:]
let currentBitRate = compressionSettings[AVVideoAverageBitRateKey] as? NSNumber
ReactLogger.log(level: .info, message: "Setting Video Bit-Rate from \(currentBitRate?.doubleValue.description ?? "nil") bps to \(bitsPerSecond) bps...")
compressionSettings[AVVideoAverageBitRateKey] = NSNumber(value: bitsPerSecond)
settings[AVVideoCompressionPropertiesKey] = compressionSettings
}
if let bitRateMultiplier = options.bitRateMultiplier {
// Check if the bit-rate even exists in the settings
if var compressionSettings = settings[AVVideoCompressionPropertiesKey] as? [String: Any],
let currentBitRate = compressionSettings[AVVideoAverageBitRateKey] as? NSNumber {
// Multiply the current value by the given multiplier
let newBitRate = Int(currentBitRate.doubleValue * bitRateMultiplier)
ReactLogger.log(level: .info, message: "Setting Video Bit-Rate from \(currentBitRate) bps to \(newBitRate) bps...")
compressionSettings[AVVideoAverageBitRateKey] = NSNumber(value: newBitRate)
settings[AVVideoCompressionPropertiesKey] = compressionSettings
}
}
return settings

19
package/ios/Types/RecordVideoOptions.swift
View File

@ -14,9 +14,14 @@ struct RecordVideoOptions {
var flash: Torch = .off
var codec: AVVideoCodecType?
/**
Bit-Rate of the Video, in Megabits per second (Mbps)
* Full Bit-Rate override for the Video Encoder, in Megabits per second (Mbps)
*/
var bitRate: Double?
var bitRateOverride: Double?
/**
* A multiplier applied to whatever the currently set bit-rate is, whether it's automatically computed by the OS Encoder,
* or set via bitRate, in Megabits per second (Mbps)
*/
var bitRateMultiplier: Double?
init(fromJSValue dictionary: NSDictionary) throws {
// File Type (.mov or .mp4)
@ -31,9 +36,13 @@ struct RecordVideoOptions {
if let codecOption = dictionary["videoCodec"] as? String {
codec = try AVVideoCodecType(withString: codecOption)
}
// BitRate
if let parsed = dictionary["videoBitRate"] as? Double {
bitRate = parsed
// BitRate Override
if let parsed = dictionary["videoBitRateOverride"] as? Double {
bitRateOverride = parsed
}
// BitRate Multiplier
if let parsed = dictionary["videoBitRateMultiplier"] as? Double {
bitRateMultiplier = parsed
}
}
}

51
package/src/Camera.tsx
View File

@ -35,6 +35,10 @@ type NativeCameraViewProps = Omit<CameraProps, 'device' | 'onInitialized' | 'onE
onCodeScanned?: (event: NativeSyntheticEvent<OnCodeScannedEvent>) => void
onViewReady: () => void
}
type NativeRecordVideoOptions = Omit<RecordVideoOptions, 'onRecordingError' | 'onRecordingFinished' | 'videoBitRate'> & {
videoBitRateOverride?: number
videoBitRateMultiplier?: number
}
type RefType = React.Component<NativeCameraViewProps> & Readonly<NativeMethods>
//#endregion
@ -122,30 +126,15 @@ export class Camera extends React.PureComponent<CameraProps> {
}
}
private calculateBitRate(bitRate: 'low' | 'normal' | 'high', codec: 'h264' | 'h265' = 'h264'): number {
const format = this.props.format
if (format == null) {
throw new CameraRuntimeError(
'parameter/invalid-combination',
`A videoBitRate of '${bitRate}' can only be used in combination with a 'format'!`,
)
private getBitRateMultiplier(bitRate: RecordVideoOptions['videoBitRate']): number {
switch (bitRate) {
case 'low':
return 0.8
case 'high':
return 1.2
default:
return 1
}
const factor = {
low: 0.8,
normal: 1,
high: 1.2,
}[bitRate]
let result = (30 / (3840 * 2160 * 0.75)) * (format.videoWidth * format.videoHeight)
// FPS - 30 is default, 60 would be 2x, 120 would be 4x
const fps = this.props.fps ?? Math.min(format.maxFps, 30)
result = (result / 30) * fps
// H.265 (HEVC) codec is 20% more efficient
if (codec === 'h265') result = result * 0.8
// 10-Bit Video HDR takes up 20% more pixels than standard range (8-bit SDR)
if (this.props.videoHdr) result = result * 1.2
// Return overall result
return result * factor
}
/**
@ -165,12 +154,20 @@ export class Camera extends React.PureComponent<CameraProps> {
* ```
*/
public startRecording(options: RecordVideoOptions): void {
const { onRecordingError, onRecordingFinished, ...passThroughOptions } = options
const { onRecordingError, onRecordingFinished, videoBitRate, ...passThruOptions } = options
if (typeof onRecordingError !== 'function' || typeof onRecordingFinished !== 'function')
throw new CameraRuntimeError('parameter/invalid-parameter', 'The onRecordingError or onRecordingFinished functions were not set!')
const videoBitRate = passThroughOptions.videoBitRate
if (typeof videoBitRate === 'string') passThroughOptions.videoBitRate = this.calculateBitRate(videoBitRate, options.videoCodec)
const nativeOptions: NativeRecordVideoOptions = passThruOptions
if (typeof videoBitRate === 'string') {
// If the user passed 'low'/'normal'/'high', we need to apply this as a multiplier to the native bitrate instead of absolutely setting it
delete nativeOptions.videoBitRateOverride
nativeOptions.videoBitRateMultiplier = this.getBitRateMultiplier(videoBitRate)
} else {
// If the user passed an absolute number as a bit-rate, we just use this as a full override.
delete nativeOptions.videoBitRateOverride
nativeOptions.videoBitRateOverride = videoBitRate
}
const onRecordCallback = (video?: VideoFile, error?: CameraCaptureError): void => {
if (error != null) return onRecordingError(error)
@ -178,7 +175,7 @@ export class Camera extends React.PureComponent<CameraProps> {
}
try {
// TODO: Use TurboModules to make this awaitable.
CameraModule.startRecording(this.handle, passThroughOptions, onRecordCallback)
CameraModule.startRecording(this.handle, nativeOptions, onRecordCallback)
} catch (e) {
throw tryParseNativeCameraError(e)
}