railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: Fix Video Recording delay and improve startRecording() performance (#2192)

Browse Source 
* fix: Add more logs

* perf: Init audio AVAssetWriter in parallel

* chore: Log Audio Session activation

* perf: Init Audio Session asynchronously

* chore: Log when category changed

* fix: Revert lazy audio initialization

* Update Podfile.lock

* Pass `sourceFormatHint` to video and audio AVAssetWriter

* fix: Remove `sourceFormatHint` from Video Asset Writer

* Use default options for Audio Asset Writer

* Update Podfile.lock

* Revert "Use default options for Audio Asset Writer"

This reverts commit e575a14c5342ddc7f9db557d5e3915328ed9e798.

* Add time logs

* fix: Don't synchronize audio buffers, they are already in sync

* shouldOptimizeForNetworkUse = false

* fix: Only update `latestTimestamp` once video buffer has been written

* perf: Use `AVAssetWriterInput` instead of `AVAssetWriterInputPixelBufferAdaptor`

* fix: Fix Audio not being synchronized with Video

* Remove logs add comments

* Format

* feat: Set `.videoRecording` AVAudioSession mode

* Refactor `startRecording()` a bit

* Format

* chore: Throw error directly instead of double-checking
This commit is contained in:
Marc Rousavy
2023-11-22 17:53:10 +01:00
committed by GitHub
parent 49d58d0d0c
commit cf8f3d05e3
11 changed files with 155 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
package/ios/Core/CameraSession+Audio.swift
View File

@@ -23,6 +23,7 @@ extension CameraSession {
let audioSession = AVAudioSession.sharedInstance()
try audioSession.updateCategory(AVAudioSession.Category.playAndRecord,
mode: .videoRecording,
options: [.mixWithOthers,
.allowBluetoothA2DP,
.defaultToSpeaker,
@@ -39,6 +40,7 @@ extension CameraSession {
}
audioCaptureSession.startRunning()
ReactLogger.log(level: .info, message: "Audio Session activated!")
} catch let error as NSError {
ReactLogger.log(level: .error, message: "Failed to activate audio session! Error \(error.code): \(error.description)")
switch error.code {
@@ -54,6 +56,7 @@ extension CameraSession {
ReactLogger.log(level: .info, message: "Deactivating Audio Session...")
audioCaptureSession.stopRunning()
ReactLogger.log(level: .info, message: "Audio Session deactivated!")
}
@objc

81
package/ios/Core/CameraSession+Video.swift
View File

@@ -19,6 +19,7 @@ extension CameraSession {
onError: @escaping (_ error: CameraError) -> Void) {
// Run on Camera Queue
CameraQueues.cameraQueue.async {
let start = DispatchTime.now()
ReactLogger.log(level: .info, message: "Starting Video recording...")
if options.flash != .off {
@@ -59,8 +60,8 @@ extension CameraSession {
}
}
self.recordingSession = nil
self.isRecording = false
self.recordingSession = nil
ReactLogger.log(level: .info, message: "RecordingSession finished with status \(status.descriptor).")
if let error = error as NSError? {
@@ -93,47 +94,20 @@ extension CameraSession {
return
}
ReactLogger.log(level: .info, message: "File path: \(tempFilePath)")
ReactLogger.log(level: .info, message: "Will record to temporary file: \(tempFilePath)")
let tempURL = URL(string: "file://\(tempFilePath)")!
let recordingSession: RecordingSession
do {
recordingSession = try RecordingSession(url: tempURL,
fileType: options.fileType,
completion: onFinish)
} catch let error as NSError {
onError(.capture(.createRecorderError(message: error.description)))
return
}
self.recordingSession = recordingSession
// Create RecordingSession for the temp file
let recordingSession = try RecordingSession(url: tempURL,
fileType: options.fileType,
completion: onFinish)
// Init Video
guard var videoSettings = self.recommendedVideoSettings(videoOutput: videoOutput,
fileType: options.fileType,
videoCodec: options.codec),
!videoSettings.isEmpty else {
onError(.capture(.createRecorderError(message: "Failed to get video settings!")))
return
}
ReactLogger.log(level: .trace, message: "Recommended Video Settings: \(videoSettings.description)")
// Custom Video Bit Rate
if let videoBitRate = options.bitRate {
// Convert from Mbps -> bps
let bitsPerSecond = videoBitRate * 1_000_000
videoSettings[AVVideoCompressionPropertiesKey] = [
AVVideoAverageBitRateKey: NSNumber(value: bitsPerSecond),
]
}
// get pixel format (420f, 420v, x420)
let pixelFormat = videoOutput.pixelFormat
recordingSession.initializeVideoWriter(withSettings: videoSettings,
pixelFormat: pixelFormat)
// Enable/Activate Audio Session (optional)
if enableAudio {
if let audioOutput = self.audioOutput {
// Init Audio + Activate Audio Session (optional)
if enableAudio,
let audioOutput = self.audioOutput,
let audioInput = self.audioDeviceInput {
ReactLogger.log(level: .trace, message: "Enabling Audio for Recording...")
// Activate Audio Session asynchronously
CameraQueues.audioQueue.async {
do {
@@ -145,16 +119,27 @@ extension CameraSession {
// Initialize audio asset writer
let audioSettings = audioOutput.recommendedAudioSettingsForAssetWriter(writingTo: options.fileType)
recordingSession.initializeAudioWriter(withSettings: audioSettings)
recordingSession.initializeAudioWriter(withSettings: audioSettings,
format: audioInput.device.activeFormat.formatDescription)
}
}
// start recording session with or without audio.
do {
// Init Video
let videoSettings = try videoOutput.recommendedVideoSettings(forOptions: options)
recordingSession.initializeVideoWriter(withSettings: videoSettings)
// start recording session with or without audio.
try recordingSession.startAssetWriter()
self.recordingSession = recordingSession
self.isRecording = true
let end = DispatchTime.now()
ReactLogger.log(level: .info, message: "RecordingSesssion started in \(Double(end.uptimeNanoseconds - start.uptimeNanoseconds) / 1_000_000)ms!")
} catch let error as NSError {
onError(.capture(.createRecorderError(message: "RecordingSession failed to start asset writer. \(error.description)")))
if let error = error as? CameraError {
onError(error)
} else {
onError(.capture(.createRecorderError(message: "RecordingSession failed with unknown error: \(error.description)")))
}
return
}
}
@@ -208,14 +193,4 @@ extension CameraSession {
}
}
}
private func recommendedVideoSettings(videoOutput: AVCaptureVideoDataOutput,
fileType: AVFileType,
videoCodec: AVVideoCodecType?) -> [String: Any]? {
if videoCodec != nil {
return videoOutput.recommendedVideoSettings(forVideoCodecType: videoCodec!, assetWriterOutputFileType: fileType)
} else {
return videoOutput.recommendedVideoSettingsForAssetWriter(writingTo: fileType)
}
}
}

10
package/ios/Core/CameraSession.swift
View File

@@ -272,12 +272,12 @@ class CameraSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate, AVC
switch captureOutput {
case is AVCaptureVideoDataOutput:
recordingSession.appendBuffer(sampleBuffer, type: .video, timestamp: CMSampleBufferGetPresentationTimeStamp(sampleBuffer))
// Write the Video Buffer to the .mov/.mp4 file, this is the first timestamp if nothing has been recorded yet
recordingSession.appendBuffer(sampleBuffer, type: .video)
case is AVCaptureAudioDataOutput:
let timestamp = CMSyncConvertTime(CMSampleBufferGetPresentationTimeStamp(sampleBuffer),
from: audioCaptureSession.masterClock ?? CMClockGetHostTimeClock(),
to: captureSession.masterClock ?? CMClockGetHostTimeClock())
recordingSession.appendBuffer(sampleBuffer, type: .audio, timestamp: timestamp)
// Synchronize the Audio Buffer with the Video Session's time because it's two separate AVCaptureSessions
audioCaptureSession.synchronizeBuffer(sampleBuffer, toSession: captureSession)
recordingSession.appendBuffer(sampleBuffer, type: .audio)
default:
break
}

62
package/ios/Core/RecordingSession.swift
View File

@@ -16,18 +16,12 @@ enum BufferType {
case video
}
// MARK: - RecordingSessionError
enum RecordingSessionError: Error {
case failedToStartSession
}
// MARK: - RecordingSession
class RecordingSession {
private let assetWriter: AVAssetWriter
private var audioWriter: AVAssetWriterInput?
private var bufferAdaptor: AVAssetWriterInputPixelBufferAdaptor?
private var videoWriter: AVAssetWriterInput?
private let completionHandler: (RecordingSession, AVAssetWriter.Status, Error?) -> Void
private var initialTimestamp: CMTime?
@@ -61,6 +55,7 @@ class RecordingSession {
do {
assetWriter = try AVAssetWriter(outputURL: url, fileType: fileType)
assetWriter.shouldOptimizeForNetworkUse = false
} catch let error as NSError {
throw CameraError.capture(.createRecorderError(message: error.description))
}
@@ -76,36 +71,38 @@ class RecordingSession {
/**
Initializes an AssetWriter for video frames (CMSampleBuffers).
*/
func initializeVideoWriter(withSettings settings: [String: Any], pixelFormat: OSType) {
func initializeVideoWriter(withSettings settings: [String: Any]) {
guard !settings.isEmpty else {
ReactLogger.log(level: .error, message: "Tried to initialize Video Writer with empty settings!")
return
}
guard bufferAdaptor == nil else {
guard videoWriter == nil else {
ReactLogger.log(level: .error, message: "Tried to add Video Writer twice!")
return
}
let videoWriter = AVAssetWriterInput(mediaType: .video, outputSettings: settings)
videoWriter.expectsMediaDataInRealTime = true
assetWriter.add(videoWriter)
bufferAdaptor = AVAssetWriterInputPixelBufferAdaptor(assetWriterInput: videoWriter,
withVideoSettings: settings,
pixelFormat: pixelFormat)
ReactLogger.log(level: .info, message: "Initializing Video AssetWriter with settings: \(settings.description)")
videoWriter = AVAssetWriterInput(mediaType: .video, outputSettings: settings)
videoWriter!.expectsMediaDataInRealTime = true
assetWriter.add(videoWriter!)
ReactLogger.log(level: .info, message: "Initialized Video AssetWriter.")
}
/**
Initializes an AssetWriter for audio frames (CMSampleBuffers).
*/
func initializeAudioWriter(withSettings settings: [String: Any]?) {
func initializeAudioWriter(withSettings settings: [String: Any]?, format: CMFormatDescription) {
guard audioWriter == nil else {
ReactLogger.log(level: .error, message: "Tried to add Audio Writer twice!")
return
}
audioWriter = AVAssetWriterInput(mediaType: .audio, outputSettings: settings)
if let settings = settings {
ReactLogger.log(level: .info, message: "Initializing Audio AssetWriter with settings: \(settings.description)")
} else {
ReactLogger.log(level: .info, message: "Initializing Audio AssetWriter default settings...")
}
audioWriter = AVAssetWriterInput(mediaType: .audio, outputSettings: settings, sourceFormatHint: format)
audioWriter!.expectsMediaDataInRealTime = true
assetWriter.add(audioWriter!)
ReactLogger.log(level: .info, message: "Initialized Audio AssetWriter.")
@@ -118,17 +115,18 @@ class RecordingSession {
ReactLogger.log(level: .info, message: "Starting Asset Writer(s)...")
let success = assetWriter.startWriting()
if !success {
if success {
ReactLogger.log(level: .info, message: "Asset Writer(s) started!")
} else {
ReactLogger.log(level: .error, message: "Failed to start Asset Writer(s)!")
throw RecordingSessionError.failedToStartSession
throw CameraError.capture(.createRecorderError(message: "Failed to start Asset Writer(s)!"))
}
}
/**
Appends a new CMSampleBuffer to the Asset Writer. Use bufferType to specify if this is a video or audio frame.
The timestamp parameter represents the presentation timestamp of the buffer, which should be synchronized across video and audio frames.
*/
func appendBuffer(_ buffer: CMSampleBuffer, type bufferType: BufferType, timestamp: CMTime) {
func appendBuffer(_ buffer: CMSampleBuffer, type bufferType: BufferType) {
guard assetWriter.status == .writing else {
ReactLogger.log(level: .error, message: "Frame arrived, but AssetWriter status is \(assetWriter.status.descriptor)!")
return
@@ -138,23 +136,18 @@ class RecordingSession {
return
}
latestTimestamp = timestamp
switch bufferType {
case .video:
guard let bufferAdaptor = bufferAdaptor else {
guard let videoWriter = videoWriter else {
ReactLogger.log(level: .error, message: "Video Frame arrived but VideoWriter was nil!")
return
}
if !bufferAdaptor.assetWriterInput.isReadyForMoreMediaData {
if !videoWriter.isReadyForMoreMediaData {
ReactLogger.log(level: .warning,
message: "The Video AVAssetWriterInput was not ready for more data! Is your frame rate too high?")
return
}
guard let imageBuffer = CMSampleBufferGetImageBuffer(buffer) else {
ReactLogger.log(level: .error, message: "Failed to get the CVImageBuffer!")
return
}
let timestamp = CMSampleBufferGetPresentationTimeStamp(buffer)
// Start the writing session before we write the first video frame
if !hasStartedWritingSession {
initialTimestamp = timestamp
@@ -162,7 +155,10 @@ class RecordingSession {
ReactLogger.log(level: .info, message: "Started RecordingSession at \(timestamp.seconds) seconds.")
hasStartedWritingSession = true
}
bufferAdaptor.append(imageBuffer, withPresentationTime: timestamp)
// Write Video Buffer!
videoWriter.append(buffer)
// Update state
latestTimestamp = timestamp
if !hasWrittenFirstVideoFrame {
hasWrittenFirstVideoFrame = true
}
@@ -178,9 +174,11 @@ class RecordingSession {
// first video frame has not been written yet, so skip this audio frame.
return
}
// Write Audio Sample!
audioWriter.append(buffer)
}
// If we failed to write the frames, stop the Recording
if assetWriter.status == .failed {
ReactLogger.log(level: .error,
message: "AssetWriter failed to write buffer! Error: \(assetWriter.error?.localizedDescription ?? "none")")
@@ -206,7 +204,7 @@ class RecordingSession {
completionHandler(self, .failed, error)
} else if assetWriter.status == .writing {
isFinishing = true
bufferAdaptor?.assetWriterInput.markAsFinished()
videoWriter?.markAsFinished()
audioWriter?.markAsFinished()
assetWriter.finishWriting {
self.isFinishing = false