railbird/react-native-vision-camera
9
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Simplify ChunkedRecorder

Browse Source
This commit is contained in:
Ivan Malison 2024-01-27 19:55:20 -07:00
parent d95057fa47
commit 08f37070a4
7 changed files with 172 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

201
package/android/src/main/java/com/mrousavy/camera/core/ChunkedRecorder.kt
View File

@ -1,39 +1,45 @@
package com.mrousavy.camera.core package com.mrousavy.camera.core
import kotlinx.coroutines.*
import kotlinx.coroutines.channels.Channel
import android.media.MediaCodec import android.media.MediaCodec
import android.media.MediaCodec.BufferInfo
import android.media.MediaCodecInfo import android.media.MediaCodecInfo
import android.media.MediaFormat import android.media.MediaFormat
import android.media.MediaMuxer import android.media.MediaMuxer
import android.util.Log import android.util.Log
import android.util.Size import android.util.Size
import android.view.Surface import android.view.Surface
import java.io.File
import com.mrousavy.camera.types.Orientation import com.mrousavy.camera.types.Orientation
import com.mrousavy.camera.types.RecordVideoOptions import com.mrousavy.camera.types.RecordVideoOptions
import java.io.File
import java.nio.ByteBuffer import java.nio.ByteBuffer
import kotlinx.coroutines.*
class ChunkedRecordingManager( class ChunkedRecordingManager(private val encoder: MediaCodec, private val outputDirectory: File, private val orientationHint: Int, private val iFrameInterval: Int) :
private val encoder: MediaCodec, MediaCodec.Callback() {
private val outputDirectory: File,
private val orientationHint: Int,
) {
companion object { companion object {
private const val TAG = "ChunkedRecorder" private const val TAG = "ChunkedRecorder"
private const val targetDurationUs = 10 * 1000000
fun fromParams( fun fromParams(
size: Size, size: Size,
enableAudio: Boolean, enableAudio: Boolean,
fps: Int? = null, fps: Int? = null,
orientation: Orientation, orientation: Orientation,
bitRate: Int,
options: RecordVideoOptions, options: RecordVideoOptions,
outputDirectory: File, outputDirectory: File,
iFrameInterval: Int = 3
): ChunkedRecordingManager { ): ChunkedRecordingManager {
val mimeType = options.videoCodec.toMimeType() val mimeType = options.videoCodec.toMimeType()
val format = MediaFormat.createVideoFormat(mimeType, size.width, size.height) var width = size.width
var height = size.height
val orientationDegrees = orientation.toDegrees()
if (orientationDegrees == 90 || orientationDegrees == 270) {
width = size.height
height = size.width
}
val format = MediaFormat.createVideoFormat(mimeType, width, height)
val codec = MediaCodec.createEncoderByType(mimeType) val codec = MediaCodec.createEncoderByType(mimeType)
@ -47,22 +53,24 @@ class ChunkedRecordingManager(
format.setInteger(MediaFormat.KEY_FRAME_RATE, this) format.setInteger(MediaFormat.KEY_FRAME_RATE, this)
} }
// TODO: Pull this out into configuration // TODO: Pull this out into configuration
format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1) format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, iFrameInterval)
format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate)
Log.i(TAG, "Video Format: $format")
Log.d(TAG, "Video Format: $format")
// Create a MediaCodec encoder, and configure it with our format. Get a Surface // Create a MediaCodec encoder, and configure it with our format. Get a Surface
// we can use for input and wrap it with a class that handles the EGL work. // we can use for input and wrap it with a class that handles the EGL work.
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
return ChunkedRecordingManager(codec, outputDirectory, orientation.toDegrees()) return ChunkedRecordingManager(codec, outputDirectory, orientationDegrees, iFrameInterval)
} }
} }
// In flight details // In flight details
private val bufferInfo = MediaCodec.BufferInfo()
private var currentFrameNumber: Int = 0 private var currentFrameNumber: Int = 0
private var chunkIndex = 0 private var chunkIndex = -1
private var encodedFormat: MediaFormat? = null private var encodedFormat: MediaFormat? = null
private var recording = false;
private val targetDurationUs = iFrameInterval * 1000000
val surface: Surface = encoder.createInputSurface() val surface: Surface = encoder.createInputSurface()
@ -70,15 +78,11 @@ class ChunkedRecordingManager(
if (!this.outputDirectory.exists()) { if (!this.outputDirectory.exists()) {
this.outputDirectory.mkdirs() this.outputDirectory.mkdirs()
} }
encoder.setCallback(this)
} }
// Muxer specific // Muxer specific
private class MuxerContext( private class MuxerContext(val muxer: MediaMuxer, startTimeUs: Long, encodedFormat: MediaFormat) {
muxer: MediaMuxer,
startTimeUs: Long,
encodedFormat: MediaFormat
) {
val muxer = muxer
val videoTrack: Int = muxer.addTrack(encodedFormat) val videoTrack: Int = muxer.addTrack(encodedFormat)
val startTimeUs: Long = startTimeUs val startTimeUs: Long = startTimeUs
@ -86,135 +90,80 @@ class ChunkedRecordingManager(
muxer.start() muxer.start()
} }
fun finish() { fun finish() {
muxer.stop() muxer.stop()
muxer.release() muxer.release()
} }
} }
private lateinit var muxerContext: MuxerContext private var muxerContext: MuxerContext? = null
private fun createNextMuxer() { private fun createNextMuxer(bufferInfo: BufferInfo) {
if (::muxerContext.isInitialized) { muxerContext?.finish()
muxerContext.finish()
chunkIndex++ chunkIndex++
}
val newFileName = "$chunkIndex.mp4" val newFileName = "$chunkIndex.mp4"
val newOutputFile = File(this.outputDirectory, newFileName) val newOutputFile = File(this.outputDirectory, newFileName)
Log.d(TAG, "Creating new muxer for file: $newFileName") Log.i(TAG, "Creating new muxer for file: $newFileName")
val muxer = MediaMuxer( val muxer = MediaMuxer(
newOutputFile.absolutePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4 newOutputFile.absolutePath,
MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4
) )
muxer.setOrientationHint(orientationHint)
muxerContext = MuxerContext( muxerContext = MuxerContext(
muxer, bufferInfo.presentationTimeUs, this.encodedFormat!! muxer, bufferInfo.presentationTimeUs, this.encodedFormat!!
) )
muxer.setOrientationHint(orientationHint)
} }
private fun atKeyframe(): Boolean { private fun atKeyframe(bufferInfo: BufferInfo): Boolean {
return (bufferInfo.flags and MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0 return (bufferInfo.flags and MediaCodec.BUFFER_FLAG_KEY_FRAME) != 0
} }
private fun chunkLengthUs(): Long { private fun chunkLengthUs(bufferInfo: BufferInfo): Long {
return bufferInfo.presentationTimeUs - muxerContext.startTimeUs return bufferInfo.presentationTimeUs - muxerContext!!.startTimeUs
} }
fun drainEncoder(): Boolean { fun start() {
val timeout: Long = 0 encoder.start()
var frameWasEncoded = false recording = true
while (true) {
var encoderStatus: Int = encoder.dequeueOutputBuffer(bufferInfo, timeout)
if (encoderStatus < 0) {
Log.w(
TAG, "Unexpected result from encoder.dequeueOutputBuffer: $encoderStatus"
)
}
when (encoderStatus) {
MediaCodec.INFO_TRY_AGAIN_LATER -> break;
MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {
// Should happen before receiving buffers, and should only happen once. The MediaFormat
// contains the csd-0 and csd-1 keys, which we'll need for MediaMuxer. It's unclear what
// else MediaMuxer might want, so rather than extract the codec-specific data and
// reconstruct a new MediaFormat later, we just grab it here and keep it around.
encodedFormat = encoder.outputFormat
Log.d(TAG, "encoder output format changed: $encodedFormat")
}
else -> {
var encodedData: ByteBuffer = encoder.getOutputBuffer(encoderStatus)
?: throw RuntimeException("encoderOutputBuffer $encoderStatus was null")
if ((bufferInfo.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
// The codec config data was pulled out when we got the
// INFO_OUTPUT_FORMAT_CHANGED status. The MediaMuxer won't accept
// a single big blob -- it wants separate csd-0/csd-1 chunks --
// so simply saving this off won't work.
Log.d(TAG, "ignoring BUFFER_FLAG_CODEC_CONFIG")
bufferInfo.size = 0
}
if (bufferInfo.size != 0) {
// adjust the ByteBuffer values to match BufferInfo (not needed?)
encodedData.position(bufferInfo.offset)
encodedData.limit(bufferInfo.offset + bufferInfo.size)
if (!::muxerContext.isInitialized || (atKeyframe() && chunkLengthUs() >= targetDurationUs)) {
this.createNextMuxer()
}
// TODO: we should probably add the presentation time stamp
// mEncBuffer.add(encodedData, bufferInfo.flags, bufferInfo.presentationTimeUs)
muxerContext.muxer.writeSampleData(muxerContext.videoTrack, encodedData, bufferInfo)
frameWasEncoded = true
}
encoder.releaseOutputBuffer(encoderStatus, false)
if ((bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
Log.w(TAG, "reached end of stream unexpectedly")
break
}
}
}
}
return frameWasEncoded
} }
fun finish() { fun finish() {
if (::muxerContext.isInitialized) { synchronized(this) {
muxerContext.finish() muxerContext?.finish()
recording = false
muxerContext = null
encoder.stop()
} }
} }
}
// MediaCodec.Callback methods
class ChunkedRecorder(private val manager: ChunkedRecordingManager) { override fun onInputBufferAvailable(codec: MediaCodec, index: Int) {
private val messageChannel = Channel<Message>() }
init { override fun onOutputBufferAvailable(codec: MediaCodec, index: Int, bufferInfo: MediaCodec.BufferInfo) {
CoroutineScope(Dispatchers.Default).launch { synchronized(this) {
for (msg in messageChannel) { if (!recording) {
when (msg) { return
is Message.FrameAvailable -> manager.drainEncoder() }
is Message.Shutdown -> manager.finish() val encodedData: ByteBuffer = encoder.getOutputBuffer(index)
} ?: throw RuntimeException("getOutputBuffer was null")
}
} if (muxerContext == null || (atKeyframe(bufferInfo) && chunkLengthUs(bufferInfo) >= targetDurationUs)) {
} this.createNextMuxer(bufferInfo)
}
fun sendFrameAvailable() { muxerContext!!.muxer.writeSampleData(muxerContext!!.videoTrack, encodedData, bufferInfo)
messageChannel.trySend(Message.FrameAvailable) encoder.releaseOutputBuffer(index, false)
} }
}
fun sendShutdown() {
messageChannel.trySend(Message.Shutdown) override fun onError(codec: MediaCodec, e: MediaCodec.CodecException) {
} // Implement error handling
Log.e(TAG, "Codec error: ${e.message}")
sealed class Message { }
object FrameAvailable : Message()
object Shutdown : Message() override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) {
encodedFormat = format
} }
} }

32
package/android/src/main/java/com/mrousavy/camera/core/RecordingSession.kt
View File

@ -10,7 +10,10 @@ import com.mrousavy.camera.types.Orientation
import com.mrousavy.camera.types.RecordVideoOptions import com.mrousavy.camera.types.RecordVideoOptions
import com.mrousavy.camera.utils.FileUtils import com.mrousavy.camera.utils.FileUtils
import java.io.File import java.io.File
import android.os.Environment
import java.text.SimpleDateFormat
import java.util.Locale
import java.util.Date
class RecordingSession( class RecordingSession(
context: Context, context: Context,
val cameraId: String, val cameraId: String,
@ -33,23 +36,34 @@ class RecordingSession(
data class Video(val path: String, val durationMs: Long, val size: Size) data class Video(val path: String, val durationMs: Long, val size: Size)
private val outputPath = File.createTempFile("mrousavy", options.fileType.toExtension(), context.cacheDir) private val outputPath = run {
val videoDir = Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_MOVIES)
val sdf = SimpleDateFormat("yyyy_MM_dd_HH_mm_ss_SSS", Locale.US)
val videoFileName = "VID_${sdf.format(Date())}"
File(videoDir!!, videoFileName)
}
private val bitRate = getBitRate() private val bitRate = getBitRate()
private val recordingManager = ChunkedRecordingManager.fromParams( private val recorder = ChunkedRecordingManager.fromParams(
size, enableAudio, fps, orientation, options, outputPath size,
enableAudio,
fps,
orientation,
bitRate,
options,
outputPath
) )
private val recorder: ChunkedRecorder = ChunkedRecorder(recordingManager)
private var startTime: Long? = null private var startTime: Long? = null
val surface: Surface val surface: Surface
get() { get() {
return recordingManager.surface return recorder.surface
} }
fun start() { fun start() {
synchronized(this) { synchronized(this) {
Log.i(TAG, "Starting RecordingSession..") Log.i(TAG, "Starting RecordingSession..")
startTime = System.currentTimeMillis() startTime = System.currentTimeMillis()
recorder.start()
} }
} }
@ -57,14 +71,15 @@ class RecordingSession(
synchronized(this) { synchronized(this) {
Log.i(TAG, "Stopping RecordingSession..") Log.i(TAG, "Stopping RecordingSession..")
try { try {
recorder.sendShutdown() recorder.finish()
} catch (e: Error) { } catch (e: Error) {
Log.e(TAG, "Failed to stop MediaRecorder!", e) Log.e(TAG, "Failed to stop MediaRecorder!", e)
} }
val stopTime = System.currentTimeMillis() val stopTime = System.currentTimeMillis()
val durationMs = stopTime - (startTime ?: stopTime) val durationMs = stopTime - (startTime ?: stopTime)
//callback(Video(outputFile.absolutePath, durationMs, size)) Log.i(TAG, "Finished recording video at $outputPath")
callback(Video(outputPath.absolutePath, durationMs, size))
} }
} }
@ -113,6 +128,5 @@ class RecordingSession(
} }
fun onFrame() { fun onFrame() {
recorder.sendFrameAvailable()
} }
} }

5
package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt
View File

@ -91,7 +91,7 @@ class VideoPipeline(
imageWriter = ImageWriter.newInstance(glSurface, MAX_IMAGES) imageWriter = ImageWriter.newInstance(glSurface, MAX_IMAGES)
} }
imageReader!!.setOnImageAvailableListener({ reader -> imageReader!!.setOnImageAvailableListener({ reader ->
Log.i(TAG, "ImageReader::onImageAvailable!") // Log.i(TAG, "ImageReader::onImageAvailable!")s
val image = reader.acquireNextImage() ?: return@setOnImageAvailableListener val image = reader.acquireNextImage() ?: return@setOnImageAvailableListener
// TODO: Get correct orientation and isMirrored // TODO: Get correct orientation and isMirrored
@ -153,7 +153,8 @@ class VideoPipeline(
// 5. Draw it with applied rotation/mirroring // 5. Draw it with applied rotation/mirroring
onFrame(transformMatrix) onFrame(transformMatrix)
recording?.onFrame() // 6. Notify the recording session.
recordingSession?.onFrame()
} }
} }

2
package/android/src/main/java/com/mrousavy/camera/types/VideoCodec.kt
View File

@ -1,7 +1,7 @@
package com.mrousavy.camera.types package com.mrousavy.camera.types
import android.media.MediaRecorder
import android.media.MediaFormat import android.media.MediaFormat
import android.media.MediaRecorder
enum class VideoCodec(override val unionValue: String) : JSUnionValue { enum class VideoCodec(override val unionValue: String) : JSUnionValue {
H264("h264"), H264("h264"),

6
package/example/src/CameraPage.tsx
View File

@ -160,9 +160,9 @@ export function CameraPage({ navigation }: Props): React.ReactElement {
const frameProcessor = useFrameProcessor((frame) => { const frameProcessor = useFrameProcessor((frame) => {
'worklet' 'worklet'
console.log(`${frame.timestamp}: ${frame.width}x${frame.height} ${frame.pixelFormat} Frame (${frame.orientation})`) // console.log(`${frame.timestamp}: ${frame.width}x${frame.height} ${frame.pixelFormat} Frame (${frame.orientation})`)
examplePlugin(frame) // examplePlugin(frame)
exampleKotlinSwiftPlugin(frame) // exampleKotlinSwiftPlugin(frame)
}, []) }, [])
return ( return (

2
package/example/src/MediaPage.tsx
View File

@ -74,7 +74,7 @@ export function MediaPage({ navigation, route }: Props): React.ReactElement {
} }
}, [path, type]) }, [path, type])
const source = useMemo(() => ({ uri: `file://${path}` }), [path]) const source = useMemo(() => ({ uri: `file://${path}/1.mp4` }), [path])
const screenStyle = useMemo(() => ({ opacity: hasMediaLoaded ? 1 : 0 }), [hasMediaLoaded]) const screenStyle = useMemo(() => ({ opacity: hasMediaLoaded ? 1 : 0 }), [hasMediaLoaded])

24
package/flake.nix
View File

@ -11,17 +11,23 @@
url = "github:tadfisher/android-nixpkgs"; url = "github:tadfisher/android-nixpkgs";
}; };
}; };
outputs = { self, nixpkgs, flake-utils, gitignore, android-nixpkgs, ... }: outputs = {
flake-utils.lib.eachDefaultSystem (system: self,
let nixpkgs,
pkgs = import nixpkgs { inherit system; }; flake-utils,
gitignore,
android-nixpkgs,
...
}:
flake-utils.lib.eachDefaultSystem (system: let
pkgs = import nixpkgs {inherit system;};
nodejs = pkgs.nodejs-18_x; nodejs = pkgs.nodejs-18_x;
# NOTE: this does not work # NOTE: this does not work
appBuild = pkgs.stdenv.mkDerivation { appBuild = pkgs.stdenv.mkDerivation {
name = "example-ts-node"; name = "example-ts-node";
version = "0.1.0"; version = "0.1.0";
src = gitignore.lib.gitignoreSource ./.; # uses the gitignore in the repo to only copy files git would see src = gitignore.lib.gitignoreSource ./.; # uses the gitignore in the repo to only copy files git would see
buildInputs = [ nodejs ]; buildInputs = [nodejs];
# https://nixos.org/manual/nixpkgs/stable/#sec-stdenv-phases # https://nixos.org/manual/nixpkgs/stable/#sec-stdenv-phases
buildPhase = '' buildPhase = ''
# each phase has pre/postHooks. When you make your own phase be sure to still call the hooks # each phase has pre/postHooks. When you make your own phase be sure to still call the hooks
@ -38,7 +44,8 @@
runHook postInstall runHook postInstall
''; '';
}; };
android-sdk = android-nixpkgs.sdk.${system} (sdkPkgs: with sdkPkgs; [ android-sdk = android-nixpkgs.sdk.${system} (sdkPkgs:
with sdkPkgs; [
cmdline-tools-latest cmdline-tools-latest
build-tools-30-0-3 build-tools-30-0-3
build-tools-33-0-0 build-tools-33-0-0
@ -53,10 +60,11 @@
system-images-android-33-google-apis-x86-64 system-images-android-33-google-apis-x86-64
system-images-android-34-google-apis-x86-64 system-images-android-34-google-apis-x86-64
]); ]);
in with pkgs; { in
with pkgs; {
defaultPackage = appBuild; defaultPackage = appBuild;
devShell = mkShell { devShell = mkShell {
buildInputs = [ nodejs yarn watchman gradle_7 alejandra nodePackages.prettier ]; buildInputs = [nodejs yarn watchman gradle_7 alejandra nodePackages.prettier ktlint kotlin-language-server];
ANDROID_SDK_BIN = android-sdk; ANDROID_SDK_BIN = android-sdk;
shellHook = '' shellHook = ''
export JAVA_HOME=${pkgs.jdk17.home} export JAVA_HOME=${pkgs.jdk17.home}