Compare commits

..

6 Commits

Author SHA1 Message Date
de5e15b347 use window manager to determine device rotation in android 2026-01-02 09:55:17 -08:00
c0b352dc1d fix fps (alter tfhd and trun size, add logs 2025-12-30 15:01:41 -05:00
a307f9b7e8 normalize timestamps and fix framerate metadata in init file 2025-12-29 16:48:47 -05:00
b79f876114 Fix orientation issues 2025-12-28 01:14:44 -08:00
e60c1a4eb1 Write our own muxer to make hls uupload actually work 2025-12-21 16:45:04 -08:00
a2d218580c feat: Add fragmented MP4 (fMP4) support for Android
Implements HLS-compatible fragmented MP4 recording on Android using
AndroidX Media3 FragmentedMp4Muxer, matching the iOS implementation.

Changes:
- Add FragmentedRecordingManager for fMP4 segment output
- Add ChunkedRecorderInterface to abstract recorder implementations
- Add onInitSegmentReady callback for init segment (init.mp4)
- Update onVideoChunkReady to include segment duration
- RecordingSession now uses FragmentedRecordingManager by default

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
2025-12-18 12:29:03 -08:00
19 changed files with 111 additions and 302 deletions

View File

@@ -17,18 +17,15 @@
namespace vision { namespace vision {
std::unique_ptr<OpenGLRenderer> OpenGLRenderer::CreateWithWindowSurface(std::shared_ptr<OpenGLContext> context, std::unique_ptr<OpenGLRenderer> OpenGLRenderer::CreateWithWindowSurface(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface) {
ANativeWindow* surface, return std::unique_ptr<OpenGLRenderer>(new OpenGLRenderer(std::move(context), surface));
int rotationDegrees) {
return std::unique_ptr<OpenGLRenderer>(new OpenGLRenderer(std::move(context), surface, rotationDegrees));
} }
OpenGLRenderer::OpenGLRenderer(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface, int rotationDegrees) { OpenGLRenderer::OpenGLRenderer(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface) {
_context = std::move(context); _context = std::move(context);
_outputSurface = surface; _outputSurface = surface;
_width = ANativeWindow_getWidth(surface); _width = ANativeWindow_getWidth(surface);
_height = ANativeWindow_getHeight(surface); _height = ANativeWindow_getHeight(surface);
_rotationDegrees = rotationDegrees;
} }
OpenGLRenderer::~OpenGLRenderer() { OpenGLRenderer::~OpenGLRenderer() {
@@ -69,7 +66,7 @@ void OpenGLRenderer::renderTextureToSurface(const OpenGLTexture& texture, float*
glTexParameteri(texture.target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glTexParameteri(texture.target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
// 4. Draw it using the pass-through shader which also applies transforms // 4. Draw it using the pass-through shader which also applies transforms
_passThroughShader.draw(texture, transformMatrix, _rotationDegrees); _passThroughShader.draw(texture, transformMatrix);
// 5. Swap buffers to pass it to the window surface // 5. Swap buffers to pass it to the window surface
eglSwapBuffers(_context->display, _surface); eglSwapBuffers(_context->display, _surface);

View File

@@ -24,7 +24,7 @@ public:
* Note: The `surface` is considered moved, and the OpenGL context will release it when it is * Note: The `surface` is considered moved, and the OpenGL context will release it when it is
* being deleted. * being deleted.
*/ */
static std::unique_ptr<OpenGLRenderer> CreateWithWindowSurface(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface, int rotationDegrees); static std::unique_ptr<OpenGLRenderer> CreateWithWindowSurface(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface);
/** /**
* Destroy the OpenGL Context. This needs to be called on the same thread that `use()` was called. * Destroy the OpenGL Context. This needs to be called on the same thread that `use()` was called.
*/ */
@@ -43,11 +43,10 @@ public:
void destroy(); void destroy();
private: private:
explicit OpenGLRenderer(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface, int rotationDegrees); explicit OpenGLRenderer(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface);
private: private:
int _width = 0, _height = 0; int _width = 0, _height = 0;
int _rotationDegrees = 0;
std::shared_ptr<OpenGLContext> _context; std::shared_ptr<OpenGLContext> _context;
ANativeWindow* _outputSurface; ANativeWindow* _outputSurface;
EGLSurface _surface = EGL_NO_SURFACE; EGLSurface _surface = EGL_NO_SURFACE;

View File

@@ -7,76 +7,11 @@
#include <EGL/egl.h> #include <EGL/egl.h>
#include <GLES2/gl2.h> #include <GLES2/gl2.h>
#include <GLES2/gl2ext.h> #include <GLES2/gl2ext.h>
#include <cmath>
#include <memory> #include <memory>
#include <string> #include <string>
namespace vision { namespace vision {
namespace {
void setIdentity(float* matrix) {
for (int i = 0; i < 16; i++) {
matrix[i] = 0.0f;
}
matrix[0] = 1.0f;
matrix[5] = 1.0f;
matrix[10] = 1.0f;
matrix[15] = 1.0f;
}
void multiply4x4(const float* left, const float* right, float* out) {
for (int column = 0; column < 4; column++) {
for (int row = 0; row < 4; row++) {
float sum = 0.0f;
for (int k = 0; k < 4; k++) {
sum += left[k * 4 + row] * right[column * 4 + k];
}
out[column * 4 + row] = sum;
}
}
}
void makeTranslation(float tx, float ty, float* matrix) {
setIdentity(matrix);
matrix[12] = tx;
matrix[13] = ty;
}
void makeRotation(float degrees, float* matrix) {
setIdentity(matrix);
const float radians = degrees * static_cast<float>(M_PI) / 180.0f;
const float cosine = std::cos(radians);
const float sine = std::sin(radians);
matrix[0] = cosine;
matrix[1] = sine;
matrix[4] = -sine;
matrix[5] = cosine;
}
void makeCenteredRotation(int rotationDegrees, float* matrix) {
const int normalized = ((rotationDegrees % 360) + 360) % 360;
if (normalized == 0) {
setIdentity(matrix);
return;
}
float translateToOrigin[16];
float rotation[16];
float translateBack[16];
float temp[16];
makeTranslation(-0.5f, -0.5f, translateToOrigin);
makeRotation(static_cast<float>(normalized), rotation);
makeTranslation(0.5f, 0.5f, translateBack);
multiply4x4(rotation, translateToOrigin, temp);
multiply4x4(translateBack, temp, matrix);
}
} // namespace
PassThroughShader::~PassThroughShader() { PassThroughShader::~PassThroughShader() {
if (_programId != NO_SHADER) { if (_programId != NO_SHADER) {
glDeleteProgram(_programId); glDeleteProgram(_programId);
@@ -89,7 +24,7 @@ PassThroughShader::~PassThroughShader() {
} }
} }
void PassThroughShader::draw(const OpenGLTexture& texture, float* transformMatrix, int rotationDegrees) { void PassThroughShader::draw(const OpenGLTexture& texture, float* transformMatrix) {
// 1. Set up Shader Program // 1. Set up Shader Program
if (_programId == NO_SHADER) { if (_programId == NO_SHADER) {
_programId = createProgram(); _programId = createProgram();
@@ -122,12 +57,7 @@ void PassThroughShader::draw(const OpenGLTexture& texture, float* transformMatri
glVertexAttribPointer(_vertexParameters.aTexCoord, 2, GL_FLOAT, GL_FALSE, sizeof(Vertex), glVertexAttribPointer(_vertexParameters.aTexCoord, 2, GL_FLOAT, GL_FALSE, sizeof(Vertex),
reinterpret_cast<void*>(offsetof(Vertex, texCoord))); reinterpret_cast<void*>(offsetof(Vertex, texCoord)));
float outputRotationMatrix[16]; glUniformMatrix4fv(_vertexParameters.uTransformMatrix, 1, GL_FALSE, transformMatrix);
float combinedTransformMatrix[16];
makeCenteredRotation(rotationDegrees, outputRotationMatrix);
multiply4x4(transformMatrix, outputRotationMatrix, combinedTransformMatrix);
glUniformMatrix4fv(_vertexParameters.uTransformMatrix, 1, GL_FALSE, combinedTransformMatrix);
// 4. Pass texture to fragment shader // 4. Pass texture to fragment shader
glActiveTexture(GL_TEXTURE0); glActiveTexture(GL_TEXTURE0);

View File

@@ -29,7 +29,7 @@ public:
* Draw the texture using this shader. * Draw the texture using this shader.
* Note: At the moment, only EXTERNAL textures are supported by the Shader. * Note: At the moment, only EXTERNAL textures are supported by the Shader.
*/ */
void draw(const OpenGLTexture& texture, float* transformMatrix, int rotationDegrees); void draw(const OpenGLTexture& texture, float* transformMatrix);
private: private:
// Loading // Loading

View File

@@ -47,13 +47,13 @@ void VideoPipeline::removeRecordingSessionOutputSurface() {
_recordingSessionOutput = nullptr; _recordingSessionOutput = nullptr;
} }
void VideoPipeline::setRecordingSessionOutputSurface(jobject surface, int rotationDegrees) { void VideoPipeline::setRecordingSessionOutputSurface(jobject surface) {
// 1. Delete existing output surface // 1. Delete existing output surface
removeRecordingSessionOutputSurface(); removeRecordingSessionOutputSurface();
// 2. Set new output surface if it is not null // 2. Set new output surface if it is not null
ANativeWindow* window = ANativeWindow_fromSurface(jni::Environment::current(), surface); ANativeWindow* window = ANativeWindow_fromSurface(jni::Environment::current(), surface);
_recordingSessionOutput = OpenGLRenderer::CreateWithWindowSurface(_context, window, rotationDegrees); _recordingSessionOutput = OpenGLRenderer::CreateWithWindowSurface(_context, window);
} }
int VideoPipeline::getInputTextureId() { int VideoPipeline::getInputTextureId() {
@@ -78,6 +78,7 @@ void VideoPipeline::onFrame(jni::alias_ref<jni::JArrayFloat> transformMatrixPara
OpenGLTexture& texture = _inputTexture.value(); OpenGLTexture& texture = _inputTexture.value();
if (_recordingSessionOutput) { if (_recordingSessionOutput) {
__android_log_print(ANDROID_LOG_INFO, TAG, "Rendering to RecordingSession..");
_recordingSessionOutput->renderTextureToSurface(texture, transformMatrix); _recordingSessionOutput->renderTextureToSurface(texture, transformMatrix);
} }
} }

View File

@@ -31,7 +31,7 @@ public:
int getInputTextureId(); int getInputTextureId();
// <- MediaRecorder output // <- MediaRecorder output
void setRecordingSessionOutputSurface(jobject surface, int rotationDegrees); void setRecordingSessionOutputSurface(jobject surface);
void removeRecordingSessionOutputSurface(); void removeRecordingSessionOutputSurface();
// Frame callbacks // Frame callbacks

View File

@@ -30,15 +30,8 @@ class ChunkedRecordingManager(private val encoder: MediaCodec, private val outpu
iFrameInterval: Int = 5 iFrameInterval: Int = 5
): ChunkedRecordingManager { ): ChunkedRecordingManager {
val mimeType = options.videoCodec.toMimeType() val mimeType = options.videoCodec.toMimeType()
// Use cameraOrientation (from WindowManager) for rotation metadata val cameraOrientationDegrees = cameraOrientation.toDegrees()
// The options.orientation from JavaScript is unreliable on Android when rotating between landscape modes val recordingOrientationDegrees = (options.orientation ?: Orientation.PORTRAIT).toDegrees();
// Note: MediaMuxer.setOrientationHint() uses opposite convention from HlsMuxer's rotation matrix
// We need to invert the rotation: 90 <-> 270, while 0 and 180 stay the same
val orientationDegrees = when (cameraOrientation.toDegrees()) {
90 -> 270
270 -> 90
else -> cameraOrientation.toDegrees()
}
val (width, height) = if (cameraOrientation.isLandscape()) { val (width, height) = if (cameraOrientation.isLandscape()) {
size.height to size.width size.height to size.width
} else { } else {
@@ -62,12 +55,12 @@ class ChunkedRecordingManager(private val encoder: MediaCodec, private val outpu
format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, iFrameInterval) format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, iFrameInterval)
format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate) format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate)
Log.d(TAG, "Video Format: $format, orientation: $orientationDegrees") Log.d(TAG, "Video Format: $format, camera orientation $cameraOrientationDegrees, recordingOrientation: $recordingOrientationDegrees")
// Create a MediaCodec encoder, and configure it with our format. Get a Surface // Create a MediaCodec encoder, and configure it with our format. Get a Surface
// we can use for input and wrap it with a class that handles the EGL work. // we can use for input and wrap it with a class that handles the EGL work.
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
return ChunkedRecordingManager( return ChunkedRecordingManager(
codec, outputDirectory, orientationDegrees, iFrameInterval, callbacks codec, outputDirectory, recordingOrientationDegrees, iFrameInterval, callbacks
) )
} }
} }
@@ -98,13 +91,12 @@ class ChunkedRecordingManager(private val encoder: MediaCodec, private val outpu
muxer.start() muxer.start()
} }
fun writeSample(buffer: java.nio.ByteBuffer, bufferInfo: BufferInfo) {
muxer.writeSampleData(videoTrack, buffer, bufferInfo)
}
fun finish() { fun finish() {
muxer.stop() muxer.stop()
muxer.release() muxer.release()
// Calculate duration from start time - this is approximate
// The new FragmentedRecordingManager provides accurate duration
callbacks.onVideoChunkReady(filepath, chunkIndex, null) callbacks.onVideoChunkReady(filepath, chunkIndex, null)
} }
} }
@@ -178,7 +170,7 @@ class ChunkedRecordingManager(private val encoder: MediaCodec, private val outpu
encoder.releaseOutputBuffer(index, false) encoder.releaseOutputBuffer(index, false)
return return
} }
context.writeSample(encodedData, bufferInfo) context.muxer.writeSampleData(context.videoTrack, encodedData, bufferInfo)
encoder.releaseOutputBuffer(index, false) encoder.releaseOutputBuffer(index, false)
} }
} }

View File

@@ -39,11 +39,18 @@ class FragmentedRecordingManager(
segmentDurationSeconds: Int = DEFAULT_SEGMENT_DURATION_SECONDS segmentDurationSeconds: Int = DEFAULT_SEGMENT_DURATION_SECONDS
): FragmentedRecordingManager { ): FragmentedRecordingManager {
val mimeType = options.videoCodec.toMimeType() val mimeType = options.videoCodec.toMimeType()
val cameraOrientationDegrees = cameraOrientation.toDegrees() // Use cameraOrientation (from WindowManager) for rotation metadata
val orientationDegrees = 0 // The options.orientation from JavaScript is unreliable on Android when rotating between landscape modes
val (width, height) = size.width to size.height val orientationDegrees = cameraOrientation.toDegrees()
Log.d(TAG, "Recording: ${width}x${height}, orientation=$orientationDegrees° (cameraOrientation=$cameraOrientationDegrees°)") // Swap dimensions based on camera orientation, same as ChunkedRecordingManager
val (width, height) = if (cameraOrientation.isLandscape()) {
size.height to size.width
} else {
size.width to size.height
}
Log.d(TAG, "Recording: ${width}x${height}, orientation=$orientationDegrees°")
val format = MediaFormat.createVideoFormat(mimeType, width, height) val format = MediaFormat.createVideoFormat(mimeType, width, height)
val codec = MediaCodec.createEncoderByType(mimeType) val codec = MediaCodec.createEncoderByType(mimeType)

View File

@@ -753,32 +753,17 @@ class HlsMuxer(
dos.writeShort(-1) // pre-defined dos.writeShort(-1) // pre-defined
output.write(buildAvcCBox(sps, pps)) output.write(buildAvcCBox(sps, pps))
output.write(buildPaspBox())
return wrapBox("avc1", output.toByteArray()) return wrapBox("avc1", output.toByteArray())
} }
/**
* Builds pixel aspect ratio box to explicitly declare square pixels (1:1).
* This helps players correctly interpret video dimensions without SAR scaling.
*/
private fun buildPaspBox(): ByteArray {
val output = ByteArrayOutputStream()
val dos = DataOutputStream(output)
dos.writeInt(1) // hSpacing (horizontal)
dos.writeInt(1) // vSpacing (vertical)
return wrapBox("pasp", output.toByteArray())
}
private fun buildAvcCBox(sps: ByteArray, pps: ByteArray): ByteArray { private fun buildAvcCBox(sps: ByteArray, pps: ByteArray): ByteArray {
val output = ByteArrayOutputStream() val output = ByteArrayOutputStream()
val dos = DataOutputStream(output) val dos = DataOutputStream(output)
// SPS NAL unit format: [NAL header, profile_idc, constraint_flags, level_idc, ...] val profileIdc = if (sps.isNotEmpty()) sps[0].toInt() and 0xFF else 0x42
// Skip byte 0 (NAL header, typically 0x67) to get the actual profile data val profileCompat = if (sps.size > 1) sps[1].toInt() and 0xFF else 0x00
val profileIdc = if (sps.size > 1) sps[1].toInt() and 0xFF else 0x42 val levelIdc = if (sps.size > 2) sps[2].toInt() and 0xFF else 0x1F
val profileCompat = if (sps.size > 2) sps[2].toInt() and 0xFF else 0x00
val levelIdc = if (sps.size > 3) sps[3].toInt() and 0xFF else 0x1F
dos.writeByte(1) // configuration version dos.writeByte(1) // configuration version
dos.writeByte(profileIdc) // AVC profile dos.writeByte(profileIdc) // AVC profile

View File

@@ -235,19 +235,10 @@ class PersistentCameraCaptureSession(private val cameraManager: CameraManager, p
// 1. Run a precapture sequence for AF, AE and AWB. // 1. Run a precapture sequence for AF, AE and AWB.
focusJob = coroutineScope.launch { focusJob = coroutineScope.launch {
try {
val request = repeatingRequest.createCaptureRequest(device, deviceDetails, outputs) val request = repeatingRequest.createCaptureRequest(device, deviceDetails, outputs)
val options = val options =
PrecaptureOptions(listOf(PrecaptureTrigger.AF, PrecaptureTrigger.AE), Flash.OFF, listOf(point), false, FOCUS_RESET_TIMEOUT) PrecaptureOptions(listOf(PrecaptureTrigger.AF, PrecaptureTrigger.AE), Flash.OFF, listOf(point), false, FOCUS_RESET_TIMEOUT)
session.precapture(request, deviceDetails, options) session.precapture(request, deviceDetails, options)
} catch (e: CaptureTimedOutError) {
// Focus timed out - this is non-fatal, just log and continue
Log.w(TAG, "Focus timed out at point $point, continuing without focus lock")
} catch (e: IllegalStateException) {
Log.w(TAG, "Focus failed, camera device was already closed: ${e.message}")
} catch (e: CameraAccessException) {
Log.w(TAG, "Focus failed, camera not accessible: ${e.message}")
}
} }
focusJob?.join() focusJob?.join()
@@ -263,15 +254,9 @@ class PersistentCameraCaptureSession(private val cameraManager: CameraManager, p
return@launch return@launch
} }
Log.i(TAG, "Resetting focus to auto-focus...") Log.i(TAG, "Resetting focus to auto-focus...")
try {
repeatingRequest.createCaptureRequest(device, deviceDetails, outputs).also { request -> repeatingRequest.createCaptureRequest(device, deviceDetails, outputs).also { request ->
session.setRepeatingRequest(request.build(), null, null) session.setRepeatingRequest(request.build(), null, null)
} }
} catch (e: IllegalStateException) {
Log.w(TAG, "Failed to reset focus, camera device was already closed: ${e.message}")
} catch (e: CameraAccessException) {
Log.w(TAG, "Failed to reset focus, camera not accessible: ${e.message}")
}
} }
} }
} }

View File

@@ -5,7 +5,7 @@ import android.content.Context
import android.content.res.Configuration import android.content.res.Configuration
import android.graphics.Point import android.graphics.Point
import android.os.Handler import android.os.Handler
import android.os.HandlerThread import android.os.Looper
import android.util.Log import android.util.Log
import android.util.Size import android.util.Size
import android.view.PixelCopy import android.view.PixelCopy
@@ -25,72 +25,58 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.suspendCancellableCoroutine import kotlinx.coroutines.suspendCancellableCoroutine
import kotlinx.coroutines.withContext import kotlinx.coroutines.withContext
import android.graphics.Bitmap import android.graphics.Bitmap
import android.graphics.Matrix
fun rotateBitmap90CounterClockwise(source: Bitmap): Bitmap {
val width = source.width
val height = source.height
// Create a new Bitmap with swapped width and height
val rotatedBitmap = Bitmap.createBitmap(height, width, source.config ?: Bitmap.Config.ARGB_8888)
for (y in 0 until height) {
for (x in 0 until width) {
// Set the pixel in the new position
rotatedBitmap.setPixel(y, width - 1 - x, source.getPixel(x, y))
}
}
return rotatedBitmap
}
fun Bitmap.transformBitmap(orientation: Orientation): Bitmap { fun Bitmap.transformBitmap(orientation: Orientation): Bitmap {
return when (orientation) { return when (orientation) {
Orientation.PORTRAIT -> this // No transformation needed Orientation.PORTRAIT -> this // No transformation needed
Orientation.LANDSCAPE_LEFT -> { Orientation.LANDSCAPE_LEFT -> {
val srcWidth = width // Transpose (swap width and height)
val srcHeight = height val transposedBitmap = Bitmap.createBitmap(height, width, config ?: Bitmap.Config.ARGB_8888)
val sourcePixels = IntArray(srcWidth * srcHeight) for (y in 0 until height) {
getPixels(sourcePixels, 0, srcWidth, 0, 0, srcWidth, srcHeight) for (x in 0 until width) {
transposedBitmap.setPixel(y, width - 1 - x, getPixel(x, y))
val dstWidth = srcHeight
val dstHeight = srcWidth
val destinationPixels = IntArray(dstWidth * dstHeight)
for (y in 0 until srcHeight) {
for (x in 0 until srcWidth) {
val dstX = y
val dstY = srcWidth - 1 - x
destinationPixels[dstY * dstWidth + dstX] = sourcePixels[y * srcWidth + x]
} }
} }
transposedBitmap
val transformedBitmap = Bitmap.createBitmap(dstWidth, dstHeight, config ?: Bitmap.Config.ARGB_8888)
transformedBitmap.setPixels(destinationPixels, 0, dstWidth, 0, 0, dstWidth, dstHeight)
transformedBitmap
} }
Orientation.PORTRAIT_UPSIDE_DOWN -> { Orientation.PORTRAIT_UPSIDE_DOWN -> {
val srcWidth = width // Invert vertically and horizontally (180-degree rotation)
val srcHeight = height val invertedBitmap = Bitmap.createBitmap(width, height, config ?: Bitmap.Config.ARGB_8888)
val sourcePixels = IntArray(srcWidth * srcHeight) for (y in 0 until height) {
getPixels(sourcePixels, 0, srcWidth, 0, 0, srcWidth, srcHeight) for (x in 0 until width) {
invertedBitmap.setPixel(width - 1 - x, height - 1 - y, getPixel(x, y))
val dstWidth = srcWidth
val dstHeight = srcHeight
val destinationPixels = IntArray(dstWidth * dstHeight)
for (y in 0 until srcHeight) {
for (x in 0 until srcWidth) {
val dstX = srcWidth - 1 - x
val dstY = srcHeight - 1 - y
destinationPixels[dstY * dstWidth + dstX] = sourcePixels[y * srcWidth + x]
} }
} }
invertedBitmap
val transformedBitmap = Bitmap.createBitmap(dstWidth, dstHeight, config ?: Bitmap.Config.ARGB_8888)
transformedBitmap.setPixels(destinationPixels, 0, dstWidth, 0, 0, dstWidth, dstHeight)
transformedBitmap
} }
Orientation.LANDSCAPE_RIGHT -> { Orientation.LANDSCAPE_RIGHT -> {
val srcWidth = width // Transpose (swap width and height) and invert vertically
val srcHeight = height val transposedBitmap = Bitmap.createBitmap(height, width, config ?: Bitmap.Config.ARGB_8888)
val sourcePixels = IntArray(srcWidth * srcHeight) for (y in 0 until height) {
getPixels(sourcePixels, 0, srcWidth, 0, 0, srcWidth, srcHeight) for (x in 0 until width) {
transposedBitmap.setPixel(height - 1 - y, x, getPixel(x, y))
val dstWidth = srcHeight
val dstHeight = srcWidth
val destinationPixels = IntArray(dstWidth * dstHeight)
for (y in 0 until srcHeight) {
for (x in 0 until srcWidth) {
val dstX = srcHeight - 1 - y
val dstY = x
destinationPixels[dstY * dstWidth + dstX] = sourcePixels[y * srcWidth + x]
} }
} }
transposedBitmap
val transformedBitmap = Bitmap.createBitmap(dstWidth, dstHeight, config ?: Bitmap.Config.ARGB_8888)
transformedBitmap.setPixels(destinationPixels, 0, dstWidth, 0, 0, dstWidth, dstHeight)
transformedBitmap
} }
} }
} }
@@ -200,7 +186,7 @@ class PreviewView(context: Context, callback: SurfaceHolder.Callback) :
) )
} }
}, },
pixelCopyHandler Handler(Looper.getMainLooper())
) )
} }
} }
@@ -212,10 +198,8 @@ class PreviewView(context: Context, callback: SurfaceHolder.Callback) :
val viewOrientation = Orientation.PORTRAIT val viewOrientation = Orientation.PORTRAIT
val rotated = point.rotatedBy(viewSize, cameraSize, viewOrientation, sensorOrientation) val rotated = point.rotatedBy(viewSize, cameraSize, viewOrientation, sensorOrientation)
// Clamp to valid camera coordinates (must be non-negative for MeteringRectangle) Log.i(TAG, "Converted layer point $point to camera point $rotated! ($sensorOrientation, $cameraSize -> $viewSize)")
val clamped = Point(maxOf(0, rotated.x), maxOf(0, rotated.y)) return rotated
Log.i(TAG, "Converted layer point $point to camera point $clamped! ($sensorOrientation, $cameraSize -> $viewSize)")
return clamped
} }
private fun updateLayout() { private fun updateLayout() {
@@ -270,10 +254,5 @@ class PreviewView(context: Context, callback: SurfaceHolder.Callback) :
companion object { companion object {
private const val TAG = "PreviewView" private const val TAG = "PreviewView"
private val pixelCopyHandler: Handler by lazy {
val handlerThread = HandlerThread("VisionCamera.PixelCopy")
handlerThread.start()
Handler(handlerThread.looper)
}
} }
} }

View File

@@ -8,7 +8,6 @@ import com.facebook.common.statfs.StatFsHelper
import com.mrousavy.camera.extensions.getRecommendedBitRate import com.mrousavy.camera.extensions.getRecommendedBitRate
import com.mrousavy.camera.types.Orientation import com.mrousavy.camera.types.Orientation
import com.mrousavy.camera.types.RecordVideoOptions import com.mrousavy.camera.types.RecordVideoOptions
import com.mrousavy.camera.types.StreamSegmentType
import com.mrousavy.camera.utils.FileUtils import com.mrousavy.camera.utils.FileUtils
import java.io.File import java.io.File
import android.os.Environment import android.os.Environment
@@ -28,7 +27,9 @@ class RecordingSession(
private val filePath: String, private val filePath: String,
private val callback: (video: Video) -> Unit, private val callback: (video: Video) -> Unit,
private val onError: (error: CameraError) -> Unit, private val onError: (error: CameraError) -> Unit,
private val allCallbacks: CameraSession.Callback private val allCallbacks: CameraSession.Callback,
// Use FragmentedRecordingManager for HLS-compatible fMP4 output
private val useFragmentedMp4: Boolean = true
) { ) {
companion object { companion object {
private const val TAG = "RecordingSession" private const val TAG = "RecordingSession"
@@ -43,26 +44,20 @@ class RecordingSession(
data class Video(val path: String, val durationMs: Long, val size: Size) data class Video(val path: String, val durationMs: Long, val size: Size)
val outputRotationDegrees: Int = // Strip file:// prefix if present (expo-file-system returns URIs with this prefix)
if (options.streamSegmentType == StreamSegmentType.FRAGMENTED_MP4) { private val outputPath: File = File(
when (cameraOrientation.toDegrees()) { if (filePath.startsWith("file://")) {
90 -> 270 filePath.removePrefix("file://")
270 -> 90
else -> cameraOrientation.toDegrees()
}
} else { } else {
0 filePath
} }
)
// Normalize path - expo-file-system passes file:// URIs but File expects raw paths
// Handle both file:// and file:/ variants
private val outputPath: File = File(filePath.replace(Regex("^file:/+"), "/"))
private val bitRate = getBitRate() private val bitRate = getBitRate()
// Use FragmentedRecordingManager for HLS-compatible fMP4 output, // Use FragmentedRecordingManager for HLS-compatible fMP4 output,
// or fall back to ChunkedRecordingManager for regular MP4 chunks // or fall back to ChunkedRecordingManager for regular MP4 chunks
private val recorder: ChunkedRecorderInterface = if (options.streamSegmentType == StreamSegmentType.FRAGMENTED_MP4) { private val recorder: ChunkedRecorderInterface = if (useFragmentedMp4) {
FragmentedRecordingManager.fromParams( FragmentedRecordingManager.fromParams(
allCallbacks, allCallbacks,
size, size,
@@ -94,7 +89,7 @@ class RecordingSession(
fun start() { fun start() {
synchronized(this) { synchronized(this) {
Log.i(TAG, "Starting RecordingSession with ${options.streamSegmentType} recorder..") Log.i(TAG, "Starting RecordingSession..")
startTime = System.currentTimeMillis() startTime = System.currentTimeMillis()
recorder.start() recorder.start()
} }

View File

@@ -186,7 +186,7 @@ class VideoPipeline(
if (recordingSession != null) { if (recordingSession != null) {
// Configure OpenGL pipeline to stream Frames into the Recording Session's surface // Configure OpenGL pipeline to stream Frames into the Recording Session's surface
Log.i(TAG, "Setting ${recordingSession.size} RecordingSession Output...") Log.i(TAG, "Setting ${recordingSession.size} RecordingSession Output...")
setRecordingSessionOutputSurface(recordingSession.surface, recordingSession.outputRotationDegrees) setRecordingSessionOutputSurface(recordingSession.surface)
this.recordingSession = recordingSession this.recordingSession = recordingSession
} else { } else {
// Configure OpenGL pipeline to stop streaming Frames into the Recording Session's surface // Configure OpenGL pipeline to stop streaming Frames into the Recording Session's surface
@@ -250,7 +250,7 @@ class VideoPipeline(
private external fun getInputTextureId(): Int private external fun getInputTextureId(): Int
private external fun onBeforeFrame() private external fun onBeforeFrame()
private external fun onFrame(transformMatrix: FloatArray) private external fun onFrame(transformMatrix: FloatArray)
private external fun setRecordingSessionOutputSurface(surface: Any, rotationDegrees: Int) private external fun setRecordingSessionOutputSurface(surface: Any)
private external fun removeRecordingSessionOutputSurface() private external fun removeRecordingSessionOutputSurface()
private external fun initHybrid(width: Int, height: Int): HybridData private external fun initHybrid(width: Int, height: Int): HybridData
} }

View File

@@ -9,7 +9,6 @@ class RecordVideoOptions(map: ReadableMap) {
var videoBitRateOverride: Double? = null var videoBitRateOverride: Double? = null
var videoBitRateMultiplier: Double? = null var videoBitRateMultiplier: Double? = null
var orientation: Orientation? = null var orientation: Orientation? = null
var streamSegmentType: StreamSegmentType = StreamSegmentType.FRAGMENTED_MP4
init { init {
if (map.hasKey("fileType")) { if (map.hasKey("fileType")) {
@@ -30,8 +29,5 @@ class RecordVideoOptions(map: ReadableMap) {
if (map.hasKey("orientation")) { if (map.hasKey("orientation")) {
orientation = Orientation.fromUnionValue(map.getString("orientation")) orientation = Orientation.fromUnionValue(map.getString("orientation"))
} }
if (map.hasKey("streamSegmentType")) {
streamSegmentType = StreamSegmentType.fromUnionValue(map.getString("streamSegmentType"))
}
} }
} }

View File

@@ -1,15 +0,0 @@
package com.mrousavy.camera.types
enum class StreamSegmentType(override val unionValue: String) : JSUnionValue {
FRAGMENTED_MP4("FRAGMENTED_MP4"),
RB_CHUNKED_MP4("RB_CHUNKED_MP4");
companion object : JSUnionValue.Companion<StreamSegmentType> {
override fun fromUnionValue(unionValue: String?): StreamSegmentType =
when (unionValue) {
"FRAGMENTED_MP4" -> FRAGMENTED_MP4
"RB_CHUNKED_MP4" -> RB_CHUNKED_MP4
else -> FRAGMENTED_MP4 // Default to fMP4
}
}
}

View File

@@ -20,9 +20,6 @@ extension CameraSession {
onError: @escaping (_ error: CameraError) -> Void) { onError: @escaping (_ error: CameraError) -> Void) {
// Run on Camera Queue // Run on Camera Queue
CameraQueues.cameraQueue.async { CameraQueues.cameraQueue.async {
// Normalize path - expo-file-system passes file:// URIs but FileManager expects raw paths
let normalizedPath = filePath.hasPrefix("file://") ? String(filePath.dropFirst(7)) : filePath
let start = DispatchTime.now() let start = DispatchTime.now()
ReactLogger.log(level: .info, message: "Starting Video recording...") ReactLogger.log(level: .info, message: "Starting Video recording...")
@@ -41,27 +38,11 @@ extension CameraSession {
// Callback for when new chunks are ready // Callback for when new chunks are ready
let onChunkReady: (ChunkedRecorder.Chunk) -> Void = { chunk in let onChunkReady: (ChunkedRecorder.Chunk) -> Void = { chunk in
guard let delegate = self.delegate else { guard let delegate = self.delegate else {
ReactLogger.log(level: .warning, message: "Chunk ready but delegate is nil, dropping chunk: \(chunk)")
return return
} }
delegate.onVideoChunkReady(chunk: chunk) delegate.onVideoChunkReady(chunk: chunk)
} }
// Callback for when a chunk write fails (e.g. init file write failure)
let onChunkError: (Error) -> Void = { error in
ReactLogger.log(level: .error, message: "Chunk write error, stopping recording: \(error.localizedDescription)")
// Stop recording immediately
if let session = self.recordingSession {
session.stop(clock: self.captureSession.clock)
}
// Surface error to RN
if let cameraError = error as? CameraError {
onError(cameraError)
} else {
onError(.capture(.fileError))
}
}
// Callback for when the recording ends // Callback for when the recording ends
let onFinish = { (recordingSession: RecordingSession, status: AVAssetWriter.Status, error: Error?) in let onFinish = { (recordingSession: RecordingSession, status: AVAssetWriter.Status, error: Error?) in
defer { defer {
@@ -101,23 +82,22 @@ extension CameraSession {
} }
} }
if !FileManager.default.fileExists(atPath: normalizedPath) { if !FileManager.default.fileExists(atPath: filePath) {
do { do {
try FileManager.default.createDirectory(atPath: normalizedPath, withIntermediateDirectories: true) try FileManager.default.createDirectory(atPath: filePath, withIntermediateDirectories: true)
} catch { } catch {
onError(.capture(.createRecordingDirectoryError(message: error.localizedDescription))) onError(.capture(.createRecordingDirectoryError(message: error.localizedDescription)))
return return
} }
} }
ReactLogger.log(level: .info, message: "Will record to temporary file: \(normalizedPath)") ReactLogger.log(level: .info, message: "Will record to temporary file: \(filePath)")
do { do {
// Create RecordingSession for the temp file // Create RecordingSession for the temp file
let recordingSession = try RecordingSession(outputDiretory: normalizedPath, let recordingSession = try RecordingSession(outputDiretory: filePath,
fileType: options.fileType, fileType: options.fileType,
onChunkReady: onChunkReady, onChunkReady: onChunkReady,
onChunkError: onChunkError,
completion: onFinish) completion: onFinish)
// Init Audio + Activate Audio Session (optional) // Init Audio + Activate Audio Session (optional)

View File

@@ -24,14 +24,12 @@ class ChunkedRecorder: NSObject {
let outputURL: URL let outputURL: URL
let onChunkReady: ((Chunk) -> Void) let onChunkReady: ((Chunk) -> Void)
let onError: ((Error) -> Void)?
private var chunkIndex: UInt64 = 0 private var chunkIndex: UInt64 = 0
init(outputURL: URL, onChunkReady: @escaping ((Chunk) -> Void), onError: ((Error) -> Void)? = nil) throws { init(outputURL: URL, onChunkReady: @escaping ((Chunk) -> Void)) throws {
self.outputURL = outputURL self.outputURL = outputURL
self.onChunkReady = onChunkReady self.onChunkReady = onChunkReady
self.onError = onError
guard FileManager.default.fileExists(atPath: outputURL.path) else { guard FileManager.default.fileExists(atPath: outputURL.path) else {
throw CameraError.unknown(message: "output directory does not exist at: \(outputURL.path)", cause: nil) throw CameraError.unknown(message: "output directory does not exist at: \(outputURL.path)", cause: nil)
} }
@@ -58,19 +56,14 @@ extension ChunkedRecorder: AVAssetWriterDelegate {
private func saveInitSegment(_ data: Data) { private func saveInitSegment(_ data: Data) {
let url = outputURL.appendingPathComponent("init.mp4") let url = outputURL.appendingPathComponent("init.mp4")
do { save(data: data, url: url)
try data.write(to: url)
onChunkReady(url: url, type: .initialization) onChunkReady(url: url, type: .initialization)
} catch {
ReactLogger.log(level: .error, message: "Failed to write init file \(url): \(error.localizedDescription)")
onError?(CameraError.capture(.fileError))
}
} }
private func saveSegment(_ data: Data, report: AVAssetSegmentReport?) { private func saveSegment(_ data: Data, report: AVAssetSegmentReport?) {
let name = "\(chunkIndex).mp4" let name = "\(chunkIndex).mp4"
let url = outputURL.appendingPathComponent(name) let url = outputURL.appendingPathComponent(name)
if save(data: data, url: url) { save(data: data, url: url)
let duration = report? let duration = report?
.trackReports .trackReports
.filter { $0.mediaType == .video } .filter { $0.mediaType == .video }
@@ -79,15 +72,12 @@ extension ChunkedRecorder: AVAssetWriterDelegate {
onChunkReady(url: url, type: .data(index: chunkIndex, duration: duration)) onChunkReady(url: url, type: .data(index: chunkIndex, duration: duration))
chunkIndex += 1 chunkIndex += 1
} }
}
private func save(data: Data, url: URL) -> Bool { private func save(data: Data, url: URL) {
do { do {
try data.write(to: url) try data.write(to: url)
return true
} catch { } catch {
ReactLogger.log(level: .error, message: "Unable to write \(url): \(error.localizedDescription)") ReactLogger.log(level: .error, message: "Unable to write \(url): \(error.localizedDescription)")
return false
} }
} }

View File

@@ -74,13 +74,12 @@ class RecordingSession {
init(outputDiretory: String, init(outputDiretory: String,
fileType: AVFileType, fileType: AVFileType,
onChunkReady: @escaping ((ChunkedRecorder.Chunk) -> Void), onChunkReady: @escaping ((ChunkedRecorder.Chunk) -> Void),
onChunkError: ((Error) -> Void)? = nil,
completion: @escaping (RecordingSession, AVAssetWriter.Status, Error?) -> Void) throws { completion: @escaping (RecordingSession, AVAssetWriter.Status, Error?) -> Void) throws {
completionHandler = completion completionHandler = completion
do { do {
let outputURL = URL(fileURLWithPath: outputDiretory) let outputURL = URL(fileURLWithPath: outputDiretory)
recorder = try ChunkedRecorder(outputURL: outputURL, onChunkReady: onChunkReady, onError: onChunkError) recorder = try ChunkedRecorder(outputURL: outputURL, onChunkReady: onChunkReady)
assetWriter = AVAssetWriter(contentType: UTType(fileType.rawValue)!) assetWriter = AVAssetWriter(contentType: UTType(fileType.rawValue)!)
assetWriter.shouldOptimizeForNetworkUse = false assetWriter.shouldOptimizeForNetworkUse = false
assetWriter.outputFileTypeProfile = .mpeg4AppleHLS assetWriter.outputFileTypeProfile = .mpeg4AppleHLS

View File

@@ -41,17 +41,6 @@ export interface RecordVideoOptions {
* @default 'normal' * @default 'normal'
*/ */
videoBitRate?: 'extra-low' | 'low' | 'normal' | 'high' | 'extra-high' | number videoBitRate?: 'extra-low' | 'low' | 'normal' | 'high' | 'extra-high' | number
/**
* The stream segment type for recording on Android.
* - `FRAGMENTED_MP4`: HLS-compatible segments (init.mp4 + numbered segments)
* - `RB_CHUNKED_MP4`: Legacy chunked MP4 format
*
* iOS always uses FRAGMENTED_MP4 regardless of this setting.
*
* @platform android
* @default 'FRAGMENTED_MP4'
*/
streamSegmentType?: 'FRAGMENTED_MP4' | 'RB_CHUNKED_MP4'
} }
/** /**