From 49fba9ed60d3c8c6db02beae2c56a308194722f2 Mon Sep 17 00:00:00 2001 From: Loewy Date: Mon, 22 Dec 2025 18:48:12 -0500 Subject: [PATCH] Fix fMP4 video orientation by using raw sensor frames with Y-flip transform --- .../android/src/main/cpp/OpenGLRenderer.cpp | 1 + .../android/src/main/cpp/VideoPipeline.cpp | 12 ++++++++- .../com/mrousavy/camera/core/CameraSession.kt | 10 +++---- .../camera/core/FragmentedRecordingManager.kt | 26 ++++++++++++------- 4 files changed, 33 insertions(+), 16 deletions(-) diff --git a/package/android/src/main/cpp/OpenGLRenderer.cpp b/package/android/src/main/cpp/OpenGLRenderer.cpp index e9d6bd3..13a33f6 100644 --- a/package/android/src/main/cpp/OpenGLRenderer.cpp +++ b/package/android/src/main/cpp/OpenGLRenderer.cpp @@ -26,6 +26,7 @@ OpenGLRenderer::OpenGLRenderer(std::shared_ptr context, ANativeWi _outputSurface = surface; _width = ANativeWindow_getWidth(surface); _height = ANativeWindow_getHeight(surface); + __android_log_print(ANDROID_LOG_INFO, TAG, "ROTATION_DEBUG OpenGLRenderer created with output surface dimensions: %dx%d", _width, _height); } OpenGLRenderer::~OpenGLRenderer() { diff --git a/package/android/src/main/cpp/VideoPipeline.cpp b/package/android/src/main/cpp/VideoPipeline.cpp index 8160daa..1aa746f 100644 --- a/package/android/src/main/cpp/VideoPipeline.cpp +++ b/package/android/src/main/cpp/VideoPipeline.cpp @@ -79,7 +79,17 @@ void VideoPipeline::onFrame(jni::alias_ref transformMatrixPara if (_recordingSessionOutput) { __android_log_print(ANDROID_LOG_INFO, TAG, "Rendering to RecordingSession.."); - _recordingSessionOutput->renderTextureToSurface(texture, transformMatrix); + // For recording, use a simple Y-flip matrix instead of the display transform. + // The display transform includes rotations for preview which we don't want in recordings. + // We'll rely on rotation metadata in the MP4 container for playback orientation. + // This matrix flips Y (OpenGL origin is bottom-left, video expects top-left). + float recordingMatrix[16] = { + 1.0f, 0.0f, 0.0f, 0.0f, // row 0 + 0.0f, -1.0f, 0.0f, 0.0f, // row 1 (Y flip) + 0.0f, 0.0f, 1.0f, 0.0f, // row 2 + 0.0f, 1.0f, 0.0f, 1.0f // row 3 (translate Y by 1 after flip) + }; + _recordingSessionOutput->renderTextureToSurface(texture, recordingMatrix); } } diff --git a/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt b/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt index 770812f..cda1ff9 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt @@ -428,18 +428,16 @@ class CameraSession(private val context: Context, private val cameraManager: Cam // Get actual device rotation from WindowManager since the React Native orientation hook // doesn't update when rotating between landscape-left and landscape-right on Android. - // Map device rotation to the correct orientationHint for video recording: - // - Counter-clockwise (ROTATION_90) → 270° hint - // - Clockwise (ROTATION_270) → 90° hint val windowManager = context.getSystemService(Context.WINDOW_SERVICE) as WindowManager val deviceRotation = windowManager.defaultDisplay.rotation val recordingOrientation = when (deviceRotation) { Surface.ROTATION_0 -> Orientation.PORTRAIT - Surface.ROTATION_90 -> Orientation.LANDSCAPE_RIGHT + Surface.ROTATION_90 -> Orientation.LANDSCAPE_LEFT // CCW rotation, top to left Surface.ROTATION_180 -> Orientation.PORTRAIT_UPSIDE_DOWN - Surface.ROTATION_270 -> Orientation.LANDSCAPE_LEFT + Surface.ROTATION_270 -> Orientation.LANDSCAPE_RIGHT // CW rotation, top to right else -> Orientation.PORTRAIT } + Log.i(TAG, "ROTATION_DEBUG: deviceRotation=$deviceRotation, recordingOrientation=$recordingOrientation, options.orientation=${options.orientation}") val recording = RecordingSession( context, @@ -448,7 +446,7 @@ class CameraSession(private val context: Context, private val cameraManager: Cam enableAudio, fps, videoOutput.enableHdr, - orientation, + recordingOrientation, options, filePath, callback, diff --git a/package/android/src/main/java/com/mrousavy/camera/core/FragmentedRecordingManager.kt b/package/android/src/main/java/com/mrousavy/camera/core/FragmentedRecordingManager.kt index 6157808..2e19a02 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/FragmentedRecordingManager.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/FragmentedRecordingManager.kt @@ -51,14 +51,21 @@ class FragmentedRecordingManager( segmentDurationSeconds: Int = 6 ): FragmentedRecordingManager { val mimeType = options.videoCodec.toMimeType() - val cameraOrientationDegrees = cameraOrientation.toDegrees() - val recordingOrientationDegrees = (options.orientation ?: Orientation.PORTRAIT).toDegrees() - - val (width, height) = if (cameraOrientation.isLandscape()) { - size.height to size.width - } else { - size.width to size.height + // For fragmented MP4: DON'T swap dimensions, use camera's native dimensions. + // The C++ VideoPipeline now uses a simple Y-flip matrix (not the display transform). + // This gives us raw sensor frames, and we rely on rotation metadata for playback. + val cameraOrientationDegrees = when (cameraOrientation) { + Orientation.LANDSCAPE_LEFT -> 0 // CCW landscape - works! + Orientation.LANDSCAPE_RIGHT -> 0 // CW landscape - same as CCW (Y-flip handles it) + Orientation.PORTRAIT -> 90 // Portrait typically needs 90° on Android + Orientation.PORTRAIT_UPSIDE_DOWN -> 270 } + Log.i(TAG, "ROTATION_DEBUG FragmentedRecordingManager: cameraOrientation=$cameraOrientation, cameraOrientationDegrees=$cameraOrientationDegrees, inputSize=${size.width}x${size.height}") + + // Keep original dimensions - don't swap. Let rotation metadata handle orientation. + val width = size.width + val height = size.height + Log.i(TAG, "ROTATION_DEBUG FragmentedRecordingManager: outputDimensions=${width}x${height} (no swap)") val format = MediaFormat.createVideoFormat(mimeType, width, height) val codec = MediaCodec.createEncoderByType(mimeType) @@ -74,14 +81,14 @@ class FragmentedRecordingManager( format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, segmentDurationSeconds) format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate) - Log.d(TAG, "Video Format: $format, camera orientation $cameraOrientationDegrees, recordingOrientation: $recordingOrientationDegrees") + Log.d(TAG, "Video Format: $format, camera orientation $cameraOrientationDegrees") codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) return FragmentedRecordingManager( codec, outputDirectory, - recordingOrientationDegrees, + cameraOrientationDegrees, segmentDurationSeconds * 1_000_000L, callbacks ) @@ -317,6 +324,7 @@ class FragmentedRecordingManager( initData.add(bytes) } + Log.i(TAG, "ROTATION_DEBUG convertToMedia3Format: orientationDegrees=$orientationDegrees, width=$width, height=$height") return Format.Builder() .setSampleMimeType(mimeType) .setWidth(width)