railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

Fix fMP4 video orientation by using raw sensor frames with Y-flip transform

Browse Source
This commit is contained in:
Loewy
2025-12-22 18:48:12 -05:00
parent a2d218580c
commit 49fba9ed60
4 changed files with 33 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
package/android/src/main/cpp/OpenGLRenderer.cpp
View File

@@ -26,6 +26,7 @@ OpenGLRenderer::OpenGLRenderer(std::shared_ptr<OpenGLContext> context, ANativeWi
_outputSurface = surface;
_width = ANativeWindow_getWidth(surface);
_height = ANativeWindow_getHeight(surface);
__android_log_print(ANDROID_LOG_INFO, TAG, "ROTATION_DEBUG OpenGLRenderer created with output surface dimensions: %dx%d", _width, _height);
}
OpenGLRenderer::~OpenGLRenderer() {

12
package/android/src/main/cpp/VideoPipeline.cpp
View File

@@ -79,7 +79,17 @@ void VideoPipeline::onFrame(jni::alias_ref<jni::JArrayFloat> transformMatrixPara
if (_recordingSessionOutput) {
__android_log_print(ANDROID_LOG_INFO, TAG, "Rendering to RecordingSession..");
_recordingSessionOutput->renderTextureToSurface(texture, transformMatrix);
// For recording, use a simple Y-flip matrix instead of the display transform.
// The display transform includes rotations for preview which we don't want in recordings.
// We'll rely on rotation metadata in the MP4 container for playback orientation.
// This matrix flips Y (OpenGL origin is bottom-left, video expects top-left).
float recordingMatrix[16] = {
1.0f, 0.0f, 0.0f, 0.0f, // row 0
0.0f, -1.0f, 0.0f, 0.0f, // row 1 (Y flip)
0.0f, 0.0f, 1.0f, 0.0f, // row 2
0.0f, 1.0f, 0.0f, 1.0f // row 3 (translate Y by 1 after flip)
};
_recordingSessionOutput->renderTextureToSurface(texture, recordingMatrix);
}
}

10
package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt
View File

@@ -428,18 +428,16 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
// Get actual device rotation from WindowManager since the React Native orientation hook
// doesn't update when rotating between landscape-left and landscape-right on Android.
// Map device rotation to the correct orientationHint for video recording:
// - Counter-clockwise (ROTATION_90) → 270° hint
// - Clockwise (ROTATION_270) → 90° hint
val windowManager = context.getSystemService(Context.WINDOW_SERVICE) as WindowManager
val deviceRotation = windowManager.defaultDisplay.rotation
val recordingOrientation = when (deviceRotation) {
Surface.ROTATION_0 -> Orientation.PORTRAIT
Surface.ROTATION_90 -> Orientation.LANDSCAPE_RIGHT
Surface.ROTATION_90 -> Orientation.LANDSCAPE_LEFT // CCW rotation, top to left
Surface.ROTATION_180 -> Orientation.PORTRAIT_UPSIDE_DOWN
Surface.ROTATION_270 -> Orientation.LANDSCAPE_LEFT
Surface.ROTATION_270 -> Orientation.LANDSCAPE_RIGHT // CW rotation, top to right
else -> Orientation.PORTRAIT
}
Log.i(TAG, "ROTATION_DEBUG: deviceRotation=$deviceRotation, recordingOrientation=$recordingOrientation, options.orientation=${options.orientation}")
val recording = RecordingSession(
context,
@@ -448,7 +446,7 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
enableAudio,
fps,
videoOutput.enableHdr,
orientation,
recordingOrientation,
options,
filePath,
callback,

26
package/android/src/main/java/com/mrousavy/camera/core/FragmentedRecordingManager.kt
View File

@@ -51,14 +51,21 @@ class FragmentedRecordingManager(
segmentDurationSeconds: Int = 6
): FragmentedRecordingManager {
val mimeType = options.videoCodec.toMimeType()
val cameraOrientationDegrees = cameraOrientation.toDegrees()
val recordingOrientationDegrees = (options.orientation ?: Orientation.PORTRAIT).toDegrees()
val (width, height) = if (cameraOrientation.isLandscape()) {
size.height to size.width
} else {
size.width to size.height
// For fragmented MP4: DON'T swap dimensions, use camera's native dimensions.
// The C++ VideoPipeline now uses a simple Y-flip matrix (not the display transform).
// This gives us raw sensor frames, and we rely on rotation metadata for playback.
val cameraOrientationDegrees = when (cameraOrientation) {
Orientation.LANDSCAPE_LEFT -> 0 // CCW landscape - works!
Orientation.LANDSCAPE_RIGHT -> 0 // CW landscape - same as CCW (Y-flip handles it)
Orientation.PORTRAIT -> 90 // Portrait typically needs 90° on Android
Orientation.PORTRAIT_UPSIDE_DOWN -> 270
}
Log.i(TAG, "ROTATION_DEBUG FragmentedRecordingManager: cameraOrientation=$cameraOrientation, cameraOrientationDegrees=$cameraOrientationDegrees, inputSize=${size.width}x${size.height}")
// Keep original dimensions - don't swap. Let rotation metadata handle orientation.
val width = size.width
val height = size.height
Log.i(TAG, "ROTATION_DEBUG FragmentedRecordingManager: outputDimensions=${width}x${height} (no swap)")
val format = MediaFormat.createVideoFormat(mimeType, width, height)
val codec = MediaCodec.createEncoderByType(mimeType)
@@ -74,14 +81,14 @@ class FragmentedRecordingManager(
format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, segmentDurationSeconds)
format.setInteger(MediaFormat.KEY_BIT_RATE, bitRate)
Log.d(TAG, "Video Format: $format, camera orientation $cameraOrientationDegrees, recordingOrientation: $recordingOrientationDegrees")
Log.d(TAG, "Video Format: $format, camera orientation $cameraOrientationDegrees")
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
return FragmentedRecordingManager(
codec,
outputDirectory,
recordingOrientationDegrees,
cameraOrientationDegrees,
segmentDurationSeconds * 1_000_000L,
callbacks
)
@@ -317,6 +324,7 @@ class FragmentedRecordingManager(
initData.add(bytes)
}
Log.i(TAG, "ROTATION_DEBUG convertToMedia3Format: orientationDegrees=$orientationDegrees, width=$width, height=$height")
return Format.Builder()
.setSampleMimeType(mimeType)
.setWidth(width)