From fde8a47a2d5c18dd6c6887cb28409e367cd35ef2 Mon Sep 17 00:00:00 2001 From: Loewy Date: Thu, 26 Mar 2026 15:54:20 -0700 Subject: [PATCH] rotate frag recording output instead of relying on muxer metadata --- .../android/src/main/cpp/OpenGLRenderer.cpp | 11 ++- package/android/src/main/cpp/OpenGLRenderer.h | 5 +- .../src/main/cpp/PassThroughShader.cpp | 74 ++++++++++++++++++- .../android/src/main/cpp/PassThroughShader.h | 2 +- .../android/src/main/cpp/VideoPipeline.cpp | 5 +- package/android/src/main/cpp/VideoPipeline.h | 2 +- .../camera/core/FragmentedRecordingManager.kt | 15 +--- .../mrousavy/camera/core/RecordingSession.kt | 11 +++ .../com/mrousavy/camera/core/VideoPipeline.kt | 4 +- 9 files changed, 103 insertions(+), 26 deletions(-) diff --git a/package/android/src/main/cpp/OpenGLRenderer.cpp b/package/android/src/main/cpp/OpenGLRenderer.cpp index e9d6bd3..7a588fe 100644 --- a/package/android/src/main/cpp/OpenGLRenderer.cpp +++ b/package/android/src/main/cpp/OpenGLRenderer.cpp @@ -17,15 +17,18 @@ namespace vision { -std::unique_ptr OpenGLRenderer::CreateWithWindowSurface(std::shared_ptr context, ANativeWindow* surface) { - return std::unique_ptr(new OpenGLRenderer(std::move(context), surface)); +std::unique_ptr OpenGLRenderer::CreateWithWindowSurface(std::shared_ptr context, + ANativeWindow* surface, + int rotationDegrees) { + return std::unique_ptr(new OpenGLRenderer(std::move(context), surface, rotationDegrees)); } -OpenGLRenderer::OpenGLRenderer(std::shared_ptr context, ANativeWindow* surface) { +OpenGLRenderer::OpenGLRenderer(std::shared_ptr context, ANativeWindow* surface, int rotationDegrees) { _context = std::move(context); _outputSurface = surface; _width = ANativeWindow_getWidth(surface); _height = ANativeWindow_getHeight(surface); + _rotationDegrees = rotationDegrees; } OpenGLRenderer::~OpenGLRenderer() { @@ -66,7 +69,7 @@ void OpenGLRenderer::renderTextureToSurface(const OpenGLTexture& texture, float* glTexParameteri(texture.target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); // 4. Draw it using the pass-through shader which also applies transforms - _passThroughShader.draw(texture, transformMatrix); + _passThroughShader.draw(texture, transformMatrix, _rotationDegrees); // 5. Swap buffers to pass it to the window surface eglSwapBuffers(_context->display, _surface); diff --git a/package/android/src/main/cpp/OpenGLRenderer.h b/package/android/src/main/cpp/OpenGLRenderer.h index b2b96cb..8837b0c 100644 --- a/package/android/src/main/cpp/OpenGLRenderer.h +++ b/package/android/src/main/cpp/OpenGLRenderer.h @@ -24,7 +24,7 @@ public: * Note: The `surface` is considered moved, and the OpenGL context will release it when it is * being deleted. */ - static std::unique_ptr CreateWithWindowSurface(std::shared_ptr context, ANativeWindow* surface); + static std::unique_ptr CreateWithWindowSurface(std::shared_ptr context, ANativeWindow* surface, int rotationDegrees); /** * Destroy the OpenGL Context. This needs to be called on the same thread that `use()` was called. */ @@ -43,10 +43,11 @@ public: void destroy(); private: - explicit OpenGLRenderer(std::shared_ptr context, ANativeWindow* surface); + explicit OpenGLRenderer(std::shared_ptr context, ANativeWindow* surface, int rotationDegrees); private: int _width = 0, _height = 0; + int _rotationDegrees = 0; std::shared_ptr _context; ANativeWindow* _outputSurface; EGLSurface _surface = EGL_NO_SURFACE; diff --git a/package/android/src/main/cpp/PassThroughShader.cpp b/package/android/src/main/cpp/PassThroughShader.cpp index ad51fdf..6b065cf 100644 --- a/package/android/src/main/cpp/PassThroughShader.cpp +++ b/package/android/src/main/cpp/PassThroughShader.cpp @@ -7,11 +7,76 @@ #include #include #include +#include #include #include namespace vision { +namespace { + +void setIdentity(float* matrix) { + for (int i = 0; i < 16; i++) { + matrix[i] = 0.0f; + } + matrix[0] = 1.0f; + matrix[5] = 1.0f; + matrix[10] = 1.0f; + matrix[15] = 1.0f; +} + +void multiply4x4(const float* left, const float* right, float* out) { + for (int column = 0; column < 4; column++) { + for (int row = 0; row < 4; row++) { + float sum = 0.0f; + for (int k = 0; k < 4; k++) { + sum += left[k * 4 + row] * right[column * 4 + k]; + } + out[column * 4 + row] = sum; + } + } +} + +void makeTranslation(float tx, float ty, float* matrix) { + setIdentity(matrix); + matrix[12] = tx; + matrix[13] = ty; +} + +void makeRotation(float degrees, float* matrix) { + setIdentity(matrix); + const float radians = degrees * static_cast(M_PI) / 180.0f; + const float cosine = std::cos(radians); + const float sine = std::sin(radians); + + matrix[0] = cosine; + matrix[1] = sine; + matrix[4] = -sine; + matrix[5] = cosine; +} + +void makeCenteredRotation(int rotationDegrees, float* matrix) { + const int normalized = ((rotationDegrees % 360) + 360) % 360; + if (normalized == 0) { + setIdentity(matrix); + return; + } + + float translateToOrigin[16]; + float rotation[16]; + float translateBack[16]; + float temp[16]; + + makeTranslation(-0.5f, -0.5f, translateToOrigin); + makeRotation(static_cast(normalized), rotation); + makeTranslation(0.5f, 0.5f, translateBack); + + multiply4x4(rotation, translateToOrigin, temp); + multiply4x4(translateBack, temp, matrix); +} + +} // namespace + PassThroughShader::~PassThroughShader() { if (_programId != NO_SHADER) { glDeleteProgram(_programId); @@ -24,7 +89,7 @@ PassThroughShader::~PassThroughShader() { } } -void PassThroughShader::draw(const OpenGLTexture& texture, float* transformMatrix) { +void PassThroughShader::draw(const OpenGLTexture& texture, float* transformMatrix, int rotationDegrees) { // 1. Set up Shader Program if (_programId == NO_SHADER) { _programId = createProgram(); @@ -57,7 +122,12 @@ void PassThroughShader::draw(const OpenGLTexture& texture, float* transformMatri glVertexAttribPointer(_vertexParameters.aTexCoord, 2, GL_FLOAT, GL_FALSE, sizeof(Vertex), reinterpret_cast(offsetof(Vertex, texCoord))); - glUniformMatrix4fv(_vertexParameters.uTransformMatrix, 1, GL_FALSE, transformMatrix); + float outputRotationMatrix[16]; + float combinedTransformMatrix[16]; + makeCenteredRotation(rotationDegrees, outputRotationMatrix); + multiply4x4(transformMatrix, outputRotationMatrix, combinedTransformMatrix); + + glUniformMatrix4fv(_vertexParameters.uTransformMatrix, 1, GL_FALSE, combinedTransformMatrix); // 4. Pass texture to fragment shader glActiveTexture(GL_TEXTURE0); diff --git a/package/android/src/main/cpp/PassThroughShader.h b/package/android/src/main/cpp/PassThroughShader.h index b199b3d..08d43b3 100644 --- a/package/android/src/main/cpp/PassThroughShader.h +++ b/package/android/src/main/cpp/PassThroughShader.h @@ -29,7 +29,7 @@ public: * Draw the texture using this shader. * Note: At the moment, only EXTERNAL textures are supported by the Shader. */ - void draw(const OpenGLTexture& texture, float* transformMatrix); + void draw(const OpenGLTexture& texture, float* transformMatrix, int rotationDegrees); private: // Loading diff --git a/package/android/src/main/cpp/VideoPipeline.cpp b/package/android/src/main/cpp/VideoPipeline.cpp index 8160daa..dd7141e 100644 --- a/package/android/src/main/cpp/VideoPipeline.cpp +++ b/package/android/src/main/cpp/VideoPipeline.cpp @@ -47,13 +47,13 @@ void VideoPipeline::removeRecordingSessionOutputSurface() { _recordingSessionOutput = nullptr; } -void VideoPipeline::setRecordingSessionOutputSurface(jobject surface) { +void VideoPipeline::setRecordingSessionOutputSurface(jobject surface, int rotationDegrees) { // 1. Delete existing output surface removeRecordingSessionOutputSurface(); // 2. Set new output surface if it is not null ANativeWindow* window = ANativeWindow_fromSurface(jni::Environment::current(), surface); - _recordingSessionOutput = OpenGLRenderer::CreateWithWindowSurface(_context, window); + _recordingSessionOutput = OpenGLRenderer::CreateWithWindowSurface(_context, window, rotationDegrees); } int VideoPipeline::getInputTextureId() { @@ -78,7 +78,6 @@ void VideoPipeline::onFrame(jni::alias_ref transformMatrixPara OpenGLTexture& texture = _inputTexture.value(); if (_recordingSessionOutput) { - __android_log_print(ANDROID_LOG_INFO, TAG, "Rendering to RecordingSession.."); _recordingSessionOutput->renderTextureToSurface(texture, transformMatrix); } } diff --git a/package/android/src/main/cpp/VideoPipeline.h b/package/android/src/main/cpp/VideoPipeline.h index 67f0725..b4d3bdf 100644 --- a/package/android/src/main/cpp/VideoPipeline.h +++ b/package/android/src/main/cpp/VideoPipeline.h @@ -31,7 +31,7 @@ public: int getInputTextureId(); // <- MediaRecorder output - void setRecordingSessionOutputSurface(jobject surface); + void setRecordingSessionOutputSurface(jobject surface, int rotationDegrees); void removeRecordingSessionOutputSurface(); // Frame callbacks diff --git a/package/android/src/main/java/com/mrousavy/camera/core/FragmentedRecordingManager.kt b/package/android/src/main/java/com/mrousavy/camera/core/FragmentedRecordingManager.kt index fc44441..f9713e7 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/FragmentedRecordingManager.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/FragmentedRecordingManager.kt @@ -39,18 +39,11 @@ class FragmentedRecordingManager( segmentDurationSeconds: Int = DEFAULT_SEGMENT_DURATION_SECONDS ): FragmentedRecordingManager { val mimeType = options.videoCodec.toMimeType() - // Use cameraOrientation (from WindowManager) for rotation metadata - // The options.orientation from JavaScript is unreliable on Android when rotating between landscape modes - val orientationDegrees = cameraOrientation.toDegrees() + val cameraOrientationDegrees = cameraOrientation.toDegrees() + val orientationDegrees = 0 + val (width, height) = size.width to size.height - // Swap dimensions based on camera orientation, same as ChunkedRecordingManager - val (width, height) = if (cameraOrientation.isLandscape()) { - size.height to size.width - } else { - size.width to size.height - } - - Log.d(TAG, "Recording: ${width}x${height}, orientation=$orientationDegrees°") + Log.d(TAG, "Recording: ${width}x${height}, orientation=$orientationDegrees° (cameraOrientation=$cameraOrientationDegrees°)") val format = MediaFormat.createVideoFormat(mimeType, width, height) val codec = MediaCodec.createEncoderByType(mimeType) diff --git a/package/android/src/main/java/com/mrousavy/camera/core/RecordingSession.kt b/package/android/src/main/java/com/mrousavy/camera/core/RecordingSession.kt index 96047f9..a6bf377 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/RecordingSession.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/RecordingSession.kt @@ -43,6 +43,17 @@ class RecordingSession( data class Video(val path: String, val durationMs: Long, val size: Size) + val outputRotationDegrees: Int = + if (options.streamSegmentType == StreamSegmentType.FRAGMENTED_MP4) { + when (cameraOrientation.toDegrees()) { + 90 -> 270 + 270 -> 90 + else -> cameraOrientation.toDegrees() + } + } else { + 0 + } + // Normalize path - expo-file-system passes file:// URIs but File expects raw paths // Handle both file:// and file:/ variants private val outputPath: File = File(filePath.replace(Regex("^file:/+"), "/")) diff --git a/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt b/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt index 395d396..fe13e8f 100644 --- a/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt +++ b/package/android/src/main/java/com/mrousavy/camera/core/VideoPipeline.kt @@ -186,7 +186,7 @@ class VideoPipeline( if (recordingSession != null) { // Configure OpenGL pipeline to stream Frames into the Recording Session's surface Log.i(TAG, "Setting ${recordingSession.size} RecordingSession Output...") - setRecordingSessionOutputSurface(recordingSession.surface) + setRecordingSessionOutputSurface(recordingSession.surface, recordingSession.outputRotationDegrees) this.recordingSession = recordingSession } else { // Configure OpenGL pipeline to stop streaming Frames into the Recording Session's surface @@ -250,7 +250,7 @@ class VideoPipeline( private external fun getInputTextureId(): Int private external fun onBeforeFrame() private external fun onFrame(transformMatrix: FloatArray) - private external fun setRecordingSessionOutputSurface(surface: Any) + private external fun setRecordingSessionOutputSurface(surface: Any, rotationDegrees: Int) private external fun removeRecordingSessionOutputSurface() private external fun initHybrid(width: Int, height: Int): HybridData }