Compare commits

..

2 Commits

Author SHA1 Message Date
6d99d1f17f catch negative values set by out of preview bound press 2026-01-21 12:57:46 -08:00
2f7b511ce8 catch focus timeout error on android 2026-01-21 12:01:56 -08:00
12 changed files with 70 additions and 191 deletions

View File

@@ -17,18 +17,15 @@
namespace vision {
std::unique_ptr<OpenGLRenderer> OpenGLRenderer::CreateWithWindowSurface(std::shared_ptr<OpenGLContext> context,
ANativeWindow* surface,
int rotationDegrees) {
return std::unique_ptr<OpenGLRenderer>(new OpenGLRenderer(std::move(context), surface, rotationDegrees));
std::unique_ptr<OpenGLRenderer> OpenGLRenderer::CreateWithWindowSurface(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface) {
return std::unique_ptr<OpenGLRenderer>(new OpenGLRenderer(std::move(context), surface));
}
OpenGLRenderer::OpenGLRenderer(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface, int rotationDegrees) {
OpenGLRenderer::OpenGLRenderer(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface) {
_context = std::move(context);
_outputSurface = surface;
_width = ANativeWindow_getWidth(surface);
_height = ANativeWindow_getHeight(surface);
_rotationDegrees = rotationDegrees;
}
OpenGLRenderer::~OpenGLRenderer() {
@@ -69,7 +66,7 @@ void OpenGLRenderer::renderTextureToSurface(const OpenGLTexture& texture, float*
glTexParameteri(texture.target, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
// 4. Draw it using the pass-through shader which also applies transforms
_passThroughShader.draw(texture, transformMatrix, _rotationDegrees);
_passThroughShader.draw(texture, transformMatrix);
// 5. Swap buffers to pass it to the window surface
eglSwapBuffers(_context->display, _surface);

View File

@@ -24,7 +24,7 @@ public:
* Note: The `surface` is considered moved, and the OpenGL context will release it when it is
* being deleted.
*/
static std::unique_ptr<OpenGLRenderer> CreateWithWindowSurface(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface, int rotationDegrees);
static std::unique_ptr<OpenGLRenderer> CreateWithWindowSurface(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface);
/**
* Destroy the OpenGL Context. This needs to be called on the same thread that `use()` was called.
*/
@@ -43,11 +43,10 @@ public:
void destroy();
private:
explicit OpenGLRenderer(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface, int rotationDegrees);
explicit OpenGLRenderer(std::shared_ptr<OpenGLContext> context, ANativeWindow* surface);
private:
int _width = 0, _height = 0;
int _rotationDegrees = 0;
std::shared_ptr<OpenGLContext> _context;
ANativeWindow* _outputSurface;
EGLSurface _surface = EGL_NO_SURFACE;

View File

@@ -7,76 +7,11 @@
#include <EGL/egl.h>
#include <GLES2/gl2.h>
#include <GLES2/gl2ext.h>
#include <cmath>
#include <memory>
#include <string>
namespace vision {
namespace {
void setIdentity(float* matrix) {
for (int i = 0; i < 16; i++) {
matrix[i] = 0.0f;
}
matrix[0] = 1.0f;
matrix[5] = 1.0f;
matrix[10] = 1.0f;
matrix[15] = 1.0f;
}
void multiply4x4(const float* left, const float* right, float* out) {
for (int column = 0; column < 4; column++) {
for (int row = 0; row < 4; row++) {
float sum = 0.0f;
for (int k = 0; k < 4; k++) {
sum += left[k * 4 + row] * right[column * 4 + k];
}
out[column * 4 + row] = sum;
}
}
}
void makeTranslation(float tx, float ty, float* matrix) {
setIdentity(matrix);
matrix[12] = tx;
matrix[13] = ty;
}
void makeRotation(float degrees, float* matrix) {
setIdentity(matrix);
const float radians = degrees * static_cast<float>(M_PI) / 180.0f;
const float cosine = std::cos(radians);
const float sine = std::sin(radians);
matrix[0] = cosine;
matrix[1] = sine;
matrix[4] = -sine;
matrix[5] = cosine;
}
void makeCenteredRotation(int rotationDegrees, float* matrix) {
const int normalized = ((rotationDegrees % 360) + 360) % 360;
if (normalized == 0) {
setIdentity(matrix);
return;
}
float translateToOrigin[16];
float rotation[16];
float translateBack[16];
float temp[16];
makeTranslation(-0.5f, -0.5f, translateToOrigin);
makeRotation(static_cast<float>(normalized), rotation);
makeTranslation(0.5f, 0.5f, translateBack);
multiply4x4(rotation, translateToOrigin, temp);
multiply4x4(translateBack, temp, matrix);
}
} // namespace
PassThroughShader::~PassThroughShader() {
if (_programId != NO_SHADER) {
glDeleteProgram(_programId);
@@ -89,7 +24,7 @@ PassThroughShader::~PassThroughShader() {
}
}
void PassThroughShader::draw(const OpenGLTexture& texture, float* transformMatrix, int rotationDegrees) {
void PassThroughShader::draw(const OpenGLTexture& texture, float* transformMatrix) {
// 1. Set up Shader Program
if (_programId == NO_SHADER) {
_programId = createProgram();
@@ -122,12 +57,7 @@ void PassThroughShader::draw(const OpenGLTexture& texture, float* transformMatri
glVertexAttribPointer(_vertexParameters.aTexCoord, 2, GL_FLOAT, GL_FALSE, sizeof(Vertex),
reinterpret_cast<void*>(offsetof(Vertex, texCoord)));
float outputRotationMatrix[16];
float combinedTransformMatrix[16];
makeCenteredRotation(rotationDegrees, outputRotationMatrix);
multiply4x4(transformMatrix, outputRotationMatrix, combinedTransformMatrix);
glUniformMatrix4fv(_vertexParameters.uTransformMatrix, 1, GL_FALSE, combinedTransformMatrix);
glUniformMatrix4fv(_vertexParameters.uTransformMatrix, 1, GL_FALSE, transformMatrix);
// 4. Pass texture to fragment shader
glActiveTexture(GL_TEXTURE0);

View File

@@ -29,7 +29,7 @@ public:
* Draw the texture using this shader.
* Note: At the moment, only EXTERNAL textures are supported by the Shader.
*/
void draw(const OpenGLTexture& texture, float* transformMatrix, int rotationDegrees);
void draw(const OpenGLTexture& texture, float* transformMatrix);
private:
// Loading

View File

@@ -47,13 +47,13 @@ void VideoPipeline::removeRecordingSessionOutputSurface() {
_recordingSessionOutput = nullptr;
}
void VideoPipeline::setRecordingSessionOutputSurface(jobject surface, int rotationDegrees) {
void VideoPipeline::setRecordingSessionOutputSurface(jobject surface) {
// 1. Delete existing output surface
removeRecordingSessionOutputSurface();
// 2. Set new output surface if it is not null
ANativeWindow* window = ANativeWindow_fromSurface(jni::Environment::current(), surface);
_recordingSessionOutput = OpenGLRenderer::CreateWithWindowSurface(_context, window, rotationDegrees);
_recordingSessionOutput = OpenGLRenderer::CreateWithWindowSurface(_context, window);
}
int VideoPipeline::getInputTextureId() {
@@ -78,6 +78,7 @@ void VideoPipeline::onFrame(jni::alias_ref<jni::JArrayFloat> transformMatrixPara
OpenGLTexture& texture = _inputTexture.value();
if (_recordingSessionOutput) {
__android_log_print(ANDROID_LOG_INFO, TAG, "Rendering to RecordingSession..");
_recordingSessionOutput->renderTextureToSurface(texture, transformMatrix);
}
}

View File

@@ -31,7 +31,7 @@ public:
int getInputTextureId();
// <- MediaRecorder output
void setRecordingSessionOutputSurface(jobject surface, int rotationDegrees);
void setRecordingSessionOutputSurface(jobject surface);
void removeRecordingSessionOutputSurface();
// Frame callbacks

View File

@@ -39,11 +39,18 @@ class FragmentedRecordingManager(
segmentDurationSeconds: Int = DEFAULT_SEGMENT_DURATION_SECONDS
): FragmentedRecordingManager {
val mimeType = options.videoCodec.toMimeType()
val cameraOrientationDegrees = cameraOrientation.toDegrees()
val orientationDegrees = 0
val (width, height) = size.width to size.height
// Use cameraOrientation (from WindowManager) for rotation metadata
// The options.orientation from JavaScript is unreliable on Android when rotating between landscape modes
val orientationDegrees = cameraOrientation.toDegrees()
Log.d(TAG, "Recording: ${width}x${height}, orientation=$orientationDegrees° (cameraOrientation=$cameraOrientationDegrees°)")
// Swap dimensions based on camera orientation, same as ChunkedRecordingManager
val (width, height) = if (cameraOrientation.isLandscape()) {
size.height to size.width
} else {
size.width to size.height
}
Log.d(TAG, "Recording: ${width}x${height}, orientation=$orientationDegrees°")
val format = MediaFormat.createVideoFormat(mimeType, width, height)
val codec = MediaCodec.createEncoderByType(mimeType)

View File

@@ -753,32 +753,17 @@ class HlsMuxer(
dos.writeShort(-1) // pre-defined
output.write(buildAvcCBox(sps, pps))
output.write(buildPaspBox())
return wrapBox("avc1", output.toByteArray())
}
/**
* Builds pixel aspect ratio box to explicitly declare square pixels (1:1).
* This helps players correctly interpret video dimensions without SAR scaling.
*/
private fun buildPaspBox(): ByteArray {
val output = ByteArrayOutputStream()
val dos = DataOutputStream(output)
dos.writeInt(1) // hSpacing (horizontal)
dos.writeInt(1) // vSpacing (vertical)
return wrapBox("pasp", output.toByteArray())
}
private fun buildAvcCBox(sps: ByteArray, pps: ByteArray): ByteArray {
val output = ByteArrayOutputStream()
val dos = DataOutputStream(output)
// SPS NAL unit format: [NAL header, profile_idc, constraint_flags, level_idc, ...]
// Skip byte 0 (NAL header, typically 0x67) to get the actual profile data
val profileIdc = if (sps.size > 1) sps[1].toInt() and 0xFF else 0x42
val profileCompat = if (sps.size > 2) sps[2].toInt() and 0xFF else 0x00
val levelIdc = if (sps.size > 3) sps[3].toInt() and 0xFF else 0x1F
val profileIdc = if (sps.isNotEmpty()) sps[0].toInt() and 0xFF else 0x42
val profileCompat = if (sps.size > 1) sps[1].toInt() and 0xFF else 0x00
val levelIdc = if (sps.size > 2) sps[2].toInt() and 0xFF else 0x1F
dos.writeByte(1) // configuration version
dos.writeByte(profileIdc) // AVC profile

View File

@@ -243,10 +243,6 @@ class PersistentCameraCaptureSession(private val cameraManager: CameraManager, p
} catch (e: CaptureTimedOutError) {
// Focus timed out - this is non-fatal, just log and continue
Log.w(TAG, "Focus timed out at point $point, continuing without focus lock")
} catch (e: IllegalStateException) {
Log.w(TAG, "Focus failed, camera device was already closed: ${e.message}")
} catch (e: CameraAccessException) {
Log.w(TAG, "Focus failed, camera not accessible: ${e.message}")
}
}
focusJob?.join()
@@ -263,14 +259,8 @@ class PersistentCameraCaptureSession(private val cameraManager: CameraManager, p
return@launch
}
Log.i(TAG, "Resetting focus to auto-focus...")
try {
repeatingRequest.createCaptureRequest(device, deviceDetails, outputs).also { request ->
session.setRepeatingRequest(request.build(), null, null)
}
} catch (e: IllegalStateException) {
Log.w(TAG, "Failed to reset focus, camera device was already closed: ${e.message}")
} catch (e: CameraAccessException) {
Log.w(TAG, "Failed to reset focus, camera not accessible: ${e.message}")
repeatingRequest.createCaptureRequest(device, deviceDetails, outputs).also { request ->
session.setRepeatingRequest(request.build(), null, null)
}
}
}

View File

@@ -5,7 +5,7 @@ import android.content.Context
import android.content.res.Configuration
import android.graphics.Point
import android.os.Handler
import android.os.HandlerThread
import android.os.Looper
import android.util.Log
import android.util.Size
import android.view.PixelCopy
@@ -25,72 +25,58 @@ import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.suspendCancellableCoroutine
import kotlinx.coroutines.withContext
import android.graphics.Bitmap
import android.graphics.Matrix
fun rotateBitmap90CounterClockwise(source: Bitmap): Bitmap {
val width = source.width
val height = source.height
// Create a new Bitmap with swapped width and height
val rotatedBitmap = Bitmap.createBitmap(height, width, source.config ?: Bitmap.Config.ARGB_8888)
for (y in 0 until height) {
for (x in 0 until width) {
// Set the pixel in the new position
rotatedBitmap.setPixel(y, width - 1 - x, source.getPixel(x, y))
}
}
return rotatedBitmap
}
fun Bitmap.transformBitmap(orientation: Orientation): Bitmap {
return when (orientation) {
Orientation.PORTRAIT -> this // No transformation needed
Orientation.LANDSCAPE_LEFT -> {
val srcWidth = width
val srcHeight = height
val sourcePixels = IntArray(srcWidth * srcHeight)
getPixels(sourcePixels, 0, srcWidth, 0, 0, srcWidth, srcHeight)
val dstWidth = srcHeight
val dstHeight = srcWidth
val destinationPixels = IntArray(dstWidth * dstHeight)
for (y in 0 until srcHeight) {
for (x in 0 until srcWidth) {
val dstX = y
val dstY = srcWidth - 1 - x
destinationPixels[dstY * dstWidth + dstX] = sourcePixels[y * srcWidth + x]
// Transpose (swap width and height)
val transposedBitmap = Bitmap.createBitmap(height, width, config ?: Bitmap.Config.ARGB_8888)
for (y in 0 until height) {
for (x in 0 until width) {
transposedBitmap.setPixel(y, width - 1 - x, getPixel(x, y))
}
}
val transformedBitmap = Bitmap.createBitmap(dstWidth, dstHeight, config ?: Bitmap.Config.ARGB_8888)
transformedBitmap.setPixels(destinationPixels, 0, dstWidth, 0, 0, dstWidth, dstHeight)
transformedBitmap
transposedBitmap
}
Orientation.PORTRAIT_UPSIDE_DOWN -> {
val srcWidth = width
val srcHeight = height
val sourcePixels = IntArray(srcWidth * srcHeight)
getPixels(sourcePixels, 0, srcWidth, 0, 0, srcWidth, srcHeight)
val dstWidth = srcWidth
val dstHeight = srcHeight
val destinationPixels = IntArray(dstWidth * dstHeight)
for (y in 0 until srcHeight) {
for (x in 0 until srcWidth) {
val dstX = srcWidth - 1 - x
val dstY = srcHeight - 1 - y
destinationPixels[dstY * dstWidth + dstX] = sourcePixels[y * srcWidth + x]
// Invert vertically and horizontally (180-degree rotation)
val invertedBitmap = Bitmap.createBitmap(width, height, config ?: Bitmap.Config.ARGB_8888)
for (y in 0 until height) {
for (x in 0 until width) {
invertedBitmap.setPixel(width - 1 - x, height - 1 - y, getPixel(x, y))
}
}
val transformedBitmap = Bitmap.createBitmap(dstWidth, dstHeight, config ?: Bitmap.Config.ARGB_8888)
transformedBitmap.setPixels(destinationPixels, 0, dstWidth, 0, 0, dstWidth, dstHeight)
transformedBitmap
invertedBitmap
}
Orientation.LANDSCAPE_RIGHT -> {
val srcWidth = width
val srcHeight = height
val sourcePixels = IntArray(srcWidth * srcHeight)
getPixels(sourcePixels, 0, srcWidth, 0, 0, srcWidth, srcHeight)
val dstWidth = srcHeight
val dstHeight = srcWidth
val destinationPixels = IntArray(dstWidth * dstHeight)
for (y in 0 until srcHeight) {
for (x in 0 until srcWidth) {
val dstX = srcHeight - 1 - y
val dstY = x
destinationPixels[dstY * dstWidth + dstX] = sourcePixels[y * srcWidth + x]
// Transpose (swap width and height) and invert vertically
val transposedBitmap = Bitmap.createBitmap(height, width, config ?: Bitmap.Config.ARGB_8888)
for (y in 0 until height) {
for (x in 0 until width) {
transposedBitmap.setPixel(height - 1 - y, x, getPixel(x, y))
}
}
val transformedBitmap = Bitmap.createBitmap(dstWidth, dstHeight, config ?: Bitmap.Config.ARGB_8888)
transformedBitmap.setPixels(destinationPixels, 0, dstWidth, 0, 0, dstWidth, dstHeight)
transformedBitmap
transposedBitmap
}
}
}
@@ -200,7 +186,7 @@ class PreviewView(context: Context, callback: SurfaceHolder.Callback) :
)
}
},
pixelCopyHandler
Handler(Looper.getMainLooper())
)
}
}
@@ -270,10 +256,5 @@ class PreviewView(context: Context, callback: SurfaceHolder.Callback) :
companion object {
private const val TAG = "PreviewView"
private val pixelCopyHandler: Handler by lazy {
val handlerThread = HandlerThread("VisionCamera.PixelCopy")
handlerThread.start()
Handler(handlerThread.looper)
}
}
}

View File

@@ -43,17 +43,6 @@ class RecordingSession(
data class Video(val path: String, val durationMs: Long, val size: Size)
val outputRotationDegrees: Int =
if (options.streamSegmentType == StreamSegmentType.FRAGMENTED_MP4) {
when (cameraOrientation.toDegrees()) {
90 -> 270
270 -> 90
else -> cameraOrientation.toDegrees()
}
} else {
0
}
// Normalize path - expo-file-system passes file:// URIs but File expects raw paths
// Handle both file:// and file:/ variants
private val outputPath: File = File(filePath.replace(Regex("^file:/+"), "/"))

View File

@@ -186,7 +186,7 @@ class VideoPipeline(
if (recordingSession != null) {
// Configure OpenGL pipeline to stream Frames into the Recording Session's surface
Log.i(TAG, "Setting ${recordingSession.size} RecordingSession Output...")
setRecordingSessionOutputSurface(recordingSession.surface, recordingSession.outputRotationDegrees)
setRecordingSessionOutputSurface(recordingSession.surface)
this.recordingSession = recordingSession
} else {
// Configure OpenGL pipeline to stop streaming Frames into the Recording Session's surface
@@ -250,7 +250,7 @@ class VideoPipeline(
private external fun getInputTextureId(): Int
private external fun onBeforeFrame()
private external fun onFrame(transformMatrix: FloatArray)
private external fun setRecordingSessionOutputSurface(surface: Any, rotationDegrees: Int)
private external fun setRecordingSessionOutputSurface(surface: Any)
private external fun removeRecordingSessionOutputSurface()
private external fun initHybrid(width: Int, height: Int): HybridData
}