fix: Take Orientation into account for PreviewView (#2565)

* fix: Take Orientation into account for `PreviewView`

* Log

* Take aspect ratio into account

* Reorganize code a bit

* Set LANDSCAPE_LEFT as default

* chore: Format

package/android/src/main/java/com/mrousavy/camera/core/CameraDeviceDetails.kt

@@ -1,5 +1,6 @@
 package com.mrousavy.camera.core
 
+import android.content.res.Resources
 import android.graphics.ImageFormat
 import android.hardware.camera2.CameraCharacteristics
 import android.hardware.camera2.CameraExtensionCharacteristics
@@ -9,11 +10,14 @@ import android.os.Build
 import android.util.Log
 import android.util.Range
 import android.util.Size
+import android.view.SurfaceHolder
 import com.facebook.react.bridge.Arguments
 import com.facebook.react.bridge.ReadableArray
 import com.facebook.react.bridge.ReadableMap
+import com.mrousavy.camera.extensions.bigger
 import com.mrousavy.camera.extensions.getPhotoSizes
 import com.mrousavy.camera.extensions.getVideoSizes
+import com.mrousavy.camera.extensions.smaller
 import com.mrousavy.camera.extensions.toJSValue
 import com.mrousavy.camera.types.AutoFocusSystem
 import com.mrousavy.camera.types.DeviceType
@@ -29,6 +33,20 @@ import kotlin.math.sqrt
 class CameraDeviceDetails(private val cameraManager: CameraManager, val cameraId: String) {
   companion object {
     private const val TAG = "CameraDeviceDetails"
+
+    fun getMaximumPreviewSize(): Size {
+      // See https://developer.android.com/reference/android/hardware/camera2/params/StreamConfigurationMap
+      // According to the Android Developer documentation, PREVIEW streams can have a resolution
+      // of up to the phone's display's resolution, with a maximum of 1920x1080.
+      val display1080p = Size(1920, 1080)
+      val displaySize = Size(
+        Resources.getSystem().displayMetrics.widthPixels,
+        Resources.getSystem().displayMetrics.heightPixels
+      )
+      val isHighResScreen = displaySize.bigger >= display1080p.bigger || displaySize.smaller >= display1080p.smaller
+
+      return if (isHighResScreen) display1080p else displaySize
+    }
   }
 
   val characteristics by lazy { cameraManager.getCameraCharacteristics(cameraId) }
@@ -50,7 +68,10 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, val cameraId
   val sensorSize by lazy { characteristics.get(CameraCharacteristics.SENSOR_INFO_PHYSICAL_SIZE)!! }
   val activeSize
     get() = characteristics.get(CameraCharacteristics.SENSOR_INFO_ACTIVE_ARRAY_SIZE)!!
-  val sensorOrientation = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0
+  val sensorOrientation by lazy {
+    val degrees = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0
+    return@lazy Orientation.fromRotationDegrees(degrees)
+  }
   val minFocusDistance by lazy { getMinFocusDistanceCm() }
   val name by lazy {
     val info = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) characteristics.get(CameraCharacteristics.INFO_VERSION) else null
@@ -121,6 +142,7 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, val cameraId
 
   // TODO: Also add 10-bit YUV here?
   val videoFormat = ImageFormat.YUV_420_888
+  val photoFormat = ImageFormat.JPEG
 
   // get extensions (HDR, Night Mode, ..)
   private fun getSupportedExtensions(): List<Int> =
@@ -214,13 +236,18 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, val cameraId
     return getFieldOfView(smallestFocalLength)
   }
 
-  private fun getVideoSizes(): List<Size> = characteristics.getVideoSizes(cameraId, videoFormat)
+  fun getVideoSizes(format: Int): List<Size> = characteristics.getVideoSizes(cameraId, format)
-  private fun getPhotoSizes(): List<Size> = characteristics.getPhotoSizes(ImageFormat.JPEG)
+  fun getPhotoSizes(): List<Size> = characteristics.getPhotoSizes(photoFormat)
+  fun getPreviewSizes(): List<Size> {
+    val maximumPreviewSize = getMaximumPreviewSize()
+    return cameraConfig.getOutputSizes(SurfaceHolder::class.java)
+      .filter { it.bigger <= maximumPreviewSize.bigger && it.smaller <= maximumPreviewSize.smaller }
+  }
 
   private fun getFormats(): ReadableArray {
     val array = Arguments.createArray()
 
-    val videoSizes = getVideoSizes()
+    val videoSizes = getVideoSizes(videoFormat)
     val photoSizes = getPhotoSizes()
 
     videoSizes.forEach { videoSize ->
@@ -294,7 +321,7 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, val cameraId
     map.putDouble("minExposure", exposureRange.lower.toDouble())
     map.putDouble("maxExposure", exposureRange.upper.toDouble())
     map.putString("hardwareLevel", hardwareLevel.unionValue)
-    map.putString("sensorOrientation", Orientation.fromRotationDegrees(sensorOrientation).unionValue)
+    map.putString("sensorOrientation", sensorOrientation.unionValue)
     map.putArray("formats", getFormats())
     return map
   }

package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt

@@ -23,9 +23,6 @@ import com.mrousavy.camera.core.outputs.PhotoOutput
 import com.mrousavy.camera.core.outputs.SurfaceOutput
 import com.mrousavy.camera.core.outputs.VideoPipelineOutput
 import com.mrousavy.camera.extensions.closestToOrMax
-import com.mrousavy.camera.extensions.getPhotoSizes
-import com.mrousavy.camera.extensions.getPreviewTargetSize
-import com.mrousavy.camera.extensions.getVideoSizes
 import com.mrousavy.camera.frameprocessor.Frame
 import com.mrousavy.camera.types.Flash
 import com.mrousavy.camera.types.LensFacing
@@ -245,20 +242,20 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
     codeScannerOutput = null
     isRunning = false
 
-    val characteristics = cameraManager.getCameraCharacteristics(cameraId)
+    val deviceDetails = CameraDeviceDetails(cameraManager, cameraId)
     val format = configuration.format
 
     Log.i(TAG, "Creating outputs for Camera #$cameraId...")
 
-    val isSelfie = characteristics.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT
+    val isSelfie = deviceDetails.lensFacing == LensFacing.FRONT
 
     val outputs = mutableListOf<SurfaceOutput>()
 
     // Photo Output
     val photo = configuration.photo as? CameraConfiguration.Output.Enabled<CameraConfiguration.Photo>
     if (photo != null) {
-      val imageFormat = ImageFormat.JPEG
+      val imageFormat = deviceDetails.photoFormat
-      val sizes = characteristics.getPhotoSizes(imageFormat)
+      val sizes = deviceDetails.getPhotoSizes()
       val size = sizes.closestToOrMax(format?.photoSize)
       val maxImages = 10
 
@@ -278,7 +275,7 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
     val video = configuration.video as? CameraConfiguration.Output.Enabled<CameraConfiguration.Video>
     if (video != null) {
       val imageFormat = video.config.pixelFormat.toImageFormat()
-      val sizes = characteristics.getVideoSizes(cameraId, imageFormat)
+      val sizes = deviceDetails.getVideoSizes(imageFormat)
       val size = sizes.closestToOrMax(format?.videoSize)
 
       Log.i(TAG, "Adding ${size.width}x${size.height} Video Output in ${ImageFormatUtils.imageFormatToString(imageFormat)}...")
@@ -301,7 +298,8 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
     if (preview != null) {
       // Compute Preview Size based on chosen video size
       val videoSize = videoOutput?.size ?: format?.videoSize
-      val size = characteristics.getPreviewTargetSize(videoSize)
+      val sizes = deviceDetails.getPreviewSizes()
+      val size = sizes.closestToOrMax(videoSize)
 
       val enableHdr = video?.config?.enableHdr ?: false
 
@@ -314,7 +312,7 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
       )
       outputs.add(output)
       // Size is usually landscape, so we flip it here
-      previewView?.setSurfaceSize(size.width, size.height)
+      previewView?.setSurfaceSize(size.width, size.height, deviceDetails.sensorOrientation)
     }
 
     // CodeScanner Output
@@ -327,7 +325,7 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
       }
 
       val imageFormat = ImageFormat.YUV_420_888
-      val sizes = characteristics.getVideoSizes(cameraId, imageFormat)
+      val sizes = deviceDetails.getVideoSizes(imageFormat)
       val size = sizes.closestToOrMax(Size(1280, 720))
 
       Log.i(TAG, "Adding ${size.width}x${size.height} CodeScanner Output in ${ImageFormatUtils.imageFormatToString(imageFormat)}...")

package/android/src/main/java/com/mrousavy/camera/core/PreviewView.kt

@@ -10,9 +10,9 @@ import android.view.SurfaceHolder
 import android.view.SurfaceView
 import android.widget.FrameLayout
 import com.facebook.react.bridge.UiThreadUtil
-import com.mrousavy.camera.extensions.getMaximumPreviewSize
 import com.mrousavy.camera.extensions.installHierarchyFitter
 import com.mrousavy.camera.extensions.resize
+import com.mrousavy.camera.extensions.rotatedBy
 import com.mrousavy.camera.types.Orientation
 import com.mrousavy.camera.types.ResizeMode
 import kotlin.math.roundToInt
@@ -23,7 +23,7 @@ import kotlinx.coroutines.withContext
 class PreviewView(context: Context, callback: SurfaceHolder.Callback) :
   FrameLayout(context),
   SurfaceHolder.Callback {
-  var size: Size = getMaximumPreviewSize()
+  var size: Size = CameraDeviceDetails.getMaximumPreviewSize()
     private set
   var resizeMode: ResizeMode = ResizeMode.COVER
     set(value) {
@@ -34,6 +34,15 @@ class PreviewView(context: Context, callback: SurfaceHolder.Callback) :
         invalidate()
       }
     }
+  private var inputOrientation: Orientation = Orientation.LANDSCAPE_LEFT
+    set(value) {
+      field = value
+      UiThreadUtil.runOnUiThread {
+        Log.i(TAG, "Camera Input Orientation changed to $value!")
+        requestLayout()
+        invalidate()
+      }
+    }
   private val viewSize: Size
     get() {
       val displayMetrics = context.resources.displayMetrics
@@ -66,25 +75,26 @@ class PreviewView(context: Context, callback: SurfaceHolder.Callback) :
     invalidate()
   }
 
-  suspend fun setSurfaceSize(width: Int, height: Int) {
+  suspend fun setSurfaceSize(width: Int, height: Int, cameraSensorOrientation: Orientation) {
     withContext(Dispatchers.Main) {
+      inputOrientation = cameraSensorOrientation
       surfaceView.holder.resize(width, height)
     }
   }
 
   fun convertLayerPointToCameraCoordinates(point: Point, cameraDeviceDetails: CameraDeviceDetails): Point {
-    val sensorOrientation = Orientation.fromRotationDegrees(cameraDeviceDetails.sensorOrientation)
+    val sensorOrientation = cameraDeviceDetails.sensorOrientation
     val cameraSize = Size(cameraDeviceDetails.activeSize.width(), cameraDeviceDetails.activeSize.height())
     val viewOrientation = Orientation.PORTRAIT
 
-    val rotated = Orientation.rotatePoint(point, viewSize, cameraSize, viewOrientation, sensorOrientation)
+    val rotated = point.rotatedBy(viewSize, cameraSize, viewOrientation, sensorOrientation)
-    Log.i(TAG, "$point -> $sensorOrientation (in $cameraSize -> $viewSize) -> $rotated")
+    Log.i(TAG, "Converted layer point $point to camera point $rotated! ($sensorOrientation, $cameraSize -> $viewSize)")
     return rotated
   }
 
   private fun getSize(contentSize: Size, containerSize: Size, resizeMode: ResizeMode): Size {
     // TODO: Take sensor orientation into account here
-    val contentAspectRatio = contentSize.height.toDouble() / contentSize.width
+    val contentAspectRatio = contentSize.width.toDouble() / contentSize.height
     val containerAspectRatio = containerSize.width.toDouble() / containerSize.height
 
     val widthOverHeight = when (resizeMode) {
@@ -108,9 +118,10 @@ class PreviewView(context: Context, callback: SurfaceHolder.Callback) :
     super.onMeasure(widthMeasureSpec, heightMeasureSpec)
 
     val viewSize = Size(MeasureSpec.getSize(widthMeasureSpec), MeasureSpec.getSize(heightMeasureSpec))
-    val fittedSize = getSize(size, viewSize, resizeMode)
+    val surfaceSize = size.rotatedBy(inputOrientation)
+    val fittedSize = getSize(surfaceSize, viewSize, resizeMode)
 
-    Log.i(TAG, "PreviewView is $viewSize, rendering $size content. Resizing to: $fittedSize ($resizeMode)")
+    Log.i(TAG, "PreviewView is $viewSize, rendering $surfaceSize content ($inputOrientation). Resizing to: $fittedSize ($resizeMode)")
     setMeasuredDimension(fittedSize.width, fittedSize.height)
   }
 

package/android/src/main/java/com/mrousavy/camera/extensions/CameraCharacteristics+getPreviewSize.kt

@@ -1,29 +0,0 @@
-package com.mrousavy.camera.extensions
-
-import android.content.res.Resources
-import android.hardware.camera2.CameraCharacteristics
-import android.util.Size
-import android.view.SurfaceHolder
-
-fun getMaximumPreviewSize(): Size {
-  // See https://developer.android.com/reference/android/hardware/camera2/params/StreamConfigurationMap
-  // According to the Android Developer documentation, PREVIEW streams can have a resolution
-  // of up to the phone's display's resolution, with a maximum of 1920x1080.
-  val display1080p = Size(1920, 1080)
-  val displaySize = Size(
-    Resources.getSystem().displayMetrics.widthPixels,
-    Resources.getSystem().displayMetrics.heightPixels
-  )
-  val isHighResScreen = displaySize.bigger >= display1080p.bigger || displaySize.smaller >= display1080p.smaller
-
-  return if (isHighResScreen) display1080p else displaySize
-}
-
-fun CameraCharacteristics.getPreviewTargetSize(targetSize: Size?): Size {
-  val config = this.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!!
-  val maximumPreviewSize = getMaximumPreviewSize()
-  val outputSizes = config.getOutputSizes(SurfaceHolder::class.java)
-    .filter { it.bigger <= maximumPreviewSize.bigger && it.smaller <= maximumPreviewSize.smaller }
-
-  return outputSizes.closestToOrMax(targetSize)
-}

package/android/src/main/java/com/mrousavy/camera/extensions/Point+rotatedBy.kt

@@ -0,0 +1,25 @@
+package com.mrousavy.camera.extensions
+
+import android.graphics.Point
+import android.graphics.PointF
+import android.util.Log
+import android.util.Size
+import com.mrousavy.camera.types.Orientation
+
+fun Point.rotatedBy(fromSize: Size, toSize: Size, fromOrientation: Orientation, toOrientation: Orientation): Point {
+  val differenceDegrees = (fromOrientation.toDegrees() + toOrientation.toDegrees()) % 360
+  val difference = Orientation.fromRotationDegrees(differenceDegrees)
+  val normalizedPoint = PointF(this.x / fromSize.width.toFloat(), this.y / fromSize.height.toFloat())
+
+  val rotatedNormalizedPoint = when (difference) {
+    Orientation.PORTRAIT -> normalizedPoint
+    Orientation.PORTRAIT_UPSIDE_DOWN -> PointF(1 - normalizedPoint.x, 1 - normalizedPoint.y)
+    Orientation.LANDSCAPE_LEFT -> PointF(normalizedPoint.y, 1 - normalizedPoint.x)
+    Orientation.LANDSCAPE_RIGHT -> PointF(1 - normalizedPoint.y, normalizedPoint.x)
+  }
+
+  val rotatedX = rotatedNormalizedPoint.x * toSize.width
+  val rotatedY = rotatedNormalizedPoint.y * toSize.height
+  Log.i("ROTATE", "$this -> $normalizedPoint -> $difference -> $rotatedX, $rotatedY")
+  return Point(rotatedX.toInt(), rotatedY.toInt())
+}

package/android/src/main/java/com/mrousavy/camera/types/Orientation.kt

@@ -1,9 +1,5 @@
 package com.mrousavy.camera.types
 
-import android.graphics.Point
-import android.graphics.PointF
-import android.util.Log
-import android.util.Size
 import com.mrousavy.camera.core.CameraDeviceDetails
 
 enum class Orientation(override val unionValue: String) : JSUnionValue {
@@ -30,7 +26,7 @@ enum class Orientation(override val unionValue: String) : JSUnionValue {
     }
 
     // Rotate sensor rotation by target rotation
-    val newRotationDegrees = (deviceDetails.sensorOrientation + rotationDegrees + 360) % 360
+    val newRotationDegrees = (deviceDetails.sensorOrientation.toDegrees() + rotationDegrees + 360) % 360
 
     return fromRotationDegrees(newRotationDegrees)
   }
@@ -52,29 +48,5 @@ enum class Orientation(override val unionValue: String) : JSUnionValue {
         in 225..315 -> LANDSCAPE_RIGHT
         else -> PORTRAIT
       }
-
-    fun rotatePoint(
-      point: Point,
-      fromSize: Size,
-      toSize: Size,
-      fromOrientation: Orientation,
-      toOrientation: Orientation
-    ): Point {
-      val differenceDegrees = (fromOrientation.toDegrees() + toOrientation.toDegrees()) % 360
-      val difference = Orientation.fromRotationDegrees(differenceDegrees)
-      val normalizedPoint = PointF(point.x / fromSize.width.toFloat(), point.y / fromSize.height.toFloat())
-
-      val rotatedNormalizedPoint = when (difference) {
-        PORTRAIT -> normalizedPoint
-        PORTRAIT_UPSIDE_DOWN -> PointF(1 - normalizedPoint.x, 1 - normalizedPoint.y)
-        LANDSCAPE_LEFT -> PointF(normalizedPoint.y, 1 - normalizedPoint.x)
-        LANDSCAPE_RIGHT -> PointF(1 - normalizedPoint.y, normalizedPoint.x)
-      }
-
-      val rotatedX = rotatedNormalizedPoint.x * toSize.width
-      val rotatedY = rotatedNormalizedPoint.y * toSize.height
-      Log.i("ROTATE", "$point -> $normalizedPoint -> $difference -> $rotatedX, $rotatedY")
-      return Point(rotatedX.toInt(), rotatedY.toInt())
-    }
   }
 }