fix: Take Orientation into account for PreviewView (#2565)

* fix: Take Orientation into account for `PreviewView` * Log * Take aspect ratio into account * Reorganize code a bit * Set LANDSCAPE_LEFT as default * chore: Format
2024-02-15 13:30:14 +01:00
parent 5df5ca9adf
commit 83c0184796
6 changed files with 87 additions and 83 deletions
--- a/package/android/src/main/java/com/mrousavy/camera/core/CameraDeviceDetails.kt
+++ b/package/android/src/main/java/com/mrousavy/camera/core/CameraDeviceDetails.kt
@@ -1,5 +1,6 @@
 package com.mrousavy.camera.core

+import android.content.res.Resources
 import android.graphics.ImageFormat
 import android.hardware.camera2.CameraCharacteristics
 import android.hardware.camera2.CameraExtensionCharacteristics
@@ -9,11 +10,14 @@ import android.os.Build
 import android.util.Log
 import android.util.Range
 import android.util.Size
+import android.view.SurfaceHolder
 import com.facebook.react.bridge.Arguments
 import com.facebook.react.bridge.ReadableArray
 import com.facebook.react.bridge.ReadableMap
+import com.mrousavy.camera.extensions.bigger
 import com.mrousavy.camera.extensions.getPhotoSizes
 import com.mrousavy.camera.extensions.getVideoSizes
+import com.mrousavy.camera.extensions.smaller
 import com.mrousavy.camera.extensions.toJSValue
 import com.mrousavy.camera.types.AutoFocusSystem
 import com.mrousavy.camera.types.DeviceType
@@ -29,6 +33,20 @@ import kotlin.math.sqrt
 class CameraDeviceDetails(private val cameraManager: CameraManager, val cameraId: String) {
  companion object {
    private const val TAG = "CameraDeviceDetails"
+
+    fun getMaximumPreviewSize(): Size {
+      // See https://developer.android.com/reference/android/hardware/camera2/params/StreamConfigurationMap
+      // According to the Android Developer documentation, PREVIEW streams can have a resolution
+      // of up to the phone's display's resolution, with a maximum of 1920x1080.
+      val display1080p = Size(1920, 1080)
+      val displaySize = Size(
+        Resources.getSystem().displayMetrics.widthPixels,
+        Resources.getSystem().displayMetrics.heightPixels
+      )
+      val isHighResScreen = displaySize.bigger >= display1080p.bigger || displaySize.smaller >= display1080p.smaller
+
+      return if (isHighResScreen) display1080p else displaySize
+    }
  }

  val characteristics by lazy { cameraManager.getCameraCharacteristics(cameraId) }
@@ -50,7 +68,10 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, val cameraId
  val sensorSize by lazy { characteristics.get(CameraCharacteristics.SENSOR_INFO_PHYSICAL_SIZE)!! }
  val activeSize
    get() = characteristics.get(CameraCharacteristics.SENSOR_INFO_ACTIVE_ARRAY_SIZE)!!
-  val sensorOrientation = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0
+  val sensorOrientation by lazy {
+    val degrees = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0
+    return@lazy Orientation.fromRotationDegrees(degrees)
+  }
  val minFocusDistance by lazy { getMinFocusDistanceCm() }
  val name by lazy {
    val info = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) characteristics.get(CameraCharacteristics.INFO_VERSION) else null
@@ -121,6 +142,7 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, val cameraId

  // TODO: Also add 10-bit YUV here?
  val videoFormat = ImageFormat.YUV_420_888
+  val photoFormat = ImageFormat.JPEG

  // get extensions (HDR, Night Mode, ..)
  private fun getSupportedExtensions(): List<Int> =
@@ -214,13 +236,18 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, val cameraId
    return getFieldOfView(smallestFocalLength)
  }

-  private fun getVideoSizes(): List<Size> = characteristics.getVideoSizes(cameraId, videoFormat)
-  private fun getPhotoSizes(): List<Size> = characteristics.getPhotoSizes(ImageFormat.JPEG)
+  fun getVideoSizes(format: Int): List<Size> = characteristics.getVideoSizes(cameraId, format)
+  fun getPhotoSizes(): List<Size> = characteristics.getPhotoSizes(photoFormat)
+  fun getPreviewSizes(): List<Size> {
+    val maximumPreviewSize = getMaximumPreviewSize()
+    return cameraConfig.getOutputSizes(SurfaceHolder::class.java)
+      .filter { it.bigger <= maximumPreviewSize.bigger && it.smaller <= maximumPreviewSize.smaller }
+  }

  private fun getFormats(): ReadableArray {
    val array = Arguments.createArray()

-    val videoSizes = getVideoSizes()
+    val videoSizes = getVideoSizes(videoFormat)
    val photoSizes = getPhotoSizes()

    videoSizes.forEach { videoSize ->
@@ -294,7 +321,7 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, val cameraId
    map.putDouble("minExposure", exposureRange.lower.toDouble())
    map.putDouble("maxExposure", exposureRange.upper.toDouble())
    map.putString("hardwareLevel", hardwareLevel.unionValue)
-    map.putString("sensorOrientation", Orientation.fromRotationDegrees(sensorOrientation).unionValue)
+    map.putString("sensorOrientation", sensorOrientation.unionValue)
    map.putArray("formats", getFormats())
    return map
  }
--- a/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt
+++ b/package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt
@@ -23,9 +23,6 @@ import com.mrousavy.camera.core.outputs.PhotoOutput
 import com.mrousavy.camera.core.outputs.SurfaceOutput
 import com.mrousavy.camera.core.outputs.VideoPipelineOutput
 import com.mrousavy.camera.extensions.closestToOrMax
-import com.mrousavy.camera.extensions.getPhotoSizes
-import com.mrousavy.camera.extensions.getPreviewTargetSize
-import com.mrousavy.camera.extensions.getVideoSizes
 import com.mrousavy.camera.frameprocessor.Frame
 import com.mrousavy.camera.types.Flash
 import com.mrousavy.camera.types.LensFacing
@@ -245,20 +242,20 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
    codeScannerOutput = null
    isRunning = false

-    val characteristics = cameraManager.getCameraCharacteristics(cameraId)
+    val deviceDetails = CameraDeviceDetails(cameraManager, cameraId)
    val format = configuration.format

    Log.i(TAG, "Creating outputs for Camera #$cameraId...")

-    val isSelfie = characteristics.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT
+    val isSelfie = deviceDetails.lensFacing == LensFacing.FRONT

    val outputs = mutableListOf<SurfaceOutput>()

    // Photo Output
    val photo = configuration.photo as? CameraConfiguration.Output.Enabled<CameraConfiguration.Photo>
    if (photo != null) {
-      val imageFormat = ImageFormat.JPEG
-      val sizes = characteristics.getPhotoSizes(imageFormat)
+      val imageFormat = deviceDetails.photoFormat
+      val sizes = deviceDetails.getPhotoSizes()
      val size = sizes.closestToOrMax(format?.photoSize)
      val maxImages = 10

@@ -278,7 +275,7 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
    val video = configuration.video as? CameraConfiguration.Output.Enabled<CameraConfiguration.Video>
    if (video != null) {
      val imageFormat = video.config.pixelFormat.toImageFormat()
-      val sizes = characteristics.getVideoSizes(cameraId, imageFormat)
+      val sizes = deviceDetails.getVideoSizes(imageFormat)
      val size = sizes.closestToOrMax(format?.videoSize)

      Log.i(TAG, "Adding ${size.width}x${size.height} Video Output in ${ImageFormatUtils.imageFormatToString(imageFormat)}...")
@@ -301,7 +298,8 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
    if (preview != null) {
      // Compute Preview Size based on chosen video size
      val videoSize = videoOutput?.size ?: format?.videoSize
-      val size = characteristics.getPreviewTargetSize(videoSize)
+      val sizes = deviceDetails.getPreviewSizes()
+      val size = sizes.closestToOrMax(videoSize)

      val enableHdr = video?.config?.enableHdr ?: false

@@ -314,7 +312,7 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
      )
      outputs.add(output)
      // Size is usually landscape, so we flip it here
-      previewView?.setSurfaceSize(size.width, size.height)
+      previewView?.setSurfaceSize(size.width, size.height, deviceDetails.sensorOrientation)
    }

    // CodeScanner Output
@@ -327,7 +325,7 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
      }

      val imageFormat = ImageFormat.YUV_420_888
-      val sizes = characteristics.getVideoSizes(cameraId, imageFormat)
+      val sizes = deviceDetails.getVideoSizes(imageFormat)
      val size = sizes.closestToOrMax(Size(1280, 720))

      Log.i(TAG, "Adding ${size.width}x${size.height} CodeScanner Output in ${ImageFormatUtils.imageFormatToString(imageFormat)}...")
--- a/package/android/src/main/java/com/mrousavy/camera/core/PreviewView.kt
+++ b/package/android/src/main/java/com/mrousavy/camera/core/PreviewView.kt
@@ -10,9 +10,9 @@ import android.view.SurfaceHolder
 import android.view.SurfaceView
 import android.widget.FrameLayout
 import com.facebook.react.bridge.UiThreadUtil
-import com.mrousavy.camera.extensions.getMaximumPreviewSize
 import com.mrousavy.camera.extensions.installHierarchyFitter
 import com.mrousavy.camera.extensions.resize
+import com.mrousavy.camera.extensions.rotatedBy
 import com.mrousavy.camera.types.Orientation
 import com.mrousavy.camera.types.ResizeMode
 import kotlin.math.roundToInt
@@ -23,7 +23,7 @@ import kotlinx.coroutines.withContext
 class PreviewView(context: Context, callback: SurfaceHolder.Callback) :
  FrameLayout(context),
  SurfaceHolder.Callback {
-  var size: Size = getMaximumPreviewSize()
+  var size: Size = CameraDeviceDetails.getMaximumPreviewSize()
    private set
  var resizeMode: ResizeMode = ResizeMode.COVER
    set(value) {
@@ -34,6 +34,15 @@ class PreviewView(context: Context, callback: SurfaceHolder.Callback) :
        invalidate()
      }
    }
+  private var inputOrientation: Orientation = Orientation.LANDSCAPE_LEFT
+    set(value) {
+      field = value
+      UiThreadUtil.runOnUiThread {
+        Log.i(TAG, "Camera Input Orientation changed to $value!")
+        requestLayout()
+        invalidate()
+      }
+    }
  private val viewSize: Size
    get() {
      val displayMetrics = context.resources.displayMetrics
@@ -66,25 +75,26 @@ class PreviewView(context: Context, callback: SurfaceHolder.Callback) :
    invalidate()
  }

-  suspend fun setSurfaceSize(width: Int, height: Int) {
+  suspend fun setSurfaceSize(width: Int, height: Int, cameraSensorOrientation: Orientation) {
    withContext(Dispatchers.Main) {
+      inputOrientation = cameraSensorOrientation
      surfaceView.holder.resize(width, height)
    }
  }

  fun convertLayerPointToCameraCoordinates(point: Point, cameraDeviceDetails: CameraDeviceDetails): Point {
-    val sensorOrientation = Orientation.fromRotationDegrees(cameraDeviceDetails.sensorOrientation)
+    val sensorOrientation = cameraDeviceDetails.sensorOrientation
    val cameraSize = Size(cameraDeviceDetails.activeSize.width(), cameraDeviceDetails.activeSize.height())
    val viewOrientation = Orientation.PORTRAIT

-    val rotated = Orientation.rotatePoint(point, viewSize, cameraSize, viewOrientation, sensorOrientation)
-    Log.i(TAG, "$point -> $sensorOrientation (in $cameraSize -> $viewSize) -> $rotated")
+    val rotated = point.rotatedBy(viewSize, cameraSize, viewOrientation, sensorOrientation)
+    Log.i(TAG, "Converted layer point $point to camera point $rotated! ($sensorOrientation, $cameraSize -> $viewSize)")
    return rotated
  }

  private fun getSize(contentSize: Size, containerSize: Size, resizeMode: ResizeMode): Size {
    // TODO: Take sensor orientation into account here
-    val contentAspectRatio = contentSize.height.toDouble() / contentSize.width
+    val contentAspectRatio = contentSize.width.toDouble() / contentSize.height
    val containerAspectRatio = containerSize.width.toDouble() / containerSize.height

    val widthOverHeight = when (resizeMode) {
@@ -108,9 +118,10 @@ class PreviewView(context: Context, callback: SurfaceHolder.Callback) :
    super.onMeasure(widthMeasureSpec, heightMeasureSpec)

    val viewSize = Size(MeasureSpec.getSize(widthMeasureSpec), MeasureSpec.getSize(heightMeasureSpec))
-    val fittedSize = getSize(size, viewSize, resizeMode)
+    val surfaceSize = size.rotatedBy(inputOrientation)
+    val fittedSize = getSize(surfaceSize, viewSize, resizeMode)

-    Log.i(TAG, "PreviewView is $viewSize, rendering $size content. Resizing to: $fittedSize ($resizeMode)")
+    Log.i(TAG, "PreviewView is $viewSize, rendering $surfaceSize content ($inputOrientation). Resizing to: $fittedSize ($resizeMode)")
    setMeasuredDimension(fittedSize.width, fittedSize.height)
  }

--- a/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCharacteristics+getPreviewSize.kt
+++ b/package/android/src/main/java/com/mrousavy/camera/extensions/CameraCharacteristics+getPreviewSize.kt
@@ -1,29 +0,0 @@
-package com.mrousavy.camera.extensions
-
-import android.content.res.Resources
-import android.hardware.camera2.CameraCharacteristics
-import android.util.Size
-import android.view.SurfaceHolder
-
-fun getMaximumPreviewSize(): Size {
-  // See https://developer.android.com/reference/android/hardware/camera2/params/StreamConfigurationMap
-  // According to the Android Developer documentation, PREVIEW streams can have a resolution
-  // of up to the phone's display's resolution, with a maximum of 1920x1080.
-  val display1080p = Size(1920, 1080)
-  val displaySize = Size(
-    Resources.getSystem().displayMetrics.widthPixels,
-    Resources.getSystem().displayMetrics.heightPixels
-  )
-  val isHighResScreen = displaySize.bigger >= display1080p.bigger || displaySize.smaller >= display1080p.smaller
-
-  return if (isHighResScreen) display1080p else displaySize
-}
-
-fun CameraCharacteristics.getPreviewTargetSize(targetSize: Size?): Size {
-  val config = this.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!!
-  val maximumPreviewSize = getMaximumPreviewSize()
-  val outputSizes = config.getOutputSizes(SurfaceHolder::class.java)
-    .filter { it.bigger <= maximumPreviewSize.bigger && it.smaller <= maximumPreviewSize.smaller }
-
-  return outputSizes.closestToOrMax(targetSize)
-}
--- a/package/android/src/main/java/com/mrousavy/camera/extensions/Point+rotatedBy.kt
+++ b/package/android/src/main/java/com/mrousavy/camera/extensions/Point+rotatedBy.kt
@@ -0,0 +1,25 @@
+package com.mrousavy.camera.extensions
+
+import android.graphics.Point
+import android.graphics.PointF
+import android.util.Log
+import android.util.Size
+import com.mrousavy.camera.types.Orientation
+
+fun Point.rotatedBy(fromSize: Size, toSize: Size, fromOrientation: Orientation, toOrientation: Orientation): Point {
+  val differenceDegrees = (fromOrientation.toDegrees() + toOrientation.toDegrees()) % 360
+  val difference = Orientation.fromRotationDegrees(differenceDegrees)
+  val normalizedPoint = PointF(this.x / fromSize.width.toFloat(), this.y / fromSize.height.toFloat())
+
+  val rotatedNormalizedPoint = when (difference) {
+    Orientation.PORTRAIT -> normalizedPoint
+    Orientation.PORTRAIT_UPSIDE_DOWN -> PointF(1 - normalizedPoint.x, 1 - normalizedPoint.y)
+    Orientation.LANDSCAPE_LEFT -> PointF(normalizedPoint.y, 1 - normalizedPoint.x)
+    Orientation.LANDSCAPE_RIGHT -> PointF(1 - normalizedPoint.y, normalizedPoint.x)
+  }
+
+  val rotatedX = rotatedNormalizedPoint.x * toSize.width
+  val rotatedY = rotatedNormalizedPoint.y * toSize.height
+  Log.i("ROTATE", "$this -> $normalizedPoint -> $difference -> $rotatedX, $rotatedY")
+  return Point(rotatedX.toInt(), rotatedY.toInt())
+}
--- a/package/android/src/main/java/com/mrousavy/camera/types/Orientation.kt
+++ b/package/android/src/main/java/com/mrousavy/camera/types/Orientation.kt
@@ -1,9 +1,5 @@
 package com.mrousavy.camera.types

-import android.graphics.Point
-import android.graphics.PointF
-import android.util.Log
-import android.util.Size
 import com.mrousavy.camera.core.CameraDeviceDetails

 enum class Orientation(override val unionValue: String) : JSUnionValue {
@@ -30,7 +26,7 @@ enum class Orientation(override val unionValue: String) : JSUnionValue {
    }

    // Rotate sensor rotation by target rotation
-    val newRotationDegrees = (deviceDetails.sensorOrientation + rotationDegrees + 360) % 360
+    val newRotationDegrees = (deviceDetails.sensorOrientation.toDegrees() + rotationDegrees + 360) % 360

    return fromRotationDegrees(newRotationDegrees)
  }
@@ -52,29 +48,5 @@ enum class Orientation(override val unionValue: String) : JSUnionValue {
        in 225..315 -> LANDSCAPE_RIGHT
        else -> PORTRAIT
      }
-
-    fun rotatePoint(
-      point: Point,
-      fromSize: Size,
-      toSize: Size,
-      fromOrientation: Orientation,
-      toOrientation: Orientation
-    ): Point {
-      val differenceDegrees = (fromOrientation.toDegrees() + toOrientation.toDegrees()) % 360
-      val difference = Orientation.fromRotationDegrees(differenceDegrees)
-      val normalizedPoint = PointF(point.x / fromSize.width.toFloat(), point.y / fromSize.height.toFloat())
-
-      val rotatedNormalizedPoint = when (difference) {
-        PORTRAIT -> normalizedPoint
-        PORTRAIT_UPSIDE_DOWN -> PointF(1 - normalizedPoint.x, 1 - normalizedPoint.y)
-        LANDSCAPE_LEFT -> PointF(normalizedPoint.y, 1 - normalizedPoint.x)
-        LANDSCAPE_RIGHT -> PointF(1 - normalizedPoint.y, normalizedPoint.x)
-      }
-
-      val rotatedX = rotatedNormalizedPoint.x * toSize.width
-      val rotatedY = rotatedNormalizedPoint.y * toSize.height
-      Log.i("ROTATE", "$point -> $normalizedPoint -> $difference -> $rotatedX, $rotatedY")
-      return Point(rotatedX.toInt(), rotatedY.toInt())
-    }
  }
 }