fix: Take Orientation into account for PreviewView (#2565)

* fix: Take Orientation into account for `PreviewView`

* Log

* Take aspect ratio into account

* Reorganize code a bit

* Set LANDSCAPE_LEFT as default

* chore: Format
This commit is contained in:
Marc Rousavy 2024-02-15 13:30:14 +01:00 committed by GitHub
parent 5df5ca9adf
commit 83c0184796
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 87 additions and 83 deletions

View File

@ -1,5 +1,6 @@
package com.mrousavy.camera.core package com.mrousavy.camera.core
import android.content.res.Resources
import android.graphics.ImageFormat import android.graphics.ImageFormat
import android.hardware.camera2.CameraCharacteristics import android.hardware.camera2.CameraCharacteristics
import android.hardware.camera2.CameraExtensionCharacteristics import android.hardware.camera2.CameraExtensionCharacteristics
@ -9,11 +10,14 @@ import android.os.Build
import android.util.Log import android.util.Log
import android.util.Range import android.util.Range
import android.util.Size import android.util.Size
import android.view.SurfaceHolder
import com.facebook.react.bridge.Arguments import com.facebook.react.bridge.Arguments
import com.facebook.react.bridge.ReadableArray import com.facebook.react.bridge.ReadableArray
import com.facebook.react.bridge.ReadableMap import com.facebook.react.bridge.ReadableMap
import com.mrousavy.camera.extensions.bigger
import com.mrousavy.camera.extensions.getPhotoSizes import com.mrousavy.camera.extensions.getPhotoSizes
import com.mrousavy.camera.extensions.getVideoSizes import com.mrousavy.camera.extensions.getVideoSizes
import com.mrousavy.camera.extensions.smaller
import com.mrousavy.camera.extensions.toJSValue import com.mrousavy.camera.extensions.toJSValue
import com.mrousavy.camera.types.AutoFocusSystem import com.mrousavy.camera.types.AutoFocusSystem
import com.mrousavy.camera.types.DeviceType import com.mrousavy.camera.types.DeviceType
@ -29,6 +33,20 @@ import kotlin.math.sqrt
class CameraDeviceDetails(private val cameraManager: CameraManager, val cameraId: String) { class CameraDeviceDetails(private val cameraManager: CameraManager, val cameraId: String) {
companion object { companion object {
private const val TAG = "CameraDeviceDetails" private const val TAG = "CameraDeviceDetails"
fun getMaximumPreviewSize(): Size {
// See https://developer.android.com/reference/android/hardware/camera2/params/StreamConfigurationMap
// According to the Android Developer documentation, PREVIEW streams can have a resolution
// of up to the phone's display's resolution, with a maximum of 1920x1080.
val display1080p = Size(1920, 1080)
val displaySize = Size(
Resources.getSystem().displayMetrics.widthPixels,
Resources.getSystem().displayMetrics.heightPixels
)
val isHighResScreen = displaySize.bigger >= display1080p.bigger || displaySize.smaller >= display1080p.smaller
return if (isHighResScreen) display1080p else displaySize
}
} }
val characteristics by lazy { cameraManager.getCameraCharacteristics(cameraId) } val characteristics by lazy { cameraManager.getCameraCharacteristics(cameraId) }
@ -50,7 +68,10 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, val cameraId
val sensorSize by lazy { characteristics.get(CameraCharacteristics.SENSOR_INFO_PHYSICAL_SIZE)!! } val sensorSize by lazy { characteristics.get(CameraCharacteristics.SENSOR_INFO_PHYSICAL_SIZE)!! }
val activeSize val activeSize
get() = characteristics.get(CameraCharacteristics.SENSOR_INFO_ACTIVE_ARRAY_SIZE)!! get() = characteristics.get(CameraCharacteristics.SENSOR_INFO_ACTIVE_ARRAY_SIZE)!!
val sensorOrientation = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0 val sensorOrientation by lazy {
val degrees = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION) ?: 0
return@lazy Orientation.fromRotationDegrees(degrees)
}
val minFocusDistance by lazy { getMinFocusDistanceCm() } val minFocusDistance by lazy { getMinFocusDistanceCm() }
val name by lazy { val name by lazy {
val info = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) characteristics.get(CameraCharacteristics.INFO_VERSION) else null val info = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) characteristics.get(CameraCharacteristics.INFO_VERSION) else null
@ -121,6 +142,7 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, val cameraId
// TODO: Also add 10-bit YUV here? // TODO: Also add 10-bit YUV here?
val videoFormat = ImageFormat.YUV_420_888 val videoFormat = ImageFormat.YUV_420_888
val photoFormat = ImageFormat.JPEG
// get extensions (HDR, Night Mode, ..) // get extensions (HDR, Night Mode, ..)
private fun getSupportedExtensions(): List<Int> = private fun getSupportedExtensions(): List<Int> =
@ -214,13 +236,18 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, val cameraId
return getFieldOfView(smallestFocalLength) return getFieldOfView(smallestFocalLength)
} }
private fun getVideoSizes(): List<Size> = characteristics.getVideoSizes(cameraId, videoFormat) fun getVideoSizes(format: Int): List<Size> = characteristics.getVideoSizes(cameraId, format)
private fun getPhotoSizes(): List<Size> = characteristics.getPhotoSizes(ImageFormat.JPEG) fun getPhotoSizes(): List<Size> = characteristics.getPhotoSizes(photoFormat)
fun getPreviewSizes(): List<Size> {
val maximumPreviewSize = getMaximumPreviewSize()
return cameraConfig.getOutputSizes(SurfaceHolder::class.java)
.filter { it.bigger <= maximumPreviewSize.bigger && it.smaller <= maximumPreviewSize.smaller }
}
private fun getFormats(): ReadableArray { private fun getFormats(): ReadableArray {
val array = Arguments.createArray() val array = Arguments.createArray()
val videoSizes = getVideoSizes() val videoSizes = getVideoSizes(videoFormat)
val photoSizes = getPhotoSizes() val photoSizes = getPhotoSizes()
videoSizes.forEach { videoSize -> videoSizes.forEach { videoSize ->
@ -294,7 +321,7 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, val cameraId
map.putDouble("minExposure", exposureRange.lower.toDouble()) map.putDouble("minExposure", exposureRange.lower.toDouble())
map.putDouble("maxExposure", exposureRange.upper.toDouble()) map.putDouble("maxExposure", exposureRange.upper.toDouble())
map.putString("hardwareLevel", hardwareLevel.unionValue) map.putString("hardwareLevel", hardwareLevel.unionValue)
map.putString("sensorOrientation", Orientation.fromRotationDegrees(sensorOrientation).unionValue) map.putString("sensorOrientation", sensorOrientation.unionValue)
map.putArray("formats", getFormats()) map.putArray("formats", getFormats())
return map return map
} }

View File

@ -23,9 +23,6 @@ import com.mrousavy.camera.core.outputs.PhotoOutput
import com.mrousavy.camera.core.outputs.SurfaceOutput import com.mrousavy.camera.core.outputs.SurfaceOutput
import com.mrousavy.camera.core.outputs.VideoPipelineOutput import com.mrousavy.camera.core.outputs.VideoPipelineOutput
import com.mrousavy.camera.extensions.closestToOrMax import com.mrousavy.camera.extensions.closestToOrMax
import com.mrousavy.camera.extensions.getPhotoSizes
import com.mrousavy.camera.extensions.getPreviewTargetSize
import com.mrousavy.camera.extensions.getVideoSizes
import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.Frame
import com.mrousavy.camera.types.Flash import com.mrousavy.camera.types.Flash
import com.mrousavy.camera.types.LensFacing import com.mrousavy.camera.types.LensFacing
@ -245,20 +242,20 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
codeScannerOutput = null codeScannerOutput = null
isRunning = false isRunning = false
val characteristics = cameraManager.getCameraCharacteristics(cameraId) val deviceDetails = CameraDeviceDetails(cameraManager, cameraId)
val format = configuration.format val format = configuration.format
Log.i(TAG, "Creating outputs for Camera #$cameraId...") Log.i(TAG, "Creating outputs for Camera #$cameraId...")
val isSelfie = characteristics.get(CameraCharacteristics.LENS_FACING) == CameraCharacteristics.LENS_FACING_FRONT val isSelfie = deviceDetails.lensFacing == LensFacing.FRONT
val outputs = mutableListOf<SurfaceOutput>() val outputs = mutableListOf<SurfaceOutput>()
// Photo Output // Photo Output
val photo = configuration.photo as? CameraConfiguration.Output.Enabled<CameraConfiguration.Photo> val photo = configuration.photo as? CameraConfiguration.Output.Enabled<CameraConfiguration.Photo>
if (photo != null) { if (photo != null) {
val imageFormat = ImageFormat.JPEG val imageFormat = deviceDetails.photoFormat
val sizes = characteristics.getPhotoSizes(imageFormat) val sizes = deviceDetails.getPhotoSizes()
val size = sizes.closestToOrMax(format?.photoSize) val size = sizes.closestToOrMax(format?.photoSize)
val maxImages = 10 val maxImages = 10
@ -278,7 +275,7 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
val video = configuration.video as? CameraConfiguration.Output.Enabled<CameraConfiguration.Video> val video = configuration.video as? CameraConfiguration.Output.Enabled<CameraConfiguration.Video>
if (video != null) { if (video != null) {
val imageFormat = video.config.pixelFormat.toImageFormat() val imageFormat = video.config.pixelFormat.toImageFormat()
val sizes = characteristics.getVideoSizes(cameraId, imageFormat) val sizes = deviceDetails.getVideoSizes(imageFormat)
val size = sizes.closestToOrMax(format?.videoSize) val size = sizes.closestToOrMax(format?.videoSize)
Log.i(TAG, "Adding ${size.width}x${size.height} Video Output in ${ImageFormatUtils.imageFormatToString(imageFormat)}...") Log.i(TAG, "Adding ${size.width}x${size.height} Video Output in ${ImageFormatUtils.imageFormatToString(imageFormat)}...")
@ -301,7 +298,8 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
if (preview != null) { if (preview != null) {
// Compute Preview Size based on chosen video size // Compute Preview Size based on chosen video size
val videoSize = videoOutput?.size ?: format?.videoSize val videoSize = videoOutput?.size ?: format?.videoSize
val size = characteristics.getPreviewTargetSize(videoSize) val sizes = deviceDetails.getPreviewSizes()
val size = sizes.closestToOrMax(videoSize)
val enableHdr = video?.config?.enableHdr ?: false val enableHdr = video?.config?.enableHdr ?: false
@ -314,7 +312,7 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
) )
outputs.add(output) outputs.add(output)
// Size is usually landscape, so we flip it here // Size is usually landscape, so we flip it here
previewView?.setSurfaceSize(size.width, size.height) previewView?.setSurfaceSize(size.width, size.height, deviceDetails.sensorOrientation)
} }
// CodeScanner Output // CodeScanner Output
@ -327,7 +325,7 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
} }
val imageFormat = ImageFormat.YUV_420_888 val imageFormat = ImageFormat.YUV_420_888
val sizes = characteristics.getVideoSizes(cameraId, imageFormat) val sizes = deviceDetails.getVideoSizes(imageFormat)
val size = sizes.closestToOrMax(Size(1280, 720)) val size = sizes.closestToOrMax(Size(1280, 720))
Log.i(TAG, "Adding ${size.width}x${size.height} CodeScanner Output in ${ImageFormatUtils.imageFormatToString(imageFormat)}...") Log.i(TAG, "Adding ${size.width}x${size.height} CodeScanner Output in ${ImageFormatUtils.imageFormatToString(imageFormat)}...")

View File

@ -10,9 +10,9 @@ import android.view.SurfaceHolder
import android.view.SurfaceView import android.view.SurfaceView
import android.widget.FrameLayout import android.widget.FrameLayout
import com.facebook.react.bridge.UiThreadUtil import com.facebook.react.bridge.UiThreadUtil
import com.mrousavy.camera.extensions.getMaximumPreviewSize
import com.mrousavy.camera.extensions.installHierarchyFitter import com.mrousavy.camera.extensions.installHierarchyFitter
import com.mrousavy.camera.extensions.resize import com.mrousavy.camera.extensions.resize
import com.mrousavy.camera.extensions.rotatedBy
import com.mrousavy.camera.types.Orientation import com.mrousavy.camera.types.Orientation
import com.mrousavy.camera.types.ResizeMode import com.mrousavy.camera.types.ResizeMode
import kotlin.math.roundToInt import kotlin.math.roundToInt
@ -23,7 +23,7 @@ import kotlinx.coroutines.withContext
class PreviewView(context: Context, callback: SurfaceHolder.Callback) : class PreviewView(context: Context, callback: SurfaceHolder.Callback) :
FrameLayout(context), FrameLayout(context),
SurfaceHolder.Callback { SurfaceHolder.Callback {
var size: Size = getMaximumPreviewSize() var size: Size = CameraDeviceDetails.getMaximumPreviewSize()
private set private set
var resizeMode: ResizeMode = ResizeMode.COVER var resizeMode: ResizeMode = ResizeMode.COVER
set(value) { set(value) {
@ -34,6 +34,15 @@ class PreviewView(context: Context, callback: SurfaceHolder.Callback) :
invalidate() invalidate()
} }
} }
private var inputOrientation: Orientation = Orientation.LANDSCAPE_LEFT
set(value) {
field = value
UiThreadUtil.runOnUiThread {
Log.i(TAG, "Camera Input Orientation changed to $value!")
requestLayout()
invalidate()
}
}
private val viewSize: Size private val viewSize: Size
get() { get() {
val displayMetrics = context.resources.displayMetrics val displayMetrics = context.resources.displayMetrics
@ -66,25 +75,26 @@ class PreviewView(context: Context, callback: SurfaceHolder.Callback) :
invalidate() invalidate()
} }
suspend fun setSurfaceSize(width: Int, height: Int) { suspend fun setSurfaceSize(width: Int, height: Int, cameraSensorOrientation: Orientation) {
withContext(Dispatchers.Main) { withContext(Dispatchers.Main) {
inputOrientation = cameraSensorOrientation
surfaceView.holder.resize(width, height) surfaceView.holder.resize(width, height)
} }
} }
fun convertLayerPointToCameraCoordinates(point: Point, cameraDeviceDetails: CameraDeviceDetails): Point { fun convertLayerPointToCameraCoordinates(point: Point, cameraDeviceDetails: CameraDeviceDetails): Point {
val sensorOrientation = Orientation.fromRotationDegrees(cameraDeviceDetails.sensorOrientation) val sensorOrientation = cameraDeviceDetails.sensorOrientation
val cameraSize = Size(cameraDeviceDetails.activeSize.width(), cameraDeviceDetails.activeSize.height()) val cameraSize = Size(cameraDeviceDetails.activeSize.width(), cameraDeviceDetails.activeSize.height())
val viewOrientation = Orientation.PORTRAIT val viewOrientation = Orientation.PORTRAIT
val rotated = Orientation.rotatePoint(point, viewSize, cameraSize, viewOrientation, sensorOrientation) val rotated = point.rotatedBy(viewSize, cameraSize, viewOrientation, sensorOrientation)
Log.i(TAG, "$point -> $sensorOrientation (in $cameraSize -> $viewSize) -> $rotated") Log.i(TAG, "Converted layer point $point to camera point $rotated! ($sensorOrientation, $cameraSize -> $viewSize)")
return rotated return rotated
} }
private fun getSize(contentSize: Size, containerSize: Size, resizeMode: ResizeMode): Size { private fun getSize(contentSize: Size, containerSize: Size, resizeMode: ResizeMode): Size {
// TODO: Take sensor orientation into account here // TODO: Take sensor orientation into account here
val contentAspectRatio = contentSize.height.toDouble() / contentSize.width val contentAspectRatio = contentSize.width.toDouble() / contentSize.height
val containerAspectRatio = containerSize.width.toDouble() / containerSize.height val containerAspectRatio = containerSize.width.toDouble() / containerSize.height
val widthOverHeight = when (resizeMode) { val widthOverHeight = when (resizeMode) {
@ -108,9 +118,10 @@ class PreviewView(context: Context, callback: SurfaceHolder.Callback) :
super.onMeasure(widthMeasureSpec, heightMeasureSpec) super.onMeasure(widthMeasureSpec, heightMeasureSpec)
val viewSize = Size(MeasureSpec.getSize(widthMeasureSpec), MeasureSpec.getSize(heightMeasureSpec)) val viewSize = Size(MeasureSpec.getSize(widthMeasureSpec), MeasureSpec.getSize(heightMeasureSpec))
val fittedSize = getSize(size, viewSize, resizeMode) val surfaceSize = size.rotatedBy(inputOrientation)
val fittedSize = getSize(surfaceSize, viewSize, resizeMode)
Log.i(TAG, "PreviewView is $viewSize, rendering $size content. Resizing to: $fittedSize ($resizeMode)") Log.i(TAG, "PreviewView is $viewSize, rendering $surfaceSize content ($inputOrientation). Resizing to: $fittedSize ($resizeMode)")
setMeasuredDimension(fittedSize.width, fittedSize.height) setMeasuredDimension(fittedSize.width, fittedSize.height)
} }

View File

@ -1,29 +0,0 @@
package com.mrousavy.camera.extensions
import android.content.res.Resources
import android.hardware.camera2.CameraCharacteristics
import android.util.Size
import android.view.SurfaceHolder
fun getMaximumPreviewSize(): Size {
// See https://developer.android.com/reference/android/hardware/camera2/params/StreamConfigurationMap
// According to the Android Developer documentation, PREVIEW streams can have a resolution
// of up to the phone's display's resolution, with a maximum of 1920x1080.
val display1080p = Size(1920, 1080)
val displaySize = Size(
Resources.getSystem().displayMetrics.widthPixels,
Resources.getSystem().displayMetrics.heightPixels
)
val isHighResScreen = displaySize.bigger >= display1080p.bigger || displaySize.smaller >= display1080p.smaller
return if (isHighResScreen) display1080p else displaySize
}
fun CameraCharacteristics.getPreviewTargetSize(targetSize: Size?): Size {
val config = this.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP)!!
val maximumPreviewSize = getMaximumPreviewSize()
val outputSizes = config.getOutputSizes(SurfaceHolder::class.java)
.filter { it.bigger <= maximumPreviewSize.bigger && it.smaller <= maximumPreviewSize.smaller }
return outputSizes.closestToOrMax(targetSize)
}

View File

@ -0,0 +1,25 @@
package com.mrousavy.camera.extensions
import android.graphics.Point
import android.graphics.PointF
import android.util.Log
import android.util.Size
import com.mrousavy.camera.types.Orientation
fun Point.rotatedBy(fromSize: Size, toSize: Size, fromOrientation: Orientation, toOrientation: Orientation): Point {
val differenceDegrees = (fromOrientation.toDegrees() + toOrientation.toDegrees()) % 360
val difference = Orientation.fromRotationDegrees(differenceDegrees)
val normalizedPoint = PointF(this.x / fromSize.width.toFloat(), this.y / fromSize.height.toFloat())
val rotatedNormalizedPoint = when (difference) {
Orientation.PORTRAIT -> normalizedPoint
Orientation.PORTRAIT_UPSIDE_DOWN -> PointF(1 - normalizedPoint.x, 1 - normalizedPoint.y)
Orientation.LANDSCAPE_LEFT -> PointF(normalizedPoint.y, 1 - normalizedPoint.x)
Orientation.LANDSCAPE_RIGHT -> PointF(1 - normalizedPoint.y, normalizedPoint.x)
}
val rotatedX = rotatedNormalizedPoint.x * toSize.width
val rotatedY = rotatedNormalizedPoint.y * toSize.height
Log.i("ROTATE", "$this -> $normalizedPoint -> $difference -> $rotatedX, $rotatedY")
return Point(rotatedX.toInt(), rotatedY.toInt())
}

View File

@ -1,9 +1,5 @@
package com.mrousavy.camera.types package com.mrousavy.camera.types
import android.graphics.Point
import android.graphics.PointF
import android.util.Log
import android.util.Size
import com.mrousavy.camera.core.CameraDeviceDetails import com.mrousavy.camera.core.CameraDeviceDetails
enum class Orientation(override val unionValue: String) : JSUnionValue { enum class Orientation(override val unionValue: String) : JSUnionValue {
@ -30,7 +26,7 @@ enum class Orientation(override val unionValue: String) : JSUnionValue {
} }
// Rotate sensor rotation by target rotation // Rotate sensor rotation by target rotation
val newRotationDegrees = (deviceDetails.sensorOrientation + rotationDegrees + 360) % 360 val newRotationDegrees = (deviceDetails.sensorOrientation.toDegrees() + rotationDegrees + 360) % 360
return fromRotationDegrees(newRotationDegrees) return fromRotationDegrees(newRotationDegrees)
} }
@ -52,29 +48,5 @@ enum class Orientation(override val unionValue: String) : JSUnionValue {
in 225..315 -> LANDSCAPE_RIGHT in 225..315 -> LANDSCAPE_RIGHT
else -> PORTRAIT else -> PORTRAIT
} }
fun rotatePoint(
point: Point,
fromSize: Size,
toSize: Size,
fromOrientation: Orientation,
toOrientation: Orientation
): Point {
val differenceDegrees = (fromOrientation.toDegrees() + toOrientation.toDegrees()) % 360
val difference = Orientation.fromRotationDegrees(differenceDegrees)
val normalizedPoint = PointF(point.x / fromSize.width.toFloat(), point.y / fromSize.height.toFloat())
val rotatedNormalizedPoint = when (difference) {
PORTRAIT -> normalizedPoint
PORTRAIT_UPSIDE_DOWN -> PointF(1 - normalizedPoint.x, 1 - normalizedPoint.y)
LANDSCAPE_LEFT -> PointF(normalizedPoint.y, 1 - normalizedPoint.x)
LANDSCAPE_RIGHT -> PointF(1 - normalizedPoint.y, normalizedPoint.x)
}
val rotatedX = rotatedNormalizedPoint.x * toSize.width
val rotatedY = rotatedNormalizedPoint.y * toSize.height
Log.i("ROTATE", "$point -> $normalizedPoint -> $difference -> $rotatedX, $rotatedY")
return Point(rotatedX.toInt(), rotatedY.toInt())
}
} }
} }