railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Split videoHdr and photoHdr into two settings (#2161)

Browse Source 
* feat: Split `videoHdr` and `photoHdr` into two settings

* fix: Rename all `hdr`

* fix: Fix HDR on Android

* Update CameraDeviceDetails.kt

* Update CameraDeviceDetails.kt

* fix: Correctly configure `pixelFormat` AFTER `format`

* Update CameraSession+Configuration.swift

* fix: Also after format changed
This commit is contained in:
Marc Rousavy
2023-11-15 18:33:12 +01:00
committed by GitHub
parent 75fd924899
commit c5dfb6c247
26 changed files with 129 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
package/android/src/main/java/com/mrousavy/camera/CameraView.kt
View File

@@ -70,7 +70,8 @@ class CameraView(context: Context) :
var format: ReadableMap? = null
var fps: Int? = null
var videoStabilizationMode: VideoStabilizationMode? = null
var hdr: Boolean? = null // nullable bool
var videoHdr = false
var photoHdr = false
var lowLightBoost: Boolean? = null // nullable bool
// other props
@@ -177,6 +178,10 @@ class CameraView(context: Context) :
// Orientation
config.orientation = orientation
// HDR
config.videoHdr = videoHdr
config.photoHdr = photoHdr
// Format
val format = format
if (format != null) {
@@ -188,7 +193,6 @@ class CameraView(context: Context) :
// Side-Props
config.fps = fps
config.enableLowLightBoost = lowLightBoost ?: false
config.enableHdr = hdr ?: false
config.torch = torch
// Zoom

13
package/android/src/main/java/com/mrousavy/camera/CameraViewManager.kt
View File

@@ -107,9 +107,14 @@ class CameraViewManager : ViewGroupManager<CameraView>() {
view.fps = if (fps > 0) fps else null
}
@ReactProp(name = "hdr")
fun setHdr(view: CameraView, hdr: Boolean?) {
view.hdr = hdr
@ReactProp(name = "photoHdr", defaultBoolean = false)
fun setPhotoHdr(view: CameraView, photoHdr: Boolean) {
view.photoHdr = photoHdr
}
@ReactProp(name = "videoHdr", defaultBoolean = false)
fun setVideoHdr(view: CameraView, videoHdr: Boolean) {
view.videoHdr = videoHdr
}
@ReactProp(name = "lowLightBoost")
@@ -117,7 +122,7 @@ class CameraViewManager : ViewGroupManager<CameraView>() {
view.lowLightBoost = lowLightBoost
}
@ReactProp(name = "isActive")
@ReactProp(name = "isActive", defaultBoolean = false)
fun setIsActive(view: CameraView, isActive: Boolean) {
view.isActive = isActive
}

7
package/android/src/main/java/com/mrousavy/camera/core/CameraConfiguration.kt
View File

@@ -17,7 +17,10 @@ data class CameraConfiguration(
var photo: Output<Photo> = Output.Disabled.create(),
var video: Output<Video> = Output.Disabled.create(),
var codeScanner: Output<CodeScanner> = Output.Disabled.create(),
var enableHdr: Boolean = false,
// HDR
var videoHdr: Boolean = false,
var photoHdr: Boolean = false,
// Orientation
var orientation: Orientation = Orientation.PORTRAIT,
@@ -87,7 +90,7 @@ data class CameraConfiguration(
val outputsChanged = deviceChanged || // input device
left?.photo != right.photo || left.video != right.video || left.codeScanner != right.codeScanner ||
left.preview != right.preview || // outputs
left.enableHdr != right.enableHdr || left.format != right.format // props that affect the outputs (hdr, format, ..)
left.videoHdr != right.videoHdr || left.photoHdr != right.photoHdr || left.format != right.format // props that affect the outputs
val sidePropsChanged = outputsChanged || // depend on outputs
left?.torch != right.torch || left.enableLowLightBoost != right.enableLowLightBoost || left.fps != right.fps ||

11
package/android/src/main/java/com/mrousavy/camera/core/CameraDeviceDetails.kt
View File

@@ -4,7 +4,6 @@ import android.graphics.ImageFormat
import android.hardware.camera2.CameraCharacteristics
import android.hardware.camera2.CameraManager
import android.hardware.camera2.CameraMetadata
import android.hardware.camera2.params.DynamicRangeProfiles
import android.os.Build
import android.util.Range
import android.util.Size
@@ -90,10 +89,8 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val
private fun getHasVideoHdr(): Boolean {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
if (capabilities.contains(CameraMetadata.REQUEST_AVAILABLE_CAPABILITIES_DYNAMIC_RANGE_TEN_BIT)) {
val availableProfiles = characteristics.get(CameraCharacteristics.REQUEST_AVAILABLE_DYNAMIC_RANGE_PROFILES)
?: DynamicRangeProfiles(LongArray(0))
return availableProfiles.supportedProfiles.contains(DynamicRangeProfiles.HLG10) ||
availableProfiles.supportedProfiles.contains(DynamicRangeProfiles.HDR10)
val recommendedHdrProfile = characteristics.get(CameraCharacteristics.REQUEST_RECOMMENDED_TEN_BIT_DYNAMIC_RANGE_PROFILE)
return recommendedHdrProfile != null
}
}
return false
@@ -181,8 +178,8 @@ class CameraDeviceDetails(private val cameraManager: CameraManager, private val
map.putInt("maxFps", fpsRange.upper)
map.putDouble("maxZoom", maxZoom)
map.putDouble("fieldOfView", getMaxFieldOfView())
map.putBoolean("supportsVideoHDR", supportsVideoHdr)
map.putBoolean("supportsPhotoHDR", supportsPhotoHdr)
map.putBoolean("supportsVideoHdr", supportsVideoHdr)
map.putBoolean("supportsPhotoHdr", supportsPhotoHdr)
map.putBoolean("supportsDepthCapture", supportsDepthCapture)
map.putString("autoFocusSystem", AutoFocusSystem.CONTRAST_DETECTION.unionValue)
map.putArray("videoStabilizationModes", createStabilizationModes())

10
package/android/src/main/java/com/mrousavy/camera/core/CameraSession.kt
View File

@@ -285,7 +285,7 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
val image = reader.acquireLatestImage()
onPhotoCaptured(image)
}, CameraQueues.cameraQueue.handler)
val output = PhotoOutput(imageReader, configuration.enableHdr)
val output = PhotoOutput(imageReader, configuration.photoHdr)
outputs.add(output.toOutputConfiguration(characteristics))
photoOutput = output
}
@@ -305,7 +305,7 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
isSelfie,
video.config.enableFrameProcessor
)
val output = VideoPipelineOutput(videoPipeline, configuration.enableHdr)
val output = VideoPipelineOutput(videoPipeline, configuration.videoHdr)
outputs.add(output.toOutputConfiguration(characteristics))
videoOutput = output
}
@@ -327,7 +327,7 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
preview.config.surface,
size,
SurfaceOutput.OutputType.PREVIEW,
configuration.enableHdr
configuration.videoHdr
)
outputs.add(output.toOutputConfiguration(characteristics))
previewOutput = output
@@ -398,7 +398,7 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
if (fps != null) {
if (!CAN_DO_60_FPS) {
// If we can't do 60 FPS, we clamp it to 30 FPS - that's always supported.
fps = Math.min(30, fps)
fps = 30.coerceAtMost(fps)
}
captureRequest.set(CaptureRequest.CONTROL_AE_TARGET_FPS_RANGE, Range(fps, fps))
}
@@ -427,7 +427,7 @@ class CameraSession(private val context: Context, private val cameraManager: Cam
// Set HDR
// TODO: Check if that value is even supported
if (config.enableHdr) {
if (config.videoHdr) {
captureRequest.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_HDR)
} else if (config.enableLowLightBoost) {
captureRequest.set(CaptureRequest.CONTROL_SCENE_MODE, CaptureRequest.CONTROL_SCENE_MODE_NIGHT)

8
package/android/src/main/java/com/mrousavy/camera/types/CameraDeviceFormat.kt
View File

@@ -17,8 +17,8 @@ data class CameraDeviceFormat(
val maxZoom: Double,
val videoStabilizationModes: List<VideoStabilizationMode>,
val autoFocusSystem: AutoFocusSystem,
val supportsVideoHDR: Boolean,
val supportsPhotoHDR: Boolean,
val supportsVideoHdr: Boolean,
val supportsPhotoHdr: Boolean,
val pixelFormats: List<PixelFormat>,
val supportsDepthCapture: Boolean
) {
@@ -50,8 +50,8 @@ data class CameraDeviceFormat(
value.getDouble("maxZoom"),
videoStabilizationModes,
autoFocusSystem,
value.getBoolean("supportsVideoHDR"),
value.getBoolean("supportsPhotoHDR"),
value.getBoolean("supportsVideoHdr"),
value.getBoolean("supportsPhotoHdr"),
pixelFormats,
value.getBoolean("supportsDepthCapture")
)

5
package/example/src/CameraPage.tsx
View File

@@ -68,7 +68,7 @@ export function CameraPage({ navigation }: Props): React.ReactElement {
const fps = Math.min(format?.maxFps ?? 1, targetFps)
const supportsFlash = device?.hasFlash ?? false
const supportsHdr = format?.supportsPhotoHDR
const supportsHdr = format?.supportsPhotoHdr
const supports60Fps = useMemo(() => device?.formats.some((f) => f.maxFps >= 60), [device?.formats])
const canToggleNightMode = device?.supportsLowLightBoost ?? false
@@ -181,7 +181,8 @@ export function CameraPage({ navigation }: Props): React.ReactElement {
device={device}
format={format}
fps={fps}
hdr={enableHdr}
photoHdr={enableHdr}
videoHdr={enableHdr}
lowLightBoost={device.supportsLowLightBoost && enableNightMode}
isActive={isActive}
onInitialized={onInitialized}

5
package/ios/CameraView.swift
View File

@@ -38,7 +38,8 @@ public final class CameraView: UIView, CameraSessionDelegate {
// props that require format reconfiguring
@objc var format: NSDictionary?
@objc var fps: NSNumber?
@objc var hdr = false
@objc var videoHdr = false
@objc var photoHdr = false
@objc var lowLightBoost = false
@objc var orientation: NSString?
// other props
@@ -164,7 +165,7 @@ public final class CameraView: UIView, CameraSessionDelegate {
if video || enableFrameProcessor {
config.video = .enabled(config: CameraConfiguration.Video(pixelFormat: getPixelFormat(),
enableBufferCompression: enableBufferCompression,
enableHdr: hdr,
enableHdr: videoHdr,
enableFrameProcessor: enableFrameProcessor))
} else {
config.video = .disabled

3
package/ios/CameraViewManager.m
View File

@@ -36,7 +36,8 @@ RCT_EXPORT_VIEW_PROPERTY(enableFrameProcessor, BOOL);
// device format
RCT_EXPORT_VIEW_PROPERTY(format, NSDictionary);
RCT_EXPORT_VIEW_PROPERTY(fps, NSNumber);
RCT_EXPORT_VIEW_PROPERTY(hdr, BOOL);
RCT_EXPORT_VIEW_PROPERTY(videoHdr, BOOL);
RCT_EXPORT_VIEW_PROPERTY(photoHdr, BOOL);
RCT_EXPORT_VIEW_PROPERTY(lowLightBoost, BOOL);
RCT_EXPORT_VIEW_PROPERTY(videoStabilizationMode, NSString);
RCT_EXPORT_VIEW_PROPERTY(pixelFormat, NSString);

2
package/ios/Core/CameraConfiguration.swift
View File

@@ -207,7 +207,7 @@ extension CameraConfiguration.Video {
// Find the best matching format
guard let format = videoOutput.findPixelFormat(firstOf: targetFormats) else {
throw CameraError.format(.invalidHdr)
throw CameraError.format(.invalidVideoHdr)
}
// YUV 4:2:0 10-bit (compressed/uncompressed)
return format

10
package/ios/Core/CameraError.swift
View File

@@ -108,7 +108,7 @@ enum DeviceError: String {
enum FormatError {
case invalidFps(fps: Int)
case invalidHdr
case invalidVideoHdr
case invalidFormat
case incompatiblePixelFormatWithHDR
@@ -118,8 +118,8 @@ enum FormatError {
return "invalid-format"
case .invalidFps:
return "invalid-fps"
case .invalidHdr:
return "invalid-hdr"
case .invalidVideoHdr:
return "invalid-video-hdr"
case .incompatiblePixelFormatWithHDR:
return "incompatible-pixel-format-with-hdr-setting"
}
@@ -131,8 +131,8 @@ enum FormatError {
return "The given format was invalid. Did you check if the current device supports the given format in `device.formats`?"
case let .invalidFps(fps):
return "The given format cannot run at \(fps) FPS! Make sure your FPS is lower than `format.maxFps` but higher than `format.minFps`."
case .invalidHdr:
return "The currently selected format does not support HDR capture! Make sure you select a format which includes `supportsPhotoHDR`/`supportsVideoHDR`!"
case .invalidVideoHdr:
return "The currently selected format does not support 10-bit Video HDR streaming! Make sure you select a format which includes `supportsVideoHdr`!"
case .incompatiblePixelFormatWithHDR:
return "The currently selected pixelFormat is not compatible with HDR! HDR only works with the `yuv` pixelFormat."
}

22
package/ios/Core/CameraSession+Configuration.swift
View File

@@ -100,7 +100,7 @@ extension CameraSession {
}
// Video Output + Frame Processor
if case let .enabled(video) = configuration.video {
if case .enabled = configuration.video {
ReactLogger.log(level: .info, message: "Adding Video Data output...")
// 1. Add
@@ -113,11 +113,6 @@ extension CameraSession {
// 2. Configure
videoOutput.setSampleBufferDelegate(self, queue: CameraQueues.videoQueue)
videoOutput.alwaysDiscardsLateVideoFrames = true
let pixelFormatType = try video.getPixelFormat(for: videoOutput)
videoOutput.videoSettings = [
String(kCVPixelBufferPixelFormatTypeKey): pixelFormatType,
]
self.videoOutput = videoOutput
}
@@ -210,6 +205,21 @@ extension CameraSession {
ReactLogger.log(level: .info, message: "Successfully configured Format!")
}
func configurePixelFormat(configuration: CameraConfiguration) throws {
guard case let .enabled(video) = configuration.video,
let videoOutput else {
// Video is not enabled
return
}
// Configure the VideoOutput Settings to use the given Pixel Format.
// We need to run this after device.activeFormat has been set, otherwise the VideoOutput can't stream the given Pixel Format.
let pixelFormatType = try video.getPixelFormat(for: videoOutput)
videoOutput.videoSettings = [
String(kCVPixelBufferPixelFormatTypeKey): pixelFormatType,
]
}
// pragma MARK: Side-Props
/**

13
package/ios/Core/CameraSession.swift
View File

@@ -141,21 +141,26 @@ class CameraSession: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate, AVC
if difference.formatChanged {
try self.configureFormat(configuration: config, device: device)
}
// 5. Configure side-props (fps, lowLightBoost)
// 5. After step 2. and 4., we also need to configure the PixelFormat.
// This needs to be done AFTER we updated the `format`, as this controls the supported PixelFormats.
if difference.outputsChanged || difference.formatChanged {
try self.configurePixelFormat(configuration: config)
}
// 6. Configure side-props (fps, lowLightBoost)
if difference.sidePropsChanged {
try self.configureSideProps(configuration: config, device: device)
}
// 6. Configure zoom
// 7. Configure zoom
if difference.zoomChanged {
self.configureZoom(configuration: config, device: device)
}
}
}
// 7. Start or stop the session if needed
// 8. Start or stop the session if needed
self.checkIsActive(configuration: config)
// 8. Enable or disable the Torch if needed (requires session to be running)
// 9. Enable or disable the Torch if needed (requires session to be running)
if difference.torchChanged {
try self.withDeviceLock { device in
try self.configureTorch(configuration: config, device: device)

4
package/ios/Extensions/AVCaptureDevice.Format+toDictionary.swift
View File

@@ -28,7 +28,7 @@ extension AVCaptureDevice.Format {
return maxRange?.maxFrameRate ?? 0
}
var supportsVideoHDR: Bool {
var supportsVideoHdr: Bool {
let pixelFormat = CMFormatDescriptionGetMediaSubType(formatDescription)
let hdrFormats = [
kCVPixelFormatType_420YpCbCr10BiPlanarFullRange,
@@ -38,7 +38,7 @@ extension AVCaptureDevice.Format {
return hdrFormats.contains(pixelFormat)
}
var supportsPhotoHDR: Bool {
var supportsPhotoHdr: Bool {
// TODO: Supports Photo HDR on iOS?
return false
}

16
package/ios/Types/CameraDeviceFormat.swift
View File

@@ -31,8 +31,8 @@ struct CameraDeviceFormat: Equatable, CustomStringConvertible {
let videoStabilizationModes: [VideoStabilizationMode]
let autoFocusSystem: AutoFocusSystem
let supportsVideoHDR: Bool
let supportsPhotoHDR: Bool
let supportsVideoHdr: Bool
let supportsPhotoHdr: Bool
let pixelFormats: [PixelFormat]
@@ -51,8 +51,8 @@ struct CameraDeviceFormat: Equatable, CustomStringConvertible {
maxZoom = format.videoMaxZoomFactor
videoStabilizationModes = format.videoStabilizationModes.map { VideoStabilizationMode(from: $0) }
autoFocusSystem = AutoFocusSystem(fromFocusSystem: format.autoFocusSystem)
supportsVideoHDR = format.supportsVideoHDR
supportsPhotoHDR = format.supportsPhotoHDR
supportsVideoHdr = format.supportsVideoHdr
supportsPhotoHdr = format.supportsPhotoHdr
pixelFormats = CameraDeviceFormat.getAllPixelFormats()
supportsDepthCapture = format.supportsDepthCapture
}
@@ -73,8 +73,8 @@ struct CameraDeviceFormat: Equatable, CustomStringConvertible {
videoStabilizationModes = try jsVideoStabilizationModes.map { try VideoStabilizationMode(jsValue: $0) }
let jsAutoFocusSystem = jsValue["autoFocusSystem"] as! String
autoFocusSystem = try AutoFocusSystem(jsValue: jsAutoFocusSystem)
supportsVideoHDR = jsValue["supportsVideoHDR"] as! Bool
supportsPhotoHDR = jsValue["supportsPhotoHDR"] as! Bool
supportsVideoHdr = jsValue["supportsVideoHdr"] as! Bool
supportsPhotoHdr = jsValue["supportsPhotoHdr"] as! Bool
let jsPixelFormats = jsValue["pixelFormats"] as! [String]
pixelFormats = try jsPixelFormats.map { try PixelFormat(jsValue: $0) }
supportsDepthCapture = jsValue["supportsDepthCapture"] as! Bool
@@ -98,8 +98,8 @@ struct CameraDeviceFormat: Equatable, CustomStringConvertible {
"minISO": minISO,
"fieldOfView": fieldOfView,
"maxZoom": maxZoom,
"supportsVideoHDR": supportsVideoHDR,
"supportsPhotoHDR": supportsPhotoHDR,
"supportsVideoHdr": supportsVideoHdr,
"supportsPhotoHdr": supportsPhotoHdr,
"minFps": minFps,
"maxFps": maxFps,
"pixelFormats": pixelFormats.map(\.jsValue),

4
package/src/Camera.tsx
View File

@@ -142,8 +142,8 @@ export class Camera extends React.PureComponent<CameraProps> {
result = (result / 30) * fps
// H.265 (HEVC) codec is 20% more efficient
if (codec === 'h265') result = result * 0.8
// HDR (10-bit) instead of SDR (8-bit) takes up 20% more pixels
if (this.props.hdr) result = result * 1.2
// 10-Bit Video HDR takes up 20% more pixels than standard range (8-bit SDR)
if (this.props.videoHdr) result = result * 1.2
// Return overall result
return result * factor
}

4
package/src/CameraDevice.ts
View File

@@ -91,11 +91,11 @@ export interface CameraDeviceFormat {
/**
* Specifies whether this format supports HDR mode for video capture
*/
supportsVideoHDR: boolean
supportsVideoHdr: boolean
/**
* Specifies whether this format supports HDR mode for photo capture
*/
supportsPhotoHDR: boolean
supportsPhotoHdr: boolean
/**
* Specifies whether this format supports delivering depth data for photo or video capture.
*/

2
package/src/CameraError.ts
View File

@@ -15,7 +15,7 @@ export type DeviceError =
| 'device/camera-not-available-on-simulator'
export type FormatError =
| 'format/invalid-fps'
| 'format/invalid-hdr'
| 'format/invalid-video-hdr'
| 'format/incompatible-pixel-format-with-hdr-setting'
| 'format/invalid-format'
export type SessionError =

15
package/src/CameraProps.ts
View File

@@ -116,7 +116,8 @@ export interface CameraProps extends ViewProps {
*
* The format defines the possible values for properties like:
* - {@linkcode fps}: `format.minFps`...`format.maxFps`
* - {@linkcode hdr}: `format.supportsVideoHDR`
* - {@linkcode videoHdr}: `format.supportsVideoHdr`
* - {@linkcode photoHdr}: `format.supportsPhotoHdr`
* - {@linkcode pixelFormat}: `format.pixelFormats``
* - {@linkcode enableDepthData}: `format.supportsDepthCapture``
* - {@linkcode videoStabilizationMode}: `format.videoStabilizationModes``
@@ -139,11 +140,17 @@ export interface CameraProps extends ViewProps {
*/
fps?: number
/**
* Enables or disables HDR streaming.
* Enables or disables HDR Video Streaming for Preview, Video and Frame Processor via a 10-bit wide-color pixel format.
*
* Make sure the given {@linkcode format} supports HDR (see {@linkcode CameraDeviceFormat.supportsVideoHDR format.supportsVideoHDR}).
* Make sure the given {@linkcode format} supports HDR (see {@linkcode CameraDeviceFormat.supportsVideoHdr format.supportsVideoHdr}).
*/
hdr?: boolean
videoHdr?: boolean
/**
* Enables or disables HDR Photo Capture via a double capture routine that combines low- and high exposure photos.
*
* Make sure the given {@linkcode format} supports HDR (see {@linkcode CameraDeviceFormat.supportsPhotoHdr format.supportsPhotoHdr}).
*/
photoHdr?: boolean
/**
* Enables or disables lossless buffer compression for the video stream.
* If you only use {@linkcode video} or a {@linkcode frameProcessor}, this

16
package/src/devices/getCameraFormat.ts
View File

@@ -63,11 +63,11 @@ export interface FormatFilter {
/**
* Whether you want to find a format that supports Photo HDR.
*/
photoHDR?: boolean
photoHdr?: boolean
/**
* Whether you want to find a format that supports Photo HDR.
*/
videoHDR?: boolean
videoHdr?: boolean
}
type FilterWithPriority<T> = {
@@ -198,15 +198,15 @@ export function getCameraFormat(device: CameraDevice, filters: FormatFilter[]):
}
// Find Photo HDR formats
if (filter.photoHDR != null) {
if (bestFormat.supportsPhotoHDR === filter.photoHDR.target) leftPoints++
if (format.supportsPhotoHDR === filter.photoHDR.target) rightPoints++
if (filter.photoHdr != null) {
if (bestFormat.supportsPhotoHdr === filter.photoHdr.target) leftPoints++
if (format.supportsPhotoHdr === filter.photoHdr.target) rightPoints++
}
// Find Video HDR formats
if (filter.videoHDR != null) {
if (bestFormat.supportsVideoHDR === filter.videoHDR.target) leftPoints++
if (format.supportsVideoHDR === filter.videoHDR.target) rightPoints++
if (filter.videoHdr != null) {
if (bestFormat.supportsVideoHdr === filter.videoHdr.target) leftPoints++
if (format.supportsVideoHdr === filter.videoHdr.target) rightPoints++
}
if (rightPoints > leftPoints) bestFormat = format