From 49d58d0d0ce7f5538643d28cfbf53a1103bf0528 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Wed, 22 Nov 2023 16:19:29 +0100 Subject: [PATCH] perf: Use `lossy` compressed YUV buffers if available (#2195) --- docs/docs/guides/PERFORMANCE.mdx | 4 ++-- docs/docs/guides/PIXEL_FORMATS.mdx | 2 +- package/ios/Core/CameraConfiguration.swift | 18 +++++++++--------- .../AVCaptureDevice.Format+toDictionary.swift | 2 +- package/ios/Frame Processor/FrameHostObject.mm | 8 ++++---- package/ios/Types/PixelFormat.swift | 8 ++++---- package/src/CameraProps.ts | 4 ++-- 7 files changed, 23 insertions(+), 23 deletions(-) diff --git a/docs/docs/guides/PERFORMANCE.mdx b/docs/docs/guides/PERFORMANCE.mdx index d945b8b..4d55c4e 100644 --- a/docs/docs/guides/PERFORMANCE.mdx +++ b/docs/docs/guides/PERFORMANCE.mdx @@ -11,7 +11,7 @@ import useBaseUrl from '@docusaurus/useBaseUrl' ## Performance of VisionCamera VisionCamera is highly optimized to be **as fast as a native Camera app**, and is sometimes even faster than that. -I am using highly efficient native GPU buffer formats (such as YUV 4:2:0, or lossless compressed YUV 4:2:0), running the video pipelines in parallel, using C++ for the Frame Processors implementation, and other tricks to make sure VisionCamera is as efficient as possible. +I am using highly efficient native GPU buffer formats (such as YUV 4:2:0, or lossy compressed YUV 4:2:0), running the video pipelines in parallel, using C++ for the Frame Processors implementation, and other tricks to make sure VisionCamera is as efficient as possible. ## Making it faster @@ -66,7 +66,7 @@ Video HDR uses 10-bit formats and/or additional processing steps that come with ### Buffer Compression -Enable Buffer Compression ([`enableBufferCompression`](/docs/api/interfaces/CameraProps#enablebuffercompression)) to use lossless-compressed buffers for the Camera's video pipeline. These buffers can use less memory and are more efficient. +Enable Buffer Compression ([`enableBufferCompression`](/docs/api/interfaces/CameraProps#enablebuffercompression)) to use lossy-compressed buffers for the Camera's video pipeline. These buffers can use less memory and are more efficient. Note: When not using a `frameProcessor`, buffer compression is automatically enabled. diff --git a/docs/docs/guides/PIXEL_FORMATS.mdx b/docs/docs/guides/PIXEL_FORMATS.mdx index 4b701b2..dc6e8b5 100644 --- a/docs/docs/guides/PIXEL_FORMATS.mdx +++ b/docs/docs/guides/PIXEL_FORMATS.mdx @@ -77,4 +77,4 @@ Instead of [`kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange`](https://developer [Buffer Compression](/docs/guides/performance#buffer-compression) is automatically enabled if you are not using a Frame Processor. If you are using a Frame Processor, buffer compression will be turned off, as it essentially uses a different format than YUV. See ["Understanding YpCbCr Image Formats"](https://developer.apple.com/documentation/accelerate/conversion/understanding_ypcbcr_image_formats) for more information. -Instead of [`kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange`](https://developer.apple.com/documentation/corevideo/kcvpixelformattype_420ypcbcr8biplanarvideorange), it uses [`kCVPixelFormatType_Lossless_420YpCbCr8BiPlanarVideoRange`](https://developer.apple.com/documentation/corevideo/3746862-anonymous/kcvpixelformattype_lossless_420ypcbcr8biplanarvideorange), same for full-range. +Instead of [`kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange`](https://developer.apple.com/documentation/corevideo/kcvpixelformattype_420ypcbcr8biplanarvideorange), it uses [`kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarVideoRange`](https://developer.apple.com/documentation/corevideo/3746862-anonymous/kcvpixelformattype_lossy_420ypcbcr8biplanarvideorange), same for full-range. diff --git a/package/ios/Core/CameraConfiguration.swift b/package/ios/Core/CameraConfiguration.swift index a5964c2..020d16e 100644 --- a/package/ios/Core/CameraConfiguration.swift +++ b/package/ios/Core/CameraConfiguration.swift @@ -188,14 +188,14 @@ extension CameraConfiguration.Video { if enableBufferCompression { // use compressed format instead if we enabled buffer compression if defaultFormat == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange && - videoOutput.availableVideoPixelFormatTypes.contains(kCVPixelFormatType_Lossless_420YpCbCr8BiPlanarVideoRange) { + videoOutput.availableVideoPixelFormatTypes.contains(kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarVideoRange) { // YUV 4:2:0 8-bit (limited video colors; compressed) - defaultFormat = kCVPixelFormatType_Lossless_420YpCbCr8BiPlanarVideoRange + defaultFormat = kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarVideoRange } if defaultFormat == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange && - videoOutput.availableVideoPixelFormatTypes.contains(kCVPixelFormatType_Lossless_420YpCbCr8BiPlanarFullRange) { + videoOutput.availableVideoPixelFormatTypes.contains(kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarFullRange) { // YUV 4:2:0 8-bit (full video colors; compressed) - defaultFormat = kCVPixelFormatType_Lossless_420YpCbCr8BiPlanarFullRange + defaultFormat = kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarFullRange } } @@ -208,8 +208,8 @@ extension CameraConfiguration.Video { var targetFormats = [kCVPixelFormatType_420YpCbCr10BiPlanarFullRange, kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange] if enableBufferCompression { - // If we enable buffer compression, try to use a lossless compressed YUV format first, otherwise fall back to the others. - targetFormats.insert(kCVPixelFormatType_Lossless_420YpCbCr10PackedBiPlanarVideoRange, at: 0) + // If we enable buffer compression, try to use a lossy compressed YUV format first, otherwise fall back to the others. + targetFormats.insert(kCVPixelFormatType_Lossy_420YpCbCr10PackedBiPlanarVideoRange, at: 0) } // Find the best matching format @@ -228,8 +228,8 @@ extension CameraConfiguration.Video { kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange] if enableBufferCompression { // YUV 4:2:0 8-bit (full/limited video colors; compressed) - targetFormats.insert(kCVPixelFormatType_Lossless_420YpCbCr8BiPlanarVideoRange, at: 0) - targetFormats.insert(kCVPixelFormatType_Lossless_420YpCbCr8BiPlanarFullRange, at: 0) + targetFormats.insert(kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarVideoRange, at: 0) + targetFormats.insert(kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarFullRange, at: 0) } guard let format = videoOutput.findPixelFormat(firstOf: targetFormats) else { throw CameraError.device(.pixelFormatNotSupported) @@ -240,7 +240,7 @@ extension CameraConfiguration.Video { var targetFormats = [kCVPixelFormatType_32BGRA] if enableBufferCompression { // RGBA 8-bit (compressed) - targetFormats.insert(kCVPixelFormatType_Lossless_32BGRA, at: 0) + targetFormats.insert(kCVPixelFormatType_Lossy_32BGRA, at: 0) } guard let format = videoOutput.findPixelFormat(firstOf: targetFormats) else { throw CameraError.device(.pixelFormatNotSupported) diff --git a/package/ios/Extensions/AVCaptureDevice.Format+toDictionary.swift b/package/ios/Extensions/AVCaptureDevice.Format+toDictionary.swift index 791d824..b56cac6 100644 --- a/package/ios/Extensions/AVCaptureDevice.Format+toDictionary.swift +++ b/package/ios/Extensions/AVCaptureDevice.Format+toDictionary.swift @@ -33,7 +33,7 @@ extension AVCaptureDevice.Format { let hdrFormats = [ kCVPixelFormatType_420YpCbCr10BiPlanarFullRange, kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange, - kCVPixelFormatType_Lossless_420YpCbCr10PackedBiPlanarVideoRange, + kCVPixelFormatType_Lossy_420YpCbCr10PackedBiPlanarVideoRange, ] return hdrFormats.contains(pixelFormat) } diff --git a/package/ios/Frame Processor/FrameHostObject.mm b/package/ios/Frame Processor/FrameHostObject.mm index 5b5aa3d..23b8838 100644 --- a/package/ios/Frame Processor/FrameHostObject.mm +++ b/package/ios/Frame Processor/FrameHostObject.mm @@ -154,15 +154,15 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr auto mediaType = CMFormatDescriptionGetMediaSubType(format); switch (mediaType) { case kCVPixelFormatType_32BGRA: - case kCVPixelFormatType_Lossless_32BGRA: + case kCVPixelFormatType_Lossy_32BGRA: return jsi::String::createFromUtf8(runtime, "rgb"); case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange: case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange: - case kCVPixelFormatType_Lossless_420YpCbCr8BiPlanarFullRange: - case kCVPixelFormatType_Lossless_420YpCbCr8BiPlanarVideoRange: - case kCVPixelFormatType_Lossless_420YpCbCr10PackedBiPlanarVideoRange: + case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarFullRange: + case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarVideoRange: + case kCVPixelFormatType_Lossy_420YpCbCr10PackedBiPlanarVideoRange: return jsi::String::createFromUtf8(runtime, "yuv"); default: return jsi::String::createFromUtf8(runtime, "unknown"); diff --git a/package/ios/Types/PixelFormat.swift b/package/ios/Types/PixelFormat.swift index 97e0687..6693e1b 100644 --- a/package/ios/Types/PixelFormat.swift +++ b/package/ios/Types/PixelFormat.swift @@ -33,11 +33,11 @@ enum PixelFormat: String, JSUnionValue { kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, kCVPixelFormatType_420YpCbCr10BiPlanarFullRange, kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange, - kCVPixelFormatType_Lossless_420YpCbCr8BiPlanarFullRange, - kCVPixelFormatType_Lossless_420YpCbCr8BiPlanarVideoRange, - kCVPixelFormatType_Lossless_420YpCbCr10PackedBiPlanarVideoRange: + kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarFullRange, + kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarVideoRange, + kCVPixelFormatType_Lossy_420YpCbCr10PackedBiPlanarVideoRange: self = .yuv - case kCVPixelFormatType_32BGRA, kCVPixelFormatType_Lossless_32BGRA: + case kCVPixelFormatType_32BGRA, kCVPixelFormatType_Lossy_32BGRA: self = .rgb default: self = .unknown diff --git a/package/src/CameraProps.ts b/package/src/CameraProps.ts index 42b008d..b6edda8 100644 --- a/package/src/CameraProps.ts +++ b/package/src/CameraProps.ts @@ -165,12 +165,12 @@ export interface CameraProps extends ViewProps { */ photoHdr?: boolean /** - * Enables or disables lossless buffer compression for the video stream. + * Enables or disables lossy buffer compression for the video stream. * If you only use {@linkcode video} or a {@linkcode frameProcessor}, this * can increase the efficiency and lower memory usage of the Camera. * * If buffer compression is enabled, the video pipeline will try to use a - * lossless-compressed pixel format instead of the normal one. + * lossy-compressed pixel format instead of the normal one. * * If you use a {@linkcode frameProcessor}, you might need to change how pixels * are read inside your native frame processor function as this is different