From 934de142ee26660927ff7e755ca782a1a7ff8302 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 10 Dec 2021 09:52:40 +0100 Subject: [PATCH] docs: Update `Format.pixelFormat` documentation --- docs/docs/guides/FRAME_PROCESSORS.mdx | 7 +++++++ ios/CameraView+RecordVideo.swift | 2 +- src/CameraDevice.ts | 4 +++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/docs/guides/FRAME_PROCESSORS.mdx b/docs/docs/guides/FRAME_PROCESSORS.mdx index a1af31c..1e04e08 100644 --- a/docs/docs/guides/FRAME_PROCESSORS.mdx +++ b/docs/docs/guides/FRAME_PROCESSORS.mdx @@ -160,6 +160,13 @@ const frameProcessor = useFrameProcessor((frame) => { Check out [**Frame Processor community plugins**](/docs/guides/frame-processor-plugin-list) to discover plugins, or [**start creating a plugin yourself**](/docs/guides/frame-processors-plugins-overview)! +### Selecting a Format for a Frame Processor + +When running frame processors, it is often important to choose an appropriate [format](/docs/guides/formats). Here are some general tips to consider: + +* If you are running heavy AI/ML calculations in your frame processor, make sure to [select a format](/docs/guides/formats) that has a lower resolution to optimize it's performance. +* Sometimes a frame processor plugin only works with specific [pixel formats](/docs/api/interfaces/CameraDeviceFormat#pixelformat). Some plugins (like MLKit) don't work with `x420`. + ### Benchmarks Frame Processors are _really_ fast. I have used [MLKit Vision Image Labeling](https://firebase.google.com/docs/ml-kit/ios/label-images) to label 4k Camera frames in realtime, and measured the following results: diff --git a/ios/CameraView+RecordVideo.swift b/ios/CameraView+RecordVideo.swift index ed40ffd..bd8ba1f 100644 --- a/ios/CameraView+RecordVideo.swift +++ b/ios/CameraView+RecordVideo.swift @@ -110,7 +110,7 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud callback.reject(error: .capture(.createRecorderError(message: "Failed to get video settings!"))) return } - // get pixel format (420f, 420v) + // get pixel format (420f, 420v, x420) let pixelFormat = CMFormatDescriptionGetMediaSubType(videoInput.device.activeFormat.formatDescription) self.recordingSession!.initializeVideoWriter(withSettings: videoSettings, pixelFormat: pixelFormat) diff --git a/src/CameraDevice.ts b/src/CameraDevice.ts index 92602e8..2e2c766 100644 --- a/src/CameraDevice.ts +++ b/src/CameraDevice.ts @@ -173,7 +173,9 @@ export interface CameraDeviceFormat { */ videoStabilizationModes: VideoStabilizationMode[]; /** - * Specifies the pixel format on iOS. e.g. 420v, 420f + * Specifies this format's pixel format. The pixel format specifies how the individual pixels are interpreted as a visual image. + * + * The most common format is `420v`. Some formats (like `x420`) are not compatible with some frame processor plugins (e.g. MLKit) */ pixelFormat: PixelFormat; }