From 4038db2e284c2d71b6a45fc3e86495ba0303f04d Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Tue, 8 Jun 2021 14:20:07 +0200 Subject: [PATCH] feat: Frame Processors: Allow returning `Frame`s (support for resize and other frame manipulations) (#185) * batch * Init Frame as box * Use ObjC syntax * Fix access * Revert "Fix access" This reverts commit 7de09e52739d4c2b53f485d5ed696f1665fa5737. * Revert "Use ObjC syntax" This reverts commit e33f05ae8451cc4ee24af41d14dc76a57c157554. * Revert "Init Frame as box" This reverts commit 5adafb6109bfbf7fddb8ddc4af7d306b7b76b476. * use holder * convert buffer <-> jsi object * add docs * add more docs * Update JSIUtils.mm * Update FRAME_PROCESSORS_CREATE_OVERVIEW.mdx * Update CameraView+RecordVideo.swift --- VisionCamera.podspec | 1 + .../FRAME_PROCESSORS_CREATE_OVERVIEW.mdx | 47 ++++++++++++++----- example/ios/Podfile.lock | 4 +- ios/CameraView+RecordVideo.swift | 5 +- ios/Frame Processor/CMSampleBufferRefHolder.h | 22 +++++++++ ios/Frame Processor/CMSampleBufferRefHolder.m | 25 ++++++++++ ios/Frame Processor/Frame.h | 20 -------- ios/Frame Processor/FrameHostObject.h | 8 ++-- ios/React Utils/JSIUtils.mm | 14 ++++++ ios/VisionCamera.xcodeproj/project.pbxproj | 6 ++- 10 files changed, 112 insertions(+), 40 deletions(-) create mode 100644 ios/Frame Processor/CMSampleBufferRefHolder.h create mode 100644 ios/Frame Processor/CMSampleBufferRefHolder.m delete mode 100644 ios/Frame Processor/Frame.h diff --git a/VisionCamera.podspec b/VisionCamera.podspec index 74bc7fd..d585c08 100644 --- a/VisionCamera.podspec +++ b/VisionCamera.podspec @@ -25,6 +25,7 @@ Pod::Spec.new do |s| s.source_files = [ "ios/**/*.{m,mm,swift}", "ios/CameraBridge.h", + "ios/Frame Processor/CMSampleBufferRefHolder.h", "ios/Frame Processor/FrameProcessorCallback.h", "ios/Frame Processor/FrameProcessorRuntimeManager.h", "ios/Frame Processor/FrameProcessorPluginRegistry.h", diff --git a/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx b/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx index 3f430a2..62c5558 100644 --- a/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx +++ b/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx @@ -32,15 +32,16 @@ To achieve **maximum performance**, the `scanQRCodes` function is written in a n The Frame Processor Plugin Registry API automatically manages type conversion from JS <-> native. They are converted into the most efficient data-structures, as seen here: -| JS Type | Objective-C Type | Java Type | -|----------------------|--------------------------|----------------------------| -| `number` | `NSNumber` (double) | `double` | -| `boolean` | `NSNumber` (boolean) | `boolean` | -| `string` | `NSString` | `String` | -| `[]` | `NSArray` | `Array` | -| `{}` | `NSDictionary` | `HashMap` | -| `undefined` | `nil` / `NSNull` | `null` | -| `(any, any) => void` | `RCTResponseSenderBlock` | `(Object, Object) -> void` | +| JS Type | Objective-C Type | Java Type | +|----------------------|---------------------------|----------------------------| +| `number` | `NSNumber*` (double) | `double` | +| `boolean` | `NSNumber*` (boolean) | `boolean` | +| `string` | `NSString*` | `String` | +| `[]` | `NSArray*` | `Array` | +| `{}` | `NSDictionary*` | `HashMap` | +| `undefined` / `null` | `nil` / `NSNull` | `null` | +| `(any, any) => void` | `RCTResponseSenderBlock` | `(Object, Object) -> void` | +| `Frame` | `CMSampleBufferRefHolder` | `ImageProxy` | ### Return values @@ -56,12 +57,36 @@ Returns a `string` in JS: ```js export function detectObject(frame: Frame): string { - 'worklet'; - const result = __detectObject(frame); + 'worklet' + const result = __detectObject(frame) _log(result) // <-- "cat" } ``` +You can also manipulate the buffer and return it (or a copy) by using the `CMSampleBufferRefHolder` class: + +```objc +static inline id resize(CMSampleBufferRef buffer, NSArray args) { + NSNumber* width = [arguments objectAtIndex:0]; + NSNumber* height = [arguments objectAtIndex:1]; + + CMSampleBufferRef resizedBuffer = CMSampleBufferCopyAndResize(buffer, width, height); + return [[CMSampleBufferRefHolder alloc] initWithBuffer:resizedBuffer]; +} +``` + +Which returns a `Frame` in JS: + +```js +const frameProcessor = useFrameProcessor((frame) => { + 'worklet'; + // by downscaling the frame, the `detectObjects` function runs faster. + const resizedFrame = resize(frame, 720, 480) + const objects = detectObjects(resizedFrame) + _log(objects) +}, []) +``` + ### Parameters Frame Processors can also accept parameters, following the same type convention as [return values](#return-values): diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 0b3a5b8..cbc14f5 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -322,7 +322,7 @@ PODS: - React - RNVectorIcons (8.1.0): - React-Core - - VisionCamera (2.1.0): + - VisionCamera (2.2.0): - React - React-callinvoker - React-Core @@ -490,7 +490,7 @@ SPEC CHECKSUMS: RNReanimated: 9c13c86454bfd54dab7505c1a054470bfecd2563 RNStaticSafeAreaInsets: 6103cf09647fa427186d30f67b0f5163c1ae8252 RNVectorIcons: 31cebfcf94e8cf8686eb5303ae0357da64d7a5a4 - VisionCamera: 478f70a945846254699cfd33c2ac852e79258b83 + VisionCamera: 9886518481961e1c5d94cedb9b7513c28b8368c1 Yoga: 575c581c63e0d35c9a83f4b46d01d63abc1100ac PODFILE CHECKSUM: 4b093c1d474775c2eac3268011e4b0b80929d3a2 diff --git a/ios/CameraView+RecordVideo.swift b/ios/CameraView+RecordVideo.swift index 4f2c6d3..2412048 100644 --- a/ios/CameraView+RecordVideo.swift +++ b/ios/CameraView+RecordVideo.swift @@ -203,8 +203,9 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud if frameProcessorCallback != nil && !hasLoggedFrameDropWarning && captureOutput is AVCaptureVideoDataOutput { let reason = findFrameDropReason(inBuffer: buffer) ReactLogger.log(level: .warning, - message: "Dropped a Frame. This might indicate that your Frame Processor is doing too much work. " + - "Either throttle the frame processor's frame rate, or optimize your frame processor's execution speed. Frame drop reason: \(reason)", + message: "Dropped a Frame - This might indicate that your Frame Processor is doing too much work. " + + "Either throttle the frame processor's frame rate using the `frameProcessorFps` prop, or optimize " + + "your frame processor's execution speed. Frame drop reason: \(reason)", alsoLogToJS: true) hasLoggedFrameDropWarning = true } diff --git a/ios/Frame Processor/CMSampleBufferRefHolder.h b/ios/Frame Processor/CMSampleBufferRefHolder.h new file mode 100644 index 0000000..557b22b --- /dev/null +++ b/ios/Frame Processor/CMSampleBufferRefHolder.h @@ -0,0 +1,22 @@ +// +// CMSampleBufferRefHolder.h +// VisionCamera +// +// Created by Marc Rousavy on 15.03.21. +// Copyright © 2021 mrousavy. All rights reserved. +// + +#pragma once + +#import +#import + +@interface CMSampleBufferRefHolder : NSObject { + CMSampleBufferRef buffer; +} + +- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer; + +@property (nonatomic) CMSampleBufferRef buffer; + +@end diff --git a/ios/Frame Processor/CMSampleBufferRefHolder.m b/ios/Frame Processor/CMSampleBufferRefHolder.m new file mode 100644 index 0000000..12c88d6 --- /dev/null +++ b/ios/Frame Processor/CMSampleBufferRefHolder.m @@ -0,0 +1,25 @@ +// +// CMSampleBufferRefHolder.m +// VisionCamera +// +// Created by Marc Rousavy on 08.06.21. +// Copyright © 2021 mrousavy. All rights reserved. +// + +#import "CMSampleBufferRefHolder.h" +#import +#import + +@implementation CMSampleBufferRefHolder + +- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer { + self = [super init]; + if (self) { + self.buffer = buffer; + } + return self; +} + +@synthesize buffer; + +@end diff --git a/ios/Frame Processor/Frame.h b/ios/Frame Processor/Frame.h deleted file mode 100644 index 8bd97a7..0000000 --- a/ios/Frame Processor/Frame.h +++ /dev/null @@ -1,20 +0,0 @@ -// -// Frame.h -// VisionCamera -// -// Created by Marc Rousavy on 15.03.21. -// Copyright © 2021 mrousavy. All rights reserved. -// - -#pragma once - -#import - -// TODO: Make this Objective-C so it can be imported in Swift? -class Frame { - public: - explicit Frame(CMSampleBufferRef buffer): buffer(buffer) {} - - public: - CMSampleBufferRef buffer; -}; diff --git a/ios/Frame Processor/FrameHostObject.h b/ios/Frame Processor/FrameHostObject.h index e1ca637..3e453b1 100644 --- a/ios/Frame Processor/FrameHostObject.h +++ b/ios/Frame Processor/FrameHostObject.h @@ -8,19 +8,21 @@ #pragma once -#import "Frame.h" #import #import using namespace facebook; -class JSI_EXPORT FrameHostObject: public Frame, public jsi::HostObject { +class JSI_EXPORT FrameHostObject: public jsi::HostObject { public: - explicit FrameHostObject(CMSampleBufferRef buffer): Frame(buffer) {} + explicit FrameHostObject(CMSampleBufferRef buffer): buffer(buffer) {} ~FrameHostObject(); public: jsi::Value get(jsi::Runtime&, const jsi::PropNameID& name) override; std::vector getPropertyNames(jsi::Runtime& rt) override; void destroyBuffer(); + +public: + CMSampleBufferRef buffer; }; diff --git a/ios/React Utils/JSIUtils.mm b/ios/React Utils/JSIUtils.mm index ce42897..359216b 100644 --- a/ios/React Utils/JSIUtils.mm +++ b/ios/React Utils/JSIUtils.mm @@ -12,6 +12,8 @@ #import #import #import +#import "../Frame Processor/CMSampleBufferRefHolder.h" +#import "../Frame Processor/FrameHostObject.h" using namespace facebook; using namespace facebook::react; @@ -66,6 +68,11 @@ jsi::Value convertObjCObjectToJSIValue(jsi::Runtime &runtime, id value) return convertNSArrayToJSIArray(runtime, (NSArray *)value); } else if (value == (id)kCFNull) { return jsi::Value::null(); + } else if ([value isKindOfClass:[CMSampleBufferRefHolder class]]) { + // it's boxed in a CMSampleBufferRefHolder because CMSampleBufferRef is not an NSObject + CMSampleBufferRef buffer = [(CMSampleBufferRefHolder*)value buffer]; + auto frame = std::make_shared(buffer); + return jsi::Object::createFromHostObject(runtime, frame); } return jsi::Value::undefined(); } @@ -144,6 +151,13 @@ id convertJSIValueToObjCObject(jsi::Runtime &runtime, const jsi::Value &value, s if (o.isFunction(runtime)) { return convertJSIFunctionToCallback(runtime, std::move(o.getFunction(runtime)), jsInvoker); } + if (o.isHostObject(runtime)) { + auto hostObject = o.asHostObject(runtime); + auto frame = dynamic_cast(hostObject.get()); + if (frame != nullptr) { + return [[CMSampleBufferRefHolder alloc] initWithBuffer:frame->buffer]; + } + } return convertJSIObjectToNSDictionary(runtime, o, jsInvoker); } diff --git a/ios/VisionCamera.xcodeproj/project.pbxproj b/ios/VisionCamera.xcodeproj/project.pbxproj index a942b8b..e150642 100644 --- a/ios/VisionCamera.xcodeproj/project.pbxproj +++ b/ios/VisionCamera.xcodeproj/project.pbxproj @@ -81,7 +81,7 @@ B80E069F266632F000728644 /* AVAudioSession+updateCategory.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVAudioSession+updateCategory.swift"; sourceTree = ""; }; B8103E1B25FF553B007A1684 /* FrameProcessorUtils.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = FrameProcessorUtils.mm; sourceTree = ""; }; B8103E1E25FF5550007A1684 /* FrameProcessorUtils.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorUtils.h; sourceTree = ""; }; - B8103E5725FF56F0007A1684 /* Frame.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Frame.h; sourceTree = ""; }; + B8103E5725FF56F0007A1684 /* CMSampleBufferRefHolder.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = CMSampleBufferRefHolder.h; sourceTree = ""; }; B81D41EF263C86F900B041FD /* JSIUtils.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = JSIUtils.h; sourceTree = ""; }; B82FBA942614B69D00909718 /* RCTBridge+runOnJS.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "RCTBridge+runOnJS.h"; sourceTree = ""; }; B82FBA952614B69D00909718 /* RCTBridge+runOnJS.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = "RCTBridge+runOnJS.mm"; sourceTree = ""; }; @@ -140,6 +140,7 @@ B8DB3BCB263DC97E004C18D7 /* AVFileType+descriptor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVFileType+descriptor.swift"; sourceTree = ""; }; B8DCF09125EA7BEE00EA5C72 /* SpeedChecker.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SpeedChecker.h; sourceTree = ""; }; B8DCF14425EA817D00EA5C72 /* MakeJSIRuntime.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MakeJSIRuntime.h; sourceTree = ""; }; + B8F7DDD1266F715D00120533 /* CMSampleBufferRefHolder.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = CMSampleBufferRefHolder.m; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -262,7 +263,8 @@ B80D67A825FA25380008FE8D /* FrameProcessorCallback.h */, B8103E1E25FF5550007A1684 /* FrameProcessorUtils.h */, B8103E1B25FF553B007A1684 /* FrameProcessorUtils.mm */, - B8103E5725FF56F0007A1684 /* Frame.h */, + B8103E5725FF56F0007A1684 /* CMSampleBufferRefHolder.h */, + B8F7DDD1266F715D00120533 /* CMSampleBufferRefHolder.m */, B84760A22608EE38004C3180 /* FrameHostObject.h */, B84760A52608EE7C004C3180 /* FrameHostObject.mm */, B8A751D62609E4980011C623 /* FrameProcessorRuntimeManager.h */,