diff --git a/VisionCamera.podspec b/VisionCamera.podspec index d585c08..efd7a70 100644 --- a/VisionCamera.podspec +++ b/VisionCamera.podspec @@ -25,7 +25,7 @@ Pod::Spec.new do |s| s.source_files = [ "ios/**/*.{m,mm,swift}", "ios/CameraBridge.h", - "ios/Frame Processor/CMSampleBufferRefHolder.h", + "ios/Frame Processor/Frame.h", "ios/Frame Processor/FrameProcessorCallback.h", "ios/Frame Processor/FrameProcessorRuntimeManager.h", "ios/Frame Processor/FrameProcessorPluginRegistry.h", diff --git a/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx b/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx index 925bb32..db37576 100644 --- a/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx +++ b/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx @@ -32,23 +32,23 @@ To achieve **maximum performance**, the `scanQRCodes` function is written in a n The Frame Processor Plugin Registry API automatically manages type conversion from JS <-> native. They are converted into the most efficient data-structures, as seen here: -| JS Type | Objective-C Type | Java Type | -|----------------------|---------------------------|----------------------------| -| `number` | `NSNumber*` (double) | `double` | -| `boolean` | `NSNumber*` (boolean) | `boolean` | -| `string` | `NSString*` | `String` | -| `[]` | `NSArray*` | `Array` | -| `{}` | `NSDictionary*` | `HashMap` | -| `undefined` / `null` | `nil` | `null` | -| `(any, any) => void` | `RCTResponseSenderBlock` | `(Object, Object) -> void` | -| `Frame` | `CMSampleBufferRefHolder` | `ImageProxy` | +| JS Type | Objective-C Type | Java Type | +|----------------------|-------------------------------|----------------------------| +| `number` | `NSNumber*` (double) | `double` | +| `boolean` | `NSNumber*` (boolean) | `boolean` | +| `string` | `NSString*` | `String` | +| `[]` | `NSArray*` | `Array` | +| `{}` | `NSDictionary*` | `HashMap` | +| `undefined` / `null` | `nil` | `null` | +| `(any, any) => void` | [`RCTResponseSenderBlock`][4] | `(Object, Object) -> void` | +| [`Frame`][1] | [`Frame*`][2] | [`ImageProxy`][3] | ### Return values Return values will automatically be converted to JS values, assuming they are representable in the ["Types" table](#types). So the following Objective-C frame processor: ```objc -static inline id detectObject(CMSampleBufferRef buffer, NSArray args) { +static inline id detectObject(Frame* frame, NSArray args) { return @"cat"; } ``` @@ -63,15 +63,17 @@ export function detectObject(frame: Frame): string { } ``` -You can also manipulate the buffer and return it (or a copy) by using the `CMSampleBufferRefHolder` class: +You can also manipulate the buffer and return it (or a copy) by using the `Frame` class: ```objc -static inline id resize(CMSampleBufferRef buffer, NSArray args) { +#import + +static inline id resize(Frame* frame, NSArray args) { NSNumber* width = [arguments objectAtIndex:0]; NSNumber* height = [arguments objectAtIndex:1]; - CMSampleBufferRef resizedBuffer = CMSampleBufferCopyAndResize(buffer, width, height); - return [[CMSampleBufferRefHolder alloc] initWithBuffer:resizedBuffer]; + CMSampleBufferRef resizedBuffer = CMSampleBufferCopyAndResize(frame.buffer, width, height); + return [[Frame alloc] initWithBuffer:resizedBuffer orientation:frame.orientation]; } ``` @@ -116,9 +118,9 @@ For example, a realtime video chat application might use WebRTC to send the fram ```objc static dispatch_queue_t queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0ul); -static inline id sendFrameToWebRTC(CMSampleBufferRef buffer, NSArray args) { +static inline id sendFrameToWebRTC(Frame* frame, NSArray args) { CMSampleBufferRef bufferCopy; - CMSampleBufferCreateCopy(kCFAllocatorDefault, buffer, &bufferCopy); + CMSampleBufferCreateCopy(kCFAllocatorDefault, frame.buffer, &bufferCopy); dispatch_async(queue, ^{ NSString* serverURL = (NSString*)args[0]; @@ -171,3 +173,8 @@ Your Frame Processor Plugins have to be fast. VisionCamera automatically detects
#### 🚀 Create your first Frame Processor Plugin for [iOS](frame-processors-plugins-ios) or [Android](frame-processors-plugins-android)! + +[1]: https://github.com/cuvent/react-native-vision-camera/blob/main/src/Frame.ts +[2]: https://github.com/cuvent/react-native-vision-camera/blob/main/ios/Frame%20Processor/Frame.h +[3]: https://developer.android.com/reference/androidx/camera/core/ImageProxy +[4]: https://github.com/facebook/react-native/blob/9a43eac7a32a6ba3164a048960101022a92fcd5a/React/Base/RCTBridgeModule.h#L20-L24 diff --git a/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx b/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx index a315f0f..0aacb47 100644 --- a/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx +++ b/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx @@ -27,15 +27,18 @@ iOS Frame Processor Plugins can be written in either **Objective-C** or **Swift* 2. Create an Objective-C source file, for the QR Code Plugin this will be called `QRCodeFrameProcessorPlugin.m`. 3. Add the following code: -```objc {9} +```objc {11} #import +#import @interface QRCodeFrameProcessorPlugin : NSObject @end @implementation QRCodeFrameProcessorPlugin -static inline id scanQRCodes(CMSampleBufferRef buffer, NSArray args) { +static inline id scanQRCodes(Frame* frame, NSArray args) { + CMSampleBufferRef buffer = frame.buffer; + UIImageOrientation orientation = frame.orientation; // code goes here return @[]; } @@ -62,6 +65,7 @@ The JS function name will be equal to the Objective-C function name you choose ( ```objc #import +#import ``` 3. Create an Objective-C source file with the same name as the Swift file, for the QR Code Plugin this will be `QRCodeFrameProcessorPlugin.m`. Add the following code: @@ -79,12 +83,14 @@ The first parameter in the Macro specifies the JS function name. Make sure it is 4. In the Swift file, add the following code: -```swift {6} +```swift {8} @objc(QRCodeFrameProcessorPlugin) public class QRCodeFrameProcessorPlugin: NSObject, FrameProcessorPluginBase { @objc - public static func callback(_: CMSampleBuffer!, withArgs _: [Any]!) -> Any! { + public static func callback(_ frame: Frame!, withArgs _: [Any]!) -> Any! { + let buffer = frame.buffer + let orientation = frame.orientation // code goes here return [] } diff --git a/example/ios/Frame Processor Plugins/QR Code Plugin (Objective-C)/QRCodeFrameProcessorPluginObjC.m b/example/ios/Frame Processor Plugins/QR Code Plugin (Objective-C)/QRCodeFrameProcessorPluginObjC.m index fd0c1fd..dd254a4 100644 --- a/example/ios/Frame Processor Plugins/QR Code Plugin (Objective-C)/QRCodeFrameProcessorPluginObjC.m +++ b/example/ios/Frame Processor Plugins/QR Code Plugin (Objective-C)/QRCodeFrameProcessorPluginObjC.m @@ -7,7 +7,7 @@ #import #import -#import +#import // Example for an Objective-C Frame Processor plugin @@ -16,8 +16,8 @@ @implementation QRCodeFrameProcessorPluginObjC -static inline id exampleObjC___scanQRCodes(CMSampleBufferRef buffer, NSArray* arguments) { - // TODO: Use some AI to detect QR codes in the CMSampleBufferRef +static inline id exampleObjC___scanQRCodes(Frame* frame, NSArray* arguments) { + // TODO: Use some AI to detect QR codes in the frame return @[]; } diff --git a/example/ios/Frame Processor Plugins/QR Code Plugin (Swift)/QRCodeFrameProcessorPluginSwift.swift b/example/ios/Frame Processor Plugins/QR Code Plugin (Swift)/QRCodeFrameProcessorPluginSwift.swift index 0f78df1..84fd29d 100644 --- a/example/ios/Frame Processor Plugins/QR Code Plugin (Swift)/QRCodeFrameProcessorPluginSwift.swift +++ b/example/ios/Frame Processor Plugins/QR Code Plugin (Swift)/QRCodeFrameProcessorPluginSwift.swift @@ -12,7 +12,7 @@ import Vision @objc(QRCodeFrameProcessorPluginSwift) public class QRCodeFrameProcessorPluginSwift: NSObject, FrameProcessorPluginBase { @objc - public static func callback(_: CMSampleBuffer!, withArgs _: [Any]!) -> Any! { + public static func callback(_: Frame!, withArgs _: [Any]!) -> Any! { // TODO: Use some AI to detect QR codes in the CMSampleBufferRef [] } diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index cbc14f5..882737d 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -490,7 +490,7 @@ SPEC CHECKSUMS: RNReanimated: 9c13c86454bfd54dab7505c1a054470bfecd2563 RNStaticSafeAreaInsets: 6103cf09647fa427186d30f67b0f5163c1ae8252 RNVectorIcons: 31cebfcf94e8cf8686eb5303ae0357da64d7a5a4 - VisionCamera: 9886518481961e1c5d94cedb9b7513c28b8368c1 + VisionCamera: 60f51b9c8e5074fda9952a603311338039f7bf28 Yoga: 575c581c63e0d35c9a83f4b46d01d63abc1100ac PODFILE CHECKSUM: 4b093c1d474775c2eac3268011e4b0b80929d3a2 diff --git a/ios/CameraBridge.h b/ios/CameraBridge.h index aacc750..6f6c96f 100644 --- a/ios/CameraBridge.h +++ b/ios/CameraBridge.h @@ -15,6 +15,7 @@ #import "FrameProcessorCallback.h" #import "FrameProcessorRuntimeManager.h" +#import "Frame.h" #import "RCTBridge+runOnJS.h" #import "JSConsoleHelper.h" diff --git a/ios/CameraView+RecordVideo.swift b/ios/CameraView+RecordVideo.swift index 2412048..8588688 100644 --- a/ios/CameraView+RecordVideo.swift +++ b/ios/CameraView+RecordVideo.swift @@ -169,10 +169,12 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud } public final func captureOutput(_ captureOutput: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from _: AVCaptureConnection) { + // Video Recording runs in the same queue if isRecording { guard let recordingSession = recordingSession else { return invokeOnError(.capture(.unknown(message: "isRecording was true but the RecordingSession was null!"))) } + switch captureOutput { case is AVCaptureVideoDataOutput: recordingSession.appendBuffer(sampleBuffer, type: .video, timestamp: CMSampleBufferGetPresentationTimeStamp(sampleBuffer)) @@ -191,8 +193,10 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud let diff = DispatchTime.now().uptimeNanoseconds - lastFrameProcessorCall.uptimeNanoseconds let secondsPerFrame = 1.0 / frameProcessorFps.doubleValue let nanosecondsPerFrame = secondsPerFrame * 1_000_000_000.0 + if diff > UInt64(nanosecondsPerFrame) { - frameProcessor(sampleBuffer) + let frame = Frame(buffer: sampleBuffer, orientation: bufferOrientation) + frameProcessor(frame) lastFrameProcessorCall = DispatchTime.now() } } @@ -221,4 +225,32 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud return String(describing: reason) } #endif + + /** + Gets the orientation of the CameraView's images (CMSampleBuffers). + */ + var bufferOrientation: UIImage.Orientation { + guard let cameraPosition = videoDeviceInput?.device.position else { + return .up + } + + switch UIDevice.current.orientation { + case .portrait: + return cameraPosition == .front ? .leftMirrored : .right + + case .landscapeLeft: + return cameraPosition == .front ? .downMirrored : .up + + case .portraitUpsideDown: + return cameraPosition == .front ? .rightMirrored : .left + + case .landscapeRight: + return cameraPosition == .front ? .upMirrored : .down + + case .unknown, .faceUp, .faceDown: + fallthrough + @unknown default: + return .up + } + } } diff --git a/ios/Frame Processor/CMSampleBufferRefHolder.h b/ios/Frame Processor/CMSampleBufferRefHolder.h deleted file mode 100644 index 557b22b..0000000 --- a/ios/Frame Processor/CMSampleBufferRefHolder.h +++ /dev/null @@ -1,22 +0,0 @@ -// -// CMSampleBufferRefHolder.h -// VisionCamera -// -// Created by Marc Rousavy on 15.03.21. -// Copyright © 2021 mrousavy. All rights reserved. -// - -#pragma once - -#import -#import - -@interface CMSampleBufferRefHolder : NSObject { - CMSampleBufferRef buffer; -} - -- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer; - -@property (nonatomic) CMSampleBufferRef buffer; - -@end diff --git a/ios/Frame Processor/CMSampleBufferRefHolder.m b/ios/Frame Processor/CMSampleBufferRefHolder.m deleted file mode 100644 index 12c88d6..0000000 --- a/ios/Frame Processor/CMSampleBufferRefHolder.m +++ /dev/null @@ -1,25 +0,0 @@ -// -// CMSampleBufferRefHolder.m -// VisionCamera -// -// Created by Marc Rousavy on 08.06.21. -// Copyright © 2021 mrousavy. All rights reserved. -// - -#import "CMSampleBufferRefHolder.h" -#import -#import - -@implementation CMSampleBufferRefHolder - -- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer { - self = [super init]; - if (self) { - self.buffer = buffer; - } - return self; -} - -@synthesize buffer; - -@end diff --git a/ios/Frame Processor/Frame.h b/ios/Frame Processor/Frame.h new file mode 100644 index 0000000..70d2179 --- /dev/null +++ b/ios/Frame Processor/Frame.h @@ -0,0 +1,22 @@ +// +// Frame.h +// VisionCamera +// +// Created by Marc Rousavy on 15.03.21. +// Copyright © 2021 mrousavy. All rights reserved. +// + +#pragma once + +#import +#import +#import + +@interface Frame : NSObject + +- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer orientation:(UIImageOrientation)orientation; + +@property (nonatomic, readonly) CMSampleBufferRef buffer; +@property (nonatomic, readonly) UIImageOrientation orientation; + +@end diff --git a/ios/Frame Processor/Frame.m b/ios/Frame Processor/Frame.m new file mode 100644 index 0000000..25e9ff7 --- /dev/null +++ b/ios/Frame Processor/Frame.m @@ -0,0 +1,30 @@ +// +// Frame.m +// VisionCamera +// +// Created by Marc Rousavy on 08.06.21. +// Copyright © 2021 mrousavy. All rights reserved. +// + +#import "Frame.h" +#import +#import + +@implementation Frame { + CMSampleBufferRef buffer; + UIImageOrientation orientation; +} + +- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer orientation:(UIImageOrientation)orientation { + self = [super init]; + if (self) { + _buffer = buffer; + _orientation = orientation; + } + return self; +} + +@synthesize buffer = _buffer; +@synthesize orientation = _orientation; + +@end diff --git a/ios/Frame Processor/FrameHostObject.h b/ios/Frame Processor/FrameHostObject.h index 3e453b1..60d4a13 100644 --- a/ios/Frame Processor/FrameHostObject.h +++ b/ios/Frame Processor/FrameHostObject.h @@ -10,12 +10,13 @@ #import #import +#import "Frame.h" using namespace facebook; class JSI_EXPORT FrameHostObject: public jsi::HostObject { public: - explicit FrameHostObject(CMSampleBufferRef buffer): buffer(buffer) {} + explicit FrameHostObject(Frame* frame): frame(frame) {} ~FrameHostObject(); public: @@ -24,5 +25,5 @@ public: void destroyBuffer(); public: - CMSampleBufferRef buffer; + Frame* frame; }; diff --git a/ios/Frame Processor/FrameHostObject.mm b/ios/Frame Processor/FrameHostObject.mm index 48c6cd6..1d1ce94 100644 --- a/ios/Frame Processor/FrameHostObject.mm +++ b/ios/Frame Processor/FrameHostObject.mm @@ -37,7 +37,7 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr } if (name == "toString") { auto toString = [this] (jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value { - auto imageBuffer = CMSampleBufferGetImageBuffer(buffer); + auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer); auto width = CVPixelBufferGetWidth(imageBuffer); auto height = CVPixelBufferGetHeight(imageBuffer); @@ -48,30 +48,30 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr } if (name == "isValid") { - auto isValid = buffer != nil && CMSampleBufferIsValid(buffer); + auto isValid = frame != nil && CMSampleBufferIsValid(frame.buffer); return jsi::Value(isValid); } if (name == "isReady") { - auto isReady = buffer != nil && CMSampleBufferDataIsReady(buffer); + auto isReady = frame != nil && CMSampleBufferDataIsReady(frame.buffer); return jsi::Value(isReady); } if (name == "width") { - auto imageBuffer = CMSampleBufferGetImageBuffer(buffer); + auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer); auto width = CVPixelBufferGetWidth(imageBuffer); return jsi::Value((double) width); } if (name == "height") { - auto imageBuffer = CMSampleBufferGetImageBuffer(buffer); + auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer); auto height = CVPixelBufferGetHeight(imageBuffer); return jsi::Value((double) height); } if (name == "bytesPerRow") { - auto imageBuffer = CMSampleBufferGetImageBuffer(buffer); + auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer); auto bytesPerRow = CVPixelBufferGetPlaneCount(imageBuffer); return jsi::Value((double) bytesPerRow); } if (name == "planesCount") { - auto imageBuffer = CMSampleBufferGetImageBuffer(buffer); + auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer); auto planesCount = CVPixelBufferGetPlaneCount(imageBuffer); return jsi::Value((double) planesCount); } @@ -85,5 +85,5 @@ FrameHostObject::~FrameHostObject() { void FrameHostObject::destroyBuffer() { // ARC will hopefully delete it lol - this->buffer = nil; + this->frame = nil; } diff --git a/ios/Frame Processor/FrameProcessorCallback.h b/ios/Frame Processor/FrameProcessorCallback.h index 6e588c1..6fd0737 100644 --- a/ios/Frame Processor/FrameProcessorCallback.h +++ b/ios/Frame Processor/FrameProcessorCallback.h @@ -9,6 +9,6 @@ #pragma once #import -#import +#import "Frame.h" -typedef void (^FrameProcessorCallback) (CMSampleBufferRef buffer); +typedef void (^FrameProcessorCallback) (Frame* frame); diff --git a/ios/Frame Processor/FrameProcessorPlugin.h b/ios/Frame Processor/FrameProcessorPlugin.h index 21b1391..a2ccdcb 100644 --- a/ios/Frame Processor/FrameProcessorPlugin.h +++ b/ios/Frame Processor/FrameProcessorPlugin.h @@ -11,10 +11,10 @@ #import #import "FrameProcessorPluginRegistry.h" -#import +#import "Frame.h" @protocol FrameProcessorPluginBase -+ (id) callback:(CMSampleBufferRef)buffer withArgs:(NSArray*)args; ++ (id) callback:(Frame*)frame withArgs:(NSArray*)args; @end @@ -23,7 +23,7 @@ /** * Use this Macro to register the given function as a Frame Processor. - * * Make sure the given function is a C-style function with the following signature: static inline id callback(CMSampleBufferRef buffer) + * * Make sure the given function is a C-style function with the following signature: static inline id callback(Frame* frame, NSArray* args) * * Make sure the given function's name is unique across other frame processor plugins * * Make sure your frame processor returns a Value that can be converted to JS * * Make sure to use this Macro in an @implementation, not @interface @@ -35,8 +35,8 @@ \ +(void)load \ { \ - [FrameProcessorPluginRegistry addFrameProcessorPlugin:@"__" @ #frame_processor callback:^id(CMSampleBufferRef buffer, NSArray* args) { \ - return frame_processor(buffer, args); \ + [FrameProcessorPluginRegistry addFrameProcessorPlugin:@"__" @ #frame_processor callback:^id(Frame* frame, NSArray* args) { \ + return frame_processor(frame, args); \ }]; \ } @@ -55,8 +55,8 @@ objc_name : NSObject \ __attribute__((constructor)) static void VISION_CONCAT(initialize_, objc_name)() \ { \ - [FrameProcessorPluginRegistry addFrameProcessorPlugin:@"__" @ #name callback:^id(CMSampleBufferRef buffer, NSArray* args) { \ - return [objc_name callback:buffer withArgs:args]; \ + [FrameProcessorPluginRegistry addFrameProcessorPlugin:@"__" @ #name callback:^id(Frame* frame, NSArray* args) { \ + return [objc_name callback:frame withArgs:args]; \ }]; \ } diff --git a/ios/Frame Processor/FrameProcessorPluginRegistry.h b/ios/Frame Processor/FrameProcessorPluginRegistry.h index 4ac4c0a..cb72c1a 100644 --- a/ios/Frame Processor/FrameProcessorPluginRegistry.h +++ b/ios/Frame Processor/FrameProcessorPluginRegistry.h @@ -9,9 +9,9 @@ #pragma once #import -#import +#import "Frame.h" -typedef id (^FrameProcessorPlugin) (CMSampleBufferRef buffer, NSArray* arguments); +typedef id (^FrameProcessorPlugin) (Frame* frame, NSArray* arguments); @interface FrameProcessorPluginRegistry : NSObject diff --git a/ios/Frame Processor/FrameProcessorRuntimeManager.mm b/ios/Frame Processor/FrameProcessorRuntimeManager.mm index 70bfb04..7f7f7aa 100644 --- a/ios/Frame Processor/FrameProcessorRuntimeManager.mm +++ b/ios/Frame Processor/FrameProcessorRuntimeManager.mm @@ -85,15 +85,22 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView"))) NSLog(@"FrameProcessorBindings: Installing Frame Processor plugin \"%s\"...", pluginName); FrameProcessorPlugin callback = [[FrameProcessorPluginRegistry frameProcessorPlugins] valueForKey:pluginKey]; - auto function = [callback, callInvoker](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value { + auto function = [callback, callInvoker](jsi::Runtime& runtime, + const jsi::Value& thisValue, + const jsi::Value* arguments, + size_t count) -> jsi::Value { + auto frameHostObject = arguments[0].asObject(runtime).asHostObject(runtime); auto frame = static_cast(frameHostObject.get()); + auto args = convertJSICStyleArrayToNSArray(runtime, arguments + 1, // start at index 1 since first arg = Frame count - 1, // use smaller count callInvoker); - id result = callback(frame->buffer, args); + id result = callback(frame->frame, args); + return convertObjCObjectToJSIValue(runtime, result); + }; visionGlobal.setProperty(visionRuntime, pluginName, jsi::Function::createFromHostFunction(visionRuntime, @@ -129,7 +136,10 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView"))) NSLog(@"FrameProcessorBindings: Installing global functions..."); // setFrameProcessor(viewTag: number, frameProcessor: (frame: Frame) => void) - auto setFrameProcessor = [self](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value { + auto setFrameProcessor = [self](jsi::Runtime& runtime, + const jsi::Value& thisValue, + const jsi::Value* arguments, + size_t count) -> jsi::Value { NSLog(@"FrameProcessorBindings: Setting new frame processor..."); if (!arguments[0].isNumber()) throw jsi::JSError(runtime, "Camera::setFrameProcessor: First argument ('viewTag') must be a number!"); if (!arguments[1].isObject()) throw jsi::JSError(runtime, "Camera::setFrameProcessor: Second argument ('frameProcessor') must be a function!"); @@ -163,7 +173,10 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView"))) setFrameProcessor)); // unsetFrameProcessor(viewTag: number) - auto unsetFrameProcessor = [](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value { + auto unsetFrameProcessor = [](jsi::Runtime& runtime, + const jsi::Value& thisValue, + const jsi::Value* arguments, + size_t count) -> jsi::Value { NSLog(@"FrameProcessorBindings: Removing frame processor..."); if (!arguments[0].isNumber()) throw jsi::JSError(runtime, "Camera::unsetFrameProcessor: First argument ('viewTag') must be a number!"); auto viewTag = arguments[0].asNumber(); diff --git a/ios/Frame Processor/FrameProcessorUtils.mm b/ios/Frame Processor/FrameProcessorUtils.mm index 240ae1f..6690787 100644 --- a/ios/Frame Processor/FrameProcessorUtils.mm +++ b/ios/Frame Processor/FrameProcessorUtils.mm @@ -7,22 +7,22 @@ // #import "FrameProcessorUtils.h" -#import #import #import #import "FrameHostObject.h" +#import "Frame.h" FrameProcessorCallback convertJSIFunctionToFrameProcessorCallback(jsi::Runtime &runtime, const jsi::Function &value) { __block auto cb = value.getFunction(runtime); - return ^(CMSampleBufferRef buffer) { + return ^(Frame* frame) { #if DEBUG std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now(); #endif - auto frame = std::make_shared(buffer); + auto frameHostObject = std::make_shared(frame); try { - cb.call(runtime, jsi::Object::createFromHostObject(runtime, frame)); + cb.call(runtime, jsi::Object::createFromHostObject(runtime, frameHostObject)); } catch (jsi::JSError& jsError) { NSLog(@"Frame Processor threw an error: %s", jsError.getMessage().c_str()); } @@ -39,6 +39,6 @@ FrameProcessorCallback convertJSIFunctionToFrameProcessorCallback(jsi::Runtime & // 1. we are sure we don't need it anymore, the frame processor worklet has finished executing. // 2. we don't know when the JS runtime garbage collects this object, it might be holding it for a few more frames // which then blocks the camera queue from pushing new frames (memory limit) - frame->destroyBuffer(); + frameHostObject->destroyBuffer(); }; } diff --git a/ios/React Utils/JSIUtils.mm b/ios/React Utils/JSIUtils.mm index 359216b..21d643f 100644 --- a/ios/React Utils/JSIUtils.mm +++ b/ios/React Utils/JSIUtils.mm @@ -12,7 +12,7 @@ #import #import #import -#import "../Frame Processor/CMSampleBufferRefHolder.h" +#import "../Frame Processor/Frame.h" #import "../Frame Processor/FrameHostObject.h" using namespace facebook; @@ -68,11 +68,9 @@ jsi::Value convertObjCObjectToJSIValue(jsi::Runtime &runtime, id value) return convertNSArrayToJSIArray(runtime, (NSArray *)value); } else if (value == (id)kCFNull) { return jsi::Value::null(); - } else if ([value isKindOfClass:[CMSampleBufferRefHolder class]]) { - // it's boxed in a CMSampleBufferRefHolder because CMSampleBufferRef is not an NSObject - CMSampleBufferRef buffer = [(CMSampleBufferRefHolder*)value buffer]; - auto frame = std::make_shared(buffer); - return jsi::Object::createFromHostObject(runtime, frame); + } else if ([value isKindOfClass:[Frame class]]) { + auto frameHostObject = std::make_shared((Frame*)value); + return jsi::Object::createFromHostObject(runtime, frameHostObject); } return jsi::Value::undefined(); } @@ -155,7 +153,7 @@ id convertJSIValueToObjCObject(jsi::Runtime &runtime, const jsi::Value &value, s auto hostObject = o.asHostObject(runtime); auto frame = dynamic_cast(hostObject.get()); if (frame != nullptr) { - return [[CMSampleBufferRefHolder alloc] initWithBuffer:frame->buffer]; + return frame->frame; } } return convertJSIObjectToNSDictionary(runtime, o, jsInvoker); diff --git a/ios/VisionCamera.xcodeproj/project.pbxproj b/ios/VisionCamera.xcodeproj/project.pbxproj index e150642..c2956a8 100644 --- a/ios/VisionCamera.xcodeproj/project.pbxproj +++ b/ios/VisionCamera.xcodeproj/project.pbxproj @@ -81,7 +81,7 @@ B80E069F266632F000728644 /* AVAudioSession+updateCategory.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVAudioSession+updateCategory.swift"; sourceTree = ""; }; B8103E1B25FF553B007A1684 /* FrameProcessorUtils.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = FrameProcessorUtils.mm; sourceTree = ""; }; B8103E1E25FF5550007A1684 /* FrameProcessorUtils.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorUtils.h; sourceTree = ""; }; - B8103E5725FF56F0007A1684 /* CMSampleBufferRefHolder.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = CMSampleBufferRefHolder.h; sourceTree = ""; }; + B8103E5725FF56F0007A1684 /* Frame.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Frame.h; sourceTree = ""; }; B81D41EF263C86F900B041FD /* JSIUtils.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = JSIUtils.h; sourceTree = ""; }; B82FBA942614B69D00909718 /* RCTBridge+runOnJS.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "RCTBridge+runOnJS.h"; sourceTree = ""; }; B82FBA952614B69D00909718 /* RCTBridge+runOnJS.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = "RCTBridge+runOnJS.mm"; sourceTree = ""; }; @@ -140,7 +140,7 @@ B8DB3BCB263DC97E004C18D7 /* AVFileType+descriptor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVFileType+descriptor.swift"; sourceTree = ""; }; B8DCF09125EA7BEE00EA5C72 /* SpeedChecker.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SpeedChecker.h; sourceTree = ""; }; B8DCF14425EA817D00EA5C72 /* MakeJSIRuntime.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MakeJSIRuntime.h; sourceTree = ""; }; - B8F7DDD1266F715D00120533 /* CMSampleBufferRefHolder.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = CMSampleBufferRefHolder.m; sourceTree = ""; }; + B8F7DDD1266F715D00120533 /* Frame.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = Frame.m; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -263,8 +263,8 @@ B80D67A825FA25380008FE8D /* FrameProcessorCallback.h */, B8103E1E25FF5550007A1684 /* FrameProcessorUtils.h */, B8103E1B25FF553B007A1684 /* FrameProcessorUtils.mm */, - B8103E5725FF56F0007A1684 /* CMSampleBufferRefHolder.h */, - B8F7DDD1266F715D00120533 /* CMSampleBufferRefHolder.m */, + B8103E5725FF56F0007A1684 /* Frame.h */, + B8F7DDD1266F715D00120533 /* Frame.m */, B84760A22608EE38004C3180 /* FrameHostObject.h */, B84760A52608EE7C004C3180 /* FrameHostObject.mm */, B8A751D62609E4980011C623 /* FrameProcessorRuntimeManager.h */,