From a291642c53def738a06a8fd2b910870fa26a7779 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20M=C4=99drek?= Date: Thu, 19 Oct 2023 10:35:14 +0200 Subject: [PATCH] feat: Reintroduce Macros for Frame Processor Plugin registration (#2027) in VisionCamera v1 & v2 there were two ObjC macros that were helping in creation/registration of Frame Processors, but these were removed with v3 This PR reintroduces such macros, which will not only make FP development easier, but also it will also fix issues people had with registration of Swift Frame Processors (+load vs +initialize issues) Docs were also updated to reflect that the macros should be used to correctly initialize and register ObjC/Swift Frame Processors --- .../FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx | 10 +++--- .../FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx | 36 ++++++------------- .../frameprocessor/FrameProcessorPlugin.java | 8 +++++ .../example/ExampleFrameProcessorPlugin.java | 4 +-- .../ExampleKotlinFrameProcessorPlugin.kt | 6 +++- .../camera/example/MainApplication.java | 4 +-- .../ExampleFrameProcessorPlugin.m | 14 ++++---- .../ExampleSwiftFrameProcessor.m | 18 ++-------- .../ExampleSwiftFrameProcessor.swift | 6 ++++ package/example/ios/Podfile.lock | 4 +-- package/example/src/CameraPage.tsx | 2 ++ .../ExampleKotlinSwiftPlugin.ts | 2 +- .../Frame Processor/FrameProcessorPlugin.h | 31 ++++++++++++++++ .../Frame Processor/FrameProcessorPlugin.m | 5 +++ package/src/FrameProcessorPlugins.ts | 2 +- 15 files changed, 92 insertions(+), 60 deletions(-) diff --git a/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx b/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx index a0b3702..d9f7bd7 100644 --- a/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx +++ b/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx @@ -62,6 +62,8 @@ import com.mrousavy.camera.frameprocessor.Frame; import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin; public class FaceDetectorFrameProcessorPlugin extends FrameProcessorPlugin { + FaceDetectorFrameProcessorPlugin(@Nullable Map options) {} + @Nullable @Override public Object callback(@NonNull Frame frame, @Nullable Map arguments) { @@ -87,7 +89,7 @@ import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry; public class FaceDetectorFrameProcessorPluginPackage implements ReactPackage { // highlight-start FaceDetectorFrameProcessorPluginPackage() { - FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces", options -> new FaceDetectorFrameProcessorPlugin()); + FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces", options -> new FaceDetectorFrameProcessorPlugin(options)); } // highlight-end @@ -134,9 +136,9 @@ The Frame Processor Plugin will be exposed to JS through the `VisionCameraProxy` import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin -class FaceDetectorFrameProcessorPlugin: FrameProcessorPlugin() { +class FaceDetectorFrameProcessorPlugin(options: Map?): FrameProcessorPlugin(options) { - override fun callback(frame: Frame, arguments: Map?): Any? { + override fun callback(frame: Frame, arguments: Map?): Any? { // highlight-next-line // code goes here return null @@ -158,7 +160,7 @@ class FaceDetectorFrameProcessorPluginPackage : ReactPackage { // highlight-start init { FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces") { options -> - FaceDetectorFrameProcessorPlugin() + FaceDetectorFrameProcessorPlugin(options) } } // highlight-end diff --git a/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx b/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx index c49352b..c35b538 100644 --- a/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx +++ b/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx @@ -52,7 +52,7 @@ For reference see the [CLI's docs](https://github.com/mateusz1913/vision-camera- @implementation FaceDetectorFrameProcessorPlugin - (instancetype) initWithOptions:(NSDictionary*)options; { - self = [super init]; + self = [super initWithOptions:options]; return self; } @@ -63,14 +63,9 @@ For reference see the [CLI's docs](https://github.com/mateusz1913/vision-camera- return nil; } -+ (void) load { - // highlight-start - [FrameProcessorPluginRegistry addFrameProcessorPlugin:@"detectFaces" - withInitializer:^FrameProcessorPlugin*(NSDictionary* options) { - return [[FaceDetectorFrameProcessorPlugin alloc] initWithOptions:options]; - }]; - // highlight-end -} +// highlight-start +VISION_EXPORT_FRAME_PROCESSOR(FaceDetectorFrameProcessorPlugin, detectFaces) +// highlight-end @end ``` @@ -96,6 +91,10 @@ import VisionCamera @objc(FaceDetectorFrameProcessorPlugin) public class FaceDetectorFrameProcessorPlugin: FrameProcessorPlugin { + public override init(options: [AnyHashable : Any]! = [:]) { + super.init(options: options) + } + public override func callback(_ frame: Frame, withArguments arguments: [AnyHashable : Any]?) -> Any { let buffer = frame.buffer let orientation = frame.orientation @@ -113,22 +112,9 @@ public class FaceDetectorFrameProcessorPlugin: FrameProcessorPlugin { #import "YOUR_XCODE_PROJECT_NAME-Swift.h" // <--- replace "YOUR_XCODE_PROJECT_NAME" with the actual value of your xcode project name -@interface FaceDetectorFrameProcessorPlugin (FrameProcessorPluginLoader) -@end - -@implementation FaceDetectorFrameProcessorPlugin (FrameProcessorPluginLoader) - -+ (void)load -{ - // highlight-start - [FrameProcessorPluginRegistry addFrameProcessorPlugin:@"detectFaces" - withInitializer:^FrameProcessorPlugin* (NSDictionary* options) { - return [[FaceDetectorFrameProcessorPlugin alloc] initWithOptions:options]; - }]; - // highlight-end -} - -@end +// highlight-start +VISION_EXPORT_SWIFT_FRAME_PROCESSOR(FaceDetectorFrameProcessorPlugin, detectFaces) +// highlight-end ``` 5. **Implement your frame processing.** See [Example Plugin (Swift)](https://github.com/mrousavy/react-native-vision-camera/blob/main/package/example/ios/Frame%20Processor%20Plugins/Example%20Plugin%20%28Swift%29) for reference. diff --git a/package/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java b/package/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java index 4793706..c7455de 100644 --- a/package/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java +++ b/package/android/src/main/java/com/mrousavy/camera/frameprocessor/FrameProcessorPlugin.java @@ -12,6 +12,14 @@ import java.util.Map; @DoNotStrip @Keep public abstract class FrameProcessorPlugin { + public FrameProcessorPlugin() {} + + /** + * The initializer for a Frame Processor Plugin class that takes optional object that consists + * options passed from JS layer + */ + public FrameProcessorPlugin(@Nullable Map options) {} + /** * The actual Frame Processor plugin callback. Called for every frame the ImageAnalyzer receives. * @param frame The Frame from the Camera. Don't call .close() on this, as VisionCamera handles that. diff --git a/package/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java b/package/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java index d877807..40d585d 100644 --- a/package/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java +++ b/package/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java @@ -40,7 +40,7 @@ public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin { return map; } - ExampleFrameProcessorPlugin() { - + ExampleFrameProcessorPlugin(@Nullable Map options) { + Log.d("ExamplePlugin", " - options: " + options.toString()); } } diff --git a/package/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleKotlinFrameProcessorPlugin.kt b/package/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleKotlinFrameProcessorPlugin.kt index 5afa023..c6b8ebf 100644 --- a/package/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleKotlinFrameProcessorPlugin.kt +++ b/package/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleKotlinFrameProcessorPlugin.kt @@ -4,7 +4,11 @@ import android.util.Log import com.mrousavy.camera.frameprocessor.Frame import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin -class ExampleKotlinFrameProcessorPlugin: FrameProcessorPlugin() { +class ExampleKotlinFrameProcessorPlugin(options: Map?): FrameProcessorPlugin(options) { + init { + Log.d("ExampleKotlinPlugin", " - options" + options?.toString()) + } + override fun callback(frame: Frame, params: Map?): Any? { if (params == null) { return null diff --git a/package/example/android/app/src/main/java/com/mrousavy/camera/example/MainApplication.java b/package/example/android/app/src/main/java/com/mrousavy/camera/example/MainApplication.java index 1d9ace4..ba52164 100644 --- a/package/example/android/app/src/main/java/com/mrousavy/camera/example/MainApplication.java +++ b/package/example/android/app/src/main/java/com/mrousavy/camera/example/MainApplication.java @@ -65,7 +65,7 @@ public class MainApplication extends Application implements ReactApplication { DefaultNewArchitectureEntryPoint.load(); } - FrameProcessorPluginRegistry.addFrameProcessorPlugin("example_plugin", options -> new ExampleFrameProcessorPlugin()); - FrameProcessorPluginRegistry.addFrameProcessorPlugin("example_kotlin_swift_plugin", options -> new ExampleKotlinFrameProcessorPlugin()); + FrameProcessorPluginRegistry.addFrameProcessorPlugin("example_plugin", options -> new ExampleFrameProcessorPlugin(options)); + FrameProcessorPluginRegistry.addFrameProcessorPlugin("example_kotlin_swift_plugin", options -> new ExampleKotlinFrameProcessorPlugin(options)); } } diff --git a/package/example/ios/Frame Processor Plugins/Example Plugin/ExampleFrameProcessorPlugin.m b/package/example/ios/Frame Processor Plugins/Example Plugin/ExampleFrameProcessorPlugin.m index e50d3e3..24d78a7 100644 --- a/package/example/ios/Frame Processor Plugins/Example Plugin/ExampleFrameProcessorPlugin.m +++ b/package/example/ios/Frame Processor Plugins/Example Plugin/ExampleFrameProcessorPlugin.m @@ -18,6 +18,13 @@ @implementation ExampleFrameProcessorPlugin +- (instancetype)initWithOptions:(NSDictionary * _Nullable)options +{ + self = [super initWithOptions:options]; + NSLog(@"ExamplePlugin - options: %@", options); + return self; +} + - (id)callback:(Frame *)frame withArguments:(NSDictionary *)arguments { CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer); NSLog(@"ExamplePlugin: %zu x %zu Image. Logging %lu parameters:", CVPixelBufferGetWidth(imageBuffer), CVPixelBufferGetHeight(imageBuffer), (unsigned long)arguments.count); @@ -38,12 +45,7 @@ }; } -+ (void) load { - [FrameProcessorPluginRegistry addFrameProcessorPlugin:@"example_plugin" - withInitializer:^FrameProcessorPlugin*(NSDictionary* options) { - return [[ExampleFrameProcessorPlugin alloc] init]; - }]; -} +VISION_EXPORT_FRAME_PROCESSOR(ExampleFrameProcessorPlugin, example_plugin) @end #endif diff --git a/package/example/ios/Frame Processor Plugins/Example Swift Plugin/ExampleSwiftFrameProcessor.m b/package/example/ios/Frame Processor Plugins/Example Swift Plugin/ExampleSwiftFrameProcessor.m index b2e0454..ac36332 100644 --- a/package/example/ios/Frame Processor Plugins/Example Swift Plugin/ExampleSwiftFrameProcessor.m +++ b/package/example/ios/Frame Processor Plugins/Example Swift Plugin/ExampleSwiftFrameProcessor.m @@ -6,26 +6,12 @@ // #if __has_include() -#import #import #import -#import #import "VisionCameraExample-Swift.h" -// Example for a Swift Frame Processor plugin automatic registration -@interface ExampleSwiftFrameProcessorPlugin (FrameProcessorPluginLoader) -@end - -@implementation ExampleSwiftFrameProcessorPlugin (FrameProcessorPluginLoader) - -+ (void)initialize { - [FrameProcessorPluginRegistry addFrameProcessorPlugin:@"example_kotlin_swift_plugin" - withInitializer:^FrameProcessorPlugin* _Nonnull(NSDictionary* _Nullable options) { - return [[ExampleSwiftFrameProcessorPlugin alloc] init]; - }]; -} - -@end +// // Example for a Swift Frame Processor plugin automatic registration +VISION_EXPORT_SWIFT_FRAME_PROCESSOR(ExampleSwiftFrameProcessorPlugin, example_kotlin_swift_plugin) #endif diff --git a/package/example/ios/Frame Processor Plugins/Example Swift Plugin/ExampleSwiftFrameProcessor.swift b/package/example/ios/Frame Processor Plugins/Example Swift Plugin/ExampleSwiftFrameProcessor.swift index 92222ed..92fe376 100644 --- a/package/example/ios/Frame Processor Plugins/Example Swift Plugin/ExampleSwiftFrameProcessor.swift +++ b/package/example/ios/Frame Processor Plugins/Example Swift Plugin/ExampleSwiftFrameProcessor.swift @@ -11,6 +11,12 @@ import VisionCamera // Example for a Swift Frame Processor plugin @objc(ExampleSwiftFrameProcessorPlugin) public class ExampleSwiftFrameProcessorPlugin: FrameProcessorPlugin { + public override init(options: [AnyHashable: Any]! = [:]) { + super.init(options: options) + + print("ExampleSwiftPlugin - options: \(String(describing: options))") + } + public override func callback(_ frame: Frame, withArguments arguments: [AnyHashable: Any]?) -> Any? { let imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer) diff --git a/package/example/ios/Podfile.lock b/package/example/ios/Podfile.lock index 9e453ce..2889596 100644 --- a/package/example/ios/Podfile.lock +++ b/package/example/ios/Podfile.lock @@ -507,7 +507,7 @@ PODS: - libwebp (~> 1.0) - SDWebImage/Core (~> 5.10) - SocketRocket (0.6.1) - - VisionCamera (3.3.1): + - VisionCamera (3.4.0): - React - React-callinvoker - React-Core @@ -747,7 +747,7 @@ SPEC CHECKSUMS: SDWebImage: a7f831e1a65eb5e285e3fb046a23fcfbf08e696d SDWebImageWebPCoder: 908b83b6adda48effe7667cd2b7f78c897e5111d SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17 - VisionCamera: f386aee60abb07d979c506ea9e6d4831e596cafe + VisionCamera: eead9df29ac5935d5685b5ecaea3ae8b6da84bff Yoga: 8796b55dba14d7004f980b54bcc9833ee45b28ce PODFILE CHECKSUM: 27f53791141a3303d814e09b55770336416ff4eb diff --git a/package/example/src/CameraPage.tsx b/package/example/src/CameraPage.tsx index da1f850..cc1d836 100644 --- a/package/example/src/CameraPage.tsx +++ b/package/example/src/CameraPage.tsx @@ -17,6 +17,7 @@ import type { Routes } from './Routes' import type { NativeStackScreenProps } from '@react-navigation/native-stack' import { useIsFocused } from '@react-navigation/core' import { examplePlugin } from './frame-processors/ExamplePlugin' +import { exampleKotlinSwiftPlugin } from './frame-processors/ExampleKotlinSwiftPlugin' import { usePreferredCameraDevice } from './hooks/usePreferredCameraDevice' const ReanimatedCamera = Reanimated.createAnimatedComponent(Camera) @@ -166,6 +167,7 @@ export function CameraPage({ navigation }: Props): React.ReactElement { console.log(`${frame.timestamp}: ${frame.width}x${frame.height} ${frame.pixelFormat} Frame (${frame.orientation})`) examplePlugin(frame) + exampleKotlinSwiftPlugin(frame) }, []) return ( diff --git a/package/example/src/frame-processors/ExampleKotlinSwiftPlugin.ts b/package/example/src/frame-processors/ExampleKotlinSwiftPlugin.ts index 5d790a4..4b07af6 100644 --- a/package/example/src/frame-processors/ExampleKotlinSwiftPlugin.ts +++ b/package/example/src/frame-processors/ExampleKotlinSwiftPlugin.ts @@ -1,6 +1,6 @@ import { VisionCameraProxy, Frame } from 'react-native-vision-camera' -const plugin = VisionCameraProxy.getFrameProcessorPlugin('example_kotlin_swift_plugin') +const plugin = VisionCameraProxy.getFrameProcessorPlugin('example_kotlin_swift_plugin', { foo: 'bar' }) export function exampleKotlinSwiftPlugin(frame: Frame): string[] { 'worklet' diff --git a/package/ios/Frame Processor/FrameProcessorPlugin.h b/package/ios/Frame Processor/FrameProcessorPlugin.h index da4eb24..de7d818 100644 --- a/package/ios/Frame Processor/FrameProcessorPlugin.h +++ b/package/ios/Frame Processor/FrameProcessorPlugin.h @@ -21,8 +21,39 @@ /// VisionCamera Runtime. @interface FrameProcessorPlugin : NSObject +/// The initializer for a Frame Processor Plugin class that takes optional object that consists +/// options passed from JS layer +- (instancetype _Nonnull)initWithOptions:(NSDictionary* _Nullable)options; + /// The actual callback when calling this plugin. Any Frame Processing should be handled there. /// Make sure your code is optimized, as this is a hot path. - (id _Nullable)callback:(Frame* _Nonnull)frame withArguments:(NSDictionary* _Nullable)arguments; @end + +#define VISION_CONCAT2(A, B) A##B +#define VISION_CONCAT(A, B) VISION_CONCAT2(A, B) + +#define VISION_EXPORT_FRAME_PROCESSOR(frame_processor_class, frame_processor_plugin_name) \ + +(void)load { \ + [FrameProcessorPluginRegistry addFrameProcessorPlugin:@ #frame_processor_plugin_name \ + withInitializer:^FrameProcessorPlugin*(NSDictionary* _Nullable options) { \ + return [[frame_processor_class alloc] initWithOptions:options]; \ + }]; \ + } + +#define VISION_EXPORT_SWIFT_FRAME_PROCESSOR(frame_processor_class, frame_processor_plugin_name) \ + \ + @interface frame_processor_class (FrameProcessorPluginLoader) \ + @end \ + \ + @implementation frame_processor_class (FrameProcessorPluginLoader) \ + \ + __attribute__((constructor)) static void VISION_CONCAT(initialize_, frame_processor_plugin_name)(void) { \ + [FrameProcessorPluginRegistry addFrameProcessorPlugin:@ #frame_processor_plugin_name \ + withInitializer:^FrameProcessorPlugin* _Nonnull(NSDictionary* _Nullable options) { \ + return [[frame_processor_class alloc] initWithOptions:options]; \ + }]; \ + } \ + \ + @end diff --git a/package/ios/Frame Processor/FrameProcessorPlugin.m b/package/ios/Frame Processor/FrameProcessorPlugin.m index 6f504eb..2f61726 100644 --- a/package/ios/Frame Processor/FrameProcessorPlugin.m +++ b/package/ios/Frame Processor/FrameProcessorPlugin.m @@ -11,6 +11,11 @@ // Base implementation (empty) @implementation FrameProcessorPlugin +- (instancetype)initWithOptions:(NSDictionary* _Nullable)options { + self = [super init]; + return self; +} + - (id _Nullable)callback:(Frame* _Nonnull)frame withArguments:(NSDictionary* _Nullable)arguments { [NSException raise:NSInternalInconsistencyException format:@"Frame Processor Plugin does not override the `callback(frame:withArguments:)` method!"]; diff --git a/package/src/FrameProcessorPlugins.ts b/package/src/FrameProcessorPlugins.ts index 606f4a5..c190421 100644 --- a/package/src/FrameProcessorPlugins.ts +++ b/package/src/FrameProcessorPlugins.ts @@ -27,7 +27,7 @@ interface TVisionCameraProxy { * Creates a new instance of a Frame Processor Plugin. * The Plugin has to be registered on the native side, otherwise this returns `undefined` */ - getFrameProcessorPlugin: (name: string) => FrameProcessorPlugin | undefined + getFrameProcessorPlugin: (name: string, options?: Record) => FrameProcessorPlugin | undefined } let hasWorklets = false