From bdd81cf2fb8c712b70b071deb5f5e0cbc5ba91f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20M=C4=99drek?= Date: Tue, 3 Oct 2023 11:36:55 +0200 Subject: [PATCH] chore: Improve native Frame Processor Plugin documentation (#1877) --- .github/workflows/build-ios.yml | 4 +- .../FRAME_PROCESSORS_CREATE_OVERVIEW.mdx | 24 +++++++---- .../guides/FRAME_PROCESSOR_CREATE_FINAL.mdx | 1 + .../FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx | 30 +++++++------ .../FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx | 43 ++++++++----------- 5 files changed, 54 insertions(+), 48 deletions(-) diff --git a/.github/workflows/build-ios.yml b/.github/workflows/build-ios.yml index 47e06d9..71ebaa0 100644 --- a/.github/workflows/build-ios.yml +++ b/.github/workflows/build-ios.yml @@ -56,7 +56,7 @@ jobs: uses: actions/cache@v3 with: path: | - example/ios/Pods + package/example/ios/Pods ~/Library/Caches/CocoaPods ~/.cocoapods key: ${{ runner.os }}-pods-${{ hashFiles('**/Podfile.lock') }} @@ -117,7 +117,7 @@ jobs: uses: actions/cache@v3 with: path: | - example/ios/Pods + package/example/ios/Pods ~/Library/Caches/CocoaPods ~/.cocoapods key: ${{ runner.os }}-pods-${{ hashFiles('**/Podfile.lock') }} diff --git a/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx b/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx index fb362b1..d14afa1 100644 --- a/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx +++ b/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx @@ -51,8 +51,9 @@ Similar to a TurboModule, the Frame Processor Plugin Registry API automatically Return values will automatically be converted to JS values, assuming they are representable in the ["Types" table](#types). So the following Java Frame Processor Plugin: ```java +@Nullable @Override -public Object callback(Frame frame, Object[] params) { +public Object callback(@NonNull Frame frame, @Nullable Map arguments) { return "cat"; } ``` @@ -70,8 +71,9 @@ export function detectObject(frame: Frame): string { You can also manipulate the buffer and return it (or a copy of it) by returning a [`Frame`][2]/[`Frame`][3] instance: ```java +@Nullable @Override -public Object callback(Frame frame, Object[] params) { +public Object callback(@NonNull Frame frame, @Nullable Map arguments) { Frame resizedFrame = new Frame(/* ... */); return resizedFrame; } @@ -107,12 +109,13 @@ const frameProcessor = useFrameProcessor((frame) => { To let the user know that something went wrong you can use Exceptions: ```java +@Nullable @Override -public Object callback(Frame frame, Object[] params) { - if (params[0] instanceof String) { +public Object callback(@NonNull Frame frame, @Nullable Map arguments) { + if (arguments != null && arguments.get("codeType") instanceof String) { // ... } else { - throw new Exception("First argument has to be a string!"); + throw new Exception("codeType property has to be a string!"); } } ``` @@ -123,7 +126,7 @@ Which will throw a JS-error: const frameProcessor = useFrameProcessor((frame) => { 'worklet' try { - const codes = scanCodes(frame, true) + const codes = scanCodes(frame, { codeType: 1234 }) } catch (e) { console.log(`Error: ${e.message}`) } @@ -143,9 +146,14 @@ If your Frame Processor takes longer than a single frame interval to execute, or For example, a realtime video chat application might use WebRTC to send the frames to the server. I/O operations (networking) are asynchronous, and we don't _need_ to wait for the upload to succeed before pushing the next frame, so we copy the frame and perform the upload on another Thread. ```java +@Nullable @Override -public Object callback(Frame frame, Object[] params) { - String serverURL = (String)params[0]; +public Object callback(@NonNull Frame frame, @Nullable Map arguments) { + if (arguments == null) { + return null; + } + + String serverURL = (String)arguments.get("serverURL"); Frame frameCopy = new Frame(/* ... */); uploaderQueue.runAsync(() -> { diff --git a/docs/docs/guides/FRAME_PROCESSOR_CREATE_FINAL.mdx b/docs/docs/guides/FRAME_PROCESSOR_CREATE_FINAL.mdx index c8ad4f6..8bfe5a7 100644 --- a/docs/docs/guides/FRAME_PROCESSOR_CREATE_FINAL.mdx +++ b/docs/docs/guides/FRAME_PROCESSOR_CREATE_FINAL.mdx @@ -18,6 +18,7 @@ const plugin = VisionCameraProxy.getFrameProcessorPlugin('scanFaces') */ export function scanFaces(frame: Frame): object { 'worklet' + if (plugin == null) throw new Error('Failed to load Frame Processor Plugin "scanFaces"!') return plugin.call(frame) } ``` diff --git a/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx b/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx index 5e4c104..a0b3702 100644 --- a/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx +++ b/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx @@ -34,7 +34,7 @@ For reference see the [CLI's docs](https://github.com/mateusz1913/vision-camera- protected List getPackages() { @SuppressWarnings("UnnecessaryLocalVariable") List packages = new PackageList(this).getPackages(); - ... + // ... // highlight-next-line packages.add(new FaceDetectorFrameProcessorPluginPackage()); // <- add return packages; @@ -56,13 +56,16 @@ For reference see the [CLI's docs](https://github.com/mateusz1913/vision-camera- 3. Add the following code: ```java +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; import com.mrousavy.camera.frameprocessor.Frame; import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin; public class FaceDetectorFrameProcessorPlugin extends FrameProcessorPlugin { - + @Nullable @Override - public Object callback(Frame frame, Map arguments) { + public Object callback(@NonNull Frame frame, @Nullable Map arguments) { +// highlight-next-line // code goes here return null; } @@ -73,20 +76,20 @@ public class FaceDetectorFrameProcessorPlugin extends FrameProcessorPlugin { 5. Create a new Java file which registers the Frame Processor Plugin in a React Package, for the Face Detector plugin this file will be called `FaceDetectorFrameProcessorPluginPackage.java`: ```java +import androidx.annotation.NonNull; import com.facebook.react.ReactPackage; import com.facebook.react.bridge.NativeModule; import com.facebook.react.bridge.ReactApplicationContext; import com.facebook.react.uimanager.ViewManager; import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin; import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry; -import javax.annotation.Nonnull; public class FaceDetectorFrameProcessorPluginPackage implements ReactPackage { + // highlight-start FaceDetectorFrameProcessorPluginPackage() { - // highlight-start FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces", options -> new FaceDetectorFrameProcessorPlugin()); - // highlight-end } + // highlight-end @NonNull @Override @@ -94,9 +97,9 @@ public class FaceDetectorFrameProcessorPluginPackage implements ReactPackage { return Collections.emptyList(); } - @Nonnull + @NonNull @Override - public List createViewManagers(@Nonnull ReactApplicationContext reactContext) { + public List createViewManagers(@NonNull ReactApplicationContext reactContext) { return Collections.emptyList(); } } @@ -113,7 +116,7 @@ The Frame Processor Plugin will be exposed to JS through the `VisionCameraProxy` protected List getPackages() { @SuppressWarnings("UnnecessaryLocalVariable") List packages = new PackageList(this).getPackages(); - ... + // ... // highlight-next-line packages.add(new FaceDetectorFrameProcessorPluginPackage()); // <- add return packages; @@ -133,7 +136,8 @@ import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin class FaceDetectorFrameProcessorPlugin: FrameProcessorPlugin() { - override fun callback(frame: Frame, arguments: Map): Any? { + override fun callback(frame: Frame, arguments: Map?): Any? { +// highlight-next-line // code goes here return null } @@ -151,13 +155,13 @@ import com.facebook.react.uimanager.ViewManager import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin class FaceDetectorFrameProcessorPluginPackage : ReactPackage { + // highlight-start init { - // highlight-start FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces") { options -> FaceDetectorFrameProcessorPlugin() } - // highlight-end } + // highlight-end override fun createNativeModules(reactContext: ReactApplicationContext): List { return emptyList() @@ -180,7 +184,7 @@ The Frame Processor Plugin will be exposed to JS through the `VisionCameraProxy` protected List getPackages() { @SuppressWarnings("UnnecessaryLocalVariable") List packages = new PackageList(this).getPackages(); - ... + // ... // highlight-next-line packages.add(new FaceDetectorFrameProcessorPluginPackage()); // <- add return packages; diff --git a/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx b/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx index f608c1f..c49352b 100644 --- a/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx +++ b/docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx @@ -60,7 +60,7 @@ For reference see the [CLI's docs](https://github.com/mateusz1913/vision-camera- CMSampleBufferRef buffer = frame.buffer; UIImageOrientation orientation = frame.orientation; // code goes here - return @[]; + return nil; } + (void) load { @@ -89,56 +89,49 @@ The Frame Processor Plugin will be exposed to JS through the `VisionCameraProxy` ![Xcode "Create Bridging Header" alert](https://docs-assets.developer.apple.com/published/7ebca7212c/2a065d1a-7e53-4907-a889-b7fa4f2206c9.png) -3. Inside the newly created Bridging Header, add the following code: - -```objc -#import -#import -``` - -4. In the Swift file, add the following code: +3. In the Swift file, add the following code: ```swift +import VisionCamera + @objc(FaceDetectorFrameProcessorPlugin) public class FaceDetectorFrameProcessorPlugin: FrameProcessorPlugin { - - public override func callback(_ frame: Frame!, - withArguments arguments: [String:Any]) -> Any { + public override func callback(_ frame: Frame, withArguments arguments: [AnyHashable : Any]?) -> Any { let buffer = frame.buffer let orientation = frame.orientation // code goes here - return [] + return nil } } ``` -5. In your `AppDelegate.m`, add the following imports: +4. Create an Objective-C source file that will be used to automatically register your plugin ```objc -#import "YOUR_XCODE_PROJECT_NAME-Swift.h" #import #import -``` -6. In your `AppDelegate.m`, add the following code to `application:didFinishLaunchingWithOptions:`: +#import "YOUR_XCODE_PROJECT_NAME-Swift.h" // <--- replace "YOUR_XCODE_PROJECT_NAME" with the actual value of your xcode project name -```objc -- (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)launchOptions +@interface FaceDetectorFrameProcessorPlugin (FrameProcessorPluginLoader) +@end + +@implementation FaceDetectorFrameProcessorPlugin (FrameProcessorPluginLoader) + ++ (void)load { - // ... - // highlight-start [FrameProcessorPluginRegistry addFrameProcessorPlugin:@"detectFaces" - withInitializer:^FrameProcessorPlugin*(NSDictionary* options) { + withInitializer:^FrameProcessorPlugin* (NSDictionary* options) { return [[FaceDetectorFrameProcessorPlugin alloc] initWithOptions:options]; }]; // highlight-end - - return [super application:application didFinishLaunchingWithOptions:launchOptions]; } + +@end ``` -7. **Implement your frame processing.** See [Example Plugin (Swift)](https://github.com/mrousavy/react-native-vision-camera/blob/main/package/example/ios/Frame%20Processor%20Plugins/Example%20Plugin%20%28Swift%29) for reference. +5. **Implement your frame processing.** See [Example Plugin (Swift)](https://github.com/mrousavy/react-native-vision-camera/blob/main/package/example/ios/Frame%20Processor%20Plugins/Example%20Plugin%20%28Swift%29) for reference.