chore: Improve native Frame Processor Plugin documentation (#1877)

This commit is contained in:
Mateusz Mędrek 2023-10-03 11:36:55 +02:00 committed by GitHub
parent 62e786ad04
commit bdd81cf2fb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 54 additions and 48 deletions

View File

@ -56,7 +56,7 @@ jobs:
uses: actions/cache@v3 uses: actions/cache@v3
with: with:
path: | path: |
example/ios/Pods package/example/ios/Pods
~/Library/Caches/CocoaPods ~/Library/Caches/CocoaPods
~/.cocoapods ~/.cocoapods
key: ${{ runner.os }}-pods-${{ hashFiles('**/Podfile.lock') }} key: ${{ runner.os }}-pods-${{ hashFiles('**/Podfile.lock') }}
@ -117,7 +117,7 @@ jobs:
uses: actions/cache@v3 uses: actions/cache@v3
with: with:
path: | path: |
example/ios/Pods package/example/ios/Pods
~/Library/Caches/CocoaPods ~/Library/Caches/CocoaPods
~/.cocoapods ~/.cocoapods
key: ${{ runner.os }}-pods-${{ hashFiles('**/Podfile.lock') }} key: ${{ runner.os }}-pods-${{ hashFiles('**/Podfile.lock') }}

View File

@ -51,8 +51,9 @@ Similar to a TurboModule, the Frame Processor Plugin Registry API automatically
Return values will automatically be converted to JS values, assuming they are representable in the ["Types" table](#types). So the following Java Frame Processor Plugin: Return values will automatically be converted to JS values, assuming they are representable in the ["Types" table](#types). So the following Java Frame Processor Plugin:
```java ```java
@Nullable
@Override @Override
public Object callback(Frame frame, Object[] params) { public Object callback(@NonNull Frame frame, @Nullable Map<String, Object> arguments) {
return "cat"; return "cat";
} }
``` ```
@ -70,8 +71,9 @@ export function detectObject(frame: Frame): string {
You can also manipulate the buffer and return it (or a copy of it) by returning a [`Frame`][2]/[`Frame`][3] instance: You can also manipulate the buffer and return it (or a copy of it) by returning a [`Frame`][2]/[`Frame`][3] instance:
```java ```java
@Nullable
@Override @Override
public Object callback(Frame frame, Object[] params) { public Object callback(@NonNull Frame frame, @Nullable Map<String, Object> arguments) {
Frame resizedFrame = new Frame(/* ... */); Frame resizedFrame = new Frame(/* ... */);
return resizedFrame; return resizedFrame;
} }
@ -107,12 +109,13 @@ const frameProcessor = useFrameProcessor((frame) => {
To let the user know that something went wrong you can use Exceptions: To let the user know that something went wrong you can use Exceptions:
```java ```java
@Nullable
@Override @Override
public Object callback(Frame frame, Object[] params) { public Object callback(@NonNull Frame frame, @Nullable Map<String, Object> arguments) {
if (params[0] instanceof String) { if (arguments != null && arguments.get("codeType") instanceof String) {
// ... // ...
} else { } else {
throw new Exception("First argument has to be a string!"); throw new Exception("codeType property has to be a string!");
} }
} }
``` ```
@ -123,7 +126,7 @@ Which will throw a JS-error:
const frameProcessor = useFrameProcessor((frame) => { const frameProcessor = useFrameProcessor((frame) => {
'worklet' 'worklet'
try { try {
const codes = scanCodes(frame, true) const codes = scanCodes(frame, { codeType: 1234 })
} catch (e) { } catch (e) {
console.log(`Error: ${e.message}`) console.log(`Error: ${e.message}`)
} }
@ -143,9 +146,14 @@ If your Frame Processor takes longer than a single frame interval to execute, or
For example, a realtime video chat application might use WebRTC to send the frames to the server. I/O operations (networking) are asynchronous, and we don't _need_ to wait for the upload to succeed before pushing the next frame, so we copy the frame and perform the upload on another Thread. For example, a realtime video chat application might use WebRTC to send the frames to the server. I/O operations (networking) are asynchronous, and we don't _need_ to wait for the upload to succeed before pushing the next frame, so we copy the frame and perform the upload on another Thread.
```java ```java
@Nullable
@Override @Override
public Object callback(Frame frame, Object[] params) { public Object callback(@NonNull Frame frame, @Nullable Map<String, Object> arguments) {
String serverURL = (String)params[0]; if (arguments == null) {
return null;
}
String serverURL = (String)arguments.get("serverURL");
Frame frameCopy = new Frame(/* ... */); Frame frameCopy = new Frame(/* ... */);
uploaderQueue.runAsync(() -> { uploaderQueue.runAsync(() -> {

View File

@ -18,6 +18,7 @@ const plugin = VisionCameraProxy.getFrameProcessorPlugin('scanFaces')
*/ */
export function scanFaces(frame: Frame): object { export function scanFaces(frame: Frame): object {
'worklet' 'worklet'
if (plugin == null) throw new Error('Failed to load Frame Processor Plugin "scanFaces"!')
return plugin.call(frame) return plugin.call(frame)
} }
``` ```

View File

@ -34,7 +34,7 @@ For reference see the [CLI's docs](https://github.com/mateusz1913/vision-camera-
protected List<ReactPackage> getPackages() { protected List<ReactPackage> getPackages() {
@SuppressWarnings("UnnecessaryLocalVariable") @SuppressWarnings("UnnecessaryLocalVariable")
List<ReactPackage> packages = new PackageList(this).getPackages(); List<ReactPackage> packages = new PackageList(this).getPackages();
... // ...
// highlight-next-line // highlight-next-line
packages.add(new FaceDetectorFrameProcessorPluginPackage()); // <- add packages.add(new FaceDetectorFrameProcessorPluginPackage()); // <- add
return packages; return packages;
@ -56,13 +56,16 @@ For reference see the [CLI's docs](https://github.com/mateusz1913/vision-camera-
3. Add the following code: 3. Add the following code:
```java ```java
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import com.mrousavy.camera.frameprocessor.Frame; import com.mrousavy.camera.frameprocessor.Frame;
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin; import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin;
public class FaceDetectorFrameProcessorPlugin extends FrameProcessorPlugin { public class FaceDetectorFrameProcessorPlugin extends FrameProcessorPlugin {
@Nullable
@Override @Override
public Object callback(Frame frame, Map<String, Object> arguments) { public Object callback(@NonNull Frame frame, @Nullable Map<String, Object> arguments) {
// highlight-next-line
// code goes here // code goes here
return null; return null;
} }
@ -73,20 +76,20 @@ public class FaceDetectorFrameProcessorPlugin extends FrameProcessorPlugin {
5. Create a new Java file which registers the Frame Processor Plugin in a React Package, for the Face Detector plugin this file will be called `FaceDetectorFrameProcessorPluginPackage.java`: 5. Create a new Java file which registers the Frame Processor Plugin in a React Package, for the Face Detector plugin this file will be called `FaceDetectorFrameProcessorPluginPackage.java`:
```java ```java
import androidx.annotation.NonNull;
import com.facebook.react.ReactPackage; import com.facebook.react.ReactPackage;
import com.facebook.react.bridge.NativeModule; import com.facebook.react.bridge.NativeModule;
import com.facebook.react.bridge.ReactApplicationContext; import com.facebook.react.bridge.ReactApplicationContext;
import com.facebook.react.uimanager.ViewManager; import com.facebook.react.uimanager.ViewManager;
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin; import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin;
import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry; import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry;
import javax.annotation.Nonnull;
public class FaceDetectorFrameProcessorPluginPackage implements ReactPackage { public class FaceDetectorFrameProcessorPluginPackage implements ReactPackage {
FaceDetectorFrameProcessorPluginPackage() {
// highlight-start // highlight-start
FaceDetectorFrameProcessorPluginPackage() {
FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces", options -> new FaceDetectorFrameProcessorPlugin()); FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces", options -> new FaceDetectorFrameProcessorPlugin());
// highlight-end
} }
// highlight-end
@NonNull @NonNull
@Override @Override
@ -94,9 +97,9 @@ public class FaceDetectorFrameProcessorPluginPackage implements ReactPackage {
return Collections.emptyList(); return Collections.emptyList();
} }
@Nonnull @NonNull
@Override @Override
public List<ViewManager> createViewManagers(@Nonnull ReactApplicationContext reactContext) { public List<ViewManager> createViewManagers(@NonNull ReactApplicationContext reactContext) {
return Collections.emptyList(); return Collections.emptyList();
} }
} }
@ -113,7 +116,7 @@ The Frame Processor Plugin will be exposed to JS through the `VisionCameraProxy`
protected List<ReactPackage> getPackages() { protected List<ReactPackage> getPackages() {
@SuppressWarnings("UnnecessaryLocalVariable") @SuppressWarnings("UnnecessaryLocalVariable")
List<ReactPackage> packages = new PackageList(this).getPackages(); List<ReactPackage> packages = new PackageList(this).getPackages();
... // ...
// highlight-next-line // highlight-next-line
packages.add(new FaceDetectorFrameProcessorPluginPackage()); // <- add packages.add(new FaceDetectorFrameProcessorPluginPackage()); // <- add
return packages; return packages;
@ -133,7 +136,8 @@ import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin
class FaceDetectorFrameProcessorPlugin: FrameProcessorPlugin() { class FaceDetectorFrameProcessorPlugin: FrameProcessorPlugin() {
override fun callback(frame: Frame, arguments: Map<String, Object>): Any? { override fun callback(frame: Frame, arguments: Map<String, Object>?): Any? {
// highlight-next-line
// code goes here // code goes here
return null return null
} }
@ -151,13 +155,13 @@ import com.facebook.react.uimanager.ViewManager
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin
class FaceDetectorFrameProcessorPluginPackage : ReactPackage { class FaceDetectorFrameProcessorPluginPackage : ReactPackage {
init {
// highlight-start // highlight-start
init {
FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces") { options -> FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces") { options ->
FaceDetectorFrameProcessorPlugin() FaceDetectorFrameProcessorPlugin()
} }
// highlight-end
} }
// highlight-end
override fun createNativeModules(reactContext: ReactApplicationContext): List<NativeModule> { override fun createNativeModules(reactContext: ReactApplicationContext): List<NativeModule> {
return emptyList() return emptyList()
@ -180,7 +184,7 @@ The Frame Processor Plugin will be exposed to JS through the `VisionCameraProxy`
protected List<ReactPackage> getPackages() { protected List<ReactPackage> getPackages() {
@SuppressWarnings("UnnecessaryLocalVariable") @SuppressWarnings("UnnecessaryLocalVariable")
List<ReactPackage> packages = new PackageList(this).getPackages(); List<ReactPackage> packages = new PackageList(this).getPackages();
... // ...
// highlight-next-line // highlight-next-line
packages.add(new FaceDetectorFrameProcessorPluginPackage()); // <- add packages.add(new FaceDetectorFrameProcessorPluginPackage()); // <- add
return packages; return packages;

View File

@ -60,7 +60,7 @@ For reference see the [CLI's docs](https://github.com/mateusz1913/vision-camera-
CMSampleBufferRef buffer = frame.buffer; CMSampleBufferRef buffer = frame.buffer;
UIImageOrientation orientation = frame.orientation; UIImageOrientation orientation = frame.orientation;
// code goes here // code goes here
return @[]; return nil;
} }
+ (void) load { + (void) load {
@ -89,56 +89,49 @@ The Frame Processor Plugin will be exposed to JS through the `VisionCameraProxy`
![Xcode "Create Bridging Header" alert](https://docs-assets.developer.apple.com/published/7ebca7212c/2a065d1a-7e53-4907-a889-b7fa4f2206c9.png) ![Xcode "Create Bridging Header" alert](https://docs-assets.developer.apple.com/published/7ebca7212c/2a065d1a-7e53-4907-a889-b7fa4f2206c9.png)
3. Inside the newly created Bridging Header, add the following code: 3. In the Swift file, add the following code:
```objc
#import <VisionCamera/FrameProcessorPlugin.h>
#import <VisionCamera/Frame.h>
```
4. In the Swift file, add the following code:
```swift ```swift
import VisionCamera
@objc(FaceDetectorFrameProcessorPlugin) @objc(FaceDetectorFrameProcessorPlugin)
public class FaceDetectorFrameProcessorPlugin: FrameProcessorPlugin { public class FaceDetectorFrameProcessorPlugin: FrameProcessorPlugin {
public override func callback(_ frame: Frame, withArguments arguments: [AnyHashable : Any]?) -> Any {
public override func callback(_ frame: Frame!,
withArguments arguments: [String:Any]) -> Any {
let buffer = frame.buffer let buffer = frame.buffer
let orientation = frame.orientation let orientation = frame.orientation
// code goes here // code goes here
return [] return nil
} }
} }
``` ```
5. In your `AppDelegate.m`, add the following imports: 4. Create an Objective-C source file that will be used to automatically register your plugin
```objc ```objc
#import "YOUR_XCODE_PROJECT_NAME-Swift.h"
#import <VisionCamera/FrameProcessorPlugin.h> #import <VisionCamera/FrameProcessorPlugin.h>
#import <VisionCamera/FrameProcessorPluginRegistry.h> #import <VisionCamera/FrameProcessorPluginRegistry.h>
```
6. In your `AppDelegate.m`, add the following code to `application:didFinishLaunchingWithOptions:`: #import "YOUR_XCODE_PROJECT_NAME-Swift.h" // <--- replace "YOUR_XCODE_PROJECT_NAME" with the actual value of your xcode project name
```objc @interface FaceDetectorFrameProcessorPlugin (FrameProcessorPluginLoader)
- (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)launchOptions @end
@implementation FaceDetectorFrameProcessorPlugin (FrameProcessorPluginLoader)
+ (void)load
{ {
// ...
// highlight-start // highlight-start
[FrameProcessorPluginRegistry addFrameProcessorPlugin:@"detectFaces" [FrameProcessorPluginRegistry addFrameProcessorPlugin:@"detectFaces"
withInitializer:^FrameProcessorPlugin*(NSDictionary* options) { withInitializer:^FrameProcessorPlugin* (NSDictionary* options) {
return [[FaceDetectorFrameProcessorPlugin alloc] initWithOptions:options]; return [[FaceDetectorFrameProcessorPlugin alloc] initWithOptions:options];
}]; }];
// highlight-end // highlight-end
return [super application:application didFinishLaunchingWithOptions:launchOptions];
} }
@end
``` ```
7. **Implement your frame processing.** See [Example Plugin (Swift)](https://github.com/mrousavy/react-native-vision-camera/blob/main/package/example/ios/Frame%20Processor%20Plugins/Example%20Plugin%20%28Swift%29) for reference. 5. **Implement your frame processing.** See [Example Plugin (Swift)](https://github.com/mrousavy/react-native-vision-camera/blob/main/package/example/ios/Frame%20Processor%20Plugins/Example%20Plugin%20%28Swift%29) for reference.
</TabItem> </TabItem>