feat: Add zero-copy SharedArray type to Frame Processor Plugins (#2383)

* feat: Create `TypedArray` class for Frame Processor Plugins * Type * feat: Pass `VisionCameraProxy` along (BREAKING) * feat: Finish implementation * Log a bit * feat: Successfully convert JSI <> JNI buffers * Wrap buffer * fix: Fix using wrong Runtime * feat: Add docs * add zero copy example * Format C++ * Create iOS base * feat: Finish iOS implementation * chore: Format * fix: Use `NSData` instead of `NSMutableData` * Format * fix: Fix build when Frame Processors are disabled * chore: Rename `TypedArray` to `SharedArray` * fix: Fix Swift typings for Array * Remove a few default inits * fix: Fix Android build * fix: Use `NSInteger` * Update SharedArray.mm * fix: Expose bytes directly on iOS (NSData was immutable)
2024-01-12 16:00:36 +01:00
parent 56cecaa814
commit 29fe98cc44
35 changed files with 491 additions and 65 deletions
--- a/package/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java
+++ b/package/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java
@@ -5,15 +5,21 @@ import android.util.Log;

 import com.mrousavy.camera.frameprocessor.Frame;
 import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin;
+import com.mrousavy.camera.frameprocessor.SharedArray;
+import com.mrousavy.camera.frameprocessor.VisionCameraProxy;
+
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;

+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;

 public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin {
+    SharedArray _sharedArray;
+
    @Override
    public Object callback(@NotNull Frame frame, @Nullable Map<String, Object> params) {
        if (params == null) return null;
@@ -37,11 +43,17 @@ public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin {
        array.add(17.38);

        map.put("example_array", array);
+
+        ByteBuffer byteBuffer = _sharedArray.getByteBuffer();
+        byteBuffer.put(0, (byte)(Math.random() * 100));
+        map.put("example_array_buffer", _sharedArray);
+
        return map;
    }

-    ExampleFrameProcessorPlugin(@Nullable Map<String, Object> options) {
-        super(options);
+    ExampleFrameProcessorPlugin(VisionCameraProxy proxy, @Nullable Map<String, Object> options) {
+        super();
+        _sharedArray = new SharedArray(proxy, SharedArray.Type.Uint8Array, 5);
        Log.d("ExamplePlugin", "ExampleFrameProcessorPlugin initialized with options: " + options);
    }
 }
--- a/package/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleKotlinFrameProcessorPlugin.kt
+++ b/package/example/android/app/src/main/java/com/mrousavy/camera/example/ExampleKotlinFrameProcessorPlugin.kt
@@ -3,8 +3,9 @@ package com.mrousavy.camera.example
 import android.util.Log
 import com.mrousavy.camera.frameprocessor.Frame
 import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin
+import com.mrousavy.camera.frameprocessor.VisionCameraProxy

-class ExampleKotlinFrameProcessorPlugin(options: Map<String, Any>?): FrameProcessorPlugin(options) {
+class ExampleKotlinFrameProcessorPlugin(proxy: VisionCameraProxy, options: Map<String, Any>?): FrameProcessorPlugin() {
    init {
        Log.d("ExampleKotlinPlugin", "ExampleKotlinFrameProcessorPlugin initialized with options: " + options?.toString())
    }
--- a/package/example/android/app/src/main/java/com/mrousavy/camera/example/MainApplication.java
+++ b/package/example/android/app/src/main/java/com/mrousavy/camera/example/MainApplication.java
@@ -20,8 +20,8 @@ import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry;
 public class MainApplication extends Application implements ReactApplication {
  // Register the Frame Processor Plugins for our app
  static {
-    FrameProcessorPluginRegistry.addFrameProcessorPlugin("example_plugin", options -> new ExampleFrameProcessorPlugin(options));
-    FrameProcessorPluginRegistry.addFrameProcessorPlugin("example_kotlin_swift_plugin", options -> new ExampleKotlinFrameProcessorPlugin(options));
+    FrameProcessorPluginRegistry.addFrameProcessorPlugin("example_plugin", ExampleFrameProcessorPlugin::new);
+    FrameProcessorPluginRegistry.addFrameProcessorPlugin("example_kotlin_swift_plugin", ExampleKotlinFrameProcessorPlugin::new);
  }

  private final ReactNativeHost mReactNativeHost =
@@ -33,7 +33,6 @@ public class MainApplication extends Application implements ReactApplication {

        @Override
        protected List<ReactPackage> getPackages() {
-          @SuppressWarnings("UnnecessaryLocalVariable")
          List<ReactPackage> packages = new PackageList(this).getPackages();
          // Packages that cannot be autolinked yet can be added manually here, for VisionCameraExample:
          packages.add(new CameraPackage());
--- a/Plugin/ExampleFrameProcessorPlugin.m
+++ b/Plugin/ExampleFrameProcessorPlugin.m
@@ -10,18 +10,25 @@
 #import <VisionCamera/FrameProcessorPlugin.h>
 #import <VisionCamera/FrameProcessorPluginRegistry.h>
 #import <VisionCamera/Frame.h>
+#import <VisionCamera/SharedArray.h>

 // Example for an Objective-C Frame Processor plugin

@interface ExampleFrameProcessorPlugin : FrameProcessorPlugin
@end

-@implementation ExampleFrameProcessorPlugin
+@implementation ExampleFrameProcessorPlugin {
+  SharedArray* _sharedArray;
+}

- (instancetype)initWithOptions:(NSDictionary* _Nullable)options
-{
-  self = [super initWithOptions:options];
-  NSLog(@"ExampleFrameProcessorPlugin initialized with options: %@", options);
+- (instancetype)initWithProxy:(VisionCameraProxyHolder*)proxy
+                  withOptions:(NSDictionary* _Nullable)options {
+  if (self = [super initWithProxy:proxy withOptions:options]) {
+    _sharedArray = [[SharedArray alloc] initWithProxy:proxy
+                                                 type:Int8Array
+                                                 size:5];
+    NSLog(@"ExampleFrameProcessorPlugin initialized with options: %@", options);
+  }
  return self;
 }

@@ -33,6 +40,9 @@
    NSLog(@"ExamplePlugin:   -> %@ (%@)", param == nil ? @"(nil)" : [param description], NSStringFromClass([param classForCoder]));
  }

+  uint8_t* data = _sharedArray.data;
+  data[0] = (uint8_t)(random() * 100);
+
  return @{
    @"example_str": @"Test",
    @"example_bool": @(YES),
@@ -41,7 +51,8 @@
      @"Hello",
      @(YES),
      @17.38
-    ]
+    ],
+    @"example_array_buffer": _sharedArray
  };
 }

--- a/Plugin/ExampleSwiftFrameProcessor.swift
+++ b/Plugin/ExampleSwiftFrameProcessor.swift
@@ -11,8 +11,8 @@ import VisionCamera
 // Example for a Swift Frame Processor plugin
@objc(ExampleSwiftFrameProcessorPlugin)
 public class ExampleSwiftFrameProcessorPlugin: FrameProcessorPlugin {
-  public override init(options: [AnyHashable: Any]! = [:]) {
-    super.init(options: options)
+  public override init(proxy: VisionCameraProxyHolder, options: [AnyHashable: Any]! = [:]) {
+    super.init(proxy: proxy, options: options)

    print("ExampleSwiftFrameProcessorPlugin initialized with options: \(String(describing: options))")
  }
--- a/package/example/ios/Podfile.lock
+++ b/package/example/ios/Podfile.lock
@@ -484,7 +484,7 @@ PODS:
    - libwebp (~> 1.0)
    - SDWebImage/Core (~> 5.10)
  - SocketRocket (0.6.1)
-  - VisionCamera (3.6.17):
+  - VisionCamera (3.7.1):
    - React
    - React-callinvoker
    - React-Core
@@ -724,9 +724,9 @@ SPEC CHECKSUMS:
  SDWebImage: a7f831e1a65eb5e285e3fb046a23fcfbf08e696d
  SDWebImageWebPCoder: 908b83b6adda48effe7667cd2b7f78c897e5111d
  SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17
-  VisionCamera: 361df29347b7b7ecc47b3d173daa17751a11ffc1
+  VisionCamera: ac444079a315b38ec664cf77ed548c384554f0ca
  Yoga: 4c3aa327e4a6a23eeacd71f61c81df1bcdf677d5

 PODFILE CHECKSUM: 27f53791141a3303d814e09b55770336416ff4eb

-COCOAPODS: 1.11.3
+COCOAPODS: 1.14.3
--- a/package/example/src/frame-processors/ExamplePlugin.ts
+++ b/package/example/src/frame-processors/ExamplePlugin.ts
@@ -4,6 +4,7 @@ const plugin = VisionCameraProxy.initFrameProcessorPlugin('example_plugin')

 interface Result {
  example_array: (string | number | boolean)[]
+  example_array_buffer: ArrayBuffer
  example_str: string
  example_bool: boolean
  example_double: number