railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Add zero-copy SharedArray type to Frame Processor Plugins (#2383)

Browse Source 
* feat: Create `TypedArray` class for Frame Processor Plugins

* Type

* feat: Pass `VisionCameraProxy` along (BREAKING)

* feat: Finish implementation

* Log a bit

* feat: Successfully convert JSI <> JNI buffers

* Wrap buffer

* fix: Fix using wrong Runtime

* feat: Add docs

* add zero copy example

* Format C++

* Create iOS base

* feat: Finish iOS implementation

* chore: Format

* fix: Use `NSData` instead of `NSMutableData`

* Format

* fix: Fix build when Frame Processors are disabled

* chore: Rename `TypedArray` to `SharedArray`

* fix: Fix Swift typings for Array

* Remove a few default inits

* fix: Fix Android build

* fix: Use `NSInteger`

* Update SharedArray.mm

* fix: Expose bytes directly on iOS (NSData was immutable)
This commit is contained in:
Marc Rousavy
2024-01-12 16:00:36 +01:00
committed by GitHub
parent 56cecaa814
commit 29fe98cc44
35 changed files with 491 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx
View File

@@ -44,6 +44,7 @@ Similar to a TurboModule, the Frame Processor Plugin Registry API automatically
| `{}` | `NSDictionary*` | `Map<String, Object>` |
| `undefined` / `null` | `nil` | `null` |
| `(any, any) => void` | [`RCTResponseSenderBlock`][4] | `(Object, Object) -> void` |
| `ArrayBuffer` | [`SharedArray`][7] | [`SharedArray`][8] |
| [`Frame`][1] | [`Frame*`][2] | [`Frame`][3] |
### Return values
@@ -207,3 +208,5 @@ Your Frame Processor Plugins have to be fast. Use the FPS Graph (`enableFpsGraph
[4]: https://github.com/facebook/react-native/blob/9a43eac7a32a6ba3164a048960101022a92fcd5a/React/Base/RCTBridgeModule.h#L20-L24
[5]: https://developer.apple.com/documentation/coremedia/cmsamplebuffer
[6]: https://developer.android.com/reference/androidx/camera/core/ImageProxy
[7]: https://github.com/mrousavy/react-native-vision-camera/blob/main/package/ios/Frame%20Processor/SharedArray.h
[8]: https://github.com/mrousavy/react-native-vision-camera/blob/main/package/android/src/main/java/com/mrousavy/camera/frameprocessor/SharedArray.java

12
docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_ANDROID.mdx
View File

@@ -60,9 +60,10 @@ import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import com.mrousavy.camera.frameprocessor.Frame;
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin;
import com.mrousavy.camera.frameprocessor.VisionCameraProxy;
public class FaceDetectorFrameProcessorPlugin extends FrameProcessorPlugin {
FaceDetectorFrameProcessorPlugin(@Nullable Map<String, Object> options) {}
FaceDetectorFrameProcessorPlugin(@NonNull VisionCameraProxy proxy, @Nullable Map<String, Object> options) {}
@Nullable
@Override
@@ -89,7 +90,7 @@ import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry;
public class FaceDetectorFrameProcessorPluginPackage implements ReactPackage {
// highlight-start
static {
FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces", options -> new FaceDetectorFrameProcessorPlugin(options));
FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces", FaceDetectorFrameProcessorPlugin::new);
}
// highlight-end
@@ -135,8 +136,9 @@ The Frame Processor Plugin will be exposed to JS through the `VisionCameraProxy`
```kotlin
import com.mrousavy.camera.frameprocessor.Frame
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin
import com.mrousavy.camera.frameprocessor.VisionCameraProxy
class FaceDetectorFrameProcessorPlugin(options: Map<String, Any>?): FrameProcessorPlugin(options) {
class FaceDetectorFrameProcessorPlugin(proxy: VisionCameraProxy, options: Map<String, Any>?): FrameProcessorPlugin() {
override fun callback(frame: Frame, arguments: Map<String, Any>?): Any? {
// highlight-next-line
@@ -161,8 +163,8 @@ class FaceDetectorFrameProcessorPluginPackage : ReactPackage {
// highlight-start
companion object {
init {
FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces") { options ->
FaceDetectorFrameProcessorPlugin(options)
FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces") { proxy, options ->
FaceDetectorFrameProcessorPlugin(proxy, options)
}
}
}

5
docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx
View File

@@ -51,8 +51,9 @@ For reference see the [CLI's docs](https://github.com/mateusz1913/vision-camera-
@implementation FaceDetectorFrameProcessorPlugin
- (instancetype) initWithOptions:(NSDictionary* _Nullable)options; {
self = [super initWithOptions:options];
- (instancetype) initWithProxy:(VisionCameraProxyHolder*)proxy
withOptions:(NSDictionary* _Nullable)options {
self = [super initWithProxy:proxy options:options];
return self;
}