railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Expose unified VisionCameraProxy object, make FrameProcessorPlugins object-oriented (#1660)

Browse Source 
* feat: Replace `FrameProcessorRuntimeManager` with `VisionCameraProxy` (iOS)

* Make `FrameProcessorPlugin` a constructable HostObject

* fix: Fix `name` override

* Simplify `useFrameProcessor

* fix: Fix lint errors

* Remove FrameProcessorPlugin::name

* JSIUtils -> JSINSObjectConversion
This commit is contained in:
Marc Rousavy
2023-07-21 17:52:30 +02:00
committed by GitHub
parent 375e894038
commit 44ed42d5d6
41 changed files with 762 additions and 607 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
example/ios/Frame Processor Plugins/Example Plugin (Swift)/ExamplePluginSwift.swift
View File

@@ -1,48 +0,0 @@
//
// ExamplePluginSwift.swift
// VisionCamera
//
// Created by Marc Rousavy on 30.04.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
import AVKit
import Vision
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
@objc
public class ExamplePluginSwift : FrameProcessorPlugin {
override public func name() -> String! {
return "example_plugin_swift"
}
public override func callback(_ frame: Frame!, withArguments arguments: [Any]!) -> Any! {
guard let imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer) else {
return nil
}
NSLog("ExamplePlugin: \(CVPixelBufferGetWidth(imageBuffer)) x \(CVPixelBufferGetHeight(imageBuffer)) Image. Logging \(arguments.count) parameters:")
arguments.forEach { arg in
var string = "\(arg)"
if let array = arg as? NSArray {
string = (array as Array).description
} else if let map = arg as? NSDictionary {
string = (map as Dictionary).description
}
NSLog("ExamplePlugin: -> \(string) (\(type(of: arg)))")
}
return [
"example_str": "Test",
"example_bool": true,
"example_double": 5.3,
"example_array": [
"Hello",
true,
17.38,
],
]
}
}
#endif

14
example/ios/Frame Processor Plugins/Example Plugin (Objective-C)/ExampleFrameProcessorPlugin.m → example/ios/Frame Processor Plugins/Example Plugin/ExampleFrameProcessorPlugin.m
View File

@@ -8,6 +8,7 @@
#if __has_include(<VisionCamera/FrameProcessorPlugin.h>)
#import <Foundation/Foundation.h>
#import <VisionCamera/FrameProcessorPlugin.h>
#import <VisionCamera/FrameProcessorPluginRegistry.h>
#import <VisionCamera/Frame.h>
// Example for an Objective-C Frame Processor plugin
@@ -17,18 +18,14 @@
@implementation ExampleFrameProcessorPlugin
- (NSString *)name {
return @"example_plugin";
}
- (id)callback:(Frame *)frame withArguments:(NSArray<id> *)arguments {
CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
NSLog(@"ExamplePlugin: %zu x %zu Image. Logging %lu parameters:", CVPixelBufferGetWidth(imageBuffer), CVPixelBufferGetHeight(imageBuffer), (unsigned long)arguments.count);
for (id param in arguments) {
NSLog(@"ExamplePlugin: -> %@ (%@)", param == nil ? @"(nil)" : [param description], NSStringFromClass([param classForCoder]));
}
return @{
@"example_str": @"Test",
@"example_bool": @true,
@@ -42,7 +39,10 @@
}
+ (void) load {
[self registerPlugin:[[ExampleFrameProcessorPlugin alloc] init]];
[FrameProcessorPluginRegistry addFrameProcessorPlugin:@"example_plugin"
withInitializer:^FrameProcessorPlugin*(NSDictionary* options) {
return [[ExampleFrameProcessorPlugin alloc] initWithOptions:options];
}];
}
@end

2
example/ios/Podfile.lock
View File

@@ -713,7 +713,7 @@ SPEC CHECKSUMS:
RNStaticSafeAreaInsets: 055ddbf5e476321720457cdaeec0ff2ba40ec1b8
RNVectorIcons: fcc2f6cb32f5735b586e66d14103a74ce6ad61f8
SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17
VisionCamera: b4e91836f577249470ae42707782f4b44d875cd9
VisionCamera: 29727c3ed48328b246e3a7448f7c14cc12d2fd11
Yoga: 65286bb6a07edce5e4fe8c90774da977ae8fc009
PODFILE CHECKSUM: ab9c06b18c63e741c04349c0fd630c6d3145081c

18
example/ios/VisionCameraExample.xcodeproj/project.pbxproj
View File

@@ -13,7 +13,6 @@
81AB9BB82411601600AC10FF /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 81AB9BB72411601600AC10FF /* LaunchScreen.storyboard */; };
B8DB3BD5263DE8B7004C18D7 /* BuildFile in Sources */ = {isa = PBXBuildFile; };
B8DB3BDC263DEA31004C18D7 /* ExampleFrameProcessorPlugin.m in Sources */ = {isa = PBXBuildFile; fileRef = B8DB3BD8263DEA31004C18D7 /* ExampleFrameProcessorPlugin.m */; };
B8DB3BDD263DEA31004C18D7 /* ExamplePluginSwift.swift in Sources */ = {isa = PBXBuildFile; fileRef = B8DB3BDA263DEA31004C18D7 /* ExamplePluginSwift.swift */; };
B8F0E10825E0199F00586F16 /* File.swift in Sources */ = {isa = PBXBuildFile; fileRef = B8F0E10725E0199F00586F16 /* File.swift */; };
C0B129659921D2EA967280B2 /* libPods-VisionCameraExample.a in Frameworks */ = {isa = PBXBuildFile; fileRef = 3CDCFE89C25C89320B98945E /* libPods-VisionCameraExample.a */; };
/* End PBXBuildFile section */
@@ -29,7 +28,6 @@
3CDCFE89C25C89320B98945E /* libPods-VisionCameraExample.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-VisionCameraExample.a"; sourceTree = BUILT_PRODUCTS_DIR; };
81AB9BB72411601600AC10FF /* LaunchScreen.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; name = LaunchScreen.storyboard; path = VisionCameraExample/LaunchScreen.storyboard; sourceTree = "<group>"; };
B8DB3BD8263DEA31004C18D7 /* ExampleFrameProcessorPlugin.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ExampleFrameProcessorPlugin.m; sourceTree = "<group>"; };
B8DB3BDA263DEA31004C18D7 /* ExamplePluginSwift.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = ExamplePluginSwift.swift; sourceTree = "<group>"; };
B8F0E10625E0199F00586F16 /* VisionCameraExample-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "VisionCameraExample-Bridging-Header.h"; sourceTree = "<group>"; };
B8F0E10725E0199F00586F16 /* File.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = File.swift; sourceTree = "<group>"; };
C1D342AD8210E7627A632602 /* Pods-VisionCameraExample.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-VisionCameraExample.debug.xcconfig"; path = "Target Support Files/Pods-VisionCameraExample/Pods-VisionCameraExample.debug.xcconfig"; sourceTree = "<group>"; };
@@ -118,26 +116,17 @@
B8DB3BD6263DEA31004C18D7 /* Frame Processor Plugins */ = {
isa = PBXGroup;
children = (
B8DB3BD7263DEA31004C18D7 /* Example Plugin (Objective-C) */,
B8DB3BD9263DEA31004C18D7 /* Example Plugin (Swift) */,
B8DB3BD7263DEA31004C18D7 /* Example Plugin */,
);
path = "Frame Processor Plugins";
sourceTree = "<group>";
};
B8DB3BD7263DEA31004C18D7 /* Example Plugin (Objective-C) */ = {
B8DB3BD7263DEA31004C18D7 /* Example Plugin */ = {
isa = PBXGroup;
children = (
B8DB3BD8263DEA31004C18D7 /* ExampleFrameProcessorPlugin.m */,
);
path = "Example Plugin (Objective-C)";
sourceTree = "<group>";
};
B8DB3BD9263DEA31004C18D7 /* Example Plugin (Swift) */ = {
isa = PBXGroup;
children = (
B8DB3BDA263DEA31004C18D7 /* ExamplePluginSwift.swift */,
);
path = "Example Plugin (Swift)";
path = "Example Plugin";
sourceTree = "<group>";
};
/* End PBXGroup section */
@@ -381,7 +370,6 @@
13B07FBC1A68108700A75B9A /* AppDelegate.mm in Sources */,
B8DB3BDC263DEA31004C18D7 /* ExampleFrameProcessorPlugin.m in Sources */,
B8DB3BD5263DE8B7004C18D7 /* BuildFile in Sources */,
B8DB3BDD263DEA31004C18D7 /* ExamplePluginSwift.swift in Sources */,
B8F0E10825E0199F00586F16 /* File.swift in Sources */,
13B07FC11A68108700A75B9A /* main.m in Sources */,
);

23
example/src/frame-processors/ExamplePlugin.ts
View File

@@ -1,16 +1,17 @@
import { FrameProcessorPlugins, Frame } from 'react-native-vision-camera';
import { VisionCameraProxy, Frame } from 'react-native-vision-camera';
export function examplePluginSwift(frame: Frame): string[] {
'worklet';
// @ts-expect-error because this function is dynamically injected by VisionCamera
return FrameProcessorPlugins.example_plugin_swift(frame, 'hello!', 'parameter2', true, 42, { test: 0, second: 'test' }, [
'another test',
5,
]);
}
const plugin = VisionCameraProxy.getFrameProcessorPlugin('example_plugin');
export function examplePlugin(frame: Frame): string[] {
'worklet';
// @ts-expect-error because this function is dynamically injected by VisionCamera
return FrameProcessorPlugins.example_plugin(frame, 'hello!', 'parameter2', true, 42, { test: 0, second: 'test' }, ['another test', 5]);
if (plugin == null) throw new Error('Failed to load Frame Processor Plugin "example_plugin"!');
return plugin.call(frame, {
someString: 'hello!',
someBoolean: true,
someNumber: 42,
someObject: { test: 0, second: 'test' },
someArray: ['another test', 5],
}) as string[];
}