docs: Frame Processor Java Types (#183)
* add a few spaces * Update FRAME_PROCESSORS_CREATE_OVERVIEW.mdx * Update FRAME_PROCESSORS_CREATE_OVERVIEW.mdx * Update FRAME_PROCESSORS_CREATE_OVERVIEW.mdx * Update FRAME_PROCESSORS_CREATE_OVERVIEW.mdx
This commit is contained in:
parent
af805bd331
commit
be30d55df5
@ -28,13 +28,13 @@ function App() {
|
|||||||
|
|
||||||
To achieve **maximum performance**, the `scanQRCodes` function is written in a native language (e.g. Objective-C), but it will be directly called from the VisionCamera Frame Processor JavaScript-Runtime.
|
To achieve **maximum performance**, the `scanQRCodes` function is written in a native language (e.g. Objective-C), but it will be directly called from the VisionCamera Frame Processor JavaScript-Runtime.
|
||||||
|
|
||||||
### Return Types
|
### Types
|
||||||
|
|
||||||
Frame Processors can return any primitive value that is representable in JS. So for Objective-C that maps to:
|
The Frame Processor Plugin Registry API automatically manages type conversion from JS <-> native. They are converted into the most efficient data-structures, as seen here:
|
||||||
|
|
||||||
| JS Type | Objective-C Type | Java Type |
|
| JS Type | Objective-C Type | Java Type |
|
||||||
|----------------------|--------------------------|----------------------------|
|
|----------------------|--------------------------|----------------------------|
|
||||||
| `number` | `NSNumber` | `double` |
|
| `number` | `NSNumber` (double) | `double` |
|
||||||
| `boolean` | `NSNumber` (boolean) | `boolean` |
|
| `boolean` | `NSNumber` (boolean) | `boolean` |
|
||||||
| `string` | `NSString` | `String` |
|
| `string` | `NSString` | `String` |
|
||||||
| `[]` | `NSArray` | `Array<Object>` |
|
| `[]` | `NSArray` | `Array<Object>` |
|
||||||
@ -42,7 +42,9 @@ Frame Processors can return any primitive value that is representable in JS. So
|
|||||||
| `undefined` | `nil` / `NSNull` | `null` |
|
| `undefined` | `nil` / `NSNull` | `null` |
|
||||||
| `(any, any) => void` | `RCTResponseSenderBlock` | `(Object, Object) -> void` |
|
| `(any, any) => void` | `RCTResponseSenderBlock` | `(Object, Object) -> void` |
|
||||||
|
|
||||||
The values will automatically be converted to JS values, so the following Objective-C frame processor:
|
### Return values
|
||||||
|
|
||||||
|
Return values will automatically be converted to JS values, assuming they are representable in the ["Types" table](#types). So the following Objective-C frame processor:
|
||||||
|
|
||||||
```objc
|
```objc
|
||||||
static inline id detectObject(CMSampleBufferRef buffer, NSArray args) {
|
static inline id detectObject(CMSampleBufferRef buffer, NSArray args) {
|
||||||
@ -62,7 +64,7 @@ export function detectObject(frame: Frame): string {
|
|||||||
|
|
||||||
### Parameters
|
### Parameters
|
||||||
|
|
||||||
Frame Processors can also accept parameters, following the same type convention as [return values](#return-types):
|
Frame Processors can also accept parameters, following the same type convention as [return values](#return-values):
|
||||||
|
|
||||||
```ts
|
```ts
|
||||||
const frameProcessor = useFrameProcessor((frame) => {
|
const frameProcessor = useFrameProcessor((frame) => {
|
||||||
|
@ -63,9 +63,11 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
|
|||||||
#ifdef ENABLE_FRAME_PROCESSORS
|
#ifdef ENABLE_FRAME_PROCESSORS
|
||||||
NSLog(@"FrameProcessorBindings: Creating Runtime Manager...");
|
NSLog(@"FrameProcessorBindings: Creating Runtime Manager...");
|
||||||
weakBridge = bridge;
|
weakBridge = bridge;
|
||||||
|
|
||||||
auto runtime = vision::makeJSIRuntime();
|
auto runtime = vision::makeJSIRuntime();
|
||||||
reanimated::RuntimeDecorator::decorateRuntime(*runtime, "FRAME_PROCESSOR");
|
reanimated::RuntimeDecorator::decorateRuntime(*runtime, "FRAME_PROCESSOR");
|
||||||
runtime->global().setProperty(*runtime, "_FRAME_PROCESSOR", jsi::Value(true));
|
runtime->global().setProperty(*runtime, "_FRAME_PROCESSOR", jsi::Value(true));
|
||||||
|
|
||||||
auto callInvoker = bridge.jsCallInvoker;
|
auto callInvoker = bridge.jsCallInvoker;
|
||||||
auto scheduler = std::make_shared<reanimated::REAIOSScheduler>(callInvoker);
|
auto scheduler = std::make_shared<reanimated::REAIOSScheduler>(callInvoker);
|
||||||
runtimeManager = std::make_unique<reanimated::RuntimeManager>(std::move(runtime),
|
runtimeManager = std::make_unique<reanimated::RuntimeManager>(std::move(runtime),
|
||||||
@ -76,6 +78,7 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
|
|||||||
NSLog(@"FrameProcessorBindings: Installing Frame Processor plugins...");
|
NSLog(@"FrameProcessorBindings: Installing Frame Processor plugins...");
|
||||||
auto& visionRuntime = *runtimeManager->runtime;
|
auto& visionRuntime = *runtimeManager->runtime;
|
||||||
auto visionGlobal = visionRuntime.global();
|
auto visionGlobal = visionRuntime.global();
|
||||||
|
|
||||||
for (NSString* pluginKey in [FrameProcessorPluginRegistry frameProcessorPlugins]) {
|
for (NSString* pluginKey in [FrameProcessorPluginRegistry frameProcessorPlugins]) {
|
||||||
auto pluginName = [pluginKey UTF8String];
|
auto pluginName = [pluginKey UTF8String];
|
||||||
|
|
||||||
@ -98,7 +101,9 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
|
|||||||
1, // frame
|
1, // frame
|
||||||
function));
|
function));
|
||||||
}
|
}
|
||||||
|
|
||||||
[FrameProcessorPluginRegistry markInvalid];
|
[FrameProcessorPluginRegistry markInvalid];
|
||||||
|
|
||||||
NSLog(@"FrameProcessorBindings: Frame Processor plugins installed!");
|
NSLog(@"FrameProcessorBindings: Frame Processor plugins installed!");
|
||||||
#else
|
#else
|
||||||
NSLog(@"Reanimated not found, Frame Processors are disabled.");
|
NSLog(@"Reanimated not found, Frame Processors are disabled.");
|
||||||
@ -113,11 +118,13 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
|
|||||||
NSLog(@"FrameProcessorBindings: Failed to install Frame Processor Bindings - bridge was null!");
|
NSLog(@"FrameProcessorBindings: Failed to install Frame Processor Bindings - bridge was null!");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
NSLog(@"FrameProcessorBindings: Installing Frame Processor Bindings for Bridge...");
|
NSLog(@"FrameProcessorBindings: Installing Frame Processor Bindings for Bridge...");
|
||||||
RCTCxxBridge *cxxBridge = (RCTCxxBridge *)weakBridge;
|
RCTCxxBridge *cxxBridge = (RCTCxxBridge *)weakBridge;
|
||||||
if (!cxxBridge.runtime) {
|
if (!cxxBridge.runtime) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
jsi::Runtime& jsiRuntime = *(jsi::Runtime*)cxxBridge.runtime;
|
jsi::Runtime& jsiRuntime = *(jsi::Runtime*)cxxBridge.runtime;
|
||||||
NSLog(@"FrameProcessorBindings: Installing global functions...");
|
NSLog(@"FrameProcessorBindings: Installing global functions...");
|
||||||
|
|
||||||
@ -142,6 +149,7 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
|
|||||||
NSLog(@"FrameProcessorBindings: Converting worklet to Objective-C callback...");
|
NSLog(@"FrameProcessorBindings: Converting worklet to Objective-C callback...");
|
||||||
auto& rt = *runtimeManager->runtime;
|
auto& rt = *runtimeManager->runtime;
|
||||||
auto function = worklet->getValue(rt).asObject(rt).asFunction(rt);
|
auto function = worklet->getValue(rt).asObject(rt).asFunction(rt);
|
||||||
|
|
||||||
view.frameProcessorCallback = convertJSIFunctionToFrameProcessorCallback(rt, function);
|
view.frameProcessorCallback = convertJSIFunctionToFrameProcessorCallback(rt, function);
|
||||||
NSLog(@"FrameProcessorBindings: Frame processor set!");
|
NSLog(@"FrameProcessorBindings: Frame processor set!");
|
||||||
});
|
});
|
||||||
@ -163,8 +171,10 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
|
|||||||
RCTExecuteOnMainQueue(^{
|
RCTExecuteOnMainQueue(^{
|
||||||
auto currentBridge = [RCTBridge currentBridge];
|
auto currentBridge = [RCTBridge currentBridge];
|
||||||
if (!currentBridge) return;
|
if (!currentBridge) return;
|
||||||
|
|
||||||
auto anonymousView = [currentBridge.uiManager viewForReactTag:[NSNumber numberWithDouble:viewTag]];
|
auto anonymousView = [currentBridge.uiManager viewForReactTag:[NSNumber numberWithDouble:viewTag]];
|
||||||
if (!anonymousView) return;
|
if (!anonymousView) return;
|
||||||
|
|
||||||
auto view = static_cast<CameraView*>(anonymousView);
|
auto view = static_cast<CameraView*>(anonymousView);
|
||||||
view.frameProcessorCallback = nil;
|
view.frameProcessorCallback = nil;
|
||||||
NSLog(@"FrameProcessorBindings: Frame processor removed!");
|
NSLog(@"FrameProcessorBindings: Frame processor removed!");
|
||||||
|
Loading…
Reference in New Issue
Block a user