docs: Frame Processor Java Types (#183)

* add a few spaces

* Update FRAME_PROCESSORS_CREATE_OVERVIEW.mdx

* Update FRAME_PROCESSORS_CREATE_OVERVIEW.mdx

* Update FRAME_PROCESSORS_CREATE_OVERVIEW.mdx

* Update FRAME_PROCESSORS_CREATE_OVERVIEW.mdx
This commit is contained in:
Marc Rousavy 2021-06-08 10:15:34 +02:00 committed by GitHub
parent af805bd331
commit be30d55df5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 17 additions and 5 deletions

View File

@ -28,13 +28,13 @@ function App() {
To achieve **maximum performance**, the `scanQRCodes` function is written in a native language (e.g. Objective-C), but it will be directly called from the VisionCamera Frame Processor JavaScript-Runtime. To achieve **maximum performance**, the `scanQRCodes` function is written in a native language (e.g. Objective-C), but it will be directly called from the VisionCamera Frame Processor JavaScript-Runtime.
### Return Types ### Types
Frame Processors can return any primitive value that is representable in JS. So for Objective-C that maps to: The Frame Processor Plugin Registry API automatically manages type conversion from JS <-> native. They are converted into the most efficient data-structures, as seen here:
| JS Type | Objective-C Type | Java Type | | JS Type | Objective-C Type | Java Type |
|----------------------|--------------------------|----------------------------| |----------------------|--------------------------|----------------------------|
| `number` | `NSNumber` | `double` | | `number` | `NSNumber` (double) | `double` |
| `boolean` | `NSNumber` (boolean) | `boolean` | | `boolean` | `NSNumber` (boolean) | `boolean` |
| `string` | `NSString` | `String` | | `string` | `NSString` | `String` |
| `[]` | `NSArray` | `Array<Object>` | | `[]` | `NSArray` | `Array<Object>` |
@ -42,7 +42,9 @@ Frame Processors can return any primitive value that is representable in JS. So
| `undefined` | `nil` / `NSNull` | `null` | | `undefined` | `nil` / `NSNull` | `null` |
| `(any, any) => void` | `RCTResponseSenderBlock` | `(Object, Object) -> void` | | `(any, any) => void` | `RCTResponseSenderBlock` | `(Object, Object) -> void` |
The values will automatically be converted to JS values, so the following Objective-C frame processor: ### Return values
Return values will automatically be converted to JS values, assuming they are representable in the ["Types" table](#types). So the following Objective-C frame processor:
```objc ```objc
static inline id detectObject(CMSampleBufferRef buffer, NSArray args) { static inline id detectObject(CMSampleBufferRef buffer, NSArray args) {
@ -62,7 +64,7 @@ export function detectObject(frame: Frame): string {
### Parameters ### Parameters
Frame Processors can also accept parameters, following the same type convention as [return values](#return-types): Frame Processors can also accept parameters, following the same type convention as [return values](#return-values):
```ts ```ts
const frameProcessor = useFrameProcessor((frame) => { const frameProcessor = useFrameProcessor((frame) => {

View File

@ -63,9 +63,11 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
#ifdef ENABLE_FRAME_PROCESSORS #ifdef ENABLE_FRAME_PROCESSORS
NSLog(@"FrameProcessorBindings: Creating Runtime Manager..."); NSLog(@"FrameProcessorBindings: Creating Runtime Manager...");
weakBridge = bridge; weakBridge = bridge;
auto runtime = vision::makeJSIRuntime(); auto runtime = vision::makeJSIRuntime();
reanimated::RuntimeDecorator::decorateRuntime(*runtime, "FRAME_PROCESSOR"); reanimated::RuntimeDecorator::decorateRuntime(*runtime, "FRAME_PROCESSOR");
runtime->global().setProperty(*runtime, "_FRAME_PROCESSOR", jsi::Value(true)); runtime->global().setProperty(*runtime, "_FRAME_PROCESSOR", jsi::Value(true));
auto callInvoker = bridge.jsCallInvoker; auto callInvoker = bridge.jsCallInvoker;
auto scheduler = std::make_shared<reanimated::REAIOSScheduler>(callInvoker); auto scheduler = std::make_shared<reanimated::REAIOSScheduler>(callInvoker);
runtimeManager = std::make_unique<reanimated::RuntimeManager>(std::move(runtime), runtimeManager = std::make_unique<reanimated::RuntimeManager>(std::move(runtime),
@ -76,6 +78,7 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
NSLog(@"FrameProcessorBindings: Installing Frame Processor plugins..."); NSLog(@"FrameProcessorBindings: Installing Frame Processor plugins...");
auto& visionRuntime = *runtimeManager->runtime; auto& visionRuntime = *runtimeManager->runtime;
auto visionGlobal = visionRuntime.global(); auto visionGlobal = visionRuntime.global();
for (NSString* pluginKey in [FrameProcessorPluginRegistry frameProcessorPlugins]) { for (NSString* pluginKey in [FrameProcessorPluginRegistry frameProcessorPlugins]) {
auto pluginName = [pluginKey UTF8String]; auto pluginName = [pluginKey UTF8String];
@ -98,7 +101,9 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
1, // frame 1, // frame
function)); function));
} }
[FrameProcessorPluginRegistry markInvalid]; [FrameProcessorPluginRegistry markInvalid];
NSLog(@"FrameProcessorBindings: Frame Processor plugins installed!"); NSLog(@"FrameProcessorBindings: Frame Processor plugins installed!");
#else #else
NSLog(@"Reanimated not found, Frame Processors are disabled."); NSLog(@"Reanimated not found, Frame Processors are disabled.");
@ -113,11 +118,13 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
NSLog(@"FrameProcessorBindings: Failed to install Frame Processor Bindings - bridge was null!"); NSLog(@"FrameProcessorBindings: Failed to install Frame Processor Bindings - bridge was null!");
return; return;
} }
NSLog(@"FrameProcessorBindings: Installing Frame Processor Bindings for Bridge..."); NSLog(@"FrameProcessorBindings: Installing Frame Processor Bindings for Bridge...");
RCTCxxBridge *cxxBridge = (RCTCxxBridge *)weakBridge; RCTCxxBridge *cxxBridge = (RCTCxxBridge *)weakBridge;
if (!cxxBridge.runtime) { if (!cxxBridge.runtime) {
return; return;
} }
jsi::Runtime& jsiRuntime = *(jsi::Runtime*)cxxBridge.runtime; jsi::Runtime& jsiRuntime = *(jsi::Runtime*)cxxBridge.runtime;
NSLog(@"FrameProcessorBindings: Installing global functions..."); NSLog(@"FrameProcessorBindings: Installing global functions...");
@ -142,6 +149,7 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
NSLog(@"FrameProcessorBindings: Converting worklet to Objective-C callback..."); NSLog(@"FrameProcessorBindings: Converting worklet to Objective-C callback...");
auto& rt = *runtimeManager->runtime; auto& rt = *runtimeManager->runtime;
auto function = worklet->getValue(rt).asObject(rt).asFunction(rt); auto function = worklet->getValue(rt).asObject(rt).asFunction(rt);
view.frameProcessorCallback = convertJSIFunctionToFrameProcessorCallback(rt, function); view.frameProcessorCallback = convertJSIFunctionToFrameProcessorCallback(rt, function);
NSLog(@"FrameProcessorBindings: Frame processor set!"); NSLog(@"FrameProcessorBindings: Frame processor set!");
}); });
@ -163,8 +171,10 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
RCTExecuteOnMainQueue(^{ RCTExecuteOnMainQueue(^{
auto currentBridge = [RCTBridge currentBridge]; auto currentBridge = [RCTBridge currentBridge];
if (!currentBridge) return; if (!currentBridge) return;
auto anonymousView = [currentBridge.uiManager viewForReactTag:[NSNumber numberWithDouble:viewTag]]; auto anonymousView = [currentBridge.uiManager viewForReactTag:[NSNumber numberWithDouble:viewTag]];
if (!anonymousView) return; if (!anonymousView) return;
auto view = static_cast<CameraView*>(anonymousView); auto view = static_cast<CameraView*>(anonymousView);
view.frameProcessorCallback = nil; view.frameProcessorCallback = nil;
NSLog(@"FrameProcessorBindings: Frame processor removed!"); NSLog(@"FrameProcessorBindings: Frame processor removed!");