diff --git a/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx b/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx index 7c4e264..3f430a2 100644 --- a/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx +++ b/docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx @@ -28,13 +28,13 @@ function App() { To achieve **maximum performance**, the `scanQRCodes` function is written in a native language (e.g. Objective-C), but it will be directly called from the VisionCamera Frame Processor JavaScript-Runtime. -### Return Types +### Types -Frame Processors can return any primitive value that is representable in JS. So for Objective-C that maps to: +The Frame Processor Plugin Registry API automatically manages type conversion from JS <-> native. They are converted into the most efficient data-structures, as seen here: | JS Type | Objective-C Type | Java Type | |----------------------|--------------------------|----------------------------| -| `number` | `NSNumber` | `double` | +| `number` | `NSNumber` (double) | `double` | | `boolean` | `NSNumber` (boolean) | `boolean` | | `string` | `NSString` | `String` | | `[]` | `NSArray` | `Array` | @@ -42,7 +42,9 @@ Frame Processors can return any primitive value that is representable in JS. So | `undefined` | `nil` / `NSNull` | `null` | | `(any, any) => void` | `RCTResponseSenderBlock` | `(Object, Object) -> void` | -The values will automatically be converted to JS values, so the following Objective-C frame processor: +### Return values + +Return values will automatically be converted to JS values, assuming they are representable in the ["Types" table](#types). So the following Objective-C frame processor: ```objc static inline id detectObject(CMSampleBufferRef buffer, NSArray args) { @@ -62,7 +64,7 @@ export function detectObject(frame: Frame): string { ### Parameters -Frame Processors can also accept parameters, following the same type convention as [return values](#return-types): +Frame Processors can also accept parameters, following the same type convention as [return values](#return-values): ```ts const frameProcessor = useFrameProcessor((frame) => { diff --git a/ios/Frame Processor/FrameProcessorRuntimeManager.mm b/ios/Frame Processor/FrameProcessorRuntimeManager.mm index e07cb8b..70bfb04 100644 --- a/ios/Frame Processor/FrameProcessorRuntimeManager.mm +++ b/ios/Frame Processor/FrameProcessorRuntimeManager.mm @@ -63,9 +63,11 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView"))) #ifdef ENABLE_FRAME_PROCESSORS NSLog(@"FrameProcessorBindings: Creating Runtime Manager..."); weakBridge = bridge; + auto runtime = vision::makeJSIRuntime(); reanimated::RuntimeDecorator::decorateRuntime(*runtime, "FRAME_PROCESSOR"); runtime->global().setProperty(*runtime, "_FRAME_PROCESSOR", jsi::Value(true)); + auto callInvoker = bridge.jsCallInvoker; auto scheduler = std::make_shared(callInvoker); runtimeManager = std::make_unique(std::move(runtime), @@ -76,6 +78,7 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView"))) NSLog(@"FrameProcessorBindings: Installing Frame Processor plugins..."); auto& visionRuntime = *runtimeManager->runtime; auto visionGlobal = visionRuntime.global(); + for (NSString* pluginKey in [FrameProcessorPluginRegistry frameProcessorPlugins]) { auto pluginName = [pluginKey UTF8String]; @@ -98,7 +101,9 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView"))) 1, // frame function)); } + [FrameProcessorPluginRegistry markInvalid]; + NSLog(@"FrameProcessorBindings: Frame Processor plugins installed!"); #else NSLog(@"Reanimated not found, Frame Processors are disabled."); @@ -113,11 +118,13 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView"))) NSLog(@"FrameProcessorBindings: Failed to install Frame Processor Bindings - bridge was null!"); return; } + NSLog(@"FrameProcessorBindings: Installing Frame Processor Bindings for Bridge..."); RCTCxxBridge *cxxBridge = (RCTCxxBridge *)weakBridge; if (!cxxBridge.runtime) { return; } + jsi::Runtime& jsiRuntime = *(jsi::Runtime*)cxxBridge.runtime; NSLog(@"FrameProcessorBindings: Installing global functions..."); @@ -142,6 +149,7 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView"))) NSLog(@"FrameProcessorBindings: Converting worklet to Objective-C callback..."); auto& rt = *runtimeManager->runtime; auto function = worklet->getValue(rt).asObject(rt).asFunction(rt); + view.frameProcessorCallback = convertJSIFunctionToFrameProcessorCallback(rt, function); NSLog(@"FrameProcessorBindings: Frame processor set!"); }); @@ -163,8 +171,10 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView"))) RCTExecuteOnMainQueue(^{ auto currentBridge = [RCTBridge currentBridge]; if (!currentBridge) return; + auto anonymousView = [currentBridge.uiManager viewForReactTag:[NSNumber numberWithDouble:viewTag]]; if (!anonymousView) return; + auto view = static_cast(anonymousView); view.frameProcessorCallback = nil; NSLog(@"FrameProcessorBindings: Frame processor removed!");