railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Complete iOS Codebase rewrite (#1647)

Browse Source 
* Make Frame Processors an extra subspec

* Update VisionCamera.podspec

* Make optional

* Make VisionCamera compile without Skia

* Fix

* Add skia again

* Update VisionCamera.podspec

* Make VisionCamera build without Frame Processors

* Rename error to `system/frame-processors-unavailable`

* Fix Frame Processor returning early

* Remove `preset`, FP partial rewrite

* Only warn on frame drop

* Fix wrong queue

* fix: Run on CameraQueue again

* Update CameraView.swift

* fix: Activate audio session asynchronously on audio queue

* Update CameraView+RecordVideo.swift

* Update PreviewView.h

* Cleanups

* Cleanup

* fix cast

* feat: Add LiDAR Depth Camera support

* Upgrade Ruby

* Add vector icons type

* Update Gemfile.lock

* fix: Stop queues on deinit

* Also load `builtInTrueDepthCamera`

* Update CameraViewManager.swift

* Update SkImageHelpers.mm

* Extract FrameProcessorCallback to FrameProcessor

Holds more context now :)

* Rename to .m

* fix: Add `RCTLog` import

* Create SkiaFrameProcessor

* Update CameraBridge.h

* Call Frame Processor

* Fix defines

* fix: Allow deleting callback funcs

* fix Skia build

* batch

* Just call `setSkiaFrameProcessor`

* Rewrite in Swift

* Pass `SkiaRenderer`

* Fix Import

* Move `PreviewView` to Swift

* Fix Layer

* Set Skia Canvas to Frame Host Object

* Make `DrawableFrameHostObject` subclass

* Fix TS types

* Use same MTLDevice and apply scale

* Make getter

* Extract `setTorch` and `Preview`

* fix: Fix nil metal device

* Don't wait for session stop in deinit

* Use main pixel ratio

* Use unique_ptr for Render Contexts

* fix: Fix SkiaPreviewDisplayLink broken after deinit

* inline `getTextureCache`

* Update CameraPage.tsx

* chore: Format iOS

* perf: Allow MTLLayer to be optimized for only frame buffers

* Add RN Video types

* fix: Fix Frame Processors if guard

* Find nodeModules recursively

* Create `Frame.isDrawable`

* Add `cocoapods-check` dependency
This commit is contained in:
Marc Rousavy
2023-07-20 15:30:04 +02:00
committed by GitHub
parent 5fb594ce6b
commit 375e894038
78 changed files with 1278 additions and 1245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
ios/Frame Processor/Frame.h
View File

@@ -14,9 +14,9 @@
@interface Frame : NSObject
- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer orientation:(UIImageOrientation)orientation;
- (instancetype _Nonnull) initWithBuffer:(CMSampleBufferRef _Nonnull)buffer orientation:(UIImageOrientation)orientation;
@property (nonatomic, readonly) CMSampleBufferRef buffer;
@property (nonatomic, readonly) CMSampleBufferRef _Nonnull buffer;
@property (nonatomic, readonly) UIImageOrientation orientation;
@end

4
ios/Frame Processor/Frame.m
View File

@@ -11,11 +11,11 @@
#import <CoreMedia/CMSampleBuffer.h>
@implementation Frame {
CMSampleBufferRef buffer;
CMSampleBufferRef _Nonnull buffer;
UIImageOrientation orientation;
}
- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer orientation:(UIImageOrientation)orientation {
- (instancetype) initWithBuffer:(CMSampleBufferRef _Nonnull)buffer orientation:(UIImageOrientation)orientation {
self = [super init];
if (self) {
_buffer = buffer;

7
ios/Frame Processor/FrameHostObject.h
View File

@@ -13,17 +13,11 @@
#import "Frame.h"
#import "SkCanvas.h"
#import "JsiSkCanvas.h"
using namespace facebook;
class JSI_EXPORT FrameHostObject: public jsi::HostObject {
public:
explicit FrameHostObject(Frame* frame): frame(frame) {}
explicit FrameHostObject(Frame* frame,
std::shared_ptr<RNSkia::JsiSkCanvas> canvas):
frame(frame), canvas(canvas) {}
public:
jsi::Value get(jsi::Runtime&, const jsi::PropNameID& name) override;
@@ -31,5 +25,4 @@ public:
public:
Frame* frame;
std::shared_ptr<RNSkia::JsiSkCanvas> canvas;
};

54
ios/Frame Processor/FrameHostObject.mm
View File

@@ -11,8 +11,6 @@
#import <jsi/jsi.h>
#import "WKTJsiHostObject.h"
#import "SkCanvas.h"
#import "../Skia Render Layer/SkImageHelpers.h"
#import "../../cpp/JSITypedArray.h"
std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt) {
@@ -24,6 +22,7 @@ std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt)
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("orientation")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isMirrored")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("timestamp")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isDrawable")));
// Conversion
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toString")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toArrayBuffer")));
@@ -31,27 +30,10 @@ std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt)
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isValid")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("incrementRefCount")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("decrementRefCount")));
// Skia
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("render")));
if (canvas != nullptr) {
auto canvasPropNames = canvas->getPropertyNames(rt);
for (auto& prop : canvasPropNames) {
result.push_back(std::move(prop));
}
}
return result;
}
SkRect inscribe(SkSize size, SkRect rect) {
auto halfWidthDelta = (rect.width() - size.width()) / 2.0;
auto halfHeightDelta = (rect.height() - size.height()) / 2.0;
return SkRect::MakeXYWH(rect.x() + halfWidthDelta,
rect.y() + halfHeightDelta, size.width(),
size.height());
}
jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
auto name = propName.utf8(runtime);
@@ -80,7 +62,6 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
0,
incrementRefCount);
}
if (name == "decrementRefCount") {
auto decrementRefCount = JSI_HOST_FUNCTION_LAMBDA {
// Decrement retain count by one. If the retain count is zero, ARC will destroy the Frame Buffer.
@@ -92,31 +73,6 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
0,
decrementRefCount);
}
if (name == "render") {
auto render = JSI_HOST_FUNCTION_LAMBDA {
if (canvas == nullptr) {
throw jsi::JSError(runtime, "Trying to render a Frame without a Skia Canvas! Did you install Skia?");
}
// convert CMSampleBuffer to SkImage
auto context = canvas->getCanvas()->recordingContext();
auto image = SkImageHelpers::convertCMSampleBufferToSkImage(context, frame.buffer);
// draw SkImage
if (count > 0) {
// ..with paint/shader
auto paintHostObject = arguments[0].asObject(runtime).asHostObject<RNSkia::JsiSkPaint>(runtime);
auto paint = paintHostObject->getObject();
canvas->getCanvas()->drawImage(image, 0, 0, SkSamplingOptions(), paint.get());
} else {
// ..without paint/shader
canvas->getCanvas()->drawImage(image, 0, 0);
}
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "render"), 1, render);
}
if (name == "toArrayBuffer") {
auto toArrayBuffer = JSI_HOST_FUNCTION_LAMBDA {
auto pixelBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
@@ -146,6 +102,9 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toArrayBuffer"), 0, toArrayBuffer);
}
if (name == "isDrawable") {
return jsi::Value(false);
}
if (name == "isValid") {
auto isValid = frame != nil && frame.buffer != nil && CFGetRetainCount(frame.buffer) > 0 && CMSampleBufferIsValid(frame.buffer);
return jsi::Value(isValid);
@@ -206,11 +165,6 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
return jsi::Value((double) planesCount);
}
if (canvas != nullptr) {
// If we have a Canvas, try to access the property on there.
return canvas->get(runtime, propName);
}
// fallback to base implementation
return HostObject::get(runtime, propName);
}

33
ios/Frame Processor/FrameProcessor.h Normal file
View File

@@ -0,0 +1,33 @@
//
// FrameProcessorContext.h
// VisionCamera
//
// Created by Marc Rousavy on 13.07.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
#pragma once
#import <Foundation/Foundation.h>
#import <AVFoundation/AVFoundation.h>
#import "Frame.h"
#ifdef __cplusplus
#import "WKTJsiWorklet.h"
#import <jsi/jsi.h>
#import "FrameHostObject.h"
#import <memory.h>
#endif
@interface FrameProcessor : NSObject
#ifdef __cplusplus
- (instancetype _Nonnull)initWithWorklet:(std::shared_ptr<RNWorklet::JsiWorkletContext>)context
worklet:(std::shared_ptr<RNWorklet::JsiWorklet>)worklet;
- (void)callWithFrameHostObject:(std::shared_ptr<FrameHostObject>)frameHostObject;
#endif
- (void)call:(Frame* _Nonnull)frame;
@end

61
ios/Frame Processor/FrameProcessor.mm Normal file
View File

@@ -0,0 +1,61 @@
//
// FrameProcessor.mm
// VisionCamera
//
// Created by Marc Rousavy on 13.07.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
#import <Foundation/Foundation.h>
#import "FrameProcessor.h"
#import <memory>
#import <jsi/jsi.h>
#import "WKTJsiWorklet.h"
#import "FrameHostObject.h"
using namespace facebook;
@implementation FrameProcessor {
std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;
std::shared_ptr<RNWorklet::WorkletInvoker> _workletInvoker;
}
- (instancetype)initWithWorklet:(std::shared_ptr<RNWorklet::JsiWorkletContext>)context
worklet:(std::shared_ptr<RNWorklet::JsiWorklet>)worklet {
if (self = [super init]) {
_workletContext = context;
_workletInvoker = std::make_shared<RNWorklet::WorkletInvoker>(worklet);
}
return self;
}
- (void)callWithFrameHostObject:(std::shared_ptr<FrameHostObject>)frameHostObject {
// Call the Frame Processor on the Worklet Runtime
jsi::Runtime& runtime = _workletContext->getWorkletRuntime();
try {
// Wrap HostObject as JSI Value
auto argument = jsi::Object::createFromHostObject(runtime, frameHostObject);
jsi::Value jsValue(std::move(argument));
// Call the Worklet with the Frame JS Host Object as an argument
_workletInvoker->call(runtime, jsi::Value::undefined(), &jsValue, 1);
} catch (jsi::JSError& jsError) {
// JS Error occured, print it to console.
auto message = jsError.getMessage();
_workletContext->invokeOnJsThread([message](jsi::Runtime& jsRuntime) {
auto logFn = jsRuntime.global().getPropertyAsObject(jsRuntime, "console").getPropertyAsFunction(jsRuntime, "error");
logFn.call(jsRuntime, jsi::String::createFromUtf8(jsRuntime, "Frame Processor threw an error: " + message));
});
}
}
- (void)call:(Frame* _Nonnull)frame {
// Create the Frame Host Object wrapping the internal Frame
auto frameHostObject = std::make_shared<FrameHostObject>(frame);
[self callWithFrameHostObject:frameHostObject];
}
@end

14
ios/Frame Processor/FrameProcessorCallback.h
View File

@@ -1,14 +0,0 @@
//
// FrameProcessorCallback.h
// VisionCamera
//
// Created by Marc Rousavy on 11.03.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
#pragma once
#import <Foundation/Foundation.h>
#import "Frame.h"
typedef void (^FrameProcessorCallback) (Frame* frame, void* skCanvas);

6
ios/Frame Processor/FrameProcessorPlugin.h
View File

@@ -20,13 +20,13 @@
/// Get the name of the Frame Processor Plugin.
/// This will be exposed to JS under the `FrameProcessorPlugins` Proxy object.
- (NSString *)name;
- (NSString * _Nonnull)name;
/// The actual callback when calling this plugin. Any Frame Processing should be handled there.
/// Make sure your code is optimized, as this is a hot path.
- (id) callback:(Frame*)frame withArguments:(NSArray<id>*)arguments;
- (id _Nullable) callback:(Frame* _Nonnull)frame withArguments:(NSArray<id>* _Nullable)arguments;
/// Register the given plugin in the Plugin Registry. This should be called on App Startup.
+ (void) registerPlugin:(FrameProcessorPlugin*)plugin;
+ (void) registerPlugin:(FrameProcessorPlugin* _Nonnull)plugin;
@end

4
ios/Frame Processor/FrameProcessorPlugin.m
View File

@@ -18,13 +18,13 @@
return nil;
}
- (id)callback:(Frame *)frame withArguments:(NSArray<id> *)arguments {
- (id _Nullable)callback:(Frame* _Nonnull)frame withArguments:(NSArray<id>* _Nullable)arguments {
[NSException raise:NSInternalInconsistencyException
format:@"Frame Processor Plugin \"%@\" does not override the `callback(frame:withArguments:)` method!", [self name]];
return nil;
}
+ (void)registerPlugin:(FrameProcessorPlugin *)plugin {
+ (void)registerPlugin:(FrameProcessorPlugin* _Nonnull)plugin {
[FrameProcessorPluginRegistry addFrameProcessorPlugin:plugin];
}

2
ios/Frame Processor/FrameProcessorPluginRegistry.h
View File

@@ -15,6 +15,6 @@
@interface FrameProcessorPluginRegistry : NSObject
+ (NSMutableDictionary<NSString*, FrameProcessorPlugin*>*)frameProcessorPlugins;
+ (void) addFrameProcessorPlugin:(FrameProcessorPlugin*)plugin;
+ (void) addFrameProcessorPlugin:(FrameProcessorPlugin* _Nonnull)plugin;
@end

0
ios/Frame Processor/FrameProcessorPluginRegistry.mm → ios/Frame Processor/FrameProcessorPluginRegistry.m
View File

2
ios/Frame Processor/FrameProcessorRuntimeManager.h
View File

@@ -13,8 +13,6 @@
@interface FrameProcessorRuntimeManager : NSObject
- (instancetype) init;
- (void) installFrameProcessorBindings;
@end

49
ios/Frame Processor/FrameProcessorRuntimeManager.mm
View File

@@ -10,6 +10,7 @@
#import "FrameProcessorRuntimeManager.h"
#import "FrameProcessorPluginRegistry.h"
#import "FrameProcessorPlugin.h"
#import "FrameProcessor.h"
#import "FrameHostObject.h"
#import <memory>
@@ -21,15 +22,15 @@
#import <ReactCommon/RCTTurboModuleManager.h>
#import "WKTJsiWorkletContext.h"
#import "WKTJsiWorkletApi.h"
#import "WKTJsiWorklet.h"
#import "WKTJsiHostObject.h"
#import "FrameProcessorUtils.h"
#import "FrameProcessorCallback.h"
#import "../React Utils/JSIUtils.h"
#import "../../cpp/JSITypedArray.h"
#if VISION_CAMERA_ENABLE_SKIA
#import "../Skia Render Layer/SkiaFrameProcessor.h"
#endif
// Forward declarations for the Swift classes
__attribute__((objc_runtime_name("_TtC12VisionCamera12CameraQueues")))
@interface CameraQueues : NSObject
@@ -37,21 +38,15 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera12CameraQueues")))
@end
__attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
@interface CameraView : UIView
@property (nonatomic, copy) FrameProcessorCallback _Nullable frameProcessorCallback;
@property (nonatomic, copy) FrameProcessor* _Nullable frameProcessor;
- (SkiaRenderer* _Nonnull)getSkiaRenderer;
@end
@implementation FrameProcessorRuntimeManager {
// Running Frame Processors on camera's video thread (synchronously)
// Separate Camera Worklet Context
std::shared_ptr<RNWorklet::JsiWorkletContext> workletContext;
}
- (instancetype)init {
if (self = [super init]) {
// Initialize self
}
return self;
}
- (void) setupWorkletContext:(jsi::Runtime&)runtime {
NSLog(@"FrameProcessorBindings: Creating Worklet Context...");
@@ -136,7 +131,7 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
// HostObject that attaches the cache to the lifecycle of the Runtime. On Runtime destroy, we destroy the cache.
auto propNameCacheObject = std::make_shared<vision::InvalidateCacheOnDestroy>(jsiRuntime);
jsiRuntime.global().setProperty(jsiRuntime,
"__visionCameraPropNameCache",
"__visionCameraArrayBufferCache",
jsi::Object::createFromHostObject(jsiRuntime, propNameCacheObject));
// Install the Worklet Runtime in the main React JS Runtime
@@ -148,14 +143,30 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
auto setFrameProcessor = JSI_HOST_FUNCTION_LAMBDA {
NSLog(@"FrameProcessorBindings: Setting new frame processor...");
auto viewTag = arguments[0].asNumber();
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, arguments[1]);
auto object = arguments[1].asObject(runtime);
auto frameProcessorType = object.getProperty(runtime, "type").asString(runtime).utf8(runtime);
auto worklet = std::make_shared<RNWorklet::JsiWorklet>(runtime, object.getProperty(runtime, "frameProcessor"));
RCTExecuteOnMainQueue(^{
auto currentBridge = [RCTBridge currentBridge];
auto anonymousView = [currentBridge.uiManager viewForReactTag:[NSNumber numberWithDouble:viewTag]];
auto view = static_cast<CameraView*>(anonymousView);
auto callback = convertWorkletToFrameProcessorCallback(self->workletContext->getWorkletRuntime(), worklet);
view.frameProcessorCallback = callback;
if (frameProcessorType == "frame-processor") {
view.frameProcessor = [[FrameProcessor alloc] initWithWorklet:self->workletContext
worklet:worklet];
} else if (frameProcessorType == "skia-frame-processor") {
#if VISION_CAMERA_ENABLE_SKIA
SkiaRenderer* skiaRenderer = [view getSkiaRenderer];
view.frameProcessor = [[SkiaFrameProcessor alloc] initWithWorklet:self->workletContext
worklet:worklet
skiaRenderer:skiaRenderer];
#else
throw std::runtime_error("system/skia-unavailable: Skia is not installed!");
#endif
} else {
throw std::runtime_error("Unknown FrameProcessor.type passed! Received: " + frameProcessorType);
}
});
return jsi::Value::undefined();
@@ -175,10 +186,8 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
if (!currentBridge) return;
auto anonymousView = [currentBridge.uiManager viewForReactTag:[NSNumber numberWithDouble:viewTag]];
if (!anonymousView) return;
auto view = static_cast<CameraView*>(anonymousView);
view.frameProcessorCallback = nil;
view.frameProcessor = nil;
});
return jsi::Value::undefined();

25
ios/Frame Processor/FrameProcessorUtils.h
View File

@@ -1,25 +0,0 @@
//
// FrameProcessorUtils.h
// VisionCamera
//
// Created by Marc Rousavy on 15.03.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
#pragma once
#import <Foundation/Foundation.h>
#import <React/RCTBridgeModule.h>
#import "FrameProcessorCallback.h"
#ifndef __cplusplus
#error FrameProcessorUtils.h has to be compiled with C++!
#endif
#import <jsi/jsi.h>
#import "WKTJsiWorklet.h"
#import <memory>
using namespace facebook;
FrameProcessorCallback convertWorkletToFrameProcessorCallback(jsi::Runtime& runtime, std::shared_ptr<RNWorklet::JsiWorklet> worklet);

72
ios/Frame Processor/FrameProcessorUtils.mm
View File

@@ -1,72 +0,0 @@
//
// FrameProcessorUtils.m
// VisionCamera
//
// Created by Marc Rousavy on 15.03.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
#import "FrameProcessorUtils.h"
#import <chrono>
#import <memory>
#import <regex>
#import "FrameHostObject.h"
#import "Frame.h"
#import <React/RCTBridge.h>
#import <React/RCTBridge+Private.h>
#import "JSConsoleHelper.h"
#import <ReactCommon/RCTTurboModule.h>
#import "WKTJsiWorklet.h"
#import "RNSkPlatformContext.h"
#import "RNSkiOSPlatformContext.h"
#import "JsiSkCanvas.h"
FrameProcessorCallback convertWorkletToFrameProcessorCallback(jsi::Runtime& runtime, std::shared_ptr<RNWorklet::JsiWorklet> worklet) {
// Wrap Worklet call in invoker
auto workletInvoker = std::make_shared<RNWorklet::WorkletInvoker>(worklet);
// Create cached Skia Canvas object
auto skiaPlatformContext = std::make_shared<RNSkia::RNSkiOSPlatformContext>(&runtime, RCTBridge.currentBridge);
auto canvasHostObject = std::make_shared<RNSkia::JsiSkCanvas>(skiaPlatformContext);
// Converts a Worklet to a callable Objective-C block function
return ^(Frame* frame, void* skiaCanvas) {
try {
// create HostObject which holds the Frame
auto frameHostObject = std::make_shared<FrameHostObject>(frame);
// Update cached Canvas object
if (skiaCanvas != nullptr) {
canvasHostObject->setCanvas((SkCanvas*)skiaCanvas);
frameHostObject->canvas = canvasHostObject;
} else {
frameHostObject->canvas = nullptr;
}
auto argument = jsi::Object::createFromHostObject(runtime, frameHostObject);
jsi::Value jsValue(std::move(argument));
// Call the Worklet with the Frame JS Host Object as an argument
workletInvoker->call(runtime, jsi::Value::undefined(), &jsValue, 1);
// After the sync Frame Processor finished executing, remove the Canvas on that Frame instance. It can no longer draw.
frameHostObject->canvas = nullptr;
} catch (jsi::JSError& jsError) {
// JS Error occured, print it to console.
auto stack = std::regex_replace(jsError.getStack(), std::regex("\n"), "\n ");
auto message = [NSString stringWithFormat:@"Frame Processor threw an error: %s\nIn: %s", jsError.getMessage().c_str(), stack.c_str()];
RCTBridge* bridge = [RCTBridge currentBridge];
if (bridge != nil && bridge.jsCallInvoker != nullptr) {
bridge.jsCallInvoker->invokeAsync([bridge, message]() {
auto logFn = [JSConsoleHelper getLogFunctionForBridge:bridge];
logFn(RCTLogLevelError, message);
});
} else {
NSLog(@"%@", message);
}
}
};
}