railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Complete iOS Codebase rewrite (#1647)

Browse Source 
* Make Frame Processors an extra subspec

* Update VisionCamera.podspec

* Make optional

* Make VisionCamera compile without Skia

* Fix

* Add skia again

* Update VisionCamera.podspec

* Make VisionCamera build without Frame Processors

* Rename error to `system/frame-processors-unavailable`

* Fix Frame Processor returning early

* Remove `preset`, FP partial rewrite

* Only warn on frame drop

* Fix wrong queue

* fix: Run on CameraQueue again

* Update CameraView.swift

* fix: Activate audio session asynchronously on audio queue

* Update CameraView+RecordVideo.swift

* Update PreviewView.h

* Cleanups

* Cleanup

* fix cast

* feat: Add LiDAR Depth Camera support

* Upgrade Ruby

* Add vector icons type

* Update Gemfile.lock

* fix: Stop queues on deinit

* Also load `builtInTrueDepthCamera`

* Update CameraViewManager.swift

* Update SkImageHelpers.mm

* Extract FrameProcessorCallback to FrameProcessor

Holds more context now :)

* Rename to .m

* fix: Add `RCTLog` import

* Create SkiaFrameProcessor

* Update CameraBridge.h

* Call Frame Processor

* Fix defines

* fix: Allow deleting callback funcs

* fix Skia build

* batch

* Just call `setSkiaFrameProcessor`

* Rewrite in Swift

* Pass `SkiaRenderer`

* Fix Import

* Move `PreviewView` to Swift

* Fix Layer

* Set Skia Canvas to Frame Host Object

* Make `DrawableFrameHostObject` subclass

* Fix TS types

* Use same MTLDevice and apply scale

* Make getter

* Extract `setTorch` and `Preview`

* fix: Fix nil metal device

* Don't wait for session stop in deinit

* Use main pixel ratio

* Use unique_ptr for Render Contexts

* fix: Fix SkiaPreviewDisplayLink broken after deinit

* inline `getTextureCache`

* Update CameraPage.tsx

* chore: Format iOS

* perf: Allow MTLLayer to be optimized for only frame buffers

* Add RN Video types

* fix: Fix Frame Processors if guard

* Find nodeModules recursively

* Create `Frame.isDrawable`

* Add `cocoapods-check` dependency
This commit is contained in:
Marc Rousavy
2023-07-20 15:30:04 +02:00
committed by GitHub
parent 5fb594ce6b
commit 375e894038
78 changed files with 1278 additions and 1245 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
ios/Skia Render Layer/DrawableFrameHostObject.h Normal file
View File

@@ -0,0 +1,35 @@
//
// DrawableFrameHostObject.h
// VisionCamera
//
// Created by Marc Rousavy on 20.07.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
#pragma once
#import <jsi/jsi.h>
#import "../Frame Processor/FrameHostObject.h"
#import "../Frame Processor/Frame.h"
#import <CoreMedia/CMSampleBuffer.h>
#import "SkCanvas.h"
#import "JsiSkCanvas.h"
using namespace facebook;
class JSI_EXPORT DrawableFrameHostObject: public FrameHostObject {
public:
explicit DrawableFrameHostObject(Frame* frame,
std::shared_ptr<RNSkia::JsiSkCanvas> canvas):
FrameHostObject(frame), _canvas(canvas) {}
public:
jsi::Value get(jsi::Runtime&, const jsi::PropNameID& name) override;
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& rt) override;
void invalidateCanvas();
private:
std::shared_ptr<RNSkia::JsiSkCanvas> _canvas;
};

83
ios/Skia Render Layer/DrawableFrameHostObject.mm Normal file
View File

@@ -0,0 +1,83 @@
//
// DrawableFrameHostObject.mm
// VisionCamera
//
// Created by Marc Rousavy on 20.07.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
#import "DrawableFrameHostObject.h"
#import "SkCanvas.h"
#import "SkImageHelpers.h"
std::vector<jsi::PropNameID> DrawableFrameHostObject::getPropertyNames(jsi::Runtime& rt) {
auto result = FrameHostObject::getPropertyNames(rt);
// Skia - Render Frame
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("render")));
if (_canvas != nullptr) {
auto canvasPropNames = _canvas->getPropertyNames(rt);
for (auto& prop : canvasPropNames) {
result.push_back(std::move(prop));
}
}
return result;
}
SkRect inscribe(SkSize size, SkRect rect) {
auto halfWidthDelta = (rect.width() - size.width()) / 2.0;
auto halfHeightDelta = (rect.height() - size.height()) / 2.0;
return SkRect::MakeXYWH(rect.x() + halfWidthDelta,
rect.y() + halfHeightDelta, size.width(),
size.height());
}
jsi::Value DrawableFrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
auto name = propName.utf8(runtime);
if (name == "render") {
auto render = JSI_HOST_FUNCTION_LAMBDA {
if (_canvas == nullptr) {
throw jsi::JSError(runtime, "Trying to render a Frame without a Skia Canvas! Did you install Skia?");
}
// convert CMSampleBuffer to SkImage
auto context = _canvas->getCanvas()->recordingContext();
auto image = SkImageHelpers::convertCMSampleBufferToSkImage(context, frame.buffer);
// draw SkImage
if (count > 0) {
// ..with paint/shader
auto paintHostObject = arguments[0].asObject(runtime).asHostObject<RNSkia::JsiSkPaint>(runtime);
auto paint = paintHostObject->getObject();
_canvas->getCanvas()->drawImage(image, 0, 0, SkSamplingOptions(), paint.get());
} else {
// ..without paint/shader
_canvas->getCanvas()->drawImage(image, 0, 0);
}
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "render"), 1, render);
}
if (name == "isDrawable") {
return jsi::Value(_canvas != nullptr);
}
if (_canvas != nullptr) {
// If we have a Canvas, try to access the property on there.
auto result = _canvas->get(runtime, propName);
if (!result.isUndefined()) {
return result;
}
}
// fallback to base implementation
return FrameHostObject::get(runtime, propName);
}
void DrawableFrameHostObject::invalidateCanvas() {
_canvas = nullptr;
}

26
ios/Skia Render Layer/PreviewSkiaView.h
View File

@@ -1,26 +0,0 @@
//
// PreviewSkiaView.h
// VisionCamera
//
// Created by Marc Rousavy on 17.11.22.
// Copyright © 2022 mrousavy. All rights reserved.
//
#ifndef PreviewSkiaView_h
#define PreviewSkiaView_h
#import <UIKit/UIKit.h>
#import <AVFoundation/AVFoundation.h>
#import "FrameProcessorCallback.h"
typedef void (^DrawCallback) (void* _Nonnull skCanvas);
@interface PreviewSkiaView: UIView
// Call to pass a new Frame to be drawn by the Skia Canvas
- (void) drawFrame:(_Nonnull CMSampleBufferRef)buffer withCallback:(DrawCallback _Nonnull)callback;
@end
#endif /* PreviewSkiaView_h */

60
ios/Skia Render Layer/PreviewSkiaView.mm
View File

@@ -1,60 +0,0 @@
//
// PreviewSkiaView.mm
// VisionCamera
//
// Created by Marc Rousavy on 17.11.22.
// Copyright © 2022 mrousavy. All rights reserved.
//
#import "PreviewSkiaView.h"
#import <Foundation/Foundation.h>
#import "SkiaMetalCanvasProvider.h"
#include <include/core/SkCanvas.h>
#include <exception>
#include <string>
#if SHOW_FPS
#import <React/RCTFPSGraph.h>
#endif
@implementation PreviewSkiaView {
std::shared_ptr<SkiaMetalCanvasProvider> _canvasProvider;
}
- (void)drawFrame:(CMSampleBufferRef)buffer withCallback:(DrawCallback _Nonnull)callback {
if (_canvasProvider == nullptr) {
throw std::runtime_error("Cannot draw new Frame to Canvas when SkiaMetalCanvasProvider is null!");
}
_canvasProvider->renderFrameToCanvas(buffer, ^(SkCanvas* canvas) {
callback((void*)canvas);
});
}
- (void) willMoveToSuperview:(UIView *)newWindow {
if (newWindow == NULL) {
// Remove implementation view when the parent view is not set
if (_canvasProvider != nullptr) {
[_canvasProvider->getLayer() removeFromSuperlayer];
_canvasProvider = nullptr;
}
} else {
// Create implementation view when the parent view is set
if (_canvasProvider == nullptr) {
_canvasProvider = std::make_shared<SkiaMetalCanvasProvider>();
[self.layer addSublayer: _canvasProvider->getLayer()];
_canvasProvider->start();
}
}
}
- (void) layoutSubviews {
if (_canvasProvider != nullptr) {
_canvasProvider->setSize(self.bounds.size.width, self.bounds.size.height);
}
}
@end

66
ios/Skia Render Layer/SkImageHelpers.mm
View File

@@ -25,18 +25,18 @@
# define FourCC2Str(fourcc) (const char[]){*(((char*)&fourcc)+3), *(((char*)&fourcc)+2), *(((char*)&fourcc)+1), *(((char*)&fourcc)+0),0}
#endif
CVMetalTextureCacheRef getTextureCache(GrRecordingContext* context) {
inline CVMetalTextureCacheRef getTextureCache() {
static CVMetalTextureCacheRef textureCache = nil;
if (textureCache == nil) {
// Create a new Texture Cache
auto result = CVMetalTextureCacheCreate(kCFAllocatorDefault,
nil,
MTLCreateSystemDefaultDevice(),
nil,
&textureCache);
if (result != kCVReturnSuccess || textureCache == nil) {
throw std::runtime_error("Failed to create Metal Texture Cache!");
}
auto result = CVMetalTextureCacheCreate(kCFAllocatorDefault,
nil,
MTLCreateSystemDefaultDevice(),
nil,
&textureCache);
if (result != kCVReturnSuccess || textureCache == nil) {
throw std::runtime_error("Failed to create Metal Texture Cache!");
}
}
return textureCache;
}
@@ -45,46 +45,34 @@ sk_sp<SkImage> SkImageHelpers::convertCMSampleBufferToSkImage(GrRecordingContext
auto pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
double width = CVPixelBufferGetWidth(pixelBuffer);
double height = CVPixelBufferGetHeight(pixelBuffer);
// Make sure the format is RGB (BGRA_8888)
auto format = CVPixelBufferGetPixelFormatType(pixelBuffer);
if (format != kCVPixelFormatType_32BGRA) {
auto fourCharCode = @(FourCC2Str(format));
auto error = std::string("VisionCamera: Frame has unknown Pixel Format (") + fourCharCode.UTF8String + std::string(") - cannot convert to SkImage!");
auto error = std::string("VisionCamera: Frame has unknown Pixel Format (") + FourCC2Str(format) + std::string(") - cannot convert to SkImage!");
throw std::runtime_error(error);
}
auto textureCache = getTextureCache(context);
auto textureCache = getTextureCache();
// Convert CMSampleBuffer* -> CVMetalTexture*
CVMetalTextureRef cvTexture;
CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
textureCache,
pixelBuffer,
nil,
MTLPixelFormatBGRA8Unorm,
width,
height,
0, // plane index
&cvTexture);
GrMtlTextureInfo textureInfo;
textureCache,
pixelBuffer,
nil,
MTLPixelFormatBGRA8Unorm,
width,
height,
0, // plane index
&cvTexture);
auto mtlTexture = CVMetalTextureGetTexture(cvTexture);
textureInfo.fTexture.retain((__bridge void*)mtlTexture);
// Wrap it in a GrBackendTexture
GrBackendTexture texture(width, height, GrMipmapped::kNo, textureInfo);
auto image = convertMTLTextureToSkImage(context, mtlTexture);
// Create an SkImage from the existing texture
auto image = SkImages::AdoptTextureFrom(context,
texture,
kTopLeft_GrSurfaceOrigin,
kBGRA_8888_SkColorType,
kOpaque_SkAlphaType,
SkColorSpace::MakeSRGB());
// Release the Texture wrapper (it will still be strong)
CFRelease(cvTexture);
return image;
}
@@ -92,7 +80,11 @@ sk_sp<SkImage> SkImageHelpers::convertMTLTextureToSkImage(GrRecordingContext* co
// Convert the rendered MTLTexture to an SkImage
GrMtlTextureInfo textureInfo;
textureInfo.fTexture.retain((__bridge void*)texture);
GrBackendTexture backendTexture(texture.width, texture.height, GrMipmapped::kNo, textureInfo);
GrBackendTexture backendTexture((int)texture.width,
(int)texture.height,
GrMipmapped::kNo,
textureInfo);
// TODO: Adopt or Borrow?
auto image = SkImages::AdoptTextureFrom(context,
backendTexture,
kTopLeft_GrSurfaceOrigin,
@@ -109,7 +101,7 @@ SkRect SkImageHelpers::createCenterCropRect(SkRect sourceRect, SkRect destinatio
} else {
src = SkSize::Make((sourceRect.height() * destinationRect.width()) / destinationRect.height(), sourceRect.height());
}
return inscribe(src, sourceRect);
}

27
ios/Skia Render Layer/SkiaFrameProcessor.h Normal file
View File

@@ -0,0 +1,27 @@
//
// SkiaFrameProcessor.h
// VisionCamera
//
// Created by Marc Rousavy on 14.07.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
#pragma once
#import <Foundation/Foundation.h>
#import "FrameProcessor.h"
#import "SkiaRenderer.h"
#ifdef __cplusplus
#import "WKTJsiWorklet.h"
#endif
@interface SkiaFrameProcessor: FrameProcessor
#ifdef __cplusplus
- (instancetype _Nonnull) initWithWorklet:(std::shared_ptr<RNWorklet::JsiWorkletContext>)context
worklet:(std::shared_ptr<RNWorklet::JsiWorklet>)worklet
skiaRenderer:(SkiaRenderer* _Nonnull)skiaRenderer;
#endif
@end

56
ios/Skia Render Layer/SkiaFrameProcessor.mm Normal file
View File

@@ -0,0 +1,56 @@
//
// SkiaFrameProcessor.mm
// VisionCamera
//
// Created by Marc Rousavy on 14.07.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
#import <Foundation/Foundation.h>
#import "SkiaFrameProcessor.h"
#import "SkiaRenderer.h"
#import <memory>
#import <jsi/jsi.h>
#import "DrawableFrameHostObject.h"
#import <react-native-skia/JsiSkCanvas.h>
#import <react-native-skia/RNSkiOSPlatformContext.h>
using namespace facebook;
@implementation SkiaFrameProcessor {
SkiaRenderer* _skiaRenderer;
std::shared_ptr<RNSkia::JsiSkCanvas> _skiaCanvas;
}
- (instancetype _Nonnull)initWithWorklet:(std::shared_ptr<RNWorklet::JsiWorkletContext>)context
worklet:(std::shared_ptr<RNWorklet::JsiWorklet>)worklet
skiaRenderer:(SkiaRenderer* _Nonnull)skiaRenderer {
if (self = [super initWithWorklet:context
worklet:worklet]) {
_skiaRenderer = skiaRenderer;
auto platformContext = std::make_shared<RNSkia::RNSkiOSPlatformContext>(context->getJsRuntime(),
RCTBridge.currentBridge);
_skiaCanvas = std::make_shared<RNSkia::JsiSkCanvas>(platformContext);
}
return self;
}
- (void)call:(Frame*)frame {
[_skiaRenderer renderCameraFrameToOffscreenCanvas:frame.buffer
withDrawCallback:^(SkiaCanvas _Nonnull canvas) {
// Create the Frame Host Object wrapping the internal Frame and Skia Canvas
self->_skiaCanvas->setCanvas(static_cast<SkCanvas*>(canvas));
auto frameHostObject = std::make_shared<DrawableFrameHostObject>(frame, self->_skiaCanvas);
// Call JS Frame Processor
[self callWithFrameHostObject:frameHostObject];
// Remove Skia Canvas from Host Object because it is no longer valid
frameHostObject->invalidateCanvas();
}];
}
@end

57
ios/Skia Render Layer/SkiaMetalCanvasProvider.h
View File

@@ -1,57 +0,0 @@
#pragma once
#ifndef __cplusplus
#error This header has to be compiled with C++!
#endif
#import <MetalKit/MetalKit.h>
#import <QuartzCore/CAMetalLayer.h>
#import <AVFoundation/AVFoundation.h>
#include <include/gpu/GrDirectContext.h>
#include <include/core/SkCanvas.h>
#include <functional>
#include <mutex>
#include <memory>
#include <atomic>
#import "VisionDisplayLink.h"
#import "SkiaMetalRenderContext.h"
class SkiaMetalCanvasProvider: public std::enable_shared_from_this<SkiaMetalCanvasProvider> {
public:
SkiaMetalCanvasProvider();
~SkiaMetalCanvasProvider();
// Render a Camera Frame to the off-screen canvas
void renderFrameToCanvas(CMSampleBufferRef sampleBuffer, const std::function<void(SkCanvas*)>& drawCallback);
// Start updating the DisplayLink (runLoop @ screen refresh rate) and draw Frames to the Layer
void start();
// Update the size of the View (Layer)
void setSize(int width, int height);
CALayer* getLayer();
private:
bool _isValid = false;
float _width = -1;
float _height = -1;
// For rendering Camera Frame -> off-screen MTLTexture
OffscreenRenderContext _offscreenContext;
// For rendering off-screen MTLTexture -> on-screen CAMetalLayer
LayerRenderContext _layerContext;
// For synchronization between the two Threads/Contexts
std::mutex _textureMutex;
std::atomic<bool> _hasNewFrame = false;
private:
void render();
id<MTLTexture> getTexture(int width, int height);
float getPixelDensity();
};

51
ios/Skia Render Layer/SkiaPreviewDisplayLink.swift Normal file
View File

@@ -0,0 +1,51 @@
//
// SkiaPreviewDisplayLink.swift
// VisionCamera
//
// Created by Marc Rousavy on 19.07.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
import Foundation
class SkiaPreviewDisplayLink {
private var displayLink: CADisplayLink?
private let callback: (_ timestamp: Double) -> Void
init(callback: @escaping (_ timestamp: Double) -> Void) {
self.callback = callback
}
deinit {
stop()
}
@objc
func update(_ displayLink: CADisplayLink) {
callback(displayLink.timestamp)
}
func start() {
if displayLink == nil {
let displayLink = CADisplayLink(target: self, selector: #selector(update))
let queue = DispatchQueue(label: "mrousavy/VisionCamera.preview",
qos: .userInteractive,
attributes: [],
autoreleaseFrequency: .inherit,
target: nil)
queue.async {
displayLink.add(to: .current, forMode: .common)
self.displayLink = displayLink
ReactLogger.log(level: .info, message: "Starting Skia Preview Display Link...")
RunLoop.current.run()
ReactLogger.log(level: .info, message: "Skia Preview Display Link stopped.")
}
}
}
func stop() {
displayLink?.invalidate()
displayLink = nil
}
}

82
ios/Skia Render Layer/SkiaPreviewView.swift Normal file
View File

@@ -0,0 +1,82 @@
//
// SkiaPreviewView.swift
// VisionCamera
//
// Created by Marc Rousavy on 19.07.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
import Foundation
// MARK: - SkiaPreviewLayer
@available(iOS 13.0, *)
class SkiaPreviewLayer: CAMetalLayer {
private var pixelRatio: CGFloat {
return UIScreen.main.scale
}
init(device: MTLDevice) {
super.init()
framebufferOnly = true
self.device = device
isOpaque = false
pixelFormat = .bgra8Unorm
contentsScale = pixelRatio
}
@available(*, unavailable)
required init?(coder _: NSCoder) {
fatalError("init(coder:) has not been implemented")
}
func setSize(width: CGFloat, height: CGFloat) {
frame = CGRect(x: 0, y: 0, width: width, height: height)
drawableSize = CGSize(width: width * pixelRatio,
height: height * pixelRatio)
}
}
// MARK: - SkiaPreviewView
class SkiaPreviewView: PreviewView {
private let skiaRenderer: SkiaRenderer
private let previewLayer: SkiaPreviewLayer
private lazy var displayLink = SkiaPreviewDisplayLink(callback: { [weak self] _ in
// Called everytime to render the screen - e.g. 60 FPS
if let self = self {
self.skiaRenderer.renderLatestFrame(to: self.previewLayer)
}
})
init(frame: CGRect, skiaRenderer: SkiaRenderer) {
self.skiaRenderer = skiaRenderer
previewLayer = SkiaPreviewLayer(device: skiaRenderer.metalDevice)
super.init(frame: frame)
}
deinit {
self.displayLink.stop()
}
@available(*, unavailable)
required init?(coder _: NSCoder) {
fatalError("init(coder:) has not been implemented")
}
override func willMove(toSuperview newSuperview: UIView?) {
if newSuperview != nil {
layer.addSublayer(previewLayer)
displayLink.start()
} else {
previewLayer.removeFromSuperlayer()
displayLink.stop()
}
}
override func layoutSubviews() {
previewLayer.setSize(width: bounds.size.width,
height: bounds.size.height)
}
}

22
ios/Skia Render Layer/SkiaMetalRenderContext.h → ios/Skia Render Layer/SkiaRenderContext.h
View File

@@ -1,18 +1,15 @@
//
// SkiaMetalRenderContext.h
// SkiaRenderContext.h
// VisionCamera
//
// Created by Marc Rousavy on 02.12.22.
// Copyright © 2022 mrousavy. All rights reserved.
//
#ifndef SkiaMetalRenderContext_h
#define SkiaMetalRenderContext_h
#pragma once
#import <MetalKit/MetalKit.h>
#import <QuartzCore/CAMetalLayer.h>
#import <AVFoundation/AVFoundation.h>
#include <include/gpu/GrDirectContext.h>
#import <include/gpu/GrDirectContext.h>
struct RenderContext {
id<MTLDevice> device;
@@ -26,16 +23,3 @@ struct RenderContext {
(__bridge void*)commandQueue);
}
};
// For rendering to an off-screen in-memory Metal Texture (MTLTexture)
struct OffscreenRenderContext: public RenderContext {
id<MTLTexture> texture;
};
// For rendering to a Metal Layer (CAMetalLayer)
struct LayerRenderContext: public RenderContext {
CAMetalLayer* layer;
VisionDisplayLink* displayLink;
};
#endif /* SkiaMetalRenderContext_h */

45
ios/Skia Render Layer/SkiaRenderer.h Normal file
View File

@@ -0,0 +1,45 @@
//
// SkiaRenderer.h
// VisionCamera
//
// Created by Marc Rousavy on 19.07.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
#pragma once
#import <Foundation/Foundation.h>
#import <AVFoundation/AVFoundation.h>
#import <Metal/Metal.h>
typedef void* SkiaCanvas;
typedef void(^draw_callback_t)(SkiaCanvas _Nonnull);
/**
A Camera Frame Renderer powered by Skia.
It provides two Contexts, one offscreen and one onscreen.
- Offscreen Context: Allows you to render a Frame into a Skia Canvas and draw onto it using Skia commands
- Onscreen Context: Allows you to render a Frame from the offscreen context onto a Layer allowing it to be displayed for Preview.
The two contexts may run at different Frame Rates.
*/
@interface SkiaRenderer : NSObject
/**
Renders the given Camera Frame to the offscreen Skia Canvas.
The given callback will be executed with a reference to the Skia Canvas
for the user to perform draw operations on (in this case, through a JS Frame Processor)
*/
- (void)renderCameraFrameToOffscreenCanvas:(CMSampleBufferRef _Nonnull)sampleBuffer withDrawCallback:(draw_callback_t _Nonnull)callback;
/**
Renders the latest Frame to the onscreen Layer.
This should be called everytime you want the UI to update, e.g. for 60 FPS; every 16.66ms.
*/
- (void)renderLatestFrameToLayer:(CALayer* _Nonnull)layer;
/**
The Metal Device used for Rendering to the Layer
*/
@property (nonatomic, readonly) id<MTLDevice> _Nonnull metalDevice;
@end

260
ios/Skia Render Layer/SkiaMetalCanvasProvider.mm → ios/Skia Render Layer/SkiaRenderer.mm
View File

@@ -1,72 +1,137 @@
#import "SkiaMetalCanvasProvider.h"
//
// SkiaRenderer.mm
// VisionCamera
//
// Created by Marc Rousavy on 19.07.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
#import <Foundation/Foundation.h>
#import "SkiaRenderer.h"
#import <AVFoundation/AVFoundation.h>
#import <Metal/Metal.h>
#import <include/core/SkColorSpace.h>
#import "SkiaRenderContext.h"
#import <include/core/SkSurface.h>
#import <include/core/SkCanvas.h>
#import <include/core/SkFont.h>
#import <include/core/SkColorSpace.h>
#import <include/gpu/ganesh/SkImageGanesh.h>
#import <include/gpu/GrDirectContext.h>
#import "SkImageHelpers.h"
#include <memory>
#import <system_error>
#import <memory>
#import <mutex>
SkiaMetalCanvasProvider::SkiaMetalCanvasProvider(): std::enable_shared_from_this<SkiaMetalCanvasProvider>() {
// Configure Metal Layer
_layerContext.layer = [CAMetalLayer layer];
_layerContext.layer.framebufferOnly = NO;
_layerContext.layer.device = _layerContext.device;
_layerContext.layer.opaque = false;
_layerContext.layer.contentsScale = getPixelDensity();
_layerContext.layer.pixelFormat = MTLPixelFormatBGRA8Unorm;
// Set up DisplayLink
_layerContext.displayLink = [[VisionDisplayLink alloc] init];
_isValid = true;
@implementation SkiaRenderer {
// The context we draw each Frame on
std::unique_ptr<RenderContext> _offscreenContext;
// The context the preview runs on
std::unique_ptr<RenderContext> _layerContext;
// The texture holding the drawn-to Frame
id<MTLTexture> _texture;
// For synchronization between the two Threads/Contexts
std::mutex _textureMutex;
std::atomic<bool> _hasNewFrame;
}
SkiaMetalCanvasProvider::~SkiaMetalCanvasProvider() {
_isValid = false;
NSLog(@"VisionCamera: Stopping SkiaMetalCanvasProvider DisplayLink...");
[_layerContext.displayLink stop];
- (instancetype)init {
if (self = [super init]) {
_offscreenContext = std::make_unique<RenderContext>();
_layerContext = std::make_unique<RenderContext>();
_texture = nil;
_hasNewFrame = false;
}
return self;
}
void SkiaMetalCanvasProvider::start() {
NSLog(@"VisionCamera: Starting SkiaMetalCanvasProvider DisplayLink...");
[_layerContext.displayLink start:[weakThis = weak_from_this()](double time) {
auto thiz = weakThis.lock();
if (thiz) {
thiz->render();
}
}];
- (id<MTLDevice>)metalDevice {
return _layerContext->device;
}
id<MTLTexture> SkiaMetalCanvasProvider::getTexture(int width, int height) {
if (_offscreenContext.texture == nil
|| _offscreenContext.texture.width != width
|| _offscreenContext.texture.height != height) {
- (id<MTLTexture>)getTexture:(NSUInteger)width height:(NSUInteger)height {
if (_texture == nil
|| _texture.width != width
|| _texture.height != height) {
// Create new texture with the given width and height
MTLTextureDescriptor* textureDescriptor = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm
width:width
height:height
mipmapped:NO];
textureDescriptor.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead;
_offscreenContext.texture = [_offscreenContext.device newTextureWithDescriptor:textureDescriptor];
_texture = [_offscreenContext->device newTextureWithDescriptor:textureDescriptor];
}
return _offscreenContext.texture;
return _texture;
}
/**
Callback from the DisplayLink - renders the current in-memory off-screen texture to the on-screen CAMetalLayer
*/
void SkiaMetalCanvasProvider::render() {
if (_width == -1 && _height == -1) {
return;
}
- (void)renderCameraFrameToOffscreenCanvas:(CMSampleBufferRef)sampleBuffer withDrawCallback:(draw_callback_t)callback {
// Wrap in auto release pool since we want the system to clean up after rendering
@autoreleasepool {
// Get the Frame's PixelBuffer
CVImageBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
if (pixelBuffer == nil) {
throw std::runtime_error("SkiaRenderer: Pixel Buffer is corrupt/empty.");
}
// Lock Mutex to block the runLoop from overwriting the _currentDrawable
std::unique_lock lock(_textureMutex);
// Get the Metal Texture we use for in-memory drawing
auto texture = [self getTexture:CVPixelBufferGetWidth(pixelBuffer)
height:CVPixelBufferGetHeight(pixelBuffer)];
// Get & Lock the writeable Texture from the Metal Drawable
GrMtlTextureInfo textureInfo;
textureInfo.fTexture.retain((__bridge void*)texture);
GrBackendRenderTarget backendRenderTarget((int)texture.width,
(int)texture.height,
1,
textureInfo);
auto context = _offscreenContext->skiaContext.get();
// Create a Skia Surface from the writable Texture
auto surface = SkSurface::MakeFromBackendRenderTarget(context,
backendRenderTarget,
kTopLeft_GrSurfaceOrigin,
kBGRA_8888_SkColorType,
SkColorSpace::MakeSRGB(),
nullptr);
if (surface == nullptr || surface->getCanvas() == nullptr) {
throw std::runtime_error("Skia surface could not be created from parameters.");
}
// Converts the CMSampleBuffer to an SkImage - RGB.
CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
auto image = SkImageHelpers::convertCMSampleBufferToSkImage(context, sampleBuffer);
auto canvas = surface->getCanvas();
// Clear everything so we keep it at a clean state
canvas->clear(SkColors::kBlack);
// Draw the Image into the Frame (aspectRatio: cover)
// The Frame Processor might draw the Frame again (through render()) to pass a custom paint/shader,
// but that'll just overwrite the existing one - no need to worry.
canvas->drawImage(image, 0, 0);
// Call the draw callback - probably a JS Frame Processor.
callback(static_cast<void*>(canvas));
// Flush all appended operations on the canvas and commit it to the SkSurface
surface->flushAndSubmit();
// Set dirty & free locks
_hasNewFrame = true;
lock.unlock();
CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
}
}
- (void)renderLatestFrameToLayer:(CALayer* _Nonnull)layer {
if (!_hasNewFrame) {
// No new Frame has arrived in the meantime.
// We don't need to re-draw the texture to the screen if nothing has changed, abort.
@@ -74,12 +139,12 @@ void SkiaMetalCanvasProvider::render() {
}
@autoreleasepool {
auto context = _layerContext.skiaContext.get();
auto context = _layerContext->skiaContext.get();
// Create a Skia Surface from the CAMetalLayer (use to draw to the View)
GrMTLHandle drawableHandle;
auto surface = SkSurface::MakeFromCAMetalLayer(context,
(__bridge GrMTLHandle)_layerContext.layer,
(__bridge GrMTLHandle)layer,
kTopLeft_GrSurfaceOrigin,
1,
kBGRA_8888_SkColorType,
@@ -91,15 +156,14 @@ void SkiaMetalCanvasProvider::render() {
}
auto canvas = surface->getCanvas();
// Lock the Mutex so we can operate on the Texture atomically without
// renderFrameToCanvas() overwriting in between from a different thread
std::unique_lock lock(_textureMutex);
// Get the texture
auto texture = _offscreenContext.texture;
auto texture = _texture;
if (texture == nil) return;
// Calculate Center Crop (aspectRatio: cover) transform
auto sourceRect = SkRect::MakeXYWH(0, 0, texture.width, texture.height);
auto destinationRect = SkRect::MakeXYWH(0, 0, surface->width(), surface->height());
@@ -130,104 +194,14 @@ void SkiaMetalCanvasProvider::render() {
// Pass the drawable into the Metal Command Buffer and submit it to the GPU
id<CAMetalDrawable> drawable = (__bridge id<CAMetalDrawable>)drawableHandle;
id<MTLCommandBuffer> commandBuffer([_layerContext.commandQueue commandBuffer]);
id<MTLCommandBuffer> commandBuffer([_layerContext->commandQueue commandBuffer]);
[commandBuffer presentDrawable:drawable];
[commandBuffer commit];
// Set flag back to false
_hasNewFrame = false;
lock.unlock();
}
}
float SkiaMetalCanvasProvider::getPixelDensity() {
return UIScreen.mainScreen.scale;
}
/**
Render to a canvas. This uses the current in-memory off-screen texture and draws to it.
The buffer is expected to be in RGB (`BGRA_8888`) format.
While rendering, `drawCallback` will be invoked with a Skia Canvas instance which can be used for Frame Processing (JS).
*/
void SkiaMetalCanvasProvider::renderFrameToCanvas(CMSampleBufferRef sampleBuffer, const std::function<void(SkCanvas*)>& drawCallback) {
if (_width == -1 && _height == -1) {
return;
}
// Wrap in auto release pool since we want the system to clean up after rendering
// and not wait until later - we've seen some example of memory usage growing very
// fast in the simulator without this.
@autoreleasepool {
// Get the Frame's PixelBuffer
CVImageBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
if (pixelBuffer == nil) {
throw std::runtime_error("drawFrame: Pixel Buffer is corrupt/empty.");
}
// Lock Mutex to block the runLoop from overwriting the _currentDrawable
std::unique_lock lock(_textureMutex);
// Get the Metal Texture we use for in-memory drawing
auto texture = getTexture(CVPixelBufferGetWidth(pixelBuffer),
CVPixelBufferGetHeight(pixelBuffer));
// Get & Lock the writeable Texture from the Metal Drawable
GrMtlTextureInfo fbInfo;
fbInfo.fTexture.retain((__bridge void*)texture);
GrBackendRenderTarget backendRT(texture.width,
texture.height,
1,
fbInfo);
auto context = _offscreenContext.skiaContext.get();
// Create a Skia Surface from the writable Texture
auto surface = SkSurface::MakeFromBackendRenderTarget(context,
backendRT,
kTopLeft_GrSurfaceOrigin,
kBGRA_8888_SkColorType,
nullptr,
nullptr);
if (surface == nullptr || surface->getCanvas() == nullptr) {
throw std::runtime_error("Skia surface could not be created from parameters.");
}
// Lock the Frame's PixelBuffer for the duration of the Frame Processor so the user can safely do operations on it
CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
// Converts the CMSampleBuffer to an SkImage - RGB.
auto image = SkImageHelpers::convertCMSampleBufferToSkImage(context, sampleBuffer);
auto canvas = surface->getCanvas();
// Clear everything so we keep it at a clean state
canvas->clear(SkColors::kBlack);
// Draw the Image into the Frame (aspectRatio: cover)
// The Frame Processor might draw the Frame again (through render()) to pass a custom paint/shader,
// but that'll just overwrite the existing one - no need to worry.
canvas->drawImage(image, 0, 0);
// Call the JS Frame Processor.
drawCallback(canvas);
// Flush all appended operations on the canvas and commit it to the SkSurface
surface->flushAndSubmit();
_hasNewFrame = true;
lock.unlock();
CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
}
}
void SkiaMetalCanvasProvider::setSize(int width, int height) {
_width = width;
_height = height;
_layerContext.layer.frame = CGRectMake(0, 0, width, height);
_layerContext.layer.drawableSize = CGSizeMake(width * getPixelDensity(),
height* getPixelDensity());
}
CALayer* SkiaMetalCanvasProvider::getLayer() { return _layerContext.layer; }
@end

38
ios/Skia Render Layer/VisionDisplayLink.h
View File

@@ -1,38 +0,0 @@
//
// VisionDisplayLink.h
// VisionCamera
//
// Created by Marc Rousavy on 28.11.22.
// Copyright © 2022 mrousavy. All rights reserved.
//
#ifndef DisplayLink_h
#define DisplayLink_h
#import <CoreFoundation/CoreFoundation.h>
#import <UIKit/UIKit.h>
typedef void (^block_t)(double);
@interface VisionDisplayLink : NSObject {
CADisplayLink *_displayLink;
double _currentFps;
double _previousFrameTimestamp;
}
@property(nonatomic, copy) block_t updateBlock;
// Start the DisplayLink's runLoop
- (void)start:(block_t)block;
// Stop the DisplayLink's runLoop
- (void)stop;
// Get the current FPS value
- (double)currentFps;
// The FPS value this DisplayLink is targeting
- (double)targetFps;
@end
#endif /* VisionDisplayLink_h */

63
ios/Skia Render Layer/VisionDisplayLink.m
View File

@@ -1,63 +0,0 @@
//
// VisionDisplayLink.m
// VisionCamera
//
// Created by Marc Rousavy on 28.11.22.
// Copyright © 2022 mrousavy. All rights reserved.
//
#import "VisionDisplayLink.h"
#import <Foundation/Foundation.h>
@implementation VisionDisplayLink
- (void)start:(block_t)block {
self.updateBlock = block;
// check whether the loop is already running
if (_displayLink == nil) {
// specify update method
_displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(update:)];
// Start a new Queue/Thread that will run the runLoop
dispatch_queue_attr_t qos = dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, -1);
dispatch_queue_t queue = dispatch_queue_create("mrousavy/VisionCamera.preview", qos);
dispatch_async(queue, ^{
// Add the display link to the current run loop (thread on which we're currently running on)
NSRunLoop* loop = [NSRunLoop currentRunLoop];
[self->_displayLink addToRunLoop:loop forMode:NSRunLoopCommonModes];
// Run the runLoop (blocking)
[loop run];
NSLog(@"VisionCamera: DisplayLink runLoop ended.");
});
}
}
- (void)stop {
// check whether the loop is already stopped
if (_displayLink != nil) {
// if the display link is present, it gets invalidated (loop stops)
[_displayLink invalidate];
_displayLink = nil;
}
}
- (void)update:(CADisplayLink *)sender {
double time = sender.timestamp;
double diff = time - _previousFrameTimestamp;
_currentFps = 1.0 / diff;
_previousFrameTimestamp = time;
_updateBlock(time);
}
- (double)targetFps {
return 1.0 / _displayLink.duration;
}
- (double)currentFps {
return _currentFps;
}
@end