feat: Draw onto Frame as if it was a Skia Canvas (#1479)

* Create Shaders.ts

* Add `previewType` and `enableFpsGraph`

* Add RN Skia native dependency

* Add Skia Preview View on iOS

* Pass 1

* Update FrameHostObject.mm

* Wrap Canvas

* Lockfiles

* fix: Fix stuff

* chore: Upgrade RNWorklets

* Add `previewType` to set the Preview

* feat: Add Example

* Update project.pbxproj

* `enableFpsGraph`

* Cache the `std::shared_ptr<FrameHostObject>`

* Update CameraView+RecordVideo.swift

* Update SkiaMetalCanvasProvider.mm

* Android: Integrate Skia Dependency

* fix: Use new Prefix

* Add example for rendering shader

* chore: Upgrade CameraX

* Remove KTX

* Enable `viewBinding`

* Revert "Enable `viewBinding`"

This reverts commit f2a603f53b33ea4311a296422ffd1a910ce03f9e.

* Revert "chore: Upgrade CameraX"

This reverts commit 8dc832cf8754490d31a6192e6c1a1f11cdcd94fe.

* Remove unneeded `ProcessCameraProvider.getInstance()` call

* fix: Add REA hotfix patch

* fix: Fix FrameHostObject dead in runAsync

* fix: Make `runAsync` run truly async by dropping new Frames while executing

* chore: Upgrade RN Worklets to latest

* chore: Upgrade RN Skia

* Revert "Remove KTX"

This reverts commit 253f586633f7af2da992d2279fc206dc62597129.

* Make Skia optional in CMake

* Fix import

* Update CMakeLists.txt

* Update build.gradle

* Update CameraView.kt

* Update CameraView.kt

* Update CameraView.kt

* Update Shaders.ts

* Center Blur

* chore: Upgrade RN Worklets

* feat: Add `toByteArray()`, `orientation`, `isMirrored` and `timestamp` to `Frame` (#1487)

* feat: Implement `orientation` and `isMirrored` on Frame

* feat: Add `toArrayBuffer()` func

* perf: Do faster buffer copy

* feat: Implement `toArrayBuffer()` on Android

* feat: Add `orientation` and `isMirrored` to Android

* feat: Add `timestamp` to Frame

* Update Frame.ts

* Update JImageProxy.h

* Update FrameHostObject.cpp

* Update FrameHostObject.cpp

* Update CameraPage.tsx

* fix: Format Swift

ios/Skia Render Layer/PreviewSkiaView.h

@@ -0,0 +1,26 @@
+//
+//  PreviewSkiaView.h
+//  VisionCamera
+//
+//  Created by Marc Rousavy on 17.11.22.
+//  Copyright © 2022 mrousavy. All rights reserved.
+//
+
+#ifndef PreviewSkiaView_h
+#define PreviewSkiaView_h
+
+#import <UIKit/UIKit.h>
+#import <AVFoundation/AVFoundation.h>
+#import "FrameProcessorCallback.h"
+
+typedef void (^DrawCallback) (void* _Nonnull skCanvas);
+
+@interface PreviewSkiaView: UIView
+
+// Call to pass a new Frame to be drawn by the Skia Canvas
+- (void) drawFrame:(_Nonnull CMSampleBufferRef)buffer withCallback:(DrawCallback _Nonnull)callback;
+
+@end
+
+
+#endif /* PreviewSkiaView_h */

ios/Skia Render Layer/PreviewSkiaView.mm

@@ -0,0 +1,59 @@
+//
+//  PreviewSkiaView.mm
+//  VisionCamera
+//
+//  Created by Marc Rousavy on 17.11.22.
+//  Copyright © 2022 mrousavy. All rights reserved.
+//
+
+#import "PreviewSkiaView.h"
+#import <Foundation/Foundation.h>
+
+#import "SkiaMetalCanvasProvider.h"
+
+#include <exception>
+#include <string>
+
+#if SHOW_FPS
+#import <React/RCTFPSGraph.h>
+#endif
+
+@implementation PreviewSkiaView {
+  std::shared_ptr<SkiaMetalCanvasProvider> _canvasProvider;
+}
+
+- (void)drawFrame:(CMSampleBufferRef)buffer withCallback:(DrawCallback _Nonnull)callback {
+  if (_canvasProvider == nullptr) {
+    throw std::runtime_error("Cannot draw new Frame to Canvas when SkiaMetalCanvasProvider is null!");
+  }
+  
+  _canvasProvider->renderFrameToCanvas(buffer, ^(SkCanvas* canvas) {
+    callback((void*)canvas);
+  });
+}
+
+- (void) willMoveToSuperview:(UIView *)newWindow {
+  if (newWindow == NULL) {
+    // Remove implementation view when the parent view is not set
+    if (_canvasProvider != nullptr) {
+      [_canvasProvider->getLayer() removeFromSuperlayer];
+      _canvasProvider = nullptr;
+    }
+  } else {
+    // Create implementation view when the parent view is set
+    if (_canvasProvider == nullptr) {
+      _canvasProvider = std::make_shared<SkiaMetalCanvasProvider>();
+      [self.layer addSublayer: _canvasProvider->getLayer()];
+      _canvasProvider->start();
+    }
+  }
+}
+
+- (void) layoutSubviews {
+  if (_canvasProvider != nullptr) {
+    _canvasProvider->setSize(self.bounds.size.width, self.bounds.size.height);
+  }
+}
+
+@end
+

ios/Skia Render Layer/SkImageHelpers.h

@@ -0,0 +1,39 @@
+//
+//  SkImageHelpers.h
+//  VisionCamera
+//
+//  Created by Marc Rousavy on 23.11.22.
+//  Copyright © 2022 mrousavy. All rights reserved.
+//
+
+#ifndef SkImageHelpers_h
+#define SkImageHelpers_h
+
+#import <Foundation/Foundation.h>
+#import <AVFoundation/AVFoundation.h>
+#import <include/gpu/GrRecordingContext.h>
+
+#import "SkImage.h"
+#import "SkSize.h"
+#import "SkRect.h"
+
+class SkImageHelpers {
+public:
+  SkImageHelpers() = delete;
+  
+public:
+  /**
+   Convert a CMSampleBuffer to an SkImage. Format has to be RGB.
+   */
+  static sk_sp<SkImage> convertCMSampleBufferToSkImage(GrRecordingContext* context, CMSampleBufferRef sampleBuffer);
+  /**
+   Creates a Center Crop Transformation Rect so that the source rect fills (aspectRatio: cover) the destination rect.
+   The return value should be passed as a sourceRect to a canvas->draw...Rect(..) function, destinationRect should stay the same.
+   */
+  static SkRect createCenterCropRect(SkRect source, SkRect destination);
+  
+private:
+  static SkRect inscribe(SkSize size, SkRect rect);
+};
+
+#endif /* SkImageHelpers_h */

ios/Skia Render Layer/SkImageHelpers.mm

@@ -0,0 +1,108 @@
+//
+//  CMSampleBuffer+toSkImage.m
+//  VisionCamera
+//
+//  Created by Marc Rousavy on 23.11.22.
+//  Copyright © 2022 mrousavy. All rights reserved.
+//
+
+#import "SkImageHelpers.h"
+#import <AVFoundation/AVFoundation.h>
+#import <Metal/Metal.h>
+
+#import <include/core/SkColorSpace.h>
+#import <include/core/SkSurface.h>
+#import <include/core/SkCanvas.h>
+#import <include/core/SkData.h>
+#import <include/gpu/GrRecordingContext.h>
+
+#include <TargetConditionals.h>
+#if TARGET_RT_BIG_ENDIAN
+#   define FourCC2Str(fourcc) (const char[]){*((char*)&fourcc), *(((char*)&fourcc)+1), *(((char*)&fourcc)+2), *(((char*)&fourcc)+3),0}
+#else
+#   define FourCC2Str(fourcc) (const char[]){*(((char*)&fourcc)+3), *(((char*)&fourcc)+2), *(((char*)&fourcc)+1), *(((char*)&fourcc)+0),0}
+#endif
+
+CVMetalTextureCacheRef getTextureCache(GrRecordingContext* context) {
+  static CVMetalTextureCacheRef textureCache = nil;
+  if (textureCache == nil) {
+    // Create a new Texture Cache
+     auto result = CVMetalTextureCacheCreate(kCFAllocatorDefault,
+                                             nil,
+                                             MTLCreateSystemDefaultDevice(),
+                                             nil,
+                                             &textureCache);
+     if (result != kCVReturnSuccess || textureCache == nil) {
+       throw std::runtime_error("Failed to create Metal Texture Cache!");
+     }
+  }
+  return textureCache;
+}
+
+sk_sp<SkImage> SkImageHelpers::convertCMSampleBufferToSkImage(GrRecordingContext* context, CMSampleBufferRef sampleBuffer) {
+  auto pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
+  double width = CVPixelBufferGetWidth(pixelBuffer);
+  double height = CVPixelBufferGetHeight(pixelBuffer);
+
+  // Make sure the format is RGB (BGRA_8888)
+  auto format = CVPixelBufferGetPixelFormatType(pixelBuffer);
+  if (format != kCVPixelFormatType_32BGRA) {
+    auto fourCharCode = @(FourCC2Str(format));
+    auto error = std::string("VisionCamera: Frame has unknown Pixel Format (") + fourCharCode.UTF8String + std::string(") - cannot convert to SkImage!");
+    throw std::runtime_error(error);
+  }
+  
+  auto textureCache = getTextureCache(context);
+
+  // Convert CMSampleBuffer* -> CVMetalTexture*
+  CVMetalTextureRef cvTexture;
+  CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault,
+                                           textureCache,
+                                           pixelBuffer,
+                                           nil,
+                                           MTLPixelFormatBGRA8Unorm,
+                                           width,
+                                           height,
+                                           0, // plane index
+                                           &cvTexture);
+  GrMtlTextureInfo textureInfo;
+  auto mtlTexture = CVMetalTextureGetTexture(cvTexture);
+  textureInfo.fTexture.retain((__bridge void*)mtlTexture);
+
+  // Wrap it in a GrBackendTexture
+  GrBackendTexture texture(width, height, GrMipmapped::kNo, textureInfo);
+
+  // Create an SkImage from the existing texture
+  auto image = SkImage::MakeFromTexture(context,
+                                        texture,
+                                        kTopLeft_GrSurfaceOrigin,
+                                        kBGRA_8888_SkColorType,
+                                        kOpaque_SkAlphaType,
+                                        SkColorSpace::MakeSRGB());
+
+  // Release the Texture wrapper (it will still be strong)
+  CFRelease(cvTexture);
+
+  return image;
+}
+
+SkRect SkImageHelpers::createCenterCropRect(SkRect sourceRect, SkRect destinationRect) {
+  SkSize src;
+  if (destinationRect.width() / destinationRect.height() > sourceRect.width() / sourceRect.height()) {
+    src = SkSize::Make(sourceRect.width(), (sourceRect.width() * destinationRect.height()) / destinationRect.width());
+  } else {
+    src = SkSize::Make((sourceRect.height() * destinationRect.width()) / destinationRect.height(), sourceRect.height());
+  }
+
+  return inscribe(src, sourceRect);
+}
+
+SkRect SkImageHelpers::inscribe(SkSize size, SkRect rect) {
+  auto halfWidthDelta = (rect.width() - size.width()) / 2.0;
+  auto halfHeightDelta = (rect.height() - size.height()) / 2.0;
+  return SkRect::MakeXYWH(rect.x() + halfWidthDelta,
+                          rect.y() + halfHeightDelta,
+                          size.width(),
+                          size.height());
+}
+

ios/Skia Render Layer/SkiaMetalCanvasProvider.h

@@ -0,0 +1,56 @@
+#pragma once
+
+#ifndef __cplusplus
+#error This header has to be compiled with C++!
+#endif
+
+#import <MetalKit/MetalKit.h>
+#import <QuartzCore/CAMetalLayer.h>
+
+#import <AVFoundation/AVFoundation.h>
+
+#include <functional>
+#include <include/gpu/GrDirectContext.h>
+#include <mutex>
+#include <memory>
+#include <atomic>
+
+#import "VisionDisplayLink.h"
+#import "SkiaMetalRenderContext.h"
+
+class SkiaMetalCanvasProvider: public std::enable_shared_from_this<SkiaMetalCanvasProvider> {
+public:
+  SkiaMetalCanvasProvider();
+  ~SkiaMetalCanvasProvider();
+
+  // Render a Camera Frame to the off-screen canvas
+  void renderFrameToCanvas(CMSampleBufferRef sampleBuffer, const std::function<void(SkCanvas*)>& drawCallback);
+
+  // Start updating the DisplayLink (runLoop @ screen refresh rate) and draw Frames to the Layer
+  void start();
+  // Update the size of the View (Layer)
+  void setSize(int width, int height);
+  CALayer* getLayer();
+
+private:
+  bool _isValid = false;
+  float _width = -1;
+  float _height = -1;
+  
+  // For rendering Camera Frame -> off-screen MTLTexture
+  OffscreenRenderContext _offscreenContext;
+
+  // For rendering off-screen MTLTexture -> on-screen CAMetalLayer
+  LayerRenderContext _layerContext;
+  
+  // For synchronization between the two Threads/Contexts
+  std::mutex _textureMutex;
+  std::atomic<bool> _hasNewFrame = false;
+  
+private:
+  void render();
+  id<MTLTexture> getTexture(int width, int height);
+
+  float getPixelDensity();
+};
+

ios/Skia Render Layer/SkiaMetalCanvasProvider.mm

@@ -0,0 +1,240 @@
+#import "SkiaMetalCanvasProvider.h"
+
+#import <AVFoundation/AVFoundation.h>
+#import <Metal/Metal.h>
+
+#import <include/core/SkColorSpace.h>
+#import <include/core/SkSurface.h>
+#import <include/core/SkCanvas.h>
+#import <include/core/SkFont.h>
+#import <include/gpu/GrDirectContext.h>
+
+#import "SkImageHelpers.h"
+
+#include <memory>
+
+SkiaMetalCanvasProvider::SkiaMetalCanvasProvider(): std::enable_shared_from_this<SkiaMetalCanvasProvider>() {
+  // Configure Metal Layer
+  _layerContext.layer = [CAMetalLayer layer];
+  _layerContext.layer.framebufferOnly = NO;
+  _layerContext.layer.device = _layerContext.device;
+  _layerContext.layer.opaque = false;
+  _layerContext.layer.contentsScale = getPixelDensity();
+  _layerContext.layer.pixelFormat = MTLPixelFormatBGRA8Unorm;
+  // Set up DisplayLink
+  _layerContext.displayLink = [[VisionDisplayLink alloc] init];
+  
+  _isValid = true;
+}
+
+SkiaMetalCanvasProvider::~SkiaMetalCanvasProvider() {
+  _isValid = false;
+  NSLog(@"VisionCamera: Stopping SkiaMetalCanvasProvider DisplayLink...");
+  [_layerContext.displayLink stop];
+}
+
+void SkiaMetalCanvasProvider::start() {
+  NSLog(@"VisionCamera: Starting SkiaMetalCanvasProvider DisplayLink...");
+  [_layerContext.displayLink start:[weakThis = weak_from_this()](double time) {
+    auto thiz = weakThis.lock();
+    if (thiz) {
+      thiz->render();
+    }
+  }];
+}
+
+id<MTLTexture> SkiaMetalCanvasProvider::getTexture(int width, int height) {
+  if (_offscreenContext.texture == nil
+      || _offscreenContext.texture.width != width
+      || _offscreenContext.texture.height != height) {
+    // Create new texture with the given width and height
+    MTLTextureDescriptor* textureDescriptor = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm
+                                                                                                 width:width
+                                                                                                height:height
+                                                                                             mipmapped:NO];
+    textureDescriptor.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead;
+    _offscreenContext.texture = [_offscreenContext.device newTextureWithDescriptor:textureDescriptor];
+  }
+  return _offscreenContext.texture;
+}
+
+/**
+ Callback from the DisplayLink - renders the current in-memory off-screen texture to the on-screen CAMetalLayer
+ */
+void SkiaMetalCanvasProvider::render() {
+  if (_width == -1 && _height == -1) {
+    return;
+  }
+  
+  if (!_hasNewFrame) {
+    // No new Frame has arrived in the meantime.
+    // We don't need to re-draw the texture to the screen if nothing has changed, abort.
+    return;
+  }
+  
+  @autoreleasepool {
+    auto context = _layerContext.skiaContext.get();
+    
+    // Create a Skia Surface from the CAMetalLayer (use to draw to the View)
+    GrMTLHandle drawableHandle;
+    auto surface = SkSurface::MakeFromCAMetalLayer(context,
+                                                   (__bridge GrMTLHandle)_layerContext.layer,
+                                                   kTopLeft_GrSurfaceOrigin,
+                                                   1,
+                                                   kBGRA_8888_SkColorType,
+                                                   nullptr,
+                                                   nullptr,
+                                                   &drawableHandle);
+    if (surface == nullptr || surface->getCanvas() == nullptr) {
+      throw std::runtime_error("Skia surface could not be created from parameters.");
+    }
+    
+    auto canvas = surface->getCanvas();
+    
+    // Lock the Mutex so we can operate on the Texture atomically without
+    // renderFrameToCanvas() overwriting in between from a different thread
+    std::unique_lock lock(_textureMutex);
+  
+    // Get the texture
+    auto texture = _offscreenContext.texture;
+    if (texture == nil) return;
+    
+    // Calculate Center Crop (aspectRatio: cover) transform
+    auto sourceRect = SkRect::MakeXYWH(0, 0, texture.width, texture.height);
+    auto destinationRect = SkRect::MakeXYWH(0, 0, surface->width(), surface->height());
+    sourceRect = SkImageHelpers::createCenterCropRect(sourceRect, destinationRect);
+    auto offsetX = -sourceRect.left();
+    auto offsetY = -sourceRect.top();
+
+    // The Canvas is equal to the View size, where-as the Frame has a different size (e.g. 4k)
+    // We scale the Canvas to the exact dimensions of the Frame so that the user can use the Frame as a coordinate system
+    canvas->save();
+    
+    auto scaleW = static_cast<double>(surface->width()) / texture.width;
+    auto scaleH = static_cast<double>(surface->height()) / texture.height;
+    auto scale = MAX(scaleW, scaleH);
+    canvas->scale(scale, scale);
+    canvas->translate(offsetX, offsetY);
+    
+    // Convert the rendered MTLTexture to an SkImage
+    GrMtlTextureInfo textureInfo;
+    textureInfo.fTexture.retain((__bridge void*)texture);
+    GrBackendTexture backendTexture(texture.width, texture.height, GrMipmapped::kNo, textureInfo);
+    auto image = SkImage::MakeFromTexture(context,
+                                          backendTexture,
+                                          kTopLeft_GrSurfaceOrigin,
+                                          kBGRA_8888_SkColorType,
+                                          kOpaque_SkAlphaType,
+                                          SkColorSpace::MakeSRGB());
+    
+    // Draw the Texture (Frame) to the Canvas
+    canvas->drawImage(image, 0, 0);
+    
+    // Restore the scale & transform
+    canvas->restore();
+    
+    surface->flushAndSubmit();
+    
+    // Pass the drawable into the Metal Command Buffer and submit it to the GPU
+    id<CAMetalDrawable> drawable = (__bridge id<CAMetalDrawable>)drawableHandle;
+    id<MTLCommandBuffer> commandBuffer([_layerContext.commandQueue commandBuffer]);
+    [commandBuffer presentDrawable:drawable];
+    [commandBuffer commit];
+    
+    _hasNewFrame = false;
+    
+    lock.unlock();
+  }
+}
+
+float SkiaMetalCanvasProvider::getPixelDensity() {
+  return UIScreen.mainScreen.scale;
+}
+
+/**
+ Render to a canvas. This uses the current in-memory off-screen texture and draws to it.
+ The buffer is expected to be in RGB (`BGRA_8888`) format.
+ While rendering, `drawCallback` will be invoked with a Skia Canvas instance which can be used for Frame Processing (JS).
+ */
+void SkiaMetalCanvasProvider::renderFrameToCanvas(CMSampleBufferRef sampleBuffer, const std::function<void(SkCanvas*)>& drawCallback) {
+  if (_width == -1 && _height == -1) {
+    return;
+  }
+
+  // Wrap in auto release pool since we want the system to clean up after rendering
+  // and not wait until later - we've seen some example of memory usage growing very
+  // fast in the simulator without this.
+  @autoreleasepool {
+    // Get the Frame's PixelBuffer
+    CVImageBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
+    if (pixelBuffer == nil) {
+      throw std::runtime_error("drawFrame: Pixel Buffer is corrupt/empty.");
+    }
+    
+    // Lock Mutex to block the runLoop from overwriting the _currentDrawable
+    std::unique_lock lock(_textureMutex);
+    
+    // Get the Metal Texture we use for in-memory drawing
+    auto texture = getTexture(CVPixelBufferGetWidth(pixelBuffer),
+                              CVPixelBufferGetHeight(pixelBuffer));
+    
+    // Get & Lock the writeable Texture from the Metal Drawable
+    GrMtlTextureInfo fbInfo;
+    fbInfo.fTexture.retain((__bridge void*)texture);
+    GrBackendRenderTarget backendRT(texture.width,
+                                    texture.height,
+                                    1,
+                                    fbInfo);
+    
+    auto context = _offscreenContext.skiaContext.get();
+    
+    // Create a Skia Surface from the writable Texture
+    auto surface = SkSurface::MakeFromBackendRenderTarget(context,
+                                                          backendRT,
+                                                          kTopLeft_GrSurfaceOrigin,
+                                                          kBGRA_8888_SkColorType,
+                                                          nullptr,
+                                                          nullptr);
+    
+    if (surface == nullptr || surface->getCanvas() == nullptr) {
+      throw std::runtime_error("Skia surface could not be created from parameters.");
+    }
+    
+    // Lock the Frame's PixelBuffer for the duration of the Frame Processor so the user can safely do operations on it
+    CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
+    
+    // Converts the CMSampleBuffer to an SkImage - RGB.
+    auto image = SkImageHelpers::convertCMSampleBufferToSkImage(context, sampleBuffer);
+    
+    auto canvas = surface->getCanvas();
+    
+    // Clear everything so we keep it at a clean state
+    canvas->clear(SkColors::kBlack);
+    
+    // Draw the Image into the Frame (aspectRatio: cover)
+    // The Frame Processor might draw the Frame again (through render()) to pass a custom paint/shader,
+    // but that'll just overwrite the existing one - no need to worry.
+    canvas->drawImage(image, 0, 0);
+    
+    // Call the JS Frame Processor.
+    drawCallback(canvas);
+    
+    // Flush all appended operations on the canvas and commit it to the SkSurface
+    surface->flushAndSubmit();
+    
+    _hasNewFrame = true;
+    
+    lock.unlock();
+    CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
+  }
+}
+
+void SkiaMetalCanvasProvider::setSize(int width, int height) {
+  _width = width;
+  _height = height;
+  _layerContext.layer.frame = CGRectMake(0, 0, width, height);
+  _layerContext.layer.drawableSize = CGSizeMake(width * getPixelDensity(),
+                                                height* getPixelDensity());
+}
+
+CALayer* SkiaMetalCanvasProvider::getLayer() { return _layerContext.layer; }

ios/Skia Render Layer/SkiaMetalRenderContext.h

@@ -0,0 +1,41 @@
+//
+//  SkiaMetalRenderContext.h
+//  VisionCamera
+//
+//  Created by Marc Rousavy on 02.12.22.
+//  Copyright © 2022 mrousavy. All rights reserved.
+//
+
+#ifndef SkiaMetalRenderContext_h
+#define SkiaMetalRenderContext_h
+
+#import <MetalKit/MetalKit.h>
+#import <QuartzCore/CAMetalLayer.h>
+#import <AVFoundation/AVFoundation.h>
+#include <include/gpu/GrDirectContext.h>
+
+struct RenderContext {
+  id<MTLDevice> device;
+  id<MTLCommandQueue> commandQueue;
+  sk_sp<GrDirectContext> skiaContext;
+  
+  RenderContext() {
+    device = MTLCreateSystemDefaultDevice();
+    commandQueue = id<MTLCommandQueue>(CFRetain((GrMTLHandle)[device newCommandQueue]));
+    skiaContext = GrDirectContext::MakeMetal((__bridge void*)device,
+                                             (__bridge void*)commandQueue);
+  }
+};
+
+// For rendering to an off-screen in-memory Metal Texture (MTLTexture)
+struct OffscreenRenderContext: public RenderContext {
+  id<MTLTexture> texture;
+};
+
+// For rendering to a Metal Layer (CAMetalLayer)
+struct LayerRenderContext: public RenderContext {
+  CAMetalLayer* layer;
+  VisionDisplayLink* displayLink;
+};
+
+#endif /* SkiaMetalRenderContext_h */

ios/Skia Render Layer/VisionDisplayLink.h

@@ -0,0 +1,38 @@
+//
+//  VisionDisplayLink.h
+//  VisionCamera
+//
+//  Created by Marc Rousavy on 28.11.22.
+//  Copyright © 2022 mrousavy. All rights reserved.
+//
+
+#ifndef DisplayLink_h
+#define DisplayLink_h
+
+#import <CoreFoundation/CoreFoundation.h>
+#import <UIKit/UIKit.h>
+
+typedef void (^block_t)(double);
+@interface VisionDisplayLink : NSObject {
+  CADisplayLink *_displayLink;
+  double _currentFps;
+  double _previousFrameTimestamp;
+}
+
+@property(nonatomic, copy) block_t updateBlock;
+
+// Start the DisplayLink's runLoop
+- (void)start:(block_t)block;
+
+// Stop the DisplayLink's runLoop
+- (void)stop;
+
+// Get the current FPS value
+- (double)currentFps;
+
+// The FPS value this DisplayLink is targeting
+- (double)targetFps;
+
+@end
+
+#endif /* VisionDisplayLink_h */

ios/Skia Render Layer/VisionDisplayLink.m

@@ -0,0 +1,63 @@
+//
+//  VisionDisplayLink.m
+//  VisionCamera
+//
+//  Created by Marc Rousavy on 28.11.22.
+//  Copyright © 2022 mrousavy. All rights reserved.
+//
+
+#import "VisionDisplayLink.h"
+#import <Foundation/Foundation.h>
+
+@implementation VisionDisplayLink
+
+- (void)start:(block_t)block {
+  self.updateBlock = block;
+  // check whether the loop is already running
+  if (_displayLink == nil) {
+    // specify update method
+    _displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(update:)];
+
+    // Start a new Queue/Thread that will run the runLoop
+    dispatch_queue_attr_t qos = dispatch_queue_attr_make_with_qos_class(DISPATCH_QUEUE_SERIAL, QOS_CLASS_USER_INTERACTIVE, -1);
+    dispatch_queue_t queue = dispatch_queue_create("mrousavy/VisionCamera.preview", qos);
+    dispatch_async(queue, ^{
+      // Add the display link to the current run loop (thread on which we're currently running on)
+      NSRunLoop* loop = [NSRunLoop currentRunLoop];
+      [self->_displayLink addToRunLoop:loop forMode:NSRunLoopCommonModes];
+      // Run the runLoop (blocking)
+      [loop run];
+      NSLog(@"VisionCamera: DisplayLink runLoop ended.");
+    });
+  }
+}
+
+- (void)stop {
+  // check whether the loop is already stopped
+  if (_displayLink != nil) {
+    // if the display link is present, it gets invalidated (loop stops)
+    
+    [_displayLink invalidate];
+    _displayLink = nil;
+  }
+}
+
+- (void)update:(CADisplayLink *)sender {
+  double time = sender.timestamp;
+  
+  double diff = time - _previousFrameTimestamp;
+  _currentFps = 1.0 / diff;
+  _previousFrameTimestamp = time;
+  
+  _updateBlock(time);
+}
+
+- (double)targetFps {
+  return 1.0 / _displayLink.duration;
+}
+
+- (double)currentFps {
+  return _currentFps;
+}
+
+@end