feat: native Frame type to provide Orientation (#186)

* Use Frame.h

* Add orientation

* Determine buffer orientation

* Replace plugins

* fix calls

* Update FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx

* Update FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx

* format

* Update CameraPage.tsx

* Update FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx

* Add links to docs

* Use `.` syntax

* Make properties `readonly`

* Fix `@synthesize` backing store

VisionCamera.podspec

@@ -25,7 +25,7 @@ Pod::Spec.new do |s|
   s.source_files = [
     "ios/**/*.{m,mm,swift}",
     "ios/CameraBridge.h",
-    "ios/Frame Processor/CMSampleBufferRefHolder.h",
+    "ios/Frame Processor/Frame.h",
     "ios/Frame Processor/FrameProcessorCallback.h",
     "ios/Frame Processor/FrameProcessorRuntimeManager.h",
     "ios/Frame Processor/FrameProcessorPluginRegistry.h",

docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx

@@ -32,23 +32,23 @@ To achieve **maximum performance**, the `scanQRCodes` function is written in a n
 
 The Frame Processor Plugin Registry API automatically manages type conversion from JS <-> native. They are converted into the most efficient data-structures, as seen here:
 
-| JS Type              | Objective-C Type          | Java Type                  |
+| JS Type              | Objective-C Type              | Java Type                  |
-|----------------------|---------------------------|----------------------------|
+|----------------------|-------------------------------|----------------------------|
-| `number`             | `NSNumber*` (double)      | `double`                   |
+| `number`             | `NSNumber*` (double)          | `double`                   |
-| `boolean`            | `NSNumber*` (boolean)     | `boolean`                  |
+| `boolean`            | `NSNumber*` (boolean)         | `boolean`                  |
-| `string`             | `NSString*`               | `String`                   |
+| `string`             | `NSString*`                   | `String`                   |
-| `[]`                 | `NSArray*`                | `Array<Object>`            |
+| `[]`                 | `NSArray*`                    | `Array<Object>`            |
-| `{}`                 | `NSDictionary*`           | `HashMap<Object>`          |
+| `{}`                 | `NSDictionary*`               | `HashMap<Object>`          |
-| `undefined` / `null` | `nil`                     | `null`                     |
+| `undefined` / `null` | `nil`                         | `null`                     |
-| `(any, any) => void` | `RCTResponseSenderBlock`  | `(Object, Object) -> void` |
+| `(any, any) => void` | [`RCTResponseSenderBlock`][4] | `(Object, Object) -> void` |
-| `Frame`              | `CMSampleBufferRefHolder` | `ImageProxy`               |
+| [`Frame`][1]         | [`Frame*`][2]                 | [`ImageProxy`][3]          |
 
 ### Return values
 
 Return values will automatically be converted to JS values, assuming they are representable in the ["Types" table](#types). So the following Objective-C frame processor:
 
 ```objc
-static inline id detectObject(CMSampleBufferRef buffer, NSArray args) {
+static inline id detectObject(Frame* frame, NSArray args) {
   return @"cat";
 }
 ```
@@ -63,15 +63,17 @@ export function detectObject(frame: Frame): string {
 }
 ```
 
-You can also manipulate the buffer and return it (or a copy) by using the `CMSampleBufferRefHolder` class:
+You can also manipulate the buffer and return it (or a copy) by using the `Frame` class:
 
 ```objc
-static inline id resize(CMSampleBufferRef buffer, NSArray args) {
+#import <VisionCamera/Frame.h>
+
+static inline id resize(Frame* frame, NSArray args) {
   NSNumber* width = [arguments objectAtIndex:0];
   NSNumber* height = [arguments objectAtIndex:1];
 
-  CMSampleBufferRef resizedBuffer = CMSampleBufferCopyAndResize(buffer, width, height);
+  CMSampleBufferRef resizedBuffer = CMSampleBufferCopyAndResize(frame.buffer, width, height);
-  return [[CMSampleBufferRefHolder alloc] initWithBuffer:resizedBuffer];
+  return [[Frame alloc] initWithBuffer:resizedBuffer orientation:frame.orientation];
 }
 ```
 
@@ -116,9 +118,9 @@ For example, a realtime video chat application might use WebRTC to send the fram
 ```objc
 static dispatch_queue_t queue = dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0ul);
 
-static inline id sendFrameToWebRTC(CMSampleBufferRef buffer, NSArray args) {
+static inline id sendFrameToWebRTC(Frame* frame, NSArray args) {
   CMSampleBufferRef bufferCopy;
-  CMSampleBufferCreateCopy(kCFAllocatorDefault, buffer, &bufferCopy);
+  CMSampleBufferCreateCopy(kCFAllocatorDefault, frame.buffer, &bufferCopy);
 
   dispatch_async(queue, ^{
     NSString* serverURL = (NSString*)args[0];
@@ -171,3 +173,8 @@ Your Frame Processor Plugins have to be fast. VisionCamera automatically detects
 <br />
 
 #### 🚀 Create your first Frame Processor Plugin for [iOS](frame-processors-plugins-ios) or [Android](frame-processors-plugins-android)!
+
+[1]: https://github.com/cuvent/react-native-vision-camera/blob/main/src/Frame.ts
+[2]: https://github.com/cuvent/react-native-vision-camera/blob/main/ios/Frame%20Processor/Frame.h
+[3]: https://developer.android.com/reference/androidx/camera/core/ImageProxy
+[4]: https://github.com/facebook/react-native/blob/9a43eac7a32a6ba3164a048960101022a92fcd5a/React/Base/RCTBridgeModule.h#L20-L24

docs/docs/guides/FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx

@@ -27,15 +27,18 @@ iOS Frame Processor Plugins can be written in either **Objective-C** or **Swift*
 2. Create an Objective-C source file, for the QR Code Plugin this will be called `QRCodeFrameProcessorPlugin.m`.
 3. Add the following code:
 
-```objc {9}
+```objc {11}
 #import <VisionCamera/FrameProcessorPlugin.h>
+#import <VisionCamera/Frame.h>
 
 @interface QRCodeFrameProcessorPlugin : NSObject
 @end
 
 @implementation QRCodeFrameProcessorPlugin
 
-static inline id scanQRCodes(CMSampleBufferRef buffer, NSArray args) {
+static inline id scanQRCodes(Frame* frame, NSArray args) {
+  CMSampleBufferRef buffer = frame.buffer;
+  UIImageOrientation orientation = frame.orientation;
   // code goes here
   return @[];
 }
@@ -62,6 +65,7 @@ The JS function name will be equal to the Objective-C function name you choose (
 
 ```objc
 #import <VisionCamera/FrameProcessorPlugin.h>
+#import <VisionCamera/Frame.h>
 ```
 
 3. Create an Objective-C source file with the same name as the Swift file, for the QR Code Plugin this will be `QRCodeFrameProcessorPlugin.m`. Add the following code:
@@ -79,12 +83,14 @@ The first parameter in the Macro specifies the JS function name. Make sure it is
 
 4. In the Swift file, add the following code:
 
-```swift {6}
+```swift {8}
 @objc(QRCodeFrameProcessorPlugin)
 public class QRCodeFrameProcessorPlugin: NSObject, FrameProcessorPluginBase {
 
   @objc
-  public static func callback(_: CMSampleBuffer!, withArgs _: [Any]!) -> Any! {
+  public static func callback(_ frame: Frame!, withArgs _: [Any]!) -> Any! {
+    let buffer = frame.buffer
+    let orientation = frame.orientation
     // code goes here
     return []
   }

example/ios/Frame Processor Plugins/QR Code Plugin (Objective-C)/QRCodeFrameProcessorPluginObjC.m

@@ -7,7 +7,7 @@
 
 #import <Foundation/Foundation.h>
 #import <VisionCamera/FrameProcessorPlugin.h>
-#import <Vision/VNDetectBarcodesRequest.h>
+#import <VisionCamera/Frame.h>
 
 // Example for an Objective-C Frame Processor plugin
 
@@ -16,8 +16,8 @@
 
 @implementation QRCodeFrameProcessorPluginObjC
 
-static inline id exampleObjC___scanQRCodes(CMSampleBufferRef buffer, NSArray* arguments) {
+static inline id exampleObjC___scanQRCodes(Frame* frame, NSArray* arguments) {
-  // TODO: Use some AI to detect QR codes in the CMSampleBufferRef
+  // TODO: Use some AI to detect QR codes in the frame
   return @[];
 }
 

example/ios/Frame Processor Plugins/QR Code Plugin (Swift)/QRCodeFrameProcessorPluginSwift.swift

@@ -12,7 +12,7 @@ import Vision
 @objc(QRCodeFrameProcessorPluginSwift)
 public class QRCodeFrameProcessorPluginSwift: NSObject, FrameProcessorPluginBase {
     @objc
-    public static func callback(_: CMSampleBuffer!, withArgs _: [Any]!) -> Any! {
+    public static func callback(_: Frame!, withArgs _: [Any]!) -> Any! {
         // TODO: Use some AI to detect QR codes in the CMSampleBufferRef
         []
     }

example/ios/Podfile.lock

@@ -490,7 +490,7 @@ SPEC CHECKSUMS:
   RNReanimated: 9c13c86454bfd54dab7505c1a054470bfecd2563
   RNStaticSafeAreaInsets: 6103cf09647fa427186d30f67b0f5163c1ae8252
   RNVectorIcons: 31cebfcf94e8cf8686eb5303ae0357da64d7a5a4
-  VisionCamera: 9886518481961e1c5d94cedb9b7513c28b8368c1
+  VisionCamera: 60f51b9c8e5074fda9952a603311338039f7bf28
   Yoga: 575c581c63e0d35c9a83f4b46d01d63abc1100ac
 
 PODFILE CHECKSUM: 4b093c1d474775c2eac3268011e4b0b80929d3a2

ios/CameraBridge.h

@@ -15,6 +15,7 @@
 
 #import "FrameProcessorCallback.h"
 #import "FrameProcessorRuntimeManager.h"
+#import "Frame.h"
 #import "RCTBridge+runOnJS.h"
 #import "JSConsoleHelper.h"
 

ios/CameraView+RecordVideo.swift

@@ -169,10 +169,12 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud
   }
 
   public final func captureOutput(_ captureOutput: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from _: AVCaptureConnection) {
+    // Video Recording runs in the same queue
     if isRecording {
       guard let recordingSession = recordingSession else {
         return invokeOnError(.capture(.unknown(message: "isRecording was true but the RecordingSession was null!")))
       }
+
       switch captureOutput {
       case is AVCaptureVideoDataOutput:
         recordingSession.appendBuffer(sampleBuffer, type: .video, timestamp: CMSampleBufferGetPresentationTimeStamp(sampleBuffer))
@@ -191,8 +193,10 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud
       let diff = DispatchTime.now().uptimeNanoseconds - lastFrameProcessorCall.uptimeNanoseconds
       let secondsPerFrame = 1.0 / frameProcessorFps.doubleValue
       let nanosecondsPerFrame = secondsPerFrame * 1_000_000_000.0
+
       if diff > UInt64(nanosecondsPerFrame) {
-        frameProcessor(sampleBuffer)
+        let frame = Frame(buffer: sampleBuffer, orientation: bufferOrientation)
+        frameProcessor(frame)
         lastFrameProcessorCall = DispatchTime.now()
       }
     }
@@ -221,4 +225,32 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud
       return String(describing: reason)
     }
   #endif
+
+  /**
+   Gets the orientation of the CameraView's images (CMSampleBuffers).
+   */
+  var bufferOrientation: UIImage.Orientation {
+    guard let cameraPosition = videoDeviceInput?.device.position else {
+      return .up
+    }
+
+    switch UIDevice.current.orientation {
+    case .portrait:
+      return cameraPosition == .front ? .leftMirrored : .right
+
+    case .landscapeLeft:
+      return cameraPosition == .front ? .downMirrored : .up
+
+    case .portraitUpsideDown:
+      return cameraPosition == .front ? .rightMirrored : .left
+
+    case .landscapeRight:
+      return cameraPosition == .front ? .upMirrored : .down
+
+    case .unknown, .faceUp, .faceDown:
+      fallthrough
+    @unknown default:
+      return .up
+    }
+  }
 }

ios/Frame Processor/CMSampleBufferRefHolder.h

@@ -1,22 +0,0 @@
-//
-//  CMSampleBufferRefHolder.h
-//  VisionCamera
-//
-//  Created by Marc Rousavy on 15.03.21.
-//  Copyright © 2021 mrousavy. All rights reserved.
-//
-
-#pragma once
-
-#import <Foundation/Foundation.h>
-#import <CoreMedia/CMSampleBuffer.h>
-
-@interface CMSampleBufferRefHolder : NSObject {
-  CMSampleBufferRef buffer;
-}
-
-- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer;
-
-@property (nonatomic) CMSampleBufferRef buffer;
-
-@end

ios/Frame Processor/CMSampleBufferRefHolder.m

@@ -1,25 +0,0 @@
-//
-//  CMSampleBufferRefHolder.m
-//  VisionCamera
-//
-//  Created by Marc Rousavy on 08.06.21.
-//  Copyright © 2021 mrousavy. All rights reserved.
-//
-
-#import "CMSampleBufferRefHolder.h"
-#import <Foundation/Foundation.h>
-#import <CoreMedia/CMSampleBuffer.h>
-
-@implementation CMSampleBufferRefHolder
-
-- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer {
-  self = [super init];
-  if (self) {
-    self.buffer = buffer;
-  }
-  return self;
-}
-
-@synthesize buffer;
-
-@end

ios/Frame Processor/Frame.h

@@ -0,0 +1,22 @@
+//
+//  Frame.h
+//  VisionCamera
+//
+//  Created by Marc Rousavy on 15.03.21.
+//  Copyright © 2021 mrousavy. All rights reserved.
+//
+
+#pragma once
+
+#import <Foundation/Foundation.h>
+#import <CoreMedia/CMSampleBuffer.h>
+#import <UIKit/UIImage.h>
+
+@interface Frame : NSObject
+
+- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer orientation:(UIImageOrientation)orientation;
+
+@property (nonatomic, readonly) CMSampleBufferRef buffer;
+@property (nonatomic, readonly) UIImageOrientation orientation;
+
+@end

ios/Frame Processor/Frame.m

@@ -0,0 +1,30 @@
+//
+//  Frame.m
+//  VisionCamera
+//
+//  Created by Marc Rousavy on 08.06.21.
+//  Copyright © 2021 mrousavy. All rights reserved.
+//
+
+#import "Frame.h"
+#import <Foundation/Foundation.h>
+#import <CoreMedia/CMSampleBuffer.h>
+
+@implementation Frame {
+  CMSampleBufferRef buffer;
+  UIImageOrientation orientation;
+}
+
+- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer orientation:(UIImageOrientation)orientation {
+  self = [super init];
+  if (self) {
+    _buffer = buffer;
+    _orientation = orientation;
+  }
+  return self;
+}
+
+@synthesize buffer = _buffer;
+@synthesize orientation = _orientation;
+
+@end

ios/Frame Processor/FrameHostObject.h

@@ -10,12 +10,13 @@
 
 #import <jsi/jsi.h>
 #import <CoreMedia/CMSampleBuffer.h>
+#import "Frame.h"
 
 using namespace facebook;
 
 class JSI_EXPORT FrameHostObject: public jsi::HostObject {
 public:
-  explicit FrameHostObject(CMSampleBufferRef buffer): buffer(buffer) {}
+  explicit FrameHostObject(Frame* frame): frame(frame) {}
   ~FrameHostObject();
 
 public:
@@ -24,5 +25,5 @@ public:
   void destroyBuffer();
   
 public:
-  CMSampleBufferRef buffer;
+  Frame* frame;
 };

ios/Frame Processor/FrameHostObject.mm

@@ -37,7 +37,7 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
   }
   if (name == "toString") {
     auto toString = [this] (jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
-      auto imageBuffer = CMSampleBufferGetImageBuffer(buffer);
+      auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
       auto width = CVPixelBufferGetWidth(imageBuffer);
       auto height = CVPixelBufferGetHeight(imageBuffer);
 
@@ -48,30 +48,30 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
   }
 
   if (name == "isValid") {
-    auto isValid = buffer != nil && CMSampleBufferIsValid(buffer);
+    auto isValid = frame != nil && CMSampleBufferIsValid(frame.buffer);
     return jsi::Value(isValid);
   }
   if (name == "isReady") {
-    auto isReady = buffer != nil && CMSampleBufferDataIsReady(buffer);
+    auto isReady = frame != nil && CMSampleBufferDataIsReady(frame.buffer);
     return jsi::Value(isReady);
   }
   if (name == "width") {
-    auto imageBuffer = CMSampleBufferGetImageBuffer(buffer);
+    auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
     auto width = CVPixelBufferGetWidth(imageBuffer);
     return jsi::Value((double) width);
   }
   if (name == "height") {
-    auto imageBuffer = CMSampleBufferGetImageBuffer(buffer);
+    auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
     auto height = CVPixelBufferGetHeight(imageBuffer);
     return jsi::Value((double) height);
   }
   if (name == "bytesPerRow") {
-    auto imageBuffer = CMSampleBufferGetImageBuffer(buffer);
+    auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
     auto bytesPerRow = CVPixelBufferGetPlaneCount(imageBuffer);
     return jsi::Value((double) bytesPerRow);
   }
   if (name == "planesCount") {
-    auto imageBuffer = CMSampleBufferGetImageBuffer(buffer);
+    auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
     auto planesCount = CVPixelBufferGetPlaneCount(imageBuffer);
     return jsi::Value((double) planesCount);
   }
@@ -85,5 +85,5 @@ FrameHostObject::~FrameHostObject() {
 
 void FrameHostObject::destroyBuffer() {
   // ARC will hopefully delete it lol
-  this->buffer = nil;
+  this->frame = nil;
 }

ios/Frame Processor/FrameProcessorCallback.h

@@ -9,6 +9,6 @@
 #pragma once
 
 #import <Foundation/Foundation.h>
-#import <CoreMedia/CMSampleBuffer.h>
+#import "Frame.h"
 
-typedef void (^FrameProcessorCallback) (CMSampleBufferRef buffer);
+typedef void (^FrameProcessorCallback) (Frame* frame);

ios/Frame Processor/FrameProcessorPlugin.h

@@ -11,10 +11,10 @@
 
 #import <Foundation/Foundation.h>
 #import "FrameProcessorPluginRegistry.h"
-#import <CoreMedia/CMSampleBuffer.h>
+#import "Frame.h"
 
 @protocol FrameProcessorPluginBase
-+ (id) callback:(CMSampleBufferRef)buffer withArgs:(NSArray<id>*)args;
++ (id) callback:(Frame*)frame withArgs:(NSArray<id>*)args;
 @end
 
 
@@ -23,7 +23,7 @@
 
 /**
  * Use this Macro to register the given function as a Frame Processor.
- * * Make sure the given function is a C-style function with the following signature: static inline id callback(CMSampleBufferRef buffer)
+ * * Make sure the given function is a C-style function with the following signature: static inline id callback(Frame* frame, NSArray* args)
  * * Make sure the given function's name is unique across other frame processor plugins
  * * Make sure your frame processor returns a Value that can be converted to JS
  * * Make sure to use this Macro in an @implementation, not @interface
@@ -35,8 +35,8 @@
                                                                                     \
 +(void)load                                                                         \
 {                                                                                   \
-  [FrameProcessorPluginRegistry addFrameProcessorPlugin:@"__" @ #frame_processor callback:^id(CMSampleBufferRef buffer, NSArray<id>* args) { \
+  [FrameProcessorPluginRegistry addFrameProcessorPlugin:@"__" @ #frame_processor callback:^id(Frame* frame, NSArray<id>* args) { \
-    return frame_processor(buffer, args);                                           \
+    return frame_processor(frame, args);                                           \
   }];                                                                               \
 }
 
@@ -55,8 +55,8 @@ objc_name : NSObject<FrameProcessorPluginBase>
                                                                                     \
 __attribute__((constructor)) static void VISION_CONCAT(initialize_, objc_name)()    \
 {                                                                                   \
-  [FrameProcessorPluginRegistry addFrameProcessorPlugin:@"__" @ #name callback:^id(CMSampleBufferRef buffer, NSArray<id>* args) {    \
+  [FrameProcessorPluginRegistry addFrameProcessorPlugin:@"__" @ #name callback:^id(Frame* frame, NSArray<id>* args) {    \
-    return [objc_name callback:buffer withArgs:args];                               \
+    return [objc_name callback:frame withArgs:args];                               \
   }];                                                                               \
 }
 

ios/Frame Processor/FrameProcessorPluginRegistry.h

@@ -9,9 +9,9 @@
 #pragma once
 
 #import <Foundation/Foundation.h>
-#import <CoreMedia/CMSampleBuffer.h>
+#import "Frame.h"
 
-typedef id (^FrameProcessorPlugin) (CMSampleBufferRef buffer, NSArray<id>* arguments);
+typedef id (^FrameProcessorPlugin) (Frame* frame, NSArray<id>* arguments);
 
 @interface FrameProcessorPluginRegistry : NSObject
 

ios/Frame Processor/FrameProcessorRuntimeManager.mm

@@ -85,15 +85,22 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
       NSLog(@"FrameProcessorBindings: Installing Frame Processor plugin \"%s\"...", pluginName);
       FrameProcessorPlugin callback = [[FrameProcessorPluginRegistry frameProcessorPlugins] valueForKey:pluginKey];
 
-      auto function = [callback, callInvoker](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
+      auto function = [callback, callInvoker](jsi::Runtime& runtime,
+                                              const jsi::Value& thisValue,
+                                              const jsi::Value* arguments,
+                                              size_t count) -> jsi::Value {
+        
         auto frameHostObject = arguments[0].asObject(runtime).asHostObject(runtime);
         auto frame = static_cast<FrameHostObject*>(frameHostObject.get());
+        
         auto args = convertJSICStyleArrayToNSArray(runtime,
                                                    arguments + 1, // start at index 1 since first arg = Frame
                                                    count - 1, // use smaller count
                                                    callInvoker);
-        id result = callback(frame->buffer, args);
+        id result = callback(frame->frame, args);
+        
         return convertObjCObjectToJSIValue(runtime, result);
+        
       };
 
       visionGlobal.setProperty(visionRuntime, pluginName, jsi::Function::createFromHostFunction(visionRuntime,
@@ -129,7 +136,10 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
   NSLog(@"FrameProcessorBindings: Installing global functions...");
 
   // setFrameProcessor(viewTag: number, frameProcessor: (frame: Frame) => void)
-  auto setFrameProcessor = [self](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
+  auto setFrameProcessor = [self](jsi::Runtime& runtime,
+                                  const jsi::Value& thisValue,
+                                  const jsi::Value* arguments,
+                                  size_t count) -> jsi::Value {
     NSLog(@"FrameProcessorBindings: Setting new frame processor...");
     if (!arguments[0].isNumber()) throw jsi::JSError(runtime, "Camera::setFrameProcessor: First argument ('viewTag') must be a number!");
     if (!arguments[1].isObject()) throw jsi::JSError(runtime, "Camera::setFrameProcessor: Second argument ('frameProcessor') must be a function!");
@@ -163,7 +173,10 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
                                                                                                          setFrameProcessor));
 
   // unsetFrameProcessor(viewTag: number)
-  auto unsetFrameProcessor = [](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
+  auto unsetFrameProcessor = [](jsi::Runtime& runtime,
+                                const jsi::Value& thisValue,
+                                const jsi::Value* arguments,
+                                size_t count) -> jsi::Value {
     NSLog(@"FrameProcessorBindings: Removing frame processor...");
     if (!arguments[0].isNumber()) throw jsi::JSError(runtime, "Camera::unsetFrameProcessor: First argument ('viewTag') must be a number!");
     auto viewTag = arguments[0].asNumber();

ios/Frame Processor/FrameProcessorUtils.mm

@@ -7,22 +7,22 @@
 //
 
 #import "FrameProcessorUtils.h"
-#import <CoreMedia/CMSampleBuffer.h>
 #import <chrono>
 #import <memory>
 #import "FrameHostObject.h"
+#import "Frame.h"
 
 FrameProcessorCallback convertJSIFunctionToFrameProcessorCallback(jsi::Runtime &runtime, const jsi::Function &value) {
   __block auto cb = value.getFunction(runtime);
 
-  return ^(CMSampleBufferRef buffer) {
+  return ^(Frame* frame) {
 #if DEBUG
     std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now();
 #endif
 
-    auto frame = std::make_shared<FrameHostObject>(buffer);
+    auto frameHostObject = std::make_shared<FrameHostObject>(frame);
     try {
-      cb.call(runtime, jsi::Object::createFromHostObject(runtime, frame));
+      cb.call(runtime, jsi::Object::createFromHostObject(runtime, frameHostObject));
     } catch (jsi::JSError& jsError) {
       NSLog(@"Frame Processor threw an error: %s", jsError.getMessage().c_str());
     }
@@ -39,6 +39,6 @@ FrameProcessorCallback convertJSIFunctionToFrameProcessorCallback(jsi::Runtime &
     //  1. we are sure we don't need it anymore, the frame processor worklet has finished executing.
     //  2. we don't know when the JS runtime garbage collects this object, it might be holding it for a few more frames
     //     which then blocks the camera queue from pushing new frames (memory limit)
-    frame->destroyBuffer();
+    frameHostObject->destroyBuffer();
   };
 }

ios/React Utils/JSIUtils.mm

@@ -12,7 +12,7 @@
 #import <ReactCommon/CallInvoker.h>
 #import <React/RCTBridge.h>
 #import <ReactCommon/TurboModuleUtils.h>
-#import "../Frame Processor/CMSampleBufferRefHolder.h"
+#import "../Frame Processor/Frame.h"
 #import "../Frame Processor/FrameHostObject.h"
 
 using namespace facebook;
@@ -68,11 +68,9 @@ jsi::Value convertObjCObjectToJSIValue(jsi::Runtime &runtime, id value)
     return convertNSArrayToJSIArray(runtime, (NSArray *)value);
   } else if (value == (id)kCFNull) {
     return jsi::Value::null();
-  } else if ([value isKindOfClass:[CMSampleBufferRefHolder class]]) {
+  } else if ([value isKindOfClass:[Frame class]]) {
-    // it's boxed in a CMSampleBufferRefHolder because CMSampleBufferRef is not an NSObject
+    auto frameHostObject = std::make_shared<FrameHostObject>((Frame*)value);
-    CMSampleBufferRef buffer = [(CMSampleBufferRefHolder*)value buffer];
+    return jsi::Object::createFromHostObject(runtime, frameHostObject);
-    auto frame = std::make_shared<FrameHostObject>(buffer);
-    return jsi::Object::createFromHostObject(runtime, frame);
   }
   return jsi::Value::undefined();
 }
@@ -155,7 +153,7 @@ id convertJSIValueToObjCObject(jsi::Runtime &runtime, const jsi::Value &value, s
       auto hostObject = o.asHostObject(runtime);
       auto frame = dynamic_cast<FrameHostObject*>(hostObject.get());
       if (frame != nullptr) {
-        return [[CMSampleBufferRefHolder alloc] initWithBuffer:frame->buffer];
+        return frame->frame;
       }
     }
     return convertJSIObjectToNSDictionary(runtime, o, jsInvoker);

ios/VisionCamera.xcodeproj/project.pbxproj

@@ -81,7 +81,7 @@
 		B80E069F266632F000728644 /* AVAudioSession+updateCategory.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVAudioSession+updateCategory.swift"; sourceTree = "<group>"; };
 		B8103E1B25FF553B007A1684 /* FrameProcessorUtils.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = FrameProcessorUtils.mm; sourceTree = "<group>"; };
 		B8103E1E25FF5550007A1684 /* FrameProcessorUtils.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorUtils.h; sourceTree = "<group>"; };
-		B8103E5725FF56F0007A1684 /* CMSampleBufferRefHolder.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = CMSampleBufferRefHolder.h; sourceTree = "<group>"; };
+		B8103E5725FF56F0007A1684 /* Frame.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Frame.h; sourceTree = "<group>"; };
 		B81D41EF263C86F900B041FD /* JSIUtils.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = JSIUtils.h; sourceTree = "<group>"; };
 		B82FBA942614B69D00909718 /* RCTBridge+runOnJS.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "RCTBridge+runOnJS.h"; sourceTree = "<group>"; };
 		B82FBA952614B69D00909718 /* RCTBridge+runOnJS.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = "RCTBridge+runOnJS.mm"; sourceTree = "<group>"; };
@@ -140,7 +140,7 @@
 		B8DB3BCB263DC97E004C18D7 /* AVFileType+descriptor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVFileType+descriptor.swift"; sourceTree = "<group>"; };
 		B8DCF09125EA7BEE00EA5C72 /* SpeedChecker.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SpeedChecker.h; sourceTree = "<group>"; };
 		B8DCF14425EA817D00EA5C72 /* MakeJSIRuntime.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MakeJSIRuntime.h; sourceTree = "<group>"; };
-		B8F7DDD1266F715D00120533 /* CMSampleBufferRefHolder.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = CMSampleBufferRefHolder.m; sourceTree = "<group>"; };
+		B8F7DDD1266F715D00120533 /* Frame.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = Frame.m; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
@@ -263,8 +263,8 @@
 				B80D67A825FA25380008FE8D /* FrameProcessorCallback.h */,
 				B8103E1E25FF5550007A1684 /* FrameProcessorUtils.h */,
 				B8103E1B25FF553B007A1684 /* FrameProcessorUtils.mm */,
-				B8103E5725FF56F0007A1684 /* CMSampleBufferRefHolder.h */,
+				B8103E5725FF56F0007A1684 /* Frame.h */,
-				B8F7DDD1266F715D00120533 /* CMSampleBufferRefHolder.m */,
+				B8F7DDD1266F715D00120533 /* Frame.m */,
 				B84760A22608EE38004C3180 /* FrameHostObject.h */,
 				B84760A52608EE7C004C3180 /* FrameHostObject.mm */,
 				B8A751D62609E4980011C623 /* FrameProcessorRuntimeManager.h */,