feat: Frame Processors: Allow returning Frames (support for resize and other frame manipulations) (#185)

* batch

* Init Frame as box

* Use ObjC syntax

* Fix access

* Revert "Fix access"

This reverts commit 7de09e52739d4c2b53f485d5ed696f1665fa5737.

* Revert "Use ObjC syntax"

This reverts commit e33f05ae8451cc4ee24af41d14dc76a57c157554.

* Revert "Init Frame as box"

This reverts commit 5adafb6109bfbf7fddb8ddc4af7d306b7b76b476.

* use holder

* convert buffer <-> jsi object

* add docs

* add more docs

* Update JSIUtils.mm

* Update FRAME_PROCESSORS_CREATE_OVERVIEW.mdx

* Update CameraView+RecordVideo.swift

VisionCamera.podspec

@@ -25,6 +25,7 @@ Pod::Spec.new do |s|
   s.source_files = [
     "ios/**/*.{m,mm,swift}",
     "ios/CameraBridge.h",
+    "ios/Frame Processor/CMSampleBufferRefHolder.h",
     "ios/Frame Processor/FrameProcessorCallback.h",
     "ios/Frame Processor/FrameProcessorRuntimeManager.h",
     "ios/Frame Processor/FrameProcessorPluginRegistry.h",

docs/docs/guides/FRAME_PROCESSORS_CREATE_OVERVIEW.mdx

@@ -33,14 +33,15 @@ To achieve **maximum performance**, the `scanQRCodes` function is written in a n
 The Frame Processor Plugin Registry API automatically manages type conversion from JS <-> native. They are converted into the most efficient data-structures, as seen here:
 
 | JS Type              | Objective-C Type          | Java Type                  |
-|----------------------|--------------------------|----------------------------|
+|----------------------|---------------------------|----------------------------|
-| `number`             | `NSNumber` (double)      | `double`                   |
+| `number`             | `NSNumber*` (double)      | `double`                   |
-| `boolean`            | `NSNumber` (boolean)     | `boolean`                  |
+| `boolean`            | `NSNumber*` (boolean)     | `boolean`                  |
-| `string`             | `NSString`               | `String`                   |
+| `string`             | `NSString*`               | `String`                   |
-| `[]`                 | `NSArray`                | `Array<Object>`            |
+| `[]`                 | `NSArray*`                | `Array<Object>`            |
-| `{}`                 | `NSDictionary`           | `HashMap<Object>`          |
+| `{}`                 | `NSDictionary*`           | `HashMap<Object>`          |
-| `undefined`          | `nil` / `NSNull`         | `null`                     |
+| `undefined` / `null` | `nil` / `NSNull`          | `null`                     |
 | `(any, any) => void` | `RCTResponseSenderBlock`  | `(Object, Object) -> void` |
+| `Frame`              | `CMSampleBufferRefHolder` | `ImageProxy`               |
 
 ### Return values
 
@@ -56,12 +57,36 @@ Returns a `string` in JS:
 
 ```js
 export function detectObject(frame: Frame): string {
-  'worklet';
+  'worklet'
-  const result =  __detectObject(frame);
+  const result = __detectObject(frame)
   _log(result) // <-- "cat"
 }
 ```
 
+You can also manipulate the buffer and return it (or a copy) by using the `CMSampleBufferRefHolder` class:
+
+```objc
+static inline id resize(CMSampleBufferRef buffer, NSArray args) {
+  NSNumber* width = [arguments objectAtIndex:0];
+  NSNumber* height = [arguments objectAtIndex:1];
+
+  CMSampleBufferRef resizedBuffer = CMSampleBufferCopyAndResize(buffer, width, height);
+  return [[CMSampleBufferRefHolder alloc] initWithBuffer:resizedBuffer];
+}
+```
+
+Which returns a `Frame` in JS:
+
+```js
+const frameProcessor = useFrameProcessor((frame) => {
+  'worklet';
+  // by downscaling the frame, the `detectObjects` function runs faster.
+  const resizedFrame = resize(frame, 720, 480)
+  const objects = detectObjects(resizedFrame)
+  _log(objects)
+}, [])
+```
+
 ### Parameters
 
 Frame Processors can also accept parameters, following the same type convention as [return values](#return-values):

example/ios/Podfile.lock

@@ -322,7 +322,7 @@ PODS:
     - React
   - RNVectorIcons (8.1.0):
     - React-Core
-  - VisionCamera (2.1.0):
+  - VisionCamera (2.2.0):
     - React
     - React-callinvoker
     - React-Core
@@ -490,7 +490,7 @@ SPEC CHECKSUMS:
   RNReanimated: 9c13c86454bfd54dab7505c1a054470bfecd2563
   RNStaticSafeAreaInsets: 6103cf09647fa427186d30f67b0f5163c1ae8252
   RNVectorIcons: 31cebfcf94e8cf8686eb5303ae0357da64d7a5a4
-  VisionCamera: 478f70a945846254699cfd33c2ac852e79258b83
+  VisionCamera: 9886518481961e1c5d94cedb9b7513c28b8368c1
   Yoga: 575c581c63e0d35c9a83f4b46d01d63abc1100ac
 
 PODFILE CHECKSUM: 4b093c1d474775c2eac3268011e4b0b80929d3a2

ios/CameraView+RecordVideo.swift

@@ -203,8 +203,9 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud
       if frameProcessorCallback != nil && !hasLoggedFrameDropWarning && captureOutput is AVCaptureVideoDataOutput {
         let reason = findFrameDropReason(inBuffer: buffer)
         ReactLogger.log(level: .warning,
-                        message: "Dropped a Frame. This might indicate that your Frame Processor is doing too much work. " +
+                        message: "Dropped a Frame - This might indicate that your Frame Processor is doing too much work. " +
-                          "Either throttle the frame processor's frame rate, or optimize your frame processor's execution speed. Frame drop reason: \(reason)",
+                          "Either throttle the frame processor's frame rate using the `frameProcessorFps` prop, or optimize " +
+                          "your frame processor's execution speed. Frame drop reason: \(reason)",
                         alsoLogToJS: true)
         hasLoggedFrameDropWarning = true
       }

ios/Frame Processor/CMSampleBufferRefHolder.h

@@ -0,0 +1,22 @@
+//
+//  CMSampleBufferRefHolder.h
+//  VisionCamera
+//
+//  Created by Marc Rousavy on 15.03.21.
+//  Copyright © 2021 mrousavy. All rights reserved.
+//
+
+#pragma once
+
+#import <Foundation/Foundation.h>
+#import <CoreMedia/CMSampleBuffer.h>
+
+@interface CMSampleBufferRefHolder : NSObject {
+  CMSampleBufferRef buffer;
+}
+
+- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer;
+
+@property (nonatomic) CMSampleBufferRef buffer;
+
+@end

ios/Frame Processor/CMSampleBufferRefHolder.m

@@ -0,0 +1,25 @@
+//
+//  CMSampleBufferRefHolder.m
+//  VisionCamera
+//
+//  Created by Marc Rousavy on 08.06.21.
+//  Copyright © 2021 mrousavy. All rights reserved.
+//
+
+#import "CMSampleBufferRefHolder.h"
+#import <Foundation/Foundation.h>
+#import <CoreMedia/CMSampleBuffer.h>
+
+@implementation CMSampleBufferRefHolder
+
+- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer {
+  self = [super init];
+  if (self) {
+    self.buffer = buffer;
+  }
+  return self;
+}
+
+@synthesize buffer;
+
+@end

ios/Frame Processor/Frame.h

@@ -1,20 +0,0 @@
-//
-//  Frame.h
-//  VisionCamera
-//
-//  Created by Marc Rousavy on 15.03.21.
-//  Copyright © 2021 mrousavy. All rights reserved.
-//
-
-#pragma once
-
-#import <CoreMedia/CMSampleBuffer.h>
-
-// TODO: Make this Objective-C so it can be imported in Swift?
-class Frame {
- public:
-  explicit Frame(CMSampleBufferRef buffer): buffer(buffer) {}
-
- public:
-  CMSampleBufferRef buffer;
-};

ios/Frame Processor/FrameHostObject.h

@@ -8,19 +8,21 @@
 
 #pragma once
 
-#import "Frame.h"
 #import <jsi/jsi.h>
 #import <CoreMedia/CMSampleBuffer.h>
 
 using namespace facebook;
 
-class JSI_EXPORT FrameHostObject: public Frame, public jsi::HostObject {
+class JSI_EXPORT FrameHostObject: public jsi::HostObject {
 public:
-  explicit FrameHostObject(CMSampleBufferRef buffer): Frame(buffer) {}
+  explicit FrameHostObject(CMSampleBufferRef buffer): buffer(buffer) {}
   ~FrameHostObject();
 
 public:
   jsi::Value get(jsi::Runtime&, const jsi::PropNameID& name) override;
   std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& rt) override;
   void destroyBuffer();
+  
+public:
+  CMSampleBufferRef buffer;
 };

ios/React Utils/JSIUtils.mm

@@ -12,6 +12,8 @@
 #import <ReactCommon/CallInvoker.h>
 #import <React/RCTBridge.h>
 #import <ReactCommon/TurboModuleUtils.h>
+#import "../Frame Processor/CMSampleBufferRefHolder.h"
+#import "../Frame Processor/FrameHostObject.h"
 
 using namespace facebook;
 using namespace facebook::react;
@@ -66,6 +68,11 @@ jsi::Value convertObjCObjectToJSIValue(jsi::Runtime &runtime, id value)
     return convertNSArrayToJSIArray(runtime, (NSArray *)value);
   } else if (value == (id)kCFNull) {
     return jsi::Value::null();
+  } else if ([value isKindOfClass:[CMSampleBufferRefHolder class]]) {
+    // it's boxed in a CMSampleBufferRefHolder because CMSampleBufferRef is not an NSObject
+    CMSampleBufferRef buffer = [(CMSampleBufferRefHolder*)value buffer];
+    auto frame = std::make_shared<FrameHostObject>(buffer);
+    return jsi::Object::createFromHostObject(runtime, frame);
   }
   return jsi::Value::undefined();
 }
@@ -144,6 +151,13 @@ id convertJSIValueToObjCObject(jsi::Runtime &runtime, const jsi::Value &value, s
     if (o.isFunction(runtime)) {
       return convertJSIFunctionToCallback(runtime, std::move(o.getFunction(runtime)), jsInvoker);
     }
+    if (o.isHostObject(runtime)) {
+      auto hostObject = o.asHostObject(runtime);
+      auto frame = dynamic_cast<FrameHostObject*>(hostObject.get());
+      if (frame != nullptr) {
+        return [[CMSampleBufferRefHolder alloc] initWithBuffer:frame->buffer];
+      }
+    }
     return convertJSIObjectToNSDictionary(runtime, o, jsInvoker);
   }
 

ios/VisionCamera.xcodeproj/project.pbxproj

@@ -81,7 +81,7 @@
 		B80E069F266632F000728644 /* AVAudioSession+updateCategory.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVAudioSession+updateCategory.swift"; sourceTree = "<group>"; };
 		B8103E1B25FF553B007A1684 /* FrameProcessorUtils.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = FrameProcessorUtils.mm; sourceTree = "<group>"; };
 		B8103E1E25FF5550007A1684 /* FrameProcessorUtils.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorUtils.h; sourceTree = "<group>"; };
-		B8103E5725FF56F0007A1684 /* Frame.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Frame.h; sourceTree = "<group>"; };
+		B8103E5725FF56F0007A1684 /* CMSampleBufferRefHolder.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = CMSampleBufferRefHolder.h; sourceTree = "<group>"; };
 		B81D41EF263C86F900B041FD /* JSIUtils.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = JSIUtils.h; sourceTree = "<group>"; };
 		B82FBA942614B69D00909718 /* RCTBridge+runOnJS.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "RCTBridge+runOnJS.h"; sourceTree = "<group>"; };
 		B82FBA952614B69D00909718 /* RCTBridge+runOnJS.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = "RCTBridge+runOnJS.mm"; sourceTree = "<group>"; };
@@ -140,6 +140,7 @@
 		B8DB3BCB263DC97E004C18D7 /* AVFileType+descriptor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVFileType+descriptor.swift"; sourceTree = "<group>"; };
 		B8DCF09125EA7BEE00EA5C72 /* SpeedChecker.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SpeedChecker.h; sourceTree = "<group>"; };
 		B8DCF14425EA817D00EA5C72 /* MakeJSIRuntime.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MakeJSIRuntime.h; sourceTree = "<group>"; };
+		B8F7DDD1266F715D00120533 /* CMSampleBufferRefHolder.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = CMSampleBufferRefHolder.m; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
@@ -262,7 +263,8 @@
 				B80D67A825FA25380008FE8D /* FrameProcessorCallback.h */,
 				B8103E1E25FF5550007A1684 /* FrameProcessorUtils.h */,
 				B8103E1B25FF553B007A1684 /* FrameProcessorUtils.mm */,
-				B8103E5725FF56F0007A1684 /* Frame.h */,
+				B8103E5725FF56F0007A1684 /* CMSampleBufferRefHolder.h */,
+				B8F7DDD1266F715D00120533 /* CMSampleBufferRefHolder.m */,
 				B84760A22608EE38004C3180 /* FrameHostObject.h */,
 				B84760A52608EE7C004C3180 /* FrameHostObject.mm */,
 				B8A751D62609E4980011C623 /* FrameProcessorRuntimeManager.h */,