feat: Frame Processors: Allow returning Frame
s (support for resize and other frame manipulations) (#185)
* batch * Init Frame as box * Use ObjC syntax * Fix access * Revert "Fix access" This reverts commit 7de09e52739d4c2b53f485d5ed696f1665fa5737. * Revert "Use ObjC syntax" This reverts commit e33f05ae8451cc4ee24af41d14dc76a57c157554. * Revert "Init Frame as box" This reverts commit 5adafb6109bfbf7fddb8ddc4af7d306b7b76b476. * use holder * convert buffer <-> jsi object * add docs * add more docs * Update JSIUtils.mm * Update FRAME_PROCESSORS_CREATE_OVERVIEW.mdx * Update CameraView+RecordVideo.swift
This commit is contained in:
parent
1b08c0cbae
commit
4038db2e28
@ -25,6 +25,7 @@ Pod::Spec.new do |s|
|
||||
s.source_files = [
|
||||
"ios/**/*.{m,mm,swift}",
|
||||
"ios/CameraBridge.h",
|
||||
"ios/Frame Processor/CMSampleBufferRefHolder.h",
|
||||
"ios/Frame Processor/FrameProcessorCallback.h",
|
||||
"ios/Frame Processor/FrameProcessorRuntimeManager.h",
|
||||
"ios/Frame Processor/FrameProcessorPluginRegistry.h",
|
||||
|
@ -33,14 +33,15 @@ To achieve **maximum performance**, the `scanQRCodes` function is written in a n
|
||||
The Frame Processor Plugin Registry API automatically manages type conversion from JS <-> native. They are converted into the most efficient data-structures, as seen here:
|
||||
|
||||
| JS Type | Objective-C Type | Java Type |
|
||||
|----------------------|--------------------------|----------------------------|
|
||||
| `number` | `NSNumber` (double) | `double` |
|
||||
| `boolean` | `NSNumber` (boolean) | `boolean` |
|
||||
| `string` | `NSString` | `String` |
|
||||
| `[]` | `NSArray` | `Array<Object>` |
|
||||
| `{}` | `NSDictionary` | `HashMap<Object>` |
|
||||
| `undefined` | `nil` / `NSNull` | `null` |
|
||||
|----------------------|---------------------------|----------------------------|
|
||||
| `number` | `NSNumber*` (double) | `double` |
|
||||
| `boolean` | `NSNumber*` (boolean) | `boolean` |
|
||||
| `string` | `NSString*` | `String` |
|
||||
| `[]` | `NSArray*` | `Array<Object>` |
|
||||
| `{}` | `NSDictionary*` | `HashMap<Object>` |
|
||||
| `undefined` / `null` | `nil` / `NSNull` | `null` |
|
||||
| `(any, any) => void` | `RCTResponseSenderBlock` | `(Object, Object) -> void` |
|
||||
| `Frame` | `CMSampleBufferRefHolder` | `ImageProxy` |
|
||||
|
||||
### Return values
|
||||
|
||||
@ -56,12 +57,36 @@ Returns a `string` in JS:
|
||||
|
||||
```js
|
||||
export function detectObject(frame: Frame): string {
|
||||
'worklet';
|
||||
const result = __detectObject(frame);
|
||||
'worklet'
|
||||
const result = __detectObject(frame)
|
||||
_log(result) // <-- "cat"
|
||||
}
|
||||
```
|
||||
|
||||
You can also manipulate the buffer and return it (or a copy) by using the `CMSampleBufferRefHolder` class:
|
||||
|
||||
```objc
|
||||
static inline id resize(CMSampleBufferRef buffer, NSArray args) {
|
||||
NSNumber* width = [arguments objectAtIndex:0];
|
||||
NSNumber* height = [arguments objectAtIndex:1];
|
||||
|
||||
CMSampleBufferRef resizedBuffer = CMSampleBufferCopyAndResize(buffer, width, height);
|
||||
return [[CMSampleBufferRefHolder alloc] initWithBuffer:resizedBuffer];
|
||||
}
|
||||
```
|
||||
|
||||
Which returns a `Frame` in JS:
|
||||
|
||||
```js
|
||||
const frameProcessor = useFrameProcessor((frame) => {
|
||||
'worklet';
|
||||
// by downscaling the frame, the `detectObjects` function runs faster.
|
||||
const resizedFrame = resize(frame, 720, 480)
|
||||
const objects = detectObjects(resizedFrame)
|
||||
_log(objects)
|
||||
}, [])
|
||||
```
|
||||
|
||||
### Parameters
|
||||
|
||||
Frame Processors can also accept parameters, following the same type convention as [return values](#return-values):
|
||||
|
@ -322,7 +322,7 @@ PODS:
|
||||
- React
|
||||
- RNVectorIcons (8.1.0):
|
||||
- React-Core
|
||||
- VisionCamera (2.1.0):
|
||||
- VisionCamera (2.2.0):
|
||||
- React
|
||||
- React-callinvoker
|
||||
- React-Core
|
||||
@ -490,7 +490,7 @@ SPEC CHECKSUMS:
|
||||
RNReanimated: 9c13c86454bfd54dab7505c1a054470bfecd2563
|
||||
RNStaticSafeAreaInsets: 6103cf09647fa427186d30f67b0f5163c1ae8252
|
||||
RNVectorIcons: 31cebfcf94e8cf8686eb5303ae0357da64d7a5a4
|
||||
VisionCamera: 478f70a945846254699cfd33c2ac852e79258b83
|
||||
VisionCamera: 9886518481961e1c5d94cedb9b7513c28b8368c1
|
||||
Yoga: 575c581c63e0d35c9a83f4b46d01d63abc1100ac
|
||||
|
||||
PODFILE CHECKSUM: 4b093c1d474775c2eac3268011e4b0b80929d3a2
|
||||
|
@ -203,8 +203,9 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud
|
||||
if frameProcessorCallback != nil && !hasLoggedFrameDropWarning && captureOutput is AVCaptureVideoDataOutput {
|
||||
let reason = findFrameDropReason(inBuffer: buffer)
|
||||
ReactLogger.log(level: .warning,
|
||||
message: "Dropped a Frame. This might indicate that your Frame Processor is doing too much work. " +
|
||||
"Either throttle the frame processor's frame rate, or optimize your frame processor's execution speed. Frame drop reason: \(reason)",
|
||||
message: "Dropped a Frame - This might indicate that your Frame Processor is doing too much work. " +
|
||||
"Either throttle the frame processor's frame rate using the `frameProcessorFps` prop, or optimize " +
|
||||
"your frame processor's execution speed. Frame drop reason: \(reason)",
|
||||
alsoLogToJS: true)
|
||||
hasLoggedFrameDropWarning = true
|
||||
}
|
||||
|
22
ios/Frame Processor/CMSampleBufferRefHolder.h
Normal file
22
ios/Frame Processor/CMSampleBufferRefHolder.h
Normal file
@ -0,0 +1,22 @@
|
||||
//
|
||||
// CMSampleBufferRefHolder.h
|
||||
// VisionCamera
|
||||
//
|
||||
// Created by Marc Rousavy on 15.03.21.
|
||||
// Copyright © 2021 mrousavy. All rights reserved.
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#import <Foundation/Foundation.h>
|
||||
#import <CoreMedia/CMSampleBuffer.h>
|
||||
|
||||
@interface CMSampleBufferRefHolder : NSObject {
|
||||
CMSampleBufferRef buffer;
|
||||
}
|
||||
|
||||
- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer;
|
||||
|
||||
@property (nonatomic) CMSampleBufferRef buffer;
|
||||
|
||||
@end
|
25
ios/Frame Processor/CMSampleBufferRefHolder.m
Normal file
25
ios/Frame Processor/CMSampleBufferRefHolder.m
Normal file
@ -0,0 +1,25 @@
|
||||
//
|
||||
// CMSampleBufferRefHolder.m
|
||||
// VisionCamera
|
||||
//
|
||||
// Created by Marc Rousavy on 08.06.21.
|
||||
// Copyright © 2021 mrousavy. All rights reserved.
|
||||
//
|
||||
|
||||
#import "CMSampleBufferRefHolder.h"
|
||||
#import <Foundation/Foundation.h>
|
||||
#import <CoreMedia/CMSampleBuffer.h>
|
||||
|
||||
@implementation CMSampleBufferRefHolder
|
||||
|
||||
- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer {
|
||||
self = [super init];
|
||||
if (self) {
|
||||
self.buffer = buffer;
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
@synthesize buffer;
|
||||
|
||||
@end
|
@ -1,20 +0,0 @@
|
||||
//
|
||||
// Frame.h
|
||||
// VisionCamera
|
||||
//
|
||||
// Created by Marc Rousavy on 15.03.21.
|
||||
// Copyright © 2021 mrousavy. All rights reserved.
|
||||
//
|
||||
|
||||
#pragma once
|
||||
|
||||
#import <CoreMedia/CMSampleBuffer.h>
|
||||
|
||||
// TODO: Make this Objective-C so it can be imported in Swift?
|
||||
class Frame {
|
||||
public:
|
||||
explicit Frame(CMSampleBufferRef buffer): buffer(buffer) {}
|
||||
|
||||
public:
|
||||
CMSampleBufferRef buffer;
|
||||
};
|
@ -8,19 +8,21 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#import "Frame.h"
|
||||
#import <jsi/jsi.h>
|
||||
#import <CoreMedia/CMSampleBuffer.h>
|
||||
|
||||
using namespace facebook;
|
||||
|
||||
class JSI_EXPORT FrameHostObject: public Frame, public jsi::HostObject {
|
||||
class JSI_EXPORT FrameHostObject: public jsi::HostObject {
|
||||
public:
|
||||
explicit FrameHostObject(CMSampleBufferRef buffer): Frame(buffer) {}
|
||||
explicit FrameHostObject(CMSampleBufferRef buffer): buffer(buffer) {}
|
||||
~FrameHostObject();
|
||||
|
||||
public:
|
||||
jsi::Value get(jsi::Runtime&, const jsi::PropNameID& name) override;
|
||||
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& rt) override;
|
||||
void destroyBuffer();
|
||||
|
||||
public:
|
||||
CMSampleBufferRef buffer;
|
||||
};
|
||||
|
@ -12,6 +12,8 @@
|
||||
#import <ReactCommon/CallInvoker.h>
|
||||
#import <React/RCTBridge.h>
|
||||
#import <ReactCommon/TurboModuleUtils.h>
|
||||
#import "../Frame Processor/CMSampleBufferRefHolder.h"
|
||||
#import "../Frame Processor/FrameHostObject.h"
|
||||
|
||||
using namespace facebook;
|
||||
using namespace facebook::react;
|
||||
@ -66,6 +68,11 @@ jsi::Value convertObjCObjectToJSIValue(jsi::Runtime &runtime, id value)
|
||||
return convertNSArrayToJSIArray(runtime, (NSArray *)value);
|
||||
} else if (value == (id)kCFNull) {
|
||||
return jsi::Value::null();
|
||||
} else if ([value isKindOfClass:[CMSampleBufferRefHolder class]]) {
|
||||
// it's boxed in a CMSampleBufferRefHolder because CMSampleBufferRef is not an NSObject
|
||||
CMSampleBufferRef buffer = [(CMSampleBufferRefHolder*)value buffer];
|
||||
auto frame = std::make_shared<FrameHostObject>(buffer);
|
||||
return jsi::Object::createFromHostObject(runtime, frame);
|
||||
}
|
||||
return jsi::Value::undefined();
|
||||
}
|
||||
@ -144,6 +151,13 @@ id convertJSIValueToObjCObject(jsi::Runtime &runtime, const jsi::Value &value, s
|
||||
if (o.isFunction(runtime)) {
|
||||
return convertJSIFunctionToCallback(runtime, std::move(o.getFunction(runtime)), jsInvoker);
|
||||
}
|
||||
if (o.isHostObject(runtime)) {
|
||||
auto hostObject = o.asHostObject(runtime);
|
||||
auto frame = dynamic_cast<FrameHostObject*>(hostObject.get());
|
||||
if (frame != nullptr) {
|
||||
return [[CMSampleBufferRefHolder alloc] initWithBuffer:frame->buffer];
|
||||
}
|
||||
}
|
||||
return convertJSIObjectToNSDictionary(runtime, o, jsInvoker);
|
||||
}
|
||||
|
||||
|
@ -81,7 +81,7 @@
|
||||
B80E069F266632F000728644 /* AVAudioSession+updateCategory.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVAudioSession+updateCategory.swift"; sourceTree = "<group>"; };
|
||||
B8103E1B25FF553B007A1684 /* FrameProcessorUtils.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = FrameProcessorUtils.mm; sourceTree = "<group>"; };
|
||||
B8103E1E25FF5550007A1684 /* FrameProcessorUtils.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorUtils.h; sourceTree = "<group>"; };
|
||||
B8103E5725FF56F0007A1684 /* Frame.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Frame.h; sourceTree = "<group>"; };
|
||||
B8103E5725FF56F0007A1684 /* CMSampleBufferRefHolder.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = CMSampleBufferRefHolder.h; sourceTree = "<group>"; };
|
||||
B81D41EF263C86F900B041FD /* JSIUtils.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = JSIUtils.h; sourceTree = "<group>"; };
|
||||
B82FBA942614B69D00909718 /* RCTBridge+runOnJS.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "RCTBridge+runOnJS.h"; sourceTree = "<group>"; };
|
||||
B82FBA952614B69D00909718 /* RCTBridge+runOnJS.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = "RCTBridge+runOnJS.mm"; sourceTree = "<group>"; };
|
||||
@ -140,6 +140,7 @@
|
||||
B8DB3BCB263DC97E004C18D7 /* AVFileType+descriptor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVFileType+descriptor.swift"; sourceTree = "<group>"; };
|
||||
B8DCF09125EA7BEE00EA5C72 /* SpeedChecker.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SpeedChecker.h; sourceTree = "<group>"; };
|
||||
B8DCF14425EA817D00EA5C72 /* MakeJSIRuntime.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MakeJSIRuntime.h; sourceTree = "<group>"; };
|
||||
B8F7DDD1266F715D00120533 /* CMSampleBufferRefHolder.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = CMSampleBufferRefHolder.m; sourceTree = "<group>"; };
|
||||
/* End PBXFileReference section */
|
||||
|
||||
/* Begin PBXFrameworksBuildPhase section */
|
||||
@ -262,7 +263,8 @@
|
||||
B80D67A825FA25380008FE8D /* FrameProcessorCallback.h */,
|
||||
B8103E1E25FF5550007A1684 /* FrameProcessorUtils.h */,
|
||||
B8103E1B25FF553B007A1684 /* FrameProcessorUtils.mm */,
|
||||
B8103E5725FF56F0007A1684 /* Frame.h */,
|
||||
B8103E5725FF56F0007A1684 /* CMSampleBufferRefHolder.h */,
|
||||
B8F7DDD1266F715D00120533 /* CMSampleBufferRefHolder.m */,
|
||||
B84760A22608EE38004C3180 /* FrameHostObject.h */,
|
||||
B84760A52608EE7C004C3180 /* FrameHostObject.mm */,
|
||||
B8A751D62609E4980011C623 /* FrameProcessorRuntimeManager.h */,
|
||||
|
Loading…
Reference in New Issue
Block a user