railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: native Frame type to provide Orientation (#186)

Browse Source 
* Use Frame.h

* Add orientation

* Determine buffer orientation

* Replace plugins

* fix calls

* Update FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx

* Update FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx

* format

* Update CameraPage.tsx

* Update FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx

* Add links to docs

* Use `.` syntax

* Make properties `readonly`

* Fix `@synthesize` backing store
This commit is contained in:
Marc Rousavy
2021-06-09 10:57:05 +02:00
committed by GitHub
parent 7025fc1cbe
commit 68a716b506
21 changed files with 179 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
ios/CameraBridge.h
View File

@@ -15,6 +15,7 @@
#import "FrameProcessorCallback.h"
#import "FrameProcessorRuntimeManager.h"
#import "Frame.h"
#import "RCTBridge+runOnJS.h"
#import "JSConsoleHelper.h"

34
ios/CameraView+RecordVideo.swift
View File

@@ -169,10 +169,12 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud
}
public final func captureOutput(_ captureOutput: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from _: AVCaptureConnection) {
// Video Recording runs in the same queue
if isRecording {
guard let recordingSession = recordingSession else {
return invokeOnError(.capture(.unknown(message: "isRecording was true but the RecordingSession was null!")))
}
switch captureOutput {
case is AVCaptureVideoDataOutput:
recordingSession.appendBuffer(sampleBuffer, type: .video, timestamp: CMSampleBufferGetPresentationTimeStamp(sampleBuffer))
@@ -191,8 +193,10 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud
let diff = DispatchTime.now().uptimeNanoseconds - lastFrameProcessorCall.uptimeNanoseconds
let secondsPerFrame = 1.0 / frameProcessorFps.doubleValue
let nanosecondsPerFrame = secondsPerFrame * 1_000_000_000.0
if diff > UInt64(nanosecondsPerFrame) {
frameProcessor(sampleBuffer)
let frame = Frame(buffer: sampleBuffer, orientation: bufferOrientation)
frameProcessor(frame)
lastFrameProcessorCall = DispatchTime.now()
}
}
@@ -221,4 +225,32 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud
return String(describing: reason)
}
#endif
/**
Gets the orientation of the CameraView's images (CMSampleBuffers).
*/
var bufferOrientation: UIImage.Orientation {
guard let cameraPosition = videoDeviceInput?.device.position else {
return .up
}
switch UIDevice.current.orientation {
case .portrait:
return cameraPosition == .front ? .leftMirrored : .right
case .landscapeLeft:
return cameraPosition == .front ? .downMirrored : .up
case .portraitUpsideDown:
return cameraPosition == .front ? .rightMirrored : .left
case .landscapeRight:
return cameraPosition == .front ? .upMirrored : .down
case .unknown, .faceUp, .faceDown:
fallthrough
@unknown default:
return .up
}
}
}

22
ios/Frame Processor/CMSampleBufferRefHolder.h
View File

@@ -1,22 +0,0 @@
//
// CMSampleBufferRefHolder.h
// VisionCamera
//
// Created by Marc Rousavy on 15.03.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
#pragma once
#import <Foundation/Foundation.h>
#import <CoreMedia/CMSampleBuffer.h>
@interface CMSampleBufferRefHolder : NSObject {
CMSampleBufferRef buffer;
}
- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer;
@property (nonatomic) CMSampleBufferRef buffer;
@end

25
ios/Frame Processor/CMSampleBufferRefHolder.m
View File

@@ -1,25 +0,0 @@
//
// CMSampleBufferRefHolder.m
// VisionCamera
//
// Created by Marc Rousavy on 08.06.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
#import "CMSampleBufferRefHolder.h"
#import <Foundation/Foundation.h>
#import <CoreMedia/CMSampleBuffer.h>
@implementation CMSampleBufferRefHolder
- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer {
self = [super init];
if (self) {
self.buffer = buffer;
}
return self;
}
@synthesize buffer;
@end

22
ios/Frame Processor/Frame.h Normal file
View File

@@ -0,0 +1,22 @@
//
// Frame.h
// VisionCamera
//
// Created by Marc Rousavy on 15.03.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
#pragma once
#import <Foundation/Foundation.h>
#import <CoreMedia/CMSampleBuffer.h>
#import <UIKit/UIImage.h>
@interface Frame : NSObject
- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer orientation:(UIImageOrientation)orientation;
@property (nonatomic, readonly) CMSampleBufferRef buffer;
@property (nonatomic, readonly) UIImageOrientation orientation;
@end

30
ios/Frame Processor/Frame.m Normal file
View File

@@ -0,0 +1,30 @@
//
// Frame.m
// VisionCamera
//
// Created by Marc Rousavy on 08.06.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
#import "Frame.h"
#import <Foundation/Foundation.h>
#import <CoreMedia/CMSampleBuffer.h>
@implementation Frame {
CMSampleBufferRef buffer;
UIImageOrientation orientation;
}
- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer orientation:(UIImageOrientation)orientation {
self = [super init];
if (self) {
_buffer = buffer;
_orientation = orientation;
}
return self;
}
@synthesize buffer = _buffer;
@synthesize orientation = _orientation;
@end

5
ios/Frame Processor/FrameHostObject.h
View File

@@ -10,12 +10,13 @@
#import <jsi/jsi.h>
#import <CoreMedia/CMSampleBuffer.h>
#import "Frame.h"
using namespace facebook;
class JSI_EXPORT FrameHostObject: public jsi::HostObject {
public:
explicit FrameHostObject(CMSampleBufferRef buffer): buffer(buffer) {}
explicit FrameHostObject(Frame* frame): frame(frame) {}
~FrameHostObject();
public:
@@ -24,5 +25,5 @@ public:
void destroyBuffer();
public:
CMSampleBufferRef buffer;
Frame* frame;
};

16
ios/Frame Processor/FrameHostObject.mm
View File

@@ -37,7 +37,7 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
}
if (name == "toString") {
auto toString = [this] (jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
auto imageBuffer = CMSampleBufferGetImageBuffer(buffer);
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto width = CVPixelBufferGetWidth(imageBuffer);
auto height = CVPixelBufferGetHeight(imageBuffer);
@@ -48,30 +48,30 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
}
if (name == "isValid") {
auto isValid = buffer != nil && CMSampleBufferIsValid(buffer);
auto isValid = frame != nil && CMSampleBufferIsValid(frame.buffer);
return jsi::Value(isValid);
}
if (name == "isReady") {
auto isReady = buffer != nil && CMSampleBufferDataIsReady(buffer);
auto isReady = frame != nil && CMSampleBufferDataIsReady(frame.buffer);
return jsi::Value(isReady);
}
if (name == "width") {
auto imageBuffer = CMSampleBufferGetImageBuffer(buffer);
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto width = CVPixelBufferGetWidth(imageBuffer);
return jsi::Value((double) width);
}
if (name == "height") {
auto imageBuffer = CMSampleBufferGetImageBuffer(buffer);
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto height = CVPixelBufferGetHeight(imageBuffer);
return jsi::Value((double) height);
}
if (name == "bytesPerRow") {
auto imageBuffer = CMSampleBufferGetImageBuffer(buffer);
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto bytesPerRow = CVPixelBufferGetPlaneCount(imageBuffer);
return jsi::Value((double) bytesPerRow);
}
if (name == "planesCount") {
auto imageBuffer = CMSampleBufferGetImageBuffer(buffer);
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto planesCount = CVPixelBufferGetPlaneCount(imageBuffer);
return jsi::Value((double) planesCount);
}
@@ -85,5 +85,5 @@ FrameHostObject::~FrameHostObject() {
void FrameHostObject::destroyBuffer() {
// ARC will hopefully delete it lol
this->buffer = nil;
this->frame = nil;
}

4
ios/Frame Processor/FrameProcessorCallback.h
View File

@@ -9,6 +9,6 @@
#pragma once
#import <Foundation/Foundation.h>
#import <CoreMedia/CMSampleBuffer.h>
#import "Frame.h"
typedef void (^FrameProcessorCallback) (CMSampleBufferRef buffer);
typedef void (^FrameProcessorCallback) (Frame* frame);

14
ios/Frame Processor/FrameProcessorPlugin.h
View File

@@ -11,10 +11,10 @@
#import <Foundation/Foundation.h>
#import "FrameProcessorPluginRegistry.h"
#import <CoreMedia/CMSampleBuffer.h>
#import "Frame.h"
@protocol FrameProcessorPluginBase
+ (id) callback:(CMSampleBufferRef)buffer withArgs:(NSArray<id>*)args;
+ (id) callback:(Frame*)frame withArgs:(NSArray<id>*)args;
@end
@@ -23,7 +23,7 @@
/**
* Use this Macro to register the given function as a Frame Processor.
* * Make sure the given function is a C-style function with the following signature: static inline id callback(CMSampleBufferRef buffer)
* * Make sure the given function is a C-style function with the following signature: static inline id callback(Frame* frame, NSArray* args)
* * Make sure the given function's name is unique across other frame processor plugins
* * Make sure your frame processor returns a Value that can be converted to JS
* * Make sure to use this Macro in an @implementation, not @interface
@@ -35,8 +35,8 @@
\
+(void)load \
{ \
[FrameProcessorPluginRegistry addFrameProcessorPlugin:@"__" @ #frame_processor callback:^id(CMSampleBufferRef buffer, NSArray<id>* args) { \
return frame_processor(buffer, args); \
[FrameProcessorPluginRegistry addFrameProcessorPlugin:@"__" @ #frame_processor callback:^id(Frame* frame, NSArray<id>* args) { \
return frame_processor(frame, args); \
}]; \
}
@@ -55,8 +55,8 @@ objc_name : NSObject<FrameProcessorPluginBase>
\
__attribute__((constructor)) static void VISION_CONCAT(initialize_, objc_name)() \
{ \
[FrameProcessorPluginRegistry addFrameProcessorPlugin:@"__" @ #name callback:^id(CMSampleBufferRef buffer, NSArray<id>* args) { \
return [objc_name callback:buffer withArgs:args]; \
[FrameProcessorPluginRegistry addFrameProcessorPlugin:@"__" @ #name callback:^id(Frame* frame, NSArray<id>* args) { \
return [objc_name callback:frame withArgs:args]; \
}]; \
}

4
ios/Frame Processor/FrameProcessorPluginRegistry.h
View File

@@ -9,9 +9,9 @@
#pragma once
#import <Foundation/Foundation.h>
#import <CoreMedia/CMSampleBuffer.h>
#import "Frame.h"
typedef id (^FrameProcessorPlugin) (CMSampleBufferRef buffer, NSArray<id>* arguments);
typedef id (^FrameProcessorPlugin) (Frame* frame, NSArray<id>* arguments);
@interface FrameProcessorPluginRegistry : NSObject

21
ios/Frame Processor/FrameProcessorRuntimeManager.mm
View File

@@ -85,15 +85,22 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
NSLog(@"FrameProcessorBindings: Installing Frame Processor plugin \"%s\"...", pluginName);
FrameProcessorPlugin callback = [[FrameProcessorPluginRegistry frameProcessorPlugins] valueForKey:pluginKey];
auto function = [callback, callInvoker](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
auto function = [callback, callInvoker](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
size_t count) -> jsi::Value {
auto frameHostObject = arguments[0].asObject(runtime).asHostObject(runtime);
auto frame = static_cast<FrameHostObject*>(frameHostObject.get());
auto args = convertJSICStyleArrayToNSArray(runtime,
arguments + 1, // start at index 1 since first arg = Frame
count - 1, // use smaller count
callInvoker);
id result = callback(frame->buffer, args);
id result = callback(frame->frame, args);
return convertObjCObjectToJSIValue(runtime, result);
};
visionGlobal.setProperty(visionRuntime, pluginName, jsi::Function::createFromHostFunction(visionRuntime,
@@ -129,7 +136,10 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
NSLog(@"FrameProcessorBindings: Installing global functions...");
// setFrameProcessor(viewTag: number, frameProcessor: (frame: Frame) => void)
auto setFrameProcessor = [self](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
auto setFrameProcessor = [self](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
size_t count) -> jsi::Value {
NSLog(@"FrameProcessorBindings: Setting new frame processor...");
if (!arguments[0].isNumber()) throw jsi::JSError(runtime, "Camera::setFrameProcessor: First argument ('viewTag') must be a number!");
if (!arguments[1].isObject()) throw jsi::JSError(runtime, "Camera::setFrameProcessor: Second argument ('frameProcessor') must be a function!");
@@ -163,7 +173,10 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
setFrameProcessor));
// unsetFrameProcessor(viewTag: number)
auto unsetFrameProcessor = [](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
auto unsetFrameProcessor = [](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
size_t count) -> jsi::Value {
NSLog(@"FrameProcessorBindings: Removing frame processor...");
if (!arguments[0].isNumber()) throw jsi::JSError(runtime, "Camera::unsetFrameProcessor: First argument ('viewTag') must be a number!");
auto viewTag = arguments[0].asNumber();

10
ios/Frame Processor/FrameProcessorUtils.mm
View File

@@ -7,22 +7,22 @@
//
#import "FrameProcessorUtils.h"
#import <CoreMedia/CMSampleBuffer.h>
#import <chrono>
#import <memory>
#import "FrameHostObject.h"
#import "Frame.h"
FrameProcessorCallback convertJSIFunctionToFrameProcessorCallback(jsi::Runtime &runtime, const jsi::Function &value) {
__block auto cb = value.getFunction(runtime);
return ^(CMSampleBufferRef buffer) {
return ^(Frame* frame) {
#if DEBUG
std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now();
#endif
auto frame = std::make_shared<FrameHostObject>(buffer);
auto frameHostObject = std::make_shared<FrameHostObject>(frame);
try {
cb.call(runtime, jsi::Object::createFromHostObject(runtime, frame));
cb.call(runtime, jsi::Object::createFromHostObject(runtime, frameHostObject));
} catch (jsi::JSError& jsError) {
NSLog(@"Frame Processor threw an error: %s", jsError.getMessage().c_str());
}
@@ -39,6 +39,6 @@ FrameProcessorCallback convertJSIFunctionToFrameProcessorCallback(jsi::Runtime &
// 1. we are sure we don't need it anymore, the frame processor worklet has finished executing.
// 2. we don't know when the JS runtime garbage collects this object, it might be holding it for a few more frames
// which then blocks the camera queue from pushing new frames (memory limit)
frame->destroyBuffer();
frameHostObject->destroyBuffer();
};
}

12
ios/React Utils/JSIUtils.mm
View File

@@ -12,7 +12,7 @@
#import <ReactCommon/CallInvoker.h>
#import <React/RCTBridge.h>
#import <ReactCommon/TurboModuleUtils.h>
#import "../Frame Processor/CMSampleBufferRefHolder.h"
#import "../Frame Processor/Frame.h"
#import "../Frame Processor/FrameHostObject.h"
using namespace facebook;
@@ -68,11 +68,9 @@ jsi::Value convertObjCObjectToJSIValue(jsi::Runtime &runtime, id value)
return convertNSArrayToJSIArray(runtime, (NSArray *)value);
} else if (value == (id)kCFNull) {
return jsi::Value::null();
} else if ([value isKindOfClass:[CMSampleBufferRefHolder class]]) {
// it's boxed in a CMSampleBufferRefHolder because CMSampleBufferRef is not an NSObject
CMSampleBufferRef buffer = [(CMSampleBufferRefHolder*)value buffer];
auto frame = std::make_shared<FrameHostObject>(buffer);
return jsi::Object::createFromHostObject(runtime, frame);
} else if ([value isKindOfClass:[Frame class]]) {
auto frameHostObject = std::make_shared<FrameHostObject>((Frame*)value);
return jsi::Object::createFromHostObject(runtime, frameHostObject);
}
return jsi::Value::undefined();
}
@@ -155,7 +153,7 @@ id convertJSIValueToObjCObject(jsi::Runtime &runtime, const jsi::Value &value, s
auto hostObject = o.asHostObject(runtime);
auto frame = dynamic_cast<FrameHostObject*>(hostObject.get());
if (frame != nullptr) {
return [[CMSampleBufferRefHolder alloc] initWithBuffer:frame->buffer];
return frame->frame;
}
}
return convertJSIObjectToNSDictionary(runtime, o, jsInvoker);

8
ios/VisionCamera.xcodeproj/project.pbxproj
View File

@@ -81,7 +81,7 @@
B80E069F266632F000728644 /* AVAudioSession+updateCategory.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVAudioSession+updateCategory.swift"; sourceTree = "<group>"; };
B8103E1B25FF553B007A1684 /* FrameProcessorUtils.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = FrameProcessorUtils.mm; sourceTree = "<group>"; };
B8103E1E25FF5550007A1684 /* FrameProcessorUtils.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorUtils.h; sourceTree = "<group>"; };
B8103E5725FF56F0007A1684 /* CMSampleBufferRefHolder.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = CMSampleBufferRefHolder.h; sourceTree = "<group>"; };
B8103E5725FF56F0007A1684 /* Frame.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Frame.h; sourceTree = "<group>"; };
B81D41EF263C86F900B041FD /* JSIUtils.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = JSIUtils.h; sourceTree = "<group>"; };
B82FBA942614B69D00909718 /* RCTBridge+runOnJS.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "RCTBridge+runOnJS.h"; sourceTree = "<group>"; };
B82FBA952614B69D00909718 /* RCTBridge+runOnJS.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = "RCTBridge+runOnJS.mm"; sourceTree = "<group>"; };
@@ -140,7 +140,7 @@
B8DB3BCB263DC97E004C18D7 /* AVFileType+descriptor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVFileType+descriptor.swift"; sourceTree = "<group>"; };
B8DCF09125EA7BEE00EA5C72 /* SpeedChecker.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SpeedChecker.h; sourceTree = "<group>"; };
B8DCF14425EA817D00EA5C72 /* MakeJSIRuntime.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MakeJSIRuntime.h; sourceTree = "<group>"; };
B8F7DDD1266F715D00120533 /* CMSampleBufferRefHolder.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = CMSampleBufferRefHolder.m; sourceTree = "<group>"; };
B8F7DDD1266F715D00120533 /* Frame.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = Frame.m; sourceTree = "<group>"; };
/* End PBXFileReference section */
/* Begin PBXFrameworksBuildPhase section */
@@ -263,8 +263,8 @@
B80D67A825FA25380008FE8D /* FrameProcessorCallback.h */,
B8103E1E25FF5550007A1684 /* FrameProcessorUtils.h */,
B8103E1B25FF553B007A1684 /* FrameProcessorUtils.mm */,
B8103E5725FF56F0007A1684 /* CMSampleBufferRefHolder.h */,
B8F7DDD1266F715D00120533 /* CMSampleBufferRefHolder.m */,
B8103E5725FF56F0007A1684 /* Frame.h */,
B8F7DDD1266F715D00120533 /* Frame.m */,
B84760A22608EE38004C3180 /* FrameHostObject.h */,
B84760A52608EE7C004C3180 /* FrameHostObject.mm */,
B8A751D62609E4980011C623 /* FrameProcessorRuntimeManager.h */,