railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

feat: native Frame type to provide Orientation (#186)

Browse Source 
* Use Frame.h

* Add orientation

* Determine buffer orientation

* Replace plugins

* fix calls

* Update FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx

* Update FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx

* format

* Update CameraPage.tsx

* Update FRAME_PROCESSOR_CREATE_PLUGIN_IOS.mdx

* Add links to docs

* Use `.` syntax

* Make properties `readonly`

* Fix `@synthesize` backing store
This commit is contained in:
Marc Rousavy
2021-06-09 10:57:05 +02:00
committed by GitHub
parent 7025fc1cbe
commit 68a716b506
21 changed files with 179 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
ios/Frame Processor/CMSampleBufferRefHolder.h
View File

@@ -1,22 +0,0 @@
//
// CMSampleBufferRefHolder.h
// VisionCamera
//
// Created by Marc Rousavy on 15.03.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
#pragma once
#import <Foundation/Foundation.h>
#import <CoreMedia/CMSampleBuffer.h>
@interface CMSampleBufferRefHolder : NSObject {
CMSampleBufferRef buffer;
}
- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer;
@property (nonatomic) CMSampleBufferRef buffer;
@end

25
ios/Frame Processor/CMSampleBufferRefHolder.m
View File

@@ -1,25 +0,0 @@
//
// CMSampleBufferRefHolder.m
// VisionCamera
//
// Created by Marc Rousavy on 08.06.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
#import "CMSampleBufferRefHolder.h"
#import <Foundation/Foundation.h>
#import <CoreMedia/CMSampleBuffer.h>
@implementation CMSampleBufferRefHolder
- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer {
self = [super init];
if (self) {
self.buffer = buffer;
}
return self;
}
@synthesize buffer;
@end

22
ios/Frame Processor/Frame.h Normal file
View File

@@ -0,0 +1,22 @@
//
// Frame.h
// VisionCamera
//
// Created by Marc Rousavy on 15.03.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
#pragma once
#import <Foundation/Foundation.h>
#import <CoreMedia/CMSampleBuffer.h>
#import <UIKit/UIImage.h>
@interface Frame : NSObject
- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer orientation:(UIImageOrientation)orientation;
@property (nonatomic, readonly) CMSampleBufferRef buffer;
@property (nonatomic, readonly) UIImageOrientation orientation;
@end

30
ios/Frame Processor/Frame.m Normal file
View File

@@ -0,0 +1,30 @@
//
// Frame.m
// VisionCamera
//
// Created by Marc Rousavy on 08.06.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
#import "Frame.h"
#import <Foundation/Foundation.h>
#import <CoreMedia/CMSampleBuffer.h>
@implementation Frame {
CMSampleBufferRef buffer;
UIImageOrientation orientation;
}
- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer orientation:(UIImageOrientation)orientation {
self = [super init];
if (self) {
_buffer = buffer;
_orientation = orientation;
}
return self;
}
@synthesize buffer = _buffer;
@synthesize orientation = _orientation;
@end

5
ios/Frame Processor/FrameHostObject.h
View File

@@ -10,12 +10,13 @@
#import <jsi/jsi.h>
#import <CoreMedia/CMSampleBuffer.h>
#import "Frame.h"
using namespace facebook;
class JSI_EXPORT FrameHostObject: public jsi::HostObject {
public:
explicit FrameHostObject(CMSampleBufferRef buffer): buffer(buffer) {}
explicit FrameHostObject(Frame* frame): frame(frame) {}
~FrameHostObject();
public:
@@ -24,5 +25,5 @@ public:
void destroyBuffer();
public:
CMSampleBufferRef buffer;
Frame* frame;
};

16
ios/Frame Processor/FrameHostObject.mm
View File

@@ -37,7 +37,7 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
}
if (name == "toString") {
auto toString = [this] (jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
auto imageBuffer = CMSampleBufferGetImageBuffer(buffer);
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto width = CVPixelBufferGetWidth(imageBuffer);
auto height = CVPixelBufferGetHeight(imageBuffer);
@@ -48,30 +48,30 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
}
if (name == "isValid") {
auto isValid = buffer != nil && CMSampleBufferIsValid(buffer);
auto isValid = frame != nil && CMSampleBufferIsValid(frame.buffer);
return jsi::Value(isValid);
}
if (name == "isReady") {
auto isReady = buffer != nil && CMSampleBufferDataIsReady(buffer);
auto isReady = frame != nil && CMSampleBufferDataIsReady(frame.buffer);
return jsi::Value(isReady);
}
if (name == "width") {
auto imageBuffer = CMSampleBufferGetImageBuffer(buffer);
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto width = CVPixelBufferGetWidth(imageBuffer);
return jsi::Value((double) width);
}
if (name == "height") {
auto imageBuffer = CMSampleBufferGetImageBuffer(buffer);
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto height = CVPixelBufferGetHeight(imageBuffer);
return jsi::Value((double) height);
}
if (name == "bytesPerRow") {
auto imageBuffer = CMSampleBufferGetImageBuffer(buffer);
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto bytesPerRow = CVPixelBufferGetPlaneCount(imageBuffer);
return jsi::Value((double) bytesPerRow);
}
if (name == "planesCount") {
auto imageBuffer = CMSampleBufferGetImageBuffer(buffer);
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto planesCount = CVPixelBufferGetPlaneCount(imageBuffer);
return jsi::Value((double) planesCount);
}
@@ -85,5 +85,5 @@ FrameHostObject::~FrameHostObject() {
void FrameHostObject::destroyBuffer() {
// ARC will hopefully delete it lol
this->buffer = nil;
this->frame = nil;
}

4
ios/Frame Processor/FrameProcessorCallback.h
View File

@@ -9,6 +9,6 @@
#pragma once
#import <Foundation/Foundation.h>
#import <CoreMedia/CMSampleBuffer.h>
#import "Frame.h"
typedef void (^FrameProcessorCallback) (CMSampleBufferRef buffer);
typedef void (^FrameProcessorCallback) (Frame* frame);

14
ios/Frame Processor/FrameProcessorPlugin.h
View File

@@ -11,10 +11,10 @@
#import <Foundation/Foundation.h>
#import "FrameProcessorPluginRegistry.h"
#import <CoreMedia/CMSampleBuffer.h>
#import "Frame.h"
@protocol FrameProcessorPluginBase
+ (id) callback:(CMSampleBufferRef)buffer withArgs:(NSArray<id>*)args;
+ (id) callback:(Frame*)frame withArgs:(NSArray<id>*)args;
@end
@@ -23,7 +23,7 @@
/**
* Use this Macro to register the given function as a Frame Processor.
* * Make sure the given function is a C-style function with the following signature: static inline id callback(CMSampleBufferRef buffer)
* * Make sure the given function is a C-style function with the following signature: static inline id callback(Frame* frame, NSArray* args)
* * Make sure the given function's name is unique across other frame processor plugins
* * Make sure your frame processor returns a Value that can be converted to JS
* * Make sure to use this Macro in an @implementation, not @interface
@@ -35,8 +35,8 @@
\
+(void)load \
{ \
[FrameProcessorPluginRegistry addFrameProcessorPlugin:@"__" @ #frame_processor callback:^id(CMSampleBufferRef buffer, NSArray<id>* args) { \
return frame_processor(buffer, args); \
[FrameProcessorPluginRegistry addFrameProcessorPlugin:@"__" @ #frame_processor callback:^id(Frame* frame, NSArray<id>* args) { \
return frame_processor(frame, args); \
}]; \
}
@@ -55,8 +55,8 @@ objc_name : NSObject<FrameProcessorPluginBase>
\
__attribute__((constructor)) static void VISION_CONCAT(initialize_, objc_name)() \
{ \
[FrameProcessorPluginRegistry addFrameProcessorPlugin:@"__" @ #name callback:^id(CMSampleBufferRef buffer, NSArray<id>* args) { \
return [objc_name callback:buffer withArgs:args]; \
[FrameProcessorPluginRegistry addFrameProcessorPlugin:@"__" @ #name callback:^id(Frame* frame, NSArray<id>* args) { \
return [objc_name callback:frame withArgs:args]; \
}]; \
}

4
ios/Frame Processor/FrameProcessorPluginRegistry.h
View File

@@ -9,9 +9,9 @@
#pragma once
#import <Foundation/Foundation.h>
#import <CoreMedia/CMSampleBuffer.h>
#import "Frame.h"
typedef id (^FrameProcessorPlugin) (CMSampleBufferRef buffer, NSArray<id>* arguments);
typedef id (^FrameProcessorPlugin) (Frame* frame, NSArray<id>* arguments);
@interface FrameProcessorPluginRegistry : NSObject

21
ios/Frame Processor/FrameProcessorRuntimeManager.mm
View File

@@ -85,15 +85,22 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
NSLog(@"FrameProcessorBindings: Installing Frame Processor plugin \"%s\"...", pluginName);
FrameProcessorPlugin callback = [[FrameProcessorPluginRegistry frameProcessorPlugins] valueForKey:pluginKey];
auto function = [callback, callInvoker](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
auto function = [callback, callInvoker](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
size_t count) -> jsi::Value {
auto frameHostObject = arguments[0].asObject(runtime).asHostObject(runtime);
auto frame = static_cast<FrameHostObject*>(frameHostObject.get());
auto args = convertJSICStyleArrayToNSArray(runtime,
arguments + 1, // start at index 1 since first arg = Frame
count - 1, // use smaller count
callInvoker);
id result = callback(frame->buffer, args);
id result = callback(frame->frame, args);
return convertObjCObjectToJSIValue(runtime, result);
};
visionGlobal.setProperty(visionRuntime, pluginName, jsi::Function::createFromHostFunction(visionRuntime,
@@ -129,7 +136,10 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
NSLog(@"FrameProcessorBindings: Installing global functions...");
// setFrameProcessor(viewTag: number, frameProcessor: (frame: Frame) => void)
auto setFrameProcessor = [self](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
auto setFrameProcessor = [self](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
size_t count) -> jsi::Value {
NSLog(@"FrameProcessorBindings: Setting new frame processor...");
if (!arguments[0].isNumber()) throw jsi::JSError(runtime, "Camera::setFrameProcessor: First argument ('viewTag') must be a number!");
if (!arguments[1].isObject()) throw jsi::JSError(runtime, "Camera::setFrameProcessor: Second argument ('frameProcessor') must be a function!");
@@ -163,7 +173,10 @@ __attribute__((objc_runtime_name("_TtC12VisionCamera10CameraView")))
setFrameProcessor));
// unsetFrameProcessor(viewTag: number)
auto unsetFrameProcessor = [](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
auto unsetFrameProcessor = [](jsi::Runtime& runtime,
const jsi::Value& thisValue,
const jsi::Value* arguments,
size_t count) -> jsi::Value {
NSLog(@"FrameProcessorBindings: Removing frame processor...");
if (!arguments[0].isNumber()) throw jsi::JSError(runtime, "Camera::unsetFrameProcessor: First argument ('viewTag') must be a number!");
auto viewTag = arguments[0].asNumber();

10
ios/Frame Processor/FrameProcessorUtils.mm
View File

@@ -7,22 +7,22 @@
//
#import "FrameProcessorUtils.h"
#import <CoreMedia/CMSampleBuffer.h>
#import <chrono>
#import <memory>
#import "FrameHostObject.h"
#import "Frame.h"
FrameProcessorCallback convertJSIFunctionToFrameProcessorCallback(jsi::Runtime &runtime, const jsi::Function &value) {
__block auto cb = value.getFunction(runtime);
return ^(CMSampleBufferRef buffer) {
return ^(Frame* frame) {
#if DEBUG
std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now();
#endif
auto frame = std::make_shared<FrameHostObject>(buffer);
auto frameHostObject = std::make_shared<FrameHostObject>(frame);
try {
cb.call(runtime, jsi::Object::createFromHostObject(runtime, frame));
cb.call(runtime, jsi::Object::createFromHostObject(runtime, frameHostObject));
} catch (jsi::JSError& jsError) {
NSLog(@"Frame Processor threw an error: %s", jsError.getMessage().c_str());
}
@@ -39,6 +39,6 @@ FrameProcessorCallback convertJSIFunctionToFrameProcessorCallback(jsi::Runtime &
// 1. we are sure we don't need it anymore, the frame processor worklet has finished executing.
// 2. we don't know when the JS runtime garbage collects this object, it might be holding it for a few more frames
// which then blocks the camera queue from pushing new frames (memory limit)
frame->destroyBuffer();
frameHostObject->destroyBuffer();
};
}