railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Frame Processors: Allow returning Frames (support for resize and other frame manipulations) (#185)

Browse Source 
* batch

* Init Frame as box

* Use ObjC syntax

* Fix access

* Revert "Fix access"

This reverts commit 7de09e52739d4c2b53f485d5ed696f1665fa5737.

* Revert "Use ObjC syntax"

This reverts commit e33f05ae8451cc4ee24af41d14dc76a57c157554.

* Revert "Init Frame as box"

This reverts commit 5adafb6109bfbf7fddb8ddc4af7d306b7b76b476.

* use holder

* convert buffer <-> jsi object

* add docs

* add more docs

* Update JSIUtils.mm

* Update FRAME_PROCESSORS_CREATE_OVERVIEW.mdx

* Update CameraView+RecordVideo.swift
This commit is contained in:
Marc Rousavy
2021-06-08 14:20:07 +02:00
committed by GitHub
parent 1b08c0cbae
commit 4038db2e28
10 changed files with 112 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
ios/CameraView+RecordVideo.swift
View File

@@ -203,8 +203,9 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud
if frameProcessorCallback != nil && !hasLoggedFrameDropWarning && captureOutput is AVCaptureVideoDataOutput {
let reason = findFrameDropReason(inBuffer: buffer)
ReactLogger.log(level: .warning,
message: "Dropped a Frame. This might indicate that your Frame Processor is doing too much work. " +
"Either throttle the frame processor's frame rate, or optimize your frame processor's execution speed. Frame drop reason: \(reason)",
message: "Dropped a Frame - This might indicate that your Frame Processor is doing too much work. " +
"Either throttle the frame processor's frame rate using the `frameProcessorFps` prop, or optimize " +
"your frame processor's execution speed. Frame drop reason: \(reason)",
alsoLogToJS: true)
hasLoggedFrameDropWarning = true
}

22
ios/Frame Processor/CMSampleBufferRefHolder.h Normal file
View File

@@ -0,0 +1,22 @@
//
// CMSampleBufferRefHolder.h
// VisionCamera
//
// Created by Marc Rousavy on 15.03.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
#pragma once
#import <Foundation/Foundation.h>
#import <CoreMedia/CMSampleBuffer.h>
@interface CMSampleBufferRefHolder : NSObject {
CMSampleBufferRef buffer;
}
- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer;
@property (nonatomic) CMSampleBufferRef buffer;
@end

25
ios/Frame Processor/CMSampleBufferRefHolder.m Normal file
View File

@@ -0,0 +1,25 @@
//
// CMSampleBufferRefHolder.m
// VisionCamera
//
// Created by Marc Rousavy on 08.06.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
#import "CMSampleBufferRefHolder.h"
#import <Foundation/Foundation.h>
#import <CoreMedia/CMSampleBuffer.h>
@implementation CMSampleBufferRefHolder
- (instancetype) initWithBuffer:(CMSampleBufferRef)buffer {
self = [super init];
if (self) {
self.buffer = buffer;
}
return self;
}
@synthesize buffer;
@end

20
ios/Frame Processor/Frame.h
View File

@@ -1,20 +0,0 @@
//
// Frame.h
// VisionCamera
//
// Created by Marc Rousavy on 15.03.21.
// Copyright © 2021 mrousavy. All rights reserved.
//
#pragma once
#import <CoreMedia/CMSampleBuffer.h>
// TODO: Make this Objective-C so it can be imported in Swift?
class Frame {
public:
explicit Frame(CMSampleBufferRef buffer): buffer(buffer) {}
public:
CMSampleBufferRef buffer;
};

8
ios/Frame Processor/FrameHostObject.h
View File

@@ -8,19 +8,21 @@
#pragma once
#import "Frame.h"
#import <jsi/jsi.h>
#import <CoreMedia/CMSampleBuffer.h>
using namespace facebook;
class JSI_EXPORT FrameHostObject: public Frame, public jsi::HostObject {
class JSI_EXPORT FrameHostObject: public jsi::HostObject {
public:
explicit FrameHostObject(CMSampleBufferRef buffer): Frame(buffer) {}
explicit FrameHostObject(CMSampleBufferRef buffer): buffer(buffer) {}
~FrameHostObject();
public:
jsi::Value get(jsi::Runtime&, const jsi::PropNameID& name) override;
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime& rt) override;
void destroyBuffer();
public:
CMSampleBufferRef buffer;
};

14
ios/React Utils/JSIUtils.mm
View File

@@ -12,6 +12,8 @@
#import <ReactCommon/CallInvoker.h>
#import <React/RCTBridge.h>
#import <ReactCommon/TurboModuleUtils.h>
#import "../Frame Processor/CMSampleBufferRefHolder.h"
#import "../Frame Processor/FrameHostObject.h"
using namespace facebook;
using namespace facebook::react;
@@ -66,6 +68,11 @@ jsi::Value convertObjCObjectToJSIValue(jsi::Runtime &runtime, id value)
return convertNSArrayToJSIArray(runtime, (NSArray *)value);
} else if (value == (id)kCFNull) {
return jsi::Value::null();
} else if ([value isKindOfClass:[CMSampleBufferRefHolder class]]) {
// it's boxed in a CMSampleBufferRefHolder because CMSampleBufferRef is not an NSObject
CMSampleBufferRef buffer = [(CMSampleBufferRefHolder*)value buffer];
auto frame = std::make_shared<FrameHostObject>(buffer);
return jsi::Object::createFromHostObject(runtime, frame);
}
return jsi::Value::undefined();
}
@@ -144,6 +151,13 @@ id convertJSIValueToObjCObject(jsi::Runtime &runtime, const jsi::Value &value, s
if (o.isFunction(runtime)) {
return convertJSIFunctionToCallback(runtime, std::move(o.getFunction(runtime)), jsInvoker);
}
if (o.isHostObject(runtime)) {
auto hostObject = o.asHostObject(runtime);
auto frame = dynamic_cast<FrameHostObject*>(hostObject.get());
if (frame != nullptr) {
return [[CMSampleBufferRefHolder alloc] initWithBuffer:frame->buffer];
}
}
return convertJSIObjectToNSDictionary(runtime, o, jsInvoker);
}

6
ios/VisionCamera.xcodeproj/project.pbxproj
View File

@@ -81,7 +81,7 @@
B80E069F266632F000728644 /* AVAudioSession+updateCategory.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVAudioSession+updateCategory.swift"; sourceTree = "<group>"; };
B8103E1B25FF553B007A1684 /* FrameProcessorUtils.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = FrameProcessorUtils.mm; sourceTree = "<group>"; };
B8103E1E25FF5550007A1684 /* FrameProcessorUtils.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorUtils.h; sourceTree = "<group>"; };
B8103E5725FF56F0007A1684 /* Frame.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Frame.h; sourceTree = "<group>"; };
B8103E5725FF56F0007A1684 /* CMSampleBufferRefHolder.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = CMSampleBufferRefHolder.h; sourceTree = "<group>"; };
B81D41EF263C86F900B041FD /* JSIUtils.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = JSIUtils.h; sourceTree = "<group>"; };
B82FBA942614B69D00909718 /* RCTBridge+runOnJS.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = "RCTBridge+runOnJS.h"; sourceTree = "<group>"; };
B82FBA952614B69D00909718 /* RCTBridge+runOnJS.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = "RCTBridge+runOnJS.mm"; sourceTree = "<group>"; };
@@ -140,6 +140,7 @@
B8DB3BCB263DC97E004C18D7 /* AVFileType+descriptor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVFileType+descriptor.swift"; sourceTree = "<group>"; };
B8DCF09125EA7BEE00EA5C72 /* SpeedChecker.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = SpeedChecker.h; sourceTree = "<group>"; };
B8DCF14425EA817D00EA5C72 /* MakeJSIRuntime.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = MakeJSIRuntime.h; sourceTree = "<group>"; };
B8F7DDD1266F715D00120533 /* CMSampleBufferRefHolder.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = CMSampleBufferRefHolder.m; sourceTree = "<group>"; };
/* End PBXFileReference section */
/* Begin PBXFrameworksBuildPhase section */
@@ -262,7 +263,8 @@
B80D67A825FA25380008FE8D /* FrameProcessorCallback.h */,
B8103E1E25FF5550007A1684 /* FrameProcessorUtils.h */,
B8103E1B25FF553B007A1684 /* FrameProcessorUtils.mm */,
B8103E5725FF56F0007A1684 /* Frame.h */,
B8103E5725FF56F0007A1684 /* CMSampleBufferRefHolder.h */,
B8F7DDD1266F715D00120533 /* CMSampleBufferRefHolder.m */,
B84760A22608EE38004C3180 /* FrameHostObject.h */,
B84760A52608EE7C004C3180 /* FrameHostObject.mm */,
B8A751D62609E4980011C623 /* FrameProcessorRuntimeManager.h */,