railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Make Frame thread-safe and improve error messages (#2327)

Browse Source 
* fix: Fix multi-Thread access on Java

* fix: Thread-lock access on iOS as well

* whoops add missing header impl

* Update Podfile.lock

* fix: Don't use `CFGetRetainCount`

* fix: Lock access on iOS as well

* C++ format

* More detailed error

* chore: Move getters into `Frame`

* Format c++

* Use enum `orientation` again

* format

* fix: Synchronize `isValid` on Java

* Also log pixelformat

* feat: Use Java enums in C++

* Format C++
This commit is contained in:
Marc Rousavy
2023-12-29 14:09:56 +01:00
committed by GitHub
parent e4393cd83a
commit 895f3ec889
14 changed files with 338 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
package/ios/Frame Processor/Frame.h
View File

@@ -19,4 +19,14 @@
@property(nonatomic, readonly) CMSampleBufferRef _Nonnull buffer;
@property(nonatomic, readonly) UIImageOrientation orientation;
// Getters
- (NSString* _Nonnull)pixelFormat;
- (BOOL)isMirrored;
- (BOOL)isValid;
- (size_t)width;
- (size_t)height;
- (double)timestamp;
- (size_t)bytesPerRow;
- (size_t)planesCount;
@end

69
package/ios/Frame Processor/Frame.m
View File

@@ -20,11 +20,80 @@
if (self) {
_buffer = buffer;
_orientation = orientation;
CFRetain(buffer);
}
return self;
}
- (void)dealloc {
CFRelease(_buffer);
}
@synthesize buffer = _buffer;
@synthesize orientation = _orientation;
- (NSString*)pixelFormat {
CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(_buffer);
FourCharCode mediaType = CMFormatDescriptionGetMediaSubType(format);
switch (mediaType) {
case kCVPixelFormatType_32BGRA:
case kCVPixelFormatType_Lossy_32BGRA:
return @"rgb";
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange:
case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange:
case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarFullRange:
case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarVideoRange:
case kCVPixelFormatType_Lossy_420YpCbCr10PackedBiPlanarVideoRange:
return @"yuv";
default:
return @"unknown";
}
}
- (BOOL)isMirrored {
switch (_orientation) {
case UIImageOrientationUp:
case UIImageOrientationDown:
case UIImageOrientationLeft:
case UIImageOrientationRight:
return false;
case UIImageOrientationDownMirrored:
case UIImageOrientationUpMirrored:
case UIImageOrientationLeftMirrored:
case UIImageOrientationRightMirrored:
return true;
}
}
- (BOOL)isValid {
return _buffer != nil && CMSampleBufferIsValid(_buffer);
}
- (size_t)width {
CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_buffer);
return CVPixelBufferGetWidth(imageBuffer);
}
- (size_t)height {
CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_buffer);
return CVPixelBufferGetHeight(imageBuffer);
}
- (double)timestamp {
CMTime timestamp = CMSampleBufferGetPresentationTimeStamp(_buffer);
return CMTimeGetSeconds(timestamp) * 1000.0;
}
- (size_t)bytesPerRow {
CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_buffer);
return CVPixelBufferGetBytesPerRow(imageBuffer);
}
- (size_t)planesCount {
CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_buffer);
return CVPixelBufferGetPlaneCount(imageBuffer);
}
@end

8
package/ios/Frame Processor/FrameHostObject.h
View File

@@ -10,6 +10,7 @@
#import <CoreMedia/CMSampleBuffer.h>
#import <jsi/jsi.h>
#import <mutex>
#import "Frame.h"
@@ -25,4 +26,11 @@ public:
public:
Frame* frame;
private:
Frame* getFrame();
private:
std::mutex _mutex;
size_t _refCount = 0;
};

168
package/ios/Frame Processor/FrameHostObject.mm
View File

@@ -7,6 +7,7 @@
//
#import "FrameHostObject.h"
#import "UIImageOrientation+descriptor.h"
#import "WKTJsiHostObject.h"
#import <Foundation/Foundation.h>
#import <jsi/jsi.h>
@@ -34,42 +35,70 @@ std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt)
return result;
}
Frame* FrameHostObject::getFrame() {
Frame* frame = this->frame;
if (frame == nil || !CMSampleBufferIsValid(frame.buffer)) {
throw std::runtime_error("Frame is already closed! "
"Are you trying to access the Image data outside of a Frame Processor's lifetime?\n"
"- If you want to use `console.log(frame)`, use `console.log(frame.toString())` instead.\n"
"- If you want to do async processing, use `runAsync(...)` instead.\n"
"- If you want to use runOnJS, increment it's ref-count: `frame.incrementRefCount()`");
}
return frame;
}
jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
auto name = propName.utf8(runtime);
if (name == "toString") {
auto toString = JSI_HOST_FUNCTION_LAMBDA {
if (this->frame == nil) {
return jsi::String::createFromUtf8(runtime, "[closed frame]");
}
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto width = CVPixelBufferGetWidth(imageBuffer);
auto height = CVPixelBufferGetHeight(imageBuffer);
auto toString = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
// Lock Frame so it cannot be deallocated while we access it
std::lock_guard lock(this->_mutex);
NSMutableString* string = [NSMutableString stringWithFormat:@"%lu x %lu Frame", width, height];
// Print debug description (width, height)
Frame* frame = this->getFrame();
NSMutableString* string = [NSMutableString stringWithFormat:@"%lu x %lu %@ Frame", frame.width, frame.height, frame.pixelFormat];
return jsi::String::createFromUtf8(runtime, string.UTF8String);
};
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
}
if (name == "incrementRefCount") {
auto incrementRefCount = JSI_HOST_FUNCTION_LAMBDA {
// Increment retain count by one so ARC doesn't destroy the Frame Buffer.
CFRetain(frame.buffer);
auto incrementRefCount = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
size_t count) -> jsi::Value {
// Lock Frame so it cannot be deallocated while we access it
std::lock_guard lock(this->_mutex);
// Increment our self-counted ref count by one.
_refCount++;
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "incrementRefCount"), 0, incrementRefCount);
}
if (name == "decrementRefCount") {
auto decrementRefCount = JSI_HOST_FUNCTION_LAMBDA {
// Decrement retain count by one. If the retain count is zero, ARC will destroy the Frame
// Buffer.
CFRelease(frame.buffer);
auto decrementRefCount = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
size_t count) -> jsi::Value {
// Lock Frame so it cannot be deallocated while we access it
std::lock_guard lock(this->_mutex);
// Decrement our self-counted ref count by one.
_refCount--;
if (_refCount < 1) {
// ARC will then delete the Frame and the underlying Frame Buffer.
this->frame = nil;
}
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "decrementRefCount"), 0, decrementRefCount);
}
if (name == "toArrayBuffer") {
auto toArrayBuffer = JSI_HOST_FUNCTION_LAMBDA {
auto toArrayBuffer = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
size_t count) -> jsi::Value {
// Lock Frame so it cannot be deallocated while we access it
std::lock_guard lock(this->_mutex);
// Get CPU readable Pixel Buffer from Frame and write it to a jsi::ArrayBuffer
Frame* frame = this->getFrame();
auto pixelBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer);
auto height = CVPixelBufferGetHeight(pixelBuffer);
@@ -101,82 +130,69 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
}
if (name == "isValid") {
auto isValid = frame != nil && frame.buffer != nil && CFGetRetainCount(frame.buffer) > 0 && CMSampleBufferIsValid(frame.buffer);
return jsi::Value(isValid);
// Lock Frame so it cannot be deallocated while we access it
std::lock_guard lock(this->_mutex);
// unsafely access the Frame and try to see if it's valid
Frame* frame = this->frame;
return jsi::Value(frame != nil && frame.isValid);
}
if (name == "width") {
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto width = CVPixelBufferGetWidth(imageBuffer);
return jsi::Value((double)width);
// Lock Frame so it cannot be deallocated while we access it
std::lock_guard lock(this->_mutex);
Frame* frame = this->getFrame();
return jsi::Value((double)frame.width);
}
if (name == "height") {
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto height = CVPixelBufferGetHeight(imageBuffer);
return jsi::Value((double)height);
// Lock Frame so it cannot be deallocated while we access it
std::lock_guard lock(this->_mutex);
Frame* frame = this->getFrame();
return jsi::Value((double)frame.height);
}
if (name == "orientation") {
switch (frame.orientation) {
case UIImageOrientationUp:
case UIImageOrientationUpMirrored:
return jsi::String::createFromUtf8(runtime, "portrait");
case UIImageOrientationDown:
case UIImageOrientationDownMirrored:
return jsi::String::createFromUtf8(runtime, "portrait-upside-down");
case UIImageOrientationLeft:
case UIImageOrientationLeftMirrored:
return jsi::String::createFromUtf8(runtime, "landscape-left");
case UIImageOrientationRight:
case UIImageOrientationRightMirrored:
return jsi::String::createFromUtf8(runtime, "landscape-right");
}
// Lock Frame so it cannot be deallocated while we access it
std::lock_guard lock(this->_mutex);
Frame* frame = this->getFrame();
NSString* orientation = [NSString stringWithParsed:frame.orientation];
return jsi::String::createFromUtf8(runtime, orientation.UTF8String);
}
if (name == "isMirrored") {
switch (frame.orientation) {
case UIImageOrientationUp:
case UIImageOrientationDown:
case UIImageOrientationLeft:
case UIImageOrientationRight:
return jsi::Value(false);
case UIImageOrientationDownMirrored:
case UIImageOrientationUpMirrored:
case UIImageOrientationLeftMirrored:
case UIImageOrientationRightMirrored:
return jsi::Value(true);
}
// Lock Frame so it cannot be deallocated while we access it
std::lock_guard lock(this->_mutex);
Frame* frame = this->getFrame();
return jsi::Value(frame.isMirrored);
}
if (name == "timestamp") {
auto timestamp = CMSampleBufferGetPresentationTimeStamp(frame.buffer);
auto seconds = static_cast<double>(CMTimeGetSeconds(timestamp));
return jsi::Value(seconds * 1000.0);
// Lock Frame so it cannot be deallocated while we access it
std::lock_guard lock(this->_mutex);
Frame* frame = this->getFrame();
return jsi::Value(frame.timestamp);
}
if (name == "pixelFormat") {
auto format = CMSampleBufferGetFormatDescription(frame.buffer);
auto mediaType = CMFormatDescriptionGetMediaSubType(format);
switch (mediaType) {
case kCVPixelFormatType_32BGRA:
case kCVPixelFormatType_Lossy_32BGRA:
return jsi::String::createFromUtf8(runtime, "rgb");
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange:
case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange:
case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarFullRange:
case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarVideoRange:
case kCVPixelFormatType_Lossy_420YpCbCr10PackedBiPlanarVideoRange:
return jsi::String::createFromUtf8(runtime, "yuv");
default:
return jsi::String::createFromUtf8(runtime, "unknown");
}
// Lock Frame so it cannot be deallocated while we access it
std::lock_guard lock(this->_mutex);
Frame* frame = this->getFrame();
return jsi::String::createFromUtf8(runtime, frame.pixelFormat.UTF8String);
}
if (name == "bytesPerRow") {
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer);
return jsi::Value((double)bytesPerRow);
// Lock Frame so it cannot be deallocated while we access it
std::lock_guard lock(this->_mutex);
Frame* frame = this->getFrame();
return jsi::Value((double)frame.bytesPerRow);
}
if (name == "planesCount") {
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto planesCount = CVPixelBufferGetPlaneCount(imageBuffer);
return jsi::Value((double)planesCount);
// Lock Frame so it cannot be deallocated while we access it
std::lock_guard lock(this->_mutex);
Frame* frame = this->getFrame();
return jsi::Value((double)frame.planesCount);
}
// fallback to base implementation

39
package/ios/Frame Processor/UIImageOrientation+descriptor.h Normal file
View File

@@ -0,0 +1,39 @@
//
// UIImageOrientation+descriptor.h
// VisionCamera
//
// Created by Marc Rousavy on 29.12.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
#pragma once
#import <Foundation/Foundation.h>
#import <UIKit/UIImage.h>
@interface NSString (UIImageOrientationJSDescriptor)
+ (NSString*)stringWithParsed:(UIImageOrientation)orientation;
@end
@implementation NSString (UIImageOrientationJSDescriptor)
+ (NSString*)stringWithParsed:(UIImageOrientation)orientation {
switch (orientation) {
case UIImageOrientationUp:
case UIImageOrientationUpMirrored:
return @"portrait";
case UIImageOrientationDown:
case UIImageOrientationDownMirrored:
return @"portrait-upside-down";
case UIImageOrientationLeft:
case UIImageOrientationLeftMirrored:
return @"landscape-left";
case UIImageOrientationRight:
case UIImageOrientationRightMirrored:
return @"landscape-right";
}
}
@end

2
package/ios/VisionCamera.xcodeproj/project.pbxproj
View File

@@ -160,6 +160,7 @@
B887518425E0102000DB86D6 /* CameraView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CameraView.swift; sourceTree = "<group>"; };
B88873E5263D46C7008B1D0E /* FrameProcessorPlugin.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorPlugin.h; sourceTree = "<group>"; };
B8994E6B263F03E100069589 /* JSINSObjectConversion.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = JSINSObjectConversion.mm; sourceTree = "<group>"; };
B89A79692B3EF60F005E0357 /* UIImageOrientation+descriptor.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "UIImageOrientation+descriptor.h"; sourceTree = "<group>"; };
B8A1AEC32AD7EDE800169C0D /* AVCaptureVideoDataOutput+pixelFormat.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVCaptureVideoDataOutput+pixelFormat.swift"; sourceTree = "<group>"; };
B8A1AEC52AD7F08E00169C0D /* CameraView+Focus.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CameraView+Focus.swift"; sourceTree = "<group>"; };
B8A1AEC72AD8005400169C0D /* CameraSession+Configuration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CameraSession+Configuration.swift"; sourceTree = "<group>"; };
@@ -334,6 +335,7 @@
B81D41EF263C86F900B041FD /* JSINSObjectConversion.h */,
B8994E6B263F03E100069589 /* JSINSObjectConversion.mm */,
B85F7AE82A77BB680089C539 /* FrameProcessorPlugin.m */,
B89A79692B3EF60F005E0357 /* UIImageOrientation+descriptor.h */,
);
path = "Frame Processor";
sourceTree = "<group>";