feat: Make Frame thread-safe and improve error messages (#2327)

* fix: Fix multi-Thread access on Java

* fix: Thread-lock access on iOS as well

* whoops add missing header impl

* Update Podfile.lock

* fix: Don't use `CFGetRetainCount`

* fix: Lock access on iOS as well

* C++ format

* More detailed error

* chore: Move getters into `Frame`

* Format c++

* Use enum `orientation` again

* format

* fix: Synchronize `isValid` on Java

* Also log pixelformat

* feat: Use Java enums in C++

* Format C++
This commit is contained in:
Marc Rousavy 2023-12-29 14:09:56 +01:00 committed by GitHub
parent e4393cd83a
commit 895f3ec889
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 338 additions and 114 deletions

View File

@ -77,7 +77,9 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
} }
auto width = this->frame->getWidth(); auto width = this->frame->getWidth();
auto height = this->frame->getHeight(); auto height = this->frame->getHeight();
auto str = std::to_string(width) + " x " + std::to_string(height) + " Frame"; auto format = this->frame->getPixelFormat();
auto formatString = format->getUnionValue();
auto str = std::to_string(width) + " x " + std::to_string(height) + " " + formatString->toString() + " Frame";
return jsi::String::createFromUtf8(runtime, str); return jsi::String::createFromUtf8(runtime, str);
}; };
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString); return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
@ -141,11 +143,13 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
return jsi::Value(this->frame->getIsMirrored()); return jsi::Value(this->frame->getIsMirrored());
} }
if (name == "orientation") { if (name == "orientation") {
auto string = this->frame->getOrientation(); auto orientation = this->frame->getOrientation();
auto string = orientation->getUnionValue();
return jsi::String::createFromUtf8(runtime, string->toStdString()); return jsi::String::createFromUtf8(runtime, string->toStdString());
} }
if (name == "pixelFormat") { if (name == "pixelFormat") {
auto string = this->frame->getPixelFormat(); auto pixelFormat = this->frame->getPixelFormat();
auto string = pixelFormat->getUnionValue();
return jsi::String::createFromUtf8(runtime, string->toStdString()); return jsi::String::createFromUtf8(runtime, string->toStdString());
} }
if (name == "timestamp") { if (name == "timestamp") {

View File

@ -4,6 +4,8 @@
#include "JFrame.h" #include "JFrame.h"
#include "JOrientation.h"
#include "JPixelFormat.h"
#include <fbjni/fbjni.h> #include <fbjni/fbjni.h>
#include <jni.h> #include <jni.h>
@ -39,13 +41,13 @@ jlong JFrame::getTimestamp() const {
return getTimestampMethod(self()); return getTimestampMethod(self());
} }
local_ref<JString> JFrame::getOrientation() const { local_ref<JOrientation> JFrame::getOrientation() const {
static const auto getOrientationMethod = getClass()->getMethod<JString()>("getOrientation"); static const auto getOrientationMethod = getClass()->getMethod<JOrientation()>("getOrientation");
return getOrientationMethod(self()); return getOrientationMethod(self());
} }
local_ref<JString> JFrame::getPixelFormat() const { local_ref<JPixelFormat> JFrame::getPixelFormat() const {
static const auto getPixelFormatMethod = getClass()->getMethod<JString()>("getPixelFormat"); static const auto getPixelFormatMethod = getClass()->getMethod<JPixelFormat()>("getPixelFormat");
return getPixelFormatMethod(self()); return getPixelFormatMethod(self());
} }
@ -77,9 +79,4 @@ void JFrame::decrementRefCount() {
decrementRefCountMethod(self()); decrementRefCountMethod(self());
} }
void JFrame::close() {
static const auto closeMethod = getClass()->getMethod<void()>("close");
closeMethod(self());
}
} // namespace vision } // namespace vision

View File

@ -4,6 +4,8 @@
#pragma once #pragma once
#include "JOrientation.h"
#include "JPixelFormat.h"
#include <fbjni/fbjni.h> #include <fbjni/fbjni.h>
#include <jni.h> #include <jni.h>
@ -25,15 +27,14 @@ public:
int getPlanesCount() const; int getPlanesCount() const;
int getBytesPerRow() const; int getBytesPerRow() const;
jlong getTimestamp() const; jlong getTimestamp() const;
local_ref<JString> getOrientation() const; local_ref<JOrientation> getOrientation() const;
local_ref<JString> getPixelFormat() const; local_ref<JPixelFormat> getPixelFormat() const;
#if __ANDROID_API__ >= 26 #if __ANDROID_API__ >= 26
AHardwareBuffer* getHardwareBuffer() const; AHardwareBuffer* getHardwareBuffer() const;
#endif #endif
void incrementRefCount(); void incrementRefCount();
void decrementRefCount(); void decrementRefCount();
void close();
}; };
} // namespace vision } // namespace vision

View File

@ -0,0 +1,24 @@
//
// Created by Marc Rousavy on 29.12.23.
//
#pragma once
#include <fbjni/fbjni.h>
#include <jni.h>
namespace vision {
using namespace facebook;
using namespace jni;
struct JJSUnionValue : public JavaClass<JJSUnionValue> {
static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/types/JSUnionValue;";
local_ref<JString> getUnionValue() {
const auto getUnionValueMethod = getClass()->getMethod<JString()>("getUnionValue");
return getUnionValueMethod(self());
}
};
} // namespace vision

View File

@ -0,0 +1,20 @@
//
// Created by Marc Rousavy on 29.12.23.
//
#pragma once
#include "JJSUnionValue.h"
#include <fbjni/fbjni.h>
#include <jni.h>
namespace vision {
using namespace facebook;
using namespace jni;
struct JOrientation : public JavaClass<JOrientation, JJSUnionValue> {
static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/types/Orientation;";
};
} // namespace vision

View File

@ -0,0 +1,20 @@
//
// Created by Marc Rousavy on 29.12.23.
//
#pragma once
#include "JJSUnionValue.h"
#include <fbjni/fbjni.h>
#include <jni.h>
namespace vision {
using namespace facebook;
using namespace jni;
struct JPixelFormat : public JavaClass<JPixelFormat, JJSUnionValue> {
static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/types/PixelFormat;";
};
} // namespace vision

View File

@ -7,6 +7,7 @@ import com.facebook.proguard.annotations.DoNotStrip;
import com.mrousavy.camera.core.HardwareBuffersNotAvailableError; import com.mrousavy.camera.core.HardwareBuffersNotAvailableError;
import com.mrousavy.camera.types.PixelFormat; import com.mrousavy.camera.types.PixelFormat;
import com.mrousavy.camera.types.Orientation; import com.mrousavy.camera.types.Orientation;
import java.lang.IllegalStateException;
public class Frame { public class Frame {
private final Image image; private final Image image;
@ -24,30 +25,44 @@ public class Frame {
} }
public Image getImage() { public Image getImage() {
return image; synchronized (this) {
Image img = image;
if (!getIsImageValid(img)) {
throw new RuntimeException("Frame is already closed! " +
"Are you trying to access the Image data outside of a Frame Processor's lifetime?\n" +
"- If you want to use `console.log(frame)`, use `console.log(frame.toString())` instead.\n" +
"- If you want to do async processing, use `runAsync(...)` instead.\n" +
"- If you want to use runOnJS, increment it's ref-count: `frame.incrementRefCount()`");
}
return img;
}
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@DoNotStrip @DoNotStrip
public int getWidth() { public int getWidth() {
return image.getWidth(); return getImage().getWidth();
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@DoNotStrip @DoNotStrip
public int getHeight() { public int getHeight() {
return image.getHeight(); return getImage().getHeight();
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@DoNotStrip @DoNotStrip
public boolean getIsValid() { public boolean getIsValid() {
return getIsImageValid(getImage());
}
private boolean getIsImageValid(Image image) {
try { try {
// will throw an exception if the image is already closed // will throw an exception if the image is already closed
image.getCropRect(); synchronized (this) { image.getFormat(); }
// no exception thrown, image must still be valid. // no exception thrown, image must still be valid.
return true; return true;
} catch (Exception e) { } catch (IllegalStateException e) {
// exception thrown, image has already been closed. // exception thrown, image has already been closed.
return false; return false;
} }
@ -67,27 +82,26 @@ public class Frame {
@SuppressWarnings("unused") @SuppressWarnings("unused")
@DoNotStrip @DoNotStrip
public String getOrientation() { public Orientation getOrientation() {
return orientation.getUnionValue(); return orientation;
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@DoNotStrip @DoNotStrip
public String getPixelFormat() { public PixelFormat getPixelFormat() {
PixelFormat format = PixelFormat.Companion.fromImageFormat(image.getFormat()); return PixelFormat.Companion.fromImageFormat(getImage().getFormat());
return format.getUnionValue();
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@DoNotStrip @DoNotStrip
public int getPlanesCount() { public int getPlanesCount() {
return image.getPlanes().length; return getImage().getPlanes().length;
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@DoNotStrip @DoNotStrip
public int getBytesPerRow() { public int getBytesPerRow() {
return image.getPlanes()[0].getRowStride(); return getImage().getPlanes()[0].getRowStride();
} }
@SuppressWarnings("unused") @SuppressWarnings("unused")
@ -101,7 +115,7 @@ public class Frame {
throw new HardwareBuffersNotAvailableError(); throw new HardwareBuffersNotAvailableError();
} }
if (hardwareBuffer == null) { if (hardwareBuffer == null) {
hardwareBuffer = image.getHardwareBuffer(); hardwareBuffer = getImage().getHardwareBuffer();
} }
return hardwareBuffer; return hardwareBuffer;
} }
@ -121,17 +135,17 @@ public class Frame {
refCount--; refCount--;
if (refCount <= 0) { if (refCount <= 0) {
// If no reference is held on this Image, close it. // If no reference is held on this Image, close it.
image.close(); close();
} }
} }
} }
@SuppressWarnings("unused") private synchronized void close() {
@DoNotStrip synchronized (this) {
private void close() { if (hardwareBuffer != null) {
if (hardwareBuffer != null) { hardwareBuffer.close();
hardwareBuffer.close(); }
image.close();
} }
image.close();
} }
} }

View File

@ -484,7 +484,7 @@ PODS:
- libwebp (~> 1.0) - libwebp (~> 1.0)
- SDWebImage/Core (~> 5.10) - SDWebImage/Core (~> 5.10)
- SocketRocket (0.6.1) - SocketRocket (0.6.1)
- VisionCamera (3.6.16): - VisionCamera (3.6.17):
- React - React
- React-callinvoker - React-callinvoker
- React-Core - React-Core
@ -724,9 +724,9 @@ SPEC CHECKSUMS:
SDWebImage: a7f831e1a65eb5e285e3fb046a23fcfbf08e696d SDWebImage: a7f831e1a65eb5e285e3fb046a23fcfbf08e696d
SDWebImageWebPCoder: 908b83b6adda48effe7667cd2b7f78c897e5111d SDWebImageWebPCoder: 908b83b6adda48effe7667cd2b7f78c897e5111d
SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17 SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17
VisionCamera: 06f5b990082ec783a58dcadc5bac677974f7a580 VisionCamera: 361df29347b7b7ecc47b3d173daa17751a11ffc1
Yoga: 4c3aa327e4a6a23eeacd71f61c81df1bcdf677d5 Yoga: 4c3aa327e4a6a23eeacd71f61c81df1bcdf677d5
PODFILE CHECKSUM: 27f53791141a3303d814e09b55770336416ff4eb PODFILE CHECKSUM: 27f53791141a3303d814e09b55770336416ff4eb
COCOAPODS: 1.11.3 COCOAPODS: 1.14.3

View File

@ -19,4 +19,14 @@
@property(nonatomic, readonly) CMSampleBufferRef _Nonnull buffer; @property(nonatomic, readonly) CMSampleBufferRef _Nonnull buffer;
@property(nonatomic, readonly) UIImageOrientation orientation; @property(nonatomic, readonly) UIImageOrientation orientation;
// Getters
- (NSString* _Nonnull)pixelFormat;
- (BOOL)isMirrored;
- (BOOL)isValid;
- (size_t)width;
- (size_t)height;
- (double)timestamp;
- (size_t)bytesPerRow;
- (size_t)planesCount;
@end @end

View File

@ -20,11 +20,80 @@
if (self) { if (self) {
_buffer = buffer; _buffer = buffer;
_orientation = orientation; _orientation = orientation;
CFRetain(buffer);
} }
return self; return self;
} }
- (void)dealloc {
CFRelease(_buffer);
}
@synthesize buffer = _buffer; @synthesize buffer = _buffer;
@synthesize orientation = _orientation; @synthesize orientation = _orientation;
- (NSString*)pixelFormat {
CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(_buffer);
FourCharCode mediaType = CMFormatDescriptionGetMediaSubType(format);
switch (mediaType) {
case kCVPixelFormatType_32BGRA:
case kCVPixelFormatType_Lossy_32BGRA:
return @"rgb";
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange:
case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange:
case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarFullRange:
case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarVideoRange:
case kCVPixelFormatType_Lossy_420YpCbCr10PackedBiPlanarVideoRange:
return @"yuv";
default:
return @"unknown";
}
}
- (BOOL)isMirrored {
switch (_orientation) {
case UIImageOrientationUp:
case UIImageOrientationDown:
case UIImageOrientationLeft:
case UIImageOrientationRight:
return false;
case UIImageOrientationDownMirrored:
case UIImageOrientationUpMirrored:
case UIImageOrientationLeftMirrored:
case UIImageOrientationRightMirrored:
return true;
}
}
- (BOOL)isValid {
return _buffer != nil && CMSampleBufferIsValid(_buffer);
}
- (size_t)width {
CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_buffer);
return CVPixelBufferGetWidth(imageBuffer);
}
- (size_t)height {
CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_buffer);
return CVPixelBufferGetHeight(imageBuffer);
}
- (double)timestamp {
CMTime timestamp = CMSampleBufferGetPresentationTimeStamp(_buffer);
return CMTimeGetSeconds(timestamp) * 1000.0;
}
- (size_t)bytesPerRow {
CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_buffer);
return CVPixelBufferGetBytesPerRow(imageBuffer);
}
- (size_t)planesCount {
CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_buffer);
return CVPixelBufferGetPlaneCount(imageBuffer);
}
@end @end

View File

@ -10,6 +10,7 @@
#import <CoreMedia/CMSampleBuffer.h> #import <CoreMedia/CMSampleBuffer.h>
#import <jsi/jsi.h> #import <jsi/jsi.h>
#import <mutex>
#import "Frame.h" #import "Frame.h"
@ -25,4 +26,11 @@ public:
public: public:
Frame* frame; Frame* frame;
private:
Frame* getFrame();
private:
std::mutex _mutex;
size_t _refCount = 0;
}; };

View File

@ -7,6 +7,7 @@
// //
#import "FrameHostObject.h" #import "FrameHostObject.h"
#import "UIImageOrientation+descriptor.h"
#import "WKTJsiHostObject.h" #import "WKTJsiHostObject.h"
#import <Foundation/Foundation.h> #import <Foundation/Foundation.h>
#import <jsi/jsi.h> #import <jsi/jsi.h>
@ -34,42 +35,70 @@ std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt)
return result; return result;
} }
Frame* FrameHostObject::getFrame() {
Frame* frame = this->frame;
if (frame == nil || !CMSampleBufferIsValid(frame.buffer)) {
throw std::runtime_error("Frame is already closed! "
"Are you trying to access the Image data outside of a Frame Processor's lifetime?\n"
"- If you want to use `console.log(frame)`, use `console.log(frame.toString())` instead.\n"
"- If you want to do async processing, use `runAsync(...)` instead.\n"
"- If you want to use runOnJS, increment it's ref-count: `frame.incrementRefCount()`");
}
return frame;
}
jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) { jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
auto name = propName.utf8(runtime); auto name = propName.utf8(runtime);
if (name == "toString") { if (name == "toString") {
auto toString = JSI_HOST_FUNCTION_LAMBDA { auto toString = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
if (this->frame == nil) { // Lock Frame so it cannot be deallocated while we access it
return jsi::String::createFromUtf8(runtime, "[closed frame]"); std::lock_guard lock(this->_mutex);
}
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto width = CVPixelBufferGetWidth(imageBuffer);
auto height = CVPixelBufferGetHeight(imageBuffer);
NSMutableString* string = [NSMutableString stringWithFormat:@"%lu x %lu Frame", width, height]; // Print debug description (width, height)
Frame* frame = this->getFrame();
NSMutableString* string = [NSMutableString stringWithFormat:@"%lu x %lu %@ Frame", frame.width, frame.height, frame.pixelFormat];
return jsi::String::createFromUtf8(runtime, string.UTF8String); return jsi::String::createFromUtf8(runtime, string.UTF8String);
}; };
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString); return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
} }
if (name == "incrementRefCount") { if (name == "incrementRefCount") {
auto incrementRefCount = JSI_HOST_FUNCTION_LAMBDA { auto incrementRefCount = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
// Increment retain count by one so ARC doesn't destroy the Frame Buffer. size_t count) -> jsi::Value {
CFRetain(frame.buffer); // Lock Frame so it cannot be deallocated while we access it
std::lock_guard lock(this->_mutex);
// Increment our self-counted ref count by one.
_refCount++;
return jsi::Value::undefined(); return jsi::Value::undefined();
}; };
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "incrementRefCount"), 0, incrementRefCount); return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "incrementRefCount"), 0, incrementRefCount);
} }
if (name == "decrementRefCount") { if (name == "decrementRefCount") {
auto decrementRefCount = JSI_HOST_FUNCTION_LAMBDA { auto decrementRefCount = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
// Decrement retain count by one. If the retain count is zero, ARC will destroy the Frame size_t count) -> jsi::Value {
// Buffer. // Lock Frame so it cannot be deallocated while we access it
CFRelease(frame.buffer); std::lock_guard lock(this->_mutex);
// Decrement our self-counted ref count by one.
_refCount--;
if (_refCount < 1) {
// ARC will then delete the Frame and the underlying Frame Buffer.
this->frame = nil;
}
return jsi::Value::undefined(); return jsi::Value::undefined();
}; };
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "decrementRefCount"), 0, decrementRefCount); return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "decrementRefCount"), 0, decrementRefCount);
} }
if (name == "toArrayBuffer") { if (name == "toArrayBuffer") {
auto toArrayBuffer = JSI_HOST_FUNCTION_LAMBDA { auto toArrayBuffer = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
size_t count) -> jsi::Value {
// Lock Frame so it cannot be deallocated while we access it
std::lock_guard lock(this->_mutex);
// Get CPU readable Pixel Buffer from Frame and write it to a jsi::ArrayBuffer
Frame* frame = this->getFrame();
auto pixelBuffer = CMSampleBufferGetImageBuffer(frame.buffer); auto pixelBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
auto bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer); auto bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer);
auto height = CVPixelBufferGetHeight(pixelBuffer); auto height = CVPixelBufferGetHeight(pixelBuffer);
@ -101,82 +130,69 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
} }
if (name == "isValid") { if (name == "isValid") {
auto isValid = frame != nil && frame.buffer != nil && CFGetRetainCount(frame.buffer) > 0 && CMSampleBufferIsValid(frame.buffer); // Lock Frame so it cannot be deallocated while we access it
return jsi::Value(isValid); std::lock_guard lock(this->_mutex);
// unsafely access the Frame and try to see if it's valid
Frame* frame = this->frame;
return jsi::Value(frame != nil && frame.isValid);
} }
if (name == "width") { if (name == "width") {
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer); // Lock Frame so it cannot be deallocated while we access it
auto width = CVPixelBufferGetWidth(imageBuffer); std::lock_guard lock(this->_mutex);
return jsi::Value((double)width);
Frame* frame = this->getFrame();
return jsi::Value((double)frame.width);
} }
if (name == "height") { if (name == "height") {
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer); // Lock Frame so it cannot be deallocated while we access it
auto height = CVPixelBufferGetHeight(imageBuffer); std::lock_guard lock(this->_mutex);
return jsi::Value((double)height);
Frame* frame = this->getFrame();
return jsi::Value((double)frame.height);
} }
if (name == "orientation") { if (name == "orientation") {
switch (frame.orientation) { // Lock Frame so it cannot be deallocated while we access it
case UIImageOrientationUp: std::lock_guard lock(this->_mutex);
case UIImageOrientationUpMirrored:
return jsi::String::createFromUtf8(runtime, "portrait"); Frame* frame = this->getFrame();
case UIImageOrientationDown: NSString* orientation = [NSString stringWithParsed:frame.orientation];
case UIImageOrientationDownMirrored: return jsi::String::createFromUtf8(runtime, orientation.UTF8String);
return jsi::String::createFromUtf8(runtime, "portrait-upside-down");
case UIImageOrientationLeft:
case UIImageOrientationLeftMirrored:
return jsi::String::createFromUtf8(runtime, "landscape-left");
case UIImageOrientationRight:
case UIImageOrientationRightMirrored:
return jsi::String::createFromUtf8(runtime, "landscape-right");
}
} }
if (name == "isMirrored") { if (name == "isMirrored") {
switch (frame.orientation) { // Lock Frame so it cannot be deallocated while we access it
case UIImageOrientationUp: std::lock_guard lock(this->_mutex);
case UIImageOrientationDown:
case UIImageOrientationLeft: Frame* frame = this->getFrame();
case UIImageOrientationRight: return jsi::Value(frame.isMirrored);
return jsi::Value(false);
case UIImageOrientationDownMirrored:
case UIImageOrientationUpMirrored:
case UIImageOrientationLeftMirrored:
case UIImageOrientationRightMirrored:
return jsi::Value(true);
}
} }
if (name == "timestamp") { if (name == "timestamp") {
auto timestamp = CMSampleBufferGetPresentationTimeStamp(frame.buffer); // Lock Frame so it cannot be deallocated while we access it
auto seconds = static_cast<double>(CMTimeGetSeconds(timestamp)); std::lock_guard lock(this->_mutex);
return jsi::Value(seconds * 1000.0);
Frame* frame = this->getFrame();
return jsi::Value(frame.timestamp);
} }
if (name == "pixelFormat") { if (name == "pixelFormat") {
auto format = CMSampleBufferGetFormatDescription(frame.buffer); // Lock Frame so it cannot be deallocated while we access it
auto mediaType = CMFormatDescriptionGetMediaSubType(format); std::lock_guard lock(this->_mutex);
switch (mediaType) {
case kCVPixelFormatType_32BGRA: Frame* frame = this->getFrame();
case kCVPixelFormatType_Lossy_32BGRA: return jsi::String::createFromUtf8(runtime, frame.pixelFormat.UTF8String);
return jsi::String::createFromUtf8(runtime, "rgb");
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange:
case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange:
case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarFullRange:
case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarVideoRange:
case kCVPixelFormatType_Lossy_420YpCbCr10PackedBiPlanarVideoRange:
return jsi::String::createFromUtf8(runtime, "yuv");
default:
return jsi::String::createFromUtf8(runtime, "unknown");
}
} }
if (name == "bytesPerRow") { if (name == "bytesPerRow") {
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer); // Lock Frame so it cannot be deallocated while we access it
auto bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer); std::lock_guard lock(this->_mutex);
return jsi::Value((double)bytesPerRow);
Frame* frame = this->getFrame();
return jsi::Value((double)frame.bytesPerRow);
} }
if (name == "planesCount") { if (name == "planesCount") {
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer); // Lock Frame so it cannot be deallocated while we access it
auto planesCount = CVPixelBufferGetPlaneCount(imageBuffer); std::lock_guard lock(this->_mutex);
return jsi::Value((double)planesCount);
Frame* frame = this->getFrame();
return jsi::Value((double)frame.planesCount);
} }
// fallback to base implementation // fallback to base implementation

View File

@ -0,0 +1,39 @@
//
// UIImageOrientation+descriptor.h
// VisionCamera
//
// Created by Marc Rousavy on 29.12.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
#pragma once
#import <Foundation/Foundation.h>
#import <UIKit/UIImage.h>
@interface NSString (UIImageOrientationJSDescriptor)
+ (NSString*)stringWithParsed:(UIImageOrientation)orientation;
@end
@implementation NSString (UIImageOrientationJSDescriptor)
+ (NSString*)stringWithParsed:(UIImageOrientation)orientation {
switch (orientation) {
case UIImageOrientationUp:
case UIImageOrientationUpMirrored:
return @"portrait";
case UIImageOrientationDown:
case UIImageOrientationDownMirrored:
return @"portrait-upside-down";
case UIImageOrientationLeft:
case UIImageOrientationLeftMirrored:
return @"landscape-left";
case UIImageOrientationRight:
case UIImageOrientationRightMirrored:
return @"landscape-right";
}
}
@end

View File

@ -160,6 +160,7 @@
B887518425E0102000DB86D6 /* CameraView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CameraView.swift; sourceTree = "<group>"; }; B887518425E0102000DB86D6 /* CameraView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CameraView.swift; sourceTree = "<group>"; };
B88873E5263D46C7008B1D0E /* FrameProcessorPlugin.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorPlugin.h; sourceTree = "<group>"; }; B88873E5263D46C7008B1D0E /* FrameProcessorPlugin.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorPlugin.h; sourceTree = "<group>"; };
B8994E6B263F03E100069589 /* JSINSObjectConversion.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = JSINSObjectConversion.mm; sourceTree = "<group>"; }; B8994E6B263F03E100069589 /* JSINSObjectConversion.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = JSINSObjectConversion.mm; sourceTree = "<group>"; };
B89A79692B3EF60F005E0357 /* UIImageOrientation+descriptor.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "UIImageOrientation+descriptor.h"; sourceTree = "<group>"; };
B8A1AEC32AD7EDE800169C0D /* AVCaptureVideoDataOutput+pixelFormat.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVCaptureVideoDataOutput+pixelFormat.swift"; sourceTree = "<group>"; }; B8A1AEC32AD7EDE800169C0D /* AVCaptureVideoDataOutput+pixelFormat.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVCaptureVideoDataOutput+pixelFormat.swift"; sourceTree = "<group>"; };
B8A1AEC52AD7F08E00169C0D /* CameraView+Focus.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CameraView+Focus.swift"; sourceTree = "<group>"; }; B8A1AEC52AD7F08E00169C0D /* CameraView+Focus.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CameraView+Focus.swift"; sourceTree = "<group>"; };
B8A1AEC72AD8005400169C0D /* CameraSession+Configuration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CameraSession+Configuration.swift"; sourceTree = "<group>"; }; B8A1AEC72AD8005400169C0D /* CameraSession+Configuration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CameraSession+Configuration.swift"; sourceTree = "<group>"; };
@ -334,6 +335,7 @@
B81D41EF263C86F900B041FD /* JSINSObjectConversion.h */, B81D41EF263C86F900B041FD /* JSINSObjectConversion.h */,
B8994E6B263F03E100069589 /* JSINSObjectConversion.mm */, B8994E6B263F03E100069589 /* JSINSObjectConversion.mm */,
B85F7AE82A77BB680089C539 /* FrameProcessorPlugin.m */, B85F7AE82A77BB680089C539 /* FrameProcessorPlugin.m */,
B89A79692B3EF60F005E0357 /* UIImageOrientation+descriptor.h */,
); );
path = "Frame Processor"; path = "Frame Processor";
sourceTree = "<group>"; sourceTree = "<group>";