From 895f3ec889bdf87cdd06a5d4ec024a3a479b4105 Mon Sep 17 00:00:00 2001 From: Marc Rousavy Date: Fri, 29 Dec 2023 14:09:56 +0100 Subject: [PATCH] feat: Make `Frame` thread-safe and improve error messages (#2327) * fix: Fix multi-Thread access on Java * fix: Thread-lock access on iOS as well * whoops add missing header impl * Update Podfile.lock * fix: Don't use `CFGetRetainCount` * fix: Lock access on iOS as well * C++ format * More detailed error * chore: Move getters into `Frame` * Format c++ * Use enum `orientation` again * format * fix: Synchronize `isValid` on Java * Also log pixelformat * feat: Use Java enums in C++ * Format C++ --- .../cpp/frameprocessor/FrameHostObject.cpp | 10 +- .../frameprocessor/java-bindings/JFrame.cpp | 15 +- .../cpp/frameprocessor/java-bindings/JFrame.h | 7 +- .../java-bindings/JJSUnionValue.h | 24 +++ .../java-bindings/JOrientation.h | 20 +++ .../java-bindings/JPixelFormat.h | 20 +++ .../mrousavy/camera/frameprocessor/Frame.java | 54 +++--- package/example/ios/Podfile.lock | 6 +- package/ios/Frame Processor/Frame.h | 10 ++ package/ios/Frame Processor/Frame.m | 69 +++++++ package/ios/Frame Processor/FrameHostObject.h | 8 + .../ios/Frame Processor/FrameHostObject.mm | 168 ++++++++++-------- .../UIImageOrientation+descriptor.h | 39 ++++ .../VisionCamera.xcodeproj/project.pbxproj | 2 + 14 files changed, 338 insertions(+), 114 deletions(-) create mode 100644 package/android/src/main/cpp/frameprocessor/java-bindings/JJSUnionValue.h create mode 100644 package/android/src/main/cpp/frameprocessor/java-bindings/JOrientation.h create mode 100644 package/android/src/main/cpp/frameprocessor/java-bindings/JPixelFormat.h create mode 100644 package/ios/Frame Processor/UIImageOrientation+descriptor.h diff --git a/package/android/src/main/cpp/frameprocessor/FrameHostObject.cpp b/package/android/src/main/cpp/frameprocessor/FrameHostObject.cpp index 83e99f7..9a2ed04 100644 --- a/package/android/src/main/cpp/frameprocessor/FrameHostObject.cpp +++ b/package/android/src/main/cpp/frameprocessor/FrameHostObject.cpp @@ -77,7 +77,9 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr } auto width = this->frame->getWidth(); auto height = this->frame->getHeight(); - auto str = std::to_string(width) + " x " + std::to_string(height) + " Frame"; + auto format = this->frame->getPixelFormat(); + auto formatString = format->getUnionValue(); + auto str = std::to_string(width) + " x " + std::to_string(height) + " " + formatString->toString() + " Frame"; return jsi::String::createFromUtf8(runtime, str); }; return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString); @@ -141,11 +143,13 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr return jsi::Value(this->frame->getIsMirrored()); } if (name == "orientation") { - auto string = this->frame->getOrientation(); + auto orientation = this->frame->getOrientation(); + auto string = orientation->getUnionValue(); return jsi::String::createFromUtf8(runtime, string->toStdString()); } if (name == "pixelFormat") { - auto string = this->frame->getPixelFormat(); + auto pixelFormat = this->frame->getPixelFormat(); + auto string = pixelFormat->getUnionValue(); return jsi::String::createFromUtf8(runtime, string->toStdString()); } if (name == "timestamp") { diff --git a/package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.cpp b/package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.cpp index b534bcc..ceebeb1 100644 --- a/package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.cpp +++ b/package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.cpp @@ -4,6 +4,8 @@ #include "JFrame.h" +#include "JOrientation.h" +#include "JPixelFormat.h" #include #include @@ -39,13 +41,13 @@ jlong JFrame::getTimestamp() const { return getTimestampMethod(self()); } -local_ref JFrame::getOrientation() const { - static const auto getOrientationMethod = getClass()->getMethod("getOrientation"); +local_ref JFrame::getOrientation() const { + static const auto getOrientationMethod = getClass()->getMethod("getOrientation"); return getOrientationMethod(self()); } -local_ref JFrame::getPixelFormat() const { - static const auto getPixelFormatMethod = getClass()->getMethod("getPixelFormat"); +local_ref JFrame::getPixelFormat() const { + static const auto getPixelFormatMethod = getClass()->getMethod("getPixelFormat"); return getPixelFormatMethod(self()); } @@ -77,9 +79,4 @@ void JFrame::decrementRefCount() { decrementRefCountMethod(self()); } -void JFrame::close() { - static const auto closeMethod = getClass()->getMethod("close"); - closeMethod(self()); -} - } // namespace vision diff --git a/package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.h b/package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.h index 310f749..19d8b5d 100644 --- a/package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.h +++ b/package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.h @@ -4,6 +4,8 @@ #pragma once +#include "JOrientation.h" +#include "JPixelFormat.h" #include #include @@ -25,15 +27,14 @@ public: int getPlanesCount() const; int getBytesPerRow() const; jlong getTimestamp() const; - local_ref getOrientation() const; - local_ref getPixelFormat() const; + local_ref getOrientation() const; + local_ref getPixelFormat() const; #if __ANDROID_API__ >= 26 AHardwareBuffer* getHardwareBuffer() const; #endif void incrementRefCount(); void decrementRefCount(); - void close(); }; } // namespace vision diff --git a/package/android/src/main/cpp/frameprocessor/java-bindings/JJSUnionValue.h b/package/android/src/main/cpp/frameprocessor/java-bindings/JJSUnionValue.h new file mode 100644 index 0000000..16c9bad --- /dev/null +++ b/package/android/src/main/cpp/frameprocessor/java-bindings/JJSUnionValue.h @@ -0,0 +1,24 @@ +// +// Created by Marc Rousavy on 29.12.23. +// + +#pragma once + +#include +#include + +namespace vision { + +using namespace facebook; +using namespace jni; + +struct JJSUnionValue : public JavaClass { + static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/types/JSUnionValue;"; + + local_ref getUnionValue() { + const auto getUnionValueMethod = getClass()->getMethod("getUnionValue"); + return getUnionValueMethod(self()); + } +}; + +} // namespace vision \ No newline at end of file diff --git a/package/android/src/main/cpp/frameprocessor/java-bindings/JOrientation.h b/package/android/src/main/cpp/frameprocessor/java-bindings/JOrientation.h new file mode 100644 index 0000000..458ed1b --- /dev/null +++ b/package/android/src/main/cpp/frameprocessor/java-bindings/JOrientation.h @@ -0,0 +1,20 @@ +// +// Created by Marc Rousavy on 29.12.23. +// + +#pragma once + +#include "JJSUnionValue.h" +#include +#include + +namespace vision { + +using namespace facebook; +using namespace jni; + +struct JOrientation : public JavaClass { + static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/types/Orientation;"; +}; + +} // namespace vision diff --git a/package/android/src/main/cpp/frameprocessor/java-bindings/JPixelFormat.h b/package/android/src/main/cpp/frameprocessor/java-bindings/JPixelFormat.h new file mode 100644 index 0000000..8b04c53 --- /dev/null +++ b/package/android/src/main/cpp/frameprocessor/java-bindings/JPixelFormat.h @@ -0,0 +1,20 @@ +// +// Created by Marc Rousavy on 29.12.23. +// + +#pragma once + +#include "JJSUnionValue.h" +#include +#include + +namespace vision { + +using namespace facebook; +using namespace jni; + +struct JPixelFormat : public JavaClass { + static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/types/PixelFormat;"; +}; + +} // namespace vision \ No newline at end of file diff --git a/package/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java b/package/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java index 39add95..0c6907a 100644 --- a/package/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java +++ b/package/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java @@ -7,6 +7,7 @@ import com.facebook.proguard.annotations.DoNotStrip; import com.mrousavy.camera.core.HardwareBuffersNotAvailableError; import com.mrousavy.camera.types.PixelFormat; import com.mrousavy.camera.types.Orientation; +import java.lang.IllegalStateException; public class Frame { private final Image image; @@ -24,30 +25,44 @@ public class Frame { } public Image getImage() { - return image; + synchronized (this) { + Image img = image; + if (!getIsImageValid(img)) { + throw new RuntimeException("Frame is already closed! " + + "Are you trying to access the Image data outside of a Frame Processor's lifetime?\n" + + "- If you want to use `console.log(frame)`, use `console.log(frame.toString())` instead.\n" + + "- If you want to do async processing, use `runAsync(...)` instead.\n" + + "- If you want to use runOnJS, increment it's ref-count: `frame.incrementRefCount()`"); + } + return img; + } } @SuppressWarnings("unused") @DoNotStrip public int getWidth() { - return image.getWidth(); + return getImage().getWidth(); } @SuppressWarnings("unused") @DoNotStrip public int getHeight() { - return image.getHeight(); + return getImage().getHeight(); } @SuppressWarnings("unused") @DoNotStrip public boolean getIsValid() { + return getIsImageValid(getImage()); + } + + private boolean getIsImageValid(Image image) { try { // will throw an exception if the image is already closed - image.getCropRect(); + synchronized (this) { image.getFormat(); } // no exception thrown, image must still be valid. return true; - } catch (Exception e) { + } catch (IllegalStateException e) { // exception thrown, image has already been closed. return false; } @@ -67,27 +82,26 @@ public class Frame { @SuppressWarnings("unused") @DoNotStrip - public String getOrientation() { - return orientation.getUnionValue(); + public Orientation getOrientation() { + return orientation; } @SuppressWarnings("unused") @DoNotStrip - public String getPixelFormat() { - PixelFormat format = PixelFormat.Companion.fromImageFormat(image.getFormat()); - return format.getUnionValue(); + public PixelFormat getPixelFormat() { + return PixelFormat.Companion.fromImageFormat(getImage().getFormat()); } @SuppressWarnings("unused") @DoNotStrip public int getPlanesCount() { - return image.getPlanes().length; + return getImage().getPlanes().length; } @SuppressWarnings("unused") @DoNotStrip public int getBytesPerRow() { - return image.getPlanes()[0].getRowStride(); + return getImage().getPlanes()[0].getRowStride(); } @SuppressWarnings("unused") @@ -101,7 +115,7 @@ public class Frame { throw new HardwareBuffersNotAvailableError(); } if (hardwareBuffer == null) { - hardwareBuffer = image.getHardwareBuffer(); + hardwareBuffer = getImage().getHardwareBuffer(); } return hardwareBuffer; } @@ -121,17 +135,17 @@ public class Frame { refCount--; if (refCount <= 0) { // If no reference is held on this Image, close it. - image.close(); + close(); } } } - @SuppressWarnings("unused") - @DoNotStrip - private void close() { - if (hardwareBuffer != null) { - hardwareBuffer.close(); + private synchronized void close() { + synchronized (this) { + if (hardwareBuffer != null) { + hardwareBuffer.close(); + } + image.close(); } - image.close(); } } diff --git a/package/example/ios/Podfile.lock b/package/example/ios/Podfile.lock index 9fac47f..397096d 100644 --- a/package/example/ios/Podfile.lock +++ b/package/example/ios/Podfile.lock @@ -484,7 +484,7 @@ PODS: - libwebp (~> 1.0) - SDWebImage/Core (~> 5.10) - SocketRocket (0.6.1) - - VisionCamera (3.6.16): + - VisionCamera (3.6.17): - React - React-callinvoker - React-Core @@ -724,9 +724,9 @@ SPEC CHECKSUMS: SDWebImage: a7f831e1a65eb5e285e3fb046a23fcfbf08e696d SDWebImageWebPCoder: 908b83b6adda48effe7667cd2b7f78c897e5111d SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17 - VisionCamera: 06f5b990082ec783a58dcadc5bac677974f7a580 + VisionCamera: 361df29347b7b7ecc47b3d173daa17751a11ffc1 Yoga: 4c3aa327e4a6a23eeacd71f61c81df1bcdf677d5 PODFILE CHECKSUM: 27f53791141a3303d814e09b55770336416ff4eb -COCOAPODS: 1.11.3 +COCOAPODS: 1.14.3 diff --git a/package/ios/Frame Processor/Frame.h b/package/ios/Frame Processor/Frame.h index 45775ce..edbcd0a 100644 --- a/package/ios/Frame Processor/Frame.h +++ b/package/ios/Frame Processor/Frame.h @@ -19,4 +19,14 @@ @property(nonatomic, readonly) CMSampleBufferRef _Nonnull buffer; @property(nonatomic, readonly) UIImageOrientation orientation; +// Getters +- (NSString* _Nonnull)pixelFormat; +- (BOOL)isMirrored; +- (BOOL)isValid; +- (size_t)width; +- (size_t)height; +- (double)timestamp; +- (size_t)bytesPerRow; +- (size_t)planesCount; + @end diff --git a/package/ios/Frame Processor/Frame.m b/package/ios/Frame Processor/Frame.m index 0d5e8d7..a441f2d 100644 --- a/package/ios/Frame Processor/Frame.m +++ b/package/ios/Frame Processor/Frame.m @@ -20,11 +20,80 @@ if (self) { _buffer = buffer; _orientation = orientation; + CFRetain(buffer); } return self; } +- (void)dealloc { + CFRelease(_buffer); +} + @synthesize buffer = _buffer; @synthesize orientation = _orientation; +- (NSString*)pixelFormat { + CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(_buffer); + FourCharCode mediaType = CMFormatDescriptionGetMediaSubType(format); + switch (mediaType) { + case kCVPixelFormatType_32BGRA: + case kCVPixelFormatType_Lossy_32BGRA: + return @"rgb"; + case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: + case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: + case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange: + case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange: + case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarFullRange: + case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarVideoRange: + case kCVPixelFormatType_Lossy_420YpCbCr10PackedBiPlanarVideoRange: + return @"yuv"; + default: + return @"unknown"; + } +} + +- (BOOL)isMirrored { + switch (_orientation) { + case UIImageOrientationUp: + case UIImageOrientationDown: + case UIImageOrientationLeft: + case UIImageOrientationRight: + return false; + case UIImageOrientationDownMirrored: + case UIImageOrientationUpMirrored: + case UIImageOrientationLeftMirrored: + case UIImageOrientationRightMirrored: + return true; + } +} + +- (BOOL)isValid { + return _buffer != nil && CMSampleBufferIsValid(_buffer); +} + +- (size_t)width { + CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_buffer); + return CVPixelBufferGetWidth(imageBuffer); +} + +- (size_t)height { + CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_buffer); + return CVPixelBufferGetHeight(imageBuffer); +} + +- (double)timestamp { + CMTime timestamp = CMSampleBufferGetPresentationTimeStamp(_buffer); + return CMTimeGetSeconds(timestamp) * 1000.0; +} + +- (size_t)bytesPerRow { + CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_buffer); + return CVPixelBufferGetBytesPerRow(imageBuffer); +} + +- (size_t)planesCount { + CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_buffer); + return CVPixelBufferGetPlaneCount(imageBuffer); +} + @end diff --git a/package/ios/Frame Processor/FrameHostObject.h b/package/ios/Frame Processor/FrameHostObject.h index 7f9abe6..aa010ba 100644 --- a/package/ios/Frame Processor/FrameHostObject.h +++ b/package/ios/Frame Processor/FrameHostObject.h @@ -10,6 +10,7 @@ #import #import +#import #import "Frame.h" @@ -25,4 +26,11 @@ public: public: Frame* frame; + +private: + Frame* getFrame(); + +private: + std::mutex _mutex; + size_t _refCount = 0; }; diff --git a/package/ios/Frame Processor/FrameHostObject.mm b/package/ios/Frame Processor/FrameHostObject.mm index 23b8838..776be3a 100644 --- a/package/ios/Frame Processor/FrameHostObject.mm +++ b/package/ios/Frame Processor/FrameHostObject.mm @@ -7,6 +7,7 @@ // #import "FrameHostObject.h" +#import "UIImageOrientation+descriptor.h" #import "WKTJsiHostObject.h" #import #import @@ -34,42 +35,70 @@ std::vector FrameHostObject::getPropertyNames(jsi::Runtime& rt) return result; } +Frame* FrameHostObject::getFrame() { + Frame* frame = this->frame; + if (frame == nil || !CMSampleBufferIsValid(frame.buffer)) { + throw std::runtime_error("Frame is already closed! " + "Are you trying to access the Image data outside of a Frame Processor's lifetime?\n" + "- If you want to use `console.log(frame)`, use `console.log(frame.toString())` instead.\n" + "- If you want to do async processing, use `runAsync(...)` instead.\n" + "- If you want to use runOnJS, increment it's ref-count: `frame.incrementRefCount()`"); + } + return frame; +} + jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) { auto name = propName.utf8(runtime); if (name == "toString") { - auto toString = JSI_HOST_FUNCTION_LAMBDA { - if (this->frame == nil) { - return jsi::String::createFromUtf8(runtime, "[closed frame]"); - } - auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer); - auto width = CVPixelBufferGetWidth(imageBuffer); - auto height = CVPixelBufferGetHeight(imageBuffer); + auto toString = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value { + // Lock Frame so it cannot be deallocated while we access it + std::lock_guard lock(this->_mutex); - NSMutableString* string = [NSMutableString stringWithFormat:@"%lu x %lu Frame", width, height]; + // Print debug description (width, height) + Frame* frame = this->getFrame(); + NSMutableString* string = [NSMutableString stringWithFormat:@"%lu x %lu %@ Frame", frame.width, frame.height, frame.pixelFormat]; return jsi::String::createFromUtf8(runtime, string.UTF8String); }; return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString); } if (name == "incrementRefCount") { - auto incrementRefCount = JSI_HOST_FUNCTION_LAMBDA { - // Increment retain count by one so ARC doesn't destroy the Frame Buffer. - CFRetain(frame.buffer); + auto incrementRefCount = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, + size_t count) -> jsi::Value { + // Lock Frame so it cannot be deallocated while we access it + std::lock_guard lock(this->_mutex); + + // Increment our self-counted ref count by one. + _refCount++; return jsi::Value::undefined(); }; return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "incrementRefCount"), 0, incrementRefCount); } if (name == "decrementRefCount") { - auto decrementRefCount = JSI_HOST_FUNCTION_LAMBDA { - // Decrement retain count by one. If the retain count is zero, ARC will destroy the Frame - // Buffer. - CFRelease(frame.buffer); + auto decrementRefCount = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, + size_t count) -> jsi::Value { + // Lock Frame so it cannot be deallocated while we access it + std::lock_guard lock(this->_mutex); + + // Decrement our self-counted ref count by one. + _refCount--; + if (_refCount < 1) { + // ARC will then delete the Frame and the underlying Frame Buffer. + this->frame = nil; + } + return jsi::Value::undefined(); }; return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "decrementRefCount"), 0, decrementRefCount); } if (name == "toArrayBuffer") { - auto toArrayBuffer = JSI_HOST_FUNCTION_LAMBDA { + auto toArrayBuffer = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, + size_t count) -> jsi::Value { + // Lock Frame so it cannot be deallocated while we access it + std::lock_guard lock(this->_mutex); + + // Get CPU readable Pixel Buffer from Frame and write it to a jsi::ArrayBuffer + Frame* frame = this->getFrame(); auto pixelBuffer = CMSampleBufferGetImageBuffer(frame.buffer); auto bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer); auto height = CVPixelBufferGetHeight(pixelBuffer); @@ -101,82 +130,69 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr } if (name == "isValid") { - auto isValid = frame != nil && frame.buffer != nil && CFGetRetainCount(frame.buffer) > 0 && CMSampleBufferIsValid(frame.buffer); - return jsi::Value(isValid); + // Lock Frame so it cannot be deallocated while we access it + std::lock_guard lock(this->_mutex); + + // unsafely access the Frame and try to see if it's valid + Frame* frame = this->frame; + return jsi::Value(frame != nil && frame.isValid); } if (name == "width") { - auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer); - auto width = CVPixelBufferGetWidth(imageBuffer); - return jsi::Value((double)width); + // Lock Frame so it cannot be deallocated while we access it + std::lock_guard lock(this->_mutex); + + Frame* frame = this->getFrame(); + return jsi::Value((double)frame.width); } if (name == "height") { - auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer); - auto height = CVPixelBufferGetHeight(imageBuffer); - return jsi::Value((double)height); + // Lock Frame so it cannot be deallocated while we access it + std::lock_guard lock(this->_mutex); + + Frame* frame = this->getFrame(); + return jsi::Value((double)frame.height); } if (name == "orientation") { - switch (frame.orientation) { - case UIImageOrientationUp: - case UIImageOrientationUpMirrored: - return jsi::String::createFromUtf8(runtime, "portrait"); - case UIImageOrientationDown: - case UIImageOrientationDownMirrored: - return jsi::String::createFromUtf8(runtime, "portrait-upside-down"); - case UIImageOrientationLeft: - case UIImageOrientationLeftMirrored: - return jsi::String::createFromUtf8(runtime, "landscape-left"); - case UIImageOrientationRight: - case UIImageOrientationRightMirrored: - return jsi::String::createFromUtf8(runtime, "landscape-right"); - } + // Lock Frame so it cannot be deallocated while we access it + std::lock_guard lock(this->_mutex); + + Frame* frame = this->getFrame(); + NSString* orientation = [NSString stringWithParsed:frame.orientation]; + return jsi::String::createFromUtf8(runtime, orientation.UTF8String); } if (name == "isMirrored") { - switch (frame.orientation) { - case UIImageOrientationUp: - case UIImageOrientationDown: - case UIImageOrientationLeft: - case UIImageOrientationRight: - return jsi::Value(false); - case UIImageOrientationDownMirrored: - case UIImageOrientationUpMirrored: - case UIImageOrientationLeftMirrored: - case UIImageOrientationRightMirrored: - return jsi::Value(true); - } + // Lock Frame so it cannot be deallocated while we access it + std::lock_guard lock(this->_mutex); + + Frame* frame = this->getFrame(); + return jsi::Value(frame.isMirrored); } if (name == "timestamp") { - auto timestamp = CMSampleBufferGetPresentationTimeStamp(frame.buffer); - auto seconds = static_cast(CMTimeGetSeconds(timestamp)); - return jsi::Value(seconds * 1000.0); + // Lock Frame so it cannot be deallocated while we access it + std::lock_guard lock(this->_mutex); + + Frame* frame = this->getFrame(); + return jsi::Value(frame.timestamp); } if (name == "pixelFormat") { - auto format = CMSampleBufferGetFormatDescription(frame.buffer); - auto mediaType = CMFormatDescriptionGetMediaSubType(format); - switch (mediaType) { - case kCVPixelFormatType_32BGRA: - case kCVPixelFormatType_Lossy_32BGRA: - return jsi::String::createFromUtf8(runtime, "rgb"); - case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: - case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: - case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange: - case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange: - case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarFullRange: - case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarVideoRange: - case kCVPixelFormatType_Lossy_420YpCbCr10PackedBiPlanarVideoRange: - return jsi::String::createFromUtf8(runtime, "yuv"); - default: - return jsi::String::createFromUtf8(runtime, "unknown"); - } + // Lock Frame so it cannot be deallocated while we access it + std::lock_guard lock(this->_mutex); + + Frame* frame = this->getFrame(); + return jsi::String::createFromUtf8(runtime, frame.pixelFormat.UTF8String); } if (name == "bytesPerRow") { - auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer); - auto bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer); - return jsi::Value((double)bytesPerRow); + // Lock Frame so it cannot be deallocated while we access it + std::lock_guard lock(this->_mutex); + + Frame* frame = this->getFrame(); + return jsi::Value((double)frame.bytesPerRow); } if (name == "planesCount") { - auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer); - auto planesCount = CVPixelBufferGetPlaneCount(imageBuffer); - return jsi::Value((double)planesCount); + // Lock Frame so it cannot be deallocated while we access it + std::lock_guard lock(this->_mutex); + + Frame* frame = this->getFrame(); + return jsi::Value((double)frame.planesCount); } // fallback to base implementation diff --git a/package/ios/Frame Processor/UIImageOrientation+descriptor.h b/package/ios/Frame Processor/UIImageOrientation+descriptor.h new file mode 100644 index 0000000..f65b68f --- /dev/null +++ b/package/ios/Frame Processor/UIImageOrientation+descriptor.h @@ -0,0 +1,39 @@ +// +// UIImageOrientation+descriptor.h +// VisionCamera +// +// Created by Marc Rousavy on 29.12.23. +// Copyright © 2023 mrousavy. All rights reserved. +// + +#pragma once + +#import +#import + +@interface NSString (UIImageOrientationJSDescriptor) + ++ (NSString*)stringWithParsed:(UIImageOrientation)orientation; + +@end + +@implementation NSString (UIImageOrientationJSDescriptor) + ++ (NSString*)stringWithParsed:(UIImageOrientation)orientation { + switch (orientation) { + case UIImageOrientationUp: + case UIImageOrientationUpMirrored: + return @"portrait"; + case UIImageOrientationDown: + case UIImageOrientationDownMirrored: + return @"portrait-upside-down"; + case UIImageOrientationLeft: + case UIImageOrientationLeftMirrored: + return @"landscape-left"; + case UIImageOrientationRight: + case UIImageOrientationRightMirrored: + return @"landscape-right"; + } +} + +@end diff --git a/package/ios/VisionCamera.xcodeproj/project.pbxproj b/package/ios/VisionCamera.xcodeproj/project.pbxproj index 768739e..d6966c2 100644 --- a/package/ios/VisionCamera.xcodeproj/project.pbxproj +++ b/package/ios/VisionCamera.xcodeproj/project.pbxproj @@ -160,6 +160,7 @@ B887518425E0102000DB86D6 /* CameraView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CameraView.swift; sourceTree = ""; }; B88873E5263D46C7008B1D0E /* FrameProcessorPlugin.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorPlugin.h; sourceTree = ""; }; B8994E6B263F03E100069589 /* JSINSObjectConversion.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = JSINSObjectConversion.mm; sourceTree = ""; }; + B89A79692B3EF60F005E0357 /* UIImageOrientation+descriptor.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "UIImageOrientation+descriptor.h"; sourceTree = ""; }; B8A1AEC32AD7EDE800169C0D /* AVCaptureVideoDataOutput+pixelFormat.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVCaptureVideoDataOutput+pixelFormat.swift"; sourceTree = ""; }; B8A1AEC52AD7F08E00169C0D /* CameraView+Focus.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CameraView+Focus.swift"; sourceTree = ""; }; B8A1AEC72AD8005400169C0D /* CameraSession+Configuration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CameraSession+Configuration.swift"; sourceTree = ""; }; @@ -334,6 +335,7 @@ B81D41EF263C86F900B041FD /* JSINSObjectConversion.h */, B8994E6B263F03E100069589 /* JSINSObjectConversion.mm */, B85F7AE82A77BB680089C539 /* FrameProcessorPlugin.m */, + B89A79692B3EF60F005E0357 /* UIImageOrientation+descriptor.h */, ); path = "Frame Processor"; sourceTree = "";