feat: Make Frame thread-safe and improve error messages (#2327)

* fix: Fix multi-Thread access on Java

* fix: Thread-lock access on iOS as well

* whoops add missing header impl

* Update Podfile.lock

* fix: Don't use `CFGetRetainCount`

* fix: Lock access on iOS as well

* C++ format

* More detailed error

* chore: Move getters into `Frame`

* Format c++

* Use enum `orientation` again

* format

* fix: Synchronize `isValid` on Java

* Also log pixelformat

* feat: Use Java enums in C++

* Format C++

package/android/src/main/cpp/frameprocessor/FrameHostObject.cpp

@@ -77,7 +77,9 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
       }
       auto width = this->frame->getWidth();
       auto height = this->frame->getHeight();
-      auto str = std::to_string(width) + " x " + std::to_string(height) + " Frame";
+      auto format = this->frame->getPixelFormat();
+      auto formatString = format->getUnionValue();
+      auto str = std::to_string(width) + " x " + std::to_string(height) + " " + formatString->toString() + " Frame";
       return jsi::String::createFromUtf8(runtime, str);
     };
     return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
@@ -141,11 +143,13 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
     return jsi::Value(this->frame->getIsMirrored());
   }
   if (name == "orientation") {
-    auto string = this->frame->getOrientation();
+    auto orientation = this->frame->getOrientation();
+    auto string = orientation->getUnionValue();
     return jsi::String::createFromUtf8(runtime, string->toStdString());
   }
   if (name == "pixelFormat") {
-    auto string = this->frame->getPixelFormat();
+    auto pixelFormat = this->frame->getPixelFormat();
+    auto string = pixelFormat->getUnionValue();
     return jsi::String::createFromUtf8(runtime, string->toStdString());
   }
   if (name == "timestamp") {

package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.cpp

@@ -4,6 +4,8 @@
 
 #include "JFrame.h"
 
+#include "JOrientation.h"
+#include "JPixelFormat.h"
 #include <fbjni/fbjni.h>
 #include <jni.h>
 
@@ -39,13 +41,13 @@ jlong JFrame::getTimestamp() const {
   return getTimestampMethod(self());
 }
 
-local_ref<JString> JFrame::getOrientation() const {
-  static const auto getOrientationMethod = getClass()->getMethod<JString()>("getOrientation");
+local_ref<JOrientation> JFrame::getOrientation() const {
+  static const auto getOrientationMethod = getClass()->getMethod<JOrientation()>("getOrientation");
   return getOrientationMethod(self());
 }
 
-local_ref<JString> JFrame::getPixelFormat() const {
-  static const auto getPixelFormatMethod = getClass()->getMethod<JString()>("getPixelFormat");
+local_ref<JPixelFormat> JFrame::getPixelFormat() const {
+  static const auto getPixelFormatMethod = getClass()->getMethod<JPixelFormat()>("getPixelFormat");
   return getPixelFormatMethod(self());
 }
 
@@ -77,9 +79,4 @@ void JFrame::decrementRefCount() {
   decrementRefCountMethod(self());
 }
 
-void JFrame::close() {
-  static const auto closeMethod = getClass()->getMethod<void()>("close");
-  closeMethod(self());
-}
-
 } // namespace vision

package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.h

@@ -4,6 +4,8 @@
 
 #pragma once
 
+#include "JOrientation.h"
+#include "JPixelFormat.h"
 #include <fbjni/fbjni.h>
 #include <jni.h>
 
@@ -25,15 +27,14 @@ public:
   int getPlanesCount() const;
   int getBytesPerRow() const;
   jlong getTimestamp() const;
-  local_ref<JString> getOrientation() const;
-  local_ref<JString> getPixelFormat() const;
+  local_ref<JOrientation> getOrientation() const;
+  local_ref<JPixelFormat> getPixelFormat() const;
 #if __ANDROID_API__ >= 26
   AHardwareBuffer* getHardwareBuffer() const;
 #endif
 
   void incrementRefCount();
   void decrementRefCount();
-  void close();
 };
 
 } // namespace vision

package/android/src/main/cpp/frameprocessor/java-bindings/JJSUnionValue.h

@@ -0,0 +1,24 @@
+//
+// Created by Marc Rousavy on 29.12.23.
+//
+
+#pragma once
+
+#include <fbjni/fbjni.h>
+#include <jni.h>
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+struct JJSUnionValue : public JavaClass<JJSUnionValue> {
+  static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/types/JSUnionValue;";
+
+  local_ref<JString> getUnionValue() {
+    const auto getUnionValueMethod = getClass()->getMethod<JString()>("getUnionValue");
+    return getUnionValueMethod(self());
+  }
+};
+
+} // namespace vision

package/android/src/main/cpp/frameprocessor/java-bindings/JOrientation.h

@@ -0,0 +1,20 @@
+//
+// Created by Marc Rousavy on 29.12.23.
+//
+
+#pragma once
+
+#include "JJSUnionValue.h"
+#include <fbjni/fbjni.h>
+#include <jni.h>
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+struct JOrientation : public JavaClass<JOrientation, JJSUnionValue> {
+  static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/types/Orientation;";
+};
+
+} // namespace vision

package/android/src/main/cpp/frameprocessor/java-bindings/JPixelFormat.h

@@ -0,0 +1,20 @@
+//
+// Created by Marc Rousavy on 29.12.23.
+//
+
+#pragma once
+
+#include "JJSUnionValue.h"
+#include <fbjni/fbjni.h>
+#include <jni.h>
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+struct JPixelFormat : public JavaClass<JPixelFormat, JJSUnionValue> {
+  static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/types/PixelFormat;";
+};
+
+} // namespace vision

package/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java

@@ -7,6 +7,7 @@ import com.facebook.proguard.annotations.DoNotStrip;
 import com.mrousavy.camera.core.HardwareBuffersNotAvailableError;
 import com.mrousavy.camera.types.PixelFormat;
 import com.mrousavy.camera.types.Orientation;
+import java.lang.IllegalStateException;
 
 public class Frame {
     private final Image image;
@@ -24,30 +25,44 @@ public class Frame {
     }
 
     public Image getImage() {
-        return image;
+        synchronized (this) {
+            Image img = image;
+            if (!getIsImageValid(img)) {
+                throw new RuntimeException("Frame is already closed! " +
+                    "Are you trying to access the Image data outside of a Frame Processor's lifetime?\n" +
+                    "- If you want to use `console.log(frame)`, use `console.log(frame.toString())` instead.\n" +
+                    "- If you want to do async processing, use `runAsync(...)` instead.\n" +
+                    "- If you want to use runOnJS, increment it's ref-count: `frame.incrementRefCount()`");
+            }
+            return img;
+        }
     }
 
     @SuppressWarnings("unused")
     @DoNotStrip
     public int getWidth() {
-        return image.getWidth();
+        return getImage().getWidth();
     }
 
     @SuppressWarnings("unused")
     @DoNotStrip
     public int getHeight() {
-        return image.getHeight();
+        return getImage().getHeight();
     }
 
     @SuppressWarnings("unused")
     @DoNotStrip
     public boolean getIsValid() {
+        return getIsImageValid(getImage());
+    }
+
+    private boolean getIsImageValid(Image image) {
         try {
             // will throw an exception if the image is already closed
-            image.getCropRect();
+            synchronized (this) { image.getFormat(); }
             // no exception thrown, image must still be valid.
             return true;
-        } catch (Exception e) {
+        } catch (IllegalStateException e) {
             // exception thrown, image has already been closed.
             return false;
         }
@@ -67,27 +82,26 @@ public class Frame {
 
     @SuppressWarnings("unused")
     @DoNotStrip
-    public String getOrientation() {
-        return orientation.getUnionValue();
+    public Orientation getOrientation() {
+        return orientation;
     }
 
     @SuppressWarnings("unused")
     @DoNotStrip
-    public String getPixelFormat() {
-        PixelFormat format = PixelFormat.Companion.fromImageFormat(image.getFormat());
-        return format.getUnionValue();
+    public PixelFormat getPixelFormat() {
+        return PixelFormat.Companion.fromImageFormat(getImage().getFormat());
     }
 
     @SuppressWarnings("unused")
     @DoNotStrip
     public int getPlanesCount() {
-        return image.getPlanes().length;
+        return getImage().getPlanes().length;
     }
 
     @SuppressWarnings("unused")
     @DoNotStrip
     public int getBytesPerRow() {
-        return image.getPlanes()[0].getRowStride();
+        return getImage().getPlanes()[0].getRowStride();
     }
 
     @SuppressWarnings("unused")
@@ -101,7 +115,7 @@ public class Frame {
             throw new HardwareBuffersNotAvailableError();
         }
         if (hardwareBuffer == null) {
-            hardwareBuffer = image.getHardwareBuffer();
+            hardwareBuffer = getImage().getHardwareBuffer();
         }
         return hardwareBuffer;
     }
@@ -121,17 +135,17 @@ public class Frame {
             refCount--;
             if (refCount <= 0) {
                 // If no reference is held on this Image, close it.
-                image.close();
+                close();
             }
         }
     }
 
-    @SuppressWarnings("unused")
-    @DoNotStrip
-    private void close() {
-        if (hardwareBuffer != null) {
-            hardwareBuffer.close();
+    private synchronized void close() {
+        synchronized (this) {
+            if (hardwareBuffer != null) {
+                hardwareBuffer.close();
+            }
+            image.close();
         }
-        image.close();
     }
 }

package/example/ios/Podfile.lock

@@ -484,7 +484,7 @@ PODS:
     - libwebp (~> 1.0)
     - SDWebImage/Core (~> 5.10)
   - SocketRocket (0.6.1)
-  - VisionCamera (3.6.16):
+  - VisionCamera (3.6.17):
     - React
     - React-callinvoker
     - React-Core
@@ -724,9 +724,9 @@ SPEC CHECKSUMS:
   SDWebImage: a7f831e1a65eb5e285e3fb046a23fcfbf08e696d
   SDWebImageWebPCoder: 908b83b6adda48effe7667cd2b7f78c897e5111d
   SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17
-  VisionCamera: 06f5b990082ec783a58dcadc5bac677974f7a580
+  VisionCamera: 361df29347b7b7ecc47b3d173daa17751a11ffc1
   Yoga: 4c3aa327e4a6a23eeacd71f61c81df1bcdf677d5
 
 PODFILE CHECKSUM: 27f53791141a3303d814e09b55770336416ff4eb
 
-COCOAPODS: 1.11.3
+COCOAPODS: 1.14.3

package/ios/Frame Processor/Frame.h

@@ -19,4 +19,14 @@
 @property(nonatomic, readonly) CMSampleBufferRef _Nonnull buffer;
 @property(nonatomic, readonly) UIImageOrientation orientation;
 
+// Getters
+- (NSString* _Nonnull)pixelFormat;
+- (BOOL)isMirrored;
+- (BOOL)isValid;
+- (size_t)width;
+- (size_t)height;
+- (double)timestamp;
+- (size_t)bytesPerRow;
+- (size_t)planesCount;
+
 @end

package/ios/Frame Processor/Frame.m

@@ -20,11 +20,80 @@
   if (self) {
     _buffer = buffer;
     _orientation = orientation;
+    CFRetain(buffer);
   }
   return self;
 }
 
+- (void)dealloc {
+  CFRelease(_buffer);
+}
+
 @synthesize buffer = _buffer;
 @synthesize orientation = _orientation;
 
+- (NSString*)pixelFormat {
+  CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(_buffer);
+  FourCharCode mediaType = CMFormatDescriptionGetMediaSubType(format);
+  switch (mediaType) {
+    case kCVPixelFormatType_32BGRA:
+    case kCVPixelFormatType_Lossy_32BGRA:
+      return @"rgb";
+    case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
+    case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
+    case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange:
+    case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange:
+    case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarFullRange:
+    case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarVideoRange:
+    case kCVPixelFormatType_Lossy_420YpCbCr10PackedBiPlanarVideoRange:
+      return @"yuv";
+    default:
+      return @"unknown";
+  }
+}
+
+- (BOOL)isMirrored {
+  switch (_orientation) {
+    case UIImageOrientationUp:
+    case UIImageOrientationDown:
+    case UIImageOrientationLeft:
+    case UIImageOrientationRight:
+      return false;
+    case UIImageOrientationDownMirrored:
+    case UIImageOrientationUpMirrored:
+    case UIImageOrientationLeftMirrored:
+    case UIImageOrientationRightMirrored:
+      return true;
+  }
+}
+
+- (BOOL)isValid {
+  return _buffer != nil && CMSampleBufferIsValid(_buffer);
+}
+
+- (size_t)width {
+  CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_buffer);
+  return CVPixelBufferGetWidth(imageBuffer);
+}
+
+- (size_t)height {
+  CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_buffer);
+  return CVPixelBufferGetHeight(imageBuffer);
+}
+
+- (double)timestamp {
+  CMTime timestamp = CMSampleBufferGetPresentationTimeStamp(_buffer);
+  return CMTimeGetSeconds(timestamp) * 1000.0;
+}
+
+- (size_t)bytesPerRow {
+  CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_buffer);
+  return CVPixelBufferGetBytesPerRow(imageBuffer);
+}
+
+- (size_t)planesCount {
+  CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_buffer);
+  return CVPixelBufferGetPlaneCount(imageBuffer);
+}
+
 @end

package/ios/Frame Processor/FrameHostObject.h

@@ -10,6 +10,7 @@
 
 #import <CoreMedia/CMSampleBuffer.h>
 #import <jsi/jsi.h>
+#import <mutex>
 
 #import "Frame.h"
 
@@ -25,4 +26,11 @@ public:
 
 public:
   Frame* frame;
+
+private:
+  Frame* getFrame();
+
+private:
+  std::mutex _mutex;
+  size_t _refCount = 0;
 };

package/ios/Frame Processor/FrameHostObject.mm

@@ -7,6 +7,7 @@
 //
 
 #import "FrameHostObject.h"
+#import "UIImageOrientation+descriptor.h"
 #import "WKTJsiHostObject.h"
 #import <Foundation/Foundation.h>
 #import <jsi/jsi.h>
@@ -34,42 +35,70 @@ std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt)
   return result;
 }
 
+Frame* FrameHostObject::getFrame() {
+  Frame* frame = this->frame;
+  if (frame == nil || !CMSampleBufferIsValid(frame.buffer)) {
+    throw std::runtime_error("Frame is already closed! "
+                             "Are you trying to access the Image data outside of a Frame Processor's lifetime?\n"
+                             "- If you want to use `console.log(frame)`, use `console.log(frame.toString())` instead.\n"
+                             "- If you want to do async processing, use `runAsync(...)` instead.\n"
+                             "- If you want to use runOnJS, increment it's ref-count: `frame.incrementRefCount()`");
+  }
+  return frame;
+}
+
 jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
   auto name = propName.utf8(runtime);
 
   if (name == "toString") {
-    auto toString = JSI_HOST_FUNCTION_LAMBDA {
-      if (this->frame == nil) {
-        return jsi::String::createFromUtf8(runtime, "[closed frame]");
-      }
-      auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
-      auto width = CVPixelBufferGetWidth(imageBuffer);
-      auto height = CVPixelBufferGetHeight(imageBuffer);
+    auto toString = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
+      // Lock Frame so it cannot be deallocated while we access it
+      std::lock_guard lock(this->_mutex);
 
-      NSMutableString* string = [NSMutableString stringWithFormat:@"%lu x %lu Frame", width, height];
+      // Print debug description (width, height)
+      Frame* frame = this->getFrame();
+      NSMutableString* string = [NSMutableString stringWithFormat:@"%lu x %lu %@ Frame", frame.width, frame.height, frame.pixelFormat];
       return jsi::String::createFromUtf8(runtime, string.UTF8String);
     };
     return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
   }
   if (name == "incrementRefCount") {
-    auto incrementRefCount = JSI_HOST_FUNCTION_LAMBDA {
-      // Increment retain count by one so ARC doesn't destroy the Frame Buffer.
-      CFRetain(frame.buffer);
+    auto incrementRefCount = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
+                                    size_t count) -> jsi::Value {
+      // Lock Frame so it cannot be deallocated while we access it
+      std::lock_guard lock(this->_mutex);
+
+      // Increment our self-counted ref count by one.
+      _refCount++;
       return jsi::Value::undefined();
     };
     return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "incrementRefCount"), 0, incrementRefCount);
   }
   if (name == "decrementRefCount") {
-    auto decrementRefCount = JSI_HOST_FUNCTION_LAMBDA {
-      // Decrement retain count by one. If the retain count is zero, ARC will destroy the Frame
-      // Buffer.
-      CFRelease(frame.buffer);
+    auto decrementRefCount = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
+                                    size_t count) -> jsi::Value {
+      // Lock Frame so it cannot be deallocated while we access it
+      std::lock_guard lock(this->_mutex);
+
+      // Decrement our self-counted ref count by one.
+      _refCount--;
+      if (_refCount < 1) {
+        // ARC will then delete the Frame and the underlying Frame Buffer.
+        this->frame = nil;
+      }
+
       return jsi::Value::undefined();
     };
     return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "decrementRefCount"), 0, decrementRefCount);
   }
   if (name == "toArrayBuffer") {
-    auto toArrayBuffer = JSI_HOST_FUNCTION_LAMBDA {
+    auto toArrayBuffer = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
+                                size_t count) -> jsi::Value {
+      // Lock Frame so it cannot be deallocated while we access it
+      std::lock_guard lock(this->_mutex);
+
+      // Get CPU readable Pixel Buffer from Frame and write it to a jsi::ArrayBuffer
+      Frame* frame = this->getFrame();
       auto pixelBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
       auto bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer);
       auto height = CVPixelBufferGetHeight(pixelBuffer);
@@ -101,82 +130,69 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
   }
 
   if (name == "isValid") {
-    auto isValid = frame != nil && frame.buffer != nil && CFGetRetainCount(frame.buffer) > 0 && CMSampleBufferIsValid(frame.buffer);
-    return jsi::Value(isValid);
+    // Lock Frame so it cannot be deallocated while we access it
+    std::lock_guard lock(this->_mutex);
+
+    // unsafely access the Frame and try to see if it's valid
+    Frame* frame = this->frame;
+    return jsi::Value(frame != nil && frame.isValid);
   }
   if (name == "width") {
-    auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
-    auto width = CVPixelBufferGetWidth(imageBuffer);
-    return jsi::Value((double)width);
+    // Lock Frame so it cannot be deallocated while we access it
+    std::lock_guard lock(this->_mutex);
+
+    Frame* frame = this->getFrame();
+    return jsi::Value((double)frame.width);
   }
   if (name == "height") {
-    auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
-    auto height = CVPixelBufferGetHeight(imageBuffer);
-    return jsi::Value((double)height);
+    // Lock Frame so it cannot be deallocated while we access it
+    std::lock_guard lock(this->_mutex);
+
+    Frame* frame = this->getFrame();
+    return jsi::Value((double)frame.height);
   }
   if (name == "orientation") {
-    switch (frame.orientation) {
-      case UIImageOrientationUp:
-      case UIImageOrientationUpMirrored:
-        return jsi::String::createFromUtf8(runtime, "portrait");
-      case UIImageOrientationDown:
-      case UIImageOrientationDownMirrored:
-        return jsi::String::createFromUtf8(runtime, "portrait-upside-down");
-      case UIImageOrientationLeft:
-      case UIImageOrientationLeftMirrored:
-        return jsi::String::createFromUtf8(runtime, "landscape-left");
-      case UIImageOrientationRight:
-      case UIImageOrientationRightMirrored:
-        return jsi::String::createFromUtf8(runtime, "landscape-right");
-    }
+    // Lock Frame so it cannot be deallocated while we access it
+    std::lock_guard lock(this->_mutex);
+
+    Frame* frame = this->getFrame();
+    NSString* orientation = [NSString stringWithParsed:frame.orientation];
+    return jsi::String::createFromUtf8(runtime, orientation.UTF8String);
   }
   if (name == "isMirrored") {
-    switch (frame.orientation) {
-      case UIImageOrientationUp:
-      case UIImageOrientationDown:
-      case UIImageOrientationLeft:
-      case UIImageOrientationRight:
-        return jsi::Value(false);
-      case UIImageOrientationDownMirrored:
-      case UIImageOrientationUpMirrored:
-      case UIImageOrientationLeftMirrored:
-      case UIImageOrientationRightMirrored:
-        return jsi::Value(true);
-    }
+    // Lock Frame so it cannot be deallocated while we access it
+    std::lock_guard lock(this->_mutex);
+
+    Frame* frame = this->getFrame();
+    return jsi::Value(frame.isMirrored);
   }
   if (name == "timestamp") {
-    auto timestamp = CMSampleBufferGetPresentationTimeStamp(frame.buffer);
-    auto seconds = static_cast<double>(CMTimeGetSeconds(timestamp));
-    return jsi::Value(seconds * 1000.0);
+    // Lock Frame so it cannot be deallocated while we access it
+    std::lock_guard lock(this->_mutex);
+
+    Frame* frame = this->getFrame();
+    return jsi::Value(frame.timestamp);
   }
   if (name == "pixelFormat") {
-    auto format = CMSampleBufferGetFormatDescription(frame.buffer);
-    auto mediaType = CMFormatDescriptionGetMediaSubType(format);
-    switch (mediaType) {
-      case kCVPixelFormatType_32BGRA:
-      case kCVPixelFormatType_Lossy_32BGRA:
-        return jsi::String::createFromUtf8(runtime, "rgb");
-      case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
-      case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
-      case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange:
-      case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange:
-      case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarFullRange:
-      case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarVideoRange:
-      case kCVPixelFormatType_Lossy_420YpCbCr10PackedBiPlanarVideoRange:
-        return jsi::String::createFromUtf8(runtime, "yuv");
-      default:
-        return jsi::String::createFromUtf8(runtime, "unknown");
-    }
+    // Lock Frame so it cannot be deallocated while we access it
+    std::lock_guard lock(this->_mutex);
+
+    Frame* frame = this->getFrame();
+    return jsi::String::createFromUtf8(runtime, frame.pixelFormat.UTF8String);
   }
   if (name == "bytesPerRow") {
-    auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
-    auto bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer);
-    return jsi::Value((double)bytesPerRow);
+    // Lock Frame so it cannot be deallocated while we access it
+    std::lock_guard lock(this->_mutex);
+
+    Frame* frame = this->getFrame();
+    return jsi::Value((double)frame.bytesPerRow);
   }
   if (name == "planesCount") {
-    auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
-    auto planesCount = CVPixelBufferGetPlaneCount(imageBuffer);
-    return jsi::Value((double)planesCount);
+    // Lock Frame so it cannot be deallocated while we access it
+    std::lock_guard lock(this->_mutex);
+
+    Frame* frame = this->getFrame();
+    return jsi::Value((double)frame.planesCount);
   }
 
   // fallback to base implementation

package/ios/Frame Processor/UIImageOrientation+descriptor.h

@@ -0,0 +1,39 @@
+//
+//  UIImageOrientation+descriptor.h
+//  VisionCamera
+//
+//  Created by Marc Rousavy on 29.12.23.
+//  Copyright © 2023 mrousavy. All rights reserved.
+//
+
+#pragma once
+
+#import <Foundation/Foundation.h>
+#import <UIKit/UIImage.h>
+
+@interface NSString (UIImageOrientationJSDescriptor)
+
++ (NSString*)stringWithParsed:(UIImageOrientation)orientation;
+
+@end
+
+@implementation NSString (UIImageOrientationJSDescriptor)
+
++ (NSString*)stringWithParsed:(UIImageOrientation)orientation {
+  switch (orientation) {
+    case UIImageOrientationUp:
+    case UIImageOrientationUpMirrored:
+      return @"portrait";
+    case UIImageOrientationDown:
+    case UIImageOrientationDownMirrored:
+      return @"portrait-upside-down";
+    case UIImageOrientationLeft:
+    case UIImageOrientationLeftMirrored:
+      return @"landscape-left";
+    case UIImageOrientationRight:
+    case UIImageOrientationRightMirrored:
+      return @"landscape-right";
+  }
+}
+
+@end

package/ios/VisionCamera.xcodeproj/project.pbxproj

@@ -160,6 +160,7 @@
 		B887518425E0102000DB86D6 /* CameraView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CameraView.swift; sourceTree = "<group>"; };
 		B88873E5263D46C7008B1D0E /* FrameProcessorPlugin.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorPlugin.h; sourceTree = "<group>"; };
 		B8994E6B263F03E100069589 /* JSINSObjectConversion.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = JSINSObjectConversion.mm; sourceTree = "<group>"; };
+		B89A79692B3EF60F005E0357 /* UIImageOrientation+descriptor.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "UIImageOrientation+descriptor.h"; sourceTree = "<group>"; };
 		B8A1AEC32AD7EDE800169C0D /* AVCaptureVideoDataOutput+pixelFormat.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVCaptureVideoDataOutput+pixelFormat.swift"; sourceTree = "<group>"; };
 		B8A1AEC52AD7F08E00169C0D /* CameraView+Focus.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CameraView+Focus.swift"; sourceTree = "<group>"; };
 		B8A1AEC72AD8005400169C0D /* CameraSession+Configuration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CameraSession+Configuration.swift"; sourceTree = "<group>"; };
@@ -334,6 +335,7 @@
 				B81D41EF263C86F900B041FD /* JSINSObjectConversion.h */,
 				B8994E6B263F03E100069589 /* JSINSObjectConversion.mm */,
 				B85F7AE82A77BB680089C539 /* FrameProcessorPlugin.m */,
+				B89A79692B3EF60F005E0357 /* UIImageOrientation+descriptor.h */,
 			);
 			path = "Frame Processor";
 			sourceTree = "<group>";