feat: Make Frame thread-safe and improve error messages (#2327)

* fix: Fix multi-Thread access on Java

* fix: Thread-lock access on iOS as well

* whoops add missing header impl

* Update Podfile.lock

* fix: Don't use `CFGetRetainCount`

* fix: Lock access on iOS as well

* C++ format

* More detailed error

* chore: Move getters into `Frame`

* Format c++

* Use enum `orientation` again

* format

* fix: Synchronize `isValid` on Java

* Also log pixelformat

* feat: Use Java enums in C++

* Format C++

package/android/src/main/cpp/frameprocessor/FrameHostObject.cpp

@@ -77,7 +77,9 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
       }
       auto width = this->frame->getWidth();
       auto height = this->frame->getHeight();
-      auto str = std::to_string(width) + " x " + std::to_string(height) + " Frame";
+      auto format = this->frame->getPixelFormat();
+      auto formatString = format->getUnionValue();
+      auto str = std::to_string(width) + " x " + std::to_string(height) + " " + formatString->toString() + " Frame";
       return jsi::String::createFromUtf8(runtime, str);
     };
     return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
@@ -141,11 +143,13 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
     return jsi::Value(this->frame->getIsMirrored());
   }
   if (name == "orientation") {
-    auto string = this->frame->getOrientation();
+    auto orientation = this->frame->getOrientation();
+    auto string = orientation->getUnionValue();
     return jsi::String::createFromUtf8(runtime, string->toStdString());
   }
   if (name == "pixelFormat") {
-    auto string = this->frame->getPixelFormat();
+    auto pixelFormat = this->frame->getPixelFormat();
+    auto string = pixelFormat->getUnionValue();
     return jsi::String::createFromUtf8(runtime, string->toStdString());
   }
   if (name == "timestamp") {

package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.cpp

@@ -4,6 +4,8 @@
 
 #include "JFrame.h"
 
+#include "JOrientation.h"
+#include "JPixelFormat.h"
 #include <fbjni/fbjni.h>
 #include <jni.h>
 
@@ -39,13 +41,13 @@ jlong JFrame::getTimestamp() const {
   return getTimestampMethod(self());
 }
 
-local_ref<JString> JFrame::getOrientation() const {
-  static const auto getOrientationMethod = getClass()->getMethod<JString()>("getOrientation");
+local_ref<JOrientation> JFrame::getOrientation() const {
+  static const auto getOrientationMethod = getClass()->getMethod<JOrientation()>("getOrientation");
   return getOrientationMethod(self());
 }
 
-local_ref<JString> JFrame::getPixelFormat() const {
-  static const auto getPixelFormatMethod = getClass()->getMethod<JString()>("getPixelFormat");
+local_ref<JPixelFormat> JFrame::getPixelFormat() const {
+  static const auto getPixelFormatMethod = getClass()->getMethod<JPixelFormat()>("getPixelFormat");
   return getPixelFormatMethod(self());
 }
 
@@ -77,9 +79,4 @@ void JFrame::decrementRefCount() {
   decrementRefCountMethod(self());
 }
 
-void JFrame::close() {
-  static const auto closeMethod = getClass()->getMethod<void()>("close");
-  closeMethod(self());
-}
-
 } // namespace vision

package/android/src/main/cpp/frameprocessor/java-bindings/JFrame.h

@@ -4,6 +4,8 @@
 
 #pragma once
 
+#include "JOrientation.h"
+#include "JPixelFormat.h"
 #include <fbjni/fbjni.h>
 #include <jni.h>
 
@@ -25,15 +27,14 @@ public:
   int getPlanesCount() const;
   int getBytesPerRow() const;
   jlong getTimestamp() const;
-  local_ref<JString> getOrientation() const;
-  local_ref<JString> getPixelFormat() const;
+  local_ref<JOrientation> getOrientation() const;
+  local_ref<JPixelFormat> getPixelFormat() const;
 #if __ANDROID_API__ >= 26
   AHardwareBuffer* getHardwareBuffer() const;
 #endif
 
   void incrementRefCount();
   void decrementRefCount();
-  void close();
 };
 
 } // namespace vision

package/android/src/main/cpp/frameprocessor/java-bindings/JJSUnionValue.h

@@ -0,0 +1,24 @@
+//
+// Created by Marc Rousavy on 29.12.23.
+//
+
+#pragma once
+
+#include <fbjni/fbjni.h>
+#include <jni.h>
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+struct JJSUnionValue : public JavaClass<JJSUnionValue> {
+  static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/types/JSUnionValue;";
+
+  local_ref<JString> getUnionValue() {
+    const auto getUnionValueMethod = getClass()->getMethod<JString()>("getUnionValue");
+    return getUnionValueMethod(self());
+  }
+};
+
+} // namespace vision

package/android/src/main/cpp/frameprocessor/java-bindings/JOrientation.h

@@ -0,0 +1,20 @@
+//
+// Created by Marc Rousavy on 29.12.23.
+//
+
+#pragma once
+
+#include "JJSUnionValue.h"
+#include <fbjni/fbjni.h>
+#include <jni.h>
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+struct JOrientation : public JavaClass<JOrientation, JJSUnionValue> {
+  static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/types/Orientation;";
+};
+
+} // namespace vision

package/android/src/main/cpp/frameprocessor/java-bindings/JPixelFormat.h

@@ -0,0 +1,20 @@
+//
+// Created by Marc Rousavy on 29.12.23.
+//
+
+#pragma once
+
+#include "JJSUnionValue.h"
+#include <fbjni/fbjni.h>
+#include <jni.h>
+
+namespace vision {
+
+using namespace facebook;
+using namespace jni;
+
+struct JPixelFormat : public JavaClass<JPixelFormat, JJSUnionValue> {
+  static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/types/PixelFormat;";
+};
+
+} // namespace vision

package/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java

@@ -7,6 +7,7 @@ import com.facebook.proguard.annotations.DoNotStrip;
 import com.mrousavy.camera.core.HardwareBuffersNotAvailableError;
 import com.mrousavy.camera.types.PixelFormat;
 import com.mrousavy.camera.types.Orientation;
+import java.lang.IllegalStateException;
 
 public class Frame {
     private final Image image;
@@ -24,30 +25,44 @@ public class Frame {
     }
 
     public Image getImage() {
-        return image;
+        synchronized (this) {
+            Image img = image;
+            if (!getIsImageValid(img)) {
+                throw new RuntimeException("Frame is already closed! " +
+                    "Are you trying to access the Image data outside of a Frame Processor's lifetime?\n" +
+                    "- If you want to use `console.log(frame)`, use `console.log(frame.toString())` instead.\n" +
+                    "- If you want to do async processing, use `runAsync(...)` instead.\n" +
+                    "- If you want to use runOnJS, increment it's ref-count: `frame.incrementRefCount()`");
+            }
+            return img;
+        }
     }
 
     @SuppressWarnings("unused")
     @DoNotStrip
     public int getWidth() {
-        return image.getWidth();
+        return getImage().getWidth();
     }
 
     @SuppressWarnings("unused")
     @DoNotStrip
     public int getHeight() {
-        return image.getHeight();
+        return getImage().getHeight();
     }
 
     @SuppressWarnings("unused")
     @DoNotStrip
     public boolean getIsValid() {
+        return getIsImageValid(getImage());
+    }
+
+    private boolean getIsImageValid(Image image) {
         try {
             // will throw an exception if the image is already closed
-            image.getCropRect();
+            synchronized (this) { image.getFormat(); }
             // no exception thrown, image must still be valid.
             return true;
-        } catch (Exception e) {
+        } catch (IllegalStateException e) {
             // exception thrown, image has already been closed.
             return false;
         }
@@ -67,27 +82,26 @@ public class Frame {
 
     @SuppressWarnings("unused")
     @DoNotStrip
-    public String getOrientation() {
-        return orientation.getUnionValue();
+    public Orientation getOrientation() {
+        return orientation;
     }
 
     @SuppressWarnings("unused")
     @DoNotStrip
-    public String getPixelFormat() {
-        PixelFormat format = PixelFormat.Companion.fromImageFormat(image.getFormat());
-        return format.getUnionValue();
+    public PixelFormat getPixelFormat() {
+        return PixelFormat.Companion.fromImageFormat(getImage().getFormat());
     }
 
     @SuppressWarnings("unused")
     @DoNotStrip
     public int getPlanesCount() {
-        return image.getPlanes().length;
+        return getImage().getPlanes().length;
     }
 
     @SuppressWarnings("unused")
     @DoNotStrip
     public int getBytesPerRow() {
-        return image.getPlanes()[0].getRowStride();
+        return getImage().getPlanes()[0].getRowStride();
     }
 
     @SuppressWarnings("unused")
@@ -101,7 +115,7 @@ public class Frame {
             throw new HardwareBuffersNotAvailableError();
         }
         if (hardwareBuffer == null) {
-            hardwareBuffer = image.getHardwareBuffer();
+            hardwareBuffer = getImage().getHardwareBuffer();
         }
         return hardwareBuffer;
     }
@@ -121,17 +135,17 @@ public class Frame {
             refCount--;
             if (refCount <= 0) {
                 // If no reference is held on this Image, close it.
-                image.close();
+                close();
             }
         }
     }
 
-    @SuppressWarnings("unused")
-    @DoNotStrip
-    private void close() {
-        if (hardwareBuffer != null) {
-            hardwareBuffer.close();
+    private synchronized void close() {
+        synchronized (this) {
+            if (hardwareBuffer != null) {
+                hardwareBuffer.close();
+            }
+            image.close();
         }
-        image.close();
     }
 }