feat: Make Frame
thread-safe and improve error messages (#2327)
* fix: Fix multi-Thread access on Java * fix: Thread-lock access on iOS as well * whoops add missing header impl * Update Podfile.lock * fix: Don't use `CFGetRetainCount` * fix: Lock access on iOS as well * C++ format * More detailed error * chore: Move getters into `Frame` * Format c++ * Use enum `orientation` again * format * fix: Synchronize `isValid` on Java * Also log pixelformat * feat: Use Java enums in C++ * Format C++
This commit is contained in:
parent
e4393cd83a
commit
895f3ec889
@ -77,7 +77,9 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
|
|||||||
}
|
}
|
||||||
auto width = this->frame->getWidth();
|
auto width = this->frame->getWidth();
|
||||||
auto height = this->frame->getHeight();
|
auto height = this->frame->getHeight();
|
||||||
auto str = std::to_string(width) + " x " + std::to_string(height) + " Frame";
|
auto format = this->frame->getPixelFormat();
|
||||||
|
auto formatString = format->getUnionValue();
|
||||||
|
auto str = std::to_string(width) + " x " + std::to_string(height) + " " + formatString->toString() + " Frame";
|
||||||
return jsi::String::createFromUtf8(runtime, str);
|
return jsi::String::createFromUtf8(runtime, str);
|
||||||
};
|
};
|
||||||
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
|
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
|
||||||
@ -141,11 +143,13 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
|
|||||||
return jsi::Value(this->frame->getIsMirrored());
|
return jsi::Value(this->frame->getIsMirrored());
|
||||||
}
|
}
|
||||||
if (name == "orientation") {
|
if (name == "orientation") {
|
||||||
auto string = this->frame->getOrientation();
|
auto orientation = this->frame->getOrientation();
|
||||||
|
auto string = orientation->getUnionValue();
|
||||||
return jsi::String::createFromUtf8(runtime, string->toStdString());
|
return jsi::String::createFromUtf8(runtime, string->toStdString());
|
||||||
}
|
}
|
||||||
if (name == "pixelFormat") {
|
if (name == "pixelFormat") {
|
||||||
auto string = this->frame->getPixelFormat();
|
auto pixelFormat = this->frame->getPixelFormat();
|
||||||
|
auto string = pixelFormat->getUnionValue();
|
||||||
return jsi::String::createFromUtf8(runtime, string->toStdString());
|
return jsi::String::createFromUtf8(runtime, string->toStdString());
|
||||||
}
|
}
|
||||||
if (name == "timestamp") {
|
if (name == "timestamp") {
|
||||||
|
@ -4,6 +4,8 @@
|
|||||||
|
|
||||||
#include "JFrame.h"
|
#include "JFrame.h"
|
||||||
|
|
||||||
|
#include "JOrientation.h"
|
||||||
|
#include "JPixelFormat.h"
|
||||||
#include <fbjni/fbjni.h>
|
#include <fbjni/fbjni.h>
|
||||||
#include <jni.h>
|
#include <jni.h>
|
||||||
|
|
||||||
@ -39,13 +41,13 @@ jlong JFrame::getTimestamp() const {
|
|||||||
return getTimestampMethod(self());
|
return getTimestampMethod(self());
|
||||||
}
|
}
|
||||||
|
|
||||||
local_ref<JString> JFrame::getOrientation() const {
|
local_ref<JOrientation> JFrame::getOrientation() const {
|
||||||
static const auto getOrientationMethod = getClass()->getMethod<JString()>("getOrientation");
|
static const auto getOrientationMethod = getClass()->getMethod<JOrientation()>("getOrientation");
|
||||||
return getOrientationMethod(self());
|
return getOrientationMethod(self());
|
||||||
}
|
}
|
||||||
|
|
||||||
local_ref<JString> JFrame::getPixelFormat() const {
|
local_ref<JPixelFormat> JFrame::getPixelFormat() const {
|
||||||
static const auto getPixelFormatMethod = getClass()->getMethod<JString()>("getPixelFormat");
|
static const auto getPixelFormatMethod = getClass()->getMethod<JPixelFormat()>("getPixelFormat");
|
||||||
return getPixelFormatMethod(self());
|
return getPixelFormatMethod(self());
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -77,9 +79,4 @@ void JFrame::decrementRefCount() {
|
|||||||
decrementRefCountMethod(self());
|
decrementRefCountMethod(self());
|
||||||
}
|
}
|
||||||
|
|
||||||
void JFrame::close() {
|
|
||||||
static const auto closeMethod = getClass()->getMethod<void()>("close");
|
|
||||||
closeMethod(self());
|
|
||||||
}
|
|
||||||
|
|
||||||
} // namespace vision
|
} // namespace vision
|
||||||
|
@ -4,6 +4,8 @@
|
|||||||
|
|
||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
|
#include "JOrientation.h"
|
||||||
|
#include "JPixelFormat.h"
|
||||||
#include <fbjni/fbjni.h>
|
#include <fbjni/fbjni.h>
|
||||||
#include <jni.h>
|
#include <jni.h>
|
||||||
|
|
||||||
@ -25,15 +27,14 @@ public:
|
|||||||
int getPlanesCount() const;
|
int getPlanesCount() const;
|
||||||
int getBytesPerRow() const;
|
int getBytesPerRow() const;
|
||||||
jlong getTimestamp() const;
|
jlong getTimestamp() const;
|
||||||
local_ref<JString> getOrientation() const;
|
local_ref<JOrientation> getOrientation() const;
|
||||||
local_ref<JString> getPixelFormat() const;
|
local_ref<JPixelFormat> getPixelFormat() const;
|
||||||
#if __ANDROID_API__ >= 26
|
#if __ANDROID_API__ >= 26
|
||||||
AHardwareBuffer* getHardwareBuffer() const;
|
AHardwareBuffer* getHardwareBuffer() const;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
void incrementRefCount();
|
void incrementRefCount();
|
||||||
void decrementRefCount();
|
void decrementRefCount();
|
||||||
void close();
|
|
||||||
};
|
};
|
||||||
|
|
||||||
} // namespace vision
|
} // namespace vision
|
||||||
|
@ -0,0 +1,24 @@
|
|||||||
|
//
|
||||||
|
// Created by Marc Rousavy on 29.12.23.
|
||||||
|
//
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <fbjni/fbjni.h>
|
||||||
|
#include <jni.h>
|
||||||
|
|
||||||
|
namespace vision {
|
||||||
|
|
||||||
|
using namespace facebook;
|
||||||
|
using namespace jni;
|
||||||
|
|
||||||
|
struct JJSUnionValue : public JavaClass<JJSUnionValue> {
|
||||||
|
static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/types/JSUnionValue;";
|
||||||
|
|
||||||
|
local_ref<JString> getUnionValue() {
|
||||||
|
const auto getUnionValueMethod = getClass()->getMethod<JString()>("getUnionValue");
|
||||||
|
return getUnionValueMethod(self());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace vision
|
@ -0,0 +1,20 @@
|
|||||||
|
//
|
||||||
|
// Created by Marc Rousavy on 29.12.23.
|
||||||
|
//
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "JJSUnionValue.h"
|
||||||
|
#include <fbjni/fbjni.h>
|
||||||
|
#include <jni.h>
|
||||||
|
|
||||||
|
namespace vision {
|
||||||
|
|
||||||
|
using namespace facebook;
|
||||||
|
using namespace jni;
|
||||||
|
|
||||||
|
struct JOrientation : public JavaClass<JOrientation, JJSUnionValue> {
|
||||||
|
static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/types/Orientation;";
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace vision
|
@ -0,0 +1,20 @@
|
|||||||
|
//
|
||||||
|
// Created by Marc Rousavy on 29.12.23.
|
||||||
|
//
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "JJSUnionValue.h"
|
||||||
|
#include <fbjni/fbjni.h>
|
||||||
|
#include <jni.h>
|
||||||
|
|
||||||
|
namespace vision {
|
||||||
|
|
||||||
|
using namespace facebook;
|
||||||
|
using namespace jni;
|
||||||
|
|
||||||
|
struct JPixelFormat : public JavaClass<JPixelFormat, JJSUnionValue> {
|
||||||
|
static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/types/PixelFormat;";
|
||||||
|
};
|
||||||
|
|
||||||
|
} // namespace vision
|
@ -7,6 +7,7 @@ import com.facebook.proguard.annotations.DoNotStrip;
|
|||||||
import com.mrousavy.camera.core.HardwareBuffersNotAvailableError;
|
import com.mrousavy.camera.core.HardwareBuffersNotAvailableError;
|
||||||
import com.mrousavy.camera.types.PixelFormat;
|
import com.mrousavy.camera.types.PixelFormat;
|
||||||
import com.mrousavy.camera.types.Orientation;
|
import com.mrousavy.camera.types.Orientation;
|
||||||
|
import java.lang.IllegalStateException;
|
||||||
|
|
||||||
public class Frame {
|
public class Frame {
|
||||||
private final Image image;
|
private final Image image;
|
||||||
@ -24,30 +25,44 @@ public class Frame {
|
|||||||
}
|
}
|
||||||
|
|
||||||
public Image getImage() {
|
public Image getImage() {
|
||||||
return image;
|
synchronized (this) {
|
||||||
|
Image img = image;
|
||||||
|
if (!getIsImageValid(img)) {
|
||||||
|
throw new RuntimeException("Frame is already closed! " +
|
||||||
|
"Are you trying to access the Image data outside of a Frame Processor's lifetime?\n" +
|
||||||
|
"- If you want to use `console.log(frame)`, use `console.log(frame.toString())` instead.\n" +
|
||||||
|
"- If you want to do async processing, use `runAsync(...)` instead.\n" +
|
||||||
|
"- If you want to use runOnJS, increment it's ref-count: `frame.incrementRefCount()`");
|
||||||
|
}
|
||||||
|
return img;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unused")
|
@SuppressWarnings("unused")
|
||||||
@DoNotStrip
|
@DoNotStrip
|
||||||
public int getWidth() {
|
public int getWidth() {
|
||||||
return image.getWidth();
|
return getImage().getWidth();
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unused")
|
@SuppressWarnings("unused")
|
||||||
@DoNotStrip
|
@DoNotStrip
|
||||||
public int getHeight() {
|
public int getHeight() {
|
||||||
return image.getHeight();
|
return getImage().getHeight();
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unused")
|
@SuppressWarnings("unused")
|
||||||
@DoNotStrip
|
@DoNotStrip
|
||||||
public boolean getIsValid() {
|
public boolean getIsValid() {
|
||||||
|
return getIsImageValid(getImage());
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean getIsImageValid(Image image) {
|
||||||
try {
|
try {
|
||||||
// will throw an exception if the image is already closed
|
// will throw an exception if the image is already closed
|
||||||
image.getCropRect();
|
synchronized (this) { image.getFormat(); }
|
||||||
// no exception thrown, image must still be valid.
|
// no exception thrown, image must still be valid.
|
||||||
return true;
|
return true;
|
||||||
} catch (Exception e) {
|
} catch (IllegalStateException e) {
|
||||||
// exception thrown, image has already been closed.
|
// exception thrown, image has already been closed.
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
@ -67,27 +82,26 @@ public class Frame {
|
|||||||
|
|
||||||
@SuppressWarnings("unused")
|
@SuppressWarnings("unused")
|
||||||
@DoNotStrip
|
@DoNotStrip
|
||||||
public String getOrientation() {
|
public Orientation getOrientation() {
|
||||||
return orientation.getUnionValue();
|
return orientation;
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unused")
|
@SuppressWarnings("unused")
|
||||||
@DoNotStrip
|
@DoNotStrip
|
||||||
public String getPixelFormat() {
|
public PixelFormat getPixelFormat() {
|
||||||
PixelFormat format = PixelFormat.Companion.fromImageFormat(image.getFormat());
|
return PixelFormat.Companion.fromImageFormat(getImage().getFormat());
|
||||||
return format.getUnionValue();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unused")
|
@SuppressWarnings("unused")
|
||||||
@DoNotStrip
|
@DoNotStrip
|
||||||
public int getPlanesCount() {
|
public int getPlanesCount() {
|
||||||
return image.getPlanes().length;
|
return getImage().getPlanes().length;
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unused")
|
@SuppressWarnings("unused")
|
||||||
@DoNotStrip
|
@DoNotStrip
|
||||||
public int getBytesPerRow() {
|
public int getBytesPerRow() {
|
||||||
return image.getPlanes()[0].getRowStride();
|
return getImage().getPlanes()[0].getRowStride();
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unused")
|
@SuppressWarnings("unused")
|
||||||
@ -101,7 +115,7 @@ public class Frame {
|
|||||||
throw new HardwareBuffersNotAvailableError();
|
throw new HardwareBuffersNotAvailableError();
|
||||||
}
|
}
|
||||||
if (hardwareBuffer == null) {
|
if (hardwareBuffer == null) {
|
||||||
hardwareBuffer = image.getHardwareBuffer();
|
hardwareBuffer = getImage().getHardwareBuffer();
|
||||||
}
|
}
|
||||||
return hardwareBuffer;
|
return hardwareBuffer;
|
||||||
}
|
}
|
||||||
@ -121,17 +135,17 @@ public class Frame {
|
|||||||
refCount--;
|
refCount--;
|
||||||
if (refCount <= 0) {
|
if (refCount <= 0) {
|
||||||
// If no reference is held on this Image, close it.
|
// If no reference is held on this Image, close it.
|
||||||
image.close();
|
close();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@SuppressWarnings("unused")
|
private synchronized void close() {
|
||||||
@DoNotStrip
|
synchronized (this) {
|
||||||
private void close() {
|
if (hardwareBuffer != null) {
|
||||||
if (hardwareBuffer != null) {
|
hardwareBuffer.close();
|
||||||
hardwareBuffer.close();
|
}
|
||||||
|
image.close();
|
||||||
}
|
}
|
||||||
image.close();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -484,7 +484,7 @@ PODS:
|
|||||||
- libwebp (~> 1.0)
|
- libwebp (~> 1.0)
|
||||||
- SDWebImage/Core (~> 5.10)
|
- SDWebImage/Core (~> 5.10)
|
||||||
- SocketRocket (0.6.1)
|
- SocketRocket (0.6.1)
|
||||||
- VisionCamera (3.6.16):
|
- VisionCamera (3.6.17):
|
||||||
- React
|
- React
|
||||||
- React-callinvoker
|
- React-callinvoker
|
||||||
- React-Core
|
- React-Core
|
||||||
@ -724,9 +724,9 @@ SPEC CHECKSUMS:
|
|||||||
SDWebImage: a7f831e1a65eb5e285e3fb046a23fcfbf08e696d
|
SDWebImage: a7f831e1a65eb5e285e3fb046a23fcfbf08e696d
|
||||||
SDWebImageWebPCoder: 908b83b6adda48effe7667cd2b7f78c897e5111d
|
SDWebImageWebPCoder: 908b83b6adda48effe7667cd2b7f78c897e5111d
|
||||||
SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17
|
SocketRocket: f32cd54efbe0f095c4d7594881e52619cfe80b17
|
||||||
VisionCamera: 06f5b990082ec783a58dcadc5bac677974f7a580
|
VisionCamera: 361df29347b7b7ecc47b3d173daa17751a11ffc1
|
||||||
Yoga: 4c3aa327e4a6a23eeacd71f61c81df1bcdf677d5
|
Yoga: 4c3aa327e4a6a23eeacd71f61c81df1bcdf677d5
|
||||||
|
|
||||||
PODFILE CHECKSUM: 27f53791141a3303d814e09b55770336416ff4eb
|
PODFILE CHECKSUM: 27f53791141a3303d814e09b55770336416ff4eb
|
||||||
|
|
||||||
COCOAPODS: 1.11.3
|
COCOAPODS: 1.14.3
|
||||||
|
@ -19,4 +19,14 @@
|
|||||||
@property(nonatomic, readonly) CMSampleBufferRef _Nonnull buffer;
|
@property(nonatomic, readonly) CMSampleBufferRef _Nonnull buffer;
|
||||||
@property(nonatomic, readonly) UIImageOrientation orientation;
|
@property(nonatomic, readonly) UIImageOrientation orientation;
|
||||||
|
|
||||||
|
// Getters
|
||||||
|
- (NSString* _Nonnull)pixelFormat;
|
||||||
|
- (BOOL)isMirrored;
|
||||||
|
- (BOOL)isValid;
|
||||||
|
- (size_t)width;
|
||||||
|
- (size_t)height;
|
||||||
|
- (double)timestamp;
|
||||||
|
- (size_t)bytesPerRow;
|
||||||
|
- (size_t)planesCount;
|
||||||
|
|
||||||
@end
|
@end
|
||||||
|
@ -20,11 +20,80 @@
|
|||||||
if (self) {
|
if (self) {
|
||||||
_buffer = buffer;
|
_buffer = buffer;
|
||||||
_orientation = orientation;
|
_orientation = orientation;
|
||||||
|
CFRetain(buffer);
|
||||||
}
|
}
|
||||||
return self;
|
return self;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
- (void)dealloc {
|
||||||
|
CFRelease(_buffer);
|
||||||
|
}
|
||||||
|
|
||||||
@synthesize buffer = _buffer;
|
@synthesize buffer = _buffer;
|
||||||
@synthesize orientation = _orientation;
|
@synthesize orientation = _orientation;
|
||||||
|
|
||||||
|
- (NSString*)pixelFormat {
|
||||||
|
CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(_buffer);
|
||||||
|
FourCharCode mediaType = CMFormatDescriptionGetMediaSubType(format);
|
||||||
|
switch (mediaType) {
|
||||||
|
case kCVPixelFormatType_32BGRA:
|
||||||
|
case kCVPixelFormatType_Lossy_32BGRA:
|
||||||
|
return @"rgb";
|
||||||
|
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
|
||||||
|
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
|
||||||
|
case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange:
|
||||||
|
case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange:
|
||||||
|
case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarFullRange:
|
||||||
|
case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarVideoRange:
|
||||||
|
case kCVPixelFormatType_Lossy_420YpCbCr10PackedBiPlanarVideoRange:
|
||||||
|
return @"yuv";
|
||||||
|
default:
|
||||||
|
return @"unknown";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
- (BOOL)isMirrored {
|
||||||
|
switch (_orientation) {
|
||||||
|
case UIImageOrientationUp:
|
||||||
|
case UIImageOrientationDown:
|
||||||
|
case UIImageOrientationLeft:
|
||||||
|
case UIImageOrientationRight:
|
||||||
|
return false;
|
||||||
|
case UIImageOrientationDownMirrored:
|
||||||
|
case UIImageOrientationUpMirrored:
|
||||||
|
case UIImageOrientationLeftMirrored:
|
||||||
|
case UIImageOrientationRightMirrored:
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
- (BOOL)isValid {
|
||||||
|
return _buffer != nil && CMSampleBufferIsValid(_buffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
- (size_t)width {
|
||||||
|
CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_buffer);
|
||||||
|
return CVPixelBufferGetWidth(imageBuffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
- (size_t)height {
|
||||||
|
CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_buffer);
|
||||||
|
return CVPixelBufferGetHeight(imageBuffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
- (double)timestamp {
|
||||||
|
CMTime timestamp = CMSampleBufferGetPresentationTimeStamp(_buffer);
|
||||||
|
return CMTimeGetSeconds(timestamp) * 1000.0;
|
||||||
|
}
|
||||||
|
|
||||||
|
- (size_t)bytesPerRow {
|
||||||
|
CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_buffer);
|
||||||
|
return CVPixelBufferGetBytesPerRow(imageBuffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
- (size_t)planesCount {
|
||||||
|
CVPixelBufferRef imageBuffer = CMSampleBufferGetImageBuffer(_buffer);
|
||||||
|
return CVPixelBufferGetPlaneCount(imageBuffer);
|
||||||
|
}
|
||||||
|
|
||||||
@end
|
@end
|
||||||
|
@ -10,6 +10,7 @@
|
|||||||
|
|
||||||
#import <CoreMedia/CMSampleBuffer.h>
|
#import <CoreMedia/CMSampleBuffer.h>
|
||||||
#import <jsi/jsi.h>
|
#import <jsi/jsi.h>
|
||||||
|
#import <mutex>
|
||||||
|
|
||||||
#import "Frame.h"
|
#import "Frame.h"
|
||||||
|
|
||||||
@ -25,4 +26,11 @@ public:
|
|||||||
|
|
||||||
public:
|
public:
|
||||||
Frame* frame;
|
Frame* frame;
|
||||||
|
|
||||||
|
private:
|
||||||
|
Frame* getFrame();
|
||||||
|
|
||||||
|
private:
|
||||||
|
std::mutex _mutex;
|
||||||
|
size_t _refCount = 0;
|
||||||
};
|
};
|
||||||
|
@ -7,6 +7,7 @@
|
|||||||
//
|
//
|
||||||
|
|
||||||
#import "FrameHostObject.h"
|
#import "FrameHostObject.h"
|
||||||
|
#import "UIImageOrientation+descriptor.h"
|
||||||
#import "WKTJsiHostObject.h"
|
#import "WKTJsiHostObject.h"
|
||||||
#import <Foundation/Foundation.h>
|
#import <Foundation/Foundation.h>
|
||||||
#import <jsi/jsi.h>
|
#import <jsi/jsi.h>
|
||||||
@ -34,42 +35,70 @@ std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt)
|
|||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Frame* FrameHostObject::getFrame() {
|
||||||
|
Frame* frame = this->frame;
|
||||||
|
if (frame == nil || !CMSampleBufferIsValid(frame.buffer)) {
|
||||||
|
throw std::runtime_error("Frame is already closed! "
|
||||||
|
"Are you trying to access the Image data outside of a Frame Processor's lifetime?\n"
|
||||||
|
"- If you want to use `console.log(frame)`, use `console.log(frame.toString())` instead.\n"
|
||||||
|
"- If you want to do async processing, use `runAsync(...)` instead.\n"
|
||||||
|
"- If you want to use runOnJS, increment it's ref-count: `frame.incrementRefCount()`");
|
||||||
|
}
|
||||||
|
return frame;
|
||||||
|
}
|
||||||
|
|
||||||
jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
|
jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
|
||||||
auto name = propName.utf8(runtime);
|
auto name = propName.utf8(runtime);
|
||||||
|
|
||||||
if (name == "toString") {
|
if (name == "toString") {
|
||||||
auto toString = JSI_HOST_FUNCTION_LAMBDA {
|
auto toString = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
|
||||||
if (this->frame == nil) {
|
// Lock Frame so it cannot be deallocated while we access it
|
||||||
return jsi::String::createFromUtf8(runtime, "[closed frame]");
|
std::lock_guard lock(this->_mutex);
|
||||||
}
|
|
||||||
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
|
|
||||||
auto width = CVPixelBufferGetWidth(imageBuffer);
|
|
||||||
auto height = CVPixelBufferGetHeight(imageBuffer);
|
|
||||||
|
|
||||||
NSMutableString* string = [NSMutableString stringWithFormat:@"%lu x %lu Frame", width, height];
|
// Print debug description (width, height)
|
||||||
|
Frame* frame = this->getFrame();
|
||||||
|
NSMutableString* string = [NSMutableString stringWithFormat:@"%lu x %lu %@ Frame", frame.width, frame.height, frame.pixelFormat];
|
||||||
return jsi::String::createFromUtf8(runtime, string.UTF8String);
|
return jsi::String::createFromUtf8(runtime, string.UTF8String);
|
||||||
};
|
};
|
||||||
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
|
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
|
||||||
}
|
}
|
||||||
if (name == "incrementRefCount") {
|
if (name == "incrementRefCount") {
|
||||||
auto incrementRefCount = JSI_HOST_FUNCTION_LAMBDA {
|
auto incrementRefCount = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
|
||||||
// Increment retain count by one so ARC doesn't destroy the Frame Buffer.
|
size_t count) -> jsi::Value {
|
||||||
CFRetain(frame.buffer);
|
// Lock Frame so it cannot be deallocated while we access it
|
||||||
|
std::lock_guard lock(this->_mutex);
|
||||||
|
|
||||||
|
// Increment our self-counted ref count by one.
|
||||||
|
_refCount++;
|
||||||
return jsi::Value::undefined();
|
return jsi::Value::undefined();
|
||||||
};
|
};
|
||||||
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "incrementRefCount"), 0, incrementRefCount);
|
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "incrementRefCount"), 0, incrementRefCount);
|
||||||
}
|
}
|
||||||
if (name == "decrementRefCount") {
|
if (name == "decrementRefCount") {
|
||||||
auto decrementRefCount = JSI_HOST_FUNCTION_LAMBDA {
|
auto decrementRefCount = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
|
||||||
// Decrement retain count by one. If the retain count is zero, ARC will destroy the Frame
|
size_t count) -> jsi::Value {
|
||||||
// Buffer.
|
// Lock Frame so it cannot be deallocated while we access it
|
||||||
CFRelease(frame.buffer);
|
std::lock_guard lock(this->_mutex);
|
||||||
|
|
||||||
|
// Decrement our self-counted ref count by one.
|
||||||
|
_refCount--;
|
||||||
|
if (_refCount < 1) {
|
||||||
|
// ARC will then delete the Frame and the underlying Frame Buffer.
|
||||||
|
this->frame = nil;
|
||||||
|
}
|
||||||
|
|
||||||
return jsi::Value::undefined();
|
return jsi::Value::undefined();
|
||||||
};
|
};
|
||||||
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "decrementRefCount"), 0, decrementRefCount);
|
return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "decrementRefCount"), 0, decrementRefCount);
|
||||||
}
|
}
|
||||||
if (name == "toArrayBuffer") {
|
if (name == "toArrayBuffer") {
|
||||||
auto toArrayBuffer = JSI_HOST_FUNCTION_LAMBDA {
|
auto toArrayBuffer = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
|
||||||
|
size_t count) -> jsi::Value {
|
||||||
|
// Lock Frame so it cannot be deallocated while we access it
|
||||||
|
std::lock_guard lock(this->_mutex);
|
||||||
|
|
||||||
|
// Get CPU readable Pixel Buffer from Frame and write it to a jsi::ArrayBuffer
|
||||||
|
Frame* frame = this->getFrame();
|
||||||
auto pixelBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
|
auto pixelBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
|
||||||
auto bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer);
|
auto bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer);
|
||||||
auto height = CVPixelBufferGetHeight(pixelBuffer);
|
auto height = CVPixelBufferGetHeight(pixelBuffer);
|
||||||
@ -101,82 +130,69 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (name == "isValid") {
|
if (name == "isValid") {
|
||||||
auto isValid = frame != nil && frame.buffer != nil && CFGetRetainCount(frame.buffer) > 0 && CMSampleBufferIsValid(frame.buffer);
|
// Lock Frame so it cannot be deallocated while we access it
|
||||||
return jsi::Value(isValid);
|
std::lock_guard lock(this->_mutex);
|
||||||
|
|
||||||
|
// unsafely access the Frame and try to see if it's valid
|
||||||
|
Frame* frame = this->frame;
|
||||||
|
return jsi::Value(frame != nil && frame.isValid);
|
||||||
}
|
}
|
||||||
if (name == "width") {
|
if (name == "width") {
|
||||||
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
|
// Lock Frame so it cannot be deallocated while we access it
|
||||||
auto width = CVPixelBufferGetWidth(imageBuffer);
|
std::lock_guard lock(this->_mutex);
|
||||||
return jsi::Value((double)width);
|
|
||||||
|
Frame* frame = this->getFrame();
|
||||||
|
return jsi::Value((double)frame.width);
|
||||||
}
|
}
|
||||||
if (name == "height") {
|
if (name == "height") {
|
||||||
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
|
// Lock Frame so it cannot be deallocated while we access it
|
||||||
auto height = CVPixelBufferGetHeight(imageBuffer);
|
std::lock_guard lock(this->_mutex);
|
||||||
return jsi::Value((double)height);
|
|
||||||
|
Frame* frame = this->getFrame();
|
||||||
|
return jsi::Value((double)frame.height);
|
||||||
}
|
}
|
||||||
if (name == "orientation") {
|
if (name == "orientation") {
|
||||||
switch (frame.orientation) {
|
// Lock Frame so it cannot be deallocated while we access it
|
||||||
case UIImageOrientationUp:
|
std::lock_guard lock(this->_mutex);
|
||||||
case UIImageOrientationUpMirrored:
|
|
||||||
return jsi::String::createFromUtf8(runtime, "portrait");
|
Frame* frame = this->getFrame();
|
||||||
case UIImageOrientationDown:
|
NSString* orientation = [NSString stringWithParsed:frame.orientation];
|
||||||
case UIImageOrientationDownMirrored:
|
return jsi::String::createFromUtf8(runtime, orientation.UTF8String);
|
||||||
return jsi::String::createFromUtf8(runtime, "portrait-upside-down");
|
|
||||||
case UIImageOrientationLeft:
|
|
||||||
case UIImageOrientationLeftMirrored:
|
|
||||||
return jsi::String::createFromUtf8(runtime, "landscape-left");
|
|
||||||
case UIImageOrientationRight:
|
|
||||||
case UIImageOrientationRightMirrored:
|
|
||||||
return jsi::String::createFromUtf8(runtime, "landscape-right");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (name == "isMirrored") {
|
if (name == "isMirrored") {
|
||||||
switch (frame.orientation) {
|
// Lock Frame so it cannot be deallocated while we access it
|
||||||
case UIImageOrientationUp:
|
std::lock_guard lock(this->_mutex);
|
||||||
case UIImageOrientationDown:
|
|
||||||
case UIImageOrientationLeft:
|
Frame* frame = this->getFrame();
|
||||||
case UIImageOrientationRight:
|
return jsi::Value(frame.isMirrored);
|
||||||
return jsi::Value(false);
|
|
||||||
case UIImageOrientationDownMirrored:
|
|
||||||
case UIImageOrientationUpMirrored:
|
|
||||||
case UIImageOrientationLeftMirrored:
|
|
||||||
case UIImageOrientationRightMirrored:
|
|
||||||
return jsi::Value(true);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (name == "timestamp") {
|
if (name == "timestamp") {
|
||||||
auto timestamp = CMSampleBufferGetPresentationTimeStamp(frame.buffer);
|
// Lock Frame so it cannot be deallocated while we access it
|
||||||
auto seconds = static_cast<double>(CMTimeGetSeconds(timestamp));
|
std::lock_guard lock(this->_mutex);
|
||||||
return jsi::Value(seconds * 1000.0);
|
|
||||||
|
Frame* frame = this->getFrame();
|
||||||
|
return jsi::Value(frame.timestamp);
|
||||||
}
|
}
|
||||||
if (name == "pixelFormat") {
|
if (name == "pixelFormat") {
|
||||||
auto format = CMSampleBufferGetFormatDescription(frame.buffer);
|
// Lock Frame so it cannot be deallocated while we access it
|
||||||
auto mediaType = CMFormatDescriptionGetMediaSubType(format);
|
std::lock_guard lock(this->_mutex);
|
||||||
switch (mediaType) {
|
|
||||||
case kCVPixelFormatType_32BGRA:
|
Frame* frame = this->getFrame();
|
||||||
case kCVPixelFormatType_Lossy_32BGRA:
|
return jsi::String::createFromUtf8(runtime, frame.pixelFormat.UTF8String);
|
||||||
return jsi::String::createFromUtf8(runtime, "rgb");
|
|
||||||
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
|
|
||||||
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
|
|
||||||
case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange:
|
|
||||||
case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange:
|
|
||||||
case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarFullRange:
|
|
||||||
case kCVPixelFormatType_Lossy_420YpCbCr8BiPlanarVideoRange:
|
|
||||||
case kCVPixelFormatType_Lossy_420YpCbCr10PackedBiPlanarVideoRange:
|
|
||||||
return jsi::String::createFromUtf8(runtime, "yuv");
|
|
||||||
default:
|
|
||||||
return jsi::String::createFromUtf8(runtime, "unknown");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (name == "bytesPerRow") {
|
if (name == "bytesPerRow") {
|
||||||
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
|
// Lock Frame so it cannot be deallocated while we access it
|
||||||
auto bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer);
|
std::lock_guard lock(this->_mutex);
|
||||||
return jsi::Value((double)bytesPerRow);
|
|
||||||
|
Frame* frame = this->getFrame();
|
||||||
|
return jsi::Value((double)frame.bytesPerRow);
|
||||||
}
|
}
|
||||||
if (name == "planesCount") {
|
if (name == "planesCount") {
|
||||||
auto imageBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
|
// Lock Frame so it cannot be deallocated while we access it
|
||||||
auto planesCount = CVPixelBufferGetPlaneCount(imageBuffer);
|
std::lock_guard lock(this->_mutex);
|
||||||
return jsi::Value((double)planesCount);
|
|
||||||
|
Frame* frame = this->getFrame();
|
||||||
|
return jsi::Value((double)frame.planesCount);
|
||||||
}
|
}
|
||||||
|
|
||||||
// fallback to base implementation
|
// fallback to base implementation
|
||||||
|
39
package/ios/Frame Processor/UIImageOrientation+descriptor.h
Normal file
39
package/ios/Frame Processor/UIImageOrientation+descriptor.h
Normal file
@ -0,0 +1,39 @@
|
|||||||
|
//
|
||||||
|
// UIImageOrientation+descriptor.h
|
||||||
|
// VisionCamera
|
||||||
|
//
|
||||||
|
// Created by Marc Rousavy on 29.12.23.
|
||||||
|
// Copyright © 2023 mrousavy. All rights reserved.
|
||||||
|
//
|
||||||
|
|
||||||
|
#pragma once
|
||||||
|
|
||||||
|
#import <Foundation/Foundation.h>
|
||||||
|
#import <UIKit/UIImage.h>
|
||||||
|
|
||||||
|
@interface NSString (UIImageOrientationJSDescriptor)
|
||||||
|
|
||||||
|
+ (NSString*)stringWithParsed:(UIImageOrientation)orientation;
|
||||||
|
|
||||||
|
@end
|
||||||
|
|
||||||
|
@implementation NSString (UIImageOrientationJSDescriptor)
|
||||||
|
|
||||||
|
+ (NSString*)stringWithParsed:(UIImageOrientation)orientation {
|
||||||
|
switch (orientation) {
|
||||||
|
case UIImageOrientationUp:
|
||||||
|
case UIImageOrientationUpMirrored:
|
||||||
|
return @"portrait";
|
||||||
|
case UIImageOrientationDown:
|
||||||
|
case UIImageOrientationDownMirrored:
|
||||||
|
return @"portrait-upside-down";
|
||||||
|
case UIImageOrientationLeft:
|
||||||
|
case UIImageOrientationLeftMirrored:
|
||||||
|
return @"landscape-left";
|
||||||
|
case UIImageOrientationRight:
|
||||||
|
case UIImageOrientationRightMirrored:
|
||||||
|
return @"landscape-right";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
@end
|
@ -160,6 +160,7 @@
|
|||||||
B887518425E0102000DB86D6 /* CameraView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CameraView.swift; sourceTree = "<group>"; };
|
B887518425E0102000DB86D6 /* CameraView.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = CameraView.swift; sourceTree = "<group>"; };
|
||||||
B88873E5263D46C7008B1D0E /* FrameProcessorPlugin.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorPlugin.h; sourceTree = "<group>"; };
|
B88873E5263D46C7008B1D0E /* FrameProcessorPlugin.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FrameProcessorPlugin.h; sourceTree = "<group>"; };
|
||||||
B8994E6B263F03E100069589 /* JSINSObjectConversion.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = JSINSObjectConversion.mm; sourceTree = "<group>"; };
|
B8994E6B263F03E100069589 /* JSINSObjectConversion.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = JSINSObjectConversion.mm; sourceTree = "<group>"; };
|
||||||
|
B89A79692B3EF60F005E0357 /* UIImageOrientation+descriptor.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "UIImageOrientation+descriptor.h"; sourceTree = "<group>"; };
|
||||||
B8A1AEC32AD7EDE800169C0D /* AVCaptureVideoDataOutput+pixelFormat.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVCaptureVideoDataOutput+pixelFormat.swift"; sourceTree = "<group>"; };
|
B8A1AEC32AD7EDE800169C0D /* AVCaptureVideoDataOutput+pixelFormat.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "AVCaptureVideoDataOutput+pixelFormat.swift"; sourceTree = "<group>"; };
|
||||||
B8A1AEC52AD7F08E00169C0D /* CameraView+Focus.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CameraView+Focus.swift"; sourceTree = "<group>"; };
|
B8A1AEC52AD7F08E00169C0D /* CameraView+Focus.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CameraView+Focus.swift"; sourceTree = "<group>"; };
|
||||||
B8A1AEC72AD8005400169C0D /* CameraSession+Configuration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CameraSession+Configuration.swift"; sourceTree = "<group>"; };
|
B8A1AEC72AD8005400169C0D /* CameraSession+Configuration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "CameraSession+Configuration.swift"; sourceTree = "<group>"; };
|
||||||
@ -334,6 +335,7 @@
|
|||||||
B81D41EF263C86F900B041FD /* JSINSObjectConversion.h */,
|
B81D41EF263C86F900B041FD /* JSINSObjectConversion.h */,
|
||||||
B8994E6B263F03E100069589 /* JSINSObjectConversion.mm */,
|
B8994E6B263F03E100069589 /* JSINSObjectConversion.mm */,
|
||||||
B85F7AE82A77BB680089C539 /* FrameProcessorPlugin.m */,
|
B85F7AE82A77BB680089C539 /* FrameProcessorPlugin.m */,
|
||||||
|
B89A79692B3EF60F005E0357 /* UIImageOrientation+descriptor.h */,
|
||||||
);
|
);
|
||||||
path = "Frame Processor";
|
path = "Frame Processor";
|
||||||
sourceTree = "<group>";
|
sourceTree = "<group>";
|
||||||
|
Loading…
Reference in New Issue
Block a user