feat: Use JSI's ArrayBuffer instead of TypedArray (#2408)

* feat: Use JSI's `ArrayBuffer` instead of `TypedArray`

* fix: Fix move memory

* feat: Implement iOS

* Format

* Update JSIJNIConversion.cpp

* fix: Fix Android `toArrayBuffer` and other

* Catch FP call errors

* Update return type

* Use `CPU_READ_OFTEN` flag as well

* CPU flag

* Run destructors under `jni::ThreadScope`

* Update FrameProcessorPluginHostObject.cpp

* fix: Fix `toArrayBuffer()` crash

* Update Frame.ts
This commit is contained in:
Marc Rousavy
2024-01-17 20:18:46 +01:00
committed by GitHub
parent 2f21609e39
commit ba1d7eec9c
26 changed files with 296 additions and 620 deletions

View File

@@ -1,314 +0,0 @@
//
// JSITypedArray.cpp
// VisionCamera
//
// Created by Marc Rousavy on 21.02.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
// Copied & Adapted from
// https://github.com/expo/expo/blob/main/packages/expo-gl/common/EXTypedArrayApi.cpp Credits to
// Expo
#include "JSITypedArray.h"
#include <algorithm>
#include <memory>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
namespace vision {
template <TypedArrayKind T> using ContentType = typename typedArrayTypeMap<T>::type;
enum class Prop {
Buffer, // "buffer"
Constructor, // "constructor"
Name, // "name"
Proto, // "__proto__"
Length, // "length"
ByteLength, // "byteLength"
ByteOffset, // "offset"
IsView, // "isView"
ArrayBuffer, // "ArrayBuffer"
Int8Array, // "Int8Array"
Int16Array, // "Int16Array"
Int32Array, // "Int32Array"
Uint8Array, // "Uint8Array"
Uint8ClampedArray, // "Uint8ClampedArray"
Uint16Array, // "Uint16Array"
Uint32Array, // "Uint32Array"
Float32Array, // "Float32Array"
Float64Array, // "Float64Array"
};
class PropNameIDCache {
public:
const jsi::PropNameID& get(jsi::Runtime& runtime, Prop prop) {
auto key = reinterpret_cast<uintptr_t>(&runtime);
if (this->props.find(key) == this->props.end()) {
this->props[key] = std::unordered_map<Prop, std::unique_ptr<jsi::PropNameID>>();
}
if (!this->props[key][prop]) {
this->props[key][prop] = std::make_unique<jsi::PropNameID>(createProp(runtime, prop));
}
return *(this->props[key][prop]);
}
const jsi::PropNameID& getConstructorNameProp(jsi::Runtime& runtime, TypedArrayKind kind);
void invalidate(uintptr_t key) {
if (props.find(key) != props.end()) {
props[key].clear();
}
}
private:
std::unordered_map<uintptr_t, std::unordered_map<Prop, std::unique_ptr<jsi::PropNameID>>> props;
jsi::PropNameID createProp(jsi::Runtime& runtime, Prop prop);
};
PropNameIDCache propNameIDCache;
void invalidateArrayBufferCache(jsi::Runtime& runtime) {
auto key = reinterpret_cast<uintptr_t>(&runtime);
propNameIDCache.invalidate(key);
}
TypedArrayKind getTypedArrayKindForName(const std::string& name);
TypedArrayBase::TypedArrayBase(jsi::Runtime& runtime, size_t size, TypedArrayKind kind)
: TypedArrayBase(runtime, runtime.global()
.getProperty(runtime, propNameIDCache.getConstructorNameProp(runtime, kind))
.asObject(runtime)
.asFunction(runtime)
.callAsConstructor(runtime, {static_cast<double>(size)})
.asObject(runtime)) {}
TypedArrayBase::TypedArrayBase(jsi::Runtime& runtime, const jsi::Object& obj) : jsi::Object(jsi::Value(runtime, obj).asObject(runtime)) {}
TypedArrayKind TypedArrayBase::getKind(jsi::Runtime& runtime) const {
auto constructorName = this->getProperty(runtime, propNameIDCache.get(runtime, Prop::Constructor))
.asObject(runtime)
.getProperty(runtime, propNameIDCache.get(runtime, Prop::Name))
.asString(runtime)
.utf8(runtime);
return getTypedArrayKindForName(constructorName);
}
size_t TypedArrayBase::size(jsi::Runtime& runtime) const {
return getProperty(runtime, propNameIDCache.get(runtime, Prop::Length)).asNumber();
}
size_t TypedArrayBase::length(jsi::Runtime& runtime) const {
return getProperty(runtime, propNameIDCache.get(runtime, Prop::Length)).asNumber();
}
size_t TypedArrayBase::byteLength(jsi::Runtime& runtime) const {
return getProperty(runtime, propNameIDCache.get(runtime, Prop::ByteLength)).asNumber();
}
size_t TypedArrayBase::byteOffset(jsi::Runtime& runtime) const {
return getProperty(runtime, propNameIDCache.get(runtime, Prop::ByteOffset)).asNumber();
}
bool TypedArrayBase::hasBuffer(jsi::Runtime& runtime) const {
auto buffer = getProperty(runtime, propNameIDCache.get(runtime, Prop::Buffer));
return buffer.isObject() && buffer.asObject(runtime).isArrayBuffer(runtime);
}
std::vector<uint8_t> TypedArrayBase::toVector(jsi::Runtime& runtime) {
auto start = reinterpret_cast<uint8_t*>(getBuffer(runtime).data(runtime) + byteOffset(runtime));
auto end = start + byteLength(runtime);
return std::vector<uint8_t>(start, end);
}
jsi::ArrayBuffer TypedArrayBase::getBuffer(jsi::Runtime& runtime) const {
auto buffer = getProperty(runtime, propNameIDCache.get(runtime, Prop::Buffer));
if (buffer.isObject() && buffer.asObject(runtime).isArrayBuffer(runtime)) {
return buffer.asObject(runtime).getArrayBuffer(runtime);
} else {
throw std::runtime_error("no ArrayBuffer attached");
}
}
bool isTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj) {
auto jsVal = runtime.global()
.getProperty(runtime, propNameIDCache.get(runtime, Prop::ArrayBuffer))
.asObject(runtime)
.getProperty(runtime, propNameIDCache.get(runtime, Prop::IsView))
.asObject(runtime)
.asFunction(runtime)
.callWithThis(runtime, runtime.global(), {jsi::Value(runtime, jsObj)});
if (jsVal.isBool()) {
return jsVal.getBool();
} else {
throw std::runtime_error("value is not a boolean");
}
}
TypedArrayBase getTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj) {
auto jsVal = runtime.global()
.getProperty(runtime, propNameIDCache.get(runtime, Prop::ArrayBuffer))
.asObject(runtime)
.getProperty(runtime, propNameIDCache.get(runtime, Prop::IsView))
.asObject(runtime)
.asFunction(runtime)
.callWithThis(runtime, runtime.global(), {jsi::Value(runtime, jsObj)});
if (jsVal.isBool()) {
return TypedArrayBase(runtime, jsObj);
} else {
throw std::runtime_error("value is not a boolean");
}
}
std::vector<uint8_t> arrayBufferToVector(jsi::Runtime& runtime, jsi::Object& jsObj) {
if (!jsObj.isArrayBuffer(runtime)) {
throw std::runtime_error("Object is not an ArrayBuffer");
}
auto jsArrayBuffer = jsObj.getArrayBuffer(runtime);
uint8_t* dataBlock = jsArrayBuffer.data(runtime);
size_t blockSize = jsArrayBuffer.getProperty(runtime, propNameIDCache.get(runtime, Prop::ByteLength)).asNumber();
return std::vector<uint8_t>(dataBlock, dataBlock + blockSize);
}
void arrayBufferUpdate(jsi::Runtime& runtime, jsi::ArrayBuffer& buffer, std::vector<uint8_t> data, size_t offset) {
uint8_t* dataBlock = buffer.data(runtime);
size_t blockSize = buffer.size(runtime);
if (data.size() > blockSize) {
throw jsi::JSError(runtime, "ArrayBuffer is to small to fit data");
}
std::copy(data.begin(), data.end(), dataBlock + offset);
}
template <TypedArrayKind T> TypedArray<T>::TypedArray(jsi::Runtime& runtime, size_t size) : TypedArrayBase(runtime, size, T) {}
template <TypedArrayKind T>
TypedArray<T>::TypedArray(jsi::Runtime& runtime, std::vector<ContentType<T>> data) : TypedArrayBase(runtime, data.size(), T) {
update(runtime, data);
}
template <TypedArrayKind T> TypedArray<T>::TypedArray(TypedArrayBase&& base) : TypedArrayBase(std::move(base)) {}
template <TypedArrayKind T> std::vector<ContentType<T>> TypedArray<T>::toVector(jsi::Runtime& runtime) {
auto start = reinterpret_cast<ContentType<T>*>(getBuffer(runtime).data(runtime) + byteOffset(runtime));
auto end = start + size(runtime);
return std::vector<ContentType<T>>(start, end);
}
template <TypedArrayKind T> void TypedArray<T>::update(jsi::Runtime& runtime, const std::vector<ContentType<T>>& data) {
if (data.size() != size(runtime)) {
throw jsi::JSError(runtime, "TypedArray can only be updated with a vector of the same size");
}
uint8_t* rawData = getBuffer(runtime).data(runtime) + byteOffset(runtime);
std::copy(data.begin(), data.end(), reinterpret_cast<ContentType<T>*>(rawData));
}
template <TypedArrayKind T> void TypedArray<T>::updateUnsafe(jsi::Runtime& runtime, ContentType<T>* data, size_t length) {
if (length != size(runtime)) {
throw jsi::JSError(runtime, "TypedArray can only be updated with an array of the same size");
}
uint8_t* rawData = getBuffer(runtime).data(runtime) + byteOffset(runtime);
memcpy(rawData, data, length);
}
template <TypedArrayKind T> uint8_t* TypedArray<T>::data(jsi::Runtime& runtime) {
return getBuffer(runtime).data(runtime) + byteOffset(runtime);
}
const jsi::PropNameID& PropNameIDCache::getConstructorNameProp(jsi::Runtime& runtime, TypedArrayKind kind) {
switch (kind) {
case TypedArrayKind::Int8Array:
return get(runtime, Prop::Int8Array);
case TypedArrayKind::Int16Array:
return get(runtime, Prop::Int16Array);
case TypedArrayKind::Int32Array:
return get(runtime, Prop::Int32Array);
case TypedArrayKind::Uint8Array:
return get(runtime, Prop::Uint8Array);
case TypedArrayKind::Uint8ClampedArray:
return get(runtime, Prop::Uint8ClampedArray);
case TypedArrayKind::Uint16Array:
return get(runtime, Prop::Uint16Array);
case TypedArrayKind::Uint32Array:
return get(runtime, Prop::Uint32Array);
case TypedArrayKind::Float32Array:
return get(runtime, Prop::Float32Array);
case TypedArrayKind::Float64Array:
return get(runtime, Prop::Float64Array);
}
}
jsi::PropNameID PropNameIDCache::createProp(jsi::Runtime& runtime, Prop prop) {
auto create = [&](const std::string& propName) { return jsi::PropNameID::forUtf8(runtime, propName); };
switch (prop) {
case Prop::Buffer:
return create("buffer");
case Prop::Constructor:
return create("constructor");
case Prop::Name:
return create("name");
case Prop::Proto:
return create("__proto__");
case Prop::Length:
return create("length");
case Prop::ByteLength:
return create("byteLength");
case Prop::ByteOffset:
return create("byteOffset");
case Prop::IsView:
return create("isView");
case Prop::ArrayBuffer:
return create("ArrayBuffer");
case Prop::Int8Array:
return create("Int8Array");
case Prop::Int16Array:
return create("Int16Array");
case Prop::Int32Array:
return create("Int32Array");
case Prop::Uint8Array:
return create("Uint8Array");
case Prop::Uint8ClampedArray:
return create("Uint8ClampedArray");
case Prop::Uint16Array:
return create("Uint16Array");
case Prop::Uint32Array:
return create("Uint32Array");
case Prop::Float32Array:
return create("Float32Array");
case Prop::Float64Array:
return create("Float64Array");
}
}
std::unordered_map<std::string, TypedArrayKind> nameToKindMap = {
{"Int8Array", TypedArrayKind::Int8Array},
{"Int16Array", TypedArrayKind::Int16Array},
{"Int32Array", TypedArrayKind::Int32Array},
{"Uint8Array", TypedArrayKind::Uint8Array},
{"Uint8ClampedArray", TypedArrayKind::Uint8ClampedArray},
{"Uint16Array", TypedArrayKind::Uint16Array},
{"Uint32Array", TypedArrayKind::Uint32Array},
{"Float32Array", TypedArrayKind::Float32Array},
{"Float64Array", TypedArrayKind::Float64Array},
};
TypedArrayKind getTypedArrayKindForName(const std::string& name) {
return nameToKindMap.at(name);
}
template class TypedArray<TypedArrayKind::Int8Array>;
template class TypedArray<TypedArrayKind::Int16Array>;
template class TypedArray<TypedArrayKind::Int32Array>;
template class TypedArray<TypedArrayKind::Uint8Array>;
template class TypedArray<TypedArrayKind::Uint8ClampedArray>;
template class TypedArray<TypedArrayKind::Uint16Array>;
template class TypedArray<TypedArrayKind::Uint32Array>;
template class TypedArray<TypedArrayKind::Float32Array>;
template class TypedArray<TypedArrayKind::Float64Array>;
} // namespace vision

View File

@@ -1,141 +0,0 @@
//
// JSITypedArray.h
// VisionCamera
//
// Created by Marc Rousavy on 21.02.23.
// Copyright © 2023 mrousavy. All rights reserved.
//
// Copied & Adapted from
// https://github.com/expo/expo/blob/main/packages/expo-gl/common/EXTypedArrayApi.h Credits to Expo
#pragma once
#include <jsi/jsi.h>
#include <utility>
#include <vector>
namespace jsi = facebook::jsi;
namespace vision {
enum class TypedArrayKind {
Int8Array,
Int16Array,
Int32Array,
Uint8Array,
Uint8ClampedArray,
Uint16Array,
Uint32Array,
Float32Array,
Float64Array,
};
template <TypedArrayKind T> class TypedArray;
template <TypedArrayKind T> struct typedArrayTypeMap;
template <> struct typedArrayTypeMap<TypedArrayKind::Int8Array> {
typedef int8_t type;
};
template <> struct typedArrayTypeMap<TypedArrayKind::Int16Array> {
typedef int16_t type;
};
template <> struct typedArrayTypeMap<TypedArrayKind::Int32Array> {
typedef int32_t type;
};
template <> struct typedArrayTypeMap<TypedArrayKind::Uint8Array> {
typedef uint8_t type;
};
template <> struct typedArrayTypeMap<TypedArrayKind::Uint8ClampedArray> {
typedef uint8_t type;
};
template <> struct typedArrayTypeMap<TypedArrayKind::Uint16Array> {
typedef uint16_t type;
};
template <> struct typedArrayTypeMap<TypedArrayKind::Uint32Array> {
typedef uint32_t type;
};
template <> struct typedArrayTypeMap<TypedArrayKind::Float32Array> {
typedef float type;
};
template <> struct typedArrayTypeMap<TypedArrayKind::Float64Array> {
typedef double type;
};
void invalidateArrayBufferCache(jsi::Runtime& runtime);
class TypedArrayBase : public jsi::Object {
public:
template <TypedArrayKind T> using ContentType = typename typedArrayTypeMap<T>::type;
TypedArrayBase(jsi::Runtime&, size_t, TypedArrayKind);
TypedArrayBase(jsi::Runtime&, const jsi::Object&);
TypedArrayBase(TypedArrayBase&&) = default;
TypedArrayBase& operator=(TypedArrayBase&&) = default;
TypedArrayKind getKind(jsi::Runtime& runtime) const;
template <TypedArrayKind T> TypedArray<T> get(jsi::Runtime& runtime) const&;
template <TypedArrayKind T> TypedArray<T> get(jsi::Runtime& runtime) &&;
template <TypedArrayKind T> TypedArray<T> as(jsi::Runtime& runtime) const&;
template <TypedArrayKind T> TypedArray<T> as(jsi::Runtime& runtime) &&;
size_t size(jsi::Runtime& runtime) const;
size_t length(jsi::Runtime& runtime) const;
size_t byteLength(jsi::Runtime& runtime) const;
size_t byteOffset(jsi::Runtime& runtime) const;
bool hasBuffer(jsi::Runtime& runtime) const;
std::vector<uint8_t> toVector(jsi::Runtime& runtime);
jsi::ArrayBuffer getBuffer(jsi::Runtime& runtime) const;
private:
template <TypedArrayKind> friend class TypedArray;
};
bool isTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj);
TypedArrayBase getTypedArray(jsi::Runtime& runtime, const jsi::Object& jsObj);
std::vector<uint8_t> arrayBufferToVector(jsi::Runtime& runtime, jsi::Object& jsObj);
void arrayBufferUpdate(jsi::Runtime& runtime, jsi::ArrayBuffer& buffer, std::vector<uint8_t> data, size_t offset);
template <TypedArrayKind T> class TypedArray : public TypedArrayBase {
public:
explicit TypedArray(TypedArrayBase&& base);
TypedArray(jsi::Runtime& runtime, size_t size);
TypedArray(jsi::Runtime& runtime, std::vector<ContentType<T>> data);
TypedArray(TypedArray&&) = default;
TypedArray& operator=(TypedArray&&) = default;
std::vector<ContentType<T>> toVector(jsi::Runtime& runtime);
void update(jsi::Runtime& runtime, const std::vector<ContentType<T>>& data);
void updateUnsafe(jsi::Runtime& runtime, ContentType<T>* data, size_t length);
uint8_t* data(jsi::Runtime& runtime);
};
template <TypedArrayKind T> TypedArray<T> TypedArrayBase::get(jsi::Runtime& runtime) const& {
assert(getKind(runtime) == T);
(void)runtime; // when assert is disabled we need to mark this as used
return TypedArray<T>(jsi::Value(runtime, jsi::Value(runtime, *this).asObject(runtime)));
}
template <TypedArrayKind T> TypedArray<T> TypedArrayBase::get(jsi::Runtime& runtime) && {
assert(getKind(runtime) == T);
(void)runtime; // when assert is disabled we need to mark this as used
return TypedArray<T>(std::move(*this));
}
template <TypedArrayKind T> TypedArray<T> TypedArrayBase::as(jsi::Runtime& runtime) const& {
if (getKind(runtime) != T) {
throw jsi::JSError(runtime, "Object is not a TypedArray");
}
return get<T>(runtime);
}
template <TypedArrayKind T> TypedArray<T> TypedArrayBase::as(jsi::Runtime& runtime) && {
if (getKind(runtime) != T) {
throw jsi::JSError(runtime, "Object is not a TypedArray");
}
return std::move(*this).get<T>(runtime);
}
} // namespace vision

View File

@@ -0,0 +1,36 @@
//
// MutableRawBuffer.cpp
// VisionCamera
//
// Created by Marc Rousavy on 17.01.24.
// Copyright © 2024 mrousavy. All rights reserved.
//
#include "MutableRawBuffer.h"
#include <functional>
#include <memory>
namespace vision {
MutableRawBuffer::MutableRawBuffer(uint8_t* data, size_t size, std::function<void()> cleanup)
: _data(data), _size(size), _cleanup(std::move(cleanup)) {}
MutableRawBuffer::MutableRawBuffer(size_t size) {
_size = size;
_data = (uint8_t*)malloc(size * sizeof(uint8_t));
_cleanup = [=]() { free(_data); };
}
MutableRawBuffer::~MutableRawBuffer() {
_cleanup();
}
size_t MutableRawBuffer::size() const {
return _size;
}
uint8_t* MutableRawBuffer::data() {
return _data;
}
} // namespace vision

View File

@@ -0,0 +1,36 @@
//
// MutableRawBuffer.h
// VisionCamera
//
// Created by Marc Rousavy on 17.01.24.
// Copyright © 2024 mrousavy. All rights reserved.
//
#pragma once
#include <functional>
#include <jsi/jsi.h>
#include <memory>
namespace vision {
using namespace facebook;
class MutableRawBuffer : public jsi::MutableBuffer {
public:
explicit MutableRawBuffer(size_t size);
explicit MutableRawBuffer(uint8_t* data, size_t size, std::function<void()> cleanup);
~MutableRawBuffer();
public:
uint8_t* data() override;
size_t size() const override;
private:
uint8_t* _data;
size_t _size;
std::function<void()> _cleanup;
};
} // namespace vision