railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

feat: Skia for Android (#1731)

Browse Source 
* feat: Call Skia Renderer

* Use default NativePreviewView for Skia

* Render to separate FBO

* It appears once

* Refactor a lot lol

* Pass width/height

* Read width/heights

* Update SkiaRenderer.cpp

* Read stencil/samples

* Use switch for target

* Clear full red

* Update VideoPipeline.cpp

* fix: Use `BorrowTextureFrom` instead of `AdoptTextureFrom`

* Get it to work

* Draw Camera Frame again (only works for first frame)

* glDisable(GL_BLEND)

* Use Frame Buffer again

* Simplify Skia offscreen surface creation

* fix: Get it to kinda work?

* fix: Remove `sampler2D` shader

Only the EXTERNAL_OES one kinda works

* Revert "fix: Remove `sampler2D` shader"

This reverts commit bf241a82f440f5a442f23a2b10329b813e7cdb3e.

* Revert "fix: Get it to kinda work?"

This reverts commit ea6a8784ad8dc7d05e8076591874f021b51dd84a.

* fix: Use Skia for rendering

* Simplify drawing code a lot

* Clean up drawing loop a bit more

* Some docs

* Update SkiaRenderer.cpp

* Surface

* try to use Matrix

* Use BottomLeft as a surface origin again

* Get actual surface dimensions

* Use 1x1 pbuffer instead

* Update SkiaRenderer.cpp

* Update SkiaRenderer.cpp

* feat: Implement Skia Frame Processor (#1735)

* feat: Implement JS Skia Frame Processor

* Update SkiaRenderer.cpp

* push

* Create Frame from C++

* compile

* Compile

* Update VideoPipeline.cpp

* Fix JNI local ref

* Use `HardwareBuffer` for implementation

* feat: Custom `Frame` implementation that uses CPU `ByteBuffer` (#1736)

* feat: Implement JS Skia Frame Processor

* Update SkiaRenderer.cpp

* push

* Create Frame from C++

* compile

* Compile

* Update VideoPipeline.cpp

* Fix JNI local ref

* Use `HardwareBuffer` for implementation

* try: Try to just create a CPU based ByteBuffer

* fix: Fix Java Type

* fix remaining errors

* try fixing FrameFactory

* Use `free`

* fix: Fix scene mode crash on some emulators

* fix: Fix scene mode crash on some emulators

* Fix getting pixels

* fix: Fix buffer not being freed

* Add some docs to `Frame`

* Test Skia again

* Use `getCurrentPresentationTime()`

* Remove `FrameFactory.cpp`

* Update VideoPipeline.h

* Update VideoPipeline.cpp
This commit is contained in:
Marc Rousavy
2023-09-01 10:43:19 +02:00
committed by GitHub
parent 6bbb44d541
commit a7c137da07
47 changed files with 1099 additions and 962 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
android/src/main/cpp/frameprocessor/FrameHostObject.cpp
View File

@@ -31,7 +31,6 @@ std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt)
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("width")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("height")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("bytesPerRow")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("planesCount")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("orientation")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isMirrored")));
result.push_back(jsi::PropNameID::forUtf8(rt, std::string("timestamp")));
@@ -55,7 +54,7 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
const jsi::Value* args,
size_t count) -> jsi::Value {
// Increment retain count by one.
this->frame->incrementRefCount();
this->frame->cthis()->incrementRefCount();
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime,
@@ -69,7 +68,7 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
const jsi::Value* args,
size_t count) -> jsi::Value {
// Decrement retain count by one. If the retain count is zero, the Frame gets closed.
this->frame->decrementRefCount();
this->frame->cthis()->decrementRefCount();
return jsi::Value::undefined();
};
return jsi::Function::createFromHostFunction(runtime,
@@ -85,8 +84,8 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
if (!this->frame) {
return jsi::String::createFromUtf8(runtime, "[closed frame]");
}
auto width = this->frame->getWidth();
auto height = this->frame->getHeight();
auto width = this->frame->cthis()->getWidth();
auto height = this->frame->cthis()->getHeight();
auto str = std::to_string(width) + " x " + std::to_string(height) + " Frame";
return jsi::String::createFromUtf8(runtime, str);
};
@@ -97,11 +96,8 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
const jsi::Value& thisArg,
const jsi::Value* args,
size_t count) -> jsi::Value {
auto buffer = this->frame->toByteBuffer();
if (!buffer->isDirect()) {
throw std::runtime_error("Failed to get byte content of Frame - array is not direct ByteBuffer!");
}
auto size = buffer->getDirectSize();
size_t size = frame->cthis()->pixelsSize;
uint8_t* pixels = frame->cthis()->pixels;
static constexpr auto ARRAYBUFFER_CACHE_PROP_NAME = "__frameArrayBufferCache";
if (!runtime.global().hasProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME)) {
@@ -119,7 +115,7 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
// directly write to C++ JSI ArrayBuffer
auto destinationBuffer = arrayBuffer.data(runtime);
memcpy(destinationBuffer, buffer->getDirectAddress(), sizeof(uint8_t) * size);
memcpy(destinationBuffer, pixels, sizeof(uint8_t) * size);
return arrayBuffer;
};
@@ -127,33 +123,30 @@ jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& pr
}
if (name == "isValid") {
return jsi::Value(this->frame && this->frame->getIsValid());
return jsi::Value(this->frame && this->frame->cthis()->getIsValid());
}
if (name == "width") {
return jsi::Value(this->frame->getWidth());
return jsi::Value(this->frame->cthis()->getWidth());
}
if (name == "height") {
return jsi::Value(this->frame->getHeight());
return jsi::Value(this->frame->cthis()->getHeight());
}
if (name == "isMirrored") {
return jsi::Value(this->frame->getIsMirrored());
return jsi::Value(this->frame->cthis()->getIsMirrored());
}
if (name == "orientation") {
auto string = this->frame->getOrientation();
auto string = this->frame->cthis()->getOrientation();
return jsi::String::createFromUtf8(runtime, string->toStdString());
}
if (name == "pixelFormat") {
auto string = this->frame->getPixelFormat();
auto string = this->frame->cthis()->getPixelFormat();
return jsi::String::createFromUtf8(runtime, string->toStdString());
}
if (name == "timestamp") {
return jsi::Value(static_cast<double>(this->frame->getTimestamp()));
return jsi::Value(static_cast<double>(this->frame->cthis()->getTimestamp()));
}
if (name == "bytesPerRow") {
return jsi::Value(this->frame->getBytesPerRow());
}
if (name == "planesCount") {
return jsi::Value(this->frame->getPlanesCount());
return jsi::Value(this->frame->cthis()->getBytesPerRow());
}
// fallback to base implementation

2
android/src/main/cpp/frameprocessor/FrameHostObject.h
View File

@@ -26,7 +26,7 @@ class JSI_EXPORT FrameHostObject : public jsi::HostObject {
std::vector<jsi::PropNameID> getPropertyNames(jsi::Runtime &rt) override;
public:
jni::global_ref<JFrame> frame;
jni::global_ref<JFrame::javaobject> frame;
};
} // namespace vision

10
android/src/main/cpp/frameprocessor/JSIJNIConversion.cpp
View File

@@ -111,10 +111,10 @@ jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, c
return jsi::String::createFromUtf8(runtime, object->toString());
} else if (object->isInstanceOf(JList<jobject>::javaClassStatic())) {
} else if (object->isInstanceOf(jni::JList<jobject>::javaClassStatic())) {
// List<E>
auto arrayList = static_ref_cast<JList<jobject>>(object);
auto arrayList = jni::static_ref_cast<jni::JList<jobject>>(object);
auto size = arrayList->size();
auto result = jsi::Array(runtime, size);
@@ -125,10 +125,10 @@ jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, c
}
return result;
} else if (object->isInstanceOf(JMap<jstring, jobject>::javaClassStatic())) {
} else if (object->isInstanceOf(jni::JMap<jstring, jobject>::javaClassStatic())) {
// Map<K, V>
auto map = static_ref_cast<JMap<jstring, jobject>>(object);
auto map = jni::static_ref_cast<jni::JMap<jstring, jobject>>(object);
auto result = jsi::Object(runtime);
for (const auto& entry : *map) {
@@ -140,7 +140,7 @@ jsi::Value JSIJNIConversion::convertJNIObjectToJSIValue(jsi::Runtime &runtime, c
return result;
} else if (object->isInstanceOf(JFrame::javaClassStatic())) {
// Frame
auto frame = static_ref_cast<JFrame>(object);
auto frame = jni::static_ref_cast<JFrame::javaobject>(object);
// box into HostObject
auto hostObject = std::make_shared<FrameHostObject>(frame);

104
android/src/main/cpp/frameprocessor/java-bindings/JFrame.cpp
View File

@@ -11,71 +11,85 @@
namespace vision {
using namespace facebook;
using namespace jni;
int JFrame::getWidth() const {
static const auto getWidthMethod = getClass()->getMethod<jint()>("getWidth");
return getWidthMethod(self());
void JFrame::registerNatives() {
registerHybrid({
makeNativeMethod("getWidth", JFrame::getWidth),
makeNativeMethod("getHeight", JFrame::getHeight),
makeNativeMethod("getBytesPerRow", JFrame::getBytesPerRow),
makeNativeMethod("getTimestamp", JFrame::getTimestamp),
makeNativeMethod("getOrientation", JFrame::getOrientation),
makeNativeMethod("getIsMirrored", JFrame::getIsMirrored),
makeNativeMethod("getPixelFormat", JFrame::getPixelFormat),
makeNativeMethod("getByteBuffer", JFrame::getByteBuffer),
makeNativeMethod("getIsValid", JFrame::getIsValid),
});
}
int JFrame::getHeight() const {
static const auto getWidthMethod = getClass()->getMethod<jint()>("getHeight");
return getWidthMethod(self());
jni::local_ref<JFrame::javaobject> JFrame::create(int width,
int height,
int bytesPerRow,
long timestamp,
const std::string& orientation,
bool isMirrored) {
return newObjectCxxArgs(width,
height,
bytesPerRow,
timestamp,
orientation,
isMirrored);
}
bool JFrame::getIsValid() const {
static const auto getIsValidMethod = getClass()->getMethod<jboolean()>("getIsValid");
return getIsValidMethod(self());
JFrame::JFrame(int width,
int height,
int bytesPerRow,
long timestamp,
const std::string& orientation,
bool isMirrored) {
_width = width;
_height = height;
_bytesPerRow = bytesPerRow;
_timestamp = timestamp;
_orientation = orientation;
_isMirrored = isMirrored;
_refCount = 0;
pixelsSize = height * bytesPerRow;
pixels = (uint8_t*) malloc(pixelsSize);
}
bool JFrame::getIsMirrored() const {
static const auto getIsMirroredMethod = getClass()->getMethod<jboolean()>("getIsMirrored");
return getIsMirroredMethod(self());
JFrame::~JFrame() noexcept {
close();
}
jlong JFrame::getTimestamp() const {
static const auto getTimestampMethod = getClass()->getMethod<jlong()>("getTimestamp");
return getTimestampMethod(self());
bool JFrame::getIsValid() {
return _refCount > 0 && !_isClosed;
}
local_ref<JString> JFrame::getOrientation() const {
static const auto getOrientationMethod = getClass()->getMethod<JString()>("getOrientation");
return getOrientationMethod(self());
}
local_ref<JString> JFrame::getPixelFormat() const {
static const auto getPixelFormatMethod = getClass()->getMethod<JString()>("getPixelFormat");
return getPixelFormatMethod(self());
}
int JFrame::getPlanesCount() const {
static const auto getPlanesCountMethod = getClass()->getMethod<jint()>("getPlanesCount");
return getPlanesCountMethod(self());
}
int JFrame::getBytesPerRow() const {
static const auto getBytesPerRowMethod = getClass()->getMethod<jint()>("getBytesPerRow");
return getBytesPerRowMethod(self());
}
local_ref<JByteBuffer> JFrame::toByteBuffer() const {
static const auto toByteBufferMethod = getClass()->getMethod<JByteBuffer()>("toByteBuffer");
return toByteBufferMethod(self());
jni::local_ref<jni::JByteBuffer> JFrame::getByteBuffer() {
if (!getIsValid()) {
[[unlikely]]
throw std::runtime_error("Frame is no longer valid, cannot access getByteBuffer!");
}
return jni::JByteBuffer::wrapBytes(pixels, pixelsSize);
}
void JFrame::incrementRefCount() {
static const auto incrementRefCountMethod = getClass()->getMethod<void()>("incrementRefCount");
incrementRefCountMethod(self());
std::unique_lock lock(_mutex);
_refCount++;
}
void JFrame::decrementRefCount() {
static const auto decrementRefCountMethod = getClass()->getMethod<void()>("decrementRefCount");
decrementRefCountMethod(self());
std::unique_lock lock(_mutex);
_refCount--;
if (_refCount <= 0) {
this->close();
}
}
void JFrame::close() {
static const auto closeMethod = getClass()->getMethod<void()>("close");
closeMethod(self());
_isClosed = true;
free(pixels);
pixels = nullptr;
}
} // namespace vision

65
android/src/main/cpp/frameprocessor/java-bindings/JFrame.h
View File

@@ -7,29 +7,70 @@
#include <jni.h>
#include <fbjni/fbjni.h>
#include <fbjni/ByteBuffer.h>
#include <android/hardware_buffer.h>
#include <android/hardware_buffer_jni.h>
#include <mutex>
namespace vision {
using namespace facebook;
using namespace jni;
struct JFrame : public JavaClass<JFrame> {
class JFrame : public jni::HybridClass<JFrame> {
public:
static constexpr auto kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/Frame;";
static void registerNatives();
static jni::local_ref<JFrame::javaobject> create(int width,
int height,
int bytesPerRow,
long timestamp,
const std::string& orientation,
bool isMirrored);
~JFrame() noexcept;
protected:
friend HybridBase;
explicit JFrame(int width,
int height,
int bytesPerRow,
long timestamp,
const std::string& orientation,
bool isMirrored);
public:
int getWidth() const;
int getHeight() const;
bool getIsValid() const;
bool getIsMirrored() const;
int getPlanesCount() const;
int getBytesPerRow() const;
jlong getTimestamp() const;
local_ref<JString> getOrientation() const;
local_ref<JString> getPixelFormat() const;
local_ref<JByteBuffer> toByteBuffer() const;
int getWidth() { return _width; }
int getHeight() { return _height; }
int getBytesPerRow() { return _bytesPerRow; }
jlong getTimestamp() { return _timestamp; }
jni::local_ref<jni::JString> getOrientation() { return jni::make_jstring(_orientation); }
bool getIsMirrored() { return _isMirrored; }
// TODO: Can this be something other than RGB?
jni::local_ref<jni::JString> getPixelFormat() { return jni::make_jstring("rgb"); }
bool getIsValid();
jni::local_ref<jni::JByteBuffer> getByteBuffer();
void incrementRefCount();
void decrementRefCount();
void close();
// Backing byte data
uint8_t* pixels = nullptr;
size_t pixelsSize = 0;
private:
// Frame info
int _width = 0;
int _height = 0;
int _bytesPerRow = 0;
long _timestamp = 0;
std::string _orientation = {};
bool _isMirrored = false;
// Ref-counting
int _refCount = 0;
bool _isClosed = false;
std::mutex _mutex;
};
} // namespace vision

3
android/src/main/cpp/frameprocessor/java-bindings/JFrameProcessor.cpp
View File

@@ -17,9 +17,6 @@ using namespace facebook;
using namespace jni;
void JFrameProcessor::registerNatives() {
registerHybrid({
makeNativeMethod("call", JFrameProcessor::call)
});
}
using TSelf = jni::local_ref<JFrameProcessor::javaobject>;

19
android/src/main/cpp/frameprocessor/java-bindings/JFrameProcessor.h
View File

@@ -21,7 +21,7 @@ namespace vision {
using namespace facebook;
struct JFrameProcessor : public jni::HybridClass<JFrameProcessor> {
class JFrameProcessor : public jni::HybridClass<JFrameProcessor> {
public:
static auto constexpr kJavaDescriptor = "Lcom/mrousavy/camera/frameprocessor/FrameProcessor;";
static void registerNatives();
@@ -30,20 +30,25 @@ struct JFrameProcessor : public jni::HybridClass<JFrameProcessor> {
public:
/**
* Call the JS Frame Processor.
* Wrap the Frame in a HostObject and call the Frame Processor.
*/
void call(alias_ref<JFrame::javaobject> frame);
void call(jni::alias_ref<JFrame::javaobject> frame);
private:
// Private constructor. Use `create(..)` to create new instances.
protected:
friend HybridBase;
// C++ only constructor. Use `create(..)` to create new instances.
explicit JFrameProcessor(std::shared_ptr<RNWorklet::JsiWorklet> worklet,
std::shared_ptr<RNWorklet::JsiWorkletContext> context);
JFrameProcessor(const JFrameProcessor &) = delete;
JFrameProcessor &operator=(const JFrameProcessor &) = delete;
private:
protected:
/**
* Call the JS Frame Processor with the given Frame Host Object.
*/
void callWithFrameHostObject(const std::shared_ptr<FrameHostObject>& frameHostObject) const;
private:
friend HybridBase;
std::shared_ptr<RNWorklet::WorkletInvoker> _workletInvoker;
std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;
};

7
android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraProxy.cpp
View File

@@ -18,6 +18,10 @@
#include <react-native-worklets-core/WKTJsiWorkletContext.h>
#endif
#if VISION_CAMERA_ENABLE_SKIA
#include "JSkiaFrameProcessor.h"
#endif
namespace vision {
using TSelf = local_ref<HybridClass<JVisionCameraProxy>::jhybriddata>;
@@ -31,6 +35,7 @@ JVisionCameraProxy::JVisionCameraProxy(const jni::alias_ref<JVisionCameraProxy::
const jni::global_ref<JVisionCameraScheduler::javaobject>& scheduler) {
_javaPart = make_global(javaThis);
_runtime = runtime;
_callInvoker = callInvoker;
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
__android_log_write(ANDROID_LOG_INFO, TAG, "Creating Worklet Context...");
@@ -84,7 +89,7 @@ void JVisionCameraProxy::setFrameProcessor(int viewTag,
frameProcessor = JFrameProcessor::create(worklet, _workletContext);
} else if (frameProcessorType == "skia-frame-processor") {
#if VISION_CAMERA_ENABLE_SKIA
throw std::runtime_error("system/skia-unavailable: Skia is not yet implemented on Android!");
frameProcessor = JSkiaFrameProcessor::create(worklet, _workletContext, _callInvoker);
#else
throw std::runtime_error("system/skia-unavailable: Skia is not installed!");
#endif

2
android/src/main/cpp/frameprocessor/java-bindings/JVisionCameraProxy.h
View File

@@ -36,11 +36,13 @@ class JVisionCameraProxy : public jni::HybridClass<JVisionCameraProxy> {
jni::local_ref<JMap<jstring, jobject>> options);
jsi::Runtime* getJSRuntime() { return _runtime; }
std::shared_ptr<react::CallInvoker> getCallInvoker() { return _callInvoker; }
private:
friend HybridBase;
jni::global_ref<JVisionCameraProxy::javaobject> _javaPart;
jsi::Runtime* _runtime;
std::shared_ptr<react::CallInvoker> _callInvoker;
#if VISION_CAMERA_ENABLE_FRAME_PROCESSORS
std::shared_ptr<RNWorklet::JsiWorkletContext> _workletContext;
#endif