react-native-vision-camera/package/ios/Frame Processor/FrameHostObject.mm

//
//  FrameHostObject.m
//  VisionCamera
//
//  Created by Marc Rousavy on 22.03.21.
//  Copyright © 2021 mrousavy. All rights reserved.
//

#import "FrameHostObject.h"
#import "UIImageOrientation+descriptor.h"
#import "WKTJsiHostObject.h"
#import <Foundation/Foundation.h>
#import <jsi/jsi.h>

#import "../../cpp/JSITypedArray.h"

std::vector<jsi::PropNameID> FrameHostObject::getPropertyNames(jsi::Runtime& rt) {
  std::vector<jsi::PropNameID> result;
  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("width")));
  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("height")));
  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("bytesPerRow")));
  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("planesCount")));
  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("orientation")));
  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isMirrored")));
  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("timestamp")));
  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("pixelFormat")));
  // Conversion
  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toString")));
  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("toArrayBuffer")));
  // Ref Management
  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("isValid")));
  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("incrementRefCount")));
  result.push_back(jsi::PropNameID::forUtf8(rt, std::string("decrementRefCount")));

  return result;
}

Frame* FrameHostObject::getFrame() {
  Frame* frame = this->frame;
  if (frame == nil || !CMSampleBufferIsValid(frame.buffer)) {
    throw std::runtime_error("Frame is already closed! "
                             "Are you trying to access the Image data outside of a Frame Processor's lifetime?\n"
                             "- If you want to use `console.log(frame)`, use `console.log(frame.toString())` instead.\n"
                             "- If you want to do async processing, use `runAsync(...)` instead.\n"
                             "- If you want to use runOnJS, increment it's ref-count: `frame.incrementRefCount()`");
  }
  return frame;
}

jsi::Value FrameHostObject::get(jsi::Runtime& runtime, const jsi::PropNameID& propName) {
  auto name = propName.utf8(runtime);

  if (name == "toString") {
    auto toString = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments, size_t count) -> jsi::Value {
      // Lock Frame so it cannot be deallocated while we access it
      std::lock_guard lock(this->_mutex);

      // Print debug description (width, height)
      Frame* frame = this->getFrame();
      NSMutableString* string = [NSMutableString stringWithFormat:@"%lu x %lu %@ Frame", frame.width, frame.height, frame.pixelFormat];
      return jsi::String::createFromUtf8(runtime, string.UTF8String);
    };
    return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toString"), 0, toString);
  }
  if (name == "incrementRefCount") {
    auto incrementRefCount = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
                                    size_t count) -> jsi::Value {
      // Lock Frame so it cannot be deallocated while we access it
      std::lock_guard lock(this->_mutex);

      // Increment our self-counted ref count by one.
      _refCount++;
      return jsi::Value::undefined();
    };
    return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "incrementRefCount"), 0, incrementRefCount);
  }
  if (name == "decrementRefCount") {
    auto decrementRefCount = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
                                    size_t count) -> jsi::Value {
      // Lock Frame so it cannot be deallocated while we access it
      std::lock_guard lock(this->_mutex);

      // Decrement our self-counted ref count by one.
      _refCount--;
      if (_refCount < 1) {
        // ARC will then delete the Frame and the underlying Frame Buffer.
        this->frame = nil;
      }

      return jsi::Value::undefined();
    };
    return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "decrementRefCount"), 0, decrementRefCount);
  }
  if (name == "toArrayBuffer") {
    auto toArrayBuffer = [this](jsi::Runtime& runtime, const jsi::Value& thisValue, const jsi::Value* arguments,
                                size_t count) -> jsi::Value {
      // Lock Frame so it cannot be deallocated while we access it
      std::lock_guard lock(this->_mutex);

      // Get CPU readable Pixel Buffer from Frame and write it to a jsi::ArrayBuffer
      Frame* frame = this->getFrame();
      auto pixelBuffer = CMSampleBufferGetImageBuffer(frame.buffer);
      auto bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer);
      auto height = CVPixelBufferGetHeight(pixelBuffer);

      auto arraySize = bytesPerRow * height;

      static constexpr auto ARRAYBUFFER_CACHE_PROP_NAME = "__frameArrayBufferCache";
      if (!runtime.global().hasProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME)) {
        vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray> arrayBuffer(runtime, arraySize);
        runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer);
      }

      auto arrayBufferCache = runtime.global().getPropertyAsObject(runtime, ARRAYBUFFER_CACHE_PROP_NAME);
      auto arrayBuffer = vision::getTypedArray(runtime, arrayBufferCache).get<vision::TypedArrayKind::Uint8ClampedArray>(runtime);

      if (arrayBuffer.size(runtime) != arraySize) {
        arrayBuffer = vision::TypedArray<vision::TypedArrayKind::Uint8ClampedArray>(runtime, arraySize);
        runtime.global().setProperty(runtime, ARRAYBUFFER_CACHE_PROP_NAME, arrayBuffer);
      }

      CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);
      auto buffer = (uint8_t*)CVPixelBufferGetBaseAddress(pixelBuffer);
      arrayBuffer.updateUnsafe(runtime, buffer, arraySize);
      CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly);

      return arrayBuffer;
    };
    return jsi::Function::createFromHostFunction(runtime, jsi::PropNameID::forUtf8(runtime, "toArrayBuffer"), 0, toArrayBuffer);
  }

  if (name == "isValid") {
    // Lock Frame so it cannot be deallocated while we access it
    std::lock_guard lock(this->_mutex);

    // unsafely access the Frame and try to see if it's valid
    Frame* frame = this->frame;
    return jsi::Value(frame != nil && frame.isValid);
  }
  if (name == "width") {
    // Lock Frame so it cannot be deallocated while we access it
    std::lock_guard lock(this->_mutex);

    Frame* frame = this->getFrame();
    return jsi::Value((double)frame.width);
  }
  if (name == "height") {
    // Lock Frame so it cannot be deallocated while we access it
    std::lock_guard lock(this->_mutex);

    Frame* frame = this->getFrame();
    return jsi::Value((double)frame.height);
  }
  if (name == "orientation") {
    // Lock Frame so it cannot be deallocated while we access it
    std::lock_guard lock(this->_mutex);

    Frame* frame = this->getFrame();
    NSString* orientation = [NSString stringWithParsed:frame.orientation];
    return jsi::String::createFromUtf8(runtime, orientation.UTF8String);
  }
  if (name == "isMirrored") {
    // Lock Frame so it cannot be deallocated while we access it
    std::lock_guard lock(this->_mutex);

    Frame* frame = this->getFrame();
    return jsi::Value(frame.isMirrored);
  }
  if (name == "timestamp") {
    // Lock Frame so it cannot be deallocated while we access it
    std::lock_guard lock(this->_mutex);

    Frame* frame = this->getFrame();
    return jsi::Value(frame.timestamp);
  }
  if (name == "pixelFormat") {
    // Lock Frame so it cannot be deallocated while we access it
    std::lock_guard lock(this->_mutex);

    Frame* frame = this->getFrame();
    return jsi::String::createFromUtf8(runtime, frame.pixelFormat.UTF8String);
  }
  if (name == "bytesPerRow") {
    // Lock Frame so it cannot be deallocated while we access it
    std::lock_guard lock(this->_mutex);

    Frame* frame = this->getFrame();
    return jsi::Value((double)frame.bytesPerRow);
  }
  if (name == "planesCount") {
    // Lock Frame so it cannot be deallocated while we access it
    std::lock_guard lock(this->_mutex);

    Frame* frame = this->getFrame();
    return jsi::Value((double)frame.planesCount);
  }

  // fallback to base implementation
  return HostObject::get(runtime, propName);
}