feat: Draw onto Frame as if it was a Skia Canvas (#1479)

* Create Shaders.ts * Add `previewType` and `enableFpsGraph` * Add RN Skia native dependency * Add Skia Preview View on iOS * Pass 1 * Update FrameHostObject.mm * Wrap Canvas * Lockfiles * fix: Fix stuff * chore: Upgrade RNWorklets * Add `previewType` to set the Preview * feat: Add Example * Update project.pbxproj * `enableFpsGraph` * Cache the `std::shared_ptr<FrameHostObject>` * Update CameraView+RecordVideo.swift * Update SkiaMetalCanvasProvider.mm * Android: Integrate Skia Dependency * fix: Use new Prefix * Add example for rendering shader * chore: Upgrade CameraX * Remove KTX * Enable `viewBinding` * Revert "Enable `viewBinding`" This reverts commit f2a603f53b33ea4311a296422ffd1a910ce03f9e. * Revert "chore: Upgrade CameraX" This reverts commit 8dc832cf8754490d31a6192e6c1a1f11cdcd94fe. * Remove unneeded `ProcessCameraProvider.getInstance()` call * fix: Add REA hotfix patch * fix: Fix FrameHostObject dead in runAsync * fix: Make `runAsync` run truly async by dropping new Frames while executing * chore: Upgrade RN Worklets to latest * chore: Upgrade RN Skia * Revert "Remove KTX" This reverts commit 253f586633f7af2da992d2279fc206dc62597129. * Make Skia optional in CMake * Fix import * Update CMakeLists.txt * Update build.gradle * Update CameraView.kt * Update CameraView.kt * Update CameraView.kt * Update Shaders.ts * Center Blur * chore: Upgrade RN Worklets * feat: Add `toByteArray()`, `orientation`, `isMirrored` and `timestamp` to `Frame` (#1487) * feat: Implement `orientation` and `isMirrored` on Frame * feat: Add `toArrayBuffer()` func * perf: Do faster buffer copy * feat: Implement `toArrayBuffer()` on Android * feat: Add `orientation` and `isMirrored` to Android * feat: Add `timestamp` to Frame * Update Frame.ts * Update JImageProxy.h * Update FrameHostObject.cpp * Update FrameHostObject.cpp * Update CameraPage.tsx * fix: Format Swift
2023-02-21 15:00:48 +01:00
parent 1f7a2e07f2
commit 12f850c8e1
49 changed files with 2166 additions and 85 deletions
--- a/example/ios/Podfile.lock
+++ b/example/ios/Podfile.lock
@@ -276,6 +276,35 @@ PODS:
    - RCTTypeSafety
    - React-Core
    - ReactCommon/turbomodule/core
+  - react-native-skia (0.1.175):
+    - React
+    - React-callinvoker
+    - React-Core
+    - react-native-skia/Api (= 0.1.175)
+    - react-native-skia/Jsi (= 0.1.175)
+    - react-native-skia/RNSkia (= 0.1.175)
+    - react-native-skia/SkiaHeaders (= 0.1.175)
+    - react-native-skia/Utils (= 0.1.175)
+  - react-native-skia/Api (0.1.175):
+    - React
+    - React-callinvoker
+    - React-Core
+  - react-native-skia/Jsi (0.1.175):
+    - React
+    - React-callinvoker
+    - React-Core
+  - react-native-skia/RNSkia (0.1.175):
+    - React
+    - React-callinvoker
+    - React-Core
+  - react-native-skia/SkiaHeaders (0.1.175):
+    - React
+    - React-callinvoker
+    - React-Core
+  - react-native-skia/Utils (0.1.175):
+    - React
+    - React-callinvoker
+    - React-Core
  - react-native-slider (4.4.2):
    - React-Core
  - react-native-video (5.2.1):
@@ -407,10 +436,11 @@ PODS:
    - React-Core
  - RNVectorIcons (9.2.0):
    - React-Core
-  - VisionCamera (2.15.4):
+  - VisionCamera (3.0.0-rc.1):
    - React
    - React-callinvoker
    - React-Core
+    - react-native-skia
    - react-native-worklets
  - Yoga (1.14.0)

@@ -440,6 +470,7 @@ DEPENDENCIES:
  - "react-native-blur (from `../node_modules/@react-native-community/blur`)"
  - "react-native-cameraroll (from `../node_modules/@react-native-camera-roll/camera-roll`)"
  - react-native-safe-area-context (from `../node_modules/react-native-safe-area-context`)
+  - "react-native-skia (from `../node_modules/@shopify/react-native-skia`)"
  - "react-native-slider (from `../node_modules/@react-native-community/slider`)"
  - react-native-video (from `../node_modules/react-native-video`)
  - react-native-worklets (from `../node_modules/react-native-worklets`)
@@ -516,6 +547,8 @@ EXTERNAL SOURCES:
    :path: "../node_modules/@react-native-camera-roll/camera-roll"
  react-native-safe-area-context:
    :path: "../node_modules/react-native-safe-area-context"
+  react-native-skia:
+    :path: "../node_modules/@shopify/react-native-skia"
  react-native-slider:
    :path: "../node_modules/@react-native-community/slider"
  react-native-video:
@@ -589,6 +622,7 @@ SPEC CHECKSUMS:
  react-native-blur: 50c9feabacbc5f49b61337ebc32192c6be7ec3c3
  react-native-cameraroll: 5b25d0be40185d02e522bf2abf8a1ba4e8faa107
  react-native-safe-area-context: 39c2d8be3328df5d437ac1700f4f3a4f75716acc
+  react-native-skia: 51f4a6586c362814f677df4ac4226f13c634bcfa
  react-native-slider: 33b8d190b59d4f67a541061bb91775d53d617d9d
  react-native-video: c26780b224543c62d5e1b2a7244a5cd1b50e8253
  react-native-worklets: c7576ad4ad0f030ff41e8d74ad0077c96054a6c1
@@ -610,7 +644,7 @@ SPEC CHECKSUMS:
  RNScreens: 218801c16a2782546d30bd2026bb625c0302d70f
  RNStaticSafeAreaInsets: 055ddbf5e476321720457cdaeec0ff2ba40ec1b8
  RNVectorIcons: fcc2f6cb32f5735b586e66d14103a74ce6ad61f8
-  VisionCamera: 312151eb95370d1d764720de3b7dad33d8c7fb40
+  VisionCamera: 0d154cd0ab9043a3c8a4908fb57ad65c9e1f3baf
  Yoga: 5ed1699acbba8863755998a4245daa200ff3817b

 PODFILE CHECKSUM: d53724fe402c2547f1dd1cc571bbe77d9820e636
--- a/example/package.json
+++ b/example/package.json
@@ -19,6 +19,7 @@
    "@react-native-community/slider": "^4.4.2",
    "@react-navigation/native": "^6.1.3",
    "@react-navigation/native-stack": "^6.9.9",
+    "@shopify/react-native-skia": "^0.1.175",
    "react": "^18.2.0",
    "react-native": "^0.71.3",
    "react-native-gesture-handler": "^2.9.0",
@@ -29,7 +30,7 @@
    "react-native-static-safe-area-insets": "^2.2.0",
    "react-native-vector-icons": "^9.2.0",
    "react-native-video": "^5.2.1",
-    "react-native-worklets": "https://github.com/chrfalch/react-native-worklets#15d52dd"
+    "react-native-worklets": "https://github.com/chrfalch/react-native-worklets#d62d76c"
  },
  "devDependencies": {
    "@babel/core": "^7.20.12",
--- a/example/src/CameraPage.tsx
+++ b/example/src/CameraPage.tsx
@@ -1,6 +1,6 @@
 import * as React from 'react';
 import { useRef, useState, useMemo, useCallback } from 'react';
-import { StyleSheet, Text, View } from 'react-native';
+import { Platform, StyleSheet, Text, View } from 'react-native';
 import { PinchGestureHandler, PinchGestureHandlerGestureEvent, TapGestureHandler } from 'react-native-gesture-handler';
 import {
  CameraDeviceFormat,
@@ -25,6 +25,8 @@ import { examplePlugin } from './frame-processors/ExamplePlugin';
 import type { Routes } from './Routes';
 import type { NativeStackScreenProps } from '@react-navigation/native-stack';
 import { useIsFocused } from '@react-navigation/core';
+import { Skia } from '@shopify/react-native-skia';
+import { FACE_SHADER } from './Shaders';

 const ReanimatedCamera = Reanimated.createAnimatedComponent(Camera);
 Reanimated.addWhitelistedNativeProps({
@@ -196,11 +198,37 @@ export function CameraPage({ navigation }: Props): React.ReactElement {
    console.log('re-rendering camera page without active camera');
  }

-  const frameProcessor = useFrameProcessor((frame) => {
-    'worklet';
-    const values = examplePlugin(frame);
-    console.log(`Return Values: ${JSON.stringify(values)}`);
-  }, []);
+  const radius = (format?.videoHeight ?? 1080) * 0.1;
+  const width = radius;
+  const height = radius;
+  const x = (format?.videoHeight ?? 1080) / 2 - radius / 2;
+  const y = (format?.videoWidth ?? 1920) / 2 - radius / 2;
+  const centerX = x + width / 2;
+  const centerY = y + height / 2;
+
+  const runtimeEffect = Skia.RuntimeEffect.Make(FACE_SHADER);
+  if (runtimeEffect == null) throw new Error('Shader failed to compile!');
+  const shaderBuilder = Skia.RuntimeShaderBuilder(runtimeEffect);
+  shaderBuilder.setUniform('r', [width]);
+  shaderBuilder.setUniform('x', [centerX]);
+  shaderBuilder.setUniform('y', [centerY]);
+  shaderBuilder.setUniform('resolution', [1920, 1080]);
+  const imageFilter = Skia.ImageFilter.MakeRuntimeShader(shaderBuilder, null, null);
+
+  const paint = Skia.Paint();
+  paint.setImageFilter(imageFilter);
+
+  const isIOS = Platform.OS === 'ios';
+  const frameProcessor = useFrameProcessor(
+    (frame) => {
+      'worklet';
+      console.log(`Width: ${frame.width}`);
+
+      if (isIOS) frame.render(paint);
+      else console.log('Drawing to the Frame is not yet available on Android. WIP PR');
+    },
+    [isIOS, paint],
+  );

  return (
    <View style={styles.container}>
@@ -224,6 +252,8 @@ export function CameraPage({ navigation }: Props): React.ReactElement {
                photo={true}
                video={true}
                audio={hasMicrophonePermission}
+                enableFpsGraph={true}
+                previewType="skia"
                frameProcessor={device.supportsParallelVideoProcessing ? frameProcessor : undefined}
                orientation="portrait"
              />
--- a/example/src/Shaders.ts
+++ b/example/src/Shaders.ts
@@ -0,0 +1,89 @@
+export const INVERTED_COLORS_SHADER = `
+uniform shader image;
+
+half4 main(vec2 pos) {
+  vec4 color = image.eval(pos);
+  return vec4(1.0 - color.rgb, 1.0);
+}
+`;
+
+export const CHROMATIC_ABERRATION_SHADER = `
+uniform shader image;
+
+vec4 chromatic(vec2 pos, float offset) {
+  float r = image.eval(pos).r;
+  float g = image.eval(vec2(pos.x + offset, pos.y)).g;
+  float b = image.eval(vec2(pos.x + offset * 2.0, pos.y)).b;
+  return vec4(r, g, b, 1.0);
+}
+
+half4 main(vec2 pos) {
+  float offset = 50.0;
+  return chromatic(pos, offset);
+}
+`;
+
+export const NO_SHADER = `
+half4 main(vec2 pos) {
+  return vec4(1.0);
+}
+`;
+
+export const BLUR_SHADER = `
+const int samples = 35,
+          LOD = 2,         // gaussian done on MIPmap at scale LOD
+          sLOD = 1 << LOD; // tile size = 2^LOD
+const float sigma = float(samples) * .25;
+
+float gaussian(vec2 i) {
+    return exp( -.5* dot(i/=sigma,i) ) / ( 6.28 * sigma*sigma );
+}
+
+vec4 blur(sampler2D sp, vec2 U, vec2 scale) {
+    vec4 O = vec4(0);
+    int s = samples/sLOD;
+
+    for ( int i = 0; i < s*s; i++ ) {
+        vec2 d = vec2(i%s, i/s)*float(sLOD) - float(samples)/2.;
+        O += gaussian(d) * textureLod( sp, U + scale * d , float(LOD) );
+    }
+
+    return O / O.a;
+}
+
+void mainImage(out vec4 O, vec2 U) {
+    O = blur( iChannel0, U/iResolution.xy, 1./iChannelResolution[0].xy );
+}
+`;
+
+export const FACE_SHADER = `
+uniform shader image;
+uniform float x;
+uniform float y;
+uniform float r;
+uniform vec2 resolution;
+
+const float samples = 3.0;
+const float radius = 40.0;
+const float weight = 1.0;
+
+half4 main(vec2 pos) {
+  float delta = pow((pow(pos.x - x, 2) + pow(pos.y - y, 2)), 0.5);
+  if (delta < r) {
+    vec3 sum = vec3(0.0);
+    vec3 accumulation = vec3(0);
+    vec3 weightedsum = vec3(0);
+    for (float deltaX = -samples * radius; deltaX <= samples * radius; deltaX += radius / samples) {
+      for (float deltaY = -samples * radius; deltaY <= samples * radius; deltaY += radius / samples) {
+        accumulation += image.eval(vec2(pos.x + deltaX, pos.y + deltaY)).rgb;
+        weightedsum += weight;
+      }
+    }
+    sum = accumulation / weightedsum;
+    return vec4(sum, 1.0);
+  }
+  else {
+    return image.eval(pos);
+  }
+}
+`;
--- a/example/yarn.lock
+++ b/example/yarn.lock
@@ -1183,6 +1183,20 @@
  dependencies:
    nanoid "^3.1.23"

+"@shopify/react-native-skia@^0.1.175":
+  version "0.1.175"
+  resolved "https://registry.yarnpkg.com/@shopify/react-native-skia/-/react-native-skia-0.1.175.tgz#4fc6b30f7d47d3dc9192791021d99e5d11f75739"
+  integrity sha512-vA5YPGu7GmBi5qliLyMzbpkH9mmCWAZoaoGhM9/g5o9zX8xAmUYcGgg3MOqxtnxCnfTmqFFBj43s+QGgMRTpqg==
+  dependencies:
+    "@types/pixelmatch" "^5.2.4"
+    "@types/pngjs" "^6.0.1"
+    "@types/ws" "^8.5.3"
+    canvaskit-wasm "0.38.0"
+    pixelmatch "^5.3.0"
+    pngjs "^6.0.0"
+    react-reconciler "^0.27.0"
+    ws "^8.11.0"
+
 "@sideway/address@^4.1.3":
  version "4.1.4"
  resolved "https://registry.yarnpkg.com/@sideway/address/-/address-4.1.4.tgz#03dccebc6ea47fdc226f7d3d1ad512955d4783f0"
@@ -1253,6 +1267,20 @@
  resolved "https://registry.yarnpkg.com/@types/node/-/node-18.13.0.tgz#0400d1e6ce87e9d3032c19eb6c58205b0d3f7850"
  integrity sha512-gC3TazRzGoOnoKAhUx+Q0t8S9Tzs74z7m0ipwGpSqQrleP14hKxP4/JUeEQcD3W1/aIpnWl8pHowI7WokuZpXg==

+"@types/pixelmatch@^5.2.4":
+  version "5.2.4"
+  resolved "https://registry.yarnpkg.com/@types/pixelmatch/-/pixelmatch-5.2.4.tgz#ca145cc5ede1388c71c68edf2d1f5190e5ddd0f6"
+  integrity sha512-HDaSHIAv9kwpMN7zlmwfTv6gax0PiporJOipcrGsVNF3Ba+kryOZc0Pio5pn6NhisgWr7TaajlPEKTbTAypIBQ==
+  dependencies:
+    "@types/node" "*"
+
+"@types/pngjs@^6.0.1":
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/@types/pngjs/-/pngjs-6.0.1.tgz#c711ec3fbbf077fed274ecccaf85dd4673130072"
+  integrity sha512-J39njbdW1U/6YyVXvC9+1iflZghP8jgRf2ndYghdJb5xL49LYDB+1EuAxfbuJ2IBbWIL3AjHPQhgaTxT3YaYeg==
+  dependencies:
+    "@types/node" "*"
+
 "@types/prop-types@*":
  version "15.7.5"
  resolved "https://registry.yarnpkg.com/@types/prop-types/-/prop-types-15.7.5.tgz#5f19d2b85a98e9558036f6a3cacc8819420f05cf"
@@ -1312,6 +1340,13 @@
  resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.1.tgz#20f18294f797f2209b5f65c8e3b5c8e8261d127c"
  integrity sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==

+"@types/ws@^8.5.3":
+  version "8.5.4"
+  resolved "https://registry.yarnpkg.com/@types/ws/-/ws-8.5.4.tgz#bb10e36116d6e570dd943735f86c933c1587b8a5"
+  integrity sha512-zdQDHKUgcX/zBc4GrwsE/7dVdAD8JR4EuiAXiiUhhfyIJXXb2+PrGshFyeXWQPMmmZ2XxgaqclgpIC7eTXc1mg==
+  dependencies:
+    "@types/node" "*"
+
 "@types/yargs-parser@*":
  version "21.0.0"
  resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-21.0.0.tgz#0c60e537fa790f5f9472ed2776c2b71ec117351b"
@@ -1874,6 +1909,11 @@ caniuse-lite@^1.0.30001449:
  resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001452.tgz#dff7b8bb834b3a91808f0a9ff0453abb1fbba02a"
  integrity sha512-Lkp0vFjMkBB3GTpLR8zk4NwW5EdRdnitwYJHDOOKIU85x4ckYCPQ+9WlVvSVClHxVReefkUMtWZH2l9KGlD51w==

+canvaskit-wasm@0.38.0:
+  version "0.38.0"
+  resolved "https://registry.yarnpkg.com/canvaskit-wasm/-/canvaskit-wasm-0.38.0.tgz#83e6c46f3015c2ff3f6503157f47453af76a7be7"
+  integrity sha512-ZEG6lucpbQ4Ld+mY8C1Ng+PMLVP+/AX02jS0Sdl28NyMxuKSa9uKB8oGd1BYp1XWPyO2Jgr7U8pdyjJ/F3xR5Q==
+
 chalk@^2.0.0:
  version "2.4.2"
  resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424"
@@ -4866,6 +4906,13 @@ pirates@^4.0.5:
  resolved "https://registry.yarnpkg.com/pirates/-/pirates-4.0.5.tgz#feec352ea5c3268fb23a37c702ab1699f35a5f3b"
  integrity sha512-8V9+HQPupnaXMA23c5hvl69zXvTwTzyAYasnkb0Tts4XvO4CliqONMOnvlq26rkhLC3nWDFBJf73LU1e1VZLaQ==

+pixelmatch@^5.3.0:
+  version "5.3.0"
+  resolved "https://registry.yarnpkg.com/pixelmatch/-/pixelmatch-5.3.0.tgz#5e5321a7abedfb7962d60dbf345deda87cb9560a"
+  integrity sha512-o8mkY4E/+LNUf6LzX96ht6k6CEDi65k9G2rjMtBe9Oo+VPKSvl+0GKHuH/AlG+GA5LPG/i5hrekkxUc3s2HU+Q==
+  dependencies:
+    pngjs "^6.0.0"
+
 pkg-dir@^3.0.0:
  version "3.0.0"
  resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-3.0.0.tgz#2749020f239ed990881b1f71210d51eb6523bea3"
@@ -4880,6 +4927,11 @@ pkg-up@^3.1.0:
  dependencies:
    find-up "^3.0.0"

+pngjs@^6.0.0:
+  version "6.0.0"
+  resolved "https://registry.yarnpkg.com/pngjs/-/pngjs-6.0.0.tgz#ca9e5d2aa48db0228a52c419c3308e87720da821"
+  integrity sha512-TRzzuFRRmEoSW/p1KVAmiOgPco2Irlah+bGFCeNfJXxxYGwSw7YwAOAcd7X28K/m5bjBWKsC29KyoMfHbypayg==
+
 posix-character-classes@^0.1.0:
  version "0.1.1"
  resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab"
@@ -5090,9 +5142,9 @@ react-native-video@^5.2.1:
    prop-types "^15.7.2"
    shaka-player "^2.5.9"

-"react-native-worklets@https://github.com/chrfalch/react-native-worklets#15d52dd":
+"react-native-worklets@https://github.com/chrfalch/react-native-worklets#d62d76c":
  version "0.1.0"
-  resolved "https://github.com/chrfalch/react-native-worklets#15d52dd1289831cecc7906823f613172e0c6cd2e"
+  resolved "https://github.com/chrfalch/react-native-worklets#d62d76c20ed7a3bbfebe5623bc976e5c2d9beabd"

 react-native@^0.71.3:
  version "0.71.3"
@@ -5134,6 +5186,14 @@ react-native@^0.71.3:
    whatwg-fetch "^3.0.0"
    ws "^6.2.2"

+react-reconciler@^0.27.0:
+  version "0.27.0"
+  resolved "https://registry.yarnpkg.com/react-reconciler/-/react-reconciler-0.27.0.tgz#360124fdf2d76447c7491ee5f0e04503ed9acf5b"
+  integrity sha512-HmMDKciQjYmBRGuuhIaKA1ba/7a+UsM5FzOZsMO2JYHt9Jh8reCb7j1eDC95NOyUlKM9KRyvdx0flBuDvYSBoA==
+  dependencies:
+    loose-envify "^1.1.0"
+    scheduler "^0.21.0"
+
 react-refresh@^0.4.0:
  version "0.4.3"
  resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.4.3.tgz#966f1750c191672e76e16c2efa569150cc73ab53"
@@ -5377,6 +5437,13 @@ safe-regex@^1.1.0:
  dependencies:
    ret "~0.1.10"

+scheduler@^0.21.0:
+  version "0.21.0"
+  resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.21.0.tgz#6fd2532ff5a6d877b6edb12f00d8ab7e8f308820"
+  integrity sha512-1r87x5fz9MXqswA2ERLo0EbOAU74DpIUO090gIasYTqlVoJeMcl+Z1Rg7WHz+qtPujhS/hGIt9kxZOYBV3faRQ==
+  dependencies:
+    loose-envify "^1.1.0"
+
 scheduler@^0.23.0:
  version "0.23.0"
  resolved "https://registry.yarnpkg.com/scheduler/-/scheduler-0.23.0.tgz#ba8041afc3d30eb206a487b6b384002e4e61fdfe"
@@ -6184,6 +6251,11 @@ ws@^7, ws@^7.5.1:
  resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591"
  integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q==

+ws@^8.11.0:
+  version "8.12.1"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-8.12.1.tgz#c51e583d79140b5e42e39be48c934131942d4a8f"
+  integrity sha512-1qo+M9Ba+xNhPB+YTWUlK6M17brTut5EXbcBaMRN5pH5dFrXz7lzz1ChFSUq3bOUl8yEvSenhHmYUNJxFzdJew==
+
 xtend@~4.0.1:
  version "4.0.2"
  resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"