railbird/react-native-vision-camera
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Draw onto Frame as if it was a Skia Canvas (#1479)

Browse Source 
* Create Shaders.ts

* Add `previewType` and `enableFpsGraph`

* Add RN Skia native dependency

* Add Skia Preview View on iOS

* Pass 1

* Update FrameHostObject.mm

* Wrap Canvas

* Lockfiles

* fix: Fix stuff

* chore: Upgrade RNWorklets

* Add `previewType` to set the Preview

* feat: Add Example

* Update project.pbxproj

* `enableFpsGraph`

* Cache the `std::shared_ptr<FrameHostObject>`

* Update CameraView+RecordVideo.swift

* Update SkiaMetalCanvasProvider.mm

* Android: Integrate Skia Dependency

* fix: Use new Prefix

* Add example for rendering shader

* chore: Upgrade CameraX

* Remove KTX

* Enable `viewBinding`

* Revert "Enable `viewBinding`"

This reverts commit f2a603f53b33ea4311a296422ffd1a910ce03f9e.

* Revert "chore: Upgrade CameraX"

This reverts commit 8dc832cf8754490d31a6192e6c1a1f11cdcd94fe.

* Remove unneeded `ProcessCameraProvider.getInstance()` call

* fix: Add REA hotfix patch

* fix: Fix FrameHostObject dead in runAsync

* fix: Make `runAsync` run truly async by dropping new Frames while executing

* chore: Upgrade RN Worklets to latest

* chore: Upgrade RN Skia

* Revert "Remove KTX"

This reverts commit 253f586633f7af2da992d2279fc206dc62597129.

* Make Skia optional in CMake

* Fix import

* Update CMakeLists.txt

* Update build.gradle

* Update CameraView.kt

* Update CameraView.kt

* Update CameraView.kt

* Update Shaders.ts

* Center Blur

* chore: Upgrade RN Worklets

* feat: Add `toByteArray()`, `orientation`, `isMirrored` and `timestamp` to `Frame` (#1487)

* feat: Implement `orientation` and `isMirrored` on Frame

* feat: Add `toArrayBuffer()` func

* perf: Do faster buffer copy

* feat: Implement `toArrayBuffer()` on Android

* feat: Add `orientation` and `isMirrored` to Android

* feat: Add `timestamp` to Frame

* Update Frame.ts

* Update JImageProxy.h

* Update FrameHostObject.cpp

* Update FrameHostObject.cpp

* Update CameraPage.tsx

* fix: Format Swift
This commit is contained in:
Marc Rousavy
2023-02-21 15:00:48 +01:00
committed by GitHub
parent 1f7a2e07f2
commit 12f850c8e1
49 changed files with 2166 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
example/src/CameraPage.tsx
View File

@@ -1,6 +1,6 @@
import * as React from 'react';
import { useRef, useState, useMemo, useCallback } from 'react';
import { StyleSheet, Text, View } from 'react-native';
import { Platform, StyleSheet, Text, View } from 'react-native';
import { PinchGestureHandler, PinchGestureHandlerGestureEvent, TapGestureHandler } from 'react-native-gesture-handler';
import {
CameraDeviceFormat,
@@ -25,6 +25,8 @@ import { examplePlugin } from './frame-processors/ExamplePlugin';
import type { Routes } from './Routes';
import type { NativeStackScreenProps } from '@react-navigation/native-stack';
import { useIsFocused } from '@react-navigation/core';
import { Skia } from '@shopify/react-native-skia';
import { FACE_SHADER } from './Shaders';
const ReanimatedCamera = Reanimated.createAnimatedComponent(Camera);
Reanimated.addWhitelistedNativeProps({
@@ -196,11 +198,37 @@ export function CameraPage({ navigation }: Props): React.ReactElement {
console.log('re-rendering camera page without active camera');
}
const frameProcessor = useFrameProcessor((frame) => {
'worklet';
const values = examplePlugin(frame);
console.log(`Return Values: ${JSON.stringify(values)}`);
}, []);
const radius = (format?.videoHeight ?? 1080) * 0.1;
const width = radius;
const height = radius;
const x = (format?.videoHeight ?? 1080) / 2 - radius / 2;
const y = (format?.videoWidth ?? 1920) / 2 - radius / 2;
const centerX = x + width / 2;
const centerY = y + height / 2;
const runtimeEffect = Skia.RuntimeEffect.Make(FACE_SHADER);
if (runtimeEffect == null) throw new Error('Shader failed to compile!');
const shaderBuilder = Skia.RuntimeShaderBuilder(runtimeEffect);
shaderBuilder.setUniform('r', [width]);
shaderBuilder.setUniform('x', [centerX]);
shaderBuilder.setUniform('y', [centerY]);
shaderBuilder.setUniform('resolution', [1920, 1080]);
const imageFilter = Skia.ImageFilter.MakeRuntimeShader(shaderBuilder, null, null);
const paint = Skia.Paint();
paint.setImageFilter(imageFilter);
const isIOS = Platform.OS === 'ios';
const frameProcessor = useFrameProcessor(
(frame) => {
'worklet';
console.log(`Width: ${frame.width}`);
if (isIOS) frame.render(paint);
else console.log('Drawing to the Frame is not yet available on Android. WIP PR');
},
[isIOS, paint],
);
return (
<View style={styles.container}>
@@ -224,6 +252,8 @@ export function CameraPage({ navigation }: Props): React.ReactElement {
photo={true}
video={true}
audio={hasMicrophonePermission}
enableFpsGraph={true}
previewType="skia"
frameProcessor={device.supportsParallelVideoProcessing ? frameProcessor : undefined}
orientation="portrait"
/>

89
example/src/Shaders.ts Normal file
View File

@@ -0,0 +1,89 @@
export const INVERTED_COLORS_SHADER = `
uniform shader image;
half4 main(vec2 pos) {
vec4 color = image.eval(pos);
return vec4(1.0 - color.rgb, 1.0);
}
`;
export const CHROMATIC_ABERRATION_SHADER = `
uniform shader image;
vec4 chromatic(vec2 pos, float offset) {
float r = image.eval(pos).r;
float g = image.eval(vec2(pos.x + offset, pos.y)).g;
float b = image.eval(vec2(pos.x + offset * 2.0, pos.y)).b;
return vec4(r, g, b, 1.0);
}
half4 main(vec2 pos) {
float offset = 50.0;
return chromatic(pos, offset);
}
`;
export const NO_SHADER = `
half4 main(vec2 pos) {
return vec4(1.0);
}
`;
export const BLUR_SHADER = `
const int samples = 35,
LOD = 2, // gaussian done on MIPmap at scale LOD
sLOD = 1 << LOD; // tile size = 2^LOD
const float sigma = float(samples) * .25;
float gaussian(vec2 i) {
return exp( -.5* dot(i/=sigma,i) ) / ( 6.28 * sigma*sigma );
}
vec4 blur(sampler2D sp, vec2 U, vec2 scale) {
vec4 O = vec4(0);
int s = samples/sLOD;
for ( int i = 0; i < s*s; i++ ) {
vec2 d = vec2(i%s, i/s)*float(sLOD) - float(samples)/2.;
O += gaussian(d) * textureLod( sp, U + scale * d , float(LOD) );
}
return O / O.a;
}
void mainImage(out vec4 O, vec2 U) {
O = blur( iChannel0, U/iResolution.xy, 1./iChannelResolution[0].xy );
}
`;
export const FACE_SHADER = `
uniform shader image;
uniform float x;
uniform float y;
uniform float r;
uniform vec2 resolution;
const float samples = 3.0;
const float radius = 40.0;
const float weight = 1.0;
half4 main(vec2 pos) {
float delta = pow((pow(pos.x - x, 2) + pow(pos.y - y, 2)), 0.5);
if (delta < r) {
vec3 sum = vec3(0.0);
vec3 accumulation = vec3(0);
vec3 weightedsum = vec3(0);
for (float deltaX = -samples * radius; deltaX <= samples * radius; deltaX += radius / samples) {
for (float deltaY = -samples * radius; deltaY <= samples * radius; deltaY += radius / samples) {
accumulation += image.eval(vec2(pos.x + deltaX, pos.y + deltaY)).rgb;
weightedsum += weight;
}
}
sum = accumulation / weightedsum;
return vec4(sum, 1.0);
}
else {
return image.eval(pos);
}
}
`;