feat: Rewrite Android C++ part (VisionCameraProxy + JFrame) (#1661)

* First Android rewrite

* Rewrite Android C++ backend

* Pass `ReadableNativeMap`, fix build error

* fix: Fix FrameProcessor init

* Make a bunch of stuff const reference to avoid copies

* Indents

* Cleanup

* indents

* docs: Update Android docs

* Update CameraView.kt

* fix: Format C++ code

example/android/app/src/main/java/com/mrousavy/camera/example/ExampleFrameProcessorPlugin.java

@@ -1,19 +1,30 @@
 package com.mrousavy.camera.example;
 
 import android.util.Log;
+
 import androidx.camera.core.ImageProxy;
+
+import com.facebook.react.bridge.ReadableNativeMap;
 import com.facebook.react.bridge.WritableNativeArray;
 import com.facebook.react.bridge.WritableNativeMap;
+import com.mrousavy.camera.frameprocessor.Frame;
 import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.HashMap;
 
 public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin {
     @Override
-    public Object callback(@NotNull ImageProxy image, @NotNull Object[] params) {
-        Log.d("ExamplePlugin", image.getWidth() + " x " + image.getHeight() + " Image with format #" + image.getFormat() + ". Logging " + params.length + " parameters:");
+    public Object callback(@NotNull Frame frame, @Nullable ReadableNativeMap params) {
+        HashMap<String, Object> hashMap = params != null ? params.toHashMap() : new HashMap<>();
+        ImageProxy image = frame.getImageProxy();
 
-        for (Object param : params) {
-            Log.d("ExamplePlugin", "  -> " + (param == null ? "(null)" : param.toString() + " (" + param.getClass().getName() + ")"));
+        Log.d("ExamplePlugin", image.getWidth() + " x " + image.getHeight() + " Image with format #" + image.getFormat() + ". Logging " + hashMap.size() + " parameters:");
+
+        for (String key : hashMap.keySet()) {
+            Object value = hashMap.get(key);
+            Log.d("ExamplePlugin", "  -> " + (value == null ? "(null)" : value.toString() + " (" + value.getClass().getName() + ")"));
         }
 
         WritableNativeMap map = new WritableNativeMap();
@@ -31,6 +42,6 @@ public class ExampleFrameProcessorPlugin extends FrameProcessorPlugin {
     }
 
     ExampleFrameProcessorPlugin() {
-        super("example_plugin");
+
     }
 }

example/android/app/src/main/java/com/mrousavy/camera/example/MainApplication.java

@@ -1,10 +1,14 @@
 package com.mrousavy.camera.example;
 
 import android.app.Application;
+
+import androidx.annotation.Nullable;
+
 import com.facebook.react.PackageList;
 import com.facebook.react.ReactApplication;
 import com.facebook.react.ReactNativeHost;
 import com.facebook.react.ReactPackage;
+import com.facebook.react.bridge.ReadableNativeMap;
 import com.facebook.soloader.SoLoader;
 import java.util.List;
 import com.facebook.react.defaults.DefaultNewArchitectureEntryPoint;
@@ -12,6 +16,7 @@ import com.facebook.react.defaults.DefaultReactNativeHost;
 
 import com.mrousavy.camera.CameraPackage;
 import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin;
+import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry;
 
 public class MainApplication extends Application implements ReactApplication {
 
@@ -61,6 +66,6 @@ public class MainApplication extends Application implements ReactApplication {
       DefaultNewArchitectureEntryPoint.load();
     }
 
-    FrameProcessorPlugin.register(new ExampleFrameProcessorPlugin());
+    FrameProcessorPluginRegistry.addFrameProcessorPlugin("example_plugin", options -> new ExampleFrameProcessorPlugin());
   }
 }

example/src/CameraPage.tsx

@@ -1,6 +1,6 @@
 import * as React from 'react';
 import { useRef, useState, useMemo, useCallback } from 'react';
-import { Platform, StyleSheet, Text, View } from 'react-native';
+import { StyleSheet, Text, View } from 'react-native';
 import { PinchGestureHandler, PinchGestureHandlerGestureEvent, TapGestureHandler } from 'react-native-gesture-handler';
 import {
   CameraDeviceFormat,
@@ -8,7 +8,7 @@ import {
   PhotoFile,
   sortFormats,
   useCameraDevices,
-  useSkiaFrameProcessor,
+  useFrameProcessor,
   VideoFile,
 } from 'react-native-vision-camera';
 import { Camera, frameRateIncluded } from 'react-native-vision-camera';
@@ -26,6 +26,7 @@ import type { NativeStackScreenProps } from '@react-navigation/native-stack';
 import { useIsFocused } from '@react-navigation/core';
 import { Skia } from '@shopify/react-native-skia';
 import { FACE_SHADER } from './Shaders';
+import { examplePlugin } from './frame-processors/ExamplePlugin';
 
 const ReanimatedCamera = Reanimated.createAnimatedComponent(Camera);
 Reanimated.addWhitelistedNativeProps({
@@ -217,16 +218,13 @@ export function CameraPage({ navigation }: Props): React.ReactElement {
   const paint = Skia.Paint();
   paint.setImageFilter(imageFilter);
 
-  const isIOS = Platform.OS === 'ios';
-  const frameProcessor = useSkiaFrameProcessor(
-    (frame) => {
-      'worklet';
-      console.log(`Width: ${frame.width}`);
+  const frameProcessor = useFrameProcessor((frame) => {
+    'worklet';
 
-      if (frame.isDrawable) frame.render(paint);
-    },
-    [isIOS, paint],
-  );
+    console.log(`Width: ${frame.width}`);
+    const result = examplePlugin(frame);
+    console.log('Example Plugin: ', result);
+  }, []);
 
   return (
     <View style={styles.container}>
@@ -247,12 +245,10 @@ export function CameraPage({ navigation }: Props): React.ReactElement {
                 onError={onError}
                 enableZoomGesture={false}
                 animatedProps={cameraAnimatedProps}
-                photo={true}
-                video={true}
                 audio={hasMicrophonePermission}
                 enableFpsGraph={true}
                 orientation="portrait"
-                frameProcessor={device.supportsParallelVideoProcessing ? frameProcessor : undefined}
+                frameProcessor={frameProcessor}
               />
             </TapGestureHandler>
           </Reanimated.View>