feat: Rewrite Android C++ part (VisionCameraProxy + JFrame) (#1661)

* First Android rewrite

* Rewrite Android C++ backend

* Pass `ReadableNativeMap`, fix build error

* fix: Fix FrameProcessor init

* Make a bunch of stuff const reference to avoid copies

* Indents

* Cleanup

* indents

* docs: Update Android docs

* Update CameraView.kt

* fix: Format C++ code
This commit is contained in:
Marc Rousavy
2023-07-22 00:15:11 +02:00
committed by GitHub
parent 44ed42d5d6
commit 86dd703c2b
45 changed files with 985 additions and 859 deletions

View File

@@ -131,7 +131,7 @@ const frameProcessor = useFrameProcessor((frame) => {
## What's possible?
You can run any native code you want in a Frame Processor Plugin. Just like in the native iOS and Android Camera APIs, you will receive a frame (`CMSampleBuffer` on iOS, `ImageProxy` on Android) which you can use however you want. In other words; **everything is possible**.
You can run any native code you want in a Frame Processor Plugin. Just like in the native iOS and Android Camera APIs, you will receive a frame ([`CMSampleBuffer`][5] on iOS, [`ImageProxy`][6] on Android) which you can use however you want. In other words; **everything is possible**.
## Implementations
@@ -194,5 +194,7 @@ Your Frame Processor Plugins have to be fast. Use the FPS Graph (`enableFpsGraph
[1]: https://github.com/mrousavy/react-native-vision-camera/blob/main/src/Frame.ts
[2]: https://github.com/mrousavy/react-native-vision-camera/blob/main/ios/Frame%20Processor/Frame.h
[3]: https://developer.android.com/reference/androidx/camera/core/ImageProxy
[3]: https://github.com/mrousavy/react-native-vision-camera/blob/main/android/src/main/java/com/mrousavy/camera/frameprocessor/Frame.java
[4]: https://github.com/facebook/react-native/blob/9a43eac7a32a6ba3164a048960101022a92fcd5a/React/Base/RCTBridgeModule.h#L20-L24
[5]: https://developer.apple.com/documentation/coremedia/cmsamplebuffer
[6]: https://developer.android.com/reference/androidx/camera/core/ImageProxy

View File

@@ -55,21 +55,16 @@ For reference see the [CLI's docs](https://github.com/mateusz1913/vision-camera-
3. Add the following code:
```java {8}
import androidx.camera.core.ImageProxy;
import com.mrousavy.camera.frameprocessor.Frame;
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin;
public class FaceDetectorFrameProcessorPlugin extends FrameProcessorPlugin {
@Override
public Object callback(ImageProxy image, ReadableNativeMap arguments) {
public Object callback(Frame frame, ReadableNativeMap arguments) {
// code goes here
return null;
}
@Override
public String getName() {
return "detectFaces";
}
}
```
@@ -86,13 +81,14 @@ import com.facebook.react.bridge.NativeModule;
import com.facebook.react.bridge.ReactApplicationContext;
import com.facebook.react.uimanager.ViewManager;
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin;
import com.mrousavy.camera.frameprocessor.FrameProcessorPluginRegistry;
import javax.annotation.Nonnull;
public class FaceDetectorFrameProcessorPluginPackage implements ReactPackage {
@NonNull
@Override
public List<NativeModule> createNativeModules(@NonNull ReactApplicationContext reactContext) {
FrameProcessorPlugin.register(new FaceDetectorFrameProcessorPlugin());
FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces", options -> new FaceDetectorFrameProcessorPlugin());
return Collections.emptyList();
}
@@ -125,19 +121,15 @@ public class FaceDetectorFrameProcessorPluginPackage implements ReactPackage {
3. Add the following code:
```kotlin {7}
import androidx.camera.core.ImageProxy
import com.mrousavy.camera.frameprocessor.Frame
import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin
class FaceDetectorFrameProcessorPlugin: FrameProcessorPlugin() {
override fun callback(image: ImageProxy, arguments: ReadableNativeMap): Any? {
override fun callback(frame: Frame, arguments: ReadableNativeMap): Any? {
// code goes here
return null
}
override fun getName(): String {
return "detectFaces"
}
}
```
@@ -157,7 +149,9 @@ import com.mrousavy.camera.frameprocessor.FrameProcessorPlugin
class FaceDetectorFrameProcessorPluginPackage : ReactPackage {
override fun createNativeModules(reactContext: ReactApplicationContext): List<NativeModule> {
FrameProcessorPlugin.register(FaceDetectorFrameProcessorPlugin())
FrameProcessorPluginRegistry.addFrameProcessorPlugin("detectFaces") { options ->
FaceDetectorFrameProcessorPlugin()
}
return emptyList()
}