feat: Draw onto Frame as if it was a Skia Canvas (#1479)

* Create Shaders.ts

* Add `previewType` and `enableFpsGraph`

* Add RN Skia native dependency

* Add Skia Preview View on iOS

* Pass 1

* Update FrameHostObject.mm

* Wrap Canvas

* Lockfiles

* fix: Fix stuff

* chore: Upgrade RNWorklets

* Add `previewType` to set the Preview

* feat: Add Example

* Update project.pbxproj

* `enableFpsGraph`

* Cache the `std::shared_ptr<FrameHostObject>`

* Update CameraView+RecordVideo.swift

* Update SkiaMetalCanvasProvider.mm

* Android: Integrate Skia Dependency

* fix: Use new Prefix

* Add example for rendering shader

* chore: Upgrade CameraX

* Remove KTX

* Enable `viewBinding`

* Revert "Enable `viewBinding`"

This reverts commit f2a603f53b33ea4311a296422ffd1a910ce03f9e.

* Revert "chore: Upgrade CameraX"

This reverts commit 8dc832cf8754490d31a6192e6c1a1f11cdcd94fe.

* Remove unneeded `ProcessCameraProvider.getInstance()` call

* fix: Add REA hotfix patch

* fix: Fix FrameHostObject dead in runAsync

* fix: Make `runAsync` run truly async by dropping new Frames while executing

* chore: Upgrade RN Worklets to latest

* chore: Upgrade RN Skia

* Revert "Remove KTX"

This reverts commit 253f586633f7af2da992d2279fc206dc62597129.

* Make Skia optional in CMake

* Fix import

* Update CMakeLists.txt

* Update build.gradle

* Update CameraView.kt

* Update CameraView.kt

* Update CameraView.kt

* Update Shaders.ts

* Center Blur

* chore: Upgrade RN Worklets

* feat: Add `toByteArray()`, `orientation`, `isMirrored` and `timestamp` to `Frame` (#1487)

* feat: Implement `orientation` and `isMirrored` on Frame

* feat: Add `toArrayBuffer()` func

* perf: Do faster buffer copy

* feat: Implement `toArrayBuffer()` on Android

* feat: Add `orientation` and `isMirrored` to Android

* feat: Add `timestamp` to Frame

* Update Frame.ts

* Update JImageProxy.h

* Update FrameHostObject.cpp

* Update FrameHostObject.cpp

* Update CameraPage.tsx

* fix: Format Swift
This commit is contained in:
Marc Rousavy
2023-02-21 15:00:48 +01:00
committed by GitHub
parent 1f7a2e07f2
commit 12f850c8e1
49 changed files with 2166 additions and 85 deletions

View File

@@ -123,8 +123,6 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
internal var activeVideoRecording: Recording? = null
private var lastFrameProcessorCall = System.currentTimeMillis()
private var extensionsManager: ExtensionsManager? = null
private val scaleGestureListener: ScaleGestureDetector.SimpleOnScaleGestureListener
@@ -326,7 +324,7 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
/**
* Configures the camera capture session. This should only be called when the camera device changes.
*/
@SuppressLint("RestrictedApi")
@SuppressLint("RestrictedApi", "UnsafeOptInUsageError")
private suspend fun configureSession() {
try {
val startTime = System.currentTimeMillis()
@@ -461,10 +459,11 @@ class CameraView(context: Context, private val frameProcessorThread: ExecutorSer
if (enableFrameProcessor) {
Log.i(TAG, "Adding ImageAnalysis use-case...")
imageAnalysis = imageAnalysisBuilder.build().apply {
setAnalyzer(cameraExecutor, { image ->
setAnalyzer(cameraExecutor) { image ->
// Call JS Frame Processor
frameProcessorCallback(image)
})
// frame gets closed in FrameHostObject implementation (JS ref counting)
}
}
useCases.add(imageAnalysis!!)
}

View File

@@ -172,7 +172,6 @@ class CameraViewModule(reactContext: ReactApplicationContext) : ReactContextBase
withPromise(promise) {
val cameraProvider = ProcessCameraProvider.getInstance(reactApplicationContext).await()
val extensionsManager = ExtensionsManager.getInstanceAsync(reactApplicationContext, cameraProvider).await()
ProcessCameraProvider.getInstance(reactApplicationContext).await()
val manager = reactApplicationContext.getSystemService(Context.CAMERA_SERVICE) as? CameraManager
?: throw CameraManagerUnavailableError()

View File

@@ -1,12 +1,16 @@
package com.mrousavy.camera.frameprocessor;
import android.annotation.SuppressLint;
import android.graphics.ImageFormat;
import android.graphics.Matrix;
import android.media.Image;
import androidx.annotation.Keep;
import androidx.camera.core.ImageProxy;
import com.facebook.proguard.annotations.DoNotStrip;
import java.nio.ByteBuffer;
@SuppressWarnings("unused") // used through JNI
@DoNotStrip
@Keep
@@ -28,6 +32,33 @@ public class ImageProxyUtils {
}
}
@DoNotStrip
@Keep
public static boolean isImageProxyMirrored(ImageProxy imageProxy) {
Matrix matrix = imageProxy.getImageInfo().getSensorToBufferTransformMatrix();
// TODO: Figure out how to get isMirrored from ImageProxy
return false;
}
@DoNotStrip
@Keep
public static String getOrientation(ImageProxy imageProxy) {
int rotation = imageProxy.getImageInfo().getRotationDegrees();
if (rotation >= 45 && rotation < 135)
return "landscapeRight";
if (rotation >= 135 && rotation < 225)
return "portraitUpsideDown";
if (rotation >= 225 && rotation < 315)
return "landscapeLeft";
return "portrait";
}
@DoNotStrip
@Keep
public static long getTimestamp(ImageProxy imageProxy) {
return imageProxy.getImageInfo().getTimestamp();
}
@DoNotStrip
@Keep
public static int getPlanesCount(ImageProxy imageProxy) {
@@ -39,4 +70,29 @@ public class ImageProxyUtils {
public static int getBytesPerRow(ImageProxy imageProxy) {
return imageProxy.getPlanes()[0].getRowStride();
}
private static byte[] byteArrayCache;
@DoNotStrip
@Keep
public static byte[] toByteArray(ImageProxy imageProxy) {
switch (imageProxy.getFormat()) {
case ImageFormat.YUV_420_888:
ByteBuffer yBuffer = imageProxy.getPlanes()[0].getBuffer();
ByteBuffer vuBuffer = imageProxy.getPlanes()[2].getBuffer();
int ySize = yBuffer.remaining();
int vuSize = vuBuffer.remaining();
if (byteArrayCache == null || byteArrayCache.length != ySize + vuSize) {
byteArrayCache = new byte[ySize + vuSize];
}
yBuffer.get(byteArrayCache, 0, ySize);
vuBuffer.get(byteArrayCache, ySize, vuSize);
return byteArrayCache;
default:
throw new RuntimeException("Cannot convert Frame with Format " + imageProxy.getFormat() + " to byte array!");
}
}
}