feat: Draw onto Frame as if it was a Skia Canvas (#1479)
* Create Shaders.ts * Add `previewType` and `enableFpsGraph` * Add RN Skia native dependency * Add Skia Preview View on iOS * Pass 1 * Update FrameHostObject.mm * Wrap Canvas * Lockfiles * fix: Fix stuff * chore: Upgrade RNWorklets * Add `previewType` to set the Preview * feat: Add Example * Update project.pbxproj * `enableFpsGraph` * Cache the `std::shared_ptr<FrameHostObject>` * Update CameraView+RecordVideo.swift * Update SkiaMetalCanvasProvider.mm * Android: Integrate Skia Dependency * fix: Use new Prefix * Add example for rendering shader * chore: Upgrade CameraX * Remove KTX * Enable `viewBinding` * Revert "Enable `viewBinding`" This reverts commit f2a603f53b33ea4311a296422ffd1a910ce03f9e. * Revert "chore: Upgrade CameraX" This reverts commit 8dc832cf8754490d31a6192e6c1a1f11cdcd94fe. * Remove unneeded `ProcessCameraProvider.getInstance()` call * fix: Add REA hotfix patch * fix: Fix FrameHostObject dead in runAsync * fix: Make `runAsync` run truly async by dropping new Frames while executing * chore: Upgrade RN Worklets to latest * chore: Upgrade RN Skia * Revert "Remove KTX" This reverts commit 253f586633f7af2da992d2279fc206dc62597129. * Make Skia optional in CMake * Fix import * Update CMakeLists.txt * Update build.gradle * Update CameraView.kt * Update CameraView.kt * Update CameraView.kt * Update Shaders.ts * Center Blur * chore: Upgrade RN Worklets * feat: Add `toByteArray()`, `orientation`, `isMirrored` and `timestamp` to `Frame` (#1487) * feat: Implement `orientation` and `isMirrored` on Frame * feat: Add `toArrayBuffer()` func * perf: Do faster buffer copy * feat: Implement `toArrayBuffer()` on Android * feat: Add `orientation` and `isMirrored` to Android * feat: Add `timestamp` to Frame * Update Frame.ts * Update JImageProxy.h * Update FrameHostObject.cpp * Update FrameHostObject.cpp * Update CameraPage.tsx * fix: Format Swift
This commit is contained in:
@@ -190,8 +190,26 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud
|
||||
}
|
||||
|
||||
public final func captureOutput(_ captureOutput: AVCaptureOutput, didOutput sampleBuffer: CMSampleBuffer, from _: AVCaptureConnection) {
|
||||
// Draw Frame to Preview View Canvas (and call Frame Processor)
|
||||
if captureOutput is AVCaptureVideoDataOutput {
|
||||
if let previewView = previewView as? PreviewSkiaView {
|
||||
// Render to Skia PreviewView
|
||||
previewView.drawFrame(sampleBuffer) { canvas in
|
||||
// Call JS Frame Processor before passing Frame to GPU - allows user to draw
|
||||
guard let frameProcessor = self.frameProcessorCallback else { return }
|
||||
let frame = Frame(buffer: sampleBuffer, orientation: self.bufferOrientation)
|
||||
frameProcessor(frame, canvas)
|
||||
}
|
||||
} else {
|
||||
// Call JS Frame Processor. User cannot draw, since we don't have a Skia Canvas.
|
||||
guard let frameProcessor = frameProcessorCallback else { return }
|
||||
let frame = Frame(buffer: sampleBuffer, orientation: bufferOrientation)
|
||||
frameProcessor(frame, nil)
|
||||
}
|
||||
}
|
||||
|
||||
// Record Video Frame/Audio Sample to File
|
||||
if isRecording {
|
||||
// Write Video / Audio frame to file
|
||||
guard let recordingSession = recordingSession else {
|
||||
invokeOnError(.capture(.unknown(message: "isRecording was true but the RecordingSession was null!")))
|
||||
return
|
||||
@@ -210,14 +228,21 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud
|
||||
}
|
||||
}
|
||||
|
||||
if let frameProcessor = frameProcessorCallback, captureOutput is AVCaptureVideoDataOutput {
|
||||
// Call the JavaScript Frame Processor func (worklet)
|
||||
let frame = Frame(buffer: sampleBuffer, orientation: bufferOrientation)
|
||||
frameProcessor(frame)
|
||||
}
|
||||
#if DEBUG
|
||||
if captureOutput is AVCaptureVideoDataOutput {
|
||||
// Update FPS Graph per Frame
|
||||
if let fpsGraph = fpsGraph {
|
||||
DispatchQueue.main.async {
|
||||
fpsGraph.onTick(CACurrentMediaTime())
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
private func recommendedVideoSettings(videoOutput: AVCaptureVideoDataOutput, fileType: AVFileType, videoCodec: AVVideoCodecType?) -> [String: Any]? {
|
||||
private func recommendedVideoSettings(videoOutput: AVCaptureVideoDataOutput,
|
||||
fileType: AVFileType,
|
||||
videoCodec: AVVideoCodecType?) -> [String: Any]? {
|
||||
if videoCodec != nil {
|
||||
return videoOutput.recommendedVideoSettings(forVideoCodecType: videoCodec!, assetWriterOutputFileType: fileType)
|
||||
} else {
|
||||
@@ -233,7 +258,7 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud
|
||||
return .up
|
||||
}
|
||||
|
||||
switch UIDevice.current.orientation {
|
||||
switch outputOrientation {
|
||||
case .portrait:
|
||||
return cameraPosition == .front ? .leftMirrored : .right
|
||||
|
||||
@@ -246,8 +271,8 @@ extension CameraView: AVCaptureVideoDataOutputSampleBufferDelegate, AVCaptureAud
|
||||
case .landscapeRight:
|
||||
return cameraPosition == .front ? .upMirrored : .down
|
||||
|
||||
case .unknown, .faceUp, .faceDown:
|
||||
fallthrough
|
||||
case .unknown:
|
||||
return .up
|
||||
@unknown default:
|
||||
return .up
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user