diff --git a/ios/Skia Render Layer/SkImageHelpers.h b/ios/Skia Render Layer/SkImageHelpers.h index bb95c19..7eb38a1 100644 --- a/ios/Skia Render Layer/SkImageHelpers.h +++ b/ios/Skia Render Layer/SkImageHelpers.h @@ -11,6 +11,8 @@ #import #import +#import + #import #import "SkImage.h" @@ -26,6 +28,10 @@ public: Convert a CMSampleBuffer to an SkImage. Format has to be RGB. */ static sk_sp convertCMSampleBufferToSkImage(GrRecordingContext* context, CMSampleBufferRef sampleBuffer); + /** + Convert a MTLTexture to an SkImage. Format has to be RGB. + */ + static sk_sp convertMTLTextureToSkImage(GrRecordingContext* context, id mtlTexture); /** Creates a Center Crop Transformation Rect so that the source rect fills (aspectRatio: cover) the destination rect. The return value should be passed as a sourceRect to a canvas->draw...Rect(..) function, destinationRect should stay the same. diff --git a/ios/Skia Render Layer/SkImageHelpers.mm b/ios/Skia Render Layer/SkImageHelpers.mm index 94e041b..65ad4c0 100644 --- a/ios/Skia Render Layer/SkImageHelpers.mm +++ b/ios/Skia Render Layer/SkImageHelpers.mm @@ -86,6 +86,20 @@ sk_sp SkImageHelpers::convertCMSampleBufferToSkImage(GrRecordingContext return image; } +sk_sp SkImageHelpers::convertMTLTextureToSkImage(GrRecordingContext* context, id texture) { + // Convert the rendered MTLTexture to an SkImage + GrMtlTextureInfo textureInfo; + textureInfo.fTexture.retain((__bridge void*)texture); + GrBackendTexture backendTexture(texture.width, texture.height, GrMipmapped::kNo, textureInfo); + auto image = SkImages::AdoptTextureFrom(context, + backendTexture, + kTopLeft_GrSurfaceOrigin, + kBGRA_8888_SkColorType, + kOpaque_SkAlphaType, + SkColorSpace::MakeSRGB()); + return image; +} + SkRect SkImageHelpers::createCenterCropRect(SkRect sourceRect, SkRect destinationRect) { SkSize src; if (destinationRect.width() / destinationRect.height() > sourceRect.width() / sourceRect.height()) { diff --git a/ios/Skia Render Layer/SkiaMetalCanvasProvider.mm b/ios/Skia Render Layer/SkiaMetalCanvasProvider.mm index 517025b..e96329a 100644 --- a/ios/Skia Render Layer/SkiaMetalCanvasProvider.mm +++ b/ios/Skia Render Layer/SkiaMetalCanvasProvider.mm @@ -23,7 +23,7 @@ SkiaMetalCanvasProvider::SkiaMetalCanvasProvider(): std::enable_shared_from_this _layerContext.layer.pixelFormat = MTLPixelFormatBGRA8Unorm; // Set up DisplayLink _layerContext.displayLink = [[VisionDisplayLink alloc] init]; - + _isValid = true; } @@ -65,16 +65,16 @@ void SkiaMetalCanvasProvider::render() { if (_width == -1 && _height == -1) { return; } - + if (!_hasNewFrame) { // No new Frame has arrived in the meantime. // We don't need to re-draw the texture to the screen if nothing has changed, abort. return; } - + @autoreleasepool { auto context = _layerContext.skiaContext.get(); - + // Create a Skia Surface from the CAMetalLayer (use to draw to the View) GrMTLHandle drawableHandle; auto surface = SkSurface::MakeFromCAMetalLayer(context, @@ -88,17 +88,17 @@ void SkiaMetalCanvasProvider::render() { if (surface == nullptr || surface->getCanvas() == nullptr) { throw std::runtime_error("Skia surface could not be created from parameters."); } - + auto canvas = surface->getCanvas(); - + // Lock the Mutex so we can operate on the Texture atomically without // renderFrameToCanvas() overwriting in between from a different thread std::unique_lock lock(_textureMutex); - + // Get the texture auto texture = _offscreenContext.texture; if (texture == nil) return; - + // Calculate Center Crop (aspectRatio: cover) transform auto sourceRect = SkRect::MakeXYWH(0, 0, texture.width, texture.height); auto destinationRect = SkRect::MakeXYWH(0, 0, surface->width(), surface->height()); @@ -109,40 +109,32 @@ void SkiaMetalCanvasProvider::render() { // The Canvas is equal to the View size, where-as the Frame has a different size (e.g. 4k) // We scale the Canvas to the exact dimensions of the Frame so that the user can use the Frame as a coordinate system canvas->save(); - + auto scaleW = static_cast(surface->width()) / texture.width; auto scaleH = static_cast(surface->height()) / texture.height; auto scale = MAX(scaleW, scaleH); canvas->scale(scale, scale); canvas->translate(offsetX, offsetY); - + // Convert the rendered MTLTexture to an SkImage - GrMtlTextureInfo textureInfo; - textureInfo.fTexture.retain((__bridge void*)texture); - GrBackendTexture backendTexture(texture.width, texture.height, GrMipmapped::kNo, textureInfo); - auto image = SkImage::MakeFromTexture(context, - backendTexture, - kTopLeft_GrSurfaceOrigin, - kBGRA_8888_SkColorType, - kOpaque_SkAlphaType, - SkColorSpace::MakeSRGB()); - + auto image = SkImageHelpers::convertMTLTextureToSkImage(context, texture); + // Draw the Texture (Frame) to the Canvas canvas->drawImage(image, 0, 0); - + // Restore the scale & transform canvas->restore(); - + surface->flushAndSubmit(); - + // Pass the drawable into the Metal Command Buffer and submit it to the GPU id drawable = (__bridge id)drawableHandle; id commandBuffer([_layerContext.commandQueue commandBuffer]); [commandBuffer presentDrawable:drawable]; [commandBuffer commit]; - + _hasNewFrame = false; - + lock.unlock(); } } @@ -170,14 +162,14 @@ void SkiaMetalCanvasProvider::renderFrameToCanvas(CMSampleBufferRef sampleBuffer if (pixelBuffer == nil) { throw std::runtime_error("drawFrame: Pixel Buffer is corrupt/empty."); } - + // Lock Mutex to block the runLoop from overwriting the _currentDrawable std::unique_lock lock(_textureMutex); - + // Get the Metal Texture we use for in-memory drawing auto texture = getTexture(CVPixelBufferGetWidth(pixelBuffer), CVPixelBufferGetHeight(pixelBuffer)); - + // Get & Lock the writeable Texture from the Metal Drawable GrMtlTextureInfo fbInfo; fbInfo.fTexture.retain((__bridge void*)texture); @@ -185,9 +177,9 @@ void SkiaMetalCanvasProvider::renderFrameToCanvas(CMSampleBufferRef sampleBuffer texture.height, 1, fbInfo); - + auto context = _offscreenContext.skiaContext.get(); - + // Create a Skia Surface from the writable Texture auto surface = SkSurface::MakeFromBackendRenderTarget(context, backendRT, @@ -195,35 +187,35 @@ void SkiaMetalCanvasProvider::renderFrameToCanvas(CMSampleBufferRef sampleBuffer kBGRA_8888_SkColorType, nullptr, nullptr); - + if (surface == nullptr || surface->getCanvas() == nullptr) { throw std::runtime_error("Skia surface could not be created from parameters."); } - + // Lock the Frame's PixelBuffer for the duration of the Frame Processor so the user can safely do operations on it CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); - + // Converts the CMSampleBuffer to an SkImage - RGB. auto image = SkImageHelpers::convertCMSampleBufferToSkImage(context, sampleBuffer); - + auto canvas = surface->getCanvas(); - + // Clear everything so we keep it at a clean state canvas->clear(SkColors::kBlack); - + // Draw the Image into the Frame (aspectRatio: cover) // The Frame Processor might draw the Frame again (through render()) to pass a custom paint/shader, // but that'll just overwrite the existing one - no need to worry. canvas->drawImage(image, 0, 0); - + // Call the JS Frame Processor. drawCallback(canvas); - + // Flush all appended operations on the canvas and commit it to the SkSurface surface->flushAndSubmit(); - + _hasNewFrame = true; - + lock.unlock(); CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); }