// // SkiaRenderer.mm // VisionCamera // // Created by Marc Rousavy on 19.07.23. // Copyright © 2023 mrousavy. All rights reserved. // #import #import "SkiaRenderer.h" #import #import #import "SkiaRenderContext.h" #import #import #import #import #import #import #import #import "SkImageHelpers.h" #import #import #import @implementation SkiaRenderer { // The context we draw each Frame on std::unique_ptr _offscreenContext; // The context the preview runs on std::unique_ptr _layerContext; // The texture holding the drawn-to Frame id _texture; // For synchronization between the two Threads/Contexts std::mutex _textureMutex; std::atomic _hasNewFrame; } - (instancetype)init { if (self = [super init]) { _offscreenContext = std::make_unique(); _layerContext = std::make_unique(); _texture = nil; _hasNewFrame = false; } return self; } - (id)metalDevice { return _layerContext->device; } - (id)getTexture:(NSUInteger)width height:(NSUInteger)height { if (_texture == nil || _texture.width != width || _texture.height != height) { // Create new texture with the given width and height MTLTextureDescriptor* textureDescriptor = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm width:width height:height mipmapped:NO]; textureDescriptor.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead; _texture = [_offscreenContext->device newTextureWithDescriptor:textureDescriptor]; } return _texture; } - (void)renderCameraFrameToOffscreenCanvas:(CMSampleBufferRef)sampleBuffer withDrawCallback:(draw_callback_t)callback { // Wrap in auto release pool since we want the system to clean up after rendering @autoreleasepool { // Get the Frame's PixelBuffer CVImageBufferRef pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); if (pixelBuffer == nil) { throw std::runtime_error("SkiaRenderer: Pixel Buffer is corrupt/empty."); } // Lock Mutex to block the runLoop from overwriting the _currentDrawable std::unique_lock lock(_textureMutex); // Get the Metal Texture we use for in-memory drawing auto texture = [self getTexture:CVPixelBufferGetWidth(pixelBuffer) height:CVPixelBufferGetHeight(pixelBuffer)]; // Get & Lock the writeable Texture from the Metal Drawable GrMtlTextureInfo textureInfo; textureInfo.fTexture.retain((__bridge void*)texture); GrBackendRenderTarget backendRenderTarget((int)texture.width, (int)texture.height, 1, textureInfo); auto context = _offscreenContext->skiaContext.get(); // Create a Skia Surface from the writable Texture auto surface = SkSurfaces::WrapBackendRenderTarget(context, backendRenderTarget, kTopLeft_GrSurfaceOrigin, kBGRA_8888_SkColorType, SkColorSpace::MakeSRGB(), nullptr); if (surface == nullptr || surface->getCanvas() == nullptr) { throw std::runtime_error("Skia surface could not be created from parameters."); } // Converts the CMSampleBuffer to an SkImage - RGB. CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); auto image = SkImageHelpers::convertCMSampleBufferToSkImage(context, sampleBuffer); auto canvas = surface->getCanvas(); // Clear everything so we keep it at a clean state canvas->clear(SkColors::kBlack); // Draw the Image into the Frame (aspectRatio: cover) // The Frame Processor might draw the Frame again (through render()) to pass a custom paint/shader, // but that'll just overwrite the existing one - no need to worry. canvas->drawImage(image, 0, 0); // Call the draw callback - probably a JS Frame Processor. callback(static_cast(canvas)); // Flush all appended operations on the canvas and commit it to the SkSurface surface->flushAndSubmit(); // Set dirty & free locks _hasNewFrame = true; lock.unlock(); CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); } } - (void)renderLatestFrameToLayer:(CALayer* _Nonnull)layer { if (!_hasNewFrame) { // No new Frame has arrived in the meantime. // We don't need to re-draw the texture to the screen if nothing has changed, abort. return; } @autoreleasepool { auto context = _layerContext->skiaContext.get(); // Create a Skia Surface from the CAMetalLayer (use to draw to the View) GrMTLHandle drawableHandle; auto surface = SkSurfaces::WrapCAMetalLayer(context, (__bridge GrMTLHandle)layer, kTopLeft_GrSurfaceOrigin, 1, kBGRA_8888_SkColorType, nullptr, nullptr, &drawableHandle); if (surface == nullptr || surface->getCanvas() == nullptr) { throw std::runtime_error("Skia surface could not be created from parameters."); } auto canvas = surface->getCanvas(); // Lock the Mutex so we can operate on the Texture atomically without // renderFrameToCanvas() overwriting in between from a different thread std::unique_lock lock(_textureMutex); auto texture = _texture; if (texture == nil) return; // Calculate Center Crop (aspectRatio: cover) transform auto sourceRect = SkRect::MakeXYWH(0, 0, texture.width, texture.height); auto destinationRect = SkRect::MakeXYWH(0, 0, surface->width(), surface->height()); sourceRect = SkImageHelpers::createCenterCropRect(sourceRect, destinationRect); auto offsetX = -sourceRect.left(); auto offsetY = -sourceRect.top(); // The Canvas is equal to the View size, where-as the Frame has a different size (e.g. 4k) // We scale the Canvas to the exact dimensions of the Frame so that the user can use the Frame as a coordinate system canvas->save(); auto scaleW = static_cast(surface->width()) / texture.width; auto scaleH = static_cast(surface->height()) / texture.height; auto scale = MAX(scaleW, scaleH); canvas->scale(scale, scale); canvas->translate(offsetX, offsetY); // Convert the rendered MTLTexture to an SkImage auto image = SkImageHelpers::convertMTLTextureToSkImage(context, texture); // Draw the Texture (Frame) to the Canvas canvas->drawImage(image, 0, 0); // Restore the scale & transform canvas->restore(); surface->flushAndSubmit(); // Pass the drawable into the Metal Command Buffer and submit it to the GPU id drawable = (__bridge id)drawableHandle; id commandBuffer([_layerContext->commandQueue commandBuffer]); [commandBuffer presentDrawable:drawable]; [commandBuffer commit]; // Set flag back to false _hasNewFrame = false; lock.unlock(); } } @end