swift - 如何使用 CVPixelBuffer/IOSurface 更新 CALayer？

Question

我有一个 IOSurface 支持的 CVPixelBuffer，它以 30fps 从外部源更新。我想在 NSView 中呈现图像数据的预览——对我来说最好的方法是什么？
我可以直接在 View 上设置 CALayer 的 .contents ，但这只会在我的 View 第一次更新时更新(或者说，如果我调整了 View 的大小)。我一直在仔细研究文档，但我无法弄清楚在图层或 View 上正确调用需求显示以让 View 基础结构知道刷新自身，尤其是当更新来自 View 外部时。
理想情况下，我只需将 IOSurface 绑定(bind)到我的层，我对其所做的任何更改都会被传播，但我不确定这是否可能。

class VideoPreviewController: NSViewController, VideoFeedConsumer {
    let customLayer : CALayer = CALayer()
    
    override func viewDidLoad() {
        super.viewDidLoad()
        // Do view setup here.
        print("Loaded our video preview")
        
        view.layer?.addSublayer(customLayer)
        customLayer.frame = view.frame
        
        // register our view with the browser service
        VideoFeedBrowser.instance.registerConsumer(self)
    }
    
    override func viewWillDisappear() {
        // deregister our view from the video feed
        VideoFeedBrowser.instance.deregisterConsumer(self)

        super.viewWillDisappear()
    }
    
    // This callback gets called at 30fps whenever the pixelbuffer is updated
    @objc func updateFrame(pixelBuffer: CVPixelBuffer) {

        guard let surface = CVPixelBufferGetIOSurface(pixelBuffer)?.takeUnretainedValue() else {
            print("pixelbuffer isn't IOsurface backed! noooooo!")
            return;
        }

        // Try and tell the view to redraw itself with new contents?
        // These methods don't work
        //self.view.setNeedsDisplay(self.view.visibleRect)
        //self.customLayer.setNeedsDisplay()
        self.customLayer.contents = surface

    }
    
}

这是我对基于 NSView 而不是基于 NSViewController 的缩放版本的尝试，它也不能正确更新(或就此问题正确缩放):

class VideoPreviewThumbnail: NSView, VideoFeedConsumer {
   

    required init?(coder decoder: NSCoder) {
        super.init(coder: decoder)
        self.wantsLayer = true
        
        // register our view with the browser service
        VideoFeedBrowser.instance.registerConsumer(self)
    }
    
    override init(frame frameRect: NSRect) {
        super.init(frame: frameRect)
        self.wantsLayer = true
        
        // register our view with the browser service
        VideoFeedBrowser.instance.registerConsumer(self)
    }
    
    deinit{
        VideoFeedBrowser.instance.deregisterConsumer(self)
    }
    
    override func updateLayer() {
        // Do I need to put something here?
        print("update layer")
    }
    
    @objc
    func updateFrame(pixelBuffer: CVPixelBuffer) {
        guard let surface = CVPixelBufferGetIOSurface(pixelBuffer)?.takeUnretainedValue() else {
            print("pixelbuffer isn't IOsurface backed! noooooo!")
            return;
        }
        self.layer?.contents = surface
        self.layer?.transform = CATransform3DMakeScale(
            self.frame.width / CGFloat(CVPixelBufferGetWidth(pixelBuffer)),
            self.frame.height / CGFloat(CVPixelBufferGetHeight(pixelBuffer)),
            CGFloat(1))
    }

}

我错过了什么？

Answer 1

也许我错了，但我认为你是在后台线程上更新你的 NSView 。 (我想 updateFrame 的回调在后台线程上)
如果我是对的，当你想更新 NSView 时，转换你的 pixelBuffer到任何你想要的(NSImage？)，然后在主线程上调度它。
伪代码(我不经常使用 CVPixelBuffer，所以我不确定这是转换为 NSImage 的正确方法)

let ciImage = CIImage(cvImageBuffer: pixelBuffer)
let context = CIContext(options: nil)

let width = CVPixelBufferGetWidth(pixelBuffer)
let height = CVPixelBufferGetHeight(pixelBuffer)

let cgImage = context.createCGImage(ciImage, from: CGRect(x: 0, y: 0, width: width, height: height))

let nsImage = NSImage(cgImage: cgImage, size: CGSize(width: width, height: height))

DispatchQueue.main.async {
    // assign the NSImage to your NSView here
}

另一个捕获 : 我做了一些测试，似乎你不能将 IOSurface 直接分配给 CALayer 的内容。
我试过这个:

    let textureImageWidth = 1024
    let textureImageHeight = 1024

    let macPixelFormatString = "ARGB"
    var macPixelFormat: UInt32 = 0
    for c in macPixelFormatString.utf8.reversed() {
       macPixelFormat *= 256
       macPixelFormat += UInt32(c)
    }

    let ioSurface = IOSurfaceCreate([kIOSurfaceWidth: textureImageWidth,
                    kIOSurfaceHeight: textureImageHeight,
                    kIOSurfaceBytesPerElement: 4,
                    kIOSurfaceBytesPerRow: textureImageWidth * 4,
                    kIOSurfaceAllocSize: textureImageWidth * textureImageHeight * 4,
                    kIOSurfacePixelFormat: macPixelFormat] as CFDictionary)!

    IOSurfaceLock(ioSurface, IOSurfaceLockOptions.readOnly, nil)
    let test = CIImage(ioSurface: ioSurface)
    IOSurfaceUnlock(ioSurface, IOSurfaceLockOptions.readOnly, nil)
    
    v1?.layer?.contents = ioSurface

v1 是我的观点。没有效果
即使使用 CIImage 也没有效果(仅最后几行)

    IOSurfaceLock(ioSurface, IOSurfaceLockOptions.readOnly, nil)
    let test = CIImage(ioSurface: ioSurface)
    IOSurfaceUnlock(ioSurface, IOSurfaceLockOptions.readOnly, nil)
    
    v1?.layer?.contents = test

如果我创建一个 CGImage 它可以工作

    IOSurfaceLock(ioSurface, IOSurfaceLockOptions.readOnly, nil)
    let test = CIImage(ioSurface: ioSurface)
    IOSurfaceUnlock(ioSurface, IOSurfaceLockOptions.readOnly, nil)
    
    let context = CIContext.init()
    let img = context.createCGImage(test, from: test.extent)
    v1?.layer?.contents = img