swift - 对象跟踪在 iOS 11 中使用视觉框架

Question

我想检测物体并使用视觉框架跟踪该物体。我成功地完成了检测对象的工作，也成功地完成了一些跟踪工作，但我的跟踪工作并没有那么准确。

我希望在转换帧时具有更高的准确性，因为它在跟踪对象时经常会丢失准确性。

请检查以下代码以检测和跟踪对象:

import UIKit 
import AVFoundation
import Vision

class ViewController: UIViewController {

private lazy var captureSession: AVCaptureSession = {
   let session = AVCaptureSession()
   session.sessionPreset = AVCaptureSession.Preset.photo
   guard let backCamera = AVCaptureDevice.default(for: .video),
       let input = try? AVCaptureDeviceInput(device: backCamera) else 
           {
            return session
           }
    session.addInput(input)
    return session
 }()

private lazy var cameraLayer: AVCaptureVideoPreviewLayer = 
AVCaptureVideoPreviewLayer(session: self.captureSession)

private let handler = VNSequenceRequestHandler()
fileprivate var lastObservation: VNDetectedObjectObservation?

lazy var highlightView: UIView = {
    let view = UIView()
    view.layer.borderColor = UIColor.red.cgColor
    view.layer.borderWidth = 4
    view.backgroundColor = .clear
    return view
}()

override func viewDidLoad() {
   super.viewDidLoad()

    view.layer.addSublayer(cameraLayer)
   view.addSubview(highlightView)

   let output = AVCaptureVideoDataOutput()
   output.setSampleBufferDelegate(self, queue: DispatchQueue(label: 
"queue"))
   captureSession.addOutput(output)

   captureSession.startRunning()

   let tapGestureRecognizer = UITapGestureRecognizer(target: self, 
action: #selector(tapAction))
    view.addGestureRecognizer(tapGestureRecognizer)
}

override func viewDidLayoutSubviews() {
    super.viewDidLayoutSubviews()
    cameraLayer.frame = view.bounds
}

   // MARK: - Actions

  @objc private func tapAction(recognizer: UITapGestureRecognizer) {
    highlightView.frame.size = CGSize(width: 120, height: 120)
    highlightView.center = recognizer.location(in: view)

    let originalRect = highlightView.frame
    var convertedRect = 
    cameraLayer.metadataOutputRectConverted(fromLayerRect: 
    originalRect)
    convertedRect.origin.y = 1 - convertedRect.origin.y

     lastObservation = VNDetectedObjectObservation(boundingBox: 
     convertedRect)
}

fileprivate func handle(_ request: VNRequest, error: Error?) {
    DispatchQueue.main.async {
        guard let newObservation = request.results?.first as? 
VNDetectedObjectObservation else {
           return
        }
        self.lastObservation = newObservation
        var transformedRect = newObservation.boundingBox
        transformedRect.origin.y = 1 - transformedRect.origin.y
        let convertedRect = 
        self.cameraLayer.layerRectConverted(fromMetadataOutputRect: 
       transformedRect)
        self.highlightView.frame = convertedRect
    }
}
}

 extension ViewController: 
 AVCaptureVideoDataOutputSampleBufferDelegate {

func captureOutput(_ output: AVCaptureOutput, didOutput sampleBuffer: 
   CMSampleBuffer, from connection: AVCaptureConnection) {
        guard let pixelBuffer = 
    CMSampleBufferGetImageBuffer(sampleBuffer),
            let observation = lastObservation else {
                return
        } 
        let request = VNTrackObjectRequest(detectedObjectObservation: 
    observation) { [unowned self] request, error in
            self.handle(request, error: error)
       }
       request.trackingLevel = .accurate
        do {
            try handler.perform([request], on: pixelBuffer)
        }
        catch {
            print(error)
        }
       }
       }

任何帮助将不胜感激!!谢谢。

Answer 1

我不太擅长 vision 和 core ml，但显然你的代码看起来不错。您可以做的一件事是检查视觉何时没有在缓冲区中获得任何跟踪，如果跟踪请求置信度值降至 0，则必须将其属性标记为 isLastFrame true。

  if !trackingRequest.isLastFrame {
      if observation.confidence > 0.7 {
          trackingRequest.inputObservation = observation
      } else {
                trackingRequest.isLastFrame = true
            }
          newTrackingRequests.append(trackingRequest)

   }

这样可以很容易地找出视觉跟踪请求是否丢失了跟踪对象或者它只是跟踪了错误的对象。