c++ - 从 Direct3D11CaptureFrame 转换为 OpenCV 的 cv::Mat

Question

我正在尝试从 Direct3D11CaptureFrame 进行转换，该帧从 Microsoft's Screen Capture documentation 的示例中获取(并通过将其保存在磁盘上验证其正确性)到 cv::Mat。我通过以下方式将框架的表面转换为 ID3D11Texture2D，它似乎有效:

void processFrame(const winrt::Windows::Graphics::Capture::Direct3D11CaptureFrame &frame) {
       winrt::com_ptr<Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess> access =
          frame.Surface().as<Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess>();
      winrt::com_ptr<ID3D11Texture2D> frameSurface;
      winrt::check_hresult(access->GetInterface(winrt::guid_of<ID3D11Texture2D>(), frameSurface.put_void()));
      (...)
}

从这点来看opencv's d3d interop samples ，我看到我有两个选择。第一个包括使用 cv::directx::convertFromD3D11Texture2D 例如

void processFrame(const winrt::Windows::Graphics::Capture::Direct3D11CaptureFrame &frame) {
       winrt::com_ptr<Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess> access =
          frame.Surface().as<Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess>();
      winrt::com_ptr<ID3D11Texture2D> frameSurface;
      winrt::check_hresult(access->GetInterface(winrt::guid_of<ID3D11Texture2D>(), frameSurface.put_void()));

  cv::Mat img;
  // throws exception in the following line
  //
  cv::directx::convertFromD3D11Texture2D(frameSurface.get(), img);
}

但它在调用输入数组的析构函数(frameSurface.get())后抛出异常。此方法在代码示例中用于 GPU 工作，但我没有在 the documentation 中看到它被声明为专用于 GPU。 .

第二个选项要经过几个步骤:

void processFrame(const winrt::Windows::Graphics::Capture::Direct3D11CaptureFrame &frame, int height, int width) {
  auto access =
      frame.Surface().as<Windows::Graphics::DirectX::Direct3D11::IDirect3DDxgiInterfaceAccess>();
  winrt::com_ptr<ID3D11Texture2D> frameSurface;
  winrt::check_hresult(
      access->GetInterface(winrt::guid_of<ID3D11Texture2D>(), frameSurface.put_void()));

  ID3D11Device *deviceD3D;
  frameSurface->GetDevice(&deviceD3D);
  ID3D11DeviceContext *m_pD3D11Ctx;
  deviceD3D->GetImmediateContext(&m_pD3D11Ctx);

  auto subResource = ::D3D11CalcSubresource(0, 0, 1);
  D3D11_MAPPED_SUBRESOURCE mappedTex;
  auto r = m_pD3D11Ctx->Map(frameSurface.get(), subResource, D3D11_MAP_WRITE_DISCARD, 0, &mappedTex);
  // FAILS here
  //
  if (FAILED(r)) {
    throw std::runtime_error("surface mapping failed!");
  }
  cv::Mat m(height, width, CV_8UC4, mappedTex.pData, mappedTex.RowPitch);
}

...但是在将上下文映射到子资源时失败。

考虑到我对 DirectX 的经验不足，我一直在调整我能想到的一切，并试图找出我遗漏了什么，但我无法为这两个选项中的任何一个找到解决方案。有人看到我的错误或对我的问题有更好的解决方案吗？

Answer 1

1.你需要新的 D3D11_TEXTURE2D_DESC 并设置:CPUAccessFlags = D3D11_CPU_ACCESS_READ

2.使用 D3D11_TEXTURE2D_DESC 创建一个新的 CreateTexture2D

3.复制frameSurface到你的新CreateTexture2D

void SimpleCapture::OnFrameArrived(
Direct3D11CaptureFramePool const& sender,
winrt::Windows::Foundation::IInspectable const&) {
    auto newSize = false;
    auto frame = sender.TryGetNextFrame();
    auto frameContentSize = frame.ContentSize();

    D3D11_TEXTURE2D_DESC desc;
    desc.MipLevels = 1;
    desc.ArraySize = 1;
    desc.Format = DXGI_FORMAT_B8G8R8A8_UNORM;
    desc.SampleDesc = { 1,0 };
    desc.Usage = D3D11_USAGE_STAGING;
    desc.BindFlags = 0;
    desc.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
    desc.MiscFlags = 0;
    
    if (frameContentSize.Width != m_lastSize.Width ||
        frameContentSize.Height != m_lastSize.Height) {
        newSize = true;
        m_lastSize = frameContentSize;
    }
    desc.Width = m_lastSize.Width;
    desc.Height = m_lastSize.Height;

    ID3D11Texture2D* stagingTexture;
    d3dDevice.get()->CreateTexture2D(&desc, nullptr, &stagingTexture);
    
    auto frameSurface = GetDXGIInterfaceFromObject<ID3D11Texture2D>(frame.Surface());
    m_d3dContext->CopyResource(stagingTexture, frameSurface.get());

    D3D11_MAPPED_SUBRESOURCE mappedTex;
    m_d3dContext->Map(stagingTexture, 0, D3D11_MAP_READ, 0, &mappedTex);
    m_d3dContext->Unmap(stagingTexture, 0);

    cv::Mat c = cv::Mat(desc.Height, desc.Width, CV_8UC4, mappedTex.pData, mappedTex.RowPitch);
    cv::imshow("show", c);

    stagingTexture->Release();


    DXGI_PRESENT_PARAMETERS presentParameters = { 0 };

    if (newSize)
    {
        m_framePool.Recreate(
            m_device,
            DirectXPixelFormat::B8G8R8A8UIntNormalized,
            2,
            m_lastSize);
    }
}