c++ - IMFTransform::ProcessInput() "The buffer was too small to carry out the requested action."

Question

我正在尝试使用 IMFTransform 将纹理编码为 H264。我可以使用 SinkWriter 将纹理写入和编码到文件中，并且可以播放视频和所有内容，效果很好。但我正在尝试学习如何使用 IMFTransform 以便我可以自己访问编码的 IMFSamples。
不幸的是，我并没有走得太远，因为 ProcessInput 失败了 "The buffer was too small to carry out the requested action."作为 HRESULT。
我不知道它指的是哪个“缓冲区”，并且对该错误进行搜索绝对没有结果。除了 ProcessInput() 之外，没有其他调用返回错误的 HRESULT ，并且 SinkWriter 工作正常。所以我完全零线索问题是什么。

#include "main.h"
#include "WinDesktopDup.h"
#include <iostream>
#include <wmcodecdsp.h>

WinDesktopDup dup;

void SetupDpiAwareness()
{
    if (!SetProcessDpiAwarenessContext(DPI_AWARENESS_CONTEXT_SYSTEM_AWARE))
        printf("SetProcessDpiAwarenessContext failed\n");
}

const UINT32 VIDEO_WIDTH = 3840;
const UINT32 VIDEO_HEIGHT = 2160;
const UINT32 VIDEO_FPS = 120;
const UINT64 VIDEO_FRAME_DURATION = 10 * 1000 * 1000 / VIDEO_FPS;
const UINT32 VIDEO_BIT_RATE = 800000;
const GUID   VIDEO_ENCODING_FORMAT = MFVideoFormat_H264;
const GUID   VIDEO_INPUT_FORMAT = MFVideoFormat_ARGB32;
const UINT32 VIDEO_PELS = VIDEO_WIDTH * VIDEO_HEIGHT;
const UINT32 VIDEO_FRAME_COUNT = 20 * VIDEO_FPS;

template <class T>
void SafeRelease(T** ppT) {
    if (*ppT) {
        (*ppT)->Release();
        *ppT = NULL;
    }
}

bool usingEncoder;
IMFMediaType* pMediaTypeOut = NULL;
IMFMediaType* pMediaTypeIn = NULL;
HRESULT SetMediaType()
{
    // Set the output media type.
    HRESULT hr = MFCreateMediaType(&pMediaTypeOut);
    if (!SUCCEEDED(hr)) { printf("MFCreateMediaType failed\n"); }
    hr = pMediaTypeOut->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
    if (!SUCCEEDED(hr)) { printf("SetGUID failed\n"); }
    hr = pMediaTypeOut->SetGUID(MF_MT_SUBTYPE, VIDEO_ENCODING_FORMAT);
    if (!SUCCEEDED(hr)) { printf("SetGUID (2) failed\n"); }
    hr = pMediaTypeOut->SetUINT32(MF_MT_AVG_BITRATE, VIDEO_BIT_RATE);
    if (!SUCCEEDED(hr)) { printf("SetUINT32 (3) failed\n"); }
    hr = pMediaTypeOut->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
    if (!SUCCEEDED(hr)) { printf("SetUINT32 (4) failed\n"); }
    hr = MFSetAttributeSize(pMediaTypeOut, MF_MT_FRAME_SIZE, VIDEO_WIDTH, VIDEO_HEIGHT);
    if (!SUCCEEDED(hr)) { printf("MFSetAttributeSize failed\n"); }
    hr = MFSetAttributeRatio(pMediaTypeOut, MF_MT_FRAME_RATE, VIDEO_FPS, 1);
    if (!SUCCEEDED(hr)) { printf("MFSetAttributeRatio failed\n"); }
    hr = MFSetAttributeRatio(pMediaTypeOut, MF_MT_PIXEL_ASPECT_RATIO, 1, 1);
    if (!SUCCEEDED(hr)) { printf("MFSetAttributeRatio (2) failed\n"); }
    

    // Set the input media type.
    hr = MFCreateMediaType(&pMediaTypeIn);
    if (!SUCCEEDED(hr)) { printf("MFCreateMediaType failed\n"); }
    hr = pMediaTypeIn->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video);
    if (!SUCCEEDED(hr)) { printf("SetGUID (3) failed\n"); }
    hr = pMediaTypeIn->SetGUID(MF_MT_SUBTYPE, VIDEO_INPUT_FORMAT);
    if (!SUCCEEDED(hr)) { printf("SetGUID (4) failed\n"); }
    hr = pMediaTypeIn->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive);
    if (!SUCCEEDED(hr)) { printf("SetUINT32 (5) failed\n"); }
    hr = MFSetAttributeSize(pMediaTypeIn, MF_MT_FRAME_SIZE, VIDEO_WIDTH, VIDEO_HEIGHT);
    if (!SUCCEEDED(hr)) { printf("MFSetAttributeSize (2) failed\n"); }
    hr = MFSetAttributeRatio(pMediaTypeIn, MF_MT_FRAME_RATE, VIDEO_FPS, 1);
    if (!SUCCEEDED(hr)) { printf("MFSetAttributeRatio (3) failed\n"); }
    hr = MFSetAttributeRatio(pMediaTypeIn, MF_MT_PIXEL_ASPECT_RATIO, 1, 1);
    if (!SUCCEEDED(hr)) { printf("MFSetAttributeRatio (4) failed\n"); }
    
    return hr;
}

HRESULT InitializeSinkWriter(IMFSinkWriter** ppWriter, DWORD* pStreamIndex)
{
    IMFDXGIDeviceManager* pDeviceManager = NULL;
    UINT                  resetToken;
    IMFAttributes* attributes;

    *ppWriter = NULL;
    *pStreamIndex = NULL;

    IMFSinkWriter* pSinkWriter = NULL;
    
    DWORD          streamIndex;

    HRESULT hr = MFCreateDXGIDeviceManager(&resetToken, &pDeviceManager);
    if (!SUCCEEDED(hr)) { printf("MFCreateDXGIDeviceManager failed\n"); }
    hr = pDeviceManager->ResetDevice(dup.D3DDevice, resetToken);
    if (!SUCCEEDED(hr)) { printf("ResetDevice failed\n"); }

    hr = MFCreateAttributes(&attributes, 3);
    if (!SUCCEEDED(hr)) { printf("MFCreateAttributes failed\n"); }
    hr = attributes->SetUINT32(MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, 1);
    if (!SUCCEEDED(hr)) { printf("SetUINT32 failed\n"); }
    hr = attributes->SetUINT32(MF_LOW_LATENCY, 1);
    if (!SUCCEEDED(hr)) { printf("SetUINT32 (2) failed\n"); }
    hr = attributes->SetUnknown(MF_SINK_WRITER_D3D_MANAGER, pDeviceManager);
    if (!SUCCEEDED(hr)) { printf("SetUnknown failed\n"); }
    hr = MFCreateSinkWriterFromURL(L"output.mp4", NULL, attributes, &pSinkWriter);
    if (!SUCCEEDED(hr)) { printf("MFCreateSinkWriterFromURL failed\n"); }

    hr = pSinkWriter->AddStream(pMediaTypeOut, &streamIndex);
    if (!SUCCEEDED(hr)) { printf("AddStream failed\n"); }

    hr = pSinkWriter->SetInputMediaType(streamIndex, pMediaTypeIn, NULL);
    if (!SUCCEEDED(hr)) { printf("SetInputMediaType failed\n"); }

    // Tell the sink writer to start accepting data.
    hr = pSinkWriter->BeginWriting();
    if (!SUCCEEDED(hr)) { printf("BeginWriting failed\n"); }

    // Return the pointer to the caller.
    *ppWriter = pSinkWriter;
    (*ppWriter)->AddRef();
    *pStreamIndex = streamIndex;

    SafeRelease(&pSinkWriter);
    SafeRelease(&pMediaTypeOut);
    SafeRelease(&pMediaTypeIn);
    return hr;
}

IUnknown* _transformUnk;
IMFTransform* pMFTransform;
HRESULT InitializeEncoder(DWORD* pStreamIndex)
{
    HRESULT hr = CoCreateInstance(CLSID_CMSH264EncoderMFT, NULL, CLSCTX_INPROC_SERVER, IID_IUnknown, (void**)&_transformUnk);
    if (!SUCCEEDED(hr)) { printf("CoCreateInstance failed\n"); }
    hr = _transformUnk->QueryInterface(IID_PPV_ARGS(&pMFTransform));
    if (!SUCCEEDED(hr)) { printf("QueryInterface failed\n"); }
    
    hr = pMFTransform->SetOutputType(0, pMediaTypeOut, 0);
    if (!SUCCEEDED(hr)) { printf("SetOutputType failed\n"); }

    hr = pMFTransform->SetInputType(0, pMediaTypeIn, 0);
    if (!SUCCEEDED(hr)) { printf("SetInputType failed\n"); }


    DWORD mftStatus = 0;
    hr = pMFTransform->GetInputStatus(0, &mftStatus);
    if (!SUCCEEDED(hr)) { printf("GetInputStatus failed\n"); }

    if (MFT_INPUT_STATUS_ACCEPT_DATA != mftStatus)
        printf("MFT_INPUT_STATUS_ACCEPT_DATA\n");

    hr = pMFTransform->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, NULL);
    if (!SUCCEEDED(hr)) { printf("MFT_MESSAGE_NOTIFY_BEGIN_STREAMING failed\n"); }
    hr = pMFTransform->ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, NULL);
    if (!SUCCEEDED(hr)) { printf("MFT_MESSAGE_NOTIFY_START_OF_STREAM failed\n"); }

    SafeRelease(&pSinkWriter);
    SafeRelease(&pMediaTypeOut);
    SafeRelease(&pMediaTypeIn);
    return hr;
}

ID3D11Texture2D* texture;

HRESULT WriteFrame(IMFSinkWriter* pWriter, DWORD streamIndex, const LONGLONG& rtStart)
{
    IMFSample* pSample = NULL;
    IMFMediaBuffer* pBuffer = NULL;

    HRESULT hr;
    
    hr = MFCreateDXGISurfaceBuffer(__uuidof(ID3D11Texture2D), texture, 0, false, &pBuffer);
    if (!SUCCEEDED(hr)) { printf("MFCreateDXGISurfaceBuffer failed\n"); }

    DWORD len;
    hr = ((IMF2DBuffer*)pBuffer)->GetContiguousLength(&len);
    if (!SUCCEEDED(hr)) { printf("GetContiguousLength failed\n"); }

    hr = pBuffer->SetCurrentLength(len);
    if (!SUCCEEDED(hr)) { printf("SetCurrentLength failed\n"); }

    // Create a media sample and add the buffer to the sample.
    hr = MFCreateSample(&pSample);
    if (!SUCCEEDED(hr)) { printf("MFCreateSample failed\n"); }

    hr = pSample->AddBuffer(pBuffer);
    if (!SUCCEEDED(hr)) { printf("AddBuffer failed\n"); }

    // Set the time stamp and the duration.
    hr = pSample->SetSampleTime(rtStart);
    if (!SUCCEEDED(hr)) { printf("SetSampleTime failed\n"); }

    hr = pSample->SetSampleDuration(VIDEO_FRAME_DURATION);
    if (!SUCCEEDED(hr)) { printf("SetSampleDuration failed\n"); }

    // Send the sample to the Sink Writer or Encoder.

    if (!usingEncoder)
    {
        hr = pWriter->WriteSample(streamIndex, pSample);
        if (!SUCCEEDED(hr)) { printf("WriteSample failed\n"); }
    }
    else
    {
        hr = pMFTransform->ProcessInput(0, pSample, 0);
        if (!SUCCEEDED(hr)) { printf("ProcessInput failed\n"); }
    }
    
    SafeRelease(&pSample);
    SafeRelease(&pBuffer);
    return hr;
}

int APIENTRY main(HINSTANCE hInstance, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nShowCmd)
{
    SetupDpiAwareness();
    auto err = dup.Initialize();

    // Initialize MF
    CoInitializeEx(0, COINIT_APARTMENTTHREADED); // Need to call this once when a thread is using COM or it wont work
    MFStartup(MF_VERSION);                       // Need to call this too for Media Foundation related memes

    IMFSinkWriter* pSinkWriter = NULL;
    DWORD          stream = 0;
    LONGLONG       rtStart = 0;

    usingEncoder = true; // True if we want to encode with IMFTransform, false if we want to write with SinkWriter
    
    HRESULT        hr = SetMediaType();
    if (!SUCCEEDED(hr)) { printf("SetMediaType failed\n"); }

    if (!usingEncoder)
    {
        hr = InitializeSinkWriter(&pSinkWriter, &stream);
        if (!SUCCEEDED(hr)) { printf("InitializeSinkWriter failed\n"); }
    }
    else
    {
        hr = pMediaTypeIn->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_IYUV); // Using MFVideoFormat_ARGB32 causes SetInputType() to fail
        hr = InitializeEncoder(&stream);
        if (!SUCCEEDED(hr)) { printf("InitializeEncoder failed\n"); }
    }
    
    const int CAPTURE_LENGTH = 10;

    int total_frames = VIDEO_FPS * CAPTURE_LENGTH;

    for (int i = 0; i < 1; i++)
    {
        texture = dup.CaptureNext();
        if (texture != nullptr)
        {
            hr = WriteFrame(pSinkWriter, stream, rtStart);
            if (!SUCCEEDED(hr))
                printf("WriteFrame failed\n");
            rtStart += VIDEO_FRAME_DURATION;
            texture->Release();
        }
        else
        {
            i--;
        }
    }

    if (FAILED(hr))
    {
        std::cout << "Failure" << std::endl;
    }

    if (SUCCEEDED(hr)) {
        hr = pSinkWriter->Finalize();
    }

    SafeRelease(&pSinkWriter);
    MFShutdown();
    CoUninitialize();
}

Answer 1

Here’s documentation对于您在代码中使用的 Microsoft 基于 CPU 的软件 h.264 编码器。
不支持 MFVideoFormat_ARGB32在输入上。它根本不支持任何 RGB 格式。该转换仅支持输入视频的 YUV 格式。
顺便说一句，如果您用硬件编码器替换 MFT，它们很可能会公开与 Microsoft 软件相同的一组功能，我认为它们不支持 RGB。而且，因为所有硬件转换都是异步的，所以你需要稍微不同的工作流程来直接驱动它们。
sink writer 工作正常的原因，它在引擎盖下创建并托管了 2 个 MFT，从 RGB 到 YUV 的格式转换器，另一个是编码器。
您有以下选择。

在将帧传递给编码器之前，使用另一个 MFT 将 RGBA 转换为 NV12。

使用像素着色器(使用 2 个不同的像素着色器将纹理四边形渲染到 NV12 纹理的 2 个平面)或使用单个计算着色器(为视频的每个 2x2 块调度 1 个线程，为每个块写入 6 个字节， 4 进入 R8_UNORM 输出纹理与亮度，其他 2 个字节进入 R8G8_UNORM 输出纹理与颜色数据)。

使用接收器编写器，但使用 MFCreateSinkWriterFromMediaSink 创建它API 而不是 MFCreateSinkWriterFromURL .实现 IMFMediaSink COM 接口(interface)，还有 IMFStreamSink对于它的视频流，框架将调用 IMFStreamSink.ProcessSample一旦可用，就会在系统内存中为您提供编码的视频样本。