c# - MF SinkWriter写入样本失败

Question

我正在尝试使用MediaFoundation将ID3D11Texture2D编码为mp4。下面是我当前的代码。

初始化接收器编写器

private int InitializeSinkWriter(String outputFile, int videoWidth, int videoHeight)
    {
        IMFMediaType mediaTypeIn = null;
        IMFMediaType mediaTypeOut = null;
        IMFAttributes attributes = null;

        int hr = 0;

        if (Succeeded(hr)) hr = (int)MFExtern.MFCreateAttributes(out attributes, 1);
        if (Succeeded(hr)) hr = (int)attributes.SetUINT32(MFAttributesClsid.MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, 1);            
        if (Succeeded(hr)) hr = (int)attributes.SetUINT32(MFAttributesClsid.MF_LOW_LATENCY, 1);

        // Create the sink writer 
        if (Succeeded(hr)) hr = (int)MFExtern.MFCreateSinkWriterFromURL(outputFile, null, attributes, out sinkWriter);

        // Create the output type
        if (Succeeded(hr)) hr = (int)MFExtern.MFCreateMediaType(out mediaTypeOut);
        if (Succeeded(hr)) hr = (int)mediaTypeOut.SetGUID(MFAttributesClsid.MF_MT_MAJOR_TYPE, MFMediaType.Video);
        if (Succeeded(hr)) hr = (int)mediaTypeOut.SetGUID(MFAttributesClsid.MF_TRANSCODE_CONTAINERTYPE, MFTranscodeContainerType.MPEG4);
        if (Succeeded(hr)) hr = (int)mediaTypeOut.SetGUID(MFAttributesClsid.MF_MT_SUBTYPE, MFMediaType.H264);
        if (Succeeded(hr)) hr = (int)mediaTypeOut.SetUINT32(MFAttributesClsid.MF_MT_AVG_BITRATE, videoBitRate);
        if (Succeeded(hr)) hr = (int)mediaTypeOut.SetUINT32(MFAttributesClsid.MF_MT_INTERLACE_MODE, (int)MFVideoInterlaceMode.Progressive);            

        if (Succeeded(hr)) hr = (int)MFExtern.MFSetAttributeSize(mediaTypeOut, MFAttributesClsid.MF_MT_FRAME_SIZE, videoWidth, videoHeight);
        if (Succeeded(hr)) hr = (int)MFExtern.MFSetAttributeRatio(mediaTypeOut, MFAttributesClsid.MF_MT_FRAME_RATE, VIDEO_FPS, 1);
        if (Succeeded(hr)) hr = (int)MFExtern.MFSetAttributeRatio(mediaTypeOut, MFAttributesClsid.MF_MT_PIXEL_ASPECT_RATIO, 1, 1);
        if (Succeeded(hr)) hr = (int)sinkWriter.AddStream(mediaTypeOut, out streamIndex);



        // Create the input type 
        if (Succeeded(hr)) hr = (int)MFExtern.MFCreateMediaType(out mediaTypeIn);
        if (Succeeded(hr)) hr = (int)mediaTypeIn.SetGUID(MFAttributesClsid.MF_MT_MAJOR_TYPE, MFMediaType.Video);
        if (Succeeded(hr)) hr = (int)mediaTypeIn.SetGUID(MFAttributesClsid.MF_MT_SUBTYPE, MFMediaType.ARGB32);
        if (Succeeded(hr)) hr = (int)mediaTypeIn.SetUINT32(MFAttributesClsid.MF_SA_D3D11_AWARE, 1);
        if (Succeeded(hr)) hr = (int)mediaTypeIn.SetUINT32(MFAttributesClsid.MF_MT_INTERLACE_MODE, (int)MFVideoInterlaceMode.Progressive);
        if (Succeeded(hr)) hr = (int)MFExtern.MFSetAttributeSize(mediaTypeIn, MFAttributesClsid.MF_MT_FRAME_SIZE, videoWidth, videoHeight);
        if (Succeeded(hr)) hr = (int)MFExtern.MFSetAttributeRatio(mediaTypeIn, MFAttributesClsid.MF_MT_FRAME_RATE, VIDEO_FPS, 1);
        if (Succeeded(hr)) hr = (int)MFExtern.MFSetAttributeRatio(mediaTypeIn, MFAttributesClsid.MF_MT_PIXEL_ASPECT_RATIO, 1, 1);
        if (Succeeded(hr)) hr = (int)sinkWriter.SetInputMediaType(streamIndex, mediaTypeIn, null);


        // Start accepting data
        if (Succeeded(hr)) hr = (int)sinkWriter.BeginWriting();


        COMBase.SafeRelease(mediaTypeOut);
        COMBase.SafeRelease(mediaTypeIn);

        return hr;
    }

书写框架

 int hr = 0;
        IMFSample sample = null;
        IMFMediaBuffer buffer = null;
        IMF2DBuffer p2Dbuffer = null;
        object texNativeObject = Marshal.GetObjectForIUnknown(surface.NativePointer);

        if (Succeeded(hr)) hr = (int)MFExtern.MFCreateDXGISurfaceBuffer(new Guid("6f15aaf2-d208-4e89-9ab4-489535d34f9c"), texNativeObject, 0, false, out p2Dbuffer);

        buffer = MFVideoEncoderST.ReinterpretCast<IMF2DBuffer,IMFMediaBuffer>(p2Dbuffer);
        int length=0;
        if (Succeeded(hr)) hr = (int)p2Dbuffer.GetContiguousLength(out length);
        if (Succeeded(hr)) hr = (int)buffer.SetCurrentLength(length);


        if (Succeeded(hr)) hr = (int)MFExtern.MFCreateVideoSampleFromSurface(null, out sample);

        if (Succeeded(hr)) hr = (int)sample.AddBuffer(buffer);
        if (Succeeded(hr)) hr = (int)sample.SetSampleTime(prevRecordingDuration);
        if (Succeeded(hr)) hr = (int)sample.SetSampleDuration((recordDuration - prevRecordingDuration));

        if (Succeeded(hr)) hr = (int)sinkWriter.WriteSample(streamIndex, sample);


        COMBase.SafeRelease(sample);
        COMBase.SafeRelease(buffer);

使用MFTRACE我得到以下错误。

    02:48:04.99463 CMFSinkWriterDetours::WriteSample @024BEA18 Stream Index 0x0, Sample @17CEACE0, Time 571ms, Duration 16ms, Buffers 1, Size 4196352B,2088,2008 02:48:04.99465 CMFSinkWriterDetours::WriteSample @024BEA18 failed hr=0x887A0005 (null)2088,2008 
02:48:05.01090 CMFSinkWriterDetours::WriteSample @024BEA18 Stream Index 0x0, Sample @17CE9FC0, Time 587ms, Duration 17ms, Buffers 1, Size 4196352B,2088,2008 02:48:05.01091 CMFSinkWriterDetours::WriteSample @024BEA18 failed hr=0x887A0005 (null)2088,2008 
02:48:05.02712 CMFSinkWriterDetours::WriteSample @024BEA18 Stream Index 0x0, Sample @17CEACE0, Time 604ms, Duration 16ms, Buffers 1, Size 4196352B,2088,2008 02:48:05.02713 CMFSinkWriterDetours::WriteSample @024BEA18 failed hr=0x887A0005 (null)

谁能告诉我我的代码有什么问题吗？我只能产生0个字节的mp4文件。

Answer 1

我在这里遇到了一些潜在的问题。罗曼（Roman）提到了这两个大公司，因此我将详细说明。我也有其他一些批评/建议给您。

不使用IMFDXGIDeviceManager

为了在Media Foundation中使用硬件加速，您需要创建DirectX设备管理器对象，对于DX9，可以是IDirect3DDeviceManager9，对于DXGI，则可以是IMFDXGIDeviceManager。我强烈建议阅读该接口的所有MSDN文档。之所以有此必要，是因为同一DX设备必须在所有正在使用的协作硬件MF转换之间共享，因为它们都需要访问该设备控制的共享GPU内存，并且每个DX设备在工作时都需要对该设备的独占控制权，因此需要一个锁定系统。设备管理器对象提供了该锁定系统，也是向一个或多个转换提供DX设备的标准方式。对于DXGI，可使用MFCreateDXGIDeviceManager创建它。

从那里，您需要创建DX11设备，并用DX11设备调用IMFDXGIDeviceManager::ResetDevice。然后，您需要为Sink Writer本身设置设备管理器，而在上面提供的代码中则没有这样做。这样完成：

// ... inside your InitializeSinkWriter function that you listed above

// I'm assuming you've already created and set up the DXGI device manager elsewhere
IMFDXGIDeviceManager pDeviceManager;

// Passing 3 as the argument because we're adding 3 attributes immediately, saves re-allocations
if (Succeeded(hr)) hr = (int)MFExtern.MFCreateAttributes(out attributes, 3);
if (Succeeded(hr)) hr = (int)attributes.SetUINT32(MFAttributesClsid.MF_READWRITE_ENABLE_HARDWARE_TRANSFORMS, 1);            
if (Succeeded(hr)) hr = (int)attributes.SetUINT32(MFAttributesClsid.MF_LOW_LATENCY, 1);

// Here's the key piece!
if (Succeeded(hr)) hr = (int)attributes.SetUnknown(MFAttributesClsid.MF_SINK_WRITER_D3D_MANAGER, pDeviceManager);

// Create the sink writer 
if (Succeeded(hr)) hr = (int)MFExtern.MFCreateSinkWriterFromURL(outputFile, null, attributes, out sinkWriter);

这实际上将启用对硬件编码器的D3D11支持，并允许其读取您传入的 Texture2D。值得注意的是， MF_SINK_WRITER_D3D_MANAGER对于DX9和DXGI设备管理器均适用。


编码器缓冲相同纹理的多个 IMFSample实例

这也是造成您问题的潜在原因-至少，即使这不是造成明显问题的原因，也至少会导致很多意想不到的行为。基于Roman的注释，许多编码器会在其编码过程中缓冲多个帧。使用Sink Writer时，您看不到该行为，因为它可以为您处理所有细节工作。但是，您要完成的工作（即，将D3D11纹理作为输入帧发送）足够低，您不得不担心接收器写入器使用的编码器MFT的内部细节。

大多数视频编码器MFT将使用某个大小的内部缓冲区来存储通过 IMFTransform::ProcessInput提供的最后N个样本。这样做的副作用是，在生成任何输出之前，必须提供多个样本作为输入。视频编码器需要按顺序访问多个样本，因为它们使用后续帧来确定如何对当前帧进行编码。换句话说，如果解码器正在处理帧0，则可能还需要查看帧1、2和3。从技术角度来看，这是由于诸如 inter-frame prediction和运动估计之类的。编码器完成对最旧样本的处理后，将生成一个输出缓冲区（另一个 IMFSample对象，但这一次是通过 IMFTransform::ProcessOutput在输出端），然后丢弃它正在处理的输入样本（通过调用 IUnknown::Release） ，然后请求更多输入，并最终移至下一帧。您可以在MSDN文章 Processing Data in the Encoder中阅读有关此过程的更多信息。

正如罗马人所暗示的，这意味着您将 ID3D11Texture2D封装在 IMFMediaBuffer内的 IMFSample内，然后将其传递给接收器编写器。该样本可能会在编码过程中被编码器缓冲。当编码器工作时，该 Texture2D的内容可能正在更改，这可能会导致各种问题。即使那没有引起程序错误，也肯定会导致非常奇怪的编码视频输出。想象一下，如果编码器试图预测下一帧的视觉内容在下一帧中如何变化，然后从编码器下方更新出两帧的实际视觉内容！

发生此特定问题的原因是，编码器仅具有指向您的 IMFSample实例的指针引用，而该实例最终只是指向您的 ID3D11Texture2D对象的指针，而该对象是对可变图形内存的一种指针引用。最终，由于程序的其他部分，该图形内存的内容会发生变化，但是由于始终更新相同的GPU纹理，因此将编码器指向的每个样本都指向相同的单个纹理。这意味着每当您通过更改GPU内存来更新纹理时，所有活动的 IMFSample对象都将反映这些更改，因为它们都有效地指向同一GPU纹理。

要解决此问题，您需要分配多个 ID3D11Texture2D对象，以便在将其提供给Sink Writer时可以将一个纹理与一个 IMFSample配对。通过使每个样本指向唯一的纹理，这将解决所有样本指向相同的单个GPU纹理的问题。但是，您不一定知道需要创建多少个纹理，因此处理此问题的最安全方法是编写自己的纹理分配器。仍然可以在C＃中完成它的价值，MediaFoundation.NET定义了您需要使用的接口。

分配器应维护“空闲” SharpDX.Texture2D对象的列表-接收器编写器/编码器当前未使用的对象。您的程序应该能够从分配器中请求新的纹理对象，在这种情况下，它将从空闲列表中返回一个对象，或者创建一个新的纹理来适应该请求。

下一个问题是知道编码器何时丢弃了 IMFSample对象，以便可以将附加的纹理添加回空闲列表中。碰巧的是，您当前使用的 MFCreateVideoSampleFromSurface函数会分配实现 IMFTrackedSample接口的样本。您将需要该接口，以便在释放示例时收到通知，以便可以回收 Texture2D对象。

诀窍是您必须告诉样本您是分配器。首先，您的分配器类需要实现 IMFAsyncCallback。如果通过 IMFTrackedSample::SetAllocator在样本上设置分配器类，则只要编码器释放样本，就会调用分配器的 IMFAsyncCallback::Invoke方法，并以 IMFAsyncResult作为参数传递。这是分配器类的外观的一般示例。

sealed class TextureAllocator : IMFAsyncCallback, IDisposable
{
    private ConcurrentStack<SharpDX.Direct3D11.Texture2D> m_freeStack;
    private static readonly Guid s_IID_ID3D11Texture2D = new Guid("6f15aaf2-d208-4e89-9ab4-489535d34f9c");

    // If all textures are the exact same size and color format,
    // consider making those parameters private class members and
    // requiring they be specified as arguments to the constructor.
    public TextureAllocator()
    {
        m_freeStack = new ConcurrentStack<SharpDX.Direct3D11.Texture2D>();
    }

    private bool disposedValue = false;
    private void Dispose(bool disposing)
    {
        if(!disposedValue)
        {
            if(disposing)
            {
                // Dispose managed resources here
            }

            if(m_freeStack != null)
            {
                SharpDX.Direct3D11.Texture2D texture;
                while(m_freeStack.TryPop(out texture))
                {
                    texture.Dispose();
                }
                m_freeStack = null;
            }

            disposedValue = true;
        }
    }

    public void Dispose()
    {
        Dispose(true);
        GC.SuppressFinalize(this);
    }

    ~TextureAllocator()
    {
        Dispose(false);
    }

    private SharpDX.Direct3D11.Texture2D InternalAllocateNewTexture()
    {
        // Allocate a new texture with your format, size, etc here.
    }

    public SharpDX.Direct3D11.Texture2D AllocateTexture()
    {
        SharpDX.Direct3D11.Texture2D existingTexture;
        if(m_freeStack.TryPop(out existingTexture))
        {
            return existingTexture;
        }
        else
        {
            return InternalAllocateNewTexture();
        }
    }

    public IMFSample CreateSampleAndAllocateTexture()
    {
        IMFSample pSample;
        IMFTrackedSample pTrackedSample;
        HResult hr;

        // Create the video sample. This function returns an IMFTrackedSample per MSDN
        hr = MFExtern.MFCreateVideoSampleFromSurface(null, out pSample);
        MFError.ThrowExceptionForHR(hr);

        // Query the IMFSample to see if it implements IMFTrackedSample
        pTrackedSample = pSample as IMFTrackedSample;
        if(pTrackedSample == null)
        {
            // Throw an exception if we didn't get an IMFTrackedSample
            // but this shouldn't happen in practice.
            throw new InvalidCastException("MFCreateVideoSampleFromSurface returned a sample that did not implement IMFTrackedSample");
        }

        // Use our own class to allocate a texture
        SharpDX.Direct3D11.Texture2D availableTexture = AllocateTexture();
        // Convert the texture's native ID3D11Texture2D pointer into
        // an IUnknown (represented as as System.Object)
        object texNativeObject = Marshal.GetObjectForIUnknown(availableTexture.NativePointer);

        // Create the media buffer from the texture
        IMFMediaBuffer p2DBuffer;
        hr = MFExtern.MFCreateDXGISurfaceBuffer(s_IID_ID3D11Texture2D, texNativeObject, 0, false, out p2DBuffer);
        // Release the object-as-IUnknown we created above
        COMBase.SafeRelease(texNativeObject);
        // If media buffer creation failed, throw an exception
        MFError.ThrowExceptionForHR(hr);

        // Set the owning instance of this class as the allocator
        // for IMFTrackedSample to notify when the sample is released
        pTrackedSample.SetAllocator(this, null);

        // Attach the created buffer to the sample
        pTrackedSample.AddBuffer(p2DBuffer);

        return pTrackedSample;
    }

    // This is public so any textures you allocate but don't make IMFSamples 
    // out of can be returned to the allocator manually.
    public void ReturnFreeTexture(SharpDX.Direct3D11.Texture2D freeTexture)
    {
        m_freeStack.Push(freeTexture);
    }

    // IMFAsyncCallback.GetParameters
    // This is allowed to return E_NOTIMPL as a way of specifying
    // there are no special parameters.
    public HResult GetParameters(out MFAsync pdwFlags, out MFAsyncCallbackQueue pdwQueue)
    {
        pdwFlags = MFAsync.None;
        pdwQueue = MFAsyncCallbackQueue.Standard;
        return HResult.E_NOTIMPL;
    }

    public HResult Invoke(IMFAsyncResult pResult)
    {
        object pUnkObject;
        IMFSample pSample = null;
        IMFMediaBuffer pBuffer = null;
        IMFDXGIBuffer pDXGIBuffer = null;

        // Get the IUnknown out of the IMFAsyncResult if there is one
        HResult hr = pResult.GetObject(out pUnkObject);
        if(Succeeded(hr))
        {
            pSample = pUnkObject as IMFSample;
        }

        if(pSample != null)
        {
            // Based on your implementation, there should only be one 
            // buffer attached to one sample, so we can always grab the
            // first buffer. You could add some error checking here to make
            // sure the sample has a buffer count that is 1.
            hr = pSample.GetBufferByIndex(0, out pBuffer);
        }

        if(Succeeded(hr))
        {
            // Query the IMFMediaBuffer to see if it implements IMFDXGIBuffer
            pDXGIBuffer = pBuffer as IMFDXGIBuffer;
        }

        if(pDXGIBuffer != null)
        {
           // Got an IMFDXGIBuffer, so we can extract the internal 
           // ID3D11Texture2D and make a new SharpDX.Texture2D wrapper.
           hr = pDXGIBuffer.GetResource(s_IID_ID3D11Texture2D, out pUnkObject);
        }

        if(Succeeded(hr))
        {
           // If we got here, pUnkObject is the native D3D11 Texture2D as
           // a System.Object, but it's unlikely you have an interface 
           // definition for ID3D11Texture2D handy, so we can't just cast
           // the object to the proper interface.

           // Happily, SharpDX supports wrapping System.Object within
           // SharpDX.ComObject which makes things pretty easy.
           SharpDX.ComObject comWrapper = new SharpDX.ComObject(pUnkObject);

           // If this doesn't work, or you're using something like SlimDX
           // which doesn't support object wrapping the same way, the below
           // code is an alternative way.
           /*
           IntPtr pD3DTexture2D = Marshal.GetIUnknownForObject(pUnkObject);
           // Create your wrapper object here, like this for SharpDX
           SharpDX.ComObject comWrapper = new SharpDX.ComObject(pD3DTexture2D);
           // or like this for SlimDX
           SlimDX.Direct3D11.Texture2D.FromPointer(pD3DTexture2D);
           Marshal.Release(pD3DTexture2D);
           */

           // You might need to query comWrapper for a SharpDX.DXGI.Resource
           // first, then query that for the SharpDX.Direct3D11.Texture2D.
           SharpDX.Direct3D11.Texture2D texture = comWrapper.QueryInterface<SharpDX.Direct3D11.Texture2D>();
           if(texture != null)
           {
               // Now you can add "texture" back to the allocator's free list
               ReturnFreeTexture(texture);
           }
        }
    }
}

在Sink Writer输入媒体类型上设置 MF_SA_D3D_AWARE

我不认为这会导致您得到不良的 HRESULT，但是无论如何都不是正确的选择。 MF_SA_D3D_AWARE（及其DX11对应的 MF_SA_D3D11_AWARE）是由 IMFTransform对象设置的属性，以通知您转换分别支持通过DX9或DX11进行图形加速。无需在Sink Writer的输入媒体类型上进行设置。


SafeRelease上没有 texNativeObject

建议您在 COMBase.SafeRelease()上调用 texNativeObject，否则可能会泄漏内存。那样，否则您将不必要地延长该COM对象的寿命，直到GC为您清除引用计数为止


不必要的铸造

这是上面代码的一部分：

buffer = MFVideoEncoderST.ReinterpretCast<IMF2DBuffer,IMFMediaBuffer>(p2Dbuffer);
int length=0;
if (Succeeded(hr)) hr = (int)p2Dbuffer.GetContiguousLength(out length);
if (Succeeded(hr)) hr = (int)buffer.SetCurrentLength(length);

我不确定您的 ReinterpretCast函数在做什么，但是如果您确实需要在C＃中执行 QueryInterface样式转换，则可以只使用 as运算符或常规转换。

// pMediaBuffer is of type IMFMediaBuffer and has been created elsewhere
IMF2DBuffer p2DBuffer = pMediaBuffer as IMF2DBuffer;
if(p2DBuffer != null)
{
    // pMediaBuffer is an IMFMediaBuffer that also implements IMF2DBuffer
}
else
{
    // pMediaBuffer does not implement IMF2DBuffer
}