c++ - 媒体基础音频/视频捕获到 MPEG4 FileSink 产生不正确的持续时间

Question

我正在使用媒体基础框架开发媒体流应用程序。我使用了一些来自互联网和 Anton Polinger 书中的样本。不幸的是，将流保存到 mp4 文件后，文件的元数据已损坏。它有不正确的持续时间(根据我的 PC 的工作时间，例如 30 小时)，错误的比特率。经过长时间的努力，我已经为单流(视频或音频)修复了它，但是当我尝试同时录制音频和视频时，这个问题再次出现。我的拓扑有问题，但我不明白是什么，可能这里有一些专家？

我获取音频和视频源，将其包装到 IMFCollection 中，通过 MFCreateAggregateSource 创建聚合源。我为聚合源中的每个源创建源节点:

Com::IMFTopologyNodePtr 
TopologyBuilder::CreateSourceNode(Com::IMFStreamDescriptorPtr 
streamDescriptor)
{
    HRESULT hr = S_OK;
    Com::IMFTopologyNodePtr pNode;
    // Create the topology node, indicating that it must be a source node.
    hr = MFCreateTopologyNode(MF_TOPOLOGY_SOURCESTREAM_NODE, &pNode);
    THROW_ON_FAIL(hr, "Unable to create topology node for source");

    // Associate the node with the source by passing in a pointer to the media source,
    // and indicating that it is the source
    hr = pNode->SetUnknown(MF_TOPONODE_SOURCE, _sourceDefinition->GetMediaSource());
    THROW_ON_FAIL(hr, "Unable to set source as object for topology node");

    // Set the node presentation descriptor attribute of the node by passing
    // in a pointer to the presentation descriptor
    hr = pNode->SetUnknown(MF_TOPONODE_PRESENTATION_DESCRIPTOR, _sourceDefinition->GetPresentationDescriptor());
    THROW_ON_FAIL(hr, "Unable to set MF_TOPONODE_PRESENTATION_DESCRIPTOR to node");

    // Set the node stream descriptor attribute by passing in a pointer to the stream
    // descriptor
    hr = pNode->SetUnknown(MF_TOPONODE_STREAM_DESCRIPTOR, streamDescriptor);
    THROW_ON_FAIL(hr, "Unable to set MF_TOPONODE_STREAM_DESCRIPTOR to node");

    return pNode;
}

之后我将每个源连接到转换(H264 编码器和 AAC 编码器)和 MPEG4FileSink:

void TopologyBuilder::CreateFileSinkOutputNode(PCWSTR filePath)
{
    HRESULT hr = S_OK;
    DWORD sink_count;

    Com::IMFByteStreamPtr byte_stream;
    Com::IMFTransformPtr transform;

    LPCWSTR lpcwstrFilePath = filePath;
    hr = MFCreateFile(
    MF_ACCESSMODE_WRITE, MF_OPENMODE_FAIL_IF_NOT_EXIST, MF_FILEFLAGS_NONE,
    lpcwstrFilePath, &byte_stream);
    THROW_ON_FAIL(hr, L"Unable to create and open file");

// Video stream
    Com::IMFMediaTypePtr in_mf_video_media_type = _sourceDefinition->GetCurrentVideoMediaType();

    Com::IMFMediaTypePtr out_mf_media_type = CreateMediaType(MFMediaType_Video, MFVideoFormat_H264);
    hr = CopyType(in_mf_video_media_type, out_mf_media_type);
    THROW_ON_FAIL(hr, L"Unable to copy type parameters");

    if (GetSubtype(in_mf_video_media_type) != MEDIASUBTYPE_H264)
    {
        transform.Attach(CreateAndInitCoderMft(MFT_CATEGORY_VIDEO_ENCODER, out_mf_media_type));
        THROW_ON_NULL(transform);
    }

    if (transform)
    {
        Com::IMFMediaTypePtr transformMediaType;
        hr = transform->GetOutputCurrentType(0, &transformMediaType);
        THROW_ON_FAIL(hr, L"Unable to get current output type");

        UINT32 pcbBlobSize = 0;
        hr = transformMediaType->GetBlobSize(MF_MT_MPEG_SEQUENCE_HEADER, &pcbBlobSize);
        THROW_ON_FAIL(hr, L"Unable to get blob size of MF_MT_MPEG_SEQUENCE_HEADER");

        std::vector<UINT8> blob(pcbBlobSize);
        hr = transformMediaType->GetBlob(MF_MT_MPEG_SEQUENCE_HEADER, &blob.front(), blob.size(), NULL);
        THROW_ON_FAIL(hr, L"Unable to get blob MF_MT_MPEG_SEQUENCE_HEADER");

        hr = out_mf_media_type->SetBlob(MF_MT_MPEG_SEQUENCE_HEADER, &blob.front(), blob.size());
        THROW_ON_FAIL(hr, L"Unable to set blob of MF_MT_MPEG_SEQUENCE_HEADER");
    }

    // Audio stream
    Com::IMFMediaTypePtr out_mf_audio_media_type;
    Com::IMFTransformPtr transformAudio;
    Com::IMFMediaTypePtr mediaTypeTmp = _sourceDefinition->GetCurrentAudioMediaType();
    Com::IMFMediaTypePtr in_mf_audio_media_type;
    if (mediaTypeTmp != NULL)
    {
        std::unique_ptr<MediaTypesFactory> factory(new MediaTypesFactory());
        if (!IsMediaTypeSupportedByAacEncoder(mediaTypeTmp))
        {
            UINT32 channels;
            hr = mediaTypeTmp->GetUINT32(MF_MT_AUDIO_NUM_CHANNELS, &channels);
            THROW_ON_FAIL(hr, L"Unable to get MF_MT_AUDIO_NUM_CHANNELS fron source media type");
            in_mf_audio_media_type = factory->CreatePCM(factory->DEFAULT_SAMPLE_RATE, channels);
        }
        else
        {
            in_mf_audio_media_type.Attach(mediaTypeTmp.Detach());
        }

        out_mf_audio_media_type = factory->CreateAAC(in_mf_audio_media_type, factory->HIGH_ENCODED_BITRATE);
        GUID subType = GetSubtype(in_mf_audio_media_type);
        if (GetSubtype(in_mf_audio_media_type) != MFAudioFormat_AAC)
        {
            // add encoder to Aac
        transformAudio.Attach(CreateAndInitCoderMft(MFT_CATEGORY_AUDIO_ENCODER, out_mf_audio_media_type));
        }
    }

    Com::IMFMediaSinkPtr pFileSink;
    hr = MFCreateMPEG4MediaSink(byte_stream, out_mf_media_type,     out_mf_audio_media_type, &pFileSink);
    THROW_ON_FAIL(hr, L"Unable to create mpeg4 media sink");

    Com::IMFTopologyNodePtr pOutputNodeVideo;
    hr = MFCreateTopologyNode(MF_TOPOLOGY_OUTPUT_NODE, &pOutputNodeVideo);
    THROW_ON_FAIL(hr, L"Unable to create output node");

    hr = pFileSink->GetStreamSinkCount(&sink_count);
    THROW_ON_FAIL(hr, L"Unable to get stream sink count from mediasink");

    if (sink_count == 0)
    {
        THROW_ON_FAIL(E_UNEXPECTED, L"Sink count should be greater than 0");
    }

    Com::IMFStreamSinkPtr stream_sink_video;
    hr = pFileSink->GetStreamSinkByIndex(0, &stream_sink_video);
    THROW_ON_FAIL(hr, L"Unable to get stream sink by index");

    hr = pOutputNodeVideo->SetObject(stream_sink_video);
    THROW_ON_FAIL(hr, L"Unable to set stream sink as output node object");

    hr = _pTopology->AddNode(pOutputNodeVideo);
    THROW_ON_FAIL(hr, L"Unable to add file sink output node");

    pOutputNodeVideo = AddEncoderIfNeed(_pTopology, transform, in_mf_video_media_type, pOutputNodeVideo);

    _outVideoNodes.push_back(pOutputNodeVideo);

    Com::IMFTopologyNodePtr pOutputNodeAudio;

    if (in_mf_audio_media_type != NULL)
    {
        hr = MFCreateTopologyNode(MF_TOPOLOGY_OUTPUT_NODE, &pOutputNodeAudio);
        THROW_ON_FAIL(hr, L"Unable to create output node");

        Com::IMFStreamSinkPtr stream_sink_audio;
        hr = pFileSink->GetStreamSinkByIndex(1, &stream_sink_audio);
        THROW_ON_FAIL(hr, L"Unable to get stream sink by index");

        hr = pOutputNodeAudio->SetObject(stream_sink_audio);
        THROW_ON_FAIL(hr, L"Unable to set stream sink as output node object");

        hr = _pTopology->AddNode(pOutputNodeAudio);
        THROW_ON_FAIL(hr, L"Unable to add file sink output node");

        if (transformAudio)
        {
            Com::IMFTopologyNodePtr outputTransformNodeAudio;
            AddTransformNode(_pTopology, transformAudio, pOutputNodeAudio, &outputTransformNodeAudio);

            _outAudioNode = outputTransformNodeAudio;
        }
        else
    {
            _outAudioNode = pOutputNodeAudio;
        }
    }
}

当输出类型应用于音频转换时，它有 15 个属性而不是 8 个，包括 MF_MT_AVG_BITRATE，据我所知应该应用于视频。在我的例子中，它是 192000，它与视频流上的 MF_MT_AVG_BITRATE 不同。我的 AAC 媒体类型是通过这种方法创建的:

HRESULT MediaTypesFactory::CopyAudioTypeBasicAttributes(IMFMediaType * in_media_type, IMFMediaType * out_mf_media_type) {
    HRESULT hr = S_OK;
    static const GUID AUDIO_MAJORTYPE = MFMediaType_Audio;
    static const GUID AUDIO_SUBTYPE = MFAudioFormat_PCM;

    out_mf_media_type->SetUINT32(MF_MT_AUDIO_BITS_PER_SAMPLE, AUDIO_BITS_PER_SAMPLE);

    WAVEFORMATEX *in_wfx;
    UINT32 wfx_size;

    MFCreateWaveFormatExFromMFMediaType(in_media_type, &in_wfx, &wfx_size);

    hr = out_mf_media_type->SetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND, in_wfx->nSamplesPerSec);
    DEBUG_ON_FAIL(hr);

    hr = out_mf_media_type->SetUINT32(MF_MT_AUDIO_NUM_CHANNELS, in_wfx->nChannels);
    DEBUG_ON_FAIL(hr);

    hr = out_mf_media_type->SetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, in_wfx->nAvgBytesPerSec);
    DEBUG_ON_FAIL(hr);

    hr = out_mf_media_type->SetUINT32(MF_MT_AUDIO_BLOCK_ALIGNMENT, in_wfx->nBlockAlign);
    DEBUG_ON_FAIL(hr);

    return hr;
}

如果有人能帮助我或解释我错在哪里，那就太棒了。谢谢。

Answer 1

在我的项目中 CaptureManager我遇到了类似的问题 - 虽然我已经编写了用于将来自许多网络摄像头的实时视频录制到一个文件中的代码。经过对 Media Foundation 的长期研究，我发现了两个重要的事实:1. 实时源 - 网络摄像头和麦克风不是从 0 开始的 - 根据它们的规范样本应该从 0 时间戳开始 - Live Sources - “第一个样本的时间戳应该为零。” - 但实时源设置当前系统时间。2.我从你的代码中看到你使用Media Session - 它是一个具有 IMFMediaSession 接口(interface)的对象。我认为您是通过 MFCreateMediaSession 函数创建的。此函数创建 session 的默认版本，该 session 针对从文件播放媒体进行了优化，默认情况下从 0 开始采样。在我看来，主要问题是默认媒体 session 不检查源媒体样本的时间戳，因为媒体文件从零或 StartPosition 开始。然而，实时源不是从 0 开始的——它们应该，或者必须，但不是。所以，我的建议 - 使用 IMFTransform 编写类，这将是源和编码器之间的“代理”转换 - 这个“代理”转换必须修复来自实时源的媒体样本的时间戳:1. 当它首先接收时来自直播源的媒体样本，它保存第一个媒体样本的实际时间戳，如引用时间，并将第一个媒体样本的时间戳设置为零，所有时间戳来自这个直播源的下一个媒体样本必须减去这个引用时间并设置为媒体样本的时间戳。另外，检查调用 IMFFinalizableMediaSink 的代码.

问候。