c - FFmpeg - pts 和 dts 不能正确增加视频，但可以增加音频

Question

我正在尝试拍摄两个视频并将它们放在一个视频中。但是，当我运行代码时，在解码/编码第二个视频时出现此错误:

Application provided invalid, non monotonically increasing dts to muxer in stream 0

代码完成后，第一个视频完全没问题，但第二个视频不是。我能够产生的最佳结果是第二个视频的后半部分紧接在第一个视频之后。有趣的是，音频完全没有问题，而且应该是这样。

过去，我能够使用我的解码/编码代码来简单地复制视频(视频和音频)。

我已经在网上搜索了这个特定问题并尝试了建议的解决方案，但似乎没有一个能解决我的问题。这些是我一直在寻找的线程:

FFmpeg - What does non monotonically increasing dts mean?

How to use libavformat to concat 2 video files with same codec (re-muxing)?

Non monotonically increasing dts to muxer in stream

这是我编写的当前代码:

Video 和 ClipSequence 结构:

typedef struct Video {
    char* filename;
    AVFormatContext* inputContext;
    AVFormatContext* outputContext;
    AVCodec* videoCodec;
    AVCodec* audioCodec;
    AVStream* inputStream;
    AVStream* outputStream;
    AVCodecContext* videoCodecContext_I; // Input
    AVCodecContext* audioCodecContext_I; // Input
    AVCodecContext* videoCodecContext_O; // Output
    AVCodecContext* audioCodecContext_O; // Output
    int videoStream;
    int audioStream;
    SwrContext* swrContext;
} Video;

typedef struct ClipSequence {
    VideoList* videos;
    AVFormatContext* outputContext;
    AVStream* outputStream;
    int64_t lastpts, lastdts;
    int64_t currentpts, currentdts;
} ClipSequence;

解码和编码(音频代码相同):

int decodeVideoSequence(ClipSequence* sequence, Video* video, AVPacket* packet, AVFrame* frame) {
    int response = avcodec_send_packet(video->videoCodecContext_I, packet);
    if (response < 0) {
        printf("[ERROR] Failed to send video packet to decoder\n");
        return response;
    }
    while (response >= 0) {
        response = avcodec_receive_frame(video->videoCodecContext_I, frame);
        if (response == AVERROR(EAGAIN) || response == AVERROR_EOF) {
            break;
        } else if (response < 0) {
            printf("[ERROR] Failed to receive video frame from decoder\n");
            return response;
        }
        if (response >= 0) {
            // Do stuff and encode
            sequence->currentpts = packet->pts; // Store decoded packet's pts and dts
            sequence->currentdts = packet->dts;
            if (encodeVideoSequence(sequence, video, frame) < 0) {
                printf("[ERROR] Failed to encode new video\n");
                return -1;
            }
        }
        av_frame_unref(frame);
    }
    return 0;
}

int encodeVideoSequence(ClipSequence* sequence, Video* video, AVFrame* frame) {
    AVPacket* packet = av_packet_alloc();
    if (!packet) {
        printf("[ERROR] Could not allocate memory for video output packet\n");
        return -1;
    }
    int response = avcodec_send_frame(video->videoCodecContext_O, frame);
    if (response < 0) {
        printf("[ERROR] Failed to send video frame for encoding\n");
        return response;
    }
    while (response >= 0) {
        response = avcodec_receive_packet(video->videoCodecContext_O, packet);
        if (response == AVERROR(EAGAIN) || response == AVERROR_EOF) {
            break;
        } else if (response < 0) {
            printf("[ERROR] Failed to receive video packet from encoder\n");
            return response;
        }
        // Set packet to have pts and dts based on the previous video's pts and dts
        packet->flags |= AV_PKT_FLAG_KEY;
        packet->pts = sequence->currentpts + sequence->lastpts;
        packet->dts = sequence->currentdts + sequence->lastdts;
        packet->stream_index = video->videoStream;
        packet->duration = 1000; // 60 fps
        response = av_interleaved_write_frame(sequence->outputContext, packet);
        if (response < 0) {
            printf("[ERROR] Failed to write video packet\n");
            break;
        }
    }
    av_packet_unref(packet);
    av_packet_free(&packet);
    return 0;
}

读取帧:

int readSequenceFrames(ClipSequence* sequence, Video* video, AVPacket* packet, AVFrame* frame) {
    if (!packet) {
        printf("[ERROR] Packet not allocated to be read\n");
        return -1;
    }
    if (!frame) {
        printf("[ERROR] Frame not allocated to be read\n");
        return -1;
    }
    // Sets video and audio codec context parameters
    if (prepareVideoOutStream(video) < 0) {
        printf("[ERROR] Failed to prepare output video stream\n");
        return -1;
    }
    if (prepareAudioOutStream(video) < 0) {
        printf("[ERROR] Failed to prepare output audio stream\n");
        return -1;
    }
    // Prepares audio resampling
    if (initResampler(video->audioCodecContext_I, video->audioCodecContext_O, &(video->swrContext)) < 0) {
        printf("[ERROR] Failed to init audio resampler\n");
        return -1;
    }
    // Read packets
    int frameNum = 0;
    while (av_read_frame(video->inputContext, packet) >= 0) {
        printf("[READ] Reading frame %i\n", frameNum);
        if (packet->stream_index == video->videoStream) {
            if (decodeVideoSequence(sequence, video, packet, frame) < 0) {
                printf("[ERROR] Failed to decode and encode video\n");
                return -1;
            }
        } else if (packet->stream_index == video->audioStream) {
            if (decodeAudioSequence(sequence, video, packet, frame) < 0) {
                printf("[ERROR] Failed to decode and encode audio\n");
                return -1;
            }
        }
        av_packet_unref(packet);
        frameNum++;
    }
    // Increment pts and dts by the last pts and dts in the current video
    sequence->lastpts += sequence->currentpts;
    sequence->lastdts += sequence->currentdts;
    return 0;
}

我相信当我增加 pts 和 dts 时我有正确的逻辑。我不确定我到底错过了什么。

谢谢。

Answer 1

   // Increment pts and dts by the last pts and dts in the current video
    sequence->lastpts += sequence->currentpts;
    sequence->lastdts += sequence->currentdts;

这是错误的。首先，暂时忽略 PTS，只处理 DTS。

DTS 不是相对数，而是绝对数。将递增的数字相加会创建一个指数序列。例如:如果这是一个 30 fps 的视频，并且时基为 1/30，则 DTS 将每帧前进一帧。例如0、1、2、3、4、5、6、7、9

如果你继续把它们加在一起，你会得到:0、1、3、6、10、15、21、28、36、45

所以， sequence->lastdts = sequence->currentdts;不是 sequence->lastdts += sequence->currentdts;
设置新的 DTS 时，需要添加帧时长
例如 packet->dts = sequence->lastdts + frame_duration;
否则这一帧将与前一帧具有相同的 DTS，当它需要多 1 帧时

下一个 PTS:

PTS 不是单调的，可以在时间上倒退。您并不能以这种方式跟踪它，因为下一个 PTS 可能是较低的时间戳。要解决此问题，您需要将 PTS 转换为 CTS，然后再返回:

auto cts = packet->pts - packet->dts
packet->dts = // Whatever you are updating the dts to
packet->pts = packet->dts + cts

同时设置 packet->flags |= AV_PKT_FLAG_KEY;在每一帧上寻找时都会引起问题