android - 使用 Android MediaCodec 以不同的比特率重新编码 h.264 内容

Question

我正在尝试使用 4.2 中引入的 Android MediaCodec API 以不同的比特率重新编码 h.264 mp4 文件。

我能够使用 MediaCodec API 解码和播放内容(在更改比特率之前)，但是如果我在解码之前尝试使用不同的比特率重新编码内容，我会得到乱码(绿色灰色像素化屏幕)。

我使用的代码基于 Android 测试用例 android.media.cts.DecoderTest Android Test Case android.media.cts.DecoderTest :

public void encodeDecodeVideoFile(AssetFileDescriptor assetFileDescriptor) {

    int bitRate = 500000;
    int frameRate = 30;
    int width = 480;
    int height = 368;
    String mimeType = "video/avc";

    MediaCodec encoder, decoder = null;
    ByteBuffer[] encoderInputBuffers;
    ByteBuffer[] encoderOutputBuffers;
    ByteBuffer[] decoderInputBuffers = null;
    ByteBuffer[] decoderOutputBuffers = null;

    // Find a code that supports the mime type
    int numCodecs = MediaCodecList.getCodecCount();
    MediaCodecInfo codecInfo = null;
    for (int i = 0; i < numCodecs && codecInfo == null; i++) {
        MediaCodecInfo info = MediaCodecList.getCodecInfoAt(i);
        if (!info.isEncoder()) {
            continue;
        }
        String[] types = info.getSupportedTypes();
        boolean found = false;
        for (int j = 0; j < types.length && !found; j++) {
            if (types[j].equals(mimeType))
                found = true;
        }
        if (!found)
            continue;
        codecInfo = info;
    }
    Log.d(TAG, "Found " + codecInfo.getName() + " supporting " + mimeType);

    // Find a color profile that the codec supports
    int colorFormat = 0;
    MediaCodecInfo.CodecCapabilities capabilities = codecInfo.getCapabilitiesForType(mimeType);
    for (int i = 0; i < capabilities.colorFormats.length && colorFormat == 0; i++) {
        int format = capabilities.colorFormats[i];
        switch (format) {
        case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Planar:
        case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedPlanar:
        case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar:
        case MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedSemiPlanar:
        case MediaCodecInfo.CodecCapabilities.COLOR_TI_FormatYUV420PackedSemiPlanar:
            colorFormat = format;
            break;
        default:
            Log.d(TAG, "Skipping unsupported color format " + format);
            break;
        }
    }
    Log.d(TAG, "Using color format " + colorFormat);

    // Determine width, height and slice sizes
    if (codecInfo.getName().equals("OMX.TI.DUCATI1.VIDEO.H264E")) {
        // This codec doesn't support a width not a multiple of 16,
        // so round down.
        width &= ~15;
    }
    int stride = width;
    int sliceHeight = height;
    if (codecInfo.getName().startsWith("OMX.Nvidia.")) {
        stride = (stride + 15) / 16 * 16;
        sliceHeight = (sliceHeight + 15) / 16 * 16;
    }

    // Used MediaExtractor to select the first track from the h.264 content
    MediaExtractor extractor  = new MediaExtractor();
    extractor.setDataSource(assetFileDescriptor.getFileDescriptor(), assetFileDescriptor.getStartOffset(), assetFileDescriptor.getLength());
    MediaFormat extractedFormat = extractor.getTrackFormat(0);
    String mime = extractedFormat.getString(MediaFormat.KEY_MIME);
    Log.d(TAG, "Extartced Mime " + mime);
    extractor.selectTrack(0);

    // Create an encoder
    encoder = MediaCodec.createByCodecName(codecInfo.getName());
    MediaFormat inputFormat = MediaFormat.createVideoFormat(mimeType, width, height);
    inputFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitRate);
    inputFormat.setInteger(MediaFormat.KEY_FRAME_RATE, frameRate);
    inputFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
    inputFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 5);
    inputFormat.setInteger("stride", stride);
    inputFormat.setInteger("slice-height", sliceHeight);
    Log.d(TAG, "Configuring encoder with input format " + inputFormat);
    encoder.configure(inputFormat, null /* surface */, null /* crypto */, MediaCodec.CONFIGURE_FLAG_ENCODE);
    encoder.start();
    encoderInputBuffers = encoder.getInputBuffers();
    encoderOutputBuffers = encoder.getOutputBuffers();

    // start encoding + decoding
    final long kTimeOutUs = 5000;
    MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
    boolean sawInputEOS = false;
    boolean sawOutputEOS = false;
    MediaFormat oformat = null;
    long startMs = System.currentTimeMillis();
    while (!sawOutputEOS) {
        if (!sawInputEOS) {
            int inputBufIndex = encoder.dequeueInputBuffer(kTimeOutUs);

            if (inputBufIndex >= 0) {

                ByteBuffer dstBuf = encoderInputBuffers[inputBufIndex];

                int sampleSize = extractor.readSampleData(dstBuf, 0 /* offset */);

                long presentationTimeUs = 0;

                if (sampleSize < 0) {
                    Log.d(TAG, "saw input EOS.");
                    sawInputEOS = true;
                    sampleSize = 0;
                } else {
                    presentationTimeUs = extractor.getSampleTime();
                }

                encoder.queueInputBuffer(inputBufIndex, 0 /* offset */, sampleSize, presentationTimeUs, sawInputEOS ? MediaCodec.BUFFER_FLAG_END_OF_STREAM : 0);

                if (!sawInputEOS) {
                    extractor.advance();
                }
            }
        }

        int res = encoder.dequeueOutputBuffer(info, kTimeOutUs);

        if (res >= 0) {
            int outputBufIndex = res;
            ByteBuffer buf = encoderOutputBuffers[outputBufIndex];

            buf.position(info.offset);
            buf.limit(info.offset + info.size);

            if ((info.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {

                // create a decoder
                decoder = MediaCodec.createDecoderByType(mimeType);
                MediaFormat format = MediaFormat.createVideoFormat(mimeType, width, height);
                format.setInteger(MediaFormat.KEY_COLOR_FORMAT, colorFormat);
                format.setByteBuffer("csd-0", buf);
                decoder.configure(format, surface /* surface */, null /* crypto */, 0 /* flags */);
                decoder.start();
                decoderInputBuffers = decoder.getInputBuffers();
                decoderOutputBuffers = decoder.getOutputBuffers();
            } else {
                int decIndex = decoder.dequeueInputBuffer(-1);
                decoderInputBuffers[decIndex].clear();
                decoderInputBuffers[decIndex].put(buf);
                decoder.queueInputBuffer(decIndex, 0, info.size, info.presentationTimeUs, info.flags);
            }

            encoder.releaseOutputBuffer(outputBufIndex, false /* render */);
        } else if (res == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
            encoderOutputBuffers = encoder.getOutputBuffers();

            Log.d(TAG, "encoder output buffers have changed.");
        } else if (res == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
            MediaFormat encformat = encoder.getOutputFormat();

            Log.d(TAG, "encoder output format has changed to " + encformat);
        }

        if (decoder == null)
            res = MediaCodec.INFO_TRY_AGAIN_LATER;
        else
            res = decoder.dequeueOutputBuffer(info, kTimeOutUs);

        if (res >= 0) {
            int outputBufIndex = res;
            ByteBuffer buf = decoderOutputBuffers[outputBufIndex];

            buf.position(info.offset);
            buf.limit(info.offset + info.size);

            // The worlds simplest FPS implementation
            while (info.presentationTimeUs / 1000 > System.currentTimeMillis() - startMs) {
                try {
                    Thread.sleep(10);
                } catch (InterruptedException e) {
                    e.printStackTrace();
                    break;
                }
            }

            decoder.releaseOutputBuffer(outputBufIndex, true /* render */);

            if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
                Log.d(TAG, "saw output EOS.");
                sawOutputEOS = true;
            }
        } else if (res == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
            decoderOutputBuffers = decoder.getOutputBuffers();

            Log.d(TAG, "decoder output buffers have changed.");
        } else if (res == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
            oformat = decoder.getOutputFormat();

            Log.d(TAG, "decoder output format has changed to " + oformat);
        }

    }

    encoder.stop();
    encoder.release();
    decoder.stop();
    decoder.release();
}

我尝试编码的文件来自 Android cts 测试项目:

R.raw.video_480x360_mp4_h264_1000kbps_30fps_aac_stereo_128kbps_44100hz

我猜问题与我在编码器 MediaCodec 中指定的格式参数有关，但我无法找出不正确/缺失的地方。

Answer 1

在您的程序中，存在设计问题。在循环 while (!sawOutputEOS) { 您尝试重新编码的地方，

int inputBufIndex = encoder.dequeueInputBuffer(kTimeOutUs);

从编码器的输入/输入端口出列一个缓冲区

 ByteBuffer dstBuf = encoderInputBuffers[inputBufIndex];

指向出列缓冲区的缓冲区指针

int sampleSize = extractor.readSampleData(dstBuf, 0 /* offset */);

出列缓冲区中填充了来自提取器的数据。提取器的输出是压缩比特流。这不是 不是 YUV 未压缩的帧。

encoder.queueInputBuffer(inputBufIndex,....)

在这里，压缩比特流被编码。由于这不是 YUV 帧，编码器将尝试尽其所能执行压缩，因此，您会在编码器的输出端观察到一个难以辨认的绿色帧。我假设您在屏幕上观察到了这一点，因为您在解码相同内容的代码的以下部分。即使编码器的输出被写入文件并通过不同的播放器或在 PC 上播放，也会观察到这种情况。

根据程序，我假设您的预期设计是 Decode ==> Encode ==> Live Decode 您的图形应该是

MediaExtractor ==> MediaCodec (Decoder) ==> MediaCodec (Encoder) ==> MediaCodec (Decoder) ==> Display

P.S:您在运行该程序时是否观察到任何内存违规？