android - 音视频Muxing问题

Question

我正在修改同时写入音频和视频的音频编码器示例。我正在尝试通过MediaProjection从设备显示中获取视频原始数据，并通过AudioRecord从麦克风中获取音频原始数据。然后，我将它们发送给MediaCodec(我将设置两个Codec实例)。

之后，我将视频数据和音频数据发送到MediaMux以获取mp4文件。我有任何问题:

结果，我得到一个具有完美视频轨道但音轨糟糕的文件。音频轨道的播放在轨道的开头具有一定的间隔，没有暂停，但在结尾处非常快。

当我使用android MXPlayer播放视频都可以，但是当我使用PC播放器(Windows Media或Media Player Classic)播放时，音频在MXPlayer上播放，但视频不播放-仅显示第一帧。

Sample of result video:

部分代码:

/**
* Method run of EncoderTask
*/
public void run() {
    if (mIsInitialized) {
        switch (type) {
            case ENCODE_AUDIO_FRAME:
                if (!mStopReceived) {
                    _offerAudioEncoder(mAudioData, presentationTimeNs);
                    mDrainHandler.postDelayed(new Runnable() {
                        @Override
                        public void run() {
                            mEncodingService.submit(new EncoderTask(Encoder.this, EncoderTaskType.ENCODER_VIDEO_FRAME));
                        }
                    }, DELAY_MILLIS); // 10 milliseconds
                }
                break;
            case ENCODER_VIDEO_FRAME:
                if (!mStopReceived) {
                    encoder._offerVideoEncoder();
                    mDrainHandler.postDelayed(new Runnable() {
                        @Override
                        public void run() {
                            mEncodingService.submit(new EncoderTask(Encoder.this, EncoderTaskType.ENCODE_AUDIO_FRAME));
                        }
                    }, DELAY_MILLIS); // 10 milliseconds
                }
                break;
            case FINALIZE_ENCODER:
                finalizeEncoder();
                break;

        }
        // prevent multiple execution of same task
        mIsInitialized = false;
        mEncodingServiceQueueLength -= 1;
    } else {
        Log.e(TAG, "run() called but EncoderTask not initialized");
    }
}

public void _offerVideoEncoder() {
    Log.d(TAG, "Offer video");
    if (mStopReceived) {
        closeVideoEncoder();
        Log.d(TAG, "Offer video - stop");
    } else {
        drainEncoder(mVideoEncoder, mVideoBufferInfo, mVideoTrackIndex, false, "video");
        Log.d(TAG, "Offer video - drain");
        if (mStopReceived) {
            closeVideoEncoder();
            Log.d(TAG, "Offer video - stop");
        }
    }
}   

public void processAudioFrame() {
    long audioPresentationTimeNs = System.nanoTime();
    byte[] thisBuffer;
    if (mDataBuffer.isEmpty()) {
        thisBuffer = new byte[mSamplesPerFrame];
    } else {
        thisBuffer = mDataBuffer.poll();
    }

    mReadResult = mAudioRecorder.read(thisBuffer, 0, mSamplesPerFrame);
    if (VERBOSE) Log.i(TAG, "FillBuffer       real: " + String.valueOf(mBufferWriteIndex)
                + " - " + String.valueOf(mBufferWriteIndex + mReadResult - 1));

    if (mReadResult != AudioRecord.ERROR_BAD_VALUE && mReadResult != AudioRecord.ERROR_INVALID_OPERATION) {
        mBufferWriteIndex = mBufferWriteIndex + mReadResult - 1;
        mTotalFramesWritten++;
        if (mAudioEncoder != null) {
            mAudioEncoder.offerEncoder(thisBuffer, audioPresentationTimeNs);
        }


        if (!mIsRecording && mAudioRecorder != null) {
            mAudioRecorder.setRecordPositionUpdateListener(null);
            mAudioRecorder.release();
            mAudioRecorder = null;
            Log.i(TAG, "stopped");
        }
    } else {
        Log.e(TAG, "Read error");
    }
}


private void _offerAudioEncoder(byte[] input, long presentationTimeNs) {
    if (audioBytesReceived == 0) {
        mAudioStartTime = presentationTimeNs;
    }

    mTotalInputAudioFrameCount++;
    audioBytesReceived += input.length;
    if (mEosSentToAudioEncoder && mStopReceived || input == null) {
        logStatistics();
        if (mEosReceived) {
            Log.d(TAG, "EOS received in offerAudioEncoder");
            closeAudioEncoder();
            mEosSentToAudioEncoder = true;
            if (!mStopReceived) {
                prepareAudioEncoder();
            } else {
                mEncodingService.shutdown();
            }
        }
        return;
    }
    // transfer previously encoded data to muxer
    drainEncoder(mAudioEncoder, mAudioBufferInfo, mAudioTrackIndex, false, "audio");
    sendFrameToEncoder(input, presentationTimeNs);// send current frame data to encoder
}

private void sendFrameToEncoder(byte[] input, long presentationTimeNs) {
    try {
        ByteBuffer[] inputBuffers = mAudioEncoder.getInputBuffers();
        int inputBufferIndex = mAudioEncoder.dequeueInputBuffer(-1);
        if (inputBufferIndex >= 0) {
            ByteBuffer inputBuffer = inputBuffers[inputBufferIndex];
            inputBuffer.clear();
            inputBuffer.put(input);
            if (mAudioSoftwarePoller != null) {
                mAudioSoftwarePoller.recycleInputBuffer(input);
            }
            long presentationTimeUs = (presentationTimeNs - mAudioStartTime) / 1000; // in microseconds
            if (mEosReceived) {
                mAudioEncoder.queueInputBuffer(inputBufferIndex, 0, input.length, presentationTimeUs, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
                closeAudioEncoder();
                mEosSentToAudioEncoder = true;
                if (mStopReceived) {
                    mEncodingService.shutdown();
                }
            } else {
                mAudioEncoder.queueInputBuffer(inputBufferIndex, 0, input.length, presentationTimeUs, 0);
            }
        }
    } catch (Throwable t) {
        t.printStackTrace();
    }
}

private void drainEncoder(MediaCodec encoder, MediaCodec.BufferInfo bufferInfo, TrackIndex trackIndex, boolean endOfStream, String type) {
    final int TIMEOUT_USEC = 100;
    ByteBuffer[] encoderOutputBuffers = encoder.getOutputBuffers();
    while (true) {
        int encoderStatus = encoder.dequeueOutputBuffer(bufferInfo, TIMEOUT_USEC);
        if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) {
            if (!endOfStream) {
                if (VERBOSE) Log.d(TAG, "INFO_TRY_AGAIN_LATER " + type + " out of while");
                break;      // out of while
            } else {
                if (VERBOSE) Log.d(TAG, "no " + type + " output available, spinning to await EOS");
            }
        } else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
            // not expected for an encoder
            if (VERBOSE) Log.d(TAG, "INFO_OUTPUT_BUFFERS_CHANGED " + type);
            encoderOutputBuffers = encoder.getOutputBuffers();

        } else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
            // should happen before receiving buffers, and should only happen once
            if (mMuxerStarted) {
                throw new RuntimeException("format changed after muxer start");
            }
            MediaFormat newFormat = encoder.getOutputFormat();
            Log.d(TAG, "encoder output format changed: " + newFormat + ".");

            // now that we have the Magic Goodies, start the muxer
            synchronized (mMuxer) {
                trackIndex.index = mMuxer.addTrack(newFormat);
                numTracksAdded++;
                Log.d(TAG, "Added " + type + " track index: " + trackIndex.index);
                if (numTracksAdded == TOTAL_NUM_TRACKS) {
                    mMuxer.start();
                    mMuxerStarted = true;
                    Log.d(TAG, numTracksAdded + " tracks added. Muxer started");
                    break;
                }
            }
        } else if (encoderStatus < 0) {
            Log.w(TAG, "unexpected result from " + type + " encoder.dequeueOutputBuffer: " +
                    encoderStatus);
        } else {
            if (encodedData == null) {
                if (VERBOSE) Log.d(TAG, "drainEncoder(" + endOfStream + ") " + type + " encodedData == null");
                throw new RuntimeException("encoderOutputBuffer " + encoderStatus +
                        " was null");
            }
            if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) {
                // The codec config data was pulled out and fed to the muxer when we got
                // the INFO_OUTPUT_FORMAT_CHANGED status.  Ignore it.
                if (VERBOSE) Log.d(TAG, "ignoring BUFFER_FLAG_CODEC_CONFIG");
                bufferInfo.size = 0;
            }
            if (bufferInfo.size != 0) {
                if (!mMuxerStarted) {
                    if (VERBOSE) Log.d(TAG, "drainEncoder(" + endOfStream + ") " + type + " Muxer not started");
                    throw new RuntimeException("muxer hasn't started");
                }
                // adjust the ByteBuffer values to match BufferInfo (not needed?)
                encodedData.position(bufferInfo.offset);
                encodedData.limit(bufferInfo.offset + bufferInfo.size);
                synchronized (mMuxer) {
                    mMuxer.writeSampleData(trackIndex.index, encodedData, bufferInfo);
                }
            }

            encoder.releaseOutputBuffer(encoderStatus, false);

            if ((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
                if (!endOfStream) {
                    Log.w(TAG, "reached end of stream unexpectedly");
                } else {
                    if (VERBOSE) Log.d(TAG, "end of stream reached");
                }
                break;
            }
        }
    }
    long endTime = System.nanoTime();
}

Answer 1

我认为您在漏水编码器功能中错过了ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];之前
如果(encodedData == null)