javascript - 如何在 Canvas 上没有新绘图的情况下录制 Canvas + 音频

Question

我想录制一段在 HTML Canvas 上运行的脚本的视频，同时录制来自麦克风的音频。我正在使用 canvas.captureStream() 从 Canvas 创建视频流。然后，我从 getUserMedia() 创建一个音频流，并将两个流混合在一个流中，然后传递给一个新的 MediaRecorder。

问题是，默认情况下，如果没有在 Canvas 上绘制任何内容， Canvas 流中将不会捕获任何帧。因此，在 Canvas 不活动的部分，视频和音频不同步。

我使用的代码如下所示:

let recording = false;
let mediaRecorder;
let chunks = [];

let newStream, canvasStream


let constrains = {
  audio: {
    channelCount: { ideal: 2, min: 1 },
    sampleRate: 48000,
    sampleSize: 16,
    volume: 1,
    echoCancellation: true,
    noiseSuppression: true,
    autoGainControl: true,
  },
  video: false
};

recButton.addEventListener("click", () => {
  recording = !recording;
  if (recording) {
    recButton.textContent = "Click to Stop";
    recButton.style.color = "red";
    canvasStream = canvas.captureStream(40); // ---> captureStream()


    navigator.mediaDevices.getUserMedia(constrains) // ---> audioStream
      .then(audioStream => {
     
        canvasStream.addTrack(audioStream.getTracks()[0]);   // --> joint the two streams
    

        mediaRecorder = new MediaRecorder(canvasStream, {
          mimeType: 'video/webm; codecs=vp9',
          // ignoreMutedMedia: false
        });

        mediaRecorder.ondataavailable = e => {
          if (e.data.size > 0) {
            chunks.push(e.data);  // chunks = [];   already defined above....
          }
        };
        mediaRecorder.start();
      })
      .catch(err => {
        console.log(err.name, err.message);
      });
  } else {
    recButton.textContent = "Record";
    recButton.style.color = "black";
    mediaRecorder.stop()

    stopTracks(canvasStream);  // function stop defined below
    console.log(mediaRecorder.state)
    setTimeout(() => {                // why this is necessary?
      const blob = new Blob(chunks, {
        type: "video/mp4"
      });
      const url = URL.createObjectURL(blob);
      const a = document.createElement("a");
      a.href = url;
      a.download = "recording.mp4";
      a.click();
      URL.revokeObjectURL(url);
    }, 200);
  }
});

let stopTracks = stream => stream.getTracks().forEach(track => track.stop());

Answer 1

这有点像 Chrome 错误，因为即使视频轨道在这种情况下确实应该没有产生数据，但音频轨道有并且 MediaRecorder 应该捕获它。

解决此问题的唯一方法是至少在与调用 canvas.captureStream() 相同的绘图帧中在 Canvas 上执行新绘图。
对于 2D 上下文，较少干扰的方法是绘制一个透明的 1x1 矩形。

let recording = false;
let mediaRecorder;
const chunks = [];

let newStream, canvasStream

const constrains = {
  audio: {
    channelCount: { ideal: 2, min: 1 },
    sampleRate: 48000,
    sampleSize: 16,
    volume: 1,
    echoCancellation: true,
    noiseSuppression: true,
    autoGainControl: true,
  },
  video: false
};

recButton.addEventListener("click", () => {
  recording = !recording;
  if (recording) {
    recButton.textContent = "Click to Stop";
    recButton.style.color = "red";
    canvasStream = canvas.captureStream(40);
    // force activate the Canvas track
    forceEmptyDrawing(canvas);
    navigator.mediaDevices.getUserMedia(constrains) // ---> audioStream
      .then(audioStream => {

        canvasStream.addTrack(audioStream.getTracks()[0]); // --> joint the two streams
        mediaRecorder = new MediaRecorder(canvasStream, {
          // don't forget the audio codec
          mimeType: "video/webm; codecs=vp9,opus",
          // ignoreMutedMedia: false
        });

        mediaRecorder.ondataavailable = e => {
          if (e.data.size > 0) {
            chunks.push(e.data);
          }
        };
        mediaRecorder.start(10);
      })
      .catch(err => {
        console.log(err.name, err.message);
      });
  } else {
    recButton.textContent = "Record";
    recButton.style.color = "black";
    mediaRecorder.stop()
    // wait for the stop event
    // don't rely on some magic timeout
    mediaRecorder.onstop = () => {
      // only when the recorder is completely stopped
      // you can stop the tracks
      stopTracks(canvasStream);
      const blob = new Blob(chunks, {
        type: "video/webm" // you asked for webm, not mp4
      });
      // just to verify we did record something
      console.log(`recorded a ${ blob.size }b file`);
    };
  }
});

const stopTracks = stream => stream.getTracks().forEach(track => track.stop());

// draws a 1x1 transparent rectangle
function forceEmptyDrawing(canvas) {
  const ctx = canvas.getContext("2d");
  const alpha = ctx.globalAlpha;
  ctx.globalAlpha = 0;
  ctx.fillRect(0, 0, 1, 1);
  ctx.globalAlpha = alpha;
}

As a fiddle因为 StackSnippets 不能使用 gUM(仅适用于 Chrome，Firefox 和 Safari 不支持提供的 mimeType)。