javascript - 在浏览器中将 Canvas 转换为音频以进行数据传输

Question

我想在浏览器中进行一些数据移动，并希望将 ImageData 从 Canvas 加载到 AudioData 中，因此我可以使用一些音频库(如金枪鱼)在数据上添加效果。

这是我迄今为止尝试过的

const canvasToBlob = (canvas) => {
  let dataUrl = canvas.toDataURL('image/png');
  return dataURItoBlob(dataUrl);
};

const dataURItoBlob = (dataURI) => {
  // convert base64 to raw binary data held in a string
  // doesn't handle URLEncoded DataURIs - see SO answer #6850276 for code that does this
  let byteString = atob(dataURI.split(',')[1]);

  // separate out the mime component
  let mime = dataURI.split(',')[0].split(':')[1].split(';')[0];
  console.log(mime);

  // write the bytes of the string to an ArrayBuffer
  let ab = new ArrayBuffer(byteString.length);
  let ia = new Uint8Array(ab);
  for (let i = 0; i < byteString.length; i++) {
    ia[i] = byteString.charCodeAt(i);
  };

  // write the ArrayBuffer to a blob, and you're done
  let blob = new Blob([ab], {type: 'audio/wav'});
  return blob;
};

export default function audioConvert (canvas) {
  const imageBlob = canvasToBlob(canvas);
  let audioCtx = new AudioContext();
  let fileReader = new FileReader();

  fileReader.onloadend = () => {
    console.log(fileReader.result);
    audioCtx.decodeAudioData(fileReader.result).then((decodedData) => {
      console.log(decodedData);
    });
  };
  fileReader.readAsArrayBuffer(imageBlob);
};

不幸的是我只收到错误

Uncaught (in promise) DOMException: Unable to decode audio data

如何更改代码来获取 AudioContext。

或者 AudioContext 是正确的方法吗？

我想添加到图像中的一些效果是“回声”或“哇哇”，如 f.e. 所见。在大胆。

Answer 1

图像到 Canvas 到音频上下文声音缓冲区示例

还没有答案，所以我刚刚添加了一个非常快速的加载图像并转换为声音缓冲区并播放。

// Stereo
var log = (data)=>console.log(data);
var audioCtx = new AudioContext(); // get audio context
var pixelPlayTime,sBuffers,ctx,can,startTime,source;  // sBuffers is the wav form array
// load image
var image = new Image;
image.setAttribute('crossOrigin', 'anonymous');
image.src = "https://upload.wikimedia.org/wikipedia/commons/5/5a/Hoverfly07.jpg";

// when loaded convert to wave
image.onload = function(){
    displayit.innerHTML = "";

    // create a canvas to read pixel data
    can = document.createElement("canvas");
    can.width = this.width;
    can.height = this.height;
    ctx = can.getContext("2d");
    ctx.drawImage(this,0,0);
    var data = ctx.getImageData(0,0,this.width,this.height).data;

    // get image size and workout sound buffer sizes
    var pixelCount = this.width * this.height;
    pixelPlayTime = pixelCount / audioCtx.sampleRate;
    sBuffers = audioCtx.createBuffer(2, pixelCount, audioCtx.sampleRate);

    // left gets red full vol and blue half
    var channel1 = sBuffers.getChannelData(0); // get channel 0
    var i = 0;
    while(i < pixelCount){
        // get red and some of green for this channel
        var sample = ((data[i*4]/128) - 1 + (data[i*4+2]/255) - 0.5) / 1.5;
        channel1[i] = sample < -1 ? -1 : sample > 1 ? 1 : sample;
        i ++
    }

    // Right gets green full vol and blue half
    var channel2 = sBuffers.getChannelData(1); // get channel 1
    i = 0;
    while(i < pixelCount){
        // get blue and some of green for this channel
        var sample = ((data[i*4 + 1]/128) - 1 + (data[i*4+2]/255) - 0.5) / 1.5;
        channel2[i] = sample < -1 ? -1 : sample > 1 ? 1 : sample;
        i ++
    }

    // sound buffers created prep to play
    source = audioCtx.createBufferSource(); // for playing the buffer
    source.buffer = sBuffers;              // point to the buffer
    source.connect(audioCtx.destination);       // connect

    // show image as canvas
    can.style.width = Math.floor(this.width / 4) + "px"
    can.style.height = Math.floor(this.height / 4) + "px"
    displayit.innerHTML = "Click image to hear it<br>";
    displayit.appendChild(can);

    // Wait for clien click
    can.addEventListener("click",()=>{
      requestAnimationFrame(startSounds);
     });

}
// start playing the sount
function startSounds(time){
    if(startTime === undefined){
      startTime = time;
      pixelPlayTime *= 1000; // convert from seconds to micro seconds
      source.start();                             // and play dat phunky fly
      requestAnimationFrame(update);
      ctx.fillStyle = "white";
    }
}

// display where the sound data is on the image.
// Basic timing use to show at what line the sound is being generated from.
function update(time){
    var pos = (time - startTime) / pixelPlayTime;
    var posY = Math.floor(pos * can.height);
    ctx.globalCompositeOperation = "lighter";
    ctx.fillRect(0,posY,can.width,4);
    ctx.globalCompositeOperation = "source-over";
    ctx.drawImage(image,0,posY-4,can.width,4,0,posY-4,can.width,4);
    if(pos < 1){
        requestAnimationFrame(update);
    }
}

<div id="displayit" style="font-family:arial"><h2>Please wait loading image</h2></div>

<div style="font-family:arial">
<b><a href="https://en.wikipedia.org/wiki/User:Fir0002"  title="en:User:Fir0002">fir0002</a> | <a  href="http://www.flagstaffotos.com.au">flagstaffotos.com.au</a></b>



Image attribution
<table>
<tbody><tr>
<td style="width:90px;"><img alt="" src="https://upload.wikimedia.org/wikipedia/commons/thumb/7/79/CC_some_rights_reserved.svg/90px-CC_some_rights_reserved.svg.png" width="90" height="36" srcset="https://upload.wikimedia.org/wikipedia/commons/thumb/7/79/CC_some_rights_reserved.svg/135px-CC_some_rights_reserved.svg.png 1.5x, https://upload.wikimedia.org/wikipedia/commons/thumb/7/79/CC_some_rights_reserved.svg/180px-CC_some_rights_reserved.svg.png 2x" data-file-width="744" data-file-height="300"></td>
<td><span xml:lang="en"><i>This file is published under the following <a href="https://en.wikipedia.org/wiki/Creative_Commons" title="w:Creative Commons">Creative Commons</a> license:<br>
<a href="//creativecommons.org/licenses/by-nc/3.0/">Attribution <b>NonCommercial</b> Unported 3.0</a></i></span></td>
</tr>
</tbody></table>
<div style="font-family:arial">