javascript - 移相器 3 : Shader with microphone input (Web)

Question

我想用 Phaser 3 做一个音频输入可视化器，我想把麦克风输入到着色器，但我找不到让它工作的方法。

我对着色器有基本的了解，我可以处理图像纹理，但我真的不知道如何提供声音。我检查了一个用 three.js 制作的工作示例:three.js webaudio - visualizer并且我已经设法从麦克风获取声音输入作为 1024 个数字的 Uint8Array。

这是我正在使用的着色器:

// simplesound.gsl.js
#ifdef GL_ES
precision highp float;
#endif

precision mediump float;
uniform vec2 resolution;
uniform sampler2D iChannel0;

varying vec2 fragCoord;

void main() {
  vec2 uv = fragCoord.xy / resolution.xy;
  vec2 mu = texture2D(iChannel0, uv).rg;

  float y = uv.y - mu.x;
  y = smoothstep(0., 0.02, abs(y - 0.1));

  gl_FragColor = vec4(y);
}

这是我的场景代码，试图让它工作:

import Phaser from 'phaser';
// This will provide the array mentioned above with code that will use `navigator.getUserMedia`.
import { setupAudioContext } from '../audiostream';

export default class MainScene2 extends Phaser.Scene {
  constructor() {
    super({ key: 'MainScene2' });
  }

  preload() {
    this.load.glsl('simplesound', '/static/simplesound.glsl.js');
  }

  create() {
    this.shader = this.add.shader('simplesound', 400, 300, 800, 600);

    // When the user presses the 'g' key we will start listening for mic input
    const GKey = this.input.keyboard.addKey('G');

    GKey.on('down', () => {
      setupAudioContext((array) => {
        // this array is the array mentioned above, in the three.js example they do something like creating
        // a texture from this input and providing that texture to the shader uniform. I tried different things but
        // nothing worked :(
        //
        // I tried using this.shader.setChannel0 and this.shader.setUniform but nothing seems to work as well.
      });
    });
  }
}

我一直在努力让这个工作有一段时间了，但一无所获:(

Answer 1

对于没有着色器的可能解决方案，仅使用相位器和 javascript 可能看起来像这样(真的没有着色器，但我也很感兴趣着色器版本的外观)。

在此演示中，我使用的是音频文件中的数据。 为了使其适用于您的用例，您只需将麦克风数据插入 data 变量即可。

演示:
(代码中的注释，是为了突出主要思想)
单击并等待几秒钟。 顺便说一句:我添加了一些屏幕抖动，使演示更加生动。

document.body.style = 'margin:0;';
    
    var data = [];
    var playing =  -1;
    var audioContext = new (window.AudioContext || window.webkitAudioContext)();
    var analyser = audioContext.createAnalyser();
    var buffer;
    var source;
    var url = 'https://labs.phaser.io/assets/audio/Rossini - William Tell Overture (8 Bits Version)/left.ogg'

    // START Audio part for Demo
    function loadAudio() {
        var request = new XMLHttpRequest();
        request.open('GET', url, true);
        request.responseType = 'arraybuffer';
        request.onload = function() {
            audioContext.decodeAudioData(request.response, function(buf) {
                buffer = buf;
                playAudio();
            });
        };
        request.send();
    }
    
    function playAudio() {
        source = audioContext.createBufferSource();
        source.buffer = buffer;
        source.connect(audioContext.destination);
        source.connect(analyser);
        source.start(0);
    }

    // END Audio part for Demo

    var config = {
        type: Phaser.AUTO,
        width: 536,
        height: 183,
        scene: {
            create,
            update
        },
        banner: false
    }; 

    var game = new Phaser.Game(config);

    // this would be the varibale that should be updated from the audio source
    var markers;
    var createRandomData = true;

    function create () {
        // Start create Marker texture 
        // this could be remove if you want to load an actual image
        let g = this.make.graphics({x: 0, y: 0, add: false});
        
        g.lineStyle(10, 0xffffff);
        g.beginPath();
        g.moveTo(0, 0);
        g.lineTo(50, 0);

        g.strokePath();

        g.generateTexture('marker', 30, 10);
        // End create Marker texture 
    
        // Create the markers
        // the repeat property sets how many markers you want to display, if you want all 1024 => that would be your value
        markers = this.add.group({ key: 'marker', repeat: 50,
            setXY: { x: 10, y: 10, stepX: 35 }, setOrigin: { x: 0, y: 0}});
        
        this.add.rectangle(10, 10, 180, 20, 0).setOrigin(0);
        let label = this.add.text( 10, 10, 'Click to start music', {color: 'red', fontSize:'20px', fontStyle:'bold'} )
        
        // start and stop the playback of music     
        this.input.on('pointerdown', function () {
            switch (playing) {
                case -1:
                    loadAudio();
                    playing = 1;
                    label.setText('Click to stop music');
                    break;
                case 0:
                    playAudio();
                    playing = 1;
                    label.setText('Click to stop music');
                    break;
                case 1:
                    source.stop();
                    playing = 0;
                    label.setText('Click to start music');
                    break;
            }   
        }); 

    }

    function update(){
        if (markers){
            // here we update the y-position of the marker in depending on the value of the data. 
            // ( min y = 10 and max y ~ 245)
            markers.children.iterate(function (child, idx) {
                child.y = 10 + (config.height - 20) / 255 * data[idx];
                
                // you could even add some camera shake, for more effect
                if(idx < 3 && data[idx] > 253){
                    this.cameras.main.shake(30);
                }
            }, this);
            
            // if the analyser is valid and updates the data variable
            // this part could some where else, I just wanted to keep the code concise
            if(analyser){
              var spectrums = new Uint8Array(analyser.frequencyBinCount);
              analyser.getByteFrequencyData(spectrums);

              // convert data to a plain array and updating the data variable
              data = [].slice.call(spectrums);
            }
        }
    }

<script src="https://cdn.jsdelivr.net/npm/phaser@3.55.2/dist/phaser.js"></script>

Basically this application "only" alters the Y-positions of each marker, based on the value returned from the audio array, which is loaded from the audio file.
Disclaimer: this is rough demo code, and could use some cleanup/improvement, if it should be used in production.