c++ - 如何使用 PortAudio 和 OpenCV 避免不一致的音频播放？

Question

我使用 opencv(用于对象识别)结合 portaudio 来播放基于视频输入的声音。本质上，我的目标是以不同的速率播放特定音高/频率的正弦波音调。它有效，但结果非常不可预测。有时音频播放工作(程序运行缓慢，但它工作)，其他时候没有音频播放发生。简而言之/流程这就是我的程序所做的:

启动网络摄像头源 -> 获取网络摄像头图像 -> 选择图像中的区域 -> 返回视频源 -> while(frame exists) -> 跟踪对象位置 -> 初始化端口音频工具 -> 根据位置播放声音 ->终止 Portaudio 工具

我似乎无法弄清楚为什么音频播放不一致。大家有什么窍门吗？我一直在阅读，我的想法是这是一个延迟问题，但我真的没有这方面的经验。当我在没有 opencv 的情况下使用 portaudio 时，不会出现延迟问题，所以我知道这与结合两者有关。感谢您的帮助。

while (frame)
{
    cvCopyImage(frame, drawImg);

    // process
    track(frame);

    // get result
    CvRect r;
    float  confidence;
    bool   valid;
    /* getRoi tells us if the region being tracked on the screen
     * is the same region that we chose prior to entering this while loop
     */
    getRoi(&r, &confidence, &valid); 

    // show
    cvDrawRect(drawImg, cvPoint(r.x, r.y), 
        cvPoint(r.x + r.width - 1, r.y + r.height - 1),
        valid ? cvScalar(0, 255, 0) : cvScalar(0, 255, 255),
        2
    );
    writeLogo(drawImg,"USC-IRIS");
    int xpos = r.x;
    int ypos = r.y;



    cvShowImage("Tracking", drawImg);
    cout << "valid " << valid << endl;
    cout << "conf val " << confidence << endl;
    cout << "xpos, ypos " << xpos << ", " << ypos << endl;
            //If the region on the screen is the region we chose
            //then we should play specific sounds
    if(valid){

        sI->soundWrite(xpos, ypos);
        float freq = sI->getFreq();
        int amp = sI->getAmp();
        float pulse = sI->getPulse();

        switch(amp){
            case 0:
                //printf("Hear sound in both ears.\n");
                data.targetBalance = .5;
                break;
            case 1:
                //printf("Hear sound in left ear.\n");
                data.targetBalance = 0;
                break;
            case 2:
                //printf("Hear sound in right ear.\n");
                data.targetBalance = 1;
                break;
            default:
                //printf("Incorrect value for amp (left/right sound indicator)");
                data.targetBalance = .5;
                break;
        }



        err = Pa_Initialize(); //scan for available devices i.e. audio jack, headphones
        if(err != paNoError) {
            printf("init\n");
            goto error;
        }
        //open the sound stream for processing
        err =  Pa_OpenDefaultStream( &stream, 0, 2, paFloat32, SAMPLE_RATE, 
            256, patestCallback, &data ); //open the sound stream for processing
        if( err != paNoError ) {
            printf("open\n");
            goto error;
        }

        //start the stream (i.e. play sound) if no errors
        err = Pa_StartStream(stream);
        if(err != paNoError) {
            printf("start\n");
            goto error;
        }

        //check which ear(s) the sound should be played to



        //hold that tone for a certain amount of time (pulse*200 millisec)
        Pa_Sleep(pulse*200);
        cout << "pulse: " << pulse <<  endl << "freq: " << freq << endl;
        cout << "amp: " << amp << endl;

        //stop the stream (i.e. stop playing sound)
        err = Pa_StopStream(stream);
        if(err != paNoError) {
            printf("stop\n");
            goto error;
        }

        err = Pa_CloseStream( stream );
        if( err != paNoError ) {
            printf("close\n");
            goto error;
        }

        err = Pa_Terminate();
        if( err != paNoError ) {
            printf("term\n");
            goto error;
        }
    }
    int key = cvWaitKey(1);
    // write
    if (output_txt)
        fprintf(output_txt, "%d %d %d %d\n", r.x, r.y, r.width, r.height);
    if (output_avi)
        cvWriteFrame(output_avi, drawImg);

    // next
    if (key == 'q'||key=='Q')
        break;
    frame = cvQueryFrame(capture);
}

Answer 1

看来，音频播放不一致是由于另一段代码没有显示在我上面的问题中。下面是错误的代码。我认为该错误与此函数中的第一个 if 语句和最后一个 forloop 有关。我认为变量 framesToCalc 没有被正确计算。因此，第一个 for 循环没有将任何数据放入 outputBuffer/out 变量。然后，最后我将剩余未使用的缓冲区空间归零。因此，由于缓冲区归零而没有声音。我的解决方案是删除第一个 if else 和最后一个 forloop。此外，我执行了第一个从 i=0 到 framesPerBuffer 的 for 循环。现在它完美地工作了。

static int patestCallback(const void *inputBuffer, void *outputBuffer, unsigned long framesPerBuffer, const PaStreamCallbackTimeInfo *timeInfo, PaStreamCallbackFlags statusFlags, void *userData){
paTestData *data = (paTestData*)userData;
SAMPLE_t *out = (SAMPLE_t *)outputBuffer;
int i;
int framesToCalc;
int finished = 0;
(void) inputBuffer; 
int left_phase = data->left_phase;
int right_phase = data->right_phase;


if( data->framesToGo < framesPerBuffer )
{
    framesToCalc = data->framesToGo;
    data->framesToGo = 0;
    finished = 1;
}
else
{
    framesToCalc = framesPerBuffer;
    data->framesToGo -= framesPerBuffer;
}

for( i=0; i<framesToCalc; i++ )
{
    if( data->currentBalance < data->targetBalance )
    {
        data->currentBalance += BALANCE_DELTA;
    }
    else if( data->currentBalance > data->targetBalance )
    {
        data->currentBalance -= BALANCE_DELTA;
    }
    left_phase += (LEFT_FREQ / SAMPLE_RATE);
    right_phase += (RIGHT_FREQ / SAMPLE_RATE);
    if( fabs(data->currentBalance - .5)  < .001){
        //left_phase += (double)(LEFT_FREQ / SAMPLE_RATE);
        if( left_phase > 1.0) left_phase -= 1.0;

        *out++ = DOUBLE_TO_SAMPLE( AMPLITUDE * sin( (left_phase * M_PI * 2. )));

        //right_phase += (double)(RIGHT_FREQ / SAMPLE_RATE);
        if( right_phase > 1.0) right_phase -= 1.0;
        *out++ = DOUBLE_TO_SAMPLE( AMPLITUDE * sin( (right_phase * M_PI * 2. )));
    }else{
        //left_phase += (double)(LEFT_FREQ / SAMPLE_RATE);
        if( left_phase > 1.0) left_phase -= 1.0;

        *out++ = DOUBLE_TO_SAMPLE( AMPLITUDE * sin( (left_phase * M_PI * 2. ))*(1.0 - data->currentBalance));

        //right_phase += (double)(RIGHT_FREQ / SAMPLE_RATE);
        if( right_phase > 1.0) right_phase -= 1.0;
        *out++ = DOUBLE_TO_SAMPLE( AMPLITUDE * sin( (right_phase * M_PI * 2. ))*data->currentBalance);
    }

}
    // zero remainder of final buffer
    for( ; i<(int)framesPerBuffer; i++ )
    {
        *out++ = SAMPLE_ZERO; //left
        *out++ = SAMPLE_ZERO; //right
    }
    data->left_phase = left_phase;
    data->right_phase = right_phase;
    return finished;
}