c++ - 消除光流算法中的噪声？

Question

我正在使用函数 cvCalcOpticalFlowPyrLK() 来检测和跟踪视频中的移动对象。我使用 Canny() 函数来跟踪特征。但结果并不好，它有很多来自 cvCalcOpticalFlowPyrLK() 的噪音。

请看下面我的代码:

#include <windows.h>
#include <stdio.h>
#include <vector>
#include <opencv/cv.h>
#include <opencv/highgui.h>
#include <opencv2/core/core.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/imgproc/imgproc.hpp>

using namespace std;
using namespace cv;

double Distance(Point& p1, Point& p2)
{
    return abs(p2.x - p1.x) + abs(p2.y - p1.y);
}

VOID EliminateBgrLines(vector<vector<Point>>* srcCountours, vector<vector<Point>>* dstCountours,
    CvPoint2D32f* frame1_features, CvPoint2D32f* frame2_features, char* optical_flow_found_feature)
{
    Point p, q;
    int featureIndex = 0;
    int numContours = srcCountours->size();
    int* removeContoursIndex = new int[numContours];
    int countRemoveContours = 0;

    vector<vector<Point>>::iterator Iter;
    int iterIndex = 0;
    for(Iter = srcCountours->begin() ; Iter != srcCountours->end() ; Iter++)
    {
        vector<Point> contour = *Iter;
        int numPoints = contour.size();
        int countBgrPoints = 0;
        int countTotalPointsFound = 0;
        double totalDistance = 0;

        for(int k = 0 ; k < numPoints; k++)
        {               
            if(optical_flow_found_feature[featureIndex] == 0)  
            {
                featureIndex ++;
                countBgrPoints ++; //The points that can not find in the next frame will be treated as background point
                continue;       
            }
            countTotalPointsFound ++;

            p. x = (int)frame1_features[featureIndex].x;
            p. y = (int)frame1_features[featureIndex].y;
            q. x = (int)frame2_features[featureIndex].x;
            q. y = (int)frame2_features[featureIndex].y;            
            featureIndex ++;

            double d = Distance(p, q);
            totalDistance += d;
            double opticalFlowThreshold = 1;
            if(d < opticalFlowThreshold)
                countBgrPoints ++;
        }

        int eliminateBgrLineThreshold = 40; //40 %
        if ( (double)countBgrPoints/numPoints > (eliminateBgrLineThreshold/100) ) //eliminateBgrLineThreshold(%) is bgr point
        {
            removeContoursIndex[countRemoveContours] = iterIndex;
            countRemoveContours ++;
        }
        iterIndex++;
    }       
    for(int i = 0 ; i < numContours; i++)
    {           
        if(trackingCoreGlobal->ISContainInArray(i, removeContoursIndex, countRemoveContours) == false)
            dstCountours->insert(dstCountours->end(), srcCountours->at(i));
    }
    delete []removeContoursIndex;       
}

int main()
{
    long current_frame = 0 ;
    CvSize frame_size;
    long number_of_frames = 0;
    CvCapture *input_video = NULL;

    input_video = cvCaptureFromFile(videoPath);
    if  (input_video == NULL)
        return false;
    cvQueryFrame(input_video);

    frame_size.height = (int)cvGetCaptureProperty(input_video, CV_CAP_PROP_FRAME_HEIGHT);
    frame_size.width = (int)cvGetCaptureProperty(input_video, CV_CAP_PROP_FRAME_WIDTH);

    /*  Go to the end of the AVI (The fraction is "1") */
    cvSetCaptureProperty( input_video, CV_CAP_PROP_POS_AVI_RATIO, 1.0) ;
    number_of_frames = (int)cvGetCaptureProperty( input_video, CV_CAP_PROP_POS_FRAMES) ;
    /*  Return to the beginning */
    cvSetCaptureProperty( input_video, CV_CAP_PROP_POS_FRAMES , 0.0) ;

    int number_of_features = 0;
    CvPoint2D32f* frame1_features = new CvPoint2D32f[NUMBER_OF_FEATURES];
    CvPoint2D32f* frame2_features = new CvPoint2D32f[NUMBER_OF_FEATURES];
    char* optical_flow_found_feature = new char[NUMBER_OF_FEATURES];
    float* optical_flow_feature_error = new float[NUMBER_OF_FEATURES];

    while(isStopped == false)
    {
        static  IplImage * frame  = NULL, * frame1_1C = NULL, * frame2_1C = 
            NULL, * eig_image  = NULL, * temp_image  = NULL, * pyramid1 = NULL, * pyramid2 = NULL;
        /*  Go to the frame we want.   Important if multiple frames are queried in
        * the loop which they of course are for optical flow.   Note that the very
        * first call to this is actually not needed . ( Because the correct position
        * is set outsite the for ()  loop.)
        */
        cvSetCaptureProperty( input_video, CV_CAP_PROP_POS_FRAMES, current_frame) ;
        /*  Get the next frame of the video.
        * IMPORTANT !   cvQueryFrame()  always returns a pointer to the _ same_
        * memory location.   So successive calls :
        * frame1 = cvQueryFrame();
        * frame2 = cvQueryFrame();
        * frame3 = cvQueryFrame();
        * will result in ( frame1 ==  frame2 &&  frame2 ==  frame3 ) being true.
        * The solution is to make a copy of the cvQueryFrame() output .
        */

        frame = cvQueryFrame(input_video);
        if(frame == NULL)
            return false;
        frame1_1C = cvCreateImage(frame_size, IPL_DEPTH_8U, 1);
        if(frame1_1C ==  NULL)
            return false;
        cvConvertImage(frame, frame1_1C);

        /* Get the second frame of video. Sample principles as the first */
        frame = cvQueryFrame(input_video);
        if(frame == NULL)
            return false;
        frame2_1C = cvCreateImage(frame_size, IPL_DEPTH_8U, 1);
        if(frame2_1C ==  NULL)
            return false;
        cvConvertImage(frame, frame2_1C);

        /*Preparation : Allocate the necessary storage*/
        eig_image = cvCreateImage(frame_size, IPL_DEPTH_32F, 1);
        if(eig_image ==  NULL)
            return false;
        temp_image = cvCreateImage(frame_size, IPL_DEPTH_32F, 1);
        if(temp_image ==  NULL)
            return false;

        number_of_features = 0;

        /*  Actually run the Shi and Tomasi algorithm !!
        * "frame1 _ 1 C " is the input image.
        * "eig _ image" and " temp_ image" are just workspace for the algorithm .
        * The first ".01" specifies the minimum quality of the features  ( based on the 
        eigenvalues ).
        * The second  ".01" specifies the minimum Euclidean distance between features .
        * "NULL" means use the entire input image.   You could point to a part of the 
        image.
        * WHEN THE ALGORITHM RETURNS:
        * "frame1 _ features" will contain the feature points.
        * "number _ of_ features" will be set to a value < = NUMBER_OF_FEATURES indicating the number of 
        feature points found.
        */

        //Get Features by using Canny Edges - Begin
        int ratio = 3;
        int kernel_size = 3;
        int cannyThreshold = 40;
        Mat detected_edges;
        Mat frame1_1C_Mat = cvarrToMat(frame1_1C);
        // Reduce noise with a kernel 3x3
        blur(frame1_1C_Mat, detected_edges, Size(3,3)) ;
        // Canny detector
        Canny(detected_edges, detected_edges, cannyThreshold, cannyThreshold*ratio, kernel_size);

        RNG rng(12345);
        vector<vector<Point>> contours;
        vector<Vec4i> hierarchy;
        findContours(detected_edges, contours, hierarchy, CV_RETR_TREE, CV_CHAIN_APPROX_SIMPLE, Point(0, 0));

        DrawContours(contours, "Before Removing Bgr Contours"); //This function will draw the result contours

        //Copy feature for tracking by optical flow
        int numContours = contours.size();
        for(int i = 0 ; i < numContours; i++)
        {
            vector<Point> contour = contours.at(i);
            int numPoints = contour.size();
            for(int k = 0 ; k < numPoints; k++)
            {
                Point point = contour.at(k);
                frame1_features[number_of_features].x = point.x;
                frame1_features[number_of_features].y = point.y;
                number_of_features++;
            }
        }
        /*
        This termination criteria tells the algorithm to stop when it has either done 
        20 iterations or when epsilon is better than 0.3, you can play with these parameters 
        for speed vs accuracy but these values work pretty well in many situations.
        */
        CvTermCriteria optical_flow_termination_criteria = cvTermCriteria(CV_TERMCRIT_ITER|CV_TERMCRIT_EPS, 20, 0.3) ;

        /*  This is some workspace for the algorithm.
        The algorithm actually carves the image into pyramids of different resolutions
        */
        pyramid1 = cvCreateImage(frame_size, IPL_DEPTH_8U, 1);
        if(pyramid1 ==  NULL)
            return false;
        pyramid2 = cvCreateImage(frame_size, IPL_DEPTH_8U, 1);
        if(pyramid2 ==  NULL)
            return false;

        /*Actually run Pyramidal Lucas Kanade Optical Flow!!
        * "frame1_1C" is the first frame with the known features.
        * "frame2_1C" is the second frame where we want to find the first frame's features.
        * "pyramid1" and "pyramid2" are workspace for the algorithm.
        * "frame1_features" are the features from the first frame.
        * "frame2_features" is the (outputted) locations of those features in the second frame.
        * "number_ of_features" is the number of features in the frame 1 _ features array .
        * "optical_flow_window" is the size of the window to use to avoid the aperture problem.
        * "5" is the maximum number of pyramids to use. 0 would be just one level.
        * "optical_flow_found_feature" is as described above (non-zero iff feature found by the flow).
        * "optical_flow_feature_error" is as described above (error in the flow for this feature).
        * "optical_flow_termination_criteria" is as described above (how long the algorithm should look).
        * "0" means disable enhancements. (For example , the second array isn't pre-initialized with guesses)
        */      

        /*This is the window size to use to avoid the aperture problem (see slide "Optical Flow: Overview")*/
        int opticalFlowWindowSize = 21;
        int pyramidLevel = 5;
        CvSize optical_flow_window = cvSize(opticalFlowWindowSize, opticalFlowWindowSize);      
        cvCalcOpticalFlowPyrLK(frame1_1C, frame2_1C, pyramid1, pyramid2, frame1_features, 
            frame2_features, number_of_features, optical_flow_window, pyramidLevel, 
            optical_flow_found_feature, optical_flow_feature_error, 
            optical_flow_termination_criteria, 0) ;     

        vector<vector<Point>> afterRemoveBgrItemContours;
        EliminateBgrLines(&contours, &afterRemoveBgrItemContours, frame1_features, frame2_features, optical_flow_found_feature);

        DrawContours(afterRemoveBgrItemContours, "After Removing Bgr Contours"); //This function will draw the result contours

        current_frame++;
        if (current_frame >= number_of_frames - 1)  
            current_frame = 0;

        detected_edges.release();
        frame1_1C_Mat.release();
        detected_edges.release();
        cvReleaseImage(&frame1_1C);
        cvReleaseImage(&frame2_1C);
        cvReleaseImage(&eig_image);
        cvReleaseImage(&temp_image);
        cvReleaseImage(&pyramid1);
        cvReleaseImage(&pyramid2);      
    } //End while(true)

    delete []frame1_features;
    delete []frame2_features;
    delete []optical_flow_found_feature;
    delete []optical_flow_feature_error;
}

我试过很多阈值，但都不能很好地消除噪声。

这是我尝试过的一些结果:

我尝试过许多其他阈值，但它仍然有很多噪音，有人可以告诉我改进算法以获得更好结果的方法吗？

非常感谢，

大同

Answer 1

Canny edges 不能为 Optical Flow 提供很好的特征。你应该试试名字很好的 goodFeaturesToTrack()相反。