python - 计算图像片段(超像素)的特征向量

Question

对于图像聚类或分类等任务，我们通常将图像转换为数字特征向量。现在，我不想为整个图像计算特征向量，而是想为图像的片段生成特征(不限于矩形片段)。例如，使用 SLIC 算法 ( skimage.segmentation.slic ) 我可以将图像分割成超像素。现在我想为每个段生成特征(区域大小、位置、颜色、形状和纹理特征)，如第 5.3 节所述

Gould, Stephen, et al. "Multi-class segmentation with relative location prior." International Journal of Computer Vision 80.3 (2008): 300-316.

在给定图像和片段掩码的情况下，python 中是否存在可以帮助我生成这些特征的现有库？我可以使用 skimage 执行此操作吗？

Answer 1

我不知道有任何这样的库。但是，前段时间我需要自己计算特征，您可以在下面找到一些代码片段。尽管代码不是 Python 语言，但它可能对您有所帮助。请注意，我试验过超体素；因此，您可能会在其中找到一些 PCL 引用。

如果您开始自己实现功能，请查看以下出版物以获取一些想法(两种情况都在表 1 中):

Derek Hoiem, Andrew N. Stein, Alexei A. Efros, Martial Hebert:
Recovering Occlusion Boundaries from a Single Image. ICCV 2007: 1-8
Joseph Tighe, Svetlana Lazebnik:
Superparsing - Scalable Nonparametric Image Parsing with Superpixels. International Journal of Computer Vision 101(2): 329-349 (2013)

请注意，并非头文件中的所有定义都已实际实现；然而，它们可以作为灵感。

标题:

#ifndef SUPERPIXELFEATURES_H
#define SUPERPIXELFEATURES_H

#include <opencv2/opencv.hpp>
#include <pcl/point_cloud.h>
#include <pcl/point_types.h>
#include <Eigen/Dense>
#include <string>

namespace features {

    /**
     * Class SuperpixelFeatures represents a set of features computed for
     * each superpixel in a given image.
     */
    class SuperpixelFeatures {

    public:

        /**
         * Construct superpixel features form only an image.
         * 
         * @param image
         * @param labels
         */
        SuperpixelFeatures(const cv::Mat &image, int** labels);

        /**
         * Construct superpixel features from the image and its depth and
         * a given superpixel segmentation.
         * 
         * @param image
         * @param depth
         * @param labels
         */
        SuperpixelFeatures(const cv::Mat &image, const cv::Mat &depth, int** labels);

        /**
         * Constructu superpixel features form the image and a point cloud and
         * a given superpixel segmentation.
         * 
         * @param image
         * @param pointCloud
         */
        SuperpixelFeatures(const cv::Mat &image, pcl::PointCloud<pcl::PointXYZ>::Ptr pointCloud, int** labels);

        /**
         * Destructor.
         */
        ~SuperpixelFeatures();

        /**
         * Add maximum color in each channel to the features.
         * 
         * @return 
         */
        Eigen::Vector2i addMaximumColor();

        /**
         * Add minimum color in each channel to the features.
         * 
         * @return 
         */
        Eigen::Vector2i addMinimumColor();

        /**
         * Add mean color to the features.
         * 
         * @return 
         */
        Eigen::Vector2i addMeanBGRColor();

        /**
         * Add mean position to the features.
         * 
         * @return 
         */
        Eigen::Vector2i addMean3DPosition();

        /**
         * Add mean position (pixel coordinates) to the features.
         * 
         * @return 
         */
        Eigen::Vector2i addMean2DPosition();

        /**
         * Add the surface normal (mean normal) to the features.
         * 
         * @return 
         */
        Eigen::Vector2i addMeanNormal();

        /**
         * Add a 3D bounding box of the superpixel to the features.
         * 
         * @return 
         */
        Eigen::Vector2i addBoundingBox();

        /**
         * Add the compactness of the superpixel in its 2D sens to the features.
         * 
         * @return 
         */
        Eigen::Vector2i addCompactness();

        /**
         * Add the area in pixels to the features.
         * 
         * @return 
         */
        Eigen::Vector2i addArea();

        /**
         * Add the color covariance matrix to the features.
         * 
         * @return 
         */
        Eigen::Vector2i addColorCovariance();

        /**
         * Add the position covariance matrix to the features.
         * @return 
         */
        Eigen::Vector2i addPositionCovariance();

        /**
         * Add point-ness, curve-ness and surface-ness to the features.
         * 
         * @return 
         */
        Eigen::Vector2i addSuperpixelStatistics();

        /**
         * Add a color histogram of the given number of bins to the features.
         * 
         * @param bins
         * @return 
         */
        Eigen::Vector2i addColorHistogram(int bins);

        /**
         * Add the ground truth label to the features.
         * 
         * @param labels
         * @return 
         */
        Eigen::Vector2i addGroundTruth(int** labels);

        /**
         * Get the dimension of the computed features.
         * 
         * @return
         */
        int getFeatureDimension() const;

        /**
         * Get the total number of superpixels.
         * 
         * @return 
         */
        int getNumberOfSuperpixels() const;

        /**
         * Get pointer to comptued features.
         * 
         * @return
         */
        Eigen::MatrixXd* getFeatures() const;

    protected:

        void appendFeatures(Eigen::MatrixXd features);

        cv::Mat* image;
        int height;
        int width;

        int** labels;
        int numberOfSuperpixels;

        pcl::PointCloud<pcl::PointXYZ>::Ptr pointCloud;
        bool pointCloudAvailable;

        Eigen::MatrixXd* features;

    };
}

来源:

#include <pcl/features/normal_3d.h>
#include <pcl/features/integral_image_normal.h>
#include "Tools.h"
#include "SuperpixelFeatures.h"

SuperpixelFeatures::SuperpixelFeatures(const cv::Mat &image, int** labels) {

    this->image = new cv::Mat();
    int channels = image.channels();

    assert(channels == 1 || channels == 3);

    if (channels == 1) {
        image.convertTo(*this->image, CV_8UC1);
    }
    else if (channels == 3) {
        image.convertTo(*this->image, CV_8UC3);
        cv::cvtColor(*this->image, *this->image, SEEDS_REVISED_OPENCV_BGR2Lab, 3);
    }

    this->height = image.rows;
    this->width = image.cols;

    this->pointCloudAvailable = false;

    // Copy labels.
    this->labels = new int*[this->height];
    for (int i = 0; i < this->height; ++i) {
        this->labels[i] = new int[this->width];

        for (int j = 0; j < this->width; ++j) {
            this->labels[i][j] = labels[i][j];
        }
    }

    this->numberOfSuperpixels = seeds_revised::tools::Integrity::countSuperpixels(this->labels, this->height, this->width);
    seeds_revised::tools::Integrity::relabel(this->labels, this->height, this->width);

    this->features = new Eigen::MatrixXd(this->numberOfSuperpixels, 1);

    // Initialize first column with labels.
    for (int label = 0; label < this->numberOfSuperpixels; ++label) {
        (*this->features)(label, 0) = label;
    }
}

SuperpixelFeatures::SuperpixelFeatures(const cv::Mat &image, pcl::PointCloud<pcl::PointXYZ>::Ptr pointCloud, int** labels) {
    assert(image.rows == (int) pointCloud->height);
    assert(image.cols == (int) pointCloud->width);

    this->image = new cv::Mat();
    int channels = image.channels();

    assert(channels == 1 || channels == 3);

    if (channels == 1) {
        image.convertTo(*this->image, CV_8UC1);
    }
    else if (channels == 3) {
        image.convertTo(*this->image, CV_8UC3);
        cv::cvtColor(*this->image, *this->image, SEEDS_REVISED_OPENCV_BGR2Lab, 3);
    }

    this->pointCloud = pointCloud;
    this->height = pointCloud->height;
    this->width = pointCloud->width;
    this->pointCloudAvailable = true;

    // Copy labels.
    this->labels = new int*[this->height];
    for (int i = 0; i < this->height; ++i) {
        this->labels[i] = new int[this->width];

        for (int j = 0; j < this->width; ++j) {
            this->labels[i][j] = labels[i][j];
        }
    }

    this->numberOfSuperpixels = seeds_revised::tools::Integrity::countSuperpixels(this->labels, this->height, this->width);
    seeds_revised::tools::Integrity::relabel(this->labels, this->height, this->width);

    this->features = new Eigen::MatrixXd(this->numberOfSuperpixels, 1);

    // Initialize first column with labels.
    for (int label = 0; label < this->numberOfSuperpixels; ++label) {
        (*this->features)(label, 0) = label;
    }
}

SuperpixelFeatures::~SuperpixelFeatures() {
    delete this->image;

    for (int i = 0; i < this->height; ++i) {
        delete[] this->labels[i];
    }

    delete[] this->labels;
}

Eigen::Vector2i SuperpixelFeatures::addMeanBGRColor() {
    int cols = this->features->cols();
    this->features->resize(this->numberOfSuperpixels, cols + 3);

    double meanB = 0;
    double meanG = 0;
    double meanR = 0;
    int count = 0;

    for (int label = 0; label < this->numberOfSuperpixels; ++label) {

        meanB = 0;
        meanG = 0;
        meanR = 0;
        count = 0;

        for (int i = 0; i < this->height; ++i) {
            for (int j = 0; j < this->width; ++j) {
                if (this->labels[i][j] == label) {
                    meanB += this->image->at<cv::Vec3b>(i, j)[0];
                    meanG += this->image->at<cv::Vec3b>(i, j)[1];
                    meanR += this->image->at<cv::Vec3b>(i, j)[2];
                    ++count;
                }
            }
        }

        (*this->features)(label, cols) = meanB/count;
        (*this->features)(label, cols + 1) = meanG/count;
        (*this->features)(label, cols + 2) = meanR/count;
    }

    return Eigen::Vector2i(cols, cols + 2);
}

Eigen::Vector2i SuperpixelFeatures::addMean3DPosition() {
    assert(this->pointCloudAvailable);

    int cols = this->features->cols();
    this->features->resize(this->numberOfSuperpixels, cols + 3);

    double meanX = 0;
    double meanY = 0;
    double meanZ = 0;
    int count = 0;

    for (int label = 0; label < this->numberOfSuperpixels; ++label) {

        meanX = 0;
        meanY = 0;
        meanZ = 0;
        count = 0;

        for (int i = 0; i < this->height; ++i) {
            for (int j = 0; j < this->width; ++j) {
                if (this->labels[i][j] == label) {
                    meanX += (*this->pointCloud)(j, i).x;
                    meanY += (*this->pointCloud)(j, i).y;
                    meanZ += (*this->pointCloud)(j, i).z;
                    ++count;
                }
            }
        }

        (*this->features)(label, cols) = meanX/count;
        (*this->features)(label, cols + 1) = meanY/count;
        (*this->features)(label, cols + 2) = meanZ/count;
    }

    return Eigen::Vector2i(cols, cols + 2);
}

Eigen::Vector2i SuperpixelFeatures::addMean2DPosition() {

    int cols = this->features->cols();
    this->features->resize(this->numberOfSuperpixels, cols + 2);

    double meanX = 0;
    double meanY = 0;
    int count = 0;

    for (int label = 0; label < this->numberOfSuperpixels; ++label) {

        meanX = 0;
        meanY = 0;
        count = 0;

        for (int i = 0; i < this->height; ++i) {
            for (int j = 0; j < this->width; ++j) {
                if (this->labels[i][j] == label) {
                    meanX += j;
                    meanY += i;
                    ++count;
                }
            }
        }

        (*this->features)(label, cols) = meanX/count;
        (*this->features)(label, cols + 1) = meanY/count;
    }

    return Eigen::Vector2i(cols, cols + 1);
}

Eigen::Vector2i SuperpixelFeatures::addMeanNormal() {
    int cols = this->features->cols();
    this->features->resize(this->numberOfSuperpixels, cols + 3);

    for (int label = 0; label < this->numberOfSuperpixels; ++label) {
        std::vector<int> indices;

        for (int i = 0; i < this->height; ++i) {
            for (int j = 0; j < this->width; ++j) {
                if (this->labels[i][j] == label) {
                    indices.push_back(i*cols + j);
                }
            }
        }

        Eigen::Vector4f superpixelCentroid;
        Eigen::Matrix3f superpixelCovariance;
        Eigen::Vector3f superpixelNormal;

        pcl::compute3DCentroid(*pointCloud, indices, superpixelCentroid);
        pcl::computeCovarianceMatrix(*pointCloud, indices, superpixelCentroid, superpixelCovariance);
        Eigen::SelfAdjointEigenSolver<Eigen::Matrix3f> superpixelEigenValues(superpixelCovariance);
        superpixelNormal = superpixelEigenValues.eigenvectors().col(0);

        (*this->features)(label, cols) = superpixelNormal(0);
        (*this->features)(label, cols + 1) = superpixelNormal(1);
        (*this->features)(label, cols + 2) = superpixelNormal(2);
    }

    return Eigen::Vector2i(cols, cols + 2);
}

Eigen::Vector2i SuperpixelFeatures::addArea() {
    int cols = this->features->cols();
    this->features->resize(this->numberOfSuperpixels, cols + 1);

    int area = 0;
    for (int label = 0; label < this->numberOfSuperpixels; ++label) {

        area = 0;
        for (int i = 0; i < this->height; ++i) {
            for (int j = 0; j < this->width; ++j) {
                if (this->labels[i][j] == label) {
                    ++area;
                }
            }
        }

        (*this->features)(label, cols) = area;
    }

    return Eigen::Vector2i(cols, cols);
}

Eigen::Vector2i SuperpixelFeatures::addSuperpixelStatistics() {
    assert(this->pointCloudAvailable);

    int cols = this->features->cols();
    this->features->resize(this->numberOfSuperpixels, cols + 3);

    for (int label = 0; label < this->numberOfSuperpixels; ++label) {
        std::vector<int> indices;

        for (int i = 0; i < this->height; ++i) {
            for (int j = 0; j < this->width; ++j) {
                if (this->labels[i][j] == label) {
                    indices.push_back(i*cols + j);
                }
            }
        }

        Eigen::Vector4f superpixelCentroid;
        Eigen::Matrix3f superpixelCovariance;
        Eigen::Vector3f superpixelNormal;

        pcl::compute3DCentroid(*pointCloud, indices, superpixelCentroid);
        pcl::computeCovarianceMatrix(*pointCloud, indices, superpixelCentroid, superpixelCovariance);
        Eigen::SelfAdjointEigenSolver<Eigen::Matrix3f> superpixelEigenValues(superpixelCovariance);



        // Point-ness:
        (*this->features)(label, cols) = superpixelEigenValues.eigenvalues()(0);
        (*this->features)(label, cols + 1) = superpixelEigenValues.eigenvalues()(2) - superpixelEigenValues.eigenvalues()(1);
        (*this->features)(label, cols + 2) = superpixelEigenValues.eigenvalues()(1) - superpixelEigenValues.eigenvalues()(0);
    }

    return Eigen::Vector2i(cols, cols + 2);
}

Eigen::Vector2i SuperpixelFeatures::addColorHistogram(int bins) {
    assert(bins > 0 && bins < 10);

    int histogramSize = std::pow(bins, 3);
    int cols = this->features->cols();
    this->features->resize(this->numberOfSuperpixels, cols + histogramSize);

    int* normalization = new int[this->numberOfSuperpixels];
    for (int label = 0; label < this->numberOfSuperpixels; ++label) {
        normalization[label] = 0;

        for (int k = 0; k < histogramSize; ++k) {
            (*this->features)(label, cols + k) = 0;
        }
    }


    int denominator = ceil(256./((double) bins));
    for (int i = 0; i < this->height; ++i) {
        for (int j = 0; j < this->width; ++j) {
            int bin = this->image->at<cv::Vec3b>(i, j)[0]/denominator + bins*(this->image->at<cv::Vec3b>(i, j)[1]/denominator) + bins*bins*(this->image->at<cv::Vec3b>(i, j)[2]/denominator);
            ++(*this->features)(this->labels[i][j], cols + bin);
            ++normalization[this->labels[i][j]];
        }  
    }

    for (int label = 0; label < this->numberOfSuperpixels; ++label) {
        for (int k = 0; k < histogramSize; ++k) {
            (*this->features)(label, cols + k) /= normalization[label];
        }
    }

    return Eigen::Vector2i(cols, cols + histogramSize);
}

Eigen::Vector2i SuperpixelFeatures::addGroundTruth(int** labels) {
    int numberOfLabels = 0;
    for (int i = 0; i < this->height; ++i) {
        for (int j = 0; j < this->width; ++j) {
            if (labels[i][j] > numberOfLabels) {
                numberOfLabels = labels[i][j];
            }
        }
    }

    // Remember that zero may be a label as well.
    numberOfLabels = numberOfLabels + 1;

    Eigen::MatrixXi intersection(this->numberOfSuperpixels, numberOfLabels);
    for (int i = 0; i < this->height; ++i) {
        for (int j = 0; j < this->width; ++j) {
            assert(this->labels[i][j] < this->numberOfSuperpixels);
            assert(labels[i][j] < numberOfLabels);

            ++intersection(this->labels[i][j], labels[i][j]);
        }
    }

    for (int label = 0; label < this->numberOfSuperpixels; ++label) {

        int maxIntersection = 0;
        int maxGTLabel = 0;
        for (int gtLabel = 0; gtLabel < numberOfLabels; ++gtLabel) {
            if (intersection(label, gtLabel) > maxIntersection) {
                maxIntersection = intersection(label, gtLabel);
                maxGTLabel = gtLabel;
            }
        }

        (*this->features)(label, 0) = maxGTLabel;
    }

    return Eigen::Vector2i(0, 0);
}

int SuperpixelFeatures::getFeatureDimension() const {
    return this->features->cols();
}

Eigen::MatrixXd* SuperpixelFeatures::getFeatures() const {
    return this->features;
}