Python视频流到C++服务器

Question

我正在通过一个项目学习 C++，在这个项目中，我将视频从一台设备流式传输到另一台设备上进行显示。我有一个带有以下客户端代码的 PiCamera，它捕获帧并通过 TCP 流发送它们。

import io
import socket
import struct
import time
import picamera


def stream():
    servver_ip = "0.0.0.0" #intentionally left out
    client_socket = socket.socket()
    client_socket.connect((server_ip, 8123))
    connection = client_socket.makefile("wb") 

    try:
        camera = picamera.PiCamera()
        camera.resolution = (640, 480)
        camera.start_preview()
        time.sleep(2)
        start = time.time()
        stream = io.BytesIO()
        for image in camera.capture_continuous(stream, 'jpeg'):
            connection.write(struct.pack("<L", stream.tell()))
            connection.flush()
            stream.seek(0)
            connection.write(stream.read()) 
            stream.seek(0)
            stream.truncate()
        connection.write(struct.pack("<L", 0)) 
    except Exception as e:
        raise e
    finally:
        connection.close()
        client_socket.close()

def main():
    while True:
        try:
            stream()
        except ConnectionRefusedError as e:
            print("Connection refused. It the server on?")
            time.sleep(2)
        except Exception as e:
            print(e)
            break 

if __name__ == "__main__":
    main()

上面的代码直接来自 PiCamera recipie，当我在另一端有 Python 脚本时效果很好。但是，当我尝试使用以下 C++ 代码接收和显示流时，我只得到部分帧并且它根本无法正常流动。有时我会得到 0 个数据、1 个帧或缓慢的断断续续的困惑。更改 usleep 增量似乎得到了一些改进，但我担心这不是正确的答案。我也试过在 recv 中使用 MSG_WAITALL 标志，但这似乎阻止了任何数据通过，这可能表明我的缓冲区大小值不正确。

// the standard stuff
#include <iostream>
#include <string>
#include <unistd.h>

// opencv
#include <opencv2/opencv.hpp>
#include <opencv2/highgui/highgui.hpp>
#include <opencv2/video/video.hpp>

//socket stuffs
#include <sys/socket.h>
#include <sys/types.h>
#include <netdb.h>
#include <netinet/in.h>


int main(int argc, const char * argv[]) {

    int key;

    int yes = 1;

    int height = 480;
    int width = 640;

    char* listenPort = "8123";

    int bytes = 0;

    int status;
    struct addrinfo hints; // define our interface
    struct addrinfo *serviceinfo; // will point to results
    struct sockaddr_storage their_addr;
    memset(&hints, 0, sizeof(hints)); // make sure the struct is empy
    hints.ai_family = AF_UNSPEC; // don't care if IP4 or IP6
    hints.ai_socktype = SOCK_STREAM; // TCP
    hints.ai_flags = AI_PASSIVE; // fill in my IP for me

    // look up address info for this machine. Will populate service info
    if ((getaddrinfo(NULL, listenPort, &hints, &serviceinfo)) == -1){
        fprintf(stderr, "getaddinfo error: %s\n", gai_strerror(status));
        exit(1);
    }

    // make the socket
    int sockfd = socket(serviceinfo->ai_family, serviceinfo->ai_socktype, serviceinfo->ai_protocol);

    //bind to the correct port
    if ((bind(sockfd, serviceinfo->ai_addr, serviceinfo->ai_addrlen)) == -1){
        fprintf(stderr, "failed to bind: %s\n", gai_strerror(status));
        exit(1);
    }

    // allow us to reuse the port
    if ((setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes))) == -1){
        perror("setsockopt");
        exit(1);
    }

    // number of connections to let in the queue
    int backlog = 1;

    // start listening for incoming connections
    if ((listen(sockfd, backlog)) == -1){
        perror("listen");
        exit(1);
    }

    // start accepting incoming connections
    socklen_t addr_size = sizeof(their_addr);
    int new_fd = accept(sockfd, (struct sockaddr *)&their_addr, &addr_size); // use this for all send and receive calls
    if (new_fd == -1){
        perror("accept");
        exit(1);
    }

    // empty image object
    cv::Mat img = cv::Mat::zeros(height, width, CV_8UC1);

    if (!img.isContinuous()){
        img = img.clone();
    }

    int image_size = img.total() * img.elemSize();

    cv::Mat rawImage = cv::Mat::zeros(1, image_size, CV_8UC1);

    if (!rawImage.isContinuous()){
        rawImage = rawImage.clone();
    }
    cv::Mat flipped = cv::Mat::zeros(height, width, CV_8UC1);

    std::cout << "Press q to quit" << std::endl;
    cv::namedWindow("Sentry", cv::WINDOW_AUTOSIZE);

    while (key != 'q'){
        std::cout << "Capturing" << std::endl;

        if ((bytes = recv(new_fd, rawImage.data, image_size, 0)) == -1){
            perror("Failed to receive bytes!");
            exit(1);
        }

        img = cv::imdecode(rawImage, CV_LOAD_IMAGE_COLOR);
        cv::flip(img, flipped, 1);


        if (flipped.data != NULL) {
            // show image. using a global here
            cv::imshow("Sentry", flipped);
        }

        memset(rawImage.data, 0x0, sizeof(rawImage));

        // look for quit key
        key = cv::waitKey(10);

        // pause for half a second
        usleep(500000);
    };

    cv::destroyAllWindows();

    freeaddrinfo(serviceinfo); // clear the linked list

    close(sockfd);

    return 0;

}

我正在寻找任何提示、答案或只是为了指明正确的方向。提前致谢。

编辑:工作解决方案
首先，感谢凯文帮助我。我的问题是我没有意识到我最初的 Python 客户端正在发送图像大小。通过搜索和处理下面的答案，我开始获取前 4 个字节并开始调整我的 cv::Mat rawImage 的大小。我没有编写逻辑来检查 recv 以确保获取所有数据，而是使用 MSG_WAITALL。该标志与获得正确的有效载荷大小相结合，使一切顺利进行。非常谦卑的经历。

#define CHUNKSIZE 500

int main(int argc, const char * argv[]) {

    // skipping code from before

    int32_t image_size = 0;
    cv::Mat rawImage;
    long data_received = 0;
    long success;

    cv::namedWindow("Sentry", cv::WINDOW_AUTOSIZE);

    while (key != 'q'){
        std::cout << "Capturing" << std::endl;

        // Very important!! Grab the image_size here
        success = recv(new_fd, &image_size, 4 , NULL);
        if (success == -1){
            perror("Failed to receive file size!");
            exit(1);
        }
        // if your image size is extremely large, it's probably 
        // because you're grabing the wrong number of bytes above
        if (image_size > 300000){
            std::cout << "Image size is too large " << image_size << std::endl;
            exit(1);
        }

        if (image_size > 0) {

            //
            rawImage = cv::Mat::zeros(1, image_size, CV_8UC1);
            success = recv(new_fd, rawImage.data, image_size, MSG_WAITALL);
            if (success == -1){
                perror("Failed to receive");
                exit(1);
            }

            img = cv::imdecode(rawImage, CV_LOAD_IMAGE_COLOR);

            if (img.data != NULL) {
                // show image. using a global here
                cv::imshow("Sentry", img);
            } else {
                std::cout << "Image is null!" << std::endl;
            }

            memset(&rawImage, 0x0, sizeof(rawImage));

            // look for quit key
            key = cv::waitKey(10);
        } else {
            std::cout << "No message yet" << std::endl;

        }
        image_size = 0;
        // pause for half a second
        usleep(500000);
    };

    cv::destroyAllWindows();

    freeaddrinfo(serviceinfo); // clear the linked list

    close(sockfd);

    return 0;

}

Answer 1

当我使用带有 C++ 的 wifi 在我的 PC 上创建实时流式传输 Raspberry Pi 时，我也遇到了同样的问题。

我通过只发送图像 block 解决了这个问题。测试大约 500 字节的 block ，然后在看到它工作后增加它。不要发送整个文件，否则程序将只获取帧的一部分。

此外，不要像@Sam 所说的那样读取 send() 或 recv() 函数的返回码，我实际上发现它在创建实时流时不可靠。只需分块发送即可。

如果你想看的话，我有我的源代码，我知道使用 C++ 创建直播服务器非常困难，因为几乎没有文档，所以我想帮助你，因为它给了我 sooo很头疼!如果您想要源代码，请告诉我，祝您好运!

客户端应该看起来像这样，这不会编译!我试图让它易于阅读:

//Client    
#define CHUNK_SIZE 500

int main()
{
    //do all the connection stuff here and connect to it
    int filesize;
    int dataReceived = 0;
    char received_message[10];
    std::vector<char> imageData;
    while (1)
    {
        recv(Connection, received_message, 11, NULL); //recieved File size of image
        filesize = atoi(received_message); //Turning File size char into an integer 

        imageData.resize(filesize); //Setting up vector to recieve image data

        while (dataReceived < filesize) //keep running until we get enough bytes
        {
            if (filesize - dataReceived >= CHUNK_SIZE) //if the amount of bytes left to recieve is greater than or equal to chunk size, then lets receive a CHUNK_SIZE of the image
            {
                recv(Connection, &imageData[dataReceived], CHUNK_SIZE, 0);
                dataReceived += CHUNK_SIZE;
            }
            else //Else, if the amount of bytes left to recieve is smaller than the CHUNK_SIZE then lets receive a specific amount of bytes
            {
                recv(Connection, &imageData[dataReceived], fileSize - dataReceived, 0);
                dataReceived += fileSize - dataReceived;
            }
         Sleep(Amount of sleep); //Note, python code will always be slower than C++ so you must set a Sleep() function.
        }
        std::cout << "Got the image, it is stored in the imageData vector!" << std::endl;
        std::cout << "Press enter to get another image!" << std::endl;
        std::cin.get(); 

    }
}