c++ - 具有不同消息大小的多播性能

Question

我写了一个 ping pong 测试程序来测试我的套接字库的性能，并且我发现无论传输数据包的大小如何，多播性能都是恒定的。所以我想知道我是不是对 socket 做错了什么，或者这是我应该预料到的？

这是我的测试结果:

buffer size:56 packet size:56 iteration:10000000 RRT/message: 12234ns
buffer size:128 packet size:128 iteration:10000000 RRT/message: 12316ns
buffer size:256 packet size:256 iteration:10000000 RRT/message: 12598ns
buffer size:512 packet size:512 iteration:10000000 RRT/message: 12782ns
buffer size:1024 packet size:1024 iteration:10000000 RRT/message: 17599ns

我的代码部分:

mmsghdr* packets;
iovec* iovecs;
....
packets = new mmsghdr[batch_recv_size];
std::memset(packets, 0, sizeof(packets));
iovecs = new iovec[batch_recv_size];
std::memset(iovecs, 0, sizeof(iovecs));
for (int i = 0; i < batch_recv_size; i++)
{
    iovecs[i] =
    {};
    iovecs[i].iov_base = pool.malloc();
    iovecs[i].iov_len = max_msg_size;
    packets[i] =
    {};
    packets[i].msg_hdr.msg_iov = &iovecs[i];
    packets[i].msg_hdr.msg_iovlen = 1;
}
....
int fd = socket(PF_INET, SOCK_DGRAM, IPPROTO_UDP);
if (fd < 0)
{
    error_code = errno;
    std::snprintf(error, sizeof(error), "Failed to create socket. ErrorNo: %d, ErrorText: %s.", error_code, strerror(error_code));
    if (user_handler.on_error)
    user_handler.on_error(std::string(error));
    return;
}

// Set reuse address.
if (!asio::socket_util::set_reuseaddr(fd, &error_code))
{
    close(fd);
    std::snprintf(error, sizeof(error), "Failed to set reuse address. ErrorNo: %d, ErrorText: %s.", error_code, strerror(error_code));
    if (user_handler.on_error)
    user_handler.on_error(std::string(error));
    return;
}

// Bind.
// Do not bind to INADDR_ANY, otherwise you will receive data from two multicast ip with same port.
sockaddr_in listen_sockaddr;
listen_sockaddr.sin_family = AF_INET;
listen_sockaddr.sin_addr.s_addr = inet_addr(mcast_ip.c_str());
listen_sockaddr.sin_port = htons(mcast_port);
if (bind(fd, (sockaddr*) &listen_sockaddr, sizeof(sockaddr)) < 0)
{
    error_code = errno;
    close(fd);
    std::snprintf(error, sizeof(error), "Failed to bind. ErrorNo: %d, ErrorText: %s.", error_code, strerror(error_code));
    if (user_handler.on_error)
    user_handler.on_error(std::string(error));
    return;
}

// Set non-blocking.
if (!asio::socket_util::set_nonblock(fd, &error_code))
{
    close(fd);
    std::snprintf(error, sizeof(error), "Failed to set non-blocking. ErrorNo: %d, ErrorText: %s.", error_code, strerror(error_code));
    if (user_handler.on_error)
    user_handler.on_error(std::string(error));
    return;
}

// Set socket receive buffer size.
socklen_t opt_len = sizeof(socket_recv_buffer_size);
if (setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &socket_recv_buffer_size, opt_len) < 0)
{
    error_code = errno;
    close(fd);
    std::snprintf(error, sizeof(error), "Failed to set socket receive buffer size. ErrorNo: %d, ErrorText: %s.", error_code,
            strerror(error_code));
    if (user_handler.on_error)
    user_handler.on_error(std::string(error));
    return;
}

// Get the socket receive buffer size.
int recv_buffer_size_set = 0;
if (getsockopt(fd, SOL_SOCKET, SO_RCVBUF, &recv_buffer_size_set, &opt_len) < 0)
{
    error_code = errno;
    close(fd);
    std::snprintf(error, sizeof(error), "Failed to get socket receive buffer size. ErrorNo: %d, ErrorText: %s.", error_code,
            strerror(error_code));
    if (user_handler.on_error)
    user_handler.on_error(std::string(error));
    return;
}

// Join multicast group.
ip_mreq mreq =
{};
mreq.imr_multiaddr.s_addr = inet_addr(mcast_ip.c_str());
mreq.imr_interface.s_addr = inet_addr(mcast_interface_ip.c_str());
if (setsockopt(fd, IPPROTO_IP, IP_ADD_MEMBERSHIP, &mreq, sizeof(mreq)) < 0)
{
    error_code = errno;
    close(fd);
    std::snprintf(error, sizeof(error), "Failed to join to multicast address: %s ErrorNo: %d, ErrorText: %s",mcast_interface_ip.c_str(), error_code,
            strerror(error_code));
    if (user_handler.on_error)
    user_handler.on_error(std::string(error));
    return;
}

//When receive signal from epoll, on_event triggered
void on_event(uint32_t events)
{
    while (true)
    {
        int packet_count = recvmmsg(mcast_fd, packets, batch_recv_size, 0, NULL);
        if (packet_count > 0)
        {
            unsigned long long recv_time = date_util::get_nanosecs_since_epoch();
            for (int i = 0; i < packet_count; i++)
            {
                user_handler.on_recv(recv_time, reinterpret_cast<uint8_t*>(iovecs[i].iov_base), iovecs[i].iov_len);
            }
        }
        else
        {
            break;
        }
    }
}

Answer 1

你没有做错任何事。在您超出路径 MTU 之前，性能可能大致相同测试中使用的主机之间的网络路由的大小。如果它们都在同一个以太网上，那么这个大小可能约为 1500 字节。一旦超过 PMTU 大小，您将在 IP 层出现碎片，这将影响性能。

在 Linux 上，使用 tracepath实用程序发现主机之间的 PTMU 并相应地调整您的测试参数。