c - Linux，UDP数据报和内核时间戳: Lots of examples and stackoversflow entries later,，仍然根本无法获得时间戳

Question

我一直在尝试并未能使Linux(内核4.1.4)为我提供发送和接收UDP数据报的时间戳。我已经阅读了原始的内核文档(https://www.kernel.org/doc/Documentation/networking/timestamping.txt)，以及许多示例和许多stackoverflow条目。我可以在发送方和接收方之间毫无问题地发送数据报。但是我无法获得发送或接收数据报的时间戳，也无法弄清楚自己在做什么错。

一件奇怪的事情是，当我使用MSG_ERRQUEUE channel 获取已发送数据报上的时间戳信息时，我确实获得了原始传出数据包，并且得到了第一个辅助消息(SOL_IP，IP_RECVERR)，但是没有得到第二条消息(应为SOL_SOCKET级别，键入SCM_TIMESTAMPING)。

在另一个关于获取已发送数据包的时间戳(Timestamp outgoing packets)的stackoverflow条目中，有人提到某些驱动程序可能未实现对skb_tx_timestamp的调用，但我检查了我的(Realtek)，并且该调用肯定在其中。

这是我设置UDP接收器的方式(未显示错误处理代码):

inf->fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);

timestampOn = SOF_TIMESTAMPING_RX_SOFTWARE | SOF_TIMESTAMPING_RX_HARDWARE;
r = setsockopt(inf->fd, SOL_SOCKET, SO_TIMESTAMPING, &timestampOn, sizeof(timestampOn));

r = setsockopt(inf->fd, SOL_SOCKET, SO_REUSEPORT, &on, sizeof(on));

memset(&(inf->local), 0, sizeof(struct sockaddr_in));
inf->local.sin_family = AF_INET;
inf->local.sin_port = htons(port);
inf->local.sin_addr.s_addr = htonl(INADDR_ANY);

r = bind(inf->fd, (struct sockaddr *)&(inf->local), sizeof(struct sockaddr_in));

是否使用SO_REUSEPORT似乎无关紧要。

对于接收，我的理解是我们不使用MSG_ERRQUEUE。仅当我们需要已发送消息的时间戳记时。此外，当我将MSG_ERRQUEUE与recvmsg一起使用时，会得到“资源暂时不可用”。这是我接收数据报的方式:

int recv_len;
struct msghdr   msg;
struct iovec    iov;

memset(&msg, 0, sizeof(msg));
memset(&iov, 0, sizeof(iov));

// Space for control message info plus timestamp
char ctrl[2048];
memset(ctrl, 0, sizeof(ctrl));
//struct cmsghdr *cmsg = (struct cmsghdr *) &ctrl;

// Ancillary data buffer and length
msg.msg_control      = (char *) ctrl;
msg.msg_controllen   = sizeof(ctrl);

// Dest address info
msg.msg_name         = (struct sockaddr *) &(inf->remote);
msg.msg_namelen      = sizeof(struct sockaddr_in);

// Array of data buffers (scatter/gather)
msg.msg_iov          = &iov;
msg.msg_iovlen       = 1;

// Data buffer pointer and length
iov.iov_base         = buf;
iov.iov_len          = len;

recv_len = recvmsg(inf->fd, &msg, 0);

然后，我将指向msg的指针传递给执行此操作的另一个函数( handle_time):

struct timespec* ts = NULL;
struct cmsghdr* cmsg;
struct sock_extended_err *ext;

for( cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg,cmsg) ) {
    printf("level=%d, type=%d, len=%zu\n", cmsg->cmsg_level, cmsg->cmsg_type, cmsg->cmsg_len);
}

收到零消息。 所以这是第一个问题。我上面的设置代码与我在网上找到的其他六个示例相匹配，但是我没有从中获得任何辅助数据。

接下来，让我们转向发送数据报。设置如下:

inf->port = port;
inf->fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);

memset(&(inf->remote), 0, sizeof(struct sockaddr_in));
inf->remote.sin_family = AF_INET;
inf->remote.sin_port = htons(port);

timestampOn = SOF_TIMESTAMPING_TX_SOFTWARE | SOF_TIMESTAMPING_TX_HARDWARE;
r = setsockopt(inf->fd, SOL_SOCKET, SO_TIMESTAMPING, &timestampOn, sizeof(timestampOn));

on = 1;
r = setsockopt(inf->fd, SOL_SOCKET, SO_BROADCAST, &on, sizeof(on));

r = inet_aton(address, &(inf->remote.sin_addr));

这就是我发送数据报的方式:

int send_len, r, i;
struct msghdr   msg;
struct iovec    iov;

memset(&msg, 0, sizeof(msg));
memset(&iov, 0, sizeof(iov));

// Space for control message info plus timestamp
char ctrl[2048];
memset(ctrl, 0, sizeof(ctrl));
//struct cmsghdr *cmsg = (struct cmsghdr *) &ctrl;

// Ancillary data buffer and length
//msg.msg_control      = (char *) ctrl;
//msg.msg_controllen   = sizeof(ctrl);    

// Dest address info
msg.msg_name         = (struct sockaddr *) &(inf->remote);
msg.msg_namelen      = sizeof(struct sockaddr_in);

// Array of data buffers (scatter/gather)
msg.msg_iov          = &iov;
msg.msg_iovlen       = 1;

// Data buffer pointer and length
iov.iov_base         = buf;
iov.iov_len          = len;

send_len = sendmsg(inf->fd, &msg, 0);

我已经看到了重用msg和iov数据结构的示例，但是在我的实验中，我添加了代码以确保已清除所有内容，以防万一send留下了任何东西，尽管这没有任何区别。这是获取时间戳的代码:

memset(&msg, 0, sizeof(msg));
memset(&iov, 0, sizeof(iov));
memset(ctrl, 0, sizeof(ctrl));
msg.msg_control      = (char *) ctrl;
msg.msg_controllen   = sizeof(ctrl);
msg.msg_name         = (struct sockaddr *) &(inf->remote);
msg.msg_namelen      = sizeof(struct sockaddr_in);
msg.msg_iov          = &iov;
msg.msg_iovlen       = 1;
iov.iov_base         = junk_buf;
iov.iov_len          = sizeof(junk_buf);

for (;;) {
    r = recvmsg(inf->fd, &msg, MSG_ERRQUEUE);
    if (r<0) {
        fprintf(stderr, "Didn't get kernel time\n");
        return send_len;
    }

    printf("recvmsg returned %d\n", r);
    handle_time(&msg);
}

数据缓冲区包含预期的原始数据报。我得到的辅助数据仅包含一条消息，handle_time的输出为:

level=0, type=11, len=48

这是SOL_IP级别，类型为IP_RECVERR，根据文档要求。查看有效负载(结构sock_extended_err)，错误号为42(ENOMSG，没有所需类型的消息)，来源为4(SO_EE_ORIGIN_TXSTATUS)。从文档来看，这是应该发生的，并且表明实际上我确实设法通知内核我想要TX状态消息。但是没有第二条辅助信息!

我试图查看是否有任何内核编译选项可能会禁用此功能，但我没有找到任何选项。所以我对此完全感到困惑。谁能帮我弄清楚我做错了什么？

谢谢!

更新:我尝试在另一台Linux机器上运行相同的代码，这次是CentOS 7(内核3.10.0-693.2.2.el7.x86_64)。我无法弄清楚该计算机具有哪种类型的NIC，但是当我尝试发送数据报时，会出现一些其他奇怪的行为。对于第一个数据报，当我启动该程序时，我将获得消息和一条辅助消息，就像上面一样。对于随后的每个 sendmsg调用，errno告诉我出现“无效参数”错误。如果我没有在套接字上启用时间戳，此错误将消失。

更新2:我发现我并没有进行必要的ioctl来启用驱动程序中的时间戳。不幸的是，当我执行此调用时，我从errno(没有此类设备)获得ENODEV。这是我尝试执行的操作(这是我从 https://github.com/majek/openonload/blob/master/src/tests/onload/hwtimestamping/tx_timestamping.c模仿的):

struct ifreq ifr;
struct hwtstamp_config hwc;

inf->fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);

memset(&ifr, 0, sizeof(ifr));
hwc.flags = 0;
hwc.tx_type = HWTSTAMP_TX_ON;
hwc.rx_filter = HWTSTAMP_FILTER_ALL;
ifr.ifr_data = (char*)&hwc;
r = ioctl(inf->fd, SIOCSHWTSTAMP, &ifr);

话虽这么说，我对软件时间戳比较满意，不需要该调用。因此，我不确定这是否有帮助。

更新3:请求了一个可编译的示例。整个程序非常简单，因此我将其放在以下的pastebin中: https://pastebin.com/qd0gspRc

另外，这是ethtool的输出:

Time stamping parameters for eth0:
Capabilities:
        software-transmit     (SOF_TIMESTAMPING_TX_SOFTWARE)
        software-receive      (SOF_TIMESTAMPING_RX_SOFTWARE)
        software-system-clock (SOF_TIMESTAMPING_SOFTWARE)
PTP Hardware Clock: none
Hardware Transmit Timestamp Modes: none
Hardware Receive Filter Modes: none

由于这显然不支持硬件时间戳，因此ioctl毫无意义。我尝试将发送方和接收方的SO_TIMESTAMPING设置更改为SOF_TIMESTAMPING_TX_SOFTWARE和SOF_TIMESTAMPING_RX_SOFTWARE。那没有帮助。

然后，我尝试将SOF_TIMESTAMPING_SOFTWARE添加到两者中。我终于开始得到一些东西:

level=1, type=37, len=64

级别1是SOL_SOCKET，类型37是SCM_TIMESTAMPING。我将返回文档，并弄清楚如何解释这一点。它说了一些关于传递三个时间结构的数组的事情。驱动程序对 skb_tx_timestamp的调用应该已经足够，这样就不需要我启用“假”软件时间戳来获取信息。

Answer 1

就像我在评论中说的，必须使用SOF_TIMESTAMPING_SOFTWARE和SOF_TIMESTAMPING_RAW_HARDWARE，因为如果我正确理解the documentation，则有些位会生成时间戳，有些位会在控制消息中报告它们:

1.3.1 Timestamp Generation

Some bits are requests to the stack to try to generate timestamps. Any combination of them is valid. Changes to these bits apply to newly created packets, not to packets already in the stack. As a result, it is possible to selectively request timestamps for a subset of packets (e.g., for sampling) by embedding an send() call within two setsockopt calls, one to enable timestamp generation and one to disable it. Timestamps may also be generated for reasons other than being requested by a particular socket, such as when receive timestamping is enabled system wide, as explained earlier.

1.3.2 Timestamp Reporting

The other three bits control which timestamps will be reported in a generated control message. Changes to the bits take immediate effect at the timestamp reporting locations in the stack. Timestamps are only reported for packets that also have the relevant timestamp generation request set.

之后，要使用数据文档，请说:

2.1 SCM_TIMESTAMPING records

These timestamps are returned in a control message with cmsg_level SOL_SOCKET, cmsg_type SCM_TIMESTAMPING, and payload of type

struct scm_timestamping { struct timespec ts[3]; };

...

The structure can return up to three timestamps. This is a legacy feature. At least one field is non-zero at any time. Most timestamps are passed in ts[0]. Hardware timestamps are passed in ts[2].

要获取传输时间戳，这需要一些配置，首先您需要知道 that software timestamp are not always available，我只实现了获取硬件传输时间戳。但是我不是这些领域的专家，我只是尝试使用发现的信息来实现时间戳。

其次，我需要使用 linuxptp tool激活硬件功能，我使用 hwstamp_cli:

hwstamp_ctl -i eth0 -r 1 -t 1

对此进行了一些修改，我实现了获取硬件传输时间戳的功能，但仅使用ethX接口(interface)，因为lo接口(interface)没有这些功能AFAIK，因此最终代码为:

#include <arpa/inet.h>
#include <errno.h>
#include <inttypes.h>
#include <linux/errqueue.h>
#include <linux/net_tstamp.h>
#include <linux/sockios.h>
#include <net/if.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <sys/time.h>
#include <sys/types.h>
#include <unistd.h>

#define UDP_MAX_LENGTH 1500

typedef struct {
  int fd;
  int port;
  int err_no;
  struct sockaddr_in local;
  struct sockaddr_in remote;
  struct timeval time_kernel;
  struct timeval time_user;
  int64_t prev_serialnum;
} socket_info;

static int setup_udp_receiver(socket_info *inf, int port) {
  inf->port = port;
  inf->fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
  if (inf->fd < 0) {
    inf->err_no = errno;
    fprintf(stderr, "setup_udp_server: socket failed: %s\n",
            strerror(inf->err_no));
    return inf->fd;
  }

  int timestampOn =
      SOF_TIMESTAMPING_RX_SOFTWARE | SOF_TIMESTAMPING_TX_SOFTWARE |
      SOF_TIMESTAMPING_SOFTWARE | SOF_TIMESTAMPING_RX_HARDWARE |
      SOF_TIMESTAMPING_TX_HARDWARE | SOF_TIMESTAMPING_RAW_HARDWARE |
      // SOF_TIMESTAMPING_OPT_TSONLY |
      0;
  int r = setsockopt(inf->fd, SOL_SOCKET, SO_TIMESTAMPING, &timestampOn,
                     sizeof timestampOn);
  if (r < 0) {
    inf->err_no = errno;
    fprintf(stderr, "setup_udp_server: setsockopt failed: %s\n",
            strerror(inf->err_no));
    return r;
  }

  int on = 1;
  r = setsockopt(inf->fd, SOL_SOCKET, SO_REUSEPORT, &on, sizeof on);
  if (r < 0) {
    inf->err_no = errno;
    fprintf(stderr, "setup_udp_server: setsockopt2 failed: %s\n",
            strerror(inf->err_no));
    return r;
  }

  inf->local = (struct sockaddr_in){.sin_family = AF_INET,
                                    .sin_port = htons((uint16_t)port),
                                    .sin_addr.s_addr = htonl(INADDR_ANY)};
  r = bind(inf->fd, (struct sockaddr *)&inf->local, sizeof inf->local);
  if (r < 0) {
    inf->err_no = errno;
    fprintf(stderr, "setup_udp_server: bind failed: %s\n",
            strerror(inf->err_no));
    return r;
  }

  inf->prev_serialnum = -1;

  return 0;
}

static int setup_udp_sender(socket_info *inf, int port, char *address) {
  inf->port = port;
  inf->fd = socket(AF_INET, SOCK_DGRAM, IPPROTO_UDP);
  if (inf->fd < 0) {
    inf->err_no = errno;
    fprintf(stderr, "setup_udp_client: socket failed: %s\n",
            strerror(inf->err_no));
    return inf->fd;
  }

  int timestampOn =
      SOF_TIMESTAMPING_RX_SOFTWARE | SOF_TIMESTAMPING_TX_SOFTWARE |
      SOF_TIMESTAMPING_SOFTWARE | SOF_TIMESTAMPING_RX_HARDWARE |
      SOF_TIMESTAMPING_TX_HARDWARE | SOF_TIMESTAMPING_RAW_HARDWARE |
      // SOF_TIMESTAMPING_OPT_TSONLY |
      0;
  int r = setsockopt(inf->fd, SOL_SOCKET, SO_TIMESTAMPING, &timestampOn,
                     sizeof timestampOn);
  if (r < 0) {
    inf->err_no = errno;
    fprintf(stderr, "setup_udp_server: setsockopt failed: %s\n",
            strerror(inf->err_no));
    return r;
  }

  inf->remote = (struct sockaddr_in){.sin_family = AF_INET,
                                     .sin_port = htons((uint16_t)port)};
  r = inet_aton(address, &inf->remote.sin_addr);
  if (r == 0) {
    fprintf(stderr, "setup_udp_client: inet_aton failed\n");
    inf->err_no = 0;
    return -1;
  }

  inf->local = (struct sockaddr_in){.sin_family = AF_INET,
                                    .sin_port = htons(0),
                                    .sin_addr.s_addr = htonl(INADDR_ANY)};
  inf->prev_serialnum = -1;

  return 0;
}

static void handle_scm_timestamping(struct scm_timestamping *ts) {
  for (size_t i = 0; i < sizeof ts->ts / sizeof *ts->ts; i++) {
    printf("timestamp: %lld.%.9lds\n", (long long)ts->ts[i].tv_sec,
           ts->ts[i].tv_nsec);
  }
}

static void handle_time(struct msghdr *msg) {

  for (struct cmsghdr *cmsg = CMSG_FIRSTHDR(msg); cmsg;
       cmsg = CMSG_NXTHDR(msg, cmsg)) {
    printf("level=%d, type=%d, len=%zu\n", cmsg->cmsg_level, cmsg->cmsg_type,
           cmsg->cmsg_len);

    if (cmsg->cmsg_level == SOL_IP && cmsg->cmsg_type == IP_RECVERR) {
      struct sock_extended_err *ext =
          (struct sock_extended_err *)CMSG_DATA(cmsg);
      printf("errno=%d, origin=%d\n", ext->ee_errno, ext->ee_origin);
      continue;
    }

    if (cmsg->cmsg_level != SOL_SOCKET)
      continue;

    switch (cmsg->cmsg_type) {
    case SO_TIMESTAMPNS: {
      struct scm_timestamping *ts = (struct scm_timestamping *)CMSG_DATA(cmsg);
      handle_scm_timestamping(ts);
    } break;
    case SO_TIMESTAMPING: {
      struct scm_timestamping *ts = (struct scm_timestamping *)CMSG_DATA(cmsg);
      handle_scm_timestamping(ts);
    } break;
    default:
      /* Ignore other cmsg options */
      break;
    }
  }
  printf("End messages\n");
}

static ssize_t udp_receive(socket_info *inf, char *buf, size_t len) {
  char ctrl[2048];
  struct iovec iov = (struct iovec){.iov_base = buf, .iov_len = len};
  struct msghdr msg = (struct msghdr){.msg_control = ctrl,
                                      .msg_controllen = sizeof ctrl,
                                      .msg_name = &inf->remote,
                                      .msg_namelen = sizeof inf->remote,
                                      .msg_iov = &iov,
                                      .msg_iovlen = 1};
  ssize_t recv_len = recvmsg(inf->fd, &msg, 0);
  gettimeofday(&inf->time_user, NULL);

  if (recv_len < 0) {
    inf->err_no = errno;
    fprintf(stderr, "udp_receive: recvfrom failed: %s\n",
            strerror(inf->err_no));
  }

  handle_time(&msg);

  return recv_len;
}

static ssize_t udp_send(socket_info *inf, char *buf, size_t len) {
  struct iovec iov = (struct iovec){.iov_base = buf, .iov_len = len};
  struct msghdr msg = (struct msghdr){.msg_name = &inf->remote,
                                      .msg_namelen = sizeof inf->remote,
                                      .msg_iov = &iov,
                                      .msg_iovlen = 1};
  gettimeofday(&inf->time_user, NULL);
  ssize_t send_len = sendmsg(inf->fd, &msg, 0);
  if (send_len < 0) {
    inf->err_no = errno;
    fprintf(stderr, "udp_send: sendmsg failed: %s\n", strerror(inf->err_no));
  }

  return send_len;
}

static ssize_t meq_receive(socket_info *inf, char *buf, size_t len) {
  struct iovec iov = (struct iovec){.iov_base = buf, .iov_len = len};
  char ctrl[2048];
  struct msghdr msg = (struct msghdr){.msg_control = ctrl,
                                      .msg_controllen = sizeof ctrl,
                                      .msg_name = &inf->remote,
                                      .msg_namelen = sizeof inf->remote,
                                      .msg_iov = &iov,
                                      .msg_iovlen = 1};
  ssize_t recv_len = recvmsg(inf->fd, &msg, MSG_ERRQUEUE);
  if (recv_len < 0) {
    inf->err_no = errno;
    if (errno != EAGAIN) {
      fprintf(stderr, "meq_receive: recvmsg failed: %s\n",
              strerror(inf->err_no));
    }
    return recv_len;
  }
  handle_time(&msg);

  return recv_len;
}

typedef struct {
  int64_t serialnum;

  int64_t user_time_serialnum;
  int64_t user_time;

  int64_t kernel_time_serialnum;
  int64_t kernel_time;

  size_t message_bytes;
} message_header;

static const size_t payload_max = UDP_MAX_LENGTH - sizeof(message_header);

static ssize_t generate_random_message(socket_info *inf, char *buf,
                                       size_t len) {
  if (len < sizeof(message_header)) {
    return -1;
  }
  message_header *header = (message_header *)buf;
  char *payload = (char *)(header + 1);
  size_t payload_len = (size_t)random() % (payload_max + 1);
  if (payload_len > len - sizeof(message_header)) {
    payload_len = len - sizeof(message_header);
  }
  for (size_t i = 0; i < payload_len; i++) {
    payload[i] = (char)random();
  }

  static int64_t serial_num = 0;
  *header = (message_header){
      .user_time_serialnum = inf->prev_serialnum,
      .user_time = inf->time_user.tv_sec * 1000000000L + inf->time_user.tv_usec,
      .kernel_time_serialnum = inf->prev_serialnum,
      .kernel_time =
          inf->time_kernel.tv_sec * 1000000000L + inf->time_kernel.tv_usec,
      .serialnum = serial_num,
      .message_bytes = payload_len};
  size_t total = payload_len + sizeof *header;

  printf("uts%5" PRId64 ": kt=%" PRId64 ", ut=%" PRId64 ", sn=%" PRId64
         ": s=%zu\n",
         header->user_time_serialnum, header->kernel_time, header->user_time,
         header->serialnum, total);

  inf->prev_serialnum = serial_num++;

  return (ssize_t)total;
}

static void sender_loop(char *host) {
  socket_info inf;
  int ret = setup_udp_sender(&inf, 8000, host);
  if (ret < 0) {
    return;
  }

  for (int i = 0; i < 2000; i++) {
    useconds_t t = random() % 2000000;
    usleep(t);
    char packet_buffer[4096];
    ssize_t len =
        generate_random_message(&inf, packet_buffer, sizeof packet_buffer);
    if (len < 0) {
      return;
    }
    udp_send(&inf, packet_buffer, (size_t)len);
    while (meq_receive(&inf, packet_buffer, sizeof packet_buffer) != -1) {
    }
  }
}

static void receiver_loop(void) {
  socket_info inf;
  int ret = setup_udp_receiver(&inf, 8000);
  if (ret < 0) {
    return;
  }

  for (int i = 0; i < 1000; i++) {
    char packet_buffer[4096];
    udp_receive(&inf, packet_buffer, sizeof packet_buffer);
  }
}

#define USAGE "Usage: %s [-r | -s host]\n"

int main(int argc, char *argv[]) {
  if (argc < 2) {
    fprintf(stderr, USAGE, argv[0]);
    return 0;
  }

  if (0 == strcmp(argv[1], "-s")) {
    if (argc < 3) {
      fprintf(stderr, USAGE, argv[0]);
      return 0;
    }
    sender_loop(argv[2]);
  } else if (0 == strcmp(argv[1], "-r")) {
    receiver_loop();
  } else {
    fprintf(stderr, USAGE, argv[0]);
  }
}

示例输出:

$ ./a.out -r
level=1, type=37, len=64
timestamp: 1511196758.087209387s
timestamp: 0.000000000s
timestamp: 0.000000000s
End messages
level=1, type=37, len=64
timestamp: 1511196759.333507671s
timestamp: 0.000000000s
timestamp: 0.000000000s
End messages
$ ./a.out -s "8.8.8.8"
uts   -1: kt=238059712, ut=140918979990070, sn=0: s=482
uts    0: kt=238059712, ut=1511197522000237457, sn=1: s=132
level=1, type=37, len=64
timestamp: 0.000000000s
timestamp: 0.000000000s
timestamp: 1511197359.637050597s
level=0, type=11, len=48
errno=42, origin=4
End messages
uts    1: kt=238059712, ut=1511197523000483805, sn=2: s=1454
level=1, type=37, len=64
timestamp: 0.000000000s
timestamp: 0.000000000s
timestamp: 1511197360.883295397s
level=0, type=11, len=48
errno=42, origin=4
End messages

现场测试: sender， receiver