From c75a2220f3027eddb2f3fa908a54c14fa13047cb Mon Sep 17 00:00:00 2001 From: yazimut Date: Wed, 16 Oct 2019 13:12:35 +0300 Subject: [PATCH] Added Generic Segmentation Offload for UDP --- docs/invoking.rst | 9 +++- src/iperf.h | 2 + src/iperf_api.c | 63 +++++++++++++++++++++++++ src/iperf_api.h | 3 ++ src/iperf_error.c | 7 +++ src/iperf_locale.c | 5 ++ src/iperf_udp.c | 114 ++++++++++++++++++++++++++++++++++++++++----- src/net.c | 39 ++++++++++++++++ src/net.h | 1 + 9 files changed, 231 insertions(+), 12 deletions(-) diff --git a/docs/invoking.rst b/docs/invoking.rst index 37677424a..7801dac07 100644 --- a/docs/invoking.rst +++ b/docs/invoking.rst @@ -388,7 +388,14 @@ the executable. --rsa-public-key-path file path to the RSA public key used to encrypt authentication cre- dentials (if built with OpenSSL support) - + + --udp-gso N + enables Generic Segmentation Offload for UDP protocol. "N" is a + segment size, it cannot be greater than MTU including headers. + Use "--udp-lso 0" to use blocksize ("-l" option) as a segment + size. Attention! Spliting into too many fragments can lead to a + crash of "iperf" or statistics errors. Supported only by Linux! + EXAMPLES Authentication - RSA Keypair diff --git a/src/iperf.h b/src/iperf.h index 6ce77f59b..b8e01e1fc 100644 --- a/src/iperf.h +++ b/src/iperf.h @@ -154,6 +154,8 @@ struct iperf_settings EVP_PKEY *client_rsa_pubkey; #endif // HAVE_SSL int connect_timeout; /* socket connection timeout, in ms */ + int udp_gso; /* Generic Segmentation Offload for UDP protocol */ + uint16_t udp_gso_segsize; /* GSO segment size */ }; struct iperf_test; diff --git a/src/iperf_api.c b/src/iperf_api.c index f979de1c8..4e9e5ebc4 100644 --- a/src/iperf_api.c +++ b/src/iperf_api.c @@ -44,6 +44,9 @@ #include #include #include +#include +#include +#include #include #include #ifdef HAVE_STDINT_H @@ -57,6 +60,9 @@ #include #include #include +#if defined(__linux__) +#include +#endif #if defined(HAVE_CPUSET_SETAFFINITY) #include @@ -87,6 +93,14 @@ #include "iperf_auth.h" #endif /* HAVE_SSL */ +#ifndef UDP_SEGMENT + #define UDP_SEGMENT 103 /* Set GSO segmentation size */ +#endif + +#ifndef ETH_MAX_MTU + #define ETH_MAX_MTU 0xFFFF +#endif + /* Forwards. */ static int send_parameters(struct iperf_test *test); static int get_parameters(struct iperf_test *test); @@ -853,6 +867,7 @@ iperf_parse_arguments(struct iperf_test *test, int argc, char **argv) {"connect-timeout", required_argument, NULL, OPT_CONNECT_TIMEOUT}, {"debug", no_argument, NULL, 'd'}, {"help", no_argument, NULL, 'h'}, + {"udp-gso", required_argument, NULL, OPT_UDP_GSO}, {NULL, 0, NULL, 0} }; int flag; @@ -1242,6 +1257,12 @@ iperf_parse_arguments(struct iperf_test *test, int argc, char **argv) case 'h': usage_long(stdout); exit(0); + case OPT_UDP_GSO: +#if defined(__linux__) + test->settings->udp_gso = 1; + test->settings->udp_gso_segsize = unit_atoi(optarg); +#endif + break; default: usage_long(stderr); exit(1); @@ -1376,6 +1397,48 @@ iperf_parse_arguments(struct iperf_test *test, int argc, char **argv) warning("Debug output (-d) may interfere with JSON output (-J)"); } +#if defined(__linux__) + if (test->settings->udp_gso) { + int prev_blksize = test->settings->blksize; + uint16_t iphdr_len = 0; + switch (test->settings->domain) { + case PF_UNSPEC: + warning("Domain is unspecified! Using IPv4 by default."); + case PF_INET: + iphdr_len = sizeof(struct iphdr); + break; + case PF_INET6: + iphdr_len = sizeof(struct ip6_hdr); + break; + default: + i_errno = IEDOMAIN; + perror("Current domain is unsupported by iperf GSO now. Disabling UDP GSO"); + test->settings->udp_gso = 0; + test->settings->udp_gso_segsize = 0; + break; + } + + if (iphdr_len) { + if (!test->settings->udp_gso_segsize) { + test->settings->udp_gso_segsize = test->settings->blksize; + test->settings->blksize = ETH_MAX_MTU - iphdr_len - sizeof(struct udphdr); + } + + if (test->settings->udp_gso_segsize > ETH_DATA_LEN - iphdr_len - sizeof(struct udphdr)) { + warning("UDP GSO segment size exceeds maximum. Using maximum value instead of current."); + test->settings->udp_gso_segsize = ETH_DATA_LEN - iphdr_len - sizeof(struct udphdr); + } + + if (test->settings->udp_gso_segsize >= test->settings->blksize) { + warning("UDP GSO segment size >= block size. The use of UDP GSO is impractival and amy cause errors. Disabling UDP GSO."); + test->settings->blksize = prev_blksize; + test->settings->udp_gso = 0; + test->settings->udp_gso_segsize = 0; + } + } + } +#endif + return 0; } diff --git a/src/iperf_api.h b/src/iperf_api.h index a077f2894..35f6fcd77 100644 --- a/src/iperf_api.h +++ b/src/iperf_api.h @@ -73,6 +73,7 @@ struct iperf_time; #define OPT_REPEATING_PAYLOAD 18 #define OPT_EXTRA_DATA 19 #define OPT_BIDIRECTIONAL 20 +#define OPT_UDP_GSO 25 /* states */ #define TEST_START 1 @@ -345,6 +346,8 @@ enum { IEBADFORMAT = 24, // Bad format argument to -f IEREVERSEBIDIR = 25, // Iperf cannot be both reverse and bidirectional IEBADPORT = 26, // Bad port number + IEDOMAIN = 27, // The specified domain doesn't supported by some functions (check perror) + IEUDPGSO = 28, // The configuration doesn't LSO for UDP protocol /* Test errors */ IENEWTEST = 100, // Unable to create a new test (check perror) IEINITTEST = 101, // Test initialization failed (check perror) diff --git a/src/iperf_error.c b/src/iperf_error.c index fd3cccca6..6322c691b 100644 --- a/src/iperf_error.c +++ b/src/iperf_error.c @@ -385,6 +385,13 @@ iperf_strerror(int int_errno) case IEREVERSEBIDIR: snprintf(errstr, len, "cannot be both reverse and bidirectional"); break; + case IEDOMAIN: + snprintf(errstr, len, "invalid domain"); + perr = 1; + break; + case IEUDPGSO: + snprintf(errstr, len, "your configuration doesn't support GSO for UDP protocol"); + break; } diff --git a/src/iperf_locale.c b/src/iperf_locale.c index 0d8a7ecc5..9d30b0c36 100644 --- a/src/iperf_locale.c +++ b/src/iperf_locale.c @@ -181,6 +181,11 @@ const char usage_longstr[] = "Usage: iperf3 [-s|-c host] [options]\n" " --rsa-public-key-path path to the RSA public key used to encrypt\n" " authentication credentials\n" #endif //HAVESSL + " --udp-gso N enables Generic Segmentation Offload for UDP protocol. \"N\" is a\n" + " segment size, it cannot be greater than MTU including headers.\n" + " Use \"--udp-gso 0\" to use blocksize (\"-l\" option) as a segment\n" + " size. Attention! Splitting into too many fragments can lead to a\n" + " crash of \"iperf\" or statistics errors. Supported only by Linux!\n" #ifdef NOT_YET_SUPPORTED /* still working on these */ #endif diff --git a/src/iperf_udp.c b/src/iperf_udp.c index 36235045f..3806d1195 100644 --- a/src/iperf_udp.c +++ b/src/iperf_udp.c @@ -34,6 +34,7 @@ #include #include #include +#include #ifdef HAVE_STDINT_H #include #endif @@ -55,6 +56,10 @@ # define PRIu64 "llu" #endif +#ifndef UDP_SEGMENT + #define UDP_SEGMENT 103 /* Set GSO segmentation size */ +#endif + /* iperf_udp_recv * * receives the data for UDP @@ -64,6 +69,7 @@ iperf_udp_recv(struct iperf_stream *sp) { uint32_t sec, usec; uint64_t pcount; + uint16_t udp_gso_seg; int r; int size = sp->settings->blksize; double transit = 0, d = 0; @@ -88,9 +94,11 @@ iperf_udp_recv(struct iperf_stream *sp) memcpy(&sec, sp->buffer, sizeof(sec)); memcpy(&usec, sp->buffer+4, sizeof(usec)); memcpy(&pcount, sp->buffer+8, sizeof(pcount)); + memcpy(&udp_gso_seg, sp->buffer+16, sizeof(udp_gso_seg)); sec = ntohl(sec); usec = ntohl(usec); pcount = be64toh(pcount); + udp_gso_seg = ntohs(udp_gso_seg); sent_time.secs = sec; sent_time.usecs = usec; } @@ -99,9 +107,11 @@ iperf_udp_recv(struct iperf_stream *sp) memcpy(&sec, sp->buffer, sizeof(sec)); memcpy(&usec, sp->buffer+4, sizeof(usec)); memcpy(&pc, sp->buffer+8, sizeof(pc)); + memcpy(&udp_gso_seg, sp->buffer+12, sizeof(udp_gso_seg)); sec = ntohl(sec); usec = ntohl(usec); pcount = ntohl(pc); + udp_gso_seg = ntohs(udp_gso_seg); sent_time.secs = sec; sent_time.usecs = usec; } @@ -122,6 +132,7 @@ iperf_udp_recv(struct iperf_stream *sp) * far (so we're expecting to see the packet with sequence number * sp->packet_count + 1 arrive next). */ + if (!udp_gso_seg) { if (pcount >= sp->packet_count + 1) { /* Forward, but is there a gap in sequence numbers? */ @@ -152,6 +163,7 @@ iperf_udp_recv(struct iperf_stream *sp) if (sp->test->debug) fprintf(stderr, "OUT OF ORDER - incoming packet sequence %" PRIu64 " but expected sequence %d on stream %d", pcount, sp->packet_count, sp->socket); } + } /* * jitter measurement @@ -202,31 +214,98 @@ iperf_udp_send(struct iperf_stream *sp) uint32_t sec, usec; uint64_t pcount; + uint16_t udp_gso_seg = 0; sec = htonl(before.secs); usec = htonl(before.usecs); pcount = htobe64(sp->packet_count); + udp_gso_seg = htons(udp_gso_seg); - memcpy(sp->buffer, &sec, sizeof(sec)); - memcpy(sp->buffer+4, &usec, sizeof(usec)); - memcpy(sp->buffer+8, &pcount, sizeof(pcount)); + if (sp->settings->udp_gso) { + /* Protection from lose segment with counters. */ + int i = 0; + int counters_len = sizeof(sec) + sizeof(usec) + sizeof(pcount) + sizeof(udp_gso_seg); + char* buf = sp->buffer; + do { + memcpy(buf, &sec, sizeof(sec)); + memcpy(buf+4, &usec, sizeof(usec)); + memcpy(buf+8, &pcount, sizeof(pcount)); + memcpy(buf+16, &udp_gso_seg, sizeof(udp_gso_seg)); + + buf += sp->settings->udp_gso_segsize; + i += sp->settings->udp_gso_segsize; + udp_gso_seg = htons(ntohs(udp_gso_seg) + 1); + } while (i + counters_len <= size); + } else { + memcpy(sp->buffer, &sec, sizeof(sec)); + memcpy(sp->buffer+4, &usec, sizeof(usec)); + memcpy(sp->buffer+8, &pcount, sizeof(pcount)); + memcpy(sp->buffer+16, &udp_gso_seg, sizeof(udp_gso_seg)); + } - } - else { + } + else { uint32_t sec, usec, pcount; + uint16_t udp_gso_seg = 0; sec = htonl(before.secs); usec = htonl(before.usecs); pcount = htonl(sp->packet_count); + udp_gso_seg = htons(udp_gso_seg); + + if (sp->settings->udp_gso) { + /* Protection from lose segment with counters. */ + int i = 0; + int counters_len = sizeof(sec) + sizeof(usec) + sizeof(pcount) + sizeof(udp_gso_seg); + char* buf = sp->buffer; + do { + memcpy(buf, &sec, sizeof(sec)); + memcpy(buf+4, &usec, sizeof(usec)); + memcpy(buf+8, &pcount, sizeof(pcount)); + memcpy(buf+12, &udp_gso_seg, sizeof(udp_gso_seg)); + + buf += sp->settings->udp_gso_segsize; + i += sp->settings->udp_gso_segsize; + udp_gso_seg = htons(ntohs(udp_gso_seg) + 1); + } while (i + counters_len <= size); + } else { + memcpy(sp->buffer, &sec, sizeof(sec)); + memcpy(sp->buffer+4, &usec, sizeof(usec)); + memcpy(sp->buffer+8, &pcount, sizeof(pcount)); + memcpy(sp->buffer+12, &udp_gso_seg, sizeof(udp_gso_seg)); + } - memcpy(sp->buffer, &sec, sizeof(sec)); - memcpy(sp->buffer+4, &usec, sizeof(usec)); - memcpy(sp->buffer+8, &pcount, sizeof(pcount)); - - } + } - r = Nwrite(sp->socket, sp->buffer, size, Pudp); + if (sp->settings->udp_gso) { +#if defined(__linux__) + char control[CMSG_SPACE(sizeof(sp->settings->udp_gso_segsize))] = { 0 }; + struct msghdr msg = { 0 }; + struct iovec iov = { 0 }; + struct cmsghdr *cm = NULL; + + iov.iov_base = sp->buffer; + iov.iov_len = size; + + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + + msg.msg_control = control; + msg.msg_controllen = sizeof(control); + + cm = CMSG_FIRSTHDR(&msg); + cm->cmsg_level = SOL_UDP; + cm->cmsg_type = UDP_SEGMENT; + cm->cmsg_len = CMSG_LEN(sizeof(sp->settings->udp_gso_segsize)); + uint16_t *valp = (uint16_t *)CMSG_DATA(cm); + *valp = sp->settings->udp_gso_segsize; + + r = Nsendmsg(sp->socket, &msg, Pudp); +#endif + } else { + r = Nwrite(sp->socket, sp->buffer, size, Pudp); + } if (r < 0) return r; @@ -553,6 +632,19 @@ iperf_udp_connect(struct iperf_test *test) return -1; } +#if defined(__linux__) + if (test->settings->udp_gso) { + uint16_t t_udp_gso_value; + socklen_t t_udp_gso_size; + if (getsockopt(s, SOL_UDP, UDP_SEGMENT, &t_udp_gso_value, &t_udp_gso_size)) { + i_errno = IEUDPGSO; + test->settings->udp_gso = 0; + test->settings->udp_gso_segsize = 0; + return -1; + } + } +#endif + return s; } diff --git a/src/net.c b/src/net.c index b475ed026..a0017b669 100644 --- a/src/net.c +++ b/src/net.c @@ -375,6 +375,45 @@ Nwrite(int fd, const char *buf, size_t count, int prot) } +int +Nsendmsg(int fd, struct msghdr *msg, int prot) +{ + register ssize_t r; + register ssize_t total_sent = 0; + ssize_t req = msg->msg_iov->iov_len; + + if (prot == SOCK_DGRAM) { + while (msg->msg_iov->iov_len > 0) { + r = sendmsg(fd, msg, NULL); + + if (r < 0) { + switch (errno) { + case EINTR: + case EAGAIN: +#if (EAGAIN != EWOULDBLOCK) + case EWOULDBLOCK: +#endif + break; + case ENOBUFS: + return NET_SOFTERROR; + default: + return NET_HARDERROR; + } + } else if (r == 0) { + return NET_HARDERROR; + } + + msg->msg_iov->iov_base += r; + msg->msg_iov->iov_len -= r; + total_sent += r; + msg->msg_flags = NULL; + } + + return total_sent; + } +} + + int has_sendfile(void) { diff --git a/src/net.h b/src/net.h index 3738d6ae8..b12af3e72 100644 --- a/src/net.h +++ b/src/net.h @@ -32,6 +32,7 @@ int netdial(int domain, int proto, char *local, int local_port, char *server, in int netannounce(int domain, int proto, char *local, int port); int Nread(int fd, char *buf, size_t count, int prot); int Nwrite(int fd, const char *buf, size_t count, int prot) /* __attribute__((hot)) */; +int Nsendmsg(int fd, struct msghdr *msg, int prot); int has_sendfile(void); int Nsendfile(int fromfd, int tofd, const char *buf, size_t count) /* __attribute__((hot)) */; int setnonblocking(int fd, int nonblocking);