1*bc36442eSArseniy Krasnov // SPDX-License-Identifier: GPL-2.0-only 2*bc36442eSArseniy Krasnov /* MSG_ZEROCOPY feature tests for vsock 3*bc36442eSArseniy Krasnov * 4*bc36442eSArseniy Krasnov * Copyright (C) 2023 SberDevices. 5*bc36442eSArseniy Krasnov * 6*bc36442eSArseniy Krasnov * Author: Arseniy Krasnov <avkrasnov@salutedevices.com> 7*bc36442eSArseniy Krasnov */ 8*bc36442eSArseniy Krasnov 9*bc36442eSArseniy Krasnov #include <stdio.h> 10*bc36442eSArseniy Krasnov #include <stdlib.h> 11*bc36442eSArseniy Krasnov #include <string.h> 12*bc36442eSArseniy Krasnov #include <sys/mman.h> 13*bc36442eSArseniy Krasnov #include <unistd.h> 14*bc36442eSArseniy Krasnov #include <poll.h> 15*bc36442eSArseniy Krasnov #include <linux/errqueue.h> 16*bc36442eSArseniy Krasnov #include <linux/kernel.h> 17*bc36442eSArseniy Krasnov #include <errno.h> 18*bc36442eSArseniy Krasnov 19*bc36442eSArseniy Krasnov #include "control.h" 20*bc36442eSArseniy Krasnov #include "vsock_test_zerocopy.h" 21*bc36442eSArseniy Krasnov #include "msg_zerocopy_common.h" 22*bc36442eSArseniy Krasnov 23*bc36442eSArseniy Krasnov #ifndef PAGE_SIZE 24*bc36442eSArseniy Krasnov #define PAGE_SIZE 4096 25*bc36442eSArseniy Krasnov #endif 26*bc36442eSArseniy Krasnov 27*bc36442eSArseniy Krasnov #define VSOCK_TEST_DATA_MAX_IOV 3 28*bc36442eSArseniy Krasnov 29*bc36442eSArseniy Krasnov struct vsock_test_data { 30*bc36442eSArseniy Krasnov /* This test case if for SOCK_STREAM only. */ 31*bc36442eSArseniy Krasnov bool stream_only; 32*bc36442eSArseniy Krasnov /* Data must be zerocopied. This field is checked against 33*bc36442eSArseniy Krasnov * field 'ee_code' of the 'struct sock_extended_err', which 34*bc36442eSArseniy Krasnov * contains bit to detect that zerocopy transmission was 35*bc36442eSArseniy Krasnov * fallbacked to copy mode. 36*bc36442eSArseniy Krasnov */ 37*bc36442eSArseniy Krasnov bool zerocopied; 38*bc36442eSArseniy Krasnov /* Enable SO_ZEROCOPY option on the socket. Without enabled 39*bc36442eSArseniy Krasnov * SO_ZEROCOPY, every MSG_ZEROCOPY transmission will behave 40*bc36442eSArseniy Krasnov * like without MSG_ZEROCOPY flag. 41*bc36442eSArseniy Krasnov */ 42*bc36442eSArseniy Krasnov bool so_zerocopy; 43*bc36442eSArseniy Krasnov /* 'errno' after 'sendmsg()' call. */ 44*bc36442eSArseniy Krasnov int sendmsg_errno; 45*bc36442eSArseniy Krasnov /* Number of valid elements in 'vecs'. */ 46*bc36442eSArseniy Krasnov int vecs_cnt; 47*bc36442eSArseniy Krasnov struct iovec vecs[VSOCK_TEST_DATA_MAX_IOV]; 48*bc36442eSArseniy Krasnov }; 49*bc36442eSArseniy Krasnov 50*bc36442eSArseniy Krasnov static struct vsock_test_data test_data_array[] = { 51*bc36442eSArseniy Krasnov /* Last element has non-page aligned size. */ 52*bc36442eSArseniy Krasnov { 53*bc36442eSArseniy Krasnov .zerocopied = true, 54*bc36442eSArseniy Krasnov .so_zerocopy = true, 55*bc36442eSArseniy Krasnov .sendmsg_errno = 0, 56*bc36442eSArseniy Krasnov .vecs_cnt = 3, 57*bc36442eSArseniy Krasnov { 58*bc36442eSArseniy Krasnov { NULL, PAGE_SIZE }, 59*bc36442eSArseniy Krasnov { NULL, PAGE_SIZE }, 60*bc36442eSArseniy Krasnov { NULL, 200 } 61*bc36442eSArseniy Krasnov } 62*bc36442eSArseniy Krasnov }, 63*bc36442eSArseniy Krasnov /* All elements have page aligned base and size. */ 64*bc36442eSArseniy Krasnov { 65*bc36442eSArseniy Krasnov .zerocopied = true, 66*bc36442eSArseniy Krasnov .so_zerocopy = true, 67*bc36442eSArseniy Krasnov .sendmsg_errno = 0, 68*bc36442eSArseniy Krasnov .vecs_cnt = 3, 69*bc36442eSArseniy Krasnov { 70*bc36442eSArseniy Krasnov { NULL, PAGE_SIZE }, 71*bc36442eSArseniy Krasnov { NULL, PAGE_SIZE * 2 }, 72*bc36442eSArseniy Krasnov { NULL, PAGE_SIZE * 3 } 73*bc36442eSArseniy Krasnov } 74*bc36442eSArseniy Krasnov }, 75*bc36442eSArseniy Krasnov /* All elements have page aligned base and size. But 76*bc36442eSArseniy Krasnov * data length is bigger than 64Kb. 77*bc36442eSArseniy Krasnov */ 78*bc36442eSArseniy Krasnov { 79*bc36442eSArseniy Krasnov .zerocopied = true, 80*bc36442eSArseniy Krasnov .so_zerocopy = true, 81*bc36442eSArseniy Krasnov .sendmsg_errno = 0, 82*bc36442eSArseniy Krasnov .vecs_cnt = 3, 83*bc36442eSArseniy Krasnov { 84*bc36442eSArseniy Krasnov { NULL, PAGE_SIZE * 16 }, 85*bc36442eSArseniy Krasnov { NULL, PAGE_SIZE * 16 }, 86*bc36442eSArseniy Krasnov { NULL, PAGE_SIZE * 16 } 87*bc36442eSArseniy Krasnov } 88*bc36442eSArseniy Krasnov }, 89*bc36442eSArseniy Krasnov /* Middle element has both non-page aligned base and size. */ 90*bc36442eSArseniy Krasnov { 91*bc36442eSArseniy Krasnov .zerocopied = true, 92*bc36442eSArseniy Krasnov .so_zerocopy = true, 93*bc36442eSArseniy Krasnov .sendmsg_errno = 0, 94*bc36442eSArseniy Krasnov .vecs_cnt = 3, 95*bc36442eSArseniy Krasnov { 96*bc36442eSArseniy Krasnov { NULL, PAGE_SIZE }, 97*bc36442eSArseniy Krasnov { (void *)1, 100 }, 98*bc36442eSArseniy Krasnov { NULL, PAGE_SIZE } 99*bc36442eSArseniy Krasnov } 100*bc36442eSArseniy Krasnov }, 101*bc36442eSArseniy Krasnov /* Middle element is unmapped. */ 102*bc36442eSArseniy Krasnov { 103*bc36442eSArseniy Krasnov .zerocopied = false, 104*bc36442eSArseniy Krasnov .so_zerocopy = true, 105*bc36442eSArseniy Krasnov .sendmsg_errno = ENOMEM, 106*bc36442eSArseniy Krasnov .vecs_cnt = 3, 107*bc36442eSArseniy Krasnov { 108*bc36442eSArseniy Krasnov { NULL, PAGE_SIZE }, 109*bc36442eSArseniy Krasnov { MAP_FAILED, PAGE_SIZE }, 110*bc36442eSArseniy Krasnov { NULL, PAGE_SIZE } 111*bc36442eSArseniy Krasnov } 112*bc36442eSArseniy Krasnov }, 113*bc36442eSArseniy Krasnov /* Valid data, but SO_ZEROCOPY is off. This 114*bc36442eSArseniy Krasnov * will trigger fallback to copy. 115*bc36442eSArseniy Krasnov */ 116*bc36442eSArseniy Krasnov { 117*bc36442eSArseniy Krasnov .zerocopied = false, 118*bc36442eSArseniy Krasnov .so_zerocopy = false, 119*bc36442eSArseniy Krasnov .sendmsg_errno = 0, 120*bc36442eSArseniy Krasnov .vecs_cnt = 1, 121*bc36442eSArseniy Krasnov { 122*bc36442eSArseniy Krasnov { NULL, PAGE_SIZE } 123*bc36442eSArseniy Krasnov } 124*bc36442eSArseniy Krasnov }, 125*bc36442eSArseniy Krasnov /* Valid data, but message is bigger than peer's 126*bc36442eSArseniy Krasnov * buffer, so this will trigger fallback to copy. 127*bc36442eSArseniy Krasnov * This test is for SOCK_STREAM only, because 128*bc36442eSArseniy Krasnov * for SOCK_SEQPACKET, 'sendmsg()' returns EMSGSIZE. 129*bc36442eSArseniy Krasnov */ 130*bc36442eSArseniy Krasnov { 131*bc36442eSArseniy Krasnov .stream_only = true, 132*bc36442eSArseniy Krasnov .zerocopied = false, 133*bc36442eSArseniy Krasnov .so_zerocopy = true, 134*bc36442eSArseniy Krasnov .sendmsg_errno = 0, 135*bc36442eSArseniy Krasnov .vecs_cnt = 1, 136*bc36442eSArseniy Krasnov { 137*bc36442eSArseniy Krasnov { NULL, 100 * PAGE_SIZE } 138*bc36442eSArseniy Krasnov } 139*bc36442eSArseniy Krasnov }, 140*bc36442eSArseniy Krasnov }; 141*bc36442eSArseniy Krasnov 142*bc36442eSArseniy Krasnov #define POLL_TIMEOUT_MS 100 143*bc36442eSArseniy Krasnov 144*bc36442eSArseniy Krasnov static void test_client(const struct test_opts *opts, 145*bc36442eSArseniy Krasnov const struct vsock_test_data *test_data, 146*bc36442eSArseniy Krasnov bool sock_seqpacket) 147*bc36442eSArseniy Krasnov { 148*bc36442eSArseniy Krasnov struct pollfd fds = { 0 }; 149*bc36442eSArseniy Krasnov struct msghdr msg = { 0 }; 150*bc36442eSArseniy Krasnov ssize_t sendmsg_res; 151*bc36442eSArseniy Krasnov struct iovec *iovec; 152*bc36442eSArseniy Krasnov int fd; 153*bc36442eSArseniy Krasnov 154*bc36442eSArseniy Krasnov if (sock_seqpacket) 155*bc36442eSArseniy Krasnov fd = vsock_seqpacket_connect(opts->peer_cid, 1234); 156*bc36442eSArseniy Krasnov else 157*bc36442eSArseniy Krasnov fd = vsock_stream_connect(opts->peer_cid, 1234); 158*bc36442eSArseniy Krasnov 159*bc36442eSArseniy Krasnov if (fd < 0) { 160*bc36442eSArseniy Krasnov perror("connect"); 161*bc36442eSArseniy Krasnov exit(EXIT_FAILURE); 162*bc36442eSArseniy Krasnov } 163*bc36442eSArseniy Krasnov 164*bc36442eSArseniy Krasnov if (test_data->so_zerocopy) 165*bc36442eSArseniy Krasnov enable_so_zerocopy(fd); 166*bc36442eSArseniy Krasnov 167*bc36442eSArseniy Krasnov iovec = alloc_test_iovec(test_data->vecs, test_data->vecs_cnt); 168*bc36442eSArseniy Krasnov 169*bc36442eSArseniy Krasnov msg.msg_iov = iovec; 170*bc36442eSArseniy Krasnov msg.msg_iovlen = test_data->vecs_cnt; 171*bc36442eSArseniy Krasnov 172*bc36442eSArseniy Krasnov errno = 0; 173*bc36442eSArseniy Krasnov 174*bc36442eSArseniy Krasnov sendmsg_res = sendmsg(fd, &msg, MSG_ZEROCOPY); 175*bc36442eSArseniy Krasnov if (errno != test_data->sendmsg_errno) { 176*bc36442eSArseniy Krasnov fprintf(stderr, "expected 'errno' == %i, got %i\n", 177*bc36442eSArseniy Krasnov test_data->sendmsg_errno, errno); 178*bc36442eSArseniy Krasnov exit(EXIT_FAILURE); 179*bc36442eSArseniy Krasnov } 180*bc36442eSArseniy Krasnov 181*bc36442eSArseniy Krasnov if (!errno) { 182*bc36442eSArseniy Krasnov if (sendmsg_res != iovec_bytes(iovec, test_data->vecs_cnt)) { 183*bc36442eSArseniy Krasnov fprintf(stderr, "expected 'sendmsg()' == %li, got %li\n", 184*bc36442eSArseniy Krasnov iovec_bytes(iovec, test_data->vecs_cnt), 185*bc36442eSArseniy Krasnov sendmsg_res); 186*bc36442eSArseniy Krasnov exit(EXIT_FAILURE); 187*bc36442eSArseniy Krasnov } 188*bc36442eSArseniy Krasnov } 189*bc36442eSArseniy Krasnov 190*bc36442eSArseniy Krasnov fds.fd = fd; 191*bc36442eSArseniy Krasnov fds.events = 0; 192*bc36442eSArseniy Krasnov 193*bc36442eSArseniy Krasnov if (poll(&fds, 1, POLL_TIMEOUT_MS) < 0) { 194*bc36442eSArseniy Krasnov perror("poll"); 195*bc36442eSArseniy Krasnov exit(EXIT_FAILURE); 196*bc36442eSArseniy Krasnov } 197*bc36442eSArseniy Krasnov 198*bc36442eSArseniy Krasnov if (fds.revents & POLLERR) { 199*bc36442eSArseniy Krasnov vsock_recv_completion(fd, &test_data->zerocopied); 200*bc36442eSArseniy Krasnov } else if (test_data->so_zerocopy && !test_data->sendmsg_errno) { 201*bc36442eSArseniy Krasnov /* If we don't have data in the error queue, but 202*bc36442eSArseniy Krasnov * SO_ZEROCOPY was enabled and 'sendmsg()' was 203*bc36442eSArseniy Krasnov * successful - this is an error. 204*bc36442eSArseniy Krasnov */ 205*bc36442eSArseniy Krasnov fprintf(stderr, "POLLERR expected\n"); 206*bc36442eSArseniy Krasnov exit(EXIT_FAILURE); 207*bc36442eSArseniy Krasnov } 208*bc36442eSArseniy Krasnov 209*bc36442eSArseniy Krasnov if (!test_data->sendmsg_errno) 210*bc36442eSArseniy Krasnov control_writeulong(iovec_hash_djb2(iovec, test_data->vecs_cnt)); 211*bc36442eSArseniy Krasnov else 212*bc36442eSArseniy Krasnov control_writeulong(0); 213*bc36442eSArseniy Krasnov 214*bc36442eSArseniy Krasnov control_writeln("DONE"); 215*bc36442eSArseniy Krasnov free_test_iovec(test_data->vecs, iovec, test_data->vecs_cnt); 216*bc36442eSArseniy Krasnov close(fd); 217*bc36442eSArseniy Krasnov } 218*bc36442eSArseniy Krasnov 219*bc36442eSArseniy Krasnov void test_stream_msgzcopy_client(const struct test_opts *opts) 220*bc36442eSArseniy Krasnov { 221*bc36442eSArseniy Krasnov int i; 222*bc36442eSArseniy Krasnov 223*bc36442eSArseniy Krasnov for (i = 0; i < ARRAY_SIZE(test_data_array); i++) 224*bc36442eSArseniy Krasnov test_client(opts, &test_data_array[i], false); 225*bc36442eSArseniy Krasnov } 226*bc36442eSArseniy Krasnov 227*bc36442eSArseniy Krasnov void test_seqpacket_msgzcopy_client(const struct test_opts *opts) 228*bc36442eSArseniy Krasnov { 229*bc36442eSArseniy Krasnov int i; 230*bc36442eSArseniy Krasnov 231*bc36442eSArseniy Krasnov for (i = 0; i < ARRAY_SIZE(test_data_array); i++) { 232*bc36442eSArseniy Krasnov if (test_data_array[i].stream_only) 233*bc36442eSArseniy Krasnov continue; 234*bc36442eSArseniy Krasnov 235*bc36442eSArseniy Krasnov test_client(opts, &test_data_array[i], true); 236*bc36442eSArseniy Krasnov } 237*bc36442eSArseniy Krasnov } 238*bc36442eSArseniy Krasnov 239*bc36442eSArseniy Krasnov static void test_server(const struct test_opts *opts, 240*bc36442eSArseniy Krasnov const struct vsock_test_data *test_data, 241*bc36442eSArseniy Krasnov bool sock_seqpacket) 242*bc36442eSArseniy Krasnov { 243*bc36442eSArseniy Krasnov unsigned long remote_hash; 244*bc36442eSArseniy Krasnov unsigned long local_hash; 245*bc36442eSArseniy Krasnov ssize_t total_bytes_rec; 246*bc36442eSArseniy Krasnov unsigned char *data; 247*bc36442eSArseniy Krasnov size_t data_len; 248*bc36442eSArseniy Krasnov int fd; 249*bc36442eSArseniy Krasnov 250*bc36442eSArseniy Krasnov if (sock_seqpacket) 251*bc36442eSArseniy Krasnov fd = vsock_seqpacket_accept(VMADDR_CID_ANY, 1234, NULL); 252*bc36442eSArseniy Krasnov else 253*bc36442eSArseniy Krasnov fd = vsock_stream_accept(VMADDR_CID_ANY, 1234, NULL); 254*bc36442eSArseniy Krasnov 255*bc36442eSArseniy Krasnov if (fd < 0) { 256*bc36442eSArseniy Krasnov perror("accept"); 257*bc36442eSArseniy Krasnov exit(EXIT_FAILURE); 258*bc36442eSArseniy Krasnov } 259*bc36442eSArseniy Krasnov 260*bc36442eSArseniy Krasnov data_len = iovec_bytes(test_data->vecs, test_data->vecs_cnt); 261*bc36442eSArseniy Krasnov 262*bc36442eSArseniy Krasnov data = malloc(data_len); 263*bc36442eSArseniy Krasnov if (!data) { 264*bc36442eSArseniy Krasnov perror("malloc"); 265*bc36442eSArseniy Krasnov exit(EXIT_FAILURE); 266*bc36442eSArseniy Krasnov } 267*bc36442eSArseniy Krasnov 268*bc36442eSArseniy Krasnov total_bytes_rec = 0; 269*bc36442eSArseniy Krasnov 270*bc36442eSArseniy Krasnov while (total_bytes_rec != data_len) { 271*bc36442eSArseniy Krasnov ssize_t bytes_rec; 272*bc36442eSArseniy Krasnov 273*bc36442eSArseniy Krasnov bytes_rec = read(fd, data + total_bytes_rec, 274*bc36442eSArseniy Krasnov data_len - total_bytes_rec); 275*bc36442eSArseniy Krasnov if (bytes_rec <= 0) 276*bc36442eSArseniy Krasnov break; 277*bc36442eSArseniy Krasnov 278*bc36442eSArseniy Krasnov total_bytes_rec += bytes_rec; 279*bc36442eSArseniy Krasnov } 280*bc36442eSArseniy Krasnov 281*bc36442eSArseniy Krasnov if (test_data->sendmsg_errno == 0) 282*bc36442eSArseniy Krasnov local_hash = hash_djb2(data, data_len); 283*bc36442eSArseniy Krasnov else 284*bc36442eSArseniy Krasnov local_hash = 0; 285*bc36442eSArseniy Krasnov 286*bc36442eSArseniy Krasnov free(data); 287*bc36442eSArseniy Krasnov 288*bc36442eSArseniy Krasnov /* Waiting for some result. */ 289*bc36442eSArseniy Krasnov remote_hash = control_readulong(); 290*bc36442eSArseniy Krasnov if (remote_hash != local_hash) { 291*bc36442eSArseniy Krasnov fprintf(stderr, "hash mismatch\n"); 292*bc36442eSArseniy Krasnov exit(EXIT_FAILURE); 293*bc36442eSArseniy Krasnov } 294*bc36442eSArseniy Krasnov 295*bc36442eSArseniy Krasnov control_expectln("DONE"); 296*bc36442eSArseniy Krasnov close(fd); 297*bc36442eSArseniy Krasnov } 298*bc36442eSArseniy Krasnov 299*bc36442eSArseniy Krasnov void test_stream_msgzcopy_server(const struct test_opts *opts) 300*bc36442eSArseniy Krasnov { 301*bc36442eSArseniy Krasnov int i; 302*bc36442eSArseniy Krasnov 303*bc36442eSArseniy Krasnov for (i = 0; i < ARRAY_SIZE(test_data_array); i++) 304*bc36442eSArseniy Krasnov test_server(opts, &test_data_array[i], false); 305*bc36442eSArseniy Krasnov } 306*bc36442eSArseniy Krasnov 307*bc36442eSArseniy Krasnov void test_seqpacket_msgzcopy_server(const struct test_opts *opts) 308*bc36442eSArseniy Krasnov { 309*bc36442eSArseniy Krasnov int i; 310*bc36442eSArseniy Krasnov 311*bc36442eSArseniy Krasnov for (i = 0; i < ARRAY_SIZE(test_data_array); i++) { 312*bc36442eSArseniy Krasnov if (test_data_array[i].stream_only) 313*bc36442eSArseniy Krasnov continue; 314*bc36442eSArseniy Krasnov 315*bc36442eSArseniy Krasnov test_server(opts, &test_data_array[i], true); 316*bc36442eSArseniy Krasnov } 317*bc36442eSArseniy Krasnov } 318*bc36442eSArseniy Krasnov 319*bc36442eSArseniy Krasnov void test_stream_msgzcopy_empty_errq_client(const struct test_opts *opts) 320*bc36442eSArseniy Krasnov { 321*bc36442eSArseniy Krasnov struct msghdr msg = { 0 }; 322*bc36442eSArseniy Krasnov char cmsg_data[128]; 323*bc36442eSArseniy Krasnov ssize_t res; 324*bc36442eSArseniy Krasnov int fd; 325*bc36442eSArseniy Krasnov 326*bc36442eSArseniy Krasnov fd = vsock_stream_connect(opts->peer_cid, 1234); 327*bc36442eSArseniy Krasnov if (fd < 0) { 328*bc36442eSArseniy Krasnov perror("connect"); 329*bc36442eSArseniy Krasnov exit(EXIT_FAILURE); 330*bc36442eSArseniy Krasnov } 331*bc36442eSArseniy Krasnov 332*bc36442eSArseniy Krasnov msg.msg_control = cmsg_data; 333*bc36442eSArseniy Krasnov msg.msg_controllen = sizeof(cmsg_data); 334*bc36442eSArseniy Krasnov 335*bc36442eSArseniy Krasnov res = recvmsg(fd, &msg, MSG_ERRQUEUE); 336*bc36442eSArseniy Krasnov if (res != -1) { 337*bc36442eSArseniy Krasnov fprintf(stderr, "expected 'recvmsg(2)' failure, got %zi\n", 338*bc36442eSArseniy Krasnov res); 339*bc36442eSArseniy Krasnov exit(EXIT_FAILURE); 340*bc36442eSArseniy Krasnov } 341*bc36442eSArseniy Krasnov 342*bc36442eSArseniy Krasnov control_writeln("DONE"); 343*bc36442eSArseniy Krasnov close(fd); 344*bc36442eSArseniy Krasnov } 345*bc36442eSArseniy Krasnov 346*bc36442eSArseniy Krasnov void test_stream_msgzcopy_empty_errq_server(const struct test_opts *opts) 347*bc36442eSArseniy Krasnov { 348*bc36442eSArseniy Krasnov int fd; 349*bc36442eSArseniy Krasnov 350*bc36442eSArseniy Krasnov fd = vsock_stream_accept(VMADDR_CID_ANY, 1234, NULL); 351*bc36442eSArseniy Krasnov if (fd < 0) { 352*bc36442eSArseniy Krasnov perror("accept"); 353*bc36442eSArseniy Krasnov exit(EXIT_FAILURE); 354*bc36442eSArseniy Krasnov } 355*bc36442eSArseniy Krasnov 356*bc36442eSArseniy Krasnov control_expectln("DONE"); 357*bc36442eSArseniy Krasnov close(fd); 358*bc36442eSArseniy Krasnov } 359