1bc36442eSArseniy Krasnov // SPDX-License-Identifier: GPL-2.0-only
2bc36442eSArseniy Krasnov /* MSG_ZEROCOPY feature tests for vsock
3bc36442eSArseniy Krasnov *
4bc36442eSArseniy Krasnov * Copyright (C) 2023 SberDevices.
5bc36442eSArseniy Krasnov *
6bc36442eSArseniy Krasnov * Author: Arseniy Krasnov <avkrasnov@salutedevices.com>
7bc36442eSArseniy Krasnov */
8bc36442eSArseniy Krasnov
9bc36442eSArseniy Krasnov #include <stdio.h>
10bc36442eSArseniy Krasnov #include <stdlib.h>
11bc36442eSArseniy Krasnov #include <string.h>
12bc36442eSArseniy Krasnov #include <sys/mman.h>
13bc36442eSArseniy Krasnov #include <unistd.h>
14bc36442eSArseniy Krasnov #include <poll.h>
15bc36442eSArseniy Krasnov #include <linux/errqueue.h>
16bc36442eSArseniy Krasnov #include <linux/kernel.h>
17bc36442eSArseniy Krasnov #include <errno.h>
18bc36442eSArseniy Krasnov
19bc36442eSArseniy Krasnov #include "control.h"
20bc36442eSArseniy Krasnov #include "vsock_test_zerocopy.h"
21bc36442eSArseniy Krasnov #include "msg_zerocopy_common.h"
22bc36442eSArseniy Krasnov
23bc36442eSArseniy Krasnov #ifndef PAGE_SIZE
24bc36442eSArseniy Krasnov #define PAGE_SIZE 4096
25bc36442eSArseniy Krasnov #endif
26bc36442eSArseniy Krasnov
27bc36442eSArseniy Krasnov #define VSOCK_TEST_DATA_MAX_IOV 3
28bc36442eSArseniy Krasnov
29bc36442eSArseniy Krasnov struct vsock_test_data {
30bc36442eSArseniy Krasnov /* This test case if for SOCK_STREAM only. */
31bc36442eSArseniy Krasnov bool stream_only;
32bc36442eSArseniy Krasnov /* Data must be zerocopied. This field is checked against
33bc36442eSArseniy Krasnov * field 'ee_code' of the 'struct sock_extended_err', which
34bc36442eSArseniy Krasnov * contains bit to detect that zerocopy transmission was
35bc36442eSArseniy Krasnov * fallbacked to copy mode.
36bc36442eSArseniy Krasnov */
37bc36442eSArseniy Krasnov bool zerocopied;
38bc36442eSArseniy Krasnov /* Enable SO_ZEROCOPY option on the socket. Without enabled
39bc36442eSArseniy Krasnov * SO_ZEROCOPY, every MSG_ZEROCOPY transmission will behave
40bc36442eSArseniy Krasnov * like without MSG_ZEROCOPY flag.
41bc36442eSArseniy Krasnov */
42bc36442eSArseniy Krasnov bool so_zerocopy;
43bc36442eSArseniy Krasnov /* 'errno' after 'sendmsg()' call. */
44bc36442eSArseniy Krasnov int sendmsg_errno;
45bc36442eSArseniy Krasnov /* Number of valid elements in 'vecs'. */
46bc36442eSArseniy Krasnov int vecs_cnt;
47bc36442eSArseniy Krasnov struct iovec vecs[VSOCK_TEST_DATA_MAX_IOV];
48bc36442eSArseniy Krasnov };
49bc36442eSArseniy Krasnov
50bc36442eSArseniy Krasnov static struct vsock_test_data test_data_array[] = {
51bc36442eSArseniy Krasnov /* Last element has non-page aligned size. */
52bc36442eSArseniy Krasnov {
53bc36442eSArseniy Krasnov .zerocopied = true,
54bc36442eSArseniy Krasnov .so_zerocopy = true,
55bc36442eSArseniy Krasnov .sendmsg_errno = 0,
56bc36442eSArseniy Krasnov .vecs_cnt = 3,
57bc36442eSArseniy Krasnov {
58bc36442eSArseniy Krasnov { NULL, PAGE_SIZE },
59bc36442eSArseniy Krasnov { NULL, PAGE_SIZE },
60bc36442eSArseniy Krasnov { NULL, 200 }
61bc36442eSArseniy Krasnov }
62bc36442eSArseniy Krasnov },
63bc36442eSArseniy Krasnov /* All elements have page aligned base and size. */
64bc36442eSArseniy Krasnov {
65bc36442eSArseniy Krasnov .zerocopied = true,
66bc36442eSArseniy Krasnov .so_zerocopy = true,
67bc36442eSArseniy Krasnov .sendmsg_errno = 0,
68bc36442eSArseniy Krasnov .vecs_cnt = 3,
69bc36442eSArseniy Krasnov {
70bc36442eSArseniy Krasnov { NULL, PAGE_SIZE },
71bc36442eSArseniy Krasnov { NULL, PAGE_SIZE * 2 },
72bc36442eSArseniy Krasnov { NULL, PAGE_SIZE * 3 }
73bc36442eSArseniy Krasnov }
74bc36442eSArseniy Krasnov },
75bc36442eSArseniy Krasnov /* All elements have page aligned base and size. But
76bc36442eSArseniy Krasnov * data length is bigger than 64Kb.
77bc36442eSArseniy Krasnov */
78bc36442eSArseniy Krasnov {
79bc36442eSArseniy Krasnov .zerocopied = true,
80bc36442eSArseniy Krasnov .so_zerocopy = true,
81bc36442eSArseniy Krasnov .sendmsg_errno = 0,
82bc36442eSArseniy Krasnov .vecs_cnt = 3,
83bc36442eSArseniy Krasnov {
84bc36442eSArseniy Krasnov { NULL, PAGE_SIZE * 16 },
85bc36442eSArseniy Krasnov { NULL, PAGE_SIZE * 16 },
86bc36442eSArseniy Krasnov { NULL, PAGE_SIZE * 16 }
87bc36442eSArseniy Krasnov }
88bc36442eSArseniy Krasnov },
89bc36442eSArseniy Krasnov /* Middle element has both non-page aligned base and size. */
90bc36442eSArseniy Krasnov {
91bc36442eSArseniy Krasnov .zerocopied = true,
92bc36442eSArseniy Krasnov .so_zerocopy = true,
93bc36442eSArseniy Krasnov .sendmsg_errno = 0,
94bc36442eSArseniy Krasnov .vecs_cnt = 3,
95bc36442eSArseniy Krasnov {
96bc36442eSArseniy Krasnov { NULL, PAGE_SIZE },
97bc36442eSArseniy Krasnov { (void *)1, 100 },
98bc36442eSArseniy Krasnov { NULL, PAGE_SIZE }
99bc36442eSArseniy Krasnov }
100bc36442eSArseniy Krasnov },
101bc36442eSArseniy Krasnov /* Middle element is unmapped. */
102bc36442eSArseniy Krasnov {
103bc36442eSArseniy Krasnov .zerocopied = false,
104bc36442eSArseniy Krasnov .so_zerocopy = true,
105bc36442eSArseniy Krasnov .sendmsg_errno = ENOMEM,
106bc36442eSArseniy Krasnov .vecs_cnt = 3,
107bc36442eSArseniy Krasnov {
108bc36442eSArseniy Krasnov { NULL, PAGE_SIZE },
109bc36442eSArseniy Krasnov { MAP_FAILED, PAGE_SIZE },
110bc36442eSArseniy Krasnov { NULL, PAGE_SIZE }
111bc36442eSArseniy Krasnov }
112bc36442eSArseniy Krasnov },
113bc36442eSArseniy Krasnov /* Valid data, but SO_ZEROCOPY is off. This
114bc36442eSArseniy Krasnov * will trigger fallback to copy.
115bc36442eSArseniy Krasnov */
116bc36442eSArseniy Krasnov {
117bc36442eSArseniy Krasnov .zerocopied = false,
118bc36442eSArseniy Krasnov .so_zerocopy = false,
119bc36442eSArseniy Krasnov .sendmsg_errno = 0,
120bc36442eSArseniy Krasnov .vecs_cnt = 1,
121bc36442eSArseniy Krasnov {
122bc36442eSArseniy Krasnov { NULL, PAGE_SIZE }
123bc36442eSArseniy Krasnov }
124bc36442eSArseniy Krasnov },
125bc36442eSArseniy Krasnov /* Valid data, but message is bigger than peer's
126bc36442eSArseniy Krasnov * buffer, so this will trigger fallback to copy.
127bc36442eSArseniy Krasnov * This test is for SOCK_STREAM only, because
128bc36442eSArseniy Krasnov * for SOCK_SEQPACKET, 'sendmsg()' returns EMSGSIZE.
129bc36442eSArseniy Krasnov */
130bc36442eSArseniy Krasnov {
131bc36442eSArseniy Krasnov .stream_only = true,
132bc36442eSArseniy Krasnov .zerocopied = false,
133bc36442eSArseniy Krasnov .so_zerocopy = true,
134bc36442eSArseniy Krasnov .sendmsg_errno = 0,
135bc36442eSArseniy Krasnov .vecs_cnt = 1,
136bc36442eSArseniy Krasnov {
137bc36442eSArseniy Krasnov { NULL, 100 * PAGE_SIZE }
138bc36442eSArseniy Krasnov }
139bc36442eSArseniy Krasnov },
140bc36442eSArseniy Krasnov };
141bc36442eSArseniy Krasnov
142bc36442eSArseniy Krasnov #define POLL_TIMEOUT_MS 100
143bc36442eSArseniy Krasnov
test_client(const struct test_opts * opts,const struct vsock_test_data * test_data,bool sock_seqpacket)144bc36442eSArseniy Krasnov static void test_client(const struct test_opts *opts,
145bc36442eSArseniy Krasnov const struct vsock_test_data *test_data,
146bc36442eSArseniy Krasnov bool sock_seqpacket)
147bc36442eSArseniy Krasnov {
148bc36442eSArseniy Krasnov struct pollfd fds = { 0 };
149bc36442eSArseniy Krasnov struct msghdr msg = { 0 };
150bc36442eSArseniy Krasnov ssize_t sendmsg_res;
151bc36442eSArseniy Krasnov struct iovec *iovec;
152bc36442eSArseniy Krasnov int fd;
153bc36442eSArseniy Krasnov
154bc36442eSArseniy Krasnov if (sock_seqpacket)
155*e18c7092SArseniy Krasnov fd = vsock_seqpacket_connect(opts->peer_cid, opts->peer_port);
156bc36442eSArseniy Krasnov else
157*e18c7092SArseniy Krasnov fd = vsock_stream_connect(opts->peer_cid, opts->peer_port);
158bc36442eSArseniy Krasnov
159bc36442eSArseniy Krasnov if (fd < 0) {
160bc36442eSArseniy Krasnov perror("connect");
161bc36442eSArseniy Krasnov exit(EXIT_FAILURE);
162bc36442eSArseniy Krasnov }
163bc36442eSArseniy Krasnov
164bc36442eSArseniy Krasnov if (test_data->so_zerocopy)
165bc36442eSArseniy Krasnov enable_so_zerocopy(fd);
166bc36442eSArseniy Krasnov
167bc36442eSArseniy Krasnov iovec = alloc_test_iovec(test_data->vecs, test_data->vecs_cnt);
168bc36442eSArseniy Krasnov
169bc36442eSArseniy Krasnov msg.msg_iov = iovec;
170bc36442eSArseniy Krasnov msg.msg_iovlen = test_data->vecs_cnt;
171bc36442eSArseniy Krasnov
172bc36442eSArseniy Krasnov errno = 0;
173bc36442eSArseniy Krasnov
174bc36442eSArseniy Krasnov sendmsg_res = sendmsg(fd, &msg, MSG_ZEROCOPY);
175bc36442eSArseniy Krasnov if (errno != test_data->sendmsg_errno) {
176bc36442eSArseniy Krasnov fprintf(stderr, "expected 'errno' == %i, got %i\n",
177bc36442eSArseniy Krasnov test_data->sendmsg_errno, errno);
178bc36442eSArseniy Krasnov exit(EXIT_FAILURE);
179bc36442eSArseniy Krasnov }
180bc36442eSArseniy Krasnov
181bc36442eSArseniy Krasnov if (!errno) {
182bc36442eSArseniy Krasnov if (sendmsg_res != iovec_bytes(iovec, test_data->vecs_cnt)) {
183bc36442eSArseniy Krasnov fprintf(stderr, "expected 'sendmsg()' == %li, got %li\n",
184bc36442eSArseniy Krasnov iovec_bytes(iovec, test_data->vecs_cnt),
185bc36442eSArseniy Krasnov sendmsg_res);
186bc36442eSArseniy Krasnov exit(EXIT_FAILURE);
187bc36442eSArseniy Krasnov }
188bc36442eSArseniy Krasnov }
189bc36442eSArseniy Krasnov
190bc36442eSArseniy Krasnov fds.fd = fd;
191bc36442eSArseniy Krasnov fds.events = 0;
192bc36442eSArseniy Krasnov
193bc36442eSArseniy Krasnov if (poll(&fds, 1, POLL_TIMEOUT_MS) < 0) {
194bc36442eSArseniy Krasnov perror("poll");
195bc36442eSArseniy Krasnov exit(EXIT_FAILURE);
196bc36442eSArseniy Krasnov }
197bc36442eSArseniy Krasnov
198bc36442eSArseniy Krasnov if (fds.revents & POLLERR) {
199bc36442eSArseniy Krasnov vsock_recv_completion(fd, &test_data->zerocopied);
200bc36442eSArseniy Krasnov } else if (test_data->so_zerocopy && !test_data->sendmsg_errno) {
201bc36442eSArseniy Krasnov /* If we don't have data in the error queue, but
202bc36442eSArseniy Krasnov * SO_ZEROCOPY was enabled and 'sendmsg()' was
203bc36442eSArseniy Krasnov * successful - this is an error.
204bc36442eSArseniy Krasnov */
205bc36442eSArseniy Krasnov fprintf(stderr, "POLLERR expected\n");
206bc36442eSArseniy Krasnov exit(EXIT_FAILURE);
207bc36442eSArseniy Krasnov }
208bc36442eSArseniy Krasnov
209bc36442eSArseniy Krasnov if (!test_data->sendmsg_errno)
210bc36442eSArseniy Krasnov control_writeulong(iovec_hash_djb2(iovec, test_data->vecs_cnt));
211bc36442eSArseniy Krasnov else
212bc36442eSArseniy Krasnov control_writeulong(0);
213bc36442eSArseniy Krasnov
214bc36442eSArseniy Krasnov control_writeln("DONE");
215bc36442eSArseniy Krasnov free_test_iovec(test_data->vecs, iovec, test_data->vecs_cnt);
216bc36442eSArseniy Krasnov close(fd);
217bc36442eSArseniy Krasnov }
218bc36442eSArseniy Krasnov
test_stream_msgzcopy_client(const struct test_opts * opts)219bc36442eSArseniy Krasnov void test_stream_msgzcopy_client(const struct test_opts *opts)
220bc36442eSArseniy Krasnov {
221bc36442eSArseniy Krasnov int i;
222bc36442eSArseniy Krasnov
223bc36442eSArseniy Krasnov for (i = 0; i < ARRAY_SIZE(test_data_array); i++)
224bc36442eSArseniy Krasnov test_client(opts, &test_data_array[i], false);
225bc36442eSArseniy Krasnov }
226bc36442eSArseniy Krasnov
test_seqpacket_msgzcopy_client(const struct test_opts * opts)227bc36442eSArseniy Krasnov void test_seqpacket_msgzcopy_client(const struct test_opts *opts)
228bc36442eSArseniy Krasnov {
229bc36442eSArseniy Krasnov int i;
230bc36442eSArseniy Krasnov
231bc36442eSArseniy Krasnov for (i = 0; i < ARRAY_SIZE(test_data_array); i++) {
232bc36442eSArseniy Krasnov if (test_data_array[i].stream_only)
233bc36442eSArseniy Krasnov continue;
234bc36442eSArseniy Krasnov
235bc36442eSArseniy Krasnov test_client(opts, &test_data_array[i], true);
236bc36442eSArseniy Krasnov }
237bc36442eSArseniy Krasnov }
238bc36442eSArseniy Krasnov
test_server(const struct test_opts * opts,const struct vsock_test_data * test_data,bool sock_seqpacket)239bc36442eSArseniy Krasnov static void test_server(const struct test_opts *opts,
240bc36442eSArseniy Krasnov const struct vsock_test_data *test_data,
241bc36442eSArseniy Krasnov bool sock_seqpacket)
242bc36442eSArseniy Krasnov {
243bc36442eSArseniy Krasnov unsigned long remote_hash;
244bc36442eSArseniy Krasnov unsigned long local_hash;
245bc36442eSArseniy Krasnov ssize_t total_bytes_rec;
246bc36442eSArseniy Krasnov unsigned char *data;
247bc36442eSArseniy Krasnov size_t data_len;
248bc36442eSArseniy Krasnov int fd;
249bc36442eSArseniy Krasnov
250bc36442eSArseniy Krasnov if (sock_seqpacket)
251*e18c7092SArseniy Krasnov fd = vsock_seqpacket_accept(VMADDR_CID_ANY, opts->peer_port, NULL);
252bc36442eSArseniy Krasnov else
253*e18c7092SArseniy Krasnov fd = vsock_stream_accept(VMADDR_CID_ANY, opts->peer_port, NULL);
254bc36442eSArseniy Krasnov
255bc36442eSArseniy Krasnov if (fd < 0) {
256bc36442eSArseniy Krasnov perror("accept");
257bc36442eSArseniy Krasnov exit(EXIT_FAILURE);
258bc36442eSArseniy Krasnov }
259bc36442eSArseniy Krasnov
260bc36442eSArseniy Krasnov data_len = iovec_bytes(test_data->vecs, test_data->vecs_cnt);
261bc36442eSArseniy Krasnov
262bc36442eSArseniy Krasnov data = malloc(data_len);
263bc36442eSArseniy Krasnov if (!data) {
264bc36442eSArseniy Krasnov perror("malloc");
265bc36442eSArseniy Krasnov exit(EXIT_FAILURE);
266bc36442eSArseniy Krasnov }
267bc36442eSArseniy Krasnov
268bc36442eSArseniy Krasnov total_bytes_rec = 0;
269bc36442eSArseniy Krasnov
270bc36442eSArseniy Krasnov while (total_bytes_rec != data_len) {
271bc36442eSArseniy Krasnov ssize_t bytes_rec;
272bc36442eSArseniy Krasnov
273bc36442eSArseniy Krasnov bytes_rec = read(fd, data + total_bytes_rec,
274bc36442eSArseniy Krasnov data_len - total_bytes_rec);
275bc36442eSArseniy Krasnov if (bytes_rec <= 0)
276bc36442eSArseniy Krasnov break;
277bc36442eSArseniy Krasnov
278bc36442eSArseniy Krasnov total_bytes_rec += bytes_rec;
279bc36442eSArseniy Krasnov }
280bc36442eSArseniy Krasnov
281bc36442eSArseniy Krasnov if (test_data->sendmsg_errno == 0)
282bc36442eSArseniy Krasnov local_hash = hash_djb2(data, data_len);
283bc36442eSArseniy Krasnov else
284bc36442eSArseniy Krasnov local_hash = 0;
285bc36442eSArseniy Krasnov
286bc36442eSArseniy Krasnov free(data);
287bc36442eSArseniy Krasnov
288bc36442eSArseniy Krasnov /* Waiting for some result. */
289bc36442eSArseniy Krasnov remote_hash = control_readulong();
290bc36442eSArseniy Krasnov if (remote_hash != local_hash) {
291bc36442eSArseniy Krasnov fprintf(stderr, "hash mismatch\n");
292bc36442eSArseniy Krasnov exit(EXIT_FAILURE);
293bc36442eSArseniy Krasnov }
294bc36442eSArseniy Krasnov
295bc36442eSArseniy Krasnov control_expectln("DONE");
296bc36442eSArseniy Krasnov close(fd);
297bc36442eSArseniy Krasnov }
298bc36442eSArseniy Krasnov
test_stream_msgzcopy_server(const struct test_opts * opts)299bc36442eSArseniy Krasnov void test_stream_msgzcopy_server(const struct test_opts *opts)
300bc36442eSArseniy Krasnov {
301bc36442eSArseniy Krasnov int i;
302bc36442eSArseniy Krasnov
303bc36442eSArseniy Krasnov for (i = 0; i < ARRAY_SIZE(test_data_array); i++)
304bc36442eSArseniy Krasnov test_server(opts, &test_data_array[i], false);
305bc36442eSArseniy Krasnov }
306bc36442eSArseniy Krasnov
test_seqpacket_msgzcopy_server(const struct test_opts * opts)307bc36442eSArseniy Krasnov void test_seqpacket_msgzcopy_server(const struct test_opts *opts)
308bc36442eSArseniy Krasnov {
309bc36442eSArseniy Krasnov int i;
310bc36442eSArseniy Krasnov
311bc36442eSArseniy Krasnov for (i = 0; i < ARRAY_SIZE(test_data_array); i++) {
312bc36442eSArseniy Krasnov if (test_data_array[i].stream_only)
313bc36442eSArseniy Krasnov continue;
314bc36442eSArseniy Krasnov
315bc36442eSArseniy Krasnov test_server(opts, &test_data_array[i], true);
316bc36442eSArseniy Krasnov }
317bc36442eSArseniy Krasnov }
318bc36442eSArseniy Krasnov
test_stream_msgzcopy_empty_errq_client(const struct test_opts * opts)319bc36442eSArseniy Krasnov void test_stream_msgzcopy_empty_errq_client(const struct test_opts *opts)
320bc36442eSArseniy Krasnov {
321bc36442eSArseniy Krasnov struct msghdr msg = { 0 };
322bc36442eSArseniy Krasnov char cmsg_data[128];
323bc36442eSArseniy Krasnov ssize_t res;
324bc36442eSArseniy Krasnov int fd;
325bc36442eSArseniy Krasnov
326*e18c7092SArseniy Krasnov fd = vsock_stream_connect(opts->peer_cid, opts->peer_port);
327bc36442eSArseniy Krasnov if (fd < 0) {
328bc36442eSArseniy Krasnov perror("connect");
329bc36442eSArseniy Krasnov exit(EXIT_FAILURE);
330bc36442eSArseniy Krasnov }
331bc36442eSArseniy Krasnov
332bc36442eSArseniy Krasnov msg.msg_control = cmsg_data;
333bc36442eSArseniy Krasnov msg.msg_controllen = sizeof(cmsg_data);
334bc36442eSArseniy Krasnov
335bc36442eSArseniy Krasnov res = recvmsg(fd, &msg, MSG_ERRQUEUE);
336bc36442eSArseniy Krasnov if (res != -1) {
337bc36442eSArseniy Krasnov fprintf(stderr, "expected 'recvmsg(2)' failure, got %zi\n",
338bc36442eSArseniy Krasnov res);
339bc36442eSArseniy Krasnov exit(EXIT_FAILURE);
340bc36442eSArseniy Krasnov }
341bc36442eSArseniy Krasnov
342bc36442eSArseniy Krasnov control_writeln("DONE");
343bc36442eSArseniy Krasnov close(fd);
344bc36442eSArseniy Krasnov }
345bc36442eSArseniy Krasnov
test_stream_msgzcopy_empty_errq_server(const struct test_opts * opts)346bc36442eSArseniy Krasnov void test_stream_msgzcopy_empty_errq_server(const struct test_opts *opts)
347bc36442eSArseniy Krasnov {
348bc36442eSArseniy Krasnov int fd;
349bc36442eSArseniy Krasnov
350*e18c7092SArseniy Krasnov fd = vsock_stream_accept(VMADDR_CID_ANY, opts->peer_port, NULL);
351bc36442eSArseniy Krasnov if (fd < 0) {
352bc36442eSArseniy Krasnov perror("accept");
353bc36442eSArseniy Krasnov exit(EXIT_FAILURE);
354bc36442eSArseniy Krasnov }
355bc36442eSArseniy Krasnov
356bc36442eSArseniy Krasnov control_expectln("DONE");
357bc36442eSArseniy Krasnov close(fd);
358bc36442eSArseniy Krasnov }
359