1#!/bin/sh 2 3# 4# SPDX-License-Identifier: BSD-2-Clause 5# 6# Copyright (c) 2021 Jean-S�bastien P�dron <dumbbell@FreeBSD.org> 7# 8# Redistribution and use in source and binary forms, with or without 9# modification, are permitted provided that the following conditions 10# are met: 11# 1. Redistributions of source code must retain the above copyright 12# notice, this list of conditions and the following disclaimer. 13# 2. Redistributions in binary form must reproduce the above copyright 14# notice, this list of conditions and the following disclaimer in the 15# documentation and/or other materials provided with the distribution. 16# 17# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27# SUCH DAMAGE. 28# 29 30# "Written file doesn't match memory buffer" seen on main-n244961-e6bb49f12ca 31# https://reviews.freebsd.org/D28811 32 33. ../default.cfg 34kldstat -v | grep -q zfs.ko || { kldload zfs.ko || 35 exit 0; loaded=1; } 36 37cat > /tmp/write_vs_sendfile.c <<EOF 38#include <sys/errno.h> 39#include <sys/socket.h> 40#include <sys/stat.h> 41#include <sys/types.h> 42#include <sys/uio.h> 43#include <netinet/in.h> 44 45#include <assert.h> 46#include <fcntl.h> 47#include <netdb.h> 48#include <stdio.h> 49#include <stdlib.h> 50#include <string.h> 51#include <pthread.h> 52#include <unistd.h> 53 54#define FILENAME "myfile.bin" 55#define MAX_SIZE 50 * 1000 * 1000 56 57int tcp_port; 58int chunk_size; 59 60static void * 61sender_start(void *buffer __unused) 62{ 63 int fd, sock, ret; 64 struct sockaddr_in sa = { 0 }; 65 off_t cursor; 66 67 printf("Sender: opening connection to TCP port %d\n", tcp_port); 68 sock = socket(AF_INET, SOCK_STREAM, 0); 69 if (sock < 0) { 70 perror("Sender: failed to create socket"); 71 return (NULL); 72 } 73 74 sa.sin_family = AF_INET; 75 sa.sin_port = htons(tcp_port); 76 sa.sin_addr.s_addr = htonl((((((127 << 8) | 0) << 8) | 0) << 8) | 1); 77 78 ret = connect(sock, (struct sockaddr *)&sa, sizeof(sa)); 79 if (ret < 0) { 80 perror("Sender: failed to connect to localhost"); 81 close(sock); 82 return (NULL); 83 } 84 85 printf("Sender: opening %s\n", FILENAME); 86 fd = open(FILENAME, O_CREAT|O_RDONLY, 0644); 87 if (fd < 0) { 88 perror("Sender: failed to open file"); 89 close(sock); 90 return (NULL); 91 } 92 93 printf("Sender: starting sendfile(2) loop\n"); 94 cursor = 0; 95 do { 96 size_t to_send = chunk_size; 97 off_t sbytes = 0; 98 99 do { 100#if defined(__FreeBSD__) 101 ret = sendfile(fd, sock, cursor, to_send, 102 NULL, &sbytes, 0); 103 if (ret == 0) { 104 cursor += sbytes; 105 to_send -= sbytes; 106 } 107#elif defined(__APPLE__) 108 sbytes = to_send; 109 ret = sendfile(fd, sock, cursor, &sbytes, NULL, 0); 110 if (ret < 0 && (errno == EAGAIN || errno == EINTR)) { 111 ret = 0; 112 } 113 if (ret == 0) { 114 ret = 0; 115 cursor += sbytes; 116 to_send -= sbytes; 117 } 118#else 119#error Not implemented 120#endif 121 } while (ret == 0 && to_send > 0); 122 } while (cursor < MAX_SIZE); 123 124 printf("Sender: closing socket\n"); 125 close(fd); 126 printf("Sender: closing %s\n", FILENAME); 127 close(sock); 128 129 return (NULL); 130} 131 132static void * 133writer_start(void *buffer) 134{ 135 int fd, cursor, ret; 136 137 printf("Writer: opening %s\n", FILENAME); 138 fd = open(FILENAME, O_CREAT|O_RDWR|O_TRUNC|O_DIRECT, 0644); 139 if (fd < 0) { 140 perror("Writer: failed to open file"); 141 return (NULL); 142 } 143 144 /* We sleep one second to give a head start to the sendfile(2) thread 145 * above. */ 146 sleep(1); 147 148 printf( 149 "Writer: writing chunks of %u bytes to a max of %u bytes\n", 150 chunk_size, MAX_SIZE); 151 cursor = 0; 152 do { 153 ret = write(fd, buffer, chunk_size); 154 if (ret < 0) { 155 perror("Writer: failed to write file"); 156 break; 157 } 158 assert(ret == chunk_size); 159 160 cursor += ret; 161 } while (cursor < MAX_SIZE); 162 163 printf("Writer: closing %s\n", FILENAME); 164 close(fd); 165 166 return (NULL); 167} 168 169int 170check_file(void *buffer, int flags) 171{ 172 int fd, ret, cursor; 173 void *read_buffer; 174 175 printf("Writer: opening %s\n", FILENAME); 176 fd = open(FILENAME, O_RDONLY | flags | O_DIRECT); 177 if (fd < 0) { 178 perror("Checker: failed to open file"); 179 return (1); 180 } 181 182 read_buffer = malloc(chunk_size); 183 if (buffer == NULL) { 184 perror("Checker: failed to allocate buffer"); 185 close(fd); 186 return (1); 187 } 188 189 cursor = 0; 190 do { 191 ret = read(fd, read_buffer, chunk_size); 192 if (ret < 0) { 193 perror("Checker: failed to read file"); 194 close(fd); 195 free(read_buffer); 196 return (1); 197 } 198 assert(ret == chunk_size); 199 200 cursor += ret; 201 202 ret = memcmp(buffer, read_buffer, chunk_size); 203 } while (ret == 0 && cursor < MAX_SIZE); 204 205 return (ret); 206} 207 208int 209main(int argc, char *argv[]) 210{ 211 int ret; 212 void *buffer; 213 pthread_t sender; 214 pthread_t writer; 215 216 /* The sender thread will connect to the TCP port on the local host. 217 * The user is responsible for starting an instance of netcat like 218 * this: 219 * 220 * nc -k -l 8080 221 */ 222 tcp_port = argc >= 2 ? atoi(argv[1]) : 8080; 223 chunk_size = argc >= 3 ? atoi(argv[2]) : 32128; 224 225 /* We initialize a buffer and fill it with 0xff bytes. The buffer is 226 * written many times to a file by the writer thread (see below). */ 227 buffer = malloc(chunk_size); 228 if (buffer == NULL) { 229 perror("Main: failed to allocate buffer"); 230 return (1); 231 } 232 233 memset(buffer, 255, chunk_size); 234 235 unlink(FILENAME); 236 237 /* The sender thread is responsible for sending the file written by the 238 * writer thread to a local TCP port. The goal is always try to send 239 * data which is not written to the file yet. */ 240 ret = pthread_create(&sender, NULL, &sender_start, buffer); 241 if (ret != 0) { 242 free(buffer); 243 return (2); 244 } 245 246 /* The writer thread is responsible for writing the allocated buffer to 247 * the file until it reaches a size of 50 MB. */ 248 ret = pthread_create(&writer, NULL, &writer_start, buffer); 249 if (ret != 0) { 250 pthread_cancel(sender); 251 pthread_join(sender, NULL); 252 free(buffer); 253 return (2); 254 } 255 256 pthread_join(writer, NULL); 257 pthread_cancel(sender); 258 pthread_join(sender, NULL); 259 260 /* Now that both threads terminated, we check the content of the 261 * written file. The bug on ZFS on FreeBSD is that some portions of the 262 * file contains zeros instead of the expected 0xff bytes. */ 263 ret = check_file(buffer, 0); 264 free(buffer); 265 266 if (ret != 0) { 267 fprintf(stderr, "\033[1;31mWritten file doesn't match memory buffer\033[0m\n"); 268 } 269 270 return (ret); 271} 272EOF 273 274mycc -o /tmp/write_vs_sendfile -Wall -Wextra -O2 /tmp/write_vs_sendfile.c -lpthread || exit 1 275rm /tmp/write_vs_sendfile.c 276 277u1=$mdstart 278u2=$((u1 + 1)) 279 280mdconfig -l | grep -q md$u1 && mdconfig -d -u $u1 281mdconfig -l | grep -q md$u2 && mdconfig -d -u $u2 282 283mdconfig -s 2g -u $u1 284mdconfig -s 2g -u $u2 285 286zpool list | egrep -q "^stress2_tank" && zpool destroy stress2_tank 287[ -d /stress2_tank ] && rm -rf /stress2_tank 288zpool create stress2_tank raidz md$u1 md$u2 289zfs create stress2_tank/test 290 291here=`pwd` 292cd /stress2_tank/test 293nc -k -l 8080 >/dev/null & 294sleep .5 295/tmp/write_vs_sendfile; s=$? 296kill $! 297wait 298cd $here 299 300zfs umount stress2_tank/test 301zfs destroy -r stress2_tank 302zpool destroy stress2_tank 303 304mdconfig -d -u $u1 305mdconfig -d -u $u2 306[ -n "$loaded" ] && kldunload zfs.ko 307rm /tmp/write_vs_sendfile 308exit $s 309