1#!/bin/sh 2 3# 4# Copyright (c) 2017 Dell EMC Isilon 5# All rights reserved. 6# 7# Redistribution and use in source and binary forms, with or without 8# modification, are permitted provided that the following conditions 9# are met: 10# 1. Redistributions of source code must retain the above copyright 11# notice, this list of conditions and the following disclaimer. 12# 2. Redistributions in binary form must reproduce the above copyright 13# notice, this list of conditions and the following disclaimer in the 14# documentation and/or other materials provided with the distribution. 15# 16# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26# SUCH DAMAGE. 27# 28 29# Scenario: backup of a number (> maxvnodes) of small files. 30 31# Test how vnlru impacts open(2). 32 33# FreeBSD 12.0-CURRENT #0 r312620: Mon Jan 23 23:27:46 CET 2017 34# /usr/src/sys/amd64/compile/BENCH amd64 35# ./tvnlru.sh 36# FAIL 328/306502 37# files = 500000, maxvnodes = 500000, ave=0.000018, max=0.000328, elapsed 4 38# files = 500000, maxvnodes = 500000, ave=0.000018, max=0.000155, elapsed 4 39# files = 500000, maxvnodes = 500000, ave=0.000018, max=0.000227, elapsed 4 40# files = 500000, maxvnodes = 500000, ave=0.000014, max=0.000126, elapsed 3 41# 42# files = 1000000, maxvnodes = 500000, ave=0.000035, max=0.205627, elapsed 14 43# files = 1000000, maxvnodes = 500000, ave=0.000033, max=0.205185, elapsed 14 44# files = 1000000, maxvnodes = 500000, ave=0.000038, max=0.306502, elapsed 14 45# files = 1000000, maxvnodes = 500000, ave=0.000037, max=0.205177, elapsed 14 46 47. ../default.cfg 48[ `id -u ` -ne 0 ] && echo "Must be root!" && exit 1 49[ `sysctl -n hw.physmem` -lt $(( 4 * 1024 * 1024 * 1024)) ] && exit 0 50 51files=1000000 52[ `sysctl -n kern.maxvnodes` -lt $files ] && exit 0 53 54log=/tmp/tvnlru.log 55dir=/tmp 56odir=`pwd` 57cd $dir 58sed '1,/^EOF/d' < $odir/$0 > $dir/tvnlru.c 59mycc -o tvnlru -Wall -Wextra -O0 -g tvnlru.c || exit 1 60rm -f tvnlru.c 61cd $odir 62 63mount | grep -q "on $mntpoint " && umount -f $mntpoint 64[ -c /dev/md$mdstart ] && mdconfig -d -u $mdstart 65mdconfig -a -t swap -s 2g -u $mdstart || exit 1 66bsdlabel -w md$mdstart auto 67newfs -n -b 4096 -f 512 -i 512 md${mdstart}$part > /dev/null 68mount -o async /dev/md${mdstart}$part $mntpoint 69 70ncpu=`sysctl -n hw.ncpu` 71[ $ncpu -lt 4 ] && { rm /tmp/tvnlru; exit 0; } 72ncpu=4 73inodes=`df -i $mntpoint | tail -1 | awk "{print \\$7 - $ncpu - 1}"` 74oldmx=`sysctl -n kern.maxvnodes` 75[ $files -gt $inodes ] && { echo "Disk too small"; files=$inodes; } 76[ $files -gt $oldmx ] && 77 { echo "$files exceed old maxvnods"; files=$oldmx; } 78newmaxvnodes=$((files / 2)) 79trap "sysctl kern.maxvnodes=$oldmx > /dev/null" EXIT SIGINT 80sysctl kern.maxvnodes=$newmaxvnodes > /dev/null 81 82# warmup 83cd $mntpoint 84t1=`/tmp/tvnlru $ncpu $newmaxvnodes $newmaxvnodes $mntpoint 2>/dev/null` 85cd $odir 86umount $mntpoint 87newfs -n -b 4096 -f 512 -i 512 md${mdstart}$part > /dev/null 88mount -o async /dev/md${mdstart}$part $mntpoint 89 90cd $mntpoint 91t1=`/tmp/tvnlru $ncpu $newmaxvnodes $newmaxvnodes $mntpoint 2>$log` 92cd $odir 93umount $mntpoint 94newfs -n -b 4096 -f 512 -i 512 md${mdstart}$part > /dev/null 95mount -o async /dev/md${mdstart}$part $mntpoint 96 97cd $mntpoint 98echo >> $log 99t2=`/tmp/tvnlru $ncpu $files $newmaxvnodes $mntpoint 2>>$log` 100s=$? 101cd $odir 102 103s=0 104for i in `jot 10`; do 105 mount | grep -q "on $mntpoint " || break 106 umount $mntpoint || sleep 2 107done 108mount | grep -q "on $mntpoint " && { s=2; umount -f $mntpoint; } 109mdconfig -d -u $mdstart 110[ $t2 -gt $((t1 * 3)) ] && { s=3; echo "Fail $t1/$t2"; cat $log; } 111rm -rf /tmp/tvnlru /tmp/tvnlru.log 112exit $s 113 114EOF 115#include <sys/param.h> 116#include <sys/mman.h> 117#include <sys/time.h> 118#include <sys/stat.h> 119#include <sys/wait.h> 120 121#include <machine/atomic.h> 122 123#include <err.h> 124#include <errno.h> 125#include <fcntl.h> 126#include <stdio.h> 127#include <stdlib.h> 128#include <time.h> 129#include <unistd.h> 130 131static volatile u_int *share; 132 133#define SYNC 0 134 135static long maxvnodes, parallel, tvnodes, vnodes; 136static char *mp; 137 138static void 139test(int idx) 140{ 141 struct timeval diff, start, stop; 142 time_t st; 143 uint64_t mx, tot, usec; 144 pid_t pid; 145 int fd, i, n; 146 char dir[80], file[80], help[80]; 147 148 atomic_add_int(&share[SYNC], 1); 149 while (share[SYNC] != (unsigned int)parallel) 150 ; 151 152 pid = getpid(); 153 snprintf(dir, sizeof(dir), "d%09ld", (long)pid); 154 if (mkdir(dir, 0700) == -1) 155 err(1, "mkdir(%s)", dir); 156 if (chdir(dir) == -1) 157 err(1, "chdir(%s)", dir); 158 159 for (i = 0; i < vnodes; i++) { 160 snprintf(file, sizeof(file), "f%09d", i); 161 if ((fd = open(file, O_RDWR | O_CREAT, DEFFILEMODE)) == -1) 162 err(1, "open(%s)", file); 163 close(fd); 164 } 165 166 snprintf(help, sizeof(help), "umount %s > /dev/null 2>&1", mp); 167 system(help); /* flush the cache */ 168 169 mx = 0; 170 n = 0; 171 st = time(NULL); 172 tot = 0; 173 for (i = 0; i < vnodes; i++) { 174 snprintf(file, sizeof(file), "f%09d", i); 175 gettimeofday(&start, NULL); 176 if ((fd = open(file, O_RDONLY)) == -1) 177 err(1, "open(%s)", file); 178 gettimeofday(&stop, NULL); 179 timersub(&stop, &start, &diff); 180 usec = ((uint64_t)1000000 * diff.tv_sec + diff.tv_usec); 181 tot += usec; 182 n++; 183 if (mx < usec) 184 mx = usec; 185 close(fd); 186 } 187 fprintf(stderr, 188 "files = %7ld, maxvnodes = %ld, ave=%.6f, max=%.6f, " 189 "elapsed %2ld\n", 190 tvnodes, maxvnodes, (double)tot / 1000000 / n, (double)mx / 191 1000000, time(NULL) - st); 192 share[idx] = mx; 193 194 for (i = 0; i < vnodes; i++) { 195 snprintf(file, sizeof(file), "f%09d", i); 196 if (unlink(file) == -1) 197 err(1, "unlink(%s)", file); 198 } 199 chdir(".."); 200 if (rmdir(dir) == -1) 201 err(1, "rmdir(%s)", dir); 202 203 _exit(0); 204} 205 206int 207main(int argc, char *argv[]) 208{ 209 size_t len; 210 pid_t *pids; 211 int e, i, status; 212 u_int mx; 213 214 if (argc != 5) { 215 fprintf(stderr, "Usage: %s <ncpu> <inodes> <maxvnodes> <mount point>\n", 216 argv[0]); 217 exit(1); 218 } 219 parallel = atol(argv[1]); 220 pids = calloc(parallel, sizeof(pid_t)); 221 tvnodes = atol(argv[2]); 222 vnodes = tvnodes / parallel; 223 maxvnodes = atol(argv[3]); 224 mp = argv[4]; 225 e = 0; 226 len = PAGE_SIZE; 227 if ((share = mmap(NULL, len, PROT_READ | PROT_WRITE, 228 MAP_ANON | MAP_SHARED, -1, 0)) == MAP_FAILED) 229 err(1, "mmap"); 230 231 for (i = 0; i < parallel; i++) { 232 if ((pids[i] = fork()) == 0) 233 test(i + 1); 234 } 235 236 for (i = 0; i < parallel; i++) { 237 if (waitpid(pids[i], &status, 0) == -1) 238 err(1, "waitpid(%d)", pids[i]); 239 e += status == 0 ? 0 : 1; 240 } 241 242 mx = 0; 243 for (i = 0; i < parallel; i++) { 244// fprintf(stderr, "share[%d] = %u\n", i + 1, share[i + 1]); 245 if (mx < share[i + 1]) 246 mx = share[i + 1]; 247 } 248 fprintf(stdout, "%lu\n", (unsigned long)mx); 249 250 return (e); 251} 252