1 /***************************************************************************
2 * tools/benchmark_files.cpp
3 *
4 * Part of the STXXL. See http://stxxl.sourceforge.net
5 *
6 * Copyright (C) 2002-2003 Roman Dementiev <dementiev@mpi-sb.mpg.de>
7 * Copyright (C) 2007-2011 Andreas Beckmann <beckmann@cs.uni-frankfurt.de>
8 * Copyright (C) 2009 Johannes Singler <singler@ira.uka.de>
9 * Copyright (C) 2013 Timo Bingmann <tb@panthema.net>
10 *
11 * Distributed under the Boost Software License, Version 1.0.
12 * (See accompanying file LICENSE_1_0.txt or copy at
13 * http://www.boost.org/LICENSE_1_0.txt)
14 **************************************************************************/
15
16 /*
17 example gnuplot command for the output of this program:
18 (x-axis: file offset in GiB, y-axis: bandwidth in MiB/s)
19
20 plot \
21 "file.log" using ($3/1024):($14) w l title "read", \
22 "file.log" using ($3/1024):($7) w l title "write"
23 */
24
25 #include <iomanip>
26 #include <vector>
27
28 #include <stxxl/io>
29 #include <stxxl/aligned_alloc>
30 #include <stxxl/timer>
31 #include <stxxl/bits/version.h>
32 #include <stxxl/bits/common/cmdline.h>
33
34 using stxxl::request_ptr;
35 using stxxl::file;
36 using stxxl::timestamp;
37 using stxxl::unsigned_type;
38 using stxxl::uint64;
39
40 #ifdef BLOCK_ALIGN
41 #undef BLOCK_ALIGN
42 #endif
43
44 #define BLOCK_ALIGN 4096
45
46 #define POLL_DELAY 1000
47
48 #if STXXL_WINDOWS
49 const char* default_file_type = "wincall";
50 #else
51 const char* default_file_type = "syscall";
52 #endif
53
54 #ifdef WATCH_TIMES
watch_times(request_ptr reqs[],unsigned n,double * out)55 void watch_times(request_ptr reqs[], unsigned n, double* out)
56 {
57 bool* finished = new bool[n];
58 unsigned count = 0;
59 for (unsigned i = 0; i < n; i++)
60 finished[i] = false;
61
62 while (count != n)
63 {
64 usleep(POLL_DELAY);
65 unsigned i = 0;
66 for (i = 0; i < n; i++)
67 {
68 if (!finished[i])
69 if (reqs[i]->poll())
70 {
71 finished[i] = true;
72 out[i] = timestamp();
73 count++;
74 }
75 }
76 }
77 delete[] finished;
78 }
79
out_stat(double start,double end,double * times,unsigned n,const std::vector<std::string> & names)80 void out_stat(double start, double end, double* times, unsigned n, const std::vector<std::string>& names)
81 {
82 for (unsigned i = 0; i < n; i++)
83 {
84 std::cout << i << " " << names[i] << " took " <<
85 100. * (times[i] - start) / (end - start) << " %" << std::endl;
86 }
87 }
88 #endif
89
90 #define MB (1024 * 1024)
91
92 // returns throughput in MiB/s
throughput(stxxl::int64 bytes,double seconds)93 static inline double throughput(stxxl::int64 bytes, double seconds)
94 {
95 if (seconds == 0.0)
96 return 0.0;
97 return (double)bytes / (1024 * 1024) / seconds;
98 }
99
benchmark_files(int argc,char * argv[])100 int benchmark_files(int argc, char* argv[])
101 {
102 uint64 offset = 0, length = 0;
103
104 bool no_direct_io = false;
105 bool sync_io = false;
106 bool resize_after_open = false;
107 std::string file_type = default_file_type;
108 unsigned_type block_size = 0;
109 unsigned int batch_size = 1;
110 std::string opstr = "wv";
111 unsigned pattern = 0;
112
113 std::vector<std::string> files_arr;
114
115 stxxl::cmdline_parser cp;
116
117 cp.add_param_bytes("length", "Length to write in file.", length);
118 cp.add_param_stringlist("filename", "File path to run benchmark on.", files_arr);
119
120 cp.add_bytes('o', "offset", "Starting offset to write in file.", offset);
121
122 cp.add_flag(0, "no-direct", "open files without O_DIRECT", no_direct_io);
123 cp.add_flag(0, "sync", "open files with O_SYNC|O_DSYNC|O_RSYNC", sync_io);
124 cp.add_flag(0, "resize", "resize the file size after opening, needed e.g. for creating mmap files", resize_after_open);
125
126 cp.add_bytes(0, "block_size", "block size for operations (default 8 MiB)", block_size);
127 cp.add_uint(0, "batch_size", "increase (default 1) to submit several I/Os at once and report average rate", batch_size);
128
129 cp.add_string('f', "file-type",
130 "Method to open file (syscall|mmap|wincall|boostfd|...) default: " + file_type, file_type);
131
132 cp.add_string('p', "operations",
133 "[w]rite pattern, [r]ead without verification, read and [v]erify pattern (default: 'wv')", opstr);
134
135 cp.add_uint(0, "pattern",
136 "32-bit pattern to write (default: block index)", pattern);
137
138 cp.set_description("Open a file using one of STXXL's file abstractions and perform write/read/verify tests on the file. "
139 "Block sizes and batch size can be adjusted via command line. "
140 "If length == 0 , then operation will continue till end of space (please ignore the write error). "
141 "Memory consumption: block_size * batch_size * num_files");
142
143 if (!cp.process(argc, argv))
144 return -1;
145
146 uint64 endpos = offset + length;
147
148 if (block_size == 0)
149 block_size = 8 * MB;
150
151 if (batch_size == 0)
152 batch_size = 1;
153
154 bool do_read = false, do_write = false, do_verify = false;
155
156 // deprecated, use --no-direct instead
157 if (opstr.find("nd") != std::string::npos || opstr.find("ND") != std::string::npos) {
158 no_direct_io = true;
159 }
160
161 if (opstr.find('r') != std::string::npos || opstr.find('R') != std::string::npos) {
162 do_read = true;
163 }
164 if (opstr.find('v') != std::string::npos || opstr.find('V') != std::string::npos) {
165 do_verify = true;
166 }
167 if (opstr.find('w') != std::string::npos || opstr.find('W') != std::string::npos) {
168 do_write = true;
169 }
170
171 const char* myself = strrchr(argv[0], '/');
172 if (!myself || !*(++myself))
173 myself = argv[0];
174 std::cout << "# " << myself << " " << stxxl::get_version_string_long();
175 #if STXXL_DIRECT_IO_OFF
176 std::cout << " STXXL_DIRECT_IO_OFF";
177 #endif
178 std::cout << std::endl;
179
180 for (size_t ii = 0; ii < files_arr.size(); ii++)
181 {
182 std::cout << "# Add file: " << files_arr[ii] << std::endl;
183 }
184
185 const size_t nfiles = files_arr.size();
186 bool verify_failed = false;
187
188 const unsigned_type step_size = block_size * batch_size;
189 const unsigned_type block_size_int = block_size / sizeof(int);
190 const uint64 step_size_int = step_size / sizeof(int);
191
192 unsigned* buffer = (unsigned*)stxxl::aligned_alloc<BLOCK_ALIGN>(step_size * nfiles);
193 file** files = new file*[nfiles];
194 request_ptr* reqs = new request_ptr[nfiles * batch_size];
195
196 #ifdef WATCH_TIMES
197 double* r_finish_times = new double[nfiles];
198 double* w_finish_times = new double[nfiles];
199 #endif
200
201 double totaltimeread = 0, totaltimewrite = 0;
202 stxxl::int64 totalsizeread = 0, totalsizewrite = 0;
203
204 // fill buffer with pattern
205 for (unsigned i = 0; i < nfiles * step_size_int; i++)
206 buffer[i] = (pattern ? pattern : i);
207
208 // open files
209 for (unsigned i = 0; i < nfiles; i++)
210 {
211 int openmode = file::CREAT | file::RDWR;
212 if (!no_direct_io) {
213 openmode |= file::DIRECT;
214 }
215 if (sync_io) {
216 openmode |= file::SYNC;
217 }
218
219 files[i] = stxxl::create_file(file_type, files_arr[i], openmode, i);
220 if (resize_after_open)
221 files[i]->set_size(endpos);
222 }
223
224 std::cout << "# Step size: "
225 << step_size << " bytes per file ("
226 << batch_size << " block" << (batch_size == 1 ? "" : "s") << " of "
227 << block_size << " bytes)"
228 << " file_type=" << file_type
229 << " O_DIRECT=" << (no_direct_io ? "no" : "yes")
230 << " O_SYNC=" << (sync_io ? "yes" : "no")
231 << std::endl;
232
233 stxxl::timer t_total(true);
234 try {
235 while (offset + uint64(step_size) <= endpos || length == 0)
236 {
237 const uint64 current_step_size = (length == 0) ? stxxl::int64(step_size) : std::min<stxxl::int64>(step_size, endpos - offset);
238 const uint64 current_step_size_int = current_step_size / sizeof(int);
239 const unsigned_type current_num_blocks = (unsigned_type)stxxl::div_ceil(current_step_size, block_size);
240
241 std::cout << "File offset " << std::setw(8) << offset / MB << " MiB: " << std::fixed;
242
243 double begin = timestamp(), end = begin, elapsed;
244
245 if (do_write)
246 {
247 // write block number (512 byte blocks) into each block at position 42 * sizeof(unsigned)
248 for (uint64 j = 42, b = offset >> 9; j < current_step_size_int; j += 512 / sizeof(unsigned), ++b)
249 {
250 for (unsigned i = 0; i < nfiles; i++)
251 buffer[current_step_size_int * i + j] = (unsigned int)b;
252 }
253
254 for (unsigned i = 0; i < nfiles; i++)
255 {
256 for (unsigned_type j = 0; j < current_num_blocks; j++)
257 reqs[i * current_num_blocks + j] =
258 files[i]->awrite(buffer + current_step_size_int * i + j * block_size_int,
259 offset + j * block_size,
260 (unsigned_type)block_size);
261 }
262
263 #ifdef WATCH_TIMES
264 watch_times(reqs, nfiles, w_finish_times);
265 #else
266 wait_all(reqs, nfiles * current_num_blocks);
267 #endif
268
269 end = timestamp();
270 elapsed = end - begin;
271 totalsizewrite += current_step_size;
272 totaltimewrite += elapsed;
273 }
274 else {
275 elapsed = 0.0;
276 }
277
278 #if 0
279 std::cout << "# WRITE\nFiles: " << nfiles
280 << " \nElapsed time: " << end - begin
281 << " \nThroughput: " << int(double(current_step_size * nfiles) / MB / (end - begin))
282 << " MiB/s \nPer one file:"
283 << int(double(current_step_size) / MB / (end - begin)) << " MiB/s"
284 << std::endl;
285 #endif
286
287 #ifdef WATCH_TIMES
288 out_stat(begin, end, w_finish_times, nfiles, files_arr);
289 #endif
290 std::cout << std::setw(2) << nfiles << " * "
291 << std::setw(8) << std::setprecision(3)
292 << (throughput(current_step_size, elapsed)) << " = "
293 << std::setw(8) << std::setprecision(3)
294 << (throughput(current_step_size, elapsed) * (double)nfiles) << " MiB/s write,";
295
296 begin = end = timestamp();
297
298 if (do_read || do_verify)
299 {
300 for (unsigned i = 0; i < nfiles; i++)
301 {
302 for (unsigned j = 0; j < current_num_blocks; j++)
303 reqs[i * current_num_blocks + j] =
304 files[i]->aread(buffer + current_step_size_int * i + j * block_size_int,
305 offset + j * block_size,
306 (unsigned_type)block_size);
307 }
308
309 #ifdef WATCH_TIMES
310 watch_times(reqs, nfiles, r_finish_times);
311 #else
312 wait_all(reqs, nfiles * current_num_blocks);
313 #endif
314
315 end = timestamp();
316 elapsed = end - begin;
317 totalsizeread += current_step_size;
318 totaltimeread += elapsed;
319 }
320 else {
321 elapsed = 0.0;
322 }
323
324 #if 0
325 std::cout << "# READ\nFiles: " << nfiles
326 << " \nElapsed time: " << end - begin
327 << " \nThroughput: " << int(double(current_step_size * nfiles) / MB / (end - begin))
328 << " MiB/s \nPer one file:"
329 << int(double(current_step_size) / MB / (end - begin)) << " MiB/s"
330 << std::endl;
331 #endif
332
333 std::cout << std::setw(2) << nfiles << " * "
334 << std::setw(8) << std::setprecision(3)
335 << (throughput(current_step_size, elapsed)) << " = "
336 << std::setw(8) << std::setprecision(3)
337 << (throughput(current_step_size, elapsed) * (double)nfiles) << " MiB/s read";
338
339 #ifdef WATCH_TIMES
340 out_stat(begin, end, r_finish_times, nfiles, files_arr);
341 #endif
342
343 if (do_verify)
344 {
345 for (unsigned d = 0; d < nfiles; ++d)
346 {
347 for (unsigned s = 0; s < (current_step_size >> 9); ++s) {
348 uint64 i = d * current_step_size_int + s * (512 / sizeof(unsigned)) + 42;
349 uint64 b = (offset >> 9) + s;
350 if (buffer[i] != b)
351 {
352 verify_failed = true;
353 std::cout << "Error on file " << d << " sector " << std::hex << std::setw(8) << b
354 << " got: " << std::hex << std::setw(8) << buffer[i] << " wanted: " << std::hex << std::setw(8) << b
355 << std::dec << std::endl;
356 }
357 buffer[i] = (pattern ? pattern : (unsigned int)i);
358 }
359 }
360
361 for (uint64 i = 0; i < nfiles * current_step_size_int; i++)
362 {
363 if (buffer[i] != (pattern ? pattern : i))
364 {
365 stxxl::int64 ibuf = i / current_step_size_int;
366 uint64 pos = i % current_step_size_int;
367
368 std::cout << std::endl
369 << "Error on file " << ibuf << " position " << std::hex << std::setw(8) << offset + pos * sizeof(int)
370 << " got: " << std::hex << std::setw(8) << buffer[i] << " wanted: " << std::hex << std::setw(8) << i
371 << std::dec << std::endl;
372
373 i = (ibuf + 1) * current_step_size_int; // jump to next
374
375 verify_failed = true;
376 }
377 }
378 }
379 std::cout << std::endl;
380
381 offset += current_step_size;
382 }
383 }
384 catch (const std::exception& ex)
385 {
386 std::cout << std::endl;
387 STXXL_ERRMSG(ex.what());
388 }
389 t_total.stop();
390
391 std::cout << "=============================================================================================" << std::endl;
392 // the following line of output is parsed by misc/filebench-avgplot.sh
393 std::cout << "# Average over " << std::setw(8) << stxxl::STXXL_MAX(totalsizewrite, totalsizeread) / MB << " MiB: ";
394 std::cout << std::setw(2) << nfiles << " * "
395 << std::setw(8) << std::setprecision(3)
396 << (throughput(totalsizewrite, totaltimewrite)) << " = "
397 << std::setw(8) << std::setprecision(3)
398 << (throughput(totalsizewrite, totaltimewrite) * (double)nfiles) << " MiB/s write,";
399
400 std::cout << std::setw(2) << nfiles << " * "
401 << std::setw(8) << std::setprecision(3)
402 << (throughput(totalsizeread, totaltimeread)) << " = "
403 << std::setw(8) << std::setprecision(3)
404 << (throughput(totalsizeread, totaltimeread) * (double)nfiles) << " MiB/s read"
405 << std::endl;
406
407 if (totaltimewrite != 0.0)
408 std::cout << "# Write time " << std::setw(8) << std::setprecision(3) << totaltimewrite << " s" << std::endl;
409 if (totaltimeread != 0.0)
410 std::cout << "# Read time " << std::setw(8) << std::setprecision(3) << totaltimeread << " s" << std::endl;
411
412 std::cout << "# Non-I/O time " << std::setw(8) << std::setprecision(3)
413 << (t_total.seconds() - totaltimewrite - totaltimeread) << " s, average throughput "
414 << std::setw(8) << std::setprecision(3)
415 << (throughput(totalsizewrite + totalsizeread, t_total.seconds() - totaltimewrite - totaltimeread) * (double)nfiles) << " MiB/s"
416 << std::endl;
417
418 std::cout << "# Total time " << std::setw(8) << std::setprecision(3) << t_total.seconds() << " s, average throughput "
419 << std::setw(8) << std::setprecision(3)
420 << (throughput(totalsizewrite + totalsizeread, t_total.seconds()) * (double)nfiles) << " MiB/s"
421 << std::endl;
422
423 if (do_verify)
424 {
425 std::cout << "# Verify: " << (verify_failed ? "FAILED." : "all okay.") << std::endl;
426 }
427
428 #ifdef WATCH_TIMES
429 delete[] r_finish_times;
430 delete[] w_finish_times;
431 #endif
432 delete[] reqs;
433 for (unsigned i = 0; i < nfiles; i++)
434 delete files[i];
435 delete[] files;
436 stxxl::aligned_dealloc<BLOCK_ALIGN>(buffer);
437
438 return (verify_failed ? 1 : 0);
439 }
440
441 // vim: et:ts=4:sw=4
442