1 /***************************************************************************
2 * tools/benchmark_disks.cpp
3 *
4 * Part of FOXXLL. See http://foxxll.org
5 *
6 * Copyright (C) 2009 Johannes Singler <singler@ira.uka.de>
7 * Copyright (C) 2013 Timo Bingmann <tb@panthema.net>
8 *
9 * Distributed under the Boost Software License, Version 1.0.
10 * (See accompanying file LICENSE_1_0.txt or copy at
11 * http://www.boost.org/LICENSE_1_0.txt)
12 **************************************************************************/
13
14 /*
15 This programm will benchmark the disks configured via .foxxll disk
16 configuration files. The block manager is used to read and write blocks using
17 the different allocation strategies.
18 */
19
20 /*
21 example gnuplot command for the output of this program:
22 (x-axis: offset in GiB, y-axis: bandwidth in MiB/s)
23
24 plot \
25 "disk.log" using ($2/1024):($7) w l title "read", \
26 "disk.log" using ($2/1024):($4) w l title "write"
27 */
28
29 #include <algorithm>
30 #include <iomanip>
31 #include <limits>
32 #include <sstream>
33 #include <string>
34 #include <vector>
35
36 #include <tlx/cmdline_parser.hpp>
37 #include <tlx/logger.hpp>
38
39 #include <foxxll/io.hpp>
40 #include <foxxll/mng.hpp>
41
42 using foxxll::timestamp;
43 using foxxll::external_size_type;
44
45 #ifdef BLOCK_ALIGN
46 #undef BLOCK_ALIGN
47 #endif
48
49 #define BLOCK_ALIGN 4096
50
51 #define POLL_DELAY 1000
52
53 #define CHECK_AFTER_READ 0
54
55 #define KiB (1024)
56 #define MiB (1024 * 1024)
57
58 bool g_quiet = false;
59 double g_time_limit = NAN;
60
61 template <typename AllocStrategy>
benchmark_disks_alloc(external_size_type size,external_size_type start_offset,size_t batch_size,const size_t raw_block_size,const std::string & optrw)62 int benchmark_disks_alloc(
63 external_size_type size, external_size_type start_offset,
64 size_t batch_size, const size_t raw_block_size,
65 const std::string& optrw)
66 {
67 external_size_type endpos = start_offset + size;
68
69 if (size == 0)
70 endpos = std::numeric_limits<external_size_type>::max();
71
72 bool do_read = (optrw.find('r') != std::string::npos);
73 bool do_write = (optrw.find('w') != std::string::npos);
74
75 // initialize disk configuration
76 foxxll::block_manager::get_instance();
77
78 // construct block type
79
80 const size_t block_size = raw_block_size / sizeof(uint32_t);
81
82 using block_type = uint32_t *;
83 using BID = foxxll::BID<0>;
84
85 if (batch_size == 0)
86 batch_size = foxxll::config::get_instance()->disks_number();
87
88 // calculate total bytes processed in a batch
89 batch_size = raw_block_size * batch_size;
90
91 size_t num_blocks_per_batch = foxxll::div_ceil(batch_size, raw_block_size);
92 batch_size = num_blocks_per_batch * raw_block_size;
93
94 std::vector<block_type> buffer(num_blocks_per_batch);
95 foxxll::request_ptr* reqs = new foxxll::request_ptr[num_blocks_per_batch];
96 std::vector<BID> bids;
97 double total_time_read = 0, total_time_write = 0;
98 external_size_type total_size_read = 0, total_size_write = 0;
99
100 LOG1 << "# Batch size: "
101 << foxxll::add_IEC_binary_multiplier(batch_size, "B") << " ("
102 << num_blocks_per_batch << " blocks of "
103 << foxxll::add_IEC_binary_multiplier(raw_block_size, "B") << ")"
104 << " using " << AllocStrategy().name();
105
106 // allocate data blocks
107 for (size_t j = 0; j < num_blocks_per_batch; ++j) {
108 buffer[j] = reinterpret_cast<block_type>(
109 foxxll::aligned_alloc<4096>(raw_block_size));
110 }
111
112 // touch data, so it is actually allocated
113 for (size_t j = 0; j < num_blocks_per_batch; ++j) {
114 for (size_t i = 0; i < block_size; ++i)
115 buffer[j][i] = static_cast<uint32_t>(j * block_size + i);
116 }
117
118 try {
119 AllocStrategy alloc;
120 size_t current_batch_size;
121
122 double ts_begin = timestamp();
123
124 for (external_size_type offset = 0; offset < endpos; offset += current_batch_size)
125 {
126 std::stringstream ss;
127
128 current_batch_size = static_cast<size_t>(
129 std::min<external_size_type>(batch_size, endpos - offset));
130 #if CHECK_AFTER_READ
131 const size_t current_batch_size_int = current_batch_size / sizeof(uint32_t);
132 #endif
133 const size_t current_num_blocks_per_batch =
134 foxxll::div_ceil(current_batch_size, raw_block_size);
135
136 size_t num_total_blocks = bids.size();
137 bids.resize(num_total_blocks + current_num_blocks_per_batch);
138
139 // fill in block size of BID<0> variable blocks
140 for (BID& b : bids) b.size = raw_block_size;
141
142 foxxll::block_manager::get_instance()->new_blocks(
143 alloc, bids.begin() + num_total_blocks, bids.end()
144 );
145
146 if (offset < start_offset)
147 continue;
148
149 if (!g_quiet)
150 ss << "Offset " << std::setw(7) << (offset / MiB) << " MiB: "
151 << std::fixed;
152
153 double begin = timestamp(), end, elapsed;
154
155 if (do_write)
156 {
157 for (size_t j = 0; j < current_num_blocks_per_batch; j++)
158 reqs[j] = bids[num_total_blocks + j].write(buffer[j], raw_block_size);
159
160 wait_all(reqs, current_num_blocks_per_batch);
161
162 end = timestamp();
163 elapsed = end - begin;
164 total_size_write += current_batch_size;
165 total_time_write += elapsed;
166 }
167 else
168 elapsed = 0.0;
169
170 if (!g_quiet)
171 ss << std::setw(5) << std::setprecision(1)
172 << (double(current_batch_size) / MiB / elapsed)
173 << " MiB/s write, ";
174
175 begin = timestamp();
176
177 if (do_read)
178 {
179 for (size_t j = 0; j < current_num_blocks_per_batch; j++)
180 reqs[j] = bids[num_total_blocks + j].read(
181 buffer[j], raw_block_size
182 );
183
184 wait_all(reqs, current_num_blocks_per_batch);
185
186 end = timestamp();
187 elapsed = end - begin;
188 total_size_read += current_batch_size;
189 total_time_read += elapsed;
190 }
191 else
192 elapsed = 0.0;
193
194 if (!g_quiet)
195 ss << std::setw(5) << std::setprecision(1)
196 << (double(current_batch_size) / MiB / elapsed)
197 << " MiB/s read";
198
199 if (!g_quiet)
200 LOG1 << ss.str();
201
202 #if CHECK_AFTER_READ
203 for (size_t j = 0; j < current_num_blocks_per_batch; j++)
204 {
205 for (size_t i = 0; i < block_size; i++)
206 {
207 if (buffer[j][i] != static_cast<uint32_t>(j * block_size + i))
208 {
209 size_t ibuf = i / current_batch_size_int;
210 size_t pos = i % current_batch_size_int;
211
212 LOG1 << "Error on disk " << ibuf << " position "
213 << std::hex << std::setw(8)
214 << offset + pos * sizeof(uint32_t)
215 << " got: "
216 << std::hex << std::setw(8)
217 << buffer[j][i]
218 << " wanted: "
219 << std::hex << std::setw(8)
220 << (j * block_size + i)
221 << std::dec << std::endl;
222
223 i = (ibuf + 1) * current_batch_size_int; // jump to next
224 }
225 }
226 }
227 #endif
228 if (timestamp() - ts_begin > g_time_limit)
229 break;
230 }
231 }
232 catch (const std::exception& ex)
233 {
234 LOG1 << ex.what();
235 }
236
237 LOG1 << "=============================================================================================\n"
238 << "# Average over " << std::setw(7) << total_size_write / MiB << " MiB: "
239 << std::setw(5) << std::setprecision(1)
240 << (double(total_size_write) / MiB / total_time_write) << " MiB/s write, "
241 << std::setw(5) << std::setprecision(1)
242 << (double(total_size_read) / MiB / total_time_read) << " MiB/s read";
243
244 std::cout << "RESULT"
245 << (getenv("RESULT") ? getenv("RESULT") : "")
246 << " size=" << size
247 << " op=" << optrw
248 << " block_size=" << raw_block_size
249 << " batch_size=" << batch_size / raw_block_size
250 << " offset=" << start_offset
251 << " write_time=" << total_time_write
252 << " read_time=" << total_time_read
253 << " write_size=" << total_size_write
254 << " read_size=" << total_size_read
255 << " time=" << (total_time_write + total_time_read)
256 << " total_size=" << (total_size_write + total_size_read)
257 << std::endl;
258
259 delete[] reqs;
260
261 for (size_t j = 0; j < num_blocks_per_batch; ++j)
262 foxxll::aligned_dealloc<4096>(buffer[j]);
263
264 return 0;
265 }
266
benchmark_disks(int argc,char * argv[])267 int benchmark_disks(int argc, char* argv[])
268 {
269 // parse command line
270
271 tlx::CmdlineParser cp;
272
273 external_size_type size = 0, offset = 0;
274 unsigned int batch_size = 0;
275 external_size_type block_size = 8 * MiB;
276 std::string optrw = "rw", allocstr;
277
278 cp.add_flag(
279 'q', "quiet", g_quiet, "quiet processing"
280 );
281
282 cp.add_param_bytes(
283 "size", size,
284 "Amount of data to write/read from disks (e.g. 10GiB)"
285 );
286 cp.add_opt_param_string(
287 "r|w", optrw,
288 "Only read or write blocks (default: both write and read)"
289 );
290 cp.add_opt_param_string(
291 "alloc", allocstr,
292 "Block allocation strategy: random_cyclic, simple_random, fully_random, striping. (default: random_cyclic)"
293 );
294
295 cp.add_unsigned(
296 'b', "batch", batch_size,
297 "Number of blocks written/read in one batch (default: D * B)"
298 );
299 cp.add_bytes(
300 'B', "block_size", block_size,
301 "Size of blocks written in one syscall. (default: B = 8MiB)"
302 );
303 cp.add_bytes(
304 'o', "offset", offset,
305 "Starting offset of operation range. (default: 0)"
306 );
307 cp.add_double(
308 'T', "time-limit", g_time_limit,
309 "limit time of experiment (seconds)"
310 );
311
312 cp.set_description(
313 "This program will benchmark the disks configured by the standard "
314 ".foxxll disk configuration files mechanism. Blocks of 8 MiB are "
315 "written and/or read in sequence using the block manager. The batch "
316 "size describes how many blocks are written/read in one batch. The "
317 "are taken from block_manager using given the specified allocation "
318 "strategy. If size == 0, then writing/reading operation are done "
319 "until an error occurs. "
320 );
321
322 if (!cp.process(argc, argv))
323 return -1;
324
325 if (allocstr.size())
326 {
327 if (allocstr == "random_cyclic")
328 return benchmark_disks_alloc<foxxll::random_cyclic>(
329 size, offset, batch_size, block_size, optrw
330 );
331 if (allocstr == "simple_random")
332 return benchmark_disks_alloc<foxxll::simple_random>(
333 size, offset, batch_size, block_size, optrw
334 );
335 if (allocstr == "fully_random")
336 return benchmark_disks_alloc<foxxll::fully_random>(
337 size, offset, batch_size, block_size, optrw
338 );
339 if (allocstr == "striping")
340 return benchmark_disks_alloc<foxxll::striping>(
341 size, offset, batch_size, block_size, optrw
342 );
343
344 LOG1 << "Unknown allocation strategy '" << allocstr << "'";
345 cp.print_usage();
346 return -1;
347 }
348
349 return benchmark_disks_alloc<foxxll::default_alloc_strategy>(
350 size, offset, batch_size, block_size, optrw
351 );
352 }
353
354 /**************************************************************************/
355