1 /*-
2 * Public Domain 2014-2018 MongoDB, Inc.
3 * Public Domain 2008-2014 WiredTiger, Inc.
4 *
5 * This is free and unencumbered software released into the public domain.
6 *
7 * Anyone is free to copy, modify, publish, use, compile, sell, or
8 * distribute this software, either in source code form or as a compiled
9 * binary, for any purpose, commercial or non-commercial, and by any
10 * means.
11 *
12 * In jurisdictions that recognize copyright laws, the author or authors
13 * of this software dedicate any and all copyright interest in the
14 * software to the public domain. We make this dedication for the benefit
15 * of the public at large and to the detriment of our heirs and
16 * successors. We intend this dedication to be an overt act of
17 * relinquishment in perpetuity of all present and future rights to this
18 * software under copyright law.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
21 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
23 * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
24 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
25 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
26 * OTHER DEALINGS IN THE SOFTWARE.
27 */
28
29 #ifndef HAVE_WTPERF_H
30 #define HAVE_WTPERF_H
31
32 #include "test_util.h"
33
34 #include <assert.h>
35 #include <math.h>
36
37 #include "config_opt.h"
38
39 typedef struct __wtperf WTPERF;
40 typedef struct __wtperf_thread WTPERF_THREAD;
41 typedef struct __truncate_queue_entry TRUNCATE_QUEUE_ENTRY;
42
43 #define EXT_PFX ",extensions=("
44 #define EXT_SFX ")"
45 #define EXTPATH "../../ext/compressors/" /* Extensions path */
46 #define BLKCMP_PFX "block_compressor="
47
48 #define LZ4_BLK BLKCMP_PFX "lz4"
49 #define LZ4_EXT \
50 EXT_PFX EXTPATH "lz4/.libs/libwiredtiger_lz4.so" EXT_SFX
51 #define SNAPPY_BLK BLKCMP_PFX "snappy"
52 #define SNAPPY_EXT \
53 EXT_PFX EXTPATH "snappy/.libs/libwiredtiger_snappy.so" EXT_SFX
54 #define ZLIB_BLK BLKCMP_PFX "zlib"
55 #define ZLIB_EXT \
56 EXT_PFX EXTPATH "zlib/.libs/libwiredtiger_zlib.so" EXT_SFX
57 #define ZSTD_BLK BLKCMP_PFX "zstd"
58 #define ZSTD_EXT \
59 EXT_PFX EXTPATH "zstd/.libs/libwiredtiger_zstd.so" EXT_SFX
60
61 typedef struct {
62 int64_t threads; /* Thread count */
63 int64_t insert; /* Insert ratio */
64 int64_t read; /* Read ratio */
65 int64_t update; /* Update ratio */
66 uint64_t throttle; /* Maximum operations/second */
67 /* Number of operations per transaction. Zero for autocommit */
68 int64_t ops_per_txn;
69 int64_t pause; /* Time between scans */
70 int64_t read_range; /* Range of reads */
71 int32_t table_index; /* Table to focus ops on */
72 int64_t truncate; /* Truncate ratio */
73 uint64_t truncate_pct; /* Truncate Percent */
74 uint64_t truncate_count; /* Truncate Count */
75 int64_t update_delta; /* Value size change on update */
76
77 #define WORKER_INSERT 1 /* Insert */
78 #define WORKER_INSERT_RMW 2 /* Insert with read-modify-write */
79 #define WORKER_READ 3 /* Read */
80 #define WORKER_TRUNCATE 4 /* Truncate */
81 #define WORKER_UPDATE 5 /* Update */
82 uint8_t ops[100]; /* Operation schedule */
83 } WORKLOAD;
84
85 /* Steering items for the truncate workload */
86 typedef struct {
87 uint64_t stone_gap;
88 uint64_t needed_stones;
89 uint64_t expected_total;
90 uint64_t total_inserts;
91 uint64_t last_total_inserts;
92 uint64_t num_stones;
93 uint64_t last_key;
94 uint64_t catchup_multiplier;
95 } TRUNCATE_CONFIG;
96
97 /* Queue entry for use with the Truncate Logic */
98 struct __truncate_queue_entry {
99 char *key; /* Truncation point */
100 uint64_t diff; /* Number of items to be truncated*/
101 TAILQ_ENTRY(__truncate_queue_entry) q;
102 };
103
104 /* Steering for the throttle configuration */
105 typedef struct {
106 struct timespec last_increment; /* Time that we last added more ops */
107 uint64_t ops_count; /* The number of ops this increment */
108 uint64_t ops_per_increment; /* Ops to add per increment */
109 uint64_t usecs_increment; /* Time interval of each increment */
110 } THROTTLE_CONFIG;
111
112 #define LOG_PARTIAL_CONFIG ",log=(enabled=false)"
113 #define READONLY_CONFIG ",readonly=true"
114 struct __wtperf { /* Per-database structure */
115 char *home; /* WiredTiger home */
116 char *monitor_dir; /* Monitor output dir */
117 char *partial_config; /* Config string for partial logging */
118 char *reopen_config; /* Config string for conn reopen */
119 char *log_table_uri; /* URI for log table */
120 char **uris; /* URIs */
121
122 WT_CONNECTION *conn; /* Database connection */
123
124 FILE *logf; /* Logging handle */
125
126 char *async_config; /* Config string for async */
127 bool use_asyncops; /* Use async operations */
128
129 const char *compress_ext; /* Compression extension for conn */
130 const char *compress_table; /* Compression arg to table create */
131
132 WTPERF_THREAD *ckptthreads; /* Checkpoint threads */
133 WTPERF_THREAD *popthreads; /* Populate threads */
134
135 #define WORKLOAD_MAX 50
136 WTPERF_THREAD *workers; /* Worker threads */
137 u_int workers_cnt;
138
139 WORKLOAD *workload; /* Workloads */
140 u_int workload_cnt;
141
142 /* State tracking variables. */
143 uint64_t ckpt_ops; /* checkpoint operations */
144 uint64_t insert_ops; /* insert operations */
145 uint64_t read_ops; /* read operations */
146 uint64_t truncate_ops; /* truncate operations */
147 uint64_t update_ops; /* update operations */
148
149 uint64_t insert_key; /* insert key */
150 uint64_t log_like_table_key; /* used to allocate IDs for log table */
151
152 volatile bool ckpt; /* checkpoint in progress */
153 volatile bool error; /* thread error */
154 volatile bool stop; /* notify threads to stop */
155 volatile bool in_warmup; /* running warmup phase */
156
157 volatile bool idle_cycle_run; /* Signal for idle cycle thread */
158
159 volatile uint32_t totalsec; /* total seconds running */
160
161 #define CFG_GROW 0x0001 /* There is a grow workload */
162 #define CFG_SHRINK 0x0002 /* There is a shrink workload */
163 #define CFG_TRUNCATE 0x0004 /* There is a truncate workload */
164 uint32_t flags; /* flags */
165
166 /* Queue head for use with the Truncate Logic */
167 TAILQ_HEAD(__truncate_qh, __truncate_queue_entry) stone_head;
168
169 CONFIG_OPTS *opts; /* Global configuration */
170 };
171
172 #define ELEMENTS(a) (sizeof(a) / sizeof(a[0]))
173
174 #define READ_RANGE_OPS 10
175 #define THROTTLE_OPS 100
176
177 #define THOUSAND (1000ULL)
178 #define MILLION (1000000ULL)
179 #define BILLION (1000000000ULL)
180
181 #define NSEC_PER_SEC BILLION
182 #define USEC_PER_SEC MILLION
183 #define MSEC_PER_SEC THOUSAND
184
185 #define ns_to_ms(v) ((v) / MILLION)
186 #define ns_to_sec(v) ((v) / BILLION)
187 #define ns_to_us(v) ((v) / THOUSAND)
188
189 #define us_to_ms(v) ((v) / THOUSAND)
190 #define us_to_ns(v) ((v) * THOUSAND)
191 #define us_to_sec(v) ((v) / MILLION)
192
193 #define ms_to_ns(v) ((v) * MILLION)
194 #define ms_to_us(v) ((v) * THOUSAND)
195 #define ms_to_sec(v) ((v) / THOUSAND)
196
197 #define sec_to_ns(v) ((v) * BILLION)
198 #define sec_to_us(v) ((v) * MILLION)
199 #define sec_to_ms(v) ((v) * THOUSAND)
200
201 typedef struct {
202 /*
203 * Threads maintain the total thread operation and total latency they've
204 * experienced; the monitor thread periodically copies these values into
205 * the last_XXX fields.
206 */
207 uint64_t ops; /* Total operations */
208 uint64_t latency_ops; /* Total ops sampled for latency */
209 uint64_t latency; /* Total latency */
210
211 uint64_t last_latency_ops; /* Last read by monitor thread */
212 uint64_t last_latency;
213
214 /*
215 * Minimum/maximum latency, shared with the monitor thread, that is, the
216 * monitor thread clears it so it's recalculated again for each period.
217 */
218 uint32_t min_latency; /* Minimum latency (uS) */
219 uint32_t max_latency; /* Maximum latency (uS) */
220
221 /*
222 * Latency buckets.
223 */
224 uint32_t us[1000]; /* < 1us ... 1000us */
225 uint32_t ms[1000]; /* < 1ms ... 1000ms */
226 uint32_t sec[100]; /* < 1s 2s ... 100s */
227 } TRACK;
228
229 struct __wtperf_thread { /* Per-thread structure */
230 WTPERF *wtperf; /* Enclosing configuration */
231 WT_CURSOR *rand_cursor; /* Random key cursor */
232
233 WT_RAND_STATE rnd; /* Random number generation state */
234
235 wt_thread_t handle; /* Handle */
236
237 char *key_buf, *value_buf; /* Key/value memory */
238
239 WORKLOAD *workload; /* Workload */
240
241 THROTTLE_CONFIG throttle_cfg; /* Throttle configuration */
242
243 TRUNCATE_CONFIG trunc_cfg; /* Truncate configuration */
244
245 TRACK ckpt; /* Checkpoint operations */
246 TRACK insert; /* Insert operations */
247 TRACK read; /* Read operations */
248 TRACK update; /* Update operations */
249 TRACK truncate; /* Truncate operations */
250 TRACK truncate_sleep; /* Truncate sleep operations */
251 };
252
253 void cleanup_truncate_config(WTPERF *);
254 int config_opt_file(WTPERF *, const char *);
255 void config_opt_cleanup(CONFIG_OPTS *);
256 void config_opt_init(CONFIG_OPTS **);
257 void config_opt_log(CONFIG_OPTS *, const char *);
258 int config_opt_name_value(WTPERF *, const char *, const char *);
259 void config_opt_print(WTPERF *);
260 int config_opt_str(WTPERF *, const char *);
261 void config_opt_usage(void);
262 int config_sanity(WTPERF *);
263 void latency_insert(WTPERF *, uint32_t *, uint32_t *, uint32_t *);
264 void latency_print(WTPERF *);
265 void latency_read(WTPERF *, uint32_t *, uint32_t *, uint32_t *);
266 void latency_update(WTPERF *, uint32_t *, uint32_t *, uint32_t *);
267 int run_truncate(
268 WTPERF *, WTPERF_THREAD *, WT_CURSOR *, WT_SESSION *, int *);
269 int setup_log_file(WTPERF *);
270 void setup_throttle(WTPERF_THREAD *);
271 void setup_truncate(WTPERF *, WTPERF_THREAD *, WT_SESSION *);
272 void start_idle_table_cycle(WTPERF *, wt_thread_t *);
273 void stop_idle_table_cycle(WTPERF *, wt_thread_t);
274 void worker_throttle(WTPERF_THREAD *);
275 uint64_t sum_ckpt_ops(WTPERF *);
276 uint64_t sum_insert_ops(WTPERF *);
277 uint64_t sum_pop_ops(WTPERF *);
278 uint64_t sum_read_ops(WTPERF *);
279 uint64_t sum_truncate_ops(WTPERF *);
280 uint64_t sum_update_ops(WTPERF *);
281
282 void lprintf(const WTPERF *, int err, uint32_t, const char *, ...)
283 #if defined(__GNUC__)
284 __attribute__((format (printf, 4, 5)))
285 #endif
286 ;
287
288 static inline void
generate_key(CONFIG_OPTS * opts,char * key_buf,uint64_t keyno)289 generate_key(CONFIG_OPTS *opts, char *key_buf, uint64_t keyno)
290 {
291 u64_to_string_zf(keyno, key_buf, opts->key_sz);
292 }
293
294 static inline void
extract_key(char * key_buf,uint64_t * keynop)295 extract_key(char *key_buf, uint64_t *keynop)
296 {
297 (void)sscanf(key_buf, "%" SCNu64, keynop);
298 }
299
300 /*
301 * die --
302 * Print message and exit on failure.
303 */
304 static inline void
305 die(int, const char *)
306 WT_GCC_FUNC_DECL_ATTRIBUTE((noreturn));
307 static inline void
die(int e,const char * str)308 die(int e, const char *str)
309 {
310 fprintf(stderr, "Call to %s failed: %s", str, wiredtiger_strerror(e));
311 exit(EXIT_FAILURE);
312 }
313 #endif
314