1 /* Copyright (c) 2015-2018 Dovecot authors, see the included COPYING file */
2 
3 #include "lib.h"
4 #include "istream.h"
5 #include "istream-private.h"
6 #include "istream-concat.h"
7 #include "istream-failure-at.h"
8 #include "ostream-failure-at.h"
9 #include "ostream.h"
10 #include "fs-api-private.h"
11 
12 
13 #define RANDOMFAIL_ERROR "Random failure injection"
14 
15 static const char *fs_op_names[FS_OP_COUNT] = {
16 	"wait", "metadata", "prefetch", "read", "write", "lock", "exists",
17 	"stat", "copy", "rename", "delete", "iter"
18 };
19 
20 struct randomfail_fs {
21 	struct fs fs;
22 	unsigned int op_probability[FS_OP_COUNT];
23 	uoff_t range_start[FS_OP_COUNT], range_end[FS_OP_COUNT];
24 };
25 
26 struct randomfail_fs_file {
27 	struct fs_file file;
28 	struct fs_file *super_read;
29 	struct istream *input;
30 	bool op_pending[FS_OP_COUNT];
31 
32 	struct ostream *super_output;
33 };
34 
35 struct randomfail_fs_iter {
36 	struct fs_iter iter;
37 	struct fs_iter *super;
38 	unsigned int fail_pos;
39 };
40 
41 #define RANDOMFAIL_FS(ptr)	container_of((ptr), struct randomfail_fs, fs)
42 #define RANDOMFAIL_FILE(ptr)	container_of((ptr), struct randomfail_fs_file, file)
43 #define RANDOMFAIL_ITER(ptr)	container_of((ptr), struct randomfail_fs_iter, iter)
44 
fs_randomfail_alloc(void)45 static struct fs *fs_randomfail_alloc(void)
46 {
47 	struct randomfail_fs *fs;
48 
49 	fs = i_new(struct randomfail_fs, 1);
50 	fs->fs = fs_class_randomfail;
51 	return &fs->fs;
52 }
53 
fs_op_find(const char * str,enum fs_op * op_r)54 static bool fs_op_find(const char *str, enum fs_op *op_r)
55 {
56 	enum fs_op op;
57 
58 	for (op = 0; op < FS_OP_COUNT; op++) {
59 		if (strcmp(fs_op_names[op], str) == 0) {
60 			*op_r = op;
61 			return TRUE;
62 		}
63 	}
64 	return FALSE;
65 }
66 
67 static int
fs_randomfail_add_probability(struct randomfail_fs * fs,const char * key,const char * value,const char ** error_r)68 fs_randomfail_add_probability(struct randomfail_fs *fs,
69 			      const char *key, const char *value,
70 			      const char **error_r)
71 {
72 	unsigned int num;
73 	enum fs_op op;
74 	bool invalid_value = FALSE;
75 
76 	if (str_to_uint(value, &num) < 0 || num > 100)
77 		invalid_value = TRUE;
78 	if (fs_op_find(key, &op)) {
79 		if (invalid_value) {
80 			*error_r = "Invalid probability value";
81 			return -1;
82 		}
83 		fs->op_probability[op] = num;
84 		return 1;
85 	}
86 	if (strcmp(key, "all") == 0) {
87 		if (invalid_value) {
88 			*error_r = "Invalid probability value";
89 			return -1;
90 		}
91 		for (op = 0; op < FS_OP_COUNT; op++)
92 			fs->op_probability[op] = num;
93 		return 1;
94 	}
95 	return 0;
96 }
97 
98 static int
fs_randomfail_add_probability_range(struct randomfail_fs * fs,const char * key,const char * value,const char ** error_r)99 fs_randomfail_add_probability_range(struct randomfail_fs *fs,
100 				    const char *key, const char *value,
101 				    const char **error_r)
102 {
103 	enum fs_op op;
104 	const char *p;
105 	uoff_t num1, num2;
106 
107 	if (strcmp(key, "read-range") == 0)
108 		op = FS_OP_READ;
109 	else if (strcmp(key, "write-range") == 0)
110 		op = FS_OP_WRITE;
111 	else if (strcmp(key, "iter-range") == 0)
112 		op = FS_OP_ITER;
113 	else
114 		return 0;
115 
116 	p = strchr(value, '-');
117 	if (p == NULL) {
118 		if (str_to_uoff(value, &num1) < 0) {
119 			*error_r = "Invalid range value";
120 			return -1;
121 		}
122 		num2 = num1;
123 	} else if (str_to_uoff(t_strdup_until(value, p), &num1) < 0 ||
124 		   str_to_uoff(p+1, &num2) < 0 || num1 > num2) {
125 		*error_r = "Invalid range values";
126 		return -1;
127 	}
128 	fs->range_start[op] = num1;
129 	fs->range_end[op] = num2;
130 	return 1;
131 }
132 
fs_randomfail_parse_params(struct randomfail_fs * fs,const char * params,const char ** error_r)133 static int fs_randomfail_parse_params(struct randomfail_fs *fs,
134 				      const char *params, const char **error_r)
135 {
136 	const char *const *tmp;
137 	int ret;
138 
139 	for (tmp = t_strsplit_spaces(params, ","); *tmp != NULL; tmp++) {
140 		const char *key = *tmp;
141 		const char *value = strchr(key, '=');
142 
143 		if (value == NULL) {
144 			*error_r = "Missing '='";
145 			return -1;
146 		}
147 		key = t_strdup_until(key, value++);
148 		if ((ret = fs_randomfail_add_probability(fs, key, value, error_r)) != 0) {
149 			if (ret < 0)
150 				return -1;
151 			continue;
152 		}
153 		if ((ret = fs_randomfail_add_probability_range(fs, key, value, error_r)) != 0) {
154 			if (ret < 0)
155 				return -1;
156 			continue;
157 		}
158 		*error_r = t_strdup_printf("Unknown key '%s'", key);
159 		return -1;
160 	}
161 	return 0;
162 }
163 
164 static int
fs_randomfail_init(struct fs * _fs,const char * args,const struct fs_settings * set,const char ** error_r)165 fs_randomfail_init(struct fs *_fs, const char *args,
166 		   const struct fs_settings *set, const char **error_r)
167 {
168 	struct randomfail_fs *fs = RANDOMFAIL_FS(_fs);
169 	const char *p, *parent_name, *parent_args, *error;
170 
171 	p = strchr(args, ':');
172 	if (p == NULL) {
173 		*error_r = "Randomfail parameters missing";
174 		return -1;
175 	}
176 	if (fs_randomfail_parse_params(fs, t_strdup_until(args, p++), &error) < 0) {
177 		*error_r = t_strdup_printf(
178 			"Invalid randomfail parameters: %s", error);
179 		return -1;
180 	}
181 	args = p;
182 
183 	if (*args == '\0') {
184 		*error_r = "Parent filesystem not given as parameter";
185 		return -1;
186 	}
187 
188 	parent_args = strchr(args, ':');
189 	if (parent_args == NULL) {
190 		parent_name = args;
191 		parent_args = "";
192 	} else {
193 		parent_name = t_strdup_until(args, parent_args);
194 		parent_args++;
195 	}
196 	if (fs_init(parent_name, parent_args, set, &_fs->parent, error_r) < 0)
197 		return -1;
198 	return 0;
199 }
200 
fs_randomfail_free(struct fs * _fs)201 static void fs_randomfail_free(struct fs *_fs)
202 {
203 	struct randomfail_fs *fs = RANDOMFAIL_FS(_fs);
204 
205 	i_free(fs);
206 }
207 
fs_randomfail_get_properties(struct fs * _fs)208 static enum fs_properties fs_randomfail_get_properties(struct fs *_fs)
209 {
210 	return fs_get_properties(_fs->parent);
211 }
212 
fs_randomfail_file_alloc(void)213 static struct fs_file *fs_randomfail_file_alloc(void)
214 {
215 	struct randomfail_fs_file *file = i_new(struct randomfail_fs_file, 1);
216 	return &file->file;
217 }
218 
219 static void
fs_randomfail_file_init(struct fs_file * _file,const char * path,enum fs_open_mode mode,enum fs_open_flags flags)220 fs_randomfail_file_init(struct fs_file *_file, const char *path,
221 			enum fs_open_mode mode, enum fs_open_flags flags)
222 {
223 	struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
224 
225 	file->file.path = i_strdup(path);
226 	file->file.parent = fs_file_init_parent(_file, path, mode, flags);
227 }
228 
fs_randomfail_file_deinit(struct fs_file * _file)229 static void fs_randomfail_file_deinit(struct fs_file *_file)
230 {
231 	struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
232 
233 	fs_file_free(_file);
234 	i_free(file->file.path);
235 	i_free(file);
236 }
237 
fs_random_fail(struct fs * _fs,struct event * event,int divider,enum fs_op op)238 static bool fs_random_fail(struct fs *_fs, struct event *event,
239 			   int divider, enum fs_op op)
240 {
241 	struct randomfail_fs *fs = RANDOMFAIL_FS(_fs);
242 
243 	if (fs->op_probability[op] == 0)
244 		return FALSE;
245 	if ((unsigned int)i_rand_limit(100 * divider) <= fs->op_probability[op]) {
246 		fs_set_error(event, EIO, RANDOMFAIL_ERROR);
247 		return TRUE;
248 	}
249 	return FALSE;
250 }
251 
252 static bool
fs_file_random_fail_begin(struct randomfail_fs_file * file,enum fs_op op)253 fs_file_random_fail_begin(struct randomfail_fs_file *file, enum fs_op op)
254 {
255 	if (!file->op_pending[op]) {
256 		if (fs_random_fail(file->file.fs, file->file.event, 2, op))
257 			return TRUE;
258 	}
259 	file->op_pending[op] = TRUE;
260 	return FALSE;
261 }
262 
263 static int
fs_file_random_fail_end(struct randomfail_fs_file * file,int ret,enum fs_op op)264 fs_file_random_fail_end(struct randomfail_fs_file *file,
265 			int ret, enum fs_op op)
266 {
267 	if (ret == 0 || errno != EAGAIN) {
268 		if (fs_random_fail(file->file.fs, file->file.event, 2, op))
269 			return -1;
270 		file->op_pending[op] = FALSE;
271 	}
272 	return ret;
273 }
274 
275 static bool
fs_random_fail_range(struct fs * _fs,struct event * event,enum fs_op op,uoff_t * offset_r)276 fs_random_fail_range(struct fs *_fs, struct event *event,
277 		     enum fs_op op, uoff_t *offset_r)
278 {
279 	struct randomfail_fs *fs = RANDOMFAIL_FS(_fs);
280 
281 	if (!fs_random_fail(_fs, event, 1, op))
282 		return FALSE;
283 	*offset_r = i_rand_minmax(fs->range_start[op], fs->range_end[op]);
284 	return TRUE;
285 }
286 
287 static int
fs_randomfail_get_metadata(struct fs_file * _file,enum fs_get_metadata_flags flags,const ARRAY_TYPE (fs_metadata)** metadata_r)288 fs_randomfail_get_metadata(struct fs_file *_file,
289 			   enum fs_get_metadata_flags flags,
290 			   const ARRAY_TYPE(fs_metadata) **metadata_r)
291 {
292 	struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
293 	int ret;
294 
295 	if (fs_file_random_fail_begin(file, FS_OP_METADATA))
296 		return -1;
297 	ret = fs_get_metadata_full(_file->parent, flags, metadata_r);
298 	return fs_file_random_fail_end(file, ret, FS_OP_METADATA);
299 }
300 
fs_randomfail_prefetch(struct fs_file * _file,uoff_t length)301 static bool fs_randomfail_prefetch(struct fs_file *_file, uoff_t length)
302 {
303 	if (fs_random_fail(_file->fs, _file->event, 1, FS_OP_PREFETCH))
304 		return TRUE;
305 	return fs_prefetch(_file->parent, length);
306 }
307 
fs_randomfail_read(struct fs_file * _file,void * buf,size_t size)308 static ssize_t fs_randomfail_read(struct fs_file *_file, void *buf, size_t size)
309 {
310 	struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
311 	int ret;
312 
313 	if (fs_file_random_fail_begin(file, FS_OP_READ))
314 		return -1;
315 	ret = fs_read(_file->parent, buf, size);
316 	if (fs_file_random_fail_end(file, ret < 0 ? -1 : 0, FS_OP_READ) < 0)
317 		return -1;
318 	return ret;
319 }
320 
321 static struct istream *
fs_randomfail_read_stream(struct fs_file * _file,size_t max_buffer_size)322 fs_randomfail_read_stream(struct fs_file *_file, size_t max_buffer_size)
323 {
324 	struct istream *input, *input2;
325 	uoff_t offset;
326 
327 	input = fs_read_stream(_file->parent, max_buffer_size);
328 	if (!fs_random_fail_range(_file->fs, _file->event, FS_OP_READ, &offset))
329 		return input;
330 	input2 = i_stream_create_failure_at(input, offset, EIO, RANDOMFAIL_ERROR);
331 	i_stream_unref(&input);
332 	return input2;
333 }
334 
fs_randomfail_write(struct fs_file * _file,const void * data,size_t size)335 static int fs_randomfail_write(struct fs_file *_file, const void *data, size_t size)
336 {
337 	struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
338 	int ret;
339 
340 	if (fs_file_random_fail_begin(file, FS_OP_WRITE))
341 		return -1;
342 	ret = fs_write(_file->parent, data, size);
343 	return fs_file_random_fail_end(file, ret, FS_OP_EXISTS);
344 }
345 
fs_randomfail_write_stream(struct fs_file * _file)346 static void fs_randomfail_write_stream(struct fs_file *_file)
347 {
348 	struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
349 	uoff_t offset;
350 
351 	i_assert(_file->output == NULL);
352 
353 	file->super_output = fs_write_stream(_file->parent);
354 	if (!fs_random_fail_range(_file->fs, _file->event, FS_OP_WRITE, &offset))
355 		_file->output = file->super_output;
356 	else {
357 		_file->output = o_stream_create_failure_at(file->super_output, offset,
358 							   RANDOMFAIL_ERROR);
359 	}
360 }
361 
fs_randomfail_write_stream_finish(struct fs_file * _file,bool success)362 static int fs_randomfail_write_stream_finish(struct fs_file *_file, bool success)
363 {
364 	struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
365 
366 	if (_file->output != NULL) {
367 		if (_file->output == file->super_output)
368 			_file->output = NULL;
369 		else
370 			o_stream_unref(&_file->output);
371 		if (!success) {
372 			fs_write_stream_abort_parent(_file, &file->super_output);
373 			return -1;
374 		}
375 		if (fs_random_fail(_file->fs, _file->event, 1, FS_OP_WRITE)) {
376 			fs_write_stream_abort_error(_file->parent, &file->super_output, RANDOMFAIL_ERROR);
377 			return -1;
378 		}
379 	}
380 	return fs_write_stream_finish(_file->parent, &file->super_output);
381 }
382 
383 static int
fs_randomfail_lock(struct fs_file * _file,unsigned int secs,struct fs_lock ** lock_r)384 fs_randomfail_lock(struct fs_file *_file, unsigned int secs, struct fs_lock **lock_r)
385 {
386 	if (fs_random_fail(_file->fs, _file->event, 1, FS_OP_LOCK))
387 		return -1;
388 	return fs_lock(_file->parent, secs, lock_r);
389 }
390 
fs_randomfail_unlock(struct fs_lock * _lock ATTR_UNUSED)391 static void fs_randomfail_unlock(struct fs_lock *_lock ATTR_UNUSED)
392 {
393 	i_unreached();
394 }
395 
fs_randomfail_exists(struct fs_file * _file)396 static int fs_randomfail_exists(struct fs_file *_file)
397 {
398 	struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
399 	int ret;
400 
401 	if (fs_file_random_fail_begin(file, FS_OP_EXISTS))
402 		return -1;
403 	ret = fs_exists(_file->parent);
404 	return fs_file_random_fail_end(file, ret, FS_OP_EXISTS);
405 }
406 
fs_randomfail_stat(struct fs_file * _file,struct stat * st_r)407 static int fs_randomfail_stat(struct fs_file *_file, struct stat *st_r)
408 {
409 	struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
410 	int ret;
411 
412 	if (fs_file_random_fail_begin(file, FS_OP_STAT))
413 		return -1;
414 	ret = fs_stat(_file->parent, st_r);
415 	return fs_file_random_fail_end(file, ret, FS_OP_STAT);
416 }
417 
fs_randomfail_get_nlinks(struct fs_file * _file,nlink_t * nlinks_r)418 static int fs_randomfail_get_nlinks(struct fs_file *_file, nlink_t *nlinks_r)
419 {
420 	struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
421 	int ret;
422 
423 	if (fs_file_random_fail_begin(file, FS_OP_STAT))
424 		return -1;
425 	ret = fs_get_nlinks(_file->parent, nlinks_r);
426 	return fs_file_random_fail_end(file, ret, FS_OP_STAT);
427 }
428 
fs_randomfail_copy(struct fs_file * _src,struct fs_file * _dest)429 static int fs_randomfail_copy(struct fs_file *_src, struct fs_file *_dest)
430 {
431 	struct randomfail_fs_file *dest = RANDOMFAIL_FILE(_dest);
432 	int ret;
433 
434 	if (fs_file_random_fail_begin(dest, FS_OP_COPY))
435 		return -1;
436 
437 	if (_src != NULL)
438 		ret = fs_copy(_src->parent, _dest->parent);
439 	else
440 		ret = fs_copy_finish_async(_dest->parent);
441 	return fs_file_random_fail_end(dest, ret, FS_OP_COPY);
442 }
443 
fs_randomfail_rename(struct fs_file * _src,struct fs_file * _dest)444 static int fs_randomfail_rename(struct fs_file *_src, struct fs_file *_dest)
445 {
446 	struct randomfail_fs_file *dest = RANDOMFAIL_FILE(_dest);
447 	int ret;
448 
449 	if (fs_file_random_fail_begin(dest, FS_OP_RENAME))
450 		return -1;
451 	ret = fs_rename(_src->parent, _dest->parent);
452 	return fs_file_random_fail_end(dest, ret, FS_OP_RENAME);
453 }
454 
fs_randomfail_delete(struct fs_file * _file)455 static int fs_randomfail_delete(struct fs_file *_file)
456 {
457 	struct randomfail_fs_file *file = RANDOMFAIL_FILE(_file);
458 	int ret;
459 
460 	if (fs_file_random_fail_begin(file, FS_OP_DELETE))
461 		return -1;
462 	ret = fs_delete(_file->parent);
463 	return fs_file_random_fail_end(file, ret, FS_OP_DELETE);
464 }
465 
fs_randomfail_iter_alloc(void)466 static struct fs_iter *fs_randomfail_iter_alloc(void)
467 {
468 	struct randomfail_fs_iter *iter = i_new(struct randomfail_fs_iter, 1);
469 	return &iter->iter;
470 }
471 
472 static void
fs_randomfail_iter_init(struct fs_iter * _iter,const char * path,enum fs_iter_flags flags)473 fs_randomfail_iter_init(struct fs_iter *_iter, const char *path,
474 			enum fs_iter_flags flags)
475 {
476 	struct randomfail_fs_iter *iter = RANDOMFAIL_ITER(_iter);
477 	uoff_t pos;
478 
479 	iter->super = fs_iter_init_parent(_iter, path, flags);
480 	if (fs_random_fail_range(_iter->fs, _iter->event, FS_OP_ITER, &pos))
481 		iter->fail_pos = pos + 1;
482 }
483 
fs_randomfail_iter_next(struct fs_iter * _iter)484 static const char *fs_randomfail_iter_next(struct fs_iter *_iter)
485 {
486 	struct randomfail_fs_iter *iter = RANDOMFAIL_ITER(_iter);
487 	const char *fname;
488 
489 	if (iter->fail_pos > 0) {
490 		if (iter->fail_pos == 1)
491 			return NULL;
492 		iter->fail_pos--;
493 	}
494 
495 	iter->super->async_callback = _iter->async_callback;
496 	iter->super->async_context = _iter->async_context;
497 
498 	fname = fs_iter_next(iter->super);
499 	_iter->async_have_more = iter->super->async_have_more;
500 	return fname;
501 }
502 
fs_randomfail_iter_deinit(struct fs_iter * _iter)503 static int fs_randomfail_iter_deinit(struct fs_iter *_iter)
504 {
505 	struct randomfail_fs_iter *iter = RANDOMFAIL_ITER(_iter);
506 	const char *error;
507 	int ret;
508 
509 	if ((ret = fs_iter_deinit(&iter->super, &error)) < 0)
510 		fs_set_error_errno(_iter->event, "%s", error);
511 	if (iter->fail_pos == 1) {
512 		fs_set_error(_iter->event, EIO, RANDOMFAIL_ERROR);
513 		ret = -1;
514 	}
515 	return ret;
516 }
517 
518 const struct fs fs_class_randomfail = {
519 	.name = "randomfail",
520 	.v = {
521 		fs_randomfail_alloc,
522 		fs_randomfail_init,
523 		NULL,
524 		fs_randomfail_free,
525 		fs_randomfail_get_properties,
526 		fs_randomfail_file_alloc,
527 		fs_randomfail_file_init,
528 		fs_randomfail_file_deinit,
529 		fs_wrapper_file_close,
530 		fs_wrapper_file_get_path,
531 		fs_wrapper_set_async_callback,
532 		fs_wrapper_wait_async,
533 		fs_wrapper_set_metadata,
534 		fs_randomfail_get_metadata,
535 		fs_randomfail_prefetch,
536 		fs_randomfail_read,
537 		fs_randomfail_read_stream,
538 		fs_randomfail_write,
539 		fs_randomfail_write_stream,
540 		fs_randomfail_write_stream_finish,
541 		fs_randomfail_lock,
542 		fs_randomfail_unlock,
543 		fs_randomfail_exists,
544 		fs_randomfail_stat,
545 		fs_randomfail_copy,
546 		fs_randomfail_rename,
547 		fs_randomfail_delete,
548 		fs_randomfail_iter_alloc,
549 		fs_randomfail_iter_init,
550 		fs_randomfail_iter_next,
551 		fs_randomfail_iter_deinit,
552 		NULL,
553 		fs_randomfail_get_nlinks,
554 	}
555 };
556