xref: /linux/fs/bcachefs/util.h (revision afefc986)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _BCACHEFS_UTIL_H
3 #define _BCACHEFS_UTIL_H
4 
5 #include <linux/bio.h>
6 #include <linux/blkdev.h>
7 #include <linux/closure.h>
8 #include <linux/errno.h>
9 #include <linux/freezer.h>
10 #include <linux/kernel.h>
11 #include <linux/min_heap.h>
12 #include <linux/sched/clock.h>
13 #include <linux/llist.h>
14 #include <linux/log2.h>
15 #include <linux/percpu.h>
16 #include <linux/preempt.h>
17 #include <linux/ratelimit.h>
18 #include <linux/slab.h>
19 #include <linux/vmalloc.h>
20 #include <linux/workqueue.h>
21 
22 #include "mean_and_variance.h"
23 
24 #include "darray.h"
25 #include "time_stats.h"
26 
27 struct closure;
28 
29 #ifdef CONFIG_BCACHEFS_DEBUG
30 #define EBUG_ON(cond)		BUG_ON(cond)
31 #else
32 #define EBUG_ON(cond)
33 #endif
34 
35 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
36 #define CPU_BIG_ENDIAN		0
37 #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
38 #define CPU_BIG_ENDIAN		1
39 #endif
40 
41 /* type hackery */
42 
43 #define type_is_exact(_val, _type)					\
44 	__builtin_types_compatible_p(typeof(_val), _type)
45 
46 #define type_is(_val, _type)						\
47 	(__builtin_types_compatible_p(typeof(_val), _type) ||		\
48 	 __builtin_types_compatible_p(typeof(_val), const _type))
49 
50 /* Userspace doesn't align allocations as nicely as the kernel allocators: */
buf_pages(void * p,size_t len)51 static inline size_t buf_pages(void *p, size_t len)
52 {
53 	return DIV_ROUND_UP(len +
54 			    ((unsigned long) p & (PAGE_SIZE - 1)),
55 			    PAGE_SIZE);
56 }
57 
58 #define init_heap(heap, _size, gfp)					\
59 ({									\
60 	(heap)->nr = 0;						\
61 	(heap)->size = (_size);						\
62 	(heap)->data = kvmalloc((heap)->size * sizeof((heap)->data[0]),\
63 				 (gfp));				\
64 })
65 
66 #define free_heap(heap)							\
67 do {									\
68 	kvfree((heap)->data);						\
69 	(heap)->data = NULL;						\
70 } while (0)
71 
72 #define ANYSINT_MAX(t)							\
73 	((((t) 1 << (sizeof(t) * 8 - 2)) - (t) 1) * (t) 2 + (t) 1)
74 
75 #include "printbuf.h"
76 
77 #define prt_vprintf(_out, ...)		bch2_prt_vprintf(_out, __VA_ARGS__)
78 #define prt_printf(_out, ...)		bch2_prt_printf(_out, __VA_ARGS__)
79 #define printbuf_str(_buf)		bch2_printbuf_str(_buf)
80 #define printbuf_exit(_buf)		bch2_printbuf_exit(_buf)
81 
82 #define printbuf_tabstops_reset(_buf)	bch2_printbuf_tabstops_reset(_buf)
83 #define printbuf_tabstop_pop(_buf)	bch2_printbuf_tabstop_pop(_buf)
84 #define printbuf_tabstop_push(_buf, _n)	bch2_printbuf_tabstop_push(_buf, _n)
85 
86 #define printbuf_indent_add(_out, _n)	bch2_printbuf_indent_add(_out, _n)
87 #define printbuf_indent_sub(_out, _n)	bch2_printbuf_indent_sub(_out, _n)
88 
89 #define prt_newline(_out)		bch2_prt_newline(_out)
90 #define prt_tab(_out)			bch2_prt_tab(_out)
91 #define prt_tab_rjust(_out)		bch2_prt_tab_rjust(_out)
92 
93 #define prt_bytes_indented(...)		bch2_prt_bytes_indented(__VA_ARGS__)
94 #define prt_u64(_out, _v)		prt_printf(_out, "%llu", (u64) (_v))
95 #define prt_human_readable_u64(...)	bch2_prt_human_readable_u64(__VA_ARGS__)
96 #define prt_human_readable_s64(...)	bch2_prt_human_readable_s64(__VA_ARGS__)
97 #define prt_units_u64(...)		bch2_prt_units_u64(__VA_ARGS__)
98 #define prt_units_s64(...)		bch2_prt_units_s64(__VA_ARGS__)
99 #define prt_string_option(...)		bch2_prt_string_option(__VA_ARGS__)
100 #define prt_bitflags(...)		bch2_prt_bitflags(__VA_ARGS__)
101 #define prt_bitflags_vector(...)	bch2_prt_bitflags_vector(__VA_ARGS__)
102 
103 void bch2_pr_time_units(struct printbuf *, u64);
104 void bch2_prt_datetime(struct printbuf *, time64_t);
105 
106 #ifdef __KERNEL__
uuid_unparse_lower(u8 * uuid,char * out)107 static inline void uuid_unparse_lower(u8 *uuid, char *out)
108 {
109 	sprintf(out, "%pUb", uuid);
110 }
111 #else
112 #include <uuid/uuid.h>
113 #endif
114 
pr_uuid(struct printbuf * out,u8 * uuid)115 static inline void pr_uuid(struct printbuf *out, u8 *uuid)
116 {
117 	char uuid_str[40];
118 
119 	uuid_unparse_lower(uuid, uuid_str);
120 	prt_printf(out, "%s", uuid_str);
121 }
122 
123 int bch2_strtoint_h(const char *, int *);
124 int bch2_strtouint_h(const char *, unsigned int *);
125 int bch2_strtoll_h(const char *, long long *);
126 int bch2_strtoull_h(const char *, unsigned long long *);
127 int bch2_strtou64_h(const char *, u64 *);
128 
bch2_strtol_h(const char * cp,long * res)129 static inline int bch2_strtol_h(const char *cp, long *res)
130 {
131 #if BITS_PER_LONG == 32
132 	return bch2_strtoint_h(cp, (int *) res);
133 #else
134 	return bch2_strtoll_h(cp, (long long *) res);
135 #endif
136 }
137 
bch2_strtoul_h(const char * cp,long * res)138 static inline int bch2_strtoul_h(const char *cp, long *res)
139 {
140 #if BITS_PER_LONG == 32
141 	return bch2_strtouint_h(cp, (unsigned int *) res);
142 #else
143 	return bch2_strtoull_h(cp, (unsigned long long *) res);
144 #endif
145 }
146 
147 #define strtoi_h(cp, res)						\
148 	( type_is(*res, int)		? bch2_strtoint_h(cp, (void *) res)\
149 	: type_is(*res, long)		? bch2_strtol_h(cp, (void *) res)\
150 	: type_is(*res, long long)	? bch2_strtoll_h(cp, (void *) res)\
151 	: type_is(*res, unsigned)	? bch2_strtouint_h(cp, (void *) res)\
152 	: type_is(*res, unsigned long)	? bch2_strtoul_h(cp, (void *) res)\
153 	: type_is(*res, unsigned long long) ? bch2_strtoull_h(cp, (void *) res)\
154 	: -EINVAL)
155 
156 #define strtoul_safe(cp, var)						\
157 ({									\
158 	unsigned long _v;						\
159 	int _r = kstrtoul(cp, 10, &_v);					\
160 	if (!_r)							\
161 		var = _v;						\
162 	_r;								\
163 })
164 
165 #define strtoul_safe_clamp(cp, var, min, max)				\
166 ({									\
167 	unsigned long _v;						\
168 	int _r = kstrtoul(cp, 10, &_v);					\
169 	if (!_r)							\
170 		var = clamp_t(typeof(var), _v, min, max);		\
171 	_r;								\
172 })
173 
174 #define strtoul_safe_restrict(cp, var, min, max)			\
175 ({									\
176 	unsigned long _v;						\
177 	int _r = kstrtoul(cp, 10, &_v);					\
178 	if (!_r && _v >= min && _v <= max)				\
179 		var = _v;						\
180 	else								\
181 		_r = -EINVAL;						\
182 	_r;								\
183 })
184 
185 #define snprint(out, var)						\
186 	prt_printf(out,							\
187 		   type_is(var, int)		? "%i\n"		\
188 		 : type_is(var, unsigned)	? "%u\n"		\
189 		 : type_is(var, long)		? "%li\n"		\
190 		 : type_is(var, unsigned long)	? "%lu\n"		\
191 		 : type_is(var, s64)		? "%lli\n"		\
192 		 : type_is(var, u64)		? "%llu\n"		\
193 		 : type_is(var, char *)		? "%s\n"		\
194 		 : "%i\n", var)
195 
196 bool bch2_is_zero(const void *, size_t);
197 
198 u64 bch2_read_flag_list(const char *, const char * const[]);
199 
200 void bch2_prt_u64_base2_nbits(struct printbuf *, u64, unsigned);
201 void bch2_prt_u64_base2(struct printbuf *, u64);
202 
203 void bch2_print_string_as_lines(const char *prefix, const char *lines);
204 void bch2_print_string_as_lines_nonblocking(const char *prefix, const char *lines);
205 
206 typedef DARRAY(unsigned long) bch_stacktrace;
207 int bch2_save_backtrace(bch_stacktrace *stack, struct task_struct *, unsigned, gfp_t);
208 void bch2_prt_backtrace(struct printbuf *, bch_stacktrace *);
209 int bch2_prt_task_backtrace(struct printbuf *, struct task_struct *, unsigned, gfp_t);
210 
prt_bdevname(struct printbuf * out,struct block_device * bdev)211 static inline void prt_bdevname(struct printbuf *out, struct block_device *bdev)
212 {
213 #ifdef __KERNEL__
214 	prt_printf(out, "%pg", bdev);
215 #else
216 	prt_str(out, bdev->name);
217 #endif
218 }
219 
220 void bch2_time_stats_to_text(struct printbuf *, struct bch2_time_stats *);
221 
222 #define ewma_add(ewma, val, weight)					\
223 ({									\
224 	typeof(ewma) _ewma = (ewma);					\
225 	typeof(weight) _weight = (weight);				\
226 									\
227 	(((_ewma << _weight) - _ewma) + (val)) >> _weight;		\
228 })
229 
230 struct bch_ratelimit {
231 	/* Next time we want to do some work, in nanoseconds */
232 	u64			next;
233 
234 	/*
235 	 * Rate at which we want to do work, in units per nanosecond
236 	 * The units here correspond to the units passed to
237 	 * bch2_ratelimit_increment()
238 	 */
239 	unsigned		rate;
240 };
241 
bch2_ratelimit_reset(struct bch_ratelimit * d)242 static inline void bch2_ratelimit_reset(struct bch_ratelimit *d)
243 {
244 	d->next = local_clock();
245 }
246 
247 u64 bch2_ratelimit_delay(struct bch_ratelimit *);
248 void bch2_ratelimit_increment(struct bch_ratelimit *, u64);
249 
250 struct bch_pd_controller {
251 	struct bch_ratelimit	rate;
252 	unsigned long		last_update;
253 
254 	s64			last_actual;
255 	s64			smoothed_derivative;
256 
257 	unsigned		p_term_inverse;
258 	unsigned		d_smooth;
259 	unsigned		d_term;
260 
261 	/* for exporting to sysfs (no effect on behavior) */
262 	s64			last_derivative;
263 	s64			last_proportional;
264 	s64			last_change;
265 	s64			last_target;
266 
267 	/*
268 	 * If true, the rate will not increase if bch2_ratelimit_delay()
269 	 * is not being called often enough.
270 	 */
271 	bool			backpressure;
272 };
273 
274 void bch2_pd_controller_update(struct bch_pd_controller *, s64, s64, int);
275 void bch2_pd_controller_init(struct bch_pd_controller *);
276 void bch2_pd_controller_debug_to_text(struct printbuf *, struct bch_pd_controller *);
277 
278 #define sysfs_pd_controller_attribute(name)				\
279 	rw_attribute(name##_rate);					\
280 	rw_attribute(name##_rate_bytes);				\
281 	rw_attribute(name##_rate_d_term);				\
282 	rw_attribute(name##_rate_p_term_inverse);			\
283 	read_attribute(name##_rate_debug)
284 
285 #define sysfs_pd_controller_files(name)					\
286 	&sysfs_##name##_rate,						\
287 	&sysfs_##name##_rate_bytes,					\
288 	&sysfs_##name##_rate_d_term,					\
289 	&sysfs_##name##_rate_p_term_inverse,				\
290 	&sysfs_##name##_rate_debug
291 
292 #define sysfs_pd_controller_show(name, var)				\
293 do {									\
294 	sysfs_hprint(name##_rate,		(var)->rate.rate);	\
295 	sysfs_print(name##_rate_bytes,		(var)->rate.rate);	\
296 	sysfs_print(name##_rate_d_term,		(var)->d_term);		\
297 	sysfs_print(name##_rate_p_term_inverse,	(var)->p_term_inverse);	\
298 									\
299 	if (attr == &sysfs_##name##_rate_debug)				\
300 		bch2_pd_controller_debug_to_text(out, var);		\
301 } while (0)
302 
303 #define sysfs_pd_controller_store(name, var)				\
304 do {									\
305 	sysfs_strtoul_clamp(name##_rate,				\
306 			    (var)->rate.rate, 1, UINT_MAX);		\
307 	sysfs_strtoul_clamp(name##_rate_bytes,				\
308 			    (var)->rate.rate, 1, UINT_MAX);		\
309 	sysfs_strtoul(name##_rate_d_term,	(var)->d_term);		\
310 	sysfs_strtoul_clamp(name##_rate_p_term_inverse,			\
311 			    (var)->p_term_inverse, 1, INT_MAX);		\
312 } while (0)
313 
314 #define container_of_or_null(ptr, type, member)				\
315 ({									\
316 	typeof(ptr) _ptr = ptr;						\
317 	_ptr ? container_of(_ptr, type, member) : NULL;			\
318 })
319 
320 /* Does linear interpolation between powers of two */
fract_exp_two(unsigned x,unsigned fract_bits)321 static inline unsigned fract_exp_two(unsigned x, unsigned fract_bits)
322 {
323 	unsigned fract = x & ~(~0 << fract_bits);
324 
325 	x >>= fract_bits;
326 	x   = 1 << x;
327 	x  += (x * fract) >> fract_bits;
328 
329 	return x;
330 }
331 
332 void bch2_bio_map(struct bio *bio, void *base, size_t);
333 int bch2_bio_alloc_pages(struct bio *, size_t, gfp_t);
334 
335 #define closure_bio_submit(bio, cl)					\
336 do {									\
337 	closure_get(cl);						\
338 	submit_bio(bio);						\
339 } while (0)
340 
341 #define kthread_wait(cond)						\
342 ({									\
343 	int _ret = 0;							\
344 									\
345 	while (1) {							\
346 		set_current_state(TASK_INTERRUPTIBLE);			\
347 		if (kthread_should_stop()) {				\
348 			_ret = -1;					\
349 			break;						\
350 		}							\
351 									\
352 		if (cond)						\
353 			break;						\
354 									\
355 		schedule();						\
356 	}								\
357 	set_current_state(TASK_RUNNING);				\
358 	_ret;								\
359 })
360 
361 #define kthread_wait_freezable(cond)					\
362 ({									\
363 	int _ret = 0;							\
364 	while (1) {							\
365 		set_current_state(TASK_INTERRUPTIBLE);			\
366 		if (kthread_should_stop()) {				\
367 			_ret = -1;					\
368 			break;						\
369 		}							\
370 									\
371 		if (cond)						\
372 			break;						\
373 									\
374 		schedule();						\
375 		try_to_freeze();					\
376 	}								\
377 	set_current_state(TASK_RUNNING);				\
378 	_ret;								\
379 })
380 
381 size_t bch2_rand_range(size_t);
382 
383 void memcpy_to_bio(struct bio *, struct bvec_iter, const void *);
384 void memcpy_from_bio(void *, struct bio *, struct bvec_iter);
385 
memcpy_u64s_small(void * dst,const void * src,unsigned u64s)386 static inline void memcpy_u64s_small(void *dst, const void *src,
387 				     unsigned u64s)
388 {
389 	u64 *d = dst;
390 	const u64 *s = src;
391 
392 	while (u64s--)
393 		*d++ = *s++;
394 }
395 
__memcpy_u64s(void * dst,const void * src,unsigned u64s)396 static inline void __memcpy_u64s(void *dst, const void *src,
397 				 unsigned u64s)
398 {
399 #ifdef CONFIG_X86_64
400 	long d0, d1, d2;
401 
402 	asm volatile("rep ; movsq"
403 		     : "=&c" (d0), "=&D" (d1), "=&S" (d2)
404 		     : "0" (u64s), "1" (dst), "2" (src)
405 		     : "memory");
406 #else
407 	u64 *d = dst;
408 	const u64 *s = src;
409 
410 	while (u64s--)
411 		*d++ = *s++;
412 #endif
413 }
414 
memcpy_u64s(void * dst,const void * src,unsigned u64s)415 static inline void memcpy_u64s(void *dst, const void *src,
416 			       unsigned u64s)
417 {
418 	EBUG_ON(!(dst >= src + u64s * sizeof(u64) ||
419 		 dst + u64s * sizeof(u64) <= src));
420 
421 	__memcpy_u64s(dst, src, u64s);
422 }
423 
__memmove_u64s_down(void * dst,const void * src,unsigned u64s)424 static inline void __memmove_u64s_down(void *dst, const void *src,
425 				       unsigned u64s)
426 {
427 	__memcpy_u64s(dst, src, u64s);
428 }
429 
memmove_u64s_down(void * dst,const void * src,unsigned u64s)430 static inline void memmove_u64s_down(void *dst, const void *src,
431 				     unsigned u64s)
432 {
433 	EBUG_ON(dst > src);
434 
435 	__memmove_u64s_down(dst, src, u64s);
436 }
437 
__memmove_u64s_down_small(void * dst,const void * src,unsigned u64s)438 static inline void __memmove_u64s_down_small(void *dst, const void *src,
439 				       unsigned u64s)
440 {
441 	memcpy_u64s_small(dst, src, u64s);
442 }
443 
memmove_u64s_down_small(void * dst,const void * src,unsigned u64s)444 static inline void memmove_u64s_down_small(void *dst, const void *src,
445 				     unsigned u64s)
446 {
447 	EBUG_ON(dst > src);
448 
449 	__memmove_u64s_down_small(dst, src, u64s);
450 }
451 
__memmove_u64s_up_small(void * _dst,const void * _src,unsigned u64s)452 static inline void __memmove_u64s_up_small(void *_dst, const void *_src,
453 					   unsigned u64s)
454 {
455 	u64 *dst = (u64 *) _dst + u64s;
456 	u64 *src = (u64 *) _src + u64s;
457 
458 	while (u64s--)
459 		*--dst = *--src;
460 }
461 
memmove_u64s_up_small(void * dst,const void * src,unsigned u64s)462 static inline void memmove_u64s_up_small(void *dst, const void *src,
463 					 unsigned u64s)
464 {
465 	EBUG_ON(dst < src);
466 
467 	__memmove_u64s_up_small(dst, src, u64s);
468 }
469 
__memmove_u64s_up(void * _dst,const void * _src,unsigned u64s)470 static inline void __memmove_u64s_up(void *_dst, const void *_src,
471 				     unsigned u64s)
472 {
473 	u64 *dst = (u64 *) _dst + u64s - 1;
474 	u64 *src = (u64 *) _src + u64s - 1;
475 
476 #ifdef CONFIG_X86_64
477 	long d0, d1, d2;
478 
479 	asm volatile("std ;\n"
480 		     "rep ; movsq\n"
481 		     "cld ;\n"
482 		     : "=&c" (d0), "=&D" (d1), "=&S" (d2)
483 		     : "0" (u64s), "1" (dst), "2" (src)
484 		     : "memory");
485 #else
486 	while (u64s--)
487 		*dst-- = *src--;
488 #endif
489 }
490 
memmove_u64s_up(void * dst,const void * src,unsigned u64s)491 static inline void memmove_u64s_up(void *dst, const void *src,
492 				   unsigned u64s)
493 {
494 	EBUG_ON(dst < src);
495 
496 	__memmove_u64s_up(dst, src, u64s);
497 }
498 
memmove_u64s(void * dst,const void * src,unsigned u64s)499 static inline void memmove_u64s(void *dst, const void *src,
500 				unsigned u64s)
501 {
502 	if (dst < src)
503 		__memmove_u64s_down(dst, src, u64s);
504 	else
505 		__memmove_u64s_up(dst, src, u64s);
506 }
507 
508 /* Set the last few bytes up to a u64 boundary given an offset into a buffer. */
memset_u64s_tail(void * s,int c,unsigned bytes)509 static inline void memset_u64s_tail(void *s, int c, unsigned bytes)
510 {
511 	unsigned rem = round_up(bytes, sizeof(u64)) - bytes;
512 
513 	memset(s + bytes, c, rem);
514 }
515 
516 /* just the memmove, doesn't update @_nr */
517 #define __array_insert_item(_array, _nr, _pos)				\
518 	memmove(&(_array)[(_pos) + 1],					\
519 		&(_array)[(_pos)],					\
520 		sizeof((_array)[0]) * ((_nr) - (_pos)))
521 
522 #define array_insert_item(_array, _nr, _pos, _new_item)			\
523 do {									\
524 	__array_insert_item(_array, _nr, _pos);				\
525 	(_nr)++;							\
526 	(_array)[(_pos)] = (_new_item);					\
527 } while (0)
528 
529 #define array_remove_items(_array, _nr, _pos, _nr_to_remove)		\
530 do {									\
531 	(_nr) -= (_nr_to_remove);					\
532 	memmove(&(_array)[(_pos)],					\
533 		&(_array)[(_pos) + (_nr_to_remove)],			\
534 		sizeof((_array)[0]) * ((_nr) - (_pos)));		\
535 } while (0)
536 
537 #define array_remove_item(_array, _nr, _pos)				\
538 	array_remove_items(_array, _nr, _pos, 1)
539 
__move_gap(void * array,size_t element_size,size_t nr,size_t size,size_t old_gap,size_t new_gap)540 static inline void __move_gap(void *array, size_t element_size,
541 			      size_t nr, size_t size,
542 			      size_t old_gap, size_t new_gap)
543 {
544 	size_t gap_end = old_gap + size - nr;
545 
546 	if (new_gap < old_gap) {
547 		size_t move = old_gap - new_gap;
548 
549 		memmove(array + element_size * (gap_end - move),
550 			array + element_size * (old_gap - move),
551 				element_size * move);
552 	} else if (new_gap > old_gap) {
553 		size_t move = new_gap - old_gap;
554 
555 		memmove(array + element_size * old_gap,
556 			array + element_size * gap_end,
557 				element_size * move);
558 	}
559 }
560 
561 /* Move the gap in a gap buffer: */
562 #define move_gap(_d, _new_gap)						\
563 do {									\
564 	BUG_ON(_new_gap > (_d)->nr);					\
565 	BUG_ON((_d)->gap > (_d)->nr);					\
566 									\
567 	__move_gap((_d)->data, sizeof((_d)->data[0]),			\
568 		   (_d)->nr, (_d)->size, (_d)->gap, _new_gap);		\
569 	(_d)->gap = _new_gap;						\
570 } while (0)
571 
572 #define bubble_sort(_base, _nr, _cmp)					\
573 do {									\
574 	ssize_t _i, _last;						\
575 	bool _swapped = true;						\
576 									\
577 	for (_last= (ssize_t) (_nr) - 1; _last > 0 && _swapped; --_last) {\
578 		_swapped = false;					\
579 		for (_i = 0; _i < _last; _i++)				\
580 			if (_cmp((_base)[_i], (_base)[_i + 1]) > 0) {	\
581 				swap((_base)[_i], (_base)[_i + 1]);	\
582 				_swapped = true;			\
583 			}						\
584 	}								\
585 } while (0)
586 
587 #define per_cpu_sum(_p)							\
588 ({									\
589 	typeof(*_p) _ret = 0;						\
590 									\
591 	int cpu;							\
592 	for_each_possible_cpu(cpu)					\
593 		_ret += *per_cpu_ptr(_p, cpu);				\
594 	_ret;								\
595 })
596 
percpu_u64_get(u64 __percpu * src)597 static inline u64 percpu_u64_get(u64 __percpu *src)
598 {
599 	return per_cpu_sum(src);
600 }
601 
percpu_u64_set(u64 __percpu * dst,u64 src)602 static inline void percpu_u64_set(u64 __percpu *dst, u64 src)
603 {
604 	int cpu;
605 
606 	for_each_possible_cpu(cpu)
607 		*per_cpu_ptr(dst, cpu) = 0;
608 	this_cpu_write(*dst, src);
609 }
610 
acc_u64s(u64 * acc,const u64 * src,unsigned nr)611 static inline void acc_u64s(u64 *acc, const u64 *src, unsigned nr)
612 {
613 	for (unsigned i = 0; i < nr; i++)
614 		acc[i] += src[i];
615 }
616 
acc_u64s_percpu(u64 * acc,const u64 __percpu * src,unsigned nr)617 static inline void acc_u64s_percpu(u64 *acc, const u64 __percpu *src,
618 				   unsigned nr)
619 {
620 	int cpu;
621 
622 	for_each_possible_cpu(cpu)
623 		acc_u64s(acc, per_cpu_ptr(src, cpu), nr);
624 }
625 
percpu_memset(void __percpu * p,int c,size_t bytes)626 static inline void percpu_memset(void __percpu *p, int c, size_t bytes)
627 {
628 	int cpu;
629 
630 	for_each_possible_cpu(cpu)
631 		memset(per_cpu_ptr(p, cpu), c, bytes);
632 }
633 
634 u64 *bch2_acc_percpu_u64s(u64 __percpu *, unsigned);
635 
636 #define cmp_int(l, r)		((l > r) - (l < r))
637 
u8_cmp(u8 l,u8 r)638 static inline int u8_cmp(u8 l, u8 r)
639 {
640 	return cmp_int(l, r);
641 }
642 
cmp_le32(__le32 l,__le32 r)643 static inline int cmp_le32(__le32 l, __le32 r)
644 {
645 	return cmp_int(le32_to_cpu(l), le32_to_cpu(r));
646 }
647 
648 #include <linux/uuid.h>
649 
650 #define QSTR(n) { { { .len = strlen(n) } }, .name = n }
651 
qstr_eq(const struct qstr l,const struct qstr r)652 static inline bool qstr_eq(const struct qstr l, const struct qstr r)
653 {
654 	return l.len == r.len && !memcmp(l.name, r.name, l.len);
655 }
656 
657 void bch2_darray_str_exit(darray_str *);
658 int bch2_split_devs(const char *, darray_str *);
659 
660 #ifdef __KERNEL__
661 
662 __must_check
copy_to_user_errcode(void __user * to,const void * from,unsigned long n)663 static inline int copy_to_user_errcode(void __user *to, const void *from, unsigned long n)
664 {
665 	return copy_to_user(to, from, n) ? -EFAULT : 0;
666 }
667 
668 __must_check
copy_from_user_errcode(void * to,const void __user * from,unsigned long n)669 static inline int copy_from_user_errcode(void *to, const void __user *from, unsigned long n)
670 {
671 	return copy_from_user(to, from, n) ? -EFAULT : 0;
672 }
673 
674 #endif
675 
mod_bit(long nr,volatile unsigned long * addr,bool v)676 static inline void mod_bit(long nr, volatile unsigned long *addr, bool v)
677 {
678 	if (v)
679 		set_bit(nr, addr);
680 	else
681 		clear_bit(nr, addr);
682 }
683 
__set_bit_le64(size_t bit,__le64 * addr)684 static inline void __set_bit_le64(size_t bit, __le64 *addr)
685 {
686 	addr[bit / 64] |= cpu_to_le64(BIT_ULL(bit % 64));
687 }
688 
__clear_bit_le64(size_t bit,__le64 * addr)689 static inline void __clear_bit_le64(size_t bit, __le64 *addr)
690 {
691 	addr[bit / 64] &= ~cpu_to_le64(BIT_ULL(bit % 64));
692 }
693 
test_bit_le64(size_t bit,__le64 * addr)694 static inline bool test_bit_le64(size_t bit, __le64 *addr)
695 {
696 	return (addr[bit / 64] & cpu_to_le64(BIT_ULL(bit % 64))) != 0;
697 }
698 
699 #endif /* _BCACHEFS_UTIL_H */
700