1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_WORD_AT_A_TIME_H 3 #define _ASM_WORD_AT_A_TIME_H 4 5 #include <linux/kernel.h> 6 #include <asm/asm-extable.h> 7 #include <asm/bitsperlong.h> 8 9 struct word_at_a_time { 10 const unsigned long bits; 11 }; 12 13 #define WORD_AT_A_TIME_CONSTANTS { REPEAT_BYTE(0x7f) } 14 15 static inline unsigned long prep_zero_mask(unsigned long val, unsigned long data, const struct word_at_a_time *c) 16 { 17 return data; 18 } 19 20 static inline unsigned long create_zero_mask(unsigned long data) 21 { 22 return __fls(data); 23 } 24 25 static inline unsigned long find_zero(unsigned long data) 26 { 27 return (data ^ (BITS_PER_LONG - 1)) >> 3; 28 } 29 30 static inline unsigned long has_zero(unsigned long val, unsigned long *data, const struct word_at_a_time *c) 31 { 32 unsigned long mask = (val & c->bits) + c->bits; 33 34 *data = ~(mask | val | c->bits); 35 return *data; 36 } 37 38 static inline unsigned long zero_bytemask(unsigned long data) 39 { 40 return ~1UL << data; 41 } 42 43 /* 44 * Load an unaligned word from kernel space. 45 * 46 * In the (very unlikely) case of the word being a page-crosser 47 * and the next page not being mapped, take the exception and 48 * return zeroes in the non-existing part. 49 */ 50 static inline unsigned long load_unaligned_zeropad(const void *addr) 51 { 52 unsigned long data; 53 54 asm volatile( 55 "0: lg %[data],0(%[addr])\n" 56 "1: nopr %%r7\n" 57 EX_TABLE_ZEROPAD(0b, 1b, %[data], %[addr]) 58 EX_TABLE_ZEROPAD(1b, 1b, %[data], %[addr]) 59 : [data] "=d" (data) 60 : [addr] "a" (addr), "m" (*(unsigned long *)addr)); 61 return data; 62 } 63 64 #endif /* _ASM_WORD_AT_A_TIME_H */ 65