1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2021 Vladimir Kondratyev <wulf@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions are 8 * met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in 13 * the documentation and/or other materials provided with the 14 * distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #ifndef _LINUXKPI_LINUX_SEQLOCK_H__ 30 #define _LINUXKPI_LINUX_SEQLOCK_H__ 31 32 #include <sys/param.h> 33 #include <sys/systm.h> 34 #include <sys/lock.h> 35 #include <sys/mutex.h> 36 #include <sys/rwlock.h> 37 #include <sys/seqc.h> 38 39 struct lock_class_key; 40 41 struct seqcount { 42 seqc_t seqc; 43 }; 44 typedef struct seqcount seqcount_t; 45 46 struct seqlock { 47 struct mtx seql_lock; 48 struct seqcount seql_count; 49 }; 50 typedef struct seqlock seqlock_t; 51 52 struct seqcount_mutex { 53 seqc_t seqc; 54 }; 55 typedef struct seqcount_mutex seqcount_mutex_t; 56 typedef struct seqcount_mutex seqcount_ww_mutex_t; 57 58 static inline void 59 __seqcount_init(struct seqcount *seqcount, const char *name __unused, 60 struct lock_class_key *key __unused) 61 { 62 seqcount->seqc = 0; 63 } 64 #define seqcount_init(seqcount) __seqcount_init(seqcount, NULL, NULL) 65 66 static inline void 67 seqcount_mutex_init(struct seqcount_mutex *seqcount, void *mutex __unused) 68 { 69 seqcount->seqc = 0; 70 } 71 72 #define seqcount_ww_mutex_init(seqcount, ww_mutex) \ 73 seqcount_mutex_init((seqcount), (ww_mutex)) 74 75 #define write_seqcount_begin(s) \ 76 _Generic(*(s), \ 77 struct seqcount: seqc_sleepable_write_begin, \ 78 struct seqcount_mutex: seqc_write_begin \ 79 )(&(s)->seqc) 80 81 #define write_seqcount_end(s) \ 82 _Generic(*(s), \ 83 struct seqcount: seqc_sleepable_write_end, \ 84 struct seqcount_mutex: seqc_write_end \ 85 )(&(s)->seqc) 86 87 #define read_seqcount_begin(s) seqc_read(&(s)->seqc) 88 #define raw_read_seqcount(s) seqc_read_any(&(s)->seqc) 89 90 /* 91 * XXX: Are predicts from inline functions still not honored by clang? 92 */ 93 #define __read_seqcount_retry(seqcount, gen) \ 94 (!seqc_consistent_no_fence(&(seqcount)->seqc, gen)) 95 #define read_seqcount_retry(seqcount, gen) \ 96 (!seqc_consistent(&(seqcount)->seqc, gen)) 97 98 static inline void 99 seqlock_init(struct seqlock *seqlock) 100 { 101 /* 102 * Don't enroll to witness(4) to avoid orphaned references after struct 103 * seqlock has been freed. There is no seqlock destructor exists so we 104 * can't expect automatic mtx_destroy() execution before free(). 105 */ 106 mtx_init(&seqlock->seql_lock, "seqlock", NULL, MTX_DEF|MTX_NOWITNESS); 107 seqcount_init(&seqlock->seql_count); 108 } 109 110 static inline void 111 lkpi_write_seqlock(struct seqlock *seqlock, const bool irqsave) 112 { 113 mtx_lock(&seqlock->seql_lock); 114 if (irqsave) 115 critical_enter(); 116 write_seqcount_begin(&seqlock->seql_count); 117 } 118 119 static inline void 120 write_seqlock(struct seqlock *seqlock) 121 { 122 lkpi_write_seqlock(seqlock, false); 123 } 124 125 static inline void 126 lkpi_write_sequnlock(struct seqlock *seqlock, const bool irqsave) 127 { 128 write_seqcount_end(&seqlock->seql_count); 129 if (irqsave) 130 critical_exit(); 131 mtx_unlock(&seqlock->seql_lock); 132 } 133 134 static inline void 135 write_sequnlock(struct seqlock *seqlock) 136 { 137 lkpi_write_sequnlock(seqlock, false); 138 } 139 140 /* 141 * Disable preemption when the consumer wants to disable interrupts. This 142 * ensures that the caller won't be starved if it is preempted by a 143 * higher-priority reader, but assumes that the caller won't perform any 144 * blocking operations while holding the write lock; probably a safe 145 * assumption. 146 */ 147 #define write_seqlock_irqsave(seqlock, flags) do { \ 148 (flags) = 0; \ 149 lkpi_write_seqlock(seqlock, true); \ 150 } while (0) 151 152 static inline void 153 write_sequnlock_irqrestore(struct seqlock *seqlock, 154 unsigned long flags __unused) 155 { 156 lkpi_write_sequnlock(seqlock, true); 157 } 158 159 static inline unsigned 160 read_seqbegin(const struct seqlock *seqlock) 161 { 162 return (read_seqcount_begin(&seqlock->seql_count)); 163 } 164 165 #define read_seqretry(seqlock, gen) \ 166 read_seqcount_retry(&(seqlock)->seql_count, gen) 167 168 #endif /* _LINUXKPI_LINUX_SEQLOCK_H__ */ 169