1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2019, 2020 Jeffrey Roberson <jeff@FreeBSD.org> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice unmodified, this list of conditions, and the following 11 * disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 * 27 * $FreeBSD$ 28 * 29 */ 30 31 #ifndef _SYS_SMR_H_ 32 #define _SYS_SMR_H_ 33 34 #include <sys/_smr.h> 35 36 /* 37 * Safe memory reclamation. See subr_smr.c for a description of the 38 * algorithm. 39 * 40 * Readers synchronize with smr_enter()/exit() and writers may either 41 * free directly to a SMR UMA zone or use smr_synchronize or wait. 42 */ 43 44 /* 45 * Modular arithmetic for comparing sequence numbers that have 46 * potentially wrapped. Copied from tcp_seq.h. 47 */ 48 #define SMR_SEQ_LT(a, b) ((int32_t)((a)-(b)) < 0) 49 #define SMR_SEQ_LEQ(a, b) ((int32_t)((a)-(b)) <= 0) 50 #define SMR_SEQ_GT(a, b) ((int32_t)((a)-(b)) > 0) 51 #define SMR_SEQ_GEQ(a, b) ((int32_t)((a)-(b)) >= 0) 52 #define SMR_SEQ_DELTA(a, b) ((int32_t)((a)-(b))) 53 54 #define SMR_SEQ_INVALID 0 55 56 /* Shared SMR state. */ 57 struct smr_shared { 58 const char *s_name; /* Name for debugging/reporting. */ 59 smr_seq_t s_wr_seq; /* Current write sequence #. */ 60 smr_seq_t s_rd_seq; /* Minimum observed read sequence. */ 61 }; 62 typedef struct smr_shared *smr_shared_t; 63 64 /* Per-cpu SMR state. */ 65 struct smr { 66 smr_seq_t c_seq; /* Current observed sequence. */ 67 smr_shared_t c_shared; /* Shared SMR state. */ 68 int c_deferred; /* Deferred advance counter. */ 69 }; 70 71 #define SMR_ENTERED(smr) \ 72 (curthread->td_critnest != 0 && zpcpu_get((smr))->c_seq != SMR_SEQ_INVALID) 73 74 #define SMR_ASSERT_ENTERED(smr) \ 75 KASSERT(SMR_ENTERED(smr), ("Not in smr section")) 76 77 #define SMR_ASSERT_NOT_ENTERED(smr) \ 78 KASSERT(!SMR_ENTERED(smr), ("In smr section.")); 79 80 /* 81 * Return the current write sequence number. 82 */ 83 static inline smr_seq_t 84 smr_shared_current(smr_shared_t s) 85 { 86 87 return (atomic_load_int(&s->s_wr_seq)); 88 } 89 90 static inline smr_seq_t 91 smr_current(smr_t smr) 92 { 93 94 return (smr_shared_current(zpcpu_get(smr)->c_shared)); 95 } 96 97 /* 98 * Enter a read section. 99 */ 100 static inline void 101 smr_enter(smr_t smr) 102 { 103 104 critical_enter(); 105 smr = zpcpu_get(smr); 106 KASSERT(smr->c_seq == 0, 107 ("smr_enter(%s) does not support recursion.", 108 smr->c_shared->s_name)); 109 110 /* 111 * Store the current observed write sequence number in our 112 * per-cpu state so that it can be queried via smr_poll(). 113 * Frees that are newer than this stored value will be 114 * deferred until we call smr_exit(). 115 * 116 * An acquire barrier is used to synchronize with smr_exit() 117 * and smr_poll(). 118 * 119 * It is possible that a long delay between loading the wr_seq 120 * and storing the c_seq could create a situation where the 121 * rd_seq advances beyond our stored c_seq. In this situation 122 * only the observed wr_seq is stale, the fence still orders 123 * the load. See smr_poll() for details on how this condition 124 * is detected and handled there. 125 */ 126 /* This is an add because we do not have atomic_store_acq_int */ 127 atomic_add_acq_int(&smr->c_seq, smr_shared_current(smr->c_shared)); 128 } 129 130 /* 131 * Exit a read section. 132 */ 133 static inline void 134 smr_exit(smr_t smr) 135 { 136 137 smr = zpcpu_get(smr); 138 CRITICAL_ASSERT(curthread); 139 KASSERT(smr->c_seq != SMR_SEQ_INVALID, 140 ("smr_exit(%s) not in a smr section.", smr->c_shared->s_name)); 141 142 /* 143 * Clear the recorded sequence number. This allows poll() to 144 * detect CPUs not in read sections. 145 * 146 * Use release semantics to retire any stores before the sequence 147 * number is cleared. 148 */ 149 atomic_store_rel_int(&smr->c_seq, SMR_SEQ_INVALID); 150 critical_exit(); 151 } 152 153 /* 154 * Advances the write sequence number. Returns the sequence number 155 * required to ensure that all modifications are visible to readers. 156 */ 157 smr_seq_t smr_advance(smr_t smr); 158 159 /* 160 * Advances the write sequence number only after N calls. Returns 161 * the correct goal for a wr_seq that has not yet occurred. Used to 162 * minimize shared cacheline invalidations for frequent writers. 163 */ 164 smr_seq_t smr_advance_deferred(smr_t smr, int limit); 165 166 /* 167 * Returns true if a goal sequence has been reached. If 168 * wait is true this will busy loop until success. 169 */ 170 bool smr_poll(smr_t smr, smr_seq_t goal, bool wait); 171 172 /* Create a new SMR context. */ 173 smr_t smr_create(const char *name); 174 void smr_destroy(smr_t smr); 175 176 /* 177 * Blocking wait for all readers to observe 'goal'. 178 */ 179 static inline bool 180 smr_wait(smr_t smr, smr_seq_t goal) 181 { 182 183 return (smr_poll(smr, goal, true)); 184 } 185 186 /* 187 * Synchronize advances the write sequence and returns when all 188 * readers have observed it. 189 * 190 * If your application can cache a sequence number returned from 191 * smr_advance() and poll or wait at a later time there will 192 * be less chance of busy looping while waiting for readers. 193 */ 194 static inline void 195 smr_synchronize(smr_t smr) 196 { 197 198 smr_wait(smr, smr_advance(smr)); 199 } 200 201 /* Only at startup. */ 202 void smr_init(void); 203 204 #endif /* _SYS_SMR_H_ */ 205