1 #ifndef QEMU_RCU_H 2 #define QEMU_RCU_H 3 4 /* 5 * urcu-mb.h 6 * 7 * Userspace RCU header with explicit memory barrier. 8 * 9 * This library is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU Lesser General Public 11 * License as published by the Free Software Foundation; either 12 * version 2.1 of the License, or (at your option) any later version. 13 * 14 * This library is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 17 * Lesser General Public License for more details. 18 * 19 * You should have received a copy of the GNU Lesser General Public 20 * License along with this library; if not, write to the Free Software 21 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 22 * 23 * IBM's contributions to this file may be relicensed under LGPLv2 or later. 24 */ 25 26 27 #include "qemu/thread.h" 28 #include "qemu/queue.h" 29 #include "qemu/atomic.h" 30 #include "qemu/notify.h" 31 #include "qemu/sys_membarrier.h" 32 33 #ifdef __cplusplus 34 extern "C" { 35 #endif 36 37 /* 38 * Important ! 39 * 40 * Each thread containing read-side critical sections must be registered 41 * with rcu_register_thread() before calling rcu_read_lock(). 42 * rcu_unregister_thread() should be called before the thread exits. 43 */ 44 45 #ifdef DEBUG_RCU 46 #define rcu_assert(args...) assert(args) 47 #else 48 #define rcu_assert(args...) 49 #endif 50 51 /* 52 * Global quiescent period counter with low-order bits unused. 53 * Using a int rather than a char to eliminate false register dependencies 54 * causing stalls on some architectures. 55 */ 56 extern unsigned long rcu_gp_ctr; 57 58 extern QemuEvent rcu_gp_event; 59 60 struct rcu_reader_data { 61 /* Data used by both reader and synchronize_rcu() */ 62 unsigned long ctr; 63 bool waiting; 64 65 /* Data used by reader only */ 66 unsigned depth; 67 68 /* Data used for registry, protected by rcu_registry_lock */ 69 QLIST_ENTRY(rcu_reader_data) node; 70 71 /* 72 * NotifierList used to force an RCU grace period. Accessed under 73 * rcu_registry_lock. Note that the notifier is called _outside_ 74 * the thread! 75 */ 76 NotifierList force_rcu; 77 }; 78 79 extern __thread struct rcu_reader_data rcu_reader; 80 81 static inline void rcu_read_lock(void) 82 { 83 struct rcu_reader_data *p_rcu_reader = &rcu_reader; 84 unsigned ctr; 85 86 if (p_rcu_reader->depth++ > 0) { 87 return; 88 } 89 90 ctr = qatomic_read(&rcu_gp_ctr); 91 qatomic_set(&p_rcu_reader->ctr, ctr); 92 93 /* Write p_rcu_reader->ctr before reading RCU-protected pointers. */ 94 smp_mb_placeholder(); 95 } 96 97 static inline void rcu_read_unlock(void) 98 { 99 struct rcu_reader_data *p_rcu_reader = &rcu_reader; 100 101 assert(p_rcu_reader->depth != 0); 102 if (--p_rcu_reader->depth > 0) { 103 return; 104 } 105 106 /* Ensure that the critical section is seen to precede the 107 * store to p_rcu_reader->ctr. Together with the following 108 * smp_mb_placeholder(), this ensures writes to p_rcu_reader->ctr 109 * are sequentially consistent. 110 */ 111 qatomic_store_release(&p_rcu_reader->ctr, 0); 112 113 /* Write p_rcu_reader->ctr before reading p_rcu_reader->waiting. */ 114 smp_mb_placeholder(); 115 if (unlikely(qatomic_read(&p_rcu_reader->waiting))) { 116 qatomic_set(&p_rcu_reader->waiting, false); 117 qemu_event_set(&rcu_gp_event); 118 } 119 } 120 121 extern void synchronize_rcu(void); 122 123 /* 124 * Reader thread registration. 125 */ 126 extern void rcu_register_thread(void); 127 extern void rcu_unregister_thread(void); 128 129 /* 130 * Support for fork(). fork() support is enabled at startup. 131 */ 132 extern void rcu_enable_atfork(void); 133 extern void rcu_disable_atfork(void); 134 135 struct rcu_head; 136 typedef void RCUCBFunc(struct rcu_head *head); 137 138 struct rcu_head { 139 struct rcu_head *next; 140 RCUCBFunc *func; 141 }; 142 143 extern void call_rcu1(struct rcu_head *head, RCUCBFunc *func); 144 extern void drain_call_rcu(void); 145 146 /* The operands of the minus operator must have the same type, 147 * which must be the one that we specify in the cast. 148 */ 149 #define call_rcu(head, func, field) \ 150 call_rcu1(({ \ 151 char __attribute__((unused)) \ 152 offset_must_be_zero[-offsetof(typeof(*(head)), field)], \ 153 func_type_invalid = (func) - (void (*)(typeof(head)))(func); \ 154 &(head)->field; \ 155 }), \ 156 (RCUCBFunc *)(func)) 157 158 #define g_free_rcu(obj, field) \ 159 call_rcu1(({ \ 160 char __attribute__((unused)) \ 161 offset_must_be_zero[-offsetof(typeof(*(obj)), field)]; \ 162 &(obj)->field; \ 163 }), \ 164 (RCUCBFunc *)g_free); 165 166 typedef void RCUReadAuto; 167 static inline RCUReadAuto *rcu_read_auto_lock(void) 168 { 169 rcu_read_lock(); 170 /* Anything non-NULL causes the cleanup function to be called */ 171 return (void *)(uintptr_t)0x1; 172 } 173 174 static inline void rcu_read_auto_unlock(RCUReadAuto *r) 175 { 176 rcu_read_unlock(); 177 } 178 179 G_DEFINE_AUTOPTR_CLEANUP_FUNC(RCUReadAuto, rcu_read_auto_unlock) 180 181 #define WITH_RCU_READ_LOCK_GUARD() \ 182 WITH_RCU_READ_LOCK_GUARD_(glue(_rcu_read_auto, __COUNTER__)) 183 184 #define WITH_RCU_READ_LOCK_GUARD_(var) \ 185 for (g_autoptr(RCUReadAuto) var = rcu_read_auto_lock(); \ 186 (var); rcu_read_auto_unlock(var), (var) = NULL) 187 188 #define RCU_READ_LOCK_GUARD() \ 189 g_autoptr(RCUReadAuto) _rcu_read_auto __attribute__((unused)) = rcu_read_auto_lock() 190 191 /* 192 * Force-RCU notifiers tell readers that they should exit their 193 * read-side critical section. 194 */ 195 void rcu_add_force_rcu_notifier(Notifier *n); 196 void rcu_remove_force_rcu_notifier(Notifier *n); 197 198 #ifdef __cplusplus 199 } 200 #endif 201 202 #endif /* QEMU_RCU_H */ 203