1 #ifndef QEMU_RCU_H 2 #define QEMU_RCU_H 3 4 /* 5 * urcu-mb.h 6 * 7 * Userspace RCU header with explicit memory barrier. 8 * 9 * This library is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU Lesser General Public 11 * License as published by the Free Software Foundation; either 12 * version 2.1 of the License, or (at your option) any later version. 13 * 14 * This library is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 17 * Lesser General Public License for more details. 18 * 19 * You should have received a copy of the GNU Lesser General Public 20 * License along with this library; if not, write to the Free Software 21 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 22 * 23 * IBM's contributions to this file may be relicensed under LGPLv2 or later. 24 */ 25 26 27 #include "qemu/thread.h" 28 #include "qemu/queue.h" 29 #include "qemu/atomic.h" 30 #include "qemu/notify.h" 31 #include "qemu/sys_membarrier.h" 32 #include "qemu/coroutine-tls.h" 33 34 #ifdef __cplusplus 35 extern "C" { 36 #endif 37 38 /* 39 * Important ! 40 * 41 * Each thread containing read-side critical sections must be registered 42 * with rcu_register_thread() before calling rcu_read_lock(). 43 * rcu_unregister_thread() should be called before the thread exits. 44 */ 45 46 #ifdef DEBUG_RCU 47 #define rcu_assert(args...) assert(args) 48 #else 49 #define rcu_assert(args...) 50 #endif 51 52 /* 53 * Global quiescent period counter with low-order bits unused. 54 * Using a int rather than a char to eliminate false register dependencies 55 * causing stalls on some architectures. 56 */ 57 extern unsigned long rcu_gp_ctr; 58 59 extern QemuEvent rcu_gp_event; 60 61 struct rcu_reader_data { 62 /* Data used by both reader and synchronize_rcu() */ 63 unsigned long ctr; 64 bool waiting; 65 66 /* Data used by reader only */ 67 unsigned depth; 68 69 /* Data used for registry, protected by rcu_registry_lock */ 70 QLIST_ENTRY(rcu_reader_data) node; 71 72 /* 73 * NotifierList used to force an RCU grace period. Accessed under 74 * rcu_registry_lock. Note that the notifier is called _outside_ 75 * the thread! 76 */ 77 NotifierList force_rcu; 78 }; 79 80 QEMU_DECLARE_CO_TLS(struct rcu_reader_data, rcu_reader) 81 82 static inline void rcu_read_lock(void) 83 { 84 struct rcu_reader_data *p_rcu_reader = get_ptr_rcu_reader(); 85 unsigned ctr; 86 87 if (p_rcu_reader->depth++ > 0) { 88 return; 89 } 90 91 ctr = qatomic_read(&rcu_gp_ctr); 92 qatomic_set(&p_rcu_reader->ctr, ctr); 93 94 /* Write p_rcu_reader->ctr before reading RCU-protected pointers. */ 95 smp_mb_placeholder(); 96 } 97 98 static inline void rcu_read_unlock(void) 99 { 100 struct rcu_reader_data *p_rcu_reader = get_ptr_rcu_reader(); 101 102 assert(p_rcu_reader->depth != 0); 103 if (--p_rcu_reader->depth > 0) { 104 return; 105 } 106 107 /* Ensure that the critical section is seen to precede the 108 * store to p_rcu_reader->ctr. Together with the following 109 * smp_mb_placeholder(), this ensures writes to p_rcu_reader->ctr 110 * are sequentially consistent. 111 */ 112 qatomic_store_release(&p_rcu_reader->ctr, 0); 113 114 /* Write p_rcu_reader->ctr before reading p_rcu_reader->waiting. */ 115 smp_mb_placeholder(); 116 if (unlikely(qatomic_read(&p_rcu_reader->waiting))) { 117 qatomic_set(&p_rcu_reader->waiting, false); 118 qemu_event_set(&rcu_gp_event); 119 } 120 } 121 122 extern void synchronize_rcu(void); 123 124 /* 125 * Reader thread registration. 126 */ 127 extern void rcu_register_thread(void); 128 extern void rcu_unregister_thread(void); 129 130 /* 131 * Support for fork(). fork() support is enabled at startup. 132 */ 133 extern void rcu_enable_atfork(void); 134 extern void rcu_disable_atfork(void); 135 136 struct rcu_head; 137 typedef void RCUCBFunc(struct rcu_head *head); 138 139 struct rcu_head { 140 struct rcu_head *next; 141 RCUCBFunc *func; 142 }; 143 144 extern void call_rcu1(struct rcu_head *head, RCUCBFunc *func); 145 extern void drain_call_rcu(void); 146 147 /* The operands of the minus operator must have the same type, 148 * which must be the one that we specify in the cast. 149 */ 150 #define call_rcu(head, func, field) \ 151 call_rcu1(({ \ 152 char __attribute__((unused)) \ 153 offset_must_be_zero[-offsetof(typeof(*(head)), field)], \ 154 func_type_invalid = (func) - (void (*)(typeof(head)))(func); \ 155 &(head)->field; \ 156 }), \ 157 (RCUCBFunc *)(func)) 158 159 #define g_free_rcu(obj, field) \ 160 call_rcu1(({ \ 161 char __attribute__((unused)) \ 162 offset_must_be_zero[-offsetof(typeof(*(obj)), field)]; \ 163 &(obj)->field; \ 164 }), \ 165 (RCUCBFunc *)g_free); 166 167 typedef void RCUReadAuto; 168 static inline RCUReadAuto *rcu_read_auto_lock(void) 169 { 170 rcu_read_lock(); 171 /* Anything non-NULL causes the cleanup function to be called */ 172 return (void *)(uintptr_t)0x1; 173 } 174 175 static inline void rcu_read_auto_unlock(RCUReadAuto *r) 176 { 177 rcu_read_unlock(); 178 } 179 180 G_DEFINE_AUTOPTR_CLEANUP_FUNC(RCUReadAuto, rcu_read_auto_unlock) 181 182 #define WITH_RCU_READ_LOCK_GUARD() \ 183 WITH_RCU_READ_LOCK_GUARD_(glue(_rcu_read_auto, __COUNTER__)) 184 185 #define WITH_RCU_READ_LOCK_GUARD_(var) \ 186 for (g_autoptr(RCUReadAuto) var = rcu_read_auto_lock(); \ 187 (var); rcu_read_auto_unlock(var), (var) = NULL) 188 189 #define RCU_READ_LOCK_GUARD() \ 190 g_autoptr(RCUReadAuto) _rcu_read_auto __attribute__((unused)) = rcu_read_auto_lock() 191 192 /* 193 * Force-RCU notifiers tell readers that they should exit their 194 * read-side critical section. 195 */ 196 void rcu_add_force_rcu_notifier(Notifier *n); 197 void rcu_remove_force_rcu_notifier(Notifier *n); 198 199 #ifdef __cplusplus 200 } 201 #endif 202 203 #endif /* QEMU_RCU_H */ 204