1 #ifndef QEMU_RCU_H 2 #define QEMU_RCU_H 3 4 /* 5 * urcu-mb.h 6 * 7 * Userspace RCU header with explicit memory barrier. 8 * 9 * This library is free software; you can redistribute it and/or 10 * modify it under the terms of the GNU Lesser General Public 11 * License as published by the Free Software Foundation; either 12 * version 2.1 of the License, or (at your option) any later version. 13 * 14 * This library is distributed in the hope that it will be useful, 15 * but WITHOUT ANY WARRANTY; without even the implied warranty of 16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 17 * Lesser General Public License for more details. 18 * 19 * You should have received a copy of the GNU Lesser General Public 20 * License along with this library; if not, write to the Free Software 21 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 22 * 23 * IBM's contributions to this file may be relicensed under LGPLv2 or later. 24 */ 25 26 27 #include "qemu/thread.h" 28 #include "qemu/queue.h" 29 #include "qemu/atomic.h" 30 #include "qemu/notify.h" 31 #include "qemu/sys_membarrier.h" 32 #include "qemu/coroutine-tls.h" 33 34 /* 35 * Important ! 36 * 37 * Each thread containing read-side critical sections must be registered 38 * with rcu_register_thread() before calling rcu_read_lock(). 39 * rcu_unregister_thread() should be called before the thread exits. 40 */ 41 42 #ifdef DEBUG_RCU 43 #define rcu_assert(args...) assert(args) 44 #else 45 #define rcu_assert(args...) 46 #endif 47 48 /* 49 * Global quiescent period counter with low-order bits unused. 50 * Using a int rather than a char to eliminate false register dependencies 51 * causing stalls on some architectures. 52 */ 53 extern unsigned long rcu_gp_ctr; 54 55 extern QemuEvent rcu_gp_event; 56 57 struct rcu_reader_data { 58 /* Data used by both reader and synchronize_rcu() */ 59 unsigned long ctr; 60 bool waiting; 61 62 /* Data used by reader only */ 63 unsigned depth; 64 65 /* Data used for registry, protected by rcu_registry_lock */ 66 QLIST_ENTRY(rcu_reader_data) node; 67 68 /* 69 * NotifierList used to force an RCU grace period. Accessed under 70 * rcu_registry_lock. Note that the notifier is called _outside_ 71 * the thread! 72 */ 73 NotifierList force_rcu; 74 }; 75 76 QEMU_DECLARE_CO_TLS(struct rcu_reader_data, rcu_reader) 77 78 static inline void rcu_read_lock(void) 79 { 80 struct rcu_reader_data *p_rcu_reader = get_ptr_rcu_reader(); 81 unsigned ctr; 82 83 if (p_rcu_reader->depth++ > 0) { 84 return; 85 } 86 87 ctr = qatomic_read(&rcu_gp_ctr); 88 qatomic_set(&p_rcu_reader->ctr, ctr); 89 90 /* Write p_rcu_reader->ctr before reading RCU-protected pointers. */ 91 smp_mb_placeholder(); 92 } 93 94 static inline void rcu_read_unlock(void) 95 { 96 struct rcu_reader_data *p_rcu_reader = get_ptr_rcu_reader(); 97 98 assert(p_rcu_reader->depth != 0); 99 if (--p_rcu_reader->depth > 0) { 100 return; 101 } 102 103 /* Ensure that the critical section is seen to precede the 104 * store to p_rcu_reader->ctr. Together with the following 105 * smp_mb_placeholder(), this ensures writes to p_rcu_reader->ctr 106 * are sequentially consistent. 107 */ 108 qatomic_store_release(&p_rcu_reader->ctr, 0); 109 110 /* Write p_rcu_reader->ctr before reading p_rcu_reader->waiting. */ 111 smp_mb_placeholder(); 112 if (unlikely(qatomic_read(&p_rcu_reader->waiting))) { 113 qatomic_set(&p_rcu_reader->waiting, false); 114 qemu_event_set(&rcu_gp_event); 115 } 116 } 117 118 extern void synchronize_rcu(void); 119 120 /* 121 * Reader thread registration. 122 */ 123 extern void rcu_register_thread(void); 124 extern void rcu_unregister_thread(void); 125 126 /* 127 * Support for fork(). fork() support is enabled at startup. 128 */ 129 extern void rcu_enable_atfork(void); 130 extern void rcu_disable_atfork(void); 131 132 struct rcu_head; 133 typedef void RCUCBFunc(struct rcu_head *head); 134 135 struct rcu_head { 136 struct rcu_head *next; 137 RCUCBFunc *func; 138 }; 139 140 extern void call_rcu1(struct rcu_head *head, RCUCBFunc *func); 141 extern void drain_call_rcu(void); 142 143 /* The operands of the minus operator must have the same type, 144 * which must be the one that we specify in the cast. 145 */ 146 #define call_rcu(head, func, field) \ 147 call_rcu1(({ \ 148 char __attribute__((unused)) \ 149 offset_must_be_zero[-offsetof(typeof(*(head)), field)], \ 150 func_type_invalid = (func) - (void (*)(typeof(head)))(func); \ 151 &(head)->field; \ 152 }), \ 153 (RCUCBFunc *)(func)) 154 155 #define g_free_rcu(obj, field) \ 156 call_rcu1(({ \ 157 char __attribute__((unused)) \ 158 offset_must_be_zero[-offsetof(typeof(*(obj)), field)]; \ 159 &(obj)->field; \ 160 }), \ 161 (RCUCBFunc *)g_free); 162 163 typedef void RCUReadAuto; 164 static inline RCUReadAuto *rcu_read_auto_lock(void) 165 { 166 rcu_read_lock(); 167 /* Anything non-NULL causes the cleanup function to be called */ 168 return (void *)(uintptr_t)0x1; 169 } 170 171 static inline void rcu_read_auto_unlock(RCUReadAuto *r) 172 { 173 rcu_read_unlock(); 174 } 175 176 G_DEFINE_AUTOPTR_CLEANUP_FUNC(RCUReadAuto, rcu_read_auto_unlock) 177 178 #define WITH_RCU_READ_LOCK_GUARD() \ 179 WITH_RCU_READ_LOCK_GUARD_(glue(_rcu_read_auto, __COUNTER__)) 180 181 #define WITH_RCU_READ_LOCK_GUARD_(var) \ 182 for (g_autoptr(RCUReadAuto) var = rcu_read_auto_lock(); \ 183 (var); rcu_read_auto_unlock(var), (var) = NULL) 184 185 #define RCU_READ_LOCK_GUARD() \ 186 g_autoptr(RCUReadAuto) _rcu_read_auto __attribute__((unused)) = rcu_read_auto_lock() 187 188 /* 189 * Force-RCU notifiers tell readers that they should exit their 190 * read-side critical section. 191 */ 192 void rcu_add_force_rcu_notifier(Notifier *n); 193 void rcu_remove_force_rcu_notifier(Notifier *n); 194 195 #endif /* QEMU_RCU_H */ 196