1 /* 2 * QEMU Thread Context 3 * 4 * Copyright Red Hat Inc., 2022 5 * 6 * Authors: 7 * David Hildenbrand <david@redhat.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2 or later. 10 * See the COPYING file in the top-level directory. 11 */ 12 13 #include "qemu/osdep.h" 14 #include "qemu/thread-context.h" 15 #include "qapi/error.h" 16 #include "qapi/qapi-builtin-visit.h" 17 #include "qapi/visitor.h" 18 #include "qemu/config-file.h" 19 #include "qapi/qapi-builtin-visit.h" 20 #include "qom/object_interfaces.h" 21 #include "qemu/module.h" 22 #include "qemu/bitmap.h" 23 24 #ifdef CONFIG_NUMA 25 #include <numa.h> 26 #endif 27 28 enum { 29 TC_CMD_NONE = 0, 30 TC_CMD_STOP, 31 TC_CMD_NEW, 32 }; 33 34 typedef struct ThreadContextCmdNew { 35 QemuThread *thread; 36 const char *name; 37 void *(*start_routine)(void *); 38 void *arg; 39 int mode; 40 } ThreadContextCmdNew; 41 42 static void *thread_context_run(void *opaque) 43 { 44 ThreadContext *tc = opaque; 45 46 tc->thread_id = qemu_get_thread_id(); 47 qemu_sem_post(&tc->sem); 48 49 while (true) { 50 /* 51 * Threads inherit the CPU affinity of the creating thread. For this 52 * reason, we create new (especially short-lived) threads from our 53 * persistent context thread. 54 * 55 * Especially when QEMU is not allowed to set the affinity itself, 56 * management tools can simply set the affinity of the context thread 57 * after creating the context, to have new threads created via 58 * the context inherit the CPU affinity automatically. 59 */ 60 switch (tc->thread_cmd) { 61 case TC_CMD_NONE: 62 break; 63 case TC_CMD_STOP: 64 tc->thread_cmd = TC_CMD_NONE; 65 qemu_sem_post(&tc->sem); 66 return NULL; 67 case TC_CMD_NEW: { 68 ThreadContextCmdNew *cmd_new = tc->thread_cmd_data; 69 70 qemu_thread_create(cmd_new->thread, cmd_new->name, 71 cmd_new->start_routine, cmd_new->arg, 72 cmd_new->mode); 73 tc->thread_cmd = TC_CMD_NONE; 74 tc->thread_cmd_data = NULL; 75 qemu_sem_post(&tc->sem); 76 break; 77 } 78 default: 79 g_assert_not_reached(); 80 } 81 qemu_sem_wait(&tc->sem_thread); 82 } 83 } 84 85 static void thread_context_set_cpu_affinity(Object *obj, Visitor *v, 86 const char *name, void *opaque, 87 Error **errp) 88 { 89 ThreadContext *tc = THREAD_CONTEXT(obj); 90 uint16List *l, *host_cpus = NULL; 91 unsigned long *bitmap = NULL; 92 int nbits = 0, ret; 93 Error *err = NULL; 94 95 if (tc->init_cpu_bitmap) { 96 error_setg(errp, "Mixing CPU and node affinity not supported"); 97 return; 98 } 99 100 visit_type_uint16List(v, name, &host_cpus, &err); 101 if (err) { 102 error_propagate(errp, err); 103 return; 104 } 105 106 if (!host_cpus) { 107 error_setg(errp, "CPU list is empty"); 108 goto out; 109 } 110 111 for (l = host_cpus; l; l = l->next) { 112 nbits = MAX(nbits, l->value + 1); 113 } 114 bitmap = bitmap_new(nbits); 115 for (l = host_cpus; l; l = l->next) { 116 set_bit(l->value, bitmap); 117 } 118 119 if (tc->thread_id != -1) { 120 /* 121 * Note: we won't be adjusting the affinity of any thread that is still 122 * around, but only the affinity of the context thread. 123 */ 124 ret = qemu_thread_set_affinity(&tc->thread, bitmap, nbits); 125 if (ret) { 126 error_setg(errp, "Setting CPU affinity failed: %s", strerror(ret)); 127 } 128 } else { 129 tc->init_cpu_bitmap = bitmap; 130 bitmap = NULL; 131 tc->init_cpu_nbits = nbits; 132 } 133 out: 134 g_free(bitmap); 135 qapi_free_uint16List(host_cpus); 136 } 137 138 static void thread_context_get_cpu_affinity(Object *obj, Visitor *v, 139 const char *name, void *opaque, 140 Error **errp) 141 { 142 unsigned long *bitmap, nbits, value; 143 ThreadContext *tc = THREAD_CONTEXT(obj); 144 uint16List *host_cpus = NULL; 145 uint16List **tail = &host_cpus; 146 int ret; 147 148 if (tc->thread_id == -1) { 149 error_setg(errp, "Object not initialized yet"); 150 return; 151 } 152 153 ret = qemu_thread_get_affinity(&tc->thread, &bitmap, &nbits); 154 if (ret) { 155 error_setg(errp, "Getting CPU affinity failed: %s", strerror(ret)); 156 return; 157 } 158 159 value = find_first_bit(bitmap, nbits); 160 while (value < nbits) { 161 QAPI_LIST_APPEND(tail, value); 162 163 value = find_next_bit(bitmap, nbits, value + 1); 164 } 165 g_free(bitmap); 166 167 visit_type_uint16List(v, name, &host_cpus, errp); 168 qapi_free_uint16List(host_cpus); 169 } 170 171 static void thread_context_set_node_affinity(Object *obj, Visitor *v, 172 const char *name, void *opaque, 173 Error **errp) 174 { 175 #ifdef CONFIG_NUMA 176 const int nbits = numa_num_possible_cpus(); 177 ThreadContext *tc = THREAD_CONTEXT(obj); 178 uint16List *l, *host_nodes = NULL; 179 unsigned long *bitmap = NULL; 180 struct bitmask *tmp_cpus; 181 Error *err = NULL; 182 int ret, i; 183 184 if (tc->init_cpu_bitmap) { 185 error_setg(errp, "Mixing CPU and node affinity not supported"); 186 return; 187 } 188 189 visit_type_uint16List(v, name, &host_nodes, &err); 190 if (err) { 191 error_propagate(errp, err); 192 return; 193 } 194 195 if (!host_nodes) { 196 error_setg(errp, "Node list is empty"); 197 goto out; 198 } 199 200 bitmap = bitmap_new(nbits); 201 tmp_cpus = numa_allocate_cpumask(); 202 for (l = host_nodes; l; l = l->next) { 203 numa_bitmask_clearall(tmp_cpus); 204 ret = numa_node_to_cpus(l->value, tmp_cpus); 205 if (ret) { 206 /* We ignore any errors, such as impossible nodes. */ 207 continue; 208 } 209 for (i = 0; i < nbits; i++) { 210 if (numa_bitmask_isbitset(tmp_cpus, i)) { 211 set_bit(i, bitmap); 212 } 213 } 214 } 215 numa_free_cpumask(tmp_cpus); 216 217 if (bitmap_empty(bitmap, nbits)) { 218 error_setg(errp, "The nodes select no CPUs"); 219 goto out; 220 } 221 222 if (tc->thread_id != -1) { 223 /* 224 * Note: we won't be adjusting the affinity of any thread that is still 225 * around for now, but only the affinity of the context thread. 226 */ 227 ret = qemu_thread_set_affinity(&tc->thread, bitmap, nbits); 228 if (ret) { 229 error_setg(errp, "Setting CPU affinity failed: %s", strerror(ret)); 230 } 231 } else { 232 tc->init_cpu_bitmap = bitmap; 233 bitmap = NULL; 234 tc->init_cpu_nbits = nbits; 235 } 236 out: 237 g_free(bitmap); 238 qapi_free_uint16List(host_nodes); 239 #else 240 error_setg(errp, "NUMA node affinity is not supported by this QEMU"); 241 #endif 242 } 243 244 static void thread_context_get_thread_id(Object *obj, Visitor *v, 245 const char *name, void *opaque, 246 Error **errp) 247 { 248 ThreadContext *tc = THREAD_CONTEXT(obj); 249 uint64_t value = tc->thread_id; 250 251 visit_type_uint64(v, name, &value, errp); 252 } 253 254 static void thread_context_instance_complete(UserCreatable *uc, Error **errp) 255 { 256 ThreadContext *tc = THREAD_CONTEXT(uc); 257 char *thread_name; 258 int ret; 259 260 thread_name = g_strdup_printf("TC %s", 261 object_get_canonical_path_component(OBJECT(uc))); 262 qemu_thread_create(&tc->thread, thread_name, thread_context_run, tc, 263 QEMU_THREAD_JOINABLE); 264 g_free(thread_name); 265 266 /* Wait until initialization of the thread is done. */ 267 while (tc->thread_id == -1) { 268 qemu_sem_wait(&tc->sem); 269 } 270 271 if (tc->init_cpu_bitmap) { 272 ret = qemu_thread_set_affinity(&tc->thread, tc->init_cpu_bitmap, 273 tc->init_cpu_nbits); 274 if (ret) { 275 error_setg(errp, "Setting CPU affinity failed: %s", strerror(ret)); 276 } 277 g_free(tc->init_cpu_bitmap); 278 tc->init_cpu_bitmap = NULL; 279 } 280 } 281 282 static void thread_context_class_init(ObjectClass *oc, void *data) 283 { 284 UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc); 285 286 ucc->complete = thread_context_instance_complete; 287 object_class_property_add(oc, "thread-id", "int", 288 thread_context_get_thread_id, NULL, NULL, 289 NULL); 290 object_class_property_add(oc, "cpu-affinity", "int", 291 thread_context_get_cpu_affinity, 292 thread_context_set_cpu_affinity, NULL, NULL); 293 object_class_property_add(oc, "node-affinity", "int", NULL, 294 thread_context_set_node_affinity, NULL, NULL); 295 } 296 297 static void thread_context_instance_init(Object *obj) 298 { 299 ThreadContext *tc = THREAD_CONTEXT(obj); 300 301 tc->thread_id = -1; 302 qemu_sem_init(&tc->sem, 0); 303 qemu_sem_init(&tc->sem_thread, 0); 304 qemu_mutex_init(&tc->mutex); 305 } 306 307 static void thread_context_instance_finalize(Object *obj) 308 { 309 ThreadContext *tc = THREAD_CONTEXT(obj); 310 311 if (tc->thread_id != -1) { 312 tc->thread_cmd = TC_CMD_STOP; 313 qemu_sem_post(&tc->sem_thread); 314 qemu_thread_join(&tc->thread); 315 } 316 qemu_sem_destroy(&tc->sem); 317 qemu_sem_destroy(&tc->sem_thread); 318 qemu_mutex_destroy(&tc->mutex); 319 } 320 321 static const TypeInfo thread_context_info = { 322 .name = TYPE_THREAD_CONTEXT, 323 .parent = TYPE_OBJECT, 324 .class_init = thread_context_class_init, 325 .instance_size = sizeof(ThreadContext), 326 .instance_init = thread_context_instance_init, 327 .instance_finalize = thread_context_instance_finalize, 328 .interfaces = (InterfaceInfo[]) { 329 { TYPE_USER_CREATABLE }, 330 { } 331 } 332 }; 333 334 static void thread_context_register_types(void) 335 { 336 type_register_static(&thread_context_info); 337 } 338 type_init(thread_context_register_types) 339 340 void thread_context_create_thread(ThreadContext *tc, QemuThread *thread, 341 const char *name, 342 void *(*start_routine)(void *), void *arg, 343 int mode) 344 { 345 ThreadContextCmdNew data = { 346 .thread = thread, 347 .name = name, 348 .start_routine = start_routine, 349 .arg = arg, 350 .mode = mode, 351 }; 352 353 qemu_mutex_lock(&tc->mutex); 354 tc->thread_cmd = TC_CMD_NEW; 355 tc->thread_cmd_data = &data; 356 qemu_sem_post(&tc->sem_thread); 357 358 while (tc->thread_cmd != TC_CMD_NONE) { 359 qemu_sem_wait(&tc->sem); 360 } 361 qemu_mutex_unlock(&tc->mutex); 362 } 363