1 /* 2 * plugin-gen.c - TCG-related bits of plugin infrastructure 3 * 4 * Copyright (C) 2018, Emilio G. Cota <cota@braap.org> 5 * License: GNU GPL, version 2 or later. 6 * See the COPYING file in the top-level directory. 7 * 8 * We support instrumentation at an instruction granularity. That is, 9 * if a plugin wants to instrument the memory accesses performed by a 10 * particular instruction, it can just do that instead of instrumenting 11 * all memory accesses. Thus, in order to do this we first have to 12 * translate a TB, so that plugins can decide what/where to instrument. 13 * 14 * Injecting the desired instrumentation could be done with a second 15 * translation pass that combined the instrumentation requests, but that 16 * would be ugly and inefficient since we would decode the guest code twice. 17 * Instead, during TB translation we add "empty" instrumentation calls for all 18 * possible instrumentation events, and then once we collect the instrumentation 19 * requests from plugins, we either "fill in" those empty events or remove them 20 * if they have no requests. 21 * 22 * When "filling in" an event we first copy the empty callback's TCG ops. This 23 * might seem unnecessary, but it is done to support an arbitrary number 24 * of callbacks per event. Take for example a regular instruction callback. 25 * We first generate a callback to an empty helper function. Then, if two 26 * plugins register one callback each for this instruction, we make two copies 27 * of the TCG ops generated for the empty callback, substituting the function 28 * pointer that points to the empty helper function with the plugins' desired 29 * callback functions. After that we remove the empty callback's ops. 30 * 31 * Note that the location in TCGOp.args[] of the pointer to a helper function 32 * varies across different guest and host architectures. Instead of duplicating 33 * the logic that figures this out, we rely on the fact that the empty 34 * callbacks point to empty functions that are unique pointers in the program. 35 * Thus, to find the right location we just have to look for a match in 36 * TCGOp.args[]. This is the main reason why we first copy an empty callback's 37 * TCG ops and then fill them in; regardless of whether we have one or many 38 * callbacks for that event, the logic to add all of them is the same. 39 * 40 * When generating more than one callback per event, we make a small 41 * optimization to avoid generating redundant operations. For instance, for the 42 * second and all subsequent callbacks of an event, we do not need to reload the 43 * CPU's index into a TCG temp, since the first callback did it already. 44 */ 45 #include "qemu/osdep.h" 46 #include "qemu/plugin.h" 47 #include "cpu.h" 48 #include "tcg/tcg.h" 49 #include "tcg/tcg-temp-internal.h" 50 #include "tcg/tcg-op.h" 51 #include "exec/exec-all.h" 52 #include "exec/plugin-gen.h" 53 #include "exec/translator.h" 54 #include "exec/helper-proto-common.h" 55 56 #define HELPER_H "accel/tcg/plugin-helpers.h" 57 #include "exec/helper-info.c.inc" 58 #undef HELPER_H 59 60 /* 61 * plugin_cb_start TCG op args[]: 62 * 0: enum plugin_gen_from 63 * 1: enum plugin_gen_cb 64 * 2: set to 1 for mem callback that is a write, 0 otherwise. 65 */ 66 67 enum plugin_gen_from { 68 PLUGIN_GEN_FROM_TB, 69 PLUGIN_GEN_FROM_INSN, 70 PLUGIN_GEN_FROM_MEM, 71 PLUGIN_GEN_AFTER_INSN, 72 PLUGIN_GEN_N_FROMS, 73 }; 74 75 enum plugin_gen_cb { 76 PLUGIN_GEN_CB_UDATA, 77 PLUGIN_GEN_CB_UDATA_R, 78 PLUGIN_GEN_CB_INLINE, 79 PLUGIN_GEN_CB_MEM, 80 PLUGIN_GEN_ENABLE_MEM_HELPER, 81 PLUGIN_GEN_DISABLE_MEM_HELPER, 82 PLUGIN_GEN_N_CBS, 83 }; 84 85 /* 86 * These helpers are stubs that get dynamically switched out for calls 87 * direct to the plugin if they are subscribed to. 88 */ 89 void HELPER(plugin_vcpu_udata_cb_no_wg)(uint32_t cpu_index, void *udata) 90 { } 91 92 void HELPER(plugin_vcpu_udata_cb_no_rwg)(uint32_t cpu_index, void *udata) 93 { } 94 95 void HELPER(plugin_vcpu_mem_cb)(unsigned int vcpu_index, 96 qemu_plugin_meminfo_t info, uint64_t vaddr, 97 void *userdata) 98 { } 99 100 static void gen_empty_udata_cb(void (*gen_helper)(TCGv_i32, TCGv_ptr)) 101 { 102 TCGv_i32 cpu_index = tcg_temp_ebb_new_i32(); 103 TCGv_ptr udata = tcg_temp_ebb_new_ptr(); 104 105 tcg_gen_movi_ptr(udata, 0); 106 tcg_gen_ld_i32(cpu_index, tcg_env, 107 -offsetof(ArchCPU, env) + offsetof(CPUState, cpu_index)); 108 gen_helper(cpu_index, udata); 109 110 tcg_temp_free_ptr(udata); 111 tcg_temp_free_i32(cpu_index); 112 } 113 114 static void gen_empty_udata_cb_no_wg(void) 115 { 116 gen_empty_udata_cb(gen_helper_plugin_vcpu_udata_cb_no_wg); 117 } 118 119 static void gen_empty_udata_cb_no_rwg(void) 120 { 121 gen_empty_udata_cb(gen_helper_plugin_vcpu_udata_cb_no_rwg); 122 } 123 124 /* 125 * For now we only support addi_i64. 126 * When we support more ops, we can generate one empty inline cb for each. 127 */ 128 static void gen_empty_inline_cb(void) 129 { 130 TCGv_i32 cpu_index = tcg_temp_ebb_new_i32(); 131 TCGv_ptr cpu_index_as_ptr = tcg_temp_ebb_new_ptr(); 132 TCGv_i64 val = tcg_temp_ebb_new_i64(); 133 TCGv_ptr ptr = tcg_temp_ebb_new_ptr(); 134 135 tcg_gen_ld_i32(cpu_index, tcg_env, 136 -offsetof(ArchCPU, env) + offsetof(CPUState, cpu_index)); 137 /* second operand will be replaced by immediate value */ 138 tcg_gen_mul_i32(cpu_index, cpu_index, cpu_index); 139 tcg_gen_ext_i32_ptr(cpu_index_as_ptr, cpu_index); 140 141 tcg_gen_movi_ptr(ptr, 0); 142 tcg_gen_add_ptr(ptr, ptr, cpu_index_as_ptr); 143 tcg_gen_ld_i64(val, ptr, 0); 144 /* second operand will be replaced by immediate value */ 145 tcg_gen_add_i64(val, val, val); 146 147 tcg_gen_st_i64(val, ptr, 0); 148 tcg_temp_free_ptr(ptr); 149 tcg_temp_free_i64(val); 150 tcg_temp_free_ptr(cpu_index_as_ptr); 151 tcg_temp_free_i32(cpu_index); 152 } 153 154 static void gen_empty_mem_cb(TCGv_i64 addr, uint32_t info) 155 { 156 TCGv_i32 cpu_index = tcg_temp_ebb_new_i32(); 157 TCGv_i32 meminfo = tcg_temp_ebb_new_i32(); 158 TCGv_ptr udata = tcg_temp_ebb_new_ptr(); 159 160 tcg_gen_movi_i32(meminfo, info); 161 tcg_gen_movi_ptr(udata, 0); 162 tcg_gen_ld_i32(cpu_index, tcg_env, 163 -offsetof(ArchCPU, env) + offsetof(CPUState, cpu_index)); 164 165 gen_helper_plugin_vcpu_mem_cb(cpu_index, meminfo, addr, udata); 166 167 tcg_temp_free_ptr(udata); 168 tcg_temp_free_i32(meminfo); 169 tcg_temp_free_i32(cpu_index); 170 } 171 172 /* 173 * Share the same function for enable/disable. When enabling, the NULL 174 * pointer will be overwritten later. 175 */ 176 static void gen_empty_mem_helper(void) 177 { 178 TCGv_ptr ptr = tcg_temp_ebb_new_ptr(); 179 180 tcg_gen_movi_ptr(ptr, 0); 181 tcg_gen_st_ptr(ptr, tcg_env, offsetof(CPUState, plugin_mem_cbs) - 182 offsetof(ArchCPU, env)); 183 tcg_temp_free_ptr(ptr); 184 } 185 186 static void gen_plugin_cb_start(enum plugin_gen_from from, 187 enum plugin_gen_cb type, unsigned wr) 188 { 189 tcg_gen_plugin_cb_start(from, type, wr); 190 } 191 192 static void gen_wrapped(enum plugin_gen_from from, 193 enum plugin_gen_cb type, void (*func)(void)) 194 { 195 gen_plugin_cb_start(from, type, 0); 196 func(); 197 tcg_gen_plugin_cb_end(); 198 } 199 200 static void plugin_gen_empty_callback(enum plugin_gen_from from) 201 { 202 switch (from) { 203 case PLUGIN_GEN_AFTER_INSN: 204 case PLUGIN_GEN_FROM_TB: 205 tcg_gen_plugin_cb(from); 206 break; 207 case PLUGIN_GEN_FROM_INSN: 208 /* 209 * Note: plugin_gen_inject() relies on ENABLE_MEM_HELPER being 210 * the first callback of an instruction 211 */ 212 gen_wrapped(from, PLUGIN_GEN_ENABLE_MEM_HELPER, 213 gen_empty_mem_helper); 214 gen_wrapped(from, PLUGIN_GEN_CB_UDATA, gen_empty_udata_cb_no_rwg); 215 gen_wrapped(from, PLUGIN_GEN_CB_UDATA_R, gen_empty_udata_cb_no_wg); 216 gen_wrapped(from, PLUGIN_GEN_CB_INLINE, gen_empty_inline_cb); 217 break; 218 default: 219 g_assert_not_reached(); 220 } 221 } 222 223 void plugin_gen_empty_mem_callback(TCGv_i64 addr, uint32_t info) 224 { 225 enum qemu_plugin_mem_rw rw = get_plugin_meminfo_rw(info); 226 227 gen_plugin_cb_start(PLUGIN_GEN_FROM_MEM, PLUGIN_GEN_CB_MEM, rw); 228 gen_empty_mem_cb(addr, info); 229 tcg_gen_plugin_cb_end(); 230 231 gen_plugin_cb_start(PLUGIN_GEN_FROM_MEM, PLUGIN_GEN_CB_INLINE, rw); 232 gen_empty_inline_cb(); 233 tcg_gen_plugin_cb_end(); 234 } 235 236 static TCGOp *find_op(TCGOp *op, TCGOpcode opc) 237 { 238 while (op) { 239 if (op->opc == opc) { 240 return op; 241 } 242 op = QTAILQ_NEXT(op, link); 243 } 244 return NULL; 245 } 246 247 static TCGOp *rm_ops_range(TCGOp *begin, TCGOp *end) 248 { 249 TCGOp *ret = QTAILQ_NEXT(end, link); 250 251 QTAILQ_REMOVE_SEVERAL(&tcg_ctx->ops, begin, end, link); 252 return ret; 253 } 254 255 /* remove all ops until (and including) plugin_cb_end */ 256 static TCGOp *rm_ops(TCGOp *op) 257 { 258 TCGOp *end_op = find_op(op, INDEX_op_plugin_cb_end); 259 260 tcg_debug_assert(end_op); 261 return rm_ops_range(op, end_op); 262 } 263 264 static TCGOp *copy_op_nocheck(TCGOp **begin_op, TCGOp *op) 265 { 266 TCGOp *old_op = QTAILQ_NEXT(*begin_op, link); 267 unsigned nargs = old_op->nargs; 268 269 *begin_op = old_op; 270 op = tcg_op_insert_after(tcg_ctx, op, old_op->opc, nargs); 271 memcpy(op->args, old_op->args, sizeof(op->args[0]) * nargs); 272 273 return op; 274 } 275 276 static TCGOp *copy_op(TCGOp **begin_op, TCGOp *op, TCGOpcode opc) 277 { 278 op = copy_op_nocheck(begin_op, op); 279 tcg_debug_assert((*begin_op)->opc == opc); 280 return op; 281 } 282 283 static TCGOp *copy_const_ptr(TCGOp **begin_op, TCGOp *op, void *ptr) 284 { 285 if (UINTPTR_MAX == UINT32_MAX) { 286 /* mov_i32 */ 287 op = copy_op(begin_op, op, INDEX_op_mov_i32); 288 op->args[1] = tcgv_i32_arg(tcg_constant_i32((uintptr_t)ptr)); 289 } else { 290 /* mov_i64 */ 291 op = copy_op(begin_op, op, INDEX_op_mov_i64); 292 op->args[1] = tcgv_i64_arg(tcg_constant_i64((uintptr_t)ptr)); 293 } 294 return op; 295 } 296 297 static TCGOp *copy_ld_i32(TCGOp **begin_op, TCGOp *op) 298 { 299 return copy_op(begin_op, op, INDEX_op_ld_i32); 300 } 301 302 static TCGOp *copy_ext_i32_ptr(TCGOp **begin_op, TCGOp *op) 303 { 304 if (UINTPTR_MAX == UINT32_MAX) { 305 op = copy_op(begin_op, op, INDEX_op_mov_i32); 306 } else { 307 op = copy_op(begin_op, op, INDEX_op_ext_i32_i64); 308 } 309 return op; 310 } 311 312 static TCGOp *copy_add_ptr(TCGOp **begin_op, TCGOp *op) 313 { 314 if (UINTPTR_MAX == UINT32_MAX) { 315 op = copy_op(begin_op, op, INDEX_op_add_i32); 316 } else { 317 op = copy_op(begin_op, op, INDEX_op_add_i64); 318 } 319 return op; 320 } 321 322 static TCGOp *copy_ld_i64(TCGOp **begin_op, TCGOp *op) 323 { 324 if (TCG_TARGET_REG_BITS == 32) { 325 /* 2x ld_i32 */ 326 op = copy_ld_i32(begin_op, op); 327 op = copy_ld_i32(begin_op, op); 328 } else { 329 /* ld_i64 */ 330 op = copy_op(begin_op, op, INDEX_op_ld_i64); 331 } 332 return op; 333 } 334 335 static TCGOp *copy_st_i64(TCGOp **begin_op, TCGOp *op) 336 { 337 if (TCG_TARGET_REG_BITS == 32) { 338 /* 2x st_i32 */ 339 op = copy_op(begin_op, op, INDEX_op_st_i32); 340 op = copy_op(begin_op, op, INDEX_op_st_i32); 341 } else { 342 /* st_i64 */ 343 op = copy_op(begin_op, op, INDEX_op_st_i64); 344 } 345 return op; 346 } 347 348 static TCGOp *copy_add_i64(TCGOp **begin_op, TCGOp *op, uint64_t v) 349 { 350 if (TCG_TARGET_REG_BITS == 32) { 351 /* all 32-bit backends must implement add2_i32 */ 352 g_assert(TCG_TARGET_HAS_add2_i32); 353 op = copy_op(begin_op, op, INDEX_op_add2_i32); 354 op->args[4] = tcgv_i32_arg(tcg_constant_i32(v)); 355 op->args[5] = tcgv_i32_arg(tcg_constant_i32(v >> 32)); 356 } else { 357 op = copy_op(begin_op, op, INDEX_op_add_i64); 358 op->args[2] = tcgv_i64_arg(tcg_constant_i64(v)); 359 } 360 return op; 361 } 362 363 static TCGOp *copy_mul_i32(TCGOp **begin_op, TCGOp *op, uint32_t v) 364 { 365 op = copy_op(begin_op, op, INDEX_op_mul_i32); 366 op->args[2] = tcgv_i32_arg(tcg_constant_i32(v)); 367 return op; 368 } 369 370 static TCGOp *copy_st_ptr(TCGOp **begin_op, TCGOp *op) 371 { 372 if (UINTPTR_MAX == UINT32_MAX) { 373 /* st_i32 */ 374 op = copy_op(begin_op, op, INDEX_op_st_i32); 375 } else { 376 /* st_i64 */ 377 op = copy_st_i64(begin_op, op); 378 } 379 return op; 380 } 381 382 static TCGOp *copy_call(TCGOp **begin_op, TCGOp *op, void *func, int *cb_idx) 383 { 384 TCGOp *old_op; 385 int func_idx; 386 387 /* copy all ops until the call */ 388 do { 389 op = copy_op_nocheck(begin_op, op); 390 } while (op->opc != INDEX_op_call); 391 392 /* fill in the op call */ 393 old_op = *begin_op; 394 TCGOP_CALLI(op) = TCGOP_CALLI(old_op); 395 TCGOP_CALLO(op) = TCGOP_CALLO(old_op); 396 tcg_debug_assert(op->life == 0); 397 398 func_idx = TCGOP_CALLO(op) + TCGOP_CALLI(op); 399 *cb_idx = func_idx; 400 op->args[func_idx] = (uintptr_t)func; 401 402 return op; 403 } 404 405 /* 406 * When we append/replace ops here we are sensitive to changing patterns of 407 * TCGOps generated by the tcg_gen_FOO calls when we generated the 408 * empty callbacks. This will assert very quickly in a debug build as 409 * we assert the ops we are replacing are the correct ones. 410 */ 411 static TCGOp *append_udata_cb(const struct qemu_plugin_dyn_cb *cb, 412 TCGOp *begin_op, TCGOp *op, int *cb_idx) 413 { 414 /* const_ptr */ 415 op = copy_const_ptr(&begin_op, op, cb->userp); 416 417 /* copy the ld_i32, but note that we only have to copy it once */ 418 if (*cb_idx == -1) { 419 op = copy_op(&begin_op, op, INDEX_op_ld_i32); 420 } else { 421 begin_op = QTAILQ_NEXT(begin_op, link); 422 tcg_debug_assert(begin_op && begin_op->opc == INDEX_op_ld_i32); 423 } 424 425 /* call */ 426 op = copy_call(&begin_op, op, cb->regular.f.vcpu_udata, cb_idx); 427 428 return op; 429 } 430 431 static TCGOp *append_inline_cb(const struct qemu_plugin_dyn_cb *cb, 432 TCGOp *begin_op, TCGOp *op, 433 int *unused) 434 { 435 char *ptr = cb->inline_insn.entry.score->data->data; 436 size_t elem_size = g_array_get_element_size( 437 cb->inline_insn.entry.score->data); 438 size_t offset = cb->inline_insn.entry.offset; 439 440 op = copy_ld_i32(&begin_op, op); 441 op = copy_mul_i32(&begin_op, op, elem_size); 442 op = copy_ext_i32_ptr(&begin_op, op); 443 op = copy_const_ptr(&begin_op, op, ptr + offset); 444 op = copy_add_ptr(&begin_op, op); 445 op = copy_ld_i64(&begin_op, op); 446 op = copy_add_i64(&begin_op, op, cb->inline_insn.imm); 447 op = copy_st_i64(&begin_op, op); 448 return op; 449 } 450 451 static TCGOp *append_mem_cb(const struct qemu_plugin_dyn_cb *cb, 452 TCGOp *begin_op, TCGOp *op, int *cb_idx) 453 { 454 enum plugin_gen_cb type = begin_op->args[1]; 455 456 tcg_debug_assert(type == PLUGIN_GEN_CB_MEM); 457 458 /* const_i32 == mov_i32 ("info", so it remains as is) */ 459 op = copy_op(&begin_op, op, INDEX_op_mov_i32); 460 461 /* const_ptr */ 462 op = copy_const_ptr(&begin_op, op, cb->userp); 463 464 /* copy the ld_i32, but note that we only have to copy it once */ 465 if (*cb_idx == -1) { 466 op = copy_op(&begin_op, op, INDEX_op_ld_i32); 467 } else { 468 begin_op = QTAILQ_NEXT(begin_op, link); 469 tcg_debug_assert(begin_op && begin_op->opc == INDEX_op_ld_i32); 470 } 471 472 if (type == PLUGIN_GEN_CB_MEM) { 473 /* call */ 474 op = copy_call(&begin_op, op, cb->regular.f.vcpu_udata, cb_idx); 475 } 476 477 return op; 478 } 479 480 typedef TCGOp *(*inject_fn)(const struct qemu_plugin_dyn_cb *cb, 481 TCGOp *begin_op, TCGOp *op, int *intp); 482 typedef bool (*op_ok_fn)(const TCGOp *op, const struct qemu_plugin_dyn_cb *cb); 483 484 static bool op_ok(const TCGOp *op, const struct qemu_plugin_dyn_cb *cb) 485 { 486 return true; 487 } 488 489 static bool op_rw(const TCGOp *op, const struct qemu_plugin_dyn_cb *cb) 490 { 491 int w; 492 493 w = op->args[2]; 494 return !!(cb->rw & (w + 1)); 495 } 496 497 static void inject_cb_type(const GArray *cbs, TCGOp *begin_op, 498 inject_fn inject, op_ok_fn ok) 499 { 500 TCGOp *end_op; 501 TCGOp *op; 502 int cb_idx = -1; 503 int i; 504 505 if (!cbs || cbs->len == 0) { 506 rm_ops(begin_op); 507 return; 508 } 509 510 end_op = find_op(begin_op, INDEX_op_plugin_cb_end); 511 tcg_debug_assert(end_op); 512 513 op = end_op; 514 for (i = 0; i < cbs->len; i++) { 515 struct qemu_plugin_dyn_cb *cb = 516 &g_array_index(cbs, struct qemu_plugin_dyn_cb, i); 517 518 if (!ok(begin_op, cb)) { 519 continue; 520 } 521 op = inject(cb, begin_op, op, &cb_idx); 522 } 523 rm_ops_range(begin_op, end_op); 524 } 525 526 static void 527 inject_udata_cb(const GArray *cbs, TCGOp *begin_op) 528 { 529 inject_cb_type(cbs, begin_op, append_udata_cb, op_ok); 530 } 531 532 static void 533 inject_inline_cb(const GArray *cbs, TCGOp *begin_op, op_ok_fn ok) 534 { 535 inject_cb_type(cbs, begin_op, append_inline_cb, ok); 536 } 537 538 static void 539 inject_mem_cb(const GArray *cbs, TCGOp *begin_op) 540 { 541 inject_cb_type(cbs, begin_op, append_mem_cb, op_rw); 542 } 543 544 /* we could change the ops in place, but we can reuse more code by copying */ 545 static void inject_mem_helper(TCGOp *begin_op, GArray *arr) 546 { 547 TCGOp *orig_op = begin_op; 548 TCGOp *end_op; 549 TCGOp *op; 550 551 end_op = find_op(begin_op, INDEX_op_plugin_cb_end); 552 tcg_debug_assert(end_op); 553 554 /* const ptr */ 555 op = copy_const_ptr(&begin_op, end_op, arr); 556 557 /* st_ptr */ 558 op = copy_st_ptr(&begin_op, op); 559 560 rm_ops_range(orig_op, end_op); 561 } 562 563 /* 564 * Tracking memory accesses performed from helpers requires extra work. 565 * If an instruction is emulated with helpers, we do two things: 566 * (1) copy the CB descriptors, and keep track of it so that they can be 567 * freed later on, and (2) point CPUState.plugin_mem_cbs to the descriptors, so 568 * that we can read them at run-time (i.e. when the helper executes). 569 * This run-time access is performed from qemu_plugin_vcpu_mem_cb. 570 * 571 * Note that plugin_gen_disable_mem_helpers undoes (2). Since it 572 * is possible that the code we generate after the instruction is 573 * dead, we also add checks before generating tb_exit etc. 574 */ 575 static void inject_mem_enable_helper(struct qemu_plugin_tb *ptb, 576 struct qemu_plugin_insn *plugin_insn, 577 TCGOp *begin_op) 578 { 579 GArray *cbs[2]; 580 GArray *arr; 581 size_t n_cbs, i; 582 583 cbs[0] = plugin_insn->cbs[PLUGIN_CB_MEM][PLUGIN_CB_REGULAR]; 584 cbs[1] = plugin_insn->cbs[PLUGIN_CB_MEM][PLUGIN_CB_INLINE]; 585 586 n_cbs = 0; 587 for (i = 0; i < ARRAY_SIZE(cbs); i++) { 588 n_cbs += cbs[i]->len; 589 } 590 591 plugin_insn->mem_helper = plugin_insn->calls_helpers && n_cbs; 592 if (likely(!plugin_insn->mem_helper)) { 593 rm_ops(begin_op); 594 return; 595 } 596 ptb->mem_helper = true; 597 598 arr = g_array_sized_new(false, false, 599 sizeof(struct qemu_plugin_dyn_cb), n_cbs); 600 601 for (i = 0; i < ARRAY_SIZE(cbs); i++) { 602 g_array_append_vals(arr, cbs[i]->data, cbs[i]->len); 603 } 604 605 qemu_plugin_add_dyn_cb_arr(arr); 606 inject_mem_helper(begin_op, arr); 607 } 608 609 /* called before finishing a TB with exit_tb, goto_tb or goto_ptr */ 610 void plugin_gen_disable_mem_helpers(void) 611 { 612 /* 613 * We could emit the clearing unconditionally and be done. However, this can 614 * be wasteful if for instance plugins don't track memory accesses, or if 615 * most TBs don't use helpers. Instead, emit the clearing iff the TB calls 616 * helpers that might access guest memory. 617 * 618 * Note: we do not reset plugin_tb->mem_helper here; a TB might have several 619 * exit points, and we want to emit the clearing from all of them. 620 */ 621 if (!tcg_ctx->plugin_tb->mem_helper) { 622 return; 623 } 624 tcg_gen_st_ptr(tcg_constant_ptr(NULL), tcg_env, 625 offsetof(CPUState, plugin_mem_cbs) - offsetof(ArchCPU, env)); 626 } 627 628 static void plugin_gen_insn_udata(const struct qemu_plugin_tb *ptb, 629 TCGOp *begin_op, int insn_idx) 630 { 631 struct qemu_plugin_insn *insn = g_ptr_array_index(ptb->insns, insn_idx); 632 633 inject_udata_cb(insn->cbs[PLUGIN_CB_INSN][PLUGIN_CB_REGULAR], begin_op); 634 } 635 636 static void plugin_gen_insn_udata_r(const struct qemu_plugin_tb *ptb, 637 TCGOp *begin_op, int insn_idx) 638 { 639 struct qemu_plugin_insn *insn = g_ptr_array_index(ptb->insns, insn_idx); 640 641 inject_udata_cb(insn->cbs[PLUGIN_CB_INSN][PLUGIN_CB_REGULAR_R], begin_op); 642 } 643 644 static void plugin_gen_insn_inline(const struct qemu_plugin_tb *ptb, 645 TCGOp *begin_op, int insn_idx) 646 { 647 struct qemu_plugin_insn *insn = g_ptr_array_index(ptb->insns, insn_idx); 648 inject_inline_cb(insn->cbs[PLUGIN_CB_INSN][PLUGIN_CB_INLINE], 649 begin_op, op_ok); 650 } 651 652 static void plugin_gen_mem_regular(const struct qemu_plugin_tb *ptb, 653 TCGOp *begin_op, int insn_idx) 654 { 655 struct qemu_plugin_insn *insn = g_ptr_array_index(ptb->insns, insn_idx); 656 inject_mem_cb(insn->cbs[PLUGIN_CB_MEM][PLUGIN_CB_REGULAR], begin_op); 657 } 658 659 static void plugin_gen_mem_inline(const struct qemu_plugin_tb *ptb, 660 TCGOp *begin_op, int insn_idx) 661 { 662 const GArray *cbs; 663 struct qemu_plugin_insn *insn = g_ptr_array_index(ptb->insns, insn_idx); 664 665 cbs = insn->cbs[PLUGIN_CB_MEM][PLUGIN_CB_INLINE]; 666 inject_inline_cb(cbs, begin_op, op_rw); 667 } 668 669 static void plugin_gen_enable_mem_helper(struct qemu_plugin_tb *ptb, 670 TCGOp *begin_op, int insn_idx) 671 { 672 struct qemu_plugin_insn *insn = g_ptr_array_index(ptb->insns, insn_idx); 673 inject_mem_enable_helper(ptb, insn, begin_op); 674 } 675 676 static void gen_disable_mem_helper(struct qemu_plugin_tb *ptb, 677 struct qemu_plugin_insn *insn) 678 { 679 if (insn->mem_helper) { 680 tcg_gen_st_ptr(tcg_constant_ptr(0), tcg_env, 681 offsetof(CPUState, plugin_mem_cbs) - 682 offsetof(ArchCPU, env)); 683 } 684 } 685 686 static void gen_udata_cb(struct qemu_plugin_dyn_cb *cb) 687 { 688 TCGv_i32 cpu_index = tcg_temp_ebb_new_i32(); 689 690 tcg_gen_ld_i32(cpu_index, tcg_env, 691 -offsetof(ArchCPU, env) + offsetof(CPUState, cpu_index)); 692 tcg_gen_call2(cb->regular.f.vcpu_udata, cb->regular.info, NULL, 693 tcgv_i32_temp(cpu_index), 694 tcgv_ptr_temp(tcg_constant_ptr(cb->userp))); 695 tcg_temp_free_i32(cpu_index); 696 } 697 698 static void gen_inline_cb(struct qemu_plugin_dyn_cb *cb) 699 { 700 GArray *arr = cb->inline_insn.entry.score->data; 701 size_t offset = cb->inline_insn.entry.offset; 702 TCGv_i32 cpu_index = tcg_temp_ebb_new_i32(); 703 TCGv_i64 val = tcg_temp_ebb_new_i64(); 704 TCGv_ptr ptr = tcg_temp_ebb_new_ptr(); 705 706 tcg_gen_ld_i32(cpu_index, tcg_env, 707 -offsetof(ArchCPU, env) + offsetof(CPUState, cpu_index)); 708 tcg_gen_muli_i32(cpu_index, cpu_index, g_array_get_element_size(arr)); 709 tcg_gen_ext_i32_ptr(ptr, cpu_index); 710 tcg_temp_free_i32(cpu_index); 711 712 tcg_gen_addi_ptr(ptr, ptr, (intptr_t)arr->data); 713 tcg_gen_ld_i64(val, ptr, offset); 714 tcg_gen_addi_i64(val, val, cb->inline_insn.imm); 715 tcg_gen_st_i64(val, ptr, offset); 716 717 tcg_temp_free_i64(val); 718 tcg_temp_free_ptr(ptr); 719 } 720 721 /* #define DEBUG_PLUGIN_GEN_OPS */ 722 static void pr_ops(void) 723 { 724 #ifdef DEBUG_PLUGIN_GEN_OPS 725 TCGOp *op; 726 int i = 0; 727 728 QTAILQ_FOREACH(op, &tcg_ctx->ops, link) { 729 const char *name = ""; 730 const char *type = ""; 731 732 if (op->opc == INDEX_op_plugin_cb_start) { 733 switch (op->args[0]) { 734 case PLUGIN_GEN_FROM_TB: 735 name = "tb"; 736 break; 737 case PLUGIN_GEN_FROM_INSN: 738 name = "insn"; 739 break; 740 case PLUGIN_GEN_FROM_MEM: 741 name = "mem"; 742 break; 743 case PLUGIN_GEN_AFTER_INSN: 744 name = "after insn"; 745 break; 746 default: 747 break; 748 } 749 switch (op->args[1]) { 750 case PLUGIN_GEN_CB_UDATA: 751 type = "udata"; 752 break; 753 case PLUGIN_GEN_CB_INLINE: 754 type = "inline"; 755 break; 756 case PLUGIN_GEN_CB_MEM: 757 type = "mem"; 758 break; 759 case PLUGIN_GEN_ENABLE_MEM_HELPER: 760 type = "enable mem helper"; 761 break; 762 case PLUGIN_GEN_DISABLE_MEM_HELPER: 763 type = "disable mem helper"; 764 break; 765 default: 766 break; 767 } 768 } 769 printf("op[%2i]: %s %s %s\n", i, tcg_op_defs[op->opc].name, name, type); 770 i++; 771 } 772 #endif 773 } 774 775 static void plugin_gen_inject(struct qemu_plugin_tb *plugin_tb) 776 { 777 TCGOp *op, *next; 778 int insn_idx = -1; 779 780 pr_ops(); 781 782 /* 783 * While injecting code, we cannot afford to reuse any ebb temps 784 * that might be live within the existing opcode stream. 785 * The simplest solution is to release them all and create new. 786 */ 787 memset(tcg_ctx->free_temps, 0, sizeof(tcg_ctx->free_temps)); 788 789 QTAILQ_FOREACH_SAFE(op, &tcg_ctx->ops, link, next) { 790 switch (op->opc) { 791 case INDEX_op_insn_start: 792 insn_idx++; 793 break; 794 795 case INDEX_op_plugin_cb: 796 { 797 enum plugin_gen_from from = op->args[0]; 798 struct qemu_plugin_insn *insn = NULL; 799 const GArray *cbs; 800 int i, n; 801 802 if (insn_idx >= 0) { 803 insn = g_ptr_array_index(plugin_tb->insns, insn_idx); 804 } 805 806 tcg_ctx->emit_before_op = op; 807 808 switch (from) { 809 case PLUGIN_GEN_AFTER_INSN: 810 assert(insn != NULL); 811 gen_disable_mem_helper(plugin_tb, insn); 812 break; 813 814 case PLUGIN_GEN_FROM_TB: 815 assert(insn == NULL); 816 817 cbs = plugin_tb->cbs[PLUGIN_CB_REGULAR]; 818 for (i = 0, n = (cbs ? cbs->len : 0); i < n; i++) { 819 struct qemu_plugin_dyn_cb *cb = 820 &g_array_index(cbs, struct qemu_plugin_dyn_cb, i); 821 gen_udata_cb(cb); 822 } 823 824 cbs = plugin_tb->cbs[PLUGIN_CB_INLINE]; 825 for (i = 0, n = (cbs ? cbs->len : 0); i < n; i++) { 826 struct qemu_plugin_dyn_cb *cb = 827 &g_array_index(cbs, struct qemu_plugin_dyn_cb, i); 828 gen_inline_cb(cb); 829 } 830 break; 831 832 default: 833 g_assert_not_reached(); 834 } 835 836 tcg_ctx->emit_before_op = NULL; 837 tcg_op_remove(tcg_ctx, op); 838 break; 839 } 840 841 case INDEX_op_plugin_cb_start: 842 { 843 enum plugin_gen_from from = op->args[0]; 844 enum plugin_gen_cb type = op->args[1]; 845 846 switch (from) { 847 case PLUGIN_GEN_FROM_INSN: 848 { 849 g_assert(insn_idx >= 0); 850 851 switch (type) { 852 case PLUGIN_GEN_CB_UDATA: 853 plugin_gen_insn_udata(plugin_tb, op, insn_idx); 854 break; 855 case PLUGIN_GEN_CB_UDATA_R: 856 plugin_gen_insn_udata_r(plugin_tb, op, insn_idx); 857 break; 858 case PLUGIN_GEN_CB_INLINE: 859 plugin_gen_insn_inline(plugin_tb, op, insn_idx); 860 break; 861 case PLUGIN_GEN_ENABLE_MEM_HELPER: 862 plugin_gen_enable_mem_helper(plugin_tb, op, insn_idx); 863 break; 864 default: 865 g_assert_not_reached(); 866 } 867 break; 868 } 869 case PLUGIN_GEN_FROM_MEM: 870 { 871 g_assert(insn_idx >= 0); 872 873 switch (type) { 874 case PLUGIN_GEN_CB_MEM: 875 plugin_gen_mem_regular(plugin_tb, op, insn_idx); 876 break; 877 case PLUGIN_GEN_CB_INLINE: 878 plugin_gen_mem_inline(plugin_tb, op, insn_idx); 879 break; 880 default: 881 g_assert_not_reached(); 882 } 883 884 break; 885 } 886 default: 887 g_assert_not_reached(); 888 } 889 break; 890 } 891 default: 892 /* plugins don't care about any other ops */ 893 break; 894 } 895 } 896 pr_ops(); 897 } 898 899 bool plugin_gen_tb_start(CPUState *cpu, const DisasContextBase *db, 900 bool mem_only) 901 { 902 bool ret = false; 903 904 if (test_bit(QEMU_PLUGIN_EV_VCPU_TB_TRANS, cpu->plugin_state->event_mask)) { 905 struct qemu_plugin_tb *ptb = tcg_ctx->plugin_tb; 906 int i; 907 908 /* reset callbacks */ 909 for (i = 0; i < PLUGIN_N_CB_SUBTYPES; i++) { 910 if (ptb->cbs[i]) { 911 g_array_set_size(ptb->cbs[i], 0); 912 } 913 } 914 ptb->n = 0; 915 916 ret = true; 917 918 ptb->vaddr = db->pc_first; 919 ptb->vaddr2 = -1; 920 ptb->haddr1 = db->host_addr[0]; 921 ptb->haddr2 = NULL; 922 ptb->mem_only = mem_only; 923 ptb->mem_helper = false; 924 925 plugin_gen_empty_callback(PLUGIN_GEN_FROM_TB); 926 } 927 928 tcg_ctx->plugin_insn = NULL; 929 930 return ret; 931 } 932 933 void plugin_gen_insn_start(CPUState *cpu, const DisasContextBase *db) 934 { 935 struct qemu_plugin_tb *ptb = tcg_ctx->plugin_tb; 936 struct qemu_plugin_insn *pinsn; 937 938 pinsn = qemu_plugin_tb_insn_get(ptb, db->pc_next); 939 tcg_ctx->plugin_insn = pinsn; 940 plugin_gen_empty_callback(PLUGIN_GEN_FROM_INSN); 941 942 /* 943 * Detect page crossing to get the new host address. 944 * Note that we skip this when haddr1 == NULL, e.g. when we're 945 * fetching instructions from a region not backed by RAM. 946 */ 947 if (ptb->haddr1 == NULL) { 948 pinsn->haddr = NULL; 949 } else if (is_same_page(db, db->pc_next)) { 950 pinsn->haddr = ptb->haddr1 + pinsn->vaddr - ptb->vaddr; 951 } else { 952 if (ptb->vaddr2 == -1) { 953 ptb->vaddr2 = TARGET_PAGE_ALIGN(db->pc_first); 954 get_page_addr_code_hostp(cpu_env(cpu), ptb->vaddr2, &ptb->haddr2); 955 } 956 pinsn->haddr = ptb->haddr2 + pinsn->vaddr - ptb->vaddr2; 957 } 958 } 959 960 void plugin_gen_insn_end(void) 961 { 962 plugin_gen_empty_callback(PLUGIN_GEN_AFTER_INSN); 963 } 964 965 /* 966 * There are cases where we never get to finalise a translation - for 967 * example a page fault during translation. As a result we shouldn't 968 * do any clean-up here and make sure things are reset in 969 * plugin_gen_tb_start. 970 */ 971 void plugin_gen_tb_end(CPUState *cpu, size_t num_insns) 972 { 973 struct qemu_plugin_tb *ptb = tcg_ctx->plugin_tb; 974 975 /* translator may have removed instructions, update final count */ 976 g_assert(num_insns <= ptb->n); 977 ptb->n = num_insns; 978 979 /* collect instrumentation requests */ 980 qemu_plugin_tb_trans_cb(cpu, ptb); 981 982 /* inject the instrumentation at the appropriate places */ 983 plugin_gen_inject(ptb); 984 } 985