1 #include "qemu/osdep.h" 2 #include "qemu/units.h" 3 #include "qemu/error-report.h" 4 #include "qapi/qapi-commands-cxl.h" 5 #include "hw/mem/memory-device.h" 6 #include "hw/mem/pc-dimm.h" 7 #include "hw/pci/pci.h" 8 #include "hw/qdev-properties.h" 9 #include "qapi/error.h" 10 #include "qemu/log.h" 11 #include "qemu/module.h" 12 #include "qemu/pmem.h" 13 #include "qemu/range.h" 14 #include "qemu/rcu.h" 15 #include "sysemu/hostmem.h" 16 #include "sysemu/numa.h" 17 #include "hw/cxl/cxl.h" 18 #include "hw/pci/msix.h" 19 20 #define DWORD_BYTE 4 21 22 /* Default CDAT entries for a memory region */ 23 enum { 24 CT3_CDAT_DSMAS, 25 CT3_CDAT_DSLBIS0, 26 CT3_CDAT_DSLBIS1, 27 CT3_CDAT_DSLBIS2, 28 CT3_CDAT_DSLBIS3, 29 CT3_CDAT_DSEMTS, 30 CT3_CDAT_NUM_ENTRIES 31 }; 32 33 static int ct3_build_cdat_entries_for_mr(CDATSubHeader **cdat_table, 34 int dsmad_handle, MemoryRegion *mr) 35 { 36 g_autofree CDATDsmas *dsmas = NULL; 37 g_autofree CDATDslbis *dslbis0 = NULL; 38 g_autofree CDATDslbis *dslbis1 = NULL; 39 g_autofree CDATDslbis *dslbis2 = NULL; 40 g_autofree CDATDslbis *dslbis3 = NULL; 41 g_autofree CDATDsemts *dsemts = NULL; 42 43 dsmas = g_malloc(sizeof(*dsmas)); 44 if (!dsmas) { 45 return -ENOMEM; 46 } 47 *dsmas = (CDATDsmas) { 48 .header = { 49 .type = CDAT_TYPE_DSMAS, 50 .length = sizeof(*dsmas), 51 }, 52 .DSMADhandle = dsmad_handle, 53 .flags = CDAT_DSMAS_FLAG_NV, 54 .DPA_base = 0, 55 .DPA_length = int128_get64(mr->size), 56 }; 57 58 /* For now, no memory side cache, plausiblish numbers */ 59 dslbis0 = g_malloc(sizeof(*dslbis0)); 60 if (!dslbis0) { 61 return -ENOMEM; 62 } 63 *dslbis0 = (CDATDslbis) { 64 .header = { 65 .type = CDAT_TYPE_DSLBIS, 66 .length = sizeof(*dslbis0), 67 }, 68 .handle = dsmad_handle, 69 .flags = HMAT_LB_MEM_MEMORY, 70 .data_type = HMAT_LB_DATA_READ_LATENCY, 71 .entry_base_unit = 10000, /* 10ns base */ 72 .entry[0] = 15, /* 150ns */ 73 }; 74 75 dslbis1 = g_malloc(sizeof(*dslbis1)); 76 if (!dslbis1) { 77 return -ENOMEM; 78 } 79 *dslbis1 = (CDATDslbis) { 80 .header = { 81 .type = CDAT_TYPE_DSLBIS, 82 .length = sizeof(*dslbis1), 83 }, 84 .handle = dsmad_handle, 85 .flags = HMAT_LB_MEM_MEMORY, 86 .data_type = HMAT_LB_DATA_WRITE_LATENCY, 87 .entry_base_unit = 10000, 88 .entry[0] = 25, /* 250ns */ 89 }; 90 91 dslbis2 = g_malloc(sizeof(*dslbis2)); 92 if (!dslbis2) { 93 return -ENOMEM; 94 } 95 *dslbis2 = (CDATDslbis) { 96 .header = { 97 .type = CDAT_TYPE_DSLBIS, 98 .length = sizeof(*dslbis2), 99 }, 100 .handle = dsmad_handle, 101 .flags = HMAT_LB_MEM_MEMORY, 102 .data_type = HMAT_LB_DATA_READ_BANDWIDTH, 103 .entry_base_unit = 1000, /* GB/s */ 104 .entry[0] = 16, 105 }; 106 107 dslbis3 = g_malloc(sizeof(*dslbis3)); 108 if (!dslbis3) { 109 return -ENOMEM; 110 } 111 *dslbis3 = (CDATDslbis) { 112 .header = { 113 .type = CDAT_TYPE_DSLBIS, 114 .length = sizeof(*dslbis3), 115 }, 116 .handle = dsmad_handle, 117 .flags = HMAT_LB_MEM_MEMORY, 118 .data_type = HMAT_LB_DATA_WRITE_BANDWIDTH, 119 .entry_base_unit = 1000, /* GB/s */ 120 .entry[0] = 16, 121 }; 122 123 dsemts = g_malloc(sizeof(*dsemts)); 124 if (!dsemts) { 125 return -ENOMEM; 126 } 127 *dsemts = (CDATDsemts) { 128 .header = { 129 .type = CDAT_TYPE_DSEMTS, 130 .length = sizeof(*dsemts), 131 }, 132 .DSMAS_handle = dsmad_handle, 133 /* Reserved - the non volatile from DSMAS matters */ 134 .EFI_memory_type_attr = 2, 135 .DPA_offset = 0, 136 .DPA_length = int128_get64(mr->size), 137 }; 138 139 /* Header always at start of structure */ 140 cdat_table[CT3_CDAT_DSMAS] = g_steal_pointer(&dsmas); 141 cdat_table[CT3_CDAT_DSLBIS0] = g_steal_pointer(&dslbis0); 142 cdat_table[CT3_CDAT_DSLBIS1] = g_steal_pointer(&dslbis1); 143 cdat_table[CT3_CDAT_DSLBIS2] = g_steal_pointer(&dslbis2); 144 cdat_table[CT3_CDAT_DSLBIS3] = g_steal_pointer(&dslbis3); 145 cdat_table[CT3_CDAT_DSEMTS] = g_steal_pointer(&dsemts); 146 147 return 0; 148 } 149 150 static int ct3_build_cdat_table(CDATSubHeader ***cdat_table, void *priv) 151 { 152 g_autofree CDATSubHeader **table = NULL; 153 MemoryRegion *nonvolatile_mr; 154 CXLType3Dev *ct3d = priv; 155 int dsmad_handle = 0; 156 int rc; 157 158 if (!ct3d->hostmem) { 159 return 0; 160 } 161 162 nonvolatile_mr = host_memory_backend_get_memory(ct3d->hostmem); 163 if (!nonvolatile_mr) { 164 return -EINVAL; 165 } 166 167 table = g_malloc0(CT3_CDAT_NUM_ENTRIES * sizeof(*table)); 168 if (!table) { 169 return -ENOMEM; 170 } 171 172 rc = ct3_build_cdat_entries_for_mr(table, dsmad_handle++, nonvolatile_mr); 173 if (rc < 0) { 174 return rc; 175 } 176 177 *cdat_table = g_steal_pointer(&table); 178 179 return CT3_CDAT_NUM_ENTRIES; 180 } 181 182 static void ct3_free_cdat_table(CDATSubHeader **cdat_table, int num, void *priv) 183 { 184 int i; 185 186 for (i = 0; i < num; i++) { 187 g_free(cdat_table[i]); 188 } 189 g_free(cdat_table); 190 } 191 192 static bool cxl_doe_cdat_rsp(DOECap *doe_cap) 193 { 194 CDATObject *cdat = &CXL_TYPE3(doe_cap->pdev)->cxl_cstate.cdat; 195 uint16_t ent; 196 void *base; 197 uint32_t len; 198 CDATReq *req = pcie_doe_get_write_mbox_ptr(doe_cap); 199 CDATRsp rsp; 200 201 assert(cdat->entry_len); 202 203 /* Discard if request length mismatched */ 204 if (pcie_doe_get_obj_len(req) < 205 DIV_ROUND_UP(sizeof(CDATReq), DWORD_BYTE)) { 206 return false; 207 } 208 209 ent = req->entry_handle; 210 base = cdat->entry[ent].base; 211 len = cdat->entry[ent].length; 212 213 rsp = (CDATRsp) { 214 .header = { 215 .vendor_id = CXL_VENDOR_ID, 216 .data_obj_type = CXL_DOE_TABLE_ACCESS, 217 .reserved = 0x0, 218 .length = DIV_ROUND_UP((sizeof(rsp) + len), DWORD_BYTE), 219 }, 220 .rsp_code = CXL_DOE_TAB_RSP, 221 .table_type = CXL_DOE_TAB_TYPE_CDAT, 222 .entry_handle = (ent < cdat->entry_len - 1) ? 223 ent + 1 : CXL_DOE_TAB_ENT_MAX, 224 }; 225 226 memcpy(doe_cap->read_mbox, &rsp, sizeof(rsp)); 227 memcpy(doe_cap->read_mbox + DIV_ROUND_UP(sizeof(rsp), DWORD_BYTE), 228 base, len); 229 230 doe_cap->read_mbox_len += rsp.header.length; 231 232 return true; 233 } 234 235 static uint32_t ct3d_config_read(PCIDevice *pci_dev, uint32_t addr, int size) 236 { 237 CXLType3Dev *ct3d = CXL_TYPE3(pci_dev); 238 uint32_t val; 239 240 if (pcie_doe_read_config(&ct3d->doe_cdat, addr, size, &val)) { 241 return val; 242 } 243 244 return pci_default_read_config(pci_dev, addr, size); 245 } 246 247 static void ct3d_config_write(PCIDevice *pci_dev, uint32_t addr, uint32_t val, 248 int size) 249 { 250 CXLType3Dev *ct3d = CXL_TYPE3(pci_dev); 251 252 pcie_doe_write_config(&ct3d->doe_cdat, addr, val, size); 253 pci_default_write_config(pci_dev, addr, val, size); 254 pcie_aer_write_config(pci_dev, addr, val, size); 255 } 256 257 /* 258 * Null value of all Fs suggested by IEEE RA guidelines for use of 259 * EU, OUI and CID 260 */ 261 #define UI64_NULL ~(0ULL) 262 263 static void build_dvsecs(CXLType3Dev *ct3d) 264 { 265 CXLComponentState *cxl_cstate = &ct3d->cxl_cstate; 266 uint8_t *dvsec; 267 268 dvsec = (uint8_t *)&(CXLDVSECDevice){ 269 .cap = 0x1e, 270 .ctrl = 0x2, 271 .status2 = 0x2, 272 .range1_size_hi = ct3d->hostmem->size >> 32, 273 .range1_size_lo = (2 << 5) | (2 << 2) | 0x3 | 274 (ct3d->hostmem->size & 0xF0000000), 275 .range1_base_hi = 0, 276 .range1_base_lo = 0, 277 }; 278 cxl_component_create_dvsec(cxl_cstate, CXL2_TYPE3_DEVICE, 279 PCIE_CXL_DEVICE_DVSEC_LENGTH, 280 PCIE_CXL_DEVICE_DVSEC, 281 PCIE_CXL2_DEVICE_DVSEC_REVID, dvsec); 282 283 dvsec = (uint8_t *)&(CXLDVSECRegisterLocator){ 284 .rsvd = 0, 285 .reg0_base_lo = RBI_COMPONENT_REG | CXL_COMPONENT_REG_BAR_IDX, 286 .reg0_base_hi = 0, 287 .reg1_base_lo = RBI_CXL_DEVICE_REG | CXL_DEVICE_REG_BAR_IDX, 288 .reg1_base_hi = 0, 289 }; 290 cxl_component_create_dvsec(cxl_cstate, CXL2_TYPE3_DEVICE, 291 REG_LOC_DVSEC_LENGTH, REG_LOC_DVSEC, 292 REG_LOC_DVSEC_REVID, dvsec); 293 dvsec = (uint8_t *)&(CXLDVSECDeviceGPF){ 294 .phase2_duration = 0x603, /* 3 seconds */ 295 .phase2_power = 0x33, /* 0x33 miliwatts */ 296 }; 297 cxl_component_create_dvsec(cxl_cstate, CXL2_TYPE3_DEVICE, 298 GPF_DEVICE_DVSEC_LENGTH, GPF_DEVICE_DVSEC, 299 GPF_DEVICE_DVSEC_REVID, dvsec); 300 301 dvsec = (uint8_t *)&(CXLDVSECPortFlexBus){ 302 .cap = 0x26, /* 68B, IO, Mem, non-MLD */ 303 .ctrl = 0x02, /* IO always enabled */ 304 .status = 0x26, /* same as capabilities */ 305 .rcvd_mod_ts_data_phase1 = 0xef, /* WTF? */ 306 }; 307 cxl_component_create_dvsec(cxl_cstate, CXL2_TYPE3_DEVICE, 308 PCIE_FLEXBUS_PORT_DVSEC_LENGTH_2_0, 309 PCIE_FLEXBUS_PORT_DVSEC, 310 PCIE_FLEXBUS_PORT_DVSEC_REVID_2_0, dvsec); 311 } 312 313 static void hdm_decoder_commit(CXLType3Dev *ct3d, int which) 314 { 315 ComponentRegisters *cregs = &ct3d->cxl_cstate.crb; 316 uint32_t *cache_mem = cregs->cache_mem_registers; 317 318 assert(which == 0); 319 320 /* TODO: Sanity checks that the decoder is possible */ 321 ARRAY_FIELD_DP32(cache_mem, CXL_HDM_DECODER0_CTRL, COMMIT, 0); 322 ARRAY_FIELD_DP32(cache_mem, CXL_HDM_DECODER0_CTRL, ERR, 0); 323 324 ARRAY_FIELD_DP32(cache_mem, CXL_HDM_DECODER0_CTRL, COMMITTED, 1); 325 } 326 327 static int ct3d_qmp_uncor_err_to_cxl(CxlUncorErrorType qmp_err) 328 { 329 switch (qmp_err) { 330 case CXL_UNCOR_ERROR_TYPE_CACHE_DATA_PARITY: 331 return CXL_RAS_UNC_ERR_CACHE_DATA_PARITY; 332 case CXL_UNCOR_ERROR_TYPE_CACHE_ADDRESS_PARITY: 333 return CXL_RAS_UNC_ERR_CACHE_ADDRESS_PARITY; 334 case CXL_UNCOR_ERROR_TYPE_CACHE_BE_PARITY: 335 return CXL_RAS_UNC_ERR_CACHE_BE_PARITY; 336 case CXL_UNCOR_ERROR_TYPE_CACHE_DATA_ECC: 337 return CXL_RAS_UNC_ERR_CACHE_DATA_ECC; 338 case CXL_UNCOR_ERROR_TYPE_MEM_DATA_PARITY: 339 return CXL_RAS_UNC_ERR_MEM_DATA_PARITY; 340 case CXL_UNCOR_ERROR_TYPE_MEM_ADDRESS_PARITY: 341 return CXL_RAS_UNC_ERR_MEM_ADDRESS_PARITY; 342 case CXL_UNCOR_ERROR_TYPE_MEM_BE_PARITY: 343 return CXL_RAS_UNC_ERR_MEM_BE_PARITY; 344 case CXL_UNCOR_ERROR_TYPE_MEM_DATA_ECC: 345 return CXL_RAS_UNC_ERR_MEM_DATA_ECC; 346 case CXL_UNCOR_ERROR_TYPE_REINIT_THRESHOLD: 347 return CXL_RAS_UNC_ERR_REINIT_THRESHOLD; 348 case CXL_UNCOR_ERROR_TYPE_RSVD_ENCODING: 349 return CXL_RAS_UNC_ERR_RSVD_ENCODING; 350 case CXL_UNCOR_ERROR_TYPE_POISON_RECEIVED: 351 return CXL_RAS_UNC_ERR_POISON_RECEIVED; 352 case CXL_UNCOR_ERROR_TYPE_RECEIVER_OVERFLOW: 353 return CXL_RAS_UNC_ERR_RECEIVER_OVERFLOW; 354 case CXL_UNCOR_ERROR_TYPE_INTERNAL: 355 return CXL_RAS_UNC_ERR_INTERNAL; 356 case CXL_UNCOR_ERROR_TYPE_CXL_IDE_TX: 357 return CXL_RAS_UNC_ERR_CXL_IDE_TX; 358 case CXL_UNCOR_ERROR_TYPE_CXL_IDE_RX: 359 return CXL_RAS_UNC_ERR_CXL_IDE_RX; 360 default: 361 return -EINVAL; 362 } 363 } 364 365 static int ct3d_qmp_cor_err_to_cxl(CxlCorErrorType qmp_err) 366 { 367 switch (qmp_err) { 368 case CXL_COR_ERROR_TYPE_CACHE_DATA_ECC: 369 return CXL_RAS_COR_ERR_CACHE_DATA_ECC; 370 case CXL_COR_ERROR_TYPE_MEM_DATA_ECC: 371 return CXL_RAS_COR_ERR_MEM_DATA_ECC; 372 case CXL_COR_ERROR_TYPE_CRC_THRESHOLD: 373 return CXL_RAS_COR_ERR_CRC_THRESHOLD; 374 case CXL_COR_ERROR_TYPE_RETRY_THRESHOLD: 375 return CXL_RAS_COR_ERR_RETRY_THRESHOLD; 376 case CXL_COR_ERROR_TYPE_CACHE_POISON_RECEIVED: 377 return CXL_RAS_COR_ERR_CACHE_POISON_RECEIVED; 378 case CXL_COR_ERROR_TYPE_MEM_POISON_RECEIVED: 379 return CXL_RAS_COR_ERR_MEM_POISON_RECEIVED; 380 case CXL_COR_ERROR_TYPE_PHYSICAL: 381 return CXL_RAS_COR_ERR_PHYSICAL; 382 default: 383 return -EINVAL; 384 } 385 } 386 387 static void ct3d_reg_write(void *opaque, hwaddr offset, uint64_t value, 388 unsigned size) 389 { 390 CXLComponentState *cxl_cstate = opaque; 391 ComponentRegisters *cregs = &cxl_cstate->crb; 392 CXLType3Dev *ct3d = container_of(cxl_cstate, CXLType3Dev, cxl_cstate); 393 uint32_t *cache_mem = cregs->cache_mem_registers; 394 bool should_commit = false; 395 int which_hdm = -1; 396 397 assert(size == 4); 398 g_assert(offset < CXL2_COMPONENT_CM_REGION_SIZE); 399 400 switch (offset) { 401 case A_CXL_HDM_DECODER0_CTRL: 402 should_commit = FIELD_EX32(value, CXL_HDM_DECODER0_CTRL, COMMIT); 403 which_hdm = 0; 404 break; 405 case A_CXL_RAS_UNC_ERR_STATUS: 406 { 407 uint32_t capctrl = ldl_le_p(cache_mem + R_CXL_RAS_ERR_CAP_CTRL); 408 uint32_t fe = FIELD_EX32(capctrl, CXL_RAS_ERR_CAP_CTRL, FIRST_ERROR_POINTER); 409 CXLError *cxl_err; 410 uint32_t unc_err; 411 412 /* 413 * If single bit written that corresponds to the first error 414 * pointer being cleared, update the status and header log. 415 */ 416 if (!QTAILQ_EMPTY(&ct3d->error_list)) { 417 if ((1 << fe) ^ value) { 418 CXLError *cxl_next; 419 /* 420 * Software is using wrong flow for multiple header recording 421 * Following behavior in PCIe r6.0 and assuming multiple 422 * header support. Implementation defined choice to clear all 423 * matching records if more than one bit set - which corresponds 424 * closest to behavior of hardware not capable of multiple 425 * header recording. 426 */ 427 QTAILQ_FOREACH_SAFE(cxl_err, &ct3d->error_list, node, cxl_next) { 428 if ((1 << cxl_err->type) & value) { 429 QTAILQ_REMOVE(&ct3d->error_list, cxl_err, node); 430 g_free(cxl_err); 431 } 432 } 433 } else { 434 /* Done with previous FE, so drop from list */ 435 cxl_err = QTAILQ_FIRST(&ct3d->error_list); 436 QTAILQ_REMOVE(&ct3d->error_list, cxl_err, node); 437 g_free(cxl_err); 438 } 439 440 /* 441 * If there is another FE, then put that in place and update 442 * the header log 443 */ 444 if (!QTAILQ_EMPTY(&ct3d->error_list)) { 445 uint32_t *header_log = &cache_mem[R_CXL_RAS_ERR_HEADER0]; 446 int i; 447 448 cxl_err = QTAILQ_FIRST(&ct3d->error_list); 449 for (i = 0; i < CXL_RAS_ERR_HEADER_NUM; i++) { 450 stl_le_p(header_log + i, cxl_err->header[i]); 451 } 452 capctrl = FIELD_DP32(capctrl, CXL_RAS_ERR_CAP_CTRL, 453 FIRST_ERROR_POINTER, cxl_err->type); 454 } else { 455 /* 456 * If no more errors, then follow recomendation of PCI spec 457 * r6.0 6.2.4.2 to set the first error pointer to a status 458 * bit that will never be used. 459 */ 460 capctrl = FIELD_DP32(capctrl, CXL_RAS_ERR_CAP_CTRL, 461 FIRST_ERROR_POINTER, 462 CXL_RAS_UNC_ERR_CXL_UNUSED); 463 } 464 stl_le_p((uint8_t *)cache_mem + A_CXL_RAS_ERR_CAP_CTRL, capctrl); 465 } 466 unc_err = 0; 467 QTAILQ_FOREACH(cxl_err, &ct3d->error_list, node) { 468 unc_err |= 1 << cxl_err->type; 469 } 470 stl_le_p((uint8_t *)cache_mem + offset, unc_err); 471 472 return; 473 } 474 case A_CXL_RAS_COR_ERR_STATUS: 475 { 476 uint32_t rw1c = value; 477 uint32_t temp = ldl_le_p((uint8_t *)cache_mem + offset); 478 temp &= ~rw1c; 479 stl_le_p((uint8_t *)cache_mem + offset, temp); 480 return; 481 } 482 default: 483 break; 484 } 485 486 stl_le_p((uint8_t *)cache_mem + offset, value); 487 if (should_commit) { 488 hdm_decoder_commit(ct3d, which_hdm); 489 } 490 } 491 492 static bool cxl_setup_memory(CXLType3Dev *ct3d, Error **errp) 493 { 494 DeviceState *ds = DEVICE(ct3d); 495 MemoryRegion *mr; 496 char *name; 497 498 if (!ct3d->hostmem) { 499 error_setg(errp, "memdev property must be set"); 500 return false; 501 } 502 503 mr = host_memory_backend_get_memory(ct3d->hostmem); 504 if (!mr) { 505 error_setg(errp, "memdev property must be set"); 506 return false; 507 } 508 memory_region_set_nonvolatile(mr, true); 509 memory_region_set_enabled(mr, true); 510 host_memory_backend_set_mapped(ct3d->hostmem, true); 511 512 if (ds->id) { 513 name = g_strdup_printf("cxl-type3-dpa-space:%s", ds->id); 514 } else { 515 name = g_strdup("cxl-type3-dpa-space"); 516 } 517 address_space_init(&ct3d->hostmem_as, mr, name); 518 g_free(name); 519 520 ct3d->cxl_dstate.pmem_size = ct3d->hostmem->size; 521 522 if (!ct3d->lsa) { 523 error_setg(errp, "lsa property must be set"); 524 return false; 525 } 526 527 return true; 528 } 529 530 static DOEProtocol doe_cdat_prot[] = { 531 { CXL_VENDOR_ID, CXL_DOE_TABLE_ACCESS, cxl_doe_cdat_rsp }, 532 { } 533 }; 534 535 static void ct3_realize(PCIDevice *pci_dev, Error **errp) 536 { 537 CXLType3Dev *ct3d = CXL_TYPE3(pci_dev); 538 CXLComponentState *cxl_cstate = &ct3d->cxl_cstate; 539 ComponentRegisters *regs = &cxl_cstate->crb; 540 MemoryRegion *mr = ®s->component_registers; 541 uint8_t *pci_conf = pci_dev->config; 542 unsigned short msix_num = 1; 543 int i, rc; 544 545 QTAILQ_INIT(&ct3d->error_list); 546 547 if (!cxl_setup_memory(ct3d, errp)) { 548 return; 549 } 550 551 pci_config_set_prog_interface(pci_conf, 0x10); 552 553 pcie_endpoint_cap_init(pci_dev, 0x80); 554 if (ct3d->sn != UI64_NULL) { 555 pcie_dev_ser_num_init(pci_dev, 0x100, ct3d->sn); 556 cxl_cstate->dvsec_offset = 0x100 + 0x0c; 557 } else { 558 cxl_cstate->dvsec_offset = 0x100; 559 } 560 561 ct3d->cxl_cstate.pdev = pci_dev; 562 build_dvsecs(ct3d); 563 564 regs->special_ops = g_new0(MemoryRegionOps, 1); 565 regs->special_ops->write = ct3d_reg_write; 566 567 cxl_component_register_block_init(OBJECT(pci_dev), cxl_cstate, 568 TYPE_CXL_TYPE3); 569 570 pci_register_bar( 571 pci_dev, CXL_COMPONENT_REG_BAR_IDX, 572 PCI_BASE_ADDRESS_SPACE_MEMORY | PCI_BASE_ADDRESS_MEM_TYPE_64, mr); 573 574 cxl_device_register_block_init(OBJECT(pci_dev), &ct3d->cxl_dstate); 575 pci_register_bar(pci_dev, CXL_DEVICE_REG_BAR_IDX, 576 PCI_BASE_ADDRESS_SPACE_MEMORY | 577 PCI_BASE_ADDRESS_MEM_TYPE_64, 578 &ct3d->cxl_dstate.device_registers); 579 580 /* MSI(-X) Initailization */ 581 rc = msix_init_exclusive_bar(pci_dev, msix_num, 4, NULL); 582 if (rc) { 583 goto err_address_space_free; 584 } 585 for (i = 0; i < msix_num; i++) { 586 msix_vector_use(pci_dev, i); 587 } 588 589 /* DOE Initailization */ 590 pcie_doe_init(pci_dev, &ct3d->doe_cdat, 0x190, doe_cdat_prot, true, 0); 591 592 cxl_cstate->cdat.build_cdat_table = ct3_build_cdat_table; 593 cxl_cstate->cdat.free_cdat_table = ct3_free_cdat_table; 594 cxl_cstate->cdat.private = ct3d; 595 cxl_doe_cdat_init(cxl_cstate, errp); 596 597 pcie_cap_deverr_init(pci_dev); 598 /* Leave a bit of room for expansion */ 599 rc = pcie_aer_init(pci_dev, PCI_ERR_VER, 0x200, PCI_ERR_SIZEOF, NULL); 600 if (rc) { 601 goto err_release_cdat; 602 } 603 604 return; 605 606 err_release_cdat: 607 cxl_doe_cdat_release(cxl_cstate); 608 g_free(regs->special_ops); 609 err_address_space_free: 610 address_space_destroy(&ct3d->hostmem_as); 611 return; 612 } 613 614 static void ct3_exit(PCIDevice *pci_dev) 615 { 616 CXLType3Dev *ct3d = CXL_TYPE3(pci_dev); 617 CXLComponentState *cxl_cstate = &ct3d->cxl_cstate; 618 ComponentRegisters *regs = &cxl_cstate->crb; 619 620 pcie_aer_exit(pci_dev); 621 cxl_doe_cdat_release(cxl_cstate); 622 g_free(regs->special_ops); 623 address_space_destroy(&ct3d->hostmem_as); 624 } 625 626 /* TODO: Support multiple HDM decoders and DPA skip */ 627 static bool cxl_type3_dpa(CXLType3Dev *ct3d, hwaddr host_addr, uint64_t *dpa) 628 { 629 uint32_t *cache_mem = ct3d->cxl_cstate.crb.cache_mem_registers; 630 uint64_t decoder_base, decoder_size, hpa_offset; 631 uint32_t hdm0_ctrl; 632 int ig, iw; 633 634 decoder_base = (((uint64_t)cache_mem[R_CXL_HDM_DECODER0_BASE_HI] << 32) | 635 cache_mem[R_CXL_HDM_DECODER0_BASE_LO]); 636 if ((uint64_t)host_addr < decoder_base) { 637 return false; 638 } 639 640 hpa_offset = (uint64_t)host_addr - decoder_base; 641 642 decoder_size = ((uint64_t)cache_mem[R_CXL_HDM_DECODER0_SIZE_HI] << 32) | 643 cache_mem[R_CXL_HDM_DECODER0_SIZE_LO]; 644 if (hpa_offset >= decoder_size) { 645 return false; 646 } 647 648 hdm0_ctrl = cache_mem[R_CXL_HDM_DECODER0_CTRL]; 649 iw = FIELD_EX32(hdm0_ctrl, CXL_HDM_DECODER0_CTRL, IW); 650 ig = FIELD_EX32(hdm0_ctrl, CXL_HDM_DECODER0_CTRL, IG); 651 652 *dpa = (MAKE_64BIT_MASK(0, 8 + ig) & hpa_offset) | 653 ((MAKE_64BIT_MASK(8 + ig + iw, 64 - 8 - ig - iw) & hpa_offset) >> iw); 654 655 return true; 656 } 657 658 MemTxResult cxl_type3_read(PCIDevice *d, hwaddr host_addr, uint64_t *data, 659 unsigned size, MemTxAttrs attrs) 660 { 661 CXLType3Dev *ct3d = CXL_TYPE3(d); 662 uint64_t dpa_offset; 663 MemoryRegion *mr; 664 665 /* TODO support volatile region */ 666 mr = host_memory_backend_get_memory(ct3d->hostmem); 667 if (!mr) { 668 return MEMTX_ERROR; 669 } 670 671 if (!cxl_type3_dpa(ct3d, host_addr, &dpa_offset)) { 672 return MEMTX_ERROR; 673 } 674 675 if (dpa_offset > int128_get64(mr->size)) { 676 return MEMTX_ERROR; 677 } 678 679 return address_space_read(&ct3d->hostmem_as, dpa_offset, attrs, data, size); 680 } 681 682 MemTxResult cxl_type3_write(PCIDevice *d, hwaddr host_addr, uint64_t data, 683 unsigned size, MemTxAttrs attrs) 684 { 685 CXLType3Dev *ct3d = CXL_TYPE3(d); 686 uint64_t dpa_offset; 687 MemoryRegion *mr; 688 689 mr = host_memory_backend_get_memory(ct3d->hostmem); 690 if (!mr) { 691 return MEMTX_OK; 692 } 693 694 if (!cxl_type3_dpa(ct3d, host_addr, &dpa_offset)) { 695 return MEMTX_OK; 696 } 697 698 if (dpa_offset > int128_get64(mr->size)) { 699 return MEMTX_OK; 700 } 701 return address_space_write(&ct3d->hostmem_as, dpa_offset, attrs, 702 &data, size); 703 } 704 705 static void ct3d_reset(DeviceState *dev) 706 { 707 CXLType3Dev *ct3d = CXL_TYPE3(dev); 708 uint32_t *reg_state = ct3d->cxl_cstate.crb.cache_mem_registers; 709 uint32_t *write_msk = ct3d->cxl_cstate.crb.cache_mem_regs_write_mask; 710 711 cxl_component_register_init_common(reg_state, write_msk, CXL2_TYPE3_DEVICE); 712 cxl_device_register_init_common(&ct3d->cxl_dstate); 713 } 714 715 static Property ct3_props[] = { 716 DEFINE_PROP_LINK("memdev", CXLType3Dev, hostmem, TYPE_MEMORY_BACKEND, 717 HostMemoryBackend *), 718 DEFINE_PROP_LINK("lsa", CXLType3Dev, lsa, TYPE_MEMORY_BACKEND, 719 HostMemoryBackend *), 720 DEFINE_PROP_UINT64("sn", CXLType3Dev, sn, UI64_NULL), 721 DEFINE_PROP_STRING("cdat", CXLType3Dev, cxl_cstate.cdat.filename), 722 DEFINE_PROP_END_OF_LIST(), 723 }; 724 725 static uint64_t get_lsa_size(CXLType3Dev *ct3d) 726 { 727 MemoryRegion *mr; 728 729 mr = host_memory_backend_get_memory(ct3d->lsa); 730 return memory_region_size(mr); 731 } 732 733 static void validate_lsa_access(MemoryRegion *mr, uint64_t size, 734 uint64_t offset) 735 { 736 assert(offset + size <= memory_region_size(mr)); 737 assert(offset + size > offset); 738 } 739 740 static uint64_t get_lsa(CXLType3Dev *ct3d, void *buf, uint64_t size, 741 uint64_t offset) 742 { 743 MemoryRegion *mr; 744 void *lsa; 745 746 mr = host_memory_backend_get_memory(ct3d->lsa); 747 validate_lsa_access(mr, size, offset); 748 749 lsa = memory_region_get_ram_ptr(mr) + offset; 750 memcpy(buf, lsa, size); 751 752 return size; 753 } 754 755 static void set_lsa(CXLType3Dev *ct3d, const void *buf, uint64_t size, 756 uint64_t offset) 757 { 758 MemoryRegion *mr; 759 void *lsa; 760 761 mr = host_memory_backend_get_memory(ct3d->lsa); 762 validate_lsa_access(mr, size, offset); 763 764 lsa = memory_region_get_ram_ptr(mr) + offset; 765 memcpy(lsa, buf, size); 766 memory_region_set_dirty(mr, offset, size); 767 768 /* 769 * Just like the PMEM, if the guest is not allowed to exit gracefully, label 770 * updates will get lost. 771 */ 772 } 773 774 /* For uncorrectable errors include support for multiple header recording */ 775 void qmp_cxl_inject_uncorrectable_errors(const char *path, 776 CXLUncorErrorRecordList *errors, 777 Error **errp) 778 { 779 Object *obj = object_resolve_path(path, NULL); 780 static PCIEAERErr err = {}; 781 CXLType3Dev *ct3d; 782 CXLError *cxl_err; 783 uint32_t *reg_state; 784 uint32_t unc_err; 785 bool first; 786 787 if (!obj) { 788 error_setg(errp, "Unable to resolve path"); 789 return; 790 } 791 792 if (!object_dynamic_cast(obj, TYPE_CXL_TYPE3)) { 793 error_setg(errp, "Path does not point to a CXL type 3 device"); 794 return; 795 } 796 797 err.status = PCI_ERR_UNC_INTN; 798 err.source_id = pci_requester_id(PCI_DEVICE(obj)); 799 err.flags = 0; 800 801 ct3d = CXL_TYPE3(obj); 802 803 first = QTAILQ_EMPTY(&ct3d->error_list); 804 reg_state = ct3d->cxl_cstate.crb.cache_mem_registers; 805 while (errors) { 806 uint32List *header = errors->value->header; 807 uint8_t header_count = 0; 808 int cxl_err_code; 809 810 cxl_err_code = ct3d_qmp_uncor_err_to_cxl(errors->value->type); 811 if (cxl_err_code < 0) { 812 error_setg(errp, "Unknown error code"); 813 return; 814 } 815 816 /* If the error is masked, nothing to do here */ 817 if (!((1 << cxl_err_code) & 818 ~ldl_le_p(reg_state + R_CXL_RAS_UNC_ERR_MASK))) { 819 errors = errors->next; 820 continue; 821 } 822 823 cxl_err = g_malloc0(sizeof(*cxl_err)); 824 if (!cxl_err) { 825 return; 826 } 827 828 cxl_err->type = cxl_err_code; 829 while (header && header_count < 32) { 830 cxl_err->header[header_count++] = header->value; 831 header = header->next; 832 } 833 if (header_count > 32) { 834 error_setg(errp, "Header must be 32 DWORD or less"); 835 return; 836 } 837 QTAILQ_INSERT_TAIL(&ct3d->error_list, cxl_err, node); 838 839 errors = errors->next; 840 } 841 842 if (first && !QTAILQ_EMPTY(&ct3d->error_list)) { 843 uint32_t *cache_mem = ct3d->cxl_cstate.crb.cache_mem_registers; 844 uint32_t capctrl = ldl_le_p(cache_mem + R_CXL_RAS_ERR_CAP_CTRL); 845 uint32_t *header_log = &cache_mem[R_CXL_RAS_ERR_HEADER0]; 846 int i; 847 848 cxl_err = QTAILQ_FIRST(&ct3d->error_list); 849 for (i = 0; i < CXL_RAS_ERR_HEADER_NUM; i++) { 850 stl_le_p(header_log + i, cxl_err->header[i]); 851 } 852 853 capctrl = FIELD_DP32(capctrl, CXL_RAS_ERR_CAP_CTRL, 854 FIRST_ERROR_POINTER, cxl_err->type); 855 stl_le_p(cache_mem + R_CXL_RAS_ERR_CAP_CTRL, capctrl); 856 } 857 858 unc_err = 0; 859 QTAILQ_FOREACH(cxl_err, &ct3d->error_list, node) { 860 unc_err |= (1 << cxl_err->type); 861 } 862 if (!unc_err) { 863 return; 864 } 865 866 stl_le_p(reg_state + R_CXL_RAS_UNC_ERR_STATUS, unc_err); 867 pcie_aer_inject_error(PCI_DEVICE(obj), &err); 868 869 return; 870 } 871 872 void qmp_cxl_inject_correctable_error(const char *path, CxlCorErrorType type, 873 Error **errp) 874 { 875 static PCIEAERErr err = {}; 876 Object *obj = object_resolve_path(path, NULL); 877 CXLType3Dev *ct3d; 878 uint32_t *reg_state; 879 uint32_t cor_err; 880 int cxl_err_type; 881 882 if (!obj) { 883 error_setg(errp, "Unable to resolve path"); 884 return; 885 } 886 if (!object_dynamic_cast(obj, TYPE_CXL_TYPE3)) { 887 error_setg(errp, "Path does not point to a CXL type 3 device"); 888 return; 889 } 890 891 err.status = PCI_ERR_COR_INTERNAL; 892 err.source_id = pci_requester_id(PCI_DEVICE(obj)); 893 err.flags = PCIE_AER_ERR_IS_CORRECTABLE; 894 895 ct3d = CXL_TYPE3(obj); 896 reg_state = ct3d->cxl_cstate.crb.cache_mem_registers; 897 cor_err = ldl_le_p(reg_state + R_CXL_RAS_COR_ERR_STATUS); 898 899 cxl_err_type = ct3d_qmp_cor_err_to_cxl(type); 900 if (cxl_err_type < 0) { 901 error_setg(errp, "Invalid COR error"); 902 return; 903 } 904 /* If the error is masked, nothting to do here */ 905 if (!((1 << cxl_err_type) & ~ldl_le_p(reg_state + R_CXL_RAS_COR_ERR_MASK))) { 906 return; 907 } 908 909 cor_err |= (1 << cxl_err_type); 910 stl_le_p(reg_state + R_CXL_RAS_COR_ERR_STATUS, cor_err); 911 912 pcie_aer_inject_error(PCI_DEVICE(obj), &err); 913 } 914 915 static void ct3_class_init(ObjectClass *oc, void *data) 916 { 917 DeviceClass *dc = DEVICE_CLASS(oc); 918 PCIDeviceClass *pc = PCI_DEVICE_CLASS(oc); 919 CXLType3Class *cvc = CXL_TYPE3_CLASS(oc); 920 921 pc->realize = ct3_realize; 922 pc->exit = ct3_exit; 923 pc->class_id = PCI_CLASS_MEMORY_CXL; 924 pc->vendor_id = PCI_VENDOR_ID_INTEL; 925 pc->device_id = 0xd93; /* LVF for now */ 926 pc->revision = 1; 927 928 pc->config_write = ct3d_config_write; 929 pc->config_read = ct3d_config_read; 930 931 set_bit(DEVICE_CATEGORY_STORAGE, dc->categories); 932 dc->desc = "CXL PMEM Device (Type 3)"; 933 dc->reset = ct3d_reset; 934 device_class_set_props(dc, ct3_props); 935 936 cvc->get_lsa_size = get_lsa_size; 937 cvc->get_lsa = get_lsa; 938 cvc->set_lsa = set_lsa; 939 } 940 941 static const TypeInfo ct3d_info = { 942 .name = TYPE_CXL_TYPE3, 943 .parent = TYPE_PCI_DEVICE, 944 .class_size = sizeof(struct CXLType3Class), 945 .class_init = ct3_class_init, 946 .instance_size = sizeof(CXLType3Dev), 947 .interfaces = (InterfaceInfo[]) { 948 { INTERFACE_CXL_DEVICE }, 949 { INTERFACE_PCIE_DEVICE }, 950 {} 951 }, 952 }; 953 954 static void ct3d_registers(void) 955 { 956 type_register_static(&ct3d_info); 957 } 958 959 type_init(ct3d_registers); 960