1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/pci.h> 3 #include <linux/module.h> 4 #include <linux/slab.h> 5 #include <linux/ioport.h> 6 #include <linux/wait.h> 7 8 #include "pci.h" 9 10 /* 11 * This interrupt-safe spinlock protects all accesses to PCI 12 * configuration space. 13 */ 14 15 DEFINE_RAW_SPINLOCK(pci_lock); 16 17 /* 18 * Wrappers for all PCI configuration access functions. They just check 19 * alignment, do locking and call the low-level functions pointed to 20 * by pci_dev->ops. 21 */ 22 23 #define PCI_byte_BAD 0 24 #define PCI_word_BAD (pos & 1) 25 #define PCI_dword_BAD (pos & 3) 26 27 #ifdef CONFIG_PCI_LOCKLESS_CONFIG 28 # define pci_lock_config(f) do { (void)(f); } while (0) 29 # define pci_unlock_config(f) do { (void)(f); } while (0) 30 #else 31 # define pci_lock_config(f) raw_spin_lock_irqsave(&pci_lock, f) 32 # define pci_unlock_config(f) raw_spin_unlock_irqrestore(&pci_lock, f) 33 #endif 34 35 #define PCI_OP_READ(size, type, len) \ 36 int noinline pci_bus_read_config_##size \ 37 (struct pci_bus *bus, unsigned int devfn, int pos, type *value) \ 38 { \ 39 int res; \ 40 unsigned long flags; \ 41 u32 data = 0; \ 42 if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER; \ 43 pci_lock_config(flags); \ 44 res = bus->ops->read(bus, devfn, pos, len, &data); \ 45 if (res) \ 46 PCI_SET_ERROR_RESPONSE(value); \ 47 else \ 48 *value = (type)data; \ 49 pci_unlock_config(flags); \ 50 return res; \ 51 } 52 53 #define PCI_OP_WRITE(size, type, len) \ 54 int noinline pci_bus_write_config_##size \ 55 (struct pci_bus *bus, unsigned int devfn, int pos, type value) \ 56 { \ 57 int res; \ 58 unsigned long flags; \ 59 if (PCI_##size##_BAD) return PCIBIOS_BAD_REGISTER_NUMBER; \ 60 pci_lock_config(flags); \ 61 res = bus->ops->write(bus, devfn, pos, len, value); \ 62 pci_unlock_config(flags); \ 63 return res; \ 64 } 65 66 PCI_OP_READ(byte, u8, 1) 67 PCI_OP_READ(word, u16, 2) 68 PCI_OP_READ(dword, u32, 4) 69 PCI_OP_WRITE(byte, u8, 1) 70 PCI_OP_WRITE(word, u16, 2) 71 PCI_OP_WRITE(dword, u32, 4) 72 73 EXPORT_SYMBOL(pci_bus_read_config_byte); 74 EXPORT_SYMBOL(pci_bus_read_config_word); 75 EXPORT_SYMBOL(pci_bus_read_config_dword); 76 EXPORT_SYMBOL(pci_bus_write_config_byte); 77 EXPORT_SYMBOL(pci_bus_write_config_word); 78 EXPORT_SYMBOL(pci_bus_write_config_dword); 79 80 int pci_generic_config_read(struct pci_bus *bus, unsigned int devfn, 81 int where, int size, u32 *val) 82 { 83 void __iomem *addr; 84 85 addr = bus->ops->map_bus(bus, devfn, where); 86 if (!addr) 87 return PCIBIOS_DEVICE_NOT_FOUND; 88 89 if (size == 1) 90 *val = readb(addr); 91 else if (size == 2) 92 *val = readw(addr); 93 else 94 *val = readl(addr); 95 96 return PCIBIOS_SUCCESSFUL; 97 } 98 EXPORT_SYMBOL_GPL(pci_generic_config_read); 99 100 int pci_generic_config_write(struct pci_bus *bus, unsigned int devfn, 101 int where, int size, u32 val) 102 { 103 void __iomem *addr; 104 105 addr = bus->ops->map_bus(bus, devfn, where); 106 if (!addr) 107 return PCIBIOS_DEVICE_NOT_FOUND; 108 109 if (size == 1) 110 writeb(val, addr); 111 else if (size == 2) 112 writew(val, addr); 113 else 114 writel(val, addr); 115 116 return PCIBIOS_SUCCESSFUL; 117 } 118 EXPORT_SYMBOL_GPL(pci_generic_config_write); 119 120 int pci_generic_config_read32(struct pci_bus *bus, unsigned int devfn, 121 int where, int size, u32 *val) 122 { 123 void __iomem *addr; 124 125 addr = bus->ops->map_bus(bus, devfn, where & ~0x3); 126 if (!addr) 127 return PCIBIOS_DEVICE_NOT_FOUND; 128 129 *val = readl(addr); 130 131 if (size <= 2) 132 *val = (*val >> (8 * (where & 3))) & ((1 << (size * 8)) - 1); 133 134 return PCIBIOS_SUCCESSFUL; 135 } 136 EXPORT_SYMBOL_GPL(pci_generic_config_read32); 137 138 int pci_generic_config_write32(struct pci_bus *bus, unsigned int devfn, 139 int where, int size, u32 val) 140 { 141 void __iomem *addr; 142 u32 mask, tmp; 143 144 addr = bus->ops->map_bus(bus, devfn, where & ~0x3); 145 if (!addr) 146 return PCIBIOS_DEVICE_NOT_FOUND; 147 148 if (size == 4) { 149 writel(val, addr); 150 return PCIBIOS_SUCCESSFUL; 151 } 152 153 /* 154 * In general, hardware that supports only 32-bit writes on PCI is 155 * not spec-compliant. For example, software may perform a 16-bit 156 * write. If the hardware only supports 32-bit accesses, we must 157 * do a 32-bit read, merge in the 16 bits we intend to write, 158 * followed by a 32-bit write. If the 16 bits we *don't* intend to 159 * write happen to have any RW1C (write-one-to-clear) bits set, we 160 * just inadvertently cleared something we shouldn't have. 161 */ 162 if (!bus->unsafe_warn) { 163 dev_warn(&bus->dev, "%d-byte config write to %04x:%02x:%02x.%d offset %#x may corrupt adjacent RW1C bits\n", 164 size, pci_domain_nr(bus), bus->number, 165 PCI_SLOT(devfn), PCI_FUNC(devfn), where); 166 bus->unsafe_warn = 1; 167 } 168 169 mask = ~(((1 << (size * 8)) - 1) << ((where & 0x3) * 8)); 170 tmp = readl(addr) & mask; 171 tmp |= val << ((where & 0x3) * 8); 172 writel(tmp, addr); 173 174 return PCIBIOS_SUCCESSFUL; 175 } 176 EXPORT_SYMBOL_GPL(pci_generic_config_write32); 177 178 /** 179 * pci_bus_set_ops - Set raw operations of pci bus 180 * @bus: pci bus struct 181 * @ops: new raw operations 182 * 183 * Return previous raw operations 184 */ 185 struct pci_ops *pci_bus_set_ops(struct pci_bus *bus, struct pci_ops *ops) 186 { 187 struct pci_ops *old_ops; 188 unsigned long flags; 189 190 raw_spin_lock_irqsave(&pci_lock, flags); 191 old_ops = bus->ops; 192 bus->ops = ops; 193 raw_spin_unlock_irqrestore(&pci_lock, flags); 194 return old_ops; 195 } 196 EXPORT_SYMBOL(pci_bus_set_ops); 197 198 /* 199 * The following routines are to prevent the user from accessing PCI config 200 * space when it's unsafe to do so. Some devices require this during BIST and 201 * we're required to prevent it during D-state transitions. 202 * 203 * We have a bit per device to indicate it's blocked and a global wait queue 204 * for callers to sleep on until devices are unblocked. 205 */ 206 static DECLARE_WAIT_QUEUE_HEAD(pci_cfg_wait); 207 208 static noinline void pci_wait_cfg(struct pci_dev *dev) 209 __must_hold(&pci_lock) 210 { 211 do { 212 raw_spin_unlock_irq(&pci_lock); 213 wait_event(pci_cfg_wait, !dev->block_cfg_access); 214 raw_spin_lock_irq(&pci_lock); 215 } while (dev->block_cfg_access); 216 } 217 218 /* Returns 0 on success, negative values indicate error. */ 219 #define PCI_USER_READ_CONFIG(size, type) \ 220 int pci_user_read_config_##size \ 221 (struct pci_dev *dev, int pos, type *val) \ 222 { \ 223 int ret = PCIBIOS_SUCCESSFUL; \ 224 u32 data = -1; \ 225 if (PCI_##size##_BAD) \ 226 return -EINVAL; \ 227 raw_spin_lock_irq(&pci_lock); \ 228 if (unlikely(dev->block_cfg_access)) \ 229 pci_wait_cfg(dev); \ 230 ret = dev->bus->ops->read(dev->bus, dev->devfn, \ 231 pos, sizeof(type), &data); \ 232 raw_spin_unlock_irq(&pci_lock); \ 233 if (ret) \ 234 PCI_SET_ERROR_RESPONSE(val); \ 235 else \ 236 *val = (type)data; \ 237 return pcibios_err_to_errno(ret); \ 238 } \ 239 EXPORT_SYMBOL_GPL(pci_user_read_config_##size); 240 241 /* Returns 0 on success, negative values indicate error. */ 242 #define PCI_USER_WRITE_CONFIG(size, type) \ 243 int pci_user_write_config_##size \ 244 (struct pci_dev *dev, int pos, type val) \ 245 { \ 246 int ret = PCIBIOS_SUCCESSFUL; \ 247 if (PCI_##size##_BAD) \ 248 return -EINVAL; \ 249 raw_spin_lock_irq(&pci_lock); \ 250 if (unlikely(dev->block_cfg_access)) \ 251 pci_wait_cfg(dev); \ 252 ret = dev->bus->ops->write(dev->bus, dev->devfn, \ 253 pos, sizeof(type), val); \ 254 raw_spin_unlock_irq(&pci_lock); \ 255 return pcibios_err_to_errno(ret); \ 256 } \ 257 EXPORT_SYMBOL_GPL(pci_user_write_config_##size); 258 259 PCI_USER_READ_CONFIG(byte, u8) 260 PCI_USER_READ_CONFIG(word, u16) 261 PCI_USER_READ_CONFIG(dword, u32) 262 PCI_USER_WRITE_CONFIG(byte, u8) 263 PCI_USER_WRITE_CONFIG(word, u16) 264 PCI_USER_WRITE_CONFIG(dword, u32) 265 266 /** 267 * pci_cfg_access_lock - Lock PCI config reads/writes 268 * @dev: pci device struct 269 * 270 * When access is locked, any userspace reads or writes to config 271 * space and concurrent lock requests will sleep until access is 272 * allowed via pci_cfg_access_unlock() again. 273 */ 274 void pci_cfg_access_lock(struct pci_dev *dev) 275 { 276 might_sleep(); 277 278 raw_spin_lock_irq(&pci_lock); 279 if (dev->block_cfg_access) 280 pci_wait_cfg(dev); 281 dev->block_cfg_access = 1; 282 raw_spin_unlock_irq(&pci_lock); 283 } 284 EXPORT_SYMBOL_GPL(pci_cfg_access_lock); 285 286 /** 287 * pci_cfg_access_trylock - try to lock PCI config reads/writes 288 * @dev: pci device struct 289 * 290 * Same as pci_cfg_access_lock, but will return 0 if access is 291 * already locked, 1 otherwise. This function can be used from 292 * atomic contexts. 293 */ 294 bool pci_cfg_access_trylock(struct pci_dev *dev) 295 { 296 unsigned long flags; 297 bool locked = true; 298 299 raw_spin_lock_irqsave(&pci_lock, flags); 300 if (dev->block_cfg_access) 301 locked = false; 302 else 303 dev->block_cfg_access = 1; 304 raw_spin_unlock_irqrestore(&pci_lock, flags); 305 306 return locked; 307 } 308 EXPORT_SYMBOL_GPL(pci_cfg_access_trylock); 309 310 /** 311 * pci_cfg_access_unlock - Unlock PCI config reads/writes 312 * @dev: pci device struct 313 * 314 * This function allows PCI config accesses to resume. 315 */ 316 void pci_cfg_access_unlock(struct pci_dev *dev) 317 { 318 unsigned long flags; 319 320 raw_spin_lock_irqsave(&pci_lock, flags); 321 322 /* 323 * This indicates a problem in the caller, but we don't need 324 * to kill them, unlike a double-block above. 325 */ 326 WARN_ON(!dev->block_cfg_access); 327 328 dev->block_cfg_access = 0; 329 raw_spin_unlock_irqrestore(&pci_lock, flags); 330 331 wake_up_all(&pci_cfg_wait); 332 } 333 EXPORT_SYMBOL_GPL(pci_cfg_access_unlock); 334 335 static inline int pcie_cap_version(const struct pci_dev *dev) 336 { 337 return pcie_caps_reg(dev) & PCI_EXP_FLAGS_VERS; 338 } 339 340 bool pcie_cap_has_lnkctl(const struct pci_dev *dev) 341 { 342 int type = pci_pcie_type(dev); 343 344 return type == PCI_EXP_TYPE_ENDPOINT || 345 type == PCI_EXP_TYPE_LEG_END || 346 type == PCI_EXP_TYPE_ROOT_PORT || 347 type == PCI_EXP_TYPE_UPSTREAM || 348 type == PCI_EXP_TYPE_DOWNSTREAM || 349 type == PCI_EXP_TYPE_PCI_BRIDGE || 350 type == PCI_EXP_TYPE_PCIE_BRIDGE; 351 } 352 353 static inline bool pcie_cap_has_sltctl(const struct pci_dev *dev) 354 { 355 return pcie_downstream_port(dev) && 356 pcie_caps_reg(dev) & PCI_EXP_FLAGS_SLOT; 357 } 358 359 bool pcie_cap_has_rtctl(const struct pci_dev *dev) 360 { 361 int type = pci_pcie_type(dev); 362 363 return type == PCI_EXP_TYPE_ROOT_PORT || 364 type == PCI_EXP_TYPE_RC_EC; 365 } 366 367 static bool pcie_capability_reg_implemented(struct pci_dev *dev, int pos) 368 { 369 if (!pci_is_pcie(dev)) 370 return false; 371 372 switch (pos) { 373 case PCI_EXP_FLAGS: 374 return true; 375 case PCI_EXP_DEVCAP: 376 case PCI_EXP_DEVCTL: 377 case PCI_EXP_DEVSTA: 378 return true; 379 case PCI_EXP_LNKCAP: 380 case PCI_EXP_LNKCTL: 381 case PCI_EXP_LNKSTA: 382 return pcie_cap_has_lnkctl(dev); 383 case PCI_EXP_SLTCAP: 384 case PCI_EXP_SLTCTL: 385 case PCI_EXP_SLTSTA: 386 return pcie_cap_has_sltctl(dev); 387 case PCI_EXP_RTCTL: 388 case PCI_EXP_RTCAP: 389 case PCI_EXP_RTSTA: 390 return pcie_cap_has_rtctl(dev); 391 case PCI_EXP_DEVCAP2: 392 case PCI_EXP_DEVCTL2: 393 case PCI_EXP_LNKCAP2: 394 case PCI_EXP_LNKCTL2: 395 case PCI_EXP_LNKSTA2: 396 return pcie_cap_version(dev) > 1; 397 default: 398 return false; 399 } 400 } 401 402 /* 403 * Note that these accessor functions are only for the "PCI Express 404 * Capability" (see PCIe spec r3.0, sec 7.8). They do not apply to the 405 * other "PCI Express Extended Capabilities" (AER, VC, ACS, MFVC, etc.) 406 */ 407 int pcie_capability_read_word(struct pci_dev *dev, int pos, u16 *val) 408 { 409 int ret; 410 411 *val = 0; 412 if (pos & 1) 413 return PCIBIOS_BAD_REGISTER_NUMBER; 414 415 if (pcie_capability_reg_implemented(dev, pos)) { 416 ret = pci_read_config_word(dev, pci_pcie_cap(dev) + pos, val); 417 /* 418 * Reset *val to 0 if pci_read_config_word() fails; it may 419 * have been written as 0xFFFF (PCI_ERROR_RESPONSE) if the 420 * config read failed on PCI. 421 */ 422 if (ret) 423 *val = 0; 424 return ret; 425 } 426 427 /* 428 * For Functions that do not implement the Slot Capabilities, 429 * Slot Status, and Slot Control registers, these spaces must 430 * be hardwired to 0b, with the exception of the Presence Detect 431 * State bit in the Slot Status register of Downstream Ports, 432 * which must be hardwired to 1b. (PCIe Base Spec 3.0, sec 7.8) 433 */ 434 if (pci_is_pcie(dev) && pcie_downstream_port(dev) && 435 pos == PCI_EXP_SLTSTA) 436 *val = PCI_EXP_SLTSTA_PDS; 437 438 return 0; 439 } 440 EXPORT_SYMBOL(pcie_capability_read_word); 441 442 int pcie_capability_read_dword(struct pci_dev *dev, int pos, u32 *val) 443 { 444 int ret; 445 446 *val = 0; 447 if (pos & 3) 448 return PCIBIOS_BAD_REGISTER_NUMBER; 449 450 if (pcie_capability_reg_implemented(dev, pos)) { 451 ret = pci_read_config_dword(dev, pci_pcie_cap(dev) + pos, val); 452 /* 453 * Reset *val to 0 if pci_read_config_dword() fails; it may 454 * have been written as 0xFFFFFFFF (PCI_ERROR_RESPONSE) if 455 * the config read failed on PCI. 456 */ 457 if (ret) 458 *val = 0; 459 return ret; 460 } 461 462 if (pci_is_pcie(dev) && pcie_downstream_port(dev) && 463 pos == PCI_EXP_SLTSTA) 464 *val = PCI_EXP_SLTSTA_PDS; 465 466 return 0; 467 } 468 EXPORT_SYMBOL(pcie_capability_read_dword); 469 470 int pcie_capability_write_word(struct pci_dev *dev, int pos, u16 val) 471 { 472 if (pos & 1) 473 return PCIBIOS_BAD_REGISTER_NUMBER; 474 475 if (!pcie_capability_reg_implemented(dev, pos)) 476 return 0; 477 478 return pci_write_config_word(dev, pci_pcie_cap(dev) + pos, val); 479 } 480 EXPORT_SYMBOL(pcie_capability_write_word); 481 482 int pcie_capability_write_dword(struct pci_dev *dev, int pos, u32 val) 483 { 484 if (pos & 3) 485 return PCIBIOS_BAD_REGISTER_NUMBER; 486 487 if (!pcie_capability_reg_implemented(dev, pos)) 488 return 0; 489 490 return pci_write_config_dword(dev, pci_pcie_cap(dev) + pos, val); 491 } 492 EXPORT_SYMBOL(pcie_capability_write_dword); 493 494 int pcie_capability_clear_and_set_word(struct pci_dev *dev, int pos, 495 u16 clear, u16 set) 496 { 497 int ret; 498 u16 val; 499 500 ret = pcie_capability_read_word(dev, pos, &val); 501 if (!ret) { 502 val &= ~clear; 503 val |= set; 504 ret = pcie_capability_write_word(dev, pos, val); 505 } 506 507 return ret; 508 } 509 EXPORT_SYMBOL(pcie_capability_clear_and_set_word); 510 511 int pcie_capability_clear_and_set_dword(struct pci_dev *dev, int pos, 512 u32 clear, u32 set) 513 { 514 int ret; 515 u32 val; 516 517 ret = pcie_capability_read_dword(dev, pos, &val); 518 if (!ret) { 519 val &= ~clear; 520 val |= set; 521 ret = pcie_capability_write_dword(dev, pos, val); 522 } 523 524 return ret; 525 } 526 EXPORT_SYMBOL(pcie_capability_clear_and_set_dword); 527 528 int pci_read_config_byte(const struct pci_dev *dev, int where, u8 *val) 529 { 530 if (pci_dev_is_disconnected(dev)) { 531 PCI_SET_ERROR_RESPONSE(val); 532 return PCIBIOS_DEVICE_NOT_FOUND; 533 } 534 return pci_bus_read_config_byte(dev->bus, dev->devfn, where, val); 535 } 536 EXPORT_SYMBOL(pci_read_config_byte); 537 538 int pci_read_config_word(const struct pci_dev *dev, int where, u16 *val) 539 { 540 if (pci_dev_is_disconnected(dev)) { 541 PCI_SET_ERROR_RESPONSE(val); 542 return PCIBIOS_DEVICE_NOT_FOUND; 543 } 544 return pci_bus_read_config_word(dev->bus, dev->devfn, where, val); 545 } 546 EXPORT_SYMBOL(pci_read_config_word); 547 548 int pci_read_config_dword(const struct pci_dev *dev, int where, 549 u32 *val) 550 { 551 if (pci_dev_is_disconnected(dev)) { 552 PCI_SET_ERROR_RESPONSE(val); 553 return PCIBIOS_DEVICE_NOT_FOUND; 554 } 555 return pci_bus_read_config_dword(dev->bus, dev->devfn, where, val); 556 } 557 EXPORT_SYMBOL(pci_read_config_dword); 558 559 int pci_write_config_byte(const struct pci_dev *dev, int where, u8 val) 560 { 561 if (pci_dev_is_disconnected(dev)) 562 return PCIBIOS_DEVICE_NOT_FOUND; 563 return pci_bus_write_config_byte(dev->bus, dev->devfn, where, val); 564 } 565 EXPORT_SYMBOL(pci_write_config_byte); 566 567 int pci_write_config_word(const struct pci_dev *dev, int where, u16 val) 568 { 569 if (pci_dev_is_disconnected(dev)) 570 return PCIBIOS_DEVICE_NOT_FOUND; 571 return pci_bus_write_config_word(dev->bus, dev->devfn, where, val); 572 } 573 EXPORT_SYMBOL(pci_write_config_word); 574 575 int pci_write_config_dword(const struct pci_dev *dev, int where, 576 u32 val) 577 { 578 if (pci_dev_is_disconnected(dev)) 579 return PCIBIOS_DEVICE_NOT_FOUND; 580 return pci_bus_write_config_dword(dev->bus, dev->devfn, where, val); 581 } 582 EXPORT_SYMBOL(pci_write_config_dword); 583