1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 1997, Stefan Esser <se@freebsd.org> 5 * Copyright (c) 2000, Michael Smith <msmith@freebsd.org> 6 * Copyright (c) 2000, BSDi 7 * Copyright (c) 2004, Scott Long <scottl@freebsd.org> 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice unmodified, this list of conditions, and the following 15 * disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 21 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 22 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 23 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 24 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 25 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 29 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/bus.h> 38 #include <sys/lock.h> 39 #include <sys/kernel.h> 40 #include <sys/mutex.h> 41 #include <sys/malloc.h> 42 #include <sys/queue.h> 43 #include <sys/sysctl.h> 44 #include <dev/pci/pcivar.h> 45 #include <dev/pci/pcireg.h> 46 #include <machine/pci_cfgreg.h> 47 #include <machine/pc/bios.h> 48 49 #include <vm/vm.h> 50 #include <vm/vm_param.h> 51 #include <vm/vm_kern.h> 52 #include <vm/vm_extern.h> 53 #include <vm/pmap.h> 54 55 #define PRVERB(a) do { \ 56 if (bootverbose) \ 57 printf a ; \ 58 } while(0) 59 60 #define PCIE_CACHE 8 61 struct pcie_cfg_elem { 62 TAILQ_ENTRY(pcie_cfg_elem) elem; 63 vm_offset_t vapage; 64 vm_paddr_t papage; 65 }; 66 67 SYSCTL_DECL(_hw_pci); 68 69 static TAILQ_HEAD(pcie_cfg_list, pcie_cfg_elem) pcie_list[MAXCPU]; 70 static uint64_t pcie_base; 71 static int pcie_minbus, pcie_maxbus; 72 static uint32_t pcie_badslots; 73 int cfgmech; 74 static int devmax; 75 static struct mtx pcicfg_mtx; 76 static int mcfg_enable = 1; 77 SYSCTL_INT(_hw_pci, OID_AUTO, mcfg, CTLFLAG_RDTUN, &mcfg_enable, 0, 78 "Enable support for PCI-e memory mapped config access"); 79 80 static uint32_t pci_docfgregread(int bus, int slot, int func, int reg, 81 int bytes); 82 static int pcireg_cfgread(int bus, int slot, int func, int reg, int bytes); 83 static void pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes); 84 static int pcireg_cfgopen(void); 85 static int pciereg_cfgread(int bus, unsigned slot, unsigned func, 86 unsigned reg, unsigned bytes); 87 static void pciereg_cfgwrite(int bus, unsigned slot, unsigned func, 88 unsigned reg, int data, unsigned bytes); 89 90 /* 91 * Some BIOS writers seem to want to ignore the spec and put 92 * 0 in the intline rather than 255 to indicate none. Some use 93 * numbers in the range 128-254 to indicate something strange and 94 * apparently undocumented anywhere. Assume these are completely bogus 95 * and map them to 255, which means "none". 96 */ 97 static __inline int 98 pci_i386_map_intline(int line) 99 { 100 if (line == 0 || line >= 128) 101 return (PCI_INVALID_IRQ); 102 return (line); 103 } 104 105 static u_int16_t 106 pcibios_get_version(void) 107 { 108 struct bios_regs args; 109 110 if (PCIbios.ventry == 0) { 111 PRVERB(("pcibios: No call entry point\n")); 112 return (0); 113 } 114 args.eax = PCIBIOS_BIOS_PRESENT; 115 if (bios32(&args, PCIbios.ventry, GSEL(GCODE_SEL, SEL_KPL))) { 116 PRVERB(("pcibios: BIOS_PRESENT call failed\n")); 117 return (0); 118 } 119 if (args.edx != 0x20494350) { 120 PRVERB(("pcibios: BIOS_PRESENT didn't return 'PCI ' in edx\n")); 121 return (0); 122 } 123 return (args.ebx & 0xffff); 124 } 125 126 /* 127 * Initialise access to PCI configuration space 128 */ 129 int 130 pci_cfgregopen(void) 131 { 132 uint16_t v; 133 static int opened = 0; 134 135 if (opened) 136 return (1); 137 138 if (cfgmech == CFGMECH_NONE && pcireg_cfgopen() == 0) 139 return (0); 140 141 v = pcibios_get_version(); 142 if (v > 0) 143 PRVERB(("pcibios: BIOS version %x.%02x\n", (v & 0xff00) >> 8, 144 v & 0xff)); 145 mtx_init(&pcicfg_mtx, "pcicfg", NULL, MTX_SPIN); 146 opened = 1; 147 148 /* $PIR requires PCI BIOS 2.10 or greater. */ 149 if (v >= 0x0210) 150 pci_pir_open(); 151 152 return (1); 153 } 154 155 static uint32_t 156 pci_docfgregread(int bus, int slot, int func, int reg, int bytes) 157 { 158 159 if (cfgmech == CFGMECH_PCIE && 160 (bus >= pcie_minbus && bus <= pcie_maxbus) && 161 (bus != 0 || !(1 << slot & pcie_badslots))) 162 return (pciereg_cfgread(bus, slot, func, reg, bytes)); 163 else 164 return (pcireg_cfgread(bus, slot, func, reg, bytes)); 165 } 166 167 /* 168 * Read configuration space register 169 */ 170 u_int32_t 171 pci_cfgregread(int bus, int slot, int func, int reg, int bytes) 172 { 173 uint32_t line; 174 175 /* 176 * Some BIOS writers seem to want to ignore the spec and put 177 * 0 in the intline rather than 255 to indicate none. The rest of 178 * the code uses 255 as an invalid IRQ. 179 */ 180 if (reg == PCIR_INTLINE && bytes == 1) { 181 line = pci_docfgregread(bus, slot, func, PCIR_INTLINE, 1); 182 return (pci_i386_map_intline(line)); 183 } 184 return (pci_docfgregread(bus, slot, func, reg, bytes)); 185 } 186 187 /* 188 * Write configuration space register 189 */ 190 void 191 pci_cfgregwrite(int bus, int slot, int func, int reg, u_int32_t data, int bytes) 192 { 193 194 if (cfgmech == CFGMECH_PCIE && 195 (bus >= pcie_minbus && bus <= pcie_maxbus) && 196 (bus != 0 || !(1 << slot & pcie_badslots))) 197 pciereg_cfgwrite(bus, slot, func, reg, data, bytes); 198 else 199 pcireg_cfgwrite(bus, slot, func, reg, data, bytes); 200 } 201 202 /* 203 * Configuration space access using direct register operations 204 */ 205 206 /* enable configuration space accesses and return data port address */ 207 static int 208 pci_cfgenable(unsigned bus, unsigned slot, unsigned func, int reg, int bytes) 209 { 210 int dataport = 0; 211 212 if (bus <= PCI_BUSMAX 213 && slot < devmax 214 && func <= PCI_FUNCMAX 215 && (unsigned)reg <= PCI_REGMAX 216 && bytes != 3 217 && (unsigned)bytes <= 4 218 && (reg & (bytes - 1)) == 0) { 219 switch (cfgmech) { 220 case CFGMECH_PCIE: 221 case CFGMECH_1: 222 outl(CONF1_ADDR_PORT, (1U << 31) 223 | (bus << 16) | (slot << 11) 224 | (func << 8) | (reg & ~0x03)); 225 dataport = CONF1_DATA_PORT + (reg & 0x03); 226 break; 227 case CFGMECH_2: 228 outb(CONF2_ENABLE_PORT, 0xf0 | (func << 1)); 229 outb(CONF2_FORWARD_PORT, bus); 230 dataport = 0xc000 | (slot << 8) | reg; 231 break; 232 } 233 } 234 return (dataport); 235 } 236 237 /* disable configuration space accesses */ 238 static void 239 pci_cfgdisable(void) 240 { 241 switch (cfgmech) { 242 case CFGMECH_PCIE: 243 case CFGMECH_1: 244 /* 245 * Do nothing for the config mechanism 1 case. 246 * Writing a 0 to the address port can apparently 247 * confuse some bridges and cause spurious 248 * access failures. 249 */ 250 break; 251 case CFGMECH_2: 252 outb(CONF2_ENABLE_PORT, 0); 253 break; 254 } 255 } 256 257 static int 258 pcireg_cfgread(int bus, int slot, int func, int reg, int bytes) 259 { 260 int data = -1; 261 int port; 262 263 mtx_lock_spin(&pcicfg_mtx); 264 port = pci_cfgenable(bus, slot, func, reg, bytes); 265 if (port != 0) { 266 switch (bytes) { 267 case 1: 268 data = inb(port); 269 break; 270 case 2: 271 data = inw(port); 272 break; 273 case 4: 274 data = inl(port); 275 break; 276 } 277 pci_cfgdisable(); 278 } 279 mtx_unlock_spin(&pcicfg_mtx); 280 return (data); 281 } 282 283 static void 284 pcireg_cfgwrite(int bus, int slot, int func, int reg, int data, int bytes) 285 { 286 int port; 287 288 mtx_lock_spin(&pcicfg_mtx); 289 port = pci_cfgenable(bus, slot, func, reg, bytes); 290 if (port != 0) { 291 switch (bytes) { 292 case 1: 293 outb(port, data); 294 break; 295 case 2: 296 outw(port, data); 297 break; 298 case 4: 299 outl(port, data); 300 break; 301 } 302 pci_cfgdisable(); 303 } 304 mtx_unlock_spin(&pcicfg_mtx); 305 } 306 307 /* check whether the configuration mechanism has been correctly identified */ 308 static int 309 pci_cfgcheck(int maxdev) 310 { 311 uint32_t id, class; 312 uint8_t header; 313 uint8_t device; 314 int port; 315 316 if (bootverbose) 317 printf("pci_cfgcheck:\tdevice "); 318 319 for (device = 0; device < maxdev; device++) { 320 if (bootverbose) 321 printf("%d ", device); 322 323 port = pci_cfgenable(0, device, 0, 0, 4); 324 id = inl(port); 325 if (id == 0 || id == 0xffffffff) 326 continue; 327 328 port = pci_cfgenable(0, device, 0, 8, 4); 329 class = inl(port) >> 8; 330 if (bootverbose) 331 printf("[class=%06x] ", class); 332 if (class == 0 || (class & 0xf870ff) != 0) 333 continue; 334 335 port = pci_cfgenable(0, device, 0, 14, 1); 336 header = inb(port); 337 if (bootverbose) 338 printf("[hdr=%02x] ", header); 339 if ((header & 0x7e) != 0) 340 continue; 341 342 if (bootverbose) 343 printf("is there (id=%08x)\n", id); 344 345 pci_cfgdisable(); 346 return (1); 347 } 348 if (bootverbose) 349 printf("-- nothing found\n"); 350 351 pci_cfgdisable(); 352 return (0); 353 } 354 355 static int 356 pcireg_cfgopen(void) 357 { 358 uint32_t mode1res, oldval1; 359 uint8_t mode2res, oldval2; 360 361 /* Check for type #1 first. */ 362 oldval1 = inl(CONF1_ADDR_PORT); 363 364 if (bootverbose) { 365 printf("pci_open(1):\tmode 1 addr port (0x0cf8) is 0x%08x\n", 366 oldval1); 367 } 368 369 cfgmech = CFGMECH_1; 370 devmax = 32; 371 372 outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK); 373 DELAY(1); 374 mode1res = inl(CONF1_ADDR_PORT); 375 outl(CONF1_ADDR_PORT, oldval1); 376 377 if (bootverbose) 378 printf("pci_open(1a):\tmode1res=0x%08x (0x%08lx)\n", mode1res, 379 CONF1_ENABLE_CHK); 380 381 if (mode1res) { 382 if (pci_cfgcheck(32)) 383 return (cfgmech); 384 } 385 386 outl(CONF1_ADDR_PORT, CONF1_ENABLE_CHK1); 387 mode1res = inl(CONF1_ADDR_PORT); 388 outl(CONF1_ADDR_PORT, oldval1); 389 390 if (bootverbose) 391 printf("pci_open(1b):\tmode1res=0x%08x (0x%08lx)\n", mode1res, 392 CONF1_ENABLE_CHK1); 393 394 if ((mode1res & CONF1_ENABLE_MSK1) == CONF1_ENABLE_RES1) { 395 if (pci_cfgcheck(32)) 396 return (cfgmech); 397 } 398 399 /* Type #1 didn't work, so try type #2. */ 400 oldval2 = inb(CONF2_ENABLE_PORT); 401 402 if (bootverbose) { 403 printf("pci_open(2):\tmode 2 enable port (0x0cf8) is 0x%02x\n", 404 oldval2); 405 } 406 407 if ((oldval2 & 0xf0) == 0) { 408 409 cfgmech = CFGMECH_2; 410 devmax = 16; 411 412 outb(CONF2_ENABLE_PORT, CONF2_ENABLE_CHK); 413 mode2res = inb(CONF2_ENABLE_PORT); 414 outb(CONF2_ENABLE_PORT, oldval2); 415 416 if (bootverbose) 417 printf("pci_open(2a):\tmode2res=0x%02x (0x%02x)\n", 418 mode2res, CONF2_ENABLE_CHK); 419 420 if (mode2res == CONF2_ENABLE_RES) { 421 if (bootverbose) 422 printf("pci_open(2a):\tnow trying mechanism 2\n"); 423 424 if (pci_cfgcheck(16)) 425 return (cfgmech); 426 } 427 } 428 429 /* Nothing worked, so punt. */ 430 cfgmech = CFGMECH_NONE; 431 devmax = 0; 432 return (cfgmech); 433 } 434 435 int 436 pcie_cfgregopen(uint64_t base, uint8_t minbus, uint8_t maxbus) 437 { 438 struct pcie_cfg_list *pcielist; 439 struct pcie_cfg_elem *pcie_array, *elem; 440 #ifdef SMP 441 struct pcpu *pc; 442 #endif 443 vm_offset_t va; 444 uint32_t val1, val2; 445 int i, slot; 446 447 if (!mcfg_enable) 448 return (0); 449 450 if (minbus != 0) 451 return (0); 452 453 if (!pae_mode && base >= 0x100000000) { 454 if (bootverbose) 455 printf( 456 "PCI: Memory Mapped PCI configuration area base 0x%jx too high\n", 457 (uintmax_t)base); 458 return (0); 459 } 460 461 if (bootverbose) 462 printf("PCIe: Memory Mapped configuration base @ 0x%jx\n", 463 (uintmax_t)base); 464 465 #ifdef SMP 466 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) 467 #endif 468 { 469 470 pcie_array = malloc(sizeof(struct pcie_cfg_elem) * PCIE_CACHE, 471 M_DEVBUF, M_NOWAIT); 472 if (pcie_array == NULL) 473 return (0); 474 475 va = kva_alloc(PCIE_CACHE * PAGE_SIZE); 476 if (va == 0) { 477 free(pcie_array, M_DEVBUF); 478 return (0); 479 } 480 481 #ifdef SMP 482 pcielist = &pcie_list[pc->pc_cpuid]; 483 #else 484 pcielist = &pcie_list[0]; 485 #endif 486 TAILQ_INIT(pcielist); 487 for (i = 0; i < PCIE_CACHE; i++) { 488 elem = &pcie_array[i]; 489 elem->vapage = va + (i * PAGE_SIZE); 490 elem->papage = 0; 491 TAILQ_INSERT_HEAD(pcielist, elem, elem); 492 } 493 } 494 495 pcie_base = base; 496 pcie_minbus = minbus; 497 pcie_maxbus = maxbus; 498 cfgmech = CFGMECH_PCIE; 499 devmax = 32; 500 501 /* 502 * On some AMD systems, some of the devices on bus 0 are 503 * inaccessible using memory-mapped PCI config access. Walk 504 * bus 0 looking for such devices. For these devices, we will 505 * fall back to using type 1 config access instead. 506 */ 507 if (pci_cfgregopen() != 0) { 508 for (slot = 0; slot <= PCI_SLOTMAX; slot++) { 509 val1 = pcireg_cfgread(0, slot, 0, 0, 4); 510 if (val1 == 0xffffffff) 511 continue; 512 513 val2 = pciereg_cfgread(0, slot, 0, 0, 4); 514 if (val2 != val1) 515 pcie_badslots |= (1 << slot); 516 } 517 } 518 519 return (1); 520 } 521 522 #define PCIE_PADDR(base, reg, bus, slot, func) \ 523 ((base) + \ 524 ((((bus) & 0xff) << 20) | \ 525 (((slot) & 0x1f) << 15) | \ 526 (((func) & 0x7) << 12) | \ 527 ((reg) & 0xfff))) 528 529 static __inline vm_offset_t 530 pciereg_findaddr(int bus, unsigned slot, unsigned func, unsigned reg) 531 { 532 struct pcie_cfg_list *pcielist; 533 struct pcie_cfg_elem *elem; 534 vm_paddr_t pa, papage; 535 536 pa = PCIE_PADDR(pcie_base, reg, bus, slot, func); 537 papage = pa & ~PAGE_MASK; 538 539 /* 540 * Find an element in the cache that matches the physical page desired, 541 * or create a new mapping from the least recently used element. 542 * A very simple LRU algorithm is used here, does it need to be more 543 * efficient? 544 */ 545 pcielist = &pcie_list[PCPU_GET(cpuid)]; 546 TAILQ_FOREACH(elem, pcielist, elem) { 547 if (elem->papage == papage) 548 break; 549 } 550 551 if (elem == NULL) { 552 elem = TAILQ_LAST(pcielist, pcie_cfg_list); 553 if (elem->papage != 0) { 554 pmap_kremove(elem->vapage); 555 invlpg(elem->vapage); 556 } 557 pmap_kenter(elem->vapage, papage); 558 elem->papage = papage; 559 } 560 561 if (elem != TAILQ_FIRST(pcielist)) { 562 TAILQ_REMOVE(pcielist, elem, elem); 563 TAILQ_INSERT_HEAD(pcielist, elem, elem); 564 } 565 return (elem->vapage | (pa & PAGE_MASK)); 566 } 567 568 /* 569 * AMD BIOS And Kernel Developer's Guides for CPU families starting with 10h 570 * have a requirement that all accesses to the memory mapped PCI configuration 571 * space are done using AX class of registers. 572 * Since other vendors do not currently have any contradicting requirements 573 * the AMD access pattern is applied universally. 574 */ 575 576 static int 577 pciereg_cfgread(int bus, unsigned slot, unsigned func, unsigned reg, 578 unsigned bytes) 579 { 580 vm_offset_t va; 581 int data = -1; 582 583 if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX || 584 func > PCI_FUNCMAX || reg > PCIE_REGMAX) 585 return (-1); 586 587 critical_enter(); 588 va = pciereg_findaddr(bus, slot, func, reg); 589 590 switch (bytes) { 591 case 4: 592 __asm("movl %1, %0" : "=a" (data) 593 : "m" (*(volatile uint32_t *)va)); 594 break; 595 case 2: 596 __asm("movzwl %1, %0" : "=a" (data) 597 : "m" (*(volatile uint16_t *)va)); 598 break; 599 case 1: 600 __asm("movzbl %1, %0" : "=a" (data) 601 : "m" (*(volatile uint8_t *)va)); 602 break; 603 } 604 605 critical_exit(); 606 return (data); 607 } 608 609 static void 610 pciereg_cfgwrite(int bus, unsigned slot, unsigned func, unsigned reg, int data, 611 unsigned bytes) 612 { 613 vm_offset_t va; 614 615 if (bus < pcie_minbus || bus > pcie_maxbus || slot > PCI_SLOTMAX || 616 func > PCI_FUNCMAX || reg > PCIE_REGMAX) 617 return; 618 619 critical_enter(); 620 va = pciereg_findaddr(bus, slot, func, reg); 621 622 switch (bytes) { 623 case 4: 624 __asm("movl %1, %0" : "=m" (*(volatile uint32_t *)va) 625 : "a" (data)); 626 break; 627 case 2: 628 __asm("movw %1, %0" : "=m" (*(volatile uint16_t *)va) 629 : "a" ((uint16_t)data)); 630 break; 631 case 1: 632 __asm("movb %1, %0" : "=m" (*(volatile uint8_t *)va) 633 : "a" ((uint8_t)data)); 634 break; 635 } 636 637 critical_exit(); 638 } 639