1047043c2SRobert Mustacchi /* 2047043c2SRobert Mustacchi * This file and its contents are supplied under the terms of the 3047043c2SRobert Mustacchi * Common Development and Distribution License ("CDDL"), version 1.0. 4047043c2SRobert Mustacchi * You may only use this file in accordance with the terms of version 5047043c2SRobert Mustacchi * 1.0 of the CDDL. 6047043c2SRobert Mustacchi * 7047043c2SRobert Mustacchi * A full copy of the text of the CDDL should have accompanied this 8047043c2SRobert Mustacchi * source. A copy of the CDDL is also available via the Internet at 9047043c2SRobert Mustacchi * http://www.illumos.org/license/CDDL. 10047043c2SRobert Mustacchi */ 11047043c2SRobert Mustacchi 12047043c2SRobert Mustacchi /* 13047043c2SRobert Mustacchi * Copyright 2019, Joyent, Inc. 141e0464b8SRobert Mustacchi * Copyright 2024 Oxide Computer Company 15047043c2SRobert Mustacchi */ 16047043c2SRobert Mustacchi 17047043c2SRobert Mustacchi /* 18047043c2SRobert Mustacchi * Nexus Driver for AMD Zen family systems. The purpose of this driver is to 19047043c2SRobert Mustacchi * provide access to the following resources in a single, centralized fashion: 20047043c2SRobert Mustacchi * 21047043c2SRobert Mustacchi * - The per-chip Data Fabric 22047043c2SRobert Mustacchi * - The North Bridge 23047043c2SRobert Mustacchi * - The System Management Network (SMN) 24047043c2SRobert Mustacchi * 25047043c2SRobert Mustacchi * This is a nexus driver as once we have attached to all the requisite 26047043c2SRobert Mustacchi * components, we will enumerate child devices which consume this functionality. 27047043c2SRobert Mustacchi * 28047043c2SRobert Mustacchi * ------------------------ 29047043c2SRobert Mustacchi * Mapping Devices Together 30047043c2SRobert Mustacchi * ------------------------ 31047043c2SRobert Mustacchi * 32047043c2SRobert Mustacchi * The operating system needs to expose things like temperature sensors and DRAM 3319040b41SDan Cross * configuration registers in terms of things that are meaningful to the system 3419040b41SDan Cross * such as logical CPUs, cores, etc. This driver attaches to the PCI devices 3519040b41SDan Cross * that represent the northbridge, data fabrics, and dies. Note that there are 3619040b41SDan Cross * multiple northbridge and DF devices (one each per die) and this driver maps 3719040b41SDan Cross * all of these three things together. Unfortunately, this requires some 3819040b41SDan Cross * acrobatics as there is no direct way to map a northbridge to its 3919040b41SDan Cross * corresponding die. Instead, we map a CPU die to a data fabric PCI device and 4019040b41SDan Cross * a data fabric PCI device to a corresponding northbridge PCI device. This 4119040b41SDan Cross * transitive relationship allows us to map from between northbridge and die. 4219040b41SDan Cross * 4319040b41SDan Cross * As each data fabric device is attached, based on vendor and device portions 4419040b41SDan Cross * of the PCI ID, we add it to the DF stubs list in the global amdzen_t 4519040b41SDan Cross * structure, amdzen_data->azn_df_stubs. We must now map these to logical CPUs. 46047043c2SRobert Mustacchi * 47047043c2SRobert Mustacchi * In current Zen based products, there is a direct mapping between processor 4819040b41SDan Cross * nodes and a data fabric PCI device: all of the devices are on PCI Bus 0 and 4919040b41SDan Cross * start from Device 0x18, so device 0x18 maps to processor node 0, 0x19 to 50047043c2SRobert Mustacchi * processor node 1, etc. This means that to map a logical CPU to a data fabric 51047043c2SRobert Mustacchi * device, we take its processor node id, add it to 0x18 and find the PCI device 5219040b41SDan Cross * that is on bus 0 with that ID number. We already discovered the DF devices as 5319040b41SDan Cross * described above. 54047043c2SRobert Mustacchi * 5519040b41SDan Cross * The northbridge PCI device has a well-defined device and function, but the 5619040b41SDan Cross * bus that it is on varies. Each die has its own set of assigned PCI buses and 5719040b41SDan Cross * its northbridge device is on the first die-specific bus. This implies that 5819040b41SDan Cross * the northbridges do not show up on PCI bus 0, as that is the PCI bus that all 5919040b41SDan Cross * of the data fabric devices are on and is not assigned to any particular die. 6019040b41SDan Cross * Additionally, while the northbridge on the lowest-numbered PCI bus 6119040b41SDan Cross * intuitively corresponds to processor node zero, hardware does not guarantee 6219040b41SDan Cross * this. Because we don't want to be at the mercy of firmware, we don't rely on 6319040b41SDan Cross * this ordering assumption, though we have yet to find a system that deviates 6419040b41SDan Cross * from it, either. 65047043c2SRobert Mustacchi * 66047043c2SRobert Mustacchi * One of the registers in the data fabric device's function 0 6719040b41SDan Cross * (AMDZEN_DF_F0_CFG_ADDR_CTL) happens to identify the first PCI bus that is 68047043c2SRobert Mustacchi * associated with the processor node. This means that we can map a data fabric 6919040b41SDan Cross * device to a northbridge by finding the northbridge whose PCI bus ID matches 7019040b41SDan Cross * the value in the corresponding data fabric's AMDZEN_DF_F0_CFG_ADDR_CTL. 71047043c2SRobert Mustacchi * 7219040b41SDan Cross * Given all of the above, we can map a northbridge to a data fabric device and 7319040b41SDan Cross * a die to a data fabric device. Because these are 1:1 mappings, there is a 7419040b41SDan Cross * transitive relationship from northbridge to die. and therefore we know which 7519040b41SDan Cross * northbridge is associated with which processor die. This is summarized in the 7619040b41SDan Cross * following image: 77047043c2SRobert Mustacchi * 78047043c2SRobert Mustacchi * +-------+ +------------------------------------+ +--------------+ 7919040b41SDan Cross * | Die 0 |---->| Data Fabric PCI BDF 0/18/0 |---->| Northbridge | 8019040b41SDan Cross * +-------+ | AMDZEN_DF_F0_CFG_ADDR_CTL: bus 10 | | PCI 10/0/0 | 8119040b41SDan Cross * ... +------------------------------------+ +--------------+ 8219040b41SDan Cross * +-------+ +------------------------------------+ +--------------+ 8319040b41SDan Cross * | Die n |---->| Data Fabric PCI BDF 0/18+n/0 |---->| Northbridge | 84047043c2SRobert Mustacchi * +-------+ | AMDZEN_DF_F0_CFG_ADDR_CTL: bus 133 | | PCI 133/0/0 | 85047043c2SRobert Mustacchi * +------------------------------------+ +--------------+ 86047043c2SRobert Mustacchi * 8719040b41SDan Cross * Note, the PCI buses used by the northbridges here are arbitrary examples that 8819040b41SDan Cross * do not necessarily reflect actual hardware values; however, the 8919040b41SDan Cross * bus/device/function (BDF) of the data fabric accurately models hardware. All 9019040b41SDan Cross * BDF values are in hex. 91047043c2SRobert Mustacchi * 92047043c2SRobert Mustacchi * Starting with the Rome generation of processors (Family 17h Model 30-3Fh), 9319040b41SDan Cross * AMD has multiple northbridges on a given die. All of these northbridges share 9419040b41SDan Cross * the same data fabric and system management network port. From our perspective 9519040b41SDan Cross * this means that some of the northbridge devices will be redundant and that we 9619040b41SDan Cross * no longer have a 1:1 mapping between the northbridge and the data fabric 9719040b41SDan Cross * devices. Every data fabric will have a northbridge, but not every northbridge 9819040b41SDan Cross * will have a data fabric device mapped. Because we're always trying to map 9919040b41SDan Cross * from a die to a northbridge and not the reverse, the fact that there are 10019040b41SDan Cross * extra northbridge devices hanging around that we don't know about shouldn't 10119040b41SDan Cross * be a problem. 102047043c2SRobert Mustacchi * 103047043c2SRobert Mustacchi * ------------------------------- 104047043c2SRobert Mustacchi * Attach and Detach Complications 105047043c2SRobert Mustacchi * ------------------------------- 106047043c2SRobert Mustacchi * 10719040b41SDan Cross * We need to map different PCI devices together. Each device is attached to a 10819040b41SDan Cross * amdzen_stub driver to facilitate integration with the rest of the kernel PCI 10919040b41SDan Cross * machinery and so we have to manage multiple dev_info_t structures, each of 11019040b41SDan Cross * which may be independently attached and detached. 111047043c2SRobert Mustacchi * 11219040b41SDan Cross * This is not particularly complex for attach: our _init routine allocates the 11319040b41SDan Cross * necessary mutex and list structures at module load time, and as each stub is 11419040b41SDan Cross * attached, it calls into this code to be added to the appropriate list. When 11519040b41SDan Cross * the nexus itself is attached, we walk the PCI device tree accumulating a 11619040b41SDan Cross * counter for all devices we expect to be attached. Once the scan is complete 11719040b41SDan Cross * and all such devices are accounted for (stub registration may be happening 11819040b41SDan Cross * asynchronously with respect to nexus attach), we initialize the nexus device 11919040b41SDan Cross * and the attach is complete. 12019040b41SDan Cross * 12119040b41SDan Cross * Most other device drivers support instances that can be brought back after 12219040b41SDan Cross * detach, provided they are associated with an active minor node in the 12319040b41SDan Cross * /devices file system. This driver is different. Once a stub device has been 12419040b41SDan Cross * attached, we do not permit detaching the nexus driver instance, as the kernel 12519040b41SDan Cross * does not give us interlocking guarantees between nexus and stub driver attach 12619040b41SDan Cross * and detach. It is simplest to just unconditionally fail detach once a stub 12719040b41SDan Cross * has attached. 128047043c2SRobert Mustacchi * 129047043c2SRobert Mustacchi * --------------- 130047043c2SRobert Mustacchi * Exposed Devices 131047043c2SRobert Mustacchi * --------------- 132047043c2SRobert Mustacchi * 133047043c2SRobert Mustacchi * Rather than try and have all of the different functions that could be 13419040b41SDan Cross * provided in one driver, we have a nexus driver that tries to load child 13519040b41SDan Cross * pseudo-device drivers that provide specific pieces of functionality. 13671815ce7SRobert Mustacchi * 13771815ce7SRobert Mustacchi * ------- 13871815ce7SRobert Mustacchi * Locking 13971815ce7SRobert Mustacchi * ------- 14071815ce7SRobert Mustacchi * 14119040b41SDan Cross * The amdzen_data structure contains a single lock, azn_mutex. 14271815ce7SRobert Mustacchi * 14319040b41SDan Cross * The various client functions here are intended for our nexus's direct 14419040b41SDan Cross * children, but have been designed in case someone else should depends on this 14519040b41SDan Cross * driver. Once a DF has been discovered, the set of entities inside of it 14619040b41SDan Cross * (adf_nents, adf_ents[]) is considered static, constant data, and iteration 14719040b41SDan Cross * over them does not require locking. However, the discovery of the amd_df_t 14819040b41SDan Cross * does. In addition, locking is required whenever performing register accesses 14919040b41SDan Cross * to the DF or SMN. 15019040b41SDan Cross * 15119040b41SDan Cross * To summarize, one must hold the lock in the following circumstances: 15219040b41SDan Cross * 15319040b41SDan Cross * - Looking up DF structures 15419040b41SDan Cross * - Reading or writing to DF registers 15519040b41SDan Cross * - Reading or writing to SMN registers 15671815ce7SRobert Mustacchi * 15771815ce7SRobert Mustacchi * In general, it is preferred that the lock be held across an entire client 15871815ce7SRobert Mustacchi * operation if possible. The only time this becomes an issue are when we have 15919040b41SDan Cross * callbacks into our callers (ala amdzen_c_df_iter()) as they may recursively 16019040b41SDan Cross * call into us. 161047043c2SRobert Mustacchi */ 162047043c2SRobert Mustacchi 163047043c2SRobert Mustacchi #include <sys/modctl.h> 164047043c2SRobert Mustacchi #include <sys/conf.h> 165047043c2SRobert Mustacchi #include <sys/devops.h> 166047043c2SRobert Mustacchi #include <sys/ddi.h> 167047043c2SRobert Mustacchi #include <sys/sunddi.h> 168047043c2SRobert Mustacchi #include <sys/pci.h> 169047043c2SRobert Mustacchi #include <sys/sysmacros.h> 170047043c2SRobert Mustacchi #include <sys/sunndi.h> 171047043c2SRobert Mustacchi #include <sys/x86_archext.h> 172047043c2SRobert Mustacchi #include <sys/cpuvar.h> 173dd23d762SRobert Mustacchi #include <sys/policy.h> 174dd23d762SRobert Mustacchi #include <sys/stat.h> 175dd23d762SRobert Mustacchi #include <sys/sunddi.h> 176dd23d762SRobert Mustacchi #include <sys/bitmap.h> 1771e0464b8SRobert Mustacchi #include <sys/stdbool.h> 178047043c2SRobert Mustacchi 17971815ce7SRobert Mustacchi #include <sys/amdzen/df.h> 180dd23d762SRobert Mustacchi #include <sys/amdzen/ccd.h> 181047043c2SRobert Mustacchi #include "amdzen.h" 182dd23d762SRobert Mustacchi #include "amdzen_client.h" 183dd23d762SRobert Mustacchi #include "amdzen_topo.h" 184047043c2SRobert Mustacchi 185047043c2SRobert Mustacchi amdzen_t *amdzen_data; 186047043c2SRobert Mustacchi 187047043c2SRobert Mustacchi /* 188dd23d762SRobert Mustacchi * Internal minor nodes for devices that the nexus provides itself. 189dd23d762SRobert Mustacchi */ 190dd23d762SRobert Mustacchi #define AMDZEN_MINOR_TOPO 0 191dd23d762SRobert Mustacchi 192dd23d762SRobert Mustacchi /* 193047043c2SRobert Mustacchi * Array of northbridge IDs that we care about. 194047043c2SRobert Mustacchi */ 195047043c2SRobert Mustacchi static const uint16_t amdzen_nb_ids[] = { 196047043c2SRobert Mustacchi /* Family 17h Ryzen, Epyc Models 00h-0fh (Zen uarch) */ 197047043c2SRobert Mustacchi 0x1450, 198becd642cSRobert Mustacchi /* Family 17h Raven Ridge, Kestrel, Dali Models 10h-2fh (Zen uarch) */ 199047043c2SRobert Mustacchi 0x15d0, 200e9abe9d6SRobert Mustacchi /* Family 17h/19h Rome, Milan, Matisse, Vermeer Zen 2/Zen 3 uarch */ 201becd642cSRobert Mustacchi 0x1480, 202f8e9c7b3SRobert Mustacchi /* Family 17h/19h Renoir, Cezanne, Van Gogh Zen 2/3 uarch */ 203f8e9c7b3SRobert Mustacchi 0x1630, 204e6f89c3aSRobert Mustacchi /* Family 19h Genoa and Bergamo */ 205f8e9c7b3SRobert Mustacchi 0x14a4, 206f8e9c7b3SRobert Mustacchi /* Family 17h Mendocino, Family 19h Rembrandt */ 207f8e9c7b3SRobert Mustacchi 0x14b5, 208f8e9c7b3SRobert Mustacchi /* Family 19h Raphael */ 209e6f89c3aSRobert Mustacchi 0x14d8, 210e6f89c3aSRobert Mustacchi /* Family 19h Phoenix */ 211e6f89c3aSRobert Mustacchi 0x14e8 212047043c2SRobert Mustacchi }; 213047043c2SRobert Mustacchi 214047043c2SRobert Mustacchi typedef struct { 215047043c2SRobert Mustacchi char *acd_name; 216047043c2SRobert Mustacchi amdzen_child_t acd_addr; 2171e0464b8SRobert Mustacchi /* 2181e0464b8SRobert Mustacchi * This indicates whether or not we should issue warnings to users when 2191e0464b8SRobert Mustacchi * something happens specific to this instance. The main reason we don't 2201e0464b8SRobert Mustacchi * want to is for optional devices that may not be installed as they are 2211e0464b8SRobert Mustacchi * for development purposes (e.g. usmn, zen_udf); however, if there is 2221e0464b8SRobert Mustacchi * an issue with the others we still want to know. 2231e0464b8SRobert Mustacchi */ 2241e0464b8SRobert Mustacchi bool acd_warn; 225047043c2SRobert Mustacchi } amdzen_child_data_t; 226047043c2SRobert Mustacchi 227047043c2SRobert Mustacchi static const amdzen_child_data_t amdzen_children[] = { 2281e0464b8SRobert Mustacchi { "smntemp", AMDZEN_C_SMNTEMP, true }, 2291e0464b8SRobert Mustacchi { "usmn", AMDZEN_C_USMN, false }, 2301e0464b8SRobert Mustacchi { "zen_udf", AMDZEN_C_ZEN_UDF, false }, 2311e0464b8SRobert Mustacchi { "zen_umc", AMDZEN_C_ZEN_UMC, true } 232047043c2SRobert Mustacchi }; 233047043c2SRobert Mustacchi 2344adf43b0SKeith M Wesolowski static uint8_t 2354adf43b0SKeith M Wesolowski amdzen_stub_get8(amdzen_stub_t *stub, off_t reg) 2364adf43b0SKeith M Wesolowski { 2374adf43b0SKeith M Wesolowski return (pci_config_get8(stub->azns_cfgspace, reg)); 2384adf43b0SKeith M Wesolowski } 2394adf43b0SKeith M Wesolowski 2404adf43b0SKeith M Wesolowski static uint16_t 2414adf43b0SKeith M Wesolowski amdzen_stub_get16(amdzen_stub_t *stub, off_t reg) 2424adf43b0SKeith M Wesolowski { 2434adf43b0SKeith M Wesolowski return (pci_config_get16(stub->azns_cfgspace, reg)); 2444adf43b0SKeith M Wesolowski } 2454adf43b0SKeith M Wesolowski 246047043c2SRobert Mustacchi static uint32_t 247047043c2SRobert Mustacchi amdzen_stub_get32(amdzen_stub_t *stub, off_t reg) 248047043c2SRobert Mustacchi { 249047043c2SRobert Mustacchi return (pci_config_get32(stub->azns_cfgspace, reg)); 250047043c2SRobert Mustacchi } 251047043c2SRobert Mustacchi 252549e0fd3SRobert Mustacchi static uint64_t 253549e0fd3SRobert Mustacchi amdzen_stub_get64(amdzen_stub_t *stub, off_t reg) 254549e0fd3SRobert Mustacchi { 255549e0fd3SRobert Mustacchi return (pci_config_get64(stub->azns_cfgspace, reg)); 256549e0fd3SRobert Mustacchi } 257549e0fd3SRobert Mustacchi 258047043c2SRobert Mustacchi static void 2594adf43b0SKeith M Wesolowski amdzen_stub_put8(amdzen_stub_t *stub, off_t reg, uint8_t val) 2604adf43b0SKeith M Wesolowski { 2614adf43b0SKeith M Wesolowski pci_config_put8(stub->azns_cfgspace, reg, val); 2624adf43b0SKeith M Wesolowski } 2634adf43b0SKeith M Wesolowski 2644adf43b0SKeith M Wesolowski static void 2654adf43b0SKeith M Wesolowski amdzen_stub_put16(amdzen_stub_t *stub, off_t reg, uint16_t val) 2664adf43b0SKeith M Wesolowski { 2674adf43b0SKeith M Wesolowski pci_config_put16(stub->azns_cfgspace, reg, val); 2684adf43b0SKeith M Wesolowski } 2694adf43b0SKeith M Wesolowski 2704adf43b0SKeith M Wesolowski static void 271047043c2SRobert Mustacchi amdzen_stub_put32(amdzen_stub_t *stub, off_t reg, uint32_t val) 272047043c2SRobert Mustacchi { 273047043c2SRobert Mustacchi pci_config_put32(stub->azns_cfgspace, reg, val); 274047043c2SRobert Mustacchi } 275047043c2SRobert Mustacchi 27671815ce7SRobert Mustacchi static uint64_t 27771815ce7SRobert Mustacchi amdzen_df_read_regdef(amdzen_t *azn, amdzen_df_t *df, const df_reg_def_t def, 27871815ce7SRobert Mustacchi uint8_t inst, boolean_t do_64) 27971815ce7SRobert Mustacchi { 28071815ce7SRobert Mustacchi df_reg_def_t ficaa; 28171815ce7SRobert Mustacchi df_reg_def_t ficad; 28271815ce7SRobert Mustacchi uint32_t val = 0; 28371815ce7SRobert Mustacchi df_rev_t df_rev = azn->azn_dfs[0].adf_rev; 28471815ce7SRobert Mustacchi 28571815ce7SRobert Mustacchi VERIFY(MUTEX_HELD(&azn->azn_mutex)); 28671815ce7SRobert Mustacchi ASSERT3U(def.drd_gens & df_rev, ==, df_rev); 28771815ce7SRobert Mustacchi val = DF_FICAA_V2_SET_TARG_INST(val, 1); 28871815ce7SRobert Mustacchi val = DF_FICAA_V2_SET_FUNC(val, def.drd_func); 28971815ce7SRobert Mustacchi val = DF_FICAA_V2_SET_INST(val, inst); 29071815ce7SRobert Mustacchi val = DF_FICAA_V2_SET_64B(val, do_64 ? 1 : 0); 29171815ce7SRobert Mustacchi 29271815ce7SRobert Mustacchi switch (df_rev) { 29371815ce7SRobert Mustacchi case DF_REV_2: 29471815ce7SRobert Mustacchi case DF_REV_3: 29571815ce7SRobert Mustacchi case DF_REV_3P5: 29671815ce7SRobert Mustacchi ficaa = DF_FICAA_V2; 29771815ce7SRobert Mustacchi ficad = DF_FICAD_LO_V2; 29871815ce7SRobert Mustacchi /* 29971815ce7SRobert Mustacchi * Both here and in the DFv4 case, the register ignores the 30071815ce7SRobert Mustacchi * lower 2 bits. That is we can only address and encode things 30171815ce7SRobert Mustacchi * in units of 4 bytes. 30271815ce7SRobert Mustacchi */ 30371815ce7SRobert Mustacchi val = DF_FICAA_V2_SET_REG(val, def.drd_reg >> 2); 30471815ce7SRobert Mustacchi break; 30571815ce7SRobert Mustacchi case DF_REV_4: 30671815ce7SRobert Mustacchi ficaa = DF_FICAA_V4; 30771815ce7SRobert Mustacchi ficad = DF_FICAD_LO_V4; 30871815ce7SRobert Mustacchi val = DF_FICAA_V4_SET_REG(val, def.drd_reg >> 2); 30971815ce7SRobert Mustacchi break; 31071815ce7SRobert Mustacchi default: 31171815ce7SRobert Mustacchi panic("encountered unexpected DF rev: %u", df_rev); 31271815ce7SRobert Mustacchi } 31371815ce7SRobert Mustacchi 31471815ce7SRobert Mustacchi amdzen_stub_put32(df->adf_funcs[ficaa.drd_func], ficaa.drd_reg, val); 31571815ce7SRobert Mustacchi if (do_64) { 31671815ce7SRobert Mustacchi return (amdzen_stub_get64(df->adf_funcs[ficad.drd_func], 31771815ce7SRobert Mustacchi ficad.drd_reg)); 31871815ce7SRobert Mustacchi } else { 31971815ce7SRobert Mustacchi return (amdzen_stub_get32(df->adf_funcs[ficad.drd_func], 32071815ce7SRobert Mustacchi ficad.drd_reg)); 32171815ce7SRobert Mustacchi } 32271815ce7SRobert Mustacchi } 32371815ce7SRobert Mustacchi 324047043c2SRobert Mustacchi /* 325047043c2SRobert Mustacchi * Perform a targeted 32-bit indirect read to a specific instance and function. 326047043c2SRobert Mustacchi */ 327047043c2SRobert Mustacchi static uint32_t 32871815ce7SRobert Mustacchi amdzen_df_read32(amdzen_t *azn, amdzen_df_t *df, uint8_t inst, 32971815ce7SRobert Mustacchi const df_reg_def_t def) 330047043c2SRobert Mustacchi { 33171815ce7SRobert Mustacchi return (amdzen_df_read_regdef(azn, df, def, inst, B_FALSE)); 332047043c2SRobert Mustacchi } 333047043c2SRobert Mustacchi 334549e0fd3SRobert Mustacchi /* 33571815ce7SRobert Mustacchi * For a broadcast read, just go to the underlying PCI function and perform a 33671815ce7SRobert Mustacchi * read. At this point in time, we don't believe we need to use the FICAA/FICAD 33771815ce7SRobert Mustacchi * to access it (though it does have a broadcast mode). 338549e0fd3SRobert Mustacchi */ 33971815ce7SRobert Mustacchi static uint32_t 34071815ce7SRobert Mustacchi amdzen_df_read32_bcast(amdzen_t *azn, amdzen_df_t *df, const df_reg_def_t def) 341549e0fd3SRobert Mustacchi { 342549e0fd3SRobert Mustacchi VERIFY(MUTEX_HELD(&azn->azn_mutex)); 34371815ce7SRobert Mustacchi return (amdzen_stub_get32(df->adf_funcs[def.drd_func], def.drd_reg)); 344549e0fd3SRobert Mustacchi } 345549e0fd3SRobert Mustacchi 346047043c2SRobert Mustacchi static uint32_t 3474adf43b0SKeith M Wesolowski amdzen_smn_read(amdzen_t *azn, amdzen_df_t *df, const smn_reg_t reg) 348047043c2SRobert Mustacchi { 3494adf43b0SKeith M Wesolowski const uint32_t base_addr = SMN_REG_ADDR_BASE(reg); 3504adf43b0SKeith M Wesolowski const uint32_t addr_off = SMN_REG_ADDR_OFF(reg); 3514adf43b0SKeith M Wesolowski 3524adf43b0SKeith M Wesolowski VERIFY(SMN_REG_IS_NATURALLY_ALIGNED(reg)); 353047043c2SRobert Mustacchi VERIFY(MUTEX_HELD(&azn->azn_mutex)); 3544adf43b0SKeith M Wesolowski amdzen_stub_put32(df->adf_nb, AMDZEN_NB_SMN_ADDR, base_addr); 3554adf43b0SKeith M Wesolowski 3564adf43b0SKeith M Wesolowski switch (SMN_REG_SIZE(reg)) { 3574adf43b0SKeith M Wesolowski case 1: 3584adf43b0SKeith M Wesolowski return ((uint32_t)amdzen_stub_get8(df->adf_nb, 3594adf43b0SKeith M Wesolowski AMDZEN_NB_SMN_DATA + addr_off)); 3604adf43b0SKeith M Wesolowski case 2: 3614adf43b0SKeith M Wesolowski return ((uint32_t)amdzen_stub_get16(df->adf_nb, 3624adf43b0SKeith M Wesolowski AMDZEN_NB_SMN_DATA + addr_off)); 3634adf43b0SKeith M Wesolowski case 4: 364047043c2SRobert Mustacchi return (amdzen_stub_get32(df->adf_nb, AMDZEN_NB_SMN_DATA)); 3654adf43b0SKeith M Wesolowski default: 3664adf43b0SKeith M Wesolowski panic("unreachable invalid SMN register size %u", 3674adf43b0SKeith M Wesolowski SMN_REG_SIZE(reg)); 3684adf43b0SKeith M Wesolowski } 369047043c2SRobert Mustacchi } 370047043c2SRobert Mustacchi 371f198607dSRobert Mustacchi static void 3724adf43b0SKeith M Wesolowski amdzen_smn_write(amdzen_t *azn, amdzen_df_t *df, const smn_reg_t reg, 373ba215efeSKeith M Wesolowski const uint32_t val) 374f198607dSRobert Mustacchi { 3754adf43b0SKeith M Wesolowski const uint32_t base_addr = SMN_REG_ADDR_BASE(reg); 3764adf43b0SKeith M Wesolowski const uint32_t addr_off = SMN_REG_ADDR_OFF(reg); 3774adf43b0SKeith M Wesolowski 3784adf43b0SKeith M Wesolowski VERIFY(SMN_REG_IS_NATURALLY_ALIGNED(reg)); 3794adf43b0SKeith M Wesolowski VERIFY(SMN_REG_VALUE_FITS(reg, val)); 380f198607dSRobert Mustacchi VERIFY(MUTEX_HELD(&azn->azn_mutex)); 3814adf43b0SKeith M Wesolowski amdzen_stub_put32(df->adf_nb, AMDZEN_NB_SMN_ADDR, base_addr); 3824adf43b0SKeith M Wesolowski 3834adf43b0SKeith M Wesolowski switch (SMN_REG_SIZE(reg)) { 3844adf43b0SKeith M Wesolowski case 1: 3854adf43b0SKeith M Wesolowski amdzen_stub_put8(df->adf_nb, AMDZEN_NB_SMN_DATA + addr_off, 3864adf43b0SKeith M Wesolowski (uint8_t)val); 3874adf43b0SKeith M Wesolowski break; 3884adf43b0SKeith M Wesolowski case 2: 3894adf43b0SKeith M Wesolowski amdzen_stub_put16(df->adf_nb, AMDZEN_NB_SMN_DATA + addr_off, 3904adf43b0SKeith M Wesolowski (uint16_t)val); 3914adf43b0SKeith M Wesolowski break; 3924adf43b0SKeith M Wesolowski case 4: 393f198607dSRobert Mustacchi amdzen_stub_put32(df->adf_nb, AMDZEN_NB_SMN_DATA, val); 3944adf43b0SKeith M Wesolowski break; 3954adf43b0SKeith M Wesolowski default: 3964adf43b0SKeith M Wesolowski panic("unreachable invalid SMN register size %u", 3974adf43b0SKeith M Wesolowski SMN_REG_SIZE(reg)); 3984adf43b0SKeith M Wesolowski } 399f198607dSRobert Mustacchi } 400f198607dSRobert Mustacchi 401047043c2SRobert Mustacchi static amdzen_df_t * 402047043c2SRobert Mustacchi amdzen_df_find(amdzen_t *azn, uint_t dfno) 403047043c2SRobert Mustacchi { 404047043c2SRobert Mustacchi uint_t i; 405047043c2SRobert Mustacchi 406047043c2SRobert Mustacchi ASSERT(MUTEX_HELD(&azn->azn_mutex)); 407047043c2SRobert Mustacchi if (dfno >= azn->azn_ndfs) { 408047043c2SRobert Mustacchi return (NULL); 409047043c2SRobert Mustacchi } 410047043c2SRobert Mustacchi 411047043c2SRobert Mustacchi for (i = 0; i < azn->azn_ndfs; i++) { 412047043c2SRobert Mustacchi amdzen_df_t *df = &azn->azn_dfs[i]; 413047043c2SRobert Mustacchi if ((df->adf_flags & AMDZEN_DF_F_VALID) == 0) { 414047043c2SRobert Mustacchi continue; 415047043c2SRobert Mustacchi } 416047043c2SRobert Mustacchi 417047043c2SRobert Mustacchi if (dfno == 0) { 418047043c2SRobert Mustacchi return (df); 419047043c2SRobert Mustacchi } 420047043c2SRobert Mustacchi dfno--; 421047043c2SRobert Mustacchi } 422047043c2SRobert Mustacchi 423047043c2SRobert Mustacchi return (NULL); 424047043c2SRobert Mustacchi } 425047043c2SRobert Mustacchi 426dd23d762SRobert Mustacchi static amdzen_df_ent_t * 427dd23d762SRobert Mustacchi amdzen_df_ent_find_by_instid(amdzen_df_t *df, uint8_t instid) 428dd23d762SRobert Mustacchi { 429dd23d762SRobert Mustacchi for (uint_t i = 0; i < df->adf_nents; i++) { 430dd23d762SRobert Mustacchi amdzen_df_ent_t *ent = &df->adf_ents[i]; 431dd23d762SRobert Mustacchi 432dd23d762SRobert Mustacchi if ((ent->adfe_flags & AMDZEN_DFE_F_ENABLED) == 0) { 433dd23d762SRobert Mustacchi continue; 434dd23d762SRobert Mustacchi } 435dd23d762SRobert Mustacchi 436dd23d762SRobert Mustacchi if (ent->adfe_inst_id == instid) { 437dd23d762SRobert Mustacchi return (ent); 438dd23d762SRobert Mustacchi } 439dd23d762SRobert Mustacchi } 440dd23d762SRobert Mustacchi 441dd23d762SRobert Mustacchi return (NULL); 442dd23d762SRobert Mustacchi } 443dd23d762SRobert Mustacchi 444047043c2SRobert Mustacchi /* 445047043c2SRobert Mustacchi * Client functions that are used by nexus children. 446047043c2SRobert Mustacchi */ 447047043c2SRobert Mustacchi int 4484adf43b0SKeith M Wesolowski amdzen_c_smn_read(uint_t dfno, const smn_reg_t reg, uint32_t *valp) 449047043c2SRobert Mustacchi { 450047043c2SRobert Mustacchi amdzen_df_t *df; 451047043c2SRobert Mustacchi amdzen_t *azn = amdzen_data; 452047043c2SRobert Mustacchi 4534adf43b0SKeith M Wesolowski if (!SMN_REG_SIZE_IS_VALID(reg)) 4544adf43b0SKeith M Wesolowski return (EINVAL); 4554adf43b0SKeith M Wesolowski if (!SMN_REG_IS_NATURALLY_ALIGNED(reg)) 4564adf43b0SKeith M Wesolowski return (EINVAL); 4574adf43b0SKeith M Wesolowski 458047043c2SRobert Mustacchi mutex_enter(&azn->azn_mutex); 459047043c2SRobert Mustacchi df = amdzen_df_find(azn, dfno); 460047043c2SRobert Mustacchi if (df == NULL) { 461047043c2SRobert Mustacchi mutex_exit(&azn->azn_mutex); 462047043c2SRobert Mustacchi return (ENOENT); 463047043c2SRobert Mustacchi } 464047043c2SRobert Mustacchi 465047043c2SRobert Mustacchi if ((df->adf_flags & AMDZEN_DF_F_FOUND_NB) == 0) { 466047043c2SRobert Mustacchi mutex_exit(&azn->azn_mutex); 467047043c2SRobert Mustacchi return (ENXIO); 468047043c2SRobert Mustacchi } 469047043c2SRobert Mustacchi 4704adf43b0SKeith M Wesolowski *valp = amdzen_smn_read(azn, df, reg); 471047043c2SRobert Mustacchi mutex_exit(&azn->azn_mutex); 472047043c2SRobert Mustacchi return (0); 473047043c2SRobert Mustacchi } 474047043c2SRobert Mustacchi 475f198607dSRobert Mustacchi int 4764adf43b0SKeith M Wesolowski amdzen_c_smn_write(uint_t dfno, const smn_reg_t reg, const uint32_t val) 477f198607dSRobert Mustacchi { 478f198607dSRobert Mustacchi amdzen_df_t *df; 479f198607dSRobert Mustacchi amdzen_t *azn = amdzen_data; 480f198607dSRobert Mustacchi 4814adf43b0SKeith M Wesolowski if (!SMN_REG_SIZE_IS_VALID(reg)) 4824adf43b0SKeith M Wesolowski return (EINVAL); 4834adf43b0SKeith M Wesolowski if (!SMN_REG_IS_NATURALLY_ALIGNED(reg)) 4844adf43b0SKeith M Wesolowski return (EINVAL); 4854adf43b0SKeith M Wesolowski if (!SMN_REG_VALUE_FITS(reg, val)) 4864adf43b0SKeith M Wesolowski return (EOVERFLOW); 4874adf43b0SKeith M Wesolowski 488f198607dSRobert Mustacchi mutex_enter(&azn->azn_mutex); 489f198607dSRobert Mustacchi df = amdzen_df_find(azn, dfno); 490f198607dSRobert Mustacchi if (df == NULL) { 491f198607dSRobert Mustacchi mutex_exit(&azn->azn_mutex); 492f198607dSRobert Mustacchi return (ENOENT); 493f198607dSRobert Mustacchi } 494f198607dSRobert Mustacchi 495f198607dSRobert Mustacchi if ((df->adf_flags & AMDZEN_DF_F_FOUND_NB) == 0) { 496f198607dSRobert Mustacchi mutex_exit(&azn->azn_mutex); 497f198607dSRobert Mustacchi return (ENXIO); 498f198607dSRobert Mustacchi } 499f198607dSRobert Mustacchi 5004adf43b0SKeith M Wesolowski amdzen_smn_write(azn, df, reg, val); 501f198607dSRobert Mustacchi mutex_exit(&azn->azn_mutex); 502f198607dSRobert Mustacchi return (0); 503f198607dSRobert Mustacchi } 504f198607dSRobert Mustacchi 505047043c2SRobert Mustacchi uint_t 506047043c2SRobert Mustacchi amdzen_c_df_count(void) 507047043c2SRobert Mustacchi { 508047043c2SRobert Mustacchi uint_t ret; 509047043c2SRobert Mustacchi amdzen_t *azn = amdzen_data; 510047043c2SRobert Mustacchi 511047043c2SRobert Mustacchi mutex_enter(&azn->azn_mutex); 512047043c2SRobert Mustacchi ret = azn->azn_ndfs; 513047043c2SRobert Mustacchi mutex_exit(&azn->azn_mutex); 514047043c2SRobert Mustacchi return (ret); 515047043c2SRobert Mustacchi } 516047043c2SRobert Mustacchi 51771815ce7SRobert Mustacchi df_rev_t 51871815ce7SRobert Mustacchi amdzen_c_df_rev(void) 51971815ce7SRobert Mustacchi { 52071815ce7SRobert Mustacchi amdzen_df_t *df; 52171815ce7SRobert Mustacchi amdzen_t *azn = amdzen_data; 52271815ce7SRobert Mustacchi df_rev_t rev; 52371815ce7SRobert Mustacchi 52471815ce7SRobert Mustacchi /* 52571815ce7SRobert Mustacchi * Always use the first DF instance to determine what we're using. Our 52671815ce7SRobert Mustacchi * current assumption, which seems to generally be true, is that the 52771815ce7SRobert Mustacchi * given DF revisions are the same in a given system when the DFs are 52871815ce7SRobert Mustacchi * directly connected. 52971815ce7SRobert Mustacchi */ 53071815ce7SRobert Mustacchi mutex_enter(&azn->azn_mutex); 53171815ce7SRobert Mustacchi df = amdzen_df_find(azn, 0); 53271815ce7SRobert Mustacchi if (df == NULL) { 53371815ce7SRobert Mustacchi rev = DF_REV_UNKNOWN; 53471815ce7SRobert Mustacchi } else { 53571815ce7SRobert Mustacchi rev = df->adf_rev; 53671815ce7SRobert Mustacchi } 53771815ce7SRobert Mustacchi mutex_exit(&azn->azn_mutex); 53871815ce7SRobert Mustacchi 53971815ce7SRobert Mustacchi return (rev); 54071815ce7SRobert Mustacchi } 54171815ce7SRobert Mustacchi 542549e0fd3SRobert Mustacchi int 54371815ce7SRobert Mustacchi amdzen_c_df_read32(uint_t dfno, uint8_t inst, const df_reg_def_t def, 54471815ce7SRobert Mustacchi uint32_t *valp) 545549e0fd3SRobert Mustacchi { 546549e0fd3SRobert Mustacchi amdzen_df_t *df; 547549e0fd3SRobert Mustacchi amdzen_t *azn = amdzen_data; 548549e0fd3SRobert Mustacchi 549549e0fd3SRobert Mustacchi mutex_enter(&azn->azn_mutex); 550549e0fd3SRobert Mustacchi df = amdzen_df_find(azn, dfno); 551549e0fd3SRobert Mustacchi if (df == NULL) { 552549e0fd3SRobert Mustacchi mutex_exit(&azn->azn_mutex); 553549e0fd3SRobert Mustacchi return (ENOENT); 554549e0fd3SRobert Mustacchi } 555549e0fd3SRobert Mustacchi 556*09dcaaa3SRobert Mustacchi if (df->adf_rev == DF_REV_UNKNOWN) { 557*09dcaaa3SRobert Mustacchi mutex_exit(&azn->azn_mutex); 558*09dcaaa3SRobert Mustacchi return (ENOTSUP); 559*09dcaaa3SRobert Mustacchi } 560*09dcaaa3SRobert Mustacchi 56171815ce7SRobert Mustacchi *valp = amdzen_df_read_regdef(azn, df, def, inst, B_FALSE); 562549e0fd3SRobert Mustacchi mutex_exit(&azn->azn_mutex); 563549e0fd3SRobert Mustacchi 564549e0fd3SRobert Mustacchi return (0); 565549e0fd3SRobert Mustacchi } 566549e0fd3SRobert Mustacchi 567549e0fd3SRobert Mustacchi int 56871815ce7SRobert Mustacchi amdzen_c_df_read64(uint_t dfno, uint8_t inst, const df_reg_def_t def, 56971815ce7SRobert Mustacchi uint64_t *valp) 570549e0fd3SRobert Mustacchi { 571549e0fd3SRobert Mustacchi amdzen_df_t *df; 572549e0fd3SRobert Mustacchi amdzen_t *azn = amdzen_data; 573549e0fd3SRobert Mustacchi 574549e0fd3SRobert Mustacchi mutex_enter(&azn->azn_mutex); 575549e0fd3SRobert Mustacchi df = amdzen_df_find(azn, dfno); 576549e0fd3SRobert Mustacchi if (df == NULL) { 577549e0fd3SRobert Mustacchi mutex_exit(&azn->azn_mutex); 578549e0fd3SRobert Mustacchi return (ENOENT); 579549e0fd3SRobert Mustacchi } 580549e0fd3SRobert Mustacchi 581*09dcaaa3SRobert Mustacchi if (df->adf_rev == DF_REV_UNKNOWN) { 582*09dcaaa3SRobert Mustacchi mutex_exit(&azn->azn_mutex); 583*09dcaaa3SRobert Mustacchi return (ENOTSUP); 584*09dcaaa3SRobert Mustacchi } 585*09dcaaa3SRobert Mustacchi 58671815ce7SRobert Mustacchi *valp = amdzen_df_read_regdef(azn, df, def, inst, B_TRUE); 587549e0fd3SRobert Mustacchi mutex_exit(&azn->azn_mutex); 588549e0fd3SRobert Mustacchi 589549e0fd3SRobert Mustacchi return (0); 590549e0fd3SRobert Mustacchi } 591047043c2SRobert Mustacchi 59271815ce7SRobert Mustacchi int 59371815ce7SRobert Mustacchi amdzen_c_df_iter(uint_t dfno, zen_df_type_t type, amdzen_c_iter_f func, 59471815ce7SRobert Mustacchi void *arg) 59571815ce7SRobert Mustacchi { 59671815ce7SRobert Mustacchi amdzen_df_t *df; 59771815ce7SRobert Mustacchi amdzen_t *azn = amdzen_data; 59871815ce7SRobert Mustacchi df_type_t df_type; 59971815ce7SRobert Mustacchi uint8_t df_subtype; 60071815ce7SRobert Mustacchi 60171815ce7SRobert Mustacchi /* 60271815ce7SRobert Mustacchi * Unlike other calls here, we hold our lock only to find the DF here. 60371815ce7SRobert Mustacchi * The main reason for this is the nature of the callback function. 60471815ce7SRobert Mustacchi * Folks are iterating over instances so they can call back into us. If 60571815ce7SRobert Mustacchi * you look at the locking statement, the thing that is most volatile 60671815ce7SRobert Mustacchi * right here and what we need to protect is the DF itself and 60771815ce7SRobert Mustacchi * subsequent register accesses to it. The actual data about which 60871815ce7SRobert Mustacchi * entities exist is static and so once we have found a DF we should 60971815ce7SRobert Mustacchi * hopefully be in good shape as they only come, but don't go. 61071815ce7SRobert Mustacchi */ 61171815ce7SRobert Mustacchi mutex_enter(&azn->azn_mutex); 61271815ce7SRobert Mustacchi df = amdzen_df_find(azn, dfno); 61371815ce7SRobert Mustacchi if (df == NULL) { 61471815ce7SRobert Mustacchi mutex_exit(&azn->azn_mutex); 61571815ce7SRobert Mustacchi return (ENOENT); 61671815ce7SRobert Mustacchi } 61771815ce7SRobert Mustacchi mutex_exit(&azn->azn_mutex); 61871815ce7SRobert Mustacchi 61971815ce7SRobert Mustacchi switch (type) { 62071815ce7SRobert Mustacchi case ZEN_DF_TYPE_CS_UMC: 62171815ce7SRobert Mustacchi df_type = DF_TYPE_CS; 62271815ce7SRobert Mustacchi /* 62371815ce7SRobert Mustacchi * In the original Zeppelin DFv2 die there was no subtype field 62471815ce7SRobert Mustacchi * used for the CS. The UMC is the only type and has a subtype 62571815ce7SRobert Mustacchi * of zero. 62671815ce7SRobert Mustacchi */ 62771815ce7SRobert Mustacchi if (df->adf_rev != DF_REV_2) { 62871815ce7SRobert Mustacchi df_subtype = DF_CS_SUBTYPE_UMC; 62971815ce7SRobert Mustacchi } else { 63071815ce7SRobert Mustacchi df_subtype = 0; 63171815ce7SRobert Mustacchi } 63271815ce7SRobert Mustacchi break; 63371815ce7SRobert Mustacchi case ZEN_DF_TYPE_CCM_CPU: 63471815ce7SRobert Mustacchi /* 635dd23d762SRobert Mustacchi * Because the CCM CPU subtype has always remained zero, we can 636dd23d762SRobert Mustacchi * use that regardless of the generation. 63771815ce7SRobert Mustacchi */ 638f8e9c7b3SRobert Mustacchi df_type = DF_TYPE_CCM; 639dd23d762SRobert Mustacchi df_subtype = DF_CCM_SUBTYPE_CPU; 64071815ce7SRobert Mustacchi break; 64171815ce7SRobert Mustacchi default: 64271815ce7SRobert Mustacchi return (EINVAL); 64371815ce7SRobert Mustacchi } 64471815ce7SRobert Mustacchi 64571815ce7SRobert Mustacchi for (uint_t i = 0; i < df->adf_nents; i++) { 64671815ce7SRobert Mustacchi amdzen_df_ent_t *ent = &df->adf_ents[i]; 64771815ce7SRobert Mustacchi 64871815ce7SRobert Mustacchi /* 64971815ce7SRobert Mustacchi * Some DF components are not considered enabled and therefore 65071815ce7SRobert Mustacchi * will end up having bogus values in their ID fields. If we do 65171815ce7SRobert Mustacchi * not have an enable flag set, we must skip this node. 65271815ce7SRobert Mustacchi */ 65371815ce7SRobert Mustacchi if ((ent->adfe_flags & AMDZEN_DFE_F_ENABLED) == 0) 65471815ce7SRobert Mustacchi continue; 65571815ce7SRobert Mustacchi 65671815ce7SRobert Mustacchi if (ent->adfe_type == df_type && 65771815ce7SRobert Mustacchi ent->adfe_subtype == df_subtype) { 65871815ce7SRobert Mustacchi int ret = func(dfno, ent->adfe_fabric_id, 65971815ce7SRobert Mustacchi ent->adfe_inst_id, arg); 66071815ce7SRobert Mustacchi if (ret != 0) { 66171815ce7SRobert Mustacchi return (ret); 66271815ce7SRobert Mustacchi } 66371815ce7SRobert Mustacchi } 66471815ce7SRobert Mustacchi } 66571815ce7SRobert Mustacchi 66671815ce7SRobert Mustacchi return (0); 66771815ce7SRobert Mustacchi } 66871815ce7SRobert Mustacchi 66971815ce7SRobert Mustacchi int 67071815ce7SRobert Mustacchi amdzen_c_df_fabric_decomp(df_fabric_decomp_t *decomp) 67171815ce7SRobert Mustacchi { 67271815ce7SRobert Mustacchi const amdzen_df_t *df; 67371815ce7SRobert Mustacchi amdzen_t *azn = amdzen_data; 67471815ce7SRobert Mustacchi 67571815ce7SRobert Mustacchi mutex_enter(&azn->azn_mutex); 67671815ce7SRobert Mustacchi df = amdzen_df_find(azn, 0); 67771815ce7SRobert Mustacchi if (df == NULL) { 67871815ce7SRobert Mustacchi mutex_exit(&azn->azn_mutex); 67971815ce7SRobert Mustacchi return (ENOENT); 68071815ce7SRobert Mustacchi } 68171815ce7SRobert Mustacchi 68271815ce7SRobert Mustacchi *decomp = df->adf_decomp; 68371815ce7SRobert Mustacchi mutex_exit(&azn->azn_mutex); 68471815ce7SRobert Mustacchi return (0); 68571815ce7SRobert Mustacchi } 68671815ce7SRobert Mustacchi 687047043c2SRobert Mustacchi static boolean_t 688047043c2SRobert Mustacchi amdzen_create_child(amdzen_t *azn, const amdzen_child_data_t *acd) 689047043c2SRobert Mustacchi { 690047043c2SRobert Mustacchi int ret; 691047043c2SRobert Mustacchi dev_info_t *child; 692047043c2SRobert Mustacchi 693047043c2SRobert Mustacchi if (ndi_devi_alloc(azn->azn_dip, acd->acd_name, 694047043c2SRobert Mustacchi (pnode_t)DEVI_SID_NODEID, &child) != NDI_SUCCESS) { 695549e0fd3SRobert Mustacchi dev_err(azn->azn_dip, CE_WARN, "!failed to allocate child " 696047043c2SRobert Mustacchi "dip for %s", acd->acd_name); 697047043c2SRobert Mustacchi return (B_FALSE); 698047043c2SRobert Mustacchi } 699047043c2SRobert Mustacchi 700047043c2SRobert Mustacchi ddi_set_parent_data(child, (void *)acd); 701047043c2SRobert Mustacchi if ((ret = ndi_devi_online(child, 0)) != NDI_SUCCESS) { 7021e0464b8SRobert Mustacchi if (acd->acd_warn) { 7031e0464b8SRobert Mustacchi dev_err(azn->azn_dip, CE_WARN, "!failed to online " 7041e0464b8SRobert Mustacchi "child dip %s: %d", acd->acd_name, ret); 7051e0464b8SRobert Mustacchi } 706047043c2SRobert Mustacchi return (B_FALSE); 707047043c2SRobert Mustacchi } 708047043c2SRobert Mustacchi 709047043c2SRobert Mustacchi return (B_TRUE); 710047043c2SRobert Mustacchi } 711047043c2SRobert Mustacchi 712047043c2SRobert Mustacchi static boolean_t 713047043c2SRobert Mustacchi amdzen_map_dfs(amdzen_t *azn) 714047043c2SRobert Mustacchi { 715047043c2SRobert Mustacchi amdzen_stub_t *stub; 716047043c2SRobert Mustacchi 717047043c2SRobert Mustacchi ASSERT(MUTEX_HELD(&azn->azn_mutex)); 718047043c2SRobert Mustacchi 719047043c2SRobert Mustacchi for (stub = list_head(&azn->azn_df_stubs); stub != NULL; 720047043c2SRobert Mustacchi stub = list_next(&azn->azn_df_stubs, stub)) { 721047043c2SRobert Mustacchi amdzen_df_t *df; 722047043c2SRobert Mustacchi uint_t dfno; 723047043c2SRobert Mustacchi 724047043c2SRobert Mustacchi dfno = stub->azns_dev - AMDZEN_DF_FIRST_DEVICE; 725047043c2SRobert Mustacchi if (dfno > AMDZEN_MAX_DFS) { 726047043c2SRobert Mustacchi dev_err(stub->azns_dip, CE_WARN, "encountered df " 727047043c2SRobert Mustacchi "device with illegal DF PCI b/d/f: 0x%x/%x/%x", 728047043c2SRobert Mustacchi stub->azns_bus, stub->azns_dev, stub->azns_func); 729047043c2SRobert Mustacchi goto err; 730047043c2SRobert Mustacchi } 731047043c2SRobert Mustacchi 732047043c2SRobert Mustacchi df = &azn->azn_dfs[dfno]; 733047043c2SRobert Mustacchi 734047043c2SRobert Mustacchi if (stub->azns_func >= AMDZEN_MAX_DF_FUNCS) { 735047043c2SRobert Mustacchi dev_err(stub->azns_dip, CE_WARN, "encountered df " 736047043c2SRobert Mustacchi "device with illegal DF PCI b/d/f: 0x%x/%x/%x", 737047043c2SRobert Mustacchi stub->azns_bus, stub->azns_dev, stub->azns_func); 738047043c2SRobert Mustacchi goto err; 739047043c2SRobert Mustacchi } 740047043c2SRobert Mustacchi 741047043c2SRobert Mustacchi if (df->adf_funcs[stub->azns_func] != NULL) { 742047043c2SRobert Mustacchi dev_err(stub->azns_dip, CE_WARN, "encountered " 743047043c2SRobert Mustacchi "duplicate df device with DF PCI b/d/f: 0x%x/%x/%x", 744047043c2SRobert Mustacchi stub->azns_bus, stub->azns_dev, stub->azns_func); 745047043c2SRobert Mustacchi goto err; 746047043c2SRobert Mustacchi } 747047043c2SRobert Mustacchi df->adf_funcs[stub->azns_func] = stub; 748047043c2SRobert Mustacchi } 749047043c2SRobert Mustacchi 750047043c2SRobert Mustacchi return (B_TRUE); 751047043c2SRobert Mustacchi 752047043c2SRobert Mustacchi err: 753047043c2SRobert Mustacchi azn->azn_flags |= AMDZEN_F_DEVICE_ERROR; 754047043c2SRobert Mustacchi return (B_FALSE); 755047043c2SRobert Mustacchi } 756047043c2SRobert Mustacchi 757047043c2SRobert Mustacchi static boolean_t 758047043c2SRobert Mustacchi amdzen_check_dfs(amdzen_t *azn) 759047043c2SRobert Mustacchi { 760047043c2SRobert Mustacchi uint_t i; 761047043c2SRobert Mustacchi boolean_t ret = B_TRUE; 762047043c2SRobert Mustacchi 763047043c2SRobert Mustacchi for (i = 0; i < AMDZEN_MAX_DFS; i++) { 764047043c2SRobert Mustacchi amdzen_df_t *df = &azn->azn_dfs[i]; 765047043c2SRobert Mustacchi uint_t count = 0; 766047043c2SRobert Mustacchi 767047043c2SRobert Mustacchi /* 768047043c2SRobert Mustacchi * We require all platforms to have DFs functions 0-6. Not all 769047043c2SRobert Mustacchi * platforms have DF function 7. 770047043c2SRobert Mustacchi */ 771047043c2SRobert Mustacchi for (uint_t func = 0; func < AMDZEN_MAX_DF_FUNCS - 1; func++) { 772047043c2SRobert Mustacchi if (df->adf_funcs[func] != NULL) { 773047043c2SRobert Mustacchi count++; 774047043c2SRobert Mustacchi } 775047043c2SRobert Mustacchi } 776047043c2SRobert Mustacchi 777047043c2SRobert Mustacchi if (count == 0) 778047043c2SRobert Mustacchi continue; 779047043c2SRobert Mustacchi 780047043c2SRobert Mustacchi if (count != 7) { 781047043c2SRobert Mustacchi ret = B_FALSE; 782047043c2SRobert Mustacchi dev_err(azn->azn_dip, CE_WARN, "df %u devices " 783047043c2SRobert Mustacchi "incomplete", i); 784047043c2SRobert Mustacchi } else { 785047043c2SRobert Mustacchi df->adf_flags |= AMDZEN_DF_F_VALID; 786047043c2SRobert Mustacchi azn->azn_ndfs++; 787047043c2SRobert Mustacchi } 788047043c2SRobert Mustacchi } 789047043c2SRobert Mustacchi 790047043c2SRobert Mustacchi return (ret); 791047043c2SRobert Mustacchi } 792047043c2SRobert Mustacchi 793047043c2SRobert Mustacchi static const uint8_t amdzen_df_rome_ids[0x2b] = { 794047043c2SRobert Mustacchi 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16, 17, 18, 19, 20, 21, 22, 23, 795047043c2SRobert Mustacchi 24, 25, 26, 27, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 796047043c2SRobert Mustacchi 44, 45, 46, 47, 48 797047043c2SRobert Mustacchi }; 798047043c2SRobert Mustacchi 799047043c2SRobert Mustacchi /* 800e9abe9d6SRobert Mustacchi * Check the first df entry to see if it belongs to Rome or Milan. If so, then 801e9abe9d6SRobert Mustacchi * it uses the disjoint ID space. 802e9abe9d6SRobert Mustacchi */ 803e9abe9d6SRobert Mustacchi static boolean_t 804e9abe9d6SRobert Mustacchi amdzen_is_rome_style(uint_t id) 805e9abe9d6SRobert Mustacchi { 806e9abe9d6SRobert Mustacchi return (id == 0x1490 || id == 0x1650); 807e9abe9d6SRobert Mustacchi } 808e9abe9d6SRobert Mustacchi 809e9abe9d6SRobert Mustacchi /* 81071815ce7SRobert Mustacchi * To be able to do most other things we want to do, we must first determine 81171815ce7SRobert Mustacchi * what revision of the DF (data fabric) that we're using. 81271815ce7SRobert Mustacchi * 81371815ce7SRobert Mustacchi * Snapshot the df version. This was added explicitly in DFv4.0, around the Zen 81471815ce7SRobert Mustacchi * 4 timeframe and allows us to tell apart different version of the DF register 81571815ce7SRobert Mustacchi * set, most usefully when various subtypes were added. 81671815ce7SRobert Mustacchi * 81771815ce7SRobert Mustacchi * Older versions can theoretically be told apart based on usage of reserved 81871815ce7SRobert Mustacchi * registers. We walk these in the following order, starting with the newest rev 81971815ce7SRobert Mustacchi * and walking backwards to tell things apart: 82071815ce7SRobert Mustacchi * 82171815ce7SRobert Mustacchi * o v3.5 -> Check function 1, register 0x150. This was reserved prior 82271815ce7SRobert Mustacchi * to this point. This is actually DF_FIDMASK0_V3P5. We are supposed 82371815ce7SRobert Mustacchi * to check bits [7:0]. 82471815ce7SRobert Mustacchi * 82571815ce7SRobert Mustacchi * o v3.0 -> Check function 1, register 0x208. The low byte (7:0) was 82671815ce7SRobert Mustacchi * changed to indicate a component mask. This is non-zero 82771815ce7SRobert Mustacchi * in the 3.0 generation. This is actually DF_FIDMASK_V2. 82871815ce7SRobert Mustacchi * 82971815ce7SRobert Mustacchi * o v2.0 -> This is just the not that case. Presumably v1 wasn't part 83071815ce7SRobert Mustacchi * of the Zen generation. 83171815ce7SRobert Mustacchi * 83271815ce7SRobert Mustacchi * Because we don't know what version we are yet, we do not use the normal 83371815ce7SRobert Mustacchi * versioned register accesses which would check what DF version we are and 83471815ce7SRobert Mustacchi * would want to use the normal indirect register accesses (which also require 83571815ce7SRobert Mustacchi * us to know the version). We instead do direct broadcast reads. 83671815ce7SRobert Mustacchi */ 83771815ce7SRobert Mustacchi static void 83871815ce7SRobert Mustacchi amdzen_determine_df_vers(amdzen_t *azn, amdzen_df_t *df) 83971815ce7SRobert Mustacchi { 84071815ce7SRobert Mustacchi uint32_t val; 84171815ce7SRobert Mustacchi df_reg_def_t rd = DF_FBICNT; 84271815ce7SRobert Mustacchi 84371815ce7SRobert Mustacchi val = amdzen_stub_get32(df->adf_funcs[rd.drd_func], rd.drd_reg); 84471815ce7SRobert Mustacchi df->adf_major = DF_FBICNT_V4_GET_MAJOR(val); 84571815ce7SRobert Mustacchi df->adf_minor = DF_FBICNT_V4_GET_MINOR(val); 84671815ce7SRobert Mustacchi if (df->adf_major == 0 && df->adf_minor == 0) { 84771815ce7SRobert Mustacchi rd = DF_FIDMASK0_V3P5; 84871815ce7SRobert Mustacchi val = amdzen_stub_get32(df->adf_funcs[rd.drd_func], rd.drd_reg); 84971815ce7SRobert Mustacchi if (bitx32(val, 7, 0) != 0) { 85071815ce7SRobert Mustacchi df->adf_major = 3; 85171815ce7SRobert Mustacchi df->adf_minor = 5; 85271815ce7SRobert Mustacchi df->adf_rev = DF_REV_3P5; 85371815ce7SRobert Mustacchi } else { 85471815ce7SRobert Mustacchi rd = DF_FIDMASK_V2; 85571815ce7SRobert Mustacchi val = amdzen_stub_get32(df->adf_funcs[rd.drd_func], 85671815ce7SRobert Mustacchi rd.drd_reg); 85771815ce7SRobert Mustacchi if (bitx32(val, 7, 0) != 0) { 85871815ce7SRobert Mustacchi df->adf_major = 3; 85971815ce7SRobert Mustacchi df->adf_minor = 0; 86071815ce7SRobert Mustacchi df->adf_rev = DF_REV_3; 86171815ce7SRobert Mustacchi } else { 86271815ce7SRobert Mustacchi df->adf_major = 2; 86371815ce7SRobert Mustacchi df->adf_minor = 0; 86471815ce7SRobert Mustacchi df->adf_rev = DF_REV_2; 86571815ce7SRobert Mustacchi } 86671815ce7SRobert Mustacchi } 86771815ce7SRobert Mustacchi } else if (df->adf_major == 4 && df->adf_minor == 0) { 86871815ce7SRobert Mustacchi df->adf_rev = DF_REV_4; 86971815ce7SRobert Mustacchi } else { 87071815ce7SRobert Mustacchi df->adf_rev = DF_REV_UNKNOWN; 87171815ce7SRobert Mustacchi } 87271815ce7SRobert Mustacchi } 87371815ce7SRobert Mustacchi 87471815ce7SRobert Mustacchi /* 87571815ce7SRobert Mustacchi * All of the different versions of the DF have different ways of getting at and 87671815ce7SRobert Mustacchi * answering the question of how do I break a fabric ID into a corresponding 87771815ce7SRobert Mustacchi * socket, die, and component. Importantly the goal here is to obtain, cache, 87871815ce7SRobert Mustacchi * and normalize: 87971815ce7SRobert Mustacchi * 88071815ce7SRobert Mustacchi * o The DF System Configuration 88171815ce7SRobert Mustacchi * o The various Mask registers 88271815ce7SRobert Mustacchi * o The Node ID 88371815ce7SRobert Mustacchi */ 88471815ce7SRobert Mustacchi static void 88571815ce7SRobert Mustacchi amdzen_determine_fabric_decomp(amdzen_t *azn, amdzen_df_t *df) 88671815ce7SRobert Mustacchi { 88771815ce7SRobert Mustacchi uint32_t mask; 88871815ce7SRobert Mustacchi df_fabric_decomp_t *decomp = &df->adf_decomp; 88971815ce7SRobert Mustacchi 89071815ce7SRobert Mustacchi switch (df->adf_rev) { 89171815ce7SRobert Mustacchi case DF_REV_2: 89271815ce7SRobert Mustacchi df->adf_syscfg = amdzen_df_read32_bcast(azn, df, DF_SYSCFG_V2); 89371815ce7SRobert Mustacchi switch (DF_SYSCFG_V2_GET_MY_TYPE(df->adf_syscfg)) { 89471815ce7SRobert Mustacchi case DF_DIE_TYPE_CPU: 89571815ce7SRobert Mustacchi mask = amdzen_df_read32_bcast(azn, df, 89671815ce7SRobert Mustacchi DF_DIEMASK_CPU_V2); 89771815ce7SRobert Mustacchi break; 89871815ce7SRobert Mustacchi case DF_DIE_TYPE_APU: 89971815ce7SRobert Mustacchi mask = amdzen_df_read32_bcast(azn, df, 90071815ce7SRobert Mustacchi DF_DIEMASK_APU_V2); 90171815ce7SRobert Mustacchi break; 90271815ce7SRobert Mustacchi default: 90371815ce7SRobert Mustacchi panic("DF thinks we're not on a CPU!"); 90471815ce7SRobert Mustacchi } 90571815ce7SRobert Mustacchi df->adf_mask0 = mask; 90671815ce7SRobert Mustacchi 90771815ce7SRobert Mustacchi /* 90871815ce7SRobert Mustacchi * DFv2 is a bit different in how the fabric mask register is 90971815ce7SRobert Mustacchi * phrased. Logically a fabric ID is broken into something that 91071815ce7SRobert Mustacchi * uniquely identifies a "node" (a particular die on a socket) 91171815ce7SRobert Mustacchi * and something that identifies a "component", e.g. a memory 91271815ce7SRobert Mustacchi * controller. 91371815ce7SRobert Mustacchi * 91471815ce7SRobert Mustacchi * Starting with DFv3, these registers logically called out how 91571815ce7SRobert Mustacchi * to separate the fabric ID first into a node and a component. 91671815ce7SRobert Mustacchi * Then the node was then broken down into a socket and die. In 91771815ce7SRobert Mustacchi * DFv2, there is no separate mask and shift of a node. Instead 91871815ce7SRobert Mustacchi * the socket and die are absolute offsets into the fabric ID 91971815ce7SRobert Mustacchi * rather than relative offsets into the node ID. As such, when 92071815ce7SRobert Mustacchi * we encounter DFv2, we fake up a node mask and shift and make 92171815ce7SRobert Mustacchi * it look like DFv3+. 92271815ce7SRobert Mustacchi */ 92371815ce7SRobert Mustacchi decomp->dfd_node_mask = DF_DIEMASK_V2_GET_SOCK_MASK(mask) | 92471815ce7SRobert Mustacchi DF_DIEMASK_V2_GET_DIE_MASK(mask); 92571815ce7SRobert Mustacchi decomp->dfd_node_shift = DF_DIEMASK_V2_GET_DIE_SHIFT(mask); 92671815ce7SRobert Mustacchi decomp->dfd_comp_mask = DF_DIEMASK_V2_GET_COMP_MASK(mask); 92771815ce7SRobert Mustacchi decomp->dfd_comp_shift = 0; 92871815ce7SRobert Mustacchi 92971815ce7SRobert Mustacchi decomp->dfd_sock_mask = DF_DIEMASK_V2_GET_SOCK_MASK(mask) >> 93071815ce7SRobert Mustacchi decomp->dfd_node_shift; 93171815ce7SRobert Mustacchi decomp->dfd_die_mask = DF_DIEMASK_V2_GET_DIE_MASK(mask) >> 93271815ce7SRobert Mustacchi decomp->dfd_node_shift; 93371815ce7SRobert Mustacchi decomp->dfd_sock_shift = DF_DIEMASK_V2_GET_SOCK_SHIFT(mask) - 93471815ce7SRobert Mustacchi decomp->dfd_node_shift; 93571815ce7SRobert Mustacchi decomp->dfd_die_shift = DF_DIEMASK_V2_GET_DIE_SHIFT(mask) - 93671815ce7SRobert Mustacchi decomp->dfd_node_shift; 93771815ce7SRobert Mustacchi ASSERT3U(decomp->dfd_die_shift, ==, 0); 938dd23d762SRobert Mustacchi 939dd23d762SRobert Mustacchi /* 940dd23d762SRobert Mustacchi * There is no register in the actual data fabric with the node 941dd23d762SRobert Mustacchi * ID in DFv2 that we have found. Instead we take the first 942dd23d762SRobert Mustacchi * entity's fabric ID and transform it into the node id. 943dd23d762SRobert Mustacchi */ 944dd23d762SRobert Mustacchi df->adf_nodeid = (df->adf_ents[0].adfe_fabric_id & 945dd23d762SRobert Mustacchi decomp->dfd_node_mask) >> decomp->dfd_node_shift; 94671815ce7SRobert Mustacchi break; 94771815ce7SRobert Mustacchi case DF_REV_3: 94871815ce7SRobert Mustacchi df->adf_syscfg = amdzen_df_read32_bcast(azn, df, DF_SYSCFG_V3); 94971815ce7SRobert Mustacchi df->adf_mask0 = amdzen_df_read32_bcast(azn, df, 95071815ce7SRobert Mustacchi DF_FIDMASK0_V3); 95171815ce7SRobert Mustacchi df->adf_mask1 = amdzen_df_read32_bcast(azn, df, 95271815ce7SRobert Mustacchi DF_FIDMASK1_V3); 95371815ce7SRobert Mustacchi 95471815ce7SRobert Mustacchi decomp->dfd_sock_mask = 95571815ce7SRobert Mustacchi DF_FIDMASK1_V3_GET_SOCK_MASK(df->adf_mask1); 95671815ce7SRobert Mustacchi decomp->dfd_sock_shift = 95771815ce7SRobert Mustacchi DF_FIDMASK1_V3_GET_SOCK_SHIFT(df->adf_mask1); 95871815ce7SRobert Mustacchi decomp->dfd_die_mask = 95971815ce7SRobert Mustacchi DF_FIDMASK1_V3_GET_DIE_MASK(df->adf_mask1); 96071815ce7SRobert Mustacchi decomp->dfd_die_shift = 0; 96171815ce7SRobert Mustacchi decomp->dfd_node_mask = 96271815ce7SRobert Mustacchi DF_FIDMASK0_V3_GET_NODE_MASK(df->adf_mask0); 96371815ce7SRobert Mustacchi decomp->dfd_node_shift = 96471815ce7SRobert Mustacchi DF_FIDMASK1_V3_GET_NODE_SHIFT(df->adf_mask1); 96571815ce7SRobert Mustacchi decomp->dfd_comp_mask = 96671815ce7SRobert Mustacchi DF_FIDMASK0_V3_GET_COMP_MASK(df->adf_mask0); 96771815ce7SRobert Mustacchi decomp->dfd_comp_shift = 0; 968dd23d762SRobert Mustacchi 969dd23d762SRobert Mustacchi df->adf_nodeid = DF_SYSCFG_V3_GET_NODE_ID(df->adf_syscfg); 97071815ce7SRobert Mustacchi break; 97171815ce7SRobert Mustacchi case DF_REV_3P5: 97271815ce7SRobert Mustacchi df->adf_syscfg = amdzen_df_read32_bcast(azn, df, 97371815ce7SRobert Mustacchi DF_SYSCFG_V3P5); 97471815ce7SRobert Mustacchi df->adf_mask0 = amdzen_df_read32_bcast(azn, df, 97571815ce7SRobert Mustacchi DF_FIDMASK0_V3P5); 97671815ce7SRobert Mustacchi df->adf_mask1 = amdzen_df_read32_bcast(azn, df, 97771815ce7SRobert Mustacchi DF_FIDMASK1_V3P5); 97871815ce7SRobert Mustacchi df->adf_mask2 = amdzen_df_read32_bcast(azn, df, 97971815ce7SRobert Mustacchi DF_FIDMASK2_V3P5); 98071815ce7SRobert Mustacchi 98171815ce7SRobert Mustacchi decomp->dfd_sock_mask = 98271815ce7SRobert Mustacchi DF_FIDMASK2_V3P5_GET_SOCK_MASK(df->adf_mask2); 98371815ce7SRobert Mustacchi decomp->dfd_sock_shift = 98471815ce7SRobert Mustacchi DF_FIDMASK1_V3P5_GET_SOCK_SHIFT(df->adf_mask1); 98571815ce7SRobert Mustacchi decomp->dfd_die_mask = 98671815ce7SRobert Mustacchi DF_FIDMASK2_V3P5_GET_DIE_MASK(df->adf_mask2); 98771815ce7SRobert Mustacchi decomp->dfd_die_shift = 0; 98871815ce7SRobert Mustacchi decomp->dfd_node_mask = 98971815ce7SRobert Mustacchi DF_FIDMASK0_V3P5_GET_NODE_MASK(df->adf_mask0); 99071815ce7SRobert Mustacchi decomp->dfd_node_shift = 99171815ce7SRobert Mustacchi DF_FIDMASK1_V3P5_GET_NODE_SHIFT(df->adf_mask1); 99271815ce7SRobert Mustacchi decomp->dfd_comp_mask = 99371815ce7SRobert Mustacchi DF_FIDMASK0_V3P5_GET_COMP_MASK(df->adf_mask0); 99471815ce7SRobert Mustacchi decomp->dfd_comp_shift = 0; 995dd23d762SRobert Mustacchi 996dd23d762SRobert Mustacchi df->adf_nodeid = DF_SYSCFG_V3P5_GET_NODE_ID(df->adf_syscfg); 99771815ce7SRobert Mustacchi break; 99871815ce7SRobert Mustacchi case DF_REV_4: 99971815ce7SRobert Mustacchi df->adf_syscfg = amdzen_df_read32_bcast(azn, df, DF_SYSCFG_V4); 100071815ce7SRobert Mustacchi df->adf_mask0 = amdzen_df_read32_bcast(azn, df, 100171815ce7SRobert Mustacchi DF_FIDMASK0_V4); 100271815ce7SRobert Mustacchi df->adf_mask1 = amdzen_df_read32_bcast(azn, df, 100371815ce7SRobert Mustacchi DF_FIDMASK1_V4); 100471815ce7SRobert Mustacchi df->adf_mask2 = amdzen_df_read32_bcast(azn, df, 100571815ce7SRobert Mustacchi DF_FIDMASK2_V4); 100671815ce7SRobert Mustacchi 100771815ce7SRobert Mustacchi /* 100871815ce7SRobert Mustacchi * The DFv4 registers are at a different location in the DF; 100971815ce7SRobert Mustacchi * however, the actual layout of fields is the same as DFv3.5. 101071815ce7SRobert Mustacchi * This is why you see V3P5 below. 101171815ce7SRobert Mustacchi */ 101271815ce7SRobert Mustacchi decomp->dfd_sock_mask = 101371815ce7SRobert Mustacchi DF_FIDMASK2_V3P5_GET_SOCK_MASK(df->adf_mask2); 101471815ce7SRobert Mustacchi decomp->dfd_sock_shift = 101571815ce7SRobert Mustacchi DF_FIDMASK1_V3P5_GET_SOCK_SHIFT(df->adf_mask1); 101671815ce7SRobert Mustacchi decomp->dfd_die_mask = 101771815ce7SRobert Mustacchi DF_FIDMASK2_V3P5_GET_DIE_MASK(df->adf_mask2); 101871815ce7SRobert Mustacchi decomp->dfd_die_shift = 0; 101971815ce7SRobert Mustacchi decomp->dfd_node_mask = 102071815ce7SRobert Mustacchi DF_FIDMASK0_V3P5_GET_NODE_MASK(df->adf_mask0); 102171815ce7SRobert Mustacchi decomp->dfd_node_shift = 102271815ce7SRobert Mustacchi DF_FIDMASK1_V3P5_GET_NODE_SHIFT(df->adf_mask1); 102371815ce7SRobert Mustacchi decomp->dfd_comp_mask = 102471815ce7SRobert Mustacchi DF_FIDMASK0_V3P5_GET_COMP_MASK(df->adf_mask0); 102571815ce7SRobert Mustacchi decomp->dfd_comp_shift = 0; 1026dd23d762SRobert Mustacchi 1027dd23d762SRobert Mustacchi df->adf_nodeid = DF_SYSCFG_V4_GET_NODE_ID(df->adf_syscfg); 102871815ce7SRobert Mustacchi break; 102971815ce7SRobert Mustacchi default: 103071815ce7SRobert Mustacchi panic("encountered suspicious, previously rejected DF " 103171815ce7SRobert Mustacchi "rev: 0x%x", df->adf_rev); 103271815ce7SRobert Mustacchi } 103371815ce7SRobert Mustacchi } 103471815ce7SRobert Mustacchi 103571815ce7SRobert Mustacchi /* 1036dd23d762SRobert Mustacchi * The purpose of this function is to map CCMs to the corresponding CCDs that 1037dd23d762SRobert Mustacchi * exist. This is not an obvious thing as there is no direct mapping in the data 1038dd23d762SRobert Mustacchi * fabric between these IDs. 1039dd23d762SRobert Mustacchi * 1040dd23d762SRobert Mustacchi * Prior to DFv4, a given CCM was only ever connected to at most one CCD. 1041dd23d762SRobert Mustacchi * Starting in DFv4 a given CCM may have one or two SDP (scalable data ports) 1042dd23d762SRobert Mustacchi * that connect to CCDs. These may be connected to the same CCD or a different 1043dd23d762SRobert Mustacchi * one. When both ports are enabled we must check whether or not the port is 1044dd23d762SRobert Mustacchi * considered to be in wide mode. When wide mode is enabled then the two ports 1045dd23d762SRobert Mustacchi * are connected to a single CCD. If wide mode is disabled then the two ports 1046dd23d762SRobert Mustacchi * are connected to separate CCDs. 1047dd23d762SRobert Mustacchi * 1048dd23d762SRobert Mustacchi * The physical number of a CCD, which is how we determine the SMN aperture to 1049dd23d762SRobert Mustacchi * use, is based on the CCM ID. In most sockets we have seen up to a maximum of 1050dd23d762SRobert Mustacchi * 8 CCMs. When a CCM is connected to more than one CCD we have determined based 1051dd23d762SRobert Mustacchi * on some hints from AMD's ACPI information that the numbering is assumed to be 1052dd23d762SRobert Mustacchi * that CCM's number plus the total number of CCMs. 1053dd23d762SRobert Mustacchi * 1054dd23d762SRobert Mustacchi * More concretely, the SP5 Genoa/Bergamo Zen 4 platform has 8 CCMs. When there 1055dd23d762SRobert Mustacchi * are more than 8 CCDs installed then CCM 0 maps to CCDs 0 and 8. CCM 1 to CCDs 1056dd23d762SRobert Mustacchi * 1 and 9, etc. CCMs 4-7 map 1:1 to CCDs 4-7. However, the placement of CCDs 1057dd23d762SRobert Mustacchi * within the package has changed across generations. 1058dd23d762SRobert Mustacchi * 1059dd23d762SRobert Mustacchi * Notably in Rome and Milan (Zen 2/3) it appears that each quadrant had an 1060dd23d762SRobert Mustacchi * increasing number of CCDs. So CCDs 0/1 were together, 2/3, 4/5, and 6/7. This 1061dd23d762SRobert Mustacchi * meant that in cases where only a subset of CCDs were populated it'd forcibly 1062dd23d762SRobert Mustacchi * disable the higher CCD in a group (but with DFv3 the CCM would still be 1063dd23d762SRobert Mustacchi * enabled). So a 4 CCD config would generally enable CCDs 0, 2, 4, and 6 say. 1064dd23d762SRobert Mustacchi * This was almost certainly done to balance the NUMA config. 1065dd23d762SRobert Mustacchi * 1066dd23d762SRobert Mustacchi * Instead, starting in Genoa (Zen 4) the CCMs are round-robined around the 1067dd23d762SRobert Mustacchi * quadrants so CCMs (CCDs) 0 (0/8) and 4 (4) are together, 1 (1/9) and 5 (5), 1068dd23d762SRobert Mustacchi * etc. This is also why we more often see disabled CCMs in Genoa, but not in 1069dd23d762SRobert Mustacchi * Rome/Milan. 1070dd23d762SRobert Mustacchi * 1071dd23d762SRobert Mustacchi * When we're operating in wide mode and therefore both SDPs are connected to a 1072dd23d762SRobert Mustacchi * single CCD, we've always found that the lower CCD index will be used by the 1073dd23d762SRobert Mustacchi * system and the higher one is not considered present. Therefore, when 1074dd23d762SRobert Mustacchi * operating in wide mode, we need to make sure that whenever we have a non-zero 1075dd23d762SRobert Mustacchi * value for SDPs being connected that we rewrite this to only appear as a 1076dd23d762SRobert Mustacchi * single CCD is present. It's conceivable (though hard to imagine) that we 1077dd23d762SRobert Mustacchi * could get a value of 0b10 indicating that only the upper SDP link is active 1078dd23d762SRobert Mustacchi * for some reason. 1079dd23d762SRobert Mustacchi */ 1080dd23d762SRobert Mustacchi 1081dd23d762SRobert Mustacchi static void 1082dd23d762SRobert Mustacchi amdzen_setup_df_ccm(amdzen_t *azn, amdzen_df_t *df, amdzen_df_ent_t *dfe, 1083dd23d762SRobert Mustacchi uint32_t ccmno) 1084dd23d762SRobert Mustacchi { 1085dd23d762SRobert Mustacchi amdzen_ccm_data_t *ccm = &dfe->adfe_data.aded_ccm; 1086dd23d762SRobert Mustacchi uint32_t ccd_en; 1087dd23d762SRobert Mustacchi 1088dd23d762SRobert Mustacchi if (df->adf_rev >= DF_REV_4) { 1089dd23d762SRobert Mustacchi uint32_t val = amdzen_df_read32(azn, df, dfe->adfe_inst_id, 1090dd23d762SRobert Mustacchi DF_CCD_EN_V4); 1091dd23d762SRobert Mustacchi ccd_en = DF_CCD_EN_V4_GET_CCD_EN(val); 1092dd23d762SRobert Mustacchi 1093dd23d762SRobert Mustacchi val = amdzen_df_read32(azn, df, dfe->adfe_inst_id, 1094dd23d762SRobert Mustacchi DF_CCMCFG4_V4); 1095dd23d762SRobert Mustacchi if (DF_CCMCFG4_V4_GET_WIDE_EN(val) != 0 && ccd_en != 0) { 1096dd23d762SRobert Mustacchi ccd_en = 0x1; 1097dd23d762SRobert Mustacchi } 1098dd23d762SRobert Mustacchi } else { 1099dd23d762SRobert Mustacchi ccd_en = 0x1; 1100dd23d762SRobert Mustacchi } 1101dd23d762SRobert Mustacchi 1102dd23d762SRobert Mustacchi for (uint32_t i = 0; i < DF_MAX_CCDS_PER_CCM; i++) { 1103dd23d762SRobert Mustacchi ccm->acd_ccd_en[i] = (ccd_en & (1 << i)) != 0; 1104dd23d762SRobert Mustacchi if (ccm->acd_ccd_en[i] == 0) 1105dd23d762SRobert Mustacchi continue; 1106dd23d762SRobert Mustacchi ccm->acd_ccd_id[i] = ccmno + i * df->adf_nccm; 1107dd23d762SRobert Mustacchi ccm->acd_nccds++; 1108dd23d762SRobert Mustacchi } 1109dd23d762SRobert Mustacchi } 1110dd23d762SRobert Mustacchi 1111dd23d762SRobert Mustacchi /* 1112047043c2SRobert Mustacchi * Initialize our knowledge about a given series of nodes on the data fabric. 1113047043c2SRobert Mustacchi */ 1114047043c2SRobert Mustacchi static void 1115047043c2SRobert Mustacchi amdzen_setup_df(amdzen_t *azn, amdzen_df_t *df) 1116047043c2SRobert Mustacchi { 1117047043c2SRobert Mustacchi uint_t i; 1118dd23d762SRobert Mustacchi uint32_t val, ccmno; 1119047043c2SRobert Mustacchi 112071815ce7SRobert Mustacchi amdzen_determine_df_vers(azn, df); 112171815ce7SRobert Mustacchi 112271815ce7SRobert Mustacchi switch (df->adf_rev) { 112371815ce7SRobert Mustacchi case DF_REV_2: 112471815ce7SRobert Mustacchi case DF_REV_3: 112571815ce7SRobert Mustacchi case DF_REV_3P5: 112671815ce7SRobert Mustacchi val = amdzen_df_read32_bcast(azn, df, DF_CFG_ADDR_CTL_V2); 112771815ce7SRobert Mustacchi break; 112871815ce7SRobert Mustacchi case DF_REV_4: 112971815ce7SRobert Mustacchi val = amdzen_df_read32_bcast(azn, df, DF_CFG_ADDR_CTL_V4); 113071815ce7SRobert Mustacchi break; 113171815ce7SRobert Mustacchi default: 113271815ce7SRobert Mustacchi dev_err(azn->azn_dip, CE_WARN, "encountered unsupported DF " 113371815ce7SRobert Mustacchi "revision: 0x%x", df->adf_rev); 113471815ce7SRobert Mustacchi return; 113571815ce7SRobert Mustacchi } 113671815ce7SRobert Mustacchi df->adf_nb_busno = DF_CFG_ADDR_CTL_GET_BUS_NUM(val); 113771815ce7SRobert Mustacchi val = amdzen_df_read32_bcast(azn, df, DF_FBICNT); 113871815ce7SRobert Mustacchi df->adf_nents = DF_FBICNT_GET_COUNT(val); 1139047043c2SRobert Mustacchi if (df->adf_nents == 0) 1140047043c2SRobert Mustacchi return; 1141047043c2SRobert Mustacchi df->adf_ents = kmem_zalloc(sizeof (amdzen_df_ent_t) * df->adf_nents, 1142047043c2SRobert Mustacchi KM_SLEEP); 1143047043c2SRobert Mustacchi 1144047043c2SRobert Mustacchi for (i = 0; i < df->adf_nents; i++) { 1145047043c2SRobert Mustacchi amdzen_df_ent_t *dfe = &df->adf_ents[i]; 1146047043c2SRobert Mustacchi uint8_t inst = i; 1147047043c2SRobert Mustacchi 1148047043c2SRobert Mustacchi /* 1149047043c2SRobert Mustacchi * Unfortunately, Rome uses a discontinuous instance ID pattern 1150047043c2SRobert Mustacchi * while everything else we can find uses a contiguous instance 1151047043c2SRobert Mustacchi * ID pattern. This means that for Rome, we need to adjust the 1152047043c2SRobert Mustacchi * indexes that we iterate over, though the total number of 115371815ce7SRobert Mustacchi * entries is right. This was carried over into Milan, but not 115471815ce7SRobert Mustacchi * Genoa. 1155047043c2SRobert Mustacchi */ 1156e9abe9d6SRobert Mustacchi if (amdzen_is_rome_style(df->adf_funcs[0]->azns_did)) { 11577c3513e0SRobert Mustacchi if (inst >= ARRAY_SIZE(amdzen_df_rome_ids)) { 1158047043c2SRobert Mustacchi dev_err(azn->azn_dip, CE_WARN, "Rome family " 1159047043c2SRobert Mustacchi "processor reported more ids than the PPR, " 1160e9abe9d6SRobert Mustacchi "resetting %u to instance zero", inst); 1161047043c2SRobert Mustacchi inst = 0; 1162047043c2SRobert Mustacchi } else { 1163047043c2SRobert Mustacchi inst = amdzen_df_rome_ids[inst]; 1164047043c2SRobert Mustacchi } 1165047043c2SRobert Mustacchi } 1166047043c2SRobert Mustacchi 1167047043c2SRobert Mustacchi dfe->adfe_drvid = inst; 116871815ce7SRobert Mustacchi dfe->adfe_info0 = amdzen_df_read32(azn, df, inst, DF_FBIINFO0); 116971815ce7SRobert Mustacchi dfe->adfe_info1 = amdzen_df_read32(azn, df, inst, DF_FBIINFO1); 117071815ce7SRobert Mustacchi dfe->adfe_info2 = amdzen_df_read32(azn, df, inst, DF_FBIINFO2); 117171815ce7SRobert Mustacchi dfe->adfe_info3 = amdzen_df_read32(azn, df, inst, DF_FBIINFO3); 1172047043c2SRobert Mustacchi 117371815ce7SRobert Mustacchi dfe->adfe_type = DF_FBIINFO0_GET_TYPE(dfe->adfe_info0); 117471815ce7SRobert Mustacchi dfe->adfe_subtype = DF_FBIINFO0_GET_SUBTYPE(dfe->adfe_info0); 117571815ce7SRobert Mustacchi 117671815ce7SRobert Mustacchi /* 117771815ce7SRobert Mustacchi * The enabled flag was not present in Zen 1. Simulate it by 117871815ce7SRobert Mustacchi * checking for a non-zero register instead. 117971815ce7SRobert Mustacchi */ 118071815ce7SRobert Mustacchi if (DF_FBIINFO0_V3_GET_ENABLED(dfe->adfe_info0) || 118171815ce7SRobert Mustacchi (df->adf_rev == DF_REV_2 && dfe->adfe_info0 != 0)) { 1182047043c2SRobert Mustacchi dfe->adfe_flags |= AMDZEN_DFE_F_ENABLED; 1183047043c2SRobert Mustacchi } 118471815ce7SRobert Mustacchi if (DF_FBIINFO0_GET_HAS_MCA(dfe->adfe_info0)) { 1185047043c2SRobert Mustacchi dfe->adfe_flags |= AMDZEN_DFE_F_MCA; 1186047043c2SRobert Mustacchi } 118771815ce7SRobert Mustacchi dfe->adfe_inst_id = DF_FBIINFO3_GET_INSTID(dfe->adfe_info3); 118871815ce7SRobert Mustacchi switch (df->adf_rev) { 118971815ce7SRobert Mustacchi case DF_REV_2: 1190047043c2SRobert Mustacchi dfe->adfe_fabric_id = 119171815ce7SRobert Mustacchi DF_FBIINFO3_V2_GET_BLOCKID(dfe->adfe_info3); 119271815ce7SRobert Mustacchi break; 119371815ce7SRobert Mustacchi case DF_REV_3: 119471815ce7SRobert Mustacchi dfe->adfe_fabric_id = 119571815ce7SRobert Mustacchi DF_FBIINFO3_V3_GET_BLOCKID(dfe->adfe_info3); 119671815ce7SRobert Mustacchi break; 119771815ce7SRobert Mustacchi case DF_REV_3P5: 119871815ce7SRobert Mustacchi dfe->adfe_fabric_id = 119971815ce7SRobert Mustacchi DF_FBIINFO3_V3P5_GET_BLOCKID(dfe->adfe_info3); 120071815ce7SRobert Mustacchi break; 120171815ce7SRobert Mustacchi case DF_REV_4: 120271815ce7SRobert Mustacchi dfe->adfe_fabric_id = 120371815ce7SRobert Mustacchi DF_FBIINFO3_V4_GET_BLOCKID(dfe->adfe_info3); 120471815ce7SRobert Mustacchi break; 120571815ce7SRobert Mustacchi default: 120671815ce7SRobert Mustacchi panic("encountered suspicious, previously rejected DF " 120771815ce7SRobert Mustacchi "rev: 0x%x", df->adf_rev); 120871815ce7SRobert Mustacchi } 1209dd23d762SRobert Mustacchi 1210dd23d762SRobert Mustacchi /* 1211dd23d762SRobert Mustacchi * Record information about a subset of DF entities that we've 1212dd23d762SRobert Mustacchi * found. Currently we're tracking this only for CCMs. 1213dd23d762SRobert Mustacchi */ 1214dd23d762SRobert Mustacchi if ((dfe->adfe_flags & AMDZEN_DFE_F_ENABLED) == 0) 1215dd23d762SRobert Mustacchi continue; 1216dd23d762SRobert Mustacchi 1217dd23d762SRobert Mustacchi if (dfe->adfe_type == DF_TYPE_CCM && 1218dd23d762SRobert Mustacchi dfe->adfe_subtype == DF_CCM_SUBTYPE_CPU) { 1219dd23d762SRobert Mustacchi df->adf_nccm++; 1220dd23d762SRobert Mustacchi } 1221dd23d762SRobert Mustacchi } 1222dd23d762SRobert Mustacchi 1223dd23d762SRobert Mustacchi /* 1224dd23d762SRobert Mustacchi * Now that we have filled in all of our info, attempt to fill in 1225dd23d762SRobert Mustacchi * specific information about different types of instances. 1226dd23d762SRobert Mustacchi */ 1227dd23d762SRobert Mustacchi ccmno = 0; 1228dd23d762SRobert Mustacchi for (uint_t i = 0; i < df->adf_nents; i++) { 1229dd23d762SRobert Mustacchi amdzen_df_ent_t *dfe = &df->adf_ents[i]; 1230dd23d762SRobert Mustacchi 1231dd23d762SRobert Mustacchi if ((dfe->adfe_flags & AMDZEN_DFE_F_ENABLED) == 0) 1232dd23d762SRobert Mustacchi continue; 1233dd23d762SRobert Mustacchi 1234dd23d762SRobert Mustacchi /* 1235dd23d762SRobert Mustacchi * Perform type and sub-type specific initialization. Currently 1236dd23d762SRobert Mustacchi * limited to CCMs. 1237dd23d762SRobert Mustacchi */ 1238dd23d762SRobert Mustacchi switch (dfe->adfe_type) { 1239dd23d762SRobert Mustacchi case DF_TYPE_CCM: 1240dd23d762SRobert Mustacchi amdzen_setup_df_ccm(azn, df, dfe, ccmno); 1241dd23d762SRobert Mustacchi ccmno++; 1242dd23d762SRobert Mustacchi break; 1243dd23d762SRobert Mustacchi default: 1244dd23d762SRobert Mustacchi break; 1245dd23d762SRobert Mustacchi } 1246047043c2SRobert Mustacchi } 1247047043c2SRobert Mustacchi 124871815ce7SRobert Mustacchi amdzen_determine_fabric_decomp(azn, df); 1249047043c2SRobert Mustacchi } 1250047043c2SRobert Mustacchi 1251047043c2SRobert Mustacchi static void 1252047043c2SRobert Mustacchi amdzen_find_nb(amdzen_t *azn, amdzen_df_t *df) 1253047043c2SRobert Mustacchi { 1254047043c2SRobert Mustacchi amdzen_stub_t *stub; 1255047043c2SRobert Mustacchi 1256047043c2SRobert Mustacchi for (stub = list_head(&azn->azn_nb_stubs); stub != NULL; 1257047043c2SRobert Mustacchi stub = list_next(&azn->azn_nb_stubs, stub)) { 1258047043c2SRobert Mustacchi if (stub->azns_bus == df->adf_nb_busno) { 1259047043c2SRobert Mustacchi df->adf_flags |= AMDZEN_DF_F_FOUND_NB; 1260047043c2SRobert Mustacchi df->adf_nb = stub; 1261047043c2SRobert Mustacchi return; 1262047043c2SRobert Mustacchi } 1263047043c2SRobert Mustacchi } 1264047043c2SRobert Mustacchi } 1265047043c2SRobert Mustacchi 1266047043c2SRobert Mustacchi static void 1267dd23d762SRobert Mustacchi amdzen_initpkg_to_apic(amdzen_t *azn, const uint32_t pkg0, const uint32_t pkg7) 1268dd23d762SRobert Mustacchi { 1269dd23d762SRobert Mustacchi uint32_t nsock, nccd, nccx, ncore, nthr, extccx; 1270dd23d762SRobert Mustacchi uint32_t nsock_bits, nccd_bits, nccx_bits, ncore_bits, nthr_bits; 1271dd23d762SRobert Mustacchi amdzen_apic_decomp_t *apic = &azn->azn_apic_decomp; 1272dd23d762SRobert Mustacchi 1273dd23d762SRobert Mustacchi /* 1274dd23d762SRobert Mustacchi * These are all 0 based values, meaning that we need to add one to each 1275dd23d762SRobert Mustacchi * of them. However, we skip this because to calculate the number of 1276dd23d762SRobert Mustacchi * bits to cover an entity we would subtract one. 1277dd23d762SRobert Mustacchi */ 1278dd23d762SRobert Mustacchi nthr = SCFCTP_PMREG_INITPKG0_GET_SMTEN(pkg0); 1279dd23d762SRobert Mustacchi ncore = SCFCTP_PMREG_INITPKG7_GET_N_CORES(pkg7); 1280dd23d762SRobert Mustacchi nccx = SCFCTP_PMREG_INITPKG7_GET_N_CCXS(pkg7); 1281dd23d762SRobert Mustacchi nccd = SCFCTP_PMREG_INITPKG7_GET_N_DIES(pkg7); 1282dd23d762SRobert Mustacchi nsock = SCFCTP_PMREG_INITPKG7_GET_N_SOCKETS(pkg7); 1283dd23d762SRobert Mustacchi 1284dd23d762SRobert Mustacchi if (uarchrev_uarch(cpuid_getuarchrev(CPU)) >= X86_UARCH_AMD_ZEN4) { 1285dd23d762SRobert Mustacchi extccx = SCFCTP_PMREG_INITPKG7_ZEN4_GET_16TAPIC(pkg7); 1286dd23d762SRobert Mustacchi } else { 1287dd23d762SRobert Mustacchi extccx = 0; 1288dd23d762SRobert Mustacchi } 1289dd23d762SRobert Mustacchi 1290dd23d762SRobert Mustacchi nthr_bits = highbit(nthr); 1291dd23d762SRobert Mustacchi ncore_bits = highbit(ncore); 1292dd23d762SRobert Mustacchi nccx_bits = highbit(nccx); 1293dd23d762SRobert Mustacchi nccd_bits = highbit(nccd); 1294dd23d762SRobert Mustacchi nsock_bits = highbit(nsock); 1295dd23d762SRobert Mustacchi 1296dd23d762SRobert Mustacchi apic->aad_thread_shift = 0; 1297dd23d762SRobert Mustacchi apic->aad_thread_mask = (1 << nthr_bits) - 1; 1298dd23d762SRobert Mustacchi 1299dd23d762SRobert Mustacchi apic->aad_core_shift = nthr_bits; 1300dd23d762SRobert Mustacchi if (ncore_bits > 0) { 1301dd23d762SRobert Mustacchi apic->aad_core_mask = (1 << ncore_bits) - 1; 1302dd23d762SRobert Mustacchi apic->aad_core_mask <<= apic->aad_core_shift; 1303dd23d762SRobert Mustacchi } else { 1304dd23d762SRobert Mustacchi apic->aad_core_mask = 0; 1305dd23d762SRobert Mustacchi } 1306dd23d762SRobert Mustacchi 1307dd23d762SRobert Mustacchi /* 1308dd23d762SRobert Mustacchi * The APIC_16T_MODE bit indicates that the total shift to start the CCX 1309dd23d762SRobert Mustacchi * should be at 4 bits if it's not. It doesn't mean that the CCX portion 1310dd23d762SRobert Mustacchi * of the value should take up four bits. In the common Genoa case, 1311dd23d762SRobert Mustacchi * nccx_bits will be zero. 1312dd23d762SRobert Mustacchi */ 1313dd23d762SRobert Mustacchi apic->aad_ccx_shift = apic->aad_core_shift + ncore_bits; 1314dd23d762SRobert Mustacchi if (extccx != 0 && apic->aad_ccx_shift < 4) { 1315dd23d762SRobert Mustacchi apic->aad_ccx_shift = 4; 1316dd23d762SRobert Mustacchi } 1317dd23d762SRobert Mustacchi if (nccx_bits > 0) { 1318dd23d762SRobert Mustacchi apic->aad_ccx_mask = (1 << nccx_bits) - 1; 1319dd23d762SRobert Mustacchi apic->aad_ccx_mask <<= apic->aad_ccx_shift; 1320dd23d762SRobert Mustacchi } else { 1321dd23d762SRobert Mustacchi apic->aad_ccx_mask = 0; 1322dd23d762SRobert Mustacchi } 1323dd23d762SRobert Mustacchi 1324dd23d762SRobert Mustacchi apic->aad_ccd_shift = apic->aad_ccx_shift + nccx_bits; 1325dd23d762SRobert Mustacchi if (nccd_bits > 0) { 1326dd23d762SRobert Mustacchi apic->aad_ccd_mask = (1 << nccd_bits) - 1; 1327dd23d762SRobert Mustacchi apic->aad_ccd_mask <<= apic->aad_ccd_shift; 1328dd23d762SRobert Mustacchi } else { 1329dd23d762SRobert Mustacchi apic->aad_ccd_mask = 0; 1330dd23d762SRobert Mustacchi } 1331dd23d762SRobert Mustacchi 1332dd23d762SRobert Mustacchi apic->aad_sock_shift = apic->aad_ccd_shift + nccd_bits; 1333dd23d762SRobert Mustacchi if (nsock_bits > 0) { 1334dd23d762SRobert Mustacchi apic->aad_sock_mask = (1 << nsock_bits) - 1; 1335dd23d762SRobert Mustacchi apic->aad_sock_mask <<= apic->aad_sock_shift; 1336dd23d762SRobert Mustacchi } else { 1337dd23d762SRobert Mustacchi apic->aad_sock_mask = 0; 1338dd23d762SRobert Mustacchi } 1339dd23d762SRobert Mustacchi 1340dd23d762SRobert Mustacchi /* 1341dd23d762SRobert Mustacchi * Currently all supported Zen 2+ platforms only have a single die per 1342dd23d762SRobert Mustacchi * socket as compared to Zen 1. So this is always kept at zero. 1343dd23d762SRobert Mustacchi */ 1344dd23d762SRobert Mustacchi apic->aad_die_mask = 0; 1345dd23d762SRobert Mustacchi apic->aad_die_shift = 0; 1346dd23d762SRobert Mustacchi } 1347dd23d762SRobert Mustacchi 1348dd23d762SRobert Mustacchi /* 1349dd23d762SRobert Mustacchi * We would like to determine what the logical APIC decomposition is on Zen 3 1350dd23d762SRobert Mustacchi * and newer family parts. While there is information added to CPUID in the form 1351dd23d762SRobert Mustacchi * of leaf 8X26, that isn't present in Zen 3, so instead we go to what we 1352dd23d762SRobert Mustacchi * believe is the underlying source of the CPUID data. 1353dd23d762SRobert Mustacchi * 1354dd23d762SRobert Mustacchi * Fundamentally there are a series of registers in SMN space that relate to the 1355dd23d762SRobert Mustacchi * SCFCTP. Coincidentally, there is one of these for each core and there are a 1356dd23d762SRobert Mustacchi * pair of related SMN registers. L3::SCFCTP::PMREG_INITPKG0 contains 1357dd23d762SRobert Mustacchi * information about a given's core logical and physical IDs. More interestingly 1358dd23d762SRobert Mustacchi * for this particular case, L3::SCFCTP::PMREG_INITPKG7, contains the overall 1359dd23d762SRobert Mustacchi * total number of logical entities. We've been promised that this has to be 1360dd23d762SRobert Mustacchi * the same across the fabric. That's all well and good, but this begs the 1361dd23d762SRobert Mustacchi * question of how do we actually get there. The above is a core-specific 1362dd23d762SRobert Mustacchi * register and requires that we understand information about which CCDs and 1363dd23d762SRobert Mustacchi * CCXs are actually present. 1364dd23d762SRobert Mustacchi * 1365dd23d762SRobert Mustacchi * So we are starting with a data fabric that has some CCM present. The CCM 1366dd23d762SRobert Mustacchi * entries in the data fabric may be tagged with our ENABLED flag. 1367dd23d762SRobert Mustacchi * Unfortunately, that can be true regardless of whether or not it's actually 1368dd23d762SRobert Mustacchi * present or not. As a result, we go to another chunk of SMN space registers, 1369dd23d762SRobert Mustacchi * SMU::PWR. These contain information about the CCDs, the physical cores that 1370dd23d762SRobert Mustacchi * are enabled, and related. So we will first walk the DF entities and see if we 1371dd23d762SRobert Mustacchi * can read its SMN::PWR::CCD_DIE_ID. If we get back a value of all 1s then 1372dd23d762SRobert Mustacchi * there is nothing present. Otherwise, we should get back something that 1373dd23d762SRobert Mustacchi * matches information in the data fabric. 1374dd23d762SRobert Mustacchi * 1375dd23d762SRobert Mustacchi * With that in hand, we can read the SMU::PWR::CORE_ENABLE register to 1376dd23d762SRobert Mustacchi * determine which physical cores are enabled in the CCD/CCX. That will finally 1377dd23d762SRobert Mustacchi * give us an index to get to our friend INITPKG7. 1378dd23d762SRobert Mustacchi */ 1379dd23d762SRobert Mustacchi static boolean_t 1380dd23d762SRobert Mustacchi amdzen_determine_apic_decomp_initpkg(amdzen_t *azn) 1381dd23d762SRobert Mustacchi { 1382dd23d762SRobert Mustacchi amdzen_df_t *df = &azn->azn_dfs[0]; 1383dd23d762SRobert Mustacchi uint32_t ccdno = 0; 1384dd23d762SRobert Mustacchi 1385dd23d762SRobert Mustacchi for (uint_t i = 0; i < df->adf_nents; i++) { 1386dd23d762SRobert Mustacchi const amdzen_df_ent_t *ent = &df->adf_ents[i]; 1387dd23d762SRobert Mustacchi if ((ent->adfe_flags & AMDZEN_DFE_F_ENABLED) == 0) 1388dd23d762SRobert Mustacchi continue; 1389dd23d762SRobert Mustacchi 1390dd23d762SRobert Mustacchi if (ent->adfe_type == DF_TYPE_CCM && 1391dd23d762SRobert Mustacchi ent->adfe_subtype == DF_CCM_SUBTYPE_CPU) { 1392dd23d762SRobert Mustacchi uint32_t val, nccx, pkg7, pkg0; 1393dd23d762SRobert Mustacchi smn_reg_t die_reg, thrcfg_reg, core_reg; 1394dd23d762SRobert Mustacchi smn_reg_t pkg7_reg, pkg0_reg; 1395dd23d762SRobert Mustacchi int core_bit; 1396dd23d762SRobert Mustacchi uint8_t pccxno, pcoreno; 1397dd23d762SRobert Mustacchi 1398dd23d762SRobert Mustacchi die_reg = SMUPWR_CCD_DIE_ID(ccdno); 1399dd23d762SRobert Mustacchi val = amdzen_smn_read(azn, df, die_reg); 1400dd23d762SRobert Mustacchi if (val == SMN_EINVAL32) { 1401dd23d762SRobert Mustacchi ccdno++; 1402dd23d762SRobert Mustacchi continue; 1403dd23d762SRobert Mustacchi } 1404dd23d762SRobert Mustacchi 1405dd23d762SRobert Mustacchi ASSERT3U(SMUPWR_CCD_DIE_ID_GET(val), ==, ccdno); 1406dd23d762SRobert Mustacchi 1407dd23d762SRobert Mustacchi /* 1408dd23d762SRobert Mustacchi * This die actually exists. Switch over to the core 1409dd23d762SRobert Mustacchi * enable register to find one to ask about physically. 1410dd23d762SRobert Mustacchi */ 1411dd23d762SRobert Mustacchi thrcfg_reg = SMUPWR_THREAD_CFG(ccdno); 1412dd23d762SRobert Mustacchi val = amdzen_smn_read(azn, df, thrcfg_reg); 1413dd23d762SRobert Mustacchi nccx = SMUPWR_THREAD_CFG_GET_COMPLEX_COUNT(val) + 1; 1414dd23d762SRobert Mustacchi core_reg = SMUPWR_CORE_EN(ccdno); 1415dd23d762SRobert Mustacchi val = amdzen_smn_read(azn, df, core_reg); 1416dd23d762SRobert Mustacchi if (val == 0) { 1417dd23d762SRobert Mustacchi ccdno++; 1418dd23d762SRobert Mustacchi continue; 1419dd23d762SRobert Mustacchi } 1420dd23d762SRobert Mustacchi 1421dd23d762SRobert Mustacchi /* 1422dd23d762SRobert Mustacchi * There exists an enabled physical core. Find the first 1423dd23d762SRobert Mustacchi * index of it and map it to the corresponding CCD and 1424dd23d762SRobert Mustacchi * CCX. ddi_ffs is the bit index, but we want the 1425dd23d762SRobert Mustacchi * physical core number, hence the -1. 1426dd23d762SRobert Mustacchi */ 1427dd23d762SRobert Mustacchi core_bit = ddi_ffs(val); 1428dd23d762SRobert Mustacchi ASSERT3S(core_bit, !=, 0); 1429dd23d762SRobert Mustacchi pcoreno = core_bit - 1; 1430dd23d762SRobert Mustacchi 1431dd23d762SRobert Mustacchi /* 1432dd23d762SRobert Mustacchi * Unfortunately SMU::PWR::THREAD_CONFIGURATION gives us 1433dd23d762SRobert Mustacchi * the Number of logical cores that are present in the 1434dd23d762SRobert Mustacchi * complex, not the total number of physical cores. So 1435dd23d762SRobert Mustacchi * here we need to encode that in Zen 3+ the number of 1436dd23d762SRobert Mustacchi * cores per CCX is a maximum of 8. Right now we do 1437dd23d762SRobert Mustacchi * assume that the physical and logical ccx numbering is 1438dd23d762SRobert Mustacchi * equivalent (we have no other way of knowing if it is 1439dd23d762SRobert Mustacchi * or isn't right now) and that we'd always have CCX0 1440dd23d762SRobert Mustacchi * before CCX1. AMD seems to suggest we can assume this, 1441dd23d762SRobert Mustacchi * though it is a worrisome assumption. 1442dd23d762SRobert Mustacchi */ 1443dd23d762SRobert Mustacchi pccxno = pcoreno / 8; 1444dd23d762SRobert Mustacchi ASSERT3U(pccxno, <, nccx); 1445dd23d762SRobert Mustacchi pkg7_reg = SCFCTP_PMREG_INITPKG7(ccdno, pccxno, 1446dd23d762SRobert Mustacchi pcoreno); 1447dd23d762SRobert Mustacchi pkg7 = amdzen_smn_read(azn, df, pkg7_reg); 1448dd23d762SRobert Mustacchi pkg0_reg = SCFCTP_PMREG_INITPKG0(ccdno, pccxno, 1449dd23d762SRobert Mustacchi pcoreno); 1450dd23d762SRobert Mustacchi pkg0 = amdzen_smn_read(azn, df, pkg0_reg); 1451dd23d762SRobert Mustacchi amdzen_initpkg_to_apic(azn, pkg0, pkg7); 1452dd23d762SRobert Mustacchi return (B_TRUE); 1453dd23d762SRobert Mustacchi } 1454dd23d762SRobert Mustacchi } 1455dd23d762SRobert Mustacchi 1456dd23d762SRobert Mustacchi return (B_FALSE); 1457dd23d762SRobert Mustacchi } 1458dd23d762SRobert Mustacchi 1459dd23d762SRobert Mustacchi /* 1460dd23d762SRobert Mustacchi * We have the fun job of trying to figure out what the correct form of the APIC 1461dd23d762SRobert Mustacchi * decomposition should be and how to break that into its logical components. 1462dd23d762SRobert Mustacchi * The way that we get at this is generation-specific unfortunately. Here's how 1463dd23d762SRobert Mustacchi * it works out: 1464dd23d762SRobert Mustacchi * 1465dd23d762SRobert Mustacchi * Zen 1-2 This era of CPUs are deceptively simple. The PPR for a given 1466dd23d762SRobert Mustacchi * family defines exactly how the APIC ID is broken into logical 1467dd23d762SRobert Mustacchi * components and it's fixed. That is, depending on whether or 1468dd23d762SRobert Mustacchi * not SMT is enabled. Zen 1 and Zen 2 use different schemes for 1469dd23d762SRobert Mustacchi * constructing this. The way that we're supposed to check if SMT 1470dd23d762SRobert Mustacchi * is enabled is to use AMD leaf 8X1E and ask how many threads per 1471dd23d762SRobert Mustacchi * core there are. We use the x86 feature set to determine that 1472dd23d762SRobert Mustacchi * instead. 1473dd23d762SRobert Mustacchi * 1474dd23d762SRobert Mustacchi * More specifically the Zen 1 scheme is 7 bits long. The bits have 1475dd23d762SRobert Mustacchi * the following meanings. 1476dd23d762SRobert Mustacchi * 1477dd23d762SRobert Mustacchi * [6] Socket ID 1478dd23d762SRobert Mustacchi * [5:4] Node ID 1479dd23d762SRobert Mustacchi * [3] Logical CCX ID 1480dd23d762SRobert Mustacchi * With SMT Without SMT 1481dd23d762SRobert Mustacchi * [2:1] Logical Core ID [2] hardcoded to zero 1482dd23d762SRobert Mustacchi * [0] Thread ID [1:0] Logical Core ID 1483dd23d762SRobert Mustacchi * 1484dd23d762SRobert Mustacchi * The following is the Zen 2 scheme assuming SMT. The Zen 2 scheme 1485dd23d762SRobert Mustacchi * without SMT shifts everything to the right by one bit. 1486dd23d762SRobert Mustacchi * 1487dd23d762SRobert Mustacchi * [7] Socket ID 1488dd23d762SRobert Mustacchi * [6:4] Logical CCD ID 1489dd23d762SRobert Mustacchi * [3] Logical CCX ID 1490dd23d762SRobert Mustacchi * [2:1] Logical Core ID 1491dd23d762SRobert Mustacchi * [0] Thread ID 1492dd23d762SRobert Mustacchi * 1493dd23d762SRobert Mustacchi * Zen 3 Zen 3 CPUs moved past the fixed APIC ID format that Zen 1 and 1494dd23d762SRobert Mustacchi * Zen 2 had, but also don't give us the nice way of discovering 1495dd23d762SRobert Mustacchi * this via CPUID that Zen 4 did. The APIC ID id uses a given 1496dd23d762SRobert Mustacchi * number of bits for each logical component that exists, but the 1497dd23d762SRobert Mustacchi * exact number varies based on what's actually present. To get at 1498dd23d762SRobert Mustacchi * this we use a piece of data that is embedded in the SCFCTP 1499dd23d762SRobert Mustacchi * (Scalable Control Fabric, Clocks, Test, Power Gating). This can 1500dd23d762SRobert Mustacchi * be used to determine how many logical entities of each kind the 1501dd23d762SRobert Mustacchi * system thinks exist. While we could use the various CPUID 1502dd23d762SRobert Mustacchi * topology items to try to speed this up, they don't tell us the 1503dd23d762SRobert Mustacchi * die information that we need to do this. 1504dd23d762SRobert Mustacchi * 1505dd23d762SRobert Mustacchi * Zen 4+ Zen 4 introduced CPUID leaf 8000_0026h which gives us a means 1506dd23d762SRobert Mustacchi * for determining how to extract the CCD, CCX, and related pieces 1507dd23d762SRobert Mustacchi * out of the device. One thing we have to be aware of is that when 1508dd23d762SRobert Mustacchi * the CCD and CCX shift are the same, that means that there is 1509dd23d762SRobert Mustacchi * only a single CCX and therefore have to take that into account 1510dd23d762SRobert Mustacchi * appropriately. This is the case generally on Zen 4 platforms, 1511dd23d762SRobert Mustacchi * but not on Bergamo. Until we can confirm the actual CPUID leaf 1512dd23d762SRobert Mustacchi * values that we receive in the cases of Bergamo and others, we 1513dd23d762SRobert Mustacchi * opt instead to use the same SCFCTP scheme as Zen 3. 1514dd23d762SRobert Mustacchi */ 1515dd23d762SRobert Mustacchi static boolean_t 1516dd23d762SRobert Mustacchi amdzen_determine_apic_decomp(amdzen_t *azn) 1517dd23d762SRobert Mustacchi { 1518dd23d762SRobert Mustacchi x86_uarchrev_t uarchrev = cpuid_getuarchrev(CPU); 1519dd23d762SRobert Mustacchi amdzen_apic_decomp_t *apic = &azn->azn_apic_decomp; 1520dd23d762SRobert Mustacchi boolean_t smt = is_x86_feature(x86_featureset, X86FSET_HTT); 1521dd23d762SRobert Mustacchi 1522dd23d762SRobert Mustacchi switch (uarchrev_uarch(uarchrev)) { 1523dd23d762SRobert Mustacchi case X86_UARCH_AMD_ZEN1: 1524dd23d762SRobert Mustacchi case X86_UARCH_AMD_ZENPLUS: 1525dd23d762SRobert Mustacchi apic->aad_sock_mask = 0x40; 1526dd23d762SRobert Mustacchi apic->aad_sock_shift = 6; 1527dd23d762SRobert Mustacchi apic->aad_die_mask = 0x30; 1528dd23d762SRobert Mustacchi apic->aad_die_shift = 4; 1529dd23d762SRobert Mustacchi apic->aad_ccd_mask = 0; 1530dd23d762SRobert Mustacchi apic->aad_ccd_shift = 0; 1531dd23d762SRobert Mustacchi apic->aad_ccx_mask = 0x08; 1532dd23d762SRobert Mustacchi apic->aad_ccx_shift = 3; 1533dd23d762SRobert Mustacchi 1534dd23d762SRobert Mustacchi if (smt) { 1535dd23d762SRobert Mustacchi apic->aad_core_mask = 0x06; 1536dd23d762SRobert Mustacchi apic->aad_core_shift = 1; 1537dd23d762SRobert Mustacchi apic->aad_thread_mask = 0x1; 1538dd23d762SRobert Mustacchi apic->aad_thread_shift = 0; 1539dd23d762SRobert Mustacchi } else { 1540dd23d762SRobert Mustacchi apic->aad_core_mask = 0x03; 1541dd23d762SRobert Mustacchi apic->aad_core_shift = 0; 1542dd23d762SRobert Mustacchi apic->aad_thread_mask = 0; 1543dd23d762SRobert Mustacchi apic->aad_thread_shift = 0; 1544dd23d762SRobert Mustacchi } 1545dd23d762SRobert Mustacchi break; 1546dd23d762SRobert Mustacchi case X86_UARCH_AMD_ZEN2: 1547dd23d762SRobert Mustacchi if (smt) { 1548dd23d762SRobert Mustacchi apic->aad_sock_mask = 0x80; 1549dd23d762SRobert Mustacchi apic->aad_sock_shift = 7; 1550dd23d762SRobert Mustacchi apic->aad_die_mask = 0; 1551dd23d762SRobert Mustacchi apic->aad_die_shift = 0; 1552dd23d762SRobert Mustacchi apic->aad_ccd_mask = 0x70; 1553dd23d762SRobert Mustacchi apic->aad_ccd_shift = 4; 1554dd23d762SRobert Mustacchi apic->aad_ccx_mask = 0x08; 1555dd23d762SRobert Mustacchi apic->aad_ccx_shift = 3; 1556dd23d762SRobert Mustacchi apic->aad_core_mask = 0x06; 1557dd23d762SRobert Mustacchi apic->aad_core_shift = 1; 1558dd23d762SRobert Mustacchi apic->aad_thread_mask = 0x01; 1559dd23d762SRobert Mustacchi apic->aad_thread_shift = 0; 1560dd23d762SRobert Mustacchi } else { 1561dd23d762SRobert Mustacchi apic->aad_sock_mask = 0x40; 1562dd23d762SRobert Mustacchi apic->aad_sock_shift = 6; 1563dd23d762SRobert Mustacchi apic->aad_die_mask = 0; 1564dd23d762SRobert Mustacchi apic->aad_die_shift = 0; 1565dd23d762SRobert Mustacchi apic->aad_ccd_mask = 0x38; 1566dd23d762SRobert Mustacchi apic->aad_ccd_shift = 3; 1567dd23d762SRobert Mustacchi apic->aad_ccx_mask = 0x04; 1568dd23d762SRobert Mustacchi apic->aad_ccx_shift = 2; 1569dd23d762SRobert Mustacchi apic->aad_core_mask = 0x3; 1570dd23d762SRobert Mustacchi apic->aad_core_shift = 0; 1571dd23d762SRobert Mustacchi apic->aad_thread_mask = 0; 1572dd23d762SRobert Mustacchi apic->aad_thread_shift = 0; 1573dd23d762SRobert Mustacchi } 1574dd23d762SRobert Mustacchi break; 1575dd23d762SRobert Mustacchi case X86_UARCH_AMD_ZEN3: 1576dd23d762SRobert Mustacchi case X86_UARCH_AMD_ZEN4: 1577dd23d762SRobert Mustacchi return (amdzen_determine_apic_decomp_initpkg(azn)); 1578dd23d762SRobert Mustacchi default: 1579dd23d762SRobert Mustacchi return (B_FALSE); 1580dd23d762SRobert Mustacchi } 1581dd23d762SRobert Mustacchi return (B_TRUE); 1582dd23d762SRobert Mustacchi } 1583dd23d762SRobert Mustacchi 1584dd23d762SRobert Mustacchi /* 1585dd23d762SRobert Mustacchi * Snapshot the number of cores that can exist in a CCX based on the Zen 1586dd23d762SRobert Mustacchi * microarchitecture revision. In Zen 1-4 this has been a constant number 1587dd23d762SRobert Mustacchi * regardless of the actual CPU Family. 1588dd23d762SRobert Mustacchi */ 1589dd23d762SRobert Mustacchi static void 1590dd23d762SRobert Mustacchi amdzen_determine_ncore_per_ccx(amdzen_t *azn) 1591dd23d762SRobert Mustacchi { 1592dd23d762SRobert Mustacchi x86_uarchrev_t uarchrev = cpuid_getuarchrev(CPU); 1593dd23d762SRobert Mustacchi 1594dd23d762SRobert Mustacchi switch (uarchrev_uarch(uarchrev)) { 1595dd23d762SRobert Mustacchi case X86_UARCH_AMD_ZEN1: 1596dd23d762SRobert Mustacchi case X86_UARCH_AMD_ZENPLUS: 1597dd23d762SRobert Mustacchi case X86_UARCH_AMD_ZEN2: 1598dd23d762SRobert Mustacchi azn->azn_ncore_per_ccx = 4; 1599dd23d762SRobert Mustacchi break; 1600dd23d762SRobert Mustacchi case X86_UARCH_AMD_ZEN3: 1601dd23d762SRobert Mustacchi case X86_UARCH_AMD_ZEN4: 1602dd23d762SRobert Mustacchi azn->azn_ncore_per_ccx = 8; 1603dd23d762SRobert Mustacchi break; 1604dd23d762SRobert Mustacchi default: 1605dd23d762SRobert Mustacchi panic("asked about non-Zen uarch"); 1606dd23d762SRobert Mustacchi } 1607dd23d762SRobert Mustacchi } 1608dd23d762SRobert Mustacchi 1609dd23d762SRobert Mustacchi /* 1610dd23d762SRobert Mustacchi * We need to be careful using this function as different AMD generations have 1611dd23d762SRobert Mustacchi * acted in different ways when there is a missing CCD. We've found that in 1612dd23d762SRobert Mustacchi * hardware where the CCM is enabled but there is no CCD attached, it generally 1613dd23d762SRobert Mustacchi * is safe (i.e. DFv3 on Rome), but on DFv4 if we ask for a CCD that would 1614dd23d762SRobert Mustacchi * correspond to a disabled CCM then the firmware may inject a fatal error 1615dd23d762SRobert Mustacchi * (which is hopefully something missing in our RAS/MCA-X enablement). 1616dd23d762SRobert Mustacchi * 1617dd23d762SRobert Mustacchi * Put differently if this doesn't correspond to an Enabled CCM and you know the 1618dd23d762SRobert Mustacchi * number of valid CCDs on this, don't use it. 1619dd23d762SRobert Mustacchi */ 1620dd23d762SRobert Mustacchi static boolean_t 1621dd23d762SRobert Mustacchi amdzen_ccd_present(amdzen_t *azn, amdzen_df_t *df, uint32_t ccdno) 1622dd23d762SRobert Mustacchi { 1623dd23d762SRobert Mustacchi smn_reg_t die_reg = SMUPWR_CCD_DIE_ID(ccdno); 1624dd23d762SRobert Mustacchi uint32_t val = amdzen_smn_read(azn, df, die_reg); 1625dd23d762SRobert Mustacchi if (val == SMN_EINVAL32) { 1626dd23d762SRobert Mustacchi return (B_FALSE); 1627dd23d762SRobert Mustacchi } 1628dd23d762SRobert Mustacchi 1629dd23d762SRobert Mustacchi ASSERT3U(ccdno, ==, SMUPWR_CCD_DIE_ID_GET(val)); 1630dd23d762SRobert Mustacchi return (B_TRUE); 1631dd23d762SRobert Mustacchi } 1632dd23d762SRobert Mustacchi 1633dd23d762SRobert Mustacchi /* 1634dd23d762SRobert Mustacchi * Attempt to determine a logical CCD number of a given CCD where we don't have 1635dd23d762SRobert Mustacchi * hardware support for L3::SCFCTP::PMREG_INITPKG* (e.g. pre-Zen 3 systems). 1636dd23d762SRobert Mustacchi * The CCD numbers that we have are the in the physical space. Likely beacuse of 1637dd23d762SRobert Mustacchi * how the orientation of CCM numbers map to physical locations and the layout 1638dd23d762SRobert Mustacchi * of them within the pacakge, we haven't found a good way using the core DFv3 1639dd23d762SRobert Mustacchi * registers to determine if a given CCD is actually present or not as generally 1640dd23d762SRobert Mustacchi * all the CCMs are left enabled. Instead we use SMU::PWR::DIE_ID as a proxy to 1641dd23d762SRobert Mustacchi * determine CCD presence. 1642dd23d762SRobert Mustacchi */ 1643dd23d762SRobert Mustacchi static uint32_t 1644dd23d762SRobert Mustacchi amdzen_ccd_log_id_zen2(amdzen_t *azn, amdzen_df_t *df, 1645dd23d762SRobert Mustacchi const amdzen_df_ent_t *targ) 1646dd23d762SRobert Mustacchi { 1647dd23d762SRobert Mustacchi uint32_t smnid = 0; 1648dd23d762SRobert Mustacchi uint32_t logid = 0; 1649dd23d762SRobert Mustacchi 1650dd23d762SRobert Mustacchi for (uint_t i = 0; i < df->adf_nents; i++) { 1651dd23d762SRobert Mustacchi const amdzen_df_ent_t *ent = &df->adf_ents[i]; 1652dd23d762SRobert Mustacchi 1653dd23d762SRobert Mustacchi if ((ent->adfe_flags & AMDZEN_DFE_F_ENABLED) == 0) { 1654dd23d762SRobert Mustacchi continue; 1655dd23d762SRobert Mustacchi } 1656dd23d762SRobert Mustacchi 1657dd23d762SRobert Mustacchi if (ent->adfe_inst_id == targ->adfe_inst_id) { 1658dd23d762SRobert Mustacchi return (logid); 1659dd23d762SRobert Mustacchi } 1660dd23d762SRobert Mustacchi 1661dd23d762SRobert Mustacchi if (ent->adfe_type == targ->adfe_type && 1662dd23d762SRobert Mustacchi ent->adfe_subtype == targ->adfe_subtype) { 1663dd23d762SRobert Mustacchi boolean_t present = amdzen_ccd_present(azn, df, smnid); 1664dd23d762SRobert Mustacchi smnid++; 1665dd23d762SRobert Mustacchi if (present) { 1666dd23d762SRobert Mustacchi logid++; 1667dd23d762SRobert Mustacchi } 1668dd23d762SRobert Mustacchi } 1669dd23d762SRobert Mustacchi } 1670dd23d762SRobert Mustacchi 1671dd23d762SRobert Mustacchi panic("asked to match against invalid DF entity %p in df %p", targ, df); 1672dd23d762SRobert Mustacchi } 1673dd23d762SRobert Mustacchi 1674dd23d762SRobert Mustacchi static void 1675dd23d762SRobert Mustacchi amdzen_ccd_fill_core_initpkg0(amdzen_t *azn, amdzen_df_t *df, 1676dd23d762SRobert Mustacchi amdzen_topo_ccd_t *ccd, amdzen_topo_ccx_t *ccx, amdzen_topo_core_t *core, 1677dd23d762SRobert Mustacchi boolean_t *ccd_set, boolean_t *ccx_set) 1678dd23d762SRobert Mustacchi { 1679dd23d762SRobert Mustacchi smn_reg_t pkg0_reg; 1680dd23d762SRobert Mustacchi uint32_t pkg0; 1681dd23d762SRobert Mustacchi 1682dd23d762SRobert Mustacchi pkg0_reg = SCFCTP_PMREG_INITPKG0(ccd->atccd_phys_no, ccx->atccx_phys_no, 1683dd23d762SRobert Mustacchi core->atcore_phys_no); 1684dd23d762SRobert Mustacchi pkg0 = amdzen_smn_read(azn, df, pkg0_reg); 1685dd23d762SRobert Mustacchi core->atcore_log_no = SCFCTP_PMREG_INITPKG0_GET_LOG_CORE(pkg0); 1686dd23d762SRobert Mustacchi 1687dd23d762SRobert Mustacchi if (!*ccx_set) { 1688dd23d762SRobert Mustacchi ccx->atccx_log_no = SCFCTP_PMREG_INITPKG0_GET_LOG_CCX(pkg0); 1689dd23d762SRobert Mustacchi *ccx_set = B_TRUE; 1690dd23d762SRobert Mustacchi } 1691dd23d762SRobert Mustacchi 1692dd23d762SRobert Mustacchi if (!*ccd_set) { 1693dd23d762SRobert Mustacchi ccd->atccd_log_no = SCFCTP_PMREG_INITPKG0_GET_LOG_DIE(pkg0); 1694dd23d762SRobert Mustacchi *ccd_set = B_TRUE; 1695dd23d762SRobert Mustacchi } 1696dd23d762SRobert Mustacchi } 1697dd23d762SRobert Mustacchi 1698dd23d762SRobert Mustacchi /* 1699dd23d762SRobert Mustacchi * Attempt to fill in the physical topology information for this given CCD. 1700dd23d762SRobert Mustacchi * There are a few steps to this that we undertake to perform this as follows: 1701dd23d762SRobert Mustacchi * 1702dd23d762SRobert Mustacchi * 1) First we determine whether the CCD is actually present or not by reading 1703dd23d762SRobert Mustacchi * SMU::PWR::DIE_ID. CCDs that are not installed will still have an enabled DF 1704dd23d762SRobert Mustacchi * entry it appears, but the request for the die ID will returns an invalid 1705dd23d762SRobert Mustacchi * read (all 1s). This die ID should match what we think of as the SMN number 1706dd23d762SRobert Mustacchi * below. If not, we're in trouble and the rest of this is in question. 1707dd23d762SRobert Mustacchi * 1708dd23d762SRobert Mustacchi * 2) We use the SMU::PWR registers to determine how many logical and physical 1709dd23d762SRobert Mustacchi * cores are present in this CCD and how they are split amongst the CCX. Here we 1710dd23d762SRobert Mustacchi * need to encode the CPU to CCX core size rankings. Through this process we 1711dd23d762SRobert Mustacchi * determine and fill out which threads and cores are enabled. 1712dd23d762SRobert Mustacchi * 1713dd23d762SRobert Mustacchi * 3) In Zen 3+ we then will read each core's INITPK0 values to ensure that we 1714dd23d762SRobert Mustacchi * have a proper physical to logical mapping, at which point we can fill in the 1715dd23d762SRobert Mustacchi * APIC IDs. For Zen 2, we will set the AMDZEN_TOPO_CCD_F_CORE_PHYS_UNKNOWN to 1716dd23d762SRobert Mustacchi * indicate that we just mapped the first logical processor to the first enabled 1717dd23d762SRobert Mustacchi * core. 1718dd23d762SRobert Mustacchi * 1719dd23d762SRobert Mustacchi * 4) Once we have the logical IDs determined we will construct the APIC ID that 1720dd23d762SRobert Mustacchi * we expect this to have. 1721dd23d762SRobert Mustacchi * 1722dd23d762SRobert Mustacchi * Steps (2) - (4) are intertwined and done together. 1723dd23d762SRobert Mustacchi */ 1724dd23d762SRobert Mustacchi static void 1725dd23d762SRobert Mustacchi amdzen_ccd_fill_topo(amdzen_t *azn, amdzen_df_t *df, amdzen_df_ent_t *ent, 1726dd23d762SRobert Mustacchi amdzen_topo_ccd_t *ccd) 1727dd23d762SRobert Mustacchi { 1728dd23d762SRobert Mustacchi uint32_t val, nccx, core_en, thread_en; 1729dd23d762SRobert Mustacchi uint32_t nlcore_per_ccx, nthreads_per_core; 1730dd23d762SRobert Mustacchi uint32_t sockid, dieid, compid; 1731dd23d762SRobert Mustacchi const uint32_t ccdno = ccd->atccd_phys_no; 1732dd23d762SRobert Mustacchi const x86_uarch_t uarch = uarchrev_uarch(cpuid_getuarchrev(CPU)); 1733dd23d762SRobert Mustacchi boolean_t smt, pkg0_ids, logccd_set = B_FALSE; 1734dd23d762SRobert Mustacchi smn_reg_t reg; 1735dd23d762SRobert Mustacchi 1736dd23d762SRobert Mustacchi ASSERT(MUTEX_HELD(&azn->azn_mutex)); 1737dd23d762SRobert Mustacchi if (!amdzen_ccd_present(azn, df, ccdno)) { 1738dd23d762SRobert Mustacchi ccd->atccd_err = AMDZEN_TOPO_CCD_E_CCD_MISSING; 1739dd23d762SRobert Mustacchi return; 1740dd23d762SRobert Mustacchi } 1741dd23d762SRobert Mustacchi 1742dd23d762SRobert Mustacchi reg = SMUPWR_THREAD_CFG(ccdno); 1743dd23d762SRobert Mustacchi val = amdzen_smn_read(azn, df, reg); 1744dd23d762SRobert Mustacchi nccx = SMUPWR_THREAD_CFG_GET_COMPLEX_COUNT(val) + 1; 1745dd23d762SRobert Mustacchi nlcore_per_ccx = SMUPWR_THREAD_CFG_GET_COMPLEX_COUNT(val) + 1; 1746dd23d762SRobert Mustacchi smt = SMUPWR_THREAD_CFG_GET_SMT_MODE(val); 1747dd23d762SRobert Mustacchi ASSERT3U(nccx, <=, AMDZEN_TOPO_CCD_MAX_CCX); 1748dd23d762SRobert Mustacchi if (smt == SMUPWR_THREAD_CFG_SMT_MODE_SMT) { 1749dd23d762SRobert Mustacchi nthreads_per_core = 2; 1750dd23d762SRobert Mustacchi } else { 1751dd23d762SRobert Mustacchi nthreads_per_core = 1; 1752dd23d762SRobert Mustacchi } 1753dd23d762SRobert Mustacchi 1754dd23d762SRobert Mustacchi reg = SMUPWR_CORE_EN(ccdno); 1755dd23d762SRobert Mustacchi core_en = amdzen_smn_read(azn, df, reg); 1756dd23d762SRobert Mustacchi reg = SMUPWR_THREAD_EN(ccdno); 1757dd23d762SRobert Mustacchi thread_en = amdzen_smn_read(azn, df, reg); 1758dd23d762SRobert Mustacchi 1759dd23d762SRobert Mustacchi /* 1760dd23d762SRobert Mustacchi * The BSP is never enabled in a conventional sense and therefore the 1761dd23d762SRobert Mustacchi * bit is reserved and left as 0. As the BSP should be in the first CCD, 1762dd23d762SRobert Mustacchi * we go through and OR back in the bit lest we think the thread isn't 1763dd23d762SRobert Mustacchi * enabled. 1764dd23d762SRobert Mustacchi */ 1765dd23d762SRobert Mustacchi if (ccdno == 0) { 1766dd23d762SRobert Mustacchi thread_en |= 1; 1767dd23d762SRobert Mustacchi } 1768dd23d762SRobert Mustacchi 1769dd23d762SRobert Mustacchi ccd->atccd_phys_no = ccdno; 1770dd23d762SRobert Mustacchi if (uarch >= X86_UARCH_AMD_ZEN3) { 1771dd23d762SRobert Mustacchi pkg0_ids = B_TRUE; 1772dd23d762SRobert Mustacchi } else { 1773dd23d762SRobert Mustacchi ccd->atccd_flags |= AMDZEN_TOPO_CCD_F_CORE_PHYS_UNKNOWN; 1774dd23d762SRobert Mustacchi pkg0_ids = B_FALSE; 1775dd23d762SRobert Mustacchi 1776dd23d762SRobert Mustacchi /* 1777dd23d762SRobert Mustacchi * Determine the CCD logical ID for Zen 2 now since this doesn't 1778dd23d762SRobert Mustacchi * rely upon needing a valid physical core. 1779dd23d762SRobert Mustacchi */ 1780dd23d762SRobert Mustacchi ccd->atccd_log_no = amdzen_ccd_log_id_zen2(azn, df, ent); 1781dd23d762SRobert Mustacchi logccd_set = B_TRUE; 1782dd23d762SRobert Mustacchi } 1783dd23d762SRobert Mustacchi 1784dd23d762SRobert Mustacchi /* 1785dd23d762SRobert Mustacchi * To construct the APIC ID we need to know the socket and die (not CCD) 1786dd23d762SRobert Mustacchi * this is on. We deconstruct the CCD's fabric ID to determine that. 1787dd23d762SRobert Mustacchi */ 1788dd23d762SRobert Mustacchi zen_fabric_id_decompose(&df->adf_decomp, ent->adfe_fabric_id, &sockid, 1789dd23d762SRobert Mustacchi &dieid, &compid); 1790dd23d762SRobert Mustacchi 1791dd23d762SRobert Mustacchi /* 1792dd23d762SRobert Mustacchi * At this point we have all the information about the CCD, the number 1793dd23d762SRobert Mustacchi * of CCX instances, and which physical cores and threads are enabled. 1794dd23d762SRobert Mustacchi * Currently we assume that if we have one CCX enabled, then it is 1795dd23d762SRobert Mustacchi * always CCX0. We cannot find evidence of a two CCX supporting part 1796dd23d762SRobert Mustacchi * that doesn't always ship with both CCXs present and enabled. 1797dd23d762SRobert Mustacchi */ 1798dd23d762SRobert Mustacchi ccd->atccd_nlog_ccx = ccd->atccd_nphys_ccx = nccx; 1799dd23d762SRobert Mustacchi for (uint32_t ccxno = 0; ccxno < nccx; ccxno++) { 1800dd23d762SRobert Mustacchi amdzen_topo_ccx_t *ccx = &ccd->atccd_ccx[ccxno]; 1801dd23d762SRobert Mustacchi const uint32_t core_mask = (1 << azn->azn_ncore_per_ccx) - 1; 1802dd23d762SRobert Mustacchi const uint32_t core_shift = ccxno * azn->azn_ncore_per_ccx; 1803dd23d762SRobert Mustacchi const uint32_t ccx_core_en = (core_en >> core_shift) & 1804dd23d762SRobert Mustacchi core_mask; 1805dd23d762SRobert Mustacchi boolean_t logccx_set = B_FALSE; 1806dd23d762SRobert Mustacchi 1807dd23d762SRobert Mustacchi ccd->atccd_ccx_en[ccxno] = 1; 1808dd23d762SRobert Mustacchi ccx->atccx_phys_no = ccxno; 1809dd23d762SRobert Mustacchi ccx->atccx_nphys_cores = azn->azn_ncore_per_ccx; 1810dd23d762SRobert Mustacchi ccx->atccx_nlog_cores = nlcore_per_ccx; 1811dd23d762SRobert Mustacchi 1812dd23d762SRobert Mustacchi if (!pkg0_ids) { 1813dd23d762SRobert Mustacchi ccx->atccx_log_no = ccx->atccx_phys_no; 1814dd23d762SRobert Mustacchi logccx_set = B_TRUE; 1815dd23d762SRobert Mustacchi } 1816dd23d762SRobert Mustacchi 1817dd23d762SRobert Mustacchi for (uint32_t coreno = 0, logcorezen2 = 0; 1818dd23d762SRobert Mustacchi coreno < azn->azn_ncore_per_ccx; coreno++) { 1819dd23d762SRobert Mustacchi amdzen_topo_core_t *core = &ccx->atccx_cores[coreno]; 1820dd23d762SRobert Mustacchi 1821dd23d762SRobert Mustacchi if ((ccx_core_en & (1 << coreno)) == 0) { 1822dd23d762SRobert Mustacchi continue; 1823dd23d762SRobert Mustacchi } 1824dd23d762SRobert Mustacchi 1825dd23d762SRobert Mustacchi ccx->atccx_core_en[coreno] = 1; 1826dd23d762SRobert Mustacchi core->atcore_phys_no = coreno; 1827dd23d762SRobert Mustacchi 1828dd23d762SRobert Mustacchi /* 1829dd23d762SRobert Mustacchi * Now that we have the physical core number present, we 1830dd23d762SRobert Mustacchi * must determine the logical core number and fill out 1831dd23d762SRobert Mustacchi * the logical CCX/CCD if it has not been set. We must 1832dd23d762SRobert Mustacchi * do this before we attempt to look at which threads 1833dd23d762SRobert Mustacchi * are enabled, because that operates based upon logical 1834dd23d762SRobert Mustacchi * core number. 1835dd23d762SRobert Mustacchi * 1836dd23d762SRobert Mustacchi * For Zen 2 we do not have INITPKG0 at our disposal. We 1837dd23d762SRobert Mustacchi * currently assume (and tag for userland with the 1838dd23d762SRobert Mustacchi * AMDZEN_TOPO_CCD_F_CORE_PHYS_UNKNOWN flag) that we are 1839dd23d762SRobert Mustacchi * mapping logical cores to physicals in the order of 1840dd23d762SRobert Mustacchi * appearance. 1841dd23d762SRobert Mustacchi */ 1842dd23d762SRobert Mustacchi if (pkg0_ids) { 1843dd23d762SRobert Mustacchi amdzen_ccd_fill_core_initpkg0(azn, df, ccd, ccx, 1844dd23d762SRobert Mustacchi core, &logccd_set, &logccx_set); 1845dd23d762SRobert Mustacchi } else { 1846dd23d762SRobert Mustacchi core->atcore_log_no = logcorezen2; 1847dd23d762SRobert Mustacchi logcorezen2++; 1848dd23d762SRobert Mustacchi } 1849dd23d762SRobert Mustacchi 1850dd23d762SRobert Mustacchi /* 1851dd23d762SRobert Mustacchi * Determining which bits to use for the thread is a bit 1852dd23d762SRobert Mustacchi * weird here. Thread IDs within a CCX are logical, but 1853dd23d762SRobert Mustacchi * there are always physically spaced CCX sizes. See the 1854dd23d762SRobert Mustacchi * comment at the definition for SMU::PWR::THREAD_ENABLE 1855dd23d762SRobert Mustacchi * for more information. 1856dd23d762SRobert Mustacchi */ 1857dd23d762SRobert Mustacchi const uint32_t thread_shift = (ccx->atccx_nphys_cores * 1858dd23d762SRobert Mustacchi ccx->atccx_log_no + core->atcore_log_no) * 1859dd23d762SRobert Mustacchi nthreads_per_core; 1860dd23d762SRobert Mustacchi const uint32_t thread_mask = (nthreads_per_core << 1) - 1861dd23d762SRobert Mustacchi 1; 1862dd23d762SRobert Mustacchi const uint32_t core_thread_en = (thread_en >> 1863dd23d762SRobert Mustacchi thread_shift) & thread_mask; 1864dd23d762SRobert Mustacchi core->atcore_nthreads = nthreads_per_core; 1865dd23d762SRobert Mustacchi core->atcore_thr_en[0] = core_thread_en & 0x01; 1866dd23d762SRobert Mustacchi core->atcore_thr_en[1] = core_thread_en & 0x02; 1867dd23d762SRobert Mustacchi #ifdef DEBUG 1868dd23d762SRobert Mustacchi if (nthreads_per_core == 1) { 1869dd23d762SRobert Mustacchi VERIFY0(core->atcore_thr_en[1]); 1870dd23d762SRobert Mustacchi } 1871dd23d762SRobert Mustacchi #endif 1872dd23d762SRobert Mustacchi for (uint32_t thrno = 0; thrno < core->atcore_nthreads; 1873dd23d762SRobert Mustacchi thrno++) { 1874dd23d762SRobert Mustacchi ASSERT3U(core->atcore_thr_en[thrno], !=, 0); 1875dd23d762SRobert Mustacchi 1876dd23d762SRobert Mustacchi zen_apic_id_compose(&azn->azn_apic_decomp, 1877dd23d762SRobert Mustacchi sockid, dieid, ccd->atccd_log_no, 1878dd23d762SRobert Mustacchi ccx->atccx_log_no, core->atcore_log_no, 1879dd23d762SRobert Mustacchi thrno, &core->atcore_apicids[thrno]); 1880dd23d762SRobert Mustacchi 1881dd23d762SRobert Mustacchi } 1882dd23d762SRobert Mustacchi } 1883dd23d762SRobert Mustacchi 1884dd23d762SRobert Mustacchi ASSERT3U(logccx_set, ==, B_TRUE); 1885dd23d762SRobert Mustacchi ASSERT3U(logccd_set, ==, B_TRUE); 1886dd23d762SRobert Mustacchi } 1887dd23d762SRobert Mustacchi } 1888dd23d762SRobert Mustacchi 1889dd23d762SRobert Mustacchi static void 1890047043c2SRobert Mustacchi amdzen_nexus_init(void *arg) 1891047043c2SRobert Mustacchi { 1892047043c2SRobert Mustacchi uint_t i; 1893047043c2SRobert Mustacchi amdzen_t *azn = arg; 1894047043c2SRobert Mustacchi 1895047043c2SRobert Mustacchi /* 1896047043c2SRobert Mustacchi * First go through all of the stubs and assign the DF entries. 1897047043c2SRobert Mustacchi */ 1898047043c2SRobert Mustacchi mutex_enter(&azn->azn_mutex); 1899047043c2SRobert Mustacchi if (!amdzen_map_dfs(azn) || !amdzen_check_dfs(azn)) { 1900047043c2SRobert Mustacchi azn->azn_flags |= AMDZEN_F_MAP_ERROR; 1901047043c2SRobert Mustacchi goto done; 1902047043c2SRobert Mustacchi } 1903047043c2SRobert Mustacchi 1904047043c2SRobert Mustacchi for (i = 0; i < AMDZEN_MAX_DFS; i++) { 1905047043c2SRobert Mustacchi amdzen_df_t *df = &azn->azn_dfs[i]; 1906047043c2SRobert Mustacchi 1907047043c2SRobert Mustacchi if ((df->adf_flags & AMDZEN_DF_F_VALID) == 0) 1908047043c2SRobert Mustacchi continue; 1909047043c2SRobert Mustacchi amdzen_setup_df(azn, df); 1910047043c2SRobert Mustacchi amdzen_find_nb(azn, df); 1911047043c2SRobert Mustacchi } 1912047043c2SRobert Mustacchi 1913dd23d762SRobert Mustacchi if (amdzen_determine_apic_decomp(azn)) { 1914dd23d762SRobert Mustacchi azn->azn_flags |= AMDZEN_F_APIC_DECOMP_VALID; 1915dd23d762SRobert Mustacchi } 1916dd23d762SRobert Mustacchi 1917dd23d762SRobert Mustacchi amdzen_determine_ncore_per_ccx(azn); 1918dd23d762SRobert Mustacchi 1919047043c2SRobert Mustacchi /* 1920047043c2SRobert Mustacchi * Not all children may be installed. As such, we do not treat the 1921047043c2SRobert Mustacchi * failure of a child as fatal to the driver. 1922047043c2SRobert Mustacchi */ 1923047043c2SRobert Mustacchi mutex_exit(&azn->azn_mutex); 1924047043c2SRobert Mustacchi for (i = 0; i < ARRAY_SIZE(amdzen_children); i++) { 1925047043c2SRobert Mustacchi (void) amdzen_create_child(azn, &amdzen_children[i]); 1926047043c2SRobert Mustacchi } 1927047043c2SRobert Mustacchi mutex_enter(&azn->azn_mutex); 1928047043c2SRobert Mustacchi 1929047043c2SRobert Mustacchi done: 1930047043c2SRobert Mustacchi azn->azn_flags &= ~AMDZEN_F_ATTACH_DISPATCHED; 1931047043c2SRobert Mustacchi azn->azn_flags |= AMDZEN_F_ATTACH_COMPLETE; 1932047043c2SRobert Mustacchi azn->azn_taskqid = TASKQID_INVALID; 1933047043c2SRobert Mustacchi cv_broadcast(&azn->azn_cv); 1934047043c2SRobert Mustacchi mutex_exit(&azn->azn_mutex); 1935047043c2SRobert Mustacchi } 1936047043c2SRobert Mustacchi 1937047043c2SRobert Mustacchi static int 1938047043c2SRobert Mustacchi amdzen_stub_scan_cb(dev_info_t *dip, void *arg) 1939047043c2SRobert Mustacchi { 1940047043c2SRobert Mustacchi amdzen_t *azn = arg; 1941047043c2SRobert Mustacchi uint16_t vid, did; 1942047043c2SRobert Mustacchi int *regs; 1943047043c2SRobert Mustacchi uint_t nregs, i; 1944047043c2SRobert Mustacchi boolean_t match = B_FALSE; 1945047043c2SRobert Mustacchi 1946047043c2SRobert Mustacchi if (dip == ddi_root_node()) { 1947047043c2SRobert Mustacchi return (DDI_WALK_CONTINUE); 1948047043c2SRobert Mustacchi } 1949047043c2SRobert Mustacchi 1950047043c2SRobert Mustacchi /* 1951047043c2SRobert Mustacchi * If a node in question is not a pci node, then we have no interest in 1952047043c2SRobert Mustacchi * it as all the stubs that we care about are related to pci devices. 1953047043c2SRobert Mustacchi */ 1954047043c2SRobert Mustacchi if (strncmp("pci", ddi_get_name(dip), 3) != 0) { 1955047043c2SRobert Mustacchi return (DDI_WALK_PRUNECHILD); 1956047043c2SRobert Mustacchi } 1957047043c2SRobert Mustacchi 1958047043c2SRobert Mustacchi /* 1959047043c2SRobert Mustacchi * If we can't get a device or vendor ID and prove that this is an AMD 1960047043c2SRobert Mustacchi * part, then we don't care about it. 1961047043c2SRobert Mustacchi */ 1962047043c2SRobert Mustacchi vid = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 1963047043c2SRobert Mustacchi "vendor-id", PCI_EINVAL16); 1964047043c2SRobert Mustacchi did = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 1965047043c2SRobert Mustacchi "device-id", PCI_EINVAL16); 1966047043c2SRobert Mustacchi if (vid == PCI_EINVAL16 || did == PCI_EINVAL16) { 1967047043c2SRobert Mustacchi return (DDI_WALK_CONTINUE); 1968047043c2SRobert Mustacchi } 1969047043c2SRobert Mustacchi 19709b0429a1SPu Wen if (vid != AMDZEN_PCI_VID_AMD && vid != AMDZEN_PCI_VID_HYGON) { 1971047043c2SRobert Mustacchi return (DDI_WALK_CONTINUE); 1972047043c2SRobert Mustacchi } 1973047043c2SRobert Mustacchi 1974047043c2SRobert Mustacchi for (i = 0; i < ARRAY_SIZE(amdzen_nb_ids); i++) { 1975047043c2SRobert Mustacchi if (amdzen_nb_ids[i] == did) { 1976047043c2SRobert Mustacchi match = B_TRUE; 1977047043c2SRobert Mustacchi } 1978047043c2SRobert Mustacchi } 1979047043c2SRobert Mustacchi 1980047043c2SRobert Mustacchi if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 1981047043c2SRobert Mustacchi "reg", ®s, &nregs) != DDI_PROP_SUCCESS) { 1982047043c2SRobert Mustacchi return (DDI_WALK_CONTINUE); 1983047043c2SRobert Mustacchi } 1984047043c2SRobert Mustacchi 1985047043c2SRobert Mustacchi if (nregs == 0) { 1986047043c2SRobert Mustacchi ddi_prop_free(regs); 1987047043c2SRobert Mustacchi return (DDI_WALK_CONTINUE); 1988047043c2SRobert Mustacchi } 1989047043c2SRobert Mustacchi 1990047043c2SRobert Mustacchi if (PCI_REG_BUS_G(regs[0]) == AMDZEN_DF_BUSNO && 1991047043c2SRobert Mustacchi PCI_REG_DEV_G(regs[0]) >= AMDZEN_DF_FIRST_DEVICE) { 1992047043c2SRobert Mustacchi match = B_TRUE; 1993047043c2SRobert Mustacchi } 1994047043c2SRobert Mustacchi 1995047043c2SRobert Mustacchi ddi_prop_free(regs); 1996047043c2SRobert Mustacchi if (match) { 1997047043c2SRobert Mustacchi mutex_enter(&azn->azn_mutex); 1998047043c2SRobert Mustacchi azn->azn_nscanned++; 1999047043c2SRobert Mustacchi mutex_exit(&azn->azn_mutex); 2000047043c2SRobert Mustacchi } 2001047043c2SRobert Mustacchi 2002047043c2SRobert Mustacchi return (DDI_WALK_CONTINUE); 2003047043c2SRobert Mustacchi } 2004047043c2SRobert Mustacchi 2005047043c2SRobert Mustacchi static void 2006047043c2SRobert Mustacchi amdzen_stub_scan(void *arg) 2007047043c2SRobert Mustacchi { 2008047043c2SRobert Mustacchi amdzen_t *azn = arg; 2009047043c2SRobert Mustacchi 2010047043c2SRobert Mustacchi mutex_enter(&azn->azn_mutex); 2011047043c2SRobert Mustacchi azn->azn_nscanned = 0; 2012047043c2SRobert Mustacchi mutex_exit(&azn->azn_mutex); 2013047043c2SRobert Mustacchi 2014047043c2SRobert Mustacchi ddi_walk_devs(ddi_root_node(), amdzen_stub_scan_cb, azn); 2015047043c2SRobert Mustacchi 2016047043c2SRobert Mustacchi mutex_enter(&azn->azn_mutex); 2017047043c2SRobert Mustacchi azn->azn_flags &= ~AMDZEN_F_SCAN_DISPATCHED; 2018047043c2SRobert Mustacchi azn->azn_flags |= AMDZEN_F_SCAN_COMPLETE; 2019047043c2SRobert Mustacchi 2020047043c2SRobert Mustacchi if (azn->azn_nscanned == 0) { 2021047043c2SRobert Mustacchi azn->azn_flags |= AMDZEN_F_UNSUPPORTED; 2022047043c2SRobert Mustacchi azn->azn_taskqid = TASKQID_INVALID; 2023047043c2SRobert Mustacchi cv_broadcast(&azn->azn_cv); 2024047043c2SRobert Mustacchi } else if (azn->azn_npresent == azn->azn_nscanned) { 2025047043c2SRobert Mustacchi azn->azn_flags |= AMDZEN_F_ATTACH_DISPATCHED; 2026047043c2SRobert Mustacchi azn->azn_taskqid = taskq_dispatch(system_taskq, 2027047043c2SRobert Mustacchi amdzen_nexus_init, azn, TQ_SLEEP); 2028047043c2SRobert Mustacchi } 2029047043c2SRobert Mustacchi mutex_exit(&azn->azn_mutex); 2030047043c2SRobert Mustacchi } 2031047043c2SRobert Mustacchi 2032047043c2SRobert Mustacchi /* 2033047043c2SRobert Mustacchi * Unfortunately we can't really let the stubs detach as we may need them to be 2034047043c2SRobert Mustacchi * available for client operations. We may be able to improve this if we know 2035047043c2SRobert Mustacchi * that the actual nexus is going away. However, as long as it's active, we need 2036047043c2SRobert Mustacchi * all the stubs. 2037047043c2SRobert Mustacchi */ 2038047043c2SRobert Mustacchi int 2039047043c2SRobert Mustacchi amdzen_detach_stub(dev_info_t *dip, ddi_detach_cmd_t cmd) 2040047043c2SRobert Mustacchi { 2041047043c2SRobert Mustacchi if (cmd == DDI_SUSPEND) { 2042047043c2SRobert Mustacchi return (DDI_SUCCESS); 2043047043c2SRobert Mustacchi } 2044047043c2SRobert Mustacchi 2045047043c2SRobert Mustacchi return (DDI_FAILURE); 2046047043c2SRobert Mustacchi } 2047047043c2SRobert Mustacchi 2048047043c2SRobert Mustacchi int 2049047043c2SRobert Mustacchi amdzen_attach_stub(dev_info_t *dip, ddi_attach_cmd_t cmd) 2050047043c2SRobert Mustacchi { 2051047043c2SRobert Mustacchi int *regs, reg; 2052047043c2SRobert Mustacchi uint_t nregs, i; 2053047043c2SRobert Mustacchi uint16_t vid, did; 2054047043c2SRobert Mustacchi amdzen_stub_t *stub; 2055047043c2SRobert Mustacchi amdzen_t *azn = amdzen_data; 2056047043c2SRobert Mustacchi boolean_t valid = B_FALSE; 2057047043c2SRobert Mustacchi boolean_t nb = B_FALSE; 2058047043c2SRobert Mustacchi 2059047043c2SRobert Mustacchi if (cmd == DDI_RESUME) { 2060047043c2SRobert Mustacchi return (DDI_SUCCESS); 2061047043c2SRobert Mustacchi } else if (cmd != DDI_ATTACH) { 2062047043c2SRobert Mustacchi return (DDI_FAILURE); 2063047043c2SRobert Mustacchi } 2064047043c2SRobert Mustacchi 2065047043c2SRobert Mustacchi /* 2066047043c2SRobert Mustacchi * Make sure that the stub that we've been asked to attach is a pci type 2067047043c2SRobert Mustacchi * device. If not, then there is no reason for us to proceed. 2068047043c2SRobert Mustacchi */ 2069047043c2SRobert Mustacchi if (strncmp("pci", ddi_get_name(dip), 3) != 0) { 2070047043c2SRobert Mustacchi dev_err(dip, CE_WARN, "asked to attach a bad AMD Zen nexus " 2071047043c2SRobert Mustacchi "stub: %s", ddi_get_name(dip)); 2072047043c2SRobert Mustacchi return (DDI_FAILURE); 2073047043c2SRobert Mustacchi } 2074047043c2SRobert Mustacchi vid = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 2075047043c2SRobert Mustacchi "vendor-id", PCI_EINVAL16); 2076047043c2SRobert Mustacchi did = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 2077047043c2SRobert Mustacchi "device-id", PCI_EINVAL16); 2078047043c2SRobert Mustacchi if (vid == PCI_EINVAL16 || did == PCI_EINVAL16) { 2079047043c2SRobert Mustacchi dev_err(dip, CE_WARN, "failed to get PCI ID properties"); 2080047043c2SRobert Mustacchi return (DDI_FAILURE); 2081047043c2SRobert Mustacchi } 2082047043c2SRobert Mustacchi 20839b0429a1SPu Wen if (vid != AMDZEN_PCI_VID_AMD && vid != AMDZEN_PCI_VID_HYGON) { 20849b0429a1SPu Wen dev_err(dip, CE_WARN, "expected vendor ID (0x%x), found 0x%x", 20859b0429a1SPu Wen cpuid_getvendor(CPU) == X86_VENDOR_HYGON ? 20869b0429a1SPu Wen AMDZEN_PCI_VID_HYGON : AMDZEN_PCI_VID_AMD, vid); 2087047043c2SRobert Mustacchi return (DDI_FAILURE); 2088047043c2SRobert Mustacchi } 2089047043c2SRobert Mustacchi 2090047043c2SRobert Mustacchi if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 2091047043c2SRobert Mustacchi "reg", ®s, &nregs) != DDI_PROP_SUCCESS) { 2092047043c2SRobert Mustacchi dev_err(dip, CE_WARN, "failed to get 'reg' property"); 2093047043c2SRobert Mustacchi return (DDI_FAILURE); 2094047043c2SRobert Mustacchi } 2095047043c2SRobert Mustacchi 2096047043c2SRobert Mustacchi if (nregs == 0) { 2097047043c2SRobert Mustacchi ddi_prop_free(regs); 2098047043c2SRobert Mustacchi dev_err(dip, CE_WARN, "missing 'reg' property values"); 2099047043c2SRobert Mustacchi return (DDI_FAILURE); 2100047043c2SRobert Mustacchi } 2101047043c2SRobert Mustacchi reg = *regs; 2102047043c2SRobert Mustacchi ddi_prop_free(regs); 2103047043c2SRobert Mustacchi 2104047043c2SRobert Mustacchi for (i = 0; i < ARRAY_SIZE(amdzen_nb_ids); i++) { 2105047043c2SRobert Mustacchi if (amdzen_nb_ids[i] == did) { 2106047043c2SRobert Mustacchi valid = B_TRUE; 2107047043c2SRobert Mustacchi nb = B_TRUE; 2108047043c2SRobert Mustacchi } 2109047043c2SRobert Mustacchi } 2110047043c2SRobert Mustacchi 2111047043c2SRobert Mustacchi if (!valid && PCI_REG_BUS_G(reg) == AMDZEN_DF_BUSNO && 2112047043c2SRobert Mustacchi PCI_REG_DEV_G(reg) >= AMDZEN_DF_FIRST_DEVICE) { 2113047043c2SRobert Mustacchi valid = B_TRUE; 2114047043c2SRobert Mustacchi nb = B_FALSE; 2115047043c2SRobert Mustacchi } 2116047043c2SRobert Mustacchi 2117047043c2SRobert Mustacchi if (!valid) { 2118047043c2SRobert Mustacchi dev_err(dip, CE_WARN, "device %s didn't match the nexus list", 2119047043c2SRobert Mustacchi ddi_get_name(dip)); 2120047043c2SRobert Mustacchi return (DDI_FAILURE); 2121047043c2SRobert Mustacchi } 2122047043c2SRobert Mustacchi 2123047043c2SRobert Mustacchi stub = kmem_alloc(sizeof (amdzen_stub_t), KM_SLEEP); 2124047043c2SRobert Mustacchi if (pci_config_setup(dip, &stub->azns_cfgspace) != DDI_SUCCESS) { 2125047043c2SRobert Mustacchi dev_err(dip, CE_WARN, "failed to set up config space"); 2126047043c2SRobert Mustacchi kmem_free(stub, sizeof (amdzen_stub_t)); 2127047043c2SRobert Mustacchi return (DDI_FAILURE); 2128047043c2SRobert Mustacchi } 2129047043c2SRobert Mustacchi 2130047043c2SRobert Mustacchi stub->azns_dip = dip; 2131047043c2SRobert Mustacchi stub->azns_vid = vid; 2132047043c2SRobert Mustacchi stub->azns_did = did; 2133047043c2SRobert Mustacchi stub->azns_bus = PCI_REG_BUS_G(reg); 2134047043c2SRobert Mustacchi stub->azns_dev = PCI_REG_DEV_G(reg); 2135047043c2SRobert Mustacchi stub->azns_func = PCI_REG_FUNC_G(reg); 2136047043c2SRobert Mustacchi ddi_set_driver_private(dip, stub); 2137047043c2SRobert Mustacchi 2138047043c2SRobert Mustacchi mutex_enter(&azn->azn_mutex); 2139047043c2SRobert Mustacchi azn->azn_npresent++; 2140047043c2SRobert Mustacchi if (nb) { 2141047043c2SRobert Mustacchi list_insert_tail(&azn->azn_nb_stubs, stub); 2142047043c2SRobert Mustacchi } else { 2143047043c2SRobert Mustacchi list_insert_tail(&azn->azn_df_stubs, stub); 2144047043c2SRobert Mustacchi } 2145047043c2SRobert Mustacchi 2146047043c2SRobert Mustacchi if ((azn->azn_flags & AMDZEN_F_TASKQ_MASK) == AMDZEN_F_SCAN_COMPLETE && 2147047043c2SRobert Mustacchi azn->azn_nscanned == azn->azn_npresent) { 2148047043c2SRobert Mustacchi azn->azn_flags |= AMDZEN_F_ATTACH_DISPATCHED; 2149047043c2SRobert Mustacchi azn->azn_taskqid = taskq_dispatch(system_taskq, 2150047043c2SRobert Mustacchi amdzen_nexus_init, azn, TQ_SLEEP); 2151047043c2SRobert Mustacchi } 2152047043c2SRobert Mustacchi mutex_exit(&azn->azn_mutex); 2153047043c2SRobert Mustacchi 2154047043c2SRobert Mustacchi return (DDI_SUCCESS); 2155047043c2SRobert Mustacchi } 2156047043c2SRobert Mustacchi 2157047043c2SRobert Mustacchi static int 2158047043c2SRobert Mustacchi amdzen_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 2159047043c2SRobert Mustacchi void *arg, void *result) 2160047043c2SRobert Mustacchi { 2161047043c2SRobert Mustacchi char buf[32]; 2162047043c2SRobert Mustacchi dev_info_t *child; 2163047043c2SRobert Mustacchi const amdzen_child_data_t *acd; 2164047043c2SRobert Mustacchi 2165047043c2SRobert Mustacchi switch (ctlop) { 2166047043c2SRobert Mustacchi case DDI_CTLOPS_REPORTDEV: 2167047043c2SRobert Mustacchi if (rdip == NULL) { 2168047043c2SRobert Mustacchi return (DDI_FAILURE); 2169047043c2SRobert Mustacchi } 2170047043c2SRobert Mustacchi cmn_err(CE_CONT, "amdzen nexus: %s@%s, %s%d\n", 2171047043c2SRobert Mustacchi ddi_node_name(rdip), ddi_get_name_addr(rdip), 2172047043c2SRobert Mustacchi ddi_driver_name(rdip), ddi_get_instance(rdip)); 2173047043c2SRobert Mustacchi break; 2174047043c2SRobert Mustacchi case DDI_CTLOPS_INITCHILD: 2175047043c2SRobert Mustacchi child = arg; 2176047043c2SRobert Mustacchi if (child == NULL) { 2177047043c2SRobert Mustacchi dev_err(dip, CE_WARN, "!no child passed for " 2178047043c2SRobert Mustacchi "DDI_CTLOPS_INITCHILD"); 2179047043c2SRobert Mustacchi } 2180047043c2SRobert Mustacchi 2181047043c2SRobert Mustacchi acd = ddi_get_parent_data(child); 2182047043c2SRobert Mustacchi if (acd == NULL) { 2183047043c2SRobert Mustacchi dev_err(dip, CE_WARN, "!missing child parent data"); 2184047043c2SRobert Mustacchi return (DDI_FAILURE); 2185047043c2SRobert Mustacchi } 2186047043c2SRobert Mustacchi 2187047043c2SRobert Mustacchi if (snprintf(buf, sizeof (buf), "%d", acd->acd_addr) >= 2188047043c2SRobert Mustacchi sizeof (buf)) { 2189047043c2SRobert Mustacchi dev_err(dip, CE_WARN, "!failed to construct device " 2190047043c2SRobert Mustacchi "addr due to overflow"); 2191047043c2SRobert Mustacchi return (DDI_FAILURE); 2192047043c2SRobert Mustacchi } 2193047043c2SRobert Mustacchi 2194047043c2SRobert Mustacchi ddi_set_name_addr(child, buf); 2195047043c2SRobert Mustacchi break; 2196047043c2SRobert Mustacchi case DDI_CTLOPS_UNINITCHILD: 2197047043c2SRobert Mustacchi child = arg; 2198047043c2SRobert Mustacchi if (child == NULL) { 2199047043c2SRobert Mustacchi dev_err(dip, CE_WARN, "!no child passed for " 2200047043c2SRobert Mustacchi "DDI_CTLOPS_UNINITCHILD"); 2201047043c2SRobert Mustacchi } 2202047043c2SRobert Mustacchi 2203047043c2SRobert Mustacchi ddi_set_name_addr(child, NULL); 2204047043c2SRobert Mustacchi break; 2205047043c2SRobert Mustacchi default: 2206047043c2SRobert Mustacchi return (ddi_ctlops(dip, rdip, ctlop, arg, result)); 2207047043c2SRobert Mustacchi } 2208047043c2SRobert Mustacchi return (DDI_SUCCESS); 2209047043c2SRobert Mustacchi } 2210047043c2SRobert Mustacchi 2211047043c2SRobert Mustacchi static int 2212dd23d762SRobert Mustacchi amdzen_topo_open(dev_t *devp, int flag, int otyp, cred_t *credp) 2213dd23d762SRobert Mustacchi { 2214dd23d762SRobert Mustacchi minor_t m; 2215dd23d762SRobert Mustacchi amdzen_t *azn = amdzen_data; 2216dd23d762SRobert Mustacchi 2217dd23d762SRobert Mustacchi if (crgetzoneid(credp) != GLOBAL_ZONEID || 2218dd23d762SRobert Mustacchi secpolicy_sys_config(credp, B_FALSE) != 0) { 2219dd23d762SRobert Mustacchi return (EPERM); 2220dd23d762SRobert Mustacchi } 2221dd23d762SRobert Mustacchi 2222dd23d762SRobert Mustacchi if ((flag & (FEXCL | FNDELAY | FNONBLOCK)) != 0) { 2223dd23d762SRobert Mustacchi return (EINVAL); 2224dd23d762SRobert Mustacchi } 2225dd23d762SRobert Mustacchi 2226dd23d762SRobert Mustacchi if (otyp != OTYP_CHR) { 2227dd23d762SRobert Mustacchi return (EINVAL); 2228dd23d762SRobert Mustacchi } 2229dd23d762SRobert Mustacchi 2230dd23d762SRobert Mustacchi m = getminor(*devp); 2231dd23d762SRobert Mustacchi if (m != AMDZEN_MINOR_TOPO) { 2232dd23d762SRobert Mustacchi return (ENXIO); 2233dd23d762SRobert Mustacchi } 2234dd23d762SRobert Mustacchi 2235dd23d762SRobert Mustacchi mutex_enter(&azn->azn_mutex); 2236dd23d762SRobert Mustacchi if ((azn->azn_flags & AMDZEN_F_IOCTL_MASK) != 2237dd23d762SRobert Mustacchi AMDZEN_F_ATTACH_COMPLETE) { 2238dd23d762SRobert Mustacchi mutex_exit(&azn->azn_mutex); 2239dd23d762SRobert Mustacchi return (ENOTSUP); 2240dd23d762SRobert Mustacchi } 2241dd23d762SRobert Mustacchi mutex_exit(&azn->azn_mutex); 2242dd23d762SRobert Mustacchi 2243dd23d762SRobert Mustacchi return (0); 2244dd23d762SRobert Mustacchi } 2245dd23d762SRobert Mustacchi 2246dd23d762SRobert Mustacchi static int 2247dd23d762SRobert Mustacchi amdzen_topo_ioctl_base(amdzen_t *azn, intptr_t arg, int mode) 2248dd23d762SRobert Mustacchi { 2249dd23d762SRobert Mustacchi amdzen_topo_base_t base; 2250dd23d762SRobert Mustacchi 2251dd23d762SRobert Mustacchi bzero(&base, sizeof (base)); 2252dd23d762SRobert Mustacchi mutex_enter(&azn->azn_mutex); 2253dd23d762SRobert Mustacchi base.atb_ndf = azn->azn_ndfs; 2254dd23d762SRobert Mustacchi 2255dd23d762SRobert Mustacchi if ((azn->azn_flags & AMDZEN_F_APIC_DECOMP_VALID) == 0) { 2256dd23d762SRobert Mustacchi mutex_exit(&azn->azn_mutex); 2257dd23d762SRobert Mustacchi return (ENOTSUP); 2258dd23d762SRobert Mustacchi } 2259dd23d762SRobert Mustacchi 2260dd23d762SRobert Mustacchi base.atb_apic_decomp = azn->azn_apic_decomp; 2261dd23d762SRobert Mustacchi for (uint_t i = 0; i < azn->azn_ndfs; i++) { 2262dd23d762SRobert Mustacchi const amdzen_df_t *df = &azn->azn_dfs[i]; 2263dd23d762SRobert Mustacchi 2264dd23d762SRobert Mustacchi base.atb_maxdfent = MAX(base.atb_maxdfent, df->adf_nents); 2265dd23d762SRobert Mustacchi if (i == 0) { 2266dd23d762SRobert Mustacchi base.atb_rev = df->adf_rev; 2267dd23d762SRobert Mustacchi base.atb_df_decomp = df->adf_decomp; 2268dd23d762SRobert Mustacchi } 2269dd23d762SRobert Mustacchi } 2270dd23d762SRobert Mustacchi mutex_exit(&azn->azn_mutex); 2271dd23d762SRobert Mustacchi 2272dd23d762SRobert Mustacchi if (ddi_copyout(&base, (void *)(uintptr_t)arg, sizeof (base), 2273dd23d762SRobert Mustacchi mode & FKIOCTL) != 0) { 2274dd23d762SRobert Mustacchi return (EFAULT); 2275dd23d762SRobert Mustacchi } 2276dd23d762SRobert Mustacchi 2277dd23d762SRobert Mustacchi return (0); 2278dd23d762SRobert Mustacchi } 2279dd23d762SRobert Mustacchi 2280dd23d762SRobert Mustacchi /* 2281dd23d762SRobert Mustacchi * Fill in the peers. The way we do is this is to just fill in all the entries 2282dd23d762SRobert Mustacchi * and then zero out the ones that aren't valid. 2283dd23d762SRobert Mustacchi */ 2284dd23d762SRobert Mustacchi static void 2285dd23d762SRobert Mustacchi amdzen_topo_ioctl_df_fill_peers(const amdzen_df_ent_t *ent, 2286dd23d762SRobert Mustacchi amdzen_topo_df_ent_t *topo_ent) 2287dd23d762SRobert Mustacchi { 2288dd23d762SRobert Mustacchi topo_ent->atde_npeers = DF_FBIINFO0_GET_FTI_PCNT(ent->adfe_info0); 2289dd23d762SRobert Mustacchi topo_ent->atde_peers[0] = DF_FBINFO1_GET_FTI0_NINSTID(ent->adfe_info1); 2290dd23d762SRobert Mustacchi topo_ent->atde_peers[1] = DF_FBINFO1_GET_FTI1_NINSTID(ent->adfe_info1); 2291dd23d762SRobert Mustacchi topo_ent->atde_peers[2] = DF_FBINFO1_GET_FTI2_NINSTID(ent->adfe_info1); 2292dd23d762SRobert Mustacchi topo_ent->atde_peers[3] = DF_FBINFO1_GET_FTI3_NINSTID(ent->adfe_info1); 2293dd23d762SRobert Mustacchi topo_ent->atde_peers[4] = DF_FBINFO2_GET_FTI4_NINSTID(ent->adfe_info2); 2294dd23d762SRobert Mustacchi topo_ent->atde_peers[5] = DF_FBINFO2_GET_FTI5_NINSTID(ent->adfe_info2); 2295dd23d762SRobert Mustacchi 2296dd23d762SRobert Mustacchi for (uint32_t i = topo_ent->atde_npeers; i < AMDZEN_TOPO_DF_MAX_PEERS; 2297dd23d762SRobert Mustacchi i++) { 2298dd23d762SRobert Mustacchi topo_ent->atde_peers[i] = 0; 2299dd23d762SRobert Mustacchi } 2300dd23d762SRobert Mustacchi } 2301dd23d762SRobert Mustacchi 2302dd23d762SRobert Mustacchi static void 2303dd23d762SRobert Mustacchi amdzen_topo_ioctl_df_fill_ccm(const amdzen_df_ent_t *ent, 2304dd23d762SRobert Mustacchi amdzen_topo_df_ent_t *topo_ent) 2305dd23d762SRobert Mustacchi { 2306dd23d762SRobert Mustacchi const amdzen_ccm_data_t *ccm = &ent->adfe_data.aded_ccm; 2307dd23d762SRobert Mustacchi amdzen_topo_ccm_data_t *topo_ccm = &topo_ent->atde_data.atded_ccm; 2308dd23d762SRobert Mustacchi 2309dd23d762SRobert Mustacchi topo_ccm->atcd_nccds = ccm->acd_nccds; 2310dd23d762SRobert Mustacchi for (uint32_t i = 0; i < DF_MAX_CCDS_PER_CCM; i++) { 2311dd23d762SRobert Mustacchi topo_ccm->atcd_ccd_en[i] = ccm->acd_ccd_en[i]; 2312dd23d762SRobert Mustacchi topo_ccm->atcd_ccd_ids[i] = ccm->acd_ccd_id[i]; 2313dd23d762SRobert Mustacchi } 2314dd23d762SRobert Mustacchi } 2315dd23d762SRobert Mustacchi 2316dd23d762SRobert Mustacchi static int 2317dd23d762SRobert Mustacchi amdzen_topo_ioctl_df(amdzen_t *azn, intptr_t arg, int mode) 2318dd23d762SRobert Mustacchi { 2319dd23d762SRobert Mustacchi uint_t model; 2320dd23d762SRobert Mustacchi uint32_t max_ents, nwritten; 2321dd23d762SRobert Mustacchi const amdzen_df_t *df; 2322dd23d762SRobert Mustacchi amdzen_topo_df_t topo_df; 2323dd23d762SRobert Mustacchi #ifdef _MULTI_DATAMODEL 2324dd23d762SRobert Mustacchi amdzen_topo_df32_t topo_df32; 2325dd23d762SRobert Mustacchi #endif 2326dd23d762SRobert Mustacchi 2327dd23d762SRobert Mustacchi model = ddi_model_convert_from(mode); 2328dd23d762SRobert Mustacchi switch (model) { 2329dd23d762SRobert Mustacchi #ifdef _MULTI_DATAMODEL 2330dd23d762SRobert Mustacchi case DDI_MODEL_ILP32: 2331dd23d762SRobert Mustacchi if (ddi_copyin((void *)(uintptr_t)arg, &topo_df32, 2332dd23d762SRobert Mustacchi sizeof (topo_df32), mode & FKIOCTL) != 0) { 2333dd23d762SRobert Mustacchi return (EFAULT); 2334dd23d762SRobert Mustacchi } 2335dd23d762SRobert Mustacchi bzero(&topo_df, sizeof (topo_df)); 2336dd23d762SRobert Mustacchi topo_df.atd_dfno = topo_df32.atd_dfno; 2337dd23d762SRobert Mustacchi topo_df.atd_df_buf_nents = topo_df32.atd_df_buf_nents; 2338dd23d762SRobert Mustacchi topo_df.atd_df_ents = (void *)(uintptr_t)topo_df32.atd_df_ents; 2339dd23d762SRobert Mustacchi break; 2340dd23d762SRobert Mustacchi #endif 2341dd23d762SRobert Mustacchi case DDI_MODEL_NONE: 2342dd23d762SRobert Mustacchi if (ddi_copyin((void *)(uintptr_t)arg, &topo_df, 2343dd23d762SRobert Mustacchi sizeof (topo_df), mode & FKIOCTL) != 0) { 2344dd23d762SRobert Mustacchi return (EFAULT); 2345dd23d762SRobert Mustacchi } 2346dd23d762SRobert Mustacchi break; 2347dd23d762SRobert Mustacchi default: 2348dd23d762SRobert Mustacchi return (ENOTSUP); 2349dd23d762SRobert Mustacchi } 2350dd23d762SRobert Mustacchi 2351dd23d762SRobert Mustacchi mutex_enter(&azn->azn_mutex); 2352dd23d762SRobert Mustacchi if (topo_df.atd_dfno >= azn->azn_ndfs) { 2353dd23d762SRobert Mustacchi mutex_exit(&azn->azn_mutex); 2354dd23d762SRobert Mustacchi return (EINVAL); 2355dd23d762SRobert Mustacchi } 2356dd23d762SRobert Mustacchi 2357dd23d762SRobert Mustacchi df = &azn->azn_dfs[topo_df.atd_dfno]; 2358dd23d762SRobert Mustacchi topo_df.atd_nodeid = df->adf_nodeid; 2359dd23d762SRobert Mustacchi topo_df.atd_sockid = (df->adf_nodeid & df->adf_decomp.dfd_sock_mask) >> 2360dd23d762SRobert Mustacchi df->adf_decomp.dfd_sock_shift; 2361dd23d762SRobert Mustacchi topo_df.atd_dieid = (df->adf_nodeid & df->adf_decomp.dfd_die_mask) >> 2362dd23d762SRobert Mustacchi df->adf_decomp.dfd_die_shift; 2363dd23d762SRobert Mustacchi topo_df.atd_rev = df->adf_rev; 2364dd23d762SRobert Mustacchi topo_df.atd_df_act_nents = df->adf_nents; 2365dd23d762SRobert Mustacchi max_ents = MIN(topo_df.atd_df_buf_nents, df->adf_nents); 2366dd23d762SRobert Mustacchi 2367dd23d762SRobert Mustacchi if (topo_df.atd_df_ents == NULL) { 2368dd23d762SRobert Mustacchi topo_df.atd_df_buf_nvalid = 0; 2369dd23d762SRobert Mustacchi mutex_exit(&azn->azn_mutex); 2370dd23d762SRobert Mustacchi goto copyout; 2371dd23d762SRobert Mustacchi } 2372dd23d762SRobert Mustacchi 2373dd23d762SRobert Mustacchi nwritten = 0; 2374dd23d762SRobert Mustacchi for (uint32_t i = 0; i < max_ents; i++) { 2375dd23d762SRobert Mustacchi amdzen_topo_df_ent_t topo_ent; 2376dd23d762SRobert Mustacchi const amdzen_df_ent_t *ent = &df->adf_ents[i]; 2377dd23d762SRobert Mustacchi 2378dd23d762SRobert Mustacchi /* 2379dd23d762SRobert Mustacchi * We opt not to include disabled elements right now. They 2380dd23d762SRobert Mustacchi * generally don't have a valid type and there isn't much useful 2381dd23d762SRobert Mustacchi * information we can get from them. This can be changed if we 2382dd23d762SRobert Mustacchi * find a use case for them for userland topo. 2383dd23d762SRobert Mustacchi */ 2384dd23d762SRobert Mustacchi if ((ent->adfe_flags & AMDZEN_DFE_F_ENABLED) == 0) 2385dd23d762SRobert Mustacchi continue; 2386dd23d762SRobert Mustacchi 2387dd23d762SRobert Mustacchi bzero(&topo_ent, sizeof (topo_ent)); 2388dd23d762SRobert Mustacchi topo_ent.atde_type = ent->adfe_type; 2389dd23d762SRobert Mustacchi topo_ent.atde_subtype = ent->adfe_subtype; 2390dd23d762SRobert Mustacchi topo_ent.atde_fabric_id = ent->adfe_fabric_id; 2391dd23d762SRobert Mustacchi topo_ent.atde_inst_id = ent->adfe_inst_id; 2392dd23d762SRobert Mustacchi amdzen_topo_ioctl_df_fill_peers(ent, &topo_ent); 2393dd23d762SRobert Mustacchi 2394dd23d762SRobert Mustacchi if (ent->adfe_type == DF_TYPE_CCM && 2395dd23d762SRobert Mustacchi ent->adfe_subtype == DF_CCM_SUBTYPE_CPU) { 2396dd23d762SRobert Mustacchi amdzen_topo_ioctl_df_fill_ccm(ent, &topo_ent); 2397dd23d762SRobert Mustacchi } 2398dd23d762SRobert Mustacchi 2399dd23d762SRobert Mustacchi if (ddi_copyout(&topo_ent, &topo_df.atd_df_ents[nwritten], 2400dd23d762SRobert Mustacchi sizeof (topo_ent), mode & FKIOCTL) != 0) { 2401dd23d762SRobert Mustacchi mutex_exit(&azn->azn_mutex); 2402dd23d762SRobert Mustacchi return (EFAULT); 2403dd23d762SRobert Mustacchi } 2404dd23d762SRobert Mustacchi nwritten++; 2405dd23d762SRobert Mustacchi } 2406dd23d762SRobert Mustacchi mutex_exit(&azn->azn_mutex); 2407dd23d762SRobert Mustacchi 2408dd23d762SRobert Mustacchi topo_df.atd_df_buf_nvalid = nwritten; 2409dd23d762SRobert Mustacchi copyout: 2410dd23d762SRobert Mustacchi switch (model) { 2411dd23d762SRobert Mustacchi #ifdef _MULTI_DATAMODEL 2412dd23d762SRobert Mustacchi case DDI_MODEL_ILP32: 2413dd23d762SRobert Mustacchi topo_df32.atd_nodeid = topo_df.atd_nodeid; 2414dd23d762SRobert Mustacchi topo_df32.atd_sockid = topo_df.atd_sockid; 2415dd23d762SRobert Mustacchi topo_df32.atd_dieid = topo_df.atd_dieid; 2416dd23d762SRobert Mustacchi topo_df32.atd_rev = topo_df.atd_rev; 2417dd23d762SRobert Mustacchi topo_df32.atd_df_buf_nvalid = topo_df.atd_df_buf_nvalid; 2418dd23d762SRobert Mustacchi topo_df32.atd_df_act_nents = topo_df.atd_df_act_nents; 2419dd23d762SRobert Mustacchi 2420dd23d762SRobert Mustacchi if (ddi_copyout(&topo_df32, (void *)(uintptr_t)arg, 2421dd23d762SRobert Mustacchi sizeof (topo_df32), mode & FKIOCTL) != 0) { 2422dd23d762SRobert Mustacchi return (EFAULT); 2423dd23d762SRobert Mustacchi } 2424dd23d762SRobert Mustacchi break; 2425dd23d762SRobert Mustacchi #endif 2426dd23d762SRobert Mustacchi case DDI_MODEL_NONE: 2427dd23d762SRobert Mustacchi if (ddi_copyout(&topo_df, (void *)(uintptr_t)arg, 2428dd23d762SRobert Mustacchi sizeof (topo_df), mode & FKIOCTL) != 0) { 2429dd23d762SRobert Mustacchi return (EFAULT); 2430dd23d762SRobert Mustacchi } 2431dd23d762SRobert Mustacchi break; 2432dd23d762SRobert Mustacchi default: 2433dd23d762SRobert Mustacchi break; 2434dd23d762SRobert Mustacchi } 2435dd23d762SRobert Mustacchi 2436dd23d762SRobert Mustacchi 2437dd23d762SRobert Mustacchi return (0); 2438dd23d762SRobert Mustacchi } 2439dd23d762SRobert Mustacchi 2440dd23d762SRobert Mustacchi static int 2441dd23d762SRobert Mustacchi amdzen_topo_ioctl_ccd(amdzen_t *azn, intptr_t arg, int mode) 2442dd23d762SRobert Mustacchi { 2443dd23d762SRobert Mustacchi amdzen_topo_ccd_t ccd, *ccdp; 2444dd23d762SRobert Mustacchi amdzen_df_t *df; 2445dd23d762SRobert Mustacchi amdzen_df_ent_t *ent; 2446dd23d762SRobert Mustacchi amdzen_ccm_data_t *ccm; 2447dd23d762SRobert Mustacchi uint32_t ccdno; 2448dd23d762SRobert Mustacchi size_t copyin_size = offsetof(amdzen_topo_ccd_t, atccd_err); 2449dd23d762SRobert Mustacchi 2450dd23d762SRobert Mustacchi /* 2451dd23d762SRobert Mustacchi * Only copy in the identifying information so that way we can ensure 2452dd23d762SRobert Mustacchi * the rest of the structure we return to the user doesn't contain 2453dd23d762SRobert Mustacchi * anything unexpected in it. 2454dd23d762SRobert Mustacchi */ 2455dd23d762SRobert Mustacchi bzero(&ccd, sizeof (ccd)); 2456dd23d762SRobert Mustacchi if (ddi_copyin((void *)(uintptr_t)arg, &ccd, copyin_size, 2457dd23d762SRobert Mustacchi mode & FKIOCTL) != 0) { 2458dd23d762SRobert Mustacchi return (EFAULT); 2459dd23d762SRobert Mustacchi } 2460dd23d762SRobert Mustacchi 2461dd23d762SRobert Mustacchi mutex_enter(&azn->azn_mutex); 2462dd23d762SRobert Mustacchi if ((azn->azn_flags & AMDZEN_F_APIC_DECOMP_VALID) == 0) { 2463dd23d762SRobert Mustacchi ccd.atccd_err = AMDZEN_TOPO_CCD_E_NO_APIC_DECOMP; 2464dd23d762SRobert Mustacchi goto copyout; 2465dd23d762SRobert Mustacchi } 2466dd23d762SRobert Mustacchi 2467dd23d762SRobert Mustacchi df = amdzen_df_find(azn, ccd.atccd_dfno); 2468dd23d762SRobert Mustacchi if (df == NULL) { 2469dd23d762SRobert Mustacchi ccd.atccd_err = AMDZEN_TOPO_CCD_E_BAD_DFNO; 2470dd23d762SRobert Mustacchi goto copyout; 2471dd23d762SRobert Mustacchi } 2472dd23d762SRobert Mustacchi 2473dd23d762SRobert Mustacchi /* 2474dd23d762SRobert Mustacchi * We don't have enough information to know how to construct this 2475dd23d762SRobert Mustacchi * information in Zen 1 at this time, so refuse. 2476dd23d762SRobert Mustacchi */ 2477dd23d762SRobert Mustacchi if (df->adf_rev <= DF_REV_2) { 2478dd23d762SRobert Mustacchi ccd.atccd_err = AMDZEN_TOPO_CCD_E_SOC_UNSUPPORTED; 2479dd23d762SRobert Mustacchi goto copyout; 2480dd23d762SRobert Mustacchi } 2481dd23d762SRobert Mustacchi 2482dd23d762SRobert Mustacchi ent = amdzen_df_ent_find_by_instid(df, ccd.atccd_instid); 2483dd23d762SRobert Mustacchi if (ent == NULL) { 2484dd23d762SRobert Mustacchi ccd.atccd_err = AMDZEN_TOPO_CCD_E_BAD_INSTID; 2485dd23d762SRobert Mustacchi goto copyout; 2486dd23d762SRobert Mustacchi } 2487dd23d762SRobert Mustacchi 2488dd23d762SRobert Mustacchi if (ent->adfe_type != DF_TYPE_CCM || 2489dd23d762SRobert Mustacchi ent->adfe_subtype != DF_CCM_SUBTYPE_CPU) { 2490dd23d762SRobert Mustacchi ccd.atccd_err = AMDZEN_TOPO_CCD_E_NOT_A_CCD; 2491dd23d762SRobert Mustacchi goto copyout; 2492dd23d762SRobert Mustacchi } 2493dd23d762SRobert Mustacchi 2494dd23d762SRobert Mustacchi ccm = &ent->adfe_data.aded_ccm; 2495dd23d762SRobert Mustacchi for (ccdno = 0; ccdno < DF_MAX_CCDS_PER_CCM; ccdno++) { 2496dd23d762SRobert Mustacchi if (ccm->acd_ccd_en[ccdno] != 0 && 2497dd23d762SRobert Mustacchi ccm->acd_ccd_id[ccdno] == ccd.atccd_phys_no) { 2498dd23d762SRobert Mustacchi break; 2499dd23d762SRobert Mustacchi } 2500dd23d762SRobert Mustacchi } 2501dd23d762SRobert Mustacchi 2502dd23d762SRobert Mustacchi if (ccdno == DF_MAX_CCDS_PER_CCM) { 2503dd23d762SRobert Mustacchi ccd.atccd_err = AMDZEN_TOPO_CCD_E_NOT_A_CCD; 2504dd23d762SRobert Mustacchi goto copyout; 2505dd23d762SRobert Mustacchi } 2506dd23d762SRobert Mustacchi 2507dd23d762SRobert Mustacchi if (ccm->acd_ccd_data[ccdno] == NULL) { 2508dd23d762SRobert Mustacchi /* 2509dd23d762SRobert Mustacchi * We don't actually have this data. Go fill it out and save it 2510dd23d762SRobert Mustacchi * for future use. 2511dd23d762SRobert Mustacchi */ 2512dd23d762SRobert Mustacchi ccdp = kmem_zalloc(sizeof (amdzen_topo_ccd_t), KM_NOSLEEP_LAZY); 2513dd23d762SRobert Mustacchi if (ccdp == NULL) { 2514dd23d762SRobert Mustacchi mutex_exit(&azn->azn_mutex); 2515dd23d762SRobert Mustacchi return (ENOMEM); 2516dd23d762SRobert Mustacchi } 2517dd23d762SRobert Mustacchi 2518dd23d762SRobert Mustacchi ccdp->atccd_dfno = ccd.atccd_dfno; 2519dd23d762SRobert Mustacchi ccdp->atccd_instid = ccd.atccd_instid; 2520dd23d762SRobert Mustacchi ccdp->atccd_phys_no = ccd.atccd_phys_no; 2521dd23d762SRobert Mustacchi amdzen_ccd_fill_topo(azn, df, ent, ccdp); 2522dd23d762SRobert Mustacchi ccm->acd_ccd_data[ccdno] = ccdp; 2523dd23d762SRobert Mustacchi } 2524dd23d762SRobert Mustacchi ASSERT3P(ccm->acd_ccd_data[ccdno], !=, NULL); 2525dd23d762SRobert Mustacchi bcopy(ccm->acd_ccd_data[ccdno], &ccd, sizeof (ccd)); 2526dd23d762SRobert Mustacchi 2527dd23d762SRobert Mustacchi copyout: 2528dd23d762SRobert Mustacchi mutex_exit(&azn->azn_mutex); 2529dd23d762SRobert Mustacchi if (ddi_copyout(&ccd, (void *)(uintptr_t)arg, sizeof (ccd), 2530dd23d762SRobert Mustacchi mode & FKIOCTL) != 0) { 2531dd23d762SRobert Mustacchi return (EFAULT); 2532dd23d762SRobert Mustacchi } 2533dd23d762SRobert Mustacchi 2534dd23d762SRobert Mustacchi return (0); 2535dd23d762SRobert Mustacchi } 2536dd23d762SRobert Mustacchi 2537dd23d762SRobert Mustacchi static int 2538dd23d762SRobert Mustacchi amdzen_topo_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, 2539dd23d762SRobert Mustacchi cred_t *credp, int *rvalp) 2540dd23d762SRobert Mustacchi { 2541dd23d762SRobert Mustacchi int ret; 2542dd23d762SRobert Mustacchi amdzen_t *azn = amdzen_data; 2543dd23d762SRobert Mustacchi 2544dd23d762SRobert Mustacchi if (getminor(dev) != AMDZEN_MINOR_TOPO) { 2545dd23d762SRobert Mustacchi return (ENXIO); 2546dd23d762SRobert Mustacchi } 2547dd23d762SRobert Mustacchi 2548dd23d762SRobert Mustacchi if ((mode & FREAD) == 0) { 2549dd23d762SRobert Mustacchi return (EBADF); 2550dd23d762SRobert Mustacchi } 2551dd23d762SRobert Mustacchi 2552dd23d762SRobert Mustacchi switch (cmd) { 2553dd23d762SRobert Mustacchi case AMDZEN_TOPO_IOCTL_BASE: 2554dd23d762SRobert Mustacchi ret = amdzen_topo_ioctl_base(azn, arg, mode); 2555dd23d762SRobert Mustacchi break; 2556dd23d762SRobert Mustacchi case AMDZEN_TOPO_IOCTL_DF: 2557dd23d762SRobert Mustacchi ret = amdzen_topo_ioctl_df(azn, arg, mode); 2558dd23d762SRobert Mustacchi break; 2559dd23d762SRobert Mustacchi case AMDZEN_TOPO_IOCTL_CCD: 2560dd23d762SRobert Mustacchi ret = amdzen_topo_ioctl_ccd(azn, arg, mode); 2561dd23d762SRobert Mustacchi break; 2562dd23d762SRobert Mustacchi default: 2563dd23d762SRobert Mustacchi ret = ENOTTY; 2564dd23d762SRobert Mustacchi break; 2565dd23d762SRobert Mustacchi } 2566dd23d762SRobert Mustacchi 2567dd23d762SRobert Mustacchi return (ret); 2568dd23d762SRobert Mustacchi } 2569dd23d762SRobert Mustacchi 2570dd23d762SRobert Mustacchi static int 2571dd23d762SRobert Mustacchi amdzen_topo_close(dev_t dev, int flag, int otyp, cred_t *credp) 2572dd23d762SRobert Mustacchi { 2573dd23d762SRobert Mustacchi if (otyp != OTYP_CHR) { 2574dd23d762SRobert Mustacchi return (EINVAL); 2575dd23d762SRobert Mustacchi } 2576dd23d762SRobert Mustacchi 2577dd23d762SRobert Mustacchi if (getminor(dev) != AMDZEN_MINOR_TOPO) { 2578dd23d762SRobert Mustacchi return (ENXIO); 2579dd23d762SRobert Mustacchi } 2580dd23d762SRobert Mustacchi 2581dd23d762SRobert Mustacchi return (0); 2582dd23d762SRobert Mustacchi } 2583dd23d762SRobert Mustacchi 2584dd23d762SRobert Mustacchi static int 2585047043c2SRobert Mustacchi amdzen_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 2586047043c2SRobert Mustacchi { 2587047043c2SRobert Mustacchi amdzen_t *azn = amdzen_data; 2588047043c2SRobert Mustacchi 2589047043c2SRobert Mustacchi if (cmd == DDI_RESUME) { 2590047043c2SRobert Mustacchi return (DDI_SUCCESS); 2591047043c2SRobert Mustacchi } else if (cmd != DDI_ATTACH) { 2592047043c2SRobert Mustacchi return (DDI_FAILURE); 2593047043c2SRobert Mustacchi } 2594047043c2SRobert Mustacchi 2595047043c2SRobert Mustacchi mutex_enter(&azn->azn_mutex); 2596047043c2SRobert Mustacchi if (azn->azn_dip != NULL) { 2597047043c2SRobert Mustacchi dev_err(dip, CE_WARN, "driver is already attached!"); 2598047043c2SRobert Mustacchi mutex_exit(&azn->azn_mutex); 2599047043c2SRobert Mustacchi return (DDI_FAILURE); 2600047043c2SRobert Mustacchi } 2601047043c2SRobert Mustacchi 2602dd23d762SRobert Mustacchi if (ddi_create_minor_node(dip, "topo", S_IFCHR, AMDZEN_MINOR_TOPO, 2603dd23d762SRobert Mustacchi DDI_PSEUDO, 0) != 0) { 2604dd23d762SRobert Mustacchi dev_err(dip, CE_WARN, "failed to create topo minor node!"); 2605dd23d762SRobert Mustacchi mutex_exit(&azn->azn_mutex); 2606dd23d762SRobert Mustacchi return (DDI_FAILURE); 2607dd23d762SRobert Mustacchi } 2608dd23d762SRobert Mustacchi 2609047043c2SRobert Mustacchi azn->azn_dip = dip; 2610047043c2SRobert Mustacchi azn->azn_taskqid = taskq_dispatch(system_taskq, amdzen_stub_scan, 2611047043c2SRobert Mustacchi azn, TQ_SLEEP); 2612047043c2SRobert Mustacchi azn->azn_flags |= AMDZEN_F_SCAN_DISPATCHED; 2613047043c2SRobert Mustacchi mutex_exit(&azn->azn_mutex); 2614047043c2SRobert Mustacchi 2615047043c2SRobert Mustacchi return (DDI_SUCCESS); 2616047043c2SRobert Mustacchi } 2617047043c2SRobert Mustacchi 2618047043c2SRobert Mustacchi static int 2619047043c2SRobert Mustacchi amdzen_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 2620047043c2SRobert Mustacchi { 2621047043c2SRobert Mustacchi amdzen_t *azn = amdzen_data; 2622047043c2SRobert Mustacchi 2623047043c2SRobert Mustacchi if (cmd == DDI_SUSPEND) { 2624047043c2SRobert Mustacchi return (DDI_SUCCESS); 2625047043c2SRobert Mustacchi } else if (cmd != DDI_DETACH) { 2626047043c2SRobert Mustacchi return (DDI_FAILURE); 2627047043c2SRobert Mustacchi } 2628047043c2SRobert Mustacchi 2629047043c2SRobert Mustacchi mutex_enter(&azn->azn_mutex); 2630047043c2SRobert Mustacchi while (azn->azn_taskqid != TASKQID_INVALID) { 2631047043c2SRobert Mustacchi cv_wait(&azn->azn_cv, &azn->azn_mutex); 2632047043c2SRobert Mustacchi } 2633047043c2SRobert Mustacchi 2634047043c2SRobert Mustacchi /* 2635047043c2SRobert Mustacchi * If we've attached any stub drivers, e.g. this platform is important 2636047043c2SRobert Mustacchi * for us, then we fail detach. 2637047043c2SRobert Mustacchi */ 2638047043c2SRobert Mustacchi if (!list_is_empty(&azn->azn_df_stubs) || 2639047043c2SRobert Mustacchi !list_is_empty(&azn->azn_nb_stubs)) { 2640047043c2SRobert Mustacchi mutex_exit(&azn->azn_mutex); 2641047043c2SRobert Mustacchi return (DDI_FAILURE); 2642047043c2SRobert Mustacchi } 2643047043c2SRobert Mustacchi 2644dd23d762SRobert Mustacchi ddi_remove_minor_node(azn->azn_dip, NULL); 2645047043c2SRobert Mustacchi azn->azn_dip = NULL; 2646047043c2SRobert Mustacchi mutex_exit(&azn->azn_mutex); 2647047043c2SRobert Mustacchi 2648047043c2SRobert Mustacchi return (DDI_SUCCESS); 2649047043c2SRobert Mustacchi } 2650047043c2SRobert Mustacchi 2651047043c2SRobert Mustacchi static void 2652047043c2SRobert Mustacchi amdzen_free(void) 2653047043c2SRobert Mustacchi { 2654047043c2SRobert Mustacchi if (amdzen_data == NULL) { 2655047043c2SRobert Mustacchi return; 2656047043c2SRobert Mustacchi } 2657047043c2SRobert Mustacchi 2658047043c2SRobert Mustacchi VERIFY(list_is_empty(&amdzen_data->azn_df_stubs)); 2659047043c2SRobert Mustacchi list_destroy(&amdzen_data->azn_df_stubs); 2660047043c2SRobert Mustacchi VERIFY(list_is_empty(&amdzen_data->azn_nb_stubs)); 2661047043c2SRobert Mustacchi list_destroy(&amdzen_data->azn_nb_stubs); 2662047043c2SRobert Mustacchi cv_destroy(&amdzen_data->azn_cv); 2663047043c2SRobert Mustacchi mutex_destroy(&amdzen_data->azn_mutex); 2664047043c2SRobert Mustacchi kmem_free(amdzen_data, sizeof (amdzen_t)); 2665047043c2SRobert Mustacchi amdzen_data = NULL; 2666047043c2SRobert Mustacchi } 2667047043c2SRobert Mustacchi 2668047043c2SRobert Mustacchi static void 2669047043c2SRobert Mustacchi amdzen_alloc(void) 2670047043c2SRobert Mustacchi { 2671047043c2SRobert Mustacchi amdzen_data = kmem_zalloc(sizeof (amdzen_t), KM_SLEEP); 2672047043c2SRobert Mustacchi mutex_init(&amdzen_data->azn_mutex, NULL, MUTEX_DRIVER, NULL); 2673047043c2SRobert Mustacchi list_create(&amdzen_data->azn_df_stubs, sizeof (amdzen_stub_t), 2674047043c2SRobert Mustacchi offsetof(amdzen_stub_t, azns_link)); 2675047043c2SRobert Mustacchi list_create(&amdzen_data->azn_nb_stubs, sizeof (amdzen_stub_t), 2676047043c2SRobert Mustacchi offsetof(amdzen_stub_t, azns_link)); 2677047043c2SRobert Mustacchi cv_init(&amdzen_data->azn_cv, NULL, CV_DRIVER, NULL); 2678047043c2SRobert Mustacchi } 2679047043c2SRobert Mustacchi 2680dd23d762SRobert Mustacchi static struct cb_ops amdzen_topo_cb_ops = { 2681dd23d762SRobert Mustacchi .cb_open = amdzen_topo_open, 2682dd23d762SRobert Mustacchi .cb_close = amdzen_topo_close, 2683dd23d762SRobert Mustacchi .cb_strategy = nodev, 2684dd23d762SRobert Mustacchi .cb_print = nodev, 2685dd23d762SRobert Mustacchi .cb_dump = nodev, 2686dd23d762SRobert Mustacchi .cb_read = nodev, 2687dd23d762SRobert Mustacchi .cb_write = nodev, 2688dd23d762SRobert Mustacchi .cb_ioctl = amdzen_topo_ioctl, 2689dd23d762SRobert Mustacchi .cb_devmap = nodev, 2690dd23d762SRobert Mustacchi .cb_mmap = nodev, 2691dd23d762SRobert Mustacchi .cb_segmap = nodev, 2692dd23d762SRobert Mustacchi .cb_chpoll = nochpoll, 2693dd23d762SRobert Mustacchi .cb_prop_op = ddi_prop_op, 2694dd23d762SRobert Mustacchi .cb_flag = D_MP, 2695dd23d762SRobert Mustacchi .cb_rev = CB_REV, 2696dd23d762SRobert Mustacchi .cb_aread = nodev, 2697dd23d762SRobert Mustacchi .cb_awrite = nodev 2698dd23d762SRobert Mustacchi }; 2699dd23d762SRobert Mustacchi 2700047043c2SRobert Mustacchi struct bus_ops amdzen_bus_ops = { 2701047043c2SRobert Mustacchi .busops_rev = BUSO_REV, 2702047043c2SRobert Mustacchi .bus_map = nullbusmap, 2703047043c2SRobert Mustacchi .bus_dma_map = ddi_no_dma_map, 2704047043c2SRobert Mustacchi .bus_dma_allochdl = ddi_no_dma_allochdl, 2705047043c2SRobert Mustacchi .bus_dma_freehdl = ddi_no_dma_freehdl, 2706047043c2SRobert Mustacchi .bus_dma_bindhdl = ddi_no_dma_bindhdl, 2707047043c2SRobert Mustacchi .bus_dma_unbindhdl = ddi_no_dma_unbindhdl, 2708047043c2SRobert Mustacchi .bus_dma_flush = ddi_no_dma_flush, 2709047043c2SRobert Mustacchi .bus_dma_win = ddi_no_dma_win, 2710047043c2SRobert Mustacchi .bus_dma_ctl = ddi_no_dma_mctl, 2711047043c2SRobert Mustacchi .bus_prop_op = ddi_bus_prop_op, 2712047043c2SRobert Mustacchi .bus_ctl = amdzen_bus_ctl 2713047043c2SRobert Mustacchi }; 2714047043c2SRobert Mustacchi 2715047043c2SRobert Mustacchi static struct dev_ops amdzen_dev_ops = { 2716047043c2SRobert Mustacchi .devo_rev = DEVO_REV, 2717047043c2SRobert Mustacchi .devo_refcnt = 0, 2718047043c2SRobert Mustacchi .devo_getinfo = nodev, 2719047043c2SRobert Mustacchi .devo_identify = nulldev, 2720047043c2SRobert Mustacchi .devo_probe = nulldev, 2721047043c2SRobert Mustacchi .devo_attach = amdzen_attach, 2722047043c2SRobert Mustacchi .devo_detach = amdzen_detach, 2723047043c2SRobert Mustacchi .devo_reset = nodev, 2724047043c2SRobert Mustacchi .devo_quiesce = ddi_quiesce_not_needed, 2725dd23d762SRobert Mustacchi .devo_bus_ops = &amdzen_bus_ops, 2726dd23d762SRobert Mustacchi .devo_cb_ops = &amdzen_topo_cb_ops 2727047043c2SRobert Mustacchi }; 2728047043c2SRobert Mustacchi 2729047043c2SRobert Mustacchi static struct modldrv amdzen_modldrv = { 2730047043c2SRobert Mustacchi .drv_modops = &mod_driverops, 2731047043c2SRobert Mustacchi .drv_linkinfo = "AMD Zen Nexus Driver", 2732047043c2SRobert Mustacchi .drv_dev_ops = &amdzen_dev_ops 2733047043c2SRobert Mustacchi }; 2734047043c2SRobert Mustacchi 2735047043c2SRobert Mustacchi static struct modlinkage amdzen_modlinkage = { 2736047043c2SRobert Mustacchi .ml_rev = MODREV_1, 2737047043c2SRobert Mustacchi .ml_linkage = { &amdzen_modldrv, NULL } 2738047043c2SRobert Mustacchi }; 2739047043c2SRobert Mustacchi 2740047043c2SRobert Mustacchi int 2741047043c2SRobert Mustacchi _init(void) 2742047043c2SRobert Mustacchi { 2743047043c2SRobert Mustacchi int ret; 2744047043c2SRobert Mustacchi 27459b0429a1SPu Wen if (cpuid_getvendor(CPU) != X86_VENDOR_AMD && 27469b0429a1SPu Wen cpuid_getvendor(CPU) != X86_VENDOR_HYGON) { 2747047043c2SRobert Mustacchi return (ENOTSUP); 2748047043c2SRobert Mustacchi } 2749047043c2SRobert Mustacchi 2750047043c2SRobert Mustacchi if ((ret = mod_install(&amdzen_modlinkage)) == 0) { 2751047043c2SRobert Mustacchi amdzen_alloc(); 2752047043c2SRobert Mustacchi } 2753047043c2SRobert Mustacchi 2754047043c2SRobert Mustacchi return (ret); 2755047043c2SRobert Mustacchi } 2756047043c2SRobert Mustacchi 2757047043c2SRobert Mustacchi int 2758047043c2SRobert Mustacchi _info(struct modinfo *modinfop) 2759047043c2SRobert Mustacchi { 2760047043c2SRobert Mustacchi return (mod_info(&amdzen_modlinkage, modinfop)); 2761047043c2SRobert Mustacchi } 2762047043c2SRobert Mustacchi 2763047043c2SRobert Mustacchi int 2764047043c2SRobert Mustacchi _fini(void) 2765047043c2SRobert Mustacchi { 2766047043c2SRobert Mustacchi int ret; 2767047043c2SRobert Mustacchi 2768047043c2SRobert Mustacchi if ((ret = mod_remove(&amdzen_modlinkage)) == 0) { 2769047043c2SRobert Mustacchi amdzen_free(); 2770047043c2SRobert Mustacchi } 2771047043c2SRobert Mustacchi 2772047043c2SRobert Mustacchi return (ret); 2773047043c2SRobert Mustacchi } 2774