1 /* 2 * Copyright 2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * 23 */ 24 #include <linux/list.h> 25 #include "amdgpu.h" 26 #include "amdgpu_xgmi.h" 27 #include "amdgpu_smu.h" 28 #include "df/df_3_6_offset.h" 29 30 static DEFINE_MUTEX(xgmi_mutex); 31 32 #define AMDGPU_MAX_XGMI_HIVE 8 33 #define AMDGPU_MAX_XGMI_DEVICE_PER_HIVE 4 34 35 static struct amdgpu_hive_info xgmi_hives[AMDGPU_MAX_XGMI_HIVE]; 36 static unsigned hive_count = 0; 37 38 void *amdgpu_xgmi_hive_try_lock(struct amdgpu_hive_info *hive) 39 { 40 return &hive->device_list; 41 } 42 43 /** 44 * DOC: AMDGPU XGMI Support 45 * 46 * XGMI is a high speed interconnect that joins multiple GPU cards 47 * into a homogeneous memory space that is organized by a collective 48 * hive ID and individual node IDs, both of which are 64-bit numbers. 49 * 50 * The file xgmi_device_id contains the unique per GPU device ID and 51 * is stored in the /sys/class/drm/card${cardno}/device/ directory. 52 * 53 * Inside the device directory a sub-directory 'xgmi_hive_info' is 54 * created which contains the hive ID and the list of nodes. 55 * 56 * The hive ID is stored in: 57 * /sys/class/drm/card${cardno}/device/xgmi_hive_info/xgmi_hive_id 58 * 59 * The node information is stored in numbered directories: 60 * /sys/class/drm/card${cardno}/device/xgmi_hive_info/node${nodeno}/xgmi_device_id 61 * 62 * Each device has their own xgmi_hive_info direction with a mirror 63 * set of node sub-directories. 64 * 65 * The XGMI memory space is built by contiguously adding the power of 66 * two padded VRAM space from each node to each other. 67 * 68 */ 69 70 71 static ssize_t amdgpu_xgmi_show_hive_id(struct device *dev, 72 struct device_attribute *attr, char *buf) 73 { 74 struct amdgpu_hive_info *hive = 75 container_of(attr, struct amdgpu_hive_info, dev_attr); 76 77 return snprintf(buf, PAGE_SIZE, "%llu\n", hive->hive_id); 78 } 79 80 static int amdgpu_xgmi_sysfs_create(struct amdgpu_device *adev, 81 struct amdgpu_hive_info *hive) 82 { 83 int ret = 0; 84 85 if (WARN_ON(hive->kobj)) 86 return -EINVAL; 87 88 hive->kobj = kobject_create_and_add("xgmi_hive_info", &adev->dev->kobj); 89 if (!hive->kobj) { 90 dev_err(adev->dev, "XGMI: Failed to allocate sysfs entry!\n"); 91 return -EINVAL; 92 } 93 94 hive->dev_attr = (struct device_attribute) { 95 .attr = { 96 .name = "xgmi_hive_id", 97 .mode = S_IRUGO, 98 99 }, 100 .show = amdgpu_xgmi_show_hive_id, 101 }; 102 103 ret = sysfs_create_file(hive->kobj, &hive->dev_attr.attr); 104 if (ret) { 105 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_hive_id\n"); 106 kobject_del(hive->kobj); 107 kobject_put(hive->kobj); 108 hive->kobj = NULL; 109 } 110 111 return ret; 112 } 113 114 static void amdgpu_xgmi_sysfs_destroy(struct amdgpu_device *adev, 115 struct amdgpu_hive_info *hive) 116 { 117 sysfs_remove_file(hive->kobj, &hive->dev_attr.attr); 118 kobject_del(hive->kobj); 119 kobject_put(hive->kobj); 120 hive->kobj = NULL; 121 } 122 123 static ssize_t amdgpu_xgmi_show_device_id(struct device *dev, 124 struct device_attribute *attr, 125 char *buf) 126 { 127 struct drm_device *ddev = dev_get_drvdata(dev); 128 struct amdgpu_device *adev = ddev->dev_private; 129 130 return snprintf(buf, PAGE_SIZE, "%llu\n", adev->gmc.xgmi.node_id); 131 132 } 133 134 #define AMDGPU_XGMI_SET_FICAA(o) ((o) | 0x456801) 135 static ssize_t amdgpu_xgmi_show_error(struct device *dev, 136 struct device_attribute *attr, 137 char *buf) 138 { 139 struct drm_device *ddev = dev_get_drvdata(dev); 140 struct amdgpu_device *adev = ddev->dev_private; 141 uint32_t ficaa_pie_ctl_in, ficaa_pie_status_in; 142 uint64_t fica_out; 143 unsigned int error_count = 0; 144 145 ficaa_pie_ctl_in = AMDGPU_XGMI_SET_FICAA(0x200); 146 ficaa_pie_status_in = AMDGPU_XGMI_SET_FICAA(0x208); 147 148 fica_out = adev->df_funcs->get_fica(adev, ficaa_pie_ctl_in); 149 if (fica_out != 0x1f) 150 pr_err("xGMI error counters not enabled!\n"); 151 152 fica_out = adev->df_funcs->get_fica(adev, ficaa_pie_status_in); 153 154 if ((fica_out & 0xffff) == 2) 155 error_count = ((fica_out >> 62) & 0x1) + (fica_out >> 63); 156 157 adev->df_funcs->set_fica(adev, ficaa_pie_status_in, 0, 0); 158 159 return snprintf(buf, PAGE_SIZE, "%d\n", error_count); 160 } 161 162 163 static DEVICE_ATTR(xgmi_device_id, S_IRUGO, amdgpu_xgmi_show_device_id, NULL); 164 static DEVICE_ATTR(xgmi_error, S_IRUGO, amdgpu_xgmi_show_error, NULL); 165 166 static int amdgpu_xgmi_sysfs_add_dev_info(struct amdgpu_device *adev, 167 struct amdgpu_hive_info *hive) 168 { 169 int ret = 0; 170 char node[10] = { 0 }; 171 172 /* Create xgmi device id file */ 173 ret = device_create_file(adev->dev, &dev_attr_xgmi_device_id); 174 if (ret) { 175 dev_err(adev->dev, "XGMI: Failed to create device file xgmi_device_id\n"); 176 return ret; 177 } 178 179 /* Create xgmi error file */ 180 ret = device_create_file(adev->dev, &dev_attr_xgmi_error); 181 if (ret) 182 pr_err("failed to create xgmi_error\n"); 183 184 185 /* Create sysfs link to hive info folder on the first device */ 186 if (adev != hive->adev) { 187 ret = sysfs_create_link(&adev->dev->kobj, hive->kobj, 188 "xgmi_hive_info"); 189 if (ret) { 190 dev_err(adev->dev, "XGMI: Failed to create link to hive info"); 191 goto remove_file; 192 } 193 } 194 195 sprintf(node, "node%d", hive->number_devices); 196 /* Create sysfs link form the hive folder to yourself */ 197 ret = sysfs_create_link(hive->kobj, &adev->dev->kobj, node); 198 if (ret) { 199 dev_err(adev->dev, "XGMI: Failed to create link from hive info"); 200 goto remove_link; 201 } 202 203 goto success; 204 205 206 remove_link: 207 sysfs_remove_link(&adev->dev->kobj, adev->ddev->unique); 208 209 remove_file: 210 device_remove_file(adev->dev, &dev_attr_xgmi_device_id); 211 212 success: 213 return ret; 214 } 215 216 static void amdgpu_xgmi_sysfs_rem_dev_info(struct amdgpu_device *adev, 217 struct amdgpu_hive_info *hive) 218 { 219 device_remove_file(adev->dev, &dev_attr_xgmi_device_id); 220 sysfs_remove_link(&adev->dev->kobj, adev->ddev->unique); 221 sysfs_remove_link(hive->kobj, adev->ddev->unique); 222 } 223 224 225 226 struct amdgpu_hive_info *amdgpu_get_xgmi_hive(struct amdgpu_device *adev, int lock) 227 { 228 int i; 229 struct amdgpu_hive_info *tmp; 230 231 if (!adev->gmc.xgmi.hive_id) 232 return NULL; 233 234 mutex_lock(&xgmi_mutex); 235 236 for (i = 0 ; i < hive_count; ++i) { 237 tmp = &xgmi_hives[i]; 238 if (tmp->hive_id == adev->gmc.xgmi.hive_id) { 239 if (lock) 240 mutex_lock(&tmp->hive_lock); 241 mutex_unlock(&xgmi_mutex); 242 return tmp; 243 } 244 } 245 if (i >= AMDGPU_MAX_XGMI_HIVE) { 246 mutex_unlock(&xgmi_mutex); 247 return NULL; 248 } 249 250 /* initialize new hive if not exist */ 251 tmp = &xgmi_hives[hive_count++]; 252 253 if (amdgpu_xgmi_sysfs_create(adev, tmp)) { 254 mutex_unlock(&xgmi_mutex); 255 return NULL; 256 } 257 258 tmp->adev = adev; 259 tmp->hive_id = adev->gmc.xgmi.hive_id; 260 INIT_LIST_HEAD(&tmp->device_list); 261 mutex_init(&tmp->hive_lock); 262 mutex_init(&tmp->reset_lock); 263 264 if (lock) 265 mutex_lock(&tmp->hive_lock); 266 tmp->pstate = -1; 267 mutex_unlock(&xgmi_mutex); 268 269 return tmp; 270 } 271 272 int amdgpu_xgmi_set_pstate(struct amdgpu_device *adev, int pstate) 273 { 274 int ret = 0; 275 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev, 0); 276 277 if (!hive) 278 return 0; 279 280 if (hive->pstate == pstate) 281 return 0; 282 283 dev_dbg(adev->dev, "Set xgmi pstate %d.\n", pstate); 284 285 if (is_support_sw_smu_xgmi(adev)) 286 ret = smu_set_xgmi_pstate(&adev->smu, pstate); 287 if (ret) 288 dev_err(adev->dev, 289 "XGMI: Set pstate failure on device %llx, hive %llx, ret %d", 290 adev->gmc.xgmi.node_id, 291 adev->gmc.xgmi.hive_id, ret); 292 293 return ret; 294 } 295 296 int amdgpu_xgmi_update_topology(struct amdgpu_hive_info *hive, struct amdgpu_device *adev) 297 { 298 int ret = -EINVAL; 299 300 /* Each psp need to set the latest topology */ 301 ret = psp_xgmi_set_topology_info(&adev->psp, 302 hive->number_devices, 303 &adev->psp.xgmi_context.top_info); 304 if (ret) 305 dev_err(adev->dev, 306 "XGMI: Set topology failure on device %llx, hive %llx, ret %d", 307 adev->gmc.xgmi.node_id, 308 adev->gmc.xgmi.hive_id, ret); 309 310 return ret; 311 } 312 313 314 int amdgpu_xgmi_get_hops_count(struct amdgpu_device *adev, 315 struct amdgpu_device *peer_adev) 316 { 317 struct psp_xgmi_topology_info *top = &adev->psp.xgmi_context.top_info; 318 int i; 319 320 for (i = 0 ; i < top->num_nodes; ++i) 321 if (top->nodes[i].node_id == peer_adev->gmc.xgmi.node_id) 322 return top->nodes[i].num_hops; 323 return -EINVAL; 324 } 325 326 int amdgpu_xgmi_add_device(struct amdgpu_device *adev) 327 { 328 struct psp_xgmi_topology_info *top_info; 329 struct amdgpu_hive_info *hive; 330 struct amdgpu_xgmi *entry; 331 struct amdgpu_device *tmp_adev = NULL; 332 333 int count = 0, ret = 0; 334 335 if (!adev->gmc.xgmi.supported) 336 return 0; 337 338 if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) { 339 ret = psp_xgmi_get_hive_id(&adev->psp, &adev->gmc.xgmi.hive_id); 340 if (ret) { 341 dev_err(adev->dev, 342 "XGMI: Failed to get hive id\n"); 343 return ret; 344 } 345 346 ret = psp_xgmi_get_node_id(&adev->psp, &adev->gmc.xgmi.node_id); 347 if (ret) { 348 dev_err(adev->dev, 349 "XGMI: Failed to get node id\n"); 350 return ret; 351 } 352 } else { 353 adev->gmc.xgmi.hive_id = 16; 354 adev->gmc.xgmi.node_id = adev->gmc.xgmi.physical_node_id + 16; 355 } 356 357 hive = amdgpu_get_xgmi_hive(adev, 1); 358 if (!hive) { 359 ret = -EINVAL; 360 dev_err(adev->dev, 361 "XGMI: node 0x%llx, can not match hive 0x%llx in the hive list.\n", 362 adev->gmc.xgmi.node_id, adev->gmc.xgmi.hive_id); 363 goto exit; 364 } 365 366 top_info = &adev->psp.xgmi_context.top_info; 367 368 list_add_tail(&adev->gmc.xgmi.head, &hive->device_list); 369 list_for_each_entry(entry, &hive->device_list, head) 370 top_info->nodes[count++].node_id = entry->node_id; 371 top_info->num_nodes = count; 372 hive->number_devices = count; 373 374 if (amdgpu_device_ip_get_ip_block(adev, AMD_IP_BLOCK_TYPE_PSP)) { 375 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { 376 /* update node list for other device in the hive */ 377 if (tmp_adev != adev) { 378 top_info = &tmp_adev->psp.xgmi_context.top_info; 379 top_info->nodes[count - 1].node_id = 380 adev->gmc.xgmi.node_id; 381 top_info->num_nodes = count; 382 } 383 ret = amdgpu_xgmi_update_topology(hive, tmp_adev); 384 if (ret) 385 goto exit; 386 } 387 388 /* get latest topology info for each device from psp */ 389 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) { 390 ret = psp_xgmi_get_topology_info(&tmp_adev->psp, count, 391 &tmp_adev->psp.xgmi_context.top_info); 392 if (ret) { 393 dev_err(tmp_adev->dev, 394 "XGMI: Get topology failure on device %llx, hive %llx, ret %d", 395 tmp_adev->gmc.xgmi.node_id, 396 tmp_adev->gmc.xgmi.hive_id, ret); 397 /* To do : continue with some node failed or disable the whole hive */ 398 goto exit; 399 } 400 } 401 } 402 403 if (!ret) 404 ret = amdgpu_xgmi_sysfs_add_dev_info(adev, hive); 405 406 407 mutex_unlock(&hive->hive_lock); 408 exit: 409 if (!ret) 410 dev_info(adev->dev, "XGMI: Add node %d, hive 0x%llx.\n", 411 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id); 412 else 413 dev_err(adev->dev, "XGMI: Failed to add node %d, hive 0x%llx ret: %d\n", 414 adev->gmc.xgmi.physical_node_id, adev->gmc.xgmi.hive_id, 415 ret); 416 417 return ret; 418 } 419 420 void amdgpu_xgmi_remove_device(struct amdgpu_device *adev) 421 { 422 struct amdgpu_hive_info *hive; 423 424 if (!adev->gmc.xgmi.supported) 425 return; 426 427 hive = amdgpu_get_xgmi_hive(adev, 1); 428 if (!hive) 429 return; 430 431 if (!(hive->number_devices--)) { 432 amdgpu_xgmi_sysfs_destroy(adev, hive); 433 mutex_destroy(&hive->hive_lock); 434 mutex_destroy(&hive->reset_lock); 435 } else { 436 amdgpu_xgmi_sysfs_rem_dev_info(adev, hive); 437 mutex_unlock(&hive->hive_lock); 438 } 439 } 440