1 /*- 2 * Copyright (c) 2017 The FreeBSD Foundation 3 * All rights reserved. 4 * Copyright (c) 2018, 2019 Intel Corporation 5 * 6 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 7 * under sponsorship from the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "opt_acpi.h" 35 #include "opt_ddb.h" 36 37 #include <sys/param.h> 38 #include <sys/bio.h> 39 #include <sys/bitstring.h> 40 #include <sys/bus.h> 41 #include <sys/kernel.h> 42 #include <sys/lock.h> 43 #include <sys/malloc.h> 44 #include <sys/module.h> 45 #include <sys/uuid.h> 46 #include <contrib/dev/acpica/include/acpi.h> 47 #include <contrib/dev/acpica/include/accommon.h> 48 #include <contrib/dev/acpica/include/acuuid.h> 49 #include <dev/acpica/acpivar.h> 50 #include <dev/nvdimm/nvdimm_var.h> 51 52 #define _COMPONENT ACPI_OEM 53 ACPI_MODULE_NAME("NVDIMM") 54 55 static struct uuid intel_nvdimm_dsm_uuid = 56 {0x4309AC30,0x0D11,0x11E4,0x91,0x91,{0x08,0x00,0x20,0x0C,0x9A,0x66}}; 57 #define INTEL_NVDIMM_DSM_REV 1 58 #define INTEL_NVDIMM_DSM_GET_LABEL_SIZE 4 59 #define INTEL_NVDIMM_DSM_GET_LABEL_DATA 5 60 61 static devclass_t nvdimm_devclass; 62 static devclass_t nvdimm_root_devclass; 63 MALLOC_DEFINE(M_NVDIMM, "nvdimm", "NVDIMM driver memory"); 64 65 static int 66 read_label_area_size(struct nvdimm_dev *nv) 67 { 68 ACPI_OBJECT *result_buffer; 69 ACPI_HANDLE handle; 70 ACPI_STATUS status; 71 ACPI_BUFFER result; 72 uint32_t *out; 73 int error; 74 75 handle = nvdimm_root_get_acpi_handle(nv->nv_dev); 76 if (handle == NULL) 77 return (ENODEV); 78 result.Length = ACPI_ALLOCATE_BUFFER; 79 result.Pointer = NULL; 80 status = acpi_EvaluateDSM(handle, (uint8_t *)&intel_nvdimm_dsm_uuid, 81 INTEL_NVDIMM_DSM_REV, INTEL_NVDIMM_DSM_GET_LABEL_SIZE, NULL, 82 &result); 83 error = ENXIO; 84 if (ACPI_SUCCESS(status) && result.Pointer != NULL && 85 result.Length >= sizeof(ACPI_OBJECT)) { 86 result_buffer = result.Pointer; 87 if (result_buffer->Type == ACPI_TYPE_BUFFER && 88 result_buffer->Buffer.Length >= 12) { 89 out = (uint32_t *)result_buffer->Buffer.Pointer; 90 nv->label_area_size = out[1]; 91 nv->max_label_xfer = out[2]; 92 error = 0; 93 } 94 } 95 if (result.Pointer != NULL) 96 AcpiOsFree(result.Pointer); 97 return (error); 98 } 99 100 static int 101 read_label_area(struct nvdimm_dev *nv, uint8_t *dest, off_t offset, 102 off_t length) 103 { 104 ACPI_BUFFER result; 105 ACPI_HANDLE handle; 106 ACPI_OBJECT params_pkg, params_buf, *result_buf; 107 ACPI_STATUS status; 108 uint32_t params[2]; 109 off_t to_read; 110 int error; 111 112 error = 0; 113 handle = nvdimm_root_get_acpi_handle(nv->nv_dev); 114 if (offset < 0 || length <= 0 || 115 offset + length > nv->label_area_size || 116 handle == NULL) 117 return (ENODEV); 118 params_pkg.Type = ACPI_TYPE_PACKAGE; 119 params_pkg.Package.Count = 1; 120 params_pkg.Package.Elements = ¶ms_buf; 121 params_buf.Type = ACPI_TYPE_BUFFER; 122 params_buf.Buffer.Length = sizeof(params); 123 params_buf.Buffer.Pointer = (UINT8 *)params; 124 while (length > 0) { 125 to_read = MIN(length, nv->max_label_xfer); 126 params[0] = offset; 127 params[1] = to_read; 128 result.Length = ACPI_ALLOCATE_BUFFER; 129 result.Pointer = NULL; 130 status = acpi_EvaluateDSM(handle, 131 (uint8_t *)&intel_nvdimm_dsm_uuid, INTEL_NVDIMM_DSM_REV, 132 INTEL_NVDIMM_DSM_GET_LABEL_DATA, ¶ms_pkg, &result); 133 if (ACPI_FAILURE(status) || 134 result.Length < sizeof(ACPI_OBJECT) || 135 result.Pointer == NULL) { 136 error = ENXIO; 137 break; 138 } 139 result_buf = (ACPI_OBJECT *)result.Pointer; 140 if (result_buf->Type != ACPI_TYPE_BUFFER || 141 result_buf->Buffer.Pointer == NULL || 142 result_buf->Buffer.Length != 4 + to_read || 143 ((uint16_t *)result_buf->Buffer.Pointer)[0] != 0) { 144 error = ENXIO; 145 break; 146 } 147 bcopy(result_buf->Buffer.Pointer + 4, dest, to_read); 148 dest += to_read; 149 offset += to_read; 150 length -= to_read; 151 if (result.Pointer != NULL) { 152 AcpiOsFree(result.Pointer); 153 result.Pointer = NULL; 154 } 155 } 156 if (result.Pointer != NULL) 157 AcpiOsFree(result.Pointer); 158 return (error); 159 } 160 161 static uint64_t 162 fletcher64(const void *data, size_t length) 163 { 164 size_t i; 165 uint32_t a, b; 166 const uint32_t *d; 167 168 a = 0; 169 b = 0; 170 d = (const uint32_t *)data; 171 length = length / sizeof(uint32_t); 172 for (i = 0; i < length; i++) { 173 a += d[i]; 174 b += a; 175 } 176 return ((uint64_t)b << 32 | a); 177 } 178 179 static bool 180 label_index_is_valid(struct nvdimm_label_index *index, uint32_t max_labels, 181 size_t size, size_t offset) 182 { 183 uint64_t checksum; 184 185 index = (struct nvdimm_label_index *)((uint8_t *)index + offset); 186 if (strcmp(index->signature, NVDIMM_INDEX_BLOCK_SIGNATURE) != 0) 187 return false; 188 checksum = index->checksum; 189 index->checksum = 0; 190 if (checksum != fletcher64(index, size) || 191 index->this_offset != size * offset || index->this_size != size || 192 index->other_offset != size * (offset == 0 ? 1 : 0) || 193 index->seq == 0 || index->seq > 3 || index->slot_cnt > max_labels || 194 index->label_size != 1) 195 return false; 196 return true; 197 } 198 199 static int 200 read_label(struct nvdimm_dev *nv, int num) 201 { 202 struct nvdimm_label_entry *entry, *i, *next; 203 uint64_t checksum; 204 off_t offset; 205 int error; 206 207 offset = nv->label_index->label_offset + 208 num * (128 << nv->label_index->label_size); 209 entry = malloc(sizeof(*entry), M_NVDIMM, M_WAITOK); 210 error = read_label_area(nv, (uint8_t *)&entry->label, offset, 211 sizeof(struct nvdimm_label)); 212 if (error != 0) { 213 free(entry, M_NVDIMM); 214 return (error); 215 } 216 checksum = entry->label.checksum; 217 entry->label.checksum = 0; 218 if (checksum != fletcher64(&entry->label, sizeof(entry->label)) || 219 entry->label.slot != num) { 220 free(entry, M_NVDIMM); 221 return (ENXIO); 222 } 223 224 /* Insertion ordered by dimm_phys_addr */ 225 if (SLIST_EMPTY(&nv->labels) || 226 entry->label.dimm_phys_addr <= 227 SLIST_FIRST(&nv->labels)->label.dimm_phys_addr) { 228 SLIST_INSERT_HEAD(&nv->labels, entry, link); 229 return (0); 230 } 231 SLIST_FOREACH_SAFE(i, &nv->labels, link, next) { 232 if (next == NULL || 233 entry->label.dimm_phys_addr <= next->label.dimm_phys_addr) { 234 SLIST_INSERT_AFTER(i, entry, link); 235 return (0); 236 } 237 } 238 __unreachable(); 239 } 240 241 static int 242 read_labels(struct nvdimm_dev *nv) 243 { 244 struct nvdimm_label_index *indices; 245 size_t bitfield_size, index_size, num_labels; 246 int error, n; 247 bool index_0_valid, index_1_valid; 248 249 for (index_size = 256; ; index_size += 256) { 250 num_labels = 8 * (index_size - 251 sizeof(struct nvdimm_label_index)); 252 if (index_size + num_labels * sizeof(struct nvdimm_label) >= 253 nv->label_area_size) 254 break; 255 } 256 num_labels = (nv->label_area_size - index_size) / 257 sizeof(struct nvdimm_label); 258 bitfield_size = roundup2(num_labels, 8) / 8; 259 indices = malloc(2 * index_size, M_NVDIMM, M_WAITOK); 260 error = read_label_area(nv, (void *)indices, 0, 2 * index_size); 261 if (error != 0) { 262 free(indices, M_NVDIMM); 263 return (error); 264 } 265 index_0_valid = label_index_is_valid(indices, num_labels, index_size, 266 0); 267 index_1_valid = label_index_is_valid(indices, num_labels, index_size, 268 1); 269 if (!index_0_valid && !index_1_valid) { 270 free(indices, M_NVDIMM); 271 return (ENXIO); 272 } 273 if (index_0_valid && index_1_valid && 274 (indices[1].seq > indices[0].seq || 275 (indices[1].seq == 1 && indices[0].seq == 3))) 276 index_0_valid = false; 277 nv->label_index = malloc(index_size, M_NVDIMM, M_WAITOK); 278 bcopy(indices + (index_0_valid ? 0 : 1), nv->label_index, index_size); 279 free(indices, M_NVDIMM); 280 for (bit_ffc_at((bitstr_t *)nv->label_index->free, 0, num_labels, &n); 281 n >= 0; 282 bit_ffc_at((bitstr_t *)nv->label_index->free, n + 1, num_labels, 283 &n)) { 284 read_label(nv, n); 285 } 286 return (0); 287 } 288 289 struct nvdimm_dev * 290 nvdimm_find_by_handle(nfit_handle_t nv_handle) 291 { 292 struct nvdimm_dev *res; 293 device_t *dimms; 294 int i, error, num_dimms; 295 296 res = NULL; 297 error = devclass_get_devices(nvdimm_devclass, &dimms, &num_dimms); 298 if (error != 0) 299 return (NULL); 300 for (i = 0; i < num_dimms; i++) { 301 if (nvdimm_root_get_device_handle(dimms[i]) == nv_handle) { 302 res = device_get_softc(dimms[i]); 303 break; 304 } 305 } 306 free(dimms, M_TEMP); 307 return (res); 308 } 309 310 static int 311 nvdimm_probe(device_t dev) 312 { 313 314 return (BUS_PROBE_NOWILDCARD); 315 } 316 317 static int 318 nvdimm_attach(device_t dev) 319 { 320 struct nvdimm_dev *nv; 321 ACPI_TABLE_NFIT *nfitbl; 322 ACPI_HANDLE handle; 323 ACPI_STATUS status; 324 int error; 325 326 nv = device_get_softc(dev); 327 handle = nvdimm_root_get_acpi_handle(dev); 328 if (handle == NULL) 329 return (EINVAL); 330 nv->nv_dev = dev; 331 nv->nv_handle = nvdimm_root_get_device_handle(dev); 332 333 status = AcpiGetTable(ACPI_SIG_NFIT, 1, (ACPI_TABLE_HEADER **)&nfitbl); 334 if (ACPI_FAILURE(status)) { 335 if (bootverbose) 336 device_printf(dev, "cannot get NFIT\n"); 337 return (ENXIO); 338 } 339 acpi_nfit_get_flush_addrs(nfitbl, nv->nv_handle, &nv->nv_flush_addr, 340 &nv->nv_flush_addr_cnt); 341 AcpiPutTable(&nfitbl->Header); 342 error = read_label_area_size(nv); 343 if (error == 0) { 344 /* 345 * Ignoring errors reading labels. Not all NVDIMMs 346 * support labels and namespaces. 347 */ 348 read_labels(nv); 349 } 350 return (0); 351 } 352 353 static int 354 nvdimm_detach(device_t dev) 355 { 356 struct nvdimm_dev *nv; 357 struct nvdimm_label_entry *label, *next; 358 359 nv = device_get_softc(dev); 360 free(nv->nv_flush_addr, M_NVDIMM); 361 free(nv->label_index, M_NVDIMM); 362 SLIST_FOREACH_SAFE(label, &nv->labels, link, next) { 363 SLIST_REMOVE_HEAD(&nv->labels, link); 364 free(label, M_NVDIMM); 365 } 366 return (0); 367 } 368 369 static int 370 nvdimm_suspend(device_t dev) 371 { 372 373 return (0); 374 } 375 376 static int 377 nvdimm_resume(device_t dev) 378 { 379 380 return (0); 381 } 382 383 static ACPI_STATUS 384 find_dimm(ACPI_HANDLE handle, UINT32 nesting_level, void *context, 385 void **return_value) 386 { 387 ACPI_DEVICE_INFO *device_info; 388 ACPI_STATUS status; 389 390 status = AcpiGetObjectInfo(handle, &device_info); 391 if (ACPI_FAILURE(status)) 392 return_ACPI_STATUS(AE_ERROR); 393 if (device_info->Address == (uintptr_t)context) { 394 *(ACPI_HANDLE *)return_value = handle; 395 return_ACPI_STATUS(AE_CTRL_TERMINATE); 396 } 397 return_ACPI_STATUS(AE_OK); 398 } 399 400 static ACPI_HANDLE 401 get_dimm_acpi_handle(ACPI_HANDLE root_handle, nfit_handle_t adr) 402 { 403 ACPI_HANDLE res; 404 ACPI_STATUS status; 405 406 res = NULL; 407 status = AcpiWalkNamespace(ACPI_TYPE_DEVICE, root_handle, 1, find_dimm, 408 NULL, (void *)(uintptr_t)adr, &res); 409 if (ACPI_FAILURE(status)) 410 res = NULL; 411 return (res); 412 } 413 414 static int 415 nvdimm_root_create_devs(device_t dev, ACPI_TABLE_NFIT *nfitbl) 416 { 417 ACPI_HANDLE root_handle, dimm_handle; 418 device_t child; 419 nfit_handle_t *dimm_ids, *dimm; 420 uintptr_t *ivars; 421 int num_dimm_ids; 422 423 root_handle = acpi_get_handle(dev); 424 acpi_nfit_get_dimm_ids(nfitbl, &dimm_ids, &num_dimm_ids); 425 for (dimm = dimm_ids; dimm < dimm_ids + num_dimm_ids; dimm++) { 426 dimm_handle = get_dimm_acpi_handle(root_handle, *dimm); 427 child = BUS_ADD_CHILD(dev, 100, "nvdimm", -1); 428 if (child == NULL) { 429 device_printf(dev, "failed to create nvdimm\n"); 430 return (ENXIO); 431 } 432 ivars = mallocarray(NVDIMM_ROOT_IVAR_MAX, sizeof(uintptr_t), 433 M_NVDIMM, M_ZERO | M_WAITOK); 434 device_set_ivars(child, ivars); 435 nvdimm_root_set_acpi_handle(child, dimm_handle); 436 nvdimm_root_set_device_handle(child, *dimm); 437 } 438 free(dimm_ids, M_NVDIMM); 439 return (0); 440 } 441 442 static int 443 nvdimm_root_create_spas(struct nvdimm_root_dev *dev, ACPI_TABLE_NFIT *nfitbl) 444 { 445 ACPI_NFIT_SYSTEM_ADDRESS **spas, **spa; 446 struct SPA_mapping *spa_mapping; 447 enum SPA_mapping_type spa_type; 448 int error, num_spas; 449 450 error = 0; 451 acpi_nfit_get_spa_ranges(nfitbl, &spas, &num_spas); 452 for (spa = spas; spa < spas + num_spas; spa++) { 453 spa_type = nvdimm_spa_type_from_uuid( 454 (struct uuid *)(*spa)->RangeGuid); 455 if (spa_type == SPA_TYPE_UNKNOWN) 456 continue; 457 spa_mapping = malloc(sizeof(struct SPA_mapping), M_NVDIMM, 458 M_WAITOK | M_ZERO); 459 error = nvdimm_spa_init(spa_mapping, *spa, spa_type); 460 if (error != 0) { 461 nvdimm_spa_fini(spa_mapping); 462 free(spa, M_NVDIMM); 463 break; 464 } 465 nvdimm_create_namespaces(spa_mapping, nfitbl); 466 SLIST_INSERT_HEAD(&dev->spas, spa_mapping, link); 467 } 468 free(spas, M_NVDIMM); 469 return (error); 470 } 471 472 static char *nvdimm_root_id[] = {"ACPI0012", NULL}; 473 474 static int 475 nvdimm_root_probe(device_t dev) 476 { 477 int rv; 478 479 if (acpi_disabled("nvdimm")) 480 return (ENXIO); 481 rv = ACPI_ID_PROBE(device_get_parent(dev), dev, nvdimm_root_id, NULL); 482 if (rv <= 0) 483 device_set_desc(dev, "ACPI NVDIMM root device"); 484 485 return (rv); 486 } 487 488 static int 489 nvdimm_root_attach(device_t dev) 490 { 491 struct nvdimm_root_dev *root; 492 ACPI_TABLE_NFIT *nfitbl; 493 ACPI_STATUS status; 494 int error; 495 496 status = AcpiGetTable(ACPI_SIG_NFIT, 1, (ACPI_TABLE_HEADER **)&nfitbl); 497 if (ACPI_FAILURE(status)) { 498 device_printf(dev, "cannot get NFIT\n"); 499 return (ENXIO); 500 } 501 error = nvdimm_root_create_devs(dev, nfitbl); 502 if (error != 0) 503 return (error); 504 error = bus_generic_attach(dev); 505 if (error != 0) 506 return (error); 507 root = device_get_softc(dev); 508 error = nvdimm_root_create_spas(root, nfitbl); 509 AcpiPutTable(&nfitbl->Header); 510 return (error); 511 } 512 513 static int 514 nvdimm_root_detach(device_t dev) 515 { 516 struct nvdimm_root_dev *root; 517 struct SPA_mapping *spa, *next; 518 device_t *children; 519 int i, error, num_children; 520 521 root = device_get_softc(dev); 522 SLIST_FOREACH_SAFE(spa, &root->spas, link, next) { 523 nvdimm_destroy_namespaces(spa); 524 nvdimm_spa_fini(spa); 525 SLIST_REMOVE_HEAD(&root->spas, link); 526 free(spa, M_NVDIMM); 527 } 528 error = bus_generic_detach(dev); 529 if (error != 0) 530 return (error); 531 error = device_get_children(dev, &children, &num_children); 532 if (error != 0) 533 return (error); 534 for (i = 0; i < num_children; i++) 535 free(device_get_ivars(children[i]), M_NVDIMM); 536 free(children, M_TEMP); 537 error = device_delete_children(dev); 538 return (error); 539 } 540 541 static int 542 nvdimm_root_read_ivar(device_t dev, device_t child, int index, 543 uintptr_t *result) 544 { 545 546 if (index < 0 || index >= NVDIMM_ROOT_IVAR_MAX) 547 return (ENOENT); 548 *result = ((uintptr_t *)device_get_ivars(child))[index]; 549 return (0); 550 } 551 552 static int 553 nvdimm_root_write_ivar(device_t dev, device_t child, int index, 554 uintptr_t value) 555 { 556 557 if (index < 0 || index >= NVDIMM_ROOT_IVAR_MAX) 558 return (ENOENT); 559 ((uintptr_t *)device_get_ivars(child))[index] = value; 560 return (0); 561 } 562 563 static device_method_t nvdimm_methods[] = { 564 DEVMETHOD(device_probe, nvdimm_probe), 565 DEVMETHOD(device_attach, nvdimm_attach), 566 DEVMETHOD(device_detach, nvdimm_detach), 567 DEVMETHOD(device_suspend, nvdimm_suspend), 568 DEVMETHOD(device_resume, nvdimm_resume), 569 DEVMETHOD_END 570 }; 571 572 static driver_t nvdimm_driver = { 573 "nvdimm", 574 nvdimm_methods, 575 sizeof(struct nvdimm_dev), 576 }; 577 578 static device_method_t nvdimm_root_methods[] = { 579 DEVMETHOD(device_probe, nvdimm_root_probe), 580 DEVMETHOD(device_attach, nvdimm_root_attach), 581 DEVMETHOD(device_detach, nvdimm_root_detach), 582 DEVMETHOD(bus_add_child, bus_generic_add_child), 583 DEVMETHOD(bus_read_ivar, nvdimm_root_read_ivar), 584 DEVMETHOD(bus_write_ivar, nvdimm_root_write_ivar), 585 DEVMETHOD_END 586 }; 587 588 static driver_t nvdimm_root_driver = { 589 "nvdimm_root", 590 nvdimm_root_methods, 591 sizeof(struct nvdimm_root_dev), 592 }; 593 594 DRIVER_MODULE(nvdimm_root, acpi, nvdimm_root_driver, nvdimm_root_devclass, NULL, 595 NULL); 596 DRIVER_MODULE(nvdimm, nvdimm_root, nvdimm_driver, nvdimm_devclass, NULL, NULL); 597 MODULE_DEPEND(nvdimm, acpi, 1, 1, 1); 598