1 /** @file 2 3 Utility functions used by virtio device drivers. 4 5 Copyright (C) 2012-2016, Red Hat, Inc. 6 Portion of Copyright (C) 2013, ARM Ltd. 7 Copyright (C) 2017, AMD Inc, All rights reserved.<BR> 8 9 SPDX-License-Identifier: BSD-2-Clause-Patent 10 11 **/ 12 13 #include <Library/BaseLib.h> 14 #include <Library/BaseMemoryLib.h> 15 #include <Library/DebugLib.h> 16 #include <Library/UefiBootServicesTableLib.h> 17 18 #include <Library/VirtioLib.h> 19 20 21 /** 22 23 Configure a virtio ring. 24 25 This function sets up internal storage (the guest-host communication area) 26 and lays out several "navigation" (ie. no-ownership) pointers to parts of 27 that storage. 28 29 Relevant sections from the virtio-0.9.5 spec: 30 - 1.1 Virtqueues, 31 - 2.3 Virtqueue Configuration. 32 33 @param[in] VirtIo The virtio device which will use the ring. 34 35 @param[in] The number of descriptors to allocate for the 36 virtio ring, as requested by the host. 37 38 @param[out] Ring The virtio ring to set up. 39 40 @return Status codes propagated from 41 VirtIo->AllocateSharedPages(). 42 43 @retval EFI_SUCCESS Allocation and setup successful. Ring->Base 44 (and nothing else) is responsible for 45 deallocation. 46 47 **/ 48 EFI_STATUS 49 EFIAPI 50 VirtioRingInit ( 51 IN VIRTIO_DEVICE_PROTOCOL *VirtIo, 52 IN UINT16 QueueSize, 53 OUT VRING *Ring 54 ) 55 { 56 EFI_STATUS Status; 57 UINTN RingSize; 58 volatile UINT8 *RingPagesPtr; 59 60 RingSize = ALIGN_VALUE ( 61 sizeof *Ring->Desc * QueueSize + 62 sizeof *Ring->Avail.Flags + 63 sizeof *Ring->Avail.Idx + 64 sizeof *Ring->Avail.Ring * QueueSize + 65 sizeof *Ring->Avail.UsedEvent, 66 EFI_PAGE_SIZE); 67 68 RingSize += ALIGN_VALUE ( 69 sizeof *Ring->Used.Flags + 70 sizeof *Ring->Used.Idx + 71 sizeof *Ring->Used.UsedElem * QueueSize + 72 sizeof *Ring->Used.AvailEvent, 73 EFI_PAGE_SIZE); 74 75 // 76 // Allocate a shared ring buffer 77 // 78 Ring->NumPages = EFI_SIZE_TO_PAGES (RingSize); 79 Status = VirtIo->AllocateSharedPages ( 80 VirtIo, 81 Ring->NumPages, 82 &Ring->Base 83 ); 84 if (EFI_ERROR (Status)) { 85 return Status; 86 } 87 SetMem (Ring->Base, RingSize, 0x00); 88 RingPagesPtr = Ring->Base; 89 90 Ring->Desc = (volatile VOID *) RingPagesPtr; 91 RingPagesPtr += sizeof *Ring->Desc * QueueSize; 92 93 Ring->Avail.Flags = (volatile VOID *) RingPagesPtr; 94 RingPagesPtr += sizeof *Ring->Avail.Flags; 95 96 Ring->Avail.Idx = (volatile VOID *) RingPagesPtr; 97 RingPagesPtr += sizeof *Ring->Avail.Idx; 98 99 Ring->Avail.Ring = (volatile VOID *) RingPagesPtr; 100 RingPagesPtr += sizeof *Ring->Avail.Ring * QueueSize; 101 102 Ring->Avail.UsedEvent = (volatile VOID *) RingPagesPtr; 103 RingPagesPtr += sizeof *Ring->Avail.UsedEvent; 104 105 RingPagesPtr = (volatile UINT8 *) Ring->Base + 106 ALIGN_VALUE (RingPagesPtr - (volatile UINT8 *) Ring->Base, 107 EFI_PAGE_SIZE); 108 109 Ring->Used.Flags = (volatile VOID *) RingPagesPtr; 110 RingPagesPtr += sizeof *Ring->Used.Flags; 111 112 Ring->Used.Idx = (volatile VOID *) RingPagesPtr; 113 RingPagesPtr += sizeof *Ring->Used.Idx; 114 115 Ring->Used.UsedElem = (volatile VOID *) RingPagesPtr; 116 RingPagesPtr += sizeof *Ring->Used.UsedElem * QueueSize; 117 118 Ring->Used.AvailEvent = (volatile VOID *) RingPagesPtr; 119 RingPagesPtr += sizeof *Ring->Used.AvailEvent; 120 121 Ring->QueueSize = QueueSize; 122 return EFI_SUCCESS; 123 } 124 125 126 /** 127 128 Tear down the internal resources of a configured virtio ring. 129 130 The caller is responsible to stop the host from using this ring before 131 invoking this function: the VSTAT_DRIVER_OK bit must be clear in 132 VhdrDeviceStatus. 133 134 @param[in] VirtIo The virtio device which was using the ring. 135 136 @param[out] Ring The virtio ring to clean up. 137 138 **/ 139 VOID 140 EFIAPI 141 VirtioRingUninit ( 142 IN VIRTIO_DEVICE_PROTOCOL *VirtIo, 143 IN OUT VRING *Ring 144 ) 145 { 146 VirtIo->FreeSharedPages (VirtIo, Ring->NumPages, Ring->Base); 147 SetMem (Ring, sizeof *Ring, 0x00); 148 } 149 150 151 /** 152 153 Turn off interrupt notifications from the host, and prepare for appending 154 multiple descriptors to the virtio ring. 155 156 The calling driver must be in VSTAT_DRIVER_OK state. 157 158 @param[in,out] Ring The virtio ring we intend to append descriptors to. 159 160 @param[out] Indices The DESC_INDICES structure to initialize. 161 162 **/ 163 VOID 164 EFIAPI 165 VirtioPrepare ( 166 IN OUT VRING *Ring, 167 OUT DESC_INDICES *Indices 168 ) 169 { 170 // 171 // Prepare for virtio-0.9.5, 2.4.2 Receiving Used Buffers From the Device. 172 // We're going to poll the answer, the host should not send an interrupt. 173 // 174 *Ring->Avail.Flags = (UINT16) VRING_AVAIL_F_NO_INTERRUPT; 175 176 // 177 // Prepare for virtio-0.9.5, 2.4.1 Supplying Buffers to the Device. 178 // 179 // Since we support only one in-flight descriptor chain, we can always build 180 // that chain starting at entry #0 of the descriptor table. 181 // 182 Indices->HeadDescIdx = 0; 183 Indices->NextDescIdx = Indices->HeadDescIdx; 184 } 185 186 /** 187 188 Append a contiguous buffer for transmission / reception via the virtio ring. 189 190 This function implements the following section from virtio-0.9.5: 191 - 2.4.1.1 Placing Buffers into the Descriptor Table 192 193 Free space is taken as granted, since the individual drivers support only 194 synchronous requests and host side status is processed in lock-step with 195 request submission. It is the calling driver's responsibility to verify the 196 ring size in advance. 197 198 The caller is responsible for initializing *Indices with VirtioPrepare() 199 first. 200 201 @param[in,out] Ring The virtio ring to append the buffer to, 202 as a descriptor. 203 204 @param[in] BufferDeviceAddress (Bus master device) start address of the 205 transmit / receive buffer. 206 207 @param[in] BufferSize Number of bytes to transmit or receive. 208 209 @param[in] Flags A bitmask of VRING_DESC_F_* flags. The 210 caller computes this mask dependent on 211 further buffers to append and transfer 212 direction. VRING_DESC_F_INDIRECT is 213 unsupported. The VRING_DESC.Next field is 214 always set, but the host only interprets 215 it dependent on VRING_DESC_F_NEXT. 216 217 @param[in,out] Indices Indices->HeadDescIdx is not accessed. 218 On input, Indices->NextDescIdx identifies 219 the next descriptor to carry the buffer. 220 On output, Indices->NextDescIdx is 221 incremented by one, modulo 2^16. 222 223 **/ 224 VOID 225 EFIAPI 226 VirtioAppendDesc ( 227 IN OUT VRING *Ring, 228 IN UINT64 BufferDeviceAddress, 229 IN UINT32 BufferSize, 230 IN UINT16 Flags, 231 IN OUT DESC_INDICES *Indices 232 ) 233 { 234 volatile VRING_DESC *Desc; 235 236 Desc = &Ring->Desc[Indices->NextDescIdx++ % Ring->QueueSize]; 237 Desc->Addr = BufferDeviceAddress; 238 Desc->Len = BufferSize; 239 Desc->Flags = Flags; 240 Desc->Next = Indices->NextDescIdx % Ring->QueueSize; 241 } 242 243 244 /** 245 246 Notify the host about the descriptor chain just built, and wait until the 247 host processes it. 248 249 @param[in] VirtIo The target virtio device to notify. 250 251 @param[in] VirtQueueId Identifies the queue for the target device. 252 253 @param[in,out] Ring The virtio ring with descriptors to submit. 254 255 @param[in] Indices Indices->NextDescIdx is not accessed. 256 Indices->HeadDescIdx identifies the head descriptor 257 of the descriptor chain. 258 259 @param[out] UsedLen On success, the total number of bytes, consecutively 260 across the buffers linked by the descriptor chain, 261 that the host wrote. May be NULL if the caller 262 doesn't care, or can compute the same information 263 from device-specific request structures linked by the 264 descriptor chain. 265 266 @return Error code from VirtIo->SetQueueNotify() if it fails. 267 268 @retval EFI_SUCCESS Otherwise, the host processed all descriptors. 269 270 **/ 271 EFI_STATUS 272 EFIAPI 273 VirtioFlush ( 274 IN VIRTIO_DEVICE_PROTOCOL *VirtIo, 275 IN UINT16 VirtQueueId, 276 IN OUT VRING *Ring, 277 IN DESC_INDICES *Indices, 278 OUT UINT32 *UsedLen OPTIONAL 279 ) 280 { 281 UINT16 NextAvailIdx; 282 UINT16 LastUsedIdx; 283 EFI_STATUS Status; 284 UINTN PollPeriodUsecs; 285 286 // 287 // virtio-0.9.5, 2.4.1.2 Updating the Available Ring 288 // 289 // It is not exactly clear from the wording of the virtio-0.9.5 290 // specification, but each entry in the Available Ring references only the 291 // head descriptor of any given descriptor chain. 292 // 293 NextAvailIdx = *Ring->Avail.Idx; 294 // 295 // (Due to our lock-step progress, this is where the host will produce the 296 // used element with the head descriptor's index in it.) 297 // 298 LastUsedIdx = NextAvailIdx; 299 Ring->Avail.Ring[NextAvailIdx++ % Ring->QueueSize] = 300 Indices->HeadDescIdx % Ring->QueueSize; 301 302 // 303 // virtio-0.9.5, 2.4.1.3 Updating the Index Field 304 // 305 MemoryFence(); 306 *Ring->Avail.Idx = NextAvailIdx; 307 308 // 309 // virtio-0.9.5, 2.4.1.4 Notifying the Device -- gratuitous notifications are 310 // OK. 311 // 312 MemoryFence(); 313 Status = VirtIo->SetQueueNotify (VirtIo, VirtQueueId); 314 if (EFI_ERROR (Status)) { 315 return Status; 316 } 317 318 // 319 // virtio-0.9.5, 2.4.2 Receiving Used Buffers From the Device 320 // Wait until the host processes and acknowledges our descriptor chain. The 321 // condition we use for polling is greatly simplified and relies on the 322 // synchronous, lock-step progress. 323 // 324 // Keep slowing down until we reach a poll period of slightly above 1 ms. 325 // 326 PollPeriodUsecs = 1; 327 MemoryFence(); 328 while (*Ring->Used.Idx != NextAvailIdx) { 329 gBS->Stall (PollPeriodUsecs); // calls AcpiTimerLib::MicroSecondDelay 330 331 if (PollPeriodUsecs < 1024) { 332 PollPeriodUsecs *= 2; 333 } 334 MemoryFence(); 335 } 336 337 MemoryFence(); 338 339 if (UsedLen != NULL) { 340 volatile CONST VRING_USED_ELEM *UsedElem; 341 342 UsedElem = &Ring->Used.UsedElem[LastUsedIdx % Ring->QueueSize]; 343 ASSERT (UsedElem->Id == Indices->HeadDescIdx); 344 *UsedLen = UsedElem->Len; 345 } 346 347 return EFI_SUCCESS; 348 } 349 350 351 /** 352 353 Report the feature bits to the VirtIo 1.0 device that the VirtIo 1.0 driver 354 understands. 355 356 In VirtIo 1.0, a device can reject a self-inconsistent feature bitmap through 357 the new VSTAT_FEATURES_OK status bit. (For example if the driver requests a 358 higher level feature but clears a prerequisite feature.) This function is a 359 small wrapper around VIRTIO_DEVICE_PROTOCOL.SetGuestFeatures() that also 360 verifies if the VirtIo 1.0 device accepts the feature bitmap. 361 362 @param[in] VirtIo Report feature bits to this device. 363 364 @param[in] Features The set of feature bits that the driver wishes 365 to report. The caller is responsible to perform 366 any masking before calling this function; the 367 value is directly written with 368 VIRTIO_DEVICE_PROTOCOL.SetGuestFeatures(). 369 370 @param[in,out] DeviceStatus On input, the status byte most recently written 371 to the device's status register. On output (even 372 on error), DeviceStatus will be updated so that 373 it is suitable for further status bit 374 manipulation and writing to the device's status 375 register. 376 377 @retval EFI_SUCCESS The device accepted the configuration in Features. 378 379 @return EFI_UNSUPPORTED The device rejected the configuration in Features. 380 381 @retval EFI_UNSUPPORTED VirtIo->Revision is smaller than 1.0.0. 382 383 @return Error codes from the SetGuestFeatures(), 384 SetDeviceStatus(), GetDeviceStatus() member 385 functions. 386 387 **/ 388 EFI_STATUS 389 EFIAPI 390 Virtio10WriteFeatures ( 391 IN VIRTIO_DEVICE_PROTOCOL *VirtIo, 392 IN UINT64 Features, 393 IN OUT UINT8 *DeviceStatus 394 ) 395 { 396 EFI_STATUS Status; 397 398 if (VirtIo->Revision < VIRTIO_SPEC_REVISION (1, 0, 0)) { 399 return EFI_UNSUPPORTED; 400 } 401 402 Status = VirtIo->SetGuestFeatures (VirtIo, Features); 403 if (EFI_ERROR (Status)) { 404 return Status; 405 } 406 407 *DeviceStatus |= VSTAT_FEATURES_OK; 408 Status = VirtIo->SetDeviceStatus (VirtIo, *DeviceStatus); 409 if (EFI_ERROR (Status)) { 410 return Status; 411 } 412 413 Status = VirtIo->GetDeviceStatus (VirtIo, DeviceStatus); 414 if (EFI_ERROR (Status)) { 415 return Status; 416 } 417 418 if ((*DeviceStatus & VSTAT_FEATURES_OK) == 0) { 419 Status = EFI_UNSUPPORTED; 420 } 421 422 return Status; 423 } 424 425 /** 426 Provides the virtio device address required to access system memory from a 427 DMA bus master. 428 429 The interface follows the same usage pattern as defined in UEFI spec 2.6 430 (Section 13.2 PCI Root Bridge I/O Protocol) 431 432 The VirtioMapAllBytesInSharedBuffer() is similar to VIRTIO_MAP_SHARED 433 with exception that NumberOfBytes is IN-only parameter. The function 434 maps all the bytes specified in NumberOfBytes param in one consecutive 435 range. 436 437 @param[in] VirtIo The virtio device for which the mapping is 438 requested. 439 440 @param[in] Operation Indicates if the bus master is going to 441 read or write to system memory. 442 443 @param[in] HostAddress The system memory address to map to shared 444 buffer address. 445 446 @param[in] NumberOfBytes Number of bytes to map. 447 448 @param[out] DeviceAddress The resulting shared map address for the 449 bus master to access the hosts HostAddress. 450 451 @param[out] Mapping A resulting token to pass to 452 VIRTIO_UNMAP_SHARED. 453 454 455 @retval EFI_SUCCESS The NumberOfBytes is successfully mapped. 456 @retval EFI_UNSUPPORTED The HostAddress cannot be mapped as a 457 common buffer. 458 @retval EFI_INVALID_PARAMETER One or more parameters are invalid. 459 @retval EFI_OUT_OF_RESOURCES The request could not be completed due to 460 a lack of resources. This includes the case 461 when NumberOfBytes bytes cannot be mapped 462 in one consecutive range. 463 @retval EFI_DEVICE_ERROR The system hardware could not map the 464 requested address. 465 **/ 466 EFI_STATUS 467 EFIAPI 468 VirtioMapAllBytesInSharedBuffer ( 469 IN VIRTIO_DEVICE_PROTOCOL *VirtIo, 470 IN VIRTIO_MAP_OPERATION Operation, 471 IN VOID *HostAddress, 472 IN UINTN NumberOfBytes, 473 OUT EFI_PHYSICAL_ADDRESS *DeviceAddress, 474 OUT VOID **Mapping 475 ) 476 { 477 EFI_STATUS Status; 478 VOID *MapInfo; 479 UINTN Size; 480 EFI_PHYSICAL_ADDRESS PhysicalAddress; 481 482 Size = NumberOfBytes; 483 Status = VirtIo->MapSharedBuffer ( 484 VirtIo, 485 Operation, 486 HostAddress, 487 &Size, 488 &PhysicalAddress, 489 &MapInfo 490 ); 491 if (EFI_ERROR (Status)) { 492 return Status; 493 } 494 495 if (Size < NumberOfBytes) { 496 goto Failed; 497 } 498 499 *Mapping = MapInfo; 500 *DeviceAddress = PhysicalAddress; 501 502 return EFI_SUCCESS; 503 504 Failed: 505 VirtIo->UnmapSharedBuffer (VirtIo, MapInfo); 506 return EFI_OUT_OF_RESOURCES; 507 } 508 509 /** 510 511 Map the ring buffer so that it can be accessed equally by both guest 512 and hypervisor. 513 514 @param[in] VirtIo The virtio device instance. 515 516 @param[in] Ring The virtio ring to map. 517 518 @param[out] RingBaseShift A resulting translation offset, to be 519 passed to VirtIo->SetQueueAddress(). 520 521 @param[out] Mapping A resulting token to pass to 522 VirtIo->UnmapSharedBuffer(). 523 524 @return Status code from VirtIo->MapSharedBuffer() 525 **/ 526 EFI_STATUS 527 EFIAPI 528 VirtioRingMap ( 529 IN VIRTIO_DEVICE_PROTOCOL *VirtIo, 530 IN VRING *Ring, 531 OUT UINT64 *RingBaseShift, 532 OUT VOID **Mapping 533 ) 534 { 535 EFI_STATUS Status; 536 EFI_PHYSICAL_ADDRESS DeviceAddress; 537 538 Status = VirtioMapAllBytesInSharedBuffer ( 539 VirtIo, 540 VirtioOperationBusMasterCommonBuffer, 541 Ring->Base, 542 EFI_PAGES_TO_SIZE (Ring->NumPages), 543 &DeviceAddress, 544 Mapping 545 ); 546 if (EFI_ERROR (Status)) { 547 return Status; 548 } 549 550 *RingBaseShift = DeviceAddress - (UINT64)(UINTN)Ring->Base; 551 return EFI_SUCCESS; 552 } 553