1 /*++ 2 3 Copyright (c) Microsoft Corporation 4 5 Module Name: 6 7 wdfpool.c 8 9 Abstract: 10 11 This module implements the driver frameworks pool routines. 12 13 Author: 14 15 16 17 18 Environment: 19 20 Both kernel and user mode 21 22 Revision History: 23 24 25 26 27 28 29 --*/ 30 31 #include "fxobjectpch.hpp" 32 33 // We use DoTraceMessage 34 extern "C" { 35 36 #if defined(EVENT_TRACING) 37 #include "wdfpool.tmh" 38 #endif 39 40 } 41 42 BOOLEAN 43 FxIsPagedPoolType( 44 __in POOL_TYPE Type 45 ) 46 /*++ 47 48 Routine Description: 49 50 Return whether paged pool is specified by POOL_TYPE 51 52 Arguments: 53 54 Type - POOL_TYPE 55 56 Returns: 57 TRUE - Paged Pool,FALSE - Non-Paged Pool 58 59 --*/ 60 { 61 // 62 // Cleaner than doing (Type & 0x01) 63 // 64 switch( Type & (~POOL_COLD_ALLOCATION) ) { 65 case PagedPool: 66 case PagedPoolCacheAligned: 67 return TRUE; 68 69 default: 70 return FALSE; 71 } 72 } 73 74 75 PVOID 76 FxPoolAllocator( 77 __in PFX_DRIVER_GLOBALS FxDriverGlobals, 78 __in PFX_POOL Pool, 79 __in POOL_TYPE Type, 80 __in SIZE_T Size, 81 __in ULONG Tag, 82 __in PVOID Caller 83 ) 84 /*++ 85 86 Routine Description: 87 88 Allocates system pool tracked in a FX_POOL tracking object. 89 90 Arguments: 91 92 Pool - FX_POOL object for tracking allocations 93 94 Type - POOL_TYPE from ntddk.h 95 96 Size - Size in bytes of the allocation 97 98 Tag - Caller specified additional tag value for debugging/tracing 99 100 Caller - Caller's address 101 102 Returns: 103 104 NULL - Could not allocate pool 105 !NULL - Pointer to pool of minimum Size bytes 106 107 Remarks: 108 109 In kernel mode this routine conditionally adds header on top iff the 110 allocation size is < PAGE_SIZE. If the allocation size is >= PAGE_SIZE 111 the caller would expect a page aligned pointer, hence no header is added. 112 In addition, ExAllocatePool* functions guarantee that a buffer < PAGE_SIZE 113 doesn't straddle page boundary. This allows FxPoolFree to determine whether 114 a header is added to buffer or not based on whether the pointer passed in 115 is page aligned or not. (In addition, when pool tracking is ON, this 116 routine adds pool tracking header based on whether additional space for this 117 header will push the buffer size beyond PAGE_SIZE, which is an optimization.) 118 119 Such guarantees are not available with user mode allocator, hence in case 120 of user mode we always add the header. (In user mode a buffer < PAGE_SIZE 121 can straddle page boundary and the pointer returned may happen to be page 122 aligned, causing FxPoolFree to free the wrong pointer.) 123 124 --*/ 125 { 126 PVOID ptr; 127 PCHAR pTrueBase; 128 PFX_POOL_TRACKER pTracker; 129 PFX_POOL_HEADER pHeader; 130 NTSTATUS status; 131 SIZE_T allocationSize; 132 133 134 ptr = NULL; 135 136 // 137 // Allocations of a zero request size are invalid. 138 // 139 // Besides, with a zero request size, special pool could place us 140 // at the end of a page, and adding our header would give us a page 141 // aligned address, which is ambiguous with large allocations. 142 // 143 if (Size == 0) { 144 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGPOOL, 145 "Invalid Allocation Size of 0 requested"); 146 FxVerifierDbgBreakPoint(FxDriverGlobals); 147 return NULL; 148 } 149 150 if (FxDriverGlobals->IsPoolTrackingOn()) { 151 152 if (FxDriverGlobals->FxVerifierOn && 153 (FxDriverGlobals->WdfVerifierAllocateFailCount != -1L)) { 154 155 // 156 // If the registry key VerifierAllocateFailCount is set, all allocations 157 // after the specified count are failed. 158 // 159 // This is a brutal test, but also ensures the framework can cleanup 160 // under low memory conditions as well. 161 // 162 if (FxDriverGlobals->WdfVerifierAllocateFailCount == 0) { 163 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGPOOL, 164 "Allocation Fail Count exceeded"); 165 return NULL; 166 } 167 168 // Decrement the count 169 InterlockedDecrement(&FxDriverGlobals->WdfVerifierAllocateFailCount); 170 } 171 172 // 173 // (Kernel mode only) PAGE_SIZE or greater allocations can not have our 174 // header since this would break the system allocators contract 175 // that PAGE_SIZE or greater allocations start on a whole page boundary 176 // 177 178 // 179 // For allocations less than a page size that will not fit with our 180 // header, we round up to a non-tracked whole page allocation so 181 // we don't burn two pages for this boundary condition. 182 // 183 184 // This if is the same as 185 // Size + sizeof(FX_POOL_TRACKER) + FX_POOL_HEADER_SIZE >= PAGE_SIZE 186 // BUT with no integer overflow 187 if (Mx::IsKM() && 188 (Size >= PAGE_SIZE - sizeof(FX_POOL_TRACKER) - FX_POOL_HEADER_SIZE) 189 ) { 190 191 // 192 // Ensure that we ask for at least a page to ensure the 193 // allocation starts on a whole page. 194 // 195 if (Size < PAGE_SIZE) { 196 Size = PAGE_SIZE; 197 } 198 199 ptr = MxMemory::MxAllocatePoolWithTag(Type, Size, Tag); 200 201 // 202 // The current system allocator returns paged aligned memory 203 // in this case, which we rely on to detect whether our header 204 // is present or not in FxPoolFree 205 // 206 ASSERT(((ULONG_PTR)ptr & (PAGE_SIZE-1)) == 0); 207 } 208 else { 209 210 status = RtlSIZETAdd(Size, 211 sizeof(FX_POOL_TRACKER) + FX_POOL_HEADER_SIZE, 212 &allocationSize); 213 214 if (!NT_SUCCESS(status)) { 215 DoTraceLevelMessage( 216 FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGPOOL, 217 "overflow: allocation tracker (%d) + header (%d) + pool " 218 "request (%I64d)", sizeof(FX_POOL_TRACKER), 219 FX_POOL_HEADER_SIZE, Size); 220 221 return NULL; 222 } 223 224 pTrueBase = (PCHAR) MxMemory::MxAllocatePoolWithTag( 225 Type, 226 allocationSize, 227 Tag 228 ); 229 230 if (pTrueBase == NULL) { 231 return NULL; 232 } 233 234 pTracker = (PFX_POOL_TRACKER) pTrueBase; 235 pHeader = WDF_PTR_ADD_OFFSET_TYPE(pTrueBase, 236 sizeof(FX_POOL_TRACKER), 237 PFX_POOL_HEADER); 238 pHeader->Base = pTrueBase; 239 pHeader->FxDriverGlobals = FxDriverGlobals; 240 241 // 242 // Adjust the pointer to what we return to the driver 243 // 244 ptr = &pHeader->AllocationStart[0]; 245 246 // 247 // Ensure the pointer we are returning is aligned on the proper 248 // boundary. 249 // 250 ASSERT( ((ULONG_PTR) ptr & (MEMORY_ALLOCATION_ALIGNMENT-1)) == 0); 251 252 // 253 // Ensure the pointer is still not page aligned after 254 // our adjustment. Otherwise the pool free code will 255 // get confused and call ExFreePool on the wrong ptr. 256 // 257 if (Mx::IsKM()) { 258 ASSERT(((ULONG_PTR)ptr & (PAGE_SIZE-1)) != 0 ); 259 } 260 261 // 262 // We must separate paged and non-paged pool since 263 // the lock held differs as to whether we can accept 264 // page faults and block in the allocator. 265 // 266 if (FxIsPagedPoolType(Type)) { 267 // 268 // Format and insert the Tracker in the PagedHeader list. 269 // 270 FxPoolInsertPagedAllocateTracker(Pool, 271 pTracker, 272 Size, 273 Tag, 274 Caller); 275 } 276 else { 277 // 278 // Format and insert the Tracker in the NonPagedHeader list. 279 // 280 FxPoolInsertNonPagedAllocateTracker(Pool, 281 pTracker, 282 Size, 283 Tag, 284 Caller); 285 } 286 } 287 } 288 else { 289 // 290 // No pool tracking... 291 // 292 293 if ((Size < PAGE_SIZE) || Mx::IsUM()) 294 { 295 // 296 // (Kernel mode only) See if adding our header promotes us past a 297 // page boundary 298 // 299 status = RtlSIZETAdd(Size, 300 FX_POOL_HEADER_SIZE, 301 &allocationSize); 302 303 if (!NT_SUCCESS(status)) { 304 DoTraceLevelMessage( 305 FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGPOOL, 306 "overflow: header + pool request (%I64d)", Size); 307 308 return NULL; 309 } 310 311 } 312 else { 313 // 314 // Is the raw request for alloc >= PAGE_SIZE ? Then just use it. 315 // 316 allocationSize = Size; 317 } 318 319 // 320 // Is cooked size for alloc >= PAGE_SIZE ? Then just do it. 321 // 322 if (allocationSize >= PAGE_SIZE && Mx::IsKM()) 323 { 324 // 325 // Important to use allocationSize so that we get a page aligned 326 // allocation so that we know to just free the memory pointer as is 327 // when it is freed. 328 // 329 ptr = MxMemory::MxAllocatePoolWithTag(Type, allocationSize, Tag); 330 ASSERT(((ULONG_PTR)ptr & (PAGE_SIZE-1)) == 0); 331 } 332 else { 333 pTrueBase = (PCHAR) MxMemory::MxAllocatePoolWithTag(Type, 334 allocationSize, 335 Tag); 336 337 if (pTrueBase != NULL) { 338 339 pHeader = (PFX_POOL_HEADER) pTrueBase; 340 pHeader->Base = pTrueBase; 341 pHeader->FxDriverGlobals = FxDriverGlobals; 342 343 ptr = &pHeader->AllocationStart[0]; 344 345 if (Mx::IsKM()) { 346 // 347 // Ensure the pointer is still not page aligned after 348 // our adjustment. Otherwise the pool free code will 349 // get confused and call ExFreePool on the wrong ptr. 350 // 351 ASSERT( ((ULONG_PTR)ptr & (PAGE_SIZE-1)) != 0 ); 352 } 353 } 354 } 355 } 356 357 return ptr; 358 } 359 360 VOID 361 FxPoolFree( 362 __in_xcount(ptr is at an offset from AllocationStart) PVOID ptr 363 ) 364 /*++ 365 366 Routine Description: 367 368 Release tracked pool 369 370 Arguments: 371 372 Pool - FX_POOL object allocation is tracked in 373 374 ptr - Pointer to pool to release 375 376 Returns: 377 378 Remarks: 379 In kernel mode the pointer passed in may or may not have a header before 380 it depending upon whether the pointer is page aligned or not. 381 382 In user mode the pointer passed in always has a header before it. See 383 remarks for FxPoolAllocator. 384 385 --*/ 386 { 387 PFX_POOL_HEADER pHeader; 388 PVOID pTrueBase; 389 PFX_POOL_TRACKER pTracker; 390 391 // 392 // Null pointers are always bad 393 // 394 if( ptr == NULL ) { 395 ASSERTMSG("NULL pointer freed\n", FALSE); 396 Mx::MxBugCheckEx(WDF_VIOLATION, 397 WDF_REQUIRED_PARAMETER_IS_NULL, 398 (ULONG_PTR)NULL, 399 (ULONG_PTR)_ReturnAddress(), 400 (ULONG_PTR)NULL 401 ); 402 } 403 404 // 405 // (Kernel mode only) If ptr is aligned on page boundry (indicates 406 // it was > PAGE_SIZE allocation) 407 // then there will be no common header...just free the memory without 408 // further processing. 409 // 410 if( Mx::IsKM() && ( ((ULONG_PTR)ptr & (PAGE_SIZE-1)) == 0 ) ) { 411 MxMemory::MxFreePool(ptr); 412 return; 413 } 414 415 // 416 // Ensure the pointer we are returning is aligned on the proper 417 // boundary. 418 // 419 ASSERT( ((ULONG_PTR) ptr & (MEMORY_ALLOCATION_ALIGNMENT-1)) == 0); 420 421 // 422 // Dereference the Common header which all <PAGE_SIZE allcations will have. 423 // 424 pHeader = CONTAINING_RECORD(ptr, FX_POOL_HEADER, AllocationStart); 425 pTrueBase = pHeader->Base; 426 427 // 428 // If PoolTracker is on then Base must point to it's header. 429 // This is currently the only option for this area...may change later. 430 // 431 if (pHeader->FxDriverGlobals->IsPoolTrackingOn()) { 432 433 pTracker = (PFX_POOL_TRACKER) pTrueBase; 434 435 if (FxIsPagedPoolType(pTracker->PoolType)) { 436 // 437 // Decommission this Paged Allocation tracker 438 // 439 FxPoolRemovePagedAllocateTracker(pTracker); 440 } 441 else { 442 // 443 // Decommission this NonPaged Allocation tracker 444 // 445 FxPoolRemoveNonPagedAllocateTracker(pTracker); 446 } 447 448 // 449 // Scrub the pool to zeros to catch destructed objects 450 // by NULL'ing the v-table ptr 451 // 452 RtlZeroMemory(pTracker, pTracker->Size + sizeof(FX_POOL_TRACKER)); 453 } 454 455 MxMemory::MxFreePool(pTrueBase); 456 } 457 458 NTSTATUS 459 FxPoolDump( 460 __in PFX_DRIVER_GLOBALS FxDriverGlobals, 461 __in PFX_POOL Pool 462 ) 463 /*++ 464 465 Routine Description: 466 467 Dump the FX_POOL tracking object 468 469 Arguments: 470 471 Pool - FX_POOL object for tracking allocations 472 473 Returns: 474 475 STATUS_SUCCESS 476 477 --*/ 478 { 479 PFX_POOL_TRACKER pTracker; 480 PLIST_ENTRY ple; 481 KIRQL oldIrql; 482 BOOLEAN leak; 483 484 // 485 // Dump usage information 486 // 487 DoTraceLevelMessage( 488 FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDEVICE, 489 "FxPoolDump: " 490 "NonPagedBytes %I64d, PagedBytes %I64d, " 491 "NonPagedAllocations %d, PagedAllocations %d," 492 "PeakNonPagedBytes %I64d, PeakPagedBytes %I64d," 493 "FxPoolDump: PeakNonPagedAllocations %d, PeakPagedAllocations %d", 494 Pool->NonPagedBytes, Pool->PagedBytes, 495 Pool->NonPagedAllocations, Pool->PagedAllocations, 496 Pool->PeakNonPagedBytes, Pool->PeakPagedBytes, 497 Pool->PeakNonPagedAllocations, Pool->PeakPagedAllocations 498 ); 499 500 leak = FALSE; 501 502 // 503 // Check paged pool for leaks 504 // 505 Pool->PagedLock.Acquire(); 506 507 for (ple = Pool->PagedHead.Flink; ple != &Pool->PagedHead; ple = ple->Flink) { 508 pTracker = CONTAINING_RECORD(ple, FX_POOL_TRACKER, Link); 509 510 // Leaker 511 leak = TRUE; 512 513 DoTraceLevelMessage( 514 FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDEVICE, 515 "FX_POOL 0x%p leaked paged memory alloc 0x%p (tracking block %p)", 516 Pool, pTracker + 1, pTracker); 517 } 518 519 Pool->PagedLock.Release(); 520 521 // 522 // Check non-paged pool for leaks 523 // 524 525 Pool->NonPagedLock.Acquire(&oldIrql); 526 527 for (ple = Pool->NonPagedHead.Flink; 528 ple != &Pool->NonPagedHead; 529 ple = ple->Flink) { 530 pTracker = CONTAINING_RECORD(ple, FX_POOL_TRACKER, Link ); 531 532 // Leaker 533 leak = TRUE; 534 535 DoTraceLevelMessage( 536 FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDEVICE, 537 "FX_POOL 0x%p leaked non-paged memory alloc 0x%p (tracking block %p)", 538 Pool, pTracker+1, pTracker); 539 } 540 541 Pool->NonPagedLock.Release(oldIrql); 542 543 if (leak) { 544 FxVerifierDbgBreakPoint(FxDriverGlobals); 545 return STATUS_MORE_ENTRIES; 546 } 547 else { 548 return STATUS_SUCCESS; 549 } 550 } 551 552 _Must_inspect_result_ 553 NTSTATUS 554 FxPoolInitialize( 555 __in PFX_DRIVER_GLOBALS FxDriverGlobals, 556 __in PFX_POOL Pool 557 ) 558 /*++ 559 560 Routine Description: 561 Initialize the FX_POOL tracking object 562 563 Arguments: 564 Pool - FX_POOL object for tracking allocations 565 566 Returns: 567 STATUS_SUCCESS 568 569 --*/ 570 { 571 NTSTATUS status = STATUS_SUCCESS; 572 573 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGPOOL, 574 "Initializing Pool 0x%p, Tracking %d", 575 Pool, FxDriverGlobals->IsPoolTrackingOn()); 576 577 Pool->NonPagedLock.Initialize(); 578 579 InitializeListHead( &Pool->NonPagedHead ); 580 581 status = Pool->PagedLock.Initialize(); 582 if (!NT_SUCCESS(status)) { 583 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGPOOL, 584 "Initializing paged lock failed for Pool 0x%p, " 585 "status %!STATUS!", 586 Pool, status); 587 goto exit; 588 } 589 590 InitializeListHead( &Pool->PagedHead ); 591 592 // Pool usage information 593 Pool->NonPagedBytes = 0; 594 Pool->PagedBytes = 0; 595 596 Pool->NonPagedAllocations = 0; 597 Pool->PagedAllocations = 0; 598 599 Pool->PeakNonPagedBytes = 0; 600 Pool->PeakPagedBytes = 0; 601 602 Pool->PeakNonPagedAllocations = 0; 603 Pool->PeakPagedAllocations = 0; 604 605 exit: 606 if (!NT_SUCCESS(status)) { 607 // 608 // We disable pool tracking if we could not initialize the locks needed 609 // 610 // If we don't do this we would need another flag to make FxPoolDestroy 611 // not access the locks 612 // 613 FxDriverGlobals->FxPoolTrackingOn = FALSE; 614 } 615 616 // 617 // FxPoolDestroy will always be called even if we fail FxPoolInitialize 618 // 619 // FxPoolDestroy will uninitialize locks both in success and failure 620 // cases 621 // 622 623 return status; 624 } 625 626 VOID 627 FxPoolDestroy( 628 __in PFX_DRIVER_GLOBALS FxDriverGlobals, 629 __in PFX_POOL Pool 630 ) 631 /*++ 632 633 Routine Description: 634 Destroy the FX_POOL tracking object 635 636 Arguments: 637 Pool - FX_POOL object for tracking allocations 638 639 Returns: 640 STATUS_SUCCESS 641 642 --*/ 643 { 644 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGPOOL, 645 "Destroying Pool 0x%p", Pool); 646 647 if (FxDriverGlobals->IsPoolTrackingOn()) { 648 FxPoolDump(FxDriverGlobals, Pool); 649 650 #if FX_CORE_MODE==FX_CORE_KERNEL_MODE 651 FxMdlDump(FxDriverGlobals); 652 #endif 653 // 654 // We don't automatically free memory items since we don't 655 // know what they contain, and who is still referencing them. 656 // 657 } 658 659 Pool->PagedLock.Uninitialize(); 660 Pool->NonPagedLock.Uninitialize(); 661 662 return; 663 } 664 665 _Must_inspect_result_ 666 NTSTATUS 667 FxPoolPackageInitialize( 668 __in PFX_DRIVER_GLOBALS FxDriverGlobals 669 ) 670 /*++ 671 672 Routine Description: 673 Initialize the pool support package at startup time. 674 675 This must be called before the first allocation. 676 677 Arguments: 678 FxDriverGlobals - DriverGlobals 679 680 Returns: 681 STATUS_SUCCESS 682 683 --*/ 684 { 685 return FxPoolInitialize(FxDriverGlobals, &FxDriverGlobals->FxPoolFrameworks); 686 } 687 688 VOID 689 FxPoolPackageDestroy( 690 __in PFX_DRIVER_GLOBALS FxDriverGlobals 691 ) 692 /*++ 693 694 Routine Description: 695 Destroy the pool support package at unload time 696 697 This must be after the last free 698 699 Arguments: 700 FxDriverGlobals - Driver's globals 701 702 Returns: 703 STATUS_SUCCESS 704 705 --*/ 706 { 707 FxPoolDestroy(FxDriverGlobals, &FxDriverGlobals->FxPoolFrameworks); 708 return; 709 } 710 711