1 /*++
2
3 Copyright (c) Microsoft Corporation
4
5 Module Name:
6
7 wdfpool.c
8
9 Abstract:
10
11 This module implements the driver frameworks pool routines.
12
13 Author:
14
15
16
17
18 Environment:
19
20 Both kernel and user mode
21
22 Revision History:
23
24
25
26
27
28
29 --*/
30
31 #include "fxobjectpch.hpp"
32
33 // We use DoTraceMessage
34 extern "C" {
35
36 #if defined(EVENT_TRACING)
37 #include "wdfpool.tmh"
38 #endif
39
40 }
41
42 BOOLEAN
FxIsPagedPoolType(__in POOL_TYPE Type)43 FxIsPagedPoolType(
44 __in POOL_TYPE Type
45 )
46 /*++
47
48 Routine Description:
49
50 Return whether paged pool is specified by POOL_TYPE
51
52 Arguments:
53
54 Type - POOL_TYPE
55
56 Returns:
57 TRUE - Paged Pool,FALSE - Non-Paged Pool
58
59 --*/
60 {
61 //
62 // Cleaner than doing (Type & 0x01)
63 //
64 switch( Type & (~POOL_COLD_ALLOCATION) ) {
65 case PagedPool:
66 case PagedPoolCacheAligned:
67 return TRUE;
68
69 default:
70 return FALSE;
71 }
72 }
73
74
75 PVOID
FxPoolAllocator(__in PFX_DRIVER_GLOBALS FxDriverGlobals,__in PFX_POOL Pool,__in POOL_TYPE Type,__in SIZE_T Size,__in ULONG Tag,__in PVOID Caller)76 FxPoolAllocator(
77 __in PFX_DRIVER_GLOBALS FxDriverGlobals,
78 __in PFX_POOL Pool,
79 __in POOL_TYPE Type,
80 __in SIZE_T Size,
81 __in ULONG Tag,
82 __in PVOID Caller
83 )
84 /*++
85
86 Routine Description:
87
88 Allocates system pool tracked in a FX_POOL tracking object.
89
90 Arguments:
91
92 Pool - FX_POOL object for tracking allocations
93
94 Type - POOL_TYPE from ntddk.h
95
96 Size - Size in bytes of the allocation
97
98 Tag - Caller specified additional tag value for debugging/tracing
99
100 Caller - Caller's address
101
102 Returns:
103
104 NULL - Could not allocate pool
105 !NULL - Pointer to pool of minimum Size bytes
106
107 Remarks:
108
109 In kernel mode this routine conditionally adds header on top iff the
110 allocation size is < PAGE_SIZE. If the allocation size is >= PAGE_SIZE
111 the caller would expect a page aligned pointer, hence no header is added.
112 In addition, ExAllocatePool* functions guarantee that a buffer < PAGE_SIZE
113 doesn't straddle page boundary. This allows FxPoolFree to determine whether
114 a header is added to buffer or not based on whether the pointer passed in
115 is page aligned or not. (In addition, when pool tracking is ON, this
116 routine adds pool tracking header based on whether additional space for this
117 header will push the buffer size beyond PAGE_SIZE, which is an optimization.)
118
119 Such guarantees are not available with user mode allocator, hence in case
120 of user mode we always add the header. (In user mode a buffer < PAGE_SIZE
121 can straddle page boundary and the pointer returned may happen to be page
122 aligned, causing FxPoolFree to free the wrong pointer.)
123
124 --*/
125 {
126 PVOID ptr;
127 PCHAR pTrueBase;
128 PFX_POOL_TRACKER pTracker;
129 PFX_POOL_HEADER pHeader;
130 NTSTATUS status;
131 SIZE_T allocationSize;
132
133
134 ptr = NULL;
135
136 //
137 // Allocations of a zero request size are invalid.
138 //
139 // Besides, with a zero request size, special pool could place us
140 // at the end of a page, and adding our header would give us a page
141 // aligned address, which is ambiguous with large allocations.
142 //
143 if (Size == 0) {
144 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGPOOL,
145 "Invalid Allocation Size of 0 requested");
146 FxVerifierDbgBreakPoint(FxDriverGlobals);
147 return NULL;
148 }
149
150 if (FxDriverGlobals->IsPoolTrackingOn()) {
151
152 if (FxDriverGlobals->FxVerifierOn &&
153 (FxDriverGlobals->WdfVerifierAllocateFailCount != -1L)) {
154
155 //
156 // If the registry key VerifierAllocateFailCount is set, all allocations
157 // after the specified count are failed.
158 //
159 // This is a brutal test, but also ensures the framework can cleanup
160 // under low memory conditions as well.
161 //
162 if (FxDriverGlobals->WdfVerifierAllocateFailCount == 0) {
163 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGPOOL,
164 "Allocation Fail Count exceeded");
165 return NULL;
166 }
167
168 // Decrement the count
169 InterlockedDecrement(&FxDriverGlobals->WdfVerifierAllocateFailCount);
170 }
171
172 //
173 // (Kernel mode only) PAGE_SIZE or greater allocations can not have our
174 // header since this would break the system allocators contract
175 // that PAGE_SIZE or greater allocations start on a whole page boundary
176 //
177
178 //
179 // For allocations less than a page size that will not fit with our
180 // header, we round up to a non-tracked whole page allocation so
181 // we don't burn two pages for this boundary condition.
182 //
183
184 // This if is the same as
185 // Size + sizeof(FX_POOL_TRACKER) + FX_POOL_HEADER_SIZE >= PAGE_SIZE
186 // BUT with no integer overflow
187 if (Mx::IsKM() &&
188 (Size >= PAGE_SIZE - sizeof(FX_POOL_TRACKER) - FX_POOL_HEADER_SIZE)
189 ) {
190
191 //
192 // Ensure that we ask for at least a page to ensure the
193 // allocation starts on a whole page.
194 //
195 if (Size < PAGE_SIZE) {
196 Size = PAGE_SIZE;
197 }
198
199 ptr = MxMemory::MxAllocatePoolWithTag(Type, Size, Tag);
200
201 //
202 // The current system allocator returns paged aligned memory
203 // in this case, which we rely on to detect whether our header
204 // is present or not in FxPoolFree
205 //
206 ASSERT(((ULONG_PTR)ptr & (PAGE_SIZE-1)) == 0);
207 }
208 else {
209
210 status = RtlSIZETAdd(Size,
211 sizeof(FX_POOL_TRACKER) + FX_POOL_HEADER_SIZE,
212 &allocationSize);
213
214 if (!NT_SUCCESS(status)) {
215 DoTraceLevelMessage(
216 FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGPOOL,
217 "overflow: allocation tracker (%d) + header (%d) + pool "
218 "request (%I64d)", sizeof(FX_POOL_TRACKER),
219 FX_POOL_HEADER_SIZE, Size);
220
221 return NULL;
222 }
223
224 pTrueBase = (PCHAR) MxMemory::MxAllocatePoolWithTag(
225 Type,
226 allocationSize,
227 Tag
228 );
229
230 if (pTrueBase == NULL) {
231 return NULL;
232 }
233
234 pTracker = (PFX_POOL_TRACKER) pTrueBase;
235 pHeader = WDF_PTR_ADD_OFFSET_TYPE(pTrueBase,
236 sizeof(FX_POOL_TRACKER),
237 PFX_POOL_HEADER);
238 pHeader->Base = pTrueBase;
239 pHeader->FxDriverGlobals = FxDriverGlobals;
240
241 //
242 // Adjust the pointer to what we return to the driver
243 //
244 ptr = &pHeader->AllocationStart[0];
245
246 //
247 // Ensure the pointer we are returning is aligned on the proper
248 // boundary.
249 //
250 ASSERT( ((ULONG_PTR) ptr & (MEMORY_ALLOCATION_ALIGNMENT-1)) == 0);
251
252 //
253 // Ensure the pointer is still not page aligned after
254 // our adjustment. Otherwise the pool free code will
255 // get confused and call ExFreePool on the wrong ptr.
256 //
257 if (Mx::IsKM()) {
258 ASSERT(((ULONG_PTR)ptr & (PAGE_SIZE-1)) != 0 );
259 }
260
261 //
262 // We must separate paged and non-paged pool since
263 // the lock held differs as to whether we can accept
264 // page faults and block in the allocator.
265 //
266 if (FxIsPagedPoolType(Type)) {
267 //
268 // Format and insert the Tracker in the PagedHeader list.
269 //
270 FxPoolInsertPagedAllocateTracker(Pool,
271 pTracker,
272 Size,
273 Tag,
274 Caller);
275 }
276 else {
277 //
278 // Format and insert the Tracker in the NonPagedHeader list.
279 //
280 FxPoolInsertNonPagedAllocateTracker(Pool,
281 pTracker,
282 Size,
283 Tag,
284 Caller);
285 }
286 }
287 }
288 else {
289 //
290 // No pool tracking...
291 //
292
293 if ((Size < PAGE_SIZE) || Mx::IsUM())
294 {
295 //
296 // (Kernel mode only) See if adding our header promotes us past a
297 // page boundary
298 //
299 status = RtlSIZETAdd(Size,
300 FX_POOL_HEADER_SIZE,
301 &allocationSize);
302
303 if (!NT_SUCCESS(status)) {
304 DoTraceLevelMessage(
305 FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGPOOL,
306 "overflow: header + pool request (%I64d)", Size);
307
308 return NULL;
309 }
310
311 }
312 else {
313 //
314 // Is the raw request for alloc >= PAGE_SIZE ? Then just use it.
315 //
316 allocationSize = Size;
317 }
318
319 //
320 // Is cooked size for alloc >= PAGE_SIZE ? Then just do it.
321 //
322 if (allocationSize >= PAGE_SIZE && Mx::IsKM())
323 {
324 //
325 // Important to use allocationSize so that we get a page aligned
326 // allocation so that we know to just free the memory pointer as is
327 // when it is freed.
328 //
329 ptr = MxMemory::MxAllocatePoolWithTag(Type, allocationSize, Tag);
330 ASSERT(((ULONG_PTR)ptr & (PAGE_SIZE-1)) == 0);
331 }
332 else {
333 pTrueBase = (PCHAR) MxMemory::MxAllocatePoolWithTag(Type,
334 allocationSize,
335 Tag);
336
337 if (pTrueBase != NULL) {
338
339 pHeader = (PFX_POOL_HEADER) pTrueBase;
340 pHeader->Base = pTrueBase;
341 pHeader->FxDriverGlobals = FxDriverGlobals;
342
343 ptr = &pHeader->AllocationStart[0];
344
345 if (Mx::IsKM()) {
346 //
347 // Ensure the pointer is still not page aligned after
348 // our adjustment. Otherwise the pool free code will
349 // get confused and call ExFreePool on the wrong ptr.
350 //
351 ASSERT( ((ULONG_PTR)ptr & (PAGE_SIZE-1)) != 0 );
352 }
353 }
354 }
355 }
356
357 return ptr;
358 }
359
360 VOID
FxPoolFree(__in_xcount (ptr is at an offset from AllocationStart)PVOID ptr)361 FxPoolFree(
362 __in_xcount(ptr is at an offset from AllocationStart) PVOID ptr
363 )
364 /*++
365
366 Routine Description:
367
368 Release tracked pool
369
370 Arguments:
371
372 Pool - FX_POOL object allocation is tracked in
373
374 ptr - Pointer to pool to release
375
376 Returns:
377
378 Remarks:
379 In kernel mode the pointer passed in may or may not have a header before
380 it depending upon whether the pointer is page aligned or not.
381
382 In user mode the pointer passed in always has a header before it. See
383 remarks for FxPoolAllocator.
384
385 --*/
386 {
387 PFX_POOL_HEADER pHeader;
388 PVOID pTrueBase;
389 PFX_POOL_TRACKER pTracker;
390
391 //
392 // Null pointers are always bad
393 //
394 if( ptr == NULL ) {
395 ASSERTMSG("NULL pointer freed\n", FALSE);
396 Mx::MxBugCheckEx(WDF_VIOLATION,
397 WDF_REQUIRED_PARAMETER_IS_NULL,
398 (ULONG_PTR)NULL,
399 (ULONG_PTR)_ReturnAddress(),
400 (ULONG_PTR)NULL
401 );
402 }
403
404 //
405 // (Kernel mode only) If ptr is aligned on page boundry (indicates
406 // it was > PAGE_SIZE allocation)
407 // then there will be no common header...just free the memory without
408 // further processing.
409 //
410 if( Mx::IsKM() && ( ((ULONG_PTR)ptr & (PAGE_SIZE-1)) == 0 ) ) {
411 MxMemory::MxFreePool(ptr);
412 return;
413 }
414
415 //
416 // Ensure the pointer we are returning is aligned on the proper
417 // boundary.
418 //
419 ASSERT( ((ULONG_PTR) ptr & (MEMORY_ALLOCATION_ALIGNMENT-1)) == 0);
420
421 //
422 // Dereference the Common header which all <PAGE_SIZE allcations will have.
423 //
424 pHeader = CONTAINING_RECORD(ptr, FX_POOL_HEADER, AllocationStart);
425 pTrueBase = pHeader->Base;
426
427 //
428 // If PoolTracker is on then Base must point to it's header.
429 // This is currently the only option for this area...may change later.
430 //
431 if (pHeader->FxDriverGlobals->IsPoolTrackingOn()) {
432
433 pTracker = (PFX_POOL_TRACKER) pTrueBase;
434
435 if (FxIsPagedPoolType(pTracker->PoolType)) {
436 //
437 // Decommission this Paged Allocation tracker
438 //
439 FxPoolRemovePagedAllocateTracker(pTracker);
440 }
441 else {
442 //
443 // Decommission this NonPaged Allocation tracker
444 //
445 FxPoolRemoveNonPagedAllocateTracker(pTracker);
446 }
447
448 //
449 // Scrub the pool to zeros to catch destructed objects
450 // by NULL'ing the v-table ptr
451 //
452 RtlZeroMemory(pTracker, pTracker->Size + sizeof(FX_POOL_TRACKER));
453 }
454
455 MxMemory::MxFreePool(pTrueBase);
456 }
457
458 NTSTATUS
FxPoolDump(__in PFX_DRIVER_GLOBALS FxDriverGlobals,__in PFX_POOL Pool)459 FxPoolDump(
460 __in PFX_DRIVER_GLOBALS FxDriverGlobals,
461 __in PFX_POOL Pool
462 )
463 /*++
464
465 Routine Description:
466
467 Dump the FX_POOL tracking object
468
469 Arguments:
470
471 Pool - FX_POOL object for tracking allocations
472
473 Returns:
474
475 STATUS_SUCCESS
476
477 --*/
478 {
479 PFX_POOL_TRACKER pTracker;
480 PLIST_ENTRY ple;
481 KIRQL oldIrql;
482 BOOLEAN leak;
483
484 //
485 // Dump usage information
486 //
487 DoTraceLevelMessage(
488 FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDEVICE,
489 "FxPoolDump: "
490 "NonPagedBytes %I64d, PagedBytes %I64d, "
491 "NonPagedAllocations %d, PagedAllocations %d,"
492 "PeakNonPagedBytes %I64d, PeakPagedBytes %I64d,"
493 "FxPoolDump: PeakNonPagedAllocations %d, PeakPagedAllocations %d",
494 Pool->NonPagedBytes, Pool->PagedBytes,
495 Pool->NonPagedAllocations, Pool->PagedAllocations,
496 Pool->PeakNonPagedBytes, Pool->PeakPagedBytes,
497 Pool->PeakNonPagedAllocations, Pool->PeakPagedAllocations
498 );
499
500 leak = FALSE;
501
502 //
503 // Check paged pool for leaks
504 //
505 Pool->PagedLock.Acquire();
506
507 for (ple = Pool->PagedHead.Flink; ple != &Pool->PagedHead; ple = ple->Flink) {
508 pTracker = CONTAINING_RECORD(ple, FX_POOL_TRACKER, Link);
509
510 // Leaker
511 leak = TRUE;
512
513 DoTraceLevelMessage(
514 FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDEVICE,
515 "FX_POOL 0x%p leaked paged memory alloc 0x%p (tracking block %p)",
516 Pool, pTracker + 1, pTracker);
517 }
518
519 Pool->PagedLock.Release();
520
521 //
522 // Check non-paged pool for leaks
523 //
524
525 Pool->NonPagedLock.Acquire(&oldIrql);
526
527 for (ple = Pool->NonPagedHead.Flink;
528 ple != &Pool->NonPagedHead;
529 ple = ple->Flink) {
530 pTracker = CONTAINING_RECORD(ple, FX_POOL_TRACKER, Link );
531
532 // Leaker
533 leak = TRUE;
534
535 DoTraceLevelMessage(
536 FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGDEVICE,
537 "FX_POOL 0x%p leaked non-paged memory alloc 0x%p (tracking block %p)",
538 Pool, pTracker+1, pTracker);
539 }
540
541 Pool->NonPagedLock.Release(oldIrql);
542
543 if (leak) {
544 FxVerifierDbgBreakPoint(FxDriverGlobals);
545 return STATUS_MORE_ENTRIES;
546 }
547 else {
548 return STATUS_SUCCESS;
549 }
550 }
551
552 _Must_inspect_result_
553 NTSTATUS
FxPoolInitialize(__in PFX_DRIVER_GLOBALS FxDriverGlobals,__in PFX_POOL Pool)554 FxPoolInitialize(
555 __in PFX_DRIVER_GLOBALS FxDriverGlobals,
556 __in PFX_POOL Pool
557 )
558 /*++
559
560 Routine Description:
561 Initialize the FX_POOL tracking object
562
563 Arguments:
564 Pool - FX_POOL object for tracking allocations
565
566 Returns:
567 STATUS_SUCCESS
568
569 --*/
570 {
571 NTSTATUS status = STATUS_SUCCESS;
572
573 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGPOOL,
574 "Initializing Pool 0x%p, Tracking %d",
575 Pool, FxDriverGlobals->IsPoolTrackingOn());
576
577 Pool->NonPagedLock.Initialize();
578
579 InitializeListHead( &Pool->NonPagedHead );
580
581 status = Pool->PagedLock.Initialize();
582 if (!NT_SUCCESS(status)) {
583 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_ERROR, TRACINGPOOL,
584 "Initializing paged lock failed for Pool 0x%p, "
585 "status %!STATUS!",
586 Pool, status);
587 goto exit;
588 }
589
590 InitializeListHead( &Pool->PagedHead );
591
592 // Pool usage information
593 Pool->NonPagedBytes = 0;
594 Pool->PagedBytes = 0;
595
596 Pool->NonPagedAllocations = 0;
597 Pool->PagedAllocations = 0;
598
599 Pool->PeakNonPagedBytes = 0;
600 Pool->PeakPagedBytes = 0;
601
602 Pool->PeakNonPagedAllocations = 0;
603 Pool->PeakPagedAllocations = 0;
604
605 exit:
606 if (!NT_SUCCESS(status)) {
607 //
608 // We disable pool tracking if we could not initialize the locks needed
609 //
610 // If we don't do this we would need another flag to make FxPoolDestroy
611 // not access the locks
612 //
613 FxDriverGlobals->FxPoolTrackingOn = FALSE;
614 }
615
616 //
617 // FxPoolDestroy will always be called even if we fail FxPoolInitialize
618 //
619 // FxPoolDestroy will uninitialize locks both in success and failure
620 // cases
621 //
622
623 return status;
624 }
625
626 VOID
FxPoolDestroy(__in PFX_DRIVER_GLOBALS FxDriverGlobals,__in PFX_POOL Pool)627 FxPoolDestroy(
628 __in PFX_DRIVER_GLOBALS FxDriverGlobals,
629 __in PFX_POOL Pool
630 )
631 /*++
632
633 Routine Description:
634 Destroy the FX_POOL tracking object
635
636 Arguments:
637 Pool - FX_POOL object for tracking allocations
638
639 Returns:
640 STATUS_SUCCESS
641
642 --*/
643 {
644 DoTraceLevelMessage(FxDriverGlobals, TRACE_LEVEL_VERBOSE, TRACINGPOOL,
645 "Destroying Pool 0x%p", Pool);
646
647 if (FxDriverGlobals->IsPoolTrackingOn()) {
648 FxPoolDump(FxDriverGlobals, Pool);
649
650 #if FX_CORE_MODE==FX_CORE_KERNEL_MODE
651 FxMdlDump(FxDriverGlobals);
652 #endif
653 //
654 // We don't automatically free memory items since we don't
655 // know what they contain, and who is still referencing them.
656 //
657 }
658
659 Pool->PagedLock.Uninitialize();
660 Pool->NonPagedLock.Uninitialize();
661
662 return;
663 }
664
665 _Must_inspect_result_
666 NTSTATUS
FxPoolPackageInitialize(__in PFX_DRIVER_GLOBALS FxDriverGlobals)667 FxPoolPackageInitialize(
668 __in PFX_DRIVER_GLOBALS FxDriverGlobals
669 )
670 /*++
671
672 Routine Description:
673 Initialize the pool support package at startup time.
674
675 This must be called before the first allocation.
676
677 Arguments:
678 FxDriverGlobals - DriverGlobals
679
680 Returns:
681 STATUS_SUCCESS
682
683 --*/
684 {
685 return FxPoolInitialize(FxDriverGlobals, &FxDriverGlobals->FxPoolFrameworks);
686 }
687
688 VOID
FxPoolPackageDestroy(__in PFX_DRIVER_GLOBALS FxDriverGlobals)689 FxPoolPackageDestroy(
690 __in PFX_DRIVER_GLOBALS FxDriverGlobals
691 )
692 /*++
693
694 Routine Description:
695 Destroy the pool support package at unload time
696
697 This must be after the last free
698
699 Arguments:
700 FxDriverGlobals - Driver's globals
701
702 Returns:
703 STATUS_SUCCESS
704
705 --*/
706 {
707 FxPoolDestroy(FxDriverGlobals, &FxDriverGlobals->FxPoolFrameworks);
708 return;
709 }
710
711