1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2015-2020 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #if defined(NVRM)
25 # include "os/os.h"
26 #else
27 # include "shrdebug.h"
28 # include "nvos.h"
29 #endif
30 #include "containers/eheap_old.h"
31
32 #if !defined(SRT_BUILD)
33 #include "os/os.h"
34 #endif
35
36 static void initPublicObjectFunctionPointers_EHeap(OBJEHEAP *pHeap);
37 static NV_STATUS eheapInit(OBJEHEAP *, NvU64, NvU64, NvU32, NvU32);
38 static NV_STATUS eheapDestruct(OBJEHEAP *);
39 static NV_STATUS eheapAlloc(OBJEHEAP *, NvU32, NvU32 *, NvU64 *, NvU64 *,NvU64, NvU64, EMEMBLOCK **, void*, EHeapOwnershipComparator*);
40 static NV_STATUS eheapFree(OBJEHEAP *, NvU64);
41 static void eheapInfo(OBJEHEAP *, NvU64 *, NvU64 *, NvU64 *, NvU64 *, NvU32 *, NvU64 *);
42 static void eheapInfoForRange(OBJEHEAP *, NV_RANGE, NvU64 *, NvU64 *, NvU32 *, NvU64 *);
43 static NV_STATUS eheapGetSize(OBJEHEAP *, NvU64 *);
44 static NV_STATUS eheapGetFree(OBJEHEAP *, NvU64 *);
45 static NV_STATUS eheapGetBase(OBJEHEAP *, NvU64 *);
46 static EMEMBLOCK *eheapGetBlock(OBJEHEAP *, NvU64, NvBool);
47 static NV_STATUS eheapSetAllocRange(OBJEHEAP *, NvU64, NvU64);
48 static NV_STATUS eheapTraverse(OBJEHEAP *, void *, EHeapTraversalFn, NvS32);
49 static NV_STATUS _eheapBlockFree(OBJEHEAP *pHeap, EMEMBLOCK *block);
50 static NvU32 eheapGetNumBlocks(OBJEHEAP *);
51 static NV_STATUS eheapGetBlockInfo(OBJEHEAP *, NvU32, NVOS32_HEAP_DUMP_BLOCK *);
52 static NV_STATUS eheapSetOwnerIsolation(OBJEHEAP *, NvBool, NvU32);
53 static NvBool _eheapCheckOwnership(OBJEHEAP *, void*, NvU64, NvU64, EMEMBLOCK *, EHeapOwnershipComparator*);
54
55 void
constructObjEHeap(OBJEHEAP * pHeap,NvU64 Base,NvU64 LimitPlusOne,NvU32 sizeofMemBlock,NvU32 numPreAllocMemStruct)56 constructObjEHeap(OBJEHEAP *pHeap, NvU64 Base, NvU64 LimitPlusOne, NvU32 sizeofMemBlock, NvU32 numPreAllocMemStruct)
57 {
58 initPublicObjectFunctionPointers_EHeap(pHeap);
59
60 eheapInit(pHeap, Base, LimitPlusOne, sizeofMemBlock, numPreAllocMemStruct);
61 }
62
63 static void
initPublicObjectFunctionPointers_EHeap(OBJEHEAP * pHeap)64 initPublicObjectFunctionPointers_EHeap(OBJEHEAP *pHeap)
65 {
66 pHeap->eheapDestruct = eheapDestruct;
67 pHeap->eheapAlloc = eheapAlloc;
68 pHeap->eheapFree = eheapFree;
69 pHeap->eheapInfo = eheapInfo;
70 pHeap->eheapInfoForRange = eheapInfoForRange;
71 pHeap->eheapGetSize = eheapGetSize;
72 pHeap->eheapGetFree = eheapGetFree;
73 pHeap->eheapGetBase = eheapGetBase;
74 pHeap->eheapGetBlock = eheapGetBlock;
75 pHeap->eheapSetAllocRange = eheapSetAllocRange;
76 pHeap->eheapTraverse = eheapTraverse;
77 pHeap->eheapGetNumBlocks = eheapGetNumBlocks;
78 pHeap->eheapGetBlockInfo = eheapGetBlockInfo;
79 pHeap->eheapSetOwnerIsolation = eheapSetOwnerIsolation;
80 }
81
82 static NV_STATUS
_eheapAllocMemStruct(OBJEHEAP * pHeap,EMEMBLOCK ** ppMemBlock)83 _eheapAllocMemStruct
84 (
85 OBJEHEAP *pHeap,
86 EMEMBLOCK **ppMemBlock
87 )
88 {
89 if (pHeap->numPreAllocMemStruct > 0)
90 {
91 // We are out of pre-allocated mem data structs
92 if (NULL == pHeap->pFreeMemStructList)
93 {
94 NV_ASSERT(0);
95 return NV_ERR_OPERATING_SYSTEM;
96 }
97
98 *ppMemBlock = pHeap->pFreeMemStructList;
99 pHeap->pFreeMemStructList = pHeap->pFreeMemStructList->next;
100 }
101 else
102 {
103 *ppMemBlock = portMemAllocNonPaged(pHeap->sizeofMemBlock);
104
105 if (*ppMemBlock == NULL)
106 {
107 NV_ASSERT(0);
108 return NV_ERR_OPERATING_SYSTEM;
109 }
110 portMemSet(*ppMemBlock, 0, pHeap->sizeofMemBlock);
111 }
112
113 return NV_OK;
114 }
115
116 static NV_STATUS
_eheapFreeMemStruct(OBJEHEAP * pHeap,EMEMBLOCK ** ppMemBlock)117 _eheapFreeMemStruct
118 (
119 OBJEHEAP *pHeap,
120 EMEMBLOCK **ppMemBlock
121 )
122 {
123 if (pHeap->numPreAllocMemStruct > 0)
124 {
125 portMemSet(*ppMemBlock, 0, pHeap->sizeofMemBlock);
126
127 (*ppMemBlock)->next = pHeap->pFreeMemStructList;
128 pHeap->pFreeMemStructList = *ppMemBlock;
129
130 *ppMemBlock = NULL;
131 }
132 else
133 {
134 portMemFree(*ppMemBlock);
135 *ppMemBlock = NULL;
136 }
137
138 return NV_OK;
139 }
140
141 //
142 // Create a heap. Even though we can return error here the resultant
143 // object must be self consistent (zero pointers, etc) if there were
144 // alloc failures, etc.
145 //
146 static NV_STATUS
eheapInit(OBJEHEAP * pHeap,NvU64 Base,NvU64 LimitPlusOne,NvU32 sizeofData,NvU32 numPreAllocMemStruct)147 eheapInit
148 (
149 OBJEHEAP *pHeap,
150 NvU64 Base,
151 NvU64 LimitPlusOne,
152 NvU32 sizeofData,
153 NvU32 numPreAllocMemStruct
154 )
155 {
156 EMEMBLOCK *block;
157 NvU32 i;
158
159 //
160 // Simply create a free heap.
161 //
162 pHeap->base = Base;
163 pHeap->total = LimitPlusOne - Base;
164 pHeap->rangeLo = pHeap->base;
165 pHeap->rangeHi = pHeap->base + pHeap->total - 1;
166 pHeap->free = pHeap->total;
167 pHeap->sizeofMemBlock = sizeofData + sizeof(EMEMBLOCK);
168
169 pHeap->numPreAllocMemStruct = 0;
170 pHeap->pPreAllocAddr = NULL;
171 pHeap->pBlockList = NULL;
172 pHeap->pFreeBlockList = NULL;
173 pHeap->pFreeMemStructList = NULL;
174 pHeap->numBlocks = 0;
175 pHeap->pBlockTree = NULL;
176 pHeap->bOwnerIsolation = NV_FALSE;
177 pHeap->ownerGranularity = 0;
178
179 //
180 // User requested a static eheap that has a list of pre-allocated
181 // EMEMBLOCK data structure.
182 //
183 if (numPreAllocMemStruct > 0)
184 {
185 ++numPreAllocMemStruct; // reserve one for us - see below
186
187 pHeap->pPreAllocAddr = portMemAllocNonPaged(pHeap->sizeofMemBlock * numPreAllocMemStruct);
188
189 if (pHeap->pPreAllocAddr)
190 {
191 pHeap->numPreAllocMemStruct = numPreAllocMemStruct;
192 pHeap->pFreeMemStructList = pHeap->pPreAllocAddr;
193
194 portMemSet(pHeap->pFreeMemStructList, 0, pHeap->sizeofMemBlock * numPreAllocMemStruct);
195
196 //
197 // Form the list of free mem structures. Just need to utilize the next field of EMEMBLOCK.
198 //
199 for (i = 0; i < numPreAllocMemStruct - 1; i++)
200 {
201 ((EMEMBLOCK *)((NvU8 *)pHeap->pFreeMemStructList + (i * pHeap->sizeofMemBlock)))->next
202 = (EMEMBLOCK *)((NvU8 *)pHeap->pFreeMemStructList + (i + 1) * pHeap->sizeofMemBlock);
203 }
204 }
205 }
206
207 if (_eheapAllocMemStruct(pHeap, &block) != NV_OK)
208 {
209 return NV_ERR_OPERATING_SYSTEM;
210 }
211
212 block->owner = NVOS32_BLOCK_TYPE_FREE;
213 block->refCount = 0;
214 block->begin = Base;
215 block->align = Base;
216 block->end = LimitPlusOne - 1;
217 block->prevFree = block;
218 block->nextFree = block;
219 block->next = block;
220 block->prev = block;
221 block->pData = (void*)(block+1);
222
223 //
224 // Fill in the heap bank info.
225 //
226 pHeap->pBlockList = block;
227 pHeap->pFreeBlockList = block;
228 pHeap->numBlocks = 1;
229
230 portMemSet((void *)&block->node, 0, sizeof(NODE));
231 block->node.keyStart = block->begin;
232 block->node.keyEnd = block->end;
233 block->node.Data = (void *)block;
234 if (btreeInsert(&block->node, &pHeap->pBlockTree) != NV_OK)
235 {
236 eheapDestruct(pHeap);
237 return NV_ERR_OPERATING_SYSTEM;
238 }
239
240 return NV_OK;
241 }
242
243 static NV_STATUS
eheapDestruct(OBJEHEAP * pHeap)244 eheapDestruct
245 (
246 OBJEHEAP *pHeap
247 )
248 {
249 EMEMBLOCK *block, *blockFirst, *blockNext;
250 NvBool headptr_updated;
251
252 if (!pHeap->pBlockList)
253 return NV_OK;
254
255 //
256 // Free all allocated blocks
257 //
258 do {
259 block = blockFirst = pHeap->pBlockList;
260 headptr_updated = NV_FALSE;
261
262 do {
263 blockNext = block->next;
264
265 _eheapBlockFree(pHeap, block);
266
267 // restart scanning the list, if the heap->pBlockList changed
268 if (blockFirst != pHeap->pBlockList) {
269 headptr_updated = NV_TRUE;
270 break;
271 }
272
273 block = blockNext;
274
275 } while (block != pHeap->pBlockList);
276
277 } while (headptr_updated);
278
279 if (pHeap->numPreAllocMemStruct > 0)
280 {
281 // free static blocks
282 portMemFree(pHeap->pPreAllocAddr);
283 pHeap->pPreAllocAddr = NULL;
284 }
285 else
286 {
287 portMemFree(pHeap->pBlockList);
288 pHeap->pBlockList = NULL;
289 }
290
291 return NV_OK;
292 }
293
294 // 'flags' using NVOS32_ALLOC_FLAGS_* though some are n/a
295 static NV_STATUS
eheapAlloc(OBJEHEAP * pHeap,NvU32 owner,NvU32 * flags,NvU64 * offset,NvU64 * size,NvU64 offsetAlign,NvU64 sizeAlign,EMEMBLOCK ** ppMemBlock,void * pIsolationID,EHeapOwnershipComparator * checker)296 eheapAlloc
297 (
298 OBJEHEAP *pHeap,
299 NvU32 owner,
300 NvU32 *flags,
301 NvU64 *offset,
302 NvU64 *size,
303 NvU64 offsetAlign,
304 NvU64 sizeAlign,
305 EMEMBLOCK **ppMemBlock, // not generally useful over e.g. a split!
306 void *pIsolationID,
307 EHeapOwnershipComparator *checker
308 )
309 {
310 NvU64 allocLo, allocAl, allocHi;
311 EMEMBLOCK *blockFirstFree, *blockFree;
312 EMEMBLOCK *blockNew = NULL, *blockSplit = NULL;
313 NvU64 desiredOffset;
314 NvU64 allocSize;
315 NvU64 rangeLo, rangeHi;
316
317 if ((*flags & NVOS32_ALLOC_FLAGS_FORCE_INTERNAL_INDEX) &&
318 (*flags & NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE))
319 {
320 return NV_ERR_INVALID_ARGUMENT;
321 }
322
323 // Save the offset for fixed address requests, or it's likely uninitialized.
324 desiredOffset = (*flags & NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE) ? *offset: 0;
325
326 //
327 // zero result so that apps that ignore return code have another
328 // chance to see the error of their ways...
329 //
330 *offset = 0;
331
332 //
333 // Check for valid size.
334 //
335 if (*size == 0)
336 return NV_ERR_INVALID_ARGUMENT;
337
338 //
339 // Range-limited the request.
340 //
341 rangeLo = pHeap->rangeLo;
342 rangeHi = pHeap->rangeHi;
343
344 if (rangeLo == 0 && rangeHi == 0) {
345 rangeLo = pHeap->base;
346 rangeHi = pHeap->base + pHeap->total - 1;
347 }
348 if (rangeHi > pHeap->base + pHeap->total - 1) {
349 rangeHi = pHeap->base + pHeap->total - 1;
350 }
351 if (rangeLo > rangeHi)
352 return NV_ERR_INVALID_ARGUMENT;
353
354 // Align size up.
355 allocSize = ((*size + (sizeAlign - 1)) / sizeAlign) * sizeAlign;
356
357 //
358 // Trivial reject size vs. free.
359 //
360 if (pHeap->free < allocSize)
361 return NV_ERR_NO_MEMORY;
362
363 /* This flag will force an exclusive allocation of the request
364 * within the range of ownerGranularity
365 */
366
367 if ( *flags & NVOS32_ALLOC_FLAGS_FORCE_INTERNAL_INDEX )
368 {
369 NvU64 desiredOffsetLo, desiredOffsetHi;
370
371 NV_ASSERT_OR_RETURN(pHeap->ownerGranularity, NV_ERR_INVALID_ARGUMENT);
372 NV_ASSERT_OR_RETURN(pHeap->bOwnerIsolation && checker, NV_ERR_INVALID_ARGUMENT);
373
374 blockFree = pHeap->pFreeBlockList;
375
376 if (blockFree == NULL)
377 goto failed;
378
379 do
380 {
381 desiredOffset = NV_ALIGN_DOWN(blockFree->begin, pHeap->ownerGranularity) + offsetAlign;
382
383 while (desiredOffset + allocSize - 1 <= blockFree->end)
384 {
385 desiredOffsetLo = NV_ALIGN_DOWN(desiredOffset, pHeap->ownerGranularity);
386 desiredOffsetHi = (((desiredOffset % pHeap->ownerGranularity) == 0) ?
387 NV_ALIGN_UP((desiredOffset + 1), pHeap->ownerGranularity) :
388 NV_ALIGN_UP(desiredOffset, pHeap->ownerGranularity));
389
390 if ((desiredOffset >= blockFree->begin) &&
391 ((desiredOffsetLo >= blockFree->begin) &&
392 (desiredOffsetHi <= blockFree->end)))
393 {
394 if (_eheapCheckOwnership(pHeap, pIsolationID, desiredOffset,
395 desiredOffset + allocSize - 1, blockFree, checker))
396 {
397 allocLo = desiredOffset;
398 allocHi = desiredOffset + allocSize - 1;
399 allocAl = allocLo;
400 goto got_one;
401 }
402 }
403
404 desiredOffset += pHeap->ownerGranularity;
405 }
406
407 blockFree = blockFree->nextFree;
408
409 } while (blockFree != pHeap->pFreeBlockList);
410
411 /* return error if can't get that particular address */
412 goto failed;
413 }
414
415 // Ensure a valid allocation type was passed in
416 //if (type > NVOS32_NUM_MEM_TYPES - 1)
417 //return NV_ERR_INVALID_ARGUMENT;
418
419 //
420 // Check for fixed address request.
421 // This allows caller to say: I really want this memory at a particular
422 // offset. Returns error if can't get that offset.
423 //
424 if ( *flags & NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE )
425 {
426 // is our desired offset suitably aligned?
427 if (desiredOffset % offsetAlign)
428 goto failed;
429
430 blockFree = pHeap->pFreeBlockList;
431
432 if (blockFree == NULL)
433 {
434 goto failed;
435 }
436
437 do
438 {
439 //
440 // Allocate from the bottom of the memory block.
441 //
442 blockFree = blockFree->nextFree;
443
444 // Does this block contain our desired range?
445 if ( (desiredOffset >= blockFree->begin) &&
446 (desiredOffset + allocSize - 1) <= blockFree->end )
447 {
448 //
449 // Make sure no allocated block between ALIGN_DOWN(allocLo, granularity)
450 // and ALIGN_UP(allocHi, granularity) have a different owner than the current allocation
451 //
452 if (pHeap->bOwnerIsolation)
453 {
454 NV_ASSERT(NULL != checker);
455 if (!_eheapCheckOwnership(pHeap, pIsolationID, desiredOffset,
456 desiredOffset + allocSize - 1, blockFree, checker))
457 {
458 break;
459 }
460 }
461
462 // we have a match, now remove it from the pool
463 allocLo = desiredOffset;
464 allocHi = desiredOffset + allocSize - 1;
465 allocAl = allocLo;
466 goto got_one;
467 }
468
469 } while (blockFree != pHeap->pFreeBlockList);
470
471 // return error if can't get that particular address
472 goto failed;
473 }
474
475 blockFirstFree = pHeap->pFreeBlockList;
476 if (!blockFirstFree)
477 goto failed;
478
479 //
480 // When scanning upwards, start at the bottom - 1 so the following loop looks symmetric.
481 //
482 if ( *flags & NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_DOWN ) {
483 blockFirstFree = blockFirstFree->prevFree;
484 }
485 blockFree = blockFirstFree;
486 do
487 {
488 NvU64 blockLo;
489 NvU64 blockHi;
490
491 //
492 // Is this block completely out of range?
493 //
494 if ( ( blockFree->end < rangeLo ) || ( blockFree->begin > rangeHi ) )
495 {
496 if ( *flags & NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_DOWN )
497 blockFree = blockFree->prevFree;
498 else
499 blockFree = blockFree->nextFree;
500 continue;
501 }
502
503 //
504 // Find the intersection of the free block and the specified range.
505 //
506 blockLo = (rangeLo > blockFree->begin) ? rangeLo : blockFree->begin;
507 blockHi = (rangeHi < blockFree->end) ? rangeHi : blockFree->end;
508
509 if ( *flags & NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_DOWN )
510 {
511 //
512 // Allocate from the top of the memory block.
513 //
514 allocLo = (blockHi - allocSize + 1) / offsetAlign * offsetAlign;
515 allocAl = allocLo;
516 allocHi = allocAl + allocSize - 1;
517 }
518 else
519 {
520 //
521 // Allocate from the bottom of the memory block.
522 //
523 allocAl = (blockLo + (offsetAlign - 1)) / offsetAlign * offsetAlign;
524 allocLo = allocAl;
525 allocHi = allocAl + allocSize - 1;
526 }
527
528 //
529 // Make sure no allocated block between ALIGN_DOWN(allocLo, granularity)
530 // and ALIGN_UP(allocHi, granularity) have a different owner than the current allocation
531 //
532 if (pHeap->bOwnerIsolation)
533 {
534 NV_ASSERT(NULL != checker);
535
536 if (_eheapCheckOwnership(pHeap, pIsolationID, allocLo, allocHi, blockFree, checker))
537 {
538 goto alloc_done;
539 }
540
541 //
542 // Try realloc if we still have enough free memory in current free block
543 //
544 if (*flags & NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_DOWN)
545 {
546 NvU64 checkLo = NV_ALIGN_DOWN(allocLo, pHeap->ownerGranularity);
547
548 if (checkLo > blockFree->begin)
549 {
550 blockHi = checkLo;
551
552 allocLo = (blockHi - allocSize + 1) / offsetAlign * offsetAlign;
553 allocAl = allocLo;
554 allocHi = allocAl + allocSize - 1;
555
556 if (_eheapCheckOwnership(pHeap, pIsolationID, allocLo, allocHi, blockFree, checker))
557 {
558 goto alloc_done;
559 }
560 }
561 }
562 else
563 {
564 NvU64 checkHi = NV_ALIGN_UP(allocHi, pHeap->ownerGranularity);
565
566 if (checkHi < blockFree->end)
567 {
568 blockLo = checkHi;
569
570 allocAl = (blockLo + (offsetAlign - 1)) / offsetAlign * offsetAlign;
571 allocLo = allocAl;
572 allocHi = allocAl + allocSize - 1;
573
574 if (_eheapCheckOwnership(pHeap, pIsolationID, allocLo, allocHi, blockFree, checker))
575 {
576 goto alloc_done;
577 }
578 }
579 }
580
581 //
582 // Cannot find any available memory in current free block, go to the next
583 //
584 goto next_free;
585 }
586
587 alloc_done:
588 //
589 // Does the desired range fall completely within this block?
590 // Also make sure it does not wrap-around.
591 // Also make sure it is within the desired range.
592 //
593 if ((allocLo >= blockFree->begin) && (allocHi <= blockFree->end))
594 {
595 if (allocLo <= allocHi)
596 if ((allocLo >= rangeLo) && (allocHi <= rangeHi))
597 goto got_one;
598
599 }
600
601 next_free:
602 if ( *flags & NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_DOWN )
603 blockFree = blockFree->prevFree;
604 else
605 blockFree = blockFree->nextFree;
606
607 } while (blockFree != blockFirstFree);
608
609 //
610 // Out of memory.
611 //
612 goto failed;
613
614 //
615 // We have a match. Now link it in, trimming or splitting
616 // any slop from the enclosing block as needed.
617 //
618
619 got_one:
620
621 if ((allocLo == blockFree->begin) && (allocHi == blockFree->end))
622 {
623 //
624 // Wow, exact match so replace free block.
625 // Remove from free list.
626 //
627 blockFree->nextFree->prevFree = blockFree->prevFree;
628 blockFree->prevFree->nextFree = blockFree->nextFree;
629 if (pHeap->pFreeBlockList == blockFree)
630 {
631 //
632 // This could be the last free block.
633 //
634 if (blockFree->nextFree == blockFree)
635 pHeap->pFreeBlockList = NULL;
636 else
637 pHeap->pFreeBlockList = blockFree->nextFree;
638 }
639
640 //
641 // Set owner/type values here. Don't move because some fields are unions.
642 //
643 blockFree->owner = owner;
644 blockFree->refCount = 1;
645 blockFree->align = allocAl;
646
647 // tail end code below assumes 'blockFree' is the new block
648 blockNew = blockFree;
649 }
650 else if ((allocLo >= blockFree->begin) && (allocHi <= blockFree->end))
651 {
652 //
653 // Found a fit.
654 // It isn't exact, so we'll have to do a split
655 //
656 if (_eheapAllocMemStruct(pHeap, &blockNew) != NV_OK)
657 {
658 goto failed;
659 }
660
661 blockNew->owner = owner;
662 blockNew->refCount = 1;
663 blockNew->begin = allocLo;
664 blockNew->align = allocAl;
665 blockNew->end = allocHi;
666
667 if ((blockFree->begin < blockNew->begin) && (blockFree->end > blockNew->end))
668 {
669 //
670 // Split free block in two.
671 //
672 if (_eheapAllocMemStruct(pHeap, &blockSplit) != NV_OK)
673 {
674 goto failed;
675 }
676
677 //
678 // Remove free block from rb-tree since node's range will be
679 // changed.
680 //
681 if (btreeUnlink(&blockFree->node, &pHeap->pBlockTree) != NV_OK)
682 {
683 goto failed;
684 }
685
686 blockSplit->owner = NVOS32_BLOCK_TYPE_FREE;
687 blockSplit->refCount = 0;
688 blockSplit->begin = blockNew->end + 1;
689 blockSplit->align = blockSplit->begin;
690 blockSplit->end = blockFree->end;
691 blockSplit->pData = (void*)(blockNew+1);
692 blockFree->end = blockNew->begin - 1;
693 //
694 // Insert free split block into free list.
695 //
696 blockSplit->nextFree = blockFree->nextFree;
697 blockSplit->prevFree = blockFree;
698 blockSplit->nextFree->prevFree = blockSplit;
699 blockFree->nextFree = blockSplit;
700 //
701 // Insert new and split blocks into block list.
702 //
703 blockNew->next = blockSplit;
704 blockNew->prev = blockFree;
705 blockSplit->next = blockFree->next;
706 blockSplit->prev = blockNew;
707 blockFree->next = blockNew;
708 blockSplit->next->prev = blockSplit;
709
710 // update numBlocks count
711 pHeap->numBlocks++;
712
713 // re-insert updated free block into rb-tree
714 blockFree->node.keyEnd = blockFree->end;
715 if (btreeInsert(&blockFree->node, &pHeap->pBlockTree) != NV_OK)
716 {
717 goto failed;
718 }
719
720 // insert new and split blocks into rb-tree
721 portMemSet((void *)&blockNew->node, 0, sizeof(NODE));
722 portMemSet((void *)&blockSplit->node, 0, sizeof(NODE));
723 blockNew->node.keyStart = blockNew->begin;
724 blockNew->node.keyEnd = blockNew->end;
725 blockNew->node.Data = (void *)blockNew;
726 blockSplit->node.keyStart = blockSplit->begin;
727 blockSplit->node.keyEnd = blockSplit->end;
728 blockSplit->node.Data = (void *)blockSplit;
729 if (btreeInsert(&blockNew->node, &pHeap->pBlockTree) != NV_OK)
730 {
731 goto failed;
732 }
733 if (btreeInsert(&blockSplit->node, &pHeap->pBlockTree) != NV_OK)
734 {
735 goto failed;
736 }
737 }
738 else if (blockFree->end == blockNew->end)
739 {
740 //
741 // Remove free block from rb-tree since node's range will be
742 // changed.
743 //
744 if (btreeUnlink(&blockFree->node, &pHeap->pBlockTree) != NV_OK)
745 {
746 goto failed;
747 }
748
749 //
750 // New block inserted after free block.
751 //
752 blockFree->end = blockNew->begin - 1;
753 blockNew->next = blockFree->next;
754 blockNew->prev = blockFree;
755 blockFree->next->prev = blockNew;
756 blockFree->next = blockNew;
757
758 // re-insert updated free block into rb-tree
759 blockFree->node.keyEnd = blockFree->end;
760 if (btreeInsert(&blockFree->node, &pHeap->pBlockTree) != NV_OK)
761 {
762 goto failed;
763 }
764
765 // insert new block into rb-tree
766 portMemSet((void *)&blockNew->node, 0, sizeof(NODE));
767 blockNew->node.keyStart = blockNew->begin;
768 blockNew->node.keyEnd = blockNew->end;
769 blockNew->node.Data = (void *)blockNew;
770 if (btreeInsert(&blockNew->node, &pHeap->pBlockTree) != NV_OK)
771 {
772 goto failed;
773 }
774 }
775 else if (blockFree->begin == blockNew->begin)
776 {
777 //
778 // Remove free block from rb-tree since node's range will be
779 // changed.
780 //
781 if (btreeUnlink(&blockFree->node, &pHeap->pBlockTree) != NV_OK)
782 {
783 goto failed;
784 }
785
786 //
787 // New block inserted before free block.
788 //
789 blockFree->begin = blockNew->end + 1;
790 blockFree->align = blockFree->begin;
791 blockNew->next = blockFree;
792 blockNew->prev = blockFree->prev;
793 blockFree->prev->next = blockNew;
794 blockFree->prev = blockNew;
795 if (pHeap->pBlockList == blockFree)
796 pHeap->pBlockList = blockNew;
797
798 // re-insert updated free block into rb-tree
799 blockFree->node.keyStart = blockFree->begin;
800 if (btreeInsert(&blockFree->node, &pHeap->pBlockTree) != NV_OK)
801 {
802 goto failed;
803 }
804
805 // insert new block into rb-tree
806 portMemSet((void *)&blockNew->node, 0, sizeof(NODE));
807 blockNew->node.keyStart = blockNew->begin;
808 blockNew->node.keyEnd = blockNew->end;
809 blockNew->node.Data = (void *)blockNew;
810 if (btreeInsert(&blockNew->node, &pHeap->pBlockTree) != NV_OK)
811 {
812 goto failed;
813 }
814 }
815 else
816 {
817 failed:
818 if (blockNew) _eheapFreeMemStruct(pHeap, &blockNew);
819 if (blockSplit) _eheapFreeMemStruct(pHeap, &blockSplit);
820 return NV_ERR_NO_MEMORY;
821 }
822
823 pHeap->numBlocks++;
824 }
825
826 NV_ASSERT(blockNew != NULL); // assert is for Coverity
827 pHeap->free -= blockNew->end - blockNew->begin + 1; // Reduce free amount by allocated block size.
828
829 // Initialize a pointer to the outer wrapper's specific control structure, tacked to the end of the EMEMBLOCK
830 blockNew->pData = (void*)(blockNew+1);
831
832 // Return values
833 *size = allocSize;
834 *offset = blockNew->align;
835 if ( ppMemBlock) *ppMemBlock = blockNew;
836
837 return NV_OK;
838 }
839
840 static NV_STATUS
_eheapBlockFree(OBJEHEAP * pHeap,EMEMBLOCK * block)841 _eheapBlockFree
842 (
843 OBJEHEAP *pHeap,
844 EMEMBLOCK *block
845 )
846 {
847 EMEMBLOCK *blockTmp;
848
849 //
850 // Check for valid owner.
851 //
852 if (block->owner == NVOS32_BLOCK_TYPE_FREE) return NV_ERR_INVALID_ARGUMENT;
853
854 //
855 // Check refCount.
856 //
857 if (--block->refCount != 0)
858 return NV_OK;
859
860 //
861 // Update free count.
862 //
863 pHeap->free += block->end - block->begin + 1;
864
865 //
866 //
867 // Can this merge with any surrounding free blocks?
868 //
869 if ((block->prev->owner == NVOS32_BLOCK_TYPE_FREE) && (block != pHeap->pBlockList))
870 {
871 //
872 // Remove block to be freed and previous one since nodes will be
873 // combined into single one.
874 //
875 if (btreeUnlink(&block->node, &pHeap->pBlockTree) != NV_OK)
876 {
877 return NV_ERR_INVALID_OFFSET;
878 }
879 if (btreeUnlink(&block->prev->node, &pHeap->pBlockTree) != NV_OK)
880 {
881 return NV_ERR_INVALID_OFFSET;
882 }
883
884 //
885 // Merge with previous block.
886 //
887 block->prev->next = block->next;
888 block->next->prev = block->prev;
889 block->prev->end = block->end;
890 blockTmp = block;
891 block = block->prev;
892 pHeap->numBlocks--;
893 _eheapFreeMemStruct(pHeap, &blockTmp);
894
895 // re-insert updated free block into rb-tree
896 block->node.keyEnd = block->end;
897 if (btreeInsert(&block->node, &pHeap->pBlockTree) != NV_OK)
898 {
899 return NV_ERR_INVALID_OFFSET;
900 }
901 }
902 if ((block->next->owner == NVOS32_BLOCK_TYPE_FREE) && (block->next != pHeap->pBlockList))
903 {
904 //
905 // Remove block to be freed and next one since nodes will be
906 // combined into single one.
907 //
908 if (btreeUnlink(&block->node, &pHeap->pBlockTree) != NV_OK)
909 {
910 return NV_ERR_INVALID_OFFSET;
911 }
912 if (btreeUnlink(&block->next->node, &pHeap->pBlockTree) != NV_OK)
913 {
914 return NV_ERR_INVALID_OFFSET;
915 }
916
917 //
918 // Merge with next block.
919 //
920 block->prev->next = block->next;
921 block->next->prev = block->prev;
922 block->next->begin = block->begin;
923 if (pHeap->pBlockList == block)
924 pHeap->pBlockList = block->next;
925 if (block->owner == NVOS32_BLOCK_TYPE_FREE)
926 {
927 if (pHeap->pFreeBlockList == block)
928 pHeap->pFreeBlockList = block->nextFree;
929 block->nextFree->prevFree = block->prevFree;
930 block->prevFree->nextFree = block->nextFree;
931 }
932 blockTmp = block;
933 block = block->next;
934 pHeap->numBlocks--;
935 _eheapFreeMemStruct(pHeap, &blockTmp);
936
937 // re-insert updated free block into rb-tree
938 block->node.keyStart = block->begin;
939 if (btreeInsert(&block->node, &pHeap->pBlockTree) != NV_OK)
940 {
941 return NV_ERR_INVALID_OFFSET;
942 }
943 }
944 if (block->owner != NVOS32_BLOCK_TYPE_FREE)
945 {
946 //
947 // Nothing was merged. Add to free list.
948 //
949 blockTmp = pHeap->pFreeBlockList;
950 if (!blockTmp)
951 {
952 pHeap->pFreeBlockList = block;
953 block->nextFree = block;
954 block->prevFree = block;
955 }
956 else
957 {
958 if (blockTmp->begin > block->begin)
959 //
960 // Insert into beginning of free list.
961 //
962 pHeap->pFreeBlockList = block;
963 else if (blockTmp->prevFree->begin > block->begin)
964 //
965 // Insert into free list.
966 //
967 do
968 {
969 blockTmp = blockTmp->nextFree;
970 } while (blockTmp->begin < block->begin);
971 /*
972 else
973 * Insert at end of list.
974 */
975 block->nextFree = blockTmp;
976 block->prevFree = blockTmp->prevFree;
977 block->prevFree->nextFree = block;
978 blockTmp->prevFree = block;
979 }
980 }
981 block->owner = NVOS32_BLOCK_TYPE_FREE;
982 //block->mhandle = 0x0;
983 block->align = block->begin;
984
985 portMemSet((block+1), 0, pHeap->sizeofMemBlock - sizeof(EMEMBLOCK));
986
987 return NV_OK;
988 }
989
990 static NV_STATUS
eheapFree(OBJEHEAP * pHeap,NvU64 offset)991 eheapFree
992 (
993 OBJEHEAP *pHeap,
994 NvU64 offset
995 )
996 {
997 EMEMBLOCK *block;
998
999 block = (EMEMBLOCK *) eheapGetBlock(pHeap, offset, 0);
1000 if (!block)
1001 return NV_ERR_INVALID_OFFSET;
1002
1003 return _eheapBlockFree(pHeap, block);
1004 }
1005
1006 static EMEMBLOCK *
eheapGetBlock(OBJEHEAP * pHeap,NvU64 offset,NvBool bReturnFreeBlock)1007 eheapGetBlock
1008 (
1009 OBJEHEAP *pHeap,
1010 NvU64 offset,
1011 NvBool bReturnFreeBlock
1012 )
1013 {
1014 EMEMBLOCK *block;
1015 PNODE pNode;
1016
1017 if (btreeSearch(offset, &pNode, pHeap->pBlockTree) != NV_OK)
1018 {
1019 return NULL;
1020 }
1021
1022 block = (EMEMBLOCK *)pNode->Data;
1023 if ((block->owner == NVOS32_BLOCK_TYPE_FREE ) && !bReturnFreeBlock)
1024 {
1025 return NULL;
1026 }
1027
1028 return block;
1029 }
1030
1031 static NV_STATUS
eheapGetSize(OBJEHEAP * pHeap,NvU64 * size)1032 eheapGetSize
1033 (
1034 OBJEHEAP *pHeap,
1035 NvU64 *size
1036 )
1037 {
1038 *size = pHeap->total;
1039 return NV_OK;
1040 }
1041
1042 static NV_STATUS
eheapGetFree(OBJEHEAP * pHeap,NvU64 * free)1043 eheapGetFree
1044 (
1045 OBJEHEAP *pHeap,
1046 NvU64 *free
1047 )
1048 {
1049 *free = pHeap->free;
1050 return NV_OK;
1051 }
1052
1053 static NV_STATUS
eheapGetBase(OBJEHEAP * pHeap,NvU64 * base)1054 eheapGetBase
1055 (
1056 OBJEHEAP *pHeap,
1057 NvU64 *base
1058 )
1059 {
1060 *base = pHeap->base;
1061 return NV_OK;
1062 }
1063
1064 static void
eheapInfo(OBJEHEAP * pHeap,NvU64 * pBytesFree,NvU64 * pBytesTotal,NvU64 * pLargestFreeOffset,NvU64 * pLargestFreeSize,NvU32 * pNumFreeBlocks,NvU64 * pUsableBytesFree)1065 eheapInfo
1066 (
1067 OBJEHEAP *pHeap,
1068 NvU64 *pBytesFree, // in all of the space managed
1069 NvU64 *pBytesTotal, // in all of the space managed
1070 NvU64 *pLargestFreeOffset, // constrained to pHeap->rangeLo, pHeap->rangeHi
1071 NvU64 *pLargestFreeSize, // constrained to pHeap->rangeLo, pHeap->rangeHi
1072 NvU32 *pNumFreeBlocks,
1073 NvU64 *pUsableBytesFree // constrained to pHeap->rangeLo, pHeap->rangeHi
1074 )
1075 {
1076 NV_RANGE range = rangeMake(pHeap->rangeLo, pHeap->rangeHi);
1077
1078 if (pBytesFree)
1079 {
1080 *pBytesFree = pHeap->free;
1081 }
1082 if (pBytesTotal)
1083 {
1084 *pBytesTotal = pHeap->total;
1085 }
1086 eheapInfoForRange(pHeap, range, pLargestFreeOffset, pLargestFreeSize, pNumFreeBlocks, pUsableBytesFree);
1087 }
1088
1089 static void
eheapInfoForRange(OBJEHEAP * pHeap,NV_RANGE range,NvU64 * pLargestFreeOffset,NvU64 * pLargestFreeSize,NvU32 * pNumFreeBlocks,NvU64 * pUsableBytesFree)1090 eheapInfoForRange
1091 (
1092 OBJEHEAP *pHeap,
1093 NV_RANGE range,
1094 NvU64 *pLargestFreeOffset, // constrained to rangeLo, rangeHi
1095 NvU64 *pLargestFreeSize, // constrained to rangeLo, rangeHi
1096 NvU32 *pNumFreeBlocks,
1097 NvU64 *pUsableBytesFree // constrained to rangeLo, rangeHi
1098 )
1099 {
1100 EMEMBLOCK *blockFirstFree, *blockFree;
1101 NvU64 freeBlockSize = 0;
1102 NvU64 largestFreeOffset = 0;
1103 NvU64 largestFreeSize = 0;
1104 NvU32 numFreeBlocks = 0;
1105
1106 if (pUsableBytesFree)
1107 *pUsableBytesFree = 0;
1108
1109 blockFirstFree = pHeap->pFreeBlockList;
1110 if (blockFirstFree)
1111 {
1112 NV_ASSERT( range.lo <= range.hi );
1113
1114 blockFree = blockFirstFree;
1115 do {
1116 NvU64 clampedBlockBegin = (blockFree->begin >= range.lo) ?
1117 blockFree->begin : range.lo;
1118 NvU64 clampedBlockEnd = (blockFree->end <= range.hi) ?
1119 blockFree->end : range.hi;
1120 if (clampedBlockBegin <= clampedBlockEnd)
1121 {
1122 numFreeBlocks++;
1123 freeBlockSize = clampedBlockEnd - clampedBlockBegin + 1;
1124
1125 if (pUsableBytesFree)
1126 *pUsableBytesFree += freeBlockSize;
1127
1128 if ( freeBlockSize > largestFreeSize )
1129 {
1130 largestFreeOffset = clampedBlockBegin;
1131 largestFreeSize = freeBlockSize;
1132 }
1133 }
1134 blockFree = blockFree->nextFree;
1135 } while (blockFree != blockFirstFree);
1136 }
1137
1138 if (pLargestFreeOffset)
1139 {
1140 *pLargestFreeOffset = largestFreeOffset;
1141 }
1142 if (pLargestFreeSize)
1143 {
1144 *pLargestFreeSize = largestFreeSize;
1145 }
1146 if (pNumFreeBlocks)
1147 {
1148 *pNumFreeBlocks = numFreeBlocks;
1149 }
1150 }
1151
1152 static NV_STATUS
eheapSetAllocRange(OBJEHEAP * pHeap,NvU64 rangeLo,NvU64 rangeHi)1153 eheapSetAllocRange
1154 (
1155 OBJEHEAP *pHeap,
1156 NvU64 rangeLo,
1157 NvU64 rangeHi
1158 )
1159 {
1160
1161 if ( rangeLo < pHeap->base )
1162 rangeLo = pHeap->base;
1163
1164 if ( rangeHi > (pHeap->base + pHeap->total - 1) )
1165 rangeHi = (pHeap->base + pHeap->total - 1);
1166
1167 if ( rangeHi < rangeLo )
1168 return NV_ERR_INVALID_ARGUMENT;
1169
1170 pHeap->rangeLo = rangeLo;
1171 pHeap->rangeHi = rangeHi;
1172
1173 return NV_OK;
1174 }
1175
1176 static NV_STATUS
eheapTraverse(OBJEHEAP * pHeap,void * pEnv,EHeapTraversalFn traversalFn,NvS32 direction)1177 eheapTraverse
1178 (
1179 OBJEHEAP *pHeap,
1180 void *pEnv,
1181 EHeapTraversalFn traversalFn,
1182 NvS32 direction
1183 )
1184 {
1185 NvU32 cont = 1, backAtFirstBlock = 0;
1186 EMEMBLOCK *pBlock, *pBlockNext;
1187 NV_STATUS rc;
1188 NvU64 cursorOffset; // for dealing with cursor invalidates.
1189 NvU64 firstBlockBegin, firstBlockEnd; // we'll never call the traversal fn twice on the same (sub)extent.
1190
1191 pBlock = (direction > 0) ? pHeap->pBlockList : pHeap->pBlockList->prev;
1192 NV_ASSERT_OR_RETURN(pBlock != NULL, NV_ERR_INVALID_STATE);
1193
1194 //
1195 // Cursor invalidates mean we can't compare with 'pHeap->pBlockList'.
1196 // Instead we'll compare with the extent. If we intersect it at all in
1197 // a later block then we'll consider that as having returned to the first block.
1198 //
1199 firstBlockBegin = pBlock->begin;
1200 firstBlockEnd = pBlock->end;
1201
1202 do
1203 {
1204 NvU32 invalCursor = 0;
1205
1206 if ( direction > 0 )
1207 {
1208 pBlockNext = pBlock->next;
1209 cursorOffset = pBlockNext->begin;
1210 }
1211 else
1212 {
1213 pBlockNext = pBlock->prev;
1214 cursorOffset = pBlockNext->end;
1215 }
1216
1217 rc = traversalFn(pHeap, pEnv, pBlock, &cont, &invalCursor);
1218
1219 if ( invalCursor )
1220 {
1221 // A block was added at or freed. So far only freeing the current block.
1222 pBlock = eheapGetBlock(pHeap, cursorOffset, 1 /*return even if it is a free block*/);
1223
1224 // Advance to the next block if the cursor block was merged.
1225 if ((direction > 0) && (pBlock->begin < cursorOffset))
1226 {
1227 pBlock = pBlock->next;
1228 }
1229 else if ((direction <= 0) && (pBlock->end > cursorOffset))
1230 {
1231 pBlock = pBlock->prev;
1232 }
1233 }
1234 else
1235 {
1236 // No change to the list, use the fast way to find the next block.
1237 pBlock = pBlockNext;
1238
1239 }
1240
1241 NV_ASSERT_OR_RETURN(pBlock != NULL, NV_ERR_INVALID_STATE); // 1. list is circular, 2. cursorOffset should always be found unless the list is badly malformed.
1242
1243 //
1244 // Back to first block? Defined as being at a block for which the
1245 // intersection with the original first block is non-null.
1246 //
1247 if ( ((firstBlockBegin >= pBlock->begin ) && (firstBlockBegin <= pBlock->end)) ||
1248 ((firstBlockEnd <= pBlock->end ) && (firstBlockEnd >= pBlock->begin)) )
1249 {
1250 backAtFirstBlock = 1;
1251 }
1252
1253 } while (cont && !backAtFirstBlock);
1254
1255 return rc;
1256 }
1257
1258 /*!
1259 * @brief returns number of blocks in eHeap.
1260 *
1261 * @param[in] pHeap: pointer to eHeap struct to get data from
1262 *
1263 * @returns the number of blocks (free or allocated) currently in the heap
1264 */
1265 static NvU32
eheapGetNumBlocks(OBJEHEAP * pHeap)1266 eheapGetNumBlocks
1267 (
1268 OBJEHEAP *pHeap
1269 )
1270 {
1271 return pHeap->numBlocks;
1272 }
1273
1274 /*!
1275 * @brief Copies over block information for each block
1276 * in the heap into the provided buffer.
1277 *
1278 * @param[in] pHeap: pointer to eHeap struct to get data from
1279 * @param[in] numBlocks: number of blocks passed in block buffer
1280 * @param[out] pBlockBuffer: pointer to buffer where info will be copied to
1281 *
1282 * @return 'NV_OK' Operation completed successfully
1283 * 'NV_ERR_INVALID_ARGUMENT' size of buffer passed in is
1284 * incorrect
1285 * 'NV_ERR_INVALID_STATE' if the blocklist doesn't match the
1286 * heapSize
1287 */
1288 static NV_STATUS
eheapGetBlockInfo(OBJEHEAP * pHeap,NvU32 numBlocks,NVOS32_HEAP_DUMP_BLOCK * pBlockBuffer)1289 eheapGetBlockInfo
1290 (
1291 OBJEHEAP *pHeap,
1292 NvU32 numBlocks,
1293 NVOS32_HEAP_DUMP_BLOCK *pBlockBuffer
1294 )
1295 {
1296 EMEMBLOCK *pBlock;
1297 NvU32 heapSize, i;
1298 NV_STATUS rmStatus = NV_OK;
1299
1300 // ensure buffer is the same numBlocks
1301 heapSize = eheapGetNumBlocks(pHeap);
1302 NV_ASSERT_OR_RETURN(heapSize == numBlocks, NV_ERR_INVALID_ARGUMENT);
1303
1304 pBlock = pHeap->pBlockList;
1305 for (i = 0; i < heapSize; i++)
1306 {
1307 pBlockBuffer->begin = pBlock->begin;
1308 pBlockBuffer->align = pBlock->align;
1309 pBlockBuffer->end = pBlock->end;
1310 pBlockBuffer->owner = pBlock->owner;
1311 pBlockBuffer->format = 0; // EMEMBLOCK does not have format, ignore for now
1312 pBlock = pBlock->next;
1313 if (pBlock == NULL)
1314 {
1315 return NV_ERR_INVALID_STATE;
1316 }
1317 pBlockBuffer++;
1318 }
1319
1320 return rmStatus;
1321 }
1322
1323 /**
1324 * @brief Set up block owner isolation
1325 *
1326 * Owner isolation means that no two block owners can own allocations which live within a specified range.
1327 *
1328 * @param[in] pHeap pointer to EHEAP object
1329 * @param[in] bEnable NV_TRUE to enable the allocation isolation
1330 * @param[in] granularity allocation granularity
1331 *
1332 * @return NV_OK on success
1333 */
1334 NV_STATUS
eheapSetOwnerIsolation(OBJEHEAP * pHeap,NvBool bEnable,NvU32 granularity)1335 eheapSetOwnerIsolation
1336 (
1337 OBJEHEAP *pHeap,
1338 NvBool bEnable,
1339 NvU32 granularity
1340 )
1341 {
1342 // This can only be set before any allocations have occurred.
1343 if (pHeap->free != pHeap->total)
1344 {
1345 return NV_ERR_INVALID_STATE;
1346 }
1347 // Saying no 2 block owners can share the same block doesn't make sense.
1348 if (bEnable && granularity < 2)
1349 {
1350 return NV_ERR_INVALID_ARGUMENT;
1351 }
1352
1353 if (bEnable && (granularity & (granularity-1)))
1354 {
1355 return NV_ERR_INVALID_ARGUMENT;
1356 }
1357 pHeap->bOwnerIsolation = bEnable;
1358 pHeap->ownerGranularity = granularity;
1359
1360 return NV_OK;
1361 }
1362
1363 /**
1364 * @brief Check heap block ownership
1365 *
1366 * @param[in] pHeap Pointer to EHEAP object
1367 * @param[in] pIsolationID Unique isolation ID constructed by the caller
1368 * @param[in] allocLo Allocated range low
1369 * @param[in] allocHi Allocated range high
1370 * @param[in] blockFree Free block list
1371 * @param[in] pChecker Caller defined ownership ID comparator
1372 *
1373 * @return NV_TRUE if success
1374 */
1375 static NvBool
_eheapCheckOwnership(OBJEHEAP * pHeap,void * pIsolationID,NvU64 allocLo,NvU64 allocHi,EMEMBLOCK * blockFree,EHeapOwnershipComparator * pComparator)1376 _eheapCheckOwnership
1377 (
1378 OBJEHEAP *pHeap,
1379 void *pIsolationID,
1380 NvU64 allocLo,
1381 NvU64 allocHi,
1382 EMEMBLOCK *blockFree,
1383 EHeapOwnershipComparator *pComparator
1384 )
1385 {
1386 EMEMBLOCK *pTmpBlock;
1387 NvU64 checkLo = NV_ALIGN_DOWN(allocLo, pHeap->ownerGranularity);
1388 NvU64 checkHi = (((allocHi % pHeap->ownerGranularity) == 0) ?
1389 NV_ALIGN_UP((allocHi + 1), pHeap->ownerGranularity) :
1390 NV_ALIGN_UP(allocHi, pHeap->ownerGranularity));
1391 NvU64 check;
1392
1393 checkLo = (checkLo <= pHeap->base) ? pHeap->base : checkLo;
1394 checkHi = (checkHi >= pHeap->base + pHeap->total - 1) ? (pHeap->base + pHeap->total - 1) : checkHi;
1395
1396 NV_ASSERT(NULL != blockFree);
1397
1398 if (blockFree->begin > checkLo || blockFree->end < checkHi)
1399 {
1400 for (check = checkLo; check < checkHi; /* in-loop */)
1401 {
1402 pTmpBlock = pHeap->eheapGetBlock(pHeap, check, NV_TRUE);
1403 NV_ASSERT(pTmpBlock);
1404
1405 if (pTmpBlock->owner != NVOS32_BLOCK_TYPE_FREE)
1406 {
1407 if (!pComparator(pIsolationID, pTmpBlock->pData))
1408 {
1409 return NV_FALSE;
1410 }
1411 }
1412
1413 check = pTmpBlock->end + 1;
1414 }
1415 }
1416
1417 return NV_TRUE;
1418 }
1419