1 /*
2  * Copyright (C) 2019-2021 Intel Corporation
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  */
7 
8 #pragma once
9 #include "shared/source/helpers/constants.h"
10 #include "shared/source/helpers/heap_assigner.h"
11 #include "shared/source/os_interface/os_memory.h"
12 #include "shared/source/utilities/heap_allocator.h"
13 
14 #include <array>
15 #include <map>
16 
17 namespace NEO {
18 
19 enum class HeapIndex : uint32_t {
20     HEAP_INTERNAL_DEVICE_MEMORY = 0u,
21     HEAP_INTERNAL = 1u,
22     HEAP_EXTERNAL_DEVICE_MEMORY = 2u,
23     HEAP_EXTERNAL = 3u,
24     HEAP_STANDARD,
25     HEAP_STANDARD64KB,
26     HEAP_STANDARD2MB,
27     HEAP_SVM,
28     HEAP_EXTENDED,
29     HEAP_EXTERNAL_FRONT_WINDOW,
30     HEAP_EXTERNAL_DEVICE_FRONT_WINDOW,
31     HEAP_INTERNAL_FRONT_WINDOW,
32     HEAP_INTERNAL_DEVICE_FRONT_WINDOW,
33 
34     // Please put new heap indexes above this line
35     TOTAL_HEAPS
36 };
37 
38 class GfxPartition {
39   public:
40     GfxPartition(OSMemory::ReservedCpuAddressRange &sharedReservedCpuAddressRange);
41     MOCKABLE_VIRTUAL ~GfxPartition();
42 
init(uint64_t gpuAddressSpace,size_t cpuAddressRangeSizeToReserve,uint32_t rootDeviceIndex,size_t numRootDevices)43     bool init(uint64_t gpuAddressSpace, size_t cpuAddressRangeSizeToReserve, uint32_t rootDeviceIndex, size_t numRootDevices) {
44         return init(gpuAddressSpace, cpuAddressRangeSizeToReserve, rootDeviceIndex, numRootDevices, false);
45     }
46     MOCKABLE_VIRTUAL bool init(uint64_t gpuAddressSpace, size_t cpuAddressRangeSizeToReserve, uint32_t rootDeviceIndex, size_t numRootDevices, bool useExternalFrontWindowPool);
47 
heapInit(HeapIndex heapIndex,uint64_t base,uint64_t size)48     void heapInit(HeapIndex heapIndex, uint64_t base, uint64_t size) {
49         getHeap(heapIndex).init(base, size, MemoryConstants::pageSize);
50     }
51 
heapInitWithAllocationAlignment(HeapIndex heapIndex,uint64_t base,uint64_t size,size_t allocationAlignment)52     void heapInitWithAllocationAlignment(HeapIndex heapIndex, uint64_t base, uint64_t size, size_t allocationAlignment) {
53         getHeap(heapIndex).init(base, size, allocationAlignment);
54     }
55 
heapInitExternalWithFrontWindow(HeapIndex heapIndex,uint64_t base,uint64_t size)56     void heapInitExternalWithFrontWindow(HeapIndex heapIndex, uint64_t base, uint64_t size) {
57         getHeap(heapIndex).initExternalWithFrontWindow(base, size);
58     }
59 
heapInitWithFrontWindow(HeapIndex heapIndex,uint64_t base,uint64_t size,uint64_t frontWindowSize)60     void heapInitWithFrontWindow(HeapIndex heapIndex, uint64_t base, uint64_t size, uint64_t frontWindowSize) {
61         getHeap(heapIndex).initWithFrontWindow(base, size, frontWindowSize);
62     }
63 
heapInitFrontWindow(HeapIndex heapIndex,uint64_t base,uint64_t size)64     void heapInitFrontWindow(HeapIndex heapIndex, uint64_t base, uint64_t size) {
65         getHeap(heapIndex).initFrontWindow(base, size);
66     }
67 
heapAllocate(HeapIndex heapIndex,size_t & size)68     MOCKABLE_VIRTUAL uint64_t heapAllocate(HeapIndex heapIndex, size_t &size) {
69         return getHeap(heapIndex).allocate(size);
70     }
71 
heapAllocateWithCustomAlignment(HeapIndex heapIndex,size_t & size,size_t alignment)72     uint64_t heapAllocateWithCustomAlignment(HeapIndex heapIndex, size_t &size, size_t alignment) {
73         return getHeap(heapIndex).allocateWithCustomAlignment(size, alignment);
74     }
75 
heapFree(HeapIndex heapIndex,uint64_t ptr,size_t size)76     MOCKABLE_VIRTUAL void heapFree(HeapIndex heapIndex, uint64_t ptr, size_t size) {
77         getHeap(heapIndex).free(ptr, size);
78     }
79 
80     MOCKABLE_VIRTUAL void freeGpuAddressRange(uint64_t ptr, size_t size);
81 
getHeapBase(HeapIndex heapIndex)82     uint64_t getHeapBase(HeapIndex heapIndex) {
83         return getHeap(heapIndex).getBase();
84     }
85 
getHeapLimit(HeapIndex heapIndex)86     uint64_t getHeapLimit(HeapIndex heapIndex) {
87         return getHeap(heapIndex).getLimit();
88     }
89 
getHeapMinimalAddress(HeapIndex heapIndex)90     uint64_t getHeapMinimalAddress(HeapIndex heapIndex) {
91         if (heapIndex == HeapIndex::HEAP_SVM ||
92             heapIndex == HeapIndex::HEAP_EXTERNAL_DEVICE_FRONT_WINDOW ||
93             heapIndex == HeapIndex::HEAP_EXTERNAL_FRONT_WINDOW ||
94             heapIndex == HeapIndex::HEAP_INTERNAL_DEVICE_FRONT_WINDOW ||
95             heapIndex == HeapIndex::HEAP_INTERNAL_FRONT_WINDOW) {
96             return getHeapBase(heapIndex);
97         } else {
98             if ((heapIndex == HeapIndex::HEAP_EXTERNAL ||
99                  heapIndex == HeapIndex::HEAP_EXTERNAL_DEVICE_MEMORY) &&
100                 (getHeapLimit(HeapAssigner::mapExternalWindowIndex(heapIndex)) != 0)) {
101                 return getHeapBase(heapIndex) + GfxPartition::externalFrontWindowPoolSize;
102             } else if (heapIndex == HeapIndex::HEAP_INTERNAL ||
103                        heapIndex == HeapIndex::HEAP_INTERNAL_DEVICE_MEMORY) {
104                 return getHeapBase(heapIndex) + GfxPartition::internalFrontWindowPoolSize;
105             } else if (heapIndex == HeapIndex::HEAP_STANDARD2MB) {
106                 return getHeapBase(heapIndex) + GfxPartition::heapGranularity2MB;
107             }
108             return getHeapBase(heapIndex) + GfxPartition::heapGranularity;
109         }
110     }
111 
isLimitedRange()112     bool isLimitedRange() { return getHeap(HeapIndex::HEAP_SVM).getSize() == 0ull; }
113 
114     static constexpr uint64_t heapGranularity = MemoryConstants::pageSize64k;
115     static constexpr uint64_t heapGranularity2MB = 2 * MemoryConstants::megaByte;
116     static constexpr size_t externalFrontWindowPoolSize = 16 * MemoryConstants::megaByte;
117     static constexpr size_t internalFrontWindowPoolSize = 1 * MemoryConstants::megaByte;
118 
119     static const std::array<HeapIndex, 4> heap32Names;
120     static const std::array<HeapIndex, 8> heapNonSvmNames;
121 
122   protected:
123     bool initAdditionalRange(uint32_t cpuAddressWidth, uint64_t gpuAddressSpace, uint64_t &gfxBase, uint64_t &gfxTop, uint32_t rootDeviceIndex, size_t numRootDevices);
124 
125     class Heap {
126       public:
127         Heap() = default;
128         void init(uint64_t base, uint64_t size, size_t allocationAlignment);
129         void initExternalWithFrontWindow(uint64_t base, uint64_t size);
130         void initWithFrontWindow(uint64_t base, uint64_t size, uint64_t frontWindowSize);
131         void initFrontWindow(uint64_t base, uint64_t size);
getBase()132         uint64_t getBase() const { return base; }
getSize()133         uint64_t getSize() const { return size; }
getLimit()134         uint64_t getLimit() const { return size ? base + size - 1 : 0; }
allocate(size_t & size)135         uint64_t allocate(size_t &size) { return alloc->allocate(size); }
allocateWithCustomAlignment(size_t & sizeToAllocate,size_t alignment)136         uint64_t allocateWithCustomAlignment(size_t &sizeToAllocate, size_t alignment) { return alloc->allocateWithCustomAlignment(sizeToAllocate, alignment); }
free(uint64_t ptr,size_t size)137         void free(uint64_t ptr, size_t size) { alloc->free(ptr, size); }
138 
139       protected:
140         uint64_t base = 0, size = 0;
141         std::unique_ptr<HeapAllocator> alloc;
142     };
143 
getHeap(HeapIndex heapIndex)144     Heap &getHeap(HeapIndex heapIndex) {
145         return heaps[static_cast<uint32_t>(heapIndex)];
146     }
147 
148     std::array<Heap, static_cast<uint32_t>(HeapIndex::TOTAL_HEAPS)> heaps;
149 
150     OSMemory::ReservedCpuAddressRange &reservedCpuAddressRange;
151     std::unique_ptr<OSMemory> osMemory;
152 };
153 
154 } // namespace NEO
155