1/* 2 * Copyright (C) 2021 Intel Corporation 3 * 4 * SPDX-License-Identifier: MIT 5 * 6 */ 7 8#include "shared/source/utilities/tag_allocator.h" 9 10namespace NEO { 11template <typename TagType> 12TagAllocator<TagType>::TagAllocator(const std::vector<uint32_t> &rootDeviceIndices, MemoryManager *memMngr, size_t tagCount, size_t tagAlignment, 13 size_t tagSize, bool doNotReleaseNodes, DeviceBitfield deviceBitfield) 14 : TagAllocatorBase(rootDeviceIndices, memMngr, tagCount, tagAlignment, tagSize, doNotReleaseNodes, deviceBitfield) { 15 16 populateFreeTags(); 17} 18 19template <typename TagType> 20TagNodeBase *TagAllocator<TagType>::getTag() { 21 if (freeTags.peekIsEmpty()) { 22 releaseDeferredTags(); 23 } 24 auto node = freeTags.removeFrontOne().release(); 25 if (!node) { 26 std::unique_lock<std::mutex> lock(allocatorMutex); 27 populateFreeTags(); 28 node = freeTags.removeFrontOne().release(); 29 } 30 usedTags.pushFrontOne(*node); 31 node->incRefCount(); 32 node->initialize(); 33 return node; 34} 35 36template <typename TagType> 37void TagAllocator<TagType>::returnTagToFreePool(TagNodeBase *node) { 38 auto nodeT = static_cast<NodeType *>(node); 39 [[maybe_unused]] auto usedNode = usedTags.removeOne(*nodeT).release(); 40 DEBUG_BREAK_IF(usedNode == nullptr); 41 42 freeTags.pushFrontOne(*nodeT); 43} 44 45template <typename TagType> 46void TagAllocator<TagType>::returnTagToDeferredPool(TagNodeBase *node) { 47 auto nodeT = static_cast<NodeType *>(node); 48 auto usedNode = usedTags.removeOne(*nodeT).release(); 49 DEBUG_BREAK_IF(!usedNode); 50 deferredTags.pushFrontOne(*usedNode); 51} 52 53template <typename TagType> 54void TagAllocator<TagType>::releaseDeferredTags() { 55 IDList<NodeType, false> pendingFreeTags; 56 IDList<NodeType, false> pendingDeferredTags; 57 auto currentNode = deferredTags.detachNodes(); 58 59 while (currentNode != nullptr) { 60 auto nextNode = currentNode->next; 61 if (currentNode->canBeReleased()) { 62 pendingFreeTags.pushFrontOne(*currentNode); 63 } else { 64 pendingDeferredTags.pushFrontOne(*currentNode); 65 } 66 currentNode = nextNode; 67 } 68 69 if (!pendingFreeTags.peekIsEmpty()) { 70 freeTags.splice(*pendingFreeTags.detachNodes()); 71 } 72 if (!pendingDeferredTags.peekIsEmpty()) { 73 deferredTags.splice(*pendingDeferredTags.detachNodes()); 74 } 75} 76 77template <typename TagType> 78void TagAllocator<TagType>::populateFreeTags() { 79 size_t allocationSizeRequired = tagCount * tagSize; 80 81 void *baseCpuAddress = nullptr; 82 uint64_t baseGpuAddress = 0; 83 84 auto multiGraphicsAllocation = new MultiGraphicsAllocation(maxRootDeviceIndex); 85 AllocationProperties allocationProperties{rootDeviceIndices[0], allocationSizeRequired, TagType::getAllocationType(), deviceBitfield}; 86 87 if (rootDeviceIndices.size() == 1) { 88 GraphicsAllocation *graphicsAllocation = memoryManager->allocateGraphicsMemoryWithProperties(allocationProperties); 89 90 baseCpuAddress = graphicsAllocation->getUnderlyingBuffer(); 91 baseGpuAddress = graphicsAllocation->getGpuAddress(); 92 93 multiGraphicsAllocation->addAllocation(graphicsAllocation); 94 } else { 95 allocationProperties.subDevicesBitfield = systemMemoryBitfield; 96 97 baseCpuAddress = memoryManager->createMultiGraphicsAllocationInSystemMemoryPool(rootDeviceIndices, allocationProperties, *multiGraphicsAllocation); 98 baseGpuAddress = castToUint64(baseCpuAddress); 99 } 100 101 gfxAllocations.emplace_back(multiGraphicsAllocation); 102 103 auto nodesMemory = std::make_unique<NodeType[]>(tagCount); 104 105 for (size_t i = 0; i < tagCount; ++i) { 106 auto tagOffset = i * tagSize; 107 108 nodesMemory[i].allocator = this; 109 nodesMemory[i].gfxAllocation = multiGraphicsAllocation; 110 nodesMemory[i].tagForCpuAccess = reinterpret_cast<TagType *>(ptrOffset(baseCpuAddress, tagOffset)); 111 nodesMemory[i].gpuAddress = baseGpuAddress + tagOffset; 112 nodesMemory[i].setDoNotReleaseNodes(doNotReleaseNodes); 113 114 freeTags.pushTailOne(nodesMemory[i]); 115 } 116 117 tagPoolMemory.push_back(std::move(nodesMemory)); 118} 119 120template <typename TagType> 121void TagAllocator<TagType>::returnTag(TagNodeBase *node) { 122 if (node->refCountFetchSub(1) == 1) { 123 if (node->canBeReleased()) { 124 returnTagToFreePool(node); 125 } else { 126 returnTagToDeferredPool(node); 127 } 128 } 129} 130 131template <typename TagType> 132size_t TagNode<TagType>::getGlobalStartOffset() const { 133 if constexpr (TagType::getTagNodeType() == TagNodeType::TimestampPacket) { 134 return TagType::getGlobalStartOffset(); 135 } else { 136 UNRECOVERABLE_IF(true); 137 } 138} 139 140template <typename TagType> 141size_t TagNode<TagType>::getContextStartOffset() const { 142 if constexpr (TagType::getTagNodeType() == TagNodeType::TimestampPacket) { 143 return TagType::getContextStartOffset(); 144 } else { 145 UNRECOVERABLE_IF(true); 146 } 147} 148 149template <typename TagType> 150size_t TagNode<TagType>::getContextEndOffset() const { 151 if constexpr (TagType::getTagNodeType() == TagNodeType::TimestampPacket) { 152 return TagType::getContextEndOffset(); 153 } else { 154 UNRECOVERABLE_IF(true); 155 } 156} 157 158template <typename TagType> 159size_t TagNode<TagType>::getGlobalEndOffset() const { 160 if constexpr (TagType::getTagNodeType() == TagNodeType::TimestampPacket) { 161 return TagType::getGlobalEndOffset(); 162 } else { 163 UNRECOVERABLE_IF(true); 164 } 165} 166 167template <typename TagType> 168uint64_t TagNode<TagType>::getContextStartValue([[maybe_unused]] uint32_t packetIndex) const { 169 if constexpr (TagType::getTagNodeType() != TagNodeType::HwPerfCounter) { 170 return tagForCpuAccess->getContextStartValue(packetIndex); 171 } else { 172 UNRECOVERABLE_IF(true); 173 } 174} 175 176template <typename TagType> 177uint64_t TagNode<TagType>::getGlobalStartValue([[maybe_unused]] uint32_t packetIndex) const { 178 if constexpr (TagType::getTagNodeType() != TagNodeType::HwPerfCounter) { 179 return tagForCpuAccess->getGlobalStartValue(packetIndex); 180 } else { 181 UNRECOVERABLE_IF(true); 182 } 183} 184 185template <typename TagType> 186uint64_t TagNode<TagType>::getContextEndValue([[maybe_unused]] uint32_t packetIndex) const { 187 if constexpr (TagType::getTagNodeType() != TagNodeType::HwPerfCounter) { 188 return tagForCpuAccess->getContextEndValue(packetIndex); 189 } else { 190 UNRECOVERABLE_IF(true); 191 } 192} 193 194template <typename TagType> 195uint64_t TagNode<TagType>::getGlobalEndValue([[maybe_unused]] uint32_t packetIndex) const { 196 if constexpr (TagType::getTagNodeType() != TagNodeType::HwPerfCounter) { 197 return tagForCpuAccess->getGlobalEndValue(packetIndex); 198 } else { 199 UNRECOVERABLE_IF(true); 200 } 201} 202 203template <typename TagType> 204void const *TagNode<TagType>::getContextEndAddress([[maybe_unused]] uint32_t packetIndex) const { 205 if constexpr (TagType::getTagNodeType() == TagNodeType::TimestampPacket) { 206 return tagForCpuAccess->getContextEndAddress(packetIndex); 207 } else { 208 UNRECOVERABLE_IF(true); 209 } 210} 211 212template <typename TagType> 213uint64_t &TagNode<TagType>::getContextCompleteRef() const { 214 if constexpr (TagType::getTagNodeType() == TagNodeType::HwTimeStamps) { 215 return tagForCpuAccess->ContextCompleteTS; 216 } else { 217 UNRECOVERABLE_IF(true); 218 } 219} 220 221template <typename TagType> 222uint64_t &TagNode<TagType>::getGlobalEndRef() const { 223 if constexpr (TagType::getTagNodeType() == TagNodeType::HwTimeStamps) { 224 return tagForCpuAccess->GlobalEndTS; 225 } else { 226 UNRECOVERABLE_IF(true); 227 } 228} 229 230template <typename TagType> 231size_t TagNode<TagType>::getSinglePacketSize() const { 232 if constexpr (TagType::getTagNodeType() == TagNodeType::TimestampPacket) { 233 return TagType::getSinglePacketSize(); 234 } else { 235 UNRECOVERABLE_IF(true); 236 } 237} 238 239template <typename TagType> 240void TagNode<TagType>::assignDataToAllTimestamps([[maybe_unused]] uint32_t packetIndex, [[maybe_unused]] void *source) { 241 if constexpr (TagType::getTagNodeType() == TagNodeType::TimestampPacket) { 242 return tagForCpuAccess->assignDataToAllTimestamps(packetIndex, source); 243 } else { 244 UNRECOVERABLE_IF(true); 245 } 246} 247 248template <typename TagType> 249MetricsLibraryApi::QueryHandle_1_0 &TagNode<TagType>::getQueryHandleRef() const { 250 if constexpr (TagType::getTagNodeType() == TagNodeType::HwPerfCounter) { 251 return tagForCpuAccess->query.handle; 252 } else { 253 UNRECOVERABLE_IF(true); 254 } 255} 256 257} // namespace NEO 258