1/*
2 * Copyright (C) 2021 Intel Corporation
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 */
7
8#include "shared/source/utilities/tag_allocator.h"
9
10namespace NEO {
11template <typename TagType>
12TagAllocator<TagType>::TagAllocator(const std::vector<uint32_t> &rootDeviceIndices, MemoryManager *memMngr, size_t tagCount, size_t tagAlignment,
13                                    size_t tagSize, bool doNotReleaseNodes, DeviceBitfield deviceBitfield)
14    : TagAllocatorBase(rootDeviceIndices, memMngr, tagCount, tagAlignment, tagSize, doNotReleaseNodes, deviceBitfield) {
15
16    populateFreeTags();
17}
18
19template <typename TagType>
20TagNodeBase *TagAllocator<TagType>::getTag() {
21    if (freeTags.peekIsEmpty()) {
22        releaseDeferredTags();
23    }
24    auto node = freeTags.removeFrontOne().release();
25    if (!node) {
26        std::unique_lock<std::mutex> lock(allocatorMutex);
27        populateFreeTags();
28        node = freeTags.removeFrontOne().release();
29    }
30    usedTags.pushFrontOne(*node);
31    node->incRefCount();
32    node->initialize();
33    return node;
34}
35
36template <typename TagType>
37void TagAllocator<TagType>::returnTagToFreePool(TagNodeBase *node) {
38    auto nodeT = static_cast<NodeType *>(node);
39    [[maybe_unused]] auto usedNode = usedTags.removeOne(*nodeT).release();
40    DEBUG_BREAK_IF(usedNode == nullptr);
41
42    freeTags.pushFrontOne(*nodeT);
43}
44
45template <typename TagType>
46void TagAllocator<TagType>::returnTagToDeferredPool(TagNodeBase *node) {
47    auto nodeT = static_cast<NodeType *>(node);
48    auto usedNode = usedTags.removeOne(*nodeT).release();
49    DEBUG_BREAK_IF(!usedNode);
50    deferredTags.pushFrontOne(*usedNode);
51}
52
53template <typename TagType>
54void TagAllocator<TagType>::releaseDeferredTags() {
55    IDList<NodeType, false> pendingFreeTags;
56    IDList<NodeType, false> pendingDeferredTags;
57    auto currentNode = deferredTags.detachNodes();
58
59    while (currentNode != nullptr) {
60        auto nextNode = currentNode->next;
61        if (currentNode->canBeReleased()) {
62            pendingFreeTags.pushFrontOne(*currentNode);
63        } else {
64            pendingDeferredTags.pushFrontOne(*currentNode);
65        }
66        currentNode = nextNode;
67    }
68
69    if (!pendingFreeTags.peekIsEmpty()) {
70        freeTags.splice(*pendingFreeTags.detachNodes());
71    }
72    if (!pendingDeferredTags.peekIsEmpty()) {
73        deferredTags.splice(*pendingDeferredTags.detachNodes());
74    }
75}
76
77template <typename TagType>
78void TagAllocator<TagType>::populateFreeTags() {
79    size_t allocationSizeRequired = tagCount * tagSize;
80
81    void *baseCpuAddress = nullptr;
82    uint64_t baseGpuAddress = 0;
83
84    auto multiGraphicsAllocation = new MultiGraphicsAllocation(maxRootDeviceIndex);
85    AllocationProperties allocationProperties{rootDeviceIndices[0], allocationSizeRequired, TagType::getAllocationType(), deviceBitfield};
86
87    if (rootDeviceIndices.size() == 1) {
88        GraphicsAllocation *graphicsAllocation = memoryManager->allocateGraphicsMemoryWithProperties(allocationProperties);
89
90        baseCpuAddress = graphicsAllocation->getUnderlyingBuffer();
91        baseGpuAddress = graphicsAllocation->getGpuAddress();
92
93        multiGraphicsAllocation->addAllocation(graphicsAllocation);
94    } else {
95        allocationProperties.subDevicesBitfield = systemMemoryBitfield;
96
97        baseCpuAddress = memoryManager->createMultiGraphicsAllocationInSystemMemoryPool(rootDeviceIndices, allocationProperties, *multiGraphicsAllocation);
98        baseGpuAddress = castToUint64(baseCpuAddress);
99    }
100
101    gfxAllocations.emplace_back(multiGraphicsAllocation);
102
103    auto nodesMemory = std::make_unique<NodeType[]>(tagCount);
104
105    for (size_t i = 0; i < tagCount; ++i) {
106        auto tagOffset = i * tagSize;
107
108        nodesMemory[i].allocator = this;
109        nodesMemory[i].gfxAllocation = multiGraphicsAllocation;
110        nodesMemory[i].tagForCpuAccess = reinterpret_cast<TagType *>(ptrOffset(baseCpuAddress, tagOffset));
111        nodesMemory[i].gpuAddress = baseGpuAddress + tagOffset;
112        nodesMemory[i].setDoNotReleaseNodes(doNotReleaseNodes);
113
114        freeTags.pushTailOne(nodesMemory[i]);
115    }
116
117    tagPoolMemory.push_back(std::move(nodesMemory));
118}
119
120template <typename TagType>
121void TagAllocator<TagType>::returnTag(TagNodeBase *node) {
122    if (node->refCountFetchSub(1) == 1) {
123        if (node->canBeReleased()) {
124            returnTagToFreePool(node);
125        } else {
126            returnTagToDeferredPool(node);
127        }
128    }
129}
130
131template <typename TagType>
132size_t TagNode<TagType>::getGlobalStartOffset() const {
133    if constexpr (TagType::getTagNodeType() == TagNodeType::TimestampPacket) {
134        return TagType::getGlobalStartOffset();
135    } else {
136        UNRECOVERABLE_IF(true);
137    }
138}
139
140template <typename TagType>
141size_t TagNode<TagType>::getContextStartOffset() const {
142    if constexpr (TagType::getTagNodeType() == TagNodeType::TimestampPacket) {
143        return TagType::getContextStartOffset();
144    } else {
145        UNRECOVERABLE_IF(true);
146    }
147}
148
149template <typename TagType>
150size_t TagNode<TagType>::getContextEndOffset() const {
151    if constexpr (TagType::getTagNodeType() == TagNodeType::TimestampPacket) {
152        return TagType::getContextEndOffset();
153    } else {
154        UNRECOVERABLE_IF(true);
155    }
156}
157
158template <typename TagType>
159size_t TagNode<TagType>::getGlobalEndOffset() const {
160    if constexpr (TagType::getTagNodeType() == TagNodeType::TimestampPacket) {
161        return TagType::getGlobalEndOffset();
162    } else {
163        UNRECOVERABLE_IF(true);
164    }
165}
166
167template <typename TagType>
168uint64_t TagNode<TagType>::getContextStartValue([[maybe_unused]] uint32_t packetIndex) const {
169    if constexpr (TagType::getTagNodeType() != TagNodeType::HwPerfCounter) {
170        return tagForCpuAccess->getContextStartValue(packetIndex);
171    } else {
172        UNRECOVERABLE_IF(true);
173    }
174}
175
176template <typename TagType>
177uint64_t TagNode<TagType>::getGlobalStartValue([[maybe_unused]] uint32_t packetIndex) const {
178    if constexpr (TagType::getTagNodeType() != TagNodeType::HwPerfCounter) {
179        return tagForCpuAccess->getGlobalStartValue(packetIndex);
180    } else {
181        UNRECOVERABLE_IF(true);
182    }
183}
184
185template <typename TagType>
186uint64_t TagNode<TagType>::getContextEndValue([[maybe_unused]] uint32_t packetIndex) const {
187    if constexpr (TagType::getTagNodeType() != TagNodeType::HwPerfCounter) {
188        return tagForCpuAccess->getContextEndValue(packetIndex);
189    } else {
190        UNRECOVERABLE_IF(true);
191    }
192}
193
194template <typename TagType>
195uint64_t TagNode<TagType>::getGlobalEndValue([[maybe_unused]] uint32_t packetIndex) const {
196    if constexpr (TagType::getTagNodeType() != TagNodeType::HwPerfCounter) {
197        return tagForCpuAccess->getGlobalEndValue(packetIndex);
198    } else {
199        UNRECOVERABLE_IF(true);
200    }
201}
202
203template <typename TagType>
204void const *TagNode<TagType>::getContextEndAddress([[maybe_unused]] uint32_t packetIndex) const {
205    if constexpr (TagType::getTagNodeType() == TagNodeType::TimestampPacket) {
206        return tagForCpuAccess->getContextEndAddress(packetIndex);
207    } else {
208        UNRECOVERABLE_IF(true);
209    }
210}
211
212template <typename TagType>
213uint64_t &TagNode<TagType>::getContextCompleteRef() const {
214    if constexpr (TagType::getTagNodeType() == TagNodeType::HwTimeStamps) {
215        return tagForCpuAccess->ContextCompleteTS;
216    } else {
217        UNRECOVERABLE_IF(true);
218    }
219}
220
221template <typename TagType>
222uint64_t &TagNode<TagType>::getGlobalEndRef() const {
223    if constexpr (TagType::getTagNodeType() == TagNodeType::HwTimeStamps) {
224        return tagForCpuAccess->GlobalEndTS;
225    } else {
226        UNRECOVERABLE_IF(true);
227    }
228}
229
230template <typename TagType>
231size_t TagNode<TagType>::getSinglePacketSize() const {
232    if constexpr (TagType::getTagNodeType() == TagNodeType::TimestampPacket) {
233        return TagType::getSinglePacketSize();
234    } else {
235        UNRECOVERABLE_IF(true);
236    }
237}
238
239template <typename TagType>
240void TagNode<TagType>::assignDataToAllTimestamps([[maybe_unused]] uint32_t packetIndex, [[maybe_unused]] void *source) {
241    if constexpr (TagType::getTagNodeType() == TagNodeType::TimestampPacket) {
242        return tagForCpuAccess->assignDataToAllTimestamps(packetIndex, source);
243    } else {
244        UNRECOVERABLE_IF(true);
245    }
246}
247
248template <typename TagType>
249MetricsLibraryApi::QueryHandle_1_0 &TagNode<TagType>::getQueryHandleRef() const {
250    if constexpr (TagType::getTagNodeType() == TagNodeType::HwPerfCounter) {
251        return tagForCpuAccess->query.handle;
252    } else {
253        UNRECOVERABLE_IF(true);
254    }
255}
256
257} // namespace NEO
258