1 /*
2 * Copyright (C) 2018-2021 Intel Corporation
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 */
7
8 #include "shared/source/os_interface/linux/drm_buffer_object.h"
9
10 #include "shared/source/helpers/aligned_memory.h"
11 #include "shared/source/helpers/debug_helpers.h"
12 #include "shared/source/os_interface/linux/drm_memory_manager.h"
13 #include "shared/source/os_interface/linux/drm_memory_operations_handler.h"
14 #include "shared/source/os_interface/linux/drm_neo.h"
15 #include "shared/source/os_interface/linux/os_context_linux.h"
16 #include "shared/source/os_interface/linux/os_time_linux.h"
17 #include "shared/source/os_interface/os_context.h"
18 #include "shared/source/utilities/stackvec.h"
19
20 #include "drm/i915_drm.h"
21
22 #include <errno.h>
23 #include <fcntl.h>
24 #include <map>
25 #include <stdarg.h>
26 #include <string.h>
27 #include <sys/ioctl.h>
28 #include <sys/mman.h>
29 #include <sys/syscall.h>
30 #include <sys/types.h>
31 #include <unistd.h>
32
33 namespace NEO {
34
BufferObject(Drm * drm,int handle,size_t size,size_t maxOsContextCount)35 BufferObject::BufferObject(Drm *drm, int handle, size_t size, size_t maxOsContextCount) : drm(drm), refCount(1), handle(handle), size(size), isReused(false) {
36 this->tiling_mode = I915_TILING_NONE;
37 this->lockedAddress = nullptr;
38
39 perContextVmsUsed = drm->isPerContextVMRequired();
40 requiresExplicitResidency = drm->hasPageFaultSupport();
41
42 if (perContextVmsUsed) {
43 bindInfo.resize(maxOsContextCount);
44 for (auto &iter : bindInfo) {
45 iter.fill(false);
46 }
47 } else {
48 bindInfo.resize(1);
49 bindInfo[0].fill(false);
50 }
51 }
52
getRefCount() const53 uint32_t BufferObject::getRefCount() const {
54 return this->refCount.load();
55 }
56
close()57 bool BufferObject::close() {
58 drm_gem_close close = {};
59 close.handle = this->handle;
60
61 PRINT_DEBUG_STRING(DebugManager.flags.PrintBOCreateDestroyResult.get(), stdout, "Calling gem close on handle: BO-%d\n", this->handle);
62
63 int ret = this->drm->ioctl(DRM_IOCTL_GEM_CLOSE, &close);
64 if (ret != 0) {
65 int err = errno;
66 PRINT_DEBUG_STRING(DebugManager.flags.PrintDebugMessages.get(), stderr, "ioctl(GEM_CLOSE) failed with %d. errno=%d(%s)\n", ret, err, strerror(err));
67 DEBUG_BREAK_IF(true);
68 return false;
69 }
70
71 this->handle = -1;
72
73 return true;
74 }
75
wait(int64_t timeoutNs)76 int BufferObject::wait(int64_t timeoutNs) {
77 if (this->drm->isVmBindAvailable()) {
78 return 0;
79 }
80
81 int ret = this->drm->waitHandle(this->handle, -1);
82 UNRECOVERABLE_IF(ret != 0);
83
84 return ret;
85 }
86
setTiling(uint32_t mode,uint32_t stride)87 bool BufferObject::setTiling(uint32_t mode, uint32_t stride) {
88 if (this->tiling_mode == mode) {
89 return true;
90 }
91
92 drm_i915_gem_set_tiling set_tiling = {};
93 set_tiling.handle = this->handle;
94 set_tiling.tiling_mode = mode;
95 set_tiling.stride = stride;
96
97 if (this->drm->ioctl(DRM_IOCTL_I915_GEM_SET_TILING, &set_tiling) != 0) {
98 return false;
99 }
100
101 this->tiling_mode = set_tiling.tiling_mode;
102
103 return set_tiling.tiling_mode == mode;
104 }
105
getOsContextId(OsContext * osContext)106 uint32_t BufferObject::getOsContextId(OsContext *osContext) {
107 return perContextVmsUsed ? osContext->getContextId() : 0u;
108 }
109
fillExecObject(drm_i915_gem_exec_object2 & execObject,OsContext * osContext,uint32_t vmHandleId,uint32_t drmContextId)110 void BufferObject::fillExecObject(drm_i915_gem_exec_object2 &execObject, OsContext *osContext, uint32_t vmHandleId, uint32_t drmContextId) {
111 execObject.handle = this->handle;
112 execObject.relocation_count = 0; //No relocations, we are SoftPinning
113 execObject.relocs_ptr = 0ul;
114 execObject.alignment = 0;
115 execObject.offset = this->gpuAddress;
116 execObject.flags = EXEC_OBJECT_PINNED | EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
117
118 if (DebugManager.flags.UseAsyncDrmExec.get() == 1) {
119 execObject.flags |= EXEC_OBJECT_ASYNC;
120 }
121
122 if (this->isMarkedForCapture()) {
123 execObject.flags |= EXEC_OBJECT_CAPTURE;
124 }
125 execObject.rsvd1 = drmContextId;
126 execObject.rsvd2 = 0;
127
128 this->fillExecObjectImpl(execObject, osContext, vmHandleId);
129 }
130
exec(uint32_t used,size_t startOffset,unsigned int flags,bool requiresCoherency,OsContext * osContext,uint32_t vmHandleId,uint32_t drmContextId,BufferObject * const residency[],size_t residencyCount,drm_i915_gem_exec_object2 * execObjectsStorage)131 int BufferObject::exec(uint32_t used, size_t startOffset, unsigned int flags, bool requiresCoherency, OsContext *osContext, uint32_t vmHandleId, uint32_t drmContextId, BufferObject *const residency[], size_t residencyCount, drm_i915_gem_exec_object2 *execObjectsStorage) {
132 for (size_t i = 0; i < residencyCount; i++) {
133 residency[i]->fillExecObject(execObjectsStorage[i], osContext, vmHandleId, drmContextId);
134 }
135 this->fillExecObject(execObjectsStorage[residencyCount], osContext, vmHandleId, drmContextId);
136
137 drm_i915_gem_execbuffer2 execbuf{};
138 execbuf.buffers_ptr = reinterpret_cast<uintptr_t>(execObjectsStorage);
139 execbuf.buffer_count = static_cast<uint32_t>(residencyCount + 1u);
140 execbuf.batch_start_offset = static_cast<uint32_t>(startOffset);
141 execbuf.batch_len = alignUp(used, 8);
142 execbuf.flags = flags;
143 execbuf.rsvd1 = drmContextId;
144
145 if (DebugManager.flags.PrintExecutionBuffer.get()) {
146 PRINT_DEBUG_STRING(DebugManager.flags.PrintExecutionBuffer.get(), stdout, "Exec called with drmVmId = %u\n",
147 static_cast<const OsContextLinux *>(osContext)->getDrmVmIds().size() ? static_cast<const OsContextLinux *>(osContext)->getDrmVmIds()[vmHandleId] : 0);
148
149 printExecutionBuffer(execbuf, residencyCount, execObjectsStorage, residency);
150 }
151
152 int ret = this->drm->ioctl(DRM_IOCTL_I915_GEM_EXECBUFFER2, &execbuf);
153
154 if (ret != 0) {
155 int err = this->drm->getErrno();
156 if (err == EOPNOTSUPP) {
157 PRINT_DEBUG_STRING(DebugManager.flags.PrintDebugMessages.get(), stderr, "ioctl(I915_GEM_EXECBUFFER2) failed with %d. errno=%d(%s)\n", ret, err, strerror(err));
158 return err;
159 }
160
161 static_cast<DrmMemoryOperationsHandler *>(this->drm->getRootDeviceEnvironment().memoryOperationsInterface.get())->evictUnusedAllocations(false, true);
162 ret = this->drm->ioctl(DRM_IOCTL_I915_GEM_EXECBUFFER2, &execbuf);
163 }
164
165 if (ret != 0) {
166 static_cast<DrmMemoryOperationsHandler *>(this->drm->getRootDeviceEnvironment().memoryOperationsInterface.get())->evictUnusedAllocations(true, true);
167 ret = this->drm->ioctl(DRM_IOCTL_I915_GEM_EXECBUFFER2, &execbuf);
168 }
169
170 if (ret == 0) {
171 return 0;
172 }
173
174 int err = this->drm->getErrno();
175 PRINT_DEBUG_STRING(DebugManager.flags.PrintDebugMessages.get(), stderr, "ioctl(I915_GEM_EXECBUFFER2) failed with %d. errno=%d(%s)\n", ret, err, strerror(err));
176 return err;
177 }
178
bind(OsContext * osContext,uint32_t vmHandleId)179 int BufferObject::bind(OsContext *osContext, uint32_t vmHandleId) {
180 int retVal = 0;
181 auto contextId = getOsContextId(osContext);
182 if (!this->bindInfo[contextId][vmHandleId]) {
183 retVal = this->drm->bindBufferObject(osContext, vmHandleId, this);
184 auto err = this->drm->getErrno();
185
186 PRINT_DEBUG_STRING(DebugManager.flags.PrintBOBindingResult.get(), stderr, "bind BO-%d to VM %u, drmVmId = %u, range: %llx - %llx, size: %lld, result: %d, errno: %d(%s)\n",
187 this->handle, vmHandleId, static_cast<const OsContextLinux *>(osContext)->getDrmVmIds().size() ? static_cast<const OsContextLinux *>(osContext)->getDrmVmIds()[vmHandleId] : 0, this->gpuAddress, ptrOffset(this->gpuAddress, this->size), this->size, retVal, err, strerror(err));
188
189 if (!retVal) {
190 this->bindInfo[contextId][vmHandleId] = true;
191 }
192 }
193 return retVal;
194 }
195
unbind(OsContext * osContext,uint32_t vmHandleId)196 int BufferObject::unbind(OsContext *osContext, uint32_t vmHandleId) {
197 int retVal = 0;
198 auto contextId = getOsContextId(osContext);
199 if (this->bindInfo[contextId][vmHandleId]) {
200 retVal = this->drm->unbindBufferObject(osContext, vmHandleId, this);
201 auto err = this->drm->getErrno();
202
203 PRINT_DEBUG_STRING(DebugManager.flags.PrintBOBindingResult.get(), stderr, "unbind BO-%d from VM %u, drmVmId = %u, range: %llx - %llx, size: %lld, result: %d, errno: %d(%s)\n",
204 this->handle, vmHandleId, static_cast<const OsContextLinux *>(osContext)->getDrmVmIds().size() ? static_cast<const OsContextLinux *>(osContext)->getDrmVmIds()[vmHandleId] : 0, this->gpuAddress, ptrOffset(this->gpuAddress, this->size), this->size, retVal, err, strerror(err));
205
206 if (!retVal) {
207 this->bindInfo[contextId][vmHandleId] = false;
208 }
209 }
210 return retVal;
211 }
212
printExecutionBuffer(drm_i915_gem_execbuffer2 & execbuf,const size_t & residencyCount,drm_i915_gem_exec_object2 * execObjectsStorage,BufferObject * const residency[])213 void BufferObject::printExecutionBuffer(drm_i915_gem_execbuffer2 &execbuf, const size_t &residencyCount, drm_i915_gem_exec_object2 *execObjectsStorage, BufferObject *const residency[]) {
214 std::stringstream logger;
215 logger << "drm_i915_gem_execbuffer2 { "
216 << "buffer_ptr: " + std::to_string(execbuf.buffers_ptr)
217 << ", buffer_count: " + std::to_string(execbuf.buffer_count)
218 << ", batch_start_offset: " + std::to_string(execbuf.batch_start_offset)
219 << ", batch_len: " + std::to_string(execbuf.batch_len)
220 << ", flags: " + std::to_string(execbuf.flags)
221 << ", rsvd1: " + std::to_string(execbuf.rsvd1)
222 << " }\n";
223
224 size_t i;
225 for (i = 0; i < residencyCount; i++) {
226 logger << "Buffer Object = { handle: BO-" << execObjectsStorage[i].handle
227 << ", address range: 0x" << (void *)execObjectsStorage[i].offset
228 << " - 0x" << (void *)ptrOffset(execObjectsStorage[i].offset, residency[i]->peekSize())
229 << ", flags: " << std::hex << execObjectsStorage[i].flags << std::dec
230 << ", size: " << residency[i]->peekSize() << " }\n";
231 }
232 logger << "Command Buffer Object = { handle: BO-" << execObjectsStorage[i].handle
233 << ", address range: 0x" << (void *)execObjectsStorage[i].offset
234 << " - 0x" << (void *)ptrOffset(execObjectsStorage[i].offset, this->peekSize())
235 << ", flags: " << std::hex << execObjectsStorage[i].flags << std::dec
236 << ", size: " << this->peekSize() << " }\n";
237
238 printf("%s\n", logger.str().c_str());
239 }
240
bindBOsWithinContext(BufferObject * const boToPin[],size_t numberOfBos,OsContext * osContext,uint32_t vmHandleId)241 int bindBOsWithinContext(BufferObject *const boToPin[], size_t numberOfBos, OsContext *osContext, uint32_t vmHandleId) {
242 auto retVal = 0;
243
244 for (auto drmIterator = 0u; drmIterator < osContext->getDeviceBitfield().size(); drmIterator++) {
245 if (osContext->getDeviceBitfield().test(drmIterator)) {
246 for (size_t i = 0; i < numberOfBos; i++) {
247 retVal |= boToPin[i]->bind(osContext, drmIterator);
248 }
249 }
250 }
251
252 return retVal;
253 }
254
pin(BufferObject * const boToPin[],size_t numberOfBos,OsContext * osContext,uint32_t vmHandleId,uint32_t drmContextId)255 int BufferObject::pin(BufferObject *const boToPin[], size_t numberOfBos, OsContext *osContext, uint32_t vmHandleId, uint32_t drmContextId) {
256 auto retVal = 0;
257
258 if (this->drm->isVmBindAvailable()) {
259 retVal = bindBOsWithinContext(boToPin, numberOfBos, osContext, vmHandleId);
260 } else {
261 StackVec<drm_i915_gem_exec_object2, maxFragmentsCount + 1> execObject(numberOfBos + 1);
262 retVal = this->exec(4u, 0u, 0u, false, osContext, vmHandleId, drmContextId, boToPin, numberOfBos, &execObject[0]);
263 }
264
265 return retVal;
266 }
267
validateHostPtr(BufferObject * const boToPin[],size_t numberOfBos,OsContext * osContext,uint32_t vmHandleId,uint32_t drmContextId)268 int BufferObject::validateHostPtr(BufferObject *const boToPin[], size_t numberOfBos, OsContext *osContext, uint32_t vmHandleId, uint32_t drmContextId) {
269 auto retVal = 0;
270
271 if (this->drm->isVmBindAvailable()) {
272 for (size_t i = 0; i < numberOfBos; i++) {
273 retVal = boToPin[i]->bind(osContext, vmHandleId);
274 if (retVal) {
275 break;
276 }
277 }
278 } else {
279 StackVec<drm_i915_gem_exec_object2, maxFragmentsCount + 1> execObject(numberOfBos + 1);
280 retVal = this->exec(4u, 0u, 0u, false, osContext, vmHandleId, drmContextId, boToPin, numberOfBos, &execObject[0]);
281 }
282
283 return retVal;
284 }
285
addBindExtHandle(uint32_t handle)286 void BufferObject::addBindExtHandle(uint32_t handle) {
287 bindExtHandles.push_back(handle);
288 }
289
290 } // namespace NEO
291