1 /*
2  * Copyright (C) 2018-2021 Intel Corporation
3  *
4  * SPDX-License-Identifier: MIT
5  *
6  */
7 
8 #include "shared/source/os_interface/linux/drm_buffer_object.h"
9 
10 #include "shared/source/helpers/aligned_memory.h"
11 #include "shared/source/helpers/debug_helpers.h"
12 #include "shared/source/os_interface/linux/drm_memory_manager.h"
13 #include "shared/source/os_interface/linux/drm_memory_operations_handler.h"
14 #include "shared/source/os_interface/linux/drm_neo.h"
15 #include "shared/source/os_interface/linux/os_context_linux.h"
16 #include "shared/source/os_interface/linux/os_time_linux.h"
17 #include "shared/source/os_interface/os_context.h"
18 #include "shared/source/utilities/stackvec.h"
19 
20 #include "drm/i915_drm.h"
21 
22 #include <errno.h>
23 #include <fcntl.h>
24 #include <map>
25 #include <stdarg.h>
26 #include <string.h>
27 #include <sys/ioctl.h>
28 #include <sys/mman.h>
29 #include <sys/syscall.h>
30 #include <sys/types.h>
31 #include <unistd.h>
32 
33 namespace NEO {
34 
BufferObject(Drm * drm,int handle,size_t size,size_t maxOsContextCount)35 BufferObject::BufferObject(Drm *drm, int handle, size_t size, size_t maxOsContextCount) : drm(drm), refCount(1), handle(handle), size(size), isReused(false) {
36     this->tiling_mode = I915_TILING_NONE;
37     this->lockedAddress = nullptr;
38 
39     perContextVmsUsed = drm->isPerContextVMRequired();
40     requiresExplicitResidency = drm->hasPageFaultSupport();
41 
42     if (perContextVmsUsed) {
43         bindInfo.resize(maxOsContextCount);
44         for (auto &iter : bindInfo) {
45             iter.fill(false);
46         }
47     } else {
48         bindInfo.resize(1);
49         bindInfo[0].fill(false);
50     }
51 }
52 
getRefCount() const53 uint32_t BufferObject::getRefCount() const {
54     return this->refCount.load();
55 }
56 
close()57 bool BufferObject::close() {
58     drm_gem_close close = {};
59     close.handle = this->handle;
60 
61     PRINT_DEBUG_STRING(DebugManager.flags.PrintBOCreateDestroyResult.get(), stdout, "Calling gem close on handle: BO-%d\n", this->handle);
62 
63     int ret = this->drm->ioctl(DRM_IOCTL_GEM_CLOSE, &close);
64     if (ret != 0) {
65         int err = errno;
66         PRINT_DEBUG_STRING(DebugManager.flags.PrintDebugMessages.get(), stderr, "ioctl(GEM_CLOSE) failed with %d. errno=%d(%s)\n", ret, err, strerror(err));
67         DEBUG_BREAK_IF(true);
68         return false;
69     }
70 
71     this->handle = -1;
72 
73     return true;
74 }
75 
wait(int64_t timeoutNs)76 int BufferObject::wait(int64_t timeoutNs) {
77     if (this->drm->isVmBindAvailable()) {
78         return 0;
79     }
80 
81     int ret = this->drm->waitHandle(this->handle, -1);
82     UNRECOVERABLE_IF(ret != 0);
83 
84     return ret;
85 }
86 
setTiling(uint32_t mode,uint32_t stride)87 bool BufferObject::setTiling(uint32_t mode, uint32_t stride) {
88     if (this->tiling_mode == mode) {
89         return true;
90     }
91 
92     drm_i915_gem_set_tiling set_tiling = {};
93     set_tiling.handle = this->handle;
94     set_tiling.tiling_mode = mode;
95     set_tiling.stride = stride;
96 
97     if (this->drm->ioctl(DRM_IOCTL_I915_GEM_SET_TILING, &set_tiling) != 0) {
98         return false;
99     }
100 
101     this->tiling_mode = set_tiling.tiling_mode;
102 
103     return set_tiling.tiling_mode == mode;
104 }
105 
getOsContextId(OsContext * osContext)106 uint32_t BufferObject::getOsContextId(OsContext *osContext) {
107     return perContextVmsUsed ? osContext->getContextId() : 0u;
108 }
109 
fillExecObject(drm_i915_gem_exec_object2 & execObject,OsContext * osContext,uint32_t vmHandleId,uint32_t drmContextId)110 void BufferObject::fillExecObject(drm_i915_gem_exec_object2 &execObject, OsContext *osContext, uint32_t vmHandleId, uint32_t drmContextId) {
111     execObject.handle = this->handle;
112     execObject.relocation_count = 0; //No relocations, we are SoftPinning
113     execObject.relocs_ptr = 0ul;
114     execObject.alignment = 0;
115     execObject.offset = this->gpuAddress;
116     execObject.flags = EXEC_OBJECT_PINNED | EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
117 
118     if (DebugManager.flags.UseAsyncDrmExec.get() == 1) {
119         execObject.flags |= EXEC_OBJECT_ASYNC;
120     }
121 
122     if (this->isMarkedForCapture()) {
123         execObject.flags |= EXEC_OBJECT_CAPTURE;
124     }
125     execObject.rsvd1 = drmContextId;
126     execObject.rsvd2 = 0;
127 
128     this->fillExecObjectImpl(execObject, osContext, vmHandleId);
129 }
130 
exec(uint32_t used,size_t startOffset,unsigned int flags,bool requiresCoherency,OsContext * osContext,uint32_t vmHandleId,uint32_t drmContextId,BufferObject * const residency[],size_t residencyCount,drm_i915_gem_exec_object2 * execObjectsStorage)131 int BufferObject::exec(uint32_t used, size_t startOffset, unsigned int flags, bool requiresCoherency, OsContext *osContext, uint32_t vmHandleId, uint32_t drmContextId, BufferObject *const residency[], size_t residencyCount, drm_i915_gem_exec_object2 *execObjectsStorage) {
132     for (size_t i = 0; i < residencyCount; i++) {
133         residency[i]->fillExecObject(execObjectsStorage[i], osContext, vmHandleId, drmContextId);
134     }
135     this->fillExecObject(execObjectsStorage[residencyCount], osContext, vmHandleId, drmContextId);
136 
137     drm_i915_gem_execbuffer2 execbuf{};
138     execbuf.buffers_ptr = reinterpret_cast<uintptr_t>(execObjectsStorage);
139     execbuf.buffer_count = static_cast<uint32_t>(residencyCount + 1u);
140     execbuf.batch_start_offset = static_cast<uint32_t>(startOffset);
141     execbuf.batch_len = alignUp(used, 8);
142     execbuf.flags = flags;
143     execbuf.rsvd1 = drmContextId;
144 
145     if (DebugManager.flags.PrintExecutionBuffer.get()) {
146         PRINT_DEBUG_STRING(DebugManager.flags.PrintExecutionBuffer.get(), stdout, "Exec called with drmVmId = %u\n",
147                            static_cast<const OsContextLinux *>(osContext)->getDrmVmIds().size() ? static_cast<const OsContextLinux *>(osContext)->getDrmVmIds()[vmHandleId] : 0);
148 
149         printExecutionBuffer(execbuf, residencyCount, execObjectsStorage, residency);
150     }
151 
152     int ret = this->drm->ioctl(DRM_IOCTL_I915_GEM_EXECBUFFER2, &execbuf);
153 
154     if (ret != 0) {
155         int err = this->drm->getErrno();
156         if (err == EOPNOTSUPP) {
157             PRINT_DEBUG_STRING(DebugManager.flags.PrintDebugMessages.get(), stderr, "ioctl(I915_GEM_EXECBUFFER2) failed with %d. errno=%d(%s)\n", ret, err, strerror(err));
158             return err;
159         }
160 
161         static_cast<DrmMemoryOperationsHandler *>(this->drm->getRootDeviceEnvironment().memoryOperationsInterface.get())->evictUnusedAllocations(false, true);
162         ret = this->drm->ioctl(DRM_IOCTL_I915_GEM_EXECBUFFER2, &execbuf);
163     }
164 
165     if (ret != 0) {
166         static_cast<DrmMemoryOperationsHandler *>(this->drm->getRootDeviceEnvironment().memoryOperationsInterface.get())->evictUnusedAllocations(true, true);
167         ret = this->drm->ioctl(DRM_IOCTL_I915_GEM_EXECBUFFER2, &execbuf);
168     }
169 
170     if (ret == 0) {
171         return 0;
172     }
173 
174     int err = this->drm->getErrno();
175     PRINT_DEBUG_STRING(DebugManager.flags.PrintDebugMessages.get(), stderr, "ioctl(I915_GEM_EXECBUFFER2) failed with %d. errno=%d(%s)\n", ret, err, strerror(err));
176     return err;
177 }
178 
bind(OsContext * osContext,uint32_t vmHandleId)179 int BufferObject::bind(OsContext *osContext, uint32_t vmHandleId) {
180     int retVal = 0;
181     auto contextId = getOsContextId(osContext);
182     if (!this->bindInfo[contextId][vmHandleId]) {
183         retVal = this->drm->bindBufferObject(osContext, vmHandleId, this);
184         auto err = this->drm->getErrno();
185 
186         PRINT_DEBUG_STRING(DebugManager.flags.PrintBOBindingResult.get(), stderr, "bind BO-%d to VM %u, drmVmId = %u, range: %llx - %llx, size: %lld, result: %d, errno: %d(%s)\n",
187                            this->handle, vmHandleId, static_cast<const OsContextLinux *>(osContext)->getDrmVmIds().size() ? static_cast<const OsContextLinux *>(osContext)->getDrmVmIds()[vmHandleId] : 0, this->gpuAddress, ptrOffset(this->gpuAddress, this->size), this->size, retVal, err, strerror(err));
188 
189         if (!retVal) {
190             this->bindInfo[contextId][vmHandleId] = true;
191         }
192     }
193     return retVal;
194 }
195 
unbind(OsContext * osContext,uint32_t vmHandleId)196 int BufferObject::unbind(OsContext *osContext, uint32_t vmHandleId) {
197     int retVal = 0;
198     auto contextId = getOsContextId(osContext);
199     if (this->bindInfo[contextId][vmHandleId]) {
200         retVal = this->drm->unbindBufferObject(osContext, vmHandleId, this);
201         auto err = this->drm->getErrno();
202 
203         PRINT_DEBUG_STRING(DebugManager.flags.PrintBOBindingResult.get(), stderr, "unbind BO-%d from VM %u, drmVmId = %u, range: %llx - %llx, size: %lld, result: %d, errno: %d(%s)\n",
204                            this->handle, vmHandleId, static_cast<const OsContextLinux *>(osContext)->getDrmVmIds().size() ? static_cast<const OsContextLinux *>(osContext)->getDrmVmIds()[vmHandleId] : 0, this->gpuAddress, ptrOffset(this->gpuAddress, this->size), this->size, retVal, err, strerror(err));
205 
206         if (!retVal) {
207             this->bindInfo[contextId][vmHandleId] = false;
208         }
209     }
210     return retVal;
211 }
212 
printExecutionBuffer(drm_i915_gem_execbuffer2 & execbuf,const size_t & residencyCount,drm_i915_gem_exec_object2 * execObjectsStorage,BufferObject * const residency[])213 void BufferObject::printExecutionBuffer(drm_i915_gem_execbuffer2 &execbuf, const size_t &residencyCount, drm_i915_gem_exec_object2 *execObjectsStorage, BufferObject *const residency[]) {
214     std::stringstream logger;
215     logger << "drm_i915_gem_execbuffer2 { "
216            << "buffer_ptr: " + std::to_string(execbuf.buffers_ptr)
217            << ", buffer_count: " + std::to_string(execbuf.buffer_count)
218            << ", batch_start_offset: " + std::to_string(execbuf.batch_start_offset)
219            << ", batch_len: " + std::to_string(execbuf.batch_len)
220            << ", flags: " + std::to_string(execbuf.flags)
221            << ", rsvd1: " + std::to_string(execbuf.rsvd1)
222            << " }\n";
223 
224     size_t i;
225     for (i = 0; i < residencyCount; i++) {
226         logger << "Buffer Object = { handle: BO-" << execObjectsStorage[i].handle
227                << ", address range: 0x" << (void *)execObjectsStorage[i].offset
228                << " - 0x" << (void *)ptrOffset(execObjectsStorage[i].offset, residency[i]->peekSize())
229                << ", flags: " << std::hex << execObjectsStorage[i].flags << std::dec
230                << ", size: " << residency[i]->peekSize() << " }\n";
231     }
232     logger << "Command Buffer Object = { handle: BO-" << execObjectsStorage[i].handle
233            << ", address range: 0x" << (void *)execObjectsStorage[i].offset
234            << " - 0x" << (void *)ptrOffset(execObjectsStorage[i].offset, this->peekSize())
235            << ", flags: " << std::hex << execObjectsStorage[i].flags << std::dec
236            << ", size: " << this->peekSize() << " }\n";
237 
238     printf("%s\n", logger.str().c_str());
239 }
240 
bindBOsWithinContext(BufferObject * const boToPin[],size_t numberOfBos,OsContext * osContext,uint32_t vmHandleId)241 int bindBOsWithinContext(BufferObject *const boToPin[], size_t numberOfBos, OsContext *osContext, uint32_t vmHandleId) {
242     auto retVal = 0;
243 
244     for (auto drmIterator = 0u; drmIterator < osContext->getDeviceBitfield().size(); drmIterator++) {
245         if (osContext->getDeviceBitfield().test(drmIterator)) {
246             for (size_t i = 0; i < numberOfBos; i++) {
247                 retVal |= boToPin[i]->bind(osContext, drmIterator);
248             }
249         }
250     }
251 
252     return retVal;
253 }
254 
pin(BufferObject * const boToPin[],size_t numberOfBos,OsContext * osContext,uint32_t vmHandleId,uint32_t drmContextId)255 int BufferObject::pin(BufferObject *const boToPin[], size_t numberOfBos, OsContext *osContext, uint32_t vmHandleId, uint32_t drmContextId) {
256     auto retVal = 0;
257 
258     if (this->drm->isVmBindAvailable()) {
259         retVal = bindBOsWithinContext(boToPin, numberOfBos, osContext, vmHandleId);
260     } else {
261         StackVec<drm_i915_gem_exec_object2, maxFragmentsCount + 1> execObject(numberOfBos + 1);
262         retVal = this->exec(4u, 0u, 0u, false, osContext, vmHandleId, drmContextId, boToPin, numberOfBos, &execObject[0]);
263     }
264 
265     return retVal;
266 }
267 
validateHostPtr(BufferObject * const boToPin[],size_t numberOfBos,OsContext * osContext,uint32_t vmHandleId,uint32_t drmContextId)268 int BufferObject::validateHostPtr(BufferObject *const boToPin[], size_t numberOfBos, OsContext *osContext, uint32_t vmHandleId, uint32_t drmContextId) {
269     auto retVal = 0;
270 
271     if (this->drm->isVmBindAvailable()) {
272         for (size_t i = 0; i < numberOfBos; i++) {
273             retVal = boToPin[i]->bind(osContext, vmHandleId);
274             if (retVal) {
275                 break;
276             }
277         }
278     } else {
279         StackVec<drm_i915_gem_exec_object2, maxFragmentsCount + 1> execObject(numberOfBos + 1);
280         retVal = this->exec(4u, 0u, 0u, false, osContext, vmHandleId, drmContextId, boToPin, numberOfBos, &execObject[0]);
281     }
282 
283     return retVal;
284 }
285 
addBindExtHandle(uint32_t handle)286 void BufferObject::addBindExtHandle(uint32_t handle) {
287     bindExtHandles.push_back(handle);
288 }
289 
290 } // namespace NEO
291