1 /*
2 * Copyright (C) 2020-2021 Intel Corporation
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 */
7
8 #pragma once
9
10 #include "shared/source/helpers/debug_helpers.h"
11 #include "shared/source/kernel/kernel_arg_metadata.h"
12 #include "shared/source/utilities/arrayref.h"
13 #include "shared/source/utilities/stackvec.h"
14
15 #include <cinttypes>
16 #include <cstddef>
17 #include <limits>
18
19 namespace NEO {
20
21 using CrossThreadDataOffset = uint16_t;
22 using DynamicStateHeapOffset = uint16_t;
23 using SurfaceStateHeapOffset = uint16_t;
24
25 template <typename T>
26 static constexpr T undefined = std::numeric_limits<T>::max();
27
28 template <typename T>
isUndefinedOffset(T offset)29 bool isUndefinedOffset(T offset) {
30 return undefined<T> == offset;
31 }
32
33 template <typename T>
isValidOffset(T offset)34 bool isValidOffset(T offset) {
35 return false == isUndefinedOffset<T>(offset);
36 }
37
38 struct ArgDescPointer final {
39 SurfaceStateHeapOffset bindful = undefined<SurfaceStateHeapOffset>;
40 CrossThreadDataOffset stateless = undefined<CrossThreadDataOffset>;
41 CrossThreadDataOffset bindless = undefined<CrossThreadDataOffset>;
42 CrossThreadDataOffset bufferOffset = undefined<CrossThreadDataOffset>;
43 CrossThreadDataOffset slmOffset = undefined<CrossThreadDataOffset>;
44 uint8_t requiredSlmAlignment = 0;
45 uint8_t pointerSize = 0;
46 bool accessedUsingStatelessAddressingMode = true;
47
isPureStatefulfinal48 bool isPureStateful() const {
49 return false == accessedUsingStatelessAddressingMode;
50 }
51 };
52
53 struct ArgDescImage final {
54 SurfaceStateHeapOffset bindful = undefined<SurfaceStateHeapOffset>; // stateful with BTI
55 CrossThreadDataOffset bindless = undefined<CrossThreadDataOffset>;
56 struct {
57 CrossThreadDataOffset imgWidth = undefined<CrossThreadDataOffset>;
58 CrossThreadDataOffset imgHeight = undefined<CrossThreadDataOffset>;
59 CrossThreadDataOffset imgDepth = undefined<CrossThreadDataOffset>;
60 CrossThreadDataOffset channelDataType = undefined<CrossThreadDataOffset>;
61 CrossThreadDataOffset channelOrder = undefined<CrossThreadDataOffset>;
62
63 CrossThreadDataOffset arraySize = undefined<CrossThreadDataOffset>;
64 CrossThreadDataOffset numSamples = undefined<CrossThreadDataOffset>;
65 CrossThreadDataOffset numMipLevels = undefined<CrossThreadDataOffset>;
66
67 CrossThreadDataOffset flatBaseOffset = undefined<CrossThreadDataOffset>;
68 CrossThreadDataOffset flatWidth = undefined<CrossThreadDataOffset>;
69 CrossThreadDataOffset flatHeight = undefined<CrossThreadDataOffset>;
70 CrossThreadDataOffset flatPitch = undefined<CrossThreadDataOffset>;
71 } metadataPayload;
72 };
73
74 struct ArgDescSampler final {
75 uint32_t samplerType = 0;
76 DynamicStateHeapOffset bindful = undefined<DynamicStateHeapOffset>;
77 CrossThreadDataOffset bindless = undefined<CrossThreadDataOffset>;
78 struct {
79 CrossThreadDataOffset samplerSnapWa = undefined<CrossThreadDataOffset>;
80 CrossThreadDataOffset samplerAddressingMode = undefined<CrossThreadDataOffset>;
81 CrossThreadDataOffset samplerNormalizedCoords = undefined<CrossThreadDataOffset>;
82 } metadataPayload;
83 };
84
85 struct ArgDescValue final {
86 struct Element {
87 CrossThreadDataOffset offset = undefined<CrossThreadDataOffset>;
88 uint16_t size = 0U;
89 uint16_t sourceOffset = 0U;
90 };
91 StackVec<Element, 1> elements;
92 };
93
94 struct ArgDescriptor final {
95 enum ArgType : uint8_t {
96 ArgTUnknown,
97 ArgTPointer,
98 ArgTImage,
99 ArgTSampler,
100 ArgTValue
101 };
102
103 struct ExtendedTypeInfo {
ExtendedTypeInfofinal::ExtendedTypeInfo104 ExtendedTypeInfo() {
105 packed = 0U;
106 }
107 union {
108 struct {
109 bool isAccelerator : 1;
110 bool isDeviceQueue : 1;
111 bool isMediaImage : 1;
112 bool isMediaBlockImage : 1;
113 bool isTransformable : 1;
114 bool needsPatch : 1;
115 bool hasVmeExtendedDescriptor : 1;
116 bool hasDeviceSideEnqueueExtendedDescriptor : 1;
117 };
118 uint32_t packed;
119 };
120 };
121
122 ArgDescriptor(ArgType type);
123
ArgDescriptorfinal124 ArgDescriptor()
125 : ArgDescriptor(ArgTUnknown) {
126 }
127
128 ArgDescriptor &operator=(const ArgDescriptor &rhs);
ArgDescriptorfinal129 ArgDescriptor(const ArgDescriptor &rhs) {
130 *this = rhs;
131 }
132
133 template <typename T>
134 const T &as() const;
135
136 template <typename T>
137 T &as(bool initIfUnknown = false);
138
139 template <ArgType Type>
isfinal140 bool is() const {
141 return Type == this->type;
142 }
143
getTraitsfinal144 ArgTypeTraits &getTraits() {
145 return traits;
146 }
147
getTraitsfinal148 const ArgTypeTraits &getTraits() const {
149 return traits;
150 }
151
getExtendedTypeInfofinal152 ExtendedTypeInfo &getExtendedTypeInfo() {
153 return extendedTypeInfo;
154 }
155
getExtendedTypeInfofinal156 const ExtendedTypeInfo &getExtendedTypeInfo() const {
157 return extendedTypeInfo;
158 }
159
isReadOnlyfinal160 bool isReadOnly() const {
161 switch (type) {
162 default:
163 return true;
164 case ArgTImage:
165 return (KernelArgMetadata::AccessReadOnly == traits.accessQualifier);
166 case ArgTPointer:
167 return (KernelArgMetadata::AddrConstant == traits.addressQualifier) || (traits.typeQualifiers.constQual);
168 }
169 }
170
171 protected:
172 ArgDescValue asByValue;
173 ArgTypeTraits traits;
174 union {
175 ArgDescPointer asPointer;
176 ArgDescImage asImage;
177 ArgDescSampler asSampler;
178 };
179
180 ExtendedTypeInfo extendedTypeInfo;
181
182 public:
183 ArgType type;
184 };
185
186 namespace {
187 constexpr auto ArgSize = sizeof(ArgDescriptor);
188 static_assert(ArgSize <= 64, "Keep it small");
189 } // namespace
190
191 template <>
192 inline const ArgDescPointer &ArgDescriptor::as<ArgDescPointer>() const {
193 UNRECOVERABLE_IF(type != ArgTPointer);
194 return this->asPointer;
195 }
196
197 template <>
198 inline const ArgDescImage &ArgDescriptor::as<ArgDescImage>() const {
199 UNRECOVERABLE_IF(type != ArgTImage);
200 return this->asImage;
201 }
202
203 template <>
204 inline const ArgDescSampler &ArgDescriptor::as<ArgDescSampler>() const {
205 UNRECOVERABLE_IF(type != ArgTSampler);
206 return this->asSampler;
207 }
208
209 template <>
210 inline const ArgDescValue &ArgDescriptor::as<ArgDescValue>() const {
211 UNRECOVERABLE_IF(type != ArgTValue);
212 return this->asByValue;
213 }
214
215 template <>
216 inline ArgDescPointer &ArgDescriptor::as<ArgDescPointer>(bool initIfUnknown) {
217 if ((ArgTUnknown == type) && initIfUnknown) {
218 this->type = ArgTPointer;
219 this->asPointer = {};
220 }
221 UNRECOVERABLE_IF(type != ArgTPointer);
222 return this->asPointer;
223 }
224
225 template <>
226 inline ArgDescImage &ArgDescriptor::as<ArgDescImage>(bool initIfUnknown) {
227 if ((ArgTUnknown == type) && initIfUnknown) {
228 this->type = ArgTImage;
229 this->asImage = {};
230 }
231 UNRECOVERABLE_IF(type != ArgTImage);
232 return this->asImage;
233 }
234
235 template <>
236 inline ArgDescSampler &ArgDescriptor::as<ArgDescSampler>(bool initIfUnknown) {
237 if ((ArgTUnknown == type) && initIfUnknown) {
238 this->type = ArgTSampler;
239 this->asSampler = {};
240 }
241 UNRECOVERABLE_IF(type != ArgTSampler);
242 return this->asSampler;
243 }
244
245 template <>
246 inline ArgDescValue &ArgDescriptor::as<ArgDescValue>(bool initIfUnknown) {
247 if ((ArgTUnknown == type) && initIfUnknown) {
248 this->type = ArgTValue;
249 this->asByValue = {};
250 }
251 UNRECOVERABLE_IF(type != ArgTValue);
252 return this->asByValue;
253 }
254
255 template <uint32_t VecSize, typename T>
setOffsetsVec(CrossThreadDataOffset (& dst)[VecSize],const T (& src)[VecSize])256 inline void setOffsetsVec(CrossThreadDataOffset (&dst)[VecSize], const T (&src)[VecSize]) {
257 for (uint32_t i = 0; i < VecSize; ++i) {
258 dst[i] = src[i];
259 }
260 }
261
262 template <typename T>
patchNonPointer(ArrayRef<uint8_t> buffer,CrossThreadDataOffset location,const T & value)263 inline bool patchNonPointer(ArrayRef<uint8_t> buffer, CrossThreadDataOffset location, const T &value) {
264 if (undefined<CrossThreadDataOffset> == location) {
265 return false;
266 }
267 UNRECOVERABLE_IF(location + sizeof(T) > buffer.size());
268 *reinterpret_cast<T *>(buffer.begin() + location) = value;
269 return true;
270 }
271
272 template <uint32_t VecSize, typename T>
patchVecNonPointer(ArrayRef<uint8_t> buffer,const CrossThreadDataOffset (& location)[VecSize],const T (& value)[VecSize])273 inline uint32_t patchVecNonPointer(ArrayRef<uint8_t> buffer, const CrossThreadDataOffset (&location)[VecSize], const T (&value)[VecSize]) {
274 uint32_t numPatched = 0;
275 for (uint32_t i = 0; i < VecSize; ++i) {
276 numPatched += patchNonPointer(buffer, location[i], value[i]) ? 1 : 0;
277 }
278 return numPatched;
279 }
280
patchPointer(ArrayRef<uint8_t> buffer,const ArgDescPointer & arg,uintptr_t value)281 inline bool patchPointer(ArrayRef<uint8_t> buffer, const ArgDescPointer &arg, uintptr_t value) {
282 if (arg.pointerSize == 8) {
283 return patchNonPointer(buffer, arg.stateless, static_cast<uint64_t>(value));
284 } else {
285 UNRECOVERABLE_IF((arg.pointerSize != 4) && isValidOffset(arg.stateless));
286 return patchNonPointer(buffer, arg.stateless, static_cast<uint32_t>(value));
287 }
288 }
289
290 inline ArgDescriptor &ArgDescriptor::operator=(const ArgDescriptor &rhs) {
291 this->type = ArgTUnknown;
292 switch (rhs.type) {
293 default:
294 break;
295 case ArgTPointer:
296 this->as<ArgDescPointer>(true) = rhs.as<ArgDescPointer>();
297 break;
298 case ArgTImage:
299 this->as<ArgDescImage>(true) = rhs.as<ArgDescImage>();
300 break;
301 case ArgTSampler:
302 this->as<ArgDescSampler>(true) = rhs.as<ArgDescSampler>();
303 break;
304 case ArgTValue:
305 this->as<ArgDescValue>(true) = rhs.as<ArgDescValue>();
306 break;
307 }
308 this->type = rhs.type;
309 this->traits = rhs.traits;
310 this->extendedTypeInfo = rhs.extendedTypeInfo;
311 return *this;
312 }
313
ArgDescriptor(ArgType type)314 inline ArgDescriptor::ArgDescriptor(ArgType type) : type(type) {
315 switch (type) {
316 default:
317 break;
318 case ArgTPointer:
319 this->as<ArgDescPointer>(true);
320 break;
321 case ArgTImage:
322 this->as<ArgDescImage>(true);
323 break;
324 case ArgTSampler:
325 this->as<ArgDescSampler>(true);
326 break;
327 case ArgTValue:
328 this->as<ArgDescValue>(true);
329 break;
330 }
331 }
332
333 struct ArgDescriptorExtended {
334 virtual ~ArgDescriptorExtended() = default;
335 };
336
337 } // namespace NEO
338