1 /*
2 * Copyright © 2017 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23 #include <stdio.h>
24 #include <time.h>
25 #include "pipe/p_defines.h"
26 #include "pipe/p_state.h"
27 #include "util/debug.h"
28 #include "util/ralloc.h"
29 #include "util/u_inlines.h"
30 #include "util/format/u_format.h"
31 #include "util/u_upload_mgr.h"
32 #include "drm-uapi/i915_drm.h"
33 #include "iris_context.h"
34 #include "iris_resource.h"
35 #include "iris_screen.h"
36 #include "common/intel_defines.h"
37 #include "common/intel_sample_positions.h"
38
39 /**
40 * The pipe->set_debug_callback() driver hook.
41 */
42 static void
iris_set_debug_callback(struct pipe_context * ctx,const struct pipe_debug_callback * cb)43 iris_set_debug_callback(struct pipe_context *ctx,
44 const struct pipe_debug_callback *cb)
45 {
46 struct iris_context *ice = (struct iris_context *)ctx;
47 struct iris_screen *screen = (struct iris_screen *)ctx->screen;
48
49 util_queue_finish(&screen->shader_compiler_queue);
50
51 if (cb)
52 ice->dbg = *cb;
53 else
54 memset(&ice->dbg, 0, sizeof(ice->dbg));
55 }
56
57 /**
58 * Called from the batch module when it detects a GPU hang.
59 *
60 * In this case, we've lost our GEM context, and can't rely on any existing
61 * state on the GPU. We must mark everything dirty and wipe away any saved
62 * assumptions about the last known state of the GPU.
63 */
64 void
iris_lost_context_state(struct iris_batch * batch)65 iris_lost_context_state(struct iris_batch *batch)
66 {
67 struct iris_context *ice = batch->ice;
68
69 if (batch->name == IRIS_BATCH_RENDER) {
70 batch->screen->vtbl.init_render_context(batch);
71 } else if (batch->name == IRIS_BATCH_COMPUTE) {
72 batch->screen->vtbl.init_compute_context(batch);
73 } else {
74 unreachable("unhandled batch reset");
75 }
76
77 ice->state.dirty = ~0ull;
78 ice->state.stage_dirty = ~0ull;
79 ice->state.current_hash_scale = 0;
80 memset(&ice->shaders.urb, 0, sizeof(ice->shaders.urb));
81 memset(ice->state.last_block, 0, sizeof(ice->state.last_block));
82 memset(ice->state.last_grid, 0, sizeof(ice->state.last_grid));
83 batch->last_surface_base_address = ~0ull;
84 batch->last_aux_map_state = 0;
85 batch->screen->vtbl.lost_genx_state(ice, batch);
86 }
87
88 static enum pipe_reset_status
iris_get_device_reset_status(struct pipe_context * ctx)89 iris_get_device_reset_status(struct pipe_context *ctx)
90 {
91 struct iris_context *ice = (struct iris_context *)ctx;
92
93 enum pipe_reset_status worst_reset = PIPE_NO_RESET;
94
95 /* Check the reset status of each batch's hardware context, and take the
96 * worst status (if one was guilty, proclaim guilt).
97 */
98 for (int i = 0; i < IRIS_BATCH_COUNT; i++) {
99 /* This will also recreate the hardware contexts as necessary, so any
100 * future queries will show no resets. We only want to report once.
101 */
102 enum pipe_reset_status batch_reset =
103 iris_batch_check_for_reset(&ice->batches[i]);
104
105 if (batch_reset == PIPE_NO_RESET)
106 continue;
107
108 if (worst_reset == PIPE_NO_RESET) {
109 worst_reset = batch_reset;
110 } else {
111 /* GUILTY < INNOCENT < UNKNOWN */
112 worst_reset = MIN2(worst_reset, batch_reset);
113 }
114 }
115
116 if (worst_reset != PIPE_NO_RESET && ice->reset.reset)
117 ice->reset.reset(ice->reset.data, worst_reset);
118
119 return worst_reset;
120 }
121
122 static void
iris_set_device_reset_callback(struct pipe_context * ctx,const struct pipe_device_reset_callback * cb)123 iris_set_device_reset_callback(struct pipe_context *ctx,
124 const struct pipe_device_reset_callback *cb)
125 {
126 struct iris_context *ice = (struct iris_context *)ctx;
127
128 if (cb)
129 ice->reset = *cb;
130 else
131 memset(&ice->reset, 0, sizeof(ice->reset));
132 }
133
134 static void
iris_get_sample_position(struct pipe_context * ctx,unsigned sample_count,unsigned sample_index,float * out_value)135 iris_get_sample_position(struct pipe_context *ctx,
136 unsigned sample_count,
137 unsigned sample_index,
138 float *out_value)
139 {
140 union {
141 struct {
142 float x[16];
143 float y[16];
144 } a;
145 struct {
146 float _0XOffset, _1XOffset, _2XOffset, _3XOffset,
147 _4XOffset, _5XOffset, _6XOffset, _7XOffset,
148 _8XOffset, _9XOffset, _10XOffset, _11XOffset,
149 _12XOffset, _13XOffset, _14XOffset, _15XOffset;
150 float _0YOffset, _1YOffset, _2YOffset, _3YOffset,
151 _4YOffset, _5YOffset, _6YOffset, _7YOffset,
152 _8YOffset, _9YOffset, _10YOffset, _11YOffset,
153 _12YOffset, _13YOffset, _14YOffset, _15YOffset;
154 } v;
155 } u;
156 switch (sample_count) {
157 case 1: INTEL_SAMPLE_POS_1X(u.v._); break;
158 case 2: INTEL_SAMPLE_POS_2X(u.v._); break;
159 case 4: INTEL_SAMPLE_POS_4X(u.v._); break;
160 case 8: INTEL_SAMPLE_POS_8X(u.v._); break;
161 case 16: INTEL_SAMPLE_POS_16X(u.v._); break;
162 default: unreachable("invalid sample count");
163 }
164
165 out_value[0] = u.a.x[sample_index];
166 out_value[1] = u.a.y[sample_index];
167 }
168
169 static bool
create_dirty_dmabuf_set(struct iris_context * ice)170 create_dirty_dmabuf_set(struct iris_context *ice)
171 {
172 assert(ice->dirty_dmabufs == NULL);
173
174 ice->dirty_dmabufs = _mesa_pointer_set_create(ice);
175 return ice->dirty_dmabufs != NULL;
176 }
177
178 void
iris_mark_dirty_dmabuf(struct iris_context * ice,struct pipe_resource * res)179 iris_mark_dirty_dmabuf(struct iris_context *ice,
180 struct pipe_resource *res)
181 {
182 if (!_mesa_set_search(ice->dirty_dmabufs, res)) {
183 _mesa_set_add(ice->dirty_dmabufs, res);
184 pipe_reference(NULL, &res->reference);
185 }
186 }
187
188 static void
clear_dirty_dmabuf_set(struct iris_context * ice)189 clear_dirty_dmabuf_set(struct iris_context *ice)
190 {
191 set_foreach(ice->dirty_dmabufs, entry) {
192 struct pipe_resource *res = (struct pipe_resource *)entry->key;
193 if (pipe_reference(&res->reference, NULL))
194 res->screen->resource_destroy(res->screen, res);
195 }
196
197 _mesa_set_clear(ice->dirty_dmabufs, NULL);
198 }
199
200 void
iris_flush_dirty_dmabufs(struct iris_context * ice)201 iris_flush_dirty_dmabufs(struct iris_context *ice)
202 {
203 set_foreach(ice->dirty_dmabufs, entry) {
204 struct pipe_resource *res = (struct pipe_resource *)entry->key;
205 ice->ctx.flush_resource(&ice->ctx, res);
206 }
207
208 clear_dirty_dmabuf_set(ice);
209 }
210
211
212 /**
213 * Destroy a context, freeing any associated memory.
214 */
215 void
iris_destroy_context(struct pipe_context * ctx)216 iris_destroy_context(struct pipe_context *ctx)
217 {
218 struct iris_context *ice = (struct iris_context *)ctx;
219 struct iris_screen *screen = (struct iris_screen *)ctx->screen;
220
221 if (ctx->stream_uploader)
222 u_upload_destroy(ctx->stream_uploader);
223 if (ctx->const_uploader)
224 u_upload_destroy(ctx->const_uploader);
225
226 clear_dirty_dmabuf_set(ice);
227
228 screen->vtbl.destroy_state(ice);
229
230 for (unsigned i = 0; i < ARRAY_SIZE(ice->shaders.scratch_surfs); i++)
231 pipe_resource_reference(&ice->shaders.scratch_surfs[i].res, NULL);
232
233 iris_destroy_program_cache(ice);
234 iris_destroy_border_color_pool(ice);
235 if (screen->measure.config)
236 iris_destroy_ctx_measure(ice);
237
238 u_upload_destroy(ice->state.surface_uploader);
239 u_upload_destroy(ice->state.bindless_uploader);
240 u_upload_destroy(ice->state.dynamic_uploader);
241 u_upload_destroy(ice->query_buffer_uploader);
242
243 iris_batch_free(&ice->batches[IRIS_BATCH_RENDER]);
244 iris_batch_free(&ice->batches[IRIS_BATCH_COMPUTE]);
245 iris_destroy_binder(&ice->state.binder);
246
247 slab_destroy_child(&ice->transfer_pool);
248 slab_destroy_child(&ice->transfer_pool_unsync);
249
250 ralloc_free(ice);
251 }
252
253 #define genX_call(devinfo, func, ...) \
254 switch ((devinfo)->verx10) { \
255 case 125: \
256 gfx125_##func(__VA_ARGS__); \
257 break; \
258 case 120: \
259 gfx12_##func(__VA_ARGS__); \
260 break; \
261 case 110: \
262 gfx11_##func(__VA_ARGS__); \
263 break; \
264 case 90: \
265 gfx9_##func(__VA_ARGS__); \
266 break; \
267 case 80: \
268 gfx8_##func(__VA_ARGS__); \
269 break; \
270 default: \
271 unreachable("Unknown hardware generation"); \
272 }
273
274 /**
275 * Create a context.
276 *
277 * This is where each context begins.
278 */
279 struct pipe_context *
iris_create_context(struct pipe_screen * pscreen,void * priv,unsigned flags)280 iris_create_context(struct pipe_screen *pscreen, void *priv, unsigned flags)
281 {
282 struct iris_screen *screen = (struct iris_screen*)pscreen;
283 const struct intel_device_info *devinfo = &screen->devinfo;
284 struct iris_context *ice = rzalloc(NULL, struct iris_context);
285
286 if (!ice)
287 return NULL;
288
289 struct pipe_context *ctx = &ice->ctx;
290
291 ctx->screen = pscreen;
292 ctx->priv = priv;
293
294 ctx->stream_uploader = u_upload_create_default(ctx);
295 if (!ctx->stream_uploader) {
296 free(ctx);
297 return NULL;
298 }
299 ctx->const_uploader = u_upload_create(ctx, 1024 * 1024,
300 PIPE_BIND_CONSTANT_BUFFER,
301 PIPE_USAGE_IMMUTABLE,
302 IRIS_RESOURCE_FLAG_DEVICE_MEM);
303 if (!ctx->const_uploader) {
304 u_upload_destroy(ctx->stream_uploader);
305 free(ctx);
306 return NULL;
307 }
308
309 if (!create_dirty_dmabuf_set(ice)) {
310 ralloc_free(ice);
311 return NULL;
312 }
313
314 ctx->destroy = iris_destroy_context;
315 ctx->set_debug_callback = iris_set_debug_callback;
316 ctx->set_device_reset_callback = iris_set_device_reset_callback;
317 ctx->get_device_reset_status = iris_get_device_reset_status;
318 ctx->get_sample_position = iris_get_sample_position;
319
320 iris_init_context_fence_functions(ctx);
321 iris_init_blit_functions(ctx);
322 iris_init_clear_functions(ctx);
323 iris_init_program_functions(ctx);
324 iris_init_resource_functions(ctx);
325 iris_init_flush_functions(ctx);
326 iris_init_perfquery_functions(ctx);
327
328 iris_init_program_cache(ice);
329 iris_init_border_color_pool(ice);
330 iris_init_binder(ice);
331
332 slab_create_child(&ice->transfer_pool, &screen->transfer_pool);
333 slab_create_child(&ice->transfer_pool_unsync, &screen->transfer_pool);
334
335 ice->state.surface_uploader =
336 u_upload_create(ctx, 64 * 1024, PIPE_BIND_CUSTOM, PIPE_USAGE_IMMUTABLE,
337 IRIS_RESOURCE_FLAG_SURFACE_MEMZONE |
338 IRIS_RESOURCE_FLAG_DEVICE_MEM);
339 ice->state.bindless_uploader =
340 u_upload_create(ctx, 64 * 1024, PIPE_BIND_CUSTOM, PIPE_USAGE_IMMUTABLE,
341 IRIS_RESOURCE_FLAG_BINDLESS_MEMZONE |
342 IRIS_RESOURCE_FLAG_DEVICE_MEM);
343 ice->state.dynamic_uploader =
344 u_upload_create(ctx, 64 * 1024, PIPE_BIND_CUSTOM, PIPE_USAGE_IMMUTABLE,
345 IRIS_RESOURCE_FLAG_DYNAMIC_MEMZONE |
346 IRIS_RESOURCE_FLAG_DEVICE_MEM);
347
348 ice->query_buffer_uploader =
349 u_upload_create(ctx, 16 * 1024, PIPE_BIND_CUSTOM, PIPE_USAGE_STAGING,
350 0);
351
352 genX_call(devinfo, init_state, ice);
353 genX_call(devinfo, init_blorp, ice);
354 genX_call(devinfo, init_query, ice);
355
356 int priority = 0;
357 if (flags & PIPE_CONTEXT_HIGH_PRIORITY)
358 priority = INTEL_CONTEXT_HIGH_PRIORITY;
359 if (flags & PIPE_CONTEXT_LOW_PRIORITY)
360 priority = INTEL_CONTEXT_LOW_PRIORITY;
361
362 if (INTEL_DEBUG(DEBUG_BATCH))
363 ice->state.sizes = _mesa_hash_table_u64_create(ice);
364
365 for (int i = 0; i < IRIS_BATCH_COUNT; i++) {
366 iris_init_batch(ice, (enum iris_batch_name) i, priority);
367 }
368
369 screen->vtbl.init_render_context(&ice->batches[IRIS_BATCH_RENDER]);
370 screen->vtbl.init_compute_context(&ice->batches[IRIS_BATCH_COMPUTE]);
371
372 if (!(flags & PIPE_CONTEXT_PREFER_THREADED))
373 return ctx;
374
375 /* Clover doesn't support u_threaded_context */
376 if (flags & PIPE_CONTEXT_COMPUTE_ONLY)
377 return ctx;
378
379 return threaded_context_create(ctx, &screen->transfer_pool,
380 iris_replace_buffer_storage,
381 NULL, /* TODO: asynchronous flushes? */
382 &ice->thrctx);
383 }
384