1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2018 Intel Corporation
4 */
5
6 #include <linux/prime_numbers.h>
7
8 #include "i915_selftest.h"
9 #include "intel_engine_heartbeat.h"
10 #include "intel_engine_pm.h"
11 #include "intel_reset.h"
12 #include "intel_ring.h"
13 #include "selftest_engine_heartbeat.h"
14 #include "selftests/i915_random.h"
15 #include "selftests/igt_flush_test.h"
16 #include "selftests/igt_live_test.h"
17 #include "selftests/igt_spinner.h"
18 #include "selftests/lib_sw_fence.h"
19 #include "shmem_utils.h"
20
21 #include "gem/selftests/igt_gem_utils.h"
22 #include "gem/selftests/mock_context.h"
23
24 #define CS_GPR(engine, n) ((engine)->mmio_base + 0x600 + (n) * 4)
25 #define NUM_GPR 16
26 #define NUM_GPR_DW (NUM_GPR * 2) /* each GPR is 2 dwords */
27
create_scratch(struct intel_gt * gt)28 static struct i915_vma *create_scratch(struct intel_gt *gt)
29 {
30 return __vm_create_scratch_for_read_pinned(>->ggtt->vm, PAGE_SIZE);
31 }
32
is_active(struct i915_request * rq)33 static bool is_active(struct i915_request *rq)
34 {
35 if (i915_request_is_active(rq))
36 return true;
37
38 if (i915_request_on_hold(rq))
39 return true;
40
41 if (i915_request_has_initial_breadcrumb(rq) && i915_request_started(rq))
42 return true;
43
44 return false;
45 }
46
wait_for_submit(struct intel_engine_cs * engine,struct i915_request * rq,unsigned long timeout)47 static int wait_for_submit(struct intel_engine_cs *engine,
48 struct i915_request *rq,
49 unsigned long timeout)
50 {
51 /* Ignore our own attempts to suppress excess tasklets */
52 tasklet_hi_schedule(&engine->execlists.tasklet);
53
54 timeout += jiffies;
55 do {
56 bool done = time_after(jiffies, timeout);
57
58 if (i915_request_completed(rq)) /* that was quick! */
59 return 0;
60
61 /* Wait until the HW has acknowleged the submission (or err) */
62 intel_engine_flush_submission(engine);
63 if (!READ_ONCE(engine->execlists.pending[0]) && is_active(rq))
64 return 0;
65
66 if (done)
67 return -ETIME;
68
69 cond_resched();
70 } while (1);
71 }
72
emit_semaphore_signal(struct intel_context * ce,void * slot)73 static int emit_semaphore_signal(struct intel_context *ce, void *slot)
74 {
75 const u32 offset =
76 i915_ggtt_offset(ce->engine->status_page.vma) +
77 offset_in_page(slot);
78 struct i915_request *rq;
79 u32 *cs;
80
81 rq = intel_context_create_request(ce);
82 if (IS_ERR(rq))
83 return PTR_ERR(rq);
84
85 cs = intel_ring_begin(rq, 4);
86 if (IS_ERR(cs)) {
87 i915_request_add(rq);
88 return PTR_ERR(cs);
89 }
90
91 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
92 *cs++ = offset;
93 *cs++ = 0;
94 *cs++ = 1;
95
96 intel_ring_advance(rq, cs);
97
98 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
99 i915_request_add(rq);
100 return 0;
101 }
102
context_flush(struct intel_context * ce,long timeout)103 static int context_flush(struct intel_context *ce, long timeout)
104 {
105 struct i915_request *rq;
106 struct dma_fence *fence;
107 int err = 0;
108
109 rq = intel_engine_create_kernel_request(ce->engine);
110 if (IS_ERR(rq))
111 return PTR_ERR(rq);
112
113 fence = i915_active_fence_get(&ce->timeline->last_request);
114 if (fence) {
115 i915_request_await_dma_fence(rq, fence);
116 dma_fence_put(fence);
117 }
118
119 rq = i915_request_get(rq);
120 i915_request_add(rq);
121 if (i915_request_wait(rq, 0, timeout) < 0)
122 err = -ETIME;
123 i915_request_put(rq);
124
125 rmb(); /* We know the request is written, make sure all state is too! */
126 return err;
127 }
128
live_lrc_layout(void * arg)129 static int live_lrc_layout(void *arg)
130 {
131 struct intel_gt *gt = arg;
132 struct intel_engine_cs *engine;
133 enum intel_engine_id id;
134 u32 *lrc;
135 int err;
136
137 /*
138 * Check the registers offsets we use to create the initial reg state
139 * match the layout saved by HW.
140 */
141
142 lrc = (u32 *)__get_free_page(GFP_KERNEL); /* requires page alignment */
143 if (!lrc)
144 return -ENOMEM;
145 GEM_BUG_ON(offset_in_page(lrc));
146
147 err = 0;
148 for_each_engine(engine, gt, id) {
149 u32 *hw;
150 int dw;
151
152 if (!engine->default_state)
153 continue;
154
155 hw = shmem_pin_map(engine->default_state);
156 if (IS_ERR(hw)) {
157 err = PTR_ERR(hw);
158 break;
159 }
160 hw += LRC_STATE_OFFSET / sizeof(*hw);
161
162 __lrc_init_regs(memset(lrc, POISON_INUSE, PAGE_SIZE),
163 engine->kernel_context, engine, true);
164
165 dw = 0;
166 do {
167 u32 lri = READ_ONCE(hw[dw]);
168
169 if (lri == 0) {
170 dw++;
171 continue;
172 }
173
174 if (lrc[dw] == 0) {
175 pr_debug("%s: skipped instruction %x at dword %d\n",
176 engine->name, lri, dw);
177 dw++;
178 continue;
179 }
180
181 if ((lri & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
182 pr_err("%s: Expected LRI command at dword %d, found %08x\n",
183 engine->name, dw, lri);
184 err = -EINVAL;
185 break;
186 }
187
188 if (lrc[dw] != lri) {
189 pr_err("%s: LRI command mismatch at dword %d, expected %08x found %08x\n",
190 engine->name, dw, lri, lrc[dw]);
191 err = -EINVAL;
192 break;
193 }
194
195 lri &= 0x7f;
196 lri++;
197 dw++;
198
199 while (lri) {
200 u32 offset = READ_ONCE(hw[dw]);
201
202 if (offset != lrc[dw]) {
203 pr_err("%s: Different registers found at dword %d, expected %x, found %x\n",
204 engine->name, dw, offset, lrc[dw]);
205 err = -EINVAL;
206 break;
207 }
208
209 /*
210 * Skip over the actual register value as we
211 * expect that to differ.
212 */
213 dw += 2;
214 lri -= 2;
215 }
216 } while (!err && (lrc[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
217
218 if (err) {
219 pr_info("%s: HW register image:\n", engine->name);
220 igt_hexdump(hw, PAGE_SIZE);
221
222 pr_info("%s: SW register image:\n", engine->name);
223 igt_hexdump(lrc, PAGE_SIZE);
224 }
225
226 shmem_unpin_map(engine->default_state, hw);
227 if (err)
228 break;
229 }
230
231 free_page((unsigned long)lrc);
232 return err;
233 }
234
find_offset(const u32 * lri,u32 offset)235 static int find_offset(const u32 *lri, u32 offset)
236 {
237 int i;
238
239 for (i = 0; i < PAGE_SIZE / sizeof(u32); i++)
240 if (lri[i] == offset)
241 return i;
242
243 return -1;
244 }
245
live_lrc_fixed(void * arg)246 static int live_lrc_fixed(void *arg)
247 {
248 struct intel_gt *gt = arg;
249 struct intel_engine_cs *engine;
250 enum intel_engine_id id;
251 int err = 0;
252
253 /*
254 * Check the assumed register offsets match the actual locations in
255 * the context image.
256 */
257
258 for_each_engine(engine, gt, id) {
259 const struct {
260 u32 reg;
261 u32 offset;
262 const char *name;
263 } tbl[] = {
264 {
265 i915_mmio_reg_offset(RING_START(engine->mmio_base)),
266 CTX_RING_START - 1,
267 "RING_START"
268 },
269 {
270 i915_mmio_reg_offset(RING_CTL(engine->mmio_base)),
271 CTX_RING_CTL - 1,
272 "RING_CTL"
273 },
274 {
275 i915_mmio_reg_offset(RING_HEAD(engine->mmio_base)),
276 CTX_RING_HEAD - 1,
277 "RING_HEAD"
278 },
279 {
280 i915_mmio_reg_offset(RING_TAIL(engine->mmio_base)),
281 CTX_RING_TAIL - 1,
282 "RING_TAIL"
283 },
284 {
285 i915_mmio_reg_offset(RING_MI_MODE(engine->mmio_base)),
286 lrc_ring_mi_mode(engine),
287 "RING_MI_MODE"
288 },
289 {
290 i915_mmio_reg_offset(RING_BBSTATE(engine->mmio_base)),
291 CTX_BB_STATE - 1,
292 "BB_STATE"
293 },
294 {
295 i915_mmio_reg_offset(RING_BB_PER_CTX_PTR(engine->mmio_base)),
296 lrc_ring_wa_bb_per_ctx(engine),
297 "RING_BB_PER_CTX_PTR"
298 },
299 {
300 i915_mmio_reg_offset(RING_INDIRECT_CTX(engine->mmio_base)),
301 lrc_ring_indirect_ptr(engine),
302 "RING_INDIRECT_CTX_PTR"
303 },
304 {
305 i915_mmio_reg_offset(RING_INDIRECT_CTX_OFFSET(engine->mmio_base)),
306 lrc_ring_indirect_offset(engine),
307 "RING_INDIRECT_CTX_OFFSET"
308 },
309 {
310 i915_mmio_reg_offset(RING_CTX_TIMESTAMP(engine->mmio_base)),
311 CTX_TIMESTAMP - 1,
312 "RING_CTX_TIMESTAMP"
313 },
314 {
315 i915_mmio_reg_offset(GEN8_RING_CS_GPR(engine->mmio_base, 0)),
316 lrc_ring_gpr0(engine),
317 "RING_CS_GPR0"
318 },
319 {
320 i915_mmio_reg_offset(RING_CMD_BUF_CCTL(engine->mmio_base)),
321 lrc_ring_cmd_buf_cctl(engine),
322 "RING_CMD_BUF_CCTL"
323 },
324 { },
325 }, *t;
326 u32 *hw;
327
328 if (!engine->default_state)
329 continue;
330
331 hw = shmem_pin_map(engine->default_state);
332 if (IS_ERR(hw)) {
333 err = PTR_ERR(hw);
334 break;
335 }
336 hw += LRC_STATE_OFFSET / sizeof(*hw);
337
338 for (t = tbl; t->name; t++) {
339 int dw = find_offset(hw, t->reg);
340
341 if (dw != t->offset) {
342 pr_err("%s: Offset for %s [0x%x] mismatch, found %x, expected %x\n",
343 engine->name,
344 t->name,
345 t->reg,
346 dw,
347 t->offset);
348 err = -EINVAL;
349 }
350 }
351
352 shmem_unpin_map(engine->default_state, hw);
353 }
354
355 return err;
356 }
357
__live_lrc_state(struct intel_engine_cs * engine,struct i915_vma * scratch)358 static int __live_lrc_state(struct intel_engine_cs *engine,
359 struct i915_vma *scratch)
360 {
361 struct intel_context *ce;
362 struct i915_request *rq;
363 struct i915_gem_ww_ctx ww;
364 enum {
365 RING_START_IDX = 0,
366 RING_TAIL_IDX,
367 MAX_IDX
368 };
369 u32 expected[MAX_IDX];
370 u32 *cs;
371 int err;
372 int n;
373
374 ce = intel_context_create(engine);
375 if (IS_ERR(ce))
376 return PTR_ERR(ce);
377
378 i915_gem_ww_ctx_init(&ww, false);
379 retry:
380 err = i915_gem_object_lock(scratch->obj, &ww);
381 if (!err)
382 err = intel_context_pin_ww(ce, &ww);
383 if (err)
384 goto err_put;
385
386 rq = i915_request_create(ce);
387 if (IS_ERR(rq)) {
388 err = PTR_ERR(rq);
389 goto err_unpin;
390 }
391
392 cs = intel_ring_begin(rq, 4 * MAX_IDX);
393 if (IS_ERR(cs)) {
394 err = PTR_ERR(cs);
395 i915_request_add(rq);
396 goto err_unpin;
397 }
398
399 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
400 *cs++ = i915_mmio_reg_offset(RING_START(engine->mmio_base));
401 *cs++ = i915_ggtt_offset(scratch) + RING_START_IDX * sizeof(u32);
402 *cs++ = 0;
403
404 expected[RING_START_IDX] = i915_ggtt_offset(ce->ring->vma);
405
406 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
407 *cs++ = i915_mmio_reg_offset(RING_TAIL(engine->mmio_base));
408 *cs++ = i915_ggtt_offset(scratch) + RING_TAIL_IDX * sizeof(u32);
409 *cs++ = 0;
410
411 err = i915_request_await_object(rq, scratch->obj, true);
412 if (!err)
413 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
414
415 i915_request_get(rq);
416 i915_request_add(rq);
417 if (err)
418 goto err_rq;
419
420 intel_engine_flush_submission(engine);
421 expected[RING_TAIL_IDX] = ce->ring->tail;
422
423 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
424 err = -ETIME;
425 goto err_rq;
426 }
427
428 cs = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
429 if (IS_ERR(cs)) {
430 err = PTR_ERR(cs);
431 goto err_rq;
432 }
433
434 for (n = 0; n < MAX_IDX; n++) {
435 if (cs[n] != expected[n]) {
436 pr_err("%s: Stored register[%d] value[0x%x] did not match expected[0x%x]\n",
437 engine->name, n, cs[n], expected[n]);
438 err = -EINVAL;
439 break;
440 }
441 }
442
443 i915_gem_object_unpin_map(scratch->obj);
444
445 err_rq:
446 i915_request_put(rq);
447 err_unpin:
448 intel_context_unpin(ce);
449 err_put:
450 if (err == -EDEADLK) {
451 err = i915_gem_ww_ctx_backoff(&ww);
452 if (!err)
453 goto retry;
454 }
455 i915_gem_ww_ctx_fini(&ww);
456 intel_context_put(ce);
457 return err;
458 }
459
live_lrc_state(void * arg)460 static int live_lrc_state(void *arg)
461 {
462 struct intel_gt *gt = arg;
463 struct intel_engine_cs *engine;
464 struct i915_vma *scratch;
465 enum intel_engine_id id;
466 int err = 0;
467
468 /*
469 * Check the live register state matches what we expect for this
470 * intel_context.
471 */
472
473 scratch = create_scratch(gt);
474 if (IS_ERR(scratch))
475 return PTR_ERR(scratch);
476
477 for_each_engine(engine, gt, id) {
478 err = __live_lrc_state(engine, scratch);
479 if (err)
480 break;
481 }
482
483 if (igt_flush_test(gt->i915))
484 err = -EIO;
485
486 i915_vma_unpin_and_release(&scratch, 0);
487 return err;
488 }
489
gpr_make_dirty(struct intel_context * ce)490 static int gpr_make_dirty(struct intel_context *ce)
491 {
492 struct i915_request *rq;
493 u32 *cs;
494 int n;
495
496 rq = intel_context_create_request(ce);
497 if (IS_ERR(rq))
498 return PTR_ERR(rq);
499
500 cs = intel_ring_begin(rq, 2 * NUM_GPR_DW + 2);
501 if (IS_ERR(cs)) {
502 i915_request_add(rq);
503 return PTR_ERR(cs);
504 }
505
506 *cs++ = MI_LOAD_REGISTER_IMM(NUM_GPR_DW);
507 for (n = 0; n < NUM_GPR_DW; n++) {
508 *cs++ = CS_GPR(ce->engine, n);
509 *cs++ = STACK_MAGIC;
510 }
511 *cs++ = MI_NOOP;
512
513 intel_ring_advance(rq, cs);
514
515 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
516 i915_request_add(rq);
517
518 return 0;
519 }
520
521 static struct i915_request *
__gpr_read(struct intel_context * ce,struct i915_vma * scratch,u32 * slot)522 __gpr_read(struct intel_context *ce, struct i915_vma *scratch, u32 *slot)
523 {
524 const u32 offset =
525 i915_ggtt_offset(ce->engine->status_page.vma) +
526 offset_in_page(slot);
527 struct i915_request *rq;
528 u32 *cs;
529 int err;
530 int n;
531
532 rq = intel_context_create_request(ce);
533 if (IS_ERR(rq))
534 return rq;
535
536 cs = intel_ring_begin(rq, 6 + 4 * NUM_GPR_DW);
537 if (IS_ERR(cs)) {
538 i915_request_add(rq);
539 return ERR_CAST(cs);
540 }
541
542 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
543 *cs++ = MI_NOOP;
544
545 *cs++ = MI_SEMAPHORE_WAIT |
546 MI_SEMAPHORE_GLOBAL_GTT |
547 MI_SEMAPHORE_POLL |
548 MI_SEMAPHORE_SAD_NEQ_SDD;
549 *cs++ = 0;
550 *cs++ = offset;
551 *cs++ = 0;
552
553 for (n = 0; n < NUM_GPR_DW; n++) {
554 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
555 *cs++ = CS_GPR(ce->engine, n);
556 *cs++ = i915_ggtt_offset(scratch) + n * sizeof(u32);
557 *cs++ = 0;
558 }
559
560 i915_vma_lock(scratch);
561 err = i915_request_await_object(rq, scratch->obj, true);
562 if (!err)
563 err = i915_vma_move_to_active(scratch, rq, EXEC_OBJECT_WRITE);
564 i915_vma_unlock(scratch);
565
566 i915_request_get(rq);
567 i915_request_add(rq);
568 if (err) {
569 i915_request_put(rq);
570 rq = ERR_PTR(err);
571 }
572
573 return rq;
574 }
575
__live_lrc_gpr(struct intel_engine_cs * engine,struct i915_vma * scratch,bool preempt)576 static int __live_lrc_gpr(struct intel_engine_cs *engine,
577 struct i915_vma *scratch,
578 bool preempt)
579 {
580 u32 *slot = memset32(engine->status_page.addr + 1000, 0, 4);
581 struct intel_context *ce;
582 struct i915_request *rq;
583 u32 *cs;
584 int err;
585 int n;
586
587 if (INTEL_GEN(engine->i915) < 9 && engine->class != RENDER_CLASS)
588 return 0; /* GPR only on rcs0 for gen8 */
589
590 err = gpr_make_dirty(engine->kernel_context);
591 if (err)
592 return err;
593
594 ce = intel_context_create(engine);
595 if (IS_ERR(ce))
596 return PTR_ERR(ce);
597
598 rq = __gpr_read(ce, scratch, slot);
599 if (IS_ERR(rq)) {
600 err = PTR_ERR(rq);
601 goto err_put;
602 }
603
604 err = wait_for_submit(engine, rq, HZ / 2);
605 if (err)
606 goto err_rq;
607
608 if (preempt) {
609 err = gpr_make_dirty(engine->kernel_context);
610 if (err)
611 goto err_rq;
612
613 err = emit_semaphore_signal(engine->kernel_context, slot);
614 if (err)
615 goto err_rq;
616
617 err = wait_for_submit(engine, rq, HZ / 2);
618 if (err)
619 goto err_rq;
620 } else {
621 slot[0] = 1;
622 wmb();
623 }
624
625 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
626 err = -ETIME;
627 goto err_rq;
628 }
629
630 cs = i915_gem_object_pin_map_unlocked(scratch->obj, I915_MAP_WB);
631 if (IS_ERR(cs)) {
632 err = PTR_ERR(cs);
633 goto err_rq;
634 }
635
636 for (n = 0; n < NUM_GPR_DW; n++) {
637 if (cs[n]) {
638 pr_err("%s: GPR[%d].%s was not zero, found 0x%08x!\n",
639 engine->name,
640 n / 2, n & 1 ? "udw" : "ldw",
641 cs[n]);
642 err = -EINVAL;
643 break;
644 }
645 }
646
647 i915_gem_object_unpin_map(scratch->obj);
648
649 err_rq:
650 memset32(&slot[0], -1, 4);
651 wmb();
652 i915_request_put(rq);
653 err_put:
654 intel_context_put(ce);
655 return err;
656 }
657
live_lrc_gpr(void * arg)658 static int live_lrc_gpr(void *arg)
659 {
660 struct intel_gt *gt = arg;
661 struct intel_engine_cs *engine;
662 struct i915_vma *scratch;
663 enum intel_engine_id id;
664 int err = 0;
665
666 /*
667 * Check that GPR registers are cleared in new contexts as we need
668 * to avoid leaking any information from previous contexts.
669 */
670
671 scratch = create_scratch(gt);
672 if (IS_ERR(scratch))
673 return PTR_ERR(scratch);
674
675 for_each_engine(engine, gt, id) {
676 st_engine_heartbeat_disable(engine);
677
678 err = __live_lrc_gpr(engine, scratch, false);
679 if (err)
680 goto err;
681
682 err = __live_lrc_gpr(engine, scratch, true);
683 if (err)
684 goto err;
685
686 err:
687 st_engine_heartbeat_enable(engine);
688 if (igt_flush_test(gt->i915))
689 err = -EIO;
690 if (err)
691 break;
692 }
693
694 i915_vma_unpin_and_release(&scratch, 0);
695 return err;
696 }
697
698 static struct i915_request *
create_timestamp(struct intel_context * ce,void * slot,int idx)699 create_timestamp(struct intel_context *ce, void *slot, int idx)
700 {
701 const u32 offset =
702 i915_ggtt_offset(ce->engine->status_page.vma) +
703 offset_in_page(slot);
704 struct i915_request *rq;
705 u32 *cs;
706 int err;
707
708 rq = intel_context_create_request(ce);
709 if (IS_ERR(rq))
710 return rq;
711
712 cs = intel_ring_begin(rq, 10);
713 if (IS_ERR(cs)) {
714 err = PTR_ERR(cs);
715 goto err;
716 }
717
718 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
719 *cs++ = MI_NOOP;
720
721 *cs++ = MI_SEMAPHORE_WAIT |
722 MI_SEMAPHORE_GLOBAL_GTT |
723 MI_SEMAPHORE_POLL |
724 MI_SEMAPHORE_SAD_NEQ_SDD;
725 *cs++ = 0;
726 *cs++ = offset;
727 *cs++ = 0;
728
729 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
730 *cs++ = i915_mmio_reg_offset(RING_CTX_TIMESTAMP(rq->engine->mmio_base));
731 *cs++ = offset + idx * sizeof(u32);
732 *cs++ = 0;
733
734 intel_ring_advance(rq, cs);
735
736 err = 0;
737 err:
738 i915_request_get(rq);
739 i915_request_add(rq);
740 if (err) {
741 i915_request_put(rq);
742 return ERR_PTR(err);
743 }
744
745 return rq;
746 }
747
748 struct lrc_timestamp {
749 struct intel_engine_cs *engine;
750 struct intel_context *ce[2];
751 u32 poison;
752 };
753
timestamp_advanced(u32 start,u32 end)754 static bool timestamp_advanced(u32 start, u32 end)
755 {
756 return (s32)(end - start) > 0;
757 }
758
__lrc_timestamp(const struct lrc_timestamp * arg,bool preempt)759 static int __lrc_timestamp(const struct lrc_timestamp *arg, bool preempt)
760 {
761 u32 *slot = memset32(arg->engine->status_page.addr + 1000, 0, 4);
762 struct i915_request *rq;
763 u32 timestamp;
764 int err = 0;
765
766 arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP] = arg->poison;
767 rq = create_timestamp(arg->ce[0], slot, 1);
768 if (IS_ERR(rq))
769 return PTR_ERR(rq);
770
771 err = wait_for_submit(rq->engine, rq, HZ / 2);
772 if (err)
773 goto err;
774
775 if (preempt) {
776 arg->ce[1]->lrc_reg_state[CTX_TIMESTAMP] = 0xdeadbeef;
777 err = emit_semaphore_signal(arg->ce[1], slot);
778 if (err)
779 goto err;
780 } else {
781 slot[0] = 1;
782 wmb();
783 }
784
785 /* And wait for switch to kernel (to save our context to memory) */
786 err = context_flush(arg->ce[0], HZ / 2);
787 if (err)
788 goto err;
789
790 if (!timestamp_advanced(arg->poison, slot[1])) {
791 pr_err("%s(%s): invalid timestamp on restore, context:%x, request:%x\n",
792 arg->engine->name, preempt ? "preempt" : "simple",
793 arg->poison, slot[1]);
794 err = -EINVAL;
795 }
796
797 timestamp = READ_ONCE(arg->ce[0]->lrc_reg_state[CTX_TIMESTAMP]);
798 if (!timestamp_advanced(slot[1], timestamp)) {
799 pr_err("%s(%s): invalid timestamp on save, request:%x, context:%x\n",
800 arg->engine->name, preempt ? "preempt" : "simple",
801 slot[1], timestamp);
802 err = -EINVAL;
803 }
804
805 err:
806 memset32(slot, -1, 4);
807 i915_request_put(rq);
808 return err;
809 }
810
live_lrc_timestamp(void * arg)811 static int live_lrc_timestamp(void *arg)
812 {
813 struct lrc_timestamp data = {};
814 struct intel_gt *gt = arg;
815 enum intel_engine_id id;
816 const u32 poison[] = {
817 0,
818 S32_MAX,
819 (u32)S32_MAX + 1,
820 U32_MAX,
821 };
822
823 /*
824 * We want to verify that the timestamp is saved and restore across
825 * context switches and is monotonic.
826 *
827 * So we do this with a little bit of LRC poisoning to check various
828 * boundary conditions, and see what happens if we preempt the context
829 * with a second request (carrying more poison into the timestamp).
830 */
831
832 for_each_engine(data.engine, gt, id) {
833 int i, err = 0;
834
835 st_engine_heartbeat_disable(data.engine);
836
837 for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
838 struct intel_context *tmp;
839
840 tmp = intel_context_create(data.engine);
841 if (IS_ERR(tmp)) {
842 err = PTR_ERR(tmp);
843 goto err;
844 }
845
846 err = intel_context_pin(tmp);
847 if (err) {
848 intel_context_put(tmp);
849 goto err;
850 }
851
852 data.ce[i] = tmp;
853 }
854
855 for (i = 0; i < ARRAY_SIZE(poison); i++) {
856 data.poison = poison[i];
857
858 err = __lrc_timestamp(&data, false);
859 if (err)
860 break;
861
862 err = __lrc_timestamp(&data, true);
863 if (err)
864 break;
865 }
866
867 err:
868 st_engine_heartbeat_enable(data.engine);
869 for (i = 0; i < ARRAY_SIZE(data.ce); i++) {
870 if (!data.ce[i])
871 break;
872
873 intel_context_unpin(data.ce[i]);
874 intel_context_put(data.ce[i]);
875 }
876
877 if (igt_flush_test(gt->i915))
878 err = -EIO;
879 if (err)
880 return err;
881 }
882
883 return 0;
884 }
885
886 static struct i915_vma *
create_user_vma(struct i915_address_space * vm,unsigned long size)887 create_user_vma(struct i915_address_space *vm, unsigned long size)
888 {
889 struct drm_i915_gem_object *obj;
890 struct i915_vma *vma;
891 int err;
892
893 obj = i915_gem_object_create_internal(vm->i915, size);
894 if (IS_ERR(obj))
895 return ERR_CAST(obj);
896
897 vma = i915_vma_instance(obj, vm, NULL);
898 if (IS_ERR(vma)) {
899 i915_gem_object_put(obj);
900 return vma;
901 }
902
903 err = i915_vma_pin(vma, 0, 0, PIN_USER);
904 if (err) {
905 i915_gem_object_put(obj);
906 return ERR_PTR(err);
907 }
908
909 return vma;
910 }
911
912 static struct i915_vma *
store_context(struct intel_context * ce,struct i915_vma * scratch)913 store_context(struct intel_context *ce, struct i915_vma *scratch)
914 {
915 struct i915_vma *batch;
916 u32 dw, x, *cs, *hw;
917 u32 *defaults;
918
919 batch = create_user_vma(ce->vm, SZ_64K);
920 if (IS_ERR(batch))
921 return batch;
922
923 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
924 if (IS_ERR(cs)) {
925 i915_vma_put(batch);
926 return ERR_CAST(cs);
927 }
928
929 defaults = shmem_pin_map(ce->engine->default_state);
930 if (!defaults) {
931 i915_gem_object_unpin_map(batch->obj);
932 i915_vma_put(batch);
933 return ERR_PTR(-ENOMEM);
934 }
935
936 x = 0;
937 dw = 0;
938 hw = defaults;
939 hw += LRC_STATE_OFFSET / sizeof(*hw);
940 do {
941 u32 len = hw[dw] & 0x7f;
942
943 if (hw[dw] == 0) {
944 dw++;
945 continue;
946 }
947
948 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
949 dw += len + 2;
950 continue;
951 }
952
953 dw++;
954 len = (len + 1) / 2;
955 while (len--) {
956 *cs++ = MI_STORE_REGISTER_MEM_GEN8;
957 *cs++ = hw[dw];
958 *cs++ = lower_32_bits(scratch->node.start + x);
959 *cs++ = upper_32_bits(scratch->node.start + x);
960
961 dw += 2;
962 x += 4;
963 }
964 } while (dw < PAGE_SIZE / sizeof(u32) &&
965 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
966
967 *cs++ = MI_BATCH_BUFFER_END;
968
969 shmem_unpin_map(ce->engine->default_state, defaults);
970
971 i915_gem_object_flush_map(batch->obj);
972 i915_gem_object_unpin_map(batch->obj);
973
974 return batch;
975 }
976
move_to_active(struct i915_request * rq,struct i915_vma * vma,unsigned int flags)977 static int move_to_active(struct i915_request *rq,
978 struct i915_vma *vma,
979 unsigned int flags)
980 {
981 int err;
982
983 i915_vma_lock(vma);
984 err = i915_request_await_object(rq, vma->obj, flags);
985 if (!err)
986 err = i915_vma_move_to_active(vma, rq, flags);
987 i915_vma_unlock(vma);
988
989 return err;
990 }
991
992 static struct i915_request *
record_registers(struct intel_context * ce,struct i915_vma * before,struct i915_vma * after,u32 * sema)993 record_registers(struct intel_context *ce,
994 struct i915_vma *before,
995 struct i915_vma *after,
996 u32 *sema)
997 {
998 struct i915_vma *b_before, *b_after;
999 struct i915_request *rq;
1000 u32 *cs;
1001 int err;
1002
1003 b_before = store_context(ce, before);
1004 if (IS_ERR(b_before))
1005 return ERR_CAST(b_before);
1006
1007 b_after = store_context(ce, after);
1008 if (IS_ERR(b_after)) {
1009 rq = ERR_CAST(b_after);
1010 goto err_before;
1011 }
1012
1013 rq = intel_context_create_request(ce);
1014 if (IS_ERR(rq))
1015 goto err_after;
1016
1017 err = move_to_active(rq, before, EXEC_OBJECT_WRITE);
1018 if (err)
1019 goto err_rq;
1020
1021 err = move_to_active(rq, b_before, 0);
1022 if (err)
1023 goto err_rq;
1024
1025 err = move_to_active(rq, after, EXEC_OBJECT_WRITE);
1026 if (err)
1027 goto err_rq;
1028
1029 err = move_to_active(rq, b_after, 0);
1030 if (err)
1031 goto err_rq;
1032
1033 cs = intel_ring_begin(rq, 14);
1034 if (IS_ERR(cs)) {
1035 err = PTR_ERR(cs);
1036 goto err_rq;
1037 }
1038
1039 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1040 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1041 *cs++ = lower_32_bits(b_before->node.start);
1042 *cs++ = upper_32_bits(b_before->node.start);
1043
1044 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE;
1045 *cs++ = MI_SEMAPHORE_WAIT |
1046 MI_SEMAPHORE_GLOBAL_GTT |
1047 MI_SEMAPHORE_POLL |
1048 MI_SEMAPHORE_SAD_NEQ_SDD;
1049 *cs++ = 0;
1050 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1051 offset_in_page(sema);
1052 *cs++ = 0;
1053 *cs++ = MI_NOOP;
1054
1055 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1056 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1057 *cs++ = lower_32_bits(b_after->node.start);
1058 *cs++ = upper_32_bits(b_after->node.start);
1059
1060 intel_ring_advance(rq, cs);
1061
1062 WRITE_ONCE(*sema, 0);
1063 i915_request_get(rq);
1064 i915_request_add(rq);
1065 err_after:
1066 i915_vma_put(b_after);
1067 err_before:
1068 i915_vma_put(b_before);
1069 return rq;
1070
1071 err_rq:
1072 i915_request_add(rq);
1073 rq = ERR_PTR(err);
1074 goto err_after;
1075 }
1076
load_context(struct intel_context * ce,u32 poison)1077 static struct i915_vma *load_context(struct intel_context *ce, u32 poison)
1078 {
1079 struct i915_vma *batch;
1080 u32 dw, *cs, *hw;
1081 u32 *defaults;
1082
1083 batch = create_user_vma(ce->vm, SZ_64K);
1084 if (IS_ERR(batch))
1085 return batch;
1086
1087 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
1088 if (IS_ERR(cs)) {
1089 i915_vma_put(batch);
1090 return ERR_CAST(cs);
1091 }
1092
1093 defaults = shmem_pin_map(ce->engine->default_state);
1094 if (!defaults) {
1095 i915_gem_object_unpin_map(batch->obj);
1096 i915_vma_put(batch);
1097 return ERR_PTR(-ENOMEM);
1098 }
1099
1100 dw = 0;
1101 hw = defaults;
1102 hw += LRC_STATE_OFFSET / sizeof(*hw);
1103 do {
1104 u32 len = hw[dw] & 0x7f;
1105
1106 if (hw[dw] == 0) {
1107 dw++;
1108 continue;
1109 }
1110
1111 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1112 dw += len + 2;
1113 continue;
1114 }
1115
1116 dw++;
1117 len = (len + 1) / 2;
1118 *cs++ = MI_LOAD_REGISTER_IMM(len);
1119 while (len--) {
1120 *cs++ = hw[dw];
1121 *cs++ = poison;
1122 dw += 2;
1123 }
1124 } while (dw < PAGE_SIZE / sizeof(u32) &&
1125 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1126
1127 *cs++ = MI_BATCH_BUFFER_END;
1128
1129 shmem_unpin_map(ce->engine->default_state, defaults);
1130
1131 i915_gem_object_flush_map(batch->obj);
1132 i915_gem_object_unpin_map(batch->obj);
1133
1134 return batch;
1135 }
1136
poison_registers(struct intel_context * ce,u32 poison,u32 * sema)1137 static int poison_registers(struct intel_context *ce, u32 poison, u32 *sema)
1138 {
1139 struct i915_request *rq;
1140 struct i915_vma *batch;
1141 u32 *cs;
1142 int err;
1143
1144 batch = load_context(ce, poison);
1145 if (IS_ERR(batch))
1146 return PTR_ERR(batch);
1147
1148 rq = intel_context_create_request(ce);
1149 if (IS_ERR(rq)) {
1150 err = PTR_ERR(rq);
1151 goto err_batch;
1152 }
1153
1154 err = move_to_active(rq, batch, 0);
1155 if (err)
1156 goto err_rq;
1157
1158 cs = intel_ring_begin(rq, 8);
1159 if (IS_ERR(cs)) {
1160 err = PTR_ERR(cs);
1161 goto err_rq;
1162 }
1163
1164 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE;
1165 *cs++ = MI_BATCH_BUFFER_START_GEN8 | BIT(8);
1166 *cs++ = lower_32_bits(batch->node.start);
1167 *cs++ = upper_32_bits(batch->node.start);
1168
1169 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1170 *cs++ = i915_ggtt_offset(ce->engine->status_page.vma) +
1171 offset_in_page(sema);
1172 *cs++ = 0;
1173 *cs++ = 1;
1174
1175 intel_ring_advance(rq, cs);
1176
1177 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
1178 err_rq:
1179 i915_request_add(rq);
1180 err_batch:
1181 i915_vma_put(batch);
1182 return err;
1183 }
1184
is_moving(u32 a,u32 b)1185 static bool is_moving(u32 a, u32 b)
1186 {
1187 return a != b;
1188 }
1189
compare_isolation(struct intel_engine_cs * engine,struct i915_vma * ref[2],struct i915_vma * result[2],struct intel_context * ce,u32 poison)1190 static int compare_isolation(struct intel_engine_cs *engine,
1191 struct i915_vma *ref[2],
1192 struct i915_vma *result[2],
1193 struct intel_context *ce,
1194 u32 poison)
1195 {
1196 u32 x, dw, *hw, *lrc;
1197 u32 *A[2], *B[2];
1198 u32 *defaults;
1199 int err = 0;
1200
1201 A[0] = i915_gem_object_pin_map_unlocked(ref[0]->obj, I915_MAP_WC);
1202 if (IS_ERR(A[0]))
1203 return PTR_ERR(A[0]);
1204
1205 A[1] = i915_gem_object_pin_map_unlocked(ref[1]->obj, I915_MAP_WC);
1206 if (IS_ERR(A[1])) {
1207 err = PTR_ERR(A[1]);
1208 goto err_A0;
1209 }
1210
1211 B[0] = i915_gem_object_pin_map_unlocked(result[0]->obj, I915_MAP_WC);
1212 if (IS_ERR(B[0])) {
1213 err = PTR_ERR(B[0]);
1214 goto err_A1;
1215 }
1216
1217 B[1] = i915_gem_object_pin_map_unlocked(result[1]->obj, I915_MAP_WC);
1218 if (IS_ERR(B[1])) {
1219 err = PTR_ERR(B[1]);
1220 goto err_B0;
1221 }
1222
1223 lrc = i915_gem_object_pin_map_unlocked(ce->state->obj,
1224 i915_coherent_map_type(engine->i915));
1225 if (IS_ERR(lrc)) {
1226 err = PTR_ERR(lrc);
1227 goto err_B1;
1228 }
1229 lrc += LRC_STATE_OFFSET / sizeof(*hw);
1230
1231 defaults = shmem_pin_map(ce->engine->default_state);
1232 if (!defaults) {
1233 err = -ENOMEM;
1234 goto err_lrc;
1235 }
1236
1237 x = 0;
1238 dw = 0;
1239 hw = defaults;
1240 hw += LRC_STATE_OFFSET / sizeof(*hw);
1241 do {
1242 u32 len = hw[dw] & 0x7f;
1243
1244 if (hw[dw] == 0) {
1245 dw++;
1246 continue;
1247 }
1248
1249 if ((hw[dw] & GENMASK(31, 23)) != MI_INSTR(0x22, 0)) {
1250 dw += len + 2;
1251 continue;
1252 }
1253
1254 dw++;
1255 len = (len + 1) / 2;
1256 while (len--) {
1257 if (!is_moving(A[0][x], A[1][x]) &&
1258 (A[0][x] != B[0][x] || A[1][x] != B[1][x])) {
1259 switch (hw[dw] & 4095) {
1260 case 0x30: /* RING_HEAD */
1261 case 0x34: /* RING_TAIL */
1262 break;
1263
1264 default:
1265 pr_err("%s[%d]: Mismatch for register %4x, default %08x, reference %08x, result (%08x, %08x), poison %08x, context %08x\n",
1266 engine->name, dw,
1267 hw[dw], hw[dw + 1],
1268 A[0][x], B[0][x], B[1][x],
1269 poison, lrc[dw + 1]);
1270 err = -EINVAL;
1271 }
1272 }
1273 dw += 2;
1274 x++;
1275 }
1276 } while (dw < PAGE_SIZE / sizeof(u32) &&
1277 (hw[dw] & ~BIT(0)) != MI_BATCH_BUFFER_END);
1278
1279 shmem_unpin_map(ce->engine->default_state, defaults);
1280 err_lrc:
1281 i915_gem_object_unpin_map(ce->state->obj);
1282 err_B1:
1283 i915_gem_object_unpin_map(result[1]->obj);
1284 err_B0:
1285 i915_gem_object_unpin_map(result[0]->obj);
1286 err_A1:
1287 i915_gem_object_unpin_map(ref[1]->obj);
1288 err_A0:
1289 i915_gem_object_unpin_map(ref[0]->obj);
1290 return err;
1291 }
1292
__lrc_isolation(struct intel_engine_cs * engine,u32 poison)1293 static int __lrc_isolation(struct intel_engine_cs *engine, u32 poison)
1294 {
1295 u32 *sema = memset32(engine->status_page.addr + 1000, 0, 1);
1296 struct i915_vma *ref[2], *result[2];
1297 struct intel_context *A, *B;
1298 struct i915_request *rq;
1299 int err;
1300
1301 A = intel_context_create(engine);
1302 if (IS_ERR(A))
1303 return PTR_ERR(A);
1304
1305 B = intel_context_create(engine);
1306 if (IS_ERR(B)) {
1307 err = PTR_ERR(B);
1308 goto err_A;
1309 }
1310
1311 ref[0] = create_user_vma(A->vm, SZ_64K);
1312 if (IS_ERR(ref[0])) {
1313 err = PTR_ERR(ref[0]);
1314 goto err_B;
1315 }
1316
1317 ref[1] = create_user_vma(A->vm, SZ_64K);
1318 if (IS_ERR(ref[1])) {
1319 err = PTR_ERR(ref[1]);
1320 goto err_ref0;
1321 }
1322
1323 rq = record_registers(A, ref[0], ref[1], sema);
1324 if (IS_ERR(rq)) {
1325 err = PTR_ERR(rq);
1326 goto err_ref1;
1327 }
1328
1329 WRITE_ONCE(*sema, 1);
1330 wmb();
1331
1332 if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1333 i915_request_put(rq);
1334 err = -ETIME;
1335 goto err_ref1;
1336 }
1337 i915_request_put(rq);
1338
1339 result[0] = create_user_vma(A->vm, SZ_64K);
1340 if (IS_ERR(result[0])) {
1341 err = PTR_ERR(result[0]);
1342 goto err_ref1;
1343 }
1344
1345 result[1] = create_user_vma(A->vm, SZ_64K);
1346 if (IS_ERR(result[1])) {
1347 err = PTR_ERR(result[1]);
1348 goto err_result0;
1349 }
1350
1351 rq = record_registers(A, result[0], result[1], sema);
1352 if (IS_ERR(rq)) {
1353 err = PTR_ERR(rq);
1354 goto err_result1;
1355 }
1356
1357 err = poison_registers(B, poison, sema);
1358 if (err) {
1359 WRITE_ONCE(*sema, -1);
1360 i915_request_put(rq);
1361 goto err_result1;
1362 }
1363
1364 if (i915_request_wait(rq, 0, HZ / 2) < 0) {
1365 i915_request_put(rq);
1366 err = -ETIME;
1367 goto err_result1;
1368 }
1369 i915_request_put(rq);
1370
1371 err = compare_isolation(engine, ref, result, A, poison);
1372
1373 err_result1:
1374 i915_vma_put(result[1]);
1375 err_result0:
1376 i915_vma_put(result[0]);
1377 err_ref1:
1378 i915_vma_put(ref[1]);
1379 err_ref0:
1380 i915_vma_put(ref[0]);
1381 err_B:
1382 intel_context_put(B);
1383 err_A:
1384 intel_context_put(A);
1385 return err;
1386 }
1387
skip_isolation(const struct intel_engine_cs * engine)1388 static bool skip_isolation(const struct intel_engine_cs *engine)
1389 {
1390 if (engine->class == COPY_ENGINE_CLASS && INTEL_GEN(engine->i915) == 9)
1391 return true;
1392
1393 if (engine->class == RENDER_CLASS && INTEL_GEN(engine->i915) == 11)
1394 return true;
1395
1396 return false;
1397 }
1398
live_lrc_isolation(void * arg)1399 static int live_lrc_isolation(void *arg)
1400 {
1401 struct intel_gt *gt = arg;
1402 struct intel_engine_cs *engine;
1403 enum intel_engine_id id;
1404 const u32 poison[] = {
1405 STACK_MAGIC,
1406 0x3a3a3a3a,
1407 0x5c5c5c5c,
1408 0xffffffff,
1409 0xffff0000,
1410 };
1411 int err = 0;
1412
1413 /*
1414 * Our goal is try and verify that per-context state cannot be
1415 * tampered with by another non-privileged client.
1416 *
1417 * We take the list of context registers from the LRI in the default
1418 * context image and attempt to modify that list from a remote context.
1419 */
1420
1421 for_each_engine(engine, gt, id) {
1422 int i;
1423
1424 /* Just don't even ask */
1425 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN) &&
1426 skip_isolation(engine))
1427 continue;
1428
1429 intel_engine_pm_get(engine);
1430 for (i = 0; i < ARRAY_SIZE(poison); i++) {
1431 int result;
1432
1433 result = __lrc_isolation(engine, poison[i]);
1434 if (result && !err)
1435 err = result;
1436
1437 result = __lrc_isolation(engine, ~poison[i]);
1438 if (result && !err)
1439 err = result;
1440 }
1441 intel_engine_pm_put(engine);
1442 if (igt_flush_test(gt->i915)) {
1443 err = -EIO;
1444 break;
1445 }
1446 }
1447
1448 return err;
1449 }
1450
indirect_ctx_submit_req(struct intel_context * ce)1451 static int indirect_ctx_submit_req(struct intel_context *ce)
1452 {
1453 struct i915_request *rq;
1454 int err = 0;
1455
1456 rq = intel_context_create_request(ce);
1457 if (IS_ERR(rq))
1458 return PTR_ERR(rq);
1459
1460 i915_request_get(rq);
1461 i915_request_add(rq);
1462
1463 if (i915_request_wait(rq, 0, HZ / 5) < 0)
1464 err = -ETIME;
1465
1466 i915_request_put(rq);
1467
1468 return err;
1469 }
1470
1471 #define CTX_BB_CANARY_OFFSET (3 * 1024)
1472 #define CTX_BB_CANARY_INDEX (CTX_BB_CANARY_OFFSET / sizeof(u32))
1473
1474 static u32 *
emit_indirect_ctx_bb_canary(const struct intel_context * ce,u32 * cs)1475 emit_indirect_ctx_bb_canary(const struct intel_context *ce, u32 *cs)
1476 {
1477 *cs++ = MI_STORE_REGISTER_MEM_GEN8 |
1478 MI_SRM_LRM_GLOBAL_GTT |
1479 MI_LRI_LRM_CS_MMIO;
1480 *cs++ = i915_mmio_reg_offset(RING_START(0));
1481 *cs++ = i915_ggtt_offset(ce->state) +
1482 context_wa_bb_offset(ce) +
1483 CTX_BB_CANARY_OFFSET;
1484 *cs++ = 0;
1485
1486 return cs;
1487 }
1488
1489 static void
indirect_ctx_bb_setup(struct intel_context * ce)1490 indirect_ctx_bb_setup(struct intel_context *ce)
1491 {
1492 u32 *cs = context_indirect_bb(ce);
1493
1494 cs[CTX_BB_CANARY_INDEX] = 0xdeadf00d;
1495
1496 setup_indirect_ctx_bb(ce, ce->engine, emit_indirect_ctx_bb_canary);
1497 }
1498
check_ring_start(struct intel_context * ce)1499 static bool check_ring_start(struct intel_context *ce)
1500 {
1501 const u32 * const ctx_bb = (void *)(ce->lrc_reg_state) -
1502 LRC_STATE_OFFSET + context_wa_bb_offset(ce);
1503
1504 if (ctx_bb[CTX_BB_CANARY_INDEX] == ce->lrc_reg_state[CTX_RING_START])
1505 return true;
1506
1507 pr_err("ring start mismatch: canary 0x%08x vs state 0x%08x\n",
1508 ctx_bb[CTX_BB_CANARY_INDEX],
1509 ce->lrc_reg_state[CTX_RING_START]);
1510
1511 return false;
1512 }
1513
indirect_ctx_bb_check(struct intel_context * ce)1514 static int indirect_ctx_bb_check(struct intel_context *ce)
1515 {
1516 int err;
1517
1518 err = indirect_ctx_submit_req(ce);
1519 if (err)
1520 return err;
1521
1522 if (!check_ring_start(ce))
1523 return -EINVAL;
1524
1525 return 0;
1526 }
1527
__live_lrc_indirect_ctx_bb(struct intel_engine_cs * engine)1528 static int __live_lrc_indirect_ctx_bb(struct intel_engine_cs *engine)
1529 {
1530 struct intel_context *a, *b;
1531 int err;
1532
1533 a = intel_context_create(engine);
1534 if (IS_ERR(a))
1535 return PTR_ERR(a);
1536 err = intel_context_pin(a);
1537 if (err)
1538 goto put_a;
1539
1540 b = intel_context_create(engine);
1541 if (IS_ERR(b)) {
1542 err = PTR_ERR(b);
1543 goto unpin_a;
1544 }
1545 err = intel_context_pin(b);
1546 if (err)
1547 goto put_b;
1548
1549 /* We use the already reserved extra page in context state */
1550 if (!a->wa_bb_page) {
1551 GEM_BUG_ON(b->wa_bb_page);
1552 GEM_BUG_ON(INTEL_GEN(engine->i915) == 12);
1553 goto unpin_b;
1554 }
1555
1556 /*
1557 * In order to test that our per context bb is truly per context,
1558 * and executes at the intended spot on context restoring process,
1559 * make the batch store the ring start value to memory.
1560 * As ring start is restored apriori of starting the indirect ctx bb and
1561 * as it will be different for each context, it fits to this purpose.
1562 */
1563 indirect_ctx_bb_setup(a);
1564 indirect_ctx_bb_setup(b);
1565
1566 err = indirect_ctx_bb_check(a);
1567 if (err)
1568 goto unpin_b;
1569
1570 err = indirect_ctx_bb_check(b);
1571
1572 unpin_b:
1573 intel_context_unpin(b);
1574 put_b:
1575 intel_context_put(b);
1576 unpin_a:
1577 intel_context_unpin(a);
1578 put_a:
1579 intel_context_put(a);
1580
1581 return err;
1582 }
1583
live_lrc_indirect_ctx_bb(void * arg)1584 static int live_lrc_indirect_ctx_bb(void *arg)
1585 {
1586 struct intel_gt *gt = arg;
1587 struct intel_engine_cs *engine;
1588 enum intel_engine_id id;
1589 int err = 0;
1590
1591 for_each_engine(engine, gt, id) {
1592 intel_engine_pm_get(engine);
1593 err = __live_lrc_indirect_ctx_bb(engine);
1594 intel_engine_pm_put(engine);
1595
1596 if (igt_flush_test(gt->i915))
1597 err = -EIO;
1598
1599 if (err)
1600 break;
1601 }
1602
1603 return err;
1604 }
1605
garbage_reset(struct intel_engine_cs * engine,struct i915_request * rq)1606 static void garbage_reset(struct intel_engine_cs *engine,
1607 struct i915_request *rq)
1608 {
1609 const unsigned int bit = I915_RESET_ENGINE + engine->id;
1610 unsigned long *lock = &engine->gt->reset.flags;
1611
1612 local_bh_disable();
1613 if (!test_and_set_bit(bit, lock)) {
1614 tasklet_disable(&engine->execlists.tasklet);
1615
1616 if (!rq->fence.error)
1617 __intel_engine_reset_bh(engine, NULL);
1618
1619 tasklet_enable(&engine->execlists.tasklet);
1620 clear_and_wake_up_bit(bit, lock);
1621 }
1622 local_bh_enable();
1623 }
1624
garbage(struct intel_context * ce,struct rnd_state * prng)1625 static struct i915_request *garbage(struct intel_context *ce,
1626 struct rnd_state *prng)
1627 {
1628 struct i915_request *rq;
1629 int err;
1630
1631 err = intel_context_pin(ce);
1632 if (err)
1633 return ERR_PTR(err);
1634
1635 prandom_bytes_state(prng,
1636 ce->lrc_reg_state,
1637 ce->engine->context_size -
1638 LRC_STATE_OFFSET);
1639
1640 rq = intel_context_create_request(ce);
1641 if (IS_ERR(rq)) {
1642 err = PTR_ERR(rq);
1643 goto err_unpin;
1644 }
1645
1646 i915_request_get(rq);
1647 i915_request_add(rq);
1648 return rq;
1649
1650 err_unpin:
1651 intel_context_unpin(ce);
1652 return ERR_PTR(err);
1653 }
1654
__lrc_garbage(struct intel_engine_cs * engine,struct rnd_state * prng)1655 static int __lrc_garbage(struct intel_engine_cs *engine, struct rnd_state *prng)
1656 {
1657 struct intel_context *ce;
1658 struct i915_request *hang;
1659 int err = 0;
1660
1661 ce = intel_context_create(engine);
1662 if (IS_ERR(ce))
1663 return PTR_ERR(ce);
1664
1665 hang = garbage(ce, prng);
1666 if (IS_ERR(hang)) {
1667 err = PTR_ERR(hang);
1668 goto err_ce;
1669 }
1670
1671 if (wait_for_submit(engine, hang, HZ / 2)) {
1672 i915_request_put(hang);
1673 err = -ETIME;
1674 goto err_ce;
1675 }
1676
1677 intel_context_set_banned(ce);
1678 garbage_reset(engine, hang);
1679
1680 intel_engine_flush_submission(engine);
1681 if (!hang->fence.error) {
1682 i915_request_put(hang);
1683 pr_err("%s: corrupted context was not reset\n",
1684 engine->name);
1685 err = -EINVAL;
1686 goto err_ce;
1687 }
1688
1689 if (i915_request_wait(hang, 0, HZ / 2) < 0) {
1690 pr_err("%s: corrupted context did not recover\n",
1691 engine->name);
1692 i915_request_put(hang);
1693 err = -EIO;
1694 goto err_ce;
1695 }
1696 i915_request_put(hang);
1697
1698 err_ce:
1699 intel_context_put(ce);
1700 return err;
1701 }
1702
live_lrc_garbage(void * arg)1703 static int live_lrc_garbage(void *arg)
1704 {
1705 struct intel_gt *gt = arg;
1706 struct intel_engine_cs *engine;
1707 enum intel_engine_id id;
1708
1709 /*
1710 * Verify that we can recover if one context state is completely
1711 * corrupted.
1712 */
1713
1714 if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN))
1715 return 0;
1716
1717 for_each_engine(engine, gt, id) {
1718 I915_RND_STATE(prng);
1719 int err = 0, i;
1720
1721 if (!intel_has_reset_engine(engine->gt))
1722 continue;
1723
1724 intel_engine_pm_get(engine);
1725 for (i = 0; i < 3; i++) {
1726 err = __lrc_garbage(engine, &prng);
1727 if (err)
1728 break;
1729 }
1730 intel_engine_pm_put(engine);
1731
1732 if (igt_flush_test(gt->i915))
1733 err = -EIO;
1734 if (err)
1735 return err;
1736 }
1737
1738 return 0;
1739 }
1740
__live_pphwsp_runtime(struct intel_engine_cs * engine)1741 static int __live_pphwsp_runtime(struct intel_engine_cs *engine)
1742 {
1743 struct intel_context *ce;
1744 struct i915_request *rq;
1745 IGT_TIMEOUT(end_time);
1746 int err;
1747
1748 ce = intel_context_create(engine);
1749 if (IS_ERR(ce))
1750 return PTR_ERR(ce);
1751
1752 ce->runtime.num_underflow = 0;
1753 ce->runtime.max_underflow = 0;
1754
1755 do {
1756 unsigned int loop = 1024;
1757
1758 while (loop) {
1759 rq = intel_context_create_request(ce);
1760 if (IS_ERR(rq)) {
1761 err = PTR_ERR(rq);
1762 goto err_rq;
1763 }
1764
1765 if (--loop == 0)
1766 i915_request_get(rq);
1767
1768 i915_request_add(rq);
1769 }
1770
1771 if (__igt_timeout(end_time, NULL))
1772 break;
1773
1774 i915_request_put(rq);
1775 } while (1);
1776
1777 err = i915_request_wait(rq, 0, HZ / 5);
1778 if (err < 0) {
1779 pr_err("%s: request not completed!\n", engine->name);
1780 goto err_wait;
1781 }
1782
1783 igt_flush_test(engine->i915);
1784
1785 pr_info("%s: pphwsp runtime %lluns, average %lluns\n",
1786 engine->name,
1787 intel_context_get_total_runtime_ns(ce),
1788 intel_context_get_avg_runtime_ns(ce));
1789
1790 err = 0;
1791 if (ce->runtime.num_underflow) {
1792 pr_err("%s: pphwsp underflow %u time(s), max %u cycles!\n",
1793 engine->name,
1794 ce->runtime.num_underflow,
1795 ce->runtime.max_underflow);
1796 GEM_TRACE_DUMP();
1797 err = -EOVERFLOW;
1798 }
1799
1800 err_wait:
1801 i915_request_put(rq);
1802 err_rq:
1803 intel_context_put(ce);
1804 return err;
1805 }
1806
live_pphwsp_runtime(void * arg)1807 static int live_pphwsp_runtime(void *arg)
1808 {
1809 struct intel_gt *gt = arg;
1810 struct intel_engine_cs *engine;
1811 enum intel_engine_id id;
1812 int err = 0;
1813
1814 /*
1815 * Check that cumulative context runtime as stored in the pphwsp[16]
1816 * is monotonic.
1817 */
1818
1819 for_each_engine(engine, gt, id) {
1820 err = __live_pphwsp_runtime(engine);
1821 if (err)
1822 break;
1823 }
1824
1825 if (igt_flush_test(gt->i915))
1826 err = -EIO;
1827
1828 return err;
1829 }
1830
intel_lrc_live_selftests(struct drm_i915_private * i915)1831 int intel_lrc_live_selftests(struct drm_i915_private *i915)
1832 {
1833 static const struct i915_subtest tests[] = {
1834 SUBTEST(live_lrc_layout),
1835 SUBTEST(live_lrc_fixed),
1836 SUBTEST(live_lrc_state),
1837 SUBTEST(live_lrc_gpr),
1838 SUBTEST(live_lrc_isolation),
1839 SUBTEST(live_lrc_timestamp),
1840 SUBTEST(live_lrc_garbage),
1841 SUBTEST(live_pphwsp_runtime),
1842 SUBTEST(live_lrc_indirect_ctx_bb),
1843 };
1844
1845 if (!HAS_LOGICAL_RING_CONTEXTS(i915))
1846 return 0;
1847
1848 return intel_gt_live_subtests(tests, &i915->gt);
1849 }
1850