1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2018 Intel Corporation
4 */
5
6 #include "gem/i915_gem_pm.h"
7 #include "gt/intel_engine_user.h"
8 #include "gt/intel_gt.h"
9 #include "i915_selftest.h"
10 #include "intel_reset.h"
11
12 #include "selftests/igt_flush_test.h"
13 #include "selftests/igt_reset.h"
14 #include "selftests/igt_spinner.h"
15 #include "selftests/mock_drm.h"
16
17 #include "gem/selftests/igt_gem_utils.h"
18 #include "gem/selftests/mock_context.h"
19
20 static const struct wo_register {
21 enum intel_platform platform;
22 u32 reg;
23 } wo_registers[] = {
24 { INTEL_GEMINILAKE, 0x731c }
25 };
26
27 struct wa_lists {
28 struct i915_wa_list gt_wa_list;
29 struct {
30 struct i915_wa_list wa_list;
31 struct i915_wa_list ctx_wa_list;
32 } engine[I915_NUM_ENGINES];
33 };
34
request_add_sync(struct i915_request * rq,int err)35 static int request_add_sync(struct i915_request *rq, int err)
36 {
37 i915_request_get(rq);
38 i915_request_add(rq);
39 if (i915_request_wait(rq, 0, HZ / 5) < 0)
40 err = -EIO;
41 i915_request_put(rq);
42
43 return err;
44 }
45
request_add_spin(struct i915_request * rq,struct igt_spinner * spin)46 static int request_add_spin(struct i915_request *rq, struct igt_spinner *spin)
47 {
48 int err = 0;
49
50 i915_request_get(rq);
51 i915_request_add(rq);
52 if (spin && !igt_wait_for_spinner(spin, rq))
53 err = -ETIMEDOUT;
54 i915_request_put(rq);
55
56 return err;
57 }
58
59 static void
reference_lists_init(struct intel_gt * gt,struct wa_lists * lists)60 reference_lists_init(struct intel_gt *gt, struct wa_lists *lists)
61 {
62 struct intel_engine_cs *engine;
63 enum intel_engine_id id;
64
65 memset(lists, 0, sizeof(*lists));
66
67 wa_init_start(&lists->gt_wa_list, "GT_REF", "global");
68 gt_init_workarounds(gt->i915, &lists->gt_wa_list);
69 wa_init_finish(&lists->gt_wa_list);
70
71 for_each_engine(engine, gt, id) {
72 struct i915_wa_list *wal = &lists->engine[id].wa_list;
73
74 wa_init_start(wal, "REF", engine->name);
75 engine_init_workarounds(engine, wal);
76 wa_init_finish(wal);
77
78 __intel_engine_init_ctx_wa(engine,
79 &lists->engine[id].ctx_wa_list,
80 "CTX_REF");
81 }
82 }
83
84 static void
reference_lists_fini(struct intel_gt * gt,struct wa_lists * lists)85 reference_lists_fini(struct intel_gt *gt, struct wa_lists *lists)
86 {
87 struct intel_engine_cs *engine;
88 enum intel_engine_id id;
89
90 for_each_engine(engine, gt, id)
91 intel_wa_list_free(&lists->engine[id].wa_list);
92
93 intel_wa_list_free(&lists->gt_wa_list);
94 }
95
96 static struct drm_i915_gem_object *
read_nonprivs(struct intel_context * ce)97 read_nonprivs(struct intel_context *ce)
98 {
99 struct intel_engine_cs *engine = ce->engine;
100 const u32 base = engine->mmio_base;
101 struct drm_i915_gem_object *result;
102 struct i915_request *rq;
103 struct i915_vma *vma;
104 u32 srm, *cs;
105 int err;
106 int i;
107
108 result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
109 if (IS_ERR(result))
110 return result;
111
112 i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC);
113
114 cs = i915_gem_object_pin_map_unlocked(result, I915_MAP_WB);
115 if (IS_ERR(cs)) {
116 err = PTR_ERR(cs);
117 goto err_obj;
118 }
119 memset(cs, 0xc5, PAGE_SIZE);
120 i915_gem_object_flush_map(result);
121 i915_gem_object_unpin_map(result);
122
123 vma = i915_vma_instance(result, &engine->gt->ggtt->vm, NULL);
124 if (IS_ERR(vma)) {
125 err = PTR_ERR(vma);
126 goto err_obj;
127 }
128
129 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
130 if (err)
131 goto err_obj;
132
133 rq = intel_context_create_request(ce);
134 if (IS_ERR(rq)) {
135 err = PTR_ERR(rq);
136 goto err_pin;
137 }
138
139 i915_vma_lock(vma);
140 err = i915_request_await_object(rq, vma->obj, true);
141 if (err == 0)
142 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
143 i915_vma_unlock(vma);
144 if (err)
145 goto err_req;
146
147 srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
148 if (INTEL_GEN(engine->i915) >= 8)
149 srm++;
150
151 cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS);
152 if (IS_ERR(cs)) {
153 err = PTR_ERR(cs);
154 goto err_req;
155 }
156
157 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
158 *cs++ = srm;
159 *cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i));
160 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
161 *cs++ = 0;
162 }
163 intel_ring_advance(rq, cs);
164
165 i915_request_add(rq);
166 i915_vma_unpin(vma);
167
168 return result;
169
170 err_req:
171 i915_request_add(rq);
172 err_pin:
173 i915_vma_unpin(vma);
174 err_obj:
175 i915_gem_object_put(result);
176 return ERR_PTR(err);
177 }
178
179 static u32
get_whitelist_reg(const struct intel_engine_cs * engine,unsigned int i)180 get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i)
181 {
182 i915_reg_t reg = i < engine->whitelist.count ?
183 engine->whitelist.list[i].reg :
184 RING_NOPID(engine->mmio_base);
185
186 return i915_mmio_reg_offset(reg);
187 }
188
189 static void
print_results(const struct intel_engine_cs * engine,const u32 * results)190 print_results(const struct intel_engine_cs *engine, const u32 *results)
191 {
192 unsigned int i;
193
194 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
195 u32 expected = get_whitelist_reg(engine, i);
196 u32 actual = results[i];
197
198 pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n",
199 i, expected, actual);
200 }
201 }
202
check_whitelist(struct intel_context * ce)203 static int check_whitelist(struct intel_context *ce)
204 {
205 struct intel_engine_cs *engine = ce->engine;
206 struct drm_i915_gem_object *results;
207 struct intel_wedge_me wedge;
208 u32 *vaddr;
209 int err;
210 int i;
211
212 results = read_nonprivs(ce);
213 if (IS_ERR(results))
214 return PTR_ERR(results);
215
216 err = 0;
217 i915_gem_object_lock(results, NULL);
218 intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */
219 err = i915_gem_object_set_to_cpu_domain(results, false);
220
221 if (intel_gt_is_wedged(engine->gt))
222 err = -EIO;
223 if (err)
224 goto out_put;
225
226 vaddr = i915_gem_object_pin_map(results, I915_MAP_WB);
227 if (IS_ERR(vaddr)) {
228 err = PTR_ERR(vaddr);
229 goto out_put;
230 }
231
232 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
233 u32 expected = get_whitelist_reg(engine, i);
234 u32 actual = vaddr[i];
235
236 if (expected != actual) {
237 print_results(engine, vaddr);
238 pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n",
239 i, expected, actual);
240
241 err = -EINVAL;
242 break;
243 }
244 }
245
246 i915_gem_object_unpin_map(results);
247 out_put:
248 i915_gem_object_unlock(results);
249 i915_gem_object_put(results);
250 return err;
251 }
252
do_device_reset(struct intel_engine_cs * engine)253 static int do_device_reset(struct intel_engine_cs *engine)
254 {
255 intel_gt_reset(engine->gt, engine->mask, "live_workarounds");
256 return 0;
257 }
258
do_engine_reset(struct intel_engine_cs * engine)259 static int do_engine_reset(struct intel_engine_cs *engine)
260 {
261 return intel_engine_reset(engine, "live_workarounds");
262 }
263
264 static int
switch_to_scratch_context(struct intel_engine_cs * engine,struct igt_spinner * spin)265 switch_to_scratch_context(struct intel_engine_cs *engine,
266 struct igt_spinner *spin)
267 {
268 struct intel_context *ce;
269 struct i915_request *rq;
270 int err = 0;
271
272 ce = intel_context_create(engine);
273 if (IS_ERR(ce))
274 return PTR_ERR(ce);
275
276 rq = igt_spinner_create_request(spin, ce, MI_NOOP);
277 intel_context_put(ce);
278
279 if (IS_ERR(rq)) {
280 spin = NULL;
281 err = PTR_ERR(rq);
282 goto err;
283 }
284
285 err = request_add_spin(rq, spin);
286 err:
287 if (err && spin)
288 igt_spinner_end(spin);
289
290 return err;
291 }
292
check_whitelist_across_reset(struct intel_engine_cs * engine,int (* reset)(struct intel_engine_cs *),const char * name)293 static int check_whitelist_across_reset(struct intel_engine_cs *engine,
294 int (*reset)(struct intel_engine_cs *),
295 const char *name)
296 {
297 struct intel_context *ce, *tmp;
298 struct igt_spinner spin;
299 intel_wakeref_t wakeref;
300 int err;
301
302 pr_info("Checking %d whitelisted registers on %s (RING_NONPRIV) [%s]\n",
303 engine->whitelist.count, engine->name, name);
304
305 ce = intel_context_create(engine);
306 if (IS_ERR(ce))
307 return PTR_ERR(ce);
308
309 err = igt_spinner_init(&spin, engine->gt);
310 if (err)
311 goto out_ctx;
312
313 err = check_whitelist(ce);
314 if (err) {
315 pr_err("Invalid whitelist *before* %s reset!\n", name);
316 goto out_spin;
317 }
318
319 err = switch_to_scratch_context(engine, &spin);
320 if (err)
321 goto out_spin;
322
323 with_intel_runtime_pm(engine->uncore->rpm, wakeref)
324 err = reset(engine);
325
326 igt_spinner_end(&spin);
327
328 if (err) {
329 pr_err("%s reset failed\n", name);
330 goto out_spin;
331 }
332
333 err = check_whitelist(ce);
334 if (err) {
335 pr_err("Whitelist not preserved in context across %s reset!\n",
336 name);
337 goto out_spin;
338 }
339
340 tmp = intel_context_create(engine);
341 if (IS_ERR(tmp)) {
342 err = PTR_ERR(tmp);
343 goto out_spin;
344 }
345 intel_context_put(ce);
346 ce = tmp;
347
348 err = check_whitelist(ce);
349 if (err) {
350 pr_err("Invalid whitelist *after* %s reset in fresh context!\n",
351 name);
352 goto out_spin;
353 }
354
355 out_spin:
356 igt_spinner_fini(&spin);
357 out_ctx:
358 intel_context_put(ce);
359 return err;
360 }
361
create_batch(struct i915_address_space * vm)362 static struct i915_vma *create_batch(struct i915_address_space *vm)
363 {
364 struct drm_i915_gem_object *obj;
365 struct i915_vma *vma;
366 int err;
367
368 obj = i915_gem_object_create_internal(vm->i915, 16 * PAGE_SIZE);
369 if (IS_ERR(obj))
370 return ERR_CAST(obj);
371
372 vma = i915_vma_instance(obj, vm, NULL);
373 if (IS_ERR(vma)) {
374 err = PTR_ERR(vma);
375 goto err_obj;
376 }
377
378 err = i915_vma_pin(vma, 0, 0, PIN_USER);
379 if (err)
380 goto err_obj;
381
382 return vma;
383
384 err_obj:
385 i915_gem_object_put(obj);
386 return ERR_PTR(err);
387 }
388
reg_write(u32 old,u32 new,u32 rsvd)389 static u32 reg_write(u32 old, u32 new, u32 rsvd)
390 {
391 if (rsvd == 0x0000ffff) {
392 old &= ~(new >> 16);
393 old |= new & (new >> 16);
394 } else {
395 old &= ~rsvd;
396 old |= new & rsvd;
397 }
398
399 return old;
400 }
401
wo_register(struct intel_engine_cs * engine,u32 reg)402 static bool wo_register(struct intel_engine_cs *engine, u32 reg)
403 {
404 enum intel_platform platform = INTEL_INFO(engine->i915)->platform;
405 int i;
406
407 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
408 RING_FORCE_TO_NONPRIV_ACCESS_WR)
409 return true;
410
411 for (i = 0; i < ARRAY_SIZE(wo_registers); i++) {
412 if (wo_registers[i].platform == platform &&
413 wo_registers[i].reg == reg)
414 return true;
415 }
416
417 return false;
418 }
419
timestamp(const struct intel_engine_cs * engine,u32 reg)420 static bool timestamp(const struct intel_engine_cs *engine, u32 reg)
421 {
422 reg = (reg - engine->mmio_base) & ~RING_FORCE_TO_NONPRIV_ACCESS_MASK;
423 switch (reg) {
424 case 0x358:
425 case 0x35c:
426 case 0x3a8:
427 return true;
428
429 default:
430 return false;
431 }
432 }
433
ro_register(u32 reg)434 static bool ro_register(u32 reg)
435 {
436 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
437 RING_FORCE_TO_NONPRIV_ACCESS_RD)
438 return true;
439
440 return false;
441 }
442
whitelist_writable_count(struct intel_engine_cs * engine)443 static int whitelist_writable_count(struct intel_engine_cs *engine)
444 {
445 int count = engine->whitelist.count;
446 int i;
447
448 for (i = 0; i < engine->whitelist.count; i++) {
449 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
450
451 if (ro_register(reg))
452 count--;
453 }
454
455 return count;
456 }
457
check_dirty_whitelist(struct intel_context * ce)458 static int check_dirty_whitelist(struct intel_context *ce)
459 {
460 const u32 values[] = {
461 0x00000000,
462 0x01010101,
463 0x10100101,
464 0x03030303,
465 0x30300303,
466 0x05050505,
467 0x50500505,
468 0x0f0f0f0f,
469 0xf00ff00f,
470 0x10101010,
471 0xf0f01010,
472 0x30303030,
473 0xa0a03030,
474 0x50505050,
475 0xc0c05050,
476 0xf0f0f0f0,
477 0x11111111,
478 0x33333333,
479 0x55555555,
480 0x0000ffff,
481 0x00ff00ff,
482 0xff0000ff,
483 0xffff00ff,
484 0xffffffff,
485 };
486 struct intel_engine_cs *engine = ce->engine;
487 struct i915_vma *scratch;
488 struct i915_vma *batch;
489 int err = 0, i, v, sz;
490 u32 *cs, *results;
491
492 sz = (2 * ARRAY_SIZE(values) + 1) * sizeof(u32);
493 scratch = __vm_create_scratch_for_read_pinned(ce->vm, sz);
494 if (IS_ERR(scratch))
495 return PTR_ERR(scratch);
496
497 batch = create_batch(ce->vm);
498 if (IS_ERR(batch)) {
499 err = PTR_ERR(batch);
500 goto out_scratch;
501 }
502
503 for (i = 0; i < engine->whitelist.count; i++) {
504 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
505 struct i915_gem_ww_ctx ww;
506 u64 addr = scratch->node.start;
507 struct i915_request *rq;
508 u32 srm, lrm, rsvd;
509 u32 expect;
510 int idx;
511 bool ro_reg;
512
513 if (wo_register(engine, reg))
514 continue;
515
516 if (timestamp(engine, reg))
517 continue; /* timestamps are expected to autoincrement */
518
519 ro_reg = ro_register(reg);
520
521 i915_gem_ww_ctx_init(&ww, false);
522 retry:
523 cs = NULL;
524 err = i915_gem_object_lock(scratch->obj, &ww);
525 if (!err)
526 err = i915_gem_object_lock(batch->obj, &ww);
527 if (!err)
528 err = intel_context_pin_ww(ce, &ww);
529 if (err)
530 goto out;
531
532 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
533 if (IS_ERR(cs)) {
534 err = PTR_ERR(cs);
535 goto out_ctx;
536 }
537
538 results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
539 if (IS_ERR(results)) {
540 err = PTR_ERR(results);
541 goto out_unmap_batch;
542 }
543
544 /* Clear non priv flags */
545 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
546
547 srm = MI_STORE_REGISTER_MEM;
548 lrm = MI_LOAD_REGISTER_MEM;
549 if (INTEL_GEN(engine->i915) >= 8)
550 lrm++, srm++;
551
552 pr_debug("%s: Writing garbage to %x\n",
553 engine->name, reg);
554
555 /* SRM original */
556 *cs++ = srm;
557 *cs++ = reg;
558 *cs++ = lower_32_bits(addr);
559 *cs++ = upper_32_bits(addr);
560
561 idx = 1;
562 for (v = 0; v < ARRAY_SIZE(values); v++) {
563 /* LRI garbage */
564 *cs++ = MI_LOAD_REGISTER_IMM(1);
565 *cs++ = reg;
566 *cs++ = values[v];
567
568 /* SRM result */
569 *cs++ = srm;
570 *cs++ = reg;
571 *cs++ = lower_32_bits(addr + sizeof(u32) * idx);
572 *cs++ = upper_32_bits(addr + sizeof(u32) * idx);
573 idx++;
574 }
575 for (v = 0; v < ARRAY_SIZE(values); v++) {
576 /* LRI garbage */
577 *cs++ = MI_LOAD_REGISTER_IMM(1);
578 *cs++ = reg;
579 *cs++ = ~values[v];
580
581 /* SRM result */
582 *cs++ = srm;
583 *cs++ = reg;
584 *cs++ = lower_32_bits(addr + sizeof(u32) * idx);
585 *cs++ = upper_32_bits(addr + sizeof(u32) * idx);
586 idx++;
587 }
588 GEM_BUG_ON(idx * sizeof(u32) > scratch->size);
589
590 /* LRM original -- don't leave garbage in the context! */
591 *cs++ = lrm;
592 *cs++ = reg;
593 *cs++ = lower_32_bits(addr);
594 *cs++ = upper_32_bits(addr);
595
596 *cs++ = MI_BATCH_BUFFER_END;
597
598 i915_gem_object_flush_map(batch->obj);
599 i915_gem_object_unpin_map(batch->obj);
600 intel_gt_chipset_flush(engine->gt);
601 cs = NULL;
602
603 rq = i915_request_create(ce);
604 if (IS_ERR(rq)) {
605 err = PTR_ERR(rq);
606 goto out_unmap_scratch;
607 }
608
609 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */
610 err = engine->emit_init_breadcrumb(rq);
611 if (err)
612 goto err_request;
613 }
614
615 err = i915_request_await_object(rq, batch->obj, false);
616 if (err == 0)
617 err = i915_vma_move_to_active(batch, rq, 0);
618 if (err)
619 goto err_request;
620
621 err = i915_request_await_object(rq, scratch->obj, true);
622 if (err == 0)
623 err = i915_vma_move_to_active(scratch, rq,
624 EXEC_OBJECT_WRITE);
625 if (err)
626 goto err_request;
627
628 err = engine->emit_bb_start(rq,
629 batch->node.start, PAGE_SIZE,
630 0);
631 if (err)
632 goto err_request;
633
634 err_request:
635 err = request_add_sync(rq, err);
636 if (err) {
637 pr_err("%s: Futzing %x timedout; cancelling test\n",
638 engine->name, reg);
639 intel_gt_set_wedged(engine->gt);
640 goto out_unmap_scratch;
641 }
642
643 GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff);
644 if (!ro_reg) {
645 /* detect write masking */
646 rsvd = results[ARRAY_SIZE(values)];
647 if (!rsvd) {
648 pr_err("%s: Unable to write to whitelisted register %x\n",
649 engine->name, reg);
650 err = -EINVAL;
651 goto out_unmap_scratch;
652 }
653 } else {
654 rsvd = 0;
655 }
656
657 expect = results[0];
658 idx = 1;
659 for (v = 0; v < ARRAY_SIZE(values); v++) {
660 if (ro_reg)
661 expect = results[0];
662 else
663 expect = reg_write(expect, values[v], rsvd);
664
665 if (results[idx] != expect)
666 err++;
667 idx++;
668 }
669 for (v = 0; v < ARRAY_SIZE(values); v++) {
670 if (ro_reg)
671 expect = results[0];
672 else
673 expect = reg_write(expect, ~values[v], rsvd);
674
675 if (results[idx] != expect)
676 err++;
677 idx++;
678 }
679 if (err) {
680 pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n",
681 engine->name, err, reg);
682
683 if (ro_reg)
684 pr_info("%s: Whitelisted read-only register: %x, original value %08x\n",
685 engine->name, reg, results[0]);
686 else
687 pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n",
688 engine->name, reg, results[0], rsvd);
689
690 expect = results[0];
691 idx = 1;
692 for (v = 0; v < ARRAY_SIZE(values); v++) {
693 u32 w = values[v];
694
695 if (ro_reg)
696 expect = results[0];
697 else
698 expect = reg_write(expect, w, rsvd);
699 pr_info("Wrote %08x, read %08x, expect %08x\n",
700 w, results[idx], expect);
701 idx++;
702 }
703 for (v = 0; v < ARRAY_SIZE(values); v++) {
704 u32 w = ~values[v];
705
706 if (ro_reg)
707 expect = results[0];
708 else
709 expect = reg_write(expect, w, rsvd);
710 pr_info("Wrote %08x, read %08x, expect %08x\n",
711 w, results[idx], expect);
712 idx++;
713 }
714
715 err = -EINVAL;
716 }
717 out_unmap_scratch:
718 i915_gem_object_unpin_map(scratch->obj);
719 out_unmap_batch:
720 if (cs)
721 i915_gem_object_unpin_map(batch->obj);
722 out_ctx:
723 intel_context_unpin(ce);
724 out:
725 if (err == -EDEADLK) {
726 err = i915_gem_ww_ctx_backoff(&ww);
727 if (!err)
728 goto retry;
729 }
730 i915_gem_ww_ctx_fini(&ww);
731 if (err)
732 break;
733 }
734
735 if (igt_flush_test(engine->i915))
736 err = -EIO;
737
738 i915_vma_unpin_and_release(&batch, 0);
739 out_scratch:
740 i915_vma_unpin_and_release(&scratch, 0);
741 return err;
742 }
743
live_dirty_whitelist(void * arg)744 static int live_dirty_whitelist(void *arg)
745 {
746 struct intel_gt *gt = arg;
747 struct intel_engine_cs *engine;
748 enum intel_engine_id id;
749
750 /* Can the user write to the whitelisted registers? */
751
752 if (INTEL_GEN(gt->i915) < 7) /* minimum requirement for LRI, SRM, LRM */
753 return 0;
754
755 for_each_engine(engine, gt, id) {
756 struct intel_context *ce;
757 int err;
758
759 if (engine->whitelist.count == 0)
760 continue;
761
762 ce = intel_context_create(engine);
763 if (IS_ERR(ce))
764 return PTR_ERR(ce);
765
766 err = check_dirty_whitelist(ce);
767 intel_context_put(ce);
768 if (err)
769 return err;
770 }
771
772 return 0;
773 }
774
live_reset_whitelist(void * arg)775 static int live_reset_whitelist(void *arg)
776 {
777 struct intel_gt *gt = arg;
778 struct intel_engine_cs *engine;
779 enum intel_engine_id id;
780 int err = 0;
781
782 /* If we reset the gpu, we should not lose the RING_NONPRIV */
783 igt_global_reset_lock(gt);
784
785 for_each_engine(engine, gt, id) {
786 if (engine->whitelist.count == 0)
787 continue;
788
789 if (intel_has_reset_engine(gt)) {
790 err = check_whitelist_across_reset(engine,
791 do_engine_reset,
792 "engine");
793 if (err)
794 goto out;
795 }
796
797 if (intel_has_gpu_reset(gt)) {
798 err = check_whitelist_across_reset(engine,
799 do_device_reset,
800 "device");
801 if (err)
802 goto out;
803 }
804 }
805
806 out:
807 igt_global_reset_unlock(gt);
808 return err;
809 }
810
read_whitelisted_registers(struct intel_context * ce,struct i915_vma * results)811 static int read_whitelisted_registers(struct intel_context *ce,
812 struct i915_vma *results)
813 {
814 struct intel_engine_cs *engine = ce->engine;
815 struct i915_request *rq;
816 int i, err = 0;
817 u32 srm, *cs;
818
819 rq = intel_context_create_request(ce);
820 if (IS_ERR(rq))
821 return PTR_ERR(rq);
822
823 i915_vma_lock(results);
824 err = i915_request_await_object(rq, results->obj, true);
825 if (err == 0)
826 err = i915_vma_move_to_active(results, rq, EXEC_OBJECT_WRITE);
827 i915_vma_unlock(results);
828 if (err)
829 goto err_req;
830
831 srm = MI_STORE_REGISTER_MEM;
832 if (INTEL_GEN(engine->i915) >= 8)
833 srm++;
834
835 cs = intel_ring_begin(rq, 4 * engine->whitelist.count);
836 if (IS_ERR(cs)) {
837 err = PTR_ERR(cs);
838 goto err_req;
839 }
840
841 for (i = 0; i < engine->whitelist.count; i++) {
842 u64 offset = results->node.start + sizeof(u32) * i;
843 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
844
845 /* Clear non priv flags */
846 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
847
848 *cs++ = srm;
849 *cs++ = reg;
850 *cs++ = lower_32_bits(offset);
851 *cs++ = upper_32_bits(offset);
852 }
853 intel_ring_advance(rq, cs);
854
855 err_req:
856 return request_add_sync(rq, err);
857 }
858
scrub_whitelisted_registers(struct intel_context * ce)859 static int scrub_whitelisted_registers(struct intel_context *ce)
860 {
861 struct intel_engine_cs *engine = ce->engine;
862 struct i915_request *rq;
863 struct i915_vma *batch;
864 int i, err = 0;
865 u32 *cs;
866
867 batch = create_batch(ce->vm);
868 if (IS_ERR(batch))
869 return PTR_ERR(batch);
870
871 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
872 if (IS_ERR(cs)) {
873 err = PTR_ERR(cs);
874 goto err_batch;
875 }
876
877 *cs++ = MI_LOAD_REGISTER_IMM(whitelist_writable_count(engine));
878 for (i = 0; i < engine->whitelist.count; i++) {
879 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
880
881 if (ro_register(reg))
882 continue;
883
884 /* Clear non priv flags */
885 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
886
887 *cs++ = reg;
888 *cs++ = 0xffffffff;
889 }
890 *cs++ = MI_BATCH_BUFFER_END;
891
892 i915_gem_object_flush_map(batch->obj);
893 intel_gt_chipset_flush(engine->gt);
894
895 rq = intel_context_create_request(ce);
896 if (IS_ERR(rq)) {
897 err = PTR_ERR(rq);
898 goto err_unpin;
899 }
900
901 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */
902 err = engine->emit_init_breadcrumb(rq);
903 if (err)
904 goto err_request;
905 }
906
907 i915_vma_lock(batch);
908 err = i915_request_await_object(rq, batch->obj, false);
909 if (err == 0)
910 err = i915_vma_move_to_active(batch, rq, 0);
911 i915_vma_unlock(batch);
912 if (err)
913 goto err_request;
914
915 /* Perform the writes from an unprivileged "user" batch */
916 err = engine->emit_bb_start(rq, batch->node.start, 0, 0);
917
918 err_request:
919 err = request_add_sync(rq, err);
920
921 err_unpin:
922 i915_gem_object_unpin_map(batch->obj);
923 err_batch:
924 i915_vma_unpin_and_release(&batch, 0);
925 return err;
926 }
927
928 struct regmask {
929 i915_reg_t reg;
930 unsigned long gen_mask;
931 };
932
find_reg(struct drm_i915_private * i915,i915_reg_t reg,const struct regmask * tbl,unsigned long count)933 static bool find_reg(struct drm_i915_private *i915,
934 i915_reg_t reg,
935 const struct regmask *tbl,
936 unsigned long count)
937 {
938 u32 offset = i915_mmio_reg_offset(reg);
939
940 while (count--) {
941 if (INTEL_INFO(i915)->gen_mask & tbl->gen_mask &&
942 i915_mmio_reg_offset(tbl->reg) == offset)
943 return true;
944 tbl++;
945 }
946
947 return false;
948 }
949
pardon_reg(struct drm_i915_private * i915,i915_reg_t reg)950 static bool pardon_reg(struct drm_i915_private *i915, i915_reg_t reg)
951 {
952 /* Alas, we must pardon some whitelists. Mistakes already made */
953 static const struct regmask pardon[] = {
954 { GEN9_CTX_PREEMPT_REG, INTEL_GEN_MASK(9, 9) },
955 { GEN8_L3SQCREG4, INTEL_GEN_MASK(9, 9) },
956 };
957
958 return find_reg(i915, reg, pardon, ARRAY_SIZE(pardon));
959 }
960
result_eq(struct intel_engine_cs * engine,u32 a,u32 b,i915_reg_t reg)961 static bool result_eq(struct intel_engine_cs *engine,
962 u32 a, u32 b, i915_reg_t reg)
963 {
964 if (a != b && !pardon_reg(engine->i915, reg)) {
965 pr_err("Whitelisted register 0x%4x not context saved: A=%08x, B=%08x\n",
966 i915_mmio_reg_offset(reg), a, b);
967 return false;
968 }
969
970 return true;
971 }
972
writeonly_reg(struct drm_i915_private * i915,i915_reg_t reg)973 static bool writeonly_reg(struct drm_i915_private *i915, i915_reg_t reg)
974 {
975 /* Some registers do not seem to behave and our writes unreadable */
976 static const struct regmask wo[] = {
977 { GEN9_SLICE_COMMON_ECO_CHICKEN1, INTEL_GEN_MASK(9, 9) },
978 };
979
980 return find_reg(i915, reg, wo, ARRAY_SIZE(wo));
981 }
982
result_neq(struct intel_engine_cs * engine,u32 a,u32 b,i915_reg_t reg)983 static bool result_neq(struct intel_engine_cs *engine,
984 u32 a, u32 b, i915_reg_t reg)
985 {
986 if (a == b && !writeonly_reg(engine->i915, reg)) {
987 pr_err("Whitelist register 0x%4x:%08x was unwritable\n",
988 i915_mmio_reg_offset(reg), a);
989 return false;
990 }
991
992 return true;
993 }
994
995 static int
check_whitelisted_registers(struct intel_engine_cs * engine,struct i915_vma * A,struct i915_vma * B,bool (* fn)(struct intel_engine_cs * engine,u32 a,u32 b,i915_reg_t reg))996 check_whitelisted_registers(struct intel_engine_cs *engine,
997 struct i915_vma *A,
998 struct i915_vma *B,
999 bool (*fn)(struct intel_engine_cs *engine,
1000 u32 a, u32 b,
1001 i915_reg_t reg))
1002 {
1003 u32 *a, *b;
1004 int i, err;
1005
1006 a = i915_gem_object_pin_map_unlocked(A->obj, I915_MAP_WB);
1007 if (IS_ERR(a))
1008 return PTR_ERR(a);
1009
1010 b = i915_gem_object_pin_map_unlocked(B->obj, I915_MAP_WB);
1011 if (IS_ERR(b)) {
1012 err = PTR_ERR(b);
1013 goto err_a;
1014 }
1015
1016 err = 0;
1017 for (i = 0; i < engine->whitelist.count; i++) {
1018 const struct i915_wa *wa = &engine->whitelist.list[i];
1019
1020 if (i915_mmio_reg_offset(wa->reg) &
1021 RING_FORCE_TO_NONPRIV_ACCESS_RD)
1022 continue;
1023
1024 if (!fn(engine, a[i], b[i], wa->reg))
1025 err = -EINVAL;
1026 }
1027
1028 i915_gem_object_unpin_map(B->obj);
1029 err_a:
1030 i915_gem_object_unpin_map(A->obj);
1031 return err;
1032 }
1033
live_isolated_whitelist(void * arg)1034 static int live_isolated_whitelist(void *arg)
1035 {
1036 struct intel_gt *gt = arg;
1037 struct {
1038 struct i915_vma *scratch[2];
1039 } client[2] = {};
1040 struct intel_engine_cs *engine;
1041 enum intel_engine_id id;
1042 int i, err = 0;
1043
1044 /*
1045 * Check that a write into a whitelist register works, but
1046 * invisible to a second context.
1047 */
1048
1049 if (!intel_engines_has_context_isolation(gt->i915))
1050 return 0;
1051
1052 for (i = 0; i < ARRAY_SIZE(client); i++) {
1053 client[i].scratch[0] =
1054 __vm_create_scratch_for_read_pinned(gt->vm, 4096);
1055 if (IS_ERR(client[i].scratch[0])) {
1056 err = PTR_ERR(client[i].scratch[0]);
1057 goto err;
1058 }
1059
1060 client[i].scratch[1] =
1061 __vm_create_scratch_for_read_pinned(gt->vm, 4096);
1062 if (IS_ERR(client[i].scratch[1])) {
1063 err = PTR_ERR(client[i].scratch[1]);
1064 i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1065 goto err;
1066 }
1067 }
1068
1069 for_each_engine(engine, gt, id) {
1070 struct intel_context *ce[2];
1071
1072 if (!engine->kernel_context->vm)
1073 continue;
1074
1075 if (!whitelist_writable_count(engine))
1076 continue;
1077
1078 ce[0] = intel_context_create(engine);
1079 if (IS_ERR(ce[0])) {
1080 err = PTR_ERR(ce[0]);
1081 break;
1082 }
1083 ce[1] = intel_context_create(engine);
1084 if (IS_ERR(ce[1])) {
1085 err = PTR_ERR(ce[1]);
1086 intel_context_put(ce[0]);
1087 break;
1088 }
1089
1090 /* Read default values */
1091 err = read_whitelisted_registers(ce[0], client[0].scratch[0]);
1092 if (err)
1093 goto err_ce;
1094
1095 /* Try to overwrite registers (should only affect ctx0) */
1096 err = scrub_whitelisted_registers(ce[0]);
1097 if (err)
1098 goto err_ce;
1099
1100 /* Read values from ctx1, we expect these to be defaults */
1101 err = read_whitelisted_registers(ce[1], client[1].scratch[0]);
1102 if (err)
1103 goto err_ce;
1104
1105 /* Verify that both reads return the same default values */
1106 err = check_whitelisted_registers(engine,
1107 client[0].scratch[0],
1108 client[1].scratch[0],
1109 result_eq);
1110 if (err)
1111 goto err_ce;
1112
1113 /* Read back the updated values in ctx0 */
1114 err = read_whitelisted_registers(ce[0], client[0].scratch[1]);
1115 if (err)
1116 goto err_ce;
1117
1118 /* User should be granted privilege to overwhite regs */
1119 err = check_whitelisted_registers(engine,
1120 client[0].scratch[0],
1121 client[0].scratch[1],
1122 result_neq);
1123 err_ce:
1124 intel_context_put(ce[1]);
1125 intel_context_put(ce[0]);
1126 if (err)
1127 break;
1128 }
1129
1130 err:
1131 for (i = 0; i < ARRAY_SIZE(client); i++) {
1132 i915_vma_unpin_and_release(&client[i].scratch[1], 0);
1133 i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1134 }
1135
1136 if (igt_flush_test(gt->i915))
1137 err = -EIO;
1138
1139 return err;
1140 }
1141
1142 static bool
verify_wa_lists(struct intel_gt * gt,struct wa_lists * lists,const char * str)1143 verify_wa_lists(struct intel_gt *gt, struct wa_lists *lists,
1144 const char *str)
1145 {
1146 struct intel_engine_cs *engine;
1147 enum intel_engine_id id;
1148 bool ok = true;
1149
1150 ok &= wa_list_verify(gt->uncore, &lists->gt_wa_list, str);
1151
1152 for_each_engine(engine, gt, id) {
1153 struct intel_context *ce;
1154
1155 ce = intel_context_create(engine);
1156 if (IS_ERR(ce))
1157 return false;
1158
1159 ok &= engine_wa_list_verify(ce,
1160 &lists->engine[id].wa_list,
1161 str) == 0;
1162
1163 ok &= engine_wa_list_verify(ce,
1164 &lists->engine[id].ctx_wa_list,
1165 str) == 0;
1166
1167 intel_context_put(ce);
1168 }
1169
1170 return ok;
1171 }
1172
1173 static int
live_gpu_reset_workarounds(void * arg)1174 live_gpu_reset_workarounds(void *arg)
1175 {
1176 struct intel_gt *gt = arg;
1177 intel_wakeref_t wakeref;
1178 struct wa_lists lists;
1179 bool ok;
1180
1181 if (!intel_has_gpu_reset(gt))
1182 return 0;
1183
1184 pr_info("Verifying after GPU reset...\n");
1185
1186 igt_global_reset_lock(gt);
1187 wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1188
1189 reference_lists_init(gt, &lists);
1190
1191 ok = verify_wa_lists(gt, &lists, "before reset");
1192 if (!ok)
1193 goto out;
1194
1195 intel_gt_reset(gt, ALL_ENGINES, "live_workarounds");
1196
1197 ok = verify_wa_lists(gt, &lists, "after reset");
1198
1199 out:
1200 reference_lists_fini(gt, &lists);
1201 intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1202 igt_global_reset_unlock(gt);
1203
1204 return ok ? 0 : -ESRCH;
1205 }
1206
1207 static int
live_engine_reset_workarounds(void * arg)1208 live_engine_reset_workarounds(void *arg)
1209 {
1210 struct intel_gt *gt = arg;
1211 struct intel_engine_cs *engine;
1212 enum intel_engine_id id;
1213 struct intel_context *ce;
1214 struct igt_spinner spin;
1215 struct i915_request *rq;
1216 intel_wakeref_t wakeref;
1217 struct wa_lists lists;
1218 int ret = 0;
1219
1220 if (!intel_has_reset_engine(gt))
1221 return 0;
1222
1223 igt_global_reset_lock(gt);
1224 wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1225
1226 reference_lists_init(gt, &lists);
1227
1228 for_each_engine(engine, gt, id) {
1229 bool ok;
1230
1231 pr_info("Verifying after %s reset...\n", engine->name);
1232 ce = intel_context_create(engine);
1233 if (IS_ERR(ce)) {
1234 ret = PTR_ERR(ce);
1235 break;
1236 }
1237
1238 ok = verify_wa_lists(gt, &lists, "before reset");
1239 if (!ok) {
1240 ret = -ESRCH;
1241 goto err;
1242 }
1243
1244 ret = intel_engine_reset(engine, "live_workarounds:idle");
1245 if (ret) {
1246 pr_err("%s: Reset failed while idle\n", engine->name);
1247 goto err;
1248 }
1249
1250 ok = verify_wa_lists(gt, &lists, "after idle reset");
1251 if (!ok) {
1252 ret = -ESRCH;
1253 goto err;
1254 }
1255
1256 ret = igt_spinner_init(&spin, engine->gt);
1257 if (ret)
1258 goto err;
1259
1260 rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
1261 if (IS_ERR(rq)) {
1262 ret = PTR_ERR(rq);
1263 igt_spinner_fini(&spin);
1264 goto err;
1265 }
1266
1267 ret = request_add_spin(rq, &spin);
1268 if (ret) {
1269 pr_err("%s: Spinner failed to start\n", engine->name);
1270 igt_spinner_fini(&spin);
1271 goto err;
1272 }
1273
1274 ret = intel_engine_reset(engine, "live_workarounds:active");
1275 if (ret) {
1276 pr_err("%s: Reset failed on an active spinner\n",
1277 engine->name);
1278 igt_spinner_fini(&spin);
1279 goto err;
1280 }
1281
1282 igt_spinner_end(&spin);
1283 igt_spinner_fini(&spin);
1284
1285 ok = verify_wa_lists(gt, &lists, "after busy reset");
1286 if (!ok) {
1287 ret = -ESRCH;
1288 goto err;
1289 }
1290
1291 err:
1292 intel_context_put(ce);
1293 if (ret)
1294 break;
1295 }
1296
1297 reference_lists_fini(gt, &lists);
1298 intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1299 igt_global_reset_unlock(gt);
1300
1301 igt_flush_test(gt->i915);
1302
1303 return ret;
1304 }
1305
intel_workarounds_live_selftests(struct drm_i915_private * i915)1306 int intel_workarounds_live_selftests(struct drm_i915_private *i915)
1307 {
1308 static const struct i915_subtest tests[] = {
1309 SUBTEST(live_dirty_whitelist),
1310 SUBTEST(live_reset_whitelist),
1311 SUBTEST(live_isolated_whitelist),
1312 SUBTEST(live_gpu_reset_workarounds),
1313 SUBTEST(live_engine_reset_workarounds),
1314 };
1315
1316 if (intel_gt_is_wedged(&i915->gt))
1317 return 0;
1318
1319 return intel_gt_live_subtests(tests, &i915->gt);
1320 }
1321