1 #include "qemu/osdep.h"
2 #include "qemu/cutils.h"
3 #include "cpu.h"
4 #include "helper_regs.h"
5 #include "hw/ppc/spapr.h"
6 #include "mmu-hash64.h"
7 #include "mmu-book3s-v3.h"
8
valid_ptex(PowerPCCPU * cpu,target_ulong ptex)9 static inline bool valid_ptex(PowerPCCPU *cpu, target_ulong ptex)
10 {
11 /*
12 * hash value/pteg group index is normalized by HPT mask
13 */
14 if (((ptex & ~7ULL) / HPTES_PER_GROUP) & ~ppc_hash64_hpt_mask(cpu)) {
15 return false;
16 }
17 return true;
18 }
19
h_enter(PowerPCCPU * cpu,SpaprMachineState * spapr,target_ulong opcode,target_ulong * args)20 static target_ulong h_enter(PowerPCCPU *cpu, SpaprMachineState *spapr,
21 target_ulong opcode, target_ulong *args)
22 {
23 target_ulong flags = args[0];
24 target_ulong ptex = args[1];
25 target_ulong pteh = args[2];
26 target_ulong ptel = args[3];
27 unsigned apshift;
28 target_ulong raddr;
29 target_ulong slot;
30 const ppc_hash_pte64_t *hptes;
31
32 apshift = ppc_hash64_hpte_page_shift_noslb(cpu, pteh, ptel);
33 if (!apshift) {
34 /* Bad page size encoding */
35 return H_PARAMETER;
36 }
37
38 raddr = (ptel & HPTE64_R_RPN) & ~((1ULL << apshift) - 1);
39
40 if (is_ram_address(spapr, raddr)) {
41 /* Regular RAM - should have WIMG=0010 */
42 if ((ptel & HPTE64_R_WIMG) != HPTE64_R_M) {
43 return H_PARAMETER;
44 }
45 } else {
46 target_ulong wimg_flags;
47 /* Looks like an IO address */
48 /* FIXME: What WIMG combinations could be sensible for IO?
49 * For now we allow WIMG=010x, but are there others? */
50 /* FIXME: Should we check against registered IO addresses? */
51 wimg_flags = (ptel & (HPTE64_R_W | HPTE64_R_I | HPTE64_R_M));
52
53 if (wimg_flags != HPTE64_R_I &&
54 wimg_flags != (HPTE64_R_I | HPTE64_R_M)) {
55 return H_PARAMETER;
56 }
57 }
58
59 pteh &= ~0x60ULL;
60
61 if (!valid_ptex(cpu, ptex)) {
62 return H_PARAMETER;
63 }
64
65 slot = ptex & 7ULL;
66 ptex = ptex & ~7ULL;
67
68 if (likely((flags & H_EXACT) == 0)) {
69 hptes = ppc_hash64_map_hptes(cpu, ptex, HPTES_PER_GROUP);
70 for (slot = 0; slot < 8; slot++) {
71 if (!(ppc_hash64_hpte0(cpu, hptes, slot) & HPTE64_V_VALID)) {
72 break;
73 }
74 }
75 ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
76 if (slot == 8) {
77 return H_PTEG_FULL;
78 }
79 } else {
80 hptes = ppc_hash64_map_hptes(cpu, ptex + slot, 1);
81 if (ppc_hash64_hpte0(cpu, hptes, 0) & HPTE64_V_VALID) {
82 ppc_hash64_unmap_hptes(cpu, hptes, ptex + slot, 1);
83 return H_PTEG_FULL;
84 }
85 ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
86 }
87
88 spapr_store_hpte(cpu, ptex + slot, pteh | HPTE64_V_HPTE_DIRTY, ptel);
89
90 args[0] = ptex + slot;
91 return H_SUCCESS;
92 }
93
94 typedef enum {
95 REMOVE_SUCCESS = 0,
96 REMOVE_NOT_FOUND = 1,
97 REMOVE_PARM = 2,
98 REMOVE_HW = 3,
99 } RemoveResult;
100
remove_hpte(PowerPCCPU * cpu,target_ulong ptex,target_ulong avpn,target_ulong flags,target_ulong * vp,target_ulong * rp)101 static RemoveResult remove_hpte(PowerPCCPU *cpu
102 , target_ulong ptex,
103 target_ulong avpn,
104 target_ulong flags,
105 target_ulong *vp, target_ulong *rp)
106 {
107 const ppc_hash_pte64_t *hptes;
108 target_ulong v, r;
109
110 if (!valid_ptex(cpu, ptex)) {
111 return REMOVE_PARM;
112 }
113
114 hptes = ppc_hash64_map_hptes(cpu, ptex, 1);
115 v = ppc_hash64_hpte0(cpu, hptes, 0);
116 r = ppc_hash64_hpte1(cpu, hptes, 0);
117 ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
118
119 if ((v & HPTE64_V_VALID) == 0 ||
120 ((flags & H_AVPN) && (v & ~0x7fULL) != avpn) ||
121 ((flags & H_ANDCOND) && (v & avpn) != 0)) {
122 return REMOVE_NOT_FOUND;
123 }
124 *vp = v;
125 *rp = r;
126 spapr_store_hpte(cpu, ptex, HPTE64_V_HPTE_DIRTY, 0);
127 ppc_hash64_tlb_flush_hpte(cpu, ptex, v, r);
128 return REMOVE_SUCCESS;
129 }
130
h_remove(PowerPCCPU * cpu,SpaprMachineState * spapr,target_ulong opcode,target_ulong * args)131 static target_ulong h_remove(PowerPCCPU *cpu, SpaprMachineState *spapr,
132 target_ulong opcode, target_ulong *args)
133 {
134 CPUPPCState *env = &cpu->env;
135 target_ulong flags = args[0];
136 target_ulong ptex = args[1];
137 target_ulong avpn = args[2];
138 RemoveResult ret;
139
140 ret = remove_hpte(cpu, ptex, avpn, flags,
141 &args[0], &args[1]);
142
143 switch (ret) {
144 case REMOVE_SUCCESS:
145 check_tlb_flush(env, true);
146 return H_SUCCESS;
147
148 case REMOVE_NOT_FOUND:
149 return H_NOT_FOUND;
150
151 case REMOVE_PARM:
152 return H_PARAMETER;
153
154 case REMOVE_HW:
155 return H_HARDWARE;
156 }
157
158 g_assert_not_reached();
159 }
160
161 #define H_BULK_REMOVE_TYPE 0xc000000000000000ULL
162 #define H_BULK_REMOVE_REQUEST 0x4000000000000000ULL
163 #define H_BULK_REMOVE_RESPONSE 0x8000000000000000ULL
164 #define H_BULK_REMOVE_END 0xc000000000000000ULL
165 #define H_BULK_REMOVE_CODE 0x3000000000000000ULL
166 #define H_BULK_REMOVE_SUCCESS 0x0000000000000000ULL
167 #define H_BULK_REMOVE_NOT_FOUND 0x1000000000000000ULL
168 #define H_BULK_REMOVE_PARM 0x2000000000000000ULL
169 #define H_BULK_REMOVE_HW 0x3000000000000000ULL
170 #define H_BULK_REMOVE_RC 0x0c00000000000000ULL
171 #define H_BULK_REMOVE_FLAGS 0x0300000000000000ULL
172 #define H_BULK_REMOVE_ABSOLUTE 0x0000000000000000ULL
173 #define H_BULK_REMOVE_ANDCOND 0x0100000000000000ULL
174 #define H_BULK_REMOVE_AVPN 0x0200000000000000ULL
175 #define H_BULK_REMOVE_PTEX 0x00ffffffffffffffULL
176
177 #define H_BULK_REMOVE_MAX_BATCH 4
178
h_bulk_remove(PowerPCCPU * cpu,SpaprMachineState * spapr,target_ulong opcode,target_ulong * args)179 static target_ulong h_bulk_remove(PowerPCCPU *cpu, SpaprMachineState *spapr,
180 target_ulong opcode, target_ulong *args)
181 {
182 CPUPPCState *env = &cpu->env;
183 int i;
184 target_ulong rc = H_SUCCESS;
185
186 for (i = 0; i < H_BULK_REMOVE_MAX_BATCH; i++) {
187 target_ulong *tsh = &args[i*2];
188 target_ulong tsl = args[i*2 + 1];
189 target_ulong v, r, ret;
190
191 if ((*tsh & H_BULK_REMOVE_TYPE) == H_BULK_REMOVE_END) {
192 break;
193 } else if ((*tsh & H_BULK_REMOVE_TYPE) != H_BULK_REMOVE_REQUEST) {
194 return H_PARAMETER;
195 }
196
197 *tsh &= H_BULK_REMOVE_PTEX | H_BULK_REMOVE_FLAGS;
198 *tsh |= H_BULK_REMOVE_RESPONSE;
199
200 if ((*tsh & H_BULK_REMOVE_ANDCOND) && (*tsh & H_BULK_REMOVE_AVPN)) {
201 *tsh |= H_BULK_REMOVE_PARM;
202 return H_PARAMETER;
203 }
204
205 ret = remove_hpte(cpu, *tsh & H_BULK_REMOVE_PTEX, tsl,
206 (*tsh & H_BULK_REMOVE_FLAGS) >> 26,
207 &v, &r);
208
209 *tsh |= ret << 60;
210
211 switch (ret) {
212 case REMOVE_SUCCESS:
213 *tsh |= (r & (HPTE64_R_C | HPTE64_R_R)) << 43;
214 break;
215
216 case REMOVE_PARM:
217 rc = H_PARAMETER;
218 goto exit;
219
220 case REMOVE_HW:
221 rc = H_HARDWARE;
222 goto exit;
223 }
224 }
225 exit:
226 check_tlb_flush(env, true);
227
228 return rc;
229 }
230
h_protect(PowerPCCPU * cpu,SpaprMachineState * spapr,target_ulong opcode,target_ulong * args)231 static target_ulong h_protect(PowerPCCPU *cpu, SpaprMachineState *spapr,
232 target_ulong opcode, target_ulong *args)
233 {
234 CPUPPCState *env = &cpu->env;
235 target_ulong flags = args[0];
236 target_ulong ptex = args[1];
237 target_ulong avpn = args[2];
238 const ppc_hash_pte64_t *hptes;
239 target_ulong v, r;
240
241 if (!valid_ptex(cpu, ptex)) {
242 return H_PARAMETER;
243 }
244
245 hptes = ppc_hash64_map_hptes(cpu, ptex, 1);
246 v = ppc_hash64_hpte0(cpu, hptes, 0);
247 r = ppc_hash64_hpte1(cpu, hptes, 0);
248 ppc_hash64_unmap_hptes(cpu, hptes, ptex, 1);
249
250 if ((v & HPTE64_V_VALID) == 0 ||
251 ((flags & H_AVPN) && (v & ~0x7fULL) != avpn)) {
252 return H_NOT_FOUND;
253 }
254
255 r &= ~(HPTE64_R_PP0 | HPTE64_R_PP | HPTE64_R_N |
256 HPTE64_R_KEY_HI | HPTE64_R_KEY_LO);
257 r |= (flags << 55) & HPTE64_R_PP0;
258 r |= (flags << 48) & HPTE64_R_KEY_HI;
259 r |= flags & (HPTE64_R_PP | HPTE64_R_N | HPTE64_R_KEY_LO);
260 spapr_store_hpte(cpu, ptex,
261 (v & ~HPTE64_V_VALID) | HPTE64_V_HPTE_DIRTY, 0);
262 ppc_hash64_tlb_flush_hpte(cpu, ptex, v, r);
263 /* Flush the tlb */
264 check_tlb_flush(env, true);
265 /* Don't need a memory barrier, due to qemu's global lock */
266 spapr_store_hpte(cpu, ptex, v | HPTE64_V_HPTE_DIRTY, r);
267 return H_SUCCESS;
268 }
269
h_read(PowerPCCPU * cpu,SpaprMachineState * spapr,target_ulong opcode,target_ulong * args)270 static target_ulong h_read(PowerPCCPU *cpu, SpaprMachineState *spapr,
271 target_ulong opcode, target_ulong *args)
272 {
273 target_ulong flags = args[0];
274 target_ulong ptex = args[1];
275 int i, ridx, n_entries = 1;
276 const ppc_hash_pte64_t *hptes;
277
278 if (!valid_ptex(cpu, ptex)) {
279 return H_PARAMETER;
280 }
281
282 if (flags & H_READ_4) {
283 /* Clear the two low order bits */
284 ptex &= ~(3ULL);
285 n_entries = 4;
286 }
287
288 hptes = ppc_hash64_map_hptes(cpu, ptex, n_entries);
289 for (i = 0, ridx = 0; i < n_entries; i++) {
290 args[ridx++] = ppc_hash64_hpte0(cpu, hptes, i);
291 args[ridx++] = ppc_hash64_hpte1(cpu, hptes, i);
292 }
293 ppc_hash64_unmap_hptes(cpu, hptes, ptex, n_entries);
294
295 return H_SUCCESS;
296 }
297
298 struct SpaprPendingHpt {
299 /* These fields are read-only after initialization */
300 int shift;
301 QemuThread thread;
302
303 /* These fields are protected by the BQL */
304 bool complete;
305
306 /* These fields are private to the preparation thread if
307 * !complete, otherwise protected by the BQL */
308 int ret;
309 void *hpt;
310 };
311
free_pending_hpt(SpaprPendingHpt * pending)312 static void free_pending_hpt(SpaprPendingHpt *pending)
313 {
314 if (pending->hpt) {
315 qemu_vfree(pending->hpt);
316 }
317
318 g_free(pending);
319 }
320
hpt_prepare_thread(void * opaque)321 static void *hpt_prepare_thread(void *opaque)
322 {
323 SpaprPendingHpt *pending = opaque;
324 size_t size = 1ULL << pending->shift;
325
326 pending->hpt = qemu_try_memalign(size, size);
327 if (pending->hpt) {
328 memset(pending->hpt, 0, size);
329 pending->ret = H_SUCCESS;
330 } else {
331 pending->ret = H_NO_MEM;
332 }
333
334 qemu_mutex_lock_iothread();
335
336 if (SPAPR_MACHINE(qdev_get_machine())->pending_hpt == pending) {
337 /* Ready to go */
338 pending->complete = true;
339 } else {
340 /* We've been cancelled, clean ourselves up */
341 free_pending_hpt(pending);
342 }
343
344 qemu_mutex_unlock_iothread();
345 return NULL;
346 }
347
348 /* Must be called with BQL held */
cancel_hpt_prepare(SpaprMachineState * spapr)349 static void cancel_hpt_prepare(SpaprMachineState *spapr)
350 {
351 SpaprPendingHpt *pending = spapr->pending_hpt;
352
353 /* Let the thread know it's cancelled */
354 spapr->pending_hpt = NULL;
355
356 if (!pending) {
357 /* Nothing to do */
358 return;
359 }
360
361 if (!pending->complete) {
362 /* thread will clean itself up */
363 return;
364 }
365
366 free_pending_hpt(pending);
367 }
368
softmmu_resize_hpt_prepare(PowerPCCPU * cpu,SpaprMachineState * spapr,target_ulong shift)369 target_ulong softmmu_resize_hpt_prepare(PowerPCCPU *cpu,
370 SpaprMachineState *spapr,
371 target_ulong shift)
372 {
373 SpaprPendingHpt *pending = spapr->pending_hpt;
374
375 if (pending) {
376 /* something already in progress */
377 if (pending->shift == shift) {
378 /* and it's suitable */
379 if (pending->complete) {
380 return pending->ret;
381 } else {
382 return H_LONG_BUSY_ORDER_100_MSEC;
383 }
384 }
385
386 /* not suitable, cancel and replace */
387 cancel_hpt_prepare(spapr);
388 }
389
390 if (!shift) {
391 /* nothing to do */
392 return H_SUCCESS;
393 }
394
395 /* start new prepare */
396
397 pending = g_new0(SpaprPendingHpt, 1);
398 pending->shift = shift;
399 pending->ret = H_HARDWARE;
400
401 qemu_thread_create(&pending->thread, "sPAPR HPT prepare",
402 hpt_prepare_thread, pending, QEMU_THREAD_DETACHED);
403
404 spapr->pending_hpt = pending;
405
406 /* In theory we could estimate the time more accurately based on
407 * the new size, but there's not much point */
408 return H_LONG_BUSY_ORDER_100_MSEC;
409 }
410
new_hpte_load0(void * htab,uint64_t pteg,int slot)411 static uint64_t new_hpte_load0(void *htab, uint64_t pteg, int slot)
412 {
413 uint8_t *addr = htab;
414
415 addr += pteg * HASH_PTEG_SIZE_64;
416 addr += slot * HASH_PTE_SIZE_64;
417 return ldq_p(addr);
418 }
419
new_hpte_store(void * htab,uint64_t pteg,int slot,uint64_t pte0,uint64_t pte1)420 static void new_hpte_store(void *htab, uint64_t pteg, int slot,
421 uint64_t pte0, uint64_t pte1)
422 {
423 uint8_t *addr = htab;
424
425 addr += pteg * HASH_PTEG_SIZE_64;
426 addr += slot * HASH_PTE_SIZE_64;
427
428 stq_p(addr, pte0);
429 stq_p(addr + HPTE64_DW1, pte1);
430 }
431
rehash_hpte(PowerPCCPU * cpu,const ppc_hash_pte64_t * hptes,void * old_hpt,uint64_t oldsize,void * new_hpt,uint64_t newsize,uint64_t pteg,int slot)432 static int rehash_hpte(PowerPCCPU *cpu,
433 const ppc_hash_pte64_t *hptes,
434 void *old_hpt, uint64_t oldsize,
435 void *new_hpt, uint64_t newsize,
436 uint64_t pteg, int slot)
437 {
438 uint64_t old_hash_mask = (oldsize >> 7) - 1;
439 uint64_t new_hash_mask = (newsize >> 7) - 1;
440 target_ulong pte0 = ppc_hash64_hpte0(cpu, hptes, slot);
441 target_ulong pte1;
442 uint64_t avpn;
443 unsigned base_pg_shift;
444 uint64_t hash, new_pteg, replace_pte0;
445
446 if (!(pte0 & HPTE64_V_VALID) || !(pte0 & HPTE64_V_BOLTED)) {
447 return H_SUCCESS;
448 }
449
450 pte1 = ppc_hash64_hpte1(cpu, hptes, slot);
451
452 base_pg_shift = ppc_hash64_hpte_page_shift_noslb(cpu, pte0, pte1);
453 assert(base_pg_shift); /* H_ENTER shouldn't allow a bad encoding */
454 avpn = HPTE64_V_AVPN_VAL(pte0) & ~(((1ULL << base_pg_shift) - 1) >> 23);
455
456 if (pte0 & HPTE64_V_SECONDARY) {
457 pteg = ~pteg;
458 }
459
460 if ((pte0 & HPTE64_V_SSIZE) == HPTE64_V_SSIZE_256M) {
461 uint64_t offset, vsid;
462
463 /* We only have 28 - 23 bits of offset in avpn */
464 offset = (avpn & 0x1f) << 23;
465 vsid = avpn >> 5;
466 /* We can find more bits from the pteg value */
467 if (base_pg_shift < 23) {
468 offset |= ((vsid ^ pteg) & old_hash_mask) << base_pg_shift;
469 }
470
471 hash = vsid ^ (offset >> base_pg_shift);
472 } else if ((pte0 & HPTE64_V_SSIZE) == HPTE64_V_SSIZE_1T) {
473 uint64_t offset, vsid;
474
475 /* We only have 40 - 23 bits of seg_off in avpn */
476 offset = (avpn & 0x1ffff) << 23;
477 vsid = avpn >> 17;
478 if (base_pg_shift < 23) {
479 offset |= ((vsid ^ (vsid << 25) ^ pteg) & old_hash_mask)
480 << base_pg_shift;
481 }
482
483 hash = vsid ^ (vsid << 25) ^ (offset >> base_pg_shift);
484 } else {
485 error_report("rehash_pte: Bad segment size in HPTE");
486 return H_HARDWARE;
487 }
488
489 new_pteg = hash & new_hash_mask;
490 if (pte0 & HPTE64_V_SECONDARY) {
491 assert(~pteg == (hash & old_hash_mask));
492 new_pteg = ~new_pteg;
493 } else {
494 assert(pteg == (hash & old_hash_mask));
495 }
496 assert((oldsize != newsize) || (pteg == new_pteg));
497 replace_pte0 = new_hpte_load0(new_hpt, new_pteg, slot);
498 /*
499 * Strictly speaking, we don't need all these tests, since we only
500 * ever rehash bolted HPTEs. We might in future handle non-bolted
501 * HPTEs, though so make the logic correct for those cases as
502 * well.
503 */
504 if (replace_pte0 & HPTE64_V_VALID) {
505 assert(newsize < oldsize);
506 if (replace_pte0 & HPTE64_V_BOLTED) {
507 if (pte0 & HPTE64_V_BOLTED) {
508 /* Bolted collision, nothing we can do */
509 return H_PTEG_FULL;
510 } else {
511 /* Discard this hpte */
512 return H_SUCCESS;
513 }
514 }
515 }
516
517 new_hpte_store(new_hpt, new_pteg, slot, pte0, pte1);
518 return H_SUCCESS;
519 }
520
rehash_hpt(PowerPCCPU * cpu,void * old_hpt,uint64_t oldsize,void * new_hpt,uint64_t newsize)521 static int rehash_hpt(PowerPCCPU *cpu,
522 void *old_hpt, uint64_t oldsize,
523 void *new_hpt, uint64_t newsize)
524 {
525 uint64_t n_ptegs = oldsize >> 7;
526 uint64_t pteg;
527 int slot;
528 int rc;
529
530 for (pteg = 0; pteg < n_ptegs; pteg++) {
531 hwaddr ptex = pteg * HPTES_PER_GROUP;
532 const ppc_hash_pte64_t *hptes
533 = ppc_hash64_map_hptes(cpu, ptex, HPTES_PER_GROUP);
534
535 if (!hptes) {
536 return H_HARDWARE;
537 }
538
539 for (slot = 0; slot < HPTES_PER_GROUP; slot++) {
540 rc = rehash_hpte(cpu, hptes, old_hpt, oldsize, new_hpt, newsize,
541 pteg, slot);
542 if (rc != H_SUCCESS) {
543 ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
544 return rc;
545 }
546 }
547 ppc_hash64_unmap_hptes(cpu, hptes, ptex, HPTES_PER_GROUP);
548 }
549
550 return H_SUCCESS;
551 }
552
softmmu_resize_hpt_commit(PowerPCCPU * cpu,SpaprMachineState * spapr,target_ulong flags,target_ulong shift)553 target_ulong softmmu_resize_hpt_commit(PowerPCCPU *cpu,
554 SpaprMachineState *spapr,
555 target_ulong flags,
556 target_ulong shift)
557 {
558 SpaprPendingHpt *pending = spapr->pending_hpt;
559 int rc;
560 size_t newsize;
561
562 if (flags != 0) {
563 return H_PARAMETER;
564 }
565
566 if (!pending || (pending->shift != shift)) {
567 /* no matching prepare */
568 return H_CLOSED;
569 }
570
571 if (!pending->complete) {
572 /* prepare has not completed */
573 return H_BUSY;
574 }
575
576 /* Shouldn't have got past PREPARE without an HPT */
577 g_assert(spapr->htab_shift);
578
579 newsize = 1ULL << pending->shift;
580 rc = rehash_hpt(cpu, spapr->htab, HTAB_SIZE(spapr),
581 pending->hpt, newsize);
582 if (rc == H_SUCCESS) {
583 qemu_vfree(spapr->htab);
584 spapr->htab = pending->hpt;
585 spapr->htab_shift = pending->shift;
586
587 push_sregs_to_kvm_pr(spapr);
588
589 pending->hpt = NULL; /* so it's not free()d */
590 }
591
592 /* Clean up */
593 spapr->pending_hpt = NULL;
594 free_pending_hpt(pending);
595
596 return rc;
597 }
598
hypercall_register_types(void)599 static void hypercall_register_types(void)
600 {
601 /* hcall-pft */
602 spapr_register_hypercall(H_ENTER, h_enter);
603 spapr_register_hypercall(H_REMOVE, h_remove);
604 spapr_register_hypercall(H_PROTECT, h_protect);
605 spapr_register_hypercall(H_READ, h_read);
606
607 /* hcall-bulk */
608 spapr_register_hypercall(H_BULK_REMOVE, h_bulk_remove);
609
610 }
611
612 type_init(hypercall_register_types)
613