1 /* 2 * spu_switch.c 3 * 4 * (C) Copyright IBM Corp. 2005 5 * 6 * Author: Mark Nutter <mnutter@us.ibm.com> 7 * 8 * Host-side part of SPU context switch sequence outlined in 9 * Synergistic Processor Element, Book IV. 10 * 11 * A fully premptive switch of an SPE is very expensive in terms 12 * of time and system resources. SPE Book IV indicates that SPE 13 * allocation should follow a "serially reusable device" model, 14 * in which the SPE is assigned a task until it completes. When 15 * this is not possible, this sequence may be used to premptively 16 * save, and then later (optionally) restore the context of a 17 * program executing on an SPE. 18 * 19 * 20 * This program is free software; you can redistribute it and/or modify 21 * it under the terms of the GNU General Public License as published by 22 * the Free Software Foundation; either version 2, or (at your option) 23 * any later version. 24 * 25 * This program is distributed in the hope that it will be useful, 26 * but WITHOUT ANY WARRANTY; without even the implied warranty of 27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 28 * GNU General Public License for more details. 29 * 30 * You should have received a copy of the GNU General Public License 31 * along with this program; if not, write to the Free Software 32 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 33 */ 34 35 #include <linux/module.h> 36 #include <linux/errno.h> 37 #include <linux/hardirq.h> 38 #include <linux/sched.h> 39 #include <linux/kernel.h> 40 #include <linux/mm.h> 41 #include <linux/vmalloc.h> 42 #include <linux/smp.h> 43 #include <linux/stddef.h> 44 #include <linux/unistd.h> 45 46 #include <asm/io.h> 47 #include <asm/spu.h> 48 #include <asm/spu_priv1.h> 49 #include <asm/spu_csa.h> 50 #include <asm/mmu_context.h> 51 52 #include "spufs.h" 53 54 #include "spu_save_dump.h" 55 #include "spu_restore_dump.h" 56 57 #if 0 58 #define POLL_WHILE_TRUE(_c) { \ 59 do { \ 60 } while (_c); \ 61 } 62 #else 63 #define RELAX_SPIN_COUNT 1000 64 #define POLL_WHILE_TRUE(_c) { \ 65 do { \ 66 int _i; \ 67 for (_i=0; _i<RELAX_SPIN_COUNT && (_c); _i++) { \ 68 cpu_relax(); \ 69 } \ 70 if (unlikely(_c)) yield(); \ 71 else break; \ 72 } while (_c); \ 73 } 74 #endif /* debug */ 75 76 #define POLL_WHILE_FALSE(_c) POLL_WHILE_TRUE(!(_c)) 77 78 static inline void acquire_spu_lock(struct spu *spu) 79 { 80 /* Save, Step 1: 81 * Restore, Step 1: 82 * Acquire SPU-specific mutual exclusion lock. 83 * TBD. 84 */ 85 } 86 87 static inline void release_spu_lock(struct spu *spu) 88 { 89 /* Restore, Step 76: 90 * Release SPU-specific mutual exclusion lock. 91 * TBD. 92 */ 93 } 94 95 static inline int check_spu_isolate(struct spu_state *csa, struct spu *spu) 96 { 97 struct spu_problem __iomem *prob = spu->problem; 98 u32 isolate_state; 99 100 /* Save, Step 2: 101 * Save, Step 6: 102 * If SPU_Status[E,L,IS] any field is '1', this 103 * SPU is in isolate state and cannot be context 104 * saved at this time. 105 */ 106 isolate_state = SPU_STATUS_ISOLATED_STATE | 107 SPU_STATUS_ISOLATED_LOAD_STATUS | SPU_STATUS_ISOLATED_EXIT_STATUS; 108 return (in_be32(&prob->spu_status_R) & isolate_state) ? 1 : 0; 109 } 110 111 static inline void disable_interrupts(struct spu_state *csa, struct spu *spu) 112 { 113 /* Save, Step 3: 114 * Restore, Step 2: 115 * Save INT_Mask_class0 in CSA. 116 * Write INT_MASK_class0 with value of 0. 117 * Save INT_Mask_class1 in CSA. 118 * Write INT_MASK_class1 with value of 0. 119 * Save INT_Mask_class2 in CSA. 120 * Write INT_MASK_class2 with value of 0. 121 * Synchronize all three interrupts to be sure 122 * we no longer execute a handler on another CPU. 123 */ 124 spin_lock_irq(&spu->register_lock); 125 if (csa) { 126 csa->priv1.int_mask_class0_RW = spu_int_mask_get(spu, 0); 127 csa->priv1.int_mask_class1_RW = spu_int_mask_get(spu, 1); 128 csa->priv1.int_mask_class2_RW = spu_int_mask_get(spu, 2); 129 } 130 spu_int_mask_set(spu, 0, 0ul); 131 spu_int_mask_set(spu, 1, 0ul); 132 spu_int_mask_set(spu, 2, 0ul); 133 eieio(); 134 spin_unlock_irq(&spu->register_lock); 135 136 /* 137 * This flag needs to be set before calling synchronize_irq so 138 * that the update will be visible to the relevant handlers 139 * via a simple load. 140 */ 141 set_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags); 142 synchronize_irq(spu->irqs[0]); 143 synchronize_irq(spu->irqs[1]); 144 synchronize_irq(spu->irqs[2]); 145 } 146 147 static inline void set_watchdog_timer(struct spu_state *csa, struct spu *spu) 148 { 149 /* Save, Step 4: 150 * Restore, Step 25. 151 * Set a software watchdog timer, which specifies the 152 * maximum allowable time for a context save sequence. 153 * 154 * For present, this implementation will not set a global 155 * watchdog timer, as virtualization & variable system load 156 * may cause unpredictable execution times. 157 */ 158 } 159 160 static inline void inhibit_user_access(struct spu_state *csa, struct spu *spu) 161 { 162 /* Save, Step 5: 163 * Restore, Step 3: 164 * Inhibit user-space access (if provided) to this 165 * SPU by unmapping the virtual pages assigned to 166 * the SPU memory-mapped I/O (MMIO) for problem 167 * state. TBD. 168 */ 169 } 170 171 static inline void set_switch_pending(struct spu_state *csa, struct spu *spu) 172 { 173 /* Save, Step 7: 174 * Restore, Step 5: 175 * Set a software context switch pending flag. 176 * Done above in Step 3 - disable_interrupts(). 177 */ 178 } 179 180 static inline void save_mfc_cntl(struct spu_state *csa, struct spu *spu) 181 { 182 struct spu_priv2 __iomem *priv2 = spu->priv2; 183 184 /* Save, Step 8: 185 * Suspend DMA and save MFC_CNTL. 186 */ 187 switch (in_be64(&priv2->mfc_control_RW) & 188 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) { 189 case MFC_CNTL_SUSPEND_IN_PROGRESS: 190 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 191 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) == 192 MFC_CNTL_SUSPEND_COMPLETE); 193 /* fall through */ 194 case MFC_CNTL_SUSPEND_COMPLETE: 195 if (csa) 196 csa->priv2.mfc_control_RW = 197 in_be64(&priv2->mfc_control_RW) | 198 MFC_CNTL_SUSPEND_DMA_QUEUE; 199 break; 200 case MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION: 201 out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE); 202 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 203 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) == 204 MFC_CNTL_SUSPEND_COMPLETE); 205 if (csa) 206 csa->priv2.mfc_control_RW = 207 in_be64(&priv2->mfc_control_RW) & 208 ~MFC_CNTL_SUSPEND_DMA_QUEUE & 209 ~MFC_CNTL_SUSPEND_MASK; 210 break; 211 } 212 } 213 214 static inline void save_spu_runcntl(struct spu_state *csa, struct spu *spu) 215 { 216 struct spu_problem __iomem *prob = spu->problem; 217 218 /* Save, Step 9: 219 * Save SPU_Runcntl in the CSA. This value contains 220 * the "Application Desired State". 221 */ 222 csa->prob.spu_runcntl_RW = in_be32(&prob->spu_runcntl_RW); 223 } 224 225 static inline void save_mfc_sr1(struct spu_state *csa, struct spu *spu) 226 { 227 /* Save, Step 10: 228 * Save MFC_SR1 in the CSA. 229 */ 230 csa->priv1.mfc_sr1_RW = spu_mfc_sr1_get(spu); 231 } 232 233 static inline void save_spu_status(struct spu_state *csa, struct spu *spu) 234 { 235 struct spu_problem __iomem *prob = spu->problem; 236 237 /* Save, Step 11: 238 * Read SPU_Status[R], and save to CSA. 239 */ 240 if ((in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) == 0) { 241 csa->prob.spu_status_R = in_be32(&prob->spu_status_R); 242 } else { 243 u32 stopped; 244 245 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 246 eieio(); 247 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 248 SPU_STATUS_RUNNING); 249 stopped = 250 SPU_STATUS_INVALID_INSTR | SPU_STATUS_SINGLE_STEP | 251 SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP; 252 if ((in_be32(&prob->spu_status_R) & stopped) == 0) 253 csa->prob.spu_status_R = SPU_STATUS_RUNNING; 254 else 255 csa->prob.spu_status_R = in_be32(&prob->spu_status_R); 256 } 257 } 258 259 static inline void save_mfc_stopped_status(struct spu_state *csa, 260 struct spu *spu) 261 { 262 struct spu_priv2 __iomem *priv2 = spu->priv2; 263 const u64 mask = MFC_CNTL_DECREMENTER_RUNNING | 264 MFC_CNTL_DMA_QUEUES_EMPTY; 265 266 /* Save, Step 12: 267 * Read MFC_CNTL[Ds]. Update saved copy of 268 * CSA.MFC_CNTL[Ds]. 269 * 270 * update: do the same with MFC_CNTL[Q]. 271 */ 272 csa->priv2.mfc_control_RW &= ~mask; 273 csa->priv2.mfc_control_RW |= in_be64(&priv2->mfc_control_RW) & mask; 274 } 275 276 static inline void halt_mfc_decr(struct spu_state *csa, struct spu *spu) 277 { 278 struct spu_priv2 __iomem *priv2 = spu->priv2; 279 280 /* Save, Step 13: 281 * Write MFC_CNTL[Dh] set to a '1' to halt 282 * the decrementer. 283 */ 284 out_be64(&priv2->mfc_control_RW, 285 MFC_CNTL_DECREMENTER_HALTED | MFC_CNTL_SUSPEND_MASK); 286 eieio(); 287 } 288 289 static inline void save_timebase(struct spu_state *csa, struct spu *spu) 290 { 291 /* Save, Step 14: 292 * Read PPE Timebase High and Timebase low registers 293 * and save in CSA. TBD. 294 */ 295 csa->suspend_time = get_cycles(); 296 } 297 298 static inline void remove_other_spu_access(struct spu_state *csa, 299 struct spu *spu) 300 { 301 /* Save, Step 15: 302 * Remove other SPU access to this SPU by unmapping 303 * this SPU's pages from their address space. TBD. 304 */ 305 } 306 307 static inline void do_mfc_mssync(struct spu_state *csa, struct spu *spu) 308 { 309 struct spu_problem __iomem *prob = spu->problem; 310 311 /* Save, Step 16: 312 * Restore, Step 11. 313 * Write SPU_MSSync register. Poll SPU_MSSync[P] 314 * for a value of 0. 315 */ 316 out_be64(&prob->spc_mssync_RW, 1UL); 317 POLL_WHILE_TRUE(in_be64(&prob->spc_mssync_RW) & MS_SYNC_PENDING); 318 } 319 320 static inline void issue_mfc_tlbie(struct spu_state *csa, struct spu *spu) 321 { 322 /* Save, Step 17: 323 * Restore, Step 12. 324 * Restore, Step 48. 325 * Write TLB_Invalidate_Entry[IS,VPN,L,Lp]=0 register. 326 * Then issue a PPE sync instruction. 327 */ 328 spu_tlb_invalidate(spu); 329 mb(); 330 } 331 332 static inline void handle_pending_interrupts(struct spu_state *csa, 333 struct spu *spu) 334 { 335 /* Save, Step 18: 336 * Handle any pending interrupts from this SPU 337 * here. This is OS or hypervisor specific. One 338 * option is to re-enable interrupts to handle any 339 * pending interrupts, with the interrupt handlers 340 * recognizing the software Context Switch Pending 341 * flag, to ensure the SPU execution or MFC command 342 * queue is not restarted. TBD. 343 */ 344 } 345 346 static inline void save_mfc_queues(struct spu_state *csa, struct spu *spu) 347 { 348 struct spu_priv2 __iomem *priv2 = spu->priv2; 349 int i; 350 351 /* Save, Step 19: 352 * If MFC_Cntl[Se]=0 then save 353 * MFC command queues. 354 */ 355 if ((in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DMA_QUEUES_EMPTY) == 0) { 356 for (i = 0; i < 8; i++) { 357 csa->priv2.puq[i].mfc_cq_data0_RW = 358 in_be64(&priv2->puq[i].mfc_cq_data0_RW); 359 csa->priv2.puq[i].mfc_cq_data1_RW = 360 in_be64(&priv2->puq[i].mfc_cq_data1_RW); 361 csa->priv2.puq[i].mfc_cq_data2_RW = 362 in_be64(&priv2->puq[i].mfc_cq_data2_RW); 363 csa->priv2.puq[i].mfc_cq_data3_RW = 364 in_be64(&priv2->puq[i].mfc_cq_data3_RW); 365 } 366 for (i = 0; i < 16; i++) { 367 csa->priv2.spuq[i].mfc_cq_data0_RW = 368 in_be64(&priv2->spuq[i].mfc_cq_data0_RW); 369 csa->priv2.spuq[i].mfc_cq_data1_RW = 370 in_be64(&priv2->spuq[i].mfc_cq_data1_RW); 371 csa->priv2.spuq[i].mfc_cq_data2_RW = 372 in_be64(&priv2->spuq[i].mfc_cq_data2_RW); 373 csa->priv2.spuq[i].mfc_cq_data3_RW = 374 in_be64(&priv2->spuq[i].mfc_cq_data3_RW); 375 } 376 } 377 } 378 379 static inline void save_ppu_querymask(struct spu_state *csa, struct spu *spu) 380 { 381 struct spu_problem __iomem *prob = spu->problem; 382 383 /* Save, Step 20: 384 * Save the PPU_QueryMask register 385 * in the CSA. 386 */ 387 csa->prob.dma_querymask_RW = in_be32(&prob->dma_querymask_RW); 388 } 389 390 static inline void save_ppu_querytype(struct spu_state *csa, struct spu *spu) 391 { 392 struct spu_problem __iomem *prob = spu->problem; 393 394 /* Save, Step 21: 395 * Save the PPU_QueryType register 396 * in the CSA. 397 */ 398 csa->prob.dma_querytype_RW = in_be32(&prob->dma_querytype_RW); 399 } 400 401 static inline void save_ppu_tagstatus(struct spu_state *csa, struct spu *spu) 402 { 403 struct spu_problem __iomem *prob = spu->problem; 404 405 /* Save the Prxy_TagStatus register in the CSA. 406 * 407 * It is unnecessary to restore dma_tagstatus_R, however, 408 * dma_tagstatus_R in the CSA is accessed via backing_ops, so 409 * we must save it. 410 */ 411 csa->prob.dma_tagstatus_R = in_be32(&prob->dma_tagstatus_R); 412 } 413 414 static inline void save_mfc_csr_tsq(struct spu_state *csa, struct spu *spu) 415 { 416 struct spu_priv2 __iomem *priv2 = spu->priv2; 417 418 /* Save, Step 22: 419 * Save the MFC_CSR_TSQ register 420 * in the LSCSA. 421 */ 422 csa->priv2.spu_tag_status_query_RW = 423 in_be64(&priv2->spu_tag_status_query_RW); 424 } 425 426 static inline void save_mfc_csr_cmd(struct spu_state *csa, struct spu *spu) 427 { 428 struct spu_priv2 __iomem *priv2 = spu->priv2; 429 430 /* Save, Step 23: 431 * Save the MFC_CSR_CMD1 and MFC_CSR_CMD2 432 * registers in the CSA. 433 */ 434 csa->priv2.spu_cmd_buf1_RW = in_be64(&priv2->spu_cmd_buf1_RW); 435 csa->priv2.spu_cmd_buf2_RW = in_be64(&priv2->spu_cmd_buf2_RW); 436 } 437 438 static inline void save_mfc_csr_ato(struct spu_state *csa, struct spu *spu) 439 { 440 struct spu_priv2 __iomem *priv2 = spu->priv2; 441 442 /* Save, Step 24: 443 * Save the MFC_CSR_ATO register in 444 * the CSA. 445 */ 446 csa->priv2.spu_atomic_status_RW = in_be64(&priv2->spu_atomic_status_RW); 447 } 448 449 static inline void save_mfc_tclass_id(struct spu_state *csa, struct spu *spu) 450 { 451 /* Save, Step 25: 452 * Save the MFC_TCLASS_ID register in 453 * the CSA. 454 */ 455 csa->priv1.mfc_tclass_id_RW = spu_mfc_tclass_id_get(spu); 456 } 457 458 static inline void set_mfc_tclass_id(struct spu_state *csa, struct spu *spu) 459 { 460 /* Save, Step 26: 461 * Restore, Step 23. 462 * Write the MFC_TCLASS_ID register with 463 * the value 0x10000000. 464 */ 465 spu_mfc_tclass_id_set(spu, 0x10000000); 466 eieio(); 467 } 468 469 static inline void purge_mfc_queue(struct spu_state *csa, struct spu *spu) 470 { 471 struct spu_priv2 __iomem *priv2 = spu->priv2; 472 473 /* Save, Step 27: 474 * Restore, Step 14. 475 * Write MFC_CNTL[Pc]=1 (purge queue). 476 */ 477 out_be64(&priv2->mfc_control_RW, 478 MFC_CNTL_PURGE_DMA_REQUEST | 479 MFC_CNTL_SUSPEND_MASK); 480 eieio(); 481 } 482 483 static inline void wait_purge_complete(struct spu_state *csa, struct spu *spu) 484 { 485 struct spu_priv2 __iomem *priv2 = spu->priv2; 486 487 /* Save, Step 28: 488 * Poll MFC_CNTL[Ps] until value '11' is read 489 * (purge complete). 490 */ 491 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 492 MFC_CNTL_PURGE_DMA_STATUS_MASK) == 493 MFC_CNTL_PURGE_DMA_COMPLETE); 494 } 495 496 static inline void setup_mfc_sr1(struct spu_state *csa, struct spu *spu) 497 { 498 /* Save, Step 30: 499 * Restore, Step 18: 500 * Write MFC_SR1 with MFC_SR1[D=0,S=1] and 501 * MFC_SR1[TL,R,Pr,T] set correctly for the 502 * OS specific environment. 503 * 504 * Implementation note: The SPU-side code 505 * for save/restore is privileged, so the 506 * MFC_SR1[Pr] bit is not set. 507 * 508 */ 509 spu_mfc_sr1_set(spu, (MFC_STATE1_MASTER_RUN_CONTROL_MASK | 510 MFC_STATE1_RELOCATE_MASK | 511 MFC_STATE1_BUS_TLBIE_MASK)); 512 } 513 514 static inline void save_spu_npc(struct spu_state *csa, struct spu *spu) 515 { 516 struct spu_problem __iomem *prob = spu->problem; 517 518 /* Save, Step 31: 519 * Save SPU_NPC in the CSA. 520 */ 521 csa->prob.spu_npc_RW = in_be32(&prob->spu_npc_RW); 522 } 523 524 static inline void save_spu_privcntl(struct spu_state *csa, struct spu *spu) 525 { 526 struct spu_priv2 __iomem *priv2 = spu->priv2; 527 528 /* Save, Step 32: 529 * Save SPU_PrivCntl in the CSA. 530 */ 531 csa->priv2.spu_privcntl_RW = in_be64(&priv2->spu_privcntl_RW); 532 } 533 534 static inline void reset_spu_privcntl(struct spu_state *csa, struct spu *spu) 535 { 536 struct spu_priv2 __iomem *priv2 = spu->priv2; 537 538 /* Save, Step 33: 539 * Restore, Step 16: 540 * Write SPU_PrivCntl[S,Le,A] fields reset to 0. 541 */ 542 out_be64(&priv2->spu_privcntl_RW, 0UL); 543 eieio(); 544 } 545 546 static inline void save_spu_lslr(struct spu_state *csa, struct spu *spu) 547 { 548 struct spu_priv2 __iomem *priv2 = spu->priv2; 549 550 /* Save, Step 34: 551 * Save SPU_LSLR in the CSA. 552 */ 553 csa->priv2.spu_lslr_RW = in_be64(&priv2->spu_lslr_RW); 554 } 555 556 static inline void reset_spu_lslr(struct spu_state *csa, struct spu *spu) 557 { 558 struct spu_priv2 __iomem *priv2 = spu->priv2; 559 560 /* Save, Step 35: 561 * Restore, Step 17. 562 * Reset SPU_LSLR. 563 */ 564 out_be64(&priv2->spu_lslr_RW, LS_ADDR_MASK); 565 eieio(); 566 } 567 568 static inline void save_spu_cfg(struct spu_state *csa, struct spu *spu) 569 { 570 struct spu_priv2 __iomem *priv2 = spu->priv2; 571 572 /* Save, Step 36: 573 * Save SPU_Cfg in the CSA. 574 */ 575 csa->priv2.spu_cfg_RW = in_be64(&priv2->spu_cfg_RW); 576 } 577 578 static inline void save_pm_trace(struct spu_state *csa, struct spu *spu) 579 { 580 /* Save, Step 37: 581 * Save PM_Trace_Tag_Wait_Mask in the CSA. 582 * Not performed by this implementation. 583 */ 584 } 585 586 static inline void save_mfc_rag(struct spu_state *csa, struct spu *spu) 587 { 588 /* Save, Step 38: 589 * Save RA_GROUP_ID register and the 590 * RA_ENABLE reigster in the CSA. 591 */ 592 csa->priv1.resource_allocation_groupID_RW = 593 spu_resource_allocation_groupID_get(spu); 594 csa->priv1.resource_allocation_enable_RW = 595 spu_resource_allocation_enable_get(spu); 596 } 597 598 static inline void save_ppu_mb_stat(struct spu_state *csa, struct spu *spu) 599 { 600 struct spu_problem __iomem *prob = spu->problem; 601 602 /* Save, Step 39: 603 * Save MB_Stat register in the CSA. 604 */ 605 csa->prob.mb_stat_R = in_be32(&prob->mb_stat_R); 606 } 607 608 static inline void save_ppu_mb(struct spu_state *csa, struct spu *spu) 609 { 610 struct spu_problem __iomem *prob = spu->problem; 611 612 /* Save, Step 40: 613 * Save the PPU_MB register in the CSA. 614 */ 615 csa->prob.pu_mb_R = in_be32(&prob->pu_mb_R); 616 } 617 618 static inline void save_ppuint_mb(struct spu_state *csa, struct spu *spu) 619 { 620 struct spu_priv2 __iomem *priv2 = spu->priv2; 621 622 /* Save, Step 41: 623 * Save the PPUINT_MB register in the CSA. 624 */ 625 csa->priv2.puint_mb_R = in_be64(&priv2->puint_mb_R); 626 } 627 628 static inline void save_ch_part1(struct spu_state *csa, struct spu *spu) 629 { 630 struct spu_priv2 __iomem *priv2 = spu->priv2; 631 u64 idx, ch_indices[] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL }; 632 int i; 633 634 /* Save, Step 42: 635 */ 636 637 /* Save CH 1, without channel count */ 638 out_be64(&priv2->spu_chnlcntptr_RW, 1); 639 csa->spu_chnldata_RW[1] = in_be64(&priv2->spu_chnldata_RW); 640 641 /* Save the following CH: [0,3,4,24,25,27] */ 642 for (i = 0; i < ARRAY_SIZE(ch_indices); i++) { 643 idx = ch_indices[i]; 644 out_be64(&priv2->spu_chnlcntptr_RW, idx); 645 eieio(); 646 csa->spu_chnldata_RW[idx] = in_be64(&priv2->spu_chnldata_RW); 647 csa->spu_chnlcnt_RW[idx] = in_be64(&priv2->spu_chnlcnt_RW); 648 out_be64(&priv2->spu_chnldata_RW, 0UL); 649 out_be64(&priv2->spu_chnlcnt_RW, 0UL); 650 eieio(); 651 } 652 } 653 654 static inline void save_spu_mb(struct spu_state *csa, struct spu *spu) 655 { 656 struct spu_priv2 __iomem *priv2 = spu->priv2; 657 int i; 658 659 /* Save, Step 43: 660 * Save SPU Read Mailbox Channel. 661 */ 662 out_be64(&priv2->spu_chnlcntptr_RW, 29UL); 663 eieio(); 664 csa->spu_chnlcnt_RW[29] = in_be64(&priv2->spu_chnlcnt_RW); 665 for (i = 0; i < 4; i++) { 666 csa->spu_mailbox_data[i] = in_be64(&priv2->spu_chnldata_RW); 667 } 668 out_be64(&priv2->spu_chnlcnt_RW, 0UL); 669 eieio(); 670 } 671 672 static inline void save_mfc_cmd(struct spu_state *csa, struct spu *spu) 673 { 674 struct spu_priv2 __iomem *priv2 = spu->priv2; 675 676 /* Save, Step 44: 677 * Save MFC_CMD Channel. 678 */ 679 out_be64(&priv2->spu_chnlcntptr_RW, 21UL); 680 eieio(); 681 csa->spu_chnlcnt_RW[21] = in_be64(&priv2->spu_chnlcnt_RW); 682 eieio(); 683 } 684 685 static inline void reset_ch(struct spu_state *csa, struct spu *spu) 686 { 687 struct spu_priv2 __iomem *priv2 = spu->priv2; 688 u64 ch_indices[4] = { 21UL, 23UL, 28UL, 30UL }; 689 u64 ch_counts[4] = { 16UL, 1UL, 1UL, 1UL }; 690 u64 idx; 691 int i; 692 693 /* Save, Step 45: 694 * Reset the following CH: [21, 23, 28, 30] 695 */ 696 for (i = 0; i < 4; i++) { 697 idx = ch_indices[i]; 698 out_be64(&priv2->spu_chnlcntptr_RW, idx); 699 eieio(); 700 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]); 701 eieio(); 702 } 703 } 704 705 static inline void resume_mfc_queue(struct spu_state *csa, struct spu *spu) 706 { 707 struct spu_priv2 __iomem *priv2 = spu->priv2; 708 709 /* Save, Step 46: 710 * Restore, Step 25. 711 * Write MFC_CNTL[Sc]=0 (resume queue processing). 712 */ 713 out_be64(&priv2->mfc_control_RW, MFC_CNTL_RESUME_DMA_QUEUE); 714 } 715 716 static inline void setup_mfc_slbs(struct spu_state *csa, struct spu *spu, 717 unsigned int *code, int code_size) 718 { 719 /* Save, Step 47: 720 * Restore, Step 30. 721 * If MFC_SR1[R]=1, write 0 to SLB_Invalidate_All 722 * register, then initialize SLB_VSID and SLB_ESID 723 * to provide access to SPU context save code and 724 * LSCSA. 725 * 726 * This implementation places both the context 727 * switch code and LSCSA in kernel address space. 728 * 729 * Further this implementation assumes that the 730 * MFC_SR1[R]=1 (in other words, assume that 731 * translation is desired by OS environment). 732 */ 733 spu_invalidate_slbs(spu); 734 spu_setup_kernel_slbs(spu, csa->lscsa, code, code_size); 735 } 736 737 static inline void set_switch_active(struct spu_state *csa, struct spu *spu) 738 { 739 /* Save, Step 48: 740 * Restore, Step 23. 741 * Change the software context switch pending flag 742 * to context switch active. 743 * 744 * This implementation does not uses a switch active flag. 745 */ 746 clear_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags); 747 mb(); 748 } 749 750 static inline void enable_interrupts(struct spu_state *csa, struct spu *spu) 751 { 752 unsigned long class1_mask = CLASS1_ENABLE_SEGMENT_FAULT_INTR | 753 CLASS1_ENABLE_STORAGE_FAULT_INTR; 754 755 /* Save, Step 49: 756 * Restore, Step 22: 757 * Reset and then enable interrupts, as 758 * needed by OS. 759 * 760 * This implementation enables only class1 761 * (translation) interrupts. 762 */ 763 spin_lock_irq(&spu->register_lock); 764 spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK); 765 spu_int_stat_clear(spu, 1, CLASS1_INTR_MASK); 766 spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK); 767 spu_int_mask_set(spu, 0, 0ul); 768 spu_int_mask_set(spu, 1, class1_mask); 769 spu_int_mask_set(spu, 2, 0ul); 770 spin_unlock_irq(&spu->register_lock); 771 } 772 773 static inline int send_mfc_dma(struct spu *spu, unsigned long ea, 774 unsigned int ls_offset, unsigned int size, 775 unsigned int tag, unsigned int rclass, 776 unsigned int cmd) 777 { 778 struct spu_problem __iomem *prob = spu->problem; 779 union mfc_tag_size_class_cmd command; 780 unsigned int transfer_size; 781 volatile unsigned int status = 0x0; 782 783 while (size > 0) { 784 transfer_size = 785 (size > MFC_MAX_DMA_SIZE) ? MFC_MAX_DMA_SIZE : size; 786 command.u.mfc_size = transfer_size; 787 command.u.mfc_tag = tag; 788 command.u.mfc_rclassid = rclass; 789 command.u.mfc_cmd = cmd; 790 do { 791 out_be32(&prob->mfc_lsa_W, ls_offset); 792 out_be64(&prob->mfc_ea_W, ea); 793 out_be64(&prob->mfc_union_W.all64, command.all64); 794 status = 795 in_be32(&prob->mfc_union_W.by32.mfc_class_cmd32); 796 if (unlikely(status & 0x2)) { 797 cpu_relax(); 798 } 799 } while (status & 0x3); 800 size -= transfer_size; 801 ea += transfer_size; 802 ls_offset += transfer_size; 803 } 804 return 0; 805 } 806 807 static inline void save_ls_16kb(struct spu_state *csa, struct spu *spu) 808 { 809 unsigned long addr = (unsigned long)&csa->lscsa->ls[0]; 810 unsigned int ls_offset = 0x0; 811 unsigned int size = 16384; 812 unsigned int tag = 0; 813 unsigned int rclass = 0; 814 unsigned int cmd = MFC_PUT_CMD; 815 816 /* Save, Step 50: 817 * Issue a DMA command to copy the first 16K bytes 818 * of local storage to the CSA. 819 */ 820 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 821 } 822 823 static inline void set_spu_npc(struct spu_state *csa, struct spu *spu) 824 { 825 struct spu_problem __iomem *prob = spu->problem; 826 827 /* Save, Step 51: 828 * Restore, Step 31. 829 * Write SPU_NPC[IE]=0 and SPU_NPC[LSA] to entry 830 * point address of context save code in local 831 * storage. 832 * 833 * This implementation uses SPU-side save/restore 834 * programs with entry points at LSA of 0. 835 */ 836 out_be32(&prob->spu_npc_RW, 0); 837 eieio(); 838 } 839 840 static inline void set_signot1(struct spu_state *csa, struct spu *spu) 841 { 842 struct spu_problem __iomem *prob = spu->problem; 843 union { 844 u64 ull; 845 u32 ui[2]; 846 } addr64; 847 848 /* Save, Step 52: 849 * Restore, Step 32: 850 * Write SPU_Sig_Notify_1 register with upper 32-bits 851 * of the CSA.LSCSA effective address. 852 */ 853 addr64.ull = (u64) csa->lscsa; 854 out_be32(&prob->signal_notify1, addr64.ui[0]); 855 eieio(); 856 } 857 858 static inline void set_signot2(struct spu_state *csa, struct spu *spu) 859 { 860 struct spu_problem __iomem *prob = spu->problem; 861 union { 862 u64 ull; 863 u32 ui[2]; 864 } addr64; 865 866 /* Save, Step 53: 867 * Restore, Step 33: 868 * Write SPU_Sig_Notify_2 register with lower 32-bits 869 * of the CSA.LSCSA effective address. 870 */ 871 addr64.ull = (u64) csa->lscsa; 872 out_be32(&prob->signal_notify2, addr64.ui[1]); 873 eieio(); 874 } 875 876 static inline void send_save_code(struct spu_state *csa, struct spu *spu) 877 { 878 unsigned long addr = (unsigned long)&spu_save_code[0]; 879 unsigned int ls_offset = 0x0; 880 unsigned int size = sizeof(spu_save_code); 881 unsigned int tag = 0; 882 unsigned int rclass = 0; 883 unsigned int cmd = MFC_GETFS_CMD; 884 885 /* Save, Step 54: 886 * Issue a DMA command to copy context save code 887 * to local storage and start SPU. 888 */ 889 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 890 } 891 892 static inline void set_ppu_querymask(struct spu_state *csa, struct spu *spu) 893 { 894 struct spu_problem __iomem *prob = spu->problem; 895 896 /* Save, Step 55: 897 * Restore, Step 38. 898 * Write PPU_QueryMask=1 (enable Tag Group 0) 899 * and issue eieio instruction. 900 */ 901 out_be32(&prob->dma_querymask_RW, MFC_TAGID_TO_TAGMASK(0)); 902 eieio(); 903 } 904 905 static inline void wait_tag_complete(struct spu_state *csa, struct spu *spu) 906 { 907 struct spu_problem __iomem *prob = spu->problem; 908 u32 mask = MFC_TAGID_TO_TAGMASK(0); 909 unsigned long flags; 910 911 /* Save, Step 56: 912 * Restore, Step 39. 913 * Restore, Step 39. 914 * Restore, Step 46. 915 * Poll PPU_TagStatus[gn] until 01 (Tag group 0 complete) 916 * or write PPU_QueryType[TS]=01 and wait for Tag Group 917 * Complete Interrupt. Write INT_Stat_Class0 or 918 * INT_Stat_Class2 with value of 'handled'. 919 */ 920 POLL_WHILE_FALSE(in_be32(&prob->dma_tagstatus_R) & mask); 921 922 local_irq_save(flags); 923 spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK); 924 spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK); 925 local_irq_restore(flags); 926 } 927 928 static inline void wait_spu_stopped(struct spu_state *csa, struct spu *spu) 929 { 930 struct spu_problem __iomem *prob = spu->problem; 931 unsigned long flags; 932 933 /* Save, Step 57: 934 * Restore, Step 40. 935 * Poll until SPU_Status[R]=0 or wait for SPU Class 0 936 * or SPU Class 2 interrupt. Write INT_Stat_class0 937 * or INT_Stat_class2 with value of handled. 938 */ 939 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING); 940 941 local_irq_save(flags); 942 spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK); 943 spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK); 944 local_irq_restore(flags); 945 } 946 947 static inline int check_save_status(struct spu_state *csa, struct spu *spu) 948 { 949 struct spu_problem __iomem *prob = spu->problem; 950 u32 complete; 951 952 /* Save, Step 54: 953 * If SPU_Status[P]=1 and SPU_Status[SC] = "success", 954 * context save succeeded, otherwise context save 955 * failed. 956 */ 957 complete = ((SPU_SAVE_COMPLETE << SPU_STOP_STATUS_SHIFT) | 958 SPU_STATUS_STOPPED_BY_STOP); 959 return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0; 960 } 961 962 static inline void terminate_spu_app(struct spu_state *csa, struct spu *spu) 963 { 964 /* Restore, Step 4: 965 * If required, notify the "using application" that 966 * the SPU task has been terminated. TBD. 967 */ 968 } 969 970 static inline void suspend_mfc_and_halt_decr(struct spu_state *csa, 971 struct spu *spu) 972 { 973 struct spu_priv2 __iomem *priv2 = spu->priv2; 974 975 /* Restore, Step 7: 976 * Write MFC_Cntl[Dh,Sc,Sm]='1','1','0' to suspend 977 * the queue and halt the decrementer. 978 */ 979 out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE | 980 MFC_CNTL_DECREMENTER_HALTED); 981 eieio(); 982 } 983 984 static inline void wait_suspend_mfc_complete(struct spu_state *csa, 985 struct spu *spu) 986 { 987 struct spu_priv2 __iomem *priv2 = spu->priv2; 988 989 /* Restore, Step 8: 990 * Restore, Step 47. 991 * Poll MFC_CNTL[Ss] until 11 is returned. 992 */ 993 POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) & 994 MFC_CNTL_SUSPEND_DMA_STATUS_MASK) == 995 MFC_CNTL_SUSPEND_COMPLETE); 996 } 997 998 static inline int suspend_spe(struct spu_state *csa, struct spu *spu) 999 { 1000 struct spu_problem __iomem *prob = spu->problem; 1001 1002 /* Restore, Step 9: 1003 * If SPU_Status[R]=1, stop SPU execution 1004 * and wait for stop to complete. 1005 * 1006 * Returns 1 if SPU_Status[R]=1 on entry. 1007 * 0 otherwise 1008 */ 1009 if (in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) { 1010 if (in_be32(&prob->spu_status_R) & 1011 SPU_STATUS_ISOLATED_EXIT_STATUS) { 1012 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1013 SPU_STATUS_RUNNING); 1014 } 1015 if ((in_be32(&prob->spu_status_R) & 1016 SPU_STATUS_ISOLATED_LOAD_STATUS) 1017 || (in_be32(&prob->spu_status_R) & 1018 SPU_STATUS_ISOLATED_STATE)) { 1019 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1020 eieio(); 1021 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1022 SPU_STATUS_RUNNING); 1023 out_be32(&prob->spu_runcntl_RW, 0x2); 1024 eieio(); 1025 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1026 SPU_STATUS_RUNNING); 1027 } 1028 if (in_be32(&prob->spu_status_R) & 1029 SPU_STATUS_WAITING_FOR_CHANNEL) { 1030 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1031 eieio(); 1032 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1033 SPU_STATUS_RUNNING); 1034 } 1035 return 1; 1036 } 1037 return 0; 1038 } 1039 1040 static inline void clear_spu_status(struct spu_state *csa, struct spu *spu) 1041 { 1042 struct spu_problem __iomem *prob = spu->problem; 1043 1044 /* Restore, Step 10: 1045 * If SPU_Status[R]=0 and SPU_Status[E,L,IS]=1, 1046 * release SPU from isolate state. 1047 */ 1048 if (!(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING)) { 1049 if (in_be32(&prob->spu_status_R) & 1050 SPU_STATUS_ISOLATED_EXIT_STATUS) { 1051 spu_mfc_sr1_set(spu, 1052 MFC_STATE1_MASTER_RUN_CONTROL_MASK); 1053 eieio(); 1054 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1055 eieio(); 1056 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1057 SPU_STATUS_RUNNING); 1058 } 1059 if ((in_be32(&prob->spu_status_R) & 1060 SPU_STATUS_ISOLATED_LOAD_STATUS) 1061 || (in_be32(&prob->spu_status_R) & 1062 SPU_STATUS_ISOLATED_STATE)) { 1063 spu_mfc_sr1_set(spu, 1064 MFC_STATE1_MASTER_RUN_CONTROL_MASK); 1065 eieio(); 1066 out_be32(&prob->spu_runcntl_RW, 0x2); 1067 eieio(); 1068 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1069 SPU_STATUS_RUNNING); 1070 } 1071 } 1072 } 1073 1074 static inline void reset_ch_part1(struct spu_state *csa, struct spu *spu) 1075 { 1076 struct spu_priv2 __iomem *priv2 = spu->priv2; 1077 u64 ch_indices[] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL }; 1078 u64 idx; 1079 int i; 1080 1081 /* Restore, Step 20: 1082 */ 1083 1084 /* Reset CH 1 */ 1085 out_be64(&priv2->spu_chnlcntptr_RW, 1); 1086 out_be64(&priv2->spu_chnldata_RW, 0UL); 1087 1088 /* Reset the following CH: [0,3,4,24,25,27] */ 1089 for (i = 0; i < ARRAY_SIZE(ch_indices); i++) { 1090 idx = ch_indices[i]; 1091 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1092 eieio(); 1093 out_be64(&priv2->spu_chnldata_RW, 0UL); 1094 out_be64(&priv2->spu_chnlcnt_RW, 0UL); 1095 eieio(); 1096 } 1097 } 1098 1099 static inline void reset_ch_part2(struct spu_state *csa, struct spu *spu) 1100 { 1101 struct spu_priv2 __iomem *priv2 = spu->priv2; 1102 u64 ch_indices[5] = { 21UL, 23UL, 28UL, 29UL, 30UL }; 1103 u64 ch_counts[5] = { 16UL, 1UL, 1UL, 0UL, 1UL }; 1104 u64 idx; 1105 int i; 1106 1107 /* Restore, Step 21: 1108 * Reset the following CH: [21, 23, 28, 29, 30] 1109 */ 1110 for (i = 0; i < 5; i++) { 1111 idx = ch_indices[i]; 1112 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1113 eieio(); 1114 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]); 1115 eieio(); 1116 } 1117 } 1118 1119 static inline void setup_spu_status_part1(struct spu_state *csa, 1120 struct spu *spu) 1121 { 1122 u32 status_P = SPU_STATUS_STOPPED_BY_STOP; 1123 u32 status_I = SPU_STATUS_INVALID_INSTR; 1124 u32 status_H = SPU_STATUS_STOPPED_BY_HALT; 1125 u32 status_S = SPU_STATUS_SINGLE_STEP; 1126 u32 status_S_I = SPU_STATUS_SINGLE_STEP | SPU_STATUS_INVALID_INSTR; 1127 u32 status_S_P = SPU_STATUS_SINGLE_STEP | SPU_STATUS_STOPPED_BY_STOP; 1128 u32 status_P_H = SPU_STATUS_STOPPED_BY_HALT |SPU_STATUS_STOPPED_BY_STOP; 1129 u32 status_P_I = SPU_STATUS_STOPPED_BY_STOP |SPU_STATUS_INVALID_INSTR; 1130 u32 status_code; 1131 1132 /* Restore, Step 27: 1133 * If the CSA.SPU_Status[I,S,H,P]=1 then add the correct 1134 * instruction sequence to the end of the SPU based restore 1135 * code (after the "context restored" stop and signal) to 1136 * restore the correct SPU status. 1137 * 1138 * NOTE: Rather than modifying the SPU executable, we 1139 * instead add a new 'stopped_status' field to the 1140 * LSCSA. The SPU-side restore reads this field and 1141 * takes the appropriate action when exiting. 1142 */ 1143 1144 status_code = 1145 (csa->prob.spu_status_R >> SPU_STOP_STATUS_SHIFT) & 0xFFFF; 1146 if ((csa->prob.spu_status_R & status_P_I) == status_P_I) { 1147 1148 /* SPU_Status[P,I]=1 - Illegal Instruction followed 1149 * by Stop and Signal instruction, followed by 'br -4'. 1150 * 1151 */ 1152 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_I; 1153 csa->lscsa->stopped_status.slot[1] = status_code; 1154 1155 } else if ((csa->prob.spu_status_R & status_P_H) == status_P_H) { 1156 1157 /* SPU_Status[P,H]=1 - Halt Conditional, followed 1158 * by Stop and Signal instruction, followed by 1159 * 'br -4'. 1160 */ 1161 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_H; 1162 csa->lscsa->stopped_status.slot[1] = status_code; 1163 1164 } else if ((csa->prob.spu_status_R & status_S_P) == status_S_P) { 1165 1166 /* SPU_Status[S,P]=1 - Stop and Signal instruction 1167 * followed by 'br -4'. 1168 */ 1169 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_P; 1170 csa->lscsa->stopped_status.slot[1] = status_code; 1171 1172 } else if ((csa->prob.spu_status_R & status_S_I) == status_S_I) { 1173 1174 /* SPU_Status[S,I]=1 - Illegal instruction followed 1175 * by 'br -4'. 1176 */ 1177 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_I; 1178 csa->lscsa->stopped_status.slot[1] = status_code; 1179 1180 } else if ((csa->prob.spu_status_R & status_P) == status_P) { 1181 1182 /* SPU_Status[P]=1 - Stop and Signal instruction 1183 * followed by 'br -4'. 1184 */ 1185 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P; 1186 csa->lscsa->stopped_status.slot[1] = status_code; 1187 1188 } else if ((csa->prob.spu_status_R & status_H) == status_H) { 1189 1190 /* SPU_Status[H]=1 - Halt Conditional, followed 1191 * by 'br -4'. 1192 */ 1193 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_H; 1194 1195 } else if ((csa->prob.spu_status_R & status_S) == status_S) { 1196 1197 /* SPU_Status[S]=1 - Two nop instructions. 1198 */ 1199 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S; 1200 1201 } else if ((csa->prob.spu_status_R & status_I) == status_I) { 1202 1203 /* SPU_Status[I]=1 - Illegal instruction followed 1204 * by 'br -4'. 1205 */ 1206 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_I; 1207 1208 } 1209 } 1210 1211 static inline void setup_spu_status_part2(struct spu_state *csa, 1212 struct spu *spu) 1213 { 1214 u32 mask; 1215 1216 /* Restore, Step 28: 1217 * If the CSA.SPU_Status[I,S,H,P,R]=0 then 1218 * add a 'br *' instruction to the end of 1219 * the SPU based restore code. 1220 * 1221 * NOTE: Rather than modifying the SPU executable, we 1222 * instead add a new 'stopped_status' field to the 1223 * LSCSA. The SPU-side restore reads this field and 1224 * takes the appropriate action when exiting. 1225 */ 1226 mask = SPU_STATUS_INVALID_INSTR | 1227 SPU_STATUS_SINGLE_STEP | 1228 SPU_STATUS_STOPPED_BY_HALT | 1229 SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING; 1230 if (!(csa->prob.spu_status_R & mask)) { 1231 csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_R; 1232 } 1233 } 1234 1235 static inline void restore_mfc_rag(struct spu_state *csa, struct spu *spu) 1236 { 1237 /* Restore, Step 29: 1238 * Restore RA_GROUP_ID register and the 1239 * RA_ENABLE reigster from the CSA. 1240 */ 1241 spu_resource_allocation_groupID_set(spu, 1242 csa->priv1.resource_allocation_groupID_RW); 1243 spu_resource_allocation_enable_set(spu, 1244 csa->priv1.resource_allocation_enable_RW); 1245 } 1246 1247 static inline void send_restore_code(struct spu_state *csa, struct spu *spu) 1248 { 1249 unsigned long addr = (unsigned long)&spu_restore_code[0]; 1250 unsigned int ls_offset = 0x0; 1251 unsigned int size = sizeof(spu_restore_code); 1252 unsigned int tag = 0; 1253 unsigned int rclass = 0; 1254 unsigned int cmd = MFC_GETFS_CMD; 1255 1256 /* Restore, Step 37: 1257 * Issue MFC DMA command to copy context 1258 * restore code to local storage. 1259 */ 1260 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 1261 } 1262 1263 static inline void setup_decr(struct spu_state *csa, struct spu *spu) 1264 { 1265 /* Restore, Step 34: 1266 * If CSA.MFC_CNTL[Ds]=1 (decrementer was 1267 * running) then adjust decrementer, set 1268 * decrementer running status in LSCSA, 1269 * and set decrementer "wrapped" status 1270 * in LSCSA. 1271 */ 1272 if (csa->priv2.mfc_control_RW & MFC_CNTL_DECREMENTER_RUNNING) { 1273 cycles_t resume_time = get_cycles(); 1274 cycles_t delta_time = resume_time - csa->suspend_time; 1275 1276 csa->lscsa->decr_status.slot[0] = SPU_DECR_STATUS_RUNNING; 1277 if (csa->lscsa->decr.slot[0] < delta_time) { 1278 csa->lscsa->decr_status.slot[0] |= 1279 SPU_DECR_STATUS_WRAPPED; 1280 } 1281 1282 csa->lscsa->decr.slot[0] -= delta_time; 1283 } else { 1284 csa->lscsa->decr_status.slot[0] = 0; 1285 } 1286 } 1287 1288 static inline void setup_ppu_mb(struct spu_state *csa, struct spu *spu) 1289 { 1290 /* Restore, Step 35: 1291 * Copy the CSA.PU_MB data into the LSCSA. 1292 */ 1293 csa->lscsa->ppu_mb.slot[0] = csa->prob.pu_mb_R; 1294 } 1295 1296 static inline void setup_ppuint_mb(struct spu_state *csa, struct spu *spu) 1297 { 1298 /* Restore, Step 36: 1299 * Copy the CSA.PUINT_MB data into the LSCSA. 1300 */ 1301 csa->lscsa->ppuint_mb.slot[0] = csa->priv2.puint_mb_R; 1302 } 1303 1304 static inline int check_restore_status(struct spu_state *csa, struct spu *spu) 1305 { 1306 struct spu_problem __iomem *prob = spu->problem; 1307 u32 complete; 1308 1309 /* Restore, Step 40: 1310 * If SPU_Status[P]=1 and SPU_Status[SC] = "success", 1311 * context restore succeeded, otherwise context restore 1312 * failed. 1313 */ 1314 complete = ((SPU_RESTORE_COMPLETE << SPU_STOP_STATUS_SHIFT) | 1315 SPU_STATUS_STOPPED_BY_STOP); 1316 return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0; 1317 } 1318 1319 static inline void restore_spu_privcntl(struct spu_state *csa, struct spu *spu) 1320 { 1321 struct spu_priv2 __iomem *priv2 = spu->priv2; 1322 1323 /* Restore, Step 41: 1324 * Restore SPU_PrivCntl from the CSA. 1325 */ 1326 out_be64(&priv2->spu_privcntl_RW, csa->priv2.spu_privcntl_RW); 1327 eieio(); 1328 } 1329 1330 static inline void restore_status_part1(struct spu_state *csa, struct spu *spu) 1331 { 1332 struct spu_problem __iomem *prob = spu->problem; 1333 u32 mask; 1334 1335 /* Restore, Step 42: 1336 * If any CSA.SPU_Status[I,S,H,P]=1, then 1337 * restore the error or single step state. 1338 */ 1339 mask = SPU_STATUS_INVALID_INSTR | 1340 SPU_STATUS_SINGLE_STEP | 1341 SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP; 1342 if (csa->prob.spu_status_R & mask) { 1343 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1344 eieio(); 1345 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1346 SPU_STATUS_RUNNING); 1347 } 1348 } 1349 1350 static inline void restore_status_part2(struct spu_state *csa, struct spu *spu) 1351 { 1352 struct spu_problem __iomem *prob = spu->problem; 1353 u32 mask; 1354 1355 /* Restore, Step 43: 1356 * If all CSA.SPU_Status[I,S,H,P,R]=0 then write 1357 * SPU_RunCntl[R0R1]='01', wait for SPU_Status[R]=1, 1358 * then write '00' to SPU_RunCntl[R0R1] and wait 1359 * for SPU_Status[R]=0. 1360 */ 1361 mask = SPU_STATUS_INVALID_INSTR | 1362 SPU_STATUS_SINGLE_STEP | 1363 SPU_STATUS_STOPPED_BY_HALT | 1364 SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING; 1365 if (!(csa->prob.spu_status_R & mask)) { 1366 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1367 eieio(); 1368 POLL_WHILE_FALSE(in_be32(&prob->spu_status_R) & 1369 SPU_STATUS_RUNNING); 1370 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1371 eieio(); 1372 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & 1373 SPU_STATUS_RUNNING); 1374 } 1375 } 1376 1377 static inline void restore_ls_16kb(struct spu_state *csa, struct spu *spu) 1378 { 1379 unsigned long addr = (unsigned long)&csa->lscsa->ls[0]; 1380 unsigned int ls_offset = 0x0; 1381 unsigned int size = 16384; 1382 unsigned int tag = 0; 1383 unsigned int rclass = 0; 1384 unsigned int cmd = MFC_GET_CMD; 1385 1386 /* Restore, Step 44: 1387 * Issue a DMA command to restore the first 1388 * 16kb of local storage from CSA. 1389 */ 1390 send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd); 1391 } 1392 1393 static inline void suspend_mfc(struct spu_state *csa, struct spu *spu) 1394 { 1395 struct spu_priv2 __iomem *priv2 = spu->priv2; 1396 1397 /* Restore, Step 47. 1398 * Write MFC_Cntl[Sc,Sm]='1','0' to suspend 1399 * the queue. 1400 */ 1401 out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE); 1402 eieio(); 1403 } 1404 1405 static inline void clear_interrupts(struct spu_state *csa, struct spu *spu) 1406 { 1407 /* Restore, Step 49: 1408 * Write INT_MASK_class0 with value of 0. 1409 * Write INT_MASK_class1 with value of 0. 1410 * Write INT_MASK_class2 with value of 0. 1411 * Write INT_STAT_class0 with value of -1. 1412 * Write INT_STAT_class1 with value of -1. 1413 * Write INT_STAT_class2 with value of -1. 1414 */ 1415 spin_lock_irq(&spu->register_lock); 1416 spu_int_mask_set(spu, 0, 0ul); 1417 spu_int_mask_set(spu, 1, 0ul); 1418 spu_int_mask_set(spu, 2, 0ul); 1419 spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK); 1420 spu_int_stat_clear(spu, 1, CLASS1_INTR_MASK); 1421 spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK); 1422 spin_unlock_irq(&spu->register_lock); 1423 } 1424 1425 static inline void restore_mfc_queues(struct spu_state *csa, struct spu *spu) 1426 { 1427 struct spu_priv2 __iomem *priv2 = spu->priv2; 1428 int i; 1429 1430 /* Restore, Step 50: 1431 * If MFC_Cntl[Se]!=0 then restore 1432 * MFC command queues. 1433 */ 1434 if ((csa->priv2.mfc_control_RW & MFC_CNTL_DMA_QUEUES_EMPTY_MASK) == 0) { 1435 for (i = 0; i < 8; i++) { 1436 out_be64(&priv2->puq[i].mfc_cq_data0_RW, 1437 csa->priv2.puq[i].mfc_cq_data0_RW); 1438 out_be64(&priv2->puq[i].mfc_cq_data1_RW, 1439 csa->priv2.puq[i].mfc_cq_data1_RW); 1440 out_be64(&priv2->puq[i].mfc_cq_data2_RW, 1441 csa->priv2.puq[i].mfc_cq_data2_RW); 1442 out_be64(&priv2->puq[i].mfc_cq_data3_RW, 1443 csa->priv2.puq[i].mfc_cq_data3_RW); 1444 } 1445 for (i = 0; i < 16; i++) { 1446 out_be64(&priv2->spuq[i].mfc_cq_data0_RW, 1447 csa->priv2.spuq[i].mfc_cq_data0_RW); 1448 out_be64(&priv2->spuq[i].mfc_cq_data1_RW, 1449 csa->priv2.spuq[i].mfc_cq_data1_RW); 1450 out_be64(&priv2->spuq[i].mfc_cq_data2_RW, 1451 csa->priv2.spuq[i].mfc_cq_data2_RW); 1452 out_be64(&priv2->spuq[i].mfc_cq_data3_RW, 1453 csa->priv2.spuq[i].mfc_cq_data3_RW); 1454 } 1455 } 1456 eieio(); 1457 } 1458 1459 static inline void restore_ppu_querymask(struct spu_state *csa, struct spu *spu) 1460 { 1461 struct spu_problem __iomem *prob = spu->problem; 1462 1463 /* Restore, Step 51: 1464 * Restore the PPU_QueryMask register from CSA. 1465 */ 1466 out_be32(&prob->dma_querymask_RW, csa->prob.dma_querymask_RW); 1467 eieio(); 1468 } 1469 1470 static inline void restore_ppu_querytype(struct spu_state *csa, struct spu *spu) 1471 { 1472 struct spu_problem __iomem *prob = spu->problem; 1473 1474 /* Restore, Step 52: 1475 * Restore the PPU_QueryType register from CSA. 1476 */ 1477 out_be32(&prob->dma_querytype_RW, csa->prob.dma_querytype_RW); 1478 eieio(); 1479 } 1480 1481 static inline void restore_mfc_csr_tsq(struct spu_state *csa, struct spu *spu) 1482 { 1483 struct spu_priv2 __iomem *priv2 = spu->priv2; 1484 1485 /* Restore, Step 53: 1486 * Restore the MFC_CSR_TSQ register from CSA. 1487 */ 1488 out_be64(&priv2->spu_tag_status_query_RW, 1489 csa->priv2.spu_tag_status_query_RW); 1490 eieio(); 1491 } 1492 1493 static inline void restore_mfc_csr_cmd(struct spu_state *csa, struct spu *spu) 1494 { 1495 struct spu_priv2 __iomem *priv2 = spu->priv2; 1496 1497 /* Restore, Step 54: 1498 * Restore the MFC_CSR_CMD1 and MFC_CSR_CMD2 1499 * registers from CSA. 1500 */ 1501 out_be64(&priv2->spu_cmd_buf1_RW, csa->priv2.spu_cmd_buf1_RW); 1502 out_be64(&priv2->spu_cmd_buf2_RW, csa->priv2.spu_cmd_buf2_RW); 1503 eieio(); 1504 } 1505 1506 static inline void restore_mfc_csr_ato(struct spu_state *csa, struct spu *spu) 1507 { 1508 struct spu_priv2 __iomem *priv2 = spu->priv2; 1509 1510 /* Restore, Step 55: 1511 * Restore the MFC_CSR_ATO register from CSA. 1512 */ 1513 out_be64(&priv2->spu_atomic_status_RW, csa->priv2.spu_atomic_status_RW); 1514 } 1515 1516 static inline void restore_mfc_tclass_id(struct spu_state *csa, struct spu *spu) 1517 { 1518 /* Restore, Step 56: 1519 * Restore the MFC_TCLASS_ID register from CSA. 1520 */ 1521 spu_mfc_tclass_id_set(spu, csa->priv1.mfc_tclass_id_RW); 1522 eieio(); 1523 } 1524 1525 static inline void set_llr_event(struct spu_state *csa, struct spu *spu) 1526 { 1527 u64 ch0_cnt, ch0_data; 1528 u64 ch1_data; 1529 1530 /* Restore, Step 57: 1531 * Set the Lock Line Reservation Lost Event by: 1532 * 1. OR CSA.SPU_Event_Status with bit 21 (Lr) set to 1. 1533 * 2. If CSA.SPU_Channel_0_Count=0 and 1534 * CSA.SPU_Wr_Event_Mask[Lr]=1 and 1535 * CSA.SPU_Event_Status[Lr]=0 then set 1536 * CSA.SPU_Event_Status_Count=1. 1537 */ 1538 ch0_cnt = csa->spu_chnlcnt_RW[0]; 1539 ch0_data = csa->spu_chnldata_RW[0]; 1540 ch1_data = csa->spu_chnldata_RW[1]; 1541 csa->spu_chnldata_RW[0] |= MFC_LLR_LOST_EVENT; 1542 if ((ch0_cnt == 0) && !(ch0_data & MFC_LLR_LOST_EVENT) && 1543 (ch1_data & MFC_LLR_LOST_EVENT)) { 1544 csa->spu_chnlcnt_RW[0] = 1; 1545 } 1546 } 1547 1548 static inline void restore_decr_wrapped(struct spu_state *csa, struct spu *spu) 1549 { 1550 /* Restore, Step 58: 1551 * If the status of the CSA software decrementer 1552 * "wrapped" flag is set, OR in a '1' to 1553 * CSA.SPU_Event_Status[Tm]. 1554 */ 1555 if (!(csa->lscsa->decr_status.slot[0] & SPU_DECR_STATUS_WRAPPED)) 1556 return; 1557 1558 if ((csa->spu_chnlcnt_RW[0] == 0) && 1559 (csa->spu_chnldata_RW[1] & 0x20) && 1560 !(csa->spu_chnldata_RW[0] & 0x20)) 1561 csa->spu_chnlcnt_RW[0] = 1; 1562 1563 csa->spu_chnldata_RW[0] |= 0x20; 1564 } 1565 1566 static inline void restore_ch_part1(struct spu_state *csa, struct spu *spu) 1567 { 1568 struct spu_priv2 __iomem *priv2 = spu->priv2; 1569 u64 idx, ch_indices[] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL }; 1570 int i; 1571 1572 /* Restore, Step 59: 1573 * Restore the following CH: [0,3,4,24,25,27] 1574 */ 1575 for (i = 0; i < ARRAY_SIZE(ch_indices); i++) { 1576 idx = ch_indices[i]; 1577 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1578 eieio(); 1579 out_be64(&priv2->spu_chnldata_RW, csa->spu_chnldata_RW[idx]); 1580 out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[idx]); 1581 eieio(); 1582 } 1583 } 1584 1585 static inline void restore_ch_part2(struct spu_state *csa, struct spu *spu) 1586 { 1587 struct spu_priv2 __iomem *priv2 = spu->priv2; 1588 u64 ch_indices[3] = { 9UL, 21UL, 23UL }; 1589 u64 ch_counts[3] = { 1UL, 16UL, 1UL }; 1590 u64 idx; 1591 int i; 1592 1593 /* Restore, Step 60: 1594 * Restore the following CH: [9,21,23]. 1595 */ 1596 ch_counts[0] = 1UL; 1597 ch_counts[1] = csa->spu_chnlcnt_RW[21]; 1598 ch_counts[2] = 1UL; 1599 for (i = 0; i < 3; i++) { 1600 idx = ch_indices[i]; 1601 out_be64(&priv2->spu_chnlcntptr_RW, idx); 1602 eieio(); 1603 out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]); 1604 eieio(); 1605 } 1606 } 1607 1608 static inline void restore_spu_lslr(struct spu_state *csa, struct spu *spu) 1609 { 1610 struct spu_priv2 __iomem *priv2 = spu->priv2; 1611 1612 /* Restore, Step 61: 1613 * Restore the SPU_LSLR register from CSA. 1614 */ 1615 out_be64(&priv2->spu_lslr_RW, csa->priv2.spu_lslr_RW); 1616 eieio(); 1617 } 1618 1619 static inline void restore_spu_cfg(struct spu_state *csa, struct spu *spu) 1620 { 1621 struct spu_priv2 __iomem *priv2 = spu->priv2; 1622 1623 /* Restore, Step 62: 1624 * Restore the SPU_Cfg register from CSA. 1625 */ 1626 out_be64(&priv2->spu_cfg_RW, csa->priv2.spu_cfg_RW); 1627 eieio(); 1628 } 1629 1630 static inline void restore_pm_trace(struct spu_state *csa, struct spu *spu) 1631 { 1632 /* Restore, Step 63: 1633 * Restore PM_Trace_Tag_Wait_Mask from CSA. 1634 * Not performed by this implementation. 1635 */ 1636 } 1637 1638 static inline void restore_spu_npc(struct spu_state *csa, struct spu *spu) 1639 { 1640 struct spu_problem __iomem *prob = spu->problem; 1641 1642 /* Restore, Step 64: 1643 * Restore SPU_NPC from CSA. 1644 */ 1645 out_be32(&prob->spu_npc_RW, csa->prob.spu_npc_RW); 1646 eieio(); 1647 } 1648 1649 static inline void restore_spu_mb(struct spu_state *csa, struct spu *spu) 1650 { 1651 struct spu_priv2 __iomem *priv2 = spu->priv2; 1652 int i; 1653 1654 /* Restore, Step 65: 1655 * Restore MFC_RdSPU_MB from CSA. 1656 */ 1657 out_be64(&priv2->spu_chnlcntptr_RW, 29UL); 1658 eieio(); 1659 out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[29]); 1660 for (i = 0; i < 4; i++) { 1661 out_be64(&priv2->spu_chnldata_RW, csa->spu_mailbox_data[i]); 1662 } 1663 eieio(); 1664 } 1665 1666 static inline void check_ppu_mb_stat(struct spu_state *csa, struct spu *spu) 1667 { 1668 struct spu_problem __iomem *prob = spu->problem; 1669 u32 dummy = 0; 1670 1671 /* Restore, Step 66: 1672 * If CSA.MB_Stat[P]=0 (mailbox empty) then 1673 * read from the PPU_MB register. 1674 */ 1675 if ((csa->prob.mb_stat_R & 0xFF) == 0) { 1676 dummy = in_be32(&prob->pu_mb_R); 1677 eieio(); 1678 } 1679 } 1680 1681 static inline void check_ppuint_mb_stat(struct spu_state *csa, struct spu *spu) 1682 { 1683 struct spu_priv2 __iomem *priv2 = spu->priv2; 1684 u64 dummy = 0UL; 1685 1686 /* Restore, Step 66: 1687 * If CSA.MB_Stat[I]=0 (mailbox empty) then 1688 * read from the PPUINT_MB register. 1689 */ 1690 if ((csa->prob.mb_stat_R & 0xFF0000) == 0) { 1691 dummy = in_be64(&priv2->puint_mb_R); 1692 eieio(); 1693 spu_int_stat_clear(spu, 2, CLASS2_ENABLE_MAILBOX_INTR); 1694 eieio(); 1695 } 1696 } 1697 1698 static inline void restore_mfc_sr1(struct spu_state *csa, struct spu *spu) 1699 { 1700 /* Restore, Step 69: 1701 * Restore the MFC_SR1 register from CSA. 1702 */ 1703 spu_mfc_sr1_set(spu, csa->priv1.mfc_sr1_RW); 1704 eieio(); 1705 } 1706 1707 static inline void set_int_route(struct spu_state *csa, struct spu *spu) 1708 { 1709 struct spu_context *ctx = spu->ctx; 1710 1711 spu_cpu_affinity_set(spu, ctx->last_ran); 1712 } 1713 1714 static inline void restore_other_spu_access(struct spu_state *csa, 1715 struct spu *spu) 1716 { 1717 /* Restore, Step 70: 1718 * Restore other SPU mappings to this SPU. TBD. 1719 */ 1720 } 1721 1722 static inline void restore_spu_runcntl(struct spu_state *csa, struct spu *spu) 1723 { 1724 struct spu_problem __iomem *prob = spu->problem; 1725 1726 /* Restore, Step 71: 1727 * If CSA.SPU_Status[R]=1 then write 1728 * SPU_RunCntl[R0R1]='01'. 1729 */ 1730 if (csa->prob.spu_status_R & SPU_STATUS_RUNNING) { 1731 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE); 1732 eieio(); 1733 } 1734 } 1735 1736 static inline void restore_mfc_cntl(struct spu_state *csa, struct spu *spu) 1737 { 1738 struct spu_priv2 __iomem *priv2 = spu->priv2; 1739 1740 /* Restore, Step 72: 1741 * Restore the MFC_CNTL register for the CSA. 1742 */ 1743 out_be64(&priv2->mfc_control_RW, csa->priv2.mfc_control_RW); 1744 eieio(); 1745 /* 1746 * FIXME: this is to restart a DMA that we were processing 1747 * before the save. better remember the fault information 1748 * in the csa instead. 1749 */ 1750 if ((csa->priv2.mfc_control_RW & MFC_CNTL_SUSPEND_DMA_QUEUE_MASK)) { 1751 out_be64(&priv2->mfc_control_RW, MFC_CNTL_RESTART_DMA_COMMAND); 1752 eieio(); 1753 } 1754 } 1755 1756 static inline void enable_user_access(struct spu_state *csa, struct spu *spu) 1757 { 1758 /* Restore, Step 73: 1759 * Enable user-space access (if provided) to this 1760 * SPU by mapping the virtual pages assigned to 1761 * the SPU memory-mapped I/O (MMIO) for problem 1762 * state. TBD. 1763 */ 1764 } 1765 1766 static inline void reset_switch_active(struct spu_state *csa, struct spu *spu) 1767 { 1768 /* Restore, Step 74: 1769 * Reset the "context switch active" flag. 1770 * Not performed by this implementation. 1771 */ 1772 } 1773 1774 static inline void reenable_interrupts(struct spu_state *csa, struct spu *spu) 1775 { 1776 /* Restore, Step 75: 1777 * Re-enable SPU interrupts. 1778 */ 1779 spin_lock_irq(&spu->register_lock); 1780 spu_int_mask_set(spu, 0, csa->priv1.int_mask_class0_RW); 1781 spu_int_mask_set(spu, 1, csa->priv1.int_mask_class1_RW); 1782 spu_int_mask_set(spu, 2, csa->priv1.int_mask_class2_RW); 1783 spin_unlock_irq(&spu->register_lock); 1784 } 1785 1786 static int quiece_spu(struct spu_state *prev, struct spu *spu) 1787 { 1788 /* 1789 * Combined steps 2-18 of SPU context save sequence, which 1790 * quiesce the SPU state (disable SPU execution, MFC command 1791 * queues, decrementer, SPU interrupts, etc.). 1792 * 1793 * Returns 0 on success. 1794 * 2 if failed step 2. 1795 * 6 if failed step 6. 1796 */ 1797 1798 if (check_spu_isolate(prev, spu)) { /* Step 2. */ 1799 return 2; 1800 } 1801 disable_interrupts(prev, spu); /* Step 3. */ 1802 set_watchdog_timer(prev, spu); /* Step 4. */ 1803 inhibit_user_access(prev, spu); /* Step 5. */ 1804 if (check_spu_isolate(prev, spu)) { /* Step 6. */ 1805 return 6; 1806 } 1807 set_switch_pending(prev, spu); /* Step 7. */ 1808 save_mfc_cntl(prev, spu); /* Step 8. */ 1809 save_spu_runcntl(prev, spu); /* Step 9. */ 1810 save_mfc_sr1(prev, spu); /* Step 10. */ 1811 save_spu_status(prev, spu); /* Step 11. */ 1812 save_mfc_stopped_status(prev, spu); /* Step 12. */ 1813 halt_mfc_decr(prev, spu); /* Step 13. */ 1814 save_timebase(prev, spu); /* Step 14. */ 1815 remove_other_spu_access(prev, spu); /* Step 15. */ 1816 do_mfc_mssync(prev, spu); /* Step 16. */ 1817 issue_mfc_tlbie(prev, spu); /* Step 17. */ 1818 handle_pending_interrupts(prev, spu); /* Step 18. */ 1819 1820 return 0; 1821 } 1822 1823 static void save_csa(struct spu_state *prev, struct spu *spu) 1824 { 1825 /* 1826 * Combine steps 19-44 of SPU context save sequence, which 1827 * save regions of the privileged & problem state areas. 1828 */ 1829 1830 save_mfc_queues(prev, spu); /* Step 19. */ 1831 save_ppu_querymask(prev, spu); /* Step 20. */ 1832 save_ppu_querytype(prev, spu); /* Step 21. */ 1833 save_ppu_tagstatus(prev, spu); /* NEW. */ 1834 save_mfc_csr_tsq(prev, spu); /* Step 22. */ 1835 save_mfc_csr_cmd(prev, spu); /* Step 23. */ 1836 save_mfc_csr_ato(prev, spu); /* Step 24. */ 1837 save_mfc_tclass_id(prev, spu); /* Step 25. */ 1838 set_mfc_tclass_id(prev, spu); /* Step 26. */ 1839 save_mfc_cmd(prev, spu); /* Step 26a - moved from 44. */ 1840 purge_mfc_queue(prev, spu); /* Step 27. */ 1841 wait_purge_complete(prev, spu); /* Step 28. */ 1842 setup_mfc_sr1(prev, spu); /* Step 30. */ 1843 save_spu_npc(prev, spu); /* Step 31. */ 1844 save_spu_privcntl(prev, spu); /* Step 32. */ 1845 reset_spu_privcntl(prev, spu); /* Step 33. */ 1846 save_spu_lslr(prev, spu); /* Step 34. */ 1847 reset_spu_lslr(prev, spu); /* Step 35. */ 1848 save_spu_cfg(prev, spu); /* Step 36. */ 1849 save_pm_trace(prev, spu); /* Step 37. */ 1850 save_mfc_rag(prev, spu); /* Step 38. */ 1851 save_ppu_mb_stat(prev, spu); /* Step 39. */ 1852 save_ppu_mb(prev, spu); /* Step 40. */ 1853 save_ppuint_mb(prev, spu); /* Step 41. */ 1854 save_ch_part1(prev, spu); /* Step 42. */ 1855 save_spu_mb(prev, spu); /* Step 43. */ 1856 reset_ch(prev, spu); /* Step 45. */ 1857 } 1858 1859 static void save_lscsa(struct spu_state *prev, struct spu *spu) 1860 { 1861 /* 1862 * Perform steps 46-57 of SPU context save sequence, 1863 * which save regions of the local store and register 1864 * file. 1865 */ 1866 1867 resume_mfc_queue(prev, spu); /* Step 46. */ 1868 /* Step 47. */ 1869 setup_mfc_slbs(prev, spu, spu_save_code, sizeof(spu_save_code)); 1870 set_switch_active(prev, spu); /* Step 48. */ 1871 enable_interrupts(prev, spu); /* Step 49. */ 1872 save_ls_16kb(prev, spu); /* Step 50. */ 1873 set_spu_npc(prev, spu); /* Step 51. */ 1874 set_signot1(prev, spu); /* Step 52. */ 1875 set_signot2(prev, spu); /* Step 53. */ 1876 send_save_code(prev, spu); /* Step 54. */ 1877 set_ppu_querymask(prev, spu); /* Step 55. */ 1878 wait_tag_complete(prev, spu); /* Step 56. */ 1879 wait_spu_stopped(prev, spu); /* Step 57. */ 1880 } 1881 1882 static void force_spu_isolate_exit(struct spu *spu) 1883 { 1884 struct spu_problem __iomem *prob = spu->problem; 1885 struct spu_priv2 __iomem *priv2 = spu->priv2; 1886 1887 /* Stop SPE execution and wait for completion. */ 1888 out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP); 1889 iobarrier_rw(); 1890 POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING); 1891 1892 /* Restart SPE master runcntl. */ 1893 spu_mfc_sr1_set(spu, MFC_STATE1_MASTER_RUN_CONTROL_MASK); 1894 iobarrier_w(); 1895 1896 /* Initiate isolate exit request and wait for completion. */ 1897 out_be64(&priv2->spu_privcntl_RW, 4LL); 1898 iobarrier_w(); 1899 out_be32(&prob->spu_runcntl_RW, 2); 1900 iobarrier_rw(); 1901 POLL_WHILE_FALSE((in_be32(&prob->spu_status_R) 1902 & SPU_STATUS_STOPPED_BY_STOP)); 1903 1904 /* Reset load request to normal. */ 1905 out_be64(&priv2->spu_privcntl_RW, SPU_PRIVCNT_LOAD_REQUEST_NORMAL); 1906 iobarrier_w(); 1907 } 1908 1909 /** 1910 * stop_spu_isolate 1911 * Check SPU run-control state and force isolated 1912 * exit function as necessary. 1913 */ 1914 static void stop_spu_isolate(struct spu *spu) 1915 { 1916 struct spu_problem __iomem *prob = spu->problem; 1917 1918 if (in_be32(&prob->spu_status_R) & SPU_STATUS_ISOLATED_STATE) { 1919 /* The SPU is in isolated state; the only way 1920 * to get it out is to perform an isolated 1921 * exit (clean) operation. 1922 */ 1923 force_spu_isolate_exit(spu); 1924 } 1925 } 1926 1927 static void harvest(struct spu_state *prev, struct spu *spu) 1928 { 1929 /* 1930 * Perform steps 2-25 of SPU context restore sequence, 1931 * which resets an SPU either after a failed save, or 1932 * when using SPU for first time. 1933 */ 1934 1935 disable_interrupts(prev, spu); /* Step 2. */ 1936 inhibit_user_access(prev, spu); /* Step 3. */ 1937 terminate_spu_app(prev, spu); /* Step 4. */ 1938 set_switch_pending(prev, spu); /* Step 5. */ 1939 stop_spu_isolate(spu); /* NEW. */ 1940 remove_other_spu_access(prev, spu); /* Step 6. */ 1941 suspend_mfc_and_halt_decr(prev, spu); /* Step 7. */ 1942 wait_suspend_mfc_complete(prev, spu); /* Step 8. */ 1943 if (!suspend_spe(prev, spu)) /* Step 9. */ 1944 clear_spu_status(prev, spu); /* Step 10. */ 1945 do_mfc_mssync(prev, spu); /* Step 11. */ 1946 issue_mfc_tlbie(prev, spu); /* Step 12. */ 1947 handle_pending_interrupts(prev, spu); /* Step 13. */ 1948 purge_mfc_queue(prev, spu); /* Step 14. */ 1949 wait_purge_complete(prev, spu); /* Step 15. */ 1950 reset_spu_privcntl(prev, spu); /* Step 16. */ 1951 reset_spu_lslr(prev, spu); /* Step 17. */ 1952 setup_mfc_sr1(prev, spu); /* Step 18. */ 1953 spu_invalidate_slbs(spu); /* Step 19. */ 1954 reset_ch_part1(prev, spu); /* Step 20. */ 1955 reset_ch_part2(prev, spu); /* Step 21. */ 1956 enable_interrupts(prev, spu); /* Step 22. */ 1957 set_switch_active(prev, spu); /* Step 23. */ 1958 set_mfc_tclass_id(prev, spu); /* Step 24. */ 1959 resume_mfc_queue(prev, spu); /* Step 25. */ 1960 } 1961 1962 static void restore_lscsa(struct spu_state *next, struct spu *spu) 1963 { 1964 /* 1965 * Perform steps 26-40 of SPU context restore sequence, 1966 * which restores regions of the local store and register 1967 * file. 1968 */ 1969 1970 set_watchdog_timer(next, spu); /* Step 26. */ 1971 setup_spu_status_part1(next, spu); /* Step 27. */ 1972 setup_spu_status_part2(next, spu); /* Step 28. */ 1973 restore_mfc_rag(next, spu); /* Step 29. */ 1974 /* Step 30. */ 1975 setup_mfc_slbs(next, spu, spu_restore_code, sizeof(spu_restore_code)); 1976 set_spu_npc(next, spu); /* Step 31. */ 1977 set_signot1(next, spu); /* Step 32. */ 1978 set_signot2(next, spu); /* Step 33. */ 1979 setup_decr(next, spu); /* Step 34. */ 1980 setup_ppu_mb(next, spu); /* Step 35. */ 1981 setup_ppuint_mb(next, spu); /* Step 36. */ 1982 send_restore_code(next, spu); /* Step 37. */ 1983 set_ppu_querymask(next, spu); /* Step 38. */ 1984 wait_tag_complete(next, spu); /* Step 39. */ 1985 wait_spu_stopped(next, spu); /* Step 40. */ 1986 } 1987 1988 static void restore_csa(struct spu_state *next, struct spu *spu) 1989 { 1990 /* 1991 * Combine steps 41-76 of SPU context restore sequence, which 1992 * restore regions of the privileged & problem state areas. 1993 */ 1994 1995 restore_spu_privcntl(next, spu); /* Step 41. */ 1996 restore_status_part1(next, spu); /* Step 42. */ 1997 restore_status_part2(next, spu); /* Step 43. */ 1998 restore_ls_16kb(next, spu); /* Step 44. */ 1999 wait_tag_complete(next, spu); /* Step 45. */ 2000 suspend_mfc(next, spu); /* Step 46. */ 2001 wait_suspend_mfc_complete(next, spu); /* Step 47. */ 2002 issue_mfc_tlbie(next, spu); /* Step 48. */ 2003 clear_interrupts(next, spu); /* Step 49. */ 2004 restore_mfc_queues(next, spu); /* Step 50. */ 2005 restore_ppu_querymask(next, spu); /* Step 51. */ 2006 restore_ppu_querytype(next, spu); /* Step 52. */ 2007 restore_mfc_csr_tsq(next, spu); /* Step 53. */ 2008 restore_mfc_csr_cmd(next, spu); /* Step 54. */ 2009 restore_mfc_csr_ato(next, spu); /* Step 55. */ 2010 restore_mfc_tclass_id(next, spu); /* Step 56. */ 2011 set_llr_event(next, spu); /* Step 57. */ 2012 restore_decr_wrapped(next, spu); /* Step 58. */ 2013 restore_ch_part1(next, spu); /* Step 59. */ 2014 restore_ch_part2(next, spu); /* Step 60. */ 2015 restore_spu_lslr(next, spu); /* Step 61. */ 2016 restore_spu_cfg(next, spu); /* Step 62. */ 2017 restore_pm_trace(next, spu); /* Step 63. */ 2018 restore_spu_npc(next, spu); /* Step 64. */ 2019 restore_spu_mb(next, spu); /* Step 65. */ 2020 check_ppu_mb_stat(next, spu); /* Step 66. */ 2021 check_ppuint_mb_stat(next, spu); /* Step 67. */ 2022 spu_invalidate_slbs(spu); /* Modified Step 68. */ 2023 restore_mfc_sr1(next, spu); /* Step 69. */ 2024 set_int_route(next, spu); /* NEW */ 2025 restore_other_spu_access(next, spu); /* Step 70. */ 2026 restore_spu_runcntl(next, spu); /* Step 71. */ 2027 restore_mfc_cntl(next, spu); /* Step 72. */ 2028 enable_user_access(next, spu); /* Step 73. */ 2029 reset_switch_active(next, spu); /* Step 74. */ 2030 reenable_interrupts(next, spu); /* Step 75. */ 2031 } 2032 2033 static int __do_spu_save(struct spu_state *prev, struct spu *spu) 2034 { 2035 int rc; 2036 2037 /* 2038 * SPU context save can be broken into three phases: 2039 * 2040 * (a) quiesce [steps 2-16]. 2041 * (b) save of CSA, performed by PPE [steps 17-42] 2042 * (c) save of LSCSA, mostly performed by SPU [steps 43-52]. 2043 * 2044 * Returns 0 on success. 2045 * 2,6 if failed to quiece SPU 2046 * 53 if SPU-side of save failed. 2047 */ 2048 2049 rc = quiece_spu(prev, spu); /* Steps 2-16. */ 2050 switch (rc) { 2051 default: 2052 case 2: 2053 case 6: 2054 harvest(prev, spu); 2055 return rc; 2056 break; 2057 case 0: 2058 break; 2059 } 2060 save_csa(prev, spu); /* Steps 17-43. */ 2061 save_lscsa(prev, spu); /* Steps 44-53. */ 2062 return check_save_status(prev, spu); /* Step 54. */ 2063 } 2064 2065 static int __do_spu_restore(struct spu_state *next, struct spu *spu) 2066 { 2067 int rc; 2068 2069 /* 2070 * SPU context restore can be broken into three phases: 2071 * 2072 * (a) harvest (or reset) SPU [steps 2-24]. 2073 * (b) restore LSCSA [steps 25-40], mostly performed by SPU. 2074 * (c) restore CSA [steps 41-76], performed by PPE. 2075 * 2076 * The 'harvest' step is not performed here, but rather 2077 * as needed below. 2078 */ 2079 2080 restore_lscsa(next, spu); /* Steps 24-39. */ 2081 rc = check_restore_status(next, spu); /* Step 40. */ 2082 switch (rc) { 2083 default: 2084 /* Failed. Return now. */ 2085 return rc; 2086 break; 2087 case 0: 2088 /* Fall through to next step. */ 2089 break; 2090 } 2091 restore_csa(next, spu); 2092 2093 return 0; 2094 } 2095 2096 /** 2097 * spu_save - SPU context save, with locking. 2098 * @prev: pointer to SPU context save area, to be saved. 2099 * @spu: pointer to SPU iomem structure. 2100 * 2101 * Acquire locks, perform the save operation then return. 2102 */ 2103 int spu_save(struct spu_state *prev, struct spu *spu) 2104 { 2105 int rc; 2106 2107 acquire_spu_lock(spu); /* Step 1. */ 2108 rc = __do_spu_save(prev, spu); /* Steps 2-53. */ 2109 release_spu_lock(spu); 2110 if (rc != 0 && rc != 2 && rc != 6) { 2111 panic("%s failed on SPU[%d], rc=%d.\n", 2112 __func__, spu->number, rc); 2113 } 2114 return 0; 2115 } 2116 EXPORT_SYMBOL_GPL(spu_save); 2117 2118 /** 2119 * spu_restore - SPU context restore, with harvest and locking. 2120 * @new: pointer to SPU context save area, to be restored. 2121 * @spu: pointer to SPU iomem structure. 2122 * 2123 * Perform harvest + restore, as we may not be coming 2124 * from a previous successful save operation, and the 2125 * hardware state is unknown. 2126 */ 2127 int spu_restore(struct spu_state *new, struct spu *spu) 2128 { 2129 int rc; 2130 2131 acquire_spu_lock(spu); 2132 harvest(NULL, spu); 2133 spu->slb_replace = 0; 2134 rc = __do_spu_restore(new, spu); 2135 release_spu_lock(spu); 2136 if (rc) { 2137 panic("%s failed on SPU[%d] rc=%d.\n", 2138 __func__, spu->number, rc); 2139 } 2140 return rc; 2141 } 2142 EXPORT_SYMBOL_GPL(spu_restore); 2143 2144 static void init_prob(struct spu_state *csa) 2145 { 2146 csa->spu_chnlcnt_RW[9] = 1; 2147 csa->spu_chnlcnt_RW[21] = 16; 2148 csa->spu_chnlcnt_RW[23] = 1; 2149 csa->spu_chnlcnt_RW[28] = 1; 2150 csa->spu_chnlcnt_RW[30] = 1; 2151 csa->prob.spu_runcntl_RW = SPU_RUNCNTL_STOP; 2152 csa->prob.mb_stat_R = 0x000400; 2153 } 2154 2155 static void init_priv1(struct spu_state *csa) 2156 { 2157 /* Enable decode, relocate, tlbie response, master runcntl. */ 2158 csa->priv1.mfc_sr1_RW = MFC_STATE1_LOCAL_STORAGE_DECODE_MASK | 2159 MFC_STATE1_MASTER_RUN_CONTROL_MASK | 2160 MFC_STATE1_PROBLEM_STATE_MASK | 2161 MFC_STATE1_RELOCATE_MASK | MFC_STATE1_BUS_TLBIE_MASK; 2162 2163 /* Enable OS-specific set of interrupts. */ 2164 csa->priv1.int_mask_class0_RW = CLASS0_ENABLE_DMA_ALIGNMENT_INTR | 2165 CLASS0_ENABLE_INVALID_DMA_COMMAND_INTR | 2166 CLASS0_ENABLE_SPU_ERROR_INTR; 2167 csa->priv1.int_mask_class1_RW = CLASS1_ENABLE_SEGMENT_FAULT_INTR | 2168 CLASS1_ENABLE_STORAGE_FAULT_INTR; 2169 csa->priv1.int_mask_class2_RW = CLASS2_ENABLE_SPU_STOP_INTR | 2170 CLASS2_ENABLE_SPU_HALT_INTR | 2171 CLASS2_ENABLE_SPU_DMA_TAG_GROUP_COMPLETE_INTR; 2172 } 2173 2174 static void init_priv2(struct spu_state *csa) 2175 { 2176 csa->priv2.spu_lslr_RW = LS_ADDR_MASK; 2177 csa->priv2.mfc_control_RW = MFC_CNTL_RESUME_DMA_QUEUE | 2178 MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION | 2179 MFC_CNTL_DMA_QUEUES_EMPTY_MASK; 2180 } 2181 2182 /** 2183 * spu_alloc_csa - allocate and initialize an SPU context save area. 2184 * 2185 * Allocate and initialize the contents of an SPU context save area. 2186 * This includes enabling address translation, interrupt masks, etc., 2187 * as appropriate for the given OS environment. 2188 * 2189 * Note that storage for the 'lscsa' is allocated separately, 2190 * as it is by far the largest of the context save regions, 2191 * and may need to be pinned or otherwise specially aligned. 2192 */ 2193 int spu_init_csa(struct spu_state *csa) 2194 { 2195 int rc; 2196 2197 if (!csa) 2198 return -EINVAL; 2199 memset(csa, 0, sizeof(struct spu_state)); 2200 2201 rc = spu_alloc_lscsa(csa); 2202 if (rc) 2203 return rc; 2204 2205 spin_lock_init(&csa->register_lock); 2206 2207 init_prob(csa); 2208 init_priv1(csa); 2209 init_priv2(csa); 2210 2211 return 0; 2212 } 2213 2214 void spu_fini_csa(struct spu_state *csa) 2215 { 2216 spu_free_lscsa(csa); 2217 } 2218