linux/arch/powerpc/platforms/cell/spufs/switch.c
<<
>>
Prefs
   1/*
   2 * spu_switch.c
   3 *
   4 * (C) Copyright IBM Corp. 2005
   5 *
   6 * Author: Mark Nutter <mnutter@us.ibm.com>
   7 *
   8 * Host-side part of SPU context switch sequence outlined in
   9 * Synergistic Processor Element, Book IV.
  10 *
  11 * A fully premptive switch of an SPE is very expensive in terms
  12 * of time and system resources.  SPE Book IV indicates that SPE
  13 * allocation should follow a "serially reusable device" model,
  14 * in which the SPE is assigned a task until it completes.  When
  15 * this is not possible, this sequence may be used to premptively
  16 * save, and then later (optionally) restore the context of a
  17 * program executing on an SPE.
  18 *
  19 *
  20 * This program is free software; you can redistribute it and/or modify
  21 * it under the terms of the GNU General Public License as published by
  22 * the Free Software Foundation; either version 2, or (at your option)
  23 * any later version.
  24 *
  25 * This program is distributed in the hope that it will be useful,
  26 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  27 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  28 * GNU General Public License for more details.
  29 *
  30 * You should have received a copy of the GNU General Public License
  31 * along with this program; if not, write to the Free Software
  32 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  33 */
  34
  35#include <linux/module.h>
  36#include <linux/errno.h>
  37#include <linux/hardirq.h>
  38#include <linux/sched.h>
  39#include <linux/kernel.h>
  40#include <linux/mm.h>
  41#include <linux/vmalloc.h>
  42#include <linux/smp.h>
  43#include <linux/stddef.h>
  44#include <linux/unistd.h>
  45
  46#include <asm/io.h>
  47#include <asm/spu.h>
  48#include <asm/spu_priv1.h>
  49#include <asm/spu_csa.h>
  50#include <asm/mmu_context.h>
  51
  52#include "spufs.h"
  53
  54#include "spu_save_dump.h"
  55#include "spu_restore_dump.h"
  56
  57#if 0
  58#define POLL_WHILE_TRUE(_c) {                           \
  59    do {                                                \
  60    } while (_c);                                       \
  61  }
  62#else
  63#define RELAX_SPIN_COUNT                                1000
  64#define POLL_WHILE_TRUE(_c) {                           \
  65    do {                                                \
  66        int _i;                                         \
  67        for (_i=0; _i<RELAX_SPIN_COUNT && (_c); _i++) { \
  68            cpu_relax();                                \
  69        }                                               \
  70        if (unlikely(_c)) yield();                      \
  71        else break;                                     \
  72    } while (_c);                                       \
  73  }
  74#endif                          /* debug */
  75
  76#define POLL_WHILE_FALSE(_c)    POLL_WHILE_TRUE(!(_c))
  77
  78static inline void acquire_spu_lock(struct spu *spu)
  79{
  80        /* Save, Step 1:
  81         * Restore, Step 1:
  82         *    Acquire SPU-specific mutual exclusion lock.
  83         *    TBD.
  84         */
  85}
  86
  87static inline void release_spu_lock(struct spu *spu)
  88{
  89        /* Restore, Step 76:
  90         *    Release SPU-specific mutual exclusion lock.
  91         *    TBD.
  92         */
  93}
  94
  95static inline int check_spu_isolate(struct spu_state *csa, struct spu *spu)
  96{
  97        struct spu_problem __iomem *prob = spu->problem;
  98        u32 isolate_state;
  99
 100        /* Save, Step 2:
 101         * Save, Step 6:
 102         *     If SPU_Status[E,L,IS] any field is '1', this
 103         *     SPU is in isolate state and cannot be context
 104         *     saved at this time.
 105         */
 106        isolate_state = SPU_STATUS_ISOLATED_STATE |
 107            SPU_STATUS_ISOLATED_LOAD_STATUS | SPU_STATUS_ISOLATED_EXIT_STATUS;
 108        return (in_be32(&prob->spu_status_R) & isolate_state) ? 1 : 0;
 109}
 110
 111static inline void disable_interrupts(struct spu_state *csa, struct spu *spu)
 112{
 113        /* Save, Step 3:
 114         * Restore, Step 2:
 115         *     Save INT_Mask_class0 in CSA.
 116         *     Write INT_MASK_class0 with value of 0.
 117         *     Save INT_Mask_class1 in CSA.
 118         *     Write INT_MASK_class1 with value of 0.
 119         *     Save INT_Mask_class2 in CSA.
 120         *     Write INT_MASK_class2 with value of 0.
 121         *     Synchronize all three interrupts to be sure
 122         *     we no longer execute a handler on another CPU.
 123         */
 124        spin_lock_irq(&spu->register_lock);
 125        if (csa) {
 126                csa->priv1.int_mask_class0_RW = spu_int_mask_get(spu, 0);
 127                csa->priv1.int_mask_class1_RW = spu_int_mask_get(spu, 1);
 128                csa->priv1.int_mask_class2_RW = spu_int_mask_get(spu, 2);
 129        }
 130        spu_int_mask_set(spu, 0, 0ul);
 131        spu_int_mask_set(spu, 1, 0ul);
 132        spu_int_mask_set(spu, 2, 0ul);
 133        eieio();
 134        spin_unlock_irq(&spu->register_lock);
 135        synchronize_irq(spu->irqs[0]);
 136        synchronize_irq(spu->irqs[1]);
 137        synchronize_irq(spu->irqs[2]);
 138}
 139
 140static inline void set_watchdog_timer(struct spu_state *csa, struct spu *spu)
 141{
 142        /* Save, Step 4:
 143         * Restore, Step 25.
 144         *    Set a software watchdog timer, which specifies the
 145         *    maximum allowable time for a context save sequence.
 146         *
 147         *    For present, this implementation will not set a global
 148         *    watchdog timer, as virtualization & variable system load
 149         *    may cause unpredictable execution times.
 150         */
 151}
 152
 153static inline void inhibit_user_access(struct spu_state *csa, struct spu *spu)
 154{
 155        /* Save, Step 5:
 156         * Restore, Step 3:
 157         *     Inhibit user-space access (if provided) to this
 158         *     SPU by unmapping the virtual pages assigned to
 159         *     the SPU memory-mapped I/O (MMIO) for problem
 160         *     state. TBD.
 161         */
 162}
 163
 164static inline void set_switch_pending(struct spu_state *csa, struct spu *spu)
 165{
 166        /* Save, Step 7:
 167         * Restore, Step 5:
 168         *     Set a software context switch pending flag.
 169         */
 170        set_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags);
 171        mb();
 172}
 173
 174static inline void save_mfc_cntl(struct spu_state *csa, struct spu *spu)
 175{
 176        struct spu_priv2 __iomem *priv2 = spu->priv2;
 177
 178        /* Save, Step 8:
 179         *     Suspend DMA and save MFC_CNTL.
 180         */
 181        switch (in_be64(&priv2->mfc_control_RW) &
 182               MFC_CNTL_SUSPEND_DMA_STATUS_MASK) {
 183        case MFC_CNTL_SUSPEND_IN_PROGRESS:
 184                POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) &
 185                                  MFC_CNTL_SUSPEND_DMA_STATUS_MASK) ==
 186                                 MFC_CNTL_SUSPEND_COMPLETE);
 187                /* fall through */
 188        case MFC_CNTL_SUSPEND_COMPLETE:
 189                if (csa) {
 190                        csa->priv2.mfc_control_RW =
 191                                MFC_CNTL_SUSPEND_MASK |
 192                                MFC_CNTL_SUSPEND_DMA_QUEUE;
 193                }
 194                break;
 195        case MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION:
 196                out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE);
 197                POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) &
 198                                  MFC_CNTL_SUSPEND_DMA_STATUS_MASK) ==
 199                                 MFC_CNTL_SUSPEND_COMPLETE);
 200                if (csa) {
 201                        csa->priv2.mfc_control_RW = 0;
 202                }
 203                break;
 204        }
 205}
 206
 207static inline void save_spu_runcntl(struct spu_state *csa, struct spu *spu)
 208{
 209        struct spu_problem __iomem *prob = spu->problem;
 210
 211        /* Save, Step 9:
 212         *     Save SPU_Runcntl in the CSA.  This value contains
 213         *     the "Application Desired State".
 214         */
 215        csa->prob.spu_runcntl_RW = in_be32(&prob->spu_runcntl_RW);
 216}
 217
 218static inline void save_mfc_sr1(struct spu_state *csa, struct spu *spu)
 219{
 220        /* Save, Step 10:
 221         *     Save MFC_SR1 in the CSA.
 222         */
 223        csa->priv1.mfc_sr1_RW = spu_mfc_sr1_get(spu);
 224}
 225
 226static inline void save_spu_status(struct spu_state *csa, struct spu *spu)
 227{
 228        struct spu_problem __iomem *prob = spu->problem;
 229
 230        /* Save, Step 11:
 231         *     Read SPU_Status[R], and save to CSA.
 232         */
 233        if ((in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) == 0) {
 234                csa->prob.spu_status_R = in_be32(&prob->spu_status_R);
 235        } else {
 236                u32 stopped;
 237
 238                out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
 239                eieio();
 240                POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
 241                                SPU_STATUS_RUNNING);
 242                stopped =
 243                    SPU_STATUS_INVALID_INSTR | SPU_STATUS_SINGLE_STEP |
 244                    SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP;
 245                if ((in_be32(&prob->spu_status_R) & stopped) == 0)
 246                        csa->prob.spu_status_R = SPU_STATUS_RUNNING;
 247                else
 248                        csa->prob.spu_status_R = in_be32(&prob->spu_status_R);
 249        }
 250}
 251
 252static inline void save_mfc_decr(struct spu_state *csa, struct spu *spu)
 253{
 254        struct spu_priv2 __iomem *priv2 = spu->priv2;
 255
 256        /* Save, Step 12:
 257         *     Read MFC_CNTL[Ds].  Update saved copy of
 258         *     CSA.MFC_CNTL[Ds].
 259         */
 260        csa->priv2.mfc_control_RW |=
 261                in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DECREMENTER_RUNNING;
 262}
 263
 264static inline void halt_mfc_decr(struct spu_state *csa, struct spu *spu)
 265{
 266        struct spu_priv2 __iomem *priv2 = spu->priv2;
 267
 268        /* Save, Step 13:
 269         *     Write MFC_CNTL[Dh] set to a '1' to halt
 270         *     the decrementer.
 271         */
 272        out_be64(&priv2->mfc_control_RW,
 273                 MFC_CNTL_DECREMENTER_HALTED | MFC_CNTL_SUSPEND_MASK);
 274        eieio();
 275}
 276
 277static inline void save_timebase(struct spu_state *csa, struct spu *spu)
 278{
 279        /* Save, Step 14:
 280         *    Read PPE Timebase High and Timebase low registers
 281         *    and save in CSA.  TBD.
 282         */
 283        csa->suspend_time = get_cycles();
 284}
 285
 286static inline void remove_other_spu_access(struct spu_state *csa,
 287                                           struct spu *spu)
 288{
 289        /* Save, Step 15:
 290         *     Remove other SPU access to this SPU by unmapping
 291         *     this SPU's pages from their address space.  TBD.
 292         */
 293}
 294
 295static inline void do_mfc_mssync(struct spu_state *csa, struct spu *spu)
 296{
 297        struct spu_problem __iomem *prob = spu->problem;
 298
 299        /* Save, Step 16:
 300         * Restore, Step 11.
 301         *     Write SPU_MSSync register. Poll SPU_MSSync[P]
 302         *     for a value of 0.
 303         */
 304        out_be64(&prob->spc_mssync_RW, 1UL);
 305        POLL_WHILE_TRUE(in_be64(&prob->spc_mssync_RW) & MS_SYNC_PENDING);
 306}
 307
 308static inline void issue_mfc_tlbie(struct spu_state *csa, struct spu *spu)
 309{
 310        /* Save, Step 17:
 311         * Restore, Step 12.
 312         * Restore, Step 48.
 313         *     Write TLB_Invalidate_Entry[IS,VPN,L,Lp]=0 register.
 314         *     Then issue a PPE sync instruction.
 315         */
 316        spu_tlb_invalidate(spu);
 317        mb();
 318}
 319
 320static inline void handle_pending_interrupts(struct spu_state *csa,
 321                                             struct spu *spu)
 322{
 323        /* Save, Step 18:
 324         *     Handle any pending interrupts from this SPU
 325         *     here.  This is OS or hypervisor specific.  One
 326         *     option is to re-enable interrupts to handle any
 327         *     pending interrupts, with the interrupt handlers
 328         *     recognizing the software Context Switch Pending
 329         *     flag, to ensure the SPU execution or MFC command
 330         *     queue is not restarted.  TBD.
 331         */
 332}
 333
 334static inline void save_mfc_queues(struct spu_state *csa, struct spu *spu)
 335{
 336        struct spu_priv2 __iomem *priv2 = spu->priv2;
 337        int i;
 338
 339        /* Save, Step 19:
 340         *     If MFC_Cntl[Se]=0 then save
 341         *     MFC command queues.
 342         */
 343        if ((in_be64(&priv2->mfc_control_RW) & MFC_CNTL_DMA_QUEUES_EMPTY) == 0) {
 344                for (i = 0; i < 8; i++) {
 345                        csa->priv2.puq[i].mfc_cq_data0_RW =
 346                            in_be64(&priv2->puq[i].mfc_cq_data0_RW);
 347                        csa->priv2.puq[i].mfc_cq_data1_RW =
 348                            in_be64(&priv2->puq[i].mfc_cq_data1_RW);
 349                        csa->priv2.puq[i].mfc_cq_data2_RW =
 350                            in_be64(&priv2->puq[i].mfc_cq_data2_RW);
 351                        csa->priv2.puq[i].mfc_cq_data3_RW =
 352                            in_be64(&priv2->puq[i].mfc_cq_data3_RW);
 353                }
 354                for (i = 0; i < 16; i++) {
 355                        csa->priv2.spuq[i].mfc_cq_data0_RW =
 356                            in_be64(&priv2->spuq[i].mfc_cq_data0_RW);
 357                        csa->priv2.spuq[i].mfc_cq_data1_RW =
 358                            in_be64(&priv2->spuq[i].mfc_cq_data1_RW);
 359                        csa->priv2.spuq[i].mfc_cq_data2_RW =
 360                            in_be64(&priv2->spuq[i].mfc_cq_data2_RW);
 361                        csa->priv2.spuq[i].mfc_cq_data3_RW =
 362                            in_be64(&priv2->spuq[i].mfc_cq_data3_RW);
 363                }
 364        }
 365}
 366
 367static inline void save_ppu_querymask(struct spu_state *csa, struct spu *spu)
 368{
 369        struct spu_problem __iomem *prob = spu->problem;
 370
 371        /* Save, Step 20:
 372         *     Save the PPU_QueryMask register
 373         *     in the CSA.
 374         */
 375        csa->prob.dma_querymask_RW = in_be32(&prob->dma_querymask_RW);
 376}
 377
 378static inline void save_ppu_querytype(struct spu_state *csa, struct spu *spu)
 379{
 380        struct spu_problem __iomem *prob = spu->problem;
 381
 382        /* Save, Step 21:
 383         *     Save the PPU_QueryType register
 384         *     in the CSA.
 385         */
 386        csa->prob.dma_querytype_RW = in_be32(&prob->dma_querytype_RW);
 387}
 388
 389static inline void save_ppu_tagstatus(struct spu_state *csa, struct spu *spu)
 390{
 391        struct spu_problem __iomem *prob = spu->problem;
 392
 393        /* Save the Prxy_TagStatus register in the CSA.
 394         *
 395         * It is unnecessary to restore dma_tagstatus_R, however,
 396         * dma_tagstatus_R in the CSA is accessed via backing_ops, so
 397         * we must save it.
 398         */
 399        csa->prob.dma_tagstatus_R = in_be32(&prob->dma_tagstatus_R);
 400}
 401
 402static inline void save_mfc_csr_tsq(struct spu_state *csa, struct spu *spu)
 403{
 404        struct spu_priv2 __iomem *priv2 = spu->priv2;
 405
 406        /* Save, Step 22:
 407         *     Save the MFC_CSR_TSQ register
 408         *     in the LSCSA.
 409         */
 410        csa->priv2.spu_tag_status_query_RW =
 411            in_be64(&priv2->spu_tag_status_query_RW);
 412}
 413
 414static inline void save_mfc_csr_cmd(struct spu_state *csa, struct spu *spu)
 415{
 416        struct spu_priv2 __iomem *priv2 = spu->priv2;
 417
 418        /* Save, Step 23:
 419         *     Save the MFC_CSR_CMD1 and MFC_CSR_CMD2
 420         *     registers in the CSA.
 421         */
 422        csa->priv2.spu_cmd_buf1_RW = in_be64(&priv2->spu_cmd_buf1_RW);
 423        csa->priv2.spu_cmd_buf2_RW = in_be64(&priv2->spu_cmd_buf2_RW);
 424}
 425
 426static inline void save_mfc_csr_ato(struct spu_state *csa, struct spu *spu)
 427{
 428        struct spu_priv2 __iomem *priv2 = spu->priv2;
 429
 430        /* Save, Step 24:
 431         *     Save the MFC_CSR_ATO register in
 432         *     the CSA.
 433         */
 434        csa->priv2.spu_atomic_status_RW = in_be64(&priv2->spu_atomic_status_RW);
 435}
 436
 437static inline void save_mfc_tclass_id(struct spu_state *csa, struct spu *spu)
 438{
 439        /* Save, Step 25:
 440         *     Save the MFC_TCLASS_ID register in
 441         *     the CSA.
 442         */
 443        csa->priv1.mfc_tclass_id_RW = spu_mfc_tclass_id_get(spu);
 444}
 445
 446static inline void set_mfc_tclass_id(struct spu_state *csa, struct spu *spu)
 447{
 448        /* Save, Step 26:
 449         * Restore, Step 23.
 450         *     Write the MFC_TCLASS_ID register with
 451         *     the value 0x10000000.
 452         */
 453        spu_mfc_tclass_id_set(spu, 0x10000000);
 454        eieio();
 455}
 456
 457static inline void purge_mfc_queue(struct spu_state *csa, struct spu *spu)
 458{
 459        struct spu_priv2 __iomem *priv2 = spu->priv2;
 460
 461        /* Save, Step 27:
 462         * Restore, Step 14.
 463         *     Write MFC_CNTL[Pc]=1 (purge queue).
 464         */
 465        out_be64(&priv2->mfc_control_RW, MFC_CNTL_PURGE_DMA_REQUEST);
 466        eieio();
 467}
 468
 469static inline void wait_purge_complete(struct spu_state *csa, struct spu *spu)
 470{
 471        struct spu_priv2 __iomem *priv2 = spu->priv2;
 472
 473        /* Save, Step 28:
 474         *     Poll MFC_CNTL[Ps] until value '11' is read
 475         *     (purge complete).
 476         */
 477        POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) &
 478                         MFC_CNTL_PURGE_DMA_STATUS_MASK) ==
 479                         MFC_CNTL_PURGE_DMA_COMPLETE);
 480}
 481
 482static inline void setup_mfc_sr1(struct spu_state *csa, struct spu *spu)
 483{
 484        /* Save, Step 30:
 485         * Restore, Step 18:
 486         *     Write MFC_SR1 with MFC_SR1[D=0,S=1] and
 487         *     MFC_SR1[TL,R,Pr,T] set correctly for the
 488         *     OS specific environment.
 489         *
 490         *     Implementation note: The SPU-side code
 491         *     for save/restore is privileged, so the
 492         *     MFC_SR1[Pr] bit is not set.
 493         *
 494         */
 495        spu_mfc_sr1_set(spu, (MFC_STATE1_MASTER_RUN_CONTROL_MASK |
 496                              MFC_STATE1_RELOCATE_MASK |
 497                              MFC_STATE1_BUS_TLBIE_MASK));
 498}
 499
 500static inline void save_spu_npc(struct spu_state *csa, struct spu *spu)
 501{
 502        struct spu_problem __iomem *prob = spu->problem;
 503
 504        /* Save, Step 31:
 505         *     Save SPU_NPC in the CSA.
 506         */
 507        csa->prob.spu_npc_RW = in_be32(&prob->spu_npc_RW);
 508}
 509
 510static inline void save_spu_privcntl(struct spu_state *csa, struct spu *spu)
 511{
 512        struct spu_priv2 __iomem *priv2 = spu->priv2;
 513
 514        /* Save, Step 32:
 515         *     Save SPU_PrivCntl in the CSA.
 516         */
 517        csa->priv2.spu_privcntl_RW = in_be64(&priv2->spu_privcntl_RW);
 518}
 519
 520static inline void reset_spu_privcntl(struct spu_state *csa, struct spu *spu)
 521{
 522        struct spu_priv2 __iomem *priv2 = spu->priv2;
 523
 524        /* Save, Step 33:
 525         * Restore, Step 16:
 526         *     Write SPU_PrivCntl[S,Le,A] fields reset to 0.
 527         */
 528        out_be64(&priv2->spu_privcntl_RW, 0UL);
 529        eieio();
 530}
 531
 532static inline void save_spu_lslr(struct spu_state *csa, struct spu *spu)
 533{
 534        struct spu_priv2 __iomem *priv2 = spu->priv2;
 535
 536        /* Save, Step 34:
 537         *     Save SPU_LSLR in the CSA.
 538         */
 539        csa->priv2.spu_lslr_RW = in_be64(&priv2->spu_lslr_RW);
 540}
 541
 542static inline void reset_spu_lslr(struct spu_state *csa, struct spu *spu)
 543{
 544        struct spu_priv2 __iomem *priv2 = spu->priv2;
 545
 546        /* Save, Step 35:
 547         * Restore, Step 17.
 548         *     Reset SPU_LSLR.
 549         */
 550        out_be64(&priv2->spu_lslr_RW, LS_ADDR_MASK);
 551        eieio();
 552}
 553
 554static inline void save_spu_cfg(struct spu_state *csa, struct spu *spu)
 555{
 556        struct spu_priv2 __iomem *priv2 = spu->priv2;
 557
 558        /* Save, Step 36:
 559         *     Save SPU_Cfg in the CSA.
 560         */
 561        csa->priv2.spu_cfg_RW = in_be64(&priv2->spu_cfg_RW);
 562}
 563
 564static inline void save_pm_trace(struct spu_state *csa, struct spu *spu)
 565{
 566        /* Save, Step 37:
 567         *     Save PM_Trace_Tag_Wait_Mask in the CSA.
 568         *     Not performed by this implementation.
 569         */
 570}
 571
 572static inline void save_mfc_rag(struct spu_state *csa, struct spu *spu)
 573{
 574        /* Save, Step 38:
 575         *     Save RA_GROUP_ID register and the
 576         *     RA_ENABLE reigster in the CSA.
 577         */
 578        csa->priv1.resource_allocation_groupID_RW =
 579                spu_resource_allocation_groupID_get(spu);
 580        csa->priv1.resource_allocation_enable_RW =
 581                spu_resource_allocation_enable_get(spu);
 582}
 583
 584static inline void save_ppu_mb_stat(struct spu_state *csa, struct spu *spu)
 585{
 586        struct spu_problem __iomem *prob = spu->problem;
 587
 588        /* Save, Step 39:
 589         *     Save MB_Stat register in the CSA.
 590         */
 591        csa->prob.mb_stat_R = in_be32(&prob->mb_stat_R);
 592}
 593
 594static inline void save_ppu_mb(struct spu_state *csa, struct spu *spu)
 595{
 596        struct spu_problem __iomem *prob = spu->problem;
 597
 598        /* Save, Step 40:
 599         *     Save the PPU_MB register in the CSA.
 600         */
 601        csa->prob.pu_mb_R = in_be32(&prob->pu_mb_R);
 602}
 603
 604static inline void save_ppuint_mb(struct spu_state *csa, struct spu *spu)
 605{
 606        struct spu_priv2 __iomem *priv2 = spu->priv2;
 607
 608        /* Save, Step 41:
 609         *     Save the PPUINT_MB register in the CSA.
 610         */
 611        csa->priv2.puint_mb_R = in_be64(&priv2->puint_mb_R);
 612}
 613
 614static inline void save_ch_part1(struct spu_state *csa, struct spu *spu)
 615{
 616        struct spu_priv2 __iomem *priv2 = spu->priv2;
 617        u64 idx, ch_indices[] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL };
 618        int i;
 619
 620        /* Save, Step 42:
 621         */
 622
 623        /* Save CH 1, without channel count */
 624        out_be64(&priv2->spu_chnlcntptr_RW, 1);
 625        csa->spu_chnldata_RW[1] = in_be64(&priv2->spu_chnldata_RW);
 626
 627        /* Save the following CH: [0,3,4,24,25,27] */
 628        for (i = 0; i < ARRAY_SIZE(ch_indices); i++) {
 629                idx = ch_indices[i];
 630                out_be64(&priv2->spu_chnlcntptr_RW, idx);
 631                eieio();
 632                csa->spu_chnldata_RW[idx] = in_be64(&priv2->spu_chnldata_RW);
 633                csa->spu_chnlcnt_RW[idx] = in_be64(&priv2->spu_chnlcnt_RW);
 634                out_be64(&priv2->spu_chnldata_RW, 0UL);
 635                out_be64(&priv2->spu_chnlcnt_RW, 0UL);
 636                eieio();
 637        }
 638}
 639
 640static inline void save_spu_mb(struct spu_state *csa, struct spu *spu)
 641{
 642        struct spu_priv2 __iomem *priv2 = spu->priv2;
 643        int i;
 644
 645        /* Save, Step 43:
 646         *     Save SPU Read Mailbox Channel.
 647         */
 648        out_be64(&priv2->spu_chnlcntptr_RW, 29UL);
 649        eieio();
 650        csa->spu_chnlcnt_RW[29] = in_be64(&priv2->spu_chnlcnt_RW);
 651        for (i = 0; i < 4; i++) {
 652                csa->spu_mailbox_data[i] = in_be64(&priv2->spu_chnldata_RW);
 653        }
 654        out_be64(&priv2->spu_chnlcnt_RW, 0UL);
 655        eieio();
 656}
 657
 658static inline void save_mfc_cmd(struct spu_state *csa, struct spu *spu)
 659{
 660        struct spu_priv2 __iomem *priv2 = spu->priv2;
 661
 662        /* Save, Step 44:
 663         *     Save MFC_CMD Channel.
 664         */
 665        out_be64(&priv2->spu_chnlcntptr_RW, 21UL);
 666        eieio();
 667        csa->spu_chnlcnt_RW[21] = in_be64(&priv2->spu_chnlcnt_RW);
 668        eieio();
 669}
 670
 671static inline void reset_ch(struct spu_state *csa, struct spu *spu)
 672{
 673        struct spu_priv2 __iomem *priv2 = spu->priv2;
 674        u64 ch_indices[4] = { 21UL, 23UL, 28UL, 30UL };
 675        u64 ch_counts[4] = { 16UL, 1UL, 1UL, 1UL };
 676        u64 idx;
 677        int i;
 678
 679        /* Save, Step 45:
 680         *     Reset the following CH: [21, 23, 28, 30]
 681         */
 682        for (i = 0; i < 4; i++) {
 683                idx = ch_indices[i];
 684                out_be64(&priv2->spu_chnlcntptr_RW, idx);
 685                eieio();
 686                out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]);
 687                eieio();
 688        }
 689}
 690
 691static inline void resume_mfc_queue(struct spu_state *csa, struct spu *spu)
 692{
 693        struct spu_priv2 __iomem *priv2 = spu->priv2;
 694
 695        /* Save, Step 46:
 696         * Restore, Step 25.
 697         *     Write MFC_CNTL[Sc]=0 (resume queue processing).
 698         */
 699        out_be64(&priv2->mfc_control_RW, MFC_CNTL_RESUME_DMA_QUEUE);
 700}
 701
 702static inline void setup_mfc_slbs(struct spu_state *csa, struct spu *spu,
 703                unsigned int *code, int code_size)
 704{
 705        /* Save, Step 47:
 706         * Restore, Step 30.
 707         *     If MFC_SR1[R]=1, write 0 to SLB_Invalidate_All
 708         *     register, then initialize SLB_VSID and SLB_ESID
 709         *     to provide access to SPU context save code and
 710         *     LSCSA.
 711         *
 712         *     This implementation places both the context
 713         *     switch code and LSCSA in kernel address space.
 714         *
 715         *     Further this implementation assumes that the
 716         *     MFC_SR1[R]=1 (in other words, assume that
 717         *     translation is desired by OS environment).
 718         */
 719        spu_invalidate_slbs(spu);
 720        spu_setup_kernel_slbs(spu, csa->lscsa, code, code_size);
 721}
 722
 723static inline void set_switch_active(struct spu_state *csa, struct spu *spu)
 724{
 725        /* Save, Step 48:
 726         * Restore, Step 23.
 727         *     Change the software context switch pending flag
 728         *     to context switch active.
 729         *
 730         *     This implementation does not uses a switch active flag.
 731         */
 732        clear_bit(SPU_CONTEXT_SWITCH_PENDING, &spu->flags);
 733        mb();
 734}
 735
 736static inline void enable_interrupts(struct spu_state *csa, struct spu *spu)
 737{
 738        unsigned long class1_mask = CLASS1_ENABLE_SEGMENT_FAULT_INTR |
 739            CLASS1_ENABLE_STORAGE_FAULT_INTR;
 740
 741        /* Save, Step 49:
 742         * Restore, Step 22:
 743         *     Reset and then enable interrupts, as
 744         *     needed by OS.
 745         *
 746         *     This implementation enables only class1
 747         *     (translation) interrupts.
 748         */
 749        spin_lock_irq(&spu->register_lock);
 750        spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK);
 751        spu_int_stat_clear(spu, 1, CLASS1_INTR_MASK);
 752        spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK);
 753        spu_int_mask_set(spu, 0, 0ul);
 754        spu_int_mask_set(spu, 1, class1_mask);
 755        spu_int_mask_set(spu, 2, 0ul);
 756        spin_unlock_irq(&spu->register_lock);
 757}
 758
 759static inline int send_mfc_dma(struct spu *spu, unsigned long ea,
 760                               unsigned int ls_offset, unsigned int size,
 761                               unsigned int tag, unsigned int rclass,
 762                               unsigned int cmd)
 763{
 764        struct spu_problem __iomem *prob = spu->problem;
 765        union mfc_tag_size_class_cmd command;
 766        unsigned int transfer_size;
 767        volatile unsigned int status = 0x0;
 768
 769        while (size > 0) {
 770                transfer_size =
 771                    (size > MFC_MAX_DMA_SIZE) ? MFC_MAX_DMA_SIZE : size;
 772                command.u.mfc_size = transfer_size;
 773                command.u.mfc_tag = tag;
 774                command.u.mfc_rclassid = rclass;
 775                command.u.mfc_cmd = cmd;
 776                do {
 777                        out_be32(&prob->mfc_lsa_W, ls_offset);
 778                        out_be64(&prob->mfc_ea_W, ea);
 779                        out_be64(&prob->mfc_union_W.all64, command.all64);
 780                        status =
 781                            in_be32(&prob->mfc_union_W.by32.mfc_class_cmd32);
 782                        if (unlikely(status & 0x2)) {
 783                                cpu_relax();
 784                        }
 785                } while (status & 0x3);
 786                size -= transfer_size;
 787                ea += transfer_size;
 788                ls_offset += transfer_size;
 789        }
 790        return 0;
 791}
 792
 793static inline void save_ls_16kb(struct spu_state *csa, struct spu *spu)
 794{
 795        unsigned long addr = (unsigned long)&csa->lscsa->ls[0];
 796        unsigned int ls_offset = 0x0;
 797        unsigned int size = 16384;
 798        unsigned int tag = 0;
 799        unsigned int rclass = 0;
 800        unsigned int cmd = MFC_PUT_CMD;
 801
 802        /* Save, Step 50:
 803         *     Issue a DMA command to copy the first 16K bytes
 804         *     of local storage to the CSA.
 805         */
 806        send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
 807}
 808
 809static inline void set_spu_npc(struct spu_state *csa, struct spu *spu)
 810{
 811        struct spu_problem __iomem *prob = spu->problem;
 812
 813        /* Save, Step 51:
 814         * Restore, Step 31.
 815         *     Write SPU_NPC[IE]=0 and SPU_NPC[LSA] to entry
 816         *     point address of context save code in local
 817         *     storage.
 818         *
 819         *     This implementation uses SPU-side save/restore
 820         *     programs with entry points at LSA of 0.
 821         */
 822        out_be32(&prob->spu_npc_RW, 0);
 823        eieio();
 824}
 825
 826static inline void set_signot1(struct spu_state *csa, struct spu *spu)
 827{
 828        struct spu_problem __iomem *prob = spu->problem;
 829        union {
 830                u64 ull;
 831                u32 ui[2];
 832        } addr64;
 833
 834        /* Save, Step 52:
 835         * Restore, Step 32:
 836         *    Write SPU_Sig_Notify_1 register with upper 32-bits
 837         *    of the CSA.LSCSA effective address.
 838         */
 839        addr64.ull = (u64) csa->lscsa;
 840        out_be32(&prob->signal_notify1, addr64.ui[0]);
 841        eieio();
 842}
 843
 844static inline void set_signot2(struct spu_state *csa, struct spu *spu)
 845{
 846        struct spu_problem __iomem *prob = spu->problem;
 847        union {
 848                u64 ull;
 849                u32 ui[2];
 850        } addr64;
 851
 852        /* Save, Step 53:
 853         * Restore, Step 33:
 854         *    Write SPU_Sig_Notify_2 register with lower 32-bits
 855         *    of the CSA.LSCSA effective address.
 856         */
 857        addr64.ull = (u64) csa->lscsa;
 858        out_be32(&prob->signal_notify2, addr64.ui[1]);
 859        eieio();
 860}
 861
 862static inline void send_save_code(struct spu_state *csa, struct spu *spu)
 863{
 864        unsigned long addr = (unsigned long)&spu_save_code[0];
 865        unsigned int ls_offset = 0x0;
 866        unsigned int size = sizeof(spu_save_code);
 867        unsigned int tag = 0;
 868        unsigned int rclass = 0;
 869        unsigned int cmd = MFC_GETFS_CMD;
 870
 871        /* Save, Step 54:
 872         *     Issue a DMA command to copy context save code
 873         *     to local storage and start SPU.
 874         */
 875        send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
 876}
 877
 878static inline void set_ppu_querymask(struct spu_state *csa, struct spu *spu)
 879{
 880        struct spu_problem __iomem *prob = spu->problem;
 881
 882        /* Save, Step 55:
 883         * Restore, Step 38.
 884         *     Write PPU_QueryMask=1 (enable Tag Group 0)
 885         *     and issue eieio instruction.
 886         */
 887        out_be32(&prob->dma_querymask_RW, MFC_TAGID_TO_TAGMASK(0));
 888        eieio();
 889}
 890
 891static inline void wait_tag_complete(struct spu_state *csa, struct spu *spu)
 892{
 893        struct spu_problem __iomem *prob = spu->problem;
 894        u32 mask = MFC_TAGID_TO_TAGMASK(0);
 895        unsigned long flags;
 896
 897        /* Save, Step 56:
 898         * Restore, Step 39.
 899         * Restore, Step 39.
 900         * Restore, Step 46.
 901         *     Poll PPU_TagStatus[gn] until 01 (Tag group 0 complete)
 902         *     or write PPU_QueryType[TS]=01 and wait for Tag Group
 903         *     Complete Interrupt.  Write INT_Stat_Class0 or
 904         *     INT_Stat_Class2 with value of 'handled'.
 905         */
 906        POLL_WHILE_FALSE(in_be32(&prob->dma_tagstatus_R) & mask);
 907
 908        local_irq_save(flags);
 909        spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK);
 910        spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK);
 911        local_irq_restore(flags);
 912}
 913
 914static inline void wait_spu_stopped(struct spu_state *csa, struct spu *spu)
 915{
 916        struct spu_problem __iomem *prob = spu->problem;
 917        unsigned long flags;
 918
 919        /* Save, Step 57:
 920         * Restore, Step 40.
 921         *     Poll until SPU_Status[R]=0 or wait for SPU Class 0
 922         *     or SPU Class 2 interrupt.  Write INT_Stat_class0
 923         *     or INT_Stat_class2 with value of handled.
 924         */
 925        POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING);
 926
 927        local_irq_save(flags);
 928        spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK);
 929        spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK);
 930        local_irq_restore(flags);
 931}
 932
 933static inline int check_save_status(struct spu_state *csa, struct spu *spu)
 934{
 935        struct spu_problem __iomem *prob = spu->problem;
 936        u32 complete;
 937
 938        /* Save, Step 54:
 939         *     If SPU_Status[P]=1 and SPU_Status[SC] = "success",
 940         *     context save succeeded, otherwise context save
 941         *     failed.
 942         */
 943        complete = ((SPU_SAVE_COMPLETE << SPU_STOP_STATUS_SHIFT) |
 944                    SPU_STATUS_STOPPED_BY_STOP);
 945        return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0;
 946}
 947
 948static inline void terminate_spu_app(struct spu_state *csa, struct spu *spu)
 949{
 950        /* Restore, Step 4:
 951         *    If required, notify the "using application" that
 952         *    the SPU task has been terminated.  TBD.
 953         */
 954}
 955
 956static inline void suspend_mfc_and_halt_decr(struct spu_state *csa,
 957                struct spu *spu)
 958{
 959        struct spu_priv2 __iomem *priv2 = spu->priv2;
 960
 961        /* Restore, Step 7:
 962         *     Write MFC_Cntl[Dh,Sc,Sm]='1','1','0' to suspend
 963         *     the queue and halt the decrementer.
 964         */
 965        out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE |
 966                 MFC_CNTL_DECREMENTER_HALTED);
 967        eieio();
 968}
 969
 970static inline void wait_suspend_mfc_complete(struct spu_state *csa,
 971                                             struct spu *spu)
 972{
 973        struct spu_priv2 __iomem *priv2 = spu->priv2;
 974
 975        /* Restore, Step 8:
 976         * Restore, Step 47.
 977         *     Poll MFC_CNTL[Ss] until 11 is returned.
 978         */
 979        POLL_WHILE_FALSE((in_be64(&priv2->mfc_control_RW) &
 980                         MFC_CNTL_SUSPEND_DMA_STATUS_MASK) ==
 981                         MFC_CNTL_SUSPEND_COMPLETE);
 982}
 983
 984static inline int suspend_spe(struct spu_state *csa, struct spu *spu)
 985{
 986        struct spu_problem __iomem *prob = spu->problem;
 987
 988        /* Restore, Step 9:
 989         *    If SPU_Status[R]=1, stop SPU execution
 990         *    and wait for stop to complete.
 991         *
 992         *    Returns       1 if SPU_Status[R]=1 on entry.
 993         *                  0 otherwise
 994         */
 995        if (in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING) {
 996                if (in_be32(&prob->spu_status_R) &
 997                    SPU_STATUS_ISOLATED_EXIT_STATUS) {
 998                        POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
 999                                        SPU_STATUS_RUNNING);
1000                }
1001                if ((in_be32(&prob->spu_status_R) &
1002                     SPU_STATUS_ISOLATED_LOAD_STATUS)
1003                    || (in_be32(&prob->spu_status_R) &
1004                        SPU_STATUS_ISOLATED_STATE)) {
1005                        out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
1006                        eieio();
1007                        POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1008                                        SPU_STATUS_RUNNING);
1009                        out_be32(&prob->spu_runcntl_RW, 0x2);
1010                        eieio();
1011                        POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1012                                        SPU_STATUS_RUNNING);
1013                }
1014                if (in_be32(&prob->spu_status_R) &
1015                    SPU_STATUS_WAITING_FOR_CHANNEL) {
1016                        out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
1017                        eieio();
1018                        POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1019                                        SPU_STATUS_RUNNING);
1020                }
1021                return 1;
1022        }
1023        return 0;
1024}
1025
1026static inline void clear_spu_status(struct spu_state *csa, struct spu *spu)
1027{
1028        struct spu_problem __iomem *prob = spu->problem;
1029
1030        /* Restore, Step 10:
1031         *    If SPU_Status[R]=0 and SPU_Status[E,L,IS]=1,
1032         *    release SPU from isolate state.
1033         */
1034        if (!(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING)) {
1035                if (in_be32(&prob->spu_status_R) &
1036                    SPU_STATUS_ISOLATED_EXIT_STATUS) {
1037                        spu_mfc_sr1_set(spu,
1038                                        MFC_STATE1_MASTER_RUN_CONTROL_MASK);
1039                        eieio();
1040                        out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
1041                        eieio();
1042                        POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1043                                        SPU_STATUS_RUNNING);
1044                }
1045                if ((in_be32(&prob->spu_status_R) &
1046                     SPU_STATUS_ISOLATED_LOAD_STATUS)
1047                    || (in_be32(&prob->spu_status_R) &
1048                        SPU_STATUS_ISOLATED_STATE)) {
1049                        spu_mfc_sr1_set(spu,
1050                                        MFC_STATE1_MASTER_RUN_CONTROL_MASK);
1051                        eieio();
1052                        out_be32(&prob->spu_runcntl_RW, 0x2);
1053                        eieio();
1054                        POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1055                                        SPU_STATUS_RUNNING);
1056                }
1057        }
1058}
1059
1060static inline void reset_ch_part1(struct spu_state *csa, struct spu *spu)
1061{
1062        struct spu_priv2 __iomem *priv2 = spu->priv2;
1063        u64 ch_indices[] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL };
1064        u64 idx;
1065        int i;
1066
1067        /* Restore, Step 20:
1068         */
1069
1070        /* Reset CH 1 */
1071        out_be64(&priv2->spu_chnlcntptr_RW, 1);
1072        out_be64(&priv2->spu_chnldata_RW, 0UL);
1073
1074        /* Reset the following CH: [0,3,4,24,25,27] */
1075        for (i = 0; i < ARRAY_SIZE(ch_indices); i++) {
1076                idx = ch_indices[i];
1077                out_be64(&priv2->spu_chnlcntptr_RW, idx);
1078                eieio();
1079                out_be64(&priv2->spu_chnldata_RW, 0UL);
1080                out_be64(&priv2->spu_chnlcnt_RW, 0UL);
1081                eieio();
1082        }
1083}
1084
1085static inline void reset_ch_part2(struct spu_state *csa, struct spu *spu)
1086{
1087        struct spu_priv2 __iomem *priv2 = spu->priv2;
1088        u64 ch_indices[5] = { 21UL, 23UL, 28UL, 29UL, 30UL };
1089        u64 ch_counts[5] = { 16UL, 1UL, 1UL, 0UL, 1UL };
1090        u64 idx;
1091        int i;
1092
1093        /* Restore, Step 21:
1094         *     Reset the following CH: [21, 23, 28, 29, 30]
1095         */
1096        for (i = 0; i < 5; i++) {
1097                idx = ch_indices[i];
1098                out_be64(&priv2->spu_chnlcntptr_RW, idx);
1099                eieio();
1100                out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]);
1101                eieio();
1102        }
1103}
1104
1105static inline void setup_spu_status_part1(struct spu_state *csa,
1106                                          struct spu *spu)
1107{
1108        u32 status_P = SPU_STATUS_STOPPED_BY_STOP;
1109        u32 status_I = SPU_STATUS_INVALID_INSTR;
1110        u32 status_H = SPU_STATUS_STOPPED_BY_HALT;
1111        u32 status_S = SPU_STATUS_SINGLE_STEP;
1112        u32 status_S_I = SPU_STATUS_SINGLE_STEP | SPU_STATUS_INVALID_INSTR;
1113        u32 status_S_P = SPU_STATUS_SINGLE_STEP | SPU_STATUS_STOPPED_BY_STOP;
1114        u32 status_P_H = SPU_STATUS_STOPPED_BY_HALT |SPU_STATUS_STOPPED_BY_STOP;
1115        u32 status_P_I = SPU_STATUS_STOPPED_BY_STOP |SPU_STATUS_INVALID_INSTR;
1116        u32 status_code;
1117
1118        /* Restore, Step 27:
1119         *     If the CSA.SPU_Status[I,S,H,P]=1 then add the correct
1120         *     instruction sequence to the end of the SPU based restore
1121         *     code (after the "context restored" stop and signal) to
1122         *     restore the correct SPU status.
1123         *
1124         *     NOTE: Rather than modifying the SPU executable, we
1125         *     instead add a new 'stopped_status' field to the
1126         *     LSCSA.  The SPU-side restore reads this field and
1127         *     takes the appropriate action when exiting.
1128         */
1129
1130        status_code =
1131            (csa->prob.spu_status_R >> SPU_STOP_STATUS_SHIFT) & 0xFFFF;
1132        if ((csa->prob.spu_status_R & status_P_I) == status_P_I) {
1133
1134                /* SPU_Status[P,I]=1 - Illegal Instruction followed
1135                 * by Stop and Signal instruction, followed by 'br -4'.
1136                 *
1137                 */
1138                csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_I;
1139                csa->lscsa->stopped_status.slot[1] = status_code;
1140
1141        } else if ((csa->prob.spu_status_R & status_P_H) == status_P_H) {
1142
1143                /* SPU_Status[P,H]=1 - Halt Conditional, followed
1144                 * by Stop and Signal instruction, followed by
1145                 * 'br -4'.
1146                 */
1147                csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P_H;
1148                csa->lscsa->stopped_status.slot[1] = status_code;
1149
1150        } else if ((csa->prob.spu_status_R & status_S_P) == status_S_P) {
1151
1152                /* SPU_Status[S,P]=1 - Stop and Signal instruction
1153                 * followed by 'br -4'.
1154                 */
1155                csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_P;
1156                csa->lscsa->stopped_status.slot[1] = status_code;
1157
1158        } else if ((csa->prob.spu_status_R & status_S_I) == status_S_I) {
1159
1160                /* SPU_Status[S,I]=1 - Illegal instruction followed
1161                 * by 'br -4'.
1162                 */
1163                csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S_I;
1164                csa->lscsa->stopped_status.slot[1] = status_code;
1165
1166        } else if ((csa->prob.spu_status_R & status_P) == status_P) {
1167
1168                /* SPU_Status[P]=1 - Stop and Signal instruction
1169                 * followed by 'br -4'.
1170                 */
1171                csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_P;
1172                csa->lscsa->stopped_status.slot[1] = status_code;
1173
1174        } else if ((csa->prob.spu_status_R & status_H) == status_H) {
1175
1176                /* SPU_Status[H]=1 - Halt Conditional, followed
1177                 * by 'br -4'.
1178                 */
1179                csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_H;
1180
1181        } else if ((csa->prob.spu_status_R & status_S) == status_S) {
1182
1183                /* SPU_Status[S]=1 - Two nop instructions.
1184                 */
1185                csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_S;
1186
1187        } else if ((csa->prob.spu_status_R & status_I) == status_I) {
1188
1189                /* SPU_Status[I]=1 - Illegal instruction followed
1190                 * by 'br -4'.
1191                 */
1192                csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_I;
1193
1194        }
1195}
1196
1197static inline void setup_spu_status_part2(struct spu_state *csa,
1198                                          struct spu *spu)
1199{
1200        u32 mask;
1201
1202        /* Restore, Step 28:
1203         *     If the CSA.SPU_Status[I,S,H,P,R]=0 then
1204         *     add a 'br *' instruction to the end of
1205         *     the SPU based restore code.
1206         *
1207         *     NOTE: Rather than modifying the SPU executable, we
1208         *     instead add a new 'stopped_status' field to the
1209         *     LSCSA.  The SPU-side restore reads this field and
1210         *     takes the appropriate action when exiting.
1211         */
1212        mask = SPU_STATUS_INVALID_INSTR |
1213            SPU_STATUS_SINGLE_STEP |
1214            SPU_STATUS_STOPPED_BY_HALT |
1215            SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING;
1216        if (!(csa->prob.spu_status_R & mask)) {
1217                csa->lscsa->stopped_status.slot[0] = SPU_STOPPED_STATUS_R;
1218        }
1219}
1220
1221static inline void restore_mfc_rag(struct spu_state *csa, struct spu *spu)
1222{
1223        /* Restore, Step 29:
1224         *     Restore RA_GROUP_ID register and the
1225         *     RA_ENABLE reigster from the CSA.
1226         */
1227        spu_resource_allocation_groupID_set(spu,
1228                        csa->priv1.resource_allocation_groupID_RW);
1229        spu_resource_allocation_enable_set(spu,
1230                        csa->priv1.resource_allocation_enable_RW);
1231}
1232
1233static inline void send_restore_code(struct spu_state *csa, struct spu *spu)
1234{
1235        unsigned long addr = (unsigned long)&spu_restore_code[0];
1236        unsigned int ls_offset = 0x0;
1237        unsigned int size = sizeof(spu_restore_code);
1238        unsigned int tag = 0;
1239        unsigned int rclass = 0;
1240        unsigned int cmd = MFC_GETFS_CMD;
1241
1242        /* Restore, Step 37:
1243         *     Issue MFC DMA command to copy context
1244         *     restore code to local storage.
1245         */
1246        send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
1247}
1248
1249static inline void setup_decr(struct spu_state *csa, struct spu *spu)
1250{
1251        /* Restore, Step 34:
1252         *     If CSA.MFC_CNTL[Ds]=1 (decrementer was
1253         *     running) then adjust decrementer, set
1254         *     decrementer running status in LSCSA,
1255         *     and set decrementer "wrapped" status
1256         *     in LSCSA.
1257         */
1258        if (csa->priv2.mfc_control_RW & MFC_CNTL_DECREMENTER_RUNNING) {
1259                cycles_t resume_time = get_cycles();
1260                cycles_t delta_time = resume_time - csa->suspend_time;
1261
1262                csa->lscsa->decr_status.slot[0] = SPU_DECR_STATUS_RUNNING;
1263                if (csa->lscsa->decr.slot[0] < delta_time) {
1264                        csa->lscsa->decr_status.slot[0] |=
1265                                 SPU_DECR_STATUS_WRAPPED;
1266                }
1267
1268                csa->lscsa->decr.slot[0] -= delta_time;
1269        } else {
1270                csa->lscsa->decr_status.slot[0] = 0;
1271        }
1272}
1273
1274static inline void setup_ppu_mb(struct spu_state *csa, struct spu *spu)
1275{
1276        /* Restore, Step 35:
1277         *     Copy the CSA.PU_MB data into the LSCSA.
1278         */
1279        csa->lscsa->ppu_mb.slot[0] = csa->prob.pu_mb_R;
1280}
1281
1282static inline void setup_ppuint_mb(struct spu_state *csa, struct spu *spu)
1283{
1284        /* Restore, Step 36:
1285         *     Copy the CSA.PUINT_MB data into the LSCSA.
1286         */
1287        csa->lscsa->ppuint_mb.slot[0] = csa->priv2.puint_mb_R;
1288}
1289
1290static inline int check_restore_status(struct spu_state *csa, struct spu *spu)
1291{
1292        struct spu_problem __iomem *prob = spu->problem;
1293        u32 complete;
1294
1295        /* Restore, Step 40:
1296         *     If SPU_Status[P]=1 and SPU_Status[SC] = "success",
1297         *     context restore succeeded, otherwise context restore
1298         *     failed.
1299         */
1300        complete = ((SPU_RESTORE_COMPLETE << SPU_STOP_STATUS_SHIFT) |
1301                    SPU_STATUS_STOPPED_BY_STOP);
1302        return (in_be32(&prob->spu_status_R) != complete) ? 1 : 0;
1303}
1304
1305static inline void restore_spu_privcntl(struct spu_state *csa, struct spu *spu)
1306{
1307        struct spu_priv2 __iomem *priv2 = spu->priv2;
1308
1309        /* Restore, Step 41:
1310         *     Restore SPU_PrivCntl from the CSA.
1311         */
1312        out_be64(&priv2->spu_privcntl_RW, csa->priv2.spu_privcntl_RW);
1313        eieio();
1314}
1315
1316static inline void restore_status_part1(struct spu_state *csa, struct spu *spu)
1317{
1318        struct spu_problem __iomem *prob = spu->problem;
1319        u32 mask;
1320
1321        /* Restore, Step 42:
1322         *     If any CSA.SPU_Status[I,S,H,P]=1, then
1323         *     restore the error or single step state.
1324         */
1325        mask = SPU_STATUS_INVALID_INSTR |
1326            SPU_STATUS_SINGLE_STEP |
1327            SPU_STATUS_STOPPED_BY_HALT | SPU_STATUS_STOPPED_BY_STOP;
1328        if (csa->prob.spu_status_R & mask) {
1329                out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
1330                eieio();
1331                POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1332                                SPU_STATUS_RUNNING);
1333        }
1334}
1335
1336static inline void restore_status_part2(struct spu_state *csa, struct spu *spu)
1337{
1338        struct spu_problem __iomem *prob = spu->problem;
1339        u32 mask;
1340
1341        /* Restore, Step 43:
1342         *     If all CSA.SPU_Status[I,S,H,P,R]=0 then write
1343         *     SPU_RunCntl[R0R1]='01', wait for SPU_Status[R]=1,
1344         *     then write '00' to SPU_RunCntl[R0R1] and wait
1345         *     for SPU_Status[R]=0.
1346         */
1347        mask = SPU_STATUS_INVALID_INSTR |
1348            SPU_STATUS_SINGLE_STEP |
1349            SPU_STATUS_STOPPED_BY_HALT |
1350            SPU_STATUS_STOPPED_BY_STOP | SPU_STATUS_RUNNING;
1351        if (!(csa->prob.spu_status_R & mask)) {
1352                out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
1353                eieio();
1354                POLL_WHILE_FALSE(in_be32(&prob->spu_status_R) &
1355                                 SPU_STATUS_RUNNING);
1356                out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
1357                eieio();
1358                POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) &
1359                                SPU_STATUS_RUNNING);
1360        }
1361}
1362
1363static inline void restore_ls_16kb(struct spu_state *csa, struct spu *spu)
1364{
1365        unsigned long addr = (unsigned long)&csa->lscsa->ls[0];
1366        unsigned int ls_offset = 0x0;
1367        unsigned int size = 16384;
1368        unsigned int tag = 0;
1369        unsigned int rclass = 0;
1370        unsigned int cmd = MFC_GET_CMD;
1371
1372        /* Restore, Step 44:
1373         *     Issue a DMA command to restore the first
1374         *     16kb of local storage from CSA.
1375         */
1376        send_mfc_dma(spu, addr, ls_offset, size, tag, rclass, cmd);
1377}
1378
1379static inline void suspend_mfc(struct spu_state *csa, struct spu *spu)
1380{
1381        struct spu_priv2 __iomem *priv2 = spu->priv2;
1382
1383        /* Restore, Step 47.
1384         *     Write MFC_Cntl[Sc,Sm]='1','0' to suspend
1385         *     the queue.
1386         */
1387        out_be64(&priv2->mfc_control_RW, MFC_CNTL_SUSPEND_DMA_QUEUE);
1388        eieio();
1389}
1390
1391static inline void clear_interrupts(struct spu_state *csa, struct spu *spu)
1392{
1393        /* Restore, Step 49:
1394         *     Write INT_MASK_class0 with value of 0.
1395         *     Write INT_MASK_class1 with value of 0.
1396         *     Write INT_MASK_class2 with value of 0.
1397         *     Write INT_STAT_class0 with value of -1.
1398         *     Write INT_STAT_class1 with value of -1.
1399         *     Write INT_STAT_class2 with value of -1.
1400         */
1401        spin_lock_irq(&spu->register_lock);
1402        spu_int_mask_set(spu, 0, 0ul);
1403        spu_int_mask_set(spu, 1, 0ul);
1404        spu_int_mask_set(spu, 2, 0ul);
1405        spu_int_stat_clear(spu, 0, CLASS0_INTR_MASK);
1406        spu_int_stat_clear(spu, 1, CLASS1_INTR_MASK);
1407        spu_int_stat_clear(spu, 2, CLASS2_INTR_MASK);
1408        spin_unlock_irq(&spu->register_lock);
1409}
1410
1411static inline void restore_mfc_queues(struct spu_state *csa, struct spu *spu)
1412{
1413        struct spu_priv2 __iomem *priv2 = spu->priv2;
1414        int i;
1415
1416        /* Restore, Step 50:
1417         *     If MFC_Cntl[Se]!=0 then restore
1418         *     MFC command queues.
1419         */
1420        if ((csa->priv2.mfc_control_RW & MFC_CNTL_DMA_QUEUES_EMPTY_MASK) == 0) {
1421                for (i = 0; i < 8; i++) {
1422                        out_be64(&priv2->puq[i].mfc_cq_data0_RW,
1423                                 csa->priv2.puq[i].mfc_cq_data0_RW);
1424                        out_be64(&priv2->puq[i].mfc_cq_data1_RW,
1425                                 csa->priv2.puq[i].mfc_cq_data1_RW);
1426                        out_be64(&priv2->puq[i].mfc_cq_data2_RW,
1427                                 csa->priv2.puq[i].mfc_cq_data2_RW);
1428                        out_be64(&priv2->puq[i].mfc_cq_data3_RW,
1429                                 csa->priv2.puq[i].mfc_cq_data3_RW);
1430                }
1431                for (i = 0; i < 16; i++) {
1432                        out_be64(&priv2->spuq[i].mfc_cq_data0_RW,
1433                                 csa->priv2.spuq[i].mfc_cq_data0_RW);
1434                        out_be64(&priv2->spuq[i].mfc_cq_data1_RW,
1435                                 csa->priv2.spuq[i].mfc_cq_data1_RW);
1436                        out_be64(&priv2->spuq[i].mfc_cq_data2_RW,
1437                                 csa->priv2.spuq[i].mfc_cq_data2_RW);
1438                        out_be64(&priv2->spuq[i].mfc_cq_data3_RW,
1439                                 csa->priv2.spuq[i].mfc_cq_data3_RW);
1440                }
1441        }
1442        eieio();
1443}
1444
1445static inline void restore_ppu_querymask(struct spu_state *csa, struct spu *spu)
1446{
1447        struct spu_problem __iomem *prob = spu->problem;
1448
1449        /* Restore, Step 51:
1450         *     Restore the PPU_QueryMask register from CSA.
1451         */
1452        out_be32(&prob->dma_querymask_RW, csa->prob.dma_querymask_RW);
1453        eieio();
1454}
1455
1456static inline void restore_ppu_querytype(struct spu_state *csa, struct spu *spu)
1457{
1458        struct spu_problem __iomem *prob = spu->problem;
1459
1460        /* Restore, Step 52:
1461         *     Restore the PPU_QueryType register from CSA.
1462         */
1463        out_be32(&prob->dma_querytype_RW, csa->prob.dma_querytype_RW);
1464        eieio();
1465}
1466
1467static inline void restore_mfc_csr_tsq(struct spu_state *csa, struct spu *spu)
1468{
1469        struct spu_priv2 __iomem *priv2 = spu->priv2;
1470
1471        /* Restore, Step 53:
1472         *     Restore the MFC_CSR_TSQ register from CSA.
1473         */
1474        out_be64(&priv2->spu_tag_status_query_RW,
1475                 csa->priv2.spu_tag_status_query_RW);
1476        eieio();
1477}
1478
1479static inline void restore_mfc_csr_cmd(struct spu_state *csa, struct spu *spu)
1480{
1481        struct spu_priv2 __iomem *priv2 = spu->priv2;
1482
1483        /* Restore, Step 54:
1484         *     Restore the MFC_CSR_CMD1 and MFC_CSR_CMD2
1485         *     registers from CSA.
1486         */
1487        out_be64(&priv2->spu_cmd_buf1_RW, csa->priv2.spu_cmd_buf1_RW);
1488        out_be64(&priv2->spu_cmd_buf2_RW, csa->priv2.spu_cmd_buf2_RW);
1489        eieio();
1490}
1491
1492static inline void restore_mfc_csr_ato(struct spu_state *csa, struct spu *spu)
1493{
1494        struct spu_priv2 __iomem *priv2 = spu->priv2;
1495
1496        /* Restore, Step 55:
1497         *     Restore the MFC_CSR_ATO register from CSA.
1498         */
1499        out_be64(&priv2->spu_atomic_status_RW, csa->priv2.spu_atomic_status_RW);
1500}
1501
1502static inline void restore_mfc_tclass_id(struct spu_state *csa, struct spu *spu)
1503{
1504        /* Restore, Step 56:
1505         *     Restore the MFC_TCLASS_ID register from CSA.
1506         */
1507        spu_mfc_tclass_id_set(spu, csa->priv1.mfc_tclass_id_RW);
1508        eieio();
1509}
1510
1511static inline void set_llr_event(struct spu_state *csa, struct spu *spu)
1512{
1513        u64 ch0_cnt, ch0_data;
1514        u64 ch1_data;
1515
1516        /* Restore, Step 57:
1517         *    Set the Lock Line Reservation Lost Event by:
1518         *      1. OR CSA.SPU_Event_Status with bit 21 (Lr) set to 1.
1519         *      2. If CSA.SPU_Channel_0_Count=0 and
1520         *         CSA.SPU_Wr_Event_Mask[Lr]=1 and
1521         *         CSA.SPU_Event_Status[Lr]=0 then set
1522         *         CSA.SPU_Event_Status_Count=1.
1523         */
1524        ch0_cnt = csa->spu_chnlcnt_RW[0];
1525        ch0_data = csa->spu_chnldata_RW[0];
1526        ch1_data = csa->spu_chnldata_RW[1];
1527        csa->spu_chnldata_RW[0] |= MFC_LLR_LOST_EVENT;
1528        if ((ch0_cnt == 0) && !(ch0_data & MFC_LLR_LOST_EVENT) &&
1529            (ch1_data & MFC_LLR_LOST_EVENT)) {
1530                csa->spu_chnlcnt_RW[0] = 1;
1531        }
1532}
1533
1534static inline void restore_decr_wrapped(struct spu_state *csa, struct spu *spu)
1535{
1536        /* Restore, Step 58:
1537         *     If the status of the CSA software decrementer
1538         *     "wrapped" flag is set, OR in a '1' to
1539         *     CSA.SPU_Event_Status[Tm].
1540         */
1541        if (!(csa->lscsa->decr_status.slot[0] & SPU_DECR_STATUS_WRAPPED))
1542                return;
1543
1544        if ((csa->spu_chnlcnt_RW[0] == 0) &&
1545            (csa->spu_chnldata_RW[1] & 0x20) &&
1546            !(csa->spu_chnldata_RW[0] & 0x20))
1547                csa->spu_chnlcnt_RW[0] = 1;
1548
1549        csa->spu_chnldata_RW[0] |= 0x20;
1550}
1551
1552static inline void restore_ch_part1(struct spu_state *csa, struct spu *spu)
1553{
1554        struct spu_priv2 __iomem *priv2 = spu->priv2;
1555        u64 idx, ch_indices[] = { 0UL, 3UL, 4UL, 24UL, 25UL, 27UL };
1556        int i;
1557
1558        /* Restore, Step 59:
1559         *      Restore the following CH: [0,3,4,24,25,27]
1560         */
1561        for (i = 0; i < ARRAY_SIZE(ch_indices); i++) {
1562                idx = ch_indices[i];
1563                out_be64(&priv2->spu_chnlcntptr_RW, idx);
1564                eieio();
1565                out_be64(&priv2->spu_chnldata_RW, csa->spu_chnldata_RW[idx]);
1566                out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[idx]);
1567                eieio();
1568        }
1569}
1570
1571static inline void restore_ch_part2(struct spu_state *csa, struct spu *spu)
1572{
1573        struct spu_priv2 __iomem *priv2 = spu->priv2;
1574        u64 ch_indices[3] = { 9UL, 21UL, 23UL };
1575        u64 ch_counts[3] = { 1UL, 16UL, 1UL };
1576        u64 idx;
1577        int i;
1578
1579        /* Restore, Step 60:
1580         *     Restore the following CH: [9,21,23].
1581         */
1582        ch_counts[0] = 1UL;
1583        ch_counts[1] = csa->spu_chnlcnt_RW[21];
1584        ch_counts[2] = 1UL;
1585        for (i = 0; i < 3; i++) {
1586                idx = ch_indices[i];
1587                out_be64(&priv2->spu_chnlcntptr_RW, idx);
1588                eieio();
1589                out_be64(&priv2->spu_chnlcnt_RW, ch_counts[i]);
1590                eieio();
1591        }
1592}
1593
1594static inline void restore_spu_lslr(struct spu_state *csa, struct spu *spu)
1595{
1596        struct spu_priv2 __iomem *priv2 = spu->priv2;
1597
1598        /* Restore, Step 61:
1599         *     Restore the SPU_LSLR register from CSA.
1600         */
1601        out_be64(&priv2->spu_lslr_RW, csa->priv2.spu_lslr_RW);
1602        eieio();
1603}
1604
1605static inline void restore_spu_cfg(struct spu_state *csa, struct spu *spu)
1606{
1607        struct spu_priv2 __iomem *priv2 = spu->priv2;
1608
1609        /* Restore, Step 62:
1610         *     Restore the SPU_Cfg register from CSA.
1611         */
1612        out_be64(&priv2->spu_cfg_RW, csa->priv2.spu_cfg_RW);
1613        eieio();
1614}
1615
1616static inline void restore_pm_trace(struct spu_state *csa, struct spu *spu)
1617{
1618        /* Restore, Step 63:
1619         *     Restore PM_Trace_Tag_Wait_Mask from CSA.
1620         *     Not performed by this implementation.
1621         */
1622}
1623
1624static inline void restore_spu_npc(struct spu_state *csa, struct spu *spu)
1625{
1626        struct spu_problem __iomem *prob = spu->problem;
1627
1628        /* Restore, Step 64:
1629         *     Restore SPU_NPC from CSA.
1630         */
1631        out_be32(&prob->spu_npc_RW, csa->prob.spu_npc_RW);
1632        eieio();
1633}
1634
1635static inline void restore_spu_mb(struct spu_state *csa, struct spu *spu)
1636{
1637        struct spu_priv2 __iomem *priv2 = spu->priv2;
1638        int i;
1639
1640        /* Restore, Step 65:
1641         *     Restore MFC_RdSPU_MB from CSA.
1642         */
1643        out_be64(&priv2->spu_chnlcntptr_RW, 29UL);
1644        eieio();
1645        out_be64(&priv2->spu_chnlcnt_RW, csa->spu_chnlcnt_RW[29]);
1646        for (i = 0; i < 4; i++) {
1647                out_be64(&priv2->spu_chnldata_RW, csa->spu_mailbox_data[i]);
1648        }
1649        eieio();
1650}
1651
1652static inline void check_ppu_mb_stat(struct spu_state *csa, struct spu *spu)
1653{
1654        struct spu_problem __iomem *prob = spu->problem;
1655        u32 dummy = 0;
1656
1657        /* Restore, Step 66:
1658         *     If CSA.MB_Stat[P]=0 (mailbox empty) then
1659         *     read from the PPU_MB register.
1660         */
1661        if ((csa->prob.mb_stat_R & 0xFF) == 0) {
1662                dummy = in_be32(&prob->pu_mb_R);
1663                eieio();
1664        }
1665}
1666
1667static inline void check_ppuint_mb_stat(struct spu_state *csa, struct spu *spu)
1668{
1669        struct spu_priv2 __iomem *priv2 = spu->priv2;
1670        u64 dummy = 0UL;
1671
1672        /* Restore, Step 66:
1673         *     If CSA.MB_Stat[I]=0 (mailbox empty) then
1674         *     read from the PPUINT_MB register.
1675         */
1676        if ((csa->prob.mb_stat_R & 0xFF0000) == 0) {
1677                dummy = in_be64(&priv2->puint_mb_R);
1678                eieio();
1679                spu_int_stat_clear(spu, 2, CLASS2_ENABLE_MAILBOX_INTR);
1680                eieio();
1681        }
1682}
1683
1684static inline void restore_mfc_sr1(struct spu_state *csa, struct spu *spu)
1685{
1686        /* Restore, Step 69:
1687         *     Restore the MFC_SR1 register from CSA.
1688         */
1689        spu_mfc_sr1_set(spu, csa->priv1.mfc_sr1_RW);
1690        eieio();
1691}
1692
1693static inline void restore_other_spu_access(struct spu_state *csa,
1694                                            struct spu *spu)
1695{
1696        /* Restore, Step 70:
1697         *     Restore other SPU mappings to this SPU. TBD.
1698         */
1699}
1700
1701static inline void restore_spu_runcntl(struct spu_state *csa, struct spu *spu)
1702{
1703        struct spu_problem __iomem *prob = spu->problem;
1704
1705        /* Restore, Step 71:
1706         *     If CSA.SPU_Status[R]=1 then write
1707         *     SPU_RunCntl[R0R1]='01'.
1708         */
1709        if (csa->prob.spu_status_R & SPU_STATUS_RUNNING) {
1710                out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_RUNNABLE);
1711                eieio();
1712        }
1713}
1714
1715static inline void restore_mfc_cntl(struct spu_state *csa, struct spu *spu)
1716{
1717        struct spu_priv2 __iomem *priv2 = spu->priv2;
1718
1719        /* Restore, Step 72:
1720         *    Restore the MFC_CNTL register for the CSA.
1721         */
1722        out_be64(&priv2->mfc_control_RW, csa->priv2.mfc_control_RW);
1723        eieio();
1724        /*
1725         * FIXME: this is to restart a DMA that we were processing
1726         *        before the save. better remember the fault information
1727         *        in the csa instead.
1728         */
1729        if ((csa->priv2.mfc_control_RW & MFC_CNTL_SUSPEND_DMA_QUEUE_MASK)) {
1730                out_be64(&priv2->mfc_control_RW, MFC_CNTL_RESTART_DMA_COMMAND);
1731                eieio();
1732        }
1733}
1734
1735static inline void enable_user_access(struct spu_state *csa, struct spu *spu)
1736{
1737        /* Restore, Step 73:
1738         *     Enable user-space access (if provided) to this
1739         *     SPU by mapping the virtual pages assigned to
1740         *     the SPU memory-mapped I/O (MMIO) for problem
1741         *     state. TBD.
1742         */
1743}
1744
1745static inline void reset_switch_active(struct spu_state *csa, struct spu *spu)
1746{
1747        /* Restore, Step 74:
1748         *     Reset the "context switch active" flag.
1749         *     Not performed by this implementation.
1750         */
1751}
1752
1753static inline void reenable_interrupts(struct spu_state *csa, struct spu *spu)
1754{
1755        /* Restore, Step 75:
1756         *     Re-enable SPU interrupts.
1757         */
1758        spin_lock_irq(&spu->register_lock);
1759        spu_int_mask_set(spu, 0, csa->priv1.int_mask_class0_RW);
1760        spu_int_mask_set(spu, 1, csa->priv1.int_mask_class1_RW);
1761        spu_int_mask_set(spu, 2, csa->priv1.int_mask_class2_RW);
1762        spin_unlock_irq(&spu->register_lock);
1763}
1764
1765static int quiece_spu(struct spu_state *prev, struct spu *spu)
1766{
1767        /*
1768         * Combined steps 2-18 of SPU context save sequence, which
1769         * quiesce the SPU state (disable SPU execution, MFC command
1770         * queues, decrementer, SPU interrupts, etc.).
1771         *
1772         * Returns      0 on success.
1773         *              2 if failed step 2.
1774         *              6 if failed step 6.
1775         */
1776
1777        if (check_spu_isolate(prev, spu)) {     /* Step 2. */
1778                return 2;
1779        }
1780        disable_interrupts(prev, spu);          /* Step 3. */
1781        set_watchdog_timer(prev, spu);          /* Step 4. */
1782        inhibit_user_access(prev, spu);         /* Step 5. */
1783        if (check_spu_isolate(prev, spu)) {     /* Step 6. */
1784                return 6;
1785        }
1786        set_switch_pending(prev, spu);          /* Step 7. */
1787        save_mfc_cntl(prev, spu);               /* Step 8. */
1788        save_spu_runcntl(prev, spu);            /* Step 9. */
1789        save_mfc_sr1(prev, spu);                /* Step 10. */
1790        save_spu_status(prev, spu);             /* Step 11. */
1791        save_mfc_decr(prev, spu);               /* Step 12. */
1792        halt_mfc_decr(prev, spu);               /* Step 13. */
1793        save_timebase(prev, spu);               /* Step 14. */
1794        remove_other_spu_access(prev, spu);     /* Step 15. */
1795        do_mfc_mssync(prev, spu);               /* Step 16. */
1796        issue_mfc_tlbie(prev, spu);             /* Step 17. */
1797        handle_pending_interrupts(prev, spu);   /* Step 18. */
1798
1799        return 0;
1800}
1801
1802static void save_csa(struct spu_state *prev, struct spu *spu)
1803{
1804        /*
1805         * Combine steps 19-44 of SPU context save sequence, which
1806         * save regions of the privileged & problem state areas.
1807         */
1808
1809        save_mfc_queues(prev, spu);     /* Step 19. */
1810        save_ppu_querymask(prev, spu);  /* Step 20. */
1811        save_ppu_querytype(prev, spu);  /* Step 21. */
1812        save_ppu_tagstatus(prev, spu);  /* NEW.     */
1813        save_mfc_csr_tsq(prev, spu);    /* Step 22. */
1814        save_mfc_csr_cmd(prev, spu);    /* Step 23. */
1815        save_mfc_csr_ato(prev, spu);    /* Step 24. */
1816        save_mfc_tclass_id(prev, spu);  /* Step 25. */
1817        set_mfc_tclass_id(prev, spu);   /* Step 26. */
1818        purge_mfc_queue(prev, spu);     /* Step 27. */
1819        wait_purge_complete(prev, spu); /* Step 28. */
1820        setup_mfc_sr1(prev, spu);       /* Step 30. */
1821        save_spu_npc(prev, spu);        /* Step 31. */
1822        save_spu_privcntl(prev, spu);   /* Step 32. */
1823        reset_spu_privcntl(prev, spu);  /* Step 33. */
1824        save_spu_lslr(prev, spu);       /* Step 34. */
1825        reset_spu_lslr(prev, spu);      /* Step 35. */
1826        save_spu_cfg(prev, spu);        /* Step 36. */
1827        save_pm_trace(prev, spu);       /* Step 37. */
1828        save_mfc_rag(prev, spu);        /* Step 38. */
1829        save_ppu_mb_stat(prev, spu);    /* Step 39. */
1830        save_ppu_mb(prev, spu);         /* Step 40. */
1831        save_ppuint_mb(prev, spu);      /* Step 41. */
1832        save_ch_part1(prev, spu);       /* Step 42. */
1833        save_spu_mb(prev, spu);         /* Step 43. */
1834        save_mfc_cmd(prev, spu);        /* Step 44. */
1835        reset_ch(prev, spu);            /* Step 45. */
1836}
1837
1838static void save_lscsa(struct spu_state *prev, struct spu *spu)
1839{
1840        /*
1841         * Perform steps 46-57 of SPU context save sequence,
1842         * which save regions of the local store and register
1843         * file.
1844         */
1845
1846        resume_mfc_queue(prev, spu);    /* Step 46. */
1847        /* Step 47. */
1848        setup_mfc_slbs(prev, spu, spu_save_code, sizeof(spu_save_code));
1849        set_switch_active(prev, spu);   /* Step 48. */
1850        enable_interrupts(prev, spu);   /* Step 49. */
1851        save_ls_16kb(prev, spu);        /* Step 50. */
1852        set_spu_npc(prev, spu);         /* Step 51. */
1853        set_signot1(prev, spu);         /* Step 52. */
1854        set_signot2(prev, spu);         /* Step 53. */
1855        send_save_code(prev, spu);      /* Step 54. */
1856        set_ppu_querymask(prev, spu);   /* Step 55. */
1857        wait_tag_complete(prev, spu);   /* Step 56. */
1858        wait_spu_stopped(prev, spu);    /* Step 57. */
1859}
1860
1861static void force_spu_isolate_exit(struct spu *spu)
1862{
1863        struct spu_problem __iomem *prob = spu->problem;
1864        struct spu_priv2 __iomem *priv2 = spu->priv2;
1865
1866        /* Stop SPE execution and wait for completion. */
1867        out_be32(&prob->spu_runcntl_RW, SPU_RUNCNTL_STOP);
1868        iobarrier_rw();
1869        POLL_WHILE_TRUE(in_be32(&prob->spu_status_R) & SPU_STATUS_RUNNING);
1870
1871        /* Restart SPE master runcntl. */
1872        spu_mfc_sr1_set(spu, MFC_STATE1_MASTER_RUN_CONTROL_MASK);
1873        iobarrier_w();
1874
1875        /* Initiate isolate exit request and wait for completion. */
1876        out_be64(&priv2->spu_privcntl_RW, 4LL);
1877        iobarrier_w();
1878        out_be32(&prob->spu_runcntl_RW, 2);
1879        iobarrier_rw();
1880        POLL_WHILE_FALSE((in_be32(&prob->spu_status_R)
1881                                & SPU_STATUS_STOPPED_BY_STOP));
1882
1883        /* Reset load request to normal. */
1884        out_be64(&priv2->spu_privcntl_RW, SPU_PRIVCNT_LOAD_REQUEST_NORMAL);
1885        iobarrier_w();
1886}
1887
1888/**
1889 * stop_spu_isolate
1890 *      Check SPU run-control state and force isolated
1891 *      exit function as necessary.
1892 */
1893static void stop_spu_isolate(struct spu *spu)
1894{
1895        struct spu_problem __iomem *prob = spu->problem;
1896
1897        if (in_be32(&prob->spu_status_R) & SPU_STATUS_ISOLATED_STATE) {
1898                /* The SPU is in isolated state; the only way
1899                 * to get it out is to perform an isolated
1900                 * exit (clean) operation.
1901                 */
1902                force_spu_isolate_exit(spu);
1903        }
1904}
1905
1906static void harvest(struct spu_state *prev, struct spu *spu)
1907{
1908        /*
1909         * Perform steps 2-25 of SPU context restore sequence,
1910         * which resets an SPU either after a failed save, or
1911         * when using SPU for first time.
1912         */
1913
1914        disable_interrupts(prev, spu);          /* Step 2.  */
1915        inhibit_user_access(prev, spu);         /* Step 3.  */
1916        terminate_spu_app(prev, spu);           /* Step 4.  */
1917        set_switch_pending(prev, spu);          /* Step 5.  */
1918        stop_spu_isolate(spu);                  /* NEW.     */
1919        remove_other_spu_access(prev, spu);     /* Step 6.  */
1920        suspend_mfc_and_halt_decr(prev, spu);   /* Step 7.  */
1921        wait_suspend_mfc_complete(prev, spu);   /* Step 8.  */
1922        if (!suspend_spe(prev, spu))            /* Step 9.  */
1923                clear_spu_status(prev, spu);    /* Step 10. */
1924        do_mfc_mssync(prev, spu);               /* Step 11. */
1925        issue_mfc_tlbie(prev, spu);             /* Step 12. */
1926        handle_pending_interrupts(prev, spu);   /* Step 13. */
1927        purge_mfc_queue(prev, spu);             /* Step 14. */
1928        wait_purge_complete(prev, spu);         /* Step 15. */
1929        reset_spu_privcntl(prev, spu);          /* Step 16. */
1930        reset_spu_lslr(prev, spu);              /* Step 17. */
1931        setup_mfc_sr1(prev, spu);               /* Step 18. */
1932        spu_invalidate_slbs(spu);               /* Step 19. */
1933        reset_ch_part1(prev, spu);              /* Step 20. */
1934        reset_ch_part2(prev, spu);              /* Step 21. */
1935        enable_interrupts(prev, spu);           /* Step 22. */
1936        set_switch_active(prev, spu);           /* Step 23. */
1937        set_mfc_tclass_id(prev, spu);           /* Step 24. */
1938        resume_mfc_queue(prev, spu);            /* Step 25. */
1939}
1940
1941static void restore_lscsa(struct spu_state *next, struct spu *spu)
1942{
1943        /*
1944         * Perform steps 26-40 of SPU context restore sequence,
1945         * which restores regions of the local store and register
1946         * file.
1947         */
1948
1949        set_watchdog_timer(next, spu);          /* Step 26. */
1950        setup_spu_status_part1(next, spu);      /* Step 27. */
1951        setup_spu_status_part2(next, spu);      /* Step 28. */
1952        restore_mfc_rag(next, spu);             /* Step 29. */
1953        /* Step 30. */
1954        setup_mfc_slbs(next, spu, spu_restore_code, sizeof(spu_restore_code));
1955        set_spu_npc(next, spu);                 /* Step 31. */
1956        set_signot1(next, spu);                 /* Step 32. */
1957        set_signot2(next, spu);                 /* Step 33. */
1958        setup_decr(next, spu);                  /* Step 34. */
1959        setup_ppu_mb(next, spu);                /* Step 35. */
1960        setup_ppuint_mb(next, spu);             /* Step 36. */
1961        send_restore_code(next, spu);           /* Step 37. */
1962        set_ppu_querymask(next, spu);           /* Step 38. */
1963        wait_tag_complete(next, spu);           /* Step 39. */
1964        wait_spu_stopped(next, spu);            /* Step 40. */
1965}
1966
1967static void restore_csa(struct spu_state *next, struct spu *spu)
1968{
1969        /*
1970         * Combine steps 41-76 of SPU context restore sequence, which
1971         * restore regions of the privileged & problem state areas.
1972         */
1973
1974        restore_spu_privcntl(next, spu);        /* Step 41. */
1975        restore_status_part1(next, spu);        /* Step 42. */
1976        restore_status_part2(next, spu);        /* Step 43. */
1977        restore_ls_16kb(next, spu);             /* Step 44. */
1978        wait_tag_complete(next, spu);           /* Step 45. */
1979        suspend_mfc(next, spu);                 /* Step 46. */
1980        wait_suspend_mfc_complete(next, spu);   /* Step 47. */
1981        issue_mfc_tlbie(next, spu);             /* Step 48. */
1982        clear_interrupts(next, spu);            /* Step 49. */
1983        restore_mfc_queues(next, spu);          /* Step 50. */
1984        restore_ppu_querymask(next, spu);       /* Step 51. */
1985        restore_ppu_querytype(next, spu);       /* Step 52. */
1986        restore_mfc_csr_tsq(next, spu);         /* Step 53. */
1987        restore_mfc_csr_cmd(next, spu);         /* Step 54. */
1988        restore_mfc_csr_ato(next, spu);         /* Step 55. */
1989        restore_mfc_tclass_id(next, spu);       /* Step 56. */
1990        set_llr_event(next, spu);               /* Step 57. */
1991        restore_decr_wrapped(next, spu);        /* Step 58. */
1992        restore_ch_part1(next, spu);            /* Step 59. */
1993        restore_ch_part2(next, spu);            /* Step 60. */
1994        restore_spu_lslr(next, spu);            /* Step 61. */
1995        restore_spu_cfg(next, spu);             /* Step 62. */
1996        restore_pm_trace(next, spu);            /* Step 63. */
1997        restore_spu_npc(next, spu);             /* Step 64. */
1998        restore_spu_mb(next, spu);              /* Step 65. */
1999        check_ppu_mb_stat(next, spu);           /* Step 66. */
2000        check_ppuint_mb_stat(next, spu);        /* Step 67. */
2001        spu_invalidate_slbs(spu);               /* Modified Step 68. */
2002        restore_mfc_sr1(next, spu);             /* Step 69. */
2003        restore_other_spu_access(next, spu);    /* Step 70. */
2004        restore_spu_runcntl(next, spu);         /* Step 71. */
2005        restore_mfc_cntl(next, spu);            /* Step 72. */
2006        enable_user_access(next, spu);          /* Step 73. */
2007        reset_switch_active(next, spu);         /* Step 74. */
2008        reenable_interrupts(next, spu);         /* Step 75. */
2009}
2010
2011static int __do_spu_save(struct spu_state *prev, struct spu *spu)
2012{
2013        int rc;
2014
2015        /*
2016         * SPU context save can be broken into three phases:
2017         *
2018         *     (a) quiesce [steps 2-16].
2019         *     (b) save of CSA, performed by PPE [steps 17-42]
2020         *     (c) save of LSCSA, mostly performed by SPU [steps 43-52].
2021         *
2022         * Returns      0 on success.
2023         *              2,6 if failed to quiece SPU
2024         *              53 if SPU-side of save failed.
2025         */
2026
2027        rc = quiece_spu(prev, spu);             /* Steps 2-16. */
2028        switch (rc) {
2029        default:
2030        case 2:
2031        case 6:
2032                harvest(prev, spu);
2033                return rc;
2034                break;
2035        case 0:
2036                break;
2037        }
2038        save_csa(prev, spu);                    /* Steps 17-43. */
2039        save_lscsa(prev, spu);                  /* Steps 44-53. */
2040        return check_save_status(prev, spu);    /* Step 54.     */
2041}
2042
2043static int __do_spu_restore(struct spu_state *next, struct spu *spu)
2044{
2045        int rc;
2046
2047        /*
2048         * SPU context restore can be broken into three phases:
2049         *
2050         *    (a) harvest (or reset) SPU [steps 2-24].
2051         *    (b) restore LSCSA [steps 25-40], mostly performed by SPU.
2052         *    (c) restore CSA [steps 41-76], performed by PPE.
2053         *
2054         * The 'harvest' step is not performed here, but rather
2055         * as needed below.
2056         */
2057
2058        restore_lscsa(next, spu);               /* Steps 24-39. */
2059        rc = check_restore_status(next, spu);   /* Step 40.     */
2060        switch (rc) {
2061        default:
2062                /* Failed. Return now. */
2063                return rc;
2064                break;
2065        case 0:
2066                /* Fall through to next step. */
2067                break;
2068        }
2069        restore_csa(next, spu);
2070
2071        return 0;
2072}
2073
2074/**
2075 * spu_save - SPU context save, with locking.
2076 * @prev: pointer to SPU context save area, to be saved.
2077 * @spu: pointer to SPU iomem structure.
2078 *
2079 * Acquire locks, perform the save operation then return.
2080 */
2081int spu_save(struct spu_state *prev, struct spu *spu)
2082{
2083        int rc;
2084
2085        acquire_spu_lock(spu);          /* Step 1.     */
2086        rc = __do_spu_save(prev, spu);  /* Steps 2-53. */
2087        release_spu_lock(spu);
2088        if (rc != 0 && rc != 2 && rc != 6) {
2089                panic("%s failed on SPU[%d], rc=%d.\n",
2090                      __func__, spu->number, rc);
2091        }
2092        return 0;
2093}
2094EXPORT_SYMBOL_GPL(spu_save);
2095
2096/**
2097 * spu_restore - SPU context restore, with harvest and locking.
2098 * @new: pointer to SPU context save area, to be restored.
2099 * @spu: pointer to SPU iomem structure.
2100 *
2101 * Perform harvest + restore, as we may not be coming
2102 * from a previous successful save operation, and the
2103 * hardware state is unknown.
2104 */
2105int spu_restore(struct spu_state *new, struct spu *spu)
2106{
2107        int rc;
2108
2109        acquire_spu_lock(spu);
2110        harvest(NULL, spu);
2111        spu->slb_replace = 0;
2112        rc = __do_spu_restore(new, spu);
2113        release_spu_lock(spu);
2114        if (rc) {
2115                panic("%s failed on SPU[%d] rc=%d.\n",
2116                       __func__, spu->number, rc);
2117        }
2118        return rc;
2119}
2120EXPORT_SYMBOL_GPL(spu_restore);
2121
2122static void init_prob(struct spu_state *csa)
2123{
2124        csa->spu_chnlcnt_RW[9] = 1;
2125        csa->spu_chnlcnt_RW[21] = 16;
2126        csa->spu_chnlcnt_RW[23] = 1;
2127        csa->spu_chnlcnt_RW[28] = 1;
2128        csa->spu_chnlcnt_RW[30] = 1;
2129        csa->prob.spu_runcntl_RW = SPU_RUNCNTL_STOP;
2130        csa->prob.mb_stat_R = 0x000400;
2131}
2132
2133static void init_priv1(struct spu_state *csa)
2134{
2135        /* Enable decode, relocate, tlbie response, master runcntl. */
2136        csa->priv1.mfc_sr1_RW = MFC_STATE1_LOCAL_STORAGE_DECODE_MASK |
2137            MFC_STATE1_MASTER_RUN_CONTROL_MASK |
2138            MFC_STATE1_PROBLEM_STATE_MASK |
2139            MFC_STATE1_RELOCATE_MASK | MFC_STATE1_BUS_TLBIE_MASK;
2140
2141        /* Enable OS-specific set of interrupts. */
2142        csa->priv1.int_mask_class0_RW = CLASS0_ENABLE_DMA_ALIGNMENT_INTR |
2143            CLASS0_ENABLE_INVALID_DMA_COMMAND_INTR |
2144            CLASS0_ENABLE_SPU_ERROR_INTR;
2145        csa->priv1.int_mask_class1_RW = CLASS1_ENABLE_SEGMENT_FAULT_INTR |
2146            CLASS1_ENABLE_STORAGE_FAULT_INTR;
2147        csa->priv1.int_mask_class2_RW = CLASS2_ENABLE_SPU_STOP_INTR |
2148            CLASS2_ENABLE_SPU_HALT_INTR |
2149            CLASS2_ENABLE_SPU_DMA_TAG_GROUP_COMPLETE_INTR;
2150}
2151
2152static void init_priv2(struct spu_state *csa)
2153{
2154        csa->priv2.spu_lslr_RW = LS_ADDR_MASK;
2155        csa->priv2.mfc_control_RW = MFC_CNTL_RESUME_DMA_QUEUE |
2156            MFC_CNTL_NORMAL_DMA_QUEUE_OPERATION |
2157            MFC_CNTL_DMA_QUEUES_EMPTY_MASK;
2158}
2159
2160/**
2161 * spu_alloc_csa - allocate and initialize an SPU context save area.
2162 *
2163 * Allocate and initialize the contents of an SPU context save area.
2164 * This includes enabling address translation, interrupt masks, etc.,
2165 * as appropriate for the given OS environment.
2166 *
2167 * Note that storage for the 'lscsa' is allocated separately,
2168 * as it is by far the largest of the context save regions,
2169 * and may need to be pinned or otherwise specially aligned.
2170 */
2171int spu_init_csa(struct spu_state *csa)
2172{
2173        int rc;
2174
2175        if (!csa)
2176                return -EINVAL;
2177        memset(csa, 0, sizeof(struct spu_state));
2178
2179        rc = spu_alloc_lscsa(csa);
2180        if (rc)
2181                return rc;
2182
2183        spin_lock_init(&csa->register_lock);
2184
2185        init_prob(csa);
2186        init_priv1(csa);
2187        init_priv2(csa);
2188
2189        return 0;
2190}
2191
2192void spu_fini_csa(struct spu_state *csa)
2193{
2194        spu_free_lscsa(csa);
2195}
2196
lxr.linux.no kindly hosted by Redpill Linpro AS, provider of Linux consulting and operations services since 1995.