powerpc/kvm: Fix build errors with older toolchains
[linux-2.6.git] / arch / powerpc / kvm / book3s_hv_rmhandlers.S
index 9af2648..de29501 100644 (file)
@@ -30,8 +30,6 @@
  *                                                                           *
  ****************************************************************************/
 
-#define SHADOW_VCPU_OFF                PACA_KVM_SVCPU
-
        .globl  kvmppc_skip_interrupt
 kvmppc_skip_interrupt:
        mfspr   r13,SPRN_SRR0
@@ -79,6 +77,32 @@ _GLOBAL(kvmppc_hv_entry_trampoline)
  *                                                                            *
  *****************************************************************************/
 
+#define XICS_XIRR              4
+#define XICS_QIRR              0xc
+
+/*
+ * We come in here when wakened from nap mode on a secondary hw thread.
+ * Relocation is off and most register values are lost.
+ * r13 points to the PACA.
+ */
+       .globl  kvm_start_guest
+kvm_start_guest:
+       ld      r1,PACAEMERGSP(r13)
+       subi    r1,r1,STACK_FRAME_OVERHEAD
+
+       /* get vcpu pointer */
+       ld      r4, HSTATE_KVM_VCPU(r13)
+
+       /* We got here with an IPI; clear it */
+       ld      r5, HSTATE_XICS_PHYS(r13)
+       li      r0, 0xff
+       li      r6, XICS_QIRR
+       li      r7, XICS_XIRR
+       lwzcix  r8, r5, r7              /* ack the interrupt */
+       sync
+       stbcix  r0, r5, r6              /* clear it */
+       stwcix  r8, r5, r7              /* EOI it */
+
 .global kvmppc_hv_entry
 kvmppc_hv_entry:
 
@@ -124,12 +148,20 @@ kvmppc_hv_entry:
        lwz     r7, VCPU_PMC + 12(r4)
        lwz     r8, VCPU_PMC + 16(r4)
        lwz     r9, VCPU_PMC + 20(r4)
+BEGIN_FTR_SECTION
+       lwz     r10, VCPU_PMC + 24(r4)
+       lwz     r11, VCPU_PMC + 28(r4)
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
        mtspr   SPRN_PMC1, r3
        mtspr   SPRN_PMC2, r5
        mtspr   SPRN_PMC3, r6
        mtspr   SPRN_PMC4, r7
        mtspr   SPRN_PMC5, r8
        mtspr   SPRN_PMC6, r9
+BEGIN_FTR_SECTION
+       mtspr   SPRN_PMC7, r10
+       mtspr   SPRN_PMC8, r11
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
        ld      r3, VCPU_MMCR(r4)
        ld      r5, VCPU_MMCR + 8(r4)
        ld      r6, VCPU_MMCR + 16(r4)
@@ -141,9 +173,11 @@ kvmppc_hv_entry:
        /* Load up FP, VMX and VSX registers */
        bl      kvmppc_load_fp
 
+BEGIN_FTR_SECTION
        /* Switch DSCR to guest value */
        ld      r5, VCPU_DSCR(r4)
        mtspr   SPRN_DSCR, r5
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
 
        /*
         * Set the decrementer to the guest decrementer.
@@ -166,6 +200,14 @@ kvmppc_hv_entry:
        /* Save R1 in the PACA */
        std     r1, HSTATE_HOST_R1(r13)
 
+       /* Increment yield count if they have a VPA */
+       ld      r3, VCPU_VPA(r4)
+       cmpdi   r3, 0
+       beq     25f
+       lwz     r5, LPPACA_YIELDCOUNT(r3)
+       addi    r5, r5, 1
+       stw     r5, LPPACA_YIELDCOUNT(r3)
+25:
        /* Load up DAR and DSISR */
        ld      r5, VCPU_DAR(r4)
        lwz     r6, VCPU_DSISR(r4)
@@ -178,6 +220,7 @@ kvmppc_hv_entry:
        mtspr   SPRN_DABRX,r5
        mtspr   SPRN_DABR,r6
 
+BEGIN_FTR_SECTION
        /* Restore AMR and UAMOR, set AMOR to all 1s */
        ld      r5,VCPU_AMR(r4)
        ld      r6,VCPU_UAMOR(r4)
@@ -185,6 +228,7 @@ kvmppc_hv_entry:
        mtspr   SPRN_AMR,r5
        mtspr   SPRN_UAMOR,r6
        mtspr   SPRN_AMOR,r7
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
 
        /* Clear out SLB */
        li      r6,0
@@ -192,8 +236,29 @@ kvmppc_hv_entry:
        slbia
        ptesync
 
-       /* Switch to guest partition. */
+BEGIN_FTR_SECTION
+       b       30f
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
+       /*
+        * POWER7 host -> guest partition switch code.
+        * We don't have to lock against concurrent tlbies,
+        * but we do have to coordinate across hardware threads.
+        */
+       /* Increment entry count iff exit count is zero. */
+       ld      r5,HSTATE_KVM_VCORE(r13)
+       addi    r9,r5,VCORE_ENTRY_EXIT
+21:    lwarx   r3,0,r9
+       cmpwi   r3,0x100                /* any threads starting to exit? */
+       bge     secondary_too_late      /* if so we're too late to the party */
+       addi    r3,r3,1
+       stwcx.  r3,0,r9
+       bne     21b
+
+       /* Primary thread switches to guest partition. */
        ld      r9,VCPU_KVM(r4)         /* pointer to struct kvm */
+       lwz     r6,VCPU_PTID(r4)
+       cmpwi   r6,0
+       bne     20f
        ld      r6,KVM_SDR1(r9)
        lwz     r7,KVM_LPID(r9)
        li      r0,LPID_RSVD            /* switch to reserved LPID */
@@ -202,8 +267,26 @@ kvmppc_hv_entry:
        mtspr   SPRN_SDR1,r6            /* switch to partition page table */
        mtspr   SPRN_LPID,r7
        isync
-       ld      r8,VCPU_LPCR(r4)
-       mtspr   SPRN_LPCR,r8
+       li      r0,1
+       stb     r0,VCORE_IN_GUEST(r5)   /* signal secondaries to continue */
+       b       10f
+
+       /* Secondary threads wait for primary to have done partition switch */
+20:    lbz     r0,VCORE_IN_GUEST(r5)
+       cmpwi   r0,0
+       beq     20b
+
+       /* Set LPCR.  Set the MER bit if there is a pending external irq. */
+10:    ld      r8,KVM_LPCR(r9)
+       ld      r0,VCPU_PENDING_EXC(r4)
+       li      r7,(1 << BOOK3S_IRQPRIO_EXTERNAL)
+       oris    r7,r7,(1 << BOOK3S_IRQPRIO_EXTERNAL_LEVEL)@h
+       and.    r0,r0,r7
+       beq     11f
+       ori     r8,r8,LPCR_MER
+11:    mtspr   SPRN_LPCR,r8
+       ld      r8,KVM_RMOR(r9)
+       mtspr   SPRN_RMOR,r8
        isync
 
        /* Check if HDEC expires soon */
@@ -217,10 +300,12 @@ kvmppc_hv_entry:
         * Invalidate the TLB if we could possibly have stale TLB
         * entries for this partition on this core due to the use
         * of tlbiel.
+        * XXX maybe only need this on primary thread?
         */
        ld      r9,VCPU_KVM(r4)         /* pointer to struct kvm */
        lwz     r5,VCPU_VCPUID(r4)
        lhz     r6,PACAPACAINDEX(r13)
+       rldimi  r6,r5,0,62              /* XXX map as if threads 1:1 p:v */
        lhz     r8,VCPU_LAST_CPU(r4)
        sldi    r7,r6,1                 /* see if this is the same vcpu */
        add     r7,r7,r9                /* as last ran on this pcpu */
@@ -250,9 +335,94 @@ kvmppc_hv_entry:
        ld      r8,VCPU_SPURR(r4)
        mtspr   SPRN_PURR,r7
        mtspr   SPRN_SPURR,r8
+       b       31f
+
+       /*
+        * PPC970 host -> guest partition switch code.
+        * We have to lock against concurrent tlbies,
+        * using native_tlbie_lock to lock against host tlbies
+        * and kvm->arch.tlbie_lock to lock against guest tlbies.
+        * We also have to invalidate the TLB since its
+        * entries aren't tagged with the LPID.
+        */
+30:    ld      r9,VCPU_KVM(r4)         /* pointer to struct kvm */
+
+       /* first take native_tlbie_lock */
+       .section ".toc","aw"
+toc_tlbie_lock:
+       .tc     native_tlbie_lock[TC],native_tlbie_lock
+       .previous
+       ld      r3,toc_tlbie_lock@toc(2)
+       lwz     r8,PACA_LOCK_TOKEN(r13)
+24:    lwarx   r0,0,r3
+       cmpwi   r0,0
+       bne     24b
+       stwcx.  r8,0,r3
+       bne     24b
+       isync
+
+       ld      r7,KVM_LPCR(r9)         /* use kvm->arch.lpcr to store HID4 */
+       li      r0,0x18f
+       rotldi  r0,r0,HID4_LPID5_SH     /* all lpid bits in HID4 = 1 */
+       or      r0,r7,r0
+       ptesync
+       sync
+       mtspr   SPRN_HID4,r0            /* switch to reserved LPID */
+       isync
+       li      r0,0
+       stw     r0,0(r3)                /* drop native_tlbie_lock */
+
+       /* invalidate the whole TLB */
+       li      r0,256
+       mtctr   r0
+       li      r6,0
+25:    tlbiel  r6
+       addi    r6,r6,0x1000
+       bdnz    25b
+       ptesync
+
+       /* Take the guest's tlbie_lock */
+       addi    r3,r9,KVM_TLBIE_LOCK
+24:    lwarx   r0,0,r3
+       cmpwi   r0,0
+       bne     24b
+       stwcx.  r8,0,r3
+       bne     24b
+       isync
+       ld      r6,KVM_SDR1(r9)
+       mtspr   SPRN_SDR1,r6            /* switch to partition page table */
+
+       /* Set up HID4 with the guest's LPID etc. */
+       sync
+       mtspr   SPRN_HID4,r7
+       isync
+
+       /* drop the guest's tlbie_lock */
+       li      r0,0
+       stw     r0,0(r3)
+
+       /* Check if HDEC expires soon */
+       mfspr   r3,SPRN_HDEC
+       cmpwi   r3,10
+       li      r12,BOOK3S_INTERRUPT_HV_DECREMENTER
+       mr      r9,r4
+       blt     hdec_soon
+
+       /* Enable HDEC interrupts */
+       mfspr   r0,SPRN_HID0
+       li      r3,1
+       rldimi  r0,r3, HID0_HDICE_SH, 64-HID0_HDICE_SH-1
+       sync
+       mtspr   SPRN_HID0,r0
+       mfspr   r0,SPRN_HID0
+       mfspr   r0,SPRN_HID0
+       mfspr   r0,SPRN_HID0
+       mfspr   r0,SPRN_HID0
+       mfspr   r0,SPRN_HID0
+       mfspr   r0,SPRN_HID0
 
        /* Load up guest SLB entries */
-       lwz     r5,VCPU_SLB_MAX(r4)
+31:    lwz     r5,VCPU_SLB_MAX(r4)
        cmpwi   r5,0
        beq     9f
        mtctr   r5
@@ -401,16 +571,23 @@ kvmppc_interrupt:
        cmpwi   r3,0
        bge     ignore_hdec
 2:
+       /* See if this is something we can handle in real mode */
+       cmpwi   r12,BOOK3S_INTERRUPT_SYSCALL
+       beq     hcall_try_real_mode
+hcall_real_cont:
 
        /* Check for mediated interrupts (could be done earlier really ...) */
+BEGIN_FTR_SECTION
        cmpwi   r12,BOOK3S_INTERRUPT_EXTERNAL
        bne+    1f
-       ld      r5,VCPU_LPCR(r9)
+       ld      r5,VCPU_KVM(r9)
+       ld      r5,KVM_LPCR(r5)
        andi.   r0,r11,MSR_EE
        beq     1f
        andi.   r0,r5,LPCR_MER
        bne     bounce_ext_interrupt
 1:
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
 
        /* Save DEC */
        mfspr   r5,SPRN_DEC
@@ -422,9 +599,11 @@ kvmppc_interrupt:
        /* Save HEIR (HV emulation assist reg) in last_inst
           if this is an HEI (HV emulation interrupt, e40) */
        li      r3,-1
+BEGIN_FTR_SECTION
        cmpwi   r12,BOOK3S_INTERRUPT_H_EMUL_ASSIST
        bne     11f
        mfspr   r3,SPRN_HEIR
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
 11:    stw     r3,VCPU_LAST_INST(r9)
 
        /* Save more register state  */
@@ -438,8 +617,10 @@ kvmppc_interrupt:
        stw     r7, VCPU_DSISR(r9)
        std     r8, VCPU_CTR(r9)
        /* grab HDAR & HDSISR if HV data storage interrupt (HDSI) */
+BEGIN_FTR_SECTION
        cmpwi   r12,BOOK3S_INTERRUPT_H_DATA_STORAGE
        beq     6f
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
 7:     std     r6, VCPU_FAULT_DAR(r9)
        stw     r7, VCPU_FAULT_DSISR(r9)
 
@@ -473,6 +654,7 @@ kvmppc_interrupt:
        /*
         * Save the guest PURR/SPURR
         */
+BEGIN_FTR_SECTION
        mfspr   r5,SPRN_PURR
        mfspr   r6,SPRN_SPURR
        ld      r7,VCPU_PURR(r9)
@@ -492,6 +674,7 @@ kvmppc_interrupt:
        add     r4,r4,r6
        mtspr   SPRN_PURR,r3
        mtspr   SPRN_SPURR,r4
+END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_201)
 
        /* Clear out SLB */
        li      r5,0
@@ -500,8 +683,68 @@ kvmppc_interrupt:
        ptesync
 
 hdec_soon:
-       /* Switch back to host partition */
+BEGIN_FTR_SECTION
+       b       32f
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
+       /*
+        * POWER7 guest -> host partition switch code.
+        * We don't have to lock against tlbies but we do
+        * have to coordinate the hardware threads.
+        */
+       /* Increment the threads-exiting-guest count in the 0xff00
+          bits of vcore->entry_exit_count */
+       lwsync
+       ld      r5,HSTATE_KVM_VCORE(r13)
+       addi    r6,r5,VCORE_ENTRY_EXIT
+41:    lwarx   r3,0,r6
+       addi    r0,r3,0x100
+       stwcx.  r0,0,r6
+       bne     41b
+
+       /*
+        * At this point we have an interrupt that we have to pass
+        * up to the kernel or qemu; we can't handle it in real mode.
+        * Thus we have to do a partition switch, so we have to
+        * collect the other threads, if we are the first thread
+        * to take an interrupt.  To do this, we set the HDEC to 0,
+        * which causes an HDEC interrupt in all threads within 2ns
+        * because the HDEC register is shared between all 4 threads.
+        * However, we don't need to bother if this is an HDEC
+        * interrupt, since the other threads will already be on their
+        * way here in that case.
+        */
+       cmpwi   r12,BOOK3S_INTERRUPT_HV_DECREMENTER
+       beq     40f
+       cmpwi   r3,0x100        /* Are we the first here? */
+       bge     40f
+       cmpwi   r3,1
+       ble     40f
+       li      r0,0
+       mtspr   SPRN_HDEC,r0
+40:
+
+       /* Secondary threads wait for primary to do partition switch */
        ld      r4,VCPU_KVM(r9)         /* pointer to struct kvm */
+       ld      r5,HSTATE_KVM_VCORE(r13)
+       lwz     r3,VCPU_PTID(r9)
+       cmpwi   r3,0
+       beq     15f
+       HMT_LOW
+13:    lbz     r3,VCORE_IN_GUEST(r5)
+       cmpwi   r3,0
+       bne     13b
+       HMT_MEDIUM
+       b       16f
+
+       /* Primary thread waits for all the secondaries to exit guest */
+15:    lwz     r3,VCORE_ENTRY_EXIT(r5)
+       srwi    r0,r3,8
+       clrldi  r3,r3,56
+       cmpw    r3,r0
+       bne     15b
+       isync
+
+       /* Primary thread switches back to host partition */
        ld      r6,KVM_HOST_SDR1(r4)
        lwz     r7,KVM_HOST_LPID(r4)
        li      r8,LPID_RSVD            /* switch to reserved LPID */
@@ -510,15 +753,90 @@ hdec_soon:
        mtspr   SPRN_SDR1,r6            /* switch to partition page table */
        mtspr   SPRN_LPID,r7
        isync
+       li      r0,0
+       stb     r0,VCORE_IN_GUEST(r5)
        lis     r8,0x7fff               /* MAX_INT@h */
        mtspr   SPRN_HDEC,r8
 
-       ld      r8,KVM_HOST_LPCR(r4)
+16:    ld      r8,KVM_HOST_LPCR(r4)
        mtspr   SPRN_LPCR,r8
        isync
+       b       33f
+
+       /*
+        * PPC970 guest -> host partition switch code.
+        * We have to lock against concurrent tlbies, and
+        * we have to flush the whole TLB.
+        */
+32:    ld      r4,VCPU_KVM(r9)         /* pointer to struct kvm */
+
+       /* Take the guest's tlbie_lock */
+       lwz     r8,PACA_LOCK_TOKEN(r13)
+       addi    r3,r4,KVM_TLBIE_LOCK
+24:    lwarx   r0,0,r3
+       cmpwi   r0,0
+       bne     24b
+       stwcx.  r8,0,r3
+       bne     24b
+       isync
+
+       ld      r7,KVM_HOST_LPCR(r4)    /* use kvm->arch.host_lpcr for HID4 */
+       li      r0,0x18f
+       rotldi  r0,r0,HID4_LPID5_SH     /* all lpid bits in HID4 = 1 */
+       or      r0,r7,r0
+       ptesync
+       sync
+       mtspr   SPRN_HID4,r0            /* switch to reserved LPID */
+       isync
+       li      r0,0
+       stw     r0,0(r3)                /* drop guest tlbie_lock */
+
+       /* invalidate the whole TLB */
+       li      r0,256
+       mtctr   r0
+       li      r6,0
+25:    tlbiel  r6
+       addi    r6,r6,0x1000
+       bdnz    25b
+       ptesync
+
+       /* take native_tlbie_lock */
+       ld      r3,toc_tlbie_lock@toc(2)
+24:    lwarx   r0,0,r3
+       cmpwi   r0,0
+       bne     24b
+       stwcx.  r8,0,r3
+       bne     24b
+       isync
+
+       ld      r6,KVM_HOST_SDR1(r4)
+       mtspr   SPRN_SDR1,r6            /* switch to host page table */
+
+       /* Set up host HID4 value */
+       sync
+       mtspr   SPRN_HID4,r7
+       isync
+       li      r0,0
+       stw     r0,0(r3)                /* drop native_tlbie_lock */
+
+       lis     r8,0x7fff               /* MAX_INT@h */
+       mtspr   SPRN_HDEC,r8
+
+       /* Disable HDEC interrupts */
+       mfspr   r0,SPRN_HID0
+       li      r3,0
+       rldimi  r0,r3, HID0_HDICE_SH, 64-HID0_HDICE_SH-1
+       sync
+       mtspr   SPRN_HID0,r0
+       mfspr   r0,SPRN_HID0
+       mfspr   r0,SPRN_HID0
+       mfspr   r0,SPRN_HID0
+       mfspr   r0,SPRN_HID0
+       mfspr   r0,SPRN_HID0
+       mfspr   r0,SPRN_HID0
 
        /* load host SLB entries */
-       ld      r8,PACA_SLBSHADOWPTR(r13)
+33:    ld      r8,PACA_SLBSHADOWPTR(r13)
 
        .rept   SLB_NUM_BOLTED
        ld      r5,SLBSHADOW_SAVEAREA(r8)
@@ -530,12 +848,14 @@ hdec_soon:
        .endr
 
        /* Save and reset AMR and UAMOR before turning on the MMU */
+BEGIN_FTR_SECTION
        mfspr   r5,SPRN_AMR
        mfspr   r6,SPRN_UAMOR
        std     r5,VCPU_AMR(r9)
        std     r6,VCPU_UAMOR(r9)
        li      r6,0
        mtspr   SPRN_AMR,r6
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
 
        /* Restore host DABR and DABRX */
        ld      r5,HSTATE_DABR(r13)
@@ -544,10 +864,12 @@ hdec_soon:
        mtspr   SPRN_DABRX,r6
 
        /* Switch DSCR back to host value */
+BEGIN_FTR_SECTION
        mfspr   r8, SPRN_DSCR
        ld      r7, HSTATE_DSCR(r13)
        std     r8, VCPU_DSCR(r7)
        mtspr   SPRN_DSCR, r7
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
 
        /* Save non-volatile GPRs */
        std     r14, VCPU_GPR(r14)(r9)
@@ -579,13 +901,28 @@ hdec_soon:
        std     r5, VCPU_SPRG2(r9)
        std     r6, VCPU_SPRG3(r9)
 
-       /* Save PMU registers */
+       /* Increment yield count if they have a VPA */
+       ld      r8, VCPU_VPA(r9)        /* do they have a VPA? */
+       cmpdi   r8, 0
+       beq     25f
+       lwz     r3, LPPACA_YIELDCOUNT(r8)
+       addi    r3, r3, 1
+       stw     r3, LPPACA_YIELDCOUNT(r8)
+25:
+       /* Save PMU registers if requested */
+       /* r8 and cr0.eq are live here */
        li      r3, 1
        sldi    r3, r3, 31              /* MMCR0_FC (freeze counters) bit */
        mfspr   r4, SPRN_MMCR0          /* save MMCR0 */
        mtspr   SPRN_MMCR0, r3          /* freeze all counters, disable ints */
        isync
-       mfspr   r5, SPRN_MMCR1
+       beq     21f                     /* if no VPA, save PMU stuff anyway */
+       lbz     r7, LPPACA_PMCINUSE(r8)
+       cmpwi   r7, 0                   /* did they ask for PMU stuff to be saved? */
+       bne     21f
+       std     r3, VCPU_MMCR(r9)       /* if not, set saved MMCR0 to FC */
+       b       22f
+21:    mfspr   r5, SPRN_MMCR1
        mfspr   r6, SPRN_MMCRA
        std     r4, VCPU_MMCR(r9)
        std     r5, VCPU_MMCR + 8(r9)
@@ -596,17 +933,32 @@ hdec_soon:
        mfspr   r6, SPRN_PMC4
        mfspr   r7, SPRN_PMC5
        mfspr   r8, SPRN_PMC6
+BEGIN_FTR_SECTION
+       mfspr   r10, SPRN_PMC7
+       mfspr   r11, SPRN_PMC8
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
        stw     r3, VCPU_PMC(r9)
        stw     r4, VCPU_PMC + 4(r9)
        stw     r5, VCPU_PMC + 8(r9)
        stw     r6, VCPU_PMC + 12(r9)
        stw     r7, VCPU_PMC + 16(r9)
        stw     r8, VCPU_PMC + 20(r9)
+BEGIN_FTR_SECTION
+       stw     r10, VCPU_PMC + 24(r9)
+       stw     r11, VCPU_PMC + 28(r9)
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
 22:
        /* save FP state */
        mr      r3, r9
        bl      .kvmppc_save_fp
 
+       /* Secondary threads go off to take a nap on POWER7 */
+BEGIN_FTR_SECTION
+       lwz     r0,VCPU_PTID(r3)
+       cmpwi   r0,0
+       bne     secondary_nap
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_206)
+
        /*
         * Reload DEC.  HDEC interrupts were disabled when
         * we reloaded the host's LPCR value.
@@ -627,12 +979,20 @@ hdec_soon:
        lwz     r6, HSTATE_PMC + 12(r13)
        lwz     r8, HSTATE_PMC + 16(r13)
        lwz     r9, HSTATE_PMC + 20(r13)
+BEGIN_FTR_SECTION
+       lwz     r10, HSTATE_PMC + 24(r13)
+       lwz     r11, HSTATE_PMC + 28(r13)
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
        mtspr   SPRN_PMC1, r3
        mtspr   SPRN_PMC2, r4
        mtspr   SPRN_PMC3, r5
        mtspr   SPRN_PMC4, r6
        mtspr   SPRN_PMC5, r8
        mtspr   SPRN_PMC6, r9
+BEGIN_FTR_SECTION
+       mtspr   SPRN_PMC7, r10
+       mtspr   SPRN_PMC8, r11
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
        ld      r3, HSTATE_MMCR(r13)
        ld      r4, HSTATE_MMCR + 8(r13)
        ld      r5, HSTATE_MMCR + 16(r13)
@@ -658,7 +1018,7 @@ hdec_soon:
        cmpwi   r12, BOOK3S_INTERRUPT_MACHINE_CHECK
 
        /* RFI into the highmem handler, or branch to interrupt handler */
-       mfmsr   r6
+12:    mfmsr   r6
        mtctr   r12
        li      r0, MSR_RI
        andc    r6, r6, r0
@@ -668,7 +1028,11 @@ hdec_soon:
        beqctr
        RFI
 
-11:    mtspr   SPRN_HSRR0, r8
+11:
+BEGIN_FTR_SECTION
+       b       12b
+END_FTR_SECTION_IFSET(CPU_FTR_ARCH_201)
+       mtspr   SPRN_HSRR0, r8
        mtspr   SPRN_HSRR1, r7
        ba      0x500
 
@@ -676,6 +1040,125 @@ hdec_soon:
        mfspr   r7,SPRN_HDSISR
        b       7b
 
+/*
+ * Try to handle an hcall in real mode.
+ * Returns to the guest if we handle it, or continues on up to
+ * the kernel if we can't (i.e. if we don't have a handler for
+ * it, or if the handler returns H_TOO_HARD).
+ */
+       .globl  hcall_try_real_mode
+hcall_try_real_mode:
+       ld      r3,VCPU_GPR(r3)(r9)
+       andi.   r0,r11,MSR_PR
+       bne     hcall_real_cont
+       clrrdi  r3,r3,2
+       cmpldi  r3,hcall_real_table_end - hcall_real_table
+       bge     hcall_real_cont
+       LOAD_REG_ADDR(r4, hcall_real_table)
+       lwzx    r3,r3,r4
+       cmpwi   r3,0
+       beq     hcall_real_cont
+       add     r3,r3,r4
+       mtctr   r3
+       mr      r3,r9           /* get vcpu pointer */
+       ld      r4,VCPU_GPR(r4)(r9)
+       bctrl
+       cmpdi   r3,H_TOO_HARD
+       beq     hcall_real_fallback
+       ld      r4,HSTATE_KVM_VCPU(r13)
+       std     r3,VCPU_GPR(r3)(r4)
+       ld      r10,VCPU_PC(r4)
+       ld      r11,VCPU_MSR(r4)
+       b       fast_guest_return
+
+       /* We've attempted a real mode hcall, but it's punted it back
+        * to userspace.  We need to restore some clobbered volatiles
+        * before resuming the pass-it-to-qemu path */
+hcall_real_fallback:
+       li      r12,BOOK3S_INTERRUPT_SYSCALL
+       ld      r9, HSTATE_KVM_VCPU(r13)
+       ld      r11, VCPU_MSR(r9)
+
+       b       hcall_real_cont
+
+       .globl  hcall_real_table
+hcall_real_table:
+       .long   0               /* 0 - unused */
+       .long   .kvmppc_h_remove - hcall_real_table
+       .long   .kvmppc_h_enter - hcall_real_table
+       .long   .kvmppc_h_read - hcall_real_table
+       .long   0               /* 0x10 - H_CLEAR_MOD */
+       .long   0               /* 0x14 - H_CLEAR_REF */
+       .long   .kvmppc_h_protect - hcall_real_table
+       .long   0               /* 0x1c - H_GET_TCE */
+       .long   .kvmppc_h_put_tce - hcall_real_table
+       .long   0               /* 0x24 - H_SET_SPRG0 */
+       .long   .kvmppc_h_set_dabr - hcall_real_table
+       .long   0               /* 0x2c */
+       .long   0               /* 0x30 */
+       .long   0               /* 0x34 */
+       .long   0               /* 0x38 */
+       .long   0               /* 0x3c */
+       .long   0               /* 0x40 */
+       .long   0               /* 0x44 */
+       .long   0               /* 0x48 */
+       .long   0               /* 0x4c */
+       .long   0               /* 0x50 */
+       .long   0               /* 0x54 */
+       .long   0               /* 0x58 */
+       .long   0               /* 0x5c */
+       .long   0               /* 0x60 */
+       .long   0               /* 0x64 */
+       .long   0               /* 0x68 */
+       .long   0               /* 0x6c */
+       .long   0               /* 0x70 */
+       .long   0               /* 0x74 */
+       .long   0               /* 0x78 */
+       .long   0               /* 0x7c */
+       .long   0               /* 0x80 */
+       .long   0               /* 0x84 */
+       .long   0               /* 0x88 */
+       .long   0               /* 0x8c */
+       .long   0               /* 0x90 */
+       .long   0               /* 0x94 */
+       .long   0               /* 0x98 */
+       .long   0               /* 0x9c */
+       .long   0               /* 0xa0 */
+       .long   0               /* 0xa4 */
+       .long   0               /* 0xa8 */
+       .long   0               /* 0xac */
+       .long   0               /* 0xb0 */
+       .long   0               /* 0xb4 */
+       .long   0               /* 0xb8 */
+       .long   0               /* 0xbc */
+       .long   0               /* 0xc0 */
+       .long   0               /* 0xc4 */
+       .long   0               /* 0xc8 */
+       .long   0               /* 0xcc */
+       .long   0               /* 0xd0 */
+       .long   0               /* 0xd4 */
+       .long   0               /* 0xd8 */
+       .long   0               /* 0xdc */
+       .long   0               /* 0xe0 */
+       .long   0               /* 0xe4 */
+       .long   0               /* 0xe8 */
+       .long   0               /* 0xec */
+       .long   0               /* 0xf0 */
+       .long   0               /* 0xf4 */
+       .long   0               /* 0xf8 */
+       .long   0               /* 0xfc */
+       .long   0               /* 0x100 */
+       .long   0               /* 0x104 */
+       .long   0               /* 0x108 */
+       .long   0               /* 0x10c */
+       .long   0               /* 0x110 */
+       .long   0               /* 0x114 */
+       .long   0               /* 0x118 */
+       .long   0               /* 0x11c */
+       .long   0               /* 0x120 */
+       .long   .kvmppc_h_bulk_remove - hcall_real_table
+hcall_real_table_end:
+
 ignore_hdec:
        mr      r4,r9
        b       fast_guest_return
@@ -688,10 +1171,66 @@ bounce_ext_interrupt:
        LOAD_REG_IMMEDIATE(r11,MSR_SF | MSR_ME);
        b       fast_guest_return
 
+_GLOBAL(kvmppc_h_set_dabr)
+       std     r4,VCPU_DABR(r3)
+       mtspr   SPRN_DABR,r4
+       li      r3,0
+       blr
+
+secondary_too_late:
+       ld      r5,HSTATE_KVM_VCORE(r13)
+       HMT_LOW
+13:    lbz     r3,VCORE_IN_GUEST(r5)
+       cmpwi   r3,0
+       bne     13b
+       HMT_MEDIUM
+       ld      r11,PACA_SLBSHADOWPTR(r13)
+
+       .rept   SLB_NUM_BOLTED
+       ld      r5,SLBSHADOW_SAVEAREA(r11)
+       ld      r6,SLBSHADOW_SAVEAREA+8(r11)
+       andis.  r7,r5,SLB_ESID_V@h
+       beq     1f
+       slbmte  r6,r5
+1:     addi    r11,r11,16
+       .endr
+       b       50f
+
+secondary_nap:
+       /* Clear any pending IPI */
+50:    ld      r5, HSTATE_XICS_PHYS(r13)
+       li      r0, 0xff
+       li      r6, XICS_QIRR
+       stbcix  r0, r5, r6
+
+       /* increment the nap count and then go to nap mode */
+       ld      r4, HSTATE_KVM_VCORE(r13)
+       addi    r4, r4, VCORE_NAP_COUNT
+       lwsync                          /* make previous updates visible */
+51:    lwarx   r3, 0, r4
+       addi    r3, r3, 1
+       stwcx.  r3, 0, r4
+       bne     51b
+       isync
+
+       mfspr   r4, SPRN_LPCR
+       li      r0, LPCR_PECE
+       andc    r4, r4, r0
+       ori     r4, r4, LPCR_PECE0      /* exit nap on interrupt */
+       mtspr   SPRN_LPCR, r4
+       li      r0, 0
+       std     r0, HSTATE_SCRATCH0(r13)
+       ptesync
+       ld      r0, HSTATE_SCRATCH0(r13)
+1:     cmpd    r0, r0
+       bne     1b
+       nap
+       b       .
+
 /*
  * Save away FP, VMX and VSX registers.
  * r3 = vcpu pointer
-*/
+ */
 _GLOBAL(kvmppc_save_fp)
        mfmsr   r9
        ori     r8,r9,MSR_FP
@@ -712,7 +1251,7 @@ BEGIN_FTR_SECTION
        reg = 0
        .rept   32
        li      r6,reg*16+VCPU_VSRS
-       stxvd2x reg,r6,r3
+       STXVD2X(reg,r6,r3)
        reg = reg + 1
        .endr
 FTR_SECTION_ELSE
@@ -774,7 +1313,7 @@ BEGIN_FTR_SECTION
        reg = 0
        .rept   32
        li      r7,reg*16+VCPU_VSRS
-       lxvd2x  reg,r7,r4
+       LXVD2X(reg,r7,r4)
        reg = reg + 1
        .endr
 FTR_SECTION_ELSE