powerpc/64s/exception: Move KVM test to common code

This allows more code to be moved out of unrelocated regions. The
system call KVMTEST is changed to be open-coded and remain in the
tramp area to avoid having to move it to entry_64.S. The custom nature
of the system call entry code means the hcall case can be made more
streamlined than regular interrupt handlers.

mpe: Incorporate fix from Nick:

Moving KVM test to the common entry code missed the case of HMI and
MCE, which do not do __GEN_COMMON_ENTRY (because they don't want to
switch to virt mode).

This means a MCE or HMI exception that is taken while KVM is running a
guest context will not be switched out of that context, and KVM won't
be notified. Found by running sigfuz in guest with patched host on
POWER9 DD2.3, which causes some TM related HMI interrupts (which are
expected and supposed to be handled by KVM).

This fix adds a __GEN_REALMODE_COMMON_ENTRY for those handlers to add
the KVM test. This makes them look a little more like other handlers
that all use __GEN_COMMON_ENTRY.

Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20200225173541.1549955-13-npiggin@gmail.com

diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index a47f2e5..1bc73ac 100644 (file)
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -44,7 +44,6 @@
  * EXC_VIRT_BEGIN/END  - virt (AIL), unrelocated exception vectors
  * TRAMP_REAL_BEGIN    - real, unrelocated helpers (virt may call these)
  * TRAMP_VIRT_BEGIN    - virt, unreloc helpers (in practice, real can use)
- * TRAMP_KVM_BEGIN     - KVM handlers, these are put into real, unrelocated
  * EXC_COMMON          - After switching to virtual, relocated mode.
  */
 
@@ -74,13 +73,6 @@ name:
 #define TRAMP_VIRT_BEGIN(name)                                 \
        FIXED_SECTION_ENTRY_BEGIN(virt_trampolines, name)
 
-#ifdef CONFIG_KVM_BOOK3S_64_HANDLER
-#define TRAMP_KVM_BEGIN(name)                                  \
-       TRAMP_VIRT_BEGIN(name)
-#else
-#define TRAMP_KVM_BEGIN(name)
-#endif
-
 #define EXC_REAL_NONE(start, size)                             \
        FIXED_SECTION_ENTRY_BEGIN_LOCATION(real_vectors, exc_real_##start##_##unused, start, size); \
        FIXED_SECTION_ENTRY_END_LOCATION(real_vectors, exc_real_##start##_##unused, start, size)
@@ -271,6 +263,9 @@ do_define_int n
 .endm
 
 .macro GEN_KVM name
+       .balign IFETCH_ALIGN_BYTES
+\name\()_kvm:
+
        .if IKVM_SKIP
        cmpwi   r10,KVM_GUEST_MODE_SKIP
        beq     89f
@@ -281,13 +276,18 @@ BEGIN_FTR_SECTION_NESTED(947)
 END_FTR_SECTION_NESTED(CPU_FTR_CFAR,CPU_FTR_CFAR,947)
        .endif
 
+       ld      r10,PACA_EXGEN+EX_CTR(r13)
+       mtctr   r10
 BEGIN_FTR_SECTION_NESTED(948)
        ld      r10,IAREA+EX_PPR(r13)
        std     r10,HSTATE_PPR(r13)
 END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948)
-       ld      r10,IAREA+EX_R10(r13)
+       ld      r11,IAREA+EX_R11(r13)
+       ld      r12,IAREA+EX_R12(r13)
        std     r12,HSTATE_SCRATCH0(r13)
        sldi    r12,r9,32
+       ld      r9,IAREA+EX_R9(r13)
+       ld      r10,IAREA+EX_R10(r13)
        /* HSRR variants have the 0x2 bit added to their trap number */
        .if IHSRR == EXC_HV_OR_STD
        BEGIN_FTR_SECTION
@@ -300,29 +300,16 @@ END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948)
        .else
        ori     r12,r12,(IVEC)
        .endif
-
-#ifdef CONFIG_RELOCATABLE
-       /*
-        * KVM requires __LOAD_FAR_HANDLER beause kvmppc_interrupt lives
-        * outside the head section. CONFIG_RELOCATABLE KVM expects CTR
-        * to be saved in HSTATE_SCRATCH1.
-        */
-       ld      r9,IAREA+EX_CTR(r13)
-       std     r9,HSTATE_SCRATCH1(r13)
-       __LOAD_FAR_HANDLER(r9, kvmppc_interrupt)
-       mtctr   r9
-       ld      r9,IAREA+EX_R9(r13)
-       bctr
-#else
-       ld      r9,IAREA+EX_R9(r13)
        b       kvmppc_interrupt
-#endif
-
 
        .if IKVM_SKIP
 89:    mtocrf  0x80,r9
+       ld      r10,PACA_EXGEN+EX_CTR(r13)
+       mtctr   r10
        ld      r9,IAREA+EX_R9(r13)
        ld      r10,IAREA+EX_R10(r13)
+       ld      r11,IAREA+EX_R11(r13)
+       ld      r12,IAREA+EX_R12(r13)
        .if IHSRR == EXC_HV_OR_STD
        BEGIN_FTR_SECTION
        b       kvmppc_skip_Hinterrupt
@@ -407,11 +394,6 @@ END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948)
        mfctr   r10
        std     r10,IAREA+EX_CTR(r13)
        mfcr    r9
-
-       .if (!\virt && IKVM_REAL) || (\virt && IKVM_VIRT)
-               KVMTEST \name IHSRR IVEC
-       .endif
-
        std     r11,IAREA+EX_R11(r13)
        std     r12,IAREA+EX_R12(r13)
 
@@ -469,12 +451,18 @@ END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948)
 
 /*
  * __GEN_COMMON_ENTRY is required to receive the branch from interrupt
- * entry, except in the case of the IEARLY handlers.
+ * entry, except in the case of the real-mode handlers which require
+ * __GEN_REALMODE_COMMON_ENTRY.
+ *
  * This switches to virtual mode and sets MSR[RI].
  */
 .macro __GEN_COMMON_ENTRY name
 DEFINE_FIXED_SYMBOL(\name\()_common_real)
 \name\()_common_real:
+       .if IKVM_REAL
+               KVMTEST \name IHSRR IVEC
+       .endif
+
        ld      r10,PACAKMSR(r13)       /* get MSR value for kernel */
        /* MSR[RI] is clear iff using SRR regs */
        .if IHSRR == EXC_HV_OR_STD
@@ -487,12 +475,32 @@ DEFINE_FIXED_SYMBOL(\name\()_common_real)
        mtmsrd  r10
 
        .if IVIRT
+       .if IKVM_VIRT
+       b       1f /* skip the virt test coming from real */
+       .endif
+
        .balign IFETCH_ALIGN_BYTES
 DEFINE_FIXED_SYMBOL(\name\()_common_virt)
 \name\()_common_virt:
+       .if IKVM_VIRT
+               KVMTEST \name IHSRR IVEC
+1:
+       .endif
        .endif /* IVIRT */
 .endm
 
+/*
+ * Don't switch to virt mode. Used for early MCE and HMI handlers that
+ * want to run in real mode.
+ */
+.macro __GEN_REALMODE_COMMON_ENTRY name
+DEFINE_FIXED_SYMBOL(\name\()_common_real)
+\name\()_common_real:
+       .if IKVM_REAL
+               KVMTEST \name IHSRR IVEC
+       .endif
+.endm
+
 .macro __GEN_COMMON_BODY name
        .if IMASK
                lbz     r10,PACAIRQSOFTMASK(r13)
@@ -848,8 +856,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206)
         */
 EXC_REAL_END(system_reset, 0x100, 0x100)
 EXC_VIRT_NONE(0x4100, 0x100)
-TRAMP_KVM_BEGIN(system_reset_kvm)
-       GEN_KVM system_reset
 
 #ifdef CONFIG_PPC_P7_NAP
 TRAMP_REAL_BEGIN(system_reset_idle_wake)
@@ -927,6 +933,8 @@ EXC_COMMON_BEGIN(system_reset_common)
        EXCEPTION_RESTORE_REGS EXC_STD
        RFI_TO_USER_OR_KERNEL
 
+       GEN_KVM system_reset
+
 
 INT_DEFINE_BEGIN(machine_check_early)
        IVEC=0x200
@@ -968,9 +976,6 @@ TRAMP_REAL_BEGIN(machine_check_fwnmi)
        GEN_INT_ENTRY machine_check_early, virt=0
 #endif
 
-TRAMP_KVM_BEGIN(machine_check_kvm)
-       GEN_KVM machine_check
-
 #define MACHINE_CHECK_HANDLER_WINDUP                   \
        /* Clear MSR_RI before setting SRR0 and SRR1. */\
        li      r9,0;                                   \
@@ -985,6 +990,8 @@ EXC_COMMON_BEGIN(machine_check_early_common)
        mfspr   r11,SPRN_SRR0
        mfspr   r12,SPRN_SRR1
 
+       __GEN_REALMODE_COMMON_ENTRY machine_check_early
+
        /*
         * Switch to mc_emergency stack and handle re-entrancy (we limit
         * the nested MCE upto level 4 to avoid stack overflow).
@@ -1126,6 +1133,9 @@ EXC_COMMON_BEGIN(machine_check_common)
        bl      machine_check_exception
        b       ret_from_except
 
+       GEN_KVM machine_check
+
+
 #ifdef CONFIG_PPC_P7_NAP
 /*
  * This is an idle wakeup. Low level machine check has already been
@@ -1218,8 +1228,6 @@ EXC_REAL_END(data_access, 0x300, 0x80)
 EXC_VIRT_BEGIN(data_access, 0x4300, 0x80)
        GEN_INT_ENTRY data_access, virt=1
 EXC_VIRT_END(data_access, 0x4300, 0x80)
-TRAMP_KVM_BEGIN(data_access_kvm)
-       GEN_KVM data_access
 EXC_COMMON_BEGIN(data_access_common)
        GEN_COMMON data_access
        ld      r4,_DAR(r1)
@@ -1232,6 +1240,8 @@ MMU_FTR_SECTION_ELSE
        b       handle_page_fault
 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
 
+       GEN_KVM data_access
+
 
 INT_DEFINE_BEGIN(data_access_slb)
        IVEC=0x380
@@ -1248,8 +1258,6 @@ EXC_REAL_END(data_access_slb, 0x380, 0x80)
 EXC_VIRT_BEGIN(data_access_slb, 0x4380, 0x80)
        GEN_INT_ENTRY data_access_slb, virt=1
 EXC_VIRT_END(data_access_slb, 0x4380, 0x80)
-TRAMP_KVM_BEGIN(data_access_slb_kvm)
-       GEN_KVM data_access_slb
 EXC_COMMON_BEGIN(data_access_slb_common)
        GEN_COMMON data_access_slb
        ld      r4,_DAR(r1)
@@ -1274,6 +1282,8 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
        bl      do_bad_slb_fault
        b       ret_from_except
 
+       GEN_KVM data_access_slb
+
 
 INT_DEFINE_BEGIN(instruction_access)
        IVEC=0x400
@@ -1289,8 +1299,6 @@ EXC_REAL_END(instruction_access, 0x400, 0x80)
 EXC_VIRT_BEGIN(instruction_access, 0x4400, 0x80)
        GEN_INT_ENTRY instruction_access, virt=1
 EXC_VIRT_END(instruction_access, 0x4400, 0x80)
-TRAMP_KVM_BEGIN(instruction_access_kvm)
-       GEN_KVM instruction_access
 EXC_COMMON_BEGIN(instruction_access_common)
        GEN_COMMON instruction_access
        ld      r4,_DAR(r1)
@@ -1303,6 +1311,8 @@ MMU_FTR_SECTION_ELSE
        b       handle_page_fault
 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
 
+       GEN_KVM instruction_access
+
 
 INT_DEFINE_BEGIN(instruction_access_slb)
        IVEC=0x480
@@ -1319,8 +1329,6 @@ EXC_REAL_END(instruction_access_slb, 0x480, 0x80)
 EXC_VIRT_BEGIN(instruction_access_slb, 0x4480, 0x80)
        GEN_INT_ENTRY instruction_access_slb, virt=1
 EXC_VIRT_END(instruction_access_slb, 0x4480, 0x80)
-TRAMP_KVM_BEGIN(instruction_access_slb_kvm)
-       GEN_KVM instruction_access_slb
 EXC_COMMON_BEGIN(instruction_access_slb_common)
        GEN_COMMON instruction_access_slb
        ld      r4,_DAR(r1)
@@ -1345,6 +1353,9 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
        bl      do_bad_slb_fault
        b       ret_from_except
 
+       GEN_KVM instruction_access_slb
+
+
 INT_DEFINE_BEGIN(hardware_interrupt)
        IVEC=0x500
        IHSRR=EXC_HV_OR_STD
@@ -1359,8 +1370,6 @@ EXC_REAL_END(hardware_interrupt, 0x500, 0x100)
 EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100)
        GEN_INT_ENTRY hardware_interrupt, virt=1
 EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100)
-TRAMP_KVM_BEGIN(hardware_interrupt_kvm)
-       GEN_KVM hardware_interrupt
 EXC_COMMON_BEGIN(hardware_interrupt_common)
        GEN_COMMON hardware_interrupt
        FINISH_NAP
@@ -1369,6 +1378,8 @@ EXC_COMMON_BEGIN(hardware_interrupt_common)
        bl      do_IRQ
        b       ret_from_except_lite
 
+       GEN_KVM hardware_interrupt
+
 
 INT_DEFINE_BEGIN(alignment)
        IVEC=0x600
@@ -1383,8 +1394,6 @@ EXC_REAL_END(alignment, 0x600, 0x100)
 EXC_VIRT_BEGIN(alignment, 0x4600, 0x100)
        GEN_INT_ENTRY alignment, virt=1
 EXC_VIRT_END(alignment, 0x4600, 0x100)
-TRAMP_KVM_BEGIN(alignment_kvm)
-       GEN_KVM alignment
 EXC_COMMON_BEGIN(alignment_common)
        GEN_COMMON alignment
        bl      save_nvgprs
@@ -1392,6 +1401,8 @@ EXC_COMMON_BEGIN(alignment_common)
        bl      alignment_exception
        b       ret_from_except
 
+       GEN_KVM alignment
+
 
 INT_DEFINE_BEGIN(program_check)
        IVEC=0x700
@@ -1404,8 +1415,6 @@ EXC_REAL_END(program_check, 0x700, 0x100)
 EXC_VIRT_BEGIN(program_check, 0x4700, 0x100)
        GEN_INT_ENTRY program_check, virt=1
 EXC_VIRT_END(program_check, 0x4700, 0x100)
-TRAMP_KVM_BEGIN(program_check_kvm)
-       GEN_KVM program_check
 EXC_COMMON_BEGIN(program_check_common)
        __GEN_COMMON_ENTRY program_check
 
@@ -1445,6 +1454,8 @@ EXC_COMMON_BEGIN(program_check_common)
        bl      program_check_exception
        b       ret_from_except
 
+       GEN_KVM program_check
+
 
 INT_DEFINE_BEGIN(fp_unavailable)
        IVEC=0x800
@@ -1458,8 +1469,6 @@ EXC_REAL_END(fp_unavailable, 0x800, 0x100)
 EXC_VIRT_BEGIN(fp_unavailable, 0x4800, 0x100)
        GEN_INT_ENTRY fp_unavailable, virt=1
 EXC_VIRT_END(fp_unavailable, 0x4800, 0x100)
-TRAMP_KVM_BEGIN(fp_unavailable_kvm)
-       GEN_KVM fp_unavailable
 EXC_COMMON_BEGIN(fp_unavailable_common)
        GEN_COMMON fp_unavailable
        bne     1f                      /* if from user, just load it up */
@@ -1490,6 +1499,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_TM)
        b       ret_from_except
 #endif
 
+       GEN_KVM fp_unavailable
+
 
 INT_DEFINE_BEGIN(decrementer)
        IVEC=0x900
@@ -1503,8 +1514,6 @@ EXC_REAL_END(decrementer, 0x900, 0x80)
 EXC_VIRT_BEGIN(decrementer, 0x4900, 0x80)
        GEN_INT_ENTRY decrementer, virt=1
 EXC_VIRT_END(decrementer, 0x4900, 0x80)
-TRAMP_KVM_BEGIN(decrementer_kvm)
-       GEN_KVM decrementer
 EXC_COMMON_BEGIN(decrementer_common)
        GEN_COMMON decrementer
        FINISH_NAP
@@ -1513,6 +1522,8 @@ EXC_COMMON_BEGIN(decrementer_common)
        bl      timer_interrupt
        b       ret_from_except_lite
 
+       GEN_KVM decrementer
+
 
 INT_DEFINE_BEGIN(hdecrementer)
        IVEC=0x980
@@ -1527,8 +1538,6 @@ EXC_REAL_END(hdecrementer, 0x980, 0x80)
 EXC_VIRT_BEGIN(hdecrementer, 0x4980, 0x80)
        GEN_INT_ENTRY hdecrementer, virt=1
 EXC_VIRT_END(hdecrementer, 0x4980, 0x80)
-TRAMP_KVM_BEGIN(hdecrementer_kvm)
-       GEN_KVM hdecrementer
 EXC_COMMON_BEGIN(hdecrementer_common)
        GEN_COMMON hdecrementer
        bl      save_nvgprs
@@ -1536,6 +1545,8 @@ EXC_COMMON_BEGIN(hdecrementer_common)
        bl      hdec_interrupt
        b       ret_from_except
 
+       GEN_KVM hdecrementer
+
 
 INT_DEFINE_BEGIN(doorbell_super)
        IVEC=0xa00
@@ -1549,8 +1560,6 @@ EXC_REAL_END(doorbell_super, 0xa00, 0x100)
 EXC_VIRT_BEGIN(doorbell_super, 0x4a00, 0x100)
        GEN_INT_ENTRY doorbell_super, virt=1
 EXC_VIRT_END(doorbell_super, 0x4a00, 0x100)
-TRAMP_KVM_BEGIN(doorbell_super_kvm)
-       GEN_KVM doorbell_super
 EXC_COMMON_BEGIN(doorbell_super_common)
        GEN_COMMON doorbell_super
        FINISH_NAP
@@ -1563,6 +1572,8 @@ EXC_COMMON_BEGIN(doorbell_super_common)
 #endif
        b       ret_from_except_lite
 
+       GEN_KVM doorbell_super
+
 
 EXC_REAL_NONE(0xb00, 0x100)
 EXC_VIRT_NONE(0x4b00, 0x100)
@@ -1667,6 +1678,7 @@ EXC_VIRT_BEGIN(system_call, 0x4c00, 0x100)
 EXC_VIRT_END(system_call, 0x4c00, 0x100)
 
 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
+TRAMP_REAL_BEGIN(system_call_kvm)
        /*
         * This is a hcall, so register convention is as above, with these
         * differences:
@@ -1674,20 +1686,35 @@ EXC_VIRT_END(system_call, 0x4c00, 0x100)
         * ctr = orig r13
         * orig r10 saved in PACA
         */
-TRAMP_KVM_BEGIN(system_call_kvm)
         /*
          * Save the PPR (on systems that support it) before changing to
          * HMT_MEDIUM. That allows the KVM code to save that value into the
          * guest state (it is the guest's PPR value).
          */
-       OPT_GET_SPR(r10, SPRN_PPR, CPU_FTR_HAS_PPR)
+BEGIN_FTR_SECTION_NESTED(948)
+       mfspr   r10,SPRN_PPR
+       std     r10,HSTATE_PPR(r13)
+END_FTR_SECTION_NESTED(CPU_FTR_HAS_PPR,CPU_FTR_HAS_PPR,948)
        HMT_MEDIUM
-       OPT_SAVE_REG_TO_PACA(PACA_EXGEN+EX_PPR, r10, CPU_FTR_HAS_PPR)
        mfctr   r10
        SET_SCRATCH0(r10)
-       std     r9,PACA_EXGEN+EX_R9(r13)
-       mfcr    r9
-       GEN_KVM system_call
+       mfcr    r10
+       std     r12,HSTATE_SCRATCH0(r13)
+       sldi    r12,r10,32
+       ori     r12,r12,0xc00
+#ifdef CONFIG_RELOCATABLE
+       /*
+        * Requires __LOAD_FAR_HANDLER beause kvmppc_interrupt lives
+        * outside the head section.
+        */
+       __LOAD_FAR_HANDLER(r10, kvmppc_interrupt)
+       mtctr   r10
+       ld      r10,PACA_EXGEN+EX_R10(r13)
+       bctr
+#else
+       ld      r10,PACA_EXGEN+EX_R10(r13)
+       b       kvmppc_interrupt
+#endif
 #endif
 
 
@@ -1702,8 +1729,6 @@ EXC_REAL_END(single_step, 0xd00, 0x100)
 EXC_VIRT_BEGIN(single_step, 0x4d00, 0x100)
        GEN_INT_ENTRY single_step, virt=1
 EXC_VIRT_END(single_step, 0x4d00, 0x100)
-TRAMP_KVM_BEGIN(single_step_kvm)
-       GEN_KVM single_step
 EXC_COMMON_BEGIN(single_step_common)
        GEN_COMMON single_step
        bl      save_nvgprs
@@ -1711,6 +1736,8 @@ EXC_COMMON_BEGIN(single_step_common)
        bl      single_step_exception
        b       ret_from_except
 
+       GEN_KVM single_step
+
 
 INT_DEFINE_BEGIN(h_data_storage)
        IVEC=0xe00
@@ -1728,8 +1755,6 @@ EXC_REAL_END(h_data_storage, 0xe00, 0x20)
 EXC_VIRT_BEGIN(h_data_storage, 0x4e00, 0x20)
        GEN_INT_ENTRY h_data_storage, virt=1, ool=1
 EXC_VIRT_END(h_data_storage, 0x4e00, 0x20)
-TRAMP_KVM_BEGIN(h_data_storage_kvm)
-       GEN_KVM h_data_storage
 EXC_COMMON_BEGIN(h_data_storage_common)
        GEN_COMMON h_data_storage
        bl      save_nvgprs
@@ -1743,6 +1768,8 @@ MMU_FTR_SECTION_ELSE
 ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_TYPE_RADIX)
        b       ret_from_except
 
+       GEN_KVM h_data_storage
+
 
 INT_DEFINE_BEGIN(h_instr_storage)
        IVEC=0xe20
@@ -1757,8 +1784,6 @@ EXC_REAL_END(h_instr_storage, 0xe20, 0x20)
 EXC_VIRT_BEGIN(h_instr_storage, 0x4e20, 0x20)
        GEN_INT_ENTRY h_instr_storage, virt=1, ool=1
 EXC_VIRT_END(h_instr_storage, 0x4e20, 0x20)
-TRAMP_KVM_BEGIN(h_instr_storage_kvm)
-       GEN_KVM h_instr_storage
 EXC_COMMON_BEGIN(h_instr_storage_common)
        GEN_COMMON h_instr_storage
        bl      save_nvgprs
@@ -1766,6 +1791,8 @@ EXC_COMMON_BEGIN(h_instr_storage_common)
        bl      unknown_exception
        b       ret_from_except
 
+       GEN_KVM h_instr_storage
+
 
 INT_DEFINE_BEGIN(emulation_assist)
        IVEC=0xe40
@@ -1780,8 +1807,6 @@ EXC_REAL_END(emulation_assist, 0xe40, 0x20)
 EXC_VIRT_BEGIN(emulation_assist, 0x4e40, 0x20)
        GEN_INT_ENTRY emulation_assist, virt=1, ool=1
 EXC_VIRT_END(emulation_assist, 0x4e40, 0x20)
-TRAMP_KVM_BEGIN(emulation_assist_kvm)
-       GEN_KVM emulation_assist
 EXC_COMMON_BEGIN(emulation_assist_common)
        GEN_COMMON emulation_assist
        bl      save_nvgprs
@@ -1789,6 +1814,8 @@ EXC_COMMON_BEGIN(emulation_assist_common)
        bl      emulation_assist_interrupt
        b       ret_from_except
 
+       GEN_KVM emulation_assist
+
 
 /*
  * hmi_exception trampoline is a special case. It jumps to hmi_exception_early
@@ -1816,14 +1843,13 @@ EXC_REAL_BEGIN(hmi_exception, 0xe60, 0x20)
        GEN_INT_ENTRY hmi_exception_early, virt=0, ool=1
 EXC_REAL_END(hmi_exception, 0xe60, 0x20)
 EXC_VIRT_NONE(0x4e60, 0x20)
-TRAMP_KVM_BEGIN(hmi_exception_early_kvm)
-       GEN_KVM hmi_exception_early
-TRAMP_KVM_BEGIN(hmi_exception_kvm)
-       GEN_KVM hmi_exception
 
 EXC_COMMON_BEGIN(hmi_exception_early_common)
        mfspr   r11,SPRN_HSRR0          /* Save HSRR0 */
        mfspr   r12,SPRN_HSRR1          /* Save HSRR1 */
+
+       __GEN_REALMODE_COMMON_ENTRY hmi_exception_early
+
        mr      r10,r1                  /* Save r1 */
        ld      r1,PACAEMERGSP(r13)     /* Use emergency stack for realmode */
        subi    r1,r1,INT_FRAME_SIZE    /* alloc stack frame            */
@@ -1846,6 +1872,8 @@ EXC_COMMON_BEGIN(hmi_exception_early_common)
        EXCEPTION_RESTORE_REGS EXC_HV
        GEN_INT_ENTRY hmi_exception, virt=0
 
+       GEN_KVM hmi_exception_early
+
 EXC_COMMON_BEGIN(hmi_exception_common)
        GEN_COMMON hmi_exception
        FINISH_NAP
@@ -1855,6 +1883,8 @@ EXC_COMMON_BEGIN(hmi_exception_common)
        bl      handle_hmi_exception
        b       ret_from_except
 
+       GEN_KVM hmi_exception
+
 
 INT_DEFINE_BEGIN(h_doorbell)
        IVEC=0xe80
@@ -1870,8 +1900,6 @@ EXC_REAL_END(h_doorbell, 0xe80, 0x20)
 EXC_VIRT_BEGIN(h_doorbell, 0x4e80, 0x20)
        GEN_INT_ENTRY h_doorbell, virt=1, ool=1
 EXC_VIRT_END(h_doorbell, 0x4e80, 0x20)
-TRAMP_KVM_BEGIN(h_doorbell_kvm)
-       GEN_KVM h_doorbell
 EXC_COMMON_BEGIN(h_doorbell_common)
        GEN_COMMON h_doorbell
        FINISH_NAP
@@ -1884,6 +1912,8 @@ EXC_COMMON_BEGIN(h_doorbell_common)
 #endif
        b       ret_from_except_lite
 
+       GEN_KVM h_doorbell
+
 
 INT_DEFINE_BEGIN(h_virt_irq)
        IVEC=0xea0
@@ -1899,8 +1929,6 @@ EXC_REAL_END(h_virt_irq, 0xea0, 0x20)
 EXC_VIRT_BEGIN(h_virt_irq, 0x4ea0, 0x20)
        GEN_INT_ENTRY h_virt_irq, virt=1, ool=1
 EXC_VIRT_END(h_virt_irq, 0x4ea0, 0x20)
-TRAMP_KVM_BEGIN(h_virt_irq_kvm)
-       GEN_KVM h_virt_irq
 EXC_COMMON_BEGIN(h_virt_irq_common)
        GEN_COMMON h_virt_irq
        FINISH_NAP
@@ -1909,6 +1937,8 @@ EXC_COMMON_BEGIN(h_virt_irq_common)
        bl      do_IRQ
        b       ret_from_except_lite
 
+       GEN_KVM h_virt_irq
+
 
 EXC_REAL_NONE(0xec0, 0x20)
 EXC_VIRT_NONE(0x4ec0, 0x20)
@@ -1928,8 +1958,6 @@ EXC_REAL_END(performance_monitor, 0xf00, 0x20)
 EXC_VIRT_BEGIN(performance_monitor, 0x4f00, 0x20)
        GEN_INT_ENTRY performance_monitor, virt=1, ool=1
 EXC_VIRT_END(performance_monitor, 0x4f00, 0x20)
-TRAMP_KVM_BEGIN(performance_monitor_kvm)
-       GEN_KVM performance_monitor
 EXC_COMMON_BEGIN(performance_monitor_common)
        GEN_COMMON performance_monitor
        FINISH_NAP
@@ -1938,6 +1966,8 @@ EXC_COMMON_BEGIN(performance_monitor_common)
        bl      performance_monitor_exception
        b       ret_from_except_lite
 
+       GEN_KVM performance_monitor
+
 
 INT_DEFINE_BEGIN(altivec_unavailable)
        IVEC=0xf20
@@ -1951,8 +1981,6 @@ EXC_REAL_END(altivec_unavailable, 0xf20, 0x20)
 EXC_VIRT_BEGIN(altivec_unavailable, 0x4f20, 0x20)
        GEN_INT_ENTRY altivec_unavailable, virt=1, ool=1
 EXC_VIRT_END(altivec_unavailable, 0x4f20, 0x20)
-TRAMP_KVM_BEGIN(altivec_unavailable_kvm)
-       GEN_KVM altivec_unavailable
 EXC_COMMON_BEGIN(altivec_unavailable_common)
        GEN_COMMON altivec_unavailable
 #ifdef CONFIG_ALTIVEC
@@ -1986,6 +2014,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC)
        bl      altivec_unavailable_exception
        b       ret_from_except
 
+       GEN_KVM altivec_unavailable
+
 
 INT_DEFINE_BEGIN(vsx_unavailable)
        IVEC=0xf40
@@ -1999,8 +2029,6 @@ EXC_REAL_END(vsx_unavailable, 0xf40, 0x20)
 EXC_VIRT_BEGIN(vsx_unavailable, 0x4f40, 0x20)
        GEN_INT_ENTRY vsx_unavailable, virt=1, ool=1
 EXC_VIRT_END(vsx_unavailable, 0x4f40, 0x20)
-TRAMP_KVM_BEGIN(vsx_unavailable_kvm)
-       GEN_KVM vsx_unavailable
 EXC_COMMON_BEGIN(vsx_unavailable_common)
        GEN_COMMON vsx_unavailable
 #ifdef CONFIG_VSX
@@ -2033,6 +2061,8 @@ END_FTR_SECTION_IFSET(CPU_FTR_VSX)
        bl      vsx_unavailable_exception
        b       ret_from_except
 
+       GEN_KVM vsx_unavailable
+
 
 INT_DEFINE_BEGIN(facility_unavailable)
        IVEC=0xf60
@@ -2045,8 +2075,6 @@ EXC_REAL_END(facility_unavailable, 0xf60, 0x20)
 EXC_VIRT_BEGIN(facility_unavailable, 0x4f60, 0x20)
        GEN_INT_ENTRY facility_unavailable, virt=1, ool=1
 EXC_VIRT_END(facility_unavailable, 0x4f60, 0x20)
-TRAMP_KVM_BEGIN(facility_unavailable_kvm)
-       GEN_KVM facility_unavailable
 EXC_COMMON_BEGIN(facility_unavailable_common)
        GEN_COMMON facility_unavailable
        bl      save_nvgprs
@@ -2054,6 +2082,8 @@ EXC_COMMON_BEGIN(facility_unavailable_common)
        bl      facility_unavailable_exception
        b       ret_from_except
 
+       GEN_KVM facility_unavailable
+
 
 INT_DEFINE_BEGIN(h_facility_unavailable)
        IVEC=0xf80
@@ -2068,8 +2098,6 @@ EXC_REAL_END(h_facility_unavailable, 0xf80, 0x20)
 EXC_VIRT_BEGIN(h_facility_unavailable, 0x4f80, 0x20)
        GEN_INT_ENTRY h_facility_unavailable, virt=1, ool=1
 EXC_VIRT_END(h_facility_unavailable, 0x4f80, 0x20)
-TRAMP_KVM_BEGIN(h_facility_unavailable_kvm)
-       GEN_KVM h_facility_unavailable
 EXC_COMMON_BEGIN(h_facility_unavailable_common)
        GEN_COMMON h_facility_unavailable
        bl      save_nvgprs
@@ -2077,6 +2105,8 @@ EXC_COMMON_BEGIN(h_facility_unavailable_common)
        bl      facility_unavailable_exception
        b       ret_from_except
 
+       GEN_KVM h_facility_unavailable
+
 
 EXC_REAL_NONE(0xfa0, 0x20)
 EXC_VIRT_NONE(0x4fa0, 0x20)
@@ -2102,14 +2132,15 @@ EXC_REAL_BEGIN(cbe_system_error, 0x1200, 0x100)
        GEN_INT_ENTRY cbe_system_error, virt=0
 EXC_REAL_END(cbe_system_error, 0x1200, 0x100)
 EXC_VIRT_NONE(0x5200, 0x100)
-TRAMP_KVM_BEGIN(cbe_system_error_kvm)
-       GEN_KVM cbe_system_error
 EXC_COMMON_BEGIN(cbe_system_error_common)
        GEN_COMMON cbe_system_error
        bl      save_nvgprs
        addi    r3,r1,STACK_FRAME_OVERHEAD
        bl      cbe_system_error_exception
        b       ret_from_except
+
+       GEN_KVM cbe_system_error
+
 #else /* CONFIG_CBE_RAS */
 EXC_REAL_NONE(0x1200, 0x100)
 EXC_VIRT_NONE(0x5200, 0x100)
@@ -2128,8 +2159,6 @@ EXC_REAL_END(instruction_breakpoint, 0x1300, 0x100)
 EXC_VIRT_BEGIN(instruction_breakpoint, 0x5300, 0x100)
        GEN_INT_ENTRY instruction_breakpoint, virt=1
 EXC_VIRT_END(instruction_breakpoint, 0x5300, 0x100)
-TRAMP_KVM_BEGIN(instruction_breakpoint_kvm)
-       GEN_KVM instruction_breakpoint
 EXC_COMMON_BEGIN(instruction_breakpoint_common)
        GEN_COMMON instruction_breakpoint
        bl      save_nvgprs
@@ -2137,6 +2166,8 @@ EXC_COMMON_BEGIN(instruction_breakpoint_common)
        bl      instruction_breakpoint_exception
        b       ret_from_except
 
+       GEN_KVM instruction_breakpoint
+
 
 EXC_REAL_NONE(0x1400, 0x100)
 EXC_VIRT_NONE(0x5400, 0x100)
@@ -2145,6 +2176,7 @@ INT_DEFINE_BEGIN(denorm_exception)
        IVEC=0x1500
        IHSRR=EXC_HV
        IEARLY=2
+       IKVM_REAL=1
 INT_DEFINE_END(denorm_exception)
 
 EXC_REAL_BEGIN(denorm_exception, 0x1500, 0x100)
@@ -2154,7 +2186,6 @@ EXC_REAL_BEGIN(denorm_exception, 0x1500, 0x100)
        andis.  r10,r10,(HSRR1_DENORM)@h /* denorm? */
        bne+    denorm_assist
 #endif
-       KVMTEST denorm_exception, EXC_HV, 0x1500
        mfspr   r11,SPRN_HSRR0
        mfspr   r12,SPRN_HSRR1
        GEN_BRANCH_TO_COMMON denorm_exception, virt=0
@@ -2172,8 +2203,6 @@ EXC_VIRT_END(denorm_exception, 0x5500, 0x100)
 #else
 EXC_VIRT_NONE(0x5500, 0x100)
 #endif
-TRAMP_KVM_BEGIN(denorm_exception_kvm)
-       GEN_KVM denorm_exception
 
 #ifdef CONFIG_PPC_DENORMALISATION
 TRAMP_REAL_BEGIN(denorm_assist)
@@ -2251,6 +2280,8 @@ EXC_COMMON_BEGIN(denorm_exception_common)
        bl      unknown_exception
        b       ret_from_except
 
+       GEN_KVM denorm_exception
+
 
 #ifdef CONFIG_CBE_RAS
 INT_DEFINE_BEGIN(cbe_maintenance)
@@ -2264,14 +2295,15 @@ EXC_REAL_BEGIN(cbe_maintenance, 0x1600, 0x100)
        GEN_INT_ENTRY cbe_maintenance, virt=0
 EXC_REAL_END(cbe_maintenance, 0x1600, 0x100)
 EXC_VIRT_NONE(0x5600, 0x100)
-TRAMP_KVM_BEGIN(cbe_maintenance_kvm)
-       GEN_KVM cbe_maintenance
 EXC_COMMON_BEGIN(cbe_maintenance_common)
        GEN_COMMON cbe_maintenance
        bl      save_nvgprs
        addi    r3,r1,STACK_FRAME_OVERHEAD
        bl      cbe_maintenance_exception
        b       ret_from_except
+
+       GEN_KVM cbe_maintenance
+
 #else /* CONFIG_CBE_RAS */
 EXC_REAL_NONE(0x1600, 0x100)
 EXC_VIRT_NONE(0x5600, 0x100)
@@ -2289,8 +2321,6 @@ EXC_REAL_END(altivec_assist, 0x1700, 0x100)
 EXC_VIRT_BEGIN(altivec_assist, 0x5700, 0x100)
        GEN_INT_ENTRY altivec_assist, virt=1
 EXC_VIRT_END(altivec_assist, 0x5700, 0x100)
-TRAMP_KVM_BEGIN(altivec_assist_kvm)
-       GEN_KVM altivec_assist
 EXC_COMMON_BEGIN(altivec_assist_common)
        GEN_COMMON altivec_assist
        bl      save_nvgprs
@@ -2302,6 +2332,8 @@ EXC_COMMON_BEGIN(altivec_assist_common)
 #endif
        b       ret_from_except
 
+       GEN_KVM altivec_assist
+
 
 #ifdef CONFIG_CBE_RAS
 INT_DEFINE_BEGIN(cbe_thermal)
@@ -2315,14 +2347,15 @@ EXC_REAL_BEGIN(cbe_thermal, 0x1800, 0x100)
        GEN_INT_ENTRY cbe_thermal, virt=0
 EXC_REAL_END(cbe_thermal, 0x1800, 0x100)
 EXC_VIRT_NONE(0x5800, 0x100)
-TRAMP_KVM_BEGIN(cbe_thermal_kvm)
-       GEN_KVM cbe_thermal
 EXC_COMMON_BEGIN(cbe_thermal_common)
        GEN_COMMON cbe_thermal
        bl      save_nvgprs
        addi    r3,r1,STACK_FRAME_OVERHEAD
        bl      cbe_thermal_exception
        b       ret_from_except
+
+       GEN_KVM cbe_thermal
+
 #else /* CONFIG_CBE_RAS */
 EXC_REAL_NONE(0x1800, 0x100)
 EXC_VIRT_NONE(0x5800, 0x100)
@@ -2514,17 +2547,12 @@ TRAMP_REAL_BEGIN(hrfi_flush_fallback)
        GET_SCRATCH0(r13);
        hrfid
 
-/*
- * Real mode exceptions actually use this too, but alternate
- * instruction code patches (which end up in the common .text area)
- * cannot reach these if they are put there.
- */
 USE_TEXT_SECTION()
        MASKED_INTERRUPT EXC_STD
        MASKED_INTERRUPT EXC_HV
 
 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
-TRAMP_REAL_BEGIN(kvmppc_skip_interrupt)
+kvmppc_skip_interrupt:
        /*
         * Here all GPRs are unchanged from when the interrupt happened
         * except for r13, which is saved in SPRG_SCRATCH0.
@@ -2536,7 +2564,7 @@ TRAMP_REAL_BEGIN(kvmppc_skip_interrupt)
        RFI_TO_KERNEL
        b       .
 
-TRAMP_REAL_BEGIN(kvmppc_skip_Hinterrupt)
+kvmppc_skip_Hinterrupt:
        /*
         * Here all GPRs are unchanged from when the interrupt happened
         * except for r13, which is saved in SPRG_SCRATCH0.
@@ -2549,16 +2577,6 @@ TRAMP_REAL_BEGIN(kvmppc_skip_Hinterrupt)
        b       .
 #endif
 
-/*
- * Ensure that any handlers that get invoked from the exception prologs
- * above are below the first 64KB (0x10000) of the kernel image because
- * the prologs assemble the addresses of these handlers using the
- * LOAD_HANDLER macro, which uses an ori instruction.
- */
-
-/*** Common interrupt handlers ***/
-
-
        /*
         * Relocation-on interrupts: A subset of the interrupts can be delivered
         * with IR=1/DR=1, if AIL==2 and MSR.HV won't be changed by delivering

diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
index dbc2fec..780a499 100644 (file)
--- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S
+++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S
@@ -1266,7 +1266,6 @@ kvmppc_interrupt_hv:
         * R12          = (guest CR << 32) | interrupt vector
         * R13          = PACA
         * guest R12 saved in shadow VCPU SCRATCH0
-        * guest CTR saved in shadow VCPU SCRATCH1 if RELOCATABLE
         * guest R13 saved in SPRN_SCRATCH0
         */
        std     r9, HSTATE_SCRATCH2(r13)
@@ -1367,12 +1366,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
 11:    stw     r3,VCPU_HEIR(r9)
 
        /* these are volatile across C function calls */
-#ifdef CONFIG_RELOCATABLE
-       ld      r3, HSTATE_SCRATCH1(r13)
-       mtctr   r3
-#else
        mfctr   r3
-#endif
        mfxer   r4
        std     r3, VCPU_CTR(r9)
        std     r4, VCPU_XER(r9)
@@ -3258,7 +3252,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_P9_TM_HV_ASSIST)
  * r12 is (CR << 32) | vector
  * r13 points to our PACA
  * r12 is saved in HSTATE_SCRATCH0(r13)
- * ctr is saved in HSTATE_SCRATCH1(r13) if RELOCATABLE
  * r9 is saved in HSTATE_SCRATCH2(r13)
  * r13 is saved in HSPRG1
  * cfar is saved in HSTATE_CFAR(r13)
@@ -3307,11 +3300,7 @@ kvmppc_bad_host_intr:
        ld      r5, HSTATE_CFAR(r13)
        std     r5, ORIG_GPR3(r1)
        mflr    r3
-#ifdef CONFIG_RELOCATABLE
-       ld      r4, HSTATE_SCRATCH1(r13)
-#else
        mfctr   r4
-#endif
        mfxer   r5
        lbz     r6, PACAIRQSOFTMASK(r13)
        std     r3, _LINK(r1)

diff --git a/arch/powerpc/kvm/book3s_segment.S b/arch/powerpc/kvm/book3s_segment.S
index 0169bab..1f492aa 100644 (file)
--- a/arch/powerpc/kvm/book3s_segment.S
+++ b/arch/powerpc/kvm/book3s_segment.S
@@ -167,16 +167,9 @@ kvmppc_interrupt_pr:
         * R12             = (guest CR << 32) | exit handler id
         * R13             = PACA
         * HSTATE.SCRATCH0 = guest R12
-        * HSTATE.SCRATCH1 = guest CTR if RELOCATABLE
         */
 #ifdef CONFIG_PPC64
        /* Match 32-bit entry */
-#ifdef CONFIG_RELOCATABLE
-       std     r9, HSTATE_SCRATCH2(r13)
-       ld      r9, HSTATE_SCRATCH1(r13)
-       mtctr   r9
-       ld      r9, HSTATE_SCRATCH2(r13)
-#endif
        rotldi  r12, r12, 32              /* Flip R12 halves for stw */
        stw     r12, HSTATE_SCRATCH1(r13) /* CR is now in the low half */
        srdi    r12, r12, 32              /* shift trap into low half */