Merge tag 'powerpc-cve-2020-4788' into fixes

[linux-2.6-microblaze.git] / arch / powerpc / kernel / exceptions-64s.S

diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index 8e6b2cc..4d01f09 100644 (file)
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -2952,15 +2952,8 @@ TRAMP_REAL_BEGIN(stf_barrier_fallback)
        .endr
        blr
 
-TRAMP_REAL_BEGIN(rfi_flush_fallback)
-       SET_SCRATCH0(r13);
-       GET_PACA(r13);
-       std     r1,PACA_EXRFI+EX_R12(r13)
-       ld      r1,PACAKSAVE(r13)
-       std     r9,PACA_EXRFI+EX_R9(r13)
-       std     r10,PACA_EXRFI+EX_R10(r13)
-       std     r11,PACA_EXRFI+EX_R11(r13)
-       mfctr   r9
+/* Clobbers r10, r11, ctr */
+.macro L1D_DISPLACEMENT_FLUSH
        ld      r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
        ld      r11,PACA_L1D_FLUSH_SIZE(r13)
        srdi    r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
@@ -2971,7 +2964,7 @@ TRAMP_REAL_BEGIN(rfi_flush_fallback)
        sync
 
        /*
-        * The load adresses are at staggered offsets within cachelines,
+        * The load addresses are at staggered offsets within cachelines,
         * which suits some pipelines better (on others it should not
         * hurt).
         */
@@ -2986,7 +2979,30 @@ TRAMP_REAL_BEGIN(rfi_flush_fallback)
        ld      r11,(0x80 + 8)*7(r10)
        addi    r10,r10,0x80*8
        bdnz    1b
+.endm
+
+TRAMP_REAL_BEGIN(entry_flush_fallback)
+       std     r9,PACA_EXRFI+EX_R9(r13)
+       std     r10,PACA_EXRFI+EX_R10(r13)
+       std     r11,PACA_EXRFI+EX_R11(r13)
+       mfctr   r9
+       L1D_DISPLACEMENT_FLUSH
+       mtctr   r9
+       ld      r9,PACA_EXRFI+EX_R9(r13)
+       ld      r10,PACA_EXRFI+EX_R10(r13)
+       ld      r11,PACA_EXRFI+EX_R11(r13)
+       blr
 
+TRAMP_REAL_BEGIN(rfi_flush_fallback)
+       SET_SCRATCH0(r13);
+       GET_PACA(r13);
+       std     r1,PACA_EXRFI+EX_R12(r13)
+       ld      r1,PACAKSAVE(r13)
+       std     r9,PACA_EXRFI+EX_R9(r13)
+       std     r10,PACA_EXRFI+EX_R10(r13)
+       std     r11,PACA_EXRFI+EX_R11(r13)
+       mfctr   r9
+       L1D_DISPLACEMENT_FLUSH
        mtctr   r9
        ld      r9,PACA_EXRFI+EX_R9(r13)
        ld      r10,PACA_EXRFI+EX_R10(r13)
@@ -3004,32 +3020,7 @@ TRAMP_REAL_BEGIN(hrfi_flush_fallback)
        std     r10,PACA_EXRFI+EX_R10(r13)
        std     r11,PACA_EXRFI+EX_R11(r13)
        mfctr   r9
-       ld      r10,PACA_RFI_FLUSH_FALLBACK_AREA(r13)
-       ld      r11,PACA_L1D_FLUSH_SIZE(r13)
-       srdi    r11,r11,(7 + 3) /* 128 byte lines, unrolled 8x */
-       mtctr   r11
-       DCBT_BOOK3S_STOP_ALL_STREAM_IDS(r11) /* Stop prefetch streams */
-
-       /* order ld/st prior to dcbt stop all streams with flushing */
-       sync
-
-       /*
-        * The load adresses are at staggered offsets within cachelines,
-        * which suits some pipelines better (on others it should not
-        * hurt).
-        */
-1:
-       ld      r11,(0x80 + 8)*0(r10)
-       ld      r11,(0x80 + 8)*1(r10)
-       ld      r11,(0x80 + 8)*2(r10)
-       ld      r11,(0x80 + 8)*3(r10)
-       ld      r11,(0x80 + 8)*4(r10)
-       ld      r11,(0x80 + 8)*5(r10)
-       ld      r11,(0x80 + 8)*6(r10)
-       ld      r11,(0x80 + 8)*7(r10)
-       addi    r10,r10,0x80*8
-       bdnz    1b
-
+       L1D_DISPLACEMENT_FLUSH
        mtctr   r9
        ld      r9,PACA_EXRFI+EX_R9(r13)
        ld      r10,PACA_EXRFI+EX_R10(r13)
@@ -3080,8 +3071,21 @@ TRAMP_REAL_BEGIN(rfscv_flush_fallback)
        RFSCV
 
 USE_TEXT_SECTION()
-       MASKED_INTERRUPT
-       MASKED_INTERRUPT hsrr=1
+
+_GLOBAL(do_uaccess_flush)
+       UACCESS_FLUSH_FIXUP_SECTION
+       nop
+       nop
+       nop
+       blr
+       L1D_DISPLACEMENT_FLUSH
+       blr
+_ASM_NOKPROBE_SYMBOL(do_uaccess_flush)
+EXPORT_SYMBOL(do_uaccess_flush)
+
+
+MASKED_INTERRUPT
+MASKED_INTERRUPT hsrr=1
 
 #ifdef CONFIG_KVM_BOOK3S_64_HANDLER
 kvmppc_skip_interrupt: