Merge tag 'powerpc-4.16-1' of git://git.kernel.org/pub/scm/linux/kernel/git/powerpc...

[linux-2.6-microblaze.git] / arch / powerpc / include / asm / book3s / 64 / hash-64k.h

diff --git a/arch/powerpc/include/asm/book3s/64/hash-64k.h b/arch/powerpc/include/asm/book3s/64/hash-64k.h
index cb46d10..338b7da 100644 (file)
--- a/arch/powerpc/include/asm/book3s/64/hash-64k.h
+++ b/arch/powerpc/include/asm/book3s/64/hash-64k.h
@@ -13,21 +13,17 @@
  */
 #define H_PAGE_COMBO   _RPAGE_RPN0 /* this is a combo 4k page */
 #define H_PAGE_4K_PFN  _RPAGE_RPN1 /* PFN is for a single 4k page */
+#define H_PAGE_BUSY    _RPAGE_RPN44     /* software: PTE & hash are busy */
+#define H_PAGE_HASHPTE _RPAGE_RPN43    /* PTE has associated HPTE */
+
 /*
  * We need to differentiate between explicit huge page and THP huge
  * page, since THP huge page also need to track real subpage details
  */
 #define H_PAGE_THP_HUGE  H_PAGE_4K_PFN
 
-/*
- * Used to track subpage group valid if H_PAGE_COMBO is set
- * This overloads H_PAGE_F_GIX and H_PAGE_F_SECOND
- */
-#define H_PAGE_COMBO_VALID     (H_PAGE_F_GIX | H_PAGE_F_SECOND)
-
 /* PTE flags to conserve for HPTE identification */
-#define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_F_SECOND | \
-                        H_PAGE_F_GIX | H_PAGE_HASHPTE | H_PAGE_COMBO)
+#define _PAGE_HPTEFLAGS (H_PAGE_BUSY | H_PAGE_HASHPTE | H_PAGE_COMBO)
 /*
  * we support 16 fragments per PTE page of 64K size.
  */
@@ -55,24 +51,57 @@ static inline real_pte_t __real_pte(pte_t pte, pte_t *ptep)
        unsigned long *hidxp;
 
        rpte.pte = pte;
-       rpte.hidx = 0;
-       if (pte_val(pte) & H_PAGE_COMBO) {
-               /*
-                * Make sure we order the hidx load against the H_PAGE_COMBO
-                * check. The store side ordering is done in __hash_page_4K
-                */
-               smp_rmb();
-               hidxp = (unsigned long *)(ptep + PTRS_PER_PTE);
-               rpte.hidx = *hidxp;
-       }
+
+       /*
+        * Ensure that we do not read the hidx before we read the PTE. Because
+        * the writer side is expected to finish writing the hidx first followed
+        * by the PTE, by using smp_wmb(). pte_set_hash_slot() ensures that.
+        */
+       smp_rmb();
+
+       hidxp = (unsigned long *)(ptep + PTRS_PER_PTE);
+       rpte.hidx = *hidxp;
        return rpte;
 }
 
+/*
+ * shift the hidx representation by one-modulo-0xf; i.e hidx 0 is respresented
+ * as 1, 1 as 2,... , and 0xf as 0.  This convention lets us represent a
+ * invalid hidx 0xf with a 0x0 bit value. PTEs are anyway zero'd when
+ * allocated. We dont have to zero them gain; thus save on the initialization.
+ */
+#define HIDX_UNSHIFT_BY_ONE(x) ((x + 0xfUL) & 0xfUL) /* shift backward by one */
+#define HIDX_SHIFT_BY_ONE(x) ((x + 0x1UL) & 0xfUL)   /* shift forward by one */
+#define HIDX_BITS(x, index)  (x << (index << 2))
+#define BITS_TO_HIDX(x, index)  ((x >> (index << 2)) & 0xfUL)
+#define INVALID_RPTE_HIDX  0x0UL
+
 static inline unsigned long __rpte_to_hidx(real_pte_t rpte, unsigned long index)
 {
-       if ((pte_val(rpte.pte) & H_PAGE_COMBO))
-               return (rpte.hidx >> (index<<2)) & 0xf;
-       return (pte_val(rpte.pte) >> H_PAGE_F_GIX_SHIFT) & 0xf;
+       return HIDX_UNSHIFT_BY_ONE(BITS_TO_HIDX(rpte.hidx, index));
+}
+
+/*
+ * Commit the hidx and return PTE bits that needs to be modified. The caller is
+ * expected to modify the PTE bits accordingly and commit the PTE to memory.
+ */
+static inline unsigned long pte_set_hidx(pte_t *ptep, real_pte_t rpte,
+               unsigned int subpg_index, unsigned long hidx)
+{
+       unsigned long *hidxp = (unsigned long *)(ptep + PTRS_PER_PTE);
+
+       rpte.hidx &= ~HIDX_BITS(0xfUL, subpg_index);
+       *hidxp = rpte.hidx  | HIDX_BITS(HIDX_SHIFT_BY_ONE(hidx), subpg_index);
+
+       /*
+        * Anyone reading PTE must ensure hidx bits are read after reading the
+        * PTE by using the read-side barrier smp_rmb(). __real_pte() can be
+        * used for that.
+        */
+       smp_wmb();
+
+       /* No PTE bits to be modified, return 0x0UL */
+       return 0x0UL;
 }
 
 #define __rpte_to_pte(r)       ((r).pte)