arch/s390/mm/kasan_init.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/kasan.h>
   3 #include <linux/sched/task.h>
   4 #include <linux/pgtable.h>
   5 #include <asm/pgalloc.h>
   6 #include <asm/kasan.h>
   7 #include <asm/mem_detect.h>
   8 #include <asm/processor.h>
   9 #include <asm/sclp.h>
  10 #include <asm/facility.h>
  11 #include <asm/sections.h>
  12 #include <asm/setup.h>
  13 #include <asm/uv.h>
  14
  15 static unsigned long segment_pos __initdata;
  16 static unsigned long segment_low __initdata;
  17 static bool has_edat __initdata;
  18 static bool has_nx __initdata;
  19
  20 #define __sha(x) ((unsigned long)kasan_mem_to_shadow((void *)x))
  21
  22 static void __init kasan_early_panic(const char *reason)
  23 {
  24         sclp_early_printk("The Linux kernel failed to boot with the KernelAddressSanitizer:\n");
  25         sclp_early_printk(reason);
  26         disabled_wait();
  27 }
  28
  29 static void * __init kasan_early_alloc_segment(void)
  30 {
  31         segment_pos -= _SEGMENT_SIZE;
  32
  33         if (segment_pos < segment_low)
  34                 kasan_early_panic("out of memory during initialisation\n");
  35
  36         return __va(segment_pos);
  37 }
  38
  39 static void * __init kasan_early_alloc_pages(unsigned int order)
  40 {
  41         pgalloc_pos -= (PAGE_SIZE << order);
  42
  43         if (pgalloc_pos < pgalloc_low)
  44                 kasan_early_panic("out of memory during initialisation\n");
  45
  46         return __va(pgalloc_pos);
  47 }
  48
  49 static void * __init kasan_early_crst_alloc(unsigned long val)
  50 {
  51         unsigned long *table;
  52
  53         table = kasan_early_alloc_pages(CRST_ALLOC_ORDER);
  54         if (table)
  55                 crst_table_init(table, val);
  56         return table;
  57 }
  58
  59 static pte_t * __init kasan_early_pte_alloc(void)
  60 {
  61         static void *pte_leftover;
  62         pte_t *pte;
  63
  64         BUILD_BUG_ON(_PAGE_TABLE_SIZE * 2 != PAGE_SIZE);
  65
  66         if (!pte_leftover) {
  67                 pte_leftover = kasan_early_alloc_pages(0);
  68                 pte = pte_leftover + _PAGE_TABLE_SIZE;
  69         } else {
  70                 pte = pte_leftover;
  71                 pte_leftover = NULL;
  72         }
  73         memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE);
  74         return pte;
  75 }
  76
  77 enum populate_mode {
  78         POPULATE_MAP,
  79         POPULATE_ZERO_SHADOW,
  80         POPULATE_SHALLOW
  81 };
  82
  83 static inline pgprot_t pgprot_clear_bit(pgprot_t pgprot, unsigned long bit)
  84 {
  85         return __pgprot(pgprot_val(pgprot) & ~bit);
  86 }
  87
  88 static void __init kasan_early_pgtable_populate(unsigned long address,
  89                                                 unsigned long end,
  90                                                 enum populate_mode mode)
  91 {
  92         pgprot_t pgt_prot_zero = PAGE_KERNEL_RO;
  93         pgprot_t pgt_prot = PAGE_KERNEL;
  94         pgprot_t sgt_prot = SEGMENT_KERNEL;
  95         pgd_t *pg_dir;
  96         p4d_t *p4_dir;
  97         pud_t *pu_dir;
  98         pmd_t *pm_dir;
  99         pte_t *pt_dir;
 100         pmd_t pmd;
 101         pte_t pte;
 102
 103         if (!has_nx) {
 104                 pgt_prot_zero = pgprot_clear_bit(pgt_prot_zero, _PAGE_NOEXEC);
 105                 pgt_prot = pgprot_clear_bit(pgt_prot, _PAGE_NOEXEC);
 106                 sgt_prot = pgprot_clear_bit(sgt_prot, _SEGMENT_ENTRY_NOEXEC);
 107         }
 108
 109         while (address < end) {
 110                 pg_dir = pgd_offset_k(address);
 111                 if (pgd_none(*pg_dir)) {
 112                         if (mode == POPULATE_ZERO_SHADOW &&
 113                             IS_ALIGNED(address, PGDIR_SIZE) &&
 114                             end - address >= PGDIR_SIZE) {
 115                                 pgd_populate(&init_mm, pg_dir,
 116                                                 kasan_early_shadow_p4d);
 117                                 address = (address + PGDIR_SIZE) & PGDIR_MASK;
 118                                 continue;
 119                         }
 120                         p4_dir = kasan_early_crst_alloc(_REGION2_ENTRY_EMPTY);
 121                         pgd_populate(&init_mm, pg_dir, p4_dir);
 122                 }
 123
 124                 if (mode == POPULATE_SHALLOW) {
 125                         address = (address + P4D_SIZE) & P4D_MASK;
 126                         continue;
 127                 }
 128
 129                 p4_dir = p4d_offset(pg_dir, address);
 130                 if (p4d_none(*p4_dir)) {
 131                         if (mode == POPULATE_ZERO_SHADOW &&
 132                             IS_ALIGNED(address, P4D_SIZE) &&
 133                             end - address >= P4D_SIZE) {
 134                                 p4d_populate(&init_mm, p4_dir,
 135                                                 kasan_early_shadow_pud);
 136                                 address = (address + P4D_SIZE) & P4D_MASK;
 137                                 continue;
 138                         }
 139                         pu_dir = kasan_early_crst_alloc(_REGION3_ENTRY_EMPTY);
 140                         p4d_populate(&init_mm, p4_dir, pu_dir);
 141                 }
 142
 143                 pu_dir = pud_offset(p4_dir, address);
 144                 if (pud_none(*pu_dir)) {
 145                         if (mode == POPULATE_ZERO_SHADOW &&
 146                             IS_ALIGNED(address, PUD_SIZE) &&
 147                             end - address >= PUD_SIZE) {
 148                                 pud_populate(&init_mm, pu_dir,
 149                                                 kasan_early_shadow_pmd);
 150                                 address = (address + PUD_SIZE) & PUD_MASK;
 151                                 continue;
 152                         }
 153                         pm_dir = kasan_early_crst_alloc(_SEGMENT_ENTRY_EMPTY);
 154                         pud_populate(&init_mm, pu_dir, pm_dir);
 155                 }
 156
 157                 pm_dir = pmd_offset(pu_dir, address);
 158                 if (pmd_none(*pm_dir)) {
 159                         if (IS_ALIGNED(address, PMD_SIZE) &&
 160                             end - address >= PMD_SIZE) {
 161                                 if (mode == POPULATE_ZERO_SHADOW) {
 162                                         pmd_populate(&init_mm, pm_dir, kasan_early_shadow_pte);
 163                                         address = (address + PMD_SIZE) & PMD_MASK;
 164                                         continue;
 165                                 } else if (has_edat) {
 166                                         void *page = kasan_early_alloc_segment();
 167
 168                                         memset(page, 0, _SEGMENT_SIZE);
 169                                         pmd = __pmd(__pa(page));
 170                                         pmd = set_pmd_bit(pmd, sgt_prot);
 171                                         set_pmd(pm_dir, pmd);
 172                                         address = (address + PMD_SIZE) & PMD_MASK;
 173                                         continue;
 174                                 }
 175                         }
 176                         pt_dir = kasan_early_pte_alloc();
 177                         pmd_populate(&init_mm, pm_dir, pt_dir);
 178                 } else if (pmd_large(*pm_dir)) {
 179                         address = (address + PMD_SIZE) & PMD_MASK;
 180                         continue;
 181                 }
 182
 183                 pt_dir = pte_offset_kernel(pm_dir, address);
 184                 if (pte_none(*pt_dir)) {
 185                         void *page;
 186
 187                         switch (mode) {
 188                         case POPULATE_MAP:
 189                                 page = kasan_early_alloc_pages(0);
 190                                 memset(page, 0, PAGE_SIZE);
 191                                 pte = __pte(__pa(page));
 192                                 pte = set_pte_bit(pte, pgt_prot);
 193                                 set_pte(pt_dir, pte);
 194                                 break;
 195                         case POPULATE_ZERO_SHADOW:
 196                                 page = kasan_early_shadow_page;
 197                                 pte = __pte(__pa(page));
 198                                 pte = set_pte_bit(pte, pgt_prot_zero);
 199                                 set_pte(pt_dir, pte);
 200                                 break;
 201                         case POPULATE_SHALLOW:
 202                                 /* should never happen */
 203                                 break;
 204                         }
 205                 }
 206                 address += PAGE_SIZE;
 207         }
 208 }
 209
 210 static void __init kasan_early_detect_facilities(void)
 211 {
 212         if (test_facility(8)) {
 213                 has_edat = true;
 214                 __ctl_set_bit(0, 23);
 215         }
 216         if (!noexec_disabled && test_facility(130)) {
 217                 has_nx = true;
 218                 __ctl_set_bit(0, 20);
 219         }
 220 }
 221
 222 void __init kasan_early_init(void)
 223 {
 224         pte_t pte_z = __pte(__pa(kasan_early_shadow_page) | pgprot_val(PAGE_KERNEL_RO));
 225         pmd_t pmd_z = __pmd(__pa(kasan_early_shadow_pte) | _SEGMENT_ENTRY);
 226         pud_t pud_z = __pud(__pa(kasan_early_shadow_pmd) | _REGION3_ENTRY);
 227         p4d_t p4d_z = __p4d(__pa(kasan_early_shadow_pud) | _REGION2_ENTRY);
 228         unsigned long untracked_end = MODULES_VADDR;
 229         unsigned long shadow_alloc_size;
 230         unsigned long start, end;
 231         int i;
 232
 233         kasan_early_detect_facilities();
 234         if (!has_nx)
 235                 pte_z = clear_pte_bit(pte_z, __pgprot(_PAGE_NOEXEC));
 236
 237         BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, P4D_SIZE));
 238         BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, P4D_SIZE));
 239
 240         /* init kasan zero shadow */
 241         crst_table_init((unsigned long *)kasan_early_shadow_p4d, p4d_val(p4d_z));
 242         crst_table_init((unsigned long *)kasan_early_shadow_pud, pud_val(pud_z));
 243         crst_table_init((unsigned long *)kasan_early_shadow_pmd, pmd_val(pmd_z));
 244         memset64((u64 *)kasan_early_shadow_pte, pte_val(pte_z), PTRS_PER_PTE);
 245
 246         if (has_edat) {
 247                 shadow_alloc_size = get_mem_detect_usable_total() >> KASAN_SHADOW_SCALE_SHIFT;
 248                 segment_pos = round_down(pgalloc_pos, _SEGMENT_SIZE);
 249                 segment_low = segment_pos - shadow_alloc_size;
 250                 segment_low = round_down(segment_low, _SEGMENT_SIZE);
 251                 pgalloc_pos = segment_low;
 252         }
 253         /*
 254          * Current memory layout:
 255          * +- 0 -------------+         +- shadow start -+
 256          * |1:1 ident mapping|        /|1/8 of ident map|
 257          * |                 |       / |                |
 258          * +-end of ident map+      /  +----------------+
 259          * | ... gap ...     |     /   |    kasan       |
 260          * |                 |    /    |  zero page     |
 261          * +- vmalloc area  -+   /     |   mapping      |
 262          * | vmalloc_size    |  /      | (untracked)    |
 263          * +- modules vaddr -+ /       +----------------+
 264          * | 2Gb             |/        |    unmapped    | allocated per module
 265          * +- shadow start  -+         +----------------+
 266          * | 1/8 addr space  |         | zero pg mapping| (untracked)
 267          * +- shadow end ----+---------+- shadow end ---+
 268          *
 269          * Current memory layout (KASAN_VMALLOC):
 270          * +- 0 -------------+         +- shadow start -+
 271          * |1:1 ident mapping|        /|1/8 of ident map|
 272          * |                 |       / |                |
 273          * +-end of ident map+      /  +----------------+
 274          * | ... gap ...     |     /   | kasan zero page| (untracked)
 275          * |                 |    /    | mapping        |
 276          * +- vmalloc area  -+   /     +----------------+
 277          * | vmalloc_size    |  /      |shallow populate|
 278          * +- modules vaddr -+ /       +----------------+
 279          * | 2Gb             |/        |shallow populate|
 280          * +- shadow start  -+         +----------------+
 281          * | 1/8 addr space  |         | zero pg mapping| (untracked)
 282          * +- shadow end ----+---------+- shadow end ---+
 283          */
 284         /* populate kasan shadow (for identity mapping and zero page mapping) */
 285         for_each_mem_detect_usable_block(i, &start, &end)
 286                 kasan_early_pgtable_populate(__sha(start), __sha(end), POPULATE_MAP);
 287         if (IS_ENABLED(CONFIG_KASAN_VMALLOC)) {
 288                 untracked_end = VMALLOC_START;
 289                 /* shallowly populate kasan shadow for vmalloc and modules */
 290                 kasan_early_pgtable_populate(__sha(VMALLOC_START), __sha(MODULES_END),
 291                                              POPULATE_SHALLOW);
 292         }
 293         /* populate kasan shadow for untracked memory */
 294         kasan_early_pgtable_populate(__sha(ident_map_size), __sha(untracked_end),
 295                                      POPULATE_ZERO_SHADOW);
 296         kasan_early_pgtable_populate(__sha(MODULES_END), __sha(_REGION1_SIZE),
 297                                      POPULATE_ZERO_SHADOW);
 298         /* enable kasan */
 299         init_task.kasan_depth = 0;
 300         sclp_early_printk("KernelAddressSanitizer initialized\n");
 301 }