mm/kasan/hw_tags.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * This file contains core hardware tag-based KASAN code.
   4  *
   5  * Copyright (c) 2020 Google, Inc.
   6  * Author: Andrey Konovalov <andreyknvl@google.com>
   7  */
   8
   9 #define pr_fmt(fmt) "kasan: " fmt
  10
  11 #include <linux/init.h>
  12 #include <linux/kasan.h>
  13 #include <linux/kernel.h>
  14 #include <linux/memory.h>
  15 #include <linux/mm.h>
  16 #include <linux/static_key.h>
  17 #include <linux/string.h>
  18 #include <linux/types.h>
  19
  20 #include "kasan.h"
  21
  22 enum kasan_arg {
  23         KASAN_ARG_DEFAULT,
  24         KASAN_ARG_OFF,
  25         KASAN_ARG_ON,
  26 };
  27
  28 enum kasan_arg_mode {
  29         KASAN_ARG_MODE_DEFAULT,
  30         KASAN_ARG_MODE_SYNC,
  31         KASAN_ARG_MODE_ASYNC,
  32         KASAN_ARG_MODE_ASYMM,
  33 };
  34
  35 enum kasan_arg_stacktrace {
  36         KASAN_ARG_STACKTRACE_DEFAULT,
  37         KASAN_ARG_STACKTRACE_OFF,
  38         KASAN_ARG_STACKTRACE_ON,
  39 };
  40
  41 static enum kasan_arg kasan_arg __ro_after_init;
  42 static enum kasan_arg_mode kasan_arg_mode __ro_after_init;
  43 static enum kasan_arg_stacktrace kasan_arg_stacktrace __ro_after_init;
  44
  45 /* Whether KASAN is enabled at all. */
  46 DEFINE_STATIC_KEY_FALSE(kasan_flag_enabled);
  47 EXPORT_SYMBOL(kasan_flag_enabled);
  48
  49 /* Whether the selected mode is synchronous/asynchronous/asymmetric.*/
  50 enum kasan_mode kasan_mode __ro_after_init;
  51 EXPORT_SYMBOL_GPL(kasan_mode);
  52
  53 /* Whether to collect alloc/free stack traces. */
  54 DEFINE_STATIC_KEY_FALSE(kasan_flag_stacktrace);
  55
  56 /* kasan=off/on */
  57 static int __init early_kasan_flag(char *arg)
  58 {
  59         if (!arg)
  60                 return -EINVAL;
  61
  62         if (!strcmp(arg, "off"))
  63                 kasan_arg = KASAN_ARG_OFF;
  64         else if (!strcmp(arg, "on"))
  65                 kasan_arg = KASAN_ARG_ON;
  66         else
  67                 return -EINVAL;
  68
  69         return 0;
  70 }
  71 early_param("kasan", early_kasan_flag);
  72
  73 /* kasan.mode=sync/async/asymm */
  74 static int __init early_kasan_mode(char *arg)
  75 {
  76         if (!arg)
  77                 return -EINVAL;
  78
  79         if (!strcmp(arg, "sync"))
  80                 kasan_arg_mode = KASAN_ARG_MODE_SYNC;
  81         else if (!strcmp(arg, "async"))
  82                 kasan_arg_mode = KASAN_ARG_MODE_ASYNC;
  83         else if (!strcmp(arg, "asymm"))
  84                 kasan_arg_mode = KASAN_ARG_MODE_ASYMM;
  85         else
  86                 return -EINVAL;
  87
  88         return 0;
  89 }
  90 early_param("kasan.mode", early_kasan_mode);
  91
  92 /* kasan.stacktrace=off/on */
  93 static int __init early_kasan_flag_stacktrace(char *arg)
  94 {
  95         if (!arg)
  96                 return -EINVAL;
  97
  98         if (!strcmp(arg, "off"))
  99                 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_OFF;
 100         else if (!strcmp(arg, "on"))
 101                 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_ON;
 102         else
 103                 return -EINVAL;
 104
 105         return 0;
 106 }
 107 early_param("kasan.stacktrace", early_kasan_flag_stacktrace);
 108
 109 /* kasan_init_hw_tags_cpu() is called for each CPU. */
 110 void kasan_init_hw_tags_cpu(void)
 111 {
 112         /*
 113          * There's no need to check that the hardware is MTE-capable here,
 114          * as this function is only called for MTE-capable hardware.
 115          */
 116
 117         /* If KASAN is disabled via command line, don't initialize it. */
 118         if (kasan_arg == KASAN_ARG_OFF)
 119                 return;
 120
 121         /*
 122          * Enable async or asymm modes only when explicitly requested
 123          * through the command line.
 124          */
 125         if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC)
 126                 hw_enable_tagging_async();
 127         else if (kasan_arg_mode == KASAN_ARG_MODE_ASYMM)
 128                 hw_enable_tagging_asymm();
 129         else
 130                 hw_enable_tagging_sync();
 131 }
 132
 133 /* kasan_init_hw_tags() is called once on boot CPU. */
 134 void __init kasan_init_hw_tags(void)
 135 {
 136         /* If hardware doesn't support MTE, don't initialize KASAN. */
 137         if (!system_supports_mte())
 138                 return;
 139
 140         /* If KASAN is disabled via command line, don't initialize it. */
 141         if (kasan_arg == KASAN_ARG_OFF)
 142                 return;
 143
 144         /* Enable KASAN. */
 145         static_branch_enable(&kasan_flag_enabled);
 146
 147         switch (kasan_arg_mode) {
 148         case KASAN_ARG_MODE_DEFAULT:
 149                 /*
 150                  * Default to sync mode.
 151                  */
 152                 fallthrough;
 153         case KASAN_ARG_MODE_SYNC:
 154                 /* Sync mode enabled. */
 155                 kasan_mode = KASAN_MODE_SYNC;
 156                 break;
 157         case KASAN_ARG_MODE_ASYNC:
 158                 /* Async mode enabled. */
 159                 kasan_mode = KASAN_MODE_ASYNC;
 160                 break;
 161         case KASAN_ARG_MODE_ASYMM:
 162                 /* Asymm mode enabled. */
 163                 kasan_mode = KASAN_MODE_ASYMM;
 164                 break;
 165         }
 166
 167         switch (kasan_arg_stacktrace) {
 168         case KASAN_ARG_STACKTRACE_DEFAULT:
 169                 /* Default to enabling stack trace collection. */
 170                 static_branch_enable(&kasan_flag_stacktrace);
 171                 break;
 172         case KASAN_ARG_STACKTRACE_OFF:
 173                 /* Do nothing, kasan_flag_stacktrace keeps its default value. */
 174                 break;
 175         case KASAN_ARG_STACKTRACE_ON:
 176                 static_branch_enable(&kasan_flag_stacktrace);
 177                 break;
 178         }
 179
 180         pr_info("KernelAddressSanitizer initialized\n");
 181 }
 182
 183 void kasan_alloc_pages(struct page *page, unsigned int order, gfp_t flags)
 184 {
 185         /*
 186          * This condition should match the one in post_alloc_hook() in
 187          * page_alloc.c.
 188          */
 189         bool init = !want_init_on_free() && want_init_on_alloc(flags);
 190
 191         if (flags & __GFP_SKIP_KASAN_POISON)
 192                 SetPageSkipKASanPoison(page);
 193
 194         if (flags & __GFP_ZEROTAGS) {
 195                 int i;
 196
 197                 for (i = 0; i != 1 << order; ++i)
 198                         tag_clear_highpage(page + i);
 199         } else {
 200                 kasan_unpoison_pages(page, order, init);
 201         }
 202 }
 203
 204 void kasan_free_pages(struct page *page, unsigned int order)
 205 {
 206         /*
 207          * This condition should match the one in free_pages_prepare() in
 208          * page_alloc.c.
 209          */
 210         bool init = want_init_on_free();
 211
 212         kasan_poison_pages(page, order, init);
 213 }
 214
 215 #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
 216
 217 void kasan_enable_tagging_sync(void)
 218 {
 219         hw_enable_tagging_sync();
 220 }
 221 EXPORT_SYMBOL_GPL(kasan_enable_tagging_sync);
 222
 223 void kasan_force_async_fault(void)
 224 {
 225         hw_force_async_tag_fault();
 226 }
 227 EXPORT_SYMBOL_GPL(kasan_force_async_fault);
 228
 229 #endif