mm/kasan/hw_tags.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * This file contains core hardware tag-based KASAN code.
   4  *
   5  * Copyright (c) 2020 Google, Inc.
   6  * Author: Andrey Konovalov <andreyknvl@google.com>
   7  */
   8
   9 #define pr_fmt(fmt) "kasan: " fmt
  10
  11 #include <linux/init.h>
  12 #include <linux/kasan.h>
  13 #include <linux/kernel.h>
  14 #include <linux/memory.h>
  15 #include <linux/mm.h>
  16 #include <linux/static_key.h>
  17 #include <linux/string.h>
  18 #include <linux/types.h>
  19
  20 #include "kasan.h"
  21
  22 enum kasan_arg {
  23         KASAN_ARG_DEFAULT,
  24         KASAN_ARG_OFF,
  25         KASAN_ARG_ON,
  26 };
  27
  28 enum kasan_arg_mode {
  29         KASAN_ARG_MODE_DEFAULT,
  30         KASAN_ARG_MODE_SYNC,
  31         KASAN_ARG_MODE_ASYNC,
  32         KASAN_ARG_MODE_ASYMM,
  33 };
  34
  35 enum kasan_arg_stacktrace {
  36         KASAN_ARG_STACKTRACE_DEFAULT,
  37         KASAN_ARG_STACKTRACE_OFF,
  38         KASAN_ARG_STACKTRACE_ON,
  39 };
  40
  41 static enum kasan_arg kasan_arg __ro_after_init;
  42 static enum kasan_arg_mode kasan_arg_mode __ro_after_init;
  43 static enum kasan_arg_stacktrace kasan_arg_stacktrace __ro_after_init;
  44
  45 /* Whether KASAN is enabled at all. */
  46 DEFINE_STATIC_KEY_FALSE(kasan_flag_enabled);
  47 EXPORT_SYMBOL(kasan_flag_enabled);
  48
  49 /* Whether the selected mode is synchronous/asynchronous/asymmetric.*/
  50 enum kasan_mode kasan_mode __ro_after_init;
  51 EXPORT_SYMBOL_GPL(kasan_mode);
  52
  53 /* Whether to collect alloc/free stack traces. */
  54 DEFINE_STATIC_KEY_FALSE(kasan_flag_stacktrace);
  55
  56 /* kasan=off/on */
  57 static int __init early_kasan_flag(char *arg)
  58 {
  59         if (!arg)
  60                 return -EINVAL;
  61
  62         if (!strcmp(arg, "off"))
  63                 kasan_arg = KASAN_ARG_OFF;
  64         else if (!strcmp(arg, "on"))
  65                 kasan_arg = KASAN_ARG_ON;
  66         else
  67                 return -EINVAL;
  68
  69         return 0;
  70 }
  71 early_param("kasan", early_kasan_flag);
  72
  73 /* kasan.mode=sync/async/asymm */
  74 static int __init early_kasan_mode(char *arg)
  75 {
  76         if (!arg)
  77                 return -EINVAL;
  78
  79         if (!strcmp(arg, "sync"))
  80                 kasan_arg_mode = KASAN_ARG_MODE_SYNC;
  81         else if (!strcmp(arg, "async"))
  82                 kasan_arg_mode = KASAN_ARG_MODE_ASYNC;
  83         else if (!strcmp(arg, "asymm"))
  84                 kasan_arg_mode = KASAN_ARG_MODE_ASYMM;
  85         else
  86                 return -EINVAL;
  87
  88         return 0;
  89 }
  90 early_param("kasan.mode", early_kasan_mode);
  91
  92 /* kasan.stacktrace=off/on */
  93 static int __init early_kasan_flag_stacktrace(char *arg)
  94 {
  95         if (!arg)
  96                 return -EINVAL;
  97
  98         if (!strcmp(arg, "off"))
  99                 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_OFF;
 100         else if (!strcmp(arg, "on"))
 101                 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_ON;
 102         else
 103                 return -EINVAL;
 104
 105         return 0;
 106 }
 107 early_param("kasan.stacktrace", early_kasan_flag_stacktrace);
 108
 109 static inline const char *kasan_mode_info(void)
 110 {
 111         if (kasan_mode == KASAN_MODE_ASYNC)
 112                 return "async";
 113         else if (kasan_mode == KASAN_MODE_ASYMM)
 114                 return "asymm";
 115         else
 116                 return "sync";
 117 }
 118
 119 /* kasan_init_hw_tags_cpu() is called for each CPU. */
 120 void kasan_init_hw_tags_cpu(void)
 121 {
 122         /*
 123          * There's no need to check that the hardware is MTE-capable here,
 124          * as this function is only called for MTE-capable hardware.
 125          */
 126
 127         /* If KASAN is disabled via command line, don't initialize it. */
 128         if (kasan_arg == KASAN_ARG_OFF)
 129                 return;
 130
 131         /*
 132          * Enable async or asymm modes only when explicitly requested
 133          * through the command line.
 134          */
 135         if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC)
 136                 hw_enable_tagging_async();
 137         else if (kasan_arg_mode == KASAN_ARG_MODE_ASYMM)
 138                 hw_enable_tagging_asymm();
 139         else
 140                 hw_enable_tagging_sync();
 141 }
 142
 143 /* kasan_init_hw_tags() is called once on boot CPU. */
 144 void __init kasan_init_hw_tags(void)
 145 {
 146         /* If hardware doesn't support MTE, don't initialize KASAN. */
 147         if (!system_supports_mte())
 148                 return;
 149
 150         /* If KASAN is disabled via command line, don't initialize it. */
 151         if (kasan_arg == KASAN_ARG_OFF)
 152                 return;
 153
 154         /* Enable KASAN. */
 155         static_branch_enable(&kasan_flag_enabled);
 156
 157         switch (kasan_arg_mode) {
 158         case KASAN_ARG_MODE_DEFAULT:
 159                 /*
 160                  * Default to sync mode.
 161                  */
 162                 fallthrough;
 163         case KASAN_ARG_MODE_SYNC:
 164                 /* Sync mode enabled. */
 165                 kasan_mode = KASAN_MODE_SYNC;
 166                 break;
 167         case KASAN_ARG_MODE_ASYNC:
 168                 /* Async mode enabled. */
 169                 kasan_mode = KASAN_MODE_ASYNC;
 170                 break;
 171         case KASAN_ARG_MODE_ASYMM:
 172                 /* Asymm mode enabled. */
 173                 kasan_mode = KASAN_MODE_ASYMM;
 174                 break;
 175         }
 176
 177         switch (kasan_arg_stacktrace) {
 178         case KASAN_ARG_STACKTRACE_DEFAULT:
 179                 /* Default to enabling stack trace collection. */
 180                 static_branch_enable(&kasan_flag_stacktrace);
 181                 break;
 182         case KASAN_ARG_STACKTRACE_OFF:
 183                 /* Do nothing, kasan_flag_stacktrace keeps its default value. */
 184                 break;
 185         case KASAN_ARG_STACKTRACE_ON:
 186                 static_branch_enable(&kasan_flag_stacktrace);
 187                 break;
 188         }
 189
 190         pr_info("KernelAddressSanitizer initialized (hw-tags, mode=%s, stacktrace=%s)\n",
 191                 kasan_mode_info(),
 192                 kasan_stack_collection_enabled() ? "on" : "off");
 193 }
 194
 195 void kasan_alloc_pages(struct page *page, unsigned int order, gfp_t flags)
 196 {
 197         /*
 198          * This condition should match the one in post_alloc_hook() in
 199          * page_alloc.c.
 200          */
 201         bool init = !want_init_on_free() && want_init_on_alloc(flags);
 202
 203         if (flags & __GFP_SKIP_KASAN_POISON)
 204                 SetPageSkipKASanPoison(page);
 205
 206         if (flags & __GFP_ZEROTAGS) {
 207                 int i;
 208
 209                 for (i = 0; i != 1 << order; ++i)
 210                         tag_clear_highpage(page + i);
 211         } else {
 212                 kasan_unpoison_pages(page, order, init);
 213         }
 214 }
 215
 216 void kasan_free_pages(struct page *page, unsigned int order)
 217 {
 218         /*
 219          * This condition should match the one in free_pages_prepare() in
 220          * page_alloc.c.
 221          */
 222         bool init = want_init_on_free();
 223
 224         kasan_poison_pages(page, order, init);
 225 }
 226
 227 #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
 228
 229 void kasan_enable_tagging_sync(void)
 230 {
 231         hw_enable_tagging_sync();
 232 }
 233 EXPORT_SYMBOL_GPL(kasan_enable_tagging_sync);
 234
 235 void kasan_force_async_fault(void)
 236 {
 237         hw_force_async_tag_fault();
 238 }
 239 EXPORT_SYMBOL_GPL(kasan_force_async_fault);
 240
 241 #endif