mm/kasan/hw_tags.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * This file contains core hardware tag-based KASAN code.
   4  *
   5  * Copyright (c) 2020 Google, Inc.
   6  * Author: Andrey Konovalov <andreyknvl@google.com>
   7  */
   8
   9 #define pr_fmt(fmt) "kasan: " fmt
  10
  11 #include <linux/init.h>
  12 #include <linux/kasan.h>
  13 #include <linux/kernel.h>
  14 #include <linux/memory.h>
  15 #include <linux/mm.h>
  16 #include <linux/static_key.h>
  17 #include <linux/string.h>
  18 #include <linux/types.h>
  19
  20 #include "kasan.h"
  21
  22 enum kasan_arg {
  23         KASAN_ARG_DEFAULT,
  24         KASAN_ARG_OFF,
  25         KASAN_ARG_ON,
  26 };
  27
  28 enum kasan_arg_mode {
  29         KASAN_ARG_MODE_DEFAULT,
  30         KASAN_ARG_MODE_SYNC,
  31         KASAN_ARG_MODE_ASYNC,
  32 };
  33
  34 enum kasan_arg_stacktrace {
  35         KASAN_ARG_STACKTRACE_DEFAULT,
  36         KASAN_ARG_STACKTRACE_OFF,
  37         KASAN_ARG_STACKTRACE_ON,
  38 };
  39
  40 static enum kasan_arg kasan_arg __ro_after_init;
  41 static enum kasan_arg_mode kasan_arg_mode __ro_after_init;
  42 static enum kasan_arg_stacktrace kasan_arg_stacktrace __ro_after_init;
  43
  44 /* Whether KASAN is enabled at all. */
  45 DEFINE_STATIC_KEY_FALSE(kasan_flag_enabled);
  46 EXPORT_SYMBOL(kasan_flag_enabled);
  47
  48 /* Whether the asynchronous mode is enabled. */
  49 bool kasan_flag_async __ro_after_init;
  50 EXPORT_SYMBOL_GPL(kasan_flag_async);
  51
  52 /* Whether to collect alloc/free stack traces. */
  53 DEFINE_STATIC_KEY_FALSE(kasan_flag_stacktrace);
  54
  55 /* kasan=off/on */
  56 static int __init early_kasan_flag(char *arg)
  57 {
  58         if (!arg)
  59                 return -EINVAL;
  60
  61         if (!strcmp(arg, "off"))
  62                 kasan_arg = KASAN_ARG_OFF;
  63         else if (!strcmp(arg, "on"))
  64                 kasan_arg = KASAN_ARG_ON;
  65         else
  66                 return -EINVAL;
  67
  68         return 0;
  69 }
  70 early_param("kasan", early_kasan_flag);
  71
  72 /* kasan.mode=sync/async */
  73 static int __init early_kasan_mode(char *arg)
  74 {
  75         if (!arg)
  76                 return -EINVAL;
  77
  78         if (!strcmp(arg, "sync"))
  79                 kasan_arg_mode = KASAN_ARG_MODE_SYNC;
  80         else if (!strcmp(arg, "async"))
  81                 kasan_arg_mode = KASAN_ARG_MODE_ASYNC;
  82         else
  83                 return -EINVAL;
  84
  85         return 0;
  86 }
  87 early_param("kasan.mode", early_kasan_mode);
  88
  89 /* kasan.stacktrace=off/on */
  90 static int __init early_kasan_flag_stacktrace(char *arg)
  91 {
  92         if (!arg)
  93                 return -EINVAL;
  94
  95         if (!strcmp(arg, "off"))
  96                 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_OFF;
  97         else if (!strcmp(arg, "on"))
  98                 kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_ON;
  99         else
 100                 return -EINVAL;
 101
 102         return 0;
 103 }
 104 early_param("kasan.stacktrace", early_kasan_flag_stacktrace);
 105
 106 /* kasan_init_hw_tags_cpu() is called for each CPU. */
 107 void kasan_init_hw_tags_cpu(void)
 108 {
 109         /*
 110          * There's no need to check that the hardware is MTE-capable here,
 111          * as this function is only called for MTE-capable hardware.
 112          */
 113
 114         /* If KASAN is disabled via command line, don't initialize it. */
 115         if (kasan_arg == KASAN_ARG_OFF)
 116                 return;
 117
 118         /*
 119          * Enable async mode only when explicitly requested through
 120          * the command line.
 121          */
 122         if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC)
 123                 hw_enable_tagging_async();
 124         else
 125                 hw_enable_tagging_sync();
 126 }
 127
 128 /* kasan_init_hw_tags() is called once on boot CPU. */
 129 void __init kasan_init_hw_tags(void)
 130 {
 131         /* If hardware doesn't support MTE, don't initialize KASAN. */
 132         if (!system_supports_mte())
 133                 return;
 134
 135         /* If KASAN is disabled via command line, don't initialize it. */
 136         if (kasan_arg == KASAN_ARG_OFF)
 137                 return;
 138
 139         /* Enable KASAN. */
 140         static_branch_enable(&kasan_flag_enabled);
 141
 142         switch (kasan_arg_mode) {
 143         case KASAN_ARG_MODE_DEFAULT:
 144                 /*
 145                  * Default to sync mode.
 146                  * Do nothing, kasan_flag_async keeps its default value.
 147                  */
 148                 break;
 149         case KASAN_ARG_MODE_SYNC:
 150                 /* Do nothing, kasan_flag_async keeps its default value. */
 151                 break;
 152         case KASAN_ARG_MODE_ASYNC:
 153                 /* Async mode enabled. */
 154                 kasan_flag_async = true;
 155                 break;
 156         }
 157
 158         switch (kasan_arg_stacktrace) {
 159         case KASAN_ARG_STACKTRACE_DEFAULT:
 160                 /* Default to enabling stack trace collection. */
 161                 static_branch_enable(&kasan_flag_stacktrace);
 162                 break;
 163         case KASAN_ARG_STACKTRACE_OFF:
 164                 /* Do nothing, kasan_flag_stacktrace keeps its default value. */
 165                 break;
 166         case KASAN_ARG_STACKTRACE_ON:
 167                 static_branch_enable(&kasan_flag_stacktrace);
 168                 break;
 169         }
 170
 171         pr_info("KernelAddressSanitizer initialized\n");
 172 }
 173
 174 void kasan_alloc_pages(struct page *page, unsigned int order, gfp_t flags)
 175 {
 176         /*
 177          * This condition should match the one in post_alloc_hook() in
 178          * page_alloc.c.
 179          */
 180         bool init = !want_init_on_free() && want_init_on_alloc(flags);
 181
 182         if (flags & __GFP_SKIP_KASAN_POISON)
 183                 SetPageSkipKASanPoison(page);
 184
 185         if (flags & __GFP_ZEROTAGS) {
 186                 int i;
 187
 188                 for (i = 0; i != 1 << order; ++i)
 189                         tag_clear_highpage(page + i);
 190         } else {
 191                 kasan_unpoison_pages(page, order, init);
 192         }
 193 }
 194
 195 void kasan_free_pages(struct page *page, unsigned int order)
 196 {
 197         /*
 198          * This condition should match the one in free_pages_prepare() in
 199          * page_alloc.c.
 200          */
 201         bool init = want_init_on_free();
 202
 203         kasan_poison_pages(page, order, init);
 204 }
 205
 206 #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
 207
 208 void kasan_enable_tagging_sync(void)
 209 {
 210         hw_enable_tagging_sync();
 211 }
 212 EXPORT_SYMBOL_GPL(kasan_enable_tagging_sync);
 213
 214 void kasan_force_async_fault(void)
 215 {
 216         hw_force_async_tag_fault();
 217 }
 218 EXPORT_SYMBOL_GPL(kasan_force_async_fault);
 219
 220 #endif