arch/riscv/kernel/cpufeature.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copied from arch/arm64/kernel/cpufeature.c
   4  *
   5  * Copyright (C) 2015 ARM Ltd.
   6  * Copyright (C) 2017 SiFive
   7  */
   8
   9 #include <linux/bitmap.h>
  10 #include <linux/ctype.h>
  11 #include <linux/libfdt.h>
  12 #include <linux/log2.h>
  13 #include <linux/memory.h>
  14 #include <linux/module.h>
  15 #include <linux/of.h>
  16 #include <asm/alternative.h>
  17 #include <asm/cacheflush.h>
  18 #include <asm/errata_list.h>
  19 #include <asm/hwcap.h>
  20 #include <asm/patch.h>
  21 #include <asm/pgtable.h>
  22 #include <asm/processor.h>
  23 #include <asm/smp.h>
  24 #include <asm/switch_to.h>
  25
  26 #define NUM_ALPHA_EXTS ('z' - 'a' + 1)
  27
  28 unsigned long elf_hwcap __read_mostly;
  29
  30 /* Host ISA bitmap */
  31 static DECLARE_BITMAP(riscv_isa, RISCV_ISA_EXT_MAX) __read_mostly;
  32
  33 /**
  34  * riscv_isa_extension_base() - Get base extension word
  35  *
  36  * @isa_bitmap: ISA bitmap to use
  37  * Return: base extension word as unsigned long value
  38  *
  39  * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used.
  40  */
  41 unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap)
  42 {
  43         if (!isa_bitmap)
  44                 return riscv_isa[0];
  45         return isa_bitmap[0];
  46 }
  47 EXPORT_SYMBOL_GPL(riscv_isa_extension_base);
  48
  49 /**
  50  * __riscv_isa_extension_available() - Check whether given extension
  51  * is available or not
  52  *
  53  * @isa_bitmap: ISA bitmap to use
  54  * @bit: bit position of the desired extension
  55  * Return: true or false
  56  *
  57  * NOTE: If isa_bitmap is NULL then Host ISA bitmap will be used.
  58  */
  59 bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, int bit)
  60 {
  61         const unsigned long *bmap = (isa_bitmap) ? isa_bitmap : riscv_isa;
  62
  63         if (bit >= RISCV_ISA_EXT_MAX)
  64                 return false;
  65
  66         return test_bit(bit, bmap) ? true : false;
  67 }
  68 EXPORT_SYMBOL_GPL(__riscv_isa_extension_available);
  69
  70 static bool riscv_isa_extension_check(int id)
  71 {
  72         switch (id) {
  73         case RISCV_ISA_EXT_ZICBOM:
  74                 if (!riscv_cbom_block_size) {
  75                         pr_err("Zicbom detected in ISA string, but no cbom-block-size found\n");
  76                         return false;
  77                 } else if (!is_power_of_2(riscv_cbom_block_size)) {
  78                         pr_err("cbom-block-size present, but is not a power-of-2\n");
  79                         return false;
  80                 }
  81                 return true;
  82         }
  83
  84         return true;
  85 }
  86
  87 void __init riscv_fill_hwcap(void)
  88 {
  89         struct device_node *node;
  90         const char *isa;
  91         char print_str[NUM_ALPHA_EXTS + 1];
  92         int i, j, rc;
  93         unsigned long isa2hwcap[26] = {0};
  94         unsigned long hartid;
  95
  96         isa2hwcap['i' - 'a'] = COMPAT_HWCAP_ISA_I;
  97         isa2hwcap['m' - 'a'] = COMPAT_HWCAP_ISA_M;
  98         isa2hwcap['a' - 'a'] = COMPAT_HWCAP_ISA_A;
  99         isa2hwcap['f' - 'a'] = COMPAT_HWCAP_ISA_F;
 100         isa2hwcap['d' - 'a'] = COMPAT_HWCAP_ISA_D;
 101         isa2hwcap['c' - 'a'] = COMPAT_HWCAP_ISA_C;
 102
 103         elf_hwcap = 0;
 104
 105         bitmap_zero(riscv_isa, RISCV_ISA_EXT_MAX);
 106
 107         for_each_of_cpu_node(node) {
 108                 unsigned long this_hwcap = 0;
 109                 DECLARE_BITMAP(this_isa, RISCV_ISA_EXT_MAX);
 110                 const char *temp;
 111
 112                 rc = riscv_of_processor_hartid(node, &hartid);
 113                 if (rc < 0)
 114                         continue;
 115
 116                 if (of_property_read_string(node, "riscv,isa", &isa)) {
 117                         pr_warn("Unable to find \"riscv,isa\" devicetree entry\n");
 118                         continue;
 119                 }
 120
 121                 temp = isa;
 122 #if IS_ENABLED(CONFIG_32BIT)
 123                 if (!strncmp(isa, "rv32", 4))
 124                         isa += 4;
 125 #elif IS_ENABLED(CONFIG_64BIT)
 126                 if (!strncmp(isa, "rv64", 4))
 127                         isa += 4;
 128 #endif
 129                 /* The riscv,isa DT property must start with rv64 or rv32 */
 130                 if (temp == isa)
 131                         continue;
 132                 bitmap_zero(this_isa, RISCV_ISA_EXT_MAX);
 133                 for (; *isa; ++isa) {
 134                         const char *ext = isa++;
 135                         const char *ext_end = isa;
 136                         bool ext_long = false, ext_err = false;
 137
 138                         switch (*ext) {
 139                         case 's':
 140                                 /**
 141                                  * Workaround for invalid single-letter 's' & 'u'(QEMU).
 142                                  * No need to set the bit in riscv_isa as 's' & 'u' are
 143                                  * not valid ISA extensions. It works until multi-letter
 144                                  * extension starting with "Su" appears.
 145                                  */
 146                                 if (ext[-1] != '_' && ext[1] == 'u') {
 147                                         ++isa;
 148                                         ext_err = true;
 149                                         break;
 150                                 }
 151                                 fallthrough;
 152                         case 'x':
 153                         case 'z':
 154                                 ext_long = true;
 155                                 /* Multi-letter extension must be delimited */
 156                                 for (; *isa && *isa != '_'; ++isa)
 157                                         if (unlikely(!islower(*isa)
 158                                                      && !isdigit(*isa)))
 159                                                 ext_err = true;
 160                                 /* Parse backwards */
 161                                 ext_end = isa;
 162                                 if (unlikely(ext_err))
 163                                         break;
 164                                 if (!isdigit(ext_end[-1]))
 165                                         break;
 166                                 /* Skip the minor version */
 167                                 while (isdigit(*--ext_end))
 168                                         ;
 169                                 if (ext_end[0] != 'p'
 170                                     || !isdigit(ext_end[-1])) {
 171                                         /* Advance it to offset the pre-decrement */
 172                                         ++ext_end;
 173                                         break;
 174                                 }
 175                                 /* Skip the major version */
 176                                 while (isdigit(*--ext_end))
 177                                         ;
 178                                 ++ext_end;
 179                                 break;
 180                         default:
 181                                 if (unlikely(!islower(*ext))) {
 182                                         ext_err = true;
 183                                         break;
 184                                 }
 185                                 /* Find next extension */
 186                                 if (!isdigit(*isa))
 187                                         break;
 188                                 /* Skip the minor version */
 189                                 while (isdigit(*++isa))
 190                                         ;
 191                                 if (*isa != 'p')
 192                                         break;
 193                                 if (!isdigit(*++isa)) {
 194                                         --isa;
 195                                         break;
 196                                 }
 197                                 /* Skip the major version */
 198                                 while (isdigit(*++isa))
 199                                         ;
 200                                 break;
 201                         }
 202                         if (*isa != '_')
 203                                 --isa;
 204
 205 #define SET_ISA_EXT_MAP(name, bit)                                              \
 206                         do {                                                    \
 207                                 if ((ext_end - ext == sizeof(name) - 1) &&      \
 208                                      !memcmp(ext, name, sizeof(name) - 1) &&    \
 209                                      riscv_isa_extension_check(bit))            \
 210                                         set_bit(bit, this_isa);                 \
 211                         } while (false)                                         \
 212
 213                         if (unlikely(ext_err))
 214                                 continue;
 215                         if (!ext_long) {
 216                                 int nr = *ext - 'a';
 217
 218                                 if (riscv_isa_extension_check(nr)) {
 219                                         this_hwcap |= isa2hwcap[nr];
 220                                         set_bit(nr, this_isa);
 221                                 }
 222                         } else {
 223                                 /* sorted alphabetically */
 224                                 SET_ISA_EXT_MAP("sscofpmf", RISCV_ISA_EXT_SSCOFPMF);
 225                                 SET_ISA_EXT_MAP("sstc", RISCV_ISA_EXT_SSTC);
 226                                 SET_ISA_EXT_MAP("svinval", RISCV_ISA_EXT_SVINVAL);
 227                                 SET_ISA_EXT_MAP("svnapot", RISCV_ISA_EXT_SVNAPOT);
 228                                 SET_ISA_EXT_MAP("svpbmt", RISCV_ISA_EXT_SVPBMT);
 229                                 SET_ISA_EXT_MAP("zbb", RISCV_ISA_EXT_ZBB);
 230                                 SET_ISA_EXT_MAP("zicbom", RISCV_ISA_EXT_ZICBOM);
 231                                 SET_ISA_EXT_MAP("zihintpause", RISCV_ISA_EXT_ZIHINTPAUSE);
 232                         }
 233 #undef SET_ISA_EXT_MAP
 234                 }
 235
 236                 /*
 237                  * All "okay" hart should have same isa. Set HWCAP based on
 238                  * common capabilities of every "okay" hart, in case they don't
 239                  * have.
 240                  */
 241                 if (elf_hwcap)
 242                         elf_hwcap &= this_hwcap;
 243                 else
 244                         elf_hwcap = this_hwcap;
 245
 246                 if (bitmap_empty(riscv_isa, RISCV_ISA_EXT_MAX))
 247                         bitmap_copy(riscv_isa, this_isa, RISCV_ISA_EXT_MAX);
 248                 else
 249                         bitmap_and(riscv_isa, riscv_isa, this_isa, RISCV_ISA_EXT_MAX);
 250         }
 251
 252         /* We don't support systems with F but without D, so mask those out
 253          * here. */
 254         if ((elf_hwcap & COMPAT_HWCAP_ISA_F) && !(elf_hwcap & COMPAT_HWCAP_ISA_D)) {
 255                 pr_info("This kernel does not support systems with F but not D\n");
 256                 elf_hwcap &= ~COMPAT_HWCAP_ISA_F;
 257         }
 258
 259         memset(print_str, 0, sizeof(print_str));
 260         for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++)
 261                 if (riscv_isa[0] & BIT_MASK(i))
 262                         print_str[j++] = (char)('a' + i);
 263         pr_info("riscv: base ISA extensions %s\n", print_str);
 264
 265         memset(print_str, 0, sizeof(print_str));
 266         for (i = 0, j = 0; i < NUM_ALPHA_EXTS; i++)
 267                 if (elf_hwcap & BIT_MASK(i))
 268                         print_str[j++] = (char)('a' + i);
 269         pr_info("riscv: ELF capabilities %s\n", print_str);
 270 }
 271
 272 #ifdef CONFIG_RISCV_ALTERNATIVE
 273 void __init_or_module riscv_cpufeature_patch_func(struct alt_entry *begin,
 274                                                   struct alt_entry *end,
 275                                                   unsigned int stage)
 276 {
 277         struct alt_entry *alt;
 278         void *oldptr, *altptr;
 279
 280         if (stage == RISCV_ALTERNATIVES_EARLY_BOOT)
 281                 return;
 282
 283         for (alt = begin; alt < end; alt++) {
 284                 if (alt->vendor_id != 0)
 285                         continue;
 286                 if (alt->errata_id >= RISCV_ISA_EXT_MAX) {
 287                         WARN(1, "This extension id:%d is not in ISA extension list",
 288                                 alt->errata_id);
 289                         continue;
 290                 }
 291
 292                 if (!__riscv_isa_extension_available(NULL, alt->errata_id))
 293                         continue;
 294
 295                 oldptr = ALT_OLD_PTR(alt);
 296                 altptr = ALT_ALT_PTR(alt);
 297
 298                 mutex_lock(&text_mutex);
 299                 patch_text_nosync(oldptr, altptr, alt->alt_len);
 300                 riscv_alternative_fix_offsets(oldptr, alt->alt_len, oldptr - altptr);
 301                 mutex_unlock(&text_mutex);
 302         }
 303 }
 304 #endif