arch/x86/kernel/cpu/bugs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Copyright (C) 1994  Linus Torvalds
   4  *
   5  *  Cyrix stuff, June 1998 by:
   6  *      - Rafael R. Reilova (moved everything from head.S),
   7  *        <rreilova@ececs.uc.edu>
   8  *      - Channing Corn (tests & fixes),
   9  *      - Andrew D. Balsa (code cleanup).
  10  */
  11 #include <linux/init.h>
  12 #include <linux/utsname.h>
  13 #include <linux/cpu.h>
  14 #include <linux/module.h>
  15 #include <linux/nospec.h>
  16 #include <linux/prctl.h>
  17 #include <linux/sched/smt.h>
  18
  19 #include <asm/spec-ctrl.h>
  20 #include <asm/cmdline.h>
  21 #include <asm/bugs.h>
  22 #include <asm/processor.h>
  23 #include <asm/processor-flags.h>
  24 #include <asm/fpu/internal.h>
  25 #include <asm/msr.h>
  26 #include <asm/vmx.h>
  27 #include <asm/paravirt.h>
  28 #include <asm/alternative.h>
  29 #include <asm/pgtable.h>
  30 #include <asm/set_memory.h>
  31 #include <asm/intel-family.h>
  32 #include <asm/e820/api.h>
  33 #include <asm/hypervisor.h>
  34
  35 #include "cpu.h"
  36
  37 static void __init spectre_v1_select_mitigation(void);
  38 static void __init spectre_v2_select_mitigation(void);
  39 static void __init ssb_select_mitigation(void);
  40 static void __init l1tf_select_mitigation(void);
  41 static void __init mds_select_mitigation(void);
  42 static void __init mds_print_mitigation(void);
  43 static void __init taa_select_mitigation(void);
  44
  45 /* The base value of the SPEC_CTRL MSR that always has to be preserved. */
  46 u64 x86_spec_ctrl_base;
  47 EXPORT_SYMBOL_GPL(x86_spec_ctrl_base);
  48 static DEFINE_MUTEX(spec_ctrl_mutex);
  49
  50 /*
  51  * The vendor and possibly platform specific bits which can be modified in
  52  * x86_spec_ctrl_base.
  53  */
  54 static u64 __ro_after_init x86_spec_ctrl_mask = SPEC_CTRL_IBRS;
  55
  56 /*
  57  * AMD specific MSR info for Speculative Store Bypass control.
  58  * x86_amd_ls_cfg_ssbd_mask is initialized in identify_boot_cpu().
  59  */
  60 u64 __ro_after_init x86_amd_ls_cfg_base;
  61 u64 __ro_after_init x86_amd_ls_cfg_ssbd_mask;
  62
  63 /* Control conditional STIBP in switch_to() */
  64 DEFINE_STATIC_KEY_FALSE(switch_to_cond_stibp);
  65 /* Control conditional IBPB in switch_mm() */
  66 DEFINE_STATIC_KEY_FALSE(switch_mm_cond_ibpb);
  67 /* Control unconditional IBPB in switch_mm() */
  68 DEFINE_STATIC_KEY_FALSE(switch_mm_always_ibpb);
  69
  70 /* Control MDS CPU buffer clear before returning to user space */
  71 DEFINE_STATIC_KEY_FALSE(mds_user_clear);
  72 EXPORT_SYMBOL_GPL(mds_user_clear);
  73 /* Control MDS CPU buffer clear before idling (halt, mwait) */
  74 DEFINE_STATIC_KEY_FALSE(mds_idle_clear);
  75 EXPORT_SYMBOL_GPL(mds_idle_clear);
  76
  77 void __init check_bugs(void)
  78 {
  79         identify_boot_cpu();
  80
  81         /*
  82          * identify_boot_cpu() initialized SMT support information, let the
  83          * core code know.
  84          */
  85         cpu_smt_check_topology();
  86
  87         if (!IS_ENABLED(CONFIG_SMP)) {
  88                 pr_info("CPU: ");
  89                 print_cpu_info(&boot_cpu_data);
  90         }
  91
  92         /*
  93          * Read the SPEC_CTRL MSR to account for reserved bits which may
  94          * have unknown values. AMD64_LS_CFG MSR is cached in the early AMD
  95          * init code as it is not enumerated and depends on the family.
  96          */
  97         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
  98                 rdmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
  99
 100         /* Allow STIBP in MSR_SPEC_CTRL if supported */
 101         if (boot_cpu_has(X86_FEATURE_STIBP))
 102                 x86_spec_ctrl_mask |= SPEC_CTRL_STIBP;
 103
 104         /* Select the proper CPU mitigations before patching alternatives: */
 105         spectre_v1_select_mitigation();
 106         spectre_v2_select_mitigation();
 107         ssb_select_mitigation();
 108         l1tf_select_mitigation();
 109         mds_select_mitigation();
 110         taa_select_mitigation();
 111
 112         /*
 113          * As MDS and TAA mitigations are inter-related, print MDS
 114          * mitigation until after TAA mitigation selection is done.
 115          */
 116         mds_print_mitigation();
 117
 118         arch_smt_update();
 119
 120 #ifdef CONFIG_X86_32
 121         /*
 122          * Check whether we are able to run this kernel safely on SMP.
 123          *
 124          * - i386 is no longer supported.
 125          * - In order to run on anything without a TSC, we need to be
 126          *   compiled for a i486.
 127          */
 128         if (boot_cpu_data.x86 < 4)
 129                 panic("Kernel requires i486+ for 'invlpg' and other features");
 130
 131         init_utsname()->machine[1] =
 132                 '0' + (boot_cpu_data.x86 > 6 ? 6 : boot_cpu_data.x86);
 133         alternative_instructions();
 134
 135         fpu__init_check_bugs();
 136 #else /* CONFIG_X86_64 */
 137         alternative_instructions();
 138
 139         /*
 140          * Make sure the first 2MB area is not mapped by huge pages
 141          * There are typically fixed size MTRRs in there and overlapping
 142          * MTRRs into large pages causes slow downs.
 143          *
 144          * Right now we don't do that with gbpages because there seems
 145          * very little benefit for that case.
 146          */
 147         if (!direct_gbpages)
 148                 set_memory_4k((unsigned long)__va(0), 1);
 149 #endif
 150 }
 151
 152 void
 153 x86_virt_spec_ctrl(u64 guest_spec_ctrl, u64 guest_virt_spec_ctrl, bool setguest)
 154 {
 155         u64 msrval, guestval, hostval = x86_spec_ctrl_base;
 156         struct thread_info *ti = current_thread_info();
 157
 158         /* Is MSR_SPEC_CTRL implemented ? */
 159         if (static_cpu_has(X86_FEATURE_MSR_SPEC_CTRL)) {
 160                 /*
 161                  * Restrict guest_spec_ctrl to supported values. Clear the
 162                  * modifiable bits in the host base value and or the
 163                  * modifiable bits from the guest value.
 164                  */
 165                 guestval = hostval & ~x86_spec_ctrl_mask;
 166                 guestval |= guest_spec_ctrl & x86_spec_ctrl_mask;
 167
 168                 /* SSBD controlled in MSR_SPEC_CTRL */
 169                 if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
 170                     static_cpu_has(X86_FEATURE_AMD_SSBD))
 171                         hostval |= ssbd_tif_to_spec_ctrl(ti->flags);
 172
 173                 /* Conditional STIBP enabled? */
 174                 if (static_branch_unlikely(&switch_to_cond_stibp))
 175                         hostval |= stibp_tif_to_spec_ctrl(ti->flags);
 176
 177                 if (hostval != guestval) {
 178                         msrval = setguest ? guestval : hostval;
 179                         wrmsrl(MSR_IA32_SPEC_CTRL, msrval);
 180                 }
 181         }
 182
 183         /*
 184          * If SSBD is not handled in MSR_SPEC_CTRL on AMD, update
 185          * MSR_AMD64_L2_CFG or MSR_VIRT_SPEC_CTRL if supported.
 186          */
 187         if (!static_cpu_has(X86_FEATURE_LS_CFG_SSBD) &&
 188             !static_cpu_has(X86_FEATURE_VIRT_SSBD))
 189                 return;
 190
 191         /*
 192          * If the host has SSBD mitigation enabled, force it in the host's
 193          * virtual MSR value. If its not permanently enabled, evaluate
 194          * current's TIF_SSBD thread flag.
 195          */
 196         if (static_cpu_has(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE))
 197                 hostval = SPEC_CTRL_SSBD;
 198         else
 199                 hostval = ssbd_tif_to_spec_ctrl(ti->flags);
 200
 201         /* Sanitize the guest value */
 202         guestval = guest_virt_spec_ctrl & SPEC_CTRL_SSBD;
 203
 204         if (hostval != guestval) {
 205                 unsigned long tif;
 206
 207                 tif = setguest ? ssbd_spec_ctrl_to_tif(guestval) :
 208                                  ssbd_spec_ctrl_to_tif(hostval);
 209
 210                 speculation_ctrl_update(tif);
 211         }
 212 }
 213 EXPORT_SYMBOL_GPL(x86_virt_spec_ctrl);
 214
 215 static void x86_amd_ssb_disable(void)
 216 {
 217         u64 msrval = x86_amd_ls_cfg_base | x86_amd_ls_cfg_ssbd_mask;
 218
 219         if (boot_cpu_has(X86_FEATURE_VIRT_SSBD))
 220                 wrmsrl(MSR_AMD64_VIRT_SPEC_CTRL, SPEC_CTRL_SSBD);
 221         else if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD))
 222                 wrmsrl(MSR_AMD64_LS_CFG, msrval);
 223 }
 224
 225 #undef pr_fmt
 226 #define pr_fmt(fmt)     "MDS: " fmt
 227
 228 /* Default mitigation for MDS-affected CPUs */
 229 static enum mds_mitigations mds_mitigation __ro_after_init = MDS_MITIGATION_FULL;
 230 static bool mds_nosmt __ro_after_init = false;
 231
 232 static const char * const mds_strings[] = {
 233         [MDS_MITIGATION_OFF]    = "Vulnerable",
 234         [MDS_MITIGATION_FULL]   = "Mitigation: Clear CPU buffers",
 235         [MDS_MITIGATION_VMWERV] = "Vulnerable: Clear CPU buffers attempted, no microcode",
 236 };
 237
 238 static void __init mds_select_mitigation(void)
 239 {
 240         if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off()) {
 241                 mds_mitigation = MDS_MITIGATION_OFF;
 242                 return;
 243         }
 244
 245         if (mds_mitigation == MDS_MITIGATION_FULL) {
 246                 if (!boot_cpu_has(X86_FEATURE_MD_CLEAR))
 247                         mds_mitigation = MDS_MITIGATION_VMWERV;
 248
 249                 static_branch_enable(&mds_user_clear);
 250
 251                 if (!boot_cpu_has(X86_BUG_MSBDS_ONLY) &&
 252                     (mds_nosmt || cpu_mitigations_auto_nosmt()))
 253                         cpu_smt_disable(false);
 254         }
 255 }
 256
 257 static void __init mds_print_mitigation(void)
 258 {
 259         if (!boot_cpu_has_bug(X86_BUG_MDS) || cpu_mitigations_off())
 260                 return;
 261
 262         pr_info("%s\n", mds_strings[mds_mitigation]);
 263 }
 264
 265 static int __init mds_cmdline(char *str)
 266 {
 267         if (!boot_cpu_has_bug(X86_BUG_MDS))
 268                 return 0;
 269
 270         if (!str)
 271                 return -EINVAL;
 272
 273         if (!strcmp(str, "off"))
 274                 mds_mitigation = MDS_MITIGATION_OFF;
 275         else if (!strcmp(str, "full"))
 276                 mds_mitigation = MDS_MITIGATION_FULL;
 277         else if (!strcmp(str, "full,nosmt")) {
 278                 mds_mitigation = MDS_MITIGATION_FULL;
 279                 mds_nosmt = true;
 280         }
 281
 282         return 0;
 283 }
 284 early_param("mds", mds_cmdline);
 285
 286 #undef pr_fmt
 287 #define pr_fmt(fmt)     "TAA: " fmt
 288
 289 enum taa_mitigations {
 290         TAA_MITIGATION_OFF,
 291         TAA_MITIGATION_UCODE_NEEDED,
 292         TAA_MITIGATION_VERW,
 293         TAA_MITIGATION_TSX_DISABLED,
 294 };
 295
 296 /* Default mitigation for TAA-affected CPUs */
 297 static enum taa_mitigations taa_mitigation __ro_after_init = TAA_MITIGATION_VERW;
 298 static bool taa_nosmt __ro_after_init;
 299
 300 static const char * const taa_strings[] = {
 301         [TAA_MITIGATION_OFF]            = "Vulnerable",
 302         [TAA_MITIGATION_UCODE_NEEDED]   = "Vulnerable: Clear CPU buffers attempted, no microcode",
 303         [TAA_MITIGATION_VERW]           = "Mitigation: Clear CPU buffers",
 304         [TAA_MITIGATION_TSX_DISABLED]   = "Mitigation: TSX disabled",
 305 };
 306
 307 static void __init taa_select_mitigation(void)
 308 {
 309         u64 ia32_cap;
 310
 311         if (!boot_cpu_has_bug(X86_BUG_TAA)) {
 312                 taa_mitigation = TAA_MITIGATION_OFF;
 313                 return;
 314         }
 315
 316         /* TSX previously disabled by tsx=off */
 317         if (!boot_cpu_has(X86_FEATURE_RTM)) {
 318                 taa_mitigation = TAA_MITIGATION_TSX_DISABLED;
 319                 goto out;
 320         }
 321
 322         if (cpu_mitigations_off()) {
 323                 taa_mitigation = TAA_MITIGATION_OFF;
 324                 return;
 325         }
 326
 327         /*
 328          * TAA mitigation via VERW is turned off if both
 329          * tsx_async_abort=off and mds=off are specified.
 330          */
 331         if (taa_mitigation == TAA_MITIGATION_OFF &&
 332             mds_mitigation == MDS_MITIGATION_OFF)
 333                 goto out;
 334
 335         if (boot_cpu_has(X86_FEATURE_MD_CLEAR))
 336                 taa_mitigation = TAA_MITIGATION_VERW;
 337         else
 338                 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
 339
 340         /*
 341          * VERW doesn't clear the CPU buffers when MD_CLEAR=1 and MDS_NO=1.
 342          * A microcode update fixes this behavior to clear CPU buffers. It also
 343          * adds support for MSR_IA32_TSX_CTRL which is enumerated by the
 344          * ARCH_CAP_TSX_CTRL_MSR bit.
 345          *
 346          * On MDS_NO=1 CPUs if ARCH_CAP_TSX_CTRL_MSR is not set, microcode
 347          * update is required.
 348          */
 349         ia32_cap = x86_read_arch_cap_msr();
 350         if ( (ia32_cap & ARCH_CAP_MDS_NO) &&
 351             !(ia32_cap & ARCH_CAP_TSX_CTRL_MSR))
 352                 taa_mitigation = TAA_MITIGATION_UCODE_NEEDED;
 353
 354         /*
 355          * TSX is enabled, select alternate mitigation for TAA which is
 356          * the same as MDS. Enable MDS static branch to clear CPU buffers.
 357          *
 358          * For guests that can't determine whether the correct microcode is
 359          * present on host, enable the mitigation for UCODE_NEEDED as well.
 360          */
 361         static_branch_enable(&mds_user_clear);
 362
 363         if (taa_nosmt || cpu_mitigations_auto_nosmt())
 364                 cpu_smt_disable(false);
 365
 366         /*
 367          * Update MDS mitigation, if necessary, as the mds_user_clear is
 368          * now enabled for TAA mitigation.
 369          */
 370         if (mds_mitigation == MDS_MITIGATION_OFF &&
 371             boot_cpu_has_bug(X86_BUG_MDS)) {
 372                 mds_mitigation = MDS_MITIGATION_FULL;
 373                 mds_select_mitigation();
 374         }
 375 out:
 376         pr_info("%s\n", taa_strings[taa_mitigation]);
 377 }
 378
 379 static int __init tsx_async_abort_parse_cmdline(char *str)
 380 {
 381         if (!boot_cpu_has_bug(X86_BUG_TAA))
 382                 return 0;
 383
 384         if (!str)
 385                 return -EINVAL;
 386
 387         if (!strcmp(str, "off")) {
 388                 taa_mitigation = TAA_MITIGATION_OFF;
 389         } else if (!strcmp(str, "full")) {
 390                 taa_mitigation = TAA_MITIGATION_VERW;
 391         } else if (!strcmp(str, "full,nosmt")) {
 392                 taa_mitigation = TAA_MITIGATION_VERW;
 393                 taa_nosmt = true;
 394         }
 395
 396         return 0;
 397 }
 398 early_param("tsx_async_abort", tsx_async_abort_parse_cmdline);
 399
 400 #undef pr_fmt
 401 #define pr_fmt(fmt)     "Spectre V1 : " fmt
 402
 403 enum spectre_v1_mitigation {
 404         SPECTRE_V1_MITIGATION_NONE,
 405         SPECTRE_V1_MITIGATION_AUTO,
 406 };
 407
 408 static enum spectre_v1_mitigation spectre_v1_mitigation __ro_after_init =
 409         SPECTRE_V1_MITIGATION_AUTO;
 410
 411 static const char * const spectre_v1_strings[] = {
 412         [SPECTRE_V1_MITIGATION_NONE] = "Vulnerable: __user pointer sanitization and usercopy barriers only; no swapgs barriers",
 413         [SPECTRE_V1_MITIGATION_AUTO] = "Mitigation: usercopy/swapgs barriers and __user pointer sanitization",
 414 };
 415
 416 /*
 417  * Does SMAP provide full mitigation against speculative kernel access to
 418  * userspace?
 419  */
 420 static bool smap_works_speculatively(void)
 421 {
 422         if (!boot_cpu_has(X86_FEATURE_SMAP))
 423                 return false;
 424
 425         /*
 426          * On CPUs which are vulnerable to Meltdown, SMAP does not
 427          * prevent speculative access to user data in the L1 cache.
 428          * Consider SMAP to be non-functional as a mitigation on these
 429          * CPUs.
 430          */
 431         if (boot_cpu_has(X86_BUG_CPU_MELTDOWN))
 432                 return false;
 433
 434         return true;
 435 }
 436
 437 static void __init spectre_v1_select_mitigation(void)
 438 {
 439         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V1) || cpu_mitigations_off()) {
 440                 spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
 441                 return;
 442         }
 443
 444         if (spectre_v1_mitigation == SPECTRE_V1_MITIGATION_AUTO) {
 445                 /*
 446                  * With Spectre v1, a user can speculatively control either
 447                  * path of a conditional swapgs with a user-controlled GS
 448                  * value.  The mitigation is to add lfences to both code paths.
 449                  *
 450                  * If FSGSBASE is enabled, the user can put a kernel address in
 451                  * GS, in which case SMAP provides no protection.
 452                  *
 453                  * [ NOTE: Don't check for X86_FEATURE_FSGSBASE until the
 454                  *         FSGSBASE enablement patches have been merged. ]
 455                  *
 456                  * If FSGSBASE is disabled, the user can only put a user space
 457                  * address in GS.  That makes an attack harder, but still
 458                  * possible if there's no SMAP protection.
 459                  */
 460                 if (!smap_works_speculatively()) {
 461                         /*
 462                          * Mitigation can be provided from SWAPGS itself or
 463                          * PTI as the CR3 write in the Meltdown mitigation
 464                          * is serializing.
 465                          *
 466                          * If neither is there, mitigate with an LFENCE to
 467                          * stop speculation through swapgs.
 468                          */
 469                         if (boot_cpu_has_bug(X86_BUG_SWAPGS) &&
 470                             !boot_cpu_has(X86_FEATURE_PTI))
 471                                 setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_USER);
 472
 473                         /*
 474                          * Enable lfences in the kernel entry (non-swapgs)
 475                          * paths, to prevent user entry from speculatively
 476                          * skipping swapgs.
 477                          */
 478                         setup_force_cpu_cap(X86_FEATURE_FENCE_SWAPGS_KERNEL);
 479                 }
 480         }
 481
 482         pr_info("%s\n", spectre_v1_strings[spectre_v1_mitigation]);
 483 }
 484
 485 static int __init nospectre_v1_cmdline(char *str)
 486 {
 487         spectre_v1_mitigation = SPECTRE_V1_MITIGATION_NONE;
 488         return 0;
 489 }
 490 early_param("nospectre_v1", nospectre_v1_cmdline);
 491
 492 #undef pr_fmt
 493 #define pr_fmt(fmt)     "Spectre V2 : " fmt
 494
 495 static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init =
 496         SPECTRE_V2_NONE;
 497
 498 static enum spectre_v2_user_mitigation spectre_v2_user __ro_after_init =
 499         SPECTRE_V2_USER_NONE;
 500
 501 #ifdef CONFIG_RETPOLINE
 502 static bool spectre_v2_bad_module;
 503
 504 bool retpoline_module_ok(bool has_retpoline)
 505 {
 506         if (spectre_v2_enabled == SPECTRE_V2_NONE || has_retpoline)
 507                 return true;
 508
 509         pr_err("System may be vulnerable to spectre v2\n");
 510         spectre_v2_bad_module = true;
 511         return false;
 512 }
 513
 514 static inline const char *spectre_v2_module_string(void)
 515 {
 516         return spectre_v2_bad_module ? " - vulnerable module loaded" : "";
 517 }
 518 #else
 519 static inline const char *spectre_v2_module_string(void) { return ""; }
 520 #endif
 521
 522 static inline bool match_option(const char *arg, int arglen, const char *opt)
 523 {
 524         int len = strlen(opt);
 525
 526         return len == arglen && !strncmp(arg, opt, len);
 527 }
 528
 529 /* The kernel command line selection for spectre v2 */
 530 enum spectre_v2_mitigation_cmd {
 531         SPECTRE_V2_CMD_NONE,
 532         SPECTRE_V2_CMD_AUTO,
 533         SPECTRE_V2_CMD_FORCE,
 534         SPECTRE_V2_CMD_RETPOLINE,
 535         SPECTRE_V2_CMD_RETPOLINE_GENERIC,
 536         SPECTRE_V2_CMD_RETPOLINE_AMD,
 537 };
 538
 539 enum spectre_v2_user_cmd {
 540         SPECTRE_V2_USER_CMD_NONE,
 541         SPECTRE_V2_USER_CMD_AUTO,
 542         SPECTRE_V2_USER_CMD_FORCE,
 543         SPECTRE_V2_USER_CMD_PRCTL,
 544         SPECTRE_V2_USER_CMD_PRCTL_IBPB,
 545         SPECTRE_V2_USER_CMD_SECCOMP,
 546         SPECTRE_V2_USER_CMD_SECCOMP_IBPB,
 547 };
 548
 549 static const char * const spectre_v2_user_strings[] = {
 550         [SPECTRE_V2_USER_NONE]                  = "User space: Vulnerable",
 551         [SPECTRE_V2_USER_STRICT]                = "User space: Mitigation: STIBP protection",
 552         [SPECTRE_V2_USER_STRICT_PREFERRED]      = "User space: Mitigation: STIBP always-on protection",
 553         [SPECTRE_V2_USER_PRCTL]                 = "User space: Mitigation: STIBP via prctl",
 554         [SPECTRE_V2_USER_SECCOMP]               = "User space: Mitigation: STIBP via seccomp and prctl",
 555 };
 556
 557 static const struct {
 558         const char                      *option;
 559         enum spectre_v2_user_cmd        cmd;
 560         bool                            secure;
 561 } v2_user_options[] __initconst = {
 562         { "auto",               SPECTRE_V2_USER_CMD_AUTO,               false },
 563         { "off",                SPECTRE_V2_USER_CMD_NONE,               false },
 564         { "on",                 SPECTRE_V2_USER_CMD_FORCE,              true  },
 565         { "prctl",              SPECTRE_V2_USER_CMD_PRCTL,              false },
 566         { "prctl,ibpb",         SPECTRE_V2_USER_CMD_PRCTL_IBPB,         false },
 567         { "seccomp",            SPECTRE_V2_USER_CMD_SECCOMP,            false },
 568         { "seccomp,ibpb",       SPECTRE_V2_USER_CMD_SECCOMP_IBPB,       false },
 569 };
 570
 571 static void __init spec_v2_user_print_cond(const char *reason, bool secure)
 572 {
 573         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
 574                 pr_info("spectre_v2_user=%s forced on command line.\n", reason);
 575 }
 576
 577 static enum spectre_v2_user_cmd __init
 578 spectre_v2_parse_user_cmdline(enum spectre_v2_mitigation_cmd v2_cmd)
 579 {
 580         char arg[20];
 581         int ret, i;
 582
 583         switch (v2_cmd) {
 584         case SPECTRE_V2_CMD_NONE:
 585                 return SPECTRE_V2_USER_CMD_NONE;
 586         case SPECTRE_V2_CMD_FORCE:
 587                 return SPECTRE_V2_USER_CMD_FORCE;
 588         default:
 589                 break;
 590         }
 591
 592         ret = cmdline_find_option(boot_command_line, "spectre_v2_user",
 593                                   arg, sizeof(arg));
 594         if (ret < 0)
 595                 return SPECTRE_V2_USER_CMD_AUTO;
 596
 597         for (i = 0; i < ARRAY_SIZE(v2_user_options); i++) {
 598                 if (match_option(arg, ret, v2_user_options[i].option)) {
 599                         spec_v2_user_print_cond(v2_user_options[i].option,
 600                                                 v2_user_options[i].secure);
 601                         return v2_user_options[i].cmd;
 602                 }
 603         }
 604
 605         pr_err("Unknown user space protection option (%s). Switching to AUTO select\n", arg);
 606         return SPECTRE_V2_USER_CMD_AUTO;
 607 }
 608
 609 static void __init
 610 spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd)
 611 {
 612         enum spectre_v2_user_mitigation mode = SPECTRE_V2_USER_NONE;
 613         bool smt_possible = IS_ENABLED(CONFIG_SMP);
 614         enum spectre_v2_user_cmd cmd;
 615
 616         if (!boot_cpu_has(X86_FEATURE_IBPB) && !boot_cpu_has(X86_FEATURE_STIBP))
 617                 return;
 618
 619         if (cpu_smt_control == CPU_SMT_FORCE_DISABLED ||
 620             cpu_smt_control == CPU_SMT_NOT_SUPPORTED)
 621                 smt_possible = false;
 622
 623         cmd = spectre_v2_parse_user_cmdline(v2_cmd);
 624         switch (cmd) {
 625         case SPECTRE_V2_USER_CMD_NONE:
 626                 goto set_mode;
 627         case SPECTRE_V2_USER_CMD_FORCE:
 628                 mode = SPECTRE_V2_USER_STRICT;
 629                 break;
 630         case SPECTRE_V2_USER_CMD_PRCTL:
 631         case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
 632                 mode = SPECTRE_V2_USER_PRCTL;
 633                 break;
 634         case SPECTRE_V2_USER_CMD_AUTO:
 635         case SPECTRE_V2_USER_CMD_SECCOMP:
 636         case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
 637                 if (IS_ENABLED(CONFIG_SECCOMP))
 638                         mode = SPECTRE_V2_USER_SECCOMP;
 639                 else
 640                         mode = SPECTRE_V2_USER_PRCTL;
 641                 break;
 642         }
 643
 644         /*
 645          * At this point, an STIBP mode other than "off" has been set.
 646          * If STIBP support is not being forced, check if STIBP always-on
 647          * is preferred.
 648          */
 649         if (mode != SPECTRE_V2_USER_STRICT &&
 650             boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON))
 651                 mode = SPECTRE_V2_USER_STRICT_PREFERRED;
 652
 653         /* Initialize Indirect Branch Prediction Barrier */
 654         if (boot_cpu_has(X86_FEATURE_IBPB)) {
 655                 setup_force_cpu_cap(X86_FEATURE_USE_IBPB);
 656
 657                 switch (cmd) {
 658                 case SPECTRE_V2_USER_CMD_FORCE:
 659                 case SPECTRE_V2_USER_CMD_PRCTL_IBPB:
 660                 case SPECTRE_V2_USER_CMD_SECCOMP_IBPB:
 661                         static_branch_enable(&switch_mm_always_ibpb);
 662                         break;
 663                 case SPECTRE_V2_USER_CMD_PRCTL:
 664                 case SPECTRE_V2_USER_CMD_AUTO:
 665                 case SPECTRE_V2_USER_CMD_SECCOMP:
 666                         static_branch_enable(&switch_mm_cond_ibpb);
 667                         break;
 668                 default:
 669                         break;
 670                 }
 671
 672                 pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n",
 673                         static_key_enabled(&switch_mm_always_ibpb) ?
 674                         "always-on" : "conditional");
 675         }
 676
 677         /* If enhanced IBRS is enabled no STIBP required */
 678         if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
 679                 return;
 680
 681         /*
 682          * If SMT is not possible or STIBP is not available clear the STIBP
 683          * mode.
 684          */
 685         if (!smt_possible || !boot_cpu_has(X86_FEATURE_STIBP))
 686                 mode = SPECTRE_V2_USER_NONE;
 687 set_mode:
 688         spectre_v2_user = mode;
 689         /* Only print the STIBP mode when SMT possible */
 690         if (smt_possible)
 691                 pr_info("%s\n", spectre_v2_user_strings[mode]);
 692 }
 693
 694 static const char * const spectre_v2_strings[] = {
 695         [SPECTRE_V2_NONE]                       = "Vulnerable",
 696         [SPECTRE_V2_RETPOLINE_GENERIC]          = "Mitigation: Full generic retpoline",
 697         [SPECTRE_V2_RETPOLINE_AMD]              = "Mitigation: Full AMD retpoline",
 698         [SPECTRE_V2_IBRS_ENHANCED]              = "Mitigation: Enhanced IBRS",
 699 };
 700
 701 static const struct {
 702         const char *option;
 703         enum spectre_v2_mitigation_cmd cmd;
 704         bool secure;
 705 } mitigation_options[] __initconst = {
 706         { "off",                SPECTRE_V2_CMD_NONE,              false },
 707         { "on",                 SPECTRE_V2_CMD_FORCE,             true  },
 708         { "retpoline",          SPECTRE_V2_CMD_RETPOLINE,         false },
 709         { "retpoline,amd",      SPECTRE_V2_CMD_RETPOLINE_AMD,     false },
 710         { "retpoline,generic",  SPECTRE_V2_CMD_RETPOLINE_GENERIC, false },
 711         { "auto",               SPECTRE_V2_CMD_AUTO,              false },
 712 };
 713
 714 static void __init spec_v2_print_cond(const char *reason, bool secure)
 715 {
 716         if (boot_cpu_has_bug(X86_BUG_SPECTRE_V2) != secure)
 717                 pr_info("%s selected on command line.\n", reason);
 718 }
 719
 720 static enum spectre_v2_mitigation_cmd __init spectre_v2_parse_cmdline(void)
 721 {
 722         enum spectre_v2_mitigation_cmd cmd = SPECTRE_V2_CMD_AUTO;
 723         char arg[20];
 724         int ret, i;
 725
 726         if (cmdline_find_option_bool(boot_command_line, "nospectre_v2") ||
 727             cpu_mitigations_off())
 728                 return SPECTRE_V2_CMD_NONE;
 729
 730         ret = cmdline_find_option(boot_command_line, "spectre_v2", arg, sizeof(arg));
 731         if (ret < 0)
 732                 return SPECTRE_V2_CMD_AUTO;
 733
 734         for (i = 0; i < ARRAY_SIZE(mitigation_options); i++) {
 735                 if (!match_option(arg, ret, mitigation_options[i].option))
 736                         continue;
 737                 cmd = mitigation_options[i].cmd;
 738                 break;
 739         }
 740
 741         if (i >= ARRAY_SIZE(mitigation_options)) {
 742                 pr_err("unknown option (%s). Switching to AUTO select\n", arg);
 743                 return SPECTRE_V2_CMD_AUTO;
 744         }
 745
 746         if ((cmd == SPECTRE_V2_CMD_RETPOLINE ||
 747              cmd == SPECTRE_V2_CMD_RETPOLINE_AMD ||
 748              cmd == SPECTRE_V2_CMD_RETPOLINE_GENERIC) &&
 749             !IS_ENABLED(CONFIG_RETPOLINE)) {
 750                 pr_err("%s selected but not compiled in. Switching to AUTO select\n", mitigation_options[i].option);
 751                 return SPECTRE_V2_CMD_AUTO;
 752         }
 753
 754         if (cmd == SPECTRE_V2_CMD_RETPOLINE_AMD &&
 755             boot_cpu_data.x86_vendor != X86_VENDOR_HYGON &&
 756             boot_cpu_data.x86_vendor != X86_VENDOR_AMD) {
 757                 pr_err("retpoline,amd selected but CPU is not AMD. Switching to AUTO select\n");
 758                 return SPECTRE_V2_CMD_AUTO;
 759         }
 760
 761         spec_v2_print_cond(mitigation_options[i].option,
 762                            mitigation_options[i].secure);
 763         return cmd;
 764 }
 765
 766 static void __init spectre_v2_select_mitigation(void)
 767 {
 768         enum spectre_v2_mitigation_cmd cmd = spectre_v2_parse_cmdline();
 769         enum spectre_v2_mitigation mode = SPECTRE_V2_NONE;
 770
 771         /*
 772          * If the CPU is not affected and the command line mode is NONE or AUTO
 773          * then nothing to do.
 774          */
 775         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2) &&
 776             (cmd == SPECTRE_V2_CMD_NONE || cmd == SPECTRE_V2_CMD_AUTO))
 777                 return;
 778
 779         switch (cmd) {
 780         case SPECTRE_V2_CMD_NONE:
 781                 return;
 782
 783         case SPECTRE_V2_CMD_FORCE:
 784         case SPECTRE_V2_CMD_AUTO:
 785                 if (boot_cpu_has(X86_FEATURE_IBRS_ENHANCED)) {
 786                         mode = SPECTRE_V2_IBRS_ENHANCED;
 787                         /* Force it so VMEXIT will restore correctly */
 788                         x86_spec_ctrl_base |= SPEC_CTRL_IBRS;
 789                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 790                         goto specv2_set_mode;
 791                 }
 792                 if (IS_ENABLED(CONFIG_RETPOLINE))
 793                         goto retpoline_auto;
 794                 break;
 795         case SPECTRE_V2_CMD_RETPOLINE_AMD:
 796                 if (IS_ENABLED(CONFIG_RETPOLINE))
 797                         goto retpoline_amd;
 798                 break;
 799         case SPECTRE_V2_CMD_RETPOLINE_GENERIC:
 800                 if (IS_ENABLED(CONFIG_RETPOLINE))
 801                         goto retpoline_generic;
 802                 break;
 803         case SPECTRE_V2_CMD_RETPOLINE:
 804                 if (IS_ENABLED(CONFIG_RETPOLINE))
 805                         goto retpoline_auto;
 806                 break;
 807         }
 808         pr_err("Spectre mitigation: kernel not compiled with retpoline; no mitigation available!");
 809         return;
 810
 811 retpoline_auto:
 812         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD ||
 813             boot_cpu_data.x86_vendor == X86_VENDOR_HYGON) {
 814         retpoline_amd:
 815                 if (!boot_cpu_has(X86_FEATURE_LFENCE_RDTSC)) {
 816                         pr_err("Spectre mitigation: LFENCE not serializing, switching to generic retpoline\n");
 817                         goto retpoline_generic;
 818                 }
 819                 mode = SPECTRE_V2_RETPOLINE_AMD;
 820                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE_AMD);
 821                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 822         } else {
 823         retpoline_generic:
 824                 mode = SPECTRE_V2_RETPOLINE_GENERIC;
 825                 setup_force_cpu_cap(X86_FEATURE_RETPOLINE);
 826         }
 827
 828 specv2_set_mode:
 829         spectre_v2_enabled = mode;
 830         pr_info("%s\n", spectre_v2_strings[mode]);
 831
 832         /*
 833          * If spectre v2 protection has been enabled, unconditionally fill
 834          * RSB during a context switch; this protects against two independent
 835          * issues:
 836          *
 837          *      - RSB underflow (and switch to BTB) on Skylake+
 838          *      - SpectreRSB variant of spectre v2 on X86_BUG_SPECTRE_V2 CPUs
 839          */
 840         setup_force_cpu_cap(X86_FEATURE_RSB_CTXSW);
 841         pr_info("Spectre v2 / SpectreRSB mitigation: Filling RSB on context switch\n");
 842
 843         /*
 844          * Retpoline means the kernel is safe because it has no indirect
 845          * branches. Enhanced IBRS protects firmware too, so, enable restricted
 846          * speculation around firmware calls only when Enhanced IBRS isn't
 847          * supported.
 848          *
 849          * Use "mode" to check Enhanced IBRS instead of boot_cpu_has(), because
 850          * the user might select retpoline on the kernel command line and if
 851          * the CPU supports Enhanced IBRS, kernel might un-intentionally not
 852          * enable IBRS around firmware calls.
 853          */
 854         if (boot_cpu_has(X86_FEATURE_IBRS) && mode != SPECTRE_V2_IBRS_ENHANCED) {
 855                 setup_force_cpu_cap(X86_FEATURE_USE_IBRS_FW);
 856                 pr_info("Enabling Restricted Speculation for firmware calls\n");
 857         }
 858
 859         /* Set up IBPB and STIBP depending on the general spectre V2 command */
 860         spectre_v2_user_select_mitigation(cmd);
 861 }
 862
 863 static void update_stibp_msr(void * __unused)
 864 {
 865         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
 866 }
 867
 868 /* Update x86_spec_ctrl_base in case SMT state changed. */
 869 static void update_stibp_strict(void)
 870 {
 871         u64 mask = x86_spec_ctrl_base & ~SPEC_CTRL_STIBP;
 872
 873         if (sched_smt_active())
 874                 mask |= SPEC_CTRL_STIBP;
 875
 876         if (mask == x86_spec_ctrl_base)
 877                 return;
 878
 879         pr_info("Update user space SMT mitigation: STIBP %s\n",
 880                 mask & SPEC_CTRL_STIBP ? "always-on" : "off");
 881         x86_spec_ctrl_base = mask;
 882         on_each_cpu(update_stibp_msr, NULL, 1);
 883 }
 884
 885 /* Update the static key controlling the evaluation of TIF_SPEC_IB */
 886 static void update_indir_branch_cond(void)
 887 {
 888         if (sched_smt_active())
 889                 static_branch_enable(&switch_to_cond_stibp);
 890         else
 891                 static_branch_disable(&switch_to_cond_stibp);
 892 }
 893
 894 #undef pr_fmt
 895 #define pr_fmt(fmt) fmt
 896
 897 /* Update the static key controlling the MDS CPU buffer clear in idle */
 898 static void update_mds_branch_idle(void)
 899 {
 900         /*
 901          * Enable the idle clearing if SMT is active on CPUs which are
 902          * affected only by MSBDS and not any other MDS variant.
 903          *
 904          * The other variants cannot be mitigated when SMT is enabled, so
 905          * clearing the buffers on idle just to prevent the Store Buffer
 906          * repartitioning leak would be a window dressing exercise.
 907          */
 908         if (!boot_cpu_has_bug(X86_BUG_MSBDS_ONLY))
 909                 return;
 910
 911         if (sched_smt_active())
 912                 static_branch_enable(&mds_idle_clear);
 913         else
 914                 static_branch_disable(&mds_idle_clear);
 915 }
 916
 917 #define MDS_MSG_SMT "MDS CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/mds.html for more details.\n"
 918 #define TAA_MSG_SMT "TAA CPU bug present and SMT on, data leak possible. See https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/tsx_async_abort.html for more details.\n"
 919
 920 void cpu_bugs_smt_update(void)
 921 {
 922         mutex_lock(&spec_ctrl_mutex);
 923
 924         switch (spectre_v2_user) {
 925         case SPECTRE_V2_USER_NONE:
 926                 break;
 927         case SPECTRE_V2_USER_STRICT:
 928         case SPECTRE_V2_USER_STRICT_PREFERRED:
 929                 update_stibp_strict();
 930                 break;
 931         case SPECTRE_V2_USER_PRCTL:
 932         case SPECTRE_V2_USER_SECCOMP:
 933                 update_indir_branch_cond();
 934                 break;
 935         }
 936
 937         switch (mds_mitigation) {
 938         case MDS_MITIGATION_FULL:
 939         case MDS_MITIGATION_VMWERV:
 940                 if (sched_smt_active() && !boot_cpu_has(X86_BUG_MSBDS_ONLY))
 941                         pr_warn_once(MDS_MSG_SMT);
 942                 update_mds_branch_idle();
 943                 break;
 944         case MDS_MITIGATION_OFF:
 945                 break;
 946         }
 947
 948         switch (taa_mitigation) {
 949         case TAA_MITIGATION_VERW:
 950         case TAA_MITIGATION_UCODE_NEEDED:
 951                 if (sched_smt_active())
 952                         pr_warn_once(TAA_MSG_SMT);
 953                 break;
 954         case TAA_MITIGATION_TSX_DISABLED:
 955         case TAA_MITIGATION_OFF:
 956                 break;
 957         }
 958
 959         mutex_unlock(&spec_ctrl_mutex);
 960 }
 961
 962 #undef pr_fmt
 963 #define pr_fmt(fmt)     "Speculative Store Bypass: " fmt
 964
 965 static enum ssb_mitigation ssb_mode __ro_after_init = SPEC_STORE_BYPASS_NONE;
 966
 967 /* The kernel command line selection */
 968 enum ssb_mitigation_cmd {
 969         SPEC_STORE_BYPASS_CMD_NONE,
 970         SPEC_STORE_BYPASS_CMD_AUTO,
 971         SPEC_STORE_BYPASS_CMD_ON,
 972         SPEC_STORE_BYPASS_CMD_PRCTL,
 973         SPEC_STORE_BYPASS_CMD_SECCOMP,
 974 };
 975
 976 static const char * const ssb_strings[] = {
 977         [SPEC_STORE_BYPASS_NONE]        = "Vulnerable",
 978         [SPEC_STORE_BYPASS_DISABLE]     = "Mitigation: Speculative Store Bypass disabled",
 979         [SPEC_STORE_BYPASS_PRCTL]       = "Mitigation: Speculative Store Bypass disabled via prctl",
 980         [SPEC_STORE_BYPASS_SECCOMP]     = "Mitigation: Speculative Store Bypass disabled via prctl and seccomp",
 981 };
 982
 983 static const struct {
 984         const char *option;
 985         enum ssb_mitigation_cmd cmd;
 986 } ssb_mitigation_options[]  __initconst = {
 987         { "auto",       SPEC_STORE_BYPASS_CMD_AUTO },    /* Platform decides */
 988         { "on",         SPEC_STORE_BYPASS_CMD_ON },      /* Disable Speculative Store Bypass */
 989         { "off",        SPEC_STORE_BYPASS_CMD_NONE },    /* Don't touch Speculative Store Bypass */
 990         { "prctl",      SPEC_STORE_BYPASS_CMD_PRCTL },   /* Disable Speculative Store Bypass via prctl */
 991         { "seccomp",    SPEC_STORE_BYPASS_CMD_SECCOMP }, /* Disable Speculative Store Bypass via prctl and seccomp */
 992 };
 993
 994 static enum ssb_mitigation_cmd __init ssb_parse_cmdline(void)
 995 {
 996         enum ssb_mitigation_cmd cmd = SPEC_STORE_BYPASS_CMD_AUTO;
 997         char arg[20];
 998         int ret, i;
 999
1000         if (cmdline_find_option_bool(boot_command_line, "nospec_store_bypass_disable") ||
1001             cpu_mitigations_off()) {
1002                 return SPEC_STORE_BYPASS_CMD_NONE;
1003         } else {
1004                 ret = cmdline_find_option(boot_command_line, "spec_store_bypass_disable",
1005                                           arg, sizeof(arg));
1006                 if (ret < 0)
1007                         return SPEC_STORE_BYPASS_CMD_AUTO;
1008
1009                 for (i = 0; i < ARRAY_SIZE(ssb_mitigation_options); i++) {
1010                         if (!match_option(arg, ret, ssb_mitigation_options[i].option))
1011                                 continue;
1012
1013                         cmd = ssb_mitigation_options[i].cmd;
1014                         break;
1015                 }
1016
1017                 if (i >= ARRAY_SIZE(ssb_mitigation_options)) {
1018                         pr_err("unknown option (%s). Switching to AUTO select\n", arg);
1019                         return SPEC_STORE_BYPASS_CMD_AUTO;
1020                 }
1021         }
1022
1023         return cmd;
1024 }
1025
1026 static enum ssb_mitigation __init __ssb_select_mitigation(void)
1027 {
1028         enum ssb_mitigation mode = SPEC_STORE_BYPASS_NONE;
1029         enum ssb_mitigation_cmd cmd;
1030
1031         if (!boot_cpu_has(X86_FEATURE_SSBD))
1032                 return mode;
1033
1034         cmd = ssb_parse_cmdline();
1035         if (!boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS) &&
1036             (cmd == SPEC_STORE_BYPASS_CMD_NONE ||
1037              cmd == SPEC_STORE_BYPASS_CMD_AUTO))
1038                 return mode;
1039
1040         switch (cmd) {
1041         case SPEC_STORE_BYPASS_CMD_AUTO:
1042         case SPEC_STORE_BYPASS_CMD_SECCOMP:
1043                 /*
1044                  * Choose prctl+seccomp as the default mode if seccomp is
1045                  * enabled.
1046                  */
1047                 if (IS_ENABLED(CONFIG_SECCOMP))
1048                         mode = SPEC_STORE_BYPASS_SECCOMP;
1049                 else
1050                         mode = SPEC_STORE_BYPASS_PRCTL;
1051                 break;
1052         case SPEC_STORE_BYPASS_CMD_ON:
1053                 mode = SPEC_STORE_BYPASS_DISABLE;
1054                 break;
1055         case SPEC_STORE_BYPASS_CMD_PRCTL:
1056                 mode = SPEC_STORE_BYPASS_PRCTL;
1057                 break;
1058         case SPEC_STORE_BYPASS_CMD_NONE:
1059                 break;
1060         }
1061
1062         /*
1063          * If SSBD is controlled by the SPEC_CTRL MSR, then set the proper
1064          * bit in the mask to allow guests to use the mitigation even in the
1065          * case where the host does not enable it.
1066          */
1067         if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) ||
1068             static_cpu_has(X86_FEATURE_AMD_SSBD)) {
1069                 x86_spec_ctrl_mask |= SPEC_CTRL_SSBD;
1070         }
1071
1072         /*
1073          * We have three CPU feature flags that are in play here:
1074          *  - X86_BUG_SPEC_STORE_BYPASS - CPU is susceptible.
1075          *  - X86_FEATURE_SSBD - CPU is able to turn off speculative store bypass
1076          *  - X86_FEATURE_SPEC_STORE_BYPASS_DISABLE - engage the mitigation
1077          */
1078         if (mode == SPEC_STORE_BYPASS_DISABLE) {
1079                 setup_force_cpu_cap(X86_FEATURE_SPEC_STORE_BYPASS_DISABLE);
1080                 /*
1081                  * Intel uses the SPEC CTRL MSR Bit(2) for this, while AMD may
1082                  * use a completely different MSR and bit dependent on family.
1083                  */
1084                 if (!static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) &&
1085                     !static_cpu_has(X86_FEATURE_AMD_SSBD)) {
1086                         x86_amd_ssb_disable();
1087                 } else {
1088                         x86_spec_ctrl_base |= SPEC_CTRL_SSBD;
1089                         wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
1090                 }
1091         }
1092
1093         return mode;
1094 }
1095
1096 static void ssb_select_mitigation(void)
1097 {
1098         ssb_mode = __ssb_select_mitigation();
1099
1100         if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
1101                 pr_info("%s\n", ssb_strings[ssb_mode]);
1102 }
1103
1104 #undef pr_fmt
1105 #define pr_fmt(fmt)     "Speculation prctl: " fmt
1106
1107 static void task_update_spec_tif(struct task_struct *tsk)
1108 {
1109         /* Force the update of the real TIF bits */
1110         set_tsk_thread_flag(tsk, TIF_SPEC_FORCE_UPDATE);
1111
1112         /*
1113          * Immediately update the speculation control MSRs for the current
1114          * task, but for a non-current task delay setting the CPU
1115          * mitigation until it is scheduled next.
1116          *
1117          * This can only happen for SECCOMP mitigation. For PRCTL it's
1118          * always the current task.
1119          */
1120         if (tsk == current)
1121                 speculation_ctrl_update_current();
1122 }
1123
1124 static int ssb_prctl_set(struct task_struct *task, unsigned long ctrl)
1125 {
1126         if (ssb_mode != SPEC_STORE_BYPASS_PRCTL &&
1127             ssb_mode != SPEC_STORE_BYPASS_SECCOMP)
1128                 return -ENXIO;
1129
1130         switch (ctrl) {
1131         case PR_SPEC_ENABLE:
1132                 /* If speculation is force disabled, enable is not allowed */
1133                 if (task_spec_ssb_force_disable(task))
1134                         return -EPERM;
1135                 task_clear_spec_ssb_disable(task);
1136                 task_clear_spec_ssb_noexec(task);
1137                 task_update_spec_tif(task);
1138                 break;
1139         case PR_SPEC_DISABLE:
1140                 task_set_spec_ssb_disable(task);
1141                 task_clear_spec_ssb_noexec(task);
1142                 task_update_spec_tif(task);
1143                 break;
1144         case PR_SPEC_FORCE_DISABLE:
1145                 task_set_spec_ssb_disable(task);
1146                 task_set_spec_ssb_force_disable(task);
1147                 task_clear_spec_ssb_noexec(task);
1148                 task_update_spec_tif(task);
1149                 break;
1150         case PR_SPEC_DISABLE_NOEXEC:
1151                 if (task_spec_ssb_force_disable(task))
1152                         return -EPERM;
1153                 task_set_spec_ssb_disable(task);
1154                 task_set_spec_ssb_noexec(task);
1155                 task_update_spec_tif(task);
1156                 break;
1157         default:
1158                 return -ERANGE;
1159         }
1160         return 0;
1161 }
1162
1163 static int ib_prctl_set(struct task_struct *task, unsigned long ctrl)
1164 {
1165         switch (ctrl) {
1166         case PR_SPEC_ENABLE:
1167                 if (spectre_v2_user == SPECTRE_V2_USER_NONE)
1168                         return 0;
1169                 /*
1170                  * Indirect branch speculation is always disabled in strict
1171                  * mode.
1172                  */
1173                 if (spectre_v2_user == SPECTRE_V2_USER_STRICT ||
1174                     spectre_v2_user == SPECTRE_V2_USER_STRICT_PREFERRED)
1175                         return -EPERM;
1176                 task_clear_spec_ib_disable(task);
1177                 task_update_spec_tif(task);
1178                 break;
1179         case PR_SPEC_DISABLE:
1180         case PR_SPEC_FORCE_DISABLE:
1181                 /*
1182                  * Indirect branch speculation is always allowed when
1183                  * mitigation is force disabled.
1184                  */
1185                 if (spectre_v2_user == SPECTRE_V2_USER_NONE)
1186                         return -EPERM;
1187                 if (spectre_v2_user == SPECTRE_V2_USER_STRICT ||
1188                     spectre_v2_user == SPECTRE_V2_USER_STRICT_PREFERRED)
1189                         return 0;
1190                 task_set_spec_ib_disable(task);
1191                 if (ctrl == PR_SPEC_FORCE_DISABLE)
1192                         task_set_spec_ib_force_disable(task);
1193                 task_update_spec_tif(task);
1194                 break;
1195         default:
1196                 return -ERANGE;
1197         }
1198         return 0;
1199 }
1200
1201 int arch_prctl_spec_ctrl_set(struct task_struct *task, unsigned long which,
1202                              unsigned long ctrl)
1203 {
1204         switch (which) {
1205         case PR_SPEC_STORE_BYPASS:
1206                 return ssb_prctl_set(task, ctrl);
1207         case PR_SPEC_INDIRECT_BRANCH:
1208                 return ib_prctl_set(task, ctrl);
1209         default:
1210                 return -ENODEV;
1211         }
1212 }
1213
1214 #ifdef CONFIG_SECCOMP
1215 void arch_seccomp_spec_mitigate(struct task_struct *task)
1216 {
1217         if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP)
1218                 ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE);
1219         if (spectre_v2_user == SPECTRE_V2_USER_SECCOMP)
1220                 ib_prctl_set(task, PR_SPEC_FORCE_DISABLE);
1221 }
1222 #endif
1223
1224 static int ssb_prctl_get(struct task_struct *task)
1225 {
1226         switch (ssb_mode) {
1227         case SPEC_STORE_BYPASS_DISABLE:
1228                 return PR_SPEC_DISABLE;
1229         case SPEC_STORE_BYPASS_SECCOMP:
1230         case SPEC_STORE_BYPASS_PRCTL:
1231                 if (task_spec_ssb_force_disable(task))
1232                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
1233                 if (task_spec_ssb_noexec(task))
1234                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE_NOEXEC;
1235                 if (task_spec_ssb_disable(task))
1236                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
1237                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
1238         default:
1239                 if (boot_cpu_has_bug(X86_BUG_SPEC_STORE_BYPASS))
1240                         return PR_SPEC_ENABLE;
1241                 return PR_SPEC_NOT_AFFECTED;
1242         }
1243 }
1244
1245 static int ib_prctl_get(struct task_struct *task)
1246 {
1247         if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2))
1248                 return PR_SPEC_NOT_AFFECTED;
1249
1250         switch (spectre_v2_user) {
1251         case SPECTRE_V2_USER_NONE:
1252                 return PR_SPEC_ENABLE;
1253         case SPECTRE_V2_USER_PRCTL:
1254         case SPECTRE_V2_USER_SECCOMP:
1255                 if (task_spec_ib_force_disable(task))
1256                         return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE;
1257                 if (task_spec_ib_disable(task))
1258                         return PR_SPEC_PRCTL | PR_SPEC_DISABLE;
1259                 return PR_SPEC_PRCTL | PR_SPEC_ENABLE;
1260         case SPECTRE_V2_USER_STRICT:
1261         case SPECTRE_V2_USER_STRICT_PREFERRED:
1262                 return PR_SPEC_DISABLE;
1263         default:
1264                 return PR_SPEC_NOT_AFFECTED;
1265         }
1266 }
1267
1268 int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
1269 {
1270         switch (which) {
1271         case PR_SPEC_STORE_BYPASS:
1272                 return ssb_prctl_get(task);
1273         case PR_SPEC_INDIRECT_BRANCH:
1274                 return ib_prctl_get(task);
1275         default:
1276                 return -ENODEV;
1277         }
1278 }
1279
1280 void x86_spec_ctrl_setup_ap(void)
1281 {
1282         if (boot_cpu_has(X86_FEATURE_MSR_SPEC_CTRL))
1283                 wrmsrl(MSR_IA32_SPEC_CTRL, x86_spec_ctrl_base);
1284
1285         if (ssb_mode == SPEC_STORE_BYPASS_DISABLE)
1286                 x86_amd_ssb_disable();
1287 }
1288
1289 bool itlb_multihit_kvm_mitigation;
1290 EXPORT_SYMBOL_GPL(itlb_multihit_kvm_mitigation);
1291
1292 #undef pr_fmt
1293 #define pr_fmt(fmt)     "L1TF: " fmt
1294
1295 /* Default mitigation for L1TF-affected CPUs */
1296 enum l1tf_mitigations l1tf_mitigation __ro_after_init = L1TF_MITIGATION_FLUSH;
1297 #if IS_ENABLED(CONFIG_KVM_INTEL)
1298 EXPORT_SYMBOL_GPL(l1tf_mitigation);
1299 #endif
1300 enum vmx_l1d_flush_state l1tf_vmx_mitigation = VMENTER_L1D_FLUSH_AUTO;
1301 EXPORT_SYMBOL_GPL(l1tf_vmx_mitigation);
1302
1303 /*
1304  * These CPUs all support 44bits physical address space internally in the
1305  * cache but CPUID can report a smaller number of physical address bits.
1306  *
1307  * The L1TF mitigation uses the top most address bit for the inversion of
1308  * non present PTEs. When the installed memory reaches into the top most
1309  * address bit due to memory holes, which has been observed on machines
1310  * which report 36bits physical address bits and have 32G RAM installed,
1311  * then the mitigation range check in l1tf_select_mitigation() triggers.
1312  * This is a false positive because the mitigation is still possible due to
1313  * the fact that the cache uses 44bit internally. Use the cache bits
1314  * instead of the reported physical bits and adjust them on the affected
1315  * machines to 44bit if the reported bits are less than 44.
1316  */
1317 static void override_cache_bits(struct cpuinfo_x86 *c)
1318 {
1319         if (c->x86 != 6)
1320                 return;
1321
1322         switch (c->x86_model) {
1323         case INTEL_FAM6_NEHALEM:
1324         case INTEL_FAM6_WESTMERE:
1325         case INTEL_FAM6_SANDYBRIDGE:
1326         case INTEL_FAM6_IVYBRIDGE:
1327         case INTEL_FAM6_HASWELL:
1328         case INTEL_FAM6_HASWELL_L:
1329         case INTEL_FAM6_HASWELL_G:
1330         case INTEL_FAM6_BROADWELL:
1331         case INTEL_FAM6_BROADWELL_G:
1332         case INTEL_FAM6_SKYLAKE_L:
1333         case INTEL_FAM6_SKYLAKE:
1334         case INTEL_FAM6_KABYLAKE_L:
1335         case INTEL_FAM6_KABYLAKE:
1336                 if (c->x86_cache_bits < 44)
1337                         c->x86_cache_bits = 44;
1338                 break;
1339         }
1340 }
1341
1342 static void __init l1tf_select_mitigation(void)
1343 {
1344         u64 half_pa;
1345
1346         if (!boot_cpu_has_bug(X86_BUG_L1TF))
1347                 return;
1348
1349         if (cpu_mitigations_off())
1350                 l1tf_mitigation = L1TF_MITIGATION_OFF;
1351         else if (cpu_mitigations_auto_nosmt())
1352                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
1353
1354         override_cache_bits(&boot_cpu_data);
1355
1356         switch (l1tf_mitigation) {
1357         case L1TF_MITIGATION_OFF:
1358         case L1TF_MITIGATION_FLUSH_NOWARN:
1359         case L1TF_MITIGATION_FLUSH:
1360                 break;
1361         case L1TF_MITIGATION_FLUSH_NOSMT:
1362         case L1TF_MITIGATION_FULL:
1363                 cpu_smt_disable(false);
1364                 break;
1365         case L1TF_MITIGATION_FULL_FORCE:
1366                 cpu_smt_disable(true);
1367                 break;
1368         }
1369
1370 #if CONFIG_PGTABLE_LEVELS == 2
1371         pr_warn("Kernel not compiled for PAE. No mitigation for L1TF\n");
1372         return;
1373 #endif
1374
1375         half_pa = (u64)l1tf_pfn_limit() << PAGE_SHIFT;
1376         if (l1tf_mitigation != L1TF_MITIGATION_OFF &&
1377                         e820__mapped_any(half_pa, ULLONG_MAX - half_pa, E820_TYPE_RAM)) {
1378                 pr_warn("System has more than MAX_PA/2 memory. L1TF mitigation not effective.\n");
1379                 pr_info("You may make it effective by booting the kernel with mem=%llu parameter.\n",
1380                                 half_pa);
1381                 pr_info("However, doing so will make a part of your RAM unusable.\n");
1382                 pr_info("Reading https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/l1tf.html might help you decide.\n");
1383                 return;
1384         }
1385
1386         setup_force_cpu_cap(X86_FEATURE_L1TF_PTEINV);
1387 }
1388
1389 static int __init l1tf_cmdline(char *str)
1390 {
1391         if (!boot_cpu_has_bug(X86_BUG_L1TF))
1392                 return 0;
1393
1394         if (!str)
1395                 return -EINVAL;
1396
1397         if (!strcmp(str, "off"))
1398                 l1tf_mitigation = L1TF_MITIGATION_OFF;
1399         else if (!strcmp(str, "flush,nowarn"))
1400                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOWARN;
1401         else if (!strcmp(str, "flush"))
1402                 l1tf_mitigation = L1TF_MITIGATION_FLUSH;
1403         else if (!strcmp(str, "flush,nosmt"))
1404                 l1tf_mitigation = L1TF_MITIGATION_FLUSH_NOSMT;
1405         else if (!strcmp(str, "full"))
1406                 l1tf_mitigation = L1TF_MITIGATION_FULL;
1407         else if (!strcmp(str, "full,force"))
1408                 l1tf_mitigation = L1TF_MITIGATION_FULL_FORCE;
1409
1410         return 0;
1411 }
1412 early_param("l1tf", l1tf_cmdline);
1413
1414 #undef pr_fmt
1415 #define pr_fmt(fmt) fmt
1416
1417 #ifdef CONFIG_SYSFS
1418
1419 #define L1TF_DEFAULT_MSG "Mitigation: PTE Inversion"
1420
1421 #if IS_ENABLED(CONFIG_KVM_INTEL)
1422 static const char * const l1tf_vmx_states[] = {
1423         [VMENTER_L1D_FLUSH_AUTO]                = "auto",
1424         [VMENTER_L1D_FLUSH_NEVER]               = "vulnerable",
1425         [VMENTER_L1D_FLUSH_COND]                = "conditional cache flushes",
1426         [VMENTER_L1D_FLUSH_ALWAYS]              = "cache flushes",
1427         [VMENTER_L1D_FLUSH_EPT_DISABLED]        = "EPT disabled",
1428         [VMENTER_L1D_FLUSH_NOT_REQUIRED]        = "flush not necessary"
1429 };
1430
1431 static ssize_t l1tf_show_state(char *buf)
1432 {
1433         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_AUTO)
1434                 return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
1435
1436         if (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_EPT_DISABLED ||
1437             (l1tf_vmx_mitigation == VMENTER_L1D_FLUSH_NEVER &&
1438              sched_smt_active())) {
1439                 return sprintf(buf, "%s; VMX: %s\n", L1TF_DEFAULT_MSG,
1440                                l1tf_vmx_states[l1tf_vmx_mitigation]);
1441         }
1442
1443         return sprintf(buf, "%s; VMX: %s, SMT %s\n", L1TF_DEFAULT_MSG,
1444                        l1tf_vmx_states[l1tf_vmx_mitigation],
1445                        sched_smt_active() ? "vulnerable" : "disabled");
1446 }
1447
1448 static ssize_t itlb_multihit_show_state(char *buf)
1449 {
1450         if (itlb_multihit_kvm_mitigation)
1451                 return sprintf(buf, "KVM: Mitigation: Split huge pages\n");
1452         else
1453                 return sprintf(buf, "KVM: Vulnerable\n");
1454 }
1455 #else
1456 static ssize_t l1tf_show_state(char *buf)
1457 {
1458         return sprintf(buf, "%s\n", L1TF_DEFAULT_MSG);
1459 }
1460
1461 static ssize_t itlb_multihit_show_state(char *buf)
1462 {
1463         return sprintf(buf, "Processor vulnerable\n");
1464 }
1465 #endif
1466
1467 static ssize_t mds_show_state(char *buf)
1468 {
1469         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
1470                 return sprintf(buf, "%s; SMT Host state unknown\n",
1471                                mds_strings[mds_mitigation]);
1472         }
1473
1474         if (boot_cpu_has(X86_BUG_MSBDS_ONLY)) {
1475                 return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
1476                                (mds_mitigation == MDS_MITIGATION_OFF ? "vulnerable" :
1477                                 sched_smt_active() ? "mitigated" : "disabled"));
1478         }
1479
1480         return sprintf(buf, "%s; SMT %s\n", mds_strings[mds_mitigation],
1481                        sched_smt_active() ? "vulnerable" : "disabled");
1482 }
1483
1484 static ssize_t tsx_async_abort_show_state(char *buf)
1485 {
1486         if ((taa_mitigation == TAA_MITIGATION_TSX_DISABLED) ||
1487             (taa_mitigation == TAA_MITIGATION_OFF))
1488                 return sprintf(buf, "%s\n", taa_strings[taa_mitigation]);
1489
1490         if (boot_cpu_has(X86_FEATURE_HYPERVISOR)) {
1491                 return sprintf(buf, "%s; SMT Host state unknown\n",
1492                                taa_strings[taa_mitigation]);
1493         }
1494
1495         return sprintf(buf, "%s; SMT %s\n", taa_strings[taa_mitigation],
1496                        sched_smt_active() ? "vulnerable" : "disabled");
1497 }
1498
1499 static char *stibp_state(void)
1500 {
1501         if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED)
1502                 return "";
1503
1504         switch (spectre_v2_user) {
1505         case SPECTRE_V2_USER_NONE:
1506                 return ", STIBP: disabled";
1507         case SPECTRE_V2_USER_STRICT:
1508                 return ", STIBP: forced";
1509         case SPECTRE_V2_USER_STRICT_PREFERRED:
1510                 return ", STIBP: always-on";
1511         case SPECTRE_V2_USER_PRCTL:
1512         case SPECTRE_V2_USER_SECCOMP:
1513                 if (static_key_enabled(&switch_to_cond_stibp))
1514                         return ", STIBP: conditional";
1515         }
1516         return "";
1517 }
1518
1519 static char *ibpb_state(void)
1520 {
1521         if (boot_cpu_has(X86_FEATURE_IBPB)) {
1522                 if (static_key_enabled(&switch_mm_always_ibpb))
1523                         return ", IBPB: always-on";
1524                 if (static_key_enabled(&switch_mm_cond_ibpb))
1525                         return ", IBPB: conditional";
1526                 return ", IBPB: disabled";
1527         }
1528         return "";
1529 }
1530
1531 static ssize_t cpu_show_common(struct device *dev, struct device_attribute *attr,
1532                                char *buf, unsigned int bug)
1533 {
1534         if (!boot_cpu_has_bug(bug))
1535                 return sprintf(buf, "Not affected\n");
1536
1537         switch (bug) {
1538         case X86_BUG_CPU_MELTDOWN:
1539                 if (boot_cpu_has(X86_FEATURE_PTI))
1540                         return sprintf(buf, "Mitigation: PTI\n");
1541
1542                 if (hypervisor_is_type(X86_HYPER_XEN_PV))
1543                         return sprintf(buf, "Unknown (XEN PV detected, hypervisor mitigation required)\n");
1544
1545                 break;
1546
1547         case X86_BUG_SPECTRE_V1:
1548                 return sprintf(buf, "%s\n", spectre_v1_strings[spectre_v1_mitigation]);
1549
1550         case X86_BUG_SPECTRE_V2:
1551                 return sprintf(buf, "%s%s%s%s%s%s\n", spectre_v2_strings[spectre_v2_enabled],
1552                                ibpb_state(),
1553                                boot_cpu_has(X86_FEATURE_USE_IBRS_FW) ? ", IBRS_FW" : "",
1554                                stibp_state(),
1555                                boot_cpu_has(X86_FEATURE_RSB_CTXSW) ? ", RSB filling" : "",
1556                                spectre_v2_module_string());
1557
1558         case X86_BUG_SPEC_STORE_BYPASS:
1559                 return sprintf(buf, "%s\n", ssb_strings[ssb_mode]);
1560
1561         case X86_BUG_L1TF:
1562                 if (boot_cpu_has(X86_FEATURE_L1TF_PTEINV))
1563                         return l1tf_show_state(buf);
1564                 break;
1565
1566         case X86_BUG_MDS:
1567                 return mds_show_state(buf);
1568
1569         case X86_BUG_TAA:
1570                 return tsx_async_abort_show_state(buf);
1571
1572         case X86_BUG_ITLB_MULTIHIT:
1573                 return itlb_multihit_show_state(buf);
1574
1575         default:
1576                 break;
1577         }
1578
1579         return sprintf(buf, "Vulnerable\n");
1580 }
1581
1582 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf)
1583 {
1584         return cpu_show_common(dev, attr, buf, X86_BUG_CPU_MELTDOWN);
1585 }
1586
1587 ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf)
1588 {
1589         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V1);
1590 }
1591
1592 ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf)
1593 {
1594         return cpu_show_common(dev, attr, buf, X86_BUG_SPECTRE_V2);
1595 }
1596
1597 ssize_t cpu_show_spec_store_bypass(struct device *dev, struct device_attribute *attr, char *buf)
1598 {
1599         return cpu_show_common(dev, attr, buf, X86_BUG_SPEC_STORE_BYPASS);
1600 }
1601
1602 ssize_t cpu_show_l1tf(struct device *dev, struct device_attribute *attr, char *buf)
1603 {
1604         return cpu_show_common(dev, attr, buf, X86_BUG_L1TF);
1605 }
1606
1607 ssize_t cpu_show_mds(struct device *dev, struct device_attribute *attr, char *buf)
1608 {
1609         return cpu_show_common(dev, attr, buf, X86_BUG_MDS);
1610 }
1611
1612 ssize_t cpu_show_tsx_async_abort(struct device *dev, struct device_attribute *attr, char *buf)
1613 {
1614         return cpu_show_common(dev, attr, buf, X86_BUG_TAA);
1615 }
1616
1617 ssize_t cpu_show_itlb_multihit(struct device *dev, struct device_attribute *attr, char *buf)
1618 {
1619         return cpu_show_common(dev, attr, buf, X86_BUG_ITLB_MULTIHIT);
1620 }
1621 #endif