arch/x86/kernel/fpu/xstate.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * xsave/xrstor support.
   4  *
   5  * Author: Suresh Siddha <suresh.b.siddha@intel.com>
   6  */
   7 #include <linux/compat.h>
   8 #include <linux/cpu.h>
   9 #include <linux/mman.h>
  10 #include <linux/pkeys.h>
  11 #include <linux/seq_file.h>
  12 #include <linux/proc_fs.h>
  13
  14 #include <asm/fpu/api.h>
  15 #include <asm/fpu/internal.h>
  16 #include <asm/fpu/signal.h>
  17 #include <asm/fpu/regset.h>
  18 #include <asm/fpu/xstate.h>
  19
  20 #include <asm/tlbflush.h>
  21 #include <asm/cpufeature.h>
  22
  23 /*
  24  * Although we spell it out in here, the Processor Trace
  25  * xfeature is completely unused.  We use other mechanisms
  26  * to save/restore PT state in Linux.
  27  */
  28 static const char *xfeature_names[] =
  29 {
  30         "x87 floating point registers"  ,
  31         "SSE registers"                 ,
  32         "AVX registers"                 ,
  33         "MPX bounds registers"          ,
  34         "MPX CSR"                       ,
  35         "AVX-512 opmask"                ,
  36         "AVX-512 Hi256"                 ,
  37         "AVX-512 ZMM_Hi256"             ,
  38         "Processor Trace (unused)"      ,
  39         "Protection Keys User registers",
  40         "PASID state",
  41         "unknown xstate feature"        ,
  42 };
  43
  44 static short xsave_cpuid_features[] __initdata = {
  45         X86_FEATURE_FPU,
  46         X86_FEATURE_XMM,
  47         X86_FEATURE_AVX,
  48         X86_FEATURE_MPX,
  49         X86_FEATURE_MPX,
  50         X86_FEATURE_AVX512F,
  51         X86_FEATURE_AVX512F,
  52         X86_FEATURE_AVX512F,
  53         X86_FEATURE_INTEL_PT,
  54         X86_FEATURE_PKU,
  55         X86_FEATURE_ENQCMD,
  56 };
  57
  58 /*
  59  * This represents the full set of bits that should ever be set in a kernel
  60  * XSAVE buffer, both supervisor and user xstates.
  61  */
  62 u64 xfeatures_mask_all __ro_after_init;
  63
  64 static unsigned int xstate_offsets[XFEATURE_MAX] __ro_after_init =
  65         { [ 0 ... XFEATURE_MAX - 1] = -1};
  66 static unsigned int xstate_sizes[XFEATURE_MAX] __ro_after_init =
  67         { [ 0 ... XFEATURE_MAX - 1] = -1};
  68 static unsigned int xstate_comp_offsets[XFEATURE_MAX] __ro_after_init =
  69         { [ 0 ... XFEATURE_MAX - 1] = -1};
  70 static unsigned int xstate_supervisor_only_offsets[XFEATURE_MAX] __ro_after_init =
  71         { [ 0 ... XFEATURE_MAX - 1] = -1};
  72
  73 /*
  74  * The XSAVE area of kernel can be in standard or compacted format;
  75  * it is always in standard format for user mode. This is the user
  76  * mode standard format size used for signal and ptrace frames.
  77  */
  78 unsigned int fpu_user_xstate_size __ro_after_init;
  79
  80 /*
  81  * Return whether the system supports a given xfeature.
  82  *
  83  * Also return the name of the (most advanced) feature that the caller requested:
  84  */
  85 int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name)
  86 {
  87         u64 xfeatures_missing = xfeatures_needed & ~xfeatures_mask_all;
  88
  89         if (unlikely(feature_name)) {
  90                 long xfeature_idx, max_idx;
  91                 u64 xfeatures_print;
  92                 /*
  93                  * So we use FLS here to be able to print the most advanced
  94                  * feature that was requested but is missing. So if a driver
  95                  * asks about "XFEATURE_MASK_SSE | XFEATURE_MASK_YMM" we'll print the
  96                  * missing AVX feature - this is the most informative message
  97                  * to users:
  98                  */
  99                 if (xfeatures_missing)
 100                         xfeatures_print = xfeatures_missing;
 101                 else
 102                         xfeatures_print = xfeatures_needed;
 103
 104                 xfeature_idx = fls64(xfeatures_print)-1;
 105                 max_idx = ARRAY_SIZE(xfeature_names)-1;
 106                 xfeature_idx = min(xfeature_idx, max_idx);
 107
 108                 *feature_name = xfeature_names[xfeature_idx];
 109         }
 110
 111         if (xfeatures_missing)
 112                 return 0;
 113
 114         return 1;
 115 }
 116 EXPORT_SYMBOL_GPL(cpu_has_xfeatures);
 117
 118 static bool xfeature_is_supervisor(int xfeature_nr)
 119 {
 120         /*
 121          * Extended State Enumeration Sub-leaves (EAX = 0DH, ECX = n, n > 1)
 122          * returns ECX[0] set to (1) for a supervisor state, and cleared (0)
 123          * for a user state.
 124          */
 125         u32 eax, ebx, ecx, edx;
 126
 127         cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
 128         return ecx & 1;
 129 }
 130
 131 /*
 132  * Enable the extended processor state save/restore feature.
 133  * Called once per CPU onlining.
 134  */
 135 void fpu__init_cpu_xstate(void)
 136 {
 137         if (!boot_cpu_has(X86_FEATURE_XSAVE) || !xfeatures_mask_all)
 138                 return;
 139
 140         cr4_set_bits(X86_CR4_OSXSAVE);
 141
 142         /*
 143          * XCR_XFEATURE_ENABLED_MASK (aka. XCR0) sets user features
 144          * managed by XSAVE{C, OPT, S} and XRSTOR{S}.  Only XSAVE user
 145          * states can be set here.
 146          */
 147         xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask_user());
 148
 149         /*
 150          * MSR_IA32_XSS sets supervisor states managed by XSAVES.
 151          */
 152         if (boot_cpu_has(X86_FEATURE_XSAVES)) {
 153                 wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() |
 154                                      xfeatures_mask_dynamic());
 155         }
 156 }
 157
 158 static bool xfeature_enabled(enum xfeature xfeature)
 159 {
 160         return xfeatures_mask_all & BIT_ULL(xfeature);
 161 }
 162
 163 /*
 164  * Record the offsets and sizes of various xstates contained
 165  * in the XSAVE state memory layout.
 166  */
 167 static void __init setup_xstate_features(void)
 168 {
 169         u32 eax, ebx, ecx, edx, i;
 170         /* start at the beginning of the "extended state" */
 171         unsigned int last_good_offset = offsetof(struct xregs_state,
 172                                                  extended_state_area);
 173         /*
 174          * The FP xstates and SSE xstates are legacy states. They are always
 175          * in the fixed offsets in the xsave area in either compacted form
 176          * or standard form.
 177          */
 178         xstate_offsets[XFEATURE_FP]     = 0;
 179         xstate_sizes[XFEATURE_FP]       = offsetof(struct fxregs_state,
 180                                                    xmm_space);
 181
 182         xstate_offsets[XFEATURE_SSE]    = xstate_sizes[XFEATURE_FP];
 183         xstate_sizes[XFEATURE_SSE]      = sizeof_field(struct fxregs_state,
 184                                                        xmm_space);
 185
 186         for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
 187                 if (!xfeature_enabled(i))
 188                         continue;
 189
 190                 cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);
 191
 192                 xstate_sizes[i] = eax;
 193
 194                 /*
 195                  * If an xfeature is supervisor state, the offset in EBX is
 196                  * invalid, leave it to -1.
 197                  */
 198                 if (xfeature_is_supervisor(i))
 199                         continue;
 200
 201                 xstate_offsets[i] = ebx;
 202
 203                 /*
 204                  * In our xstate size checks, we assume that the highest-numbered
 205                  * xstate feature has the highest offset in the buffer.  Ensure
 206                  * it does.
 207                  */
 208                 WARN_ONCE(last_good_offset > xstate_offsets[i],
 209                           "x86/fpu: misordered xstate at %d\n", last_good_offset);
 210
 211                 last_good_offset = xstate_offsets[i];
 212         }
 213 }
 214
 215 static void __init print_xstate_feature(u64 xstate_mask)
 216 {
 217         const char *feature_name;
 218
 219         if (cpu_has_xfeatures(xstate_mask, &feature_name))
 220                 pr_info("x86/fpu: Supporting XSAVE feature 0x%03Lx: '%s'\n", xstate_mask, feature_name);
 221 }
 222
 223 /*
 224  * Print out all the supported xstate features:
 225  */
 226 static void __init print_xstate_features(void)
 227 {
 228         print_xstate_feature(XFEATURE_MASK_FP);
 229         print_xstate_feature(XFEATURE_MASK_SSE);
 230         print_xstate_feature(XFEATURE_MASK_YMM);
 231         print_xstate_feature(XFEATURE_MASK_BNDREGS);
 232         print_xstate_feature(XFEATURE_MASK_BNDCSR);
 233         print_xstate_feature(XFEATURE_MASK_OPMASK);
 234         print_xstate_feature(XFEATURE_MASK_ZMM_Hi256);
 235         print_xstate_feature(XFEATURE_MASK_Hi16_ZMM);
 236         print_xstate_feature(XFEATURE_MASK_PKRU);
 237         print_xstate_feature(XFEATURE_MASK_PASID);
 238 }
 239
 240 /*
 241  * This check is important because it is easy to get XSTATE_*
 242  * confused with XSTATE_BIT_*.
 243  */
 244 #define CHECK_XFEATURE(nr) do {         \
 245         WARN_ON(nr < FIRST_EXTENDED_XFEATURE);  \
 246         WARN_ON(nr >= XFEATURE_MAX);    \
 247 } while (0)
 248
 249 /*
 250  * We could cache this like xstate_size[], but we only use
 251  * it here, so it would be a waste of space.
 252  */
 253 static int xfeature_is_aligned(int xfeature_nr)
 254 {
 255         u32 eax, ebx, ecx, edx;
 256
 257         CHECK_XFEATURE(xfeature_nr);
 258
 259         if (!xfeature_enabled(xfeature_nr)) {
 260                 WARN_ONCE(1, "Checking alignment of disabled xfeature %d\n",
 261                           xfeature_nr);
 262                 return 0;
 263         }
 264
 265         cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
 266         /*
 267          * The value returned by ECX[1] indicates the alignment
 268          * of state component 'i' when the compacted format
 269          * of the extended region of an XSAVE area is used:
 270          */
 271         return !!(ecx & 2);
 272 }
 273
 274 /*
 275  * This function sets up offsets and sizes of all extended states in
 276  * xsave area. This supports both standard format and compacted format
 277  * of the xsave area.
 278  */
 279 static void __init setup_xstate_comp_offsets(void)
 280 {
 281         unsigned int next_offset;
 282         int i;
 283
 284         /*
 285          * The FP xstates and SSE xstates are legacy states. They are always
 286          * in the fixed offsets in the xsave area in either compacted form
 287          * or standard form.
 288          */
 289         xstate_comp_offsets[XFEATURE_FP] = 0;
 290         xstate_comp_offsets[XFEATURE_SSE] = offsetof(struct fxregs_state,
 291                                                      xmm_space);
 292
 293         if (!boot_cpu_has(X86_FEATURE_XSAVES)) {
 294                 for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
 295                         if (xfeature_enabled(i))
 296                                 xstate_comp_offsets[i] = xstate_offsets[i];
 297                 }
 298                 return;
 299         }
 300
 301         next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE;
 302
 303         for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
 304                 if (!xfeature_enabled(i))
 305                         continue;
 306
 307                 if (xfeature_is_aligned(i))
 308                         next_offset = ALIGN(next_offset, 64);
 309
 310                 xstate_comp_offsets[i] = next_offset;
 311                 next_offset += xstate_sizes[i];
 312         }
 313 }
 314
 315 /*
 316  * Setup offsets of a supervisor-state-only XSAVES buffer:
 317  *
 318  * The offsets stored in xstate_comp_offsets[] only work for one specific
 319  * value of the Requested Feature BitMap (RFBM).  In cases where a different
 320  * RFBM value is used, a different set of offsets is required.  This set of
 321  * offsets is for when RFBM=xfeatures_mask_supervisor().
 322  */
 323 static void __init setup_supervisor_only_offsets(void)
 324 {
 325         unsigned int next_offset;
 326         int i;
 327
 328         next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE;
 329
 330         for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
 331                 if (!xfeature_enabled(i) || !xfeature_is_supervisor(i))
 332                         continue;
 333
 334                 if (xfeature_is_aligned(i))
 335                         next_offset = ALIGN(next_offset, 64);
 336
 337                 xstate_supervisor_only_offsets[i] = next_offset;
 338                 next_offset += xstate_sizes[i];
 339         }
 340 }
 341
 342 /*
 343  * Print out xstate component offsets and sizes
 344  */
 345 static void __init print_xstate_offset_size(void)
 346 {
 347         int i;
 348
 349         for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
 350                 if (!xfeature_enabled(i))
 351                         continue;
 352                 pr_info("x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n",
 353                          i, xstate_comp_offsets[i], i, xstate_sizes[i]);
 354         }
 355 }
 356
 357 /*
 358  * All supported features have either init state all zeros or are
 359  * handled in setup_init_fpu() individually. This is an explicit
 360  * feature list and does not use XFEATURE_MASK*SUPPORTED to catch
 361  * newly added supported features at build time and make people
 362  * actually look at the init state for the new feature.
 363  */
 364 #define XFEATURES_INIT_FPSTATE_HANDLED          \
 365         (XFEATURE_MASK_FP |                     \
 366          XFEATURE_MASK_SSE |                    \
 367          XFEATURE_MASK_YMM |                    \
 368          XFEATURE_MASK_OPMASK |                 \
 369          XFEATURE_MASK_ZMM_Hi256 |              \
 370          XFEATURE_MASK_Hi16_ZMM  |              \
 371          XFEATURE_MASK_PKRU |                   \
 372          XFEATURE_MASK_BNDREGS |                \
 373          XFEATURE_MASK_BNDCSR |                 \
 374          XFEATURE_MASK_PASID)
 375
 376 /*
 377  * setup the xstate image representing the init state
 378  */
 379 static void __init setup_init_fpu_buf(void)
 380 {
 381         static int on_boot_cpu __initdata = 1;
 382
 383         BUILD_BUG_ON((XFEATURE_MASK_USER_SUPPORTED |
 384                       XFEATURE_MASK_SUPERVISOR_SUPPORTED) !=
 385                      XFEATURES_INIT_FPSTATE_HANDLED);
 386
 387         WARN_ON_FPU(!on_boot_cpu);
 388         on_boot_cpu = 0;
 389
 390         if (!boot_cpu_has(X86_FEATURE_XSAVE))
 391                 return;
 392
 393         setup_xstate_features();
 394         print_xstate_features();
 395
 396         if (boot_cpu_has(X86_FEATURE_XSAVES))
 397                 init_fpstate.xsave.header.xcomp_bv = XCOMP_BV_COMPACTED_FORMAT |
 398                                                      xfeatures_mask_all;
 399
 400         /*
 401          * Init all the features state with header.xfeatures being 0x0
 402          */
 403         copy_kernel_to_xregs_booting(&init_fpstate.xsave);
 404
 405         /*
 406          * All components are now in init state. Read the state back so
 407          * that init_fpstate contains all non-zero init state. This only
 408          * works with XSAVE, but not with XSAVEOPT and XSAVES because
 409          * those use the init optimization which skips writing data for
 410          * components in init state.
 411          *
 412          * XSAVE could be used, but that would require to reshuffle the
 413          * data when XSAVES is available because XSAVES uses xstate
 414          * compaction. But doing so is a pointless exercise because most
 415          * components have an all zeros init state except for the legacy
 416          * ones (FP and SSE). Those can be saved with FXSAVE into the
 417          * legacy area. Adding new features requires to ensure that init
 418          * state is all zeroes or if not to add the necessary handling
 419          * here.
 420          */
 421         fxsave(&init_fpstate.fxsave);
 422 }
 423
 424 static int xfeature_uncompacted_offset(int xfeature_nr)
 425 {
 426         u32 eax, ebx, ecx, edx;
 427
 428         /*
 429          * Only XSAVES supports supervisor states and it uses compacted
 430          * format. Checking a supervisor state's uncompacted offset is
 431          * an error.
 432          */
 433         if (XFEATURE_MASK_SUPERVISOR_ALL & BIT_ULL(xfeature_nr)) {
 434                 WARN_ONCE(1, "No fixed offset for xstate %d\n", xfeature_nr);
 435                 return -1;
 436         }
 437
 438         CHECK_XFEATURE(xfeature_nr);
 439         cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
 440         return ebx;
 441 }
 442
 443 int xfeature_size(int xfeature_nr)
 444 {
 445         u32 eax, ebx, ecx, edx;
 446
 447         CHECK_XFEATURE(xfeature_nr);
 448         cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
 449         return eax;
 450 }
 451
 452 /* Validate an xstate header supplied by userspace (ptrace or sigreturn) */
 453 static int validate_user_xstate_header(const struct xstate_header *hdr)
 454 {
 455         /* No unknown or supervisor features may be set */
 456         if (hdr->xfeatures & ~xfeatures_mask_user())
 457                 return -EINVAL;
 458
 459         /* Userspace must use the uncompacted format */
 460         if (hdr->xcomp_bv)
 461                 return -EINVAL;
 462
 463         /*
 464          * If 'reserved' is shrunken to add a new field, make sure to validate
 465          * that new field here!
 466          */
 467         BUILD_BUG_ON(sizeof(hdr->reserved) != 48);
 468
 469         /* No reserved bits may be set */
 470         if (memchr_inv(hdr->reserved, 0, sizeof(hdr->reserved)))
 471                 return -EINVAL;
 472
 473         return 0;
 474 }
 475
 476 static void __xstate_dump_leaves(void)
 477 {
 478         int i;
 479         u32 eax, ebx, ecx, edx;
 480         static int should_dump = 1;
 481
 482         if (!should_dump)
 483                 return;
 484         should_dump = 0;
 485         /*
 486          * Dump out a few leaves past the ones that we support
 487          * just in case there are some goodies up there
 488          */
 489         for (i = 0; i < XFEATURE_MAX + 10; i++) {
 490                 cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);
 491                 pr_warn("CPUID[%02x, %02x]: eax=%08x ebx=%08x ecx=%08x edx=%08x\n",
 492                         XSTATE_CPUID, i, eax, ebx, ecx, edx);
 493         }
 494 }
 495
 496 #define XSTATE_WARN_ON(x) do {                                                  \
 497         if (WARN_ONCE(x, "XSAVE consistency problem, dumping leaves")) {        \
 498                 __xstate_dump_leaves();                                         \
 499         }                                                                       \
 500 } while (0)
 501
 502 #define XCHECK_SZ(sz, nr, nr_macro, __struct) do {                      \
 503         if ((nr == nr_macro) &&                                         \
 504             WARN_ONCE(sz != sizeof(__struct),                           \
 505                 "%s: struct is %zu bytes, cpu state %d bytes\n",        \
 506                 __stringify(nr_macro), sizeof(__struct), sz)) {         \
 507                 __xstate_dump_leaves();                                 \
 508         }                                                               \
 509 } while (0)
 510
 511 /*
 512  * We have a C struct for each 'xstate'.  We need to ensure
 513  * that our software representation matches what the CPU
 514  * tells us about the state's size.
 515  */
 516 static void check_xstate_against_struct(int nr)
 517 {
 518         /*
 519          * Ask the CPU for the size of the state.
 520          */
 521         int sz = xfeature_size(nr);
 522         /*
 523          * Match each CPU state with the corresponding software
 524          * structure.
 525          */
 526         XCHECK_SZ(sz, nr, XFEATURE_YMM,       struct ymmh_struct);
 527         XCHECK_SZ(sz, nr, XFEATURE_BNDREGS,   struct mpx_bndreg_state);
 528         XCHECK_SZ(sz, nr, XFEATURE_BNDCSR,    struct mpx_bndcsr_state);
 529         XCHECK_SZ(sz, nr, XFEATURE_OPMASK,    struct avx_512_opmask_state);
 530         XCHECK_SZ(sz, nr, XFEATURE_ZMM_Hi256, struct avx_512_zmm_uppers_state);
 531         XCHECK_SZ(sz, nr, XFEATURE_Hi16_ZMM,  struct avx_512_hi16_state);
 532         XCHECK_SZ(sz, nr, XFEATURE_PKRU,      struct pkru_state);
 533         XCHECK_SZ(sz, nr, XFEATURE_PASID,     struct ia32_pasid_state);
 534
 535         /*
 536          * Make *SURE* to add any feature numbers in below if
 537          * there are "holes" in the xsave state component
 538          * numbers.
 539          */
 540         if ((nr < XFEATURE_YMM) ||
 541             (nr >= XFEATURE_MAX) ||
 542             (nr == XFEATURE_PT_UNIMPLEMENTED_SO_FAR) ||
 543             ((nr >= XFEATURE_RSRVD_COMP_11) && (nr <= XFEATURE_LBR))) {
 544                 WARN_ONCE(1, "no structure for xstate: %d\n", nr);
 545                 XSTATE_WARN_ON(1);
 546         }
 547 }
 548
 549 /*
 550  * This essentially double-checks what the cpu told us about
 551  * how large the XSAVE buffer needs to be.  We are recalculating
 552  * it to be safe.
 553  *
 554  * Dynamic XSAVE features allocate their own buffers and are not
 555  * covered by these checks. Only the size of the buffer for task->fpu
 556  * is checked here.
 557  */
 558 static void do_extra_xstate_size_checks(void)
 559 {
 560         int paranoid_xstate_size = FXSAVE_SIZE + XSAVE_HDR_SIZE;
 561         int i;
 562
 563         for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
 564                 if (!xfeature_enabled(i))
 565                         continue;
 566
 567                 check_xstate_against_struct(i);
 568                 /*
 569                  * Supervisor state components can be managed only by
 570                  * XSAVES.
 571                  */
 572                 if (!cpu_feature_enabled(X86_FEATURE_XSAVES))
 573                         XSTATE_WARN_ON(xfeature_is_supervisor(i));
 574
 575                 /* Align from the end of the previous feature */
 576                 if (xfeature_is_aligned(i))
 577                         paranoid_xstate_size = ALIGN(paranoid_xstate_size, 64);
 578                 /*
 579                  * The offset of a given state in the non-compacted
 580                  * format is given to us in a CPUID leaf.  We check
 581                  * them for being ordered (increasing offsets) in
 582                  * setup_xstate_features(). XSAVES uses compacted format.
 583                  */
 584                 if (!cpu_feature_enabled(X86_FEATURE_XSAVES))
 585                         paranoid_xstate_size = xfeature_uncompacted_offset(i);
 586                 /*
 587                  * The compacted-format offset always depends on where
 588                  * the previous state ended.
 589                  */
 590                 paranoid_xstate_size += xfeature_size(i);
 591         }
 592         XSTATE_WARN_ON(paranoid_xstate_size != fpu_kernel_xstate_size);
 593 }
 594
 595
 596 /*
 597  * Get total size of enabled xstates in XCR0 | IA32_XSS.
 598  *
 599  * Note the SDM's wording here.  "sub-function 0" only enumerates
 600  * the size of the *user* states.  If we use it to size a buffer
 601  * that we use 'XSAVES' on, we could potentially overflow the
 602  * buffer because 'XSAVES' saves system states too.
 603  */
 604 static unsigned int __init get_xsaves_size(void)
 605 {
 606         unsigned int eax, ebx, ecx, edx;
 607         /*
 608          * - CPUID function 0DH, sub-function 1:
 609          *    EBX enumerates the size (in bytes) required by
 610          *    the XSAVES instruction for an XSAVE area
 611          *    containing all the state components
 612          *    corresponding to bits currently set in
 613          *    XCR0 | IA32_XSS.
 614          */
 615         cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx);
 616         return ebx;
 617 }
 618
 619 /*
 620  * Get the total size of the enabled xstates without the dynamic supervisor
 621  * features.
 622  */
 623 static unsigned int __init get_xsaves_size_no_dynamic(void)
 624 {
 625         u64 mask = xfeatures_mask_dynamic();
 626         unsigned int size;
 627
 628         if (!mask)
 629                 return get_xsaves_size();
 630
 631         /* Disable dynamic features. */
 632         wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor());
 633
 634         /*
 635          * Ask the hardware what size is required of the buffer.
 636          * This is the size required for the task->fpu buffer.
 637          */
 638         size = get_xsaves_size();
 639
 640         /* Re-enable dynamic features so XSAVES will work on them again. */
 641         wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | mask);
 642
 643         return size;
 644 }
 645
 646 static unsigned int __init get_xsave_size(void)
 647 {
 648         unsigned int eax, ebx, ecx, edx;
 649         /*
 650          * - CPUID function 0DH, sub-function 0:
 651          *    EBX enumerates the size (in bytes) required by
 652          *    the XSAVE instruction for an XSAVE area
 653          *    containing all the *user* state components
 654          *    corresponding to bits currently set in XCR0.
 655          */
 656         cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
 657         return ebx;
 658 }
 659
 660 /*
 661  * Will the runtime-enumerated 'xstate_size' fit in the init
 662  * task's statically-allocated buffer?
 663  */
 664 static bool is_supported_xstate_size(unsigned int test_xstate_size)
 665 {
 666         if (test_xstate_size <= sizeof(union fpregs_state))
 667                 return true;
 668
 669         pr_warn("x86/fpu: xstate buffer too small (%zu < %d), disabling xsave\n",
 670                         sizeof(union fpregs_state), test_xstate_size);
 671         return false;
 672 }
 673
 674 static int __init init_xstate_size(void)
 675 {
 676         /* Recompute the context size for enabled features: */
 677         unsigned int possible_xstate_size;
 678         unsigned int xsave_size;
 679
 680         xsave_size = get_xsave_size();
 681
 682         if (boot_cpu_has(X86_FEATURE_XSAVES))
 683                 possible_xstate_size = get_xsaves_size_no_dynamic();
 684         else
 685                 possible_xstate_size = xsave_size;
 686
 687         /* Ensure we have the space to store all enabled: */
 688         if (!is_supported_xstate_size(possible_xstate_size))
 689                 return -EINVAL;
 690
 691         /*
 692          * The size is OK, we are definitely going to use xsave,
 693          * make it known to the world that we need more space.
 694          */
 695         fpu_kernel_xstate_size = possible_xstate_size;
 696         do_extra_xstate_size_checks();
 697
 698         /*
 699          * User space is always in standard format.
 700          */
 701         fpu_user_xstate_size = xsave_size;
 702         return 0;
 703 }
 704
 705 /*
 706  * We enabled the XSAVE hardware, but something went wrong and
 707  * we can not use it.  Disable it.
 708  */
 709 static void fpu__init_disable_system_xstate(void)
 710 {
 711         xfeatures_mask_all = 0;
 712         cr4_clear_bits(X86_CR4_OSXSAVE);
 713         setup_clear_cpu_cap(X86_FEATURE_XSAVE);
 714 }
 715
 716 /*
 717  * Enable and initialize the xsave feature.
 718  * Called once per system bootup.
 719  */
 720 void __init fpu__init_system_xstate(void)
 721 {
 722         unsigned int eax, ebx, ecx, edx;
 723         static int on_boot_cpu __initdata = 1;
 724         u64 xfeatures;
 725         int err;
 726         int i;
 727
 728         WARN_ON_FPU(!on_boot_cpu);
 729         on_boot_cpu = 0;
 730
 731         if (!boot_cpu_has(X86_FEATURE_FPU)) {
 732                 pr_info("x86/fpu: No FPU detected\n");
 733                 return;
 734         }
 735
 736         if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
 737                 pr_info("x86/fpu: x87 FPU will use %s\n",
 738                         boot_cpu_has(X86_FEATURE_FXSR) ? "FXSAVE" : "FSAVE");
 739                 return;
 740         }
 741
 742         if (boot_cpu_data.cpuid_level < XSTATE_CPUID) {
 743                 WARN_ON_FPU(1);
 744                 return;
 745         }
 746
 747         /*
 748          * Find user xstates supported by the processor.
 749          */
 750         cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
 751         xfeatures_mask_all = eax + ((u64)edx << 32);
 752
 753         /*
 754          * Find supervisor xstates supported by the processor.
 755          */
 756         cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx);
 757         xfeatures_mask_all |= ecx + ((u64)edx << 32);
 758
 759         if ((xfeatures_mask_user() & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) {
 760                 /*
 761                  * This indicates that something really unexpected happened
 762                  * with the enumeration.  Disable XSAVE and try to continue
 763                  * booting without it.  This is too early to BUG().
 764                  */
 765                 pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n",
 766                        xfeatures_mask_all);
 767                 goto out_disable;
 768         }
 769
 770         /*
 771          * Clear XSAVE features that are disabled in the normal CPUID.
 772          */
 773         for (i = 0; i < ARRAY_SIZE(xsave_cpuid_features); i++) {
 774                 if (!boot_cpu_has(xsave_cpuid_features[i]))
 775                         xfeatures_mask_all &= ~BIT_ULL(i);
 776         }
 777
 778         xfeatures_mask_all &= XFEATURE_MASK_USER_SUPPORTED |
 779                               XFEATURE_MASK_SUPERVISOR_SUPPORTED;
 780
 781         /* Store it for paranoia check at the end */
 782         xfeatures = xfeatures_mask_all;
 783
 784         /* Enable xstate instructions to be able to continue with initialization: */
 785         fpu__init_cpu_xstate();
 786         err = init_xstate_size();
 787         if (err)
 788                 goto out_disable;
 789
 790         /*
 791          * Update info used for ptrace frames; use standard-format size and no
 792          * supervisor xstates:
 793          */
 794         update_regset_xstate_info(fpu_user_xstate_size, xfeatures_mask_user());
 795
 796         fpu__init_prepare_fx_sw_frame();
 797         setup_init_fpu_buf();
 798         setup_xstate_comp_offsets();
 799         setup_supervisor_only_offsets();
 800
 801         /*
 802          * Paranoia check whether something in the setup modified the
 803          * xfeatures mask.
 804          */
 805         if (xfeatures != xfeatures_mask_all) {
 806                 pr_err("x86/fpu: xfeatures modified from 0x%016llx to 0x%016llx during init, disabling XSAVE\n",
 807                        xfeatures, xfeatures_mask_all);
 808                 goto out_disable;
 809         }
 810
 811         print_xstate_offset_size();
 812         pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n",
 813                 xfeatures_mask_all,
 814                 fpu_kernel_xstate_size,
 815                 boot_cpu_has(X86_FEATURE_XSAVES) ? "compacted" : "standard");
 816         return;
 817
 818 out_disable:
 819         /* something went wrong, try to boot without any XSAVE support */
 820         fpu__init_disable_system_xstate();
 821 }
 822
 823 /*
 824  * Restore minimal FPU state after suspend:
 825  */
 826 void fpu__resume_cpu(void)
 827 {
 828         /*
 829          * Restore XCR0 on xsave capable CPUs:
 830          */
 831         if (boot_cpu_has(X86_FEATURE_XSAVE))
 832                 xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask_user());
 833
 834         /*
 835          * Restore IA32_XSS. The same CPUID bit enumerates support
 836          * of XSAVES and MSR_IA32_XSS.
 837          */
 838         if (boot_cpu_has(X86_FEATURE_XSAVES)) {
 839                 wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor()  |
 840                                      xfeatures_mask_dynamic());
 841         }
 842 }
 843
 844 /*
 845  * Given an xstate feature nr, calculate where in the xsave
 846  * buffer the state is.  Callers should ensure that the buffer
 847  * is valid.
 848  */
 849 static void *__raw_xsave_addr(struct xregs_state *xsave, int xfeature_nr)
 850 {
 851         if (!xfeature_enabled(xfeature_nr)) {
 852                 WARN_ON_FPU(1);
 853                 return NULL;
 854         }
 855
 856         return (void *)xsave + xstate_comp_offsets[xfeature_nr];
 857 }
 858 /*
 859  * Given the xsave area and a state inside, this function returns the
 860  * address of the state.
 861  *
 862  * This is the API that is called to get xstate address in either
 863  * standard format or compacted format of xsave area.
 864  *
 865  * Note that if there is no data for the field in the xsave buffer
 866  * this will return NULL.
 867  *
 868  * Inputs:
 869  *      xstate: the thread's storage area for all FPU data
 870  *      xfeature_nr: state which is defined in xsave.h (e.g. XFEATURE_FP,
 871  *      XFEATURE_SSE, etc...)
 872  * Output:
 873  *      address of the state in the xsave area, or NULL if the
 874  *      field is not present in the xsave buffer.
 875  */
 876 void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr)
 877 {
 878         /*
 879          * Do we even *have* xsave state?
 880          */
 881         if (!boot_cpu_has(X86_FEATURE_XSAVE))
 882                 return NULL;
 883
 884         /*
 885          * We should not ever be requesting features that we
 886          * have not enabled.
 887          */
 888         WARN_ONCE(!(xfeatures_mask_all & BIT_ULL(xfeature_nr)),
 889                   "get of unsupported state");
 890         /*
 891          * This assumes the last 'xsave*' instruction to
 892          * have requested that 'xfeature_nr' be saved.
 893          * If it did not, we might be seeing and old value
 894          * of the field in the buffer.
 895          *
 896          * This can happen because the last 'xsave' did not
 897          * request that this feature be saved (unlikely)
 898          * or because the "init optimization" caused it
 899          * to not be saved.
 900          */
 901         if (!(xsave->header.xfeatures & BIT_ULL(xfeature_nr)))
 902                 return NULL;
 903
 904         return __raw_xsave_addr(xsave, xfeature_nr);
 905 }
 906 EXPORT_SYMBOL_GPL(get_xsave_addr);
 907
 908 #ifdef CONFIG_ARCH_HAS_PKEYS
 909
 910 /*
 911  * This will go out and modify PKRU register to set the access
 912  * rights for @pkey to @init_val.
 913  */
 914 int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
 915                               unsigned long init_val)
 916 {
 917         u32 old_pkru, new_pkru_bits = 0;
 918         int pkey_shift;
 919
 920         /*
 921          * This check implies XSAVE support.  OSPKE only gets
 922          * set if we enable XSAVE and we enable PKU in XCR0.
 923          */
 924         if (!boot_cpu_has(X86_FEATURE_OSPKE))
 925                 return -EINVAL;
 926
 927         /*
 928          * This code should only be called with valid 'pkey'
 929          * values originating from in-kernel users.  Complain
 930          * if a bad value is observed.
 931          */
 932         if (WARN_ON_ONCE(pkey >= arch_max_pkey()))
 933                 return -EINVAL;
 934
 935         /* Set the bits we need in PKRU:  */
 936         if (init_val & PKEY_DISABLE_ACCESS)
 937                 new_pkru_bits |= PKRU_AD_BIT;
 938         if (init_val & PKEY_DISABLE_WRITE)
 939                 new_pkru_bits |= PKRU_WD_BIT;
 940
 941         /* Shift the bits in to the correct place in PKRU for pkey: */
 942         pkey_shift = pkey * PKRU_BITS_PER_PKEY;
 943         new_pkru_bits <<= pkey_shift;
 944
 945         /* Get old PKRU and mask off any old bits in place: */
 946         old_pkru = read_pkru();
 947         old_pkru &= ~((PKRU_AD_BIT|PKRU_WD_BIT) << pkey_shift);
 948
 949         /* Write old part along with new part: */
 950         write_pkru(old_pkru | new_pkru_bits);
 951
 952         return 0;
 953 }
 954 #endif /* ! CONFIG_ARCH_HAS_PKEYS */
 955
 956 /*
 957  * Weird legacy quirk: SSE and YMM states store information in the
 958  * MXCSR and MXCSR_FLAGS fields of the FP area. That means if the FP
 959  * area is marked as unused in the xfeatures header, we need to copy
 960  * MXCSR and MXCSR_FLAGS if either SSE or YMM are in use.
 961  */
 962 static inline bool xfeatures_mxcsr_quirk(u64 xfeatures)
 963 {
 964         if (!(xfeatures & (XFEATURE_MASK_SSE|XFEATURE_MASK_YMM)))
 965                 return false;
 966
 967         if (xfeatures & XFEATURE_MASK_FP)
 968                 return false;
 969
 970         return true;
 971 }
 972
 973 static void copy_feature(bool from_xstate, struct membuf *to, void *xstate,
 974                          void *init_xstate, unsigned int size)
 975 {
 976         membuf_write(to, from_xstate ? xstate : init_xstate, size);
 977 }
 978
 979 /**
 980  * copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer
 981  * @to:         membuf descriptor
 982  * @xsave:      The kernel xstate buffer to copy from
 983  * @copy_mode:  The requested copy mode
 984  *
 985  * Converts from kernel XSAVE or XSAVES compacted format to UABI conforming
 986  * format, i.e. from the kernel internal hardware dependent storage format
 987  * to the requested @mode. UABI XSTATE is always uncompacted!
 988  *
 989  * It supports partial copy but @to.pos always starts from zero.
 990  */
 991 void copy_xstate_to_uabi_buf(struct membuf to, struct xregs_state *xsave,
 992                              enum xstate_copy_mode copy_mode)
 993 {
 994         const unsigned int off_mxcsr = offsetof(struct fxregs_state, mxcsr);
 995         struct xregs_state *xinit = &init_fpstate.xsave;
 996         struct xstate_header header;
 997         unsigned int zerofrom;
 998         int i;
 999
1000         header.xfeatures = xsave->header.xfeatures;
1001
1002         /* Mask out the feature bits depending on copy mode */
1003         switch (copy_mode) {
1004         case XSTATE_COPY_FP:
1005                 header.xfeatures &= XFEATURE_MASK_FP;
1006                 break;
1007
1008         case XSTATE_COPY_FX:
1009                 header.xfeatures &= XFEATURE_MASK_FP | XFEATURE_MASK_SSE;
1010                 break;
1011
1012         case XSTATE_COPY_XSAVE:
1013                 header.xfeatures &= xfeatures_mask_user();
1014                 break;
1015         }
1016
1017         /* Copy FP state up to MXCSR */
1018         copy_feature(header.xfeatures & XFEATURE_MASK_FP, &to, &xsave->i387,
1019                      &xinit->i387, off_mxcsr);
1020
1021         /* Copy MXCSR when SSE or YMM are set in the feature mask */
1022         copy_feature(header.xfeatures & (XFEATURE_MASK_SSE | XFEATURE_MASK_YMM),
1023                      &to, &xsave->i387.mxcsr, &xinit->i387.mxcsr,
1024                      MXCSR_AND_FLAGS_SIZE);
1025
1026         /* Copy the remaining FP state */
1027         copy_feature(header.xfeatures & XFEATURE_MASK_FP,
1028                      &to, &xsave->i387.st_space, &xinit->i387.st_space,
1029                      sizeof(xsave->i387.st_space));
1030
1031         /* Copy the SSE state - shared with YMM, but independently managed */
1032         copy_feature(header.xfeatures & XFEATURE_MASK_SSE,
1033                      &to, &xsave->i387.xmm_space, &xinit->i387.xmm_space,
1034                      sizeof(xsave->i387.xmm_space));
1035
1036         if (copy_mode != XSTATE_COPY_XSAVE)
1037                 goto out;
1038
1039         /* Zero the padding area */
1040         membuf_zero(&to, sizeof(xsave->i387.padding));
1041
1042         /* Copy xsave->i387.sw_reserved */
1043         membuf_write(&to, xstate_fx_sw_bytes, sizeof(xsave->i387.sw_reserved));
1044
1045         /* Copy the user space relevant state of @xsave->header */
1046         membuf_write(&to, &header, sizeof(header));
1047
1048         zerofrom = offsetof(struct xregs_state, extended_state_area);
1049
1050         for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
1051                 /*
1052                  * The ptrace buffer is in non-compacted XSAVE format.
1053                  * In non-compacted format disabled features still occupy
1054                  * state space, but there is no state to copy from in the
1055                  * compacted init_fpstate. The gap tracking will zero this
1056                  * later.
1057                  */
1058                 if (!(xfeatures_mask_user() & BIT_ULL(i)))
1059                         continue;
1060
1061                 /*
1062                  * If there was a feature or alignment gap, zero the space
1063                  * in the destination buffer.
1064                  */
1065                 if (zerofrom < xstate_offsets[i])
1066                         membuf_zero(&to, xstate_offsets[i] - zerofrom);
1067
1068                 copy_feature(header.xfeatures & BIT_ULL(i), &to,
1069                              __raw_xsave_addr(xsave, i),
1070                              __raw_xsave_addr(xinit, i),
1071                              xstate_sizes[i]);
1072
1073                 /*
1074                  * Keep track of the last copied state in the non-compacted
1075                  * target buffer for gap zeroing.
1076                  */
1077                 zerofrom = xstate_offsets[i] + xstate_sizes[i];
1078         }
1079
1080 out:
1081         if (to.left)
1082                 membuf_zero(&to, to.left);
1083 }
1084
1085 static inline bool mxcsr_valid(struct xstate_header *hdr, const u32 *mxcsr)
1086 {
1087         u64 mask = XFEATURE_MASK_FP | XFEATURE_MASK_SSE | XFEATURE_MASK_YMM;
1088
1089         /* Only check if it is in use */
1090         if (hdr->xfeatures & mask) {
1091                 /* Reserved bits in MXCSR must be zero. */
1092                 if (*mxcsr & ~mxcsr_feature_mask)
1093                         return false;
1094         }
1095         return true;
1096 }
1097
1098 /*
1099  * Convert from a ptrace standard-format kernel buffer to kernel XSAVE[S] format
1100  * and copy to the target thread. This is called from xstateregs_set().
1101  */
1102 int copy_kernel_to_xstate(struct xregs_state *xsave, const void *kbuf)
1103 {
1104         unsigned int offset, size;
1105         int i;
1106         struct xstate_header hdr;
1107
1108         offset = offsetof(struct xregs_state, header);
1109         size = sizeof(hdr);
1110
1111         memcpy(&hdr, kbuf + offset, size);
1112
1113         if (validate_user_xstate_header(&hdr))
1114                 return -EINVAL;
1115
1116         if (!mxcsr_valid(&hdr, kbuf + offsetof(struct fxregs_state, mxcsr)))
1117                 return -EINVAL;
1118
1119         for (i = 0; i < XFEATURE_MAX; i++) {
1120                 u64 mask = ((u64)1 << i);
1121
1122                 if (hdr.xfeatures & mask) {
1123                         void *dst = __raw_xsave_addr(xsave, i);
1124
1125                         offset = xstate_offsets[i];
1126                         size = xstate_sizes[i];
1127
1128                         memcpy(dst, kbuf + offset, size);
1129                 }
1130         }
1131
1132         if (xfeatures_mxcsr_quirk(hdr.xfeatures)) {
1133                 offset = offsetof(struct fxregs_state, mxcsr);
1134                 size = MXCSR_AND_FLAGS_SIZE;
1135                 memcpy(&xsave->i387.mxcsr, kbuf + offset, size);
1136         }
1137
1138         /*
1139          * The state that came in from userspace was user-state only.
1140          * Mask all the user states out of 'xfeatures':
1141          */
1142         xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR_ALL;
1143
1144         /*
1145          * Add back in the features that came in from userspace:
1146          */
1147         xsave->header.xfeatures |= hdr.xfeatures;
1148
1149         return 0;
1150 }
1151
1152 /*
1153  * Convert from a sigreturn standard-format user-space buffer to kernel
1154  * XSAVE[S] format and copy to the target thread. This is called from the
1155  * sigreturn() and rt_sigreturn() system calls.
1156  */
1157 int copy_user_to_xstate(struct xregs_state *xsave, const void __user *ubuf)
1158 {
1159         unsigned int offset, size;
1160         int i;
1161         struct xstate_header hdr;
1162
1163         offset = offsetof(struct xregs_state, header);
1164         size = sizeof(hdr);
1165
1166         if (copy_from_user(&hdr, ubuf + offset, size))
1167                 return -EFAULT;
1168
1169         if (validate_user_xstate_header(&hdr))
1170                 return -EINVAL;
1171
1172         for (i = 0; i < XFEATURE_MAX; i++) {
1173                 u64 mask = ((u64)1 << i);
1174
1175                 if (hdr.xfeatures & mask) {
1176                         void *dst = __raw_xsave_addr(xsave, i);
1177
1178                         offset = xstate_offsets[i];
1179                         size = xstate_sizes[i];
1180
1181                         if (copy_from_user(dst, ubuf + offset, size))
1182                                 return -EFAULT;
1183                 }
1184         }
1185
1186         if (xfeatures_mxcsr_quirk(hdr.xfeatures)) {
1187                 offset = offsetof(struct fxregs_state, mxcsr);
1188                 size = MXCSR_AND_FLAGS_SIZE;
1189                 if (copy_from_user(&xsave->i387.mxcsr, ubuf + offset, size))
1190                         return -EFAULT;
1191         }
1192
1193         /*
1194          * The state that came in from userspace was user-state only.
1195          * Mask all the user states out of 'xfeatures':
1196          */
1197         xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR_ALL;
1198
1199         /*
1200          * Add back in the features that came in from userspace:
1201          */
1202         xsave->header.xfeatures |= hdr.xfeatures;
1203
1204         return 0;
1205 }
1206
1207 /**
1208  * copy_dynamic_supervisor_to_kernel() - Save dynamic supervisor states to
1209  *                                       an xsave area
1210  * @xstate: A pointer to an xsave area
1211  * @mask: Represent the dynamic supervisor features saved into the xsave area
1212  *
1213  * Only the dynamic supervisor states sets in the mask are saved into the xsave
1214  * area (See the comment in XFEATURE_MASK_DYNAMIC for the details of dynamic
1215  * supervisor feature). Besides the dynamic supervisor states, the legacy
1216  * region and XSAVE header are also saved into the xsave area. The supervisor
1217  * features in the XFEATURE_MASK_SUPERVISOR_SUPPORTED and
1218  * XFEATURE_MASK_SUPERVISOR_UNSUPPORTED are not saved.
1219  *
1220  * The xsave area must be 64-bytes aligned.
1221  */
1222 void copy_dynamic_supervisor_to_kernel(struct xregs_state *xstate, u64 mask)
1223 {
1224         u64 dynamic_mask = xfeatures_mask_dynamic() & mask;
1225         u32 lmask, hmask;
1226         int err;
1227
1228         if (WARN_ON_FPU(!boot_cpu_has(X86_FEATURE_XSAVES)))
1229                 return;
1230
1231         if (WARN_ON_FPU(!dynamic_mask))
1232                 return;
1233
1234         lmask = dynamic_mask;
1235         hmask = dynamic_mask >> 32;
1236
1237         XSTATE_OP(XSAVES, xstate, lmask, hmask, err);
1238
1239         /* Should never fault when copying to a kernel buffer */
1240         WARN_ON_FPU(err);
1241 }
1242
1243 /**
1244  * copy_kernel_to_dynamic_supervisor() - Restore dynamic supervisor states from
1245  *                                       an xsave area
1246  * @xstate: A pointer to an xsave area
1247  * @mask: Represent the dynamic supervisor features restored from the xsave area
1248  *
1249  * Only the dynamic supervisor states sets in the mask are restored from the
1250  * xsave area (See the comment in XFEATURE_MASK_DYNAMIC for the details of
1251  * dynamic supervisor feature). Besides the dynamic supervisor states, the
1252  * legacy region and XSAVE header are also restored from the xsave area. The
1253  * supervisor features in the XFEATURE_MASK_SUPERVISOR_SUPPORTED and
1254  * XFEATURE_MASK_SUPERVISOR_UNSUPPORTED are not restored.
1255  *
1256  * The xsave area must be 64-bytes aligned.
1257  */
1258 void copy_kernel_to_dynamic_supervisor(struct xregs_state *xstate, u64 mask)
1259 {
1260         u64 dynamic_mask = xfeatures_mask_dynamic() & mask;
1261         u32 lmask, hmask;
1262         int err;
1263
1264         if (WARN_ON_FPU(!boot_cpu_has(X86_FEATURE_XSAVES)))
1265                 return;
1266
1267         if (WARN_ON_FPU(!dynamic_mask))
1268                 return;
1269
1270         lmask = dynamic_mask;
1271         hmask = dynamic_mask >> 32;
1272
1273         XSTATE_OP(XRSTORS, xstate, lmask, hmask, err);
1274
1275         /* Should never fault when copying from a kernel buffer */
1276         WARN_ON_FPU(err);
1277 }
1278
1279 #ifdef CONFIG_PROC_PID_ARCH_STATUS
1280 /*
1281  * Report the amount of time elapsed in millisecond since last AVX512
1282  * use in the task.
1283  */
1284 static void avx512_status(struct seq_file *m, struct task_struct *task)
1285 {
1286         unsigned long timestamp = READ_ONCE(task->thread.fpu.avx512_timestamp);
1287         long delta;
1288
1289         if (!timestamp) {
1290                 /*
1291                  * Report -1 if no AVX512 usage
1292                  */
1293                 delta = -1;
1294         } else {
1295                 delta = (long)(jiffies - timestamp);
1296                 /*
1297                  * Cap to LONG_MAX if time difference > LONG_MAX
1298                  */
1299                 if (delta < 0)
1300                         delta = LONG_MAX;
1301                 delta = jiffies_to_msecs(delta);
1302         }
1303
1304         seq_put_decimal_ll(m, "AVX512_elapsed_ms:\t", delta);
1305         seq_putc(m, '\n');
1306 }
1307
1308 /*
1309  * Report architecture specific information
1310  */
1311 int proc_pid_arch_status(struct seq_file *m, struct pid_namespace *ns,
1312                         struct pid *pid, struct task_struct *task)
1313 {
1314         /*
1315          * Report AVX512 state if the processor and build option supported.
1316          */
1317         if (cpu_feature_enabled(X86_FEATURE_AVX512F))
1318                 avx512_status(m, task);
1319
1320         return 0;
1321 }
1322 #endif /* CONFIG_PROC_PID_ARCH_STATUS */