1 // SPDX-License-Identifier: GPL-2.0-only
3 * xsave/xrstor support.
5 * Author: Suresh Siddha <suresh.b.siddha@intel.com>
7 #include <linux/compat.h>
9 #include <linux/mman.h>
10 #include <linux/pkeys.h>
11 #include <linux/seq_file.h>
12 #include <linux/proc_fs.h>
14 #include <asm/fpu/api.h>
15 #include <asm/fpu/internal.h>
16 #include <asm/fpu/signal.h>
17 #include <asm/fpu/regset.h>
18 #include <asm/fpu/xstate.h>
20 #include <asm/tlbflush.h>
21 #include <asm/cpufeature.h>
24 * Although we spell it out in here, the Processor Trace
25 * xfeature is completely unused. We use other mechanisms
26 * to save/restore PT state in Linux.
28 static const char *xfeature_names[] =
30 "x87 floating point registers" ,
33 "MPX bounds registers" ,
38 "Processor Trace (unused)" ,
39 "Protection Keys User registers",
41 "unknown xstate feature" ,
44 static short xsave_cpuid_features[] __initdata = {
59 * This represents the full set of bits that should ever be set in a kernel
60 * XSAVE buffer, both supervisor and user xstates.
62 u64 xfeatures_mask_all __ro_after_init;
64 static unsigned int xstate_offsets[XFEATURE_MAX] __ro_after_init =
65 { [ 0 ... XFEATURE_MAX - 1] = -1};
66 static unsigned int xstate_sizes[XFEATURE_MAX] __ro_after_init =
67 { [ 0 ... XFEATURE_MAX - 1] = -1};
68 static unsigned int xstate_comp_offsets[XFEATURE_MAX] __ro_after_init =
69 { [ 0 ... XFEATURE_MAX - 1] = -1};
70 static unsigned int xstate_supervisor_only_offsets[XFEATURE_MAX] __ro_after_init =
71 { [ 0 ... XFEATURE_MAX - 1] = -1};
74 * The XSAVE area of kernel can be in standard or compacted format;
75 * it is always in standard format for user mode. This is the user
76 * mode standard format size used for signal and ptrace frames.
78 unsigned int fpu_user_xstate_size __ro_after_init;
81 * Return whether the system supports a given xfeature.
83 * Also return the name of the (most advanced) feature that the caller requested:
85 int cpu_has_xfeatures(u64 xfeatures_needed, const char **feature_name)
87 u64 xfeatures_missing = xfeatures_needed & ~xfeatures_mask_all;
89 if (unlikely(feature_name)) {
90 long xfeature_idx, max_idx;
93 * So we use FLS here to be able to print the most advanced
94 * feature that was requested but is missing. So if a driver
95 * asks about "XFEATURE_MASK_SSE | XFEATURE_MASK_YMM" we'll print the
96 * missing AVX feature - this is the most informative message
99 if (xfeatures_missing)
100 xfeatures_print = xfeatures_missing;
102 xfeatures_print = xfeatures_needed;
104 xfeature_idx = fls64(xfeatures_print)-1;
105 max_idx = ARRAY_SIZE(xfeature_names)-1;
106 xfeature_idx = min(xfeature_idx, max_idx);
108 *feature_name = xfeature_names[xfeature_idx];
111 if (xfeatures_missing)
116 EXPORT_SYMBOL_GPL(cpu_has_xfeatures);
118 static bool xfeature_is_supervisor(int xfeature_nr)
121 * Extended State Enumeration Sub-leaves (EAX = 0DH, ECX = n, n > 1)
122 * returns ECX[0] set to (1) for a supervisor state, and cleared (0)
125 u32 eax, ebx, ecx, edx;
127 cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
132 * Enable the extended processor state save/restore feature.
133 * Called once per CPU onlining.
135 void fpu__init_cpu_xstate(void)
137 if (!boot_cpu_has(X86_FEATURE_XSAVE) || !xfeatures_mask_all)
140 cr4_set_bits(X86_CR4_OSXSAVE);
143 * XCR_XFEATURE_ENABLED_MASK (aka. XCR0) sets user features
144 * managed by XSAVE{C, OPT, S} and XRSTOR{S}. Only XSAVE user
145 * states can be set here.
147 xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask_user());
150 * MSR_IA32_XSS sets supervisor states managed by XSAVES.
152 if (boot_cpu_has(X86_FEATURE_XSAVES)) {
153 wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() |
154 xfeatures_mask_dynamic());
158 static bool xfeature_enabled(enum xfeature xfeature)
160 return xfeatures_mask_all & BIT_ULL(xfeature);
164 * Record the offsets and sizes of various xstates contained
165 * in the XSAVE state memory layout.
167 static void __init setup_xstate_features(void)
169 u32 eax, ebx, ecx, edx, i;
170 /* start at the beginning of the "extended state" */
171 unsigned int last_good_offset = offsetof(struct xregs_state,
172 extended_state_area);
174 * The FP xstates and SSE xstates are legacy states. They are always
175 * in the fixed offsets in the xsave area in either compacted form
178 xstate_offsets[XFEATURE_FP] = 0;
179 xstate_sizes[XFEATURE_FP] = offsetof(struct fxregs_state,
182 xstate_offsets[XFEATURE_SSE] = xstate_sizes[XFEATURE_FP];
183 xstate_sizes[XFEATURE_SSE] = sizeof_field(struct fxregs_state,
186 for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
187 if (!xfeature_enabled(i))
190 cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);
192 xstate_sizes[i] = eax;
195 * If an xfeature is supervisor state, the offset in EBX is
196 * invalid, leave it to -1.
198 if (xfeature_is_supervisor(i))
201 xstate_offsets[i] = ebx;
204 * In our xstate size checks, we assume that the highest-numbered
205 * xstate feature has the highest offset in the buffer. Ensure
208 WARN_ONCE(last_good_offset > xstate_offsets[i],
209 "x86/fpu: misordered xstate at %d\n", last_good_offset);
211 last_good_offset = xstate_offsets[i];
215 static void __init print_xstate_feature(u64 xstate_mask)
217 const char *feature_name;
219 if (cpu_has_xfeatures(xstate_mask, &feature_name))
220 pr_info("x86/fpu: Supporting XSAVE feature 0x%03Lx: '%s'\n", xstate_mask, feature_name);
224 * Print out all the supported xstate features:
226 static void __init print_xstate_features(void)
228 print_xstate_feature(XFEATURE_MASK_FP);
229 print_xstate_feature(XFEATURE_MASK_SSE);
230 print_xstate_feature(XFEATURE_MASK_YMM);
231 print_xstate_feature(XFEATURE_MASK_BNDREGS);
232 print_xstate_feature(XFEATURE_MASK_BNDCSR);
233 print_xstate_feature(XFEATURE_MASK_OPMASK);
234 print_xstate_feature(XFEATURE_MASK_ZMM_Hi256);
235 print_xstate_feature(XFEATURE_MASK_Hi16_ZMM);
236 print_xstate_feature(XFEATURE_MASK_PKRU);
237 print_xstate_feature(XFEATURE_MASK_PASID);
241 * This check is important because it is easy to get XSTATE_*
242 * confused with XSTATE_BIT_*.
244 #define CHECK_XFEATURE(nr) do { \
245 WARN_ON(nr < FIRST_EXTENDED_XFEATURE); \
246 WARN_ON(nr >= XFEATURE_MAX); \
250 * We could cache this like xstate_size[], but we only use
251 * it here, so it would be a waste of space.
253 static int xfeature_is_aligned(int xfeature_nr)
255 u32 eax, ebx, ecx, edx;
257 CHECK_XFEATURE(xfeature_nr);
259 if (!xfeature_enabled(xfeature_nr)) {
260 WARN_ONCE(1, "Checking alignment of disabled xfeature %d\n",
265 cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
267 * The value returned by ECX[1] indicates the alignment
268 * of state component 'i' when the compacted format
269 * of the extended region of an XSAVE area is used:
275 * This function sets up offsets and sizes of all extended states in
276 * xsave area. This supports both standard format and compacted format
279 static void __init setup_xstate_comp_offsets(void)
281 unsigned int next_offset;
285 * The FP xstates and SSE xstates are legacy states. They are always
286 * in the fixed offsets in the xsave area in either compacted form
289 xstate_comp_offsets[XFEATURE_FP] = 0;
290 xstate_comp_offsets[XFEATURE_SSE] = offsetof(struct fxregs_state,
293 if (!boot_cpu_has(X86_FEATURE_XSAVES)) {
294 for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
295 if (xfeature_enabled(i))
296 xstate_comp_offsets[i] = xstate_offsets[i];
301 next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE;
303 for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
304 if (!xfeature_enabled(i))
307 if (xfeature_is_aligned(i))
308 next_offset = ALIGN(next_offset, 64);
310 xstate_comp_offsets[i] = next_offset;
311 next_offset += xstate_sizes[i];
316 * Setup offsets of a supervisor-state-only XSAVES buffer:
318 * The offsets stored in xstate_comp_offsets[] only work for one specific
319 * value of the Requested Feature BitMap (RFBM). In cases where a different
320 * RFBM value is used, a different set of offsets is required. This set of
321 * offsets is for when RFBM=xfeatures_mask_supervisor().
323 static void __init setup_supervisor_only_offsets(void)
325 unsigned int next_offset;
328 next_offset = FXSAVE_SIZE + XSAVE_HDR_SIZE;
330 for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
331 if (!xfeature_enabled(i) || !xfeature_is_supervisor(i))
334 if (xfeature_is_aligned(i))
335 next_offset = ALIGN(next_offset, 64);
337 xstate_supervisor_only_offsets[i] = next_offset;
338 next_offset += xstate_sizes[i];
343 * Print out xstate component offsets and sizes
345 static void __init print_xstate_offset_size(void)
349 for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
350 if (!xfeature_enabled(i))
352 pr_info("x86/fpu: xstate_offset[%d]: %4d, xstate_sizes[%d]: %4d\n",
353 i, xstate_comp_offsets[i], i, xstate_sizes[i]);
358 * All supported features have either init state all zeros or are
359 * handled in setup_init_fpu() individually. This is an explicit
360 * feature list and does not use XFEATURE_MASK*SUPPORTED to catch
361 * newly added supported features at build time and make people
362 * actually look at the init state for the new feature.
364 #define XFEATURES_INIT_FPSTATE_HANDLED \
365 (XFEATURE_MASK_FP | \
366 XFEATURE_MASK_SSE | \
367 XFEATURE_MASK_YMM | \
368 XFEATURE_MASK_OPMASK | \
369 XFEATURE_MASK_ZMM_Hi256 | \
370 XFEATURE_MASK_Hi16_ZMM | \
371 XFEATURE_MASK_PKRU | \
372 XFEATURE_MASK_BNDREGS | \
373 XFEATURE_MASK_BNDCSR | \
377 * setup the xstate image representing the init state
379 static void __init setup_init_fpu_buf(void)
381 static int on_boot_cpu __initdata = 1;
383 BUILD_BUG_ON((XFEATURE_MASK_USER_SUPPORTED |
384 XFEATURE_MASK_SUPERVISOR_SUPPORTED) !=
385 XFEATURES_INIT_FPSTATE_HANDLED);
387 WARN_ON_FPU(!on_boot_cpu);
390 if (!boot_cpu_has(X86_FEATURE_XSAVE))
393 setup_xstate_features();
394 print_xstate_features();
396 if (boot_cpu_has(X86_FEATURE_XSAVES))
397 init_fpstate.xsave.header.xcomp_bv = XCOMP_BV_COMPACTED_FORMAT |
401 * Init all the features state with header.xfeatures being 0x0
403 copy_kernel_to_xregs_booting(&init_fpstate.xsave);
406 * All components are now in init state. Read the state back so
407 * that init_fpstate contains all non-zero init state. This only
408 * works with XSAVE, but not with XSAVEOPT and XSAVES because
409 * those use the init optimization which skips writing data for
410 * components in init state.
412 * XSAVE could be used, but that would require to reshuffle the
413 * data when XSAVES is available because XSAVES uses xstate
414 * compaction. But doing so is a pointless exercise because most
415 * components have an all zeros init state except for the legacy
416 * ones (FP and SSE). Those can be saved with FXSAVE into the
417 * legacy area. Adding new features requires to ensure that init
418 * state is all zeroes or if not to add the necessary handling
421 fxsave(&init_fpstate.fxsave);
424 static int xfeature_uncompacted_offset(int xfeature_nr)
426 u32 eax, ebx, ecx, edx;
429 * Only XSAVES supports supervisor states and it uses compacted
430 * format. Checking a supervisor state's uncompacted offset is
433 if (XFEATURE_MASK_SUPERVISOR_ALL & BIT_ULL(xfeature_nr)) {
434 WARN_ONCE(1, "No fixed offset for xstate %d\n", xfeature_nr);
438 CHECK_XFEATURE(xfeature_nr);
439 cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
443 int xfeature_size(int xfeature_nr)
445 u32 eax, ebx, ecx, edx;
447 CHECK_XFEATURE(xfeature_nr);
448 cpuid_count(XSTATE_CPUID, xfeature_nr, &eax, &ebx, &ecx, &edx);
453 * 'XSAVES' implies two different things:
454 * 1. saving of supervisor/system state
455 * 2. using the compacted format
457 * Use this function when dealing with the compacted format so
458 * that it is obvious which aspect of 'XSAVES' is being handled
459 * by the calling code.
461 int using_compacted_format(void)
463 return boot_cpu_has(X86_FEATURE_XSAVES);
466 /* Validate an xstate header supplied by userspace (ptrace or sigreturn) */
467 static int validate_user_xstate_header(const struct xstate_header *hdr)
469 /* No unknown or supervisor features may be set */
470 if (hdr->xfeatures & ~xfeatures_mask_user())
473 /* Userspace must use the uncompacted format */
478 * If 'reserved' is shrunken to add a new field, make sure to validate
479 * that new field here!
481 BUILD_BUG_ON(sizeof(hdr->reserved) != 48);
483 /* No reserved bits may be set */
484 if (memchr_inv(hdr->reserved, 0, sizeof(hdr->reserved)))
490 static void __xstate_dump_leaves(void)
493 u32 eax, ebx, ecx, edx;
494 static int should_dump = 1;
500 * Dump out a few leaves past the ones that we support
501 * just in case there are some goodies up there
503 for (i = 0; i < XFEATURE_MAX + 10; i++) {
504 cpuid_count(XSTATE_CPUID, i, &eax, &ebx, &ecx, &edx);
505 pr_warn("CPUID[%02x, %02x]: eax=%08x ebx=%08x ecx=%08x edx=%08x\n",
506 XSTATE_CPUID, i, eax, ebx, ecx, edx);
510 #define XSTATE_WARN_ON(x) do { \
511 if (WARN_ONCE(x, "XSAVE consistency problem, dumping leaves")) { \
512 __xstate_dump_leaves(); \
516 #define XCHECK_SZ(sz, nr, nr_macro, __struct) do { \
517 if ((nr == nr_macro) && \
518 WARN_ONCE(sz != sizeof(__struct), \
519 "%s: struct is %zu bytes, cpu state %d bytes\n", \
520 __stringify(nr_macro), sizeof(__struct), sz)) { \
521 __xstate_dump_leaves(); \
526 * We have a C struct for each 'xstate'. We need to ensure
527 * that our software representation matches what the CPU
528 * tells us about the state's size.
530 static void check_xstate_against_struct(int nr)
533 * Ask the CPU for the size of the state.
535 int sz = xfeature_size(nr);
537 * Match each CPU state with the corresponding software
540 XCHECK_SZ(sz, nr, XFEATURE_YMM, struct ymmh_struct);
541 XCHECK_SZ(sz, nr, XFEATURE_BNDREGS, struct mpx_bndreg_state);
542 XCHECK_SZ(sz, nr, XFEATURE_BNDCSR, struct mpx_bndcsr_state);
543 XCHECK_SZ(sz, nr, XFEATURE_OPMASK, struct avx_512_opmask_state);
544 XCHECK_SZ(sz, nr, XFEATURE_ZMM_Hi256, struct avx_512_zmm_uppers_state);
545 XCHECK_SZ(sz, nr, XFEATURE_Hi16_ZMM, struct avx_512_hi16_state);
546 XCHECK_SZ(sz, nr, XFEATURE_PKRU, struct pkru_state);
547 XCHECK_SZ(sz, nr, XFEATURE_PASID, struct ia32_pasid_state);
550 * Make *SURE* to add any feature numbers in below if
551 * there are "holes" in the xsave state component
554 if ((nr < XFEATURE_YMM) ||
555 (nr >= XFEATURE_MAX) ||
556 (nr == XFEATURE_PT_UNIMPLEMENTED_SO_FAR) ||
557 ((nr >= XFEATURE_RSRVD_COMP_11) && (nr <= XFEATURE_LBR))) {
558 WARN_ONCE(1, "no structure for xstate: %d\n", nr);
564 * This essentially double-checks what the cpu told us about
565 * how large the XSAVE buffer needs to be. We are recalculating
568 * Dynamic XSAVE features allocate their own buffers and are not
569 * covered by these checks. Only the size of the buffer for task->fpu
572 static void do_extra_xstate_size_checks(void)
574 int paranoid_xstate_size = FXSAVE_SIZE + XSAVE_HDR_SIZE;
577 for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
578 if (!xfeature_enabled(i))
581 check_xstate_against_struct(i);
583 * Supervisor state components can be managed only by
584 * XSAVES, which is compacted-format only.
586 if (!using_compacted_format())
587 XSTATE_WARN_ON(xfeature_is_supervisor(i));
589 /* Align from the end of the previous feature */
590 if (xfeature_is_aligned(i))
591 paranoid_xstate_size = ALIGN(paranoid_xstate_size, 64);
593 * The offset of a given state in the non-compacted
594 * format is given to us in a CPUID leaf. We check
595 * them for being ordered (increasing offsets) in
596 * setup_xstate_features().
598 if (!using_compacted_format())
599 paranoid_xstate_size = xfeature_uncompacted_offset(i);
601 * The compacted-format offset always depends on where
602 * the previous state ended.
604 paranoid_xstate_size += xfeature_size(i);
606 XSTATE_WARN_ON(paranoid_xstate_size != fpu_kernel_xstate_size);
611 * Get total size of enabled xstates in XCR0 | IA32_XSS.
613 * Note the SDM's wording here. "sub-function 0" only enumerates
614 * the size of the *user* states. If we use it to size a buffer
615 * that we use 'XSAVES' on, we could potentially overflow the
616 * buffer because 'XSAVES' saves system states too.
618 static unsigned int __init get_xsaves_size(void)
620 unsigned int eax, ebx, ecx, edx;
622 * - CPUID function 0DH, sub-function 1:
623 * EBX enumerates the size (in bytes) required by
624 * the XSAVES instruction for an XSAVE area
625 * containing all the state components
626 * corresponding to bits currently set in
629 cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx);
634 * Get the total size of the enabled xstates without the dynamic supervisor
637 static unsigned int __init get_xsaves_size_no_dynamic(void)
639 u64 mask = xfeatures_mask_dynamic();
643 return get_xsaves_size();
645 /* Disable dynamic features. */
646 wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor());
649 * Ask the hardware what size is required of the buffer.
650 * This is the size required for the task->fpu buffer.
652 size = get_xsaves_size();
654 /* Re-enable dynamic features so XSAVES will work on them again. */
655 wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() | mask);
660 static unsigned int __init get_xsave_size(void)
662 unsigned int eax, ebx, ecx, edx;
664 * - CPUID function 0DH, sub-function 0:
665 * EBX enumerates the size (in bytes) required by
666 * the XSAVE instruction for an XSAVE area
667 * containing all the *user* state components
668 * corresponding to bits currently set in XCR0.
670 cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
675 * Will the runtime-enumerated 'xstate_size' fit in the init
676 * task's statically-allocated buffer?
678 static bool is_supported_xstate_size(unsigned int test_xstate_size)
680 if (test_xstate_size <= sizeof(union fpregs_state))
683 pr_warn("x86/fpu: xstate buffer too small (%zu < %d), disabling xsave\n",
684 sizeof(union fpregs_state), test_xstate_size);
688 static int __init init_xstate_size(void)
690 /* Recompute the context size for enabled features: */
691 unsigned int possible_xstate_size;
692 unsigned int xsave_size;
694 xsave_size = get_xsave_size();
696 if (boot_cpu_has(X86_FEATURE_XSAVES))
697 possible_xstate_size = get_xsaves_size_no_dynamic();
699 possible_xstate_size = xsave_size;
701 /* Ensure we have the space to store all enabled: */
702 if (!is_supported_xstate_size(possible_xstate_size))
706 * The size is OK, we are definitely going to use xsave,
707 * make it known to the world that we need more space.
709 fpu_kernel_xstate_size = possible_xstate_size;
710 do_extra_xstate_size_checks();
713 * User space is always in standard format.
715 fpu_user_xstate_size = xsave_size;
720 * We enabled the XSAVE hardware, but something went wrong and
721 * we can not use it. Disable it.
723 static void fpu__init_disable_system_xstate(void)
725 xfeatures_mask_all = 0;
726 cr4_clear_bits(X86_CR4_OSXSAVE);
727 setup_clear_cpu_cap(X86_FEATURE_XSAVE);
731 * Enable and initialize the xsave feature.
732 * Called once per system bootup.
734 void __init fpu__init_system_xstate(void)
736 unsigned int eax, ebx, ecx, edx;
737 static int on_boot_cpu __initdata = 1;
742 WARN_ON_FPU(!on_boot_cpu);
745 if (!boot_cpu_has(X86_FEATURE_FPU)) {
746 pr_info("x86/fpu: No FPU detected\n");
750 if (!boot_cpu_has(X86_FEATURE_XSAVE)) {
751 pr_info("x86/fpu: x87 FPU will use %s\n",
752 boot_cpu_has(X86_FEATURE_FXSR) ? "FXSAVE" : "FSAVE");
756 if (boot_cpu_data.cpuid_level < XSTATE_CPUID) {
762 * Find user xstates supported by the processor.
764 cpuid_count(XSTATE_CPUID, 0, &eax, &ebx, &ecx, &edx);
765 xfeatures_mask_all = eax + ((u64)edx << 32);
768 * Find supervisor xstates supported by the processor.
770 cpuid_count(XSTATE_CPUID, 1, &eax, &ebx, &ecx, &edx);
771 xfeatures_mask_all |= ecx + ((u64)edx << 32);
773 if ((xfeatures_mask_user() & XFEATURE_MASK_FPSSE) != XFEATURE_MASK_FPSSE) {
775 * This indicates that something really unexpected happened
776 * with the enumeration. Disable XSAVE and try to continue
777 * booting without it. This is too early to BUG().
779 pr_err("x86/fpu: FP/SSE not present amongst the CPU's xstate features: 0x%llx.\n",
785 * Clear XSAVE features that are disabled in the normal CPUID.
787 for (i = 0; i < ARRAY_SIZE(xsave_cpuid_features); i++) {
788 if (!boot_cpu_has(xsave_cpuid_features[i]))
789 xfeatures_mask_all &= ~BIT_ULL(i);
792 xfeatures_mask_all &= XFEATURE_MASK_USER_SUPPORTED |
793 XFEATURE_MASK_SUPERVISOR_SUPPORTED;
795 /* Store it for paranoia check at the end */
796 xfeatures = xfeatures_mask_all;
798 /* Enable xstate instructions to be able to continue with initialization: */
799 fpu__init_cpu_xstate();
800 err = init_xstate_size();
805 * Update info used for ptrace frames; use standard-format size and no
806 * supervisor xstates:
808 update_regset_xstate_info(fpu_user_xstate_size, xfeatures_mask_user());
810 fpu__init_prepare_fx_sw_frame();
811 setup_init_fpu_buf();
812 setup_xstate_comp_offsets();
813 setup_supervisor_only_offsets();
816 * Paranoia check whether something in the setup modified the
819 if (xfeatures != xfeatures_mask_all) {
820 pr_err("x86/fpu: xfeatures modified from 0x%016llx to 0x%016llx during init, disabling XSAVE\n",
821 xfeatures, xfeatures_mask_all);
825 print_xstate_offset_size();
826 pr_info("x86/fpu: Enabled xstate features 0x%llx, context size is %d bytes, using '%s' format.\n",
828 fpu_kernel_xstate_size,
829 boot_cpu_has(X86_FEATURE_XSAVES) ? "compacted" : "standard");
833 /* something went wrong, try to boot without any XSAVE support */
834 fpu__init_disable_system_xstate();
838 * Restore minimal FPU state after suspend:
840 void fpu__resume_cpu(void)
843 * Restore XCR0 on xsave capable CPUs:
845 if (boot_cpu_has(X86_FEATURE_XSAVE))
846 xsetbv(XCR_XFEATURE_ENABLED_MASK, xfeatures_mask_user());
849 * Restore IA32_XSS. The same CPUID bit enumerates support
850 * of XSAVES and MSR_IA32_XSS.
852 if (boot_cpu_has(X86_FEATURE_XSAVES)) {
853 wrmsrl(MSR_IA32_XSS, xfeatures_mask_supervisor() |
854 xfeatures_mask_dynamic());
859 * Given an xstate feature nr, calculate where in the xsave
860 * buffer the state is. Callers should ensure that the buffer
863 static void *__raw_xsave_addr(struct xregs_state *xsave, int xfeature_nr)
865 if (!xfeature_enabled(xfeature_nr)) {
870 return (void *)xsave + xstate_comp_offsets[xfeature_nr];
873 * Given the xsave area and a state inside, this function returns the
874 * address of the state.
876 * This is the API that is called to get xstate address in either
877 * standard format or compacted format of xsave area.
879 * Note that if there is no data for the field in the xsave buffer
880 * this will return NULL.
883 * xstate: the thread's storage area for all FPU data
884 * xfeature_nr: state which is defined in xsave.h (e.g. XFEATURE_FP,
885 * XFEATURE_SSE, etc...)
887 * address of the state in the xsave area, or NULL if the
888 * field is not present in the xsave buffer.
890 void *get_xsave_addr(struct xregs_state *xsave, int xfeature_nr)
893 * Do we even *have* xsave state?
895 if (!boot_cpu_has(X86_FEATURE_XSAVE))
899 * We should not ever be requesting features that we
902 WARN_ONCE(!(xfeatures_mask_all & BIT_ULL(xfeature_nr)),
903 "get of unsupported state");
905 * This assumes the last 'xsave*' instruction to
906 * have requested that 'xfeature_nr' be saved.
907 * If it did not, we might be seeing and old value
908 * of the field in the buffer.
910 * This can happen because the last 'xsave' did not
911 * request that this feature be saved (unlikely)
912 * or because the "init optimization" caused it
915 if (!(xsave->header.xfeatures & BIT_ULL(xfeature_nr)))
918 return __raw_xsave_addr(xsave, xfeature_nr);
920 EXPORT_SYMBOL_GPL(get_xsave_addr);
922 #ifdef CONFIG_ARCH_HAS_PKEYS
925 * This will go out and modify PKRU register to set the access
926 * rights for @pkey to @init_val.
928 int arch_set_user_pkey_access(struct task_struct *tsk, int pkey,
929 unsigned long init_val)
932 int pkey_shift = (pkey * PKRU_BITS_PER_PKEY);
933 u32 new_pkru_bits = 0;
936 * This check implies XSAVE support. OSPKE only gets
937 * set if we enable XSAVE and we enable PKU in XCR0.
939 if (!boot_cpu_has(X86_FEATURE_OSPKE))
943 * This code should only be called with valid 'pkey'
944 * values originating from in-kernel users. Complain
945 * if a bad value is observed.
947 WARN_ON_ONCE(pkey >= arch_max_pkey());
949 /* Set the bits we need in PKRU: */
950 if (init_val & PKEY_DISABLE_ACCESS)
951 new_pkru_bits |= PKRU_AD_BIT;
952 if (init_val & PKEY_DISABLE_WRITE)
953 new_pkru_bits |= PKRU_WD_BIT;
955 /* Shift the bits in to the correct place in PKRU for pkey: */
956 new_pkru_bits <<= pkey_shift;
958 /* Get old PKRU and mask off any old bits in place: */
959 old_pkru = read_pkru();
960 old_pkru &= ~((PKRU_AD_BIT|PKRU_WD_BIT) << pkey_shift);
962 /* Write old part along with new part: */
963 write_pkru(old_pkru | new_pkru_bits);
967 #endif /* ! CONFIG_ARCH_HAS_PKEYS */
970 * Weird legacy quirk: SSE and YMM states store information in the
971 * MXCSR and MXCSR_FLAGS fields of the FP area. That means if the FP
972 * area is marked as unused in the xfeatures header, we need to copy
973 * MXCSR and MXCSR_FLAGS if either SSE or YMM are in use.
975 static inline bool xfeatures_mxcsr_quirk(u64 xfeatures)
977 if (!(xfeatures & (XFEATURE_MASK_SSE|XFEATURE_MASK_YMM)))
980 if (xfeatures & XFEATURE_MASK_FP)
986 static void copy_feature(bool from_xstate, struct membuf *to, void *xstate,
987 void *init_xstate, unsigned int size)
989 membuf_write(to, from_xstate ? xstate : init_xstate, size);
993 * copy_xstate_to_uabi_buf - Copy kernel saved xstate to a UABI buffer
994 * @to: membuf descriptor
995 * @xsave: The kernel xstate buffer to copy from
996 * @copy_mode: The requested copy mode
998 * Converts from kernel XSAVE or XSAVES compacted format to UABI conforming
999 * format, i.e. from the kernel internal hardware dependent storage format
1000 * to the requested @mode. UABI XSTATE is always uncompacted!
1002 * It supports partial copy but @to.pos always starts from zero.
1004 void copy_xstate_to_uabi_buf(struct membuf to, struct xregs_state *xsave,
1005 enum xstate_copy_mode copy_mode)
1007 const unsigned int off_mxcsr = offsetof(struct fxregs_state, mxcsr);
1008 struct xregs_state *xinit = &init_fpstate.xsave;
1009 struct xstate_header header;
1010 unsigned int zerofrom;
1013 header.xfeatures = xsave->header.xfeatures;
1015 /* Mask out the feature bits depending on copy mode */
1016 switch (copy_mode) {
1017 case XSTATE_COPY_FP:
1018 header.xfeatures &= XFEATURE_MASK_FP;
1021 case XSTATE_COPY_FX:
1022 header.xfeatures &= XFEATURE_MASK_FP | XFEATURE_MASK_SSE;
1025 case XSTATE_COPY_XSAVE:
1026 header.xfeatures &= xfeatures_mask_user();
1030 /* Copy FP state up to MXCSR */
1031 copy_feature(header.xfeatures & XFEATURE_MASK_FP, &to, &xsave->i387,
1032 &xinit->i387, off_mxcsr);
1034 /* Copy MXCSR when SSE or YMM are set in the feature mask */
1035 copy_feature(header.xfeatures & (XFEATURE_MASK_SSE | XFEATURE_MASK_YMM),
1036 &to, &xsave->i387.mxcsr, &xinit->i387.mxcsr,
1037 MXCSR_AND_FLAGS_SIZE);
1039 /* Copy the remaining FP state */
1040 copy_feature(header.xfeatures & XFEATURE_MASK_FP,
1041 &to, &xsave->i387.st_space, &xinit->i387.st_space,
1042 sizeof(xsave->i387.st_space));
1044 /* Copy the SSE state - shared with YMM, but independently managed */
1045 copy_feature(header.xfeatures & XFEATURE_MASK_SSE,
1046 &to, &xsave->i387.xmm_space, &xinit->i387.xmm_space,
1047 sizeof(xsave->i387.xmm_space));
1049 if (copy_mode != XSTATE_COPY_XSAVE)
1052 /* Zero the padding area */
1053 membuf_zero(&to, sizeof(xsave->i387.padding));
1055 /* Copy xsave->i387.sw_reserved */
1056 membuf_write(&to, xstate_fx_sw_bytes, sizeof(xsave->i387.sw_reserved));
1058 /* Copy the user space relevant state of @xsave->header */
1059 membuf_write(&to, &header, sizeof(header));
1061 zerofrom = offsetof(struct xregs_state, extended_state_area);
1063 for (i = FIRST_EXTENDED_XFEATURE; i < XFEATURE_MAX; i++) {
1065 * The ptrace buffer is in non-compacted XSAVE format.
1066 * In non-compacted format disabled features still occupy
1067 * state space, but there is no state to copy from in the
1068 * compacted init_fpstate. The gap tracking will zero this
1071 if (!(xfeatures_mask_user() & BIT_ULL(i)))
1075 * If there was a feature or alignment gap, zero the space
1076 * in the destination buffer.
1078 if (zerofrom < xstate_offsets[i])
1079 membuf_zero(&to, xstate_offsets[i] - zerofrom);
1081 copy_feature(header.xfeatures & BIT_ULL(i), &to,
1082 __raw_xsave_addr(xsave, i),
1083 __raw_xsave_addr(xinit, i),
1087 * Keep track of the last copied state in the non-compacted
1088 * target buffer for gap zeroing.
1090 zerofrom = xstate_offsets[i] + xstate_sizes[i];
1095 membuf_zero(&to, to.left);
1098 static inline bool mxcsr_valid(struct xstate_header *hdr, const u32 *mxcsr)
1100 u64 mask = XFEATURE_MASK_FP | XFEATURE_MASK_SSE | XFEATURE_MASK_YMM;
1102 /* Only check if it is in use */
1103 if (hdr->xfeatures & mask) {
1104 /* Reserved bits in MXCSR must be zero. */
1105 if (*mxcsr & ~mxcsr_feature_mask)
1112 * Convert from a ptrace standard-format kernel buffer to kernel XSAVE[S] format
1113 * and copy to the target thread. This is called from xstateregs_set().
1115 int copy_kernel_to_xstate(struct xregs_state *xsave, const void *kbuf)
1117 unsigned int offset, size;
1119 struct xstate_header hdr;
1121 offset = offsetof(struct xregs_state, header);
1124 memcpy(&hdr, kbuf + offset, size);
1126 if (validate_user_xstate_header(&hdr))
1129 if (!mxcsr_valid(&hdr, kbuf + offsetof(struct fxregs_state, mxcsr)))
1132 for (i = 0; i < XFEATURE_MAX; i++) {
1133 u64 mask = ((u64)1 << i);
1135 if (hdr.xfeatures & mask) {
1136 void *dst = __raw_xsave_addr(xsave, i);
1138 offset = xstate_offsets[i];
1139 size = xstate_sizes[i];
1141 memcpy(dst, kbuf + offset, size);
1145 if (xfeatures_mxcsr_quirk(hdr.xfeatures)) {
1146 offset = offsetof(struct fxregs_state, mxcsr);
1147 size = MXCSR_AND_FLAGS_SIZE;
1148 memcpy(&xsave->i387.mxcsr, kbuf + offset, size);
1152 * The state that came in from userspace was user-state only.
1153 * Mask all the user states out of 'xfeatures':
1155 xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR_ALL;
1158 * Add back in the features that came in from userspace:
1160 xsave->header.xfeatures |= hdr.xfeatures;
1166 * Convert from a sigreturn standard-format user-space buffer to kernel
1167 * XSAVE[S] format and copy to the target thread. This is called from the
1168 * sigreturn() and rt_sigreturn() system calls.
1170 int copy_user_to_xstate(struct xregs_state *xsave, const void __user *ubuf)
1172 unsigned int offset, size;
1174 struct xstate_header hdr;
1176 offset = offsetof(struct xregs_state, header);
1179 if (copy_from_user(&hdr, ubuf + offset, size))
1182 if (validate_user_xstate_header(&hdr))
1185 for (i = 0; i < XFEATURE_MAX; i++) {
1186 u64 mask = ((u64)1 << i);
1188 if (hdr.xfeatures & mask) {
1189 void *dst = __raw_xsave_addr(xsave, i);
1191 offset = xstate_offsets[i];
1192 size = xstate_sizes[i];
1194 if (copy_from_user(dst, ubuf + offset, size))
1199 if (xfeatures_mxcsr_quirk(hdr.xfeatures)) {
1200 offset = offsetof(struct fxregs_state, mxcsr);
1201 size = MXCSR_AND_FLAGS_SIZE;
1202 if (copy_from_user(&xsave->i387.mxcsr, ubuf + offset, size))
1207 * The state that came in from userspace was user-state only.
1208 * Mask all the user states out of 'xfeatures':
1210 xsave->header.xfeatures &= XFEATURE_MASK_SUPERVISOR_ALL;
1213 * Add back in the features that came in from userspace:
1215 xsave->header.xfeatures |= hdr.xfeatures;
1221 * Save only supervisor states to the kernel buffer. This blows away all
1222 * old states, and is intended to be used only in __fpu__restore_sig(), where
1223 * user states are restored from the user buffer.
1225 void copy_supervisor_to_kernel(struct xregs_state *xstate)
1227 struct xstate_header *header;
1228 u64 max_bit, min_bit;
1232 if (WARN_ON(!boot_cpu_has(X86_FEATURE_XSAVES)))
1235 if (!xfeatures_mask_supervisor())
1238 max_bit = __fls(xfeatures_mask_supervisor());
1239 min_bit = __ffs(xfeatures_mask_supervisor());
1241 lmask = xfeatures_mask_supervisor();
1242 hmask = xfeatures_mask_supervisor() >> 32;
1243 XSTATE_OP(XSAVES, xstate, lmask, hmask, err);
1245 /* We should never fault when copying to a kernel buffer: */
1246 if (WARN_ON_FPU(err))
1250 * At this point, the buffer has only supervisor states and must be
1251 * converted back to normal kernel format.
1253 header = &xstate->header;
1254 header->xcomp_bv |= xfeatures_mask_all;
1257 * This only moves states up in the buffer. Start with
1258 * the last state and move backwards so that states are
1259 * not overwritten until after they are moved. Note:
1260 * memmove() allows overlapping src/dst buffers.
1262 for (i = max_bit; i >= min_bit; i--) {
1263 u8 *xbuf = (u8 *)xstate;
1265 if (!((header->xfeatures >> i) & 1))
1268 /* Move xfeature 'i' into its normal location */
1269 memmove(xbuf + xstate_comp_offsets[i],
1270 xbuf + xstate_supervisor_only_offsets[i],
1276 * copy_dynamic_supervisor_to_kernel() - Save dynamic supervisor states to
1278 * @xstate: A pointer to an xsave area
1279 * @mask: Represent the dynamic supervisor features saved into the xsave area
1281 * Only the dynamic supervisor states sets in the mask are saved into the xsave
1282 * area (See the comment in XFEATURE_MASK_DYNAMIC for the details of dynamic
1283 * supervisor feature). Besides the dynamic supervisor states, the legacy
1284 * region and XSAVE header are also saved into the xsave area. The supervisor
1285 * features in the XFEATURE_MASK_SUPERVISOR_SUPPORTED and
1286 * XFEATURE_MASK_SUPERVISOR_UNSUPPORTED are not saved.
1288 * The xsave area must be 64-bytes aligned.
1290 void copy_dynamic_supervisor_to_kernel(struct xregs_state *xstate, u64 mask)
1292 u64 dynamic_mask = xfeatures_mask_dynamic() & mask;
1296 if (WARN_ON_FPU(!boot_cpu_has(X86_FEATURE_XSAVES)))
1299 if (WARN_ON_FPU(!dynamic_mask))
1302 lmask = dynamic_mask;
1303 hmask = dynamic_mask >> 32;
1305 XSTATE_OP(XSAVES, xstate, lmask, hmask, err);
1307 /* Should never fault when copying to a kernel buffer */
1312 * copy_kernel_to_dynamic_supervisor() - Restore dynamic supervisor states from
1314 * @xstate: A pointer to an xsave area
1315 * @mask: Represent the dynamic supervisor features restored from the xsave area
1317 * Only the dynamic supervisor states sets in the mask are restored from the
1318 * xsave area (See the comment in XFEATURE_MASK_DYNAMIC for the details of
1319 * dynamic supervisor feature). Besides the dynamic supervisor states, the
1320 * legacy region and XSAVE header are also restored from the xsave area. The
1321 * supervisor features in the XFEATURE_MASK_SUPERVISOR_SUPPORTED and
1322 * XFEATURE_MASK_SUPERVISOR_UNSUPPORTED are not restored.
1324 * The xsave area must be 64-bytes aligned.
1326 void copy_kernel_to_dynamic_supervisor(struct xregs_state *xstate, u64 mask)
1328 u64 dynamic_mask = xfeatures_mask_dynamic() & mask;
1332 if (WARN_ON_FPU(!boot_cpu_has(X86_FEATURE_XSAVES)))
1335 if (WARN_ON_FPU(!dynamic_mask))
1338 lmask = dynamic_mask;
1339 hmask = dynamic_mask >> 32;
1341 XSTATE_OP(XRSTORS, xstate, lmask, hmask, err);
1343 /* Should never fault when copying from a kernel buffer */
1347 #ifdef CONFIG_PROC_PID_ARCH_STATUS
1349 * Report the amount of time elapsed in millisecond since last AVX512
1352 static void avx512_status(struct seq_file *m, struct task_struct *task)
1354 unsigned long timestamp = READ_ONCE(task->thread.fpu.avx512_timestamp);
1359 * Report -1 if no AVX512 usage
1363 delta = (long)(jiffies - timestamp);
1365 * Cap to LONG_MAX if time difference > LONG_MAX
1369 delta = jiffies_to_msecs(delta);
1372 seq_put_decimal_ll(m, "AVX512_elapsed_ms:\t", delta);
1377 * Report architecture specific information
1379 int proc_pid_arch_status(struct seq_file *m, struct pid_namespace *ns,
1380 struct pid *pid, struct task_struct *task)
1383 * Report AVX512 state if the processor and build option supported.
1385 if (cpu_feature_enabled(X86_FEATURE_AVX512F))
1386 avx512_status(m, task);
1390 #endif /* CONFIG_PROC_PID_ARCH_STATUS */