1 // SPDX-License-Identifier: GPL-2.0-only
3 * x86 FPU boot time init code:
5 #include <asm/fpu/api.h>
6 #include <asm/tlbflush.h>
9 #include <linux/sched.h>
10 #include <linux/sched/task.h>
11 #include <linux/init.h>
18 * Initialize the registers found in all CPUs, CR0 and CR4:
20 static void fpu__init_cpu_generic(void)
23 unsigned long cr4_mask = 0;
25 if (boot_cpu_has(X86_FEATURE_FXSR))
26 cr4_mask |= X86_CR4_OSFXSR;
27 if (boot_cpu_has(X86_FEATURE_XMM))
28 cr4_mask |= X86_CR4_OSXMMEXCPT;
30 cr4_set_bits(cr4_mask);
33 cr0 &= ~(X86_CR0_TS|X86_CR0_EM); /* clear TS and EM */
34 if (!boot_cpu_has(X86_FEATURE_FPU))
38 /* Flush out any pending x87 state: */
39 #ifdef CONFIG_MATH_EMULATION
40 if (!boot_cpu_has(X86_FEATURE_FPU))
41 fpstate_init_soft(¤t->thread.fpu.fpstate->regs.soft);
44 asm volatile ("fninit");
48 * Enable all supported FPU features. Called when a CPU is brought online:
50 void fpu__init_cpu(void)
52 fpu__init_cpu_generic();
53 fpu__init_cpu_xstate();
56 static bool __init fpu__probe_without_cpuid(void)
64 cr0 &= ~(X86_CR0_TS | X86_CR0_EM);
67 asm volatile("fninit ; fnstsw %0 ; fnstcw %1" : "+m" (fsw), "+m" (fcw));
69 pr_info("x86/fpu: Probing for FPU: FSW=0x%04hx FCW=0x%04hx\n", fsw, fcw);
71 return fsw == 0 && (fcw & 0x103f) == 0x003f;
74 static void __init fpu__init_system_early_generic(void)
76 if (!boot_cpu_has(X86_FEATURE_CPUID) &&
77 !test_bit(X86_FEATURE_FPU, (unsigned long *)cpu_caps_cleared)) {
78 if (fpu__probe_without_cpuid())
79 setup_force_cpu_cap(X86_FEATURE_FPU);
81 setup_clear_cpu_cap(X86_FEATURE_FPU);
84 #ifndef CONFIG_MATH_EMULATION
85 if (!test_cpu_cap(&boot_cpu_data, X86_FEATURE_FPU)) {
86 pr_emerg("x86/fpu: Giving up, no FPU found and no math emulation present\n");
94 * Boot time FPU feature detection code:
96 unsigned int mxcsr_feature_mask __ro_after_init = 0xffffffffu;
97 EXPORT_SYMBOL_GPL(mxcsr_feature_mask);
99 static void __init fpu__init_system_mxcsr(void)
101 unsigned int mask = 0;
103 if (boot_cpu_has(X86_FEATURE_FXSR)) {
104 /* Static because GCC does not get 16-byte stack alignment right: */
105 static struct fxregs_state fxregs __initdata;
107 asm volatile("fxsave %0" : "+m" (fxregs));
109 mask = fxregs.mxcsr_mask;
112 * If zero then use the default features mask,
113 * which has all features set, except the
114 * denormals-are-zero feature bit:
119 mxcsr_feature_mask &= mask;
123 * Once per bootup FPU initialization sequences that will run on most x86 CPUs:
125 static void __init fpu__init_system_generic(void)
128 * Set up the legacy init FPU context. Will be updated when the
129 * CPU supports XSAVE[S].
131 fpstate_init_user(&init_fpstate);
133 fpu__init_system_mxcsr();
137 * Enforce that 'MEMBER' is the last field of 'TYPE'.
139 * Align the computed size with alignment of the TYPE,
140 * because that's how C aligns structs.
142 #define CHECK_MEMBER_AT_END_OF(TYPE, MEMBER) \
143 BUILD_BUG_ON(sizeof(TYPE) != \
144 ALIGN(offsetofend(TYPE, MEMBER), _Alignof(TYPE)))
147 * We append the 'struct fpu' to the task_struct:
149 static void __init fpu__init_task_struct_size(void)
151 int task_size = sizeof(struct task_struct);
154 * Subtract off the static size of the register state.
155 * It potentially has a bunch of padding.
157 task_size -= sizeof(current->thread.fpu.__fpstate.regs);
160 * Add back the dynamically-calculated register state
163 task_size += fpu_kernel_cfg.default_size;
166 * We dynamically size 'struct fpu', so we require that
167 * it be at the end of 'thread_struct' and that
168 * 'thread_struct' be at the end of 'task_struct'. If
169 * you hit a compile error here, check the structure to
170 * see if something got added to the end.
172 CHECK_MEMBER_AT_END_OF(struct fpu, __fpstate);
173 CHECK_MEMBER_AT_END_OF(struct thread_struct, fpu);
174 CHECK_MEMBER_AT_END_OF(struct task_struct, thread);
176 arch_task_struct_size = task_size;
180 * Set up the user and kernel xstate sizes based on the legacy FPU context size.
182 * We set this up first, and later it will be overwritten by
183 * fpu__init_system_xstate() if the CPU knows about xstates.
185 static void __init fpu__init_system_xstate_size_legacy(void)
190 * Note that the size configuration might be overwritten later
191 * during fpu__init_system_xstate().
193 if (!cpu_feature_enabled(X86_FEATURE_FPU)) {
194 size = sizeof(struct swregs_state);
195 } else if (cpu_feature_enabled(X86_FEATURE_FXSR)) {
196 size = sizeof(struct fxregs_state);
197 fpu_user_cfg.legacy_features = XFEATURE_MASK_FPSSE;
199 size = sizeof(struct fregs_state);
200 fpu_user_cfg.legacy_features = XFEATURE_MASK_FP;
203 fpu_kernel_cfg.max_size = size;
204 fpu_kernel_cfg.default_size = size;
205 fpu_user_cfg.max_size = size;
206 fpu_user_cfg.default_size = size;
207 fpstate_reset(¤t->thread.fpu);
211 * Called on the boot CPU once per system bootup, to set up the initial
212 * FPU state that is later cloned into all processes:
214 void __init fpu__init_system(void)
216 fpstate_reset(¤t->thread.fpu);
217 fpu__init_system_early_generic();
220 * The FPU has to be operational for some of the
221 * later FPU init activities:
225 fpu__init_system_generic();
226 fpu__init_system_xstate_size_legacy();
227 fpu__init_system_xstate(fpu_kernel_cfg.max_size);
228 fpu__init_task_struct_size();