1 /* SPDX-License-Identifier: GPL-2.0-only */
5 * Copyright (C) 2012 ARM Ltd.
6 * Author: Marc Zyngier <marc.zyngier@arm.com>
9 #include <linux/init.h>
10 #include <linux/linkage.h>
12 #include <asm/assembler.h>
13 #include <asm/el2_setup.h>
14 #include <asm/kvm_arm.h>
15 #include <asm/kvm_asm.h>
16 #include <asm/ptrace.h>
19 // Warning, hardcoded register allocation
20 // This will clobber x1 and x2, and expect x1 to contain
21 // the id register value as read from the HW
22 .macro __check_override idreg, fld, width, pass, fail
23 ubfx x1, x1, #\fld, #\width
26 adr_l x1, \idreg\()_override
27 ldr x2, [x1, FTR_OVR_VAL_OFFSET]
28 ldr x1, [x1, FTR_OVR_MASK_OFFSET]
29 ubfx x2, x2, #\fld, #\width
30 ubfx x1, x1, #\fld, #\width
38 .macro check_override idreg, fld, pass, fail
40 __check_override \idreg \fld 4 \pass \fail
44 .pushsection .hyp.text, "ax"
48 SYM_CODE_START(__hyp_stub_vectors)
49 ventry el2_sync_invalid // Synchronous EL2t
50 ventry el2_irq_invalid // IRQ EL2t
51 ventry el2_fiq_invalid // FIQ EL2t
52 ventry el2_error_invalid // Error EL2t
54 ventry elx_sync // Synchronous EL2h
55 ventry el2_irq_invalid // IRQ EL2h
56 ventry el2_fiq_invalid // FIQ EL2h
57 ventry el2_error_invalid // Error EL2h
59 ventry elx_sync // Synchronous 64-bit EL1
60 ventry el1_irq_invalid // IRQ 64-bit EL1
61 ventry el1_fiq_invalid // FIQ 64-bit EL1
62 ventry el1_error_invalid // Error 64-bit EL1
64 ventry el1_sync_invalid // Synchronous 32-bit EL1
65 ventry el1_irq_invalid // IRQ 32-bit EL1
66 ventry el1_fiq_invalid // FIQ 32-bit EL1
67 ventry el1_error_invalid // Error 32-bit EL1
68 SYM_CODE_END(__hyp_stub_vectors)
72 SYM_CODE_START_LOCAL(elx_sync)
73 cmp x0, #HVC_SET_VECTORS
78 1: cmp x0, #HVC_FINALISE_EL2
81 2: cmp x0, #HVC_SOFT_RESTART
89 3: cmp x0, #HVC_RESET_VECTORS
90 beq 9f // Nothing to reset!
92 /* Someone called kvm_call_hyp() against the hyp-stub... */
93 mov_q x0, HVC_STUB_ERR
98 SYM_CODE_END(elx_sync)
100 SYM_CODE_START_LOCAL(__finalise_el2)
101 check_override id_aa64pfr1 ID_AA64PFR1_SME_SHIFT .Linit_sme .Lskip_sme
103 .Linit_sme: /* SME register access and priority mapping */
104 mrs x0, cptr_el2 // Disable SME traps
105 bic x0, x0, #CPTR_EL2_TSM
110 orr x1, x1, #SCTLR_ELx_ENTP2 // Disable TPIDR2 traps
114 mov x1, #0 // SMCR controls
116 mrs_s x2, SYS_ID_AA64SMFR0_EL1
117 ubfx x2, x2, #ID_AA64SMFR0_FA64_SHIFT, #1 // Full FP in SM?
118 cbz x2, .Lskip_sme_fa64
120 orr x1, x1, SMCR_ELx_FA64_MASK
123 orr x1, x1, #SMCR_ELx_LEN_MASK // Enable full SME vector
124 msr_s SYS_SMCR_EL2, x1 // length for EL1.
126 mrs_s x1, SYS_SMIDR_EL1 // Priority mapping supported?
127 ubfx x1, x1, #SMIDR_EL1_SMPS_SHIFT, #1
130 msr_s SYS_SMPRIMAP_EL2, xzr // Make all priorities equal
132 mrs x1, id_aa64mmfr1_el1 // HCRX_EL2 present?
133 ubfx x1, x1, #ID_AA64MMFR1_HCX_SHIFT, #4
136 mrs_s x1, SYS_HCRX_EL2
137 orr x1, x1, #HCRX_EL2_SMPME_MASK // Enable priority mapping
138 msr_s SYS_HCRX_EL2, x1
142 // nVHE? No way! Give me the real thing!
143 // Sanity check: MMU *must* be off
147 // Needs to be VHE capable, obviously
148 check_override id_aa64mmfr1 ID_AA64MMFR1_VHE_SHIFT 2f 1f
150 1: mov_q x0, HVC_STUB_ERR
153 // Engage the VHE magic!
154 mov_q x0, HCR_HOST_VHE_FLAGS
158 // Use the EL1 allocated stack, per-cpu offset
164 // FP configuration, vectors
165 mrs_s x0, SYS_CPACR_EL12
167 mrs_s x0, SYS_VBAR_EL12
170 // Use EL2 translations for SPE & TRBE and disable access from EL1
172 bic x0, x0, #(MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT)
173 bic x0, x0, #(MDCR_EL2_E2TB_MASK << MDCR_EL2_E2TB_SHIFT)
176 // Transfer the MM state from EL1 to EL2
177 mrs_s x0, SYS_TCR_EL12
179 mrs_s x0, SYS_TTBR0_EL12
181 mrs_s x0, SYS_TTBR1_EL12
183 mrs_s x0, SYS_MAIR_EL12
187 // Hack the exception return to stay at EL2
189 and x0, x0, #~PSR_MODE_MASK
190 mov x1, #PSR_MODE_EL2h
195 SYM_CODE_END(__finalise_el2)
197 // At the point where we reach enter_vhe(), we run with
198 // the MMU off (which is enforced by __finalise_el2()).
199 // We thus need to be in the idmap, or everything will
200 // explode when enabling the MMU.
202 .pushsection .idmap.text, "ax"
204 SYM_CODE_START_LOCAL(enter_vhe)
205 // Invalidate TLBs before enabling the MMU
210 // Enable the EL2 S1 MMU, as set up from EL1
211 mrs_s x0, SYS_SCTLR_EL12
214 // Disable the EL1 S1 MMU for a good measure
215 mov_q x0, INIT_SCTLR_EL1_MMU_OFF
216 msr_s SYS_SCTLR_EL12, x0
221 SYM_CODE_END(enter_vhe)
225 .macro invalid_vector label
226 SYM_CODE_START_LOCAL(\label)
231 invalid_vector el2_sync_invalid
232 invalid_vector el2_irq_invalid
233 invalid_vector el2_fiq_invalid
234 invalid_vector el2_error_invalid
235 invalid_vector el1_sync_invalid
236 invalid_vector el1_irq_invalid
237 invalid_vector el1_fiq_invalid
238 invalid_vector el1_error_invalid
243 * __hyp_set_vectors: Call this after boot to set the initial hypervisor
244 * vectors as part of hypervisor installation. On an SMP system, this should
245 * be called on each CPU.
247 * x0 must be the physical address of the new vector table, and must be
250 * Before calling this, you must check that the stub hypervisor is installed
251 * everywhere, by waiting for any secondary CPUs to be brought up and then
252 * checking that is_hyp_mode_available() is true.
254 * If not, there is a pre-existing hypervisor, some CPUs failed to boot, or
255 * something else went wrong... in such cases, trying to install a new
256 * hypervisor is unlikely to work as desired.
258 * When you call into your shiny new hypervisor, sp_el2 will contain junk,
259 * so you will need to set that to something sensible at the new hypervisor's
260 * initialisation entry point.
263 SYM_FUNC_START(__hyp_set_vectors)
265 mov x0, #HVC_SET_VECTORS
268 SYM_FUNC_END(__hyp_set_vectors)
270 SYM_FUNC_START(__hyp_reset_vectors)
271 mov x0, #HVC_RESET_VECTORS
274 SYM_FUNC_END(__hyp_reset_vectors)
277 * Entry point to finalise EL2 and switch to VHE if deemed capable
279 * w0: boot mode, as returned by init_kernel_el()
281 SYM_FUNC_START(finalise_el2)
282 // Need to have booted at EL2
283 cmp w0, #BOOT_CPU_MODE_EL2
286 // and still be at EL1
288 cmp x0, #CurrentEL_EL1
291 mov x0, #HVC_FINALISE_EL2
295 SYM_FUNC_END(finalise_el2)