1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2017 ARM Ltd.
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
7 #include <linux/kvm_host.h>
8 #include <linux/random.h>
9 #include <linux/memblock.h>
10 #include <asm/alternative.h>
11 #include <asm/debug-monitors.h>
13 #include <asm/kvm_mmu.h>
14 #include <asm/memory.h>
17 * The LSB of the HYP VA tag
21 * The HYP VA tag value with the region bit
27 * Compute HYP VA by using the same computation as kern_hyp_va().
29 static u64 __early_kern_hyp_va(u64 addr)
32 addr |= tag_val << tag_lsb;
37 * Store a hyp VA <-> PA offset into a EL2-owned variable.
39 static void init_hyp_physvirt_offset(void)
43 /* Compute the offset from the hyp VA and PA of a random symbol. */
44 kern_va = (u64)lm_alias(__hyp_text_start);
45 hyp_va = __early_kern_hyp_va(kern_va);
46 hyp_physvirt_offset = (s64)__pa(kern_va) - (s64)hyp_va;
50 * We want to generate a hyp VA with the following format (with V ==
53 * 63 ... V | V-1 | V-2 .. tag_lsb | tag_lsb - 1 .. 0
54 * ---------------------------------------------------------
55 * | 0000000 | hyp_va_msb | random tag | kern linear VA |
56 * |--------- tag_val -----------|----- va_mask ---|
58 * which does not conflict with the idmap regions.
60 __init void kvm_compute_layout(void)
62 phys_addr_t idmap_addr = __pa_symbol(__hyp_idmap_text_start);
65 /* Where is my RAM region? */
66 hyp_va_msb = idmap_addr & BIT(vabits_actual - 1);
67 hyp_va_msb ^= BIT(vabits_actual - 1);
69 tag_lsb = fls64((u64)phys_to_virt(memblock_start_of_DRAM()) ^
70 (u64)(high_memory - 1));
72 va_mask = GENMASK_ULL(tag_lsb - 1, 0);
75 if (IS_ENABLED(CONFIG_RANDOMIZE_BASE) && tag_lsb != (vabits_actual - 1)) {
76 /* We have some free bits to insert a random tag. */
77 tag_val |= get_random_long() & GENMASK_ULL(vabits_actual - 2, tag_lsb);
81 init_hyp_physvirt_offset();
84 static u32 compute_instruction(int n, u32 rd, u32 rn)
86 u32 insn = AARCH64_BREAK_FAULT;
90 insn = aarch64_insn_gen_logical_immediate(AARCH64_INSN_LOGIC_AND,
91 AARCH64_INSN_VARIANT_64BIT,
96 /* ROR is a variant of EXTR with Rm = Rn */
97 insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT,
103 insn = aarch64_insn_gen_add_sub_imm(rd, rn,
104 tag_val & GENMASK(11, 0),
105 AARCH64_INSN_VARIANT_64BIT,
106 AARCH64_INSN_ADSB_ADD);
110 insn = aarch64_insn_gen_add_sub_imm(rd, rn,
111 tag_val & GENMASK(23, 12),
112 AARCH64_INSN_VARIANT_64BIT,
113 AARCH64_INSN_ADSB_ADD);
117 /* ROR is a variant of EXTR with Rm = Rn */
118 insn = aarch64_insn_gen_extr(AARCH64_INSN_VARIANT_64BIT,
119 rn, rn, rd, 64 - tag_lsb);
126 void __init kvm_update_va_mask(struct alt_instr *alt,
127 __le32 *origptr, __le32 *updptr, int nr_inst)
131 BUG_ON(nr_inst != 5);
133 for (i = 0; i < nr_inst; i++) {
134 u32 rd, rn, insn, oinsn;
137 * VHE doesn't need any address translation, let's NOP
140 * Alternatively, if the tag is zero (because the layout
141 * dictates it and we don't have any spare bits in the
142 * address), NOP everything after masking the kernel VA.
144 if (has_vhe() || (!tag_val && i > 0)) {
145 updptr[i] = cpu_to_le32(aarch64_insn_gen_nop());
149 oinsn = le32_to_cpu(origptr[i]);
150 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn);
151 rn = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RN, oinsn);
153 insn = compute_instruction(i, rd, rn);
154 BUG_ON(insn == AARCH64_BREAK_FAULT);
156 updptr[i] = cpu_to_le32(insn);
160 void kvm_patch_vector_branch(struct alt_instr *alt,
161 __le32 *origptr, __le32 *updptr, int nr_inst)
166 BUG_ON(nr_inst != 4);
168 if (!cpus_have_const_cap(ARM64_SPECTRE_V3A) || WARN_ON_ONCE(has_vhe()))
172 * Compute HYP VA by using the same computation as kern_hyp_va()
174 addr = __early_kern_hyp_va((u64)kvm_ksym_ref(__kvm_hyp_vector));
176 /* Use PC[10:7] to branch to the same vector in KVM */
177 addr |= ((u64)origptr & GENMASK_ULL(10, 7));
180 * Branch over the preamble in order to avoid the initial store on
181 * the stack (which we already perform in the hardening vectors).
183 addr += KVM_VECTOR_PREAMBLE;
185 /* movz x0, #(addr & 0xffff) */
186 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
189 AARCH64_INSN_VARIANT_64BIT,
190 AARCH64_INSN_MOVEWIDE_ZERO);
191 *updptr++ = cpu_to_le32(insn);
193 /* movk x0, #((addr >> 16) & 0xffff), lsl #16 */
194 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
197 AARCH64_INSN_VARIANT_64BIT,
198 AARCH64_INSN_MOVEWIDE_KEEP);
199 *updptr++ = cpu_to_le32(insn);
201 /* movk x0, #((addr >> 32) & 0xffff), lsl #32 */
202 insn = aarch64_insn_gen_movewide(AARCH64_INSN_REG_0,
205 AARCH64_INSN_VARIANT_64BIT,
206 AARCH64_INSN_MOVEWIDE_KEEP);
207 *updptr++ = cpu_to_le32(insn);
210 insn = aarch64_insn_gen_branch_reg(AARCH64_INSN_REG_0,
211 AARCH64_INSN_BRANCH_NOLINK);
212 *updptr++ = cpu_to_le32(insn);
215 static void generate_mov_q(u64 val, __le32 *origptr, __le32 *updptr, int nr_inst)
219 BUG_ON(nr_inst != 4);
221 /* Compute target register */
222 oinsn = le32_to_cpu(*origptr);
223 rd = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RD, oinsn);
225 /* movz rd, #(val & 0xffff) */
226 insn = aarch64_insn_gen_movewide(rd,
229 AARCH64_INSN_VARIANT_64BIT,
230 AARCH64_INSN_MOVEWIDE_ZERO);
231 *updptr++ = cpu_to_le32(insn);
233 /* movk rd, #((val >> 16) & 0xffff), lsl #16 */
234 insn = aarch64_insn_gen_movewide(rd,
237 AARCH64_INSN_VARIANT_64BIT,
238 AARCH64_INSN_MOVEWIDE_KEEP);
239 *updptr++ = cpu_to_le32(insn);
241 /* movk rd, #((val >> 32) & 0xffff), lsl #32 */
242 insn = aarch64_insn_gen_movewide(rd,
245 AARCH64_INSN_VARIANT_64BIT,
246 AARCH64_INSN_MOVEWIDE_KEEP);
247 *updptr++ = cpu_to_le32(insn);
249 /* movk rd, #((val >> 48) & 0xffff), lsl #48 */
250 insn = aarch64_insn_gen_movewide(rd,
253 AARCH64_INSN_VARIANT_64BIT,
254 AARCH64_INSN_MOVEWIDE_KEEP);
255 *updptr++ = cpu_to_le32(insn);
258 void kvm_update_kimg_phys_offset(struct alt_instr *alt,
259 __le32 *origptr, __le32 *updptr, int nr_inst)
261 generate_mov_q(kimage_voffset + PHYS_OFFSET, origptr, updptr, nr_inst);
264 void kvm_get_kimage_voffset(struct alt_instr *alt,
265 __le32 *origptr, __le32 *updptr, int nr_inst)
267 generate_mov_q(kimage_voffset, origptr, updptr, nr_inst);