1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2013 Huawei Ltd.
4 * Author: Jiang Liu <liuj97@gmail.com>
6 * Copyright (C) 2014-2016 Zi Shen Lim <zlim.lnx@gmail.com>
8 #include <linux/bitops.h>
10 #include <linux/printk.h>
11 #include <linux/sizes.h>
12 #include <linux/types.h>
14 #include <asm/debug-monitors.h>
15 #include <asm/errno.h>
17 #include <asm/kprobes.h>
19 #define AARCH64_INSN_SF_BIT BIT(31)
20 #define AARCH64_INSN_N_BIT BIT(22)
21 #define AARCH64_INSN_LSL_12 BIT(22)
23 static int __kprobes aarch64_get_imm_shift_mask(enum aarch64_insn_imm_type type,
24 u32 *maskp, int *shiftp)
30 case AARCH64_INSN_IMM_26:
34 case AARCH64_INSN_IMM_19:
38 case AARCH64_INSN_IMM_16:
42 case AARCH64_INSN_IMM_14:
46 case AARCH64_INSN_IMM_12:
50 case AARCH64_INSN_IMM_9:
54 case AARCH64_INSN_IMM_7:
58 case AARCH64_INSN_IMM_6:
59 case AARCH64_INSN_IMM_S:
63 case AARCH64_INSN_IMM_R:
67 case AARCH64_INSN_IMM_N:
81 #define ADR_IMM_HILOSPLIT 2
82 #define ADR_IMM_SIZE SZ_2M
83 #define ADR_IMM_LOMASK ((1 << ADR_IMM_HILOSPLIT) - 1)
84 #define ADR_IMM_HIMASK ((ADR_IMM_SIZE >> ADR_IMM_HILOSPLIT) - 1)
85 #define ADR_IMM_LOSHIFT 29
86 #define ADR_IMM_HISHIFT 5
88 u64 aarch64_insn_decode_immediate(enum aarch64_insn_imm_type type, u32 insn)
90 u32 immlo, immhi, mask;
94 case AARCH64_INSN_IMM_ADR:
96 immlo = (insn >> ADR_IMM_LOSHIFT) & ADR_IMM_LOMASK;
97 immhi = (insn >> ADR_IMM_HISHIFT) & ADR_IMM_HIMASK;
98 insn = (immhi << ADR_IMM_HILOSPLIT) | immlo;
99 mask = ADR_IMM_SIZE - 1;
102 if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) {
103 pr_err("%s: unknown immediate encoding %d\n", __func__,
109 return (insn >> shift) & mask;
112 u32 __kprobes aarch64_insn_encode_immediate(enum aarch64_insn_imm_type type,
115 u32 immlo, immhi, mask;
118 if (insn == AARCH64_BREAK_FAULT)
119 return AARCH64_BREAK_FAULT;
122 case AARCH64_INSN_IMM_ADR:
124 immlo = (imm & ADR_IMM_LOMASK) << ADR_IMM_LOSHIFT;
125 imm >>= ADR_IMM_HILOSPLIT;
126 immhi = (imm & ADR_IMM_HIMASK) << ADR_IMM_HISHIFT;
128 mask = ((ADR_IMM_LOMASK << ADR_IMM_LOSHIFT) |
129 (ADR_IMM_HIMASK << ADR_IMM_HISHIFT));
132 if (aarch64_get_imm_shift_mask(type, &mask, &shift) < 0) {
133 pr_err("%s: unknown immediate encoding %d\n", __func__,
135 return AARCH64_BREAK_FAULT;
139 /* Update the immediate field. */
140 insn &= ~(mask << shift);
141 insn |= (imm & mask) << shift;
146 u32 aarch64_insn_decode_register(enum aarch64_insn_register_type type,
152 case AARCH64_INSN_REGTYPE_RT:
153 case AARCH64_INSN_REGTYPE_RD:
156 case AARCH64_INSN_REGTYPE_RN:
159 case AARCH64_INSN_REGTYPE_RT2:
160 case AARCH64_INSN_REGTYPE_RA:
163 case AARCH64_INSN_REGTYPE_RM:
167 pr_err("%s: unknown register type encoding %d\n", __func__,
172 return (insn >> shift) & GENMASK(4, 0);
175 static u32 aarch64_insn_encode_register(enum aarch64_insn_register_type type,
177 enum aarch64_insn_register reg)
181 if (insn == AARCH64_BREAK_FAULT)
182 return AARCH64_BREAK_FAULT;
184 if (reg < AARCH64_INSN_REG_0 || reg > AARCH64_INSN_REG_SP) {
185 pr_err("%s: unknown register encoding %d\n", __func__, reg);
186 return AARCH64_BREAK_FAULT;
190 case AARCH64_INSN_REGTYPE_RT:
191 case AARCH64_INSN_REGTYPE_RD:
194 case AARCH64_INSN_REGTYPE_RN:
197 case AARCH64_INSN_REGTYPE_RT2:
198 case AARCH64_INSN_REGTYPE_RA:
201 case AARCH64_INSN_REGTYPE_RM:
202 case AARCH64_INSN_REGTYPE_RS:
206 pr_err("%s: unknown register type encoding %d\n", __func__,
208 return AARCH64_BREAK_FAULT;
211 insn &= ~(GENMASK(4, 0) << shift);
212 insn |= reg << shift;
217 static const u32 aarch64_insn_ldst_size[] = {
218 [AARCH64_INSN_SIZE_8] = 0,
219 [AARCH64_INSN_SIZE_16] = 1,
220 [AARCH64_INSN_SIZE_32] = 2,
221 [AARCH64_INSN_SIZE_64] = 3,
224 static u32 aarch64_insn_encode_ldst_size(enum aarch64_insn_size_type type,
229 if (type < AARCH64_INSN_SIZE_8 || type > AARCH64_INSN_SIZE_64) {
230 pr_err("%s: unknown size encoding %d\n", __func__, type);
231 return AARCH64_BREAK_FAULT;
234 size = aarch64_insn_ldst_size[type];
235 insn &= ~GENMASK(31, 30);
241 static inline long label_imm_common(unsigned long pc, unsigned long addr,
246 if ((pc & 0x3) || (addr & 0x3)) {
247 pr_err("%s: A64 instructions must be word aligned\n", __func__);
251 offset = ((long)addr - (long)pc);
253 if (offset < -range || offset >= range) {
254 pr_err("%s: offset out of range\n", __func__);
261 u32 __kprobes aarch64_insn_gen_branch_imm(unsigned long pc, unsigned long addr,
262 enum aarch64_insn_branch_type type)
268 * B/BL support [-128M, 128M) offset
269 * ARM64 virtual address arrangement guarantees all kernel and module
270 * texts are within +/-128M.
272 offset = label_imm_common(pc, addr, SZ_128M);
273 if (offset >= SZ_128M)
274 return AARCH64_BREAK_FAULT;
277 case AARCH64_INSN_BRANCH_LINK:
278 insn = aarch64_insn_get_bl_value();
280 case AARCH64_INSN_BRANCH_NOLINK:
281 insn = aarch64_insn_get_b_value();
284 pr_err("%s: unknown branch encoding %d\n", __func__, type);
285 return AARCH64_BREAK_FAULT;
288 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn,
292 u32 aarch64_insn_gen_comp_branch_imm(unsigned long pc, unsigned long addr,
293 enum aarch64_insn_register reg,
294 enum aarch64_insn_variant variant,
295 enum aarch64_insn_branch_type type)
300 offset = label_imm_common(pc, addr, SZ_1M);
302 return AARCH64_BREAK_FAULT;
305 case AARCH64_INSN_BRANCH_COMP_ZERO:
306 insn = aarch64_insn_get_cbz_value();
308 case AARCH64_INSN_BRANCH_COMP_NONZERO:
309 insn = aarch64_insn_get_cbnz_value();
312 pr_err("%s: unknown branch encoding %d\n", __func__, type);
313 return AARCH64_BREAK_FAULT;
317 case AARCH64_INSN_VARIANT_32BIT:
319 case AARCH64_INSN_VARIANT_64BIT:
320 insn |= AARCH64_INSN_SF_BIT;
323 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
324 return AARCH64_BREAK_FAULT;
327 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg);
329 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
333 u32 aarch64_insn_gen_cond_branch_imm(unsigned long pc, unsigned long addr,
334 enum aarch64_insn_condition cond)
339 offset = label_imm_common(pc, addr, SZ_1M);
341 insn = aarch64_insn_get_bcond_value();
343 if (cond < AARCH64_INSN_COND_EQ || cond > AARCH64_INSN_COND_AL) {
344 pr_err("%s: unknown condition encoding %d\n", __func__, cond);
345 return AARCH64_BREAK_FAULT;
349 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
353 u32 aarch64_insn_gen_branch_reg(enum aarch64_insn_register reg,
354 enum aarch64_insn_branch_type type)
359 case AARCH64_INSN_BRANCH_NOLINK:
360 insn = aarch64_insn_get_br_value();
362 case AARCH64_INSN_BRANCH_LINK:
363 insn = aarch64_insn_get_blr_value();
365 case AARCH64_INSN_BRANCH_RETURN:
366 insn = aarch64_insn_get_ret_value();
369 pr_err("%s: unknown branch encoding %d\n", __func__, type);
370 return AARCH64_BREAK_FAULT;
373 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, reg);
376 u32 aarch64_insn_gen_load_store_reg(enum aarch64_insn_register reg,
377 enum aarch64_insn_register base,
378 enum aarch64_insn_register offset,
379 enum aarch64_insn_size_type size,
380 enum aarch64_insn_ldst_type type)
385 case AARCH64_INSN_LDST_LOAD_REG_OFFSET:
386 insn = aarch64_insn_get_ldr_reg_value();
388 case AARCH64_INSN_LDST_STORE_REG_OFFSET:
389 insn = aarch64_insn_get_str_reg_value();
392 pr_err("%s: unknown load/store encoding %d\n", __func__, type);
393 return AARCH64_BREAK_FAULT;
396 insn = aarch64_insn_encode_ldst_size(size, insn);
398 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg);
400 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
403 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn,
407 u32 aarch64_insn_gen_load_store_imm(enum aarch64_insn_register reg,
408 enum aarch64_insn_register base,
410 enum aarch64_insn_size_type size,
411 enum aarch64_insn_ldst_type type)
416 if (size < AARCH64_INSN_SIZE_8 || size > AARCH64_INSN_SIZE_64) {
417 pr_err("%s: unknown size encoding %d\n", __func__, type);
418 return AARCH64_BREAK_FAULT;
421 shift = aarch64_insn_ldst_size[size];
422 if (imm & ~(BIT(12 + shift) - BIT(shift))) {
423 pr_err("%s: invalid imm: %d\n", __func__, imm);
424 return AARCH64_BREAK_FAULT;
430 case AARCH64_INSN_LDST_LOAD_IMM_OFFSET:
431 insn = aarch64_insn_get_ldr_imm_value();
433 case AARCH64_INSN_LDST_STORE_IMM_OFFSET:
434 insn = aarch64_insn_get_str_imm_value();
437 pr_err("%s: unknown load/store encoding %d\n", __func__, type);
438 return AARCH64_BREAK_FAULT;
441 insn = aarch64_insn_encode_ldst_size(size, insn);
443 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg);
445 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
448 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, imm);
451 u32 aarch64_insn_gen_load_literal(unsigned long pc, unsigned long addr,
452 enum aarch64_insn_register reg,
458 offset = label_imm_common(pc, addr, SZ_1M);
460 return AARCH64_BREAK_FAULT;
462 insn = aarch64_insn_get_ldr_lit_value();
467 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn, reg);
469 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
473 u32 aarch64_insn_gen_load_store_pair(enum aarch64_insn_register reg1,
474 enum aarch64_insn_register reg2,
475 enum aarch64_insn_register base,
477 enum aarch64_insn_variant variant,
478 enum aarch64_insn_ldst_type type)
484 case AARCH64_INSN_LDST_LOAD_PAIR_PRE_INDEX:
485 insn = aarch64_insn_get_ldp_pre_value();
487 case AARCH64_INSN_LDST_STORE_PAIR_PRE_INDEX:
488 insn = aarch64_insn_get_stp_pre_value();
490 case AARCH64_INSN_LDST_LOAD_PAIR_POST_INDEX:
491 insn = aarch64_insn_get_ldp_post_value();
493 case AARCH64_INSN_LDST_STORE_PAIR_POST_INDEX:
494 insn = aarch64_insn_get_stp_post_value();
497 pr_err("%s: unknown load/store encoding %d\n", __func__, type);
498 return AARCH64_BREAK_FAULT;
502 case AARCH64_INSN_VARIANT_32BIT:
503 if ((offset & 0x3) || (offset < -256) || (offset > 252)) {
504 pr_err("%s: offset must be multiples of 4 in the range of [-256, 252] %d\n",
506 return AARCH64_BREAK_FAULT;
510 case AARCH64_INSN_VARIANT_64BIT:
511 if ((offset & 0x7) || (offset < -512) || (offset > 504)) {
512 pr_err("%s: offset must be multiples of 8 in the range of [-512, 504] %d\n",
514 return AARCH64_BREAK_FAULT;
517 insn |= AARCH64_INSN_SF_BIT;
520 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
521 return AARCH64_BREAK_FAULT;
524 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
527 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT2, insn,
530 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
533 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_7, insn,
537 u32 aarch64_insn_gen_load_store_ex(enum aarch64_insn_register reg,
538 enum aarch64_insn_register base,
539 enum aarch64_insn_register state,
540 enum aarch64_insn_size_type size,
541 enum aarch64_insn_ldst_type type)
546 case AARCH64_INSN_LDST_LOAD_EX:
547 case AARCH64_INSN_LDST_LOAD_ACQ_EX:
548 insn = aarch64_insn_get_load_ex_value();
549 if (type == AARCH64_INSN_LDST_LOAD_ACQ_EX)
552 case AARCH64_INSN_LDST_STORE_EX:
553 case AARCH64_INSN_LDST_STORE_REL_EX:
554 insn = aarch64_insn_get_store_ex_value();
555 if (type == AARCH64_INSN_LDST_STORE_REL_EX)
559 pr_err("%s: unknown load/store exclusive encoding %d\n", __func__, type);
560 return AARCH64_BREAK_FAULT;
563 insn = aarch64_insn_encode_ldst_size(size, insn);
565 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
568 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
571 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT2, insn,
572 AARCH64_INSN_REG_ZR);
574 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn,
578 #ifdef CONFIG_ARM64_LSE_ATOMICS
579 static u32 aarch64_insn_encode_ldst_order(enum aarch64_insn_mem_order_type type,
585 case AARCH64_INSN_MEM_ORDER_NONE:
588 case AARCH64_INSN_MEM_ORDER_ACQ:
591 case AARCH64_INSN_MEM_ORDER_REL:
594 case AARCH64_INSN_MEM_ORDER_ACQREL:
598 pr_err("%s: unknown mem order %d\n", __func__, type);
599 return AARCH64_BREAK_FAULT;
602 insn &= ~GENMASK(23, 22);
608 u32 aarch64_insn_gen_atomic_ld_op(enum aarch64_insn_register result,
609 enum aarch64_insn_register address,
610 enum aarch64_insn_register value,
611 enum aarch64_insn_size_type size,
612 enum aarch64_insn_mem_atomic_op op,
613 enum aarch64_insn_mem_order_type order)
618 case AARCH64_INSN_MEM_ATOMIC_ADD:
619 insn = aarch64_insn_get_ldadd_value();
621 case AARCH64_INSN_MEM_ATOMIC_CLR:
622 insn = aarch64_insn_get_ldclr_value();
624 case AARCH64_INSN_MEM_ATOMIC_EOR:
625 insn = aarch64_insn_get_ldeor_value();
627 case AARCH64_INSN_MEM_ATOMIC_SET:
628 insn = aarch64_insn_get_ldset_value();
630 case AARCH64_INSN_MEM_ATOMIC_SWP:
631 insn = aarch64_insn_get_swp_value();
634 pr_err("%s: unimplemented mem atomic op %d\n", __func__, op);
635 return AARCH64_BREAK_FAULT;
639 case AARCH64_INSN_SIZE_32:
640 case AARCH64_INSN_SIZE_64:
643 pr_err("%s: unimplemented size encoding %d\n", __func__, size);
644 return AARCH64_BREAK_FAULT;
647 insn = aarch64_insn_encode_ldst_size(size, insn);
649 insn = aarch64_insn_encode_ldst_order(order, insn);
651 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
654 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
657 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn,
661 static u32 aarch64_insn_encode_cas_order(enum aarch64_insn_mem_order_type type,
667 case AARCH64_INSN_MEM_ORDER_NONE:
670 case AARCH64_INSN_MEM_ORDER_ACQ:
673 case AARCH64_INSN_MEM_ORDER_REL:
676 case AARCH64_INSN_MEM_ORDER_ACQREL:
677 order = BIT(15) | BIT(22);
680 pr_err("%s: unknown mem order %d\n", __func__, type);
681 return AARCH64_BREAK_FAULT;
684 insn &= ~(BIT(15) | BIT(22));
690 u32 aarch64_insn_gen_cas(enum aarch64_insn_register result,
691 enum aarch64_insn_register address,
692 enum aarch64_insn_register value,
693 enum aarch64_insn_size_type size,
694 enum aarch64_insn_mem_order_type order)
699 case AARCH64_INSN_SIZE_32:
700 case AARCH64_INSN_SIZE_64:
703 pr_err("%s: unimplemented size encoding %d\n", __func__, size);
704 return AARCH64_BREAK_FAULT;
707 insn = aarch64_insn_get_cas_value();
709 insn = aarch64_insn_encode_ldst_size(size, insn);
711 insn = aarch64_insn_encode_cas_order(order, insn);
713 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RT, insn,
716 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
719 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RS, insn,
724 u32 aarch64_insn_gen_add_sub_imm(enum aarch64_insn_register dst,
725 enum aarch64_insn_register src,
726 int imm, enum aarch64_insn_variant variant,
727 enum aarch64_insn_adsb_type type)
732 case AARCH64_INSN_ADSB_ADD:
733 insn = aarch64_insn_get_add_imm_value();
735 case AARCH64_INSN_ADSB_SUB:
736 insn = aarch64_insn_get_sub_imm_value();
738 case AARCH64_INSN_ADSB_ADD_SETFLAGS:
739 insn = aarch64_insn_get_adds_imm_value();
741 case AARCH64_INSN_ADSB_SUB_SETFLAGS:
742 insn = aarch64_insn_get_subs_imm_value();
745 pr_err("%s: unknown add/sub encoding %d\n", __func__, type);
746 return AARCH64_BREAK_FAULT;
750 case AARCH64_INSN_VARIANT_32BIT:
752 case AARCH64_INSN_VARIANT_64BIT:
753 insn |= AARCH64_INSN_SF_BIT;
756 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
757 return AARCH64_BREAK_FAULT;
760 /* We can't encode more than a 24bit value (12bit + 12bit shift) */
761 if (imm & ~(BIT(24) - 1))
764 /* If we have something in the top 12 bits... */
765 if (imm & ~(SZ_4K - 1)) {
766 /* ... and in the low 12 bits -> error */
767 if (imm & (SZ_4K - 1))
771 insn |= AARCH64_INSN_LSL_12;
774 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
776 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
778 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_12, insn, imm);
781 pr_err("%s: invalid immediate encoding %d\n", __func__, imm);
782 return AARCH64_BREAK_FAULT;
785 u32 aarch64_insn_gen_bitfield(enum aarch64_insn_register dst,
786 enum aarch64_insn_register src,
788 enum aarch64_insn_variant variant,
789 enum aarch64_insn_bitfield_type type)
795 case AARCH64_INSN_BITFIELD_MOVE:
796 insn = aarch64_insn_get_bfm_value();
798 case AARCH64_INSN_BITFIELD_MOVE_UNSIGNED:
799 insn = aarch64_insn_get_ubfm_value();
801 case AARCH64_INSN_BITFIELD_MOVE_SIGNED:
802 insn = aarch64_insn_get_sbfm_value();
805 pr_err("%s: unknown bitfield encoding %d\n", __func__, type);
806 return AARCH64_BREAK_FAULT;
810 case AARCH64_INSN_VARIANT_32BIT:
811 mask = GENMASK(4, 0);
813 case AARCH64_INSN_VARIANT_64BIT:
814 insn |= AARCH64_INSN_SF_BIT | AARCH64_INSN_N_BIT;
815 mask = GENMASK(5, 0);
818 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
819 return AARCH64_BREAK_FAULT;
823 pr_err("%s: invalid immr encoding %d\n", __func__, immr);
824 return AARCH64_BREAK_FAULT;
827 pr_err("%s: invalid imms encoding %d\n", __func__, imms);
828 return AARCH64_BREAK_FAULT;
831 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
833 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
835 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_R, insn, immr);
837 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, imms);
840 u32 aarch64_insn_gen_movewide(enum aarch64_insn_register dst,
842 enum aarch64_insn_variant variant,
843 enum aarch64_insn_movewide_type type)
848 case AARCH64_INSN_MOVEWIDE_ZERO:
849 insn = aarch64_insn_get_movz_value();
851 case AARCH64_INSN_MOVEWIDE_KEEP:
852 insn = aarch64_insn_get_movk_value();
854 case AARCH64_INSN_MOVEWIDE_INVERSE:
855 insn = aarch64_insn_get_movn_value();
858 pr_err("%s: unknown movewide encoding %d\n", __func__, type);
859 return AARCH64_BREAK_FAULT;
862 if (imm & ~(SZ_64K - 1)) {
863 pr_err("%s: invalid immediate encoding %d\n", __func__, imm);
864 return AARCH64_BREAK_FAULT;
868 case AARCH64_INSN_VARIANT_32BIT:
869 if (shift != 0 && shift != 16) {
870 pr_err("%s: invalid shift encoding %d\n", __func__,
872 return AARCH64_BREAK_FAULT;
875 case AARCH64_INSN_VARIANT_64BIT:
876 insn |= AARCH64_INSN_SF_BIT;
877 if (shift != 0 && shift != 16 && shift != 32 && shift != 48) {
878 pr_err("%s: invalid shift encoding %d\n", __func__,
880 return AARCH64_BREAK_FAULT;
884 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
885 return AARCH64_BREAK_FAULT;
888 insn |= (shift >> 4) << 21;
890 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
892 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_16, insn, imm);
895 u32 aarch64_insn_gen_add_sub_shifted_reg(enum aarch64_insn_register dst,
896 enum aarch64_insn_register src,
897 enum aarch64_insn_register reg,
899 enum aarch64_insn_variant variant,
900 enum aarch64_insn_adsb_type type)
905 case AARCH64_INSN_ADSB_ADD:
906 insn = aarch64_insn_get_add_value();
908 case AARCH64_INSN_ADSB_SUB:
909 insn = aarch64_insn_get_sub_value();
911 case AARCH64_INSN_ADSB_ADD_SETFLAGS:
912 insn = aarch64_insn_get_adds_value();
914 case AARCH64_INSN_ADSB_SUB_SETFLAGS:
915 insn = aarch64_insn_get_subs_value();
918 pr_err("%s: unknown add/sub encoding %d\n", __func__, type);
919 return AARCH64_BREAK_FAULT;
923 case AARCH64_INSN_VARIANT_32BIT:
924 if (shift & ~(SZ_32 - 1)) {
925 pr_err("%s: invalid shift encoding %d\n", __func__,
927 return AARCH64_BREAK_FAULT;
930 case AARCH64_INSN_VARIANT_64BIT:
931 insn |= AARCH64_INSN_SF_BIT;
932 if (shift & ~(SZ_64 - 1)) {
933 pr_err("%s: invalid shift encoding %d\n", __func__,
935 return AARCH64_BREAK_FAULT;
939 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
940 return AARCH64_BREAK_FAULT;
944 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
946 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
948 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
950 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift);
953 u32 aarch64_insn_gen_data1(enum aarch64_insn_register dst,
954 enum aarch64_insn_register src,
955 enum aarch64_insn_variant variant,
956 enum aarch64_insn_data1_type type)
961 case AARCH64_INSN_DATA1_REVERSE_16:
962 insn = aarch64_insn_get_rev16_value();
964 case AARCH64_INSN_DATA1_REVERSE_32:
965 insn = aarch64_insn_get_rev32_value();
967 case AARCH64_INSN_DATA1_REVERSE_64:
968 if (variant != AARCH64_INSN_VARIANT_64BIT) {
969 pr_err("%s: invalid variant for reverse64 %d\n",
971 return AARCH64_BREAK_FAULT;
973 insn = aarch64_insn_get_rev64_value();
976 pr_err("%s: unknown data1 encoding %d\n", __func__, type);
977 return AARCH64_BREAK_FAULT;
981 case AARCH64_INSN_VARIANT_32BIT:
983 case AARCH64_INSN_VARIANT_64BIT:
984 insn |= AARCH64_INSN_SF_BIT;
987 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
988 return AARCH64_BREAK_FAULT;
991 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
993 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
996 u32 aarch64_insn_gen_data2(enum aarch64_insn_register dst,
997 enum aarch64_insn_register src,
998 enum aarch64_insn_register reg,
999 enum aarch64_insn_variant variant,
1000 enum aarch64_insn_data2_type type)
1005 case AARCH64_INSN_DATA2_UDIV:
1006 insn = aarch64_insn_get_udiv_value();
1008 case AARCH64_INSN_DATA2_SDIV:
1009 insn = aarch64_insn_get_sdiv_value();
1011 case AARCH64_INSN_DATA2_LSLV:
1012 insn = aarch64_insn_get_lslv_value();
1014 case AARCH64_INSN_DATA2_LSRV:
1015 insn = aarch64_insn_get_lsrv_value();
1017 case AARCH64_INSN_DATA2_ASRV:
1018 insn = aarch64_insn_get_asrv_value();
1020 case AARCH64_INSN_DATA2_RORV:
1021 insn = aarch64_insn_get_rorv_value();
1024 pr_err("%s: unknown data2 encoding %d\n", __func__, type);
1025 return AARCH64_BREAK_FAULT;
1029 case AARCH64_INSN_VARIANT_32BIT:
1031 case AARCH64_INSN_VARIANT_64BIT:
1032 insn |= AARCH64_INSN_SF_BIT;
1035 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1036 return AARCH64_BREAK_FAULT;
1039 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
1041 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
1043 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
1046 u32 aarch64_insn_gen_data3(enum aarch64_insn_register dst,
1047 enum aarch64_insn_register src,
1048 enum aarch64_insn_register reg1,
1049 enum aarch64_insn_register reg2,
1050 enum aarch64_insn_variant variant,
1051 enum aarch64_insn_data3_type type)
1056 case AARCH64_INSN_DATA3_MADD:
1057 insn = aarch64_insn_get_madd_value();
1059 case AARCH64_INSN_DATA3_MSUB:
1060 insn = aarch64_insn_get_msub_value();
1063 pr_err("%s: unknown data3 encoding %d\n", __func__, type);
1064 return AARCH64_BREAK_FAULT;
1068 case AARCH64_INSN_VARIANT_32BIT:
1070 case AARCH64_INSN_VARIANT_64BIT:
1071 insn |= AARCH64_INSN_SF_BIT;
1074 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1075 return AARCH64_BREAK_FAULT;
1078 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
1080 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RA, insn, src);
1082 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn,
1085 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn,
1089 u32 aarch64_insn_gen_logical_shifted_reg(enum aarch64_insn_register dst,
1090 enum aarch64_insn_register src,
1091 enum aarch64_insn_register reg,
1093 enum aarch64_insn_variant variant,
1094 enum aarch64_insn_logic_type type)
1099 case AARCH64_INSN_LOGIC_AND:
1100 insn = aarch64_insn_get_and_value();
1102 case AARCH64_INSN_LOGIC_BIC:
1103 insn = aarch64_insn_get_bic_value();
1105 case AARCH64_INSN_LOGIC_ORR:
1106 insn = aarch64_insn_get_orr_value();
1108 case AARCH64_INSN_LOGIC_ORN:
1109 insn = aarch64_insn_get_orn_value();
1111 case AARCH64_INSN_LOGIC_EOR:
1112 insn = aarch64_insn_get_eor_value();
1114 case AARCH64_INSN_LOGIC_EON:
1115 insn = aarch64_insn_get_eon_value();
1117 case AARCH64_INSN_LOGIC_AND_SETFLAGS:
1118 insn = aarch64_insn_get_ands_value();
1120 case AARCH64_INSN_LOGIC_BIC_SETFLAGS:
1121 insn = aarch64_insn_get_bics_value();
1124 pr_err("%s: unknown logical encoding %d\n", __func__, type);
1125 return AARCH64_BREAK_FAULT;
1129 case AARCH64_INSN_VARIANT_32BIT:
1130 if (shift & ~(SZ_32 - 1)) {
1131 pr_err("%s: invalid shift encoding %d\n", __func__,
1133 return AARCH64_BREAK_FAULT;
1136 case AARCH64_INSN_VARIANT_64BIT:
1137 insn |= AARCH64_INSN_SF_BIT;
1138 if (shift & ~(SZ_64 - 1)) {
1139 pr_err("%s: invalid shift encoding %d\n", __func__,
1141 return AARCH64_BREAK_FAULT;
1145 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1146 return AARCH64_BREAK_FAULT;
1150 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, dst);
1152 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, src);
1154 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, reg);
1156 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_6, insn, shift);
1160 * MOV (register) is architecturally an alias of ORR (shifted register) where
1161 * MOV <*d>, <*m> is equivalent to ORR <*d>, <*ZR>, <*m>
1163 u32 aarch64_insn_gen_move_reg(enum aarch64_insn_register dst,
1164 enum aarch64_insn_register src,
1165 enum aarch64_insn_variant variant)
1167 return aarch64_insn_gen_logical_shifted_reg(dst, AARCH64_INSN_REG_ZR,
1169 AARCH64_INSN_LOGIC_ORR);
1172 u32 aarch64_insn_gen_adr(unsigned long pc, unsigned long addr,
1173 enum aarch64_insn_register reg,
1174 enum aarch64_insn_adr_type type)
1180 case AARCH64_INSN_ADR_TYPE_ADR:
1181 insn = aarch64_insn_get_adr_value();
1184 case AARCH64_INSN_ADR_TYPE_ADRP:
1185 insn = aarch64_insn_get_adrp_value();
1186 offset = (addr - ALIGN_DOWN(pc, SZ_4K)) >> 12;
1189 pr_err("%s: unknown adr encoding %d\n", __func__, type);
1190 return AARCH64_BREAK_FAULT;
1193 if (offset < -SZ_1M || offset >= SZ_1M)
1194 return AARCH64_BREAK_FAULT;
1196 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, reg);
1198 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_ADR, insn, offset);
1202 * Decode the imm field of a branch, and return the byte offset as a
1203 * signed value (so it can be used when computing a new branch
1206 s32 aarch64_get_branch_offset(u32 insn)
1210 if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn)) {
1211 imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_26, insn);
1212 return (imm << 6) >> 4;
1215 if (aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) ||
1216 aarch64_insn_is_bcond(insn)) {
1217 imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_19, insn);
1218 return (imm << 13) >> 11;
1221 if (aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn)) {
1222 imm = aarch64_insn_decode_immediate(AARCH64_INSN_IMM_14, insn);
1223 return (imm << 18) >> 16;
1226 /* Unhandled instruction */
1231 * Encode the displacement of a branch in the imm field and return the
1232 * updated instruction.
1234 u32 aarch64_set_branch_offset(u32 insn, s32 offset)
1236 if (aarch64_insn_is_b(insn) || aarch64_insn_is_bl(insn))
1237 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_26, insn,
1240 if (aarch64_insn_is_cbz(insn) || aarch64_insn_is_cbnz(insn) ||
1241 aarch64_insn_is_bcond(insn))
1242 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_19, insn,
1245 if (aarch64_insn_is_tbz(insn) || aarch64_insn_is_tbnz(insn))
1246 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_14, insn,
1249 /* Unhandled instruction */
1253 s32 aarch64_insn_adrp_get_offset(u32 insn)
1255 BUG_ON(!aarch64_insn_is_adrp(insn));
1256 return aarch64_insn_decode_immediate(AARCH64_INSN_IMM_ADR, insn) << 12;
1259 u32 aarch64_insn_adrp_set_offset(u32 insn, s32 offset)
1261 BUG_ON(!aarch64_insn_is_adrp(insn));
1262 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_ADR, insn,
1267 * Extract the Op/CR data from a msr/mrs instruction.
1269 u32 aarch64_insn_extract_system_reg(u32 insn)
1271 return (insn & 0x1FFFE0) >> 5;
1274 bool aarch32_insn_is_wide(u32 insn)
1276 return insn >= 0xe800;
1280 * Macros/defines for extracting register numbers from instruction.
1282 u32 aarch32_insn_extract_reg_num(u32 insn, int offset)
1284 return (insn & (0xf << offset)) >> offset;
1287 #define OPC2_MASK 0x7
1288 #define OPC2_OFFSET 5
1289 u32 aarch32_insn_mcr_extract_opc2(u32 insn)
1291 return (insn & (OPC2_MASK << OPC2_OFFSET)) >> OPC2_OFFSET;
1294 #define CRM_MASK 0xf
1295 u32 aarch32_insn_mcr_extract_crm(u32 insn)
1297 return insn & CRM_MASK;
1300 static bool range_of_ones(u64 val)
1302 /* Doesn't handle full ones or full zeroes */
1303 u64 sval = val >> __ffs64(val);
1305 /* One of Sean Eron Anderson's bithack tricks */
1306 return ((sval + 1) & (sval)) == 0;
1309 static u32 aarch64_encode_immediate(u64 imm,
1310 enum aarch64_insn_variant variant,
1313 unsigned int immr, imms, n, ones, ror, esz, tmp;
1317 case AARCH64_INSN_VARIANT_32BIT:
1320 case AARCH64_INSN_VARIANT_64BIT:
1321 insn |= AARCH64_INSN_SF_BIT;
1325 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1326 return AARCH64_BREAK_FAULT;
1329 mask = GENMASK(esz - 1, 0);
1331 /* Can't encode full zeroes, full ones, or value wider than the mask */
1332 if (!imm || imm == mask || imm & ~mask)
1333 return AARCH64_BREAK_FAULT;
1336 * Inverse of Replicate(). Try to spot a repeating pattern
1337 * with a pow2 stride.
1339 for (tmp = esz / 2; tmp >= 2; tmp /= 2) {
1340 u64 emask = BIT(tmp) - 1;
1342 if ((imm & emask) != ((imm >> tmp) & emask))
1349 /* N is only set if we're encoding a 64bit value */
1352 /* Trim imm to the element size */
1355 /* That's how many ones we need to encode */
1356 ones = hweight64(imm);
1359 * imms is set to (ones - 1), prefixed with a string of ones
1360 * and a zero if they fit. Cap it to 6 bits.
1363 imms |= 0xf << ffs(esz);
1366 /* Compute the rotation */
1367 if (range_of_ones(imm)) {
1369 * Pattern: 0..01..10..0
1371 * Compute how many rotate we need to align it right
1376 * Pattern: 0..01..10..01..1
1378 * Fill the unused top bits with ones, and check if
1379 * the result is a valid immediate (all ones with a
1380 * contiguous ranges of zeroes).
1383 if (!range_of_ones(~imm))
1384 return AARCH64_BREAK_FAULT;
1387 * Compute the rotation to get a continuous set of
1388 * ones, with the first bit set at position 0
1394 * immr is the number of bits we need to rotate back to the
1395 * original set of ones. Note that this is relative to the
1398 immr = (esz - ror) % esz;
1400 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_N, insn, n);
1401 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_R, insn, immr);
1402 return aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, imms);
1405 u32 aarch64_insn_gen_logical_immediate(enum aarch64_insn_logic_type type,
1406 enum aarch64_insn_variant variant,
1407 enum aarch64_insn_register Rn,
1408 enum aarch64_insn_register Rd,
1414 case AARCH64_INSN_LOGIC_AND:
1415 insn = aarch64_insn_get_and_imm_value();
1417 case AARCH64_INSN_LOGIC_ORR:
1418 insn = aarch64_insn_get_orr_imm_value();
1420 case AARCH64_INSN_LOGIC_EOR:
1421 insn = aarch64_insn_get_eor_imm_value();
1423 case AARCH64_INSN_LOGIC_AND_SETFLAGS:
1424 insn = aarch64_insn_get_ands_imm_value();
1427 pr_err("%s: unknown logical encoding %d\n", __func__, type);
1428 return AARCH64_BREAK_FAULT;
1431 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, Rd);
1432 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, Rn);
1433 return aarch64_encode_immediate(imm, variant, insn);
1436 u32 aarch64_insn_gen_extr(enum aarch64_insn_variant variant,
1437 enum aarch64_insn_register Rm,
1438 enum aarch64_insn_register Rn,
1439 enum aarch64_insn_register Rd,
1444 insn = aarch64_insn_get_extr_value();
1447 case AARCH64_INSN_VARIANT_32BIT:
1449 return AARCH64_BREAK_FAULT;
1451 case AARCH64_INSN_VARIANT_64BIT:
1453 return AARCH64_BREAK_FAULT;
1454 insn |= AARCH64_INSN_SF_BIT;
1455 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_N, insn, 1);
1458 pr_err("%s: unknown variant encoding %d\n", __func__, variant);
1459 return AARCH64_BREAK_FAULT;
1462 insn = aarch64_insn_encode_immediate(AARCH64_INSN_IMM_S, insn, lsb);
1463 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RD, insn, Rd);
1464 insn = aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RN, insn, Rn);
1465 return aarch64_insn_encode_register(AARCH64_INSN_REGTYPE_RM, insn, Rm);
1468 u32 aarch64_insn_gen_dmb(enum aarch64_insn_mb_type type)
1474 case AARCH64_INSN_MB_SY:
1477 case AARCH64_INSN_MB_ST:
1480 case AARCH64_INSN_MB_LD:
1483 case AARCH64_INSN_MB_ISH:
1486 case AARCH64_INSN_MB_ISHST:
1489 case AARCH64_INSN_MB_ISHLD:
1492 case AARCH64_INSN_MB_NSH:
1495 case AARCH64_INSN_MB_NSHST:
1498 case AARCH64_INSN_MB_NSHLD:
1502 pr_err("%s: unknown dmb type %d\n", __func__, type);
1503 return AARCH64_BREAK_FAULT;
1506 insn = aarch64_insn_get_dmb_value();
1507 insn &= ~GENMASK(11, 8);