1 /* SPDX-License-Identifier: GPL-2.0-only */
3 * FP/SIMD state saving and restoring macros
5 * Copyright (C) 2012 ARM Ltd.
6 * Author: Catalin Marinas <catalin.marinas@arm.com>
9 #include <asm/assembler.h>
11 .macro fpsimd_save state, tmpnr
12 stp q0, q1, [\state, #16 * 0]
13 stp q2, q3, [\state, #16 * 2]
14 stp q4, q5, [\state, #16 * 4]
15 stp q6, q7, [\state, #16 * 6]
16 stp q8, q9, [\state, #16 * 8]
17 stp q10, q11, [\state, #16 * 10]
18 stp q12, q13, [\state, #16 * 12]
19 stp q14, q15, [\state, #16 * 14]
20 stp q16, q17, [\state, #16 * 16]
21 stp q18, q19, [\state, #16 * 18]
22 stp q20, q21, [\state, #16 * 20]
23 stp q22, q23, [\state, #16 * 22]
24 stp q24, q25, [\state, #16 * 24]
25 stp q26, q27, [\state, #16 * 26]
26 stp q28, q29, [\state, #16 * 28]
27 stp q30, q31, [\state, #16 * 30]!
29 str w\tmpnr, [\state, #16 * 2]
31 str w\tmpnr, [\state, #16 * 2 + 4]
34 .macro fpsimd_restore_fpcr state, tmp
36 * Writes to fpcr may be self-synchronising, so avoid restoring
37 * the register if it hasn't changed.
47 .macro fpsimd_restore state, tmpnr
48 ldp q0, q1, [\state, #16 * 0]
49 ldp q2, q3, [\state, #16 * 2]
50 ldp q4, q5, [\state, #16 * 4]
51 ldp q6, q7, [\state, #16 * 6]
52 ldp q8, q9, [\state, #16 * 8]
53 ldp q10, q11, [\state, #16 * 10]
54 ldp q12, q13, [\state, #16 * 12]
55 ldp q14, q15, [\state, #16 * 14]
56 ldp q16, q17, [\state, #16 * 16]
57 ldp q18, q19, [\state, #16 * 18]
58 ldp q20, q21, [\state, #16 * 20]
59 ldp q22, q23, [\state, #16 * 22]
60 ldp q24, q25, [\state, #16 * 24]
61 ldp q26, q27, [\state, #16 * 26]
62 ldp q28, q29, [\state, #16 * 28]
63 ldp q30, q31, [\state, #16 * 30]!
64 ldr w\tmpnr, [\state, #16 * 2]
66 ldr w\tmpnr, [\state, #16 * 2 + 4]
67 fpsimd_restore_fpcr x\tmpnr, \state
70 /* Sanity-check macros to help avoid encoding garbage instructions */
72 .macro _check_general_reg nr
73 .if (\nr) < 0 || (\nr) > 30
74 .error "Bad register number \nr."
78 .macro _sve_check_zreg znr
79 .if (\znr) < 0 || (\znr) > 31
80 .error "Bad Scalable Vector Extension vector register number \znr."
84 .macro _sve_check_preg pnr
85 .if (\pnr) < 0 || (\pnr) > 15
86 .error "Bad Scalable Vector Extension predicate register number \pnr."
90 .macro _check_num n, min, max
91 .if (\n) < (\min) || (\n) > (\max)
92 .error "Number \n out of range [\min,\max]"
96 /* SVE instruction encodings for non-SVE-capable assemblers */
97 /* (pre binutils 2.28, all kernel capable clang versions support SVE) */
99 /* STR (vector): STR Z\nz, [X\nxbase, #\offset, MUL VL] */
100 .macro _sve_str_v nz, nxbase, offset=0
102 _check_general_reg \nxbase
103 _check_num (\offset), -0x100, 0xff
107 | (((\offset) & 7) << 10) \
108 | (((\offset) & 0x1f8) << 13)
111 /* LDR (vector): LDR Z\nz, [X\nxbase, #\offset, MUL VL] */
112 .macro _sve_ldr_v nz, nxbase, offset=0
114 _check_general_reg \nxbase
115 _check_num (\offset), -0x100, 0xff
119 | (((\offset) & 7) << 10) \
120 | (((\offset) & 0x1f8) << 13)
123 /* STR (predicate): STR P\np, [X\nxbase, #\offset, MUL VL] */
124 .macro _sve_str_p np, nxbase, offset=0
126 _check_general_reg \nxbase
127 _check_num (\offset), -0x100, 0xff
131 | (((\offset) & 7) << 10) \
132 | (((\offset) & 0x1f8) << 13)
135 /* LDR (predicate): LDR P\np, [X\nxbase, #\offset, MUL VL] */
136 .macro _sve_ldr_p np, nxbase, offset=0
138 _check_general_reg \nxbase
139 _check_num (\offset), -0x100, 0xff
143 | (((\offset) & 7) << 10) \
144 | (((\offset) & 0x1f8) << 13)
147 /* RDVL X\nx, #\imm */
148 .macro _sve_rdvl nx, imm
149 _check_general_reg \nx
150 _check_num (\imm), -0x20, 0x1f
153 | (((\imm) & 0x3f) << 5)
156 /* RDFFR (unpredicated): RDFFR P\np.B */
171 .macro _sve_pfalse np
177 .macro __for from:req, to:req
181 __for %\from, %((\from) + ((\to) - (\from)) / 2)
182 __for %((\from) + ((\to) - (\from)) / 2 + 1), %\to
186 .macro _for var:req, from:req, to:req, insn:vararg
187 .macro _for__body \var:req
200 /* Update ZCR_EL1.LEN with the new VQ */
201 .macro sve_load_vq xvqminus1, xtmp, xtmp2
202 mrs_s \xtmp, SYS_ZCR_EL1
203 bic \xtmp2, \xtmp, ZCR_ELx_LEN_MASK
204 orr \xtmp2, \xtmp2, \xvqminus1
207 msr_s SYS_ZCR_EL1, \xtmp2 //self-synchronising
211 /* Preserve the first 128-bits of Znz and zero the rest. */
212 .macro _sve_flush_z nz
214 mov v\nz\().16b, v\nz\().16b
218 _for n, 0, 31, _sve_flush_z \n
220 .macro sve_flush_p_ffr
221 _for n, 0, 15, _sve_pfalse \n
225 .macro sve_save nxbase, xpfpsr, nxtmp
226 _for n, 0, 31, _sve_str_v \n, \nxbase, \n - 34
227 _for n, 0, 15, _sve_str_p \n, \nxbase, \n - 16
229 _sve_str_p 0, \nxbase
230 _sve_ldr_p 0, \nxbase, -16
233 str w\nxtmp, [\xpfpsr]
235 str w\nxtmp, [\xpfpsr, #4]
238 .macro __sve_load nxbase, xpfpsr, nxtmp
239 _for n, 0, 31, _sve_ldr_v \n, \nxbase, \n - 34
240 _sve_ldr_p 0, \nxbase
242 _for n, 0, 15, _sve_ldr_p \n, \nxbase, \n - 16
244 ldr w\nxtmp, [\xpfpsr]
246 ldr w\nxtmp, [\xpfpsr, #4]
250 .macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp, xtmp2
251 sve_load_vq \xvqminus1, x\nxtmp, \xtmp2
252 __sve_load \nxbase, \xpfpsr, \nxtmp