1 /* SPDX-License-Identifier: GPL-2.0-only */
3 * Copyright (c) 2013-2021, Arm Limited.
5 * Adapted from the original at:
6 * https://github.com/ARM-software/optimized-routines/blob/98e4d6a5c13c8e54/string/aarch64/strlen.S
9 #include <linux/linkage.h>
10 #include <asm/assembler.h>
14 * ARMv8-a, AArch64, unaligned accesses, min page size 4k.
17 #define L(label) .L ## label
19 /* Arguments and results. */
23 /* Locals and temporaries. */
35 /* NUL detection works on the principle that (X - 1) & (~X) & 0x80
36 (=> (X - 1) & ~(X | 0x7f)) is non-zero iff a byte is zero, and
37 can be done in parallel across the entire word. A faster check
38 (X - 1) & 0x80 is zero for non-NUL ASCII characters, but gives
39 false hits for characters 129..255. */
41 #define REP8_01 0x0101010101010101
42 #define REP8_7f 0x7f7f7f7f7f7f7f7f
43 #define REP8_80 0x8080808080808080
45 #define MIN_PAGE_SIZE 4096
47 /* Since strings are short on average, we check the first 16 bytes
48 of the string for a NUL character. In order to do an unaligned ldp
49 safely we have to do a page cross check first. If there is a NUL
50 byte we calculate the length from the 2 8-byte words using
51 conditional select to reduce branch mispredictions (it is unlikely
52 strlen will be repeatedly called on strings with the same length).
54 If the string is longer than 16 bytes, we align src so don't need
55 further page cross checks, and process 32 bytes per iteration
56 using the fast NUL check. If we encounter non-ASCII characters,
57 fallback to a second loop using the full NUL check.
59 If the page cross check fails, we read 16 bytes from an aligned
60 address, remove any characters before the string, and continue
61 in the main loop using aligned loads. Since strings crossing a
62 page in the first 16 bytes are rare (probability of
63 16/MIN_PAGE_SIZE ~= 0.4%), this case does not need to be optimized.
65 AArch64 systems have a minimum page size of 4k. We don't bother
66 checking for larger page sizes - the cost of setting up the correct
67 page size is just not worth the extra gain from a small reduction in
68 the cases taking the slow path. Note that we only care about
69 whether the first fetch, which may be misaligned, crosses a page
72 SYM_FUNC_START_WEAK_PI(strlen)
73 and tmp1, srcin, MIN_PAGE_SIZE - 1
75 cmp tmp1, MIN_PAGE_SIZE - 16
77 ldp data1, data2, [srcin]
79 /* For big-endian, carry propagation (if the final byte in the
80 string is 0x01) means we cannot use has_nul1/2 directly.
81 Since we expect strings to be small and early-exit,
82 byte-swap the data now so has_null1/2 will be correct. */
86 sub tmp1, data1, zeroones
87 orr tmp2, data1, REP8_7f
88 sub tmp3, data2, zeroones
89 orr tmp4, data2, REP8_7f
90 bics has_nul1, tmp1, tmp2
91 bic has_nul2, tmp3, tmp4
92 ccmp has_nul2, 0, 0, eq
93 beq L(main_loop_entry)
95 /* Enter with C = has_nul1 == 0. */
96 csel has_nul1, has_nul1, has_nul2, cc
98 rev has_nul1, has_nul1
100 csel len, xzr, len, cc
101 add len, len, tmp1, lsr 3
104 /* The inner loop processes 32 bytes per iteration and uses the fast
105 NUL check. If we encounter non-ASCII characters, use a second
106 loop with the accurate NUL check. */
112 ldp data1, data2, [src, 32]!
114 sub tmp1, data1, zeroones
115 sub tmp3, data2, zeroones
117 tst tmp2, zeroones, lsl 7
119 ldp data1, data2, [src, 16]
120 sub tmp1, data1, zeroones
121 sub tmp3, data2, zeroones
123 tst tmp2, zeroones, lsl 7
127 /* The fast check failed, so do the slower, accurate NUL check. */
128 orr tmp2, data1, REP8_7f
129 orr tmp4, data2, REP8_7f
130 bics has_nul1, tmp1, tmp2
131 bic has_nul2, tmp3, tmp4
132 ccmp has_nul2, 0, 0, eq
135 /* Enter with C = has_nul1 == 0. */
138 /* For big-endian, carry propagation (if the final byte in the
139 string is 0x01) means we cannot use has_nul1/2 directly. The
140 easiest way to get the correct byte is to byte-swap the data
141 and calculate the syndrome a second time. */
142 csel data1, data1, data2, cc
144 sub tmp1, data1, zeroones
145 orr tmp2, data1, REP8_7f
146 bic has_nul1, tmp1, tmp2
148 csel has_nul1, has_nul1, has_nul2, cc
151 rev has_nul1, has_nul1
154 csel len, len, tmp2, cc
155 add len, len, tmp1, lsr 3
159 ldp data1, data2, [src, 16]!
160 sub tmp1, data1, zeroones
161 orr tmp2, data1, REP8_7f
162 sub tmp3, data2, zeroones
163 orr tmp4, data2, REP8_7f
164 bics has_nul1, tmp1, tmp2
165 bic has_nul2, tmp3, tmp4
166 ccmp has_nul2, 0, 0, eq
168 ldp data1, data2, [src, 16]!
169 sub tmp1, data1, zeroones
170 orr tmp2, data1, REP8_7f
171 sub tmp3, data2, zeroones
172 orr tmp4, data2, REP8_7f
173 bics has_nul1, tmp1, tmp2
174 bic has_nul2, tmp3, tmp4
175 ccmp has_nul2, 0, 0, eq
179 /* Load 16 bytes from [srcin & ~15] and force the bytes that precede
180 srcin to 0x7f, so we ignore any NUL bytes before the string.
181 Then continue in the aligned loop. */
184 ldp data1, data2, [src]
188 /* Big-endian. Early bytes are at MSB. */
189 lsr tmp1, tmp4, tmp1 /* Shift (tmp1 & 63). */
191 /* Little-endian. Early bytes are at LSB. */
192 lsl tmp1, tmp4, tmp1 /* Shift (tmp1 & 63). */
194 orr tmp1, tmp1, REP8_80
195 orn data1, data1, tmp1
196 orn tmp2, data2, tmp1
198 csel data1, data1, tmp4, eq
199 csel data2, data2, tmp2, eq
200 b L(page_cross_entry)
202 SYM_FUNC_END_PI(strlen)
203 EXPORT_SYMBOL_NOKASAN(strlen)