1 /* SPDX-License-Identifier: GPL-2.0-or-later */
3 * Memory copy functions for 32-bit PowerPC.
5 * Copyright (C) 1996-2005 Paul Mackerras.
7 #include <asm/processor.h>
10 #include <asm/ppc_asm.h>
11 #include <asm/export.h>
12 #include <asm/code-patching-asm.h>
13 #include <asm/kasan.h>
15 #define COPY_16_BYTES \
25 #define COPY_16_BYTES_WITHEX(n) \
43 #define COPY_16_BYTES_EXCODE(n) \
45 addi r5,r5,-(16 * n); \
48 addi r5,r5,-(16 * n); \
50 EX_TABLE(8 ## n ## 0b,9 ## n ## 0b); \
51 EX_TABLE(8 ## n ## 1b,9 ## n ## 0b); \
52 EX_TABLE(8 ## n ## 2b,9 ## n ## 0b); \
53 EX_TABLE(8 ## n ## 3b,9 ## n ## 0b); \
54 EX_TABLE(8 ## n ## 4b,9 ## n ## 1b); \
55 EX_TABLE(8 ## n ## 5b,9 ## n ## 1b); \
56 EX_TABLE(8 ## n ## 6b,9 ## n ## 1b); \
57 EX_TABLE(8 ## n ## 7b,9 ## n ## 1b)
60 .stabs "arch/powerpc/lib/",N_SO,0,0,0f
61 .stabs "copy_32.S",N_SO,0,0,0f
64 CACHELINE_BYTES = L1_CACHE_BYTES
65 LG_CACHELINE_BYTES = L1_CACHE_SHIFT
66 CACHELINE_MASK = (L1_CACHE_BYTES-1)
70 rlwinm. r0 ,r5, 31, 1, 31
73 rlwimi r4 ,r4 ,16 ,0 ,15
81 EXPORT_SYMBOL(memset16)
85 * Use dcbz on the complete cache lines in the destination
86 * to set them to zero. This requires that the destination
87 * area is cacheable. -- paulus
89 * During early init, cache might not be active yet, so dcbz cannot be used.
90 * We therefore skip the optimised bloc that uses dcbz. This jump is
91 * replaced by a nop once cache is active. This is done in machine_init()
107 * Skip optimised bloc until cache is enabled. Will be replaced
108 * by 'bne' during boot to use normal procedure if r4 is not zero
111 patch_site 5b, patch__memset_nocache
113 clrlwi r7,r6,32-LG_CACHELINE_BYTES
115 srwi r9,r8,LG_CACHELINE_BYTES
116 addic. r9,r9,-1 /* total number of complete cachelines */
118 xori r0,r7,CACHELINE_MASK & ~3
127 addi r6,r6,CACHELINE_BYTES
129 clrlwi r5,r8,32-LG_CACHELINE_BYTES
152 EXPORT_SYMBOL(memset)
153 EXPORT_SYMBOL_KASAN(memset)
156 * This version uses dcbz on the complete cache lines in the
157 * destination area to reduce memory traffic. This requires that
158 * the destination area is cacheable.
159 * We only use this version if the source and dest don't overlap.
162 * During early init, cache might not be active yet, so dcbz cannot be used.
163 * We therefore jump to generic_memcpy which doesn't use dcbz. This jump is
164 * replaced by a nop once cache is active. This is done in machine_init()
166 _GLOBAL_KASAN(memmove)
171 _GLOBAL_KASAN(memcpy)
173 patch_site 1b, patch__memcpy_nocache
175 add r7,r3,r5 /* test if the src & dst overlap */
179 crand 0,0,4 /* cr0.lt &= cr1.lt */
180 blt generic_memcpy /* if regions overlap */
185 andi. r0,r0,CACHELINE_MASK /* # bytes to start of cache line */
188 cmplw 0,r5,r0 /* is this more than total to do? */
189 blt 63f /* if not much to do */
190 andi. r8,r0,3 /* get it word-aligned first */
194 70: lbz r9,4(r4) /* do some bytes */
202 72: lwzu r9,4(r4) /* do some words */
206 58: srwi. r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */
207 clrlwi r5,r5,32-LG_CACHELINE_BYTES
214 #if L1_CACHE_BYTES >= 32
216 #if L1_CACHE_BYTES >= 64
219 #if L1_CACHE_BYTES >= 128
245 EXPORT_SYMBOL(memcpy)
246 EXPORT_SYMBOL(memmove)
247 EXPORT_SYMBOL_KASAN(memcpy)
248 EXPORT_SYMBOL_KASAN(memmove)
254 beq 2f /* if less than 8 bytes to do */
255 andi. r0,r6,3 /* get dest word aligned */
286 rlwinm. r7,r5,32-3,3,31
291 _GLOBAL(backwards_memcpy)
292 rlwinm. r7,r5,32-3,3,31 /* r0 = r5 >> 3 */
322 rlwinm. r7,r5,32-3,3,31
327 _GLOBAL(__copy_tofrom_user)
331 andi. r0,r0,CACHELINE_MASK /* # bytes to start of cache line */
334 cmplw 0,r5,r0 /* is this more than total to do? */
335 blt 63f /* if not much to do */
336 andi. r8,r0,3 /* get it word-aligned first */
339 70: lbz r9,4(r4) /* do some bytes */
348 72: lwzu r9,4(r4) /* do some words */
357 58: srwi. r0,r5,LG_CACHELINE_BYTES /* # complete cachelines */
358 clrlwi r5,r5,32-LG_CACHELINE_BYTES
362 /* Here we decide how far ahead to prefetch the source */
368 #if MAX_COPY_PREFETCH > 1
369 /* Heuristically, for large transfers we prefetch
370 MAX_COPY_PREFETCH cachelines ahead. For small transfers
371 we prefetch 1 cacheline ahead. */
372 cmpwi r0,MAX_COPY_PREFETCH
374 li r7,MAX_COPY_PREFETCH
377 addi r3,r3,CACHELINE_BYTES
381 addi r3,r3,CACHELINE_BYTES
382 #endif /* MAX_COPY_PREFETCH > 1 */
391 /* the main body of the cacheline loop */
392 COPY_16_BYTES_WITHEX(0)
393 #if L1_CACHE_BYTES >= 32
394 COPY_16_BYTES_WITHEX(1)
395 #if L1_CACHE_BYTES >= 64
396 COPY_16_BYTES_WITHEX(2)
397 COPY_16_BYTES_WITHEX(3)
398 #if L1_CACHE_BYTES >= 128
399 COPY_16_BYTES_WITHEX(4)
400 COPY_16_BYTES_WITHEX(5)
401 COPY_16_BYTES_WITHEX(6)
402 COPY_16_BYTES_WITHEX(7)
430 /* read fault, initial single-byte copy */
433 /* write fault, initial single-byte copy */
438 /* read fault, initial word copy */
441 /* write fault, initial word copy */
447 * this stuff handles faults in the cacheline loop and branches to either
448 * 104f (if in read part) or 105f (if in write part), after updating r5
450 COPY_16_BYTES_EXCODE(0)
451 #if L1_CACHE_BYTES >= 32
452 COPY_16_BYTES_EXCODE(1)
453 #if L1_CACHE_BYTES >= 64
454 COPY_16_BYTES_EXCODE(2)
455 COPY_16_BYTES_EXCODE(3)
456 #if L1_CACHE_BYTES >= 128
457 COPY_16_BYTES_EXCODE(4)
458 COPY_16_BYTES_EXCODE(5)
459 COPY_16_BYTES_EXCODE(6)
460 COPY_16_BYTES_EXCODE(7)
465 /* read fault in cacheline loop */
468 /* fault on dcbz (effectively a write fault) */
469 /* or write fault in cacheline loop */
471 92: li r3,LG_CACHELINE_BYTES
475 /* read fault in final word loop */
478 /* write fault in final word loop */
483 /* read fault in final byte loop */
486 /* write fault in final byte loop */
491 * At this stage the number of bytes not copied is
492 * r5 + (ctr << r3), and r9 is 0 for read or 1 for write.
497 beq 120f /* shouldn't happen */
500 /* for a read fault, first try to continue the copy one byte at a time */
507 /* then clear out the destination: r3 bytes starting at 4(r6) */
518 EXPORT_SYMBOL(__copy_tofrom_user)