arch/arm/lib/memmove.S

   1 /* SPDX-License-Identifier: GPL-2.0-only */
   2 /*
   3  *  linux/arch/arm/lib/memmove.S
   4  *
   5  *  Author:     Nicolas Pitre
   6  *  Created:    Sep 28, 2005
   7  *  Copyright:  (C) MontaVista Software Inc.
   8  */
   9
  10 #include <linux/linkage.h>
  11 #include <asm/assembler.h>
  12 #include <asm/unwind.h>
  13
  14                 .text
  15
  16 /*
  17  * Prototype: void *memmove(void *dest, const void *src, size_t n);
  18  *
  19  * Note:
  20  *
  21  * If the memory regions don't overlap, we simply branch to memcpy which is
  22  * normally a bit faster. Otherwise the copy is done going downwards.  This
  23  * is a transposition of the code from copy_template.S but with the copy
  24  * occurring in the opposite direction.
  25  */
  26
  27 ENTRY(memmove)
  28         UNWIND( .fnstart                        )
  29
  30                 subs    ip, r0, r1
  31                 cmphi   r2, ip
  32                 bls     memcpy
  33
  34                 stmfd   sp!, {r0, r4, lr}
  35         UNWIND( .fnend                          )
  36
  37         UNWIND( .fnstart                        )
  38         UNWIND( .save   {r0, r4, lr}            ) @ in first stmfd block
  39                 add     r1, r1, r2
  40                 add     r0, r0, r2
  41                 subs    r2, r2, #4
  42                 blt     8f
  43                 ands    ip, r0, #3
  44         PLD(    pld     [r1, #-4]               )
  45                 bne     9f
  46                 ands    ip, r1, #3
  47                 bne     10f
  48
  49 1:              subs    r2, r2, #(28)
  50                 stmfd   sp!, {r5 - r8}
  51         UNWIND( .fnend                          )
  52
  53         UNWIND( .fnstart                        )
  54         UNWIND( .save   {r0, r4, lr}            )
  55         UNWIND( .save   {r5 - r8}               ) @ in second stmfd block
  56                 blt     5f
  57
  58         CALGN(  ands    ip, r0, #31             )
  59         CALGN(  sbcsne  r4, ip, r2              )  @ C is always set here
  60         CALGN(  bcs     2f                      )
  61         CALGN(  adr     r4, 6f                  )
  62         CALGN(  subs    r2, r2, ip              )  @ C is set here
  63         CALGN(  rsb     ip, ip, #32             )
  64         CALGN(  add     pc, r4, ip              )
  65
  66         PLD(    pld     [r1, #-4]               )
  67 2:      PLD(    subs    r2, r2, #96             )
  68         PLD(    pld     [r1, #-32]              )
  69         PLD(    blt     4f                      )
  70         PLD(    pld     [r1, #-64]              )
  71         PLD(    pld     [r1, #-96]              )
  72
  73 3:      PLD(    pld     [r1, #-128]             )
  74 4:              ldmdb   r1!, {r3, r4, r5, r6, r7, r8, ip, lr}
  75                 subs    r2, r2, #32
  76                 stmdb   r0!, {r3, r4, r5, r6, r7, r8, ip, lr}
  77                 bge     3b
  78         PLD(    cmn     r2, #96                 )
  79         PLD(    bge     4b                      )
  80
  81 5:              ands    ip, r2, #28
  82                 rsb     ip, ip, #32
  83                 addne   pc, pc, ip              @ C is always clear here
  84                 b       7f
  85 6:              W(nop)
  86                 W(ldr)  r3, [r1, #-4]!
  87                 W(ldr)  r4, [r1, #-4]!
  88                 W(ldr)  r5, [r1, #-4]!
  89                 W(ldr)  r6, [r1, #-4]!
  90                 W(ldr)  r7, [r1, #-4]!
  91                 W(ldr)  r8, [r1, #-4]!
  92                 W(ldr)  lr, [r1, #-4]!
  93
  94                 add     pc, pc, ip
  95                 nop
  96                 W(nop)
  97                 W(str)  r3, [r0, #-4]!
  98                 W(str)  r4, [r0, #-4]!
  99                 W(str)  r5, [r0, #-4]!
 100                 W(str)  r6, [r0, #-4]!
 101                 W(str)  r7, [r0, #-4]!
 102                 W(str)  r8, [r0, #-4]!
 103                 W(str)  lr, [r0, #-4]!
 104
 105         CALGN(  bcs     2b                      )
 106
 107 7:              ldmfd   sp!, {r5 - r8}
 108         UNWIND( .fnend                          ) @ end of second stmfd block
 109
 110         UNWIND( .fnstart                        )
 111         UNWIND( .save   {r0, r4, lr}            ) @ still in first stmfd block
 112
 113 8:              movs    r2, r2, lsl #31
 114                 ldrbne  r3, [r1, #-1]!
 115                 ldrbcs  r4, [r1, #-1]!
 116                 ldrbcs  ip, [r1, #-1]
 117                 strbne  r3, [r0, #-1]!
 118                 strbcs  r4, [r0, #-1]!
 119                 strbcs  ip, [r0, #-1]
 120                 ldmfd   sp!, {r0, r4, pc}
 121
 122 9:              cmp     ip, #2
 123                 ldrbgt  r3, [r1, #-1]!
 124                 ldrbge  r4, [r1, #-1]!
 125                 ldrb    lr, [r1, #-1]!
 126                 strbgt  r3, [r0, #-1]!
 127                 strbge  r4, [r0, #-1]!
 128                 subs    r2, r2, ip
 129                 strb    lr, [r0, #-1]!
 130                 blt     8b
 131                 ands    ip, r1, #3
 132                 beq     1b
 133
 134 10:             bic     r1, r1, #3
 135                 cmp     ip, #2
 136                 ldr     r3, [r1, #0]
 137                 beq     17f
 138                 blt     18f
 139         UNWIND( .fnend                          )
 140
 141
 142                 .macro  backward_copy_shift push pull
 143
 144         UNWIND( .fnstart                        )
 145         UNWIND( .save   {r0, r4, lr}            ) @ still in first stmfd block
 146                 subs    r2, r2, #28
 147                 blt     14f
 148
 149         CALGN(  ands    ip, r0, #31             )
 150         CALGN(  sbcsne  r4, ip, r2              )  @ C is always set here
 151         CALGN(  subcc   r2, r2, ip              )
 152         CALGN(  bcc     15f                     )
 153
 154 11:             stmfd   sp!, {r5 - r9}
 155         UNWIND( .fnend                          )
 156
 157         UNWIND( .fnstart                        )
 158         UNWIND( .save   {r0, r4, lr}            )
 159         UNWIND( .save   {r5 - r9}               ) @ in new second stmfd block
 160
 161         PLD(    pld     [r1, #-4]               )
 162         PLD(    subs    r2, r2, #96             )
 163         PLD(    pld     [r1, #-32]              )
 164         PLD(    blt     13f                     )
 165         PLD(    pld     [r1, #-64]              )
 166         PLD(    pld     [r1, #-96]              )
 167
 168 12:     PLD(    pld     [r1, #-128]             )
 169 13:             ldmdb   r1!, {r7, r8, r9, ip}
 170                 mov     lr, r3, lspush #\push
 171                 subs    r2, r2, #32
 172                 ldmdb   r1!, {r3, r4, r5, r6}
 173                 orr     lr, lr, ip, lspull #\pull
 174                 mov     ip, ip, lspush #\push
 175                 orr     ip, ip, r9, lspull #\pull
 176                 mov     r9, r9, lspush #\push
 177                 orr     r9, r9, r8, lspull #\pull
 178                 mov     r8, r8, lspush #\push
 179                 orr     r8, r8, r7, lspull #\pull
 180                 mov     r7, r7, lspush #\push
 181                 orr     r7, r7, r6, lspull #\pull
 182                 mov     r6, r6, lspush #\push
 183                 orr     r6, r6, r5, lspull #\pull
 184                 mov     r5, r5, lspush #\push
 185                 orr     r5, r5, r4, lspull #\pull
 186                 mov     r4, r4, lspush #\push
 187                 orr     r4, r4, r3, lspull #\pull
 188                 stmdb   r0!, {r4 - r9, ip, lr}
 189                 bge     12b
 190         PLD(    cmn     r2, #96                 )
 191         PLD(    bge     13b                     )
 192
 193                 ldmfd   sp!, {r5 - r9}
 194         UNWIND( .fnend                          ) @ end of the second stmfd block
 195
 196         UNWIND( .fnstart                        )
 197         UNWIND( .save {r0, r4, lr}              ) @ still in first stmfd block
 198
 199 14:             ands    ip, r2, #28
 200                 beq     16f
 201
 202 15:             mov     lr, r3, lspush #\push
 203                 ldr     r3, [r1, #-4]!
 204                 subs    ip, ip, #4
 205                 orr     lr, lr, r3, lspull #\pull
 206                 str     lr, [r0, #-4]!
 207                 bgt     15b
 208         CALGN(  cmp     r2, #0                  )
 209         CALGN(  bge     11b                     )
 210
 211 16:             add     r1, r1, #(\pull / 8)
 212                 b       8b
 213         UNWIND( .fnend                          )
 214
 215                 .endm
 216
 217
 218                 backward_copy_shift     push=8  pull=24
 219
 220 17:             backward_copy_shift     push=16 pull=16
 221
 222 18:             backward_copy_shift     push=24 pull=8
 223
 224 ENDPROC(memmove)