arch/x86/lib/memcpy_32.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/string.h>
   3 #include <linux/export.h>
   4
   5 #undef memcpy
   6 #undef memset
   7
   8 __visible void *memcpy(void *to, const void *from, size_t n)
   9 {
  10 #if defined(CONFIG_X86_USE_3DNOW) && !defined(CONFIG_FORTIFY_SOURCE)
  11         return __memcpy3d(to, from, n);
  12 #else
  13         return __memcpy(to, from, n);
  14 #endif
  15 }
  16 EXPORT_SYMBOL(memcpy);
  17
  18 __visible void *memset(void *s, int c, size_t count)
  19 {
  20         return __memset(s, c, count);
  21 }
  22 EXPORT_SYMBOL(memset);
  23
  24 __visible void *memmove(void *dest, const void *src, size_t n)
  25 {
  26         int d0,d1,d2,d3,d4,d5;
  27         char *ret = dest;
  28
  29         __asm__ __volatile__(
  30                 /* Handle more 16 bytes in loop */
  31                 "cmp $0x10, %0\n\t"
  32                 "jb     1f\n\t"
  33
  34                 /* Decide forward/backward copy mode */
  35                 "cmp %2, %1\n\t"
  36                 "jb     2f\n\t"
  37
  38                 /*
  39                  * movs instruction have many startup latency
  40                  * so we handle small size by general register.
  41                  */
  42                 "cmp  $680, %0\n\t"
  43                 "jb 3f\n\t"
  44                 /*
  45                  * movs instruction is only good for aligned case.
  46                  */
  47                 "mov %1, %3\n\t"
  48                 "xor %2, %3\n\t"
  49                 "and $0xff, %3\n\t"
  50                 "jz 4f\n\t"
  51                 "3:\n\t"
  52                 "sub $0x10, %0\n\t"
  53
  54                 /*
  55                  * We gobble 16 bytes forward in each loop.
  56                  */
  57                 "3:\n\t"
  58                 "sub $0x10, %0\n\t"
  59                 "mov 0*4(%1), %3\n\t"
  60                 "mov 1*4(%1), %4\n\t"
  61                 "mov  %3, 0*4(%2)\n\t"
  62                 "mov  %4, 1*4(%2)\n\t"
  63                 "mov 2*4(%1), %3\n\t"
  64                 "mov 3*4(%1), %4\n\t"
  65                 "mov  %3, 2*4(%2)\n\t"
  66                 "mov  %4, 3*4(%2)\n\t"
  67                 "lea  0x10(%1), %1\n\t"
  68                 "lea  0x10(%2), %2\n\t"
  69                 "jae 3b\n\t"
  70                 "add $0x10, %0\n\t"
  71                 "jmp 1f\n\t"
  72
  73                 /*
  74                  * Handle data forward by movs.
  75                  */
  76                 ".p2align 4\n\t"
  77                 "4:\n\t"
  78                 "mov -4(%1, %0), %3\n\t"
  79                 "lea -4(%2, %0), %4\n\t"
  80                 "shr $2, %0\n\t"
  81                 "rep movsl\n\t"
  82                 "mov %3, (%4)\n\t"
  83                 "jmp 11f\n\t"
  84                 /*
  85                  * Handle data backward by movs.
  86                  */
  87                 ".p2align 4\n\t"
  88                 "6:\n\t"
  89                 "mov (%1), %3\n\t"
  90                 "mov %2, %4\n\t"
  91                 "lea -4(%1, %0), %1\n\t"
  92                 "lea -4(%2, %0), %2\n\t"
  93                 "shr $2, %0\n\t"
  94                 "std\n\t"
  95                 "rep movsl\n\t"
  96                 "mov %3,(%4)\n\t"
  97                 "cld\n\t"
  98                 "jmp 11f\n\t"
  99
 100                 /*
 101                  * Start to prepare for backward copy.
 102                  */
 103                 ".p2align 4\n\t"
 104                 "2:\n\t"
 105                 "cmp  $680, %0\n\t"
 106                 "jb 5f\n\t"
 107                 "mov %1, %3\n\t"
 108                 "xor %2, %3\n\t"
 109                 "and $0xff, %3\n\t"
 110                 "jz 6b\n\t"
 111
 112                 /*
 113                  * Calculate copy position to tail.
 114                  */
 115                 "5:\n\t"
 116                 "add %0, %1\n\t"
 117                 "add %0, %2\n\t"
 118                 "sub $0x10, %0\n\t"
 119
 120                 /*
 121                  * We gobble 16 bytes backward in each loop.
 122                  */
 123                 "7:\n\t"
 124                 "sub $0x10, %0\n\t"
 125
 126                 "mov -1*4(%1), %3\n\t"
 127                 "mov -2*4(%1), %4\n\t"
 128                 "mov  %3, -1*4(%2)\n\t"
 129                 "mov  %4, -2*4(%2)\n\t"
 130                 "mov -3*4(%1), %3\n\t"
 131                 "mov -4*4(%1), %4\n\t"
 132                 "mov  %3, -3*4(%2)\n\t"
 133                 "mov  %4, -4*4(%2)\n\t"
 134                 "lea  -0x10(%1), %1\n\t"
 135                 "lea  -0x10(%2), %2\n\t"
 136                 "jae 7b\n\t"
 137                 /*
 138                  * Calculate copy position to head.
 139                  */
 140                 "add $0x10, %0\n\t"
 141                 "sub %0, %1\n\t"
 142                 "sub %0, %2\n\t"
 143
 144                 /*
 145                  * Move data from 8 bytes to 15 bytes.
 146                  */
 147                 ".p2align 4\n\t"
 148                 "1:\n\t"
 149                 "cmp $8, %0\n\t"
 150                 "jb 8f\n\t"
 151                 "mov 0*4(%1), %3\n\t"
 152                 "mov 1*4(%1), %4\n\t"
 153                 "mov -2*4(%1, %0), %5\n\t"
 154                 "mov -1*4(%1, %0), %1\n\t"
 155
 156                 "mov  %3, 0*4(%2)\n\t"
 157                 "mov  %4, 1*4(%2)\n\t"
 158                 "mov  %5, -2*4(%2, %0)\n\t"
 159                 "mov  %1, -1*4(%2, %0)\n\t"
 160                 "jmp 11f\n\t"
 161
 162                 /*
 163                  * Move data from 4 bytes to 7 bytes.
 164                  */
 165                 ".p2align 4\n\t"
 166                 "8:\n\t"
 167                 "cmp $4, %0\n\t"
 168                 "jb 9f\n\t"
 169                 "mov 0*4(%1), %3\n\t"
 170                 "mov -1*4(%1, %0), %4\n\t"
 171                 "mov  %3, 0*4(%2)\n\t"
 172                 "mov  %4, -1*4(%2, %0)\n\t"
 173                 "jmp 11f\n\t"
 174
 175                 /*
 176                  * Move data from 2 bytes to 3 bytes.
 177                  */
 178                 ".p2align 4\n\t"
 179                 "9:\n\t"
 180                 "cmp $2, %0\n\t"
 181                 "jb 10f\n\t"
 182                 "movw 0*2(%1), %%dx\n\t"
 183                 "movw -1*2(%1, %0), %%bx\n\t"
 184                 "movw %%dx, 0*2(%2)\n\t"
 185                 "movw %%bx, -1*2(%2, %0)\n\t"
 186                 "jmp 11f\n\t"
 187
 188                 /*
 189                  * Move data for 1 byte.
 190                  */
 191                 ".p2align 4\n\t"
 192                 "10:\n\t"
 193                 "cmp $1, %0\n\t"
 194                 "jb 11f\n\t"
 195                 "movb (%1), %%cl\n\t"
 196                 "movb %%cl, (%2)\n\t"
 197                 ".p2align 4\n\t"
 198                 "11:"
 199                 : "=&c" (d0), "=&S" (d1), "=&D" (d2),
 200                   "=r" (d3),"=r" (d4), "=r"(d5)
 201                 :"0" (n),
 202                  "1" (src),
 203                  "2" (dest)
 204                 :"memory");
 205
 206         return ret;
 207
 208 }
 209 EXPORT_SYMBOL(memmove);