arch/alpha/lib/csum_partial_copy.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * csum_partial_copy - do IP checksumming and copy
   4  *
   5  * (C) Copyright 1996 Linus Torvalds
   6  * accelerated versions (and 21264 assembly versions ) contributed by
   7  *      Rick Gorton     <rick.gorton@alpha-processor.com>
   8  *
   9  * Don't look at this too closely - you'll go mad. The things
  10  * we do for performance..
  11  */
  12
  13 #include <linux/types.h>
  14 #include <linux/string.h>
  15 #include <linux/uaccess.h>
  16
  17
  18 #define ldq_u(x,y) \
  19 __asm__ __volatile__("ldq_u %0,%1":"=r" (x):"m" (*(const unsigned long *)(y)))
  20
  21 #define stq_u(x,y) \
  22 __asm__ __volatile__("stq_u %1,%0":"=m" (*(unsigned long *)(y)):"r" (x))
  23
  24 #define extql(x,y,z) \
  25 __asm__ __volatile__("extql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  26
  27 #define extqh(x,y,z) \
  28 __asm__ __volatile__("extqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  29
  30 #define mskql(x,y,z) \
  31 __asm__ __volatile__("mskql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  32
  33 #define mskqh(x,y,z) \
  34 __asm__ __volatile__("mskqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  35
  36 #define insql(x,y,z) \
  37 __asm__ __volatile__("insql %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  38
  39 #define insqh(x,y,z) \
  40 __asm__ __volatile__("insqh %1,%2,%0":"=r" (z):"r" (x),"r" (y))
  41
  42 #define __get_word(insn,x,ptr)                          \
  43 ({                                                      \
  44         long __guu_err;                                 \
  45         __asm__ __volatile__(                           \
  46         "1:     "#insn" %0,%2\n"                        \
  47         "2:\n"                                          \
  48         EXC(1b,2b,%0,%1)                                \
  49                 : "=r"(x), "=r"(__guu_err)              \
  50                 : "m"(__m(ptr)), "1"(0));               \
  51         __guu_err;                                      \
  52 })
  53
  54 static inline unsigned short from64to16(unsigned long x)
  55 {
  56         /* Using extract instructions is a bit more efficient
  57            than the original shift/bitmask version.  */
  58
  59         union {
  60                 unsigned long   ul;
  61                 unsigned int    ui[2];
  62                 unsigned short  us[4];
  63         } in_v, tmp_v, out_v;
  64
  65         in_v.ul = x;
  66         tmp_v.ul = (unsigned long) in_v.ui[0] + (unsigned long) in_v.ui[1];
  67
  68         /* Since the bits of tmp_v.sh[3] are going to always be zero,
  69            we don't have to bother to add that in.  */
  70         out_v.ul = (unsigned long) tmp_v.us[0] + (unsigned long) tmp_v.us[1]
  71                         + (unsigned long) tmp_v.us[2];
  72
  73         /* Similarly, out_v.us[2] is always zero for the final add.  */
  74         return out_v.us[0] + out_v.us[1];
  75 }
  76
  77
  78
  79 /*
  80  * Ok. This isn't fun, but this is the EASY case.
  81  */
  82 static inline unsigned long
  83 csum_partial_cfu_aligned(const unsigned long __user *src, unsigned long *dst,
  84                          long len)
  85 {
  86         unsigned long checksum = ~0U;
  87         unsigned long carry = 0;
  88
  89         while (len >= 0) {
  90                 unsigned long word;
  91                 if (__get_word(ldq, word, src))
  92                         return 0;
  93                 checksum += carry;
  94                 src++;
  95                 checksum += word;
  96                 len -= 8;
  97                 carry = checksum < word;
  98                 *dst = word;
  99                 dst++;
 100         }
 101         len += 8;
 102         checksum += carry;
 103         if (len) {
 104                 unsigned long word, tmp;
 105                 if (__get_word(ldq, word, src))
 106                         return 0;
 107                 tmp = *dst;
 108                 mskql(word, len, word);
 109                 checksum += word;
 110                 mskqh(tmp, len, tmp);
 111                 carry = checksum < word;
 112                 *dst = word | tmp;
 113                 checksum += carry;
 114         }
 115         return checksum;
 116 }
 117
 118 /*
 119  * This is even less fun, but this is still reasonably
 120  * easy.
 121  */
 122 static inline unsigned long
 123 csum_partial_cfu_dest_aligned(const unsigned long __user *src,
 124                               unsigned long *dst,
 125                               unsigned long soff,
 126                               long len)
 127 {
 128         unsigned long first;
 129         unsigned long word, carry;
 130         unsigned long lastsrc = 7+len+(unsigned long)src;
 131         unsigned long checksum = ~0U;
 132
 133         if (__get_word(ldq_u, first,src))
 134                 return 0;
 135         carry = 0;
 136         while (len >= 0) {
 137                 unsigned long second;
 138
 139                 if (__get_word(ldq_u, second, src+1))
 140                         return 0;
 141                 extql(first, soff, word);
 142                 len -= 8;
 143                 src++;
 144                 extqh(second, soff, first);
 145                 checksum += carry;
 146                 word |= first;
 147                 first = second;
 148                 checksum += word;
 149                 *dst = word;
 150                 dst++;
 151                 carry = checksum < word;
 152         }
 153         len += 8;
 154         checksum += carry;
 155         if (len) {
 156                 unsigned long tmp;
 157                 unsigned long second;
 158                 if (__get_word(ldq_u, second, lastsrc))
 159                         return 0;
 160                 tmp = *dst;
 161                 extql(first, soff, word);
 162                 extqh(second, soff, first);
 163                 word |= first;
 164                 mskql(word, len, word);
 165                 checksum += word;
 166                 mskqh(tmp, len, tmp);
 167                 carry = checksum < word;
 168                 *dst = word | tmp;
 169                 checksum += carry;
 170         }
 171         return checksum;
 172 }
 173
 174 /*
 175  * This is slightly less fun than the above..
 176  */
 177 static inline unsigned long
 178 csum_partial_cfu_src_aligned(const unsigned long __user *src,
 179                              unsigned long *dst,
 180                              unsigned long doff,
 181                              long len,
 182                              unsigned long partial_dest)
 183 {
 184         unsigned long carry = 0;
 185         unsigned long word;
 186         unsigned long second_dest;
 187         unsigned long checksum = ~0U;
 188
 189         mskql(partial_dest, doff, partial_dest);
 190         while (len >= 0) {
 191                 if (__get_word(ldq, word, src))
 192                         return 0;
 193                 len -= 8;
 194                 insql(word, doff, second_dest);
 195                 checksum += carry;
 196                 stq_u(partial_dest | second_dest, dst);
 197                 src++;
 198                 checksum += word;
 199                 insqh(word, doff, partial_dest);
 200                 carry = checksum < word;
 201                 dst++;
 202         }
 203         len += 8;
 204         if (len) {
 205                 checksum += carry;
 206                 if (__get_word(ldq, word, src))
 207                         return 0;
 208                 mskql(word, len, word);
 209                 len -= 8;
 210                 checksum += word;
 211                 insql(word, doff, second_dest);
 212                 len += doff;
 213                 carry = checksum < word;
 214                 partial_dest |= second_dest;
 215                 if (len >= 0) {
 216                         stq_u(partial_dest, dst);
 217                         if (!len) goto out;
 218                         dst++;
 219                         insqh(word, doff, partial_dest);
 220                 }
 221                 doff = len;
 222         }
 223         ldq_u(second_dest, dst);
 224         mskqh(second_dest, doff, second_dest);
 225         stq_u(partial_dest | second_dest, dst);
 226 out:
 227         checksum += carry;
 228         return checksum;
 229 }
 230
 231 /*
 232  * This is so totally un-fun that it's frightening. Don't
 233  * look at this too closely, you'll go blind.
 234  */
 235 static inline unsigned long
 236 csum_partial_cfu_unaligned(const unsigned long __user * src,
 237                            unsigned long * dst,
 238                            unsigned long soff, unsigned long doff,
 239                            long len, unsigned long partial_dest)
 240 {
 241         unsigned long carry = 0;
 242         unsigned long first;
 243         unsigned long lastsrc;
 244         unsigned long checksum = ~0U;
 245
 246         if (__get_word(ldq_u, first, src))
 247                 return 0;
 248         lastsrc = 7+len+(unsigned long)src;
 249         mskql(partial_dest, doff, partial_dest);
 250         while (len >= 0) {
 251                 unsigned long second, word;
 252                 unsigned long second_dest;
 253
 254                 if (__get_word(ldq_u, second, src+1))
 255                         return 0;
 256                 extql(first, soff, word);
 257                 checksum += carry;
 258                 len -= 8;
 259                 extqh(second, soff, first);
 260                 src++;
 261                 word |= first;
 262                 first = second;
 263                 insql(word, doff, second_dest);
 264                 checksum += word;
 265                 stq_u(partial_dest | second_dest, dst);
 266                 carry = checksum < word;
 267                 insqh(word, doff, partial_dest);
 268                 dst++;
 269         }
 270         len += doff;
 271         checksum += carry;
 272         if (len >= 0) {
 273                 unsigned long second, word;
 274                 unsigned long second_dest;
 275
 276                 if (__get_word(ldq_u, second, lastsrc))
 277                         return 0;
 278                 extql(first, soff, word);
 279                 extqh(second, soff, first);
 280                 word |= first;
 281                 first = second;
 282                 mskql(word, len-doff, word);
 283                 checksum += word;
 284                 insql(word, doff, second_dest);
 285                 carry = checksum < word;
 286                 stq_u(partial_dest | second_dest, dst);
 287                 if (len) {
 288                         ldq_u(second_dest, dst+1);
 289                         insqh(word, doff, partial_dest);
 290                         mskqh(second_dest, len, second_dest);
 291                         stq_u(partial_dest | second_dest, dst+1);
 292                 }
 293                 checksum += carry;
 294         } else {
 295                 unsigned long second, word;
 296                 unsigned long second_dest;
 297
 298                 if (__get_word(ldq_u, second, lastsrc))
 299                         return 0;
 300                 extql(first, soff, word);
 301                 extqh(second, soff, first);
 302                 word |= first;
 303                 ldq_u(second_dest, dst);
 304                 mskql(word, len-doff, word);
 305                 checksum += word;
 306                 mskqh(second_dest, len, second_dest);
 307                 carry = checksum < word;
 308                 insql(word, doff, word);
 309                 stq_u(partial_dest | word | second_dest, dst);
 310                 checksum += carry;
 311         }
 312         return checksum;
 313 }
 314
 315 static __wsum __csum_and_copy(const void __user *src, void *dst, int len)
 316 {
 317         unsigned long soff = 7 & (unsigned long) src;
 318         unsigned long doff = 7 & (unsigned long) dst;
 319         unsigned long checksum;
 320
 321         if (!doff) {
 322                 if (!soff)
 323                         checksum = csum_partial_cfu_aligned(
 324                                 (const unsigned long __user *) src,
 325                                 (unsigned long *) dst, len-8);
 326                 else
 327                         checksum = csum_partial_cfu_dest_aligned(
 328                                 (const unsigned long __user *) src,
 329                                 (unsigned long *) dst,
 330                                 soff, len-8);
 331         } else {
 332                 unsigned long partial_dest;
 333                 ldq_u(partial_dest, dst);
 334                 if (!soff)
 335                         checksum = csum_partial_cfu_src_aligned(
 336                                 (const unsigned long __user *) src,
 337                                 (unsigned long *) dst,
 338                                 doff, len-8, partial_dest);
 339                 else
 340                         checksum = csum_partial_cfu_unaligned(
 341                                 (const unsigned long __user *) src,
 342                                 (unsigned long *) dst,
 343                                 soff, doff, len-8, partial_dest);
 344         }
 345         return (__force __wsum)from64to16 (checksum);
 346 }
 347
 348 __wsum
 349 csum_and_copy_from_user(const void __user *src, void *dst, int len)
 350 {
 351         if (!access_ok(src, len))
 352                 return 0;
 353         return __csum_and_copy(src, dst, len);
 354 }
 355 EXPORT_SYMBOL(csum_and_copy_from_user);
 356
 357 __wsum
 358 csum_partial_copy_nocheck(const void *src, void *dst, int len)
 359 {
 360         return __csum_and_copy((__force const void __user *)src,
 361                                                 dst, len);
 362 }
 363 EXPORT_SYMBOL(csum_partial_copy_nocheck);