arch/arm64/crypto/chacha-neon-core.S

   1 /*
   2  * ChaCha/XChaCha NEON helper functions
   3  *
   4  * Copyright (C) 2016-2018 Linaro, Ltd. <ard.biesheuvel@linaro.org>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  *
  10  * Originally based on:
  11  * ChaCha20 256-bit cipher algorithm, RFC7539, x64 SSSE3 functions
  12  *
  13  * Copyright (C) 2015 Martin Willi
  14  *
  15  * This program is free software; you can redistribute it and/or modify
  16  * it under the terms of the GNU General Public License as published by
  17  * the Free Software Foundation; either version 2 of the License, or
  18  * (at your option) any later version.
  19  */
  20
  21 #include <linux/linkage.h>
  22 #include <asm/assembler.h>
  23 #include <asm/cache.h>
  24
  25         .text
  26         .align          6
  27
  28 /*
  29  * chacha_permute - permute one block
  30  *
  31  * Permute one 64-byte block where the state matrix is stored in the four NEON
  32  * registers v0-v3.  It performs matrix operations on four words in parallel,
  33  * but requires shuffling to rearrange the words after each round.
  34  *
  35  * The round count is given in w3.
  36  *
  37  * Clobbers: w3, x10, v4, v12
  38  */
  39 SYM_FUNC_START_LOCAL(chacha_permute)
  40
  41         adr_l           x10, ROT8
  42         ld1             {v12.4s}, [x10]
  43
  44 .Ldoubleround:
  45         // x0 += x1, x3 = rotl32(x3 ^ x0, 16)
  46         add             v0.4s, v0.4s, v1.4s
  47         eor             v3.16b, v3.16b, v0.16b
  48         rev32           v3.8h, v3.8h
  49
  50         // x2 += x3, x1 = rotl32(x1 ^ x2, 12)
  51         add             v2.4s, v2.4s, v3.4s
  52         eor             v4.16b, v1.16b, v2.16b
  53         shl             v1.4s, v4.4s, #12
  54         sri             v1.4s, v4.4s, #20
  55
  56         // x0 += x1, x3 = rotl32(x3 ^ x0, 8)
  57         add             v0.4s, v0.4s, v1.4s
  58         eor             v3.16b, v3.16b, v0.16b
  59         tbl             v3.16b, {v3.16b}, v12.16b
  60
  61         // x2 += x3, x1 = rotl32(x1 ^ x2, 7)
  62         add             v2.4s, v2.4s, v3.4s
  63         eor             v4.16b, v1.16b, v2.16b
  64         shl             v1.4s, v4.4s, #7
  65         sri             v1.4s, v4.4s, #25
  66
  67         // x1 = shuffle32(x1, MASK(0, 3, 2, 1))
  68         ext             v1.16b, v1.16b, v1.16b, #4
  69         // x2 = shuffle32(x2, MASK(1, 0, 3, 2))
  70         ext             v2.16b, v2.16b, v2.16b, #8
  71         // x3 = shuffle32(x3, MASK(2, 1, 0, 3))
  72         ext             v3.16b, v3.16b, v3.16b, #12
  73
  74         // x0 += x1, x3 = rotl32(x3 ^ x0, 16)
  75         add             v0.4s, v0.4s, v1.4s
  76         eor             v3.16b, v3.16b, v0.16b
  77         rev32           v3.8h, v3.8h
  78
  79         // x2 += x3, x1 = rotl32(x1 ^ x2, 12)
  80         add             v2.4s, v2.4s, v3.4s
  81         eor             v4.16b, v1.16b, v2.16b
  82         shl             v1.4s, v4.4s, #12
  83         sri             v1.4s, v4.4s, #20
  84
  85         // x0 += x1, x3 = rotl32(x3 ^ x0, 8)
  86         add             v0.4s, v0.4s, v1.4s
  87         eor             v3.16b, v3.16b, v0.16b
  88         tbl             v3.16b, {v3.16b}, v12.16b
  89
  90         // x2 += x3, x1 = rotl32(x1 ^ x2, 7)
  91         add             v2.4s, v2.4s, v3.4s
  92         eor             v4.16b, v1.16b, v2.16b
  93         shl             v1.4s, v4.4s, #7
  94         sri             v1.4s, v4.4s, #25
  95
  96         // x1 = shuffle32(x1, MASK(2, 1, 0, 3))
  97         ext             v1.16b, v1.16b, v1.16b, #12
  98         // x2 = shuffle32(x2, MASK(1, 0, 3, 2))
  99         ext             v2.16b, v2.16b, v2.16b, #8
 100         // x3 = shuffle32(x3, MASK(0, 3, 2, 1))
 101         ext             v3.16b, v3.16b, v3.16b, #4
 102
 103         subs            w3, w3, #2
 104         b.ne            .Ldoubleround
 105
 106         ret
 107 SYM_FUNC_END(chacha_permute)
 108
 109 SYM_FUNC_START(chacha_block_xor_neon)
 110         // x0: Input state matrix, s
 111         // x1: 1 data block output, o
 112         // x2: 1 data block input, i
 113         // w3: nrounds
 114
 115         stp             x29, x30, [sp, #-16]!
 116         mov             x29, sp
 117
 118         // x0..3 = s0..3
 119         ld1             {v0.4s-v3.4s}, [x0]
 120         ld1             {v8.4s-v11.4s}, [x0]
 121
 122         bl              chacha_permute
 123
 124         ld1             {v4.16b-v7.16b}, [x2]
 125
 126         // o0 = i0 ^ (x0 + s0)
 127         add             v0.4s, v0.4s, v8.4s
 128         eor             v0.16b, v0.16b, v4.16b
 129
 130         // o1 = i1 ^ (x1 + s1)
 131         add             v1.4s, v1.4s, v9.4s
 132         eor             v1.16b, v1.16b, v5.16b
 133
 134         // o2 = i2 ^ (x2 + s2)
 135         add             v2.4s, v2.4s, v10.4s
 136         eor             v2.16b, v2.16b, v6.16b
 137
 138         // o3 = i3 ^ (x3 + s3)
 139         add             v3.4s, v3.4s, v11.4s
 140         eor             v3.16b, v3.16b, v7.16b
 141
 142         st1             {v0.16b-v3.16b}, [x1]
 143
 144         ldp             x29, x30, [sp], #16
 145         ret
 146 SYM_FUNC_END(chacha_block_xor_neon)
 147
 148 SYM_FUNC_START(hchacha_block_neon)
 149         // x0: Input state matrix, s
 150         // x1: output (8 32-bit words)
 151         // w2: nrounds
 152
 153         stp             x29, x30, [sp, #-16]!
 154         mov             x29, sp
 155
 156         ld1             {v0.4s-v3.4s}, [x0]
 157
 158         mov             w3, w2
 159         bl              chacha_permute
 160
 161         st1             {v0.4s}, [x1], #16
 162         st1             {v3.4s}, [x1]
 163
 164         ldp             x29, x30, [sp], #16
 165         ret
 166 SYM_FUNC_END(hchacha_block_neon)
 167
 168         a0              .req    w12
 169         a1              .req    w13
 170         a2              .req    w14
 171         a3              .req    w15
 172         a4              .req    w16
 173         a5              .req    w17
 174         a6              .req    w19
 175         a7              .req    w20
 176         a8              .req    w21
 177         a9              .req    w22
 178         a10             .req    w23
 179         a11             .req    w24
 180         a12             .req    w25
 181         a13             .req    w26
 182         a14             .req    w27
 183         a15             .req    w28
 184
 185         .align          6
 186 SYM_FUNC_START(chacha_4block_xor_neon)
 187         frame_push      10
 188
 189         // x0: Input state matrix, s
 190         // x1: 4 data blocks output, o
 191         // x2: 4 data blocks input, i
 192         // w3: nrounds
 193         // x4: byte count
 194
 195         adr_l           x10, .Lpermute
 196         and             x5, x4, #63
 197         add             x10, x10, x5
 198
 199         //
 200         // This function encrypts four consecutive ChaCha blocks by loading
 201         // the state matrix in NEON registers four times. The algorithm performs
 202         // each operation on the corresponding word of each state matrix, hence
 203         // requires no word shuffling. For final XORing step we transpose the
 204         // matrix by interleaving 32- and then 64-bit words, which allows us to
 205         // do XOR in NEON registers.
 206         //
 207         // At the same time, a fifth block is encrypted in parallel using
 208         // scalar registers
 209         //
 210         adr_l           x9, CTRINC              // ... and ROT8
 211         ld1             {v30.4s-v31.4s}, [x9]
 212
 213         // x0..15[0-3] = s0..3[0..3]
 214         add             x8, x0, #16
 215         ld4r            { v0.4s- v3.4s}, [x0]
 216         ld4r            { v4.4s- v7.4s}, [x8], #16
 217         ld4r            { v8.4s-v11.4s}, [x8], #16
 218         ld4r            {v12.4s-v15.4s}, [x8]
 219
 220         mov             a0, v0.s[0]
 221         mov             a1, v1.s[0]
 222         mov             a2, v2.s[0]
 223         mov             a3, v3.s[0]
 224         mov             a4, v4.s[0]
 225         mov             a5, v5.s[0]
 226         mov             a6, v6.s[0]
 227         mov             a7, v7.s[0]
 228         mov             a8, v8.s[0]
 229         mov             a9, v9.s[0]
 230         mov             a10, v10.s[0]
 231         mov             a11, v11.s[0]
 232         mov             a12, v12.s[0]
 233         mov             a13, v13.s[0]
 234         mov             a14, v14.s[0]
 235         mov             a15, v15.s[0]
 236
 237         // x12 += counter values 1-4
 238         add             v12.4s, v12.4s, v30.4s
 239
 240 .Ldoubleround4:
 241         // x0 += x4, x12 = rotl32(x12 ^ x0, 16)
 242         // x1 += x5, x13 = rotl32(x13 ^ x1, 16)
 243         // x2 += x6, x14 = rotl32(x14 ^ x2, 16)
 244         // x3 += x7, x15 = rotl32(x15 ^ x3, 16)
 245         add             v0.4s, v0.4s, v4.4s
 246           add           a0, a0, a4
 247         add             v1.4s, v1.4s, v5.4s
 248           add           a1, a1, a5
 249         add             v2.4s, v2.4s, v6.4s
 250           add           a2, a2, a6
 251         add             v3.4s, v3.4s, v7.4s
 252           add           a3, a3, a7
 253
 254         eor             v12.16b, v12.16b, v0.16b
 255           eor           a12, a12, a0
 256         eor             v13.16b, v13.16b, v1.16b
 257           eor           a13, a13, a1
 258         eor             v14.16b, v14.16b, v2.16b
 259           eor           a14, a14, a2
 260         eor             v15.16b, v15.16b, v3.16b
 261           eor           a15, a15, a3
 262
 263         rev32           v12.8h, v12.8h
 264           ror           a12, a12, #16
 265         rev32           v13.8h, v13.8h
 266           ror           a13, a13, #16
 267         rev32           v14.8h, v14.8h
 268           ror           a14, a14, #16
 269         rev32           v15.8h, v15.8h
 270           ror           a15, a15, #16
 271
 272         // x8 += x12, x4 = rotl32(x4 ^ x8, 12)
 273         // x9 += x13, x5 = rotl32(x5 ^ x9, 12)
 274         // x10 += x14, x6 = rotl32(x6 ^ x10, 12)
 275         // x11 += x15, x7 = rotl32(x7 ^ x11, 12)
 276         add             v8.4s, v8.4s, v12.4s
 277           add           a8, a8, a12
 278         add             v9.4s, v9.4s, v13.4s
 279           add           a9, a9, a13
 280         add             v10.4s, v10.4s, v14.4s
 281           add           a10, a10, a14
 282         add             v11.4s, v11.4s, v15.4s
 283           add           a11, a11, a15
 284
 285         eor             v16.16b, v4.16b, v8.16b
 286           eor           a4, a4, a8
 287         eor             v17.16b, v5.16b, v9.16b
 288           eor           a5, a5, a9
 289         eor             v18.16b, v6.16b, v10.16b
 290           eor           a6, a6, a10
 291         eor             v19.16b, v7.16b, v11.16b
 292           eor           a7, a7, a11
 293
 294         shl             v4.4s, v16.4s, #12
 295         shl             v5.4s, v17.4s, #12
 296         shl             v6.4s, v18.4s, #12
 297         shl             v7.4s, v19.4s, #12
 298
 299         sri             v4.4s, v16.4s, #20
 300           ror           a4, a4, #20
 301         sri             v5.4s, v17.4s, #20
 302           ror           a5, a5, #20
 303         sri             v6.4s, v18.4s, #20
 304           ror           a6, a6, #20
 305         sri             v7.4s, v19.4s, #20
 306           ror           a7, a7, #20
 307
 308         // x0 += x4, x12 = rotl32(x12 ^ x0, 8)
 309         // x1 += x5, x13 = rotl32(x13 ^ x1, 8)
 310         // x2 += x6, x14 = rotl32(x14 ^ x2, 8)
 311         // x3 += x7, x15 = rotl32(x15 ^ x3, 8)
 312         add             v0.4s, v0.4s, v4.4s
 313           add           a0, a0, a4
 314         add             v1.4s, v1.4s, v5.4s
 315           add           a1, a1, a5
 316         add             v2.4s, v2.4s, v6.4s
 317           add           a2, a2, a6
 318         add             v3.4s, v3.4s, v7.4s
 319           add           a3, a3, a7
 320
 321         eor             v12.16b, v12.16b, v0.16b
 322           eor           a12, a12, a0
 323         eor             v13.16b, v13.16b, v1.16b
 324           eor           a13, a13, a1
 325         eor             v14.16b, v14.16b, v2.16b
 326           eor           a14, a14, a2
 327         eor             v15.16b, v15.16b, v3.16b
 328           eor           a15, a15, a3
 329
 330         tbl             v12.16b, {v12.16b}, v31.16b
 331           ror           a12, a12, #24
 332         tbl             v13.16b, {v13.16b}, v31.16b
 333           ror           a13, a13, #24
 334         tbl             v14.16b, {v14.16b}, v31.16b
 335           ror           a14, a14, #24
 336         tbl             v15.16b, {v15.16b}, v31.16b
 337           ror           a15, a15, #24
 338
 339         // x8 += x12, x4 = rotl32(x4 ^ x8, 7)
 340         // x9 += x13, x5 = rotl32(x5 ^ x9, 7)
 341         // x10 += x14, x6 = rotl32(x6 ^ x10, 7)
 342         // x11 += x15, x7 = rotl32(x7 ^ x11, 7)
 343         add             v8.4s, v8.4s, v12.4s
 344           add           a8, a8, a12
 345         add             v9.4s, v9.4s, v13.4s
 346           add           a9, a9, a13
 347         add             v10.4s, v10.4s, v14.4s
 348           add           a10, a10, a14
 349         add             v11.4s, v11.4s, v15.4s
 350           add           a11, a11, a15
 351
 352         eor             v16.16b, v4.16b, v8.16b
 353           eor           a4, a4, a8
 354         eor             v17.16b, v5.16b, v9.16b
 355           eor           a5, a5, a9
 356         eor             v18.16b, v6.16b, v10.16b
 357           eor           a6, a6, a10
 358         eor             v19.16b, v7.16b, v11.16b
 359           eor           a7, a7, a11
 360
 361         shl             v4.4s, v16.4s, #7
 362         shl             v5.4s, v17.4s, #7
 363         shl             v6.4s, v18.4s, #7
 364         shl             v7.4s, v19.4s, #7
 365
 366         sri             v4.4s, v16.4s, #25
 367           ror           a4, a4, #25
 368         sri             v5.4s, v17.4s, #25
 369           ror           a5, a5, #25
 370         sri             v6.4s, v18.4s, #25
 371          ror            a6, a6, #25
 372         sri             v7.4s, v19.4s, #25
 373           ror           a7, a7, #25
 374
 375         // x0 += x5, x15 = rotl32(x15 ^ x0, 16)
 376         // x1 += x6, x12 = rotl32(x12 ^ x1, 16)
 377         // x2 += x7, x13 = rotl32(x13 ^ x2, 16)
 378         // x3 += x4, x14 = rotl32(x14 ^ x3, 16)
 379         add             v0.4s, v0.4s, v5.4s
 380           add           a0, a0, a5
 381         add             v1.4s, v1.4s, v6.4s
 382           add           a1, a1, a6
 383         add             v2.4s, v2.4s, v7.4s
 384           add           a2, a2, a7
 385         add             v3.4s, v3.4s, v4.4s
 386           add           a3, a3, a4
 387
 388         eor             v15.16b, v15.16b, v0.16b
 389           eor           a15, a15, a0
 390         eor             v12.16b, v12.16b, v1.16b
 391           eor           a12, a12, a1
 392         eor             v13.16b, v13.16b, v2.16b
 393           eor           a13, a13, a2
 394         eor             v14.16b, v14.16b, v3.16b
 395           eor           a14, a14, a3
 396
 397         rev32           v15.8h, v15.8h
 398           ror           a15, a15, #16
 399         rev32           v12.8h, v12.8h
 400           ror           a12, a12, #16
 401         rev32           v13.8h, v13.8h
 402           ror           a13, a13, #16
 403         rev32           v14.8h, v14.8h
 404           ror           a14, a14, #16
 405
 406         // x10 += x15, x5 = rotl32(x5 ^ x10, 12)
 407         // x11 += x12, x6 = rotl32(x6 ^ x11, 12)
 408         // x8 += x13, x7 = rotl32(x7 ^ x8, 12)
 409         // x9 += x14, x4 = rotl32(x4 ^ x9, 12)
 410         add             v10.4s, v10.4s, v15.4s
 411           add           a10, a10, a15
 412         add             v11.4s, v11.4s, v12.4s
 413           add           a11, a11, a12
 414         add             v8.4s, v8.4s, v13.4s
 415           add           a8, a8, a13
 416         add             v9.4s, v9.4s, v14.4s
 417           add           a9, a9, a14
 418
 419         eor             v16.16b, v5.16b, v10.16b
 420           eor           a5, a5, a10
 421         eor             v17.16b, v6.16b, v11.16b
 422           eor           a6, a6, a11
 423         eor             v18.16b, v7.16b, v8.16b
 424           eor           a7, a7, a8
 425         eor             v19.16b, v4.16b, v9.16b
 426           eor           a4, a4, a9
 427
 428         shl             v5.4s, v16.4s, #12
 429         shl             v6.4s, v17.4s, #12
 430         shl             v7.4s, v18.4s, #12
 431         shl             v4.4s, v19.4s, #12
 432
 433         sri             v5.4s, v16.4s, #20
 434           ror           a5, a5, #20
 435         sri             v6.4s, v17.4s, #20
 436           ror           a6, a6, #20
 437         sri             v7.4s, v18.4s, #20
 438           ror           a7, a7, #20
 439         sri             v4.4s, v19.4s, #20
 440           ror           a4, a4, #20
 441
 442         // x0 += x5, x15 = rotl32(x15 ^ x0, 8)
 443         // x1 += x6, x12 = rotl32(x12 ^ x1, 8)
 444         // x2 += x7, x13 = rotl32(x13 ^ x2, 8)
 445         // x3 += x4, x14 = rotl32(x14 ^ x3, 8)
 446         add             v0.4s, v0.4s, v5.4s
 447           add           a0, a0, a5
 448         add             v1.4s, v1.4s, v6.4s
 449           add           a1, a1, a6
 450         add             v2.4s, v2.4s, v7.4s
 451           add           a2, a2, a7
 452         add             v3.4s, v3.4s, v4.4s
 453           add           a3, a3, a4
 454
 455         eor             v15.16b, v15.16b, v0.16b
 456           eor           a15, a15, a0
 457         eor             v12.16b, v12.16b, v1.16b
 458           eor           a12, a12, a1
 459         eor             v13.16b, v13.16b, v2.16b
 460           eor           a13, a13, a2
 461         eor             v14.16b, v14.16b, v3.16b
 462           eor           a14, a14, a3
 463
 464         tbl             v15.16b, {v15.16b}, v31.16b
 465           ror           a15, a15, #24
 466         tbl             v12.16b, {v12.16b}, v31.16b
 467           ror           a12, a12, #24
 468         tbl             v13.16b, {v13.16b}, v31.16b
 469           ror           a13, a13, #24
 470         tbl             v14.16b, {v14.16b}, v31.16b
 471           ror           a14, a14, #24
 472
 473         // x10 += x15, x5 = rotl32(x5 ^ x10, 7)
 474         // x11 += x12, x6 = rotl32(x6 ^ x11, 7)
 475         // x8 += x13, x7 = rotl32(x7 ^ x8, 7)
 476         // x9 += x14, x4 = rotl32(x4 ^ x9, 7)
 477         add             v10.4s, v10.4s, v15.4s
 478           add           a10, a10, a15
 479         add             v11.4s, v11.4s, v12.4s
 480           add           a11, a11, a12
 481         add             v8.4s, v8.4s, v13.4s
 482           add           a8, a8, a13
 483         add             v9.4s, v9.4s, v14.4s
 484           add           a9, a9, a14
 485
 486         eor             v16.16b, v5.16b, v10.16b
 487           eor           a5, a5, a10
 488         eor             v17.16b, v6.16b, v11.16b
 489           eor           a6, a6, a11
 490         eor             v18.16b, v7.16b, v8.16b
 491           eor           a7, a7, a8
 492         eor             v19.16b, v4.16b, v9.16b
 493           eor           a4, a4, a9
 494
 495         shl             v5.4s, v16.4s, #7
 496         shl             v6.4s, v17.4s, #7
 497         shl             v7.4s, v18.4s, #7
 498         shl             v4.4s, v19.4s, #7
 499
 500         sri             v5.4s, v16.4s, #25
 501           ror           a5, a5, #25
 502         sri             v6.4s, v17.4s, #25
 503           ror           a6, a6, #25
 504         sri             v7.4s, v18.4s, #25
 505           ror           a7, a7, #25
 506         sri             v4.4s, v19.4s, #25
 507           ror           a4, a4, #25
 508
 509         subs            w3, w3, #2
 510         b.ne            .Ldoubleround4
 511
 512         ld4r            {v16.4s-v19.4s}, [x0], #16
 513         ld4r            {v20.4s-v23.4s}, [x0], #16
 514
 515         // x12 += counter values 0-3
 516         add             v12.4s, v12.4s, v30.4s
 517
 518         // x0[0-3] += s0[0]
 519         // x1[0-3] += s0[1]
 520         // x2[0-3] += s0[2]
 521         // x3[0-3] += s0[3]
 522         add             v0.4s, v0.4s, v16.4s
 523           mov           w6, v16.s[0]
 524           mov           w7, v17.s[0]
 525         add             v1.4s, v1.4s, v17.4s
 526           mov           w8, v18.s[0]
 527           mov           w9, v19.s[0]
 528         add             v2.4s, v2.4s, v18.4s
 529           add           a0, a0, w6
 530           add           a1, a1, w7
 531         add             v3.4s, v3.4s, v19.4s
 532           add           a2, a2, w8
 533           add           a3, a3, w9
 534 CPU_BE(   rev           a0, a0          )
 535 CPU_BE(   rev           a1, a1          )
 536 CPU_BE(   rev           a2, a2          )
 537 CPU_BE(   rev           a3, a3          )
 538
 539         ld4r            {v24.4s-v27.4s}, [x0], #16
 540         ld4r            {v28.4s-v31.4s}, [x0]
 541
 542         // x4[0-3] += s1[0]
 543         // x5[0-3] += s1[1]
 544         // x6[0-3] += s1[2]
 545         // x7[0-3] += s1[3]
 546         add             v4.4s, v4.4s, v20.4s
 547           mov           w6, v20.s[0]
 548           mov           w7, v21.s[0]
 549         add             v5.4s, v5.4s, v21.4s
 550           mov           w8, v22.s[0]
 551           mov           w9, v23.s[0]
 552         add             v6.4s, v6.4s, v22.4s
 553           add           a4, a4, w6
 554           add           a5, a5, w7
 555         add             v7.4s, v7.4s, v23.4s
 556           add           a6, a6, w8
 557           add           a7, a7, w9
 558 CPU_BE(   rev           a4, a4          )
 559 CPU_BE(   rev           a5, a5          )
 560 CPU_BE(   rev           a6, a6          )
 561 CPU_BE(   rev           a7, a7          )
 562
 563         // x8[0-3] += s2[0]
 564         // x9[0-3] += s2[1]
 565         // x10[0-3] += s2[2]
 566         // x11[0-3] += s2[3]
 567         add             v8.4s, v8.4s, v24.4s
 568           mov           w6, v24.s[0]
 569           mov           w7, v25.s[0]
 570         add             v9.4s, v9.4s, v25.4s
 571           mov           w8, v26.s[0]
 572           mov           w9, v27.s[0]
 573         add             v10.4s, v10.4s, v26.4s
 574           add           a8, a8, w6
 575           add           a9, a9, w7
 576         add             v11.4s, v11.4s, v27.4s
 577           add           a10, a10, w8
 578           add           a11, a11, w9
 579 CPU_BE(   rev           a8, a8          )
 580 CPU_BE(   rev           a9, a9          )
 581 CPU_BE(   rev           a10, a10        )
 582 CPU_BE(   rev           a11, a11        )
 583
 584         // x12[0-3] += s3[0]
 585         // x13[0-3] += s3[1]
 586         // x14[0-3] += s3[2]
 587         // x15[0-3] += s3[3]
 588         add             v12.4s, v12.4s, v28.4s
 589           mov           w6, v28.s[0]
 590           mov           w7, v29.s[0]
 591         add             v13.4s, v13.4s, v29.4s
 592           mov           w8, v30.s[0]
 593           mov           w9, v31.s[0]
 594         add             v14.4s, v14.4s, v30.4s
 595           add           a12, a12, w6
 596           add           a13, a13, w7
 597         add             v15.4s, v15.4s, v31.4s
 598           add           a14, a14, w8
 599           add           a15, a15, w9
 600 CPU_BE(   rev           a12, a12        )
 601 CPU_BE(   rev           a13, a13        )
 602 CPU_BE(   rev           a14, a14        )
 603 CPU_BE(   rev           a15, a15        )
 604
 605         // interleave 32-bit words in state n, n+1
 606           ldp           w6, w7, [x2], #64
 607         zip1            v16.4s, v0.4s, v1.4s
 608           ldp           w8, w9, [x2, #-56]
 609           eor           a0, a0, w6
 610         zip2            v17.4s, v0.4s, v1.4s
 611           eor           a1, a1, w7
 612         zip1            v18.4s, v2.4s, v3.4s
 613           eor           a2, a2, w8
 614         zip2            v19.4s, v2.4s, v3.4s
 615           eor           a3, a3, w9
 616           ldp           w6, w7, [x2, #-48]
 617         zip1            v20.4s, v4.4s, v5.4s
 618           ldp           w8, w9, [x2, #-40]
 619           eor           a4, a4, w6
 620         zip2            v21.4s, v4.4s, v5.4s
 621           eor           a5, a5, w7
 622         zip1            v22.4s, v6.4s, v7.4s
 623           eor           a6, a6, w8
 624         zip2            v23.4s, v6.4s, v7.4s
 625           eor           a7, a7, w9
 626           ldp           w6, w7, [x2, #-32]
 627         zip1            v24.4s, v8.4s, v9.4s
 628           ldp           w8, w9, [x2, #-24]
 629           eor           a8, a8, w6
 630         zip2            v25.4s, v8.4s, v9.4s
 631           eor           a9, a9, w7
 632         zip1            v26.4s, v10.4s, v11.4s
 633           eor           a10, a10, w8
 634         zip2            v27.4s, v10.4s, v11.4s
 635           eor           a11, a11, w9
 636           ldp           w6, w7, [x2, #-16]
 637         zip1            v28.4s, v12.4s, v13.4s
 638           ldp           w8, w9, [x2, #-8]
 639           eor           a12, a12, w6
 640         zip2            v29.4s, v12.4s, v13.4s
 641           eor           a13, a13, w7
 642         zip1            v30.4s, v14.4s, v15.4s
 643           eor           a14, a14, w8
 644         zip2            v31.4s, v14.4s, v15.4s
 645           eor           a15, a15, w9
 646
 647         add             x3, x2, x4
 648         sub             x3, x3, #128            // start of last block
 649
 650         subs            x5, x4, #128
 651         csel            x2, x2, x3, ge
 652
 653         // interleave 64-bit words in state n, n+2
 654         zip1            v0.2d, v16.2d, v18.2d
 655         zip2            v4.2d, v16.2d, v18.2d
 656           stp           a0, a1, [x1], #64
 657         zip1            v8.2d, v17.2d, v19.2d
 658         zip2            v12.2d, v17.2d, v19.2d
 659           stp           a2, a3, [x1, #-56]
 660
 661         subs            x6, x4, #192
 662         ld1             {v16.16b-v19.16b}, [x2], #64
 663         csel            x2, x2, x3, ge
 664
 665         zip1            v1.2d, v20.2d, v22.2d
 666         zip2            v5.2d, v20.2d, v22.2d
 667           stp           a4, a5, [x1, #-48]
 668         zip1            v9.2d, v21.2d, v23.2d
 669         zip2            v13.2d, v21.2d, v23.2d
 670           stp           a6, a7, [x1, #-40]
 671
 672         subs            x7, x4, #256
 673         ld1             {v20.16b-v23.16b}, [x2], #64
 674         csel            x2, x2, x3, ge
 675
 676         zip1            v2.2d, v24.2d, v26.2d
 677         zip2            v6.2d, v24.2d, v26.2d
 678           stp           a8, a9, [x1, #-32]
 679         zip1            v10.2d, v25.2d, v27.2d
 680         zip2            v14.2d, v25.2d, v27.2d
 681           stp           a10, a11, [x1, #-24]
 682
 683         subs            x8, x4, #320
 684         ld1             {v24.16b-v27.16b}, [x2], #64
 685         csel            x2, x2, x3, ge
 686
 687         zip1            v3.2d, v28.2d, v30.2d
 688         zip2            v7.2d, v28.2d, v30.2d
 689           stp           a12, a13, [x1, #-16]
 690         zip1            v11.2d, v29.2d, v31.2d
 691         zip2            v15.2d, v29.2d, v31.2d
 692           stp           a14, a15, [x1, #-8]
 693
 694         tbnz            x5, #63, .Lt128
 695         ld1             {v28.16b-v31.16b}, [x2]
 696
 697         // xor with corresponding input, write to output
 698         eor             v16.16b, v16.16b, v0.16b
 699         eor             v17.16b, v17.16b, v1.16b
 700         eor             v18.16b, v18.16b, v2.16b
 701         eor             v19.16b, v19.16b, v3.16b
 702
 703         tbnz            x6, #63, .Lt192
 704
 705         eor             v20.16b, v20.16b, v4.16b
 706         eor             v21.16b, v21.16b, v5.16b
 707         eor             v22.16b, v22.16b, v6.16b
 708         eor             v23.16b, v23.16b, v7.16b
 709
 710         st1             {v16.16b-v19.16b}, [x1], #64
 711         tbnz            x7, #63, .Lt256
 712
 713         eor             v24.16b, v24.16b, v8.16b
 714         eor             v25.16b, v25.16b, v9.16b
 715         eor             v26.16b, v26.16b, v10.16b
 716         eor             v27.16b, v27.16b, v11.16b
 717
 718         st1             {v20.16b-v23.16b}, [x1], #64
 719         tbnz            x8, #63, .Lt320
 720
 721         eor             v28.16b, v28.16b, v12.16b
 722         eor             v29.16b, v29.16b, v13.16b
 723         eor             v30.16b, v30.16b, v14.16b
 724         eor             v31.16b, v31.16b, v15.16b
 725
 726         st1             {v24.16b-v27.16b}, [x1], #64
 727         st1             {v28.16b-v31.16b}, [x1]
 728
 729 .Lout:  frame_pop
 730         ret
 731
 732         // fewer than 192 bytes of in/output
 733 .Lt192: cbz             x5, 1f                          // exactly 128 bytes?
 734         ld1             {v28.16b-v31.16b}, [x10]
 735         add             x5, x5, x1
 736         tbl             v28.16b, {v4.16b-v7.16b}, v28.16b
 737         tbl             v29.16b, {v4.16b-v7.16b}, v29.16b
 738         tbl             v30.16b, {v4.16b-v7.16b}, v30.16b
 739         tbl             v31.16b, {v4.16b-v7.16b}, v31.16b
 740
 741 0:      eor             v20.16b, v20.16b, v28.16b
 742         eor             v21.16b, v21.16b, v29.16b
 743         eor             v22.16b, v22.16b, v30.16b
 744         eor             v23.16b, v23.16b, v31.16b
 745         st1             {v20.16b-v23.16b}, [x5]         // overlapping stores
 746 1:      st1             {v16.16b-v19.16b}, [x1]
 747         b               .Lout
 748
 749         // fewer than 128 bytes of in/output
 750 .Lt128: ld1             {v28.16b-v31.16b}, [x10]
 751         add             x5, x5, x1
 752         sub             x1, x1, #64
 753         tbl             v28.16b, {v0.16b-v3.16b}, v28.16b
 754         tbl             v29.16b, {v0.16b-v3.16b}, v29.16b
 755         tbl             v30.16b, {v0.16b-v3.16b}, v30.16b
 756         tbl             v31.16b, {v0.16b-v3.16b}, v31.16b
 757         ld1             {v16.16b-v19.16b}, [x1]         // reload first output block
 758         b               0b
 759
 760         // fewer than 256 bytes of in/output
 761 .Lt256: cbz             x6, 2f                          // exactly 192 bytes?
 762         ld1             {v4.16b-v7.16b}, [x10]
 763         add             x6, x6, x1
 764         tbl             v0.16b, {v8.16b-v11.16b}, v4.16b
 765         tbl             v1.16b, {v8.16b-v11.16b}, v5.16b
 766         tbl             v2.16b, {v8.16b-v11.16b}, v6.16b
 767         tbl             v3.16b, {v8.16b-v11.16b}, v7.16b
 768
 769         eor             v28.16b, v28.16b, v0.16b
 770         eor             v29.16b, v29.16b, v1.16b
 771         eor             v30.16b, v30.16b, v2.16b
 772         eor             v31.16b, v31.16b, v3.16b
 773         st1             {v28.16b-v31.16b}, [x6]         // overlapping stores
 774 2:      st1             {v20.16b-v23.16b}, [x1]
 775         b               .Lout
 776
 777         // fewer than 320 bytes of in/output
 778 .Lt320: cbz             x7, 3f                          // exactly 256 bytes?
 779         ld1             {v4.16b-v7.16b}, [x10]
 780         add             x7, x7, x1
 781         tbl             v0.16b, {v12.16b-v15.16b}, v4.16b
 782         tbl             v1.16b, {v12.16b-v15.16b}, v5.16b
 783         tbl             v2.16b, {v12.16b-v15.16b}, v6.16b
 784         tbl             v3.16b, {v12.16b-v15.16b}, v7.16b
 785
 786         eor             v28.16b, v28.16b, v0.16b
 787         eor             v29.16b, v29.16b, v1.16b
 788         eor             v30.16b, v30.16b, v2.16b
 789         eor             v31.16b, v31.16b, v3.16b
 790         st1             {v28.16b-v31.16b}, [x7]         // overlapping stores
 791 3:      st1             {v24.16b-v27.16b}, [x1]
 792         b               .Lout
 793 SYM_FUNC_END(chacha_4block_xor_neon)
 794
 795         .section        ".rodata", "a", %progbits
 796         .align          L1_CACHE_SHIFT
 797 .Lpermute:
 798         .set            .Li, 0
 799         .rept           128
 800         .byte           (.Li - 64)
 801         .set            .Li, .Li + 1
 802         .endr
 803
 804 CTRINC: .word           1, 2, 3, 4
 805 ROT8:   .word           0x02010003, 0x06050407, 0x0a09080b, 0x0e0d0c0f