lib/raid6/neon.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * linux/lib/raid6/neon.c - RAID6 syndrome calculation using ARM NEON intrinsics
   4  *
   5  * Copyright (C) 2013 Linaro Ltd <ard.biesheuvel@linaro.org>
   6  */
   7
   8 #include <linux/raid/pq.h>
   9
  10 #ifdef __KERNEL__
  11 #include <asm/neon.h>
  12 #else
  13 #define kernel_neon_begin()
  14 #define kernel_neon_end()
  15 #define cpu_has_neon()          (1)
  16 #endif
  17
  18 /*
  19  * There are 2 reasons these wrappers are kept in a separate compilation unit
  20  * from the actual implementations in neonN.c (generated from neon.uc by
  21  * unroll.awk):
  22  * - the actual implementations use NEON intrinsics, and the GCC support header
  23  *   (arm_neon.h) is not fully compatible (type wise) with the kernel;
  24  * - the neonN.c files are compiled with -mfpu=neon and optimization enabled,
  25  *   and we have to make sure that we never use *any* NEON/VFP instructions
  26  *   outside a kernel_neon_begin()/kernel_neon_end() pair.
  27  */
  28
  29 #define RAID6_NEON_WRAPPER(_n)                                          \
  30         static void raid6_neon ## _n ## _gen_syndrome(int disks,        \
  31                                         size_t bytes, void **ptrs)      \
  32         {                                                               \
  33                 void raid6_neon ## _n  ## _gen_syndrome_real(int,       \
  34                                                 unsigned long, void**); \
  35                 kernel_neon_begin();                                    \
  36                 raid6_neon ## _n ## _gen_syndrome_real(disks,           \
  37                                         (unsigned long)bytes, ptrs);    \
  38                 kernel_neon_end();                                      \
  39         }                                                               \
  40         static void raid6_neon ## _n ## _xor_syndrome(int disks,        \
  41                                         int start, int stop,            \
  42                                         size_t bytes, void **ptrs)      \
  43         {                                                               \
  44                 void raid6_neon ## _n  ## _xor_syndrome_real(int,       \
  45                                 int, int, unsigned long, void**);       \
  46                 kernel_neon_begin();                                    \
  47                 raid6_neon ## _n ## _xor_syndrome_real(disks,           \
  48                         start, stop, (unsigned long)bytes, ptrs);       \
  49                 kernel_neon_end();                                      \
  50         }                                                               \
  51         struct raid6_calls const raid6_neonx ## _n = {                  \
  52                 raid6_neon ## _n ## _gen_syndrome,                      \
  53                 raid6_neon ## _n ## _xor_syndrome,                      \
  54                 raid6_have_neon,                                        \
  55                 "neonx" #_n,                                            \
  56                 0                                                       \
  57         }
  58
  59 static int raid6_have_neon(void)
  60 {
  61         return cpu_has_neon();
  62 }
  63
  64 RAID6_NEON_WRAPPER(1);
  65 RAID6_NEON_WRAPPER(2);
  66 RAID6_NEON_WRAPPER(4);
  67 RAID6_NEON_WRAPPER(8);