arch/arm64/include/asm/rwonce.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * Copyright (C) 2020 Google LLC.
   4  */
   5 #ifndef __ASM_RWONCE_H
   6 #define __ASM_RWONCE_H
   7
   8 #ifdef CONFIG_LTO
   9
  10 #include <linux/compiler_types.h>
  11 #include <asm/alternative-macros.h>
  12
  13 #ifndef BUILD_VDSO
  14
  15 #ifdef CONFIG_AS_HAS_LDAPR
  16 #define __LOAD_RCPC(sfx, regs...)                                       \
  17         ALTERNATIVE(                                                    \
  18                 "ldar"  #sfx "\t" #regs,                                \
  19                 ".arch_extension rcpc\n"                                \
  20                 "ldapr" #sfx "\t" #regs,                                \
  21         ARM64_HAS_LDAPR)
  22 #else
  23 #define __LOAD_RCPC(sfx, regs...)       "ldar" #sfx "\t" #regs
  24 #endif /* CONFIG_AS_HAS_LDAPR */
  25
  26 /*
  27  * When building with LTO, there is an increased risk of the compiler
  28  * converting an address dependency headed by a READ_ONCE() invocation
  29  * into a control dependency and consequently allowing for harmful
  30  * reordering by the CPU.
  31  *
  32  * Ensure that such transformations are harmless by overriding the generic
  33  * READ_ONCE() definition with one that provides RCpc acquire semantics
  34  * when building with LTO.
  35  */
  36 #define __READ_ONCE(x)                                                  \
  37 ({                                                                      \
  38         typeof(&(x)) __x = &(x);                                        \
  39         int atomic = 1;                                                 \
  40         union { __unqual_scalar_typeof(*__x) __val; char __c[1]; } __u; \
  41         switch (sizeof(x)) {                                            \
  42         case 1:                                                         \
  43                 asm volatile(__LOAD_RCPC(b, %w0, %1)                    \
  44                         : "=r" (*(__u8 *)__u.__c)                       \
  45                         : "Q" (*__x) : "memory");                       \
  46                 break;                                                  \
  47         case 2:                                                         \
  48                 asm volatile(__LOAD_RCPC(h, %w0, %1)                    \
  49                         : "=r" (*(__u16 *)__u.__c)                      \
  50                         : "Q" (*__x) : "memory");                       \
  51                 break;                                                  \
  52         case 4:                                                         \
  53                 asm volatile(__LOAD_RCPC(, %w0, %1)                     \
  54                         : "=r" (*(__u32 *)__u.__c)                      \
  55                         : "Q" (*__x) : "memory");                       \
  56                 break;                                                  \
  57         case 8:                                                         \
  58                 asm volatile(__LOAD_RCPC(, %0, %1)                      \
  59                         : "=r" (*(__u64 *)__u.__c)                      \
  60                         : "Q" (*__x) : "memory");                       \
  61                 break;                                                  \
  62         default:                                                        \
  63                 atomic = 0;                                             \
  64         }                                                               \
  65         atomic ? (typeof(*__x))__u.__val : (*(volatile typeof(__x))__x);\
  66 })
  67
  68 #endif  /* !BUILD_VDSO */
  69 #endif  /* CONFIG_LTO */
  70
  71 #include <asm-generic/rwonce.h>
  72
  73 #endif  /* __ASM_RWONCE_H */