arch/powerpc/include/asm/uaccess.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _ARCH_POWERPC_UACCESS_H
   3 #define _ARCH_POWERPC_UACCESS_H
   4
   5 #include <asm/ppc_asm.h>
   6 #include <asm/processor.h>
   7 #include <asm/page.h>
   8 #include <asm/extable.h>
   9 #include <asm/kup.h>
  10
  11 #ifdef __powerpc64__
  12 /* We use TASK_SIZE_USER64 as TASK_SIZE is not constant */
  13 #define TASK_SIZE_MAX           TASK_SIZE_USER64
  14 #else
  15 #define TASK_SIZE_MAX           TASK_SIZE
  16 #endif
  17
  18 static inline bool __access_ok(unsigned long addr, unsigned long size)
  19 {
  20         return addr < TASK_SIZE_MAX && size <= TASK_SIZE_MAX - addr;
  21 }
  22
  23 #define access_ok(addr, size)           \
  24         (__chk_user_ptr(addr),          \
  25          __access_ok((unsigned long)(addr), (size)))
  26
  27 /*
  28  * These are the main single-value transfer routines.  They automatically
  29  * use the right size if we just have the right pointer type.
  30  *
  31  * This gets kind of ugly. We want to return _two_ values in "get_user()"
  32  * and yet we don't want to do any pointers, because that is too much
  33  * of a performance impact. Thus we have a few rather ugly macros here,
  34  * and hide all the ugliness from the user.
  35  *
  36  * The "__xxx" versions of the user access functions are versions that
  37  * do not verify the address space, that must have been done previously
  38  * with a separate "access_ok()" call (this is used when we do multiple
  39  * accesses to the same area of user memory).
  40  *
  41  * As we use the same address space for kernel and user data on the
  42  * PowerPC, we can just do these as direct assignments.  (Of course, the
  43  * exception handling means that it's no longer "just"...)
  44  *
  45  */
  46 #define get_user(x, ptr) \
  47         __get_user_check((x), (ptr), sizeof(*(ptr)))
  48 #define put_user(x, ptr) \
  49         __put_user_check((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
  50
  51 #define __get_user(x, ptr) \
  52         __get_user_nocheck((x), (ptr), sizeof(*(ptr)), true)
  53 #define __put_user(x, ptr) \
  54         __put_user_nocheck((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
  55 #define __put_user_goto(x, ptr, label) \
  56         __put_user_nocheck_goto((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)), label)
  57
  58 #define __get_user_allowed(x, ptr) \
  59         __get_user_nocheck((x), (ptr), sizeof(*(ptr)), false)
  60
  61 #define __get_user_inatomic(x, ptr) \
  62         __get_user_nosleep((x), (ptr), sizeof(*(ptr)))
  63 #define __put_user_inatomic(x, ptr) \
  64         __put_user_nosleep((__typeof__(*(ptr)))(x), (ptr), sizeof(*(ptr)))
  65
  66 #ifdef CONFIG_PPC64
  67
  68 #define ___get_user_instr(gu_op, dest, ptr)                             \
  69 ({                                                                      \
  70         long __gui_ret = 0;                                             \
  71         unsigned long __gui_ptr = (unsigned long)ptr;                   \
  72         struct ppc_inst __gui_inst;                                     \
  73         unsigned int __prefix, __suffix;                                \
  74         __gui_ret = gu_op(__prefix, (unsigned int __user *)__gui_ptr);  \
  75         if (__gui_ret == 0) {                                           \
  76                 if ((__prefix >> 26) == OP_PREFIX) {                    \
  77                         __gui_ret = gu_op(__suffix,                     \
  78                                 (unsigned int __user *)__gui_ptr + 1);  \
  79                         __gui_inst = ppc_inst_prefix(__prefix,          \
  80                                                      __suffix);         \
  81                 } else {                                                \
  82                         __gui_inst = ppc_inst(__prefix);                \
  83                 }                                                       \
  84                 if (__gui_ret == 0)                                     \
  85                         (dest) = __gui_inst;                            \
  86         }                                                               \
  87         __gui_ret;                                                      \
  88 })
  89
  90 #define get_user_instr(x, ptr) \
  91         ___get_user_instr(get_user, x, ptr)
  92
  93 #define __get_user_instr(x, ptr) \
  94         ___get_user_instr(__get_user, x, ptr)
  95
  96 #define __get_user_instr_inatomic(x, ptr) \
  97         ___get_user_instr(__get_user_inatomic, x, ptr)
  98
  99 #else /* !CONFIG_PPC64 */
 100 #define get_user_instr(x, ptr) \
 101         get_user((x).val, (u32 __user *)(ptr))
 102
 103 #define __get_user_instr(x, ptr) \
 104         __get_user_nocheck((x).val, (u32 __user *)(ptr), sizeof(u32), true)
 105
 106 #define __get_user_instr_inatomic(x, ptr) \
 107         __get_user_nosleep((x).val, (u32 __user *)(ptr), sizeof(u32))
 108
 109 #endif /* CONFIG_PPC64 */
 110
 111 extern long __put_user_bad(void);
 112
 113 #define __put_user_size_allowed(x, ptr, size, retval)           \
 114 do {                                                            \
 115         __label__ __pu_failed;                                  \
 116                                                                 \
 117         retval = 0;                                             \
 118         __put_user_size_goto(x, ptr, size, __pu_failed);        \
 119         break;                                                  \
 120                                                                 \
 121 __pu_failed:                                                    \
 122         retval = -EFAULT;                                       \
 123 } while (0)
 124
 125 #define __put_user_size(x, ptr, size, retval)                   \
 126 do {                                                            \
 127         allow_write_to_user(ptr, size);                         \
 128         __put_user_size_allowed(x, ptr, size, retval);          \
 129         prevent_write_to_user(ptr, size);                       \
 130 } while (0)
 131
 132 #define __put_user_nocheck(x, ptr, size)                        \
 133 ({                                                              \
 134         long __pu_err;                                          \
 135         __typeof__(*(ptr)) __user *__pu_addr = (ptr);           \
 136         __typeof__(*(ptr)) __pu_val = (x);                      \
 137         __typeof__(size) __pu_size = (size);                    \
 138                                                                 \
 139         if (!is_kernel_addr((unsigned long)__pu_addr))          \
 140                 might_fault();                                  \
 141         __chk_user_ptr(__pu_addr);                              \
 142         __put_user_size(__pu_val, __pu_addr, __pu_size, __pu_err);      \
 143                                                                 \
 144         __pu_err;                                               \
 145 })
 146
 147 #define __put_user_check(x, ptr, size)                                  \
 148 ({                                                                      \
 149         long __pu_err = -EFAULT;                                        \
 150         __typeof__(*(ptr)) __user *__pu_addr = (ptr);                   \
 151         __typeof__(*(ptr)) __pu_val = (x);                              \
 152         __typeof__(size) __pu_size = (size);                            \
 153                                                                         \
 154         might_fault();                                                  \
 155         if (access_ok(__pu_addr, __pu_size))                            \
 156                 __put_user_size(__pu_val, __pu_addr, __pu_size, __pu_err); \
 157                                                                         \
 158         __pu_err;                                                       \
 159 })
 160
 161 #define __put_user_nosleep(x, ptr, size)                        \
 162 ({                                                              \
 163         long __pu_err;                                          \
 164         __typeof__(*(ptr)) __user *__pu_addr = (ptr);           \
 165         __typeof__(*(ptr)) __pu_val = (x);                      \
 166         __typeof__(size) __pu_size = (size);                    \
 167                                                                 \
 168         __chk_user_ptr(__pu_addr);                              \
 169         __put_user_size(__pu_val, __pu_addr, __pu_size, __pu_err); \
 170                                                                 \
 171         __pu_err;                                               \
 172 })
 173
 174
 175 /*
 176  * We don't tell gcc that we are accessing memory, but this is OK
 177  * because we do not write to any memory gcc knows about, so there
 178  * are no aliasing issues.
 179  */
 180 #define __put_user_asm_goto(x, addr, label, op)                 \
 181         asm volatile goto(                                      \
 182                 "1:     " op "%U1%X1 %0,%1      # put_user\n"   \
 183                 EX_TABLE(1b, %l2)                               \
 184                 :                                               \
 185                 : "r" (x), "m<>" (*addr)                                \
 186                 :                                               \
 187                 : label)
 188
 189 #ifdef __powerpc64__
 190 #define __put_user_asm2_goto(x, ptr, label)                     \
 191         __put_user_asm_goto(x, ptr, label, "std")
 192 #else /* __powerpc64__ */
 193 #define __put_user_asm2_goto(x, addr, label)                    \
 194         asm volatile goto(                                      \
 195                 "1:     stw%X1 %0, %1\n"                        \
 196                 "2:     stw%X1 %L0, %L1\n"                      \
 197                 EX_TABLE(1b, %l2)                               \
 198                 EX_TABLE(2b, %l2)                               \
 199                 :                                               \
 200                 : "r" (x), "m" (*addr)                          \
 201                 :                                               \
 202                 : label)
 203 #endif /* __powerpc64__ */
 204
 205 #define __put_user_size_goto(x, ptr, size, label)               \
 206 do {                                                            \
 207         switch (size) {                                         \
 208         case 1: __put_user_asm_goto(x, ptr, label, "stb"); break;       \
 209         case 2: __put_user_asm_goto(x, ptr, label, "sth"); break;       \
 210         case 4: __put_user_asm_goto(x, ptr, label, "stw"); break;       \
 211         case 8: __put_user_asm2_goto(x, ptr, label); break;     \
 212         default: __put_user_bad();                              \
 213         }                                                       \
 214 } while (0)
 215
 216 #define __put_user_nocheck_goto(x, ptr, size, label)            \
 217 do {                                                            \
 218         __typeof__(*(ptr)) __user *__pu_addr = (ptr);           \
 219         if (!is_kernel_addr((unsigned long)__pu_addr))          \
 220                 might_fault();                                  \
 221         __chk_user_ptr(ptr);                                    \
 222         __put_user_size_goto((x), __pu_addr, (size), label);    \
 223 } while (0)
 224
 225
 226 extern long __get_user_bad(void);
 227
 228 /*
 229  * This does an atomic 128 byte aligned load from userspace.
 230  * Upto caller to do enable_kernel_vmx() before calling!
 231  */
 232 #define __get_user_atomic_128_aligned(kaddr, uaddr, err)                \
 233         __asm__ __volatile__(                           \
 234                 "1:     lvx  0,0,%1     # get user\n"   \
 235                 "       stvx 0,0,%2     # put kernel\n" \
 236                 "2:\n"                                  \
 237                 ".section .fixup,\"ax\"\n"              \
 238                 "3:     li %0,%3\n"                     \
 239                 "       b 2b\n"                         \
 240                 ".previous\n"                           \
 241                 EX_TABLE(1b, 3b)                        \
 242                 : "=r" (err)                    \
 243                 : "b" (uaddr), "b" (kaddr), "i" (-EFAULT), "0" (err))
 244
 245 #define __get_user_asm(x, addr, err, op)                \
 246         __asm__ __volatile__(                           \
 247                 "1:     "op"%U2%X2 %1, %2       # get_user\n"   \
 248                 "2:\n"                                  \
 249                 ".section .fixup,\"ax\"\n"              \
 250                 "3:     li %0,%3\n"                     \
 251                 "       li %1,0\n"                      \
 252                 "       b 2b\n"                         \
 253                 ".previous\n"                           \
 254                 EX_TABLE(1b, 3b)                        \
 255                 : "=r" (err), "=r" (x)                  \
 256                 : "m<>" (*addr), "i" (-EFAULT), "0" (err))
 257
 258 #ifdef __powerpc64__
 259 #define __get_user_asm2(x, addr, err)                   \
 260         __get_user_asm(x, addr, err, "ld")
 261 #else /* __powerpc64__ */
 262 #define __get_user_asm2(x, addr, err)                   \
 263         __asm__ __volatile__(                           \
 264                 "1:     lwz%X2 %1, %2\n"                        \
 265                 "2:     lwz%X2 %L1, %L2\n"              \
 266                 "3:\n"                                  \
 267                 ".section .fixup,\"ax\"\n"              \
 268                 "4:     li %0,%3\n"                     \
 269                 "       li %1,0\n"                      \
 270                 "       li %1+1,0\n"                    \
 271                 "       b 3b\n"                         \
 272                 ".previous\n"                           \
 273                 EX_TABLE(1b, 4b)                        \
 274                 EX_TABLE(2b, 4b)                        \
 275                 : "=r" (err), "=&r" (x)                 \
 276                 : "m" (*addr), "i" (-EFAULT), "0" (err))
 277 #endif /* __powerpc64__ */
 278
 279 #define __get_user_size_allowed(x, ptr, size, retval)           \
 280 do {                                                            \
 281         retval = 0;                                             \
 282         __chk_user_ptr(ptr);                                    \
 283         if (size > sizeof(x))                                   \
 284                 (x) = __get_user_bad();                         \
 285         switch (size) {                                         \
 286         case 1: __get_user_asm(x, (u8 __user *)ptr, retval, "lbz"); break;      \
 287         case 2: __get_user_asm(x, (u16 __user *)ptr, retval, "lhz"); break;     \
 288         case 4: __get_user_asm(x, (u32 __user *)ptr, retval, "lwz"); break;     \
 289         case 8: __get_user_asm2(x, (u64 __user *)ptr, retval);  break;  \
 290         default: (x) = __get_user_bad();                        \
 291         }                                                       \
 292 } while (0)
 293
 294 #define __get_user_size(x, ptr, size, retval)                   \
 295 do {                                                            \
 296         allow_read_from_user(ptr, size);                        \
 297         __get_user_size_allowed(x, ptr, size, retval);          \
 298         prevent_read_from_user(ptr, size);                      \
 299 } while (0)
 300
 301 /*
 302  * This is a type: either unsigned long, if the argument fits into
 303  * that type, or otherwise unsigned long long.
 304  */
 305 #define __long_type(x) \
 306         __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL))
 307
 308 #define __get_user_nocheck(x, ptr, size, do_allow)                      \
 309 ({                                                              \
 310         long __gu_err;                                          \
 311         __long_type(*(ptr)) __gu_val;                           \
 312         __typeof__(*(ptr)) __user *__gu_addr = (ptr);   \
 313         __typeof__(size) __gu_size = (size);                    \
 314                                                                 \
 315         __chk_user_ptr(__gu_addr);                              \
 316         if (!is_kernel_addr((unsigned long)__gu_addr))          \
 317                 might_fault();                                  \
 318         barrier_nospec();                                       \
 319         if (do_allow)                                                           \
 320                 __get_user_size(__gu_val, __gu_addr, __gu_size, __gu_err);      \
 321         else                                                                    \
 322                 __get_user_size_allowed(__gu_val, __gu_addr, __gu_size, __gu_err); \
 323         (x) = (__typeof__(*(ptr)))__gu_val;                     \
 324                                                                 \
 325         __gu_err;                                               \
 326 })
 327
 328 #define __get_user_check(x, ptr, size)                                  \
 329 ({                                                                      \
 330         long __gu_err = -EFAULT;                                        \
 331         __long_type(*(ptr)) __gu_val = 0;                               \
 332         __typeof__(*(ptr)) __user *__gu_addr = (ptr);           \
 333         __typeof__(size) __gu_size = (size);                            \
 334                                                                         \
 335         might_fault();                                                  \
 336         if (access_ok(__gu_addr, __gu_size)) {                          \
 337                 barrier_nospec();                                       \
 338                 __get_user_size(__gu_val, __gu_addr, __gu_size, __gu_err); \
 339         }                                                               \
 340         (x) = (__force __typeof__(*(ptr)))__gu_val;                             \
 341                                                                         \
 342         __gu_err;                                                       \
 343 })
 344
 345 #define __get_user_nosleep(x, ptr, size)                        \
 346 ({                                                              \
 347         long __gu_err;                                          \
 348         __long_type(*(ptr)) __gu_val;                           \
 349         __typeof__(*(ptr)) __user *__gu_addr = (ptr);   \
 350         __typeof__(size) __gu_size = (size);                    \
 351                                                                 \
 352         __chk_user_ptr(__gu_addr);                              \
 353         barrier_nospec();                                       \
 354         __get_user_size(__gu_val, __gu_addr, __gu_size, __gu_err); \
 355         (x) = (__force __typeof__(*(ptr)))__gu_val;                     \
 356                                                                 \
 357         __gu_err;                                               \
 358 })
 359
 360
 361 /* more complex routines */
 362
 363 extern unsigned long __copy_tofrom_user(void __user *to,
 364                 const void __user *from, unsigned long size);
 365
 366 #ifdef CONFIG_ARCH_HAS_COPY_MC
 367 unsigned long __must_check
 368 copy_mc_generic(void *to, const void *from, unsigned long size);
 369
 370 static inline unsigned long __must_check
 371 copy_mc_to_kernel(void *to, const void *from, unsigned long size)
 372 {
 373         return copy_mc_generic(to, from, size);
 374 }
 375 #define copy_mc_to_kernel copy_mc_to_kernel
 376
 377 static inline unsigned long __must_check
 378 copy_mc_to_user(void __user *to, const void *from, unsigned long n)
 379 {
 380         if (likely(check_copy_size(from, n, true))) {
 381                 if (access_ok(to, n)) {
 382                         allow_write_to_user(to, n);
 383                         n = copy_mc_generic((void *)to, from, n);
 384                         prevent_write_to_user(to, n);
 385                 }
 386         }
 387
 388         return n;
 389 }
 390 #endif
 391
 392 #ifdef __powerpc64__
 393 static inline unsigned long
 394 raw_copy_in_user(void __user *to, const void __user *from, unsigned long n)
 395 {
 396         unsigned long ret;
 397
 398         barrier_nospec();
 399         allow_read_write_user(to, from, n);
 400         ret = __copy_tofrom_user(to, from, n);
 401         prevent_read_write_user(to, from, n);
 402         return ret;
 403 }
 404 #endif /* __powerpc64__ */
 405
 406 static inline unsigned long raw_copy_from_user(void *to,
 407                 const void __user *from, unsigned long n)
 408 {
 409         unsigned long ret;
 410         if (__builtin_constant_p(n) && (n <= 8)) {
 411                 ret = 1;
 412
 413                 switch (n) {
 414                 case 1:
 415                         barrier_nospec();
 416                         __get_user_size(*(u8 *)to, from, 1, ret);
 417                         break;
 418                 case 2:
 419                         barrier_nospec();
 420                         __get_user_size(*(u16 *)to, from, 2, ret);
 421                         break;
 422                 case 4:
 423                         barrier_nospec();
 424                         __get_user_size(*(u32 *)to, from, 4, ret);
 425                         break;
 426                 case 8:
 427                         barrier_nospec();
 428                         __get_user_size(*(u64 *)to, from, 8, ret);
 429                         break;
 430                 }
 431                 if (ret == 0)
 432                         return 0;
 433         }
 434
 435         barrier_nospec();
 436         allow_read_from_user(from, n);
 437         ret = __copy_tofrom_user((__force void __user *)to, from, n);
 438         prevent_read_from_user(from, n);
 439         return ret;
 440 }
 441
 442 static inline unsigned long
 443 raw_copy_to_user_allowed(void __user *to, const void *from, unsigned long n)
 444 {
 445         if (__builtin_constant_p(n) && (n <= 8)) {
 446                 unsigned long ret = 1;
 447
 448                 switch (n) {
 449                 case 1:
 450                         __put_user_size_allowed(*(u8 *)from, (u8 __user *)to, 1, ret);
 451                         break;
 452                 case 2:
 453                         __put_user_size_allowed(*(u16 *)from, (u16 __user *)to, 2, ret);
 454                         break;
 455                 case 4:
 456                         __put_user_size_allowed(*(u32 *)from, (u32 __user *)to, 4, ret);
 457                         break;
 458                 case 8:
 459                         __put_user_size_allowed(*(u64 *)from, (u64 __user *)to, 8, ret);
 460                         break;
 461                 }
 462                 if (ret == 0)
 463                         return 0;
 464         }
 465
 466         return __copy_tofrom_user(to, (__force const void __user *)from, n);
 467 }
 468
 469 static inline unsigned long
 470 raw_copy_to_user(void __user *to, const void *from, unsigned long n)
 471 {
 472         unsigned long ret;
 473
 474         allow_write_to_user(to, n);
 475         ret = raw_copy_to_user_allowed(to, from, n);
 476         prevent_write_to_user(to, n);
 477         return ret;
 478 }
 479
 480 unsigned long __arch_clear_user(void __user *addr, unsigned long size);
 481
 482 static inline unsigned long clear_user(void __user *addr, unsigned long size)
 483 {
 484         unsigned long ret = size;
 485         might_fault();
 486         if (likely(access_ok(addr, size))) {
 487                 allow_write_to_user(addr, size);
 488                 ret = __arch_clear_user(addr, size);
 489                 prevent_write_to_user(addr, size);
 490         }
 491         return ret;
 492 }
 493
 494 static inline unsigned long __clear_user(void __user *addr, unsigned long size)
 495 {
 496         return clear_user(addr, size);
 497 }
 498
 499 extern long strncpy_from_user(char *dst, const char __user *src, long count);
 500 extern __must_check long strnlen_user(const char __user *str, long n);
 501
 502 extern long __copy_from_user_flushcache(void *dst, const void __user *src,
 503                 unsigned size);
 504 extern void memcpy_page_flushcache(char *to, struct page *page, size_t offset,
 505                            size_t len);
 506
 507 static __must_check inline bool user_access_begin(const void __user *ptr, size_t len)
 508 {
 509         if (unlikely(!access_ok(ptr, len)))
 510                 return false;
 511         allow_read_write_user((void __user *)ptr, ptr, len);
 512         return true;
 513 }
 514 #define user_access_begin       user_access_begin
 515 #define user_access_end         prevent_current_access_user
 516 #define user_access_save        prevent_user_access_return
 517 #define user_access_restore     restore_user_access
 518
 519 static __must_check inline bool
 520 user_read_access_begin(const void __user *ptr, size_t len)
 521 {
 522         if (unlikely(!access_ok(ptr, len)))
 523                 return false;
 524         allow_read_from_user(ptr, len);
 525         return true;
 526 }
 527 #define user_read_access_begin  user_read_access_begin
 528 #define user_read_access_end            prevent_current_read_from_user
 529
 530 static __must_check inline bool
 531 user_write_access_begin(const void __user *ptr, size_t len)
 532 {
 533         if (unlikely(!access_ok(ptr, len)))
 534                 return false;
 535         allow_write_to_user((void __user *)ptr, len);
 536         return true;
 537 }
 538 #define user_write_access_begin user_write_access_begin
 539 #define user_write_access_end           prevent_current_write_to_user
 540
 541 #define unsafe_op_wrap(op, err) do { if (unlikely(op)) goto err; } while (0)
 542 #define unsafe_get_user(x, p, e) unsafe_op_wrap(__get_user_allowed(x, p), e)
 543 #define unsafe_put_user(x, p, e) __put_user_goto(x, p, e)
 544
 545 #define unsafe_copy_to_user(d, s, l, e) \
 546 do {                                                                    \
 547         u8 __user *_dst = (u8 __user *)(d);                             \
 548         const u8 *_src = (const u8 *)(s);                               \
 549         size_t _len = (l);                                              \
 550         int _i;                                                         \
 551                                                                         \
 552         for (_i = 0; _i < (_len & ~(sizeof(long) - 1)); _i += sizeof(long))             \
 553                 __put_user_goto(*(long*)(_src + _i), (long __user *)(_dst + _i), e);\
 554         if (IS_ENABLED(CONFIG_PPC64) && (_len & 4)) {                   \
 555                 __put_user_goto(*(u32*)(_src + _i), (u32 __user *)(_dst + _i), e);      \
 556                 _i += 4;                                                \
 557         }                                                               \
 558         if (_len & 2) {                                                 \
 559                 __put_user_goto(*(u16*)(_src + _i), (u16 __user *)(_dst + _i), e);      \
 560                 _i += 2;                                                \
 561         }                                                               \
 562         if (_len & 1) \
 563                 __put_user_goto(*(u8*)(_src + _i), (u8 __user *)(_dst + _i), e);\
 564 } while (0)
 565
 566 #define HAVE_GET_KERNEL_NOFAULT
 567
 568 #define __get_kernel_nofault(dst, src, type, err_label)                 \
 569 do {                                                                    \
 570         int __kr_err;                                                   \
 571                                                                         \
 572         __get_user_size_allowed(*((type *)(dst)), (__force type __user *)(src),\
 573                         sizeof(type), __kr_err);                        \
 574         if (unlikely(__kr_err))                                         \
 575                 goto err_label;                                         \
 576 } while (0)
 577
 578 #define __put_kernel_nofault(dst, src, type, err_label)                 \
 579         __put_user_size_goto(*((type *)(src)),                          \
 580                 (__force type __user *)(dst), sizeof(type), err_label)
 581
 582 #endif  /* _ARCH_POWERPC_UACCESS_H */