include/linux/atomic/atomic-arch-fallback.h

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 // Generated by scripts/atomic/gen-atomic-fallback.sh
   4 // DO NOT MODIFY THIS FILE DIRECTLY
   5
   6 #ifndef _LINUX_ATOMIC_FALLBACK_H
   7 #define _LINUX_ATOMIC_FALLBACK_H
   8
   9 #include <linux/compiler.h>
  10
  11 #ifndef arch_xchg_relaxed
  12 #define arch_xchg_acquire arch_xchg
  13 #define arch_xchg_release arch_xchg
  14 #define arch_xchg_relaxed arch_xchg
  15 #else /* arch_xchg_relaxed */
  16
  17 #ifndef arch_xchg_acquire
  18 #define arch_xchg_acquire(...) \
  19         __atomic_op_acquire(arch_xchg, __VA_ARGS__)
  20 #endif
  21
  22 #ifndef arch_xchg_release
  23 #define arch_xchg_release(...) \
  24         __atomic_op_release(arch_xchg, __VA_ARGS__)
  25 #endif
  26
  27 #ifndef arch_xchg
  28 #define arch_xchg(...) \
  29         __atomic_op_fence(arch_xchg, __VA_ARGS__)
  30 #endif
  31
  32 #endif /* arch_xchg_relaxed */
  33
  34 #ifndef arch_cmpxchg_relaxed
  35 #define arch_cmpxchg_acquire arch_cmpxchg
  36 #define arch_cmpxchg_release arch_cmpxchg
  37 #define arch_cmpxchg_relaxed arch_cmpxchg
  38 #else /* arch_cmpxchg_relaxed */
  39
  40 #ifndef arch_cmpxchg_acquire
  41 #define arch_cmpxchg_acquire(...) \
  42         __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)
  43 #endif
  44
  45 #ifndef arch_cmpxchg_release
  46 #define arch_cmpxchg_release(...) \
  47         __atomic_op_release(arch_cmpxchg, __VA_ARGS__)
  48 #endif
  49
  50 #ifndef arch_cmpxchg
  51 #define arch_cmpxchg(...) \
  52         __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
  53 #endif
  54
  55 #endif /* arch_cmpxchg_relaxed */
  56
  57 #ifndef arch_cmpxchg64_relaxed
  58 #define arch_cmpxchg64_acquire arch_cmpxchg64
  59 #define arch_cmpxchg64_release arch_cmpxchg64
  60 #define arch_cmpxchg64_relaxed arch_cmpxchg64
  61 #else /* arch_cmpxchg64_relaxed */
  62
  63 #ifndef arch_cmpxchg64_acquire
  64 #define arch_cmpxchg64_acquire(...) \
  65         __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)
  66 #endif
  67
  68 #ifndef arch_cmpxchg64_release
  69 #define arch_cmpxchg64_release(...) \
  70         __atomic_op_release(arch_cmpxchg64, __VA_ARGS__)
  71 #endif
  72
  73 #ifndef arch_cmpxchg64
  74 #define arch_cmpxchg64(...) \
  75         __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
  76 #endif
  77
  78 #endif /* arch_cmpxchg64_relaxed */
  79
  80 #ifndef arch_try_cmpxchg_relaxed
  81 #ifdef arch_try_cmpxchg
  82 #define arch_try_cmpxchg_acquire arch_try_cmpxchg
  83 #define arch_try_cmpxchg_release arch_try_cmpxchg
  84 #define arch_try_cmpxchg_relaxed arch_try_cmpxchg
  85 #endif /* arch_try_cmpxchg */
  86
  87 #ifndef arch_try_cmpxchg
  88 #define arch_try_cmpxchg(_ptr, _oldp, _new) \
  89 ({ \
  90         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
  91         ___r = arch_cmpxchg((_ptr), ___o, (_new)); \
  92         if (unlikely(___r != ___o)) \
  93                 *___op = ___r; \
  94         likely(___r == ___o); \
  95 })
  96 #endif /* arch_try_cmpxchg */
  97
  98 #ifndef arch_try_cmpxchg_acquire
  99 #define arch_try_cmpxchg_acquire(_ptr, _oldp, _new) \
 100 ({ \
 101         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 102         ___r = arch_cmpxchg_acquire((_ptr), ___o, (_new)); \
 103         if (unlikely(___r != ___o)) \
 104                 *___op = ___r; \
 105         likely(___r == ___o); \
 106 })
 107 #endif /* arch_try_cmpxchg_acquire */
 108
 109 #ifndef arch_try_cmpxchg_release
 110 #define arch_try_cmpxchg_release(_ptr, _oldp, _new) \
 111 ({ \
 112         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 113         ___r = arch_cmpxchg_release((_ptr), ___o, (_new)); \
 114         if (unlikely(___r != ___o)) \
 115                 *___op = ___r; \
 116         likely(___r == ___o); \
 117 })
 118 #endif /* arch_try_cmpxchg_release */
 119
 120 #ifndef arch_try_cmpxchg_relaxed
 121 #define arch_try_cmpxchg_relaxed(_ptr, _oldp, _new) \
 122 ({ \
 123         typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
 124         ___r = arch_cmpxchg_relaxed((_ptr), ___o, (_new)); \
 125         if (unlikely(___r != ___o)) \
 126                 *___op = ___r; \
 127         likely(___r == ___o); \
 128 })
 129 #endif /* arch_try_cmpxchg_relaxed */
 130
 131 #else /* arch_try_cmpxchg_relaxed */
 132
 133 #ifndef arch_try_cmpxchg_acquire
 134 #define arch_try_cmpxchg_acquire(...) \
 135         __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
 136 #endif
 137
 138 #ifndef arch_try_cmpxchg_release
 139 #define arch_try_cmpxchg_release(...) \
 140         __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)
 141 #endif
 142
 143 #ifndef arch_try_cmpxchg
 144 #define arch_try_cmpxchg(...) \
 145         __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)
 146 #endif
 147
 148 #endif /* arch_try_cmpxchg_relaxed */
 149
 150 #ifndef arch_atomic_read_acquire
 151 static __always_inline int
 152 arch_atomic_read_acquire(const atomic_t *v)
 153 {
 154         return smp_load_acquire(&(v)->counter);
 155 }
 156 #define arch_atomic_read_acquire arch_atomic_read_acquire
 157 #endif
 158
 159 #ifndef arch_atomic_set_release
 160 static __always_inline void
 161 arch_atomic_set_release(atomic_t *v, int i)
 162 {
 163         smp_store_release(&(v)->counter, i);
 164 }
 165 #define arch_atomic_set_release arch_atomic_set_release
 166 #endif
 167
 168 #ifndef arch_atomic_add_return_relaxed
 169 #define arch_atomic_add_return_acquire arch_atomic_add_return
 170 #define arch_atomic_add_return_release arch_atomic_add_return
 171 #define arch_atomic_add_return_relaxed arch_atomic_add_return
 172 #else /* arch_atomic_add_return_relaxed */
 173
 174 #ifndef arch_atomic_add_return_acquire
 175 static __always_inline int
 176 arch_atomic_add_return_acquire(int i, atomic_t *v)
 177 {
 178         int ret = arch_atomic_add_return_relaxed(i, v);
 179         __atomic_acquire_fence();
 180         return ret;
 181 }
 182 #define arch_atomic_add_return_acquire arch_atomic_add_return_acquire
 183 #endif
 184
 185 #ifndef arch_atomic_add_return_release
 186 static __always_inline int
 187 arch_atomic_add_return_release(int i, atomic_t *v)
 188 {
 189         __atomic_release_fence();
 190         return arch_atomic_add_return_relaxed(i, v);
 191 }
 192 #define arch_atomic_add_return_release arch_atomic_add_return_release
 193 #endif
 194
 195 #ifndef arch_atomic_add_return
 196 static __always_inline int
 197 arch_atomic_add_return(int i, atomic_t *v)
 198 {
 199         int ret;
 200         __atomic_pre_full_fence();
 201         ret = arch_atomic_add_return_relaxed(i, v);
 202         __atomic_post_full_fence();
 203         return ret;
 204 }
 205 #define arch_atomic_add_return arch_atomic_add_return
 206 #endif
 207
 208 #endif /* arch_atomic_add_return_relaxed */
 209
 210 #ifndef arch_atomic_fetch_add_relaxed
 211 #define arch_atomic_fetch_add_acquire arch_atomic_fetch_add
 212 #define arch_atomic_fetch_add_release arch_atomic_fetch_add
 213 #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add
 214 #else /* arch_atomic_fetch_add_relaxed */
 215
 216 #ifndef arch_atomic_fetch_add_acquire
 217 static __always_inline int
 218 arch_atomic_fetch_add_acquire(int i, atomic_t *v)
 219 {
 220         int ret = arch_atomic_fetch_add_relaxed(i, v);
 221         __atomic_acquire_fence();
 222         return ret;
 223 }
 224 #define arch_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire
 225 #endif
 226
 227 #ifndef arch_atomic_fetch_add_release
 228 static __always_inline int
 229 arch_atomic_fetch_add_release(int i, atomic_t *v)
 230 {
 231         __atomic_release_fence();
 232         return arch_atomic_fetch_add_relaxed(i, v);
 233 }
 234 #define arch_atomic_fetch_add_release arch_atomic_fetch_add_release
 235 #endif
 236
 237 #ifndef arch_atomic_fetch_add
 238 static __always_inline int
 239 arch_atomic_fetch_add(int i, atomic_t *v)
 240 {
 241         int ret;
 242         __atomic_pre_full_fence();
 243         ret = arch_atomic_fetch_add_relaxed(i, v);
 244         __atomic_post_full_fence();
 245         return ret;
 246 }
 247 #define arch_atomic_fetch_add arch_atomic_fetch_add
 248 #endif
 249
 250 #endif /* arch_atomic_fetch_add_relaxed */
 251
 252 #ifndef arch_atomic_sub_return_relaxed
 253 #define arch_atomic_sub_return_acquire arch_atomic_sub_return
 254 #define arch_atomic_sub_return_release arch_atomic_sub_return
 255 #define arch_atomic_sub_return_relaxed arch_atomic_sub_return
 256 #else /* arch_atomic_sub_return_relaxed */
 257
 258 #ifndef arch_atomic_sub_return_acquire
 259 static __always_inline int
 260 arch_atomic_sub_return_acquire(int i, atomic_t *v)
 261 {
 262         int ret = arch_atomic_sub_return_relaxed(i, v);
 263         __atomic_acquire_fence();
 264         return ret;
 265 }
 266 #define arch_atomic_sub_return_acquire arch_atomic_sub_return_acquire
 267 #endif
 268
 269 #ifndef arch_atomic_sub_return_release
 270 static __always_inline int
 271 arch_atomic_sub_return_release(int i, atomic_t *v)
 272 {
 273         __atomic_release_fence();
 274         return arch_atomic_sub_return_relaxed(i, v);
 275 }
 276 #define arch_atomic_sub_return_release arch_atomic_sub_return_release
 277 #endif
 278
 279 #ifndef arch_atomic_sub_return
 280 static __always_inline int
 281 arch_atomic_sub_return(int i, atomic_t *v)
 282 {
 283         int ret;
 284         __atomic_pre_full_fence();
 285         ret = arch_atomic_sub_return_relaxed(i, v);
 286         __atomic_post_full_fence();
 287         return ret;
 288 }
 289 #define arch_atomic_sub_return arch_atomic_sub_return
 290 #endif
 291
 292 #endif /* arch_atomic_sub_return_relaxed */
 293
 294 #ifndef arch_atomic_fetch_sub_relaxed
 295 #define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub
 296 #define arch_atomic_fetch_sub_release arch_atomic_fetch_sub
 297 #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub
 298 #else /* arch_atomic_fetch_sub_relaxed */
 299
 300 #ifndef arch_atomic_fetch_sub_acquire
 301 static __always_inline int
 302 arch_atomic_fetch_sub_acquire(int i, atomic_t *v)
 303 {
 304         int ret = arch_atomic_fetch_sub_relaxed(i, v);
 305         __atomic_acquire_fence();
 306         return ret;
 307 }
 308 #define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire
 309 #endif
 310
 311 #ifndef arch_atomic_fetch_sub_release
 312 static __always_inline int
 313 arch_atomic_fetch_sub_release(int i, atomic_t *v)
 314 {
 315         __atomic_release_fence();
 316         return arch_atomic_fetch_sub_relaxed(i, v);
 317 }
 318 #define arch_atomic_fetch_sub_release arch_atomic_fetch_sub_release
 319 #endif
 320
 321 #ifndef arch_atomic_fetch_sub
 322 static __always_inline int
 323 arch_atomic_fetch_sub(int i, atomic_t *v)
 324 {
 325         int ret;
 326         __atomic_pre_full_fence();
 327         ret = arch_atomic_fetch_sub_relaxed(i, v);
 328         __atomic_post_full_fence();
 329         return ret;
 330 }
 331 #define arch_atomic_fetch_sub arch_atomic_fetch_sub
 332 #endif
 333
 334 #endif /* arch_atomic_fetch_sub_relaxed */
 335
 336 #ifndef arch_atomic_inc
 337 static __always_inline void
 338 arch_atomic_inc(atomic_t *v)
 339 {
 340         arch_atomic_add(1, v);
 341 }
 342 #define arch_atomic_inc arch_atomic_inc
 343 #endif
 344
 345 #ifndef arch_atomic_inc_return_relaxed
 346 #ifdef arch_atomic_inc_return
 347 #define arch_atomic_inc_return_acquire arch_atomic_inc_return
 348 #define arch_atomic_inc_return_release arch_atomic_inc_return
 349 #define arch_atomic_inc_return_relaxed arch_atomic_inc_return
 350 #endif /* arch_atomic_inc_return */
 351
 352 #ifndef arch_atomic_inc_return
 353 static __always_inline int
 354 arch_atomic_inc_return(atomic_t *v)
 355 {
 356         return arch_atomic_add_return(1, v);
 357 }
 358 #define arch_atomic_inc_return arch_atomic_inc_return
 359 #endif
 360
 361 #ifndef arch_atomic_inc_return_acquire
 362 static __always_inline int
 363 arch_atomic_inc_return_acquire(atomic_t *v)
 364 {
 365         return arch_atomic_add_return_acquire(1, v);
 366 }
 367 #define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
 368 #endif
 369
 370 #ifndef arch_atomic_inc_return_release
 371 static __always_inline int
 372 arch_atomic_inc_return_release(atomic_t *v)
 373 {
 374         return arch_atomic_add_return_release(1, v);
 375 }
 376 #define arch_atomic_inc_return_release arch_atomic_inc_return_release
 377 #endif
 378
 379 #ifndef arch_atomic_inc_return_relaxed
 380 static __always_inline int
 381 arch_atomic_inc_return_relaxed(atomic_t *v)
 382 {
 383         return arch_atomic_add_return_relaxed(1, v);
 384 }
 385 #define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed
 386 #endif
 387
 388 #else /* arch_atomic_inc_return_relaxed */
 389
 390 #ifndef arch_atomic_inc_return_acquire
 391 static __always_inline int
 392 arch_atomic_inc_return_acquire(atomic_t *v)
 393 {
 394         int ret = arch_atomic_inc_return_relaxed(v);
 395         __atomic_acquire_fence();
 396         return ret;
 397 }
 398 #define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
 399 #endif
 400
 401 #ifndef arch_atomic_inc_return_release
 402 static __always_inline int
 403 arch_atomic_inc_return_release(atomic_t *v)
 404 {
 405         __atomic_release_fence();
 406         return arch_atomic_inc_return_relaxed(v);
 407 }
 408 #define arch_atomic_inc_return_release arch_atomic_inc_return_release
 409 #endif
 410
 411 #ifndef arch_atomic_inc_return
 412 static __always_inline int
 413 arch_atomic_inc_return(atomic_t *v)
 414 {
 415         int ret;
 416         __atomic_pre_full_fence();
 417         ret = arch_atomic_inc_return_relaxed(v);
 418         __atomic_post_full_fence();
 419         return ret;
 420 }
 421 #define arch_atomic_inc_return arch_atomic_inc_return
 422 #endif
 423
 424 #endif /* arch_atomic_inc_return_relaxed */
 425
 426 #ifndef arch_atomic_fetch_inc_relaxed
 427 #ifdef arch_atomic_fetch_inc
 428 #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc
 429 #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc
 430 #define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc
 431 #endif /* arch_atomic_fetch_inc */
 432
 433 #ifndef arch_atomic_fetch_inc
 434 static __always_inline int
 435 arch_atomic_fetch_inc(atomic_t *v)
 436 {
 437         return arch_atomic_fetch_add(1, v);
 438 }
 439 #define arch_atomic_fetch_inc arch_atomic_fetch_inc
 440 #endif
 441
 442 #ifndef arch_atomic_fetch_inc_acquire
 443 static __always_inline int
 444 arch_atomic_fetch_inc_acquire(atomic_t *v)
 445 {
 446         return arch_atomic_fetch_add_acquire(1, v);
 447 }
 448 #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
 449 #endif
 450
 451 #ifndef arch_atomic_fetch_inc_release
 452 static __always_inline int
 453 arch_atomic_fetch_inc_release(atomic_t *v)
 454 {
 455         return arch_atomic_fetch_add_release(1, v);
 456 }
 457 #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
 458 #endif
 459
 460 #ifndef arch_atomic_fetch_inc_relaxed
 461 static __always_inline int
 462 arch_atomic_fetch_inc_relaxed(atomic_t *v)
 463 {
 464         return arch_atomic_fetch_add_relaxed(1, v);
 465 }
 466 #define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed
 467 #endif
 468
 469 #else /* arch_atomic_fetch_inc_relaxed */
 470
 471 #ifndef arch_atomic_fetch_inc_acquire
 472 static __always_inline int
 473 arch_atomic_fetch_inc_acquire(atomic_t *v)
 474 {
 475         int ret = arch_atomic_fetch_inc_relaxed(v);
 476         __atomic_acquire_fence();
 477         return ret;
 478 }
 479 #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
 480 #endif
 481
 482 #ifndef arch_atomic_fetch_inc_release
 483 static __always_inline int
 484 arch_atomic_fetch_inc_release(atomic_t *v)
 485 {
 486         __atomic_release_fence();
 487         return arch_atomic_fetch_inc_relaxed(v);
 488 }
 489 #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
 490 #endif
 491
 492 #ifndef arch_atomic_fetch_inc
 493 static __always_inline int
 494 arch_atomic_fetch_inc(atomic_t *v)
 495 {
 496         int ret;
 497         __atomic_pre_full_fence();
 498         ret = arch_atomic_fetch_inc_relaxed(v);
 499         __atomic_post_full_fence();
 500         return ret;
 501 }
 502 #define arch_atomic_fetch_inc arch_atomic_fetch_inc
 503 #endif
 504
 505 #endif /* arch_atomic_fetch_inc_relaxed */
 506
 507 #ifndef arch_atomic_dec
 508 static __always_inline void
 509 arch_atomic_dec(atomic_t *v)
 510 {
 511         arch_atomic_sub(1, v);
 512 }
 513 #define arch_atomic_dec arch_atomic_dec
 514 #endif
 515
 516 #ifndef arch_atomic_dec_return_relaxed
 517 #ifdef arch_atomic_dec_return
 518 #define arch_atomic_dec_return_acquire arch_atomic_dec_return
 519 #define arch_atomic_dec_return_release arch_atomic_dec_return
 520 #define arch_atomic_dec_return_relaxed arch_atomic_dec_return
 521 #endif /* arch_atomic_dec_return */
 522
 523 #ifndef arch_atomic_dec_return
 524 static __always_inline int
 525 arch_atomic_dec_return(atomic_t *v)
 526 {
 527         return arch_atomic_sub_return(1, v);
 528 }
 529 #define arch_atomic_dec_return arch_atomic_dec_return
 530 #endif
 531
 532 #ifndef arch_atomic_dec_return_acquire
 533 static __always_inline int
 534 arch_atomic_dec_return_acquire(atomic_t *v)
 535 {
 536         return arch_atomic_sub_return_acquire(1, v);
 537 }
 538 #define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
 539 #endif
 540
 541 #ifndef arch_atomic_dec_return_release
 542 static __always_inline int
 543 arch_atomic_dec_return_release(atomic_t *v)
 544 {
 545         return arch_atomic_sub_return_release(1, v);
 546 }
 547 #define arch_atomic_dec_return_release arch_atomic_dec_return_release
 548 #endif
 549
 550 #ifndef arch_atomic_dec_return_relaxed
 551 static __always_inline int
 552 arch_atomic_dec_return_relaxed(atomic_t *v)
 553 {
 554         return arch_atomic_sub_return_relaxed(1, v);
 555 }
 556 #define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed
 557 #endif
 558
 559 #else /* arch_atomic_dec_return_relaxed */
 560
 561 #ifndef arch_atomic_dec_return_acquire
 562 static __always_inline int
 563 arch_atomic_dec_return_acquire(atomic_t *v)
 564 {
 565         int ret = arch_atomic_dec_return_relaxed(v);
 566         __atomic_acquire_fence();
 567         return ret;
 568 }
 569 #define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
 570 #endif
 571
 572 #ifndef arch_atomic_dec_return_release
 573 static __always_inline int
 574 arch_atomic_dec_return_release(atomic_t *v)
 575 {
 576         __atomic_release_fence();
 577         return arch_atomic_dec_return_relaxed(v);
 578 }
 579 #define arch_atomic_dec_return_release arch_atomic_dec_return_release
 580 #endif
 581
 582 #ifndef arch_atomic_dec_return
 583 static __always_inline int
 584 arch_atomic_dec_return(atomic_t *v)
 585 {
 586         int ret;
 587         __atomic_pre_full_fence();
 588         ret = arch_atomic_dec_return_relaxed(v);
 589         __atomic_post_full_fence();
 590         return ret;
 591 }
 592 #define arch_atomic_dec_return arch_atomic_dec_return
 593 #endif
 594
 595 #endif /* arch_atomic_dec_return_relaxed */
 596
 597 #ifndef arch_atomic_fetch_dec_relaxed
 598 #ifdef arch_atomic_fetch_dec
 599 #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec
 600 #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec
 601 #define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec
 602 #endif /* arch_atomic_fetch_dec */
 603
 604 #ifndef arch_atomic_fetch_dec
 605 static __always_inline int
 606 arch_atomic_fetch_dec(atomic_t *v)
 607 {
 608         return arch_atomic_fetch_sub(1, v);
 609 }
 610 #define arch_atomic_fetch_dec arch_atomic_fetch_dec
 611 #endif
 612
 613 #ifndef arch_atomic_fetch_dec_acquire
 614 static __always_inline int
 615 arch_atomic_fetch_dec_acquire(atomic_t *v)
 616 {
 617         return arch_atomic_fetch_sub_acquire(1, v);
 618 }
 619 #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
 620 #endif
 621
 622 #ifndef arch_atomic_fetch_dec_release
 623 static __always_inline int
 624 arch_atomic_fetch_dec_release(atomic_t *v)
 625 {
 626         return arch_atomic_fetch_sub_release(1, v);
 627 }
 628 #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
 629 #endif
 630
 631 #ifndef arch_atomic_fetch_dec_relaxed
 632 static __always_inline int
 633 arch_atomic_fetch_dec_relaxed(atomic_t *v)
 634 {
 635         return arch_atomic_fetch_sub_relaxed(1, v);
 636 }
 637 #define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed
 638 #endif
 639
 640 #else /* arch_atomic_fetch_dec_relaxed */
 641
 642 #ifndef arch_atomic_fetch_dec_acquire
 643 static __always_inline int
 644 arch_atomic_fetch_dec_acquire(atomic_t *v)
 645 {
 646         int ret = arch_atomic_fetch_dec_relaxed(v);
 647         __atomic_acquire_fence();
 648         return ret;
 649 }
 650 #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
 651 #endif
 652
 653 #ifndef arch_atomic_fetch_dec_release
 654 static __always_inline int
 655 arch_atomic_fetch_dec_release(atomic_t *v)
 656 {
 657         __atomic_release_fence();
 658         return arch_atomic_fetch_dec_relaxed(v);
 659 }
 660 #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
 661 #endif
 662
 663 #ifndef arch_atomic_fetch_dec
 664 static __always_inline int
 665 arch_atomic_fetch_dec(atomic_t *v)
 666 {
 667         int ret;
 668         __atomic_pre_full_fence();
 669         ret = arch_atomic_fetch_dec_relaxed(v);
 670         __atomic_post_full_fence();
 671         return ret;
 672 }
 673 #define arch_atomic_fetch_dec arch_atomic_fetch_dec
 674 #endif
 675
 676 #endif /* arch_atomic_fetch_dec_relaxed */
 677
 678 #ifndef arch_atomic_fetch_and_relaxed
 679 #define arch_atomic_fetch_and_acquire arch_atomic_fetch_and
 680 #define arch_atomic_fetch_and_release arch_atomic_fetch_and
 681 #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and
 682 #else /* arch_atomic_fetch_and_relaxed */
 683
 684 #ifndef arch_atomic_fetch_and_acquire
 685 static __always_inline int
 686 arch_atomic_fetch_and_acquire(int i, atomic_t *v)
 687 {
 688         int ret = arch_atomic_fetch_and_relaxed(i, v);
 689         __atomic_acquire_fence();
 690         return ret;
 691 }
 692 #define arch_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire
 693 #endif
 694
 695 #ifndef arch_atomic_fetch_and_release
 696 static __always_inline int
 697 arch_atomic_fetch_and_release(int i, atomic_t *v)
 698 {
 699         __atomic_release_fence();
 700         return arch_atomic_fetch_and_relaxed(i, v);
 701 }
 702 #define arch_atomic_fetch_and_release arch_atomic_fetch_and_release
 703 #endif
 704
 705 #ifndef arch_atomic_fetch_and
 706 static __always_inline int
 707 arch_atomic_fetch_and(int i, atomic_t *v)
 708 {
 709         int ret;
 710         __atomic_pre_full_fence();
 711         ret = arch_atomic_fetch_and_relaxed(i, v);
 712         __atomic_post_full_fence();
 713         return ret;
 714 }
 715 #define arch_atomic_fetch_and arch_atomic_fetch_and
 716 #endif
 717
 718 #endif /* arch_atomic_fetch_and_relaxed */
 719
 720 #ifndef arch_atomic_andnot
 721 static __always_inline void
 722 arch_atomic_andnot(int i, atomic_t *v)
 723 {
 724         arch_atomic_and(~i, v);
 725 }
 726 #define arch_atomic_andnot arch_atomic_andnot
 727 #endif
 728
 729 #ifndef arch_atomic_fetch_andnot_relaxed
 730 #ifdef arch_atomic_fetch_andnot
 731 #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot
 732 #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot
 733 #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot
 734 #endif /* arch_atomic_fetch_andnot */
 735
 736 #ifndef arch_atomic_fetch_andnot
 737 static __always_inline int
 738 arch_atomic_fetch_andnot(int i, atomic_t *v)
 739 {
 740         return arch_atomic_fetch_and(~i, v);
 741 }
 742 #define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
 743 #endif
 744
 745 #ifndef arch_atomic_fetch_andnot_acquire
 746 static __always_inline int
 747 arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
 748 {
 749         return arch_atomic_fetch_and_acquire(~i, v);
 750 }
 751 #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
 752 #endif
 753
 754 #ifndef arch_atomic_fetch_andnot_release
 755 static __always_inline int
 756 arch_atomic_fetch_andnot_release(int i, atomic_t *v)
 757 {
 758         return arch_atomic_fetch_and_release(~i, v);
 759 }
 760 #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
 761 #endif
 762
 763 #ifndef arch_atomic_fetch_andnot_relaxed
 764 static __always_inline int
 765 arch_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
 766 {
 767         return arch_atomic_fetch_and_relaxed(~i, v);
 768 }
 769 #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed
 770 #endif
 771
 772 #else /* arch_atomic_fetch_andnot_relaxed */
 773
 774 #ifndef arch_atomic_fetch_andnot_acquire
 775 static __always_inline int
 776 arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
 777 {
 778         int ret = arch_atomic_fetch_andnot_relaxed(i, v);
 779         __atomic_acquire_fence();
 780         return ret;
 781 }
 782 #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
 783 #endif
 784
 785 #ifndef arch_atomic_fetch_andnot_release
 786 static __always_inline int
 787 arch_atomic_fetch_andnot_release(int i, atomic_t *v)
 788 {
 789         __atomic_release_fence();
 790         return arch_atomic_fetch_andnot_relaxed(i, v);
 791 }
 792 #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
 793 #endif
 794
 795 #ifndef arch_atomic_fetch_andnot
 796 static __always_inline int
 797 arch_atomic_fetch_andnot(int i, atomic_t *v)
 798 {
 799         int ret;
 800         __atomic_pre_full_fence();
 801         ret = arch_atomic_fetch_andnot_relaxed(i, v);
 802         __atomic_post_full_fence();
 803         return ret;
 804 }
 805 #define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
 806 #endif
 807
 808 #endif /* arch_atomic_fetch_andnot_relaxed */
 809
 810 #ifndef arch_atomic_fetch_or_relaxed
 811 #define arch_atomic_fetch_or_acquire arch_atomic_fetch_or
 812 #define arch_atomic_fetch_or_release arch_atomic_fetch_or
 813 #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or
 814 #else /* arch_atomic_fetch_or_relaxed */
 815
 816 #ifndef arch_atomic_fetch_or_acquire
 817 static __always_inline int
 818 arch_atomic_fetch_or_acquire(int i, atomic_t *v)
 819 {
 820         int ret = arch_atomic_fetch_or_relaxed(i, v);
 821         __atomic_acquire_fence();
 822         return ret;
 823 }
 824 #define arch_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire
 825 #endif
 826
 827 #ifndef arch_atomic_fetch_or_release
 828 static __always_inline int
 829 arch_atomic_fetch_or_release(int i, atomic_t *v)
 830 {
 831         __atomic_release_fence();
 832         return arch_atomic_fetch_or_relaxed(i, v);
 833 }
 834 #define arch_atomic_fetch_or_release arch_atomic_fetch_or_release
 835 #endif
 836
 837 #ifndef arch_atomic_fetch_or
 838 static __always_inline int
 839 arch_atomic_fetch_or(int i, atomic_t *v)
 840 {
 841         int ret;
 842         __atomic_pre_full_fence();
 843         ret = arch_atomic_fetch_or_relaxed(i, v);
 844         __atomic_post_full_fence();
 845         return ret;
 846 }
 847 #define arch_atomic_fetch_or arch_atomic_fetch_or
 848 #endif
 849
 850 #endif /* arch_atomic_fetch_or_relaxed */
 851
 852 #ifndef arch_atomic_fetch_xor_relaxed
 853 #define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor
 854 #define arch_atomic_fetch_xor_release arch_atomic_fetch_xor
 855 #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor
 856 #else /* arch_atomic_fetch_xor_relaxed */
 857
 858 #ifndef arch_atomic_fetch_xor_acquire
 859 static __always_inline int
 860 arch_atomic_fetch_xor_acquire(int i, atomic_t *v)
 861 {
 862         int ret = arch_atomic_fetch_xor_relaxed(i, v);
 863         __atomic_acquire_fence();
 864         return ret;
 865 }
 866 #define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire
 867 #endif
 868
 869 #ifndef arch_atomic_fetch_xor_release
 870 static __always_inline int
 871 arch_atomic_fetch_xor_release(int i, atomic_t *v)
 872 {
 873         __atomic_release_fence();
 874         return arch_atomic_fetch_xor_relaxed(i, v);
 875 }
 876 #define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release
 877 #endif
 878
 879 #ifndef arch_atomic_fetch_xor
 880 static __always_inline int
 881 arch_atomic_fetch_xor(int i, atomic_t *v)
 882 {
 883         int ret;
 884         __atomic_pre_full_fence();
 885         ret = arch_atomic_fetch_xor_relaxed(i, v);
 886         __atomic_post_full_fence();
 887         return ret;
 888 }
 889 #define arch_atomic_fetch_xor arch_atomic_fetch_xor
 890 #endif
 891
 892 #endif /* arch_atomic_fetch_xor_relaxed */
 893
 894 #ifndef arch_atomic_xchg_relaxed
 895 #define arch_atomic_xchg_acquire arch_atomic_xchg
 896 #define arch_atomic_xchg_release arch_atomic_xchg
 897 #define arch_atomic_xchg_relaxed arch_atomic_xchg
 898 #else /* arch_atomic_xchg_relaxed */
 899
 900 #ifndef arch_atomic_xchg_acquire
 901 static __always_inline int
 902 arch_atomic_xchg_acquire(atomic_t *v, int i)
 903 {
 904         int ret = arch_atomic_xchg_relaxed(v, i);
 905         __atomic_acquire_fence();
 906         return ret;
 907 }
 908 #define arch_atomic_xchg_acquire arch_atomic_xchg_acquire
 909 #endif
 910
 911 #ifndef arch_atomic_xchg_release
 912 static __always_inline int
 913 arch_atomic_xchg_release(atomic_t *v, int i)
 914 {
 915         __atomic_release_fence();
 916         return arch_atomic_xchg_relaxed(v, i);
 917 }
 918 #define arch_atomic_xchg_release arch_atomic_xchg_release
 919 #endif
 920
 921 #ifndef arch_atomic_xchg
 922 static __always_inline int
 923 arch_atomic_xchg(atomic_t *v, int i)
 924 {
 925         int ret;
 926         __atomic_pre_full_fence();
 927         ret = arch_atomic_xchg_relaxed(v, i);
 928         __atomic_post_full_fence();
 929         return ret;
 930 }
 931 #define arch_atomic_xchg arch_atomic_xchg
 932 #endif
 933
 934 #endif /* arch_atomic_xchg_relaxed */
 935
 936 #ifndef arch_atomic_cmpxchg_relaxed
 937 #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg
 938 #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg
 939 #define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg
 940 #else /* arch_atomic_cmpxchg_relaxed */
 941
 942 #ifndef arch_atomic_cmpxchg_acquire
 943 static __always_inline int
 944 arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
 945 {
 946         int ret = arch_atomic_cmpxchg_relaxed(v, old, new);
 947         __atomic_acquire_fence();
 948         return ret;
 949 }
 950 #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire
 951 #endif
 952
 953 #ifndef arch_atomic_cmpxchg_release
 954 static __always_inline int
 955 arch_atomic_cmpxchg_release(atomic_t *v, int old, int new)
 956 {
 957         __atomic_release_fence();
 958         return arch_atomic_cmpxchg_relaxed(v, old, new);
 959 }
 960 #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release
 961 #endif
 962
 963 #ifndef arch_atomic_cmpxchg
 964 static __always_inline int
 965 arch_atomic_cmpxchg(atomic_t *v, int old, int new)
 966 {
 967         int ret;
 968         __atomic_pre_full_fence();
 969         ret = arch_atomic_cmpxchg_relaxed(v, old, new);
 970         __atomic_post_full_fence();
 971         return ret;
 972 }
 973 #define arch_atomic_cmpxchg arch_atomic_cmpxchg
 974 #endif
 975
 976 #endif /* arch_atomic_cmpxchg_relaxed */
 977
 978 #ifndef arch_atomic_try_cmpxchg_relaxed
 979 #ifdef arch_atomic_try_cmpxchg
 980 #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg
 981 #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg
 982 #define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg
 983 #endif /* arch_atomic_try_cmpxchg */
 984
 985 #ifndef arch_atomic_try_cmpxchg
 986 static __always_inline bool
 987 arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
 988 {
 989         int r, o = *old;
 990         r = arch_atomic_cmpxchg(v, o, new);
 991         if (unlikely(r != o))
 992                 *old = r;
 993         return likely(r == o);
 994 }
 995 #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
 996 #endif
 997
 998 #ifndef arch_atomic_try_cmpxchg_acquire
 999 static __always_inline bool
1000 arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
1001 {
1002         int r, o = *old;
1003         r = arch_atomic_cmpxchg_acquire(v, o, new);
1004         if (unlikely(r != o))
1005                 *old = r;
1006         return likely(r == o);
1007 }
1008 #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
1009 #endif
1010
1011 #ifndef arch_atomic_try_cmpxchg_release
1012 static __always_inline bool
1013 arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
1014 {
1015         int r, o = *old;
1016         r = arch_atomic_cmpxchg_release(v, o, new);
1017         if (unlikely(r != o))
1018                 *old = r;
1019         return likely(r == o);
1020 }
1021 #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
1022 #endif
1023
1024 #ifndef arch_atomic_try_cmpxchg_relaxed
1025 static __always_inline bool
1026 arch_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
1027 {
1028         int r, o = *old;
1029         r = arch_atomic_cmpxchg_relaxed(v, o, new);
1030         if (unlikely(r != o))
1031                 *old = r;
1032         return likely(r == o);
1033 }
1034 #define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed
1035 #endif
1036
1037 #else /* arch_atomic_try_cmpxchg_relaxed */
1038
1039 #ifndef arch_atomic_try_cmpxchg_acquire
1040 static __always_inline bool
1041 arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
1042 {
1043         bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
1044         __atomic_acquire_fence();
1045         return ret;
1046 }
1047 #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
1048 #endif
1049
1050 #ifndef arch_atomic_try_cmpxchg_release
1051 static __always_inline bool
1052 arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
1053 {
1054         __atomic_release_fence();
1055         return arch_atomic_try_cmpxchg_relaxed(v, old, new);
1056 }
1057 #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
1058 #endif
1059
1060 #ifndef arch_atomic_try_cmpxchg
1061 static __always_inline bool
1062 arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
1063 {
1064         bool ret;
1065         __atomic_pre_full_fence();
1066         ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
1067         __atomic_post_full_fence();
1068         return ret;
1069 }
1070 #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
1071 #endif
1072
1073 #endif /* arch_atomic_try_cmpxchg_relaxed */
1074
1075 #ifndef arch_atomic_sub_and_test
1076 /**
1077  * arch_atomic_sub_and_test - subtract value from variable and test result
1078  * @i: integer value to subtract
1079  * @v: pointer of type atomic_t
1080  *
1081  * Atomically subtracts @i from @v and returns
1082  * true if the result is zero, or false for all
1083  * other cases.
1084  */
1085 static __always_inline bool
1086 arch_atomic_sub_and_test(int i, atomic_t *v)
1087 {
1088         return arch_atomic_sub_return(i, v) == 0;
1089 }
1090 #define arch_atomic_sub_and_test arch_atomic_sub_and_test
1091 #endif
1092
1093 #ifndef arch_atomic_dec_and_test
1094 /**
1095  * arch_atomic_dec_and_test - decrement and test
1096  * @v: pointer of type atomic_t
1097  *
1098  * Atomically decrements @v by 1 and
1099  * returns true if the result is 0, or false for all other
1100  * cases.
1101  */
1102 static __always_inline bool
1103 arch_atomic_dec_and_test(atomic_t *v)
1104 {
1105         return arch_atomic_dec_return(v) == 0;
1106 }
1107 #define arch_atomic_dec_and_test arch_atomic_dec_and_test
1108 #endif
1109
1110 #ifndef arch_atomic_inc_and_test
1111 /**
1112  * arch_atomic_inc_and_test - increment and test
1113  * @v: pointer of type atomic_t
1114  *
1115  * Atomically increments @v by 1
1116  * and returns true if the result is zero, or false for all
1117  * other cases.
1118  */
1119 static __always_inline bool
1120 arch_atomic_inc_and_test(atomic_t *v)
1121 {
1122         return arch_atomic_inc_return(v) == 0;
1123 }
1124 #define arch_atomic_inc_and_test arch_atomic_inc_and_test
1125 #endif
1126
1127 #ifndef arch_atomic_add_negative
1128 /**
1129  * arch_atomic_add_negative - add and test if negative
1130  * @i: integer value to add
1131  * @v: pointer of type atomic_t
1132  *
1133  * Atomically adds @i to @v and returns true
1134  * if the result is negative, or false when
1135  * result is greater than or equal to zero.
1136  */
1137 static __always_inline bool
1138 arch_atomic_add_negative(int i, atomic_t *v)
1139 {
1140         return arch_atomic_add_return(i, v) < 0;
1141 }
1142 #define arch_atomic_add_negative arch_atomic_add_negative
1143 #endif
1144
1145 #ifndef arch_atomic_fetch_add_unless
1146 /**
1147  * arch_atomic_fetch_add_unless - add unless the number is already a given value
1148  * @v: pointer of type atomic_t
1149  * @a: the amount to add to v...
1150  * @u: ...unless v is equal to u.
1151  *
1152  * Atomically adds @a to @v, so long as @v was not already @u.
1153  * Returns original value of @v
1154  */
1155 static __always_inline int
1156 arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
1157 {
1158         int c = arch_atomic_read(v);
1159
1160         do {
1161                 if (unlikely(c == u))
1162                         break;
1163         } while (!arch_atomic_try_cmpxchg(v, &c, c + a));
1164
1165         return c;
1166 }
1167 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
1168 #endif
1169
1170 #ifndef arch_atomic_add_unless
1171 /**
1172  * arch_atomic_add_unless - add unless the number is already a given value
1173  * @v: pointer of type atomic_t
1174  * @a: the amount to add to v...
1175  * @u: ...unless v is equal to u.
1176  *
1177  * Atomically adds @a to @v, if @v was not already @u.
1178  * Returns true if the addition was done.
1179  */
1180 static __always_inline bool
1181 arch_atomic_add_unless(atomic_t *v, int a, int u)
1182 {
1183         return arch_atomic_fetch_add_unless(v, a, u) != u;
1184 }
1185 #define arch_atomic_add_unless arch_atomic_add_unless
1186 #endif
1187
1188 #ifndef arch_atomic_inc_not_zero
1189 /**
1190  * arch_atomic_inc_not_zero - increment unless the number is zero
1191  * @v: pointer of type atomic_t
1192  *
1193  * Atomically increments @v by 1, if @v is non-zero.
1194  * Returns true if the increment was done.
1195  */
1196 static __always_inline bool
1197 arch_atomic_inc_not_zero(atomic_t *v)
1198 {
1199         return arch_atomic_add_unless(v, 1, 0);
1200 }
1201 #define arch_atomic_inc_not_zero arch_atomic_inc_not_zero
1202 #endif
1203
1204 #ifndef arch_atomic_inc_unless_negative
1205 static __always_inline bool
1206 arch_atomic_inc_unless_negative(atomic_t *v)
1207 {
1208         int c = arch_atomic_read(v);
1209
1210         do {
1211                 if (unlikely(c < 0))
1212                         return false;
1213         } while (!arch_atomic_try_cmpxchg(v, &c, c + 1));
1214
1215         return true;
1216 }
1217 #define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative
1218 #endif
1219
1220 #ifndef arch_atomic_dec_unless_positive
1221 static __always_inline bool
1222 arch_atomic_dec_unless_positive(atomic_t *v)
1223 {
1224         int c = arch_atomic_read(v);
1225
1226         do {
1227                 if (unlikely(c > 0))
1228                         return false;
1229         } while (!arch_atomic_try_cmpxchg(v, &c, c - 1));
1230
1231         return true;
1232 }
1233 #define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive
1234 #endif
1235
1236 #ifndef arch_atomic_dec_if_positive
1237 static __always_inline int
1238 arch_atomic_dec_if_positive(atomic_t *v)
1239 {
1240         int dec, c = arch_atomic_read(v);
1241
1242         do {
1243                 dec = c - 1;
1244                 if (unlikely(dec < 0))
1245                         break;
1246         } while (!arch_atomic_try_cmpxchg(v, &c, dec));
1247
1248         return dec;
1249 }
1250 #define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
1251 #endif
1252
1253 #ifdef CONFIG_GENERIC_ATOMIC64
1254 #include <asm-generic/atomic64.h>
1255 #endif
1256
1257 #ifndef arch_atomic64_read_acquire
1258 static __always_inline s64
1259 arch_atomic64_read_acquire(const atomic64_t *v)
1260 {
1261         return smp_load_acquire(&(v)->counter);
1262 }
1263 #define arch_atomic64_read_acquire arch_atomic64_read_acquire
1264 #endif
1265
1266 #ifndef arch_atomic64_set_release
1267 static __always_inline void
1268 arch_atomic64_set_release(atomic64_t *v, s64 i)
1269 {
1270         smp_store_release(&(v)->counter, i);
1271 }
1272 #define arch_atomic64_set_release arch_atomic64_set_release
1273 #endif
1274
1275 #ifndef arch_atomic64_add_return_relaxed
1276 #define arch_atomic64_add_return_acquire arch_atomic64_add_return
1277 #define arch_atomic64_add_return_release arch_atomic64_add_return
1278 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return
1279 #else /* arch_atomic64_add_return_relaxed */
1280
1281 #ifndef arch_atomic64_add_return_acquire
1282 static __always_inline s64
1283 arch_atomic64_add_return_acquire(s64 i, atomic64_t *v)
1284 {
1285         s64 ret = arch_atomic64_add_return_relaxed(i, v);
1286         __atomic_acquire_fence();
1287         return ret;
1288 }
1289 #define arch_atomic64_add_return_acquire arch_atomic64_add_return_acquire
1290 #endif
1291
1292 #ifndef arch_atomic64_add_return_release
1293 static __always_inline s64
1294 arch_atomic64_add_return_release(s64 i, atomic64_t *v)
1295 {
1296         __atomic_release_fence();
1297         return arch_atomic64_add_return_relaxed(i, v);
1298 }
1299 #define arch_atomic64_add_return_release arch_atomic64_add_return_release
1300 #endif
1301
1302 #ifndef arch_atomic64_add_return
1303 static __always_inline s64
1304 arch_atomic64_add_return(s64 i, atomic64_t *v)
1305 {
1306         s64 ret;
1307         __atomic_pre_full_fence();
1308         ret = arch_atomic64_add_return_relaxed(i, v);
1309         __atomic_post_full_fence();
1310         return ret;
1311 }
1312 #define arch_atomic64_add_return arch_atomic64_add_return
1313 #endif
1314
1315 #endif /* arch_atomic64_add_return_relaxed */
1316
1317 #ifndef arch_atomic64_fetch_add_relaxed
1318 #define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add
1319 #define arch_atomic64_fetch_add_release arch_atomic64_fetch_add
1320 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add
1321 #else /* arch_atomic64_fetch_add_relaxed */
1322
1323 #ifndef arch_atomic64_fetch_add_acquire
1324 static __always_inline s64
1325 arch_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1326 {
1327         s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
1328         __atomic_acquire_fence();
1329         return ret;
1330 }
1331 #define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire
1332 #endif
1333
1334 #ifndef arch_atomic64_fetch_add_release
1335 static __always_inline s64
1336 arch_atomic64_fetch_add_release(s64 i, atomic64_t *v)
1337 {
1338         __atomic_release_fence();
1339         return arch_atomic64_fetch_add_relaxed(i, v);
1340 }
1341 #define arch_atomic64_fetch_add_release arch_atomic64_fetch_add_release
1342 #endif
1343
1344 #ifndef arch_atomic64_fetch_add
1345 static __always_inline s64
1346 arch_atomic64_fetch_add(s64 i, atomic64_t *v)
1347 {
1348         s64 ret;
1349         __atomic_pre_full_fence();
1350         ret = arch_atomic64_fetch_add_relaxed(i, v);
1351         __atomic_post_full_fence();
1352         return ret;
1353 }
1354 #define arch_atomic64_fetch_add arch_atomic64_fetch_add
1355 #endif
1356
1357 #endif /* arch_atomic64_fetch_add_relaxed */
1358
1359 #ifndef arch_atomic64_sub_return_relaxed
1360 #define arch_atomic64_sub_return_acquire arch_atomic64_sub_return
1361 #define arch_atomic64_sub_return_release arch_atomic64_sub_return
1362 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return
1363 #else /* arch_atomic64_sub_return_relaxed */
1364
1365 #ifndef arch_atomic64_sub_return_acquire
1366 static __always_inline s64
1367 arch_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1368 {
1369         s64 ret = arch_atomic64_sub_return_relaxed(i, v);
1370         __atomic_acquire_fence();
1371         return ret;
1372 }
1373 #define arch_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire
1374 #endif
1375
1376 #ifndef arch_atomic64_sub_return_release
1377 static __always_inline s64
1378 arch_atomic64_sub_return_release(s64 i, atomic64_t *v)
1379 {
1380         __atomic_release_fence();
1381         return arch_atomic64_sub_return_relaxed(i, v);
1382 }
1383 #define arch_atomic64_sub_return_release arch_atomic64_sub_return_release
1384 #endif
1385
1386 #ifndef arch_atomic64_sub_return
1387 static __always_inline s64
1388 arch_atomic64_sub_return(s64 i, atomic64_t *v)
1389 {
1390         s64 ret;
1391         __atomic_pre_full_fence();
1392         ret = arch_atomic64_sub_return_relaxed(i, v);
1393         __atomic_post_full_fence();
1394         return ret;
1395 }
1396 #define arch_atomic64_sub_return arch_atomic64_sub_return
1397 #endif
1398
1399 #endif /* arch_atomic64_sub_return_relaxed */
1400
1401 #ifndef arch_atomic64_fetch_sub_relaxed
1402 #define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub
1403 #define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub
1404 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub
1405 #else /* arch_atomic64_fetch_sub_relaxed */
1406
1407 #ifndef arch_atomic64_fetch_sub_acquire
1408 static __always_inline s64
1409 arch_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1410 {
1411         s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
1412         __atomic_acquire_fence();
1413         return ret;
1414 }
1415 #define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire
1416 #endif
1417
1418 #ifndef arch_atomic64_fetch_sub_release
1419 static __always_inline s64
1420 arch_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1421 {
1422         __atomic_release_fence();
1423         return arch_atomic64_fetch_sub_relaxed(i, v);
1424 }
1425 #define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release
1426 #endif
1427
1428 #ifndef arch_atomic64_fetch_sub
1429 static __always_inline s64
1430 arch_atomic64_fetch_sub(s64 i, atomic64_t *v)
1431 {
1432         s64 ret;
1433         __atomic_pre_full_fence();
1434         ret = arch_atomic64_fetch_sub_relaxed(i, v);
1435         __atomic_post_full_fence();
1436         return ret;
1437 }
1438 #define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
1439 #endif
1440
1441 #endif /* arch_atomic64_fetch_sub_relaxed */
1442
1443 #ifndef arch_atomic64_inc
1444 static __always_inline void
1445 arch_atomic64_inc(atomic64_t *v)
1446 {
1447         arch_atomic64_add(1, v);
1448 }
1449 #define arch_atomic64_inc arch_atomic64_inc
1450 #endif
1451
1452 #ifndef arch_atomic64_inc_return_relaxed
1453 #ifdef arch_atomic64_inc_return
1454 #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return
1455 #define arch_atomic64_inc_return_release arch_atomic64_inc_return
1456 #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return
1457 #endif /* arch_atomic64_inc_return */
1458
1459 #ifndef arch_atomic64_inc_return
1460 static __always_inline s64
1461 arch_atomic64_inc_return(atomic64_t *v)
1462 {
1463         return arch_atomic64_add_return(1, v);
1464 }
1465 #define arch_atomic64_inc_return arch_atomic64_inc_return
1466 #endif
1467
1468 #ifndef arch_atomic64_inc_return_acquire
1469 static __always_inline s64
1470 arch_atomic64_inc_return_acquire(atomic64_t *v)
1471 {
1472         return arch_atomic64_add_return_acquire(1, v);
1473 }
1474 #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1475 #endif
1476
1477 #ifndef arch_atomic64_inc_return_release
1478 static __always_inline s64
1479 arch_atomic64_inc_return_release(atomic64_t *v)
1480 {
1481         return arch_atomic64_add_return_release(1, v);
1482 }
1483 #define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
1484 #endif
1485
1486 #ifndef arch_atomic64_inc_return_relaxed
1487 static __always_inline s64
1488 arch_atomic64_inc_return_relaxed(atomic64_t *v)
1489 {
1490         return arch_atomic64_add_return_relaxed(1, v);
1491 }
1492 #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed
1493 #endif
1494
1495 #else /* arch_atomic64_inc_return_relaxed */
1496
1497 #ifndef arch_atomic64_inc_return_acquire
1498 static __always_inline s64
1499 arch_atomic64_inc_return_acquire(atomic64_t *v)
1500 {
1501         s64 ret = arch_atomic64_inc_return_relaxed(v);
1502         __atomic_acquire_fence();
1503         return ret;
1504 }
1505 #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1506 #endif
1507
1508 #ifndef arch_atomic64_inc_return_release
1509 static __always_inline s64
1510 arch_atomic64_inc_return_release(atomic64_t *v)
1511 {
1512         __atomic_release_fence();
1513         return arch_atomic64_inc_return_relaxed(v);
1514 }
1515 #define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
1516 #endif
1517
1518 #ifndef arch_atomic64_inc_return
1519 static __always_inline s64
1520 arch_atomic64_inc_return(atomic64_t *v)
1521 {
1522         s64 ret;
1523         __atomic_pre_full_fence();
1524         ret = arch_atomic64_inc_return_relaxed(v);
1525         __atomic_post_full_fence();
1526         return ret;
1527 }
1528 #define arch_atomic64_inc_return arch_atomic64_inc_return
1529 #endif
1530
1531 #endif /* arch_atomic64_inc_return_relaxed */
1532
1533 #ifndef arch_atomic64_fetch_inc_relaxed
1534 #ifdef arch_atomic64_fetch_inc
1535 #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc
1536 #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc
1537 #define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc
1538 #endif /* arch_atomic64_fetch_inc */
1539
1540 #ifndef arch_atomic64_fetch_inc
1541 static __always_inline s64
1542 arch_atomic64_fetch_inc(atomic64_t *v)
1543 {
1544         return arch_atomic64_fetch_add(1, v);
1545 }
1546 #define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
1547 #endif
1548
1549 #ifndef arch_atomic64_fetch_inc_acquire
1550 static __always_inline s64
1551 arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1552 {
1553         return arch_atomic64_fetch_add_acquire(1, v);
1554 }
1555 #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1556 #endif
1557
1558 #ifndef arch_atomic64_fetch_inc_release
1559 static __always_inline s64
1560 arch_atomic64_fetch_inc_release(atomic64_t *v)
1561 {
1562         return arch_atomic64_fetch_add_release(1, v);
1563 }
1564 #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
1565 #endif
1566
1567 #ifndef arch_atomic64_fetch_inc_relaxed
1568 static __always_inline s64
1569 arch_atomic64_fetch_inc_relaxed(atomic64_t *v)
1570 {
1571         return arch_atomic64_fetch_add_relaxed(1, v);
1572 }
1573 #define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed
1574 #endif
1575
1576 #else /* arch_atomic64_fetch_inc_relaxed */
1577
1578 #ifndef arch_atomic64_fetch_inc_acquire
1579 static __always_inline s64
1580 arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1581 {
1582         s64 ret = arch_atomic64_fetch_inc_relaxed(v);
1583         __atomic_acquire_fence();
1584         return ret;
1585 }
1586 #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1587 #endif
1588
1589 #ifndef arch_atomic64_fetch_inc_release
1590 static __always_inline s64
1591 arch_atomic64_fetch_inc_release(atomic64_t *v)
1592 {
1593         __atomic_release_fence();
1594         return arch_atomic64_fetch_inc_relaxed(v);
1595 }
1596 #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
1597 #endif
1598
1599 #ifndef arch_atomic64_fetch_inc
1600 static __always_inline s64
1601 arch_atomic64_fetch_inc(atomic64_t *v)
1602 {
1603         s64 ret;
1604         __atomic_pre_full_fence();
1605         ret = arch_atomic64_fetch_inc_relaxed(v);
1606         __atomic_post_full_fence();
1607         return ret;
1608 }
1609 #define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
1610 #endif
1611
1612 #endif /* arch_atomic64_fetch_inc_relaxed */
1613
1614 #ifndef arch_atomic64_dec
1615 static __always_inline void
1616 arch_atomic64_dec(atomic64_t *v)
1617 {
1618         arch_atomic64_sub(1, v);
1619 }
1620 #define arch_atomic64_dec arch_atomic64_dec
1621 #endif
1622
1623 #ifndef arch_atomic64_dec_return_relaxed
1624 #ifdef arch_atomic64_dec_return
1625 #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return
1626 #define arch_atomic64_dec_return_release arch_atomic64_dec_return
1627 #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return
1628 #endif /* arch_atomic64_dec_return */
1629
1630 #ifndef arch_atomic64_dec_return
1631 static __always_inline s64
1632 arch_atomic64_dec_return(atomic64_t *v)
1633 {
1634         return arch_atomic64_sub_return(1, v);
1635 }
1636 #define arch_atomic64_dec_return arch_atomic64_dec_return
1637 #endif
1638
1639 #ifndef arch_atomic64_dec_return_acquire
1640 static __always_inline s64
1641 arch_atomic64_dec_return_acquire(atomic64_t *v)
1642 {
1643         return arch_atomic64_sub_return_acquire(1, v);
1644 }
1645 #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
1646 #endif
1647
1648 #ifndef arch_atomic64_dec_return_release
1649 static __always_inline s64
1650 arch_atomic64_dec_return_release(atomic64_t *v)
1651 {
1652         return arch_atomic64_sub_return_release(1, v);
1653 }
1654 #define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1655 #endif
1656
1657 #ifndef arch_atomic64_dec_return_relaxed
1658 static __always_inline s64
1659 arch_atomic64_dec_return_relaxed(atomic64_t *v)
1660 {
1661         return arch_atomic64_sub_return_relaxed(1, v);
1662 }
1663 #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed
1664 #endif
1665
1666 #else /* arch_atomic64_dec_return_relaxed */
1667
1668 #ifndef arch_atomic64_dec_return_acquire
1669 static __always_inline s64
1670 arch_atomic64_dec_return_acquire(atomic64_t *v)
1671 {
1672         s64 ret = arch_atomic64_dec_return_relaxed(v);
1673         __atomic_acquire_fence();
1674         return ret;
1675 }
1676 #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
1677 #endif
1678
1679 #ifndef arch_atomic64_dec_return_release
1680 static __always_inline s64
1681 arch_atomic64_dec_return_release(atomic64_t *v)
1682 {
1683         __atomic_release_fence();
1684         return arch_atomic64_dec_return_relaxed(v);
1685 }
1686 #define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1687 #endif
1688
1689 #ifndef arch_atomic64_dec_return
1690 static __always_inline s64
1691 arch_atomic64_dec_return(atomic64_t *v)
1692 {
1693         s64 ret;
1694         __atomic_pre_full_fence();
1695         ret = arch_atomic64_dec_return_relaxed(v);
1696         __atomic_post_full_fence();
1697         return ret;
1698 }
1699 #define arch_atomic64_dec_return arch_atomic64_dec_return
1700 #endif
1701
1702 #endif /* arch_atomic64_dec_return_relaxed */
1703
1704 #ifndef arch_atomic64_fetch_dec_relaxed
1705 #ifdef arch_atomic64_fetch_dec
1706 #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec
1707 #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec
1708 #define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec
1709 #endif /* arch_atomic64_fetch_dec */
1710
1711 #ifndef arch_atomic64_fetch_dec
1712 static __always_inline s64
1713 arch_atomic64_fetch_dec(atomic64_t *v)
1714 {
1715         return arch_atomic64_fetch_sub(1, v);
1716 }
1717 #define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1718 #endif
1719
1720 #ifndef arch_atomic64_fetch_dec_acquire
1721 static __always_inline s64
1722 arch_atomic64_fetch_dec_acquire(atomic64_t *v)
1723 {
1724         return arch_atomic64_fetch_sub_acquire(1, v);
1725 }
1726 #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
1727 #endif
1728
1729 #ifndef arch_atomic64_fetch_dec_release
1730 static __always_inline s64
1731 arch_atomic64_fetch_dec_release(atomic64_t *v)
1732 {
1733         return arch_atomic64_fetch_sub_release(1, v);
1734 }
1735 #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1736 #endif
1737
1738 #ifndef arch_atomic64_fetch_dec_relaxed
1739 static __always_inline s64
1740 arch_atomic64_fetch_dec_relaxed(atomic64_t *v)
1741 {
1742         return arch_atomic64_fetch_sub_relaxed(1, v);
1743 }
1744 #define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed
1745 #endif
1746
1747 #else /* arch_atomic64_fetch_dec_relaxed */
1748
1749 #ifndef arch_atomic64_fetch_dec_acquire
1750 static __always_inline s64
1751 arch_atomic64_fetch_dec_acquire(atomic64_t *v)
1752 {
1753         s64 ret = arch_atomic64_fetch_dec_relaxed(v);
1754         __atomic_acquire_fence();
1755         return ret;
1756 }
1757 #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
1758 #endif
1759
1760 #ifndef arch_atomic64_fetch_dec_release
1761 static __always_inline s64
1762 arch_atomic64_fetch_dec_release(atomic64_t *v)
1763 {
1764         __atomic_release_fence();
1765         return arch_atomic64_fetch_dec_relaxed(v);
1766 }
1767 #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1768 #endif
1769
1770 #ifndef arch_atomic64_fetch_dec
1771 static __always_inline s64
1772 arch_atomic64_fetch_dec(atomic64_t *v)
1773 {
1774         s64 ret;
1775         __atomic_pre_full_fence();
1776         ret = arch_atomic64_fetch_dec_relaxed(v);
1777         __atomic_post_full_fence();
1778         return ret;
1779 }
1780 #define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1781 #endif
1782
1783 #endif /* arch_atomic64_fetch_dec_relaxed */
1784
1785 #ifndef arch_atomic64_fetch_and_relaxed
1786 #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and
1787 #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and
1788 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and
1789 #else /* arch_atomic64_fetch_and_relaxed */
1790
1791 #ifndef arch_atomic64_fetch_and_acquire
1792 static __always_inline s64
1793 arch_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
1794 {
1795         s64 ret = arch_atomic64_fetch_and_relaxed(i, v);
1796         __atomic_acquire_fence();
1797         return ret;
1798 }
1799 #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire
1800 #endif
1801
1802 #ifndef arch_atomic64_fetch_and_release
1803 static __always_inline s64
1804 arch_atomic64_fetch_and_release(s64 i, atomic64_t *v)
1805 {
1806         __atomic_release_fence();
1807         return arch_atomic64_fetch_and_relaxed(i, v);
1808 }
1809 #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and_release
1810 #endif
1811
1812 #ifndef arch_atomic64_fetch_and
1813 static __always_inline s64
1814 arch_atomic64_fetch_and(s64 i, atomic64_t *v)
1815 {
1816         s64 ret;
1817         __atomic_pre_full_fence();
1818         ret = arch_atomic64_fetch_and_relaxed(i, v);
1819         __atomic_post_full_fence();
1820         return ret;
1821 }
1822 #define arch_atomic64_fetch_and arch_atomic64_fetch_and
1823 #endif
1824
1825 #endif /* arch_atomic64_fetch_and_relaxed */
1826
1827 #ifndef arch_atomic64_andnot
1828 static __always_inline void
1829 arch_atomic64_andnot(s64 i, atomic64_t *v)
1830 {
1831         arch_atomic64_and(~i, v);
1832 }
1833 #define arch_atomic64_andnot arch_atomic64_andnot
1834 #endif
1835
1836 #ifndef arch_atomic64_fetch_andnot_relaxed
1837 #ifdef arch_atomic64_fetch_andnot
1838 #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot
1839 #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot
1840 #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot
1841 #endif /* arch_atomic64_fetch_andnot */
1842
1843 #ifndef arch_atomic64_fetch_andnot
1844 static __always_inline s64
1845 arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
1846 {
1847         return arch_atomic64_fetch_and(~i, v);
1848 }
1849 #define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
1850 #endif
1851
1852 #ifndef arch_atomic64_fetch_andnot_acquire
1853 static __always_inline s64
1854 arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1855 {
1856         return arch_atomic64_fetch_and_acquire(~i, v);
1857 }
1858 #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
1859 #endif
1860
1861 #ifndef arch_atomic64_fetch_andnot_release
1862 static __always_inline s64
1863 arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1864 {
1865         return arch_atomic64_fetch_and_release(~i, v);
1866 }
1867 #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
1868 #endif
1869
1870 #ifndef arch_atomic64_fetch_andnot_relaxed
1871 static __always_inline s64
1872 arch_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1873 {
1874         return arch_atomic64_fetch_and_relaxed(~i, v);
1875 }
1876 #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed
1877 #endif
1878
1879 #else /* arch_atomic64_fetch_andnot_relaxed */
1880
1881 #ifndef arch_atomic64_fetch_andnot_acquire
1882 static __always_inline s64
1883 arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1884 {
1885         s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
1886         __atomic_acquire_fence();
1887         return ret;
1888 }
1889 #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
1890 #endif
1891
1892 #ifndef arch_atomic64_fetch_andnot_release
1893 static __always_inline s64
1894 arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1895 {
1896         __atomic_release_fence();
1897         return arch_atomic64_fetch_andnot_relaxed(i, v);
1898 }
1899 #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
1900 #endif
1901
1902 #ifndef arch_atomic64_fetch_andnot
1903 static __always_inline s64
1904 arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
1905 {
1906         s64 ret;
1907         __atomic_pre_full_fence();
1908         ret = arch_atomic64_fetch_andnot_relaxed(i, v);
1909         __atomic_post_full_fence();
1910         return ret;
1911 }
1912 #define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
1913 #endif
1914
1915 #endif /* arch_atomic64_fetch_andnot_relaxed */
1916
1917 #ifndef arch_atomic64_fetch_or_relaxed
1918 #define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or
1919 #define arch_atomic64_fetch_or_release arch_atomic64_fetch_or
1920 #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or
1921 #else /* arch_atomic64_fetch_or_relaxed */
1922
1923 #ifndef arch_atomic64_fetch_or_acquire
1924 static __always_inline s64
1925 arch_atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1926 {
1927         s64 ret = arch_atomic64_fetch_or_relaxed(i, v);
1928         __atomic_acquire_fence();
1929         return ret;
1930 }
1931 #define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire
1932 #endif
1933
1934 #ifndef arch_atomic64_fetch_or_release
1935 static __always_inline s64
1936 arch_atomic64_fetch_or_release(s64 i, atomic64_t *v)
1937 {
1938         __atomic_release_fence();
1939         return arch_atomic64_fetch_or_relaxed(i, v);
1940 }
1941 #define arch_atomic64_fetch_or_release arch_atomic64_fetch_or_release
1942 #endif
1943
1944 #ifndef arch_atomic64_fetch_or
1945 static __always_inline s64
1946 arch_atomic64_fetch_or(s64 i, atomic64_t *v)
1947 {
1948         s64 ret;
1949         __atomic_pre_full_fence();
1950         ret = arch_atomic64_fetch_or_relaxed(i, v);
1951         __atomic_post_full_fence();
1952         return ret;
1953 }
1954 #define arch_atomic64_fetch_or arch_atomic64_fetch_or
1955 #endif
1956
1957 #endif /* arch_atomic64_fetch_or_relaxed */
1958
1959 #ifndef arch_atomic64_fetch_xor_relaxed
1960 #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor
1961 #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor
1962 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor
1963 #else /* arch_atomic64_fetch_xor_relaxed */
1964
1965 #ifndef arch_atomic64_fetch_xor_acquire
1966 static __always_inline s64
1967 arch_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1968 {
1969         s64 ret = arch_atomic64_fetch_xor_relaxed(i, v);
1970         __atomic_acquire_fence();
1971         return ret;
1972 }
1973 #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire
1974 #endif
1975
1976 #ifndef arch_atomic64_fetch_xor_release
1977 static __always_inline s64
1978 arch_atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1979 {
1980         __atomic_release_fence();
1981         return arch_atomic64_fetch_xor_relaxed(i, v);
1982 }
1983 #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release
1984 #endif
1985
1986 #ifndef arch_atomic64_fetch_xor
1987 static __always_inline s64
1988 arch_atomic64_fetch_xor(s64 i, atomic64_t *v)
1989 {
1990         s64 ret;
1991         __atomic_pre_full_fence();
1992         ret = arch_atomic64_fetch_xor_relaxed(i, v);
1993         __atomic_post_full_fence();
1994         return ret;
1995 }
1996 #define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
1997 #endif
1998
1999 #endif /* arch_atomic64_fetch_xor_relaxed */
2000
2001 #ifndef arch_atomic64_xchg_relaxed
2002 #define arch_atomic64_xchg_acquire arch_atomic64_xchg
2003 #define arch_atomic64_xchg_release arch_atomic64_xchg
2004 #define arch_atomic64_xchg_relaxed arch_atomic64_xchg
2005 #else /* arch_atomic64_xchg_relaxed */
2006
2007 #ifndef arch_atomic64_xchg_acquire
2008 static __always_inline s64
2009 arch_atomic64_xchg_acquire(atomic64_t *v, s64 i)
2010 {
2011         s64 ret = arch_atomic64_xchg_relaxed(v, i);
2012         __atomic_acquire_fence();
2013         return ret;
2014 }
2015 #define arch_atomic64_xchg_acquire arch_atomic64_xchg_acquire
2016 #endif
2017
2018 #ifndef arch_atomic64_xchg_release
2019 static __always_inline s64
2020 arch_atomic64_xchg_release(atomic64_t *v, s64 i)
2021 {
2022         __atomic_release_fence();
2023         return arch_atomic64_xchg_relaxed(v, i);
2024 }
2025 #define arch_atomic64_xchg_release arch_atomic64_xchg_release
2026 #endif
2027
2028 #ifndef arch_atomic64_xchg
2029 static __always_inline s64
2030 arch_atomic64_xchg(atomic64_t *v, s64 i)
2031 {
2032         s64 ret;
2033         __atomic_pre_full_fence();
2034         ret = arch_atomic64_xchg_relaxed(v, i);
2035         __atomic_post_full_fence();
2036         return ret;
2037 }
2038 #define arch_atomic64_xchg arch_atomic64_xchg
2039 #endif
2040
2041 #endif /* arch_atomic64_xchg_relaxed */
2042
2043 #ifndef arch_atomic64_cmpxchg_relaxed
2044 #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg
2045 #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg
2046 #define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg
2047 #else /* arch_atomic64_cmpxchg_relaxed */
2048
2049 #ifndef arch_atomic64_cmpxchg_acquire
2050 static __always_inline s64
2051 arch_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
2052 {
2053         s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
2054         __atomic_acquire_fence();
2055         return ret;
2056 }
2057 #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire
2058 #endif
2059
2060 #ifndef arch_atomic64_cmpxchg_release
2061 static __always_inline s64
2062 arch_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
2063 {
2064         __atomic_release_fence();
2065         return arch_atomic64_cmpxchg_relaxed(v, old, new);
2066 }
2067 #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release
2068 #endif
2069
2070 #ifndef arch_atomic64_cmpxchg
2071 static __always_inline s64
2072 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
2073 {
2074         s64 ret;
2075         __atomic_pre_full_fence();
2076         ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
2077         __atomic_post_full_fence();
2078         return ret;
2079 }
2080 #define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
2081 #endif
2082
2083 #endif /* arch_atomic64_cmpxchg_relaxed */
2084
2085 #ifndef arch_atomic64_try_cmpxchg_relaxed
2086 #ifdef arch_atomic64_try_cmpxchg
2087 #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg
2088 #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg
2089 #define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg
2090 #endif /* arch_atomic64_try_cmpxchg */
2091
2092 #ifndef arch_atomic64_try_cmpxchg
2093 static __always_inline bool
2094 arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2095 {
2096         s64 r, o = *old;
2097         r = arch_atomic64_cmpxchg(v, o, new);
2098         if (unlikely(r != o))
2099                 *old = r;
2100         return likely(r == o);
2101 }
2102 #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
2103 #endif
2104
2105 #ifndef arch_atomic64_try_cmpxchg_acquire
2106 static __always_inline bool
2107 arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2108 {
2109         s64 r, o = *old;
2110         r = arch_atomic64_cmpxchg_acquire(v, o, new);
2111         if (unlikely(r != o))
2112                 *old = r;
2113         return likely(r == o);
2114 }
2115 #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
2116 #endif
2117
2118 #ifndef arch_atomic64_try_cmpxchg_release
2119 static __always_inline bool
2120 arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2121 {
2122         s64 r, o = *old;
2123         r = arch_atomic64_cmpxchg_release(v, o, new);
2124         if (unlikely(r != o))
2125                 *old = r;
2126         return likely(r == o);
2127 }
2128 #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
2129 #endif
2130
2131 #ifndef arch_atomic64_try_cmpxchg_relaxed
2132 static __always_inline bool
2133 arch_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
2134 {
2135         s64 r, o = *old;
2136         r = arch_atomic64_cmpxchg_relaxed(v, o, new);
2137         if (unlikely(r != o))
2138                 *old = r;
2139         return likely(r == o);
2140 }
2141 #define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg_relaxed
2142 #endif
2143
2144 #else /* arch_atomic64_try_cmpxchg_relaxed */
2145
2146 #ifndef arch_atomic64_try_cmpxchg_acquire
2147 static __always_inline bool
2148 arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2149 {
2150         bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2151         __atomic_acquire_fence();
2152         return ret;
2153 }
2154 #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
2155 #endif
2156
2157 #ifndef arch_atomic64_try_cmpxchg_release
2158 static __always_inline bool
2159 arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2160 {
2161         __atomic_release_fence();
2162         return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2163 }
2164 #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
2165 #endif
2166
2167 #ifndef arch_atomic64_try_cmpxchg
2168 static __always_inline bool
2169 arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2170 {
2171         bool ret;
2172         __atomic_pre_full_fence();
2173         ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2174         __atomic_post_full_fence();
2175         return ret;
2176 }
2177 #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
2178 #endif
2179
2180 #endif /* arch_atomic64_try_cmpxchg_relaxed */
2181
2182 #ifndef arch_atomic64_sub_and_test
2183 /**
2184  * arch_atomic64_sub_and_test - subtract value from variable and test result
2185  * @i: integer value to subtract
2186  * @v: pointer of type atomic64_t
2187  *
2188  * Atomically subtracts @i from @v and returns
2189  * true if the result is zero, or false for all
2190  * other cases.
2191  */
2192 static __always_inline bool
2193 arch_atomic64_sub_and_test(s64 i, atomic64_t *v)
2194 {
2195         return arch_atomic64_sub_return(i, v) == 0;
2196 }
2197 #define arch_atomic64_sub_and_test arch_atomic64_sub_and_test
2198 #endif
2199
2200 #ifndef arch_atomic64_dec_and_test
2201 /**
2202  * arch_atomic64_dec_and_test - decrement and test
2203  * @v: pointer of type atomic64_t
2204  *
2205  * Atomically decrements @v by 1 and
2206  * returns true if the result is 0, or false for all other
2207  * cases.
2208  */
2209 static __always_inline bool
2210 arch_atomic64_dec_and_test(atomic64_t *v)
2211 {
2212         return arch_atomic64_dec_return(v) == 0;
2213 }
2214 #define arch_atomic64_dec_and_test arch_atomic64_dec_and_test
2215 #endif
2216
2217 #ifndef arch_atomic64_inc_and_test
2218 /**
2219  * arch_atomic64_inc_and_test - increment and test
2220  * @v: pointer of type atomic64_t
2221  *
2222  * Atomically increments @v by 1
2223  * and returns true if the result is zero, or false for all
2224  * other cases.
2225  */
2226 static __always_inline bool
2227 arch_atomic64_inc_and_test(atomic64_t *v)
2228 {
2229         return arch_atomic64_inc_return(v) == 0;
2230 }
2231 #define arch_atomic64_inc_and_test arch_atomic64_inc_and_test
2232 #endif
2233
2234 #ifndef arch_atomic64_add_negative
2235 /**
2236  * arch_atomic64_add_negative - add and test if negative
2237  * @i: integer value to add
2238  * @v: pointer of type atomic64_t
2239  *
2240  * Atomically adds @i to @v and returns true
2241  * if the result is negative, or false when
2242  * result is greater than or equal to zero.
2243  */
2244 static __always_inline bool
2245 arch_atomic64_add_negative(s64 i, atomic64_t *v)
2246 {
2247         return arch_atomic64_add_return(i, v) < 0;
2248 }
2249 #define arch_atomic64_add_negative arch_atomic64_add_negative
2250 #endif
2251
2252 #ifndef arch_atomic64_fetch_add_unless
2253 /**
2254  * arch_atomic64_fetch_add_unless - add unless the number is already a given value
2255  * @v: pointer of type atomic64_t
2256  * @a: the amount to add to v...
2257  * @u: ...unless v is equal to u.
2258  *
2259  * Atomically adds @a to @v, so long as @v was not already @u.
2260  * Returns original value of @v
2261  */
2262 static __always_inline s64
2263 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
2264 {
2265         s64 c = arch_atomic64_read(v);
2266
2267         do {
2268                 if (unlikely(c == u))
2269                         break;
2270         } while (!arch_atomic64_try_cmpxchg(v, &c, c + a));
2271
2272         return c;
2273 }
2274 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
2275 #endif
2276
2277 #ifndef arch_atomic64_add_unless
2278 /**
2279  * arch_atomic64_add_unless - add unless the number is already a given value
2280  * @v: pointer of type atomic64_t
2281  * @a: the amount to add to v...
2282  * @u: ...unless v is equal to u.
2283  *
2284  * Atomically adds @a to @v, if @v was not already @u.
2285  * Returns true if the addition was done.
2286  */
2287 static __always_inline bool
2288 arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
2289 {
2290         return arch_atomic64_fetch_add_unless(v, a, u) != u;
2291 }
2292 #define arch_atomic64_add_unless arch_atomic64_add_unless
2293 #endif
2294
2295 #ifndef arch_atomic64_inc_not_zero
2296 /**
2297  * arch_atomic64_inc_not_zero - increment unless the number is zero
2298  * @v: pointer of type atomic64_t
2299  *
2300  * Atomically increments @v by 1, if @v is non-zero.
2301  * Returns true if the increment was done.
2302  */
2303 static __always_inline bool
2304 arch_atomic64_inc_not_zero(atomic64_t *v)
2305 {
2306         return arch_atomic64_add_unless(v, 1, 0);
2307 }
2308 #define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero
2309 #endif
2310
2311 #ifndef arch_atomic64_inc_unless_negative
2312 static __always_inline bool
2313 arch_atomic64_inc_unless_negative(atomic64_t *v)
2314 {
2315         s64 c = arch_atomic64_read(v);
2316
2317         do {
2318                 if (unlikely(c < 0))
2319                         return false;
2320         } while (!arch_atomic64_try_cmpxchg(v, &c, c + 1));
2321
2322         return true;
2323 }
2324 #define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative
2325 #endif
2326
2327 #ifndef arch_atomic64_dec_unless_positive
2328 static __always_inline bool
2329 arch_atomic64_dec_unless_positive(atomic64_t *v)
2330 {
2331         s64 c = arch_atomic64_read(v);
2332
2333         do {
2334                 if (unlikely(c > 0))
2335                         return false;
2336         } while (!arch_atomic64_try_cmpxchg(v, &c, c - 1));
2337
2338         return true;
2339 }
2340 #define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive
2341 #endif
2342
2343 #ifndef arch_atomic64_dec_if_positive
2344 static __always_inline s64
2345 arch_atomic64_dec_if_positive(atomic64_t *v)
2346 {
2347         s64 dec, c = arch_atomic64_read(v);
2348
2349         do {
2350                 dec = c - 1;
2351                 if (unlikely(dec < 0))
2352                         break;
2353         } while (!arch_atomic64_try_cmpxchg(v, &c, dec));
2354
2355         return dec;
2356 }
2357 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
2358 #endif
2359
2360 #endif /* _LINUX_ATOMIC_FALLBACK_H */
2361 // cca554917d7ea73d5e3e7397dd70c484cad9b2c4