include/linux/atomic-arch-fallback.h

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 // Generated by scripts/atomic/gen-atomic-fallback.sh
   4 // DO NOT MODIFY THIS FILE DIRECTLY
   5
   6 #ifndef _LINUX_ATOMIC_FALLBACK_H
   7 #define _LINUX_ATOMIC_FALLBACK_H
   8
   9 #include <linux/compiler.h>
  10
  11 #ifndef arch_xchg_relaxed
  12 #define arch_xchg_relaxed               arch_xchg
  13 #define arch_xchg_acquire               arch_xchg
  14 #define arch_xchg_release               arch_xchg
  15 #else /* arch_xchg_relaxed */
  16
  17 #ifndef arch_xchg_acquire
  18 #define arch_xchg_acquire(...) \
  19         __atomic_op_acquire(arch_xchg, __VA_ARGS__)
  20 #endif
  21
  22 #ifndef arch_xchg_release
  23 #define arch_xchg_release(...) \
  24         __atomic_op_release(arch_xchg, __VA_ARGS__)
  25 #endif
  26
  27 #ifndef arch_xchg
  28 #define arch_xchg(...) \
  29         __atomic_op_fence(arch_xchg, __VA_ARGS__)
  30 #endif
  31
  32 #endif /* arch_xchg_relaxed */
  33
  34 #ifndef arch_cmpxchg_relaxed
  35 #define arch_cmpxchg_relaxed            arch_cmpxchg
  36 #define arch_cmpxchg_acquire            arch_cmpxchg
  37 #define arch_cmpxchg_release            arch_cmpxchg
  38 #else /* arch_cmpxchg_relaxed */
  39
  40 #ifndef arch_cmpxchg_acquire
  41 #define arch_cmpxchg_acquire(...) \
  42         __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)
  43 #endif
  44
  45 #ifndef arch_cmpxchg_release
  46 #define arch_cmpxchg_release(...) \
  47         __atomic_op_release(arch_cmpxchg, __VA_ARGS__)
  48 #endif
  49
  50 #ifndef arch_cmpxchg
  51 #define arch_cmpxchg(...) \
  52         __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
  53 #endif
  54
  55 #endif /* arch_cmpxchg_relaxed */
  56
  57 #ifndef arch_cmpxchg64_relaxed
  58 #define arch_cmpxchg64_relaxed          arch_cmpxchg64
  59 #define arch_cmpxchg64_acquire          arch_cmpxchg64
  60 #define arch_cmpxchg64_release          arch_cmpxchg64
  61 #else /* arch_cmpxchg64_relaxed */
  62
  63 #ifndef arch_cmpxchg64_acquire
  64 #define arch_cmpxchg64_acquire(...) \
  65         __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)
  66 #endif
  67
  68 #ifndef arch_cmpxchg64_release
  69 #define arch_cmpxchg64_release(...) \
  70         __atomic_op_release(arch_cmpxchg64, __VA_ARGS__)
  71 #endif
  72
  73 #ifndef arch_cmpxchg64
  74 #define arch_cmpxchg64(...) \
  75         __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
  76 #endif
  77
  78 #endif /* arch_cmpxchg64_relaxed */
  79
  80 #ifndef arch_atomic_read_acquire
  81 static __always_inline int
  82 arch_atomic_read_acquire(const atomic_t *v)
  83 {
  84         return smp_load_acquire(&(v)->counter);
  85 }
  86 #define arch_atomic_read_acquire arch_atomic_read_acquire
  87 #endif
  88
  89 #ifndef arch_atomic_set_release
  90 static __always_inline void
  91 arch_atomic_set_release(atomic_t *v, int i)
  92 {
  93         smp_store_release(&(v)->counter, i);
  94 }
  95 #define arch_atomic_set_release arch_atomic_set_release
  96 #endif
  97
  98 #ifndef arch_atomic_add_return_relaxed
  99 #define arch_atomic_add_return_acquire arch_atomic_add_return
 100 #define arch_atomic_add_return_release arch_atomic_add_return
 101 #define arch_atomic_add_return_relaxed arch_atomic_add_return
 102 #else /* arch_atomic_add_return_relaxed */
 103
 104 #ifndef arch_atomic_add_return_acquire
 105 static __always_inline int
 106 arch_atomic_add_return_acquire(int i, atomic_t *v)
 107 {
 108         int ret = arch_atomic_add_return_relaxed(i, v);
 109         __atomic_acquire_fence();
 110         return ret;
 111 }
 112 #define arch_atomic_add_return_acquire arch_atomic_add_return_acquire
 113 #endif
 114
 115 #ifndef arch_atomic_add_return_release
 116 static __always_inline int
 117 arch_atomic_add_return_release(int i, atomic_t *v)
 118 {
 119         __atomic_release_fence();
 120         return arch_atomic_add_return_relaxed(i, v);
 121 }
 122 #define arch_atomic_add_return_release arch_atomic_add_return_release
 123 #endif
 124
 125 #ifndef arch_atomic_add_return
 126 static __always_inline int
 127 arch_atomic_add_return(int i, atomic_t *v)
 128 {
 129         int ret;
 130         __atomic_pre_full_fence();
 131         ret = arch_atomic_add_return_relaxed(i, v);
 132         __atomic_post_full_fence();
 133         return ret;
 134 }
 135 #define arch_atomic_add_return arch_atomic_add_return
 136 #endif
 137
 138 #endif /* arch_atomic_add_return_relaxed */
 139
 140 #ifndef arch_atomic_fetch_add_relaxed
 141 #define arch_atomic_fetch_add_acquire arch_atomic_fetch_add
 142 #define arch_atomic_fetch_add_release arch_atomic_fetch_add
 143 #define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add
 144 #else /* arch_atomic_fetch_add_relaxed */
 145
 146 #ifndef arch_atomic_fetch_add_acquire
 147 static __always_inline int
 148 arch_atomic_fetch_add_acquire(int i, atomic_t *v)
 149 {
 150         int ret = arch_atomic_fetch_add_relaxed(i, v);
 151         __atomic_acquire_fence();
 152         return ret;
 153 }
 154 #define arch_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire
 155 #endif
 156
 157 #ifndef arch_atomic_fetch_add_release
 158 static __always_inline int
 159 arch_atomic_fetch_add_release(int i, atomic_t *v)
 160 {
 161         __atomic_release_fence();
 162         return arch_atomic_fetch_add_relaxed(i, v);
 163 }
 164 #define arch_atomic_fetch_add_release arch_atomic_fetch_add_release
 165 #endif
 166
 167 #ifndef arch_atomic_fetch_add
 168 static __always_inline int
 169 arch_atomic_fetch_add(int i, atomic_t *v)
 170 {
 171         int ret;
 172         __atomic_pre_full_fence();
 173         ret = arch_atomic_fetch_add_relaxed(i, v);
 174         __atomic_post_full_fence();
 175         return ret;
 176 }
 177 #define arch_atomic_fetch_add arch_atomic_fetch_add
 178 #endif
 179
 180 #endif /* arch_atomic_fetch_add_relaxed */
 181
 182 #ifndef arch_atomic_sub_return_relaxed
 183 #define arch_atomic_sub_return_acquire arch_atomic_sub_return
 184 #define arch_atomic_sub_return_release arch_atomic_sub_return
 185 #define arch_atomic_sub_return_relaxed arch_atomic_sub_return
 186 #else /* arch_atomic_sub_return_relaxed */
 187
 188 #ifndef arch_atomic_sub_return_acquire
 189 static __always_inline int
 190 arch_atomic_sub_return_acquire(int i, atomic_t *v)
 191 {
 192         int ret = arch_atomic_sub_return_relaxed(i, v);
 193         __atomic_acquire_fence();
 194         return ret;
 195 }
 196 #define arch_atomic_sub_return_acquire arch_atomic_sub_return_acquire
 197 #endif
 198
 199 #ifndef arch_atomic_sub_return_release
 200 static __always_inline int
 201 arch_atomic_sub_return_release(int i, atomic_t *v)
 202 {
 203         __atomic_release_fence();
 204         return arch_atomic_sub_return_relaxed(i, v);
 205 }
 206 #define arch_atomic_sub_return_release arch_atomic_sub_return_release
 207 #endif
 208
 209 #ifndef arch_atomic_sub_return
 210 static __always_inline int
 211 arch_atomic_sub_return(int i, atomic_t *v)
 212 {
 213         int ret;
 214         __atomic_pre_full_fence();
 215         ret = arch_atomic_sub_return_relaxed(i, v);
 216         __atomic_post_full_fence();
 217         return ret;
 218 }
 219 #define arch_atomic_sub_return arch_atomic_sub_return
 220 #endif
 221
 222 #endif /* arch_atomic_sub_return_relaxed */
 223
 224 #ifndef arch_atomic_fetch_sub_relaxed
 225 #define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub
 226 #define arch_atomic_fetch_sub_release arch_atomic_fetch_sub
 227 #define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub
 228 #else /* arch_atomic_fetch_sub_relaxed */
 229
 230 #ifndef arch_atomic_fetch_sub_acquire
 231 static __always_inline int
 232 arch_atomic_fetch_sub_acquire(int i, atomic_t *v)
 233 {
 234         int ret = arch_atomic_fetch_sub_relaxed(i, v);
 235         __atomic_acquire_fence();
 236         return ret;
 237 }
 238 #define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire
 239 #endif
 240
 241 #ifndef arch_atomic_fetch_sub_release
 242 static __always_inline int
 243 arch_atomic_fetch_sub_release(int i, atomic_t *v)
 244 {
 245         __atomic_release_fence();
 246         return arch_atomic_fetch_sub_relaxed(i, v);
 247 }
 248 #define arch_atomic_fetch_sub_release arch_atomic_fetch_sub_release
 249 #endif
 250
 251 #ifndef arch_atomic_fetch_sub
 252 static __always_inline int
 253 arch_atomic_fetch_sub(int i, atomic_t *v)
 254 {
 255         int ret;
 256         __atomic_pre_full_fence();
 257         ret = arch_atomic_fetch_sub_relaxed(i, v);
 258         __atomic_post_full_fence();
 259         return ret;
 260 }
 261 #define arch_atomic_fetch_sub arch_atomic_fetch_sub
 262 #endif
 263
 264 #endif /* arch_atomic_fetch_sub_relaxed */
 265
 266 #ifndef arch_atomic_inc
 267 static __always_inline void
 268 arch_atomic_inc(atomic_t *v)
 269 {
 270         arch_atomic_add(1, v);
 271 }
 272 #define arch_atomic_inc arch_atomic_inc
 273 #endif
 274
 275 #ifndef arch_atomic_inc_return_relaxed
 276 #ifdef arch_atomic_inc_return
 277 #define arch_atomic_inc_return_acquire arch_atomic_inc_return
 278 #define arch_atomic_inc_return_release arch_atomic_inc_return
 279 #define arch_atomic_inc_return_relaxed arch_atomic_inc_return
 280 #endif /* arch_atomic_inc_return */
 281
 282 #ifndef arch_atomic_inc_return
 283 static __always_inline int
 284 arch_atomic_inc_return(atomic_t *v)
 285 {
 286         return arch_atomic_add_return(1, v);
 287 }
 288 #define arch_atomic_inc_return arch_atomic_inc_return
 289 #endif
 290
 291 #ifndef arch_atomic_inc_return_acquire
 292 static __always_inline int
 293 arch_atomic_inc_return_acquire(atomic_t *v)
 294 {
 295         return arch_atomic_add_return_acquire(1, v);
 296 }
 297 #define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
 298 #endif
 299
 300 #ifndef arch_atomic_inc_return_release
 301 static __always_inline int
 302 arch_atomic_inc_return_release(atomic_t *v)
 303 {
 304         return arch_atomic_add_return_release(1, v);
 305 }
 306 #define arch_atomic_inc_return_release arch_atomic_inc_return_release
 307 #endif
 308
 309 #ifndef arch_atomic_inc_return_relaxed
 310 static __always_inline int
 311 arch_atomic_inc_return_relaxed(atomic_t *v)
 312 {
 313         return arch_atomic_add_return_relaxed(1, v);
 314 }
 315 #define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed
 316 #endif
 317
 318 #else /* arch_atomic_inc_return_relaxed */
 319
 320 #ifndef arch_atomic_inc_return_acquire
 321 static __always_inline int
 322 arch_atomic_inc_return_acquire(atomic_t *v)
 323 {
 324         int ret = arch_atomic_inc_return_relaxed(v);
 325         __atomic_acquire_fence();
 326         return ret;
 327 }
 328 #define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
 329 #endif
 330
 331 #ifndef arch_atomic_inc_return_release
 332 static __always_inline int
 333 arch_atomic_inc_return_release(atomic_t *v)
 334 {
 335         __atomic_release_fence();
 336         return arch_atomic_inc_return_relaxed(v);
 337 }
 338 #define arch_atomic_inc_return_release arch_atomic_inc_return_release
 339 #endif
 340
 341 #ifndef arch_atomic_inc_return
 342 static __always_inline int
 343 arch_atomic_inc_return(atomic_t *v)
 344 {
 345         int ret;
 346         __atomic_pre_full_fence();
 347         ret = arch_atomic_inc_return_relaxed(v);
 348         __atomic_post_full_fence();
 349         return ret;
 350 }
 351 #define arch_atomic_inc_return arch_atomic_inc_return
 352 #endif
 353
 354 #endif /* arch_atomic_inc_return_relaxed */
 355
 356 #ifndef arch_atomic_fetch_inc_relaxed
 357 #ifdef arch_atomic_fetch_inc
 358 #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc
 359 #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc
 360 #define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc
 361 #endif /* arch_atomic_fetch_inc */
 362
 363 #ifndef arch_atomic_fetch_inc
 364 static __always_inline int
 365 arch_atomic_fetch_inc(atomic_t *v)
 366 {
 367         return arch_atomic_fetch_add(1, v);
 368 }
 369 #define arch_atomic_fetch_inc arch_atomic_fetch_inc
 370 #endif
 371
 372 #ifndef arch_atomic_fetch_inc_acquire
 373 static __always_inline int
 374 arch_atomic_fetch_inc_acquire(atomic_t *v)
 375 {
 376         return arch_atomic_fetch_add_acquire(1, v);
 377 }
 378 #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
 379 #endif
 380
 381 #ifndef arch_atomic_fetch_inc_release
 382 static __always_inline int
 383 arch_atomic_fetch_inc_release(atomic_t *v)
 384 {
 385         return arch_atomic_fetch_add_release(1, v);
 386 }
 387 #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
 388 #endif
 389
 390 #ifndef arch_atomic_fetch_inc_relaxed
 391 static __always_inline int
 392 arch_atomic_fetch_inc_relaxed(atomic_t *v)
 393 {
 394         return arch_atomic_fetch_add_relaxed(1, v);
 395 }
 396 #define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed
 397 #endif
 398
 399 #else /* arch_atomic_fetch_inc_relaxed */
 400
 401 #ifndef arch_atomic_fetch_inc_acquire
 402 static __always_inline int
 403 arch_atomic_fetch_inc_acquire(atomic_t *v)
 404 {
 405         int ret = arch_atomic_fetch_inc_relaxed(v);
 406         __atomic_acquire_fence();
 407         return ret;
 408 }
 409 #define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
 410 #endif
 411
 412 #ifndef arch_atomic_fetch_inc_release
 413 static __always_inline int
 414 arch_atomic_fetch_inc_release(atomic_t *v)
 415 {
 416         __atomic_release_fence();
 417         return arch_atomic_fetch_inc_relaxed(v);
 418 }
 419 #define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
 420 #endif
 421
 422 #ifndef arch_atomic_fetch_inc
 423 static __always_inline int
 424 arch_atomic_fetch_inc(atomic_t *v)
 425 {
 426         int ret;
 427         __atomic_pre_full_fence();
 428         ret = arch_atomic_fetch_inc_relaxed(v);
 429         __atomic_post_full_fence();
 430         return ret;
 431 }
 432 #define arch_atomic_fetch_inc arch_atomic_fetch_inc
 433 #endif
 434
 435 #endif /* arch_atomic_fetch_inc_relaxed */
 436
 437 #ifndef arch_atomic_dec
 438 static __always_inline void
 439 arch_atomic_dec(atomic_t *v)
 440 {
 441         arch_atomic_sub(1, v);
 442 }
 443 #define arch_atomic_dec arch_atomic_dec
 444 #endif
 445
 446 #ifndef arch_atomic_dec_return_relaxed
 447 #ifdef arch_atomic_dec_return
 448 #define arch_atomic_dec_return_acquire arch_atomic_dec_return
 449 #define arch_atomic_dec_return_release arch_atomic_dec_return
 450 #define arch_atomic_dec_return_relaxed arch_atomic_dec_return
 451 #endif /* arch_atomic_dec_return */
 452
 453 #ifndef arch_atomic_dec_return
 454 static __always_inline int
 455 arch_atomic_dec_return(atomic_t *v)
 456 {
 457         return arch_atomic_sub_return(1, v);
 458 }
 459 #define arch_atomic_dec_return arch_atomic_dec_return
 460 #endif
 461
 462 #ifndef arch_atomic_dec_return_acquire
 463 static __always_inline int
 464 arch_atomic_dec_return_acquire(atomic_t *v)
 465 {
 466         return arch_atomic_sub_return_acquire(1, v);
 467 }
 468 #define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
 469 #endif
 470
 471 #ifndef arch_atomic_dec_return_release
 472 static __always_inline int
 473 arch_atomic_dec_return_release(atomic_t *v)
 474 {
 475         return arch_atomic_sub_return_release(1, v);
 476 }
 477 #define arch_atomic_dec_return_release arch_atomic_dec_return_release
 478 #endif
 479
 480 #ifndef arch_atomic_dec_return_relaxed
 481 static __always_inline int
 482 arch_atomic_dec_return_relaxed(atomic_t *v)
 483 {
 484         return arch_atomic_sub_return_relaxed(1, v);
 485 }
 486 #define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed
 487 #endif
 488
 489 #else /* arch_atomic_dec_return_relaxed */
 490
 491 #ifndef arch_atomic_dec_return_acquire
 492 static __always_inline int
 493 arch_atomic_dec_return_acquire(atomic_t *v)
 494 {
 495         int ret = arch_atomic_dec_return_relaxed(v);
 496         __atomic_acquire_fence();
 497         return ret;
 498 }
 499 #define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
 500 #endif
 501
 502 #ifndef arch_atomic_dec_return_release
 503 static __always_inline int
 504 arch_atomic_dec_return_release(atomic_t *v)
 505 {
 506         __atomic_release_fence();
 507         return arch_atomic_dec_return_relaxed(v);
 508 }
 509 #define arch_atomic_dec_return_release arch_atomic_dec_return_release
 510 #endif
 511
 512 #ifndef arch_atomic_dec_return
 513 static __always_inline int
 514 arch_atomic_dec_return(atomic_t *v)
 515 {
 516         int ret;
 517         __atomic_pre_full_fence();
 518         ret = arch_atomic_dec_return_relaxed(v);
 519         __atomic_post_full_fence();
 520         return ret;
 521 }
 522 #define arch_atomic_dec_return arch_atomic_dec_return
 523 #endif
 524
 525 #endif /* arch_atomic_dec_return_relaxed */
 526
 527 #ifndef arch_atomic_fetch_dec_relaxed
 528 #ifdef arch_atomic_fetch_dec
 529 #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec
 530 #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec
 531 #define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec
 532 #endif /* arch_atomic_fetch_dec */
 533
 534 #ifndef arch_atomic_fetch_dec
 535 static __always_inline int
 536 arch_atomic_fetch_dec(atomic_t *v)
 537 {
 538         return arch_atomic_fetch_sub(1, v);
 539 }
 540 #define arch_atomic_fetch_dec arch_atomic_fetch_dec
 541 #endif
 542
 543 #ifndef arch_atomic_fetch_dec_acquire
 544 static __always_inline int
 545 arch_atomic_fetch_dec_acquire(atomic_t *v)
 546 {
 547         return arch_atomic_fetch_sub_acquire(1, v);
 548 }
 549 #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
 550 #endif
 551
 552 #ifndef arch_atomic_fetch_dec_release
 553 static __always_inline int
 554 arch_atomic_fetch_dec_release(atomic_t *v)
 555 {
 556         return arch_atomic_fetch_sub_release(1, v);
 557 }
 558 #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
 559 #endif
 560
 561 #ifndef arch_atomic_fetch_dec_relaxed
 562 static __always_inline int
 563 arch_atomic_fetch_dec_relaxed(atomic_t *v)
 564 {
 565         return arch_atomic_fetch_sub_relaxed(1, v);
 566 }
 567 #define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed
 568 #endif
 569
 570 #else /* arch_atomic_fetch_dec_relaxed */
 571
 572 #ifndef arch_atomic_fetch_dec_acquire
 573 static __always_inline int
 574 arch_atomic_fetch_dec_acquire(atomic_t *v)
 575 {
 576         int ret = arch_atomic_fetch_dec_relaxed(v);
 577         __atomic_acquire_fence();
 578         return ret;
 579 }
 580 #define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
 581 #endif
 582
 583 #ifndef arch_atomic_fetch_dec_release
 584 static __always_inline int
 585 arch_atomic_fetch_dec_release(atomic_t *v)
 586 {
 587         __atomic_release_fence();
 588         return arch_atomic_fetch_dec_relaxed(v);
 589 }
 590 #define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
 591 #endif
 592
 593 #ifndef arch_atomic_fetch_dec
 594 static __always_inline int
 595 arch_atomic_fetch_dec(atomic_t *v)
 596 {
 597         int ret;
 598         __atomic_pre_full_fence();
 599         ret = arch_atomic_fetch_dec_relaxed(v);
 600         __atomic_post_full_fence();
 601         return ret;
 602 }
 603 #define arch_atomic_fetch_dec arch_atomic_fetch_dec
 604 #endif
 605
 606 #endif /* arch_atomic_fetch_dec_relaxed */
 607
 608 #ifndef arch_atomic_fetch_and_relaxed
 609 #define arch_atomic_fetch_and_acquire arch_atomic_fetch_and
 610 #define arch_atomic_fetch_and_release arch_atomic_fetch_and
 611 #define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and
 612 #else /* arch_atomic_fetch_and_relaxed */
 613
 614 #ifndef arch_atomic_fetch_and_acquire
 615 static __always_inline int
 616 arch_atomic_fetch_and_acquire(int i, atomic_t *v)
 617 {
 618         int ret = arch_atomic_fetch_and_relaxed(i, v);
 619         __atomic_acquire_fence();
 620         return ret;
 621 }
 622 #define arch_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire
 623 #endif
 624
 625 #ifndef arch_atomic_fetch_and_release
 626 static __always_inline int
 627 arch_atomic_fetch_and_release(int i, atomic_t *v)
 628 {
 629         __atomic_release_fence();
 630         return arch_atomic_fetch_and_relaxed(i, v);
 631 }
 632 #define arch_atomic_fetch_and_release arch_atomic_fetch_and_release
 633 #endif
 634
 635 #ifndef arch_atomic_fetch_and
 636 static __always_inline int
 637 arch_atomic_fetch_and(int i, atomic_t *v)
 638 {
 639         int ret;
 640         __atomic_pre_full_fence();
 641         ret = arch_atomic_fetch_and_relaxed(i, v);
 642         __atomic_post_full_fence();
 643         return ret;
 644 }
 645 #define arch_atomic_fetch_and arch_atomic_fetch_and
 646 #endif
 647
 648 #endif /* arch_atomic_fetch_and_relaxed */
 649
 650 #ifndef arch_atomic_andnot
 651 static __always_inline void
 652 arch_atomic_andnot(int i, atomic_t *v)
 653 {
 654         arch_atomic_and(~i, v);
 655 }
 656 #define arch_atomic_andnot arch_atomic_andnot
 657 #endif
 658
 659 #ifndef arch_atomic_fetch_andnot_relaxed
 660 #ifdef arch_atomic_fetch_andnot
 661 #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot
 662 #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot
 663 #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot
 664 #endif /* arch_atomic_fetch_andnot */
 665
 666 #ifndef arch_atomic_fetch_andnot
 667 static __always_inline int
 668 arch_atomic_fetch_andnot(int i, atomic_t *v)
 669 {
 670         return arch_atomic_fetch_and(~i, v);
 671 }
 672 #define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
 673 #endif
 674
 675 #ifndef arch_atomic_fetch_andnot_acquire
 676 static __always_inline int
 677 arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
 678 {
 679         return arch_atomic_fetch_and_acquire(~i, v);
 680 }
 681 #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
 682 #endif
 683
 684 #ifndef arch_atomic_fetch_andnot_release
 685 static __always_inline int
 686 arch_atomic_fetch_andnot_release(int i, atomic_t *v)
 687 {
 688         return arch_atomic_fetch_and_release(~i, v);
 689 }
 690 #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
 691 #endif
 692
 693 #ifndef arch_atomic_fetch_andnot_relaxed
 694 static __always_inline int
 695 arch_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
 696 {
 697         return arch_atomic_fetch_and_relaxed(~i, v);
 698 }
 699 #define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed
 700 #endif
 701
 702 #else /* arch_atomic_fetch_andnot_relaxed */
 703
 704 #ifndef arch_atomic_fetch_andnot_acquire
 705 static __always_inline int
 706 arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
 707 {
 708         int ret = arch_atomic_fetch_andnot_relaxed(i, v);
 709         __atomic_acquire_fence();
 710         return ret;
 711 }
 712 #define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
 713 #endif
 714
 715 #ifndef arch_atomic_fetch_andnot_release
 716 static __always_inline int
 717 arch_atomic_fetch_andnot_release(int i, atomic_t *v)
 718 {
 719         __atomic_release_fence();
 720         return arch_atomic_fetch_andnot_relaxed(i, v);
 721 }
 722 #define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
 723 #endif
 724
 725 #ifndef arch_atomic_fetch_andnot
 726 static __always_inline int
 727 arch_atomic_fetch_andnot(int i, atomic_t *v)
 728 {
 729         int ret;
 730         __atomic_pre_full_fence();
 731         ret = arch_atomic_fetch_andnot_relaxed(i, v);
 732         __atomic_post_full_fence();
 733         return ret;
 734 }
 735 #define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
 736 #endif
 737
 738 #endif /* arch_atomic_fetch_andnot_relaxed */
 739
 740 #ifndef arch_atomic_fetch_or_relaxed
 741 #define arch_atomic_fetch_or_acquire arch_atomic_fetch_or
 742 #define arch_atomic_fetch_or_release arch_atomic_fetch_or
 743 #define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or
 744 #else /* arch_atomic_fetch_or_relaxed */
 745
 746 #ifndef arch_atomic_fetch_or_acquire
 747 static __always_inline int
 748 arch_atomic_fetch_or_acquire(int i, atomic_t *v)
 749 {
 750         int ret = arch_atomic_fetch_or_relaxed(i, v);
 751         __atomic_acquire_fence();
 752         return ret;
 753 }
 754 #define arch_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire
 755 #endif
 756
 757 #ifndef arch_atomic_fetch_or_release
 758 static __always_inline int
 759 arch_atomic_fetch_or_release(int i, atomic_t *v)
 760 {
 761         __atomic_release_fence();
 762         return arch_atomic_fetch_or_relaxed(i, v);
 763 }
 764 #define arch_atomic_fetch_or_release arch_atomic_fetch_or_release
 765 #endif
 766
 767 #ifndef arch_atomic_fetch_or
 768 static __always_inline int
 769 arch_atomic_fetch_or(int i, atomic_t *v)
 770 {
 771         int ret;
 772         __atomic_pre_full_fence();
 773         ret = arch_atomic_fetch_or_relaxed(i, v);
 774         __atomic_post_full_fence();
 775         return ret;
 776 }
 777 #define arch_atomic_fetch_or arch_atomic_fetch_or
 778 #endif
 779
 780 #endif /* arch_atomic_fetch_or_relaxed */
 781
 782 #ifndef arch_atomic_fetch_xor_relaxed
 783 #define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor
 784 #define arch_atomic_fetch_xor_release arch_atomic_fetch_xor
 785 #define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor
 786 #else /* arch_atomic_fetch_xor_relaxed */
 787
 788 #ifndef arch_atomic_fetch_xor_acquire
 789 static __always_inline int
 790 arch_atomic_fetch_xor_acquire(int i, atomic_t *v)
 791 {
 792         int ret = arch_atomic_fetch_xor_relaxed(i, v);
 793         __atomic_acquire_fence();
 794         return ret;
 795 }
 796 #define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire
 797 #endif
 798
 799 #ifndef arch_atomic_fetch_xor_release
 800 static __always_inline int
 801 arch_atomic_fetch_xor_release(int i, atomic_t *v)
 802 {
 803         __atomic_release_fence();
 804         return arch_atomic_fetch_xor_relaxed(i, v);
 805 }
 806 #define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release
 807 #endif
 808
 809 #ifndef arch_atomic_fetch_xor
 810 static __always_inline int
 811 arch_atomic_fetch_xor(int i, atomic_t *v)
 812 {
 813         int ret;
 814         __atomic_pre_full_fence();
 815         ret = arch_atomic_fetch_xor_relaxed(i, v);
 816         __atomic_post_full_fence();
 817         return ret;
 818 }
 819 #define arch_atomic_fetch_xor arch_atomic_fetch_xor
 820 #endif
 821
 822 #endif /* arch_atomic_fetch_xor_relaxed */
 823
 824 #ifndef arch_atomic_xchg_relaxed
 825 #define arch_atomic_xchg_acquire arch_atomic_xchg
 826 #define arch_atomic_xchg_release arch_atomic_xchg
 827 #define arch_atomic_xchg_relaxed arch_atomic_xchg
 828 #else /* arch_atomic_xchg_relaxed */
 829
 830 #ifndef arch_atomic_xchg_acquire
 831 static __always_inline int
 832 arch_atomic_xchg_acquire(atomic_t *v, int i)
 833 {
 834         int ret = arch_atomic_xchg_relaxed(v, i);
 835         __atomic_acquire_fence();
 836         return ret;
 837 }
 838 #define arch_atomic_xchg_acquire arch_atomic_xchg_acquire
 839 #endif
 840
 841 #ifndef arch_atomic_xchg_release
 842 static __always_inline int
 843 arch_atomic_xchg_release(atomic_t *v, int i)
 844 {
 845         __atomic_release_fence();
 846         return arch_atomic_xchg_relaxed(v, i);
 847 }
 848 #define arch_atomic_xchg_release arch_atomic_xchg_release
 849 #endif
 850
 851 #ifndef arch_atomic_xchg
 852 static __always_inline int
 853 arch_atomic_xchg(atomic_t *v, int i)
 854 {
 855         int ret;
 856         __atomic_pre_full_fence();
 857         ret = arch_atomic_xchg_relaxed(v, i);
 858         __atomic_post_full_fence();
 859         return ret;
 860 }
 861 #define arch_atomic_xchg arch_atomic_xchg
 862 #endif
 863
 864 #endif /* arch_atomic_xchg_relaxed */
 865
 866 #ifndef arch_atomic_cmpxchg_relaxed
 867 #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg
 868 #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg
 869 #define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg
 870 #else /* arch_atomic_cmpxchg_relaxed */
 871
 872 #ifndef arch_atomic_cmpxchg_acquire
 873 static __always_inline int
 874 arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
 875 {
 876         int ret = arch_atomic_cmpxchg_relaxed(v, old, new);
 877         __atomic_acquire_fence();
 878         return ret;
 879 }
 880 #define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire
 881 #endif
 882
 883 #ifndef arch_atomic_cmpxchg_release
 884 static __always_inline int
 885 arch_atomic_cmpxchg_release(atomic_t *v, int old, int new)
 886 {
 887         __atomic_release_fence();
 888         return arch_atomic_cmpxchg_relaxed(v, old, new);
 889 }
 890 #define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release
 891 #endif
 892
 893 #ifndef arch_atomic_cmpxchg
 894 static __always_inline int
 895 arch_atomic_cmpxchg(atomic_t *v, int old, int new)
 896 {
 897         int ret;
 898         __atomic_pre_full_fence();
 899         ret = arch_atomic_cmpxchg_relaxed(v, old, new);
 900         __atomic_post_full_fence();
 901         return ret;
 902 }
 903 #define arch_atomic_cmpxchg arch_atomic_cmpxchg
 904 #endif
 905
 906 #endif /* arch_atomic_cmpxchg_relaxed */
 907
 908 #ifndef arch_atomic_try_cmpxchg_relaxed
 909 #ifdef arch_atomic_try_cmpxchg
 910 #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg
 911 #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg
 912 #define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg
 913 #endif /* arch_atomic_try_cmpxchg */
 914
 915 #ifndef arch_atomic_try_cmpxchg
 916 static __always_inline bool
 917 arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
 918 {
 919         int r, o = *old;
 920         r = arch_atomic_cmpxchg(v, o, new);
 921         if (unlikely(r != o))
 922                 *old = r;
 923         return likely(r == o);
 924 }
 925 #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
 926 #endif
 927
 928 #ifndef arch_atomic_try_cmpxchg_acquire
 929 static __always_inline bool
 930 arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
 931 {
 932         int r, o = *old;
 933         r = arch_atomic_cmpxchg_acquire(v, o, new);
 934         if (unlikely(r != o))
 935                 *old = r;
 936         return likely(r == o);
 937 }
 938 #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
 939 #endif
 940
 941 #ifndef arch_atomic_try_cmpxchg_release
 942 static __always_inline bool
 943 arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
 944 {
 945         int r, o = *old;
 946         r = arch_atomic_cmpxchg_release(v, o, new);
 947         if (unlikely(r != o))
 948                 *old = r;
 949         return likely(r == o);
 950 }
 951 #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
 952 #endif
 953
 954 #ifndef arch_atomic_try_cmpxchg_relaxed
 955 static __always_inline bool
 956 arch_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
 957 {
 958         int r, o = *old;
 959         r = arch_atomic_cmpxchg_relaxed(v, o, new);
 960         if (unlikely(r != o))
 961                 *old = r;
 962         return likely(r == o);
 963 }
 964 #define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed
 965 #endif
 966
 967 #else /* arch_atomic_try_cmpxchg_relaxed */
 968
 969 #ifndef arch_atomic_try_cmpxchg_acquire
 970 static __always_inline bool
 971 arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
 972 {
 973         bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
 974         __atomic_acquire_fence();
 975         return ret;
 976 }
 977 #define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
 978 #endif
 979
 980 #ifndef arch_atomic_try_cmpxchg_release
 981 static __always_inline bool
 982 arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
 983 {
 984         __atomic_release_fence();
 985         return arch_atomic_try_cmpxchg_relaxed(v, old, new);
 986 }
 987 #define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
 988 #endif
 989
 990 #ifndef arch_atomic_try_cmpxchg
 991 static __always_inline bool
 992 arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
 993 {
 994         bool ret;
 995         __atomic_pre_full_fence();
 996         ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
 997         __atomic_post_full_fence();
 998         return ret;
 999 }
1000 #define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
1001 #endif
1002
1003 #endif /* arch_atomic_try_cmpxchg_relaxed */
1004
1005 #ifndef arch_atomic_sub_and_test
1006 /**
1007  * arch_atomic_sub_and_test - subtract value from variable and test result
1008  * @i: integer value to subtract
1009  * @v: pointer of type atomic_t
1010  *
1011  * Atomically subtracts @i from @v and returns
1012  * true if the result is zero, or false for all
1013  * other cases.
1014  */
1015 static __always_inline bool
1016 arch_atomic_sub_and_test(int i, atomic_t *v)
1017 {
1018         return arch_atomic_sub_return(i, v) == 0;
1019 }
1020 #define arch_atomic_sub_and_test arch_atomic_sub_and_test
1021 #endif
1022
1023 #ifndef arch_atomic_dec_and_test
1024 /**
1025  * arch_atomic_dec_and_test - decrement and test
1026  * @v: pointer of type atomic_t
1027  *
1028  * Atomically decrements @v by 1 and
1029  * returns true if the result is 0, or false for all other
1030  * cases.
1031  */
1032 static __always_inline bool
1033 arch_atomic_dec_and_test(atomic_t *v)
1034 {
1035         return arch_atomic_dec_return(v) == 0;
1036 }
1037 #define arch_atomic_dec_and_test arch_atomic_dec_and_test
1038 #endif
1039
1040 #ifndef arch_atomic_inc_and_test
1041 /**
1042  * arch_atomic_inc_and_test - increment and test
1043  * @v: pointer of type atomic_t
1044  *
1045  * Atomically increments @v by 1
1046  * and returns true if the result is zero, or false for all
1047  * other cases.
1048  */
1049 static __always_inline bool
1050 arch_atomic_inc_and_test(atomic_t *v)
1051 {
1052         return arch_atomic_inc_return(v) == 0;
1053 }
1054 #define arch_atomic_inc_and_test arch_atomic_inc_and_test
1055 #endif
1056
1057 #ifndef arch_atomic_add_negative
1058 /**
1059  * arch_atomic_add_negative - add and test if negative
1060  * @i: integer value to add
1061  * @v: pointer of type atomic_t
1062  *
1063  * Atomically adds @i to @v and returns true
1064  * if the result is negative, or false when
1065  * result is greater than or equal to zero.
1066  */
1067 static __always_inline bool
1068 arch_atomic_add_negative(int i, atomic_t *v)
1069 {
1070         return arch_atomic_add_return(i, v) < 0;
1071 }
1072 #define arch_atomic_add_negative arch_atomic_add_negative
1073 #endif
1074
1075 #ifndef arch_atomic_fetch_add_unless
1076 /**
1077  * arch_atomic_fetch_add_unless - add unless the number is already a given value
1078  * @v: pointer of type atomic_t
1079  * @a: the amount to add to v...
1080  * @u: ...unless v is equal to u.
1081  *
1082  * Atomically adds @a to @v, so long as @v was not already @u.
1083  * Returns original value of @v
1084  */
1085 static __always_inline int
1086 arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
1087 {
1088         int c = arch_atomic_read(v);
1089
1090         do {
1091                 if (unlikely(c == u))
1092                         break;
1093         } while (!arch_atomic_try_cmpxchg(v, &c, c + a));
1094
1095         return c;
1096 }
1097 #define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
1098 #endif
1099
1100 #ifndef arch_atomic_add_unless
1101 /**
1102  * arch_atomic_add_unless - add unless the number is already a given value
1103  * @v: pointer of type atomic_t
1104  * @a: the amount to add to v...
1105  * @u: ...unless v is equal to u.
1106  *
1107  * Atomically adds @a to @v, if @v was not already @u.
1108  * Returns true if the addition was done.
1109  */
1110 static __always_inline bool
1111 arch_atomic_add_unless(atomic_t *v, int a, int u)
1112 {
1113         return arch_atomic_fetch_add_unless(v, a, u) != u;
1114 }
1115 #define arch_atomic_add_unless arch_atomic_add_unless
1116 #endif
1117
1118 #ifndef arch_atomic_inc_not_zero
1119 /**
1120  * arch_atomic_inc_not_zero - increment unless the number is zero
1121  * @v: pointer of type atomic_t
1122  *
1123  * Atomically increments @v by 1, if @v is non-zero.
1124  * Returns true if the increment was done.
1125  */
1126 static __always_inline bool
1127 arch_atomic_inc_not_zero(atomic_t *v)
1128 {
1129         return arch_atomic_add_unless(v, 1, 0);
1130 }
1131 #define arch_atomic_inc_not_zero arch_atomic_inc_not_zero
1132 #endif
1133
1134 #ifndef arch_atomic_inc_unless_negative
1135 static __always_inline bool
1136 arch_atomic_inc_unless_negative(atomic_t *v)
1137 {
1138         int c = arch_atomic_read(v);
1139
1140         do {
1141                 if (unlikely(c < 0))
1142                         return false;
1143         } while (!arch_atomic_try_cmpxchg(v, &c, c + 1));
1144
1145         return true;
1146 }
1147 #define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative
1148 #endif
1149
1150 #ifndef arch_atomic_dec_unless_positive
1151 static __always_inline bool
1152 arch_atomic_dec_unless_positive(atomic_t *v)
1153 {
1154         int c = arch_atomic_read(v);
1155
1156         do {
1157                 if (unlikely(c > 0))
1158                         return false;
1159         } while (!arch_atomic_try_cmpxchg(v, &c, c - 1));
1160
1161         return true;
1162 }
1163 #define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive
1164 #endif
1165
1166 #ifndef arch_atomic_dec_if_positive
1167 static __always_inline int
1168 arch_atomic_dec_if_positive(atomic_t *v)
1169 {
1170         int dec, c = arch_atomic_read(v);
1171
1172         do {
1173                 dec = c - 1;
1174                 if (unlikely(dec < 0))
1175                         break;
1176         } while (!arch_atomic_try_cmpxchg(v, &c, dec));
1177
1178         return dec;
1179 }
1180 #define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
1181 #endif
1182
1183 #ifdef CONFIG_GENERIC_ATOMIC64
1184 #include <asm-generic/atomic64.h>
1185 #endif
1186
1187 #ifndef arch_atomic64_read_acquire
1188 static __always_inline s64
1189 arch_atomic64_read_acquire(const atomic64_t *v)
1190 {
1191         return smp_load_acquire(&(v)->counter);
1192 }
1193 #define arch_atomic64_read_acquire arch_atomic64_read_acquire
1194 #endif
1195
1196 #ifndef arch_atomic64_set_release
1197 static __always_inline void
1198 arch_atomic64_set_release(atomic64_t *v, s64 i)
1199 {
1200         smp_store_release(&(v)->counter, i);
1201 }
1202 #define arch_atomic64_set_release arch_atomic64_set_release
1203 #endif
1204
1205 #ifndef arch_atomic64_add_return_relaxed
1206 #define arch_atomic64_add_return_acquire arch_atomic64_add_return
1207 #define arch_atomic64_add_return_release arch_atomic64_add_return
1208 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return
1209 #else /* arch_atomic64_add_return_relaxed */
1210
1211 #ifndef arch_atomic64_add_return_acquire
1212 static __always_inline s64
1213 arch_atomic64_add_return_acquire(s64 i, atomic64_t *v)
1214 {
1215         s64 ret = arch_atomic64_add_return_relaxed(i, v);
1216         __atomic_acquire_fence();
1217         return ret;
1218 }
1219 #define arch_atomic64_add_return_acquire arch_atomic64_add_return_acquire
1220 #endif
1221
1222 #ifndef arch_atomic64_add_return_release
1223 static __always_inline s64
1224 arch_atomic64_add_return_release(s64 i, atomic64_t *v)
1225 {
1226         __atomic_release_fence();
1227         return arch_atomic64_add_return_relaxed(i, v);
1228 }
1229 #define arch_atomic64_add_return_release arch_atomic64_add_return_release
1230 #endif
1231
1232 #ifndef arch_atomic64_add_return
1233 static __always_inline s64
1234 arch_atomic64_add_return(s64 i, atomic64_t *v)
1235 {
1236         s64 ret;
1237         __atomic_pre_full_fence();
1238         ret = arch_atomic64_add_return_relaxed(i, v);
1239         __atomic_post_full_fence();
1240         return ret;
1241 }
1242 #define arch_atomic64_add_return arch_atomic64_add_return
1243 #endif
1244
1245 #endif /* arch_atomic64_add_return_relaxed */
1246
1247 #ifndef arch_atomic64_fetch_add_relaxed
1248 #define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add
1249 #define arch_atomic64_fetch_add_release arch_atomic64_fetch_add
1250 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add
1251 #else /* arch_atomic64_fetch_add_relaxed */
1252
1253 #ifndef arch_atomic64_fetch_add_acquire
1254 static __always_inline s64
1255 arch_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1256 {
1257         s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
1258         __atomic_acquire_fence();
1259         return ret;
1260 }
1261 #define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire
1262 #endif
1263
1264 #ifndef arch_atomic64_fetch_add_release
1265 static __always_inline s64
1266 arch_atomic64_fetch_add_release(s64 i, atomic64_t *v)
1267 {
1268         __atomic_release_fence();
1269         return arch_atomic64_fetch_add_relaxed(i, v);
1270 }
1271 #define arch_atomic64_fetch_add_release arch_atomic64_fetch_add_release
1272 #endif
1273
1274 #ifndef arch_atomic64_fetch_add
1275 static __always_inline s64
1276 arch_atomic64_fetch_add(s64 i, atomic64_t *v)
1277 {
1278         s64 ret;
1279         __atomic_pre_full_fence();
1280         ret = arch_atomic64_fetch_add_relaxed(i, v);
1281         __atomic_post_full_fence();
1282         return ret;
1283 }
1284 #define arch_atomic64_fetch_add arch_atomic64_fetch_add
1285 #endif
1286
1287 #endif /* arch_atomic64_fetch_add_relaxed */
1288
1289 #ifndef arch_atomic64_sub_return_relaxed
1290 #define arch_atomic64_sub_return_acquire arch_atomic64_sub_return
1291 #define arch_atomic64_sub_return_release arch_atomic64_sub_return
1292 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return
1293 #else /* arch_atomic64_sub_return_relaxed */
1294
1295 #ifndef arch_atomic64_sub_return_acquire
1296 static __always_inline s64
1297 arch_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1298 {
1299         s64 ret = arch_atomic64_sub_return_relaxed(i, v);
1300         __atomic_acquire_fence();
1301         return ret;
1302 }
1303 #define arch_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire
1304 #endif
1305
1306 #ifndef arch_atomic64_sub_return_release
1307 static __always_inline s64
1308 arch_atomic64_sub_return_release(s64 i, atomic64_t *v)
1309 {
1310         __atomic_release_fence();
1311         return arch_atomic64_sub_return_relaxed(i, v);
1312 }
1313 #define arch_atomic64_sub_return_release arch_atomic64_sub_return_release
1314 #endif
1315
1316 #ifndef arch_atomic64_sub_return
1317 static __always_inline s64
1318 arch_atomic64_sub_return(s64 i, atomic64_t *v)
1319 {
1320         s64 ret;
1321         __atomic_pre_full_fence();
1322         ret = arch_atomic64_sub_return_relaxed(i, v);
1323         __atomic_post_full_fence();
1324         return ret;
1325 }
1326 #define arch_atomic64_sub_return arch_atomic64_sub_return
1327 #endif
1328
1329 #endif /* arch_atomic64_sub_return_relaxed */
1330
1331 #ifndef arch_atomic64_fetch_sub_relaxed
1332 #define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub
1333 #define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub
1334 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub
1335 #else /* arch_atomic64_fetch_sub_relaxed */
1336
1337 #ifndef arch_atomic64_fetch_sub_acquire
1338 static __always_inline s64
1339 arch_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1340 {
1341         s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
1342         __atomic_acquire_fence();
1343         return ret;
1344 }
1345 #define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire
1346 #endif
1347
1348 #ifndef arch_atomic64_fetch_sub_release
1349 static __always_inline s64
1350 arch_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1351 {
1352         __atomic_release_fence();
1353         return arch_atomic64_fetch_sub_relaxed(i, v);
1354 }
1355 #define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release
1356 #endif
1357
1358 #ifndef arch_atomic64_fetch_sub
1359 static __always_inline s64
1360 arch_atomic64_fetch_sub(s64 i, atomic64_t *v)
1361 {
1362         s64 ret;
1363         __atomic_pre_full_fence();
1364         ret = arch_atomic64_fetch_sub_relaxed(i, v);
1365         __atomic_post_full_fence();
1366         return ret;
1367 }
1368 #define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
1369 #endif
1370
1371 #endif /* arch_atomic64_fetch_sub_relaxed */
1372
1373 #ifndef arch_atomic64_inc
1374 static __always_inline void
1375 arch_atomic64_inc(atomic64_t *v)
1376 {
1377         arch_atomic64_add(1, v);
1378 }
1379 #define arch_atomic64_inc arch_atomic64_inc
1380 #endif
1381
1382 #ifndef arch_atomic64_inc_return_relaxed
1383 #ifdef arch_atomic64_inc_return
1384 #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return
1385 #define arch_atomic64_inc_return_release arch_atomic64_inc_return
1386 #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return
1387 #endif /* arch_atomic64_inc_return */
1388
1389 #ifndef arch_atomic64_inc_return
1390 static __always_inline s64
1391 arch_atomic64_inc_return(atomic64_t *v)
1392 {
1393         return arch_atomic64_add_return(1, v);
1394 }
1395 #define arch_atomic64_inc_return arch_atomic64_inc_return
1396 #endif
1397
1398 #ifndef arch_atomic64_inc_return_acquire
1399 static __always_inline s64
1400 arch_atomic64_inc_return_acquire(atomic64_t *v)
1401 {
1402         return arch_atomic64_add_return_acquire(1, v);
1403 }
1404 #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1405 #endif
1406
1407 #ifndef arch_atomic64_inc_return_release
1408 static __always_inline s64
1409 arch_atomic64_inc_return_release(atomic64_t *v)
1410 {
1411         return arch_atomic64_add_return_release(1, v);
1412 }
1413 #define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
1414 #endif
1415
1416 #ifndef arch_atomic64_inc_return_relaxed
1417 static __always_inline s64
1418 arch_atomic64_inc_return_relaxed(atomic64_t *v)
1419 {
1420         return arch_atomic64_add_return_relaxed(1, v);
1421 }
1422 #define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed
1423 #endif
1424
1425 #else /* arch_atomic64_inc_return_relaxed */
1426
1427 #ifndef arch_atomic64_inc_return_acquire
1428 static __always_inline s64
1429 arch_atomic64_inc_return_acquire(atomic64_t *v)
1430 {
1431         s64 ret = arch_atomic64_inc_return_relaxed(v);
1432         __atomic_acquire_fence();
1433         return ret;
1434 }
1435 #define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1436 #endif
1437
1438 #ifndef arch_atomic64_inc_return_release
1439 static __always_inline s64
1440 arch_atomic64_inc_return_release(atomic64_t *v)
1441 {
1442         __atomic_release_fence();
1443         return arch_atomic64_inc_return_relaxed(v);
1444 }
1445 #define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
1446 #endif
1447
1448 #ifndef arch_atomic64_inc_return
1449 static __always_inline s64
1450 arch_atomic64_inc_return(atomic64_t *v)
1451 {
1452         s64 ret;
1453         __atomic_pre_full_fence();
1454         ret = arch_atomic64_inc_return_relaxed(v);
1455         __atomic_post_full_fence();
1456         return ret;
1457 }
1458 #define arch_atomic64_inc_return arch_atomic64_inc_return
1459 #endif
1460
1461 #endif /* arch_atomic64_inc_return_relaxed */
1462
1463 #ifndef arch_atomic64_fetch_inc_relaxed
1464 #ifdef arch_atomic64_fetch_inc
1465 #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc
1466 #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc
1467 #define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc
1468 #endif /* arch_atomic64_fetch_inc */
1469
1470 #ifndef arch_atomic64_fetch_inc
1471 static __always_inline s64
1472 arch_atomic64_fetch_inc(atomic64_t *v)
1473 {
1474         return arch_atomic64_fetch_add(1, v);
1475 }
1476 #define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
1477 #endif
1478
1479 #ifndef arch_atomic64_fetch_inc_acquire
1480 static __always_inline s64
1481 arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1482 {
1483         return arch_atomic64_fetch_add_acquire(1, v);
1484 }
1485 #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1486 #endif
1487
1488 #ifndef arch_atomic64_fetch_inc_release
1489 static __always_inline s64
1490 arch_atomic64_fetch_inc_release(atomic64_t *v)
1491 {
1492         return arch_atomic64_fetch_add_release(1, v);
1493 }
1494 #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
1495 #endif
1496
1497 #ifndef arch_atomic64_fetch_inc_relaxed
1498 static __always_inline s64
1499 arch_atomic64_fetch_inc_relaxed(atomic64_t *v)
1500 {
1501         return arch_atomic64_fetch_add_relaxed(1, v);
1502 }
1503 #define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed
1504 #endif
1505
1506 #else /* arch_atomic64_fetch_inc_relaxed */
1507
1508 #ifndef arch_atomic64_fetch_inc_acquire
1509 static __always_inline s64
1510 arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1511 {
1512         s64 ret = arch_atomic64_fetch_inc_relaxed(v);
1513         __atomic_acquire_fence();
1514         return ret;
1515 }
1516 #define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1517 #endif
1518
1519 #ifndef arch_atomic64_fetch_inc_release
1520 static __always_inline s64
1521 arch_atomic64_fetch_inc_release(atomic64_t *v)
1522 {
1523         __atomic_release_fence();
1524         return arch_atomic64_fetch_inc_relaxed(v);
1525 }
1526 #define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
1527 #endif
1528
1529 #ifndef arch_atomic64_fetch_inc
1530 static __always_inline s64
1531 arch_atomic64_fetch_inc(atomic64_t *v)
1532 {
1533         s64 ret;
1534         __atomic_pre_full_fence();
1535         ret = arch_atomic64_fetch_inc_relaxed(v);
1536         __atomic_post_full_fence();
1537         return ret;
1538 }
1539 #define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
1540 #endif
1541
1542 #endif /* arch_atomic64_fetch_inc_relaxed */
1543
1544 #ifndef arch_atomic64_dec
1545 static __always_inline void
1546 arch_atomic64_dec(atomic64_t *v)
1547 {
1548         arch_atomic64_sub(1, v);
1549 }
1550 #define arch_atomic64_dec arch_atomic64_dec
1551 #endif
1552
1553 #ifndef arch_atomic64_dec_return_relaxed
1554 #ifdef arch_atomic64_dec_return
1555 #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return
1556 #define arch_atomic64_dec_return_release arch_atomic64_dec_return
1557 #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return
1558 #endif /* arch_atomic64_dec_return */
1559
1560 #ifndef arch_atomic64_dec_return
1561 static __always_inline s64
1562 arch_atomic64_dec_return(atomic64_t *v)
1563 {
1564         return arch_atomic64_sub_return(1, v);
1565 }
1566 #define arch_atomic64_dec_return arch_atomic64_dec_return
1567 #endif
1568
1569 #ifndef arch_atomic64_dec_return_acquire
1570 static __always_inline s64
1571 arch_atomic64_dec_return_acquire(atomic64_t *v)
1572 {
1573         return arch_atomic64_sub_return_acquire(1, v);
1574 }
1575 #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
1576 #endif
1577
1578 #ifndef arch_atomic64_dec_return_release
1579 static __always_inline s64
1580 arch_atomic64_dec_return_release(atomic64_t *v)
1581 {
1582         return arch_atomic64_sub_return_release(1, v);
1583 }
1584 #define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1585 #endif
1586
1587 #ifndef arch_atomic64_dec_return_relaxed
1588 static __always_inline s64
1589 arch_atomic64_dec_return_relaxed(atomic64_t *v)
1590 {
1591         return arch_atomic64_sub_return_relaxed(1, v);
1592 }
1593 #define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed
1594 #endif
1595
1596 #else /* arch_atomic64_dec_return_relaxed */
1597
1598 #ifndef arch_atomic64_dec_return_acquire
1599 static __always_inline s64
1600 arch_atomic64_dec_return_acquire(atomic64_t *v)
1601 {
1602         s64 ret = arch_atomic64_dec_return_relaxed(v);
1603         __atomic_acquire_fence();
1604         return ret;
1605 }
1606 #define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
1607 #endif
1608
1609 #ifndef arch_atomic64_dec_return_release
1610 static __always_inline s64
1611 arch_atomic64_dec_return_release(atomic64_t *v)
1612 {
1613         __atomic_release_fence();
1614         return arch_atomic64_dec_return_relaxed(v);
1615 }
1616 #define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1617 #endif
1618
1619 #ifndef arch_atomic64_dec_return
1620 static __always_inline s64
1621 arch_atomic64_dec_return(atomic64_t *v)
1622 {
1623         s64 ret;
1624         __atomic_pre_full_fence();
1625         ret = arch_atomic64_dec_return_relaxed(v);
1626         __atomic_post_full_fence();
1627         return ret;
1628 }
1629 #define arch_atomic64_dec_return arch_atomic64_dec_return
1630 #endif
1631
1632 #endif /* arch_atomic64_dec_return_relaxed */
1633
1634 #ifndef arch_atomic64_fetch_dec_relaxed
1635 #ifdef arch_atomic64_fetch_dec
1636 #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec
1637 #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec
1638 #define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec
1639 #endif /* arch_atomic64_fetch_dec */
1640
1641 #ifndef arch_atomic64_fetch_dec
1642 static __always_inline s64
1643 arch_atomic64_fetch_dec(atomic64_t *v)
1644 {
1645         return arch_atomic64_fetch_sub(1, v);
1646 }
1647 #define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1648 #endif
1649
1650 #ifndef arch_atomic64_fetch_dec_acquire
1651 static __always_inline s64
1652 arch_atomic64_fetch_dec_acquire(atomic64_t *v)
1653 {
1654         return arch_atomic64_fetch_sub_acquire(1, v);
1655 }
1656 #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
1657 #endif
1658
1659 #ifndef arch_atomic64_fetch_dec_release
1660 static __always_inline s64
1661 arch_atomic64_fetch_dec_release(atomic64_t *v)
1662 {
1663         return arch_atomic64_fetch_sub_release(1, v);
1664 }
1665 #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1666 #endif
1667
1668 #ifndef arch_atomic64_fetch_dec_relaxed
1669 static __always_inline s64
1670 arch_atomic64_fetch_dec_relaxed(atomic64_t *v)
1671 {
1672         return arch_atomic64_fetch_sub_relaxed(1, v);
1673 }
1674 #define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed
1675 #endif
1676
1677 #else /* arch_atomic64_fetch_dec_relaxed */
1678
1679 #ifndef arch_atomic64_fetch_dec_acquire
1680 static __always_inline s64
1681 arch_atomic64_fetch_dec_acquire(atomic64_t *v)
1682 {
1683         s64 ret = arch_atomic64_fetch_dec_relaxed(v);
1684         __atomic_acquire_fence();
1685         return ret;
1686 }
1687 #define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
1688 #endif
1689
1690 #ifndef arch_atomic64_fetch_dec_release
1691 static __always_inline s64
1692 arch_atomic64_fetch_dec_release(atomic64_t *v)
1693 {
1694         __atomic_release_fence();
1695         return arch_atomic64_fetch_dec_relaxed(v);
1696 }
1697 #define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1698 #endif
1699
1700 #ifndef arch_atomic64_fetch_dec
1701 static __always_inline s64
1702 arch_atomic64_fetch_dec(atomic64_t *v)
1703 {
1704         s64 ret;
1705         __atomic_pre_full_fence();
1706         ret = arch_atomic64_fetch_dec_relaxed(v);
1707         __atomic_post_full_fence();
1708         return ret;
1709 }
1710 #define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1711 #endif
1712
1713 #endif /* arch_atomic64_fetch_dec_relaxed */
1714
1715 #ifndef arch_atomic64_fetch_and_relaxed
1716 #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and
1717 #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and
1718 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and
1719 #else /* arch_atomic64_fetch_and_relaxed */
1720
1721 #ifndef arch_atomic64_fetch_and_acquire
1722 static __always_inline s64
1723 arch_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
1724 {
1725         s64 ret = arch_atomic64_fetch_and_relaxed(i, v);
1726         __atomic_acquire_fence();
1727         return ret;
1728 }
1729 #define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire
1730 #endif
1731
1732 #ifndef arch_atomic64_fetch_and_release
1733 static __always_inline s64
1734 arch_atomic64_fetch_and_release(s64 i, atomic64_t *v)
1735 {
1736         __atomic_release_fence();
1737         return arch_atomic64_fetch_and_relaxed(i, v);
1738 }
1739 #define arch_atomic64_fetch_and_release arch_atomic64_fetch_and_release
1740 #endif
1741
1742 #ifndef arch_atomic64_fetch_and
1743 static __always_inline s64
1744 arch_atomic64_fetch_and(s64 i, atomic64_t *v)
1745 {
1746         s64 ret;
1747         __atomic_pre_full_fence();
1748         ret = arch_atomic64_fetch_and_relaxed(i, v);
1749         __atomic_post_full_fence();
1750         return ret;
1751 }
1752 #define arch_atomic64_fetch_and arch_atomic64_fetch_and
1753 #endif
1754
1755 #endif /* arch_atomic64_fetch_and_relaxed */
1756
1757 #ifndef arch_atomic64_andnot
1758 static __always_inline void
1759 arch_atomic64_andnot(s64 i, atomic64_t *v)
1760 {
1761         arch_atomic64_and(~i, v);
1762 }
1763 #define arch_atomic64_andnot arch_atomic64_andnot
1764 #endif
1765
1766 #ifndef arch_atomic64_fetch_andnot_relaxed
1767 #ifdef arch_atomic64_fetch_andnot
1768 #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot
1769 #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot
1770 #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot
1771 #endif /* arch_atomic64_fetch_andnot */
1772
1773 #ifndef arch_atomic64_fetch_andnot
1774 static __always_inline s64
1775 arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
1776 {
1777         return arch_atomic64_fetch_and(~i, v);
1778 }
1779 #define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
1780 #endif
1781
1782 #ifndef arch_atomic64_fetch_andnot_acquire
1783 static __always_inline s64
1784 arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1785 {
1786         return arch_atomic64_fetch_and_acquire(~i, v);
1787 }
1788 #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
1789 #endif
1790
1791 #ifndef arch_atomic64_fetch_andnot_release
1792 static __always_inline s64
1793 arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1794 {
1795         return arch_atomic64_fetch_and_release(~i, v);
1796 }
1797 #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
1798 #endif
1799
1800 #ifndef arch_atomic64_fetch_andnot_relaxed
1801 static __always_inline s64
1802 arch_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1803 {
1804         return arch_atomic64_fetch_and_relaxed(~i, v);
1805 }
1806 #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed
1807 #endif
1808
1809 #else /* arch_atomic64_fetch_andnot_relaxed */
1810
1811 #ifndef arch_atomic64_fetch_andnot_acquire
1812 static __always_inline s64
1813 arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1814 {
1815         s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
1816         __atomic_acquire_fence();
1817         return ret;
1818 }
1819 #define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
1820 #endif
1821
1822 #ifndef arch_atomic64_fetch_andnot_release
1823 static __always_inline s64
1824 arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1825 {
1826         __atomic_release_fence();
1827         return arch_atomic64_fetch_andnot_relaxed(i, v);
1828 }
1829 #define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
1830 #endif
1831
1832 #ifndef arch_atomic64_fetch_andnot
1833 static __always_inline s64
1834 arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
1835 {
1836         s64 ret;
1837         __atomic_pre_full_fence();
1838         ret = arch_atomic64_fetch_andnot_relaxed(i, v);
1839         __atomic_post_full_fence();
1840         return ret;
1841 }
1842 #define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
1843 #endif
1844
1845 #endif /* arch_atomic64_fetch_andnot_relaxed */
1846
1847 #ifndef arch_atomic64_fetch_or_relaxed
1848 #define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or
1849 #define arch_atomic64_fetch_or_release arch_atomic64_fetch_or
1850 #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or
1851 #else /* arch_atomic64_fetch_or_relaxed */
1852
1853 #ifndef arch_atomic64_fetch_or_acquire
1854 static __always_inline s64
1855 arch_atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1856 {
1857         s64 ret = arch_atomic64_fetch_or_relaxed(i, v);
1858         __atomic_acquire_fence();
1859         return ret;
1860 }
1861 #define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire
1862 #endif
1863
1864 #ifndef arch_atomic64_fetch_or_release
1865 static __always_inline s64
1866 arch_atomic64_fetch_or_release(s64 i, atomic64_t *v)
1867 {
1868         __atomic_release_fence();
1869         return arch_atomic64_fetch_or_relaxed(i, v);
1870 }
1871 #define arch_atomic64_fetch_or_release arch_atomic64_fetch_or_release
1872 #endif
1873
1874 #ifndef arch_atomic64_fetch_or
1875 static __always_inline s64
1876 arch_atomic64_fetch_or(s64 i, atomic64_t *v)
1877 {
1878         s64 ret;
1879         __atomic_pre_full_fence();
1880         ret = arch_atomic64_fetch_or_relaxed(i, v);
1881         __atomic_post_full_fence();
1882         return ret;
1883 }
1884 #define arch_atomic64_fetch_or arch_atomic64_fetch_or
1885 #endif
1886
1887 #endif /* arch_atomic64_fetch_or_relaxed */
1888
1889 #ifndef arch_atomic64_fetch_xor_relaxed
1890 #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor
1891 #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor
1892 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor
1893 #else /* arch_atomic64_fetch_xor_relaxed */
1894
1895 #ifndef arch_atomic64_fetch_xor_acquire
1896 static __always_inline s64
1897 arch_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1898 {
1899         s64 ret = arch_atomic64_fetch_xor_relaxed(i, v);
1900         __atomic_acquire_fence();
1901         return ret;
1902 }
1903 #define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire
1904 #endif
1905
1906 #ifndef arch_atomic64_fetch_xor_release
1907 static __always_inline s64
1908 arch_atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1909 {
1910         __atomic_release_fence();
1911         return arch_atomic64_fetch_xor_relaxed(i, v);
1912 }
1913 #define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release
1914 #endif
1915
1916 #ifndef arch_atomic64_fetch_xor
1917 static __always_inline s64
1918 arch_atomic64_fetch_xor(s64 i, atomic64_t *v)
1919 {
1920         s64 ret;
1921         __atomic_pre_full_fence();
1922         ret = arch_atomic64_fetch_xor_relaxed(i, v);
1923         __atomic_post_full_fence();
1924         return ret;
1925 }
1926 #define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
1927 #endif
1928
1929 #endif /* arch_atomic64_fetch_xor_relaxed */
1930
1931 #ifndef arch_atomic64_xchg_relaxed
1932 #define arch_atomic64_xchg_acquire arch_atomic64_xchg
1933 #define arch_atomic64_xchg_release arch_atomic64_xchg
1934 #define arch_atomic64_xchg_relaxed arch_atomic64_xchg
1935 #else /* arch_atomic64_xchg_relaxed */
1936
1937 #ifndef arch_atomic64_xchg_acquire
1938 static __always_inline s64
1939 arch_atomic64_xchg_acquire(atomic64_t *v, s64 i)
1940 {
1941         s64 ret = arch_atomic64_xchg_relaxed(v, i);
1942         __atomic_acquire_fence();
1943         return ret;
1944 }
1945 #define arch_atomic64_xchg_acquire arch_atomic64_xchg_acquire
1946 #endif
1947
1948 #ifndef arch_atomic64_xchg_release
1949 static __always_inline s64
1950 arch_atomic64_xchg_release(atomic64_t *v, s64 i)
1951 {
1952         __atomic_release_fence();
1953         return arch_atomic64_xchg_relaxed(v, i);
1954 }
1955 #define arch_atomic64_xchg_release arch_atomic64_xchg_release
1956 #endif
1957
1958 #ifndef arch_atomic64_xchg
1959 static __always_inline s64
1960 arch_atomic64_xchg(atomic64_t *v, s64 i)
1961 {
1962         s64 ret;
1963         __atomic_pre_full_fence();
1964         ret = arch_atomic64_xchg_relaxed(v, i);
1965         __atomic_post_full_fence();
1966         return ret;
1967 }
1968 #define arch_atomic64_xchg arch_atomic64_xchg
1969 #endif
1970
1971 #endif /* arch_atomic64_xchg_relaxed */
1972
1973 #ifndef arch_atomic64_cmpxchg_relaxed
1974 #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg
1975 #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg
1976 #define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg
1977 #else /* arch_atomic64_cmpxchg_relaxed */
1978
1979 #ifndef arch_atomic64_cmpxchg_acquire
1980 static __always_inline s64
1981 arch_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1982 {
1983         s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
1984         __atomic_acquire_fence();
1985         return ret;
1986 }
1987 #define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire
1988 #endif
1989
1990 #ifndef arch_atomic64_cmpxchg_release
1991 static __always_inline s64
1992 arch_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1993 {
1994         __atomic_release_fence();
1995         return arch_atomic64_cmpxchg_relaxed(v, old, new);
1996 }
1997 #define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release
1998 #endif
1999
2000 #ifndef arch_atomic64_cmpxchg
2001 static __always_inline s64
2002 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
2003 {
2004         s64 ret;
2005         __atomic_pre_full_fence();
2006         ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
2007         __atomic_post_full_fence();
2008         return ret;
2009 }
2010 #define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
2011 #endif
2012
2013 #endif /* arch_atomic64_cmpxchg_relaxed */
2014
2015 #ifndef arch_atomic64_try_cmpxchg_relaxed
2016 #ifdef arch_atomic64_try_cmpxchg
2017 #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg
2018 #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg
2019 #define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg
2020 #endif /* arch_atomic64_try_cmpxchg */
2021
2022 #ifndef arch_atomic64_try_cmpxchg
2023 static __always_inline bool
2024 arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2025 {
2026         s64 r, o = *old;
2027         r = arch_atomic64_cmpxchg(v, o, new);
2028         if (unlikely(r != o))
2029                 *old = r;
2030         return likely(r == o);
2031 }
2032 #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
2033 #endif
2034
2035 #ifndef arch_atomic64_try_cmpxchg_acquire
2036 static __always_inline bool
2037 arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2038 {
2039         s64 r, o = *old;
2040         r = arch_atomic64_cmpxchg_acquire(v, o, new);
2041         if (unlikely(r != o))
2042                 *old = r;
2043         return likely(r == o);
2044 }
2045 #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
2046 #endif
2047
2048 #ifndef arch_atomic64_try_cmpxchg_release
2049 static __always_inline bool
2050 arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2051 {
2052         s64 r, o = *old;
2053         r = arch_atomic64_cmpxchg_release(v, o, new);
2054         if (unlikely(r != o))
2055                 *old = r;
2056         return likely(r == o);
2057 }
2058 #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
2059 #endif
2060
2061 #ifndef arch_atomic64_try_cmpxchg_relaxed
2062 static __always_inline bool
2063 arch_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
2064 {
2065         s64 r, o = *old;
2066         r = arch_atomic64_cmpxchg_relaxed(v, o, new);
2067         if (unlikely(r != o))
2068                 *old = r;
2069         return likely(r == o);
2070 }
2071 #define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg_relaxed
2072 #endif
2073
2074 #else /* arch_atomic64_try_cmpxchg_relaxed */
2075
2076 #ifndef arch_atomic64_try_cmpxchg_acquire
2077 static __always_inline bool
2078 arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2079 {
2080         bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2081         __atomic_acquire_fence();
2082         return ret;
2083 }
2084 #define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
2085 #endif
2086
2087 #ifndef arch_atomic64_try_cmpxchg_release
2088 static __always_inline bool
2089 arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2090 {
2091         __atomic_release_fence();
2092         return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2093 }
2094 #define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
2095 #endif
2096
2097 #ifndef arch_atomic64_try_cmpxchg
2098 static __always_inline bool
2099 arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2100 {
2101         bool ret;
2102         __atomic_pre_full_fence();
2103         ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2104         __atomic_post_full_fence();
2105         return ret;
2106 }
2107 #define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
2108 #endif
2109
2110 #endif /* arch_atomic64_try_cmpxchg_relaxed */
2111
2112 #ifndef arch_atomic64_sub_and_test
2113 /**
2114  * arch_atomic64_sub_and_test - subtract value from variable and test result
2115  * @i: integer value to subtract
2116  * @v: pointer of type atomic64_t
2117  *
2118  * Atomically subtracts @i from @v and returns
2119  * true if the result is zero, or false for all
2120  * other cases.
2121  */
2122 static __always_inline bool
2123 arch_atomic64_sub_and_test(s64 i, atomic64_t *v)
2124 {
2125         return arch_atomic64_sub_return(i, v) == 0;
2126 }
2127 #define arch_atomic64_sub_and_test arch_atomic64_sub_and_test
2128 #endif
2129
2130 #ifndef arch_atomic64_dec_and_test
2131 /**
2132  * arch_atomic64_dec_and_test - decrement and test
2133  * @v: pointer of type atomic64_t
2134  *
2135  * Atomically decrements @v by 1 and
2136  * returns true if the result is 0, or false for all other
2137  * cases.
2138  */
2139 static __always_inline bool
2140 arch_atomic64_dec_and_test(atomic64_t *v)
2141 {
2142         return arch_atomic64_dec_return(v) == 0;
2143 }
2144 #define arch_atomic64_dec_and_test arch_atomic64_dec_and_test
2145 #endif
2146
2147 #ifndef arch_atomic64_inc_and_test
2148 /**
2149  * arch_atomic64_inc_and_test - increment and test
2150  * @v: pointer of type atomic64_t
2151  *
2152  * Atomically increments @v by 1
2153  * and returns true if the result is zero, or false for all
2154  * other cases.
2155  */
2156 static __always_inline bool
2157 arch_atomic64_inc_and_test(atomic64_t *v)
2158 {
2159         return arch_atomic64_inc_return(v) == 0;
2160 }
2161 #define arch_atomic64_inc_and_test arch_atomic64_inc_and_test
2162 #endif
2163
2164 #ifndef arch_atomic64_add_negative
2165 /**
2166  * arch_atomic64_add_negative - add and test if negative
2167  * @i: integer value to add
2168  * @v: pointer of type atomic64_t
2169  *
2170  * Atomically adds @i to @v and returns true
2171  * if the result is negative, or false when
2172  * result is greater than or equal to zero.
2173  */
2174 static __always_inline bool
2175 arch_atomic64_add_negative(s64 i, atomic64_t *v)
2176 {
2177         return arch_atomic64_add_return(i, v) < 0;
2178 }
2179 #define arch_atomic64_add_negative arch_atomic64_add_negative
2180 #endif
2181
2182 #ifndef arch_atomic64_fetch_add_unless
2183 /**
2184  * arch_atomic64_fetch_add_unless - add unless the number is already a given value
2185  * @v: pointer of type atomic64_t
2186  * @a: the amount to add to v...
2187  * @u: ...unless v is equal to u.
2188  *
2189  * Atomically adds @a to @v, so long as @v was not already @u.
2190  * Returns original value of @v
2191  */
2192 static __always_inline s64
2193 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
2194 {
2195         s64 c = arch_atomic64_read(v);
2196
2197         do {
2198                 if (unlikely(c == u))
2199                         break;
2200         } while (!arch_atomic64_try_cmpxchg(v, &c, c + a));
2201
2202         return c;
2203 }
2204 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
2205 #endif
2206
2207 #ifndef arch_atomic64_add_unless
2208 /**
2209  * arch_atomic64_add_unless - add unless the number is already a given value
2210  * @v: pointer of type atomic64_t
2211  * @a: the amount to add to v...
2212  * @u: ...unless v is equal to u.
2213  *
2214  * Atomically adds @a to @v, if @v was not already @u.
2215  * Returns true if the addition was done.
2216  */
2217 static __always_inline bool
2218 arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
2219 {
2220         return arch_atomic64_fetch_add_unless(v, a, u) != u;
2221 }
2222 #define arch_atomic64_add_unless arch_atomic64_add_unless
2223 #endif
2224
2225 #ifndef arch_atomic64_inc_not_zero
2226 /**
2227  * arch_atomic64_inc_not_zero - increment unless the number is zero
2228  * @v: pointer of type atomic64_t
2229  *
2230  * Atomically increments @v by 1, if @v is non-zero.
2231  * Returns true if the increment was done.
2232  */
2233 static __always_inline bool
2234 arch_atomic64_inc_not_zero(atomic64_t *v)
2235 {
2236         return arch_atomic64_add_unless(v, 1, 0);
2237 }
2238 #define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero
2239 #endif
2240
2241 #ifndef arch_atomic64_inc_unless_negative
2242 static __always_inline bool
2243 arch_atomic64_inc_unless_negative(atomic64_t *v)
2244 {
2245         s64 c = arch_atomic64_read(v);
2246
2247         do {
2248                 if (unlikely(c < 0))
2249                         return false;
2250         } while (!arch_atomic64_try_cmpxchg(v, &c, c + 1));
2251
2252         return true;
2253 }
2254 #define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative
2255 #endif
2256
2257 #ifndef arch_atomic64_dec_unless_positive
2258 static __always_inline bool
2259 arch_atomic64_dec_unless_positive(atomic64_t *v)
2260 {
2261         s64 c = arch_atomic64_read(v);
2262
2263         do {
2264                 if (unlikely(c > 0))
2265                         return false;
2266         } while (!arch_atomic64_try_cmpxchg(v, &c, c - 1));
2267
2268         return true;
2269 }
2270 #define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive
2271 #endif
2272
2273 #ifndef arch_atomic64_dec_if_positive
2274 static __always_inline s64
2275 arch_atomic64_dec_if_positive(atomic64_t *v)
2276 {
2277         s64 dec, c = arch_atomic64_read(v);
2278
2279         do {
2280                 dec = c - 1;
2281                 if (unlikely(dec < 0))
2282                         break;
2283         } while (!arch_atomic64_try_cmpxchg(v, &c, dec));
2284
2285         return dec;
2286 }
2287 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
2288 #endif
2289
2290 #endif /* _LINUX_ATOMIC_FALLBACK_H */
2291 // 90cd26cfd69d2250303d654955a0cc12620fb91b