arch/ia64/kernel/fsys.S

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * This file contains the light-weight system call handlers (fsyscall-handlers).
   4  *
   5  * Copyright (C) 2003 Hewlett-Packard Co
   6  *      David Mosberger-Tang <davidm@hpl.hp.com>
   7  *
   8  * 25-Sep-03 davidm     Implement fsys_rt_sigprocmask().
   9  * 18-Feb-03 louisk     Implement fsys_gettimeofday().
  10  * 28-Feb-03 davidm     Fixed several bugs in fsys_gettimeofday().  Tuned it some more,
  11  *                      probably broke it along the way... ;-)
  12  * 13-Jul-04 clameter   Implement fsys_clock_gettime and revise fsys_gettimeofday to make
  13  *                      it capable of using memory based clocks without falling back to C code.
  14  * 08-Feb-07 Fenghua Yu Implement fsys_getcpu.
  15  *
  16  */
  17
  18 #include <asm/asmmacro.h>
  19 #include <asm/errno.h>
  20 #include <asm/asm-offsets.h>
  21 #include <asm/percpu.h>
  22 #include <asm/thread_info.h>
  23 #include <asm/sal.h>
  24 #include <asm/signal.h>
  25 #include <asm/unistd.h>
  26
  27 #include "entry.h"
  28 #include <asm/native/inst.h>
  29
  30 /*
  31  * See Documentation/ia64/fsys.rst for details on fsyscalls.
  32  *
  33  * On entry to an fsyscall handler:
  34  *   r10        = 0 (i.e., defaults to "successful syscall return")
  35  *   r11        = saved ar.pfs (a user-level value)
  36  *   r15        = system call number
  37  *   r16        = "current" task pointer (in normal kernel-mode, this is in r13)
  38  *   r32-r39    = system call arguments
  39  *   b6         = return address (a user-level value)
  40  *   ar.pfs     = previous frame-state (a user-level value)
  41  *   PSR.be     = cleared to zero (i.e., little-endian byte order is in effect)
  42  *   all other registers may contain values passed in from user-mode
  43  *
  44  * On return from an fsyscall handler:
  45  *   r11        = saved ar.pfs (as passed into the fsyscall handler)
  46  *   r15        = system call number (as passed into the fsyscall handler)
  47  *   r32-r39    = system call arguments (as passed into the fsyscall handler)
  48  *   b6         = return address (as passed into the fsyscall handler)
  49  *   ar.pfs     = previous frame-state (as passed into the fsyscall handler)
  50  */
  51
  52 ENTRY(fsys_ni_syscall)
  53         .prologue
  54         .altrp b6
  55         .body
  56         mov r8=ENOSYS
  57         mov r10=-1
  58         FSYS_RETURN
  59 END(fsys_ni_syscall)
  60
  61 ENTRY(fsys_getpid)
  62         .prologue
  63         .altrp b6
  64         .body
  65         add r17=IA64_TASK_SIGNAL_OFFSET,r16
  66         ;;
  67         ld8 r17=[r17]                           // r17 = current->signal
  68         add r9=TI_FLAGS+IA64_TASK_SIZE,r16
  69         ;;
  70         ld4 r9=[r9]
  71         add r17=IA64_SIGNAL_PIDS_TGID_OFFSET,r17
  72         ;;
  73         and r9=TIF_ALLWORK_MASK,r9
  74         ld8 r17=[r17]                           // r17 = current->signal->pids[PIDTYPE_TGID]
  75         ;;
  76         add r8=IA64_PID_LEVEL_OFFSET,r17
  77         ;;
  78         ld4 r8=[r8]                             // r8 = pid->level
  79         add r17=IA64_PID_UPID_OFFSET,r17        // r17 = &pid->numbers[0]
  80         ;;
  81         shl r8=r8,IA64_UPID_SHIFT
  82         ;;
  83         add r17=r17,r8                          // r17 = &pid->numbers[pid->level]
  84         ;;
  85         ld4 r8=[r17]                            // r8 = pid->numbers[pid->level].nr
  86         ;;
  87         mov r17=0
  88         ;;
  89         cmp.ne p8,p0=0,r9
  90 (p8)    br.spnt.many fsys_fallback_syscall
  91         FSYS_RETURN
  92 END(fsys_getpid)
  93
  94 ENTRY(fsys_set_tid_address)
  95         .prologue
  96         .altrp b6
  97         .body
  98         add r9=TI_FLAGS+IA64_TASK_SIZE,r16
  99         add r17=IA64_TASK_THREAD_PID_OFFSET,r16
 100         ;;
 101         ld4 r9=[r9]
 102         tnat.z p6,p7=r32                // check argument register for being NaT
 103         ld8 r17=[r17]                           // r17 = current->thread_pid
 104         ;;
 105         and r9=TIF_ALLWORK_MASK,r9
 106         add r8=IA64_PID_LEVEL_OFFSET,r17
 107         add r18=IA64_TASK_CLEAR_CHILD_TID_OFFSET,r16
 108         ;;
 109         ld4 r8=[r8]                             // r8 = pid->level
 110         add r17=IA64_PID_UPID_OFFSET,r17        // r17 = &pid->numbers[0]
 111         ;;
 112         shl r8=r8,IA64_UPID_SHIFT
 113         ;;
 114         add r17=r17,r8                          // r17 = &pid->numbers[pid->level]
 115         ;;
 116         ld4 r8=[r17]                            // r8 = pid->numbers[pid->level].nr
 117         ;;
 118         cmp.ne p8,p0=0,r9
 119         mov r17=-1
 120         ;;
 121 (p6)    st8 [r18]=r32
 122 (p7)    st8 [r18]=r17
 123 (p8)    br.spnt.many fsys_fallback_syscall
 124         ;;
 125         mov r17=0                       // i must not leak kernel bits...
 126         mov r18=0                       // i must not leak kernel bits...
 127         FSYS_RETURN
 128 END(fsys_set_tid_address)
 129
 130 #if IA64_GTOD_SEQ_OFFSET !=0
 131 #error fsys_gettimeofday incompatible with changes to struct fsyscall_gtod_data_t
 132 #endif
 133 #if IA64_ITC_JITTER_OFFSET !=0
 134 #error fsys_gettimeofday incompatible with changes to struct itc_jitter_data_t
 135 #endif
 136 #define CLOCK_REALTIME 0
 137 #define CLOCK_MONOTONIC 1
 138 #define CLOCK_DIVIDE_BY_1000 0x4000
 139 #define CLOCK_ADD_MONOTONIC 0x8000
 140
 141 ENTRY(fsys_gettimeofday)
 142         .prologue
 143         .altrp b6
 144         .body
 145         mov r31 = r32
 146         tnat.nz p6,p0 = r33             // guard against NaT argument
 147 (p6)    br.cond.spnt.few .fail_einval
 148         mov r30 = CLOCK_DIVIDE_BY_1000
 149         ;;
 150 .gettime:
 151         // Register map
 152         // Incoming r31 = pointer to address where to place result
 153         //          r30 = flags determining how time is processed
 154         // r2,r3 = temp r4-r7 preserved
 155         // r8 = result nanoseconds
 156         // r9 = result seconds
 157         // r10 = temporary storage for clock difference
 158         // r11 = preserved: saved ar.pfs
 159         // r12 = preserved: memory stack
 160         // r13 = preserved: thread pointer
 161         // r14 = address of mask / mask value
 162         // r15 = preserved: system call number
 163         // r16 = preserved: current task pointer
 164         // r17 = (not used)
 165         // r18 = (not used)
 166         // r19 = address of itc_lastcycle
 167         // r20 = struct fsyscall_gtod_data (= address of gtod_lock.sequence)
 168         // r21 = address of mmio_ptr
 169         // r22 = address of wall_time or monotonic_time
 170         // r23 = address of shift / value
 171         // r24 = address mult factor / cycle_last value
 172         // r25 = itc_lastcycle value
 173         // r26 = address clocksource cycle_last
 174         // r27 = (not used)
 175         // r28 = sequence number at the beginning of critical section
 176         // r29 = address of itc_jitter
 177         // r30 = time processing flags / memory address
 178         // r31 = pointer to result
 179         // Predicates
 180         // p6,p7 short term use
 181         // p8 = timesource ar.itc
 182         // p9 = timesource mmio64
 183         // p10 = timesource mmio32 - not used
 184         // p11 = timesource not to be handled by asm code
 185         // p12 = memory time source ( = p9 | p10) - not used
 186         // p13 = do cmpxchg with itc_lastcycle
 187         // p14 = Divide by 1000
 188         // p15 = Add monotonic
 189         //
 190         // Note that instructions are optimized for McKinley. McKinley can
 191         // process two bundles simultaneously and therefore we continuously
 192         // try to feed the CPU two bundles and then a stop.
 193
 194         add r2 = TI_FLAGS+IA64_TASK_SIZE,r16
 195         tnat.nz p6,p0 = r31             // guard against Nat argument
 196 (p6)    br.cond.spnt.few .fail_einval
 197         movl r20 = fsyscall_gtod_data // load fsyscall gettimeofday data address
 198         ;;
 199         ld4 r2 = [r2]                   // process work pending flags
 200         movl r29 = itc_jitter_data      // itc_jitter
 201         add r22 = IA64_GTOD_WALL_TIME_OFFSET,r20        // wall_time
 202         add r21 = IA64_CLKSRC_MMIO_OFFSET,r20
 203         mov pr = r30,0xc000     // Set predicates according to function
 204         ;;
 205         and r2 = TIF_ALLWORK_MASK,r2
 206         add r19 = IA64_ITC_LASTCYCLE_OFFSET,r29
 207 (p15)   add r22 = IA64_GTOD_MONO_TIME_OFFSET,r20        // monotonic_time
 208         ;;
 209         add r26 = IA64_CLKSRC_CYCLE_LAST_OFFSET,r20     // clksrc_cycle_last
 210         cmp.ne p6, p0 = 0, r2   // Fallback if work is scheduled
 211 (p6)    br.cond.spnt.many fsys_fallback_syscall
 212         ;;
 213         // Begin critical section
 214 .time_redo:
 215         ld4.acq r28 = [r20]     // gtod_lock.sequence, Must take first
 216         ;;
 217         and r28 = ~1,r28        // And make sequence even to force retry if odd
 218         ;;
 219         ld8 r30 = [r21]         // clocksource->mmio_ptr
 220         add r24 = IA64_CLKSRC_MULT_OFFSET,r20
 221         ld4 r2 = [r29]          // itc_jitter value
 222         add r23 = IA64_CLKSRC_SHIFT_OFFSET,r20
 223         add r14 = IA64_CLKSRC_MASK_OFFSET,r20
 224         ;;
 225         ld4 r3 = [r24]          // clocksource mult value
 226         ld8 r14 = [r14]         // clocksource mask value
 227         cmp.eq p8,p9 = 0,r30    // use cpu timer if no mmio_ptr
 228         ;;
 229         setf.sig f7 = r3        // Setup for mult scaling of counter
 230 (p8)    cmp.ne p13,p0 = r2,r0   // need itc_jitter compensation, set p13
 231         ld4 r23 = [r23]         // clocksource shift value
 232         ld8 r24 = [r26]         // get clksrc_cycle_last value
 233 (p9)    cmp.eq p13,p0 = 0,r30   // if mmio_ptr, clear p13 jitter control
 234         ;;
 235         .pred.rel.mutex p8,p9
 236         MOV_FROM_ITC(p8, p6, r2, r10)   // CPU_TIMER. 36 clocks latency!!!
 237 (p9)    ld8 r2 = [r30]          // MMIO_TIMER. Could also have latency issues..
 238 (p13)   ld8 r25 = [r19]         // get itc_lastcycle value
 239         ld8 r9 = [r22],IA64_TIME_SN_SPEC_SNSEC_OFFSET   // sec
 240         ;;
 241         ld8 r8 = [r22],-IA64_TIME_SN_SPEC_SNSEC_OFFSET  // snsec
 242 (p13)   sub r3 = r25,r2         // Diff needed before comparison (thanks davidm)
 243         ;;
 244 (p13)   cmp.gt.unc p6,p7 = r3,r0 // check if it is less than last. p6,p7 cleared
 245         sub r10 = r2,r24        // current_cycle - last_cycle
 246         ;;
 247 (p6)    sub r10 = r25,r24       // time we got was less than last_cycle
 248 (p7)    mov ar.ccv = r25        // more than last_cycle. Prep for cmpxchg
 249         ;;
 250 (p7)    cmpxchg8.rel r3 = [r19],r2,ar.ccv
 251         ;;
 252 (p7)    cmp.ne p7,p0 = r25,r3   // if cmpxchg not successful
 253         ;;
 254 (p7)    sub r10 = r3,r24        // then use new last_cycle instead
 255         ;;
 256         and r10 = r10,r14       // Apply mask
 257         ;;
 258         setf.sig f8 = r10
 259         nop.i 123
 260         ;;
 261         // fault check takes 5 cycles and we have spare time
 262 EX(.fail_efault, probe.w.fault r31, 3)
 263         xmpy.l f8 = f8,f7       // nsec_per_cyc*(counter-last_counter)
 264         ;;
 265         getf.sig r2 = f8
 266         mf
 267         ;;
 268         ld4 r10 = [r20]         // gtod_lock.sequence
 269         add r8 = r8,r2          // Add xtime.nsecs
 270         ;;
 271         shr.u r8 = r8,r23       // shift by factor
 272         cmp4.ne p7,p0 = r28,r10
 273 (p7)    br.cond.dpnt.few .time_redo     // sequence number changed, redo
 274         // End critical section.
 275         // Now r8=tv->tv_nsec and r9=tv->tv_sec
 276         mov r10 = r0
 277         movl r2 = 1000000000
 278         add r23 = IA64_TIMESPEC_TV_NSEC_OFFSET, r31
 279 (p14)   movl r3 = 2361183241434822607   // Prep for / 1000 hack
 280         ;;
 281 .time_normalize:
 282         mov r21 = r8
 283         cmp.ge p6,p0 = r8,r2
 284 (p14)   shr.u r20 = r8, 3 // We can repeat this if necessary just wasting time
 285         ;;
 286 (p14)   setf.sig f8 = r20
 287 (p6)    sub r8 = r8,r2
 288 (p6)    add r9 = 1,r9           // two nops before the branch.
 289 (p14)   setf.sig f7 = r3        // Chances for repeats are 1 in 10000 for gettod
 290 (p6)    br.cond.dpnt.few .time_normalize
 291         ;;
 292         // Divided by 8 though shift. Now divide by 125
 293         // The compiler was able to do that with a multiply
 294         // and a shift and we do the same
 295 EX(.fail_efault, probe.w.fault r23, 3)  // This also costs 5 cycles
 296 (p14)   xmpy.hu f8 = f8, f7             // xmpy has 5 cycles latency so use it
 297         ;;
 298 (p14)   getf.sig r2 = f8
 299         ;;
 300         mov r8 = r0
 301 (p14)   shr.u r21 = r2, 4
 302         ;;
 303 EX(.fail_efault, st8 [r31] = r9)
 304 EX(.fail_efault, st8 [r23] = r21)
 305         FSYS_RETURN
 306 .fail_einval:
 307         mov r8 = EINVAL
 308         mov r10 = -1
 309         FSYS_RETURN
 310 .fail_efault:
 311         mov r8 = EFAULT
 312         mov r10 = -1
 313         FSYS_RETURN
 314 END(fsys_gettimeofday)
 315
 316 ENTRY(fsys_clock_gettime)
 317         .prologue
 318         .altrp b6
 319         .body
 320         cmp4.ltu p6, p0 = CLOCK_MONOTONIC, r32
 321         // Fallback if this is not CLOCK_REALTIME or CLOCK_MONOTONIC
 322 (p6)    br.spnt.few fsys_fallback_syscall
 323         mov r31 = r33
 324         shl r30 = r32,15
 325         br.many .gettime
 326 END(fsys_clock_gettime)
 327
 328 /*
 329  * fsys_getcpu doesn't use the third parameter in this implementation. It reads
 330  * current_thread_info()->cpu and corresponding node in cpu_to_node_map.
 331  */
 332 ENTRY(fsys_getcpu)
 333         .prologue
 334         .altrp b6
 335         .body
 336         ;;
 337         add r2=TI_FLAGS+IA64_TASK_SIZE,r16
 338         tnat.nz p6,p0 = r32                     // guard against NaT argument
 339         add r3=TI_CPU+IA64_TASK_SIZE,r16
 340         ;;
 341         ld4 r3=[r3]                             // M r3 = thread_info->cpu
 342         ld4 r2=[r2]                             // M r2 = thread_info->flags
 343 (p6)    br.cond.spnt.few .fail_einval           // B
 344         ;;
 345         tnat.nz p7,p0 = r33                     // I guard against NaT argument
 346 (p7)    br.cond.spnt.few .fail_einval           // B
 347         ;;
 348         cmp.ne p6,p0=r32,r0
 349         cmp.ne p7,p0=r33,r0
 350         ;;
 351 #ifdef CONFIG_NUMA
 352         movl r17=cpu_to_node_map
 353         ;;
 354 EX(.fail_efault, (p6) probe.w.fault r32, 3)             // M This takes 5 cycles
 355 EX(.fail_efault, (p7) probe.w.fault r33, 3)             // M This takes 5 cycles
 356         shladd r18=r3,1,r17
 357         ;;
 358         ld2 r20=[r18]                           // r20 = cpu_to_node_map[cpu]
 359         and r2 = TIF_ALLWORK_MASK,r2
 360         ;;
 361         cmp.ne p8,p0=0,r2
 362 (p8)    br.spnt.many fsys_fallback_syscall
 363         ;;
 364         ;;
 365 EX(.fail_efault, (p6) st4 [r32] = r3)
 366 EX(.fail_efault, (p7) st2 [r33] = r20)
 367         mov r8=0
 368         ;;
 369 #else
 370 EX(.fail_efault, (p6) probe.w.fault r32, 3)             // M This takes 5 cycles
 371 EX(.fail_efault, (p7) probe.w.fault r33, 3)             // M This takes 5 cycles
 372         and r2 = TIF_ALLWORK_MASK,r2
 373         ;;
 374         cmp.ne p8,p0=0,r2
 375 (p8)    br.spnt.many fsys_fallback_syscall
 376         ;;
 377 EX(.fail_efault, (p6) st4 [r32] = r3)
 378 EX(.fail_efault, (p7) st2 [r33] = r0)
 379         mov r8=0
 380         ;;
 381 #endif
 382         FSYS_RETURN
 383 END(fsys_getcpu)
 384
 385 ENTRY(fsys_fallback_syscall)
 386         .prologue
 387         .altrp b6
 388         .body
 389         /*
 390          * We only get here from light-weight syscall handlers.  Thus, we already
 391          * know that r15 contains a valid syscall number.  No need to re-check.
 392          */
 393         adds r17=-1024,r15
 394         movl r14=sys_call_table
 395         ;;
 396         RSM_PSR_I(p0, r26, r27)
 397         shladd r18=r17,3,r14
 398         ;;
 399         ld8 r18=[r18]                           // load normal (heavy-weight) syscall entry-point
 400         MOV_FROM_PSR(p0, r29, r26)              // read psr (12 cyc load latency)
 401         mov r27=ar.rsc
 402         mov r21=ar.fpsr
 403         mov r26=ar.pfs
 404 END(fsys_fallback_syscall)
 405         /* FALL THROUGH */
 406 GLOBAL_ENTRY(fsys_bubble_down)
 407         .prologue
 408         .altrp b6
 409         .body
 410         /*
 411          * We get here for syscalls that don't have a lightweight
 412          * handler.  For those, we need to bubble down into the kernel
 413          * and that requires setting up a minimal pt_regs structure,
 414          * and initializing the CPU state more or less as if an
 415          * interruption had occurred.  To make syscall-restarts work,
 416          * we setup pt_regs such that cr_iip points to the second
 417          * instruction in syscall_via_break.  Decrementing the IP
 418          * hence will restart the syscall via break and not
 419          * decrementing IP will return us to the caller, as usual.
 420          * Note that we preserve the value of psr.pp rather than
 421          * initializing it from dcr.pp.  This makes it possible to
 422          * distinguish fsyscall execution from other privileged
 423          * execution.
 424          *
 425          * On entry:
 426          *      - normal fsyscall handler register usage, except
 427          *        that we also have:
 428          *      - r18: address of syscall entry point
 429          *      - r21: ar.fpsr
 430          *      - r26: ar.pfs
 431          *      - r27: ar.rsc
 432          *      - r29: psr
 433          *
 434          * We used to clear some PSR bits here but that requires slow
 435          * serialization.  Fortunately, that isn't really necessary.
 436          * The rationale is as follows: we used to clear bits
 437          * ~PSR_PRESERVED_BITS in PSR.L.  Since
 438          * PSR_PRESERVED_BITS==PSR.{UP,MFL,MFH,PK,DT,PP,SP,RT,IC}, we
 439          * ended up clearing PSR.{BE,AC,I,DFL,DFH,DI,DB,SI,TB}.
 440          * However,
 441          *
 442          * PSR.BE : already is turned off in __kernel_syscall_via_epc()
 443          * PSR.AC : don't care (kernel normally turns PSR.AC on)
 444          * PSR.I  : already turned off by the time fsys_bubble_down gets
 445          *          invoked
 446          * PSR.DFL: always 0 (kernel never turns it on)
 447          * PSR.DFH: don't care --- kernel never touches f32-f127 on its own
 448          *          initiative
 449          * PSR.DI : always 0 (kernel never turns it on)
 450          * PSR.SI : always 0 (kernel never turns it on)
 451          * PSR.DB : don't care --- kernel never enables kernel-level
 452          *          breakpoints
 453          * PSR.TB : must be 0 already; if it wasn't zero on entry to
 454          *          __kernel_syscall_via_epc, the branch to fsys_bubble_down
 455          *          will trigger a taken branch; the taken-trap-handler then
 456          *          converts the syscall into a break-based system-call.
 457          */
 458         /*
 459          * Reading psr.l gives us only bits 0-31, psr.it, and psr.mc.
 460          * The rest we have to synthesize.
 461          */
 462 #       define PSR_ONE_BITS             ((3 << IA64_PSR_CPL0_BIT)       \
 463                                          | (0x1 << IA64_PSR_RI_BIT)     \
 464                                          | IA64_PSR_BN | IA64_PSR_I)
 465
 466         invala                                  // M0|1
 467         movl r14=ia64_ret_from_syscall          // X
 468
 469         nop.m 0
 470         movl r28=__kernel_syscall_via_break     // X    create cr.iip
 471         ;;
 472
 473         mov r2=r16                              // A    get task addr to addl-addressable register
 474         adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16 // A
 475         mov r31=pr                              // I0   save pr (2 cyc)
 476         ;;
 477         st1 [r16]=r0                            // M2|3 clear current->thread.on_ustack flag
 478         addl r22=IA64_RBS_OFFSET,r2             // A    compute base of RBS
 479         add r3=TI_FLAGS+IA64_TASK_SIZE,r2       // A
 480         ;;
 481         ld4 r3=[r3]                             // M0|1 r3 = current_thread_info()->flags
 482         lfetch.fault.excl.nt1 [r22]             // M0|1 prefetch register backing-store
 483         nop.i 0
 484         ;;
 485         mov ar.rsc=0                            // M2   set enforced lazy mode, pl 0, LE, loadrs=0
 486 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
 487         MOV_FROM_ITC(p0, p6, r30, r23)          // M    get cycle for accounting
 488 #else
 489         nop.m 0
 490 #endif
 491         nop.i 0
 492         ;;
 493         mov r23=ar.bspstore                     // M2 (12 cyc) save ar.bspstore
 494         mov.m r24=ar.rnat                       // M2 (5 cyc) read ar.rnat (dual-issues!)
 495         nop.i 0
 496         ;;
 497         mov ar.bspstore=r22                     // M2 (6 cyc) switch to kernel RBS
 498         movl r8=PSR_ONE_BITS                    // X
 499         ;;
 500         mov r25=ar.unat                         // M2 (5 cyc) save ar.unat
 501         mov r19=b6                              // I0   save b6 (2 cyc)
 502         mov r20=r1                              // A    save caller's gp in r20
 503         ;;
 504         or r29=r8,r29                           // A    construct cr.ipsr value to save
 505         mov b6=r18                              // I0   copy syscall entry-point to b6 (7 cyc)
 506         addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r2 // A compute base of memory stack
 507
 508         mov r18=ar.bsp                          // M2   save (kernel) ar.bsp (12 cyc)
 509         cmp.ne pKStk,pUStk=r0,r0                // A    set pKStk <- 0, pUStk <- 1
 510         br.call.sptk.many b7=ia64_syscall_setup // B
 511         ;;
 512 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
 513         // mov.m r30=ar.itc is called in advance
 514         add r16=TI_AC_STAMP+IA64_TASK_SIZE,r2
 515         add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r2
 516         ;;
 517         ld8 r18=[r16],TI_AC_STIME-TI_AC_STAMP   // time at last check in kernel
 518         ld8 r19=[r17],TI_AC_UTIME-TI_AC_LEAVE   // time at leave kernel
 519         ;;
 520         ld8 r20=[r16],TI_AC_STAMP-TI_AC_STIME   // cumulated stime
 521         ld8 r21=[r17]                           // cumulated utime
 522         sub r22=r19,r18                         // stime before leave kernel
 523         ;;
 524         st8 [r16]=r30,TI_AC_STIME-TI_AC_STAMP   // update stamp
 525         sub r18=r30,r19                         // elapsed time in user mode
 526         ;;
 527         add r20=r20,r22                         // sum stime
 528         add r21=r21,r18                         // sum utime
 529         ;;
 530         st8 [r16]=r20                           // update stime
 531         st8 [r17]=r21                           // update utime
 532         ;;
 533 #endif
 534         mov ar.rsc=0x3                          // M2   set eager mode, pl 0, LE, loadrs=0
 535         mov rp=r14                              // I0   set the real return addr
 536         and r3=_TIF_SYSCALL_TRACEAUDIT,r3       // A
 537         ;;
 538         SSM_PSR_I(p0, p6, r22)                  // M2   we're on kernel stacks now, reenable irqs
 539         cmp.eq p8,p0=r3,r0                      // A
 540 (p10)   br.cond.spnt.many ia64_ret_from_syscall // B    return if bad call-frame or r15 is a NaT
 541
 542         nop.m 0
 543 (p8)    br.call.sptk.many b6=b6                 // B    (ignore return address)
 544         br.cond.spnt ia64_trace_syscall         // B
 545 END(fsys_bubble_down)
 546
 547         .rodata
 548         .align 8
 549         .globl fsyscall_table
 550
 551         data8 fsys_bubble_down
 552 fsyscall_table:
 553         data8 fsys_ni_syscall
 554         data8 0                         // exit                 // 1025
 555         data8 0                         // read
 556         data8 0                         // write
 557         data8 0                         // open
 558         data8 0                         // close
 559         data8 0                         // creat                // 1030
 560         data8 0                         // link
 561         data8 0                         // unlink
 562         data8 0                         // execve
 563         data8 0                         // chdir
 564         data8 0                         // fchdir               // 1035
 565         data8 0                         // utimes
 566         data8 0                         // mknod
 567         data8 0                         // chmod
 568         data8 0                         // chown
 569         data8 0                         // lseek                // 1040
 570         data8 fsys_getpid               // getpid
 571         data8 0                         // getppid
 572         data8 0                         // mount
 573         data8 0                         // umount
 574         data8 0                         // setuid               // 1045
 575         data8 0                         // getuid
 576         data8 0                         // geteuid
 577         data8 0                         // ptrace
 578         data8 0                         // access
 579         data8 0                         // sync                 // 1050
 580         data8 0                         // fsync
 581         data8 0                         // fdatasync
 582         data8 0                         // kill
 583         data8 0                         // rename
 584         data8 0                         // mkdir                // 1055
 585         data8 0                         // rmdir
 586         data8 0                         // dup
 587         data8 0                         // pipe
 588         data8 0                         // times
 589         data8 0                         // brk                  // 1060
 590         data8 0                         // setgid
 591         data8 0                         // getgid
 592         data8 0                         // getegid
 593         data8 0                         // acct
 594         data8 0                         // ioctl                // 1065
 595         data8 0                         // fcntl
 596         data8 0                         // umask
 597         data8 0                         // chroot
 598         data8 0                         // ustat
 599         data8 0                         // dup2                 // 1070
 600         data8 0                         // setreuid
 601         data8 0                         // setregid
 602         data8 0                         // getresuid
 603         data8 0                         // setresuid
 604         data8 0                         // getresgid            // 1075
 605         data8 0                         // setresgid
 606         data8 0                         // getgroups
 607         data8 0                         // setgroups
 608         data8 0                         // getpgid
 609         data8 0                         // setpgid              // 1080
 610         data8 0                         // setsid
 611         data8 0                         // getsid
 612         data8 0                         // sethostname
 613         data8 0                         // setrlimit
 614         data8 0                         // getrlimit            // 1085
 615         data8 0                         // getrusage
 616         data8 fsys_gettimeofday         // gettimeofday
 617         data8 0                         // settimeofday
 618         data8 0                         // select
 619         data8 0                         // poll                 // 1090
 620         data8 0                         // symlink
 621         data8 0                         // readlink
 622         data8 0                         // uselib
 623         data8 0                         // swapon
 624         data8 0                         // swapoff              // 1095
 625         data8 0                         // reboot
 626         data8 0                         // truncate
 627         data8 0                         // ftruncate
 628         data8 0                         // fchmod
 629         data8 0                         // fchown               // 1100
 630         data8 0                         // getpriority
 631         data8 0                         // setpriority
 632         data8 0                         // statfs
 633         data8 0                         // fstatfs
 634         data8 0                         // gettid               // 1105
 635         data8 0                         // semget
 636         data8 0                         // semop
 637         data8 0                         // semctl
 638         data8 0                         // msgget
 639         data8 0                         // msgsnd               // 1110
 640         data8 0                         // msgrcv
 641         data8 0                         // msgctl
 642         data8 0                         // shmget
 643         data8 0                         // shmat
 644         data8 0                         // shmdt                // 1115
 645         data8 0                         // shmctl
 646         data8 0                         // syslog
 647         data8 0                         // setitimer
 648         data8 0                         // getitimer
 649         data8 0                                                 // 1120
 650         data8 0
 651         data8 0
 652         data8 0                         // vhangup
 653         data8 0                         // lchown
 654         data8 0                         // remap_file_pages     // 1125
 655         data8 0                         // wait4
 656         data8 0                         // sysinfo
 657         data8 0                         // clone
 658         data8 0                         // setdomainname
 659         data8 0                         // newuname             // 1130
 660         data8 0                         // adjtimex
 661         data8 0
 662         data8 0                         // init_module
 663         data8 0                         // delete_module
 664         data8 0                                                 // 1135
 665         data8 0
 666         data8 0                         // quotactl
 667         data8 0                         // bdflush
 668         data8 0                         // sysfs
 669         data8 0                         // personality          // 1140
 670         data8 0                         // afs_syscall
 671         data8 0                         // setfsuid
 672         data8 0                         // setfsgid
 673         data8 0                         // getdents
 674         data8 0                         // flock                // 1145
 675         data8 0                         // readv
 676         data8 0                         // writev
 677         data8 0                         // pread64
 678         data8 0                         // pwrite64
 679         data8 0                         // sysctl               // 1150
 680         data8 0                         // mmap
 681         data8 0                         // munmap
 682         data8 0                         // mlock
 683         data8 0                         // mlockall
 684         data8 0                         // mprotect             // 1155
 685         data8 0                         // mremap
 686         data8 0                         // msync
 687         data8 0                         // munlock
 688         data8 0                         // munlockall
 689         data8 0                         // sched_getparam       // 1160
 690         data8 0                         // sched_setparam
 691         data8 0                         // sched_getscheduler
 692         data8 0                         // sched_setscheduler
 693         data8 0                         // sched_yield
 694         data8 0                         // sched_get_priority_max       // 1165
 695         data8 0                         // sched_get_priority_min
 696         data8 0                         // sched_rr_get_interval
 697         data8 0                         // nanosleep
 698         data8 0                         // nfsservctl
 699         data8 0                         // prctl                // 1170
 700         data8 0                         // getpagesize
 701         data8 0                         // mmap2
 702         data8 0                         // pciconfig_read
 703         data8 0                         // pciconfig_write
 704         data8 0                         // perfmonctl           // 1175
 705         data8 0                         // sigaltstack
 706         data8 0                         // rt_sigaction
 707         data8 0                         // rt_sigpending
 708         data8 0                         // rt_sigprocmask
 709         data8 0                         // rt_sigqueueinfo      // 1180
 710         data8 0                         // rt_sigreturn
 711         data8 0                         // rt_sigsuspend
 712         data8 0                         // rt_sigtimedwait
 713         data8 0                         // getcwd
 714         data8 0                         // capget               // 1185
 715         data8 0                         // capset
 716         data8 0                         // sendfile
 717         data8 0
 718         data8 0
 719         data8 0                         // socket               // 1190
 720         data8 0                         // bind
 721         data8 0                         // connect
 722         data8 0                         // listen
 723         data8 0                         // accept
 724         data8 0                         // getsockname          // 1195
 725         data8 0                         // getpeername
 726         data8 0                         // socketpair
 727         data8 0                         // send
 728         data8 0                         // sendto
 729         data8 0                         // recv                 // 1200
 730         data8 0                         // recvfrom
 731         data8 0                         // shutdown
 732         data8 0                         // setsockopt
 733         data8 0                         // getsockopt
 734         data8 0                         // sendmsg              // 1205
 735         data8 0                         // recvmsg
 736         data8 0                         // pivot_root
 737         data8 0                         // mincore
 738         data8 0                         // madvise
 739         data8 0                         // newstat              // 1210
 740         data8 0                         // newlstat
 741         data8 0                         // newfstat
 742         data8 0                         // clone2
 743         data8 0                         // getdents64
 744         data8 0                         // getunwind            // 1215
 745         data8 0                         // readahead
 746         data8 0                         // setxattr
 747         data8 0                         // lsetxattr
 748         data8 0                         // fsetxattr
 749         data8 0                         // getxattr             // 1220
 750         data8 0                         // lgetxattr
 751         data8 0                         // fgetxattr
 752         data8 0                         // listxattr
 753         data8 0                         // llistxattr
 754         data8 0                         // flistxattr           // 1225
 755         data8 0                         // removexattr
 756         data8 0                         // lremovexattr
 757         data8 0                         // fremovexattr
 758         data8 0                         // tkill
 759         data8 0                         // futex                // 1230
 760         data8 0                         // sched_setaffinity
 761         data8 0                         // sched_getaffinity
 762         data8 fsys_set_tid_address      // set_tid_address
 763         data8 0                         // fadvise64_64
 764         data8 0                         // tgkill               // 1235
 765         data8 0                         // exit_group
 766         data8 0                         // lookup_dcookie
 767         data8 0                         // io_setup
 768         data8 0                         // io_destroy
 769         data8 0                         // io_getevents         // 1240
 770         data8 0                         // io_submit
 771         data8 0                         // io_cancel
 772         data8 0                         // epoll_create
 773         data8 0                         // epoll_ctl
 774         data8 0                         // epoll_wait           // 1245
 775         data8 0                         // restart_syscall
 776         data8 0                         // semtimedop
 777         data8 0                         // timer_create
 778         data8 0                         // timer_settime
 779         data8 0                         // timer_gettime        // 1250
 780         data8 0                         // timer_getoverrun
 781         data8 0                         // timer_delete
 782         data8 0                         // clock_settime
 783         data8 fsys_clock_gettime        // clock_gettime
 784         data8 0                         // clock_getres         // 1255
 785         data8 0                         // clock_nanosleep
 786         data8 0                         // fstatfs64
 787         data8 0                         // statfs64
 788         data8 0                         // mbind
 789         data8 0                         // get_mempolicy        // 1260
 790         data8 0                         // set_mempolicy
 791         data8 0                         // mq_open
 792         data8 0                         // mq_unlink
 793         data8 0                         // mq_timedsend
 794         data8 0                         // mq_timedreceive      // 1265
 795         data8 0                         // mq_notify
 796         data8 0                         // mq_getsetattr
 797         data8 0                         // kexec_load
 798         data8 0                         // vserver
 799         data8 0                         // waitid               // 1270
 800         data8 0                         // add_key
 801         data8 0                         // request_key
 802         data8 0                         // keyctl
 803         data8 0                         // ioprio_set
 804         data8 0                         // ioprio_get           // 1275
 805         data8 0                         // move_pages
 806         data8 0                         // inotify_init
 807         data8 0                         // inotify_add_watch
 808         data8 0                         // inotify_rm_watch
 809         data8 0                         // migrate_pages        // 1280
 810         data8 0                         // openat
 811         data8 0                         // mkdirat
 812         data8 0                         // mknodat
 813         data8 0                         // fchownat
 814         data8 0                         // futimesat            // 1285
 815         data8 0                         // newfstatat
 816         data8 0                         // unlinkat
 817         data8 0                         // renameat
 818         data8 0                         // linkat
 819         data8 0                         // symlinkat            // 1290
 820         data8 0                         // readlinkat
 821         data8 0                         // fchmodat
 822         data8 0                         // faccessat
 823         data8 0
 824         data8 0                                                 // 1295
 825         data8 0                         // unshare
 826         data8 0                         // splice
 827         data8 0                         // set_robust_list
 828         data8 0                         // get_robust_list
 829         data8 0                         // sync_file_range      // 1300
 830         data8 0                         // tee
 831         data8 0                         // vmsplice
 832         data8 0
 833         data8 fsys_getcpu               // getcpu               // 1304
 834
 835         // fill in zeros for the remaining entries
 836         .zero:
 837         .space fsyscall_table + 8*NR_syscalls - .zero, 0