2
* This file contains the light-weight system call handlers (fsyscall-handlers).
4
* Copyright (C) 2003 Hewlett-Packard Co
5
* David Mosberger-Tang <davidm@hpl.hp.com>
7
* 25-Sep-03 davidm Implement fsys_rt_sigprocmask().
8
* 18-Feb-03 louisk Implement fsys_gettimeofday().
9
* 28-Feb-03 davidm Fixed several bugs in fsys_gettimeofday(). Tuned it some more,
10
* probably broke it along the way... ;-)
11
* 13-Jul-04 clameter Implement fsys_clock_gettime and revise fsys_gettimeofday to make
12
* it capable of using memory based clocks without falling back to C code.
13
* 08-Feb-07 Fenghua Yu Implement fsys_getcpu.
17
#include <asm/asmmacro.h>
18
#include <asm/errno.h>
19
#include <asm/asm-offsets.h>
20
#include <asm/percpu.h>
21
#include <asm/thread_info.h>
23
#include <asm/signal.h>
24
#include <asm/system.h>
25
#include <asm/unistd.h>
28
#include "paravirt_inst.h"
31
* See Documentation/ia64/fsys.txt for details on fsyscalls.
33
* On entry to an fsyscall handler:
34
* r10 = 0 (i.e., defaults to "successful syscall return")
35
* r11 = saved ar.pfs (a user-level value)
36
* r15 = system call number
37
* r16 = "current" task pointer (in normal kernel-mode, this is in r13)
38
* r32-r39 = system call arguments
39
* b6 = return address (a user-level value)
40
* ar.pfs = previous frame-state (a user-level value)
41
* PSR.be = cleared to zero (i.e., little-endian byte order is in effect)
42
* all other registers may contain values passed in from user-mode
44
* On return from an fsyscall handler:
45
* r11 = saved ar.pfs (as passed into the fsyscall handler)
46
* r15 = system call number (as passed into the fsyscall handler)
47
* r32-r39 = system call arguments (as passed into the fsyscall handler)
48
* b6 = return address (as passed into the fsyscall handler)
49
* ar.pfs = previous frame-state (as passed into the fsyscall handler)
52
ENTRY(fsys_ni_syscall)
65
add r17=IA64_TASK_GROUP_LEADER_OFFSET,r16
67
ld8 r17=[r17] // r17 = current->group_leader
68
add r9=TI_FLAGS+IA64_TASK_SIZE,r16
71
add r17=IA64_TASK_TGIDLINK_OFFSET,r17
73
and r9=TIF_ALLWORK_MASK,r9
74
ld8 r17=[r17] // r17 = current->group_leader->pids[PIDTYPE_PID].pid
76
add r8=IA64_PID_LEVEL_OFFSET,r17
78
ld4 r8=[r8] // r8 = pid->level
79
add r17=IA64_PID_UPID_OFFSET,r17 // r17 = &pid->numbers[0]
81
shl r8=r8,IA64_UPID_SHIFT
83
add r17=r17,r8 // r17 = &pid->numbers[pid->level]
85
ld4 r8=[r17] // r8 = pid->numbers[pid->level].nr
90
(p8) br.spnt.many fsys_fallback_syscall
98
add r17=IA64_TASK_GROUP_LEADER_OFFSET,r16
100
ld8 r17=[r17] // r17 = current->group_leader
101
add r9=TI_FLAGS+IA64_TASK_SIZE,r16
105
add r17=IA64_TASK_REAL_PARENT_OFFSET,r17 // r17 = ¤t->group_leader->real_parent
107
and r9=TIF_ALLWORK_MASK,r9
109
1: ld8 r18=[r17] // r18 = current->group_leader->real_parent
112
add r8=IA64_TASK_TGID_OFFSET,r18 // r8 = ¤t->group_leader->real_parent->tgid
116
* The .acq is needed to ensure that the read of tgid has returned its data before
117
* we re-check "real_parent".
119
ld4.acq r8=[r8] // r8 = current->group_leader->real_parent->tgid
122
* Re-read current->group_leader->real_parent.
124
ld8 r19=[r17] // r19 = current->group_leader->real_parent
125
(p8) br.spnt.many fsys_fallback_syscall
127
cmp.ne p6,p0=r18,r19 // did real_parent change?
128
mov r19=0 // i must not leak kernel bits...
129
(p6) br.cond.spnt.few 1b // yes -> redo the read of tgid and the check
131
mov r17=0 // i must not leak kernel bits...
132
mov r18=0 // i must not leak kernel bits...
134
mov r17=0 // i must not leak kernel bits...
135
mov r18=0 // i must not leak kernel bits...
136
mov r19=0 // i must not leak kernel bits...
141
ENTRY(fsys_set_tid_address)
145
add r9=TI_FLAGS+IA64_TASK_SIZE,r16
146
add r17=IA64_TASK_TGIDLINK_OFFSET,r16
149
tnat.z p6,p7=r32 // check argument register for being NaT
150
ld8 r17=[r17] // r17 = current->pids[PIDTYPE_PID].pid
152
and r9=TIF_ALLWORK_MASK,r9
153
add r8=IA64_PID_LEVEL_OFFSET,r17
154
add r18=IA64_TASK_CLEAR_CHILD_TID_OFFSET,r16
156
ld4 r8=[r8] // r8 = pid->level
157
add r17=IA64_PID_UPID_OFFSET,r17 // r17 = &pid->numbers[0]
159
shl r8=r8,IA64_UPID_SHIFT
161
add r17=r17,r8 // r17 = &pid->numbers[pid->level]
163
ld4 r8=[r17] // r8 = pid->numbers[pid->level].nr
170
(p8) br.spnt.many fsys_fallback_syscall
172
mov r17=0 // i must not leak kernel bits...
173
mov r18=0 // i must not leak kernel bits...
175
END(fsys_set_tid_address)
177
#if IA64_GTOD_LOCK_OFFSET !=0
178
#error fsys_gettimeofday incompatible with changes to struct fsyscall_gtod_data_t
180
#if IA64_ITC_JITTER_OFFSET !=0
181
#error fsys_gettimeofday incompatible with changes to struct itc_jitter_data_t
183
#define CLOCK_REALTIME 0
184
#define CLOCK_MONOTONIC 1
185
#define CLOCK_DIVIDE_BY_1000 0x4000
186
#define CLOCK_ADD_MONOTONIC 0x8000
188
ENTRY(fsys_gettimeofday)
193
tnat.nz p6,p0 = r33 // guard against NaT argument
194
(p6) br.cond.spnt.few .fail_einval
195
mov r30 = CLOCK_DIVIDE_BY_1000
199
// Incoming r31 = pointer to address where to place result
200
// r30 = flags determining how time is processed
201
// r2,r3 = temp r4-r7 preserved
202
// r8 = result nanoseconds
203
// r9 = result seconds
204
// r10 = temporary storage for clock difference
205
// r11 = preserved: saved ar.pfs
206
// r12 = preserved: memory stack
207
// r13 = preserved: thread pointer
208
// r14 = address of mask / mask value
209
// r15 = preserved: system call number
210
// r16 = preserved: current task pointer
213
// r19 = address of itc_lastcycle
214
// r20 = struct fsyscall_gtod_data (= address of gtod_lock.sequence)
215
// r21 = address of mmio_ptr
216
// r22 = address of wall_time or monotonic_time
217
// r23 = address of shift / value
218
// r24 = address mult factor / cycle_last value
219
// r25 = itc_lastcycle value
220
// r26 = address clocksource cycle_last
222
// r28 = sequence number at the beginning of critcal section
223
// r29 = address of itc_jitter
224
// r30 = time processing flags / memory address
225
// r31 = pointer to result
227
// p6,p7 short term use
228
// p8 = timesource ar.itc
229
// p9 = timesource mmio64
230
// p10 = timesource mmio32 - not used
231
// p11 = timesource not to be handled by asm code
232
// p12 = memory time source ( = p9 | p10) - not used
233
// p13 = do cmpxchg with itc_lastcycle
234
// p14 = Divide by 1000
235
// p15 = Add monotonic
237
// Note that instructions are optimized for McKinley. McKinley can
238
// process two bundles simultaneously and therefore we continuously
239
// try to feed the CPU two bundles and then a stop.
241
add r2 = TI_FLAGS+IA64_TASK_SIZE,r16
242
tnat.nz p6,p0 = r31 // guard against Nat argument
243
(p6) br.cond.spnt.few .fail_einval
244
movl r20 = fsyscall_gtod_data // load fsyscall gettimeofday data address
246
ld4 r2 = [r2] // process work pending flags
247
movl r29 = itc_jitter_data // itc_jitter
248
add r22 = IA64_GTOD_WALL_TIME_OFFSET,r20 // wall_time
249
add r21 = IA64_CLKSRC_MMIO_OFFSET,r20
250
mov pr = r30,0xc000 // Set predicates according to function
252
and r2 = TIF_ALLWORK_MASK,r2
253
add r19 = IA64_ITC_LASTCYCLE_OFFSET,r29
254
(p15) add r22 = IA64_GTOD_MONO_TIME_OFFSET,r20 // monotonic_time
256
add r26 = IA64_CLKSRC_CYCLE_LAST_OFFSET,r20 // clksrc_cycle_last
257
cmp.ne p6, p0 = 0, r2 // Fallback if work is scheduled
258
(p6) br.cond.spnt.many fsys_fallback_syscall
260
// Begin critical section
262
ld4.acq r28 = [r20] // gtod_lock.sequence, Must take first
264
and r28 = ~1,r28 // And make sequence even to force retry if odd
266
ld8 r30 = [r21] // clocksource->mmio_ptr
267
add r24 = IA64_CLKSRC_MULT_OFFSET,r20
268
ld4 r2 = [r29] // itc_jitter value
269
add r23 = IA64_CLKSRC_SHIFT_OFFSET,r20
270
add r14 = IA64_CLKSRC_MASK_OFFSET,r20
272
ld4 r3 = [r24] // clocksource mult value
273
ld8 r14 = [r14] // clocksource mask value
274
cmp.eq p8,p9 = 0,r30 // use cpu timer if no mmio_ptr
276
setf.sig f7 = r3 // Setup for mult scaling of counter
277
(p8) cmp.ne p13,p0 = r2,r0 // need itc_jitter compensation, set p13
278
ld4 r23 = [r23] // clocksource shift value
279
ld8 r24 = [r26] // get clksrc_cycle_last value
280
(p9) cmp.eq p13,p0 = 0,r30 // if mmio_ptr, clear p13 jitter control
282
.pred.rel.mutex p8,p9
283
MOV_FROM_ITC(p8, p6, r2, r10) // CPU_TIMER. 36 clocks latency!!!
284
(p9) ld8 r2 = [r30] // MMIO_TIMER. Could also have latency issues..
285
(p13) ld8 r25 = [r19] // get itc_lastcycle value
286
ld8 r9 = [r22],IA64_TIMESPEC_TV_NSEC_OFFSET // tv_sec
288
ld8 r8 = [r22],-IA64_TIMESPEC_TV_NSEC_OFFSET // tv_nsec
289
(p13) sub r3 = r25,r2 // Diff needed before comparison (thanks davidm)
291
(p13) cmp.gt.unc p6,p7 = r3,r0 // check if it is less than last. p6,p7 cleared
292
sub r10 = r2,r24 // current_cycle - last_cycle
294
(p6) sub r10 = r25,r24 // time we got was less than last_cycle
295
(p7) mov ar.ccv = r25 // more than last_cycle. Prep for cmpxchg
297
(p7) cmpxchg8.rel r3 = [r19],r2,ar.ccv
299
(p7) cmp.ne p7,p0 = r25,r3 // if cmpxchg not successful
301
(p7) sub r10 = r3,r24 // then use new last_cycle instead
303
and r10 = r10,r14 // Apply mask
308
// fault check takes 5 cycles and we have spare time
309
EX(.fail_efault, probe.w.fault r31, 3)
310
xmpy.l f8 = f8,f7 // nsec_per_cyc*(counter-last_counter)
315
ld4 r10 = [r20] // gtod_lock.sequence
316
shr.u r2 = r2,r23 // shift by factor
318
add r8 = r8,r2 // Add xtime.nsecs
319
cmp4.ne p7,p0 = r28,r10
320
(p7) br.cond.dpnt.few .time_redo // sequence number changed, redo
321
// End critical section.
322
// Now r8=tv->tv_nsec and r9=tv->tv_sec
325
add r23 = IA64_TIMESPEC_TV_NSEC_OFFSET, r31
326
(p14) movl r3 = 2361183241434822607 // Prep for / 1000 hack
331
(p14) shr.u r20 = r8, 3 // We can repeat this if necessary just wasting time
333
(p14) setf.sig f8 = r20
335
(p6) add r9 = 1,r9 // two nops before the branch.
336
(p14) setf.sig f7 = r3 // Chances for repeats are 1 in 10000 for gettod
337
(p6) br.cond.dpnt.few .time_normalize
339
// Divided by 8 though shift. Now divide by 125
340
// The compiler was able to do that with a multiply
341
// and a shift and we do the same
342
EX(.fail_efault, probe.w.fault r23, 3) // This also costs 5 cycles
343
(p14) xmpy.hu f8 = f8, f7 // xmpy has 5 cycles latency so use it
345
(p14) getf.sig r2 = f8
348
(p14) shr.u r21 = r2, 4
350
EX(.fail_efault, st8 [r31] = r9)
351
EX(.fail_efault, st8 [r23] = r21)
361
END(fsys_gettimeofday)
363
ENTRY(fsys_clock_gettime)
367
cmp4.ltu p6, p0 = CLOCK_MONOTONIC, r32
368
// Fallback if this is not CLOCK_REALTIME or CLOCK_MONOTONIC
369
(p6) br.spnt.few fsys_fallback_syscall
373
END(fsys_clock_gettime)
376
* long fsys_rt_sigprocmask (int how, sigset_t *set, sigset_t *oset, size_t sigsetsize).
379
# error Sorry, fsys_rt_sigprocmask() needs to be updated for _NSIG_WORDS != 1.
381
ENTRY(fsys_rt_sigprocmask)
386
add r2=IA64_TASK_BLOCKED_OFFSET,r16
387
add r9=TI_FLAGS+IA64_TASK_SIZE,r16
388
cmp4.ltu p6,p0=SIG_SETMASK,r32
390
cmp.ne p15,p0=r0,r34 // oset != NULL?
392
add r31=IA64_TASK_SIGHAND_OFFSET,r16
394
ld8 r3=[r2] // read/prefetch current->blocked
398
cmp.ne.or p6,p0=_NSIG_WORDS*8,r35
400
(p6) br.spnt.few .fail_einval // fail with EINVAL
403
ld8 r31=[r31] // r31 <- current->sighand
405
and r9=TIF_ALLWORK_MASK,r9
409
cmp.eq p6,p0=r0,r33 // set == NULL?
410
add r31=IA64_SIGHAND_SIGLOCK_OFFSET,r31 // r31 <- current->sighand->siglock
411
(p8) br.spnt.few .fail_efault // fail with EFAULT
412
(p7) br.spnt.many fsys_fallback_syscall // got pending kernel work...
413
(p6) br.dpnt.many .store_mask // -> short-circuit to just reading the signal mask
415
/* Argh, we actually have to do some work and _update_ the signal mask: */
417
EX(.fail_efault, probe.r.fault r33, 3) // verify user has read-access to *set
418
EX(.fail_efault, ld8 r14=[r33]) // r14 <- *set
419
mov r17=(1 << (SIGKILL - 1)) | (1 << (SIGSTOP - 1))
422
RSM_PSR_I(p0, r18, r19) // mask interrupt delivery
423
andcm r14=r14,r17 // filter out SIGKILL & SIGSTOP
424
mov r8=EINVAL // default to EINVAL
427
// __ticket_spin_trylock(r31)
437
(p6) cmpxchg4.acq r9=[r31],r19,ar.ccv
438
(p6) dep.z r20=r19,1,15 // next serving ticket for unlock
439
(p7) br.cond.spnt.many .lock_contention
443
(p7) br.cond.spnt.many .lock_contention
444
ld8 r3=[r2] // re-read current->blocked now that we hold the lock
447
ld8 r3=[r2] // re-read current->blocked now that we hold the lock
449
add r18=IA64_TASK_PENDING_OFFSET+IA64_SIGPENDING_SIGNAL_OFFSET,r16
450
add r19=IA64_TASK_SIGNAL_OFFSET,r16
451
cmp4.eq p6,p0=SIG_BLOCK,r32
453
ld8 r19=[r19] // r19 <- current->signal
454
cmp4.eq p7,p0=SIG_UNBLOCK,r32
455
cmp4.eq p8,p0=SIG_SETMASK,r32
457
ld8 r18=[r18] // r18 <- current->pending.signal
458
.pred.rel.mutex p6,p7,p8
459
(p6) or r14=r3,r14 // SIG_BLOCK
460
(p7) andcm r14=r3,r14 // SIG_UNBLOCK
462
(p8) mov r14=r14 // SIG_SETMASK
463
(p6) mov r8=0 // clear error code
464
// recalc_sigpending()
465
add r17=IA64_SIGNAL_GROUP_STOP_COUNT_OFFSET,r19
467
add r19=IA64_SIGNAL_SHARED_PENDING_OFFSET+IA64_SIGPENDING_SIGNAL_OFFSET,r19
469
ld4 r17=[r17] // r17 <- current->signal->group_stop_count
470
(p7) mov r8=0 // clear error code
472
ld8 r19=[r19] // r19 <- current->signal->shared_pending
474
cmp4.gt p6,p7=r17,r0 // p6/p7 <- (current->signal->group_stop_count > 0)?
475
(p8) mov r8=0 // clear error code
477
or r18=r18,r19 // r18 <- current->pending | current->signal->shared_pending
479
// r18 <- (current->pending | current->signal->shared_pending) & ~current->blocked:
481
add r9=TI_FLAGS+IA64_TASK_SIZE,r16
484
(p7) cmp.ne.or.andcm p6,p7=r18,r0 // p6/p7 <- signal pending
485
mov r19=0 // i must not leak kernel bits...
486
(p6) br.cond.dpnt.many .sig_pending
489
1: ld4 r17=[r9] // r17 <- current->thread_info->flags
492
and r18=~_TIF_SIGPENDING,r17 // r18 <- r17 & ~(1 << TIF_SIGPENDING)
495
st8 [r2]=r14 // update current->blocked with new mask
496
cmpxchg4.acq r8=[r9],r18,ar.ccv // current->thread_info->flags <- r18
498
cmp.ne p6,p0=r17,r8 // update failed?
499
(p6) br.cond.spnt.few 1b // yes -> retry
502
// __ticket_spin_unlock(r31)
504
mov r20=0 // i must not leak kernel bits...
506
SSM_PSR_I(p0, p9, r31)
509
srlz.d // ensure psr.i is set again
510
mov r18=0 // i must not leak kernel bits...
513
EX(.fail_efault, (p15) probe.w.fault r34, 3) // verify user has write-access to *oset
514
EX(.fail_efault, (p15) st8 [r34]=r3)
515
mov r2=0 // i must not leak kernel bits...
516
mov r3=0 // i must not leak kernel bits...
518
mov r9=0 // i must not leak kernel bits...
519
mov r14=0 // i must not leak kernel bits...
520
mov r17=0 // i must not leak kernel bits...
521
mov r31=0 // i must not leak kernel bits...
526
// __ticket_spin_unlock(r31)
527
st2.rel [r31]=r20 // release the lock
529
SSM_PSR_I(p0, p9, r17)
532
br.sptk.many fsys_fallback_syscall // with signal pending, do the heavy-weight syscall
536
/* Rather than spinning here, fall back on doing a heavy-weight syscall. */
537
SSM_PSR_I(p0, p9, r17)
540
br.sptk.many fsys_fallback_syscall
542
END(fsys_rt_sigprocmask)
545
* fsys_getcpu doesn't use the third parameter in this implementation. It reads
546
* current_thread_info()->cpu and corresponding node in cpu_to_node_map.
553
add r2=TI_FLAGS+IA64_TASK_SIZE,r16
554
tnat.nz p6,p0 = r32 // guard against NaT argument
555
add r3=TI_CPU+IA64_TASK_SIZE,r16
557
ld4 r3=[r3] // M r3 = thread_info->cpu
558
ld4 r2=[r2] // M r2 = thread_info->flags
559
(p6) br.cond.spnt.few .fail_einval // B
561
tnat.nz p7,p0 = r33 // I guard against NaT argument
562
(p7) br.cond.spnt.few .fail_einval // B
564
movl r17=cpu_to_node_map
566
EX(.fail_efault, probe.w.fault r32, 3) // M This takes 5 cycles
567
EX(.fail_efault, probe.w.fault r33, 3) // M This takes 5 cycles
570
ld2 r20=[r18] // r20 = cpu_to_node_map[cpu]
571
and r2 = TIF_ALLWORK_MASK,r2
574
(p8) br.spnt.many fsys_fallback_syscall
577
EX(.fail_efault, st4 [r32] = r3)
578
EX(.fail_efault, st2 [r33] = r20)
582
EX(.fail_efault, probe.w.fault r32, 3) // M This takes 5 cycles
583
EX(.fail_efault, probe.w.fault r33, 3) // M This takes 5 cycles
584
and r2 = TIF_ALLWORK_MASK,r2
587
(p8) br.spnt.many fsys_fallback_syscall
589
EX(.fail_efault, st4 [r32] = r3)
590
EX(.fail_efault, st2 [r33] = r0)
597
ENTRY(fsys_fallback_syscall)
602
* We only get here from light-weight syscall handlers. Thus, we already
603
* know that r15 contains a valid syscall number. No need to re-check.
606
movl r14=sys_call_table
608
RSM_PSR_I(p0, r26, r27)
611
ld8 r18=[r18] // load normal (heavy-weight) syscall entry-point
612
MOV_FROM_PSR(p0, r29, r26) // read psr (12 cyc load latency)
616
END(fsys_fallback_syscall)
618
GLOBAL_ENTRY(paravirt_fsys_bubble_down)
623
* We get here for syscalls that don't have a lightweight
624
* handler. For those, we need to bubble down into the kernel
625
* and that requires setting up a minimal pt_regs structure,
626
* and initializing the CPU state more or less as if an
627
* interruption had occurred. To make syscall-restarts work,
628
* we setup pt_regs such that cr_iip points to the second
629
* instruction in syscall_via_break. Decrementing the IP
630
* hence will restart the syscall via break and not
631
* decrementing IP will return us to the caller, as usual.
632
* Note that we preserve the value of psr.pp rather than
633
* initializing it from dcr.pp. This makes it possible to
634
* distinguish fsyscall execution from other privileged
638
* - normal fsyscall handler register usage, except
640
* - r18: address of syscall entry point
646
* We used to clear some PSR bits here but that requires slow
647
* serialization. Fortuntely, that isn't really necessary.
648
* The rationale is as follows: we used to clear bits
649
* ~PSR_PRESERVED_BITS in PSR.L. Since
650
* PSR_PRESERVED_BITS==PSR.{UP,MFL,MFH,PK,DT,PP,SP,RT,IC}, we
651
* ended up clearing PSR.{BE,AC,I,DFL,DFH,DI,DB,SI,TB}.
654
* PSR.BE : already is turned off in __kernel_syscall_via_epc()
655
* PSR.AC : don't care (kernel normally turns PSR.AC on)
656
* PSR.I : already turned off by the time paravirt_fsys_bubble_down gets
658
* PSR.DFL: always 0 (kernel never turns it on)
659
* PSR.DFH: don't care --- kernel never touches f32-f127 on its own
661
* PSR.DI : always 0 (kernel never turns it on)
662
* PSR.SI : always 0 (kernel never turns it on)
663
* PSR.DB : don't care --- kernel never enables kernel-level
665
* PSR.TB : must be 0 already; if it wasn't zero on entry to
666
* __kernel_syscall_via_epc, the branch to paravirt_fsys_bubble_down
667
* will trigger a taken branch; the taken-trap-handler then
668
* converts the syscall into a break-based system-call.
671
* Reading psr.l gives us only bits 0-31, psr.it, and psr.mc.
672
* The rest we have to synthesize.
674
# define PSR_ONE_BITS ((3 << IA64_PSR_CPL0_BIT) \
675
| (0x1 << IA64_PSR_RI_BIT) \
676
| IA64_PSR_BN | IA64_PSR_I)
679
movl r14=ia64_ret_from_syscall // X
682
movl r28=__kernel_syscall_via_break // X create cr.iip
685
mov r2=r16 // A get task addr to addl-addressable register
686
adds r16=IA64_TASK_THREAD_ON_USTACK_OFFSET,r16 // A
687
mov r31=pr // I0 save pr (2 cyc)
689
st1 [r16]=r0 // M2|3 clear current->thread.on_ustack flag
690
addl r22=IA64_RBS_OFFSET,r2 // A compute base of RBS
691
add r3=TI_FLAGS+IA64_TASK_SIZE,r2 // A
693
ld4 r3=[r3] // M0|1 r3 = current_thread_info()->flags
694
lfetch.fault.excl.nt1 [r22] // M0|1 prefetch register backing-store
697
mov ar.rsc=0 // M2 set enforced lazy mode, pl 0, LE, loadrs=0
698
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
699
MOV_FROM_ITC(p0, p6, r30, r23) // M get cycle for accounting
705
mov r23=ar.bspstore // M2 (12 cyc) save ar.bspstore
706
mov.m r24=ar.rnat // M2 (5 cyc) read ar.rnat (dual-issues!)
709
mov ar.bspstore=r22 // M2 (6 cyc) switch to kernel RBS
710
movl r8=PSR_ONE_BITS // X
712
mov r25=ar.unat // M2 (5 cyc) save ar.unat
713
mov r19=b6 // I0 save b6 (2 cyc)
714
mov r20=r1 // A save caller's gp in r20
716
or r29=r8,r29 // A construct cr.ipsr value to save
717
mov b6=r18 // I0 copy syscall entry-point to b6 (7 cyc)
718
addl r1=IA64_STK_OFFSET-IA64_PT_REGS_SIZE,r2 // A compute base of memory stack
720
mov r18=ar.bsp // M2 save (kernel) ar.bsp (12 cyc)
721
cmp.ne pKStk,pUStk=r0,r0 // A set pKStk <- 0, pUStk <- 1
722
br.call.sptk.many b7=ia64_syscall_setup // B
724
#ifdef CONFIG_VIRT_CPU_ACCOUNTING
725
// mov.m r30=ar.itc is called in advance
726
add r16=TI_AC_STAMP+IA64_TASK_SIZE,r2
727
add r17=TI_AC_LEAVE+IA64_TASK_SIZE,r2
729
ld8 r18=[r16],TI_AC_STIME-TI_AC_STAMP // time at last check in kernel
730
ld8 r19=[r17],TI_AC_UTIME-TI_AC_LEAVE // time at leave kernel
732
ld8 r20=[r16],TI_AC_STAMP-TI_AC_STIME // cumulated stime
733
ld8 r21=[r17] // cumulated utime
734
sub r22=r19,r18 // stime before leave kernel
736
st8 [r16]=r30,TI_AC_STIME-TI_AC_STAMP // update stamp
737
sub r18=r30,r19 // elapsed time in user mode
739
add r20=r20,r22 // sum stime
740
add r21=r21,r18 // sum utime
742
st8 [r16]=r20 // update stime
743
st8 [r17]=r21 // update utime
746
mov ar.rsc=0x3 // M2 set eager mode, pl 0, LE, loadrs=0
747
mov rp=r14 // I0 set the real return addr
748
and r3=_TIF_SYSCALL_TRACEAUDIT,r3 // A
750
SSM_PSR_I(p0, p6, r22) // M2 we're on kernel stacks now, reenable irqs
751
cmp.eq p8,p0=r3,r0 // A
752
(p10) br.cond.spnt.many ia64_ret_from_syscall // B return if bad call-frame or r15 is a NaT
755
(p8) br.call.sptk.many b6=b6 // B (ignore return address)
756
br.cond.spnt ia64_trace_syscall // B
757
END(paravirt_fsys_bubble_down)
761
.globl paravirt_fsyscall_table
763
data8 paravirt_fsys_bubble_down
764
paravirt_fsyscall_table:
765
data8 fsys_ni_syscall
766
data8 0 // exit // 1025
771
data8 0 // creat // 1030
776
data8 0 // fchdir // 1035
781
data8 0 // lseek // 1040
782
data8 fsys_getpid // getpid
783
data8 fsys_getppid // getppid
786
data8 0 // setuid // 1045
791
data8 0 // sync // 1050
796
data8 0 // mkdir // 1055
801
data8 0 // brk // 1060
806
data8 0 // ioctl // 1065
811
data8 0 // dup2 // 1070
816
data8 0 // getresgid // 1075
821
data8 0 // setpgid // 1080
824
data8 0 // sethostname
826
data8 0 // getrlimit // 1085
828
data8 fsys_gettimeofday // gettimeofday
829
data8 0 // settimeofday
831
data8 0 // poll // 1090
836
data8 0 // swapoff // 1095
841
data8 0 // fchown // 1100
842
data8 0 // getpriority
843
data8 0 // setpriority
846
data8 0 // gettid // 1105
851
data8 0 // msgsnd // 1110
856
data8 0 // shmdt // 1115
866
data8 0 // remap_file_pages // 1125
870
data8 0 // setdomainname
871
data8 0 // newuname // 1130
874
data8 0 // init_module
875
data8 0 // delete_module
881
data8 0 // personality // 1140
882
data8 0 // afs_syscall
886
data8 0 // flock // 1145
891
data8 0 // sysctl // 1150
896
data8 0 // mprotect // 1155
900
data8 0 // munlockall
901
data8 0 // sched_getparam // 1160
902
data8 0 // sched_setparam
903
data8 0 // sched_getscheduler
904
data8 0 // sched_setscheduler
905
data8 0 // sched_yield
906
data8 0 // sched_get_priority_max // 1165
907
data8 0 // sched_get_priority_min
908
data8 0 // sched_rr_get_interval
910
data8 0 // nfsservctl
911
data8 0 // prctl // 1170
912
data8 0 // getpagesize
914
data8 0 // pciconfig_read
915
data8 0 // pciconfig_write
916
data8 0 // perfmonctl // 1175
917
data8 0 // sigaltstack
918
data8 0 // rt_sigaction
919
data8 0 // rt_sigpending
920
data8 fsys_rt_sigprocmask // rt_sigprocmask
921
data8 0 // rt_sigqueueinfo // 1180
922
data8 0 // rt_sigreturn
923
data8 0 // rt_sigsuspend
924
data8 0 // rt_sigtimedwait
926
data8 0 // capget // 1185
931
data8 0 // socket // 1190
936
data8 0 // getsockname // 1195
937
data8 0 // getpeername
938
data8 0 // socketpair
941
data8 0 // recv // 1200
944
data8 0 // setsockopt
945
data8 0 // getsockopt
946
data8 0 // sendmsg // 1205
948
data8 0 // pivot_root
951
data8 0 // newstat // 1210
955
data8 0 // getdents64
956
data8 0 // getunwind // 1215
961
data8 0 // getxattr // 1220
965
data8 0 // llistxattr
966
data8 0 // flistxattr // 1225
967
data8 0 // removexattr
968
data8 0 // lremovexattr
969
data8 0 // fremovexattr
971
data8 0 // futex // 1230
972
data8 0 // sched_setaffinity
973
data8 0 // sched_getaffinity
974
data8 fsys_set_tid_address // set_tid_address
975
data8 0 // fadvise64_64
976
data8 0 // tgkill // 1235
977
data8 0 // exit_group
978
data8 0 // lookup_dcookie
980
data8 0 // io_destroy
981
data8 0 // io_getevents // 1240
984
data8 0 // epoll_create
986
data8 0 // epoll_wait // 1245
987
data8 0 // restart_syscall
988
data8 0 // semtimedop
989
data8 0 // timer_create
990
data8 0 // timer_settime
991
data8 0 // timer_gettime // 1250
992
data8 0 // timer_getoverrun
993
data8 0 // timer_delete
994
data8 0 // clock_settime
995
data8 fsys_clock_gettime // clock_gettime
996
data8 0 // clock_getres // 1255
997
data8 0 // clock_nanosleep
1001
data8 0 // get_mempolicy // 1260
1002
data8 0 // set_mempolicy
1004
data8 0 // mq_unlink
1005
data8 0 // mq_timedsend
1006
data8 0 // mq_timedreceive // 1265
1007
data8 0 // mq_notify
1008
data8 0 // mq_getsetattr
1009
data8 0 // kexec_load
1011
data8 0 // waitid // 1270
1013
data8 0 // request_key
1015
data8 0 // ioprio_set
1016
data8 0 // ioprio_get // 1275
1017
data8 0 // move_pages
1018
data8 0 // inotify_init
1019
data8 0 // inotify_add_watch
1020
data8 0 // inotify_rm_watch
1021
data8 0 // migrate_pages // 1280
1026
data8 0 // futimesat // 1285
1027
data8 0 // newfstatat
1031
data8 0 // symlinkat // 1290
1032
data8 0 // readlinkat
1034
data8 0 // faccessat
1039
data8 0 // set_robust_list
1040
data8 0 // get_robust_list
1041
data8 0 // sync_file_range // 1300
1045
data8 fsys_getcpu // getcpu // 1304
1047
// fill in zeros for the remaining entries
1049
.space paravirt_fsyscall_table + 8*NR_syscalls - .zero, 0