2
* arch/ia64/kernel/hyperprivop.S
4
* Copyright (C) 2005 Hewlett-Packard Co
5
* Dan Magenheimer <dan.magenheimer@hp.com>
8
#include <linux/config.h>
10
#include <asm/asmmacro.h>
11
#include <asm/kregs.h>
12
#include <asm/offsets.h>
13
#include <asm/processor.h>
14
#include <asm/system.h>
15
#include <asm/debugger.h>
16
#include <asm/asm-xsi-offsets.h>
17
#include <asm/pgtable.h>
19
#include <public/xen.h>
22
#define PERFC(n) (THIS_CPU(perfcounters) + (IA64_PERFC_ ## n) * 4)
25
#define PAGE_PHYS (__DIRTY_BITS | _PAGE_PL_PRIV | _PAGE_AR_RWX)
27
#if 1 // change to 0 to turn off all fast paths
28
# define FAST_HYPERPRIVOPS
30
# define FAST_HYPERPRIVOP_CNT
31
# define FAST_HYPERPRIVOP_PERFC(N) PERFC(fast_hyperprivop + N)
32
# define FAST_REFLECT_CNT
35
//#define FAST_TICK // mostly working (unat problems) but default off for now
36
//#define FAST_TLB_MISS_REFLECT // mostly working but default off for now
37
# undef FAST_ITC //XXX TODO fast_itc doesn't support dom0 vp yet
39
# undef FAST_ACCESS_REFLECT //XXX TODO fast_access_reflect
40
// doesn't support dom0 vp yet.
42
// TODO: Since we use callback to deliver interrupt,
43
// FAST_SSM_I needs to be rewritten.
46
# undef RFI_TO_INTERRUPT // not working yet
47
# define FAST_SET_RR0_TO_RR4
51
//#warning "FIXME: ptc.ga instruction requires spinlock for SMP"
55
// FIXME: turn off for now... but NaTs may crash Xen so re-enable soon!
56
#define HANDLE_AR_UNAT
58
// FIXME: This is defined in include/asm-ia64/hw_irq.h but this
59
// doesn't appear to be include'able from assembly?
60
#define IA64_TIMER_VECTOR 0xef
62
// Note: not hand-scheduled for now
66
// r18 == XSI_PSR_IC_OFS
69
GLOBAL_ENTRY(fast_hyperprivop)
70
adds r20=XSI_PSR_I_ADDR_OFS-XSI_PSR_IC_OFS,r18
72
// assumes domain interrupts pending, so just do it
73
cmp.eq p7,p6=HYPERPRIVOP_SSM_I,r17
74
(p7) br.sptk.many hyper_ssm_i;;
76
// Check pending event indication
77
ld8 r20=[r20] // interrupt_mask_addr
79
ld1 r22=[r20],-1 // evtchn_upcall_mask
81
ld1 r20=[r20] // evtchn_upcall_pending
84
cmp.eq p7,p6=HYPERPRIVOP_RFI,r17
85
(p7) br.sptk.many hyper_rfi
87
#ifndef FAST_HYPERPRIVOPS // see beginning of file
88
br.sptk.many dispatch_break_fault ;;
90
// if event enabled and there are pending events
93
cmp.eq.and p7,p0=r22,r0
94
(p7) br.spnt.many dispatch_break_fault
98
cmp.eq p7,p0=HYPERPRIVOP_COVER,r17
99
(p7) br.sptk.many hyper_cover
102
// HYPERPRIVOP_SSM_DT?
103
cmp.eq p7,p0=HYPERPRIVOP_SSM_DT,r17
104
(p7) br.sptk.many hyper_ssm_dt
107
// HYPERPRIVOP_RSM_DT?
108
cmp.eq p7,p0=HYPERPRIVOP_RSM_DT,r17
109
(p7) br.sptk.many hyper_rsm_dt
112
// HYPERPRIVOP_SET_ITM?
113
cmp.eq p7,p0=HYPERPRIVOP_SET_ITM,r17
114
(p7) br.sptk.many hyper_set_itm
117
// HYPERPRIVOP_SET_RR0_TO_RR4?
118
cmp.eq p7,p0=HYPERPRIVOP_SET_RR0_TO_RR4,r17
119
(p7) br.sptk.many hyper_set_rr0_to_rr4
122
// HYPERPRIVOP_SET_RR?
123
cmp.eq p7,p0=HYPERPRIVOP_SET_RR,r17
124
(p7) br.sptk.many hyper_set_rr
127
// HYPERPRIVOP_GET_RR?
128
cmp.eq p7,p0=HYPERPRIVOP_GET_RR,r17
129
(p7) br.sptk.many hyper_get_rr
132
// HYPERPRIVOP_GET_PSR?
133
cmp.eq p7,p0=HYPERPRIVOP_GET_PSR,r17
134
(p7) br.sptk.many hyper_get_psr
137
// HYPERPRIVOP_PTC_GA?
138
cmp.eq p7,p0=HYPERPRIVOP_PTC_GA,r17
139
(p7) br.sptk.many hyper_ptc_ga
142
// HYPERPRIVOP_ITC_D?
143
cmp.eq p7,p0=HYPERPRIVOP_ITC_D,r17
144
(p7) br.sptk.many hyper_itc_d
147
// HYPERPRIVOP_ITC_I?
148
cmp.eq p7,p0=HYPERPRIVOP_ITC_I,r17
149
(p7) br.sptk.many hyper_itc_i
152
// HYPERPRIVOP_THASH?
153
cmp.eq p7,p0=HYPERPRIVOP_THASH,r17
154
(p7) br.sptk.many hyper_thash
157
// HYPERPRIVOP_SET_KR?
158
cmp.eq p7,p0=HYPERPRIVOP_SET_KR,r17
159
(p7) br.sptk.many hyper_set_kr
162
// if not one of the above, give up for now and do it the slow way
163
br.sptk.many dispatch_break_fault
165
END(fast_hyperprivop)
167
// give up for now if: ipsr.be==1, ipsr.pp==1
168
// from reflect_interruption, don't need to:
169
// - printk first extint (debug only)
170
// - check for interrupt collection enabled (routine will force on)
171
// - set ifa (not valid for extint)
172
// - set iha (not valid for extint)
173
// - set itir (not valid for extint)
175
// - increment the HYPER_SSM_I fast_hyperprivop counter
176
// - set shared_mem iip to instruction after HYPER_SSM_I
177
// - set cr.iip to guest iva+0x3000
178
// - set shared_mem ipsr to [vcpu_get_ipsr_int_state]
179
// be = pp = bn = 0; dt = it = rt = 1; cpl = 3 or 0;
180
// i = shared_mem interrupt_delivery_enabled
181
// ic = shared_mem interrupt_collection_enabled
182
// ri = instruction after HYPER_SSM_I
183
// all other bits unchanged from real cr.ipsr
184
// - set cr.ipsr (DELIVER_PSR_SET/CLEAR, don't forget cpl!)
185
// - set shared_mem isr: isr.ei to instr following HYPER_SSM_I
186
// and isr.ri to cr.isr.ri (all other bits zero)
187
// - cover and set shared_mem precover_ifs to cr.ifs
188
// ^^^ MISSED THIS FOR fast_break??
189
// - set shared_mem interrupt_delivery_enabled to 0
190
// - set shared_mem interrupt_collection_enabled to 0
191
// - set r31 to SHAREDINFO_ADDR
192
// - virtual bank switch 0
193
// maybe implement later
194
// - verify that there really IS a deliverable interrupt pending
195
// - set shared_mem iva
196
// needs to be done but not implemented (in reflect_interruption)
197
// - set shared_mem iipa
198
// don't know for sure
199
// - set shared_mem unat
207
br.spnt.few dispatch_break_fault ;;
209
// give up for now if: ipsr.be==1, ipsr.pp==1
212
tbit.nz p7,p0=r30,IA64_PSR_PP_BIT
213
(p7) br.spnt.many dispatch_break_fault ;;
214
#ifdef FAST_HYPERPRIVOP_CNT
215
movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_SSM_I);;
220
// set shared_mem iip to instruction after HYPER_SSM_I
221
tbit.nz p6,p7=r30,IA64_PSR_RI_BIT+1 ;; // cr.ipsr.ri >= 2 ?
224
(p7) adds r20=1,r20 ;;
225
dep r30=r20,r30,IA64_PSR_RI_BIT,2 // adjust cr.ipsr.ri but don't save yet
226
adds r21=XSI_IIP_OFS-XSI_PSR_IC_OFS,r18 ;;
228
// set shared_mem isr
229
extr.u r16=r16,IA64_ISR_IR_BIT,1;; // grab cr.isr.ir bit
230
dep r16=r16,r0,IA64_ISR_IR_BIT,1;; // insert into cr.isr (rest of bits zero)
231
dep r16=r20,r16,IA64_PSR_RI_BIT,2 // deposit cr.isr.ri
232
adds r21=XSI_ISR_OFS-XSI_PSR_IC_OFS,r18 ;;
236
movl r28=DELIVER_PSR_SET
237
movl r27=~(DELIVER_PSR_CLR & (~IA64_PSR_CPL));;
240
// set hpsr_dfh to ipsr
241
adds r28=XSI_HPSR_DFH_OFS-XSI_PSR_IC_OFS,r18;;
243
dep r29=r28,r29,IA64_PSR_DFH_BIT,1;;
245
// set shared_mem ipsr (from ipsr in r30 with ipsr.ri already set)
246
extr.u r29=r30,IA64_PSR_CPL0_BIT,2;;
247
cmp.eq p7,p0=CONFIG_CPL0_EMUL,r29;;
248
(p7) dep r30=0,r30,IA64_PSR_CPL0_BIT,2
249
// FOR SSM_I ONLY, also turn on psr.i and psr.ic
250
movl r28=(IA64_PSR_DT|IA64_PSR_IT|IA64_PSR_RT|IA64_PSR_I|IA64_PSR_IC)
251
// movl r27=~(IA64_PSR_BE|IA64_PSR_PP|IA64_PSR_BN);;
252
movl r27=~IA64_PSR_BN;;
256
movl r22=THIS_CPU(current_psr_i_addr)
257
adds r21=XSI_IPSR_OFS-XSI_PSR_IC_OFS,r18 ;;
259
adds r27=XSI_VPSR_DFH_OFS-XSI_PSR_IC_OFS,r18;;
262
dep r30=r28,r30,IA64_PSR_DFH_BIT,1
265
// set shared_mem interrupt_delivery_enabled to 0
266
// set shared_mem interrupt_collection_enabled to 0
269
// cover and set shared_mem precover_ifs to cr.ifs
270
// set shared_mem ifs to 0
273
adds r21=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
275
adds r21=XSI_PRECOVER_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
277
// leave cr.ifs alone for later rfi
278
// set iip to go to event callback handler
279
movl r22=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
281
adds r22=IA64_VCPU_EVENT_CALLBACK_IP_OFFSET,r22;;
284
// OK, now all set to go except for switch to virtual bank0
288
adds r2=XSI_BANK1_R16_OFS-XSI_PSR_IC_OFS,r18
289
adds r3=(XSI_BANK1_R16_OFS+8)-XSI_PSR_IC_OFS,r18
290
// temporarily save ar.unat
293
// FIXME?: ar.unat is not really handled correctly,
294
// but may not matter if the OS is NaT-clean
295
.mem.offset 0,0; st8.spill [r2]=r16,16
296
.mem.offset 8,0; st8.spill [r3]=r17,16 ;;
297
.mem.offset 0,0; st8.spill [r2]=r18,16
298
.mem.offset 8,0; st8.spill [r3]=r19,16 ;;
299
.mem.offset 0,0; st8.spill [r2]=r20,16
300
.mem.offset 8,0; st8.spill [r3]=r21,16 ;;
301
.mem.offset 0,0; st8.spill [r2]=r22,16
302
.mem.offset 8,0; st8.spill [r3]=r23,16 ;;
303
.mem.offset 0,0; st8.spill [r2]=r24,16
304
.mem.offset 8,0; st8.spill [r3]=r25,16 ;;
305
.mem.offset 0,0; st8.spill [r2]=r26,16
306
.mem.offset 8,0; st8.spill [r3]=r27,16 ;;
307
.mem.offset 0,0; st8.spill [r2]=r28,16
308
.mem.offset 8,0; st8.spill [r3]=r29,16 ;;
309
.mem.offset 0,0; st8.spill [r2]=r30,16
310
.mem.offset 8,0; st8.spill [r3]=r31,16 ;;
313
adds r26=XSI_B1NATS_OFS-XSI_PSR_IC_OFS,r18 ;;
320
adds r20=XSI_BANKNUM_OFS-XSI_PSR_IC_OFS,r18 ;;
327
// reflect domain clock interrupt
331
GLOBAL_ENTRY(fast_tick_reflect)
332
#ifndef FAST_TICK // see beginning of file
333
br.cond.sptk.many rp;;
335
mov r28=IA64_TIMER_VECTOR;;
337
(p6) br.cond.spnt.few rp;;
338
movl r20=THIS_CPU(cpu_info)+IA64_CPUINFO_ITM_NEXT_OFFSET;;
341
adds r27=200,r27;; // safety margin
342
cmp.ltu p6,p0=r26,r27
343
(p6) br.cond.spnt.few rp;;
345
// slow path if: ipsr.pp==1
346
tbit.nz p6,p0=r17,IA64_PSR_PP_BIT
347
(p6) br.cond.spnt.few rp;;
348
// definitely have a domain tick
351
mov cr.itm=r26 // ensure next tick
352
#ifdef FAST_REFLECT_CNT
353
movl r20=PERFC(fast_reflect + (0x3000>>8));;
358
// vcpu_pend_timer(current)
359
movl r18=THIS_CPU(current_psr_ic_addr)
363
adds r20=XSI_ITV_OFS-XSI_PSR_IC_OFS,r18 ;;
365
cmp.eq p6,p0=r20,r0 // if cr.itv==0 done
366
(p6) br.cond.spnt.few fast_tick_reflect_done;;
367
tbit.nz p6,p0=r20,16;; // check itv.m (discard) bit
368
(p6) br.cond.spnt.few fast_tick_reflect_done;;
369
extr.u r27=r20,0,6 // r27 has low 6 bits of itv.vector
370
extr.u r26=r20,6,2 // r26 has irr index of itv.vector
371
movl r19=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
373
adds r22=IA64_VCPU_DOMAIN_ITM_LAST_OFFSET,r19
374
adds r23=IA64_VCPU_DOMAIN_ITM_OFFSET,r19;;
377
cmp.eq p6,p0=r23,r24 // skip if this tick already delivered
378
(p6) br.cond.spnt.few fast_tick_reflect_done;;
380
adds r21=IA64_VCPU_IRR0_OFFSET,r19
388
// set evtchn_upcall_pending!
389
adds r20=XSI_PSR_I_ADDR_OFS-XSI_PSR_IC_OFS,r18;;
391
adds r20=-1,r20;; // evtchn_upcall_pending
393
// if interrupted at pl0, we're done
394
extr.u r16=r17,IA64_PSR_CPL0_BIT,2;;
395
cmp.eq p6,p0=r16,r0;;
396
(p6) br.cond.spnt.few fast_tick_reflect_done;;
397
// if guest vpsr.i is off, we're done
398
movl r21=THIS_CPU(current_psr_i_addr);;
402
(p6) br.cond.spnt.few fast_tick_reflect_done;;
404
// OK, we have a clock tick to deliver to the active domain!
405
// so deliver to iva+0x3000
408
// r19 == IA64_KR(CURRENT)
412
adds r21=XSI_IIP_OFS-XSI_PSR_IC_OFS,r18 ;;
414
// set shared_mem isr
415
extr.u r16=r16,IA64_ISR_IR_BIT,1;; // grab cr.isr.ir bit
416
dep r16=r16,r0,IA64_ISR_IR_BIT,1 // insert into cr.isr (rest of bits zero)
417
extr.u r20=r17,IA64_PSR_RI_BIT,2;; // get ipsr.ri
418
dep r16=r20,r16,IA64_PSR_RI_BIT,2 // deposit cr.isr.ei
419
adds r21=XSI_ISR_OFS-XSI_PSR_IC_OFS,r18;;
421
// set cr.ipsr (make sure cpl==2!)
423
movl r28=DELIVER_PSR_SET | (CONFIG_CPL0_EMUL << IA64_PSR_CPL0_BIT)
424
movl r27=~DELIVER_PSR_CLR;;
428
// set shared_mem ipsr (from ipsr in r17 with ipsr.ri already set)
429
extr.u r29=r17,IA64_PSR_CPL0_BIT,2;;
430
cmp.eq p7,p0=CONFIG_CPL0_EMUL,r29;;
431
(p7) dep r17=0,r17,IA64_PSR_CPL0_BIT,2
432
movl r28=(IA64_PSR_DT|IA64_PSR_IT|IA64_PSR_RT)
433
movl r27=~(IA64_PSR_PP|IA64_PSR_BN|IA64_PSR_I|IA64_PSR_IC);;
438
movl r22=THIS_CPU(current_psr_i_addr);;
440
(p6) dep r17=-1,r17,IA64_PSR_IC_BIT,1 ;;
442
cmp.eq p6,p0=r16,r0;;
443
(p6) dep r17=-1,r17,IA64_PSR_I_BIT,1
445
adds r21=XSI_IPSR_OFS-XSI_PSR_IC_OFS,r18;;
447
// set shared_mem interrupt_delivery_enabled to 0
448
// set shared_mem interrupt_collection_enabled to 0
451
// cover and set shared_mem precover_ifs to cr.ifs
452
// set shared_mem ifs to 0
455
adds r21=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
457
adds r21=XSI_PRECOVER_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
459
// leave cr.ifs alone for later rfi
460
// set iip to go to domain IVA break instruction vector
461
adds r22=IA64_VCPU_IVA_OFFSET,r19;;
466
// OK, now all set to go except for switch to virtual bank0
469
#ifdef HANDLE_AR_UNAT
473
adds r2=XSI_BANK1_R16_OFS-XSI_PSR_IC_OFS,r18
474
adds r3=(XSI_BANK1_R16_OFS+8)-XSI_PSR_IC_OFS,r18
477
.mem.offset 0,0; st8.spill [r2]=r16,16
478
.mem.offset 8,0; st8.spill [r3]=r17,16 ;;
479
.mem.offset 0,0; st8.spill [r2]=r18,16
480
.mem.offset 8,0; st8.spill [r3]=r19,16 ;;
481
.mem.offset 0,0; st8.spill [r2]=r20,16
482
.mem.offset 8,0; st8.spill [r3]=r21,16 ;;
483
.mem.offset 0,0; st8.spill [r2]=r22,16
484
.mem.offset 8,0; st8.spill [r3]=r23,16 ;;
485
.mem.offset 0,0; st8.spill [r2]=r24,16
486
.mem.offset 8,0; st8.spill [r3]=r25,16 ;;
487
.mem.offset 0,0; st8.spill [r2]=r26,16
488
.mem.offset 8,0; st8.spill [r3]=r27,16 ;;
489
.mem.offset 0,0; st8.spill [r2]=r28,16
490
.mem.offset 8,0; st8.spill [r3]=r29,16 ;;
491
.mem.offset 0,0; st8.spill [r2]=r30,16
492
.mem.offset 8,0; st8.spill [r3]=r31,16 ;;
493
#ifdef HANDLE_AR_UNAT
494
// r16~r23 are preserved regsin bank0 regs, we need to restore them,
495
// r24~r31 are scratch regs, we don't need to handle NaT bit,
496
// because OS handler must assign it before access it
511
#ifdef HANDLE_AR_UNAT
515
adds r25=XSI_B1NATS_OFS-XSI_PSR_IC_OFS,r18
516
adds r20=XSI_BANKNUM_OFS-XSI_PSR_IC_OFS,r18 ;;
519
fast_tick_reflect_done:
522
END(fast_tick_reflect)
524
// reflect domain breaks directly to domain
530
GLOBAL_ENTRY(fast_break_reflect)
531
#ifndef FAST_BREAK // see beginning of file
532
br.sptk.many dispatch_break_fault ;;
536
tbit.nz p7,p0=r30,IA64_PSR_PP_BIT
537
(p7) br.spnt.few dispatch_break_fault ;;
538
movl r20=IA64_PSR_CPL ;;
541
(p7) br.spnt.many 1f ;;
543
(p7) br.spnt.few dispatch_break_fault ;;
545
movl r21=CDB_BREAK_NUM ;;
547
(p7) br.spnt.few dispatch_break_fault ;;
550
#if 1 /* special handling in case running on simulator */
551
movl r20=first_break;;
556
(p7) br.spnt.few dispatch_break_fault ;;
558
(p7) br.spnt.few dispatch_break_fault ;;
560
(p7) br.spnt.few dispatch_break_fault ;;
563
// save iim in shared_info
564
adds r21=XSI_IIM_OFS-XSI_PSR_IC_OFS,r18 ;;
567
END(fast_break_reflect)
569
// reflect to domain ivt+r20
570
// sets up isr,iip,ipsr,ifs (FIXME: do iipa too)
573
// r20 == offset into ivt
578
#ifdef FAST_REFLECT_CNT
579
movl r22=PERFC(fast_reflect)
586
// save iip in shared_info (DON'T POINT TO NEXT INSTRUCTION!)
587
adds r21=XSI_IIP_OFS-XSI_PSR_IC_OFS,r18 ;;
588
st8 [r21]=r29,XSI_ISR_OFS-XSI_IIP_OFS;;
589
// set shared_mem isr
592
movl r21=THIS_CPU(current_psr_i_addr)
595
movl r28=DELIVER_PSR_SET | (CONFIG_CPL0_EMUL << IA64_PSR_CPL0_BIT)
596
movl r27=~DELIVER_PSR_CLR;;
599
// set hpsr_dfh to ipsr
600
adds r28=XSI_HPSR_DFH_OFS-XSI_PSR_IC_OFS,r18;;
602
dep r29=r28,r29,IA64_PSR_DFH_BIT,1;;
604
// set shared_mem ipsr (from ipsr in r30 with ipsr.ri already set)
605
extr.u r29=r30,IA64_PSR_CPL0_BIT,2;;
606
cmp.eq p7,p0=CONFIG_CPL0_EMUL,r29;;
607
(p7) dep r30=0,r30,IA64_PSR_CPL0_BIT,2
608
movl r28=(IA64_PSR_DT|IA64_PSR_IT|IA64_PSR_RT)
609
movl r27=~(IA64_PSR_PP|IA64_PSR_BN);;
612
// also set shared_mem ipsr.i and ipsr.ic appropriately
615
cmp4.eq p6,p7=r24,r0;;
616
(p6) dep r30=0,r30,IA64_PSR_IC_BIT,1
617
(p7) dep r30=-1,r30,IA64_PSR_IC_BIT,1
619
cmp.ne p6,p7=r22,r0;;
620
(p6) dep r30=0,r30,IA64_PSR_I_BIT,1
621
(p7) dep r30=-1,r30,IA64_PSR_I_BIT,1
623
adds r21=XSI_IPSR_OFS-XSI_PSR_IC_OFS,r18
624
adds r27=XSI_VPSR_DFH_OFS-XSI_PSR_IC_OFS,r18;;
627
dep r30=r28,r30,IA64_PSR_DFH_BIT,1
630
// set shared_mem interrupt_delivery_enabled to 0
631
// set shared_mem interrupt_collection_enabled to 0
634
// cover and set shared_mem precover_ifs to cr.ifs
635
// set shared_mem ifs to 0
638
adds r21=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
640
adds r21=XSI_PRECOVER_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
642
// FIXME: need to save iipa and isr to be arch-compliant
643
// set iip to go to domain IVA break instruction vector
644
movl r22=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
646
adds r22=IA64_VCPU_IVA_OFFSET,r22;;
650
// OK, now all set to go except for switch to virtual bank0
653
#ifdef HANDLE_AR_UNAT
657
adds r2=XSI_BANK1_R16_OFS-XSI_PSR_IC_OFS,r18
658
adds r3=(XSI_BANK1_R16_OFS+8)-XSI_PSR_IC_OFS,r18
661
.mem.offset 0,0; st8.spill [r2]=r16,16
662
.mem.offset 8,0; st8.spill [r3]=r17,16 ;;
663
.mem.offset 0,0; st8.spill [r2]=r18,16
664
.mem.offset 8,0; st8.spill [r3]=r19,16 ;;
665
.mem.offset 0,0; st8.spill [r2]=r20,16
666
.mem.offset 8,0; st8.spill [r3]=r21,16 ;;
667
.mem.offset 0,0; st8.spill [r2]=r22,16
668
.mem.offset 8,0; st8.spill [r3]=r23,16 ;;
669
.mem.offset 0,0; st8.spill [r2]=r24,16
670
.mem.offset 8,0; st8.spill [r3]=r25,16 ;;
671
.mem.offset 0,0; st8.spill [r2]=r26,16
672
.mem.offset 8,0; st8.spill [r3]=r27,16 ;;
673
.mem.offset 0,0; st8.spill [r2]=r28,16
674
.mem.offset 8,0; st8.spill [r3]=r29,16 ;;
675
.mem.offset 0,0; st8.spill [r2]=r30,16
676
.mem.offset 8,0; st8.spill [r3]=r31,16 ;;
677
#ifdef HANDLE_AR_UNAT
678
// r16~r23 are preserved regs in bank0 regs, we need to restore them,
679
// r24~r31 are scratch regs, we don't need to handle NaT bit,
680
// because OS handler must assign it before access it
695
#ifdef HANDLE_AR_UNAT
699
adds r25=XSI_B1NATS_OFS-XSI_PSR_IC_OFS,r18
700
adds r20=XSI_BANKNUM_OFS-XSI_PSR_IC_OFS,r18 ;;
708
// reflect access faults (0x2400,0x2800,0x5300) directly to domain
711
// r19 == reflect number (only pass-thru to dispatch_reflection)
712
// r20 == offset into ivt
714
GLOBAL_ENTRY(fast_access_reflect)
715
#ifndef FAST_ACCESS_REFLECT // see beginning of file
716
br.spnt.few dispatch_reflection ;;
720
tbit.nz p7,p0=r30,IA64_PSR_PP_BIT
721
(p7) br.spnt.few dispatch_reflection ;;
722
extr.u r21=r30,IA64_PSR_CPL0_BIT,2 ;;
724
(p7) br.spnt.few dispatch_reflection ;;
725
movl r18=THIS_CPU(current_psr_ic_addr);;
729
(p7) br.spnt.few dispatch_reflection ;;
730
// set shared_mem ifa, FIXME: should we validate it?
732
adds r21=XSI_IFA_OFS-XSI_PSR_IC_OFS,r18 ;;
734
// get rr[ifa] and save to itir in shared memory (extra bits ignored)
736
adds r23=XSI_ITIR_OFS-XSI_PSR_IC_OFS,r18
737
adds r21=XSI_RR0_OFS-XSI_PSR_IC_OFS,r18 ;;
738
shladd r22=r22,3,r21;;
742
br.cond.sptk.many fast_reflect;;
743
END(fast_access_reflect)
745
// when we get to here, VHPT_CCHAIN_LOOKUP has failed and everything
746
// is as it was at the time of original miss. We want to preserve that
747
// so if we get a nested fault, we can just branch to page_fault
748
GLOBAL_ENTRY(fast_tlb_miss_reflect)
749
#ifndef FAST_TLB_MISS_REFLECT // see beginning of file
750
br.spnt.few page_fault ;;
757
// for now, always take slow path for region 0 (e.g. metaphys mode)
758
extr.u r21=r17,61,3;;
760
(p7) br.spnt.few page_fault ;;
761
// always take slow path for PL0 (e.g. __copy_from_user)
762
extr.u r21=r30,IA64_PSR_CPL0_BIT,2 ;;
764
(p7) br.spnt.few page_fault ;;
765
// slow path if strange ipsr or isr bits set
766
tbit.nz p7,p0=r30,IA64_PSR_PP_BIT,1
767
(p7) br.spnt.few page_fault ;;
768
movl r21=IA64_ISR_IR|IA64_ISR_SP|IA64_ISR_NA ;;
771
(p7) br.spnt.few page_fault ;;
772
// also take slow path if virtual psr.ic=0
773
movl r18=XSI_PSR_IC;;
776
(p7) br.spnt.few page_fault ;;
777
// OK, if we get to here, we are doing a fast vcpu_translate. Need to:
778
// 1) look in the virtual TR's (pinned), if not there
779
// 2) look in the 1-entry TLB (pinned), if not there
780
// 3) check the domain VHPT (NOT pinned, accesses domain memory!)
781
// If we find it in any of these places, we need to effectively do
784
// short-term hack for now, if in region 5-7, take slow path
785
// since all Linux TRs are in region 5 or 7, we need not check TRs
786
extr.u r21=r17,61,3;;
788
(p7) br.spnt.few page_fault ;;
789
fast_tlb_no_tr_match:
790
movl r27=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
792
tbit.nz p6,p7=r16,IA64_ISR_X_BIT;;
793
(p6) adds r25=IA64_VCPU_ITLB_OFFSET,r27
794
(p7) adds r25=IA64_VCPU_DTLB_OFFSET,r27;;
796
tbit.z p7,p0=r20,VTLB_PTE_P_BIT // present?
797
(p7) br.cond.spnt.few 1f;;
798
// if ifa is in range of tlb, don't bother to check rid, go slow path
801
extr.u r21=r21,IA64_ITIR_PS,IA64_ITIR_PS_LEN;;
804
cmp.ltu p7,p0=r17,r21
805
(p7) br.cond.sptk.many 1f;
807
cmp.ltu p7,p0=r17,r21
808
(p7) br.cond.spnt.few page_fault;;
810
1: // check the guest VHPT
811
adds r19 = XSI_PTA_OFS-XSI_PSR_IC_OFS, r18;;
813
// if (!rr.ve || !(pta & IA64_PTA_VE)) take slow way for now
814
// FIXME: later, we deliver an alt_d/i vector after thash and itir
816
adds r21=XSI_RR0_OFS-XSI_PSR_IC_OFS,r18 ;;
821
(p7) br.cond.spnt.few page_fault;;
822
tbit.z p7,p0=r19,IA64_PTA_VE_BIT
823
(p7) br.cond.spnt.few page_fault;;
824
tbit.nz p7,p0=r19,IA64_PTA_VF_BIT // long format VHPT
825
(p7) br.cond.spnt.few page_fault;;
827
// compute and save away itir (r22 & RR_PS_MASK)
828
movl r21=IA64_ITIR_PS_MASK;;
830
adds r21=XSI_ITIR_OFS-XSI_PSR_IC_OFS,r18 ;;
834
adds r21=XSI_IFA_OFS-XSI_PSR_IC_OFS,r18 ;;
836
// see vcpu_thash to save away iha
839
movl r30 = 0xe000000000000000
841
and r21 = r30, r17 // VHPT_Addr1
843
shladd r28 = r20, 3, r18
844
adds r19 = XSI_PTA_OFS-XSI_PSR_IC_OFS, r18
846
adds r27 = XSI_RR0_OFS-XSI_PSR_IC_OFS, r28
848
ld8 r24 = [r19] // pta
850
ld8 r23 = [r27] // rrs[vadr>>61]
851
extr.u r26 = r24, IA64_PTA_SIZE_BIT, IA64_PTA_SIZE_LEN
853
extr.u r22 = r23, IA64_RR_PS, IA64_RR_PS_LEN
854
shl r30 = r25, r26 // pt size
856
shr.u r19 = r17, r22 // ifa pg number
857
shr.u r29 = r24, IA64_PTA_BASE_BIT
858
adds r30 = -1, r30 // pt size mask
860
shladd r27 = r19, 3, r0 // vhpt offset
861
extr.u r26 = r30, 15, 46
871
dep.z r20 = r22, 15, 46
878
adds r23=XSI_IHA_OFS-XSI_PSR_IC_OFS,r18 ;;
880
// done with thash, check guest VHPT
882
adds r20 = XSI_PTA_OFS-XSI_PSR_IC_OFS, r18;;
883
ld8 r24 = [r20];; // pta
884
// avoid recursively walking the VHPT
885
// if (((r17=address ^ r24=pta) & ((itir_mask(pta) << 3) >> 3)) != 0) {
888
extr.u r24=r24,IA64_PTA_SIZE_BIT,IA64_PTA_SIZE_LEN;;
893
(p7) br.cond.spnt.few 1f;;
894
// __copy_from_user(&pte, r19=(void *)(*iha), sizeof(pte)=8)
895
// prepare for possible nested dtlb fault
897
movl r30=guest_vhpt_miss
898
// now go fetch the entry from the guest VHPT
900
// if we wind up here, we successfully loaded the VHPT entry
902
// this VHPT walker aborts on non-present pages instead
903
// of inserting a not-present translation, this allows
904
// vectoring directly to the miss handler
906
(p7) br.cond.spnt.few page_not_present;;
908
#ifdef FAST_REFLECT_CNT
909
movl r21=PERFC(fast_vhpt_translate);;
915
// prepare for fast_insert(PSCB(ifa),PSCB(itir),r16=pte)
917
// r17 == bit0: 1=inst, 0=data; bit1: 1=itc, 0=vcpu_translate
918
// r18 == XSI_PSR_IC_OFS
920
// r29 == saved value of b0 in case of recovery
921
// r30 == recovery ip if failure occurs
923
tbit.nz p6,p7=r16,IA64_ISR_X_BIT;;
928
movl r30=recover_and_page_fault
929
adds r21=XSI_ITIR_OFS-XSI_PSR_IC_OFS,r18 ;;
931
extr.u r24=r24,IA64_ITIR_PS,IA64_ITIR_PS_LEN
932
// IFA already in PSCB
933
br.cond.sptk.many fast_insert;;
934
END(fast_tlb_miss_reflect)
936
// we get here if fast_insert fails (e.g. due to metaphysical lookup)
937
ENTRY(recover_and_page_fault)
939
movl r21=PERFC(recover_to_page_fault);;
945
br.cond.sptk.many page_fault;;
947
// if we wind up here, we missed in guest VHPT so recover
948
// from nested dtlb fault and reflect a tlb fault to the guest
951
// fault = IA64_VHPT_FAULT
953
br.cond.sptk.many 1f;
955
// if we get to here, we are ready to reflect
956
// need to set up virtual ifa, iha, itir (fast_reflect handles
957
// virtual isr, iip, ipsr, ifs
958
// see vcpu_get_itir_on_fault: get ps,rid,(FIXME key) from rr[ifa]
960
tbit.nz p6,p7=r16,IA64_ISR_X_BIT;;
964
1: extr.u r25=r17,61,3;;
965
adds r21=XSI_RR0_OFS-XSI_PSR_IC_OFS,r18
969
extr.u r22=r22,IA64_RR_PS,IA64_RR_PS_LEN+IA64_RR_RID_LEN;;
970
dep.z r22=r22,IA64_RR_PS,IA64_RR_PS_LEN+IA64_RR_RID_LEN
971
adds r23=XSI_ITIR_OFS-XSI_PSR_IC_OFS,r18 ;;
974
// fast reflect expects
977
// r20 == offset into ivt
984
br.sptk.many fast_reflect;;
986
END(fast_tlb_miss_reflect)
989
adds r22=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18;;
992
(p6) br.spnt.few dispatch_break_fault ;;
993
// If vifs.v is set, we have two IFS to consider:
995
// * the hypervisor IFS (validated by cover)
996
// Because IFS is copied to CFM and is used to adjust AR.BSP,
997
// virtualization of rfi is not easy.
998
// Previously there was a two steps method (a first rfi jumped to
999
// a stub which performed a new rfi).
1000
// This new method discards the RS before executing the hypervisor
1001
// cover. After cover, IFS.IFM will be zero. This IFS would simply
1002
// clear CFM but not modifying AR.BSP. Therefore the guest IFS can
1003
// be used instead and there is no need of a second rfi.
1004
// Discarding the RS with the following alloc instruction just clears
1005
// CFM, which is safe because rfi will overwrite it.
1006
// There is a drawback: because the RS must be discarded before
1007
// executing C code, emulation of rfi must go through an hyperprivop
1008
// and not through normal instruction decoding.
1009
alloc r22=ar.pfs,0,0,0,0
1010
br.spnt.few dispatch_break_fault
1014
// ensure that, if giving up, registers at entry to fast_hyperprivop unchanged
1017
br.spnt.few slow_vcpu_rfi ;;
1019
// if interrupts pending and vcr.ipsr.i=1, do it the slow way
1020
adds r19=XSI_IPSR_OFS-XSI_PSR_IC_OFS,r18
1021
adds r23=XSI_METAPHYS_OFS-XSI_PSR_IC_OFS,r18
1022
cmp.ne p8,p0=r20,r0;; // evtchn_upcall_pending != 0
1023
// if (!(vpsr.dt && vpsr.rt && vpsr.it)), do it the slow way
1024
ld8 r21=[r19],XSI_IIP_OFS-XSI_IPSR_OFS // r21=vcr.ipsr
1025
movl r20=~(IA64_PSR_DT|IA64_PSR_RT|IA64_PSR_IT);;
1027
// p8 determines whether we might deliver an immediate extint
1028
(p8) tbit.nz p8,p0=r21,IA64_PSR_I_BIT;;
1030
ld4 r23=[r23] // r23=metaphysical_mode
1031
#ifndef RFI_TO_INTERRUPT // see beginning of file
1032
(p8) br.cond.spnt.few slow_vcpu_rfi
1034
(p7) br.spnt.few slow_vcpu_rfi;;
1035
// if was in metaphys mode, do it the slow way (FIXME later?)
1037
ld8 r22=[r19] // r22=vcr.iip
1038
(p7) br.spnt.few slow_vcpu_rfi;;
1039
// OK now, let's do an rfi.
1040
#ifdef FAST_HYPERPRIVOP_CNT
1041
movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_RFI);;
1046
#ifdef RFI_TO_INTERRUPT
1047
// maybe do an immediate interrupt delivery?
1048
(p8) br.cond.spnt.few rfi_check_extint;;
1052
// r18=&vpsr.i|vpsr.ic, r21==vpsr, r22=vcr.iip
1054
extr.u r19=r21,IA64_PSR_CPL0_BIT,2
1055
adds r20=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
1056
cmp.gtu p7,p0=CONFIG_CPL0_EMUL,r19
1058
(p7) mov r19=CONFIG_CPL0_EMUL
1059
dep r20=0,r20,38,25;; // ensure ifs has no reserved bits set
1061
// ipsr.cpl = max(vcr.ipsr.cpl, IA64_PSR_CPL0_BIT);
1062
movl r20=THIS_CPU(current_psr_i_addr)
1063
dep r21=r19,r21,IA64_PSR_CPL0_BIT,2;;
1064
// vpsr.i = vcr.ipsr.i; vpsr.ic = vcr.ipsr.ic
1067
tbit.nz p7,p6=r21,IA64_PSR_I_BIT
1068
tbit.nz p9,p8=r21,IA64_PSR_IC_BIT;;
1074
// force on psr.ic, i, dt, rt, it, bn
1075
movl r20=(IA64_PSR_I|IA64_PSR_IC|IA64_PSR_DT|IA64_PSR_RT| \
1076
IA64_PSR_IT|IA64_PSR_BN)
1077
// keep cr.ipsr.pp and set vPSR.pp = vIPSR.pp
1081
tbit.z p10,p11 = r22, IA64_PSR_PP_BIT
1083
adds r20=XSI_VPSR_DFH_OFS-XSI_PSR_IC_OFS,r18;;
1084
tbit.z p8,p9 = r21, IA64_PSR_DFH_BIT
1085
adds r23=XSI_VPSR_PP_OFS-XSI_PSR_IC_OFS,r18
1089
dep r21=r22,r21,IA64_PSR_PP_BIT,1
1094
(p8) adds r20=XSI_HPSR_DFH_OFS-XSI_PSR_IC_OFS,r18;;
1097
(p8) dep r21=r27,r21, IA64_PSR_DFH_BIT, 1
1100
adds r20=XSI_BANKNUM_OFS-XSI_PSR_IC_OFS,r18 ;;
1102
cmp.ne p7,p0=r21,r0 // domain already did "bank 1 switch?"
1103
(p7) br.cond.spnt.few 1f;
1104
// OK, now all set to go except for switch to virtual bank1
1110
adds r16=XSI_B1NATS_OFS-XSI_PSR_IC_OFS,r18
1111
adds r2=XSI_BANK1_R16_OFS-XSI_PSR_IC_OFS,r18
1112
adds r3=(XSI_BANK1_R16_OFS+8)-XSI_PSR_IC_OFS,r18;;
1116
// FIXME?: ar.unat is not really handled correctly,
1117
// but may not matter if the OS is NaT-clean
1118
.mem.offset 0,0; ld8.fill r16=[r2],16
1119
.mem.offset 8,0; ld8.fill r17=[r3],16 ;;
1120
.mem.offset 0,0; ld8.fill r18=[r2],16
1121
.mem.offset 0,0; ld8.fill r19=[r3],16 ;;
1122
.mem.offset 8,0; ld8.fill r20=[r2],16
1123
.mem.offset 8,0; ld8.fill r21=[r3],16 ;;
1124
.mem.offset 8,0; ld8.fill r22=[r2],16
1125
.mem.offset 8,0; ld8.fill r23=[r3],16 ;;
1126
.mem.offset 8,0; ld8.fill r24=[r2],16
1127
.mem.offset 8,0; ld8.fill r25=[r3],16 ;;
1128
.mem.offset 8,0; ld8.fill r26=[r2],16
1129
.mem.offset 8,0; ld8.fill r27=[r3],16 ;;
1130
.mem.offset 8,0; ld8.fill r28=[r2],16
1131
.mem.offset 8,0; ld8.fill r29=[r3],16 ;;
1132
.mem.offset 8,0; ld8.fill r30=[r2],16
1133
.mem.offset 8,0; ld8.fill r31=[r3],16 ;;
1144
#ifdef RFI_TO_INTERRUPT
1145
ENTRY(rfi_check_extint)
1146
//br.sptk.many dispatch_break_fault ;;
1148
// r18=&vpsr.i|vpsr.ic, r21==vpsr, r22=vcr.iip
1149
// make sure none of these get trashed in case going to just_do_rfi
1150
movl r30=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1152
adds r24=IA64_VCPU_INSVC3_OFFSET,r30
1154
adds r16=IA64_VCPU_IRR3_OFFSET,r30;;
1156
cmp.eq p6,p0=r23,r0;;
1157
(p6) adds r16=-8,r16;;
1158
(p6) adds r24=-8,r24;;
1159
(p6) adds r25=-64,r25;;
1160
(p6) ld8 r23=[r16];;
1161
(p6) cmp.eq p6,p0=r23,r0;;
1162
(p6) adds r16=-8,r16;;
1163
(p6) adds r24=-8,r24;;
1164
(p6) adds r25=-64,r25;;
1165
(p6) ld8 r23=[r16];;
1166
(p6) cmp.eq p6,p0=r23,r0;;
1167
(p6) adds r16=-8,r16;;
1168
(p6) adds r24=-8,r24;;
1169
(p6) adds r25=-64,r25;;
1170
(p6) ld8 r23=[r16];;
1172
(p6) br.cond.spnt.few just_do_rfi; // this is actually an error
1173
// r16 points to non-zero element of irr, r23 has value
1174
// r24 points to corr element of insvc, r25 has elt*64
1176
cmp.geu p6,p0=r26,r23
1177
(p6) br.cond.spnt.many just_do_rfi;
1179
// not masked by insvc, get vector number
1192
andcm r26=0xffffffffffffffff,r26;;
1195
// r26 now contains the bit index (mod 64)
1198
// r27 now contains the (within the proper word) bit mask
1200
// r26 now contains the vector [0..255]
1201
adds r20=XSI_TPR_OFS-XSI_PSR_IC_OFS,r18 ;;
1204
tbit.nz p6,p0=r20,16 // if tpr.mmi is set, just rfi
1205
(p6) br.cond.spnt.few just_do_rfi;;
1208
cmp.ge p6,p0=r29,r26 // if tpr masks interrupt, just rfi
1209
(p6) br.cond.spnt.few just_do_rfi;;
1210
END(rfi_check_extint)
1212
// this doesn't work yet (dies early after getting to user mode)
1213
// but happens relatively infrequently, so fix it later.
1214
// NOTE that these will be counted incorrectly for now (for privcnt output)
1215
ENTRY(rfi_with_interrupt)
1217
br.sptk.many dispatch_break_fault ;;
1220
// OK, have an unmasked vector, so deliver extint to vcr.iva+0x3000
1221
// r18 == XSI_PSR_IC
1222
// r21 == vipsr (ipsr in shared_mem)
1223
// r30 == IA64_KR(CURRENT)
1227
// set shared_mem isr
1228
extr.u r16=r16,IA64_ISR_IR_BIT,1;; // grab cr.isr.ir bit
1229
dep r16=r16,r0,IA64_ISR_IR_BIT,1 // insert into cr.isr (rest of bits zero)
1230
extr.u r20=r21,IA64_PSR_RI_BIT,2 ;; // get v(!)psr.ri
1231
dep r16=r20,r16,IA64_PSR_RI_BIT,2 ;; // deposit cr.isr.ei
1232
adds r22=XSI_ISR_OFS-XSI_PSR_IC_OFS,r18 ;;
1234
movl r22=THIS_CPU(current_psr_i_addr)
1235
// set cr.ipsr (make sure cpl==2!)
1237
movl r27=~DELIVER_PSR_CLR
1238
movl r28=DELIVER_PSR_SET | (CONFIG_CPL0_EMUL << IA64_PSR_CPL0_BIT)
1244
// v.ipsr and v.iip are already set (and v.iip validated) as rfi target
1245
// set shared_mem interrupt_delivery_enabled to 0
1246
// set shared_mem interrupt_collection_enabled to 0
1249
// cover and set shared_mem precover_ifs to cr.ifs
1250
// set shared_mem ifs to 0
1254
adds r22=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
1256
adds r22=XSI_PRECOVER_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
1258
// leave cr.ifs alone for later rfi
1260
adds r22=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
1263
adds r22=XSI_PRECOVER_IFS_OFS-XSI_PSR_IC_OFS,r18 ;;
1266
// set iip to go to domain IVA break instruction vector
1267
adds r22=IA64_VCPU_IVA_OFFSET,r30;;
1273
// OK, now all set to go except for switch to virtual bank0
1276
adds r2=XSI_BANK1_OFS-XSI_PSR_IC_OFS,r18
1277
adds r3=(XSI_BANK1_OFS+8)-XSI_PSR_IC_OFS,r18;;
1279
// FIXME: need to handle ar.unat!
1280
.mem.offset 0,0; st8.spill [r2]=r16,16
1281
.mem.offset 8,0; st8.spill [r3]=r17,16 ;;
1282
.mem.offset 0,0; st8.spill [r2]=r18,16
1283
.mem.offset 8,0; st8.spill [r3]=r19,16 ;;
1284
.mem.offset 0,0; st8.spill [r2]=r20,16
1285
.mem.offset 8,0; st8.spill [r3]=r21,16 ;;
1286
.mem.offset 0,0; st8.spill [r2]=r22,16
1287
.mem.offset 8,0; st8.spill [r3]=r23,16 ;;
1288
.mem.offset 0,0; st8.spill [r2]=r24,16
1289
.mem.offset 8,0; st8.spill [r3]=r25,16 ;;
1290
.mem.offset 0,0; st8.spill [r2]=r26,16
1291
.mem.offset 8,0; st8.spill [r3]=r27,16 ;;
1292
.mem.offset 0,0; st8.spill [r2]=r28,16
1293
.mem.offset 8,0; st8.spill [r3]=r29,16 ;;
1294
.mem.offset 0,0; st8.spill [r2]=r30,16
1295
.mem.offset 8,0; st8.spill [r3]=r31,16 ;;
1300
adds r20=XSI_BANKNUM_OFS-XSI_PSR_IC_OFS,r18 ;;
1304
END(rfi_with_interrupt)
1305
#endif // RFI_TO_INTERRUPT
1308
#ifdef FAST_HYPERPRIVOP_CNT
1309
movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_COVER);;
1316
// skip test for vpsr.ic.. it's a prerequisite for hyperprivops
1319
adds r22=XSI_IFS_OFS-XSI_PSR_IC_OFS,r18;;
1322
// adjust return address to skip over break instruction
1323
extr.u r26=r24,41,2 ;;
1324
cmp.eq p6,p7=2,r26 ;;
1326
(p6) adds r25=16,r25
1329
dep r24=r26,r24,IA64_PSR_RI_BIT,2
1338
// return from metaphysical mode (meta=1) to virtual mode (meta=0)
1340
#ifdef FAST_HYPERPRIVOP_CNT
1341
movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_SSM_DT);;
1348
adds r20=XSI_METAPHYS_OFS-XSI_PSR_IC_OFS,r18 ;;
1350
cmp.eq p7,p0=r21,r0 // meta==0?
1351
(p7) br.spnt.many 1f ;; // already in virtual mode
1352
movl r22=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1354
adds r22=IA64_VCPU_META_SAVED_RR0_OFFSET,r22;;
1359
// adjust return address to skip over break instruction
1360
1: extr.u r26=r24,IA64_PSR_RI_BIT,2 ;;
1361
cmp.eq p6,p7=2,r26 ;;
1363
(p6) adds r25=16,r25
1366
dep r24=r26,r24,IA64_PSR_RI_BIT,2
1375
// go to metaphysical mode (meta=1) from virtual mode (meta=0)
1377
#ifdef FAST_HYPERPRIVOP_CNT
1378
movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_RSM_DT);;
1385
adds r20=XSI_METAPHYS_OFS-XSI_PSR_IC_OFS,r18 ;;
1387
cmp.ne p7,p0=r21,r0 // meta==0?
1388
(p7) br.spnt.many 1f ;; // already in metaphysical mode
1389
movl r22=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1391
adds r22=IA64_VCPU_META_RID_DT_OFFSET,r22;;
1397
// adjust return address to skip over break instruction
1398
1: extr.u r26=r24,IA64_PSR_RI_BIT,2 ;;
1399
cmp.eq p6,p7=2,r26 ;;
1401
(p6) adds r25=16,r25
1404
dep r24=r26,r24,IA64_PSR_RI_BIT,2
1413
ENTRY(hyper_set_itm)
1414
// when we get to here r20=~=interrupts pending
1416
(p7) br.spnt.many dispatch_break_fault ;;
1417
#ifdef FAST_HYPERPRIVOP_CNT
1418
movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_SET_ITM);;
1423
movl r20=THIS_CPU(cpu_info)+IA64_CPUINFO_ITM_NEXT_OFFSET;;
1425
movl r20=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1427
adds r20=IA64_VCPU_DOMAIN_ITM_OFFSET,r20;;
1429
cmp.geu p6,p0=r21,r8;;
1431
// now "safe set" cr.itm=r21
1436
cmp.leu p6,p0=r21,r22;;
1439
(p6) br.cond.spnt.few 2b;;
1442
extr.u r26=r24,IA64_PSR_RI_BIT,2 ;;
1443
cmp.eq p6,p7=2,r26 ;;
1445
(p6) adds r25=16,r25
1448
dep r24=r26,r24,IA64_PSR_RI_BIT,2
1457
ENTRY(hyper_get_psr)
1458
#ifdef FAST_HYPERPRIVOP_CNT
1459
movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_GET_PSR);;
1465
movl r8=0xffffffff | IA64_PSR_MC | IA64_PSR_IT;;
1466
// only return PSR{36:35,31:0}
1470
dep r8=r21,r8,IA64_PSR_IC_BIT,1
1472
adds r20=XSI_VPSR_PP_OFS-XSI_PSR_IC_OFS,r18 ;;
1474
dep r8=r21,r8,IA64_PSR_PP_BIT,1
1476
adds r20=XSI_METAPHYS_OFS-XSI_PSR_IC_OFS,r18 ;;
1480
(p6) dep.z r8=r8,IA64_PSR_DT_BIT,1
1482
adds r20=XSI_PSR_I_ADDR_OFS-XSI_PSR_IC_OFS,r18 ;;
1487
(p8) dep r8=-1,r8,IA64_PSR_I_BIT,1
1488
(p9) dep r8=0,r8,IA64_PSR_I_BIT,1
1490
adds r20=XSI_VPSR_DFH_OFS-XSI_PSR_IC_OFS,r18;;
1492
dep r8=r21,r8,IA64_PSR_DFH_BIT,1
1495
extr.u r26=r24,IA64_PSR_RI_BIT,2 ;;
1496
cmp.eq p6,p7=2,r26 ;;
1498
(p6) adds r25=16,r25
1501
dep r24=r26,r24,IA64_PSR_RI_BIT,2
1512
#ifdef FAST_HYPERPRIVOP_CNT
1513
movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_GET_RR);;
1518
extr.u r25=r8,61,3;;
1519
adds r20=XSI_RR0_OFS-XSI_PSR_IC_OFS,r18
1525
extr.u r26=r24,IA64_PSR_RI_BIT,2 ;;
1526
cmp.eq p6,p7=2,r26 ;;
1528
(p6) adds r25=16,r25
1531
dep r24=r26,r24,IA64_PSR_RI_BIT,2
1541
extr.u r25=r8,61,3;;
1542
cmp.leu p7,p0=7,r25 // punt on setting rr7
1543
(p7) br.spnt.many dispatch_break_fault ;;
1544
#ifdef FAST_HYPERPRIVOP_CNT
1545
movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_SET_RR);;
1550
extr.u r26=r9,IA64_RR_RID,IA64_RR_RID_LEN // r26 = r9.rid
1551
movl r20=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1553
adds r22=IA64_VCPU_STARTING_RID_OFFSET,r20
1554
adds r23=IA64_VCPU_ENDING_RID_OFFSET,r20
1555
adds r24=IA64_VCPU_META_SAVED_RR0_OFFSET,r20
1556
adds r21=IA64_VCPU_VHPT_PG_SHIFT_OFFSET,r20;;
1561
cmp.geu p6,p0=r22,r23 // if r9.rid + starting_rid >= ending_rid
1562
(p6) br.cond.spnt.few 1f; // this is an error, but just ignore/return
1563
adds r20=XSI_RR0_OFS-XSI_PSR_IC_OFS,r18
1566
st8 [r20]=r9;; // store away exactly what was passed
1567
// but adjust value actually placed in rr[r8]
1568
// r22 contains adjusted rid, "mangle" it (see regionreg.c)
1569
// and set ps to v->arch.vhpt_pg_shift and ve to 1
1573
dep.z r23=r21,IA64_RR_PS,IA64_RR_PS_LEN;;
1574
dep r23=-1,r23,0,1;; // mangling is swapping bytes 1 & 3
1575
dep r23=r27,r23,24,8;;
1576
dep r23=r28,r23,16,8;;
1578
cmp.eq p6,p0=r25,r0;; // if rr0, save for metaphysical
1581
// done, mosey on back
1584
extr.u r26=r24,IA64_PSR_RI_BIT,2 ;;
1585
cmp.eq p6,p7=2,r26 ;;
1587
(p6) adds r25=16,r25
1590
dep r24=r26,r24,IA64_PSR_RI_BIT,2
1604
// mov rr[0x0000000000000000UL] = r8
1605
// mov rr[0x2000000000000000UL] = r9
1606
// mov rr[0x4000000000000000UL] = r10
1607
// mov rr[0x6000000000000000UL] = r11
1608
// mov rr[0x8000000000000000UL] = r14
1609
ENTRY(hyper_set_rr0_to_rr4)
1610
#ifndef FAST_SET_RR0_TO_RR4
1611
br.spnt.few dispatch_break_fault ;;
1613
#ifdef FAST_HYPERPRIVOP_CNT
1614
movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_SET_RR0_TO_RR4);;
1619
movl r17=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1622
adds r21=IA64_VCPU_STARTING_RID_OFFSET,r17
1623
adds r22=IA64_VCPU_ENDING_RID_OFFSET,r17
1624
adds r23=IA64_VCPU_VHPT_PG_SHIFT_OFFSET,r17
1626
ld4 r21=[r21] // r21 = current->starting_rid
1627
extr.u r26=r8,IA64_RR_RID,IA64_RR_RID_LEN // r26 = r8.rid
1628
extr.u r27=r9,IA64_RR_RID,IA64_RR_RID_LEN // r27 = r9.rid
1629
ld4 r22=[r22] // r22 = current->ending_rid
1630
extr.u r28=r10,IA64_RR_RID,IA64_RR_RID_LEN // r28 = r10.rid
1631
extr.u r29=r11,IA64_RR_RID,IA64_RR_RID_LEN // r29 = r11.rid
1632
adds r24=IA64_VCPU_META_SAVED_RR0_OFFSET,r17
1633
extr.u r30=r14,IA64_RR_RID,IA64_RR_RID_LEN // r30 = r14.rid
1634
ld1 r23=[r23] // r23 = current->vhpt_pg_shift
1641
dep.z r23=r23,IA64_RR_PS,IA64_RR_PS_LEN // r23 = rr.ps
1643
cmp.geu p6,p0=r16,r22 // if r8.rid + starting_rid >= ending_rid
1644
cmp.geu p7,p0=r17,r22 // if r9.rid + starting_rid >= ending_rid
1645
cmp.geu p8,p0=r19,r22 // if r10.rid + starting_rid >= ending_rid
1646
(p6) br.cond.spnt.few 1f // this is an error, but just ignore/return
1647
(p7) br.cond.spnt.few 1f // this is an error, but just ignore/return
1648
cmp.geu p9,p0=r20,r22 // if r11.rid + starting_rid >= ending_rid
1649
(p8) br.cond.spnt.few 1f // this is an error, but just ignore/return
1650
(p9) br.cond.spnt.few 1f // this is an error, but just ignore/return
1651
cmp.geu p10,p0=r21,r22 // if r14.rid + starting_rid >= ending_rid
1652
(p10) br.cond.spnt.few 1f // this is an error, but just ignore/return
1653
dep r23=-1,r23,0,1 // add rr.ve
1656
adds r22=XSI_RR0_OFS-XSI_PSR_IC_OFS,r18
1658
shl r30=r25,61 // r30 = 0x2000000000000000
1661
// simple plain version
1663
st8 [r22]=r8, 8 // current->rrs[0] = r8
1665
mov r26=0 // r26=0x0000000000000000
1668
extr.u r29=r16,16,8;;
1669
dep r25=r27,r23,24,8;; // mangling is swapping bytes 1 & 3
1670
dep r25=r28,r25,16,8;;
1671
dep r25=r29,r25,8,8;;
1672
st8 [r24]=r25 // save for metaphysical
1677
st8 [r22]=r9, 8 // current->rrs[1] = r9
1678
add r26=r26,r30 // r26 = 0x2000000000000000
1681
extr.u r29=r17,16,8;;
1682
dep r25=r27,r23,24,8;; // mangling is swapping bytes 1 & 3
1683
dep r25=r28,r25,16,8;;
1684
dep r25=r29,r25,8,8;;
1689
st8 [r22]=r10, 8 // current->rrs[2] = r10
1690
add r26=r26,r30 // r26 = 0x4000000000000000
1693
extr.u r29=r19,16,8;;
1694
dep r25=r27,r23,24,8;; // mangling is swapping bytes 1 & 3
1695
dep r25=r28,r25,16,8;;
1696
dep r25=r29,r25,8,8;;
1701
st8 [r22]=r11, 8 // current->rrs[3] = r11
1703
add r26=r26,r30 // r26 = 0x6000000000000000
1706
extr.u r29=r20,16,8;;
1707
dep r25=r27,r23,24,8;; // mangling is swapping bytes 1 & 3
1708
dep r25=r28,r25,16,8;;
1709
dep r25=r29,r25,8,8;;
1714
st8 [r22]=r14 // current->rrs[4] = r14
1716
add r26=r26,r30 // r26 = 0x8000000000000000
1719
extr.u r29=r21,16,8;;
1720
dep r25=r27,r23,24,8;; // mangling is swapping bytes 1 & 3
1721
dep r25=r28,r25,16,8;;
1722
dep r25=r29,r25,8,8;;
1728
// uses r27, r28, r29 for mangling
1729
// r25 for mangled value
1730
st8 [r22]=r8, 8 // current->rrs[0] = r8
1731
mov r26=0 // r26=0x0000000000000000
1734
extr.u r29=r16,16,8;;
1735
dep r25=r27,r23,24,8;; // mangling is swapping bytes 1 & 3
1736
dep r25=r28,r25,16,8;;
1737
dep r25=r29,r25,8,8;;
1738
st8 [r24]=r25 // save for metaphysical
1742
// r16, r24, r25 is usable.
1744
// uses r25, r28, r29 for mangling
1745
// r25 for mangled value
1748
st8 [r22]=r9, 8 // current->rrs[1] = r9
1749
extr.u r29=r17,16,8 ;;
1750
add r26=r26,r30 // r26 = 0x2000000000000000
1753
dep r25=r25,r23,24,8;; // mangling is swapping bytes 1 & 3
1754
dep r25=r28,r25,16,8;;
1755
dep r25=r29,r25,8,8;;
1759
// r16, r17, r24, r25 is usable
1761
// uses r16, r24, r29 for mangling
1762
// r17 for mangled value
1765
st8 [r22]=r10, 8 // current->rrs[2] = r10
1766
add r26=r26,r30 // r26 = 0x4000000000000000
1767
dep r17=r16,r23,24,8;; // mangling is swapping bytes 1 & 3
1768
dep r17=r24,r17,16,8;;
1769
dep r17=r29,r17,8,8;;
1773
// r16, r17, r19, r24, r25 is usable
1775
// uses r27, r28, r29 for mangling
1776
// r25 for mangled value
1779
st8 [r22]=r11, 8 // current->rrs[3] = r11
1781
add r26=r26,r30 // r26 = 0x6000000000000000
1782
dep r25=r27,r23,24,8;; // mangling is swapping bytes 1 & 3
1783
dep r25=r28,r25,16,8;;
1784
dep r25=r29,r25,8,8;;
1788
// r16, r17, r19, r20, r24, r25
1790
// uses r16, r17, r24 for mangling
1791
// r25 for mangled value
1794
st8 [r22]=r14 // current->rrs[4] = r14
1795
add r26=r26,r30 // r26 = 0x8000000000000000
1796
dep r25=r16,r23,24,8;; // mangling is swapping bytes 1 & 3
1797
dep r25=r17,r25,16,8;;
1798
dep r25=r24,r25,8,8;;
1803
// done, mosey on back
1806
extr.u r26=r24,IA64_PSR_RI_BIT,2 ;;
1807
cmp.eq p6,p7=2,r26 ;;
1809
(p6) adds r25=16,r25
1812
dep r24=r26,r24,IA64_PSR_RI_BIT,2
1819
END(hyper_set_rr0_to_rr4)
1822
extr.u r25=r8,3,61;;
1823
cmp.ne p7,p0=r0,r25 // if kr# > 7, go slow way
1824
(p7) br.spnt.many dispatch_break_fault ;;
1825
#ifdef FAST_HYPERPRIVOP_CNT
1826
movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_SET_KR);;
1831
adds r21=XSI_KR0_OFS-XSI_PSR_IC_OFS,r18
1859
// done, mosey on back
1862
extr.u r26=r24,IA64_PSR_RI_BIT,2 ;;
1863
cmp.eq p6,p7=2,r26 ;;
1865
(p6) adds r25=16,r25
1868
dep r24=r26,r24,IA64_PSR_RI_BIT,2
1877
// this routine was derived from optimized assembly output from
1878
// vcpu_thash so it is dense and difficult to read but it works
1880
// r18 == XSI_PSR_IC
1883
#ifdef FAST_HYPERPRIVOP_CNT
1884
movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_THASH);;
1891
movl r17 = 0xe000000000000000
1893
and r21 = r17, r8 // VHPT_Addr1
1895
shladd r28 = r20, 3, r18
1896
adds r19 = XSI_PTA_OFS-XSI_PSR_IC_OFS, r18
1898
adds r27 = XSI_RR0_OFS-XSI_PSR_IC_OFS, r28
1899
addl r28 = 32767, r0
1900
ld8 r24 = [r19] // pta
1902
ld8 r23 = [r27] // rrs[vadr>>61]
1903
extr.u r26 = r24, IA64_PTA_SIZE_BIT, IA64_PTA_SIZE_LEN
1905
extr.u r22 = r23, IA64_RR_PS, IA64_RR_PS_LEN
1913
shladd r27 = r19, 3, r0
1914
extr.u r26 = r17, 15, 46
1916
andcm r24 = r29, r26
1924
dep.z r20 = r22, 15, 46
1929
// done, update iip/ipsr to next instruction
1932
extr.u r26=r24,IA64_PSR_RI_BIT,2 ;;
1933
cmp.eq p6,p7=2,r26 ;;
1935
(p6) adds r25=16,r25
1938
dep r24=r26,r24,IA64_PSR_RI_BIT,2
1949
br.spnt.few dispatch_break_fault ;;
1951
// FIXME: validate not flushing Xen addresses
1952
#ifdef FAST_HYPERPRIVOP_CNT
1953
movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_PTC_GA);;
1958
movl r21=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
1960
adds r22=IA64_VCPU_VHPT_PG_SHIFT_OFFSET,r21
1962
extr.u r19=r9,2,6 // addr_range=1<<((r9&0xfc)>>2)
1965
movl r26=0x8000000000000000 // INVALID_TI_TAG
1968
ld1 r22=[r22] // current->arch.vhpt_pg_shift
1971
(p7) br.spnt.many dispatch_break_fault ;; // slow way for rr7
1973
shl r27=r22,2 // vhpt_pg_shift<<2 (for ptc.ga)
1974
shr.u r23=r19,r22 // repeat loop for n pages
1975
cmp.le p7,p0=r19,r0 // skip flush if size<=0
1976
(p7) br.cond.dpnt 2f ;;
1978
cmp.ne p7,p0=r24,r23 ;;
1979
(p7) adds r23=1,r23 ;; // n_pages<size<n_pages+1? extra iter
1981
shl r29=r20,r22;; // page_size
1986
// FIXME: should check if tag matches, not just blow it away
1987
or r24=r26,r24 ;; // vhpt_entry->ti_tag = 1
1992
br.cloop.sptk.few 1b
1998
adds r25=IA64_VCPU_DTLB_OFFSET,r21
1999
adds r26=IA64_VCPU_ITLB_OFFSET,r21;;
2004
st8 [r25]=r24 // set 1-entry i/dtlb as not present
2006
// increment to point to next instruction
2007
extr.u r26=r29,IA64_PSR_RI_BIT,2 ;;
2008
cmp.eq p6,p7=2,r26 ;;
2010
(p6) adds r30=16,r30
2013
dep r29=r26,r29,IA64_PSR_RI_BIT,2
2022
// recovery block for hyper_itc metaphysical memory lookup
2023
ENTRY(recover_and_dispatch_break_fault)
2024
#ifdef PERF_COUNTERS
2025
movl r21=PERFC(recover_to_break_fault);;
2031
br.sptk.many dispatch_break_fault;;
2032
END(recover_and_dispatch_break_fault)
2034
// Registers at entry
2035
// r17 = break immediate (HYPERPRIVOP_ITC_D or I)
2036
// r18 == XSI_PSR_IC_OFS
2040
// fall through, hyper_itc_d handles both i and d
2043
br.sptk.many dispatch_break_fault ;;
2045
// ensure itir.ps >= xen's pagesize
2046
movl r27=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
2048
adds r22=IA64_VCPU_VHPT_PG_SHIFT_OFFSET,r27
2049
adds r23=XSI_ITIR_OFS-XSI_PSR_IC_OFS,r18 ;;
2052
extr.u r24=r23,IA64_ITIR_PS,IA64_ITIR_PS_LEN;; // r24==logps
2053
cmp.gt p7,p0=r22,r24
2054
(p7) br.spnt.many dispatch_break_fault ;;
2055
adds r21=XSI_IFA_OFS-XSI_PSR_IC_OFS,r18 ;;
2057
// for now, punt on region0 inserts
2058
extr.u r21=r21,61,3;;
2060
(p7) br.spnt.many dispatch_break_fault ;;
2061
adds r27=IA64_VCPU_DOMAIN_OFFSET,r27;;
2063
// FIXME: is the global var dom0 always pinned? assume so for now
2066
// FIXME: for now, only handle dom0 (see lookup_domain_mpa below)
2067
cmp.ne p7,p0=r27,r28
2068
(p7) br.spnt.many dispatch_break_fault ;;
2069
#ifdef FAST_HYPERPRIVOP_CNT
2070
cmp.eq p6,p7=HYPERPRIVOP_ITC_D,r17;;
2071
(p6) movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_ITC_D)
2072
(p7) movl r20=FAST_HYPERPRIVOP_PERFC(HYPERPRIVOP_ITC_I);;
2080
movl r30=recover_and_dispatch_break_fault ;;
2086
#if defined(FAST_ITC) || defined (FAST_TLB_MISS_REFLECT)
2088
// fast_insert(PSCB(ifa),r24=ps,r16=pte)
2090
// r17 == bit0: 1=inst, 0=data; bit1: 1=itc, 0=vcpu_translate
2091
// r18 == XSI_PSR_IC_OFS
2093
// r29 == saved value of b0 in case of recovery
2094
// r30 == recovery ip if failure occurs
2097
// translate_domain_pte(r16=pteval,PSCB(ifa)=address,r24=itir)
2099
movl r27=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
2102
adds r23=IA64_VCPU_VHPT_PG_SHIFT_OFFSET,r27
2103
adds r20=-1,r20 // r20 == mask
2104
movl r19=_PAGE_PPN_MASK;;
2107
and r22=r16,r19;; // r22 == pteval & _PAGE_PPN_MASK
2109
shl r25=r25,r23 // -1 << current->arch.vhpt_pg_shift
2110
adds r21=XSI_IFA_OFS-XSI_PSR_IC_OFS,r18 ;;
2113
or r19=r19,r20;; // r19 == mpaddr
2114
// FIXME: for now, just do domain0 and skip mpaddr range checks
2116
movl r21=PAGE_PHYS ;;
2117
or r20=r20,r21 ;; // r20==return value from lookup_domain_mpa
2118
// r16=pteval,r20=pteval2
2119
movl r19=_PAGE_PPN_MASK
2120
movl r21=_PAGE_PL_PRIV;;
2121
andcm r25=r16,r19 // r25==pteval & ~_PAGE_PPN_MASK
2124
or r22=r22,r25;; // r22==return value from translate_domain_pte
2125
// done with translate_domain_pte
2126
// now do vcpu_itc_no_srlz(vcpu,IorD,ifa,r22=pte,r16=mppte,r24=logps)
2127
// FIXME: for now, just domain0 and skip range check
2128
// psr.ic already cleared
2129
// NOTE: r24 still contains ps (from above)
2130
shladd r24=r24,2,r0;;
2132
adds r23=XSI_IFA_OFS-XSI_PSR_IC_OFS,r18 ;;
2135
tbit.z p6,p7=r17,0;;
2139
// vhpt_insert(r23=vaddr,r22=pte,r24=logps<<2)
2149
// vcpu_set_tr_entry(trp,r22=pte|1,r24=itir,r23=ifa)
2150
// TR_ENTRY = {page_flags,itir,addr,rid}
2152
adds r28=IA64_VCPU_STARTING_RID_OFFSET,r27
2153
(p6) adds r27=IA64_VCPU_DTLB_OFFSET,r27
2154
(p7) adds r27=IA64_VCPU_ITLB_OFFSET,r27;;
2155
st8 [r27]=r22,8;; // page_flags: already has pl >= 2 and p==1
2156
st8 [r27]=r24,8 // itir
2159
st8 [r27]=r23,8 // ifa & ~0xfff
2160
adds r29 = XSI_RR0_OFS-XSI_PSR_IC_OFS,r18
2161
extr.u r25=r23,61,3;;
2162
shladd r29=r25,3,r29;;
2164
movl r20=IA64_RR_RID_MASK;;
2166
st8 [r27]=r29,-8;; // rid
2168
cmp.eq p7,p0=12<<IA64_ITIR_PS,r24
2169
(p7) br.cond.sptk.many 1f;;
2171
// trp->ppn &= ~((1UL<<(ps-12))-1); trp->vadr &= ~((1UL<<ps)-1); }
2172
extr.u r29=r24,IA64_ITIR_PS,IA64_ITIR_PS_LEN
2180
andcm r26=r29,r26 // ~((1UL<<ps)-1)
2181
andcm r25=r29,r25;; // ~((1UL<<(ps-12))-1)
2186
extr.u r28=r29,12,38;;
2187
movl r26=0xfffc000000000fff;;
2193
1: // done with vcpu_set_tr_entry
2194
//PSCBX(vcpu,i/dtlb_pte) = mp_pte
2195
movl r27=THIS_CPU(cpu_kr)+IA64_KR_CURRENT_OFFSET;;
2197
tbit.z p6,p7=r17,0;;
2198
(p6) adds r27=IA64_VCPU_DTLB_PTE_OFFSET,r27
2199
(p7) adds r27=IA64_VCPU_ITLB_PTE_OFFSET,r27;;
2201
// done with vcpu_itc_no_srlz
2203
// if hyper_itc, increment to point to next instruction
2205
(p7) br.cond.sptk.few no_inc_iip;;
2209
extr.u r26=r29,IA64_PSR_RI_BIT,2 ;;
2210
cmp.eq p6,p7=2,r26 ;;
2212
(p6) adds r30=16,r30
2215
dep r29=r26,r29,IA64_PSR_RI_BIT,2
added
removed
xen/arch/ia64/xen/hyperprivop.S
