2
# Copyright (c) 2005 Jakub Vana
3
# Copyright (c) 2005 Jakub Jermar
6
# Redistribution and use in source and binary forms, with or without
7
# modification, are permitted provided that the following conditions
10
# - Redistributions of source code must retain the above copyright
11
# notice, this list of conditions and the following disclaimer.
12
# - Redistributions in binary form must reproduce the above copyright
13
# notice, this list of conditions and the following disclaimer in the
14
# documentation and/or other materials provided with the distribution.
15
# - The name of the author may not be used to endorse or promote products
16
# derived from this software without specific prior written permission.
18
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
23
# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
#include <arch/stack.h>
31
#include <arch/register.h>
32
#include <arch/mm/page.h>
35
#define FRS_TO_SAVE 30
36
#define STACK_ITEMS (21 + FRS_TO_SAVE * 2)
37
#define STACK_FRAME_SIZE ALIGN_UP((STACK_ITEMS*STACK_ITEM_SIZE) + STACK_SCRATCH_AREA_SIZE, STACK_ALIGNMENT)
39
#if (STACK_ITEMS % 2 == 0)
40
# define STACK_FRAME_BIAS 8
42
# define STACK_FRAME_BIAS 16
45
/** Partitioning of bank 0 registers. */
50
#define R_KSTACK_BSP r22 /* keep in sync with before_thread_runs_arch() */
51
#define R_KSTACK r23 /* keep in sync with before_thread_runs_arch() */
53
/** Heavyweight interrupt handler
55
* This macro roughly follows steps from 1 to 19 described in
56
* Intel Itanium Architecture Software Developer's Manual, Chapter 3.4.2.
58
* HEAVYWEIGHT_HANDLER macro must cram into 16 bundles (48 instructions).
59
* This goal is achieved by using procedure calls after RSE becomes operational.
61
* Some steps are skipped (enabling and disabling interrupts).
63
* @param offs Offset from the beginning of IVT.
64
* @param handler Interrupt handler address.
66
.macro HEAVYWEIGHT_HANDLER offs, handler=universal_handler
69
movl R_HANDLER = \handler ;;
70
br heavyweight_handler
73
.global heavyweight_handler
75
/* 1. copy interrupt registers into bank 0 */
78
* Note that r24-r31 from bank 0 can be used only as long as PSR.ic = 0.
81
/* Set up FPU as in interrupted context. */
84
mov r26 = PSR_DFH_MASK
85
mov r27 = ~PSR_DFH_MASK ;;
99
/* 2. preserve predicate register into bank 0 */
102
/* 3. switch to kernel memory stack */
104
shr.u r31 = r12, VRN_SHIFT ;;
106
shr.u r30 = r30, PSR_CPL_SHIFT ;;
107
and r30 = PSR_CPL_MASK_SHIFTED, r30 ;;
110
* Set p3 to true if the interrupted context executed in kernel mode.
111
* Set p4 to false if the interrupted context didn't execute in kernel mode.
113
cmp.eq p3, p4 = r30, r0 ;;
114
cmp.eq p1, p2 = r30, r0 ;; /* remember IPSR setting in p1 and p2 */
117
* Set p3 to true if the stack register references kernel address space.
118
* Set p4 to false if the stack register doesn't reference kernel address space.
120
(p3) cmp.eq p3, p4 = VRN_KERNEL, r31 ;;
123
* Now, p4 is true iff the stack needs to be switched to kernel stack.
126
(p4) mov r12 = R_KSTACK ;;
128
add r31 = -STACK_FRAME_BIAS, r12 ;;
129
add r12 = -STACK_FRAME_SIZE, r12
131
/* 4. save registers in bank 0 into memory stack */
134
* If this is break_instruction handler,
135
* copy input parameters to stack.
137
mov R_TMP = 0x2c00 ;;
138
cmp.eq p6, p5 = R_OFFS, R_TMP ;;
141
* From now on, if this is break_instruction handler, p6 is true and p5
142
* is false. Otherwise p6 is false and p5 is true.
143
* Note that p5 is a preserved predicate register and we make use of it.
146
(p6) st8 [r31] = r38, -8 ;; /* save in6 */
147
(p6) st8 [r31] = r37, -8 ;; /* save in5 */
148
(p6) st8 [r31] = r36, -8 ;; /* save in4 */
149
(p6) st8 [r31] = r35, -8 ;; /* save in3 */
150
(p6) st8 [r31] = r34, -8 ;; /* save in2 */
151
(p6) st8 [r31] = r33, -8 ;; /* save in1 */
152
(p6) st8 [r31] = r32, -8 ;; /* save in0 */
153
(p5) add r31 = -56, r31 ;;
155
st8 [r31] = r30, -8 ;; /* save old stack pointer */
157
st8 [r31] = r29, -8 ;; /* save predicate registers */
159
st8 [r31] = r24, -8 ;; /* save cr.iip */
160
st8 [r31] = r25, -8 ;; /* save cr.ipsr */
161
st8 [r31] = r26, -8 ;; /* save cr.iipa */
162
st8 [r31] = r27, -8 ;; /* save cr.isr */
163
st8 [r31] = r28, -8 ;; /* save cr.ifa */
165
/* 5. RSE switch from interrupted context */
171
st8 [r31] = r24, -8 ;; /* save ar.rsc */
172
st8 [r31] = r25, -8 ;; /* save ar.pfs */
173
st8 [r31] = r26, -8 /* save ar.ifs */
175
and r24 = ~(RSC_PL_MASK), r24 ;;
176
and r30 = ~(RSC_MODE_MASK), r24 ;;
177
mov ar.rsc = r30 ;; /* update RSE state */
180
mov r28 = ar.bspstore ;;
183
* Inspect BSPSTORE to figure out whether it is necessary to switch to
186
(p1) shr.u r30 = r28, VRN_SHIFT ;;
187
(p1) cmp.eq p1, p2 = VRN_KERNEL, r30 ;;
190
* If BSPSTORE needs to be switched, p1 is false and p2 is true.
193
(p2) mov r30 = R_KSTACK_BSP ;;
194
(p2) mov ar.bspstore = r30 ;;
198
st8 [r31] = r27, -8 ;; /* save ar.rnat */
199
st8 [r31] = r30, -8 ;; /* save new value written to ar.bspstore */
200
st8 [r31] = r28, -8 ;; /* save ar.bspstore */
201
st8 [r31] = r29, -8 /* save ar.bsp */
203
mov ar.rsc = r24 /* restore RSE's setting + kernel privileges */
205
/* steps 6 - 15 are done by heavyweight_handler_inner() */
206
mov R_RET = b0 /* save b0 belonging to interrupted context */
207
br.call.sptk.many b0 = heavyweight_handler_inner
208
0: mov b0 = R_RET /* restore b0 belonging to the interrupted context */
210
/* 16. RSE switch to interrupted context */
211
cover /* allocate zero size frame (step 1 (from Intel Docs)) */
213
add r31 = (STACK_SCRATCH_AREA_SIZE + (FRS_TO_SAVE * 2 * 8)), r12 ;;
215
ld8 r30 = [r31], +8 ;; /* load ar.bsp */
216
ld8 r29 = [r31], +8 ;; /* load ar.bspstore */
217
ld8 r28 = [r31], +8 ;; /* load ar.bspstore_new */
218
sub r27 = r30 , r28 ;; /* calculate loadrs (step 2) */
223
or r24 = r30 , r27 ;;
224
mov ar.rsc = r24 ;; /* place RSE in enforced lazy mode */
226
loadrs /* (step 3) */
228
ld8 r27 = [r31], +8 ;; /* load ar.rnat */
229
ld8 r26 = [r31], +8 ;; /* load cr.ifs */
230
ld8 r25 = [r31], +8 ;; /* load ar.pfs */
231
ld8 r24 = [r31], +8 ;; /* load ar.rsc */
233
mov ar.bspstore = r29 ;; /* (step 4) */
234
mov ar.rnat = r27 /* (step 5) */
236
mov ar.pfs = r25 /* (step 6) */
239
mov ar.rsc = r24 /* (step 7) */
241
/* 17. restore interruption state from memory stack */
242
ld8 r28 = [r31], +8 ;; /* load cr.ifa */
243
ld8 r27 = [r31], +8 ;; /* load cr.isr */
244
ld8 r26 = [r31], +8 ;; /* load cr.iipa */
245
ld8 r25 = [r31], +8 ;; /* load cr.ipsr */
246
ld8 r24 = [r31], +8 ;; /* load cr.iip */
253
/* Set up FPU as in exception. */
255
mov r26 = PSR_DFH_MASK
256
mov r27 = ~PSR_DFH_MASK ;;
258
and r24 = r24, r26 ;;
262
/* 18. restore predicate registers from memory stack */
263
ld8 r29 = [r31], +8 ;; /* load predicate registers */
266
/* 19. return from interruption */
267
ld8 r12 = [r31] /* load stack pointer */
270
.global heavyweight_handler_inner
271
heavyweight_handler_inner:
273
* From this point, the rest of the interrupted context
274
* will be preserved in stacked registers and backing store.
276
alloc loc0 = ar.pfs, 0, 48, 2, 0 ;;
278
/* bank 0 is going to be shadowed, copy essential data from there */
279
mov loc1 = R_RET /* b0 belonging to interrupted context */
283
add out1 = STACK_SCRATCH_AREA_SIZE, r12
285
/* 6. switch to bank 1 and reenable PSR.ic */
290
/* 7. preserve branch and application registers */
307
/* 8. preserve general and floating-point registers */
315
(p5) mov loc24 = r8 /* only if not in break_instruction handler */
319
/* skip r12 (stack pointer) */
340
add r24 = 96 + STACK_SCRATCH_AREA_SIZE, r12
341
add r25 = 112 + STACK_SCRATCH_AREA_SIZE, r12
342
add r26 = 0 + STACK_SCRATCH_AREA_SIZE, r12
343
add r27 = 16 + STACK_SCRATCH_AREA_SIZE, r12
344
add r28 = 32 + STACK_SCRATCH_AREA_SIZE, r12
345
add r29 = 48 + STACK_SCRATCH_AREA_SIZE, r12
346
add r30 = 64 + STACK_SCRATCH_AREA_SIZE, r12
347
add r31 = 80 + STACK_SCRATCH_AREA_SIZE, r12 ;;
349
stf.spill [r26] = f2, 0x80
350
stf.spill [r27] = f3, 0x80
351
stf.spill [r28] = f4, 0x80
352
stf.spill [r29] = f5, 0x80
353
stf.spill [r30] = f6, 0x80
354
stf.spill [r31] = f7, 0x80 ;;
356
stf.spill [r24] = f8, 0x80
357
stf.spill [r25] = f9, 0x80
358
stf.spill [r26] = f10, 0x80
359
stf.spill [r27] = f11, 0x80
360
stf.spill [r28] = f12, 0x80
361
stf.spill [r29] = f13, 0x80
362
stf.spill [r30] = f14, 0x80
363
stf.spill [r31] = f15, 0x80 ;;
365
stf.spill [r24] = f16, 0x80
366
stf.spill [r25] = f17, 0x80
367
stf.spill [r26] = f18, 0x80
368
stf.spill [r27] = f19, 0x80
369
stf.spill [r28] = f20, 0x80
370
stf.spill [r29] = f21, 0x80
371
stf.spill [r30] = f22, 0x80
372
stf.spill [r31] = f23, 0x80 ;;
374
stf.spill [r24] = f24, 0x80
375
stf.spill [r25] = f25, 0x80
376
stf.spill [r26] = f26, 0x80
377
stf.spill [r27] = f27, 0x80
378
stf.spill [r28] = f28, 0x80
379
stf.spill [r29] = f29, 0x80
380
stf.spill [r30] = f30, 0x80
381
stf.spill [r31] = f31, 0x80 ;;
383
mov loc47 = ar.fpsr /* preserve floating point status register */
385
/* 9. skipped (will not enable interrupts) */
392
/* 10. call handler */
393
movl r1 = _hardcoded_load_address
396
br.call.sptk.many b0 = b1
398
/* 11. return from handler */
401
/* 12. skipped (will not disable interrupts) */
408
/* 13. restore general and floating-point registers */
409
add r24 = 96 + STACK_SCRATCH_AREA_SIZE, r12
410
add r25 = 112 + STACK_SCRATCH_AREA_SIZE, r12
411
add r26 = 0 + STACK_SCRATCH_AREA_SIZE, r12
412
add r27 = 16 + STACK_SCRATCH_AREA_SIZE, r12
413
add r28 = 32 + STACK_SCRATCH_AREA_SIZE, r12
414
add r29 = 48 + STACK_SCRATCH_AREA_SIZE, r12
415
add r30 = 64 + STACK_SCRATCH_AREA_SIZE, r12
416
add r31 = 80 + STACK_SCRATCH_AREA_SIZE, r12 ;;
418
ldf.fill f2 = [r26], 0x80
419
ldf.fill f3 = [r27], 0x80
420
ldf.fill f4 = [r28], 0x80
421
ldf.fill f5 = [r29], 0x80
422
ldf.fill f6 = [r30], 0x80
423
ldf.fill f7 = [r31], 0x80 ;;
425
ldf.fill f8 = [r24], 0x80
426
ldf.fill f9 = [r25], 0x80
427
ldf.fill f10 = [r26], 0x80
428
ldf.fill f11 = [r27], 0x80
429
ldf.fill f12 = [r28], 0x80
430
ldf.fill f13 = [r29], 0x80
431
ldf.fill f14 = [r30], 0x80
432
ldf.fill f15 = [r31], 0x80 ;;
434
ldf.fill f16 = [r24], 0x80
435
ldf.fill f17 = [r25], 0x80
436
ldf.fill f18 = [r26], 0x80
437
ldf.fill f19 = [r27], 0x80
438
ldf.fill f20 = [r28], 0x80
439
ldf.fill f21 = [r29], 0x80
440
ldf.fill f22 = [r30], 0x80
441
ldf.fill f23 = [r31], 0x80 ;;
443
ldf.fill f24 = [r24], 0x80
444
ldf.fill f25 = [r25], 0x80
445
ldf.fill f26 = [r26], 0x80
446
ldf.fill f27 = [r27], 0x80
447
ldf.fill f28 = [r28], 0x80
448
ldf.fill f29 = [r29], 0x80
449
ldf.fill f30 = [r30], 0x80
450
ldf.fill f31 = [r31], 0x80 ;;
459
(p5) mov r8 = loc24 /* only if not in break_instruction handler */
463
/* skip r12 (stack pointer) */
484
mov ar.fpsr = loc47 /* restore floating point status register */
486
/* 14. restore branch and application registers */
503
/* 15. disable PSR.ic and switch to bank 0 */
515
HEAVYWEIGHT_HANDLER 0x0000
516
HEAVYWEIGHT_HANDLER 0x0400
517
HEAVYWEIGHT_HANDLER 0x0800
518
HEAVYWEIGHT_HANDLER 0x0c00 alternate_instruction_tlb_fault
519
HEAVYWEIGHT_HANDLER 0x1000 alternate_data_tlb_fault
520
HEAVYWEIGHT_HANDLER 0x1400 data_nested_tlb_fault
521
HEAVYWEIGHT_HANDLER 0x1800
522
HEAVYWEIGHT_HANDLER 0x1c00
523
HEAVYWEIGHT_HANDLER 0x2000 data_dirty_bit_fault
524
HEAVYWEIGHT_HANDLER 0x2400 instruction_access_bit_fault
525
HEAVYWEIGHT_HANDLER 0x2800 data_access_bit_fault
526
HEAVYWEIGHT_HANDLER 0x2c00 break_instruction
527
HEAVYWEIGHT_HANDLER 0x3000 external_interrupt /* For external interrupt, heavyweight handler is used. */
528
HEAVYWEIGHT_HANDLER 0x3400
529
HEAVYWEIGHT_HANDLER 0x3800
530
HEAVYWEIGHT_HANDLER 0x3c00
531
HEAVYWEIGHT_HANDLER 0x4000
532
HEAVYWEIGHT_HANDLER 0x4400
533
HEAVYWEIGHT_HANDLER 0x4800
534
HEAVYWEIGHT_HANDLER 0x4c00
536
HEAVYWEIGHT_HANDLER 0x5000 page_not_present
537
HEAVYWEIGHT_HANDLER 0x5100
538
HEAVYWEIGHT_HANDLER 0x5200
539
HEAVYWEIGHT_HANDLER 0x5300 data_access_rights_fault
540
HEAVYWEIGHT_HANDLER 0x5400 general_exception
541
HEAVYWEIGHT_HANDLER 0x5500 disabled_fp_register
542
HEAVYWEIGHT_HANDLER 0x5600
543
HEAVYWEIGHT_HANDLER 0x5700
544
HEAVYWEIGHT_HANDLER 0x5800
545
HEAVYWEIGHT_HANDLER 0x5900
546
HEAVYWEIGHT_HANDLER 0x5a00
547
HEAVYWEIGHT_HANDLER 0x5b00
548
HEAVYWEIGHT_HANDLER 0x5c00
549
HEAVYWEIGHT_HANDLER 0x5d00
550
HEAVYWEIGHT_HANDLER 0x5e00
551
HEAVYWEIGHT_HANDLER 0x5f00
553
HEAVYWEIGHT_HANDLER 0x6000
554
HEAVYWEIGHT_HANDLER 0x6100
555
HEAVYWEIGHT_HANDLER 0x6200
556
HEAVYWEIGHT_HANDLER 0x6300
557
HEAVYWEIGHT_HANDLER 0x6400
558
HEAVYWEIGHT_HANDLER 0x6500
559
HEAVYWEIGHT_HANDLER 0x6600
560
HEAVYWEIGHT_HANDLER 0x6700
561
HEAVYWEIGHT_HANDLER 0x6800
562
HEAVYWEIGHT_HANDLER 0x6900
563
HEAVYWEIGHT_HANDLER 0x6a00
564
HEAVYWEIGHT_HANDLER 0x6b00
565
HEAVYWEIGHT_HANDLER 0x6c00
566
HEAVYWEIGHT_HANDLER 0x6d00
567
HEAVYWEIGHT_HANDLER 0x6e00
568
HEAVYWEIGHT_HANDLER 0x6f00
570
HEAVYWEIGHT_HANDLER 0x7000
571
HEAVYWEIGHT_HANDLER 0x7100
572
HEAVYWEIGHT_HANDLER 0x7200
573
HEAVYWEIGHT_HANDLER 0x7300
574
HEAVYWEIGHT_HANDLER 0x7400
575
HEAVYWEIGHT_HANDLER 0x7500
576
HEAVYWEIGHT_HANDLER 0x7600
577
HEAVYWEIGHT_HANDLER 0x7700
578
HEAVYWEIGHT_HANDLER 0x7800
579
HEAVYWEIGHT_HANDLER 0x7900
580
HEAVYWEIGHT_HANDLER 0x7a00
581
HEAVYWEIGHT_HANDLER 0x7b00
582
HEAVYWEIGHT_HANDLER 0x7c00
583
HEAVYWEIGHT_HANDLER 0x7d00
584
HEAVYWEIGHT_HANDLER 0x7e00
585
HEAVYWEIGHT_HANDLER 0x7f00