1
by Alessio Igor Bogani
Import upstream version 3.2.0 |
1 |
/*
|
2 |
* Copyright (C) 1991, 1992 Linus Torvalds
|
|
3 |
* Copyright (C) 2000, 2001, 2002 Andi Kleen, SuSE Labs
|
|
4 |
*
|
|
5 |
* Pentium III FXSR, SSE support
|
|
6 |
* Gareth Hughes <gareth@valinux.com>, May 2000
|
|
7 |
*/
|
|
8 |
||
9 |
/*
|
|
10 |
* Handle hardware traps and faults.
|
|
11 |
*/
|
|
12 |
#include <linux/interrupt.h> |
|
13 |
#include <linux/kallsyms.h> |
|
14 |
#include <linux/spinlock.h> |
|
15 |
#include <linux/kprobes.h> |
|
16 |
#include <linux/uaccess.h> |
|
17 |
#include <linux/kdebug.h> |
|
18 |
#include <linux/kgdb.h> |
|
19 |
#include <linux/kernel.h> |
|
20 |
#include <linux/module.h> |
|
21 |
#include <linux/ptrace.h> |
|
22 |
#include <linux/string.h> |
|
23 |
#include <linux/delay.h> |
|
24 |
#include <linux/errno.h> |
|
25 |
#include <linux/kexec.h> |
|
26 |
#include <linux/sched.h> |
|
27 |
#include <linux/timer.h> |
|
28 |
#include <linux/init.h> |
|
29 |
#include <linux/bug.h> |
|
30 |
#include <linux/nmi.h> |
|
31 |
#include <linux/mm.h> |
|
32 |
#include <linux/smp.h> |
|
33 |
#include <linux/io.h> |
|
34 |
||
35 |
#ifdef CONFIG_EISA
|
|
36 |
#include <linux/ioport.h> |
|
37 |
#include <linux/eisa.h> |
|
38 |
#endif
|
|
39 |
||
40 |
#ifdef CONFIG_MCA
|
|
41 |
#include <linux/mca.h> |
|
42 |
#endif
|
|
43 |
||
44 |
#if defined(CONFIG_EDAC)
|
|
45 |
#include <linux/edac.h> |
|
46 |
#endif
|
|
47 |
||
48 |
#include <asm/kmemcheck.h> |
|
49 |
#include <asm/stacktrace.h> |
|
50 |
#include <asm/processor.h> |
|
51 |
#include <asm/debugreg.h> |
|
52 |
#include <linux/atomic.h> |
|
53 |
#include <asm/system.h> |
|
54 |
#include <asm/traps.h> |
|
55 |
#include <asm/desc.h> |
|
56 |
#include <asm/i387.h> |
|
57 |
#include <asm/mce.h> |
|
58 |
||
59 |
#include <asm/mach_traps.h> |
|
60 |
||
61 |
#ifdef CONFIG_X86_64
|
|
62 |
#include <asm/x86_init.h> |
|
63 |
#include <asm/pgalloc.h> |
|
64 |
#include <asm/proto.h> |
|
65 |
#else
|
|
66 |
#include <asm/processor-flags.h> |
|
67 |
#include <asm/setup.h> |
|
68 |
||
69 |
asmlinkage int system_call(void); |
|
70 |
||
71 |
/* Do we ignore FPU interrupts ? */
|
|
72 |
char ignore_fpu_irq; |
|
73 |
||
74 |
/*
|
|
75 |
* The IDT has to be page-aligned to simplify the Pentium
|
|
76 |
* F0 0F bug workaround.
|
|
77 |
*/
|
|
78 |
gate_desc idt_table[NR_VECTORS] __page_aligned_data = { { { { 0, 0 } } }, }; |
|
79 |
#endif
|
|
80 |
||
81 |
DECLARE_BITMAP(used_vectors, NR_VECTORS); |
|
82 |
EXPORT_SYMBOL_GPL(used_vectors); |
|
83 |
||
84 |
static inline void conditional_sti(struct pt_regs *regs) |
|
85 |
{
|
|
86 |
if (regs->flags & X86_EFLAGS_IF) |
|
87 |
local_irq_enable(); |
|
88 |
}
|
|
89 |
||
90 |
static inline void preempt_conditional_sti(struct pt_regs *regs) |
|
91 |
{
|
|
92 |
inc_preempt_count(); |
|
93 |
if (regs->flags & X86_EFLAGS_IF) |
|
94 |
local_irq_enable(); |
|
95 |
}
|
|
96 |
||
97 |
static inline void conditional_cli(struct pt_regs *regs) |
|
98 |
{
|
|
99 |
if (regs->flags & X86_EFLAGS_IF) |
|
100 |
local_irq_disable(); |
|
101 |
}
|
|
102 |
||
103 |
static inline void preempt_conditional_cli(struct pt_regs *regs) |
|
104 |
{
|
|
105 |
if (regs->flags & X86_EFLAGS_IF) |
|
106 |
local_irq_disable(); |
|
107 |
dec_preempt_count(); |
|
108 |
}
|
|
109 |
||
2
by Alessio Igor Bogani
Add new lowlatency kernel flavour |
110 |
#ifdef CONFIG_X86_32
|
111 |
static inline int |
|
112 |
__compare_user_cs_desc(const struct desc_struct *desc1, |
|
113 |
const struct desc_struct *desc2) |
|
114 |
{
|
|
115 |
return ((desc1->limit0 != desc2->limit0) || |
|
116 |
(desc1->limit != desc2->limit) || |
|
117 |
(desc1->base0 != desc2->base0) || |
|
118 |
(desc1->base1 != desc2->base1) || |
|
119 |
(desc1->base2 != desc2->base2)); |
|
120 |
}
|
|
121 |
||
122 |
/*
|
|
123 |
* lazy-check for CS validity on exec-shield binaries:
|
|
124 |
*
|
|
125 |
* the original non-exec stack patch was written by
|
|
126 |
* Solar Designer <solar at openwall.com>. Thanks!
|
|
127 |
*/
|
|
128 |
static int |
|
129 |
check_lazy_exec_limit(int cpu, struct pt_regs *regs, long error_code) |
|
130 |
{
|
|
131 |
struct desc_struct *desc1, *desc2; |
|
132 |
struct vm_area_struct *vma; |
|
133 |
unsigned long limit; |
|
134 |
||
135 |
if (current->mm == NULL) |
|
136 |
return 0; |
|
137 |
||
138 |
limit = -1UL; |
|
139 |
if (current->mm->context.exec_limit != -1UL) { |
|
140 |
limit = PAGE_SIZE; |
|
141 |
spin_lock(¤t->mm->page_table_lock); |
|
142 |
for (vma = current->mm->mmap; vma; vma = vma->vm_next) |
|
143 |
if ((vma->vm_flags & VM_EXEC) && (vma->vm_end > limit)) |
|
144 |
limit = vma->vm_end; |
|
145 |
vma = get_gate_vma(current); |
|
146 |
if (vma && (vma->vm_flags & VM_EXEC) && (vma->vm_end > limit)) |
|
147 |
limit = vma->vm_end; |
|
148 |
spin_unlock(¤t->mm->page_table_lock); |
|
149 |
if (limit >= TASK_SIZE) |
|
150 |
limit = -1UL; |
|
151 |
current->mm->context.exec_limit = limit; |
|
152 |
}
|
|
153 |
set_user_cs(¤t->mm->context.user_cs, limit); |
|
154 |
||
155 |
desc1 = ¤t->mm->context.user_cs; |
|
156 |
desc2 = get_cpu_gdt_table(cpu) + GDT_ENTRY_DEFAULT_USER_CS; |
|
157 |
||
158 |
if (__compare_user_cs_desc(desc1, desc2)) { |
|
159 |
/*
|
|
160 |
* The CS was not in sync - reload it and retry the
|
|
161 |
* instruction. If the instruction still faults then
|
|
162 |
* we won't hit this branch next time around.
|
|
163 |
*/
|
|
164 |
if (print_fatal_signals >= 2) { |
|
165 |
printk(KERN_ERR "#GPF fixup (%ld[seg:%lx]) at %08lx, CPU#%d.\n", |
|
166 |
error_code, error_code/8, regs->ip, |
|
167 |
smp_processor_id()); |
|
168 |
printk(KERN_ERR "exec_limit: %08lx, user_cs: %08x/%08x, CPU_cs: %08x/%08x.\n", |
|
169 |
current->mm->context.exec_limit, |
|
170 |
desc1->a, desc1->b, desc2->a, desc2->b); |
|
171 |
}
|
|
172 |
||
173 |
load_user_cs_desc(cpu, current->mm); |
|
174 |
||
175 |
return 1; |
|
176 |
}
|
|
177 |
||
178 |
return 0; |
|
179 |
}
|
|
180 |
#endif
|
|
181 |
||
1
by Alessio Igor Bogani
Import upstream version 3.2.0 |
182 |
static void __kprobes |
183 |
do_trap(int trapnr, int signr, char *str, struct pt_regs *regs, |
|
184 |
long error_code, siginfo_t *info) |
|
185 |
{
|
|
186 |
struct task_struct *tsk = current; |
|
187 |
||
188 |
#ifdef CONFIG_X86_32
|
|
189 |
if (regs->flags & X86_VM_MASK) { |
|
190 |
/*
|
|
191 |
* traps 0, 1, 3, 4, and 5 should be forwarded to vm86.
|
|
192 |
* On nmi (interrupt 2), do_trap should not be called.
|
|
193 |
*/
|
|
194 |
if (trapnr < 6) |
|
195 |
goto vm86_trap; |
|
196 |
goto trap_signal; |
|
197 |
}
|
|
198 |
#endif
|
|
199 |
||
200 |
if (!user_mode(regs)) |
|
201 |
goto kernel_trap; |
|
202 |
||
203 |
#ifdef CONFIG_X86_32
|
|
204 |
trap_signal: |
|
205 |
#endif
|
|
206 |
/*
|
|
207 |
* We want error_code and trap_no set for userspace faults and
|
|
208 |
* kernelspace faults which result in die(), but not
|
|
209 |
* kernelspace faults which are fixed up. die() gives the
|
|
210 |
* process no chance to handle the signal and notice the
|
|
211 |
* kernel fault information, so that won't result in polluting
|
|
212 |
* the information about previously queued, but not yet
|
|
213 |
* delivered, faults. See also do_general_protection below.
|
|
214 |
*/
|
|
215 |
tsk->thread.error_code = error_code; |
|
216 |
tsk->thread.trap_no = trapnr; |
|
217 |
||
218 |
#ifdef CONFIG_X86_64
|
|
219 |
if (show_unhandled_signals && unhandled_signal(tsk, signr) && |
|
220 |
printk_ratelimit()) { |
|
221 |
printk(KERN_INFO |
|
222 |
"%s[%d] trap %s ip:%lx sp:%lx error:%lx", |
|
223 |
tsk->comm, tsk->pid, str, |
|
224 |
regs->ip, regs->sp, error_code); |
|
225 |
print_vma_addr(" in ", regs->ip); |
|
226 |
printk("\n"); |
|
227 |
}
|
|
228 |
#endif
|
|
229 |
||
230 |
if (info) |
|
231 |
force_sig_info(signr, info, tsk); |
|
232 |
else
|
|
233 |
force_sig(signr, tsk); |
|
234 |
return; |
|
235 |
||
236 |
kernel_trap: |
|
237 |
if (!fixup_exception(regs)) { |
|
238 |
tsk->thread.error_code = error_code; |
|
239 |
tsk->thread.trap_no = trapnr; |
|
240 |
die(str, regs, error_code); |
|
241 |
}
|
|
242 |
return; |
|
243 |
||
244 |
#ifdef CONFIG_X86_32
|
|
245 |
vm86_trap: |
|
246 |
if (handle_vm86_trap((struct kernel_vm86_regs *) regs, |
|
247 |
error_code, trapnr)) |
|
248 |
goto trap_signal; |
|
249 |
return; |
|
250 |
#endif
|
|
251 |
}
|
|
252 |
||
253 |
#define DO_ERROR(trapnr, signr, str, name) \
|
|
254 |
dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \
|
|
255 |
{ \
|
|
256 |
if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \
|
|
257 |
== NOTIFY_STOP) \
|
|
258 |
return; \
|
|
259 |
conditional_sti(regs); \
|
|
260 |
do_trap(trapnr, signr, str, regs, error_code, NULL); \
|
|
261 |
}
|
|
262 |
||
263 |
#define DO_ERROR_INFO(trapnr, signr, str, name, sicode, siaddr) \
|
|
264 |
dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \
|
|
265 |
{ \
|
|
266 |
siginfo_t info; \
|
|
267 |
info.si_signo = signr; \
|
|
268 |
info.si_errno = 0; \
|
|
269 |
info.si_code = sicode; \
|
|
270 |
info.si_addr = (void __user *)siaddr; \
|
|
271 |
if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) \
|
|
272 |
== NOTIFY_STOP) \
|
|
273 |
return; \
|
|
274 |
conditional_sti(regs); \
|
|
275 |
do_trap(trapnr, signr, str, regs, error_code, &info); \
|
|
276 |
}
|
|
277 |
||
278 |
DO_ERROR_INFO(0, SIGFPE, "divide error", divide_error, FPE_INTDIV, regs->ip) |
|
279 |
DO_ERROR(4, SIGSEGV, "overflow", overflow) |
|
280 |
DO_ERROR(5, SIGSEGV, "bounds", bounds) |
|
281 |
DO_ERROR_INFO(6, SIGILL, "invalid opcode", invalid_op, ILL_ILLOPN, regs->ip) |
|
282 |
DO_ERROR(9, SIGFPE, "coprocessor segment overrun", coprocessor_segment_overrun) |
|
283 |
DO_ERROR(10, SIGSEGV, "invalid TSS", invalid_TSS) |
|
284 |
DO_ERROR(11, SIGBUS, "segment not present", segment_not_present) |
|
285 |
#ifdef CONFIG_X86_32
|
|
286 |
DO_ERROR(12, SIGBUS, "stack segment", stack_segment) |
|
287 |
#endif
|
|
288 |
DO_ERROR_INFO(17, SIGBUS, "alignment check", alignment_check, BUS_ADRALN, 0) |
|
289 |
||
290 |
#ifdef CONFIG_X86_64
|
|
291 |
/* Runs on IST stack */
|
|
292 |
dotraplinkage void do_stack_segment(struct pt_regs *regs, long error_code) |
|
293 |
{
|
|
294 |
if (notify_die(DIE_TRAP, "stack segment", regs, error_code, |
|
295 |
12, SIGBUS) == NOTIFY_STOP) |
|
296 |
return; |
|
297 |
preempt_conditional_sti(regs); |
|
298 |
do_trap(12, SIGBUS, "stack segment", regs, error_code, NULL); |
|
299 |
preempt_conditional_cli(regs); |
|
300 |
}
|
|
301 |
||
302 |
dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code) |
|
303 |
{
|
|
304 |
static const char str[] = "double fault"; |
|
305 |
struct task_struct *tsk = current; |
|
306 |
||
307 |
/* Return not checked because double check cannot be ignored */
|
|
308 |
notify_die(DIE_TRAP, str, regs, error_code, 8, SIGSEGV); |
|
309 |
||
310 |
tsk->thread.error_code = error_code; |
|
311 |
tsk->thread.trap_no = 8; |
|
312 |
||
313 |
/*
|
|
314 |
* This is always a kernel trap and never fixable (and thus must
|
|
315 |
* never return).
|
|
316 |
*/
|
|
317 |
for (;;) |
|
318 |
die(str, regs, error_code); |
|
319 |
}
|
|
320 |
#endif
|
|
321 |
||
322 |
dotraplinkage void __kprobes |
|
323 |
do_general_protection(struct pt_regs *regs, long error_code) |
|
324 |
{
|
|
325 |
struct task_struct *tsk; |
|
326 |
||
327 |
conditional_sti(regs); |
|
328 |
||
329 |
#ifdef CONFIG_X86_32
|
|
330 |
if (regs->flags & X86_VM_MASK) |
|
331 |
goto gp_in_vm86; |
|
332 |
#endif
|
|
333 |
||
334 |
tsk = current; |
|
335 |
if (!user_mode(regs)) |
|
336 |
goto gp_in_kernel; |
|
337 |
||
2
by Alessio Igor Bogani
Add new lowlatency kernel flavour |
338 |
#ifdef CONFIG_X86_32
|
339 |
{
|
|
340 |
int cpu; |
|
341 |
int ok; |
|
342 |
||
343 |
cpu = get_cpu(); |
|
344 |
ok = check_lazy_exec_limit(cpu, regs, error_code); |
|
345 |
put_cpu(); |
|
346 |
||
347 |
if (ok) |
|
348 |
return; |
|
349 |
||
350 |
if (print_fatal_signals) { |
|
351 |
printk(KERN_ERR "#GPF(%ld[seg:%lx]) at %08lx, CPU#%d.\n", |
|
352 |
error_code, error_code/8, regs->ip, smp_processor_id()); |
|
353 |
printk(KERN_ERR "exec_limit: %08lx, user_cs: %08x/%08x.\n", |
|
354 |
current->mm->context.exec_limit, |
|
355 |
current->mm->context.user_cs.a, |
|
356 |
current->mm->context.user_cs.b); |
|
357 |
}
|
|
358 |
}
|
|
359 |
#endif /*CONFIG_X86_32*/ |
|
360 |
||
1
by Alessio Igor Bogani
Import upstream version 3.2.0 |
361 |
tsk->thread.error_code = error_code; |
362 |
tsk->thread.trap_no = 13; |
|
363 |
||
364 |
if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) && |
|
365 |
printk_ratelimit()) { |
|
366 |
printk(KERN_INFO |
|
367 |
"%s[%d] general protection ip:%lx sp:%lx error:%lx", |
|
368 |
tsk->comm, task_pid_nr(tsk), |
|
369 |
regs->ip, regs->sp, error_code); |
|
370 |
print_vma_addr(" in ", regs->ip); |
|
371 |
printk("\n"); |
|
372 |
}
|
|
373 |
||
374 |
force_sig(SIGSEGV, tsk); |
|
375 |
return; |
|
376 |
||
377 |
#ifdef CONFIG_X86_32
|
|
378 |
gp_in_vm86: |
|
379 |
local_irq_enable(); |
|
380 |
handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code); |
|
381 |
return; |
|
382 |
#endif
|
|
383 |
||
384 |
gp_in_kernel: |
|
385 |
if (fixup_exception(regs)) |
|
386 |
return; |
|
387 |
||
388 |
tsk->thread.error_code = error_code; |
|
389 |
tsk->thread.trap_no = 13; |
|
390 |
if (notify_die(DIE_GPF, "general protection fault", regs, |
|
391 |
error_code, 13, SIGSEGV) == NOTIFY_STOP) |
|
392 |
return; |
|
393 |
die("general protection fault", regs, error_code); |
|
394 |
}
|
|
395 |
||
396 |
/* May run on IST stack. */
|
|
397 |
dotraplinkage void __kprobes do_int3(struct pt_regs *regs, long error_code) |
|
398 |
{
|
|
399 |
#ifdef CONFIG_KGDB_LOW_LEVEL_TRAP
|
|
400 |
if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, 3, SIGTRAP) |
|
401 |
== NOTIFY_STOP) |
|
402 |
return; |
|
403 |
#endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */ |
|
404 |
#ifdef CONFIG_KPROBES
|
|
405 |
if (notify_die(DIE_INT3, "int3", regs, error_code, 3, SIGTRAP) |
|
406 |
== NOTIFY_STOP) |
|
407 |
return; |
|
408 |
#else
|
|
409 |
if (notify_die(DIE_TRAP, "int3", regs, error_code, 3, SIGTRAP) |
|
410 |
== NOTIFY_STOP) |
|
411 |
return; |
|
412 |
#endif
|
|
413 |
||
414 |
preempt_conditional_sti(regs); |
|
415 |
do_trap(3, SIGTRAP, "int3", regs, error_code, NULL); |
|
416 |
preempt_conditional_cli(regs); |
|
417 |
}
|
|
418 |
||
419 |
#ifdef CONFIG_X86_64
|
|
420 |
/*
|
|
421 |
* Help handler running on IST stack to switch back to user stack
|
|
422 |
* for scheduling or signal handling. The actual stack switch is done in
|
|
423 |
* entry.S
|
|
424 |
*/
|
|
425 |
asmlinkage __kprobes struct pt_regs *sync_regs(struct pt_regs *eregs) |
|
426 |
{
|
|
427 |
struct pt_regs *regs = eregs; |
|
428 |
/* Did already sync */
|
|
429 |
if (eregs == (struct pt_regs *)eregs->sp) |
|
430 |
;
|
|
431 |
/* Exception from user space */
|
|
432 |
else if (user_mode(eregs)) |
|
433 |
regs = task_pt_regs(current); |
|
434 |
/*
|
|
435 |
* Exception from kernel and interrupts are enabled. Move to
|
|
436 |
* kernel process stack.
|
|
437 |
*/
|
|
438 |
else if (eregs->flags & X86_EFLAGS_IF) |
|
439 |
regs = (struct pt_regs *)(eregs->sp -= sizeof(struct pt_regs)); |
|
440 |
if (eregs != regs) |
|
441 |
*regs = *eregs; |
|
442 |
return regs; |
|
443 |
}
|
|
444 |
#endif
|
|
445 |
||
446 |
/*
|
|
447 |
* Our handling of the processor debug registers is non-trivial.
|
|
448 |
* We do not clear them on entry and exit from the kernel. Therefore
|
|
449 |
* it is possible to get a watchpoint trap here from inside the kernel.
|
|
450 |
* However, the code in ./ptrace.c has ensured that the user can
|
|
451 |
* only set watchpoints on userspace addresses. Therefore the in-kernel
|
|
452 |
* watchpoint trap can only occur in code which is reading/writing
|
|
453 |
* from user space. Such code must not hold kernel locks (since it
|
|
454 |
* can equally take a page fault), therefore it is safe to call
|
|
455 |
* force_sig_info even though that claims and releases locks.
|
|
456 |
*
|
|
457 |
* Code in ./signal.c ensures that the debug control register
|
|
458 |
* is restored before we deliver any signal, and therefore that
|
|
459 |
* user code runs with the correct debug control register even though
|
|
460 |
* we clear it here.
|
|
461 |
*
|
|
462 |
* Being careful here means that we don't have to be as careful in a
|
|
463 |
* lot of more complicated places (task switching can be a bit lazy
|
|
464 |
* about restoring all the debug state, and ptrace doesn't have to
|
|
465 |
* find every occurrence of the TF bit that could be saved away even
|
|
466 |
* by user code)
|
|
467 |
*
|
|
468 |
* May run on IST stack.
|
|
469 |
*/
|
|
470 |
dotraplinkage void __kprobes do_debug(struct pt_regs *regs, long error_code) |
|
471 |
{
|
|
472 |
struct task_struct *tsk = current; |
|
473 |
int user_icebp = 0; |
|
474 |
unsigned long dr6; |
|
475 |
int si_code; |
|
476 |
||
477 |
get_debugreg(dr6, 6); |
|
478 |
||
479 |
/* Filter out all the reserved bits which are preset to 1 */
|
|
480 |
dr6 &= ~DR6_RESERVED; |
|
481 |
||
482 |
/*
|
|
483 |
* If dr6 has no reason to give us about the origin of this trap,
|
|
484 |
* then it's very likely the result of an icebp/int01 trap.
|
|
485 |
* User wants a sigtrap for that.
|
|
486 |
*/
|
|
487 |
if (!dr6 && user_mode(regs)) |
|
488 |
user_icebp = 1; |
|
489 |
||
490 |
/* Catch kmemcheck conditions first of all! */
|
|
491 |
if ((dr6 & DR_STEP) && kmemcheck_trap(regs)) |
|
492 |
return; |
|
493 |
||
494 |
/* DR6 may or may not be cleared by the CPU */
|
|
495 |
set_debugreg(0, 6); |
|
496 |
||
497 |
/*
|
|
498 |
* The processor cleared BTF, so don't mark that we need it set.
|
|
499 |
*/
|
|
500 |
clear_tsk_thread_flag(tsk, TIF_BLOCKSTEP); |
|
501 |
||
502 |
/* Store the virtualized DR6 value */
|
|
503 |
tsk->thread.debugreg6 = dr6; |
|
504 |
||
505 |
if (notify_die(DIE_DEBUG, "debug", regs, PTR_ERR(&dr6), error_code, |
|
506 |
SIGTRAP) == NOTIFY_STOP) |
|
507 |
return; |
|
508 |
||
509 |
/* It's safe to allow irq's after DR6 has been saved */
|
|
510 |
preempt_conditional_sti(regs); |
|
511 |
||
512 |
if (regs->flags & X86_VM_MASK) { |
|
513 |
handle_vm86_trap((struct kernel_vm86_regs *) regs, |
|
514 |
error_code, 1); |
|
515 |
preempt_conditional_cli(regs); |
|
516 |
return; |
|
517 |
}
|
|
518 |
||
519 |
/*
|
|
520 |
* Single-stepping through system calls: ignore any exceptions in
|
|
521 |
* kernel space, but re-enable TF when returning to user mode.
|
|
522 |
*
|
|
523 |
* We already checked v86 mode above, so we can check for kernel mode
|
|
524 |
* by just checking the CPL of CS.
|
|
525 |
*/
|
|
526 |
if ((dr6 & DR_STEP) && !user_mode(regs)) { |
|
527 |
tsk->thread.debugreg6 &= ~DR_STEP; |
|
528 |
set_tsk_thread_flag(tsk, TIF_SINGLESTEP); |
|
529 |
regs->flags &= ~X86_EFLAGS_TF; |
|
530 |
}
|
|
531 |
si_code = get_si_code(tsk->thread.debugreg6); |
|
532 |
if (tsk->thread.debugreg6 & (DR_STEP | DR_TRAP_BITS) || user_icebp) |
|
533 |
send_sigtrap(tsk, regs, error_code, si_code); |
|
534 |
preempt_conditional_cli(regs); |
|
535 |
||
536 |
return; |
|
537 |
}
|
|
538 |
||
539 |
/*
|
|
540 |
* Note that we play around with the 'TS' bit in an attempt to get
|
|
541 |
* the correct behaviour even in the presence of the asynchronous
|
|
542 |
* IRQ13 behaviour
|
|
543 |
*/
|
|
544 |
void math_error(struct pt_regs *regs, int error_code, int trapnr) |
|
545 |
{
|
|
546 |
struct task_struct *task = current; |
|
547 |
siginfo_t info; |
|
548 |
unsigned short err; |
|
549 |
char *str = (trapnr == 16) ? "fpu exception" : "simd exception"; |
|
550 |
||
551 |
if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, SIGFPE) == NOTIFY_STOP) |
|
552 |
return; |
|
553 |
conditional_sti(regs); |
|
554 |
||
555 |
if (!user_mode_vm(regs)) |
|
556 |
{
|
|
557 |
if (!fixup_exception(regs)) { |
|
558 |
task->thread.error_code = error_code; |
|
559 |
task->thread.trap_no = trapnr; |
|
560 |
die(str, regs, error_code); |
|
561 |
}
|
|
562 |
return; |
|
563 |
}
|
|
564 |
||
565 |
/*
|
|
566 |
* Save the info for the exception handler and clear the error.
|
|
567 |
*/
|
|
568 |
save_init_fpu(task); |
|
569 |
task->thread.trap_no = trapnr; |
|
570 |
task->thread.error_code = error_code; |
|
571 |
info.si_signo = SIGFPE; |
|
572 |
info.si_errno = 0; |
|
573 |
info.si_addr = (void __user *)regs->ip; |
|
574 |
if (trapnr == 16) { |
|
575 |
unsigned short cwd, swd; |
|
576 |
/*
|
|
577 |
* (~cwd & swd) will mask out exceptions that are not set to unmasked
|
|
578 |
* status. 0x3f is the exception bits in these regs, 0x200 is the
|
|
579 |
* C1 reg you need in case of a stack fault, 0x040 is the stack
|
|
580 |
* fault bit. We should only be taking one exception at a time,
|
|
581 |
* so if this combination doesn't produce any single exception,
|
|
582 |
* then we have a bad program that isn't synchronizing its FPU usage
|
|
583 |
* and it will suffer the consequences since we won't be able to
|
|
584 |
* fully reproduce the context of the exception
|
|
585 |
*/
|
|
586 |
cwd = get_fpu_cwd(task); |
|
587 |
swd = get_fpu_swd(task); |
|
588 |
||
589 |
err = swd & ~cwd; |
|
590 |
} else { |
|
591 |
/*
|
|
592 |
* The SIMD FPU exceptions are handled a little differently, as there
|
|
593 |
* is only a single status/control register. Thus, to determine which
|
|
594 |
* unmasked exception was caught we must mask the exception mask bits
|
|
595 |
* at 0x1f80, and then use these to mask the exception bits at 0x3f.
|
|
596 |
*/
|
|
597 |
unsigned short mxcsr = get_fpu_mxcsr(task); |
|
598 |
err = ~(mxcsr >> 7) & mxcsr; |
|
599 |
}
|
|
600 |
||
601 |
if (err & 0x001) { /* Invalid op */ |
|
602 |
/*
|
|
603 |
* swd & 0x240 == 0x040: Stack Underflow
|
|
604 |
* swd & 0x240 == 0x240: Stack Overflow
|
|
605 |
* User must clear the SF bit (0x40) if set
|
|
606 |
*/
|
|
607 |
info.si_code = FPE_FLTINV; |
|
608 |
} else if (err & 0x004) { /* Divide by Zero */ |
|
609 |
info.si_code = FPE_FLTDIV; |
|
610 |
} else if (err & 0x008) { /* Overflow */ |
|
611 |
info.si_code = FPE_FLTOVF; |
|
612 |
} else if (err & 0x012) { /* Denormal, Underflow */ |
|
613 |
info.si_code = FPE_FLTUND; |
|
614 |
} else if (err & 0x020) { /* Precision */ |
|
615 |
info.si_code = FPE_FLTRES; |
|
616 |
} else { |
|
617 |
/*
|
|
618 |
* If we're using IRQ 13, or supposedly even some trap 16
|
|
619 |
* implementations, it's possible we get a spurious trap...
|
|
620 |
*/
|
|
621 |
return; /* Spurious trap, no error */ |
|
622 |
}
|
|
623 |
force_sig_info(SIGFPE, &info, task); |
|
624 |
}
|
|
625 |
||
626 |
dotraplinkage void do_coprocessor_error(struct pt_regs *regs, long error_code) |
|
627 |
{
|
|
628 |
#ifdef CONFIG_X86_32
|
|
629 |
ignore_fpu_irq = 1; |
|
630 |
#endif
|
|
631 |
||
632 |
math_error(regs, error_code, 16); |
|
633 |
}
|
|
634 |
||
635 |
dotraplinkage void |
|
636 |
do_simd_coprocessor_error(struct pt_regs *regs, long error_code) |
|
637 |
{
|
|
638 |
math_error(regs, error_code, 19); |
|
639 |
}
|
|
640 |
||
641 |
dotraplinkage void |
|
642 |
do_spurious_interrupt_bug(struct pt_regs *regs, long error_code) |
|
643 |
{
|
|
644 |
conditional_sti(regs); |
|
645 |
#if 0
|
|
646 |
/* No need to warn about this any longer. */
|
|
647 |
printk(KERN_INFO "Ignoring P6 Local APIC Spurious Interrupt Bug...\n");
|
|
648 |
#endif
|
|
649 |
}
|
|
650 |
||
651 |
asmlinkage void __attribute__((weak)) smp_thermal_interrupt(void) |
|
652 |
{
|
|
653 |
}
|
|
654 |
||
655 |
asmlinkage void __attribute__((weak)) smp_threshold_interrupt(void) |
|
656 |
{
|
|
657 |
}
|
|
658 |
||
659 |
/*
|
|
3
by Luke Yelavich, Andy Whitcroft, Chase Douglas, Eugeni Dodonov, Ingo Molnar, Johannes Berg, John Johansen, Kees Cook, Leann Ogasawara, Robert Hooker, Seth Heasley, Tim Gardner, Luke Yelavich, Upstream Kernel Changes
[ Andy Whitcroft ] |
660 |
* This gets called with the process already owning the
|
661 |
* FPU state, and with CR0.TS cleared. It just needs to
|
|
662 |
* restore the FPU register state.
|
|
1
by Alessio Igor Bogani
Import upstream version 3.2.0 |
663 |
*/
|
3
by Luke Yelavich, Andy Whitcroft, Chase Douglas, Eugeni Dodonov, Ingo Molnar, Johannes Berg, John Johansen, Kees Cook, Leann Ogasawara, Robert Hooker, Seth Heasley, Tim Gardner, Luke Yelavich, Upstream Kernel Changes
[ Andy Whitcroft ] |
664 |
void __math_state_restore(struct task_struct *tsk) |
1
by Alessio Igor Bogani
Import upstream version 3.2.0 |
665 |
{
|
3
by Luke Yelavich, Andy Whitcroft, Chase Douglas, Eugeni Dodonov, Ingo Molnar, Johannes Berg, John Johansen, Kees Cook, Leann Ogasawara, Robert Hooker, Seth Heasley, Tim Gardner, Luke Yelavich, Upstream Kernel Changes
[ Andy Whitcroft ] |
666 |
/* We need a safe address that is cheap to find and that is already
|
667 |
in L1. We've just brought in "tsk->thread.has_fpu", so use that */
|
|
668 |
#define safe_address (tsk->thread.has_fpu)
|
|
669 |
||
670 |
/* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception
|
|
671 |
is pending. Clear the x87 state here by setting it to fixed
|
|
672 |
values. safe_address is a random variable that should be in L1 */
|
|
673 |
alternative_input( |
|
674 |
ASM_NOP8 ASM_NOP2, |
|
675 |
"emms\n\t" /* clear stack tags */ |
|
676 |
"fildl %P[addr]", /* set F?P to defined value */ |
|
677 |
X86_FEATURE_FXSAVE_LEAK, |
|
678 |
[addr] "m" (safe_address)); |
|
1
by Alessio Igor Bogani
Import upstream version 3.2.0 |
679 |
|
680 |
/*
|
|
681 |
* Paranoid restore. send a SIGSEGV if we fail to restore the state.
|
|
682 |
*/
|
|
683 |
if (unlikely(restore_fpu_checking(tsk))) { |
|
3
by Luke Yelavich, Andy Whitcroft, Chase Douglas, Eugeni Dodonov, Ingo Molnar, Johannes Berg, John Johansen, Kees Cook, Leann Ogasawara, Robert Hooker, Seth Heasley, Tim Gardner, Luke Yelavich, Upstream Kernel Changes
[ Andy Whitcroft ] |
684 |
__thread_fpu_end(tsk); |
1
by Alessio Igor Bogani
Import upstream version 3.2.0 |
685 |
force_sig(SIGSEGV, tsk); |
686 |
return; |
|
687 |
}
|
|
688 |
}
|
|
689 |
||
690 |
/*
|
|
691 |
* 'math_state_restore()' saves the current math information in the
|
|
692 |
* old math state array, and gets the new ones from the current task
|
|
693 |
*
|
|
694 |
* Careful.. There are problems with IBM-designed IRQ13 behaviour.
|
|
695 |
* Don't touch unless you *really* know how it works.
|
|
696 |
*
|
|
3
by Luke Yelavich, Andy Whitcroft, Chase Douglas, Eugeni Dodonov, Ingo Molnar, Johannes Berg, John Johansen, Kees Cook, Leann Ogasawara, Robert Hooker, Seth Heasley, Tim Gardner, Luke Yelavich, Upstream Kernel Changes
[ Andy Whitcroft ] |
697 |
* Must be called with kernel preemption disabled (eg with local
|
698 |
* local interrupts as in the case of do_device_not_available).
|
|
1
by Alessio Igor Bogani
Import upstream version 3.2.0 |
699 |
*/
|
3
by Luke Yelavich, Andy Whitcroft, Chase Douglas, Eugeni Dodonov, Ingo Molnar, Johannes Berg, John Johansen, Kees Cook, Leann Ogasawara, Robert Hooker, Seth Heasley, Tim Gardner, Luke Yelavich, Upstream Kernel Changes
[ Andy Whitcroft ] |
700 |
void math_state_restore(void) |
1
by Alessio Igor Bogani
Import upstream version 3.2.0 |
701 |
{
|
3
by Luke Yelavich, Andy Whitcroft, Chase Douglas, Eugeni Dodonov, Ingo Molnar, Johannes Berg, John Johansen, Kees Cook, Leann Ogasawara, Robert Hooker, Seth Heasley, Tim Gardner, Luke Yelavich, Upstream Kernel Changes
[ Andy Whitcroft ] |
702 |
struct task_struct *tsk = current; |
1
by Alessio Igor Bogani
Import upstream version 3.2.0 |
703 |
|
704 |
if (!tsk_used_math(tsk)) { |
|
705 |
local_irq_enable(); |
|
706 |
/*
|
|
707 |
* does a slab alloc which can sleep
|
|
708 |
*/
|
|
709 |
if (init_fpu(tsk)) { |
|
710 |
/*
|
|
711 |
* ran out of memory!
|
|
712 |
*/
|
|
713 |
do_group_exit(SIGKILL); |
|
714 |
return; |
|
715 |
}
|
|
716 |
local_irq_disable(); |
|
717 |
}
|
|
718 |
||
3
by Luke Yelavich, Andy Whitcroft, Chase Douglas, Eugeni Dodonov, Ingo Molnar, Johannes Berg, John Johansen, Kees Cook, Leann Ogasawara, Robert Hooker, Seth Heasley, Tim Gardner, Luke Yelavich, Upstream Kernel Changes
[ Andy Whitcroft ] |
719 |
__thread_fpu_begin(tsk); |
720 |
__math_state_restore(tsk); |
|
1
by Alessio Igor Bogani
Import upstream version 3.2.0 |
721 |
|
3
by Luke Yelavich, Andy Whitcroft, Chase Douglas, Eugeni Dodonov, Ingo Molnar, Johannes Berg, John Johansen, Kees Cook, Leann Ogasawara, Robert Hooker, Seth Heasley, Tim Gardner, Luke Yelavich, Upstream Kernel Changes
[ Andy Whitcroft ] |
722 |
tsk->fpu_counter++; |
1
by Alessio Igor Bogani
Import upstream version 3.2.0 |
723 |
}
|
724 |
EXPORT_SYMBOL_GPL(math_state_restore); |
|
725 |
||
726 |
dotraplinkage void __kprobes |
|
727 |
do_device_not_available(struct pt_regs *regs, long error_code) |
|
728 |
{
|
|
729 |
#ifdef CONFIG_MATH_EMULATION
|
|
730 |
if (read_cr0() & X86_CR0_EM) { |
|
731 |
struct math_emu_info info = { }; |
|
732 |
||
733 |
conditional_sti(regs); |
|
734 |
||
735 |
info.regs = regs; |
|
736 |
math_emulate(&info); |
|
737 |
return; |
|
738 |
}
|
|
739 |
#endif
|
|
740 |
math_state_restore(); /* interrupts still off */ |
|
741 |
#ifdef CONFIG_X86_32
|
|
742 |
conditional_sti(regs); |
|
743 |
#endif
|
|
744 |
}
|
|
745 |
||
746 |
#ifdef CONFIG_X86_32
|
|
2
by Alessio Igor Bogani
Add new lowlatency kernel flavour |
747 |
/*
|
748 |
* The fixup code for errors in iret jumps to here (iret_exc). It loses
|
|
749 |
* the original trap number and erorr code. The bogus trap 32 and error
|
|
750 |
* code 0 are what the vanilla kernel delivers via:
|
|
751 |
* DO_ERROR_INFO(32, SIGSEGV, "iret exception", iret_error, ILL_BADSTK, 0, 1)
|
|
752 |
*
|
|
753 |
* NOTE: Because of the final "1" in the macro we need to enable interrupts.
|
|
754 |
*
|
|
755 |
* In case of a general protection fault in the iret instruction, we
|
|
756 |
* need to check for a lazy CS update for exec-shield.
|
|
757 |
*/
|
|
1
by Alessio Igor Bogani
Import upstream version 3.2.0 |
758 |
dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code) |
759 |
{
|
|
2
by Alessio Igor Bogani
Add new lowlatency kernel flavour |
760 |
int ok; |
761 |
int cpu; |
|
762 |
||
1
by Alessio Igor Bogani
Import upstream version 3.2.0 |
763 |
local_irq_enable(); |
764 |
||
2
by Alessio Igor Bogani
Add new lowlatency kernel flavour |
765 |
cpu = get_cpu(); |
766 |
ok = check_lazy_exec_limit(cpu, regs, error_code); |
|
767 |
put_cpu(); |
|
768 |
||
769 |
if (!ok && notify_die(DIE_TRAP, "iret exception", regs, |
|
770 |
error_code, 32, SIGSEGV) != NOTIFY_STOP) { |
|
771 |
siginfo_t info; |
|
772 |
info.si_signo = SIGSEGV; |
|
773 |
info.si_errno = 0; |
|
774 |
info.si_code = ILL_BADSTK; |
|
775 |
info.si_addr = 0; |
|
776 |
do_trap(32, SIGSEGV, "iret exception", regs, error_code, &info); |
|
777 |
}
|
|
1
by Alessio Igor Bogani
Import upstream version 3.2.0 |
778 |
}
|
779 |
#endif
|
|
780 |
||
781 |
/* Set of traps needed for early debugging. */
|
|
782 |
void __init early_trap_init(void) |
|
783 |
{
|
|
784 |
set_intr_gate_ist(1, &debug, DEBUG_STACK); |
|
785 |
/* int3 can be called from all */
|
|
786 |
set_system_intr_gate_ist(3, &int3, DEBUG_STACK); |
|
787 |
set_intr_gate(14, &page_fault); |
|
788 |
load_idt(&idt_descr); |
|
789 |
}
|
|
790 |
||
791 |
void __init trap_init(void) |
|
792 |
{
|
|
793 |
int i; |
|
794 |
||
795 |
#ifdef CONFIG_EISA
|
|
796 |
void __iomem *p = early_ioremap(0x0FFFD9, 4); |
|
797 |
||
798 |
if (readl(p) == 'E' + ('I'<<8) + ('S'<<16) + ('A'<<24)) |
|
799 |
EISA_bus = 1; |
|
800 |
early_iounmap(p, 4); |
|
801 |
#endif
|
|
802 |
||
803 |
set_intr_gate(0, ÷_error); |
|
804 |
set_intr_gate_ist(2, &nmi, NMI_STACK); |
|
805 |
/* int4 can be called from all */
|
|
806 |
set_system_intr_gate(4, &overflow); |
|
807 |
set_intr_gate(5, &bounds); |
|
808 |
set_intr_gate(6, &invalid_op); |
|
809 |
set_intr_gate(7, &device_not_available); |
|
810 |
#ifdef CONFIG_X86_32
|
|
811 |
set_task_gate(8, GDT_ENTRY_DOUBLEFAULT_TSS); |
|
812 |
#else
|
|
813 |
set_intr_gate_ist(8, &double_fault, DOUBLEFAULT_STACK); |
|
814 |
#endif
|
|
815 |
set_intr_gate(9, &coprocessor_segment_overrun); |
|
816 |
set_intr_gate(10, &invalid_TSS); |
|
817 |
set_intr_gate(11, &segment_not_present); |
|
818 |
set_intr_gate_ist(12, &stack_segment, STACKFAULT_STACK); |
|
819 |
set_intr_gate(13, &general_protection); |
|
820 |
set_intr_gate(15, &spurious_interrupt_bug); |
|
821 |
set_intr_gate(16, &coprocessor_error); |
|
822 |
set_intr_gate(17, &alignment_check); |
|
823 |
#ifdef CONFIG_X86_MCE
|
|
824 |
set_intr_gate_ist(18, &machine_check, MCE_STACK); |
|
825 |
#endif
|
|
826 |
set_intr_gate(19, &simd_coprocessor_error); |
|
827 |
||
828 |
/* Reserve all the builtin and the syscall vector: */
|
|
829 |
for (i = 0; i < FIRST_EXTERNAL_VECTOR; i++) |
|
830 |
set_bit(i, used_vectors); |
|
831 |
||
832 |
#ifdef CONFIG_IA32_EMULATION
|
|
833 |
set_system_intr_gate(IA32_SYSCALL_VECTOR, ia32_syscall); |
|
834 |
set_bit(IA32_SYSCALL_VECTOR, used_vectors); |
|
835 |
#endif
|
|
836 |
||
837 |
#ifdef CONFIG_X86_32
|
|
838 |
set_system_trap_gate(SYSCALL_VECTOR, &system_call); |
|
839 |
set_bit(SYSCALL_VECTOR, used_vectors); |
|
840 |
#endif
|
|
841 |
||
842 |
/*
|
|
843 |
* Should be a barrier for any external CPU state:
|
|
844 |
*/
|
|
845 |
cpu_init(); |
|
846 |
||
847 |
x86_init.irqs.trap_init(); |
|
848 |
}
|