~ubuntu-branches/ubuntu/lucid/xenomai/lucid
» Revision
9
: ksrc/arch/x86/patches/adeos-ipipe-2.6.28.9-x86-2.2-07.patch
« back to all changes in this revision
Viewing changes to ksrc/arch/x86/patches/adeos-ipipe-2.6.28.9-x86-2.2-07.patch
- browse files at revision 9
- compare with another revision
- download diff
- download tarball
- view history from revision 9
- Committer: Bazaar Package Importer
- Author(s): Andres Rodriguez
- Date: 2009-06-24 22:17:01 UTC
- mfrom: (3.1.2 squeeze)
- Revision ID: james.westby@ubuntu.com-20090624221701-mwnah8aj90zmp6uj
Tags: 2.4.8-2ubuntu1
* Merge from debian unstable (LP: #391918), remaining changes:
- Add lpia to supported architectures.
- debian/rules: Create file for debhelper to pick up, use debhelper to
install it.
- debian/libxenomai1.dirs: Do not create directory.
- debian/libxenomai1.preinst: Remove symlink on upgrade, remove old udev.
rule unless modified in which case move to new name.
- debian/libxenomai1.postinst: Do not create symlink.
- debian/libxenomai1.postrm: No symlink to remove.
- Bump build-depend on debhelper to install udev rules into
/lib/udev/rules.d, add Breaks on udev to get correct version.
- Add lpia to supported architectures.
- debian/rules: Create file for debhelper to pick up, use debhelper to
install it.
- debian/libxenomai1.dirs: Do not create directory.
- debian/libxenomai1.preinst: Remove symlink on upgrade, remove old udev.
rule unless modified in which case move to new name.
- debian/libxenomai1.postinst: Do not create symlink.
- debian/libxenomai1.postrm: No symlink to remove.
- Bump build-depend on debhelper to install udev rules into
/lib/udev/rules.d, add Breaks on udev to get correct version.
- files added:
- config/libtool.m4
- files removed:
- ChangeLog
- files modified:
- CREDITS
added
removed
ksrc/arch/x86/patches/adeos-ipipe-2.6.28.9-x86-2.2-07.patch
1
diff --git a/Makefile b/Makefile
2
index 17bfe08..69cc998 100644
3
--- a/Makefile
4
+++ b/Makefile
5
@@ -525,6 +525,10 @@ endif
6
# Arch Makefiles may override this setting
7
KBUILD_CFLAGS += $(call cc-option, -fno-stack-protector)
8
9
+ifdef CONFIG_IPIPE_TRACE_MCOUNT
10
+KBUILD_CFLAGS += -pg
11
+endif
12
+
13
ifdef CONFIG_FRAME_POINTER
14
KBUILD_CFLAGS += -fno-omit-frame-pointer -fno-optimize-sibling-calls
15
else
16
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
17
index 2187c60..1179860 100644
18
--- a/arch/x86/Kconfig
19
+++ b/arch/x86/Kconfig
20
@@ -381,6 +381,7 @@ config SCHED_NO_NO_OMIT_FRAME_POINTER
21
22
menuconfig PARAVIRT_GUEST
23
bool "Paravirtualized guest support"
24
+ depends on !IPIPE
25
help
26
Say Y here to get to see options related to running Linux under
27
various hypervisors. This option alone does not add any kernel code.
28
@@ -426,7 +427,7 @@ source "arch/x86/lguest/Kconfig"
29
30
config PARAVIRT
31
bool "Enable paravirtualization code"
32
- depends on !X86_VOYAGER
33
+ depends on !(X86_VOYAGER && IPIPE)
34
help
35
This changes the kernel so it can modify itself when it is run
36
under a hypervisor, potentially improving performance significantly
37
@@ -623,6 +624,8 @@ config SCHED_MC
38
39
source "kernel/Kconfig.preempt"
40
41
+source "kernel/ipipe/Kconfig"
42
+
43
config X86_UP_APIC
44
bool "Local APIC support on uniprocessors"
45
depends on X86_32 && !SMP && !(X86_VOYAGER || X86_GENERICARCH)
46
diff --git a/arch/x86/Makefile b/arch/x86/Makefile
47
index d1a47ad..7769508 100644
48
--- a/arch/x86/Makefile
49
+++ b/arch/x86/Makefile
50
@@ -187,7 +187,7 @@ all: bzImage
51
zImage zlilo zdisk: KBUILD_IMAGE := $(boot)/zImage
52
53
zImage bzImage: vmlinux
54
- $(Q)$(MAKE) $(build)=$(boot) $(KBUILD_IMAGE)
55
+ $(Q)$(MAKE) $(build)=$(boot) mflags-y="$(mflags-y)" $(KBUILD_IMAGE)
56
$(Q)mkdir -p $(objtree)/arch/$(UTS_MACHINE)/boot
57
$(Q)ln -fsn ../../x86/boot/bzImage $(objtree)/arch/$(UTS_MACHINE)/boot/$@
58
59
diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile
60
index cd48c72..76c116d 100644
61
--- a/arch/x86/boot/Makefile
62
+++ b/arch/x86/boot/Makefile
63
@@ -115,7 +115,7 @@ $(obj)/setup.bin: $(obj)/setup.elf FORCE
64
$(call if_changed,objcopy)
65
66
$(obj)/compressed/vmlinux: FORCE
67
- $(Q)$(MAKE) $(build)=$(obj)/compressed $@
68
+ $(Q)$(MAKE) $(build)=$(obj)/compressed mflags-y="$(mflags-y)" $@
69
70
# Set this if you want to pass append arguments to the zdisk/fdimage/isoimage kernel
71
FDARGS =
72
diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
73
index 1771c80..8b0cef0 100644
74
--- a/arch/x86/boot/compressed/Makefile
75
+++ b/arch/x86/boot/compressed/Makefile
76
@@ -9,7 +9,7 @@ targets := vmlinux vmlinux.bin vmlinux.bin.gz head_$(BITS).o misc.o piggy.o
77
KBUILD_CFLAGS := -m$(BITS) -D__KERNEL__ $(LINUX_INCLUDE) -O2
78
KBUILD_CFLAGS += -fno-strict-aliasing -fPIC
79
cflags-$(CONFIG_X86_64) := -mcmodel=small
80
-KBUILD_CFLAGS += $(cflags-y)
81
+KBUILD_CFLAGS += $(cflags-y) $(mflags-y)
82
KBUILD_CFLAGS += $(call cc-option,-ffreestanding)
83
KBUILD_CFLAGS += $(call cc-option,-fno-stack-protector)
84
85
diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h
86
index 3b1510b..710ba4c 100644
87
--- a/arch/x86/include/asm/apic.h
88
+++ b/arch/x86/include/asm/apic.h
89
@@ -143,7 +143,13 @@ static inline void ack_x2APIC_irq(void)
90
#endif
91
92
93
+#ifdef CONFIG_IPIPE
94
+#define ack_APIC_irq() do { } while(0)
95
+static inline void __ack_APIC_irq(void)
96
+#else /* !CONFIG_IPIPE */
97
+#define __ack_APIC_irq() ack_APIC_irq()
98
static inline void ack_APIC_irq(void)
99
+#endif /* CONFIG_IPIPE */
100
{
101
/*
102
* ack_APIC_irq() actually gets compiled as a single instruction
103
diff --git a/arch/x86/include/asm/apicdef.h b/arch/x86/include/asm/apicdef.h
104
index 63134e3..fd9eed6 100644
105
--- a/arch/x86/include/asm/apicdef.h
106
+++ b/arch/x86/include/asm/apicdef.h
107
@@ -139,6 +139,7 @@
108
# define MAX_LOCAL_APIC 32768
109
#endif
110
111
+#ifndef __ASSEMBLY__
112
/*
113
* All x86-64 systems are xAPIC compatible.
114
* In the following, "apicid" is a physical APIC ID.
115
@@ -414,4 +415,7 @@ struct local_apic {
116
#else
117
#define BAD_APICID 0xFFFFu
118
#endif
119
+
120
+#endif /* !__ASSEMBLY__ */
121
+
122
#endif /* _ASM_X86_APICDEF_H */
123
diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h
124
index b97aecb..f3a15ec 100644
125
--- a/arch/x86/include/asm/hw_irq.h
126
+++ b/arch/x86/include/asm/hw_irq.h
127
@@ -33,6 +33,13 @@ extern void error_interrupt(void);
128
extern void spurious_interrupt(void);
129
extern void thermal_interrupt(void);
130
extern void reschedule_interrupt(void);
131
+#ifdef CONFIG_IPIPE
132
+void ipipe_ipi0(void);
133
+void ipipe_ipi1(void);
134
+void ipipe_ipi2(void);
135
+void ipipe_ipi3(void);
136
+void ipipe_ipiX(void);
137
+#endif
138
139
extern void invalidate_interrupt(void);
140
extern void invalidate_interrupt0(void);
141
diff --git a/arch/x86/include/asm/i387.h b/arch/x86/include/asm/i387.h
142
index 48f0004..e636ef3 100644
143
--- a/arch/x86/include/asm/i387.h
144
+++ b/arch/x86/include/asm/i387.h
145
@@ -283,11 +283,14 @@ static inline void __clear_fpu(struct task_struct *tsk)
146
static inline void kernel_fpu_begin(void)
147
{
148
struct thread_info *me = current_thread_info();
149
+ unsigned long flags;
150
preempt_disable();
151
+ local_irq_save_hw_cond(flags);
152
if (me->status & TS_USEDFPU)
153
__save_init_fpu(me->task);
154
else
155
clts();
156
+ local_irq_restore_hw_cond(flags);
157
}
158
159
static inline void kernel_fpu_end(void)
160
diff --git a/arch/x86/include/asm/i8259.h b/arch/x86/include/asm/i8259.h
161
index 58d7091..ac8bd15 100644
162
--- a/arch/x86/include/asm/i8259.h
163
+++ b/arch/x86/include/asm/i8259.h
164
@@ -24,7 +24,7 @@ extern unsigned int cached_irq_mask;
165
#define SLAVE_ICW4_DEFAULT 0x01
166
#define PIC_ICW4_AEOI 2
167
168
-extern spinlock_t i8259A_lock;
169
+extern ipipe_spinlock_t i8259A_lock;
170
171
extern void init_8259A(int auto_eoi);
172
extern void enable_8259A_irq(unsigned int irq);
173
diff --git a/arch/x86/include/asm/ipi.h b/arch/x86/include/asm/ipi.h
174
index f89dffb..a8779c0 100644
175
--- a/arch/x86/include/asm/ipi.h
176
+++ b/arch/x86/include/asm/ipi.h
177
@@ -66,6 +66,9 @@ static inline void __send_IPI_shortcut(unsigned int shortcut, int vector,
178
* to the APIC.
179
*/
180
unsigned int cfg;
181
+ unsigned long flags;
182
+
183
+ local_irq_save_hw(flags);
184
185
/*
186
* Wait for idle.
187
@@ -81,6 +84,8 @@ static inline void __send_IPI_shortcut(unsigned int shortcut, int vector,
188
* Send the IPI. The write to APIC_ICR fires this off.
189
*/
190
native_apic_mem_write(APIC_ICR, cfg);
191
+
192
+ local_irq_restore_hw(flags);
193
}
194
195
/*
196
@@ -90,8 +95,9 @@ static inline void __send_IPI_shortcut(unsigned int shortcut, int vector,
197
static inline void __send_IPI_dest_field(unsigned int mask, int vector,
198
unsigned int dest)
199
{
200
- unsigned long cfg;
201
+ unsigned long cfg, flags;
202
203
+ local_irq_save_hw(flags);
204
/*
205
* Wait for idle.
206
*/
207
@@ -115,11 +121,12 @@ static inline void __send_IPI_dest_field(unsigned int mask, int vector,
208
* Send the IPI. The write to APIC_ICR fires this off.
209
*/
210
native_apic_mem_write(APIC_ICR, cfg);
211
+
212
+ local_irq_restore_hw(flags);
213
}
214
215
static inline void send_IPI_mask_sequence(cpumask_t mask, int vector)
216
{
217
- unsigned long flags;
218
unsigned long query_cpu;
219
220
/*
221
@@ -127,12 +134,10 @@ static inline void send_IPI_mask_sequence(cpumask_t mask, int vector)
222
* to an arbitrary mask, so I do a unicast to each CPU instead.
223
* - mbligh
224
*/
225
- local_irq_save(flags);
226
for_each_cpu_mask_nr(query_cpu, mask) {
227
__send_IPI_dest_field(per_cpu(x86_cpu_to_apicid, query_cpu),
228
vector, APIC_DEST_PHYSICAL);
229
}
230
- local_irq_restore(flags);
231
}
232
233
#endif /* _ASM_X86_IPI_H */
234
diff --git a/arch/x86/include/asm/ipipe.h b/arch/x86/include/asm/ipipe.h
235
new file mode 100644
236
index 0000000..4bec994
237
--- /dev/null
238
+++ b/arch/x86/include/asm/ipipe.h
239
@@ -0,0 +1,156 @@
240
+/* -*- linux-c -*-
241
+ * arch/x86/include/asm/ipipe.h
242
+ *
243
+ * Copyright (C) 2007 Philippe Gerum.
244
+ *
245
+ * This program is free software; you can redistribute it and/or modify
246
+ * it under the terms of the GNU General Public License as published by
247
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
248
+ * USA; either version 2 of the License, or (at your option) any later
249
+ * version.
250
+ *
251
+ * This program is distributed in the hope that it will be useful,
252
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
253
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
254
+ * GNU General Public License for more details.
255
+ *
256
+ * You should have received a copy of the GNU General Public License
257
+ * along with this program; if not, write to the Free Software
258
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
259
+ */
260
+
261
+#ifndef __X86_IPIPE_H
262
+#define __X86_IPIPE_H
263
+
264
+#ifdef CONFIG_IPIPE
265
+
266
+#ifndef IPIPE_ARCH_STRING
267
+#define IPIPE_ARCH_STRING "2.2-07"
268
+#define IPIPE_MAJOR_NUMBER 2
269
+#define IPIPE_MINOR_NUMBER 2
270
+#define IPIPE_PATCH_NUMBER 7
271
+#endif
272
+
273
+DECLARE_PER_CPU(struct pt_regs, __ipipe_tick_regs);
274
+
275
+static inline unsigned __ipipe_get_irq_vector(int irq)
276
+{
277
+#ifdef CONFIG_X86_IO_APIC
278
+ unsigned __ipipe_get_ioapic_irq_vector(int irq);
279
+ return __ipipe_get_ioapic_irq_vector(irq);
280
+#elif defined(CONFIG_X86_LOCAL_APIC)
281
+ return irq >= IPIPE_FIRST_APIC_IRQ && irq < IPIPE_NR_XIRQS ?
282
+ ipipe_apic_irq_vector(irq) : irq + IRQ0_VECTOR;
283
+#else
284
+ return irq + IRQ0_VECTOR;
285
+#endif
286
+}
287
+
288
+#ifdef CONFIG_X86_32
289
+# include "ipipe_32.h"
290
+#else
291
+# include "ipipe_64.h"
292
+#endif
293
+
294
+/*
295
+ * The logical processor id and the current Linux task are read from the PDA,
296
+ * so this is always safe, regardless of the underlying stack.
297
+ */
298
+#define ipipe_processor_id() raw_smp_processor_id()
299
+#define ipipe_safe_current() current
300
+
301
+#define prepare_arch_switch(next) \
302
+do { \
303
+ ipipe_schedule_notify(current, next); \
304
+ local_irq_disable_hw(); \
305
+} while(0)
306
+
307
+#define task_hijacked(p) \
308
+ ({ int x = !ipipe_root_domain_p; \
309
+ __clear_bit(IPIPE_SYNC_FLAG, &ipipe_root_cpudom_var(status)); \
310
+ local_irq_enable_hw(); x; })
311
+
312
+struct ipipe_domain;
313
+
314
+struct ipipe_sysinfo {
315
+
316
+ int ncpus; /* Number of CPUs on board */
317
+ u64 cpufreq; /* CPU frequency (in Hz) */
318
+
319
+ /* Arch-dependent block */
320
+
321
+ struct {
322
+ unsigned tmirq; /* Timer tick IRQ */
323
+ u64 tmfreq; /* Timer frequency */
324
+ } archdep;
325
+};
326
+
327
+/* Private interface -- Internal use only */
328
+
329
+#define __ipipe_check_platform() do { } while(0)
330
+#define __ipipe_init_platform() do { } while(0)
331
+#define __ipipe_enable_irq(irq) irq_to_desc(irq)->chip->enable(irq)
332
+#define __ipipe_disable_irq(irq) irq_to_desc(irq)->chip->disable(irq)
333
+
334
+#ifdef CONFIG_SMP
335
+void __ipipe_hook_critical_ipi(struct ipipe_domain *ipd);
336
+#else
337
+#define __ipipe_hook_critical_ipi(ipd) do { } while(0)
338
+#endif
339
+
340
+#define __ipipe_disable_irqdesc(ipd, irq) do { } while(0)
341
+
342
+void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq);
343
+
344
+void __ipipe_enable_pipeline(void);
345
+
346
+void __ipipe_do_critical_sync(unsigned irq, void *cookie);
347
+
348
+void __ipipe_serial_debug(const char *fmt, ...);
349
+
350
+extern int __ipipe_tick_irq;
351
+
352
+#ifdef CONFIG_X86_LOCAL_APIC
353
+#define ipipe_update_tick_evtdev(evtdev) \
354
+ do { \
355
+ if (strcmp((evtdev)->name, "lapic") == 0) \
356
+ __ipipe_tick_irq = \
357
+ ipipe_apic_vector_irq(LOCAL_TIMER_VECTOR); \
358
+ else \
359
+ __ipipe_tick_irq = 0; \
360
+ } while (0)
361
+#else
362
+#define ipipe_update_tick_evtdev(evtdev) \
363
+ __ipipe_tick_irq = 0
364
+#endif
365
+
366
+int __ipipe_check_lapic(void);
367
+
368
+int __ipipe_check_tickdev(const char *devname);
369
+
370
+#define __ipipe_syscall_watched_p(p, sc) \
371
+ (((p)->flags & PF_EVNOTIFY) || (unsigned long)sc >= NR_syscalls)
372
+
373
+#define __ipipe_root_tick_p(regs) ((regs)->flags & X86_EFLAGS_IF)
374
+
375
+#ifdef CONFIG_SMP
376
+#define __ipipe_move_root_irq(irq) \
377
+ do { \
378
+ if (irq < NR_IRQS) { \
379
+ struct irq_chip *chip = irq_to_desc(irq)->chip; \
380
+ if (chip->move) \
381
+ chip->move(irq); \
382
+ } \
383
+ } while (0)
384
+#else /* !CONFIG_SMP */
385
+#define __ipipe_move_root_irq(irq) do { } while (0)
386
+#endif /* !CONFIG_SMP */
387
+
388
+#else /* !CONFIG_IPIPE */
389
+
390
+#define ipipe_update_tick_evtdev(evtdev) do { } while (0)
391
+#define task_hijacked(p) 0
392
+
393
+#endif /* CONFIG_IPIPE */
394
+
395
+#endif /* !__X86_IPIPE_H */
396
diff --git a/arch/x86/include/asm/ipipe_32.h b/arch/x86/include/asm/ipipe_32.h
397
new file mode 100644
398
index 0000000..235c846
399
--- /dev/null
400
+++ b/arch/x86/include/asm/ipipe_32.h
401
@@ -0,0 +1,151 @@
402
+/* -*- linux-c -*-
403
+ * arch/x86/include/asm/ipipe_32.h
404
+ *
405
+ * Copyright (C) 2002-2005 Philippe Gerum.
406
+ *
407
+ * This program is free software; you can redistribute it and/or modify
408
+ * it under the terms of the GNU General Public License as published by
409
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
410
+ * USA; either version 2 of the License, or (at your option) any later
411
+ * version.
412
+ *
413
+ * This program is distributed in the hope that it will be useful,
414
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
415
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
416
+ * GNU General Public License for more details.
417
+ *
418
+ * You should have received a copy of the GNU General Public License
419
+ * along with this program; if not, write to the Free Software
420
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
421
+ */
422
+
423
+#ifndef __X86_IPIPE_32_H
424
+#define __X86_IPIPE_32_H
425
+
426
+#include <linux/cpumask.h>
427
+#include <linux/list.h>
428
+#include <linux/threads.h>
429
+#include <linux/ipipe_percpu.h>
430
+#include <asm/ptrace.h>
431
+
432
+#define ipipe_read_tsc(t) __asm__ __volatile__("rdtsc" : "=A" (t))
433
+#define ipipe_cpu_freq() ({ unsigned long long __freq = cpu_has_tsc?(1000LL * cpu_khz):CLOCK_TICK_RATE; __freq; })
434
+
435
+#define ipipe_tsc2ns(t) \
436
+({ \
437
+ unsigned long long delta = (t)*1000; \
438
+ do_div(delta, cpu_khz/1000+1); \
439
+ (unsigned long)delta; \
440
+})
441
+
442
+#define ipipe_tsc2us(t) \
443
+({ \
444
+ unsigned long long delta = (t); \
445
+ do_div(delta, cpu_khz/1000+1); \
446
+ (unsigned long)delta; \
447
+})
448
+
449
+/* Private interface -- Internal use only */
450
+
451
+int __ipipe_handle_irq(struct pt_regs *regs);
452
+
453
+static inline unsigned long __ipipe_ffnz(unsigned long ul)
454
+{
455
+ __asm__("bsrl %1, %0":"=r"(ul)
456
+ : "r"(ul));
457
+ return ul;
458
+}
459
+
460
+struct irq_desc;
461
+
462
+void __ipipe_ack_edge_irq(unsigned irq, struct irq_desc *desc);
463
+
464
+void __ipipe_end_edge_irq(unsigned irq, struct irq_desc *desc);
465
+
466
+static inline void __ipipe_call_root_xirq_handler(unsigned irq,
467
+ ipipe_irq_handler_t handler)
468
+{
469
+ struct pt_regs *regs = &__raw_get_cpu_var(__ipipe_tick_regs);
470
+
471
+ regs->orig_ax = ~__ipipe_get_irq_vector(irq);
472
+
473
+ __asm__ __volatile__("pushfl\n\t"
474
+ "pushl %%cs\n\t"
475
+ "pushl $__xirq_end\n\t"
476
+ "pushl %%eax\n\t"
477
+ "pushl %%fs\n\t"
478
+ "pushl %%es\n\t"
479
+ "pushl %%ds\n\t"
480
+ "pushl %%eax\n\t"
481
+ "pushl %%ebp\n\t"
482
+ "pushl %%edi\n\t"
483
+ "pushl %%esi\n\t"
484
+ "pushl %%edx\n\t"
485
+ "pushl %%ecx\n\t"
486
+ "pushl %%ebx\n\t"
487
+ "movl %2,%%eax\n\t"
488
+ "call *%1\n\t"
489
+ "jmp ret_from_intr\n\t"
490
+ "__xirq_end: cli\n"
491
+ : /* no output */
492
+ : "a" (~irq), "r" (handler), "rm" (regs));
493
+}
494
+
495
+void irq_enter(void);
496
+void irq_exit(void);
497
+
498
+static inline void __ipipe_call_root_virq_handler(unsigned irq,
499
+ ipipe_irq_handler_t handler,
500
+ void *cookie)
501
+{
502
+ irq_enter();
503
+ __asm__ __volatile__("pushfl\n\t"
504
+ "pushl %%cs\n\t"
505
+ "pushl $__virq_end\n\t"
506
+ "pushl $-1\n\t"
507
+ "pushl %%fs\n\t"
508
+ "pushl %%es\n\t"
509
+ "pushl %%ds\n\t"
510
+ "pushl %%eax\n\t"
511
+ "pushl %%ebp\n\t"
512
+ "pushl %%edi\n\t"
513
+ "pushl %%esi\n\t"
514
+ "pushl %%edx\n\t"
515
+ "pushl %%ecx\n\t"
516
+ "pushl %%ebx\n\t"
517
+ "pushl %2\n\t"
518
+ "pushl %%eax\n\t"
519
+ "call *%1\n\t"
520
+ "addl $8,%%esp\n"
521
+ : /* no output */
522
+ : "a" (irq), "r" (handler), "d" (cookie));
523
+ irq_exit();
524
+ __asm__ __volatile__("jmp ret_from_intr\n\t"
525
+ "__virq_end: cli\n"
526
+ : /* no output */
527
+ : /* no input */);
528
+}
529
+
530
+/*
531
+ * When running handlers, enable hw interrupts for all domains but the
532
+ * one heading the pipeline, so that IRQs can never be significantly
533
+ * deferred for the latter.
534
+ */
535
+#define __ipipe_run_isr(ipd, irq) \
536
+do { \
537
+ local_irq_enable_nohead(ipd); \
538
+ if (ipd == ipipe_root_domain) { \
539
+ if (likely(!ipipe_virtual_irq_p(irq))) { \
540
+ __ipipe_move_root_irq(irq); \
541
+ __ipipe_call_root_xirq_handler(irq, ipd->irqs[irq].handler); \
542
+ } else \
543
+ __ipipe_call_root_virq_handler(irq, ipd->irqs[irq].handler, ipd->irqs[irq].cookie); \
544
+ } else { \
545
+ __clear_bit(IPIPE_SYNC_FLAG, &ipipe_cpudom_var(ipd, status)); \
546
+ ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie); \
547
+ __set_bit(IPIPE_SYNC_FLAG, &ipipe_cpudom_var(ipd, status)); \
548
+ } \
549
+ local_irq_disable_nohead(ipd); \
550
+} while(0)
551
+
552
+#endif /* !__X86_IPIPE_32_H */
553
diff --git a/arch/x86/include/asm/ipipe_64.h b/arch/x86/include/asm/ipipe_64.h
554
new file mode 100644
555
index 0000000..4e719d4
556
--- /dev/null
557
+++ b/arch/x86/include/asm/ipipe_64.h
558
@@ -0,0 +1,159 @@
559
+/* -*- linux-c -*-
560
+ * arch/x86/include/asm/ipipe_64.h
561
+ *
562
+ * Copyright (C) 2007 Philippe Gerum.
563
+ *
564
+ * This program is free software; you can redistribute it and/or modify
565
+ * it under the terms of the GNU General Public License as published by
566
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
567
+ * USA; either version 2 of the License, or (at your option) any later
568
+ * version.
569
+ *
570
+ * This program is distributed in the hope that it will be useful,
571
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
572
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
573
+ * GNU General Public License for more details.
574
+ *
575
+ * You should have received a copy of the GNU General Public License
576
+ * along with this program; if not, write to the Free Software
577
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
578
+ */
579
+
580
+#ifndef __X86_IPIPE_64_H
581
+#define __X86_IPIPE_64_H
582
+
583
+#include <asm/ptrace.h>
584
+#include <asm/irq.h>
585
+#include <linux/cpumask.h>
586
+#include <linux/list.h>
587
+#include <linux/ipipe_percpu.h>
588
+#ifdef CONFIG_SMP
589
+#include <asm/mpspec.h>
590
+#include <linux/thread_info.h>
591
+#endif
592
+
593
+#define ipipe_read_tsc(t) do { \
594
+ unsigned int __a,__d; \
595
+ asm volatile("rdtsc" : "=a" (__a), "=d" (__d)); \
596
+ (t) = ((unsigned long)__a) | (((unsigned long)__d)<<32); \
597
+} while(0)
598
+
599
+extern unsigned cpu_khz;
600
+#define ipipe_cpu_freq() ({ unsigned long __freq = (1000UL * cpu_khz); __freq; })
601
+#define ipipe_tsc2ns(t) (((t) * 1000UL) / (ipipe_cpu_freq() / 1000000UL))
602
+#define ipipe_tsc2us(t) ((t) / (ipipe_cpu_freq() / 1000000UL))
603
+
604
+/* Private interface -- Internal use only */
605
+
606
+int __ipipe_handle_irq(struct pt_regs *regs);
607
+
608
+static inline unsigned long __ipipe_ffnz(unsigned long ul)
609
+{
610
+ __asm__("bsrq %1, %0":"=r"(ul)
611
+ : "rm"(ul));
612
+ return ul;
613
+}
614
+
615
+struct irq_desc;
616
+
617
+void __ipipe_ack_edge_irq(unsigned irq, struct irq_desc *desc);
618
+
619
+void __ipipe_end_edge_irq(unsigned irq, struct irq_desc *desc);
620
+
621
+static inline void __ipipe_call_root_xirq_handler(unsigned irq,
622
+ void (*handler)(unsigned, void *))
623
+{
624
+ struct pt_regs *regs = &__raw_get_cpu_var(__ipipe_tick_regs);
625
+
626
+ regs->orig_ax = ~__ipipe_get_irq_vector(irq);
627
+
628
+ __asm__ __volatile__("movq %%rsp, %%rax\n\t"
629
+ "pushq $0\n\t"
630
+ "pushq %%rax\n\t"
631
+ "pushfq\n\t"
632
+ "pushq %[kernel_cs]\n\t"
633
+ "pushq $__xirq_end\n\t"
634
+ "pushq %[vector]\n\t"
635
+ "subq $9*8,%%rsp\n\t"
636
+ "movq %%rdi,8*8(%%rsp)\n\t"
637
+ "movq %%rsi,7*8(%%rsp)\n\t"
638
+ "movq %%rdx,6*8(%%rsp)\n\t"
639
+ "movq %%rcx,5*8(%%rsp)\n\t"
640
+ "movq %%rax,4*8(%%rsp)\n\t"
641
+ "movq %%r8,3*8(%%rsp)\n\t"
642
+ "movq %%r9,2*8(%%rsp)\n\t"
643
+ "movq %%r10,1*8(%%rsp)\n\t"
644
+ "movq %%r11,(%%rsp)\n\t"
645
+ "call *%[handler]\n\t"
646
+ "jmp exit_intr\n\t"
647
+ "__xirq_end: cli\n"
648
+ : /* no output */
649
+ : [kernel_cs] "i" (__KERNEL_CS),
650
+ [vector] "rm" (regs->orig_ax),
651
+ [handler] "r" (handler), "D" (regs)
652
+ : "rax");
653
+}
654
+
655
+void irq_enter(void);
656
+void irq_exit(void);
657
+
658
+static inline void __ipipe_call_root_virq_handler(unsigned irq,
659
+ void (*handler)(unsigned, void *),
660
+ void *cookie)
661
+{
662
+ irq_enter();
663
+ __asm__ __volatile__("movq %%rsp, %%rax\n\t"
664
+ "pushq $0\n\t"
665
+ "pushq %%rax\n\t"
666
+ "pushfq\n\t"
667
+ "pushq %[kernel_cs]\n\t"
668
+ "pushq $__virq_end\n\t"
669
+ "pushq $-1\n\t"
670
+ "subq $9*8,%%rsp\n\t"
671
+ "movq %%rdi,8*8(%%rsp)\n\t"
672
+ "movq %%rsi,7*8(%%rsp)\n\t"
673
+ "movq %%rdx,6*8(%%rsp)\n\t"
674
+ "movq %%rcx,5*8(%%rsp)\n\t"
675
+ "movq %%rax,4*8(%%rsp)\n\t"
676
+ "movq %%r8,3*8(%%rsp)\n\t"
677
+ "movq %%r9,2*8(%%rsp)\n\t"
678
+ "movq %%r10,1*8(%%rsp)\n\t"
679
+ "movq %%r11,(%%rsp)\n\t"
680
+ "call *%[handler]\n\t"
681
+ : /* no output */
682
+ : [kernel_cs] "i" (__KERNEL_CS),
683
+ [handler] "r" (handler), "D" (irq), "S" (cookie)
684
+ : "rax");
685
+ irq_exit();
686
+ __asm__ __volatile__("jmp exit_intr\n\t"
687
+ "__virq_end: cli\n"
688
+ : /* no output */
689
+ : /* no input */);
690
+}
691
+
692
+/*
693
+ * When running handlers, enable hw interrupts for all domains but the
694
+ * one heading the pipeline, so that IRQs can never be significantly
695
+ * deferred for the latter.
696
+ */
697
+#define __ipipe_run_isr(ipd, irq) \
698
+ do { \
699
+ local_irq_enable_nohead(ipd); \
700
+ if (ipd == ipipe_root_domain) { \
701
+ if (likely(!ipipe_virtual_irq_p(irq))) { \
702
+ __ipipe_move_root_irq(irq); \
703
+ __ipipe_call_root_xirq_handler( \
704
+ irq, (ipd)->irqs[irq].handler); \
705
+ } else \
706
+ __ipipe_call_root_virq_handler( \
707
+ irq, (ipd)->irqs[irq].handler, \
708
+ (ipd)->irqs[irq].cookie); \
709
+ } else { \
710
+ __clear_bit(IPIPE_SYNC_FLAG, &ipipe_cpudom_var(ipd, status)); \
711
+ ipd->irqs[irq].handler(irq, ipd->irqs[irq].cookie); \
712
+ __set_bit(IPIPE_SYNC_FLAG, &ipipe_cpudom_var(ipd, status)); \
713
+ } \
714
+ local_irq_disable_nohead(ipd); \
715
+ } while(0)
716
+
717
+#endif /* !__X86_IPIPE_64_H */
718
diff --git a/arch/x86/include/asm/ipipe_base.h b/arch/x86/include/asm/ipipe_base.h
719
new file mode 100644
720
index 0000000..05338e4
721
--- /dev/null
722
+++ b/arch/x86/include/asm/ipipe_base.h
723
@@ -0,0 +1,97 @@
724
+/* -*- linux-c -*-
725
+ * arch/x86/include/asm/ipipe_base.h
726
+ *
727
+ * Copyright (C) 2007 Philippe Gerum.
728
+ *
729
+ * This program is free software; you can redistribute it and/or modify
730
+ * it under the terms of the GNU General Public License as published by
731
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
732
+ * USA; either version 2 of the License, or (at your option) any later
733
+ * version.
734
+ *
735
+ * This program is distributed in the hope that it will be useful,
736
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
737
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
738
+ * GNU General Public License for more details.
739
+ *
740
+ * You should have received a copy of the GNU General Public License
741
+ * along with this program; if not, write to the Free Software
742
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
743
+ */
744
+
745
+#ifndef __X86_IPIPE_BASE_H
746
+#define __X86_IPIPE_BASE_H
747
+
748
+#ifdef CONFIG_X86_32
749
+# include "ipipe_base_32.h"
750
+#else
751
+# include "ipipe_base_64.h"
752
+#endif
753
+
754
+#define ex_do_divide_error 0
755
+#define ex_do_debug 1
756
+/* NMI not pipelined. */
757
+#define ex_do_int3 3
758
+#define ex_do_overflow 4
759
+#define ex_do_bounds 5
760
+#define ex_do_invalid_op 6
761
+#define ex_do_device_not_available 7
762
+/* Double fault not pipelined. */
763
+#define ex_do_coprocessor_segment_overrun 9
764
+#define ex_do_invalid_TSS 10
765
+#define ex_do_segment_not_present 11
766
+#define ex_do_stack_segment 12
767
+#define ex_do_general_protection 13
768
+#define ex_do_page_fault 14
769
+#define ex_do_spurious_interrupt_bug 15
770
+#define ex_do_coprocessor_error 16
771
+#define ex_do_alignment_check 17
772
+#define ex_machine_check_vector 18
773
+#define ex_reserved ex_machine_check_vector
774
+#define ex_do_simd_coprocessor_error 19
775
+#define ex_do_iret_error 32
776
+
777
+#if !defined(__ASSEMBLY__) && !defined(CONFIG_SMP)
778
+
779
+#if __GNUC__ >= 4
780
+/* Alias to ipipe_root_cpudom_var(status) */
781
+extern unsigned long __ipipe_root_status;
782
+#else
783
+extern unsigned long *const __ipipe_root_status_addr;
784
+#define __ipipe_root_status (*__ipipe_root_status_addr)
785
+#endif
786
+
787
+static inline void __ipipe_stall_root(void)
788
+{
789
+ volatile unsigned long *p = &__ipipe_root_status;
790
+ __asm__ __volatile__("btsl $0,%0;"
791
+ :"+m" (*p) : : "memory");
792
+}
793
+
794
+static inline unsigned long __ipipe_test_and_stall_root(void)
795
+{
796
+ volatile unsigned long *p = &__ipipe_root_status;
797
+ int oldbit;
798
+
799
+ __asm__ __volatile__("btsl $0,%1;"
800
+ "sbbl %0,%0;"
801
+ :"=r" (oldbit), "+m" (*p)
802
+ : : "memory");
803
+ return oldbit;
804
+}
805
+
806
+static inline unsigned long __ipipe_test_root(void)
807
+{
808
+ volatile unsigned long *p = &__ipipe_root_status;
809
+ int oldbit;
810
+
811
+ __asm__ __volatile__("btl $0,%1;"
812
+ "sbbl %0,%0;"
813
+ :"=r" (oldbit)
814
+ :"m" (*p));
815
+ return oldbit;
816
+}
817
+
818
+#endif /* !__ASSEMBLY__ && !CONFIG_SMP */
819
+
820
+#endif /* !__X86_IPIPE_BASE_H */
821
diff --git a/arch/x86/include/asm/ipipe_base_32.h b/arch/x86/include/asm/ipipe_base_32.h
822
new file mode 100644
823
index 0000000..0709d01
824
--- /dev/null
825
+++ b/arch/x86/include/asm/ipipe_base_32.h
826
@@ -0,0 +1,122 @@
827
+/* -*- linux-c -*-
828
+ * arch/x86/include/asm/ipipe_base_32.h
829
+ *
830
+ * Copyright (C) 2007 Philippe Gerum.
831
+ *
832
+ * This program is free software; you can redistribute it and/or modify
833
+ * it under the terms of the GNU General Public License as published by
834
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
835
+ * USA; either version 2 of the License, or (at your option) any later
836
+ * version.
837
+ *
838
+ * This program is distributed in the hope that it will be useful,
839
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
840
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
841
+ * GNU General Public License for more details.
842
+ *
843
+ * You should have received a copy of the GNU General Public License
844
+ * along with this program; if not, write to the Free Software
845
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
846
+ */
847
+
848
+#ifndef __X86_IPIPE_BASE_32_H
849
+#define __X86_IPIPE_BASE_32_H
850
+
851
+#include <linux/threads.h>
852
+#include <asm/apicdef.h>
853
+#include <asm/irq_vectors.h>
854
+
855
+#ifdef CONFIG_X86_LOCAL_APIC
856
+/* System interrupts are mapped beyond the last defined external IRQ
857
+ * number. */
858
+#define IPIPE_FIRST_APIC_IRQ NR_IRQS
859
+#define IPIPE_NR_XIRQS (NR_IRQS + 256 - FIRST_SYSTEM_VECTOR)
860
+#define ipipe_apic_irq_vector(irq) ((irq) - IPIPE_FIRST_APIC_IRQ + FIRST_SYSTEM_VECTOR)
861
+#define ipipe_apic_vector_irq(vec) ((vec) - FIRST_SYSTEM_VECTOR + IPIPE_FIRST_APIC_IRQ)
862
+/* If the APIC is enabled, then we expose four service vectors in the
863
+ * APIC space which are freely available to domains. */
864
+#define IPIPE_SERVICE_VECTOR0 0xf5
865
+#define IPIPE_SERVICE_IPI0 ipipe_apic_vector_irq(IPIPE_SERVICE_VECTOR0)
866
+#define IPIPE_SERVICE_VECTOR1 0xf6
867
+#define IPIPE_SERVICE_IPI1 ipipe_apic_vector_irq(IPIPE_SERVICE_VECTOR1)
868
+#define IPIPE_SERVICE_VECTOR2 0xf7
869
+#define IPIPE_SERVICE_IPI2 ipipe_apic_vector_irq(IPIPE_SERVICE_VECTOR2)
870
+#define IPIPE_SERVICE_VECTOR3 0xf8
871
+#define IPIPE_SERVICE_IPI3 ipipe_apic_vector_irq(IPIPE_SERVICE_VECTOR3)
872
+#define IPIPE_CRITICAL_VECTOR 0xf9 /* SMP-only: used by ipipe_critical_enter/exit() */
873
+#define IPIPE_CRITICAL_IPI ipipe_apic_vector_irq(IPIPE_CRITICAL_VECTOR)
874
+#else /* !CONFIG_X86_LOCAL_APIC */
875
+#define IPIPE_NR_XIRQS NR_IRQS
876
+#endif /* !CONFIG_X86_LOCAL_APIC */
877
+
878
+#define IPIPE_IRQ_ISHIFT 5 /* 2^5 for 32bits arch. */
879
+
880
+/* IDT fault vectors */
881
+#define IPIPE_NR_FAULTS 33 /* 32 from IDT + iret_error */
882
+/* Pseudo-vectors used for kernel events */
883
+#define IPIPE_FIRST_EVENT IPIPE_NR_FAULTS
884
+#define IPIPE_EVENT_SYSCALL (IPIPE_FIRST_EVENT)
885
+#define IPIPE_EVENT_SCHEDULE (IPIPE_FIRST_EVENT + 1)
886
+#define IPIPE_EVENT_SIGWAKE (IPIPE_FIRST_EVENT + 2)
887
+#define IPIPE_EVENT_SETSCHED (IPIPE_FIRST_EVENT + 3)
888
+#define IPIPE_EVENT_INIT (IPIPE_FIRST_EVENT + 4)
889
+#define IPIPE_EVENT_EXIT (IPIPE_FIRST_EVENT + 5)
890
+#define IPIPE_EVENT_CLEANUP (IPIPE_FIRST_EVENT + 6)
891
+#define IPIPE_LAST_EVENT IPIPE_EVENT_CLEANUP
892
+#define IPIPE_NR_EVENTS (IPIPE_LAST_EVENT + 1)
893
+
894
+#ifndef __ASSEMBLY__
895
+
896
+#include <asm/alternative.h>
897
+
898
+#ifdef CONFIG_SMP
899
+
900
+#define GET_ROOT_STATUS_ADDR \
901
+ "pushfl; cli;" \
902
+ "movl %%fs:per_cpu__this_cpu_off, %%eax;" \
903
+ "lea per_cpu__ipipe_percpu_darray(%%eax), %%eax;"
904
+#define PUT_ROOT_STATUS_ADDR "popfl;"
905
+
906
+static inline void __ipipe_stall_root(void)
907
+{
908
+ __asm__ __volatile__(GET_ROOT_STATUS_ADDR
909
+ LOCK_PREFIX
910
+ "btsl $0,(%%eax);"
911
+ PUT_ROOT_STATUS_ADDR
912
+ : : : "eax", "memory");
913
+}
914
+
915
+static inline unsigned long __ipipe_test_and_stall_root(void)
916
+{
917
+ int oldbit;
918
+
919
+ __asm__ __volatile__(GET_ROOT_STATUS_ADDR
920
+ LOCK_PREFIX
921
+ "btsl $0,(%%eax);"
922
+ "sbbl %0,%0;"
923
+ PUT_ROOT_STATUS_ADDR
924
+ :"=r" (oldbit)
925
+ : : "eax", "memory");
926
+ return oldbit;
927
+}
928
+
929
+static inline unsigned long __ipipe_test_root(void)
930
+{
931
+ int oldbit;
932
+
933
+ __asm__ __volatile__(GET_ROOT_STATUS_ADDR
934
+ "btl $0,(%%eax);"
935
+ "sbbl %0,%0;"
936
+ PUT_ROOT_STATUS_ADDR
937
+ :"=r" (oldbit)
938
+ : : "eax");
939
+ return oldbit;
940
+}
941
+
942
+#endif /* CONFIG_SMP */
943
+
944
+void __ipipe_serial_debug(const char *fmt, ...);
945
+
946
+#endif /* !__ASSEMBLY__ */
947
+
948
+#endif /* !__X86_IPIPE_BASE_32_H */
949
diff --git a/arch/x86/include/asm/ipipe_base_64.h b/arch/x86/include/asm/ipipe_base_64.h
950
new file mode 100644
951
index 0000000..02aa014
952
--- /dev/null
953
+++ b/arch/x86/include/asm/ipipe_base_64.h
954
@@ -0,0 +1,122 @@
955
+/* -*- linux-c -*-
956
+ * arch/x86/include/asm/ipipe_base_64.h
957
+ *
958
+ * Copyright (C) 2007 Philippe Gerum.
959
+ *
960
+ * This program is free software; you can redistribute it and/or modify
961
+ * it under the terms of the GNU General Public License as published by
962
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
963
+ * USA; either version 2 of the License, or (at your option) any later
964
+ * version.
965
+ *
966
+ * This program is distributed in the hope that it will be useful,
967
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
968
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
969
+ * GNU General Public License for more details.
970
+ *
971
+ * You should have received a copy of the GNU General Public License
972
+ * along with this program; if not, write to the Free Software
973
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
974
+ */
975
+
976
+#ifndef __X86_IPIPE_BASE_64_H
977
+#define __X86_IPIPE_BASE_64_H
978
+
979
+#include <linux/threads.h>
980
+
981
+/* Local APIC is always compiled in on x86_64. Reserve 32 IRQs for
982
+ APIC interrupts, we don't want them to mess with the normally
983
+ assigned interrupts. */
984
+#define IPIPE_NR_XIRQS (NR_IRQS + 32)
985
+#define IPIPE_FIRST_APIC_IRQ NR_IRQS
986
+
987
+#define ipipe_apic_irq_vector(irq) ((irq) - IPIPE_FIRST_APIC_IRQ + FIRST_SYSTEM_VECTOR)
988
+#define ipipe_apic_vector_irq(vec) ((vec) - FIRST_SYSTEM_VECTOR + IPIPE_FIRST_APIC_IRQ)
989
+
990
+/* If the APIC is enabled, then we expose four service vectors in the
991
+ APIC space which are freely available to domains. */
992
+#define IPIPE_SERVICE_VECTOR0 (INVALIDATE_TLB_VECTOR_END + 1)
993
+#define IPIPE_SERVICE_IPI0 ipipe_apic_vector_irq(IPIPE_SERVICE_VECTOR0)
994
+#define IPIPE_SERVICE_VECTOR1 (INVALIDATE_TLB_VECTOR_END + 2)
995
+#define IPIPE_SERVICE_IPI1 ipipe_apic_vector_irq(IPIPE_SERVICE_VECTOR1)
996
+#define IPIPE_SERVICE_VECTOR2 (INVALIDATE_TLB_VECTOR_END + 3)
997
+#define IPIPE_SERVICE_IPI2 ipipe_apic_vector_irq(IPIPE_SERVICE_VECTOR2)
998
+#define IPIPE_SERVICE_VECTOR3 (INVALIDATE_TLB_VECTOR_END + 4)
999
+#define IPIPE_SERVICE_IPI3 ipipe_apic_vector_irq(IPIPE_SERVICE_VECTOR3)
1000
+#ifdef CONFIG_SMP
1001
+#define IPIPE_CRITICAL_VECTOR 0xf7 /* Used by ipipe_critical_enter/exit() */
1002
+#define IPIPE_CRITICAL_IPI ipipe_apic_vector_irq(IPIPE_CRITICAL_VECTOR)
1003
+#endif
1004
+
1005
+#define IPIPE_IRQ_ISHIFT 6 /* 2^6 for 64bits arch. */
1006
+
1007
+/* IDT fault vectors */
1008
+#define IPIPE_NR_FAULTS 32
1009
+/* Pseudo-vectors used for kernel events */
1010
+#define IPIPE_FIRST_EVENT IPIPE_NR_FAULTS
1011
+#define IPIPE_EVENT_SYSCALL (IPIPE_FIRST_EVENT)
1012
+#define IPIPE_EVENT_SCHEDULE (IPIPE_FIRST_EVENT + 1)
1013
+#define IPIPE_EVENT_SIGWAKE (IPIPE_FIRST_EVENT + 2)
1014
+#define IPIPE_EVENT_SETSCHED (IPIPE_FIRST_EVENT + 3)
1015
+#define IPIPE_EVENT_INIT (IPIPE_FIRST_EVENT + 4)
1016
+#define IPIPE_EVENT_EXIT (IPIPE_FIRST_EVENT + 5)
1017
+#define IPIPE_EVENT_CLEANUP (IPIPE_FIRST_EVENT + 6)
1018
+#define IPIPE_LAST_EVENT IPIPE_EVENT_CLEANUP
1019
+#define IPIPE_NR_EVENTS (IPIPE_LAST_EVENT + 1)
1020
+
1021
+#ifndef __ASSEMBLY__
1022
+
1023
+#include <asm/alternative.h>
1024
+
1025
+#ifdef CONFIG_SMP
1026
+/*
1027
+ * Ugly: depends on x8664_pda layout and actual implementation of
1028
+ * percpu accesses.
1029
+ */
1030
+#define GET_ROOT_STATUS_ADDR \
1031
+ "pushfq; cli;" \
1032
+ "movq %%gs:8, %%rax;" /* x8664_pda.data_offset */ \
1033
+ "addq $per_cpu__ipipe_percpu_darray, %%rax;"
1034
+#define PUT_ROOT_STATUS_ADDR "popfq;"
1035
+
1036
+static inline void __ipipe_stall_root(void)
1037
+{
1038
+ __asm__ __volatile__(GET_ROOT_STATUS_ADDR
1039
+ LOCK_PREFIX
1040
+ "btsl $0,(%%rax);"
1041
+ PUT_ROOT_STATUS_ADDR
1042
+ : : : "rax", "memory");
1043
+}
1044
+
1045
+static inline unsigned long __ipipe_test_and_stall_root(void)
1046
+{
1047
+ int oldbit;
1048
+
1049
+ __asm__ __volatile__(GET_ROOT_STATUS_ADDR
1050
+ LOCK_PREFIX
1051
+ "btsl $0,(%%rax);"
1052
+ "sbbl %0,%0;"
1053
+ PUT_ROOT_STATUS_ADDR
1054
+ :"=r" (oldbit)
1055
+ : : "rax", "memory");
1056
+ return oldbit;
1057
+}
1058
+
1059
+static inline unsigned long __ipipe_test_root(void)
1060
+{
1061
+ int oldbit;
1062
+
1063
+ __asm__ __volatile__(GET_ROOT_STATUS_ADDR
1064
+ "btl $0,(%%rax);"
1065
+ "sbbl %0,%0;"
1066
+ PUT_ROOT_STATUS_ADDR
1067
+ :"=r" (oldbit)
1068
+ : : "rax");
1069
+ return oldbit;
1070
+}
1071
+
1072
+#endif /* CONFIG_SMP */
1073
+
1074
+#endif /* !__ASSEMBLY__ */
1075
+
1076
+#endif /* !__X86_IPIPE_BASE_64_H */
1077
diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h
1078
index 0005adb..3792e5d 100644
1079
--- a/arch/x86/include/asm/irq_vectors.h
1080
+++ b/arch/x86/include/asm/irq_vectors.h
1081
@@ -72,13 +72,20 @@
1082
#define THERMAL_APIC_VECTOR 0xfa
1083
#define THRESHOLD_APIC_VECTOR 0xf9
1084
#define UV_BAU_MESSAGE 0xf8
1085
+#ifdef CONFIG_IPIPE
1086
+#define INVALIDATE_TLB_VECTOR_END 0xf2
1087
+#define INVALIDATE_TLB_VECTOR_START 0xf0 /* f0-f2 used for TLB flush */
1088
+#define NUM_INVALIDATE_TLB_VECTORS 3 /* f3-f7 used by I-pipe */
1089
+#else /* !CONFIG_IPIPE */
1090
#define INVALIDATE_TLB_VECTOR_END 0xf7
1091
#define INVALIDATE_TLB_VECTOR_START 0xf0 /* f0-f7 used for TLB flush */
1092
-
1093
#define NUM_INVALIDATE_TLB_VECTORS 8
1094
+#endif
1095
1096
#endif
1097
1098
+#define FIRST_SYSTEM_VECTOR 0xef
1099
+
1100
/*
1101
* Local APIC timer IRQ vector is on a different priority level,
1102
* to work around the 'lost local interrupt if more than 2 IRQ
1103
diff --git a/arch/x86/include/asm/irqflags.h b/arch/x86/include/asm/irqflags.h
1104
index 2bdab21..bcd2f5f 100644
1105
--- a/arch/x86/include/asm/irqflags.h
1106
+++ b/arch/x86/include/asm/irqflags.h
1107
@@ -4,6 +4,10 @@
1108
#include <asm/processor-flags.h>
1109
1110
#ifndef __ASSEMBLY__
1111
+
1112
+#include <linux/ipipe_base.h>
1113
+#include <linux/ipipe_trace.h>
1114
+
1115
/*
1116
* Interrupt control:
1117
*/
1118
@@ -12,35 +16,59 @@ static inline unsigned long native_save_fl(void)
1119
{
1120
unsigned long flags;
1121
1122
+#ifdef CONFIG_IPIPE
1123
+ flags = (!__ipipe_test_root()) << 9;
1124
+ barrier();
1125
+#else
1126
asm volatile("# __raw_save_flags\n\t"
1127
"pushf ; pop %0"
1128
: "=g" (flags)
1129
: /* no input */
1130
: "memory");
1131
+#endif
1132
1133
return flags;
1134
}
1135
1136
static inline void native_restore_fl(unsigned long flags)
1137
{
1138
+#ifdef CONFIG_IPIPE
1139
+ barrier();
1140
+ __ipipe_restore_root(!(flags & X86_EFLAGS_IF));
1141
+#else
1142
asm volatile("push %0 ; popf"
1143
: /* no output */
1144
:"g" (flags)
1145
:"memory", "cc");
1146
+#endif
1147
}
1148
1149
static inline void native_irq_disable(void)
1150
{
1151
+#ifdef CONFIG_IPIPE
1152
+ ipipe_check_context(ipipe_root_domain);
1153
+ __ipipe_stall_root();
1154
+ barrier();
1155
+#else
1156
asm volatile("cli": : :"memory");
1157
+#endif
1158
}
1159
1160
static inline void native_irq_enable(void)
1161
{
1162
+#ifdef CONFIG_IPIPE
1163
+ barrier();
1164
+ __ipipe_unstall_root();
1165
+#else
1166
asm volatile("sti": : :"memory");
1167
+#endif
1168
}
1169
1170
static inline void native_safe_halt(void)
1171
{
1172
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
1173
+ ipipe_trace_end(0x8000000E);
1174
+#endif
1175
asm volatile("sti; hlt": : :"memory");
1176
}
1177
1178
@@ -66,6 +94,71 @@ static inline void raw_local_irq_restore(unsigned long flags)
1179
native_restore_fl(flags);
1180
}
1181
1182
+static inline unsigned long raw_mangle_irq_bits(int virt, unsigned long real)
1183
+{
1184
+ /*
1185
+ * Merge virtual and real interrupt mask bits into a single
1186
+ * (32bit) word.
1187
+ */
1188
+ return (real & ~(1L << 31)) | ((virt != 0) << 31);
1189
+}
1190
+
1191
+static inline int raw_demangle_irq_bits(unsigned long *x)
1192
+{
1193
+ int virt = (*x & (1L << 31)) != 0;
1194
+ *x &= ~(1L << 31);
1195
+ return virt;
1196
+}
1197
+
1198
+#define local_irq_save_hw_notrace(x) \
1199
+ __asm__ __volatile__("pushf ; pop %0 ; cli":"=g" (x): /* no input */ :"memory")
1200
+#define local_irq_restore_hw_notrace(x) \
1201
+ __asm__ __volatile__("push %0 ; popf": /* no output */ :"g" (x):"memory", "cc")
1202
+
1203
+#define local_save_flags_hw(x) __asm__ __volatile__("pushf ; pop %0":"=g" (x): /* no input */)
1204
+
1205
+#define irqs_disabled_hw() \
1206
+ ({ \
1207
+ unsigned long x; \
1208
+ local_save_flags_hw(x); \
1209
+ !((x) & X86_EFLAGS_IF); \
1210
+ })
1211
+
1212
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
1213
+#define local_irq_disable_hw() do { \
1214
+ if (!irqs_disabled_hw()) { \
1215
+ local_irq_disable_hw_notrace(); \
1216
+ ipipe_trace_begin(0x80000000); \
1217
+ } \
1218
+ } while (0)
1219
+#define local_irq_enable_hw() do { \
1220
+ if (irqs_disabled_hw()) { \
1221
+ ipipe_trace_end(0x80000000); \
1222
+ local_irq_enable_hw_notrace(); \
1223
+ } \
1224
+ } while (0)
1225
+#define local_irq_save_hw(x) do { \
1226
+ local_save_flags_hw(x); \
1227
+ if ((x) & X86_EFLAGS_IF) { \
1228
+ local_irq_disable_hw_notrace(); \
1229
+ ipipe_trace_begin(0x80000001); \
1230
+ } \
1231
+ } while (0)
1232
+#define local_irq_restore_hw(x) do { \
1233
+ if ((x) & X86_EFLAGS_IF) \
1234
+ ipipe_trace_end(0x80000001); \
1235
+ local_irq_restore_hw_notrace(x); \
1236
+ } while (0)
1237
+#else /* !CONFIG_IPIPE_TRACE_IRQSOFF */
1238
+#define local_irq_save_hw(x) local_irq_save_hw_notrace(x)
1239
+#define local_irq_restore_hw(x) local_irq_restore_hw_notrace(x)
1240
+#define local_irq_enable_hw() local_irq_enable_hw_notrace()
1241
+#define local_irq_disable_hw() local_irq_disable_hw_notrace()
1242
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
1243
+
1244
+#define local_irq_disable_hw_notrace() __asm__ __volatile__("cli": : :"memory")
1245
+#define local_irq_enable_hw_notrace() __asm__ __volatile__("sti": : :"memory")
1246
+
1247
static inline void raw_local_irq_disable(void)
1248
{
1249
native_irq_disable();
1250
@@ -99,16 +192,38 @@ static inline void halt(void)
1251
*/
1252
static inline unsigned long __raw_local_irq_save(void)
1253
{
1254
+#ifdef CONFIG_IPIPE
1255
+ unsigned long flags = (!__ipipe_test_and_stall_root()) << 9;
1256
+ barrier();
1257
+#else
1258
unsigned long flags = __raw_local_save_flags();
1259
1260
raw_local_irq_disable();
1261
+#endif
1262
1263
return flags;
1264
}
1265
#else
1266
1267
-#define ENABLE_INTERRUPTS(x) sti
1268
-#define DISABLE_INTERRUPTS(x) cli
1269
+#ifdef CONFIG_IPIPE
1270
+#ifdef CONFIG_X86_32
1271
+#define DISABLE_INTERRUPTS(clobbers) PER_CPU(ipipe_percpu_darray, %eax); btsl $0,(%eax); sti
1272
+#define ENABLE_INTERRUPTS(clobbers) call __ipipe_unstall_root
1273
+#else /* CONFIG_X86_64 */
1274
+/* Not worth virtualizing in x86_64 mode. */
1275
+#define DISABLE_INTERRUPTS(clobbers) cli
1276
+#define ENABLE_INTERRUPTS(clobbers) sti
1277
+#endif /* CONFIG_X86_64 */
1278
+#define ENABLE_INTERRUPTS_HW_COND sti
1279
+#define DISABLE_INTERRUPTS_HW(clobbers) cli
1280
+#define ENABLE_INTERRUPTS_HW(clobbers) sti
1281
+#else /* !CONFIG_IPIPE */
1282
+#define ENABLE_INTERRUPTS(x) sti
1283
+#define DISABLE_INTERRUPTS(x) cli
1284
+#define ENABLE_INTERRUPTS_HW_COND
1285
+#define DISABLE_INTERRUPTS_HW(clobbers) DISABLE_INTERRUPTS(clobbers)
1286
+#define ENABLE_INTERRUPTS_HW(clobbers) ENABLE_INTERRUPTS(clobbers)
1287
+#endif /* !CONFIG_IPIPE */
1288
1289
#ifdef CONFIG_X86_64
1290
#define SWAPGS swapgs
1291
@@ -151,8 +266,10 @@ static inline unsigned long __raw_local_irq_save(void)
1292
#define raw_local_save_flags(flags) \
1293
do { (flags) = __raw_local_save_flags(); } while (0)
1294
1295
-#define raw_local_irq_save(flags) \
1296
- do { (flags) = __raw_local_irq_save(); } while (0)
1297
+#define raw_local_irq_save(flags) do { \
1298
+ ipipe_check_context(ipipe_root_domain); \
1299
+ (flags) = __raw_local_irq_save(); \
1300
+ } while (0)
1301
1302
static inline int raw_irqs_disabled_flags(unsigned long flags)
1303
{
1304
diff --git a/arch/x86/include/asm/mmu_context.h b/arch/x86/include/asm/mmu_context.h
1305
index 8aeeb3f..dddfb84 100644
1306
--- a/arch/x86/include/asm/mmu_context.h
1307
+++ b/arch/x86/include/asm/mmu_context.h
1308
@@ -29,8 +29,11 @@ void destroy_context(struct mm_struct *mm);
1309
1310
#define activate_mm(prev, next) \
1311
do { \
1312
+ unsigned long flags; \
1313
paravirt_activate_mm((prev), (next)); \
1314
+ local_irq_save_hw_cond(flags); \
1315
switch_mm((prev), (next), NULL); \
1316
+ local_irq_restore_hw_cond(flags); \
1317
} while (0);
1318
1319
1320
diff --git a/arch/x86/include/asm/nmi.h b/arch/x86/include/asm/nmi.h
1321
index c45a0a5..f326e5b 100644
1322
--- a/arch/x86/include/asm/nmi.h
1323
+++ b/arch/x86/include/asm/nmi.h
1324
@@ -29,7 +29,7 @@ extern void setup_apic_nmi_watchdog(void *);
1325
extern void stop_apic_nmi_watchdog(void *);
1326
extern void disable_timer_nmi_watchdog(void);
1327
extern void enable_timer_nmi_watchdog(void);
1328
-extern int nmi_watchdog_tick(struct pt_regs *regs, unsigned reason);
1329
+extern int (*nmi_watchdog_tick)(struct pt_regs *regs, unsigned reason);
1330
extern void cpu_nmi_set_wd_enabled(void);
1331
1332
extern atomic_t nmi_active;
1333
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
1334
index 06d6f30..35a0e1a 100644
1335
--- a/arch/x86/include/asm/processor.h
1336
+++ b/arch/x86/include/asm/processor.h
1337
@@ -397,6 +397,7 @@ struct thread_struct {
1338
unsigned short ds;
1339
unsigned short fsindex;
1340
unsigned short gsindex;
1341
+ unsigned long rip;
1342
#endif
1343
unsigned long ip;
1344
unsigned long fs;
1345
diff --git a/arch/x86/include/asm/system.h b/arch/x86/include/asm/system.h
1346
index 2ed3f0f..4b84f80 100644
1347
--- a/arch/x86/include/asm/system.h
1348
+++ b/arch/x86/include/asm/system.h
1349
@@ -90,8 +90,10 @@ do { \
1350
#define switch_to(prev, next, last) \
1351
asm volatile(SAVE_CONTEXT \
1352
"movq %%rsp,%P[threadrsp](%[prev])\n\t" /* save RSP */ \
1353
+ "movq $thread_return,%P[threadrip](%[prev])\n\t" /* save RIP */ \
1354
"movq %P[threadrsp](%[next]),%%rsp\n\t" /* restore RSP */ \
1355
- "call __switch_to\n\t" \
1356
+ "pushq %P[threadrip](%[next])\n\t" /* restore RIP */ \
1357
+ "jmp __switch_to\n\t" \
1358
".globl thread_return\n" \
1359
"thread_return:\n\t" \
1360
"movq %%gs:%P[pda_pcurrent],%%rsi\n\t" \
1361
@@ -103,6 +105,7 @@ do { \
1362
: "=a" (last) \
1363
: [next] "S" (next), [prev] "D" (prev), \
1364
[threadrsp] "i" (offsetof(struct task_struct, thread.sp)), \
1365
+ [threadrip] "i" (offsetof(struct task_struct, thread.rip)), \
1366
[ti_flags] "i" (offsetof(struct thread_info, flags)), \
1367
[tif_fork] "i" (TIF_FORK), \
1368
[thread_info] "i" (offsetof(struct task_struct, stack)), \
1369
diff --git a/arch/x86/include/asm/unistd_32.h b/arch/x86/include/asm/unistd_32.h
1370
index f2bba78..0c75007 100644
1371
--- a/arch/x86/include/asm/unistd_32.h
1372
+++ b/arch/x86/include/asm/unistd_32.h
1373
@@ -339,6 +339,8 @@
1374
#define __NR_pipe2 331
1375
#define __NR_inotify_init1 332
1376
1377
+#define NR_syscalls 333
1378
+
1379
#ifdef __KERNEL__
1380
1381
#define __ARCH_WANT_IPC_PARSE_VERSION
1382
diff --git a/arch/x86/include/asm/unistd_64.h b/arch/x86/include/asm/unistd_64.h
1383
index d2e415e..03072c6 100644
1384
--- a/arch/x86/include/asm/unistd_64.h
1385
+++ b/arch/x86/include/asm/unistd_64.h
1386
@@ -654,6 +654,7 @@ __SYSCALL(__NR_pipe2, sys_pipe2)
1387
#define __NR_inotify_init1 294
1388
__SYSCALL(__NR_inotify_init1, sys_inotify_init1)
1389
1390
+#define NR_syscalls 295
1391
1392
#ifndef __NO_STUBS
1393
#define __ARCH_WANT_OLD_READDIR
1394
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile
1395
index b62a766..576e8a1 100644
1396
--- a/arch/x86/kernel/Makefile
1397
+++ b/arch/x86/kernel/Makefile
1398
@@ -79,6 +79,8 @@ obj-$(CONFIG_DOUBLEFAULT) += doublefault_32.o
1399
obj-$(CONFIG_KGDB) += kgdb.o
1400
obj-$(CONFIG_VM86) += vm86_32.o
1401
obj-$(CONFIG_EARLY_PRINTK) += early_printk.o
1402
+obj-$(CONFIG_IPIPE) += ipipe.o
1403
+obj-$(CONFIG_IPIPE_TRACE_MCOUNT) += mcount_$(BITS).o
1404
1405
obj-$(CONFIG_HPET_TIMER) += hpet.o
1406
1407
diff --git a/arch/x86/kernel/apic.c b/arch/x86/kernel/apic.c
1408
index 3eae1df..2587543 100644
1409
--- a/arch/x86/kernel/apic.c
1410
+++ b/arch/x86/kernel/apic.c
1411
@@ -428,7 +428,7 @@ static void lapic_timer_setup(enum clock_event_mode mode,
1412
if (evt->features & CLOCK_EVT_FEAT_DUMMY)
1413
return;
1414
1415
- local_irq_save(flags);
1416
+ local_irq_save_hw(flags);
1417
1418
switch (mode) {
1419
case CLOCK_EVT_MODE_PERIODIC:
1420
@@ -447,7 +447,7 @@ static void lapic_timer_setup(enum clock_event_mode mode,
1421
break;
1422
}
1423
1424
- local_irq_restore(flags);
1425
+ local_irq_restore_hw(flags);
1426
}
1427
1428
/*
1429
@@ -945,7 +945,7 @@ void lapic_shutdown(void)
1430
if (!cpu_has_apic)
1431
return;
1432
1433
- local_irq_save(flags);
1434
+ local_irq_save_hw(flags);
1435
1436
#ifdef CONFIG_X86_32
1437
if (!enabled_via_apicbase)
1438
@@ -955,7 +955,7 @@ void lapic_shutdown(void)
1439
disable_local_APIC();
1440
1441
1442
- local_irq_restore(flags);
1443
+ local_irq_restore_hw(flags);
1444
}
1445
1446
/*
1447
@@ -1129,6 +1129,10 @@ static void __cpuinit lapic_setup_esr(void)
1448
oldvalue, value);
1449
}
1450
1451
+int __ipipe_check_lapic(void)
1452
+{
1453
+ return !(lapic_clockevent.features & CLOCK_EVT_FEAT_DUMMY);
1454
+}
1455
1456
/**
1457
* setup_local_APIC - setup the local APIC
1458
@@ -1187,7 +1191,7 @@ void __cpuinit setup_local_APIC(void)
1459
value = apic_read(APIC_ISR + i*0x10);
1460
for (j = 31; j >= 0; j--) {
1461
if (value & (1<<j))
1462
- ack_APIC_irq();
1463
+ __ack_APIC_irq();
1464
}
1465
}
1466
1467
@@ -1693,7 +1697,7 @@ void smp_spurious_interrupt(struct pt_regs *regs)
1468
*/
1469
v = apic_read(APIC_ISR + ((SPURIOUS_APIC_VECTOR & ~0x1f) >> 1));
1470
if (v & (1 << (SPURIOUS_APIC_VECTOR & 0x1f)))
1471
- ack_APIC_irq();
1472
+ __ack_APIC_irq();
1473
1474
#ifdef CONFIG_X86_64
1475
add_pda(irq_spurious_count, 1);
1476
@@ -1970,9 +1974,9 @@ static int lapic_suspend(struct sys_device *dev, pm_message_t state)
1477
apic_pm_state.apic_thmr = apic_read(APIC_LVTTHMR);
1478
#endif
1479
1480
- local_irq_save(flags);
1481
+ local_irq_save_hw(flags);
1482
disable_local_APIC();
1483
- local_irq_restore(flags);
1484
+ local_irq_restore_hw(flags);
1485
return 0;
1486
}
1487
1488
@@ -1987,7 +1991,7 @@ static int lapic_resume(struct sys_device *dev)
1489
1490
maxlvt = lapic_get_maxlvt();
1491
1492
- local_irq_save(flags);
1493
+ local_irq_save_hw(flags);
1494
1495
#ifdef HAVE_X2APIC
1496
if (x2apic)
1497
@@ -2030,7 +2034,7 @@ static int lapic_resume(struct sys_device *dev)
1498
apic_write(APIC_ESR, 0);
1499
apic_read(APIC_ESR);
1500
1501
- local_irq_restore(flags);
1502
+ local_irq_restore_hw(flags);
1503
1504
return 0;
1505
}
1506
@@ -2236,3 +2240,4 @@ static int __init lapic_insert_resource(void)
1507
* that is using request_resource
1508
*/
1509
late_initcall(lapic_insert_resource);
1510
+
1511
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
1512
index b9c9ea0..fda09a0 100644
1513
--- a/arch/x86/kernel/cpu/common.c
1514
+++ b/arch/x86/kernel/cpu/common.c
1515
@@ -864,7 +864,7 @@ struct desc_ptr idt_descr = { 256 * 16 - 1, (unsigned long) idt_table };
1516
1517
char boot_cpu_stack[IRQSTACKSIZE] __page_aligned_bss;
1518
1519
-void __cpuinit pda_init(int cpu)
1520
+void notrace __cpuinit pda_init(int cpu)
1521
{
1522
struct x8664_pda *pda = cpu_pda(cpu);
1523
1524
@@ -956,7 +956,7 @@ struct pt_regs * __cpuinit idle_regs(struct pt_regs *regs)
1525
* A lot of state is already set up in PDA init for 64 bit
1526
*/
1527
#ifdef CONFIG_X86_64
1528
-void __cpuinit cpu_init(void)
1529
+void notrace __cpuinit cpu_init(void)
1530
{
1531
int cpu = stack_smp_processor_id();
1532
struct tss_struct *t = &per_cpu(init_tss, cpu);
1533
@@ -1076,7 +1076,7 @@ void __cpuinit cpu_init(void)
1534
1535
#else
1536
1537
-void __cpuinit cpu_init(void)
1538
+void notrace __cpuinit cpu_init(void)
1539
{
1540
int cpu = smp_processor_id();
1541
struct task_struct *curr = current;
1542
diff --git a/arch/x86/kernel/cpu/mtrr/cyrix.c b/arch/x86/kernel/cpu/mtrr/cyrix.c
1543
index ff14c32..e30e25e 100644
1544
--- a/arch/x86/kernel/cpu/mtrr/cyrix.c
1545
+++ b/arch/x86/kernel/cpu/mtrr/cyrix.c
1546
@@ -170,6 +170,7 @@ static void cyrix_set_arr(unsigned int reg, unsigned long base,
1547
unsigned long size, mtrr_type type)
1548
{
1549
unsigned char arr, arr_type, arr_size;
1550
+ unsigned long flags;
1551
1552
arr = CX86_ARR_BASE + (reg << 1) + reg; /* avoid multiplication by 3 */
1553
1554
@@ -212,6 +213,8 @@ static void cyrix_set_arr(unsigned int reg, unsigned long base,
1555
}
1556
}
1557
1558
+ local_irq_save_hw(flags);
1559
+
1560
prepare_set();
1561
1562
base <<= PAGE_SHIFT;
1563
@@ -221,6 +224,8 @@ static void cyrix_set_arr(unsigned int reg, unsigned long base,
1564
setCx86(CX86_RCR_BASE + reg, arr_type);
1565
1566
post_set();
1567
+
1568
+ local_irq_restore_hw(flags);
1569
}
1570
1571
typedef struct {
1572
@@ -238,8 +243,10 @@ static unsigned char ccr_state[7] = { 0, 0, 0, 0, 0, 0, 0 };
1573
1574
static void cyrix_set_all(void)
1575
{
1576
+ unsigned long flags;
1577
int i;
1578
1579
+ local_irq_save_hw(flags);
1580
prepare_set();
1581
1582
/* the CCRs are not contiguous */
1583
@@ -252,6 +259,7 @@ static void cyrix_set_all(void)
1584
arr_state[i].size, arr_state[i].type);
1585
1586
post_set();
1587
+ local_irq_restore_hw(flags);
1588
}
1589
1590
static struct mtrr_ops cyrix_mtrr_ops = {
1591
diff --git a/arch/x86/kernel/cpu/mtrr/generic.c b/arch/x86/kernel/cpu/mtrr/generic.c
1592
index 4e8d77f..cd4cc43 100644
1593
--- a/arch/x86/kernel/cpu/mtrr/generic.c
1594
+++ b/arch/x86/kernel/cpu/mtrr/generic.c
1595
@@ -552,7 +552,7 @@ static void generic_set_all(void)
1596
unsigned long mask, count;
1597
unsigned long flags;
1598
1599
- local_irq_save(flags);
1600
+ local_irq_save_hw(flags);
1601
prepare_set();
1602
1603
/* Actually set the state */
1604
@@ -562,7 +562,7 @@ static void generic_set_all(void)
1605
pat_init();
1606
1607
post_set();
1608
- local_irq_restore(flags);
1609
+ local_irq_restore_hw(flags);
1610
1611
/* Use the atomic bitops to update the global mask */
1612
for (count = 0; count < sizeof mask * 8; ++count) {
1613
@@ -583,12 +583,12 @@ static void generic_set_mtrr(unsigned int reg, unsigned long base,
1614
[RETURNS] Nothing.
1615
*/
1616
{
1617
- unsigned long flags;
1618
+ unsigned long flags, _flags;
1619
struct mtrr_var_range *vr;
1620
1621
vr = &mtrr_state.var_ranges[reg];
1622
1623
- local_irq_save(flags);
1624
+ local_irq_save_full(flags, _flags);
1625
prepare_set();
1626
1627
if (size == 0) {
1628
@@ -607,7 +607,7 @@ static void generic_set_mtrr(unsigned int reg, unsigned long base,
1629
}
1630
1631
post_set();
1632
- local_irq_restore(flags);
1633
+ local_irq_restore_full(flags, _flags);
1634
}
1635
1636
int generic_validate_add_page(unsigned long base, unsigned long size, unsigned int type)
1637
diff --git a/arch/x86/kernel/dumpstack_32.c b/arch/x86/kernel/dumpstack_32.c
1638
index b361475..7f24b7f 100644
1639
--- a/arch/x86/kernel/dumpstack_32.c
1640
+++ b/arch/x86/kernel/dumpstack_32.c
1641
@@ -234,6 +234,9 @@ void show_registers(struct pt_regs *regs)
1642
printk(KERN_EMERG "Process %.*s (pid: %d, ti=%p task=%p task.ti=%p)\n",
1643
TASK_COMM_LEN, current->comm, task_pid_nr(current),
1644
current_thread_info(), current, task_thread_info(current));
1645
+#ifdef CONFIG_IPIPE
1646
+ printk(KERN_EMERG "I-pipe domain %s\n", ipipe_current_domain->name);
1647
+#endif /* CONFIG_IPIPE */
1648
/*
1649
* When in-kernel, we also print out the stack and code at the
1650
* time of the fault..
1651
@@ -419,6 +422,8 @@ die_nmi(char *str, struct pt_regs *regs, int do_panic)
1652
do_exit(SIGSEGV);
1653
}
1654
1655
+EXPORT_SYMBOL_GPL(die_nmi);
1656
+
1657
static int __init oops_setup(char *s)
1658
{
1659
if (!s)
1660
diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c
1661
index 96a5db7..da14b7a 100644
1662
--- a/arch/x86/kernel/dumpstack_64.c
1663
+++ b/arch/x86/kernel/dumpstack_64.c
1664
@@ -379,6 +379,11 @@ void show_registers(struct pt_regs *regs)
1665
sp = regs->sp;
1666
printk("CPU %d ", cpu);
1667
__show_regs(regs, 1);
1668
+#ifdef CONFIG_IPIPE
1669
+ if (ipipe_current_domain != ipipe_root_domain)
1670
+ printk("I-pipe domain %s\n", ipipe_current_domain->name);
1671
+ else
1672
+#endif /* CONFIG_IPIPE */
1673
printk("Process %s (pid: %d, threadinfo %p, task %p)\n",
1674
cur->comm, cur->pid, task_thread_info(cur), cur);
1675
1676
diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S
1677
index 28b597e..fc9b2c6 100644
1678
--- a/arch/x86/kernel/entry_32.S
1679
+++ b/arch/x86/kernel/entry_32.S
1680
@@ -43,6 +43,7 @@
1681
#include <linux/linkage.h>
1682
#include <asm/thread_info.h>
1683
#include <asm/irqflags.h>
1684
+#include <asm/ipipe_base.h>
1685
#include <asm/errno.h>
1686
#include <asm/segment.h>
1687
#include <asm/smp.h>
1688
@@ -79,6 +80,59 @@
1689
1690
#define nr_syscalls ((syscall_table_size)/4)
1691
1692
+#ifdef CONFIG_IPIPE
1693
+#define EMULATE_ROOT_IRET(bypass) \
1694
+ call __ipipe_unstall_iret_root ; \
1695
+ TRACE_IRQS_ON ; \
1696
+ bypass: \
1697
+ movl PT_EAX(%esp),%eax
1698
+#define TEST_PREEMPTIBLE(regs) call __ipipe_kpreempt_root ; testl %eax,%eax
1699
+#define CATCH_ROOT_SYSCALL(bypass1,bypass2) \
1700
+ movl %esp,%eax ; \
1701
+ call __ipipe_syscall_root ; \
1702
+ testl %eax,%eax ; \
1703
+ js bypass1 ; \
1704
+ jne bypass2 ; \
1705
+ movl PT_ORIG_EAX(%esp),%eax
1706
+#define PUSH_XCODE(v) pushl $ ex_ ## v
1707
+#define PUSH_XVEC(v) pushl $ ex_ ## v
1708
+#define HANDLE_EXCEPTION(code) movl %code,%ecx ; \
1709
+ call __ipipe_handle_exception ; \
1710
+ testl %eax,%eax ; \
1711
+ jnz restore_nocheck_notrace
1712
+#define DIVERT_EXCEPTION(code) movl $(__USER_DS), %ecx ; \
1713
+ movl %ecx, %ds ; \
1714
+ movl %ecx, %es ; \
1715
+ movl %esp, %eax ; \
1716
+ movl $ex_ ## code,%edx ; \
1717
+ call __ipipe_divert_exception ; \
1718
+ testl %eax,%eax ; \
1719
+ jnz restore_nocheck_notrace
1720
+
1721
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
1722
+# define IPIPE_TRACE_IRQ_ENTER \
1723
+ lea PT_EIP-4(%esp), %ebp; \
1724
+ movl PT_ORIG_EAX(%esp), %eax; \
1725
+ call ipipe_trace_begin
1726
+# define IPIPE_TRACE_IRQ_EXIT \
1727
+ pushl %eax; \
1728
+ movl PT_ORIG_EAX+4(%esp), %eax; \
1729
+ call ipipe_trace_end; \
1730
+ popl %eax
1731
+#else /* !CONFIG_IPIPE_TRACE_IRQSOFF */
1732
+#define IPIPE_TRACE_IRQ_ENTER
1733
+#define IPIPE_TRACE_IRQ_EXIT
1734
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
1735
+#else /* !CONFIG_IPIPE */
1736
+#define EMULATE_ROOT_IRET(bypass)
1737
+#define TEST_PREEMPTIBLE(regs) testl $X86_EFLAGS_IF,PT_EFLAGS(regs)
1738
+#define CATCH_ROOT_SYSCALL(bypass1,bypass2)
1739
+#define PUSH_XCODE(v) pushl $v
1740
+#define PUSH_XVEC(v) pushl v
1741
+#define HANDLE_EXCEPTION(code) call *%code
1742
+#define DIVERT_EXCEPTION(code)
1743
+#endif /* CONFIG_IPIPE */
1744
+
1745
#ifdef CONFIG_PREEMPT
1746
#define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF
1747
#else
1748
@@ -218,6 +272,7 @@
1749
CFI_OFFSET ebx, PT_EBX-PT_OLDESP
1750
1751
ENTRY(ret_from_fork)
1752
+ ENABLE_INTERRUPTS_HW_COND
1753
CFI_STARTPROC
1754
pushl %eax
1755
CFI_ADJUST_CFA_OFFSET 4
1756
@@ -245,7 +300,7 @@ END(ret_from_fork)
1757
RING0_PTREGS_FRAME
1758
ret_from_exception:
1759
preempt_stop(CLBR_ANY)
1760
-ret_from_intr:
1761
+ENTRY(ret_from_intr)
1762
GET_THREAD_INFO(%ebp)
1763
check_userspace:
1764
movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS
1765
@@ -269,14 +324,14 @@ END(ret_from_exception)
1766
1767
#ifdef CONFIG_PREEMPT
1768
ENTRY(resume_kernel)
1769
- DISABLE_INTERRUPTS(CLBR_ANY)
1770
+ DISABLE_INTERRUPTS_HW(CLBR_ANY)
1771
cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ?
1772
jnz restore_nocheck
1773
need_resched:
1774
movl TI_flags(%ebp), %ecx # need_resched set ?
1775
testb $_TIF_NEED_RESCHED, %cl
1776
jz restore_all
1777
- testl $X86_EFLAGS_IF,PT_EFLAGS(%esp) # interrupts off (exception path) ?
1778
+ TEST_PREEMPTIBLE(%esp) # interrupts off (exception path) ?
1779
jz restore_all
1780
call preempt_schedule_irq
1781
jmp need_resched
1782
@@ -324,7 +379,7 @@ sysenter_past_esp:
1783
pushl %eax
1784
CFI_ADJUST_CFA_OFFSET 4
1785
SAVE_ALL
1786
- ENABLE_INTERRUPTS(CLBR_NONE)
1787
+ ENABLE_INTERRUPTS_HW(CLBR_NONE)
1788
1789
/*
1790
* Load the potential sixth argument from user stack.
1791
@@ -340,6 +395,7 @@ sysenter_past_esp:
1792
.previous
1793
1794
GET_THREAD_INFO(%ebp)
1795
+ CATCH_ROOT_SYSCALL(sysenter_tail,sysenter_out)
1796
1797
/* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
1798
testw $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
1799
@@ -349,6 +405,7 @@ sysenter_do_call:
1800
jae syscall_badsys
1801
call *sys_call_table(,%eax,4)
1802
movl %eax,PT_EAX(%esp)
1803
+sysenter_tail:
1804
LOCKDEP_SYS_EXIT
1805
DISABLE_INTERRUPTS(CLBR_ANY)
1806
TRACE_IRQS_OFF
1807
@@ -357,10 +414,13 @@ sysenter_do_call:
1808
jne sysexit_audit
1809
sysenter_exit:
1810
/* if something modifies registers it must also disable sysexit */
1811
+ EMULATE_ROOT_IRET(sysenter_out)
1812
movl PT_EIP(%esp), %edx
1813
movl PT_OLDESP(%esp), %ecx
1814
xorl %ebp,%ebp
1815
- TRACE_IRQS_ON
1816
+#ifndef CONFIG_IPIPE
1817
+ TRACE_IRQS_ON
1818
+#endif
1819
1: mov PT_FS(%esp), %fs
1820
ENABLE_INTERRUPTS_SYSEXIT
1821
1822
@@ -419,6 +479,7 @@ ENTRY(system_call)
1823
CFI_ADJUST_CFA_OFFSET 4
1824
SAVE_ALL
1825
GET_THREAD_INFO(%ebp)
1826
+ CATCH_ROOT_SYSCALL(syscall_exit,restore_nocheck_notrace)
1827
# system call tracing in operation / emulation
1828
/* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
1829
testw $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%ebp)
1830
@@ -450,7 +511,11 @@ restore_all:
1831
CFI_REMEMBER_STATE
1832
je ldt_ss # returning to user-space with LDT SS
1833
restore_nocheck:
1834
+#ifdef CONFIG_IPIPE
1835
+ call __ipipe_unstall_iret_root
1836
+#else /* !CONFIG_IPIPE */
1837
TRACE_IRQS_IRET
1838
+#endif /* CONFIG_IPIPE */
1839
restore_nocheck_notrace:
1840
RESTORE_REGS
1841
addl $4, %esp # skip orig_eax/error_code
1842
@@ -460,7 +525,7 @@ irq_return:
1843
.section .fixup,"ax"
1844
ENTRY(iret_exc)
1845
pushl $0 # no error code
1846
- pushl $do_iret_error
1847
+ PUSH_XCODE(do_iret_error)
1848
jmp error_code
1849
.previous
1850
.section __ex_table,"a"
1851
@@ -501,7 +566,7 @@ ldt_ss:
1852
CFI_ADJUST_CFA_OFFSET 4
1853
pushl %eax
1854
CFI_ADJUST_CFA_OFFSET 4
1855
- DISABLE_INTERRUPTS(CLBR_EAX)
1856
+ DISABLE_INTERRUPTS_HW(CLBR_EAX)
1857
TRACE_IRQS_OFF
1858
lss (%esp), %esp
1859
CFI_ADJUST_CFA_OFFSET -8
1860
@@ -516,6 +581,7 @@ work_pending:
1861
testb $_TIF_NEED_RESCHED, %cl
1862
jz work_notifysig
1863
work_resched:
1864
+ ENABLE_INTERRUPTS_HW_COND
1865
call schedule
1866
LOCKDEP_SYS_EXIT
1867
DISABLE_INTERRUPTS(CLBR_ANY) # make sure we don't miss an interrupt
1868
@@ -648,6 +714,49 @@ END(irq_entries_start)
1869
END(interrupt)
1870
.previous
1871
1872
+#ifdef CONFIG_IPIPE
1873
+ ALIGN
1874
+common_interrupt:
1875
+ SAVE_ALL
1876
+ IPIPE_TRACE_IRQ_ENTER
1877
+ movl %esp, %eax
1878
+ call __ipipe_handle_irq
1879
+ IPIPE_TRACE_IRQ_EXIT
1880
+ testl %eax,%eax
1881
+ jnz ret_from_intr
1882
+ RESTORE_REGS
1883
+ addl $4, %esp
1884
+ iret
1885
+ CFI_ENDPROC
1886
+
1887
+#define BUILD_INTERRUPT(name, nr) \
1888
+ENTRY(name) \
1889
+ RING0_INT_FRAME; \
1890
+ pushl $~(nr); \
1891
+ CFI_ADJUST_CFA_OFFSET 4; \
1892
+ SAVE_ALL; \
1893
+ IPIPE_TRACE_IRQ_ENTER; \
1894
+ movl %esp, %eax; \
1895
+ call __ipipe_handle_irq; \
1896
+ IPIPE_TRACE_IRQ_EXIT; \
1897
+ testl %eax,%eax; \
1898
+ jnz ret_from_intr; \
1899
+ RESTORE_REGS; \
1900
+ addl $4, %esp; \
1901
+ iret; \
1902
+ CFI_ENDPROC
1903
+
1904
+#ifdef CONFIG_X86_LOCAL_APIC
1905
+ BUILD_INTERRUPT(ipipe_ipi0,IPIPE_SERVICE_VECTOR0)
1906
+ BUILD_INTERRUPT(ipipe_ipi1,IPIPE_SERVICE_VECTOR1)
1907
+ BUILD_INTERRUPT(ipipe_ipi2,IPIPE_SERVICE_VECTOR2)
1908
+ BUILD_INTERRUPT(ipipe_ipi3,IPIPE_SERVICE_VECTOR3)
1909
+#ifdef CONFIG_SMP
1910
+ BUILD_INTERRUPT(ipipe_ipiX,IPIPE_CRITICAL_VECTOR)
1911
+#endif
1912
+#endif
1913
+
1914
+#else /* !CONFIG_IPIPE */
1915
/*
1916
* the CPU automatically disables interrupts when executing an IRQ vector,
1917
* so IRQ-flags tracing has to follow that:
1918
@@ -675,12 +784,14 @@ ENTRY(name) \
1919
CFI_ENDPROC; \
1920
ENDPROC(name)
1921
1922
+#endif /* !CONFIG_IPIPE */
1923
+
1924
/* The include is where all of the SMP etc. interrupts come from */
1925
#include "entry_arch.h"
1926
1927
KPROBE_ENTRY(page_fault)
1928
RING0_EC_FRAME
1929
- pushl $do_page_fault
1930
+ PUSH_XCODE(do_page_fault)
1931
CFI_ADJUST_CFA_OFFSET 4
1932
ALIGN
1933
error_code:
1934
@@ -732,7 +843,7 @@ error_code:
1935
movl %ecx, %es
1936
TRACE_IRQS_OFF
1937
movl %esp,%eax # pt_regs pointer
1938
- call *%edi
1939
+ HANDLE_EXCEPTION(edi)
1940
jmp ret_from_exception
1941
CFI_ENDPROC
1942
KPROBE_END(page_fault)
1943
@@ -741,7 +852,7 @@ ENTRY(coprocessor_error)
1944
RING0_INT_FRAME
1945
pushl $0
1946
CFI_ADJUST_CFA_OFFSET 4
1947
- pushl $do_coprocessor_error
1948
+ PUSH_XCODE(do_coprocessor_error)
1949
CFI_ADJUST_CFA_OFFSET 4
1950
jmp error_code
1951
CFI_ENDPROC
1952
@@ -751,7 +862,7 @@ ENTRY(simd_coprocessor_error)
1953
RING0_INT_FRAME
1954
pushl $0
1955
CFI_ADJUST_CFA_OFFSET 4
1956
- pushl $do_simd_coprocessor_error
1957
+ PUSH_XCODE(do_simd_coprocessor_error)
1958
CFI_ADJUST_CFA_OFFSET 4
1959
jmp error_code
1960
CFI_ENDPROC
1961
@@ -761,7 +872,7 @@ ENTRY(device_not_available)
1962
RING0_INT_FRAME
1963
pushl $-1 # mark this as an int
1964
CFI_ADJUST_CFA_OFFSET 4
1965
- pushl $do_device_not_available
1966
+ PUSH_XCODE(do_device_not_available)
1967
CFI_ADJUST_CFA_OFFSET 4
1968
jmp error_code
1969
CFI_ENDPROC
1970
@@ -805,6 +916,7 @@ debug_stack_correct:
1971
CFI_ADJUST_CFA_OFFSET 4
1972
SAVE_ALL
1973
TRACE_IRQS_OFF
1974
+ DIVERT_EXCEPTION(do_debug)
1975
xorl %edx,%edx # error code 0
1976
movl %esp,%eax # pt_regs pointer
1977
call do_debug
1978
@@ -922,6 +1034,7 @@ KPROBE_ENTRY(int3)
1979
CFI_ADJUST_CFA_OFFSET 4
1980
SAVE_ALL
1981
TRACE_IRQS_OFF
1982
+ DIVERT_EXCEPTION(do_int3)
1983
xorl %edx,%edx # zero error code
1984
movl %esp,%eax # pt_regs pointer
1985
call do_int3
1986
@@ -933,7 +1046,7 @@ ENTRY(overflow)
1987
RING0_INT_FRAME
1988
pushl $0
1989
CFI_ADJUST_CFA_OFFSET 4
1990
- pushl $do_overflow
1991
+ PUSH_XCODE(do_overflow)
1992
CFI_ADJUST_CFA_OFFSET 4
1993
jmp error_code
1994
CFI_ENDPROC
1995
@@ -943,7 +1056,7 @@ ENTRY(bounds)
1996
RING0_INT_FRAME
1997
pushl $0
1998
CFI_ADJUST_CFA_OFFSET 4
1999
- pushl $do_bounds
2000
+ PUSH_XCODE(do_bounds)
2001
CFI_ADJUST_CFA_OFFSET 4
2002
jmp error_code
2003
CFI_ENDPROC
2004
@@ -953,7 +1066,7 @@ ENTRY(invalid_op)
2005
RING0_INT_FRAME
2006
pushl $0
2007
CFI_ADJUST_CFA_OFFSET 4
2008
- pushl $do_invalid_op
2009
+ PUSH_XCODE(do_invalid_op)
2010
CFI_ADJUST_CFA_OFFSET 4
2011
jmp error_code
2012
CFI_ENDPROC
2013
@@ -963,7 +1076,7 @@ ENTRY(coprocessor_segment_overrun)
2014
RING0_INT_FRAME
2015
pushl $0
2016
CFI_ADJUST_CFA_OFFSET 4
2017
- pushl $do_coprocessor_segment_overrun
2018
+ PUSH_XCODE(do_coprocessor_segment_overrun)
2019
CFI_ADJUST_CFA_OFFSET 4
2020
jmp error_code
2021
CFI_ENDPROC
2022
@@ -971,7 +1084,7 @@ END(coprocessor_segment_overrun)
2023
2024
ENTRY(invalid_TSS)
2025
RING0_EC_FRAME
2026
- pushl $do_invalid_TSS
2027
+ PUSH_XCODE(do_invalid_TSS)
2028
CFI_ADJUST_CFA_OFFSET 4
2029
jmp error_code
2030
CFI_ENDPROC
2031
@@ -979,7 +1092,7 @@ END(invalid_TSS)
2032
2033
ENTRY(segment_not_present)
2034
RING0_EC_FRAME
2035
- pushl $do_segment_not_present
2036
+ PUSH_XCODE(do_segment_not_present)
2037
CFI_ADJUST_CFA_OFFSET 4
2038
jmp error_code
2039
CFI_ENDPROC
2040
@@ -987,7 +1100,7 @@ END(segment_not_present)
2041
2042
ENTRY(stack_segment)
2043
RING0_EC_FRAME
2044
- pushl $do_stack_segment
2045
+ PUSH_XCODE(do_stack_segment)
2046
CFI_ADJUST_CFA_OFFSET 4
2047
jmp error_code
2048
CFI_ENDPROC
2049
@@ -995,7 +1108,7 @@ END(stack_segment)
2050
2051
KPROBE_ENTRY(general_protection)
2052
RING0_EC_FRAME
2053
- pushl $do_general_protection
2054
+ PUSH_XCODE(do_general_protection)
2055
CFI_ADJUST_CFA_OFFSET 4
2056
jmp error_code
2057
CFI_ENDPROC
2058
@@ -1003,7 +1116,7 @@ KPROBE_END(general_protection)
2059
2060
ENTRY(alignment_check)
2061
RING0_EC_FRAME
2062
- pushl $do_alignment_check
2063
+ PUSH_XCODE(do_alignment_check)
2064
CFI_ADJUST_CFA_OFFSET 4
2065
jmp error_code
2066
CFI_ENDPROC
2067
@@ -1013,7 +1126,7 @@ ENTRY(divide_error)
2068
RING0_INT_FRAME
2069
pushl $0 # no error code
2070
CFI_ADJUST_CFA_OFFSET 4
2071
- pushl $do_divide_error
2072
+ PUSH_XCODE(do_divide_error)
2073
CFI_ADJUST_CFA_OFFSET 4
2074
jmp error_code
2075
CFI_ENDPROC
2076
@@ -1024,7 +1137,7 @@ ENTRY(machine_check)
2077
RING0_INT_FRAME
2078
pushl $0
2079
CFI_ADJUST_CFA_OFFSET 4
2080
- pushl machine_check_vector
2081
+ PUSH_XVEC(machine_check_vector)
2082
CFI_ADJUST_CFA_OFFSET 4
2083
jmp error_code
2084
CFI_ENDPROC
2085
@@ -1035,7 +1148,7 @@ ENTRY(spurious_interrupt_bug)
2086
RING0_INT_FRAME
2087
pushl $0
2088
CFI_ADJUST_CFA_OFFSET 4
2089
- pushl $do_spurious_interrupt_bug
2090
+ PUSH_XCODE(do_spurious_interrupt_bug)
2091
CFI_ADJUST_CFA_OFFSET 4
2092
jmp error_code
2093
CFI_ENDPROC
2094
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
2095
index b86f332..d59b479 100644
2096
--- a/arch/x86/kernel/entry_64.S
2097
+++ b/arch/x86/kernel/entry_64.S
2098
@@ -52,6 +52,7 @@
2099
#include <asm/irqflags.h>
2100
#include <asm/paravirt.h>
2101
#include <asm/ftrace.h>
2102
+#include <asm/ipipe_base.h>
2103
2104
/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */
2105
#include <linux/elf-em.h>
2106
@@ -61,6 +62,13 @@
2107
2108
.code64
2109
2110
+#ifdef CONFIG_IPIPE
2111
+#define PREEMPT_SCHEDULE_IRQ call __ipipe_preempt_schedule_irq
2112
+#else /* !CONFIG_IPIPE */
2113
+#define PREEMPT_SCHEDULE_IRQ call preempt_schedule_irq
2114
+#endif /* !CONFIG_IPIPE */
2115
+
2116
+
2117
#ifdef CONFIG_FUNCTION_TRACER
2118
#ifdef CONFIG_DYNAMIC_FTRACE
2119
ENTRY(mcount)
2120
@@ -252,6 +260,7 @@ ENTRY(ret_from_fork)
2121
CFI_ADJUST_CFA_OFFSET 8
2122
popf # reset kernel eflags
2123
CFI_ADJUST_CFA_OFFSET -8
2124
+ ENABLE_INTERRUPTS_HW_COND
2125
call schedule_tail
2126
GET_THREAD_INFO(%rcx)
2127
testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT),TI_flags(%rcx)
2128
@@ -328,6 +337,17 @@ ENTRY(system_call_after_swapgs)
2129
movq %rax,ORIG_RAX-ARGOFFSET(%rsp)
2130
movq %rcx,RIP-ARGOFFSET(%rsp)
2131
CFI_REL_OFFSET rip,RIP-ARGOFFSET
2132
+#ifdef CONFIG_IPIPE
2133
+ pushq %rdi
2134
+ pushq %rax
2135
+ leaq -(ARGOFFSET-16)(%rsp),%rdi # regs for handler
2136
+ call __ipipe_syscall_root_thunk
2137
+ testl %eax, %eax
2138
+ popq %rax
2139
+ popq %rdi
2140
+ js ret_from_sys_call
2141
+ jnz sysret_fastexit
2142
+#endif
2143
GET_THREAD_INFO(%rcx)
2144
testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags(%rcx)
2145
jnz tracesys
2146
@@ -357,6 +377,7 @@ sysret_check:
2147
* sysretq will re-enable interrupts:
2148
*/
2149
TRACE_IRQS_ON
2150
+sysret_fastexit:
2151
movq RIP-ARGOFFSET(%rsp),%rcx
2152
CFI_REGISTER rip,rcx
2153
RESTORE_ARGS 0,-ARG_SKIP,1
2154
@@ -368,6 +389,8 @@ sysret_check:
2155
/* Handle reschedules */
2156
/* edx: work, edi: workmask */
2157
sysret_careful:
2158
+ testl $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SECCOMP),%edx
2159
+ jnz ret_from_sys_call_trace
2160
bt $TIF_NEED_RESCHED,%edx
2161
jnc sysret_signal
2162
TRACE_IRQS_ON
2163
@@ -379,6 +402,16 @@ sysret_careful:
2164
CFI_ADJUST_CFA_OFFSET -8
2165
jmp sysret_check
2166
2167
+ret_from_sys_call_trace:
2168
+ TRACE_IRQS_ON
2169
+ sti
2170
+ SAVE_REST
2171
+ FIXUP_TOP_OF_STACK %rdi
2172
+ movq %rsp,%rdi
2173
+ LOAD_ARGS ARGOFFSET /* reload args from stack in case ptrace changed it */
2174
+ RESTORE_REST
2175
+ jmp int_ret_from_sys_call
2176
+
2177
/* Handle a signal */
2178
sysret_signal:
2179
TRACE_IRQS_ON
2180
@@ -667,12 +700,51 @@ END(stub_rt_sigreturn)
2181
* We entered an interrupt context - irqs are off:
2182
*/
2183
TRACE_IRQS_OFF
2184
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
2185
+ leaq RIP-8(%rdi), %rbp # make interrupted address show up in trace
2186
+ pushq %rdi
2187
+ movq ORIG_RAX(%rdi), %rdi # IRQ number
2188
+ notq %rdi # ...is inverted, fix up
2189
+ call ipipe_trace_begin
2190
+ popq %rdi
2191
+ popq %rbp
2192
+ pushq %rbp
2193
+
2194
+ call \func
2195
+
2196
+ popq %rbp
2197
+ pushq %rbp
2198
+ pushq %rax
2199
+ movq 8-ARGOFFSET+ORIG_RAX(%rbp), %rdi
2200
+ leaq 8-ARGOFFSET+RIP-8(%rbp), %rbp
2201
+ notq %rdi
2202
+ call ipipe_trace_end
2203
+ popq %rax
2204
+ popq %rbp
2205
+ pushq %rbp
2206
+#else
2207
call \func
2208
+#endif
2209
.endm
2210
2211
ENTRY(common_interrupt)
2212
+#ifdef CONFIG_IPIPE
2213
+ XCPT_FRAME
2214
+ interrupt __ipipe_handle_irq
2215
+ testl %eax, %eax
2216
+ jnz ret_from_intr
2217
+ decl %gs:pda_irqcount
2218
+ leaveq
2219
+ CFI_DEF_CFA_REGISTER rsp
2220
+ CFI_ADJUST_CFA_OFFSET -8
2221
+ testl $3,CS-ARGOFFSET(%rsp)
2222
+ jz restore_args
2223
+ swapgs
2224
+ jmp restore_args
2225
+#else /* !CONFIG_IPIPE */
2226
XCPT_FRAME
2227
interrupt do_IRQ
2228
+#endif /* !CONFIG_IPIPE */
2229
/* 0(%rsp): oldrsp-ARGOFFSET */
2230
ret_from_intr:
2231
DISABLE_INTERRUPTS(CLBR_NONE)
2232
@@ -681,7 +753,7 @@ ret_from_intr:
2233
leaveq
2234
CFI_DEF_CFA_REGISTER rsp
2235
CFI_ADJUST_CFA_OFFSET -8
2236
-exit_intr:
2237
+ENTRY(exit_intr)
2238
GET_THREAD_INFO(%rcx)
2239
testl $3,CS-ARGOFFSET(%rsp)
2240
je retint_kernel
2241
@@ -701,20 +773,20 @@ retint_check:
2242
jnz retint_careful
2243
2244
retint_swapgs: /* return to user-space */
2245
+ TRACE_IRQS_IRETQ
2246
/*
2247
* The iretq could re-enable interrupts:
2248
*/
2249
- DISABLE_INTERRUPTS(CLBR_ANY)
2250
- TRACE_IRQS_IRETQ
2251
+retint_swapgs_notrace:
2252
SWAPGS
2253
+retint_noswapgs:
2254
jmp restore_args
2255
2256
retint_restore_args: /* return to kernel space */
2257
- DISABLE_INTERRUPTS(CLBR_ANY)
2258
+ TRACE_IRQS_IRETQ
2259
/*
2260
* The iretq could re-enable interrupts:
2261
*/
2262
- TRACE_IRQS_IRETQ
2263
restore_args:
2264
RESTORE_ARGS 0,8,0
2265
2266
@@ -796,7 +868,7 @@ ENTRY(retint_kernel)
2267
jnc retint_restore_args
2268
bt $9,EFLAGS-ARGOFFSET(%rsp) /* interrupts off? */
2269
jnc retint_restore_args
2270
- call preempt_schedule_irq
2271
+ PREEMPT_SCHEDULE_IRQ
2272
jmp exit_intr
2273
#endif
2274
2275
@@ -805,7 +877,26 @@ END(common_interrupt)
2276
2277
/*
2278
* APIC interrupts.
2279
- */
2280
+ */
2281
+#ifdef CONFIG_IPIPE
2282
+ .macro apicinterrupt num,func
2283
+ INTR_FRAME
2284
+ pushq $~(\num)
2285
+ CFI_ADJUST_CFA_OFFSET 8
2286
+ interrupt __ipipe_handle_irq
2287
+ testl %eax, %eax
2288
+ jnz ret_from_intr
2289
+ decl %gs:pda_irqcount
2290
+ leaveq
2291
+ CFI_DEF_CFA_REGISTER rsp
2292
+ CFI_ADJUST_CFA_OFFSET -8
2293
+ testl $3,CS-ARGOFFSET(%rsp)
2294
+ jz restore_args
2295
+ swapgs
2296
+ jmp restore_args
2297
+ CFI_ENDPROC
2298
+ .endm
2299
+#else /* !CONFIG_IPIPE */
2300
.macro apicinterrupt num,func
2301
INTR_FRAME
2302
pushq $~(\num)
2303
@@ -814,6 +905,7 @@ END(common_interrupt)
2304
jmp ret_from_intr
2305
CFI_ENDPROC
2306
.endm
2307
+#endif /* !CONFIG_IPIPE */
2308
2309
ENTRY(thermal_interrupt)
2310
apicinterrupt THERMAL_APIC_VECTOR,smp_thermal_interrupt
2311
@@ -838,10 +930,12 @@ END(invalidate_interrupt\num)
2312
INVALIDATE_ENTRY 1
2313
INVALIDATE_ENTRY 2
2314
INVALIDATE_ENTRY 3
2315
+#ifndef CONFIG_IPIPE
2316
INVALIDATE_ENTRY 4
2317
INVALIDATE_ENTRY 5
2318
INVALIDATE_ENTRY 6
2319
INVALIDATE_ENTRY 7
2320
+#endif /* !CONFIG_IPIPE */
2321
2322
ENTRY(call_function_interrupt)
2323
apicinterrupt CALL_FUNCTION_VECTOR,smp_call_function_interrupt
2324
@@ -873,7 +967,7 @@ END(spurious_interrupt)
2325
/*
2326
* Exception entry points.
2327
*/
2328
- .macro zeroentry sym
2329
+ .macro zeroentry sym, ex_code
2330
INTR_FRAME
2331
PARAVIRT_ADJUST_EXCEPTION_FRAME
2332
pushq $0 /* push error code/oldrax */
2333
@@ -881,25 +975,33 @@ END(spurious_interrupt)
2334
pushq %rax /* push real oldrax to the rdi slot */
2335
CFI_ADJUST_CFA_OFFSET 8
2336
CFI_REL_OFFSET rax,0
2337
+#ifdef CONFIG_IPIPE
2338
+ movq $\ex_code,%rax
2339
+#else
2340
leaq \sym(%rip),%rax
2341
+#endif
2342
jmp error_entry
2343
CFI_ENDPROC
2344
.endm
2345
2346
- .macro errorentry sym
2347
+ .macro errorentry sym, ex_code
2348
XCPT_FRAME
2349
PARAVIRT_ADJUST_EXCEPTION_FRAME
2350
pushq %rax
2351
CFI_ADJUST_CFA_OFFSET 8
2352
CFI_REL_OFFSET rax,0
2353
+#ifdef CONFIG_IPIPE
2354
+ movq $\ex_code,%rax
2355
+#else
2356
leaq \sym(%rip),%rax
2357
+#endif
2358
jmp error_entry
2359
CFI_ENDPROC
2360
.endm
2361
2362
/* error code is on the stack already */
2363
/* handle NMI like exceptions that can happen everywhere */
2364
- .macro paranoidentry sym, ist=0, irqtrace=1
2365
+ .macro paranoidentry sym, ist=0, irqtrace=1, ex_code=0
2366
SAVE_ALL
2367
cld
2368
movl $1,%ebx
2369
@@ -922,6 +1024,16 @@ END(spurious_interrupt)
2370
.if \ist
2371
subq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
2372
.endif
2373
+#ifdef CONFIG_IPIPE
2374
+ .if \ex_code
2375
+ movq $\ex_code,%rsi
2376
+ call __ipipe_divert_exception /* handle(regs, ex_code) */
2377
+ testl %eax,%eax
2378
+ jnz 3f
2379
+ movq %rsp,%rdi
2380
+ movq ORIG_RAX(%rsp),%rsi
2381
+ .endif
2382
+#endif
2383
call \sym
2384
.if \ist
2385
addq $EXCEPTION_STKSZ, per_cpu__init_tss + TSS_ist + (\ist - 1) * 8(%rbp)
2386
@@ -930,6 +1042,12 @@ END(spurious_interrupt)
2387
.if \irqtrace
2388
TRACE_IRQS_OFF
2389
.endif
2390
+#ifdef CONFIG_IPIPE
2391
+ xorl %eax,%eax
2392
+ /* WARNING: paranoidexit will check %eax to decide whether the
2393
+ exception has to be propagated or not. */
2394
+#endif
2395
+3:
2396
.endm
2397
2398
/*
2399
@@ -949,6 +1067,10 @@ END(spurious_interrupt)
2400
paranoid_exit\trace:
2401
testl %ebx,%ebx /* swapgs needed? */
2402
jnz paranoid_restore\trace
2403
+#ifdef CONFIG_IPIPE
2404
+ testl %eax,%eax
2405
+ jnz paranoid_swapgs\trace
2406
+#endif
2407
testl $3,CS(%rsp)
2408
jnz paranoid_userspace\trace
2409
paranoid_swapgs\trace:
2410
@@ -1049,7 +1171,19 @@ error_sti:
2411
movq %rsp,%rdi
2412
movq ORIG_RAX(%rsp),%rsi /* get error code */
2413
movq $-1,ORIG_RAX(%rsp)
2414
+#ifdef CONFIG_IPIPE
2415
+ movq %rax, %rdx
2416
+ call __ipipe_handle_exception /* handle(regs, error_code, ex_code) */
2417
+ testl %eax, %eax
2418
+ jz error_exit
2419
+ movl %ebx,%eax
2420
+ RESTORE_REST
2421
+ testl %eax,%eax
2422
+ jne retint_noswapgs
2423
+ jmp retint_swapgs_notrace
2424
+#else /* !CONFIG_IPIPE */
2425
call *%rax
2426
+#endif /* !CONFIG_IPIPE */
2427
/* ebx: no swapgs flag (1: don't need swapgs, 0: need it) */
2428
error_exit:
2429
movl %ebx,%eax
2430
@@ -1205,19 +1339,19 @@ ENTRY(kernel_execve)
2431
ENDPROC(kernel_execve)
2432
2433
KPROBE_ENTRY(page_fault)
2434
- errorentry do_page_fault
2435
+ errorentry do_page_fault ex_do_page_fault
2436
KPROBE_END(page_fault)
2437
2438
ENTRY(coprocessor_error)
2439
- zeroentry do_coprocessor_error
2440
+ zeroentry do_coprocessor_error ex_do_coprocessor_error
2441
END(coprocessor_error)
2442
2443
ENTRY(simd_coprocessor_error)
2444
- zeroentry do_simd_coprocessor_error
2445
+ zeroentry do_simd_coprocessor_error ex_do_simd_coprocessor_error
2446
END(simd_coprocessor_error)
2447
2448
ENTRY(device_not_available)
2449
- zeroentry do_device_not_available
2450
+ zeroentry do_device_not_available ex_do_device_not_available
2451
END(device_not_available)
2452
2453
/* runs on exception stack */
2454
@@ -1226,7 +1360,7 @@ KPROBE_ENTRY(debug)
2455
PARAVIRT_ADJUST_EXCEPTION_FRAME
2456
pushq $0
2457
CFI_ADJUST_CFA_OFFSET 8
2458
- paranoidentry do_debug, DEBUG_STACK
2459
+ paranoidentry do_debug, DEBUG_STACK, 1, ex_do_debug
2460
paranoidexit
2461
KPROBE_END(debug)
2462
2463
@@ -1250,25 +1384,25 @@ KPROBE_ENTRY(int3)
2464
PARAVIRT_ADJUST_EXCEPTION_FRAME
2465
pushq $0
2466
CFI_ADJUST_CFA_OFFSET 8
2467
- paranoidentry do_int3, DEBUG_STACK
2468
+ paranoidentry do_int3, DEBUG_STACK, 1, ex_do_int3
2469
jmp paranoid_exit1
2470
CFI_ENDPROC
2471
KPROBE_END(int3)
2472
2473
ENTRY(overflow)
2474
- zeroentry do_overflow
2475
+ zeroentry do_overflow ex_do_overflow
2476
END(overflow)
2477
2478
ENTRY(bounds)
2479
- zeroentry do_bounds
2480
+ zeroentry do_bounds ex_do_bounds
2481
END(bounds)
2482
2483
ENTRY(invalid_op)
2484
- zeroentry do_invalid_op
2485
+ zeroentry do_invalid_op ex_do_invalid_op
2486
END(invalid_op)
2487
2488
ENTRY(coprocessor_segment_overrun)
2489
- zeroentry do_coprocessor_segment_overrun
2490
+ zeroentry do_coprocessor_segment_overrun ex_do_coprocessor_segment_overrun
2491
END(coprocessor_segment_overrun)
2492
2493
/* runs on exception stack */
2494
@@ -1281,11 +1415,11 @@ ENTRY(double_fault)
2495
END(double_fault)
2496
2497
ENTRY(invalid_TSS)
2498
- errorentry do_invalid_TSS
2499
+ errorentry do_invalid_TSS ex_do_invalid_TSS
2500
END(invalid_TSS)
2501
2502
ENTRY(segment_not_present)
2503
- errorentry do_segment_not_present
2504
+ errorentry do_segment_not_present ex_do_segment_not_present
2505
END(segment_not_present)
2506
2507
/* runs on exception stack */
2508
@@ -1298,19 +1432,19 @@ ENTRY(stack_segment)
2509
END(stack_segment)
2510
2511
KPROBE_ENTRY(general_protection)
2512
- errorentry do_general_protection
2513
+ errorentry do_general_protection ex_do_general_protection
2514
KPROBE_END(general_protection)
2515
2516
ENTRY(alignment_check)
2517
- errorentry do_alignment_check
2518
+ errorentry do_alignment_check ex_do_alignment_check
2519
END(alignment_check)
2520
2521
ENTRY(divide_error)
2522
- zeroentry do_divide_error
2523
+ zeroentry do_divide_error ex_do_divide_error
2524
END(divide_error)
2525
2526
ENTRY(spurious_interrupt_bug)
2527
- zeroentry do_spurious_interrupt_bug
2528
+ zeroentry do_spurious_interrupt_bug ex_do_spurious_interrupt_bug
2529
END(spurious_interrupt_bug)
2530
2531
#ifdef CONFIG_X86_MCE
2532
diff --git a/arch/x86/kernel/genapic_flat_64.c b/arch/x86/kernel/genapic_flat_64.c
2533
index c026279..8cfe478 100644
2534
--- a/arch/x86/kernel/genapic_flat_64.c
2535
+++ b/arch/x86/kernel/genapic_flat_64.c
2536
@@ -74,9 +74,9 @@ static void flat_send_IPI_mask(cpumask_t cpumask, int vector)
2537
unsigned long mask = cpus_addr(cpumask)[0];
2538
unsigned long flags;
2539
2540
- local_irq_save(flags);
2541
+ local_irq_save_hw(flags);
2542
__send_IPI_dest_field(mask, vector, APIC_DEST_LOGICAL);
2543
- local_irq_restore(flags);
2544
+ local_irq_restore_hw(flags);
2545
}
2546
2547
static void flat_send_IPI_allbutself(int vector)
2548
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
2549
index 9461dc3..6421209 100644
2550
--- a/arch/x86/kernel/head64.c
2551
+++ b/arch/x86/kernel/head64.c
2552
@@ -72,7 +72,7 @@ static void __init copy_bootdata(char *real_mode_data)
2553
}
2554
}
2555
2556
-void __init x86_64_start_kernel(char * real_mode_data)
2557
+void __init notrace x86_64_start_kernel(char * real_mode_data)
2558
{
2559
int i;
2560
2561
diff --git a/arch/x86/kernel/i8253.c b/arch/x86/kernel/i8253.c
2562
index c1b5e3e..810be3e 100644
2563
--- a/arch/x86/kernel/i8253.c
2564
+++ b/arch/x86/kernel/i8253.c
2565
@@ -8,6 +8,7 @@
2566
#include <linux/jiffies.h>
2567
#include <linux/module.h>
2568
#include <linux/spinlock.h>
2569
+#include <linux/ipipe.h>
2570
2571
#include <asm/smp.h>
2572
#include <asm/delay.h>
2573
@@ -139,6 +140,12 @@ static cycle_t pit_read(void)
2574
static int old_count;
2575
static u32 old_jifs;
2576
2577
+#ifdef CONFIG_IPIPE
2578
+ if (!__ipipe_pipeline_head_p(ipipe_root_domain))
2579
+ /* We don't really own the PIT. */
2580
+ return (cycle_t)(jiffies * LATCH) + (LATCH - 1) - old_count;
2581
+#endif /* CONFIG_IPIPE */
2582
+
2583
spin_lock_irqsave(&i8253_lock, flags);
2584
/*
2585
* Although our caller may have the read side of xtime_lock,
2586
diff --git a/arch/x86/kernel/i8259.c b/arch/x86/kernel/i8259.c
2587
index 4b8a53d..e5c525d 100644
2588
--- a/arch/x86/kernel/i8259.c
2589
+++ b/arch/x86/kernel/i8259.c
2590
@@ -33,7 +33,7 @@
2591
*/
2592
2593
static int i8259A_auto_eoi;
2594
-DEFINE_SPINLOCK(i8259A_lock);
2595
+IPIPE_DEFINE_SPINLOCK(i8259A_lock);
2596
static void mask_and_ack_8259A(unsigned int);
2597
2598
struct irq_chip i8259A_chip = {
2599
@@ -70,6 +70,7 @@ void disable_8259A_irq(unsigned int irq)
2600
unsigned long flags;
2601
2602
spin_lock_irqsave(&i8259A_lock, flags);
2603
+ ipipe_irq_lock(irq);
2604
cached_irq_mask |= mask;
2605
if (irq & 8)
2606
outb(cached_slave_mask, PIC_SLAVE_IMR);
2607
@@ -80,15 +81,18 @@ void disable_8259A_irq(unsigned int irq)
2608
2609
void enable_8259A_irq(unsigned int irq)
2610
{
2611
- unsigned int mask = ~(1 << irq);
2612
+ unsigned int mask = (1 << irq);
2613
unsigned long flags;
2614
2615
spin_lock_irqsave(&i8259A_lock, flags);
2616
- cached_irq_mask &= mask;
2617
- if (irq & 8)
2618
- outb(cached_slave_mask, PIC_SLAVE_IMR);
2619
- else
2620
- outb(cached_master_mask, PIC_MASTER_IMR);
2621
+ if (cached_irq_mask & mask) {
2622
+ cached_irq_mask &= ~mask;
2623
+ if (irq & 8)
2624
+ outb(cached_slave_mask, PIC_SLAVE_IMR);
2625
+ else
2626
+ outb(cached_master_mask, PIC_MASTER_IMR);
2627
+ ipipe_irq_unlock(irq);
2628
+ }
2629
spin_unlock_irqrestore(&i8259A_lock, flags);
2630
}
2631
2632
@@ -169,6 +173,18 @@ static void mask_and_ack_8259A(unsigned int irq)
2633
*/
2634
if (cached_irq_mask & irqmask)
2635
goto spurious_8259A_irq;
2636
+#ifdef CONFIG_IPIPE
2637
+ if (irq == 0) {
2638
+ /*
2639
+ * Fast timer ack -- don't mask (unless supposedly
2640
+ * spurious). We trace outb's in order to detect
2641
+ * broken hardware inducing large delays.
2642
+ */
2643
+ outb(0x60, PIC_MASTER_CMD); /* Specific EOI to master. */
2644
+ spin_unlock_irqrestore(&i8259A_lock, flags);
2645
+ return;
2646
+ }
2647
+#endif /* CONFIG_IPIPE */
2648
cached_irq_mask |= irqmask;
2649
2650
handle_real_irq:
2651
diff --git a/arch/x86/kernel/io_apic.c b/arch/x86/kernel/io_apic.c
2652
index 9043251..0fa9b05 100644
2653
--- a/arch/x86/kernel/io_apic.c
2654
+++ b/arch/x86/kernel/io_apic.c
2655
@@ -73,8 +73,12 @@
2656
*/
2657
int sis_apic_bug = -1;
2658
2659
-static DEFINE_SPINLOCK(ioapic_lock);
2660
-static DEFINE_SPINLOCK(vector_lock);
2661
+static IPIPE_DEFINE_SPINLOCK(ioapic_lock);
2662
+static IPIPE_DEFINE_SPINLOCK(vector_lock);
2663
+
2664
+#if defined(CONFIG_IPIPE) && defined(CONFIG_X86_32)
2665
+volatile unsigned long bugous_edge_irq_triggers[(NR_IRQS + BITS_PER_LONG - 1) / BITS_PER_LONG];
2666
+#endif
2667
2668
/*
2669
* # of IRQ routing registers
2670
@@ -151,6 +155,14 @@ static struct irq_cfg *irq_cfg_alloc(unsigned int irq)
2671
return irq_cfg(irq);
2672
}
2673
2674
+#ifdef CONFIG_IPIPE
2675
+unsigned __ipipe_get_ioapic_irq_vector(int irq)
2676
+{
2677
+ return irq >= IPIPE_FIRST_APIC_IRQ && irq < IPIPE_NR_XIRQS ?
2678
+ ipipe_apic_irq_vector(irq) : irq_cfg(irq)->vector;
2679
+}
2680
+#endif /* CONFIG_IPIPE */
2681
+
2682
/*
2683
* Rough estimation of how many shared IRQs there are, can be changed
2684
* anytime.
2685
@@ -520,6 +532,7 @@ static void mask_IO_APIC_irq (unsigned int irq)
2686
unsigned long flags;
2687
2688
spin_lock_irqsave(&ioapic_lock, flags);
2689
+ ipipe_irq_lock(irq);
2690
__mask_IO_APIC_irq(irq);
2691
spin_unlock_irqrestore(&ioapic_lock, flags);
2692
}
2693
@@ -529,7 +542,13 @@ static void unmask_IO_APIC_irq (unsigned int irq)
2694
unsigned long flags;
2695
2696
spin_lock_irqsave(&ioapic_lock, flags);
2697
+#if defined(CONFIG_IPIPE) && defined(CONFIG_X86_32)
2698
+ if (test_and_clear_bit(irq, &bugous_edge_irq_triggers[0]))
2699
+ __unmask_and_level_IO_APIC_irq(irq);
2700
+ else
2701
+#endif
2702
__unmask_IO_APIC_irq(irq);
2703
+ ipipe_irq_unlock(irq);
2704
spin_unlock_irqrestore(&ioapic_lock, flags);
2705
}
2706
2707
@@ -559,8 +578,10 @@ static void clear_IO_APIC (void)
2708
#if !defined(CONFIG_SMP) && defined(CONFIG_X86_32)
2709
void send_IPI_self(int vector)
2710
{
2711
+ unsigned long flags;
2712
unsigned int cfg;
2713
2714
+ local_irq_save_hw_cond(flags);
2715
/*
2716
* Wait for idle.
2717
*/
2718
@@ -570,6 +591,7 @@ void send_IPI_self(int vector)
2719
* Send the IPI. The write to APIC_ICR fires this off.
2720
*/
2721
apic_write(APIC_ICR, cfg);
2722
+ local_irq_restore_hw_cond(flags);
2723
}
2724
#endif /* !CONFIG_SMP && CONFIG_X86_32*/
2725
2726
@@ -2030,6 +2052,7 @@ static unsigned int startup_ioapic_irq(unsigned int irq)
2727
was_pending = 1;
2728
}
2729
__unmask_IO_APIC_irq(irq);
2730
+ ipipe_irq_unlock(irq);
2731
spin_unlock_irqrestore(&ioapic_lock, flags);
2732
2733
return was_pending;
2734
@@ -2284,11 +2307,37 @@ static void ack_x2apic_edge(unsigned int irq)
2735
}
2736
#endif
2737
2738
+#if defined(CONFIG_IPIPE) && defined(CONFIG_SMP)
2739
+static void move_apic_irq(unsigned int irq)
2740
+{
2741
+ struct irq_desc *desc = irq_to_desc(irq);
2742
+
2743
+ if (desc->handle_irq == &handle_edge_irq) {
2744
+ spin_lock(&desc->lock);
2745
+ irq_complete_move(irq);
2746
+ move_native_irq(irq);
2747
+ spin_unlock(&desc->lock);
2748
+ } else if (desc->handle_irq == &handle_fasteoi_irq) {
2749
+ spin_lock(&desc->lock);
2750
+ irq_complete_move(irq);
2751
+ if (unlikely(desc->status & IRQ_MOVE_PENDING)) {
2752
+ if (!io_apic_level_ack_pending(irq))
2753
+ move_masked_irq(irq);
2754
+ unmask_IO_APIC_irq(irq);
2755
+ }
2756
+ spin_unlock(&desc->lock);
2757
+ } else
2758
+ WARN_ON_ONCE(1);
2759
+}
2760
+#endif /* CONFIG_IPIPE */
2761
+
2762
static void ack_apic_edge(unsigned int irq)
2763
{
2764
+#ifndef CONFIG_IPIPE
2765
irq_complete_move(irq);
2766
move_native_irq(irq);
2767
- ack_APIC_irq();
2768
+#endif /* CONFIG_IPIPE */
2769
+ __ack_APIC_irq();
2770
}
2771
2772
atomic_t irq_mis_count;
2773
@@ -2299,6 +2348,7 @@ static void ack_apic_level(unsigned int irq)
2774
unsigned long v;
2775
int i;
2776
#endif
2777
+#ifndef CONFIG_IPIPE
2778
int do_unmask_irq = 0;
2779
2780
irq_complete_move(irq);
2781
@@ -2373,7 +2423,6 @@ static void ack_apic_level(unsigned int irq)
2782
move_masked_irq(irq);
2783
unmask_IO_APIC_irq(irq);
2784
}
2785
-
2786
#ifdef CONFIG_X86_32
2787
if (!(v & (1 << (i & 0x1f)))) {
2788
atomic_inc(&irq_mis_count);
2789
@@ -2383,6 +2432,30 @@ static void ack_apic_level(unsigned int irq)
2790
spin_unlock(&ioapic_lock);
2791
}
2792
#endif
2793
+#else /* CONFIG_IPIPE */
2794
+#ifdef CONFIG_X86_32
2795
+ i = irq_cfg(irq)->vector;
2796
+ v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));
2797
+ spin_lock(&ioapic_lock);
2798
+ if (unlikely(!(v & (1 << (i & 0x1f))))) {
2799
+ /* IO-APIC erratum: see comment below. */
2800
+ atomic_inc(&irq_mis_count);
2801
+ __mask_and_edge_IO_APIC_irq(irq);
2802
+ set_bit(irq, &bugous_edge_irq_triggers[0]);
2803
+ } else
2804
+ __mask_IO_APIC_irq(irq);
2805
+#else /* CONFIG_X86_64 */
2806
+ /*
2807
+ * Prevent low priority IRQs grabbed by high priority domains
2808
+ * from being delayed, waiting for a high priority interrupt
2809
+ * handler running in a low priority domain to complete.
2810
+ */
2811
+ spin_lock(&ioapic_lock);
2812
+ __mask_IO_APIC_irq(irq);
2813
+#endif /* CONFIG_X86_64 */
2814
+ spin_unlock(&ioapic_lock);
2815
+ __ack_APIC_irq();
2816
+#endif /* CONFIG_IPIPE */
2817
}
2818
2819
static struct irq_chip ioapic_chip __read_mostly = {
2820
@@ -2394,6 +2467,9 @@ static struct irq_chip ioapic_chip __read_mostly = {
2821
.eoi = ack_apic_level,
2822
#ifdef CONFIG_SMP
2823
.set_affinity = set_ioapic_affinity_irq,
2824
+#ifdef CONFIG_IPIPE
2825
+ .move = move_apic_irq,
2826
+#endif
2827
#endif
2828
.retrigger = ioapic_retrigger_irq,
2829
};
2830
@@ -2454,23 +2530,29 @@ static inline void init_IO_APIC_traps(void)
2831
2832
static void mask_lapic_irq(unsigned int irq)
2833
{
2834
- unsigned long v;
2835
+ unsigned long v, flags;
2836
2837
+ local_irq_save_hw_cond(flags);
2838
+ ipipe_irq_lock(irq);
2839
v = apic_read(APIC_LVT0);
2840
apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
2841
+ local_irq_restore_hw_cond(flags);
2842
}
2843
2844
static void unmask_lapic_irq(unsigned int irq)
2845
{
2846
- unsigned long v;
2847
+ unsigned long v, flags;
2848
2849
+ local_irq_save_hw_cond(flags);
2850
v = apic_read(APIC_LVT0);
2851
apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
2852
+ ipipe_irq_unlock(irq);
2853
+ local_irq_restore_hw_cond(flags);
2854
}
2855
2856
static void ack_lapic_irq (unsigned int irq)
2857
{
2858
- ack_APIC_irq();
2859
+ __ack_APIC_irq();
2860
}
2861
2862
static struct irq_chip lapic_chip __read_mostly = {
2863
@@ -2478,6 +2560,9 @@ static struct irq_chip lapic_chip __read_mostly = {
2864
.mask = mask_lapic_irq,
2865
.unmask = unmask_lapic_irq,
2866
.ack = ack_lapic_irq,
2867
+#if defined(CONFIG_IPIPE) && defined(CONFIG_SMP)
2868
+ .move = move_apic_irq,
2869
+#endif
2870
};
2871
2872
static void lapic_register_intr(int irq)
2873
@@ -2718,6 +2803,10 @@ static inline void __init check_timer(void)
2874
"...trying to set up timer as Virtual Wire IRQ...\n");
2875
2876
lapic_register_intr(0);
2877
+#if defined(CONFIG_IPIPE) && defined(CONFIG_X86_64)
2878
+ irq_to_desc(0)->ipipe_ack = __ipipe_ack_edge_irq;
2879
+ irq_to_desc(0)->ipipe_end = __ipipe_end_edge_irq;
2880
+#endif
2881
apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector); /* Fixed mode */
2882
enable_8259A_irq(0);
2883
2884
@@ -3122,6 +3211,9 @@ static struct irq_chip msi_chip = {
2885
.ack = ack_apic_edge,
2886
#ifdef CONFIG_SMP
2887
.set_affinity = set_msi_irq_affinity,
2888
+#ifdef CONFIG_IPIPE
2889
+ .move = move_apic_irq,
2890
+#endif
2891
#endif
2892
.retrigger = ioapic_retrigger_irq,
2893
};
2894
@@ -3347,6 +3439,9 @@ struct irq_chip dmar_msi_type = {
2895
.ack = ack_apic_edge,
2896
#ifdef CONFIG_SMP
2897
.set_affinity = dmar_msi_set_affinity,
2898
+#ifdef CONFIG_IPIPE
2899
+ .move = move_apic_irq,
2900
+#endif
2901
#endif
2902
.retrigger = ioapic_retrigger_irq,
2903
};
2904
@@ -3482,6 +3577,9 @@ static struct irq_chip ht_irq_chip = {
2905
.ack = ack_apic_edge,
2906
#ifdef CONFIG_SMP
2907
.set_affinity = set_ht_irq_affinity,
2908
+#ifdef CONFIG_IPIPE
2909
+ .move = move_apic_irq,
2910
+#endif
2911
#endif
2912
.retrigger = ioapic_retrigger_irq,
2913
};
2914
diff --git a/arch/x86/kernel/ipi.c b/arch/x86/kernel/ipi.c
2915
index f1c688e..484928c 100644
2916
--- a/arch/x86/kernel/ipi.c
2917
+++ b/arch/x86/kernel/ipi.c
2918
@@ -58,6 +58,9 @@ void __send_IPI_shortcut(unsigned int shortcut, int vector)
2919
* to the APIC.
2920
*/
2921
unsigned int cfg;
2922
+ unsigned long flags;
2923
+
2924
+ local_irq_save_hw_cond(flags);
2925
2926
/*
2927
* Wait for idle.
2928
@@ -73,6 +76,8 @@ void __send_IPI_shortcut(unsigned int shortcut, int vector)
2929
* Send the IPI. The write to APIC_ICR fires this off.
2930
*/
2931
apic_write(APIC_ICR, cfg);
2932
+
2933
+ local_irq_restore_hw_cond(flags);
2934
}
2935
2936
void send_IPI_self(int vector)
2937
@@ -86,8 +91,9 @@ void send_IPI_self(int vector)
2938
*/
2939
static inline void __send_IPI_dest_field(unsigned long mask, int vector)
2940
{
2941
- unsigned long cfg;
2942
+ unsigned long cfg, flags;
2943
2944
+ local_irq_save_hw_cond(flags);
2945
/*
2946
* Wait for idle.
2947
*/
2948
@@ -111,6 +117,8 @@ static inline void __send_IPI_dest_field(unsigned long mask, int vector)
2949
* Send the IPI. The write to APIC_ICR fires this off.
2950
*/
2951
apic_write(APIC_ICR, cfg);
2952
+
2953
+ local_irq_restore_hw_cond(flags);
2954
}
2955
2956
/*
2957
@@ -121,10 +129,10 @@ void send_IPI_mask_bitmask(cpumask_t cpumask, int vector)
2958
unsigned long mask = cpus_addr(cpumask)[0];
2959
unsigned long flags;
2960
2961
- local_irq_save(flags);
2962
+ local_irq_save_hw(flags);
2963
WARN_ON(mask & ~cpus_addr(cpu_online_map)[0]);
2964
__send_IPI_dest_field(mask, vector);
2965
- local_irq_restore(flags);
2966
+ local_irq_restore_hw(flags);
2967
}
2968
2969
void send_IPI_mask_sequence(cpumask_t mask, int vector)
2970
@@ -138,14 +146,14 @@ void send_IPI_mask_sequence(cpumask_t mask, int vector)
2971
* should be modified to do 1 message per cluster ID - mbligh
2972
*/
2973
2974
- local_irq_save(flags);
2975
+ local_irq_save_hw(flags);
2976
for_each_possible_cpu(query_cpu) {
2977
if (cpu_isset(query_cpu, mask)) {
2978
__send_IPI_dest_field(cpu_to_logical_apicid(query_cpu),
2979
vector);
2980
}
2981
}
2982
- local_irq_restore(flags);
2983
+ local_irq_restore_hw(flags);
2984
}
2985
2986
/* must come after the send_IPI functions above for inlining */
2987
diff --git a/arch/x86/kernel/ipipe.c b/arch/x86/kernel/ipipe.c
2988
new file mode 100644
2989
index 0000000..e2255b3
2990
--- /dev/null
2991
+++ b/arch/x86/kernel/ipipe.c
2992
@@ -0,0 +1,943 @@
2993
+/* -*- linux-c -*-
2994
+ * linux/arch/x86/kernel/ipipe.c
2995
+ *
2996
+ * Copyright (C) 2002-2007 Philippe Gerum.
2997
+ *
2998
+ * This program is free software; you can redistribute it and/or modify
2999
+ * it under the terms of the GNU General Public License as published by
3000
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
3001
+ * USA; either version 2 of the License, or (at your option) any later
3002
+ * version.
3003
+ *
3004
+ * This program is distributed in the hope that it will be useful,
3005
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
3006
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
3007
+ * GNU General Public License for more details.
3008
+ *
3009
+ * You should have received a copy of the GNU General Public License
3010
+ * along with this program; if not, write to the Free Software
3011
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
3012
+ *
3013
+ * Architecture-dependent I-PIPE support for x86.
3014
+ */
3015
+
3016
+#include <linux/kernel.h>
3017
+#include <linux/smp.h>
3018
+#include <linux/module.h>
3019
+#include <linux/sched.h>
3020
+#include <linux/interrupt.h>
3021
+#include <linux/slab.h>
3022
+#include <linux/irq.h>
3023
+#include <linux/clockchips.h>
3024
+#include <asm/unistd.h>
3025
+#include <asm/system.h>
3026
+#include <asm/atomic.h>
3027
+#include <asm/hw_irq.h>
3028
+#include <asm/irq.h>
3029
+#include <asm/desc.h>
3030
+#include <asm/io.h>
3031
+#ifdef CONFIG_X86_LOCAL_APIC
3032
+#include <asm/tlbflush.h>
3033
+#include <asm/fixmap.h>
3034
+#include <asm/bitops.h>
3035
+#include <asm/mpspec.h>
3036
+#ifdef CONFIG_X86_IO_APIC
3037
+#include <asm/io_apic.h>
3038
+#endif /* CONFIG_X86_IO_APIC */
3039
+#include <asm/apic.h>
3040
+#include <mach_ipi.h>
3041
+#endif /* CONFIG_X86_LOCAL_APIC */
3042
+#include <asm/traps.h>
3043
+
3044
+int __ipipe_tick_irq = 0; /* Legacy timer */
3045
+
3046
+DEFINE_PER_CPU(struct pt_regs, __ipipe_tick_regs);
3047
+
3048
+#ifdef CONFIG_SMP
3049
+
3050
+static cpumask_t __ipipe_cpu_sync_map;
3051
+
3052
+static cpumask_t __ipipe_cpu_lock_map;
3053
+
3054
+static IPIPE_DEFINE_SPINLOCK(__ipipe_cpu_barrier);
3055
+
3056
+static atomic_t __ipipe_critical_count = ATOMIC_INIT(0);
3057
+
3058
+static void (*__ipipe_cpu_sync) (void);
3059
+
3060
+#endif /* CONFIG_SMP */
3061
+
3062
+/*
3063
+ * ipipe_trigger_irq() -- Push the interrupt at front of the pipeline
3064
+ * just like if it has been actually received from a hw source. Also
3065
+ * works for virtual interrupts.
3066
+ */
3067
+int ipipe_trigger_irq(unsigned irq)
3068
+{
3069
+ struct pt_regs regs;
3070
+ unsigned long flags;
3071
+
3072
+ if (irq >= IPIPE_NR_IRQS ||
3073
+ (ipipe_virtual_irq_p(irq) &&
3074
+ !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map)))
3075
+ return -EINVAL;
3076
+
3077
+ local_irq_save_hw(flags);
3078
+ regs.flags = flags;
3079
+ regs.orig_ax = irq; /* Positive value - IRQ won't be acked */
3080
+ regs.cs = __KERNEL_CS;
3081
+ __ipipe_handle_irq(®s);
3082
+ local_irq_restore_hw(flags);
3083
+
3084
+ return 1;
3085
+}
3086
+
3087
+int ipipe_get_sysinfo(struct ipipe_sysinfo *info)
3088
+{
3089
+ info->ncpus = num_online_cpus();
3090
+ info->cpufreq = ipipe_cpu_freq();
3091
+ info->archdep.tmirq = __ipipe_tick_irq;
3092
+#ifdef CONFIG_X86_TSC
3093
+ info->archdep.tmfreq = ipipe_cpu_freq();
3094
+#else /* !CONFIG_X86_TSC */
3095
+ info->archdep.tmfreq = CLOCK_TICK_RATE;
3096
+#endif /* CONFIG_X86_TSC */
3097
+
3098
+ return 0;
3099
+}
3100
+
3101
+unsigned int do_IRQ(struct pt_regs *regs);
3102
+void smp_apic_timer_interrupt(struct pt_regs *regs);
3103
+void smp_spurious_interrupt(struct pt_regs *regs);
3104
+void smp_error_interrupt(struct pt_regs *regs);
3105
+void smp_thermal_interrupt(struct pt_regs *regs);
3106
+void smp_reschedule_interrupt(struct pt_regs *regs);
3107
+void smp_invalidate_interrupt(struct pt_regs *regs);
3108
+void smp_call_function_interrupt(struct pt_regs *regs);
3109
+void smp_call_function_single_interrupt(struct pt_regs *regs);
3110
+void mce_threshold_interrupt(struct pt_regs *regs);
3111
+void uv_bau_message_interrupt(struct pt_regs *regs);
3112
+
3113
+static void __ipipe_ack_irq(unsigned irq, struct irq_desc *desc)
3114
+{
3115
+ desc->ipipe_ack(irq, desc);
3116
+}
3117
+
3118
+void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq)
3119
+{
3120
+ irq_to_desc(irq)->status &= ~IRQ_DISABLED;
3121
+}
3122
+
3123
+#ifdef CONFIG_X86_LOCAL_APIC
3124
+
3125
+static void __ipipe_noack_apic(unsigned irq, struct irq_desc *desc)
3126
+{
3127
+}
3128
+
3129
+static void __ipipe_ack_apic(unsigned irq, struct irq_desc *desc)
3130
+{
3131
+ __ack_APIC_irq();
3132
+}
3133
+
3134
+static void __ipipe_null_handler(unsigned irq, void *cookie)
3135
+{
3136
+}
3137
+
3138
+#endif /* CONFIG_X86_LOCAL_APIC */
3139
+
3140
+/* __ipipe_enable_pipeline() -- We are running on the boot CPU, hw
3141
+ interrupts are off, and secondary CPUs are still lost in space. */
3142
+
3143
+void __init __ipipe_enable_pipeline(void)
3144
+{
3145
+ unsigned irq;
3146
+
3147
+#ifdef CONFIG_X86_LOCAL_APIC
3148
+
3149
+ /* Map the APIC system vectors. */
3150
+
3151
+ ipipe_virtualize_irq(ipipe_root_domain,
3152
+ ipipe_apic_vector_irq(LOCAL_TIMER_VECTOR),
3153
+ (ipipe_irq_handler_t)&smp_apic_timer_interrupt,
3154
+ NULL,
3155
+ &__ipipe_ack_apic,
3156
+ IPIPE_STDROOT_MASK);
3157
+
3158
+ ipipe_virtualize_irq(ipipe_root_domain,
3159
+ ipipe_apic_vector_irq(SPURIOUS_APIC_VECTOR),
3160
+ (ipipe_irq_handler_t)&smp_spurious_interrupt,
3161
+ NULL,
3162
+ &__ipipe_noack_apic,
3163
+ IPIPE_STDROOT_MASK);
3164
+
3165
+ ipipe_virtualize_irq(ipipe_root_domain,
3166
+ ipipe_apic_vector_irq(ERROR_APIC_VECTOR),
3167
+ (ipipe_irq_handler_t)&smp_error_interrupt,
3168
+ NULL,
3169
+ &__ipipe_ack_apic,
3170
+ IPIPE_STDROOT_MASK);
3171
+
3172
+ ipipe_virtualize_irq(ipipe_root_domain,
3173
+ ipipe_apic_vector_irq(IPIPE_SERVICE_VECTOR0),
3174
+ &__ipipe_null_handler,
3175
+ NULL,
3176
+ &__ipipe_ack_apic,
3177
+ IPIPE_STDROOT_MASK);
3178
+
3179
+ ipipe_virtualize_irq(ipipe_root_domain,
3180
+ ipipe_apic_vector_irq(IPIPE_SERVICE_VECTOR1),
3181
+ &__ipipe_null_handler,
3182
+ NULL,
3183
+ &__ipipe_ack_apic,
3184
+ IPIPE_STDROOT_MASK);
3185
+
3186
+ ipipe_virtualize_irq(ipipe_root_domain,
3187
+ ipipe_apic_vector_irq(IPIPE_SERVICE_VECTOR2),
3188
+ &__ipipe_null_handler,
3189
+ NULL,
3190
+ &__ipipe_ack_apic,
3191
+ IPIPE_STDROOT_MASK);
3192
+
3193
+ ipipe_virtualize_irq(ipipe_root_domain,
3194
+ ipipe_apic_vector_irq(IPIPE_SERVICE_VECTOR3),
3195
+ &__ipipe_null_handler,
3196
+ NULL,
3197
+ &__ipipe_ack_apic,
3198
+ IPIPE_STDROOT_MASK);
3199
+
3200
+#if defined(CONFIG_X86_MCE_P4THERMAL) || defined(CONFIG_X86_64)
3201
+ ipipe_virtualize_irq(ipipe_root_domain,
3202
+ ipipe_apic_vector_irq(THERMAL_APIC_VECTOR),
3203
+ (ipipe_irq_handler_t)&smp_thermal_interrupt,
3204
+ NULL,
3205
+ &__ipipe_ack_apic,
3206
+ IPIPE_STDROOT_MASK);
3207
+#endif /* CONFIG_X86_MCE_P4THERMAL || CONFIG_X86_64 */
3208
+
3209
+#ifdef CONFIG_X86_64
3210
+ ipipe_virtualize_irq(ipipe_root_domain,
3211
+ ipipe_apic_vector_irq(THRESHOLD_APIC_VECTOR),
3212
+ (ipipe_irq_handler_t)&mce_threshold_interrupt,
3213
+ NULL,
3214
+ &__ipipe_ack_apic,
3215
+ IPIPE_STDROOT_MASK);
3216
+
3217
+ ipipe_virtualize_irq(ipipe_root_domain,
3218
+ ipipe_apic_vector_irq(UV_BAU_MESSAGE),
3219
+ (ipipe_irq_handler_t)&uv_bau_message_interrupt,
3220
+ NULL,
3221
+ &__ipipe_ack_apic,
3222
+ IPIPE_STDROOT_MASK);
3223
+#endif /* CONFIG_X86_64 */
3224
+
3225
+#endif /* CONFIG_X86_LOCAL_APIC */
3226
+
3227
+#ifdef CONFIG_SMP
3228
+ ipipe_virtualize_irq(ipipe_root_domain,
3229
+ ipipe_apic_vector_irq(RESCHEDULE_VECTOR),
3230
+ (ipipe_irq_handler_t)&smp_reschedule_interrupt,
3231
+ NULL,
3232
+ &__ipipe_ack_apic,
3233
+ IPIPE_STDROOT_MASK);
3234
+
3235
+#ifdef CONFIG_X86_32
3236
+ ipipe_virtualize_irq(ipipe_root_domain,
3237
+ ipipe_apic_vector_irq(INVALIDATE_TLB_VECTOR),
3238
+ (ipipe_irq_handler_t)&smp_invalidate_interrupt,
3239
+ NULL,
3240
+ &__ipipe_ack_apic,
3241
+ IPIPE_STDROOT_MASK);
3242
+#else /* !CONFIG_X86_32 */
3243
+ {
3244
+ unsigned vector;
3245
+
3246
+ for (vector = INVALIDATE_TLB_VECTOR_START;
3247
+ vector <= INVALIDATE_TLB_VECTOR_END; ++vector)
3248
+ ipipe_virtualize_irq(ipipe_root_domain,
3249
+ ipipe_apic_vector_irq(vector),
3250
+ (ipipe_irq_handler_t)&smp_invalidate_interrupt,
3251
+ NULL,
3252
+ &__ipipe_ack_apic,
3253
+ IPIPE_STDROOT_MASK);
3254
+ }
3255
+#endif /* !CONFIG_X86_32 */
3256
+
3257
+ ipipe_virtualize_irq(ipipe_root_domain,
3258
+ ipipe_apic_vector_irq(CALL_FUNCTION_VECTOR),
3259
+ (ipipe_irq_handler_t)&smp_call_function_interrupt,
3260
+ NULL,
3261
+ &__ipipe_ack_apic,
3262
+ IPIPE_STDROOT_MASK);
3263
+
3264
+ ipipe_virtualize_irq(ipipe_root_domain,
3265
+ ipipe_apic_vector_irq(CALL_FUNCTION_SINGLE_VECTOR),
3266
+ (ipipe_irq_handler_t)&smp_call_function_single_interrupt,
3267
+ NULL,
3268
+ &__ipipe_ack_apic,
3269
+ IPIPE_STDROOT_MASK);
3270
+
3271
+#endif /* CONFIG_SMP */
3272
+
3273
+ /* Finally, virtualize the remaining ISA and IO-APIC
3274
+ * interrupts. Interrupts which have already been virtualized
3275
+ * will just beget a silent -EPERM error since
3276
+ * IPIPE_SYSTEM_MASK has been passed for them, that's ok. */
3277
+
3278
+ for (irq = 0; irq < NR_IRQS; irq++)
3279
+ /* Fails for IPIPE_CRITICAL_IPI but that's ok. */
3280
+ ipipe_virtualize_irq(ipipe_root_domain,
3281
+ irq,
3282
+ (ipipe_irq_handler_t)&do_IRQ,
3283
+ NULL,
3284
+ &__ipipe_ack_irq,
3285
+ IPIPE_STDROOT_MASK);
3286
+
3287
+#ifdef CONFIG_X86_LOCAL_APIC
3288
+ /* Eventually allow these vectors to be reprogrammed. */
3289
+ ipipe_root_domain->irqs[IPIPE_SERVICE_IPI0].control &= ~IPIPE_SYSTEM_MASK;
3290
+ ipipe_root_domain->irqs[IPIPE_SERVICE_IPI1].control &= ~IPIPE_SYSTEM_MASK;
3291
+ ipipe_root_domain->irqs[IPIPE_SERVICE_IPI2].control &= ~IPIPE_SYSTEM_MASK;
3292
+ ipipe_root_domain->irqs[IPIPE_SERVICE_IPI3].control &= ~IPIPE_SYSTEM_MASK;
3293
+#endif /* CONFIG_X86_LOCAL_APIC */
3294
+}
3295
+
3296
+#ifdef CONFIG_SMP
3297
+
3298
+cpumask_t __ipipe_set_irq_affinity(unsigned irq, cpumask_t cpumask)
3299
+{
3300
+ cpumask_t oldmask = irq_to_desc(irq)->affinity;
3301
+
3302
+ if (irq_to_desc(irq)->chip->set_affinity == NULL)
3303
+ return CPU_MASK_NONE;
3304
+
3305
+ if (cpus_empty(cpumask))
3306
+ return oldmask; /* Return mask value -- no change. */
3307
+
3308
+ cpus_and(cpumask,cpumask,cpu_online_map);
3309
+
3310
+ if (cpus_empty(cpumask))
3311
+ return CPU_MASK_NONE; /* Error -- bad mask value or non-routable IRQ. */
3312
+
3313
+ irq_to_desc(irq)->chip->set_affinity(irq,cpumask);
3314
+
3315
+ return oldmask;
3316
+}
3317
+
3318
+int __ipipe_send_ipi(unsigned ipi, cpumask_t cpumask)
3319
+{
3320
+ unsigned long flags;
3321
+ int self;
3322
+
3323
+ if (ipi != IPIPE_SERVICE_IPI0 &&
3324
+ ipi != IPIPE_SERVICE_IPI1 &&
3325
+ ipi != IPIPE_SERVICE_IPI2 &&
3326
+ ipi != IPIPE_SERVICE_IPI3)
3327
+ return -EINVAL;
3328
+
3329
+ local_irq_save_hw(flags);
3330
+
3331
+ self = cpu_isset(ipipe_processor_id(),cpumask);
3332
+ cpu_clear(ipipe_processor_id(), cpumask);
3333
+
3334
+ if (!cpus_empty(cpumask))
3335
+ send_IPI_mask(cpumask, ipipe_apic_irq_vector(ipi));
3336
+
3337
+ if (self)
3338
+ ipipe_trigger_irq(ipi);
3339
+
3340
+ local_irq_restore_hw(flags);
3341
+
3342
+ return 0;
3343
+}
3344
+
3345
+/* Always called with hw interrupts off. */
3346
+
3347
+void __ipipe_do_critical_sync(unsigned irq, void *cookie)
3348
+{
3349
+ int cpu = ipipe_processor_id();
3350
+
3351
+ cpu_set(cpu, __ipipe_cpu_sync_map);
3352
+
3353
+ /* Now we are in sync with the lock requestor running on another
3354
+ CPU. Enter a spinning wait until he releases the global
3355
+ lock. */
3356
+ spin_lock(&__ipipe_cpu_barrier);
3357
+
3358
+ /* Got it. Now get out. */
3359
+
3360
+ if (__ipipe_cpu_sync)
3361
+ /* Call the sync routine if any. */
3362
+ __ipipe_cpu_sync();
3363
+
3364
+ spin_unlock(&__ipipe_cpu_barrier);
3365
+
3366
+ cpu_clear(cpu, __ipipe_cpu_sync_map);
3367
+}
3368
+
3369
+void __ipipe_hook_critical_ipi(struct ipipe_domain *ipd)
3370
+{
3371
+ ipd->irqs[IPIPE_CRITICAL_IPI].acknowledge = &__ipipe_ack_apic;
3372
+ ipd->irqs[IPIPE_CRITICAL_IPI].handler = &__ipipe_do_critical_sync;
3373
+ ipd->irqs[IPIPE_CRITICAL_IPI].cookie = NULL;
3374
+ /* Immediately handle in the current domain but *never* pass */
3375
+ ipd->irqs[IPIPE_CRITICAL_IPI].control =
3376
+ IPIPE_HANDLE_MASK|IPIPE_STICKY_MASK|IPIPE_SYSTEM_MASK;
3377
+}
3378
+
3379
+#endif /* CONFIG_SMP */
3380
+
3381
+/* ipipe_critical_enter() -- Grab the superlock excluding all CPUs
3382
+ but the current one from a critical section. This lock is used when
3383
+ we must enforce a global critical section for a single CPU in a
3384
+ possibly SMP system whichever context the CPUs are running. */
3385
+
3386
+unsigned long ipipe_critical_enter(void (*syncfn) (void))
3387
+{
3388
+ unsigned long flags;
3389
+
3390
+ local_irq_save_hw(flags);
3391
+
3392
+#ifdef CONFIG_SMP
3393
+ if (unlikely(num_online_cpus() == 1)) /* We might be running a SMP-kernel on a UP box... */
3394
+ return flags;
3395
+
3396
+ {
3397
+ int cpu = ipipe_processor_id();
3398
+ cpumask_t lock_map;
3399
+
3400
+ if (!cpu_test_and_set(cpu, __ipipe_cpu_lock_map)) {
3401
+ while (cpu_test_and_set(BITS_PER_LONG - 1, __ipipe_cpu_lock_map)) {
3402
+ int n = 0;
3403
+ do {
3404
+ cpu_relax();
3405
+ } while (++n < cpu);
3406
+ }
3407
+
3408
+ spin_lock(&__ipipe_cpu_barrier);
3409
+
3410
+ __ipipe_cpu_sync = syncfn;
3411
+
3412
+ /* Send the sync IPI to all processors but the current one. */
3413
+ send_IPI_allbutself(IPIPE_CRITICAL_VECTOR);
3414
+
3415
+ cpus_andnot(lock_map, cpu_online_map, __ipipe_cpu_lock_map);
3416
+
3417
+ while (!cpus_equal(__ipipe_cpu_sync_map, lock_map))
3418
+ cpu_relax();
3419
+ }
3420
+
3421
+ atomic_inc(&__ipipe_critical_count);
3422
+ }
3423
+#endif /* CONFIG_SMP */
3424
+
3425
+ return flags;
3426
+}
3427
+
3428
+/* ipipe_critical_exit() -- Release the superlock. */
3429
+
3430
+void ipipe_critical_exit(unsigned long flags)
3431
+{
3432
+#ifdef CONFIG_SMP
3433
+ if (num_online_cpus() == 1)
3434
+ goto out;
3435
+
3436
+ if (atomic_dec_and_test(&__ipipe_critical_count)) {
3437
+ spin_unlock(&__ipipe_cpu_barrier);
3438
+
3439
+ while (!cpus_empty(__ipipe_cpu_sync_map))
3440
+ cpu_relax();
3441
+
3442
+ cpu_clear(ipipe_processor_id(), __ipipe_cpu_lock_map);
3443
+ cpu_clear(BITS_PER_LONG - 1, __ipipe_cpu_lock_map);
3444
+ }
3445
+out:
3446
+#endif /* CONFIG_SMP */
3447
+
3448
+ local_irq_restore_hw(flags);
3449
+}
3450
+
3451
+#ifdef CONFIG_X86_32
3452
+
3453
+static inline void __fixup_if(int s, struct pt_regs *regs)
3454
+{
3455
+ /*
3456
+ * Have the saved hw state look like the domain stall bit, so
3457
+ * that __ipipe_unstall_iret_root() restores the proper
3458
+ * pipeline state for the root stage upon exit.
3459
+ */
3460
+ if (s)
3461
+ regs->flags &= ~X86_EFLAGS_IF;
3462
+ else
3463
+ regs->flags |= X86_EFLAGS_IF;
3464
+}
3465
+
3466
+/*
3467
+ * Check the stall bit of the root domain to make sure the existing
3468
+ * preemption opportunity upon in-kernel resumption could be
3469
+ * exploited. In case a rescheduling could take place, the root stage
3470
+ * is stalled before the hw interrupts are re-enabled. This routine
3471
+ * must be called with hw interrupts off.
3472
+ */
3473
+
3474
+asmlinkage int __ipipe_kpreempt_root(struct pt_regs regs)
3475
+{
3476
+ if (test_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status)))
3477
+ /* Root stage is stalled: rescheduling denied. */
3478
+ return 0;
3479
+
3480
+ __ipipe_stall_root();
3481
+ local_irq_enable_hw_notrace();
3482
+
3483
+ return 1; /* Ok, may reschedule now. */
3484
+}
3485
+
3486
+asmlinkage void __ipipe_unstall_iret_root(struct pt_regs regs)
3487
+{
3488
+ struct ipipe_percpu_domain_data *p = ipipe_root_cpudom_ptr();
3489
+
3490
+ /* Emulate IRET's handling of the interrupt flag. */
3491
+
3492
+ local_irq_disable_hw();
3493
+
3494
+ /*
3495
+ * Restore the software state as it used to be on kernel
3496
+ * entry. CAUTION: NMIs must *not* return through this
3497
+ * emulation.
3498
+ */
3499
+ if (raw_irqs_disabled_flags(regs.flags)) {
3500
+ if (!__test_and_set_bit(IPIPE_STALL_FLAG, &p->status))
3501
+ trace_hardirqs_off();
3502
+ regs.flags |= X86_EFLAGS_IF;
3503
+ } else {
3504
+ if (test_bit(IPIPE_STALL_FLAG, &p->status)) {
3505
+ trace_hardirqs_on();
3506
+ __clear_bit(IPIPE_STALL_FLAG, &p->status);
3507
+ }
3508
+
3509
+ /*
3510
+ * Only sync virtual IRQs here, so that we don't
3511
+ * recurse indefinitely in case of an external
3512
+ * interrupt flood.
3513
+ */
3514
+ if ((p->irqpend_himask & IPIPE_IRQMASK_VIRT) != 0)
3515
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_VIRT);
3516
+ }
3517
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
3518
+ ipipe_trace_end(0x8000000D);
3519
+#endif /* CONFIG_IPIPE_TRACE_IRQSOFF */
3520
+}
3521
+
3522
+#else /* !CONFIG_X86_32 */
3523
+
3524
+static inline void __fixup_if(int s, struct pt_regs *regs)
3525
+{
3526
+}
3527
+
3528
+#ifdef CONFIG_PREEMPT
3529
+
3530
+asmlinkage void preempt_schedule_irq(void);
3531
+
3532
+void __ipipe_preempt_schedule_irq(void)
3533
+{
3534
+ struct ipipe_percpu_domain_data *p = ipipe_root_cpudom_ptr();
3535
+ int s;
3536
+
3537
+ local_irq_disable_hw();
3538
+ s = __test_and_set_bit(IPIPE_STALL_FLAG, &p->status);
3539
+ local_irq_enable_hw();
3540
+ preempt_schedule_irq(); /* Ok, may reschedule now. */
3541
+ local_irq_disable_hw();
3542
+ if (!s)
3543
+ __clear_bit(IPIPE_STALL_FLAG, &p->status);
3544
+}
3545
+#endif /* CONFIG_PREEMPT */
3546
+
3547
+#endif /* !CONFIG_X86_32 */
3548
+
3549
+static void do_machine_check_vector(struct pt_regs *regs, long error_code)
3550
+{
3551
+#ifdef CONFIG_X86_MCE
3552
+#ifdef CONFIG_X86_32
3553
+ extern void (*machine_check_vector)(struct pt_regs *, long error_code);
3554
+ machine_check_vector(regs, error_code);
3555
+#else
3556
+ do_machine_check(regs, error_code);
3557
+#endif
3558
+#endif /* CONFIG_X86_MCE */
3559
+}
3560
+
3561
+/* Work around genksyms's issue with over-qualification in decls. */
3562
+
3563
+typedef void dotraplinkage __ipipe_exhandler(struct pt_regs *, long);
3564
+
3565
+typedef __ipipe_exhandler *__ipipe_exptr;
3566
+
3567
+static __ipipe_exptr __ipipe_std_extable[] = {
3568
+
3569
+ [ex_do_divide_error] = &do_divide_error,
3570
+ [ex_do_overflow] = &do_overflow,
3571
+ [ex_do_bounds] = &do_bounds,
3572
+ [ex_do_invalid_op] = &do_invalid_op,
3573
+ [ex_do_coprocessor_segment_overrun] = &do_coprocessor_segment_overrun,
3574
+ [ex_do_invalid_TSS] = &do_invalid_TSS,
3575
+ [ex_do_segment_not_present] = &do_segment_not_present,
3576
+ [ex_do_stack_segment] = &do_stack_segment,
3577
+ [ex_do_general_protection] = do_general_protection,
3578
+ [ex_do_page_fault] = (__ipipe_exptr)&do_page_fault,
3579
+ [ex_do_spurious_interrupt_bug] = &do_spurious_interrupt_bug,
3580
+ [ex_do_coprocessor_error] = &do_coprocessor_error,
3581
+ [ex_do_alignment_check] = &do_alignment_check,
3582
+ [ex_machine_check_vector] = &do_machine_check_vector,
3583
+ [ex_do_simd_coprocessor_error] = &do_simd_coprocessor_error,
3584
+ [ex_do_device_not_available] = &do_device_not_available,
3585
+#ifdef CONFIG_X86_32
3586
+ [ex_do_iret_error] = &do_iret_error,
3587
+#endif
3588
+};
3589
+
3590
+#ifdef CONFIG_KGDB
3591
+#include <linux/kgdb.h>
3592
+
3593
+static int __ipipe_xlate_signo[] = {
3594
+
3595
+ [ex_do_divide_error] = SIGFPE,
3596
+ [ex_do_debug] = SIGTRAP,
3597
+ [2] = -1,
3598
+ [ex_do_int3] = SIGTRAP,
3599
+ [ex_do_overflow] = SIGSEGV,
3600
+ [ex_do_bounds] = SIGSEGV,
3601
+ [ex_do_invalid_op] = SIGILL,
3602
+ [ex_do_device_not_available] = -1,
3603
+ [8] = -1,
3604
+ [ex_do_coprocessor_segment_overrun] = SIGFPE,
3605
+ [ex_do_invalid_TSS] = SIGSEGV,
3606
+ [ex_do_segment_not_present] = SIGBUS,
3607
+ [ex_do_stack_segment] = SIGBUS,
3608
+ [ex_do_general_protection] = SIGSEGV,
3609
+ [ex_do_page_fault] = SIGSEGV,
3610
+ [ex_do_spurious_interrupt_bug] = -1,
3611
+ [ex_do_coprocessor_error] = -1,
3612
+ [ex_do_alignment_check] = SIGBUS,
3613
+ [ex_machine_check_vector] = -1,
3614
+ [ex_do_simd_coprocessor_error] = -1,
3615
+ [20 ... 31] = -1,
3616
+#ifdef CONFIG_X86_32
3617
+ [ex_do_iret_error] = SIGSEGV,
3618
+#endif
3619
+};
3620
+#endif /* CONFIG_KGDB */
3621
+
3622
+int __ipipe_handle_exception(struct pt_regs *regs, long error_code, int vector)
3623
+{
3624
+ unsigned long flags;
3625
+
3626
+ /* Pick up the root domain state of the interrupted context. */
3627
+ local_save_flags(flags);
3628
+
3629
+ if (ipipe_root_domain_p) {
3630
+ /*
3631
+ * Replicate hw interrupt state into the virtual mask before
3632
+ * calling the I-pipe event handler over the root domain. Also
3633
+ * required later when calling the Linux exception handler.
3634
+ */
3635
+ if (irqs_disabled_hw())
3636
+ local_irq_disable();
3637
+ }
3638
+#ifdef CONFIG_KGDB
3639
+ /* catch exception KGDB is interested in over non-root domains */
3640
+ else if (__ipipe_xlate_signo[vector] >= 0 &&
3641
+ !kgdb_handle_exception(vector, __ipipe_xlate_signo[vector],
3642
+ error_code, regs))
3643
+ return 1;
3644
+#endif /* CONFIG_KGDB */
3645
+
3646
+ if (unlikely(ipipe_trap_notify(vector, regs))) {
3647
+ local_irq_restore_nosync(flags);
3648
+ return 1;
3649
+ }
3650
+
3651
+ /*
3652
+ * 32-bit: In case we migrated to root domain inside the event
3653
+ * handler, restore the original IF from exception entry as the
3654
+ * low-level return code will evaluate it.
3655
+ */
3656
+ __fixup_if(raw_irqs_disabled_flags(flags), regs);
3657
+
3658
+ if (unlikely(!ipipe_root_domain_p)) {
3659
+ /* Detect unhandled faults over non-root domains. */
3660
+ struct ipipe_domain *ipd = ipipe_current_domain;
3661
+
3662
+ /* Switch to root so that Linux can handle the fault cleanly. */
3663
+ ipipe_current_domain = ipipe_root_domain;
3664
+
3665
+ ipipe_trace_panic_freeze();
3666
+
3667
+ /* Always warn about user land and unfixable faults. */
3668
+ if ((error_code & 4) || !search_exception_tables(instruction_pointer(regs))) {
3669
+ printk(KERN_ERR "BUG: Unhandled exception over domain"
3670
+ " %s at 0x%lx - switching to ROOT\n",
3671
+ ipd->name, instruction_pointer(regs));
3672
+ dump_stack();
3673
+ ipipe_trace_panic_dump();
3674
+#ifdef CONFIG_IPIPE_DEBUG
3675
+ /* Also report fixable ones when debugging is enabled. */
3676
+ } else {
3677
+ printk(KERN_WARNING "WARNING: Fixable exception over "
3678
+ "domain %s at 0x%lx - switching to ROOT\n",
3679
+ ipd->name, instruction_pointer(regs));
3680
+ dump_stack();
3681
+ ipipe_trace_panic_dump();
3682
+#endif /* CONFIG_IPIPE_DEBUG */
3683
+ }
3684
+ }
3685
+
3686
+ __ipipe_std_extable[vector](regs, error_code);
3687
+
3688
+ /*
3689
+ * Relevant for 64-bit: Restore root domain state as the low-level
3690
+ * return code will not align it to regs.flags.
3691
+ */
3692
+ local_irq_restore_nosync(flags);
3693
+
3694
+ return 0;
3695
+}
3696
+
3697
+int __ipipe_divert_exception(struct pt_regs *regs, int vector)
3698
+{
3699
+ unsigned long flags;
3700
+
3701
+ /* Same root state handling as in __ipipe_handle_exception. */
3702
+ local_save_flags(flags);
3703
+
3704
+ if (ipipe_root_domain_p) {
3705
+ if (irqs_disabled_hw())
3706
+ local_irq_disable();
3707
+ }
3708
+#ifdef CONFIG_KGDB
3709
+ /* catch int1 and int3 over non-root domains */
3710
+ else {
3711
+#ifdef CONFIG_X86_32
3712
+ if (vector != ex_do_device_not_available)
3713
+#endif
3714
+ {
3715
+ unsigned int condition = 0;
3716
+
3717
+ if (vector == 1)
3718
+ get_debugreg(condition, 6);
3719
+ if (!kgdb_handle_exception(vector, SIGTRAP, condition, regs))
3720
+ return 1;
3721
+ }
3722
+ }
3723
+#endif /* CONFIG_KGDB */
3724
+
3725
+ if (unlikely(ipipe_trap_notify(vector, regs))) {
3726
+ local_irq_restore_nosync(flags);
3727
+ return 1;
3728
+ }
3729
+
3730
+ /*
3731
+ * 32-bit: Due to possible migration inside the event handler, we have
3732
+ * to restore IF so that low-level return code sets the root domain
3733
+ * state correctly.
3734
+ */
3735
+ __fixup_if(raw_irqs_disabled_flags(flags), regs);
3736
+
3737
+ return 0;
3738
+}
3739
+
3740
+int __ipipe_syscall_root(struct pt_regs *regs)
3741
+{
3742
+ struct ipipe_percpu_domain_data *p;
3743
+ unsigned long flags;
3744
+ int ret;
3745
+
3746
+ /*
3747
+ * This routine either returns:
3748
+ * 0 -- if the syscall is to be passed to Linux;
3749
+ * >0 -- if the syscall should not be passed to Linux, and no
3750
+ * tail work should be performed;
3751
+ * <0 -- if the syscall should not be passed to Linux but the
3752
+ * tail work has to be performed (for handling signals etc).
3753
+ */
3754
+
3755
+ if (!__ipipe_syscall_watched_p(current, regs->orig_ax) ||
3756
+ !__ipipe_event_monitored_p(IPIPE_EVENT_SYSCALL))
3757
+ return 0;
3758
+
3759
+ ret = __ipipe_dispatch_event(IPIPE_EVENT_SYSCALL, regs);
3760
+ if (!ipipe_root_domain_p) {
3761
+#ifdef CONFIG_X86_64
3762
+ local_irq_disable_hw();
3763
+#endif
3764
+ return 1;
3765
+ }
3766
+
3767
+ p = ipipe_root_cpudom_ptr();
3768
+ __fixup_if(test_bit(IPIPE_STALL_FLAG, &p->status), regs);
3769
+ local_irq_save_hw(flags);
3770
+ /*
3771
+ * If allowed, sync pending VIRQs before _TIF_NEED_RESCHED is
3772
+ * tested.
3773
+ */
3774
+ if ((p->irqpend_himask & IPIPE_IRQMASK_VIRT) != 0)
3775
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_VIRT);
3776
+#ifdef CONFIG_X86_64
3777
+ if (!ret)
3778
+#endif
3779
+ local_irq_restore_hw(flags);
3780
+
3781
+ return -ret;
3782
+}
3783
+
3784
+/*
3785
+ * __ipipe_handle_irq() -- IPIPE's generic IRQ handler. An optimistic
3786
+ * interrupt protection log is maintained here for each domain. Hw
3787
+ * interrupts are off on entry.
3788
+ */
3789
+int __ipipe_handle_irq(struct pt_regs *regs)
3790
+{
3791
+ struct ipipe_domain *this_domain, *next_domain;
3792
+ unsigned vector = regs->orig_ax, irq;
3793
+ struct list_head *head, *pos;
3794
+ int m_ack;
3795
+
3796
+ if ((long)regs->orig_ax < 0) {
3797
+ vector = ~vector;
3798
+#ifdef CONFIG_X86_LOCAL_APIC
3799
+ if (vector >= FIRST_SYSTEM_VECTOR)
3800
+ irq = ipipe_apic_vector_irq(vector);
3801
+ else
3802
+#endif
3803
+ irq = __get_cpu_var(vector_irq)[vector];
3804
+ m_ack = 0;
3805
+ } else { /* This is a self-triggered one. */
3806
+ irq = vector;
3807
+ m_ack = 1;
3808
+ }
3809
+
3810
+ this_domain = ipipe_current_domain;
3811
+
3812
+ if (test_bit(IPIPE_STICKY_FLAG, &this_domain->irqs[irq].control))
3813
+ head = &this_domain->p_link;
3814
+ else {
3815
+ head = __ipipe_pipeline.next;
3816
+ next_domain = list_entry(head, struct ipipe_domain, p_link);
3817
+ if (likely(test_bit(IPIPE_WIRED_FLAG, &next_domain->irqs[irq].control))) {
3818
+ if (!m_ack && next_domain->irqs[irq].acknowledge)
3819
+ next_domain->irqs[irq].acknowledge(irq, irq_desc + irq);
3820
+ __ipipe_dispatch_wired(next_domain, irq);
3821
+ goto finalize_nosync;
3822
+ }
3823
+ }
3824
+
3825
+ /* Ack the interrupt. */
3826
+
3827
+ pos = head;
3828
+
3829
+ while (pos != &__ipipe_pipeline) {
3830
+ next_domain = list_entry(pos, struct ipipe_domain, p_link);
3831
+ if (test_bit(IPIPE_HANDLE_FLAG, &next_domain->irqs[irq].control)) {
3832
+ __ipipe_set_irq_pending(next_domain, irq);
3833
+ if (!m_ack && next_domain->irqs[irq].acknowledge) {
3834
+ next_domain->irqs[irq].acknowledge(irq, irq_desc + irq);
3835
+ m_ack = 1;
3836
+ }
3837
+ }
3838
+ if (!test_bit(IPIPE_PASS_FLAG, &next_domain->irqs[irq].control))
3839
+ break;
3840
+ pos = next_domain->p_link.next;
3841
+ }
3842
+
3843
+ /*
3844
+ * If the interrupt preempted the head domain, then do not
3845
+ * even try to walk the pipeline, unless an interrupt is
3846
+ * pending for it.
3847
+ */
3848
+ if (test_bit(IPIPE_AHEAD_FLAG, &this_domain->flags) &&
3849
+ ipipe_head_cpudom_var(irqpend_himask) == 0)
3850
+ goto finalize_nosync;
3851
+
3852
+ /*
3853
+ * Now walk the pipeline, yielding control to the highest
3854
+ * priority domain that has pending interrupt(s) or
3855
+ * immediately to the current domain if the interrupt has been
3856
+ * marked as 'sticky'. This search does not go beyond the
3857
+ * current domain in the pipeline.
3858
+ */
3859
+
3860
+ __ipipe_walk_pipeline(head);
3861
+
3862
+finalize_nosync:
3863
+
3864
+ /*
3865
+ * Given our deferred dispatching model for regular IRQs, we
3866
+ * only record CPU regs for the last timer interrupt, so that
3867
+ * the timer handler charges CPU times properly. It is assumed
3868
+ * that other interrupt handlers don't actually care for such
3869
+ * information.
3870
+ */
3871
+
3872
+ if (irq == __ipipe_tick_irq) {
3873
+ struct pt_regs *tick_regs = &__raw_get_cpu_var(__ipipe_tick_regs);
3874
+ tick_regs->flags = regs->flags;
3875
+ tick_regs->cs = regs->cs;
3876
+ tick_regs->ip = regs->ip;
3877
+ tick_regs->bp = regs->bp;
3878
+#ifdef CONFIG_X86_64
3879
+ tick_regs->ss = regs->ss;
3880
+ tick_regs->sp = regs->sp;
3881
+#endif
3882
+ if (!ipipe_root_domain_p)
3883
+ tick_regs->flags &= ~X86_EFLAGS_IF;
3884
+ }
3885
+
3886
+ if (!ipipe_root_domain_p ||
3887
+ test_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status)))
3888
+ return 0;
3889
+
3890
+#if defined(CONFIG_X86_32) && defined(CONFIG_SMP)
3891
+ /*
3892
+ * Prevent a spurious rescheduling from being triggered on
3893
+ * preemptible kernels along the way out through
3894
+ * ret_from_intr.
3895
+ */
3896
+ if ((long)regs->orig_ax < 0)
3897
+ __set_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status));
3898
+#endif /* CONFIG_SMP */
3899
+
3900
+ return 1;
3901
+}
3902
+
3903
+int __ipipe_check_tickdev(const char *devname)
3904
+{
3905
+#ifdef CONFIG_X86_LOCAL_APIC
3906
+ if (!strcmp(devname, "lapic"))
3907
+ return __ipipe_check_lapic();
3908
+#endif
3909
+
3910
+ return 1;
3911
+}
3912
+
3913
+EXPORT_SYMBOL(__ipipe_tick_irq);
3914
+
3915
+EXPORT_SYMBOL_GPL(irq_desc);
3916
+struct task_struct *__switch_to(struct task_struct *prev_p,
3917
+ struct task_struct *next_p);
3918
+EXPORT_SYMBOL_GPL(__switch_to);
3919
+EXPORT_SYMBOL_GPL(show_stack);
3920
+
3921
+#ifdef CONFIG_X86_32
3922
+EXPORT_PER_CPU_SYMBOL_GPL(init_tss);
3923
+#ifdef CONFIG_SMP
3924
+EXPORT_PER_CPU_SYMBOL_GPL(cpu_tlbstate);
3925
+#endif /* CONFIG_SMP */
3926
+#endif /* !CONFIG_X86_32 */
3927
+
3928
+#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
3929
+EXPORT_SYMBOL(tasklist_lock);
3930
+#endif /* CONFIG_SMP || CONFIG_DEBUG_SPINLOCK */
3931
+
3932
+#ifdef CONFIG_IPIPE_TRACE_MCOUNT
3933
+void notrace mcount(void);
3934
+EXPORT_SYMBOL(mcount);
3935
+#endif /* CONFIG_IPIPE_TRACE_MCOUNT */
3936
diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c
3937
index d1d4dc5..1789a2a 100644
3938
--- a/arch/x86/kernel/irq.c
3939
+++ b/arch/x86/kernel/irq.c
3940
@@ -31,7 +31,7 @@ void ack_bad_irq(unsigned int irq)
3941
* But only ack when the APIC is enabled -AK
3942
*/
3943
if (cpu_has_apic)
3944
- ack_APIC_irq();
3945
+ __ack_APIC_irq();
3946
#endif
3947
}
3948
3949
diff --git a/arch/x86/kernel/irqinit_32.c b/arch/x86/kernel/irqinit_32.c
3950
index 845aa98..9e46b1a 100644
3951
--- a/arch/x86/kernel/irqinit_32.c
3952
+++ b/arch/x86/kernel/irqinit_32.c
3953
@@ -151,6 +151,10 @@ void __init native_init_IRQ(void)
3954
3955
/* Low priority IPI to cleanup after moving an irq */
3956
set_intr_gate(IRQ_MOVE_CLEANUP_VECTOR, irq_move_cleanup_interrupt);
3957
+#ifdef CONFIG_IPIPE
3958
+ /* IPI for critical lock */
3959
+ set_intr_gate(IPIPE_CRITICAL_VECTOR, ipipe_ipiX);
3960
+#endif
3961
#endif
3962
3963
#ifdef CONFIG_X86_LOCAL_APIC
3964
@@ -160,6 +164,12 @@ void __init native_init_IRQ(void)
3965
/* IPI vectors for APIC spurious and error interrupts */
3966
alloc_intr_gate(SPURIOUS_APIC_VECTOR, spurious_interrupt);
3967
alloc_intr_gate(ERROR_APIC_VECTOR, error_interrupt);
3968
+#ifdef CONFIG_IPIPE
3969
+ set_intr_gate(IPIPE_SERVICE_VECTOR0, ipipe_ipi0);
3970
+ set_intr_gate(IPIPE_SERVICE_VECTOR1, ipipe_ipi1);
3971
+ set_intr_gate(IPIPE_SERVICE_VECTOR2, ipipe_ipi2);
3972
+ set_intr_gate(IPIPE_SERVICE_VECTOR3, ipipe_ipi3);
3973
+#endif
3974
#endif
3975
3976
#if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86_MCE_P4THERMAL)
3977
diff --git a/arch/x86/kernel/irqinit_64.c b/arch/x86/kernel/irqinit_64.c
3978
index ff02353..2fec83c 100644
3979
--- a/arch/x86/kernel/irqinit_64.c
3980
+++ b/arch/x86/kernel/irqinit_64.c
3981
@@ -173,11 +173,13 @@ static void __init smp_intr_init(void)
3982
alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+0, invalidate_interrupt0);
3983
alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+1, invalidate_interrupt1);
3984
alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+2, invalidate_interrupt2);
3985
+#ifndef CONFIG_IPIPE
3986
alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+3, invalidate_interrupt3);
3987
alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+4, invalidate_interrupt4);
3988
alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+5, invalidate_interrupt5);
3989
alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+6, invalidate_interrupt6);
3990
alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+7, invalidate_interrupt7);
3991
+#endif
3992
3993
/* IPI for generic function call */
3994
alloc_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
3995
diff --git a/arch/x86/kernel/mcount_32.S b/arch/x86/kernel/mcount_32.S
3996
new file mode 100644
3997
index 0000000..999097e
3998
--- /dev/null
3999
+++ b/arch/x86/kernel/mcount_32.S
4000
@@ -0,0 +1,27 @@
4001
+/*
4002
+ * linux/arch/x86/mcount_32.S
4003
+ *
4004
+ * Copyright (C) 2005, 2007 Jan Kiszka
4005
+ */
4006
+
4007
+.globl mcount
4008
+mcount:
4009
+ cmpl $0,ipipe_trace_enable
4010
+ je out
4011
+
4012
+ pushl %eax
4013
+ pushl %ecx
4014
+ pushl %edx
4015
+
4016
+ pushl $0 # no additional value (v)
4017
+ movl 4(%ebp),%ecx # __CALLER_ADDR1
4018
+ movl 16(%esp),%edx # __CALLER_ADDR0
4019
+ movl $0,%eax # IPIPE_TRACE_FUNC
4020
+ call __ipipe_trace
4021
+ popl %eax
4022
+
4023
+ popl %edx
4024
+ popl %ecx
4025
+ popl %eax
4026
+out:
4027
+ ret
4028
diff --git a/arch/x86/kernel/mcount_64.S b/arch/x86/kernel/mcount_64.S
4029
new file mode 100644
4030
index 0000000..c84a687
4031
--- /dev/null
4032
+++ b/arch/x86/kernel/mcount_64.S
4033
@@ -0,0 +1,41 @@
4034
+/*
4035
+ * linux/arch/x86/kernel/mcount_64.S
4036
+ *
4037
+ * Copyright (C) 2002 Free Software Foundation, Inc.
4038
+ * Contributed by Andreas Jaeger <aj@suse.de>.
4039
+ * Slightly adapted by Philippe Gerum for the I-pipe tracer.
4040
+ */
4041
+
4042
+#include <linux/linkage.h>
4043
+
4044
+ .code64
4045
+
4046
+ENTRY(mcount)
4047
+ cmpl $0,ipipe_trace_enable
4048
+ jz out
4049
+ subq $56,%rsp
4050
+ movq %rax,(%rsp)
4051
+ movq %rcx,8(%rsp)
4052
+ movq %rdx,16(%rsp)
4053
+ movq %rsi,24(%rsp)
4054
+ movq %rdi,32(%rsp)
4055
+ movq %r8,40(%rsp)
4056
+ movq %r9,48(%rsp)
4057
+
4058
+ movq $0,%rcx /* No additional value. */
4059
+ movq 8(%rbp),%rdx /* Parent rip. */
4060
+ movq 56(%rsp),%rsi /* Caller rip. */
4061
+ movq $0,%rdi /* IPIPE_TRACE_FN */
4062
+ call __ipipe_trace
4063
+
4064
+ movq 48(%rsp),%r9
4065
+ movq 40(%rsp),%r8
4066
+ movq 32(%rsp),%rdi
4067
+ movq 24(%rsp),%rsi
4068
+ movq 16(%rsp),%rdx
4069
+ movq 8(%rsp),%rcx
4070
+ movq (%rsp),%rax
4071
+ addq $56,%rsp
4072
+out:
4073
+ ret
4074
+END(mcount)
4075
diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c
4076
index 2c97f07..e049145 100644
4077
--- a/arch/x86/kernel/nmi.c
4078
+++ b/arch/x86/kernel/nmi.c
4079
@@ -61,6 +61,10 @@ static unsigned int nmi_hz = HZ;
4080
static DEFINE_PER_CPU(short, wd_enabled);
4081
static int endflag __initdata;
4082
4083
+static int default_nmi_watchdog_tick(struct pt_regs * regs, unsigned reason);
4084
+int (*nmi_watchdog_tick) (struct pt_regs * regs, unsigned reason) = &default_nmi_watchdog_tick;
4085
+EXPORT_SYMBOL(nmi_watchdog_tick);
4086
+
4087
static inline unsigned int get_nmi_count(int cpu)
4088
{
4089
#ifdef CONFIG_X86_64
4090
@@ -386,7 +390,7 @@ void touch_nmi_watchdog(void)
4091
EXPORT_SYMBOL(touch_nmi_watchdog);
4092
4093
notrace __kprobes int
4094
-nmi_watchdog_tick(struct pt_regs *regs, unsigned reason)
4095
+default_nmi_watchdog_tick(struct pt_regs *regs, unsigned reason)
4096
{
4097
/*
4098
* Since current_thread_info()-> is always on the stack, and we
4099
diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c
4100
index c622772..65cec6e 100644
4101
--- a/arch/x86/kernel/process.c
4102
+++ b/arch/x86/kernel/process.c
4103
@@ -26,7 +26,15 @@ int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src)
4104
return -ENOMEM;
4105
WARN_ON((unsigned long)dst->thread.xstate & 15);
4106
memcpy(dst->thread.xstate, src->thread.xstate, xstate_size);
4107
+ } else {
4108
+#ifdef CONFIG_IPIPE
4109
+ dst->thread.xstate = kmem_cache_alloc(task_xstate_cachep,
4110
+ GFP_KERNEL);
4111
+ if (!dst->thread.xstate)
4112
+ return -ENOMEM;
4113
+#endif
4114
}
4115
+
4116
return 0;
4117
}
4118
4119
@@ -50,6 +58,10 @@ void arch_task_cache_init(void)
4120
kmem_cache_create("task_xstate", xstate_size,
4121
__alignof__(union thread_xstate),
4122
SLAB_PANIC, NULL);
4123
+#ifdef CONFIG_IPIPE
4124
+ current->thread.xstate = kmem_cache_alloc(task_xstate_cachep,
4125
+ GFP_KERNEL);
4126
+#endif
4127
}
4128
4129
/*
4130
@@ -311,6 +323,11 @@ static void c1e_idle(void)
4131
4132
void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c)
4133
{
4134
+#ifdef CONFIG_IPIPE
4135
+#define default_to_mwait force_mwait
4136
+#else
4137
+#define default_to_mwait 1
4138
+#endif
4139
#ifdef CONFIG_X86_SMP
4140
if (pm_idle == poll_idle && smp_num_siblings > 1) {
4141
printk(KERN_WARNING "WARNING: polling idle and HT enabled,"
4142
@@ -320,7 +337,7 @@ void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c)
4143
if (pm_idle)
4144
return;
4145
4146
- if (cpu_has(c, X86_FEATURE_MWAIT) && mwait_usable(c)) {
4147
+ if (default_to_mwait && cpu_has(c, X86_FEATURE_MWAIT) && mwait_usable(c)) {
4148
/*
4149
* One CPU supports mwait => All CPUs supports mwait
4150
*/
4151
diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
4152
index 0a1302f..1e7c506 100644
4153
--- a/arch/x86/kernel/process_32.c
4154
+++ b/arch/x86/kernel/process_32.c
4155
@@ -113,6 +113,7 @@ void cpu_idle(void)
4156
__get_cpu_var(irq_stat).idle_timestamp = jiffies;
4157
/* Don't trace irqs off for idle */
4158
stop_critical_timings();
4159
+ ipipe_suspend_domain();
4160
pm_idle();
4161
start_critical_timings();
4162
}
4163
@@ -354,10 +355,12 @@ start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
4164
regs->cs = __USER_CS;
4165
regs->ip = new_ip;
4166
regs->sp = new_sp;
4167
+#ifndef CONFIG_IPIPE /* Lazily handled, init_fpu() will reset the state. */
4168
/*
4169
* Free the old FP and other extended state
4170
*/
4171
free_thread_xstate(current);
4172
+#endif
4173
}
4174
EXPORT_SYMBOL_GPL(start_thread);
4175
4176
@@ -552,7 +555,7 @@ struct task_struct * __switch_to(struct task_struct *prev_p, struct task_struct
4177
{
4178
struct thread_struct *prev = &prev_p->thread,
4179
*next = &next_p->thread;
4180
- int cpu = smp_processor_id();
4181
+ int cpu = raw_smp_processor_id();
4182
struct tss_struct *tss = &per_cpu(init_tss, cpu);
4183
4184
/* never put a printk in __switch_to... printk() calls wake_up*() indirectly */
4185
diff --git a/arch/x86/kernel/process_64.c b/arch/x86/kernel/process_64.c
4186
index c958120..639edb5 100644
4187
--- a/arch/x86/kernel/process_64.c
4188
+++ b/arch/x86/kernel/process_64.c
4189
@@ -55,6 +55,8 @@
4190
4191
asmlinkage extern void ret_from_fork(void);
4192
4193
+asmlinkage extern void thread_return(void);
4194
+
4195
unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED;
4196
4197
static ATOMIC_NOTIFIER_HEAD(idle_notifier);
4198
@@ -127,6 +129,7 @@ void cpu_idle(void)
4199
enter_idle();
4200
/* Don't trace irqs off for idle */
4201
stop_critical_timings();
4202
+ ipipe_suspend_domain();
4203
pm_idle();
4204
start_critical_timings();
4205
/* In many cases the interrupt that ended idle
4206
@@ -336,6 +339,7 @@ int copy_thread(int nr, unsigned long clone_flags, unsigned long sp,
4207
p->thread.sp = (unsigned long) childregs;
4208
p->thread.sp0 = (unsigned long) (childregs+1);
4209
p->thread.usersp = me->thread.usersp;
4210
+ p->thread.rip = (unsigned long) thread_return;
4211
4212
set_tsk_thread_flag(p, TIF_FORK);
4213
4214
@@ -395,10 +399,12 @@ start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp)
4215
regs->ss = __USER_DS;
4216
regs->flags = 0x200;
4217
set_fs(USER_DS);
4218
+#ifndef CONFIG_IPIPE /* Lazily handled, init_fpu() will reset the state. */
4219
/*
4220
* Free the old FP and other extended state
4221
*/
4222
free_thread_xstate(current);
4223
+#endif
4224
}
4225
EXPORT_SYMBOL_GPL(start_thread);
4226
4227
@@ -557,7 +563,7 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
4228
{
4229
struct thread_struct *prev = &prev_p->thread;
4230
struct thread_struct *next = &next_p->thread;
4231
- int cpu = smp_processor_id();
4232
+ int cpu = raw_smp_processor_id();
4233
struct tss_struct *tss = &per_cpu(init_tss, cpu);
4234
unsigned fsindex, gsindex;
4235
4236
diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c
4237
index 18f9b19..d159922 100644
4238
--- a/arch/x86/kernel/smp.c
4239
+++ b/arch/x86/kernel/smp.c
4240
@@ -142,7 +142,7 @@ void native_send_call_func_ipi(cpumask_t mask)
4241
4242
static void stop_this_cpu(void *dummy)
4243
{
4244
- local_irq_disable();
4245
+ local_irq_disable_hw();
4246
/*
4247
* Remove this CPU:
4248
*/
4249
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
4250
index f71f96f..813228c 100644
4251
--- a/arch/x86/kernel/smpboot.c
4252
+++ b/arch/x86/kernel/smpboot.c
4253
@@ -287,7 +287,7 @@ static int __cpuinitdata unsafe_smp;
4254
/*
4255
* Activate a secondary processor.
4256
*/
4257
-static void __cpuinit start_secondary(void *unused)
4258
+static void notrace __cpuinit start_secondary(void *unused)
4259
{
4260
/*
4261
* Don't put *anything* before cpu_init(), SMP booting is too
4262
@@ -965,7 +965,7 @@ restore_state:
4263
int __cpuinit native_cpu_up(unsigned int cpu)
4264
{
4265
int apicid = cpu_present_to_apicid(cpu);
4266
- unsigned long flags;
4267
+ unsigned long flags, _flags;
4268
int err;
4269
4270
WARN_ON(irqs_disabled());
4271
@@ -1017,9 +1017,9 @@ int __cpuinit native_cpu_up(unsigned int cpu)
4272
* Check TSC synchronization with the AP (keep irqs disabled
4273
* while doing so):
4274
*/
4275
- local_irq_save(flags);
4276
+ local_irq_save_full(flags, _flags);
4277
check_tsc_sync_source(cpu);
4278
- local_irq_restore(flags);
4279
+ local_irq_restore_full(flags, _flags);
4280
4281
while (!cpu_online(cpu)) {
4282
cpu_relax();
4283
diff --git a/arch/x86/kernel/time_32.c b/arch/x86/kernel/time_32.c
4284
index 77b400f..f53b944 100644
4285
--- a/arch/x86/kernel/time_32.c
4286
+++ b/arch/x86/kernel/time_32.c
4287
@@ -84,11 +84,12 @@ irqreturn_t timer_interrupt(int irq, void *dev_id)
4288
* manually to deassert NMI lines for the watchdog if run
4289
* on an 82489DX-based system.
4290
*/
4291
- spin_lock(&i8259A_lock);
4292
+ unsigned long flags;
4293
+ spin_lock_irqsave_cond(&i8259A_lock,flags);
4294
outb(0x0c, PIC_MASTER_OCW3);
4295
/* Ack the IRQ; AEOI will end it automatically. */
4296
inb(PIC_MASTER_POLL);
4297
- spin_unlock(&i8259A_lock);
4298
+ spin_unlock_irqrestore_cond(&i8259A_lock,flags);
4299
}
4300
#endif
4301
4302
diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c
4303
index 70bdbb5..5251b23 100644
4304
--- a/arch/x86/kernel/traps.c
4305
+++ b/arch/x86/kernel/traps.c
4306
@@ -877,6 +877,7 @@ asmlinkage void math_state_restore(void)
4307
{
4308
struct thread_info *thread = current_thread_info();
4309
struct task_struct *tsk = thread->task;
4310
+ unsigned long flags;
4311
4312
if (!tsk_used_math(tsk)) {
4313
local_irq_enable();
4314
@@ -893,6 +894,7 @@ asmlinkage void math_state_restore(void)
4315
local_irq_disable();
4316
}
4317
4318
+ local_irq_save_hw_cond(flags);
4319
clts(); /* Allow maths ops (or we recurse) */
4320
#ifdef CONFIG_X86_32
4321
restore_fpu(tsk);
4322
@@ -902,12 +904,14 @@ asmlinkage void math_state_restore(void)
4323
*/
4324
if (unlikely(restore_fpu_checking(tsk))) {
4325
stts();
4326
+ local_irq_restore_hw_cond(flags);
4327
force_sig(SIGSEGV, tsk);
4328
return;
4329
}
4330
#endif
4331
thread->status |= TS_USEDFPU; /* So we fnsave on switch_to() */
4332
tsk->fpu_counter++;
4333
+ local_irq_restore_hw_cond(flags);
4334
}
4335
EXPORT_SYMBOL_GPL(math_state_restore);
4336
4337
diff --git a/arch/x86/kernel/vm86_32.c b/arch/x86/kernel/vm86_32.c
4338
index 4eeb5cf..6699b26 100644
4339
--- a/arch/x86/kernel/vm86_32.c
4340
+++ b/arch/x86/kernel/vm86_32.c
4341
@@ -148,12 +148,14 @@ struct pt_regs *save_v86_state(struct kernel_vm86_regs *regs)
4342
do_exit(SIGSEGV);
4343
}
4344
4345
+ local_irq_disable_hw_cond();
4346
tss = &per_cpu(init_tss, get_cpu());
4347
current->thread.sp0 = current->thread.saved_sp0;
4348
current->thread.sysenter_cs = __KERNEL_CS;
4349
load_sp0(tss, ¤t->thread);
4350
current->thread.saved_sp0 = 0;
4351
put_cpu();
4352
+ local_irq_enable_hw_cond();
4353
4354
ret = KVM86->regs32;
4355
4356
@@ -325,12 +327,14 @@ static void do_sys_vm86(struct kernel_vm86_struct *info, struct task_struct *tsk
4357
tsk->thread.saved_fs = info->regs32->fs;
4358
savesegment(gs, tsk->thread.saved_gs);
4359
4360
+ local_irq_disable_hw_cond();
4361
tss = &per_cpu(init_tss, get_cpu());
4362
tsk->thread.sp0 = (unsigned long) &info->VM86_TSS_ESP0;
4363
if (cpu_has_sep)
4364
tsk->thread.sysenter_cs = 0;
4365
load_sp0(tss, &tsk->thread);
4366
put_cpu();
4367
+ local_irq_enable_hw_cond();
4368
4369
tsk->thread.screen_bitmap = info->screen_bitmap;
4370
if (info->flags & VM86_SCREEN_BITMAP)
4371
diff --git a/arch/x86/lib/mmx_32.c b/arch/x86/lib/mmx_32.c
4372
index c9f2d9b..78d780a 100644
4373
--- a/arch/x86/lib/mmx_32.c
4374
+++ b/arch/x86/lib/mmx_32.c
4375
@@ -30,7 +30,7 @@ void *_mmx_memcpy(void *to, const void *from, size_t len)
4376
void *p;
4377
int i;
4378
4379
- if (unlikely(in_interrupt()))
4380
+ if (unlikely(!ipipe_root_domain_p || in_interrupt()))
4381
return __memcpy(to, from, len);
4382
4383
p = to;
4384
diff --git a/arch/x86/lib/thunk_64.S b/arch/x86/lib/thunk_64.S
4385
index bf9a7d5..98609ae 100644
4386
--- a/arch/x86/lib/thunk_64.S
4387
+++ b/arch/x86/lib/thunk_64.S
4388
@@ -65,6 +65,10 @@
4389
thunk lockdep_sys_exit_thunk,lockdep_sys_exit
4390
#endif
4391
4392
+#ifdef CONFIG_IPIPE
4393
+ thunk_retrax __ipipe_syscall_root_thunk,__ipipe_syscall_root
4394
+#endif
4395
+
4396
/* SAVE_ARGS below is used only for the .cfi directives it contains. */
4397
CFI_STARTPROC
4398
SAVE_ARGS
4399
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
4400
index 68dc060..9f1c6a9 100644
4401
--- a/arch/x86/mm/fault.c
4402
+++ b/arch/x86/mm/fault.c
4403
@@ -301,6 +301,56 @@ static inline pmd_t *vmalloc_sync_one(pgd_t *pgd, unsigned long address)
4404
BUG_ON(pmd_page(*pmd) != pmd_page(*pmd_k));
4405
return pmd_k;
4406
}
4407
+#else
4408
+static inline int vmalloc_sync_one(pgd_t *pgd, unsigned long address)
4409
+{
4410
+ pgd_t *pgd_ref;
4411
+ pud_t *pud, *pud_ref;
4412
+ pmd_t *pmd, *pmd_ref;
4413
+ pte_t *pte, *pte_ref;
4414
+
4415
+ /* Make sure we are in vmalloc area */
4416
+ if (!(address >= VMALLOC_START && address < VMALLOC_END))
4417
+ return -1;
4418
+
4419
+ /* Copy kernel mappings over when needed. This can also
4420
+ happen within a race in page table update. In the later
4421
+ case just flush. */
4422
+
4423
+ pgd_ref = pgd_offset_k(address);
4424
+ if (pgd_none(*pgd_ref))
4425
+ return -1;
4426
+ if (pgd_none(*pgd))
4427
+ set_pgd(pgd, *pgd_ref);
4428
+ else
4429
+ BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
4430
+
4431
+ /* Below here mismatches are bugs because these lower tables
4432
+ are shared */
4433
+
4434
+ pud = pud_offset(pgd, address);
4435
+ pud_ref = pud_offset(pgd_ref, address);
4436
+ if (pud_none(*pud_ref))
4437
+ return -1;
4438
+ if (pud_none(*pud) || pud_page_vaddr(*pud) != pud_page_vaddr(*pud_ref))
4439
+ BUG();
4440
+ pmd = pmd_offset(pud, address);
4441
+ pmd_ref = pmd_offset(pud_ref, address);
4442
+ if (pmd_none(*pmd_ref))
4443
+ return -1;
4444
+ if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
4445
+ BUG();
4446
+ pte_ref = pte_offset_kernel(pmd_ref, address);
4447
+ if (!pte_present(*pte_ref))
4448
+ return -1;
4449
+ pte = pte_offset_kernel(pmd, address);
4450
+ /* Don't use pte_page here, because the mappings can point
4451
+ outside mem_map, and the NUMA hash lookup cannot handle
4452
+ that. */
4453
+ if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
4454
+ BUG();
4455
+ return 0;
4456
+}
4457
#endif
4458
4459
#ifdef CONFIG_X86_64
4460
@@ -493,9 +543,9 @@ static int spurious_fault(unsigned long address,
4461
*
4462
* This assumes no large pages in there.
4463
*/
4464
+#ifdef CONFIG_X86_32
4465
static int vmalloc_fault(unsigned long address)
4466
{
4467
-#ifdef CONFIG_X86_32
4468
unsigned long pgd_paddr;
4469
pmd_t *pmd_k;
4470
pte_t *pte_k;
4471
@@ -519,56 +569,14 @@ static int vmalloc_fault(unsigned long address)
4472
if (!pte_present(*pte_k))
4473
return -1;
4474
return 0;
4475
+}
4476
#else
4477
- pgd_t *pgd, *pgd_ref;
4478
- pud_t *pud, *pud_ref;
4479
- pmd_t *pmd, *pmd_ref;
4480
- pte_t *pte, *pte_ref;
4481
-
4482
- /* Make sure we are in vmalloc area */
4483
- if (!(address >= VMALLOC_START && address < VMALLOC_END))
4484
- return -1;
4485
-
4486
- /* Copy kernel mappings over when needed. This can also
4487
- happen within a race in page table update. In the later
4488
- case just flush. */
4489
-
4490
- pgd = pgd_offset(current->active_mm, address);
4491
- pgd_ref = pgd_offset_k(address);
4492
- if (pgd_none(*pgd_ref))
4493
- return -1;
4494
- if (pgd_none(*pgd))
4495
- set_pgd(pgd, *pgd_ref);
4496
- else
4497
- BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
4498
-
4499
- /* Below here mismatches are bugs because these lower tables
4500
- are shared */
4501
-
4502
- pud = pud_offset(pgd, address);
4503
- pud_ref = pud_offset(pgd_ref, address);
4504
- if (pud_none(*pud_ref))
4505
- return -1;
4506
- if (pud_none(*pud) || pud_page_vaddr(*pud) != pud_page_vaddr(*pud_ref))
4507
- BUG();
4508
- pmd = pmd_offset(pud, address);
4509
- pmd_ref = pmd_offset(pud_ref, address);
4510
- if (pmd_none(*pmd_ref))
4511
- return -1;
4512
- if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
4513
- BUG();
4514
- pte_ref = pte_offset_kernel(pmd_ref, address);
4515
- if (!pte_present(*pte_ref))
4516
- return -1;
4517
- pte = pte_offset_kernel(pmd, address);
4518
- /* Don't use pte_page here, because the mappings can point
4519
- outside mem_map, and the NUMA hash lookup cannot handle
4520
- that. */
4521
- if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
4522
- BUG();
4523
- return 0;
4524
-#endif
4525
+static int vmalloc_fault(unsigned long address)
4526
+{
4527
+ pgd_t *pgd = pgd = pgd_offset(current->active_mm, address);
4528
+ return vmalloc_sync_one(pgd, address);
4529
}
4530
+#endif
4531
4532
int show_unhandled_signals = 1;
4533
4534
@@ -592,13 +600,15 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
4535
unsigned long flags;
4536
#endif
4537
4538
+ /* get the address */
4539
+ address = read_cr2();
4540
+
4541
+ local_irq_enable_hw_cond();
4542
+
4543
tsk = current;
4544
mm = tsk->mm;
4545
prefetchw(&mm->mmap_sem);
4546
4547
- /* get the address */
4548
- address = read_cr2();
4549
-
4550
si_code = SEGV_MAPERR;
4551
4552
if (unlikely(kmmio_fault(regs, address)))
4553
@@ -944,3 +954,43 @@ void vmalloc_sync_all(void)
4554
}
4555
#endif
4556
}
4557
+
4558
+#ifdef CONFIG_IPIPE
4559
+void __ipipe_pin_range_globally(unsigned long start, unsigned long end)
4560
+{
4561
+#ifdef CONFIG_X86_32
4562
+ unsigned long next, addr = start;
4563
+
4564
+ do {
4565
+ unsigned long flags;
4566
+ struct page *page;
4567
+
4568
+ next = pgd_addr_end(addr, end);
4569
+ spin_lock_irqsave(&pgd_lock, flags);
4570
+ list_for_each_entry(page, &pgd_list, lru)
4571
+ vmalloc_sync_one(page_address(page), addr);
4572
+ spin_unlock_irqrestore(&pgd_lock, flags);
4573
+
4574
+ } while (addr = next, addr != end);
4575
+#else
4576
+ unsigned long next, addr = start;
4577
+ int ret = 0;
4578
+
4579
+ do {
4580
+ struct page *page;
4581
+
4582
+ next = pgd_addr_end(addr, end);
4583
+ spin_lock(&pgd_lock);
4584
+ list_for_each_entry(page, &pgd_list, lru) {
4585
+ pgd_t *pgd;
4586
+ pgd = (pgd_t *)page_address(page) + pgd_index(addr);
4587
+ ret = vmalloc_sync_one(pgd, addr);
4588
+ if (ret)
4589
+ break;
4590
+ }
4591
+ spin_unlock(&pgd_lock);
4592
+ addr = next;
4593
+ } while (!ret && addr != end);
4594
+#endif
4595
+}
4596
+#endif /* CONFIG_IPIPE */
4597
diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c
4598
index 5f8d746..527aeba 100644
4599
--- a/drivers/acpi/processor_idle.c
4600
+++ b/drivers/acpi/processor_idle.c
4601
@@ -218,7 +218,7 @@ static void acpi_safe_halt(void)
4602
smp_mb();
4603
if (!need_resched()) {
4604
safe_halt();
4605
- local_irq_disable();
4606
+ local_irq_disable_hw(); local_irq_disable();
4607
}
4608
current_thread_info()->status |= TS_POLLING;
4609
}
4610
@@ -402,7 +402,7 @@ static void acpi_processor_idle(void)
4611
* Interrupts must be disabled during bus mastering calculations and
4612
* for C2/C3 transitions.
4613
*/
4614
- local_irq_disable();
4615
+ local_irq_disable_hw(); local_irq_disable();
4616
4617
pr = __get_cpu_var(processors);
4618
if (!pr) {
4619
@@ -1463,7 +1463,7 @@ static int acpi_idle_enter_c1(struct cpuidle_device *dev,
4620
if (unlikely(!pr))
4621
return 0;
4622
4623
- local_irq_disable();
4624
+ local_irq_disable_hw(); local_irq_disable();
4625
4626
/* Do not access any ACPI IO ports in suspend path */
4627
if (acpi_idle_suspend) {
4628
@@ -1506,7 +1506,7 @@ static int acpi_idle_enter_simple(struct cpuidle_device *dev,
4629
if (acpi_idle_suspend)
4630
return(acpi_idle_enter_c1(dev, state));
4631
4632
- local_irq_disable();
4633
+ local_irq_disable_hw(); local_irq_disable();
4634
current_thread_info()->status &= ~TS_POLLING;
4635
/*
4636
* TS_POLLING-cleared state must be visible before we test
4637
@@ -1589,14 +1589,14 @@ static int acpi_idle_enter_bm(struct cpuidle_device *dev,
4638
dev->last_state = dev->safe_state;
4639
return dev->safe_state->enter(dev, dev->safe_state);
4640
} else {
4641
- local_irq_disable();
4642
+ local_irq_disable_hw(); local_irq_disable();
4643
acpi_safe_halt();
4644
local_irq_enable();
4645
return 0;
4646
}
4647
}
4648
4649
- local_irq_disable();
4650
+ local_irq_disable_hw(); local_irq_disable();
4651
current_thread_info()->status &= ~TS_POLLING;
4652
/*
4653
* TS_POLLING-cleared state must be visible before we test
4654
diff --git a/drivers/pci/htirq.c b/drivers/pci/htirq.c
4655
index bf7d6ce..0ffac4e 100644
4656
--- a/drivers/pci/htirq.c
4657
+++ b/drivers/pci/htirq.c
4658
@@ -21,7 +21,7 @@
4659
* With multiple simultaneous hypertransport irq devices it might pay
4660
* to make this more fine grained. But start with simple, stupid, and correct.
4661
*/
4662
-static DEFINE_SPINLOCK(ht_irq_lock);
4663
+static IPIPE_DEFINE_SPINLOCK(ht_irq_lock);
4664
4665
struct ht_irq_cfg {
4666
struct pci_dev *dev;
4667
diff --git a/drivers/serial/8250.c b/drivers/serial/8250.c
4668
index a181ccf..e494638 100644
4669
--- a/drivers/serial/8250.c
4670
+++ b/drivers/serial/8250.c
4671
@@ -2895,6 +2895,51 @@ static int serial8250_resume(struct platform_device *dev)
4672
return 0;
4673
}
4674
4675
+#if defined(CONFIG_IPIPE_DEBUG) && defined(CONFIG_SERIAL_8250_CONSOLE)
4676
+
4677
+#include <stdarg.h>
4678
+
4679
+void __ipipe_serial_debug(const char *fmt, ...)
4680
+{
4681
+ struct uart_8250_port *up = &serial8250_ports[0];
4682
+ unsigned int ier, count;
4683
+ unsigned long flags;
4684
+ char buf[128];
4685
+ va_list ap;
4686
+
4687
+ va_start(ap, fmt);
4688
+ vsprintf(buf, fmt, ap);
4689
+ va_end(ap);
4690
+ count = strlen(buf);
4691
+
4692
+ touch_nmi_watchdog();
4693
+
4694
+ local_irq_save_hw(flags);
4695
+
4696
+ /*
4697
+ * First save the IER then disable the interrupts
4698
+ */
4699
+ ier = serial_in(up, UART_IER);
4700
+
4701
+ if (up->capabilities & UART_CAP_UUE)
4702
+ serial_out(up, UART_IER, UART_IER_UUE);
4703
+ else
4704
+ serial_out(up, UART_IER, 0);
4705
+
4706
+ uart_console_write(&up->port, buf, count, serial8250_console_putchar);
4707
+
4708
+ /*
4709
+ * Finally, wait for transmitter to become empty
4710
+ * and restore the IER
4711
+ */
4712
+ wait_for_xmitr(up, BOTH_EMPTY);
4713
+ serial_out(up, UART_IER, ier);
4714
+
4715
+ local_irq_restore_hw(flags);
4716
+}
4717
+
4718
+#endif
4719
+
4720
static struct platform_driver serial8250_isa_driver = {
4721
.probe = serial8250_probe,
4722
.remove = __devexit_p(serial8250_remove),
4723
diff --git a/include/asm-generic/cmpxchg-local.h b/include/asm-generic/cmpxchg-local.h
4724
index b2ba2fc..ed01ab9 100644
4725
--- a/include/asm-generic/cmpxchg-local.h
4726
+++ b/include/asm-generic/cmpxchg-local.h
4727
@@ -20,7 +20,7 @@ static inline unsigned long __cmpxchg_local_generic(volatile void *ptr,
4728
if (size == 8 && sizeof(unsigned long) != 8)
4729
wrong_size_cmpxchg(ptr);
4730
4731
- local_irq_save(flags);
4732
+ local_irq_save_hw(flags);
4733
switch (size) {
4734
case 1: prev = *(u8 *)ptr;
4735
if (prev == old)
4736
@@ -41,7 +41,7 @@ static inline unsigned long __cmpxchg_local_generic(volatile void *ptr,
4737
default:
4738
wrong_size_cmpxchg(ptr);
4739
}
4740
- local_irq_restore(flags);
4741
+ local_irq_restore_hw(flags);
4742
return prev;
4743
}
4744
4745
@@ -54,11 +54,11 @@ static inline u64 __cmpxchg64_local_generic(volatile void *ptr,
4746
u64 prev;
4747
unsigned long flags;
4748
4749
- local_irq_save(flags);
4750
+ local_irq_save_hw(flags);
4751
prev = *(u64 *)ptr;
4752
if (prev == old)
4753
*(u64 *)ptr = new;
4754
- local_irq_restore(flags);
4755
+ local_irq_restore_hw(flags);
4756
return prev;
4757
}
4758
4759
diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
4760
index 181006c..bc69d94 100644
4761
--- a/include/linux/hardirq.h
4762
+++ b/include/linux/hardirq.h
4763
@@ -161,7 +161,22 @@ extern void irq_enter(void);
4764
*/
4765
extern void irq_exit(void);
4766
4767
-#define nmi_enter() do { lockdep_off(); __irq_enter(); } while (0)
4768
-#define nmi_exit() do { __irq_exit(); lockdep_on(); } while (0)
4769
+#define nmi_enter() \
4770
+ do { \
4771
+ ipipe_nmi_enter(); \
4772
+ if (ipipe_root_domain_p) { \
4773
+ lockdep_off(); \
4774
+ __irq_enter(); \
4775
+ } \
4776
+ } while (0)
4777
+
4778
+#define nmi_exit() \
4779
+ do { \
4780
+ if (ipipe_root_domain_p) { \
4781
+ __irq_exit(); \
4782
+ lockdep_on(); \
4783
+ } \
4784
+ ipipe_nmi_exit(); \
4785
+ } while (0)
4786
4787
#endif /* LINUX_HARDIRQ_H */
4788
diff --git a/include/linux/ipipe.h b/include/linux/ipipe.h
4789
new file mode 100644
4790
index 0000000..2b12117
4791
--- /dev/null
4792
+++ b/include/linux/ipipe.h
4793
@@ -0,0 +1,673 @@
4794
+/* -*- linux-c -*-
4795
+ * include/linux/ipipe.h
4796
+ *
4797
+ * Copyright (C) 2002-2007 Philippe Gerum.
4798
+ *
4799
+ * This program is free software; you can redistribute it and/or modify
4800
+ * it under the terms of the GNU General Public License as published by
4801
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
4802
+ * USA; either version 2 of the License, or (at your option) any later
4803
+ * version.
4804
+ *
4805
+ * This program is distributed in the hope that it will be useful,
4806
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
4807
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
4808
+ * GNU General Public License for more details.
4809
+ *
4810
+ * You should have received a copy of the GNU General Public License
4811
+ * along with this program; if not, write to the Free Software
4812
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
4813
+ */
4814
+
4815
+#ifndef __LINUX_IPIPE_H
4816
+#define __LINUX_IPIPE_H
4817
+
4818
+#include <linux/spinlock.h>
4819
+#include <linux/cache.h>
4820
+#include <linux/percpu.h>
4821
+#include <linux/mutex.h>
4822
+#include <linux/linkage.h>
4823
+#include <linux/ipipe_base.h>
4824
+#include <linux/ipipe_compat.h>
4825
+#include <asm/ipipe.h>
4826
+
4827
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
4828
+
4829
+#include <linux/cpumask.h>
4830
+#include <asm/system.h>
4831
+
4832
+static inline int ipipe_disable_context_check(int cpu)
4833
+{
4834
+ return xchg(&per_cpu(ipipe_percpu_context_check, cpu), 0);
4835
+}
4836
+
4837
+static inline void ipipe_restore_context_check(int cpu, int old_state)
4838
+{
4839
+ per_cpu(ipipe_percpu_context_check, cpu) = old_state;
4840
+}
4841
+
4842
+static inline void ipipe_context_check_off(void)
4843
+{
4844
+ int cpu;
4845
+ for_each_online_cpu(cpu)
4846
+ per_cpu(ipipe_percpu_context_check, cpu) = 0;
4847
+}
4848
+
4849
+#else /* !CONFIG_IPIPE_DEBUG_CONTEXT */
4850
+
4851
+static inline int ipipe_disable_context_check(int cpu)
4852
+{
4853
+ return 0;
4854
+}
4855
+
4856
+static inline void ipipe_restore_context_check(int cpu, int old_state) { }
4857
+
4858
+static inline void ipipe_context_check_off(void) { }
4859
+
4860
+#endif /* !CONFIG_IPIPE_DEBUG_CONTEXT */
4861
+
4862
+#ifdef CONFIG_IPIPE
4863
+
4864
+/*
4865
+ * Sanity check: IPIPE_VIRQ_BASE depends on CONFIG_NR_CPUS, and if the
4866
+ * latter gets too large, we fail to map the virtual interrupts.
4867
+ */
4868
+#if IPIPE_VIRQ_BASE / BITS_PER_LONG > BITS_PER_LONG
4869
+#error "CONFIG_NR_CPUS is too large, please lower it."
4870
+#endif
4871
+
4872
+#define IPIPE_VERSION_STRING IPIPE_ARCH_STRING
4873
+#define IPIPE_RELEASE_NUMBER ((IPIPE_MAJOR_NUMBER << 16) | \
4874
+ (IPIPE_MINOR_NUMBER << 8) | \
4875
+ (IPIPE_PATCH_NUMBER))
4876
+
4877
+#ifndef BROKEN_BUILTIN_RETURN_ADDRESS
4878
+#define __BUILTIN_RETURN_ADDRESS0 ((unsigned long)__builtin_return_address(0))
4879
+#define __BUILTIN_RETURN_ADDRESS1 ((unsigned long)__builtin_return_address(1))
4880
+#endif /* !BUILTIN_RETURN_ADDRESS */
4881
+
4882
+#define IPIPE_ROOT_PRIO 100
4883
+#define IPIPE_ROOT_ID 0
4884
+#define IPIPE_ROOT_NPTDKEYS 4 /* Must be <= BITS_PER_LONG */
4885
+
4886
+#define IPIPE_RESET_TIMER 0x1
4887
+#define IPIPE_GRAB_TIMER 0x2
4888
+
4889
+/* Global domain flags */
4890
+#define IPIPE_SPRINTK_FLAG 0 /* Synchronous printk() allowed */
4891
+#define IPIPE_AHEAD_FLAG 1 /* Domain always heads the pipeline */
4892
+
4893
+/* Interrupt control bits */
4894
+#define IPIPE_HANDLE_FLAG 0
4895
+#define IPIPE_PASS_FLAG 1
4896
+#define IPIPE_ENABLE_FLAG 2
4897
+#define IPIPE_DYNAMIC_FLAG IPIPE_HANDLE_FLAG
4898
+#define IPIPE_STICKY_FLAG 3
4899
+#define IPIPE_SYSTEM_FLAG 4
4900
+#define IPIPE_LOCK_FLAG 5
4901
+#define IPIPE_WIRED_FLAG 6
4902
+#define IPIPE_EXCLUSIVE_FLAG 7
4903
+
4904
+#define IPIPE_HANDLE_MASK (1 << IPIPE_HANDLE_FLAG)
4905
+#define IPIPE_PASS_MASK (1 << IPIPE_PASS_FLAG)
4906
+#define IPIPE_ENABLE_MASK (1 << IPIPE_ENABLE_FLAG)
4907
+#define IPIPE_DYNAMIC_MASK IPIPE_HANDLE_MASK
4908
+#define IPIPE_STICKY_MASK (1 << IPIPE_STICKY_FLAG)
4909
+#define IPIPE_SYSTEM_MASK (1 << IPIPE_SYSTEM_FLAG)
4910
+#define IPIPE_LOCK_MASK (1 << IPIPE_LOCK_FLAG)
4911
+#define IPIPE_WIRED_MASK (1 << IPIPE_WIRED_FLAG)
4912
+#define IPIPE_EXCLUSIVE_MASK (1 << IPIPE_EXCLUSIVE_FLAG)
4913
+
4914
+#define IPIPE_DEFAULT_MASK (IPIPE_HANDLE_MASK|IPIPE_PASS_MASK)
4915
+#define IPIPE_STDROOT_MASK (IPIPE_HANDLE_MASK|IPIPE_PASS_MASK|IPIPE_SYSTEM_MASK)
4916
+
4917
+#define IPIPE_EVENT_SELF 0x80000000
4918
+
4919
+#define IPIPE_NR_CPUS NR_CPUS
4920
+
4921
+#define ipipe_current_domain ipipe_cpu_var(ipipe_percpu_domain)
4922
+
4923
+#define ipipe_virtual_irq_p(irq) ((irq) >= IPIPE_VIRQ_BASE && \
4924
+ (irq) < IPIPE_NR_IRQS)
4925
+
4926
+#define IPIPE_SAME_HANDLER ((ipipe_irq_handler_t)(-1))
4927
+
4928
+struct irq_desc;
4929
+
4930
+typedef void (*ipipe_irq_ackfn_t)(unsigned irq, struct irq_desc *desc);
4931
+
4932
+typedef int (*ipipe_event_handler_t)(unsigned event,
4933
+ struct ipipe_domain *from,
4934
+ void *data);
4935
+struct ipipe_domain {
4936
+
4937
+ int slot; /* Slot number in percpu domain data array. */
4938
+ struct list_head p_link; /* Link in pipeline */
4939
+ ipipe_event_handler_t evhand[IPIPE_NR_EVENTS]; /* Event handlers. */
4940
+ unsigned long long evself; /* Self-monitored event bits. */
4941
+
4942
+ struct {
4943
+ unsigned long control;
4944
+ ipipe_irq_ackfn_t acknowledge;
4945
+ ipipe_irq_handler_t handler;
4946
+ void *cookie;
4947
+ } ____cacheline_aligned irqs[IPIPE_NR_IRQS];
4948
+
4949
+ int priority;
4950
+ void *pdd;
4951
+ unsigned long flags;
4952
+ unsigned domid;
4953
+ const char *name;
4954
+ struct mutex mutex;
4955
+};
4956
+
4957
+#define IPIPE_HEAD_PRIORITY (-1) /* For domains always heading the pipeline */
4958
+
4959
+struct ipipe_domain_attr {
4960
+
4961
+ unsigned domid; /* Domain identifier -- Magic value set by caller */
4962
+ const char *name; /* Domain name -- Warning: won't be dup'ed! */
4963
+ int priority; /* Priority in interrupt pipeline */
4964
+ void (*entry) (void); /* Domain entry point */
4965
+ void *pdd; /* Per-domain (opaque) data pointer */
4966
+};
4967
+
4968
+#ifdef CONFIG_SMP
4969
+/* These ops must start and complete on the same CPU: care for
4970
+ * migration. */
4971
+#define set_bit_safe(b, a) \
4972
+ ({ unsigned long __flags; \
4973
+ local_irq_save_hw_notrace(__flags); \
4974
+ __set_bit(b, a); \
4975
+ local_irq_restore_hw_notrace(__flags); })
4976
+#define test_and_set_bit_safe(b, a) \
4977
+ ({ unsigned long __flags, __x; \
4978
+ local_irq_save_hw_notrace(__flags); \
4979
+ __x = __test_and_set_bit(b, a); \
4980
+ local_irq_restore_hw_notrace(__flags); __x; })
4981
+#define clear_bit_safe(b, a) \
4982
+ ({ unsigned long __flags; \
4983
+ local_irq_save_hw_notrace(__flags); \
4984
+ __clear_bit(b, a); \
4985
+ local_irq_restore_hw_notrace(__flags); })
4986
+#else
4987
+#define set_bit_safe(b, a) set_bit(b, a)
4988
+#define test_and_set_bit_safe(b, a) test_and_set_bit(b, a)
4989
+#define clear_bit_safe(b, a) clear_bit(b, a)
4990
+#endif
4991
+
4992
+#define __ipipe_irq_cookie(ipd, irq) (ipd)->irqs[irq].cookie
4993
+#define __ipipe_irq_handler(ipd, irq) (ipd)->irqs[irq].handler
4994
+#define __ipipe_cpudata_irq_hits(ipd, cpu, irq) ipipe_percpudom(ipd, irqall, cpu)[irq]
4995
+
4996
+extern unsigned __ipipe_printk_virq;
4997
+
4998
+extern unsigned long __ipipe_virtual_irq_map;
4999
+
5000
+extern struct list_head __ipipe_pipeline;
5001
+
5002
+extern int __ipipe_event_monitors[];
5003
+
5004
+/* Private interface */
5005
+
5006
+void ipipe_init(void);
5007
+
5008
+#ifdef CONFIG_PROC_FS
5009
+void ipipe_init_proc(void);
5010
+
5011
+#ifdef CONFIG_IPIPE_TRACE
5012
+void __ipipe_init_tracer(void);
5013
+#else /* !CONFIG_IPIPE_TRACE */
5014
+#define __ipipe_init_tracer() do { } while(0)
5015
+#endif /* CONFIG_IPIPE_TRACE */
5016
+
5017
+#else /* !CONFIG_PROC_FS */
5018
+#define ipipe_init_proc() do { } while(0)
5019
+#endif /* CONFIG_PROC_FS */
5020
+
5021
+void __ipipe_init_stage(struct ipipe_domain *ipd);
5022
+
5023
+void __ipipe_cleanup_domain(struct ipipe_domain *ipd);
5024
+
5025
+void __ipipe_add_domain_proc(struct ipipe_domain *ipd);
5026
+
5027
+void __ipipe_remove_domain_proc(struct ipipe_domain *ipd);
5028
+
5029
+void __ipipe_flush_printk(unsigned irq, void *cookie);
5030
+
5031
+void __ipipe_walk_pipeline(struct list_head *pos);
5032
+
5033
+void __ipipe_pend_irq(unsigned irq, struct list_head *head);
5034
+
5035
+int __ipipe_dispatch_event(unsigned event, void *data);
5036
+
5037
+void __ipipe_dispatch_wired_nocheck(struct ipipe_domain *head, unsigned irq);
5038
+
5039
+void __ipipe_dispatch_wired(struct ipipe_domain *head, unsigned irq);
5040
+
5041
+void __ipipe_sync_stage(unsigned long syncmask);
5042
+
5043
+void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned irq);
5044
+
5045
+void __ipipe_lock_irq(struct ipipe_domain *ipd, int cpu, unsigned irq);
5046
+
5047
+void __ipipe_unlock_irq(struct ipipe_domain *ipd, unsigned irq);
5048
+
5049
+void __ipipe_pin_range_globally(unsigned long start, unsigned long end);
5050
+
5051
+/* Must be called hw IRQs off. */
5052
+static inline void ipipe_irq_lock(unsigned irq)
5053
+{
5054
+ __ipipe_lock_irq(ipipe_current_domain, ipipe_processor_id(), irq);
5055
+}
5056
+
5057
+/* Must be called hw IRQs off. */
5058
+static inline void ipipe_irq_unlock(unsigned irq)
5059
+{
5060
+ __ipipe_unlock_irq(ipipe_current_domain, irq);
5061
+}
5062
+
5063
+#ifndef __ipipe_sync_pipeline
5064
+#define __ipipe_sync_pipeline(syncmask) __ipipe_sync_stage(syncmask)
5065
+#endif
5066
+
5067
+#ifndef __ipipe_run_irqtail
5068
+#define __ipipe_run_irqtail() do { } while(0)
5069
+#endif
5070
+
5071
+#define __ipipe_pipeline_head_p(ipd) (&(ipd)->p_link == __ipipe_pipeline.next)
5072
+
5073
+/*
5074
+ * Keep the following as a macro, so that client code could check for
5075
+ * the support of the invariant pipeline head optimization.
5076
+ */
5077
+#define __ipipe_pipeline_head() \
5078
+ list_entry(__ipipe_pipeline.next, struct ipipe_domain, p_link)
5079
+
5080
+#define __ipipe_event_monitored_p(ev) \
5081
+ (__ipipe_event_monitors[ev] > 0 || (ipipe_current_domain->evself & (1LL << ev)))
5082
+
5083
+#ifdef CONFIG_SMP
5084
+
5085
+cpumask_t __ipipe_set_irq_affinity(unsigned irq,
5086
+ cpumask_t cpumask);
5087
+
5088
+int __ipipe_send_ipi(unsigned ipi,
5089
+ cpumask_t cpumask);
5090
+
5091
+#endif /* CONFIG_SMP */
5092
+
5093
+#define ipipe_sigwake_notify(p) \
5094
+do { \
5095
+ if (((p)->flags & PF_EVNOTIFY) && __ipipe_event_monitored_p(IPIPE_EVENT_SIGWAKE)) \
5096
+ __ipipe_dispatch_event(IPIPE_EVENT_SIGWAKE,p); \
5097
+} while(0)
5098
+
5099
+#define ipipe_exit_notify(p) \
5100
+do { \
5101
+ if (((p)->flags & PF_EVNOTIFY) && __ipipe_event_monitored_p(IPIPE_EVENT_EXIT)) \
5102
+ __ipipe_dispatch_event(IPIPE_EVENT_EXIT,p); \
5103
+} while(0)
5104
+
5105
+#define ipipe_setsched_notify(p) \
5106
+do { \
5107
+ if (((p)->flags & PF_EVNOTIFY) && __ipipe_event_monitored_p(IPIPE_EVENT_SETSCHED)) \
5108
+ __ipipe_dispatch_event(IPIPE_EVENT_SETSCHED,p); \
5109
+} while(0)
5110
+
5111
+#define ipipe_schedule_notify(prev, next) \
5112
+do { \
5113
+ if ((((prev)->flags|(next)->flags) & PF_EVNOTIFY) && \
5114
+ __ipipe_event_monitored_p(IPIPE_EVENT_SCHEDULE)) \
5115
+ __ipipe_dispatch_event(IPIPE_EVENT_SCHEDULE,next); \
5116
+} while(0)
5117
+
5118
+#define ipipe_trap_notify(ex, regs) \
5119
+({ \
5120
+ int ret = 0; \
5121
+ if ((test_bit(IPIPE_NOSTACK_FLAG, &ipipe_this_cpudom_var(status)) || \
5122
+ ((current)->flags & PF_EVNOTIFY)) && \
5123
+ __ipipe_event_monitored_p(ex)) \
5124
+ ret = __ipipe_dispatch_event(ex, regs); \
5125
+ ret; \
5126
+})
5127
+
5128
+static inline void ipipe_init_notify(struct task_struct *p)
5129
+{
5130
+ if (__ipipe_event_monitored_p(IPIPE_EVENT_INIT))
5131
+ __ipipe_dispatch_event(IPIPE_EVENT_INIT,p);
5132
+}
5133
+
5134
+struct mm_struct;
5135
+
5136
+static inline void ipipe_cleanup_notify(struct mm_struct *mm)
5137
+{
5138
+ if (__ipipe_event_monitored_p(IPIPE_EVENT_CLEANUP))
5139
+ __ipipe_dispatch_event(IPIPE_EVENT_CLEANUP,mm);
5140
+}
5141
+
5142
+/* Public interface */
5143
+
5144
+int ipipe_register_domain(struct ipipe_domain *ipd,
5145
+ struct ipipe_domain_attr *attr);
5146
+
5147
+int ipipe_unregister_domain(struct ipipe_domain *ipd);
5148
+
5149
+void ipipe_suspend_domain(void);
5150
+
5151
+int ipipe_virtualize_irq(struct ipipe_domain *ipd,
5152
+ unsigned irq,
5153
+ ipipe_irq_handler_t handler,
5154
+ void *cookie,
5155
+ ipipe_irq_ackfn_t acknowledge,
5156
+ unsigned modemask);
5157
+
5158
+int ipipe_control_irq(unsigned irq,
5159
+ unsigned clrmask,
5160
+ unsigned setmask);
5161
+
5162
+unsigned ipipe_alloc_virq(void);
5163
+
5164
+int ipipe_free_virq(unsigned virq);
5165
+
5166
+int ipipe_trigger_irq(unsigned irq);
5167
+
5168
+static inline void __ipipe_propagate_irq(unsigned irq)
5169
+{
5170
+ struct list_head *next = ipipe_current_domain->p_link.next;
5171
+ if (next == &ipipe_root.p_link) {
5172
+ /* Fast path: root must handle all interrupts. */
5173
+ __ipipe_set_irq_pending(&ipipe_root, irq);
5174
+ return;
5175
+ }
5176
+ __ipipe_pend_irq(irq, next);
5177
+}
5178
+
5179
+static inline void __ipipe_schedule_irq(unsigned irq)
5180
+{
5181
+ __ipipe_pend_irq(irq, &ipipe_current_domain->p_link);
5182
+}
5183
+
5184
+static inline void __ipipe_schedule_irq_head(unsigned irq)
5185
+{
5186
+ __ipipe_set_irq_pending(__ipipe_pipeline_head(), irq);
5187
+}
5188
+
5189
+static inline void __ipipe_schedule_irq_root(unsigned irq)
5190
+{
5191
+ __ipipe_set_irq_pending(&ipipe_root, irq);
5192
+}
5193
+
5194
+static inline void ipipe_propagate_irq(unsigned irq)
5195
+{
5196
+ unsigned long flags;
5197
+
5198
+ local_irq_save_hw(flags);
5199
+ __ipipe_propagate_irq(irq);
5200
+ local_irq_restore_hw(flags);
5201
+}
5202
+
5203
+static inline void ipipe_schedule_irq(unsigned irq)
5204
+{
5205
+ unsigned long flags;
5206
+
5207
+ local_irq_save_hw(flags);
5208
+ __ipipe_schedule_irq(irq);
5209
+ local_irq_restore_hw(flags);
5210
+}
5211
+
5212
+static inline void ipipe_schedule_irq_head(unsigned irq)
5213
+{
5214
+ unsigned long flags;
5215
+
5216
+ local_irq_save_hw(flags);
5217
+ __ipipe_schedule_irq_head(irq);
5218
+ local_irq_restore_hw(flags);
5219
+}
5220
+
5221
+static inline void ipipe_schedule_irq_root(unsigned irq)
5222
+{
5223
+ unsigned long flags;
5224
+
5225
+ local_irq_save_hw(flags);
5226
+ __ipipe_schedule_irq_root(irq);
5227
+ local_irq_restore_hw(flags);
5228
+}
5229
+
5230
+void ipipe_stall_pipeline_from(struct ipipe_domain *ipd);
5231
+
5232
+unsigned long ipipe_test_and_stall_pipeline_from(struct ipipe_domain *ipd);
5233
+
5234
+void ipipe_unstall_pipeline_from(struct ipipe_domain *ipd);
5235
+
5236
+unsigned long ipipe_test_and_unstall_pipeline_from(struct ipipe_domain *ipd);
5237
+
5238
+void ipipe_restore_pipeline_from(struct ipipe_domain *ipd,
5239
+ unsigned long x);
5240
+
5241
+static inline unsigned long ipipe_test_pipeline_from(struct ipipe_domain *ipd)
5242
+{
5243
+ return test_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status));
5244
+}
5245
+
5246
+static inline void ipipe_stall_pipeline_head(void)
5247
+{
5248
+ local_irq_disable_hw();
5249
+ __set_bit(IPIPE_STALL_FLAG, &ipipe_head_cpudom_var(status));
5250
+}
5251
+
5252
+static inline unsigned long ipipe_test_and_stall_pipeline_head(void)
5253
+{
5254
+ local_irq_disable_hw();
5255
+ return __test_and_set_bit(IPIPE_STALL_FLAG, &ipipe_head_cpudom_var(status));
5256
+}
5257
+
5258
+void ipipe_unstall_pipeline_head(void);
5259
+
5260
+void __ipipe_restore_pipeline_head(unsigned long x);
5261
+
5262
+static inline void ipipe_restore_pipeline_head(unsigned long x)
5263
+{
5264
+ /* On some archs, __test_and_set_bit() might return different
5265
+ * truth value than test_bit(), so we test the exclusive OR of
5266
+ * both statuses, assuming that the lowest bit is always set in
5267
+ * the truth value (if this is wrong, the failed optimization will
5268
+ * be caught in __ipipe_restore_pipeline_head() if
5269
+ * CONFIG_DEBUG_KERNEL is set). */
5270
+ if ((x ^ test_bit(IPIPE_STALL_FLAG, &ipipe_head_cpudom_var(status))) & 1)
5271
+ __ipipe_restore_pipeline_head(x);
5272
+}
5273
+
5274
+#define ipipe_unstall_pipeline() \
5275
+ ipipe_unstall_pipeline_from(ipipe_current_domain)
5276
+
5277
+#define ipipe_test_and_unstall_pipeline() \
5278
+ ipipe_test_and_unstall_pipeline_from(ipipe_current_domain)
5279
+
5280
+#define ipipe_test_pipeline() \
5281
+ ipipe_test_pipeline_from(ipipe_current_domain)
5282
+
5283
+#define ipipe_test_and_stall_pipeline() \
5284
+ ipipe_test_and_stall_pipeline_from(ipipe_current_domain)
5285
+
5286
+#define ipipe_stall_pipeline() \
5287
+ ipipe_stall_pipeline_from(ipipe_current_domain)
5288
+
5289
+#define ipipe_restore_pipeline(x) \
5290
+ ipipe_restore_pipeline_from(ipipe_current_domain, (x))
5291
+
5292
+void ipipe_init_attr(struct ipipe_domain_attr *attr);
5293
+
5294
+int ipipe_get_sysinfo(struct ipipe_sysinfo *sysinfo);
5295
+
5296
+unsigned long ipipe_critical_enter(void (*syncfn) (void));
5297
+
5298
+void ipipe_critical_exit(unsigned long flags);
5299
+
5300
+static inline void ipipe_set_printk_sync(struct ipipe_domain *ipd)
5301
+{
5302
+ set_bit(IPIPE_SPRINTK_FLAG, &ipd->flags);
5303
+}
5304
+
5305
+static inline void ipipe_set_printk_async(struct ipipe_domain *ipd)
5306
+{
5307
+ clear_bit(IPIPE_SPRINTK_FLAG, &ipd->flags);
5308
+}
5309
+
5310
+static inline void ipipe_set_foreign_stack(struct ipipe_domain *ipd)
5311
+{
5312
+ /* Must be called hw interrupts off. */
5313
+ __set_bit(IPIPE_NOSTACK_FLAG, &ipipe_cpudom_var(ipd, status));
5314
+}
5315
+
5316
+static inline void ipipe_clear_foreign_stack(struct ipipe_domain *ipd)
5317
+{
5318
+ /* Must be called hw interrupts off. */
5319
+ __clear_bit(IPIPE_NOSTACK_FLAG, &ipipe_cpudom_var(ipd, status));
5320
+}
5321
+
5322
+#ifndef ipipe_safe_current
5323
+#define ipipe_safe_current() \
5324
+({ \
5325
+ struct task_struct *p; \
5326
+ p = test_bit(IPIPE_NOSTACK_FLAG, \
5327
+ &ipipe_this_cpudom_var(status)) ? &init_task : current; \
5328
+ p; \
5329
+})
5330
+#endif
5331
+
5332
+ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd,
5333
+ unsigned event,
5334
+ ipipe_event_handler_t handler);
5335
+
5336
+cpumask_t ipipe_set_irq_affinity(unsigned irq,
5337
+ cpumask_t cpumask);
5338
+
5339
+int ipipe_send_ipi(unsigned ipi,
5340
+ cpumask_t cpumask);
5341
+
5342
+int ipipe_setscheduler_root(struct task_struct *p,
5343
+ int policy,
5344
+ int prio);
5345
+
5346
+int ipipe_reenter_root(struct task_struct *prev,
5347
+ int policy,
5348
+ int prio);
5349
+
5350
+int ipipe_alloc_ptdkey(void);
5351
+
5352
+int ipipe_free_ptdkey(int key);
5353
+
5354
+int ipipe_set_ptd(int key,
5355
+ void *value);
5356
+
5357
+void *ipipe_get_ptd(int key);
5358
+
5359
+int ipipe_disable_ondemand_mappings(struct task_struct *tsk);
5360
+
5361
+#define local_irq_enable_hw_cond() local_irq_enable_hw()
5362
+#define local_irq_disable_hw_cond() local_irq_disable_hw()
5363
+#define local_irq_save_hw_cond(flags) local_irq_save_hw(flags)
5364
+#define local_irq_restore_hw_cond(flags) local_irq_restore_hw(flags)
5365
+
5366
+#define local_irq_enable_nohead(ipd) \
5367
+ do { \
5368
+ if (!__ipipe_pipeline_head_p(ipd)) \
5369
+ local_irq_enable_hw(); \
5370
+ } while(0)
5371
+
5372
+#define local_irq_disable_nohead(ipd) \
5373
+ do { \
5374
+ if (!__ipipe_pipeline_head_p(ipd)) \
5375
+ local_irq_disable_hw(); \
5376
+ } while(0)
5377
+
5378
+#define local_irq_save_full(vflags, rflags) \
5379
+ do { \
5380
+ local_irq_save(vflags); \
5381
+ local_irq_save_hw(rflags); \
5382
+ } while(0)
5383
+
5384
+#define local_irq_restore_full(vflags, rflags) \
5385
+ do { \
5386
+ local_irq_restore_hw(rflags); \
5387
+ local_irq_restore(vflags); \
5388
+ } while(0)
5389
+
5390
+static inline void local_irq_restore_nosync(unsigned long x)
5391
+{
5392
+ if (raw_irqs_disabled_flags(x))
5393
+ set_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipipe_root_domain, status));
5394
+ else
5395
+ clear_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipipe_root_domain, status));
5396
+}
5397
+
5398
+#define ipipe_root_domain_p (ipipe_current_domain == ipipe_root_domain)
5399
+
5400
+static inline void ipipe_nmi_enter(void)
5401
+{
5402
+ int cpu = ipipe_processor_id();
5403
+
5404
+ per_cpu(ipipe_nmi_saved_root, cpu) = ipipe_root_cpudom_var(status);
5405
+ __set_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status));
5406
+
5407
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
5408
+ per_cpu(ipipe_saved_context_check_state, cpu) =
5409
+ ipipe_disable_context_check(cpu);
5410
+#endif /* CONFIG_IPIPE_DEBUG_CONTEXT */
5411
+}
5412
+
5413
+static inline void ipipe_nmi_exit(void)
5414
+{
5415
+ int cpu = ipipe_processor_id();
5416
+
5417
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
5418
+ ipipe_restore_context_check
5419
+ (cpu, per_cpu(ipipe_saved_context_check_state, cpu));
5420
+#endif /* CONFIG_IPIPE_DEBUG_CONTEXT */
5421
+
5422
+ if (!test_bit(IPIPE_STALL_FLAG, &per_cpu(ipipe_nmi_saved_root, cpu)))
5423
+ __clear_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status));
5424
+}
5425
+
5426
+#else /* !CONFIG_IPIPE */
5427
+
5428
+#define ipipe_init() do { } while(0)
5429
+#define ipipe_suspend_domain() do { } while(0)
5430
+#define ipipe_sigwake_notify(p) do { } while(0)
5431
+#define ipipe_setsched_notify(p) do { } while(0)
5432
+#define ipipe_init_notify(p) do { } while(0)
5433
+#define ipipe_exit_notify(p) do { } while(0)
5434
+#define ipipe_cleanup_notify(mm) do { } while(0)
5435
+#define ipipe_trap_notify(t,r) 0
5436
+#define ipipe_init_proc() do { } while(0)
5437
+
5438
+static inline void __ipipe_pin_range_globally(unsigned long start,
5439
+ unsigned long end)
5440
+{
5441
+}
5442
+
5443
+#define local_irq_enable_hw_cond() do { } while(0)
5444
+#define local_irq_disable_hw_cond() do { } while(0)
5445
+#define local_irq_save_hw_cond(flags) do { (void)(flags); } while(0)
5446
+#define local_irq_restore_hw_cond(flags) do { } while(0)
5447
+
5448
+#define ipipe_irq_lock(irq) do { } while(0)
5449
+#define ipipe_irq_unlock(irq) do { } while(0)
5450
+
5451
+#define ipipe_root_domain_p 1
5452
+#define ipipe_safe_current current
5453
+#define ipipe_processor_id() smp_processor_id()
5454
+
5455
+#define ipipe_nmi_enter() do { } while (0)
5456
+#define ipipe_nmi_exit() do { } while (0)
5457
+
5458
+#define local_irq_disable_head() local_irq_disable()
5459
+
5460
+#define local_irq_save_full(vflags, rflags) do { (void)(vflags); local_irq_save(rflags); } while(0)
5461
+#define local_irq_restore_full(vflags, rflags) do { (void)(vflags); local_irq_restore(rflags); } while(0)
5462
+#define local_irq_restore_nosync(vflags) local_irq_restore(vflags)
5463
+
5464
+#endif /* CONFIG_IPIPE */
5465
+
5466
+#endif /* !__LINUX_IPIPE_H */
5467
diff --git a/include/linux/ipipe_base.h b/include/linux/ipipe_base.h
5468
new file mode 100644
5469
index 0000000..c49fe14
5470
--- /dev/null
5471
+++ b/include/linux/ipipe_base.h
5472
@@ -0,0 +1,90 @@
5473
+/* -*- linux-c -*-
5474
+ * include/linux/ipipe_base.h
5475
+ *
5476
+ * Copyright (C) 2002-2007 Philippe Gerum.
5477
+ * 2007 Jan Kiszka.
5478
+ *
5479
+ * This program is free software; you can redistribute it and/or modify
5480
+ * it under the terms of the GNU General Public License as published by
5481
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
5482
+ * USA; either version 2 of the License, or (at your option) any later
5483
+ * version.
5484
+ *
5485
+ * This program is distributed in the hope that it will be useful,
5486
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
5487
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
5488
+ * GNU General Public License for more details.
5489
+ *
5490
+ * You should have received a copy of the GNU General Public License
5491
+ * along with this program; if not, write to the Free Software
5492
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
5493
+ */
5494
+
5495
+#ifndef __LINUX_IPIPE_BASE_H
5496
+#define __LINUX_IPIPE_BASE_H
5497
+
5498
+#ifdef CONFIG_IPIPE
5499
+
5500
+#include <linux/bitops.h>
5501
+#include <asm/ipipe_base.h>
5502
+
5503
+/* Number of virtual IRQs */
5504
+#define IPIPE_NR_VIRQS BITS_PER_LONG
5505
+/* First virtual IRQ # */
5506
+#define IPIPE_VIRQ_BASE (((IPIPE_NR_XIRQS + BITS_PER_LONG - 1) / BITS_PER_LONG) * BITS_PER_LONG)
5507
+/* Total number of IRQ slots */
5508
+#define IPIPE_NR_IRQS (IPIPE_VIRQ_BASE + IPIPE_NR_VIRQS)
5509
+/* Number of indirect words needed to map the whole IRQ space. */
5510
+#define IPIPE_IRQ_IWORDS ((IPIPE_NR_IRQS + BITS_PER_LONG - 1) / BITS_PER_LONG)
5511
+#define IPIPE_IRQ_IMASK (BITS_PER_LONG - 1)
5512
+#define IPIPE_IRQMASK_ANY (~0L)
5513
+#define IPIPE_IRQMASK_VIRT (IPIPE_IRQMASK_ANY << (IPIPE_VIRQ_BASE / BITS_PER_LONG))
5514
+
5515
+/* Per-cpu pipeline status */
5516
+#define IPIPE_STALL_FLAG 0 /* Stalls a pipeline stage -- guaranteed at bit #0 */
5517
+#define IPIPE_SYNC_FLAG 1 /* The interrupt syncer is running for the domain */
5518
+#define IPIPE_NOSTACK_FLAG 2 /* Domain currently runs on a foreign stack */
5519
+
5520
+#define IPIPE_STALL_MASK (1L << IPIPE_STALL_FLAG)
5521
+#define IPIPE_SYNC_MASK (1L << IPIPE_SYNC_FLAG)
5522
+#define IPIPE_NOSTACK_MASK (1L << IPIPE_NOSTACK_FLAG)
5523
+
5524
+typedef void (*ipipe_irq_handler_t)(unsigned irq,
5525
+ void *cookie);
5526
+
5527
+extern struct ipipe_domain ipipe_root;
5528
+
5529
+#define ipipe_root_domain (&ipipe_root)
5530
+
5531
+void __ipipe_unstall_root(void);
5532
+
5533
+void __ipipe_restore_root(unsigned long x);
5534
+
5535
+#define ipipe_preempt_disable(flags) local_irq_save_hw(flags)
5536
+#define ipipe_preempt_enable(flags) local_irq_restore_hw(flags)
5537
+
5538
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
5539
+void ipipe_check_context(struct ipipe_domain *border_ipd);
5540
+#else /* !CONFIG_IPIPE_DEBUG_CONTEXT */
5541
+static inline void ipipe_check_context(struct ipipe_domain *border_ipd) { }
5542
+#endif /* !CONFIG_IPIPE_DEBUG_CONTEXT */
5543
+
5544
+/* Generic features */
5545
+
5546
+#ifdef CONFIG_GENERIC_CLOCKEVENTS
5547
+#define __IPIPE_FEATURE_REQUEST_TICKDEV 1
5548
+#endif
5549
+#define __IPIPE_FEATURE_DELAYED_ATOMICSW 1
5550
+#define __IPIPE_FEATURE_FASTPEND_IRQ 1
5551
+#define __IPIPE_FEATURE_TRACE_EVENT 1
5552
+
5553
+#else /* !CONFIG_IPIPE */
5554
+#define ipipe_preempt_disable(flags) do { \
5555
+ preempt_disable(); \
5556
+ (void)(flags); \
5557
+ } while (0)
5558
+#define ipipe_preempt_enable(flags) preempt_enable()
5559
+#define ipipe_check_context(ipd) do { } while(0)
5560
+#endif /* CONFIG_IPIPE */
5561
+
5562
+#endif /* !__LINUX_IPIPE_BASE_H */
5563
diff --git a/include/linux/ipipe_compat.h b/include/linux/ipipe_compat.h
5564
new file mode 100644
5565
index 0000000..50a245c
5566
--- /dev/null
5567
+++ b/include/linux/ipipe_compat.h
5568
@@ -0,0 +1,54 @@
5569
+/* -*- linux-c -*-
5570
+ * include/linux/ipipe_compat.h
5571
+ *
5572
+ * Copyright (C) 2007 Philippe Gerum.
5573
+ *
5574
+ * This program is free software; you can redistribute it and/or modify
5575
+ * it under the terms of the GNU General Public License as published by
5576
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
5577
+ * USA; either version 2 of the License, or (at your option) any later
5578
+ * version.
5579
+ *
5580
+ * This program is distributed in the hope that it will be useful,
5581
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
5582
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
5583
+ * GNU General Public License for more details.
5584
+ *
5585
+ * You should have received a copy of the GNU General Public License
5586
+ * along with this program; if not, write to the Free Software
5587
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
5588
+ */
5589
+
5590
+#ifndef __LINUX_IPIPE_COMPAT_H
5591
+#define __LINUX_IPIPE_COMPAT_H
5592
+
5593
+#ifdef CONFIG_IPIPE_COMPAT
5594
+/*
5595
+ * OBSOLETE: defined only for backward compatibility. Will be removed
5596
+ * in future releases, please update client code accordingly.
5597
+ */
5598
+
5599
+#ifdef CONFIG_SMP
5600
+#define ipipe_declare_cpuid int cpuid
5601
+#define ipipe_load_cpuid() do { \
5602
+ cpuid = ipipe_processor_id(); \
5603
+ } while(0)
5604
+#define ipipe_lock_cpu(flags) do { \
5605
+ local_irq_save_hw(flags); \
5606
+ cpuid = ipipe_processor_id(); \
5607
+ } while(0)
5608
+#define ipipe_unlock_cpu(flags) local_irq_restore_hw(flags)
5609
+#define ipipe_get_cpu(flags) ipipe_lock_cpu(flags)
5610
+#define ipipe_put_cpu(flags) ipipe_unlock_cpu(flags)
5611
+#else /* !CONFIG_SMP */
5612
+#define ipipe_declare_cpuid const int cpuid = 0
5613
+#define ipipe_load_cpuid() do { } while(0)
5614
+#define ipipe_lock_cpu(flags) local_irq_save_hw(flags)
5615
+#define ipipe_unlock_cpu(flags) local_irq_restore_hw(flags)
5616
+#define ipipe_get_cpu(flags) do { (void)(flags); } while(0)
5617
+#define ipipe_put_cpu(flags) do { } while(0)
5618
+#endif /* CONFIG_SMP */
5619
+
5620
+#endif /* CONFIG_IPIPE_COMPAT */
5621
+
5622
+#endif /* !__LINUX_IPIPE_COMPAT_H */
5623
diff --git a/include/linux/ipipe_percpu.h b/include/linux/ipipe_percpu.h
5624
new file mode 100644
5625
index 0000000..2f41af3
5626
--- /dev/null
5627
+++ b/include/linux/ipipe_percpu.h
5628
@@ -0,0 +1,84 @@
5629
+/* -*- linux-c -*-
5630
+ * include/linux/ipipe_percpu.h
5631
+ *
5632
+ * Copyright (C) 2007 Philippe Gerum.
5633
+ *
5634
+ * This program is free software; you can redistribute it and/or modify
5635
+ * it under the terms of the GNU General Public License as published by
5636
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
5637
+ * USA; either version 2 of the License, or (at your option) any later
5638
+ * version.
5639
+ *
5640
+ * This program is distributed in the hope that it will be useful,
5641
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
5642
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
5643
+ * GNU General Public License for more details.
5644
+ *
5645
+ * You should have received a copy of the GNU General Public License
5646
+ * along with this program; if not, write to the Free Software
5647
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
5648
+ */
5649
+
5650
+#ifndef __LINUX_IPIPE_PERCPU_H
5651
+#define __LINUX_IPIPE_PERCPU_H
5652
+
5653
+#include <asm/percpu.h>
5654
+#include <asm/ptrace.h>
5655
+
5656
+struct ipipe_domain;
5657
+
5658
+struct ipipe_percpu_domain_data {
5659
+ unsigned long status; /* <= Must be first in struct. */
5660
+ unsigned long irqpend_himask;
5661
+ unsigned long irqpend_lomask[IPIPE_IRQ_IWORDS];
5662
+ unsigned long irqheld_mask[IPIPE_IRQ_IWORDS];
5663
+ unsigned long irqall[IPIPE_NR_IRQS];
5664
+ u64 evsync;
5665
+};
5666
+
5667
+#ifdef CONFIG_SMP
5668
+#define ipipe_percpudom_ptr(ipd, cpu) \
5669
+ (&per_cpu(ipipe_percpu_darray, cpu)[(ipd)->slot])
5670
+#define ipipe_cpudom_ptr(ipd) \
5671
+ (&__raw_get_cpu_var(ipipe_percpu_darray)[(ipd)->slot])
5672
+#else
5673
+DECLARE_PER_CPU(struct ipipe_percpu_domain_data *, ipipe_percpu_daddr[CONFIG_IPIPE_DOMAINS]);
5674
+#define ipipe_percpudom_ptr(ipd, cpu) \
5675
+ (per_cpu(ipipe_percpu_daddr, cpu)[(ipd)->slot])
5676
+#define ipipe_cpudom_ptr(ipd) \
5677
+ (__raw_get_cpu_var(ipipe_percpu_daddr)[(ipd)->slot])
5678
+#endif
5679
+#define ipipe_percpudom(ipd, var, cpu) (ipipe_percpudom_ptr(ipd, cpu)->var)
5680
+#define ipipe_cpudom_var(ipd, var) (ipipe_cpudom_ptr(ipd)->var)
5681
+
5682
+#define IPIPE_ROOT_SLOT 0
5683
+#define IPIPE_HEAD_SLOT (CONFIG_IPIPE_DOMAINS - 1)
5684
+
5685
+DECLARE_PER_CPU(struct ipipe_percpu_domain_data, ipipe_percpu_darray[CONFIG_IPIPE_DOMAINS]);
5686
+
5687
+DECLARE_PER_CPU(struct ipipe_domain *, ipipe_percpu_domain);
5688
+
5689
+DECLARE_PER_CPU(unsigned long, ipipe_nmi_saved_root);
5690
+
5691
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
5692
+DECLARE_PER_CPU(int, ipipe_percpu_context_check);
5693
+DECLARE_PER_CPU(int, ipipe_saved_context_check_state);
5694
+#endif
5695
+
5696
+#define ipipe_percpu(var, cpu) per_cpu(var, cpu)
5697
+#define ipipe_cpu_var(var) __raw_get_cpu_var(var)
5698
+
5699
+#define ipipe_root_cpudom_ptr(var) \
5700
+ (&__raw_get_cpu_var(ipipe_percpu_darray)[IPIPE_ROOT_SLOT])
5701
+
5702
+#define ipipe_root_cpudom_var(var) ipipe_root_cpudom_ptr()->var
5703
+
5704
+#define ipipe_this_cpudom_var(var) \
5705
+ ipipe_cpudom_var(ipipe_current_domain, var)
5706
+
5707
+#define ipipe_head_cpudom_ptr() \
5708
+ (&__raw_get_cpu_var(ipipe_percpu_darray)[IPIPE_HEAD_SLOT])
5709
+
5710
+#define ipipe_head_cpudom_var(var) ipipe_head_cpudom_ptr()->var
5711
+
5712
+#endif /* !__LINUX_IPIPE_PERCPU_H */
5713
diff --git a/include/linux/ipipe_tickdev.h b/include/linux/ipipe_tickdev.h
5714
new file mode 100644
5715
index 0000000..4a1cb1b
5716
--- /dev/null
5717
+++ b/include/linux/ipipe_tickdev.h
5718
@@ -0,0 +1,58 @@
5719
+/* -*- linux-c -*-
5720
+ * include/linux/ipipe_tickdev.h
5721
+ *
5722
+ * Copyright (C) 2007 Philippe Gerum.
5723
+ *
5724
+ * This program is free software; you can redistribute it and/or modify
5725
+ * it under the terms of the GNU General Public License as published by
5726
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
5727
+ * USA; either version 2 of the License, or (at your option) any later
5728
+ * version.
5729
+ *
5730
+ * This program is distributed in the hope that it will be useful,
5731
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
5732
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
5733
+ * GNU General Public License for more details.
5734
+ *
5735
+ * You should have received a copy of the GNU General Public License
5736
+ * along with this program; if not, write to the Free Software
5737
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
5738
+ */
5739
+
5740
+#ifndef __LINUX_IPIPE_TICKDEV_H
5741
+#define __LINUX_IPIPE_TICKDEV_H
5742
+
5743
+#if defined(CONFIG_IPIPE) && defined(CONFIG_GENERIC_CLOCKEVENTS)
5744
+
5745
+#include <linux/clockchips.h>
5746
+
5747
+struct tick_device;
5748
+
5749
+struct ipipe_tick_device {
5750
+
5751
+ void (*emul_set_mode)(enum clock_event_mode,
5752
+ struct clock_event_device *cdev);
5753
+ int (*emul_set_tick)(unsigned long delta,
5754
+ struct clock_event_device *cdev);
5755
+ void (*real_set_mode)(enum clock_event_mode mode,
5756
+ struct clock_event_device *cdev);
5757
+ int (*real_set_tick)(unsigned long delta,
5758
+ struct clock_event_device *cdev);
5759
+ struct tick_device *slave;
5760
+ unsigned long real_max_delta_ns;
5761
+ unsigned long real_mult;
5762
+ int real_shift;
5763
+};
5764
+
5765
+int ipipe_request_tickdev(const char *devname,
5766
+ void (*emumode)(enum clock_event_mode mode,
5767
+ struct clock_event_device *cdev),
5768
+ int (*emutick)(unsigned long evt,
5769
+ struct clock_event_device *cdev),
5770
+ int cpu, unsigned long *tmfreq);
5771
+
5772
+void ipipe_release_tickdev(int cpu);
5773
+
5774
+#endif /* CONFIG_IPIPE && CONFIG_GENERIC_CLOCKEVENTS */
5775
+
5776
+#endif /* !__LINUX_IPIPE_TICKDEV_H */
5777
diff --git a/include/linux/ipipe_trace.h b/include/linux/ipipe_trace.h
5778
new file mode 100644
5779
index 0000000..627b354
5780
--- /dev/null
5781
+++ b/include/linux/ipipe_trace.h
5782
@@ -0,0 +1,72 @@
5783
+/* -*- linux-c -*-
5784
+ * include/linux/ipipe_trace.h
5785
+ *
5786
+ * Copyright (C) 2005 Luotao Fu.
5787
+ * 2005-2007 Jan Kiszka.
5788
+ *
5789
+ * This program is free software; you can redistribute it and/or modify
5790
+ * it under the terms of the GNU General Public License as published by
5791
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
5792
+ * USA; either version 2 of the License, or (at your option) any later
5793
+ * version.
5794
+ *
5795
+ * This program is distributed in the hope that it will be useful,
5796
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
5797
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
5798
+ * GNU General Public License for more details.
5799
+ *
5800
+ * You should have received a copy of the GNU General Public License
5801
+ * along with this program; if not, write to the Free Software
5802
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
5803
+ */
5804
+
5805
+#ifndef _LINUX_IPIPE_TRACE_H
5806
+#define _LINUX_IPIPE_TRACE_H
5807
+
5808
+#ifdef CONFIG_IPIPE_TRACE
5809
+
5810
+#include <linux/types.h>
5811
+
5812
+void ipipe_trace_begin(unsigned long v);
5813
+void ipipe_trace_end(unsigned long v);
5814
+void ipipe_trace_freeze(unsigned long v);
5815
+void ipipe_trace_special(unsigned char special_id, unsigned long v);
5816
+void ipipe_trace_pid(pid_t pid, short prio);
5817
+void ipipe_trace_event(unsigned char id, unsigned long delay_tsc);
5818
+int ipipe_trace_max_reset(void);
5819
+int ipipe_trace_frozen_reset(void);
5820
+
5821
+#else /* !CONFIG_IPIPE_TRACE */
5822
+
5823
+#define ipipe_trace_begin(v) do { (void)(v); } while(0)
5824
+#define ipipe_trace_end(v) do { (void)(v); } while(0)
5825
+#define ipipe_trace_freeze(v) do { (void)(v); } while(0)
5826
+#define ipipe_trace_special(id, v) do { (void)(id); (void)(v); } while(0)
5827
+#define ipipe_trace_pid(pid, prio) do { (void)(pid); (void)(prio); } while(0)
5828
+#define ipipe_trace_event(id, delay_tsc) do { (void)(id); (void)(delay_tsc); } while(0)
5829
+#define ipipe_trace_max_reset() do { } while(0)
5830
+#define ipipe_trace_froze_reset() do { } while(0)
5831
+
5832
+#endif /* !CONFIG_IPIPE_TRACE */
5833
+
5834
+#ifdef CONFIG_IPIPE_TRACE_PANIC
5835
+void ipipe_trace_panic_freeze(void);
5836
+void ipipe_trace_panic_dump(void);
5837
+#else
5838
+static inline void ipipe_trace_panic_freeze(void) { }
5839
+static inline void ipipe_trace_panic_dump(void) { }
5840
+#endif
5841
+
5842
+#ifdef CONFIG_IPIPE_TRACE_IRQSOFF
5843
+#define ipipe_trace_irq_entry(irq) ipipe_trace_begin(irq)
5844
+#define ipipe_trace_irq_exit(irq) ipipe_trace_end(irq)
5845
+#define ipipe_trace_irqsoff() ipipe_trace_begin(0x80000000UL)
5846
+#define ipipe_trace_irqson() ipipe_trace_end(0x80000000UL)
5847
+#else
5848
+#define ipipe_trace_irq_entry(irq) do { (void)(irq);} while(0)
5849
+#define ipipe_trace_irq_exit(irq) do { (void)(irq);} while(0)
5850
+#define ipipe_trace_irqsoff() do { } while(0)
5851
+#define ipipe_trace_irqson() do { } while(0)
5852
+#endif
5853
+
5854
+#endif /* !__LINUX_IPIPE_TRACE_H */
5855
diff --git a/include/linux/irq.h b/include/linux/irq.h
5856
index 3dddfa7..6ce9115 100644
5857
--- a/include/linux/irq.h
5858
+++ b/include/linux/irq.h
5859
@@ -114,6 +114,9 @@ struct irq_chip {
5860
5861
void (*end)(unsigned int irq);
5862
void (*set_affinity)(unsigned int irq, cpumask_t dest);
5863
+#ifdef CONFIG_IPIPE
5864
+ void (*move)(unsigned int irq);
5865
+#endif /* CONFIG_IPIPE */
5866
int (*retrigger)(unsigned int irq);
5867
int (*set_type)(unsigned int irq, unsigned int flow_type);
5868
int (*set_wake)(unsigned int irq, unsigned int on);
5869
@@ -153,6 +156,12 @@ struct irq_chip {
5870
* @name: flow handler name for /proc/interrupts output
5871
*/
5872
struct irq_desc {
5873
+#ifdef CONFIG_IPIPE
5874
+ void (*ipipe_ack)(unsigned int irq,
5875
+ struct irq_desc *desc);
5876
+ void (*ipipe_end)(unsigned int irq,
5877
+ struct irq_desc *desc);
5878
+#endif /* CONFIG_IPIPE */
5879
unsigned int irq;
5880
irq_flow_handler_t handle_irq;
5881
struct irq_chip *chip;
5882
diff --git a/include/linux/kernel.h b/include/linux/kernel.h
5883
index dc7e0d0..88d5034 100644
5884
--- a/include/linux/kernel.h
5885
+++ b/include/linux/kernel.h
5886
@@ -14,6 +14,7 @@
5887
#include <linux/compiler.h>
5888
#include <linux/bitops.h>
5889
#include <linux/log2.h>
5890
+#include <linux/ipipe_base.h>
5891
#include <linux/typecheck.h>
5892
#include <linux/ratelimit.h>
5893
#include <linux/dynamic_printk.h>
5894
@@ -111,9 +112,12 @@ struct user;
5895
5896
#ifdef CONFIG_PREEMPT_VOLUNTARY
5897
extern int _cond_resched(void);
5898
-# define might_resched() _cond_resched()
5899
+# define might_resched() do { \
5900
+ ipipe_check_context(ipipe_root_domain); \
5901
+ _cond_resched(); \
5902
+ } while (0)
5903
#else
5904
-# define might_resched() do { } while (0)
5905
+# define might_resched() ipipe_check_context(ipipe_root_domain)
5906
#endif
5907
5908
#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
5909
diff --git a/include/linux/linkage.h b/include/linux/linkage.h
5910
index 9fd1f85..8018d53 100644
5911
--- a/include/linux/linkage.h
5912
+++ b/include/linux/linkage.h
5913
@@ -93,4 +93,8 @@
5914
#define ATTRIB_NORET __attribute__((noreturn))
5915
#define NORET_AND noreturn,
5916
5917
+#ifndef notrace
5918
+#define notrace __attribute__((no_instrument_function))
5919
+#endif
5920
+
5921
#endif
5922
diff --git a/include/linux/mm.h b/include/linux/mm.h
5923
index 5c7f5f5..3bd32c2 100644
5924
--- a/include/linux/mm.h
5925
+++ b/include/linux/mm.h
5926
@@ -115,6 +115,8 @@ extern unsigned int kobjsize(const void *objp);
5927
#define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */
5928
#define VM_SAO 0x20000000 /* Strong Access Ordering (powerpc) */
5929
5930
+#define VM_PINNED 0x40000000 /* Disable faults for the vma */
5931
+
5932
#ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */
5933
#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
5934
#endif
5935
diff --git a/include/linux/preempt.h b/include/linux/preempt.h
5936
index 72b1a10..80553be 100644
5937
--- a/include/linux/preempt.h
5938
+++ b/include/linux/preempt.h
5939
@@ -9,13 +9,20 @@
5940
#include <linux/thread_info.h>
5941
#include <linux/linkage.h>
5942
#include <linux/list.h>
5943
+#include <linux/ipipe_base.h>
5944
5945
#if defined(CONFIG_DEBUG_PREEMPT) || defined(CONFIG_PREEMPT_TRACER)
5946
extern void add_preempt_count(int val);
5947
extern void sub_preempt_count(int val);
5948
#else
5949
-# define add_preempt_count(val) do { preempt_count() += (val); } while (0)
5950
-# define sub_preempt_count(val) do { preempt_count() -= (val); } while (0)
5951
+# define add_preempt_count(val) do { \
5952
+ ipipe_check_context(ipipe_root_domain); \
5953
+ preempt_count() += (val); \
5954
+ } while (0)
5955
+# define sub_preempt_count(val) do { \
5956
+ ipipe_check_context(ipipe_root_domain); \
5957
+ preempt_count() -= (val); \
5958
+ } while (0)
5959
#endif
5960
5961
#define inc_preempt_count() add_preempt_count(1)
5962
diff --git a/include/linux/sched.h b/include/linux/sched.h
5963
index 3883c32..7a30b68 100644
5964
--- a/include/linux/sched.h
5965
+++ b/include/linux/sched.h
5966
@@ -59,6 +59,7 @@ struct sched_param {
5967
#include <linux/errno.h>
5968
#include <linux/nodemask.h>
5969
#include <linux/mm_types.h>
5970
+#include <linux/ipipe.h>
5971
5972
#include <asm/system.h>
5973
#include <asm/page.h>
5974
@@ -181,6 +182,13 @@ extern unsigned long long time_sync_thresh;
5975
/* in tsk->state again */
5976
#define TASK_DEAD 64
5977
#define TASK_WAKEKILL 128
5978
+#ifdef CONFIG_IPIPE
5979
+#define TASK_ATOMICSWITCH 512
5980
+#define TASK_NOWAKEUP 1024
5981
+#else /* !CONFIG_IPIPE */
5982
+#define TASK_ATOMICSWITCH 0
5983
+#define TASK_NOWAKEUP 0
5984
+#endif /* CONFIG_IPIPE */
5985
5986
/* Convenience macros for the sake of set_task_state */
5987
#define TASK_KILLABLE (TASK_WAKEKILL | TASK_UNINTERRUPTIBLE)
5988
@@ -289,6 +297,15 @@ extern void account_process_tick(struct task_struct *task, int user);
5989
extern void update_process_times(int user);
5990
extern void scheduler_tick(void);
5991
5992
+#ifdef CONFIG_IPIPE
5993
+extern void update_root_process_times(struct pt_regs *regs);
5994
+#else /* !CONFIG_IPIPE */
5995
+static inline void update_root_process_times(struct pt_regs *regs)
5996
+{
5997
+ update_process_times(user_mode(regs));
5998
+}
5999
+#endif /* CONFIG_IPIPE */
6000
+
6001
extern void sched_show_task(struct task_struct *p);
6002
6003
#ifdef CONFIG_DETECT_SOFTLOCKUP
6004
@@ -1331,6 +1348,9 @@ struct task_struct {
6005
#endif
6006
atomic_t fs_excl; /* holding fs exclusive resources */
6007
struct rcu_head rcu;
6008
+#ifdef CONFIG_IPIPE
6009
+ void *ptd[IPIPE_ROOT_NPTDKEYS];
6010
+#endif
6011
6012
/*
6013
* cache last used pipe for splice
6014
@@ -1542,6 +1562,11 @@ extern cputime_t task_gtime(struct task_struct *p);
6015
#define PF_EXITING 0x00000004 /* getting shut down */
6016
#define PF_EXITPIDONE 0x00000008 /* pi exit done on shut down */
6017
#define PF_VCPU 0x00000010 /* I'm a virtual CPU */
6018
+#ifdef CONFIG_IPIPE
6019
+#define PF_EVNOTIFY 0x00000020 /* Notify other domains about internal events */
6020
+#else
6021
+#define PF_EVNOTIFY 0
6022
+#endif /* CONFIG_IPIPE */
6023
#define PF_FORKNOEXEC 0x00000040 /* forked but didn't exec */
6024
#define PF_SUPERPRIV 0x00000100 /* used super-user privileges */
6025
#define PF_DUMPCORE 0x00000200 /* dumped core */
6026
diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h
6027
index e0c0fcc..2044388 100644
6028
--- a/include/linux/spinlock.h
6029
+++ b/include/linux/spinlock.h
6030
@@ -90,10 +90,14 @@ extern int __lockfunc generic__raw_read_trylock(raw_rwlock_t *lock);
6031
# include <linux/spinlock_up.h>
6032
#endif
6033
6034
+#undef TYPE_EQUAL
6035
+#define TYPE_EQUAL(lock, type) \
6036
+ __builtin_types_compatible_p(typeof(lock), type *)
6037
+
6038
#ifdef CONFIG_DEBUG_SPINLOCK
6039
extern void __spin_lock_init(spinlock_t *lock, const char *name,
6040
struct lock_class_key *key);
6041
-# define spin_lock_init(lock) \
6042
+# define _spin_lock_init(lock) \
6043
do { \
6044
static struct lock_class_key __key; \
6045
\
6046
@@ -101,10 +105,21 @@ do { \
6047
} while (0)
6048
6049
#else
6050
-# define spin_lock_init(lock) \
6051
+# define _spin_lock_init(lock) \
6052
do { *(lock) = SPIN_LOCK_UNLOCKED; } while (0)
6053
#endif
6054
6055
+# define spin_lock_init(lock) \
6056
+ do { \
6057
+ if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) \
6058
+ do { \
6059
+ IPIPE_DEFINE_SPINLOCK(__lock__); \
6060
+ *((ipipe_spinlock_t *)lock) = __lock__; \
6061
+ } while(0); \
6062
+ else \
6063
+ _spin_lock_init((spinlock_t *)lock); \
6064
+ } while(0)
6065
+
6066
#ifdef CONFIG_DEBUG_SPINLOCK
6067
extern void __rwlock_init(rwlock_t *lock, const char *name,
6068
struct lock_class_key *key);
6069
@@ -179,7 +194,94 @@ do { \
6070
#define read_trylock(lock) __cond_lock(lock, _read_trylock(lock))
6071
#define write_trylock(lock) __cond_lock(lock, _write_trylock(lock))
6072
6073
-#define spin_lock(lock) _spin_lock(lock)
6074
+#define PICK_SPINOP(op, lock) \
6075
+do { \
6076
+ if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) \
6077
+ __raw_spin##op(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \
6078
+ else if (TYPE_EQUAL(lock, spinlock_t)) \
6079
+ _spin##op((spinlock_t *)(lock)); \
6080
+} while (0)
6081
+
6082
+#define PICK_SPINOP_RAW(op, lock) \
6083
+do { \
6084
+ if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) \
6085
+ __raw_spin##op(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \
6086
+ else if (TYPE_EQUAL(lock, spinlock_t)) \
6087
+ __raw_spin##op(&((spinlock_t *)(lock))->raw_lock); \
6088
+} while (0)
6089
+
6090
+#define PICK_SPINLOCK_IRQ(lock) \
6091
+do { \
6092
+ if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \
6093
+ __ipipe_spin_lock_irq(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \
6094
+ } else if (TYPE_EQUAL(lock, spinlock_t)) \
6095
+ _spin_lock_irq((spinlock_t *)(lock)); \
6096
+} while (0)
6097
+
6098
+#define PICK_SPINUNLOCK_IRQ(lock) \
6099
+do { \
6100
+ if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \
6101
+ __ipipe_spin_unlock_irq(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \
6102
+ } else if (TYPE_EQUAL(lock, spinlock_t)) \
6103
+ _spin_unlock_irq((spinlock_t *)(lock)); \
6104
+} while (0)
6105
+
6106
+#define PICK_SPINLOCK_IRQ_RAW(lock) \
6107
+do { \
6108
+ if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \
6109
+ __ipipe_spin_lock_irq(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \
6110
+ } else if (TYPE_EQUAL(lock, spinlock_t)) \
6111
+ local_irq_disable(); \
6112
+ __raw_spin_lock(&((spinlock_t *)(lock))->raw_lock); \
6113
+} while (0)
6114
+
6115
+#define PICK_SPINUNLOCK_IRQ_RAW(lock) \
6116
+do { \
6117
+ if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \
6118
+ __ipipe_spin_unlock_irq(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \
6119
+ } else if (TYPE_EQUAL(lock, spinlock_t)) \
6120
+ __raw_spin_unlock(&((spinlock_t *)(lock))->raw_lock); \
6121
+ local_irq_enable(); \
6122
+} while (0)
6123
+
6124
+#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
6125
+extern int __bad_spinlock_type(void);
6126
+
6127
+#define PICK_SPINLOCK_IRQSAVE(lock, flags) \
6128
+do { \
6129
+ if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \
6130
+ (flags) = __ipipe_spin_lock_irqsave(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \
6131
+ } else if (TYPE_EQUAL(lock, spinlock_t)) \
6132
+ flags = _spin_lock_irqsave((spinlock_t *)(lock)); \
6133
+ else __bad_spinlock_type(); \
6134
+} while (0)
6135
+#define PICK_SPINLOCK_IRQSAVE_NESTED(lock, flags, subclass) \
6136
+do { \
6137
+ if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \
6138
+ (flags) = __ipipe_spin_lock_irqsave(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \
6139
+ } else if (TYPE_EQUAL(lock, spinlock_t)) \
6140
+ flags = _spin_lock_irqsave_nested((spinlock_t *)(lock), subclass); \
6141
+ else __bad_spinlock_type(); \
6142
+} while (0)
6143
+#else
6144
+#define PICK_SPINLOCK_IRQSAVE(lock, flags) \
6145
+do { \
6146
+ if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \
6147
+ (flags) = __ipipe_spin_lock_irqsave(&((__ipipe_spinlock_t *)(lock))->__raw_lock); \
6148
+ } else if (TYPE_EQUAL(lock, spinlock_t)) \
6149
+ _spin_lock_irqsave((spinlock_t *)(lock), flags); \
6150
+} while (0)
6151
+#endif
6152
+
6153
+#define PICK_SPINUNLOCK_IRQRESTORE(lock, flags) \
6154
+ do { \
6155
+ if (TYPE_EQUAL((lock), __ipipe_spinlock_t)) { \
6156
+ __ipipe_spin_unlock_irqrestore(&((__ipipe_spinlock_t *)(lock))->__raw_lock, flags); \
6157
+ } else if (TYPE_EQUAL(lock, spinlock_t)) \
6158
+ _spin_unlock_irqrestore((spinlock_t *)(lock), flags); \
6159
+} while (0)
6160
+
6161
+#define spin_lock(lock) PICK_SPINOP(_lock, lock)
6162
6163
#ifdef CONFIG_DEBUG_LOCK_ALLOC
6164
# define spin_lock_nested(lock, subclass) _spin_lock_nested(lock, subclass)
6165
@@ -201,7 +303,7 @@ do { \
6166
#define spin_lock_irqsave(lock, flags) \
6167
do { \
6168
typecheck(unsigned long, flags); \
6169
- flags = _spin_lock_irqsave(lock); \
6170
+ PICK_SPINLOCK_IRQSAVE(lock, flags); \
6171
} while (0)
6172
#define read_lock_irqsave(lock, flags) \
6173
do { \
6174
@@ -218,13 +320,13 @@ do { \
6175
#define spin_lock_irqsave_nested(lock, flags, subclass) \
6176
do { \
6177
typecheck(unsigned long, flags); \
6178
- flags = _spin_lock_irqsave_nested(lock, subclass); \
6179
+ PICK_SPINLOCK_IRQSAVE_NESTED(lock, flags, subclass); \
6180
} while (0)
6181
#else
6182
#define spin_lock_irqsave_nested(lock, flags, subclass) \
6183
do { \
6184
typecheck(unsigned long, flags); \
6185
- flags = _spin_lock_irqsave(lock); \
6186
+ PICK_SPINLOCK_IRQSAVE(lock, flags); \
6187
} while (0)
6188
#endif
6189
6190
@@ -233,7 +335,7 @@ do { \
6191
#define spin_lock_irqsave(lock, flags) \
6192
do { \
6193
typecheck(unsigned long, flags); \
6194
- _spin_lock_irqsave(lock, flags); \
6195
+ PICK_SPINLOCK_IRQSAVE(lock, flags); \
6196
} while (0)
6197
#define read_lock_irqsave(lock, flags) \
6198
do { \
6199
@@ -250,7 +352,7 @@ do { \
6200
6201
#endif
6202
6203
-#define spin_lock_irq(lock) _spin_lock_irq(lock)
6204
+#define spin_lock_irq(lock) PICK_SPINLOCK_IRQ(lock)
6205
#define spin_lock_bh(lock) _spin_lock_bh(lock)
6206
6207
#define read_lock_irq(lock) _read_lock_irq(lock)
6208
@@ -264,32 +366,40 @@ do { \
6209
*/
6210
#if defined(CONFIG_DEBUG_SPINLOCK) || defined(CONFIG_PREEMPT) || \
6211
!defined(CONFIG_SMP)
6212
-# define spin_unlock(lock) _spin_unlock(lock)
6213
+#define spin_unlock(lock) PICK_SPINOP(_unlock, lock)
6214
# define read_unlock(lock) _read_unlock(lock)
6215
# define write_unlock(lock) _write_unlock(lock)
6216
-# define spin_unlock_irq(lock) _spin_unlock_irq(lock)
6217
-# define read_unlock_irq(lock) _read_unlock_irq(lock)
6218
-# define write_unlock_irq(lock) _write_unlock_irq(lock)
6219
+# define spin_unlock_irq(lock) PICK_SPINUNLOCK_IRQ(lock)
6220
+# define read_unlock_irq(lock) _read_unlock_irq(lock)
6221
+# define write_unlock_irq(lock) _write_unlock_irq(lock)
6222
#else
6223
-# define spin_unlock(lock) \
6224
- do {__raw_spin_unlock(&(lock)->raw_lock); __release(lock); } while (0)
6225
-# define read_unlock(lock) \
6226
- do {__raw_read_unlock(&(lock)->raw_lock); __release(lock); } while (0)
6227
-# define write_unlock(lock) \
6228
- do {__raw_write_unlock(&(lock)->raw_lock); __release(lock); } while (0)
6229
-# define spin_unlock_irq(lock) \
6230
+# define spin_unlock(lock) \
6231
do { \
6232
- __raw_spin_unlock(&(lock)->raw_lock); \
6233
+ PICK_SPINOP_RAW(_unlock, lock); \
6234
+ __release(lock); \
6235
+} while(0)
6236
+# define read_unlock(lock) \
6237
+do { \
6238
+ __raw_read_unlock(&(lock)->raw_lock); \
6239
__release(lock); \
6240
- local_irq_enable(); \
6241
} while (0)
6242
-# define read_unlock_irq(lock) \
6243
+# define write_unlock(lock) \
6244
+do { \
6245
+ __raw_write_unlock(&(lock)->raw_lock); \
6246
+ __release(lock); \
6247
+} while (0)
6248
+# define spin_unlock_irq(lock) \
6249
+do { \
6250
+ PICK_SPINUNLOCK_IRQ_RAW(lock); \
6251
+ __release(lock); \
6252
+} while(0)
6253
+# define read_unlock_irq(lock) \
6254
do { \
6255
__raw_read_unlock(&(lock)->raw_lock); \
6256
__release(lock); \
6257
local_irq_enable(); \
6258
} while (0)
6259
-# define write_unlock_irq(lock) \
6260
+# define write_unlock_irq(lock) \
6261
do { \
6262
__raw_write_unlock(&(lock)->raw_lock); \
6263
__release(lock); \
6264
@@ -300,7 +410,7 @@ do { \
6265
#define spin_unlock_irqrestore(lock, flags) \
6266
do { \
6267
typecheck(unsigned long, flags); \
6268
- _spin_unlock_irqrestore(lock, flags); \
6269
+ PICK_SPINUNLOCK_IRQRESTORE(lock, flags); \
6270
} while (0)
6271
#define spin_unlock_bh(lock) _spin_unlock_bh(lock)
6272
6273
@@ -364,4 +474,29 @@ extern int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock);
6274
*/
6275
#define spin_can_lock(lock) (!spin_is_locked(lock))
6276
6277
+#ifdef CONFIG_IPIPE
6278
+void __ipipe_spin_lock_irq(raw_spinlock_t *lock);
6279
+void __ipipe_spin_unlock_irq(raw_spinlock_t *lock);
6280
+unsigned long __ipipe_spin_lock_irqsave(raw_spinlock_t *lock);
6281
+void __ipipe_spin_unlock_irqrestore(raw_spinlock_t *lock,
6282
+ unsigned long x);
6283
+void __ipipe_spin_unlock_irqbegin(ipipe_spinlock_t *lock);
6284
+void __ipipe_spin_unlock_irqcomplete(unsigned long x);
6285
+#define spin_lock_irqsave_cond(lock, flags) \
6286
+ spin_lock_irqsave(lock, flags)
6287
+#define spin_unlock_irqrestore_cond(lock, flags) \
6288
+ spin_unlock_irqrestore(lock, flags)
6289
+#else
6290
+#define spin_lock_irqsave_cond(lock, flags) \
6291
+ do { (void)(flags); spin_lock(lock); } while(0)
6292
+#define spin_unlock_irqrestore_cond(lock, flags) \
6293
+ spin_unlock(lock)
6294
+#define __ipipe_spin_lock_irq(lock) do { } while(0)
6295
+#define __ipipe_spin_unlock_irq(lock) do { } while(0)
6296
+#define __ipipe_spin_lock_irqsave(lock) 0
6297
+#define __ipipe_spin_unlock_irqrestore(lock, x) do { (void)(x); } while(0)
6298
+#define __ipipe_spin_unlock_irqbegin(lock) do { } while(0)
6299
+#define __ipipe_spin_unlock_irqcomplete(x) do { (void)(x); } while(0)
6300
+#endif
6301
+
6302
#endif /* __LINUX_SPINLOCK_H */
6303
diff --git a/include/linux/spinlock_types.h b/include/linux/spinlock_types.h
6304
index 68d88f7..f5ce3c4 100644
6305
--- a/include/linux/spinlock_types.h
6306
+++ b/include/linux/spinlock_types.h
6307
@@ -31,6 +31,10 @@ typedef struct {
6308
#endif
6309
} spinlock_t;
6310
6311
+typedef struct {
6312
+ raw_spinlock_t __raw_lock;
6313
+} __ipipe_spinlock_t;
6314
+
6315
#define SPINLOCK_MAGIC 0xdead4ead
6316
6317
typedef struct {
6318
@@ -92,9 +96,21 @@ typedef struct {
6319
* __SPIN_LOCK_UNLOCKED()/__RW_LOCK_UNLOCKED() as appropriate.
6320
*/
6321
#define SPIN_LOCK_UNLOCKED __SPIN_LOCK_UNLOCKED(old_style_spin_init)
6322
+#define IPIPE_SPIN_LOCK_UNLOCKED \
6323
+ (__ipipe_spinlock_t) { .__raw_lock = __RAW_SPIN_LOCK_UNLOCKED }
6324
#define RW_LOCK_UNLOCKED __RW_LOCK_UNLOCKED(old_style_rw_init)
6325
6326
#define DEFINE_SPINLOCK(x) spinlock_t x = __SPIN_LOCK_UNLOCKED(x)
6327
#define DEFINE_RWLOCK(x) rwlock_t x = __RW_LOCK_UNLOCKED(x)
6328
6329
+#ifdef CONFIG_IPIPE
6330
+# define ipipe_spinlock_t __ipipe_spinlock_t
6331
+# define IPIPE_DEFINE_SPINLOCK(x) ipipe_spinlock_t x = IPIPE_SPIN_LOCK_UNLOCKED
6332
+# define IPIPE_DECLARE_SPINLOCK(x) extern ipipe_spinlock_t x
6333
+#else
6334
+# define ipipe_spinlock_t spinlock_t
6335
+# define IPIPE_DEFINE_SPINLOCK(x) DEFINE_SPINLOCK(x)
6336
+# define IPIPE_DECLARE_SPINLOCK(x) extern spinlock_t x
6337
+#endif
6338
+
6339
#endif /* __LINUX_SPINLOCK_TYPES_H */
6340
diff --git a/init/Kconfig b/init/Kconfig
6341
index 83b6905..27ccba5 100644
6342
--- a/init/Kconfig
6343
+++ b/init/Kconfig
6344
@@ -73,6 +73,7 @@ config INIT_ENV_ARG_LIMIT
6345
6346
config LOCALVERSION
6347
string "Local version - append to kernel release"
6348
+ default "-ipipe"
6349
help
6350
Append an extra string to the end of your kernel version.
6351
This will show up when you type uname, for example.
6352
diff --git a/init/main.c b/init/main.c
6353
index 7e117a2..43474f4 100644
6354
--- a/init/main.c
6355
+++ b/init/main.c
6356
@@ -555,7 +555,7 @@ asmlinkage void __init start_kernel(void)
6357
debug_objects_early_init();
6358
cgroup_init_early();
6359
6360
- local_irq_disable();
6361
+ local_irq_disable_hw();
6362
early_boot_irqs_off();
6363
early_init_irq_lock_class();
6364
6365
@@ -611,6 +611,11 @@ asmlinkage void __init start_kernel(void)
6366
timekeeping_init();
6367
time_init();
6368
sched_clock_init();
6369
+ /*
6370
+ * We need to wait for the interrupt and time subsystems to be
6371
+ * initialized before enabling the pipeline.
6372
+ */
6373
+ ipipe_init();
6374
profile_init();
6375
if (!irqs_disabled())
6376
printk("start_kernel(): bug: interrupts were enabled early\n");
6377
@@ -772,6 +777,7 @@ static void __init do_basic_setup(void)
6378
usermodehelper_init();
6379
driver_init();
6380
init_irq_proc();
6381
+ ipipe_init_proc();
6382
do_initcalls();
6383
}
6384
6385
diff --git a/kernel/Makefile b/kernel/Makefile
6386
index 19fad00..e36e4e3 100644
6387
--- a/kernel/Makefile
6388
+++ b/kernel/Makefile
6389
@@ -79,6 +79,7 @@ ifeq ($(CONFIG_PREEMPT_RCU),y)
6390
obj-$(CONFIG_RCU_TRACE) += rcupreempt_trace.o
6391
endif
6392
obj-$(CONFIG_RELAY) += relay.o
6393
+obj-$(CONFIG_IPIPE) += ipipe/
6394
obj-$(CONFIG_SYSCTL) += utsname_sysctl.o
6395
obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o
6396
obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o
6397
diff --git a/kernel/exit.c b/kernel/exit.c
6398
index 10e393b..c74548a 100644
6399
--- a/kernel/exit.c
6400
+++ b/kernel/exit.c
6401
@@ -1072,6 +1072,7 @@ NORET_TYPE void do_exit(long code)
6402
acct_process();
6403
trace_sched_process_exit(tsk);
6404
6405
+ ipipe_exit_notify(tsk);
6406
exit_sem(tsk);
6407
exit_files(tsk);
6408
exit_fs(tsk);
6409
diff --git a/kernel/fork.c b/kernel/fork.c
6410
index 8f753e5..d91e8d8 100644
6411
--- a/kernel/fork.c
6412
+++ b/kernel/fork.c
6413
@@ -468,6 +468,7 @@ void mmput(struct mm_struct *mm)
6414
if (atomic_dec_and_test(&mm->mm_users)) {
6415
exit_aio(mm);
6416
exit_mmap(mm);
6417
+ ipipe_cleanup_notify(mm);
6418
set_mm_exe_file(mm, NULL);
6419
if (!list_empty(&mm->mmlist)) {
6420
spin_lock(&mmlist_lock);
6421
@@ -887,7 +888,7 @@ static void copy_flags(unsigned long clone_flags, struct task_struct *p)
6422
{
6423
unsigned long new_flags = p->flags;
6424
6425
- new_flags &= ~PF_SUPERPRIV;
6426
+ new_flags &= ~(PF_SUPERPRIV | PF_EVNOTIFY);
6427
new_flags |= PF_FORKNOEXEC;
6428
new_flags |= PF_STARTING;
6429
p->flags = new_flags;
6430
@@ -1271,6 +1272,9 @@ static struct task_struct *copy_process(unsigned long clone_flags,
6431
write_unlock_irq(&tasklist_lock);
6432
proc_fork_connector(p);
6433
cgroup_post_fork(p);
6434
+#ifdef CONFIG_IPIPE
6435
+ memset(p->ptd, 0, sizeof(p->ptd));
6436
+#endif /* CONFIG_IPIPE */
6437
return p;
6438
6439
bad_fork_free_pid:
6440
diff --git a/kernel/ipipe/Kconfig b/kernel/ipipe/Kconfig
6441
new file mode 100644
6442
index 0000000..de5e6a3
6443
--- /dev/null
6444
+++ b/kernel/ipipe/Kconfig
6445
@@ -0,0 +1,35 @@
6446
+config IPIPE
6447
+ bool "Interrupt pipeline"
6448
+ default y
6449
+ ---help---
6450
+ Activate this option if you want the interrupt pipeline to be
6451
+ compiled in.
6452
+
6453
+config IPIPE_DOMAINS
6454
+ int "Max domains"
6455
+ depends on IPIPE
6456
+ default 4
6457
+ ---help---
6458
+ The maximum number of I-pipe domains to run concurrently.
6459
+
6460
+config IPIPE_COMPAT
6461
+ bool "Maintain code compatibility with older releases"
6462
+ depends on IPIPE
6463
+ default y
6464
+ ---help---
6465
+ Activate this option if you want the compatibility code to be
6466
+ defined, so that older I-pipe clients may use obsolete
6467
+ constructs. WARNING: obsolete code will be eventually
6468
+ deprecated in future I-pipe releases, and removed from the
6469
+ compatibility support as time passes. Please fix I-pipe
6470
+ clients to get rid of such uses as soon as possible.
6471
+
6472
+config IPIPE_DELAYED_ATOMICSW
6473
+ bool
6474
+ depends on IPIPE
6475
+ default n
6476
+
6477
+config IPIPE_UNMASKED_CONTEXT_SWITCH
6478
+ bool
6479
+ depends on IPIPE
6480
+ default n
6481
diff --git a/kernel/ipipe/Kconfig.debug b/kernel/ipipe/Kconfig.debug
6482
new file mode 100644
6483
index 0000000..48dae0e
6484
--- /dev/null
6485
+++ b/kernel/ipipe/Kconfig.debug
6486
@@ -0,0 +1,88 @@
6487
+config IPIPE_DEBUG
6488
+ bool "I-pipe debugging"
6489
+ depends on IPIPE
6490
+
6491
+config IPIPE_DEBUG_CONTEXT
6492
+ bool "Check for illicit cross-domain calls"
6493
+ depends on IPIPE_DEBUG
6494
+ default y
6495
+ ---help---
6496
+ Enable this feature to arm checkpoints in the kernel that
6497
+ verify the correct invocation context. On entry of critical
6498
+ Linux services a warning is issued if the caller is not
6499
+ running over the root domain.
6500
+
6501
+config IPIPE_TRACE
6502
+ bool "Latency tracing"
6503
+ depends on IPIPE_DEBUG && !FUNCTION_TRACER
6504
+ select FRAME_POINTER
6505
+ select KALLSYMS
6506
+ select PROC_FS
6507
+ ---help---
6508
+ Activate this option if you want to use per-function tracing of
6509
+ the kernel. The tracer will collect data via instrumentation
6510
+ features like the one below or with the help of explicite calls
6511
+ of ipipe_trace_xxx(). See include/linux/ipipe_trace.h for the
6512
+ in-kernel tracing API. The collected data and runtime control
6513
+ is available via /proc/ipipe/trace/*.
6514
+
6515
+if IPIPE_TRACE
6516
+
6517
+config IPIPE_TRACE_ENABLE
6518
+ bool "Enable tracing on boot"
6519
+ default y
6520
+ ---help---
6521
+ Disable this option if you want to arm the tracer after booting
6522
+ manually ("echo 1 > /proc/ipipe/tracer/enable"). This can reduce
6523
+ boot time on slow embedded devices due to the tracer overhead.
6524
+
6525
+config IPIPE_TRACE_MCOUNT
6526
+ bool "Instrument function entries"
6527
+ default y
6528
+ ---help---
6529
+ When enabled, records every kernel function entry in the tracer
6530
+ log. While this slows down the system noticeably, it provides
6531
+ the highest level of information about the flow of events.
6532
+ However, it can be switch off in order to record only explicit
6533
+ I-pipe trace points.
6534
+
6535
+config IPIPE_TRACE_IRQSOFF
6536
+ bool "Trace IRQs-off times"
6537
+ default y
6538
+ ---help---
6539
+ Activate this option if I-pipe shall trace the longest path
6540
+ with hard-IRQs switched off.
6541
+
6542
+config IPIPE_TRACE_SHIFT
6543
+ int "Depth of trace log (14 => 16Kpoints, 15 => 32Kpoints)"
6544
+ range 10 18
6545
+ default 14
6546
+ ---help---
6547
+ The number of trace points to hold tracing data for each
6548
+ trace path, as a power of 2.
6549
+
6550
+config IPIPE_TRACE_VMALLOC
6551
+ bool "Use vmalloc'ed trace buffer"
6552
+ default y if EMBEDDED
6553
+ ---help---
6554
+ Instead of reserving static kernel data, the required buffer
6555
+ is allocated via vmalloc during boot-up when this option is
6556
+ enabled. This can help to start systems that are low on memory,
6557
+ but it slightly degrades overall performance. Try this option
6558
+ when a traced kernel hangs unexpectedly at boot time.
6559
+
6560
+config IPIPE_TRACE_PANIC
6561
+ bool "Enable panic back traces"
6562
+ default y
6563
+ ---help---
6564
+ Provides services to freeze and dump a back trace on panic
6565
+ situations. This is used on IPIPE_DEBUG_CONTEXT exceptions
6566
+ as well as ordinary kernel oopses. You can control the number
6567
+ of printed back trace points via /proc/ipipe/trace.
6568
+
6569
+config IPIPE_TRACE_ENABLE_VALUE
6570
+ int
6571
+ default 0 if !IPIPE_TRACE_ENABLE
6572
+ default 1 if IPIPE_TRACE_ENABLE
6573
+
6574
+endif
6575
diff --git a/kernel/ipipe/Makefile b/kernel/ipipe/Makefile
6576
new file mode 100644
6577
index 0000000..6257dfa
6578
--- /dev/null
6579
+++ b/kernel/ipipe/Makefile
6580
@@ -0,0 +1,3 @@
6581
+
6582
+obj-$(CONFIG_IPIPE) += core.o
6583
+obj-$(CONFIG_IPIPE_TRACE) += tracer.o
6584
diff --git a/kernel/ipipe/core.c b/kernel/ipipe/core.c
6585
new file mode 100644
6586
index 0000000..97e6aae
6587
--- /dev/null
6588
+++ b/kernel/ipipe/core.c
6589
@@ -0,0 +1,1713 @@
6590
+/* -*- linux-c -*-
6591
+ * linux/kernel/ipipe/core.c
6592
+ *
6593
+ * Copyright (C) 2002-2005 Philippe Gerum.
6594
+ *
6595
+ * This program is free software; you can redistribute it and/or modify
6596
+ * it under the terms of the GNU General Public License as published by
6597
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
6598
+ * USA; either version 2 of the License, or (at your option) any later
6599
+ * version.
6600
+ *
6601
+ * This program is distributed in the hope that it will be useful,
6602
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
6603
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
6604
+ * GNU General Public License for more details.
6605
+ *
6606
+ * You should have received a copy of the GNU General Public License
6607
+ * along with this program; if not, write to the Free Software
6608
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
6609
+ *
6610
+ * Architecture-independent I-PIPE core support.
6611
+ */
6612
+
6613
+#include <linux/version.h>
6614
+#include <linux/module.h>
6615
+#include <linux/init.h>
6616
+#include <linux/kernel.h>
6617
+#include <linux/sched.h>
6618
+#include <linux/sched.h>
6619
+#include <linux/kallsyms.h>
6620
+#include <linux/interrupt.h>
6621
+#include <linux/bitops.h>
6622
+#include <linux/tick.h>
6623
+#include <linux/prefetch.h>
6624
+#ifdef CONFIG_PROC_FS
6625
+#include <linux/proc_fs.h>
6626
+#include <linux/seq_file.h>
6627
+#endif /* CONFIG_PROC_FS */
6628
+#include <linux/ipipe_trace.h>
6629
+#include <linux/ipipe_tickdev.h>
6630
+#include <linux/irq.h>
6631
+
6632
+static int __ipipe_ptd_key_count;
6633
+
6634
+static unsigned long __ipipe_ptd_key_map;
6635
+
6636
+static unsigned long __ipipe_domain_slot_map;
6637
+
6638
+struct ipipe_domain ipipe_root;
6639
+
6640
+#ifndef CONFIG_SMP
6641
+/*
6642
+ * Create an alias to the unique root status, so that arch-dep code
6643
+ * may get simple and easy access to this percpu variable. We also
6644
+ * create an array of pointers to the percpu domain data; this tends
6645
+ * to produce a better code when reaching non-root domains. We make
6646
+ * sure that the early boot code would be able to dereference the
6647
+ * pointer to the root domain data safely by statically initializing
6648
+ * its value (local_irq*() routines depend on this).
6649
+ */
6650
+#if __GNUC__ >= 4
6651
+extern unsigned long __ipipe_root_status
6652
+__attribute__((alias(__stringify(__raw_get_cpu_var(ipipe_percpu_darray)))));
6653
+EXPORT_SYMBOL(__ipipe_root_status);
6654
+#else /* __GNUC__ < 4 */
6655
+/*
6656
+ * Work around a GCC 3.x issue making alias symbols unusable as
6657
+ * constant initializers.
6658
+ */
6659
+unsigned long *const __ipipe_root_status_addr =
6660
+ &__raw_get_cpu_var(ipipe_percpu_darray)[IPIPE_ROOT_SLOT].status;
6661
+EXPORT_SYMBOL(__ipipe_root_status_addr);
6662
+#endif /* __GNUC__ < 4 */
6663
+
6664
+DEFINE_PER_CPU(struct ipipe_percpu_domain_data *, ipipe_percpu_daddr[CONFIG_IPIPE_DOMAINS]) =
6665
+{ [IPIPE_ROOT_SLOT] = (struct ipipe_percpu_domain_data *)&__raw_get_cpu_var(ipipe_percpu_darray) };
6666
+EXPORT_PER_CPU_SYMBOL(ipipe_percpu_daddr);
6667
+#endif /* !CONFIG_SMP */
6668
+
6669
+DEFINE_PER_CPU(struct ipipe_percpu_domain_data, ipipe_percpu_darray[CONFIG_IPIPE_DOMAINS]) =
6670
+{ [IPIPE_ROOT_SLOT] = { .status = IPIPE_STALL_MASK } }; /* Root domain stalled on each CPU at startup. */
6671
+
6672
+DEFINE_PER_CPU(struct ipipe_domain *, ipipe_percpu_domain) = { &ipipe_root };
6673
+
6674
+DEFINE_PER_CPU(unsigned long, ipipe_nmi_saved_root); /* Copy of root status during NMI */
6675
+
6676
+static IPIPE_DEFINE_SPINLOCK(__ipipe_pipelock);
6677
+
6678
+LIST_HEAD(__ipipe_pipeline);
6679
+
6680
+unsigned long __ipipe_virtual_irq_map;
6681
+
6682
+#ifdef CONFIG_PRINTK
6683
+unsigned __ipipe_printk_virq;
6684
+#endif /* CONFIG_PRINTK */
6685
+
6686
+int __ipipe_event_monitors[IPIPE_NR_EVENTS];
6687
+
6688
+#ifdef CONFIG_GENERIC_CLOCKEVENTS
6689
+
6690
+DECLARE_PER_CPU(struct tick_device, tick_cpu_device);
6691
+
6692
+static DEFINE_PER_CPU(struct ipipe_tick_device, ipipe_tick_cpu_device);
6693
+
6694
+int ipipe_request_tickdev(const char *devname,
6695
+ void (*emumode)(enum clock_event_mode mode,
6696
+ struct clock_event_device *cdev),
6697
+ int (*emutick)(unsigned long delta,
6698
+ struct clock_event_device *cdev),
6699
+ int cpu, unsigned long *tmfreq)
6700
+{
6701
+ struct ipipe_tick_device *itd;
6702
+ struct tick_device *slave;
6703
+ struct clock_event_device *evtdev;
6704
+ unsigned long long freq;
6705
+ unsigned long flags;
6706
+ int status;
6707
+
6708
+ flags = ipipe_critical_enter(NULL);
6709
+
6710
+ itd = &per_cpu(ipipe_tick_cpu_device, cpu);
6711
+
6712
+ if (itd->slave != NULL) {
6713
+ status = -EBUSY;
6714
+ goto out;
6715
+ }
6716
+
6717
+ slave = &per_cpu(tick_cpu_device, cpu);
6718
+
6719
+ if (strcmp(slave->evtdev->name, devname)) {
6720
+ /*
6721
+ * No conflict so far with the current tick device,
6722
+ * check whether the requested device is sane and has
6723
+ * been blessed by the kernel.
6724
+ */
6725
+ status = __ipipe_check_tickdev(devname) ?
6726
+ CLOCK_EVT_MODE_UNUSED : CLOCK_EVT_MODE_SHUTDOWN;
6727
+ goto out;
6728
+ }
6729
+
6730
+ /*
6731
+ * Our caller asks for using the same clock event device for
6732
+ * ticking than we do, let's create a tick emulation device to
6733
+ * interpose on the set_next_event() method, so that we may
6734
+ * both manage the device in oneshot mode. Only the tick
6735
+ * emulation code will actually program the clockchip hardware
6736
+ * for the next shot, though.
6737
+ *
6738
+ * CAUTION: we still have to grab the tick device even when it
6739
+ * current runs in periodic mode, since the kernel may switch
6740
+ * to oneshot dynamically (highres/no_hz tick mode).
6741
+ */
6742
+
6743
+ evtdev = slave->evtdev;
6744
+ status = evtdev->mode;
6745
+
6746
+ if (status == CLOCK_EVT_MODE_SHUTDOWN)
6747
+ goto out;
6748
+
6749
+ itd->slave = slave;
6750
+ itd->emul_set_mode = emumode;
6751
+ itd->emul_set_tick = emutick;
6752
+ itd->real_set_mode = evtdev->set_mode;
6753
+ itd->real_set_tick = evtdev->set_next_event;
6754
+ itd->real_max_delta_ns = evtdev->max_delta_ns;
6755
+ itd->real_mult = evtdev->mult;
6756
+ itd->real_shift = evtdev->shift;
6757
+ freq = (1000000000ULL * evtdev->mult) >> evtdev->shift;
6758
+ *tmfreq = (unsigned long)freq;
6759
+ evtdev->set_mode = emumode;
6760
+ evtdev->set_next_event = emutick;
6761
+ evtdev->max_delta_ns = ULONG_MAX;
6762
+ evtdev->mult = 1;
6763
+ evtdev->shift = 0;
6764
+out:
6765
+ ipipe_critical_exit(flags);
6766
+
6767
+ return status;
6768
+}
6769
+
6770
+void ipipe_release_tickdev(int cpu)
6771
+{
6772
+ struct ipipe_tick_device *itd;
6773
+ struct tick_device *slave;
6774
+ struct clock_event_device *evtdev;
6775
+ unsigned long flags;
6776
+
6777
+ flags = ipipe_critical_enter(NULL);
6778
+
6779
+ itd = &per_cpu(ipipe_tick_cpu_device, cpu);
6780
+
6781
+ if (itd->slave != NULL) {
6782
+ slave = &per_cpu(tick_cpu_device, cpu);
6783
+ evtdev = slave->evtdev;
6784
+ evtdev->set_mode = itd->real_set_mode;
6785
+ evtdev->set_next_event = itd->real_set_tick;
6786
+ evtdev->max_delta_ns = itd->real_max_delta_ns;
6787
+ evtdev->mult = itd->real_mult;
6788
+ evtdev->shift = itd->real_shift;
6789
+ itd->slave = NULL;
6790
+ }
6791
+
6792
+ ipipe_critical_exit(flags);
6793
+}
6794
+
6795
+#endif /* CONFIG_GENERIC_CLOCKEVENTS */
6796
+
6797
+/*
6798
+ * ipipe_init() -- Initialization routine of the IPIPE layer. Called
6799
+ * by the host kernel early during the boot procedure.
6800
+ */
6801
+void __init ipipe_init(void)
6802
+{
6803
+ struct ipipe_domain *ipd = &ipipe_root;
6804
+
6805
+ __ipipe_check_platform(); /* Do platform dependent checks first. */
6806
+
6807
+ /*
6808
+ * A lightweight registration code for the root domain. We are
6809
+ * running on the boot CPU, hw interrupts are off, and
6810
+ * secondary CPUs are still lost in space.
6811
+ */
6812
+
6813
+ /* Reserve percpu data slot #0 for the root domain. */
6814
+ ipd->slot = 0;
6815
+ set_bit(0, &__ipipe_domain_slot_map);
6816
+
6817
+ ipd->name = "Linux";
6818
+ ipd->domid = IPIPE_ROOT_ID;
6819
+ ipd->priority = IPIPE_ROOT_PRIO;
6820
+
6821
+ __ipipe_init_stage(ipd);
6822
+
6823
+ INIT_LIST_HEAD(&ipd->p_link);
6824
+ list_add_tail(&ipd->p_link, &__ipipe_pipeline);
6825
+
6826
+ __ipipe_init_platform();
6827
+
6828
+#ifdef CONFIG_PRINTK
6829
+ __ipipe_printk_virq = ipipe_alloc_virq(); /* Cannot fail here. */
6830
+ ipd->irqs[__ipipe_printk_virq].handler = &__ipipe_flush_printk;
6831
+ ipd->irqs[__ipipe_printk_virq].cookie = NULL;
6832
+ ipd->irqs[__ipipe_printk_virq].acknowledge = NULL;
6833
+ ipd->irqs[__ipipe_printk_virq].control = IPIPE_HANDLE_MASK;
6834
+#endif /* CONFIG_PRINTK */
6835
+
6836
+ __ipipe_enable_pipeline();
6837
+
6838
+ printk(KERN_INFO "I-pipe %s: pipeline enabled.\n",
6839
+ IPIPE_VERSION_STRING);
6840
+}
6841
+
6842
+void __ipipe_init_stage(struct ipipe_domain *ipd)
6843
+{
6844
+ int cpu, n;
6845
+
6846
+ for_each_online_cpu(cpu) {
6847
+
6848
+ ipipe_percpudom(ipd, irqpend_himask, cpu) = 0;
6849
+
6850
+ for (n = 0; n < IPIPE_IRQ_IWORDS; n++) {
6851
+ ipipe_percpudom(ipd, irqpend_lomask, cpu)[n] = 0;
6852
+ ipipe_percpudom(ipd, irqheld_mask, cpu)[n] = 0;
6853
+ }
6854
+
6855
+ for (n = 0; n < IPIPE_NR_IRQS; n++)
6856
+ ipipe_percpudom(ipd, irqall, cpu)[n] = 0;
6857
+
6858
+ ipipe_percpudom(ipd, evsync, cpu) = 0;
6859
+ }
6860
+
6861
+ for (n = 0; n < IPIPE_NR_IRQS; n++) {
6862
+ ipd->irqs[n].acknowledge = NULL;
6863
+ ipd->irqs[n].handler = NULL;
6864
+ ipd->irqs[n].control = IPIPE_PASS_MASK; /* Pass but don't handle */
6865
+ }
6866
+
6867
+ for (n = 0; n < IPIPE_NR_EVENTS; n++)
6868
+ ipd->evhand[n] = NULL;
6869
+
6870
+ ipd->evself = 0LL;
6871
+ mutex_init(&ipd->mutex);
6872
+
6873
+ __ipipe_hook_critical_ipi(ipd);
6874
+}
6875
+
6876
+void __ipipe_cleanup_domain(struct ipipe_domain *ipd)
6877
+{
6878
+ ipipe_unstall_pipeline_from(ipd);
6879
+
6880
+#ifdef CONFIG_SMP
6881
+ {
6882
+ int cpu;
6883
+
6884
+ for_each_online_cpu(cpu) {
6885
+ while (ipipe_percpudom(ipd, irqpend_himask, cpu) != 0)
6886
+ cpu_relax();
6887
+ }
6888
+ }
6889
+#else
6890
+ __raw_get_cpu_var(ipipe_percpu_daddr)[ipd->slot] = NULL;
6891
+#endif
6892
+
6893
+ clear_bit(ipd->slot, &__ipipe_domain_slot_map);
6894
+}
6895
+
6896
+void __ipipe_unstall_root(void)
6897
+{
6898
+ struct ipipe_percpu_domain_data *p = ipipe_root_cpudom_ptr();
6899
+
6900
+#ifndef CONFIG_IPIPE_DEBUG_CONTEXT
6901
+ BUG_ON(!ipipe_root_domain_p);
6902
+#endif
6903
+
6904
+ local_irq_disable_hw();
6905
+
6906
+ __clear_bit(IPIPE_STALL_FLAG, &p->status);
6907
+
6908
+ if (unlikely(p->irqpend_himask != 0))
6909
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
6910
+
6911
+ local_irq_enable_hw();
6912
+}
6913
+
6914
+void __ipipe_restore_root(unsigned long x)
6915
+{
6916
+#ifndef CONFIG_IPIPE_DEBUG_CONTEXT
6917
+ BUG_ON(!ipipe_root_domain_p);
6918
+#endif
6919
+
6920
+ if (x)
6921
+ __ipipe_stall_root();
6922
+ else
6923
+ __ipipe_unstall_root();
6924
+}
6925
+
6926
+void ipipe_stall_pipeline_from(struct ipipe_domain *ipd)
6927
+{
6928
+ set_bit_safe(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status));
6929
+
6930
+ if (__ipipe_pipeline_head_p(ipd))
6931
+ local_irq_disable_hw();
6932
+}
6933
+
6934
+unsigned long ipipe_test_and_stall_pipeline_from(struct ipipe_domain *ipd)
6935
+{
6936
+ unsigned long x;
6937
+
6938
+ x = test_and_set_bit_safe(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status));
6939
+
6940
+ if (__ipipe_pipeline_head_p(ipd))
6941
+ local_irq_disable_hw();
6942
+
6943
+ return x;
6944
+}
6945
+
6946
+/*
6947
+ * ipipe_unstall_pipeline_from() -- Unstall the pipeline and
6948
+ * synchronize pending interrupts for a given domain. See
6949
+ * __ipipe_walk_pipeline() for more information.
6950
+ */
6951
+void ipipe_unstall_pipeline_from(struct ipipe_domain *ipd)
6952
+{
6953
+ struct list_head *pos;
6954
+ unsigned long flags;
6955
+
6956
+ local_irq_save_hw(flags);
6957
+
6958
+ __clear_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status));
6959
+
6960
+ if (ipd == ipipe_current_domain)
6961
+ pos = &ipd->p_link;
6962
+ else
6963
+ pos = __ipipe_pipeline.next;
6964
+
6965
+ __ipipe_walk_pipeline(pos);
6966
+
6967
+ if (likely(__ipipe_pipeline_head_p(ipd)))
6968
+ local_irq_enable_hw();
6969
+ else
6970
+ local_irq_restore_hw(flags);
6971
+}
6972
+
6973
+unsigned long ipipe_test_and_unstall_pipeline_from(struct ipipe_domain *ipd)
6974
+{
6975
+ unsigned long x;
6976
+
6977
+ x = test_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status));
6978
+ ipipe_unstall_pipeline_from(ipd);
6979
+
6980
+ return x;
6981
+}
6982
+
6983
+void ipipe_restore_pipeline_from(struct ipipe_domain *ipd,
6984
+ unsigned long x)
6985
+{
6986
+ if (x)
6987
+ ipipe_stall_pipeline_from(ipd);
6988
+ else
6989
+ ipipe_unstall_pipeline_from(ipd);
6990
+}
6991
+
6992
+void ipipe_unstall_pipeline_head(void)
6993
+{
6994
+ struct ipipe_percpu_domain_data *p = ipipe_head_cpudom_ptr();
6995
+
6996
+ local_irq_disable_hw();
6997
+
6998
+ __clear_bit(IPIPE_STALL_FLAG, &p->status);
6999
+
7000
+ if (unlikely(p->irqpend_himask != 0)) {
7001
+ struct ipipe_domain *head_domain = __ipipe_pipeline_head();
7002
+ if (likely(head_domain == ipipe_current_domain))
7003
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
7004
+ else
7005
+ __ipipe_walk_pipeline(&head_domain->p_link);
7006
+ }
7007
+
7008
+ local_irq_enable_hw();
7009
+}
7010
+
7011
+void __ipipe_restore_pipeline_head(unsigned long x)
7012
+{
7013
+ struct ipipe_percpu_domain_data *p = ipipe_head_cpudom_ptr();
7014
+
7015
+ local_irq_disable_hw();
7016
+
7017
+ if (x) {
7018
+#ifdef CONFIG_DEBUG_KERNEL
7019
+ static int warned;
7020
+ if (!warned && test_and_set_bit(IPIPE_STALL_FLAG, &p->status)) {
7021
+ /*
7022
+ * Already stalled albeit ipipe_restore_pipeline_head()
7023
+ * should have detected it? Send a warning once.
7024
+ */
7025
+ warned = 1;
7026
+ printk(KERN_WARNING
7027
+ "I-pipe: ipipe_restore_pipeline_head() optimization failed.\n");
7028
+ dump_stack();
7029
+ }
7030
+#else /* !CONFIG_DEBUG_KERNEL */
7031
+ set_bit(IPIPE_STALL_FLAG, &p->status);
7032
+#endif /* CONFIG_DEBUG_KERNEL */
7033
+ }
7034
+ else {
7035
+ __clear_bit(IPIPE_STALL_FLAG, &p->status);
7036
+ if (unlikely(p->irqpend_himask != 0)) {
7037
+ struct ipipe_domain *head_domain = __ipipe_pipeline_head();
7038
+ if (likely(head_domain == ipipe_current_domain))
7039
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
7040
+ else
7041
+ __ipipe_walk_pipeline(&head_domain->p_link);
7042
+ }
7043
+ local_irq_enable_hw();
7044
+ }
7045
+}
7046
+
7047
+void __ipipe_spin_lock_irq(raw_spinlock_t *lock)
7048
+{
7049
+ local_irq_disable_hw();
7050
+ __raw_spin_lock(lock);
7051
+ __set_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status));
7052
+}
7053
+
7054
+void __ipipe_spin_unlock_irq(raw_spinlock_t *lock)
7055
+{
7056
+ __raw_spin_unlock(lock);
7057
+ __clear_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status));
7058
+ local_irq_enable_hw();
7059
+}
7060
+
7061
+unsigned long __ipipe_spin_lock_irqsave(raw_spinlock_t *lock)
7062
+{
7063
+ unsigned long flags;
7064
+ int s;
7065
+
7066
+ local_irq_save_hw(flags);
7067
+ __raw_spin_lock(lock);
7068
+ s = __test_and_set_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status));
7069
+
7070
+ return raw_mangle_irq_bits(s, flags);
7071
+}
7072
+
7073
+void __ipipe_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long x)
7074
+{
7075
+ __raw_spin_unlock(lock);
7076
+ if (!raw_demangle_irq_bits(&x))
7077
+ __clear_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status));
7078
+ local_irq_restore_hw(x);
7079
+}
7080
+
7081
+void __ipipe_spin_unlock_irqbegin(ipipe_spinlock_t *lock)
7082
+{
7083
+ __raw_spin_unlock(&lock->__raw_lock);
7084
+}
7085
+
7086
+void __ipipe_spin_unlock_irqcomplete(unsigned long x)
7087
+{
7088
+ if (!raw_demangle_irq_bits(&x))
7089
+ __clear_bit(IPIPE_STALL_FLAG, &ipipe_this_cpudom_var(status));
7090
+ local_irq_restore_hw(x);
7091
+}
7092
+
7093
+/* Must be called hw IRQs off. */
7094
+void __ipipe_set_irq_pending(struct ipipe_domain *ipd, unsigned irq)
7095
+{
7096
+ int level = irq >> IPIPE_IRQ_ISHIFT, rank = irq & IPIPE_IRQ_IMASK;
7097
+ struct ipipe_percpu_domain_data *p = ipipe_cpudom_ptr(ipd);
7098
+
7099
+ prefetchw(p);
7100
+
7101
+ if (likely(!test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))) {
7102
+ __set_bit(rank, &p->irqpend_lomask[level]);
7103
+ __set_bit(level, &p->irqpend_himask);
7104
+ } else
7105
+ __set_bit(rank, &p->irqheld_mask[level]);
7106
+
7107
+ p->irqall[irq]++;
7108
+}
7109
+
7110
+/* Must be called hw IRQs off. */
7111
+void __ipipe_lock_irq(struct ipipe_domain *ipd, int cpu, unsigned irq)
7112
+{
7113
+ struct ipipe_percpu_domain_data *p;
7114
+ int level, rank;
7115
+
7116
+ if (unlikely(test_and_set_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control)))
7117
+ return;
7118
+
7119
+ level = irq >> IPIPE_IRQ_ISHIFT;
7120
+ rank = irq & IPIPE_IRQ_IMASK;
7121
+ p = ipipe_percpudom_ptr(ipd, cpu);
7122
+
7123
+ if (__test_and_clear_bit(rank, &p->irqpend_lomask[level]))
7124
+ __set_bit(rank, &p->irqheld_mask[level]);
7125
+ if (p->irqpend_lomask[level] == 0)
7126
+ __clear_bit(level, &p->irqpend_himask);
7127
+}
7128
+
7129
+/* Must be called hw IRQs off. */
7130
+void __ipipe_unlock_irq(struct ipipe_domain *ipd, unsigned irq)
7131
+{
7132
+ struct ipipe_percpu_domain_data *p;
7133
+ int cpu, level, rank;
7134
+
7135
+ if (unlikely(!test_and_clear_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control)))
7136
+ return;
7137
+
7138
+ level = irq >> IPIPE_IRQ_ISHIFT, rank = irq & IPIPE_IRQ_IMASK;
7139
+ for_each_online_cpu(cpu) {
7140
+ p = ipipe_percpudom_ptr(ipd, cpu);
7141
+ if (test_and_clear_bit(rank, &p->irqheld_mask[level])) {
7142
+ /* We need atomic ops here: */
7143
+ set_bit(rank, &p->irqpend_lomask[level]);
7144
+ set_bit(level, &p->irqpend_himask);
7145
+ }
7146
+ }
7147
+}
7148
+
7149
+/*
7150
+ * __ipipe_walk_pipeline(): Plays interrupts pending in the log. Must
7151
+ * be called with local hw interrupts disabled.
7152
+ */
7153
+void __ipipe_walk_pipeline(struct list_head *pos)
7154
+{
7155
+ struct ipipe_domain *this_domain = ipipe_current_domain, *next_domain;
7156
+ struct ipipe_percpu_domain_data *p, *np;
7157
+
7158
+ p = ipipe_cpudom_ptr(this_domain);
7159
+
7160
+ while (pos != &__ipipe_pipeline) {
7161
+
7162
+ next_domain = list_entry(pos, struct ipipe_domain, p_link);
7163
+ np = ipipe_cpudom_ptr(next_domain);
7164
+
7165
+ if (test_bit(IPIPE_STALL_FLAG, &np->status))
7166
+ break; /* Stalled stage -- do not go further. */
7167
+
7168
+ if (np->irqpend_himask) {
7169
+ if (next_domain == this_domain)
7170
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
7171
+ else {
7172
+
7173
+ p->evsync = 0;
7174
+ ipipe_current_domain = next_domain;
7175
+ ipipe_suspend_domain(); /* Sync stage and propagate interrupts. */
7176
+
7177
+ if (ipipe_current_domain == next_domain)
7178
+ ipipe_current_domain = this_domain;
7179
+ /*
7180
+ * Otherwise, something changed the current domain under our
7181
+ * feet recycling the register set; do not override the new
7182
+ * domain.
7183
+ */
7184
+
7185
+ if (p->irqpend_himask &&
7186
+ !test_bit(IPIPE_STALL_FLAG, &p->status))
7187
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
7188
+ }
7189
+ break;
7190
+ } else if (next_domain == this_domain)
7191
+ break;
7192
+
7193
+ pos = next_domain->p_link.next;
7194
+ }
7195
+}
7196
+
7197
+/*
7198
+ * ipipe_suspend_domain() -- Suspend the current domain, switching to
7199
+ * the next one which has pending work down the pipeline.
7200
+ */
7201
+void ipipe_suspend_domain(void)
7202
+{
7203
+ struct ipipe_domain *this_domain, *next_domain;
7204
+ struct ipipe_percpu_domain_data *p;
7205
+ struct list_head *ln;
7206
+ unsigned long flags;
7207
+
7208
+ local_irq_save_hw(flags);
7209
+
7210
+ this_domain = next_domain = ipipe_current_domain;
7211
+ p = ipipe_cpudom_ptr(this_domain);
7212
+ p->status &= ~(IPIPE_STALL_MASK|IPIPE_SYNC_MASK);
7213
+
7214
+ if (p->irqpend_himask != 0)
7215
+ goto sync_stage;
7216
+
7217
+ for (;;) {
7218
+ ln = next_domain->p_link.next;
7219
+
7220
+ if (ln == &__ipipe_pipeline)
7221
+ break;
7222
+
7223
+ next_domain = list_entry(ln, struct ipipe_domain, p_link);
7224
+ p = ipipe_cpudom_ptr(next_domain);
7225
+
7226
+ if (p->status & IPIPE_STALL_MASK)
7227
+ break;
7228
+
7229
+ if (p->irqpend_himask == 0)
7230
+ continue;
7231
+
7232
+ ipipe_current_domain = next_domain;
7233
+sync_stage:
7234
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
7235
+
7236
+ if (ipipe_current_domain != next_domain)
7237
+ /*
7238
+ * Something has changed the current domain under our
7239
+ * feet, recycling the register set; take note.
7240
+ */
7241
+ this_domain = ipipe_current_domain;
7242
+ }
7243
+
7244
+ ipipe_current_domain = this_domain;
7245
+
7246
+ local_irq_restore_hw(flags);
7247
+}
7248
+
7249
+
7250
+/* ipipe_alloc_virq() -- Allocate a pipelined virtual/soft interrupt.
7251
+ * Virtual interrupts are handled in exactly the same way than their
7252
+ * hw-generated counterparts wrt pipelining.
7253
+ */
7254
+unsigned ipipe_alloc_virq(void)
7255
+{
7256
+ unsigned long flags, irq = 0;
7257
+ int ipos;
7258
+
7259
+ spin_lock_irqsave(&__ipipe_pipelock, flags);
7260
+
7261
+ if (__ipipe_virtual_irq_map != ~0) {
7262
+ ipos = ffz(__ipipe_virtual_irq_map);
7263
+ set_bit(ipos, &__ipipe_virtual_irq_map);
7264
+ irq = ipos + IPIPE_VIRQ_BASE;
7265
+ }
7266
+
7267
+ spin_unlock_irqrestore(&__ipipe_pipelock, flags);
7268
+
7269
+ return irq;
7270
+}
7271
+
7272
+/* ipipe_virtualize_irq() -- Attach a handler (and optionally a hw
7273
+ acknowledge routine) to an interrupt for a given domain. */
7274
+
7275
+int ipipe_virtualize_irq(struct ipipe_domain *ipd,
7276
+ unsigned irq,
7277
+ ipipe_irq_handler_t handler,
7278
+ void *cookie,
7279
+ ipipe_irq_ackfn_t acknowledge,
7280
+ unsigned modemask)
7281
+{
7282
+ unsigned long flags;
7283
+ int err;
7284
+
7285
+ if (irq >= IPIPE_NR_IRQS)
7286
+ return -EINVAL;
7287
+
7288
+ if (ipd->irqs[irq].control & IPIPE_SYSTEM_MASK)
7289
+ return -EPERM;
7290
+
7291
+ if (!test_bit(IPIPE_AHEAD_FLAG, &ipd->flags))
7292
+ /* Silently unwire interrupts for non-heading domains. */
7293
+ modemask &= ~IPIPE_WIRED_MASK;
7294
+
7295
+ spin_lock_irqsave(&__ipipe_pipelock, flags);
7296
+
7297
+ if (handler != NULL) {
7298
+ if (handler == IPIPE_SAME_HANDLER) {
7299
+ handler = ipd->irqs[irq].handler;
7300
+ cookie = ipd->irqs[irq].cookie;
7301
+
7302
+ if (handler == NULL) {
7303
+ err = -EINVAL;
7304
+ goto unlock_and_exit;
7305
+ }
7306
+ } else if ((modemask & IPIPE_EXCLUSIVE_MASK) != 0 &&
7307
+ ipd->irqs[irq].handler != NULL) {
7308
+ err = -EBUSY;
7309
+ goto unlock_and_exit;
7310
+ }
7311
+
7312
+ /* Wired interrupts can only be delivered to domains
7313
+ * always heading the pipeline, and using dynamic
7314
+ * propagation. */
7315
+
7316
+ if ((modemask & IPIPE_WIRED_MASK) != 0) {
7317
+ if ((modemask & (IPIPE_PASS_MASK | IPIPE_STICKY_MASK)) != 0) {
7318
+ err = -EINVAL;
7319
+ goto unlock_and_exit;
7320
+ }
7321
+ modemask |= (IPIPE_HANDLE_MASK);
7322
+ }
7323
+
7324
+ if ((modemask & IPIPE_STICKY_MASK) != 0)
7325
+ modemask |= IPIPE_HANDLE_MASK;
7326
+ } else
7327
+ modemask &=
7328
+ ~(IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK |
7329
+ IPIPE_EXCLUSIVE_MASK | IPIPE_WIRED_MASK);
7330
+
7331
+ if (acknowledge == NULL && !ipipe_virtual_irq_p(irq))
7332
+ /* Acknowledge handler unspecified for a hw interrupt:
7333
+ use the Linux-defined handler instead. */
7334
+ acknowledge = ipipe_root_domain->irqs[irq].acknowledge;
7335
+
7336
+ ipd->irqs[irq].handler = handler;
7337
+ ipd->irqs[irq].cookie = cookie;
7338
+ ipd->irqs[irq].acknowledge = acknowledge;
7339
+ ipd->irqs[irq].control = modemask;
7340
+
7341
+ if (irq < NR_IRQS && handler != NULL && !ipipe_virtual_irq_p(irq)) {
7342
+ __ipipe_enable_irqdesc(ipd, irq);
7343
+
7344
+ if ((modemask & IPIPE_ENABLE_MASK) != 0) {
7345
+ if (ipd != ipipe_current_domain) {
7346
+ /* IRQ enable/disable state is domain-sensitive, so we may
7347
+ not change it for another domain. What is allowed
7348
+ however is forcing some domain to handle an interrupt
7349
+ source, by passing the proper 'ipd' descriptor which
7350
+ thus may be different from ipipe_current_domain. */
7351
+ err = -EPERM;
7352
+ goto unlock_and_exit;
7353
+ }
7354
+ __ipipe_enable_irq(irq);
7355
+ }
7356
+ }
7357
+
7358
+ err = 0;
7359
+
7360
+ unlock_and_exit:
7361
+
7362
+ spin_unlock_irqrestore(&__ipipe_pipelock, flags);
7363
+
7364
+ return err;
7365
+}
7366
+
7367
+/* ipipe_control_irq() -- Change modes of a pipelined interrupt for
7368
+ * the current domain. */
7369
+
7370
+int ipipe_control_irq(unsigned irq, unsigned clrmask, unsigned setmask)
7371
+{
7372
+ struct ipipe_domain *ipd;
7373
+ unsigned long flags;
7374
+
7375
+ if (irq >= IPIPE_NR_IRQS)
7376
+ return -EINVAL;
7377
+
7378
+ ipd = ipipe_current_domain;
7379
+
7380
+ if (ipd->irqs[irq].control & IPIPE_SYSTEM_MASK)
7381
+ return -EPERM;
7382
+
7383
+ if (ipd->irqs[irq].handler == NULL)
7384
+ setmask &= ~(IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK);
7385
+
7386
+ if ((setmask & IPIPE_STICKY_MASK) != 0)
7387
+ setmask |= IPIPE_HANDLE_MASK;
7388
+
7389
+ if ((clrmask & (IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK)) != 0) /* If one goes, both go. */
7390
+ clrmask |= (IPIPE_HANDLE_MASK | IPIPE_STICKY_MASK);
7391
+
7392
+ spin_lock_irqsave(&__ipipe_pipelock, flags);
7393
+
7394
+ ipd->irqs[irq].control &= ~clrmask;
7395
+ ipd->irqs[irq].control |= setmask;
7396
+
7397
+ if ((setmask & IPIPE_ENABLE_MASK) != 0)
7398
+ __ipipe_enable_irq(irq);
7399
+ else if ((clrmask & IPIPE_ENABLE_MASK) != 0)
7400
+ __ipipe_disable_irq(irq);
7401
+
7402
+ spin_unlock_irqrestore(&__ipipe_pipelock, flags);
7403
+
7404
+ return 0;
7405
+}
7406
+
7407
+/* __ipipe_dispatch_event() -- Low-level event dispatcher. */
7408
+
7409
+int __ipipe_dispatch_event (unsigned event, void *data)
7410
+{
7411
+ struct ipipe_domain *start_domain, *this_domain, *next_domain;
7412
+ ipipe_event_handler_t evhand;
7413
+ struct list_head *pos, *npos;
7414
+ unsigned long flags;
7415
+ int propagate = 1;
7416
+
7417
+ local_irq_save_hw(flags);
7418
+
7419
+ start_domain = this_domain = ipipe_current_domain;
7420
+
7421
+ list_for_each_safe(pos, npos, &__ipipe_pipeline) {
7422
+ /*
7423
+ * Note: Domain migration may occur while running
7424
+ * event or interrupt handlers, in which case the
7425
+ * current register set is going to be recycled for a
7426
+ * different domain than the initiating one. We do
7427
+ * care for that, always tracking the current domain
7428
+ * descriptor upon return from those handlers.
7429
+ */
7430
+ next_domain = list_entry(pos, struct ipipe_domain, p_link);
7431
+
7432
+ /*
7433
+ * Keep a cached copy of the handler's address since
7434
+ * ipipe_catch_event() may clear it under our feet.
7435
+ */
7436
+ evhand = next_domain->evhand[event];
7437
+
7438
+ if (evhand != NULL) {
7439
+ ipipe_current_domain = next_domain;
7440
+ ipipe_cpudom_var(next_domain, evsync) |= (1LL << event);
7441
+ local_irq_restore_hw(flags);
7442
+ propagate = !evhand(event, start_domain, data);
7443
+ local_irq_save_hw(flags);
7444
+ /*
7445
+ * We may have a migration issue here, if the
7446
+ * current task is migrated to another CPU on
7447
+ * behalf of the invoked handler, usually when
7448
+ * a syscall event is processed. However,
7449
+ * ipipe_catch_event() will make sure that a
7450
+ * CPU that clears a handler for any given
7451
+ * event will not attempt to wait for itself
7452
+ * to clear the evsync bit for that event,
7453
+ * which practically plugs the hole, without
7454
+ * resorting to a much more complex strategy.
7455
+ */
7456
+ ipipe_cpudom_var(next_domain, evsync) &= ~(1LL << event);
7457
+ if (ipipe_current_domain != next_domain)
7458
+ this_domain = ipipe_current_domain;
7459
+ }
7460
+
7461
+ if (next_domain != ipipe_root_domain && /* NEVER sync the root stage here. */
7462
+ ipipe_cpudom_var(next_domain, irqpend_himask) != 0 &&
7463
+ !test_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(next_domain, status))) {
7464
+ ipipe_current_domain = next_domain;
7465
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
7466
+ if (ipipe_current_domain != next_domain)
7467
+ this_domain = ipipe_current_domain;
7468
+ }
7469
+
7470
+ ipipe_current_domain = this_domain;
7471
+
7472
+ if (next_domain == this_domain || !propagate)
7473
+ break;
7474
+ }
7475
+
7476
+ local_irq_restore_hw(flags);
7477
+
7478
+ return !propagate;
7479
+}
7480
+
7481
+/*
7482
+ * __ipipe_dispatch_wired -- Wired interrupt dispatcher. Wired
7483
+ * interrupts are immediately and unconditionally delivered to the
7484
+ * domain heading the pipeline upon receipt, and such domain must have
7485
+ * been registered as an invariant head for the system (priority ==
7486
+ * IPIPE_HEAD_PRIORITY). The motivation for using wired interrupts is
7487
+ * to get an extra-fast dispatching path for those IRQs, by relying on
7488
+ * a straightforward logic based on assumptions that must always be
7489
+ * true for invariant head domains. The following assumptions are
7490
+ * made when dealing with such interrupts:
7491
+ *
7492
+ * 1- Wired interrupts are purely dynamic, i.e. the decision to
7493
+ * propagate them down the pipeline must be done from the head domain
7494
+ * ISR.
7495
+ * 2- Wired interrupts cannot be shared or sticky.
7496
+ * 3- The root domain cannot be an invariant pipeline head, in
7497
+ * consequence of what the root domain cannot handle wired
7498
+ * interrupts.
7499
+ * 4- Wired interrupts must have a valid acknowledge handler for the
7500
+ * head domain (if needed, see __ipipe_handle_irq).
7501
+ *
7502
+ * Called with hw interrupts off.
7503
+ */
7504
+
7505
+void __ipipe_dispatch_wired(struct ipipe_domain *head, unsigned irq)
7506
+{
7507
+ struct ipipe_percpu_domain_data *p = ipipe_cpudom_ptr(head);
7508
+
7509
+ prefetchw(p);
7510
+
7511
+ if (unlikely(test_bit(IPIPE_LOCK_FLAG, &head->irqs[irq].control))) {
7512
+ /*
7513
+ * If we can't process this IRQ right now, we must
7514
+ * mark it as held, so that it will get played during
7515
+ * normal log sync when the corresponding interrupt
7516
+ * source is eventually unlocked.
7517
+ */
7518
+ p->irqall[irq]++;
7519
+ __set_bit(irq & IPIPE_IRQ_IMASK, &p->irqheld_mask[irq >> IPIPE_IRQ_ISHIFT]);
7520
+ return;
7521
+ }
7522
+
7523
+ if (test_bit(IPIPE_STALL_FLAG, &p->status)) {
7524
+ __ipipe_set_irq_pending(head, irq);
7525
+ return;
7526
+ }
7527
+
7528
+ __ipipe_dispatch_wired_nocheck(head, irq);
7529
+}
7530
+
7531
+void __ipipe_dispatch_wired_nocheck(struct ipipe_domain *head, unsigned irq)
7532
+{
7533
+ struct ipipe_percpu_domain_data *p = ipipe_cpudom_ptr(head);
7534
+ struct ipipe_domain *old;
7535
+
7536
+ prefetchw(p);
7537
+
7538
+ old = ipipe_current_domain;
7539
+ ipipe_current_domain = head; /* Switch to the head domain. */
7540
+
7541
+ p->irqall[irq]++;
7542
+ __set_bit(IPIPE_STALL_FLAG, &p->status);
7543
+ head->irqs[irq].handler(irq, head->irqs[irq].cookie); /* Call the ISR. */
7544
+ __ipipe_run_irqtail();
7545
+ __clear_bit(IPIPE_STALL_FLAG, &p->status);
7546
+
7547
+ if (ipipe_current_domain == head) {
7548
+ ipipe_current_domain = old;
7549
+ if (old == head) {
7550
+ if (p->irqpend_himask)
7551
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
7552
+ return;
7553
+ }
7554
+ }
7555
+
7556
+ __ipipe_walk_pipeline(&head->p_link);
7557
+}
7558
+
7559
+/*
7560
+ * __ipipe_sync_stage() -- Flush the pending IRQs for the current
7561
+ * domain (and processor). This routine flushes the interrupt log
7562
+ * (see "Optimistic interrupt protection" from D. Stodolsky et al. for
7563
+ * more on the deferred interrupt scheme). Every interrupt that
7564
+ * occurred while the pipeline was stalled gets played. WARNING:
7565
+ * callers on SMP boxen should always check for CPU migration on
7566
+ * return of this routine. One can control the kind of interrupts
7567
+ * which are going to be sync'ed using the syncmask
7568
+ * parameter. IPIPE_IRQMASK_ANY plays them all, IPIPE_IRQMASK_VIRT
7569
+ * plays virtual interrupts only.
7570
+ *
7571
+ * This routine must be called with hw interrupts off.
7572
+ */
7573
+void __ipipe_sync_stage(unsigned long syncmask)
7574
+{
7575
+ struct ipipe_percpu_domain_data *p;
7576
+ unsigned long mask, submask;
7577
+ struct ipipe_domain *ipd;
7578
+ int level, rank, cpu;
7579
+ unsigned irq;
7580
+
7581
+ ipd = ipipe_current_domain;
7582
+ p = ipipe_cpudom_ptr(ipd);
7583
+
7584
+ if (__test_and_set_bit(IPIPE_SYNC_FLAG, &p->status))
7585
+ return;
7586
+
7587
+ cpu = ipipe_processor_id();
7588
+
7589
+ /*
7590
+ * The policy here is to keep the dispatching code interrupt-free
7591
+ * by stalling the current stage. If the upper domain handler
7592
+ * (which we call) wants to re-enable interrupts while in a safe
7593
+ * portion of the code (e.g. SA_INTERRUPT flag unset for Linux's
7594
+ * sigaction()), it will have to unstall (then stall again before
7595
+ * returning to us!) the stage when it sees fit.
7596
+ */
7597
+ while ((mask = (p->irqpend_himask & syncmask)) != 0) {
7598
+ level = __ipipe_ffnz(mask);
7599
+
7600
+ while ((submask = p->irqpend_lomask[level]) != 0) {
7601
+ rank = __ipipe_ffnz(submask);
7602
+ irq = (level << IPIPE_IRQ_ISHIFT) + rank;
7603
+
7604
+ __clear_bit(rank, &p->irqpend_lomask[level]);
7605
+
7606
+ if (p->irqpend_lomask[level] == 0)
7607
+ __clear_bit(level, &p->irqpend_himask);
7608
+ /*
7609
+ * Make sure the compiler will not postpone
7610
+ * the pending bitmask updates before calling
7611
+ * the interrupt handling routine. Otherwise,
7612
+ * those late updates could overwrite any
7613
+ * change to irqpend_hi/lomask due to a nested
7614
+ * interrupt, leaving the latter unprocessed
7615
+ * (seen on mpc836x).
7616
+ */
7617
+ barrier();
7618
+
7619
+ if (test_bit(IPIPE_LOCK_FLAG, &ipd->irqs[irq].control))
7620
+ continue;
7621
+
7622
+ __set_bit(IPIPE_STALL_FLAG, &p->status);
7623
+
7624
+ if (ipd == ipipe_root_domain)
7625
+ trace_hardirqs_off();
7626
+
7627
+ __ipipe_run_isr(ipd, irq);
7628
+ p = ipipe_cpudom_ptr(ipipe_current_domain);
7629
+#ifdef CONFIG_SMP
7630
+ {
7631
+ int newcpu = ipipe_processor_id();
7632
+
7633
+ if (newcpu != cpu) { /* Handle CPU migration. */
7634
+ /*
7635
+ * We expect any domain to clear the SYNC bit each
7636
+ * time it switches in a new task, so that preemptions
7637
+ * and/or CPU migrations (in the SMP case) over the
7638
+ * ISR do not lock out the log syncer for some
7639
+ * indefinite amount of time. In the Linux case,
7640
+ * schedule() handles this (see kernel/sched.c). For
7641
+ * this reason, we don't bother clearing it here for
7642
+ * the source CPU in the migration handling case,
7643
+ * since it must have scheduled another task in by
7644
+ * now.
7645
+ */
7646
+ __set_bit(IPIPE_SYNC_FLAG, &p->status);
7647
+ cpu = newcpu;
7648
+ }
7649
+ }
7650
+#endif /* CONFIG_SMP */
7651
+#ifdef CONFIG_TRACE_IRQFLAGS
7652
+ if (ipipe_root_domain_p &&
7653
+ test_bit(IPIPE_STALL_FLAG, &p->status))
7654
+ trace_hardirqs_on();
7655
+#endif
7656
+ __clear_bit(IPIPE_STALL_FLAG, &p->status);
7657
+ }
7658
+ }
7659
+
7660
+ __clear_bit(IPIPE_SYNC_FLAG, &p->status);
7661
+}
7662
+
7663
+/* ipipe_register_domain() -- Link a new domain to the pipeline. */
7664
+
7665
+int ipipe_register_domain(struct ipipe_domain *ipd,
7666
+ struct ipipe_domain_attr *attr)
7667
+{
7668
+ struct ipipe_domain *_ipd;
7669
+ struct list_head *pos = NULL;
7670
+ unsigned long flags;
7671
+
7672
+ if (!ipipe_root_domain_p) {
7673
+ printk(KERN_WARNING
7674
+ "I-pipe: Only the root domain may register a new domain.\n");
7675
+ return -EPERM;
7676
+ }
7677
+
7678
+ flags = ipipe_critical_enter(NULL);
7679
+
7680
+ if (attr->priority == IPIPE_HEAD_PRIORITY) {
7681
+ if (test_bit(IPIPE_HEAD_SLOT, &__ipipe_domain_slot_map)) {
7682
+ ipipe_critical_exit(flags);
7683
+ return -EAGAIN; /* Cannot override current head. */
7684
+ }
7685
+ ipd->slot = IPIPE_HEAD_SLOT;
7686
+ } else
7687
+ ipd->slot = ffz(__ipipe_domain_slot_map);
7688
+
7689
+ if (ipd->slot < CONFIG_IPIPE_DOMAINS) {
7690
+ set_bit(ipd->slot, &__ipipe_domain_slot_map);
7691
+ list_for_each(pos, &__ipipe_pipeline) {
7692
+ _ipd = list_entry(pos, struct ipipe_domain, p_link);
7693
+ if (_ipd->domid == attr->domid)
7694
+ break;
7695
+ }
7696
+ }
7697
+
7698
+ ipipe_critical_exit(flags);
7699
+
7700
+ if (pos != &__ipipe_pipeline) {
7701
+ if (ipd->slot < CONFIG_IPIPE_DOMAINS)
7702
+ clear_bit(ipd->slot, &__ipipe_domain_slot_map);
7703
+ return -EBUSY;
7704
+ }
7705
+
7706
+#ifndef CONFIG_SMP
7707
+ /*
7708
+ * Set up the perdomain pointers for direct access to the
7709
+ * percpu domain data. This saves a costly multiply each time
7710
+ * we need to refer to the contents of the percpu domain data
7711
+ * array.
7712
+ */
7713
+ __raw_get_cpu_var(ipipe_percpu_daddr)[ipd->slot] = &__raw_get_cpu_var(ipipe_percpu_darray)[ipd->slot];
7714
+#endif
7715
+
7716
+ ipd->name = attr->name;
7717
+ ipd->domid = attr->domid;
7718
+ ipd->pdd = attr->pdd;
7719
+ ipd->flags = 0;
7720
+
7721
+ if (attr->priority == IPIPE_HEAD_PRIORITY) {
7722
+ ipd->priority = INT_MAX;
7723
+ __set_bit(IPIPE_AHEAD_FLAG,&ipd->flags);
7724
+ }
7725
+ else
7726
+ ipd->priority = attr->priority;
7727
+
7728
+ __ipipe_init_stage(ipd);
7729
+
7730
+ INIT_LIST_HEAD(&ipd->p_link);
7731
+
7732
+#ifdef CONFIG_PROC_FS
7733
+ __ipipe_add_domain_proc(ipd);
7734
+#endif /* CONFIG_PROC_FS */
7735
+
7736
+ flags = ipipe_critical_enter(NULL);
7737
+
7738
+ list_for_each(pos, &__ipipe_pipeline) {
7739
+ _ipd = list_entry(pos, struct ipipe_domain, p_link);
7740
+ if (ipd->priority > _ipd->priority)
7741
+ break;
7742
+ }
7743
+
7744
+ list_add_tail(&ipd->p_link, pos);
7745
+
7746
+ ipipe_critical_exit(flags);
7747
+
7748
+ printk(KERN_INFO "I-pipe: Domain %s registered.\n", ipd->name);
7749
+
7750
+ /*
7751
+ * Finally, allow the new domain to perform its initialization
7752
+ * chores.
7753
+ */
7754
+
7755
+ if (attr->entry != NULL) {
7756
+ ipipe_current_domain = ipd;
7757
+ attr->entry();
7758
+ ipipe_current_domain = ipipe_root_domain;
7759
+
7760
+ local_irq_save_hw(flags);
7761
+
7762
+ if (ipipe_root_cpudom_var(irqpend_himask) != 0 &&
7763
+ !test_bit(IPIPE_STALL_FLAG, &ipipe_root_cpudom_var(status)))
7764
+ __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY);
7765
+
7766
+ local_irq_restore_hw(flags);
7767
+ }
7768
+
7769
+ return 0;
7770
+}
7771
+
7772
+/* ipipe_unregister_domain() -- Remove a domain from the pipeline. */
7773
+
7774
+int ipipe_unregister_domain(struct ipipe_domain *ipd)
7775
+{
7776
+ unsigned long flags;
7777
+
7778
+ if (!ipipe_root_domain_p) {
7779
+ printk(KERN_WARNING
7780
+ "I-pipe: Only the root domain may unregister a domain.\n");
7781
+ return -EPERM;
7782
+ }
7783
+
7784
+ if (ipd == ipipe_root_domain) {
7785
+ printk(KERN_WARNING
7786
+ "I-pipe: Cannot unregister the root domain.\n");
7787
+ return -EPERM;
7788
+ }
7789
+#ifdef CONFIG_SMP
7790
+ {
7791
+ unsigned irq;
7792
+ int cpu;
7793
+
7794
+ /*
7795
+ * In the SMP case, wait for the logged events to drain on
7796
+ * other processors before eventually removing the domain
7797
+ * from the pipeline.
7798
+ */
7799
+
7800
+ ipipe_unstall_pipeline_from(ipd);
7801
+
7802
+ flags = ipipe_critical_enter(NULL);
7803
+
7804
+ for (irq = 0; irq < IPIPE_NR_IRQS; irq++) {
7805
+ clear_bit(IPIPE_HANDLE_FLAG, &ipd->irqs[irq].control);
7806
+ clear_bit(IPIPE_STICKY_FLAG, &ipd->irqs[irq].control);
7807
+ set_bit(IPIPE_PASS_FLAG, &ipd->irqs[irq].control);
7808
+ }
7809
+
7810
+ ipipe_critical_exit(flags);
7811
+
7812
+ for_each_online_cpu(cpu) {
7813
+ while (ipipe_percpudom(ipd, irqpend_himask, cpu) > 0)
7814
+ cpu_relax();
7815
+ }
7816
+ }
7817
+#endif /* CONFIG_SMP */
7818
+
7819
+ mutex_lock(&ipd->mutex);
7820
+
7821
+#ifdef CONFIG_PROC_FS
7822
+ __ipipe_remove_domain_proc(ipd);
7823
+#endif /* CONFIG_PROC_FS */
7824
+
7825
+ /*
7826
+ * Simply remove the domain from the pipeline and we are almost done.
7827
+ */
7828
+
7829
+ flags = ipipe_critical_enter(NULL);
7830
+ list_del_init(&ipd->p_link);
7831
+ ipipe_critical_exit(flags);
7832
+
7833
+ __ipipe_cleanup_domain(ipd);
7834
+
7835
+ mutex_unlock(&ipd->mutex);
7836
+
7837
+ printk(KERN_INFO "I-pipe: Domain %s unregistered.\n", ipd->name);
7838
+
7839
+ return 0;
7840
+}
7841
+
7842
+/*
7843
+ * ipipe_propagate_irq() -- Force a given IRQ propagation on behalf of
7844
+ * a running interrupt handler to the next domain down the pipeline.
7845
+ * ipipe_schedule_irq() -- Does almost the same as above, but attempts
7846
+ * to pend the interrupt for the current domain first.
7847
+ * Must be called hw IRQs off.
7848
+ */
7849
+void __ipipe_pend_irq(unsigned irq, struct list_head *head)
7850
+{
7851
+ struct ipipe_domain *ipd;
7852
+ struct list_head *ln;
7853
+
7854
+#ifdef CONFIG_IPIPE_DEBUG
7855
+ BUG_ON(irq >= IPIPE_NR_IRQS ||
7856
+ (ipipe_virtual_irq_p(irq)
7857
+ && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map)));
7858
+#endif
7859
+ for (ln = head; ln != &__ipipe_pipeline; ln = ipd->p_link.next) {
7860
+ ipd = list_entry(ln, struct ipipe_domain, p_link);
7861
+ if (test_bit(IPIPE_HANDLE_FLAG, &ipd->irqs[irq].control)) {
7862
+ __ipipe_set_irq_pending(ipd, irq);
7863
+ return;
7864
+ }
7865
+ }
7866
+}
7867
+
7868
+/* ipipe_free_virq() -- Release a virtual/soft interrupt. */
7869
+
7870
+int ipipe_free_virq(unsigned virq)
7871
+{
7872
+ if (!ipipe_virtual_irq_p(virq))
7873
+ return -EINVAL;
7874
+
7875
+ clear_bit(virq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map);
7876
+
7877
+ return 0;
7878
+}
7879
+
7880
+void ipipe_init_attr(struct ipipe_domain_attr *attr)
7881
+{
7882
+ attr->name = "anon";
7883
+ attr->domid = 1;
7884
+ attr->entry = NULL;
7885
+ attr->priority = IPIPE_ROOT_PRIO;
7886
+ attr->pdd = NULL;
7887
+}
7888
+
7889
+/*
7890
+ * ipipe_catch_event() -- Interpose or remove an event handler for a
7891
+ * given domain.
7892
+ */
7893
+ipipe_event_handler_t ipipe_catch_event(struct ipipe_domain *ipd,
7894
+ unsigned event,
7895
+ ipipe_event_handler_t handler)
7896
+{
7897
+ ipipe_event_handler_t old_handler;
7898
+ unsigned long flags;
7899
+ int self = 0, cpu;
7900
+
7901
+ if (event & IPIPE_EVENT_SELF) {
7902
+ event &= ~IPIPE_EVENT_SELF;
7903
+ self = 1;
7904
+ }
7905
+
7906
+ if (event >= IPIPE_NR_EVENTS)
7907
+ return NULL;
7908
+
7909
+ flags = ipipe_critical_enter(NULL);
7910
+
7911
+ if (!(old_handler = xchg(&ipd->evhand[event],handler))) {
7912
+ if (handler) {
7913
+ if (self)
7914
+ ipd->evself |= (1LL << event);
7915
+ else
7916
+ __ipipe_event_monitors[event]++;
7917
+ }
7918
+ }
7919
+ else if (!handler) {
7920
+ if (ipd->evself & (1LL << event))
7921
+ ipd->evself &= ~(1LL << event);
7922
+ else
7923
+ __ipipe_event_monitors[event]--;
7924
+ } else if ((ipd->evself & (1LL << event)) && !self) {
7925
+ __ipipe_event_monitors[event]++;
7926
+ ipd->evself &= ~(1LL << event);
7927
+ } else if (!(ipd->evself & (1LL << event)) && self) {
7928
+ __ipipe_event_monitors[event]--;
7929
+ ipd->evself |= (1LL << event);
7930
+ }
7931
+
7932
+ ipipe_critical_exit(flags);
7933
+
7934
+ if (!handler && ipipe_root_domain_p) {
7935
+ /*
7936
+ * If we cleared a handler on behalf of the root
7937
+ * domain, we have to wait for any current invocation
7938
+ * to drain, since our caller might subsequently unmap
7939
+ * the target domain. To this aim, this code
7940
+ * synchronizes with __ipipe_dispatch_event(),
7941
+ * guaranteeing that either the dispatcher sees a null
7942
+ * handler in which case it discards the invocation
7943
+ * (which also prevents from entering a livelock), or
7944
+ * finds a valid handler and calls it. Symmetrically,
7945
+ * ipipe_catch_event() ensures that the called code
7946
+ * won't be unmapped under our feet until the event
7947
+ * synchronization flag is cleared for the given event
7948
+ * on all CPUs.
7949
+ */
7950
+ preempt_disable();
7951
+ cpu = smp_processor_id();
7952
+ /*
7953
+ * Hack: this solves the potential migration issue
7954
+ * raised in __ipipe_dispatch_event(). This is a
7955
+ * work-around which makes the assumption that other
7956
+ * CPUs will subsequently, either process at least one
7957
+ * interrupt for the target domain, or call
7958
+ * __ipipe_dispatch_event() without going through a
7959
+ * migration while running the handler at least once;
7960
+ * practically, this is safe on any normally running
7961
+ * system.
7962
+ */
7963
+ ipipe_percpudom(ipd, evsync, cpu) &= ~(1LL << event);
7964
+ preempt_enable();
7965
+
7966
+ for_each_online_cpu(cpu) {
7967
+ while (ipipe_percpudom(ipd, evsync, cpu) & (1LL << event))
7968
+ schedule_timeout_interruptible(HZ / 50);
7969
+ }
7970
+ }
7971
+
7972
+ return old_handler;
7973
+}
7974
+
7975
+cpumask_t ipipe_set_irq_affinity (unsigned irq, cpumask_t cpumask)
7976
+{
7977
+#ifdef CONFIG_SMP
7978
+ if (irq >= IPIPE_NR_XIRQS)
7979
+ /* Allow changing affinity of external IRQs only. */
7980
+ return CPU_MASK_NONE;
7981
+
7982
+ if (num_online_cpus() > 1)
7983
+ return __ipipe_set_irq_affinity(irq,cpumask);
7984
+#endif /* CONFIG_SMP */
7985
+
7986
+ return CPU_MASK_NONE;
7987
+}
7988
+
7989
+int ipipe_send_ipi (unsigned ipi, cpumask_t cpumask)
7990
+
7991
+{
7992
+#ifdef CONFIG_SMP
7993
+ return __ipipe_send_ipi(ipi,cpumask);
7994
+#else /* !CONFIG_SMP */
7995
+ return -EINVAL;
7996
+#endif /* CONFIG_SMP */
7997
+}
7998
+
7999
+int ipipe_alloc_ptdkey (void)
8000
+{
8001
+ unsigned long flags;
8002
+ int key = -1;
8003
+
8004
+ spin_lock_irqsave(&__ipipe_pipelock,flags);
8005
+
8006
+ if (__ipipe_ptd_key_count < IPIPE_ROOT_NPTDKEYS) {
8007
+ key = ffz(__ipipe_ptd_key_map);
8008
+ set_bit(key,&__ipipe_ptd_key_map);
8009
+ __ipipe_ptd_key_count++;
8010
+ }
8011
+
8012
+ spin_unlock_irqrestore(&__ipipe_pipelock,flags);
8013
+
8014
+ return key;
8015
+}
8016
+
8017
+int ipipe_free_ptdkey (int key)
8018
+{
8019
+ unsigned long flags;
8020
+
8021
+ if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS)
8022
+ return -EINVAL;
8023
+
8024
+ spin_lock_irqsave(&__ipipe_pipelock,flags);
8025
+
8026
+ if (test_and_clear_bit(key,&__ipipe_ptd_key_map))
8027
+ __ipipe_ptd_key_count--;
8028
+
8029
+ spin_unlock_irqrestore(&__ipipe_pipelock,flags);
8030
+
8031
+ return 0;
8032
+}
8033
+
8034
+int ipipe_set_ptd (int key, void *value)
8035
+
8036
+{
8037
+ if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS)
8038
+ return -EINVAL;
8039
+
8040
+ current->ptd[key] = value;
8041
+
8042
+ return 0;
8043
+}
8044
+
8045
+void *ipipe_get_ptd (int key)
8046
+
8047
+{
8048
+ if (key < 0 || key >= IPIPE_ROOT_NPTDKEYS)
8049
+ return NULL;
8050
+
8051
+ return current->ptd[key];
8052
+}
8053
+
8054
+#ifdef CONFIG_PROC_FS
8055
+
8056
+struct proc_dir_entry *ipipe_proc_root;
8057
+
8058
+static int __ipipe_version_info_proc(char *page,
8059
+ char **start,
8060
+ off_t off, int count, int *eof, void *data)
8061
+{
8062
+ int len = sprintf(page, "%s\n", IPIPE_VERSION_STRING);
8063
+
8064
+ len -= off;
8065
+
8066
+ if (len <= off + count)
8067
+ *eof = 1;
8068
+
8069
+ *start = page + off;
8070
+
8071
+ if(len > count)
8072
+ len = count;
8073
+
8074
+ if(len < 0)
8075
+ len = 0;
8076
+
8077
+ return len;
8078
+}
8079
+
8080
+static int __ipipe_common_info_show(struct seq_file *p, void *data)
8081
+{
8082
+ struct ipipe_domain *ipd = (struct ipipe_domain *)p->private;
8083
+ char handling, stickiness, lockbit, exclusive, virtuality;
8084
+
8085
+ unsigned long ctlbits;
8086
+ unsigned irq;
8087
+
8088
+ seq_printf(p, " +----- Handling ([A]ccepted, [G]rabbed, [W]ired, [D]iscarded)\n");
8089
+ seq_printf(p, " |+---- Sticky\n");
8090
+ seq_printf(p, " ||+--- Locked\n");
8091
+ seq_printf(p, " |||+-- Exclusive\n");
8092
+ seq_printf(p, " ||||+- Virtual\n");
8093
+ seq_printf(p, "[IRQ] |||||\n");
8094
+
8095
+ mutex_lock(&ipd->mutex);
8096
+
8097
+ for (irq = 0; irq < IPIPE_NR_IRQS; irq++) {
8098
+ /* Remember to protect against
8099
+ * ipipe_virtual_irq/ipipe_control_irq if more fields
8100
+ * get involved. */
8101
+ ctlbits = ipd->irqs[irq].control;
8102
+
8103
+ if (irq >= IPIPE_NR_XIRQS && !ipipe_virtual_irq_p(irq))
8104
+ /*
8105
+ * There might be a hole between the last external
8106
+ * IRQ and the first virtual one; skip it.
8107
+ */
8108
+ continue;
8109
+
8110
+ if (ipipe_virtual_irq_p(irq)
8111
+ && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map))
8112
+ /* Non-allocated virtual IRQ; skip it. */
8113
+ continue;
8114
+
8115
+ /*
8116
+ * Statuses are as follows:
8117
+ * o "accepted" means handled _and_ passed down the pipeline.
8118
+ * o "grabbed" means handled, but the interrupt might be
8119
+ * terminated _or_ passed down the pipeline depending on
8120
+ * what the domain handler asks for to the I-pipe.
8121
+ * o "wired" is basically the same as "grabbed", except that
8122
+ * the interrupt is unconditionally delivered to an invariant
8123
+ * pipeline head domain.
8124
+ * o "passed" means unhandled by the domain but passed
8125
+ * down the pipeline.
8126
+ * o "discarded" means unhandled and _not_ passed down the
8127
+ * pipeline. The interrupt merely disappears from the
8128
+ * current domain down to the end of the pipeline.
8129
+ */
8130
+ if (ctlbits & IPIPE_HANDLE_MASK) {
8131
+ if (ctlbits & IPIPE_PASS_MASK)
8132
+ handling = 'A';
8133
+ else if (ctlbits & IPIPE_WIRED_MASK)
8134
+ handling = 'W';
8135
+ else
8136
+ handling = 'G';
8137
+ } else if (ctlbits & IPIPE_PASS_MASK)
8138
+ /* Do not output if no major action is taken. */
8139
+ continue;
8140
+ else
8141
+ handling = 'D';
8142
+
8143
+ if (ctlbits & IPIPE_STICKY_MASK)
8144
+ stickiness = 'S';
8145
+ else
8146
+ stickiness = '.';
8147
+
8148
+ if (ctlbits & IPIPE_LOCK_MASK)
8149
+ lockbit = 'L';
8150
+ else
8151
+ lockbit = '.';
8152
+
8153
+ if (ctlbits & IPIPE_EXCLUSIVE_MASK)
8154
+ exclusive = 'X';
8155
+ else
8156
+ exclusive = '.';
8157
+
8158
+ if (ipipe_virtual_irq_p(irq))
8159
+ virtuality = 'V';
8160
+ else
8161
+ virtuality = '.';
8162
+
8163
+ seq_printf(p, " %3u: %c%c%c%c%c\n",
8164
+ irq, handling, stickiness, lockbit, exclusive, virtuality);
8165
+ }
8166
+
8167
+ seq_printf(p, "[Domain info]\n");
8168
+
8169
+ seq_printf(p, "id=0x%.8x\n", ipd->domid);
8170
+
8171
+ if (test_bit(IPIPE_AHEAD_FLAG,&ipd->flags))
8172
+ seq_printf(p, "priority=topmost\n");
8173
+ else
8174
+ seq_printf(p, "priority=%d\n", ipd->priority);
8175
+
8176
+ mutex_unlock(&ipd->mutex);
8177
+
8178
+ return 0;
8179
+}
8180
+
8181
+static int __ipipe_common_info_open(struct inode *inode, struct file *file)
8182
+{
8183
+ return single_open(file, __ipipe_common_info_show, PROC_I(inode)->pde->data);
8184
+}
8185
+
8186
+static struct file_operations __ipipe_info_proc_ops = {
8187
+ .owner = THIS_MODULE,
8188
+ .open = __ipipe_common_info_open,
8189
+ .read = seq_read,
8190
+ .llseek = seq_lseek,
8191
+ .release = single_release,
8192
+};
8193
+
8194
+void __ipipe_add_domain_proc(struct ipipe_domain *ipd)
8195
+{
8196
+ struct proc_dir_entry *e = create_proc_entry(ipd->name, 0444, ipipe_proc_root);
8197
+ if (e) {
8198
+ e->proc_fops = &__ipipe_info_proc_ops;
8199
+ e->data = (void*) ipd;
8200
+ }
8201
+}
8202
+
8203
+void __ipipe_remove_domain_proc(struct ipipe_domain *ipd)
8204
+{
8205
+ remove_proc_entry(ipd->name,ipipe_proc_root);
8206
+}
8207
+
8208
+void __init ipipe_init_proc(void)
8209
+{
8210
+ ipipe_proc_root = create_proc_entry("ipipe",S_IFDIR, 0);
8211
+ create_proc_read_entry("version",0444,ipipe_proc_root,&__ipipe_version_info_proc,NULL);
8212
+ __ipipe_add_domain_proc(ipipe_root_domain);
8213
+
8214
+ __ipipe_init_tracer();
8215
+}
8216
+
8217
+#endif /* CONFIG_PROC_FS */
8218
+
8219
+#ifdef CONFIG_IPIPE_DEBUG_CONTEXT
8220
+
8221
+DEFINE_PER_CPU(int, ipipe_percpu_context_check) = { 1 };
8222
+DEFINE_PER_CPU(int, ipipe_saved_context_check_state);
8223
+
8224
+void ipipe_check_context(struct ipipe_domain *border_ipd)
8225
+{
8226
+ /* Note: We don't make the per_cpu access atomic. We assume that code
8227
+ which temporarily disables the check does this in atomic context
8228
+ only. */
8229
+ if (likely(ipipe_current_domain->priority <= border_ipd->priority &&
8230
+ !test_bit(IPIPE_STALL_FLAG,
8231
+ &ipipe_head_cpudom_var(status))) ||
8232
+ !per_cpu(ipipe_percpu_context_check, ipipe_processor_id()))
8233
+ return;
8234
+
8235
+ ipipe_context_check_off();
8236
+
8237
+ ipipe_trace_panic_freeze();
8238
+ ipipe_set_printk_sync(ipipe_current_domain);
8239
+
8240
+ if (ipipe_current_domain->priority > border_ipd->priority)
8241
+ printk(KERN_ERR "I-pipe: Detected illicit call from domain "
8242
+ "'%s'\n"
8243
+ KERN_ERR " into a service reserved for domain "
8244
+ "'%s' and below.\n",
8245
+ ipipe_current_domain->name, border_ipd->name);
8246
+ else
8247
+ printk(KERN_ERR "I-pipe: Detected stalled topmost domain, "
8248
+ "probably caused by a bug.\n"
8249
+ " A critical section may have been "
8250
+ "left unterminated.\n");
8251
+ dump_stack();
8252
+ ipipe_trace_panic_dump();
8253
+}
8254
+
8255
+EXPORT_SYMBOL(ipipe_check_context);
8256
+#endif /* CONFIG_IPIPE_DEBUG_CONTEXT */
8257
+
8258
+EXPORT_SYMBOL(ipipe_virtualize_irq);
8259
+EXPORT_SYMBOL(ipipe_control_irq);
8260
+EXPORT_SYMBOL(ipipe_suspend_domain);
8261
+EXPORT_SYMBOL(ipipe_alloc_virq);
8262
+EXPORT_PER_CPU_SYMBOL(ipipe_percpu_domain);
8263
+EXPORT_PER_CPU_SYMBOL(ipipe_percpu_darray);
8264
+EXPORT_SYMBOL(ipipe_root);
8265
+EXPORT_SYMBOL(ipipe_stall_pipeline_from);
8266
+EXPORT_SYMBOL(ipipe_test_and_stall_pipeline_from);
8267
+EXPORT_SYMBOL(ipipe_unstall_pipeline_from);
8268
+EXPORT_SYMBOL(ipipe_restore_pipeline_from);
8269
+EXPORT_SYMBOL(ipipe_test_and_unstall_pipeline_from);
8270
+EXPORT_SYMBOL(ipipe_unstall_pipeline_head);
8271
+EXPORT_SYMBOL(__ipipe_restore_pipeline_head);
8272
+EXPORT_SYMBOL(__ipipe_unstall_root);
8273
+EXPORT_SYMBOL(__ipipe_restore_root);
8274
+EXPORT_SYMBOL(__ipipe_spin_lock_irq);
8275
+EXPORT_SYMBOL(__ipipe_spin_unlock_irq);
8276
+EXPORT_SYMBOL(__ipipe_spin_lock_irqsave);
8277
+EXPORT_SYMBOL(__ipipe_spin_unlock_irqrestore);
8278
+EXPORT_SYMBOL(__ipipe_pipeline);
8279
+EXPORT_SYMBOL(__ipipe_lock_irq);
8280
+EXPORT_SYMBOL(__ipipe_unlock_irq);
8281
+EXPORT_SYMBOL(ipipe_register_domain);
8282
+EXPORT_SYMBOL(ipipe_unregister_domain);
8283
+EXPORT_SYMBOL(ipipe_free_virq);
8284
+EXPORT_SYMBOL(ipipe_init_attr);
8285
+EXPORT_SYMBOL(ipipe_catch_event);
8286
+EXPORT_SYMBOL(ipipe_alloc_ptdkey);
8287
+EXPORT_SYMBOL(ipipe_free_ptdkey);
8288
+EXPORT_SYMBOL(ipipe_set_ptd);
8289
+EXPORT_SYMBOL(ipipe_get_ptd);
8290
+EXPORT_SYMBOL(ipipe_set_irq_affinity);
8291
+EXPORT_SYMBOL(ipipe_send_ipi);
8292
+EXPORT_SYMBOL(__ipipe_pend_irq);
8293
+EXPORT_SYMBOL(__ipipe_set_irq_pending);
8294
+#ifdef CONFIG_GENERIC_CLOCKEVENTS
8295
+EXPORT_SYMBOL(ipipe_request_tickdev);
8296
+EXPORT_SYMBOL(ipipe_release_tickdev);
8297
+#endif
8298
+
8299
+EXPORT_SYMBOL(ipipe_critical_enter);
8300
+EXPORT_SYMBOL(ipipe_critical_exit);
8301
+EXPORT_SYMBOL(ipipe_trigger_irq);
8302
+EXPORT_SYMBOL(ipipe_get_sysinfo);
8303
diff --git a/kernel/ipipe/tracer.c b/kernel/ipipe/tracer.c
8304
new file mode 100644
8305
index 0000000..a605672
8306
--- /dev/null
8307
+++ b/kernel/ipipe/tracer.c
8308
@@ -0,0 +1,1384 @@
8309
+/* -*- linux-c -*-
8310
+ * kernel/ipipe/tracer.c
8311
+ *
8312
+ * Copyright (C) 2005 Luotao Fu.
8313
+ * 2005-2008 Jan Kiszka.
8314
+ *
8315
+ * This program is free software; you can redistribute it and/or modify
8316
+ * it under the terms of the GNU General Public License as published by
8317
+ * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139,
8318
+ * USA; either version 2 of the License, or (at your option) any later
8319
+ * version.
8320
+ *
8321
+ * This program is distributed in the hope that it will be useful,
8322
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
8323
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
8324
+ * GNU General Public License for more details.
8325
+ *
8326
+ * You should have received a copy of the GNU General Public License
8327
+ * along with this program; if not, write to the Free Software
8328
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
8329
+ */
8330
+
8331
+#include <linux/kernel.h>
8332
+#include <linux/module.h>
8333
+#include <linux/version.h>
8334
+#include <linux/kallsyms.h>
8335
+#include <linux/seq_file.h>
8336
+#include <linux/proc_fs.h>
8337
+#include <linux/ctype.h>
8338
+#include <linux/vmalloc.h>
8339
+#include <linux/pid.h>
8340
+#include <linux/utsrelease.h>
8341
+#include <linux/sched.h>
8342
+#include <linux/ipipe.h>
8343
+#include <asm/uaccess.h>
8344
+
8345
+#define IPIPE_TRACE_PATHS 4 /* <!> Do not lower below 3 */
8346
+#define IPIPE_DEFAULT_ACTIVE 0
8347
+#define IPIPE_DEFAULT_MAX 1
8348
+#define IPIPE_DEFAULT_FROZEN 2
8349
+
8350
+#define IPIPE_TRACE_POINTS (1 << CONFIG_IPIPE_TRACE_SHIFT)
8351
+#define WRAP_POINT_NO(point) ((point) & (IPIPE_TRACE_POINTS-1))
8352
+
8353
+#define IPIPE_DEFAULT_PRE_TRACE 10
8354
+#define IPIPE_DEFAULT_POST_TRACE 10
8355
+#define IPIPE_DEFAULT_BACK_TRACE 100
8356
+
8357
+#define IPIPE_DELAY_NOTE 1000 /* in nanoseconds */
8358
+#define IPIPE_DELAY_WARN 10000 /* in nanoseconds */
8359
+
8360
+#define IPIPE_TFLG_NMI_LOCK 0x0001
8361
+#define IPIPE_TFLG_NMI_HIT 0x0002
8362
+#define IPIPE_TFLG_NMI_FREEZE_REQ 0x0004
8363
+
8364
+#define IPIPE_TFLG_HWIRQ_OFF 0x0100
8365
+#define IPIPE_TFLG_FREEZING 0x0200
8366
+#define IPIPE_TFLG_CURRDOM_SHIFT 10 /* bits 10..11: current domain */
8367
+#define IPIPE_TFLG_CURRDOM_MASK 0x0C00
8368
+#define IPIPE_TFLG_DOMSTATE_SHIFT 12 /* bits 12..15: domain stalled? */
8369
+#define IPIPE_TFLG_DOMSTATE_BITS 3
8370
+
8371
+#define IPIPE_TFLG_DOMAIN_STALLED(point, n) \
8372
+ (point->flags & (1 << (n + IPIPE_TFLG_DOMSTATE_SHIFT)))
8373
+#define IPIPE_TFLG_CURRENT_DOMAIN(point) \
8374
+ ((point->flags & IPIPE_TFLG_CURRDOM_MASK) >> IPIPE_TFLG_CURRDOM_SHIFT)
8375
+
8376
+struct ipipe_trace_point {
8377
+ short type;
8378
+ short flags;
8379
+ unsigned long eip;
8380
+ unsigned long parent_eip;
8381
+ unsigned long v;
8382
+ unsigned long long timestamp;
8383
+};
8384
+
8385
+struct ipipe_trace_path {
8386
+ volatile int flags;
8387
+ int dump_lock; /* separated from flags due to cross-cpu access */
8388
+ int trace_pos; /* next point to fill */
8389
+ int begin, end; /* finalised path begin and end */
8390
+ int post_trace; /* non-zero when in post-trace phase */
8391
+ unsigned long long length; /* max path length in cycles */
8392
+ unsigned long nmi_saved_eip; /* for deferred requests from NMIs */
8393
+ unsigned long nmi_saved_parent_eip;
8394
+ unsigned long nmi_saved_v;
8395
+ struct ipipe_trace_point point[IPIPE_TRACE_POINTS];
8396
+} ____cacheline_aligned_in_smp;
8397
+
8398
+enum ipipe_trace_type
8399
+{
8400
+ IPIPE_TRACE_FUNC = 0,
8401
+ IPIPE_TRACE_BEGIN,
8402
+ IPIPE_TRACE_END,
8403
+ IPIPE_TRACE_FREEZE,
8404
+ IPIPE_TRACE_SPECIAL,
8405
+ IPIPE_TRACE_PID,
8406
+ IPIPE_TRACE_EVENT,
8407
+};
8408
+
8409
+#define IPIPE_TYPE_MASK 0x0007
8410
+#define IPIPE_TYPE_BITS 3
8411
+
8412
+#ifdef CONFIG_IPIPE_TRACE_VMALLOC
8413
+static DEFINE_PER_CPU(struct ipipe_trace_path *, trace_path);
8414
+#else /* !CONFIG_IPIPE_TRACE_VMALLOC */
8415
+static DEFINE_PER_CPU(struct ipipe_trace_path, trace_path[IPIPE_TRACE_PATHS]) =
8416
+ { [0 ... IPIPE_TRACE_PATHS-1] = { .begin = -1, .end = -1 } };
8417
+#endif /* CONFIG_IPIPE_TRACE_VMALLOC */
8418
+
8419
+int ipipe_trace_enable = 0;
8420
+
8421
+static DEFINE_PER_CPU(int, active_path) = { IPIPE_DEFAULT_ACTIVE };
8422
+static DEFINE_PER_CPU(int, max_path) = { IPIPE_DEFAULT_MAX };
8423
+static DEFINE_PER_CPU(int, frozen_path) = { IPIPE_DEFAULT_FROZEN };
8424
+static IPIPE_DEFINE_SPINLOCK(global_path_lock);
8425
+static int pre_trace = IPIPE_DEFAULT_PRE_TRACE;
8426
+static int post_trace = IPIPE_DEFAULT_POST_TRACE;
8427
+static int back_trace = IPIPE_DEFAULT_BACK_TRACE;
8428
+static int verbose_trace = 1;
8429
+static unsigned long trace_overhead;
8430
+
8431
+static unsigned long trigger_begin;
8432
+static unsigned long trigger_end;
8433
+
8434
+static DEFINE_MUTEX(out_mutex);
8435
+static struct ipipe_trace_path *print_path;
8436
+#ifdef CONFIG_IPIPE_TRACE_PANIC
8437
+static struct ipipe_trace_path *panic_path;
8438
+#endif /* CONFIG_IPIPE_TRACE_PANIC */
8439
+static int print_pre_trace;
8440
+static int print_post_trace;
8441
+
8442
+
8443
+static long __ipipe_signed_tsc2us(long long tsc);
8444
+static void
8445
+__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point);
8446
+static void __ipipe_print_symname(struct seq_file *m, unsigned long eip);
8447
+
8448
+
8449
+static notrace void
8450
+__ipipe_store_domain_states(struct ipipe_trace_point *point)
8451
+{
8452
+ struct ipipe_domain *ipd;
8453
+ struct list_head *pos;
8454
+ int i = 0;
8455
+
8456
+ list_for_each_prev(pos, &__ipipe_pipeline) {
8457
+ ipd = list_entry(pos, struct ipipe_domain, p_link);
8458
+
8459
+ if (test_bit(IPIPE_STALL_FLAG, &ipipe_cpudom_var(ipd, status)))
8460
+ point->flags |= 1 << (i + IPIPE_TFLG_DOMSTATE_SHIFT);
8461
+
8462
+ if (ipd == ipipe_current_domain)
8463
+ point->flags |= i << IPIPE_TFLG_CURRDOM_SHIFT;
8464
+
8465
+ if (++i > IPIPE_TFLG_DOMSTATE_BITS)
8466
+ break;
8467
+ }
8468
+}
8469
+
8470
+static notrace int __ipipe_get_free_trace_path(int old, int cpu)
8471
+{
8472
+ int new_active = old;
8473
+ struct ipipe_trace_path *tp;
8474
+
8475
+ do {
8476
+ if (++new_active == IPIPE_TRACE_PATHS)
8477
+ new_active = 0;
8478
+ tp = &per_cpu(trace_path, cpu)[new_active];
8479
+ } while (new_active == per_cpu(max_path, cpu) ||
8480
+ new_active == per_cpu(frozen_path, cpu) ||
8481
+ tp->dump_lock);
8482
+
8483
+ return new_active;
8484
+}
8485
+
8486
+static notrace void
8487
+__ipipe_migrate_pre_trace(struct ipipe_trace_path *new_tp,
8488
+ struct ipipe_trace_path *old_tp, int old_pos)
8489
+{
8490
+ int i;
8491
+
8492
+ new_tp->trace_pos = pre_trace+1;
8493
+
8494
+ for (i = new_tp->trace_pos; i > 0; i--)
8495
+ memcpy(&new_tp->point[WRAP_POINT_NO(new_tp->trace_pos-i)],
8496
+ &old_tp->point[WRAP_POINT_NO(old_pos-i)],
8497
+ sizeof(struct ipipe_trace_point));
8498
+
8499
+ /* mark the end (i.e. the point before point[0]) invalid */
8500
+ new_tp->point[IPIPE_TRACE_POINTS-1].eip = 0;
8501
+}
8502
+
8503
+static notrace struct ipipe_trace_path *
8504
+__ipipe_trace_end(int cpu, struct ipipe_trace_path *tp, int pos)
8505
+{
8506
+ struct ipipe_trace_path *old_tp = tp;
8507
+ long active = per_cpu(active_path, cpu);
8508
+ unsigned long long length;
8509
+
8510
+ /* do we have a new worst case? */
8511
+ length = tp->point[tp->end].timestamp -
8512
+ tp->point[tp->begin].timestamp;
8513
+ if (length > per_cpu(trace_path, cpu)[per_cpu(max_path, cpu)].length) {
8514
+ /* we need protection here against other cpus trying
8515
+ to start a proc dump */
8516
+ spin_lock(&global_path_lock);
8517
+
8518
+ /* active path holds new worst case */
8519
+ tp->length = length;
8520
+ per_cpu(max_path, cpu) = active;
8521
+
8522
+ /* find next unused trace path */
8523
+ active = __ipipe_get_free_trace_path(active, cpu);
8524
+
8525
+ spin_unlock(&global_path_lock);
8526
+
8527
+ tp = &per_cpu(trace_path, cpu)[active];
8528
+
8529
+ /* migrate last entries for pre-tracing */
8530
+ __ipipe_migrate_pre_trace(tp, old_tp, pos);
8531
+ }
8532
+
8533
+ return tp;
8534
+}
8535
+
8536
+static notrace struct ipipe_trace_path *
8537
+__ipipe_trace_freeze(int cpu, struct ipipe_trace_path *tp, int pos)
8538
+{
8539
+ struct ipipe_trace_path *old_tp = tp;
8540
+ long active = per_cpu(active_path, cpu);
8541
+ int n;
8542
+
8543
+ /* frozen paths have no core (begin=end) */
8544
+ tp->begin = tp->end;
8545
+
8546
+ /* we need protection here against other cpus trying
8547
+ * to set their frozen path or to start a proc dump */
8548
+ spin_lock(&global_path_lock);
8549
+
8550
+ per_cpu(frozen_path, cpu) = active;
8551
+
8552
+ /* find next unused trace path */
8553
+ active = __ipipe_get_free_trace_path(active, cpu);
8554
+
8555
+ /* check if this is the first frozen path */
8556
+ for_each_possible_cpu(n) {
8557
+ if (n != cpu &&
8558
+ per_cpu(trace_path, n)[per_cpu(frozen_path, n)].end >= 0)
8559
+ tp->end = -1;
8560
+ }
8561
+
8562
+ spin_unlock(&global_path_lock);
8563
+
8564
+ tp = &per_cpu(trace_path, cpu)[active];
8565
+
8566
+ /* migrate last entries for pre-tracing */
8567
+ __ipipe_migrate_pre_trace(tp, old_tp, pos);
8568
+
8569
+ return tp;
8570
+}
8571
+
8572
+void notrace
8573
+__ipipe_trace(enum ipipe_trace_type type, unsigned long eip,
8574
+ unsigned long parent_eip, unsigned long v)
8575
+{
8576
+ struct ipipe_trace_path *tp, *old_tp;
8577
+ int pos, next_pos, begin;
8578
+ struct ipipe_trace_point *point;
8579
+ unsigned long flags;
8580
+ int cpu;
8581
+
8582
+ local_irq_save_hw_notrace(flags);
8583
+
8584
+ cpu = ipipe_processor_id();
8585
+ restart:
8586
+ tp = old_tp = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)];
8587
+
8588
+ /* here starts a race window with NMIs - catched below */
8589
+
8590
+ /* check for NMI recursion */
8591
+ if (unlikely(tp->flags & IPIPE_TFLG_NMI_LOCK)) {
8592
+ tp->flags |= IPIPE_TFLG_NMI_HIT;
8593
+
8594
+ /* first freeze request from NMI context? */
8595
+ if ((type == IPIPE_TRACE_FREEZE) &&
8596
+ !(tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)) {
8597
+ /* save arguments and mark deferred freezing */
8598
+ tp->flags |= IPIPE_TFLG_NMI_FREEZE_REQ;
8599
+ tp->nmi_saved_eip = eip;
8600
+ tp->nmi_saved_parent_eip = parent_eip;
8601
+ tp->nmi_saved_v = v;
8602
+ }
8603
+ return; /* no need for restoring flags inside IRQ */
8604
+ }
8605
+
8606
+ /* clear NMI events and set lock (atomically per cpu) */
8607
+ tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT |
8608
+ IPIPE_TFLG_NMI_FREEZE_REQ))
8609
+ | IPIPE_TFLG_NMI_LOCK;
8610
+
8611
+ /* check active_path again - some nasty NMI may have switched
8612
+ * it meanwhile */
8613
+ if (unlikely(tp !=
8614
+ &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)])) {
8615
+ /* release lock on wrong path and restart */
8616
+ tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
8617
+
8618
+ /* there is no chance that the NMI got deferred
8619
+ * => no need to check for pending freeze requests */
8620
+ goto restart;
8621
+ }
8622
+
8623
+ /* get the point buffer */
8624
+ pos = tp->trace_pos;
8625
+ point = &tp->point[pos];
8626
+
8627
+ /* store all trace point data */
8628
+ point->type = type;
8629
+ point->flags = raw_irqs_disabled_flags(flags) ? IPIPE_TFLG_HWIRQ_OFF : 0;
8630
+ point->eip = eip;
8631
+ point->parent_eip = parent_eip;
8632
+ point->v = v;
8633
+ ipipe_read_tsc(point->timestamp);
8634
+
8635
+ __ipipe_store_domain_states(point);
8636
+
8637
+ /* forward to next point buffer */
8638
+ next_pos = WRAP_POINT_NO(pos+1);
8639
+ tp->trace_pos = next_pos;
8640
+
8641
+ /* only mark beginning if we haven't started yet */
8642
+ begin = tp->begin;
8643
+ if (unlikely(type == IPIPE_TRACE_BEGIN) && (begin < 0))
8644
+ tp->begin = pos;
8645
+
8646
+ /* end of critical path, start post-trace if not already started */
8647
+ if (unlikely(type == IPIPE_TRACE_END) &&
8648
+ (begin >= 0) && !tp->post_trace)
8649
+ tp->post_trace = post_trace + 1;
8650
+
8651
+ /* freeze only if the slot is free and we are not already freezing */
8652
+ if ((unlikely(type == IPIPE_TRACE_FREEZE) ||
8653
+ (unlikely(eip >= trigger_begin && eip <= trigger_end) &&
8654
+ type == IPIPE_TRACE_FUNC)) &&
8655
+ per_cpu(trace_path, cpu)[per_cpu(frozen_path, cpu)].begin < 0 &&
8656
+ !(tp->flags & IPIPE_TFLG_FREEZING)) {
8657
+ tp->post_trace = post_trace + 1;
8658
+ tp->flags |= IPIPE_TFLG_FREEZING;
8659
+ }
8660
+
8661
+ /* enforce end of trace in case of overflow */
8662
+ if (unlikely(WRAP_POINT_NO(next_pos + 1) == begin)) {
8663
+ tp->end = pos;
8664
+ goto enforce_end;
8665
+ }
8666
+
8667
+ /* stop tracing this path if we are in post-trace and
8668
+ * a) that phase is over now or
8669
+ * b) a new TRACE_BEGIN came in but we are not freezing this path */
8670
+ if (unlikely((tp->post_trace > 0) && ((--tp->post_trace == 0) ||
8671
+ ((type == IPIPE_TRACE_BEGIN) &&
8672
+ !(tp->flags & IPIPE_TFLG_FREEZING))))) {
8673
+ /* store the path's end (i.e. excluding post-trace) */
8674
+ tp->end = WRAP_POINT_NO(pos - post_trace + tp->post_trace);
8675
+
8676
+ enforce_end:
8677
+ if (tp->flags & IPIPE_TFLG_FREEZING)
8678
+ tp = __ipipe_trace_freeze(cpu, tp, pos);
8679
+ else
8680
+ tp = __ipipe_trace_end(cpu, tp, pos);
8681
+
8682
+ /* reset the active path, maybe already start a new one */
8683
+ tp->begin = (type == IPIPE_TRACE_BEGIN) ?
8684
+ WRAP_POINT_NO(tp->trace_pos - 1) : -1;
8685
+ tp->end = -1;
8686
+ tp->post_trace = 0;
8687
+ tp->flags = 0;
8688
+
8689
+ /* update active_path not earlier to avoid races with NMIs */
8690
+ per_cpu(active_path, cpu) = tp - per_cpu(trace_path, cpu);
8691
+ }
8692
+
8693
+ /* we still have old_tp and point,
8694
+ * let's reset NMI lock and check for catches */
8695
+ old_tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
8696
+ if (unlikely(old_tp->flags & IPIPE_TFLG_NMI_HIT)) {
8697
+ /* well, this late tagging may not immediately be visible for
8698
+ * other cpus already dumping this path - a minor issue */
8699
+ point->flags |= IPIPE_TFLG_NMI_HIT;
8700
+
8701
+ /* handle deferred freezing from NMI context */
8702
+ if (old_tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)
8703
+ __ipipe_trace(IPIPE_TRACE_FREEZE, old_tp->nmi_saved_eip,
8704
+ old_tp->nmi_saved_parent_eip,
8705
+ old_tp->nmi_saved_v);
8706
+ }
8707
+
8708
+ local_irq_restore_hw_notrace(flags);
8709
+}
8710
+
8711
+static unsigned long __ipipe_global_path_lock(void)
8712
+{
8713
+ unsigned long flags;
8714
+ int cpu;
8715
+ struct ipipe_trace_path *tp;
8716
+
8717
+ spin_lock_irqsave(&global_path_lock, flags);
8718
+
8719
+ cpu = ipipe_processor_id();
8720
+ restart:
8721
+ tp = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)];
8722
+
8723
+ /* here is small race window with NMIs - catched below */
8724
+
8725
+ /* clear NMI events and set lock (atomically per cpu) */
8726
+ tp->flags = (tp->flags & ~(IPIPE_TFLG_NMI_HIT |
8727
+ IPIPE_TFLG_NMI_FREEZE_REQ))
8728
+ | IPIPE_TFLG_NMI_LOCK;
8729
+
8730
+ /* check active_path again - some nasty NMI may have switched
8731
+ * it meanwhile */
8732
+ if (tp != &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)]) {
8733
+ /* release lock on wrong path and restart */
8734
+ tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
8735
+
8736
+ /* there is no chance that the NMI got deferred
8737
+ * => no need to check for pending freeze requests */
8738
+ goto restart;
8739
+ }
8740
+
8741
+ return flags;
8742
+}
8743
+
8744
+static void __ipipe_global_path_unlock(unsigned long flags)
8745
+{
8746
+ int cpu;
8747
+ struct ipipe_trace_path *tp;
8748
+
8749
+ /* release spinlock first - it's not involved in the NMI issue */
8750
+ __ipipe_spin_unlock_irqbegin(&global_path_lock);
8751
+
8752
+ cpu = ipipe_processor_id();
8753
+ tp = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)];
8754
+
8755
+ tp->flags &= ~IPIPE_TFLG_NMI_LOCK;
8756
+
8757
+ /* handle deferred freezing from NMI context */
8758
+ if (tp->flags & IPIPE_TFLG_NMI_FREEZE_REQ)
8759
+ __ipipe_trace(IPIPE_TRACE_FREEZE, tp->nmi_saved_eip,
8760
+ tp->nmi_saved_parent_eip, tp->nmi_saved_v);
8761
+
8762
+ /* See __ipipe_spin_lock_irqsave() and friends. */
8763
+ __ipipe_spin_unlock_irqcomplete(flags);
8764
+}
8765
+
8766
+void notrace ipipe_trace_begin(unsigned long v)
8767
+{
8768
+ if (!ipipe_trace_enable)
8769
+ return;
8770
+ __ipipe_trace(IPIPE_TRACE_BEGIN, __BUILTIN_RETURN_ADDRESS0,
8771
+ __BUILTIN_RETURN_ADDRESS1, v);
8772
+}
8773
+EXPORT_SYMBOL(ipipe_trace_begin);
8774
+
8775
+void notrace ipipe_trace_end(unsigned long v)
8776
+{
8777
+ if (!ipipe_trace_enable)
8778
+ return;
8779
+ __ipipe_trace(IPIPE_TRACE_END, __BUILTIN_RETURN_ADDRESS0,
8780
+ __BUILTIN_RETURN_ADDRESS1, v);
8781
+}
8782
+EXPORT_SYMBOL(ipipe_trace_end);
8783
+
8784
+void notrace ipipe_trace_freeze(unsigned long v)
8785
+{
8786
+ if (!ipipe_trace_enable)
8787
+ return;
8788
+ __ipipe_trace(IPIPE_TRACE_FREEZE, __BUILTIN_RETURN_ADDRESS0,
8789
+ __BUILTIN_RETURN_ADDRESS1, v);
8790
+}
8791
+EXPORT_SYMBOL(ipipe_trace_freeze);
8792
+
8793
+void notrace ipipe_trace_special(unsigned char id, unsigned long v)
8794
+{
8795
+ if (!ipipe_trace_enable)
8796
+ return;
8797
+ __ipipe_trace(IPIPE_TRACE_SPECIAL | (id << IPIPE_TYPE_BITS),
8798
+ __BUILTIN_RETURN_ADDRESS0,
8799
+ __BUILTIN_RETURN_ADDRESS1, v);
8800
+}
8801
+EXPORT_SYMBOL(ipipe_trace_special);
8802
+
8803
+void notrace ipipe_trace_pid(pid_t pid, short prio)
8804
+{
8805
+ if (!ipipe_trace_enable)
8806
+ return;
8807
+ __ipipe_trace(IPIPE_TRACE_PID | (prio << IPIPE_TYPE_BITS),
8808
+ __BUILTIN_RETURN_ADDRESS0,
8809
+ __BUILTIN_RETURN_ADDRESS1, pid);
8810
+}
8811
+EXPORT_SYMBOL(ipipe_trace_pid);
8812
+
8813
+void notrace ipipe_trace_event(unsigned char id, unsigned long delay_tsc)
8814
+{
8815
+ if (!ipipe_trace_enable)
8816
+ return;
8817
+ __ipipe_trace(IPIPE_TRACE_EVENT | (id << IPIPE_TYPE_BITS),
8818
+ __BUILTIN_RETURN_ADDRESS0,
8819
+ __BUILTIN_RETURN_ADDRESS1, delay_tsc);
8820
+}
8821
+EXPORT_SYMBOL(ipipe_trace_event);
8822
+
8823
+int ipipe_trace_max_reset(void)
8824
+{
8825
+ int cpu;
8826
+ unsigned long flags;
8827
+ struct ipipe_trace_path *path;
8828
+ int ret = 0;
8829
+
8830
+ flags = __ipipe_global_path_lock();
8831
+
8832
+ for_each_possible_cpu(cpu) {
8833
+ path = &per_cpu(trace_path, cpu)[per_cpu(max_path, cpu)];
8834
+
8835
+ if (path->dump_lock) {
8836
+ ret = -EBUSY;
8837
+ break;
8838
+ }
8839
+
8840
+ path->begin = -1;
8841
+ path->end = -1;
8842
+ path->trace_pos = 0;
8843
+ path->length = 0;
8844
+ }
8845
+
8846
+ __ipipe_global_path_unlock(flags);
8847
+
8848
+ return ret;
8849
+}
8850
+EXPORT_SYMBOL(ipipe_trace_max_reset);
8851
+
8852
+int ipipe_trace_frozen_reset(void)
8853
+{
8854
+ int cpu;
8855
+ unsigned long flags;
8856
+ struct ipipe_trace_path *path;
8857
+ int ret = 0;
8858
+
8859
+ flags = __ipipe_global_path_lock();
8860
+
8861
+ for_each_online_cpu(cpu) {
8862
+ path = &per_cpu(trace_path, cpu)[per_cpu(frozen_path, cpu)];
8863
+
8864
+ if (path->dump_lock) {
8865
+ ret = -EBUSY;
8866
+ break;
8867
+ }
8868
+
8869
+ path->begin = -1;
8870
+ path->end = -1;
8871
+ path->trace_pos = 0;
8872
+ path->length = 0;
8873
+ }
8874
+
8875
+ __ipipe_global_path_unlock(flags);
8876
+
8877
+ return ret;
8878
+}
8879
+EXPORT_SYMBOL(ipipe_trace_frozen_reset);
8880
+
8881
+static void
8882
+__ipipe_get_task_info(char *task_info, struct ipipe_trace_point *point,
8883
+ int trylock)
8884
+{
8885
+ struct task_struct *task = NULL;
8886
+ char buf[8];
8887
+ int i;
8888
+ int locked = 1;
8889
+
8890
+ if (trylock) {
8891
+ if (!read_trylock(&tasklist_lock))
8892
+ locked = 0;
8893
+ } else
8894
+ read_lock(&tasklist_lock);
8895
+
8896
+ if (locked)
8897
+ task = find_task_by_pid_type_ns(PIDTYPE_PID, (pid_t)point->v, &init_pid_ns);
8898
+
8899
+ if (task)
8900
+ strncpy(task_info, task->comm, 11);
8901
+ else
8902
+ strcpy(task_info, "-<?>-");
8903
+
8904
+ if (locked)
8905
+ read_unlock(&tasklist_lock);
8906
+
8907
+ for (i = strlen(task_info); i < 11; i++)
8908
+ task_info[i] = ' ';
8909
+
8910
+ sprintf(buf, " %d ", point->type >> IPIPE_TYPE_BITS);
8911
+ strcpy(task_info + (11 - strlen(buf)), buf);
8912
+}
8913
+
8914
+static void
8915
+__ipipe_get_event_date(char *buf,struct ipipe_trace_path *path,
8916
+ struct ipipe_trace_point *point)
8917
+{
8918
+ long time;
8919
+ int type;
8920
+
8921
+ time = __ipipe_signed_tsc2us(point->timestamp -
8922
+ path->point[path->begin].timestamp + point->v);
8923
+ type = point->type >> IPIPE_TYPE_BITS;
8924
+
8925
+ if (type == 0)
8926
+ /*
8927
+ * Event type #0 is predefined, stands for the next
8928
+ * timer tick.
8929
+ */
8930
+ sprintf(buf, "tick@%-6ld", time);
8931
+ else
8932
+ sprintf(buf, "%3d@%-7ld", type, time);
8933
+}
8934
+
8935
+#ifdef CONFIG_IPIPE_TRACE_PANIC
8936
+void ipipe_trace_panic_freeze(void)
8937
+{
8938
+ unsigned long flags;
8939
+ int cpu;
8940
+
8941
+ if (!ipipe_trace_enable)
8942
+ return;
8943
+
8944
+ ipipe_trace_enable = 0;
8945
+ local_irq_save_hw_notrace(flags);
8946
+
8947
+ cpu = ipipe_processor_id();
8948
+
8949
+ panic_path = &per_cpu(trace_path, cpu)[per_cpu(active_path, cpu)];
8950
+
8951
+ local_irq_restore_hw(flags);
8952
+}
8953
+EXPORT_SYMBOL(ipipe_trace_panic_freeze);
8954
+
8955
+void ipipe_trace_panic_dump(void)
8956
+{
8957
+ int cnt = back_trace;
8958
+ int start, pos;
8959
+ char buf[16];
8960
+
8961
+ if (!panic_path)
8962
+ return;
8963
+
8964
+ ipipe_context_check_off();
8965
+
8966
+ printk("I-pipe tracer log (%d points):\n", cnt);
8967
+
8968
+ start = pos = WRAP_POINT_NO(panic_path->trace_pos-1);
8969
+
8970
+ while (cnt-- > 0) {
8971
+ struct ipipe_trace_point *point = &panic_path->point[pos];
8972
+ long time;
8973
+ char info[16];
8974
+ int i;
8975
+
8976
+ printk(" %c",
8977
+ (point->flags & IPIPE_TFLG_HWIRQ_OFF) ? '|' : ' ');
8978
+
8979
+ for (i = IPIPE_TFLG_DOMSTATE_BITS; i >= 0; i--)
8980
+ printk("%c",
8981
+ (IPIPE_TFLG_CURRENT_DOMAIN(point) == i) ?
8982
+ (IPIPE_TFLG_DOMAIN_STALLED(point, i) ?
8983
+ '#' : '+') :
8984
+ (IPIPE_TFLG_DOMAIN_STALLED(point, i) ?
8985
+ '*' : ' '));
8986
+
8987
+ if (!point->eip)
8988
+ printk("-<invalid>-\n");
8989
+ else {
8990
+ __ipipe_trace_point_type(buf, point);
8991
+ printk("%s", buf);
8992
+
8993
+ switch (point->type & IPIPE_TYPE_MASK) {
8994
+ case IPIPE_TRACE_FUNC:
8995
+ printk(" ");
8996
+ break;
8997
+
8998
+ case IPIPE_TRACE_PID:
8999
+ __ipipe_get_task_info(info,
9000
+ point, 1);
9001
+ printk("%s", info);
9002
+ break;
9003
+
9004
+ case IPIPE_TRACE_EVENT:
9005
+ __ipipe_get_event_date(info,
9006
+ panic_path, point);
9007
+ printk("%s", info);
9008
+ break;
9009
+
9010
+ default:
9011
+ printk("0x%08lx ", point->v);
9012
+ }
9013
+
9014
+ time = __ipipe_signed_tsc2us(point->timestamp -
9015
+ panic_path->point[start].timestamp);
9016
+ printk(" %5ld ", time);
9017
+
9018
+ __ipipe_print_symname(NULL, point->eip);
9019
+ printk(" (");
9020
+ __ipipe_print_symname(NULL, point->parent_eip);
9021
+ printk(")\n");
9022
+ }
9023
+ pos = WRAP_POINT_NO(pos - 1);
9024
+ }
9025
+
9026
+ panic_path = NULL;
9027
+}
9028
+EXPORT_SYMBOL(ipipe_trace_panic_dump);
9029
+#endif /* CONFIG_IPIPE_TRACE_PANIC */
9030
+
9031
+
9032
+/* --- /proc output --- */
9033
+
9034
+static notrace int __ipipe_in_critical_trpath(long point_no)
9035
+{
9036
+ return ((WRAP_POINT_NO(point_no-print_path->begin) <
9037
+ WRAP_POINT_NO(print_path->end-print_path->begin)) ||
9038
+ ((print_path->end == print_path->begin) &&
9039
+ (WRAP_POINT_NO(point_no-print_path->end) >
9040
+ print_post_trace)));
9041
+}
9042
+
9043
+static long __ipipe_signed_tsc2us(long long tsc)
9044
+{
9045
+ unsigned long long abs_tsc;
9046
+ long us;
9047
+
9048
+ /* ipipe_tsc2us works on unsigned => handle sign separately */
9049
+ abs_tsc = (tsc >= 0) ? tsc : -tsc;
9050
+ us = ipipe_tsc2us(abs_tsc);
9051
+ if (tsc < 0)
9052
+ return -us;
9053
+ else
9054
+ return us;
9055
+}
9056
+
9057
+static void
9058
+__ipipe_trace_point_type(char *buf, struct ipipe_trace_point *point)
9059
+{
9060
+ switch (point->type & IPIPE_TYPE_MASK) {
9061
+ case IPIPE_TRACE_FUNC:
9062
+ strcpy(buf, "func ");
9063
+ break;
9064
+
9065
+ case IPIPE_TRACE_BEGIN:
9066
+ strcpy(buf, "begin ");
9067
+ break;
9068
+
9069
+ case IPIPE_TRACE_END:
9070
+ strcpy(buf, "end ");
9071
+ break;
9072
+
9073
+ case IPIPE_TRACE_FREEZE:
9074
+ strcpy(buf, "freeze ");
9075
+ break;
9076
+
9077
+ case IPIPE_TRACE_SPECIAL:
9078
+ sprintf(buf, "(0x%02x) ",
9079
+ point->type >> IPIPE_TYPE_BITS);
9080
+ break;
9081
+
9082
+ case IPIPE_TRACE_PID:
9083
+ sprintf(buf, "[%5d] ", (pid_t)point->v);
9084
+ break;
9085
+
9086
+ case IPIPE_TRACE_EVENT:
9087
+ sprintf(buf, "event ");
9088
+ break;
9089
+ }
9090
+}
9091
+
9092
+static void
9093
+__ipipe_print_pathmark(struct seq_file *m, struct ipipe_trace_point *point)
9094
+{
9095
+ char mark = ' ';
9096
+ int point_no = point - print_path->point;
9097
+ int i;
9098
+
9099
+ if (print_path->end == point_no)
9100
+ mark = '<';
9101
+ else if (print_path->begin == point_no)
9102
+ mark = '>';
9103
+ else if (__ipipe_in_critical_trpath(point_no))
9104
+ mark = ':';
9105
+ seq_printf(m, "%c%c", mark,
9106
+ (point->flags & IPIPE_TFLG_HWIRQ_OFF) ? '|' : ' ');
9107
+
9108
+ if (!verbose_trace)
9109
+ return;
9110
+
9111
+ for (i = IPIPE_TFLG_DOMSTATE_BITS; i >= 0; i--)
9112
+ seq_printf(m, "%c",
9113
+ (IPIPE_TFLG_CURRENT_DOMAIN(point) == i) ?
9114
+ (IPIPE_TFLG_DOMAIN_STALLED(point, i) ?
9115
+ '#' : '+') :
9116
+ (IPIPE_TFLG_DOMAIN_STALLED(point, i) ? '*' : ' '));
9117
+}
9118
+
9119
+static void
9120
+__ipipe_print_delay(struct seq_file *m, struct ipipe_trace_point *point)
9121
+{
9122
+ unsigned long delay = 0;
9123
+ int next;
9124
+ char *mark = " ";
9125
+
9126
+ next = WRAP_POINT_NO(point+1 - print_path->point);
9127
+
9128
+ if (next != print_path->trace_pos)
9129
+ delay = ipipe_tsc2ns(print_path->point[next].timestamp -
9130
+ point->timestamp);
9131
+
9132
+ if (__ipipe_in_critical_trpath(point - print_path->point)) {
9133
+ if (delay > IPIPE_DELAY_WARN)
9134
+ mark = "! ";
9135
+ else if (delay > IPIPE_DELAY_NOTE)
9136
+ mark = "+ ";
9137
+ }
9138
+ seq_puts(m, mark);
9139
+
9140
+ if (verbose_trace)
9141
+ seq_printf(m, "%3lu.%03lu%c ", delay/1000, delay%1000,
9142
+ (point->flags & IPIPE_TFLG_NMI_HIT) ? 'N' : ' ');
9143
+ else
9144
+ seq_puts(m, " ");
9145
+}
9146
+
9147
+static void __ipipe_print_symname(struct seq_file *m, unsigned long eip)
9148
+{
9149
+ char namebuf[KSYM_NAME_LEN+1];
9150
+ unsigned long size, offset;
9151
+ const char *sym_name;
9152
+ char *modname;
9153
+
9154
+ sym_name = kallsyms_lookup(eip, &size, &offset, &modname, namebuf);
9155
+
9156
+#ifdef CONFIG_IPIPE_TRACE_PANIC
9157
+ if (!m) {
9158
+ /* panic dump */
9159
+ if (sym_name) {
9160
+ printk("%s+0x%lx", sym_name, offset);
9161
+ if (modname)
9162
+ printk(" [%s]", modname);
9163
+ }
9164
+ } else
9165
+#endif /* CONFIG_IPIPE_TRACE_PANIC */
9166
+ {
9167
+ if (sym_name) {
9168
+ if (verbose_trace) {
9169
+ seq_printf(m, "%s+0x%lx", sym_name, offset);
9170
+ if (modname)
9171
+ seq_printf(m, " [%s]", modname);
9172
+ } else
9173
+ seq_puts(m, sym_name);
9174
+ } else
9175
+ seq_printf(m, "<%08lx>", eip);
9176
+ }
9177
+}
9178
+
9179
+static void __ipipe_print_headline(struct seq_file *m)
9180
+{
9181
+ seq_printf(m, "Calibrated minimum trace-point overhead: %lu.%03lu "
9182
+ "us\n\n", trace_overhead/1000, trace_overhead%1000);
9183
+
9184
+ if (verbose_trace) {
9185
+ const char *name[4] = { [0 ... 3] = "<unused>" };
9186
+ struct list_head *pos;
9187
+ int i = 0;
9188
+
9189
+ list_for_each_prev(pos, &__ipipe_pipeline) {
9190
+ struct ipipe_domain *ipd =
9191
+ list_entry(pos, struct ipipe_domain, p_link);
9192
+
9193
+ name[i] = ipd->name;
9194
+ if (++i > 3)
9195
+ break;
9196
+ }
9197
+
9198
+ seq_printf(m,
9199
+ " +----- Hard IRQs ('|': locked)\n"
9200
+ " |+---- %s\n"
9201
+ " ||+--- %s\n"
9202
+ " |||+-- %s\n"
9203
+ " ||||+- %s%s\n"
9204
+ " ||||| +---------- "
9205
+ "Delay flag ('+': > %d us, '!': > %d us)\n"
9206
+ " ||||| | +- "
9207
+ "NMI noise ('N')\n"
9208
+ " ||||| | |\n"
9209
+ " Type User Val. Time Delay Function "
9210
+ "(Parent)\n",
9211
+ name[3], name[2], name[1], name[0],
9212
+ name[0] ? " ('*': domain stalled, '+': current, "
9213
+ "'#': current+stalled)" : "",
9214
+ IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000);
9215
+ } else
9216
+ seq_printf(m,
9217
+ " +--------------- Hard IRQs ('|': locked)\n"
9218
+ " | +- Delay flag "
9219
+ "('+': > %d us, '!': > %d us)\n"
9220
+ " | |\n"
9221
+ " Type Time Function (Parent)\n",
9222
+ IPIPE_DELAY_NOTE/1000, IPIPE_DELAY_WARN/1000);
9223
+}
9224
+
9225
+static void *__ipipe_max_prtrace_start(struct seq_file *m, loff_t *pos)
9226
+{
9227
+ loff_t n = *pos;
9228
+
9229
+ mutex_lock(&out_mutex);
9230
+
9231
+ if (!n) {
9232
+ struct ipipe_trace_path *tp;
9233
+ unsigned long length_usecs;
9234
+ int points, cpu;
9235
+ unsigned long flags;
9236
+
9237
+ /* protect against max_path/frozen_path updates while we
9238
+ * haven't locked our target path, also avoid recursively
9239
+ * taking global_path_lock from NMI context */
9240
+ flags = __ipipe_global_path_lock();
9241
+
9242
+ /* find the longest of all per-cpu paths */
9243
+ print_path = NULL;
9244
+ for_each_online_cpu(cpu) {
9245
+ tp = &per_cpu(trace_path, cpu)[per_cpu(max_path, cpu)];
9246
+ if ((print_path == NULL) ||
9247
+ (tp->length > print_path->length)) {
9248
+ print_path = tp;
9249
+ break;
9250
+ }
9251
+ }
9252
+ print_path->dump_lock = 1;
9253
+
9254
+ __ipipe_global_path_unlock(flags);
9255
+
9256
+ /* does this path actually contain data? */
9257
+ if (print_path->end == print_path->begin)
9258
+ return NULL;
9259
+
9260
+ /* number of points inside the critical path */
9261
+ points = WRAP_POINT_NO(print_path->end-print_path->begin+1);
9262
+
9263
+ /* pre- and post-tracing length, post-trace length was frozen
9264
+ in __ipipe_trace, pre-trace may have to be reduced due to
9265
+ buffer overrun */
9266
+ print_pre_trace = pre_trace;
9267
+ print_post_trace = WRAP_POINT_NO(print_path->trace_pos -
9268
+ print_path->end - 1);
9269
+ if (points+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1)
9270
+ print_pre_trace = IPIPE_TRACE_POINTS - 1 - points -
9271
+ print_post_trace;
9272
+
9273
+ length_usecs = ipipe_tsc2us(print_path->length);
9274
+ seq_printf(m, "I-pipe worst-case tracing service on %s/ipipe-%s\n"
9275
+ "------------------------------------------------------------\n",
9276
+ UTS_RELEASE, IPIPE_ARCH_STRING);
9277
+ seq_printf(m, "CPU: %d, Begin: %lld cycles, Trace Points: "
9278
+ "%d (-%d/+%d), Length: %lu us\n",
9279
+ cpu, print_path->point[print_path->begin].timestamp,
9280
+ points, print_pre_trace, print_post_trace, length_usecs);
9281
+ __ipipe_print_headline(m);
9282
+ }
9283
+
9284
+ /* check if we are inside the trace range */
9285
+ if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 +
9286
+ print_pre_trace + print_post_trace))
9287
+ return NULL;
9288
+
9289
+ /* return the next point to be shown */
9290
+ return &print_path->point[WRAP_POINT_NO(print_path->begin -
9291
+ print_pre_trace + n)];
9292
+}
9293
+
9294
+static void *__ipipe_prtrace_next(struct seq_file *m, void *p, loff_t *pos)
9295
+{
9296
+ loff_t n = ++*pos;
9297
+
9298
+ /* check if we are inside the trace range with the next entry */
9299
+ if (n >= WRAP_POINT_NO(print_path->end - print_path->begin + 1 +
9300
+ print_pre_trace + print_post_trace))
9301
+ return NULL;
9302
+
9303
+ /* return the next point to be shown */
9304
+ return &print_path->point[WRAP_POINT_NO(print_path->begin -
9305
+ print_pre_trace + *pos)];
9306
+}
9307
+
9308
+static void __ipipe_prtrace_stop(struct seq_file *m, void *p)
9309
+{
9310
+ if (print_path)
9311
+ print_path->dump_lock = 0;
9312
+ mutex_unlock(&out_mutex);
9313
+}
9314
+
9315
+static int __ipipe_prtrace_show(struct seq_file *m, void *p)
9316
+{
9317
+ long time;
9318
+ struct ipipe_trace_point *point = p;
9319
+ char buf[16];
9320
+
9321
+ if (!point->eip) {
9322
+ seq_puts(m, "-<invalid>-\n");
9323
+ return 0;
9324
+ }
9325
+
9326
+ __ipipe_print_pathmark(m, point);
9327
+ __ipipe_trace_point_type(buf, point);
9328
+ seq_puts(m, buf);
9329
+ if (verbose_trace)
9330
+ switch (point->type & IPIPE_TYPE_MASK) {
9331
+ case IPIPE_TRACE_FUNC:
9332
+ seq_puts(m, " ");
9333
+ break;
9334
+
9335
+ case IPIPE_TRACE_PID:
9336
+ __ipipe_get_task_info(buf, point, 0);
9337
+ seq_puts(m, buf);
9338
+ break;
9339
+
9340
+ case IPIPE_TRACE_EVENT:
9341
+ __ipipe_get_event_date(buf, print_path, point);
9342
+ seq_puts(m, buf);
9343
+ break;
9344
+
9345
+ default:
9346
+ seq_printf(m, "0x%08lx ", point->v);
9347
+ }
9348
+
9349
+ time = __ipipe_signed_tsc2us(point->timestamp -
9350
+ print_path->point[print_path->begin].timestamp);
9351
+ seq_printf(m, "%5ld", time);
9352
+
9353
+ __ipipe_print_delay(m, point);
9354
+ __ipipe_print_symname(m, point->eip);
9355
+ seq_puts(m, " (");
9356
+ __ipipe_print_symname(m, point->parent_eip);
9357
+ seq_puts(m, ")\n");
9358
+
9359
+ return 0;
9360
+}
9361
+
9362
+static struct seq_operations __ipipe_max_ptrace_ops = {
9363
+ .start = __ipipe_max_prtrace_start,
9364
+ .next = __ipipe_prtrace_next,
9365
+ .stop = __ipipe_prtrace_stop,
9366
+ .show = __ipipe_prtrace_show
9367
+};
9368
+
9369
+static int __ipipe_max_prtrace_open(struct inode *inode, struct file *file)
9370
+{
9371
+ return seq_open(file, &__ipipe_max_ptrace_ops);
9372
+}
9373
+
9374
+static ssize_t
9375
+__ipipe_max_reset(struct file *file, const char __user *pbuffer,
9376
+ size_t count, loff_t *data)
9377
+{
9378
+ mutex_lock(&out_mutex);
9379
+ ipipe_trace_max_reset();
9380
+ mutex_unlock(&out_mutex);
9381
+
9382
+ return count;
9383
+}
9384
+
9385
+struct file_operations __ipipe_max_prtrace_fops = {
9386
+ .open = __ipipe_max_prtrace_open,
9387
+ .read = seq_read,
9388
+ .write = __ipipe_max_reset,
9389
+ .llseek = seq_lseek,
9390
+ .release = seq_release,
9391
+};
9392
+
9393
+static void *__ipipe_frozen_prtrace_start(struct seq_file *m, loff_t *pos)
9394
+{
9395
+ loff_t n = *pos;
9396
+
9397
+ mutex_lock(&out_mutex);
9398
+
9399
+ if (!n) {
9400
+ struct ipipe_trace_path *tp;
9401
+ int cpu;
9402
+ unsigned long flags;
9403
+
9404
+ /* protect against max_path/frozen_path updates while we
9405
+ * haven't locked our target path, also avoid recursively
9406
+ * taking global_path_lock from NMI context */
9407
+ flags = __ipipe_global_path_lock();
9408
+
9409
+ /* find the first of all per-cpu frozen paths */
9410
+ print_path = NULL;
9411
+ for_each_online_cpu(cpu) {
9412
+ tp = &per_cpu(trace_path, cpu)[per_cpu(frozen_path, cpu)];
9413
+ if (tp->end >= 0) {
9414
+ print_path = tp;
9415
+ break;
9416
+ }
9417
+ }
9418
+ if (print_path)
9419
+ print_path->dump_lock = 1;
9420
+
9421
+ __ipipe_global_path_unlock(flags);
9422
+
9423
+ if (!print_path)
9424
+ return NULL;
9425
+
9426
+ /* back- and post-tracing length, post-trace length was frozen
9427
+ in __ipipe_trace, back-trace may have to be reduced due to
9428
+ buffer overrun */
9429
+ print_pre_trace = back_trace-1; /* substract freeze point */
9430
+ print_post_trace = WRAP_POINT_NO(print_path->trace_pos -
9431
+ print_path->end - 1);
9432
+ if (1+pre_trace+print_post_trace > IPIPE_TRACE_POINTS - 1)
9433
+ print_pre_trace = IPIPE_TRACE_POINTS - 2 -
9434
+ print_post_trace;
9435
+
9436
+ seq_printf(m, "I-pipe frozen back-tracing service on %s/ipipe-%s\n"
9437
+ "------------------------------------------------------"
9438
+ "------\n",
9439
+ UTS_RELEASE, IPIPE_ARCH_STRING);
9440
+ seq_printf(m, "CPU: %d, Freeze: %lld cycles, Trace Points: %d (+%d)\n",
9441
+ cpu, print_path->point[print_path->begin].timestamp,
9442
+ print_pre_trace+1, print_post_trace);
9443
+ __ipipe_print_headline(m);
9444
+ }
9445
+
9446
+ /* check if we are inside the trace range */
9447
+ if (n >= print_pre_trace + 1 + print_post_trace)
9448
+ return NULL;
9449
+
9450
+ /* return the next point to be shown */
9451
+ return &print_path->point[WRAP_POINT_NO(print_path->begin-
9452
+ print_pre_trace+n)];
9453
+}
9454
+
9455
+static struct seq_operations __ipipe_frozen_ptrace_ops = {
9456
+ .start = __ipipe_frozen_prtrace_start,
9457
+ .next = __ipipe_prtrace_next,
9458
+ .stop = __ipipe_prtrace_stop,
9459
+ .show = __ipipe_prtrace_show
9460
+};
9461
+
9462
+static int __ipipe_frozen_prtrace_open(struct inode *inode, struct file *file)
9463
+{
9464
+ return seq_open(file, &__ipipe_frozen_ptrace_ops);
9465
+}
9466
+
9467
+static ssize_t
9468
+__ipipe_frozen_ctrl(struct file *file, const char __user *pbuffer,
9469
+ size_t count, loff_t *data)
9470
+{
9471
+ char *end, buf[16];
9472
+ int val;
9473
+ int n;
9474
+
9475
+ n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count;
9476
+
9477
+ if (copy_from_user(buf, pbuffer, n))
9478
+ return -EFAULT;
9479
+
9480
+ buf[n] = '\0';
9481
+ val = simple_strtol(buf, &end, 0);
9482
+
9483
+ if (((*end != '\0') && !isspace(*end)) || (val < 0))
9484
+ return -EINVAL;
9485
+
9486
+ mutex_lock(&out_mutex);
9487
+ ipipe_trace_frozen_reset();
9488
+ if (val > 0)
9489
+ ipipe_trace_freeze(-1);
9490
+ mutex_unlock(&out_mutex);
9491
+
9492
+ return count;
9493
+}
9494
+
9495
+struct file_operations __ipipe_frozen_prtrace_fops = {
9496
+ .open = __ipipe_frozen_prtrace_open,
9497
+ .read = seq_read,
9498
+ .write = __ipipe_frozen_ctrl,
9499
+ .llseek = seq_lseek,
9500
+ .release = seq_release,
9501
+};
9502
+
9503
+static int __ipipe_rd_proc_val(char *page, char **start, off_t off,
9504
+ int count, int *eof, void *data)
9505
+{
9506
+ int len;
9507
+
9508
+ len = sprintf(page, "%u\n", *(int *)data);
9509
+ len -= off;
9510
+ if (len <= off + count)
9511
+ *eof = 1;
9512
+ *start = page + off;
9513
+ if (len > count)
9514
+ len = count;
9515
+ if (len < 0)
9516
+ len = 0;
9517
+
9518
+ return len;
9519
+}
9520
+
9521
+static int __ipipe_wr_proc_val(struct file *file, const char __user *buffer,
9522
+ unsigned long count, void *data)
9523
+{
9524
+ char *end, buf[16];
9525
+ int val;
9526
+ int n;
9527
+
9528
+ n = (count > sizeof(buf) - 1) ? sizeof(buf) - 1 : count;
9529
+
9530
+ if (copy_from_user(buf, buffer, n))
9531
+ return -EFAULT;
9532
+
9533
+ buf[n] = '\0';
9534
+ val = simple_strtol(buf, &end, 0);
9535
+
9536
+ if (((*end != '\0') && !isspace(*end)) || (val < 0))
9537
+ return -EINVAL;
9538
+
9539
+ mutex_lock(&out_mutex);
9540
+ *(int *)data = val;
9541
+ mutex_unlock(&out_mutex);
9542
+
9543
+ return count;
9544
+}
9545
+
9546
+static int __ipipe_rd_trigger(char *page, char **start, off_t off, int count,
9547
+ int *eof, void *data)
9548
+{
9549
+ int len;
9550
+
9551
+ if (!trigger_begin)
9552
+ return 0;
9553
+
9554
+ len = sprint_symbol(page, trigger_begin);
9555
+ page[len++] = '\n';
9556
+
9557
+ len -= off;
9558
+ if (len <= off + count)
9559
+ *eof = 1;
9560
+ *start = page + off;
9561
+ if (len > count)
9562
+ len = count;
9563
+ if (len < 0)
9564
+ len = 0;
9565
+
9566
+ return len;
9567
+}
9568
+
9569
+static int __ipipe_wr_trigger(struct file *file, const char __user *buffer,
9570
+ unsigned long count, void *data)
9571
+{
9572
+ char buf[KSYM_SYMBOL_LEN];
9573
+ unsigned long begin, end;
9574
+
9575
+ if (count > sizeof(buf) - 1)
9576
+ count = sizeof(buf) - 1;
9577
+ if (copy_from_user(buf, buffer, count))
9578
+ return -EFAULT;
9579
+ buf[count] = 0;
9580
+ if (buf[count-1] == '\n')
9581
+ buf[count-1] = 0;
9582
+
9583
+ begin = kallsyms_lookup_name(buf);
9584
+ if (!begin || !kallsyms_lookup_size_offset(begin, &end, NULL))
9585
+ return -ENOENT;
9586
+ end += begin - 1;
9587
+
9588
+ mutex_lock(&out_mutex);
9589
+ /* invalidate the current range before setting a new one */
9590
+ trigger_end = 0;
9591
+ wmb();
9592
+ ipipe_trace_frozen_reset();
9593
+
9594
+ /* set new range */
9595
+ trigger_begin = begin;
9596
+ wmb();
9597
+ trigger_end = end;
9598
+ mutex_unlock(&out_mutex);
9599
+
9600
+ return count;
9601
+}
9602
+
9603
+extern struct proc_dir_entry *ipipe_proc_root;
9604
+
9605
+static void __init
9606
+__ipipe_create_trace_proc_val(struct proc_dir_entry *trace_dir,
9607
+ const char *name, int *value_ptr)
9608
+{
9609
+ struct proc_dir_entry *entry;
9610
+
9611
+ entry = create_proc_entry(name, 0644, trace_dir);
9612
+ if (entry) {
9613
+ entry->data = value_ptr;
9614
+ entry->read_proc = __ipipe_rd_proc_val;
9615
+ entry->write_proc = __ipipe_wr_proc_val;
9616
+ entry->owner = THIS_MODULE;
9617
+ }
9618
+}
9619
+
9620
+void __init __ipipe_init_tracer(void)
9621
+{
9622
+ struct proc_dir_entry *trace_dir;
9623
+ struct proc_dir_entry *entry;
9624
+ unsigned long long start, end, min = ULLONG_MAX;
9625
+ int i;
9626
+#ifdef CONFIG_IPIPE_TRACE_VMALLOC
9627
+ int cpu, path;
9628
+
9629
+ for_each_possible_cpu(cpu) {
9630
+ struct ipipe_trace_path *tp_buf;
9631
+
9632
+ tp_buf = vmalloc_node(sizeof(struct ipipe_trace_path) *
9633
+ IPIPE_TRACE_PATHS, cpu_to_node(cpu));
9634
+ if (!tp_buf) {
9635
+ printk(KERN_ERR "I-pipe: "
9636
+ "insufficient memory for trace buffer.\n");
9637
+ return;
9638
+ }
9639
+ memset(tp_buf, 0,
9640
+ sizeof(struct ipipe_trace_path) * IPIPE_TRACE_PATHS);
9641
+ for (path = 0; path < IPIPE_TRACE_PATHS; path++) {
9642
+ tp_buf[path].begin = -1;
9643
+ tp_buf[path].end = -1;
9644
+ }
9645
+ per_cpu(trace_path, cpu) = tp_buf;
9646
+ }
9647
+#endif /* CONFIG_IPIPE_TRACE_VMALLOC */
9648
+ ipipe_trace_enable = CONFIG_IPIPE_TRACE_ENABLE_VALUE;
9649
+
9650
+ /* Calculate minimum overhead of __ipipe_trace() */
9651
+ local_irq_disable_hw();
9652
+ for (i = 0; i < 100; i++) {
9653
+ ipipe_read_tsc(start);
9654
+ __ipipe_trace(IPIPE_TRACE_FUNC, __BUILTIN_RETURN_ADDRESS0,
9655
+ __BUILTIN_RETURN_ADDRESS1, 0);
9656
+ ipipe_read_tsc(end);
9657
+
9658
+ end -= start;
9659
+ if (end < min)
9660
+ min = end;
9661
+ }
9662
+ local_irq_enable_hw();
9663
+ trace_overhead = ipipe_tsc2ns(min);
9664
+
9665
+ trace_dir = create_proc_entry("trace", S_IFDIR, ipipe_proc_root);
9666
+
9667
+ entry = create_proc_entry("max", 0644, trace_dir);
9668
+ if (entry)
9669
+ entry->proc_fops = &__ipipe_max_prtrace_fops;
9670
+
9671
+ entry = create_proc_entry("frozen", 0644, trace_dir);
9672
+ if (entry)
9673
+ entry->proc_fops = &__ipipe_frozen_prtrace_fops;
9674
+
9675
+ entry = create_proc_entry("trigger", 0644, trace_dir);
9676
+ if (entry) {
9677
+ entry->read_proc = __ipipe_rd_trigger;
9678
+ entry->write_proc = __ipipe_wr_trigger;
9679
+ entry->owner = THIS_MODULE;
9680
+ }
9681
+
9682
+ __ipipe_create_trace_proc_val(trace_dir, "pre_trace_points",
9683
+ &pre_trace);
9684
+ __ipipe_create_trace_proc_val(trace_dir, "post_trace_points",
9685
+ &post_trace);
9686
+ __ipipe_create_trace_proc_val(trace_dir, "back_trace_points",
9687
+ &back_trace);
9688
+ __ipipe_create_trace_proc_val(trace_dir, "verbose",
9689
+ &verbose_trace);
9690
+ __ipipe_create_trace_proc_val(trace_dir, "enable",
9691
+ &ipipe_trace_enable);
9692
+}
9693
diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c
9694
index 10b5092..4b835aa 100644
9695
--- a/kernel/irq/chip.c
9696
+++ b/kernel/irq/chip.c
9697
@@ -351,7 +351,9 @@ handle_level_irq(unsigned int irq, struct irq_desc *desc)
9698
irqreturn_t action_ret;
9699
9700
spin_lock(&desc->lock);
9701
+#ifndef CONFIG_IPIPE
9702
mask_ack_irq(desc, irq);
9703
+#endif /* CONFIG_IPIPE */
9704
9705
if (unlikely(desc->status & IRQ_INPROGRESS))
9706
goto out_unlock;
9707
@@ -427,8 +429,13 @@ handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc)
9708
9709
spin_lock(&desc->lock);
9710
desc->status &= ~IRQ_INPROGRESS;
9711
+#ifdef CONFIG_IPIPE
9712
+ desc->chip->unmask(irq);
9713
+out:
9714
+#else
9715
out:
9716
desc->chip->eoi(irq);
9717
+#endif
9718
9719
spin_unlock(&desc->lock);
9720
}
9721
@@ -469,8 +476,10 @@ handle_edge_irq(unsigned int irq, struct irq_desc *desc)
9722
}
9723
kstat_incr_irqs_this_cpu(irq, desc);
9724
9725
+#ifndef CONFIG_IPIPE
9726
/* Start handling the irq */
9727
desc->chip->ack(irq);
9728
+#endif /* CONFIG_IPIPE */
9729
9730
/* Mark the IRQ currently in progress.*/
9731
desc->status |= IRQ_INPROGRESS;
9732
@@ -510,6 +519,85 @@ out_unlock:
9733
spin_unlock(&desc->lock);
9734
}
9735
9736
+#ifdef CONFIG_IPIPE
9737
+
9738
+void __ipipe_ack_simple_irq(unsigned irq, struct irq_desc *desc)
9739
+{
9740
+}
9741
+
9742
+void __ipipe_end_simple_irq(unsigned irq, struct irq_desc *desc)
9743
+{
9744
+}
9745
+
9746
+void __ipipe_ack_level_irq(unsigned irq, struct irq_desc *desc)
9747
+{
9748
+ mask_ack_irq(desc, irq);
9749
+}
9750
+
9751
+void __ipipe_end_level_irq(unsigned irq, struct irq_desc *desc)
9752
+{
9753
+ if (desc->chip->unmask)
9754
+ desc->chip->unmask(irq);
9755
+}
9756
+
9757
+void __ipipe_ack_fasteoi_irq(unsigned irq, struct irq_desc *desc)
9758
+{
9759
+ desc->chip->eoi(irq);
9760
+}
9761
+
9762
+void __ipipe_end_fasteoi_irq(unsigned irq, struct irq_desc *desc)
9763
+{
9764
+ /*
9765
+ * Non-requestable IRQs should not be masked in EOI handler.
9766
+ */
9767
+ if (!(desc->status & IRQ_NOREQUEST))
9768
+ desc->chip->unmask(irq);
9769
+}
9770
+
9771
+void __ipipe_ack_edge_irq(unsigned irq, struct irq_desc *desc)
9772
+{
9773
+ desc->chip->ack(irq);
9774
+}
9775
+
9776
+void __ipipe_ack_percpu_irq(unsigned irq, struct irq_desc *desc)
9777
+{
9778
+ if (desc->chip->ack)
9779
+ desc->chip->ack(irq);
9780
+}
9781
+
9782
+void __ipipe_end_percpu_irq(unsigned irq, struct irq_desc *desc)
9783
+{
9784
+ if (desc->chip->eoi)
9785
+ desc->chip->eoi(irq);
9786
+}
9787
+
9788
+void __ipipe_end_edge_irq(unsigned irq, struct irq_desc *desc)
9789
+{
9790
+}
9791
+
9792
+void __ipipe_ack_bad_irq(unsigned irq, struct irq_desc *desc)
9793
+{
9794
+ static int done;
9795
+
9796
+ handle_bad_irq(irq, desc);
9797
+
9798
+ if (!done) {
9799
+ printk(KERN_WARNING "%s: unknown flow handler for IRQ %d\n",
9800
+ __FUNCTION__, irq);
9801
+ done = 1;
9802
+ }
9803
+}
9804
+
9805
+void __ipipe_noack_irq(unsigned irq, struct irq_desc *desc)
9806
+{
9807
+}
9808
+
9809
+void __ipipe_noend_irq(unsigned irq, struct irq_desc *desc)
9810
+{
9811
+}
9812
+
9813
+#endif /* CONFIG_IPIPE */
9814
+
9815
/**
9816
* handle_percpu_IRQ - Per CPU local irq handler
9817
* @irq: the interrupt number
9818
@@ -524,8 +612,10 @@ handle_percpu_irq(unsigned int irq, struct irq_desc *desc)
9819
9820
kstat_incr_irqs_this_cpu(irq, desc);
9821
9822
+#ifndef CONFIG_IPIPE
9823
if (desc->chip->ack)
9824
desc->chip->ack(irq);
9825
+#endif /* CONFIG_IPIPE */
9826
9827
action_ret = handle_IRQ_event(irq, desc->action);
9828
if (!noirqdebug)
9829
@@ -550,6 +640,30 @@ __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
9830
9831
if (!handle)
9832
handle = handle_bad_irq;
9833
+#ifdef CONFIG_IPIPE
9834
+ else if (handle == &handle_simple_irq) {
9835
+ desc->ipipe_ack = &__ipipe_ack_simple_irq;
9836
+ desc->ipipe_end = &__ipipe_end_simple_irq;
9837
+ }
9838
+ else if (handle == &handle_level_irq) {
9839
+ desc->ipipe_ack = &__ipipe_ack_level_irq;
9840
+ desc->ipipe_end = &__ipipe_end_level_irq;
9841
+ }
9842
+ else if (handle == &handle_edge_irq) {
9843
+ desc->ipipe_ack = &__ipipe_ack_edge_irq;
9844
+ desc->ipipe_end = &__ipipe_end_edge_irq;
9845
+ }
9846
+ else if (handle == &handle_fasteoi_irq) {
9847
+ desc->ipipe_ack = &__ipipe_ack_fasteoi_irq;
9848
+ desc->ipipe_end = &__ipipe_end_fasteoi_irq;
9849
+ }
9850
+#ifdef CONFIG_SMP
9851
+ else if (handle == &handle_percpu_irq) {
9852
+ desc->ipipe_ack = &__ipipe_ack_percpu_irq;
9853
+ desc->ipipe_end = &__ipipe_end_percpu_irq;
9854
+ }
9855
+#endif /* CONFIG_SMP */
9856
+#endif /* CONFIG_IPIPE */
9857
else if (desc->chip == &no_irq_chip) {
9858
printk(KERN_WARNING "Trying to install %sinterrupt handler "
9859
"for IRQ%d\n", is_chained ? "chained " : "", irq);
9860
@@ -561,7 +675,21 @@ __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
9861
* dummy_irq_chip for easy transition.
9862
*/
9863
desc->chip = &dummy_irq_chip;
9864
+#ifdef CONFIG_IPIPE
9865
+ desc->ipipe_ack = &__ipipe_noack_irq;
9866
+ desc->ipipe_end = &__ipipe_noend_irq;
9867
+#endif /* CONFIG_IPIPE */
9868
+ }
9869
+#ifdef CONFIG_IPIPE
9870
+ else if (is_chained) {
9871
+ desc->ipipe_ack = handle;
9872
+ desc->ipipe_end = &__ipipe_noend_irq;
9873
+ handle = &__ipipe_noack_irq;
9874
+ } else {
9875
+ desc->ipipe_ack = &__ipipe_ack_bad_irq;
9876
+ desc->ipipe_end = &__ipipe_noend_irq;
9877
}
9878
+#endif /* CONFIG_IPIPE */
9879
9880
spin_lock_irqsave(&desc->lock, flags);
9881
9882
@@ -571,9 +699,17 @@ __set_irq_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained,
9883
mask_ack_irq(desc, irq);
9884
desc->status |= IRQ_DISABLED;
9885
desc->depth = 1;
9886
+#ifdef CONFIG_IPIPE
9887
+ desc->ipipe_ack = &__ipipe_ack_bad_irq;
9888
+ desc->ipipe_end = &__ipipe_noend_irq;
9889
+#endif /* CONFIG_IPIPE */
9890
}
9891
desc->handle_irq = handle;
9892
desc->name = name;
9893
+#ifdef CONFIG_IPIPE
9894
+ /* Suppress intermediate trampoline routine. */
9895
+ ipipe_root_domain->irqs[irq].acknowledge = desc->ipipe_ack;
9896
+#endif /* CONFIG_IPIPE */
9897
9898
if (handle != handle_bad_irq && is_chained) {
9899
desc->status &= ~IRQ_DISABLED;
9900
diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c
9901
index c815b42..f2efd4e 100644
9902
--- a/kernel/irq/handle.c
9903
+++ b/kernel/irq/handle.c
9904
@@ -179,8 +179,10 @@ unsigned int __do_IRQ(unsigned int irq)
9905
/*
9906
* No locking required for CPU-local interrupts:
9907
*/
9908
+#ifndef CONFIG_IPIPE
9909
if (desc->chip->ack)
9910
desc->chip->ack(irq);
9911
+#endif
9912
if (likely(!(desc->status & IRQ_DISABLED))) {
9913
action_ret = handle_IRQ_event(irq, desc->action);
9914
if (!noirqdebug)
9915
@@ -191,8 +193,10 @@ unsigned int __do_IRQ(unsigned int irq)
9916
}
9917
9918
spin_lock(&desc->lock);
9919
+#ifndef CONFIG_IPIPE
9920
if (desc->chip->ack)
9921
desc->chip->ack(irq);
9922
+#endif
9923
/*
9924
* REPLAY is when Linux resends an IRQ that was dropped earlier
9925
* WAITING is used by probe to mark irqs that are being tested
9926
diff --git a/kernel/lockdep.c b/kernel/lockdep.c
9927
index 46a4041..bc57809 100644
9928
--- a/kernel/lockdep.c
9929
+++ b/kernel/lockdep.c
9930
@@ -2188,7 +2188,7 @@ void trace_hardirqs_on_caller(unsigned long ip)
9931
/* we'll do an OFF -> ON transition: */
9932
curr->hardirqs_enabled = 1;
9933
9934
- if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
9935
+ if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !irqs_disabled_hw()))
9936
return;
9937
if (DEBUG_LOCKS_WARN_ON(current->hardirq_context))
9938
return;
9939
@@ -2231,7 +2231,7 @@ void trace_hardirqs_off_caller(unsigned long ip)
9940
if (unlikely(!debug_locks || current->lockdep_recursion))
9941
return;
9942
9943
- if (DEBUG_LOCKS_WARN_ON(!irqs_disabled()))
9944
+ if (DEBUG_LOCKS_WARN_ON(!irqs_disabled() && !irqs_disabled_hw()))
9945
return;
9946
9947
if (curr->hardirqs_enabled) {
9948
diff --git a/kernel/panic.c b/kernel/panic.c
9949
index 4d50883..1afea02 100644
9950
--- a/kernel/panic.c
9951
+++ b/kernel/panic.c
9952
@@ -21,6 +21,7 @@
9953
#include <linux/debug_locks.h>
9954
#include <linux/random.h>
9955
#include <linux/kallsyms.h>
9956
+#include <linux/ipipe_trace.h>
9957
9958
int panic_on_oops;
9959
static unsigned long tainted_mask;
9960
@@ -285,6 +286,8 @@ int oops_may_print(void)
9961
*/
9962
void oops_enter(void)
9963
{
9964
+ ipipe_trace_panic_freeze();
9965
+ ipipe_disable_context_check(ipipe_processor_id());
9966
debug_locks_off(); /* can't trust the integrity of the kernel anymore */
9967
do_oops_enter_exit();
9968
}
9969
diff --git a/kernel/power/disk.c b/kernel/power/disk.c
9970
index c9d7408..e6915f2 100644
9971
--- a/kernel/power/disk.c
9972
+++ b/kernel/power/disk.c
9973
@@ -208,6 +208,7 @@ static int create_image(int platform_mode)
9974
9975
device_pm_lock();
9976
local_irq_disable();
9977
+ local_irq_disable_hw_cond();
9978
/* At this point, device_suspend() has been called, but *not*
9979
* device_power_down(). We *must* call device_power_down() now.
9980
* Otherwise, drivers for some devices (e.g. interrupt controllers)
9981
@@ -241,6 +242,7 @@ static int create_image(int platform_mode)
9982
device_power_up(in_suspend ?
9983
(error ? PMSG_RECOVER : PMSG_THAW) : PMSG_RESTORE);
9984
Enable_irqs:
9985
+ local_irq_enable_hw_cond();
9986
local_irq_enable();
9987
device_pm_unlock();
9988
return error;
9989
@@ -324,6 +326,7 @@ static int resume_target_kernel(void)
9990
9991
device_pm_lock();
9992
local_irq_disable();
9993
+ local_irq_disable_hw_cond();
9994
error = device_power_down(PMSG_QUIESCE);
9995
if (error) {
9996
printk(KERN_ERR "PM: Some devices failed to power down, "
9997
@@ -354,6 +357,7 @@ static int resume_target_kernel(void)
9998
touch_softlockup_watchdog();
9999
device_power_up(PMSG_RECOVER);
10000
Enable_irqs:
10001
+ local_irq_enable_hw_cond();
10002
local_irq_enable();
10003
device_pm_unlock();
10004
return error;
10005
diff --git a/kernel/printk.c b/kernel/printk.c
10006
index b84eec7..eab2d52 100644
10007
--- a/kernel/printk.c
10008
+++ b/kernel/printk.c
10009
@@ -533,6 +533,41 @@ static int have_callable_console(void)
10010
return 0;
10011
}
10012
10013
+#ifdef CONFIG_IPIPE
10014
+
10015
+static ipipe_spinlock_t __ipipe_printk_lock = IPIPE_SPIN_LOCK_UNLOCKED;
10016
+
10017
+static int __ipipe_printk_fill;
10018
+
10019
+static char __ipipe_printk_buf[__LOG_BUF_LEN];
10020
+
10021
+void __ipipe_flush_printk (unsigned virq, void *cookie)
10022
+{
10023
+ char *p = __ipipe_printk_buf;
10024
+ int len, lmax, out = 0;
10025
+ unsigned long flags;
10026
+
10027
+ goto start;
10028
+
10029
+ do {
10030
+ spin_unlock_irqrestore(&__ipipe_printk_lock, flags);
10031
+ start:
10032
+ lmax = __ipipe_printk_fill;
10033
+ while (out < lmax) {
10034
+ len = strlen(p) + 1;
10035
+ printk("%s",p);
10036
+ p += len;
10037
+ out += len;
10038
+ }
10039
+ spin_lock_irqsave(&__ipipe_printk_lock, flags);
10040
+ }
10041
+ while (__ipipe_printk_fill != lmax);
10042
+
10043
+ __ipipe_printk_fill = 0;
10044
+
10045
+ spin_unlock_irqrestore(&__ipipe_printk_lock, flags);
10046
+}
10047
+
10048
/**
10049
* printk - print a kernel message
10050
* @fmt: format string
10051
@@ -557,6 +592,63 @@ static int have_callable_console(void)
10052
10053
asmlinkage int printk(const char *fmt, ...)
10054
{
10055
+ int r, fbytes, oldcount;
10056
+ int sprintk = 1;
10057
+ int cs = -1;
10058
+ unsigned long flags;
10059
+ va_list args;
10060
+
10061
+ va_start(args, fmt);
10062
+
10063
+ if (test_bit(IPIPE_SPRINTK_FLAG, &ipipe_current_domain->flags) ||
10064
+ oops_in_progress)
10065
+ cs = ipipe_disable_context_check(ipipe_processor_id());
10066
+ else if (ipipe_current_domain == ipipe_root_domain) {
10067
+ struct ipipe_domain *dom;
10068
+
10069
+ local_irq_save_hw(flags);
10070
+ list_for_each_entry(dom, &__ipipe_pipeline, p_link) {
10071
+ if (dom == ipipe_root_domain)
10072
+ break;
10073
+ if (test_bit(IPIPE_STALL_FLAG,
10074
+ &ipipe_cpudom_var(dom, status)))
10075
+ sprintk = 0;
10076
+ }
10077
+ local_irq_restore_hw(flags);
10078
+ } else
10079
+ sprintk = 0;
10080
+
10081
+ if (sprintk) {
10082
+ r = vprintk(fmt, args);
10083
+ if (cs != -1)
10084
+ ipipe_restore_context_check(ipipe_processor_id(), cs);
10085
+ goto out;
10086
+ }
10087
+
10088
+ spin_lock_irqsave(&__ipipe_printk_lock, flags);
10089
+
10090
+ oldcount = __ipipe_printk_fill;
10091
+ fbytes = __LOG_BUF_LEN - oldcount;
10092
+
10093
+ if (fbytes > 1) {
10094
+ r = vscnprintf(__ipipe_printk_buf + __ipipe_printk_fill,
10095
+ fbytes, fmt, args) + 1; /* account for the null byte */
10096
+ __ipipe_printk_fill += r;
10097
+ } else
10098
+ r = 0;
10099
+
10100
+ spin_unlock_irqrestore(&__ipipe_printk_lock, flags);
10101
+
10102
+ if (oldcount == 0)
10103
+ ipipe_trigger_irq(__ipipe_printk_virq);
10104
+out:
10105
+ va_end(args);
10106
+
10107
+ return r;
10108
+}
10109
+#else /* !CONFIG_IPIPE */
10110
+asmlinkage int printk(const char *fmt, ...)
10111
+{
10112
va_list args;
10113
int r;
10114
10115
@@ -566,6 +658,7 @@ asmlinkage int printk(const char *fmt, ...)
10116
10117
return r;
10118
}
10119
+#endif /* CONFIG_IPIPE */
10120
10121
/* cpu currently holding logbuf_lock */
10122
static volatile unsigned int printk_cpu = UINT_MAX;
10123
diff --git a/kernel/sched.c b/kernel/sched.c
10124
index db66874..c5bc8dc 100644
10125
--- a/kernel/sched.c
10126
+++ b/kernel/sched.c
10127
@@ -2255,7 +2255,7 @@ static int try_to_wake_up(struct task_struct *p, unsigned int state, int sync)
10128
smp_wmb();
10129
rq = task_rq_lock(p, &flags);
10130
old_state = p->state;
10131
- if (!(old_state & state))
10132
+ if (!(old_state & state) || (old_state & (TASK_NOWAKEUP|TASK_ATOMICSWITCH)))
10133
goto out;
10134
10135
if (p->se.on_rq)
10136
@@ -2609,13 +2609,15 @@ asmlinkage void schedule_tail(struct task_struct *prev)
10137
#endif
10138
if (current->set_child_tid)
10139
put_user(task_pid_vnr(current), current->set_child_tid);
10140
+
10141
+ ipipe_init_notify(current);
10142
}
10143
10144
/*
10145
* context_switch - switch to the new MM and the new
10146
* thread's register state.
10147
*/
10148
-static inline void
10149
+static inline int
10150
context_switch(struct rq *rq, struct task_struct *prev,
10151
struct task_struct *next)
10152
{
10153
@@ -2657,12 +2659,23 @@ context_switch(struct rq *rq, struct task_struct *prev,
10154
switch_to(prev, next, prev);
10155
10156
barrier();
10157
+
10158
+#ifdef CONFIG_IPIPE_DELAYED_ATOMICSW
10159
+ current->state &= ~TASK_ATOMICSWITCH;
10160
+#else
10161
+ prev->state &= ~TASK_ATOMICSWITCH;
10162
+#endif
10163
+ if (task_hijacked(prev))
10164
+ return 1;
10165
+
10166
/*
10167
* this_rq must be evaluated again because prev may have moved
10168
* CPUs since it called schedule(), thus the 'rq' on its stack
10169
* frame will be invalid.
10170
*/
10171
finish_task_switch(this_rq(), prev);
10172
+
10173
+ return 0;
10174
}
10175
10176
/*
10177
@@ -4335,6 +4348,7 @@ EXPORT_SYMBOL(add_preempt_count);
10178
10179
void __kprobes sub_preempt_count(int val)
10180
{
10181
+ ipipe_check_context(ipipe_root_domain);
10182
#ifdef CONFIG_DEBUG_PREEMPT
10183
/*
10184
* Underflow?
10185
@@ -4383,6 +4397,7 @@ static noinline void __schedule_bug(struct task_struct *prev)
10186
*/
10187
static inline void schedule_debug(struct task_struct *prev)
10188
{
10189
+ ipipe_check_context(ipipe_root_domain);
10190
/*
10191
* Test if we are atomic. Since do_exit() needs to call into
10192
* schedule() atomically, we ignore that path for now.
10193
@@ -4451,6 +4466,9 @@ need_resched:
10194
rcu_qsctr_inc(cpu);
10195
prev = rq->curr;
10196
switch_count = &prev->nivcsw;
10197
+ if (unlikely(prev->state & TASK_ATOMICSWITCH))
10198
+ /* Pop one disable level -- one still remains. */
10199
+ preempt_enable();
10200
10201
release_kernel_lock(prev);
10202
need_resched_nonpreemptible:
10203
@@ -4490,15 +4508,18 @@ need_resched_nonpreemptible:
10204
rq->curr = next;
10205
++*switch_count;
10206
10207
- context_switch(rq, prev, next); /* unlocks the rq */
10208
+ if (context_switch(rq, prev, next)) /* unlocks the rq unless hijacked */
10209
+ return;
10210
/*
10211
* the context switch might have flipped the stack from under
10212
* us, hence refresh the local variables.
10213
*/
10214
cpu = smp_processor_id();
10215
rq = cpu_rq(cpu);
10216
- } else
10217
+ } else {
10218
+ prev->state &= ~TASK_ATOMICSWITCH;
10219
spin_unlock_irq(&rq->lock);
10220
+ }
10221
10222
if (unlikely(reacquire_kernel_lock(current) < 0))
10223
goto need_resched_nonpreemptible;
10224
@@ -5241,6 +5262,7 @@ recheck:
10225
10226
oldprio = p->prio;
10227
__setscheduler(rq, p, policy, param->sched_priority);
10228
+ ipipe_setsched_notify(p);
10229
10230
if (running)
10231
p->sched_class->set_curr_task(rq);
10232
@@ -5570,6 +5592,7 @@ static void __cond_resched(void)
10233
#ifdef CONFIG_DEBUG_SPINLOCK_SLEEP
10234
__might_sleep(__FILE__, __LINE__);
10235
#endif
10236
+ ipipe_check_context(ipipe_root_domain);
10237
/*
10238
* The BKS might be reacquired before we have dropped
10239
* PREEMPT_ACTIVE, which could trigger a second
10240
@@ -9388,3 +9411,60 @@ struct cgroup_subsys cpuacct_subsys = {
10241
.subsys_id = cpuacct_subsys_id,
10242
};
10243
#endif /* CONFIG_CGROUP_CPUACCT */
10244
+
10245
+#ifdef CONFIG_IPIPE
10246
+
10247
+int ipipe_setscheduler_root (struct task_struct *p, int policy, int prio)
10248
+{
10249
+ const struct sched_class *prev_class = p->sched_class;
10250
+ int oldprio, on_rq, running;
10251
+ unsigned long flags;
10252
+ struct rq *rq;
10253
+
10254
+ spin_lock_irqsave(&p->pi_lock, flags);
10255
+ rq = __task_rq_lock(p);
10256
+ update_rq_clock(rq);
10257
+ on_rq = p->se.on_rq;
10258
+ running = task_running(rq, p);
10259
+
10260
+ if (on_rq)
10261
+ deactivate_task(rq, p, 0);
10262
+ if (running)
10263
+ p->sched_class->put_prev_task(rq, p);
10264
+
10265
+ oldprio = p->prio;
10266
+ __setscheduler(rq, p, policy, prio);
10267
+ ipipe_setsched_notify(p);
10268
+
10269
+ if (running)
10270
+ p->sched_class->set_curr_task(rq);
10271
+ if (on_rq) {
10272
+ activate_task(rq, p, 0);
10273
+ check_class_changed(rq, p, prev_class, oldprio, running);
10274
+ }
10275
+ __task_rq_unlock(rq);
10276
+ spin_unlock_irqrestore(&p->pi_lock, flags);
10277
+
10278
+ rt_mutex_adjust_pi(p);
10279
+
10280
+ return 0;
10281
+}
10282
+
10283
+EXPORT_SYMBOL(ipipe_setscheduler_root);
10284
+
10285
+int ipipe_reenter_root (struct task_struct *prev, int policy, int prio)
10286
+{
10287
+ finish_task_switch(this_rq(), prev);
10288
+
10289
+ (void)reacquire_kernel_lock(current);
10290
+ preempt_enable_no_resched();
10291
+
10292
+ if (current->policy != policy || current->rt_priority != prio)
10293
+ return ipipe_setscheduler_root(current, policy, prio);
10294
+
10295
+ return 0;
10296
+}
10297
+
10298
+EXPORT_SYMBOL(ipipe_reenter_root);
10299
+
10300
+#endif /* CONFIG_IPIPE */
10301
diff --git a/kernel/signal.c b/kernel/signal.c
10302
index 28859a9..734c5c0 100644
10303
--- a/kernel/signal.c
10304
+++ b/kernel/signal.c
10305
@@ -494,6 +494,7 @@ void signal_wake_up(struct task_struct *t, int resume)
10306
unsigned int mask;
10307
10308
set_tsk_thread_flag(t, TIF_SIGPENDING);
10309
+ ipipe_sigwake_notify(t); /* TIF_SIGPENDING must be set first. */
10310
10311
/*
10312
* For SIGKILL, we want to wake it up in the stopped/traced/killable
10313
diff --git a/kernel/spinlock.c b/kernel/spinlock.c
10314
index 29ab207..c21e314 100644
10315
--- a/kernel/spinlock.c
10316
+++ b/kernel/spinlock.c
10317
@@ -87,7 +87,7 @@ unsigned long __lockfunc _spin_lock_irqsave(spinlock_t *lock)
10318
* _raw_spin_lock_flags() code, because lockdep assumes
10319
* that interrupts are not re-enabled during lock-acquire:
10320
*/
10321
-#ifdef CONFIG_LOCKDEP
10322
+#if defined(CONFIG_LOCKDEP) || defined(CONFIG_IPIPE)
10323
LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
10324
#else
10325
_raw_spin_lock_flags(lock, &flags);
10326
@@ -304,7 +304,7 @@ unsigned long __lockfunc _spin_lock_irqsave_nested(spinlock_t *lock, int subclas
10327
* _raw_spin_lock_flags() code, because lockdep assumes
10328
* that interrupts are not re-enabled during lock-acquire:
10329
*/
10330
-#ifdef CONFIG_LOCKDEP
10331
+#if defined(CONFIG_LOCKDEP) || defined(CONFIG_IPIPE)
10332
LOCK_CONTENDED(lock, _raw_spin_trylock, _raw_spin_lock);
10333
#else
10334
_raw_spin_lock_flags(lock, &flags);
10335
diff --git a/kernel/time/tick-common.c b/kernel/time/tick-common.c
10336
index df12434..3d63445 100644
10337
--- a/kernel/time/tick-common.c
10338
+++ b/kernel/time/tick-common.c
10339
@@ -69,7 +69,7 @@ static void tick_periodic(int cpu)
10340
write_sequnlock(&xtime_lock);
10341
}
10342
10343
- update_process_times(user_mode(get_irq_regs()));
10344
+ update_root_process_times(get_irq_regs());
10345
profile_tick(CPU_PROFILING);
10346
}
10347
10348
@@ -167,6 +167,10 @@ static void tick_setup_device(struct tick_device *td,
10349
10350
td->evtdev = newdev;
10351
10352
+ /* I-pipe: derive global tick IRQ from CPU 0 */
10353
+ if (cpu == 0)
10354
+ ipipe_update_tick_evtdev(newdev);
10355
+
10356
/*
10357
* When the device is not per cpu, pin the interrupt to the
10358
* current cpu:
10359
diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
10360
index 342fc9c..3665e47 100644
10361
--- a/kernel/time/tick-sched.c
10362
+++ b/kernel/time/tick-sched.c
10363
@@ -507,7 +507,7 @@ static void tick_nohz_handler(struct clock_event_device *dev)
10364
ts->idle_jiffies++;
10365
}
10366
10367
- update_process_times(user_mode(regs));
10368
+ update_root_process_times(regs);
10369
profile_tick(CPU_PROFILING);
10370
10371
while (tick_nohz_reprogram(ts, now)) {
10372
@@ -658,7 +658,7 @@ static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
10373
touch_softlockup_watchdog();
10374
ts->idle_jiffies++;
10375
}
10376
- update_process_times(user_mode(regs));
10377
+ update_root_process_times(regs);
10378
profile_tick(CPU_PROFILING);
10379
}
10380
10381
diff --git a/kernel/timer.c b/kernel/timer.c
10382
index a5eaea2..6ecc85e 100644
10383
--- a/kernel/timer.c
10384
+++ b/kernel/timer.c
10385
@@ -1052,6 +1052,26 @@ void update_process_times(int user_tick)
10386
run_posix_cpu_timers(p);
10387
}
10388
10389
+#ifdef CONFIG_IPIPE
10390
+
10391
+void update_root_process_times(struct pt_regs *regs)
10392
+{
10393
+ int cpu, user_tick = user_mode(regs);
10394
+
10395
+ if (__ipipe_root_tick_p(regs)) {
10396
+ update_process_times(user_tick);
10397
+ return;
10398
+ }
10399
+
10400
+ run_local_timers();
10401
+ cpu = smp_processor_id();
10402
+ if (rcu_pending(cpu))
10403
+ rcu_check_callbacks(cpu, user_tick);
10404
+ run_posix_cpu_timers(current);
10405
+}
10406
+
10407
+#endif
10408
+
10409
/*
10410
* Nr of active tasks - counted in fixed-point numbers
10411
*/
10412
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
10413
index b0f239e..fdc111b 100644
10414
--- a/lib/Kconfig.debug
10415
+++ b/lib/Kconfig.debug
10416
@@ -128,6 +128,8 @@ config DEBUG_SECTION_MISMATCH
10417
- Enable verbose reporting from modpost to help solving
10418
the section mismatches reported.
10419
10420
+source "kernel/ipipe/Kconfig.debug"
10421
+
10422
config DEBUG_KERNEL
10423
bool "Kernel debugging"
10424
help
10425
diff --git a/lib/bust_spinlocks.c b/lib/bust_spinlocks.c
10426
index 486da62..9a0f064 100644
10427
--- a/lib/bust_spinlocks.c
10428
+++ b/lib/bust_spinlocks.c
10429
@@ -12,6 +12,7 @@
10430
#include <linux/tty.h>
10431
#include <linux/wait.h>
10432
#include <linux/vt_kern.h>
10433
+#include <linux/ipipe_trace.h>
10434
10435
10436
void __attribute__((weak)) bust_spinlocks(int yes)
10437
@@ -22,6 +23,7 @@ void __attribute__((weak)) bust_spinlocks(int yes)
10438
#ifdef CONFIG_VT
10439
unblank_screen();
10440
#endif
10441
+ ipipe_trace_panic_dump();
10442
if (--oops_in_progress == 0)
10443
wake_up_klogd();
10444
}
10445
diff --git a/lib/ioremap.c b/lib/ioremap.c
10446
index 14c6078..a275469 100644
10447
--- a/lib/ioremap.c
10448
+++ b/lib/ioremap.c
10449
@@ -85,8 +85,8 @@ int ioremap_page_range(unsigned long addr,
10450
if (err)
10451
break;
10452
} while (pgd++, addr = next, addr != end);
10453
-
10454
- flush_cache_vmap(start, end);
10455
+ __ipipe_pin_range_globally(start, end);
10456
+ flush_cache_vmap(start, end);
10457
10458
return err;
10459
}
10460
diff --git a/lib/smp_processor_id.c b/lib/smp_processor_id.c
10461
index 0f8fc22..90dc46f 100644
10462
--- a/lib/smp_processor_id.c
10463
+++ b/lib/smp_processor_id.c
10464
@@ -12,10 +12,13 @@ notrace unsigned int debug_smp_processor_id(void)
10465
unsigned long preempt_count = preempt_count();
10466
int this_cpu = raw_smp_processor_id();
10467
10468
+ if (!ipipe_root_domain_p)
10469
+ goto out;
10470
+
10471
if (likely(preempt_count))
10472
goto out;
10473
10474
- if (irqs_disabled())
10475
+ if (irqs_disabled() || irqs_disabled_hw())
10476
goto out;
10477
10478
/*
10479
diff --git a/lib/spinlock_debug.c b/lib/spinlock_debug.c
10480
index 9c4b025..08f096b 100644
10481
--- a/lib/spinlock_debug.c
10482
+++ b/lib/spinlock_debug.c
10483
@@ -133,6 +133,8 @@ void _raw_spin_lock(spinlock_t *lock)
10484
debug_spin_lock_after(lock);
10485
}
10486
10487
+EXPORT_SYMBOL(_raw_spin_lock);
10488
+
10489
int _raw_spin_trylock(spinlock_t *lock)
10490
{
10491
int ret = __raw_spin_trylock(&lock->raw_lock);
10492
@@ -148,12 +150,16 @@ int _raw_spin_trylock(spinlock_t *lock)
10493
return ret;
10494
}
10495
10496
+EXPORT_SYMBOL(_raw_spin_trylock);
10497
+
10498
void _raw_spin_unlock(spinlock_t *lock)
10499
{
10500
debug_spin_unlock(lock);
10501
__raw_spin_unlock(&lock->raw_lock);
10502
}
10503
10504
+EXPORT_SYMBOL(_raw_spin_unlock);
10505
+
10506
static void rwlock_bug(rwlock_t *lock, const char *msg)
10507
{
10508
if (!debug_locks_off())
10509
@@ -199,6 +205,8 @@ void _raw_read_lock(rwlock_t *lock)
10510
__raw_read_lock(&lock->raw_lock);
10511
}
10512
10513
+EXPORT_SYMBOL(_raw_read_lock);
10514
+
10515
int _raw_read_trylock(rwlock_t *lock)
10516
{
10517
int ret = __raw_read_trylock(&lock->raw_lock);
10518
@@ -212,12 +220,16 @@ int _raw_read_trylock(rwlock_t *lock)
10519
return ret;
10520
}
10521
10522
+EXPORT_SYMBOL(_raw_read_trylock);
10523
+
10524
void _raw_read_unlock(rwlock_t *lock)
10525
{
10526
RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
10527
__raw_read_unlock(&lock->raw_lock);
10528
}
10529
10530
+EXPORT_SYMBOL(_raw_read_unlock);
10531
+
10532
static inline void debug_write_lock_before(rwlock_t *lock)
10533
{
10534
RWLOCK_BUG_ON(lock->magic != RWLOCK_MAGIC, lock, "bad magic");
10535
@@ -275,6 +287,8 @@ void _raw_write_lock(rwlock_t *lock)
10536
debug_write_lock_after(lock);
10537
}
10538
10539
+EXPORT_SYMBOL(_raw_write_lock);
10540
+
10541
int _raw_write_trylock(rwlock_t *lock)
10542
{
10543
int ret = __raw_write_trylock(&lock->raw_lock);
10544
@@ -290,8 +304,12 @@ int _raw_write_trylock(rwlock_t *lock)
10545
return ret;
10546
}
10547
10548
+EXPORT_SYMBOL(_raw_write_trylock);
10549
+
10550
void _raw_write_unlock(rwlock_t *lock)
10551
{
10552
debug_write_unlock(lock);
10553
__raw_write_unlock(&lock->raw_lock);
10554
}
10555
+
10556
+EXPORT_SYMBOL(_raw_write_unlock);
10557
diff --git a/mm/memory.c b/mm/memory.c
10558
index fe2257f..d710733 100644
10559
--- a/mm/memory.c
10560
+++ b/mm/memory.c
10561
@@ -52,6 +52,7 @@
10562
#include <linux/writeback.h>
10563
#include <linux/memcontrol.h>
10564
#include <linux/mmu_notifier.h>
10565
+#include <linux/vmalloc.h>
10566
10567
#include <asm/pgalloc.h>
10568
#include <asm/uaccess.h>
10569
@@ -482,6 +483,32 @@ out:
10570
return pfn_to_page(pfn);
10571
}
10572
10573
+static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma)
10574
+{
10575
+ /*
10576
+ * If the source page was a PFN mapping, we don't have
10577
+ * a "struct page" for it. We do a best-effort copy by
10578
+ * just copying from the original user address. If that
10579
+ * fails, we just zero-fill it. Live with it.
10580
+ */
10581
+ if (unlikely(!src)) {
10582
+ void *kaddr = kmap_atomic(dst, KM_USER0);
10583
+ void __user *uaddr = (void __user *)(va & PAGE_MASK);
10584
+
10585
+ /*
10586
+ * This really shouldn't fail, because the page is there
10587
+ * in the page tables. But it might just be unreadable,
10588
+ * in which case we just give up and fill the result with
10589
+ * zeroes.
10590
+ */
10591
+ if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE))
10592
+ memset(kaddr, 0, PAGE_SIZE);
10593
+ kunmap_atomic(kaddr, KM_USER0);
10594
+ flush_dcache_page(dst);
10595
+ } else
10596
+ copy_user_highpage(dst, src, va, vma);
10597
+}
10598
+
10599
/*
10600
* copy one vm_area from one task to the other. Assumes the page tables
10601
* already present in the new task to be cleared in the whole range
10602
@@ -490,8 +517,8 @@ out:
10603
10604
static inline void
10605
copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
10606
- pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
10607
- unsigned long addr, int *rss)
10608
+ pte_t *dst_pte, pte_t *src_pte, struct vm_area_struct *vma,
10609
+ unsigned long addr, int *rss, struct page *uncow_page)
10610
{
10611
unsigned long vm_flags = vma->vm_flags;
10612
pte_t pte = *src_pte;
10613
@@ -530,6 +557,21 @@ copy_one_pte(struct mm_struct *dst_mm, struct mm_struct *src_mm,
10614
* in the parent and the child
10615
*/
10616
if (is_cow_mapping(vm_flags)) {
10617
+#ifdef CONFIG_IPIPE
10618
+ if (uncow_page) {
10619
+ struct page *old_page = vm_normal_page(vma, addr, pte);
10620
+ cow_user_page(uncow_page, old_page, addr, vma);
10621
+ pte = mk_pte(uncow_page, vma->vm_page_prot);
10622
+
10623
+ if (vm_flags & VM_SHARED)
10624
+ pte = pte_mkclean(pte);
10625
+ pte = pte_mkold(pte);
10626
+
10627
+ page_add_new_anon_rmap(uncow_page, vma, addr);
10628
+ rss[!!PageAnon(uncow_page)]++;
10629
+ goto out_set_pte;
10630
+ }
10631
+#endif /* CONFIG_IPIPE */
10632
ptep_set_wrprotect(src_mm, addr, src_pte);
10633
pte = pte_wrprotect(pte);
10634
}
10635
@@ -560,13 +602,27 @@ static int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm,
10636
pte_t *src_pte, *dst_pte;
10637
spinlock_t *src_ptl, *dst_ptl;
10638
int progress = 0;
10639
+ struct page *uncow_page = NULL;
10640
int rss[2];
10641
-
10642
+#ifdef CONFIG_IPIPE
10643
+ int do_cow_break = 0;
10644
+again:
10645
+ if (do_cow_break) {
10646
+ uncow_page = alloc_page_vma(GFP_HIGHUSER, vma, addr);
10647
+ if (!uncow_page)
10648
+ return -ENOMEM;
10649
+ do_cow_break = 0;
10650
+ }
10651
+#else
10652
again:
10653
+#endif
10654
rss[1] = rss[0] = 0;
10655
dst_pte = pte_alloc_map_lock(dst_mm, dst_pmd, addr, &dst_ptl);
10656
- if (!dst_pte)
10657
+ if (!dst_pte) {
10658
+ if (uncow_page)
10659
+ page_cache_release(uncow_page);
10660
return -ENOMEM;
10661
+ }
10662
src_pte = pte_offset_map_nested(src_pmd, addr);
10663
src_ptl = pte_lockptr(src_mm, src_pmd);
10664
spin_lock_nested(src_ptl, SINGLE_DEPTH_NESTING);
10665
@@ -587,7 +643,26 @@ again:
10666
progress++;
10667
continue;
10668
}
10669
- copy_one_pte(dst_mm, src_mm, dst_pte, src_pte, vma, addr, rss);
10670
+#ifdef CONFIG_IPIPE
10671
+ if (likely(uncow_page == NULL) && likely(pte_present(*src_pte))) {
10672
+ if (is_cow_mapping(vma->vm_flags)) {
10673
+ if (((vma->vm_flags|src_mm->def_flags) & (VM_LOCKED|VM_PINNED))
10674
+ == (VM_LOCKED|VM_PINNED)) {
10675
+ arch_leave_lazy_mmu_mode();
10676
+ spin_unlock(src_ptl);
10677
+ pte_unmap_nested(src_pte);
10678
+ add_mm_rss(dst_mm, rss[0], rss[1]);
10679
+ pte_unmap_unlock(dst_pte, dst_ptl);
10680
+ cond_resched();
10681
+ do_cow_break = 1;
10682
+ goto again;
10683
+ }
10684
+ }
10685
+ }
10686
+#endif
10687
+ copy_one_pte(dst_mm, src_mm, dst_pte,
10688
+ src_pte, vma, addr, rss, uncow_page);
10689
+ uncow_page = NULL;
10690
progress += 8;
10691
} while (dst_pte++, src_pte++, addr += PAGE_SIZE, addr != end);
10692
10693
@@ -1734,32 +1809,6 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
10694
return pte;
10695
}
10696
10697
-static inline void cow_user_page(struct page *dst, struct page *src, unsigned long va, struct vm_area_struct *vma)
10698
-{
10699
- /*
10700
- * If the source page was a PFN mapping, we don't have
10701
- * a "struct page" for it. We do a best-effort copy by
10702
- * just copying from the original user address. If that
10703
- * fails, we just zero-fill it. Live with it.
10704
- */
10705
- if (unlikely(!src)) {
10706
- void *kaddr = kmap_atomic(dst, KM_USER0);
10707
- void __user *uaddr = (void __user *)(va & PAGE_MASK);
10708
-
10709
- /*
10710
- * This really shouldn't fail, because the page is there
10711
- * in the page tables. But it might just be unreadable,
10712
- * in which case we just give up and fill the result with
10713
- * zeroes.
10714
- */
10715
- if (__copy_from_user_inatomic(kaddr, uaddr, PAGE_SIZE))
10716
- memset(kaddr, 0, PAGE_SIZE);
10717
- kunmap_atomic(kaddr, KM_USER0);
10718
- flush_dcache_page(dst);
10719
- } else
10720
- copy_user_highpage(dst, src, va, vma);
10721
-}
10722
-
10723
/*
10724
* This routine handles present pages, when users try to write
10725
* to a shared page. It is done by copying the page to a new address
10726
@@ -3049,3 +3098,111 @@ void print_vma_addr(char *prefix, unsigned long ip)
10727
}
10728
up_read(¤t->mm->mmap_sem);
10729
}
10730
+
10731
+#ifdef CONFIG_IPIPE
10732
+
10733
+static inline int ipipe_pin_pte_range(struct mm_struct *mm, pmd_t *pmd,
10734
+ struct vm_area_struct *vma,
10735
+ unsigned long addr, unsigned long end)
10736
+{
10737
+ spinlock_t *ptl;
10738
+ pte_t *pte;
10739
+
10740
+ do {
10741
+ pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
10742
+ if (!pte)
10743
+ continue;
10744
+
10745
+ if (!pte_present(*pte) || pte_write(*pte)) {
10746
+ pte_unmap_unlock(pte, ptl);
10747
+ continue;
10748
+ }
10749
+
10750
+ if (do_wp_page(mm, vma, addr, pte, pmd, ptl, *pte) == VM_FAULT_OOM)
10751
+ return -ENOMEM;
10752
+ } while (addr += PAGE_SIZE, addr != end);
10753
+ return 0;
10754
+}
10755
+
10756
+static inline int ipipe_pin_pmd_range(struct mm_struct *mm, pud_t *pud,
10757
+ struct vm_area_struct *vma,
10758
+ unsigned long addr, unsigned long end)
10759
+{
10760
+ unsigned long next;
10761
+ pmd_t *pmd;
10762
+
10763
+ pmd = pmd_offset(pud, addr);
10764
+ do {
10765
+ next = pmd_addr_end(addr, end);
10766
+ if (pmd_none_or_clear_bad(pmd))
10767
+ continue;
10768
+ if (ipipe_pin_pte_range(mm, pmd, vma, addr, next))
10769
+ return -ENOMEM;
10770
+ } while (pmd++, addr = next, addr != end);
10771
+ return 0;
10772
+}
10773
+
10774
+static inline int ipipe_pin_pud_range(struct mm_struct *mm, pgd_t *pgd,
10775
+ struct vm_area_struct *vma,
10776
+ unsigned long addr, unsigned long end)
10777
+{
10778
+ unsigned long next;
10779
+ pud_t *pud;
10780
+
10781
+ pud = pud_offset(pgd, addr);
10782
+ do {
10783
+ next = pud_addr_end(addr, end);
10784
+ if (pud_none_or_clear_bad(pud))
10785
+ continue;
10786
+ if (ipipe_pin_pmd_range(mm, pud, vma, addr, next))
10787
+ return -ENOMEM;
10788
+ } while (pud++, addr = next, addr != end);
10789
+ return 0;
10790
+}
10791
+
10792
+int ipipe_disable_ondemand_mappings(struct task_struct *tsk)
10793
+{
10794
+ unsigned long addr, next, end;
10795
+ struct vm_area_struct *vma;
10796
+ struct mm_struct *mm;
10797
+ int result = 0;
10798
+ pgd_t *pgd;
10799
+
10800
+ mm = get_task_mm(tsk);
10801
+ if (!mm)
10802
+ return -EPERM;
10803
+
10804
+ down_write(&mm->mmap_sem);
10805
+ if (mm->def_flags & VM_PINNED)
10806
+ goto done_mm;
10807
+
10808
+ for (vma = mm->mmap; vma; vma = vma->vm_next) {
10809
+ if (!is_cow_mapping(vma->vm_flags)
10810
+ || !(vma->vm_flags & VM_WRITE))
10811
+ continue;
10812
+
10813
+ addr = vma->vm_start;
10814
+ end = vma->vm_end;
10815
+
10816
+ pgd = pgd_offset(mm, addr);
10817
+ do {
10818
+ next = pgd_addr_end(addr, end);
10819
+ if (pgd_none_or_clear_bad(pgd))
10820
+ continue;
10821
+ if (ipipe_pin_pud_range(mm, pgd, vma, addr, next)) {
10822
+ result = -ENOMEM;
10823
+ goto done_mm;
10824
+ }
10825
+ } while (pgd++, addr = next, addr != end);
10826
+ }
10827
+ mm->def_flags |= VM_PINNED;
10828
+
10829
+ done_mm:
10830
+ up_write(&mm->mmap_sem);
10831
+ mmput(mm);
10832
+ return result;
10833
+}
10834
+
10835
+EXPORT_SYMBOL(ipipe_disable_ondemand_mappings);
10836
+
10837
+#endif
10838
diff --git a/mm/mlock.c b/mm/mlock.c
10839
index 64dca47..4058ba6 100644
10840
--- a/mm/mlock.c
10841
+++ b/mm/mlock.c
10842
@@ -532,10 +532,10 @@ SYSCALL_DEFINE2(munlock, unsigned long, start, size_t, len)
10843
static int do_mlockall(int flags)
10844
{
10845
struct vm_area_struct * vma, * prev = NULL;
10846
- unsigned int def_flags = 0;
10847
+ unsigned int def_flags = current->mm->def_flags & VM_PINNED;
10848
10849
if (flags & MCL_FUTURE)
10850
- def_flags = VM_LOCKED;
10851
+ def_flags |= VM_LOCKED;
10852
current->mm->def_flags = def_flags;
10853
if (flags == MCL_FUTURE)
10854
goto out;
10855
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
10856
index 4172ce4..c828a98 100644
10857
--- a/mm/vmalloc.c
10858
+++ b/mm/vmalloc.c
10859
@@ -156,8 +156,7 @@ static int vmap_page_range(unsigned long start, unsigned long end,
10860
pgprot_t prot, struct page **pages)
10861
{
10862
pgd_t *pgd;
10863
- unsigned long next;
10864
- unsigned long addr = start;
10865
+ unsigned long next, addr = start;
10866
int err = 0;
10867
int nr = 0;
10868
10869
@@ -173,6 +172,9 @@ static int vmap_page_range(unsigned long start, unsigned long end,
10870
10871
if (unlikely(err))
10872
return err;
10873
+
10874
+ __ipipe_pin_range_globally(start, end);
10875
+
10876
return nr;
10877
}
10878
Loggerhead is a web-based interface for Breezy
Version: 2.0.1