79
82
struct rcu_state rcu_bh_state = RCU_STATE_INITIALIZER(rcu_bh_state);
80
83
DEFINE_PER_CPU(struct rcu_data, rcu_bh_data);
85
static struct rcu_state *rcu_state;
88
* The rcu_scheduler_active variable transitions from zero to one just
89
* before the first task is spawned. So when this variable is zero, RCU
90
* can assume that there is but one task, allowing RCU to (for example)
91
* optimized synchronize_sched() to a simple barrier(). When this variable
92
* is one, RCU must actually do all the hard work required to detect real
93
* grace periods. This variable is also used to suppress boot-time false
94
* positives from lockdep-RCU error checking.
82
96
int rcu_scheduler_active __read_mostly;
83
97
EXPORT_SYMBOL_GPL(rcu_scheduler_active);
100
* The rcu_scheduler_fully_active variable transitions from zero to one
101
* during the early_initcall() processing, which is after the scheduler
102
* is capable of creating new tasks. So RCU processing (for example,
103
* creating tasks for RCU priority boosting) must be delayed until after
104
* rcu_scheduler_fully_active transitions from zero to one. We also
105
* currently delay invocation of any RCU callbacks until after this point.
107
* It might later prove better for people registering RCU callbacks during
108
* early boot to take responsibility for these callbacks, but one step at
111
static int rcu_scheduler_fully_active __read_mostly;
113
#ifdef CONFIG_RCU_BOOST
116
* Control variables for per-CPU and per-rcu_node kthreads. These
117
* handle all flavors of RCU.
119
static DEFINE_PER_CPU(struct task_struct *, rcu_cpu_kthread_task);
120
DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_status);
121
DEFINE_PER_CPU(int, rcu_cpu_kthread_cpu);
122
DEFINE_PER_CPU(unsigned int, rcu_cpu_kthread_loops);
123
DEFINE_PER_CPU(char, rcu_cpu_has_work);
125
#endif /* #ifdef CONFIG_RCU_BOOST */
127
static void rcu_node_kthread_setaffinity(struct rcu_node *rnp, int outgoingcpu);
128
static void invoke_rcu_core(void);
129
static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp);
131
#define RCU_KTHREAD_PRIO 1 /* RT priority for per-CPU kthreads. */
134
* Track the rcutorture test sequence number and the update version
135
* number within a given test. The rcutorture_testseq is incremented
136
* on every rcutorture module load and unload, so has an odd value
137
* when a test is running. The rcutorture_vernum is set to zero
138
* when rcutorture starts and is incremented on each rcutorture update.
139
* These variables enable correlating rcutorture output with the
140
* RCU tracing information.
142
unsigned long rcutorture_testseq;
143
unsigned long rcutorture_vernum;
86
146
* Return true if an RCU grace period is in progress. The ACCESS_ONCE()s
87
147
* permit this function to be invoked without holding the root rcu_node
88
148
* structure's ->lock, but of course results can be subject to change.
140
201
module_param(qhimark, int, 0);
141
202
module_param(qlowmark, int, 0);
143
#ifdef CONFIG_RCU_CPU_STALL_DETECTOR
144
int rcu_cpu_stall_suppress __read_mostly = RCU_CPU_STALL_SUPPRESS_INIT;
204
int rcu_cpu_stall_suppress __read_mostly;
145
205
module_param(rcu_cpu_stall_suppress, int, 0644);
146
#endif /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
148
207
static void force_quiescent_state(struct rcu_state *rsp, int relaxed);
149
208
static int rcu_pending(int cpu);
176
235
EXPORT_SYMBOL_GPL(rcu_bh_force_quiescent_state);
238
* Record the number of times rcutorture tests have been initiated and
239
* terminated. This information allows the debugfs tracing stats to be
240
* correlated to the rcutorture messages, even when the rcutorture module
241
* is being repeatedly loaded and unloaded. In other words, we cannot
242
* store this state in rcutorture itself.
244
void rcutorture_record_test_transition(void)
246
rcutorture_testseq++;
247
rcutorture_vernum = 0;
249
EXPORT_SYMBOL_GPL(rcutorture_record_test_transition);
252
* Record the number of writer passes through the current rcutorture test.
253
* This is also used to correlate debugfs tracing stats with the rcutorture
256
void rcutorture_record_progress(unsigned long vernum)
260
EXPORT_SYMBOL_GPL(rcutorture_record_progress);
179
263
* Force a quiescent state for RCU-sched.
181
265
void rcu_sched_force_quiescent_state(void)
264
348
unsigned long flags;
265
349
struct rcu_dynticks *rdtp;
267
smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */
268
351
local_irq_save(flags);
269
352
rdtp = &__get_cpu_var(rcu_dynticks);
271
rdtp->dynticks_nesting--;
272
WARN_ON_ONCE(rdtp->dynticks & 0x1);
353
if (--rdtp->dynticks_nesting) {
354
local_irq_restore(flags);
357
/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
358
smp_mb__before_atomic_inc(); /* See above. */
359
atomic_inc(&rdtp->dynticks);
360
smp_mb__after_atomic_inc(); /* Force ordering with next sojourn. */
361
WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
273
362
local_irq_restore(flags);
364
/* If the interrupt queued a callback, get out of dyntick mode. */
366
(__get_cpu_var(rcu_sched_data).nxtlist ||
367
__get_cpu_var(rcu_bh_data).nxtlist ||
368
rcu_preempt_needs_cpu(smp_processor_id())))
287
383
local_irq_save(flags);
288
384
rdtp = &__get_cpu_var(rcu_dynticks);
290
rdtp->dynticks_nesting++;
291
WARN_ON_ONCE(!(rdtp->dynticks & 0x1));
385
if (rdtp->dynticks_nesting++) {
386
local_irq_restore(flags);
389
smp_mb__before_atomic_inc(); /* Force ordering w/previous sojourn. */
390
atomic_inc(&rdtp->dynticks);
391
/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
392
smp_mb__after_atomic_inc(); /* See above. */
393
WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
292
394
local_irq_restore(flags);
293
smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
305
406
struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);
307
if (rdtp->dynticks & 0x1)
408
if (rdtp->dynticks_nmi_nesting == 0 &&
409
(atomic_read(&rdtp->dynticks) & 0x1))
309
rdtp->dynticks_nmi++;
310
WARN_ON_ONCE(!(rdtp->dynticks_nmi & 0x1));
311
smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */
411
rdtp->dynticks_nmi_nesting++;
412
smp_mb__before_atomic_inc(); /* Force delay from prior write. */
413
atomic_inc(&rdtp->dynticks);
414
/* CPUs seeing atomic_inc() must see later RCU read-side crit sects */
415
smp_mb__after_atomic_inc(); /* See above. */
416
WARN_ON_ONCE(!(atomic_read(&rdtp->dynticks) & 0x1));
323
428
struct rcu_dynticks *rdtp = &__get_cpu_var(rcu_dynticks);
325
if (rdtp->dynticks & 0x1)
430
if (rdtp->dynticks_nmi_nesting == 0 ||
431
--rdtp->dynticks_nmi_nesting != 0)
327
smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */
328
rdtp->dynticks_nmi++;
329
WARN_ON_ONCE(rdtp->dynticks_nmi & 0x1);
433
/* CPUs seeing atomic_inc() must see prior RCU read-side crit sects */
434
smp_mb__before_atomic_inc(); /* See above. */
435
atomic_inc(&rdtp->dynticks);
436
smp_mb__after_atomic_inc(); /* Force delay to next write. */
437
WARN_ON_ONCE(atomic_read(&rdtp->dynticks) & 0x1);
379
470
static int dyntick_save_progress_counter(struct rcu_data *rdp)
385
snap = rdp->dynticks->dynticks;
386
snap_nmi = rdp->dynticks->dynticks_nmi;
387
smp_mb(); /* Order sampling of snap with end of grace period. */
388
rdp->dynticks_snap = snap;
389
rdp->dynticks_nmi_snap = snap_nmi;
390
ret = ((snap & 0x1) == 0) && ((snap_nmi & 0x1) == 0);
472
rdp->dynticks_snap = atomic_add_return(0, &rdp->dynticks->dynticks);
402
482
static int rcu_implicit_dynticks_qs(struct rcu_data *rdp)
409
curr = rdp->dynticks->dynticks;
410
snap = rdp->dynticks_snap;
411
curr_nmi = rdp->dynticks->dynticks_nmi;
412
snap_nmi = rdp->dynticks_nmi_snap;
413
smp_mb(); /* force ordering with cpu entering/leaving dynticks. */
487
curr = (unsigned long)atomic_add_return(0, &rdp->dynticks->dynticks);
488
snap = (unsigned long)rdp->dynticks_snap;
416
491
* If the CPU passed through or entered a dynticks idle phase with
538
610
static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp)
541
614
struct rcu_node *rnp;
543
616
if (rcu_cpu_stall_suppress)
545
delta = jiffies - ACCESS_ONCE(rsp->jiffies_stall);
618
j = ACCESS_ONCE(jiffies);
619
js = ACCESS_ONCE(rsp->jiffies_stall);
546
620
rnp = rdp->mynode;
547
if ((ACCESS_ONCE(rnp->qsmask) & rdp->grpmask) && delta >= 0) {
621
if ((ACCESS_ONCE(rnp->qsmask) & rdp->grpmask) && ULONG_CMP_GE(j, js)) {
549
623
/* We haven't checked in, so go dump stack. */
550
624
print_cpu_stall(rsp);
552
} else if (rcu_gp_in_progress(rsp) && delta >= RCU_STALL_RAT_DELAY) {
626
} else if (rcu_gp_in_progress(rsp) &&
627
ULONG_CMP_GE(j, js + RCU_STALL_RAT_DELAY)) {
554
/* They had two time units to dump stack, so complain. */
629
/* They had a few time units to dump stack, so complain. */
555
630
print_other_cpu_stall(rsp);
587
662
atomic_notifier_chain_register(&panic_notifier_list, &rcu_panic_block);
590
#else /* #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
592
static void record_gp_stall_check_time(struct rcu_state *rsp)
596
static void check_cpu_stall(struct rcu_state *rsp, struct rcu_data *rdp)
600
void rcu_cpu_stall_reset(void)
604
static void __init check_cpu_stall_init(void)
608
#endif /* #else #ifdef CONFIG_RCU_CPU_STALL_DETECTOR */
611
666
* Update CPU-local rcu_data state to record the newly noticed grace period.
612
667
* This is used both when we started the grace period and when we notice
864
921
static void rcu_report_qs_rsp(struct rcu_state *rsp, unsigned long flags)
865
922
__releases(rcu_get_root(rsp)->lock)
924
unsigned long gp_duration;
867
926
WARN_ON_ONCE(!rcu_gp_in_progress(rsp));
929
* Ensure that all grace-period and pre-grace-period activity
930
* is seen before the assignment to rsp->completed.
932
smp_mb(); /* See above block comment. */
933
gp_duration = jiffies - rsp->gp_start;
934
if (gp_duration > rsp->gp_max)
935
rsp->gp_max = gp_duration;
868
936
rsp->completed = rsp->gpnum;
869
937
rsp->signaled = RCU_GP_IDLE;
870
938
rcu_start_gp(rsp, flags); /* releases root node's rnp->lock. */
1392
1473
static void rcu_process_callbacks(struct softirq_action *unused)
1395
* Memory references from any prior RCU read-side critical sections
1396
* executed by the interrupted code must be seen before any RCU
1397
* grace-period manipulations below.
1399
smp_mb(); /* See above block comment. */
1401
1475
__rcu_process_callbacks(&rcu_sched_state,
1402
1476
&__get_cpu_var(rcu_sched_data));
1403
1477
__rcu_process_callbacks(&rcu_bh_state, &__get_cpu_var(rcu_bh_data));
1404
1478
rcu_preempt_process_callbacks();
1407
* Memory references from any later RCU read-side critical sections
1408
* executed by the interrupted code must be seen after any RCU
1409
* grace-period manipulations above.
1411
smp_mb(); /* See above block comment. */
1413
1480
/* If we are last CPU on way to dyntick-idle mode, accelerate it. */
1414
1481
rcu_needs_cpu_flush();
1485
* Wake up the current CPU's kthread. This replaces raise_softirq()
1486
* in earlier versions of RCU. Note that because we are running on
1487
* the current CPU with interrupts disabled, the rcu_cpu_kthread_task
1488
* cannot disappear out from under us.
1490
static void invoke_rcu_callbacks(struct rcu_state *rsp, struct rcu_data *rdp)
1492
if (unlikely(!ACCESS_ONCE(rcu_scheduler_fully_active)))
1494
if (likely(!rsp->boost)) {
1495
rcu_do_batch(rsp, rdp);
1498
invoke_rcu_callbacks_kthread();
1501
static void invoke_rcu_core(void)
1503
raise_softirq(RCU_SOFTIRQ);
1418
1507
__call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu),
1419
1508
struct rcu_state *rsp)
1820
1916
unsigned long action, void *hcpu)
1822
1918
long cpu = (long)hcpu;
1919
struct rcu_data *rdp = per_cpu_ptr(rcu_state->rda, cpu);
1920
struct rcu_node *rnp = rdp->mynode;
1824
1922
switch (action) {
1825
1923
case CPU_UP_PREPARE:
1826
1924
case CPU_UP_PREPARE_FROZEN:
1827
rcu_online_cpu(cpu);
1925
rcu_prepare_cpu(cpu);
1926
rcu_prepare_kthreads(cpu);
1929
case CPU_DOWN_FAILED:
1930
rcu_node_kthread_setaffinity(rnp, -1);
1931
rcu_cpu_kthread_setrt(cpu, 1);
1933
case CPU_DOWN_PREPARE:
1934
rcu_node_kthread_setaffinity(rnp, cpu);
1935
rcu_cpu_kthread_setrt(cpu, 0);
1829
1937
case CPU_DYING:
1830
1938
case CPU_DYING_FROZEN: