← Back to branch summary

~ubuntu-branches/debian/wheezy/linux-2.6/wheezy

« back to all changes in this revision

Viewing changes to kernel/sched_rt.c

  • Committer: Bazaar Package Importer
  • Author(s): Ben Hutchings, Ben Hutchings, Aurelien Jarno, Martin Michlmayr
  • Date: 2011-04-06 13:53:30 UTC
  • mfrom: (43.1.5 sid)
  • Revision ID: james.westby@ubuntu.com-20110406135330-wjufxhd0tvn3zx4z
Tags: 2.6.38-3
http://bugs.debian.org/620124
http://bugs.debian.org/620284
http://bugs.debian.org/620204
[ Ben Hutchings ]
* [ppc64] Add to linux-tools package architectures (Closes: #620124)
* [amd64] Save cr4 to mmu_cr4_features at boot time (Closes: #620284)
* appletalk: Fix bugs introduced when removing use of BKL
* ALSA: Fix yet another race in disconnection
* cciss: Fix lost command issue
* ath9k: Fix kernel panic in AR2427
* ses: Avoid kernel panic when lun 0 is not mapped
* PCI/ACPI: Report ASPM support to BIOS if not disabled from command line

[ Aurelien Jarno ]
* rtlwifi: fix build when PCI is not enabled.

[ Martin Michlmayr ]
* rtlwifi: Eliminate udelay calls with too large values (Closes: #620204)

Show diffs side-by-side

added added

removed removed

Lines of Context:
kernel/sched_rt.c
183
183
        return ktime_to_ns(rt_rq->tg->rt_bandwidth.rt_period);
184
184
}
185
185
 
 
186
static inline void list_add_leaf_rt_rq(struct rt_rq *rt_rq)
 
187
{
 
188
        list_add_rcu(&rt_rq->leaf_rt_rq_list,
 
189
                        &rq_of_rt_rq(rt_rq)->leaf_rt_rq_list);
 
190
}
 
191
 
 
192
static inline void list_del_leaf_rt_rq(struct rt_rq *rt_rq)
 
193
{
 
194
        list_del_rcu(&rt_rq->leaf_rt_rq_list);
 
195
}
 
196
 
186
197
#define for_each_leaf_rt_rq(rt_rq, rq) \
187
198
        list_for_each_entry_rcu(rt_rq, &rq->leaf_rt_rq_list, leaf_rt_rq_list)
188
199
 
194
205
        return rt_se->my_q;
195
206
}
196
207
 
197
 
static void enqueue_rt_entity(struct sched_rt_entity *rt_se);
 
208
static void enqueue_rt_entity(struct sched_rt_entity *rt_se, bool head);
198
209
static void dequeue_rt_entity(struct sched_rt_entity *rt_se);
199
210
 
200
211
static void sched_rt_rq_enqueue(struct rt_rq *rt_rq)
201
212
{
202
213
        struct task_struct *curr = rq_of_rt_rq(rt_rq)->curr;
203
 
        struct sched_rt_entity *rt_se = rt_rq->rt_se;
 
214
        struct sched_rt_entity *rt_se;
 
215
 
 
216
        int cpu = cpu_of(rq_of_rt_rq(rt_rq));
 
217
 
 
218
        rt_se = rt_rq->tg->rt_se[cpu];
204
219
 
205
220
        if (rt_rq->rt_nr_running) {
206
221
                if (rt_se && !on_rt_rq(rt_se))
207
 
                        enqueue_rt_entity(rt_se);
 
222
                        enqueue_rt_entity(rt_se, false);
208
223
                if (rt_rq->highest_prio.curr < curr->prio)
209
224
                        resched_task(curr);
210
225
        }
212
227
 
213
228
static void sched_rt_rq_dequeue(struct rt_rq *rt_rq)
214
229
{
215
 
        struct sched_rt_entity *rt_se = rt_rq->rt_se;
 
230
        struct sched_rt_entity *rt_se;
 
231
        int cpu = cpu_of(rq_of_rt_rq(rt_rq));
 
232
 
 
233
        rt_se = rt_rq->tg->rt_se[cpu];
216
234
 
217
235
        if (rt_se && on_rt_rq(rt_se))
218
236
                dequeue_rt_entity(rt_se);
270
288
        return ktime_to_ns(def_rt_bandwidth.rt_period);
271
289
}
272
290
 
 
291
static inline void list_add_leaf_rt_rq(struct rt_rq *rt_rq)
 
292
{
 
293
}
 
294
 
 
295
static inline void list_del_leaf_rt_rq(struct rt_rq *rt_rq)
 
296
{
 
297
}
 
298
 
273
299
#define for_each_leaf_rt_rq(rt_rq, rq) \
274
300
        for (rt_rq = &rq->rt; rt_rq; rt_rq = NULL)
275
301
 
327
353
 
328
354
        weight = cpumask_weight(rd->span);
329
355
 
330
 
        spin_lock(&rt_b->rt_runtime_lock);
 
356
        raw_spin_lock(&rt_b->rt_runtime_lock);
331
357
        rt_period = ktime_to_ns(rt_b->rt_period);
332
358
        for_each_cpu(i, rd->span) {
333
359
                struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i);
336
362
                if (iter == rt_rq)
337
363
                        continue;
338
364
 
339
 
                spin_lock(&iter->rt_runtime_lock);
 
365
                raw_spin_lock(&iter->rt_runtime_lock);
340
366
                /*
341
367
                 * Either all rqs have inf runtime and there's nothing to steal
342
368
                 * or __disable_runtime() below sets a specific rq to inf to
358
384
                        rt_rq->rt_runtime += diff;
359
385
                        more = 1;
360
386
                        if (rt_rq->rt_runtime == rt_period) {
361
 
                                spin_unlock(&iter->rt_runtime_lock);
 
387
                                raw_spin_unlock(&iter->rt_runtime_lock);
362
388
                                break;
363
389
                        }
364
390
                }
365
391
next:
366
 
                spin_unlock(&iter->rt_runtime_lock);
 
392
                raw_spin_unlock(&iter->rt_runtime_lock);
367
393
        }
368
 
        spin_unlock(&rt_b->rt_runtime_lock);
 
394
        raw_spin_unlock(&rt_b->rt_runtime_lock);
369
395
 
370
396
        return more;
371
397
}
386
412
                s64 want;
387
413
                int i;
388
414
 
389
 
                spin_lock(&rt_b->rt_runtime_lock);
390
 
                spin_lock(&rt_rq->rt_runtime_lock);
 
415
                raw_spin_lock(&rt_b->rt_runtime_lock);
 
416
                raw_spin_lock(&rt_rq->rt_runtime_lock);
391
417
                /*
392
418
                 * Either we're all inf and nobody needs to borrow, or we're
393
419
                 * already disabled and thus have nothing to do, or we have
396
422
                if (rt_rq->rt_runtime == RUNTIME_INF ||
397
423
                                rt_rq->rt_runtime == rt_b->rt_runtime)
398
424
                        goto balanced;
399
 
                spin_unlock(&rt_rq->rt_runtime_lock);
 
425
                raw_spin_unlock(&rt_rq->rt_runtime_lock);
400
426
 
401
427
                /*
402
428
                 * Calculate the difference between what we started out with
418
444
                        if (iter == rt_rq || iter->rt_runtime == RUNTIME_INF)
419
445
                                continue;
420
446
 
421
 
                        spin_lock(&iter->rt_runtime_lock);
 
447
                        raw_spin_lock(&iter->rt_runtime_lock);
422
448
                        if (want > 0) {
423
449
                                diff = min_t(s64, iter->rt_runtime, want);
424
450
                                iter->rt_runtime -= diff;
427
453
                                iter->rt_runtime -= want;
428
454
                                want -= want;
429
455
                        }
430
 
                        spin_unlock(&iter->rt_runtime_lock);
 
456
                        raw_spin_unlock(&iter->rt_runtime_lock);
431
457
 
432
458
                        if (!want)
433
459
                                break;
434
460
                }
435
461
 
436
 
                spin_lock(&rt_rq->rt_runtime_lock);
 
462
                raw_spin_lock(&rt_rq->rt_runtime_lock);
437
463
                /*
438
464
                 * We cannot be left wanting - that would mean some runtime
439
465
                 * leaked out of the system.
445
471
                 * runtime - in which case borrowing doesn't make sense.
446
472
                 */
447
473
                rt_rq->rt_runtime = RUNTIME_INF;
448
 
                spin_unlock(&rt_rq->rt_runtime_lock);
449
 
                spin_unlock(&rt_b->rt_runtime_lock);
 
474
                raw_spin_unlock(&rt_rq->rt_runtime_lock);
 
475
                raw_spin_unlock(&rt_b->rt_runtime_lock);
450
476
        }
451
477
}
452
478
 
454
480
{
455
481
        unsigned long flags;
456
482
 
457
 
        spin_lock_irqsave(&rq->lock, flags);
 
483
        raw_spin_lock_irqsave(&rq->lock, flags);
458
484
        __disable_runtime(rq);
459
 
        spin_unlock_irqrestore(&rq->lock, flags);
 
485
        raw_spin_unlock_irqrestore(&rq->lock, flags);
460
486
}
461
487
 
462
488
static void __enable_runtime(struct rq *rq)
472
498
        for_each_leaf_rt_rq(rt_rq, rq) {
473
499
                struct rt_bandwidth *rt_b = sched_rt_bandwidth(rt_rq);
474
500
 
475
 
                spin_lock(&rt_b->rt_runtime_lock);
476
 
                spin_lock(&rt_rq->rt_runtime_lock);
 
501
                raw_spin_lock(&rt_b->rt_runtime_lock);
 
502
                raw_spin_lock(&rt_rq->rt_runtime_lock);
477
503
                rt_rq->rt_runtime = rt_b->rt_runtime;
478
504
                rt_rq->rt_time = 0;
479
505
                rt_rq->rt_throttled = 0;
480
 
                spin_unlock(&rt_rq->rt_runtime_lock);
481
 
                spin_unlock(&rt_b->rt_runtime_lock);
 
506
                raw_spin_unlock(&rt_rq->rt_runtime_lock);
 
507
                raw_spin_unlock(&rt_b->rt_runtime_lock);
482
508
        }
483
509
}
484
510
 
486
512
{
487
513
        unsigned long flags;
488
514
 
489
 
        spin_lock_irqsave(&rq->lock, flags);
 
515
        raw_spin_lock_irqsave(&rq->lock, flags);
490
516
        __enable_runtime(rq);
491
 
        spin_unlock_irqrestore(&rq->lock, flags);
 
517
        raw_spin_unlock_irqrestore(&rq->lock, flags);
492
518
}
493
519
 
494
520
static int balance_runtime(struct rt_rq *rt_rq)
496
522
        int more = 0;
497
523
 
498
524
        if (rt_rq->rt_time > rt_rq->rt_runtime) {
499
 
                spin_unlock(&rt_rq->rt_runtime_lock);
 
525
                raw_spin_unlock(&rt_rq->rt_runtime_lock);
500
526
                more = do_balance_runtime(rt_rq);
501
 
                spin_lock(&rt_rq->rt_runtime_lock);
 
527
                raw_spin_lock(&rt_rq->rt_runtime_lock);
502
528
        }
503
529
 
504
530
        return more;
524
550
                struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i);
525
551
                struct rq *rq = rq_of_rt_rq(rt_rq);
526
552
 
527
 
                spin_lock(&rq->lock);
 
553
                raw_spin_lock(&rq->lock);
528
554
                if (rt_rq->rt_time) {
529
555
                        u64 runtime;
530
556
 
531
 
                        spin_lock(&rt_rq->rt_runtime_lock);
 
557
                        raw_spin_lock(&rt_rq->rt_runtime_lock);
532
558
                        if (rt_rq->rt_throttled)
533
559
                                balance_runtime(rt_rq);
534
560
                        runtime = rt_rq->rt_runtime;
539
565
                        }
540
566
                        if (rt_rq->rt_time || rt_rq->rt_nr_running)
541
567
                                idle = 0;
542
 
                        spin_unlock(&rt_rq->rt_runtime_lock);
543
 
                } else if (rt_rq->rt_nr_running)
 
568
                        raw_spin_unlock(&rt_rq->rt_runtime_lock);
 
569
                } else if (rt_rq->rt_nr_running) {
544
570
                        idle = 0;
 
571
                        if (!rt_rq_throttled(rt_rq))
 
572
                                enqueue = 1;
 
573
                }
545
574
 
546
575
                if (enqueue)
547
576
                        sched_rt_rq_enqueue(rt_rq);
548
 
                spin_unlock(&rq->lock);
 
577
                raw_spin_unlock(&rq->lock);
549
578
        }
550
579
 
551
580
        return idle;
600
629
        struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
601
630
        u64 delta_exec;
602
631
 
603
 
        if (!task_has_rt_policy(curr))
 
632
        if (curr->sched_class != &rt_sched_class)
604
633
                return;
605
634
 
606
 
        delta_exec = rq->clock - curr->se.exec_start;
 
635
        delta_exec = rq->clock_task - curr->se.exec_start;
607
636
        if (unlikely((s64)delta_exec < 0))
608
637
                delta_exec = 0;
609
638
 
610
 
        schedstat_set(curr->se.exec_max, max(curr->se.exec_max, delta_exec));
 
639
        schedstat_set(curr->se.statistics.exec_max, max(curr->se.statistics.exec_max, delta_exec));
611
640
 
612
641
        curr->se.sum_exec_runtime += delta_exec;
613
642
        account_group_exec_runtime(curr, delta_exec);
614
643
 
615
 
        curr->se.exec_start = rq->clock;
 
644
        curr->se.exec_start = rq->clock_task;
616
645
        cpuacct_charge(curr, delta_exec);
617
646
 
618
647
        sched_rt_avg_update(rq, delta_exec);
624
653
                rt_rq = rt_rq_of_se(rt_se);
625
654
 
626
655
                if (sched_rt_runtime(rt_rq) != RUNTIME_INF) {
627
 
                        spin_lock(&rt_rq->rt_runtime_lock);
 
656
                        raw_spin_lock(&rt_rq->rt_runtime_lock);
628
657
                        rt_rq->rt_time += delta_exec;
629
658
                        if (sched_rt_runtime_exceeded(rt_rq))
630
659
                                resched_task(curr);
631
 
                        spin_unlock(&rt_rq->rt_runtime_lock);
 
660
                        raw_spin_unlock(&rt_rq->rt_runtime_lock);
632
661
                }
633
662
        }
634
663
}
803
832
        dec_rt_group(rt_se, rt_rq);
804
833
}
805
834
 
806
 
static void __enqueue_rt_entity(struct sched_rt_entity *rt_se)
 
835
static void __enqueue_rt_entity(struct sched_rt_entity *rt_se, bool head)
807
836
{
808
837
        struct rt_rq *rt_rq = rt_rq_of_se(rt_se);
809
838
        struct rt_prio_array *array = &rt_rq->active;
819
848
        if (group_rq && (rt_rq_throttled(group_rq) || !group_rq->rt_nr_running))
820
849
                return;
821
850
 
822
 
        list_add_tail(&rt_se->run_list, queue);
 
851
        if (!rt_rq->rt_nr_running)
 
852
                list_add_leaf_rt_rq(rt_rq);
 
853
 
 
854
        if (head)
 
855
                list_add(&rt_se->run_list, queue);
 
856
        else
 
857
                list_add_tail(&rt_se->run_list, queue);
823
858
        __set_bit(rt_se_prio(rt_se), array->bitmap);
824
859
 
825
860
        inc_rt_tasks(rt_se, rt_rq);
835
870
                __clear_bit(rt_se_prio(rt_se), array->bitmap);
836
871
 
837
872
        dec_rt_tasks(rt_se, rt_rq);
 
873
        if (!rt_rq->rt_nr_running)
 
874
                list_del_leaf_rt_rq(rt_rq);
838
875
}
839
876
 
840
877
/*
856
893
        }
857
894
}
858
895
 
859
 
static void enqueue_rt_entity(struct sched_rt_entity *rt_se)
 
896
static void enqueue_rt_entity(struct sched_rt_entity *rt_se, bool head)
860
897
{
861
898
        dequeue_rt_stack(rt_se);
862
899
        for_each_sched_rt_entity(rt_se)
863
 
                __enqueue_rt_entity(rt_se);
 
900
                __enqueue_rt_entity(rt_se, head);
864
901
}
865
902
 
866
903
static void dequeue_rt_entity(struct sched_rt_entity *rt_se)
871
908
                struct rt_rq *rt_rq = group_rt_rq(rt_se);
872
909
 
873
910
                if (rt_rq && rt_rq->rt_nr_running)
874
 
                        __enqueue_rt_entity(rt_se);
 
911
                        __enqueue_rt_entity(rt_se, false);
875
912
        }
876
913
}
877
914
 
878
915
/*
879
916
 * Adding/removing a task to/from a priority array:
880
917
 */
881
 
static void enqueue_task_rt(struct rq *rq, struct task_struct *p, int wakeup)
 
918
static void
 
919
enqueue_task_rt(struct rq *rq, struct task_struct *p, int flags)
882
920
{
883
921
        struct sched_rt_entity *rt_se = &p->rt;
884
922
 
885
 
        if (wakeup)
 
923
        if (flags & ENQUEUE_WAKEUP)
886
924
                rt_se->timeout = 0;
887
925
 
888
 
        enqueue_rt_entity(rt_se);
 
926
        enqueue_rt_entity(rt_se, flags & ENQUEUE_HEAD);
889
927
 
890
928
        if (!task_current(rq, p) && p->rt.nr_cpus_allowed > 1)
891
929
                enqueue_pushable_task(rq, p);
892
930
}
893
931
 
894
 
static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int sleep)
 
932
static void dequeue_task_rt(struct rq *rq, struct task_struct *p, int flags)
895
933
{
896
934
        struct sched_rt_entity *rt_se = &p->rt;
897
935
 
938
976
#ifdef CONFIG_SMP
939
977
static int find_lowest_rq(struct task_struct *task);
940
978
 
941
 
static int select_task_rq_rt(struct task_struct *p, int sd_flag, int flags)
 
979
static int
 
980
select_task_rq_rt(struct rq *rq, struct task_struct *p, int sd_flag, int flags)
942
981
{
943
 
        struct rq *rq = task_rq(p);
944
 
 
945
982
        if (sd_flag != SD_BALANCE_WAKE)
946
983
                return smp_processor_id();
947
984
 
951
988
         * runqueue. Otherwise simply start this RT task
952
989
         * on its current runqueue.
953
990
         *
954
 
         * We want to avoid overloading runqueues. Even if
955
 
         * the RT task is of higher priority than the current RT task.
956
 
         * RT tasks behave differently than other tasks. If
957
 
         * one gets preempted, we try to push it off to another queue.
958
 
         * So trying to keep a preempting RT task on the same
959
 
         * cache hot CPU will force the running RT task to
960
 
         * a cold CPU. So we waste all the cache for the lower
961
 
         * RT task in hopes of saving some of a RT task
962
 
         * that is just being woken and probably will have
963
 
         * cold cache anyway.
 
991
         * We want to avoid overloading runqueues. If the woken
 
992
         * task is a higher priority, then it will stay on this CPU
 
993
         * and the lower prio task should be moved to another CPU.
 
994
         * Even though this will probably make the lower prio task
 
995
         * lose its cache, we do not want to bounce a higher task
 
996
         * around just because it gave up its CPU, perhaps for a
 
997
         * lock?
 
998
         *
 
999
         * For equal prio tasks, we just let the scheduler sort it out.
964
1000
         */
965
1001
        if (unlikely(rt_task(rq->curr)) &&
 
1002
            (rq->curr->rt.nr_cpus_allowed < 2 ||
 
1003
             rq->curr->prio < p->prio) &&
966
1004
            (p->rt.nr_cpus_allowed > 1)) {
967
1005
                int cpu = find_lowest_rq(p);
968
1006
 
1065
1103
        } while (rt_rq);
1066
1104
 
1067
1105
        p = rt_task_of(rt_se);
1068
 
        p->se.exec_start = rq->clock;
 
1106
        p->se.exec_start = rq->clock_task;
1069
1107
 
1070
1108
        return p;
1071
1109
}
1130
1168
        for_each_leaf_rt_rq(rt_rq, rq) {
1131
1169
                array = &rt_rq->active;
1132
1170
                idx = sched_find_first_bit(array->bitmap);
1133
 
 next_idx:
 
1171
next_idx:
1134
1172
                if (idx >= MAX_RT_PRIO)
1135
1173
                        continue;
1136
1174
                if (next && next->prio < idx)
1137
1175
                        continue;
1138
1176
                list_for_each_entry(rt_se, array->queue + idx, run_list) {
1139
 
                        struct task_struct *p = rt_task_of(rt_se);
 
1177
                        struct task_struct *p;
 
1178
 
 
1179
                        if (!rt_entity_is_task(rt_se))
 
1180
                                continue;
 
1181
 
 
1182
                        p = rt_task_of(rt_se);
1140
1183
                        if (pick_rt_task(rq, p, cpu)) {
1141
1184
                                next = p;
1142
1185
                                break;
1153
1196
 
1154
1197
static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask);
1155
1198
 
1156
 
static inline int pick_optimal_cpu(int this_cpu,
1157
 
                                   const struct cpumask *mask)
1158
 
{
1159
 
        int first;
1160
 
 
1161
 
        /* "this_cpu" is cheaper to preempt than a remote processor */
1162
 
        if ((this_cpu != -1) && cpumask_test_cpu(this_cpu, mask))
1163
 
                return this_cpu;
1164
 
 
1165
 
        first = cpumask_first(mask);
1166
 
        if (first < nr_cpu_ids)
1167
 
                return first;
1168
 
 
1169
 
        return -1;
1170
 
}
1171
 
 
1172
1199
static int find_lowest_rq(struct task_struct *task)
1173
1200
{
1174
1201
        struct sched_domain *sd;
1175
1202
        struct cpumask *lowest_mask = __get_cpu_var(local_cpu_mask);
1176
1203
        int this_cpu = smp_processor_id();
1177
1204
        int cpu      = task_cpu(task);
1178
 
        cpumask_var_t domain_mask;
1179
1205
 
1180
1206
        if (task->rt.nr_cpus_allowed == 1)
1181
1207
                return -1; /* No other targets possible */
1198
1224
         * Otherwise, we consult the sched_domains span maps to figure
1199
1225
         * out which cpu is logically closest to our hot cache data.
1200
1226
         */
1201
 
        if (this_cpu == cpu)
1202
 
                this_cpu = -1; /* Skip this_cpu opt if the same */
1203
 
 
1204
 
        if (alloc_cpumask_var(&domain_mask, GFP_ATOMIC)) {
1205
 
                for_each_domain(cpu, sd) {
1206
 
                        if (sd->flags & SD_WAKE_AFFINE) {
1207
 
                                int best_cpu;
1208
 
 
1209
 
                                cpumask_and(domain_mask,
1210
 
                                            sched_domain_span(sd),
1211
 
                                            lowest_mask);
1212
 
 
1213
 
                                best_cpu = pick_optimal_cpu(this_cpu,
1214
 
                                                            domain_mask);
1215
 
 
1216
 
                                if (best_cpu != -1) {
1217
 
                                        free_cpumask_var(domain_mask);
1218
 
                                        return best_cpu;
1219
 
                                }
1220
 
                        }
 
1227
        if (!cpumask_test_cpu(this_cpu, lowest_mask))
 
1228
                this_cpu = -1; /* Skip this_cpu opt if not among lowest */
 
1229
 
 
1230
        for_each_domain(cpu, sd) {
 
1231
                if (sd->flags & SD_WAKE_AFFINE) {
 
1232
                        int best_cpu;
 
1233
 
 
1234
                        /*
 
1235
                         * "this_cpu" is cheaper to preempt than a
 
1236
                         * remote processor.
 
1237
                         */
 
1238
                        if (this_cpu != -1 &&
 
1239
                            cpumask_test_cpu(this_cpu, sched_domain_span(sd)))
 
1240
                                return this_cpu;
 
1241
 
 
1242
                        best_cpu = cpumask_first_and(lowest_mask,
 
1243
                                                     sched_domain_span(sd));
 
1244
                        if (best_cpu < nr_cpu_ids)
 
1245
                                return best_cpu;
1221
1246
                }
1222
 
                free_cpumask_var(domain_mask);
1223
1247
        }
1224
1248
 
1225
1249
        /*
1227
1251
         * just give the caller *something* to work with from the compatible
1228
1252
         * locations.
1229
1253
         */
1230
 
        return pick_optimal_cpu(this_cpu, lowest_mask);
 
1254
        if (this_cpu != -1)
 
1255
                return this_cpu;
 
1256
 
 
1257
        cpu = cpumask_any(lowest_mask);
 
1258
        if (cpu < nr_cpu_ids)
 
1259
                return cpu;
 
1260
        return -1;
1231
1261
}
1232
1262
 
1233
1263
/* Will lock the rq it finds */
1259
1289
                                     task_running(rq, task) ||
1260
1290
                                     !task->se.on_rq)) {
1261
1291
 
1262
 
                                spin_unlock(&lowest_rq->lock);
 
1292
                                raw_spin_unlock(&lowest_rq->lock);
1263
1293
                                lowest_rq = NULL;
1264
1294
                                break;
1265
1295
                        }
1314
1344
        if (!next_task)
1315
1345
                return 0;
1316
1346
 
1317
 
 retry:
 
1347
retry:
1318
1348
        if (unlikely(next_task == rq->curr)) {
1319
1349
                WARN_ON(1);
1320
1350
                return 0;
1462
1492
                         * but possible)
1463
1493
                         */
1464
1494
                }
1465
 
 skip:
 
1495
skip:
1466
1496
                double_unlock_balance(this_rq, src_rq);
1467
1497
        }
1468
1498
 
1485
1515
 * If we are not running and we are not going to reschedule soon, we should
1486
1516
 * try to push tasks away now
1487
1517
 */
1488
 
static void task_wake_up_rt(struct rq *rq, struct task_struct *p)
 
1518
static void task_woken_rt(struct rq *rq, struct task_struct *p)
1489
1519
{
1490
1520
        if (!task_running(rq, p) &&
1491
1521
            !test_tsk_need_resched(rq->curr) &&
1492
1522
            has_pushable_tasks(rq) &&
1493
 
            p->rt.nr_cpus_allowed > 1)
 
1523
            p->rt.nr_cpus_allowed > 1 &&
 
1524
            rt_task(rq->curr) &&
 
1525
            (rq->curr->rt.nr_cpus_allowed < 2 ||
 
1526
             rq->curr->prio < p->prio))
1494
1527
                push_rt_tasks(rq);
1495
1528
}
1496
1529
 
1497
 
static unsigned long
1498
 
load_balance_rt(struct rq *this_rq, int this_cpu, struct rq *busiest,
1499
 
                unsigned long max_load_move,
1500
 
                struct sched_domain *sd, enum cpu_idle_type idle,
1501
 
                int *all_pinned, int *this_best_prio)
1502
 
{
1503
 
        /* don't touch RT tasks */
1504
 
        return 0;
1505
 
}
1506
 
 
1507
 
static int
1508
 
move_one_task_rt(struct rq *this_rq, int this_cpu, struct rq *busiest,
1509
 
                 struct sched_domain *sd, enum cpu_idle_type idle)
1510
 
{
1511
 
        /* don't touch RT tasks */
1512
 
        return 0;
1513
 
}
1514
 
 
1515
1530
static void set_cpus_allowed_rt(struct task_struct *p,
1516
1531
                                const struct cpumask *new_mask)
1517
1532
{
1680
1695
{
1681
1696
        unsigned long soft, hard;
1682
1697
 
1683
 
        if (!p->signal)
1684
 
                return;
1685
 
 
1686
 
        soft = p->signal->rlim[RLIMIT_RTTIME].rlim_cur;
1687
 
        hard = p->signal->rlim[RLIMIT_RTTIME].rlim_max;
 
1698
        /* max may change after cur was read, this will be fixed next tick */
 
1699
        soft = task_rlimit(p, RLIMIT_RTTIME);
 
1700
        hard = task_rlimit_max(p, RLIMIT_RTTIME);
1688
1701
 
1689
1702
        if (soft != RLIM_INFINITY) {
1690
1703
                unsigned long next;
1728
1741
{
1729
1742
        struct task_struct *p = rq->curr;
1730
1743
 
1731
 
        p->se.exec_start = rq->clock;
 
1744
        p->se.exec_start = rq->clock_task;
1732
1745
 
1733
1746
        /* The running task is never eligible for pushing */
1734
1747
        dequeue_pushable_task(rq, p);
1735
1748
}
1736
1749
 
1737
 
unsigned int get_rr_interval_rt(struct task_struct *task)
 
1750
static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task)
1738
1751
{
1739
1752
        /*
1740
1753
         * Time slice is 0 for SCHED_FIFO tasks
1759
1772
#ifdef CONFIG_SMP
1760
1773
        .select_task_rq         = select_task_rq_rt,
1761
1774
 
1762
 
        .load_balance           = load_balance_rt,
1763
 
        .move_one_task          = move_one_task_rt,
1764
1775
        .set_cpus_allowed       = set_cpus_allowed_rt,
1765
1776
        .rq_online              = rq_online_rt,
1766
1777
        .rq_offline             = rq_offline_rt,
1767
1778
        .pre_schedule           = pre_schedule_rt,
1768
1779
        .post_schedule          = post_schedule_rt,
1769
 
        .task_wake_up           = task_wake_up_rt,
 
1780
        .task_woken             = task_woken_rt,
1770
1781
        .switched_from          = switched_from_rt,
1771
1782
#endif
1772
1783