← Back to branch summary

~ubuntu-branches/ubuntu/quantal/linux-linaro-mx51/quantal

« back to all changes in this revision

Viewing changes to kernel/sched_fair.c

  • Committer: Package Import Robot
  • Author(s): John Rigby, John Rigby
  • Date: 2011-09-26 10:44:23 UTC
  • Revision ID: package-import@ubuntu.com-20110926104423-3o58a3c1bj7x00rs
Tags: 3.0.0-1007.9
https://launchpad.net/bugs/826021
[ John Rigby ]

Enable crypto modules and remove crypto-modules from
exclude-module files
LP: #826021

Show diffs side-by-side

added added

removed removed

Lines of Context:
kernel/sched_fair.c
358
358
        }
359
359
 
360
360
        cfs_rq->min_vruntime = max_vruntime(cfs_rq->min_vruntime, vruntime);
 
361
#ifndef CONFIG_64BIT
 
362
        smp_wmb();
 
363
        cfs_rq->min_vruntime_copy = cfs_rq->min_vruntime;
 
364
#endif
361
365
}
362
366
 
363
367
/*
1072
1076
        se->on_rq = 0;
1073
1077
        update_cfs_load(cfs_rq, 0);
1074
1078
        account_entity_dequeue(cfs_rq, se);
1075
 
        update_min_vruntime(cfs_rq);
1076
 
        update_cfs_shares(cfs_rq);
1077
1079
 
1078
1080
        /*
1079
1081
         * Normalize the entity after updating the min_vruntime because the
1082
1084
         */
1083
1085
        if (!(flags & DEQUEUE_SLEEP))
1084
1086
                se->vruntime -= cfs_rq->min_vruntime;
 
1087
 
 
1088
        update_min_vruntime(cfs_rq);
 
1089
        update_cfs_shares(cfs_rq);
1085
1090
}
1086
1091
 
1087
1092
/*
1340
1345
        hrtick_update(rq);
1341
1346
}
1342
1347
 
 
1348
static void set_next_buddy(struct sched_entity *se);
 
1349
 
1343
1350
/*
1344
1351
 * The dequeue_task method is called before nr_running is
1345
1352
 * decreased. We remove the task from the rbtree and
1349
1356
{
1350
1357
        struct cfs_rq *cfs_rq;
1351
1358
        struct sched_entity *se = &p->se;
 
1359
        int task_sleep = flags & DEQUEUE_SLEEP;
1352
1360
 
1353
1361
        for_each_sched_entity(se) {
1354
1362
                cfs_rq = cfs_rq_of(se);
1355
1363
                dequeue_entity(cfs_rq, se, flags);
1356
1364
 
1357
1365
                /* Don't dequeue parent if it has other entities besides us */
1358
 
                if (cfs_rq->load.weight)
 
1366
                if (cfs_rq->load.weight) {
 
1367
                        /*
 
1368
                         * Bias pick_next to pick a task from this cfs_rq, as
 
1369
                         * p is sleeping when it is within its sched_slice.
 
1370
                         */
 
1371
                        if (task_sleep && parent_entity(se))
 
1372
                                set_next_buddy(parent_entity(se));
1359
1373
                        break;
 
1374
                }
1360
1375
                flags |= DEQUEUE_SLEEP;
1361
1376
        }
1362
1377
 
1372
1387
 
1373
1388
#ifdef CONFIG_SMP
1374
1389
 
1375
 
static void task_waking_fair(struct rq *rq, struct task_struct *p)
 
1390
static void task_waking_fair(struct task_struct *p)
1376
1391
{
1377
1392
        struct sched_entity *se = &p->se;
1378
1393
        struct cfs_rq *cfs_rq = cfs_rq_of(se);
1379
 
 
1380
 
        se->vruntime -= cfs_rq->min_vruntime;
 
1394
        u64 min_vruntime;
 
1395
 
 
1396
#ifndef CONFIG_64BIT
 
1397
        u64 min_vruntime_copy;
 
1398
 
 
1399
        do {
 
1400
                min_vruntime_copy = cfs_rq->min_vruntime_copy;
 
1401
                smp_rmb();
 
1402
                min_vruntime = cfs_rq->min_vruntime;
 
1403
        } while (min_vruntime != min_vruntime_copy);
 
1404
#else
 
1405
        min_vruntime = cfs_rq->min_vruntime;
 
1406
#endif
 
1407
 
 
1408
        se->vruntime -= min_vruntime;
1381
1409
}
1382
1410
 
1383
1411
#ifdef CONFIG_FAIR_GROUP_SCHED
1557
1585
                }
1558
1586
 
1559
1587
                /* Adjust by relative CPU power of the group */
1560
 
                avg_load = (avg_load * SCHED_LOAD_SCALE) / group->cpu_power;
 
1588
                avg_load = (avg_load * SCHED_POWER_SCALE) / group->sgp->power;
1561
1589
 
1562
1590
                if (local_group) {
1563
1591
                        this_load = avg_load;
1622
1650
        /*
1623
1651
         * Otherwise, iterate the domains and find an elegible idle cpu.
1624
1652
         */
 
1653
        rcu_read_lock();
1625
1654
        for_each_domain(target, sd) {
1626
1655
                if (!(sd->flags & SD_SHARE_PKG_RESOURCES))
1627
1656
                        break;
1641
1670
                    cpumask_test_cpu(prev_cpu, sched_domain_span(sd)))
1642
1671
                        break;
1643
1672
        }
 
1673
        rcu_read_unlock();
1644
1674
 
1645
1675
        return target;
1646
1676
}
1657
1687
 * preempt must be disabled.
1658
1688
 */
1659
1689
static int
1660
 
select_task_rq_fair(struct rq *rq, struct task_struct *p, int sd_flag, int wake_flags)
 
1690
select_task_rq_fair(struct task_struct *p, int sd_flag, int wake_flags)
1661
1691
{
1662
1692
        struct sched_domain *tmp, *affine_sd = NULL, *sd = NULL;
1663
1693
        int cpu = smp_processor_id();
1673
1703
                new_cpu = prev_cpu;
1674
1704
        }
1675
1705
 
 
1706
        rcu_read_lock();
1676
1707
        for_each_domain(cpu, tmp) {
1677
1708
                if (!(tmp->flags & SD_LOAD_BALANCE))
1678
1709
                        continue;
1692
1723
                                nr_running += cpu_rq(i)->cfs.nr_running;
1693
1724
                        }
1694
1725
 
1695
 
                        capacity = DIV_ROUND_CLOSEST(power, SCHED_LOAD_SCALE);
 
1726
                        capacity = DIV_ROUND_CLOSEST(power, SCHED_POWER_SCALE);
1696
1727
 
1697
1728
                        if (tmp->flags & SD_POWERSAVINGS_BALANCE)
1698
1729
                                nr_running /= 2;
1723
1754
 
1724
1755
        if (affine_sd) {
1725
1756
                if (cpu == prev_cpu || wake_affine(affine_sd, p, sync))
1726
 
                        return select_idle_sibling(p, cpu);
1727
 
                else
1728
 
                        return select_idle_sibling(p, prev_cpu);
 
1757
                        prev_cpu = cpu;
 
1758
 
 
1759
                new_cpu = select_idle_sibling(p, prev_cpu);
 
1760
                goto unlock;
1729
1761
        }
1730
1762
 
1731
1763
        while (sd) {
1766
1798
                }
1767
1799
                /* while loop will break here if sd == NULL */
1768
1800
        }
 
1801
unlock:
 
1802
        rcu_read_unlock();
1769
1803
 
1770
1804
        return new_cpu;
1771
1805
}
1789
1823
         * This is especially important for buddies when the leftmost
1790
1824
         * task is higher priority than the buddy.
1791
1825
         */
1792
 
        if (unlikely(se->load.weight != NICE_0_LOAD))
1793
 
                gran = calc_delta_fair(gran, se);
1794
 
 
1795
 
        return gran;
 
1826
        return calc_delta_fair(gran, se);
1796
1827
}
1797
1828
 
1798
1829
/*
1826
1857
 
1827
1858
static void set_last_buddy(struct sched_entity *se)
1828
1859
{
1829
 
        if (likely(task_of(se)->policy != SCHED_IDLE)) {
1830
 
                for_each_sched_entity(se)
1831
 
                        cfs_rq_of(se)->last = se;
1832
 
        }
 
1860
        if (entity_is_task(se) && unlikely(task_of(se)->policy == SCHED_IDLE))
 
1861
                return;
 
1862
 
 
1863
        for_each_sched_entity(se)
 
1864
                cfs_rq_of(se)->last = se;
1833
1865
}
1834
1866
 
1835
1867
static void set_next_buddy(struct sched_entity *se)
1836
1868
{
1837
 
        if (likely(task_of(se)->policy != SCHED_IDLE)) {
1838
 
                for_each_sched_entity(se)
1839
 
                        cfs_rq_of(se)->next = se;
1840
 
        }
 
1869
        if (entity_is_task(se) && unlikely(task_of(se)->policy == SCHED_IDLE))
 
1870
                return;
 
1871
 
 
1872
        for_each_sched_entity(se)
 
1873
                cfs_rq_of(se)->next = se;
1841
1874
}
1842
1875
 
1843
1876
static void set_skip_buddy(struct sched_entity *se)
1844
1877
{
1845
 
        if (likely(task_of(se)->policy != SCHED_IDLE)) {
1846
 
                for_each_sched_entity(se)
1847
 
                        cfs_rq_of(se)->skip = se;
1848
 
        }
 
1878
        for_each_sched_entity(se)
 
1879
                cfs_rq_of(se)->skip = se;
1849
1880
}
1850
1881
 
1851
1882
/*
1857
1888
        struct sched_entity *se = &curr->se, *pse = &p->se;
1858
1889
        struct cfs_rq *cfs_rq = task_cfs_rq(curr);
1859
1890
        int scale = cfs_rq->nr_running >= sched_nr_latency;
 
1891
        int next_buddy_marked = 0;
1860
1892
 
1861
1893
        if (unlikely(se == pse))
1862
1894
                return;
1863
1895
 
1864
 
        if (sched_feat(NEXT_BUDDY) && scale && !(wake_flags & WF_FORK))
 
1896
        if (sched_feat(NEXT_BUDDY) && scale && !(wake_flags & WF_FORK)) {
1865
1897
                set_next_buddy(pse);
 
1898
                next_buddy_marked = 1;
 
1899
        }
1866
1900
 
1867
1901
        /*
1868
1902
         * We can come here with TIF_NEED_RESCHED already set from new task
1890
1924
        update_curr(cfs_rq);
1891
1925
        find_matching_se(&se, &pse);
1892
1926
        BUG_ON(!pse);
1893
 
        if (wakeup_preempt_entity(se, pse) == 1)
 
1927
        if (wakeup_preempt_entity(se, pse) == 1) {
 
1928
                /*
 
1929
                 * Bias pick_next to pick the sched entity that is
 
1930
                 * triggering this preemption.
 
1931
                 */
 
1932
                if (!next_buddy_marked)
 
1933
                        set_next_buddy(pse);
1894
1934
                goto preempt;
 
1935
        }
1895
1936
 
1896
1937
        return;
1897
1938
 
2102
2143
balance_tasks(struct rq *this_rq, int this_cpu, struct rq *busiest,
2103
2144
              unsigned long max_load_move, struct sched_domain *sd,
2104
2145
              enum cpu_idle_type idle, int *all_pinned,
2105
 
              int *this_best_prio, struct cfs_rq *busiest_cfs_rq)
 
2146
              struct cfs_rq *busiest_cfs_rq)
2106
2147
{
2107
2148
        int loops = 0, pulled = 0;
2108
2149
        long rem_load_move = max_load_move;
2140
2181
                 */
2141
2182
                if (rem_load_move <= 0)
2142
2183
                        break;
2143
 
 
2144
 
                if (p->prio < *this_best_prio)
2145
 
                        *this_best_prio = p->prio;
2146
2184
        }
2147
2185
out:
2148
2186
        /*
2202
2240
load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
2203
2241
                  unsigned long max_load_move,
2204
2242
                  struct sched_domain *sd, enum cpu_idle_type idle,
2205
 
                  int *all_pinned, int *this_best_prio)
 
2243
                  int *all_pinned)
2206
2244
{
2207
2245
        long rem_load_move = max_load_move;
2208
2246
        int busiest_cpu = cpu_of(busiest);
2227
2265
                rem_load = div_u64(rem_load, busiest_h_load + 1);
2228
2266
 
2229
2267
                moved_load = balance_tasks(this_rq, this_cpu, busiest,
2230
 
                                rem_load, sd, idle, all_pinned, this_best_prio,
 
2268
                                rem_load, sd, idle, all_pinned,
2231
2269
                                busiest_cfs_rq);
2232
2270
 
2233
2271
                if (!moved_load)
2253
2291
load_balance_fair(struct rq *this_rq, int this_cpu, struct rq *busiest,
2254
2292
                  unsigned long max_load_move,
2255
2293
                  struct sched_domain *sd, enum cpu_idle_type idle,
2256
 
                  int *all_pinned, int *this_best_prio)
 
2294
                  int *all_pinned)
2257
2295
{
2258
2296
        return balance_tasks(this_rq, this_cpu, busiest,
2259
2297
                        max_load_move, sd, idle, all_pinned,
2260
 
                        this_best_prio, &busiest->cfs);
 
2298
                        &busiest->cfs);
2261
2299
}
2262
2300
#endif
2263
2301
 
2274
2312
                      int *all_pinned)
2275
2313
{
2276
2314
        unsigned long total_load_moved = 0, load_moved;
2277
 
        int this_best_prio = this_rq->curr->prio;
2278
2315
 
2279
2316
        do {
2280
2317
                load_moved = load_balance_fair(this_rq, this_cpu, busiest,
2281
2318
                                max_load_move - total_load_moved,
2282
 
                                sd, idle, all_pinned, &this_best_prio);
 
2319
                                sd, idle, all_pinned);
2283
2320
 
2284
2321
                total_load_moved += load_moved;
2285
2322
 
2534
2571
 
2535
2572
unsigned long default_scale_freq_power(struct sched_domain *sd, int cpu)
2536
2573
{
2537
 
        return SCHED_LOAD_SCALE;
 
2574
        return SCHED_POWER_SCALE;
2538
2575
}
2539
2576
 
2540
2577
unsigned long __weak arch_scale_freq_power(struct sched_domain *sd, int cpu)
2571
2608
                available = total - rq->rt_avg;
2572
2609
        }
2573
2610
 
2574
 
        if (unlikely((s64)total < SCHED_LOAD_SCALE))
2575
 
                total = SCHED_LOAD_SCALE;
 
2611
        if (unlikely((s64)total < SCHED_POWER_SCALE))
 
2612
                total = SCHED_POWER_SCALE;
2576
2613
 
2577
 
        total >>= SCHED_LOAD_SHIFT;
 
2614
        total >>= SCHED_POWER_SHIFT;
2578
2615
 
2579
2616
        return div_u64(available, total);
2580
2617
}
2582
2619
static void update_cpu_power(struct sched_domain *sd, int cpu)
2583
2620
{
2584
2621
        unsigned long weight = sd->span_weight;
2585
 
        unsigned long power = SCHED_LOAD_SCALE;
 
2622
        unsigned long power = SCHED_POWER_SCALE;
2586
2623
        struct sched_group *sdg = sd->groups;
2587
2624
 
2588
2625
        if ((sd->flags & SD_SHARE_CPUPOWER) && weight > 1) {
2591
2628
                else
2592
2629
                        power *= default_scale_smt_power(sd, cpu);
2593
2630
 
2594
 
                power >>= SCHED_LOAD_SHIFT;
 
2631
                power >>= SCHED_POWER_SHIFT;
2595
2632
        }
2596
2633
 
2597
 
        sdg->cpu_power_orig = power;
 
2634
        sdg->sgp->power_orig = power;
2598
2635
 
2599
2636
        if (sched_feat(ARCH_POWER))
2600
2637
                power *= arch_scale_freq_power(sd, cpu);
2601
2638
        else
2602
2639
                power *= default_scale_freq_power(sd, cpu);
2603
2640
 
2604
 
        power >>= SCHED_LOAD_SHIFT;
 
2641
        power >>= SCHED_POWER_SHIFT;
2605
2642
 
2606
2643
        power *= scale_rt_power(cpu);
2607
 
        power >>= SCHED_LOAD_SHIFT;
 
2644
        power >>= SCHED_POWER_SHIFT;
2608
2645
 
2609
2646
        if (!power)
2610
2647
                power = 1;
2611
2648
 
2612
2649
        cpu_rq(cpu)->cpu_power = power;
2613
 
        sdg->cpu_power = power;
 
2650
        sdg->sgp->power = power;
2614
2651
}
2615
2652
 
2616
2653
static void update_group_power(struct sched_domain *sd, int cpu)
2628
2665
 
2629
2666
        group = child->groups;
2630
2667
        do {
2631
 
                power += group->cpu_power;
 
2668
                power += group->sgp->power;
2632
2669
                group = group->next;
2633
2670
        } while (group != child->groups);
2634
2671
 
2635
 
        sdg->cpu_power = power;
 
2672
        sdg->sgp->power = power;
2636
2673
}
2637
2674
 
2638
2675
/*
2646
2683
fix_small_capacity(struct sched_domain *sd, struct sched_group *group)
2647
2684
{
2648
2685
        /*
2649
 
         * Only siblings can have significantly less than SCHED_LOAD_SCALE
 
2686
         * Only siblings can have significantly less than SCHED_POWER_SCALE
2650
2687
         */
2651
 
        if (sd->level != SD_LV_SIBLING)
 
2688
        if (!(sd->flags & SD_SHARE_CPUPOWER))
2652
2689
                return 0;
2653
2690
 
2654
2691
        /*
2655
2692
         * If ~90% of the cpu_power is still there, we're good.
2656
2693
         */
2657
 
        if (group->cpu_power * 32 > group->cpu_power_orig * 29)
 
2694
        if (group->sgp->power * 32 > group->sgp->power_orig * 29)
2658
2695
                return 1;
2659
2696
 
2660
2697
        return 0;
2734
2771
        }
2735
2772
 
2736
2773
        /* Adjust by relative CPU power of the group */
2737
 
        sgs->avg_load = (sgs->group_load * SCHED_LOAD_SCALE) / group->cpu_power;
 
2774
        sgs->avg_load = (sgs->group_load*SCHED_POWER_SCALE) / group->sgp->power;
2738
2775
 
2739
2776
        /*
2740
2777
         * Consider the group unbalanced when the imbalance is larger
2751
2788
        if ((max_cpu_load - min_cpu_load) >= avg_load_per_task && max_nr_running > 1)
2752
2789
                sgs->group_imb = 1;
2753
2790
 
2754
 
        sgs->group_capacity = DIV_ROUND_CLOSEST(group->cpu_power, SCHED_LOAD_SCALE);
 
2791
        sgs->group_capacity = DIV_ROUND_CLOSEST(group->sgp->power,
 
2792
                                                SCHED_POWER_SCALE);
2755
2793
        if (!sgs->group_capacity)
2756
2794
                sgs->group_capacity = fix_small_capacity(sd, group);
2757
2795
        sgs->group_weight = group->group_weight;
2839
2877
                        return;
2840
2878
 
2841
2879
                sds->total_load += sgs.group_load;
2842
 
                sds->total_pwr += sg->cpu_power;
 
2880
                sds->total_pwr += sg->sgp->power;
2843
2881
 
2844
2882
                /*
2845
2883
                 * In case the child domain prefers tasks go to siblings
2924
2962
        if (this_cpu > busiest_cpu)
2925
2963
                return 0;
2926
2964
 
2927
 
        *imbalance = DIV_ROUND_CLOSEST(sds->max_load * sds->busiest->cpu_power,
2928
 
                                       SCHED_LOAD_SCALE);
 
2965
        *imbalance = DIV_ROUND_CLOSEST(sds->max_load * sds->busiest->sgp->power,
 
2966
                                       SCHED_POWER_SCALE);
2929
2967
        return 1;
2930
2968
}
2931
2969
 
2954
2992
                        cpu_avg_load_per_task(this_cpu);
2955
2993
 
2956
2994
        scaled_busy_load_per_task = sds->busiest_load_per_task
2957
 
                                                 * SCHED_LOAD_SCALE;
2958
 
        scaled_busy_load_per_task /= sds->busiest->cpu_power;
 
2995
                                         * SCHED_POWER_SCALE;
 
2996
        scaled_busy_load_per_task /= sds->busiest->sgp->power;
2959
2997
 
2960
2998
        if (sds->max_load - sds->this_load + scaled_busy_load_per_task >=
2961
2999
                        (scaled_busy_load_per_task * imbn)) {
2969
3007
         * moving them.
2970
3008
         */
2971
3009
 
2972
 
        pwr_now += sds->busiest->cpu_power *
 
3010
        pwr_now += sds->busiest->sgp->power *
2973
3011
                        min(sds->busiest_load_per_task, sds->max_load);
2974
 
        pwr_now += sds->this->cpu_power *
 
3012
        pwr_now += sds->this->sgp->power *
2975
3013
                        min(sds->this_load_per_task, sds->this_load);
2976
 
        pwr_now /= SCHED_LOAD_SCALE;
 
3014
        pwr_now /= SCHED_POWER_SCALE;
2977
3015
 
2978
3016
        /* Amount of load we'd subtract */
2979
 
        tmp = (sds->busiest_load_per_task * SCHED_LOAD_SCALE) /
2980
 
                sds->busiest->cpu_power;
 
3017
        tmp = (sds->busiest_load_per_task * SCHED_POWER_SCALE) /
 
3018
                sds->busiest->sgp->power;
2981
3019
        if (sds->max_load > tmp)
2982
 
                pwr_move += sds->busiest->cpu_power *
 
3020
                pwr_move += sds->busiest->sgp->power *
2983
3021
                        min(sds->busiest_load_per_task, sds->max_load - tmp);
2984
3022
 
2985
3023
        /* Amount of load we'd add */
2986
 
        if (sds->max_load * sds->busiest->cpu_power <
2987
 
                sds->busiest_load_per_task * SCHED_LOAD_SCALE)
2988
 
                tmp = (sds->max_load * sds->busiest->cpu_power) /
2989
 
                        sds->this->cpu_power;
 
3024
        if (sds->max_load * sds->busiest->sgp->power <
 
3025
                sds->busiest_load_per_task * SCHED_POWER_SCALE)
 
3026
                tmp = (sds->max_load * sds->busiest->sgp->power) /
 
3027
                        sds->this->sgp->power;
2990
3028
        else
2991
 
                tmp = (sds->busiest_load_per_task * SCHED_LOAD_SCALE) /
2992
 
                        sds->this->cpu_power;
2993
 
        pwr_move += sds->this->cpu_power *
 
3029
                tmp = (sds->busiest_load_per_task * SCHED_POWER_SCALE) /
 
3030
                        sds->this->sgp->power;
 
3031
        pwr_move += sds->this->sgp->power *
2994
3032
                        min(sds->this_load_per_task, sds->this_load + tmp);
2995
 
        pwr_move /= SCHED_LOAD_SCALE;
 
3033
        pwr_move /= SCHED_POWER_SCALE;
2996
3034
 
2997
3035
        /* Move if we gain throughput */
2998
3036
        if (pwr_move > pwr_now)
3034
3072
                load_above_capacity = (sds->busiest_nr_running -
3035
3073
                                                sds->busiest_group_capacity);
3036
3074
 
3037
 
                load_above_capacity *= (SCHED_LOAD_SCALE * SCHED_LOAD_SCALE);
 
3075
                load_above_capacity *= (SCHED_LOAD_SCALE * SCHED_POWER_SCALE);
3038
3076
 
3039
 
                load_above_capacity /= sds->busiest->cpu_power;
 
3077
                load_above_capacity /= sds->busiest->sgp->power;
3040
3078
        }
3041
3079
 
3042
3080
        /*
3052
3090
        max_pull = min(sds->max_load - sds->avg_load, load_above_capacity);
3053
3091
 
3054
3092
        /* How much load to actually move to equalise the imbalance */
3055
 
        *imbalance = min(max_pull * sds->busiest->cpu_power,
3056
 
                (sds->avg_load - sds->this_load) * sds->this->cpu_power)
3057
 
                        / SCHED_LOAD_SCALE;
 
3093
        *imbalance = min(max_pull * sds->busiest->sgp->power,
 
3094
                (sds->avg_load - sds->this_load) * sds->this->sgp->power)
 
3095
                        / SCHED_POWER_SCALE;
3058
3096
 
3059
3097
        /*
3060
3098
         * if *imbalance is less than the average load per runnable task
3123
3161
        if (!sds.busiest || sds.busiest_nr_running == 0)
3124
3162
                goto out_balanced;
3125
3163
 
3126
 
        sds.avg_load = (SCHED_LOAD_SCALE * sds.total_load) / sds.total_pwr;
 
3164
        sds.avg_load = (SCHED_POWER_SCALE * sds.total_load) / sds.total_pwr;
3127
3165
 
3128
3166
        /*
3129
3167
         * If the busiest group is imbalanced the below checks don't
3202
3240
 
3203
3241
        for_each_cpu(i, sched_group_cpus(group)) {
3204
3242
                unsigned long power = power_of(i);
3205
 
                unsigned long capacity = DIV_ROUND_CLOSEST(power, SCHED_LOAD_SCALE);
 
3243
                unsigned long capacity = DIV_ROUND_CLOSEST(power,
 
3244
                                                           SCHED_POWER_SCALE);
3206
3245
                unsigned long wl;
3207
3246
 
3208
3247
                if (!capacity)
3227
3266
                 * the load can be moved away from the cpu that is potentially
3228
3267
                 * running at a lower capacity.
3229
3268
                 */
3230
 
                wl = (wl * SCHED_LOAD_SCALE) / power;
 
3269
                wl = (wl * SCHED_POWER_SCALE) / power;
3231
3270
 
3232
3271
                if (wl > max_load) {
3233
3272
                        max_load = wl;
3465
3504
        raw_spin_unlock(&this_rq->lock);
3466
3505
 
3467
3506
        update_shares(this_cpu);
 
3507
        rcu_read_lock();
3468
3508
        for_each_domain(this_cpu, sd) {
3469
3509
                unsigned long interval;
3470
3510
                int balance = 1;
3486
3526
                        break;
3487
3527
                }
3488
3528
        }
 
3529
        rcu_read_unlock();
3489
3530
 
3490
3531
        raw_spin_lock(&this_rq->lock);
3491
3532
 
3534
3575
        double_lock_balance(busiest_rq, target_rq);
3535
3576
 
3536
3577
        /* Search for an sd spanning us and the target CPU. */
 
3578
        rcu_read_lock();
3537
3579
        for_each_domain(target_cpu, sd) {
3538
3580
                if ((sd->flags & SD_LOAD_BALANCE) &&
3539
3581
                    cpumask_test_cpu(busiest_cpu, sched_domain_span(sd)))
3549
3591
                else
3550
3592
                        schedstat_inc(sd, alb_failed);
3551
3593
        }
 
3594
        rcu_read_unlock();
3552
3595
        double_unlock_balance(busiest_rq, target_rq);
3553
3596
out_unlock:
3554
3597
        busiest_rq->active_balance = 0;
3675
3718
{
3676
3719
        struct sched_domain *sd;
3677
3720
        struct sched_group *ilb_group;
 
3721
        int ilb = nr_cpu_ids;
3678
3722
 
3679
3723
        /*
3680
3724
         * Have idle load balancer selection from semi-idle packages only
3690
3734
        if (cpumask_weight(nohz.idle_cpus_mask) < 2)
3691
3735
                goto out_done;
3692
3736
 
 
3737
        rcu_read_lock();
3693
3738
        for_each_flag_domain(cpu, sd, SD_POWERSAVINGS_BALANCE) {
3694
3739
                ilb_group = sd->groups;
3695
3740
 
3696
3741
                do {
3697
 
                        if (is_semi_idle_group(ilb_group))
3698
 
                                return cpumask_first(nohz.grp_idle_mask);
 
3742
                        if (is_semi_idle_group(ilb_group)) {
 
3743
                                ilb = cpumask_first(nohz.grp_idle_mask);
 
3744
                                goto unlock;
 
3745
                        }
3699
3746
 
3700
3747
                        ilb_group = ilb_group->next;
3701
3748
 
3702
3749
                } while (ilb_group != sd->groups);
3703
3750
        }
 
3751
unlock:
 
3752
        rcu_read_unlock();
3704
3753
 
3705
3754
out_done:
3706
 
        return nr_cpu_ids;
 
3755
        return ilb;
3707
3756
}
3708
3757
#else /*  (CONFIG_SCHED_MC || CONFIG_SCHED_SMT) */
3709
3758
static inline int find_new_ilb(int call_cpu)
3848
3897
 
3849
3898
        update_shares(cpu);
3850
3899
 
 
3900
        rcu_read_lock();
3851
3901
        for_each_domain(cpu, sd) {
3852
3902
                if (!(sd->flags & SD_LOAD_BALANCE))
3853
3903
                        continue;
3893
3943
                if (!balance)
3894
3944
                        break;
3895
3945
        }
 
3946
        rcu_read_unlock();
3896
3947
 
3897
3948
        /*
3898
3949
         * next_balance will be updated only when there is a need.