2
* Copyright (c) Red Hat Inc.
4
* Permission is hereby granted, free of charge, to any person obtaining a
5
* copy of this software and associated documentation files (the "Software"),
6
* to deal in the Software without restriction, including without limitation
7
* the rights to use, copy, modify, merge, publish, distribute, sub license,
8
* and/or sell copies of the Software, and to permit persons to whom the
9
* Software is furnished to do so, subject to the following conditions:
11
* The above copyright notice and this permission notice (including the
12
* next paragraph) shall be included in all copies or substantial portions
15
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
* FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21
* DEALINGS IN THE SOFTWARE.
23
* Authors: Dave Airlie <airlied@redhat.com>
24
* Jerome Glisse <jglisse@redhat.com>
25
* Pauli Nieminen <suokkos@gmail.com>
28
/* simple list based uncached page pool
29
* - Pool collects resently freed pages for reuse
30
* - Use page->lru to keep a free list
31
* - doesn't track currently in use pages
33
#include <linux/list.h>
34
#include <linux/spinlock.h>
35
#include <linux/highmem.h>
36
#include <linux/mm_types.h>
37
#include <linux/module.h>
39
#include <linux/seq_file.h> /* for seq_printf */
40
#include <linux/slab.h>
41
#include <linux/dma-mapping.h>
43
#include <linux/atomic.h>
45
#include "ttm/ttm_bo_driver.h"
46
#include "ttm/ttm_page_alloc.h"
52
#define NUM_PAGES_TO_ALLOC (PAGE_SIZE/sizeof(struct page *))
53
#define SMALL_ALLOCATION 16
54
#define FREE_ALL_PAGES (~0U)
55
/* times are in msecs */
56
#define PAGE_FREE_INTERVAL 1000
59
* struct ttm_page_pool - Pool to reuse recently allocated uc/wc pages.
61
* @lock: Protects the shared pool from concurrnet access. Must be used with
62
* irqsave/irqrestore variants because pool allocator maybe called from
64
* @fill_lock: Prevent concurrent calls to fill.
65
* @list: Pool of free uc/wc pages for fast reuse.
66
* @gfp_flags: Flags to pass for alloc_page.
67
* @npages: Number of pages in pool.
69
struct ttm_page_pool {
72
struct list_head list;
77
unsigned long nrefills;
81
* Limits for the pool. They are handled without locks because only place where
82
* they may change is in sysfs store. They won't have immediate effect anyway
83
* so forcing serialization to access them is pointless.
86
struct ttm_pool_opts {
95
* struct ttm_pool_manager - Holds memory pools for fst allocation
97
* Manager is read only object for pool code so it doesn't need locking.
99
* @free_interval: minimum number of jiffies between freeing pages from pool.
100
* @page_alloc_inited: reference counting for pool allocation.
101
* @work: Work that is used to shrink the pool. Work is only run when there is
102
* some pages to free.
103
* @small_allocation: Limit in number of pages what is small allocation.
105
* @pools: All pool objects in use.
107
struct ttm_pool_manager {
109
struct shrinker mm_shrink;
110
struct ttm_pool_opts options;
113
struct ttm_page_pool pools[NUM_POOLS];
115
struct ttm_page_pool wc_pool;
116
struct ttm_page_pool uc_pool;
117
struct ttm_page_pool wc_pool_dma32;
118
struct ttm_page_pool uc_pool_dma32;
123
static struct attribute ttm_page_pool_max = {
124
.name = "pool_max_size",
125
.mode = S_IRUGO | S_IWUSR
127
static struct attribute ttm_page_pool_small = {
128
.name = "pool_small_allocation",
129
.mode = S_IRUGO | S_IWUSR
131
static struct attribute ttm_page_pool_alloc_size = {
132
.name = "pool_allocation_size",
133
.mode = S_IRUGO | S_IWUSR
136
static struct attribute *ttm_pool_attrs[] = {
138
&ttm_page_pool_small,
139
&ttm_page_pool_alloc_size,
143
static void ttm_pool_kobj_release(struct kobject *kobj)
145
struct ttm_pool_manager *m =
146
container_of(kobj, struct ttm_pool_manager, kobj);
150
static ssize_t ttm_pool_store(struct kobject *kobj,
151
struct attribute *attr, const char *buffer, size_t size)
153
struct ttm_pool_manager *m =
154
container_of(kobj, struct ttm_pool_manager, kobj);
157
chars = sscanf(buffer, "%u", &val);
161
/* Convert kb to number of pages */
162
val = val / (PAGE_SIZE >> 10);
164
if (attr == &ttm_page_pool_max)
165
m->options.max_size = val;
166
else if (attr == &ttm_page_pool_small)
167
m->options.small = val;
168
else if (attr == &ttm_page_pool_alloc_size) {
169
if (val > NUM_PAGES_TO_ALLOC*8) {
170
printk(KERN_ERR TTM_PFX
171
"Setting allocation size to %lu "
172
"is not allowed. Recommended size is "
174
NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 7),
175
NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
177
} else if (val > NUM_PAGES_TO_ALLOC) {
178
printk(KERN_WARNING TTM_PFX
179
"Setting allocation size to "
180
"larger than %lu is not recommended.\n",
181
NUM_PAGES_TO_ALLOC*(PAGE_SIZE >> 10));
183
m->options.alloc_size = val;
189
static ssize_t ttm_pool_show(struct kobject *kobj,
190
struct attribute *attr, char *buffer)
192
struct ttm_pool_manager *m =
193
container_of(kobj, struct ttm_pool_manager, kobj);
196
if (attr == &ttm_page_pool_max)
197
val = m->options.max_size;
198
else if (attr == &ttm_page_pool_small)
199
val = m->options.small;
200
else if (attr == &ttm_page_pool_alloc_size)
201
val = m->options.alloc_size;
203
val = val * (PAGE_SIZE >> 10);
205
return snprintf(buffer, PAGE_SIZE, "%u\n", val);
208
static const struct sysfs_ops ttm_pool_sysfs_ops = {
209
.show = &ttm_pool_show,
210
.store = &ttm_pool_store,
213
static struct kobj_type ttm_pool_kobj_type = {
214
.release = &ttm_pool_kobj_release,
215
.sysfs_ops = &ttm_pool_sysfs_ops,
216
.default_attrs = ttm_pool_attrs,
219
static struct ttm_pool_manager *_manager;
222
static int set_pages_array_wb(struct page **pages, int addrinarray)
227
for (i = 0; i < addrinarray; i++)
228
unmap_page_from_agp(pages[i]);
233
static int set_pages_array_wc(struct page **pages, int addrinarray)
238
for (i = 0; i < addrinarray; i++)
239
map_page_into_agp(pages[i]);
244
static int set_pages_array_uc(struct page **pages, int addrinarray)
249
for (i = 0; i < addrinarray; i++)
250
map_page_into_agp(pages[i]);
257
* Select the right pool or requested caching state and ttm flags. */
258
static struct ttm_page_pool *ttm_get_pool(int flags,
259
enum ttm_caching_state cstate)
263
if (cstate == tt_cached)
271
if (flags & TTM_PAGE_FLAG_DMA32)
274
return &_manager->pools[pool_index];
277
/* set memory back to wb and free the pages. */
278
static void ttm_pages_put(struct page *pages[], unsigned npages)
281
if (set_pages_array_wb(pages, npages))
282
printk(KERN_ERR TTM_PFX "Failed to set %d pages to wb!\n",
284
for (i = 0; i < npages; ++i)
285
__free_page(pages[i]);
288
static void ttm_pool_update_free_locked(struct ttm_page_pool *pool,
289
unsigned freed_pages)
291
pool->npages -= freed_pages;
292
pool->nfrees += freed_pages;
296
* Free pages from pool.
298
* To prevent hogging the ttm_swap process we only free NUM_PAGES_TO_ALLOC
299
* number of pages in one go.
301
* @pool: to free the pages from
302
* @free_all: If set to true will free all pages in pool
304
static int ttm_page_pool_free(struct ttm_page_pool *pool, unsigned nr_free)
306
unsigned long irq_flags;
308
struct page **pages_to_free;
309
unsigned freed_pages = 0,
310
npages_to_free = nr_free;
312
if (NUM_PAGES_TO_ALLOC < nr_free)
313
npages_to_free = NUM_PAGES_TO_ALLOC;
315
pages_to_free = kmalloc(npages_to_free * sizeof(struct page *),
317
if (!pages_to_free) {
318
printk(KERN_ERR TTM_PFX
319
"Failed to allocate memory for pool free operation.\n");
324
spin_lock_irqsave(&pool->lock, irq_flags);
326
list_for_each_entry_reverse(p, &pool->list, lru) {
327
if (freed_pages >= npages_to_free)
330
pages_to_free[freed_pages++] = p;
331
/* We can only remove NUM_PAGES_TO_ALLOC at a time. */
332
if (freed_pages >= NUM_PAGES_TO_ALLOC) {
333
/* remove range of pages from the pool */
334
__list_del(p->lru.prev, &pool->list);
336
ttm_pool_update_free_locked(pool, freed_pages);
338
* Because changing page caching is costly
339
* we unlock the pool to prevent stalling.
341
spin_unlock_irqrestore(&pool->lock, irq_flags);
343
ttm_pages_put(pages_to_free, freed_pages);
344
if (likely(nr_free != FREE_ALL_PAGES))
345
nr_free -= freed_pages;
347
if (NUM_PAGES_TO_ALLOC >= nr_free)
348
npages_to_free = nr_free;
350
npages_to_free = NUM_PAGES_TO_ALLOC;
354
/* free all so restart the processing */
358
/* Not allowed to fall through or break because
359
* following context is inside spinlock while we are
367
/* remove range of pages from the pool */
369
__list_del(&p->lru, &pool->list);
371
ttm_pool_update_free_locked(pool, freed_pages);
372
nr_free -= freed_pages;
375
spin_unlock_irqrestore(&pool->lock, irq_flags);
378
ttm_pages_put(pages_to_free, freed_pages);
380
kfree(pages_to_free);
384
/* Get good estimation how many pages are free in pools */
385
static int ttm_pool_get_num_unused_pages(void)
389
for (i = 0; i < NUM_POOLS; ++i)
390
total += _manager->pools[i].npages;
396
* Callback for mm to request pool to reduce number of page held.
398
static int ttm_pool_mm_shrink(struct shrinker *shrink,
399
struct shrink_control *sc)
401
static atomic_t start_pool = ATOMIC_INIT(0);
403
unsigned pool_offset = atomic_add_return(1, &start_pool);
404
struct ttm_page_pool *pool;
405
int shrink_pages = sc->nr_to_scan;
407
pool_offset = pool_offset % NUM_POOLS;
408
/* select start pool in round robin fashion */
409
for (i = 0; i < NUM_POOLS; ++i) {
410
unsigned nr_free = shrink_pages;
411
if (shrink_pages == 0)
413
pool = &_manager->pools[(i + pool_offset)%NUM_POOLS];
414
shrink_pages = ttm_page_pool_free(pool, nr_free);
416
/* return estimated number of unused pages in pool */
417
return ttm_pool_get_num_unused_pages();
420
static void ttm_pool_mm_shrink_init(struct ttm_pool_manager *manager)
422
manager->mm_shrink.shrink = &ttm_pool_mm_shrink;
423
manager->mm_shrink.seeks = 1;
424
register_shrinker(&manager->mm_shrink);
427
static void ttm_pool_mm_shrink_fini(struct ttm_pool_manager *manager)
429
unregister_shrinker(&manager->mm_shrink);
432
static int ttm_set_pages_caching(struct page **pages,
433
enum ttm_caching_state cstate, unsigned cpages)
436
/* Set page caching */
439
r = set_pages_array_uc(pages, cpages);
441
printk(KERN_ERR TTM_PFX
442
"Failed to set %d pages to uc!\n",
446
r = set_pages_array_wc(pages, cpages);
448
printk(KERN_ERR TTM_PFX
449
"Failed to set %d pages to wc!\n",
459
* Free pages the pages that failed to change the caching state. If there is
460
* any pages that have changed their caching state already put them to the
463
static void ttm_handle_caching_state_failure(struct list_head *pages,
464
int ttm_flags, enum ttm_caching_state cstate,
465
struct page **failed_pages, unsigned cpages)
468
/* Failed pages have to be freed */
469
for (i = 0; i < cpages; ++i) {
470
list_del(&failed_pages[i]->lru);
471
__free_page(failed_pages[i]);
476
* Allocate new pages with correct caching.
478
* This function is reentrant if caller updates count depending on number of
479
* pages returned in pages array.
481
static int ttm_alloc_new_pages(struct list_head *pages, gfp_t gfp_flags,
482
int ttm_flags, enum ttm_caching_state cstate, unsigned count)
484
struct page **caching_array;
488
unsigned max_cpages = min(count,
489
(unsigned)(PAGE_SIZE/sizeof(struct page *)));
491
/* allocate array for page caching change */
492
caching_array = kmalloc(max_cpages*sizeof(struct page *), GFP_KERNEL);
494
if (!caching_array) {
495
printk(KERN_ERR TTM_PFX
496
"Unable to allocate table for new pages.");
500
for (i = 0, cpages = 0; i < count; ++i) {
501
p = alloc_page(gfp_flags);
504
printk(KERN_ERR TTM_PFX "Unable to get page %u.\n", i);
506
/* store already allocated pages in the pool after
507
* setting the caching state */
509
r = ttm_set_pages_caching(caching_array,
512
ttm_handle_caching_state_failure(pages,
514
caching_array, cpages);
520
#ifdef CONFIG_HIGHMEM
521
/* gfp flags of highmem page should never be dma32 so we
522
* we should be fine in such case
527
caching_array[cpages++] = p;
528
if (cpages == max_cpages) {
530
r = ttm_set_pages_caching(caching_array,
533
ttm_handle_caching_state_failure(pages,
535
caching_array, cpages);
542
list_add(&p->lru, pages);
546
r = ttm_set_pages_caching(caching_array, cstate, cpages);
548
ttm_handle_caching_state_failure(pages,
550
caching_array, cpages);
553
kfree(caching_array);
559
* Fill the given pool if there aren't enough pages and the requested number of
562
static void ttm_page_pool_fill_locked(struct ttm_page_pool *pool,
563
int ttm_flags, enum ttm_caching_state cstate, unsigned count,
564
unsigned long *irq_flags)
570
* Only allow one pool fill operation at a time.
571
* If pool doesn't have enough pages for the allocation new pages are
572
* allocated from outside of pool.
577
pool->fill_lock = true;
579
/* If allocation request is small and there are not enough
580
* pages in a pool we fill the pool up first. */
581
if (count < _manager->options.small
582
&& count > pool->npages) {
583
struct list_head new_pages;
584
unsigned alloc_size = _manager->options.alloc_size;
587
* Can't change page caching if in irqsave context. We have to
588
* drop the pool->lock.
590
spin_unlock_irqrestore(&pool->lock, *irq_flags);
592
INIT_LIST_HEAD(&new_pages);
593
r = ttm_alloc_new_pages(&new_pages, pool->gfp_flags, ttm_flags,
595
spin_lock_irqsave(&pool->lock, *irq_flags);
598
list_splice(&new_pages, &pool->list);
600
pool->npages += alloc_size;
602
printk(KERN_ERR TTM_PFX
603
"Failed to fill pool (%p).", pool);
604
/* If we have any pages left put them to the pool. */
605
list_for_each_entry(p, &pool->list, lru) {
608
list_splice(&new_pages, &pool->list);
609
pool->npages += cpages;
613
pool->fill_lock = false;
617
* Cut 'count' number of pages from the pool and put them on the return list.
619
* @return count of pages still required to fulfill the request.
621
static unsigned ttm_page_pool_get_pages(struct ttm_page_pool *pool,
622
struct list_head *pages, int ttm_flags,
623
enum ttm_caching_state cstate, unsigned count)
625
unsigned long irq_flags;
629
spin_lock_irqsave(&pool->lock, irq_flags);
630
ttm_page_pool_fill_locked(pool, ttm_flags, cstate, count, &irq_flags);
632
if (count >= pool->npages) {
633
/* take all pages from the pool */
634
list_splice_init(&pool->list, pages);
635
count -= pool->npages;
639
/* find the last pages to include for requested number of pages. Split
640
* pool to begin and halve it to reduce search space. */
641
if (count <= pool->npages/2) {
643
list_for_each(p, &pool->list) {
648
i = pool->npages + 1;
649
list_for_each_prev(p, &pool->list) {
654
/* Cut 'count' number of pages from the pool */
655
list_cut_position(pages, &pool->list, p);
656
pool->npages -= count;
659
spin_unlock_irqrestore(&pool->lock, irq_flags);
664
* On success pages list will hold count number of correctly
667
int ttm_get_pages(struct list_head *pages, int flags,
668
enum ttm_caching_state cstate, unsigned count,
669
dma_addr_t *dma_address)
671
struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
672
struct page *p = NULL;
673
gfp_t gfp_flags = GFP_USER;
676
/* set zero flag for page allocation if required */
677
if (flags & TTM_PAGE_FLAG_ZERO_ALLOC)
678
gfp_flags |= __GFP_ZERO;
680
/* No pool for cached pages */
682
if (flags & TTM_PAGE_FLAG_DMA32)
683
gfp_flags |= GFP_DMA32;
685
gfp_flags |= GFP_HIGHUSER;
687
for (r = 0; r < count; ++r) {
688
p = alloc_page(gfp_flags);
691
printk(KERN_ERR TTM_PFX
692
"Unable to allocate page.");
696
list_add(&p->lru, pages);
702
/* combine zero flag to pool flags */
703
gfp_flags |= pool->gfp_flags;
705
/* First we take pages from the pool */
706
count = ttm_page_pool_get_pages(pool, pages, flags, cstate, count);
708
/* clear the pages coming from the pool if requested */
709
if (flags & TTM_PAGE_FLAG_ZERO_ALLOC) {
710
list_for_each_entry(p, pages, lru) {
711
clear_page(page_address(p));
715
/* If pool didn't have enough pages allocate new one. */
717
/* ttm_alloc_new_pages doesn't reference pool so we can run
718
* multiple requests in parallel.
720
r = ttm_alloc_new_pages(pages, gfp_flags, flags, cstate, count);
722
/* If there is any pages in the list put them back to
724
printk(KERN_ERR TTM_PFX
725
"Failed to allocate extra pages "
726
"for large request.");
727
ttm_put_pages(pages, 0, flags, cstate, NULL);
736
/* Put all pages in pages list to correct pool to wait for reuse */
737
void ttm_put_pages(struct list_head *pages, unsigned page_count, int flags,
738
enum ttm_caching_state cstate, dma_addr_t *dma_address)
740
unsigned long irq_flags;
741
struct ttm_page_pool *pool = ttm_get_pool(flags, cstate);
742
struct page *p, *tmp;
745
/* No pool for this memory type so free the pages */
747
list_for_each_entry_safe(p, tmp, pages, lru) {
750
/* Make the pages list empty */
751
INIT_LIST_HEAD(pages);
754
if (page_count == 0) {
755
list_for_each_entry_safe(p, tmp, pages, lru) {
760
spin_lock_irqsave(&pool->lock, irq_flags);
761
list_splice_init(pages, &pool->list);
762
pool->npages += page_count;
763
/* Check that we don't go over the pool limit */
765
if (pool->npages > _manager->options.max_size) {
766
page_count = pool->npages - _manager->options.max_size;
767
/* free at least NUM_PAGES_TO_ALLOC number of pages
768
* to reduce calls to set_memory_wb */
769
if (page_count < NUM_PAGES_TO_ALLOC)
770
page_count = NUM_PAGES_TO_ALLOC;
772
spin_unlock_irqrestore(&pool->lock, irq_flags);
774
ttm_page_pool_free(pool, page_count);
777
static void ttm_page_pool_init_locked(struct ttm_page_pool *pool, int flags,
780
spin_lock_init(&pool->lock);
781
pool->fill_lock = false;
782
INIT_LIST_HEAD(&pool->list);
783
pool->npages = pool->nfrees = 0;
784
pool->gfp_flags = flags;
788
int ttm_page_alloc_init(struct ttm_mem_global *glob, unsigned max_pages)
794
printk(KERN_INFO TTM_PFX "Initializing pool allocator.\n");
796
_manager = kzalloc(sizeof(*_manager), GFP_KERNEL);
798
ttm_page_pool_init_locked(&_manager->wc_pool, GFP_HIGHUSER, "wc");
800
ttm_page_pool_init_locked(&_manager->uc_pool, GFP_HIGHUSER, "uc");
802
ttm_page_pool_init_locked(&_manager->wc_pool_dma32,
803
GFP_USER | GFP_DMA32, "wc dma");
805
ttm_page_pool_init_locked(&_manager->uc_pool_dma32,
806
GFP_USER | GFP_DMA32, "uc dma");
808
_manager->options.max_size = max_pages;
809
_manager->options.small = SMALL_ALLOCATION;
810
_manager->options.alloc_size = NUM_PAGES_TO_ALLOC;
812
ret = kobject_init_and_add(&_manager->kobj, &ttm_pool_kobj_type,
813
&glob->kobj, "pool");
814
if (unlikely(ret != 0)) {
815
kobject_put(&_manager->kobj);
820
ttm_pool_mm_shrink_init(_manager);
825
void ttm_page_alloc_fini(void)
829
printk(KERN_INFO TTM_PFX "Finalizing pool allocator.\n");
830
ttm_pool_mm_shrink_fini(_manager);
832
for (i = 0; i < NUM_POOLS; ++i)
833
ttm_page_pool_free(&_manager->pools[i], FREE_ALL_PAGES);
835
kobject_put(&_manager->kobj);
839
int ttm_page_alloc_debugfs(struct seq_file *m, void *data)
841
struct ttm_page_pool *p;
843
char *h[] = {"pool", "refills", "pages freed", "size"};
845
seq_printf(m, "No pool allocator running.\n");
848
seq_printf(m, "%6s %12s %13s %8s\n",
849
h[0], h[1], h[2], h[3]);
850
for (i = 0; i < NUM_POOLS; ++i) {
851
p = &_manager->pools[i];
853
seq_printf(m, "%6s %12ld %13ld %8d\n",
854
p->name, p->nrefills,
855
p->nfrees, p->npages);
859
EXPORT_SYMBOL(ttm_page_alloc_debugfs);