1
#ifndef _I386_PGALLOC_H
2
#define _I386_PGALLOC_H
4
#include <linux/config.h>
5
#include <asm/processor.h>
6
#include <asm/fixmap.h>
7
#include <asm/hypervisor.h>
8
#include <linux/threads.h>
11
* Quick lists are aligned so that least significant bits of array pointer
12
* are all zero when list is empty, and all one when list is full.
14
#define QUICKLIST_ENTRIES 256
15
#define QUICKLIST_EMPTY(_l) !((unsigned long)(_l) & ((QUICKLIST_ENTRIES*4)-1))
16
#define QUICKLIST_FULL(_l) QUICKLIST_EMPTY((_l)+1)
17
#define pgd_quicklist (current_cpu_data.pgd_quick)
18
#define pmd_quicklist (current_cpu_data.pmd_quick)
19
#define pte_quicklist (current_cpu_data.pte_quick)
20
#define pgtable_cache_size (current_cpu_data.pgtable_cache_sz)
22
#define pmd_populate(mm, pmd, pte) \
24
set_pmd(pmd, __pmd(_PAGE_TABLE + __pa(pte))); \
25
XENO_flush_page_update_queue(); \
29
* Allocate and free page tables.
32
#if defined (CONFIG_X86_PAE)
34
#error "no PAE support as yet"
37
* We can't include <linux/slab.h> here, thus these uglinesses.
41
extern struct kmem_cache_s *pae_pgd_cachep;
42
extern void *kmem_cache_alloc(struct kmem_cache_s *, int);
43
extern void kmem_cache_free(struct kmem_cache_s *, void *);
46
static inline pgd_t *get_pgd_slow(void)
49
pgd_t *pgd = kmem_cache_alloc(pae_pgd_cachep, GFP_KERNEL);
52
for (i = 0; i < USER_PTRS_PER_PGD; i++) {
53
unsigned long pmd = __get_free_page(GFP_KERNEL);
57
set_pgd(pgd + i, __pgd(1 + __pa(pmd)));
59
memcpy(pgd + USER_PTRS_PER_PGD,
60
swapper_pg_dir + USER_PTRS_PER_PGD,
61
(PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
65
for (i--; i >= 0; i--)
66
free_page((unsigned long)__va(pgd_val(pgd[i])-1));
67
kmem_cache_free(pae_pgd_cachep, pgd);
73
static inline pgd_t *get_pgd_slow(void)
75
pgd_t *pgd = (pgd_t *)__get_free_page(GFP_KERNEL);
78
memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
79
memcpy(pgd + USER_PTRS_PER_PGD,
80
init_mm.pgd + USER_PTRS_PER_PGD,
81
(PTRS_PER_PGD - USER_PTRS_PER_PGD) * sizeof(pgd_t));
82
__make_page_readonly(pgd);
83
queue_pgd_pin(__pa(pgd));
89
#endif /* CONFIG_X86_PAE */
91
static inline pgd_t *get_pgd_fast(void)
95
if ( !QUICKLIST_EMPTY(pgd_quicklist) ) {
96
ret = *(--pgd_quicklist);
100
ret = (unsigned long)get_pgd_slow();
104
static inline void free_pgd_slow(pgd_t *pgd)
106
#if defined(CONFIG_X86_PAE)
110
for (i = 0; i < USER_PTRS_PER_PGD; i++)
111
free_page((unsigned long)__va(pgd_val(pgd[i])-1));
112
kmem_cache_free(pae_pgd_cachep, pgd);
114
queue_pgd_unpin(__pa(pgd));
115
__make_page_writeable(pgd);
116
free_page((unsigned long)pgd);
120
static inline void free_pgd_fast(pgd_t *pgd)
122
if ( !QUICKLIST_FULL(pgd_quicklist) ) {
123
*(pgd_quicklist++) = (unsigned long)pgd;
124
pgtable_cache_size++;
129
static inline pte_t *pte_alloc_one(struct mm_struct *mm, unsigned long address)
133
pte = (pte_t *) __get_free_page(GFP_KERNEL);
137
__make_page_readonly(pte);
138
queue_pte_pin(__pa(pte));
144
static inline pte_t *pte_alloc_one_fast(struct mm_struct *mm,
145
unsigned long address)
147
unsigned long ret = 0;
148
if ( !QUICKLIST_EMPTY(pte_quicklist) ) {
149
ret = *(--pte_quicklist);
150
pgtable_cache_size--;
155
static __inline__ void pte_free_slow(pte_t *pte)
157
queue_pte_unpin(__pa(pte));
158
__make_page_writeable(pte);
159
free_page((unsigned long)pte);
162
static inline void pte_free_fast(pte_t *pte)
164
if ( !QUICKLIST_FULL(pte_quicklist) ) {
165
*(pte_quicklist++) = (unsigned long)pte;
166
pgtable_cache_size++;
171
#define pte_free(pte) pte_free_fast(pte)
172
#define pgd_free(pgd) free_pgd_fast(pgd)
173
#define pgd_alloc(mm) get_pgd_fast()
176
* allocating and freeing a pmd is trivial: the 1-entry pmd is
177
* inside the pgd, so has no extra memory associated with it.
178
* (In the PAE case we free the pmds as part of the pgd.)
181
#define pmd_alloc_one_fast(mm, addr) ({ BUG(); ((pmd_t *)1); })
182
#define pmd_alloc_one(mm, addr) ({ BUG(); ((pmd_t *)2); })
183
#define pmd_free_slow(x) do { } while (0)
184
#define pmd_free_fast(x) do { } while (0)
185
#define pmd_free(x) do { } while (0)
186
#define pgd_populate(mm, pmd, pte) BUG()
188
extern int do_check_pgt_cache(int, int);
193
* - flush_tlb() flushes the current mm struct TLBs
194
* - flush_tlb_all() flushes all processes TLBs
195
* - flush_tlb_mm(mm) flushes the specified mm context TLB's
196
* - flush_tlb_page(vma, vmaddr) flushes one page
197
* - flush_tlb_range(mm, start, end) flushes a range of pages
198
* - flush_tlb_pgtables(mm, start, end) flushes a range of page tables
200
* ..but the i386 has somewhat limited tlb flushing capabilities,
201
* and page-granular flushes are available only on i486 and up.
206
#define flush_tlb() __flush_tlb()
207
#define flush_tlb_all() __flush_tlb_all()
208
#define local_flush_tlb() __flush_tlb()
210
static inline void flush_tlb_mm(struct mm_struct *mm)
212
if (mm == current->active_mm) queue_tlb_flush();
213
XENO_flush_page_update_queue();
216
static inline void flush_tlb_page(struct vm_area_struct *vma,
219
if (vma->vm_mm == current->active_mm) queue_invlpg(addr);
220
XENO_flush_page_update_queue();
223
static inline void flush_tlb_range(struct mm_struct *mm,
224
unsigned long start, unsigned long end)
226
if (mm == current->active_mm) queue_tlb_flush();
227
XENO_flush_page_update_queue();
231
#error no guestos SMP support yet...
234
#define local_flush_tlb() \
237
extern void flush_tlb_all(void);
238
extern void flush_tlb_current_task(void);
239
extern void flush_tlb_mm(struct mm_struct *);
240
extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
242
#define flush_tlb() flush_tlb_current_task()
244
static inline void flush_tlb_range(struct mm_struct * mm, unsigned long start, unsigned long end)
249
#define TLBSTATE_OK 1
250
#define TLBSTATE_LAZY 2
254
struct mm_struct *active_mm;
256
} ____cacheline_aligned;
257
extern struct tlb_state cpu_tlbstate[NR_CPUS];
259
#endif /* CONFIG_SMP */
261
static inline void flush_tlb_pgtables(struct mm_struct *mm,
262
unsigned long start, unsigned long end)
264
/* i386 does not keep any page table caches in TLB */
265
XENO_flush_page_update_queue();
268
extern int direct_remap_area_pages(struct mm_struct *mm,
269
unsigned long address,
270
unsigned long machine_addr,
274
#endif /* _I386_PGALLOC_H */