~bremby/grub4helenos/main

/*
 * Copyright (c) 2006 Jakub Jermar
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright
 *   notice, this list of conditions and the following disclaimer in the
 *   documentation and/or other materials provided with the distribution.
 * - The name of the author may not be used to endorse or promote products
 *   derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

/** @addtogroup genarchmm
 * @{
 */

/**
 * @file
 * @brief Virtual Address Translation for hierarchical 4-level page tables.
 */

#include <genarch/mm/page_pt.h>
#include <mm/page.h>
#include <mm/frame.h>
#include <mm/km.h>
#include <mm/as.h>
#include <arch/mm/page.h>
#include <arch/mm/as.h>
#include <arch/barrier.h>
#include <typedefs.h>
#include <arch/asm.h>
#include <memstr.h>
#include <align.h>
#include <macros.h>
#include <bitops.h>

static void pt_mapping_insert(as_t *, uintptr_t, uintptr_t, unsigned int);
static void pt_mapping_remove(as_t *, uintptr_t);
static pte_t *pt_mapping_find(as_t *, uintptr_t, bool);
static void pt_mapping_make_global(uintptr_t, size_t);

page_mapping_operations_t pt_mapping_operations = {
	.mapping_insert = pt_mapping_insert,
	.mapping_remove = pt_mapping_remove,
	.mapping_find = pt_mapping_find,
	.mapping_make_global = pt_mapping_make_global
};

/** Map page to frame using hierarchical page tables.
 *
 * Map virtual address page to physical address frame
 * using flags.
 *
 * @param as    Address space to wich page belongs.
 * @param page  Virtual address of the page to be mapped.
 * @param frame Physical address of memory frame to which the mapping is done.
 * @param flags Flags to be used for mapping.
 *
 */
void pt_mapping_insert(as_t *as, uintptr_t page, uintptr_t frame,
    unsigned int flags)
{
	pte_t *ptl0 = (pte_t *) PA2KA((uintptr_t) as->genarch.page_table);

	ASSERT(page_table_locked(as));
	
	if (GET_PTL1_FLAGS(ptl0, PTL0_INDEX(page)) & PAGE_NOT_PRESENT) {
		pte_t *newpt = (pte_t *)
		    PA2KA(frame_alloc(PTL1_FRAMES, FRAME_LOWMEM, PTL1_SIZE - 1));
		memsetb(newpt, PTL1_SIZE, 0);
		SET_PTL1_ADDRESS(ptl0, PTL0_INDEX(page), KA2PA(newpt));
		SET_PTL1_FLAGS(ptl0, PTL0_INDEX(page),
		    PAGE_NOT_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE |
		    PAGE_WRITE);
		/*
		 * Make sure that a concurrent hardware page table walk or
		 * pt_mapping_find() will see the new PTL1 only after it is
		 * fully initialized.
		 */
		write_barrier();
		SET_PTL1_PRESENT(ptl0, PTL0_INDEX(page));
	}
	
	pte_t *ptl1 = (pte_t *) PA2KA(GET_PTL1_ADDRESS(ptl0, PTL0_INDEX(page)));
	
	if (GET_PTL2_FLAGS(ptl1, PTL1_INDEX(page)) & PAGE_NOT_PRESENT) {
		pte_t *newpt = (pte_t *)
		    PA2KA(frame_alloc(PTL2_FRAMES, FRAME_LOWMEM, PTL2_SIZE - 1));
		memsetb(newpt, PTL2_SIZE, 0);
		SET_PTL2_ADDRESS(ptl1, PTL1_INDEX(page), KA2PA(newpt));
		SET_PTL2_FLAGS(ptl1, PTL1_INDEX(page),
		    PAGE_NOT_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE |
		    PAGE_WRITE);
		/*
		 * Make the new PTL2 visible only after it is fully initialized.
		 */
		write_barrier();
		SET_PTL2_PRESENT(ptl1, PTL1_INDEX(page));
	}
	
	pte_t *ptl2 = (pte_t *) PA2KA(GET_PTL2_ADDRESS(ptl1, PTL1_INDEX(page)));
	
	if (GET_PTL3_FLAGS(ptl2, PTL2_INDEX(page)) & PAGE_NOT_PRESENT) {
		pte_t *newpt = (pte_t *)
		    PA2KA(frame_alloc(PTL3_FRAMES, FRAME_LOWMEM, PTL2_SIZE - 1));
		memsetb(newpt, PTL2_SIZE, 0);
		SET_PTL3_ADDRESS(ptl2, PTL2_INDEX(page), KA2PA(newpt));
		SET_PTL3_FLAGS(ptl2, PTL2_INDEX(page),
		    PAGE_NOT_PRESENT | PAGE_USER | PAGE_EXEC | PAGE_CACHEABLE |
		    PAGE_WRITE);
		/*
		 * Make the new PTL3 visible only after it is fully initialized.
		 */
		write_barrier();
		SET_PTL3_PRESENT(ptl2, PTL2_INDEX(page));
	}
	
	pte_t *ptl3 = (pte_t *) PA2KA(GET_PTL3_ADDRESS(ptl2, PTL2_INDEX(page)));
	
	SET_FRAME_ADDRESS(ptl3, PTL3_INDEX(page), frame);
	SET_FRAME_FLAGS(ptl3, PTL3_INDEX(page), flags | PAGE_NOT_PRESENT);
	/*
	 * Make the new mapping visible only after it is fully initialized.
	 */
	write_barrier();
	SET_FRAME_PRESENT(ptl3, PTL3_INDEX(page));
}

/** Remove mapping of page from hierarchical page tables.
 *
 * Remove any mapping of page within address space as.
 * TLB shootdown should follow in order to make effects of
 * this call visible.
 *
 * Empty page tables except PTL0 are freed.
 *
 * @param as   Address space to wich page belongs.
 * @param page Virtual address of the page to be demapped.
 *
 */
void pt_mapping_remove(as_t *as, uintptr_t page)
{
	ASSERT(page_table_locked(as));

	/*
	 * First, remove the mapping, if it exists.
	 */
	
	pte_t *ptl0 = (pte_t *) PA2KA((uintptr_t) as->genarch.page_table);
	if (GET_PTL1_FLAGS(ptl0, PTL0_INDEX(page)) & PAGE_NOT_PRESENT)
		return;
	
	pte_t *ptl1 = (pte_t *) PA2KA(GET_PTL1_ADDRESS(ptl0, PTL0_INDEX(page)));
	if (GET_PTL2_FLAGS(ptl1, PTL1_INDEX(page)) & PAGE_NOT_PRESENT)
		return;
	
	pte_t *ptl2 = (pte_t *) PA2KA(GET_PTL2_ADDRESS(ptl1, PTL1_INDEX(page)));
	if (GET_PTL3_FLAGS(ptl2, PTL2_INDEX(page)) & PAGE_NOT_PRESENT)
		return;
	
	pte_t *ptl3 = (pte_t *) PA2KA(GET_PTL3_ADDRESS(ptl2, PTL2_INDEX(page)));
	
	/*
	 * Destroy the mapping.
	 * Setting to PAGE_NOT_PRESENT is not sufficient.
	 * But we need SET_FRAME for possible PT coherence maintenance.
	 * At least on ARM.
	 */
	//TODO: Fix this inconsistency
	SET_FRAME_FLAGS(ptl3, PTL3_INDEX(page), PAGE_NOT_PRESENT);
	memsetb(&ptl3[PTL3_INDEX(page)], sizeof(pte_t), 0);
	
	/*
	 * Second, free all empty tables along the way from PTL3 down to PTL0
	 * except those needed for sharing the kernel non-identity mappings.
	 */
	
	/* Check PTL3 */
	bool empty = true;
	
	unsigned int i;
	for (i = 0; i < PTL3_ENTRIES; i++) {
		if (PTE_VALID(&ptl3[i])) {
			empty = false;
			break;
		}
	}
	
	if (empty) {
		/*
		 * PTL3 is empty.
		 * Release the frame and remove PTL3 pointer from the parent
		 * table.
		 */
#if (PTL2_ENTRIES != 0)
		memsetb(&ptl2[PTL2_INDEX(page)], sizeof(pte_t), 0);
#elif (PTL1_ENTRIES != 0)
		memsetb(&ptl1[PTL1_INDEX(page)], sizeof(pte_t), 0);
#else
		if (km_is_non_identity(page))
			return;

		memsetb(&ptl0[PTL0_INDEX(page)], sizeof(pte_t), 0);
#endif
		frame_free(KA2PA((uintptr_t) ptl3), PTL3_FRAMES);
	} else {
		/*
		 * PTL3 is not empty.
		 * Therefore, there must be a path from PTL0 to PTL3 and
		 * thus nothing to free in higher levels.
		 *
		 */
		return;
	}
	
	/* Check PTL2, empty is still true */
#if (PTL2_ENTRIES != 0)
	for (i = 0; i < PTL2_ENTRIES; i++) {
		if (PTE_VALID(&ptl2[i])) {
			empty = false;
			break;
		}
	}
	
	if (empty) {
		/*
		 * PTL2 is empty.
		 * Release the frame and remove PTL2 pointer from the parent
		 * table.
		 */
#if (PTL1_ENTRIES != 0)
		memsetb(&ptl1[PTL1_INDEX(page)], sizeof(pte_t), 0);
#else
		if (km_is_non_identity(page))
			return;

		memsetb(&ptl0[PTL0_INDEX(page)], sizeof(pte_t), 0);
#endif
		frame_free(KA2PA((uintptr_t) ptl2), PTL2_FRAMES);
	} else {
		/*
		 * PTL2 is not empty.
		 * Therefore, there must be a path from PTL0 to PTL2 and
		 * thus nothing to free in higher levels.
		 *
		 */
		return;
	}
#endif /* PTL2_ENTRIES != 0 */
	
	/* check PTL1, empty is still true */
#if (PTL1_ENTRIES != 0)
	for (i = 0; i < PTL1_ENTRIES; i++) {
		if (PTE_VALID(&ptl1[i])) {
			empty = false;
			break;
		}
	}
	
	if (empty) {
		/*
		 * PTL1 is empty.
		 * Release the frame and remove PTL1 pointer from the parent
		 * table.
		 */
		if (km_is_non_identity(page))
			return;

		memsetb(&ptl0[PTL0_INDEX(page)], sizeof(pte_t), 0);
		frame_free(KA2PA((uintptr_t) ptl1), PTL1_FRAMES);
	}
#endif /* PTL1_ENTRIES != 0 */
}

/** Find mapping for virtual page in hierarchical page tables.
 *
 * @param as     Address space to which page belongs.
 * @param page   Virtual page.
 * @param nolock True if the page tables need not be locked.
 *
 * @return NULL if there is no such mapping; entry from PTL3 describing
 *         the mapping otherwise.
 *
 */
pte_t *pt_mapping_find(as_t *as, uintptr_t page, bool nolock)
{
	ASSERT(nolock || page_table_locked(as));

	pte_t *ptl0 = (pte_t *) PA2KA((uintptr_t) as->genarch.page_table);
	if (GET_PTL1_FLAGS(ptl0, PTL0_INDEX(page)) & PAGE_NOT_PRESENT)
		return NULL;

	read_barrier();
	
	pte_t *ptl1 = (pte_t *) PA2KA(GET_PTL1_ADDRESS(ptl0, PTL0_INDEX(page)));
	if (GET_PTL2_FLAGS(ptl1, PTL1_INDEX(page)) & PAGE_NOT_PRESENT)
		return NULL;

#if (PTL1_ENTRIES != 0)
	/*
	 * Always read ptl2 only after we are sure it is present.
	 */
	read_barrier();
#endif
	
	pte_t *ptl2 = (pte_t *) PA2KA(GET_PTL2_ADDRESS(ptl1, PTL1_INDEX(page)));
	if (GET_PTL3_FLAGS(ptl2, PTL2_INDEX(page)) & PAGE_NOT_PRESENT)
		return NULL;

#if (PTL2_ENTRIES != 0)
	/*
	 * Always read ptl3 only after we are sure it is present.
	 */
	read_barrier();
#endif
	
	pte_t *ptl3 = (pte_t *) PA2KA(GET_PTL3_ADDRESS(ptl2, PTL2_INDEX(page)));
	
	return &ptl3[PTL3_INDEX(page)];
}

/** Return the size of the region mapped by a single PTL0 entry.
 *
 * @return Size of the region mapped by a single PTL0 entry.
 */
static uintptr_t ptl0_step_get(void)
{
	size_t va_bits;

	va_bits = fnzb(PTL0_ENTRIES) + fnzb(PTL1_ENTRIES) + fnzb(PTL2_ENTRIES) +
	    fnzb(PTL3_ENTRIES) + PAGE_WIDTH;

	return 1UL << (va_bits - fnzb(PTL0_ENTRIES));
}

/** Make the mappings in the given range global accross all address spaces.
 *
 * All PTL0 entries in the given range will be mapped to a next level page
 * table. The next level page table will be allocated and cleared.
 *
 * pt_mapping_remove() will never deallocate these page tables even when there
 * are no PTEs in them.
 *
 * @param as   Address space.
 * @param base Base address corresponding to the first PTL0 entry that will be
 *             altered by this function.
 * @param size Size in bytes defining the range of PTL0 entries that will be
 *             altered by this function.
 *
 */
void pt_mapping_make_global(uintptr_t base, size_t size)
{
	ASSERT(size > 0);
	
	uintptr_t ptl0 = PA2KA((uintptr_t) AS_KERNEL->genarch.page_table);
	uintptr_t ptl0_step = ptl0_step_get();
	size_t frames;
	
#if (PTL1_ENTRIES != 0)
	frames = PTL1_FRAMES;
#elif (PTL2_ENTRIES != 0)
	frames = PTL2_FRAMES;
#else
	frames = PTL3_FRAMES;
#endif
	
	for (uintptr_t addr = ALIGN_DOWN(base, ptl0_step);
	    addr - 1 < base + size - 1;
	    addr += ptl0_step) {
		uintptr_t l1 = PA2KA(frame_alloc(frames, FRAME_LOWMEM, 0));
		memsetb((void *) l1, FRAMES2SIZE(frames), 0);
		SET_PTL1_ADDRESS(ptl0, PTL0_INDEX(addr), KA2PA(l1));
		SET_PTL1_FLAGS(ptl0, PTL0_INDEX(addr),
		    PAGE_PRESENT | PAGE_USER | PAGE_CACHEABLE |
		    PAGE_EXEC | PAGE_WRITE | PAGE_READ);
	}
}

/** @}
 */