path: root/arch/x86_64/hal/paging.c

#include "paging.h"
#include "arch/x86_64/x86_64.h"
#include "hal/memory.h"
#include "jove.h"
#include "init.h"
#include "config.h"
#include "hal/obj.h"
#include "lib/string.h"
#include "lib/printf.h"
#include "lib/panic.h"
#include "idt.h"
#include "api/syscall.h"

PAGEALIGN page_t _kernel_pml4[512];
PAGEALIGN page_t _kernel_pml3[512];
PAGEALIGN page_t _kernel_pml2[512];
PAGEALIGN page_t _kernel_pml1[512];

#define IDENT_GIANTPAGES ((PHYS_HUGEPAGES_MAX + 511) / 512)
#define IDENT_MASSIVEPAGES ((IDENT_GIANTPAGES + 511) / 512)

PAGEALIGN static page_t ident_pml3[IDENT_MASSIVEPAGES][512];
PAGEALIGN static page_t ident_pml2[IDENT_GIANTPAGES][512];

void*
phys_tovirt(uintptr_t phys)
{
    return (void*)(phys | IDENT_VIRT_BASE);
}

uintptr_t
paging_get_pml4(void)
{
    uintptr_t ret = 0;
    __asm__ volatile("movq %%cr3, %0": "=r"(ret));
    return ret;
}

void
paging_pml4_switch(uintptr_t pml4)
{
    __asm__ volatile("movq %0, %%cr3":: "r"(pml4));
}

static struct int_stackframe*
s_int_handle_pf_early(struct int_stackframe *frame)
{
    extern int _kernel_panic_unlock;
    _kernel_panic_unlock = 1;

    union {
        struct {
            uint8_t p : 1;
            uint8_t wr : 1;
            uint8_t us : 1;
            uint8_t rsvd : 1;
            uint8_t id : 1;
            uint8_t pk : 1;
        };
        uint64_t value;
    } pf_error = { .value = frame->int_err };
    
    kprintf("Kernel has encountered a page fault exception early in initializaiton\n");
    kprintf("%s %s during %s\n", 
            pf_error.wr ? "Wrote to" : "Read from",
            pf_error.p ? "Present" : "Non-present",
            pf_error.id ? "Instruction fetch" : "Data access");
    uintptr_t cr2 = 0;
    __asm__ volatile("movq %%cr2, %0": "=r"(cr2));
    kprintf("Faulting address: %p\n", cr2);
    kprintf("RIP: %p | RSP: %p\n", frame->int_rip, frame->int_rsp);
    print_state(&frame->state);

    hcf();
}

static page_t*
s_pagedir_get_pml1(page_t *pml4, uintptr_t vptr)
{
    errno = 0;
    size_t pml4i = PML4I(vptr);
    if(!pml4[pml4i].p) {
        errno = ENOMAP;
        return 0;
    }

    page_t *pml3 = (page_t*)(((uintptr_t)pml4[pml4i].paddr << 12) | IDENT_VIRT_BASE);
    size_t pml3i = PML3I(vptr);
    if(!pml3[pml3i].p) {
        errno = ENOMAP;
        return 0;
    }

    page_t *pml2 = (page_t*)(((uintptr_t)pml3[pml3i].paddr << 12) | IDENT_VIRT_BASE);
    size_t pml2i = PML2I(vptr);
    if(!pml2[pml2i].p) {
        errno = ENOMAP;
        return 0;
    }

    return (page_t*)(((uintptr_t)pml2[pml2i].paddr << 12) | IDENT_VIRT_BASE);
}

int pagedir_map_frame(page_t *pml4, 
        uintptr_t pptr, uintptr_t vptr, 
        uintmax_t flg)
{
    page_t *pml1 = s_pagedir_get_pml1(pml4, vptr);
    if(!pml1)
        return errno;

    size_t pml1i = PML1I(vptr);
    if(pml1[pml1i].p) return EEXIST;

    pml1[pml1i].value = pptr | PAGE_PRESENT | flg;
    __asm__ volatile("invlpg (%0)":: "r"(vptr): "memory");
    return 0;
}

int pagedir_map_untyped_frame(
        jove_owned_obj_t *tgt, jove_owned_obj_t *untyped_obj, 
        uintptr_t vptr, struct syscall_pagedir_flags flg)
{
    page_t *pml4 = obj_deref(tgt);
    uintptr_t frame = obj_getvalue(untyped_obj);

    return pagedir_map_frame(pml4, frame, vptr, 
            (flg.writeable ? PAGE_WRITEABLE : 0) |
            (flg.executable ? PAGE_XD : 0));
}

int pagedir_unmap_untyped_frame(
        jove_owned_obj_t *tgt, jove_owned_obj_t *dst, uintptr_t vptr)
{
    page_t *pml4 = obj_deref(tgt);

    size_t pml4i = PML4I(vptr);
    if(!pml4[pml4i].p) return ENOMAP;

    page_t *pml1 = s_pagedir_get_pml1(pml4, vptr);
    size_t pml1i = PML1I(vptr);
    if(!pml1[pml1i].p) return ENOMAP;

    uintptr_t untyped_pptr = (pml1[pml1i].paddr << 12);
    
    obj_settype(dst, JOVE_OBJ_UNTYPED);
    obj_setvalue(dst, untyped_pptr);

    jove_untyped_t *untyped_data = obj_deref(dst);
    untyped_data->limit = 0x1000;
    __asm__ volatile("invlpg (%0)":: "r"(vptr): "memory");

    return 0;
}

int pagedir_map_table(page_t *pml4,
        uintptr_t pptr, uintptr_t vptr)
{
    page_t *target_table = NULL;

    void *frame_ident = (void*)(pptr | IDENT_VIRT_BASE);
    memset(frame_ident, 0, 0x1000);

    size_t pml4i = PML4I(vptr);
    if(!pml4[pml4i].p) {
        target_table = &pml4[pml4i];
        goto map_table;
    }

    page_t *pml3 = (page_t*)(((uintptr_t)pml4[pml4i].paddr << 12) | IDENT_VIRT_BASE);
    size_t pml3i = PML3I(vptr);
    if(!pml3[pml3i].p) {
        target_table = &pml3[pml3i];
        goto map_table;
    }

    page_t *pml2 = (page_t*)(((uintptr_t)pml3[pml3i].paddr << 12) | IDENT_VIRT_BASE);
    size_t pml2i = PML2I(vptr);
    if(!pml2[pml2i].p) {
        target_table = &pml2[pml2i];
        goto map_table;
    }
    return EHASMAP;

map_table:
    target_table->value = pptr |
        PAGE_PRESENT |
        (pptr > IDENT_VIRT_BASE ? 0 : PAGE_USER) |
        PAGE_WRITEABLE;
    __asm__ volatile("invlpg (%0)":: "r"(vptr): "memory");
    return 0;
}

int pagedir_map_untyped_table(
        jove_owned_obj_t *tgt, jove_owned_obj_t *untyped_obj, 
        uintptr_t vptr, struct syscall_pagedir_flags flg)
{
    page_t *pml4 = obj_deref(tgt);
    uintptr_t frame = obj_getvalue(untyped_obj);

    return pagedir_map_table(pml4, frame, vptr);
}

int pagedir_unmap_untyped_table(
        jove_owned_obj_t *tgt, jove_owned_obj_t *dst, uintptr_t vptr)
{
    page_t *target_table = NULL;
    page_t *pml4 = obj_deref(tgt);

    //If the PML4 entry does not exist at this virtual ptr, the user may be
    //trying to unmap the entire address space. We don't support this.
    size_t pml4i = PML4I(vptr);
    if(!pml4[pml4i].p) {
        return ENOMAP;
    }
    target_table = &pml4[pml4i];

    //Try to unmap the pml4 entry if the pml3 entry does not exist.
    page_t *pml3 = (page_t*)(((uintptr_t)pml4[pml4i].paddr << 12) | IDENT_VIRT_BASE);
    size_t pml3i = PML3I(vptr);
    if(!pml3[pml3i].p) {
        goto unmap_table;
    }
    target_table = &pml3[pml3i];

    //Try to unmap the pml3 entry if the pml2 entry does not exist.
    page_t *pml2 = (page_t*)(((uintptr_t)pml3[pml3i].paddr << 12) | IDENT_VIRT_BASE);
    size_t pml2i = PML2I(vptr);
    if(!pml2[pml2i].p) {
        goto unmap_table;
    }
    //We are unmapping a pml2 entry.
    target_table = &pml2[pml2i];

unmap_table:
{
    uintptr_t frame = target_table->paddr << 12;
    page_t *target_vals = (page_t*)(frame | IDENT_VIRT_BASE);

    //Check that the page table at our target entry is empty.
    //If it's not, we return an error saying that the entry contains a map.
    for(size_t i = 0; i < 512; i++) {
        if(target_vals[i].p) return EHASMAP;
    }

    //Populate the none object with an untyped object at the free'd page frame.
    obj_settype(dst, JOVE_OBJ_UNTYPED);
    obj_setvalue(dst, frame);

    jove_untyped_t *untyped = obj_deref(dst);
    untyped->limit = 0x1000;

    __asm__ volatile("invlpg (%0)":: "r"(vptr): "memory");
    return 0;
}
}

void
paging_pml4_init(void)
{
    //Setup kernel pages
    memset(_kernel_pml4, 0, 0x1000);
    memset(_kernel_pml3, 0, 0x1000);
    memset(_kernel_pml2, 0, 0x1000);
    memset(_kernel_pml1, 0, 0x1000);

    memset(ident_pml3, 0, 0x1000 * IDENT_MASSIVEPAGES);
    memset(ident_pml2, 0, 0x1000 * IDENT_GIANTPAGES);

    /*Paging entries depend on physical addresses to properly map.
     * Since the virtual kernel address differs from the physical memory address,
     * we need to convert the pointers using the physical memory pointer passed
     * to us by the bootloader.*/
    uintptr_t kernel_pml4_pptr = KPTR_TOPHYS((uintptr_t)_kernel_pml4);
    uintptr_t kernel_pml3_pptr = KPTR_TOPHYS((uintptr_t)_kernel_pml3);
    uintptr_t kernel_pml2_pptr = KPTR_TOPHYS((uintptr_t)_kernel_pml2);
    uintptr_t kernel_pml1_pptr = KPTR_TOPHYS((uintptr_t)_kernel_pml1);

    /*Maps pml4-2 as global writeable pages.*/
    _kernel_pml4[511].value = kernel_pml3_pptr | PAGE_PRESENT | PAGE_WRITEABLE | PAGE_GLOBAL;
    _kernel_pml3[510].value = kernel_pml2_pptr | PAGE_PRESENT | PAGE_WRITEABLE | PAGE_GLOBAL;
    _kernel_pml2[0].value =   kernel_pml1_pptr | PAGE_PRESENT | PAGE_WRITEABLE | PAGE_GLOBAL;

    /*text*/
    extern void *_kernel_text_start;
    extern void *_kernel_text_end;
    uintptr_t kernel_text_start_pptr = KPTR_TOPHYS((uintptr_t)&_kernel_text_start);
    uintptr_t kernel_text_end_pptr = KPTR_TOPHYS((uintptr_t)&_kernel_text_end);
    size_t kernel_text_pages = ((kernel_text_end_pptr - kernel_text_start_pptr) / 0x1000) + 1;
    
    uintptr_t kernel_text_off = (((uintptr_t)&_kernel_text_start) - KERNEL_VIRT_BASE) >> 12;

    for(size_t i = 0; i < kernel_text_pages; i++) {
        size_t off = i + kernel_text_off;
        uintptr_t page_off = kernel_text_start_pptr + (i << 12);
        _kernel_pml1[off].value = page_off | PAGE_PRESENT | PAGE_GLOBAL;
    }

    /*rodata*/
    extern void *_kernel_rodata_start;
    extern void *_kernel_rodata_end;
    uintptr_t kernel_rodata_start_pptr = KPTR_TOPHYS((uintptr_t)&_kernel_rodata_start);
    uintptr_t kernel_rodata_end_pptr = KPTR_TOPHYS((uintptr_t)&_kernel_rodata_end);
    size_t kernel_rodata_pages = ((kernel_rodata_end_pptr - kernel_rodata_start_pptr) / 0x1000) + 1;

    uintptr_t kernel_rodata_off = kernel_text_off + kernel_text_pages;
    for(size_t i = 0; i < kernel_rodata_pages; i++) {
        size_t off = i + kernel_rodata_off;
        uintptr_t page_off = kernel_rodata_start_pptr + (i << 12);
        _kernel_pml1[off].value = page_off | PAGE_PRESENT | PAGE_GLOBAL | PAGE_XD;
    }

    /*data/bss*/
    extern void *_kernel_data_start;
    extern void *_kernel_bss_end;
    uintptr_t kernel_data_start_pptr = KPTR_TOPHYS((uintptr_t)&_kernel_data_start);
    uintptr_t kernel_data_end_pptr = KPTR_TOPHYS((uintptr_t)&_kernel_bss_end);
    size_t kernel_data_pages = ((kernel_data_end_pptr - kernel_data_start_pptr) / 0x1000) + 1;

    uintptr_t kernel_data_off = kernel_rodata_off + kernel_rodata_pages;
    for(size_t i = 0; i < kernel_data_pages; i++) {
        size_t off = i + kernel_data_off;
        uintptr_t page_off = kernel_data_start_pptr + (i << 12);
        _kernel_pml1[off].value = page_off | PAGE_PRESENT | PAGE_WRITEABLE | PAGE_GLOBAL | PAGE_XD;
    }

    /*Identity pages are mapped as huge pages starting at the 256'th PML3 entry.
     *We want to support any range of physical memory up to several TiB, so
     *the mapper has to consider multiple heirarchies of huge pages.*/

    for(size_t l3 = 0; l3 < IDENT_MASSIVEPAGES; l3++) {
        uintptr_t ident_pml3_pptr = KPTR_TOPHYS(&ident_pml3[l3]);

        _kernel_pml4[256 + l3].value = ident_pml3_pptr | PAGE_PRESENT | PAGE_WRITEABLE | PAGE_GLOBAL;
    }

    for(size_t l2 = 0; l2 < IDENT_GIANTPAGES; l2++) {
        size_t l3 = l2 / 512;
        size_t l2_mod = l2 % 512;

        uintptr_t ident_pml2_pptr = KPTR_TOPHYS(&ident_pml2[l2]);
        ident_pml3[l3][l2_mod].value = ident_pml2_pptr | PAGE_PRESENT | PAGE_WRITEABLE | PAGE_GLOBAL;
    }

    for(size_t l1 = 0; l1 < PHYS_HUGEPAGES_MAX; l1++) {
        size_t l2 = l1 / 512;
        size_t l1_mod = l1 % 512;

        ident_pml2[l2][l1_mod].value = (l1 << 21) | PAGE_PRESENT | PAGE_WRITEABLE | PAGE_PS | PAGE_GLOBAL | PAGE_XD;
        ident_pml2[l2][l1_mod].xd = 1;
    }

    paging_pml4_switch(kernel_pml4_pptr);

    {
        jove_owned_obj_t *pgdir_o = objdir_getlocal(_rootdir_obj, ROOTDIR_PAGEDIR_I);
        obj_settype(pgdir_o, JOVE_OBJ_PAGEDIR);
        obj_setvalue(pgdir_o, kernel_pml4_pptr);
        obj_release(pgdir_o);
    }

    /*Before tasking, if the kernel has a bug and crashes we need some
     * information about what happened.*/
    extern struct int_stackframe *(*__int_kernel_handles[])(struct int_stackframe*);
    __int_kernel_handles[IDT_EXCEPT_PF] = s_int_handle_pf_early;
}