From ace65b453151845bc361f21f3e5b651c35f9f126 Mon Sep 17 00:00:00 2001
From: Jon Santmyer <jon@jonsantmyer.com>
Date: Wed, 22 May 2024 13:00:41 -0400
Subject: massive refactor for mp and organization
---
arch/arch.h | 8 --
arch/x86_64/cpu.c | 35 ++++++
arch/x86_64/cpu.h | 13 ---
arch/x86_64/elf.c | 55 ----------
arch/x86_64/elf.h | 78 --------------
arch/x86_64/elf_load.c | 58 ++++++++++
arch/x86_64/gdt.c | 83 ++++++--------
arch/x86_64/idt.c | 68 ++++--------
arch/x86_64/int_handler.c | 52 +++++++++
arch/x86_64/interrupt.h | 11 --
arch/x86_64/kexec.c | 50 +++++++++
arch/x86_64/msr.c | 67 ++++++++++++
arch/x86_64/page.c | 173 +++++++++++++++++++++++++++++
arch/x86_64/page.h | 28 +++++
arch/x86_64/pagedirectory.c | 138 ++++++++++++++++++++++++
arch/x86_64/pagefault.c | 39 +++++++
arch/x86_64/paging.c | 257 --------------------------------------------
arch/x86_64/paging.h | 28 -----
arch/x86_64/processor.c | 40 +++++++
arch/x86_64/restore.S | 20 ++++
arch/x86_64/savestate.S | 39 -------
arch/x86_64/serial.c | 62 -----------
arch/x86_64/serial.h | 27 -----
arch/x86_64/syscall.c | 1 -
arch/x86_64/syscall_entry.c | 25 +++++
arch/x86_64/syscall_setup.S | 41 -------
arch/x86_64/tables.c | 8 --
arch/x86_64/tables.h | 60 -----------
arch/x86_64/tasking.c | 86 ---------------
arch/x86_64/tcb_prepare.c | 9 ++
arch/x86_64/tcb_switch.c | 18 ++++
arch/x86_64/tss.h | 20 ----
arch/x86_64/uart.h | 19 ----
arch/x86_64/umode_enter.S | 8 --
arch/x86_64/umode_enter.c | 17 +++
arch/x86_64/vm_tophys.c | 23 ++++
36 files changed, 843 insertions(+), 921 deletions(-)
delete mode 100644 arch/arch.h
create mode 100644 arch/x86_64/cpu.c
delete mode 100644 arch/x86_64/cpu.h
delete mode 100644 arch/x86_64/elf.c
delete mode 100644 arch/x86_64/elf.h
create mode 100644 arch/x86_64/elf_load.c
create mode 100644 arch/x86_64/int_handler.c
delete mode 100644 arch/x86_64/interrupt.h
create mode 100644 arch/x86_64/kexec.c
create mode 100644 arch/x86_64/msr.c
create mode 100644 arch/x86_64/page.c
create mode 100644 arch/x86_64/page.h
create mode 100644 arch/x86_64/pagedirectory.c
create mode 100644 arch/x86_64/pagefault.c
delete mode 100644 arch/x86_64/paging.c
delete mode 100644 arch/x86_64/paging.h
create mode 100644 arch/x86_64/processor.c
create mode 100644 arch/x86_64/restore.S
delete mode 100644 arch/x86_64/savestate.S
delete mode 100644 arch/x86_64/serial.c
delete mode 100644 arch/x86_64/serial.h
delete mode 100644 arch/x86_64/syscall.c
create mode 100644 arch/x86_64/syscall_entry.c
delete mode 100644 arch/x86_64/syscall_setup.S
delete mode 100644 arch/x86_64/tables.c
delete mode 100644 arch/x86_64/tables.h
delete mode 100644 arch/x86_64/tasking.c
create mode 100644 arch/x86_64/tcb_prepare.c
create mode 100644 arch/x86_64/tcb_switch.c
delete mode 100644 arch/x86_64/tss.h
delete mode 100644 arch/x86_64/uart.h
delete mode 100644 arch/x86_64/umode_enter.S
create mode 100644 arch/x86_64/umode_enter.c
create mode 100644 arch/x86_64/vm_tophys.c
(limited to 'arch')
diff --git a/arch/arch.h b/arch/arch.h
deleted file mode 100644
index 6e569fd..0000000
--- a/arch/arch.h
+++ /dev/null
@@ -1,8 +0,0 @@
-#ifndef JOVE_ARCH_H
-#define JOVE_ARCH_H 1
-
-extern void serial_setup(void);
-
-extern void arch_tables_setup(void);
-
-#endif
diff --git a/arch/x86_64/cpu.c b/arch/x86_64/cpu.c
new file mode 100644
index 0000000..d088fcd
--- /dev/null
+++ b/arch/x86_64/cpu.c
@@ -0,0 +1,35 @@
+#include "arch/cpu.h"
+#include "arch/x86_64/msr.h"
+
+extern void _syscall_entry(void);
+
+extern void idt_init();
+
+void
+cpu_setup(void)
+{
+ idt_init();
+
+ cpu_arch_enable_sce();
+ cpu_set_syscall_entry((void*)_syscall_entry);
+}
+
+void cpu_arch_enable_sce(void)
+{
+ msr_efer_t feat = msr_efer_read();
+ feat.sce = 1;
+ msr_efer_write(feat);
+
+ msr_star_t star = msr_star_read();
+ star.kcs = 0x08;
+ star.ucs = 0x18;
+ msr_star_write(star);
+}
+
+void
+cpu_set_syscall_entry(void *ptr)
+{
+ msr_lstar_t lstar = (msr_lstar_t)ptr;
+ msr_lstar_write(lstar);
+}
+
diff --git a/arch/x86_64/cpu.h b/arch/x86_64/cpu.h
deleted file mode 100644
index 942c8d9..0000000
--- a/arch/x86_64/cpu.h
+++ /dev/null
@@ -1,13 +0,0 @@
-#ifndef ARCH_x86_64_CPU_H
-#define ARCH_x86_64_CPU_H 1
-
-#include <stdint.h>
-
-struct Registers
-{
- uint64_t r15, r14, r13, r12, r11, r10, r9, r8;
- uint64_t bp, di, si, dx, cx, bx, ax;
- uint64_t ip, cs, flags, sp, ss;
-};
-
-#endif
diff --git a/arch/x86_64/elf.c b/arch/x86_64/elf.c
deleted file mode 100644
index 969cbf0..0000000
--- a/arch/x86_64/elf.c
+++ /dev/null
@@ -1,55 +0,0 @@
-#include "elf.h"
-#include "lib/string.h"
-#include "usr/elf.h"
-#include "io/log.h"
-#include "mem/memory.h"
-
-void*
-elf_load(const void *data, size_t len)
-{
- if(data == NULL) return NULL;
- if(len < 4) return NULL;
-
- struct ELF_header *ehdr = (struct ELF_header*)data;
- if(ehdr->e_ident[EI_MAG0] != E_IDENT_MAG0 &&
- ehdr->e_ident[EI_MAG1] != E_IDENT_MAG1 &&
- ehdr->e_ident[EI_MAG2] != E_IDENT_MAG2 &&
- ehdr->e_ident[EI_MAG3] != E_IDENT_MAG3)
- return NULL;
-
- if(ehdr->e_ident[EI_CLASS] != E_IDENT_CLASS64) {
- klogf("Jove does not support ELF files of class CLASS32\n");
- return NULL;
- }
-
- if(ehdr->e_type != ET_EXEC) {
- klogf("Jove does not support ELF files other than ET_EXEC\n");
- return NULL;
- }
-
- uint64_t entry = ehdr->e_entry;
-
- size_t phdrc = ehdr->e_phnum;
- struct ELF_phdr *phdrs = (struct ELF_phdr*)((uintptr_t)data + ehdr->e_phoff);
-
- for(size_t phdri = 0; phdri < phdrc; phdri++)
- {
- struct ELF_phdr *phdr = &phdrs[phdri];
- void *pdata = (void*)phdr->p_vaddr;
-
- mem_ensure_range(
- phdr->p_vaddr,
- phdr->p_vaddr + phdr->p_memsz,
- (page_flags_t) {
- .present = true,
- .writeable = true,
- .useraccess = true,
- .executable = true
- });
- if(phdr->p_type == PT_LOAD)
- {
- memcpy(pdata, (void*)((uintptr_t)data + phdr->p_offset), phdr->p_filesz);
- }
- }
- return (void*)entry;
-}
diff --git a/arch/x86_64/elf.h b/arch/x86_64/elf.h
deleted file mode 100644
index ebcd228..0000000
--- a/arch/x86_64/elf.h
+++ /dev/null
@@ -1,78 +0,0 @@
-#ifndef JOVE_ARCH_x86_64_ELF_H
-#define JOVE_ARCH_x86_64_ELF_H 1
-
-#include <stdint.h>
-
-#define EI_MAG0 0
-#define EI_MAG1 1
-#define EI_MAG2 2
-#define EI_MAG3 3
-#define EI_CLASS 4
-#define EI_DATA 5
-#define EI_VERSION 6
-#define EI_OSABI 7
-#define EI_ABIVERSION 8
-#define EI_PAD 9
-
-#define E_IDENT_MAG0 0x7F
-#define E_IDENT_MAG1 'E'
-#define E_IDENT_MAG2 'L'
-#define E_IDENT_MAG3 'F'
-
-#define E_IDENT_CLASS32 1
-#define E_IDENT_CLASS64 2
-
-#define ET_EXEC 0x02
-
-struct ELF_header
-{
- uint8_t e_ident[15];
- uint16_t e_type;
- uint16_t e_machine;
- uint32_t e_version;
- uint64_t e_entry;
- uint64_t e_phoff;
- uint64_t e_shoff;
- uint32_t e_flags;
- uint16_t e_ehsize;
- uint16_t e_phentsize;
- uint16_t e_phnum;
- uint16_t e_shentsize;
- uint16_t e_shnum;
- uint16_t e_shstrndx;
-};
-
-#define PT_NULL 0
-#define PT_LOAD 1
-
-#define PF_X 1
-#define PF_W 2
-#define PF_R 4
-
-struct ELF_phdr
-{
- uint32_t p_type;
- uint32_t p_flags;
- uint64_t p_offset;
- uint64_t p_vaddr;
- uint64_t p_paddr;
- uint64_t p_filesz;
- uint64_t p_memsz;
- uint64_t p_align;
-};
-
-struct ELF_shdr
-{
- uint32_t sh_name;
- uint32_t sh_type;
- uint64_t sh_flags;
- uint64_t sh_addr;
- uint64_t sh_offset;
- uint64_t sh_size;
- uint32_t sh_link;
- uint32_t sh_info;
- uint64_t sh_addralign;
- uint64_t sh_entsize;
-};
-
-#endif
diff --git a/arch/x86_64/elf_load.c b/arch/x86_64/elf_load.c
new file mode 100644
index 0000000..d7d1b29
--- /dev/null
+++ b/arch/x86_64/elf_load.c
@@ -0,0 +1,58 @@
+#include "arch/elf.h"
+#include "string.h"
+#include "memory.h"
+#include "print.h"
+
+uintptr_t
+elf_load(const void *data, size_t len)
+{
+ if(data == NULL) return 0;
+ if(len < 4) return 0;
+
+ elf_header_t *ehdr = (elf_header_t*)data;
+ if(ehdr->ei_mag[0] != EI_MAG0 ||
+ ehdr->ei_mag[1] != EI_MAG1 ||
+ ehdr->ei_mag[2] != EI_MAG2 ||
+ ehdr->ei_mag[3] != EI_MAG3) return 0;
+
+ if(ehdr->ei_class != EI_CLASS_64) {
+ kerrf("Jove does not support ELF files of class CLASS32\n");
+ return 0;
+ }
+
+ if(ehdr->e_type != ET_EXEC) {
+ kerrf("Jove does not support ELF files other than ET_EXEC\n");
+ return 0;
+ }
+
+ uint64_t entry = ehdr->e_entry;
+
+ size_t phdrc = ehdr->e_phnum;
+ elf_phdr_t *phdrs = (elf_phdr_t*)((uintptr_t)data + ehdr->e_phoff);
+
+ for(size_t phdri = 0; phdri < phdrc; phdri++)
+ {
+ elf_phdr_t *phdr = &phdrs[phdri];
+ void *pdata = (void*)phdr->p_vaddr;
+
+ vm_ensure(
+ phdr->p_vaddr,
+ phdr->p_vaddr + phdr->p_memsz,
+ (page_flags_t) {
+ .present = true,
+ .writeable = true,
+ .useraccess = true,
+ .executable = true
+ });
+
+ if(phdr->p_type == PT_LOAD)
+ {
+ kdbgf("PT_LOAD %#016X [%#016X] -> %#016X\n",
+ (uintptr_t)data + phdr->p_offset,
+ phdr->p_filesz,
+ pdata);
+ memcpy(pdata, (void*)((uintptr_t)data + phdr->p_offset), phdr->p_filesz);
+ }
+ }
+ return entry;
+}
diff --git a/arch/x86_64/gdt.c b/arch/x86_64/gdt.c
index 07a8097..614ec30 100644
--- a/arch/x86_64/gdt.c
+++ b/arch/x86_64/gdt.c
@@ -1,24 +1,10 @@
-#include "tables.h"
-#include "tss.h"
+#include "arch/x86_64/gdt.h"
+#include "arch/x86_64/tss.h"
+#include "arch/processor.h"
+#include "string.h"
+#include "print.h"
-enum
-{
- GDT_SEGMENT_KERNEL_NULL = 0,
- GDT_SEGMENT_KERNEL_CODE,
- GDT_SEGMENT_KERNEL_DATA,
-
- GDT_SEGMENT_USER_NULL,
- GDT_SEGMENT_USER_DATA,
- GDT_SEGMENT_USER_CODE,
-
- GDT_SEGMENT_TSS_LOW,
- GDT_SEGMENT_TSS_HIGH,
-
- GDT_SEGMENT_COUNT
-};
-
-__attribute__((aligned(0x1000)))
-static struct SegmentDescriptor s_gdtd[GDT_SEGMENT_COUNT] =
+static gdt_t s_baseline_gdt =
{
{ 0 }, /* Kernel NULL */
{ /* Kernel Code (64-bit RO EX DPL0) */
@@ -99,43 +85,40 @@ static struct SegmentDescriptor s_gdtd[GDT_SEGMENT_COUNT] =
},
{ 0 }
};
-static struct XDTR s_gdtr = {
- .length = sizeof(s_gdtd) - 1,
- .address = (uintptr_t)&s_gdtd
-};
-static struct TSS s_tss = {
-
-};
-
-extern void x86_64_lgdt(struct XDTR *gdtr);
-extern void x86_64_flush_tss(void);
void
-x86_64_load_gdt(void)
+gdt_setup(processor_t *proc)
{
- {
- struct SegmentDescriptor *tss_lo = &s_gdtd[GDT_SEGMENT_TSS_LOW];
- struct SegmentDescriptor *tss_hi = &s_gdtd[GDT_SEGMENT_TSS_HIGH];
- uintptr_t tssb = (uintptr_t)&s_tss;
- tss_lo->base_0_15 = tssb & 0xFFFF;
- tss_lo->base_16_23 = (tssb >> 16) & 0xFF;
- tss_lo->base_24_31 = (tssb >> 24) & 0xFF;
- tss_hi->limit_0_15 = (tssb >> 32) & 0xFFFF;
- tss_hi->base_0_15 = (tssb >> 48) & 0xFFFF;
+ memcpy(proc->_gdt, s_baseline_gdt, sizeof(gdt_t));
+ proc->_gdtr = (struct XDTR) {
+ .length = sizeof(gdt_t) - 1,
+ .address = (uintptr_t)&proc->_gdt
+ };
+ klogf("Processor %i GDT %#016X L%i\n", proc->id, proc->_gdt, proc->_gdtr.length);
+}
- size_t tssl = sizeof(struct TSS) - 1;
- tss_lo->limit_0_15 = tssl & 0xFFFF;
- tss_lo->limit_16_19 = (tssl >> 16) & 0xF;
+void
+tss_setup(processor_t *proc)
+{
+ segment_descriptor_t *tss_lo = &proc->_gdt[GDT_SEGMENT_TSS_LOW];
+ segment_descriptor_t *tss_hi = &proc->_gdt[GDT_SEGMENT_TSS_HIGH];
+ uintptr_t tssb = (uintptr_t)&(proc->_tss);
+ tss_lo->base_0_15 = tssb & 0xFFFF;
+ tss_lo->base_16_23 = (tssb >> 16) & 0xFF;
+ tss_lo->base_24_31 = (tssb >> 24) & 0xFF;
+ tss_hi->limit_0_15 = (tssb >> 32) & 0xFFFF;
+ tss_hi->base_0_15 = (tssb >> 48) & 0xFFFF;
- s_tss.iobp = sizeof(struct TSS);
- }
+ size_t tssl = sizeof(struct TSS) - 1;
+ tss_lo->limit_0_15 = tssl & 0xFFFF;
+ tss_lo->limit_16_19 = (tssl >> 16) & 0xF;
- x86_64_lgdt(&s_gdtr);
- x86_64_flush_tss();
+ proc->_tss.iobp = sizeof(struct TSS);
}
-void tss_set_rsp(uint8_t dpl, uintptr_t rsp)
+void
+tss_set_rsp(struct TSS *tss, uint8_t dpl, uintptr_t rsp)
{
- s_tss.rsp[dpl][0] = rsp & 0xFFFFFFFF;
- s_tss.rsp[dpl][1] = (rsp >> 32) & 0xFFFFFFFF;
+ tss->rsp[dpl][0] = rsp & 0xFFFFFFFF;
+ tss->rsp[dpl][1] = (rsp >> 32) & 0xFFFFFFFF;
}
diff --git a/arch/x86_64/idt.c b/arch/x86_64/idt.c
index 05dcf43..5ddda50 100644
--- a/arch/x86_64/idt.c
+++ b/arch/x86_64/idt.c
@@ -1,64 +1,25 @@
-#include "tables.h"
-#include "cpu.h"
-#include "lib/jove.h"
-#include "io/log.h"
+#include <stddef.h>
+#include "arch/x86_64/tables.h"
+#include "arch/x86_64/idt.h"
+#include "arch/processor.h"
+#include "print.h"
+#include "jove.h"
PAGEALIGN
-static struct InterruptTrapGate s_idtd[48];
-static struct Registers *(*s_int_handlers[48])(struct Registers*);
-static struct XDTR s_idtr = {
- .length = sizeof(s_idtd) - 1,
- .address = (uintptr_t)&s_idtd
-};
+static interrupt_gate_t s_idtd[48];
uint64_t __isr_err;
uint64_t __isr_num;
-void
-int_set_handler(uint8_t code, struct Registers *(*handler)(struct Registers*))
-{
- if(code >= 48) return;
- s_int_handlers[code] = handler;
-}
-
-struct Registers*
-irq_handle(struct Registers *state)
-{
- if(__isr_num < 48) {
- if(s_int_handlers[__isr_num] != NULL) {
- return s_int_handlers[__isr_num](state);
- }
- }
- klogf("Interrupt %i\nerror code %#016X\n", __isr_num, __isr_err);
- klogf("IP: %#016X\n", state->ip);
- klogf("AX: %#016X\n", state->ax);
- klogf("BX: %#016X\n", state->bx);
- klogf("CX: %#016X\n", state->cx);
- klogf("DX: %#016X\n", state->dx);
- klogf("SI: %#016X\n", state->si);
- klogf("DI: %#016X\n", state->di);
- klogf("BP: %#016X\n", state->bp);
- klogf("R8: %#016X\n", state->r8);
- klogf("R9: %#016X\n", state->r9);
- klogf("R10: %#016X\n", state->r10);
- klogf("R11: %#016X\n", state->r11);
- klogf("R12: %#016X\n", state->r12);
- klogf("R13: %#016X\n", state->r13);
- klogf("R14: %#016X\n", state->r14);
- klogf("R15: %#016X\n", state->r15);
- kpanic("Unhandled exception\n");
- return state;
-}
-
extern void x86_64_lidt(struct XDTR *idtr);
void
-x86_64_load_idt(void)
+idt_init(void)
{
extern uintptr_t __isr_stubs[22];
for(int i = 0; i < 22; i++)
{
uintptr_t base = __isr_stubs[i];
- s_idtd[i] = (struct InterruptTrapGate){
+ s_idtd[i] = (interrupt_gate_t){
.base_0_15 = (base & 0xFFFF),
.segment_selector = 0x8,
.ist = 0,
@@ -71,7 +32,14 @@ x86_64_load_idt(void)
.base_32_63 = (base >> 32) & 0xFFFFFFFF,
.resv = 0
};
- klogf("INT %2i : %#016X (%016X)\n", i, base, s_idtd[i]);
}
- x86_64_lidt(&s_idtr);
+}
+
+void
+idt_setup(processor_t *proc)
+{
+ proc->_idtr = (struct XDTR){
+ .length = sizeof(s_idtd) - 1,
+ .address = (uintptr_t)s_idtd
+ };
}
diff --git a/arch/x86_64/int_handler.c b/arch/x86_64/int_handler.c
new file mode 100644
index 0000000..3674d1c
--- /dev/null
+++ b/arch/x86_64/int_handler.c
@@ -0,0 +1,52 @@
+#include "arch/x86_64/idt.h"
+#include "arch/processor.h"
+#include "print.h"
+#include "jove.h"
+#include "assert.h"
+#include <stddef.h>
+
+static int_handler_t s_handlers[48];
+
+int_state_t*
+irq_handle(int_state_t *state)
+{
+ if(__isr_num < 48) {
+ if(s_handlers[__isr_num] != NULL) {
+ return s_handlers[__isr_num](state);
+ }
+ }
+
+ klogf("Interrupt %i\nerror code %#016X\n", __isr_num, __isr_err);
+ int_state_print(state);
+ kpanic("Unhandled exception\n");
+ return state;
+}
+
+void
+int_handler_set(uint8_t i, int_handler_t handler)
+{
+ assert(i < 48);
+ s_handlers[i] = handler;
+}
+
+int_handler_t
+int_handler_get(uint8_t i)
+{
+ assert(i < 48);
+ return s_handlers[i];
+}
+
+void
+int_state_print(int_state_t *state)
+{
+ if(!state) return;
+ klogf("IP: %016p\n", state->ip);
+ klogf("AX: %016p BX: %016p\n", state->ax, state->bx);
+ klogf("CX: %016p DX: %016p\n", state->cx, state->dx);
+ klogf("SI: %016p DI: %016p\n", state->si, state->di);
+ klogf("SP: %016p BP: %016p\n", state->sp, state->bp);
+ klogf("R8: %016p R9: %016p\n", state->r8, state->r9);
+ klogf("R10: %016p R11: %016p\n", state->r10, state->r11);
+ klogf("R12: %016p R13: %016p\n", state->r12, state->r13);
+ klogf("R14: %016p R15: %016p\n", state->r14, state->r15);
+}
diff --git a/arch/x86_64/interrupt.h b/arch/x86_64/interrupt.h
deleted file mode 100644
index 9ae63f9..0000000
--- a/arch/x86_64/interrupt.h
+++ /dev/null
@@ -1,11 +0,0 @@
-#ifndef JOVE_ARCH_INTERRUPT_H
-#define JOVE_ARCH_INTERRUPT_H 1
-
-#include "cpu.h"
-
-/**Register a handler for a specified interrupt code.
- * @param code interrupt to handle.
- * @param handler to call.*/
-void int_set_handler(unsigned char code, struct Registers *(*handler)(struct Registers*));
-
-#endif
diff --git a/arch/x86_64/kexec.c b/arch/x86_64/kexec.c
new file mode 100644
index 0000000..406e8e7
--- /dev/null
+++ b/arch/x86_64/kexec.c
@@ -0,0 +1,50 @@
+#include "tasking.h"
+#include "memory.h"
+#include "string.h"
+#include "arch/cpu.h"
+
+#define UMODE_STACK_TOP (USERLAND_MEMORY_LIMIT)
+#define UMODE_STACKW PAGE_SIZE
+
+static void*
+s_umode_stack(void)
+{
+ uintptr_t top = UMODE_STACK_TOP;
+ uintptr_t bottom = top - (UMODE_STACKW - 1);
+
+ vm_ensure(bottom, top, (page_flags_t) {
+ .present = true,
+ .writeable = true,
+ .useraccess = true,
+ .executable = false
+ });
+ return (void*)top;
+}
+
+static inline void*
+s_stack_push(void *sp, void *data, size_t len)
+{
+ sp = (void*)((uintptr_t)sp - len);
+ memcpy(sp, data, len);
+ return sp;
+}
+
+void
+kexec(void *ip, int kargc, char **kargv, int kenvc, char **kenvp)
+{
+ /* Allocate a page for the user stack. */
+ void *sp = s_umode_stack();
+
+ intmax_t argc = kargc;
+ sp = s_stack_push(sp, &argc, sizeof(intmax_t));
+
+ /* TODO: Proper argv, envp*/
+ sp = s_stack_push(sp, &kargv, sizeof(char**));
+
+ intmax_t envc = kenvc;
+ sp = s_stack_push(sp, &envc, sizeof(intmax_t));
+
+ sp = s_stack_push(sp, &kenvp, sizeof(char**));
+
+ umode_enter(ip, sp);
+}
diff --git a/arch/x86_64/msr.c b/arch/x86_64/msr.c
new file mode 100644
index 0000000..1891126
--- /dev/null
+++ b/arch/x86_64/msr.c
@@ -0,0 +1,67 @@
+#include "arch/x86_64/msr.h"
+
+void
+msr_write(uint32_t msr, uint64_t v)
+{
+ __asm__ volatile("wrmsr":: "a"(v), "d"(v >> 32), "c"(msr));
+}
+
+uint64_t
+msr_read(uint32_t msr)
+{
+ uint32_t lo, hi;
+ __asm__ volatile("rdmsr": "=a"(lo), "=d"(hi): "c"(msr));
+ uint64_t v = hi;
+ v = (v << 32) | lo;
+ return v;
+}
+
+void
+msr_efer_write(msr_efer_t v)
+{
+ msr_write(MSR_EFER, *((uintptr_t*)&v));
+}
+
+msr_efer_t
+msr_efer_read(void)
+{
+ uint64_t rawv = msr_read(MSR_EFER);
+ return *((msr_efer_t*)&rawv);
+}
+
+void
+msr_star_write(msr_star_t v)
+{
+ msr_write(MSR_STAR, *((uintptr_t*)&v));
+}
+
+msr_star_t
+msr_star_read(void)
+{
+ uint64_t rawv = msr_read(MSR_STAR);
+ return *((msr_star_t*)&rawv);
+}
+
+void
+msr_lstar_write(msr_lstar_t v)
+{
+ msr_write(MSR_LSTAR, v);
+}
+
+msr_lstar_t
+msr_lstar_read(void)
+{
+ return msr_read(MSR_LSTAR);
+}
+
+void
+msr_gsbase_write(uintptr_t gsbase)
+{
+ msr_write(MSR_GSBASE, gsbase);
+}
+
+uintptr_t
+msr_gsbase_read(void)
+{
+ return msr_read(MSR_GSBASE);
+}
diff --git a/arch/x86_64/page.c b/arch/x86_64/page.c
new file mode 100644
index 0000000..d816f66
--- /dev/null
+++ b/arch/x86_64/page.c
@@ -0,0 +1,173 @@
+#include "arch/x86_64/page.h"
+#include "klib/rbtree.h"
+#include "arch/processor.h"
+#include "string.h"
+#include "jove.h"
+#include "memory.h"
+#include "print.h"
+
+extern void *_kernel_end;
+
+struct PageDirectoryListEntry {
+ struct PageDirectoryListEntry *next;
+ page_directory_t pd;
+};
+
+struct PageStateCache {
+ page_directory_t *pd;
+ size_t pmli[4];
+ pmle_t *pml[4];
+} s_state_cache;
+
+const uintptr_t USERLAND_MEMORY_BASE = KiB;
+const uintptr_t USERLAND_MEMORY_LIMIT = 0x00007FFFFFFFFFFFULL;
+
+const uintptr_t PHYSMAP_MEMORY_BASE = 0xFFFF800000000000ULL;
+const uintptr_t PHYSMAP_MEMORY_LIMIT = PHYSMAP_MEMORY_BASE + (1 * GiB);
+
+const uintptr_t KERNEL_MEMORY_BASE = 0xFFFF800000000000ULL;
+const uintptr_t KERNEL_MEMORY_LIMIT = 0xFFFFFFFFFFFFFFFFULL;
+
+static size_t
+s_paging_pmli(size_t l, uintptr_t addr)
+{
+ size_t shift = (PAGE_SHIFT + (9 * l));
+ return (addr & (0x1FFULL << shift)) >> shift;
+}
+
+static pmle_t*
+s_paging_fetch_table(pmle_t *pml, size_t l, uintptr_t virt)
+{
+ size_t pmli = s_paging_pmli(l, virt);
+ if(s_state_cache.pmli[l] == pmli && s_state_cache.pml[l] != NULL)
+ return s_state_cache.pml[l];
+
+ pmle_t entry = pml[pmli];
+ bool entry_new = false;
+ if(!entry.p) {
+ entry_new = true;
+ entry.value = pm_alloc(1);
+ entry.p = 1;
+ entry.rw = 1;
+ entry.us = 1;
+ pml[pmli] = entry;
+ }
+ pmle_t *table = (pmle_t*)(pm_tovirt(entry.paddr << PAGE_SHIFT));
+ if(entry_new) memset(table, 0, PAGE_SIZE);
+
+ s_state_cache.pmli[l] = pmli;
+ s_state_cache.pml[l] = table;
+ return table;
+}
+
+static void
+s_paging_cache_tables(page_directory_t *pd, uintptr_t virt)
+{
+ pmle_t *pml4 = pd->pml;
+ if(s_state_cache.pd != pd) memset(&s_state_cache, 0, sizeof(s_state_cache));
+
+ pmle_t *pml3 = s_paging_fetch_table(pml4, 3, virt);
+ pmle_t *pml2 = s_paging_fetch_table(pml3, 2, virt);
+ pmle_t *pml1 = s_paging_fetch_table(pml2, 1, virt);
+}
+
+static pmle_t*
+s_paging_get_table(pmle_t *pt, size_t l, uintptr_t virt)
+{
+ if(pt == NULL) return NULL;
+ size_t pmli = s_paging_pmli(l, virt);
+ if(s_state_cache.pmli[l] == pmli && s_state_cache.pml[l] != NULL)
+ return s_state_cache.pml[l];
+
+ pmle_t entry = pt[pmli];
+ if(!entry.p) return NULL;
+ return (pmle_t*)(pm_tovirt(entry.paddr << PAGE_SHIFT));
+}
+
+page_mapping_t
+vm_pd_mapping_get(page_directory_t *pd, uintptr_t addr)
+{
+ spinlock_acquire(pd->lock);
+ page_mapping_t mapping = { 0 };
+
+ pmle_t *pml4 = pd->pml;
+ if(s_state_cache.pd != pd) memset(&s_state_cache, 0, sizeof(s_state_cache));
+
+ pmle_t *pml3 = s_paging_get_table(pml4, 3, addr);
+ pmle_t *pml2 = s_paging_get_table(pml3, 2, addr);
+ pmle_t *pml1 = s_paging_get_table(pml2, 1, addr);
+ if(pml1 == NULL) goto release_return;
+
+ size_t pml1i = s_paging_pmli(0, addr);
+ pmle_t pml1e = pml1[pml1i];
+
+ mapping = (page_mapping_t) {
+ .phys = (pml1e.paddr << PAGE_SHIFT) & ~PAGE_MASK,
+ .pf = {
+ .present = pml1e.p,
+ .writeable = pml1e.rw,
+ .useraccess = pml1e.us,
+ .executable = !pml1e.xd
+ }
+ };
+release_return:
+ spinlock_release(pd->lock);
+ return mapping;
+}
+
+page_mapping_t vm_mapping_get(uintptr_t addr)
+{ return vm_pd_mapping_get(pd_current(), addr); }
+
+void
+mem_set_mapping_as(page_directory_t *pd, page_mapping_t mapping, uintptr_t virt)
+{
+ spinlock_acquire(pd->lock);
+ s_paging_cache_tables(pd, virt);
+ pmle_t *pml1 = s_state_cache.pml[0];
+ size_t pml1i = s_paging_pmli(0, virt);
+
+ pml1[pml1i] = (pmle_t) {
+ .p = mapping.pf.present,
+ .rw = mapping.pf.writeable,
+ .us = mapping.pf.useraccess,
+ .xd = !mapping.pf.executable,
+ .paddr = mapping.phys >> PAGE_SHIFT
+ };
+ spinlock_release(pd->lock);
+}
+
+void mem_set_mapping(page_mapping_t mapping, uintptr_t virt)
+{ mem_set_mapping_as(pd_current(), mapping, virt); }
+
+void
+vm_pd_ensure(page_directory_t *pd, uintptr_t from, uintptr_t to, page_flags_t flg)
+{
+ spinlock_acquire(pd->lock);
+ from &= ~(PAGE_SIZE - 1);
+ to += (to % PAGE_SIZE > 0 ? (PAGE_SIZE - (to % PAGE_SIZE)) : 0);
+
+ if(to < from) to = from;
+ size_t pages = (to - from) >> PAGE_SHIFT;
+ if(pages == 0) pages = 1;
+ for(size_t i = 0; i < pages; i++) {
+ uintptr_t waddr = from + (i << PAGE_SHIFT);
+ s_paging_cache_tables(pd, waddr);
+ pmle_t *pml1 = s_state_cache.pml[1];
+ size_t pml1i = s_paging_pmli(0, waddr);
+
+ if(!pml1[pml1i].p) {
+ physptr_t phys = pm_alloc(1);
+ pml1[pml1i] = (pmle_t) {
+ .p = flg.present,
+ .rw = flg.writeable,
+ .us = flg.useraccess,
+ .xd = !flg.executable,
+ .paddr = phys >> PAGE_SHIFT
+ };
+ }
+ }
+ spinlock_release(pd->lock);
+}
+
+void vm_ensure(uintptr_t from, uintptr_t to, page_flags_t flg)
+{ vm_pd_ensure(pd_current(), from, to, flg); }
diff --git a/arch/x86_64/page.h b/arch/x86_64/page.h
new file mode 100644
index 0000000..28dfad2
--- /dev/null
+++ b/arch/x86_64/page.h
@@ -0,0 +1,28 @@
+#ifndef JOVE_ARCH_x86_64_PAGING_H
+#define JOVE_ARCH_x86_64_PAGING_H 1
+
+#include <stdint.h>
+
+typedef union PageMappingLevelEntry
+{
+ struct {
+ uint8_t p : 1; /* Present */
+ uint8_t rw : 1; /* Read/write. 0 for RO.*/
+ uint8_t us : 1; /* User/supervisor. 0 for DPL3 forbid */
+ uint8_t pwt : 1;
+ uint8_t pcd : 1;
+ uint8_t a : 1; /* Accessed */
+ uint8_t d : 1; /* Dirty */
+ uint8_t ps_pat : 1;
+ uint8_t g : 1; /* Global */
+ uint8_t _r0 : 2;
+ uint8_t r : 1;
+ uint64_t paddr : 35;
+ uint8_t _r1;
+ uint8_t pk : 4;
+ uint8_t xd : 1;
+ }__attribute__((packed));
+ uint64_t value;
+} __attribute__((packed)) pmle_t;
+
+#endif
diff --git a/arch/x86_64/pagedirectory.c b/arch/x86_64/pagedirectory.c
new file mode 100644
index 0000000..cdfffeb
--- /dev/null
+++ b/arch/x86_64/pagedirectory.c
@@ -0,0 +1,138 @@
+#include "arch/page.h"
+#include "arch/processor.h"
+#include "klib/rbtree.h"
+#include "boot.h"
+#include "memory.h"
+#include "string.h"
+#include "print.h"
+
+PAGEALIGN static uint64_t s_kernel_initial_pml4[512];
+PAGEALIGN static uint64_t s_kernel_initial_pml3[2][512];
+PAGEALIGN static uint64_t s_kernel_initial_pml2[2][512];
+PAGEALIGN static uint64_t s_kernel_initial_pml1[2][512];
+static page_directory_t s_kernel_initial_pd;
+
+static rbtree_t s_page_directories;
+static intmax_t s_next_pdid = 0;
+
+page_directory_t*
+pd_new()
+{
+ page_directory_t newpd = {
+ .id = s_next_pdid++,
+ .phys = pm_alloc(1),
+ };
+ newpd.pml = (void*)pm_tovirt(newpd.phys);
+ for(size_t i = 256; i < 512; i++) {
+ pmle_t *kpe = &(s_kernel_initial_pd.pml)[i];
+ pmle_t *ppe = &(newpd.pml)[i];
+ *ppe = *kpe;
+ }
+
+ return rbtree_insert(&s_page_directories, newpd.id, &newpd);
+}
+
+static void
+s_pd_dup_pml(pmle_t *src, pmle_t *dest, size_t l, size_t i)
+{
+ pmle_t srce = src[i];
+ if(!srce.p) return;
+
+ dest[i] = srce;
+ dest[i].paddr = pm_alloc(1);
+ pmle_t dste = dest[i];
+
+ pmle_t *srct = (pmle_t*)pm_tovirt(srce.paddr << PAGE_SHIFT);
+ pmle_t *dstt = (pmle_t*)pm_tovirt(dste.paddr << PAGE_SHIFT);
+
+ if(l == 0) {
+ memcpy(dstt, srct, PAGE_SIZE);
+ return;
+ }
+
+ for(i = 0; i < 512; i++) {
+ dstt[i] = srct[i];
+ if(!srct[i].p) continue;
+ s_pd_dup_pml(srct, dstt, l - 1, i);
+ }
+}
+
+page_directory_t*
+pd_dup(page_directory_t *pd)
+{
+ page_directory_t *newpd = pd_new();
+ for(size_t i = 0; i < 256; i++) {
+ s_pd_dup_pml(pd->pml, newpd->pml, 3, i);
+ }
+ return newpd;
+}
+
+page_directory_t*
+pd_get(pdid_t pdid)
+{
+ return rbtree_find(&s_page_directories, pdid);
+}
+
+void
+pd_switch(page_directory_t *pd)
+{
+ processor_t *pc = processor_current();
+ if(pc->pd == pd) return;
+ pc->pd = pd;
+ __asm__ volatile("movq %0, %%cr3":: "r"(pd->phys));
+}
+
+void
+vm_setup_early(void)
+{
+ memset(s_kernel_initial_pml4, 0, PAGE_SIZE);
+ memset(s_kernel_initial_pml3, 0, 2 * PAGE_SIZE);
+ memset(s_kernel_initial_pml2, 0, 2 * PAGE_SIZE);
+ memset(s_kernel_initial_pml1, 0, PAGE_SIZE);
+ s_kernel_initial_pd = (page_directory_t){
+ .phys = vm_tophys_koff((uintptr_t)&s_kernel_initial_pml4),
+ .pml = (pmle_t*)&s_kernel_initial_pml4,
+ .id = s_next_pdid++
+ };
+ processor_current()->pd = &s_kernel_initial_pd;
+
+ /* Map first few GiBs */
+ s_kernel_initial_pml4[256] =
+ vm_tophys_koff((uintptr_t)&s_kernel_initial_pml3[0])
+ | 3;
+ s_kernel_initial_pml3[0][0] =
+ vm_tophys_koff((uintptr_t)&s_kernel_initial_pml2[0])
+ | 3;
+ for(int i = 0; i < 512; i++) {
+ s_kernel_initial_pml2[0][i] = (i * (PAGE_SIZE * 512)) | 0x80 | 3;
+ }
+
+ size_t kernel_pml3e = (_kernel_virtual_base >> (30)) % 512;
+ size_t kernel_pml2e = (_kernel_virtual_base >> (21)) % 512;
+ size_t kernel_npages = ((((uintptr_t)&_kernel_end) - _kernel_virtual_base) >> 12) + 1;
+ klogf("Kernel has %i pages\n", kernel_npages);
+
+ /* Map kernel pages */
+ s_kernel_initial_pml4[511] =
+ vm_tophys_koff((uintptr_t)&s_kernel_initial_pml3[1]) | 3;
+ s_kernel_initial_pml3[1][kernel_pml3e] =
+ vm_tophys_koff((uintptr_t)&s_kernel_initial_pml2[1]) | 3;
+ s_kernel_initial_pml2[1][kernel_pml2e] =
+ vm_tophys_koff((uintptr_t)&s_kernel_initial_pml1[0]) | 3;
+ for(size_t i = 0; i < kernel_npages; i++) {
+ s_kernel_initial_pml1[0][i] = (i * PAGE_SIZE) + boot_kernel_physical_address | 3;
+ }
+
+ extern int_state_t *_pagefault_handler(int_state_t*);
+ int_handler_set(14, _pagefault_handler);
+ __asm__ volatile("mov %0, %%cr3":: "r"(s_kernel_initial_pd.phys));
+}
+
+void
+vm_setup(void)
+{
+ rbtree_new(&s_page_directories, page_directory_t);
+
+ processor_t *proc = processor_current();
+ pd_switch(pd_new());
+}
diff --git a/arch/x86_64/pagefault.c b/arch/x86_64/pagefault.c
new file mode 100644
index 0000000..5bba20b
--- /dev/null
+++ b/arch/x86_64/pagefault.c
@@ -0,0 +1,39 @@
+#include "arch/x86_64/idt.h"
+#include "arch/processor.h"
+#include "memory.h"
+#include "print.h"
+#include "jove.h"
+
+#include <stdbool.h>
+
+int_state_t*
+_pagefault_handler(int_state_t *state)
+{
+ extern uint64_t __isr_err;
+
+ uintptr_t fault_addr = 0;
+ __asm__ volatile("movq %%cr2, %0": "=r"(fault_addr));
+
+ if(fault_addr < USERLAND_MEMORY_LIMIT) {
+ // Check if there is an entry in the exception return table.
+ processor_t *proc = processor_current();
+ if(proc->ert_i > 0) {
+ state->ip = (uintptr_t)_exrtab_pop();
+ return state;
+ }
+ }
+
+ bool present = __isr_err & 1;
+ bool write = __isr_err & 2;
+ bool user = __isr_err & 4;
+
+ klogf("Page fault at %016X\n", fault_addr);
+ klogf("%s %s from a %s address\n",
+ user ? "user" : "kernel",
+ write ? "wrote" : "read",
+ present ? "present" : "non-present");
+
+ int_state_print(state);
+ kpanic("Unhandled page fault\n");
+ return state;
+}
diff --git a/arch/x86_64/paging.c b/arch/x86_64/paging.c
deleted file mode 100644
index dc27ca2..0000000
--- a/arch/x86_64/paging.c
+++ /dev/null
@@ -1,257 +0,0 @@
-#include "paging.h"
-#include "interrupt.h"
-#include "io/log.h"
-#include "lib/jove.h"
-#include "lib/string.h"
-#include "lib/hashtable.h"
-#include "mem/memory.h"
-#include "boot/boot.h"
-
-extern void *_kernel_end;
-
-PAGEALIGN static uint64_t s_kernel_initial_pml4[512];
-PAGEALIGN static uint64_t s_kernel_initial_pml3[2][512];
-PAGEALIGN static uint64_t s_kernel_initial_pml2[2][512];
-PAGEALIGN static uint64_t s_kernel_initial_pml1[2][512];
-static intmax_t s_next_pdid = 0;
-static page_directory_t s_kernel_initial_pd;
-
-page_directory_t *current_page_directory;
-
-struct PageStateCache {
- page_directory_t *pd;
- size_t pmli[4];
- pmle_t *pml[4];
-} s_state_cache;
-
-physptr_t
-mem_linear_tophys_koffset(uintptr_t virt)
-{
- return (virt - boot_kernel_virtual_base) + boot_kernel_physical_address;
-}
-
-uintptr_t
-mem_phys_tolinear(physptr_t phys)
-{
- return (uintptr_t)(phys + 0xFFFF800000000000ULL);
-}
-
-static size_t
-s_paging_pmli(size_t l, uintptr_t addr)
-{
- size_t shift = (12 + (9 * l));
- return (addr & (0x1FFULL << shift)) >> shift;
-}
-
-static pmle_t*
-s_paging_fetch_table(pmle_t *pml, size_t l, uintptr_t virt)
-{
- size_t pmli = s_paging_pmli(l, virt);
- if(s_state_cache.pmli[l] == pmli && s_state_cache.pml[l] != NULL)
- return s_state_cache.pml[l];
-
- pmle_t entry = pml[pmli];
- bool entry_new = false;
- if(!entry.p) {
- entry_new = true;
- entry.value = mem_phys_alloc(1);
- entry.p = 1;
- entry.rw = 1;
- entry.us = 1;
- pml[pmli] = entry;
- }
- pmle_t *table = (pmle_t*)(mem_phys_tolinear(entry.paddr << 12));
- if(entry_new) memset(table, 0, PAGE_SIZE);
-
- s_state_cache.pmli[l] = pmli;
- s_state_cache.pml[l] = table;
- return table;
-}
-
-static void
-s_paging_cache_tables(page_directory_t *pd, uintptr_t virt)
-{
- pmle_t *pml4 = (pmle_t*)pd->virt;
- if(s_state_cache.pd != pd) memset(&s_state_cache, 0, sizeof(s_state_cache));
-
- pmle_t *pml3 = s_paging_fetch_table(pml4, 3, virt);
- pmle_t *pml2 = s_paging_fetch_table(pml3, 2, virt);
- pmle_t *pml1 = s_paging_fetch_table(pml2, 1, virt);
-}
-
-static pmle_t*
-s_paging_get_table(pmle_t *pt, size_t l, uintptr_t virt)
-{
- if(pt == NULL) return NULL;
- size_t pmli = s_paging_pmli(l, virt);
- if(s_state_cache.pmli[l] == pmli && s_state_cache.pml[l] != NULL)
- return s_state_cache.pml[l];
-
- pmle_t entry = pt[pmli];
- if(!entry.p) return NULL;
- return (pmle_t*)(mem_phys_tolinear(entry.paddr << 12));
-}
-
-page_mapping_t
-mem_get_mapping_as(page_directory_t *pd, uintptr_t addr)
-{
- spinlock_acquire(pd->lock);
- page_mapping_t mapping = { 0 };
-
- pmle_t *pml4 = (pmle_t*)pd->virt;
- if(s_state_cache.pd != pd) memset(&s_state_cache, 0, sizeof(s_state_cache));
-
- pmle_t *pml3 = s_paging_get_table(pml4, 3, addr);
- pmle_t *pml2 = s_paging_get_table(pml3, 2, addr);
- pmle_t *pml1 = s_paging_get_table(pml2, 1, addr);
- if(pml1 == NULL) goto release_return;
-
- size_t pml1i = s_paging_pmli(0, addr);
- pmle_t pml1e = pml1[pml1i];
-
- mapping = (page_mapping_t) {
- .phys = (pml1e.paddr << 12) & ~PAGE_MASK,
- .pf = {
- .present = pml1e.p,
- .writeable = pml1e.rw,
- .useraccess = pml1e.us,
- .executable = !pml1e.xd
- }
- };
-release_return:
- spinlock_release(pd->lock);
- return mapping;
-}
-
-page_mapping_t mem_get_mapping(uintptr_t addr)
-{ return mem_get_mapping_as(current_page_directory, addr); }
-
-
-bool
-mem_check_ptr(const void *ptr)
-{
- return mem_get_mapping((uintptr_t)ptr).pf.present != 0;
-}
-
-void
-mem_set_mapping_as(page_directory_t *pd, page_mapping_t mapping, uintptr_t virt)
-{
- spinlock_acquire(pd->lock);
- s_paging_cache_tables(pd, virt);
- pmle_t *pml1 = s_state_cache.pml[0];
- size_t pml1i = s_paging_pmli(0, virt);
-
- pml1[pml1i] = (pmle_t) {
- .p = mapping.pf.present,
- .rw = mapping.pf.writeable,
- .us = mapping.pf.useraccess,
- .xd = !mapping.pf.executable,
- .paddr = mapping.phys >> 12
- };
- spinlock_release(pd->lock);
-}
-
-void mem_set_mapping(page_mapping_t mapping, uintptr_t virt)
-{ mem_set_mapping_as(current_page_directory, mapping, virt); }
-
-void
-mem_ensure_range_as(page_directory_t *pd, uintptr_t from, uintptr_t to, page_flags_t flg)
-{
- spinlock_acquire(pd->lock);
- from &= ~(PAGE_SIZE - 1);
-
- if(to < from) to = from;
- size_t pages = (to - from) >> 12;
- if(pages == 0) pages = 1;
- for(size_t i = 0; i < pages; i++) {
- uintptr_t waddr = from + (i << 12);
- s_paging_cache_tables(pd, waddr);
- pmle_t *pml1 = s_state_cache.pml[1];
- size_t pml1i = s_paging_pmli(0, waddr);
-
- if(!pml1[pml1i].p) {
- physptr_t phys = mem_phys_alloc(1);
- pml1[pml1i] = (pmle_t) {
- .p = flg.present,
- .rw = flg.writeable,
- .us = flg.useraccess,
- .xd = !flg.executable,
- .paddr = phys >> 12
- };
- }
- }
- spinlock_release(pd->lock);
-}
-
-void mem_ensure_range(uintptr_t from, uintptr_t to, page_flags_t flg)
-{ mem_ensure_range_as(current_page_directory, from, to, flg); }
-
-struct Registers*
-s_pagefault_handler(struct Registers *state)
-{
- extern uint64_t __isr_err;
-
- uintptr_t fault_addr = 0;
- __asm__ volatile("movq %%cr2, %0": "=r"(fault_addr));
-
- bool present = __isr_err & 1;
- bool write = __isr_err & 2;
- bool user = __isr_err & 4;
- bool fetch = __isr_err & 16;
-
- klogf("Page fault at %016X\n", fault_addr);
- klogf("%s %s from a %s address\n",
- user ? "user" : "kernel",
- write ? "wrote" : "read",
- present ? "present" : "non-present");
-
- kpanic("Unhandled page fault at %016X\n", state->ip);
- return state;
-}
-
-void
-mem_paging_setup(void)
-{
- memset(s_kernel_initial_pml4, 0, PAGE_SIZE);
- memset(s_kernel_initial_pml3, 0, 2 * PAGE_SIZE);
- memset(s_kernel_initial_pml2, 0, 2 * PAGE_SIZE);
- memset(s_kernel_initial_pml1, 0, PAGE_SIZE);
- s_kernel_initial_pd = (page_directory_t){
- .phys = mem_linear_tophys_koffset((uintptr_t)&s_kernel_initial_pml4),
- .virt = (union PageEntry*)&s_kernel_initial_pml4,
- .ref = 1,
- .id = s_next_pdid++
- };
- current_page_directory = &s_kernel_initial_pd;
-
- /* Map first few GiBs */
- s_kernel_initial_pml4[256] =
- mem_linear_tophys_koffset((uintptr_t)&s_kernel_initial_pml3[0])
- | 3;
- s_kernel_initial_pml3[0][0] =
- mem_linear_tophys_koffset((uintptr_t)&s_kernel_initial_pml2[0])
- | 3;
- for(int i = 0; i < 512; i++) {
- s_kernel_initial_pml2[0][i] = (i * (PAGE_SIZE * 512)) | 0x80 | 3;
- }
-
- size_t kernel_pml4e = (boot_kernel_virtual_base >> (39));
- size_t kernel_pml3e = (boot_kernel_virtual_base >> (30)) % 512;
- size_t kernel_pml2e = (boot_kernel_virtual_base >> (21)) % 512;
- size_t kernel_npages = ((((uintptr_t)&_kernel_end) - boot_kernel_virtual_base) >> 12) + 1;
- klogf("Kernel has %i pages\n", kernel_npages);
-
- /* Map kernel pages */
- s_kernel_initial_pml4[511] =
- mem_linear_tophys_koffset((uintptr_t)&s_kernel_initial_pml3[1]) | 3;
- s_kernel_initial_pml3[1][kernel_pml3e] =
- mem_linear_tophys_koffset((uintptr_t)&s_kernel_initial_pml2[1]) | 3;
- s_kernel_initial_pml2[1][kernel_pml2e] =
- mem_linear_tophys_koffset((uintptr_t)&s_kernel_initial_pml1[0]) | 3;
- for(int i = 0; i < kernel_npages; i++) {
- s_kernel_initial_pml1[0][i] = (i * PAGE_SIZE) + boot_kernel_physical_address | 3;
- }
-
- int_set_handler(14, s_pagefault_handler);
- __asm__ volatile("mov %0, %%cr3":: "r"(s_kernel_initial_pd.phys));
-}
diff --git a/arch/x86_64/paging.h b/arch/x86_64/paging.h
deleted file mode 100644
index 28dfad2..0000000
--- a/arch/x86_64/paging.h
+++ /dev/null
@@ -1,28 +0,0 @@
-#ifndef JOVE_ARCH_x86_64_PAGING_H
-#define JOVE_ARCH_x86_64_PAGING_H 1
-
-#include <stdint.h>
-
-typedef union PageMappingLevelEntry
-{
- struct {
- uint8_t p : 1; /* Present */
- uint8_t rw : 1; /* Read/write. 0 for RO.*/
- uint8_t us : 1; /* User/supervisor. 0 for DPL3 forbid */
- uint8_t pwt : 1;
- uint8_t pcd : 1;
- uint8_t a : 1; /* Accessed */
- uint8_t d : 1; /* Dirty */
- uint8_t ps_pat : 1;
- uint8_t g : 1; /* Global */
- uint8_t _r0 : 2;
- uint8_t r : 1;
- uint64_t paddr : 35;
- uint8_t _r1;
- uint8_t pk : 4;
- uint8_t xd : 1;
- }__attribute__((packed));
- uint64_t value;
-} __attribute__((packed)) pmle_t;
-
-#endif
diff --git a/arch/x86_64/processor.c b/arch/x86_64/processor.c
new file mode 100644
index 0000000..44d437a
--- /dev/null
+++ b/arch/x86_64/processor.c
@@ -0,0 +1,40 @@
+#include "arch/processor.h"
+#include "arch/cpu.h"
+#include "arch/x86_64/msr.h"
+
+processor_t s_processors[PROCESSOR_MAX];
+processor_t *proc_bsp = &s_processors[0];
+
+extern void x86_64_lgdt(struct XDTR *gdtr);
+extern void x86_64_flush_tss(void);
+extern void x86_64_lidt(struct XDTR *idtr);
+
+extern void gdt_setup(processor_t*);
+extern void tss_setup(processor_t*);
+extern void idt_setup(processor_t*);
+
+void
+processor_setup(processor_t *proc)
+{
+ gdt_setup(proc);
+ tss_setup(proc);
+ idt_setup(proc);
+
+ x86_64_lgdt(&proc->_gdtr);
+ x86_64_lidt(&proc->_idtr);
+ x86_64_flush_tss();
+
+ msr_gsbase_write((uintptr_t)proc);
+}
+
+processor_t*
+processor_list(void)
+{
+ return s_processors;
+}
+
+processor_t*
+processor_current(void)
+{
+ return (processor_t*)msr_gsbase_read();
+}
diff --git a/arch/x86_64/restore.S b/arch/x86_64/restore.S
new file mode 100644
index 0000000..f4aa939
--- /dev/null
+++ b/arch/x86_64/restore.S
@@ -0,0 +1,20 @@
+.global _arch_context_restore
+.type _arch_context_restore @function
+_arch_context_restore:
+ movq %rdi, %rsp
+ popq %r15
+ popq %r14
+ popq %r13
+ popq %r12
+ popq %r11
+ popq %r10
+ popq %r9
+ popq %r8
+ popq %rbp
+ popq %rdi
+ popq %rsi
+ popq %rdx
+ popq %rcx
+ popq %rbx
+ popq %rax
+ retq
diff --git a/arch/x86_64/savestate.S b/arch/x86_64/savestate.S
deleted file mode 100644
index c89933b..0000000
--- a/arch/x86_64/savestate.S
+++ /dev/null
@@ -1,39 +0,0 @@
-.global saveregs
-.type saveregs @function
-saveregs:
- pushq %rax
- pushq %rbx
- pushq %rcx
- pushq %rdx
- pushq %rsi
- pushq %rdi
- pushq %rbp
- pushq %r8
- pushq %r9
- pushq %r10
- pushq %r11
- pushq %r12
- pushq %r13
- pushq %r14
- pushq %r15
- retq
-
-.global loadregs
-.type loadregs @function
-loadregs:
- popq %r15
- popq %r14
- popq %r13
- popq %r12
- popq %r11
- popq %r10
- popq %r9
- popq %r8
- popq %rbp
- popq %rdi
- popq %rsi
- popq %rdx
- popq %rcx
- popq %rbx
- popq %rax
-
diff --git a/arch/x86_64/serial.c b/arch/x86_64/serial.c
deleted file mode 100644
index 1b49e3f..0000000
--- a/arch/x86_64/serial.c
+++ /dev/null
@@ -1,62 +0,0 @@
-#include "serial.h"
-#include "uart.h"
-#include "io/log.h"
-
-static struct LogDevice s_serial_logdev =
- { .out = serial_out, .chain = NULL };
-
-bool serial_supported = true;
-
-void
-serial_setup(void)
-{
- /* Disable interrupts. */
- poutb(SERIAL_UART_COM_IER(SERIAL_UART_COM1), 0x0);
- /* Enable DLAB. */
- poutb(SERIAL_UART_COM_LCR(SERIAL_UART_COM1), 0x80);
- /* Set divisor to 3 (38400 baud)*/
- poutb(SERIAL_UART_COM_DLAB_DLL(SERIAL_UART_COM1), 3);
- poutb(SERIAL_UART_COM_DLAB_DLH(SERIAL_UART_COM1), 0);
- /* Set flags for LCR (8 bits, no parity, one stop)*/
- poutb(SERIAL_UART_COM_LCR(SERIAL_UART_COM1), 1 | 2);
- /* Enable & clear FIFO, 14-byte threshold */
- poutb(SERIAL_UART_COM_FCR(SERIAL_UART_COM1), 1 | 2 | 4 | 0xC0);
- /* Enable interrupts, set RTS/DSR. */
- poutb(SERIAL_UART_COM_MCR(SERIAL_UART_COM1), 1 | 2 | 8);
- /* Set loopback mode for testing. */
- poutb(SERIAL_UART_COM_MCR(SERIAL_UART_COM1), 2 | 4 | 8 | 0x10);
- /* Test serial output. */
- poutb(SERIAL_UART_COM1, 0xAE);
- if(pinb(SERIAL_UART_COM1) != 0xAE) {
- serial_supported = false;
- return;
- }
-
- /* Serial is not faulty.
- * No loopback, enable output 1 & 2.*/
- poutb(SERIAL_UART_COM_MCR(SERIAL_UART_COM1), 1 | 2 | 4 | 8);
-
- klog_newdev(&s_serial_logdev);
-}
-
-static ALWAYS_INLINE bool
-serial_transmit_empty(uint16_t com)
-{
- return pinb(SERIAL_UART_COM_LSR(com) & 0x20);
-}
-
-static inline void
-serial_outb(uint16_t com, uint8_t b)
-{
- if(b == '\n') serial_outb(com, '\r');
- while(!serial_transmit_empty(SERIAL_UART_COM1));
- poutb(com, b);
-}
-
-void
-serial_out(const char *s, size_t len)
-{
- if(!serial_supported) return;
- for(; len > 0; len--)
- serial_outb(SERIAL_UART_COM1, *(s++));
-}
diff --git a/arch/x86_64/serial.h b/arch/x86_64/serial.h
deleted file mode 100644
index ecb896d..0000000
--- a/arch/x86_64/serial.h
+++ /dev/null
@@ -1,27 +0,0 @@
-#ifndef JOVE_KERNEL_ARCH_x86_64_SERIAL_H
-#define JOVE_KERNEL_ARCH_x86_64_SERIAL_H 1
-
-#include <stdbool.h>
-#include <stddef.h>
-
-#define SERIAL_UART_COM1 0x3F8
-#define SERIAL_UART_COM2 0x2F8
-
-#define SERIAL_UART_COM_THR(COM) COM
-#define SERIAL_UART_COM_RBR(COM) COM
-#define SERIAL_UART_COM_DLAB_DLL(COM) COM
-#define SERIAL_UART_COM_IER(COM) (COM + 1)
-#define SERIAL_UART_COM_DLAB_DLH(COM) (COM + 1)
-#define SERIAL_UART_COM_IIR(COM) (COM + 2)
-#define SERIAL_UART_COM_FCR(COM) (COM + 2)
-#define SERIAL_UART_COM_LCR(COM) (COM + 3)
-#define SERIAL_UART_COM_MCR(COM) (COM + 4)
-#define SERIAL_UART_COM_LSR(COM) (COM + 5)
-#define SERIAL_UART_COM_MSR(COM) (COM + 6)
-#define SERIAL_UART_COM_SR(COM) (COM + 7)
-
-extern bool serial_supported;
-
-void serial_out(const char *s, size_t len);
-
-#endif
diff --git a/arch/x86_64/syscall.c b/arch/x86_64/syscall.c
deleted file mode 100644
index 0398f11..0000000
--- a/arch/x86_64/syscall.c
+++ /dev/null
@@ -1 +0,0 @@
-#include "usr/umode.h"
diff --git a/arch/x86_64/syscall_entry.c b/arch/x86_64/syscall_entry.c
new file mode 100644
index 0000000..56dedce
--- /dev/null
+++ b/arch/x86_64/syscall_entry.c
@@ -0,0 +1,25 @@
+#include "arch/processor.h"
+#include "syscall/handler.h"
+
+__attribute__((naked))
+void _syscall_entry()
+{
+ __asm__ volatile(" \
+ pushq %%r11; \
+ pushq %%rcx; \
+ swapgs; \
+ movq %%gs:%c[tcb], %%rax; \
+ movq %%rsp, %c[sp](%%rax); \
+ movq %c[ksp](%%rax), %%rsp; \
+ pushq %c[sp](%%rax); \
+ callq _syscall_handler; \
+ swapgs; \
+ popq %%rsp; \
+ popq %%rcx; \
+ popq %%r11; \
+ sysretq; \
+ "
+ :: [tcb] "i"(offsetof(processor_t, tcb)),
+ [sp] "i"(offsetof(tcb_t, sp)),
+ [ksp] "i"(offsetof(tcb_t, ksp)));
+}
diff --git a/arch/x86_64/syscall_setup.S b/arch/x86_64/syscall_setup.S
deleted file mode 100644
index 4f5c6f0..0000000
--- a/arch/x86_64/syscall_setup.S
+++ /dev/null
@@ -1,41 +0,0 @@
-.extern _kernel_task_sp
-.extern syscall_handler
-
-.global syscall_entry
-.type syscall_entry @function
-syscall_entry:
- swapgs
- movq %rsp, %rax
- movq _kernel_task_bp, %rsp
- pushq %rax
- pushq %rbp
- pushq %rcx
- pushq %r11
- call syscall_handler
- popq %r11
- popq %rcx
- popq %rbp
- popq %rsp
- swapgs
- sysretq
-
-.global syscall_setup_syscall
-.type syscall_setup_syscall @function
-syscall_setup_syscall:
- movq $0xc0000080, %rcx
- rdmsr
- or $1, %eax
- wrmsr
-
- movq $0xc0000081, %rcx
- rdmsr
- mov $0x001b0008, %edx
- wrmsr
-
- movq $0xc0000082, %rcx
- leaq syscall_entry, %rdx
- mov %edx, %eax
- shr $32, %rdx
- wrmsr
-
- retq
diff --git a/arch/x86_64/tables.c b/arch/x86_64/tables.c
deleted file mode 100644
index f530089..0000000
--- a/arch/x86_64/tables.c
+++ /dev/null
@@ -1,8 +0,0 @@
-#include "tables.h"
-
-void
-arch_tables_setup(void)
-{
- x86_64_load_gdt();
- x86_64_load_idt();
-}
diff --git a/arch/x86_64/tables.h b/arch/x86_64/tables.h
deleted file mode 100644
index e160c66..0000000
--- a/arch/x86_64/tables.h
+++ /dev/null
@@ -1,60 +0,0 @@
-#ifndef JOVE_ARCH_X86_64_TABLES_H
-#define JOVE_ARCH_X86_64_TABLES_H 1
-
-#include <stdint.h>
-
-#define CD_SEGMENT_TYPE_ACCESSED 1
-#define CD_SEGMENT_TYPE_WRITEABLE 2
-#define CD_SEGMENT_TYPE_DATA_EXPAND_DOWN 4
-#define CD_SEGMENT_TYPE_CODE_CONFORMING 4
-#define CD_SEGMENT_TYPE_CODE 8
-
-#define S_SEGMENT_TYPE_LDT 2
-#define S_SEGMENT_TYPE_TSS_AVAIL 9
-#define S_SEGMENT_TYPE_TSS_BUSY 11
-#define S_SEGMENT_TYPE_CALLGATE 12
-#define S_SEGMENT_TYPE_INT_GATE 14
-#define S_SEGMENT_TYPE_TRAP_GATE 15
-
-struct SegmentDescriptor
-{
- uint16_t limit_0_15; /* Segment limit. */
- uint16_t base_0_15; /* Segment base. */
- uint8_t base_16_23;
- uint8_t type : 4; /* Segment type. */
- uint8_t s : 1; /* Descriptor type (0 = system, 1 = code/data)*/
- uint8_t dpl : 2; /* Descriptor privilege level. */
- uint8_t p : 1; ; /* Present. */
- uint8_t limit_16_19 : 4;
- uint8_t avl : 1; /* Available for use by system software. */
- uint8_t l : 1; /* 64-bit segment (Ext). */
- uint8_t d_b : 1; /* Default operation size (0 = 16-bit, 1 = 32-bit)*/
- uint8_t g : 1; /* Granularity. */
- uint8_t base_24_31;
-}__attribute__((packed));
-
-struct InterruptTrapGate
-{
- uint16_t base_0_15;
- uint16_t segment_selector;
- uint8_t ist : 3;
- uint8_t zero_0 : 5;
- uint8_t type : 4;
- uint8_t zero_1 : 1;
- uint8_t dpl : 2;
- uint8_t p : 1;
- uint16_t base_16_31;
- uint32_t base_32_63;
- uint32_t resv;
-}__attribute__((packed));
-
-struct XDTR /* Generic table descriptor struct */
-{
- uint16_t length;
- uint64_t address;
-}__attribute__((packed));
-
-void x86_64_load_gdt(void);
-void x86_64_load_idt(void);
-
-#endif
diff --git a/arch/x86_64/tasking.c b/arch/x86_64/tasking.c
deleted file mode 100644
index 9b29330..0000000
--- a/arch/x86_64/tasking.c
+++ /dev/null
@@ -1,86 +0,0 @@
-#include "usr/tasking.h"
-#include "mem/memory.h"
-#include "io/log.h"
-#include "lib/hashtable.h"
-#include "lib/string.h"
-#include "paging.h"
-#include "cpu.h"
-#include "tss.h"
-
-struct TaskBody {
- struct Task base;
- struct Registers state;
-};
-
-struct Task *task_current;
-uintptr_t _kernel_task_bp = 0;
-
-//static struct SLinkedList s_tasks;
-static struct HashTable s_tasks;
-
-static struct SlabCache s_task_cache;
-static struct SlabCache s_kbp_cache;
-
-static tid_t s_task_id_next = 1;
-
-static size_t
-s_task_hash_func(const void* tid, size_t _)
-{
- return (tid_t)tid;
-}
-
-void
-tasking_setup(void)
-{
- hashtable_new(&s_tasks, struct TaskBody*, tid_t);
- s_tasks.hash_function = s_task_hash_func;
-
- mem_slabcache_new(&s_task_cache, "tasks", sizeof(struct TaskBody));
- mem_slabcache_new(&s_kbp_cache, "kernel stacks", 4096);
-
- struct TaskBody *ktask = mem_slab_alloc(&s_task_cache);
- *ktask = (struct TaskBody){
- .base.id = s_task_id_next++,
- .base.kbp = ((uintptr_t)mem_slab_alloc(&s_kbp_cache)) + 0xFF0,
- .base.perm = (size_t)-1,
- .base.pd = current_page_directory
- };
- hashtable_insert(&s_tasks, 0, ktask);
-
- task_current = (struct Task*)ktask;
- tss_set_rsp(0, task_current->kbp);
- _kernel_task_bp = task_current->kbp;
-}
-
-struct Task*
-task_new(struct Task *parent)
-{
- struct TaskBody *new = mem_slab_alloc(&s_task_cache);
- memcpy(new, parent, sizeof(struct TaskBody));
- new->base.id = s_task_id_next++;
-
- uintptr_t ksp_base = (uintptr_t)mem_slab_alloc(&s_kbp_cache);
- new->base.kbp = ksp_base + 0xFFF;
- new->base.perm = parent->perm;
-
- hashtable_insert(&s_tasks, (void*)new->base.id, new);
- return (struct Task*)new;
-}
-
-struct Task*
-task_get(tid_t id)
-{
- struct Task **task = hashtable_get(&s_tasks, (void*)id, struct Task*);
- if(task == NULL) return NULL;
- return *task;
-}
-
-void
-task_free(struct Task *task)
-{
- struct TaskBody *body = (struct TaskBody*)task;
- task->pd->ref--;
- task->kbp -= 0xFFF;
- mem_slab_free(&s_kbp_cache, (void*)(task->kbp));
- klogf("Need impl for task_free\n");
-}
diff --git a/arch/x86_64/tcb_prepare.c b/arch/x86_64/tcb_prepare.c
new file mode 100644
index 0000000..238d70d
--- /dev/null
+++ b/arch/x86_64/tcb_prepare.c
@@ -0,0 +1,9 @@
+#include "tasking.h"
+
+void
+tcb_prepare(tcb_t *tcb, void *ip)
+{
+ uintptr_t state[15] = { 0 };
+ tcb_stack_push(tcb, &ip, sizeof(void*));
+ tcb_stack_push(tcb, state, sizeof(state));
+}
diff --git a/arch/x86_64/tcb_switch.c b/arch/x86_64/tcb_switch.c
new file mode 100644
index 0000000..bc8808d
--- /dev/null
+++ b/arch/x86_64/tcb_switch.c
@@ -0,0 +1,18 @@
+#include "tasking.h"
+#include "arch/processor.h"
+
+extern void _arch_context_restore(uintptr_t rsp);
+
+void
+tcb_switch(tcb_t *to)
+{
+ processor_t *proc = processor_current();
+ tcb_t *from = proc->tcb;
+
+ proc->tcb = to;
+ if(from != to) {
+ pd_switch(to->pd);
+ }
+ tss_set_rsp(&proc->_tss, 0, to->ksp);
+ _arch_context_restore(to->ksp);
+}
diff --git a/arch/x86_64/tss.h b/arch/x86_64/tss.h
deleted file mode 100644
index dcdd01c..0000000
--- a/arch/x86_64/tss.h
+++ /dev/null
@@ -1,20 +0,0 @@
-#ifndef JOVE_ARCH_x86_64_TSS_H
-#define JOVE_ARCH_x86_64_TSS_H 1
-
-#include <stddef.h>
-#include <stdint.h>
-
-struct TSS
-{
- uint32_t resv0;
- uint32_t rsp[3][2];
- uint64_t resv1;
- uint32_t ist[8][2];
- uint64_t resv2;
- uint16_t resv3;
- uint16_t iobp;
-};
-
-void tss_set_rsp(uint8_t dpl, uintptr_t rsp);
-
-#endif
diff --git a/arch/x86_64/uart.h b/arch/x86_64/uart.h
deleted file mode 100644
index 8386eb7..0000000
--- a/arch/x86_64/uart.h
+++ /dev/null
@@ -1,19 +0,0 @@
-#ifndef JOVE_KERNEL_ARCH_x86_64_UART_H
-#define JOVE_KERNEL_ARCH_x86_64_UART_H 1
-
-#include "lib/jove.h"
-#include <stdint.h>
-
-ALWAYS_INLINE uint8_t pinb(uint16_t port)
-{
- uint8_t v;
- __asm__ volatile("inb %1, %0": "=a"(v): "Nd"(port));
- return v;
-}
-
-ALWAYS_INLINE void poutb(uint16_t port, uint8_t b)
-{
- __asm__ volatile("outb %0, %1":: "a"(b), "Nd"(port));
-}
-
-#endif
diff --git a/arch/x86_64/umode_enter.S b/arch/x86_64/umode_enter.S
deleted file mode 100644
index 29aa4e3..0000000
--- a/arch/x86_64/umode_enter.S
+++ /dev/null
@@ -1,8 +0,0 @@
-.global umode_enter
-.type umode_enter @function
-umode_enter:
- movq %rdi, %rcx
- movq %rsi, %rsp
- movq $0x0202, %r11
- cli
- sysretq
diff --git a/arch/x86_64/umode_enter.c b/arch/x86_64/umode_enter.c
new file mode 100644
index 0000000..3c7fa18
--- /dev/null
+++ b/arch/x86_64/umode_enter.c
@@ -0,0 +1,17 @@
+#include "tasking.h"
+
+__attribute__((noreturn))
+void
+umode_enter(void *ip, void *sp)
+{
+ __asm__ volatile("mov %0, %%rsp; \
+ push %%rcx; \
+ mov %1, %%rcx; \
+ mov $0x202, %%r11; \
+ cli; \
+ swapgs; \
+ sysretq"::
+ "r"(sp), "r"(ip):
+ "memory");
+ for(;;);
+}
diff --git a/arch/x86_64/vm_tophys.c b/arch/x86_64/vm_tophys.c
new file mode 100644
index 0000000..2b16880
--- /dev/null
+++ b/arch/x86_64/vm_tophys.c
@@ -0,0 +1,23 @@
+#include "memory.h"
+#include "boot.h"
+
+uintptr_t
+pm_tovirt(physptr_t phys)
+{
+ if(phys < (1 * GiB))
+ return (uintptr_t)(phys + PHYSMAP_MEMORY_BASE);
+ kpanic("Missing impl for physical addresses > 1GiB (%#016X)\n",
+ phys);
+}
+
+physptr_t
+vm_tophys(uintptr_t virt)
+{
+ return vm_mapping_get(virt).phys;
+}
+
+physptr_t
+vm_tophys_koff(uintptr_t virt)
+{
+ return (virt - _kernel_virtual_base) + boot_kernel_physical_address;
+}
--
cgit v1.2.1