diff options
| -rw-r--r-- | Makefile | 4 | ||||
| -rw-r--r-- | arch/riscv64/conf/kernel-link.S | 1 | ||||
| -rw-r--r-- | arch/riscv64/init/init.c | 148 | ||||
| -rw-r--r-- | arch/riscv64/init/start.S | 38 | ||||
| -rw-r--r-- | arch/riscv64/kernel/arch.c | 3 | ||||
| -rw-r--r-- | arch/riscv64/kernel/proc.c | 2 | ||||
| -rw-r--r-- | arch/riscv64/kernel/start.S | 36 | ||||
| -rw-r--r-- | arch/riscv64/kernel/vmem.c | 20 | ||||
| -rw-r--r-- | include/arch/proc.h | 2 | ||||
| -rw-r--r-- | include/arch/vmem.h | 5 | ||||
| -rw-r--r-- | include/kmi/initrd.h | 2 | ||||
| -rw-r--r-- | include/kmi/mem.h | 6 | ||||
| -rw-r--r-- | include/kmi/pmem.h | 2 | ||||
| -rw-r--r-- | include/kmi/proc.h | 2 | ||||
| -rw-r--r-- | include/kmi/tcb.h | 3 | ||||
| -rw-r--r-- | include/kmi/vmem.h | 2 | ||||
| -rw-r--r-- | scripts/makefile | 29 | ||||
| -rw-r--r-- | src/initrd.c | 40 | ||||
| -rw-r--r-- | src/main.c | 92 | ||||
| -rw-r--r-- | src/mem.c | 22 | ||||
| -rw-r--r-- | src/pmem.c | 178 | ||||
| -rw-r--r-- | src/proc.c | 29 | ||||
| -rw-r--r-- | src/tcb.c | 8 | ||||
| -rw-r--r-- | src/vmem.c | 35 |
24 files changed, 335 insertions, 374 deletions
@@ -19,16 +19,14 @@ all: setup setup: @echo -n > deps.mk @./scripts/gen-deps -p KERNEL -c COMPILE_KERNEL -b kernel "${KERNEL_SOURCES}" - @./scripts/gen-deps -p INIT -c COMPILE_INIT -b init "${INIT_SOURCES}" # default values, overwrite if/when needed ARCH ?= riscv64 ARCH_SOURCE = arch/$(ARCH) KERNEL_SOURCES != echo src/*.c src/uapi/*.c lib/*.c -INIT_SOURCES != echo lib/fdt*.c src/fdt.c src/string.c -CLEANUP := build deps.mk kernel.* init.* kmi.bin +CLEANUP := build deps.mk kernel.* kmi.bin CLEANUP_CMD := include arch/$(ARCH)/source.mk diff --git a/arch/riscv64/conf/kernel-link.S b/arch/riscv64/conf/kernel-link.S index 181f49d..d93b642 100644 --- a/arch/riscv64/conf/kernel-link.S +++ b/arch/riscv64/conf/kernel-link.S @@ -26,6 +26,7 @@ SECTIONS { } __kernel_end = .; + __kernel_size = __kernel_end - __kernel_start; .garbage : { *(.note*) diff --git a/arch/riscv64/init/init.c b/arch/riscv64/init/init.c deleted file mode 100644 index a4fa696..0000000 --- a/arch/riscv64/init/init.c +++ /dev/null @@ -1,148 +0,0 @@ -/* SPDX-License-Identifier: copyleft-next-0.3.1 */ -/* Copyright 2021 - 2022, Kim Kuparinen < kimi.h.kuparinen@gmail.com > */ - -/** - * @file init.c - * riscv64 'bootstrap', move actual kernel into place and jump to virtual - * memory. - */ - -#include <kmi/types.h> -#include <kmi/attrs.h> -#include <kmi/utils.h> -#include <kmi/vmem.h> -#include <arch/vmem.h> -#include <libfdt.h> - -#include "../kernel/csr.h" - -void flush_tlb_full() -{ - /** @todo could be nice to have a common set of features with kernel, - * but for now this is good enough. */ - __asm__ volatile ("sfence.vma\n" ::: "memory"); -} - -/** - * Create page table entry. - * Copy from \ref arch/riscv64/kernel/vmem.c. - * - * @param a Virtual address. - * @param f PTE flags. - */ -#define to_pte(a, f) (((a) >> 12) << 10 | (f)) - -/** - * Get RAM base address from fdt. - * @todo in case of multiple RAM banks, should try to just find one - * of them and let the kernel figure the rest out. Kernel doesn't currently - * support multiple RAM banks. - * - * @param fdt FDT pointer. - * @return Physical address of ram base. - */ -static pm_t __fdt_ram_base(void *fdt) -{ - int mem_offset = fdt_path_offset(fdt, "/memory"); - const void *mem_reg = fdt_getprop(fdt, mem_offset, "reg", NULL); - - struct cell_info ci = get_cellinfo(fdt, mem_offset); - return (pm_t)fdt_load_int_ptr(ci.addr_cells, mem_reg); -} - -/** - * Jump into virtual memory. - * - * @param load_addr Address where init has been loaded. - * @param ram_base RAM base. - */ -static void init_bootmem(uintptr_t load_addr, uintptr_t ram_base) -{ - /* set all flags on, especially A and D since MMUs are allowed to raise - * exceptions that we're not ready to handle if they're unset. */ - size_t flags = VM_V | VM_X | VM_R | VM_W | VM_D | VM_A; - - extern char *__kernel; - extern char *__kernel_size; - uintptr_t top = load_addr + (uintptr_t)&__kernel + - (uintptr_t)&__kernel_size; - - /* this could be risky, as we might overwrite some bits of initrd or fdt - * if they're allocated too close to the kernel payload. - * @todo Allocate root_branch statically? */ - struct vmem *root_branch = (struct vmem *)align_up(top, SZ_4K); - - /* direct mapping (temp) */ - for (size_t i = 0; i <= CSTACK_PAGE; ++i) - root_branch->leaf[i] = (struct vmem *)to_pte(TOP_PAGE_SIZE * i, - flags); - - /* kernel (also sort of direct mapping) */ - flags |= VM_G; - for (size_t i = KSTART_PAGE; i < IO_PAGE; ++i) - root_branch->leaf[i] = (struct vmem *)to_pte( - ram_base + TOP_PAGE_SIZE * (i - KSTART_PAGE), flags); - - /* kernel IO, map to 0 for now, will be updated in the future */ - root_branch->leaf[IO_PAGE] = (struct vmem *)to_pte(0, flags); - - uintmax_t mode = Sv39; - switch (DEFAULT_Sv_MODE) { - case Sv32: mode = SATP_MODE_Sv32; break; - case Sv39: mode = SATP_MODE_Sv39; break; - case Sv48: mode = SATP_MODE_Sv48; break; - default: break; - }; - - flush_tlb_full(); - csr_write(CSR_SATP, mode | ((uintptr_t)root_branch >> 12)); -} - -/** - * Relocate kernel proper. - * @param load_addr Address to where init has been loaded. - * Used in calculating kernel start address. - */ -static void move_kernel(uintptr_t load_addr) -{ - extern char *__kernel; - extern char *__kernel_size; - - size_t sz = (size_t)&__kernel_size; - char *src = load_addr + (char *)&__kernel; - char *dst = (char *)VM_KERN; - for (size_t i = 0; i < sz; ++i) - dst[i] = src[i]; -} - -/** - * Main driver for the init loader. - * - * @param fdt Global FDT pointer, provided by bootloader. - * @param load_addr Address to where init has been loaded. - */ -void init(void *fdt, pm_t load_addr) -{ - extern void jump_to_kernel(void *fdt, pm_t ram_base, void *k); - - pm_t ram_base = __fdt_ram_base(fdt); - - init_bootmem(load_addr, ram_base); - move_kernel(load_addr); - - jump_to_kernel(fdt, ram_base, (void *)VM_KERN); -} - -#if GENERIC_UBOOT -void init_go(int argc, char **argv, pm_t load_addr) -{ - if (argc != 2) - return; - - void *fdt = (void *)strtouintptr(argv[1]); - - /* fdt is passed as second argument, load address first but since we - * already have it due to our ingenious _start, no need to parse it */ - init(fdt, load_addr); -} -#endif diff --git a/arch/riscv64/init/start.S b/arch/riscv64/init/start.S deleted file mode 100644 index b6394f7..0000000 --- a/arch/riscv64/init/start.S +++ /dev/null @@ -1,38 +0,0 @@ -/* SPDX-License-Identifier: copyleft-next-0.3.1 */ -/* Copyright 2021 - 2022, Kim Kuparinen < kimi.h.kuparinen@gmail.com > */ - -/* generic u-boot passes argc and argv as if we were a regular - * program, so our load address must be placed third. - * If we have a 'custom' u-boot with kmi support, the fdt - * will be passed to us directly, so the load address can go second. - */ -#if GENERIC_UBOOT -# define LOAD_REG a2 -# define INIT init_go -#else -# define LOAD_REG a1 -# define INIT init -#endif - -.section .init -.global _start -/* Entry point to the kernel loader. */ -_start: -/* get load address */ -auipc LOAD_REG, 0 -/* keep using bootloader stack for now */ -//li sp, PM_STACK_TOP -/* make sure there's no garbage in tp, important for id assignment */ -li tp, 0 -call INIT -// INIT shouldn't return, but if we do we might as well just jump back to u-boot -// or wherever -ret - -.section .text -.global jump_to_kernel -jump_to_kernel: -li sp, VM_STACK_TOP // load virtual stack address -li fp, 0 -li gp, 0 -jr a2 // jump to kernel diff --git a/arch/riscv64/kernel/arch.c b/arch/riscv64/kernel/arch.c index ff2c04c..ec4cf43 100644 --- a/arch/riscv64/kernel/arch.c +++ b/arch/riscv64/kernel/arch.c @@ -19,7 +19,7 @@ id_t hartid_to_cpuid(id_t hart) if (cpuid_to_hartid(i) == hart) return i; - error("failed to match hart id %ld to cpu id\n", hart); + error("failed to match hart id %ld to cpu id\n", (long)hart); /* default to zero, though this should maybe be a panic? */ return 0; } @@ -27,7 +27,6 @@ id_t hartid_to_cpuid(id_t hart) pm_t branch_to_satp(struct vmem *branch, enum mm_mode mode) { /* Sv57 && Sv64 in the future? */ - branch = (struct vmem *)__pa(branch); pm_t pn = (pm_t)(branch) >> page_shift(); pm_t m = DEFAULT_Sv_MODE; diff --git a/arch/riscv64/kernel/proc.c b/arch/riscv64/kernel/proc.c index 4e4d7ec..55ccbb8 100644 --- a/arch/riscv64/kernel/proc.c +++ b/arch/riscv64/kernel/proc.c @@ -15,7 +15,7 @@ #include "regs.h" #include "csr.h" -void run_init(struct tcb *t, void *fdt, void *initrd) +void run_init(struct tcb *t, vm_t fdt, vm_t initrd) { /** \todo actually map fdt and initrd into the target address space */ csr_write(CSR_SSCRATCH, t); diff --git a/arch/riscv64/kernel/start.S b/arch/riscv64/kernel/start.S new file mode 100644 index 0000000..4978cc0 --- /dev/null +++ b/arch/riscv64/kernel/start.S @@ -0,0 +1,36 @@ +#if GENERIC_UBOOT +# define MAIN main_go +#else +# define LOAD_REG a1 +# define MAIN main +#endif + +.section .kernel.start +.global _start +_start: +/* get load address */ +auipc a2, 0 + +/* load initial stack */ +lla sp, riscv_init_stack +li t0, 4096 +add sp, sp, t0 + +/* set thread pointer to zero so we don't accidentally try to use a tcb */ +li tp, 0 + +/* call main, whichever one is relevant */ +call MAIN +ret + +.section .text + +/* a0 is fdt, a1 is load_addr, a2 is d, a3 is ram_base, a4 is DMAP */ +.global to_kernelspace +to_kernelspace: +lla t0, kernel +add t0, t0, a4 +sub t0, t0, a3 +add sp, sp, a4 +sub sp, sp, a3 +jr t0 diff --git a/arch/riscv64/kernel/vmem.c b/arch/riscv64/kernel/vmem.c index cc6f3ac..835f53d 100644 --- a/arch/riscv64/kernel/vmem.c +++ b/arch/riscv64/kernel/vmem.c @@ -399,13 +399,26 @@ static void __populate_dmap(struct vmem *branch) /** How many base pages we use for the rpc stack. Used fairly often so calculate * it at the start and then reference it. */ static size_t rpc_pages; +long riscv_init_stack[4096 / sizeof(long)]; +__attribute__((aligned(4096))) struct vmem bootvmem; + +struct vmem *direct_mapping() +{ + rpc_pages = order_size(MM_O1) / BASE_PAGE_SIZE; + + __populate_dmap(&bootvmem); + populate_kvmem(&bootvmem); + __use_vmem(&bootvmem, DEFAULT_Sv_MODE); + + return &bootvmem; +} + struct vmem *init_vmem(void *fdt) { UNUSED(fdt); struct vmem *b = create_vmem(); __populate_dmap(b); - rpc_pages = order_size(MM_O1) / BASE_PAGE_SIZE; /* update which memory branch to use */ use_vmem(b); return b; @@ -415,14 +428,13 @@ struct vmem *create_vmem() { struct vmem *b = (struct vmem *)alloc_page(MM_KPAGE); memset(b, 0, MM_KPAGE_SIZE); - populate_kvmem(b); return b; } stat_t use_vmem(struct vmem *b) { - __use_vmem(b, DEFAULT_Sv_MODE); + __use_vmem(__pa(b), DEFAULT_Sv_MODE); return OK; } @@ -457,7 +469,7 @@ vm_t setup_kernel_io(struct vmem *b, vm_t paddr) * modifying, or during the startup stage where we don't have a tcb yet. * I don't think checking the rpc context is necessary? */ if (!cur_tcb() || cur_tcb()->proc.vmem == b) - flush_tlb((uintptr_t)pte_addr(b->leaf[IO_PAGE])); + flush_tlb_full(); return -TOP_PAGE_SIZE + paddr - addr; } diff --git a/include/arch/proc.h b/include/arch/proc.h index 345f812..429f688 100644 --- a/include/arch/proc.h +++ b/include/arch/proc.h @@ -114,7 +114,7 @@ void adjust_syscall(struct tcb *t); * @param fdt Pointer to FDT that is passed to \c init. * @param initrd Pointer to initrd that is passed to \c init. */ -__noreturn void run_init(struct tcb *t, void *fdt, void *initrd); +__noreturn void run_init(struct tcb *t, vm_t fdt, vm_t initrd); /** * Return to userspace fast. diff --git a/include/arch/vmem.h b/include/arch/vmem.h index 118e6ff..238b12a 100644 --- a/include/arch/vmem.h +++ b/include/arch/vmem.h @@ -140,6 +140,8 @@ struct vmem *init_vmem(void *fdt); vm_t setup_kernel_io(struct vmem *b, vm_t paddr); #endif +struct vmem *direct_mapping(); + /** * Create new virtual memory space. * @@ -171,4 +173,7 @@ stat_t destroy_vmem(struct vmem *b); */ void clone_uvmem(struct vmem * restrict r, struct vmem * restrict b); +__noreturn void to_kernelspace(void *fdt, uintptr_t load_addr, + struct vmem *direct_mapping, pm_t ram_base, + pm_t dmap); #endif /* KMI_ARCH_PAGES_H */ diff --git a/include/kmi/initrd.h b/include/kmi/initrd.h index 06bbaac..58fafd5 100644 --- a/include/kmi/initrd.h +++ b/include/kmi/initrd.h @@ -45,6 +45,8 @@ pm_t get_initrdtop(const void *fdt); */ pm_t get_initrdbase(const void *fdt); +size_t get_initrdsize(const void *fdt); + /** * Move \c init program to some other region in memory. * diff --git a/include/kmi/mem.h b/include/kmi/mem.h index f43ee08..ec8d2ea 100644 --- a/include/kmi/mem.h +++ b/include/kmi/mem.h @@ -211,7 +211,7 @@ enum mm_order nearest_order(size_t size); * @param page_shift Width in bits of base page size. * \todo Should likely also be stat_t? */ -void init_mem(size_t max_order, size_t shifts[10], size_t page_shift); +void init_mem(void *mem); /** * Set RAM base address for global access. @@ -221,6 +221,8 @@ void init_mem(size_t max_order, size_t shifts[10], size_t page_shift); */ void set_ram_base(pm_t base); +void set_ram_size(size_t size); + /** * Get RAM base address. * Very much assumes set_ram_base() has been called beforehand. @@ -229,6 +231,8 @@ void set_ram_base(pm_t base); */ pm_t get_ram_base(); +size_t get_ram_size(); + /** Base page size. */ #define BASE_PAGE_SIZE (order_size(BASE_PAGE)) diff --git a/include/kmi/pmem.h b/include/kmi/pmem.h index 1917174..96a4447 100644 --- a/include/kmi/pmem.h +++ b/include/kmi/pmem.h @@ -68,6 +68,6 @@ size_t probe_pmap(pm_t ram_base, size_t ram_size, pm_t cont); * * @param fdt Global FDT pointer. */ -void init_pmem(void *fdt); +void init_pmem(void *fdt, uintptr_t load_addr); #endif /* KMI_PMEM_H */ diff --git a/include/kmi/proc.h b/include/kmi/proc.h index c49be78..4b8807f 100644 --- a/include/kmi/proc.h +++ b/include/kmi/proc.h @@ -32,6 +32,6 @@ stat_t prepare_proc(struct tcb *t, vm_t bin, vm_t interp); * @return \ref ERR_OOMEM when out of memory, \ref ERR_INVAL if loading \c init * failed, \ref OK otherwise. */ -stat_t init_proc(void *fdt); +stat_t init_proc(void *fdt, vm_t *proc_fdt, vm_t *proc_initrd); #endif /* KMI_PROC_H */ diff --git a/include/kmi/tcb.h b/include/kmi/tcb.h index df161b3..54f6e82 100644 --- a/include/kmi/tcb.h +++ b/include/kmi/tcb.h @@ -13,6 +13,7 @@ struct tcb; #include <kmi/mem_regions.h> +#include <kmi/orphanage.h> #include <kmi/syscalls.h> #include <kmi/atomic.h> #include <kmi/queue.h> @@ -26,7 +27,7 @@ struct tcb; * @param t Thread to check. * @return \c true if thread is process thread, \c false otherwise. */ -#define is_proc(t) (t->rid == t->tid) +#define is_proc(t) ((t->rid == t->tid) && !orphan(t)) /** * Check if thread is in RPC. diff --git a/include/kmi/vmem.h b/include/kmi/vmem.h index 63b931b..3205e00 100644 --- a/include/kmi/vmem.h +++ b/include/kmi/vmem.h @@ -131,6 +131,8 @@ stat_t init_uvmem(struct tcb *r, vm_t base, vm_t top); */ stat_t destroy_uvmem(struct tcb *r); +vm_t map_fixed_mem(struct tcb *r, pm_t base, size_t size, vmflags_t flags); + /** * Clone process memory. * diff --git a/scripts/makefile b/scripts/makefile index 8c0b5a4..26b33ad 100644 --- a/scripts/makefile +++ b/scripts/makefile @@ -30,7 +30,6 @@ LDFLAGS != [ "$(LLVM)" = "0" ] \ BUILD = build ARCH_KERN_BUILD = $(BUILD)/kernel/arch/$(ARCH) -ARCH_INIT_BUILD = $(BUILD)/init/arch/$(ARCH) ARCH_SOURCE = arch/$(ARCH) all: kmi.bin @@ -80,47 +79,29 @@ KERN_SIZE = wc -c kernel.bin | awk '{print $$1}' KERN_INFO = sed "s/<KERNEL_SIZE>/$$($(KERN_SIZE))/" KERNEL_LINK = arch/$(ARCH)/conf/kernel-link.S -INIT_LINK = arch/$(ARCH)/conf/init-link.S KERNEL_LD = build/kernel-link.ld -INIT_LD = build/init-link.ld UBSAN ?= 0 KERN_FLAGS != [ "$(UBSAN)" != "0" ] \ && echo -fsanitize=undefined \ || echo -INIT_FLAGS := -fpic - COMPILE_KERNEL = $(COMPILE) $(KERN_FLAGS) COMPILE_INIT = $(COMPILE) $(INIT_FLAGS) -include deps.mk -include $(KERNEL_LD).d -$(INIT_LD): kernel.bin -$(INIT_LD): $(INIT_LINK) - $(GENLINK) $(INIT_LINK) | $(STRIPLINK) | $(KERN_INFO) > $(INIT_LD) - --include $(KERNEL_LD).d $(KERNEL_LD): $(KERNEL_LINK) $(GENLINK) $(KERNEL_LINK) | $(STRIPLINK) > $(KERNEL_LD) -init.elf: $(INIT_OBJS) $(INIT_LD) - $(GENELF) $(INIT_FLAGS) -T $(INIT_LD) $(INIT_OBJS) -o init.elf $(LINK_FLAGS) - -kernel.elf: $(KERNEL_OBJS) $(KERNEL_LD) - $(GENELF) $(KERNEL_FLAGS) -T $(KERNEL_LD) $(KERNEL_OBJS) -o kernel.elf $(LINK_FLAGS) - -init.bin: init.elf - $(OBJCOPY) $(OBJCOPY_FLAGS) init.elf init.bin - -kernel.bin: kernel.elf - $(OBJCOPY) $(OBJCOPY_FLAGS) kernel.elf kernel.bin +kmi.elf: $(KERNEL_OBJS) $(KERNEL_LD) + $(GENELF) $(KERNEL_FLAGS) -T $(KERNEL_LD) $(KERNEL_OBJS) -o kmi.elf $(LINK_FLAGS) -kmi.bin: init.bin kernel.bin - cat init.bin kernel.bin > kmi.bin +kmi.bin: kmi.elf + $(OBJCOPY) $(OBJCOPY_FLAGS) kmi.elf kmi.bin # might lint some common things twice .PHONY: -lint: $(INIT_LINTS) $(KERNEL_LINTS) +lint: $(KERNEL_LINTS) diff --git a/src/initrd.c b/src/initrd.c index caff8b3..242e373 100644 --- a/src/initrd.c +++ b/src/initrd.c @@ -115,53 +115,39 @@ static size_t init_nlen = ARRAY_SIZE(init_n) - 1; /* ignore trailing NULL */ pm_t get_initrdtop(const void *fdt) { int chosen_offset = fdt_path_offset(fdt, "/chosen"); - struct cell_info ci = get_cellinfo(fdt, chosen_offset); + int len = 0; void *initrd_end_ptr = (void *)fdt_getprop(fdt, chosen_offset, - "linux,initrd-end", NULL); + "linux,initrd-end", &len); - /* fdt is only aware of physical memory pointers */ - return (pm_t)__va(fdt_load_int_ptr(ci.addr_cells, initrd_end_ptr)); + catastrophic_assert(initrd_end_ptr); + return (pm_t)fdt_load_int_ptr(len / 4, initrd_end_ptr); } pm_t get_initrdbase(const void *fdt) { const int chosen_offset = fdt_path_offset(fdt, "/chosen"); - const struct cell_info ci = get_cellinfo(fdt, chosen_offset); + int len = 0; void *initrd_base_ptr = (void *)fdt_getprop(fdt, chosen_offset, - "linux,initrd-start", NULL); + "linux,initrd-start", &len); - return (pm_t)__va(fdt_load_int_ptr(ci.addr_cells, initrd_base_ptr)); + catastrophic_assert(initrd_base_ptr); + return (pm_t)fdt_load_int_ptr(len / 4, initrd_base_ptr); } - -size_t get_init_size(const void *fdt) +size_t get_initrdsize(const void *fdt) { - char *c = (char *)get_initrdbase(fdt); - struct cpio_header *cp = __find_file(c, init_n, init_nlen); - return convnum(cp->c_filesize, 8, 16); + pm_t start = get_initrdbase(fdt); + pm_t end = get_initrdtop(fdt); + return end - start; } vm_t get_init_base(const void *fdt) { char *c = (char *)get_initrdbase(fdt); + c = __va(c); struct cpio_header *cp = __find_file(c, init_n, init_nlen); size_t name_len = convnum(cp->c_namesize, 8, 16); return ((vm_t)cp) + align_up(sizeof(struct cpio_header) + name_len, 4); } - -stat_t move_init(const void *fdt, void *target) -{ - const char *c = (const char *)get_initrdbase(fdt); - - const struct cpio_header *cp = __find_file(c, init_n, init_nlen); - size_t name_len = convnum(cp->c_namesize, 8, 16); - size_t file_len = convnum(cp->c_filesize, 8, 16); - - char *fp = (char *)cp; - fp += align_up(sizeof(struct cpio_header) + name_len, 4); - - memmove(target, fp, file_len); - return OK; -} @@ -19,6 +19,54 @@ #include <arch/smp.h> #include <libfdt.h> +static pm_t __fdt_ram_base(void *fdt) +{ + int mem_offset = fdt_path_offset(fdt, "/memory"); + const void *mem_reg = fdt_getprop(fdt, mem_offset, "reg", NULL); + + struct cell_info ci = get_cellinfo(fdt, mem_offset); + return (pm_t)fdt_load_reg_addr(ci, mem_reg, 0); +} + +static pm_t __fdt_ram_size(void *fdt) +{ + int mem_offset = fdt_path_offset(fdt, "/memory"); + const void *mem_reg = fdt_getprop(fdt, mem_offset, "reg", NULL); + + /* here we actually want the root offset because /memory itself doesn't + * have children, I guess? */ + struct cell_info ci = get_cellinfo(fdt, fdt_path_offset(fdt, "/")); + return (pm_t)fdt_load_reg_size(ci, mem_reg, 0); +} + +void kernel(void *fdt, uintptr_t load_addr, struct vmem *d) +{ + fdt = __va(fdt); + + /* dbg uses direct mapping at this point */ + init_dbg(fdt); + /* start up debugging in kernel IO */ + setup_io_dbg(d); + + dbg_fdt(fdt); + + setup_arch(fdt); + + init_pmem(fdt, load_addr); + + init_irq(fdt); + init_timer(fdt); + + vm_t proc_fdt = 0, proc_initrd = 0; + init_proc(fdt, &proc_fdt, &proc_initrd); + + /* try to bring up other cores on system */ + smp_bringup(d, fdt); + + /* start running init program */ + run_init(cur_tcb(), proc_fdt, proc_initrd); +} + /** * Boot entry of kernel actual. * @@ -29,35 +77,33 @@ * @param ram_base RAM base. * @return Should not. */ -void __main main(void *fdt, uintptr_t ram_base) +void main(unsigned long hart, void *fdt, uintptr_t load_addr) { - set_ram_base(ram_base); + /* we have our own ways to get the current hart when we need it, but we + * have to get the function signature right */ + (void)hart; - /* convert physical address to virtual address */ - fdt = __va(fdt); - - /* dbg uses direct mapping at this point */ - init_dbg(fdt); - setup_dmap_dbg(); - dbg_fdt(fdt); + pm_t ram_base = __fdt_ram_base(fdt); + pm_t ram_size = __fdt_ram_size(fdt); + set_ram_base(ram_base); + set_ram_size(ram_size); - setup_arch(fdt); + init_mem(fdt); - init_pmem(fdt); - /* setup temporary virtual memory */ - struct vmem *b = init_vmem(fdt); + struct vmem *d = direct_mapping(); - /* start up debugging in kernel IO */ - setup_io_dbg(b); + to_kernelspace(fdt, load_addr, d, ram_base, VM_DMAP); + unreachable(); +} - init_irq(fdt); - init_timer(fdt); - init_proc(fdt); +#if GENERIC_UBOOT +void main_go(size_t argc, char *argv[], uintptr_t load_addr) +{ + if (argc != 2) + return; - /* try to bring up other cores on system */ - smp_bringup(b, fdt); + void *fdt = (void *)strtouintptr(argv[1]); - /* start running init program */ - void *initrd = (void *)get_initrdbase(fdt); - run_init(cur_tcb(), fdt, initrd); + main(0, fdt, load_addr); } +#endif @@ -22,6 +22,7 @@ enum mm_order __mm_max_order; * like __mm_*. */ pm_t ram_base; +size_t ram_size; enum mm_order nearest_order(size_t size) { @@ -32,12 +33,17 @@ enum mm_order nearest_order(size_t size) return MM_O0; } -void init_mem(size_t max_order, size_t bits[10], size_t page_shift) +void init_mem(void *fdt) { + size_t max_order = 0; + size_t base_bits = 0; + size_t bits[NUM_ORDERS] = { 0 }; + stat_pmem_conf(fdt, &max_order, &base_bits, bits); + __mm_max_order = max_order; - __mm_page_shift = page_shift; + __mm_page_shift = base_bits; - __mm_shifts[0] = page_shift; + __mm_shifts[0] = __mm_page_shift; __mm_widths[0] = 1 << bits[0]; __mm_sizes[0] = 1 << __mm_page_shift; @@ -53,7 +59,17 @@ void set_ram_base(uintptr_t base) ram_base = base; } +void set_ram_size(size_t size) +{ + ram_size = size; +} + uintptr_t get_ram_base() { return ram_base; } + +size_t get_ram_size() +{ + return ram_size; +} @@ -436,77 +436,89 @@ static void __mark_area_used(pm_t base, pm_t top) mark_used(BASE_PAGE, runner); } +struct avoid_region { + pm_t base; + pm_t size; +}; + +static bool overlaps(pm_t base1, pm_t size1, pm_t base2, pm_t size2) +{ + bool b = base1 >= base2 && base1 < base2 + size2; + bool t = base1 + size1 > base2 && base1 + size1 <= base2 + size2; + return b || t; +} + /** * Mark reserved memory region used, to avoid it getting accidentally allocated. * * @param fdt Global FDT pointer. */ -static void __mark_reserved_mem(void *fdt) +static void __mark_reserved(pm_t ram_base, pm_t ram_size, size_t avoid_count, + struct avoid_region avoid[64]) +{ + for (size_t i = 0; i < avoid_count; ++i) { + pm_t base = avoid[i].base; + pm_t size = avoid[i].size; + + if (!overlaps(base, size, ram_base, ram_size)) + continue; + + pm_t top = base + size; + __mark_area_used(base, top); + info("marked [%lx - %lx] reserved\n", base, top); + } +} + +static size_t build_reserved_map(size_t exists, struct avoid_region avoid[64], + void *fdt) { int rmem_offset = fdt_path_offset(fdt, "/reserved-memory"); struct cell_info ci = get_cellinfo(fdt, rmem_offset); int node = 0; fdt_for_each_subnode(node, fdt, rmem_offset) { - uint8_t *rmem_reg = - (uint8_t *)fdt_getprop(fdt, node, "reg", NULL); + uint8_t *rmem_reg = (uint8_t *)fdt_getprop(fdt, node, "reg", + NULL); pm_t base = (pm_t)fdt_load_reg_addr(ci, rmem_reg, 0); + pm_t size = (pm_t)fdt_load_reg_size(ci, rmem_reg, 0); - /** @todo make sure the top of a reserved memory area doesn't go - * against our assumptions in FW_MAX_SIZE? */ - pm_t top = (pm_t)fdt_load_reg_size(ci, rmem_reg, 0) + base; - __mark_area_used((pm_t)__va(base), (pm_t)__va(top)); - info("marked [%lx - %lx] reserved\n", - (pm_t)__va(base), (pm_t)__va(top)); + avoid[exists++] = (struct avoid_region){(pm_t)__va(base), size}; + catastrophic_assert(exists < 64); } + + return exists; } -/** - * Read top of RAM from FDT. - * - * @param fdt Global FDT pointer. - * @return Physical address of top of RAM. - */ -static pm_t __get_ramtop(void *fdt) +static pm_t select_base(pm_t ram_base, pm_t ram_size, pm_t size, + pm_t avoid_count, + struct avoid_region avoid[64]) { - int mem_offset = fdt_path_offset(fdt, "/memory"); - const void *mem_reg = fdt_getprop(fdt, mem_offset, "reg", NULL); + for (size_t i = 0; i < avoid_count; ++i) { + /* try placing things just after each avoidance region */ + pm_t base = avoid[i].base + avoid[i].size; - /* here we actually want the root offset because /memory itself doesn't - * have children, I guess? */ - struct cell_info ci = get_cellinfo(fdt, fdt_path_offset(fdt, "/")); - pm_t base = (pm_t)fdt_load_reg_addr(ci, mem_reg, 0); - return (pm_t)fdt_load_reg_size(ci, mem_reg, 0) + base; -} + /* any better alignment than this has to be manually handled + * outside of this function */ + base = align_up(base, sizeof(long)); -/** - * Read top of FDT. - * - * @param fdt Global FDT pointer. - * @return Physical address of top of FDT. - */ -static pm_t __get_fdttop(void *fdt) -{ - const char *b = (const char *)fdt; - return (pm_t)(b + fdt_totalsize(fdt)); -} + if (!overlaps(base, size, ram_base, ram_size)) + continue; -/** - * Return base of FDT. - * - * Technically pretty useless, but here mainly for cohesion. - * - * @param fdt Global FDT pointer. - * @return \c fdt. - */ -static pm_t __get_fdtbase(void *fdt) -{ - /* lol */ - return (pm_t)fdt; + for (size_t a = 0; a < avoid_count; ++a) { + if (overlaps(base, size, avoid[a].base, avoid[a].size)) + goto retry; + + } + + return base; +retry: + } + + return 0; } -void init_pmem(void *fdt) +void init_pmem(void *fdt, uintptr_t load_addr) { /** @todo should I keep the info outputs? I suppose it's nice to see * if any assumption is being broken in the serial log, but in that case @@ -515,14 +527,10 @@ void init_pmem(void *fdt) */ info("initializing pmem\n"); - size_t max_order = 0; - size_t base_bits = 0; - size_t bits[NUM_ORDERS] = { 0 }; - stat_pmem_conf(fdt, &max_order, &base_bits, bits); - init_mem(max_order, bits, base_bits); - - pm_t ram_size = __get_ramtop(fdt) - get_ram_base(); + /* here it's a bit easier to work with virtual RAM addresses, but + * physical ones could work just as well. */ pm_t ram_base = (pm_t)__va(get_ram_base()); + pm_t ram_size = get_ram_size(); info("using ram range [%lx - %lx]\n", ram_base, ram_base + ram_size); @@ -530,23 +538,38 @@ void init_pmem(void *fdt) /** @todo could probably improve error messages on failing to get fdt * values */ pm_t initrd_base = get_initrdbase(fdt); - pm_t initrd_top = get_initrdtop(fdt); - info("found initrd at [%lx - %lx]\n", initrd_base, initrd_top); + pm_t initrd_size = get_initrdsize(fdt); + info("found initrd at [%lx - %lx]\n", initrd_base, + initrd_base + initrd_size); - pm_t fdt_top = __get_fdttop(fdt); - pm_t fdt_base = __get_fdtbase(fdt); - info("found fdt at [%lx - %lx]\n", fdt_base, fdt_top); + pm_t fdt_base = (pm_t)fdt; + pm_t fdt_size = fdt_totalsize(fdt); + info("found fdt at [%lx - %lx]\n", fdt_base, fdt_base + fdt_size); /* find probably most suitable contiguous region of ram for our physical * ram map */ - /** @todo this really should check that there's enough space in RAM - * instead of just forcing the pmap to be populated */ - pm_t pmap_base = align_up(MAX(initrd_top, fdt_top), BASE_PAGE_SIZE); - info("choosing to place pmem map at %lx\n", pmap_base); - size_t probe_size = probe_pmap(ram_base, ram_size, pmap_base); + size_t probe_size = probe_pmap(0, ram_size, 0); info("pmem map probe size returned %lu\n", probe_size); + /* linker magicry */ + extern char *__kernel_size; + /* avoidance regions, note that stack and so on is included in the + * kernel. Addresses can be outside RAM, in which case they are just + * ignored. */ + struct avoid_region avoid[64] = { + {(pm_t)__va(load_addr), (pm_t)&__kernel_size}, + {(pm_t)__va(initrd_base), initrd_size}, + {(pm_t)__va(fdt_base), fdt_size} + }; + + size_t avoid_count = build_reserved_map(4, avoid, fdt); + pm_t pmap_base = select_base(ram_base, ram_size, + probe_size, avoid_count, avoid); + + catastrophic_assert(pmap_base); + info("choosing to place pmem map at %lx\n", pmap_base); + size_t actual_size = populate_pmap(ram_base, ram_size, pmap_base); info("pmem map actual size %lu\n", actual_size); @@ -555,31 +578,10 @@ void init_pmem(void *fdt) actual_size); } - /* mark init stack, this should be unmapped once we get to executing - * processes */ - __mark_area_used(VM_STACK_BASE, VM_STACK_TOP); - info("marked stack [%lx - %lx] used\n", VM_STACK_BASE, VM_STACK_TOP); - - /* mark kernel */ - /* this could be made more explicit, I suppose. */ - __mark_area_used(VM_KERN, VM_KERN + PM_KERN_SIZE); - info("marked kernel [%lx - %lx] used\n", VM_KERN, - VM_KERN + PM_KERN_SIZE); - - /* mark fdt and initrd */ - __mark_area_used(initrd_base, initrd_top); - info("marked initrd [%lx - %lx] used\n", initrd_base, initrd_top); - - __mark_area_used(fdt_base, fdt_top); - info("marked fdt [%lx - %lx] used\n", fdt_base, fdt_top); - - /* mark pmap */ - __mark_area_used(pmap_base, pmap_base + actual_size); - info("marked pmap [%lx - %lx] used\n", pmap_base, - pmap_base + actual_size); + avoid[avoid_count++] = (struct avoid_region){pmap_base, actual_size}; /* mark reserved mem */ - __mark_reserved_mem(fdt); + __mark_reserved(ram_base, ram_size, avoid_count, avoid); init_mem_nodes(); @@ -9,12 +9,15 @@ #include <kmi/elf.h> #include <kmi/proc.h> #include <kmi/conf.h> +#include <kmi/debug.h> #include <kmi/string.h> #include <kmi/initrd.h> #include <arch/arch.h> #include <arch/proc.h> #include <arch/cpu.h> +#include <libfdt.h> + stat_t prepare_proc(struct tcb *t, vm_t bin, vm_t interp) { vm_t entry = load_elf(t, bin, interp); @@ -27,7 +30,7 @@ stat_t prepare_proc(struct tcb *t, vm_t bin, vm_t interp) return OK; } -stat_t init_proc(void *fdt) +stat_t init_proc(void *fdt, vm_t *proc_fdt, vm_t *proc_initrd) { init_tcbs(); @@ -42,18 +45,32 @@ stat_t init_proc(void *fdt) /* force tcb for core */ tcb_assign(t); - /* set current tcb */ - use_tcb(t); + t->notify_id = t->tid; /* init process has all capabilities */ set_caps(t->caps, 0, CAP_CAPS | CAP_PROC | CAP_SIGNAL | CAP_POWER | CAP_NOTIFY); - t->notify_id = t->tid; + /* we shall try to map the fdt and initrd into the new address space, so + * save them here before we switch */ + use_tcb(t); - /* allocate stacks after ELF file to make sure nothing of importance + /* allocate stacks etc after ELF file to make sure nothing of importance * clashes */ - return prepare_proc(t, get_init_base(fdt), 0); + prepare_proc(t, get_init_base(fdt), 0); /** \todo start one thread per core, with special handling for init in * that each thread starts at the entry point of init? */ + + *proc_fdt = map_fixed_mem(t, + (pm_t)fdt, fdt_totalsize(fdt), + VM_V | VM_R | VM_U); + + pm_t initrd = (pm_t)__va(get_initrdbase(fdt)); + *proc_initrd = map_fixed_mem(t, + initrd, get_initrdsize(fdt), + VM_V | VM_R | VM_U); + + info("mapped fdt at %p\n", (void *)*proc_fdt); + info("mapped initrd at %p\n", (void *)*proc_initrd); + return OK; } @@ -74,7 +74,7 @@ static id_t __alloc_tid(struct tcb *t) if (i == stop_tid) return ERR_NF; - if (get_tcb(i) || i == 0) + if (tcbs[i & (num_tids - 1)] || i == 0) continue; tcbs[i & (num_tids - 1)] = t; @@ -210,6 +210,11 @@ static stat_t __destroy_thread_data(struct tcb *t) destroy_vmem(t->rpc.vmem); /* remove ourselves from the thread pool */ + /** @todo this should be at the top of the function, and be wrapped in + * some kind of lock that checks that nobody reads the value while we're + * setting it to zero. get_tcb() should accordingly increment the + * reference count atomically. Also, an unget_tcb() is needed to + * decrement the reference count I guess? */ tcbs[t->tid] = 0; /* forcefully free last struggling bits of memory */ @@ -228,6 +233,7 @@ stat_t destroy_thread(struct tcb *t) hard_assert(!is_proc(t), ERR_INVAL); /* mark us as zombies */ + set_bits(t->state, TCB_ZOMBIE); t->rid = 0; /* remove reference to root process */ @@ -232,6 +232,34 @@ vm_t alloc_fixed_uvmem(struct tcb *t, vm_t start, size_t size, vmflags_t flags) return w; } +static vm_t map_fixed_region(struct vmem *b, vm_t v, pm_t p, size_t size, + vmflags_t flags, stat_t *status) +{ + + vm_t w = v; + stat_t stat = OK; + size_t pages = size / BASE_PAGE_SIZE; + for (size_t i = 0; i < pages; ++i) { + stat = map_vpage(b, p, v, flags, BASE_PAGE); + v += BASE_PAGE_SIZE; + p += BASE_PAGE_SIZE; + } + + if (status) + *status = stat; + + return w; +} + +vm_t map_fixed_mem(struct tcb *t, pm_t start, size_t size, vmflags_t flags) +{ + stat_t status = OK; + const vm_t v = alloc_region(&t->sp_r, size, &size, flags); + const vm_t w = map_fixed_region(t->proc.vmem, v, start, size, flags, + &status); + return w + (start % BASE_PAGE_SIZE); +} + /* free_shared_uvmem shouldn't be needed, likely to work with free_uvmem */ stat_t alloc_shared_uvmem(struct tcb *s, struct tcb *c, size_t size, vmflags_t sflags, vmflags_t cflags, @@ -341,9 +369,14 @@ stat_t copy_allocd_wrapper(struct vmem *b, pm_t *offset, vm_t vaddr, if (!new_page) return INFO_TRGN; - map_vpage(b, new_page, vaddr, flags, order); + /* set write flags temporarily */ + vmflags_t wrflags = flags | VM_W; + map_vpage(b, new_page, vaddr, wrflags, order); memcpy((void *)new_page, (void *)(paddr + *offset), order_size(order)); + /* set actual flags */ + map_vpage(b, new_page, vaddr, flags, order); + if (v_order > order) *offset += order_size(order); else |