From b36e3b83b402ee054c319f0472b16a2bd64bba7e Mon Sep 17 00:00:00 2001 From: Kimplul Date: Sun, 13 Nov 2022 07:28:52 +0200 Subject: ~1.3M ipc requests + The secret is using gravestones. I'll have to write up full documentation for the feature but essentially riscv lets us encode whatever we want into page table entries, as long as they're not active. We use this to encode highest user address that is not a zero, in that all entries in the top level are either active or gravestones. When an active entry is removed, it is either a gravestone (if there are other active entries above it) or it starts a cascade of removing entries that have been previously removed Slight runtime overhead to page mapping, pretty major advantage in rpc calls. Feature will need to be tested more thorougly, and the init program is sort of a best scenario with just one top level userspace page table entry active at a time, leading to incredibly fast context switches. Current implementation limits a process' max virtual memory to 248 GiB (in Sv39), but I don't think the missing 8 GiB is that big of a deal. --- arch/riscv64/config.h | 4 +-- arch/riscv64/kernel/vmem.c | 82 +++++++++++++++++++++++++++++++++++++++++----- common/tcb.c | 14 +++----- common/uapi/ipc.c | 2 -- include/arch/vmem.h | 2 +- 5 files changed, 80 insertions(+), 24 deletions(-) diff --git a/arch/riscv64/config.h b/arch/riscv64/config.h index 857f885..b96dc3e 100644 --- a/arch/riscv64/config.h +++ b/arch/riscv64/config.h @@ -68,13 +68,13 @@ #define KSTART_PAGE 256UL /** The RPC stack page. */ -#define CSTACK_PAGE 255UL +#define CSTACK_PAGE 248UL /** User virtual memory space start. */ #define UVMEM_START (SZ_4K) /** User virtual memory space end. */ -#define UVMEM_END (SZ_256G - SZ_1G) +#define UVMEM_END (SZ_256G - SZ_8G) /** RPC stack top. */ #define RPC_STACK_TOP (UVMEM_END + SZ_1G) diff --git a/arch/riscv64/kernel/vmem.c b/arch/riscv64/kernel/vmem.c index d30bf1f..b24be83 100644 --- a/arch/riscv64/kernel/vmem.c +++ b/arch/riscv64/kernel/vmem.c @@ -89,6 +89,13 @@ */ #define is_branch(pte) (is_active(pte) && !(pte_flags(pte) & ~VM_V)) +#define GRAVESTONE 2 + +static bool __unused(pm_t b) +{ + return b == GRAVESTONE || b == NULL; +} + /** * Find page table entry corresponding to virtual address. * @@ -106,7 +113,7 @@ static pm_t *__find_vmem(struct vmem *b, vm_t v, enum mm_order *o) size_t idx = vm_to_index(v, top); pm_t pte = (pm_t)b->leaf[idx]; - if (!pte) + if (__unused(pte)) return 0; if (is_leaf(pte)) { @@ -221,14 +228,29 @@ static void __destroy_branch(struct vmem *b) free_page(MM_KPAGE, (pm_t)__pa(b)); } +static void add_graves(struct vmem *branch, size_t idx) +{ + if (idx >= CSTACK_PAGE) + return; + + for (size_t i = idx; i < CSTACK_PAGE; ++i) { + if (!__unused((pm_t)branch->leaf[i])) + return; + + branch->leaf[i] = (struct vmem *)GRAVESTONE; + } +} + stat_t map_vpage(struct vmem *branch, pm_t paddr, vm_t vaddr, vmflags_t flags, enum mm_order order) { + struct vmem *root = branch; enum mm_order top = __mm_max_order; + while (top != order) { size_t idx = vm_to_index(vaddr, top); - if (!branch->leaf[idx]) + if (__unused((pm_t)branch->leaf[idx])) branch->leaf[idx] = __create_leaf(); branch = (struct vmem *)pte_addr(branch->leaf[idx]); @@ -243,14 +265,32 @@ stat_t map_vpage(struct vmem *branch, pm_t paddr, vm_t vaddr, vmflags_t flags, branch->leaf[idx] = (struct vmem *)to_pte((pm_t)__pa(paddr), vp_flags(flags)); + add_graves(root, vm_to_index(vaddr, __mm_max_order)); return top == __mm_max_order ? INFO_SEFF : OK; } +static void remove_graves(struct vmem *branch, size_t idx) +{ + if (idx > CSTACK_PAGE) + return; + + if (__unused((pm_t)branch->leaf[idx + 1])) + return; + + for (size_t i = idx; i < CSTACK_PAGE; ++i) { + if (!__unused((pm_t)branch->leaf[i])) + return; + + branch->leaf[i] = NULL; + } +} + stat_t unmap_vpage(struct vmem *branch, vm_t vaddr) { pm_t *pte = __find_vmem(branch, vaddr, 0); if (pte) { - *pte = 0; + *pte = GRAVESTONE; + remove_graves(branch, vm_to_index(vaddr, __mm_max_order)); return OK; } @@ -351,12 +391,36 @@ struct uvmem_sv39_map { struct vmem *leaf[CSTACK_PAGE - 1]; }; -stat_t clone_uvmem(struct vmem *r, struct vmem *b) +void clone_uvmem(struct vmem *r, struct vmem *b) { - /** \todo error checking? */ - struct uvmem_sv39_map *rm = (struct uvmem_sv39_map *)(r); - struct uvmem_sv39_map *bm = (struct uvmem_sv39_map *)(b); - *bm = *rm; + size_t i = 0; + for (; i < CSTACK_PAGE; i += 8) { + struct vmem *t = r->leaf[i + 0]; + if (t == 0) + break; + + b->leaf[i + 0] = t; + b->leaf[i + 1] = r->leaf[i + 1]; + b->leaf[i + 2] = r->leaf[i + 2]; + b->leaf[i + 3] = r->leaf[i + 3]; + b->leaf[i + 4] = r->leaf[i + 4]; + b->leaf[i + 5] = r->leaf[i + 5]; + b->leaf[i + 6] = r->leaf[i + 6]; + b->leaf[i + 7] = r->leaf[i + 7]; + } - return OK; + for (; i < CSTACK_PAGE; i += 8) { + struct vmem *t = b->leaf[i + 0]; + if (t == 0) + break; + + b->leaf[i + 0] = 0; + b->leaf[i + 1] = 0; + b->leaf[i + 2] = 0; + b->leaf[i + 3] = 0; + b->leaf[i + 4] = 0; + b->leaf[i + 5] = 0; + b->leaf[i + 6] = 0; + b->leaf[i + 7] = 0; + } } diff --git a/common/tcb.c b/common/tcb.c index f774fbc..201b15e 100644 --- a/common/tcb.c +++ b/common/tcb.c @@ -344,11 +344,8 @@ stat_t clone_rpc_maps(struct tcb *r) { hard_assert(r && is_proc(r), ERR_INVAL); struct tcb *t = r; - while ((t = t->rpc.next)) { - stat_t ret = clone_uvmem(r->proc.vmem, t->rpc.vmem); - if (ret) - return ret; - } + while ((t = t->rpc.next)) + clone_uvmem(r->proc.vmem, t->rpc.vmem); return OK; } @@ -357,11 +354,8 @@ stat_t clone_proc_maps(struct tcb *r) { hard_assert(r && is_proc(r), ERR_INVAL); struct tcb *t = r; - while ((t = t->proc.next)) { - stat_t ret = clone_uvmem(r->proc.vmem, t->proc.vmem); - if (ret) - return ret; - } + while ((t = t->proc.next)) + clone_uvmem(r->proc.vmem, t->proc.vmem); return OK; } diff --git a/common/uapi/ipc.c b/common/uapi/ipc.c index 5041ca2..e9a6e6b 100644 --- a/common/uapi/ipc.c +++ b/common/uapi/ipc.c @@ -50,8 +50,6 @@ static struct sys_ret do_ipc(sys_arg_t pid, if (!r->callback) return SYS_RET1(ERR_NOINIT); - uint64_t before, after; - clone_uvmem(r->proc.vmem, t->rpc.vmem); use_vmem(t->rpc.vmem); save_context(t); diff --git a/include/arch/vmem.h b/include/arch/vmem.h index 1a2ab7e..f6660ed 100644 --- a/include/arch/vmem.h +++ b/include/arch/vmem.h @@ -159,6 +159,6 @@ stat_t destroy_vmem(struct vmem *b); * @param b Destination virtual memory of clone. * @return \ref OK. */ -stat_t clone_uvmem(struct vmem *r, struct vmem *b); +void clone_uvmem(struct vmem *r, struct vmem *b); #endif /* APOS_ARCH_PAGES_H */ -- cgit v1.3