pretty massive virtual memory rewrite

+ The system is now a bit simpler and hopefully easier to understand, while also
  extending the shared memory to be 1:N, where there is one owner who
  may become a zombie while waiting for the N to die.

1 files changed, 142 insertions, 281 deletions

diff --git a/src/vmem.c b/src/vmem.c
index f1841e4..c713ced 100644
--- a/src/vmem.c
+++ b/src/vmem.c
@@ -6,7 +6,7 @@
  * Virtual memory handling, mainly userspace virtual memory.
  */
 
-#include <kmi/mem_regions.h>
+#include <kmi/regions.h>
 #include <kmi/assert.h>
 #include <kmi/string.h>
 #include <kmi/debug.h>
@@ -16,7 +16,9 @@
 
 stat_t init_uvmem(struct tcb *t, vm_t base, vm_t top)
 {
-	return init_region(&t->sp_r, base, top);
+	t->uvmem.owner = t->tid;
+	t->uvmem.vmem = t->proc.vmem;
+	return init_region(&t->uvmem.region, base, top);
 }
 
 /**
@@ -29,63 +31,70 @@ stat_t init_uvmem(struct tcb *t, vm_t base, vm_t top)
  *
  * @todo check shared memory regions.
  */
-static stat_t __clone_mapped_region(struct tcb *d, struct tcb *s,
-                                    struct mem_region *m)
+static stat_t __copy_mapped_region(struct tcb *d, struct tcb *s,
+                                   struct mem_region *m)
 {
 	vm_t start = m->start * order_size(BASE_PAGE);
 	vm_t end = m->end * order_size(BASE_PAGE);
 
-	size_t size = end - start, actual_size = 0;
-	vm_t va = alloc_fixed_region(&d->sp_r, start, size,
-	                             &actual_size, m->flags);
+	size_t size = end - start;
+	vm_t v = alloc_fixed_region(&d->uvmem.region, start, size, &size,
+	                            m->flags);
+	catastrophic_assert(v == start);
+	stat_t res = copy_region(d->proc.vmem, s->proc.vmem, v, v, size);
+	if (res == OK)
+		return OK;
 
-	catastrophic_assert(va == start);
-
-	if (!copy_allocd_region(d->proc.vmem, va, size, m->flags, s->proc.vmem))
-		return ERR_MISC;
-
-	return OK;
+	/* cleanup on error */
+	free_region(&d->uvmem.region, v);
+	unmap_region(d->proc.vmem, v, size);
+	return res;
 }
 
 /**
- * Unmap and free private memory region.
+ * Helper for implementing shared memory.
  *
- * @param t Current thread.
- * @param m Memory region to free.
- * @return \see unmap_freed_region().
+ * @param d 'Destination' of new mapping.
+ * @param s Owner of shared region.
+ * @param m Shared memory region to clone.
+ * @param flags Flags of new mapping.
+ * @return Address of new mapping in \p d.
  */
-static stat_t __free_mapped_private_region(struct tcb *t, struct mem_region *m)
+static vm_t __clone_shared_region(struct tcb *d, struct tcb *s,
+                                  struct mem_region *m, vmflags_t flags)
 {
-	stat_t status = OK;
-	pm_t start = __addr(m->start);
-	pm_t end = __addr(m->end);
-	if (!unmap_freed_region(t->proc.vmem, start, end - start, m->flags,
-	                        &status))
-		return ERR_MISC;
+	vm_t start = m->start * BASE_PAGE_SIZE;
+	vm_t end = m->end * BASE_PAGE_SIZE;
+
+	reference_proc(s);
 
-	return status;
+	size_t size = end - start;
+	vm_t v = alloc_shared_region(&d->uvmem.region, size, &size, m->flags,
+	                             s->rid);
+	stat_t res = clone_region(d->proc.vmem, s->proc.vmem, start, v, size,
+	                          flags);
+	if (res == OK)
+		return v;
+
+	/* cleanup on error */
+	unreference_proc(s);
+	free_region(&d->uvmem.region, v);
+	unmap_fixed_region(d->proc.vmem, v, size);
+	return NULL;
 }
 
 /**
- * Check whether process associated with shared memory is still using it.
+ * Unmap and free private memory region.
  *
- * @param pid Process to check.
- * @param start Start of memory region
- * @return \ref true if it is still in use, \ref false otherwise.
+ * @param t Current thread.
+ * @param m Memory region to free.
  */
-static bool __proc_has_region(id_t pid, vm_t start)
+static void __free_mapped_private_region(struct tcb *t, struct mem_region *m)
 {
-	/** @todo this has a slight potential to have a race condition, where
-	 * both threads want to free the same shared region at the same time. */
-	struct tcb *p = get_tcb(pid);
-	if (!p)
-		return false;
-
-	struct mem_region *m = find_used_region(&p->sp_r, start);
-	if (!m)
-		return false;
-
-	return true;
+	pm_t start = __addr(m->start);
+	pm_t end = __addr(m->end);
+	size_t size = end - start;
+	unmap_region(t->proc.vmem, start, size);
 }
 
 /**
@@ -94,29 +103,15 @@ static bool __proc_has_region(id_t pid, vm_t start)
  *
  * @param t Current thread.
  * @param m Memory region to free.
- * @return \see unmap_vpage().
  */
-static stat_t __free_mapped_shared_region(struct tcb *t, struct mem_region *m)
+static void __free_mapped_shared_region(struct tcb *t, struct mem_region *m)
 {
 	vm_t start = __addr(m->start);
-	bool in_use = __proc_has_region(m->pid, m->alt_va);
-
-	size_t osize = order_size(BASE_PAGE);
-	size_t pages = m->start - m->end;
-
-	stat_t status = OK;
-	for (size_t i = 0; i < pages; ++i) {
-		vm_t va = start + i * osize;
-
-		pm_t pa = 0;
-		stat_vpage(t->proc.vmem, va, &pa, 0, 0);
-		status = unmap_vpage(t->proc.vmem, va);
+	vm_t end = __addr(m->end);
+	unreference_proc(get_tcb(m->pid));
 
-		if (!in_use)
-			free_page(pa, BASE_PAGE);
-	}
-
-	return status;
+	size_t bytes = end - start;
+	unmap_fixed_region(t->proc.vmem, start, bytes);
 }
 
 /**
@@ -124,9 +119,8 @@ static stat_t __free_mapped_shared_region(struct tcb *t, struct mem_region *m)
  *
  * @param t Thread to work in.
  * @param m Memory region to free.
- * @return \ref OK
  */
-static stat_t __free_mapped_region(struct tcb *t, struct mem_region *m)
+static void __free_mapped_region(struct tcb *t, struct mem_region *m)
 {
 	if (m->pid != 0)
 		return __free_mapped_shared_region(t, m);
@@ -134,51 +128,72 @@ static stat_t __free_mapped_region(struct tcb *t, struct mem_region *m)
 	return __free_mapped_private_region(t, m);
 }
 
-stat_t clear_uvmem(struct tcb *t)
+void clear_uvmem(struct tcb *t)
 {
-	struct mem_region *m = find_first_region(&t->sp_r);
-	while (m) {
-		if (!is_region_kept(m))
-			__free_mapped_region(t, m);
+	if (t->uvmem.owner != t->tid)
+		return;
 
-		m = m->next;
-	}
+	struct mem_region *m = find_closest_used_region(&t->uvmem.region, 0);
+	for (; m; m = m->next) {
+		if (is_region_kept(m)) {
+			continue;
+		}
 
-	return OK;
+		if (!is_set(m->flags, MR_USED)) {
+			continue;
+		}
+
+		__free_mapped_region(t, m);
+		free_known_region(&t->uvmem.region, m);
+	}
 }
 
-stat_t purge_uvmem(struct tcb *t)
+void purge_uvmem(struct tcb *t)
 {
-	struct mem_region *m = find_first_region(&t->sp_r);
-	while (m) {
+	if (t->uvmem.owner != t->tid)
+		return;
+
+	struct mem_region *m = find_closest_used_region(&t->uvmem.region, 0);
+	for (; m; m = m->next) {
+		if (!is_set(m->flags, MR_USED))
+			continue;
+
+		/* free memory associated with region */
 		__free_mapped_region(t, m);
-		m = m->next;
 	}
 
-	return OK;
+	/* actually destroy region, will clear out all nodes automatically */
+	destroy_region(&t->uvmem.region);
 }
 
-stat_t destroy_uvmem(struct tcb *t)
+void destroy_uvmem(struct tcb *t)
 {
+	if (t->uvmem.owner != t->tid)
+		return;
+
 	/* force clear all regions */
 	purge_uvmem(t);
-	/* destroy region tree itself */
-	return destroy_region(&t->sp_r);
+	/* destroy associated virtual memory space */
+	destroy_vmem(t->uvmem.vmem);
 }
 
-stat_t clone_mem_regions(struct tcb *d, struct tcb *s)
+stat_t copy_uvmem(struct tcb *d, struct tcb *s)
 {
 	/** @todo implement some way to only iterate used regions, this loops
 	 * through all regions which is likely a slight bit slower. */
-	struct mem_region *m = find_first_region(&s->sp_r);
+	stat_t ret = OK;
+	struct mem_region *m = find_first_region(&s->uvmem.region);
 	while (m) {
 		if (is_region_used(m))
-			__clone_mapped_region(d, s, m);
+			ret = __copy_mapped_region(d, s, m);
+
+		if (ret)
+			return ret;
 
 		m = m->next;
 	}
 
-	return OK;
+	return ret;
 }
 
 vm_t alloc_uvmem(struct tcb *t, size_t size, vmflags_t flags)
@@ -186,238 +201,84 @@ vm_t alloc_uvmem(struct tcb *t, size_t size, vmflags_t flags)
 	/* t exists and is the process tcb of the current process */
 	hard_assert(t && is_proc(t), ERR_INVAL);
 
-	stat_t status = OK;
-	const vm_t v = alloc_region(&t->sp_r, size, &size, flags);
-	const vm_t w = map_allocd_region(t->proc.vmem, v, size, flags, &status);
-	return w;
-}
-
-vm_t alloc_uvpage(struct tcb *t, size_t size, vmflags_t flags, size_t *asize,
-                  pm_t *paddr)
-{
-	hard_assert(t && is_proc(t), ERR_INVAL);
-
-	enum mm_order order = nearest_order(size);
-	size_t actual_size = order_size(order);
-	stat_t status = OK;
-
-	const vm_t v = alloc_region(&t->sp_r, size, &size, flags);
-	const vm_t w = __addr(__page(v));
-
-	pm_t addr = alloc_page(order);
-	/** @todo should free region */
-	if (!addr)
+	const vm_t v = alloc_region(&t->uvmem.region, size, &size, flags);
+	if (map_region(t->proc.vmem, v, size, max_order(), flags)) {
+		unmap_region(t->proc.vmem, v, size);
+		free_region(&t->uvmem.region, v);
 		return NULL;
+	}
 
-	status = map_vpage(t->proc.vmem, addr, w, flags, order);
-	if (status)
-		return NULL;
-
-	if (asize)
-		*asize = actual_size;
-
-	if (paddr)
-		*paddr = addr;
-
-	return w;
+	return v;
 }
 
 vm_t alloc_fixed_uvmem(struct tcb *t, vm_t start, size_t size, vmflags_t flags)
 {
 	hard_assert(t && is_proc(t), ERR_INVAL);
 
-	stat_t status = OK;
-	const vm_t v = alloc_fixed_region(&t->sp_r, start, size, &size, flags);
-	const vm_t w = map_allocd_region(t->proc.vmem, v, size, flags, &status);
-	return w;
-}
-
-/**
- * Helper for \ref map_fixed_mem().
- * Maps some contiguous bit of physical memory to an allocated virtual memory region.
- *
- * @param b Virtual memory to work in.
- * @param v Start of virtual memory region.
- * @param p Start of physical memory region.
- * @param size Size of virtual memory region.
- * @param flags Flags to use for mappings.
- * @param status Is written to with the status of this function.
- * @return The start of the virtual address mapping.
- */
-static vm_t map_fixed_region(struct vmem *b, vm_t v, pm_t p, size_t size,
-                             vmflags_t flags, stat_t *status)
-{
-
-	vm_t w = v;
-	stat_t stat = OK;
-	size_t pages = size / BASE_PAGE_SIZE;
-	for (size_t i = 0; i < pages; ++i) {
-		stat = map_vpage(b, p, v, flags, BASE_PAGE);
-		v += BASE_PAGE_SIZE;
-		p += BASE_PAGE_SIZE;
+	const vm_t v = alloc_fixed_region(&t->uvmem.region, start, size, &size,
+	                                  flags);
+	if (map_region(t->proc.vmem, v, size, max_order(), flags)) {
+		unmap_region(t->proc.vmem, v, size);
+		free_region(&t->uvmem.region, v);
+		return NULL;
 	}
 
-	if (status)
-		*status = stat;
-
-	return w;
+	return v;
 }
 
-vm_t map_fixed_mem(struct tcb *t, pm_t start, size_t size, vmflags_t flags)
+vm_t map_fixed_uvmem(struct tcb *t, pm_t start, size_t size, vmflags_t flags)
 {
-	stat_t status = OK;
-	const vm_t v = alloc_region(&t->sp_r, size, &size, flags);
-	const vm_t w = map_fixed_region(t->proc.vmem, v, start, size, flags,
-	                                &status);
-	return w + (start % BASE_PAGE_SIZE);
+	const vm_t v = alloc_region(&t->uvmem.region, size, &size, flags);
+	if (map_fixed_region(t->proc.vmem, v, start, size, flags)) {
+		unmap_region(t->proc.vmem, v, size);
+		free_region(&t->uvmem.region, v);
+		return NULL;
+	}
+
+	return v + (start % BASE_PAGE_SIZE);
 }
 
 /* free_shared_uvmem shouldn't be needed, likely to work with free_uvmem */
-stat_t alloc_shared_uvmem(struct tcb *s, struct tcb *c,
-                          size_t size, vmflags_t sflags, vmflags_t cflags,
-                          vm_t *sstart, vm_t *cstart)
+vm_t alloc_shared_uvmem(struct tcb *s, size_t size, vmflags_t flags)
 {
-	hard_assert(sstart, ERR_INVAL);
-	hard_assert(cstart, ERR_INVAL);
 	hard_assert(s && is_proc(s), ERR_INVAL);
-	hard_assert(c && is_proc(c), ERR_INVAL);
-
-	size_t ssize, csize;
-	vm_t sv = alloc_shared_region(&s->sp_r, size, &ssize, sflags, c->rid);
-	vm_t cv = alloc_shared_region(&c->sp_r, size, &csize, cflags, s->rid);
-
-	/* not exactly optimal but good enough for now, I can start worrying
-	 * about hyperoptimizations whenever. */
-	set_alt_region_addr(&s->sp_r, sv, cv);
-	set_alt_region_addr(&c->sp_r, cv, sv);
-
-	if (csize != ssize) {
-		/** @todo cleanup, better errors? */
-		return ERR_INVAL;
-	}
-
-	stat_t cstatus = OK, sstatus = OK;
-	size_t osize = order_size(BASE_PAGE);
-	size_t pages = ssize / osize;
-	for (size_t i = 0; i < pages; ++i) {
-		pm_t p = alloc_page(BASE_PAGE);
-		sstatus = map_vpage(s->proc.vmem, p, sv + i * osize, sflags,
-		                    BASE_PAGE);
-		cstatus = map_vpage(c->proc.vmem, p, cv + i * osize, cflags,
-		                    BASE_PAGE);
+	const vm_t v = alloc_region(&s->uvmem.region, size, &size,
+	                            MR_SHARED | flags);
+	/* use base pages to make clone more likely to succeed */
+	if (map_region(s->proc.vmem, v, size, BASE_PAGE, flags)) {
+		unmap_region(s->proc.vmem, v, size);
+		free_region(&s->uvmem.region, v);
+		return NULL;
 	}
 
-	*sstart = sv;
-	*cstart = cv;
-
-	if (sstatus)
-		return sstatus;
-
-	if (cstatus)
-		return cstatus;
-
-	return OK;
+	return v;
 }
 
-stat_t free_uvmem(struct tcb *r, vm_t va)
+vm_t ref_shared_uvmem(struct tcb *d, struct tcb *s, vm_t v, vmflags_t flags)
 {
-	/** \todo assume tcb is root tcb? */
-	struct mem_region *m = find_used_region(&r->sp_r, va);
+	struct mem_region *m = find_used_region(&s->uvmem.region, v);
 	if (!m)
 		return ERR_NF;
 
-	stat_t status = __free_mapped_region(r, m);
-	if (status)
-		return ERR_MISC;
-
-	return free_known_region(&r->sp_r, m);
-}
-
-stat_t alloc_uvmem_wrapper(struct vmem *b, pm_t *offset, vm_t vaddr,
-                           vmflags_t flags, enum mm_order order, void *data)
-{
-	*offset = alloc_page(order);
-	if (!*offset)
-		return INFO_TRGN; /* try again */
-
-	stat_t *status = (stat_t *)data, ret;
-	ret = map_vpage(b, *offset, vaddr, flags, order);
-	if (status)
-		*status = ret;
-
-	return ret;
-}
-
-stat_t alloc_shared_wrapper(struct vmem *b, pm_t *offset, vm_t vaddr,
-                            vmflags_t flags, enum mm_order order, void *data)
-{
-	if (order != MM_O0)
-		return INFO_TRGN;
-
-	*offset = alloc_page(MM_O0);
-
-	stat_t *status = (stat_t *)data, ret;
-	ret = map_vpage(b, *offset, vaddr, flags, order);
-	if (status)
-		*status = ret;
+	if (!is_set(m->flags, MR_SHARED))
+		return ERR_INVAL;
 
-	return ret;
+	return __clone_shared_region(d, s, m, flags);
 }
 
-stat_t copy_allocd_wrapper(struct vmem *b, pm_t *offset, vm_t vaddr,
-                           vmflags_t flags, enum mm_order order, void *data)
+stat_t free_uvmem(struct tcb *r, vm_t va)
 {
-	struct vmem *s = (struct vmem *)data;
-
-	pm_t paddr = 0;
-	enum mm_order v_order = 0;
-	stat_vpage(s, vaddr, &paddr, &v_order, 0);
-	/** @todo what if we could combine multiple pages into one in the new
-	 * process? */
-	if (order > v_order)
-		return INFO_TRGN;
-
-	pm_t new_page = alloc_page(order);
-	if (!new_page)
-		return INFO_TRGN;
-
-	/* set write flags temporarily */
-	vmflags_t wrflags = flags | VM_W;
-	map_vpage(b, new_page, vaddr, wrflags, order);
-	memcpy((void *)new_page, (void *)(paddr + *offset), order_size(order));
-
-	/* set actual flags */
-	map_vpage(b, new_page, vaddr, flags, order);
-
-	if (v_order > order)
-		*offset += order_size(order);
-	else
-		*offset = 0;
+	/** \todo assume tcb is root tcb? */
+	struct mem_region *m = find_used_region(&r->uvmem.region, va);
+	if (!m)
+		return ERR_NF;
 
+	__free_mapped_region(r, m);
+	free_known_region(&r->uvmem.region, m);
 	return OK;
 }
 
-stat_t free_uvmem_wrapper(struct vmem *b, pm_t *offset, vm_t vaddr,
-                          vmflags_t flags, enum mm_order order, void *data)
+vmflags_t sanitize_uvflags(vmflags_t flags)
 {
-	UNUSED(flags);
-	UNUSED(offset);
-
-	pm_t paddr = 0;
-	enum mm_order v_order = 0;
-	stat_vpage(b, vaddr, &paddr, &v_order, 0);
-	if (order != v_order)
-		return INFO_TRGN;
-
-	/** @todo we might need to cause an ipi to flush the tlb for other
-	 * cores */
-
-	stat_t *status = (stat_t *)data, ret;
-	ret = unmap_vpage(b, vaddr);
-	if (status)
-		*status = ret;
-
-	free_page(order, paddr);
-
-	return ret;
+	return (flags & (VM_R | VM_W | VM_X)) | VM_V | VM_U;
 }