Add the ability to copy memory to non resident process.

Use/Test this by loading the user space elf from the kernel process
before it starts rather than as a part of the first thread.

This simplifies thread start a fair bit.

zion/lib/shared_ptr.h

@@ -15,12 +15,13 @@ class SharedPtr {
   }
 
   SharedPtr(const SharedPtr<T>& other)
-      : ptr_(other.ptr_), ref_cnt_(other.ref_cnt_) {
+      : init_(other.init_), ptr_(other.ptr_), ref_cnt_(other.ref_cnt_) {
     (*ref_cnt_)++;
   }
 
   SharedPtr& operator=(const SharedPtr<T>& other) {
     Cleanup();
+    init_ = other.init_;
     ptr_ = other.ptr_;
     ref_cnt_ = other.ref_cnt_;
     (*ref_cnt_)++;
@@ -29,14 +30,33 @@ class SharedPtr {
 
   ~SharedPtr() { Cleanup(); }
 
-  T& operator*() { return *ptr_; }
+  T& operator*() {
-  const T& operator*() const { return *ptr_; }
+    CheckValid();
-  T* operator->() { return ptr_; }
+    return *ptr_;
-  const T* operator->() const { return ptr_; }
+  }
+  const T& operator*() const {
+    CheckValid();
+    return *ptr_;
+  }
+  T* operator->() {
+    CheckValid();
+    return ptr_;
+  }
+  const T* operator->() const {
+    CheckValid();
+    return ptr_;
+  }
 
-  T* ptr() { return ptr_; }
+  T* ptr() {
+    CheckValid();
+    return ptr_;
+  }
 
-  bool operator==(const SharedPtr<T>& other) { return ptr_ == other.ptr_; }
+  bool operator==(const SharedPtr<T>& other) {
+    CheckValid();
+    other.CheckValid();
+    return ptr_ == other.ptr_;
+  }
 
   bool empty() { return !init_; }
 
@@ -55,6 +75,12 @@ class SharedPtr {
       delete ref_cnt_;
     }
   }
+
+  void CheckValid() const {
+    if (!init_) {
+      panic("Accessing invalid shared ptr");
+    }
+  }
 };
 
 template <typename T, class... A>

zion/loader/elf_loader.cpp

@@ -46,17 +46,9 @@ typedef struct {
   uint64_t align;
 } Elf64ProgramHeader;
 
-void badmemcpy(uint64_t base, uint64_t offset, uint64_t dest) {
-  uint8_t* ptr = reinterpret_cast<uint8_t*>(base);
-  uint8_t* dest_ptr = reinterpret_cast<uint8_t*>(dest);
-  for (uint64_t i = 0; i < offset; i++) {
-    dest_ptr[i] = ptr[i];
-  }
-}
-
 }  // namespace
 
-uint64_t LoadElfProgram(uint64_t base, uint64_t offset) {
+uint64_t LoadElfProgram(uint64_t cr3, uint64_t base, uint64_t offset) {
   Elf64Header* header = reinterpret_cast<Elf64Header*>(base);
   dbgln("phoff: %u phnum: %u", header->phoff, header->phnum);
   Elf64ProgramHeader* programs =
@@ -68,8 +60,8 @@ uint64_t LoadElfProgram(uint64_t base, uint64_t offset) {
         "filesz: %u, memsz: %u, align: %u",
         program.type, program.flags, program.offset, program.vaddr,
         program.paddr, program.filesz, program.memsz, program.align);
-    EnsureResident(program.vaddr, program.memsz);
+    CopyIntoNonResidentProcess(base + program.offset, program.filesz, cr3,
-    badmemcpy(base + program.offset, program.filesz, program.vaddr);
+                               program.vaddr);
   }
   return header->entry;
 }

zion/loader/elf_loader.h

@@ -3,4 +3,4 @@
 #include <stdint.h>
 
 // Loads the elf program and returns its entry point.
-uint64_t LoadElfProgram(uint64_t base, uint64_t length);
+uint64_t LoadElfProgram(uint64_t cr3, uint64_t base, uint64_t length);

zion/loader/init_loader.cpp

@@ -25,6 +25,11 @@ const limine_file& GetInitProgram() {
 void LoadInitProgram() {
   const limine_file& init_prog = GetInitProgram();
 
-  gProcMan->InsertProcess(
+  SharedPtr<Process> proc = MakeShared<Process>();
-      new Process(reinterpret_cast<uint64_t>(init_prog.address)));
+  gProcMan->InsertProcess(proc);
+
+  uint64_t entry =
+      LoadElfProgram(proc->cr3(), reinterpret_cast<uint64_t>(init_prog.address),
+                     init_prog.size);
+  proc->CreateThread(entry);
 }

zion/memory/paging_util.cpp

@@ -39,121 +39,155 @@ uint64_t ShiftForEntryIndexing(uint64_t addr, uint64_t offset) {
   addr &= ~0xFFFF0000'00000000;
   addr >>= offset;
   addr <<= 3;
-  return addr;
+  return addr & 0xFF8;
 }
 
-uint64_t* Pml4Entry(uint64_t addr) {
+uint64_t* Pml4Entry(uint64_t cr3, uint64_t addr) {
-  return reinterpret_cast<uint64_t*>(PML_RECURSE |
+  cr3 += boot::GetHigherHalfDirectMap();
+  return reinterpret_cast<uint64_t*>(cr3 |
                                      ShiftForEntryIndexing(addr, PML_OFFSET));
 }
 
-uint64_t* PageDirectoryPointerEntry(uint64_t addr) {
+uint64_t* PageDirectoryPointerEntry(uint64_t pdp_phys, uint64_t addr) {
-  return reinterpret_cast<uint64_t*>(PDP_RECURSE |
+  pdp_phys += boot::GetHigherHalfDirectMap();
+  pdp_phys &= ~0xFFF;
+  return reinterpret_cast<uint64_t*>(pdp_phys |
                                      ShiftForEntryIndexing(addr, PDP_OFFSET));
 }
 
-uint64_t* PageDirectoryEntry(uint64_t addr) {
+uint64_t* PageDirectoryEntry(uint64_t pd_phys, uint64_t addr) {
-  return reinterpret_cast<uint64_t*>(PD_RECURSE |
+  pd_phys += boot::GetHigherHalfDirectMap();
+  pd_phys &= ~0xFFF;
+  return reinterpret_cast<uint64_t*>(pd_phys |
                                      ShiftForEntryIndexing(addr, PD_OFFSET));
 }
 
-uint64_t* PageTableEntry(uint64_t addr) {
+uint64_t* PageTableEntry(uint64_t pt_phys, uint64_t addr) {
-  return reinterpret_cast<uint64_t*>(PT_RECURSE |
+  pt_phys += boot::GetHigherHalfDirectMap();
+  pt_phys &= ~0xFFF;
+  return reinterpret_cast<uint64_t*>(pt_phys |
                                      ShiftForEntryIndexing(addr, PT_OFFSET));
 }
 
-bool PageDirectoryPointerLoaded(uint64_t addr) {
+uint64_t PagePhysIfResident(uint64_t cr3, uint64_t virt) {
-  return *Pml4Entry(addr) & PRESENT_BIT;
+  uint64_t* pml4_entry = Pml4Entry(cr3, virt);
-}
+  if (!(*pml4_entry & PRESENT_BIT)) {
-
+    return false;
-bool PageDirectoryLoaded(uint64_t addr) {
-  return PageDirectoryPointerLoaded(addr) &&
-         (*PageDirectoryPointerEntry(addr) & PRESENT_BIT);
-}
-
-bool PageTableLoaded(uint64_t addr) {
-  return PageDirectoryLoaded(addr) && (*PageDirectoryEntry(addr) & PRESENT_BIT);
-}
-
-void MapPage(uint64_t virt, uint64_t phys) {
-  if (PageLoaded(virt)) {
-    panic("Allocating Over Existing Page: %m", virt);
   }
+  uint64_t* pdp_entry = PageDirectoryPointerEntry(*pml4_entry, virt);
+  if (!(*pdp_entry & PRESENT_BIT)) {
+    return false;
+  }
+  uint64_t* pd_entry = PageDirectoryEntry(*pdp_entry, virt);
+  if (!(*pd_entry & PRESENT_BIT)) {
+    return false;
+  }
+  uint64_t* pt_entry = PageTableEntry(*pd_entry, virt);
+  if (!(*pt_entry & PRESENT_BIT)) {
+    return false;
+  }
+  return *pt_entry & ~0xFFF;
+}
 
+uint64_t MapPage(uint64_t cr3, uint64_t virt) {
   uint64_t access_bits = PRESENT_BIT | READ_WRITE_BIT;
   uint64_t higher_half = 0xffff8000'00000000;
   if ((virt & higher_half) != higher_half) {
     access_bits |= USER_MODE_BIT;
   }
 
-  if (!PageDirectoryPointerLoaded(virt)) {
+  uint64_t* pml4_entry = Pml4Entry(cr3, virt);
+  if (!(*pml4_entry & PRESENT_BIT)) {
     uint64_t page = phys_mem::AllocatePage();
-    *Pml4Entry(virt) = page | access_bits;
+    *pml4_entry = page | access_bits;
-    ZeroOutPage(PageDirectoryPointerEntry(virt));
+    ZeroOutPage(PageDirectoryPointerEntry(*pml4_entry, virt));
   }
-  if (!PageDirectoryLoaded(virt)) {
+  uint64_t* pdp_entry = PageDirectoryPointerEntry(*pml4_entry, virt);
+  if (!(*pdp_entry & PRESENT_BIT)) {
     uint64_t page = phys_mem::AllocatePage();
-    *PageDirectoryPointerEntry(virt) = page | access_bits;
+    *pdp_entry = page | access_bits;
-    ZeroOutPage(PageDirectoryEntry(virt));
+    ZeroOutPage(PageDirectoryEntry(*pdp_entry, virt));
   }
-  if (!PageTableLoaded(virt)) {
+  uint64_t* pd_entry = PageDirectoryEntry(*pdp_entry, virt);
+  if (!(*pd_entry & PRESENT_BIT)) {
     uint64_t page = phys_mem::AllocatePage();
-    *PageDirectoryEntry(virt) = page | access_bits;
+    *(pd_entry) = page | access_bits;
-    ZeroOutPage(PageTableEntry(virt));
+    ZeroOutPage(PageTableEntry(*pd_entry, virt));
   }
 
-  *PageTableEntry(virt) = PageAlign(phys) | access_bits;
+  uint64_t* pt_entry = PageTableEntry(*pd_entry, virt);
-  ZeroOutPage(reinterpret_cast<uint64_t*>(virt));
+  if (!(*pt_entry & PRESENT_BIT)) {
+    uint64_t phys = phys_mem::AllocatePage();
+    *pt_entry = PageAlign(phys) | access_bits;
+    ZeroOutPage(reinterpret_cast<uint64_t*>(boot::GetHigherHalfDirectMap() +
+                                            PageAlign(phys)));
+    return phys;
+  } else {
+    panic("Page already allocated.");
+    return 0;
+  }
 }
 
 uint64_t Pml4Index(uint64_t addr) { return (addr >> PML_OFFSET) & 0x1FF; }
-}  // namespace
 
-void InitPaging() {
+uint64_t CurrCr3() {
   uint64_t pml4_addr = 0;
   asm volatile("mov %%cr3, %0;" : "=r"(pml4_addr));
-  uint64_t* pml4_virtual =
+  return pml4_addr;
-      reinterpret_cast<uint64_t*>(boot::GetHigherHalfDirectMap() + pml4_addr);
-
-  uint64_t recursive_entry = pml4_addr | PRESENT_BIT | READ_WRITE_BIT;
-  pml4_virtual[0x1FE] = recursive_entry;
 }
 
+}  // namespace
+
 void InitializePml4(uint64_t pml4_physical_addr) {
   uint64_t* pml4_virtual = reinterpret_cast<uint64_t*>(
       boot::GetHigherHalfDirectMap() + pml4_physical_addr);
-
+  uint64_t curr_cr3 = CurrCr3();
-  // Map the recursive entry.
-  uint64_t recursive_entry = pml4_physical_addr | PRESENT_BIT | READ_WRITE_BIT;
-  pml4_virtual[0x1FE] = recursive_entry;
 
   // Map the kernel entry.
   // This should contain the heap at 0xFFFFFFFF'40000000
   uint64_t kernel_addr = 0xFFFFFFFF'80000000;
-  pml4_virtual[Pml4Index(kernel_addr)] = *Pml4Entry(kernel_addr);
+  pml4_virtual[Pml4Index(kernel_addr)] = *Pml4Entry(curr_cr3, kernel_addr);
 
   // Map the HHDM.
   // This is necessary to access values off of the kernel stack.
   uint64_t hhdm = boot::GetHigherHalfDirectMap();
-  pml4_virtual[Pml4Index(hhdm)] = *Pml4Entry(hhdm);
+  pml4_virtual[Pml4Index(hhdm)] = *Pml4Entry(curr_cr3, hhdm);
 }
 
-void AllocatePage(uint64_t addr) {
+uint64_t AllocatePageIfNecessary(uint64_t addr, uint64_t cr3) {
-  uint64_t physical_page = phys_mem::AllocatePage();
+  if (cr3 == 0) {
-  MapPage(addr, physical_page);
+    cr3 = CurrCr3();
+  }
+  uint64_t phys = PagePhysIfResident(cr3, addr);
+  if (phys) {
+    return phys;
+  }
+  return MapPage(cr3, addr);
 }
 
 void EnsureResident(uint64_t addr, uint64_t size) {
   uint64_t max = addr + size;
   addr = PageAlign(addr);
   while (addr < max) {
-    if (!PageLoaded(addr)) {
+    AllocatePageIfNecessary(addr);
-      AllocatePage(addr);
-    }
     addr += 0x1000;
   }
 }
 
-bool PageLoaded(uint64_t addr) {
+void CopyIntoNonResidentProcess(uint64_t base, uint64_t size, uint64_t dest_cr3,
-  return PageTableLoaded(addr) && (*PageTableEntry(addr) & PRESENT_BIT);
+                                uint64_t dest_virt) {
+  if (size > 0x1000) {
+    panic("Unimplemented NR copy > 1 page");
+  }
+  if (dest_virt & 0xFFF) {
+    panic("Unimplemented NR copy to non page aligned");
+  }
+  uint64_t phys = AllocatePageIfNecessary(dest_virt, dest_cr3);
+  uint8_t* src = reinterpret_cast<uint8_t*>(base);
+  uint8_t* dest =
+      reinterpret_cast<uint8_t*>(phys + boot::GetHigherHalfDirectMap());
+
+  for (uint64_t i = 0; i < size; i++) {
+    dest[i] = src[i];
+  }
 }

zion/memory/paging_util.h

@@ -2,10 +2,10 @@
 
 #include <stdint.h>
 
-void InitPaging();
 void InitializePml4(uint64_t pml4_physical_addr);
 
-void AllocatePage(uint64_t addr);
+uint64_t AllocatePageIfNecessary(uint64_t addr, uint64_t cr3 = 0);
 void EnsureResident(uint64_t addr, uint64_t size);
 
-bool PageLoaded(uint64_t addr);
+void CopyIntoNonResidentProcess(uint64_t base, uint64_t size, uint64_t dest_cr3,
+                                uint64_t dest_virt);

zion/scheduler/process.cpp

@@ -22,14 +22,13 @@ SharedPtr<Process> Process::RootProcess() {
   return proc;
 }
 
-Process::Process(uint64_t elf_ptr) : id_(gNextId++), state_(RUNNING) {
+Process::Process() : id_(gNextId++), state_(RUNNING) {
   cr3_ = phys_mem::AllocatePage();
   InitializePml4(cr3_);
-  CreateThread(elf_ptr);
 }
 
-void Process::CreateThread(uint64_t elf_ptr) {
+void Process::CreateThread(uint64_t entry) {
-  Thread* thread = new Thread(this, next_thread_id_++, elf_ptr);
+  Thread* thread = new Thread(this, next_thread_id_++, entry);
   threads_.PushBack(thread);
   gScheduler->Enqueue(thread);
 }

zion/scheduler/process.h

@@ -17,12 +17,12 @@ class Process {
     FINISHED,
   };
   static SharedPtr<Process> RootProcess();
-  Process(uint64_t elf_ptr);
+  Process();
 
   uint64_t id() const { return id_; }
   uint64_t cr3() const { return cr3_; }
 
-  void CreateThread(uint64_t elf_ptr);
+  void CreateThread(uint64_t entry);
   SharedPtr<Thread> GetThread(uint64_t tid);
 
   // Checks the state of all child threads and transitions to

zion/scheduler/thread.cpp

@@ -23,8 +23,8 @@ SharedPtr<Thread> Thread::RootThread(Process* root_proc) {
   return new Thread(root_proc);
 }
 
-Thread::Thread(const SharedPtr<Process>& proc, uint64_t tid, uint64_t elf_ptr)
+Thread::Thread(const SharedPtr<Process>& proc, uint64_t tid, uint64_t entry)
-    : process_(proc), id_(tid), elf_ptr_(elf_ptr) {
+    : process_(proc), id_(tid), rip_(entry) {
   uint64_t* stack = new uint64_t[512];
   uint64_t* stack_ptr = stack + 511;
   // 0: rip
@@ -42,12 +42,11 @@ Thread::Thread(const SharedPtr<Process>& proc, uint64_t tid, uint64_t elf_ptr)
 uint64_t Thread::pid() { return process_->id(); }
 
 void Thread::Init() {
-  dbgln("[%u.%u]", pid(), id_);
+  dbgln("[%u.%u] thread start.", pid(), id_);
-  uint64_t rip = LoadElfProgram(elf_ptr_, 0);
   uint64_t rsp = 0x80000000;
   EnsureResident(rsp - 1, 1);
   SetRsp0(rsp0_start_);
-  jump_user_space(rip, rsp);
+  jump_user_space(rip_, rsp);
 }
 
 void Thread::Exit() {

zion/scheduler/thread.h

@@ -11,16 +11,13 @@ class Thread {
  public:
   enum State {
     UNSPECIFIED,
-    CREATED,
     RUNNING,
     RUNNABLE,
-    BLOCKED,
     FINISHED,
   };
   static SharedPtr<Thread> RootThread(Process* root_proc);
 
-  explicit Thread(const SharedPtr<Process>& proc, uint64_t tid,
+  explicit Thread(const SharedPtr<Process>& proc, uint64_t tid, uint64_t entry);
-                  uint64_t elf_ptr);
 
   uint64_t tid() { return id_; };
   uint64_t pid();
@@ -45,7 +42,8 @@ class Thread {
   uint64_t id_;
   State state_ = RUNNABLE;
 
-  uint64_t elf_ptr_;
+  // Startup Context for the thread.
+  uint64_t rip_;
 
   // Stack pointer to take on resume.
   // Stack will contain the full thread context.

zion/zion.cpp

@@ -17,9 +17,6 @@ extern "C" void zion() {
   InitGdt();
   InitIdt();
 
-  dbgln("[boot] Init Paging.");
-  InitPaging();
-
   dbgln("[boot] Init Physical Memory Manager.");
   phys_mem::InitBootstrapPageAllocation();
   KernelHeap heap(0xFFFFFFFF'40000000, 0xFFFFFFFF'80000000);