[Yunq] First pass at parsing the yunq idl for ipc.

Writing my own idl to make ipc simpler and allow easy passing of
capabilities between processes.

yunq/example.yunq

@@ -0,0 +1,14 @@
+interface File {
+  method open (OpenFileRequest) -> (FileResponse);
+}
+
+message OpenFileRequest {
+  string path;
+  u64 options;
+}
+
+message File {
+  string path;
+  u64 attrs;
+  capability mem_cap;
+}

yunq/yunq.py

@@ -0,0 +1,255 @@
+from enum import Enum
+import os
+import sys
+
+class LexemeType(Enum):
+    NONE = 0
+
+    EOF = 1
+
+    # Identifiers and Keywords
+    NAME = 2
+    
+    # Symbols
+    LEFT_BRACE = 3
+    RIGHT_BRACE = 4
+    LEFT_PAREN = 5
+    RIGHT_PAREN = 6
+    ARROW = 7
+    SEMICOLON = 8
+
+
+class Lexeme():
+    def __init__(self, lextype: LexemeType, value = None):
+        self.type = lextype
+        self.value = value
+
+    def __str__(self):
+        if self.value:
+            return "(%s, %s)" % (self.type, self.value)
+        return "(%s)" % self.type
+
+    def __repr__(self):
+        return self.__str__()
+
+
+def lexer(program: str):
+    line = 1
+    start = 0
+    current = 0
+    tokens: list[Lexeme] = [] 
+    while current < len(program):
+        # Scan next token.
+        start = current
+        curr = program[current]
+        if curr == '\n':
+            line += 1
+        elif curr == '\t' or curr == ' ' or curr == '\r':
+            pass
+        elif curr == '{':
+            tokens.append(Lexeme(LexemeType.LEFT_BRACE))
+        elif curr == '}':
+            tokens.append(Lexeme(LexemeType.RIGHT_BRACE))
+        elif curr == '(':
+            tokens.append(Lexeme(LexemeType.LEFT_PAREN))
+        elif curr == ')':
+            tokens.append(Lexeme(LexemeType.RIGHT_PAREN))
+        elif curr == ';':
+            tokens.append(Lexeme(LexemeType.SEMICOLON))
+        elif curr == '-':
+            current += 1
+            if program[current] == '>': 
+                tokens.append(Lexeme(LexemeType.ARROW))
+            else:
+                sys.exit("Expected > after - got '%s' on line %d" % (program[current], line))
+        elif curr.isalpha():
+            while program[current + 1].isalnum() or program[current + 1] == '_':
+                current += 1
+            tokens.append(Lexeme(LexemeType.NAME, program[start:current + 1]))
+        else:
+            sys.exit("Got unexpected token %s on line %s." % (curr, line))
+
+        current += 1
+
+    tokens.append(Lexeme(LexemeType.EOF))
+
+    return tokens
+
+
+class Method():
+    def __init__(self, name: str, request: str, response: str):
+        self.name = name
+        self.request = request
+        self.response = response
+
+class Interface():
+    def __init__(self, name: str, methods: list[Method]):
+        self.name = name
+        self.methods = methods
+
+class Type(Enum):
+    NONE = 0
+    U64 = 1
+    I64 = 2
+    STRING = 3
+    BYTES = 4
+    CAPABILITY = 5
+
+type_str_dict = {
+        "u64": Type.U64,
+        "i64": Type.I64,
+        "string": Type.STRING,
+        "bytes": Type.BYTES,
+        "capability": Type.CAPABILITY,
+        }
+
+class Field():
+    def __init__(self, fieldtype: Type, name: str):
+        self.type = fieldtype
+        self.name = name
+
+class Message():
+    def __init__(self, name: str, fields: list[Field]):
+        self.name = name
+        self.fields = fields
+
+Decl = Interface | Message
+
+class Parser():
+    def __init__(self, tokens: list[Lexeme]):
+        self.tokens = tokens
+        self.current = 0
+
+    def peektype(self) -> LexemeType:
+        return self.tokens[self.current].type
+
+    def peekvalue(self) -> str:
+        return self.tokens[self.current].value
+
+    def consume(self) -> Lexeme:
+        self.current += 1
+        return self.tokens[self.current - 1]
+
+    def consume_identifier(self) -> str:
+        tok = self.consume()
+        if tok.type != LexemeType.NAME:
+            sys.exit("Expected identifier got %s" % tok.type)
+        return tok.value
+
+    def consume_check(self, lex_type: LexemeType):
+        tok = self.consume()
+        if tok.type != lex_type:
+            sys.exit("Expected %s got %s" % (lex_type, tok_type))
+
+    def consume_check_identifier(self, name: str):
+        tok = self.consume()
+        if tok.type != LexemeType.NAME:
+            sys.exit("Expected '%s' got a %s" % (name, tok.type))
+        if tok.value != name:
+            sys.exit("Expected '%s' got '%s'" % (name, tok.value))
+
+    def parse(self) -> list[Decl]:
+        decls = []
+        while self.peektype() != LexemeType.EOF:
+            decls.append(self.decl())
+        return decls
+
+    def decl(self) -> Decl:
+        token = self.consume()
+        if token.type != LexemeType.NAME:
+            sys.exit("Unexpected token: %s", token)
+        if token.value == "message":
+            return self.message()
+        elif token.value == "interface":
+            return self.interface()
+        sys.exit("Unexpected identifier '%s', expected message or interface" % token.value)
+
+    def interface(self):
+        # "interface" consumed by decl.
+        name = self.consume_identifier()
+
+        self.consume_check(LexemeType.LEFT_BRACE)
+
+        methods: list[Method] = []
+        while self.peektype() != LexemeType.RIGHT_BRACE:
+            methods.append(self.method())
+
+        self.consume_check(LexemeType.RIGHT_BRACE)
+
+        return Interface(name, methods)
+
+    def method(self):
+        self.consume_check_identifier("method")
+
+        name = self.consume_identifier()
+
+        self.consume_check(LexemeType.LEFT_PAREN)
+        
+        request = self.consume_identifier()
+
+        self.consume_check(LexemeType.RIGHT_PAREN)
+        self.consume_check(LexemeType.ARROW)
+        self.consume_check(LexemeType.LEFT_PAREN)
+
+        response = self.consume_identifier()
+
+        self.consume_check(LexemeType.RIGHT_PAREN)
+        self.consume_check(LexemeType.SEMICOLON)
+
+        return Method(name, request, response)
+
+    def message(self):
+        # "message" consumed by decl.
+
+        name = self.consume_identifier()
+
+        self.consume_check(LexemeType.LEFT_BRACE)
+
+        fields: list[Field] = []
+        while self.peektype() != LexemeType.RIGHT_BRACE:
+            fields.append(self.field())
+
+        self.consume_check(LexemeType.RIGHT_BRACE)
+
+        return Message(name, fields)
+
+    def field(self):
+
+        field_type_str = self.consume_identifier()
+        if field_type_str not in type_str_dict.keys():
+            sys.exit("Expected type got '%s'" % field_type_str)
+        field_type = type_str_dict[field_type_str]
+
+        name = self.consume_identifier()
+        self.consume_check(LexemeType.SEMICOLON)
+        return Field(field_type, name)
+
+def print_ast(decls: list[Decl]):
+    for decl in decls:
+        if type(decl) is Interface:
+            print("%s (Interface)" % decl.name)
+            for method in decl.methods:
+                print("\t%s (%s -> %s)" % (method.name, method.request, method.response))
+        elif type(decl) is Message:
+            print("%s (Message)" % decl.name)
+            for field in decl.fields:
+                print("\t%s %s" % (field.type.name, field.name))
+        else:
+            print("unknown type")
+
+
+def main():
+    if len(sys.argv) != 2:
+        sys.exit("Takes name of file.")
+
+    filename = sys.argv[1]
+
+    with open(filename, mode='r') as f:
+        filedata = f.read()
+        lexemes = lexer(filedata)
+
+        parser = Parser(lexemes)
+        print_ast(parser.parse())
+
+if __name__ == "__main__":
+    main()

zion/memory/kernel_heap.cpp

@@ -18,11 +18,47 @@ KernelHeap& GetKernelHeap() {
 }  // namespace
 
 KernelHeap::KernelHeap(uint64_t lower_bound, uint64_t upper_bound)
-    : next_addr_(lower_bound), upper_bound_(upper_bound) {
+    : next_slab_addr_(lower_bound),
+      first_unsized_addr_(lower_bound + (upper_bound - lower_bound) / 2),
+      next_addr_(first_unsized_addr_),
+      upper_bound_(upper_bound) {
   gKernelHeap = this;
 }
 
+void KernelHeap::InitializeSlabAllocators() {
+  slab_8_ = glcr::MakeUnique<SlabAllocator<8>>(next_slab_addr_, 4);
+  next_slab_addr_ += 0x4000;
+  slab_16_ = glcr::MakeUnique<SlabAllocator<16>>(next_slab_addr_, 6);
+  next_slab_addr_ += 0x6000;
+  slab_32_ = glcr::MakeUnique<SlabAllocator<32>>(next_slab_addr_, 6);
+  next_slab_addr_ += 0x6000;
+}
+
 void* KernelHeap::Allocate(uint64_t size) {
+#if K_HEAP_DEBUG
+  dbgln("Alloc (%x)", size);
+#endif
+  if ((size <= 8) && slab_8_) {
+    auto ptr_or = slab_8_->Allocate();
+    if (ptr_or.ok()) {
+      return ptr_or.value();
+    }
+    dbgln("Failed allocation (slab 8): %x", ptr_or.error());
+  }
+  if ((size <= 16) && slab_16_) {
+    auto ptr_or = slab_16_->Allocate();
+    if (ptr_or.ok()) {
+      return ptr_or.value();
+    }
+    dbgln("Failed allocation (slab 16): %x", ptr_or.error());
+  }
+  if ((size <= 32) && slab_32_) {
+    auto ptr_or = slab_32_->Allocate();
+    if (ptr_or.ok()) {
+      return ptr_or.value();
+    }
+    dbgln("Failed allocation (slab 32): %x", ptr_or.error());
+  }
   if (next_addr_ + size >= upper_bound_) {
     panic("Kernel Heap Overrun (next, size, max): %m, %x, %m", next_addr_, size,
           upper_bound_);
@@ -67,5 +103,5 @@ void KernelHeap::RecordSize(uint64_t size) {
 void* operator new(uint64_t size) { return GetKernelHeap().Allocate(size); }
 void* operator new[](uint64_t size) { return GetKernelHeap().Allocate(size); }
 
-void operator delete(void*, uint64_t) {}
+void operator delete(void*, uint64_t size) {}
 void operator delete[](void*) {}

zion/memory/kernel_heap.h

@@ -1,19 +1,31 @@
 #pragma once
 
+#include <glacier/memory/unique_ptr.h>
 #include <stdint.h>
 
+#include "memory/slab_allocator.h"
+
 class KernelHeap {
  public:
   KernelHeap(uint64_t lower_bound, uint64_t upper_bound);
+  void InitializeSlabAllocators();
 
   void* Allocate(uint64_t size);
 
+  void Free(void* address);
+
   static void DumpDistribution();
 
  private:
+  uint64_t next_slab_addr_;
+  uint64_t first_unsized_addr_;
   uint64_t next_addr_;
   uint64_t upper_bound_;
 
+  glcr::UniquePtr<SlabAllocator<8>> slab_8_;
+  glcr::UniquePtr<SlabAllocator<16>> slab_16_;
+  glcr::UniquePtr<SlabAllocator<32>> slab_32_;
+
   // Distribution collection for the purpose of investigating a slab allocator.
   // 0: 0-4B
   // 1: 4B-8B

zion/memory/slab_allocator.h

@@ -0,0 +1,118 @@
+#pragma once
+
+#include <glacier/container/intrusive_list.h>
+#include <glacier/memory/ref_counted.h>
+#include <glacier/status/error.h>
+#include <glacier/status/error_or.h>
+#include <stdint.h>
+
+#include "memory/paging_util.h"
+
+constexpr uint64_t kSlabSentinel = 0xDEADBEEF'C0DEC0DE;
+
+// TODO: Add variable sized slabs (i.e. multiple page slabs.)
+template <uint64_t ElemSize>
+class Slab : public glcr::RefCounted<Slab<ElemSize>>,
+             public glcr::IntrusiveListNode<Slab<ElemSize>> {
+ public:
+  explicit Slab(uint64_t addr) : page_addr_(addr) {
+    for (uint64_t i = 0; i < kBitmapLength; i++) {
+      bitmap_[i] = 0;
+    }
+
+    bitmap_[0] = ElemSize <= 8 ? 0x3 : 0x1;
+    EnsureResident(page_addr_, 16);
+    uint64_t* first_elem = reinterpret_cast<uint64_t*>(page_addr_);
+    first_elem[0] = kSlabSentinel;
+    first_elem[1] = reinterpret_cast<uint64_t>(this);
+  }
+  Slab(Slab&) = delete;
+  Slab(Slab&&) = delete;
+
+  glcr::ErrorOr<void*> Allocate() {
+    uint64_t index = GetFirstFreeIndex();
+    if (index == 0) {
+      return glcr::EXHAUSTED;
+    }
+    bitmap_[index / 64] |= (0x1 << (index % 64));
+    return reinterpret_cast<void*>(page_addr_ + (index * ElemSize));
+  }
+
+  glcr::ErrorCode Free(void* addr) {
+    uint64_t raw_addr = reinterpret_cast<uint64_t>(addr);
+    if (raw_addr < page_addr_ || raw_addr > (page_addr_ + 0x1000)) {
+      return glcr::INVALID_ARGUMENT;
+    }
+    // FIXME: Check alignment.
+    uint64_t offset = raw_addr - page_addr_;
+    uint64_t index = offset / ElemSize;
+    if (index == 0) {
+      return glcr::FAILED_PRECONDITION;
+    }
+    bitmap_[index / 64] &= ~(0x1 << (index % 64));
+  }
+
+  bool IsFull() {
+    for (uint64_t i = 0; i < kBitmapLength; i++) {
+      if (bitmap_[i] != -1) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+ private:
+  // FIXME: Likely a bug or two here if the number of elements doesn't evenly
+  // divide in to the bitmap length.
+  static constexpr uint64_t kBitmapLength = 0x1000 / ElemSize / 64;
+  static constexpr uint64_t kMaxElements = 0x99E / ElemSize;
+  uint64_t bitmap_[kBitmapLength];
+  uint64_t page_addr_;
+
+  uint64_t GetFirstFreeIndex() {
+    uint64_t bi = 0;
+    while (bi < kBitmapLength && (bitmap_[bi] == -1)) {
+      bi++;
+    }
+    if (bi == kBitmapLength) {
+      return 0;
+    }
+    // FIXME: Use hardware bit instructions here.
+    uint64_t bo = 0;
+    uint64_t bm = bitmap_[bi];
+    while (bm & 0x1) {
+      bm >>= 1;
+      bo += 1;
+    }
+    return (bi * 64) + bo;
+  }
+};
+
+template <uint64_t ElemSize>
+class SlabAllocator {
+ public:
+  SlabAllocator() = delete;
+  SlabAllocator(SlabAllocator&) = delete;
+
+  // TODO: Add a Kernel VMMO Struct to hold things like this.
+  SlabAllocator(uint64_t base_addr, uint64_t num_pages) {
+    for (uint64_t i = 0; i < num_pages; i++, base_addr += 0x1000) {
+      slabs_.PushBack(glcr::MakeRefCounted<Slab<ElemSize>>(base_addr));
+    }
+  }
+
+  glcr::ErrorOr<void*> Allocate() {
+    glcr::RefPtr<Slab<ElemSize>> curr = slabs_.PeekFront();
+    while (curr) {
+      if (curr->IsFull()) {
+        curr = curr->next_;
+        continue;
+      }
+      return curr->Allocate();
+    }
+    return glcr::EXHAUSTED;
+  }
+
+ private:
+  glcr::IntrusiveList<Slab<ElemSize>> slabs_;
+};

zion/zion.cpp

@@ -25,6 +25,7 @@ extern "C" void zion() {
   phys_mem::InitBootstrapPageAllocation();
   KernelHeap heap(0xFFFFFFFF'40000000, 0xFFFFFFFF'80000000);
   phys_mem::InitPhysicalMemoryManager();
+  heap.InitializeSlabAllocators();
 
   dbgln("[boot] Memory allocations available now.");