[Voyageurs] XHCI Event Segment working with polling.

sys/voyageurs/CMakeLists.txt

@@ -1,5 +1,7 @@
 add_executable(voyageurs
   keyboard/keyboard_driver.cpp
+  xhci/trb.cpp
+  xhci/trb_ring.cpp
   xhci/xhci_driver.cpp
   voyageurs_server.cpp
   voyageurs.cpp)

sys/voyageurs/voyageurs.cpp

@@ -15,7 +15,7 @@ uint64_t main(uint64_t init_port) {
 
   YellowstoneClient yellowstone(gInitEndpointCap);
 
-  ASSIGN_OR_RETURN(XhciDriver xhci, XhciDriver::InitiateDriver(yellowstone));
+  ASSIGN_OR_RETURN(auto xhci, XhciDriver::InitiateDriver(yellowstone));
 
   dbgln("Initializing PS/2 Driver.");
   KeyboardDriver driver;

sys/voyageurs/xhci/trb.cpp

@@ -0,0 +1,47 @@
+#include "xhci/trb.h"
+
+constexpr uint8_t kTrb_Normal = 1;
+constexpr uint8_t kTrb_SetupStage = 2;
+constexpr uint8_t kTrb_DataStage = 3;
+constexpr uint8_t kTrb_StatusStage = 4;
+constexpr uint8_t kTrb_Isoch = 5;
+constexpr uint8_t kTrb_Link = 6;
+constexpr uint8_t kTrb_EventData = 7;
+constexpr uint8_t kTrb_NoOp = 8;
+constexpr uint8_t kTrb_EnableSlot = 9;
+constexpr uint8_t kTrb_DisableSlot = 10;
+constexpr uint8_t kTrb_NoOpCommand = 23;
+
+constexpr uint8_t kTrb_TypeOffset = 10;
+
+constexpr uint8_t kTrb_Cycle = 1;
+
+XhciTrb CreateLinkTrb(uint64_t physical_address) {
+  return {
+      .parameter = physical_address,
+      .status = 0,
+      .type_and_cycle = kTrb_Link << kTrb_TypeOffset,
+      .control = 0,
+  };
+}
+
+XhciTrb CreateEnableSlotTrb() {
+  return {
+      .parameter = 0,
+      .status = 0,
+      // FIXME: Accept Cycle Bit as a parameter.
+      .type_and_cycle = kTrb_EnableSlot << kTrb_TypeOffset | kTrb_Cycle,
+      // FIXME: Specify slot type if necessary. (XHCI Table 7-9)?
+      .control = 0,
+  };
+}
+
+XhciTrb CreateNoOpCommandTrb() {
+  return {
+      .parameter = 0,
+      .status = 0,
+      // FIXME: Accept Cycle Bit as a parameter.
+      .type_and_cycle = kTrb_NoOpCommand << kTrb_TypeOffset | kTrb_Cycle,
+      .control = 0,
+  };
+}

sys/voyageurs/xhci/trb.h

@@ -0,0 +1,8 @@
+#pragma once
+
+#include "xhci/xhci.h"
+
+XhciTrb CreateLinkTrb(uint64_t physical_address);
+
+XhciTrb CreateEnableSlotTrb();
+XhciTrb CreateNoOpCommandTrb();

sys/voyageurs/xhci/trb_ring.cpp

@@ -0,0 +1,36 @@
+#include "xhci/trb_ring.h"
+
+#include <mammoth/util/debug.h>
+
+#include "xhci/trb.h"
+
+TrbRing::TrbRing() {
+  uint64_t number_trbs = 0x1000 / sizeof(XhciTrb);
+  page_ = mmth::OwnedMemoryRegion::ContiguousPhysical(0x1000, &phys_address_);
+
+  // Zero out data.
+  uint64_t* page_ptr = reinterpret_cast<uint64_t*>(page_.vaddr());
+  for (uint64_t i = 0; i < 0x1000 / sizeof(uint64_t); i++) {
+    page_ptr[i] = 0;
+  }
+  trb_list_ = glcr::ArrayView<XhciTrb>(
+      reinterpret_cast<XhciTrb*>(page_.vaddr()), number_trbs);
+
+  // Point the end of the command ring back to the start.
+  trb_list_[trb_list_.size() - 1] = CreateLinkTrb(phys_address_);
+}
+
+void TrbRingWriter::EnqueueTrb(const XhciTrb& trb) {
+  uint64_t ptr = enqueue_ptr_++;
+  if (enqueue_ptr_ == trb_list_.size()) {
+    crash("Not implemented: queue wrapping", glcr::UNIMPLEMENTED);
+  }
+
+  trb_list_[ptr] = trb;
+}
+
+bool TrbRingReader::HasNext() {
+  return (trb_list_[dequeue_ptr_].type_and_cycle & 0x1) == cycle_bit_;
+}
+
+XhciTrb TrbRingReader::Read() { return trb_list_[dequeue_ptr_++]; }

sys/voyageurs/xhci/trb_ring.h

@@ -0,0 +1,38 @@
+#pragma once
+
+#include <glacier/container/array_view.h>
+#include <mammoth/util/memory_region.h>
+
+#include "xhci/xhci.h"
+
+class TrbRing {
+ public:
+  TrbRing();
+
+  uint64_t PhysicalAddress() { return phys_address_; }
+
+ protected:
+  uint64_t phys_address_;
+  mmth::OwnedMemoryRegion page_;
+  glcr::ArrayView<XhciTrb> trb_list_;
+};
+
+class TrbRingWriter : public TrbRing {
+ public:
+  void EnqueueTrb(const XhciTrb& trb);
+
+ private:
+  uint64_t enqueue_ptr_ = 0;
+};
+
+class TrbRingReader : public TrbRing {
+ public:
+  bool HasNext();
+  XhciTrb Read();
+
+  uint64_t DequeuePtr() { return phys_address_ + dequeue_ptr_; }
+
+ private:
+  uint64_t dequeue_ptr_ = 0;
+  uint8_t cycle_bit_ = 1;
+};

sys/voyageurs/xhci/xhci.h

@@ -73,7 +73,10 @@ struct XhciRuntime {
   uint64_t reserved3;
   uint64_t reserved4;
   XhciInterrupter interrupters[1024];
+} __attribute__((packed));
 
+struct XhciDoorbells {
+  uint32_t doorbell[256];
 } __attribute__((packed));
 
 struct XhciPort {
@@ -106,18 +109,11 @@ struct XhciDeviceContext {
   XhciEndpointContext endpoint_contexts[31];
 } __attribute__((packed));
 
-struct XhciCommandTrb {
+struct XhciTrb {
-  uint64_t reserved1;
+  uint64_t parameter;
-  uint32_t reserved2;
+  uint32_t status;
   uint16_t type_and_cycle;
-  uint16_t slot_type;
+  uint16_t control;
-} __attribute__((packed));
-
-struct XhciLinkTrb {
-  uint64_t link_address;
-  uint32_t interrupter_target;
-  uint16_t type_and_cycle;
-  uint16_t reserved;
 } __attribute__((packed));
 
 struct XhciEventRingSegmentTableEntry {

sys/voyageurs/xhci/xhci_driver.cpp

@@ -1,29 +1,67 @@
 #include "xhci/xhci_driver.h"
 
-#include <mammoth/proc/thread.h>
 #include <mammoth/util/debug.h>
 #include <mammoth/util/memory_region.h>
 #include <zcall.h>
 
+#include "xhci/trb.h"
 #include "xhci/xhci.h"
 
-glcr::ErrorOr<XhciDriver> XhciDriver::InitiateDriver(
+void interrupt_thread(void* void_driver) {
+  XhciDriver* driver = static_cast<XhciDriver*>(void_driver);
+
+  driver->InterruptLoop();
+
+  crash("Driver returned from interrupt loop", glcr::INTERNAL);
+}
+
+glcr::ErrorOr<glcr::UniquePtr<XhciDriver>> XhciDriver::InitiateDriver(
     yellowstone::YellowstoneClient& yellowstone) {
   yellowstone::XhciInfo info;
   check(yellowstone.GetXhciInfo(info));
 
   mmth::OwnedMemoryRegion pci_region =
       mmth::OwnedMemoryRegion::FromCapability(info.mutable_xhci_region());
-  XhciDriver driver(glcr::Move(pci_region));
+  // Have to make this a heap object so that the reference passed to the
-  driver.ParseMmioStructures();
+  // interrupt loop remains valid.
-  driver.DumpDebugInfo();
+  glcr::UniquePtr<XhciDriver> driver(new XhciDriver(glcr::Move(pci_region)));
-  driver.FreeExistingMemoryStructures();
+  driver->ParseMmioStructures();
-  driver.ResetController();
+  driver->FreeExistingMemoryStructures();
+  driver->ResetController();
+  driver->StartInterruptThread();
   dbgln("XHCI CONTROLLER RESET");
-  driver.DumpDebugInfo();
+  driver->NoOpCommand();
-  check(ZThreadSleep(100));
+  driver->InitiateDevices();
-  driver.DumpDebugInfo();
+  return glcr::Move(driver);
-  return driver;
+}
+
+void XhciDriver::InterruptLoop() {
+  while (true) {
+    while ((runtime_->interrupters[0].management & 0x1) != 0x1) {
+      check(ZThreadSleep(50));
+    }
+    while (event_ring_.HasNext()) {
+      XhciTrb trb = event_ring_.Read();
+      uint16_t type = trb.type_and_cycle >> 10;
+      switch (type) {
+        case 33:
+          dbgln("Command Completion Event. {x}", trb.parameter);
+          break;
+        case 34:
+          dbgln("Port Status Change Event, enabling slot.");
+          command_ring_.EnqueueTrb(CreateEnableSlotTrb());
+          doorbells_->doorbell[0] = 0;
+          break;
+        default:
+          dbgln("Unknown TRB Type {x} received.", type);
+          break;
+      }
+    }
+
+    runtime_->interrupters[0].event_ring_dequeue_pointer =
+        event_ring_.DequeuePtr() | 0x8;
+    runtime_->interrupters[0].management |= 0x1;
+  }
 }
 
 void XhciDriver::DumpDebugInfo() {
@@ -62,6 +100,7 @@ void XhciDriver::DumpDebugInfo() {
     if ((port->status_and_control & 0x3) == 0x1) {
       dbgln("Resetting: {x}", i);
       port->status_and_control |= 0x10;
+      doorbells_->doorbell[0] = 0;
     }
   }
 
@@ -88,10 +127,22 @@ glcr::ErrorCode XhciDriver::ParseMmioStructures() {
   runtime_ = reinterpret_cast<XhciRuntime*>(mmio_regions_.vaddr() +
                                             capabilities_->runtime_offset);
 
+  doorbells_ = reinterpret_cast<XhciDoorbells*>(mmio_regions_.vaddr() +
+                                                capabilities_->doorbell_offset);
+
   return glcr::OK;
 }
 
 glcr::ErrorCode XhciDriver::ResetController() {
+  // Stop the Host Controller.
+  // FIXME: Do this before freeing existing structures.
+  operational_->usb_command &= ~0x1;
+
+  while ((operational_->usb_status & 0x1) != 0x1) {
+    dbgln("Waiting XHCI Halt.");
+    RET_ERR(ZThreadSleep(50));
+  }
+
   // Host Controller Reset
   operational_->usb_command |= 0x2;
 
@@ -115,23 +166,13 @@ glcr::ErrorCode XhciDriver::ResetController() {
   return glcr::OK;
 }
 
+void XhciDriver::StartInterruptThread() {
+  interrupt_thread_ = Thread(interrupt_thread, this);
+}
+
 glcr::ErrorCode XhciDriver::InitiateCommandRing() {
-  uint64_t command_ring_phys;
+  operational_->command_ring_control = command_ring_.PhysicalAddress();
-  command_ring_mem_ =
+  dbgln("CRC: {x}", operational_->command_ring_control);
-      mmth::OwnedMemoryRegion::ContiguousPhysical(0x1000, &command_ring_phys);
-  command_trb_ = reinterpret_cast<XhciCommandTrb*>(command_ring_mem_.vaddr());
-
-  uint64_t number_trbs = 0x1000 / sizeof(XhciCommandTrb);
-
-  // Point the end of the command ring back to the start.
-  auto* link_trb =
-      reinterpret_cast<XhciLinkTrb*>(command_trb_ + number_trbs - 1);
-  link_trb->link_address = command_ring_phys;
-  // TODO: Cleaner interface for specifying a command type.
-  link_trb->type_and_cycle = 0x6 << 10;
-
-  operational_->command_ring_control = command_ring_phys;
-
   return glcr::OK;
 }
 
@@ -165,22 +206,40 @@ glcr::ErrorCode XhciDriver::InitiateEventRingSegmentTable() {
   event_ring_segment_table_ = reinterpret_cast<XhciEventRingSegmentTableEntry*>(
       event_ring_segment_table_mem_.vaddr());
 
-  uint64_t ers_phys;
+  uint64_t ers_size = 0x1000 / sizeof(XhciTrb);
-  event_ring_segment_mem_ =
+  event_ring_segment_table_[0].ring_segment_base =
-      mmth::OwnedMemoryRegion::ContiguousPhysical(0x1000, &ers_phys);
+      event_ring_.PhysicalAddress();
-  uint64_t ers_size = 0x1000 / sizeof(XhciCommandTrb);
-
-  event_ring_segment_ =
-      reinterpret_cast<XhciCommandTrb*>(event_ring_segment_mem_.vaddr());
-
-  event_ring_segment_table_[0].ring_segment_base = ers_phys;
   event_ring_segment_table_[0].ring_segment_size = ers_size & 0xFFFF;
 
-  runtime_->interrupters[0].event_ring_segment_table_base_address = erst_phys;
+  runtime_->interrupters[0].event_ring_dequeue_pointer =
-  runtime_->interrupters[0].event_ring_dequeue_pointer = erst_phys;
+      event_ring_.PhysicalAddress() | 0x8;
   runtime_->interrupters[0].event_ring_segment_table_size = 1;
+  runtime_->interrupters[0].event_ring_segment_table_base_address = erst_phys;
+
   // Enable interrupts.
   runtime_->interrupters[0].management |= 0x2;
+  runtime_->interrupters[0].moderation = 4000;
   operational_->usb_command |= 0x4;
   return glcr::OK;
 }
+
+glcr::ErrorCode XhciDriver::InitiateDevices() {
+  uint64_t max_ports = (capabilities_->hcs_params_1 & 0xFF00'0000) >> 24;
+  for (uint64_t i = 0; i < max_ports; i++) {
+    XhciPort* port = reinterpret_cast<XhciPort*>(
+        reinterpret_cast<uint64_t>(operational_) + 0x400 + (0x10 * i));
+    port->status_and_control &= ~0x10000;
+    dbgln("Port {x}: {x}", i, port->status_and_control);
+    if ((port->status_and_control & 0x3) == 0x1) {
+      dbgln("Resetting: {x}", i);
+      port->status_and_control |= 0x10;
+    }
+  }
+  return glcr::OK;
+}
+
+glcr::ErrorCode XhciDriver::NoOpCommand() {
+  command_ring_.EnqueueTrb(CreateNoOpCommandTrb());
+  doorbells_->doorbell[0] = 0;
+  return glcr::OK;
+}

sys/voyageurs/xhci/xhci_driver.h

@@ -1,33 +1,40 @@
 #pragma once
 
+#include <glacier/memory/unique_ptr.h>
 #include <glacier/status/error_or.h>
+#include <mammoth/proc/thread.h>
 #include <mammoth/util/memory_region.h>
 #include <yellowstone/yellowstone.yunq.client.h>
 
+#include "xhci/trb_ring.h"
 #include "xhci/xhci.h"
 
 class XhciDriver {
  public:
-  static glcr::ErrorOr<XhciDriver> InitiateDriver(
+  static glcr::ErrorOr<glcr::UniquePtr<XhciDriver>> InitiateDriver(
       yellowstone::YellowstoneClient& yellowstone);
 
+  XhciDriver(const XhciDriver&) = delete;
+  XhciDriver(XhciDriver&&) = default;
+
   void DumpDebugInfo();
 
+  void InterruptLoop();
+
  private:
   // MMIO Structures.
   mmth::OwnedMemoryRegion pci_region_;
   PciDeviceHeader* pci_device_header_;
 
   mmth::OwnedMemoryRegion mmio_regions_;
-  XhciCapabilities* capabilities_;
+  volatile XhciCapabilities* capabilities_;
-  XhciOperational* operational_;
+  volatile XhciOperational* operational_;
   // TODO: Extended Capabilities.
-  XhciRuntime* runtime_;
+  volatile XhciRuntime* runtime_;
-  // TODO: Doorbell Array.
+  volatile XhciDoorbells* doorbells_;
 
   // Host Memory Regions.
-  mmth::OwnedMemoryRegion command_ring_mem_;
+  TrbRingWriter command_ring_;
-  XhciCommandTrb* command_trb_;
 
   mmth::OwnedMemoryRegion device_context_base_array_mem_;
   uint64_t* device_context_base_array_;
@@ -35,8 +42,8 @@ class XhciDriver {
   mmth::OwnedMemoryRegion event_ring_segment_table_mem_;
   XhciEventRingSegmentTableEntry* event_ring_segment_table_;
 
-  mmth::OwnedMemoryRegion event_ring_segment_mem_;
+  TrbRingReader event_ring_;
-  XhciCommandTrb* event_ring_segment_;
+  Thread interrupt_thread_;
 
   XhciDriver(mmth::OwnedMemoryRegion&& pci_space);
 
@@ -44,8 +51,13 @@ class XhciDriver {
   glcr::ErrorCode FreeExistingMemoryStructures() { return glcr::OK; }
 
   glcr::ErrorCode ResetController();
+  void StartInterruptThread();
 
   glcr::ErrorCode InitiateCommandRing();
   glcr::ErrorCode InitiateDeviceContextBaseArray();
   glcr::ErrorCode InitiateEventRingSegmentTable();
+
+  glcr::ErrorCode InitiateDevices();
+
+  glcr::ErrorCode NoOpCommand();
 };