diff --git a/README b/README
index f624a8e5..4f68dba1 100644
--- a/README
+++ b/README
@@ -36,8 +36,7 @@ such as VirtualBox and Qemu:
then you likely need at least twice the size of the cdrom image.
* A harddisk or cdrom drive or support for booting from USB.
* A multiboot compliant bootloader if booting from harddisk.
-* A Parallel ATA harddisk, if you wish to access it from Sortix. The AHCI driver
- has not been merged yet.
+* ATA or AHCI harddisk.
Documentation
-------------
diff --git a/kernel/Makefile b/kernel/Makefile
index 901dbbe7..e3eec6ac 100644
--- a/kernel/Makefile
+++ b/kernel/Makefile
@@ -86,6 +86,9 @@ $(CPUDIR)/kthread.o \
crc32.o \
debugger.o \
descriptor.o \
+disk/ahci/ahci.o \
+disk/ahci/hba.o \
+disk/ahci/port.o \
disk/ata/ata.o \
disk/ata/hba.o \
disk/ata/port.o \
diff --git a/kernel/disk/ahci/ahci.cpp b/kernel/disk/ahci/ahci.cpp
new file mode 100644
index 00000000..08169156
--- /dev/null
+++ b/kernel/disk/ahci/ahci.cpp
@@ -0,0 +1,121 @@
+/*******************************************************************************
+
+ Copyright(C) Jonas 'Sortie' Termansen 2013, 2014, 2015, 2016.
+
+ This file is part of Sortix.
+
+ Sortix is free software: you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License, or (at your option) any later
+ version.
+
+ Sortix is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ details.
+
+ You should have received a copy of the GNU General Public License along with
+ Sortix. If not, see .
+
+ disk/ahci/ahci.cpp
+ Driver for the Advanced Host Controller Interface.
+
+*******************************************************************************/
+
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "ahci.h"
+#include "hba.h"
+
+namespace Sortix {
+namespace AHCI {
+
+bool WaitClear(volatile little_uint32_t* reg,
+ uint32_t bits,
+ bool any,
+ unsigned int msecs)
+{
+ struct timespec timeout = timespec_make(msecs / 1000, (msecs % 1000) * 1000000L);
+ Clock* clock = Time::GetClock(CLOCK_BOOT);
+ struct timespec begun;
+ clock->Get(&begun, NULL);
+ while ( true )
+ {
+ struct timespec now;
+ clock->Get(&now, NULL);
+ uint32_t reg_snapshop = *reg;
+ if ( !any && (reg_snapshop & bits) == 0 )
+ return true;
+ if ( any && (reg_snapshop & bits) != bits )
+ return true;
+ struct timespec elapsed = timespec_sub(now, begun);
+ if ( timespec_le(timeout, elapsed) )
+ return errno = ETIMEDOUT, false;
+ kthread_yield();
+ }
+}
+
+bool WaitSet(volatile little_uint32_t* reg,
+ uint32_t bits,
+ bool any,
+ unsigned int msecs)
+{
+ struct timespec timeout = timespec_make(msecs / 1000, (msecs % 1000) * 1000000L);
+ Clock* clock = Time::GetClock(CLOCK_BOOT);
+ struct timespec begun;
+ clock->Get(&begun, NULL);
+ while ( true )
+ {
+ struct timespec now;
+ clock->Get(&now, NULL);
+ uint32_t reg_snapshop = *reg;
+ if ( !any && (reg_snapshop & bits) == bits )
+ return true;
+ if ( any && (reg_snapshop & bits) != 0 )
+ return true;
+ struct timespec elapsed = timespec_sub(now, begun);
+ if ( timespec_le(timeout, elapsed) )
+ return errno = ETIMEDOUT, false;
+ kthread_yield();
+ }
+}
+
+static void InitializeDevice(Ref dev, const char* devpath,
+ uint32_t devaddr)
+{
+ HBA* hba = new HBA(devaddr);
+ if ( !hba )
+ PanicF("Failed to allocate ATA driver for AHCI device 0x%X", devaddr);
+
+ if ( !hba->Initialize(dev, devpath) )
+ return delete hba;
+}
+
+void Init(const char* devpath, Ref dev)
+{
+ uint32_t devaddr;
+ pcifind_t filter;
+
+ memset(&filter, 255, sizeof(filter));
+ filter.classid = 0x01;
+ filter.subclassid = 0x06;
+
+ devaddr = 0;
+ while ( (devaddr = PCI::SearchForDevices(filter, devaddr)) )
+ InitializeDevice(dev, devpath, devaddr);
+}
+
+} // namespace AHCI
+} // namespace Sortix
diff --git a/kernel/disk/ahci/ahci.h b/kernel/disk/ahci/ahci.h
new file mode 100644
index 00000000..1dfb7a2e
--- /dev/null
+++ b/kernel/disk/ahci/ahci.h
@@ -0,0 +1,49 @@
+/*******************************************************************************
+
+ Copyright(C) Jonas 'Sortie' Termansen 2013, 2014, 2015, 2016.
+
+ This file is part of Sortix.
+
+ Sortix is free software: you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License, or (at your option) any later
+ version.
+
+ Sortix is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ details.
+
+ You should have received a copy of the GNU General Public License along with
+ Sortix. If not, see .
+
+ disk/ahci/ahci.h
+ Driver for the Advanced Host Controller Interface.
+
+*******************************************************************************/
+
+#ifndef SORTIX_DISK_AHCI_AHCI_H
+#define SORTIX_DISK_AHCI_AHCI_H
+
+#include
+
+#include
+#include
+
+namespace Sortix {
+namespace AHCI {
+
+bool WaitClear(volatile little_uint32_t* reg,
+ uint32_t bits,
+ bool any,
+ unsigned int msecs);
+bool WaitSet(volatile little_uint32_t* reg,
+ uint32_t bits,
+ bool any,
+ unsigned int msecs);
+void Init(const char* devpath, Ref dev);
+
+} // namespace AHCI
+} // namespace Sortix
+
+#endif
diff --git a/kernel/disk/ahci/hba.cpp b/kernel/disk/ahci/hba.cpp
new file mode 100644
index 00000000..d788a8e0
--- /dev/null
+++ b/kernel/disk/ahci/hba.cpp
@@ -0,0 +1,280 @@
+/*******************************************************************************
+
+ Copyright(C) Jonas 'Sortie' Termansen 2013, 2014, 2015, 2016.
+
+ This file is part of Sortix.
+
+ Sortix is free software: you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License, or (at your option) any later
+ version.
+
+ Sortix is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ details.
+
+ You should have received a copy of the GNU General Public License along with
+ Sortix. If not, see .
+
+ disk/ahci/hba.cpp
+ Driver for the Advanced Host Controller Interface.
+
+*******************************************************************************/
+
+#include
+#include
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "../node.h"
+
+#include "ahci.h"
+#include "hba.h"
+#include "port.h"
+#include "registers.h"
+
+namespace Sortix {
+namespace AHCI {
+
+static inline void ahci_hba_flush(volatile struct hba_regs* hba_regs)
+{
+ (void) hba_regs->ghc;
+}
+
+static unsigned long AllocateDiskNumber()
+{
+ static unsigned long next_disk_number = 0;
+ return InterlockedIncrement(&next_disk_number).o;
+}
+
+HBA::HBA(uint32_t devaddr)
+{
+ this->devaddr = devaddr;
+ memset(&mmio_alloc, 0, sizeof(mmio_alloc));
+ memset(&ports, 0, sizeof(ports));
+ interrupt_registered = false;
+ mmio_alloced = false;
+}
+
+HBA::~HBA()
+{
+ // TODO: Destroy all the ports?
+ if ( interrupt_registered )
+ Interrupt::UnregisterHandler(interrupt_index, &interrupt_registration);
+ if ( mmio_alloced )
+ UnmapPCIBar(&mmio_alloc);
+}
+
+void HBA::LogF(const char* format, ...)
+{
+ // TODO: Print this line in an atomic manner.
+ Log::PrintF("ahci: pci 0x%X: ", devaddr);
+ va_list ap;
+ va_start(ap, format);
+ Log::PrintFV(format, ap);
+ va_end(ap);
+ Log::PrintF("\n");
+}
+
+bool HBA::Reset()
+{
+#if 0 // We probably don't want to do this?
+ // Indicate that system software is AHCI aware.
+ regs->ghc = regs->ghc | GHC_AE;
+
+ regs->ghc = regs->ghc | GHC_HR;
+ ahci_hba_flush(regs);
+
+ while ( (regs->ghc & GHC_HR) )
+ kthread_yield();
+
+ regs->ghc = regs->ghc | GHC_AE;
+ ahci_hba_flush(regs);
+
+ // TODO: Set port enable bits in Port Control and Status on Intel
+ // controllers.
+#endif
+
+ return true;
+}
+
+void HBA::OnInterrupt()
+{
+ uint32_t is = regs->is;
+ if ( !is )
+ return;
+
+ for ( size_t i = 0; i < sizeof(ports) / sizeof(ports[0]); i++ )
+ {
+ if ( !(is & (1U << i)) )
+ continue;
+ if ( ports[i] )
+ ports[i]->OnInterrupt();
+ }
+
+ regs->is = is;
+}
+
+void HBA__OnInterrupt(struct interrupt_context*, void* context)
+{
+ ((HBA*) context)->OnInterrupt();
+}
+
+bool HBA::Initialize(Ref dev, const char* devpath)
+{
+ pcibar_t mmio_bar = PCI::GetBAR(devaddr, 5);
+ if ( mmio_bar.size() < sizeof(struct hba_regs) && /* or || ? */
+ mmio_bar.size() < 1024 )
+ {
+ LogF("error: register area is too small");
+ return errno = EINVAL, false;
+ }
+
+ if ( !MapPCIBAR(&mmio_alloc, mmio_bar, 0) )
+ {
+ LogF("error: registers could not be mapped: %m");
+ return false;
+ }
+
+ regs = (volatile struct hba_regs*) mmio_alloc.from;
+
+ if ( 4 < sizeof(void*) && !(regs->cap & CAP_S64A) )
+ {
+ LogF("error: controller doesn't support 64-bit addressing");
+ return errno = EINVAL, false;
+ }
+
+ // TODO: Take control from BIOS?
+
+#if 0
+ // TODO: Is it bad to do a hard reset?
+ if ( !Reset() )
+ {
+ LogF("error: could not reset");
+ return false;
+ }
+#else
+ regs->ghc = regs->ghc | GHC_AE;
+#endif
+
+ regs->ghc = regs->ghc & ~GHC_IE;
+
+ uint32_t ports_implemented = regs->pi;
+ uint32_t num_ports = CAP_NCS(regs->cap);
+
+ uint32_t actual_ports = 0;
+
+ for ( uint32_t i = 0; i < num_ports; i++ )
+ {
+ if ( !(ports_implemented & (1 << i)) )
+ continue;
+
+ actual_ports++;
+
+ if ( regs->ports[i].pxcmd & (PXCMD_ST | PXCMD_CR |
+ PXCMD_FRE | PXCMD_FR) )
+ {
+ if ( regs->ports[i].pxcmd & (PXCMD_ST | PXCMD_CR) )
+ {
+ regs->ports[i].pxcmd = regs->ports[i].pxcmd & ~PXCMD_ST;
+ if ( !WaitClear(®s->ports[i].pxcmd, PXCMD_CR, true, 500) )
+ {
+ LogF("error: port %u: "
+ "timeout waiting for PXCMD_CR to clear", i);
+ ports_implemented &= ~(1 << i);
+ continue;
+ }
+ }
+
+ if ( regs->ports[i].pxcmd & (PXCMD_FRE | PXCMD_FR) )
+ {
+ regs->ports[i].pxcmd = regs->ports[i].pxcmd & ~PXCMD_FRE;
+ if ( !WaitClear(®s->ports[i].pxcmd, PXCMD_FR, true, 500) )
+ {
+ LogF("error: port %u: "
+ "timeout waiting for PXCMD_FR to clear", i);
+ ports_implemented &= ~(1 << i);
+ continue;
+ }
+ }
+ }
+
+ regs->ports[i].pxis = regs->ports[i].pxis;
+ regs->ports[i].pxserr = regs->ports[i].pxserr;
+ }
+
+ regs->is = regs->is;
+
+ for ( uint32_t i = 0; i < num_ports; i++ )
+ {
+ if ( !(ports_implemented & (1 << i)) )
+ continue;
+
+ if ( !(ports[i] = new Port(this, i)) )
+ {
+ LogF("error: failed to allocate port %u", i);
+ continue;
+ }
+
+ if ( !ports[i]->Initialize() )
+ {
+ delete ports[i];
+ ports[i] = NULL;
+ continue;
+ }
+ }
+
+ interrupt_index =
+ Interrupt::IRQ0 + PCI::Read8(devaddr, PCIFIELD_INTERRUPT_LINE);
+ interrupt_registration.handler = HBA__OnInterrupt;
+ interrupt_registration.context = this;
+ Interrupt::RegisterHandler(interrupt_index, &interrupt_registration);
+ interrupt_registered = true;
+
+ // Enable interrupts.
+ regs->ghc = regs->ghc | GHC_IE;
+ ahci_hba_flush(regs);
+
+ for ( uint32_t i = 0; i < num_ports; i++ )
+ {
+ if ( ports[i] && !ports[i]->FinishInitialize() )
+ {
+ //ports[i]->Destroy();
+ // TODO: WARNING: FIXME: Shut down the port here, see the function
+ // ahci_port_destroy in the OS `kiwi'!
+ // TODO: Unsafe with respect to interrupt handler.
+ delete ports[i];
+ ports[i] = NULL;
+ continue;
+ }
+ }
+
+ for ( uint32_t i = 0; i < num_ports; i++ )
+ {
+ if ( !ports[i] )
+ continue;
+ unsigned long number = AllocateDiskNumber();
+ char name[4 + sizeof(unsigned long) * 3];
+ snprintf(name, sizeof(name), "ahci%lu", number);
+ Ref node(new PortNode(ports[i], 0, 0, 0660, dev->dev, 0));
+ if ( !node )
+ PanicF("Unable to allocate memory for %s/%s inode", devpath, name);
+ ioctx_t ctx; SetupKernelIOCtx(&ctx);
+ if ( LinkInodeInDir(&ctx, dev, name, node) != 0 )
+ PanicF("Unable to link %s/%s to AHCI driver inode", devpath, name);
+ }
+
+ return true;
+}
+
+} // namespace AHCI
+} // namespace Sortix
diff --git a/kernel/disk/ahci/hba.h b/kernel/disk/ahci/hba.h
new file mode 100644
index 00000000..6f35ab59
--- /dev/null
+++ b/kernel/disk/ahci/hba.h
@@ -0,0 +1,73 @@
+/*******************************************************************************
+
+ Copyright(C) Jonas 'Sortie' Termansen 2013, 2014, 2015, 2016.
+
+ This file is part of Sortix.
+
+ Sortix is free software: you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License, or (at your option) any later
+ version.
+
+ Sortix is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ details.
+
+ You should have received a copy of the GNU General Public License along with
+ Sortix. If not, see .
+
+ disk/ahci/hba.h
+ Driver for the Advanced Host Controller Interface.
+
+*******************************************************************************/
+
+#ifndef SORTIX_DISK_AHCI_HBA_H
+#define SORTIX_DISK_AHCI_HBA_H
+
+#include
+
+#include
+#include
+#include
+
+namespace Sortix {
+namespace AHCI {
+
+struct hba_regs;
+
+class Port;
+
+class HBA
+{
+ friend class Port;
+
+public:
+ HBA(uint32_t devaddr);
+ ~HBA();
+
+public:
+ bool Initialize(Ref dev, const char* devpath);
+ void OnInterrupt();
+
+private:
+ __attribute__((format(printf, 2, 3)))
+ void LogF(const char* format, ...);
+ bool Reset();
+
+private:
+ struct interrupt_handler interrupt_registration;
+ Port* ports[32];
+ addralloc_t mmio_alloc;
+ volatile struct hba_regs* regs;
+ uint32_t devaddr;
+ unsigned int interrupt_index;
+ bool interrupt_registered;
+ bool mmio_alloced;
+
+};
+
+} // namespace AHCI
+} // namespace Sortix
+
+#endif
diff --git a/kernel/disk/ahci/port.cpp b/kernel/disk/ahci/port.cpp
new file mode 100644
index 00000000..72402505
--- /dev/null
+++ b/kernel/disk/ahci/port.cpp
@@ -0,0 +1,721 @@
+/*******************************************************************************
+
+ Copyright(C) Jonas 'Sortie' Termansen 2013, 2014, 2015, 2016.
+
+ This file is part of Sortix.
+
+ Sortix is free software: you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License, or (at your option) any later
+ version.
+
+ Sortix is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ details.
+
+ You should have received a copy of the GNU General Public License along with
+ Sortix. If not, see .
+
+ disk/ahci/port.cpp
+ Driver for the Advanced Host Controller Interface.
+
+*******************************************************************************/
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include
+#include
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+#include "ahci.h"
+#include "hba.h"
+#include "port.h"
+#include "registers.h"
+
+namespace Sortix {
+namespace AHCI {
+
+// TODO: Is this needed?
+static inline void ahci_port_flush(volatile struct port_regs* port_regs)
+{
+ (void) port_regs->pxcmd;
+}
+
+static inline void delay(int usecs)
+{
+ struct timespec delay = timespec_make(usecs / 1000000, usecs * 1000);
+ Clock* clock = Time::GetClock(CLOCK_BOOT);
+ clock->SleepDelay(delay);
+}
+
+static void copy_ata_string(char* dest, const char* src, size_t length)
+{
+ for ( size_t i = 0; i < length; i += 2 )
+ {
+ dest[i + 0] = src[i + 1];
+ dest[i + 1] = src[i + 0];
+ }
+ length = strnlen(dest, length);
+ while ( 0 < length && dest[length - 1] == ' ' )
+ length--;
+ dest[length] = '\0';
+}
+
+Port::Port(HBA* hba, uint32_t port_index)
+{
+ port_lock = KTHREAD_MUTEX_INITIALIZER;
+ memset(&control_alloc, 0, sizeof(control_alloc));
+ memset(&dma_alloc, 0, sizeof(dma_alloc));
+ this->hba = hba;
+ regs = &hba->regs->ports[port_index];
+ control_physical_frame = 0;
+ dma_physical_frame = 0;
+ this->port_index = port_index;
+ is_control_page_mapped = false;
+ is_dma_page_mapped = false;
+ interrupt_signaled = false;
+ transfer_in_progress = false;
+}
+
+Port::~Port()
+{
+ if ( transfer_in_progress )
+ FinishTransferDMA();
+ if ( is_control_page_mapped )
+ Memory::Unmap(control_alloc.from);
+ FreeKernelAddress(&control_alloc);
+ if ( is_dma_page_mapped )
+ Memory::Unmap(dma_alloc.from);
+ FreeKernelAddress(&dma_alloc);
+ if ( control_physical_frame )
+ Page::Put(control_physical_frame, PAGE_USAGE_DRIVER);
+ if ( dma_physical_frame )
+ Page::Put(dma_physical_frame, PAGE_USAGE_DRIVER);
+}
+
+void Port::LogF(const char* format, ...)
+{
+ // TODO: Print this line in an atomic manner.
+ Log::PrintF("ahci: pci 0x%X: port %u: ", hba->devaddr, port_index);
+ va_list ap;
+ va_start(ap, format);
+ Log::PrintFV(format, ap);
+ va_end(ap);
+ Log::PrintF("\n");
+}
+
+bool Port::Initialize()
+{
+ // TODO: Potentially move the wait-for-device-to-be-idle code here from hba.
+
+ // Clear interrupt status.
+ regs->pxis = regs->pxis;
+
+ // Clear error bits.
+ regs->pxserr = regs->pxserr;
+
+ if ( !(control_physical_frame = Page::Get(PAGE_USAGE_DRIVER)) )
+ {
+ LogF("error: control page allocation failure");
+ return false;
+ }
+
+ if ( !(dma_physical_frame = Page::Get(PAGE_USAGE_DRIVER)) )
+ {
+ LogF("error: dma page allocation failure");
+ return false;
+ }
+
+ if ( !AllocateKernelAddress(&control_alloc, Page::Size()) )
+ {
+ LogF("error: control page virtual address allocation failure");
+ return false;
+ }
+
+ if ( !AllocateKernelAddress(&dma_alloc, Page::Size()) )
+ {
+ LogF("error: dma page virtual address allocation failure");
+ return false;
+ }
+
+ int prot = PROT_KREAD | PROT_KWRITE;
+ is_control_page_mapped =
+ Memory::Map(control_physical_frame, control_alloc.from, prot);
+ if ( !is_control_page_mapped )
+ {
+ LogF("error: control page virtual address allocation failure");
+ return false;
+ }
+
+ Memory::Flush();
+
+ is_dma_page_mapped = Memory::Map(dma_physical_frame, dma_alloc.from, prot);
+ if ( !is_dma_page_mapped )
+ {
+
+ LogF("dma page virtual address allocation failure");
+ return false;
+ }
+
+ Memory::Flush();
+
+ return true;
+}
+
+bool Port::FinishInitialize()
+{
+ // Disable power management transitions for now (IPM = 3 = transitions to
+ // partial/slumber disabled).
+ regs->pxsctl = regs->pxsctl | 0x300 /* TODO: Magic constant? */;
+
+ // Power on and spin up the device if necessary.
+ if ( regs->pxcmd & PXCMD_CPD )
+ regs->pxcmd = regs->pxcmd | PXCMD_POD;
+ if ( hba->regs->cap & CAP_SSS )
+ regs->pxcmd = regs->pxcmd | PXCMD_SUD;
+
+ // Activate the port.
+ regs->pxcmd = (regs->pxcmd & ~PXCMD_ICC(16)) | (1 << 28);
+
+ if ( !Reset() )
+ {
+ // TODO: Is this safe?
+ return false;
+ }
+
+ // Clear interrupt status.
+ regs->pxis = regs->pxis;
+
+ // Clear error bits.
+ regs->pxserr = regs->pxserr;
+
+ uintptr_t virt = control_alloc.from;
+ uintptr_t phys = control_physical_frame;
+
+ memset((void*) virt, 0, Page::Size());
+
+ size_t offset = 0;
+
+ clist = (volatile struct command_header*) (virt + offset);
+ uint64_t pxclb_addr = phys + offset;
+ regs->pxclb = pxclb_addr >> 0;
+ regs->pxclbu = pxclb_addr >> 32;
+ offset += sizeof(struct command_header) * AHCI_COMMAND_HEADER_COUNT;
+
+ fis = (volatile struct fis*) (virt + offset);
+ uint64_t pxf_addr = phys + offset;
+ regs->pxfb = pxf_addr >> 0;
+ regs->pxfbu = pxf_addr >> 32;
+ offset += sizeof(struct fis);
+
+ ctbl = (volatile struct command_table*) (virt + offset);
+ uint64_t ctba_addr = phys + offset;
+ clist[0].ctba = ctba_addr >> 0;
+ clist[0].ctbau = ctba_addr >> 32;
+ offset += sizeof(struct command_table);
+
+ prdt = (volatile struct prd*) (virt + offset);
+ offset += sizeof(struct prd);
+ // TODO: There can be more of these, fill until end of page!
+
+ // Enable FIS receive.
+ regs->pxcmd = regs->pxcmd | PXCMD_FRE;
+ ahci_port_flush(regs);
+
+ assert(offset <= Page::Size());
+
+ uint32_t ssts = regs->pxssts;
+ uint32_t pxtfd = regs->pxtfd;
+
+ if ( (ssts & 0xF) != 0x3 )
+ return false;
+ if ( pxtfd & ATA_STATUS_BSY )
+ return false;
+ if ( pxtfd & ATA_STATUS_DRQ )
+ return false;
+
+ // TODO: ATAPI.
+ if ( regs->pxsig == 0xEB140101 )
+ return false;
+
+ // Start DMA engine.
+ if ( regs->pxcmd & PXCMD_CR )
+ {
+ if ( !WaitClear(®s->pxcmd, PXCMD_CR, false, 500) )
+ {
+ LogF("error: timeout waiting for PXCMD_CR to clear");
+ return false;
+ }
+ }
+ regs->pxcmd = regs->pxcmd | PXCMD_ST;
+ ahci_port_flush(regs);
+
+ // Set which interrupts we want to know about.
+ regs->pxie = PORT_INTR_ERROR | PXIE_DHRE | PXIE_PSE |
+ PXIE_DSE | PXIE_SDBE | PXIE_DPE;
+ ahci_port_flush(regs);
+
+ CommandDMA(ATA_CMD_IDENTIFY, 512, false);
+ if ( !AwaitInterrupt(500 /*ms*/) )
+ {
+ LogF("error: IDENTIFY timed out");
+ return false;
+ }
+ transfer_in_progress = false;
+
+ memcpy(identify_data, (void*) dma_alloc.from, sizeof(identify_data));
+
+ little_uint16_t* words = (little_uint16_t*) dma_alloc.from;
+
+ if ( words[0] & (1 << 15) )
+ return errno = EINVAL, false; // Skipping non-ATA device.
+ if ( !(words[49] & (1 << 9)) )
+ return errno = EINVAL, false; // Skipping non-LBA device.
+
+ this->is_lba48 = words[83] & (1 << 10);
+
+ copy_ata_string(serial, (const char*) &words[10], sizeof(serial) - 1);
+ copy_ata_string(revision, (const char*) &words[23], sizeof(revision) - 1);
+ copy_ata_string(model, (const char*) &words[27], sizeof(model) - 1);
+
+ uint64_t block_count;
+ if ( is_lba48 )
+ {
+ block_count = (uint64_t) words[100] << 0 |
+ (uint64_t) words[101] << 16 |
+ (uint64_t) words[102] << 32 |
+ (uint64_t) words[103] << 48;
+ }
+ else
+ {
+ block_count = (uint64_t) words[60] << 0 |
+ (uint64_t) words[61] << 16;
+ }
+
+ uint64_t block_size = 512;
+ if( (words[106] & (1 << 14)) &&
+ !(words[106] & (1 << 15)) &&
+ (words[106] & (1 << 12)) )
+ {
+ block_size = 2 * ((uint64_t) words[117] << 0 |
+ (uint64_t) words[118] << 16);
+ }
+
+ // TODO: Verify the block size is a power of two.
+
+ cylinder_count = words[1];
+ head_count = words[3];
+ sector_count = words[6];
+
+ // TODO: Verify that DMA is properly supported.
+ // See kiwi/source/drivers/bus/ata/device.c line 344.
+
+ if ( __builtin_mul_overflow(block_count, block_size, &this->device_size) )
+ {
+ LogF("error: device size overflows off_t");
+ return errno = EOVERFLOW, false;
+ }
+
+ this->block_count = (blkcnt_t) block_count;
+ this->block_size = (blkcnt_t) block_size;
+
+ return true;
+}
+
+bool Port::Reset()
+{
+ if ( regs->pxcmd & (PXCMD_ST | PXCMD_CR) )
+ {
+ regs->pxcmd = regs->pxcmd & ~PXCMD_ST;
+ if ( !WaitClear(®s->pxcmd, PXCMD_CR, false, 500) )
+ {
+ LogF("error: timeout waiting for PXCMD_CR to clear");
+ return false;
+ }
+ }
+
+ if ( regs->pxcmd & (PXCMD_FRE | PXCMD_FR) )
+ {
+ regs->pxcmd = regs->pxcmd & ~PXCMD_FRE;
+ if ( !WaitClear(®s->pxcmd, PXCMD_FR, false, 500) )
+ {
+ LogF("error: timeout waiting for PXCMD_FR to clear");
+ return false;
+ }
+ }
+
+#if 0
+ // Reset the device
+ regs->pxsctl = (regs->pxsctl & ~0xF) | 1;
+ ahci_port_flush(regs);
+ delay(1500);
+ regs->pxsctl = (regs->pxsctl & ~0xF);
+ ahci_port_flush(regs);
+#endif
+
+ // Wait for the device to be detected.
+ if ( !WaitSet(®s->pxssts, 0x1, false, 600) )
+ return false;
+
+ // Clear error.
+ regs->pxserr = regs->pxserr;
+ ahci_port_flush(regs);
+
+ // Wait for communication to be established with device
+ if ( regs->pxssts & 0x1 )
+ {
+ if ( !WaitSet(®s->pxssts, 0x3, false, 600) )
+ return false;
+ regs->pxserr = regs->pxserr;
+ ahci_port_flush(regs);
+ }
+
+ // Wait for the device to come back up.
+ if ( (regs->pxtfd & 0xFF) == 0xFF )
+ {
+ delay(500 * 1000);
+ if ( (regs->pxtfd & 0xFF) == 0xFF )
+ {
+ LogF("error: device did not come back up after reset");
+ return errno = EINVAL, false;
+ }
+ }
+
+#if 0
+ if ( !WaitClear(®s->pxtfd, ATA_STATUS_BSY, false, 1000) )
+ {
+ LogF("error: device did not unbusy");
+ return false;
+ }
+#endif
+
+ return true;
+}
+
+void Port::Seek(blkcnt_t block_index, size_t count)
+{
+ uintmax_t lba = (uintmax_t) block_index;
+ if ( is_lba48 )
+ {
+ memset((void *)&ctbl->cfis, 0, sizeof(ctbl->cfis));
+ ctbl->cfis.count_0_7 = (count >> 0) & 0xff;
+ ctbl->cfis.count_8_15 = (count >> 8) & 0xff;
+ ctbl->cfis.lba_0_7 = (lba >> 0) & 0xff;
+ ctbl->cfis.lba_8_15 = (lba >> 8) & 0xff;
+ ctbl->cfis.lba_16_23 = (lba >> 16) & 0xff;
+ ctbl->cfis.lba_24_31 = (lba >> 24) & 0xff;
+ ctbl->cfis.lba_32_39 = (lba >> 32) & 0xff;
+ ctbl->cfis.lba_40_47 = (lba >> 40) & 0xff;
+ ctbl->cfis.device = 0x40;
+ }
+ else
+ {
+ ctbl->cfis.count_0_7 = (count >> 0) & 0xff;
+ ctbl->cfis.lba_0_7 = (lba >> 0) & 0xff;
+ ctbl->cfis.lba_8_15 = (lba >> 8) & 0xff;
+ ctbl->cfis.lba_16_23 = (lba >> 16) & 0xff;
+ ctbl->cfis.device = 0x40 | ((lba >> 24) & 0xf);
+ }
+}
+
+void Port::CommandDMA(uint8_t cmd, size_t size, bool write)
+{
+ if ( 0 < size )
+ {
+ assert(size <= Page::Size());
+ assert((size & 1) == 0); /* sizes & addresses must be 2-byte aligned */
+ }
+
+ // Set up the command header.
+ uint16_t fis_length = 5 /* dwords */;
+ uint16_t prdtl = size ? 1 : 0;
+ uint16_t clist_0_dw0l = fis_length;
+ if ( write )
+ clist_0_dw0l |= COMMAND_HEADER_DW0_WRITE;
+ clist[0].dw0l = clist_0_dw0l;
+ clist[0].prdtl = prdtl;
+ clist[0].prdbc = 0;
+
+ // Set up the physical region descriptor.
+ if ( prdtl )
+ {
+ uint32_t prdt_0_dbc = size - 1;
+ uint32_t prdt_0_dw3 = prdt_0_dbc;
+ prdt[0].dba = (uint64_t) dma_physical_frame >> 0 & 0xFFFFFFFF;
+ prdt[0].dbau = (uint64_t) dma_physical_frame >> 32 & 0xFFFFFFFF;
+ prdt[0].reserved1 = 0;
+ prdt[0].dw3 = prdt_0_dw3;
+ }
+
+ // Set up the command table.
+ ctbl->cfis.type = 0x27;
+ ctbl->cfis.pm_port = 0;
+ ctbl->cfis.c_bit = 1;
+ ctbl->cfis.command = cmd;
+
+ // Anticipate we will be delivered an IRQ.
+ PrepareAwaitInterrupt();
+ transfer_in_progress = true;
+ transfer_size = size;
+ transfer_is_write = write;
+
+ // Execute the command.
+ regs->pxci = 1;
+ ahci_port_flush(regs);
+}
+
+bool Port::FinishTransferDMA()
+{
+ assert(transfer_in_progress);
+
+ // Wait for an interrupt to arrive.
+ if ( !AwaitInterrupt(10000 /*ms*/) )
+ {
+ const char* op = transfer_is_write ? "write" : "read";
+ LogF("error: %s timed out", op);
+ transfer_in_progress = false;
+ return errno = EIO, false;
+ }
+
+ while ( transfer_is_write && regs->pxtfd & ATA_STATUS_BSY )
+ kthread_yield();
+
+ transfer_in_progress = false;
+ return true;
+}
+
+off_t Port::GetSize()
+{
+ return device_size;
+}
+
+blkcnt_t Port::GetBlockCount()
+{
+ return block_count;
+}
+
+blksize_t Port::GetBlockSize()
+{
+ return block_size;
+}
+
+uint16_t Port::GetCylinderCount()
+{
+ return cylinder_count;
+}
+
+uint16_t Port::GetHeadCount()
+{
+ return head_count;
+}
+
+uint16_t Port::GetSectorCount()
+{
+ return sector_count;
+}
+
+const char* Port::GetDriver()
+{
+ return "ahci";
+}
+
+const char* Port::GetModel()
+{
+ return model;
+}
+
+const char* Port::GetSerial()
+{
+ return serial;
+}
+
+const char* Port::GetRevision()
+{
+ return revision;
+}
+
+const unsigned char* Port::GetATAIdentify(size_t* size_ptr)
+{
+ return *size_ptr = sizeof(identify_data), identify_data;
+}
+
+int Port::sync(ioctx_t* ctx)
+{
+ (void) ctx;
+ ScopedLock lock(&port_lock);
+ if ( transfer_in_progress && !FinishTransferDMA() )
+ return -1;
+ PrepareAwaitInterrupt();
+ uint8_t cmd = is_lba48 ? ATA_CMD_FLUSH_CACHE_EXT : ATA_CMD_FLUSH_CACHE;
+ CommandDMA(cmd, 0, false);
+ // TODO: This might take longer than 30 seconds according to the spec. But
+ // how long? Let's say twice that?
+ if ( !AwaitInterrupt(2 * 30000 /*ms*/) )
+ {
+ LogF("error: cache flush timed out");
+ transfer_in_progress = false;
+ return errno = EIO, -1;
+ }
+ transfer_in_progress = false;
+ if ( regs->pxtfd & (ATA_STATUS_ERR | ATA_STATUS_DF) )
+ {
+ LogF("error: IO error");
+ return errno = EIO, -1;
+ }
+ return 0;
+}
+
+ssize_t Port::pread(ioctx_t* ctx, unsigned char* buf, size_t count, off_t off)
+{
+ ScopedLock lock(&port_lock);
+ ssize_t result = 0;
+ while ( count )
+ {
+ if ( device_size <= off )
+ break;
+ if ( (uintmax_t) device_size - off < (uintmax_t) count )
+ count = (size_t) device_size - off;
+ uintmax_t block_index = (uintmax_t) off / (uintmax_t) block_size;
+ uintmax_t block_offset = (uintmax_t) off % (uintmax_t) block_size;
+ uintmax_t amount = block_offset + count;
+ if ( Page::Size() < amount )
+ amount = Page::Size();
+ size_t num_blocks = (amount + block_size - 1) / block_size;
+ uintmax_t full_amount = num_blocks * block_size;
+ // If an asynchronous operation is in progress, let it finish.
+ if ( transfer_in_progress && !FinishTransferDMA() )
+ return result ? result : -1;
+ unsigned char* dma_data = (unsigned char*) dma_alloc.from;
+ unsigned char* data = dma_data + block_offset;
+ size_t data_size = amount - block_offset;
+ Seek(block_index, num_blocks);
+ uint8_t cmd = is_lba48 ? ATA_CMD_READ_DMA_EXT : ATA_CMD_READ_DMA;
+ CommandDMA(cmd, (size_t) full_amount, false);
+ if ( !FinishTransferDMA() )
+ return result ? result : -1;
+ if ( !ctx->copy_to_dest(buf, data, data_size) )
+ return result ? result : -1;
+ buf += data_size;
+ count -= data_size;
+ result += data_size;
+ off += data_size;
+ }
+ return result;
+}
+
+ssize_t Port::pwrite(ioctx_t* ctx, const unsigned char* buf, size_t count, off_t off)
+{
+ ScopedLock lock(&port_lock);
+ ssize_t result = 0;
+ while ( count )
+ {
+ if ( device_size <= off )
+ break;
+ if ( (uintmax_t) device_size - off < (uintmax_t) count )
+ count = (size_t) device_size - off;
+ uintmax_t block_index = (uintmax_t) off / (uintmax_t) block_size;
+ uintmax_t block_offset = (uintmax_t) off % (uintmax_t) block_size;
+ uintmax_t amount = block_offset + count;
+ if ( Page::Size() < amount )
+ amount = Page::Size();
+ size_t num_blocks = (amount + block_size - 1) / block_size;
+ uintmax_t full_amount = num_blocks * block_size;
+ // If an asynchronous operation is in progress, let it finish.
+ if ( transfer_in_progress && !FinishTransferDMA() )
+ return result ? result : -1;
+ unsigned char* dma_data = (unsigned char*) dma_alloc.from;
+ unsigned char* data = dma_data + block_offset;
+ size_t data_size = amount - block_offset;
+ if ( block_offset || amount < full_amount )
+ {
+ Seek(block_index, num_blocks);
+ uint8_t cmd = is_lba48 ? ATA_CMD_READ_DMA_EXT : ATA_CMD_READ_DMA;
+ CommandDMA(cmd, (size_t) full_amount, false);
+ if ( !FinishTransferDMA() )
+ return result ? result : -1;
+ }
+ if ( !ctx->copy_from_src(data, buf, data_size) )
+ return result ? result : -1;
+ Seek(block_index, num_blocks);
+ uint8_t cmd = is_lba48 ? ATA_CMD_WRITE_DMA_EXT : ATA_CMD_WRITE_DMA;
+ CommandDMA(cmd, (size_t) full_amount, true);
+ // Let the transfer finish asynchronously so the caller can prepare the
+ // next write operation to keep the write pipeline busy.
+ buf += data_size;
+ count -= data_size;
+ result += data_size;
+ off += data_size;
+ }
+ return result;
+}
+
+void Port::PrepareAwaitInterrupt()
+{
+ interrupt_signaled = false;
+}
+
+bool Port::AwaitInterrupt(unsigned int msecs)
+{
+ struct timespec timeout = timespec_make(msecs / 1000, (msecs % 1000) * 1000000L);
+ Clock* clock = Time::GetClock(CLOCK_BOOT);
+ struct timespec begun;
+ clock->Get(&begun, NULL);
+ while ( true )
+ {
+ struct timespec now;
+ clock->Get(&now, NULL);
+ if ( interrupt_signaled )
+ return true;
+ struct timespec elapsed = timespec_sub(now, begun);
+ if ( timespec_le(timeout, elapsed) )
+ return errno = ETIMEDOUT, false;
+ // TODO: Can't safely back out here unless the pending operation is
+ // is properly cancelled.
+ //if ( Signal::IsPending() )
+ // return errno = EINTR, false;
+ kthread_yield();
+ }
+}
+
+void Port::OnInterrupt()
+{
+ // Check whether any interrupt are pending.
+ uint32_t is = regs->pxis;
+ if ( !is )
+ return;
+
+ // Clear the pending interrupts.
+ regs->pxis = is;
+
+ // Handle error interrupts.
+ if ( is & PORT_INTR_ERROR )
+ {
+ regs->pxserr = regs->pxserr;
+ // TODO: How exactly should this be handled?
+ }
+
+ if ( !interrupt_signaled )
+ {
+ interrupt_signaled = true;
+ // TODO: Priority schedule the blocking thread now.
+ }
+}
+
+} // namespace AHCI
+} // namespace Sortix
diff --git a/kernel/disk/ahci/port.h b/kernel/disk/ahci/port.h
new file mode 100644
index 00000000..90de038c
--- /dev/null
+++ b/kernel/disk/ahci/port.h
@@ -0,0 +1,122 @@
+/*******************************************************************************
+
+ Copyright(C) Jonas 'Sortie' Termansen 2013, 2014, 2015, 2016.
+
+ This file is part of Sortix.
+
+ Sortix is free software: you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License, or (at your option) any later
+ version.
+
+ Sortix is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ details.
+
+ You should have received a copy of the GNU General Public License along with
+ Sortix. If not, see .
+
+ disk/ahci/port.h
+ Driver for the Advanced Host Controller Interface.
+
+*******************************************************************************/
+
+#ifndef SORTIX_DISK_AHCI_PORT_H
+#define SORTIX_DISK_AHCI_PORT_H
+
+#include
+
+#include
+#include
+
+#include
+#include
+#include
+#include
+
+namespace Sortix {
+namespace AHCI {
+
+struct command_header;
+struct command_table;
+struct fis;
+struct port_regs;
+struct prd;
+
+class HBA;
+
+class Port : public Harddisk
+{
+public:
+ Port(HBA* hba, uint32_t port_index);
+ virtual ~Port();
+
+public:
+ virtual off_t GetSize();
+ virtual blkcnt_t GetBlockCount();
+ virtual blksize_t GetBlockSize();
+ virtual uint16_t GetCylinderCount();
+ virtual uint16_t GetHeadCount();
+ virtual uint16_t GetSectorCount();
+ virtual const char* GetDriver();
+ virtual const char* GetModel();
+ virtual const char* GetSerial();
+ virtual const char* GetRevision();
+ virtual const unsigned char* GetATAIdentify(size_t* size_ptr);
+ virtual int sync(ioctx_t* ctx);
+ virtual ssize_t pread(ioctx_t* ctx, unsigned char* buf, size_t count, off_t off);
+ virtual ssize_t pwrite(ioctx_t* ctx, const unsigned char* buf, size_t count, off_t off);
+
+public:
+ bool Initialize();
+ bool FinishInitialize();
+ void OnInterrupt();
+
+private:
+ __attribute__((format(printf, 2, 3)))
+ void LogF(const char* format, ...);
+ bool Reset();
+ void Seek(blkcnt_t block_index, size_t count);
+ void CommandDMA(uint8_t cmd, size_t size, bool write);
+ bool FinishTransferDMA();
+ void PrepareAwaitInterrupt();
+ bool AwaitInterrupt(unsigned int msescs);
+
+private:
+ kthread_mutex_t port_lock;
+ unsigned char identify_data[512];
+ char serial[20 + 1];
+ char revision[8 + 1];
+ char model[40 + 1];
+ addralloc_t control_alloc;
+ addralloc_t dma_alloc;
+ HBA* hba;
+ volatile struct port_regs* regs;
+ volatile struct command_header* clist;
+ volatile struct fis* fis;
+ volatile struct command_table* ctbl;
+ volatile struct prd* prdt;
+ addr_t control_physical_frame;
+ addr_t dma_physical_frame;
+ uint32_t port_index;
+ bool is_control_page_mapped;
+ bool is_dma_page_mapped;
+ bool is_lba48;
+ off_t device_size;
+ blksize_t block_count;
+ blkcnt_t block_size;
+ uint16_t cylinder_count;
+ uint16_t head_count;
+ uint16_t sector_count;
+ volatile bool interrupt_signaled;
+ bool transfer_in_progress;
+ size_t transfer_size;
+ bool transfer_is_write;
+
+};
+
+} // namespace AHCI
+} // namespace Sortix
+
+#endif
diff --git a/kernel/disk/ahci/registers.h b/kernel/disk/ahci/registers.h
new file mode 100644
index 00000000..67753486
--- /dev/null
+++ b/kernel/disk/ahci/registers.h
@@ -0,0 +1,256 @@
+/*******************************************************************************
+
+ Copyright(C) Jonas 'Sortie' Termansen 2013, 2014, 2015, 2016.
+
+ This file is part of Sortix.
+
+ Sortix is free software: you can redistribute it and/or modify it under the
+ terms of the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License, or (at your option) any later
+ version.
+
+ Sortix is distributed in the hope that it will be useful, but WITHOUT ANY
+ WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ details.
+
+ You should have received a copy of the GNU General Public License along with
+ Sortix. If not, see .
+
+ disk/ahci/registers.h
+ Driver for the Advanced Host Controller Interface.
+
+*******************************************************************************/
+
+#ifndef SORTIX_DISK_AHCI_REGISTERS_H
+#define SORTIX_DISK_AHCI_REGISTERS_H
+
+#include
+#include
+
+namespace Sortix {
+namespace AHCI {
+
+#define AHCI_COMMAND_HEADER_COUNT 32
+
+#if 1 /* ATA STUFF */
+#define ATA_STATUS_ERR (1 << 0) /* Error. */
+#define ATA_STATUS_DRQ (1 << 3) /* Data Request. */
+#define ATA_STATUS_DF (1 << 5) /* Device Fault. */
+#define ATA_STATUS_DRDY (1 << 6) /* Device Ready. */
+#define ATA_STATUS_BSY (1 << 7) /* Busy. */
+
+/** ATA Commands. */
+#define ATA_CMD_READ_DMA 0xC8 /**< READ DMA. */
+#define ATA_CMD_READ_DMA_EXT 0x25 /**< READ DMA EXT. */
+#define ATA_CMD_READ_SECTORS 0x20 /**< READ SECTORS. */
+#define ATA_CMD_READ_SECTORS_EXT 0x24 /**< READ SECTORS EXT. */
+#define ATA_CMD_WRITE_DMA 0xCA /**< WRITE DMA. */
+#define ATA_CMD_WRITE_DMA_EXT 0x35 /**< WRITE DMA EXT. */
+#define ATA_CMD_WRITE_SECTORS 0x30 /**< WRITE SECTORS. */
+#define ATA_CMD_WRITE_SECTORS_EXT 0x34 /**< WRITE SECTORS EXT. */
+#define ATA_CMD_PACKET 0xA0 /**< PACKET. */
+#define ATA_CMD_IDENTIFY_PACKET 0xA1 /**< IDENTIFY PACKET DEVICE. */
+#define ATA_CMD_FLUSH_CACHE 0xE7 /**< FLUSH CACHE. */
+#define ATA_CMD_FLUSH_CACHE_EXT 0xEA /**< FLUSH CACHE EXT. */
+#define ATA_CMD_IDENTIFY 0xEC /**< IDENTIFY DEVICE. */
+#endif
+
+struct fis
+{
+ little_uint8_t dsfis[0x1C]; // DMA Setup FIS.
+ little_uint8_t reserved1[0x04];
+ little_uint8_t psfis[0x14]; // PIO Setup FIS.
+ little_uint8_t reserved2[0x0C];
+ little_uint8_t rfis[0x14]; // D2H Register FIS.
+ little_uint8_t reserved3[0x04];
+ little_uint8_t sdbfis[0x08]; // Set Device Bits FIS.
+ little_uint8_t ufis[0x40]; // Unknown FIS.
+ little_uint8_t reserved4[0x60];
+};
+
+/* TODO: Less blatantly copied from Kiwi: */
+struct command_header
+{
+ /** DW0 - Description Information. */
+ little_uint16_t dw0l;
+ little_uint16_t prdtl;
+ /** DW1 - Command Status. */
+ little_uint32_t prdbc; /**< Physical Region Descriptor Byte Count. */
+ /** DW2 - Command Table Base Address.
+ * @note Bits 0-6 are reserved, must be 0. */
+ little_uint32_t ctba; /**< Command Table Descriptor Base Address. */
+ /** DW3 - Command Table Base Address Upper. */
+ little_uint32_t ctbau; /**< Command Table Descriptor Base Address Upper 32-bits. */
+ /** DW4-7 - Reserved. */
+ little_uint32_t reserved2[4];
+};
+
+static const uint32_t COMMAND_HEADER_DW0_WRITE = 1 << 6;
+
+/* TODO: Less blatantly copied from Kiwi: */
+struct prd
+{
+ /** DW0 - Data Base Address. */
+ little_uint32_t dba;
+ /** DW1 - Data Base Address Upper. */
+ little_uint32_t dbau;
+ /** DW2 - Reserved */
+ little_uint32_t reserved1;
+ /** DW3 - Description Information. */
+ /* dw3 bits 0-21: data byte count */
+ /* dw3 bits 22-30: reserved */
+ /* dw3 bits 31-31: interrupt on completion */
+ little_uint32_t dw3;
+};
+
+/* TODO: Less blatantly copied from Kiwi: */
+struct command_table
+{
+ /** Command FIS - Host to Device. */
+ struct
+ {
+ little_uint8_t type; /**< FIS Type (0x27). */
+ /*little_*/uint8_t pm_port : 4; /**< Port Multiplier Port. */
+ /*little_*/uint8_t reserved1 : 3; /**< Reserved. */
+ /*little_*/uint8_t c_bit : 1; /**< C bit. */
+ little_uint8_t command; /**< ATA Command. */
+ little_uint8_t features_0_7; /**< Features (bits 0-7). */
+ little_uint8_t lba_0_7; /**< LBA (bits 0-7). */
+ little_uint8_t lba_8_15; /**< LBA (bits 8-15). */
+ little_uint8_t lba_16_23; /**< LBA (bits 16-23). */
+ little_uint8_t device; /**< Device (bits 24-27 for LBA28). */
+ little_uint8_t lba_24_31; /**< LBA48 (bits 24-31). */
+ little_uint8_t lba_32_39; /**< LBA48 (bits 32-39). */
+ little_uint8_t lba_40_47; /**< LBA48 (bits 40-47). */
+ little_uint8_t features_8_15; /**< Features (bits 8-15). */
+ little_uint8_t count_0_7; /**< Sector Count (bits 0-7). */
+ little_uint8_t count_8_15; /**< Sector Count (bits 8-15). */
+ little_uint8_t icc; /**< Isochronous Command Completion. */
+ little_uint8_t control; /**< Device Control. */
+ little_uint32_t reserved2; /**< Reserved. */
+ little_uint8_t padding[0x2C]; /**< Padding to 64 bytes. */
+ } cfis;
+ little_uint8_t acmd[0x10]; /**< ATAPI Command (12 or 16 bytes). */
+ little_uint8_t reserved[0x30]; /**< Reserved. */
+};
+
+struct port_regs
+{
+ little_uint32_t pxclb;
+ little_uint32_t pxclbu;
+ little_uint32_t pxfb;
+ little_uint32_t pxfbu;
+ little_uint32_t pxis;
+ little_uint32_t pxie;
+ little_uint32_t pxcmd;
+ little_uint32_t reserved0;
+ little_uint32_t pxtfd;
+ little_uint32_t pxsig;
+ little_uint32_t pxssts;
+ little_uint32_t pxsctl;
+ little_uint32_t pxserr;
+ little_uint32_t pxsact;
+ little_uint32_t pxci;
+ little_uint32_t pxsntf;
+ little_uint32_t pxfbs;
+ little_uint32_t preserved1[15];
+};
+
+/** Bits in the Port x Interrupt Enable register. */
+#define PXIE_DHRE (1 << 0) /**< Device to Host Register Enable. */
+#define PXIE_PSE (1 << 1) /**< PIO Setup FIS Enable. */
+#define PXIE_DSE (1 << 2) /**< DMA Setup FIS Enable. */
+#define PXIE_SDBE (1 << 3) /**< Set Device Bits Enable. */
+#define PXIE_UFE (1 << 4) /**< Unknown FIS Enable. */
+#define PXIE_DPE (1 << 5) /**< Descriptor Processed Enable. */
+#define PXIE_PCE (1 << 6) /**< Port Connect Change Enable. */
+#define PXIE_DMPE (1 << 7) /**< Device Mechanical Presence Enable. */
+#define PXIE_PRCE (1 << 22) /**< PhyRdy Change Enable. */
+#define PXIE_IPME (1 << 23) /**< Incorrect Port Multiplier Enable. */
+#define PXIE_OFE (1 << 24) /**< Overflow Enable. */
+#define PXIE_INFE (1 << 26) /**< Interface Non-Fatal Error Enable. */
+#define PXIE_IFE (1 << 27) /**< Interface Fatal Error Enable. */
+#define PXIE_HBDE (1 << 28) /**< Host Bus Data Error Enable. */
+#define PXIE_HBFE (1 << 29) /**< Host Bus Fatal Error Enable. */
+#define PXIE_TFEE (1 << 30) /**< Task File Error Enable. */
+#define PXIE_CPDE (1 << 31) /**< Cold Port Detect Enable. */
+
+#define PORT_INTR_ERROR \
+ (PXIE_UFE | PXIE_PCE | PXIE_PRCE | PXIE_IPME | \
+ PXIE_OFE | PXIE_INFE | PXIE_IFE | PXIE_HBDE | \
+ PXIE_HBFE | PXIE_TFEE)
+
+static const uint32_t PXCMD_ST = 1 << 0; /* 0x00000001 */
+static const uint32_t PXCMD_SUD = 1 << 1; /* 0x00000002 */
+static const uint32_t PXCMD_POD = 1 << 2; /* 0x00000004 */
+static const uint32_t PXCMD_CLO = 1 << 3; /* 0x00000008 */
+static const uint32_t PXCMD_FRE = 1 << 4; /* 0x00000010 */
+static inline uint32_t PXCMD_CSS(uint32_t val) { return (val >> 8) % 32; }
+static const uint32_t PXCMD_MPSS = 1 << 13; /* 0x00002000 */
+static const uint32_t PXCMD_FR = 1 << 14; /* 0x00004000 */
+static const uint32_t PXCMD_CR = 1 << 15; /* 0x00008000 */
+static const uint32_t PXCMD_CPS = 1 << 16; /* 0x00010000 */
+static const uint32_t PXCMD_PMA = 1 << 17; /* 0x00020000 */
+static const uint32_t PXCMD_HPCP = 1 << 18; /* 0x00040000 */
+static const uint32_t PXCMD_MPSP = 1 << 19; /* 0x00080000 */
+static const uint32_t PXCMD_CPD = 1 << 20; /* 0x00100000 */
+static const uint32_t PXCMD_ESP = 1 << 21; /* 0x00200000 */
+static const uint32_t PXCMD_FBSCP = 1 << 22; /* 0x00400000 */
+static const uint32_t PXCMD_APSTE = 1 << 23; /* 0x00800000 */
+static const uint32_t PXCMD_ATAPI = 1 << 24; /* 0x01000000 */
+static const uint32_t PXCMD_DLAE = 1 << 25; /* 0x02000000 */
+static const uint32_t PXCMD_ALPE = 1 << 26; /* 0x04000000 */
+static const uint32_t PXCMD_ASP = 1 << 27; /* 0x08000000 */
+static inline uint32_t PXCMD_ICC(uint32_t val) { return (val >> 28) % 16; }
+
+struct hba_regs
+{
+ little_uint32_t cap;
+ little_uint32_t ghc;
+ little_uint32_t is;
+ little_uint32_t pi;
+ little_uint32_t vs;
+ little_uint32_t ccc_ctl;
+ little_uint32_t ccc_ports;
+ little_uint32_t em_loc;
+ little_uint32_t em_ctl;
+ little_uint32_t cap2;
+ little_uint32_t bohc;
+ little_uint32_t reserved0[53];
+ struct port_regs ports[32];
+};
+
+static_assert(sizeof(struct hba_regs) == 0x1100,
+ "sizeof(struct hba_regs) == 0x1100");
+
+static const uint32_t CAP_S64A = 1U << 31;
+static const uint32_t CAP_SNCQ = 1U << 30;
+static const uint32_t CAP_SSNTF = 1U << 29;
+static const uint32_t CAP_SMPS = 1U << 28;
+static const uint32_t CAP_SSS = 1U << 27;
+static const uint32_t CAP_SALP = 1U << 26;
+static const uint32_t CAP_SAL = 1U << 25;
+static const uint32_t CAP_SCLO = 1U << 24;
+static inline uint32_t CAP_ISS(uint32_t val) { return (val >> 20) % 16; }
+static const uint32_t CAP_SAM = 1U << 18;
+static const uint32_t CAP_SPM = 1U << 17;
+static const uint32_t CAP_FBSS = 1U << 16;
+static const uint32_t CAP_PMD = 1U << 15;
+static const uint32_t CAP_SSC = 1U << 14;
+static const uint32_t CAP_PSC = 1U << 13;
+static inline uint32_t CAP_NCS(uint32_t val) { return (val >> 8) % 32 + 1; }
+static const uint32_t CAP_CCCS = 1U << 7;
+static const uint32_t CAP_EMS = 1U << 6;
+static const uint32_t CAP_SXS = 1U << 5;
+static inline uint32_t CAP_NP(uint32_t val) { return (val >> 0) % 32 + 1; }
+
+static const uint32_t GHC_AE = 1U << 31;
+static const uint32_t GHC_MRSM = 1U << 2;
+static const uint32_t GHC_IE = 1U << 1;
+static const uint32_t GHC_HR = 1U << 0;
+
+} // namespace AHCI
+} // namespace Sortix
+
+#endif
diff --git a/kernel/kernel.cpp b/kernel/kernel.cpp
index c8e0e696..bcdc0332 100644
--- a/kernel/kernel.cpp
+++ b/kernel/kernel.cpp
@@ -73,6 +73,7 @@
#include
#include "com.h"
+#include "disk/ahci/ahci.h"
#include "disk/ata/ata.h"
#include "fs/full.h"
#include "fs/kram.h"
@@ -530,6 +531,9 @@ static void BootThread(void* /*user*/)
// Search for PCI devices and load their drivers.
PCI::Init();
+ // Initialize AHCI devices.
+ AHCI::Init("/dev", slashdev);
+
// Initialize ATA devices.
ATA::Init("/dev", slashdev);