From 29234bb23abe2ded7a9e0e36be78bfd98f0e6fff Mon Sep 17 00:00:00 2001 From: Ian Moffett Date: Wed, 10 Jul 2024 17:25:45 -0400 Subject: kernel: nvme: Add initial NVMe driver code Signed-off-by: Ian Moffett --- sys/dev/ic/nvme.c | 389 ++++++++++++++++++++++++++++++++++++++++++ sys/include/dev/ic/nvmeregs.h | 74 ++++++++ sys/include/dev/ic/nvmevar.h | 129 ++++++++++++++ 3 files changed, 592 insertions(+) create mode 100644 sys/dev/ic/nvme.c create mode 100644 sys/include/dev/ic/nvmeregs.h create mode 100644 sys/include/dev/ic/nvmevar.h (limited to 'sys') diff --git a/sys/dev/ic/nvme.c b/sys/dev/ic/nvme.c new file mode 100644 index 0000000..5f92d0e --- /dev/null +++ b/sys/dev/ic/nvme.c @@ -0,0 +1,389 @@ +/* + * Copyright (c) 2023-2024 Ian Marco Moffett and the Osmora Team. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of Hyra nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define pr_trace(fmt, ...) kprintf("nvme: " fmt, ##__VA_ARGS__) +#define pr_error(...) pr_trace(__VA_ARGS__) + +static struct pci_device *nvme_dev; +static struct timer tmr; + +static inline int +is_4k_aligned(void *ptr) +{ + return ((uintptr_t)ptr & (0x1000 - 1)) == 0; +} + +/* + * Poll register to have 'bits' set/unset. + * + * @reg: Register to poll. + * @bits: Bits to be checked. + * @pollset: True to poll as set. + */ +static int +nvme_poll_reg(struct nvme_bar *bar, volatile uint32_t *reg, uint32_t bits, + bool pollset) +{ + size_t usec_start, usec; + size_t elapsed_msec; + uint32_t val, caps; + bool tmp; + + usec_start = tmr.get_time_usec(); + caps = mmio_read32(&bar->caps); + + for (;;) { + val = mmio_read32(reg); + tmp = (pollset) ? ISSET(val, bits) : !ISSET(val, bits); + + usec = tmr.get_time_usec(); + elapsed_msec = (usec - usec_start) / 1000; + + /* If tmp is set, the register updated in time */ + if (tmp) { + break; + } + + /* Exit with an error if we timeout */ + if (elapsed_msec > CAP_TIMEOUT(caps)) { + return -ETIME; + } + } + + return val; +} + +static int +nvme_create_queue(struct nvme_bar *bar, struct nvme_queue *queue, size_t id) +{ + uint8_t dbstride; + uint16_t slots; + uint64_t caps; + uintptr_t sq_db, cq_db; + + caps = mmio_read32(&bar->caps); + dbstride = CAP_STRIDE(caps); + slots = CAP_MQES(caps); + + queue->sq = dynalloc_memalign(sizeof(void *) * slots, 0x1000); + queue->cq = dynalloc_memalign(sizeof(void *) * slots, 0x1000); + + if (queue->sq == NULL) { + return -ENOMEM; + } + + if (queue->cq == NULL) { + dynfree(queue->sq); + return -ENOMEM; + } + + memset(queue->sq, 0, sizeof(void *) * slots); + memset(queue->cq, 0, sizeof(void *) * slots); + + sq_db = (uintptr_t)bar + DEFAULT_PAGESIZE + (2 * id * (4 << dbstride)); + cq_db = (uintptr_t)bar + DEFAULT_PAGESIZE + ((2 * id + 1) * (4 << dbstride)); + + queue->sq_head = 0; + queue->sq_tail = 0; + + queue->size = slots; + queue->cq_phase = 1; + queue->sq_db = (void *)sq_db; + queue->cq_db = (void *)cq_db; + return 0; +} + +/* + * Stop and reset the NVMe controller. + */ +static int +nvme_stop_ctrl(struct nvme_bar *bar) +{ + uint32_t config, status; + + /* Do not reset if CSTS.RDY is 0 */ + status = mmio_read32(&bar->status); + if (!ISSET(status, STATUS_RDY)) { + return 0; + } + + /* Clear the enable bit to begin the reset */ + config = mmio_read32(&bar->config); + config &= ~CONFIG_EN; + mmio_write32(&bar->config, config); + + if (nvme_poll_reg(bar, &bar->status, STATUS_RDY, false) < 0) { + pr_error("Controller reset timeout\n"); + return -ETIME; + } + + return 0; +} + +/* + * Start up the controller. + */ +static int +nvme_start_ctrl(struct nvme_bar *bar) +{ + uint32_t config, status; + + /* Cannot start if already started */ + status = mmio_read32(&bar->status); + if (ISSET(status, STATUS_RDY)) { + return 0; + } + + /* Enable the controller */ + config = mmio_read32(&bar->config); + config |= CONFIG_EN; + mmio_write32(&bar->config, config); + + if (nvme_poll_reg(bar, &bar->status, STATUS_RDY, true) < 0) { + pr_error("Controller startup timeout\n"); + return -ETIME; + } + + return 0; +} + +/* + * Submit a command. + */ +static void +nvme_submit_cmd(struct nvme_queue *q, struct nvme_cmd cmd) +{ + q->sq[q->sq_tail++] = cmd; + if (q->sq_tail >= q->size) { + q->sq_tail = 0; + } + + mmio_write32(q->sq_db, q->sq_tail); +} + +/* + * Submit a command and poll for completion. + */ +static int +nvme_poll_submit_cmd(struct nvme_queue *q, struct nvme_cmd cmd) +{ + uint16_t status; + uint8_t spins = 0; + + nvme_submit_cmd(q, cmd); + + for (;;) { + /* + * If the phase bit matches the most recently submitted + * command then the command has completed + */ + status = q->cq[q->cq_head].status; + if ((status & 1) == q->cq_phase) { + break; + } + + /* Are any error bits set? */ + if ((status & ~1) != 0) { + pr_trace("Command error (bits=0x%x)\n", status >> 1); + return -EIO; + } + + /* Check for timeout */ + if (spins > 5) { + pr_error("Hang while polling phase bit, giving up\n"); + return -ETIME; + } + + tmr.msleep(150); + ++spins; + } + + return 0; +} + +static int +nvme_identify(struct nvme_ctrl *ctrl, struct nvme_id *id) +{ + struct nvme_cmd cmd = {0}; + struct nvme_identify_cmd *idcmd = &cmd.identify; + + if (!is_4k_aligned(id)) { + return -1; + } + + idcmd->opcode = NVME_OP_IDENTIFY; + idcmd->nsid = 0; + idcmd->cns = 1; /* Identify controller */ + idcmd->prp1 = VIRT_TO_PHYS(id); + idcmd->prp2 = 0; + return nvme_poll_submit_cmd(&ctrl->adminq, cmd); +} + +/* + * For debugging purposes, logs some information + * found within the controller identify data structure. + */ +static void +nvme_log_ctrl_id(struct nvme_id *id) +{ + char mn[41] = {0}; + char sn[21] = {0}; + char fr[9] = {0}; + + for (size_t i = 0; i < sizeof(id->mn); ++i) { + mn[i] = id->mn[i]; + } + + for (size_t i = 0; i < sizeof(id->fr); ++i) { + fr[i] = id->fr[i]; + } + + for (size_t i = 0; i < sizeof(id->sn); ++i) { + sn[i] = id->sn[i]; + } + + pr_trace("Model number: %s\n", mn); + pr_trace("Serial number: %s\n", sn); + pr_trace("Firmware revision: %s\n", fr); +} + +/* + * Init PCI related controller bits + */ +static void +nvme_init_pci(void) +{ + uint32_t tmp; + + /* Enable bus mastering and MMIO */ + tmp = pci_readl(nvme_dev, PCIREG_CMDSTATUS); + tmp |= (PCI_BUS_MASTERING | PCI_MEM_SPACE); + pci_writel(nvme_dev, PCIREG_CMDSTATUS, tmp); +} + +static int +nvme_init_ctrl(struct nvme_bar *bar) +{ + int error; + uint64_t caps; + uint16_t mqes; + struct nvme_ctrl ctrl = {0}; + struct nvme_queue *adminq; + struct nvme_id *id; + + /* Ensure the controller is stopped */ + if ((error = nvme_stop_ctrl(bar)) != 0) { + return error; + } + + adminq = &ctrl.adminq; + caps = mmio_read64(&bar->caps); + mqes = CAP_MQES(caps); + + /* Setup admin queues */ + nvme_create_queue(bar, adminq, 0); + mmio_write32(&bar->aqa, (mqes | mqes << 16)); + mmio_write64(&bar->asq, VIRT_TO_PHYS(adminq->sq)); + mmio_write64(&bar->acq, VIRT_TO_PHYS(adminq->cq)); + + /* Now bring the controller back up */ + if ((error = nvme_start_ctrl(bar)) != 0) { + return error; + } + + id = dynalloc_memalign(sizeof(*id), 0x1000); + if (id == NULL) { + return -ENOMEM; + } + + nvme_identify(&ctrl, id); + nvme_log_ctrl_id(id); + dynfree(id); + return 0; +} + +static int +nvme_init(void) +{ + struct pci_lookup lookup; + struct nvme_bar *bar; + int error; + + lookup.pci_class = 1; + lookup.pci_subclass = 8; + nvme_dev = pci_get_device(lookup, PCI_CLASS | PCI_SUBCLASS); + + if (nvme_dev == NULL) { + return -ENODEV; + } + + /* Try to request a general purpose timer */ + if (req_timer(TIMER_GP, &tmr) != TMRR_SUCCESS) { + pr_error("Failed to fetch general purpose timer\n"); + return -ENODEV; + } + + /* Ensure it has get_time_usec() */ + if (tmr.get_time_usec == NULL) { + pr_error("General purpose timer has no get_time_usec()\n"); + return -ENODEV; + } + + /* We also need msleep() */ + if (tmr.msleep == NULL) { + pr_error("General purpose timer has no msleep()\n"); + return -ENODEV; + } + + nvme_init_pci(); + + if ((error = pci_map_bar(nvme_dev, 0, (void *)&bar)) != 0) { + return error; + } + + return nvme_init_ctrl(bar); +} + +DRIVER_EXPORT(nvme_init); diff --git a/sys/include/dev/ic/nvmeregs.h b/sys/include/dev/ic/nvmeregs.h new file mode 100644 index 0000000..6b18e4d --- /dev/null +++ b/sys/include/dev/ic/nvmeregs.h @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2023-2024 Ian Marco Moffett and the Osmora Team. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of Hyra nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _IC_NVMEREGS_H_ +#define _IC_NVMEREGS_H_ + +#include +#include + +/* Controller Capabilities */ +#define CAP_MPSMIN(caps) ((caps >> 48) & 0xF) +#define CAP_MPSMAX(caps) ((caps >> 52) & 0xF) +#define CAP_TIMEOUT(caps) ((caps >> 24) & 0xFF) +#define CAP_STRIDE(caps) ((caps >> 32) & 0xF) +#define CAP_MQES(caps) (caps & 0xFFFF) +#define CAP_CSS(caps) (caps & 0xFF) + +/* Controller Configuration */ +#define CONFIG_EN BIT(0) +#define CONFIG_CSS_SHIFT 4 +#define CONFIG_IOSQES_SHIFT 16 +#define CONFIG_IOCQES_SHIFT 20 + +/* Controller status */ +#define STATUS_RDY BIT(0) +#define STATUS_CFS BIT(1) + +/* Command sets supported */ +#define CSS_NVM BIT(0) +#define CSS_IO BIT(6) +#define CSS_NO_IO BIT(7) + +/* NVMe controller */ +struct nvme_bar { + volatile uint64_t caps; + volatile uint32_t version; + volatile uint32_t intms; /* Interrupt mask set */ + volatile uint32_t intmc; /* Interrupt mask clear */ + volatile uint32_t config; + volatile uint32_t unused1; + volatile uint32_t status; + volatile uint32_t unused2; + volatile uint32_t aqa; /* Admin queue attributes */ + volatile uint64_t asq; /* Admin submission queue */ + volatile uint64_t acq; /* Admin completion queue */ +}; + +#endif /* !_IC_NVMEREGS_H_ */ diff --git a/sys/include/dev/ic/nvmevar.h b/sys/include/dev/ic/nvmevar.h new file mode 100644 index 0000000..aa68331 --- /dev/null +++ b/sys/include/dev/ic/nvmevar.h @@ -0,0 +1,129 @@ +/* + * Copyright (c) 2023-2024 Ian Marco Moffett and the Osmora Team. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of Hyra nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _IC_NVMEVAR_H_ +#define _IC_NVMEVAR_H_ + +#include + +/* Admin commands */ +#define NVME_OP_CREATE_IOSQ 0x01 +#define NVME_OP_CREATE_IOCQ 0x05 +#define NVME_OP_IDENTIFY 0x06 + +struct nvme_identify_cmd { + uint8_t opcode; + uint8_t flags; + uint16_t cid; + uint32_t nsid; + uint64_t unused1; + uint64_t unused2; + uint64_t prp1; + uint64_t prp2; + uint32_t cns; + uint32_t unused3[5]; +}; + +/* Command completion queue entry */ +struct nvme_cq_entry { + uint32_t res; + uint32_t unused; + uint16_t sqhead; + uint16_t sqid; + uint16_t cid; + uint16_t status; +}; + +struct nvme_cmd { + union { + struct nvme_identify_cmd identify; + }; +}; + +struct nvme_queue { + struct nvme_cmd *sq; /* Submission queue */ + struct nvme_cq_entry *cq; /* Completion queue */ + uint16_t sq_head; /* Submission queue head */ + uint16_t sq_tail; /* Submission queue tail */ + uint16_t cq_head; /* Completion queue head */ + uint8_t cq_phase : 1; /* Completion queue phase bit */ + uint16_t size; /* Size in elements */ + volatile uint32_t *sq_db; /* Submission doorbell */ + volatile uint32_t *cq_db; /* Completion doorbell */ +}; + +struct nvme_id { + uint16_t vid; + uint16_t ssvid; + char sn[20]; + char mn[40]; + char fr[8]; + uint8_t rab; + uint8_t ieee[3]; + uint8_t mic; + uint8_t mdts; + uint16_t ctrlid; + uint32_t version; + uint32_t unused1[43]; + uint16_t oacs; + uint8_t acl; + uint8_t aerl; + uint8_t fw; + uint8_t lpa; + uint8_t elpe; + uint8_t npss; + uint8_t avscc; + uint8_t apsta; + uint16_t wctemp; + uint16_t cctemp; + uint16_t unused2[121]; + uint8_t sqes; + uint8_t cqes; + uint16_t unused3; + uint32_t nn; + uint16_t oncs; + uint16_t fuses; + uint8_t fna; + uint8_t vwc; + uint16_t awun; + uint16_t awupf; + uint8_t nvscc; + uint8_t unused4; + uint16_t acwu; + uint16_t unused5; + uint32_t sgls; + uint32_t unused6[1401]; + uint8_t vs[1024]; +}; + +struct nvme_ctrl { + struct nvme_queue adminq; +}; + +#endif /* !_IC_NVMEVAR_H_ */ -- cgit v1.2.3