Merge tag 'for-3.7' of git://git.kernel.org/pub/scm/linux/kernel/git/helgaas/pci
[linux-3.10.git] / drivers / block / nvme.c
index c95ca0f..931769e 100644 (file)
@@ -79,6 +79,7 @@ struct nvme_dev {
        char serial[20];
        char model[40];
        char firmware_rev[8];
+       u32 max_hw_sectors;
 };
 
 /*
@@ -835,15 +836,15 @@ static int nvme_identify(struct nvme_dev *dev, unsigned nsid, unsigned cns,
 }
 
 static int nvme_get_features(struct nvme_dev *dev, unsigned fid,
-                               unsigned dword11, dma_addr_t dma_addr)
+                               unsigned nsid, dma_addr_t dma_addr)
 {
        struct nvme_command c;
 
        memset(&c, 0, sizeof(c));
        c.features.opcode = nvme_admin_get_features;
+       c.features.nsid = cpu_to_le32(nsid);
        c.features.prp1 = cpu_to_le64(dma_addr);
        c.features.fid = cpu_to_le32(fid);
-       c.features.dword11 = cpu_to_le32(dword11);
 
        return nvme_submit_admin_cmd(dev, &c, NULL);
 }
@@ -862,11 +863,51 @@ static int nvme_set_features(struct nvme_dev *dev, unsigned fid,
        return nvme_submit_admin_cmd(dev, &c, result);
 }
 
+/**
+ * nvme_cancel_ios - Cancel outstanding I/Os
+ * @queue: The queue to cancel I/Os on
+ * @timeout: True to only cancel I/Os which have timed out
+ */
+static void nvme_cancel_ios(struct nvme_queue *nvmeq, bool timeout)
+{
+       int depth = nvmeq->q_depth - 1;
+       struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
+       unsigned long now = jiffies;
+       int cmdid;
+
+       for_each_set_bit(cmdid, nvmeq->cmdid_data, depth) {
+               void *ctx;
+               nvme_completion_fn fn;
+               static struct nvme_completion cqe = {
+                       .status = cpu_to_le16(NVME_SC_ABORT_REQ) << 1,
+               };
+
+               if (timeout && !time_after(now, info[cmdid].timeout))
+                       continue;
+               dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d\n", cmdid);
+               ctx = cancel_cmdid(nvmeq, cmdid, &fn);
+               fn(nvmeq->dev, ctx, &cqe);
+       }
+}
+
+static void nvme_free_queue_mem(struct nvme_queue *nvmeq)
+{
+       dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
+                               (void *)nvmeq->cqes, nvmeq->cq_dma_addr);
+       dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
+                                       nvmeq->sq_cmds, nvmeq->sq_dma_addr);
+       kfree(nvmeq);
+}
+
 static void nvme_free_queue(struct nvme_dev *dev, int qid)
 {
        struct nvme_queue *nvmeq = dev->queues[qid];
        int vector = dev->entry[nvmeq->cq_vector].vector;
 
+       spin_lock_irq(&nvmeq->q_lock);
+       nvme_cancel_ios(nvmeq, false);
+       spin_unlock_irq(&nvmeq->q_lock);
+
        irq_set_affinity_hint(vector, NULL);
        free_irq(vector, nvmeq);
 
@@ -876,18 +917,15 @@ static void nvme_free_queue(struct nvme_dev *dev, int qid)
                adapter_delete_cq(dev, qid);
        }
 
-       dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
-                               (void *)nvmeq->cqes, nvmeq->cq_dma_addr);
-       dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
-                                       nvmeq->sq_cmds, nvmeq->sq_dma_addr);
-       kfree(nvmeq);
+       nvme_free_queue_mem(nvmeq);
 }
 
 static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
                                                        int depth, int vector)
 {
        struct device *dmadev = &dev->pci_dev->dev;
-       unsigned extra = (depth / 8) + (depth * sizeof(struct nvme_cmd_info));
+       unsigned extra = DIV_ROUND_UP(depth, 8) + (depth *
+                                               sizeof(struct nvme_cmd_info));
        struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq) + extra, GFP_KERNEL);
        if (!nvmeq)
                return NULL;
@@ -975,7 +1013,7 @@ static __devinit struct nvme_queue *nvme_create_queue(struct nvme_dev *dev,
 
 static int __devinit nvme_configure_admin_queue(struct nvme_dev *dev)
 {
-       int result;
+       int result = 0;
        u32 aqa;
        u64 cap;
        unsigned long timeout;
@@ -1005,17 +1043,22 @@ static int __devinit nvme_configure_admin_queue(struct nvme_dev *dev)
        timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
        dev->db_stride = NVME_CAP_STRIDE(cap);
 
-       while (!(readl(&dev->bar->csts) & NVME_CSTS_RDY)) {
+       while (!result && !(readl(&dev->bar->csts) & NVME_CSTS_RDY)) {
                msleep(100);
                if (fatal_signal_pending(current))
-                       return -EINTR;
+                       result = -EINTR;
                if (time_after(jiffies, timeout)) {
                        dev_err(&dev->pci_dev->dev,
                                "Device not ready; aborting initialisation\n");
-                       return -ENODEV;
+                       result = -ENODEV;
                }
        }
 
+       if (result) {
+               nvme_free_queue_mem(nvmeq);
+               return result;
+       }
+
        result = queue_request_irq(dev, nvmeq, "nvme admin");
        dev->queues[0] = nvmeq;
        return result;
@@ -1037,6 +1080,8 @@ static struct nvme_iod *nvme_map_user_pages(struct nvme_dev *dev, int write,
        offset = offset_in_page(addr);
        count = DIV_ROUND_UP(offset + length, PAGE_SIZE);
        pages = kcalloc(count, sizeof(*pages), GFP_KERNEL);
+       if (!pages)
+               return ERR_PTR(-ENOMEM);
 
        err = get_user_pages_fast(addr, count, 1, pages);
        if (err < count) {
@@ -1146,14 +1191,13 @@ static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
        return status;
 }
 
-static int nvme_user_admin_cmd(struct nvme_ns *ns,
+static int nvme_user_admin_cmd(struct nvme_dev *dev,
                                        struct nvme_admin_cmd __user *ucmd)
 {
-       struct nvme_dev *dev = ns->dev;
        struct nvme_admin_cmd cmd;
        struct nvme_command c;
        int status, length;
-       struct nvme_iod *iod;
+       struct nvme_iod *uninitialized_var(iod);
 
        if (!capable(CAP_SYS_ADMIN))
                return -EACCES;
@@ -1204,7 +1248,7 @@ static int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd,
        case NVME_IOCTL_ID:
                return ns->ns_id;
        case NVME_IOCTL_ADMIN_CMD:
-               return nvme_user_admin_cmd(ns, (void __user *)arg);
+               return nvme_user_admin_cmd(ns->dev, (void __user *)arg);
        case NVME_IOCTL_SUBMIT_IO:
                return nvme_submit_io(ns, (void __user *)arg);
        default:
@@ -1218,26 +1262,6 @@ static const struct block_device_operations nvme_fops = {
        .compat_ioctl   = nvme_ioctl,
 };
 
-static void nvme_timeout_ios(struct nvme_queue *nvmeq)
-{
-       int depth = nvmeq->q_depth - 1;
-       struct nvme_cmd_info *info = nvme_cmd_info(nvmeq);
-       unsigned long now = jiffies;
-       int cmdid;
-
-       for_each_set_bit(cmdid, nvmeq->cmdid_data, depth) {
-               void *ctx;
-               nvme_completion_fn fn;
-               static struct nvme_completion cqe = { .status = cpu_to_le16(NVME_SC_ABORT_REQ) << 1, };
-
-               if (!time_after(now, info[cmdid].timeout))
-                       continue;
-               dev_warn(nvmeq->q_dmadev, "Timing out I/O %d\n", cmdid);
-               ctx = cancel_cmdid(nvmeq, cmdid, &fn);
-               fn(nvmeq->dev, ctx, &cqe);
-       }
-}
-
 static void nvme_resubmit_bios(struct nvme_queue *nvmeq)
 {
        while (bio_list_peek(&nvmeq->sq_cong)) {
@@ -1269,7 +1293,7 @@ static int nvme_kthread(void *data)
                                spin_lock_irq(&nvmeq->q_lock);
                                if (nvme_process_cq(nvmeq))
                                        printk("process_cq did something\n");
-                               nvme_timeout_ios(nvmeq);
+                               nvme_cancel_ios(nvmeq, true);
                                nvme_resubmit_bios(nvmeq);
                                spin_unlock_irq(&nvmeq->q_lock);
                        }
@@ -1339,6 +1363,9 @@ static struct nvme_ns *nvme_alloc_ns(struct nvme_dev *dev, int nsid,
        ns->disk = disk;
        lbaf = id->flbas & 0xf;
        ns->lba_shift = id->lbaf[lbaf].ds;
+       blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
+       if (dev->max_hw_sectors)
+               blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors);
 
        disk->major = nvme_major;
        disk->minors = NVME_MINORS;
@@ -1383,7 +1410,7 @@ static int set_queue_count(struct nvme_dev *dev, int count)
 
 static int __devinit nvme_setup_io_queues(struct nvme_dev *dev)
 {
-       int result, cpu, i, nr_io_queues, db_bar_size;
+       int result, cpu, i, nr_io_queues, db_bar_size, q_depth;
 
        nr_io_queues = num_online_cpus();
        result = set_queue_count(dev, nr_io_queues);
@@ -1429,9 +1456,10 @@ static int __devinit nvme_setup_io_queues(struct nvme_dev *dev)
                cpu = cpumask_next(cpu, cpu_online_mask);
        }
 
+       q_depth = min_t(int, NVME_CAP_MQES(readq(&dev->bar->cap)) + 1,
+                                                               NVME_Q_DEPTH);
        for (i = 0; i < nr_io_queues; i++) {
-               dev->queues[i + 1] = nvme_create_queue(dev, i + 1,
-                                                       NVME_Q_DEPTH, i);
+               dev->queues[i + 1] = nvme_create_queue(dev, i + 1, q_depth, i);
                if (IS_ERR(dev->queues[i + 1]))
                        return PTR_ERR(dev->queues[i + 1]);
                dev->queue_count++;
@@ -1480,6 +1508,10 @@ static int __devinit nvme_dev_add(struct nvme_dev *dev)
        memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn));
        memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn));
        memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr));
+       if (ctrl->mdts) {
+               int shift = NVME_CAP_MPSMIN(readq(&dev->bar->cap)) + 12;
+               dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9);
+       }
 
        id_ns = mem;
        for (i = 1; i <= nn; i++) {
@@ -1523,8 +1555,6 @@ static int nvme_dev_remove(struct nvme_dev *dev)
        list_del(&dev->node);
        spin_unlock(&dev_list_lock);
 
-       /* TODO: wait all I/O finished or cancel them */
-
        list_for_each_entry_safe(ns, next, &dev->namespaces, list) {
                list_del(&ns->list);
                del_gendisk(ns->disk);
@@ -1560,15 +1590,33 @@ static void nvme_release_prp_pools(struct nvme_dev *dev)
        dma_pool_destroy(dev->prp_small_pool);
 }
 
-/* XXX: Use an ida or something to let remove / add work correctly */
-static void nvme_set_instance(struct nvme_dev *dev)
+static DEFINE_IDA(nvme_instance_ida);
+
+static int nvme_set_instance(struct nvme_dev *dev)
 {
-       static int instance;
-       dev->instance = instance++;
+       int instance, error;
+
+       do {
+               if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
+                       return -ENODEV;
+
+               spin_lock(&dev_list_lock);
+               error = ida_get_new(&nvme_instance_ida, &instance);
+               spin_unlock(&dev_list_lock);
+       } while (error == -EAGAIN);
+
+       if (error)
+               return -ENODEV;
+
+       dev->instance = instance;
+       return 0;
 }
 
 static void nvme_release_instance(struct nvme_dev *dev)
 {
+       spin_lock(&dev_list_lock);
+       ida_remove(&nvme_instance_ida, dev->instance);
+       spin_unlock(&dev_list_lock);
 }
 
 static int __devinit nvme_probe(struct pci_dev *pdev,
@@ -1601,7 +1649,10 @@ static int __devinit nvme_probe(struct pci_dev *pdev,
        pci_set_drvdata(pdev, dev);
        dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
        dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
-       nvme_set_instance(dev);
+       result = nvme_set_instance(dev);
+       if (result)
+               goto disable;
+
        dev->entry[0].vector = pdev->irq;
 
        result = nvme_setup_prp_pools(dev);
@@ -1704,15 +1755,17 @@ static struct pci_driver nvme_driver = {
 
 static int __init nvme_init(void)
 {
-       int result = -EBUSY;
+       int result;
 
        nvme_thread = kthread_run(nvme_kthread, NULL, "nvme");
        if (IS_ERR(nvme_thread))
                return PTR_ERR(nvme_thread);
 
-       nvme_major = register_blkdev(nvme_major, "nvme");
-       if (nvme_major <= 0)
+       result = register_blkdev(nvme_major, "nvme");
+       if (result < 0)
                goto kill_kthread;
+       else if (result > 0)
+               nvme_major = result;
 
        result = pci_register_driver(&nvme_driver);
        if (result)