From 0df5b5e14b96993dc9babe52361b00f1bff587c5 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Mon, 19 Dec 2016 19:30:55 -0800 Subject: [PATCH] nvme: move namespace scanning to common code The namespace scanning code has been mostly generic already, we just need to store a pointer to the tagset in the nvme_ctrl structure, and add a method to check if a controller is I/O incapable. The latter will hopefully be replaced by a proper controller state machine soon. Signed-off-by: Christoph Hellwig [Fixed pr conflicts] Signed-off-by: Keith Busch Signed-off-by: Jens Axboe (cherry picked from commit 5bae7f73d378a986671a3cad717c721b38f80d9e) Orabug: 25130845 Conflicts: drivers/nvme/host/nvme.h drivers/nvme/host/pci.c Signed-off-by: Ashok Vairavan Reviewed-by: Martin K. Petersen --- drivers/nvme/host/core.c | 177 +++++++++++++++++++++- drivers/nvme/host/nvme.h | 25 +++- drivers/nvme/host/pci.c | 314 ++++----------------------------------- 3 files changed, 225 insertions(+), 291 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 3d1fddf8e2ed..e2fe30abe671 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -18,6 +18,8 @@ #include #include #include +#include +#include #include #include #include @@ -29,6 +31,9 @@ #include "nvme.h" +static int nvme_major; +module_param(nvme_major, int, 0); + DEFINE_SPINLOCK(dev_list_lock); static void nvme_free_ns(struct kref *kref) @@ -44,7 +49,7 @@ static void nvme_free_ns(struct kref *kref) kfree(ns); } -void nvme_put_ns(struct nvme_ns *ns) +static void nvme_put_ns(struct nvme_ns *ns) { kref_put(&ns->kref, nvme_free_ns); } @@ -470,7 +475,7 @@ static void nvme_config_discard(struct nvme_ns *ns) queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue); } -int nvme_revalidate_disk(struct gendisk *disk) +static int nvme_revalidate_disk(struct gendisk *disk) { struct nvme_ns *ns = disk->private_data; struct nvme_id_ns *id; @@ -620,7 +625,7 @@ static const struct pr_ops nvme_pr_ops = { .pr_clear = nvme_pr_clear, }; -const struct block_device_operations nvme_fops = { +static const struct block_device_operations nvme_fops = { .owner = THIS_MODULE, .ioctl = nvme_ioctl, .compat_ioctl = nvme_compat_ioctl, @@ -800,3 +805,169 @@ void nvme_put_ctrl(struct nvme_ctrl *ctrl) kref_put(&ctrl->kref, nvme_free_ctrl); } +static int ns_cmp(void *priv, struct list_head *a, struct list_head *b) +{ + struct nvme_ns *nsa = container_of(a, struct nvme_ns, list); + struct nvme_ns *nsb = container_of(b, struct nvme_ns, list); + + return nsa->ns_id - nsb->ns_id; +} + +static struct nvme_ns *nvme_find_ns(struct nvme_ctrl *ctrl, unsigned nsid) +{ + struct nvme_ns *ns; + + list_for_each_entry(ns, &ctrl->namespaces, list) { + if (ns->ns_id == nsid) + return ns; + if (ns->ns_id > nsid) + break; + } + return NULL; +} + +static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid) +{ + struct nvme_ns *ns; + struct gendisk *disk; + int node = dev_to_node(ctrl->dev); + + ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node); + if (!ns) + return; + + ns->queue = blk_mq_init_queue(ctrl->tagset); + if (IS_ERR(ns->queue)) + goto out_free_ns; + queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue); + queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue); + ns->queue->queuedata = ns; + ns->ctrl = ctrl; + + disk = alloc_disk_node(0, node); + if (!disk) + goto out_free_queue; + + kref_init(&ns->kref); + ns->ns_id = nsid; + ns->disk = disk; + ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */ + list_add_tail(&ns->list, &ctrl->namespaces); + + blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift); + if (ctrl->max_hw_sectors) { + blk_queue_max_hw_sectors(ns->queue, ctrl->max_hw_sectors); + blk_queue_max_segments(ns->queue, + (ctrl->max_hw_sectors / (ctrl->page_size >> 9)) + 1); + } + if (ctrl->stripe_size) + blk_queue_chunk_sectors(ns->queue, ctrl->stripe_size >> 9); + if (ctrl->vwc & NVME_CTRL_VWC_PRESENT) + blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA); + + disk->major = nvme_major; + disk->first_minor = 0; + disk->fops = &nvme_fops; + disk->private_data = ns; + disk->queue = ns->queue; + disk->driverfs_dev = ctrl->device; + disk->flags = GENHD_FL_EXT_DEVT; + sprintf(disk->disk_name, "nvme%dn%d", ctrl->instance, nsid); + + /* + * Initialize capacity to 0 until we establish the namespace format and + * setup integrity extentions if necessary. The revalidate_disk after + * add_disk allows the driver to register with integrity if the format + * requires it. + */ + set_capacity(disk, 0); + if (nvme_revalidate_disk(ns->disk)) + goto out_free_disk; + + kref_get(&ctrl->kref); + + return; + out_free_disk: + kfree(disk); + list_del(&ns->list); + out_free_queue: + blk_cleanup_queue(ns->queue); + out_free_ns: + kfree(ns); +} + +static void nvme_ns_remove(struct nvme_ns *ns) +{ + bool kill = nvme_io_incapable(ns->ctrl) && + !blk_queue_dying(ns->queue); + + if (kill) + blk_set_queue_dying(ns->queue); + if (ns->disk->flags & GENHD_FL_UP) { + if (blk_get_integrity(ns->disk)) + blk_integrity_unregister(ns->disk); + del_gendisk(ns->disk); + } + if (kill || !blk_queue_dying(ns->queue)) { + blk_mq_abort_requeue_list(ns->queue); + blk_cleanup_queue(ns->queue); + } + list_del_init(&ns->list); + nvme_put_ns(ns); +} + +static void __nvme_scan_namespaces(struct nvme_ctrl *ctrl, unsigned nn) +{ + struct nvme_ns *ns, *next; + unsigned i; + + for (i = 1; i <= nn; i++) { + ns = nvme_find_ns(ctrl, i); + if (ns) { + if (revalidate_disk(ns->disk)) + nvme_ns_remove(ns); + } else + nvme_alloc_ns(ctrl, i); + } + list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) { + if (ns->ns_id > nn) + nvme_ns_remove(ns); + } + list_sort(NULL, &ctrl->namespaces, ns_cmp); +} + +void nvme_scan_namespaces(struct nvme_ctrl *ctrl) +{ + struct nvme_id_ctrl *id; + + if (nvme_identify_ctrl(ctrl, &id)) + return; + __nvme_scan_namespaces(ctrl, le32_to_cpup(&id->nn)); + kfree(id); +} + +void nvme_remove_namespaces(struct nvme_ctrl *ctrl) +{ + struct nvme_ns *ns, *next; + + list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) + nvme_ns_remove(ns); +} + +int __init nvme_core_init(void) +{ + int result; + + result = register_blkdev(nvme_major, "nvme"); + if (result < 0) + return result; + else if (result > 0) + nvme_major = result; + + return 0; +} + +void nvme_core_exit(void) +{ + unregister_blkdev(nvme_major, "nvme"); +} diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index e116e284dcc7..81acd98b9dd1 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -53,6 +53,10 @@ struct nvme_ctrl { struct device *dev; struct kref kref; int instance; + struct blk_mq_tag_set *tagset; + struct list_head namespaces; + struct device *device; /* char device */ + char name[12]; char serial[20]; char model[40]; @@ -94,6 +98,7 @@ struct nvme_ctrl_ops { int (*reg_read32)(struct nvme_ctrl *ctrl, u32 off, u32 *val); int (*reg_write32)(struct nvme_ctrl *ctrl, u32 off, u32 val); int (*reg_read64)(struct nvme_ctrl *ctrl, u32 off, u64 *val); + bool (*io_incapable)(struct nvme_ctrl *ctrl); void (*free_ctrl)(struct nvme_ctrl *ctrl); }; @@ -106,6 +111,17 @@ static inline bool nvme_ctrl_ready(struct nvme_ctrl *ctrl) return val & NVME_CSTS_RDY; } +static inline bool nvme_io_incapable(struct nvme_ctrl *ctrl) +{ + u32 val = 0; + + if (ctrl->ops->io_incapable(ctrl)) + return false; + if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &val)) + return false; + return val & NVME_CSTS_CFS; +} + static inline u64 nvme_block_nr(struct nvme_ns *ns, sector_t sector) { return (sector >> (ns->lba_shift - 9)); @@ -179,7 +195,9 @@ int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap); int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl); void nvme_put_ctrl(struct nvme_ctrl *ctrl); int nvme_init_identify(struct nvme_ctrl *ctrl); -void nvme_put_ns(struct nvme_ns *ns); + +void nvme_scan_namespaces(struct nvme_ctrl *ctrl); +void nvme_remove_namespaces(struct nvme_ctrl *ctrl); struct request *nvme_alloc_request(struct request_queue *q, struct nvme_command *cmd, unsigned int flags); @@ -203,10 +221,8 @@ int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid, int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11, dma_addr_t dma_addr, u32 *result); -extern const struct block_device_operations nvme_fops; extern spinlock_t dev_list_lock; -int nvme_revalidate_disk(struct gendisk *disk); int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns, struct nvme_passthru_cmd __user *ucmd); @@ -216,4 +232,7 @@ int nvme_sg_io(struct nvme_ns *ns, struct sg_io_hdr __user *u_hdr); int nvme_sg_io32(struct nvme_ns *ns, unsigned long arg); int nvme_sg_get_version_num(int __user *ip); +int __init nvme_core_init(void); +void nvme_core_exit(void); + #endif /* _NVME_H */ diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index d1590452f16d..ee3841a67e55 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -28,7 +28,6 @@ #include #include #include -#include #include #include #include @@ -65,9 +64,6 @@ unsigned char shutdown_timeout = 5; module_param(shutdown_timeout, byte, 0644); MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown"); -static int nvme_major; -module_param(nvme_major, int, 0); - static int nvme_char_major; module_param(nvme_char_major, int, 0); @@ -127,9 +123,7 @@ struct nvme_dev { u32 db_stride; struct msix_entry *entry; void __iomem *bar; - struct list_head namespaces; struct mutex namespaces_mutex; - struct device *device; struct work_struct reset_work; struct work_struct probe_work; struct work_struct scan_work; @@ -1579,92 +1573,6 @@ static int nvme_kthread(void *data) return 0; } -static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid) -{ - struct nvme_ns *ns; - struct gendisk *disk; - int node = dev_to_node(dev->dev); - - ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node); - if (!ns) - return; - - ns->queue = blk_mq_init_queue(&dev->tagset); - if (IS_ERR(ns->queue)) - goto out_free_ns; - queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue); - queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue); - queue_flag_set_unlocked(QUEUE_FLAG_SG_GAPS, ns->queue); - ns->ctrl = &dev->ctrl; - ns->queue->queuedata = ns; - - disk = alloc_disk_node(0, node); - if (!disk) - goto out_free_queue; - - kref_init(&ns->kref); - ns->ns_id = nsid; - ns->disk = disk; - ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */ - mutex_lock(&dev->namespaces_mutex); - list_add_tail(&ns->list, &dev->namespaces); - mutex_unlock(&dev->namespaces_mutex); - - blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift); - if (dev->max_hw_sectors) { - u32 max_segments = - (dev->max_hw_sectors / (dev->ctrl.page_size >> 9)) + 1; - - blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors); - blk_queue_max_segments(ns->queue, min_t(u32, max_segments, USHRT_MAX)); - } - if (dev->ctrl.stripe_size) - blk_queue_chunk_sectors(ns->queue, dev->ctrl.stripe_size >> 9); - if (dev->ctrl.vwc & NVME_CTRL_VWC_PRESENT) - blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA); - - disk->major = nvme_major; - disk->first_minor = 0; - disk->fops = &nvme_fops; - disk->private_data = ns; - disk->queue = ns->queue; - disk->driverfs_dev = dev->device; - disk->flags = GENHD_FL_EXT_DEVT; - sprintf(disk->disk_name, "nvme%dn%d", dev->ctrl.instance, nsid); - - /* - * Initialize capacity to 0 until we establish the namespace format and - * setup integrity extentions if necessary. The revalidate_disk after - * add_disk allows the driver to register with integrity if the format - * requires it. - */ - set_capacity(disk, 0); - if (nvme_revalidate_disk(ns->disk)) - goto out_free_disk; - - kref_get(&dev->ctrl.kref); - add_disk(ns->disk); - if (ns->ms) { - struct block_device *bd = bdget_disk(ns->disk, 0); - if (!bd) - return; - if (blkdev_get(bd, FMODE_READ, NULL)) { - bdput(bd); - return; - } - blkdev_reread_part(bd); - blkdev_put(bd, FMODE_READ); - } - return; - out_free_disk: - kfree(disk); - list_del(&ns->list); - out_free_queue: - blk_cleanup_queue(ns->queue); - out_free_ns: - kfree(ns); -} - /* * Create I/O queues. Failing to create an I/O queue is not an issue, * we can continue with less than the desired amount of queues, and @@ -1847,154 +1755,6 @@ static int nvme_setup_io_queues(struct nvme_dev *dev) return result; } -static int ns_cmp(void *priv, struct list_head *a, struct list_head *b) -{ - struct nvme_ns *nsa = container_of(a, struct nvme_ns, list); - struct nvme_ns *nsb = container_of(b, struct nvme_ns, list); - - return nsa->ns_id - nsb->ns_id; -} - -static struct nvme_ns *nvme_find_get_ns(struct nvme_dev *dev, unsigned nsid) -{ - struct nvme_ns *ns, *ret = NULL; - - mutex_lock(&dev->namespaces_mutex); - list_for_each_entry(ns, &dev->namespaces, list) { - if (ns->ns_id == nsid) { - kref_get(&ns->kref); - ret = ns; - break; - } - if (ns->ns_id > nsid) - break; - } - mutex_unlock(&dev->namespaces_mutex); - return ret; -} - -static inline bool nvme_io_incapable(struct nvme_dev *dev) -{ - return (!dev->bar || - readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_CFS || - dev->online_queues < 2); -} - -static void nvme_ns_remove(struct nvme_ns *ns) -{ - bool kill = nvme_io_incapable(to_nvme_dev(ns->ctrl)) && - !blk_queue_dying(ns->queue); - - if (kill) { - blk_set_queue_dying(ns->queue); - /* - * The controller was shutdown first if we got here through - * device removal. The shutdown may requeue outstanding - * requests. These need to be aborted immediately so - * del_gendisk doesn't block indefinitely for their completion. - */ - blk_mq_abort_requeue_list(ns->queue); - } - if (ns->disk->flags & GENHD_FL_UP) { - if (blk_get_integrity(ns->disk)) - blk_integrity_unregister(ns->disk); - - del_gendisk(ns->disk); - } - if (kill || !blk_queue_dying(ns->queue)) { - blk_mq_abort_requeue_list(ns->queue); - blk_cleanup_queue(ns->queue); - } - //mutex_lock(&ns->dev->namespaces_mutex); - list_del_init(&ns->list); - //mutex_unlock(&ns->dev->namespaces_mutex); - nvme_put_ns(ns); -} - -static void nvme_validate_ns(struct nvme_dev *dev, unsigned nsid) -{ - struct nvme_ns *ns; - - ns = nvme_find_get_ns(dev, nsid); - if (ns) { - if (revalidate_disk(ns->disk)) - nvme_ns_remove(ns); - nvme_put_ns(ns); - } else - nvme_alloc_ns(dev, nsid); -} - -static int nvme_scan_ns_list(struct nvme_dev *dev, unsigned nn) -{ - struct nvme_ns *ns; - __le32 *ns_list; - unsigned i, j, nsid, prev = 0, num_lists = DIV_ROUND_UP(nn, 1024); - int ret = 0; - - ns_list = kzalloc(0x1000, GFP_KERNEL); - if (!ns_list) - return -ENOMEM; - - for (i = 0; i < num_lists; i++) { - //ret = nvme_identify_ns_list(dev, prev, ns_list); - if (ret) - goto out; - - for (j = 0; j < min(nn, 1024U); j++) { - nsid = le32_to_cpu(ns_list[j]); - if (!nsid) - goto out; - - nvme_validate_ns(dev, nsid); - - while (++prev < nsid) { - ns = nvme_find_get_ns(dev, prev); - if (ns) { - nvme_ns_remove(ns); - nvme_put_ns(ns); - } - } - } - nn -= j; - } - out: - kfree(ns_list); - return ret; -} - -static void __nvme_scan_namespaces(struct nvme_dev *dev, unsigned nn) -{ - struct nvme_ns *ns, *next; - unsigned i; - - for (i = 1; i <= nn; i++) - nvme_validate_ns(dev, i); - - list_for_each_entry_safe(ns, next, &dev->namespaces, list) { - if (ns->ns_id > nn) - nvme_ns_remove(ns); - } -} - -void nvme_scan_namespaces(struct nvme_dev *dev) -{ - struct nvme_id_ctrl *id; - unsigned nn; - - if (nvme_identify_ctrl(&dev->ctrl, &id)) - return; - - nn = le32_to_cpu(id->nn); - if (!nvme_scan_ns_list(dev, nn)) - goto done; - __nvme_scan_namespaces(dev, le32_to_cpup(&id->nn)); -done: - mutex_lock(&dev->namespaces_mutex); - list_sort(NULL, &dev->namespaces, ns_cmp); - mutex_unlock(&dev->namespaces_mutex); - kfree(id); -} - static void nvme_set_irq_hints(struct nvme_dev *dev) { struct nvme_queue *nvmeq; @@ -2030,7 +1790,7 @@ static void nvme_dev_scan(struct work_struct *work) */ static int nvme_dev_add(struct nvme_dev *dev) { - if (!dev->tagset.tags) { + if (!dev->ctrl.tagset) { dev->tagset.ops = &nvme_mq_ops; dev->tagset.nr_hw_queues = dev->online_queues - 1; dev->tagset.timeout = NVME_IO_TIMEOUT; @@ -2043,6 +1803,7 @@ static int nvme_dev_add(struct nvme_dev *dev) if (blk_mq_alloc_tag_set(&dev->tagset)) return 0; + dev->ctrl.tagset = &dev->tagset; } schedule_work(&dev->scan_work); return 0; @@ -2310,7 +2071,7 @@ static void nvme_freeze_queues(struct nvme_dev *dev) struct nvme_ns *ns; mutex_lock(&dev->namespaces_mutex); - list_for_each_entry(ns, &dev->namespaces, list) { + list_for_each_entry(ns, &dev->ctrl.namespaces, list) { blk_mq_freeze_queue_start(ns->queue); spin_lock(ns->queue->queue_lock); @@ -2328,7 +2089,7 @@ static void nvme_unfreeze_queues(struct nvme_dev *dev) struct nvme_ns *ns; mutex_lock(&dev->namespaces_mutex); - list_for_each_entry(ns, &dev->namespaces, list) { + list_for_each_entry(ns, &dev->ctrl.namespaces, list) { queue_flag_clear_unlocked(QUEUE_FLAG_STOPPED, ns->queue); blk_mq_unfreeze_queue(ns->queue); blk_mq_start_stopped_hw_queues(ns->queue, true); @@ -2364,29 +2125,6 @@ static void nvme_dev_shutdown(struct nvme_dev *dev) nvme_clear_queue(dev->queues[i]); } -/* - * This function iterates the namespace list unlocked to allow recovery from - * controller failure. It is up to the caller to ensure the namespace list is - * not modified by scan work while this function is executing. - */ - -static void nvme_dev_remove(struct nvme_dev *dev) -{ - struct nvme_ns *ns, *next; - - if (nvme_io_incapable(dev)) { - /* - * If the device is not capable of IO (surprise hot-removal, - * for example), we need to quiesce prior to deleting the - * namespaces. This will end outstanding requests and prevent - * attempts to sync dirty data. - */ - nvme_dev_shutdown(dev); - } - list_for_each_entry_safe(ns, next, &dev->namespaces, list) - nvme_ns_remove(ns); -} - static int nvme_setup_prp_pools(struct nvme_dev *dev) { dev->prp_page_pool = dma_pool_create("prp list page", dev->dev, @@ -2444,7 +2182,7 @@ static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl) struct nvme_dev *dev = to_nvme_dev(ctrl); put_device(dev->dev); - put_device(dev->device); + put_device(ctrl->device); nvme_release_instance(dev); if (dev->tagset.tags) blk_mq_free_tag_set(&dev->tagset); @@ -2496,9 +2234,9 @@ static long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg) case NVME_IOCTL_ADMIN_CMD: return nvme_user_cmd(&dev->ctrl, NULL, (void __user *)arg); case NVME_IOCTL_IO_CMD: - if (list_empty(&dev->namespaces)) + if (list_empty(&dev->ctrl.namespaces)) return -ENOTTY; - ns = list_first_entry(&dev->namespaces, struct nvme_ns, list); + ns = list_first_entry(&dev->ctrl.namespaces, struct nvme_ns, list); return nvme_user_cmd(&dev->ctrl, ns, (void __user *)arg); case NVME_IOCTL_RESET: dev_warn(dev->dev, "resetting controller\n"); @@ -2572,7 +2310,7 @@ static void nvme_probe_work(struct work_struct *work) */ if (dev->online_queues < 2) { dev_warn(dev->dev, "IO queues not created\n"); - nvme_dev_remove(dev); + nvme_remove_namespaces(&dev->ctrl); } else { nvme_unfreeze_queues(dev); nvme_dev_add(dev); @@ -2701,10 +2439,18 @@ static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val) return 0; } +static bool nvme_pci_io_incapable(struct nvme_ctrl *ctrl) +{ + struct nvme_dev *dev = to_nvme_dev(ctrl); + + return !dev->bar || dev->online_queues < 2; +} + static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = { .reg_read32 = nvme_pci_reg_read32, .reg_write32 = nvme_pci_reg_write32, .reg_read64 = nvme_pci_reg_read64, + .io_incapable = nvme_pci_io_incapable, .free_ctrl = nvme_pci_free_ctrl, }; @@ -2729,7 +2475,7 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (!dev->queues) goto free; - INIT_LIST_HEAD(&dev->namespaces); + INIT_LIST_HEAD(&dev->ctrl.namespaces); mutex_init(&dev->namespaces_mutex); INIT_WORK(&dev->reset_work, nvme_reset_work); dev->dev = get_device(&pdev->dev); @@ -2748,17 +2494,17 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto release; kref_init(&dev->ctrl.kref); - dev->device = device_create(nvme_class, &pdev->dev, + dev->ctrl.device = device_create(nvme_class, &pdev->dev, MKDEV(nvme_char_major, dev->ctrl.instance), dev, "nvme%d", dev->ctrl.instance); - if (IS_ERR(dev->device)) { - result = PTR_ERR(dev->device); + if (IS_ERR(dev->ctrl.device)) { + result = PTR_ERR(dev->ctrl.device); goto release_pools; } - get_device(dev->device); - dev_set_drvdata(dev->device, dev); + get_device(dev->ctrl.device); + dev_set_drvdata(dev->ctrl.device, dev); - result = device_create_file(dev->device, &dev_attr_reset_controller); + result = device_create_file(dev->ctrl.device, &dev_attr_reset_controller); if (result) goto put_dev; @@ -2770,7 +2516,7 @@ static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id) put_dev: device_destroy(nvme_class, MKDEV(nvme_char_major, dev->ctrl.instance)); - put_device(dev->device); + put_device(dev->ctrl.device); release_pools: nvme_release_prp_pools(dev); release: @@ -2812,8 +2558,8 @@ static void nvme_remove(struct pci_dev *pdev) flush_work(&dev->probe_work); flush_work(&dev->reset_work); flush_work(&dev->scan_work); - device_remove_file(dev->device, &dev_attr_reset_controller); - nvme_dev_remove(dev); + device_remove_file(dev->ctrl.device, &dev_attr_reset_controller); + nvme_remove_namespaces(&dev->ctrl); nvme_dev_shutdown(dev); nvme_dev_remove_admin(dev); device_destroy(nvme_class, MKDEV(nvme_char_major, dev->ctrl.instance)); @@ -2895,11 +2641,9 @@ static int __init nvme_init(void) if (!nvme_workq) return -ENOMEM; - result = register_blkdev(nvme_major, "nvme"); + result = nvme_core_init(); if (result < 0) goto kill_workq; - else if (result > 0) - nvme_major = result; result = __register_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme", &nvme_dev_fops); @@ -2924,7 +2668,7 @@ static int __init nvme_init(void) unregister_chrdev: __unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme"); unregister_blkdev: - unregister_blkdev(nvme_major, "nvme"); + nvme_core_exit(); kill_workq: destroy_workqueue(nvme_workq); return result; @@ -2933,7 +2677,7 @@ static int __init nvme_init(void) static void __exit nvme_exit(void) { pci_unregister_driver(&nvme_driver); - unregister_blkdev(nvme_major, "nvme"); + nvme_core_exit(); destroy_workqueue(nvme_workq); class_destroy(nvme_class); __unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme"); -- 2.50.1