From 7ac86adc63023b47865da9d2db4ec1ce828c07c4 Mon Sep 17 00:00:00 2001 From: Max Gurtovoy Date: Wed, 26 Aug 2020 13:49:03 +0300 Subject: [PATCH] nvme: add support for non shared namespaces Currently, nvme-cli assume that all the namespaces in an NVM subsystem are shared between all the controllers. There is an option to create both shared and non-shared namespaces among the controllers in the same NVM subsystem. For example, in SR-IOV environment, one can attach 1 namespace to each controller that will be private and also 1 namespace that will be shared among the secondary controllers only. In this case, the output of "nvme list -v" will be wrong: NVM Express Subsystems Subsystem Subsystem-NQN Controllers ---------------- ------------------------------------------------------------------------------------------------ ---------------- nvme-subsys5 nqn.1994-11.com.samsung:nvme:PM1733:2.5-inch:S4YNNE0N700448 nvme18, nvme19, nvme20, nvme21, nvme5 NVM Express Controllers Device SN MN FR TxPort Address Subsystem Namespaces -------- -------------------- ---------------------------------------- -------- ------ -------------- ------------ ---------------- nvme18 S4YNNE0N700448 SAMSUNG MZWLJ1T9HBJR-00007 EPK98B5Q pcie 0000:0a:00.1 nvme-subsys5 nvme5n1, nvme5n2, nvme5n3, nvme5n4, nvme5n5, nvme5n6 nvme19 S4YNNE0N700448 SAMSUNG MZWLJ1T9HBJR-00007 EPK98B5Q pcie 0000:0a:00.2 nvme-subsys5 nvme5n1, nvme5n2, nvme5n3, nvme5n4, nvme5n5, nvme5n6 nvme20 S4YNNE0N700448 SAMSUNG MZWLJ1T9HBJR-00007 EPK98B5Q pcie 0000:0a:00.3 nvme-subsys5 nvme5n1, nvme5n2, nvme5n3, nvme5n4, nvme5n5, nvme5n6 nvme21 S4YNNE0N700448 SAMSUNG MZWLJ1T9HBJR-00007 EPK98B5Q pcie 0000:0a:00.4 nvme-subsys5 nvme5n1, nvme5n2, nvme5n3, nvme5n4, nvme5n5, nvme5n6 nvme5 S4YNNE0N700448 SAMSUNG MZWLJ1T9HBJR-00007 EPK98B5Q pcie 0000:0a:00.0 nvme-subsys5 nvme5n1, nvme5n2, nvme5n3, nvme5n4, nvme5n5, nvme5n6 NVM Express Namespaces Device NSID Usage Format Controllers ------------ -------- -------------------------- ---------------- ---------------- nvme5n1 1 343.60 GB / 343.60 GB 512 B + 0 B nvme18, nvme19, nvme20, nvme21, nvme5 nvme5n2 2 274.88 GB / 274.88 GB 512 B + 0 B nvme18, nvme19, nvme20, nvme21, nvme5 nvme5n3 3 283.47 GB / 283.47 GB 512 B + 0 B nvme18, nvme19, nvme20, nvme21, nvme5 nvme5n4 4 292.06 GB / 292.06 GB 512 B + 0 B nvme18, nvme19, nvme20, nvme21, nvme5 nvme5n5 5 317.83 GB / 317.83 GB 512 B + 0 B nvme18, nvme19, nvme20, nvme21, nvme5 nvme5n6 6 137.44 GB / 137.44 GB 512 B + 0 B nvme18, nvme19, nvme20, nvme21, nvme5 After the fix we'll get the following right topology: NVM Express Subsystems Subsystem Subsystem-NQN Controllers ---------------- ------------------------------------------------------------------------------------------------ ---------------- nvme-subsys5 nqn.1994-11.com.samsung:nvme:PM1733:2.5-inch:S4YNNE0N700448 nvme18, nvme19, nvme20, nvme21, nvme5 NVM Express Controllers Device SN MN FR TxPort Address Subsystem Namespaces -------- -------------------- ---------------------------------------- -------- ------ -------------- ------------ ---------------- nvme18 S4YNNE0N700448 SAMSUNG MZWLJ1T9HBJR-00007 EPK98B5Q pcie 0000:0a:00.1 nvme-subsys5 nvme5n2, nvme5n6 nvme19 S4YNNE0N700448 SAMSUNG MZWLJ1T9HBJR-00007 EPK98B5Q pcie 0000:0a:00.2 nvme-subsys5 nvme5n3, nvme5n6 nvme20 S4YNNE0N700448 SAMSUNG MZWLJ1T9HBJR-00007 EPK98B5Q pcie 0000:0a:00.3 nvme-subsys5 nvme5n4, nvme5n6 nvme21 S4YNNE0N700448 SAMSUNG MZWLJ1T9HBJR-00007 EPK98B5Q pcie 0000:0a:00.4 nvme-subsys5 nvme5n5, nvme5n6 nvme5 S4YNNE0N700448 SAMSUNG MZWLJ1T9HBJR-00007 EPK98B5Q pcie 0000:0a:00.0 nvme-subsys5 nvme5n1 NVM Express Namespaces Device NSID Usage Format Controllers ------------ -------- -------------------------- ---------------- ---------------- nvme5n1 1 343.60 GB / 343.60 GB 512 B + 0 B nvme5 nvme5n2 2 274.88 GB / 274.88 GB 512 B + 0 B nvme18 nvme5n3 3 283.47 GB / 283.47 GB 512 B + 0 B nvme19 nvme5n4 4 292.06 GB / 292.06 GB 512 B + 0 B nvme20 nvme5n5 5 317.83 GB / 317.83 GB 512 B + 0 B nvme21 nvme5n6 6 137.44 GB / 137.44 GB 512 B + 0 B nvme18, nvme19, nvme20, nvme21 Cc: Max Gurtovoy Signed-off-by: Max Gurtovoy Signed-off-by: Keith Busch --- nvme-filters.c | 16 ++++++++++++ nvme-print.c | 66 +++++++++++++++++++++++++++++++++---------------- nvme-topology.c | 66 ++++++++++++++++++++++++++++++++++++++----------- nvme.h | 1 + 4 files changed, 114 insertions(+), 35 deletions(-) diff --git a/nvme-filters.c b/nvme-filters.c index a4133f8f..17c375f5 100644 --- a/nvme-filters.c +++ b/nvme-filters.c @@ -7,6 +7,22 @@ /* global, used for controller specific namespace filter */ int current_index; +int scan_ctrl_namespace_filter(const struct dirent *d) +{ + int c, i, n; + + if (d->d_name[0] == '.') + return 0; + + if (strstr(d->d_name, "nvme")) { + if (sscanf(d->d_name, "nvme%dc%dn%d", &i, &c, &n) == 3) + return 1; + if (sscanf(d->d_name, "nvme%dn%d", &i, &n) == 2) + return 1; + } + return 0; +} + int scan_namespace_filter(const struct dirent *d) { int i, n; diff --git a/nvme-print.c b/nvme-print.c index f037a28c..5e5143a5 100644 --- a/nvme-print.c +++ b/nvme-print.c @@ -3,6 +3,7 @@ #include #include #include +#include #include "nvme-print.h" #include "util/json.h" @@ -4670,7 +4671,14 @@ static void nvme_show_list_item(struct nvme_namespace *n) const char *l_suffix = suffix_binary_get(&lba); char usage[128]; - char format[128]; + char format[128], path[256]; + struct stat st; + int ret; + + sprintf(path, "/dev/%s", n->name); + ret = stat(path, &st); + if (ret < 0) + return; sprintf(usage,"%6.2f %2sB / %6.2f %2sB", nuse, u_suffix, nsze, s_suffix); @@ -4734,17 +4742,29 @@ static void nvme_show_details_ns(struct nvme_namespace *n, bool ctrl) printf("%s", n->ctrl->name); else { struct nvme_subsystem *s = n->ctrl->subsys; - int i; + int i, j; + bool comma = false; + + for (i = 0; i < s->nr_ctrls; i++) { + struct nvme_ctrl *c = &s->ctrls[i]; - for (i = 0; i < s->nr_ctrls; i++) - printf("%s%s", i ? ", " : "", s->ctrls[i].name); + for (j = 0; j < c->nr_namespaces; j++) { + struct nvme_namespace *ns = &c->namespaces[j]; + + if (ns->nsid == n->nsid) { + printf("%s%s", comma ? ", " : "", + c->name); + comma = true; + } + } + } } printf("\n"); } static void nvme_show_detailed_list(struct nvme_topology *t) { - int i, j, k; + int i, j, k, l; printf("NVM Express Subsystems\n\n"); printf("%-16s %-96s %-.16s\n", "Subsystem", "Subsystem-NQN", "Controllers"); @@ -4779,13 +4799,14 @@ static void nvme_show_detailed_list(struct nvme_topology *t) for (k = 0; k < c->nr_namespaces; k++) { struct nvme_namespace *n = &c->namespaces[k]; - printf("%s%s", comma ? ", " : "", n->name); - comma = true; - } - for (k = 0; k < s->nr_namespaces; k++) { - struct nvme_namespace *n = &s->namespaces[k]; - printf("%s%s", comma ? ", " : "", n->name); - comma = true; + + for (l = 0; l < s->nr_namespaces; l++) { + struct nvme_namespace *ns = &s->namespaces[l]; + if (n->nsid == ns->nsid) { + printf("%s%s", comma ? ", " : "", ns->name); + comma = true; + } + } } printf("\n"); } @@ -4798,18 +4819,21 @@ static void nvme_show_detailed_list(struct nvme_topology *t) for (i = 0; i < t->nr_subsystems; i++) { struct nvme_subsystem *s = &t->subsystems[i]; - for (j = 0; j < s->nr_ctrls; j++) { - struct nvme_ctrl *c = &s->ctrls[j]; + if (s->nr_namespaces) { + for (j = 0; j < s->nr_namespaces; j++) { + struct nvme_namespace *n = &s->namespaces[j]; + nvme_show_details_ns(n, false); + } + } else { + for (j = 0; j < s->nr_ctrls; j++) { + struct nvme_ctrl *c = &s->ctrls[j]; - for (k = 0; k < c->nr_namespaces; k++) { - struct nvme_namespace *n = &c->namespaces[k]; - nvme_show_details_ns(n, true); + for (k = 0; k < c->nr_namespaces; k++) { + struct nvme_namespace *n = &c->namespaces[k]; + nvme_show_details_ns(n, true); + } } } - for (j = 0; j < s->nr_namespaces; j++) { - struct nvme_namespace *n = &s->namespaces[j]; - nvme_show_details_ns(n, false); - } } } diff --git a/nvme-topology.c b/nvme-topology.c index d24ef6b9..94a4c563 100644 --- a/nvme-topology.c +++ b/nvme-topology.c @@ -152,9 +152,11 @@ static int scan_namespace(struct nvme_namespace *n) if (fd < 0) goto free; - n->nsid = nvme_get_nsid(fd); - if (n->nsid < 0) - goto close_fd; + if (!n->nsid) { + n->nsid = nvme_get_nsid(fd); + if (n->nsid < 0) + goto close_fd; + } ret = nvme_identify_ns(fd, n->nsid, 0, &n->ns); if (ret < 0) @@ -248,7 +250,7 @@ static int scan_ctrl(struct nvme_ctrl *c, char *p, __u32 ns_instance) if (ns_instance) c->ana_state = get_nvme_ctrl_path_ana_state(path, ns_instance); - ret = scandir(path, &ns, scan_namespace_filter, alphasort); + ret = scandir(path, &ns, scan_ctrl_namespace_filter, alphasort); if (ret == -1) { fprintf(stderr, "Failed to open %s: %s\n", path, strerror(errno)); return errno; @@ -256,11 +258,36 @@ static int scan_ctrl(struct nvme_ctrl *c, char *p, __u32 ns_instance) c->nr_namespaces = ret; c->namespaces = calloc(c->nr_namespaces, sizeof(*n)); - for (i = 0; i < c->nr_namespaces; i++) { - n = &c->namespaces[i]; - n->name = strdup(ns[i]->d_name); - n->ctrl = c; - scan_namespace(n); + if (c->namespaces) { + for (i = 0; i < c->nr_namespaces; i++) { + char *ns_path, nsid[16]; + int ns_fd; + + n = &c->namespaces[i]; + n->name = strdup(ns[i]->d_name); + n->ctrl = c; + ret = asprintf(&ns_path, "%s/%s/nsid", path, n->name); + if (ret < 0) + continue; + ns_fd = open(ns_path, O_RDONLY); + if (ns_fd < 0) { + free(ns_path); + continue; + } + ret = read(ns_fd, nsid, 16); + if (ret < 0) { + close(ns_fd); + free(ns_path); + continue; + } + n->nsid = (unsigned)strtol(nsid, NULL, 10); + scan_namespace(n); + close(ns_fd); + free(ns_path); + } + } else { + i = c->nr_namespaces; + c->nr_namespaces = 0; } while (i--) @@ -327,11 +354,16 @@ static int scan_subsystem(struct nvme_subsystem *s, __u32 ns_instance) s->nr_namespaces = ret; s->namespaces = calloc(s->nr_namespaces, sizeof(*n)); - for (i = 0; i < s->nr_namespaces; i++) { - n = &s->namespaces[i]; - n->name = strdup(ns[i]->d_name); - n->ctrl = &s->ctrls[0]; - scan_namespace(n); + if (s->namespaces) { + for (i = 0; i < s->nr_namespaces; i++) { + n = &s->namespaces[i]; + n->name = strdup(ns[i]->d_name); + n->ctrl = &s->ctrls[0]; + scan_namespace(n); + } + } else { + i = s->nr_namespaces; + s->nr_namespaces = 0; } while (i--) @@ -420,6 +452,12 @@ static int legacy_list(struct nvme_topology *t) c->nr_namespaces = scandir(dev, &namespaces, scan_dev_filter, alphasort); c->namespaces = calloc(c->nr_namespaces, sizeof(*n)); + if (!c->namespaces) { + while (c->nr_namespaces--) + free(namespaces[c->nr_namespaces]); + free(namespaces); + continue; + } ret = asprintf(&path, "%s%s", dev, c->name); if (ret < 0) diff --git a/nvme.h b/nvme.h index ffc3fbdb..9a54bdb1 100644 --- a/nvme.h +++ b/nvme.h @@ -97,6 +97,7 @@ char *nvme_char_from_block(char *block); void *mmap_registers(const char *dev); extern int current_index; +int scan_ctrl_namespace_filter(const struct dirent *d); int scan_namespace_filter(const struct dirent *d); int scan_ctrl_paths_filter(const struct dirent *d); int scan_ctrls_filter(const struct dirent *d); -- 2.50.1