nvme: add support for non shared namespaces

Currently, nvme-cli assume that all the namespaces in an NVM subsystem
are shared between all the controllers. There is an option to create
both shared and non-shared namespaces among the controllers in the same
NVM subsystem. For example, in SR-IOV environment, one can attach 1
namespace to each controller that will be private and also 1 namespace
that will be shared among the secondary controllers only. In this case,
the output of "nvme list -v" will be wrong:

NVM Express Subsystems

Subsystem        Subsystem-NQN                                                                                    Controllers
---------------- ------------------------------------------------------------------------------------------------ ----------------
nvme-subsys5     nqn.1994-11.com.samsung:nvme:PM1733:2.5-inch:S4YNNE0N700448                                      nvme18, nvme19, nvme20, nvme21, nvme5

NVM Express Controllers

Device   SN                   MN                                       FR       TxPort Address        Subsystem    Namespaces
-------- -------------------- ---------------------------------------- -------- ------ -------------- ------------ ----------------
nvme18   S4YNNE0N700448       SAMSUNG MZWLJ1T9HBJR-00007               EPK98B5Q pcie   0000:0a:00.1   nvme-subsys5 nvme5n1, nvme5n2, nvme5n3, nvme5n4, nvme5n5, nvme5n6
nvme19   S4YNNE0N700448       SAMSUNG MZWLJ1T9HBJR-00007               EPK98B5Q pcie   0000:0a:00.2   nvme-subsys5 nvme5n1, nvme5n2, nvme5n3, nvme5n4, nvme5n5, nvme5n6
nvme20   S4YNNE0N700448       SAMSUNG MZWLJ1T9HBJR-00007               EPK98B5Q pcie   0000:0a:00.3   nvme-subsys5 nvme5n1, nvme5n2, nvme5n3, nvme5n4, nvme5n5, nvme5n6
nvme21   S4YNNE0N700448       SAMSUNG MZWLJ1T9HBJR-00007               EPK98B5Q pcie   0000:0a:00.4   nvme-subsys5 nvme5n1, nvme5n2, nvme5n3, nvme5n4, nvme5n5, nvme5n6
nvme5    S4YNNE0N700448       SAMSUNG MZWLJ1T9HBJR-00007               EPK98B5Q pcie   0000:0a:00.0   nvme-subsys5 nvme5n1, nvme5n2, nvme5n3, nvme5n4, nvme5n5, nvme5n6

NVM Express Namespaces

Device       NSID     Usage                      Format           Controllers
------------ -------- -------------------------- ---------------- ----------------
nvme5n1      1        343.60  GB / 343.60  GB    512   B +  0 B   nvme18, nvme19, nvme20, nvme21, nvme5
nvme5n2      2        274.88  GB / 274.88  GB    512   B +  0 B   nvme18, nvme19, nvme20, nvme21, nvme5
nvme5n3      3        283.47  GB / 283.47  GB    512   B +  0 B   nvme18, nvme19, nvme20, nvme21, nvme5
nvme5n4      4        292.06  GB / 292.06  GB    512   B +  0 B   nvme18, nvme19, nvme20, nvme21, nvme5
nvme5n5      5        317.83  GB / 317.83  GB    512   B +  0 B   nvme18, nvme19, nvme20, nvme21, nvme5
nvme5n6      6        137.44  GB / 137.44  GB    512   B +  0 B   nvme18, nvme19, nvme20, nvme21, nvme5

After the fix we'll get the following right topology:

NVM Express Subsystems

Subsystem        Subsystem-NQN                                                                                    Controllers
---------------- ------------------------------------------------------------------------------------------------ ----------------
nvme-subsys5     nqn.1994-11.com.samsung:nvme:PM1733:2.5-inch:S4YNNE0N700448                                      nvme18, nvme19, nvme20, nvme21, nvme5

NVM Express Controllers

Device   SN                   MN                                       FR       TxPort Address        Subsystem    Namespaces
-------- -------------------- ---------------------------------------- -------- ------ -------------- ------------ ----------------
nvme18   S4YNNE0N700448       SAMSUNG MZWLJ1T9HBJR-00007               EPK98B5Q pcie   0000:0a:00.1   nvme-subsys5 nvme5n2, nvme5n6
nvme19   S4YNNE0N700448       SAMSUNG MZWLJ1T9HBJR-00007               EPK98B5Q pcie   0000:0a:00.2   nvme-subsys5 nvme5n3, nvme5n6
nvme20   S4YNNE0N700448       SAMSUNG MZWLJ1T9HBJR-00007               EPK98B5Q pcie   0000:0a:00.3   nvme-subsys5 nvme5n4, nvme5n6
nvme21   S4YNNE0N700448       SAMSUNG MZWLJ1T9HBJR-00007               EPK98B5Q pcie   0000:0a:00.4   nvme-subsys5 nvme5n5, nvme5n6
nvme5    S4YNNE0N700448       SAMSUNG MZWLJ1T9HBJR-00007               EPK98B5Q pcie   0000:0a:00.0   nvme-subsys5 nvme5n1

NVM Express Namespaces

Device       NSID     Usage                      Format           Controllers
------------ -------- -------------------------- ---------------- ----------------
nvme5n1      1        343.60  GB / 343.60  GB    512   B +  0 B   nvme5
nvme5n2      2        274.88  GB / 274.88  GB    512   B +  0 B   nvme18
nvme5n3      3        283.47  GB / 283.47  GB    512   B +  0 B   nvme19
nvme5n4      4        292.06  GB / 292.06  GB    512   B +  0 B   nvme20
nvme5n5      5        317.83  GB / 317.83  GB    512   B +  0 B   nvme21
nvme5n6      6        137.44  GB / 137.44  GB    512   B +  0 B   nvme18, nvme19, nvme20, nvme21

Cc: Max Gurtovoy <maxg@mellanox.com>
Signed-off-by: Max Gurtovoy <mgurtovoy@nvidia.com>
Signed-off-by: Keith Busch <kbusch@kernel.org>

diff --git a/nvme-filters.c b/nvme-filters.c
index a4133f8f0f6bef69e84618fc978d72baaf0daee9..17c375f5a93c01980957a9c660aaef1eea6704c4 100644 (file)
--- a/nvme-filters.c
+++ b/nvme-filters.c
@@ -7,6 +7,22 @@
 /* global, used for controller specific namespace filter */
 int current_index;
 
+int scan_ctrl_namespace_filter(const struct dirent *d)
+{
+       int c, i, n;
+
+       if (d->d_name[0] == '.')
+               return 0;
+
+       if (strstr(d->d_name, "nvme")) {
+               if (sscanf(d->d_name, "nvme%dc%dn%d", &i, &c, &n) == 3)
+                       return 1;
+               if (sscanf(d->d_name, "nvme%dn%d", &i, &n) == 2)
+                       return 1;
+       }
+       return 0;
+}
+
 int scan_namespace_filter(const struct dirent *d)
 {
        int i, n;

diff --git a/nvme-print.c b/nvme-print.c
index f037a28c1c6b91cb0497a58f6be3ed1be095cffd..5e5143a56c15f31e7e92094cf5bf1bf0899c7a01 100644 (file)
--- a/nvme-print.c
+++ b/nvme-print.c
@@ -3,6 +3,7 @@
 #include <string.h>
 #include <stdlib.h>
 #include <time.h>
+#include <sys/stat.h>
 
 #include "nvme-print.h"
 #include "util/json.h"
@@ -4670,7 +4671,14 @@ static void nvme_show_list_item(struct nvme_namespace *n)
        const char *l_suffix = suffix_binary_get(&lba);
 
        char usage[128];
-       char format[128];
+       char format[128], path[256];
+       struct stat st;
+       int ret;
+
+       sprintf(path, "/dev/%s", n->name);
+       ret = stat(path, &st);
+       if (ret < 0)
+               return;
 
        sprintf(usage,"%6.2f %2sB / %6.2f %2sB", nuse, u_suffix,
                nsze, s_suffix);
@@ -4734,17 +4742,29 @@ static void nvme_show_details_ns(struct nvme_namespace *n, bool ctrl)
                printf("%s", n->ctrl->name);
        else {
                struct nvme_subsystem *s = n->ctrl->subsys;
-               int i;
+               int i, j;
+               bool comma = false;
+
+               for (i = 0; i < s->nr_ctrls; i++) {
+                       struct nvme_ctrl *c = &s->ctrls[i];
 
-               for (i = 0; i < s->nr_ctrls; i++)
-                       printf("%s%s", i ? ", " : "", s->ctrls[i].name);
+                       for (j = 0; j < c->nr_namespaces; j++) {
+                               struct nvme_namespace *ns = &c->namespaces[j];
+
+                               if (ns->nsid == n->nsid) {
+                                       printf("%s%s", comma ? ", " : "",
+                                              c->name);
+                                       comma = true;
+                               }
+                       }
+               }
        }
        printf("\n");
 }
 
 static void nvme_show_detailed_list(struct nvme_topology *t)
 {
-       int i, j, k;
+       int i, j, k, l;
 
        printf("NVM Express Subsystems\n\n");
        printf("%-16s %-96s %-.16s\n", "Subsystem", "Subsystem-NQN", "Controllers");
@@ -4779,13 +4799,14 @@ static void nvme_show_detailed_list(struct nvme_topology *t)
 
                        for (k = 0; k < c->nr_namespaces; k++) {
                                struct nvme_namespace *n = &c->namespaces[k];
-                               printf("%s%s", comma ? ", " : "", n->name);
-                               comma = true;
-                       }
-                       for (k = 0; k < s->nr_namespaces; k++) {
-                               struct nvme_namespace *n = &s->namespaces[k];
-                               printf("%s%s", comma ? ", " : "", n->name);
-                               comma = true;
+
+                               for (l = 0; l < s->nr_namespaces; l++) {
+                                       struct nvme_namespace *ns = &s->namespaces[l];
+                                       if (n->nsid == ns->nsid) {
+                                               printf("%s%s", comma ? ", " : "", ns->name);
+                                               comma = true;
+                                       }
+                               }
                        }
                        printf("\n");
                }
@@ -4798,18 +4819,21 @@ static void nvme_show_detailed_list(struct nvme_topology *t)
        for (i = 0; i < t->nr_subsystems; i++) {
                struct nvme_subsystem *s = &t->subsystems[i];
 
-               for (j = 0; j < s->nr_ctrls; j++) {
-                       struct nvme_ctrl *c = &s->ctrls[j];
+               if (s->nr_namespaces) {
+                       for (j = 0; j < s->nr_namespaces; j++) {
+                               struct nvme_namespace *n = &s->namespaces[j];
+                               nvme_show_details_ns(n, false);
+                       }
+               } else {
+                       for (j = 0; j < s->nr_ctrls; j++) {
+                               struct nvme_ctrl *c = &s->ctrls[j];
 
-                       for (k = 0; k < c->nr_namespaces; k++) {
-                               struct nvme_namespace *n = &c->namespaces[k];
-                               nvme_show_details_ns(n, true);
+                               for (k = 0; k < c->nr_namespaces; k++) {
+                                       struct nvme_namespace *n = &c->namespaces[k];
+                                       nvme_show_details_ns(n, true);
+                               }
                        }
                }
-               for (j = 0; j < s->nr_namespaces; j++) {
-                       struct nvme_namespace *n = &s->namespaces[j];
-                       nvme_show_details_ns(n, false);
-               }
        }
 }
 

diff --git a/nvme-topology.c b/nvme-topology.c
index d24ef6b91fade5066fd068031469b7694c214865..94a4c563d4751cac6bb74fc7e7d56508f5ed0d73 100644 (file)
--- a/nvme-topology.c
+++ b/nvme-topology.c
@@ -152,9 +152,11 @@ static int scan_namespace(struct nvme_namespace *n)
        if (fd < 0)
                goto free;
 
-       n->nsid = nvme_get_nsid(fd);
-       if (n->nsid < 0)
-               goto close_fd;
+       if (!n->nsid) {
+               n->nsid = nvme_get_nsid(fd);
+               if (n->nsid < 0)
+                       goto close_fd;
+       }
 
        ret = nvme_identify_ns(fd, n->nsid, 0, &n->ns);
        if (ret < 0)
@@ -248,7 +250,7 @@ static int scan_ctrl(struct nvme_ctrl *c, char *p, __u32 ns_instance)
        if (ns_instance)
                c->ana_state = get_nvme_ctrl_path_ana_state(path, ns_instance);
 
-       ret = scandir(path, &ns, scan_namespace_filter, alphasort);
+       ret = scandir(path, &ns, scan_ctrl_namespace_filter, alphasort);
        if (ret == -1) {
                fprintf(stderr, "Failed to open %s: %s\n", path, strerror(errno));
                return errno;
@@ -256,11 +258,36 @@ static int scan_ctrl(struct nvme_ctrl *c, char *p, __u32 ns_instance)
 
        c->nr_namespaces = ret;
        c->namespaces = calloc(c->nr_namespaces, sizeof(*n));
-       for (i = 0; i < c->nr_namespaces; i++) {
-               n = &c->namespaces[i];
-               n->name = strdup(ns[i]->d_name);
-               n->ctrl = c;
-               scan_namespace(n);
+       if (c->namespaces) {
+               for (i = 0; i < c->nr_namespaces; i++) {
+                       char *ns_path, nsid[16];
+                       int ns_fd;
+
+                       n = &c->namespaces[i];
+                       n->name = strdup(ns[i]->d_name);
+                       n->ctrl = c;
+                       ret = asprintf(&ns_path, "%s/%s/nsid", path, n->name);
+                       if (ret < 0)
+                               continue;
+                       ns_fd = open(ns_path, O_RDONLY);
+                       if (ns_fd < 0) {
+                               free(ns_path);
+                               continue;
+                       }
+                       ret = read(ns_fd, nsid, 16);
+                       if (ret < 0) {
+                               close(ns_fd);
+                               free(ns_path);
+                               continue;
+                       }
+                       n->nsid = (unsigned)strtol(nsid, NULL, 10);
+                       scan_namespace(n);
+                       close(ns_fd);
+                       free(ns_path);
+               }
+       } else {
+               i = c->nr_namespaces;
+               c->nr_namespaces = 0;
        }
 
        while (i--)
@@ -327,11 +354,16 @@ static int scan_subsystem(struct nvme_subsystem *s, __u32 ns_instance)
 
        s->nr_namespaces = ret;
        s->namespaces = calloc(s->nr_namespaces, sizeof(*n));
-       for (i = 0; i < s->nr_namespaces; i++) {
-               n = &s->namespaces[i];
-               n->name = strdup(ns[i]->d_name);
-               n->ctrl = &s->ctrls[0];
-               scan_namespace(n);
+       if (s->namespaces) {
+               for (i = 0; i < s->nr_namespaces; i++) {
+                       n = &s->namespaces[i];
+                       n->name = strdup(ns[i]->d_name);
+                       n->ctrl = &s->ctrls[0];
+                       scan_namespace(n);
+               }
+       } else {
+               i = s->nr_namespaces;
+               s->nr_namespaces = 0;
        }
 
        while (i--)
@@ -420,6 +452,12 @@ static int legacy_list(struct nvme_topology *t)
                c->nr_namespaces = scandir(dev, &namespaces, scan_dev_filter,
                                           alphasort);
                c->namespaces = calloc(c->nr_namespaces, sizeof(*n));
+               if (!c->namespaces) {
+                       while (c->nr_namespaces--)
+                               free(namespaces[c->nr_namespaces]);
+                       free(namespaces);
+                       continue;
+               }
 
                ret = asprintf(&path, "%s%s", dev, c->name);
                if (ret < 0)

diff --git a/nvme.h b/nvme.h
index ffc3fbdbb858371c7bba2d72ed078dbdc032486b..9a54bdb1b941dcab07329eeda796ec33dde7bf9c 100644 (file)
--- a/nvme.h
+++ b/nvme.h
@@ -97,6 +97,7 @@ char *nvme_char_from_block(char *block);
 void *mmap_registers(const char *dev);
 
 extern int current_index;
+int scan_ctrl_namespace_filter(const struct dirent *d);
 int scan_namespace_filter(const struct dirent *d);
 int scan_ctrl_paths_filter(const struct dirent *d);
 int scan_ctrls_filter(const struct dirent *d);