nvme-multipath: Add visibility for numa io-policy

This patch helps add nvme native multipath visibility for numa io-policy.
It adds a new attribute file named "numa_nodes" under namespace gendisk
device path node which prints the list of numa nodes preferred by the
given namespace path. The numa nodes value is comma delimited list of
nodes or A-B range of nodes.

For instance, if we have a shared namespace accessible from two different
controllers/paths then accessing head node of the shared namespace would
show the following output:

$ ls -l /sys/block/nvme1n1/multipath/
nvme1c1n1 -> ../../../../../pci052e:78/052e:78:00.0/nvme/nvme1/nvme1c1n1
nvme1c3n1 -> ../../../../../pci058e:78/058e:78:00.0/nvme/nvme3/nvme1c3n1

In the above example, nvme1n1 is head gendisk node created for a shared
namespace and this namespace is accessible from nvme1c1n1 and nvme1c3n1
paths. For numa io-policy we can then refer the "numa_nodes" attribute
file created under each namespace path:

$ cat /sys/block/nvme1n1/multipath/nvme1c1n1/numa_nodes
0-1

$ cat /sys/block/nvme1n1/multipath/nvme1c3n1/numa_nodes
2-3

>From the above output, we infer that I/O workload targeted at nvme1n1
and running on numa nodes 0 and 1 would prefer using path nvme1c1n1.
Similarly, I/O workload running on numa nodes 2 and 3 would prefer
using path nvme1c3n1. Reading "numa_nodes" file when configured
io-policy is anything but numa would show no output.

Reviewed-by: Sagi Grimberg <sagi@grimberg.me>
Reviewed-by: Hannes Reinecke <hare@suse.de>
Signed-off-by: Nilay Shroff <nilay@linux.ibm.com>
Signed-off-by: Keith Busch <kbusch@kernel.org>

diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
index 7c89c8da46d6ac0a6cd7a06bb26c36c3cde20dee..b47e01942ca4a659dd6bc0e59ef05a1f6fba02cf 100644 (file)
--- a/drivers/nvme/host/multipath.c
+++ b/drivers/nvme/host/multipath.c
@@ -976,6 +976,33 @@ static ssize_t ana_state_show(struct device *dev, struct device_attribute *attr,
 }
 DEVICE_ATTR_RO(ana_state);
 
+static ssize_t numa_nodes_show(struct device *dev, struct device_attribute *attr,
+               char *buf)
+{
+       int node, srcu_idx;
+       nodemask_t numa_nodes;
+       struct nvme_ns *current_ns;
+       struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
+       struct nvme_ns_head *head = ns->head;
+
+       if (head->subsys->iopolicy != NVME_IOPOLICY_NUMA)
+               return 0;
+
+       nodes_clear(numa_nodes);
+
+       srcu_idx = srcu_read_lock(&head->srcu);
+       for_each_node(node) {
+               current_ns = srcu_dereference(head->current_path[node],
+                               &head->srcu);
+               if (ns == current_ns)
+                       node_set(node, numa_nodes);
+       }
+       srcu_read_unlock(&head->srcu, srcu_idx);
+
+       return sysfs_emit(buf, "%*pbl\n", nodemask_pr_args(&numa_nodes));
+}
+DEVICE_ATTR_RO(numa_nodes);
+
 static int nvme_lookup_ana_group_desc(struct nvme_ctrl *ctrl,
                struct nvme_ana_group_desc *desc, void *data)
 {

diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
index 1eeb782947db99dd1d14b59aee985ec6586607f7..ff4bd0c2e401e71241c8487795272728b5ad2dae 100644 (file)
--- a/drivers/nvme/host/nvme.h
+++ b/drivers/nvme/host/nvme.h
@@ -984,6 +984,7 @@ static inline void nvme_trace_bio_complete(struct request *req)
 extern bool multipath;
 extern struct device_attribute dev_attr_ana_grpid;
 extern struct device_attribute dev_attr_ana_state;
+extern struct device_attribute dev_attr_numa_nodes;
 extern struct device_attribute subsys_attr_iopolicy;
 
 static inline bool nvme_disk_is_ns_head(struct gendisk *disk)

diff --git a/drivers/nvme/host/sysfs.c b/drivers/nvme/host/sysfs.c
index 477c123a4b86a8220ec5c6205137b457017ae336..96eaecda0ab915fac7d530409460547420afbf1b 100644 (file)
--- a/drivers/nvme/host/sysfs.c
+++ b/drivers/nvme/host/sysfs.c
@@ -258,6 +258,7 @@ static struct attribute *nvme_ns_attrs[] = {
 #ifdef CONFIG_NVME_MULTIPATH
        &dev_attr_ana_grpid.attr,
        &dev_attr_ana_state.attr,
+       &dev_attr_numa_nodes.attr,
 #endif
        &dev_attr_io_passthru_err_log_enabled.attr,
        NULL,
@@ -290,6 +291,10 @@ static umode_t nvme_ns_attrs_are_visible(struct kobject *kobj,
                if (!nvme_ctrl_use_ana(nvme_get_ns_from_dev(dev)->ctrl))
                        return 0;
        }
+       if (a == &dev_attr_numa_nodes.attr) {
+               if (nvme_disk_is_ns_head(dev_to_disk(dev)))
+                       return 0;
+       }
 #endif
        return a->mode;
 }