mlx4: Subscribe to PXM notifier

With patch titled: "xen/pci: Add PXM node notifier for PXM (NUMA) changes."
there is a notifier which can be used to obtain information
about which NUMA node the device is on.

Usually this kind of information is available prior to the driver
loading, but thanks to the braindead way piix4 emulation works
in QEMU there is no good way of making this work.

Upstream is working on ditching this, but it is a year off or so.

Note that this should not be needed if QEMU q35 platform
is exposed with PCIe bridges in them. The PCIe bridges would contain
the proper PXM information and the hotplugged PCIe device would
snuggly be attached there.

But in the meantime this hack is in place to expose the PXM data.

OraBug: 27200813
Acked-by: Håkon Bugge <haakon.bugge@oracle.com>
Reviewed-by: Yuval Shaia <yuval.shaia@oracle.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>

diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c
index 111639cf7f8d36d156e6498a01c8def79f5766cc..3001693d428c4765ef0d616026fb9c1db744d115 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx4/main.c
+++ b/drivers/net/ethernet/mellanox/mlx4/main.c
@@ -45,6 +45,7 @@
 
 #include <linux/mlx4/device.h>
 #include <linux/mlx4/doorbell.h>
+#include <asm/xen/pci.h>
 
 #include "mlx4.h"
 #include "fw.h"
@@ -3031,6 +3032,29 @@ static int mlx4_check_dev_cap(struct mlx4_dev *dev, struct mlx4_dev_cap *dev_cap
        return 0;
 }
 
+static int mlx4_update_numa_node(struct notifier_block *this,
+                                unsigned long event, void *ptr)
+{
+       struct pci_dev *pdev = ptr;
+       struct mlx4_dev_persistent *persist;
+       struct mlx4_dev *dev;
+       int pxm;
+
+       persist = pci_get_drvdata(pdev);
+       dev = persist->dev;
+       pxm = dev_to_node(&pdev->dev);
+
+       dev_info(&pdev->dev, "Updating PXM to %d\n", pxm);
+
+       dev->numa_node = pxm;
+
+       return NOTIFY_DONE;
+}
+
+static struct notifier_block pxm_notifier_block = {
+       .notifier_call = mlx4_update_numa_node,
+};
+
 static int mlx4_load_one(struct pci_dev *pdev, int pci_dev_data,
                         int total_vfs, int *nvfs, struct mlx4_priv *priv,
                         int reset_flow)
@@ -3366,6 +3390,10 @@ slave_start:
                atomic_dec(&pf_loading);
 
        kfree(dev_cap);
+
+       /* Update numa_node if there are changes. */
+       register_pci_pxm_handler(&pxm_notifier_block, pdev);
+
        return 0;
 
 err_port:
@@ -3649,6 +3677,7 @@ static void mlx4_unload_one(struct pci_dev *pdev)
        if (priv->removed)
                return;
 
+       unregister_pci_pxm_handler(&pxm_notifier_block, pdev);
        /* saving current ports type for further use */
        for (i = 0; i < dev->caps.num_ports; i++) {
                dev->persist->curr_port_type[i] = dev->caps.port_type[i + 1];