net/mlx5: SD, Implement devcom communication and primary election

Use devcom to communicate between the different devices. Add a new
devcom component type for this.

Each device registers itself to the devcom component <SD, group ID>.
Once all devices of a component are registered, the component becomes
ready, and a primary device is elected.

In principle, any of the devices can act as a primary, they are all
capable, and a random election would've worked. However, we aim to
achieve predictability and consistency, hence each group always choses
the same device, with the lowest PCI BUS number, as primary.

Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
Signed-off-by: Saeed Mahameed <saeedm@nvidia.com>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h
index ec32b686f58654ed9628d13d8bf5ecf292420a4a..d58032dd0df74c108b78d66bd21a89b31663c5f9 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h
@@ -10,6 +10,7 @@ enum mlx5_devcom_component {
        MLX5_DEVCOM_ESW_OFFLOADS,
        MLX5_DEVCOM_MPV,
        MLX5_DEVCOM_HCA_PORTS,
+       MLX5_DEVCOM_SD_GROUP,
        MLX5_DEVCOM_NUM_COMPONENTS,
 };
 

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c
index 9d8b1bb0c0a6deabe3f8acbb1f92d638d039da58..19e674dd1af7873e87f037ca9155a5f66f54f8fd 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c
@@ -14,6 +14,16 @@
 struct mlx5_sd {
        u32 group_id;
        u8 host_buses;
+       struct mlx5_devcom_comp_dev *devcom;
+       bool primary;
+       union {
+               struct { /* primary */
+                       struct mlx5_core_dev *secondaries[MLX5_SD_MAX_GROUP_SZ - 1];
+               };
+               struct { /* secondary */
+                       struct mlx5_core_dev *primary_dev;
+               };
+       };
 };
 
 static int mlx5_sd_get_host_buses(struct mlx5_core_dev *dev)
@@ -26,13 +36,29 @@ static int mlx5_sd_get_host_buses(struct mlx5_core_dev *dev)
        return sd->host_buses;
 }
 
+static struct mlx5_core_dev *mlx5_sd_get_primary(struct mlx5_core_dev *dev)
+{
+       struct mlx5_sd *sd = mlx5_get_sd(dev);
+
+       if (!sd)
+               return dev;
+
+       return sd->primary ? dev : sd->primary_dev;
+}
+
 struct mlx5_core_dev *
 mlx5_sd_primary_get_peer(struct mlx5_core_dev *primary, int idx)
 {
+       struct mlx5_sd *sd;
+
        if (idx == 0)
                return primary;
 
-       return NULL;
+       if (idx >= mlx5_sd_get_host_buses(primary))
+               return NULL;
+
+       sd = mlx5_get_sd(primary);
+       return sd->secondaries[idx - 1];
 }
 
 int mlx5_sd_ch_ix_get_dev_ix(struct mlx5_core_dev *dev, int ch_ix)
@@ -136,15 +162,93 @@ static void sd_cleanup(struct mlx5_core_dev *dev)
        kfree(sd);
 }
 
+static int sd_register(struct mlx5_core_dev *dev)
+{
+       struct mlx5_devcom_comp_dev *devcom, *pos;
+       struct mlx5_core_dev *peer, *primary;
+       struct mlx5_sd *sd, *primary_sd;
+       int err, i;
+
+       sd = mlx5_get_sd(dev);
+       devcom = mlx5_devcom_register_component(dev->priv.devc, MLX5_DEVCOM_SD_GROUP,
+                                               sd->group_id, NULL, dev);
+       if (!devcom)
+               return -ENOMEM;
+
+       sd->devcom = devcom;
+
+       if (mlx5_devcom_comp_get_size(devcom) != sd->host_buses)
+               return 0;
+
+       mlx5_devcom_comp_lock(devcom);
+       mlx5_devcom_comp_set_ready(devcom, true);
+       mlx5_devcom_comp_unlock(devcom);
+
+       if (!mlx5_devcom_for_each_peer_begin(devcom)) {
+               err = -ENODEV;
+               goto err_devcom_unreg;
+       }
+
+       primary = dev;
+       mlx5_devcom_for_each_peer_entry(devcom, peer, pos)
+               if (peer->pdev->bus->number < primary->pdev->bus->number)
+                       primary = peer;
+
+       primary_sd = mlx5_get_sd(primary);
+       primary_sd->primary = true;
+       i = 0;
+       /* loop the secondaries */
+       mlx5_devcom_for_each_peer_entry(primary_sd->devcom, peer, pos) {
+               struct mlx5_sd *peer_sd = mlx5_get_sd(peer);
+
+               primary_sd->secondaries[i++] = peer;
+               peer_sd->primary = false;
+               peer_sd->primary_dev = primary;
+       }
+
+       mlx5_devcom_for_each_peer_end(devcom);
+       return 0;
+
+err_devcom_unreg:
+       mlx5_devcom_comp_lock(sd->devcom);
+       mlx5_devcom_comp_set_ready(sd->devcom, false);
+       mlx5_devcom_comp_unlock(sd->devcom);
+       mlx5_devcom_unregister_component(sd->devcom);
+       return err;
+}
+
+static void sd_unregister(struct mlx5_core_dev *dev)
+{
+       struct mlx5_sd *sd = mlx5_get_sd(dev);
+
+       mlx5_devcom_comp_lock(sd->devcom);
+       mlx5_devcom_comp_set_ready(sd->devcom, false);
+       mlx5_devcom_comp_unlock(sd->devcom);
+       mlx5_devcom_unregister_component(sd->devcom);
+}
+
 int mlx5_sd_init(struct mlx5_core_dev *dev)
 {
+       struct mlx5_sd *sd = mlx5_get_sd(dev);
        int err;
 
        err = sd_init(dev);
        if (err)
                return err;
 
+       sd = mlx5_get_sd(dev);
+       if (!sd)
+               return 0;
+
+       err = sd_register(dev);
+       if (err)
+               goto err_sd_cleanup;
+
        return 0;
+
+err_sd_cleanup:
+       sd_cleanup(dev);
+       return err;
 }
 
 void mlx5_sd_cleanup(struct mlx5_core_dev *dev)
@@ -154,6 +258,7 @@ void mlx5_sd_cleanup(struct mlx5_core_dev *dev)
        if (!sd)
                return;
 
+       sd_unregister(dev);
        sd_cleanup(dev);
 }
 
@@ -161,5 +266,18 @@ struct auxiliary_device *mlx5_sd_get_adev(struct mlx5_core_dev *dev,
                                          struct auxiliary_device *adev,
                                          int idx)
 {
-       return adev;
+       struct mlx5_sd *sd = mlx5_get_sd(dev);
+       struct mlx5_core_dev *primary;
+
+       if (!sd)
+               return adev;
+
+       if (!mlx5_devcom_comp_is_ready(sd->devcom))
+               return NULL;
+
+       primary = mlx5_sd_get_primary(dev);
+       if (dev == primary)
+               return adev;
+
+       return &primary->priv.adev[idx]->adev;
 }