RDMA/mlx5: Fix affinity assignment

[ Upstream commit 617f5db1a626f18d5cbb7c7faf7bf8f9ea12be78 ]

The cited commit aimed to ensure that Virtual Functions (VFs) assign a
queue affinity to a Queue Pair (QP) to distribute traffic when
the LAG master creates a hardware LAG. If the affinity was set while
the hardware was not in LAG, the firmware would ignore the affinity value.

However, this commit unintentionally assigned an affinity to QPs on the LAG
master's VPORT even if the RDMA device was not marked as LAG-enabled.
In most cases, this was not an issue because when the hardware entered
hardware LAG configuration, the RDMA device of the LAG master would be
destroyed and a new one would be created, marked as LAG-enabled.

The problem arises when a user configures Equal-Cost Multipath (ECMP).
In ECMP mode, traffic can be directed to different physical ports based on
the queue affinity, which is intended for use by VPORTS other than the
E-Switch manager. ECMP mode is supported only if both E-Switch managers are
in switchdev mode and the appropriate route is configured via IP. In this
configuration, the RDMA device is not destroyed, and we retain the RDMA
device that is not marked as LAG-enabled.

To ensure correct behavior, Send Queues (SQs) opened by the E-Switch
manager through verbs should be assigned strict affinity. This means they
will only be able to communicate through the native physical port
associated with the E-Switch manager. This will prevent the firmware from
assigning affinity and will not allow the SQs to be remapped in case of
failover.

Fixes: 802dcc7fc5ec ("RDMA/mlx5: Support TX port affinity for VF drivers in LAG mode")
Reviewed-by: Maor Gottlieb <maorg@nvidia.com>
Signed-off-by: Mark Bloch <mbloch@nvidia.com>
Link: https://lore.kernel.org/r/425b05f4da840bc684b0f7e8ebf61aeb5cef09b0.1685960567.git.leon@kernel.org
Signed-off-by: Leon Romanovsky <leon@kernel.org>
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 91fc0cdf377d106a06312dd55e68f4741ecf6201..2dfa6f49a6f4816c7c0bad61d5928a6e38187e5f 100644 (file)
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -1598,6 +1598,9 @@ static inline bool mlx5_ib_lag_should_assign_affinity(struct mlx5_ib_dev *dev)
            MLX5_CAP_PORT_SELECTION(dev->mdev, port_select_flow_table_bypass))
                return 0;
 
+       if (mlx5_lag_is_lacp_owner(dev->mdev) && !dev->lag_active)
+               return 0;
+
        return dev->lag_active ||
                (MLX5_CAP_GEN(dev->mdev, num_lag_ports) > 1 &&
                 MLX5_CAP_GEN(dev->mdev, lag_tx_port_affinity));

diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c
index 86284aba3470dad5a4b3382465d23b07be9ab8ca..5bc63020b766d4cc8b92b16ed55751d30f8ee526 100644 (file)
--- a/drivers/infiniband/hw/mlx5/qp.c
+++ b/drivers/infiniband/hw/mlx5/qp.c
@@ -1233,6 +1233,9 @@ static int create_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
 
        MLX5_SET(create_tis_in, in, uid, to_mpd(pd)->uid);
        MLX5_SET(tisc, tisc, transport_domain, tdn);
+       if (!mlx5_ib_lag_should_assign_affinity(dev) &&
+           mlx5_lag_is_lacp_owner(dev->mdev))
+               MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1);
        if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
                MLX5_SET(tisc, tisc, underlay_qpn, qp->underlay_qpn);
 

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
index a3c5c2dab5fd725570f76f3852dfccd60d126a3e..d7ef853702b792b887dd34b4c24584a80ec30236 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h
@@ -275,18 +275,6 @@ static inline bool mlx5_sriov_is_enabled(struct mlx5_core_dev *dev)
        return pci_num_vf(dev->pdev) ? true : false;
 }
 
-static inline int mlx5_lag_is_lacp_owner(struct mlx5_core_dev *dev)
-{
-       /* LACP owner conditions:
-        * 1) Function is physical.
-        * 2) LAG is supported by FW.
-        * 3) LAG is managed by driver (currently the only option).
-        */
-       return  MLX5_CAP_GEN(dev, vport_group_manager) &&
-                  (MLX5_CAP_GEN(dev, num_lag_ports) > 1) &&
-                   MLX5_CAP_GEN(dev, lag_master);
-}
-
 int mlx5_rescan_drivers_locked(struct mlx5_core_dev *dev);
 static inline int mlx5_rescan_drivers(struct mlx5_core_dev *dev)
 {

diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index 68a3183d5d5894d93973b8f769ee2ae7ca661107..33dbe941d070cdd4021cc0534e7f3b223ba47d97 100644 (file)
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -1233,6 +1233,18 @@ static inline u16 mlx5_core_max_vfs(const struct mlx5_core_dev *dev)
        return dev->priv.sriov.max_vfs;
 }
 
+static inline int mlx5_lag_is_lacp_owner(struct mlx5_core_dev *dev)
+{
+       /* LACP owner conditions:
+        * 1) Function is physical.
+        * 2) LAG is supported by FW.
+        * 3) LAG is managed by driver (currently the only option).
+        */
+       return  MLX5_CAP_GEN(dev, vport_group_manager) &&
+                  (MLX5_CAP_GEN(dev, num_lag_ports) > 1) &&
+                   MLX5_CAP_GEN(dev, lag_master);
+}
+
 static inline int mlx5_get_gid_table_len(u16 param)
 {
        if (param > 4) {