net/mlx5: Disable roce at HCA level

Currently, when a user disables roce via the devlink param, this change
isn't passed down to the device.
If device allows disabling RoCE at device level, make use of it. This
instructs the device to skip memory allocations related to RoCE
functionality which otherwise is done by the device.

Signed-off-by: Shay Drory <shayd@nvidia.com>
Reviewed-by: Parav Pandit <parav@nvidia.com>
Signed-off-by: Saeed Mahameed <saeedm@nvidia.com>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
index a85341a41cd001d060dfa7506d262b8d753c8a1d..1c98652b244a0b9eed311d9e750a90c3e31b8e55 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
@@ -454,7 +454,8 @@ static int mlx5_devlink_enable_roce_validate(struct devlink *devlink, u32 id,
        struct mlx5_core_dev *dev = devlink_priv(devlink);
        bool new_state = val.vbool;
 
-       if (new_state && !MLX5_CAP_GEN(dev, roce)) {
+       if (new_state && !MLX5_CAP_GEN(dev, roce) &&
+           !MLX5_CAP_GEN(dev, roce_rw_supported)) {
                NL_SET_ERR_MSG_MOD(extack, "Device doesn't support RoCE");
                return -EOPNOTSUPP;
        }

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c
index 75d2842721199e1eeffa4759758f71f47ed62eed..47d92fb459edc38b23a402003b49f497d4256404 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c
@@ -558,15 +558,38 @@ static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx)
                MLX5_SET(cmd_hca_cap, set_hca_cap, num_total_dynamic_vf_msix,
                         MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix));
 
+       if (MLX5_CAP_GEN(dev, roce_rw_supported))
+               MLX5_SET(cmd_hca_cap, set_hca_cap, roce, mlx5_is_roce_init_enabled(dev));
+
        return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE);
 }
 
+/* Cached MLX5_CAP_GEN(dev, roce) can be out of sync this early in the
+ * boot process.
+ * In case RoCE cap is writable in FW and user/devlink requested to change the
+ * cap, we are yet to query the final state of the above cap.
+ * Hence, the need for this function.
+ *
+ * Returns
+ * True:
+ * 1) RoCE cap is read only in FW and already disabled
+ * OR:
+ * 2) RoCE cap is writable in FW and user/devlink requested it off.
+ *
+ * In any other case, return False.
+ */
+static bool is_roce_fw_disabled(struct mlx5_core_dev *dev)
+{
+       return (MLX5_CAP_GEN(dev, roce_rw_supported) && !mlx5_is_roce_init_enabled(dev)) ||
+               (!MLX5_CAP_GEN(dev, roce_rw_supported) && !MLX5_CAP_GEN(dev, roce));
+}
+
 static int handle_hca_cap_roce(struct mlx5_core_dev *dev, void *set_ctx)
 {
        void *set_hca_cap;
        int err;
 
-       if (!MLX5_CAP_GEN(dev, roce))
+       if (is_roce_fw_disabled(dev))
                return 0;
 
        err = mlx5_core_get_caps(dev, MLX5_CAP_ROCE);

diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h
index fb06e8870aee7001e1cd21e9713e243704df8f6d..7c8b5f06c2cd7aebd163211fef210aa3dd91e7fc 100644 (file)
--- a/include/linux/mlx5/driver.h
+++ b/include/linux/mlx5/driver.h
@@ -1251,11 +1251,12 @@ static inline bool mlx5_is_roce_init_enabled(struct mlx5_core_dev *dev)
 {
        struct devlink *devlink = priv_to_devlink(dev);
        union devlink_param_value val;
+       int err;
 
-       devlink_param_driverinit_value_get(devlink,
-                                          DEVLINK_PARAM_GENERIC_ID_ENABLE_ROCE,
-                                          &val);
-       return val.vbool;
+       err = devlink_param_driverinit_value_get(devlink,
+                                                DEVLINK_PARAM_GENERIC_ID_ENABLE_ROCE,
+                                                &val);
+       return err ? MLX5_CAP_GEN(dev, roce) : val.vbool;
 }
 
 #endif /* MLX5_DRIVER_H */

diff --git a/include/linux/mlx5/mlx5_ifc.h b/include/linux/mlx5/mlx5_ifc.h
index b8bff5109656335c5f2b4c5727d2a61ccbf4c1f9..c614ad1da44d9335a35f681f84ff986729850b65 100644 (file)
--- a/include/linux/mlx5/mlx5_ifc.h
+++ b/include/linux/mlx5/mlx5_ifc.h
@@ -1588,7 +1588,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {
        u8         log_max_tis_per_sq[0x5];
 
        u8         ext_stride_num_range[0x1];
-       u8         reserved_at_3a1[0x2];
+       u8         roce_rw_supported[0x1];
+       u8         reserved_at_3a2[0x1];
        u8         log_max_stride_sz_rq[0x5];
        u8         reserved_at_3a8[0x3];
        u8         log_min_stride_sz_rq[0x5];