net/mlx5e: Use the 'num_doorbells' devlink param

Use the new devlink param to control how many doorbells mlx5e devices
allocate and use. The maximum number of doorbells configurable is capped
to the maximum number of channels. This only applies to the Ethernet
part, the RDMA devices using mlx5 manage their own doorbells.

Signed-off-by: Cosmin Ratiu <cratiu@nvidia.com>
Reviewed-by: Dragos Tatulea <dtatulea@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
Reviewed-by: Simon Horman <horms@kernel.org>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/Documentation/networking/devlink/mlx5.rst b/Documentation/networking/devlink/mlx5.rst
index 60cc9fedf1ef70bad5c8c12f4f7a9d28f0902b87..41c9b716699eaa778f4649b377028c08036c9532 100644 (file)
--- a/Documentation/networking/devlink/mlx5.rst
+++ b/Documentation/networking/devlink/mlx5.rst
@@ -62,6 +62,15 @@ Note: permanent parameters such as ``enable_sriov`` and ``total_vfs`` require FW
    echo 1 >/sys/bus/pci/rescan
    grep ^ /sys/bus/pci/devices/0000:01:00.0/sriov_*
 
+   * - ``num_doorbells``
+     - driverinit
+     - This controls the number of channel doorbells used by the netdev. In all
+       cases, an additional doorbell is allocated and used for non-channel
+       communication (e.g. for PTP, HWS, etc.). Supported values are:
+
+       - 0: No channel-specific doorbells, use the global one for everything.
+       - [1, max_num_channels]: Spread netdev channels equally across these
+         doorbells.
 
 The ``mlx5`` driver also implements the following driver-specific
 parameters.

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
index bfa44414be82f54cd3c3b846835675c9139a3722..fceea83abbd76b9f3cfe505003d4ec82501f309f 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c
@@ -530,6 +530,25 @@ mlx5_devlink_hairpin_queue_size_validate(struct devlink *devlink, u32 id,
        return 0;
 }
 
+static int mlx5_devlink_num_doorbells_validate(struct devlink *devlink, u32 id,
+                                              union devlink_param_value val,
+                                              struct netlink_ext_ack *extack)
+{
+       struct mlx5_core_dev *mdev = devlink_priv(devlink);
+       u32 val32 = val.vu32;
+       u32 max_num_channels;
+
+       max_num_channels = mlx5e_get_max_num_channels(mdev);
+       if (val32 > max_num_channels) {
+               NL_SET_ERR_MSG_FMT_MOD(extack,
+                                      "Requested num_doorbells (%u) exceeds maximum number of channels (%u)",
+                                      val32, max_num_channels);
+               return -EINVAL;
+       }
+
+       return 0;
+}
+
 static void mlx5_devlink_hairpin_params_init_values(struct devlink *devlink)
 {
        struct mlx5_core_dev *dev = devlink_priv(devlink);
@@ -609,6 +628,9 @@ static const struct devlink_param mlx5_devlink_eth_params[] = {
                             "hairpin_queue_size", DEVLINK_PARAM_TYPE_U32,
                             BIT(DEVLINK_PARAM_CMODE_DRIVERINIT), NULL, NULL,
                             mlx5_devlink_hairpin_queue_size_validate),
+       DEVLINK_PARAM_GENERIC(NUM_DOORBELLS,
+                             BIT(DEVLINK_PARAM_CMODE_DRIVERINIT), NULL, NULL,
+                             mlx5_devlink_num_doorbells_validate),
 };
 
 static int mlx5_devlink_eth_params_register(struct devlink *devlink)
@@ -632,6 +654,10 @@ static int mlx5_devlink_eth_params_register(struct devlink *devlink)
 
        mlx5_devlink_hairpin_params_init_values(devlink);
 
+       value.vu32 = MLX5_DEFAULT_NUM_DOORBELLS;
+       devl_param_driverinit_value_set(devlink,
+                                       DEVLINK_PARAM_GENERIC_ID_NUM_DOORBELLS,
+                                       value);
        return 0;
 }
 

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c
index d13cebbc763a450cfe96e0d2d8e0a8da77e1ff4c..96b744ceaf13560eaa83acb260ff915082fc674a 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_common.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_common.c
@@ -30,6 +30,7 @@
  * SOFTWARE.
  */
 
+#include "devlink.h"
 #include "en.h"
 #include "lib/crypto.h"
 
@@ -140,6 +141,18 @@ err_close_tises:
        return err;
 }
 
+static unsigned int
+mlx5e_get_devlink_param_num_doorbells(struct mlx5_core_dev *dev)
+{
+       const u32 param_id = DEVLINK_PARAM_GENERIC_ID_NUM_DOORBELLS;
+       struct devlink *devlink = priv_to_devlink(dev);
+       union devlink_param_value val;
+       int err;
+
+       err = devl_param_driverinit_value_get(devlink, param_id, &val);
+       return err ? MLX5_DEFAULT_NUM_DOORBELLS : val.vu32;
+}
+
 int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev, bool create_tises)
 {
        struct mlx5e_hw_objs *res = &mdev->mlx5e_res.hw_objs;
@@ -164,7 +177,7 @@ int mlx5e_create_mdev_resources(struct mlx5_core_dev *mdev, bool create_tises)
                goto err_dealloc_transport_domain;
        }
 
-       num_doorbells = min(MLX5_DEFAULT_NUM_DOORBELLS,
+       num_doorbells = min(mlx5e_get_devlink_param_num_doorbells(mdev),
                            mlx5e_get_max_num_channels(mdev));
        res->bfregs = kcalloc(num_doorbells, sizeof(*res->bfregs), GFP_KERNEL);
        if (!res->bfregs) {