From 85765163952add2dcef85347a398dc0779c5c163 Mon Sep 17 00:00:00 2001 From: Mukesh Kacker Date: Mon, 16 Mar 2015 18:11:27 -0700 Subject: [PATCH] mlx4_core: More support for automatically scaling profile parameters Add a new module configuration variable "scale_profile" parameter which allows dynamic scaling of parameters. When it is not set, the Mellanox default behavior will prevail. The dynamically configured parameters are typically set to 0 in configuration - but if they are set to a specific value, a warning is printed that they are not being dynamically scaled. (This allows for make exceptions and experiments with different values). The original dynamic scaling of profile parameter num_mtt_segs (governed by log_num_mtt) is retained. In addition scaling is also introduced for parameter num_qp (governed by log_num_qp). This is not a direct port but similar in spirit to fixes done in UEK2 with following commits: 52ac96 OFED: Automatically size MTT in mlx4_core 47678c mlx4_core: increase default number of qps in mlx4_core driver 218561 mlx_core: Change log_num_mtt scaling range 497dd4 mlx4_core: change default for mlx4_scale_profile An error message improvement is borrowed from Mellanox OFED 2.4 commit 17465c net/mlx4: add explicit message if user ask too few QPs (Code for this commit is already upstream but the error message is less explicit upstream!) Signed-off-by: Mukesh Kacker --- drivers/net/ethernet/mellanox/mlx4/main.c | 49 +++++++++++++++++++++-- drivers/net/ethernet/mellanox/mlx4/qp.c | 3 +- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c index f1a790a120bb..b5ef7dd0d5a2 100644 --- a/drivers/net/ethernet/mellanox/mlx4/main.c +++ b/drivers/net/ethernet/mellanox/mlx4/main.c @@ -136,6 +136,17 @@ MODULE_PARM_DESC(log_num_vlan, "Log2 max number of VLANs per ETH port (0-7)"); #define MLX4_MIN_LOG_NUM_VLANS 0 #define MLX4_MIN_LOG_NUM_MAC 1 +/* + * mlx4_scale_profile: single tuning knob to use for tuning + * parameters based on resources to simplify configuration + * where dynamic scaling of resources makes sense. + */ +static int mlx4_scale_profile = 1; +module_param_named(scale_profile, mlx4_scale_profile, int, 0644); +MODULE_PARM_DESC(scale_profile, "Dynamically adjust default profile" + "parameters based on system resources"); +#define MLX4_SCALE_LOG_NUM_QP 20 /* 1 Meg */ + static bool use_prio; module_param_named(use_prio, use_prio, bool, 0444); MODULE_PARM_DESC(use_prio, "Enable steering by VLAN priority on ETH ports (deprecated)"); @@ -211,7 +222,26 @@ static void process_mod_param_profile(struct mlx4_profile *profile) { struct sysinfo si; - profile->num_qp = 1 << mod_param_profile.num_qp; + if (mod_param_profile.num_qp) { + if (mlx4_scale_profile) + pr_warn("mlx4_core: Both scale_profile and log_num_qp " + "are set. Ignore scale_profile.\n"); + profile->num_qp = 1 << mod_param_profile.num_qp; + } else { + /* + * Note: This could be set dynamically based on system/HCA + * resources in future. A constant for now. + */ + profile->num_qp = 1 << MLX4_SCALE_LOG_NUM_QP; + if (mlx4_scale_profile) + pr_info("mlx4_core: Scalable default profile " + "parameters are enabled. Effective log_num_qp" + " is now set to %d.\n", MLX4_SCALE_LOG_NUM_QP); + else + pr_info("mlx4_core: log_num_qp not set and scaled" + "dynamically. Effective log_num_qp" + " is now set to %d.\n", MLX4_SCALE_LOG_NUM_QP); + } profile->num_srq = 1 << mod_param_profile.num_srq; profile->rdmarc_per_qp = 1 << mod_param_profile.rdmarc_per_qp; profile->num_cq = 1 << mod_param_profile.num_cq; @@ -230,9 +260,12 @@ static void process_mod_param_profile(struct mlx4_profile *profile) * That limits us to 4TB of memory registration per HCA with * 4KB pages, which is probably OK for the next few months. */ - if (mod_param_profile.num_mtt_segs) + if (mod_param_profile.num_mtt_segs) { + if (mlx4_scale_profile) + pr_warn("mlx4_core: Both scale_profile and log_num_mtt " + "are set. Ignore scale_profile.\n"); profile->num_mtt_segs = 1 << mod_param_profile.num_mtt_segs; - else { + } else { si_meminfo(&si); profile->num_mtt_segs = roundup_pow_of_two(max_t(unsigned, @@ -246,6 +279,16 @@ static void process_mod_param_profile(struct mlx4_profile *profile) /* set the actual value, so it will be reflected to the user using the sysfs */ mod_param_profile.num_mtt_segs = ilog2(profile->num_mtt_segs); + if (mlx4_scale_profile) + pr_info("mlx4_core: Scalable default profile " + "parameters are enabled. Effective log_num_mtt" + " is now set to %d.\n", + mod_param_profile.num_mtt_segs); + else + pr_info("mlx4_core: log_num_mtt not set and scaled " + "dynamically. Effective log_num_mtt" + " is now set to %d.\n", + mod_param_profile.num_mtt_segs); } } diff --git a/drivers/net/ethernet/mellanox/mlx4/qp.c b/drivers/net/ethernet/mellanox/mlx4/qp.c index b75214a80d0e..fd81eb4611d9 100644 --- a/drivers/net/ethernet/mellanox/mlx4/qp.c +++ b/drivers/net/ethernet/mellanox/mlx4/qp.c @@ -786,7 +786,8 @@ int mlx4_init_qp_table(struct mlx4_dev *dev) */ reserved_from_bot = mlx4_num_reserved_sqps(dev); if (reserved_from_bot + reserved_from_top > dev->caps.num_qps) { - mlx4_err(dev, "Number of reserved QPs is higher than number of QPs\n"); + mlx4_err(dev, "Number of reserved QPs is higher than number " + "of QPs, increase the value of log_num_qp\n"); return -EINVAL; } -- 2.50.1