From 37888516ec933840592524f2766e6d6d34c9f66c Mon Sep 17 00:00:00 2001 From: Dotan Barak Date: Thu, 15 Nov 2012 12:53:40 +0200 Subject: [PATCH] mlx4_core: enable changing default max HCA resource limits. Enable module-initialization time modification of default HCA maximum resource limits via module parameters, as is done in mthca. Specify the log of the parameter value, rather than the value itself to avoid the hidden side-effect of rounding up values to next power-of-2. For mtt's, there is a heuristic in place to configure twice the number of MTTs required to cover host memory. This heuristic is modified slightly to guarantee a minimum of 2^20 mtt's, and is used when the user does not set the log_num_mtts module parameter. If the log_num_mtt's module parameter is set by the user, the value set for log_num_mtts will be used as-is (instead of the heuristic). V2: Adapted for rebase to kernel 3.7-rc4 Signed-off-by: Jack Morgenstein Signed-off-by: Dotan Barak Reviewed-by: Jack Morgenstein Signed-off-by: Vladimir Sokolovsky Signed-off-by: Jack Morgenstein (Ported from Mellanox OFED 2.4) Signed-off-by: Mukesh Kacker --- drivers/net/ethernet/mellanox/mlx4/main.c | 85 +++++++++++++++++--- drivers/net/ethernet/mellanox/mlx4/profile.c | 19 ----- 2 files changed, 74 insertions(+), 30 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c index ced5ecab5aa7..98577b975174 100644 --- a/drivers/net/ethernet/mellanox/mlx4/main.c +++ b/drivers/net/ethernet/mellanox/mlx4/main.c @@ -114,16 +114,6 @@ static char mlx4_version[] = DRV_NAME ": Mellanox ConnectX core driver v" DRV_VERSION " (" DRV_RELDATE ")\n"; -static struct mlx4_profile default_profile = { - .num_qp = 1 << 18, - .num_srq = 1 << 16, - .rdmarc_per_qp = 1 << 4, - .num_cq = 1 << 16, - .num_mcg = 1 << 13, - .num_mpt = 1 << 19, - .num_mtt = 1 << 20, /* It is really num mtt segements */ -}; - static struct mlx4_profile low_mem_profile = { .num_qp = 1 << 17, .num_srq = 1 << 6, @@ -168,6 +158,79 @@ struct mlx4_port_config { static atomic_t pf_loading = ATOMIC_INIT(0); +#define MLX4_LOG_NUM_MTT 20 +static struct mlx4_profile mod_param_profile = { + .num_qp = 18, + .num_srq = 16, + .rdmarc_per_qp = 4, + .num_cq = 16, + .num_mcg = 13, + .num_mpt = 19, + .num_mtt = 0, /* max(20, 2*MTTs for host memory)) */ +}; + +module_param_named(log_num_qp, mod_param_profile.num_qp, int, 0444); +MODULE_PARM_DESC(log_num_qp, "log maximum number of QPs per HCA (default: 18)"); + +module_param_named(log_num_srq, mod_param_profile.num_srq, int, 0444); +MODULE_PARM_DESC(log_num_srq, "log maximum number of SRQs per HCA " + "(default: 16)"); + +module_param_named(log_rdmarc_per_qp, mod_param_profile.rdmarc_per_qp, int, + 0444); +MODULE_PARM_DESC(log_rdmarc_per_qp, "log number of RDMARC buffers per QP " + "(default: 4)"); + +module_param_named(log_num_cq, mod_param_profile.num_cq, int, 0444); +MODULE_PARM_DESC(log_num_cq, "log maximum number of CQs per HCA (default: 16)"); + +module_param_named(log_num_mcg, mod_param_profile.num_mcg, int, 0444); +MODULE_PARM_DESC(log_num_mcg, "log maximum number of multicast groups per HCA " + "(default: 13)"); + +module_param_named(log_num_mpt, mod_param_profile.num_mpt, int, 0444); +MODULE_PARM_DESC(log_num_mpt, + "log maximum number of memory protection table entries per " + "HCA (default: 19)"); + +module_param_named(log_num_mtt, mod_param_profile.num_mtt, int, 0444); +MODULE_PARM_DESC(log_num_mtt, + "log maximum number of memory translation table segments per " + "HCA (default: max(20, 2*MTTs for register all of the host memory))"); + +static void process_mod_param_profile(struct mlx4_profile *profile) +{ + struct sysinfo si; + + profile->num_qp = 1 << mod_param_profile.num_qp; + profile->num_srq = 1 << mod_param_profile.num_srq; + profile->rdmarc_per_qp = 1 << mod_param_profile.rdmarc_per_qp; + profile->num_cq = 1 << mod_param_profile.num_cq; + profile->num_mcg = 1 << mod_param_profile.num_mcg; + profile->num_mpt = 1 << mod_param_profile.num_mpt; + /* + * We want to scale the number of MTTs with the size of the + * system memory, since it makes sense to register a lot of + * memory on a system with a lot of memory. As a heuristic, + * make sure we have enough MTTs to register twice the system + * memory (with PAGE_SIZE entries). + * + * This number has to be a power of two and fit into 32 bits + * due to device limitations, so cap this at 2^31 as well. + * That limits us to 8TB of memory registration per HCA with + * 4KB pages, which is probably OK for the next few months. + */ + if (mod_param_profile.num_mtt) + profile->num_mtt = 1 << mod_param_profile.num_mtt; + else { + si_meminfo(&si); + profile->num_mtt = + roundup_pow_of_two(max_t(unsigned, 1 << MLX4_LOG_NUM_MTT, + min(1UL << 31, + si.totalram >> (log_mtts_per_seg - 1)))); + } +} + int mlx4_check_port_params(struct mlx4_dev *dev, enum mlx4_port_type *port_type) { @@ -2031,7 +2094,7 @@ static int mlx4_init_hca(struct mlx4_dev *dev) mlx4_info(dev, "Running from within kdump kernel. Using low memory profile\n"); profile = low_mem_profile; } else { - profile = default_profile; + process_mod_param_profile(&profile); } if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) diff --git a/drivers/net/ethernet/mellanox/mlx4/profile.c b/drivers/net/ethernet/mellanox/mlx4/profile.c index 2bf437aafc53..ff5648b88ddb 100644 --- a/drivers/net/ethernet/mellanox/mlx4/profile.c +++ b/drivers/net/ethernet/mellanox/mlx4/profile.c @@ -83,31 +83,12 @@ u64 mlx4_make_profile(struct mlx4_dev *dev, u64 total_size = 0; struct mlx4_resource *profile; struct mlx4_resource tmp; - struct sysinfo si; int i, j; profile = kcalloc(MLX4_RES_NUM, sizeof(*profile), GFP_KERNEL); if (!profile) return -ENOMEM; - /* - * We want to scale the number of MTTs with the size of the - * system memory, since it makes sense to register a lot of - * memory on a system with a lot of memory. As a heuristic, - * make sure we have enough MTTs to cover twice the system - * memory (with PAGE_SIZE entries). - * - * This number has to be a power of two and fit into 32 bits - * due to device limitations, so cap this at 2^31 as well. - * That limits us to 8TB of memory registration per HCA with - * 4KB pages, which is probably OK for the next few months. - */ - si_meminfo(&si); - request->num_mtt = - roundup_pow_of_two(max_t(unsigned, request->num_mtt, - min(1UL << (31 - log_mtts_per_seg), - si.totalram >> (log_mtts_per_seg - 1)))); - profile[MLX4_RES_QP].size = dev_cap->qpc_entry_sz; profile[MLX4_RES_RDMARC].size = dev_cap->rdmarc_entry_sz; profile[MLX4_RES_ALTC].size = dev_cap->altc_entry_sz; -- 2.50.1