mlx4_core: enable changing default max HCA resource limits.

Enable module-initialization time modification of default HCA
maximum resource limits via module parameters, as is done in
mthca.

Specify the log of the parameter value, rather than the value
itself to avoid the hidden side-effect of rounding up values
to next power-of-2.

For mtt's, there is a heuristic in place to configure twice
the number of MTTs required to cover host memory.  This heuristic
is modified slightly to guarantee a minimum of 2^20 mtt's, and is
used when the user does not set the log_num_mtts module parameter.

If the log_num_mtt's module parameter is set by the user, the
value set for log_num_mtts will be used as-is (instead of the
heuristic).

V2: Adapted for rebase to kernel 3.7-rc4

Signed-off-by: Jack Morgenstein <jackm@dev.mellanox.co.il>
Signed-off-by: Dotan Barak <dotanb@dev.mellanox.co.il>
Reviewed-by: Jack Morgenstein <jackm@dev.mellanox.co.il>
Signed-off-by: Vladimir Sokolovsky <vlad@mellanox.com>
Signed-off-by: Jack Morgenstein <jackm@dev.mellanox.co.il>
(Ported from Mellanox OFED 2.4)

Signed-off-by: Mukesh Kacker <mukesh.kacker@oracle.com>

diff --git a/drivers/net/ethernet/mellanox/mlx4/main.c b/drivers/net/ethernet/mellanox/mlx4/main.c
index ced5ecab5aa754ad44ae055464608bba66d6b137..98577b975174b61dc575add3163ea502c28dd2cf 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx4/main.c
+++ b/drivers/net/ethernet/mellanox/mlx4/main.c
@@ -114,16 +114,6 @@ static char mlx4_version[] =
        DRV_NAME ": Mellanox ConnectX core driver v"
        DRV_VERSION " (" DRV_RELDATE ")\n";
 
-static struct mlx4_profile default_profile = {
-       .num_qp         = 1 << 18,
-       .num_srq        = 1 << 16,
-       .rdmarc_per_qp  = 1 << 4,
-       .num_cq         = 1 << 16,
-       .num_mcg        = 1 << 13,
-       .num_mpt        = 1 << 19,
-       .num_mtt        = 1 << 20, /* It is really num mtt segements */
-};
-
 static struct mlx4_profile low_mem_profile = {
        .num_qp         = 1 << 17,
        .num_srq        = 1 << 6,
@@ -168,6 +158,79 @@ struct mlx4_port_config {
 
 static atomic_t pf_loading = ATOMIC_INIT(0);
 
+#define MLX4_LOG_NUM_MTT 20
+static struct mlx4_profile mod_param_profile = {
+       .num_qp         = 18,
+       .num_srq        = 16,
+       .rdmarc_per_qp  = 4,
+       .num_cq         = 16,
+       .num_mcg        = 13,
+       .num_mpt        = 19,
+       .num_mtt        = 0, /* max(20, 2*MTTs for host memory)) */
+};
+
+module_param_named(log_num_qp, mod_param_profile.num_qp, int, 0444);
+MODULE_PARM_DESC(log_num_qp, "log maximum number of QPs per HCA (default: 18)");
+
+module_param_named(log_num_srq, mod_param_profile.num_srq, int, 0444);
+MODULE_PARM_DESC(log_num_srq, "log maximum number of SRQs per HCA "
+                "(default: 16)");
+
+module_param_named(log_rdmarc_per_qp, mod_param_profile.rdmarc_per_qp, int,
+                  0444);
+MODULE_PARM_DESC(log_rdmarc_per_qp, "log number of RDMARC buffers per QP "
+                "(default: 4)");
+
+module_param_named(log_num_cq, mod_param_profile.num_cq, int, 0444);
+MODULE_PARM_DESC(log_num_cq, "log maximum number of CQs per HCA (default: 16)");
+
+module_param_named(log_num_mcg, mod_param_profile.num_mcg, int, 0444);
+MODULE_PARM_DESC(log_num_mcg, "log maximum number of multicast groups per HCA "
+                "(default: 13)");
+
+module_param_named(log_num_mpt, mod_param_profile.num_mpt, int, 0444);
+MODULE_PARM_DESC(log_num_mpt,
+                "log maximum number of memory protection table entries per "
+                "HCA (default: 19)");
+
+module_param_named(log_num_mtt, mod_param_profile.num_mtt, int, 0444);
+MODULE_PARM_DESC(log_num_mtt,
+                "log maximum number of memory translation table segments per "
+                "HCA (default: max(20, 2*MTTs for register all of the host memory))");
+
+static void process_mod_param_profile(struct mlx4_profile *profile)
+{
+       struct sysinfo si;
+
+       profile->num_qp        = 1 << mod_param_profile.num_qp;
+       profile->num_srq       = 1 << mod_param_profile.num_srq;
+       profile->rdmarc_per_qp = 1 << mod_param_profile.rdmarc_per_qp;
+       profile->num_cq        = 1 << mod_param_profile.num_cq;
+       profile->num_mcg       = 1 << mod_param_profile.num_mcg;
+       profile->num_mpt       = 1 << mod_param_profile.num_mpt;
+       /*
+        * We want to scale the number of MTTs with the size of the
+        * system memory, since it makes sense to register a lot of
+        * memory on a system with a lot of memory.  As a heuristic,
+        * make sure we have enough MTTs to register twice the system
+        * memory (with PAGE_SIZE entries).
+        *
+        * This number has to be a power of two and fit into 32 bits
+        * due to device limitations, so cap this at 2^31 as well.
+        * That limits us to 8TB of memory registration per HCA with
+        * 4KB pages, which is probably OK for the next few months.
+        */
+       if (mod_param_profile.num_mtt)
+               profile->num_mtt = 1 << mod_param_profile.num_mtt;
+       else {
+               si_meminfo(&si);
+               profile->num_mtt =
+                       roundup_pow_of_two(max_t(unsigned, 1 << MLX4_LOG_NUM_MTT,
+                                                min(1UL << 31,
+                                                    si.totalram >> (log_mtts_per_seg - 1))));
+       }
+}
+
 int mlx4_check_port_params(struct mlx4_dev *dev,
                           enum mlx4_port_type *port_type)
 {
@@ -2031,7 +2094,7 @@ static int mlx4_init_hca(struct mlx4_dev *dev)
                        mlx4_info(dev, "Running from within kdump kernel. Using low memory profile\n");
                        profile = low_mem_profile;
                } else {
-                       profile = default_profile;
+                       process_mod_param_profile(&profile);
                }
                if (dev->caps.steering_mode ==
                    MLX4_STEERING_MODE_DEVICE_MANAGED)

diff --git a/drivers/net/ethernet/mellanox/mlx4/profile.c b/drivers/net/ethernet/mellanox/mlx4/profile.c
index 2bf437aafc537c8afc979f7aa3ab5f6611dcc4c3..ff5648b88ddbfbe2b9c49d643a46678465b36556 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx4/profile.c
+++ b/drivers/net/ethernet/mellanox/mlx4/profile.c
@@ -83,31 +83,12 @@ u64 mlx4_make_profile(struct mlx4_dev *dev,
        u64 total_size = 0;
        struct mlx4_resource *profile;
        struct mlx4_resource tmp;
-       struct sysinfo si;
        int i, j;
 
        profile = kcalloc(MLX4_RES_NUM, sizeof(*profile), GFP_KERNEL);
        if (!profile)
                return -ENOMEM;
 
-       /*
-        * We want to scale the number of MTTs with the size of the
-        * system memory, since it makes sense to register a lot of
-        * memory on a system with a lot of memory.  As a heuristic,
-        * make sure we have enough MTTs to cover twice the system
-        * memory (with PAGE_SIZE entries).
-        *
-        * This number has to be a power of two and fit into 32 bits
-        * due to device limitations, so cap this at 2^31 as well.
-        * That limits us to 8TB of memory registration per HCA with
-        * 4KB pages, which is probably OK for the next few months.
-        */
-       si_meminfo(&si);
-       request->num_mtt =
-               roundup_pow_of_two(max_t(unsigned, request->num_mtt,
-                                        min(1UL << (31 - log_mtts_per_seg),
-                                            si.totalram >> (log_mtts_per_seg - 1))));
-
        profile[MLX4_RES_QP].size     = dev_cap->qpc_entry_sz;
        profile[MLX4_RES_RDMARC].size = dev_cap->rdmarc_entry_sz;
        profile[MLX4_RES_ALTC].size   = dev_cap->altc_entry_sz;