RDMA/rtrs-srv: Incorporate ib_register_client into rtrs server init

The rnbd_server module's communication manager (cm) initialization depends
on the registration of the "network namespace subsystem" of the RDMA CM
agent module. As such, when the kernel is configured to load the
rnbd_server and the RDMA cma module during initialization; and if the
rnbd_server module is initialized before RDMA cma module, a null ptr
dereference occurs during the RDMA bind operation.

Call trace:

  Call Trace:
   ? xas_load+0xd/0x80
   xa_load+0x47/0x80
   cma_ps_find+0x44/0x70
   rdma_bind_addr+0x782/0x8b0
   ? get_random_bytes+0x35/0x40
   rtrs_srv_cm_init+0x50/0x80
   rtrs_srv_open+0x102/0x180
   ? rnbd_client_init+0x6e/0x6e
   rnbd_srv_init_module+0x34/0x84
   ? rnbd_client_init+0x6e/0x6e
   do_one_initcall+0x4a/0x200
   kernel_init_freeable+0x1f1/0x26e
   ? rest_init+0xb0/0xb0
   kernel_init+0xe/0x100
   ret_from_fork+0x22/0x30
  Modules linked in:
  CR2: 0000000000000015

All this happens cause the cm init is in the call chain of the module
init, which is not a preferred practice.

So remove the call to rdma_create_id() from the module init call chain.
Instead register rtrs-srv as an ib client, which makes sure that the
rdma_create_id() is called only when an ib device is added.

Fixes: 9cb837480424 ("RDMA/rtrs: server: main functionality")
Link: https://lore.kernel.org/r/20200907103106.104530-1-haris.iqbal@cloud.ionos.com
Reported-by: kernel test robot <rong.a.chen@intel.com>
Signed-off-by: Md Haris Iqbal <haris.iqbal@cloud.ionos.com>
Reviewed-by: Jack Wang <jinpu.wang@cloud.ionos.com>
Reviewed-by: Leon Romanovsky <leonro@nvidia.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>

diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.c b/drivers/infiniband/ulp/rtrs/rtrs-srv.c
index a219bd1bdbc26525b3144356c1bc76080e67a273..c28309e517e2eaac3d101a4359a09bd63afdfe9a 100644 (file)
--- a/drivers/infiniband/ulp/rtrs/rtrs-srv.c
+++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.c
@@ -16,6 +16,7 @@
 #include "rtrs-srv.h"
 #include "rtrs-log.h"
 #include <rdma/ib_cm.h>
+#include <rdma/ib_verbs.h>
 
 MODULE_DESCRIPTION("RDMA Transport Server");
 MODULE_LICENSE("GPL");
@@ -31,6 +32,7 @@ MODULE_LICENSE("GPL");
 static struct rtrs_rdma_dev_pd dev_pd;
 static mempool_t *chunk_pool;
 struct class *rtrs_dev_class;
+static struct rtrs_srv_ib_ctx ib_ctx;
 
 static int __read_mostly max_chunk_size = DEFAULT_MAX_CHUNK_SIZE;
 static int __read_mostly sess_queue_depth = DEFAULT_SESS_QUEUE_DEPTH;
@@ -2033,6 +2035,70 @@ static void free_srv_ctx(struct rtrs_srv_ctx *ctx)
        kfree(ctx);
 }
 
+static int rtrs_srv_add_one(struct ib_device *device)
+{
+       struct rtrs_srv_ctx *ctx;
+       int ret = 0;
+
+       mutex_lock(&ib_ctx.ib_dev_mutex);
+       if (ib_ctx.ib_dev_count)
+               goto out;
+
+       /*
+        * Since our CM IDs are NOT bound to any ib device we will create them
+        * only once
+        */
+       ctx = ib_ctx.srv_ctx;
+       ret = rtrs_srv_rdma_init(ctx, ib_ctx.port);
+       if (ret) {
+               /*
+                * We errored out here.
+                * According to the ib code, if we encounter an error here then the
+                * error code is ignored, and no more calls to our ops are made.
+                */
+               pr_err("Failed to initialize RDMA connection");
+               goto err_out;
+       }
+
+out:
+       /*
+        * Keep a track on the number of ib devices added
+        */
+       ib_ctx.ib_dev_count++;
+
+err_out:
+       mutex_unlock(&ib_ctx.ib_dev_mutex);
+       return ret;
+}
+
+static void rtrs_srv_remove_one(struct ib_device *device, void *client_data)
+{
+       struct rtrs_srv_ctx *ctx;
+
+       mutex_lock(&ib_ctx.ib_dev_mutex);
+       ib_ctx.ib_dev_count--;
+
+       if (ib_ctx.ib_dev_count)
+               goto out;
+
+       /*
+        * Since our CM IDs are NOT bound to any ib device we will remove them
+        * only once, when the last device is removed
+        */
+       ctx = ib_ctx.srv_ctx;
+       rdma_destroy_id(ctx->cm_id_ip);
+       rdma_destroy_id(ctx->cm_id_ib);
+
+out:
+       mutex_unlock(&ib_ctx.ib_dev_mutex);
+}
+
+static struct ib_client rtrs_srv_client = {
+       .name   = "rtrs_server",
+       .add    = rtrs_srv_add_one,
+       .remove = rtrs_srv_remove_one
+};
+
 /**
  * rtrs_srv_open() - open RTRS server context
  * @ops:               callback functions
@@ -2051,7 +2117,11 @@ struct rtrs_srv_ctx *rtrs_srv_open(struct rtrs_srv_ops *ops, u16 port)
        if (!ctx)
                return ERR_PTR(-ENOMEM);
 
-       err = rtrs_srv_rdma_init(ctx, port);
+       mutex_init(&ib_ctx.ib_dev_mutex);
+       ib_ctx.srv_ctx = ctx;
+       ib_ctx.port = port;
+
+       err = ib_register_client(&rtrs_srv_client);
        if (err) {
                free_srv_ctx(ctx);
                return ERR_PTR(err);
@@ -2090,8 +2160,8 @@ static void close_ctx(struct rtrs_srv_ctx *ctx)
  */
 void rtrs_srv_close(struct rtrs_srv_ctx *ctx)
 {
-       rdma_destroy_id(ctx->cm_id_ip);
-       rdma_destroy_id(ctx->cm_id_ib);
+       ib_unregister_client(&rtrs_srv_client);
+       mutex_destroy(&ib_ctx.ib_dev_mutex);
        close_ctx(ctx);
        free_srv_ctx(ctx);
 }

diff --git a/drivers/infiniband/ulp/rtrs/rtrs-srv.h b/drivers/infiniband/ulp/rtrs/rtrs-srv.h
index dc95b0932f0dfb2a38b06621e0d863590bc5a287..08b0b8a6eebe6ddad8f1d3acb161be59d90b09ce 100644 (file)
--- a/drivers/infiniband/ulp/rtrs/rtrs-srv.h
+++ b/drivers/infiniband/ulp/rtrs/rtrs-srv.h
@@ -118,6 +118,13 @@ struct rtrs_srv_ctx {
        struct list_head srv_list;
 };
 
+struct rtrs_srv_ib_ctx {
+       struct rtrs_srv_ctx     *srv_ctx;
+       u16                     port;
+       struct mutex            ib_dev_mutex;
+       int                     ib_dev_count;
+};
+
 extern struct class *rtrs_dev_class;
 
 void close_sess(struct rtrs_srv_sess *sess);