RDS: ActiveBonding: Make its own thread for active active

The IP related work has no relation to RDS internal workers and
the active active itself is for full HOST than just RDS. Move
all the IP specific work(s) to its own dedicated workqueue.

This will reduce queue contention and also further reduce direct
link between RDS and Active bonding and help to separate the
active active code from RDS.

Orabug: 25026643

Tested-by: Michael Nowak <michael.nowak@oracle.com>
Tested-by: Dib Chatterjee <dib.chatterjee@oracle.com>
Reviewed-by: Mukesh Kacker <mukesh.kacker@oracle.com>
Signed-off-by: Santosh Shilimkar <santosh.shilimkar@oracle.com>

diff --git a/net/rds/ib.c b/net/rds/ib.c
index 2ff8779a08a2e7abac5342642f491fd88a8194bc..798b708f956bcf2faa251930488e99ab29a05a76 100644 (file)
--- a/net/rds/ib.c
+++ b/net/rds/ib.c
@@ -119,6 +119,7 @@ static int initial_failovers_iterations; /* = 0 */
 static void rds_ib_initial_failovers(struct work_struct *workarg);
 DECLARE_DELAYED_WORK(riif_dlywork, rds_ib_initial_failovers);
 static int timeout_until_initial_failovers;
+static struct workqueue_struct *rds_ip_wq;
 
 /*
  * rds_detected_linklayer_up
@@ -1403,14 +1404,14 @@ static void rds_ib_event_handler(struct ib_event_handler *handler,
                                rds_rtd(RDS_RTD_ACT_BND,
                                        "active bonding fallback enabled\n");
                                INIT_DELAYED_WORK(&work->work, rds_ib_failback);
-                               queue_delayed_work(rds_wq, &work->work,
+                               queue_delayed_work(rds_ip_wq, &work->work,
                                        get_failback_sync_jiffies(&ip_config[port]));
                        } else
                                kfree(work);
                } else {
                        /* this_port_transition == RDSIBP_TRANSITION_DOWN */
                        INIT_DELAYED_WORK(&work->work, rds_ib_failover);
-                       queue_delayed_work(rds_wq, &work->work, 0);
+                       queue_delayed_work(rds_ip_wq, &work->work, 0);
                }
        }
 }
@@ -1507,7 +1508,7 @@ rds_ib_initial_failovers(struct work_struct *workarg)
                if (timeout_until_initial_failovers > 0) {
                        timeout_until_initial_failovers -=
                          msecs_to_jiffies(100);
-                       queue_delayed_work(rds_wq,
+                       queue_delayed_work(rds_ip_wq,
                                           &riif_dlywork,
                                           msecs_to_jiffies(100));
                        initial_failovers_iterations++;
@@ -1693,7 +1694,7 @@ sched_initial_failovers(unsigned int tot_devs,
 
        timeout_until_initial_failovers = trigger_delay_max_jiffies;
 
-       queue_delayed_work(rds_wq,
+       queue_delayed_work(rds_ip_wq,
                           &riif_dlywork,
                           trigger_delay_min_jiffies);
 }
@@ -2078,7 +2079,7 @@ static void rds_ib_unregister_client(void)
 {
        ib_unregister_client(&rds_ib_client);
        /* wait for rds_ib_dev_free() to complete */
-       flush_workqueue(rds_wq);
+       flush_workqueue(rds_ip_wq);
        flush_workqueue(rds_local_wq);
 }
 
@@ -2129,7 +2130,7 @@ static void rds_ib_joining_ip(struct work_struct *_work)
        if (in_dev && !in_dev->ifa_list && work->timeout > 0) {
                INIT_DELAYED_WORK(&work->work, rds_ib_joining_ip);
                work->timeout -= msecs_to_jiffies(100);
-               queue_delayed_work(rds_wq, &work->work, msecs_to_jiffies(100));
+               queue_delayed_work(rds_ip_wq, &work->work, msecs_to_jiffies(100));
        } else if (in_dev && in_dev->ifa_list) {
                u16 pkey = 0;
 
@@ -2247,7 +2248,7 @@ static int rds_ib_netdev_callback(struct notifier_block *self, unsigned long eve
                                work->dev = ndev;
                                work->timeout = msecs_to_jiffies(10000);
                                INIT_DELAYED_WORK(&work->work, rds_ib_joining_ip);
-                               queue_delayed_work(rds_wq, &work->work,
+                               queue_delayed_work(rds_ip_wq, &work->work,
                                                msecs_to_jiffies(100));
                        }
                }
@@ -2456,7 +2457,7 @@ static int rds_ib_netdev_callback(struct notifier_block *self, unsigned long eve
                        rds_rtd(RDS_RTD_ACT_BND,
                                "active bonding fallback enabled\n");
                        INIT_DELAYED_WORK(&work->work, rds_ib_failback);
-                       queue_delayed_work(rds_wq, &work->work,
+                       queue_delayed_work(rds_ip_wq, &work->work,
                                        get_failback_sync_jiffies(&ip_config[port]));
                        ip_config[port].port_active_ts = 0;
                } else
@@ -2466,7 +2467,7 @@ static int rds_ib_netdev_callback(struct notifier_block *self, unsigned long eve
        case RDSIBP_TRANSITION_DOWN:
                if (rds_ib_sysctl_active_bonding) {
                        INIT_DELAYED_WORK(&work->work, rds_ib_failover);
-                       queue_delayed_work(rds_wq, &work->work, 0);
+                       queue_delayed_work(rds_ip_wq, &work->work, 0);
                } else {
                        /*
                         * Note: Active bonding disabled by override
@@ -2491,6 +2492,21 @@ static struct notifier_block rds_ib_nb = {
        .notifier_call = rds_ib_netdev_callback
 };
 
+void rds_ip_thread_exit(void)
+{
+       flush_workqueue(rds_ip_wq);
+       destroy_workqueue(rds_ip_wq);
+}
+
+int rds_ip_threads_init(void)
+{
+       rds_ip_wq = create_singlethread_workqueue("krdsd_ip");
+       if (!rds_ip_wq)
+               return -ENOMEM;
+       return 0;
+}
+
+
 int rds_ib_init(void)
 {
        int ret;
@@ -2537,12 +2553,18 @@ int rds_ib_init(void)
 
        rds_info_register_func(RDS_INFO_IB_CONNECTIONS, rds_ib_ic_info);
 
+       ret = rds_ip_threads_init();
+       if (ret) {
+               printk(KERN_ERR "RDS: failed to create IP thread\n");
+               goto out_ibreg;
+       }
+
        rds_ib_ip_excl_ips_init();
 
        ret = rds_ib_ip_config_init();
        if (ret) {
                printk(KERN_ERR "RDS/IB: failed to init port\n");
-               goto out_ibreg;
+               goto out_ip_thread;
        }
 
        rds_ib_ip_failover_groups_init();
@@ -2551,6 +2573,8 @@ int rds_ib_init(void)
 
        goto out;
 
+out_ip_thread:
+       rds_ip_thread_exit();
 out_ibreg:
        rds_ib_unregister_client();
 out_recv:
@@ -2576,6 +2600,7 @@ void rds_ib_exit(void)
        destroy_workqueue(rds_aux_wq);
        rds_trans_unregister(&rds_ib_transport);
        rds_ib_fmr_exit();
+       rds_ip_thread_exit();
 
        if (ip_config)
                kfree(ip_config);