sif: During driver load, hold back events instead of ignoring them

The current semantics when a queued event is received before the driver
is done loading is to ignore it with a log warning. This was not sufficient
to implement the flush_retry_qp setup, which relies on lid change events.

Unfortunately the solution to make an exception for lid events for the
flush_retry_qp is not valid because it defeats the purpose of the
check in the first place by allowing such an event to be handled before
the data structure needed to handle it is initialized.

This commit introduces a new kernel completion that the driver completes
when the whole driver load is finished. The first EPSC event queued on the
sif work queue will now block on this completion.

This covers all the remaining cases not handled by commit
"eq: Avoid enabling interrupts on TSU EQs until the initialization is complete"
and solves the original problem that introduced the need for a fix.

Orabug: 24296729

Signed-off-by: Knut Omang <knut.omang@oracle.com>
Reviewed-by: Håkon Bugge <haakon.bugge@oracle.com>

diff --git a/drivers/infiniband/hw/sif/sif_dev.h b/drivers/infiniband/hw/sif/sif_dev.h
index fb26904d4098ae004ffdb7e59172e876131e5b30..5ef1771d7e21a967bf94a93faefeab81b7a5d6ee 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_dev.h
+++ b/drivers/infiniband/hw/sif/sif_dev.h
@@ -286,6 +286,7 @@ struct sif_dev {
        struct sif_idr pd_refs;   /* Mgmt of sif_pd allocations */
        struct sif_spqp_pool ki_spqp; /* Stencil PQPs for key invalidates */
        /* Misc settings */
+       struct completion ready_for_events; /* Set when we are ready to receive events from sif */
        bool registered;        /* Set when we are registered with the verbs layer */
        u64 min_resp_ticks;   /* expected min. hw resp.time in ticks */
 

diff --git a/drivers/infiniband/hw/sif/sif_eq.c b/drivers/infiniband/hw/sif/sif_eq.c
index 9112e43a898601cec028b2f0d89902194632f70b..1dfa00d5be9539ecd3ac2a5460de5918eac14841 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_eq.c
+++ b/drivers/infiniband/hw/sif/sif_eq.c
@@ -623,14 +623,7 @@ static void handle_event_work(struct work_struct *work)
        atomic_inc(&ew->eq->work_cnt);
 
        if (unlikely(!sdev->registered)) {
-               sif_log(sdev, SIF_INFO,
-                       "Event of type %s received before verbs framework is up - ignoring",
-                       ib_event2str(ew->ibe.event));
-
-               if ((ew->ibe.event == IB_EVENT_LID_CHANGE)
-                       && (PSIF_REVISION(sdev) <= 3))
-                       sif_r3_recreate_flush_qp(sdev, ew->ibe.element.port_num - 1);
-               goto out;
+               wait_for_completion_interruptible(&sdev->ready_for_events);
        }
 
        switch (ew->ibe.event) {
@@ -741,9 +734,8 @@ static void handle_event_work(struct work_struct *work)
                sif_log(sdev, SIF_INFO, "Unhandled event type %d", ew->ibe.event);
                break;
        }
-out:
        kfree(ew);
-       }
+}
 
 /* Generic event handler - @eqe contains little endian copy of event triggering the call
  * ib_dispatch_event dispatches directly so we have to defer the actual dispatch

diff --git a/drivers/infiniband/hw/sif/sif_ireg.c b/drivers/infiniband/hw/sif/sif_ireg.c
index b5cd96608ba568fefa3eefe3edbf92482db6bbcb..40e312c22c8f9f6cd26aa8efdcc47e93c111e813 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_ireg.c
+++ b/drivers/infiniband/hw/sif/sif_ireg.c
@@ -905,6 +905,7 @@ int sif_register_ib_device(struct sif_dev *sdev)
        sdev->ib_dev.local_dma_lkey = sdev->dma_mr->index;
 
        sdev->registered = true;
+       complete(&sdev->ready_for_events);
        sif_log(sdev, SIF_VERBS_V, "%s registered with IB", sdev->ib_dev.name);
        return 0;
 

diff --git a/drivers/infiniband/hw/sif/sif_main.c b/drivers/infiniband/hw/sif/sif_main.c
index 3ae36130bcd65c372b00d1224222f8bb8a7043f9..4c9e6e52a1d57ef53a99e1ec2ac877ceb6af0734 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_main.c
+++ b/drivers/infiniband/hw/sif/sif_main.c
@@ -223,6 +223,7 @@ static int sif_probe(struct pci_dev *pdev,
                               const struct pci_device_id *id)
 {
        int err = 0;
+       char tmp[IB_DEVICE_NAME_MAX+6];
 
        /* TBD: Zeroed memory from ib_alloc_device? */
        struct sif_dev *sdev =
@@ -239,6 +240,7 @@ static int sif_probe(struct pci_dev *pdev,
        sdev->fw_vfs = -1; /* #of VFS enabled in firmware not known yet */
        sdev->ib_dev.dma_device = &pdev->dev;
        sdev->limited_mode = sif_feature(force_limited_mode) ? true : false;
+       init_completion(&sdev->ready_for_events);
 
        strlcpy(sdev->ib_dev.name, "sif%d", IB_DEVICE_NAME_MAX);
 
@@ -255,7 +257,12 @@ static int sif_probe(struct pci_dev *pdev,
                err = -ENOMEM;
                goto wq_fail;
        }
-       sdev->misc_wq = create_singlethread_workqueue("sif_misc_wq");
+
+       /* Hold back wq event processing until everything is up */
+
+
+       sprintf(tmp, "%s_misc", sdev->ib_dev.name);
+       sdev->misc_wq = create_singlethread_workqueue(tmp);
        if (!sdev->misc_wq) {
                sif_log(sdev, SIF_INFO, "Failed to allocate sif misc work queue");
                err = -ENOMEM;