IB/hfi1: Immediately remove invalid memory from hardware

When a user expected receive page is unmapped, it should be
immediately removed from hardware rather than depend on a
reaction from user space.

Fixes: 2677a7680e77 ("IB/hfi1: Fix memory leak during unexpected shutdown")
Signed-off-by: Dean Luick <dean.luick@cornelisnetworks.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@cornelisnetworks.com>
Link: https://lore.kernel.org/r/167328548663.1472310.7871808081861622659.stgit@awfm-02.cornelisnetworks.com
Signed-off-by: Leon Romanovsky <leon@kernel.org>

diff --git a/drivers/infiniband/hw/hfi1/user_exp_rcv.c b/drivers/infiniband/hw/hfi1/user_exp_rcv.c
index 88df8ca4bb577a5c55514337cf9aa452e4c085f6..f402af1e29035958584feba0331616267d73d3a9 100644 (file)
--- a/drivers/infiniband/hw/hfi1/user_exp_rcv.c
+++ b/drivers/infiniband/hw/hfi1/user_exp_rcv.c
@@ -28,8 +28,9 @@ static int program_rcvarray(struct hfi1_filedata *fd, struct tid_user_buf *,
                            unsigned int start, u16 count,
                            u32 *tidlist, unsigned int *tididx,
                            unsigned int *pmapped);
-static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo,
-                             struct tid_group **grp);
+static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo);
+static void __clear_tid_node(struct hfi1_filedata *fd,
+                            struct tid_rb_node *node);
 static void clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node);
 
 static const struct mmu_interval_notifier_ops tid_mn_ops = {
@@ -469,7 +470,7 @@ int hfi1_user_exp_rcv_clear(struct hfi1_filedata *fd,
 
        mutex_lock(&uctxt->exp_mutex);
        for (tididx = 0; tididx < tinfo->tidcnt; tididx++) {
-               ret = unprogram_rcvarray(fd, tidinfo[tididx], NULL);
+               ret = unprogram_rcvarray(fd, tidinfo[tididx]);
                if (ret) {
                        hfi1_cdbg(TID, "Failed to unprogram rcv array %d",
                                  ret);
@@ -723,6 +724,7 @@ static int set_rcvarray_entry(struct hfi1_filedata *fd,
        }
 
        node->fdata = fd;
+       mutex_init(&node->invalidate_mutex);
        node->phys = page_to_phys(pages[0]);
        node->npages = npages;
        node->rcventry = rcventry;
@@ -762,8 +764,7 @@ out_unmap:
        return -EFAULT;
 }
 
-static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo,
-                             struct tid_group **grp)
+static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo)
 {
        struct hfi1_ctxtdata *uctxt = fd->uctxt;
        struct hfi1_devdata *dd = uctxt->dd;
@@ -786,9 +787,6 @@ static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo,
        if (!node || node->rcventry != (uctxt->expected_base + rcventry))
                return -EBADF;
 
-       if (grp)
-               *grp = node->grp;
-
        if (fd->use_mn)
                mmu_interval_notifier_remove(&node->notifier);
        cacheless_tid_rb_remove(fd, node);
@@ -796,23 +794,34 @@ static int unprogram_rcvarray(struct hfi1_filedata *fd, u32 tidinfo,
        return 0;
 }
 
-static void clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node)
+static void __clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node)
 {
        struct hfi1_ctxtdata *uctxt = fd->uctxt;
        struct hfi1_devdata *dd = uctxt->dd;
 
+       mutex_lock(&node->invalidate_mutex);
+       if (node->freed)
+               goto done;
+       node->freed = true;
+
        trace_hfi1_exp_tid_unreg(uctxt->ctxt, fd->subctxt, node->rcventry,
                                 node->npages,
                                 node->notifier.interval_tree.start, node->phys,
                                 node->dma_addr);
 
-       /*
-        * Make sure device has seen the write before we unpin the
-        * pages.
-        */
+       /* Make sure device has seen the write before pages are unpinned */
        hfi1_put_tid(dd, node->rcventry, PT_INVALID_FLUSH, 0, 0);
 
        unpin_rcv_pages(fd, NULL, node, 0, node->npages, true);
+done:
+       mutex_unlock(&node->invalidate_mutex);
+}
+
+static void clear_tid_node(struct hfi1_filedata *fd, struct tid_rb_node *node)
+{
+       struct hfi1_ctxtdata *uctxt = fd->uctxt;
+
+       __clear_tid_node(fd, node);
 
        node->grp->used--;
        node->grp->map &= ~(1 << (node->rcventry - node->grp->base));
@@ -871,10 +880,16 @@ static bool tid_rb_invalidate(struct mmu_interval_notifier *mni,
        if (node->freed)
                return true;
 
+       /* take action only if unmapping */
+       if (range->event != MMU_NOTIFY_UNMAP)
+               return true;
+
        trace_hfi1_exp_tid_inval(uctxt->ctxt, fdata->subctxt,
                                 node->notifier.interval_tree.start,
                                 node->rcventry, node->npages, node->dma_addr);
-       node->freed = true;
+
+       /* clear the hardware rcvarray entry */
+       __clear_tid_node(fdata, node);
 
        spin_lock(&fdata->invalid_lock);
        if (fdata->invalid_tid_idx < uctxt->expected_count) {

diff --git a/drivers/infiniband/hw/hfi1/user_exp_rcv.h b/drivers/infiniband/hw/hfi1/user_exp_rcv.h
index 8c53e416bf843cb662d5abab6e1ea03d11f5fc39..2ddb3dac7d916b3da997215fc00de5569a148034 100644 (file)
--- a/drivers/infiniband/hw/hfi1/user_exp_rcv.h
+++ b/drivers/infiniband/hw/hfi1/user_exp_rcv.h
@@ -27,6 +27,7 @@ struct tid_user_buf {
 struct tid_rb_node {
        struct mmu_interval_notifier notifier;
        struct hfi1_filedata *fdata;
+       struct mutex invalidate_mutex; /* covers hw removal */
        unsigned long phys;
        struct tid_group *grp;
        u32 rcventry;