sif: pd: Implement Oracle ib_core compliance shared pd

Orabug: 24713410

shared pd is not an IBTA defined feature, but an Oracle
Linux extension. Even though PSIF can share a pd easily,
it must comply with the Oracle ib_core implementation which
requires a new pd "object" when reusing a pd (via share_pd
verbs).

Without a new pd "object", it causes a NULL pointer deference
during pd clean-up phase. Thus, this patch creates a new pd
"object" when reusing a pd, and this pd "object" is pointing
to the original pd index.

Signed-off-by: Wei Lin Guay <wei.lin.guay@oracle.com>
Reviewed-by: Håkon Bugge <haakon.bugge@oracle.com>

diff --git a/drivers/infiniband/hw/sif/sif_hwi.c b/drivers/infiniband/hw/sif/sif_hwi.c
index 55fe9b76698292184c9a77a9cec280b7e8fd0e31..560fa5466ff945034a948b319f6d8894bfd97b9b 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_hwi.c
+++ b/drivers/infiniband/hw/sif/sif_hwi.c
@@ -459,7 +459,7 @@ int sif_hw_init(struct sif_dev *sdev)
                goto pd_init_failed;
 
        /* We need a kernel protection domain for resource allocation */
-       pd = alloc_pd(sdev);
+       pd = alloc_pd(sdev, false);
        if (!pd)
                goto pd_alloc_failed;
        pd->ibpd.device = &sdev->ib_dev;
@@ -501,7 +501,7 @@ pqp_failed:
        /* Release indices for qp 0 and 1 */
        for (i = 3; i >= 0; i--)
                sif_free_qp_idx(pd, i);
-       dealloc_pd(pd);
+       dealloc_pd(pd, false);
 
 pd_alloc_failed:
        sif_deinit_pd(sdev);
@@ -529,7 +529,7 @@ void sif_hw_deinit(struct sif_dev *sdev)
                        sif_free_qp_idx(sdev->pd, i);
        }
 
-       dealloc_pd(sdev->pd);
+       dealloc_pd(sdev->pd, false);
        sif_deinit_pd(sdev);
        sif_hw_kernel_cb_fini(sdev);
        sif_base_deinit(sdev);

diff --git a/drivers/infiniband/hw/sif/sif_ireg.c b/drivers/infiniband/hw/sif/sif_ireg.c
index 32682586f13768fd473fa5f885b00aaecc427b73..b4f2725adb24b2b26c106a6b2f75c0e327c63eeb 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_ireg.c
+++ b/drivers/infiniband/hw/sif/sif_ireg.c
@@ -422,7 +422,7 @@ static struct ib_ucontext *sif_alloc_ucontext(struct ib_device *ibdev,
        if (!s_uc)
                return NULL;
 
-       s_uc->pd = alloc_pd(sdev);
+       s_uc->pd = alloc_pd(sdev, false);
        if (!s_uc->pd) {
                ret = -ENOMEM;
                goto alloc_pd_failed;
@@ -486,7 +486,7 @@ static struct ib_ucontext *sif_alloc_ucontext(struct ib_device *ibdev,
 udata_copy_failed:
        release_cb(sdev, s_uc->cb);
 alloc_cb_failed:
-       dealloc_pd(s_uc->pd);
+       dealloc_pd(s_uc->pd, false);
 alloc_pd_failed:
        kfree(s_uc);
        return ERR_PTR(ret);
@@ -502,7 +502,7 @@ static int sif_dealloc_ucontext(struct ib_ucontext *ib_uc)
 
        sif_logs(SIF_VERBS_V, pd_idx = s_uc->pd->idx);
 
-       ret = dealloc_pd(s_uc->pd);
+       ret = dealloc_pd(s_uc->pd, false);
        if (ret) {
                sif_log(sdev, SIF_INFO, "Failed (status %d) to deallocate pd %d", ret, s_uc->pd->idx);
                return ret;

diff --git a/drivers/infiniband/hw/sif/sif_pd.c b/drivers/infiniband/hw/sif/sif_pd.c
index 53d37016fddfb30c9278031333ddbc4a2fa4ead6..70465832cf6d1b8af0a6cc987dcefc62c663b276 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_pd.c
+++ b/drivers/infiniband/hw/sif/sif_pd.c
@@ -45,25 +45,26 @@ inline void cancel_cb(struct psif_cb __iomem *cb)
 }
 
 
-struct sif_pd *alloc_pd(struct sif_dev *sdev)
+struct sif_pd *alloc_pd(struct sif_dev *sdev, bool shared)
 {
        struct sif_pd *pd = kzalloc(sizeof(struct sif_pd), GFP_KERNEL);
 
        if (!pd)
                return NULL;
-
-       pd->idx = sif_idr_alloc(&sdev->pd_refs, pd, GFP_KERNEL);
+       if (!shared)
+               pd->idx = sif_idr_alloc(&sdev->pd_refs, pd, GFP_KERNEL);
        spin_lock_init(&pd->lock);
        INIT_LIST_HEAD(&pd->qp_list);
        INIT_LIST_HEAD(&pd->cq_list);
        INIT_LIST_HEAD(&pd->rq_list);
 
-       sif_log(sdev, SIF_PD, "pd idx %d", pd->idx);
+       if (!shared)
+               sif_log(sdev, SIF_PD, "pd idx %d", pd->idx);
        return pd;
 }
 
 
-int dealloc_pd(struct sif_pd *pd)
+int dealloc_pd(struct sif_pd *pd, bool shared)
 {
        struct sif_dev *sdev = to_sdev(pd->ibpd.device);
 
@@ -82,7 +83,8 @@ int dealloc_pd(struct sif_pd *pd)
                return -EBUSY;
        }
 
-       sif_idr_remove(&sdev->pd_refs, pd->idx);
+       if (!shared)
+               sif_idr_remove(&sdev->pd_refs, pd->idx);
        kfree(pd);
        return 0;
 }
@@ -98,7 +100,7 @@ struct ib_pd *sif_alloc_pd(struct ib_device *ibdev,
        struct sif_pd *pd;
        int ret;
 
-       pd = alloc_pd(sdev);
+       pd = alloc_pd(sdev, false);
        if (!pd)
                return ERR_PTR(-ENOMEM);
 
@@ -111,7 +113,7 @@ struct ib_pd *sif_alloc_pd(struct ib_device *ibdev,
                resp.cb_idx = uc->cb->idx;
                ret = ib_copy_to_udata(udata, &resp, sizeof(resp));
                if (ret) {
-                       dealloc_pd(pd);
+                       dealloc_pd(pd, false);
                        return ERR_PTR(-EFAULT);
                }
        }
@@ -120,7 +122,21 @@ struct ib_pd *sif_alloc_pd(struct ib_device *ibdev,
 
 int sif_dealloc_pd(struct ib_pd *ibpd)
 {
-       return ibpd->shpd ? 0 : dealloc_pd(to_spd(ibpd));
+       if (likely(!ibpd->shpd))
+               return dealloc_pd(to_spd(ibpd), false);
+
+       if (atomic_read(&ibpd->shpd->shared) == 1) {
+               /* This is the last PD "object" of the shared PD.
+                * Thus, this PD "object" is assigned to shpd->pd
+                * in order to let sif_remove_shpd to clean up
+                * the PD "object" and remove the sif_idr.
+                */
+               struct sif_shpd *shpd = to_sshpd(ibpd->shpd);
+
+               shpd->pd = to_spd(ibpd);
+               return 0;
+       }
+       return dealloc_pd(to_spd(ibpd), true);
 }
 
 struct ib_shpd *sif_alloc_shpd(struct ib_device *ibdev,
@@ -135,6 +151,8 @@ struct ib_shpd *sif_alloc_shpd(struct ib_device *ibdev,
        if (!shpd)
                return ERR_PTR(-ENOMEM);
 
+       sif_log(sdev, SIF_PD, "alloc shared pd idx %d", pd->idx);
+
        shpd->ibshpd.device = &sdev->ib_dev;
        shpd->pd = pd;
 
@@ -146,10 +164,19 @@ struct ib_pd *sif_share_pd(struct ib_device *ibdev,
                        struct ib_udata *udata,
                        struct ib_shpd *ibshpd)
 {
+       struct sif_dev *sdev = to_sdev(ibdev);
        struct sif_shpd *shpd = to_sshpd(ibshpd);
-       struct sif_pd *pd = shpd->pd;
+       struct sif_pd *pd;
        int ret;
 
+       pd = alloc_pd(sdev, true);
+       if (!pd)
+               return ERR_PTR(-ENOMEM);
+
+       pd->idx = shpd->pd->idx;
+
+       sif_log(sdev, SIF_PD, "shared pd idx %d", pd->idx);
+
        if (udata) {
                struct sif_ucontext *uc = to_sctx(context);
                struct sif_share_pd_resp_ext resp;
@@ -169,9 +196,12 @@ int sif_remove_shpd(struct ib_device *ibdev,
                int atinit)
 {
        struct sif_shpd *shpd = to_sshpd(ibshpd);
+       struct sif_dev *sdev = to_sdev(ibdev);
 
-       if (!atinit && shpd->pd)
-               dealloc_pd(shpd->pd);
+       if (!atinit && shpd->pd) {
+               sif_log(sdev, SIF_PD, "remove shared pd idx %d", shpd->pd->idx);
+               dealloc_pd(shpd->pd, false);
+       }
 
        kfree(ibshpd);
 

diff --git a/drivers/infiniband/hw/sif/sif_pd.h b/drivers/infiniband/hw/sif/sif_pd.h
index 222d969e8efe8878b6ff9e8d101214d78370699b..ec1c24750e567a792856d1df0774db89ee99da70 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_pd.h
+++ b/drivers/infiniband/hw/sif/sif_pd.h
@@ -49,8 +49,8 @@ void sif_cb_init(struct sif_dev *sdev);
 int sif_init_pd(struct sif_dev *sdev);
 void sif_deinit_pd(struct sif_dev *sdev);
 
-struct sif_pd *alloc_pd(struct sif_dev *sdev);
-int dealloc_pd(struct sif_pd *pd);
+struct sif_pd *alloc_pd(struct sif_dev *sdev, bool shared);
+int dealloc_pd(struct sif_pd *pd, bool shared);
 
 
 /* Per protection domain table index allocations (2nd level allocation) */

diff --git a/drivers/infiniband/hw/sif/sif_xrc.c b/drivers/infiniband/hw/sif/sif_xrc.c
index 134d714fe2ae98eec40711731d6e5dfb78fa445e..f68afe046b9bd9149c13f4d00089b8d47f743014 100644 (file)
--- a/drivers/infiniband/hw/sif/sif_xrc.c
+++ b/drivers/infiniband/hw/sif/sif_xrc.c
@@ -50,7 +50,7 @@ struct ib_xrcd *sif_alloc_xrcd(struct ib_device *device,
                goto err_idr_alloc;
        }
        xrcd->index = ret;
-       xrcd->pd = alloc_pd(sdev);
+       xrcd->pd = alloc_pd(sdev, false);
        if (!xrcd->pd) {
                ret = -ENOMEM;
                sif_log(sdev, SIF_XRC, "alloc_pd failed with %d", ret);
@@ -76,7 +76,7 @@ int sif_dealloc_xrcd(struct ib_xrcd *ib_xrcd)
 
        sif_log(sdev, SIF_XRC, "index %d", xrcd->index);
 
-       dealloc_pd(xrcd->pd);
+       dealloc_pd(xrcd->pd, false);
        sif_idr_remove(&sdev->xrcd_refs, xrcd->index);
        kfree(xrcd);
        return 0;