dmaengine/idxd: Re-enable kernel workqueue under DMA API

Kernel workqueues were disabled due to flawed use of kernel VA and SVA
API. Now that we have the support for attaching PASID to the device's
default domain and the ability to reserve global PASIDs from SVA APIs,
we can re-enable the kernel work queues and use them under DMA API.

We also use non-privileged access for in-kernel DMA to be consistent
with the IOMMU settings. Consequently, interrupt for user privilege is
enabled for work completion IRQs.

Link: https://lore.kernel.org/linux-iommu/20210511194726.GP1002214@nvidia.com/
Tested-by: Tony Zhu <tony.zhu@intel.com>
Reviewed-by: Dave Jiang <dave.jiang@intel.com>
Reviewed-by: Fenghua Yu <fenghua.yu@intel.com>
Reviewed-by: Lu Baolu <baolu.lu@linux.intel.com>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Acked-by: Vinod Koul <vkoul@kernel.org>
Signed-off-by: Jacob Pan <jacob.jun.pan@linux.intel.com>
Link: https://lore.kernel.org/r/20230802212427.1497170-9-jacob.jun.pan@linux.intel.com
Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
Signed-off-by: Joerg Roedel <jroedel@suse.de>

diff --git a/drivers/dma/idxd/device.c b/drivers/dma/idxd/device.c
index 5abbcc61c5288c2251ae32816f8bd23adcb1264e..169b7ade8919f84530497aa5f75d030df32b48c9 100644 (file)
--- a/drivers/dma/idxd/device.c
+++ b/drivers/dma/idxd/device.c
@@ -299,21 +299,6 @@ void idxd_wqs_unmap_portal(struct idxd_device *idxd)
        }
 }
 
-static void __idxd_wq_set_priv_locked(struct idxd_wq *wq, int priv)
-{
-       struct idxd_device *idxd = wq->idxd;
-       union wqcfg wqcfg;
-       unsigned int offset;
-
-       offset = WQCFG_OFFSET(idxd, wq->id, WQCFG_PRIVL_IDX);
-       spin_lock(&idxd->dev_lock);
-       wqcfg.bits[WQCFG_PRIVL_IDX] = ioread32(idxd->reg_base + offset);
-       wqcfg.priv = priv;
-       wq->wqcfg->bits[WQCFG_PRIVL_IDX] = wqcfg.bits[WQCFG_PRIVL_IDX];
-       iowrite32(wqcfg.bits[WQCFG_PRIVL_IDX], idxd->reg_base + offset);
-       spin_unlock(&idxd->dev_lock);
-}
-
 static void __idxd_wq_set_pasid_locked(struct idxd_wq *wq, int pasid)
 {
        struct idxd_device *idxd = wq->idxd;
@@ -1423,15 +1408,14 @@ int drv_enable_wq(struct idxd_wq *wq)
        }
 
        /*
-        * In the event that the WQ is configurable for pasid and priv bits.
-        * For kernel wq, the driver should setup the pasid, pasid_en, and priv bit.
-        * However, for non-kernel wq, the driver should only set the pasid_en bit for
-        * shared wq. A dedicated wq that is not 'kernel' type will configure pasid and
+        * In the event that the WQ is configurable for pasid, the driver
+        * should setup the pasid, pasid_en bit. This is true for both kernel
+        * and user shared workqueues. There is no need to setup priv bit in
+        * that in-kernel DMA will also do user privileged requests.
+        * A dedicated wq that is not 'kernel' type will configure pasid and
         * pasid_en later on so there is no need to setup.
         */
        if (test_bit(IDXD_FLAG_CONFIGURABLE, &idxd->flags)) {
-               int priv = 0;
-
                if (wq_pasid_enabled(wq)) {
                        if (is_idxd_wq_kernel(wq) || wq_shared(wq)) {
                                u32 pasid = wq_dedicated(wq) ? idxd->pasid : 0;
@@ -1439,10 +1423,6 @@ int drv_enable_wq(struct idxd_wq *wq)
                                __idxd_wq_set_pasid_locked(wq, pasid);
                        }
                }
-
-               if (is_idxd_wq_kernel(wq))
-                       priv = 1;
-               __idxd_wq_set_priv_locked(wq, priv);
        }
 
        rc = 0;
@@ -1550,6 +1530,15 @@ int idxd_device_drv_probe(struct idxd_dev *idxd_dev)
        if (rc < 0)
                return -ENXIO;
 
+       /*
+        * System PASID is preserved across device disable/enable cycle, but
+        * genconfig register content gets cleared during device reset. We
+        * need to re-enable user interrupts for kernel work queue completion
+        * IRQ to function.
+        */
+       if (idxd->pasid != IOMMU_PASID_INVALID)
+               idxd_set_user_intr(idxd, 1);
+
        rc = idxd_device_evl_setup(idxd);
        if (rc < 0) {
                idxd->cmd_status = IDXD_SCMD_DEV_EVL_ERR;

diff --git a/drivers/dma/idxd/dma.c b/drivers/dma/idxd/dma.c
index eb35ca313684596831a60e662004538f6b5b25f4..07623fb0f52fc2bb160224ecf5d0dc0e625e2b00 100644 (file)
--- a/drivers/dma/idxd/dma.c
+++ b/drivers/dma/idxd/dma.c
@@ -75,9 +75,10 @@ static inline void idxd_prep_desc_common(struct idxd_wq *wq,
        hw->xfer_size = len;
        /*
         * For dedicated WQ, this field is ignored and HW will use the WQCFG.priv
-        * field instead. This field should be set to 1 for kernel descriptors.
+        * field instead. This field should be set to 0 for kernel descriptors
+        * since kernel DMA on VT-d supports "user" privilege only.
         */
-       hw->priv = 1;
+       hw->priv = 0;
        hw->completion_addr = compl;
 }
 

diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h
index 5428a2e1b1ec4c482558f4c844895c1f2cc6c03a..502be9db63f403bb5acd122b3690241ff22aa51e 100644 (file)
--- a/drivers/dma/idxd/idxd.h
+++ b/drivers/dma/idxd/idxd.h
@@ -473,6 +473,15 @@ static inline struct idxd_device *ie_to_idxd(struct idxd_irq_entry *ie)
        return container_of(ie, struct idxd_device, ie);
 }
 
+static inline void idxd_set_user_intr(struct idxd_device *idxd, bool enable)
+{
+       union gencfg_reg reg;
+
+       reg.bits = ioread32(idxd->reg_base + IDXD_GENCFG_OFFSET);
+       reg.user_int_en = enable;
+       iowrite32(reg.bits, idxd->reg_base + IDXD_GENCFG_OFFSET);
+}
+
 extern struct bus_type dsa_bus_type;
 
 extern bool support_enqcmd;

diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c
index 1aa823974cdac6c0458ea2a5029166fda2e29553..0eb1c827a215f9b5ff556dbf74652acf8fc7ed99 100644 (file)
--- a/drivers/dma/idxd/init.c
+++ b/drivers/dma/idxd/init.c
@@ -550,14 +550,59 @@ static struct idxd_device *idxd_alloc(struct pci_dev *pdev, struct idxd_driver_d
 
 static int idxd_enable_system_pasid(struct idxd_device *idxd)
 {
-       return -EOPNOTSUPP;
+       struct pci_dev *pdev = idxd->pdev;
+       struct device *dev = &pdev->dev;
+       struct iommu_domain *domain;
+       ioasid_t pasid;
+       int ret;
+
+       /*
+        * Attach a global PASID to the DMA domain so that we can use ENQCMDS
+        * to submit work on buffers mapped by DMA API.
+        */
+       domain = iommu_get_domain_for_dev(dev);
+       if (!domain)
+               return -EPERM;
+
+       pasid = iommu_alloc_global_pasid(dev);
+       if (pasid == IOMMU_PASID_INVALID)
+               return -ENOSPC;
+
+       /*
+        * DMA domain is owned by the driver, it should support all valid
+        * types such as DMA-FQ, identity, etc.
+        */
+       ret = iommu_attach_device_pasid(domain, dev, pasid);
+       if (ret) {
+               dev_err(dev, "failed to attach device pasid %d, domain type %d",
+                       pasid, domain->type);
+               iommu_free_global_pasid(pasid);
+               return ret;
+       }
+
+       /* Since we set user privilege for kernel DMA, enable completion IRQ */
+       idxd_set_user_intr(idxd, 1);
+       idxd->pasid = pasid;
+
+       return ret;
 }
 
 static void idxd_disable_system_pasid(struct idxd_device *idxd)
 {
+       struct pci_dev *pdev = idxd->pdev;
+       struct device *dev = &pdev->dev;
+       struct iommu_domain *domain;
+
+       domain = iommu_get_domain_for_dev(dev);
+       if (!domain)
+               return;
+
+       iommu_detach_device_pasid(domain, dev, idxd->pasid);
+       iommu_free_global_pasid(idxd->pasid);
 
-       iommu_sva_unbind_device(idxd->sva);
+       idxd_set_user_intr(idxd, 0);
        idxd->sva = NULL;
+       idxd->pasid = IOMMU_PASID_INVALID;
 }
 
 static int idxd_enable_sva(struct pci_dev *pdev)
@@ -600,8 +645,9 @@ static int idxd_probe(struct idxd_device *idxd)
                } else {
                        set_bit(IDXD_FLAG_USER_PASID_ENABLED, &idxd->flags);
 
-                       if (idxd_enable_system_pasid(idxd))
-                               dev_warn(dev, "No in-kernel DMA with PASID.\n");
+                       rc = idxd_enable_system_pasid(idxd);
+                       if (rc)
+                               dev_warn(dev, "No in-kernel DMA with PASID. %d\n", rc);
                        else
                                set_bit(IDXD_FLAG_PASID_ENABLED, &idxd->flags);
                }

diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c
index 293739ac55969e0a26fcf3adcc6b049583938e33..63f6966c51aa1726dea4bf465729a1b07befbb05 100644 (file)
--- a/drivers/dma/idxd/sysfs.c
+++ b/drivers/dma/idxd/sysfs.c
@@ -948,13 +948,6 @@ static ssize_t wq_name_store(struct device *dev,
        if (strlen(buf) > WQ_NAME_SIZE || strlen(buf) == 0)
                return -EINVAL;
 
-       /*
-        * This is temporarily placed here until we have SVM support for
-        * dmaengine.
-        */
-       if (wq->type == IDXD_WQT_KERNEL && device_pasid_enabled(wq->idxd))
-               return -EOPNOTSUPP;
-
        input = kstrndup(buf, count, GFP_KERNEL);
        if (!input)
                return -ENOMEM;