Interface to mark SR-IOV device ready for use by LDoms guest

Add a iov_ready file to all PCI devices (/sys/bus/pci/devices/*/iov_ready).
The iov_ready file is write only, and mapped to the pci_iov_dev_ready
hypervisor call, which is used to indicate that a PCI device is ready
or no longer ready to be shared with other domains

Write "1" to the file to indicate that the PCI device is ready.
For example:

  # echo 1 > /sys/bus/pci/devices/0001:03:00.0/iov_ready

Write "0" to the file to indicate that the PCI device is no longer ready.
For example:

  # echo 0 > /sys/bus/pci/devices/0001:03:00.0/iov_ready

Orabug: 22909608

Signed-off-by: Alexandre Chartre <alexandre.chartre@oracle.com>
Reviewed-by: Babu Moger <babu.moger@oracle.com>
Signed-off-by: Allen Pais <allen.pais@oracle.com>
(cherry picked from commit 14f6e82264924f1db2cad628edae7964caa9c03e)

diff --git a/arch/sparc/include/asm/hypervisor.h b/arch/sparc/include/asm/hypervisor.h
index c91298bdaa6920d5008bc95eaa278a5643e4fcd7..b0851469f4eb1e5ebbbd3934eeab69f987cd4a88 100644 (file)
--- a/arch/sparc/include/asm/hypervisor.h
+++ b/arch/sparc/include/asm/hypervisor.h
@@ -2339,6 +2339,29 @@ unsigned long sun4v_vintr_set_target(unsigned long dev_handle,
  */
 #define HV_FAST_PCI_MSG_SETVALID       0xd3
 
+/* pci_iov_dev_ready()
+ * TRAP:       HV_FAST_TRAP
+ * FUNCTION:   HV_FAST_PCI_IOV_DEV_READY
+ * ARG0:       devhandle
+ * ARG1:       pci_device
+ * ARG2:       ready_flag
+ * RET0:       status
+ * ERRORS:     EINVAL          Invalid devhandle or pci_device or ready_flag
+ *             ENOACCESS       No access, the caller does not own this root
+ *                             complex or device
+ *
+ * Indicate that the PCI device pci_device in the root complex specified
+ * by devhandle is ready or no longer ready to be shared with other domains.
+ * Making an endpoint ready, makes the endpoint and all nodes between it and
+ * the root complex ready.
+ *
+ * Valid values for ready_flag:
+ *     0       Not ready
+ *     1       Ready
+ *     2       Disabled
+ */
+#define HV_FAST_PCI_IOV_DEV_READY      0xf7
+
 /* Logical Domain Channel services.  */
 
 #define LDC_CHANNEL_DOWN               0

diff --git a/arch/sparc/kernel/pci.c b/arch/sparc/kernel/pci.c
index da9eadf93f74cc915b15bdd18c6f207da88f78aa..6ba977da17847b3b191f8f698eb5e7daeca2de97 100644 (file)
--- a/arch/sparc/kernel/pci.c
+++ b/arch/sparc/kernel/pci.c
@@ -1054,6 +1054,31 @@ static int __init pcibios_init(void)
 }
 subsys_initcall(pcibios_init);
 
+void pcibios_create_sysfs_dev_files(struct pci_dev *dev)
+{
+       int err;
+       struct pci_pbm_info *pbm;
+
+       pbm = dev->bus->sysdata;
+       if (!pbm || !pbm->sysfs_dev_attr_group)
+               return;
+
+       err = sysfs_create_group(&dev->dev.kobj, pbm->sysfs_dev_attr_group);
+       if (err)
+               pr_err("Failed to create sysfs entries, err=%d\n", err);
+}
+
+void pcibios_remove_sysfs_dev_files(struct pci_dev *dev)
+{
+       struct pci_pbm_info *pbm;
+
+       pbm = dev->bus->sysdata;
+       if (!pbm || !pbm->sysfs_dev_attr_group)
+               return;
+
+       sysfs_remove_group(&dev->dev.kobj, pbm->sysfs_dev_attr_group);
+}
+
 #ifdef CONFIG_SYSFS
 
 #define SLOT_NAME_SIZE  11  /* Max decimal digits + null in u32 */

diff --git a/arch/sparc/kernel/pci_impl.h b/arch/sparc/kernel/pci_impl.h
index 2f62fd133d753bda7b4b63b1b432fe4ccc305fdb..1be607e6ffe502531272d93dd6dac75169072d96 100644 (file)
--- a/arch/sparc/kernel/pci_impl.h
+++ b/arch/sparc/kernel/pci_impl.h
@@ -149,6 +149,9 @@ struct pci_pbm_info {
        /* IOMMU state, potentially shared by both PBM segments. */
        struct iommu                    *iommu;
 
+       /* Architecture-specific sysfs device attributes. */
+       const struct attribute_group    *sysfs_dev_attr_group;
+
        /* Now things for the actual PCI bus probes. */
        unsigned int                    pci_first_busno;
        unsigned int                    pci_last_busno;

diff --git a/arch/sparc/kernel/pci_sun4v.c b/arch/sparc/kernel/pci_sun4v.c
index 1bab9543f9c1f0243ccbd514b56d794921e9cf58..eaa204a6fd2716ac746fe49e8dc689ee086f60af 100644 (file)
--- a/arch/sparc/kernel/pci_sun4v.c
+++ b/arch/sparc/kernel/pci_sun4v.c
@@ -44,6 +44,13 @@ static struct vpci_version vpci_versions[] = {
        { .major = 1, .minor = 1 },
 };
 
+/*
+ * We don't use version 1 of the SDIO/IOV hypervisor API group because
+ * it was deprecated before it was ever used on Linux.
+ */
+static unsigned long iov_major = 2;
+static unsigned long iov_minor;
+
 #define PGLIST_NENTS   (PAGE_SIZE / sizeof(u64))
 
 struct iommu_batch {
@@ -1010,6 +1017,63 @@ static void pci_sun4v_msi_init(struct pci_pbm_info *pbm)
 }
 #endif /* !(CONFIG_PCI_MSI) */
 
+static ssize_t
+pci_sun4v_iov_ready_store(struct device *dev, struct device_attribute *attr,
+                        const char *buf, size_t count)
+{
+       struct pci_pbm_info *pbm;
+       struct pci_dev *pdev;
+       unsigned int device;
+       unsigned int func;
+       unsigned long err;
+       unsigned long val;
+       unsigned int bus;
+       ssize_t result;
+
+       result = kstrtoul(buf, 0, &val);
+       if (result < 0)
+               return result;
+
+       pdev = to_pci_dev(dev);
+       bus = pdev->bus->number;
+       device = PCI_SLOT(pdev->devfn);
+       func = PCI_FUNC(pdev->devfn);
+
+       pbm = pdev->bus->sysdata;
+
+       err = pci_sun4v_iov_dev_ready(pbm->devhandle,
+                                     HV_PCI_DEVICE_BUILD(bus, device, func),
+                                     val);
+
+       switch (err) {
+
+       case HV_EOK:
+               return count;
+
+       case HV_EINVAL:
+               return -EINVAL;
+
+       case HV_ENOACCESS:
+               return -EACCES;
+
+       default:
+               return -err;
+
+       }
+}
+
+static struct device_attribute pci_sun4v_iov_ready_attr =
+               __ATTR(iov_ready, S_IWUSR, NULL, pci_sun4v_iov_ready_store);
+
+static struct attribute *pci_sun4v_dev_attrs[] = {
+       &pci_sun4v_iov_ready_attr.attr,
+       NULL,
+};
+
+static const struct attribute_group pci_sun4v_dev_attr_group = {
+       .attrs = pci_sun4v_dev_attrs,
+};
+
 static int pci_sun4v_pbm_init(struct pci_pbm_info *pbm,
                              struct platform_device *op, u32 devhandle)
 {
@@ -1029,6 +1093,8 @@ static int pci_sun4v_pbm_init(struct pci_pbm_info *pbm,
 
        pbm->name = dp->full_name;
 
+       pbm->sysfs_dev_attr_group = &pci_sun4v_dev_attr_group;
+
        printk("%s: SUN4V PCI Bus Module\n", pbm->name);
        printk("%s: On NUMA node %d\n", pbm->name, pbm->numa_node);
 
@@ -1050,6 +1116,41 @@ static int pci_sun4v_pbm_init(struct pci_pbm_info *pbm,
        return 0;
 }
 
+static int pci_sun4v_hvapi_register(void)
+{
+       int i, err = -ENODEV;
+
+       for (i = 0; i < ARRAY_SIZE(vpci_versions); i++) {
+               vpci_major = vpci_versions[i].major;
+               vpci_minor = vpci_versions[i].minor;
+
+               err = sun4v_hvapi_register(HV_GRP_PCI, vpci_major,
+                   &vpci_minor);
+               if (!err)
+                       break;
+       }
+
+       if (err) {
+               pr_err(PFX "Could not register hvapi, err=%d\n", err);
+               return err;
+       }
+       pr_info(PFX "Registered hvapi major[%lu] minor[%lu]\n",
+           vpci_major, vpci_minor);
+
+       err = sun4v_hvapi_register(HV_GRP_SDIO, iov_major, &iov_minor);
+       if (err) {
+               pr_err(PFX "Could not register IOV hvapi, err=%d\n", err);
+               /* don't return an error if we fail to register the
+                * SDIO/IOV group, but SDIO/IOV hcalls won't be available.
+                */
+       } else {
+               pr_info(PFX "Registered IOV hvapi major[%lu] minor[%lu]\n",
+                       iov_major, iov_minor);
+       }
+
+       return 0;
+}
+
 static int pci_sun4v_probe(struct platform_device *op)
 {
        const struct linux_prom64_registers *regs;
@@ -1058,27 +1159,14 @@ static int pci_sun4v_probe(struct platform_device *op)
        struct device_node *dp;
        struct iommu *iommu;
        u32 devhandle;
-       int i, err = -ENODEV;
+       int i, err;
 
        dp = op->dev.of_node;
 
        if (!hvapi_negotiated++) {
-               for (i = 0; i < ARRAY_SIZE(vpci_versions); i++) {
-                       vpci_major = vpci_versions[i].major;
-                       vpci_minor = vpci_versions[i].minor;
-
-                       err = sun4v_hvapi_register(HV_GRP_PCI, vpci_major,
-                                                  &vpci_minor);
-                       if (!err)
-                               break;
-               }
-
-               if (err) {
-                       pr_err(PFX "Could not register hvapi, err=%d\n", err);
+               err = pci_sun4v_hvapi_register();
+               if (err)
                        return err;
-               }
-               pr_info(PFX "Registered hvapi major[%lu] minor[%lu]\n",
-                       vpci_major, vpci_minor);
 
                dma_ops = &sun4v_dma_ops;
        }

diff --git a/arch/sparc/kernel/pci_sun4v.h b/arch/sparc/kernel/pci_sun4v.h
index 1ef586d36b586975ce48cdbca68f9c64874cfdfe..c92eed2609b46dabe8abe155d49cf451705a396e 100644 (file)
--- a/arch/sparc/kernel/pci_sun4v.h
+++ b/arch/sparc/kernel/pci_sun4v.h
@@ -92,5 +92,8 @@ unsigned long pci_sun4v_msg_getvalid(unsigned long devhandle,
 unsigned long pci_sun4v_msg_setvalid(unsigned long devhandle,
                                     unsigned long msinum,
                                     unsigned long valid);
+unsigned long pci_sun4v_iov_dev_ready(unsigned long devhandle,
+                                     unsigned long pci_device,
+                                     unsigned long ready_flag);
 
 #endif /* !(_PCI_SUN4V_H) */

diff --git a/arch/sparc/kernel/pci_sun4v_asm.S b/arch/sparc/kernel/pci_sun4v_asm.S
index 98d11ed76d129d4f6beea755af4e5a6c1a2c382a..80fda5fd4abfaa216054f6cec730eb379ee1297b 100644 (file)
--- a/arch/sparc/kernel/pci_sun4v_asm.S
+++ b/arch/sparc/kernel/pci_sun4v_asm.S
@@ -378,3 +378,15 @@ ENTRY(pci_sun4v_msg_setvalid)
         mov    %o0, %o0
 ENDPROC(pci_sun4v_msg_setvalid)
 
+       /* %o0: devhandle
+        * %o1: pci_device
+        * %o2: ready_flag
+        *
+        * returns %o0: status
+        */
+ENTRY(pci_sun4v_iov_dev_ready)
+       mov     HV_FAST_PCI_IOV_DEV_READY, %o5
+       ta      HV_FAST_TRAP
+       retl
+        mov    %o0, %o0
+ENDPROC(pci_sun4v_iov_dev_ready)

diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c
index 312f23a8429cd9331b45afaf72a278e84bd41d83..9bed47f0096526322979d22bfd17349f6460c665 100644 (file)
--- a/drivers/pci/pci-sysfs.c
+++ b/drivers/pci/pci-sysfs.c
@@ -1360,6 +1360,14 @@ error:
        return retval;
 }
 
+void __weak pcibios_create_sysfs_dev_files(struct pci_dev *dev)
+{
+}
+
+void __weak pcibios_remove_sysfs_dev_files(struct pci_dev *dev)
+{
+}
+
 int __must_check pci_create_sysfs_dev_files(struct pci_dev *pdev)
 {
        int retval;
@@ -1413,6 +1421,8 @@ int __must_check pci_create_sysfs_dev_files(struct pci_dev *pdev)
 
        pci_create_firmware_label_files(pdev);
 
+       pcibios_create_sysfs_dev_files(pdev);
+
        return 0;
 
 err_rom_file:
@@ -1480,6 +1490,7 @@ void pci_remove_sysfs_dev_files(struct pci_dev *pdev)
 
        pci_remove_firmware_label_files(pdev);
 
+       pcibios_remove_sysfs_dev_files(pdev);
 }
 
 static int __init pci_sysfs_init(void)

diff --git a/include/linux/pci.h b/include/linux/pci.h
index 5bce728c74d0c0a13bb151f8949effa4c6f96537..5d907122b5bd00c901f3ed618410c2b9563a1596 100644 (file)
--- a/include/linux/pci.h
+++ b/include/linux/pci.h
@@ -1663,6 +1663,8 @@ int pcibios_set_pcie_reset_state(struct pci_dev *dev,
                                 enum pcie_reset_state state);
 int pcibios_add_device(struct pci_dev *dev);
 void pcibios_release_device(struct pci_dev *dev);
+void pcibios_create_sysfs_dev_files(struct pci_dev *dev);
+void pcibios_remove_sysfs_dev_files(struct pci_dev *dev);
 void pcibios_penalize_isa_irq(int irq, int active);
 
 #ifdef CONFIG_HIBERNATE_CALLBACKS