From 0fc852d81f2e9cf2b52bac2efb0e5efcc432ffa6 Mon Sep 17 00:00:00 2001 From: Alexandre Chartre Date: Thu, 17 Mar 2016 03:31:24 -0700 Subject: [PATCH] Interface to mark SR-IOV device ready for use by LDoms guest Add a iov_ready file to all PCI devices (/sys/bus/pci/devices/*/iov_ready). The iov_ready file is write only, and mapped to the pci_iov_dev_ready hypervisor call, which is used to indicate that a PCI device is ready or no longer ready to be shared with other domains Write "1" to the file to indicate that the PCI device is ready. For example: # echo 1 > /sys/bus/pci/devices/0001:03:00.0/iov_ready Write "0" to the file to indicate that the PCI device is no longer ready. For example: # echo 0 > /sys/bus/pci/devices/0001:03:00.0/iov_ready Orabug: 22909608 Signed-off-by: Alexandre Chartre Reviewed-by: Babu Moger Signed-off-by: Allen Pais (cherry picked from commit 14f6e82264924f1db2cad628edae7964caa9c03e) --- arch/sparc/include/asm/hypervisor.h | 23 ++++++ arch/sparc/kernel/pci.c | 25 ++++++ arch/sparc/kernel/pci_impl.h | 3 + arch/sparc/kernel/pci_sun4v.c | 120 ++++++++++++++++++++++++---- arch/sparc/kernel/pci_sun4v.h | 3 + arch/sparc/kernel/pci_sun4v_asm.S | 12 +++ drivers/pci/pci-sysfs.c | 11 +++ include/linux/pci.h | 2 + 8 files changed, 183 insertions(+), 16 deletions(-) diff --git a/arch/sparc/include/asm/hypervisor.h b/arch/sparc/include/asm/hypervisor.h index c91298bdaa69..b0851469f4eb 100644 --- a/arch/sparc/include/asm/hypervisor.h +++ b/arch/sparc/include/asm/hypervisor.h @@ -2339,6 +2339,29 @@ unsigned long sun4v_vintr_set_target(unsigned long dev_handle, */ #define HV_FAST_PCI_MSG_SETVALID 0xd3 +/* pci_iov_dev_ready() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_IOV_DEV_READY + * ARG0: devhandle + * ARG1: pci_device + * ARG2: ready_flag + * RET0: status + * ERRORS: EINVAL Invalid devhandle or pci_device or ready_flag + * ENOACCESS No access, the caller does not own this root + * complex or device + * + * Indicate that the PCI device pci_device in the root complex specified + * by devhandle is ready or no longer ready to be shared with other domains. + * Making an endpoint ready, makes the endpoint and all nodes between it and + * the root complex ready. + * + * Valid values for ready_flag: + * 0 Not ready + * 1 Ready + * 2 Disabled + */ +#define HV_FAST_PCI_IOV_DEV_READY 0xf7 + /* Logical Domain Channel services. */ #define LDC_CHANNEL_DOWN 0 diff --git a/arch/sparc/kernel/pci.c b/arch/sparc/kernel/pci.c index da9eadf93f74..6ba977da1784 100644 --- a/arch/sparc/kernel/pci.c +++ b/arch/sparc/kernel/pci.c @@ -1054,6 +1054,31 @@ static int __init pcibios_init(void) } subsys_initcall(pcibios_init); +void pcibios_create_sysfs_dev_files(struct pci_dev *dev) +{ + int err; + struct pci_pbm_info *pbm; + + pbm = dev->bus->sysdata; + if (!pbm || !pbm->sysfs_dev_attr_group) + return; + + err = sysfs_create_group(&dev->dev.kobj, pbm->sysfs_dev_attr_group); + if (err) + pr_err("Failed to create sysfs entries, err=%d\n", err); +} + +void pcibios_remove_sysfs_dev_files(struct pci_dev *dev) +{ + struct pci_pbm_info *pbm; + + pbm = dev->bus->sysdata; + if (!pbm || !pbm->sysfs_dev_attr_group) + return; + + sysfs_remove_group(&dev->dev.kobj, pbm->sysfs_dev_attr_group); +} + #ifdef CONFIG_SYSFS #define SLOT_NAME_SIZE 11 /* Max decimal digits + null in u32 */ diff --git a/arch/sparc/kernel/pci_impl.h b/arch/sparc/kernel/pci_impl.h index 2f62fd133d75..1be607e6ffe5 100644 --- a/arch/sparc/kernel/pci_impl.h +++ b/arch/sparc/kernel/pci_impl.h @@ -149,6 +149,9 @@ struct pci_pbm_info { /* IOMMU state, potentially shared by both PBM segments. */ struct iommu *iommu; + /* Architecture-specific sysfs device attributes. */ + const struct attribute_group *sysfs_dev_attr_group; + /* Now things for the actual PCI bus probes. */ unsigned int pci_first_busno; unsigned int pci_last_busno; diff --git a/arch/sparc/kernel/pci_sun4v.c b/arch/sparc/kernel/pci_sun4v.c index 1bab9543f9c1..eaa204a6fd27 100644 --- a/arch/sparc/kernel/pci_sun4v.c +++ b/arch/sparc/kernel/pci_sun4v.c @@ -44,6 +44,13 @@ static struct vpci_version vpci_versions[] = { { .major = 1, .minor = 1 }, }; +/* + * We don't use version 1 of the SDIO/IOV hypervisor API group because + * it was deprecated before it was ever used on Linux. + */ +static unsigned long iov_major = 2; +static unsigned long iov_minor; + #define PGLIST_NENTS (PAGE_SIZE / sizeof(u64)) struct iommu_batch { @@ -1010,6 +1017,63 @@ static void pci_sun4v_msi_init(struct pci_pbm_info *pbm) } #endif /* !(CONFIG_PCI_MSI) */ +static ssize_t +pci_sun4v_iov_ready_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + struct pci_pbm_info *pbm; + struct pci_dev *pdev; + unsigned int device; + unsigned int func; + unsigned long err; + unsigned long val; + unsigned int bus; + ssize_t result; + + result = kstrtoul(buf, 0, &val); + if (result < 0) + return result; + + pdev = to_pci_dev(dev); + bus = pdev->bus->number; + device = PCI_SLOT(pdev->devfn); + func = PCI_FUNC(pdev->devfn); + + pbm = pdev->bus->sysdata; + + err = pci_sun4v_iov_dev_ready(pbm->devhandle, + HV_PCI_DEVICE_BUILD(bus, device, func), + val); + + switch (err) { + + case HV_EOK: + return count; + + case HV_EINVAL: + return -EINVAL; + + case HV_ENOACCESS: + return -EACCES; + + default: + return -err; + + } +} + +static struct device_attribute pci_sun4v_iov_ready_attr = + __ATTR(iov_ready, S_IWUSR, NULL, pci_sun4v_iov_ready_store); + +static struct attribute *pci_sun4v_dev_attrs[] = { + &pci_sun4v_iov_ready_attr.attr, + NULL, +}; + +static const struct attribute_group pci_sun4v_dev_attr_group = { + .attrs = pci_sun4v_dev_attrs, +}; + static int pci_sun4v_pbm_init(struct pci_pbm_info *pbm, struct platform_device *op, u32 devhandle) { @@ -1029,6 +1093,8 @@ static int pci_sun4v_pbm_init(struct pci_pbm_info *pbm, pbm->name = dp->full_name; + pbm->sysfs_dev_attr_group = &pci_sun4v_dev_attr_group; + printk("%s: SUN4V PCI Bus Module\n", pbm->name); printk("%s: On NUMA node %d\n", pbm->name, pbm->numa_node); @@ -1050,6 +1116,41 @@ static int pci_sun4v_pbm_init(struct pci_pbm_info *pbm, return 0; } +static int pci_sun4v_hvapi_register(void) +{ + int i, err = -ENODEV; + + for (i = 0; i < ARRAY_SIZE(vpci_versions); i++) { + vpci_major = vpci_versions[i].major; + vpci_minor = vpci_versions[i].minor; + + err = sun4v_hvapi_register(HV_GRP_PCI, vpci_major, + &vpci_minor); + if (!err) + break; + } + + if (err) { + pr_err(PFX "Could not register hvapi, err=%d\n", err); + return err; + } + pr_info(PFX "Registered hvapi major[%lu] minor[%lu]\n", + vpci_major, vpci_minor); + + err = sun4v_hvapi_register(HV_GRP_SDIO, iov_major, &iov_minor); + if (err) { + pr_err(PFX "Could not register IOV hvapi, err=%d\n", err); + /* don't return an error if we fail to register the + * SDIO/IOV group, but SDIO/IOV hcalls won't be available. + */ + } else { + pr_info(PFX "Registered IOV hvapi major[%lu] minor[%lu]\n", + iov_major, iov_minor); + } + + return 0; +} + static int pci_sun4v_probe(struct platform_device *op) { const struct linux_prom64_registers *regs; @@ -1058,27 +1159,14 @@ static int pci_sun4v_probe(struct platform_device *op) struct device_node *dp; struct iommu *iommu; u32 devhandle; - int i, err = -ENODEV; + int i, err; dp = op->dev.of_node; if (!hvapi_negotiated++) { - for (i = 0; i < ARRAY_SIZE(vpci_versions); i++) { - vpci_major = vpci_versions[i].major; - vpci_minor = vpci_versions[i].minor; - - err = sun4v_hvapi_register(HV_GRP_PCI, vpci_major, - &vpci_minor); - if (!err) - break; - } - - if (err) { - pr_err(PFX "Could not register hvapi, err=%d\n", err); + err = pci_sun4v_hvapi_register(); + if (err) return err; - } - pr_info(PFX "Registered hvapi major[%lu] minor[%lu]\n", - vpci_major, vpci_minor); dma_ops = &sun4v_dma_ops; } diff --git a/arch/sparc/kernel/pci_sun4v.h b/arch/sparc/kernel/pci_sun4v.h index 1ef586d36b58..c92eed2609b4 100644 --- a/arch/sparc/kernel/pci_sun4v.h +++ b/arch/sparc/kernel/pci_sun4v.h @@ -92,5 +92,8 @@ unsigned long pci_sun4v_msg_getvalid(unsigned long devhandle, unsigned long pci_sun4v_msg_setvalid(unsigned long devhandle, unsigned long msinum, unsigned long valid); +unsigned long pci_sun4v_iov_dev_ready(unsigned long devhandle, + unsigned long pci_device, + unsigned long ready_flag); #endif /* !(_PCI_SUN4V_H) */ diff --git a/arch/sparc/kernel/pci_sun4v_asm.S b/arch/sparc/kernel/pci_sun4v_asm.S index 98d11ed76d12..80fda5fd4abf 100644 --- a/arch/sparc/kernel/pci_sun4v_asm.S +++ b/arch/sparc/kernel/pci_sun4v_asm.S @@ -378,3 +378,15 @@ ENTRY(pci_sun4v_msg_setvalid) mov %o0, %o0 ENDPROC(pci_sun4v_msg_setvalid) + /* %o0: devhandle + * %o1: pci_device + * %o2: ready_flag + * + * returns %o0: status + */ +ENTRY(pci_sun4v_iov_dev_ready) + mov HV_FAST_PCI_IOV_DEV_READY, %o5 + ta HV_FAST_TRAP + retl + mov %o0, %o0 +ENDPROC(pci_sun4v_iov_dev_ready) diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c index 312f23a8429c..9bed47f00965 100644 --- a/drivers/pci/pci-sysfs.c +++ b/drivers/pci/pci-sysfs.c @@ -1360,6 +1360,14 @@ error: return retval; } +void __weak pcibios_create_sysfs_dev_files(struct pci_dev *dev) +{ +} + +void __weak pcibios_remove_sysfs_dev_files(struct pci_dev *dev) +{ +} + int __must_check pci_create_sysfs_dev_files(struct pci_dev *pdev) { int retval; @@ -1413,6 +1421,8 @@ int __must_check pci_create_sysfs_dev_files(struct pci_dev *pdev) pci_create_firmware_label_files(pdev); + pcibios_create_sysfs_dev_files(pdev); + return 0; err_rom_file: @@ -1480,6 +1490,7 @@ void pci_remove_sysfs_dev_files(struct pci_dev *pdev) pci_remove_firmware_label_files(pdev); + pcibios_remove_sysfs_dev_files(pdev); } static int __init pci_sysfs_init(void) diff --git a/include/linux/pci.h b/include/linux/pci.h index 5bce728c74d0..5d907122b5bd 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -1663,6 +1663,8 @@ int pcibios_set_pcie_reset_state(struct pci_dev *dev, enum pcie_reset_state state); int pcibios_add_device(struct pci_dev *dev); void pcibios_release_device(struct pci_dev *dev); +void pcibios_create_sysfs_dev_files(struct pci_dev *dev); +void pcibios_remove_sysfs_dev_files(struct pci_dev *dev); void pcibios_penalize_isa_irq(int irq, int active); #ifdef CONFIG_HIBERNATE_CALLBACKS -- 2.50.1