From 8ae5dfb24b4bbf5f2ca60de5fe92afdbb383cccc Mon Sep 17 00:00:00 2001 From: chris hyser Date: Tue, 17 May 2016 13:05:35 -0700 Subject: [PATCH] sparc64: This patch adds PRIQ support. This patch supports INT_A through INT_D interrupts as described by the Open Firmware device tree as well as MSI vectors registered by PCIe drivers. pci=nomsi may not work though frankly that makes no sense on a SPARC machine. The command line parameter priq=off reverts to prior MSIEQ interrupt mechanism. OraBug: 22748924 Signed-off-by: Bob Picco Signed-off-by: Chris Hyser (cherry picked from commit d4c668861f91dfe6f5fa1a809218a8c46dc76c9b) (cherry picked from commit c47c2d2a53856b25843e07c78f42f45a17661d2c) Signed-off-by: Allen Pais --- arch/sparc/include/asm/hypervisor.h | 650 ++++++++++++++ arch/sparc/kernel/Makefile | 2 + arch/sparc/kernel/hvapi.c | 2 + arch/sparc/kernel/irq_64.c | 4 + arch/sparc/kernel/machine_kexec.c | 12 + arch/sparc/kernel/pci_impl.h | 8 + arch/sparc/kernel/pci_msi.c | 257 +++--- arch/sparc/kernel/pci_sun4v.c | 4 +- arch/sparc/kernel/priq_sun4v.c | 1300 +++++++++++++++++++++++++++ arch/sparc/kernel/priq_sun4v.h | 101 +++ arch/sparc/kernel/priq_sun4v_asm.S | 463 ++++++++++ arch/sparc/kernel/prom_irqtrans.c | 8 +- arch/sparc/kernel/smp_64.c | 3 + 13 files changed, 2696 insertions(+), 118 deletions(-) create mode 100644 arch/sparc/kernel/priq_sun4v.c create mode 100644 arch/sparc/kernel/priq_sun4v.h create mode 100644 arch/sparc/kernel/priq_sun4v_asm.S diff --git a/arch/sparc/include/asm/hypervisor.h b/arch/sparc/include/asm/hypervisor.h index b0851469f4eb..ce3626204398 100644 --- a/arch/sparc/include/asm/hypervisor.h +++ b/arch/sparc/include/asm/hypervisor.h @@ -76,6 +76,8 @@ #define HV_ECHANNEL 16 /* Invalid LDC channel */ #define HV_EBUSY 17 /* Resource busy */ +#define HV_EUNBOUND 19 /* Resource is unbound */ + /* mach_exit() * TRAP: HV_FAST_TRAP * FUNCTION: HV_FAST_MACH_EXIT @@ -3009,6 +3011,652 @@ unsigned long sun4v_m7_set_perfreg(unsigned long reg_num, unsigned long reg_val); #endif +/* priq_unconf() + * TRAP: HV_FAST_TRAP + * FUNCTION: PRIQ_UNCONF + * ARG0: id + * RET0: status + * ERRORS: EINVAL Invalid id. + * EBUSY: EBUSY Interrupt sources are bound to the PRIQ + * or the PRIQ is bound to a CPU. + * + * This service unconfigures a previously configured Priority Interrupt + * Queue (PRIQ). The service will fail if any interrupt sources are bound + * to the PRIQ or if the PRIQ is bound to a CPU. + */ +#define HV_FAST_PRIQ_UNCONF 0x1d2 + +/* priq_bind() + * TRAP: HV_FAST_TRAP + * FUNCTION: PRIQ_BIND + * ARG0: id + * ARG1: cpuid + * ARG2: pil + * RET0: status + * ERRORS: EINVAL Invalid id or pil value. + * ENOCPU: ENOCPU Invalid cpuid value. + * + * This service binds a previously configured Priority Interrupt + * Queue (PRIQ) specified by id to the CPU specified by cpuid and interrupt + * priority (softint level) specified by pil. + * + * If the PRIQ is not empty, the appropriate level interrupt will be triggered + * upon successful completion of this service. + */ +#define HV_FAST_PRIQ_BIND 0x1d3 + +/* priq_bind_info() + * TRAP: HV_FAST_TRAP + * FUNCTION: PRIQ_BIND_INFO + * ARG0: id + * RET0: status + * RET1: cpuid + * RET2: pil + * ERRORS: EINVAL Invalid id. + * EUNBOUND The PRIQ is unbound. + * + * This service queries the previously configured Priority Interrupt + * Queue (PRIQ) for its binding information. The PRIQ must be configured + * and bound. + */ +#define HV_FAST_PRIQ_BIND_INFO 0x1d4 + +/* priq_unbind() + * TRAP: HV_FAST_TRAP + * FUNCTION: PRIQ_UNBIND + * ARG0: id + * RET0: status + * ERRORS: EINVAL Invalid id. + * EUNBOUND The PRIQ is unbound. + * + * This service unbinds the the previously configured and bound Priority + * Interrupt Queue (PRIQ) from its CPU. + */ +#define HV_FAST_PRIQ_UNBIND 0x1d5 + +/* priq_get_head_tail() + * TRAP: HV_FAST_TRAP + * FUNCTION: PRIQ_GET_HEAD_TAIL + * ARG0: id + * RET0: status + * RET1: head + * REG2: tail + * ERRORS: EINVAL Invalid id value. + * + * This service returns the head and tail offsets associated with the + * previously configured Priority Interrupt Queue (PRIQ). + */ +#define HV_FAST_PRIQ_GET_HEAD_TAIL 0x1d6 + +/* priq_set_head() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_PRIQ_SET_HEAD + * ARG0: id + * ARG1: head + * RET0: status + * ERRORS: EINVAL Invalid id or head offset value. + * + * This service sets the head offset of the the previously configured Priority + * Interrupt Queue (PRIQ) to the value specified by head. + * + * If the PRIQ is not empty after the update, the appropriate level interrupt + * will be triggered upon successful completion of this service. + */ +#define HV_FAST_PRIQ_SET_HEAD 0x1d7 + +/* priq_status() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_PRIQ_STATUS + * ARG0: id + * RET0: status + * RET1: priq_status + * ERRORS: EINVAL Invalid id value + * + * This service returns the status of the the previously configured Priority + * Interrupt Queue (PRIQ). + */ +#define HV_FAST_PRIQ_STATUS 0x1d8 +#define PRIQ_STATUS_OK 0x0 +#define PRIQ_STATUS_OVERFLOW 0x1 /* XXX check this */ + +/* priq_conf() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PRIQ_CONF + * ARG0: type + * ARG1: arg (cpuid or devhandle) + * ARG2: raddr + * ARG3: nentries + * RET0: status + * RET1: id + * ERRORS: EINVAL Invalid nentries, type, or arg + * ENORADDR Invalid raddr + * EBADALIGN Improperly aligned raddr + * ETOOMANY No PRIQs available of the specified type + * + * This service is used to configure a Priority Interrupt Queue (PRIQ). The + * queue will be configured to start at real address raddr and contain nentries + * 64-byte records. The queue must be size aligned. + * + * Upon successful completion, the queue will be configured, its head and tail + * offsets will be initialized to equal values to indicate an empty queue, the + * PRIQ status will be set to PRIQ_STATUS_OK, and a PRIQ identifier id will + * be returned. + * + * The type parameter specifies the type of PRIQ. The arg parameter depends on + * the type of the queue. + * PRIQ_CPU - A PRIQ associated with a CPU. The argument must be a valid cpu ID. + * PRIQ_PCI - A PRIQ associated with a PCI root complex. The argument must be + * a valid devhandle of a PCI root complex. + * + * The PRIQ configured is associated with the resource indicated by the type + * and arg parameters and must be unconfigured if that resource is removed + * from the guest. + * + * XXX NOTE + * Since last examining HV source PRIQ_CPU is defined but not implemented. + */ +#define HV_FAST_PRIQ_CONF 0x1d9 +#define PRIQ_CPU 0x01 +#define PRIQ_PCI 0x02 + +/* priq_info() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PRIQ_INFO + * ARG0: id + * RET0: status + * RET1: type + * RET2: arg + * RET3: raddr + * RET4: nentries + * ERRORS: EINVAL Invalid id. + * + * This service is used to query the configuration of a previously configured + * Priority Interrupt Queue (PRIQ). + */ +#define HV_FAST_PRIQ_INFO 0x1da + +#ifndef __ASSEMBLY__ +struct hv_priq_info { + unsigned long type; + unsigned long arg; + unsigned long raddr; + unsigned long nentries; +}; +#endif /* !__ASSEMBLY__ */ +#define HV_PRIQ_INFO_TYPE_OFFSET 0x00 +#define HV_PRIQ_INFO_ARG_OFFSET 0x08 +#define HV_PRIQ_INFO_RADDR_OFFSET 0x10 +#define HV_PRIQ_INFO_NENTRIES_OFFSET 0x18 + +/* pci_priq_err_bind() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_ERR_BIND + * ARG0: devhandle + * ARG1: errtype + * ARG2: priq_id + * RET0: status + * ERRORS: EINVAL Invalid devhandle, errtype, or priq_id value + * + * This service binds error reporting for the error source specified by the + * argument errtype for the root complex defined by the argument devhandle to + * the Priority Interrupt Queue (PRIQ) specified by priq_id. + * + * The error reporting is enabled separately, see Section 24.5.23, + * "pci_priq_err_enable" . + */ +#define HV_FAST_PCI_PRIQ_ERR_BIND 0x1db + +/* pci_priq_err_info() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_ERR_INFO + * ARG0: devhandle + * ARG1: errtype + * RET0: status + * RET1: priq_id + * RET2: enabled + * ERRORS: EINVAL Invalid devhandle or errtype value. + * + * This service queries the PRIQ binding and enabled state of errors specified + * by the argument errtype for the root complex defined by the argument + * devhandle. + * + * If successful, priq_id indicates the PRIQ to which the errors are bound and + * enabled is non-zero if they are enabled. + */ +#define HV_FAST_PCI_PRIQ_ERR_INFO 0x1dc + +#ifndef __ASSEMBLY__ +struct hv_pci_priq_err_info { + unsigned long priq_id; + unsigned long enabled; +}; +#endif /* !__ASSEMBLY__ */ +#define HV_PCI_PRIQ_ERR_INFO_PRIQ_ID_OFFSET 0x00 +#define HV_PCI_PRIQ_ERR_INFO_ENABLED_OFFSET 0x08 + +/* pci_priq_err_unbind() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_UNBIND + * ARG0: devhandle + * ARG1: errtype + * RET0: status + * ERRORS: EINVAL Invalid devhandle, errtype value. + * EUNBOUND The errors are not bound to a PRIQ. + * + * This service unbinds errors specified the argument errtype for the root + * complex defined by the argument devhandle from a PRIQ. + * + * The errors are disabled as a result of this service. + */ +#define HV_FAST_PCI_PRIQ_ERR_UNBIND 0x1dd + +/* pci_priq_err_enable() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_ERR_ENABLE + * ARG0: devhandle + * ARG1: errtype + * RET0: status + * ERRORS: EINVAL Invalid devhandle or errtype value. + * EUNBOUND The errors are not bound to a PRIQ. + * + * This service enables errors specified by the argument errtype in the root + * complex specified by devhandle. + */ +#define HV_FAST_PCI_PRIQ_ERR_ENABLE 0x1de + +/* pci_priq_err_disable() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_ERR_DISABLE + * ARG0: devhandle + * ARG1: errtype + * RET0: status + * ERRORS: EINVAL Invalid devhandle or errtype value. + * + * This service disables the errors specified by the argument errtype in the + * root complex specified by devhandle. + */ +#define HV_FAST_PCI_PRIQ_ERR_DISABLE 0x1df + +/* pci_priq_msi_bind() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_MSI_BIND + * ARG0: devhandle + * ARG1: msinum + * ARG2: rid + * ARG3: priq_id + * RET0: status + * ERRORS: EINVAL Invalid devhandle, msinum, rid, or priq_id. + * + * This service binds MSI defined by the arguments devhandle, msinum, and rid + * to the Priority Interrupt Queue (PRIQ) specified by priq_id. + * + * The argument rid is the PCI Requester ID of the PCI function that will + * generate the MSI. + * + * The MSI is enabled separately, see Section 24.5.4, "pci_priq_msi_enable" + * + * If the endpoint is a PCI/X endpoint, the rid value must be the RID as seen + * in the root complex when the MSI is received. That value may not be the + * same as the RID of the endpoint device. + */ +#define HV_FAST_PCI_PRIQ_MSI_BIND 0x1e0 + +/* pci_priq_msi_info() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_MSI_INFO + * ARG0: devhandle + * ARG1: msinum + * ARG2: rid + * RET0: status + * RET1: priq_id + * RET2: enabled + * ERRORS: EINVAL Invalid id. + * + * This service queries the PRIQ binding and enabled state of the MSI specified + * by the arguments devhandle, msinum, and rid. + * + * If successful, priq_id indicates the PRIQ to which the MSI is bound and + * enabled is non-zero if the MSI is enabled. + * + * Implementation Note + * The rid value may not be validated depending on the implementation. + */ +#define HV_FAST_PCI_PRIQ_MSI_INFO 0x1e1 + +#ifndef __ASSEMBLY__ +struct hv_pci_msi_info { + unsigned long priq_id; + unsigned long enabled; +}; +#endif /* !__ASSEMBLY__ */ +#define HV_PCI_MSI_INFO_PRIQ_ID_OFFSET 0x00 +#define HV_PCI_MSI_INFO_PRIQ_ENABLE_OFFSET 0x08 + +/* pci_priq_msi_unbind() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_MSI_UNBIND + * ARG0: devhandle + * ARG1: msinum + * ARG2: rid + * RET0: status + * ERRORS: EINVAL Invalid devhandle, msinum, or rid. + * EUNBOUND The MSI is not bound to a PRIQ. + * + * This service unbinds the MSI specified by the arguments devhandle, msinum, + * and rid from a PRIQ. + * + * The MSI is disabled as a result of this service. + * + * Implementation Note + * The rid value may not be validated depending on the implementation. + */ +#define HV_FAST_PCI_PRIQ_MSI_UNBIND 0x1e2 + +/* pci_priq_msi_enable() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_MSI_ENABLE + * ARG0: devhandle + * ARG1: msinum + * ARG2: rid + * RET0: status + * ERRORS: EINVAL Invalid devhandle, msinum, or rid. + * EUNBOUND The MSI is not bound to a PRIQ. + * + * This service enables the MSI specified by the arguments devhandle, msinum, + * and rid to generate PRIQ interrupts. + * + * MSIs are implemented as memory writes to addresses designated by software + * and firmware. Enabling an MSI makes the address valid for the MSI when + * accessed by the specified RID. Depending on the implementation, generation + * of the MSI when it has not been enabled may cause errors to be reported by + * the hardware. For example, DMA errors or I/O address translation errors may + * be reported. + * + * Upon success, the MSI state for the MSI is set to PCI_MSI_DELIVERED prior + * to enabling the MSI. + * + * Implementation Note + * The rid value may not be validated depending on the implementation. + */ +#define HV_FAST_PCI_PRIQ_MSI_ENABLE 0x1e3 + +/* pci_priq_msi_disable() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_MSI_DISABLE + * ARG0: devhandle + * ARG1: msinum + * ARG2: rid + * RET0: status + * ERRORS: EINVAL Invalid devhandle, msinum, or rid. + * EUNBOUND The MSI is not bound to a PRIQ. + * + * This service disables the MSI specified by the arguments devhandle, msinum, + * and rid. + * + * Programming Note + * Depending on the system implementation, an error may occur if an MSI is + * received when it is disabled. Setting the MSI state to MSI_STATE_DELIVERED + * to prevent further interrupts may be preferable. + * + * Implementation Note + * The rid value may not be validated depending on the implementation. + */ +#define HV_FAST_PCI_PRIQ_MSI_DISABLE 0x1e4 + +/* pci_priq_msi_getstate() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_MSI_GETSTATE + * ARG0: devhandle + * ARG1: msinum + * ARG2: rid + * RET0: status + * RET1: msistate + * ERRORS: EINVAL Invalid devhandle, msinum, or rid. + * EUNBOUND The MSI is not bound to a PRIQ. + * + * This service queries the current MSI State (msistate) for the MSI + * specified by the arguments devhandle, msinum, and rid. + * + * Implementation Note + * The rid value may not be validated depending on the implementation. + */ +#define HV_FAST_PCI_PRIQ_MSI_GETSTATE 0x1e5 + +/* pci_priq_msi_setstate() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_MSI_SETSTATE + * ARG0: devhandle + * ARG1: msinum + * ARG2: rid + * ARG3: msistate + * RET0: status + * ERRORS: EINVAL Invalid devhandle, msinum, or rid. + * EUNBOUND The MSI is not bound to a PRIQ. + * + * This service sets the current MSI State (msistate) for the MSI specified + * by the arguments devhandle, msinum, and rid. + * + * Implementation Note + * The rid value may not be validated depending on the implementation. + */ +#define HV_FAST_PCI_PRIQ_MSI_SETSTATE 0x1e6 + +/* pci_priq_msg_bind() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_MSG_BIND + * ARG0: devhandle + * ARG1: msgtype + * ARG2: priq_id + * RET0: status + * ERRORS: EINVAL Invalid devhandle, msgtype, or priq_id. + * + * This service binds PCIe messages of type msgtype for the root complex + * defined by the argument devhandle to the Priority Interrupt Queue (PRIQ) + * specified by priq_id. + * + */ +#define HV_FAST_PCI_PRIQ_MSG_BIND 0x1f0 + +/* pci_priq_msg_info() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_MSG_INFO + * ARG0: devhandle + * ARG1: msgtype + * RET0: status + * RET1: priq_id + * RET2: enabled + * ERRORS: EINVAL Invalid devhandle or msgtype + * EUNBOUND The message type is not bound to a PRIQ. + * + * This service queries the PRIQ binding and enabled state of PCIe messages + * specified by the argument msgtype for the root complex defined by the + * argument devhandle. + * + * If successful, priq_id indicates the PRIQ to which the messages are bound + * and enabled is non-zero if the messages are enabled. + */ +#define HV_FAST_PCI_PRIQ_MSG_INFO 0x1f1 + +#ifndef __ASSEMBLY__ +struct hv_pci_priq_msg_info { + unsigned long priq_id; + unsigned long enabled; +}; +#endif /* !__ASSEMBLY__ */ +#define HV_PCI_PRIQ_MSG_INFO_PRIQ_ID_OFFSET 0x00 +#define HV_PCI_PRIQ_MSG_INFO_ENABLED_OFFSET 0x08 + +/* pci_priq_msg_unbind() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_MSG_UNBIND + * ARG0: devhandle + * ARG1: msgtype + * RET0: status + * ERRORS: EINVAL Invalid devhandle, msgtype. + * EUNBOUND The message type is not bound to a PRIQ. + * + * This service unbinds PCIe messages specified the argument msgtype for the + * root complex defined by he argument devhandle from a PRIQ. + * + * The messages are disabled as a result of this service. + */ +#define HV_FAST_PCI_PRIQ_MSG_UNBIND 0x1f2 + +/* pci_priq_msg_enable() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_MSG_ENABLE + * ARG0: devhandle + * ARG1: msgtype + * RET0: status + * ERRORS: EINVAL Invalid devhandle or msgtype. + * EUNBOUND The message type is not bound to a PRIQ. + * + * This service enables the PCIe messages specified by the argument msgtype + * in the root complex specified by devhandle. + */ +#define HV_FAST_PCI_PRIQ_MSG_ENABLE 0x1f3 + +/* pci_priq_msg_disable() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_MSG_DISABLE + * ARG0: devhandle + * ARG1: msgtype + * RET0: status + * ERRORS: EINVAL Invalid devhandle or msgtype. + * + * This service disables the PCIe messages specified by the argument msgtype + * in the root complex specified by devhandle. + */ +#define HV_FAST_PCI_PRIQ_MSG_DISABLE 0x1f4 + +#define HV_PCI_INTX_CLEAR 0x00UL +#define HV_PCI_INTX_SET 0x01UL + +/* pci_priq_intx_bind() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_INTX_BIND + * ARG0: devhandle + * ARG1: intx + * ARG2: priq_id + * RET0: status + * ERRORS: EINVAL Invalid devhandle, intx, or priq_id. + * + * This service binds INTx interrupts specified by intx for the root complex + * defined by the argument devhandle to the Priority Interrupt Queue (PRIQ) + * specified by priq_id. + * + * The INTx interrupt is enabled separately, see Section 24.5.16, + * “pci_priq_intx_enable”. + */ +#define HV_FAST_PCI_PRIQ_INTX_BIND 0x1f5 + +/* pci_priq_intx_info() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_INTX_INFO + * ARG0: devhandle + * ARG1: intx + * RET0: status + * RET1: priq_id + * RET2: enabled + * ERRORS: EINVAL Invalid devhandle or intx value. + * EUNBOUND The INTx interrupts are not bound to a PRIQ. + * + * This service queries the PRIQ binding and enabled state of INTx interrupts + * specified by the argument intx for the root complex defined by the argument + * devhandle. + * + * If successful, priq_id indicates the PRIQ to which the INTx interrupts are + * bound and enabled is non-zero if they are enabled. + */ +#define HV_FAST_PCI_PRIQ_INTX_INFO 0x1f6 + +#ifndef __ASSEMBLY__ +struct hv_pci_priq_intx_info { + unsigned long priq_id; + unsigned long enabled; +}; +#endif /* !__ASSEMBLY__ */ +#define HV_PCI_PRIQ_INTX_INFO_PRIQ_ID_OFFSET 0x00 +#define HV_PCI_PRIQ_INTX_INFO_ENABLED_OFFSET 0x08 + +/* pci_priq_intx_unbind() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_INTX_UNBIND + * ARG0: devhandle + * ARG1: intx + * RET0: status + * ERRORS: EINVAL Invalid devhandle or intx value. + * EUNBOUND The INTx interrupts are not bound to a PRIQ. + * + * This service unbinds INTx interrupts specified the argument intx for the + * root complex defined by the argument devhandle from a PRIQ. + * + * The INTx interrupts are disabled as a result of this service. + */ +#define HV_FAST_PCI_PRIQ_INTX_UNBIND 0x1f7 + +/* pci_priq_intx_enable() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_INTX_ENABLE + * ARG0: devhandle + * ARG1: intx + * RET0: status + * ERRORS: EINVAL Invalid devhandle or intx value. + * EUNBOUND The INTx interrupts are not bound to a PRIQ. + * + * This service enables the INTx interrupts specified by the argument intx in + * the root complex specified by devhandle. + */ +#define HV_FAST_PCI_PRIQ_INTX_ENABLE 0x1f8 + +/* pci_priq_intx_disable() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_INTX_DISABLE + * ARG0: devhandle + * ARG1: intx + * RET0: status + * ERRORS: EINVAL Invalid devhandle or intx value. + * + * This service disables the INTx interrupts specified by the argument intx + * in the root complex specified by devhandle. + */ +#define HV_FAST_PCI_PRIQ_INTX_DISABLE 0x1f9 + +/* pci_priq_intx_getstate() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_INTX_GETSTATE + * ARG0: devhandle + * ARG1: intx + * RET0: status + * RET1: intxstate + * ERRORS: EINVAL Invalid devhandle or intx + * EUNBOUND The INTx interrupts are not bound to a PRIQ. + * + * This service queries the current INTx State (msistate) for the INTx + * interrupts specified by the argument intx in the root complex specified + * by the argument devhandle. + */ +#define HV_FAST_PCI_PRIQ_INTX_GETSTATE 0x1fa + +/* pci_priq_intx_setstate() + * TRAP: HV_FAST_TRAP + * FUNCTION: HV_FAST_PCI_PRIQ_INTX_SETSTATE + * ARG0: devhandle + * ARG1: intx + * ARG2: intxstate + * RET0: status + * ERRORS: EINVAL Invalid devhandle, intx, intxstate. + * EUNBOUND The INTx interrupts are not bound to a PRIQ. + * + * This service sets the current INTx State (intxstate) for the INTx + * interrupts specified by the argument intx in the root complex specified + * by the argument devhandle. + * + * The INTx state must be cleared after an INTx interrupt is delivered in + * order to receive the next INTx interrupt of the same intx value. + */ +#define HV_FAST_PCI_PRIQ_INTX_SETSTATE 0x1fb + /* Function numbers for HV_CORE_TRAP. */ #define HV_CORE_SET_VER 0x00 #define HV_CORE_PUTCHAR 0x01 @@ -3034,6 +3682,8 @@ unsigned long sun4v_m7_set_perfreg(unsigned long reg_num, #define HV_GRP_SDIO_ERR 0x0109 #define HV_GRP_REBOOT_DATA 0x0110 #define HV_GRP_M7_PERF 0x0114 +#define HV_GRP_PRIQ 0x011b +#define HV_GRP_PRIQ_PCI 0x011c #define HV_GRP_NIAG_PERF 0x0200 #define HV_GRP_FIRE_PERF 0x0201 #define HV_GRP_N2_CPU 0x0202 diff --git a/arch/sparc/kernel/Makefile b/arch/sparc/kernel/Makefile index d096a33ee73d..c2dc38c94557 100644 --- a/arch/sparc/kernel/Makefile +++ b/arch/sparc/kernel/Makefile @@ -70,6 +70,8 @@ obj-$(CONFIG_SPARC64) += mdesc.o obj-$(CONFIG_SPARC64) += pcr.o obj-$(CONFIG_SPARC64) += nmi.o obj-$(CONFIG_SPARC64_SMP) += cpumap.o +obj-$(CONFIG_SPARC64) += priq_sun4v_asm.o +obj-$(CONFIG_SPARC64) += priq_sun4v.o obj-y += dma.o diff --git a/arch/sparc/kernel/hvapi.c b/arch/sparc/kernel/hvapi.c index 2fc2b48c2ad9..18e59ecca0f2 100644 --- a/arch/sparc/kernel/hvapi.c +++ b/arch/sparc/kernel/hvapi.c @@ -39,6 +39,8 @@ static struct api_info api_table[] = { { .group = HV_GRP_SDIO, }, { .group = HV_GRP_SDIO_ERR, }, { .group = HV_GRP_REBOOT_DATA, }, + { .group = HV_GRP_PRIQ, }, + { .group = HV_GRP_PRIQ_PCI, }, { .group = HV_GRP_NIAG_PERF, .flags = FLAG_PRE_API }, { .group = HV_GRP_FIRE_PERF, }, { .group = HV_GRP_N2_CPU, }, diff --git a/arch/sparc/kernel/irq_64.c b/arch/sparc/kernel/irq_64.c index 32a79a042679..7c15386e9201 100644 --- a/arch/sparc/kernel/irq_64.c +++ b/arch/sparc/kernel/irq_64.c @@ -47,6 +47,7 @@ #include "cpumap.h" #include "kstack.h" #include "pci_impl.h" +#include "priq_sun4v.h" struct ino_bucket *ivector_table; unsigned long ivector_table_pa; @@ -892,6 +893,8 @@ void __irq_entry handler_irq(int pil, struct pt_regs *regs) orig_sp = set_hardirq_stack(); + cpu_handle_priqs(); + while (bucket_pa) { unsigned long next_pa; unsigned int irq; @@ -1248,6 +1251,7 @@ void __init init_IRQ(void) irq_ivector_init(); map_prom_timers(); kill_prom_timer(); + sun4v_priq(); if (tlb_type == hypervisor) sun4v_init_mondo_queues(); diff --git a/arch/sparc/kernel/machine_kexec.c b/arch/sparc/kernel/machine_kexec.c index 20cca8e5a6d8..17d5e587ce2d 100644 --- a/arch/sparc/kernel/machine_kexec.c +++ b/arch/sparc/kernel/machine_kexec.c @@ -9,6 +9,7 @@ #include #include #include "kernel.h" +#include "priq_sun4v.h" static atomic_t kexec_strand_wait; static const unsigned int tte_order = 9U; @@ -132,6 +133,13 @@ out: return rc; } +static void priq_tear_down(void) +{ + int cpu = smp_processor_id(); + + priq_percpu_destroy(cpu); +} + static void mondo_tear_down(void) { int cpu = smp_processor_id(); @@ -168,6 +176,7 @@ static void machine_capture_other_strands(void *ignore) __asm__ __volatile__("flushw"); if (atomic_read(&nmi_active) > 0) stop_nmi_watchdog(NULL); + priq_tear_down(); mondo_tear_down(); atomic_dec(&kexec_strand_wait); } @@ -200,6 +209,7 @@ void machine_crash_shutdown(struct pt_regs *regs) local_irq_disable(); stop_strands(); machine_kexec_disable_irq(); + priq_percpu_destroy(smp_processor_id()); destroy_nucleus_tsb(); crash_save_cpu(regs, smp_processor_id()); } @@ -212,6 +222,7 @@ static void machine_strand_mondo_rip(void *ignore) { if (atomic_read(&nmi_active) > 0) stop_nmi_watchdog(NULL); + priq_tear_down(); mondo_tear_down(); atomic_dec(&kexec_strand_wait); } @@ -406,6 +417,7 @@ void machine_kexec(struct kimage *image) void *kexec_startp = &kexec_start; unsigned long distance = (unsigned long) (kexec_startp - trap_table); + priq_tear_down(); mondo_tear_down(); if (image == kexec_image) machine_shim_load_kexec(image); diff --git a/arch/sparc/kernel/pci_impl.h b/arch/sparc/kernel/pci_impl.h index a536e4248165..12d2cd36fc92 100644 --- a/arch/sparc/kernel/pci_impl.h +++ b/arch/sparc/kernel/pci_impl.h @@ -50,6 +50,8 @@ struct sparc64_msiq_ops { void sparc64_pbm_msi_init(struct pci_pbm_info *pbm, const struct sparc64_msiq_ops *ops); +void sparc64_pbm_free_msi(struct pci_pbm_info *pbm, int msi_num); +int sparc64_pbm_alloc_msi(struct pci_pbm_info *pbm); struct sparc64_msiq_cookie { struct pci_pbm_info *pbm; @@ -172,12 +174,18 @@ extern struct pci_pbm_info *pci_pbm_root; extern int pci_num_pbms; +static inline struct pci_pbm_info *get_pbm(struct pci_dev *pdev) +{ + return pdev->dev.archdata.host_controller; +} + /* PCI bus scanning and fixup support. */ void pci_get_pbm_props(struct pci_pbm_info *pbm); struct pci_bus *pci_scan_one_pbm(struct pci_pbm_info *pbm, struct device *parent); void pci_determine_mem_io_space(struct pci_pbm_info *pbm); void pci_register_legacy_regions(struct pci_pbm_info *pbm); +void pci_priq_msi_init(struct pci_pbm_info *pbm); /* Error reporting support. */ void pci_scan_for_target_abort(struct pci_pbm_info *, struct pci_bus *); diff --git a/arch/sparc/kernel/pci_msi.c b/arch/sparc/kernel/pci_msi.c index a5f389155e4b..cabce69097a5 100644 --- a/arch/sparc/kernel/pci_msi.c +++ b/arch/sparc/kernel/pci_msi.c @@ -92,7 +92,7 @@ static u32 pick_msiq(struct pci_pbm_info *pbm) } -static int alloc_msi(struct pci_pbm_info *pbm) +int sparc64_pbm_alloc_msi(struct pci_pbm_info *pbm) { int i; @@ -104,7 +104,7 @@ static int alloc_msi(struct pci_pbm_info *pbm) return -ENOENT; } -static void free_msi(struct pci_pbm_info *pbm, int msi_num) +void sparc64_pbm_free_msi(struct pci_pbm_info *pbm, int msi_num) { msi_num -= pbm->msi_first; clear_bit(msi_num, pbm->msi_bitmap); @@ -204,7 +204,7 @@ static int sparc64_setup_msi_irq(unsigned int *irq_p, irq_set_chip_and_handler_name(*irq_p, &msi_irq, handle_simple_irq, "MSI"); - err = alloc_msi(pbm); + err = sparc64_pbm_alloc_msi(pbm); if (unlikely(err < 0)) goto out_irq_free; @@ -236,7 +236,7 @@ static int sparc64_setup_msi_irq(unsigned int *irq_p, return 0; out_msi_free: - free_msi(pbm, msi); + sparc64_pbm_free_msi(pbm, msi); out_irq_free: irq_set_chip(*irq_p, NULL); @@ -276,7 +276,7 @@ static void sparc64_teardown_msi_irq(unsigned int irq, return; } - free_msi(pbm, msi_num); + sparc64_pbm_free_msi(pbm, msi_num); irq_set_chip(irq, NULL); irq_free(irq); @@ -418,131 +418,156 @@ static int sparc64_bringup_msi_queues(struct pci_pbm_info *pbm, return 0; } -void sparc64_pbm_msi_init(struct pci_pbm_info *pbm, - const struct sparc64_msiq_ops *ops) +static int pbm_msi_of(struct pci_pbm_info *pbm) { + const struct msi_range_prop { + u32 first_msi; + u32 num_msi; + } *mrng; + const struct addr_range_prop { + u32 msi32_high; + u32 msi32_low; + u32 msi32_len; + u32 msi64_high; + u32 msi64_low; + u32 msi64_len; + } *arng; + const struct msiq_prop { + u32 first_msiq; + u32 num_msiq; + u32 first_devino; + } *mqp; const u32 *val; int len; val = of_get_property(pbm->op->dev.of_node, "#msi-eqs", &len); if (!val || len != 4) - goto no_msi; + return -ENODEV; + pbm->msiq_num = *val; - if (pbm->msiq_num) { - const struct msiq_prop { - u32 first_msiq; - u32 num_msiq; - u32 first_devino; - } *mqp; - const struct msi_range_prop { - u32 first_msi; - u32 num_msi; - } *mrng; - const struct addr_range_prop { - u32 msi32_high; - u32 msi32_low; - u32 msi32_len; - u32 msi64_high; - u32 msi64_low; - u32 msi64_len; - } *arng; - - val = of_get_property(pbm->op->dev.of_node, "msi-eq-size", &len); - if (!val || len != 4) - goto no_msi; - - pbm->msiq_ent_count = *val; + if (!pbm->msiq_num) + return -ENODEV; + + val = of_get_property(pbm->op->dev.of_node, "msi-eq-size", &len); + if (!val || len != 4) + return -ENODEV; + + pbm->msiq_ent_count = *val; + mqp = of_get_property(pbm->op->dev.of_node, + "msi-eq-to-devino", &len); + if (!mqp) mqp = of_get_property(pbm->op->dev.of_node, - "msi-eq-to-devino", &len); - if (!mqp) - mqp = of_get_property(pbm->op->dev.of_node, - "msi-eq-devino", &len); - if (!mqp || len != sizeof(struct msiq_prop)) - goto no_msi; - - pbm->msiq_first = mqp->first_msiq; - pbm->msiq_first_devino = mqp->first_devino; - - val = of_get_property(pbm->op->dev.of_node, "#msi", &len); - if (!val || len != 4) - goto no_msi; - pbm->msi_num = *val; - - mrng = of_get_property(pbm->op->dev.of_node, "msi-ranges", &len); - if (!mrng || len != sizeof(struct msi_range_prop)) - goto no_msi; - pbm->msi_first = mrng->first_msi; - - val = of_get_property(pbm->op->dev.of_node, "msi-data-mask", &len); - if (!val || len != 4) - goto no_msi; - pbm->msi_data_mask = *val; - - val = of_get_property(pbm->op->dev.of_node, "msix-data-width", &len); - if (!val || len != 4) - goto no_msi; - pbm->msix_data_width = *val; - - arng = of_get_property(pbm->op->dev.of_node, "msi-address-ranges", - &len); - if (!arng || len != sizeof(struct addr_range_prop)) - goto no_msi; - pbm->msi32_start = ((u64)arng->msi32_high << 32) | - (u64) arng->msi32_low; - pbm->msi64_start = ((u64)arng->msi64_high << 32) | - (u64) arng->msi64_low; - pbm->msi32_len = arng->msi32_len; - pbm->msi64_len = arng->msi64_len; - - if (msi_bitmap_alloc(pbm)) - goto no_msi; - - if (msi_table_alloc(pbm)) { - msi_bitmap_free(pbm); - goto no_msi; - } + "msi-eq-devino", &len); + if (!mqp || len != sizeof(struct msiq_prop)) + return -ENODEV; - if (ops->msiq_alloc(pbm)) { - msi_table_free(pbm); - msi_bitmap_free(pbm); - goto no_msi; - } + pbm->msiq_first = mqp->first_msiq; + pbm->msiq_first_devino = mqp->first_devino; - if (sparc64_bringup_msi_queues(pbm, ops)) { - ops->msiq_free(pbm); - msi_table_free(pbm); - msi_bitmap_free(pbm); - goto no_msi; - } + val = of_get_property(pbm->op->dev.of_node, "#msi", &len); + if (!val || len != 4) + return -ENODEV; + pbm->msi_num = *val; + + mrng = of_get_property(pbm->op->dev.of_node, "msi-ranges", &len); + if (!mrng || len != sizeof(struct msi_range_prop)) + return -ENODEV; + pbm->msi_first = mrng->first_msi; + + val = of_get_property(pbm->op->dev.of_node, "msi-data-mask", &len); + if (!val || len != 4) + return -ENODEV; + pbm->msi_data_mask = *val; + + val = of_get_property(pbm->op->dev.of_node, "msix-data-width", &len); + if (!val || len != 4) + return -ENODEV; + pbm->msix_data_width = *val; + + arng = of_get_property(pbm->op->dev.of_node, "msi-address-ranges", + &len); + if (!arng || len != sizeof(struct addr_range_prop)) + return -ENODEV; + + pbm->msi32_start = ((u64)arng->msi32_high << 32) | (u64)arng->msi32_low; + pbm->msi64_start = ((u64)arng->msi64_high << 32) | (u64)arng->msi64_low; + pbm->msi32_len = arng->msi32_len; + pbm->msi64_len = arng->msi64_len; + + return 0; +} + +static void sun4v_pbm_msiq_info(struct pci_pbm_info *pbm) +{ + pr_info("%s: MSI Queue first[%u] num[%u] count[%u] devino[0x%x]\n", + pbm->name, pbm->msiq_first, pbm->msiq_num, pbm->msiq_ent_count, + pbm->msiq_first_devino); + + pr_info("%s: MSI queues at RA [%016lx]\n", pbm->name, + __pa(pbm->msi_queues)); +} + +static void sun4v_pbm_msi_info(struct pci_pbm_info *pbm) +{ + pr_info("%s: MSI first[%u] num[%u] mask[0x%x] width[%u]\n", pbm->name, + pbm->msi_first, pbm->msi_num, pbm->msi_data_mask, + pbm->msix_data_width); + + pr_info("%s: MSI addr32[0x%llx:0x%x] addr64[0x%llx:0x%x]\n", pbm->name, + pbm->msi32_start, pbm->msi32_len, pbm->msi64_start, + pbm->msi64_len); +} - printk(KERN_INFO "%s: MSI Queue first[%u] num[%u] count[%u] " - "devino[0x%x]\n", - pbm->name, - pbm->msiq_first, pbm->msiq_num, - pbm->msiq_ent_count, - pbm->msiq_first_devino); - printk(KERN_INFO "%s: MSI first[%u] num[%u] mask[0x%x] " - "width[%u]\n", - pbm->name, - pbm->msi_first, pbm->msi_num, pbm->msi_data_mask, - pbm->msix_data_width); - printk(KERN_INFO "%s: MSI addr32[0x%llx:0x%x] " - "addr64[0x%llx:0x%x]\n", - pbm->name, - pbm->msi32_start, pbm->msi32_len, - pbm->msi64_start, pbm->msi64_len); - printk(KERN_INFO "%s: MSI queues at RA [%016lx]\n", - pbm->name, - __pa(pbm->msi_queues)); - - pbm->msi_ops = ops; - pbm->setup_msi_irq = sparc64_setup_msi_irq; - pbm->teardown_msi_irq = sparc64_teardown_msi_irq; +void sparc64_pbm_msi_init(struct pci_pbm_info *pbm, + const struct sparc64_msiq_ops *ops) +{ + if (pbm_msi_of(pbm)) + goto no_msi; + + if (msi_bitmap_alloc(pbm)) + goto no_msi; + + if (msi_table_alloc(pbm)) { + msi_bitmap_free(pbm); + goto no_msi; + } + + if (ops->msiq_alloc(pbm)) { + msi_table_free(pbm); + msi_bitmap_free(pbm); + goto no_msi; } - return; + if (sparc64_bringup_msi_queues(pbm, ops)) { + ops->msiq_free(pbm); + msi_table_free(pbm); + msi_bitmap_free(pbm); + goto no_msi; + } + + sun4v_pbm_msiq_info(pbm); + sun4v_pbm_msi_info(pbm); + + pbm->msi_ops = ops; + pbm->setup_msi_irq = sparc64_setup_msi_irq; + pbm->teardown_msi_irq = sparc64_teardown_msi_irq; + return; no_msi: pbm->msiq_num = 0; - printk(KERN_INFO "%s: No MSI support.\n", pbm->name); + pr_err("%s: No MSI support.\n", pbm->name); +} + +void pci_priq_msi_init(struct pci_pbm_info *pbm) +{ + if (pbm_msi_of(pbm)) + goto out; + + if (msi_bitmap_alloc(pbm)) + goto out; + + sun4v_pbm_msi_info(pbm); + return; +out: + pr_info("%s: MSI initialization failed.\n", pbm->name); } diff --git a/arch/sparc/kernel/pci_sun4v.c b/arch/sparc/kernel/pci_sun4v.c index eaa204a6fd27..d2a34ea80a7d 100644 --- a/arch/sparc/kernel/pci_sun4v.c +++ b/arch/sparc/kernel/pci_sun4v.c @@ -26,6 +26,7 @@ #include "iommu_common.h" #include "pci_sun4v.h" +#include "priq_sun4v.h" #define DRIVER_NAME "pci_sun4v" #define PFX DRIVER_NAME ": " @@ -1009,7 +1010,8 @@ static const struct sparc64_msiq_ops pci_sun4v_msiq_ops = { static void pci_sun4v_msi_init(struct pci_pbm_info *pbm) { - sparc64_pbm_msi_init(pbm, &pci_sun4v_msiq_ops); + if (pci_sun4v_priq_msi_init(pbm)) + sparc64_pbm_msi_init(pbm, &pci_sun4v_msiq_ops); } #else /* CONFIG_PCI_MSI */ static void pci_sun4v_msi_init(struct pci_pbm_info *pbm) diff --git a/arch/sparc/kernel/priq_sun4v.c b/arch/sparc/kernel/priq_sun4v.c new file mode 100644 index 000000000000..e7554e07ffae --- /dev/null +++ b/arch/sparc/kernel/priq_sun4v.c @@ -0,0 +1,1300 @@ +/* priq_sun4v.c: SUN4V Hardware Priority Interrupt Queues - PRIQ support. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "priq_sun4v.h" +#include "pci_impl.h" + +static unsigned long priqs_eq_sizes, priqs_per_cpu, priqs_per_pcibus; +static unsigned long priq_eq_max_num_entries; +static bool priq_configured; + +/* priq_eq_sizes is a bit map of supported EQ sizes. + * If bit n is set, EQ size in bytes of 2^(13 + 3n) is supported. + * If bit n is set, #entries of 2^(7 + 3n) is supported. + * Note: Each EQ entry is 64 bytes. + * + * bit 0 = 128 entries/8KB buffer size is supported + * bit 1 = 1k entries/64KB buffer size is supported + * bit 2 = 8k entries/512KB buffer size is supported + * bit 3 = 64k entries/4MB buffer size is supported + */ +static void __init priq_max_eq_size(unsigned long priq_eq_sizes) +{ + unsigned int n; + + n = fls64(priq_eq_sizes) - 1UL; + priq_eq_max_num_entries = 1UL << (7 + 3 * n); +} + +static int __init priq_get_properties(void) +{ + struct mdesc_handle *hp = mdesc_grab(); + int rc = -ENODEV; + u64 pn, *mdp; + + pn = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform"); + + mdp = (u64 *)mdesc_get_property(hp, pn, "priq-eq-sizes", NULL); + if (!mdp) + goto out; + + priqs_eq_sizes = *mdp; + priq_max_eq_size(priqs_eq_sizes); + + mdp = (u64 *)mdesc_get_property(hp, pn, "#priqs-per-cpu", NULL); + if (mdp) + priqs_per_cpu = *mdp; + else + priqs_per_cpu = 1; + + mdp = (u64 *)mdesc_get_property(hp, pn, "#priqs-per-pcibus", NULL); + if (mdp) + priqs_per_pcibus = *mdp; + else + priqs_per_pcibus = 1; + + rc = 0; + pr_info("%s: priqs_eq_sizes=%ld priqs_per_cpu=%ld priqs_per_pcibus=%ld\n", + __func__, priqs_eq_sizes, priqs_per_cpu, priqs_per_pcibus); + +out: + mdesc_release(hp); + return rc; +} + +/* bus, device and function. */ +#define BDF(b, d, f) (((b) << 8) | ((d) << 3) | (f)) + +/* bus, device and function for intx is PEX req_id (0x0000).*/ +#define BDF_INTX BDF(0, 0, 0) + +/* INTX priq record for msidata. */ +#define PRIQ_INTA 0UL +#define PRIQ_INTB 1UL +#define PRIQ_INTC 2UL +#define PRIQ_INTD 3UL + +struct priq_irq { + struct list_head list; /* list */ + int irq; /* irq */ + unsigned int msidata; /* msi data */ + unsigned int devhandle; /* rc devhandle */ + unsigned short bdf; /* bus/device/function */ + unsigned short strand; /* strand bound too */ +}; + +static unsigned long priq_irq_to_intx(struct priq_irq *priq_irq) +{ + return priq_irq->msidata & PRIQ_INTD; +} + +static bool priq_irq_is_intx(struct priq_irq *priq_irq) +{ + return priq_irq->bdf == BDF_INTX; +} + +#define PRIQ_INTX_DEBUG 0 +#if PRIQ_INTX_DEBUG +static void priq_intx_debug(struct priq_irq *priq_irq, const char *message) +{ + unsigned long intx = priq_irq_to_intx(priq_irq); + + pr_info("%s: %s intx=%ld devhandle=0x%x irq=%d bdf=%d msidata=%u\n", + __func__, message, intx, priq_irq->devhandle, priq_irq->irq, + priq_irq->bdf, priq_irq->msidata); +} +#else +static void priq_intx_debug(struct priq_irq *priq_irq, const char *message) {} +#endif + +static const unsigned int priq_hash_node_nr = + PAGE_SIZE / sizeof(struct list_head); + +static const unsigned int priq_hash_mask = + (PAGE_SIZE / sizeof(struct list_head)) - 1; + +/* TODO should not be hardcoded but should be the order of priq_hash_mode_nr */ +#define PRIQ_HASH_BITS 9 + +/* The numa affined piece is replicated percpu for each node's strands. */ +static struct list_head *priq_hash[MAX_NUMNODES]; + +/* Should this be spotted hot then it can be done per node. + * It should only be used during MSI/IRQ hash list addition and removal. + */ +static DEFINE_SPINLOCK(priq_hash_node_lock); + +#define PRIQ_EQ_SIZE_SHIFT 6 +#define PRIQ_EQ_SIZE BIT(PRIQ_EQ_SIZE_SHIFT) + +struct priq { + unsigned long id; + unsigned long c_raddr; /* complex raddr (raddr|order) */ + unsigned long head; + unsigned long tail; + struct list_head *hash; +}; + +static void raddr_order_to_priq(struct priq *priq, unsigned long raddr, + unsigned int order) +{ + priq->c_raddr = raddr | order; +} + +static unsigned long priq_to_raddr(struct priq *priq) +{ + return priq->c_raddr & PAGE_MASK; +} + +static unsigned int priq_to_order(struct priq *priq) +{ + return priq->c_raddr & ~PAGE_MASK; +} + +static unsigned long priq_size_mask(struct priq *priq) +{ + return ~((1UL << (PAGE_SHIFT + (priq_to_order(priq)))) - 1UL); +} + +static bool priq_active(struct priq *priq) +{ + return priq->hash; +} + +static DEFINE_PER_CPU(struct priq, current_priq) = { + .id = 0UL, .c_raddr = 0UL, .head = 0UL, .tail = 0UL, + .hash = NULL}; + +/* cpus with PRIQs */ +static cpumask_t priq_cpu_mask; + +static void cpumask_set_cpu_priq(int cpu) +{ + cpumask_set_cpu(cpu, &priq_cpu_mask); +} + +static void cpumask_clear_cpu_priq(int cpu) +{ + cpumask_clear_cpu(cpu, &priq_cpu_mask); +} + +static DEFINE_SPINLOCK(priq_affine_lock); +static unsigned short next_strand[MAX_NUMNODES]; + +static int priq_choose_strand(int hint_strand) +{ + static const unsigned short strand_step = 0x08U; + int node = cpu_to_node(hint_strand); + cpumask_t *cpumask = cpumask_of_node(node); + cpumask_t possible_strands; + unsigned long flags; + int strand; + + cpumask_and(&possible_strands, &priq_cpu_mask, cpumask); + + spin_lock_irqsave(&priq_affine_lock, flags); + + strand = next_strand[node] + strand_step; + if (!cpumask_test_cpu(strand, &possible_strands)) { + strand = cpumask_next(strand + 1, &possible_strands); + if (strand >= nr_cpu_ids) + strand = cpumask_first(&possible_strands); + } + next_strand[node] = strand; + + spin_unlock_irqrestore(&priq_affine_lock, flags); + + return strand; +} + +static int priq_mask_to_strand(int hint_strand, const struct cpumask *mask) +{ + cpumask_t *cpumask = cpumask_of_node(cpu_to_node(hint_strand)); + cpumask_t tmp_mask, node_mask; + int strand; + + cpumask_and(&node_mask, &priq_cpu_mask, cpumask); + cpumask_and(&tmp_mask, &priq_cpu_mask, mask); + cpumask_and(&tmp_mask, &tmp_mask, &node_mask); + + if (cpumask_empty(&tmp_mask)) { + strand = cpumask_next(hint_strand, &node_mask); + if (strand >= nr_cpu_ids) + strand = cpumask_first(&node_mask); + } else { + strand = cpumask_next(hint_strand, &tmp_mask); + if (strand >= nr_cpu_ids) + strand = cpumask_first(&tmp_mask); + } + + return strand; +} + +static int priq_affine(int hint_strand, const struct cpumask *mask) +{ + int strand; + + if (cpumask_equal(mask, cpu_online_mask)) + strand = priq_choose_strand(hint_strand); + else + strand = priq_mask_to_strand(hint_strand, mask); + + return strand; +} + +static int priq_irq_bind_eqcb(struct irq_data *data, const struct cpumask *mask) +{ + struct priq_irq *priq_irq = irq_data_get_irq_handler_data(data); + bool is_intx = priq_irq_is_intx(priq_irq); + int strand = priq_irq->strand; + unsigned long hverror, id; + struct priq *priq; + + strand = priq_affine(strand, mask); + + if (strand >= nr_cpu_ids) + return -ENODEV; + + priq = &per_cpu(current_priq, strand); + id = priq->id; + + if (is_intx) { + unsigned long intx = priq_irq_to_intx(priq_irq); + + priq_intx_debug(priq_irq, __func__); + hverror = pci_priq_intx_bind(priq_irq->devhandle, intx, id); + } else + hverror = pci_priq_msi_bind(priq_irq->devhandle, + priq_irq->msidata, + priq_irq->bdf, id); + + if (hverror) { + pr_err("%s: Failed for bind hverror(%ld)\n", __func__, hverror); + return -ENODEV; + } + + priq_irq->strand = strand; + return 0; +} + +/* We may want to limit the number of priq per socket. For now it will 128 + * entries PRIQ and one per strand, which is 1U << (7 + 6) = 8Kb. This could + * encounter issue with nr_cpus=1, such as EQ overflow, and with unevenly + * distributed irq placement. To mitigate both of these the choices are + * larger priq EQ size and/or limit strand per core coverage. + * + * Another issue is NUMA affining priq_msi. Should a node cpu be put offline, + * then the numa affined priq_msi needs to be move from one hash to + * another. It becomes a tangled and ugly. Let's attempt to not permit this. + */ +static int priq_msi_set_affinity(struct irq_data *data, + const struct cpumask *mask, bool force) +{ + return priq_irq_bind_eqcb(data, mask); +} + +static void priq_msi_enable(struct irq_data *data) +{ + struct priq_irq *priq_msi = irq_data_get_irq_handler_data(data); + unsigned long hverror; + + hverror = priq_irq_bind_eqcb(data, data->affinity); + if (hverror) + return; + + hverror = pci_priq_msi_enable(priq_msi->devhandle, priq_msi->msidata, + priq_msi->bdf); + if (hverror) { + pr_err("%s: Failed enable hverror(%ld).\n", __func__, hverror); + return; + } + + hverror = pci_priq_msi_setstate(priq_msi->devhandle, priq_msi->msidata, + priq_msi->bdf, HV_MSISTATE_IDLE); + if (hverror) + pr_err("%s: Failed to set msi to idle, hverror(%ld).\n", + __func__, hverror); + + unmask_msi_irq(data); +} + +static void priq_msi_disable(struct irq_data *data) +{ + struct priq_irq *priq_msi = irq_data_get_irq_handler_data(data); + unsigned long hverror; + + hverror = pci_priq_msi_disable(priq_msi->devhandle, priq_msi->msidata, + priq_msi->bdf); + if (hverror) + pr_err("%s: Failed for hverror(%ld).\n", __func__, hverror); + + mask_msi_irq(data); +} + +static struct irq_chip priq_msi_chip = { + .name = "PCI-MSI", + .irq_mask = mask_msi_irq, + .irq_unmask = unmask_msi_irq, + .irq_enable = priq_msi_enable, + .irq_disable = priq_msi_disable, + .irq_set_affinity = priq_msi_set_affinity, +}; + +static int priq_intx_set_affinity(struct irq_data *data, + const struct cpumask *mask, bool force) +{ + return priq_irq_bind_eqcb(data, mask); +} + +static void priq_intx_enable(struct irq_data *data) +{ + struct priq_irq *priq_irq = irq_data_get_irq_handler_data(data); + unsigned long intx = priq_irq_to_intx(priq_irq); + unsigned long hverror; + + hverror = priq_irq_bind_eqcb(data, data->affinity); + if (hverror) + goto out; + + hverror = pci_priq_intx_enable(priq_irq->devhandle, intx); + if (hverror) + pr_err("%s: Failed for Enable hverror(%ld).\n", __func__, + hverror); + + hverror = pci_priq_intx_setstate(priq_irq->devhandle, intx, + HV_PCI_INTX_CLEAR); + if (hverror) + pr_err("%s: Failed for clear of intx, hverror(%ld).\n", + __func__, hverror); + +out: + priq_intx_debug(priq_irq, __func__); +} + +static void priq_intx_disable(struct irq_data *data) +{ + struct priq_irq *priq_irq = irq_data_get_irq_handler_data(data); + unsigned long intx = priq_irq_to_intx(priq_irq); + unsigned long hverror; + + hverror = pci_priq_intx_disable(priq_irq->devhandle, intx); + if (hverror) + pr_err("%s: Failed for hverror(%ld).\n", __func__, hverror); + + priq_intx_debug(priq_irq, __func__); +} + +static struct irq_chip priq_intx_chip = { + .name = "PCI-INTX", + .irq_enable = priq_intx_enable, + .irq_disable = priq_intx_disable, + .irq_set_affinity = priq_intx_set_affinity, +}; + +static void priq_hash_node_init(struct list_head *head) +{ + unsigned int index; + + for (index = 0U; index != priq_hash_node_nr; index++, head++) + INIT_LIST_HEAD(head); +} + +static void priq_hash_nodes_free(void) +{ + int node; + + for_each_online_node(node) + if (priq_hash[node]) { + struct list_head *head = priq_hash[node]; + + free_pages((unsigned long)head, 0U); + } +} + +static int priq_hash_nodes_init(void) +{ + int node; + + for_each_online_node(node) { + struct list_head *head; + struct page *page; + + page = alloc_pages_exact_node(node, GFP_ATOMIC, 0U); + if (!page) { + pr_err("%s: Failed to allocate priq hash list.\n", + __func__); + goto out; + } + + head = page_address(page); + priq_hash[node] = head; + priq_hash_node_init(head); + } + return 0; + +out: + priq_hash_nodes_free(); + return -ENOMEM; +} + +static unsigned int priq_hash_value(unsigned int devhandle, unsigned int bdf, + unsigned int msidata) +{ + unsigned long bits = ((unsigned long)devhandle << 44) | + ((unsigned long)msidata << 16) | bdf; + unsigned int hash_value = hash_64(bits, PRIQ_HASH_BITS); + + return hash_value; +} + +static struct list_head *priq_hvalue_to_head(struct priq *priq, + unsigned int hash_value) +{ + return &priq->hash[hash_value]; +} + +static void priq_hash_add(struct priq_irq *priq_irq, int node) +{ + unsigned int hash_value; + struct list_head *head; + unsigned long flags; + + hash_value = priq_hash_value(priq_irq->devhandle, priq_irq->bdf, + priq_irq->msidata); + head = &priq_hash[node][hash_value]; + + spin_lock_irqsave(&priq_hash_node_lock, flags); + list_add_rcu(&priq_irq->list, head); + spin_unlock_irqrestore(&priq_hash_node_lock, flags); +} + +static void priq_hash_del(struct priq_irq *priq_irq) +{ + unsigned long flags; + + spin_lock_irqsave(&priq_hash_node_lock, flags); + list_del_rcu(&priq_irq->list); + spin_unlock_irqrestore(&priq_hash_node_lock, flags); +} + +static unsigned short priq_bdf(struct pci_dev *pdev) +{ + unsigned int device = PCI_SLOT(pdev->devfn); + unsigned int func = PCI_FUNC(pdev->devfn); + struct pci_bus *bus_dev = pdev->bus; + unsigned int bus = bus_dev->number; + + return BDF(bus, device, func); +} + +/* This selection is based on node and online cpus. Should online not + * intersect the initial cpu's node then we have an issue. The issue is + * migrating the priq_msi to another node with active cpu-s. + */ +static int priq_irq_get_strand(int node) +{ + cpumask_t *nodemask = cpumask_of_node(node); + cpumask_t tmp; + + cpumask_and(&tmp, nodemask, &priq_cpu_mask); + if (cpumask_empty(&tmp)) + return cpumask_first(&priq_cpu_mask); + + return cpumask_first(&tmp); +} + +static int priq_msi_setup(unsigned int *irqp, struct pci_dev *pdev, + struct msi_desc *entry) +{ + struct pci_pbm_info *pbm = get_pbm(pdev); + struct priq_irq *priq_msi; + struct msi_msg msg; + int msi, irq; + int strand; + int node; + + node = pbm->numa_node; + strand = priq_irq_get_strand(node); + + /* if node was -1, it is now >= 0 or if all strands in original are + * offline and a different node gets chosen, it will come back correct. + */ + node = cpu_to_node(strand); + + irq = irq_alloc_descs(-1, 1, 1, node); + if (irq <= 0) + goto out; + + priq_msi = kzalloc_node(sizeof(*priq_msi), GFP_ATOMIC, node); + if (!priq_msi) + goto out_free_irq; + + msi = sparc64_pbm_alloc_msi(pbm); + if (msi < 0) + goto out_free_mem; + + priq_msi->strand = strand; + priq_msi->devhandle = pbm->devhandle; + priq_msi->msidata = msi; + priq_msi->irq = irq; + priq_msi->bdf = priq_bdf(pdev); + + /* Some of this is duplicated from pci_msi.c and should be unified. */ + if (entry->msi_attrib.is_64) { + msg.address_hi = pbm->msi64_start >> 32; + msg.address_lo = pbm->msi64_start & 0xffffffff; + } else { + msg.address_hi = 0; + msg.address_lo = pbm->msi32_start; + } + msg.data = msi; + + irq_set_msi_desc(irq, entry); + write_msi_msg(irq, &msg); + + /* Now set up chip handler name and chip handler data. */ + irq_set_chip_and_handler_name(irq, &priq_msi_chip, handle_simple_irq, + "PRIQ"); + irq_set_handler_data(irq, priq_msi); + + /* The MSI is not bound to a priq yet. */ + priq_hash_add(priq_msi, node); + + return 0; + +out_free_mem: + kfree(priq_msi); +out_free_irq: + irq_free(irq); +out: + return -ENOMEM; +} + +static void priq_msi_teardown(unsigned int irq, struct pci_dev *pdev) +{ + struct pci_pbm_info *pbm = get_pbm(pdev); + struct priq_irq *priq_msi = irq_get_handler_data(irq); + unsigned long hverror; + + hverror = pci_priq_msi_unbind(pbm->devhandle, priq_msi->msidata, + priq_msi->bdf); + if (hverror && hverror != HV_EUNBOUND) + pr_err("%s: pci_priq_msi_unbind failed with (%ld).\n", __func__, + hverror); + + priq_hash_del(priq_msi); + synchronize_rcu(); + + sparc64_pbm_free_msi(pbm, priq_msi->msidata); + irq_set_chip(irq, NULL); + irq_set_handler_data(irq, NULL); + irq_free_desc(irq); + kfree(priq_msi); +} + +/* priq msi record root complex(rc) devhandle.*/ +static unsigned int priq_pci_record_dhndl(unsigned long word4) +{ + static const unsigned long rc_mask = 0xffffffff00000000UL; + static const unsigned int rc_shift = 32U; + unsigned int rc; + + rc = (word4 & rc_mask) >> rc_shift; + return rc; +} + +/* priq msi record pci msidata (msi).*/ +static unsigned int priq_pci_record_msidata(unsigned long word6) +{ + static const unsigned int msidata_mask = 0xffffffffU; + unsigned int msidata = word6 & msidata_mask; + + return msidata; +} + +/* priq msi record pci bdf(rid).*/ +static unsigned int priq_pci_record_bdf(unsigned long word4) +{ + unsigned int bdf = word4 & 0xffffU; + + return bdf; +} + +static void priq_msi_dump_record(const char *reason, void *entry) +{ + unsigned long *word = entry; + + pr_err("%s: %s\n", __func__, reason); + pr_err("word0=0x%.16lx word1=0x%.16lx word2=0x%.16lx word3=0x%.16lx\n", + word[0], word[1], word[2], word[3]); + pr_err("word4=0x%.16lx word5=0x%.16lx word6=0x%.16lx word7=0x%.16lx\n", + word[4], word[5], word[6], word[7]); +} + +static void priq_msi_idle(struct priq_irq *priq_msi) +{ + unsigned long hverror; + + hverror = pci_priq_msi_setstate(priq_msi->devhandle, priq_msi->msidata, + priq_msi->bdf, HV_MSISTATE_IDLE); + if (hverror) + pr_err("%s: Failed to set msi to idle, hverror(%ld).\n", + __func__, hverror); +} + +#define PRIQ_TYPE_MASK 0xffUL +#define PRIQ_TYPE_MESSAGE 0x01UL +#define PRIQ_TYPE_MSI32 0x02UL +#define PRIQ_TYPE_MSI64 0x03UL +#define PRIQ_TYPE_INTX 0x08UL +#define PRIQ_TYPE_ERROR 0x0fUL + +static void process_priq_record(void *entry, int type, + struct priq_irq *priq_irq) +{ + unsigned long hverror; + unsigned long intx; + + switch (type) { + case PRIQ_TYPE_MSI32: + case PRIQ_TYPE_MSI64: + priq_msi_idle(priq_irq); + generic_handle_irq(priq_irq->irq); + break; + + case PRIQ_TYPE_INTX: + intx = priq_irq_to_intx(priq_irq); + + hverror = pci_priq_intx_setstate(priq_irq->devhandle, intx, + HV_PCI_INTX_CLEAR); + if (hverror) + pr_err("%s: intx_setstate failed for, hverror(%ld).\n", + __func__, hverror); + + generic_handle_irq(priq_irq->irq); + break; + } +} + +static void priq_msi_consume(struct priq *priq, void *entry) +{ + /* PRIQ PCI record.*/ + struct priq_pci_msi_record { + unsigned long word0; /* reserved | type */ + unsigned long word1; + unsigned long word2; + unsigned long word3; + unsigned long word4; /* devhandle | reserved | rid */ + unsigned long word5; + unsigned long word6; /* msidata */ + unsigned long word7; + } *rec = entry; + + unsigned int msidata = priq_pci_record_msidata(rec->word6); + + unsigned int bdf = priq_pci_record_bdf(rec->word4); + + unsigned int dhndl = priq_pci_record_dhndl(rec->word4); + + unsigned long type = rec->word0 & PRIQ_TYPE_MASK; + + unsigned int hash_value = priq_hash_value(dhndl, bdf, msidata); + + struct list_head *head = priq_hvalue_to_head(priq, hash_value); + + struct priq_irq *priq_irq; + bool found = false; + + if (type != PRIQ_TYPE_MSI32 && type != PRIQ_TYPE_MSI64 && + type != PRIQ_TYPE_INTX) { + priq_msi_dump_record("type not supported yet", entry); + return; + } + + rcu_read_lock(); + list_for_each_entry_rcu(priq_irq, head, list) { + if (priq_irq->devhandle == dhndl && priq_irq->bdf == bdf && + priq_irq->msidata == msidata) { + process_priq_record(entry, type, priq_irq); + found = true; + break; + } + } + rcu_read_unlock(); + + if (!found) + priq_msi_dump_record("No PRIQ MSI record found.", entry); +} + +static unsigned long priq_disable_ie(void) +{ + unsigned long pstate; + + __asm__ __volatile__("rdpr %%pstate,%0\n\t" + "wrpr %0, %1, %%pstate\n\t" + : "=&r" (pstate) + : "i" (PSTATE_IE)); + + return pstate; +} + +static void priq_enable_ie(unsigned long pstate) +{ + __asm__ __volatile__("wrpr %0, 0, %%pstate\n\t" + : /* no outputs */ + : "r" (pstate)); +} + +static unsigned long eq_distance(struct priq *priq, unsigned long head, + unsigned long tail) +{ + unsigned long distance; + + if (head > tail) { + unsigned long eq_mask = ~priq_size_mask(priq); + unsigned long eq_size = eq_mask + 1UL; + + distance = eq_size - head + tail; + } else { + distance = tail - head; + } + + return distance; +} + +static unsigned long eq_distance_percpu(struct priq *priq) +{ + unsigned long distance = eq_distance(priq, priq->head, priq->tail); + + return distance; +} + +static void handle_priq(struct priq *priq) +{ + unsigned long head, tail, hverror, pstate; + unsigned long raddr = priq_to_raddr(priq); + unsigned long mask = priq_size_mask(priq); + bool nested; + + hverror = priq_get_head_tail(priq->id, &head, &tail); + if (hverror) { + pr_err("PRIQ: %s: failed priq_get_head_tail(%ld).\n", + __func__, hverror); + return; + } + + /* this function is currently called for every interrupt, many not + * PRIQ related + */ + if (head == tail) + return; + + pstate = priq_disable_ie(); + + /* when we are done processing prior interrupts, these should be equal + * in which case we are NOT nested. + */ + if (priq->head == priq->tail) { + nested = false; + priq->tail = tail; + + } else { + nested = true; + + /* see if number of records to process has increased */ + if (eq_distance(priq, head, tail) > eq_distance_percpu(priq)) + priq->tail = tail; + } + + priq_enable_ie(pstate); + + if (nested) { + pr_err("PRIQ: bailed on nested interrupt\n"); + return; + } + +again: + while (head != tail) { + void *entry = __va(raddr + head); + + priq_msi_consume(priq, entry); + head = (head + PRIQ_EQ_SIZE) & ~mask; + } + + pstate = priq_disable_ie(); + + /* have we taken a new nested interrupt increasing the amount + * of work. if so, lets keep going + */ + if (tail != priq->tail) { + tail = priq->tail; + priq_enable_ie(pstate); + goto again; + } + + hverror = priq_set_head(priq->id, head); + priq->head = head; + + BUG_ON(priq->head != priq->tail); + priq_enable_ie(pstate); + + if (hverror) { + pr_err("%s: failed priq_set_head(%ld).\n", __func__, hverror); + pr_err("%s: Fatal error, priq is in impossible condition. Proceeding but with peril.\n", + __func__); + } +} + +/* We have at most one priq per cpu for now.*/ +void cpu_handle_priqs(void) +{ + struct priq *priq = &get_cpu_var(current_priq); + + if (priq_active(priq)) + handle_priq(priq); + + put_cpu_var(current_priq); +} + +static unsigned int priq_coarse_mask; +static bool priq_enabled = true; + +static void __init early_priq_coarse(void) +{ + static const unsigned int strands_per_core_shift = 3U; + + priq_coarse_mask = ~((1U << strands_per_core_shift) - 1U); +} + +static bool can_cpu_have_priq(int cpu) +{ + int weight; + + if (priq_coarse_mask) { + weight = num_online_cpus(); + if ((weight > 1) && (cpu & ~priq_coarse_mask)) + return false; + } + + return true; +} + +/* For now leave size at a PAGE_SIZE which is 1UL << (13-6) eq entries. + * For nr_cpu small a large size might be desirable when MR supports, + * which MR does. + */ +static bool priq_eq_max; +static size_t compute_cpu_priq_size(int cpu) +{ + size_t size; + + if ((nr_cpu_ids == 1) || !pci_msi_enabled() || priq_eq_max) + size = priq_eq_max_num_entries << PRIQ_EQ_SIZE_SHIFT; + else + size = PAGE_SIZE; + + return size; +} + +static int __init early_priq(char *p) +{ + while (p) { + char *k = strchr(p, ','); + + if (k) + *k++ = 0; + + if (*p) { + if (!strcmp(p, "max")) + priq_eq_max = true; + else if (!strcmp(p, "coarse")) + early_priq_coarse(); + else if (!strcmp(p, "off")) + priq_enabled = false; + } + + p = k; + } + return 0; +} + +early_param("priq", early_priq); + +/* Here we can decide whether a cpu will have a priq.*/ +static size_t priq_size_for_cpu(int cpu) +{ + int cpu_priq_possible = can_cpu_have_priq(cpu); + size_t priq_size = 0UL; + + if (cpu_priq_possible) + priq_size = compute_cpu_priq_size(cpu); + return priq_size; +} + +/* The priq will be numa affined and the EQ entries will be too. Should + * there be failure then the cpu will not be considered a member of priq + * online strands(cpus). + */ +void priq_percpu_setup(int cpu) +{ + struct priq *priq = &per_cpu(current_priq, cpu); + unsigned long raddr, hverror, id, nentries; + int node = cpu_to_node(cpu); + unsigned int order; + struct page *pages; + size_t size; + + if (!priq_configured) + goto out; + + size = priq_size_for_cpu(cpu); + if (!size) + goto out; + + order = get_order(size); + nentries = size >> PRIQ_EQ_SIZE_SHIFT; + + pages = alloc_pages_exact_node(node, GFP_ATOMIC, order); + if (!pages) + goto out1; + + raddr = page_to_phys(pages); + + id = priq_conf(PRIQ_CPU, cpu, raddr, nentries); + if ((long)id < 0) { + long hverror = -id; + + pr_err("%s: priq_conf failed for (%ld) one cpu(%d).\n", + __func__, hverror, cpu); + goto out2; + } + + hverror = priq_bind(id, cpu, PIL_DEVICE_IRQ); + if (hverror) { + pr_err("%s: priq_bind(id=%lx) failed for (%ld) one cpu(%d).\n", + __func__, id, hverror, cpu); + goto out3; + } + + raddr_order_to_priq(priq, raddr, order); + priq->id = id; + + /* The initialization order is important because hash is true active.*/ + priq->hash = priq_hash[node]; + + /* Mark the cpu and its associated priq online. Now binding can occur + * for PRIQ MSI. + */ + cpumask_set_cpu_priq(cpu); + goto out; + +out3: + hverror = priq_unconf(id); + if (hverror) + pr_err("%s: priq_unconf(%ld) failed for(%ld)\n", __func__, id, + hverror); +out2: + free_pages((unsigned long)__va(raddr), order); +out1: + pr_err("%s: failed to allocate priq for cpu(%d).\n", __func__, cpu); +out: + return; +} + +static void unbind_priq_msi(struct priq *priq, struct priq_irq *priq_msi) +{ + struct hv_pci_msi_info info; + unsigned long hverror; + + hverror = pci_priq_msi_info(priq_msi->devhandle, priq_msi->msidata, + priq_msi->bdf, &info); + + if (hverror == HV_EUNBOUND) { + pr_err("%s: attempt to unbind unbound MSI\n", __func__); + return; + } + + if (hverror) { + pr_err("%s: Failed to obtain priq msi info, hverror(%ld).\n", + __func__, hverror); + return; + } + + if (priq->id != info.priq_id) { + pr_err("%s: attempt to unbind MSI from wrong PRIQ\n", __func__); + return; + } + + hverror = pci_priq_msi_unbind(priq_msi->devhandle, priq_msi->msidata, + priq_msi->bdf); + if (hverror) + pr_err("%s: Failed to unbind msi irq, hverror(%ld).\n", + __func__, hverror); +} + +static void unbind_priq_intx(struct priq *priq, struct priq_irq *priq_irq) +{ + unsigned long intx = priq_irq_to_intx(priq_irq); + struct hv_pci_priq_intx_info info; + unsigned long hverror; + + hverror = pci_priq_intx_info(priq_irq->devhandle, intx, &info); + if (hverror) { + if (hverror != HV_EUNBOUND) + pr_err("%s: Failed to obtain priq intx information, hverror(%ld).\n", + __func__, hverror); + return; + } + + if (priq->id != info.priq_id) + return; + + hverror = pci_priq_intx_unbind(priq_irq->devhandle, intx); + if (hverror) + pr_err("%s: Failed to unbind intx, hverror(%ld)\n", + __func__, hverror); +} + +static void unbind_all_msi_on_priq(int cpu, struct priq *priq) +{ + struct irq_desc *desc; + int irq; + + for_each_irq_desc(irq, desc) { + struct irq_chip *chip = irq_desc_get_chip(desc); + struct priq_irq *priq_irq; + bool is_intx; + + if (chip != &priq_msi_chip && chip != &priq_intx_chip) + continue; + + priq_irq = irq_desc_get_handler_data(desc); + if (priq_irq->strand != cpu) + continue; + + is_intx = priq_irq_is_intx(priq_irq); + if (is_intx) + unbind_priq_intx(priq, priq_irq); + else + unbind_priq_msi(priq, priq_irq); + } +} + +void priq_percpu_destroy(int cpu) +{ + struct priq *priq = &per_cpu(current_priq, cpu); + unsigned long hverror; + unsigned long raddr; + unsigned int order; + + if (!priq_active(priq)) + return; + + cpumask_clear_cpu_priq(cpu); + + unbind_all_msi_on_priq(cpu, priq); + + hverror = priq_unbind(priq->id); + if (hverror) { + pr_err("%s: priq_unbind failed for (%ld)\n", __func__, hverror); + return; + } + + hverror = priq_unconf(priq->id); + if (hverror) { + pr_err("%s: priq_unconf failed with error (%ld), for cpu=%d.\n", + __func__, hverror, cpu); + return; + } + + raddr = priq_to_raddr(priq); + order = priq_to_order(priq); + + free_pages((unsigned long)__va(raddr), order); + priq->c_raddr = 0UL; + priq->hash = NULL; +} + +void __init sun4v_priq(void) +{ + int cpu = raw_smp_processor_id(); + unsigned long major = 1UL; + unsigned long minor = 1UL; + int group_rc, rc; + + if (tlb_type != hypervisor) + return; + + if (!priq_enabled) { + pr_info("PRIQ: Available but disabled by priq=off\n"); + return; + } + + rc = priq_get_properties(); + if (rc) + return; + + /* Register the HV groups.*/ + group_rc = sun4v_hvapi_register(HV_GRP_PRIQ, major, &minor); + if (group_rc) { + pr_err("%s: failed to register HV_GRP_PRIQ(%d).\n", + __func__, group_rc); + return; + } + + minor = 1UL; + group_rc = sun4v_hvapi_register(HV_GRP_PRIQ_PCI, major, &minor); + if (group_rc) { + pr_err("%s: failed to register HV_GRP_PRIQ_PCI(%d).\n", + __func__, group_rc); + return; + } + + if (priq_hash_nodes_init()) + return; + + priq_configured = true; + priq_percpu_setup(cpu); + + pr_err("PRIQ Enabled\n"); +} + +int pci_sun4v_priq_msi_init(struct pci_pbm_info *pbm) +{ + if (!priq_configured) + return -ENODEV; + + pci_priq_msi_init(pbm); + pbm->setup_msi_irq = priq_msi_setup; + pbm->teardown_msi_irq = priq_msi_teardown; + + pr_info("%s: devhandle=0x%x node=%d.\n", __func__, pbm->devhandle, + pbm->numa_node); + return 0; +} + +static int intx_shared(unsigned int rc, unsigned int msidata, int node) +{ + unsigned short bdf = BDF_INTX; + unsigned int hash_value = priq_hash_value(rc, bdf, msidata); + struct list_head *head = &priq_hash[node][hash_value]; + struct priq_irq *priq_irq; + int irq = -1; + + list_for_each_entry_rcu(priq_irq, head, list) { + if (priq_irq->devhandle == rc && priq_irq->bdf == bdf && + priq_irq->msidata == msidata) { + irq = priq_irq->irq; + break; + } + } + + return irq; +} + +static int setup_priq_intx_irq(unsigned int devhandle, unsigned int devino, + int node) +{ + unsigned int intx = devino - 1U; + unsigned short bdf = BDF_INTX; + unsigned int msidata = intx; + struct priq_irq *priq_irq; + int strand; + int irq; + + strand = priq_irq_get_strand(node); + + /* strand may not be on requested node or node might have been -1 */ + node = cpu_to_node(strand); + + irq = intx_shared(devhandle, msidata, node); + if (irq > 0) + return irq; + + irq = irq_alloc_descs(-1, 1, 1, node); + if (irq <= 0) + return irq; + + priq_irq = kzalloc_node(sizeof(*priq_irq), GFP_ATOMIC, node); + if (!priq_irq) { + pr_err("%s: failed to allocate legacy priq_irq.\n", + __func__); + irq_free(irq); + return -1; + } + + priq_irq->strand = strand; + + priq_irq->devhandle = devhandle; + priq_irq->msidata = msidata; + priq_irq->irq = irq; + priq_irq->bdf = bdf; + + irq_set_chip_and_handler_name(irq, &priq_intx_chip, handle_simple_irq, + "PRIQ"); + irq_set_handler_data(irq, priq_irq); + + priq_hash_add(priq_irq, cpu_to_node(priq_irq->strand)); + + pr_info("PRIQ: %s: devhandle=0x%x devino=0x%x node=%d irq=%d.\n", + __func__, devhandle, devino, node, irq); + + return irq; +} + +/* We will drop 0x3f (IOS_PEU_ERR_DEVINO) and 0x3e (IOS_DMU_ERR_DEVINO). + */ +int pci_priq_build_irq(unsigned int devhandle, unsigned int devino) +{ + int irq = -1; + int node; + + if (!priq_configured) + return 0; + + if (!(1 <= devino && devino <= 4)) + return irq; + + /* These INT-A ... INT-D are shared line interrupts for which the + * performance ramifications greatly exceed the advantages of exact node + * placement. As this function is called prior to PCI root complex + * structure creation, exact node calculations are ugly. + */ + node = -1; + + irq = setup_priq_intx_irq(devhandle, devino, node); + + return irq; +} + +void dump_priq_info(void) +{ + char buf[256]; + int cpu, weight; + + scnprintf(buf, sizeof(buf), "%pb", &priq_cpu_mask); + weight = cpumask_weight(&priq_cpu_mask); + + pr_info("%s: strand priq active(%d), %s\n", __func__, weight, buf); + + for_each_online_cpu(cpu) { + struct priq *priq = &per_cpu(current_priq, cpu); + unsigned long raddr = priq_to_raddr(priq); + unsigned long mask = priq_size_mask(priq); + + pr_info("cpu=%d priq id=0x%lx raddr=0x%lx mask=0x%lx hash=%p\n", + cpu, priq->id, raddr, mask, priq->hash); + } +} diff --git a/arch/sparc/kernel/priq_sun4v.h b/arch/sparc/kernel/priq_sun4v.h new file mode 100644 index 000000000000..943c5ace2e07 --- /dev/null +++ b/arch/sparc/kernel/priq_sun4v.h @@ -0,0 +1,101 @@ +/* sun4v priq interfaces. Introduced in CoreS4.*/ +#ifndef __PRIQ_SUN4V_H__ +#define __PRIQ_SUN4V_H__ + +unsigned long priq_unconf(unsigned long id); +unsigned long priq_bind(unsigned long id, + unsigned long cpuid, + unsigned long pil); +unsigned long priq_bind_info(unsigned long id, + unsigned long *cpuid, + unsigned long *pil); +unsigned long priq_unbind(unsigned long id); +unsigned long priq_get_head_tail(unsigned long id, + unsigned long *head, + unsigned long *tail); +unsigned long priq_set_head(unsigned long id, + unsigned long head); +unsigned long priq_status(unsigned long id, + unsigned long *priq_status); +unsigned long priq_conf(unsigned long type, + unsigned long arg, + unsigned long raddr, + unsigned long nentries); +unsigned long priq_info(unsigned long id, + struct hv_priq_info *hv_priq_info); +unsigned long pci_priq_err_bind(unsigned long devhandle, + unsigned long errtype, + unsigned long priq_id); +unsigned long pci_pirq__err_info(unsigned long handle, + unsigned long errtype, + struct hv_pci_priq_err_info *err_info); +unsigned long pci_priq_err_unbind(unsigned long devhandle, + unsigned long errtype); +unsigned long pci_priq_err_enable(unsigned long devhandle, + unsigned long errtype); +unsigned long pci_err_disable(unsigned long devhandle, + unsigned long errtype); +unsigned long pci_priq_msi_bind(unsigned long devhandle, + unsigned long msinum, + unsigned long rid, + unsigned long priq_id); +unsigned long pci_priq_msi_info(unsigned long devhandle, + unsigned long msinum, + unsigned long rid, + struct hv_pci_msi_info *msi_info); +unsigned long pci_priq_msi_unbind(unsigned long devhandle, + unsigned long msinum, + unsigned long rid); +unsigned long pci_priq_msi_enable(unsigned long devhandle, + unsigned long msinum, + unsigned long rid); +unsigned long pci_priq_msi_disable(unsigned long devhandle, + unsigned long msinum, + unsigned long rid); +unsigned long pci_priq_msi_getstate(unsigned long devhandle, + unsigned long msinum, + unsigned long rid, + unsigned long *msistate); +unsigned long pci_priq_msi_setstate(unsigned long devhandle, + unsigned long msinum, + unsigned long rid, + unsigned long msistate); +unsigned long pci_priq_msg_bind(unsigned long devhandle, + unsigned long msgtype, + unsigned long priq_id); +unsigned long pci_priq_msg_info(unsigned long devhandle, + unsigned long msgtype, + struct hv_pci_priq_msg_info *msg_info); +unsigned long pci_priq_msg_unbind(unsigned long devhandle, + unsigned long msgtype); +unsigned long pci_priq_msg_enable(unsigned long devhandle, + unsigned long msgtype); +unsigned long pci_priq_msg_disable(unsigned long devhandle, + unsigned long msgtype); +unsigned long pci_priq_intx_bind(unsigned long devhandle, + unsigned long intx, + unsigned long priq_id); +unsigned long pci_priq_intx_info(unsigned long devhandle, + unsigned intx, + struct hv_pci_priq_intx_info *intx_info); +unsigned long pci_priq_intx_unbind(unsigned long devhandle, + unsigned long intx); +unsigned long pci_priq_intx_enable(unsigned long devhandle, + unsigned long intx); +unsigned long pci_priq_intx_disable(unsigned long devhandle, + unsigned long intx); +unsigned long pci_priq_intx_getstate(unsigned long intx, + unsigned long *intxstate); +unsigned long pci_priq_intx_setstate(unsigned long devhandle, + unsigned long intx, + unsigned long intxstate); + +void priq_percpu_setup(int cpu); +void cpu_handle_priqs(void); +void sun4v_priq(void); +struct pci_pbm_info; +int pci_sun4v_priq_msi_init(struct pci_pbm_info *pbm); +void priq_percpu_destroy(int cpu); +struct device_node; +int pci_priq_build_irq(unsigned int devhandle, unsigned int devino); +#endif /* !__PRIQ_SUN4V_H__*/ diff --git a/arch/sparc/kernel/priq_sun4v_asm.S b/arch/sparc/kernel/priq_sun4v_asm.S new file mode 100644 index 000000000000..b31b34ff2db4 --- /dev/null +++ b/arch/sparc/kernel/priq_sun4v_asm.S @@ -0,0 +1,463 @@ +#include +#include + + /* %o0: id + * + * returns %o0: status + */ +ENTRY(priq_unconf) + or %g0, HV_FAST_PRIQ_UNCONF, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(priq_unconf) + + /* %o0: id + * %o1: cpuid + * %o2: pil + * + * returns: %o0: status + */ +ENTRY(priq_bind) + or %g0, HV_FAST_PRIQ_BIND, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(priq_bind) + + /* %o0: id + * + * returns %o0: status + * %o1: cpuid + * %o2: pil + */ +ENTRY(priq_bind_info) + or %o1, 0, %g1 + or %o2, 0, %g2 + or %g0, HV_FAST_PRIQ_BIND_INFO, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1 + 0x00] + stx %o2, [%g2 + 0x00] + retl + nop + /* +ENDPROC(priq_bind_info); + + /* %o0: id + * + * returns %o0: status + */ +ENTRY(priq_unbind) + or %g0, HV_FAST_PRIQ_UNBIND, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(priq_unbind) + + /* %o0: id + * + * returns %o0: status + * %o1: head + * %o2: tail + */ +ENTRY(priq_get_head_tail) + or %g0, %o1, %g1 + or %g0, %o2, %g2 + or %g0, HV_FAST_PRIQ_GET_HEAD_TAIL, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1 + 0x00] + stx %o2, [%g2 + 0x00] + retl + nop +ENDPROC(priq_get_head_tail) + + /* %o0: id + * %o1: head + * + * returns %o0: status + */ +ENTRY(priq_set_head) + or %g0, HV_FAST_PRIQ_SET_HEAD, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(priq_set_head) + + /* %o0: id + * + * returns %o0: status + */ +ENTRY(priq_status) + or %g0, HV_FAST_PRIQ_STATUS, %o5 + ta HV_FAST_TRAP + brnz,pn %o0, 1f + sub %g0, %o0, %o0 + or %o1, %g0, %o0 +1: retl + nop +ENDPROC(priq_status) + + /* %o0: type + * %o1: arg + * %o2: raddr + * %o3: nentries + * + * returns %o0: status + */ +ENTRY(priq_conf) + or %g0, HV_FAST_PRIQ_CONF, %o5 + ta HV_FAST_TRAP + brnz,pn %o0, 1f + sub %g0, %o0, %o0 + or %o1, %g0, %o0 +1: retl + nop +ENDPROC(priq_conf) + + /* %o0: id + * + * returns %o0: status + * %o1: type + * %o2: arg + * %o3: raddr + * %o4: nentries + */ +ENTRY(priq_info) + or %o1, %g0, %g1 + or %g0, HV_FAST_PRIQ_INFO, %o5 + ta HV_FAST_TRAP + brnz,pn %o0, 1f + stx %o1, [%g1 + HV_PRIQ_INFO_TYPE_OFFSET] + stx %o2, [%g1 + HV_PRIQ_INFO_ARG_OFFSET] + stx %o3, [%g1 + HV_PRIQ_INFO_RADDR_OFFSET] + stx %o4, [%g1 + HV_PRIQ_INFO_NENTRIES_OFFSET] +1: retl + nop +ENDPROC(priq_info) + + /* %o0: devhandle + * %o1: errtype + * %o2: priq_id + * + * returns %o0: status + */ +ENTRY(pci_priq_err_bind) + or %g0, HV_FAST_PCI_PRIQ_ERR_BIND, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(pci_priq_err_bind) + + /* %o0: devhandle + * %o1: errtype + * + * returns %o0: status + * %o1: priq_id + * %o2: enabled + */ +ENTRY(pci_priq_err_info) + or %o2, %g0, %g1 + or %g0, HV_FAST_PCI_PRIQ_ERR_INFO, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1 + HV_PCI_PRIQ_ERR_INFO_PRIQ_ID_OFFSET] + stx %o2, [%g1 + HV_PCI_PRIQ_ERR_INFO_ENABLED_OFFSET] + retl + nop +ENDPROC(pci_priq_err_info) + + /* %o0: devhandle + * %o1: errtype + * + * returns %o0: status + */ +ENTRY(pci_priq_err_unbind) + or %g0, HV_FAST_PCI_PRIQ_ERR_UNBIND, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(pci_priq_err_unbind) + + /* %o0: devhandle + * %o1: errtype + * + * returns %o0: status + */ +ENTRY(pci_priq_err_enable) + or %g0, HV_FAST_PCI_PRIQ_ERR_ENABLE, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(pci_priq_err_enable) + + /* %o0: devhandle + * %o1: errtype + * + * returns %o0: status + */ +ENTRY(pci_priq_err_disable) + or %g0, HV_FAST_PCI_PRIQ_ERR_DISABLE, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(pci_priq_err_disable) + + /* %o0: devhandle + * %o1: msinum + * %o2: rid + * %o3: priq_id + * + * returns %o0: status + */ +ENTRY(pci_priq_msi_bind) + or %g0, HV_FAST_PCI_PRIQ_MSI_BIND, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(pci_priq_msi_bind) + + /* %o0: devhandle + * %o1: msinum + * %o2: rid + * + * returns %o0: status + * %o1: priq_id + * %o2: enabled + */ +ENTRY(pci_priq_msi_info) + or %o3, %g0, %g1 + or %g0, HV_FAST_PCI_PRIQ_MSI_INFO, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1 + HV_PCI_MSI_INFO_PRIQ_ID_OFFSET] + stx %o2, [%g1 + HV_PCI_MSI_INFO_PRIQ_ENABLE_OFFSET] + retl + nop +ENDPROC(pci_priq_msi_info) + + /* %o0: devhandle + * %o1: msinum + * %o2: rid + * + * returns %o0: status + */ +ENTRY(pci_priq_msi_unbind) + or %g0, HV_FAST_PCI_PRIQ_MSI_UNBIND, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(pci_priq_msi_unbind) + + /* %o0: devhandle + * %o1: msinum + * %o2: rid + * + * returns %o0: status + */ +ENTRY(pci_priq_msi_enable) + or %g0, HV_FAST_PCI_PRIQ_MSI_ENABLE, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(pci_priq_msi_enable) + + /* %o0: devhandle + * %o1: msinum + * %o2: rid + * + * returns %o0: status + */ +ENTRY(pci_priq_msi_disable) + or %g0, HV_FAST_PCI_PRIQ_MSI_DISABLE, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(pci_priq_msi_disable) + + /* %o0: devhandle + * %o1: msinum + * %o2: rid + * + * returns %o0: status + * %o1: msistate + */ +ENTRY(pci_priq_msi_getstate) + or %o1, %g0, %g1 + or %g0, HV_FAST_PCI_PRIQ_MSI_GETSTATE, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1 + 0x00] + retl + nop +ENDPROC(pci_priq_msi_getstate) + + /* %o0: devhandle + * %o1: msinum + * %o2: rid + * %o3: msistate + * + * returns %o0: status + */ +ENTRY(pci_priq_msi_setstate) + or %g0, HV_FAST_PCI_PRIQ_MSI_SETSTATE, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(pci_priq_msi_setstate) + + /* %o0: devhandle + * %o1: msgtype + * %o2: priq_id + * + * returns %o0: status + */ +ENTRY(pci_priq_msg_bind) + or %g0, HV_FAST_PCI_PRIQ_MSG_BIND, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(pci_priq_msg_bind) + + /* %o0: devhandle + * %o1: msgtype + * + * returns: %o0: status + * %o1: priq_id + * %o2: enabled + */ +ENTRY(pci_priq_msg_info) + or %o1, %g0, %g1 + or %g0, HV_FAST_PCI_PRIQ_MSG_INFO, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1 + HV_PCI_PRIQ_MSG_INFO_PRIQ_ID_OFFSET] + stx %o2, [%g1 + HV_PCI_PRIQ_MSG_INFO_ENABLED_OFFSET] + retl + nop +ENDPROC(pci_priq_msg_info) + + /* %o0: devhandle + * %o1: msgtype + * + * returns %o0: status + */ +ENTRY(pci_priq_msg_unbind) + or %g0, HV_FAST_PCI_PRIQ_MSG_UNBIND, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(pci_priq_msg_unbind) + + /* %o0: devhandle + * %o1: msgtype + * + * returns %o0: status + */ +ENTRY(pci_priq_msg_enable) + or %g0, HV_FAST_PCI_PRIQ_MSG_ENABLE, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(pci_priq_msg_enable) + + /* %o0: devhandle + * %o1: msgtype + * + * returns %o0: status + */ +ENTRY(pci_priq_msg_disable) + or %g0, HV_FAST_PCI_PRIQ_MSG_DISABLE, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(pci_priq_msg_disable) + + /* %o0: devhandle + * %o1: intx + * %o2: priq_id + * + * returns %o0: status + */ +ENTRY(pci_priq_intx_bind) + or %g0, HV_FAST_PCI_PRIQ_INTX_BIND, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(pci_priq_intx_bind) + + /* %o0: devhandle + * %o1: intx + * + * returns %o0: status + * %o1: priq_id + * %o2: enabled + */ +ENTRY(pci_priq_intx_info) + or %o2, %g0, %g1 + or %g0, HV_FAST_PCI_PRIQ_INTX_INFO, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1 + HV_PCI_PRIQ_INTX_INFO_PRIQ_ID_OFFSET] + stx %o2, [%g1 + HV_PCI_PRIQ_INTX_INFO_ENABLED_OFFSET] + retl + nop +ENDPROC(pci_priq_intx_info) + + /* %o0: devhandle + * %o1: intx + * + * returns %o0: status + */ +ENTRY(pci_priq_intx_unbind) + or %g0, HV_FAST_PCI_PRIQ_INTX_UNBIND, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(pci_priq_intx_unbind) + + /* %o0: devhandle + * %o1: intx + * + * returns %o0: status + */ +ENTRY(pci_priq_intx_enable) + or %g0, HV_FAST_PCI_PRIQ_INTX_ENABLE, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(pci_priq_intx_enable) + + /* %o0: devhandle + * %o1: intx + * + * returns %o0: status + */ +ENTRY(pci_priq_intx_disable) + or %g0, HV_FAST_PCI_PRIQ_INTX_DISABLE, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(pci_priq_intx_disable) + + /* %o0: devhandle + * %o1: intx + * + * returns %o0: status + * %o1: intxstate + */ +ENTRY(pci_priq_intx_getstate) + or %o1, %g0, %g1 + or %g0, HV_FAST_PCI_PRIQ_INTX_GETSTATE, %o5 + ta HV_FAST_TRAP + stx %o1, [%g1 + 0x00] + retl + nop +ENDPROC(pci_priq_intx_getstate) + + /* %o0: devhandle + * %o1: intx + * %o2: intxstate + * + * returns %o0: status + */ +ENTRY(pci_priq_intx_setstate) + or %g0, HV_FAST_PCI_PRIQ_INTX_SETSTATE, %o5 + ta HV_FAST_TRAP + retl + nop +ENDPROC(pci_priq_intx_setstate) diff --git a/arch/sparc/kernel/prom_irqtrans.c b/arch/sparc/kernel/prom_irqtrans.c index 40e4936bd479..670959661fd4 100644 --- a/arch/sparc/kernel/prom_irqtrans.c +++ b/arch/sparc/kernel/prom_irqtrans.c @@ -10,6 +10,7 @@ #include #include "prom.h" +#include "priq_sun4v.h" #ifdef CONFIG_PCI /* PSYCHO interrupt mapping support. */ @@ -459,7 +460,12 @@ static unsigned int pci_sun4v_irq_build(struct device_node *dp, unsigned int devino, void *_data) { - u32 devhandle = (u32) (unsigned long) _data; + u32 devhandle = (u32)(unsigned long)_data; + int irq; + + irq = pci_priq_build_irq(devhandle, devino); + if (irq) + return irq; return sun4v_build_irq(devhandle, devino); } diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c index ad22538a196d..7ddc17859a24 100644 --- a/arch/sparc/kernel/smp_64.c +++ b/arch/sparc/kernel/smp_64.c @@ -55,6 +55,7 @@ #include #include "cpumap.h" +#include "priq_sun4v.h" #include "kernel.h" DEFINE_PER_CPU(cpumask_t, cpu_sibling_map) = CPU_MASK_NONE; @@ -133,6 +134,8 @@ void smp_callin(void) /* idle thread is expected to have preempt disabled */ preempt_disable(); + priq_percpu_setup(cpuid); + local_irq_enable(); cpu_startup_entry(CPUHP_ONLINE); -- 2.50.1