]> www.infradead.org Git - users/jedix/linux-maple.git/commitdiff
qed: VFs to try utilizing the doorbell bar
authorMintz, Yuval <Yuval.Mintz@cavium.com>
Sun, 4 Jun 2017 10:31:07 +0000 (13:31 +0300)
committerChuck Anderson <chuck.anderson@oracle.com>
Tue, 19 Sep 2017 05:32:30 +0000 (22:32 -0700)
Orabug: 26783820

VFs are currently not mapping their doorbell bar, instead relying
on the small doorbell window they have in their limited regview bar.

In order to increase the number of possible Tx connections [queues]
employeed by VF past 16, we need to start using the doorbell bar if
one such is exposed - VF would communicate this fact to PF which would
return the size-bar internally configured into chip, according to
which the VF would decide whether to actually utilize the doorbell
bar.

Signed-off-by: Yuval Mintz <Yuval.Mintz@cavium.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
[ Upstream commit 1a850bfc9e71871599ddbc0d4d4cffa2dc409855 ]
Signed-off-by: Somasundaram Krishnasamy <somasundaram.krishnasamy@oracle.com>
drivers/net/ethernet/qlogic/qed/qed.h
drivers/net/ethernet/qlogic/qed/qed_dev.c
drivers/net/ethernet/qlogic/qed/qed_main.c
drivers/net/ethernet/qlogic/qed/qed_reg_addr.h
drivers/net/ethernet/qlogic/qed/qed_sriov.c
drivers/net/ethernet/qlogic/qed/qed_vf.c
drivers/net/ethernet/qlogic/qed/qed_vf.h
include/linux/qed/qed_if.h

index 738970ea89537bd72073f74bffe92ce926ee58da..35cb126350ff0b62e682b2e4cac41235937d8de8 100644 (file)
@@ -413,6 +413,11 @@ struct qed_fw_data {
        u32                     init_ops_size;
 };
 
+enum BAR_ID {
+       BAR_ID_0,               /* used for GRC */
+       BAR_ID_1                /* Used for doorbells */
+};
+
 #define DRV_MODULE_VERSION                   \
        __stringify(QED_MAJOR_VERSION) "."    \
        __stringify(QED_MINOR_VERSION) "."    \
index e30cccd0cf02fef7b178f3d784e826f2339ee49b..1ccc1fbc37951503606096b3a7199b9807e08218 100644 (file)
@@ -69,12 +69,6 @@ static DEFINE_SPINLOCK(qm_lock);
 #define QED_MIN_DPIS            (4)
 #define QED_MIN_PWM_REGION      (QED_WID_SIZE * QED_MIN_DPIS)
 
-/* API common to all protocols */
-enum BAR_ID {
-       BAR_ID_0,       /* used for GRC */
-       BAR_ID_1        /* Used for doorbells */
-};
-
 static u32 qed_hw_bar_size(struct qed_hwfn *p_hwfn,
                           struct qed_ptt *p_ptt, enum BAR_ID bar_id)
 {
@@ -83,7 +77,7 @@ static u32 qed_hw_bar_size(struct qed_hwfn *p_hwfn,
        u32 val;
 
        if (IS_VF(p_hwfn->cdev))
-               return 1 << 17;
+               return qed_vf_hw_bar_size(p_hwfn, bar_id);
 
        val = qed_rd(p_hwfn, p_ptt, bar_reg);
        if (val)
index 4d8b132fddfce9ec58fc363e81c2e23ba2714921..ddaa5bd7daa0cf8f7cf95e9631fd37e4084a083a 100644 (file)
@@ -122,7 +122,7 @@ static void qed_free_pci(struct qed_dev *cdev)
 {
        struct pci_dev *pdev = cdev->pdev;
 
-       if (cdev->doorbells)
+       if (cdev->doorbells && cdev->db_size)
                iounmap(cdev->doorbells);
        if (cdev->regview)
                iounmap(cdev->regview);
@@ -206,16 +206,24 @@ static int qed_init_pci(struct qed_dev *cdev, struct pci_dev *pdev)
                goto err2;
        }
 
-       if (IS_PF(cdev)) {
-               cdev->db_phys_addr = pci_resource_start(cdev->pdev, 2);
-               cdev->db_size = pci_resource_len(cdev->pdev, 2);
-               cdev->doorbells = ioremap_wc(cdev->db_phys_addr, cdev->db_size);
-               if (!cdev->doorbells) {
-                       DP_NOTICE(cdev, "Cannot map doorbell space\n");
-                       return -ENOMEM;
+       cdev->db_phys_addr = pci_resource_start(cdev->pdev, 2);
+       cdev->db_size = pci_resource_len(cdev->pdev, 2);
+       if (!cdev->db_size) {
+               if (IS_PF(cdev)) {
+                       DP_NOTICE(cdev, "No Doorbell bar available\n");
+                       return -EINVAL;
+               } else {
+                       return 0;
                }
        }
 
+       cdev->doorbells = ioremap_wc(cdev->db_phys_addr, cdev->db_size);
+
+       if (!cdev->doorbells) {
+               DP_NOTICE(cdev, "Cannot map doorbell space\n");
+               return -ENOMEM;
+       }
+
        return 0;
 
 err2:
index 67172d7a786849dec03b7a802cb3f61edaa4fdec..7e4639c9207a8bc5a7f745db192e53c1c4930c8e 100644 (file)
        0x2aae60UL
 #define PGLUE_B_REG_PF_BAR1_SIZE \
        0x2aae64UL
+#define PGLUE_B_REG_VF_BAR1_SIZE 0x2aae68UL
 #define PRS_REG_ENCAPSULATION_TYPE_EN  0x1f0730UL
 #define PRS_REG_GRE_PROTOCOL           0x1f0734UL
 #define PRS_REG_VXLAN_PORT             0x1f0738UL
index 1f55f6a6e6efc8d2e21c7eeda0a6106e20645364..069019e2f0bfbd03b16e8dfb6b51b9d32be31cb1 100644 (file)
@@ -1377,6 +1377,60 @@ static void qed_iov_vf_cleanup(struct qed_hwfn *p_hwfn,
        qed_iov_clean_vf(p_hwfn, p_vf->relative_vf_id);
 }
 
+/* Returns either 0, or log(size) */
+static u32 qed_iov_vf_db_bar_size(struct qed_hwfn *p_hwfn,
+                                 struct qed_ptt *p_ptt)
+{
+       u32 val = qed_rd(p_hwfn, p_ptt, PGLUE_B_REG_VF_BAR1_SIZE);
+
+       if (val)
+               return val + 11;
+       return 0;
+}
+
+static void
+qed_iov_vf_mbx_acquire_resc_cids(struct qed_hwfn *p_hwfn,
+                                struct qed_ptt *p_ptt,
+                                struct qed_vf_info *p_vf,
+                                struct vf_pf_resc_request *p_req,
+                                struct pf_vf_resc *p_resp)
+{
+       u8 num_vf_cons = p_hwfn->pf_params.eth_pf_params.num_vf_cons;
+       u8 db_size = qed_db_addr_vf(1, DQ_DEMS_LEGACY) -
+                    qed_db_addr_vf(0, DQ_DEMS_LEGACY);
+       u32 bar_size;
+
+       p_resp->num_cids = min_t(u8, p_req->num_cids, num_vf_cons);
+
+       /* If VF didn't bother asking for QIDs than don't bother limiting
+        * number of CIDs. The VF doesn't care about the number, and this
+        * has the likely result of causing an additional acquisition.
+        */
+       if (!(p_vf->acquire.vfdev_info.capabilities &
+             VFPF_ACQUIRE_CAP_QUEUE_QIDS))
+               return;
+
+       /* If doorbell bar was mapped by VF, limit the VF CIDs to an amount
+        * that would make sure doorbells for all CIDs fall within the bar.
+        * If it doesn't, make sure regview window is sufficient.
+        */
+       if (p_vf->acquire.vfdev_info.capabilities &
+           VFPF_ACQUIRE_CAP_PHYSICAL_BAR) {
+               bar_size = qed_iov_vf_db_bar_size(p_hwfn, p_ptt);
+               if (bar_size)
+                       bar_size = 1 << bar_size;
+
+               if (p_hwfn->cdev->num_hwfns > 1)
+                       bar_size /= 2;
+       } else {
+               bar_size = PXP_VF_BAR0_DQ_LENGTH;
+       }
+
+       if (bar_size / db_size < 256)
+               p_resp->num_cids = min_t(u8, p_resp->num_cids,
+                                        (u8)(bar_size / db_size));
+}
+
 static u8 qed_iov_vf_mbx_acquire_resc(struct qed_hwfn *p_hwfn,
                                      struct qed_ptt *p_ptt,
                                      struct qed_vf_info *p_vf,
@@ -1410,6 +1464,8 @@ static u8 qed_iov_vf_mbx_acquire_resc(struct qed_hwfn *p_hwfn,
        p_resp->num_vlan_filters = min_t(u8, p_vf->num_vlan_filters,
                                         p_req->num_vlan_filters);
 
+       qed_iov_vf_mbx_acquire_resc_cids(p_hwfn, p_ptt, p_vf, p_req, p_resp);
+
        /* This isn't really needed/enforced, but some legacy VFs might depend
         * on the correct filling of this field.
         */
@@ -1552,6 +1608,9 @@ static void qed_iov_vf_mbx_acquire(struct qed_hwfn *p_hwfn,
        if (p_hwfn->cdev->num_hwfns > 1)
                pfdev_info->capabilities |= PFVF_ACQUIRE_CAP_100G;
 
+       /* Share the sizes of the bars with VF */
+       resp->pfdev_info.bar_size = qed_iov_vf_db_bar_size(p_hwfn, p_ptt);
+
        qed_iov_vf_mbx_acquire_stats(p_hwfn, &pfdev_info->stats_info);
 
        memcpy(pfdev_info->port_mac, p_hwfn->hw_info.hw_mac_addr, ETH_ALEN);
index 8afe4dc01fcec56e20053fc8f6794bc8fbdd9088..4ddc89e1f00bc3b2ec13ba2a8639805401b601c2 100644 (file)
@@ -153,6 +153,61 @@ static int qed_send_msg2pf(struct qed_hwfn *p_hwfn, u8 *done, u32 resp_size)
        return rc;
 }
 
+int _qed_vf_pf_release(struct qed_hwfn *p_hwfn, bool b_final)
+{
+       struct qed_vf_iov *p_iov = p_hwfn->vf_iov_info;
+       struct pfvf_def_resp_tlv *resp;
+       struct vfpf_first_tlv *req;
+       u32 size;
+       int rc;
+
+       /* clear mailbox and prep first tlv */
+       req = qed_vf_pf_prep(p_hwfn, CHANNEL_TLV_RELEASE, sizeof(*req));
+
+       /* add list termination tlv */
+       qed_add_tlv(p_hwfn, &p_iov->offset,
+                   CHANNEL_TLV_LIST_END, sizeof(struct channel_list_end_tlv));
+
+       resp = &p_iov->pf2vf_reply->default_resp;
+       rc = qed_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp));
+
+       if (!rc && resp->hdr.status != PFVF_STATUS_SUCCESS)
+               rc = -EAGAIN;
+
+       qed_vf_pf_req_end(p_hwfn, rc);
+       if (!b_final)
+               return rc;
+
+       p_hwfn->b_int_enabled = 0;
+
+       if (p_iov->vf2pf_request)
+               dma_free_coherent(&p_hwfn->cdev->pdev->dev,
+                                 sizeof(union vfpf_tlvs),
+                                 p_iov->vf2pf_request,
+                                 p_iov->vf2pf_request_phys);
+       if (p_iov->pf2vf_reply)
+               dma_free_coherent(&p_hwfn->cdev->pdev->dev,
+                                 sizeof(union pfvf_tlvs),
+                                 p_iov->pf2vf_reply, p_iov->pf2vf_reply_phys);
+
+       if (p_iov->bulletin.p_virt) {
+               size = sizeof(struct qed_bulletin_content);
+               dma_free_coherent(&p_hwfn->cdev->pdev->dev,
+                                 size,
+                                 p_iov->bulletin.p_virt, p_iov->bulletin.phys);
+       }
+
+       kfree(p_hwfn->vf_iov_info);
+       p_hwfn->vf_iov_info = NULL;
+
+       return rc;
+}
+
+int qed_vf_pf_release(struct qed_hwfn *p_hwfn)
+{
+       return _qed_vf_pf_release(p_hwfn, true);
+}
+
 #define VF_ACQUIRE_THRESH 3
 static void qed_vf_pf_acquire_reduce_resc(struct qed_hwfn *p_hwfn,
                                          struct vf_pf_resc_request *p_req,
@@ -216,6 +271,11 @@ static int qed_vf_pf_acquire(struct qed_hwfn *p_hwfn)
        /* Fill capability field with any non-deprecated config we support */
        req->vfdev_info.capabilities |= VFPF_ACQUIRE_CAP_100G;
 
+       /* If we've mapped the doorbell bar, try using queue qids */
+       if (p_iov->b_doorbell_bar)
+               req->vfdev_info.capabilities |= VFPF_ACQUIRE_CAP_PHYSICAL_BAR |
+                                               VFPF_ACQUIRE_CAP_QUEUE_QIDS;
+
        /* pf 2 vf bulletin board address */
        req->bulletin_addr = p_iov->bulletin.phys;
        req->bulletin_size = p_iov->bulletin.size;
@@ -338,10 +398,27 @@ exit:
        return rc;
 }
 
+u32 qed_vf_hw_bar_size(struct qed_hwfn *p_hwfn, enum BAR_ID bar_id)
+{
+       u32 bar_size;
+
+       /* Regview size is fixed */
+       if (bar_id == BAR_ID_0)
+               return 1 << 17;
+
+       /* Doorbell is received from PF */
+       bar_size = p_hwfn->vf_iov_info->acquire_resp.pfdev_info.bar_size;
+       if (bar_size)
+               return 1 << bar_size;
+       return 0;
+}
+
 int qed_vf_hw_prepare(struct qed_hwfn *p_hwfn)
 {
+       struct qed_hwfn *p_lead = QED_LEADING_HWFN(p_hwfn->cdev);
        struct qed_vf_iov *p_iov;
        u32 reg;
+       int rc;
 
        /* Set number of hwfns - might be overriden once leading hwfn learns
         * actual configuration from PF.
@@ -349,10 +426,6 @@ int qed_vf_hw_prepare(struct qed_hwfn *p_hwfn)
        if (IS_LEAD_HWFN(p_hwfn))
                p_hwfn->cdev->num_hwfns = 1;
 
-       /* Set the doorbell bar. Assumption: regview is set */
-       p_hwfn->doorbells = (u8 __iomem *)p_hwfn->regview +
-                                         PXP_VF_BAR0_START_DQ;
-
        reg = PXP_VF_BAR0_ME_OPAQUE_ADDRESS;
        p_hwfn->hw_info.opaque_fid = (u16)REG_RD(p_hwfn, reg);
 
@@ -364,6 +437,30 @@ int qed_vf_hw_prepare(struct qed_hwfn *p_hwfn)
        if (!p_iov)
                return -ENOMEM;
 
+       /* Doorbells are tricky; Upper-layer has alreday set the hwfn doorbell
+        * value, but there are several incompatibily scenarios where that
+        * would be incorrect and we'd need to override it.
+        */
+       if (!p_hwfn->doorbells) {
+               p_hwfn->doorbells = (u8 __iomem *)p_hwfn->regview +
+                                                 PXP_VF_BAR0_START_DQ;
+       } else if (p_hwfn == p_lead) {
+               /* For leading hw-function, value is always correct, but need
+                * to handle scenario where legacy PF would not support 100g
+                * mapped bars later.
+                */
+               p_iov->b_doorbell_bar = true;
+       } else {
+               /* here, value would be correct ONLY if the leading hwfn
+                * received indication that mapped-bars are supported.
+                */
+               if (p_lead->vf_iov_info->b_doorbell_bar)
+                       p_iov->b_doorbell_bar = true;
+               else
+                       p_hwfn->doorbells = (u8 __iomem *)
+                           p_hwfn->regview + PXP_VF_BAR0_START_DQ;
+       }
+
        /* Allocate vf2pf msg */
        p_iov->vf2pf_request = dma_alloc_coherent(&p_hwfn->cdev->pdev->dev,
                                                  sizeof(union vfpf_tlvs),
@@ -403,7 +500,33 @@ int qed_vf_hw_prepare(struct qed_hwfn *p_hwfn)
 
        p_hwfn->hw_info.personality = QED_PCI_ETH;
 
-       return qed_vf_pf_acquire(p_hwfn);
+       rc = qed_vf_pf_acquire(p_hwfn);
+
+       /* If VF is 100g using a mapped bar and PF is too old to support that,
+        * acquisition would succeed - but the VF would have no way knowing
+        * the size of the doorbell bar configured in HW and thus will not
+        * know how to split it for 2nd hw-function.
+        * In this case we re-try without the indication of the mapped
+        * doorbell.
+        */
+       if (!rc && p_iov->b_doorbell_bar &&
+           !qed_vf_hw_bar_size(p_hwfn, BAR_ID_1) &&
+           (p_hwfn->cdev->num_hwfns > 1)) {
+               rc = _qed_vf_pf_release(p_hwfn, false);
+               if (rc)
+                       return rc;
+
+               p_iov->b_doorbell_bar = false;
+               p_hwfn->doorbells = (u8 __iomem *)p_hwfn->regview +
+                                                 PXP_VF_BAR0_START_DQ;
+               rc = qed_vf_pf_acquire(p_hwfn);
+       }
+
+       DP_VERBOSE(p_hwfn, QED_MSG_IOV,
+                  "Regview [%p], Doorbell [%p], Device-doorbell [%p]\n",
+                  p_hwfn->regview, p_hwfn->doorbells, p_hwfn->cdev->doorbells);
+
+       return rc;
 
 free_vf2pf_request:
        dma_free_coherent(&p_hwfn->cdev->pdev->dev,
@@ -1090,54 +1213,6 @@ exit:
        return rc;
 }
 
-int qed_vf_pf_release(struct qed_hwfn *p_hwfn)
-{
-       struct qed_vf_iov *p_iov = p_hwfn->vf_iov_info;
-       struct pfvf_def_resp_tlv *resp;
-       struct vfpf_first_tlv *req;
-       u32 size;
-       int rc;
-
-       /* clear mailbox and prep first tlv */
-       req = qed_vf_pf_prep(p_hwfn, CHANNEL_TLV_RELEASE, sizeof(*req));
-
-       /* add list termination tlv */
-       qed_add_tlv(p_hwfn, &p_iov->offset,
-                   CHANNEL_TLV_LIST_END, sizeof(struct channel_list_end_tlv));
-
-       resp = &p_iov->pf2vf_reply->default_resp;
-       rc = qed_send_msg2pf(p_hwfn, &resp->hdr.status, sizeof(*resp));
-
-       if (!rc && resp->hdr.status != PFVF_STATUS_SUCCESS)
-               rc = -EAGAIN;
-
-       qed_vf_pf_req_end(p_hwfn, rc);
-
-       p_hwfn->b_int_enabled = 0;
-
-       if (p_iov->vf2pf_request)
-               dma_free_coherent(&p_hwfn->cdev->pdev->dev,
-                                 sizeof(union vfpf_tlvs),
-                                 p_iov->vf2pf_request,
-                                 p_iov->vf2pf_request_phys);
-       if (p_iov->pf2vf_reply)
-               dma_free_coherent(&p_hwfn->cdev->pdev->dev,
-                                 sizeof(union pfvf_tlvs),
-                                 p_iov->pf2vf_reply, p_iov->pf2vf_reply_phys);
-
-       if (p_iov->bulletin.p_virt) {
-               size = sizeof(struct qed_bulletin_content);
-               dma_free_coherent(&p_hwfn->cdev->pdev->dev,
-                                 size,
-                                 p_iov->bulletin.p_virt, p_iov->bulletin.phys);
-       }
-
-       kfree(p_hwfn->vf_iov_info);
-       p_hwfn->vf_iov_info = NULL;
-
-       return rc;
-}
-
 void qed_vf_pf_filter_mcast(struct qed_hwfn *p_hwfn,
                            struct qed_filter_mcast *p_filter_cmd)
 {
index d7b9c90b2f604f91e7127a708fd51721f0190c1d..794c0eefc1aff8922e98cbfbe2dcf569edca66f6 100644 (file)
@@ -46,6 +46,7 @@ struct vf_pf_resc_request {
        u8 num_mac_filters;
        u8 num_vlan_filters;
        u8 num_mc_filters;
+       u8 num_cids;
        u16 padding;
 };
 
@@ -113,6 +114,17 @@ struct vfpf_acquire_tlv {
        struct vf_pf_vfdev_info {
 #define VFPF_ACQUIRE_CAP_PRE_FP_HSI     (1 << 0) /* VF pre-FP hsi version */
 #define VFPF_ACQUIRE_CAP_100G          (1 << 1) /* VF can support 100g */
+       /* A requirement for supporting multi-Tx queues on a single queue-zone,
+        * VF would pass qids as additional information whenever passing queue
+        * references.
+        */
+#define VFPF_ACQUIRE_CAP_QUEUE_QIDS     BIT(2)
+
+       /* The VF is using the physical bar. While this is mostly internal
+        * to the VF, might affect the number of CIDs supported assuming
+        * QUEUE_QIDS is set.
+        */
+#define VFPF_ACQUIRE_CAP_PHYSICAL_BAR   BIT(3)
                u64 capabilities;
                u8 fw_major;
                u8 fw_minor;
@@ -193,7 +205,8 @@ struct pfvf_acquire_resp_tlv {
                u16 chip_rev;
                u8 dev_type;
 
-               u8 padding;
+               /* Doorbell bar size configured in HW: log(size) or 0 */
+               u8 bar_size;
 
                struct pfvf_stats_info stats_info;
 
@@ -221,6 +234,7 @@ struct pfvf_acquire_resp_tlv {
                u8 num_mac_filters;
                u8 num_vlan_filters;
                u8 num_mc_filters;
+               u8 num_cids;
                u8 padding[2];
        } resc;
 
@@ -635,6 +649,11 @@ struct qed_vf_iov {
         * compatibility [with older PFs] we'd still need to store these.
         */
        struct qed_sb_info *sbs_info[PFVF_MAX_SBS_PER_VF];
+
+       /* Determines whether VF utilizes doorbells via limited register
+        * bar or via the doorbell bar.
+        */
+       bool b_doorbell_bar;
 };
 
 #ifdef CONFIG_QED_SRIOV
@@ -943,6 +962,8 @@ void qed_iov_vf_task(struct work_struct *work);
 void qed_vf_set_vf_start_tunn_update_param(struct qed_tunnel_info *p_tun);
 int qed_vf_pf_tunnel_param_update(struct qed_hwfn *p_hwfn,
                                  struct qed_tunnel_info *p_tunn);
+
+u32 qed_vf_hw_bar_size(struct qed_hwfn *p_hwfn, enum BAR_ID bar_id);
 #else
 static inline void qed_vf_get_link_params(struct qed_hwfn *p_hwfn,
                                          struct qed_mcp_link_params *params)
@@ -1119,6 +1140,13 @@ static inline int qed_vf_pf_tunnel_param_update(struct qed_hwfn *p_hwfn,
 {
        return -EINVAL;
 }
+
+static inline u32
+qed_vf_hw_bar_size(struct qed_hwfn  *p_hwfn,
+                  enum BAR_ID bar_id)
+{
+       return 0;
+}
 #endif
 
 #endif
index e7c7946a8f3ab50dd8896122f937d598578f5946..28f5a5e4b4d27d0e47e766a52b9ab0100fc74754 100644 (file)
@@ -185,6 +185,9 @@ struct qed_eth_pf_params {
         */
        u16 num_cons;
 
+       /* per-VF number of CIDs */
+       u8 num_vf_cons;
+
        /* To enable arfs, previous to HW-init a positive number needs to be
         * set [as filters require allocated searcher ILT memory].
         * This will set the maximal number of configured steering-filters.