be2net: Add functionality to support RoCE driver

- Increase MSI-X vectors by 5 for RoCE traffic.
- Add macro to check roce support on a device.
- Add device-specific doorbell and MSI-X vector fields shared with NIC
  functionality.
- Provide RoCE driver registration and deregistration functions.
- Add support functions which will be invoked on adapter add/remove
  and port up/down events.
- Traverse through the list of adapters to invoke callback functions.

Signed-off-by: Parav Pandit <parav.pandit@emulex.com>
Acked-by: David S. Miller <davem@davemloft.net>
Signed-off-by: Roland Dreier <roland@purestorage.com>

diff --git a/drivers/net/benet/Makefile b/drivers/net/benet/Makefile
index a60cd8051135356c076cc0fbdc4878668063e584..1a91b276940d2203b65d9e210f13154cc1515cd3 100644 (file)
--- a/drivers/net/benet/Makefile
+++ b/drivers/net/benet/Makefile
@@ -4,4 +4,4 @@
 
 obj-$(CONFIG_BE2NET) += be2net.o
 
-be2net-y :=  be_main.o be_cmds.o be_ethtool.o
+be2net-y :=  be_main.o be_cmds.o be_ethtool.o be_roce.o

diff --git a/drivers/net/benet/be.h b/drivers/net/benet/be.h
index e8d86d333e268a69726c219d6725eb23c7772594..963bbf840ff0cefd2f8e1f5f3f952fcad0f3b7a3 100644 (file)
--- a/drivers/net/benet/be.h
+++ b/drivers/net/benet/be.h
@@ -33,6 +33,7 @@
 #include <linux/u64_stats_sync.h>
 
 #include "be_hw.h"
+#include "be_roce.h"
 
 #define DRV_VER                        "4.2.220o"
 #define DRV_NAME               "be2net"
@@ -103,7 +104,8 @@ static inline char *nic_name(struct pci_dev *pdev)
 #define MAX_RX_QS              (MAX_RSS_QS + 1) /* RSS qs + 1 def Rx */
 
 #define MAX_TX_QS              8
-#define MAX_MSIX_VECTORS       MAX_RSS_QS
+#define MAX_ROCE_EQS           5
+#define MAX_MSIX_VECTORS       (MAX_RSS_QS + MAX_ROCE_EQS) /* RSS qs + RoCE */
 #define BE_TX_BUDGET           256
 #define BE_NAPI_WEIGHT         64
 #define MAX_RX_POST            BE_NAPI_WEIGHT /* Frags posted at a time */
@@ -398,6 +400,17 @@ struct be_adapter {
        u32 tx_fc;              /* Tx flow control */
        bool stats_cmd_sent;
        u8 generation;          /* BladeEngine ASIC generation */
+       u32 if_type;
+       struct {
+               u8 __iomem *base;       /* Door Bell */
+               u32 size;
+               u32 total_size;
+               u64 io_addr;
+       } roce_db;
+       u32 num_msix_roce_vec;
+       struct ocrdma_dev *ocrdma_dev;
+       struct list_head entry;
+
        u32 flash_status;
        struct completion flash_compl;
 
@@ -428,6 +441,10 @@ struct be_adapter {
 #define lancer_chip(adapter)   ((adapter->pdev->device == OC_DEVICE_ID3) || \
                                 (adapter->pdev->device == OC_DEVICE_ID4))
 
+#define be_roce_supported(adapter) ((adapter->if_type == SLI_INTF_TYPE_3 || \
+                               adapter->sli_family == SKYHAWK_SLI_FAMILY) && \
+                               (adapter->function_mode & RDMA_ENABLED))
+
 extern const struct ethtool_ops be_ethtool_ops;
 
 #define msix_enabled(adapter)          (adapter->num_msix_vec > 0)
@@ -592,6 +609,12 @@ static inline bool be_is_wol_excluded(struct be_adapter *adapter)
        }
 }
 
+static inline bool be_type_2_3(struct be_adapter *adapter)
+{
+       return (adapter->if_type == SLI_INTF_TYPE_2 ||
+               adapter->if_type == SLI_INTF_TYPE_3) ? true : false;
+}
+
 extern void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm,
                u16 num_popped);
 extern void be_link_status_update(struct be_adapter *adapter, u8 link_status);
@@ -599,4 +622,17 @@ extern void be_parse_stats(struct be_adapter *adapter);
 extern int be_load_fw(struct be_adapter *adapter, u8 *func);
 extern bool be_is_wol_supported(struct be_adapter *adapter);
 extern bool be_pause_supported(struct be_adapter *adapter);
+
+/*
+ * internal function to initialize-cleanup roce device.
+ */
+extern void be_roce_dev_add(struct be_adapter *);
+extern void be_roce_dev_remove(struct be_adapter *);
+
+/*
+ * internal function to open-close roce device during ifup-ifdown.
+ */
+extern void be_roce_dev_open(struct be_adapter *);
+extern void be_roce_dev_close(struct be_adapter *);
+
 #endif                         /* BE_H */

diff --git a/drivers/net/benet/be_cmds.h b/drivers/net/benet/be_cmds.h
index 3766269786b5b3dc764587ad426efb9eb91c6a8c..fed54467f6e9d3ce5395a5beb87e031159b67f3c 100644 (file)
--- a/drivers/net/benet/be_cmds.h
+++ b/drivers/net/benet/be_cmds.h
@@ -1060,6 +1060,7 @@ struct be_cmd_resp_modify_eq_delay {
 /* The HW can come up in either of the following multi-channel modes
  * based on the skew/IPL.
  */
+#define RDMA_ENABLED                           0x4
 #define FLEX10_MODE                            0x400
 #define VNIC_MODE                              0x20000
 #define UMC_ENABLED                            0x1000000

diff --git a/drivers/net/benet/be_hw.h b/drivers/net/benet/be_hw.h
index 0949aa609164edc4b01610aabeafbdca064449c9..7c8a710eac2fa6c3da9e633da2d124373c739db8 100644 (file)
--- a/drivers/net/benet/be_hw.h
+++ b/drivers/net/benet/be_hw.h
@@ -98,11 +98,13 @@
 #define SLI_INTF_REV_SHIFT                     4
 #define SLI_INTF_FT_MASK                       0x00000001
 
+#define SLI_INTF_TYPE_2                2
+#define SLI_INTF_TYPE_3                3
 
 /* SLI family */
 #define BE_SLI_FAMILY          0x0
 #define LANCER_A0_SLI_FAMILY   0xA
-
+#define SKYHAWK_SLI_FAMILY      0x2
 
 /********* ISR0 Register offset **********/
 #define CEV_ISR0_OFFSET                        0xC18

diff --git a/drivers/net/benet/be_main.c b/drivers/net/benet/be_main.c
index 79daca3758aeb1fe823c8721e8c19870ec0f64c3..378619b544886560e4b90f42dc18a5ba69d38716 100644 (file)
--- a/drivers/net/benet/be_main.c
+++ b/drivers/net/benet/be_main.c
@@ -2100,10 +2100,17 @@ static uint be_num_rss_want(struct be_adapter *adapter)
 static void be_msix_enable(struct be_adapter *adapter)
 {
 #define BE_MIN_MSIX_VECTORS            1
-       int i, status, num_vec;
+       int i, status, num_vec, num_roce_vec = 0;
 
        /* If RSS queues are not used, need a vec for default RX Q */
        num_vec = min(be_num_rss_want(adapter), num_online_cpus());
+       if (be_roce_supported(adapter)) {
+               num_roce_vec = min_t(u32, MAX_ROCE_MSIX_VECTORS,
+                                       (num_online_cpus() + 1));
+               num_roce_vec = min(num_roce_vec, MAX_ROCE_EQS);
+               num_vec += num_roce_vec;
+               num_vec = min(num_vec, MAX_MSIX_VECTORS);
+       }
        num_vec = max(num_vec, BE_MIN_MSIX_VECTORS);
 
        for (i = 0; i < num_vec; i++)
@@ -2120,7 +2127,17 @@ static void be_msix_enable(struct be_adapter *adapter)
        }
        return;
 done:
-       adapter->num_msix_vec = num_vec;
+       if (be_roce_supported(adapter)) {
+               if (num_vec > num_roce_vec) {
+                       adapter->num_msix_vec = num_vec - num_roce_vec;
+                       adapter->num_msix_roce_vec =
+                               num_vec - adapter->num_msix_vec;
+               } else {
+                       adapter->num_msix_vec = num_vec;
+                       adapter->num_msix_roce_vec = 0;
+               }
+       } else
+               adapter->num_msix_vec = num_vec;
        return;
 }
 
@@ -2279,6 +2296,8 @@ static int be_close(struct net_device *netdev)
        struct be_eq_obj *eqo;
        int i;
 
+       be_roce_dev_close(adapter);
+
        be_async_mcc_disable(adapter);
 
        if (!lancer_chip(adapter))
@@ -2388,6 +2407,7 @@ static int be_open(struct net_device *netdev)
        if (!status)
                be_link_status_update(adapter, link_status);
 
+       be_roce_dev_open(adapter);
        return 0;
 err:
        be_close(adapter->netdev);
@@ -3150,6 +3170,24 @@ static void be_unmap_pci_bars(struct be_adapter *adapter)
                iounmap(adapter->csr);
        if (adapter->db)
                iounmap(adapter->db);
+       if (adapter->roce_db.base)
+               pci_iounmap(adapter->pdev, adapter->roce_db.base);
+}
+
+static int lancer_roce_map_pci_bars(struct be_adapter *adapter)
+{
+       struct pci_dev *pdev = adapter->pdev;
+       u8 __iomem *addr;
+
+       addr = pci_iomap(pdev, 2, 0);
+       if (addr == NULL)
+               return -ENOMEM;
+
+       adapter->roce_db.base = addr;
+       adapter->roce_db.io_addr = pci_resource_start(pdev, 2);
+       adapter->roce_db.size = 8192;
+       adapter->roce_db.total_size = pci_resource_len(pdev, 2);
+       return 0;
 }
 
 static int be_map_pci_bars(struct be_adapter *adapter)
@@ -3158,11 +3196,18 @@ static int be_map_pci_bars(struct be_adapter *adapter)
        int db_reg;
 
        if (lancer_chip(adapter)) {
-               addr = ioremap_nocache(pci_resource_start(adapter->pdev, 0),
-                       pci_resource_len(adapter->pdev, 0));
-               if (addr == NULL)
-                       return -ENOMEM;
-               adapter->db = addr;
+               if (be_type_2_3(adapter)) {
+                       addr = ioremap_nocache(
+                                       pci_resource_start(adapter->pdev, 0),
+                                       pci_resource_len(adapter->pdev, 0));
+                       if (addr == NULL)
+                               return -ENOMEM;
+                       adapter->db = addr;
+               }
+               if (adapter->if_type == SLI_INTF_TYPE_3) {
+                       if (lancer_roce_map_pci_bars(adapter))
+                               goto pci_map_err;
+               }
                return 0;
        }
 
@@ -3187,14 +3232,19 @@ static int be_map_pci_bars(struct be_adapter *adapter)
        if (addr == NULL)
                goto pci_map_err;
        adapter->db = addr;
-
+       if (adapter->sli_family == SKYHAWK_SLI_FAMILY) {
+               adapter->roce_db.size = 4096;
+               adapter->roce_db.io_addr =
+                               pci_resource_start(adapter->pdev, db_reg);
+               adapter->roce_db.total_size =
+                               pci_resource_len(adapter->pdev, db_reg);
+       }
        return 0;
 pci_map_err:
        be_unmap_pci_bars(adapter);
        return -ENOMEM;
 }
 
-
 static void be_ctrl_cleanup(struct be_adapter *adapter)
 {
        struct be_dma_mem *mem = &adapter->mbox_mem_alloced;
@@ -3302,6 +3352,8 @@ static void __devexit be_remove(struct pci_dev *pdev)
 
        cancel_delayed_work_sync(&adapter->work);
 
+       be_roce_dev_remove(adapter);
+
        unregister_netdev(adapter->netdev);
 
        be_clear(adapter);
@@ -3369,17 +3421,27 @@ static int be_dev_family_check(struct be_adapter *adapter)
                break;
        case BE_DEVICE_ID2:
        case OC_DEVICE_ID2:
-       case OC_DEVICE_ID5:
                adapter->generation = BE_GEN3;
                break;
        case OC_DEVICE_ID3:
        case OC_DEVICE_ID4:
                pci_read_config_dword(pdev, SLI_INTF_REG_OFFSET, &sli_intf);
+               adapter->if_type = (sli_intf & SLI_INTF_IF_TYPE_MASK) >>
+                                               SLI_INTF_IF_TYPE_SHIFT;
                if_type = (sli_intf & SLI_INTF_IF_TYPE_MASK) >>
                                                SLI_INTF_IF_TYPE_SHIFT;
-
                if (((sli_intf & SLI_INTF_VALID_MASK) != SLI_INTF_VALID) ||
-                       if_type != 0x02) {
+                       !be_type_2_3(adapter)) {
+                       dev_err(&pdev->dev, "SLI_INTF reg val is not valid\n");
+                       return -EINVAL;
+               }
+               adapter->sli_family = ((sli_intf & SLI_INTF_FAMILY_MASK) >>
+                                        SLI_INTF_FAMILY_SHIFT);
+               adapter->generation = BE_GEN3;
+               break;
+       case OC_DEVICE_ID5:
+               pci_read_config_dword(pdev, SLI_INTF_REG_OFFSET, &sli_intf);
+               if ((sli_intf & SLI_INTF_VALID_MASK) != SLI_INTF_VALID) {
                        dev_err(&pdev->dev, "SLI_INTF reg val is not valid\n");
                        return -EINVAL;
                }
@@ -3639,6 +3701,8 @@ static int __devinit be_probe(struct pci_dev *pdev,
        if (status != 0)
                goto unsetup;
 
+       be_roce_dev_add(adapter);
+
        dev_info(&pdev->dev, "%s: %s port %d\n", netdev->name, nic_name(pdev),
                adapter->port_num);