RDMA/erdma: Probe the erdma RoCEv2 device

Currently, the erdma driver supports both the iWARP and RoCEv2 protocols.
The erdma driver reads the ERDMA_REGS_DEV_PROTO_REG register to identify
the protocol used by the erdma device. Since each protocol requires
different ib_device_ops, we introduce the erdma_device_ops_iwarp and
erdma_device_ops_rocev2 for iWARP and RoCEv2 protocols, respectively.

Signed-off-by: Boshi Yu <boshiyu@linux.alibaba.com>
Link: https://patch.msgid.link/20241211020930.68833-2-boshiyu@linux.alibaba.com
Reviewed-by: Cheng Xu <chengyou@linux.alibaba.com>
Signed-off-by: Leon Romanovsky <leon@kernel.org>

diff --git a/drivers/infiniband/hw/erdma/Kconfig b/drivers/infiniband/hw/erdma/Kconfig
index 169038e3ceb12fbbf9937ff236d56e3e5d290a2f..267fc1f3c42af69f685af17289902cc3ef2918a3 100644 (file)
--- a/drivers/infiniband/hw/erdma/Kconfig
+++ b/drivers/infiniband/hw/erdma/Kconfig
@@ -5,7 +5,7 @@ config INFINIBAND_ERDMA
        depends on INFINIBAND_ADDR_TRANS
        depends on INFINIBAND_USER_ACCESS
        help
-         This is a RDMA/iWarp driver for Alibaba Elastic RDMA Adapter(ERDMA),
+         This is a RDMA driver for Alibaba Elastic RDMA Adapter(ERDMA),
          which supports RDMA features in Alibaba cloud environment.
 
          To compile this driver as module, choose M here. The module will be

diff --git a/drivers/infiniband/hw/erdma/erdma.h b/drivers/infiniband/hw/erdma/erdma.h
index 3c166359448d061f5ed510c079af4e73015e7b5b..ad4dc1a4bdc72e881a4a9cd632f875b6485da15d 100644 (file)
--- a/drivers/infiniband/hw/erdma/erdma.h
+++ b/drivers/infiniband/hw/erdma/erdma.h
@@ -16,7 +16,7 @@
 #include "erdma_hw.h"
 
 #define DRV_MODULE_NAME "erdma"
-#define ERDMA_NODE_DESC "Elastic RDMA(iWARP) stack"
+#define ERDMA_NODE_DESC "Elastic RDMA Adapter stack"
 
 struct erdma_eq {
        void *qbuf;
@@ -215,6 +215,7 @@ struct erdma_dev {
 
        struct dma_pool *db_pool;
        struct dma_pool *resp_pool;
+       enum erdma_proto_type proto;
 };
 
 static inline void *get_queue_entry(void *qbuf, u32 idx, u32 depth, u32 shift)

diff --git a/drivers/infiniband/hw/erdma/erdma_hw.h b/drivers/infiniband/hw/erdma/erdma_hw.h
index 05978f3b14759ad87fde92f8cf88405a7972b672..970b392d4fb4eafd8d80291143ffc32ce0431472 100644 (file)
--- a/drivers/infiniband/hw/erdma/erdma_hw.h
+++ b/drivers/infiniband/hw/erdma/erdma_hw.h
@@ -21,8 +21,15 @@
 #define ERDMA_NUM_MSIX_VEC 32U
 #define ERDMA_MSIX_VECTOR_CMDQ 0
 
+/* erdma device protocol type */
+enum erdma_proto_type {
+       ERDMA_PROTO_IWARP = 0,
+       ERDMA_PROTO_ROCEV2 = 1,
+};
+
 /* PCIe Bar0 Registers. */
 #define ERDMA_REGS_VERSION_REG 0x0
+#define ERDMA_REGS_DEV_PROTO_REG 0xC
 #define ERDMA_REGS_DEV_CTRL_REG 0x10
 #define ERDMA_REGS_DEV_ST_REG 0x14
 #define ERDMA_REGS_NETDEV_MAC_L_REG 0x18

diff --git a/drivers/infiniband/hw/erdma/erdma_main.c b/drivers/infiniband/hw/erdma/erdma_main.c
index 62f497a710041d95909af4f1cfcd3604ea0d0c22..cf97bb79e59515a70e98719e6b153360dc77a1f1 100644 (file)
--- a/drivers/infiniband/hw/erdma/erdma_main.c
+++ b/drivers/infiniband/hw/erdma/erdma_main.c
@@ -172,6 +172,8 @@ static int erdma_device_init(struct erdma_dev *dev, struct pci_dev *pdev)
 {
        int ret;
 
+       dev->proto = erdma_reg_read32(dev, ERDMA_REGS_DEV_PROTO_REG);
+
        dev->resp_pool = dma_pool_create("erdma_resp_pool", &pdev->dev,
                                         ERDMA_HW_RESP_SIZE, ERDMA_HW_RESP_SIZE,
                                         0);
@@ -474,6 +476,21 @@ static void erdma_res_cb_free(struct erdma_dev *dev)
                bitmap_free(dev->res_cb[i].bitmap);
 }
 
+static const struct ib_device_ops erdma_device_ops_rocev2 = {
+       .get_link_layer = erdma_get_link_layer,
+};
+
+static const struct ib_device_ops erdma_device_ops_iwarp = {
+       .iw_accept = erdma_accept,
+       .iw_add_ref = erdma_qp_get_ref,
+       .iw_connect = erdma_connect,
+       .iw_create_listen = erdma_create_listen,
+       .iw_destroy_listen = erdma_destroy_listen,
+       .iw_get_qp = erdma_get_ibqp,
+       .iw_reject = erdma_reject,
+       .iw_rem_ref = erdma_qp_put_ref,
+};
+
 static const struct ib_device_ops erdma_device_ops = {
        .owner = THIS_MODULE,
        .driver_id = RDMA_DRIVER_ERDMA,
@@ -494,14 +511,6 @@ static const struct ib_device_ops erdma_device_ops = {
        .get_dma_mr = erdma_get_dma_mr,
        .get_hw_stats = erdma_get_hw_stats,
        .get_port_immutable = erdma_get_port_immutable,
-       .iw_accept = erdma_accept,
-       .iw_add_ref = erdma_qp_get_ref,
-       .iw_connect = erdma_connect,
-       .iw_create_listen = erdma_create_listen,
-       .iw_destroy_listen = erdma_destroy_listen,
-       .iw_get_qp = erdma_get_ibqp,
-       .iw_reject = erdma_reject,
-       .iw_rem_ref = erdma_qp_put_ref,
        .map_mr_sg = erdma_map_mr_sg,
        .mmap = erdma_mmap,
        .mmap_free = erdma_mmap_free,
@@ -537,7 +546,14 @@ static int erdma_ib_device_add(struct pci_dev *pdev)
        if (ret)
                return ret;
 
-       ibdev->node_type = RDMA_NODE_RNIC;
+       if (erdma_device_iwarp(dev)) {
+               ibdev->node_type = RDMA_NODE_RNIC;
+               ib_set_device_ops(ibdev, &erdma_device_ops_iwarp);
+       } else {
+               ibdev->node_type = RDMA_NODE_IB_CA;
+               ib_set_device_ops(ibdev, &erdma_device_ops_rocev2);
+       }
+
        memcpy(ibdev->node_desc, ERDMA_NODE_DESC, sizeof(ERDMA_NODE_DESC));
 
        /*

diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c
index 51d619edb6c5d2722914f51cfc768080f855ec0e..3b7e55515cfd90db8f739692795020982c70fea8 100644 (file)
--- a/drivers/infiniband/hw/erdma/erdma_verbs.c
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.c
@@ -395,8 +395,17 @@ out:
 int erdma_get_port_immutable(struct ib_device *ibdev, u32 port,
                             struct ib_port_immutable *port_immutable)
 {
+       struct erdma_dev *dev = to_edev(ibdev);
+
+       if (erdma_device_iwarp(dev)) {
+               port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
+       } else {
+               port_immutable->core_cap_flags =
+                       RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
+               port_immutable->max_mad_size = IB_MGMT_MAD_SIZE;
+       }
+
        port_immutable->gid_tbl_len = 1;
-       port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
 
        return 0;
 }
@@ -1839,3 +1848,8 @@ int erdma_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
 
        return stats->num_counters;
 }
+
+enum rdma_link_layer erdma_get_link_layer(struct ib_device *ibdev, u32 port_num)
+{
+       return IB_LINK_LAYER_ETHERNET;
+}

diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h
index c998acd39a7889f8088fe4fb3609186e961fb911..90e2b35a0973a6cd3bb597a7b3fd71f852131dd3 100644 (file)
--- a/drivers/infiniband/hw/erdma/erdma_verbs.h
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.h
@@ -291,6 +291,16 @@ int erdma_modify_qp_internal(struct erdma_qp *qp, struct erdma_qp_attrs *attrs,
 void erdma_qp_llp_close(struct erdma_qp *qp);
 void erdma_qp_cm_drop(struct erdma_qp *qp);
 
+static inline bool erdma_device_iwarp(struct erdma_dev *dev)
+{
+       return dev->proto == ERDMA_PROTO_IWARP;
+}
+
+static inline bool erdma_device_rocev2(struct erdma_dev *dev)
+{
+       return dev->proto == ERDMA_PROTO_ROCEV2;
+}
+
 static inline struct erdma_ucontext *to_ectx(struct ib_ucontext *ibctx)
 {
        return container_of(ibctx, struct erdma_ucontext, ibucontext);
@@ -370,5 +380,7 @@ struct rdma_hw_stats *erdma_alloc_hw_port_stats(struct ib_device *device,
                                                u32 port_num);
 int erdma_get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats,
                       u32 port, int index);
+enum rdma_link_layer erdma_get_link_layer(struct ib_device *ibdev,
+                                         u32 port_num);
 
 #endif