RDMA/hns: Get udp sport num dynamically instead of using a fixed value

The UDP source port number in RoCE v2 is used to create entropy for
network routers (ECMP), load balancers and 802.3ad link aggregation
switching that are not aware of RoCE IB headers. Considering that the IB
core has achieved a new interface to get a hashed value of it, the fixed
value of it in QPC and UD WQE in hns driver could be fixed and the port
number is to be set dynamically now.

For QPC of RC, the value could be hashed from flow_lable if the user pass
it in or from remote qpn and local qpn. For WQE of UD, it is set according
to fl or as a random value.

Link: https://lore.kernel.org/r/1598002289-8611-1-git-send-email-liweihang@huawei.com
Signed-off-by: Weihang Li <liweihang@huawei.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>

diff --git a/drivers/infiniband/hw/hns/hns_roce_ah.c b/drivers/infiniband/hw/hns/hns_roce_ah.c
index 5b2f9314edd39d2a37d237a4bc158ccce815426a..54cadbc0724e458c6b0c6c83189a40080ff9fa39 100644 (file)
--- a/drivers/infiniband/hw/hns/hns_roce_ah.c
+++ b/drivers/infiniband/hw/hns/hns_roce_ah.c
@@ -39,6 +39,22 @@
 #define HNS_ROCE_VLAN_SL_BIT_MASK      7
 #define HNS_ROCE_VLAN_SL_SHIFT         13
 
+static inline u16 get_ah_udp_sport(const struct rdma_ah_attr *ah_attr)
+{
+       u32 fl = ah_attr->grh.flow_label;
+       u16 sport;
+
+       if (!fl)
+               sport = get_random_u32() %
+                       (IB_ROCE_UDP_ENCAP_VALID_PORT_MAX + 1 -
+                        IB_ROCE_UDP_ENCAP_VALID_PORT_MIN) +
+                       IB_ROCE_UDP_ENCAP_VALID_PORT_MIN;
+       else
+               sport = rdma_flow_label_to_udp_sport(fl);
+
+       return sport;
+}
+
 int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
                       struct ib_udata *udata)
 {
@@ -79,6 +95,8 @@ int hns_roce_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
 
        memcpy(ah->av.dgid, grh->dgid.raw, HNS_ROCE_GID_SIZE);
        ah->av.sl = rdma_ah_get_sl(ah_attr);
+       ah->av.flowlabel = grh->flow_label;
+       ah->av.udp_sport = get_ah_udp_sport(ah_attr);
 
        return 0;
 }

diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index da9888deff8c21c6745910232b0536f1bebf3804..8d92d8d82573ca8e1a553fd44ea6d396de644af3 100644 (file)
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -537,17 +537,18 @@ struct hns_roce_raq_table {
 };
 
 struct hns_roce_av {
-       u8          port;
-       u8          gid_index;
-       u8          stat_rate;
-       u8          hop_limit;
-       u32         flowlabel;
-       u8          sl;
-       u8          tclass;
-       u8          dgid[HNS_ROCE_GID_SIZE];
-       u8          mac[ETH_ALEN];
-       u16         vlan_id;
-       bool        vlan_en;
+       u8 port;
+       u8 gid_index;
+       u8 stat_rate;
+       u8 hop_limit;
+       u32 flowlabel;
+       u16 udp_sport;
+       u8 sl;
+       u8 tclass;
+       u8 dgid[HNS_ROCE_GID_SIZE];
+       u8 mac[ETH_ALEN];
+       u16 vlan_id;
+       bool vlan_en;
 };
 
 struct hns_roce_ah {

diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
index d2968594664bbbf95b5217249c420ba595aff599..4da553d435e4bac7cf7391c851d858f748ad9c2b 100644 (file)
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
@@ -369,7 +369,7 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp,
                       curr_idx & (qp->sge.sge_cnt - 1));
 
        roce_set_field(ud_sq_wqe->byte_24, V2_UD_SEND_WQE_BYTE_24_UDPSPN_M,
-                      V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, 0);
+                      V2_UD_SEND_WQE_BYTE_24_UDPSPN_S, ah->av.udp_sport);
        ud_sq_wqe->qkey = cpu_to_le32(ud_wr(wr)->remote_qkey & 0x80000000 ?
                          qp->qkey : ud_wr(wr)->remote_qkey);
        roce_set_field(ud_sq_wqe->byte_32, V2_UD_SEND_WQE_BYTE_32_DQPN_M,
@@ -4165,6 +4165,14 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp,
        return 0;
 }
 
+static inline u16 get_udp_sport(u32 fl, u32 lqpn, u32 rqpn)
+{
+       if (!fl)
+               fl = rdma_calc_flow_label(lqpn, rqpn);
+
+       return rdma_flow_label_to_udp_sport(fl);
+}
+
 static int hns_roce_v2_set_path(struct ib_qp *ibqp,
                                const struct ib_qp_attr *attr,
                                int attr_mask,
@@ -4228,7 +4236,8 @@ static int hns_roce_v2_set_path(struct ib_qp *ibqp,
 
        roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_UDPSPN_M,
                       V2_QPC_BYTE_52_UDPSPN_S,
-                      is_udp ? 0x12b7 : 0);
+                      is_udp ? get_udp_sport(grh->flow_label, ibqp->qp_num,
+                                             attr->dest_qp_num) : 0);
 
        roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_UDPSPN_M,
                       V2_QPC_BYTE_52_UDPSPN_S, 0);

diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 55dfe0e9d1d6ca44b867d8a9215adb7441e2189a..c868609a4ffaed63ea10e773ee5a95f8e2ec8c60 100644 (file)
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -4706,6 +4706,7 @@ bool rdma_dev_access_netns(const struct ib_device *device,
                           const struct net *net);
 
 #define IB_ROCE_UDP_ENCAP_VALID_PORT_MIN (0xC000)
+#define IB_ROCE_UDP_ENCAP_VALID_PORT_MAX (0xFFFF)
 #define IB_GRH_FLOWLABEL_MASK (0x000FFFFF)
 
 /**