net/mlx5e: Add flow rules for the decrypted ESP packets

The previous commit introduced two new flow groups to enable L4 RSS
for decrypted IPsec traffic. This commit implements the logic to
populate these groups with the necessary steering rules.

The rules are created dynamically whenever the first IPSec offload
rule is configured via the xfrm subsystem and the decryption tables
for RX are created. Each rule matches a specific decrypted traffic
type based on its ip version (or ethertype) and outer/inner
l4_type_ext, directing it to the appropriate L4 RSS-enabled TIR.

The lifecycle of these steering rules is tied directly to the RX
tables. They are deleted when the RX tables are destroyed.

Signed-off-by: Jianbo Liu <jianbol@nvidia.com>
Reviewed-by: Dragos Tatulea <dtatulea@nvidia.com>
Signed-off-by: Tariq Toukan <tariqt@nvidia.com>
Link: https://patch.msgid.link/1758179963-649455-5-git-send-email-tariqt@nvidia.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c
index 417c8b6548803ae93f61cbac69f5ba4c06a59998..ef2878f0c20e2e9c78288e7640829c1e3888af94 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c
@@ -61,6 +61,7 @@ struct mlx5e_ipsec_rx {
        struct mlx5_flow_table *pol_miss_ft;
        struct mlx5_flow_handle *pol_miss_rule;
        u8 allow_tunnel_mode : 1;
+       u8 ttc_rules_added : 1;
 };
 
 /* IPsec RX flow steering */
@@ -683,10 +684,13 @@ static void ipsec_mpv_work_handler(struct work_struct *_work)
        complete(&work->master_priv->ipsec->comp);
 }
 
-static void ipsec_rx_ft_disconnect(struct mlx5e_ipsec *ipsec, u32 family)
+static void ipsec_rx_ft_disconnect(struct mlx5e_ipsec *ipsec,
+                                  struct mlx5e_ipsec_rx *rx, u32 family)
 {
        struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false);
 
+       if (rx->ttc_rules_added)
+               mlx5_ttc_destroy_ipsec_rules(ttc);
        mlx5_ttc_fwd_default_dest(ttc, family2tt(family));
 }
 
@@ -721,7 +725,7 @@ static void rx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec,
 {
        /* disconnect */
        if (rx != ipsec->rx_esw)
-               ipsec_rx_ft_disconnect(ipsec, family);
+               ipsec_rx_ft_disconnect(ipsec, rx, family);
 
        mlx5_del_flow_rules(rx->sa.rule);
        mlx5_destroy_flow_group(rx->sa.group);
@@ -821,10 +825,16 @@ static void ipsec_rx_ft_connect(struct mlx5e_ipsec *ipsec,
                                struct mlx5e_ipsec_rx_create_attr *attr)
 {
        struct mlx5_flow_destination dest = {};
+       struct mlx5_ttc_table *ttc, *inner_ttc;
 
        dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
        dest.ft = rx->ft.sa;
-       mlx5_ttc_fwd_dest(attr->ttc, family2tt(attr->family), &dest);
+       if (mlx5_ttc_fwd_dest(attr->ttc, family2tt(attr->family), &dest))
+               return;
+
+       ttc = mlx5e_fs_get_ttc(ipsec->fs, false);
+       inner_ttc = mlx5e_fs_get_ttc(ipsec->fs, true);
+       rx->ttc_rules_added = !mlx5_ttc_create_ipsec_rules(ttc, inner_ttc);
 }
 
 static int ipsec_rx_chains_create_miss(struct mlx5e_ipsec *ipsec,

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.c
index 3cd5de6f714f85b3056b536dfb8641cfa1997f7c..7adad784ad46221e7be614804a8766edea247ced 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.c
@@ -32,10 +32,13 @@ static int mlx5_fs_ttc_table_size(const struct mlx5_fs_ttc_groups *groups)
 struct mlx5_ttc_table {
        int num_groups;
        const struct mlx5_fs_ttc_groups *groups;
+       struct mlx5_core_dev *mdev;
        struct mlx5_flow_table *t;
        struct mlx5_flow_group **g;
        struct mlx5_ttc_rule rules[MLX5_NUM_TT];
        struct mlx5_flow_handle *tunnel_rules[MLX5_NUM_TUNNEL_TT];
+       u32 refcnt;
+       struct mutex mutex; /* Protect adding rules for ipsec crypto offload */
 };
 
 struct mlx5_flow_table *mlx5_get_ttc_flow_table(struct mlx5_ttc_table *ttc)
@@ -302,6 +305,31 @@ static u8 mlx5_etype_to_ipv(u16 ethertype)
        return 0;
 }
 
+static void mlx5_fs_ttc_set_match_ipv_outer(struct mlx5_core_dev *mdev,
+                                           struct mlx5_flow_spec *spec,
+                                           u16 etype)
+{
+       int match_ipv_outer =
+               MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
+                                         ft_field_support.outer_ip_version);
+       u8 ipv;
+
+       ipv = mlx5_etype_to_ipv(etype);
+       if (match_ipv_outer && ipv) {
+               MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
+                                outer_headers.ip_version);
+               MLX5_SET(fte_match_param, spec->match_value,
+                        outer_headers.ip_version, ipv);
+       } else {
+               MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
+                                outer_headers.ethertype);
+               MLX5_SET(fte_match_param, spec->match_value,
+                        outer_headers.ethertype, etype);
+       }
+
+       spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
+}
+
 static void mlx5_fs_ttc_set_match_proto(void *headers_c, void *headers_v,
                                        u8 proto, bool use_l4_type)
 {
@@ -326,14 +354,10 @@ mlx5_generate_ttc_rule(struct mlx5_core_dev *dev, struct mlx5_flow_table *ft,
                       struct mlx5_flow_destination *dest, u16 etype, u8 proto,
                       bool use_l4_type, bool ipsec_rss)
 {
-       int match_ipv_outer =
-               MLX5_CAP_FLOWTABLE_NIC_RX(dev,
-                                         ft_field_support.outer_ip_version);
        MLX5_DECLARE_FLOW_ACT(flow_act);
        struct mlx5_flow_handle *rule;
        struct mlx5_flow_spec *spec;
        int err = 0;
-       u8 ipv;
 
        spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
        if (!spec)
@@ -350,16 +374,8 @@ mlx5_generate_ttc_rule(struct mlx5_core_dev *dev, struct mlx5_flow_table *ft,
                                            proto, use_l4_type);
        }
 
-       ipv = mlx5_etype_to_ipv(etype);
-       if (match_ipv_outer && ipv) {
-               spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
-               MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
-               MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, ipv);
-       } else if (etype) {
-               spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
-               MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype);
-               MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, etype);
-       }
+       if (etype)
+               mlx5_fs_ttc_set_match_ipv_outer(dev, spec, etype);
 
        if (ipsec_rss && proto == IPPROTO_ESP) {
                MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
@@ -838,6 +854,7 @@ void mlx5_destroy_ttc_table(struct mlx5_ttc_table *ttc)
 
        kfree(ttc->g);
        mlx5_destroy_flow_table(ttc->t);
+       mutex_destroy(&ttc->mutex);
        kvfree(ttc);
 }
 
@@ -894,6 +911,9 @@ struct mlx5_ttc_table *mlx5_create_ttc_table(struct mlx5_core_dev *dev,
        if (err)
                goto destroy_ft;
 
+       ttc->mdev = dev;
+       mutex_init(&ttc->mutex);
+
        return ttc;
 
 destroy_ft:
@@ -927,3 +947,194 @@ int mlx5_ttc_fwd_default_dest(struct mlx5_ttc_table *ttc,
 
        return mlx5_ttc_fwd_dest(ttc, type, &dest);
 }
+
+static void _mlx5_ttc_destroy_ipsec_rules(struct mlx5_ttc_table *ttc)
+{
+       enum mlx5_traffic_types i;
+
+       for (i = MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_TCP;
+            i <= MLX5_TT_DECRYPTED_ESP_INNER_IPV6_UDP; i++) {
+               if (!ttc->rules[i].rule)
+                       continue;
+
+               mlx5_del_flow_rules(ttc->rules[i].rule);
+               ttc->rules[i].rule = NULL;
+       }
+}
+
+void mlx5_ttc_destroy_ipsec_rules(struct mlx5_ttc_table *ttc)
+{
+       if (!mlx5_ttc_has_esp_flow_group(ttc))
+               return;
+
+       mutex_lock(&ttc->mutex);
+       if (--ttc->refcnt)
+               goto unlock;
+
+       _mlx5_ttc_destroy_ipsec_rules(ttc);
+unlock:
+       mutex_unlock(&ttc->mutex);
+}
+
+static int mlx5_ttc_get_tt_attrs(enum mlx5_traffic_types type,
+                                u16 *etype, int *l4_type_ext,
+                                enum mlx5_traffic_types *tir_tt)
+{
+       switch (type) {
+       case MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_TCP:
+       case MLX5_TT_DECRYPTED_ESP_INNER_IPV4_TCP:
+               *etype = ETH_P_IP;
+               *l4_type_ext = MLX5_PACKET_L4_TYPE_EXT_TCP;
+               *tir_tt = MLX5_TT_IPV4_TCP;
+               break;
+       case MLX5_TT_DECRYPTED_ESP_OUTER_IPV6_TCP:
+       case MLX5_TT_DECRYPTED_ESP_INNER_IPV6_TCP:
+               *etype = ETH_P_IPV6;
+               *l4_type_ext = MLX5_PACKET_L4_TYPE_EXT_TCP;
+               *tir_tt = MLX5_TT_IPV6_TCP;
+               break;
+       case MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_UDP:
+       case MLX5_TT_DECRYPTED_ESP_INNER_IPV4_UDP:
+               *etype = ETH_P_IP;
+               *l4_type_ext = MLX5_PACKET_L4_TYPE_EXT_UDP;
+               *tir_tt = MLX5_TT_IPV4_UDP;
+               break;
+       case MLX5_TT_DECRYPTED_ESP_OUTER_IPV6_UDP:
+       case MLX5_TT_DECRYPTED_ESP_INNER_IPV6_UDP:
+               *etype = ETH_P_IPV6;
+               *l4_type_ext = MLX5_PACKET_L4_TYPE_EXT_UDP;
+               *tir_tt = MLX5_TT_IPV6_UDP;
+               break;
+       default:
+               return -EINVAL;
+       }
+
+       return 0;
+}
+
+static struct mlx5_flow_handle *
+mlx5_ttc_create_ipsec_outer_rule(struct mlx5_ttc_table *ttc,
+                                enum mlx5_traffic_types type)
+{
+       struct mlx5_flow_destination dest;
+       MLX5_DECLARE_FLOW_ACT(flow_act);
+       enum mlx5_traffic_types tir_tt;
+       struct mlx5_flow_handle *rule;
+       struct mlx5_flow_spec *spec;
+       int l4_type_ext;
+       u16 etype;
+       int err;
+
+       err = mlx5_ttc_get_tt_attrs(type, &etype, &l4_type_ext, &tir_tt);
+       if (err)
+               return ERR_PTR(err);
+
+       spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+       if (!spec)
+               return ERR_PTR(-ENOMEM);
+
+       mlx5_fs_ttc_set_match_ipv_outer(ttc->mdev, spec, etype);
+
+       MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
+                        outer_headers.l4_type_ext);
+       MLX5_SET(fte_match_param, spec->match_value,
+                outer_headers.l4_type_ext, l4_type_ext);
+
+       dest = mlx5_ttc_get_default_dest(ttc, tir_tt);
+
+       rule = mlx5_add_flow_rules(ttc->t, spec, &flow_act, &dest, 1);
+       if (IS_ERR(rule)) {
+               err = PTR_ERR(rule);
+               mlx5_core_err(ttc->mdev, "%s: add rule failed\n", __func__);
+       }
+
+       kvfree(spec);
+       return err ? ERR_PTR(err) : rule;
+}
+
+static struct mlx5_flow_handle *
+mlx5_ttc_create_ipsec_inner_rule(struct mlx5_ttc_table *ttc,
+                                struct mlx5_ttc_table *inner_ttc,
+                                enum mlx5_traffic_types type)
+{
+       struct mlx5_flow_destination dest;
+       MLX5_DECLARE_FLOW_ACT(flow_act);
+       enum mlx5_traffic_types tir_tt;
+       struct mlx5_flow_handle *rule;
+       struct mlx5_flow_spec *spec;
+       int l4_type_ext;
+       u16 etype;
+       int err;
+
+       err = mlx5_ttc_get_tt_attrs(type, &etype, &l4_type_ext, &tir_tt);
+       if (err)
+               return ERR_PTR(err);
+
+       spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
+       if (!spec)
+               return ERR_PTR(-ENOMEM);
+
+       MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
+                        inner_headers.ip_version);
+       MLX5_SET(fte_match_param, spec->match_value,
+                inner_headers.ip_version, mlx5_etype_to_ipv(etype));
+       MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
+                        inner_headers.l4_type_ext);
+       MLX5_SET(fte_match_param, spec->match_value,
+                inner_headers.l4_type_ext, l4_type_ext);
+
+       dest = mlx5_ttc_get_default_dest(inner_ttc, tir_tt);
+
+       spec->match_criteria_enable = MLX5_MATCH_INNER_HEADERS;
+
+       rule = mlx5_add_flow_rules(ttc->t, spec, &flow_act, &dest, 1);
+       if (IS_ERR(rule)) {
+               err = PTR_ERR(rule);
+               mlx5_core_err(ttc->mdev, "%s: add rule failed\n", __func__);
+       }
+
+       kvfree(spec);
+       return err ? ERR_PTR(err) : rule;
+}
+
+int mlx5_ttc_create_ipsec_rules(struct mlx5_ttc_table *ttc,
+                               struct mlx5_ttc_table *inner_ttc)
+{
+       struct mlx5_flow_handle *rule;
+       enum mlx5_traffic_types i;
+
+       if (!mlx5_ttc_has_esp_flow_group(ttc))
+               return 0;
+
+       mutex_lock(&ttc->mutex);
+       if (ttc->refcnt)
+               goto skip;
+
+       for (i = MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_TCP;
+            i <= MLX5_TT_DECRYPTED_ESP_OUTER_IPV6_UDP; i++) {
+               rule = mlx5_ttc_create_ipsec_outer_rule(ttc, i);
+               if (IS_ERR(rule))
+                       goto err_out;
+
+               ttc->rules[i].rule = rule;
+       }
+
+       for (i = MLX5_TT_DECRYPTED_ESP_INNER_IPV4_TCP;
+            i <= MLX5_TT_DECRYPTED_ESP_INNER_IPV6_UDP; i++) {
+               rule = mlx5_ttc_create_ipsec_inner_rule(ttc, inner_ttc, i);
+               if (IS_ERR(rule))
+                       goto err_out;
+
+               ttc->rules[i].rule = rule;
+       }
+
+skip:
+       ttc->refcnt++;
+       mutex_unlock(&ttc->mutex);
+       return 0;
+
+err_out:
+       _mlx5_ttc_destroy_ipsec_rules(ttc);
+       mutex_unlock(&ttc->mutex);
+       return PTR_ERR(rule);
+}

diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.h
index cae6a8ba0491338bc24b334cee8a34f423c34552..95f6e56724a27484f2e43cc9a31b35370b208dcf 100644 (file)
--- a/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/fs_ttc.h
@@ -80,6 +80,9 @@ bool mlx5_tunnel_inner_ft_supported(struct mlx5_core_dev *mdev);
 u8 mlx5_get_proto_by_tunnel_type(enum mlx5_tunnel_types tt);
 
 bool mlx5_ttc_has_esp_flow_group(struct mlx5_ttc_table *ttc);
+int mlx5_ttc_create_ipsec_rules(struct mlx5_ttc_table *ttc,
+                               struct mlx5_ttc_table *inner_ttc);
+void mlx5_ttc_destroy_ipsec_rules(struct mlx5_ttc_table *ttc);
 static inline bool mlx5_ttc_is_decrypted_esp_tt(enum mlx5_traffic_types tt)
 {
        return tt >= MLX5_TT_DECRYPTED_ESP_OUTER_IPV4_TCP &&