#include "internal.h"
-#define HWS_CLEAR_MATCH_PARAM(mask, field) \
- MLX5_SET(fte_match_param, (mask)->match_buf, field, 0)
-
-#define HWS_SZ_MATCH_PARAM (MLX5_ST_SZ_DW_MATCH_PARAM * 4)
-
-static const struct rhashtable_params hws_refcount_hash = {
- .key_len = sizeof_field(struct mlx5hws_bwc_complex_rule_hash_node,
- match_buf),
- .key_offset = offsetof(struct mlx5hws_bwc_complex_rule_hash_node,
- match_buf),
- .head_offset = offsetof(struct mlx5hws_bwc_complex_rule_hash_node,
- hash_node),
- .automatic_shrinking = true,
- .min_size = 1,
+/* We chain submatchers by applying three rules on a subrule: modify header (to
+ * set register C6), jump to table (to the next submatcher) and the mandatory
+ * last rule.
+ */
+#define HWS_NUM_CHAIN_ACTIONS 3
+
+static const struct rhashtable_params hws_rules_hash_params = {
+ .key_len = sizeof_field(struct mlx5hws_bwc_complex_subrule_data,
+ match_tag),
+ .key_offset =
+ offsetof(struct mlx5hws_bwc_complex_subrule_data, match_tag),
+ .head_offset =
+ offsetof(struct mlx5hws_bwc_complex_subrule_data, hash_node),
+ .automatic_shrinking = true, .min_size = 1,
};
-bool mlx5hws_bwc_match_params_is_complex(struct mlx5hws_context *ctx,
- u8 match_criteria_enable,
- struct mlx5hws_match_parameters *mask)
+static bool
+hws_match_params_exceeds_definer(struct mlx5hws_context *ctx,
+ u8 match_criteria_enable,
+ struct mlx5hws_match_parameters *mask,
+ bool allow_jumbo)
{
struct mlx5hws_definer match_layout = {0};
struct mlx5hws_match_template *mt;
mask->match_sz,
match_criteria_enable);
if (!mt) {
- mlx5hws_err(ctx, "BWC: failed creating match template\n");
+ mlx5hws_err(ctx, "Complex matcher: failed creating match template\n");
return false;
}
- ret = mlx5hws_definer_calc_layout(ctx, mt, &match_layout);
+ ret = mlx5hws_definer_calc_layout(ctx, mt, &match_layout, allow_jumbo);
if (ret) {
/* The only case that we're interested in is E2BIG,
* which means that the match parameters need to be
return is_complex;
}
-static void
-hws_bwc_matcher_complex_params_clear_fld(struct mlx5hws_context *ctx,
- enum mlx5hws_definer_fname fname,
+bool mlx5hws_bwc_match_params_is_complex(struct mlx5hws_context *ctx,
+ u8 match_criteria_enable,
struct mlx5hws_match_parameters *mask)
{
- struct mlx5hws_cmd_query_caps *caps = ctx->caps;
-
- switch (fname) {
- case MLX5HWS_DEFINER_FNAME_ETH_TYPE_O:
- case MLX5HWS_DEFINER_FNAME_ETH_TYPE_I:
- case MLX5HWS_DEFINER_FNAME_ETH_L3_TYPE_O:
- case MLX5HWS_DEFINER_FNAME_ETH_L3_TYPE_I:
- case MLX5HWS_DEFINER_FNAME_IP_VERSION_O:
- case MLX5HWS_DEFINER_FNAME_IP_VERSION_I:
- /* Because of the strict requirements for IP address matching
- * that require ethtype/ip_version matching as well, don't clear
- * these fields - have them in both parts of the complex matcher
- */
- break;
- case MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_O:
- HWS_CLEAR_MATCH_PARAM(mask, outer_headers.smac_47_16);
- break;
- case MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_I:
- HWS_CLEAR_MATCH_PARAM(mask, inner_headers.smac_47_16);
- break;
- case MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_O:
- HWS_CLEAR_MATCH_PARAM(mask, outer_headers.smac_15_0);
- break;
- case MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_I:
- HWS_CLEAR_MATCH_PARAM(mask, inner_headers.smac_15_0);
- break;
- case MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_O:
- HWS_CLEAR_MATCH_PARAM(mask, outer_headers.dmac_47_16);
- break;
- case MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_I:
- HWS_CLEAR_MATCH_PARAM(mask, inner_headers.dmac_47_16);
- break;
- case MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_O:
- HWS_CLEAR_MATCH_PARAM(mask, outer_headers.dmac_15_0);
- break;
- case MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_I:
- HWS_CLEAR_MATCH_PARAM(mask, inner_headers.dmac_15_0);
- break;
- case MLX5HWS_DEFINER_FNAME_VLAN_TYPE_O:
- HWS_CLEAR_MATCH_PARAM(mask, outer_headers.cvlan_tag);
- HWS_CLEAR_MATCH_PARAM(mask, outer_headers.svlan_tag);
- break;
- case MLX5HWS_DEFINER_FNAME_VLAN_TYPE_I:
- HWS_CLEAR_MATCH_PARAM(mask, inner_headers.cvlan_tag);
- HWS_CLEAR_MATCH_PARAM(mask, inner_headers.svlan_tag);
- break;
- case MLX5HWS_DEFINER_FNAME_VLAN_FIRST_PRIO_O:
- HWS_CLEAR_MATCH_PARAM(mask, outer_headers.first_prio);
- break;
- case MLX5HWS_DEFINER_FNAME_VLAN_FIRST_PRIO_I:
- HWS_CLEAR_MATCH_PARAM(mask, inner_headers.first_prio);
- break;
- case MLX5HWS_DEFINER_FNAME_VLAN_CFI_O:
- HWS_CLEAR_MATCH_PARAM(mask, outer_headers.first_cfi);
- break;
- case MLX5HWS_DEFINER_FNAME_VLAN_CFI_I:
- HWS_CLEAR_MATCH_PARAM(mask, inner_headers.first_cfi);
- break;
- case MLX5HWS_DEFINER_FNAME_VLAN_ID_O:
- HWS_CLEAR_MATCH_PARAM(mask, outer_headers.first_vid);
- break;
- case MLX5HWS_DEFINER_FNAME_VLAN_ID_I:
- HWS_CLEAR_MATCH_PARAM(mask, inner_headers.first_vid);
- break;
- case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_TYPE_O:
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters.outer_second_cvlan_tag);
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters.outer_second_svlan_tag);
- break;
- case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_TYPE_I:
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters.inner_second_cvlan_tag);
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters.inner_second_svlan_tag);
- break;
- case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_PRIO_O:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.outer_second_prio);
- break;
- case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_PRIO_I:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.inner_second_prio);
- break;
- case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_CFI_O:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.outer_second_cfi);
- break;
- case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_CFI_I:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.inner_second_cfi);
- break;
- case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_ID_O:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.outer_second_vid);
- break;
- case MLX5HWS_DEFINER_FNAME_VLAN_SECOND_ID_I:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.inner_second_vid);
- break;
- case MLX5HWS_DEFINER_FNAME_IPV4_IHL_O:
- HWS_CLEAR_MATCH_PARAM(mask, outer_headers.ipv4_ihl);
- break;
- case MLX5HWS_DEFINER_FNAME_IPV4_IHL_I:
- HWS_CLEAR_MATCH_PARAM(mask, inner_headers.ipv4_ihl);
- break;
- case MLX5HWS_DEFINER_FNAME_IP_DSCP_O:
- HWS_CLEAR_MATCH_PARAM(mask, outer_headers.ip_dscp);
- break;
- case MLX5HWS_DEFINER_FNAME_IP_DSCP_I:
- HWS_CLEAR_MATCH_PARAM(mask, inner_headers.ip_dscp);
- break;
- case MLX5HWS_DEFINER_FNAME_IP_ECN_O:
- HWS_CLEAR_MATCH_PARAM(mask, outer_headers.ip_ecn);
- break;
- case MLX5HWS_DEFINER_FNAME_IP_ECN_I:
- HWS_CLEAR_MATCH_PARAM(mask, inner_headers.ip_ecn);
- break;
- case MLX5HWS_DEFINER_FNAME_IP_TTL_O:
- HWS_CLEAR_MATCH_PARAM(mask, outer_headers.ttl_hoplimit);
- break;
- case MLX5HWS_DEFINER_FNAME_IP_TTL_I:
- HWS_CLEAR_MATCH_PARAM(mask, inner_headers.ttl_hoplimit);
- break;
- case MLX5HWS_DEFINER_FNAME_IPV4_DST_O:
- HWS_CLEAR_MATCH_PARAM(mask,
- outer_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_31_0);
- break;
- case MLX5HWS_DEFINER_FNAME_IPV4_SRC_O:
- HWS_CLEAR_MATCH_PARAM(mask,
- outer_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_31_0);
- break;
- case MLX5HWS_DEFINER_FNAME_IPV4_DST_I:
- HWS_CLEAR_MATCH_PARAM(mask,
- inner_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_31_0);
- break;
- case MLX5HWS_DEFINER_FNAME_IPV4_SRC_I:
- HWS_CLEAR_MATCH_PARAM(mask,
- inner_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_31_0);
- break;
- case MLX5HWS_DEFINER_FNAME_IP_FRAG_O:
- HWS_CLEAR_MATCH_PARAM(mask, outer_headers.frag);
- break;
- case MLX5HWS_DEFINER_FNAME_IP_FRAG_I:
- HWS_CLEAR_MATCH_PARAM(mask, inner_headers.frag);
- break;
- case MLX5HWS_DEFINER_FNAME_IPV6_FLOW_LABEL_O:
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters.outer_ipv6_flow_label);
- break;
- case MLX5HWS_DEFINER_FNAME_IPV6_FLOW_LABEL_I:
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters.inner_ipv6_flow_label);
- break;
- case MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_O:
- case MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_O:
- case MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_O:
- case MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_O:
- HWS_CLEAR_MATCH_PARAM(mask,
- outer_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_127_96);
- HWS_CLEAR_MATCH_PARAM(mask,
- outer_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_95_64);
- HWS_CLEAR_MATCH_PARAM(mask,
- outer_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_63_32);
- HWS_CLEAR_MATCH_PARAM(mask,
- outer_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_31_0);
- break;
- case MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_O:
- case MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_O:
- case MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_O:
- case MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_O:
- HWS_CLEAR_MATCH_PARAM(mask,
- outer_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_127_96);
- HWS_CLEAR_MATCH_PARAM(mask,
- outer_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_95_64);
- HWS_CLEAR_MATCH_PARAM(mask,
- outer_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_63_32);
- HWS_CLEAR_MATCH_PARAM(mask,
- outer_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_31_0);
- break;
- case MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_I:
- case MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_I:
- case MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_I:
- case MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_I:
- HWS_CLEAR_MATCH_PARAM(mask,
- inner_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_127_96);
- HWS_CLEAR_MATCH_PARAM(mask,
- inner_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_95_64);
- HWS_CLEAR_MATCH_PARAM(mask,
- inner_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_63_32);
- HWS_CLEAR_MATCH_PARAM(mask,
- inner_headers.dst_ipv4_dst_ipv6.ipv6_simple_layout.ipv6_31_0);
- break;
- case MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_I:
- case MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_I:
- case MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_I:
- case MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_I:
- HWS_CLEAR_MATCH_PARAM(mask,
- inner_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_127_96);
- HWS_CLEAR_MATCH_PARAM(mask,
- inner_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_95_64);
- HWS_CLEAR_MATCH_PARAM(mask,
- inner_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_63_32);
- HWS_CLEAR_MATCH_PARAM(mask,
- inner_headers.src_ipv4_src_ipv6.ipv6_simple_layout.ipv6_31_0);
- break;
- case MLX5HWS_DEFINER_FNAME_IP_PROTOCOL_O:
- HWS_CLEAR_MATCH_PARAM(mask, outer_headers.ip_protocol);
- break;
- case MLX5HWS_DEFINER_FNAME_IP_PROTOCOL_I:
- HWS_CLEAR_MATCH_PARAM(mask, inner_headers.ip_protocol);
- break;
- case MLX5HWS_DEFINER_FNAME_L4_SPORT_O:
- HWS_CLEAR_MATCH_PARAM(mask, outer_headers.tcp_sport);
- HWS_CLEAR_MATCH_PARAM(mask, outer_headers.udp_sport);
- break;
- case MLX5HWS_DEFINER_FNAME_L4_SPORT_I:
- HWS_CLEAR_MATCH_PARAM(mask, inner_headers.tcp_dport);
- HWS_CLEAR_MATCH_PARAM(mask, inner_headers.udp_dport);
- break;
- case MLX5HWS_DEFINER_FNAME_L4_DPORT_O:
- HWS_CLEAR_MATCH_PARAM(mask, outer_headers.tcp_dport);
- HWS_CLEAR_MATCH_PARAM(mask, outer_headers.udp_dport);
- break;
- case MLX5HWS_DEFINER_FNAME_L4_DPORT_I:
- HWS_CLEAR_MATCH_PARAM(mask, inner_headers.tcp_dport);
- HWS_CLEAR_MATCH_PARAM(mask, inner_headers.udp_dport);
- break;
- case MLX5HWS_DEFINER_FNAME_TCP_FLAGS_O:
- HWS_CLEAR_MATCH_PARAM(mask, outer_headers.tcp_flags);
- break;
- case MLX5HWS_DEFINER_FNAME_TCP_ACK_NUM:
- case MLX5HWS_DEFINER_FNAME_TCP_SEQ_NUM:
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters_3.outer_tcp_seq_num);
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters_3.outer_tcp_ack_num);
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters_3.inner_tcp_seq_num);
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters_3.inner_tcp_ack_num);
- break;
- case MLX5HWS_DEFINER_FNAME_GTP_TEID:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.gtpu_teid);
- break;
- case MLX5HWS_DEFINER_FNAME_GTP_MSG_TYPE:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.gtpu_msg_type);
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.gtpu_msg_flags);
- break;
- case MLX5HWS_DEFINER_FNAME_GTPU_FIRST_EXT_DW0:
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters_3.gtpu_first_ext_dw_0);
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.gtpu_dw_0);
- break;
- case MLX5HWS_DEFINER_FNAME_GTPU_DW2:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.gtpu_dw_2);
- break;
- case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_0:
- case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_1:
- case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_2:
- case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_3:
- case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_4:
- case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_5:
- case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_6:
- case MLX5HWS_DEFINER_FNAME_FLEX_PARSER_7:
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters_2.outer_first_mpls_over_gre);
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters_2.outer_first_mpls_over_udp);
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters_3.geneve_tlv_option_0_data);
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters_4.prog_sample_field_id_0);
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters_4.prog_sample_field_value_0);
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters_4.prog_sample_field_value_1);
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters_4.prog_sample_field_id_2);
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters_4.prog_sample_field_value_2);
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters_4.prog_sample_field_id_3);
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters_4.prog_sample_field_value_3);
- break;
- case MLX5HWS_DEFINER_FNAME_VXLAN_VNI:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.vxlan_vni);
- break;
- case MLX5HWS_DEFINER_FNAME_VXLAN_GPE_FLAGS:
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters_3.outer_vxlan_gpe_flags);
- break;
- case MLX5HWS_DEFINER_FNAME_VXLAN_GPE_RSVD0:
- break;
- case MLX5HWS_DEFINER_FNAME_VXLAN_GPE_PROTO:
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters_3.outer_vxlan_gpe_next_protocol);
- break;
- case MLX5HWS_DEFINER_FNAME_VXLAN_GPE_VNI:
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters_3.outer_vxlan_gpe_vni);
- break;
- case MLX5HWS_DEFINER_FNAME_GENEVE_OPT_LEN:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.geneve_opt_len);
- break;
- case MLX5HWS_DEFINER_FNAME_GENEVE_OAM:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.geneve_oam);
- break;
- case MLX5HWS_DEFINER_FNAME_GENEVE_PROTO:
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters.geneve_protocol_type);
- break;
- case MLX5HWS_DEFINER_FNAME_GENEVE_VNI:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.geneve_vni);
- break;
- case MLX5HWS_DEFINER_FNAME_SOURCE_QP:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.source_sqn);
- break;
- case MLX5HWS_DEFINER_FNAME_SOURCE_GVMI:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.source_port);
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters.source_eswitch_owner_vhca_id);
- break;
- case MLX5HWS_DEFINER_FNAME_REG_0:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_0);
- break;
- case MLX5HWS_DEFINER_FNAME_REG_1:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_1);
- break;
- case MLX5HWS_DEFINER_FNAME_REG_2:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_2);
- break;
- case MLX5HWS_DEFINER_FNAME_REG_3:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_3);
- break;
- case MLX5HWS_DEFINER_FNAME_REG_4:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_4);
- break;
- case MLX5HWS_DEFINER_FNAME_REG_5:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_5);
- break;
- case MLX5HWS_DEFINER_FNAME_REG_7:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_c_7);
- break;
- case MLX5HWS_DEFINER_FNAME_REG_A:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.metadata_reg_a);
- break;
- case MLX5HWS_DEFINER_FNAME_GRE_C:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.gre_c_present);
- break;
- case MLX5HWS_DEFINER_FNAME_GRE_K:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.gre_k_present);
- break;
- case MLX5HWS_DEFINER_FNAME_GRE_S:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.gre_s_present);
- break;
- case MLX5HWS_DEFINER_FNAME_GRE_PROTOCOL:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.gre_protocol);
- break;
- case MLX5HWS_DEFINER_FNAME_GRE_OPT_KEY:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters.gre_key.key);
- break;
- case MLX5HWS_DEFINER_FNAME_ICMP_DW1:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.icmp_header_data);
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.icmp_type);
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.icmp_code);
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters_3.icmpv6_header_data);
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.icmpv6_type);
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_3.icmpv6_code);
- break;
- case MLX5HWS_DEFINER_FNAME_MPLS0_O:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.outer_first_mpls);
- break;
- case MLX5HWS_DEFINER_FNAME_MPLS0_I:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_2.inner_first_mpls);
- break;
- case MLX5HWS_DEFINER_FNAME_TNL_HDR_0:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_5.tunnel_header_0);
- break;
- case MLX5HWS_DEFINER_FNAME_TNL_HDR_1:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_5.tunnel_header_1);
- break;
- case MLX5HWS_DEFINER_FNAME_TNL_HDR_2:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_5.tunnel_header_2);
- break;
- case MLX5HWS_DEFINER_FNAME_TNL_HDR_3:
- HWS_CLEAR_MATCH_PARAM(mask, misc_parameters_5.tunnel_header_3);
- break;
- case MLX5HWS_DEFINER_FNAME_FLEX_PARSER0_OK:
- case MLX5HWS_DEFINER_FNAME_FLEX_PARSER1_OK:
- case MLX5HWS_DEFINER_FNAME_FLEX_PARSER2_OK:
- case MLX5HWS_DEFINER_FNAME_FLEX_PARSER3_OK:
- case MLX5HWS_DEFINER_FNAME_FLEX_PARSER4_OK:
- case MLX5HWS_DEFINER_FNAME_FLEX_PARSER5_OK:
- case MLX5HWS_DEFINER_FNAME_FLEX_PARSER6_OK:
- case MLX5HWS_DEFINER_FNAME_FLEX_PARSER7_OK:
- /* assuming this is flex parser for geneve option */
- if ((fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER0_OK &&
- ctx->caps->flex_parser_id_geneve_tlv_option_0 != 0) ||
- (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER1_OK &&
- ctx->caps->flex_parser_id_geneve_tlv_option_0 != 1) ||
- (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER2_OK &&
- ctx->caps->flex_parser_id_geneve_tlv_option_0 != 2) ||
- (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER3_OK &&
- ctx->caps->flex_parser_id_geneve_tlv_option_0 != 3) ||
- (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER4_OK &&
- ctx->caps->flex_parser_id_geneve_tlv_option_0 != 4) ||
- (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER5_OK &&
- ctx->caps->flex_parser_id_geneve_tlv_option_0 != 5) ||
- (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER6_OK &&
- ctx->caps->flex_parser_id_geneve_tlv_option_0 != 6) ||
- (fname == MLX5HWS_DEFINER_FNAME_FLEX_PARSER7_OK &&
- ctx->caps->flex_parser_id_geneve_tlv_option_0 != 7)) {
- mlx5hws_err(ctx,
- "Complex params: unsupported field %s (%d), flex parser ID for geneve is %d\n",
- mlx5hws_definer_fname_to_str(fname), fname,
- caps->flex_parser_id_geneve_tlv_option_0);
- break;
- }
- HWS_CLEAR_MATCH_PARAM(mask,
- misc_parameters.geneve_tlv_option_0_exist);
- break;
- case MLX5HWS_DEFINER_FNAME_REG_6:
- default:
- mlx5hws_err(ctx, "Complex params: unsupported field %s (%d)\n",
- mlx5hws_definer_fname_to_str(fname), fname);
- break;
- }
+ return hws_match_params_exceeds_definer(ctx, match_criteria_enable,
+ mask, true);
}
-static bool
-hws_bwc_matcher_complex_params_comb_is_valid(struct mlx5hws_definer_fc *fc,
- int fc_sz,
- u32 combination_num)
+static int
+hws_get_last_set_dword_idx(const struct mlx5hws_match_parameters *mask)
{
- bool m1[MLX5HWS_DEFINER_FNAME_MAX] = {0};
- bool m2[MLX5HWS_DEFINER_FNAME_MAX] = {0};
- bool is_first_matcher;
int i;
- for (i = 0; i < fc_sz; i++) {
- is_first_matcher = !(combination_num & BIT(i));
- if (is_first_matcher)
- m1[fc[i].fname] = true;
- else
- m2[fc[i].fname] = true;
- }
-
- /* Not all the fields can be split into separate matchers.
- * Some should be together on the same matcher.
- * For example, IPv6 parts - the whole IPv6 address should be on the
- * same matcher in order for us to deduce if it's IPv6 or IPv4 address.
- */
- if (m1[MLX5HWS_DEFINER_FNAME_IP_FRAG_O] &&
- (m2[MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_O] ||
- m2[MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_O] ||
- m2[MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_O] ||
- m2[MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_O]))
- return false;
-
- if (m2[MLX5HWS_DEFINER_FNAME_IP_FRAG_O] &&
- (m1[MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_O] ||
- m1[MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_O] ||
- m1[MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_O] ||
- m1[MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_O]))
- return false;
+ for (i = mask->match_sz / 4 - 1; i >= 0; i--)
+ if (mask->match_buf[i])
+ return i;
- if (m1[MLX5HWS_DEFINER_FNAME_IP_FRAG_I] &&
- (m2[MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_I] ||
- m2[MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_I] ||
- m2[MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_I] ||
- m2[MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_I]))
- return false;
+ return -1;
+}
- if (m2[MLX5HWS_DEFINER_FNAME_IP_FRAG_I] &&
- (m1[MLX5HWS_DEFINER_FNAME_ETH_SMAC_47_16_I] ||
- m1[MLX5HWS_DEFINER_FNAME_ETH_SMAC_15_0_I] ||
- m1[MLX5HWS_DEFINER_FNAME_ETH_DMAC_47_16_I] ||
- m1[MLX5HWS_DEFINER_FNAME_ETH_DMAC_15_0_I]))
- return false;
+static bool hws_match_mask_is_empty(const struct mlx5hws_match_parameters *mask)
+{
+ return hws_get_last_set_dword_idx(mask) == -1;
+}
- /* Don't split outer IPv6 dest address. */
- if ((m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_O] ||
- m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_O] ||
- m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_O] ||
- m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_O]) &&
- (m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_O] ||
- m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_O] ||
- m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_O] ||
- m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_O]))
- return false;
+static bool hws_dword_is_inner_ipaddr_off(int dword_off)
+{
+ /* IPv4 and IPv6 addresses share the same entry via a union, and the
+ * source and dest addresses are contiguous in the fte_match_param. So
+ * we need to check 8 words.
+ */
+ static const int inner_ip_dword_off =
+ __mlx5_dw_off(fte_match_param, inner_headers.src_ipv4_src_ipv6);
- /* Don't split outer IPv6 source address. */
- if ((m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_O] ||
- m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_O] ||
- m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_O] ||
- m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_O]) &&
- (m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_O] ||
- m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_O] ||
- m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_O] ||
- m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_O]))
- return false;
+ return dword_off >= inner_ip_dword_off &&
+ dword_off < inner_ip_dword_off + 8;
+}
- /* Don't split inner IPv6 dest address. */
- if ((m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_I] ||
- m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_I] ||
- m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_I] ||
- m1[MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_I]) &&
- (m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_127_96_I] ||
- m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_95_64_I] ||
- m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_63_32_I] ||
- m2[MLX5HWS_DEFINER_FNAME_IPV6_DST_31_0_I]))
- return false;
+static bool hws_dword_is_outer_ipaddr_off(int dword_off)
+{
+ static const int outer_ip_dword_off =
+ __mlx5_dw_off(fte_match_param, outer_headers.src_ipv4_src_ipv6);
- /* Don't split inner IPv6 source address. */
- if ((m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_I] ||
- m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_I] ||
- m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_I] ||
- m1[MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_I]) &&
- (m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_127_96_I] ||
- m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_95_64_I] ||
- m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_63_32_I] ||
- m2[MLX5HWS_DEFINER_FNAME_IPV6_SRC_31_0_I]))
- return false;
+ return dword_off >= outer_ip_dword_off &&
+ dword_off < outer_ip_dword_off + 8;
+}
- /* Don't split GRE parameters. */
- if ((m1[MLX5HWS_DEFINER_FNAME_GRE_C] ||
- m1[MLX5HWS_DEFINER_FNAME_GRE_K] ||
- m1[MLX5HWS_DEFINER_FNAME_GRE_S] ||
- m1[MLX5HWS_DEFINER_FNAME_GRE_PROTOCOL]) &&
- (m2[MLX5HWS_DEFINER_FNAME_GRE_C] ||
- m2[MLX5HWS_DEFINER_FNAME_GRE_K] ||
- m2[MLX5HWS_DEFINER_FNAME_GRE_S] ||
- m2[MLX5HWS_DEFINER_FNAME_GRE_PROTOCOL]))
- return false;
+static void hws_add_dword_to_mask(struct mlx5hws_match_parameters *mask,
+ const struct mlx5hws_match_parameters *orig,
+ int dword_idx, bool *added_inner_ipv,
+ bool *added_outer_ipv)
+{
+ mask->match_buf[dword_idx] |= orig->match_buf[dword_idx];
- /* Don't split TCP ack/seq numbers. */
- if ((m1[MLX5HWS_DEFINER_FNAME_TCP_ACK_NUM] ||
- m1[MLX5HWS_DEFINER_FNAME_TCP_SEQ_NUM]) &&
- (m2[MLX5HWS_DEFINER_FNAME_TCP_ACK_NUM] ||
- m2[MLX5HWS_DEFINER_FNAME_TCP_SEQ_NUM]))
- return false;
+ *added_inner_ipv = false;
+ *added_outer_ipv = false;
- /* Don't split flex parser. */
- if ((m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_0] ||
- m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_1] ||
- m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_2] ||
- m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_3] ||
- m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_4] ||
- m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_5] ||
- m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_6] ||
- m1[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_7]) &&
- (m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_0] ||
- m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_1] ||
- m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_2] ||
- m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_3] ||
- m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_4] ||
- m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_5] ||
- m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_6] ||
- m2[MLX5HWS_DEFINER_FNAME_FLEX_PARSER_7]))
- return false;
+ /* Any IP address fragment must be accompanied by a match on IP version.
+ * Use the `added_ipv` variables to keep track if we added IP versions
+ * specifically for this dword, so that we can roll them back if the
+ * match params become too large to fit into a definer.
+ */
+ if (hws_dword_is_inner_ipaddr_off(dword_idx) &&
+ !MLX5_GET(fte_match_param, mask->match_buf,
+ inner_headers.ip_version)) {
+ MLX5_SET(fte_match_param, mask->match_buf,
+ inner_headers.ip_version, 0xf);
+ *added_inner_ipv = true;
+ }
+ if (hws_dword_is_outer_ipaddr_off(dword_idx) &&
+ !MLX5_GET(fte_match_param, mask->match_buf,
+ outer_headers.ip_version)) {
+ MLX5_SET(fte_match_param, mask->match_buf,
+ outer_headers.ip_version, 0xf);
+ *added_outer_ipv = true;
+ }
+}
- return true;
+static void hws_remove_dword_from_mask(struct mlx5hws_match_parameters *mask,
+ int dword_idx, bool added_inner_ipv,
+ bool added_outer_ipv)
+{
+ mask->match_buf[dword_idx] = 0;
+ if (added_inner_ipv)
+ MLX5_SET(fte_match_param, mask->match_buf,
+ inner_headers.ip_version, 0);
+ if (added_outer_ipv)
+ MLX5_SET(fte_match_param, mask->match_buf,
+ outer_headers.ip_version, 0);
}
-static void
-hws_bwc_matcher_complex_params_comb_create(struct mlx5hws_context *ctx,
- struct mlx5hws_match_parameters *m,
- struct mlx5hws_match_parameters *m1,
- struct mlx5hws_match_parameters *m2,
- struct mlx5hws_definer_fc *fc,
- int fc_sz,
- u32 combination_num)
+/* Avoid leaving a single lower dword in `mask` if there are others present in
+ * `orig`. Splitting IPv6 addresses like this causes them to be interpreted as
+ * IPv4.
+ */
+static void hws_avoid_ipv6_split_of(struct mlx5hws_match_parameters *orig,
+ struct mlx5hws_match_parameters *mask,
+ int off)
{
- bool is_first_matcher;
- int i;
+ /* Masks are allocated to a full fte_match_param, but it can't hurt to
+ * double check.
+ */
+ if (orig->match_sz <= off + 3 || mask->match_sz <= off + 3)
+ return;
- memcpy(m1->match_buf, m->match_buf, m->match_sz);
- memcpy(m2->match_buf, m->match_buf, m->match_sz);
+ /* Lower dword is not set, nothing to do. */
+ if (!mask->match_buf[off + 3])
+ return;
- for (i = 0; i < fc_sz; i++) {
- is_first_matcher = !(combination_num & BIT(i));
- hws_bwc_matcher_complex_params_clear_fld(ctx,
- fc[i].fname,
- is_first_matcher ?
- m2 : m1);
- }
+ /* Higher dwords also present in `mask`, no ambiguity. */
+ if (mask->match_buf[off] || mask->match_buf[off + 1] ||
+ mask->match_buf[off + 2])
+ return;
+
+ /* There are no higher dwords in `orig`, i.e. we match on IPv4. */
+ if (!orig->match_buf[off] && !orig->match_buf[off + 1] &&
+ !orig->match_buf[off + 2])
+ return;
- MLX5_SET(fte_match_param, m2->match_buf,
- misc_parameters_2.metadata_reg_c_6, -1);
+ /* Put the lower dword back in `orig`. It is always safe to do this, the
+ * dword will just be picked up in the next submask.
+ */
+ orig->match_buf[off + 3] = mask->match_buf[off + 3];
+ mask->match_buf[off + 3] = 0;
}
-static void
-hws_bwc_matcher_complex_params_destroy(struct mlx5hws_match_parameters *mask_1,
- struct mlx5hws_match_parameters *mask_2)
+static void hws_avoid_ipv6_split(struct mlx5hws_match_parameters *orig,
+ struct mlx5hws_match_parameters *mask)
{
- kfree(mask_1->match_buf);
- kfree(mask_2->match_buf);
+ hws_avoid_ipv6_split_of(orig, mask,
+ __mlx5_dw_off(fte_match_param,
+ outer_headers.src_ipv4_src_ipv6));
+ hws_avoid_ipv6_split_of(orig, mask,
+ __mlx5_dw_off(fte_match_param,
+ outer_headers.dst_ipv4_dst_ipv6));
+ hws_avoid_ipv6_split_of(orig, mask,
+ __mlx5_dw_off(fte_match_param,
+ inner_headers.src_ipv4_src_ipv6));
+ hws_avoid_ipv6_split_of(orig, mask,
+ __mlx5_dw_off(fte_match_param,
+ inner_headers.dst_ipv4_dst_ipv6));
}
-static int
-hws_bwc_matcher_complex_params_create(struct mlx5hws_context *ctx,
- u8 match_criteria,
- struct mlx5hws_match_parameters *mask,
- struct mlx5hws_match_parameters *mask_1,
- struct mlx5hws_match_parameters *mask_2)
+/* Build a subset of the `orig` match parameters into `mask`. This subset is
+ * guaranteed to fit in a single definer an as such is a candidate for being a
+ * part of a complex matcher. Upon successful execution, the match params that
+ * go into `mask` are cleared from `orig`.
+ */
+static int hws_get_simple_params(struct mlx5hws_context *ctx, u8 match_criteria,
+ struct mlx5hws_match_parameters *orig,
+ struct mlx5hws_match_parameters *mask)
{
- struct mlx5hws_definer_fc *fc;
- u32 num_of_combinations;
- int fc_sz = 0;
- int res = 0;
- u32 i;
-
- if (MLX5_GET(fte_match_param, mask->match_buf,
- misc_parameters_2.metadata_reg_c_6)) {
- mlx5hws_err(ctx, "Complex matcher: REG_C_6 matching is reserved\n");
- res = -EINVAL;
- goto out;
- }
+ bool added_inner_ipv, added_outer_ipv;
+ int dword_idx;
+ u32 *backup;
+ int ret;
- mask_1->match_buf = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param),
- GFP_KERNEL);
- mask_2->match_buf = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param),
- GFP_KERNEL);
- if (!mask_1->match_buf || !mask_2->match_buf) {
- mlx5hws_err(ctx, "Complex matcher: failed to allocate match_param\n");
- res = -ENOMEM;
- goto free_params;
- }
+ dword_idx = hws_get_last_set_dword_idx(orig);
+ /* Nothing to do, we consumed all of the match params before. */
+ if (dword_idx == -1)
+ return 0;
- mask_1->match_sz = mask->match_sz;
- mask_2->match_sz = mask->match_sz;
+ backup = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
+ if (!backup)
+ return -ENOMEM;
- fc = mlx5hws_definer_conv_match_params_to_compressed_fc(ctx,
- match_criteria,
- mask->match_buf,
- &fc_sz);
- if (!fc) {
- res = -ENOMEM;
- goto free_params;
- }
+ while (1) {
+ dword_idx = hws_get_last_set_dword_idx(orig);
+ /* Nothing to do, we consumed all of the original match params
+ * into this subset, which still fits into a single matcher.
+ */
+ if (dword_idx == -1) {
+ ret = 0;
+ goto free_backup;
+ }
- if (fc_sz >= sizeof(num_of_combinations) * BITS_PER_BYTE) {
- mlx5hws_err(ctx,
- "Complex matcher: too many match parameters (%d)\n",
- fc_sz);
- res = -EINVAL;
- goto free_fc;
+ memcpy(backup, mask->match_buf, mask->match_sz);
+
+ /* Try to add this dword to the current subset. */
+ hws_add_dword_to_mask(mask, orig, dword_idx, &added_inner_ipv,
+ &added_outer_ipv);
+
+ if (hws_match_params_exceeds_definer(ctx, match_criteria, mask,
+ false)) {
+ /* We just added a match param that makes the definer
+ * too large. Revert and return what we had before.
+ * Note that we can't just zero out the affected fields,
+ * because it's possible that the dword we're looking at
+ * wasn't zero before (e.g. it included auto-added
+ * matches in IP version. This is why we employ the
+ * rather cumbersome memcpy for backing up.
+ */
+ memcpy(mask->match_buf, backup, mask->match_sz);
+ /* Possible future improvement: We can't add any more
+ * dwords, but it may be possible to squeeze in
+ * individual bytes, as definers have special slots for
+ * those.
+ *
+ * For now, keep the code simple. This results in an
+ * extra submatcher in some cases, but it's good enough.
+ */
+ ret = 0;
+ break;
+ }
+
+ /* The current subset of match params still fits in a single
+ * definer. Remove the dword from the original mask.
+ *
+ * Also remove any explicit match on IP version if we just
+ * included one here. We will still automatically add it to
+ * accompany any IP address fragment, but do not need to
+ * consider it by itself.
+ */
+ hws_remove_dword_from_mask(orig, dword_idx, added_inner_ipv,
+ added_outer_ipv);
}
- /* We have list of all the match fields from the match parameter.
- * Now try all the possibilities of splitting them into two match
- * buffers and look for the supported combination.
+ /* Make sure we have not picked up a single lower dword of an IPv6
+ * address, as the firmware will erroneously treat it as an IPv4
+ * address.
*/
- num_of_combinations = 1 << fc_sz;
+ hws_avoid_ipv6_split(orig, mask);
- /* Start from combination at index 1 - we know that 0 is unsupported */
- for (i = 1; i < num_of_combinations; i++) {
- if (!hws_bwc_matcher_complex_params_comb_is_valid(fc, fc_sz, i))
- continue;
+free_backup:
+ kfree(backup);
- hws_bwc_matcher_complex_params_comb_create(ctx,
- mask, mask_1, mask_2,
- fc, fc_sz, i);
- /* We now have two separate sets of match params.
- * Check if each of them can be used in its own matcher.
+ return ret;
+}
+
+static int
+hws_bwc_matcher_split_mask(struct mlx5hws_context *ctx, u8 match_criteria,
+ const struct mlx5hws_match_parameters *mask,
+ struct mlx5hws_match_parameters *submasks,
+ int *num_submasks)
+{
+ struct mlx5hws_match_parameters mask_copy;
+ int ret, i = 0;
+
+ mask_copy.match_sz = MLX5_ST_SZ_BYTES(fte_match_param);
+ mask_copy.match_buf = kzalloc(mask_copy.match_sz, GFP_KERNEL);
+ if (!mask_copy.match_buf)
+ return -ENOMEM;
+
+ memcpy(mask_copy.match_buf, mask->match_buf, mask->match_sz);
+
+ while (!hws_match_mask_is_empty(&mask_copy)) {
+ if (i >= MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS) {
+ mlx5hws_err(ctx,
+ "Complex matcher: mask too large for %d matchers\n",
+ MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS);
+ ret = -E2BIG;
+ goto free_copy;
+ }
+ /* All but the first matcher need to match on register C6 to
+ * connect pieces of the complex rule together.
*/
- if (!mlx5hws_bwc_match_params_is_complex(ctx,
- match_criteria,
- mask_1) &&
- !mlx5hws_bwc_match_params_is_complex(ctx,
- match_criteria,
- mask_2))
- break;
+ if (i > 0) {
+ MLX5_SET(fte_match_param, submasks[i].match_buf,
+ misc_parameters_2.metadata_reg_c_6, -1);
+ match_criteria |= MLX5HWS_DEFINER_MATCH_CRITERIA_MISC2;
+ }
+ ret = hws_get_simple_params(ctx, match_criteria, &mask_copy,
+ &submasks[i]);
+ if (ret < 0)
+ goto free_copy;
+ i++;
}
- if (i == num_of_combinations) {
- /* We've scanned all the combinations, but to no avail */
- mlx5hws_err(ctx, "Complex matcher: couldn't find match params combination\n");
- res = -EINVAL;
- goto free_fc;
- }
+ *num_submasks = i;
+ ret = 0;
- kfree(fc);
- return 0;
+free_copy:
+ kfree(mask_copy.match_buf);
-free_fc:
- kfree(fc);
-free_params:
- hws_bwc_matcher_complex_params_destroy(mask_1, mask_2);
-out:
- return res;
+ return ret;
}
-static int
-hws_bwc_isolated_table_create(struct mlx5hws_bwc_matcher *bwc_matcher,
- struct mlx5hws_table *table)
+static struct mlx5hws_table *
+hws_isolated_table_create(const struct mlx5hws_bwc_matcher *cmatcher)
{
+ struct mlx5hws_bwc_complex_submatcher *first_subm;
struct mlx5hws_cmd_ft_modify_attr ft_attr = {0};
- struct mlx5hws_context *ctx = table->ctx;
struct mlx5hws_table_attr tbl_attr = {0};
- struct mlx5hws_table *isolated_tbl;
- int ret = 0;
+ struct mlx5hws_table *orig_tbl;
+ struct mlx5hws_context *ctx;
+ struct mlx5hws_table *tbl;
+ int ret;
- tbl_attr.type = table->type;
- tbl_attr.level = table->level;
+ first_subm = &cmatcher->complex->submatchers[0];
+ orig_tbl = first_subm->tbl;
+ ctx = orig_tbl->ctx;
- bwc_matcher->complex->isolated_tbl =
- mlx5hws_table_create(ctx, &tbl_attr);
- isolated_tbl = bwc_matcher->complex->isolated_tbl;
- if (!isolated_tbl)
- return -EINVAL;
+ tbl_attr.type = orig_tbl->type;
+ tbl_attr.level = orig_tbl->level;
+ tbl = mlx5hws_table_create(ctx, &tbl_attr);
+ if (!tbl)
+ return ERR_PTR(-EINVAL);
- /* Set the default miss of the isolated table to
- * point to the end anchor of the original matcher.
+ /* Set the default miss of the isolated table to point
+ * to the end anchor of the original matcher.
*/
- mlx5hws_cmd_set_attr_connect_miss_tbl(ctx,
- isolated_tbl->fw_ft_type,
- isolated_tbl->type,
- &ft_attr);
- ft_attr.table_miss_id = bwc_matcher->matcher->end_ft_id;
-
- ret = mlx5hws_cmd_flow_table_modify(ctx->mdev,
- &ft_attr,
- isolated_tbl->ft_id);
+ mlx5hws_cmd_set_attr_connect_miss_tbl(ctx, tbl->fw_ft_type,
+ tbl->type, &ft_attr);
+ ft_attr.table_miss_id = first_subm->bwc_matcher->matcher->end_ft_id;
+
+ ret = mlx5hws_cmd_flow_table_modify(ctx->mdev, &ft_attr, tbl->ft_id);
if (ret) {
- mlx5hws_err(ctx, "Failed setting isolated tbl default miss\n");
+ mlx5hws_err(ctx, "Complex matcher: failed to set isolated tbl default miss\n");
goto destroy_tbl;
}
- return 0;
+ return tbl;
destroy_tbl:
- mlx5hws_table_destroy(isolated_tbl);
- return ret;
+ mlx5hws_table_destroy(tbl);
+
+ return ERR_PTR(ret);
}
-static void hws_bwc_isolated_table_destroy(struct mlx5hws_table *isolated_tbl)
+static int hws_submatcher_init_first(struct mlx5hws_bwc_matcher *cmatcher,
+ struct mlx5hws_table *table, u32 priority,
+ u8 match_criteria,
+ struct mlx5hws_match_parameters *mask)
{
- /* This table is isolated - no table is pointing to it, no need to
- * disconnect it from anywhere, it won't affect any other table's miss.
+ enum mlx5hws_action_type action_types[HWS_NUM_CHAIN_ACTIONS];
+ struct mlx5hws_bwc_complex_submatcher *subm;
+ int ret;
+
+ subm = &cmatcher->complex->submatchers[0];
+
+ /* The first submatcher lives in the original table and does not have an
+ * associated jump to table action. It also points to the outer complex
+ * matcher.
*/
- mlx5hws_table_destroy(isolated_tbl);
+ subm->tbl = table;
+ subm->action_tbl = NULL;
+ subm->bwc_matcher = cmatcher;
+
+ action_types[0] = MLX5HWS_ACTION_TYP_MODIFY_HDR;
+ action_types[1] = MLX5HWS_ACTION_TYP_TBL;
+ action_types[2] = MLX5HWS_ACTION_TYP_LAST;
+
+ ret = mlx5hws_bwc_matcher_create_simple(subm->bwc_matcher, subm->tbl,
+ priority, match_criteria, mask,
+ action_types);
+ if (ret)
+ return ret;
+
+ subm->bwc_matcher->matcher_type = MLX5HWS_BWC_MATCHER_COMPLEX_FIRST;
+
+ ret = rhashtable_init(&subm->rules_hash, &hws_rules_hash_params);
+ if (ret)
+ goto destroy_matcher;
+ mutex_init(&subm->hash_lock);
+ ida_init(&subm->chain_ida);
+
+ return 0;
+
+destroy_matcher:
+ mlx5hws_bwc_matcher_destroy_simple(subm->bwc_matcher);
+
+ return ret;
}
-static int
-hws_bwc_isolated_matcher_create(struct mlx5hws_bwc_matcher *bwc_matcher,
- struct mlx5hws_table *table,
- u8 match_criteria_enable,
- struct mlx5hws_match_parameters *mask)
+static int hws_submatcher_init(struct mlx5hws_bwc_matcher *cmatcher, int idx,
+ struct mlx5hws_table *table, u32 priority,
+ u8 match_criteria,
+ struct mlx5hws_match_parameters *mask)
{
- struct mlx5hws_table *isolated_tbl = bwc_matcher->complex->isolated_tbl;
- struct mlx5hws_bwc_matcher *isolated_bwc_matcher;
- struct mlx5hws_context *ctx = table->ctx;
+ enum mlx5hws_action_type action_types[HWS_NUM_CHAIN_ACTIONS];
+ struct mlx5hws_bwc_complex_submatcher *subm;
+ bool is_last;
int ret;
- isolated_bwc_matcher = kzalloc(sizeof(*bwc_matcher), GFP_KERNEL);
- if (!isolated_bwc_matcher)
- return -ENOMEM;
+ if (!idx)
+ return hws_submatcher_init_first(cmatcher, table, priority,
+ match_criteria, mask);
+
+ subm = &cmatcher->complex->submatchers[idx];
+ is_last = idx == cmatcher->complex->num_submatchers - 1;
+
+ subm->tbl = hws_isolated_table_create(cmatcher);
+ if (IS_ERR(subm->tbl))
+ return PTR_ERR(subm->tbl);
+
+ subm->action_tbl =
+ mlx5hws_action_create_dest_table(subm->tbl->ctx, subm->tbl,
+ MLX5HWS_ACTION_FLAG_HWS_FDB);
+ if (!subm->action_tbl) {
+ ret = -EINVAL;
+ goto destroy_tbl;
+ }
+
+ subm->bwc_matcher = kzalloc(sizeof(*subm->bwc_matcher), GFP_KERNEL);
+ if (!subm->bwc_matcher) {
+ ret = -ENOMEM;
+ goto destroy_action;
+ }
- bwc_matcher->complex->isolated_bwc_matcher = isolated_bwc_matcher;
+ /* Every matcher other than the first also matches of register C6 to
+ * bind subrules together in the complex rule using the chain ids.
+ */
+ match_criteria |= MLX5HWS_DEFINER_MATCH_CRITERIA_MISC2;
- /* Isolated BWC matcher needs access to the first BWC matcher */
- isolated_bwc_matcher->complex_first_bwc_matcher = bwc_matcher;
+ action_types[0] = MLX5HWS_ACTION_TYP_MODIFY_HDR;
+ action_types[1] = MLX5HWS_ACTION_TYP_TBL;
+ action_types[2] = MLX5HWS_ACTION_TYP_LAST;
- /* Isolated matcher needs to match on REG_C_6,
- * so make sure its criteria bit is on.
+ /* Every matcher other than the last sets register C6 and jumps to the
+ * next submatcher's table. The final submatcher will use the
+ * user-supplied actions and will attach an action template at rule
+ * insertion time.
*/
- match_criteria_enable |= MLX5HWS_DEFINER_MATCH_CRITERIA_MISC2;
-
- ret = mlx5hws_bwc_matcher_create_simple(isolated_bwc_matcher,
- isolated_tbl,
- 0,
- match_criteria_enable,
- mask,
- NULL);
- if (ret) {
- mlx5hws_err(ctx, "Complex matcher: failed creating isolated BWC matcher\n");
+ ret = mlx5hws_bwc_matcher_create_simple(subm->bwc_matcher, subm->tbl,
+ priority, match_criteria, mask,
+ is_last ? NULL : action_types);
+ if (ret)
goto free_matcher;
- }
+
+ subm->bwc_matcher->matcher_type =
+ MLX5HWS_BWC_MATCHER_COMPLEX_SUBMATCHER;
+
+ ret = rhashtable_init(&subm->rules_hash, &hws_rules_hash_params);
+ if (ret)
+ goto destroy_matcher;
+ mutex_init(&subm->hash_lock);
+ ida_init(&subm->chain_ida);
return 0;
+destroy_matcher:
+ mlx5hws_bwc_matcher_destroy_simple(subm->bwc_matcher);
free_matcher:
- kfree(bwc_matcher->complex->isolated_bwc_matcher);
+ kfree(subm->bwc_matcher);
+destroy_action:
+ mlx5hws_action_destroy(subm->action_tbl);
+destroy_tbl:
+ mlx5hws_table_destroy(subm->tbl);
+
return ret;
}
-static void
-hws_bwc_isolated_matcher_destroy(struct mlx5hws_bwc_matcher *bwc_matcher)
+static void hws_submatcher_destroy(struct mlx5hws_bwc_matcher *cmatcher,
+ int idx)
{
- mlx5hws_bwc_matcher_destroy_simple(bwc_matcher);
- kfree(bwc_matcher);
+ struct mlx5hws_bwc_complex_submatcher *subm;
+
+ subm = &cmatcher->complex->submatchers[idx];
+
+ ida_destroy(&subm->chain_ida);
+ mutex_destroy(&subm->hash_lock);
+ rhashtable_destroy(&subm->rules_hash);
+
+ if (subm->bwc_matcher) {
+ mlx5hws_bwc_matcher_destroy_simple(subm->bwc_matcher);
+ if (idx)
+ kfree(subm->bwc_matcher);
+ }
+
+ /* We own all of the isolated tables, but not the original one. */
+ if (idx) {
+ mlx5hws_action_destroy(subm->action_tbl);
+ mlx5hws_table_destroy(subm->tbl);
+ }
}
static int
-hws_bwc_isolated_actions_create(struct mlx5hws_bwc_matcher *bwc_matcher,
- struct mlx5hws_table *table)
+hws_complex_data_actions_init(struct mlx5hws_bwc_matcher_complex_data *cdata)
{
- struct mlx5hws_table *isolated_tbl = bwc_matcher->complex->isolated_tbl;
+ struct mlx5hws_context *ctx = cdata->submatchers[0].tbl->ctx;
u8 modify_hdr_action[MLX5_ST_SZ_BYTES(set_action_in)] = {0};
- struct mlx5hws_context *ctx = table->ctx;
struct mlx5hws_action_mh_pattern ptrn;
int ret = 0;
- /* Create action to jump to isolated table */
-
- bwc_matcher->complex->action_go_to_tbl =
- mlx5hws_action_create_dest_table(ctx,
- isolated_tbl,
- MLX5HWS_ACTION_FLAG_HWS_FDB);
- if (!bwc_matcher->complex->action_go_to_tbl) {
- mlx5hws_err(ctx, "Complex matcher: failed to create go-to-tbl action\n");
- return -EINVAL;
- }
-
/* Create modify header action to set REG_C_6 */
-
MLX5_SET(set_action_in, modify_hdr_action,
action_type, MLX5_MODIFICATION_TYPE_SET);
MLX5_SET(set_action_in, modify_hdr_action,
ptrn.data = (void *)modify_hdr_action;
ptrn.sz = MLX5HWS_ACTION_DOUBLE_SIZE;
- bwc_matcher->complex->action_metadata =
+ cdata->action_metadata =
mlx5hws_action_create_modify_header(ctx, 1, &ptrn, 0,
MLX5HWS_ACTION_FLAG_HWS_FDB);
- if (!bwc_matcher->complex->action_metadata) {
- ret = -EINVAL;
- goto destroy_action_go_to_tbl;
+ if (!cdata->action_metadata) {
+ mlx5hws_err(ctx, "Complex matcher: failed to create set reg C6 action\n");
+ return -EINVAL;
}
/* Create last action */
-
- bwc_matcher->complex->action_last =
+ cdata->action_last =
mlx5hws_action_create_last(ctx, MLX5HWS_ACTION_FLAG_HWS_FDB);
- if (!bwc_matcher->complex->action_last) {
+ if (!cdata->action_last) {
mlx5hws_err(ctx, "Complex matcher: failed to create last action\n");
ret = -EINVAL;
goto destroy_action_metadata;
return 0;
destroy_action_metadata:
- mlx5hws_action_destroy(bwc_matcher->complex->action_metadata);
-destroy_action_go_to_tbl:
- mlx5hws_action_destroy(bwc_matcher->complex->action_go_to_tbl);
+ mlx5hws_action_destroy(cdata->action_metadata);
+
return ret;
}
static void
-hws_bwc_isolated_actions_destroy(struct mlx5hws_bwc_matcher *bwc_matcher)
+hws_complex_data_actions_destroy(struct mlx5hws_bwc_matcher_complex_data *cdata)
{
- mlx5hws_action_destroy(bwc_matcher->complex->action_last);
- mlx5hws_action_destroy(bwc_matcher->complex->action_metadata);
- mlx5hws_action_destroy(bwc_matcher->complex->action_go_to_tbl);
+ mlx5hws_action_destroy(cdata->action_last);
+ mlx5hws_action_destroy(cdata->action_metadata);
}
int mlx5hws_bwc_matcher_create_complex(struct mlx5hws_bwc_matcher *bwc_matcher,
struct mlx5hws_table *table,
- u32 priority,
- u8 match_criteria_enable,
+ u32 priority, u8 match_criteria_enable,
struct mlx5hws_match_parameters *mask)
{
- enum mlx5hws_action_type complex_init_action_types[3];
- struct mlx5hws_bwc_matcher *isolated_bwc_matcher;
- struct mlx5hws_match_parameters mask_1 = {0};
- struct mlx5hws_match_parameters mask_2 = {0};
+ struct mlx5hws_match_parameters
+ submasks[MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS] = {0};
+ struct mlx5hws_bwc_matcher_complex_data *cdata;
struct mlx5hws_context *ctx = table->ctx;
- int ret;
-
- ret = hws_bwc_matcher_complex_params_create(table->ctx,
- match_criteria_enable,
- mask, &mask_1, &mask_2);
- if (ret)
- goto err;
-
- bwc_matcher->complex =
- kzalloc(sizeof(*bwc_matcher->complex), GFP_KERNEL);
- if (!bwc_matcher->complex) {
- ret = -ENOMEM;
- goto free_masks;
- }
+ int num_submatchers;
+ int i, ret;
- ret = rhashtable_init(&bwc_matcher->complex->refcount_hash,
- &hws_refcount_hash);
- if (ret) {
- mlx5hws_err(ctx, "Complex matcher: failed to initialize rhashtable\n");
- goto free_complex;
+ for (i = 0; i < ARRAY_SIZE(submasks); i++) {
+ submasks[i].match_sz = MLX5_ST_SZ_BYTES(fte_match_param);
+ submasks[i].match_buf = kzalloc(submasks[i].match_sz,
+ GFP_KERNEL);
+ if (!submasks[i].match_buf) {
+ ret = -ENOMEM;
+ goto free_submasks;
+ }
}
- mutex_init(&bwc_matcher->complex->hash_lock);
- ida_init(&bwc_matcher->complex->metadata_ida);
-
- /* Create initial action template for the first matcher.
- * Usually the initial AT is just dummy, but in case of complex
- * matcher we know exactly which actions should it have.
- */
-
- complex_init_action_types[0] = MLX5HWS_ACTION_TYP_MODIFY_HDR;
- complex_init_action_types[1] = MLX5HWS_ACTION_TYP_TBL;
- complex_init_action_types[2] = MLX5HWS_ACTION_TYP_LAST;
-
- /* Create the first matcher */
-
- ret = mlx5hws_bwc_matcher_create_simple(bwc_matcher,
- table,
- priority,
- match_criteria_enable,
- &mask_1,
- complex_init_action_types);
+ ret = hws_bwc_matcher_split_mask(ctx, match_criteria_enable, mask,
+ submasks, &num_submatchers);
if (ret)
- goto destroy_ida;
-
- /* Create isolated table to hold the second isolated matcher */
+ goto free_submasks;
- ret = hws_bwc_isolated_table_create(bwc_matcher, table);
- if (ret) {
- mlx5hws_err(ctx, "Complex matcher: failed creating isolated table\n");
- goto destroy_first_matcher;
+ cdata = kzalloc(sizeof(*cdata), GFP_KERNEL);
+ if (!cdata) {
+ ret = -ENOMEM;
+ goto free_submasks;
}
- /* Now create the second BWC matcher - the isolated one */
+ bwc_matcher->complex = cdata;
+ cdata->num_submatchers = num_submatchers;
- ret = hws_bwc_isolated_matcher_create(bwc_matcher, table,
- match_criteria_enable, &mask_2);
- if (ret) {
- mlx5hws_err(ctx, "Complex matcher: failed creating isolated matcher\n");
- goto destroy_isolated_tbl;
+ for (i = 0; i < num_submatchers; i++) {
+ ret = hws_submatcher_init(bwc_matcher, i, table, priority,
+ match_criteria_enable, &submasks[i]);
+ if (ret)
+ goto destroy_submatchers;
}
- /* Create action for isolated matcher's rules */
-
- ret = hws_bwc_isolated_actions_create(bwc_matcher, table);
- if (ret) {
- mlx5hws_err(ctx, "Complex matcher: failed creating isolated actions\n");
- goto destroy_isolated_matcher;
- }
+ ret = hws_complex_data_actions_init(cdata);
+ if (ret)
+ goto destroy_submatchers;
- hws_bwc_matcher_complex_params_destroy(&mask_1, &mask_2);
- return 0;
+ ret = 0;
+ goto free_submasks;
-destroy_isolated_matcher:
- isolated_bwc_matcher = bwc_matcher->complex->isolated_bwc_matcher;
- hws_bwc_isolated_matcher_destroy(isolated_bwc_matcher);
-destroy_isolated_tbl:
- hws_bwc_isolated_table_destroy(bwc_matcher->complex->isolated_tbl);
-destroy_first_matcher:
- mlx5hws_bwc_matcher_destroy_simple(bwc_matcher);
-destroy_ida:
- ida_destroy(&bwc_matcher->complex->metadata_ida);
- mutex_destroy(&bwc_matcher->complex->hash_lock);
- rhashtable_destroy(&bwc_matcher->complex->refcount_hash);
-free_complex:
- kfree(bwc_matcher->complex);
+destroy_submatchers:
+ while (i--)
+ hws_submatcher_destroy(bwc_matcher, i);
+ kfree(cdata);
bwc_matcher->complex = NULL;
-free_masks:
- hws_bwc_matcher_complex_params_destroy(&mask_1, &mask_2);
-err:
+
+free_submasks:
+ for (i = 0; i < ARRAY_SIZE(submasks); i++)
+ kfree(submasks[i].match_buf);
+
return ret;
}
void
mlx5hws_bwc_matcher_destroy_complex(struct mlx5hws_bwc_matcher *bwc_matcher)
{
- struct mlx5hws_bwc_matcher *isolated_bwc_matcher =
- bwc_matcher->complex->isolated_bwc_matcher;
-
- hws_bwc_isolated_actions_destroy(bwc_matcher);
- hws_bwc_isolated_matcher_destroy(isolated_bwc_matcher);
- hws_bwc_isolated_table_destroy(bwc_matcher->complex->isolated_tbl);
- mlx5hws_bwc_matcher_destroy_simple(bwc_matcher);
- ida_destroy(&bwc_matcher->complex->metadata_ida);
- mutex_destroy(&bwc_matcher->complex->hash_lock);
- rhashtable_destroy(&bwc_matcher->complex->refcount_hash);
+ int i;
+
+ hws_complex_data_actions_destroy(bwc_matcher->complex);
+ for (i = 0; i < bwc_matcher->complex->num_submatchers; i++)
+ hws_submatcher_destroy(bwc_matcher, i);
kfree(bwc_matcher->complex);
bwc_matcher->complex = NULL;
}
-static void
-hws_bwc_matcher_complex_hash_lock(struct mlx5hws_bwc_matcher *bwc_matcher)
-{
- mutex_lock(&bwc_matcher->complex->hash_lock);
-}
-
-static void
-hws_bwc_matcher_complex_hash_unlock(struct mlx5hws_bwc_matcher *bwc_matcher)
-{
- mutex_unlock(&bwc_matcher->complex->hash_lock);
-}
-
static int
-hws_bwc_rule_complex_hash_node_get(struct mlx5hws_bwc_rule *bwc_rule,
- struct mlx5hws_match_parameters *params)
+hws_complex_get_subrule_data(struct mlx5hws_bwc_rule *bwc_rule,
+ struct mlx5hws_bwc_complex_submatcher *subm,
+ u32 *match_params)
+__must_hold(&subm->hash_lock)
{
- struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
- struct mlx5hws_bwc_complex_rule_hash_node *node, *old_node;
- struct rhashtable *refcount_hash;
- int ret, i;
-
- bwc_rule->complex_hash_node = NULL;
+ struct mlx5hws_bwc_matcher *bwc_matcher = subm->bwc_matcher;
+ struct mlx5hws_bwc_complex_subrule_data *sr_data, *old_data;
+ struct mlx5hws_match_template *mt;
+ int ret;
- node = kzalloc(sizeof(*node), GFP_KERNEL);
- if (unlikely(!node))
+ sr_data = kzalloc(sizeof(*sr_data), GFP_KERNEL);
+ if (!sr_data)
return -ENOMEM;
- ret = ida_alloc(&bwc_matcher->complex->metadata_ida, GFP_KERNEL);
+ ret = ida_alloc(&subm->chain_ida, GFP_KERNEL);
if (ret < 0)
- goto err_free_node;
- node->tag = ret;
+ goto free_sr_data;
+ sr_data->chain_id = ret;
- refcount_set(&node->refcount, 1);
+ refcount_set(&sr_data->refcount, 1);
- /* Clear match buffer - turn off all the unrelated fields
- * in accordance with the match params mask for the first
- * matcher out of the two parts of the complex matcher.
- * The resulting mask is the key for the hash.
- */
- for (i = 0; i < MLX5_ST_SZ_DW_MATCH_PARAM; i++)
- node->match_buf[i] = params->match_buf[i] &
- bwc_matcher->mt->match_param[i];
-
- refcount_hash = &bwc_matcher->complex->refcount_hash;
- old_node = rhashtable_lookup_get_insert_fast(refcount_hash,
- &node->hash_node,
- hws_refcount_hash);
- if (IS_ERR(old_node)) {
- ret = PTR_ERR(old_node);
- goto err_free_ida;
+ mt = bwc_matcher->matcher->mt;
+ mlx5hws_definer_create_tag(match_params, mt->fc, mt->fc_sz,
+ (u8 *)&sr_data->match_tag);
+
+ old_data = rhashtable_lookup_get_insert_fast(&subm->rules_hash,
+ &sr_data->hash_node,
+ hws_rules_hash_params);
+ if (IS_ERR(old_data)) {
+ ret = PTR_ERR(old_data);
+ goto free_ida;
}
- if (old_node) {
+ if (old_data) {
/* Rule with the same tag already exists - update refcount */
- refcount_inc(&old_node->refcount);
+ refcount_inc(&old_data->refcount);
/* Let the new rule use the same tag as the existing rule.
* Note that we don't have any indication for the rule creation
* process that a rule with similar matching params already
* There's some performance advantage in skipping such cases,
* so this is left for future optimizations.
*/
- ida_free(&bwc_matcher->complex->metadata_ida, node->tag);
- kfree(node);
- node = old_node;
+ bwc_rule->subrule_data = old_data;
+ ret = 0;
+ goto free_ida;
}
- bwc_rule->complex_hash_node = node;
+ bwc_rule->subrule_data = sr_data;
return 0;
-err_free_ida:
- ida_free(&bwc_matcher->complex->metadata_ida, node->tag);
-err_free_node:
- kfree(node);
+free_ida:
+ ida_free(&subm->chain_ida, sr_data->chain_id);
+free_sr_data:
+ kfree(sr_data);
+
return ret;
}
static void
-hws_bwc_rule_complex_hash_node_put(struct mlx5hws_bwc_rule *bwc_rule,
- bool *is_last_rule)
+hws_complex_put_subrule_data(struct mlx5hws_bwc_rule *bwc_rule,
+ struct mlx5hws_bwc_complex_submatcher *subm,
+ bool *is_last_rule)
+__must_hold(&subm->hash_lock)
{
- struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
- struct mlx5hws_bwc_complex_rule_hash_node *node;
+ struct mlx5hws_bwc_complex_subrule_data *sr_data;
if (is_last_rule)
*is_last_rule = false;
- node = bwc_rule->complex_hash_node;
- if (refcount_dec_and_test(&node->refcount)) {
- rhashtable_remove_fast(&bwc_matcher->complex->refcount_hash,
- &node->hash_node,
- hws_refcount_hash);
- ida_free(&bwc_matcher->complex->metadata_ida, node->tag);
- kfree(node);
+ sr_data = bwc_rule->subrule_data;
+ if (refcount_dec_and_test(&sr_data->refcount)) {
+ rhashtable_remove_fast(&subm->rules_hash,
+ &sr_data->hash_node,
+ hws_rules_hash_params);
+ ida_free(&subm->chain_ida, sr_data->chain_id);
+ kfree(sr_data);
if (is_last_rule)
*is_last_rule = true;
}
- bwc_rule->complex_hash_node = NULL;
+ bwc_rule->subrule_data = NULL;
}
-int mlx5hws_bwc_rule_create_complex(struct mlx5hws_bwc_rule *bwc_rule,
- struct mlx5hws_match_parameters *params,
- u32 flow_source,
- struct mlx5hws_rule_action rule_actions[],
- u16 bwc_queue_idx)
+static int hws_complex_subrule_create(struct mlx5hws_bwc_matcher *cmatcher,
+ struct mlx5hws_bwc_rule *subrule,
+ u32 *match_params, u32 flow_source,
+ int bwc_queue_idx, int subm_idx,
+ struct mlx5hws_rule_action *actions,
+ u32 *chain_id)
{
- struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
- struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
+ struct mlx5hws_rule_action chain_actions[HWS_NUM_CHAIN_ACTIONS] = {0};
u8 modify_hdr_action[MLX5_ST_SZ_BYTES(set_action_in)] = {0};
- struct mlx5hws_rule_action rule_actions_1[3] = {0};
- struct mlx5hws_bwc_matcher *isolated_bwc_matcher;
- u32 *match_buf_2;
- u32 metadata_val;
- int ret = 0;
+ struct mlx5hws_bwc_matcher_complex_data *cdata;
+ struct mlx5hws_bwc_complex_submatcher *subm;
+ int ret;
- isolated_bwc_matcher = bwc_matcher->complex->isolated_bwc_matcher;
- bwc_rule->isolated_bwc_rule =
- mlx5hws_bwc_rule_alloc(isolated_bwc_matcher);
- if (unlikely(!bwc_rule->isolated_bwc_rule))
- return -ENOMEM;
+ cdata = cmatcher->complex;
+ subm = &cdata->submatchers[subm_idx];
- hws_bwc_matcher_complex_hash_lock(bwc_matcher);
+ mutex_lock(&subm->hash_lock);
- /* Get a new hash node for this complex rule.
- * If this is a unique set of match params for the first matcher,
- * we will get a new hash node with newly allocated IDA.
- * Otherwise we will get an existing node with IDA and updated refcount.
- */
- ret = hws_bwc_rule_complex_hash_node_get(bwc_rule, params);
- if (unlikely(ret)) {
- mlx5hws_err(ctx, "Complex rule: failed getting RHT node for this rule\n");
- goto free_isolated_rule;
+ ret = hws_complex_get_subrule_data(subrule, subm, match_params);
+ if (ret)
+ goto unlock;
+
+ *chain_id = subrule->subrule_data->chain_id;
+
+ if (!actions) {
+ MLX5_SET(set_action_in, modify_hdr_action, data, *chain_id);
+ chain_actions[0].action = cdata->action_metadata;
+ chain_actions[0].modify_header.data = modify_hdr_action;
+ chain_actions[1].action =
+ cdata->submatchers[subm_idx + 1].action_tbl;
+ chain_actions[2].action = cdata->action_last;
+ actions = chain_actions;
}
- /* No need to clear match buffer's fields in accordance to what
- * will actually be matched on first and second matchers.
- * Both matchers were created with the appropriate masks
- * and each of them holds the appropriate field copy array,
- * so rule creation will use only the fields that will be copied
- * in accordance with setters in field copy array.
- * We do, however, need to temporary allocate match buffer
- * for the second (isolated) rule in order to not modify
- * user's match params buffer.
- */
-
- match_buf_2 = kmemdup(params->match_buf,
- MLX5_ST_SZ_BYTES(fte_match_param),
- GFP_KERNEL);
- if (unlikely(!match_buf_2)) {
- mlx5hws_err(ctx, "Complex rule: failed allocating match_buf\n");
- ret = -ENOMEM;
- goto hash_node_put;
- }
+ ret = mlx5hws_bwc_rule_create_simple(subrule, match_params, actions,
+ flow_source, bwc_queue_idx);
+ if (ret)
+ goto put_subrule_data;
- /* On 2nd matcher, use unique 32-bit ID as a matching tag */
- metadata_val = bwc_rule->complex_hash_node->tag;
- MLX5_SET(fte_match_param, match_buf_2,
- misc_parameters_2.metadata_reg_c_6, metadata_val);
-
- /* Isolated rule's rule_actions contain all the original actions */
- ret = mlx5hws_bwc_rule_create_simple(bwc_rule->isolated_bwc_rule,
- match_buf_2,
- rule_actions,
- flow_source,
- bwc_queue_idx);
- kfree(match_buf_2);
- if (unlikely(ret)) {
- mlx5hws_err(ctx,
- "Complex rule: failed creating isolated BWC rule (%d)\n",
- ret);
- goto hash_node_put;
- }
+ ret = 0;
+ goto unlock;
- /* First rule's rule_actions contain setting metadata and
- * jump to isolated table that contains the second matcher.
- * Set metadata value to a unique value for this rule.
- */
+put_subrule_data:
+ hws_complex_put_subrule_data(subrule, subm, NULL);
+unlock:
+ mutex_unlock(&subm->hash_lock);
- MLX5_SET(set_action_in, modify_hdr_action,
- action_type, MLX5_MODIFICATION_TYPE_SET);
- MLX5_SET(set_action_in, modify_hdr_action,
- field, MLX5_MODI_META_REG_C_6);
- MLX5_SET(set_action_in, modify_hdr_action,
- length, 0); /* zero means length of 32 */
- MLX5_SET(set_action_in, modify_hdr_action,
- offset, 0);
- MLX5_SET(set_action_in, modify_hdr_action,
- data, metadata_val);
+ return ret;
+}
- rule_actions_1[0].action = bwc_matcher->complex->action_metadata;
- rule_actions_1[0].modify_header.offset = 0;
- rule_actions_1[0].modify_header.data = modify_hdr_action;
+static int hws_complex_subrule_destroy(struct mlx5hws_bwc_rule *bwc_rule,
+ struct mlx5hws_bwc_matcher *cmatcher,
+ int subm_idx)
+{
+ struct mlx5hws_bwc_matcher_complex_data *cdata;
+ struct mlx5hws_bwc_complex_submatcher *subm;
+ struct mlx5hws_context *ctx;
+ bool is_last_rule;
+ int ret = 0;
- rule_actions_1[1].action = bwc_matcher->complex->action_go_to_tbl;
- rule_actions_1[2].action = bwc_matcher->complex->action_last;
+ cdata = cmatcher->complex;
+ subm = &cdata->submatchers[subm_idx];
+ ctx = subm->tbl->ctx;
- ret = mlx5hws_bwc_rule_create_simple(bwc_rule,
- params->match_buf,
- rule_actions_1,
- flow_source,
- bwc_queue_idx);
+ mutex_lock(&subm->hash_lock);
- if (unlikely(ret)) {
+ hws_complex_put_subrule_data(bwc_rule, subm, &is_last_rule);
+ bwc_rule->rule->skip_delete = !is_last_rule;
+ ret = mlx5hws_bwc_rule_destroy_simple(bwc_rule);
+ if (unlikely(ret))
mlx5hws_err(ctx,
- "Complex rule: failed creating first BWC rule (%d)\n",
- ret);
- goto destroy_isolated_rule;
- }
+ "Complex rule: failed to delete subrule %d (%d)\n",
+ subm_idx, ret);
- hws_bwc_matcher_complex_hash_unlock(bwc_matcher);
+ if (subm_idx)
+ mlx5hws_bwc_rule_free(bwc_rule);
- return 0;
+ mutex_unlock(&subm->hash_lock);
-destroy_isolated_rule:
- mlx5hws_bwc_rule_destroy_simple(bwc_rule->isolated_bwc_rule);
-hash_node_put:
- hws_bwc_rule_complex_hash_node_put(bwc_rule, NULL);
-free_isolated_rule:
- hws_bwc_matcher_complex_hash_unlock(bwc_matcher);
- mlx5hws_bwc_rule_free(bwc_rule->isolated_bwc_rule);
return ret;
}
-int mlx5hws_bwc_rule_destroy_complex(struct mlx5hws_bwc_rule *bwc_rule)
+int mlx5hws_bwc_rule_create_complex(struct mlx5hws_bwc_rule *bwc_rule,
+ struct mlx5hws_match_parameters *params,
+ u32 flow_source,
+ struct mlx5hws_rule_action rule_actions[],
+ u16 bwc_queue_idx)
{
- struct mlx5hws_context *ctx = bwc_rule->bwc_matcher->matcher->tbl->ctx;
- struct mlx5hws_bwc_rule *isolated_bwc_rule;
- int ret_isolated, ret;
- bool is_last_rule;
+ struct mlx5hws_bwc_rule
+ *subrules[MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS] = {0};
+ struct mlx5hws_bwc_matcher *cmatcher = bwc_rule->bwc_matcher;
+ struct mlx5hws_bwc_matcher_complex_data *cdata;
+ struct mlx5hws_rule_action *subrule_actions;
+ struct mlx5hws_bwc_complex_submatcher *subm;
+ struct mlx5hws_bwc_rule *subrule;
+ u32 *match_params;
+ u32 chain_id;
+ int i, ret;
- hws_bwc_matcher_complex_hash_lock(bwc_rule->bwc_matcher);
+ cdata = cmatcher->complex;
+ if (!cdata)
+ return -EINVAL;
- hws_bwc_rule_complex_hash_node_put(bwc_rule, &is_last_rule);
- bwc_rule->rule->skip_delete = !is_last_rule;
+ /* Duplicate user data because we will modify it to set register C6
+ * values. For the same reason, make sure that we allocate a full
+ * match_param even if the user gave us fewer bytes. We need to ensure
+ * there is space for the match on C6.
+ */
+ match_params = kzalloc(MLX5_ST_SZ_BYTES(fte_match_param), GFP_KERNEL);
+ if (!match_params)
+ return -ENOMEM;
- ret = mlx5hws_bwc_rule_destroy_simple(bwc_rule);
- if (unlikely(ret))
- mlx5hws_err(ctx, "BWC complex rule: failed destroying first rule\n");
+ memcpy(match_params, params->match_buf, params->match_sz);
+
+ ret = hws_complex_subrule_create(cmatcher, bwc_rule, match_params,
+ flow_source, bwc_queue_idx, 0,
+ NULL, &chain_id);
+ if (ret)
+ goto free_match_params;
+ subrules[0] = bwc_rule;
+
+ for (i = 1; i < cdata->num_submatchers; i++) {
+ subm = &cdata->submatchers[i];
+ subrule = mlx5hws_bwc_rule_alloc(subm->bwc_matcher);
+ if (!subrule) {
+ ret = -ENOMEM;
+ goto destroy_subrules;
+ }
+
+ /* Match on the previous subrule's chain_id. This is how
+ * subrules are connected in steering.
+ */
+ MLX5_SET(fte_match_param, match_params,
+ misc_parameters_2.metadata_reg_c_6, chain_id);
+
+ /* The last subrule uses the complex rule's user-specified
+ * actions. Everything else uses the chaining rules based on the
+ * next table and chain_id.
+ */
+ subrule_actions =
+ i == cdata->num_submatchers - 1 ? rule_actions : NULL;
+
+ ret = hws_complex_subrule_create(cmatcher, subrule,
+ match_params, flow_source,
+ bwc_queue_idx, i,
+ subrule_actions, &chain_id);
+ if (ret) {
+ mlx5hws_bwc_rule_free(subrule);
+ goto destroy_subrules;
+ }
+
+ subrules[i] = subrule;
+ }
+
+ for (i = 0; i < cdata->num_submatchers - 1; i++)
+ subrules[i]->next_subrule = subrules[i + 1];
- isolated_bwc_rule = bwc_rule->isolated_bwc_rule;
- ret_isolated = mlx5hws_bwc_rule_destroy_simple(isolated_bwc_rule);
- if (unlikely(ret_isolated))
- mlx5hws_err(ctx, "BWC complex rule: failed destroying second (isolated) rule\n");
+ kfree(match_params);
- hws_bwc_matcher_complex_hash_unlock(bwc_rule->bwc_matcher);
+ return 0;
- mlx5hws_bwc_rule_free(isolated_bwc_rule);
+destroy_subrules:
+ while (i--)
+ hws_complex_subrule_destroy(subrules[i], cmatcher, i);
+free_match_params:
+ kfree(match_params);
- return ret || ret_isolated;
+ return ret;
}
-static void
-hws_bwc_matcher_clear_hash_rtcs(struct mlx5hws_bwc_matcher *bwc_matcher)
+int mlx5hws_bwc_rule_destroy_complex(struct mlx5hws_bwc_rule *bwc_rule)
{
- struct mlx5hws_bwc_complex_rule_hash_node *node;
- struct rhashtable_iter iter;
+ struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher;
+ struct mlx5hws_bwc_rule
+ *subrules[MLX5HWS_BWC_COMPLEX_MAX_SUBMATCHERS] = {0};
+ struct mlx5hws_bwc_matcher_complex_data *cdata;
+ int i, err, ret_val;
+
+ cdata = bwc_matcher->complex;
+
+ /* Construct a list of all the subrules we need to destroy. */
+ subrules[0] = bwc_rule;
+ for (i = 1; i < cdata->num_submatchers; i++)
+ subrules[i] = subrules[i - 1]->next_subrule;
+
+ ret_val = 0;
+ for (i = 0; i < cdata->num_submatchers; i++) {
+ err = hws_complex_subrule_destroy(subrules[i], bwc_matcher, i);
+ /* If something goes wrong, plow along to destroy all of the
+ * subrules but return an error upstack.
+ */
+ if (unlikely(err))
+ ret_val = err;
+ }
- rhashtable_walk_enter(&bwc_matcher->complex->refcount_hash, &iter);
- rhashtable_walk_start(&iter);
+ return ret_val;
+}
- while ((node = rhashtable_walk_next(&iter)) != NULL) {
- if (IS_ERR(node))
+static void
+hws_bwc_matcher_init_move(struct mlx5hws_bwc_matcher *bwc_matcher)
+{
+ struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
+ u16 bwc_queues = mlx5hws_bwc_queues(ctx);
+ struct mlx5hws_bwc_rule *bwc_rule;
+ struct list_head *rules_list;
+ int i;
+
+ for (i = 0; i < bwc_queues; i++) {
+ rules_list = &bwc_matcher->rules[i];
+ if (list_empty(rules_list))
continue;
- node->rtc_valid = false;
- }
- rhashtable_walk_stop(&iter);
- rhashtable_walk_exit(&iter);
+ list_for_each_entry(bwc_rule, rules_list, list_node) {
+ if (!bwc_rule->subrule_data)
+ continue;
+ bwc_rule->subrule_data->was_moved = false;
+ }
+ }
}
-int
-mlx5hws_bwc_matcher_move_all_complex(struct mlx5hws_bwc_matcher *bwc_matcher)
+int mlx5hws_bwc_matcher_complex_move(struct mlx5hws_bwc_matcher *bwc_matcher)
{
struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
struct mlx5hws_matcher *matcher = bwc_matcher->matcher;
u16 bwc_queues = mlx5hws_bwc_queues(ctx);
struct mlx5hws_bwc_rule *tmp_bwc_rule;
struct mlx5hws_rule_attr rule_attr;
- struct mlx5hws_table *isolated_tbl;
int move_error = 0, poll_error = 0;
struct mlx5hws_rule *tmp_rule;
struct list_head *rules_list;
u32 expected_completions = 1;
- u32 end_ft_id;
- int i, ret;
+ int i, ret = 0;
- /* We are rehashing the matcher that is the first part of the complex
- * matcher. Need to update the isolated matcher to point to the end_ft
- * of this new matcher. This needs to be done before moving any rules
- * to prevent possible steering loops.
- */
- isolated_tbl = bwc_matcher->complex->isolated_tbl;
- end_ft_id = bwc_matcher->matcher->resize_dst->end_ft_id;
- ret = mlx5hws_matcher_update_end_ft_isolated(isolated_tbl, end_ft_id);
- if (ret) {
- mlx5hws_err(ctx,
- "Failed updating end_ft of isolated matcher (%d)\n",
- ret);
- return ret;
- }
-
- hws_bwc_matcher_clear_hash_rtcs(bwc_matcher);
+ hws_bwc_matcher_init_move(bwc_matcher);
mlx5hws_bwc_rule_fill_attr(bwc_matcher, 0, 0, &rule_attr);
/* Check if a rule with similar tag has already
* been moved.
*/
- if (tmp_bwc_rule->complex_hash_node->rtc_valid) {
- /* This rule is a duplicate of rule with similar
- * tag that has already been moved earlier.
- * Just update this rule's RTCs.
+ if (tmp_bwc_rule->subrule_data->was_moved) {
+ /* This rule is a duplicate of rule with
+ * identical tag that has already been moved
+ * earlier. Just update this rule's RTCs.
*/
tmp_bwc_rule->rule->rtc_0 =
- tmp_bwc_rule->complex_hash_node->rtc_0;
+ tmp_bwc_rule->subrule_data->rtc_0;
tmp_bwc_rule->rule->rtc_1 =
- tmp_bwc_rule->complex_hash_node->rtc_1;
+ tmp_bwc_rule->subrule_data->rtc_1;
tmp_bwc_rule->rule->matcher =
tmp_bwc_rule->rule->matcher->resize_dst;
continue;
/* Done moving the rule to the new matcher,
* now update RTCs for all the duplicated rules.
*/
- tmp_bwc_rule->complex_hash_node->rtc_0 =
+ tmp_bwc_rule->subrule_data->rtc_0 =
tmp_bwc_rule->rule->rtc_0;
- tmp_bwc_rule->complex_hash_node->rtc_1 =
+ tmp_bwc_rule->subrule_data->rtc_1 =
tmp_bwc_rule->rule->rtc_1;
- tmp_bwc_rule->complex_hash_node->rtc_valid = true;
+ tmp_bwc_rule->subrule_data->was_moved = true;
}
}
return ret;
}
+
+int
+mlx5hws_bwc_matcher_complex_move_first(struct mlx5hws_bwc_matcher *bwc_matcher)
+{
+ struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx;
+ struct mlx5hws_bwc_matcher_complex_data *cdata;
+ struct mlx5hws_table *isolated_tbl;
+ u32 end_ft_id;
+ int i, ret;
+
+ cdata = bwc_matcher->complex;
+
+ /* We are rehashing the first submatcher. We need to update the
+ * subsequent submatchers to point to the end_ft of this new matcher.
+ * This needs to be done before moving any rules to prevent possible
+ * steering loops.
+ */
+ end_ft_id = bwc_matcher->matcher->resize_dst->end_ft_id;
+ for (i = 1; i < cdata->num_submatchers; i++) {
+ isolated_tbl = cdata->submatchers[i].tbl;
+ ret = mlx5hws_matcher_update_end_ft_isolated(isolated_tbl,
+ end_ft_id);
+ if (ret) {
+ mlx5hws_err(ctx,
+ "Complex matcher: failed updating end_ft of isolated matcher (%d)\n",
+ ret);
+ return ret;
+ }
+ }
+
+ return mlx5hws_bwc_matcher_complex_move(bwc_matcher);
+}
projects / users / hch / misc.git / commitdiff