selftests: mptcp: connect: format subtests results in TAP

The current selftests infrastructure formats the results in TAP 13. This
version doesn't support subtests and only the end result of each
selftest is taken into account. It means that a single issue in a
subtest of a selftest containing multiple subtests forces the whole
selftest to be marked as failed. It also means that subtests results are
not tracked by CIs executing selftests.

MPTCP selftests run hundreds of various subtests. It is then important
to track each of them and not one result per selftest.

It is particularly interesting to do that when validating stable kernels
with the last version of the test suite: tests might fail because a
feature is not supported but the test didn't skip that part. In this
case, if subtests are not tracked, the whole selftest will be marked as
failed making the other subtests useless because their results are
ignored.

This patch formats subtests results in TAP in mptcp_connect.sh selftest.

Link: https://github.com/multipath-tcp/mptcp_net-next/issues/368
Acked-by: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: Matthieu Baerts <matthieu.baerts@tessares.net>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/tools/testing/selftests/net/mptcp/mptcp_connect.sh b/tools/testing/selftests/net/mptcp/mptcp_connect.sh
index 0f060af01a847a3060d5e9f343a2d1cda5b71b47..b1fc8afd072dc6ddde8d561a675a5549a9a37dba 100755 (executable)
--- a/tools/testing/selftests/net/mptcp/mptcp_connect.sh
+++ b/tools/testing/selftests/net/mptcp/mptcp_connect.sh
@@ -129,6 +129,7 @@ ns3="ns3-$rndh"
 ns4="ns4-$rndh"
 
 TEST_COUNT=0
+TEST_GROUP=""
 
 cleanup()
 {
@@ -286,6 +287,7 @@ check_mptcp_disabled()
        # net.mptcp.enabled should be enabled by default
        if [ "$(ip netns exec ${disabled_ns} sysctl net.mptcp.enabled | awk '{ print $3 }')" -ne 1 ]; then
                echo -e "net.mptcp.enabled sysctl is not 1 by default\t\t[ FAIL ]"
+               mptcp_lib_result_fail "net.mptcp.enabled sysctl is not 1 by default"
                ret=1
                return 1
        fi
@@ -298,11 +300,13 @@ check_mptcp_disabled()
 
        if [ ${err} -eq 0 ]; then
                echo -e "New MPTCP socket cannot be blocked via sysctl\t\t[ FAIL ]"
+               mptcp_lib_result_fail "New MPTCP socket cannot be blocked via sysctl"
                ret=1
                return 1
        fi
 
        echo -e "New MPTCP socket can be blocked via sysctl\t\t[ OK ]"
+       mptcp_lib_result_pass "New MPTCP socket can be blocked via sysctl"
        return 0
 }
 
@@ -318,14 +322,16 @@ do_ping()
        local connector_ns="$2"
        local connect_addr="$3"
        local ping_args="-q -c 1"
+       local rc=0
 
        if is_v6 "${connect_addr}"; then
                $ipv6 || return 0
                ping_args="${ping_args} -6"
        fi
 
-       ip netns exec ${connector_ns} ping ${ping_args} $connect_addr >/dev/null
-       if [ $? -ne 0 ] ; then
+       ip netns exec ${connector_ns} ping ${ping_args} $connect_addr >/dev/null || rc=1
+
+       if [ $rc -ne 0 ] ; then
                echo "$listener_ns -> $connect_addr connectivity [ FAIL ]" 1>&2
                ret=1
 
@@ -404,7 +410,9 @@ do_transfer()
 
        local addr_port
        addr_port=$(printf "%s:%d" ${connect_addr} ${port})
-       printf "%.3s %-5s -> %.3s (%-20s) %-5s\t" ${connector_ns} ${cl_proto} ${listener_ns} ${addr_port} ${srv_proto}
+       local result_msg
+       result_msg="$(printf "%.3s %-5s -> %.3s (%-20s) %-5s" ${connector_ns} ${cl_proto} ${listener_ns} ${addr_port} ${srv_proto})"
+       printf "%s\t" "${result_msg}"
 
        if $capture; then
                local capuser
@@ -479,6 +487,7 @@ do_transfer()
 
        local duration
        duration=$((stop-start))
+       result_msg+=" # time=${duration}ms"
        printf "(duration %05sms) " "${duration}"
        if [ ${rets} -ne 0 ] || [ ${retc} -ne 0 ]; then
                echo "[ FAIL ] client exit code $retc, server $rets" 1>&2
@@ -491,6 +500,7 @@ do_transfer()
 
                echo
                cat "$capout"
+               mptcp_lib_result_fail "${TEST_GROUP}: ${result_msg}"
                return 1
        fi
 
@@ -550,6 +560,9 @@ do_transfer()
 
        if [ $retc -eq 0 ] && [ $rets -eq 0 ]; then
                printf "[ OK ]"
+               mptcp_lib_result_pass "${TEST_GROUP}: ${result_msg}"
+       else
+               mptcp_lib_result_fail "${TEST_GROUP}: ${result_msg}"
        fi
 
        if [ $cookies -eq 2 ];then
@@ -692,6 +705,8 @@ run_test_transparent()
        local lret=0
        local r6flag=""
 
+       TEST_GROUP="${msg}"
+
        # skip if we don't want v6
        if ! $ipv6 && is_v6 "${connect_addr}"; then
                return 0
@@ -703,6 +718,7 @@ run_test_transparent()
        # checking for a specific kernel version.
        if ! mptcp_lib_kallsyms_has "T __ip_sock_set_tos$"; then
                echo "INFO: ${msg} not supported by the kernel: SKIP"
+               mptcp_lib_result_skip "${TEST_GROUP}"
                return
        fi
 
@@ -720,6 +736,7 @@ EOF
        if [ $? -ne 0 ]; then
                echo "SKIP: $msg, could not load nft ruleset"
                mptcp_lib_fail_if_expected_feature "nft rules"
+               mptcp_lib_result_skip "${TEST_GROUP}"
                return
        fi
 
@@ -736,6 +753,7 @@ EOF
                ip netns exec "$listener_ns" nft flush ruleset
                echo "SKIP: $msg, ip $r6flag rule failed"
                mptcp_lib_fail_if_expected_feature "ip rule"
+               mptcp_lib_result_skip "${TEST_GROUP}"
                return
        fi
 
@@ -745,6 +763,7 @@ EOF
                ip -net "$listener_ns" $r6flag rule del fwmark 1 lookup 100
                echo "SKIP: $msg, ip route add local $local_addr failed"
                mptcp_lib_fail_if_expected_feature "ip route"
+               mptcp_lib_result_skip "${TEST_GROUP}"
                return
        fi
 
@@ -774,6 +793,7 @@ run_tests_peekmode()
 {
        local peekmode="$1"
 
+       TEST_GROUP="peek mode: ${peekmode}"
        echo "INFO: with peek mode: ${peekmode}"
        run_tests_lo "$ns1" "$ns1" 10.0.1.1 1 "-P ${peekmode}"
        run_tests_lo "$ns1" "$ns1" dead:beef:1::1 1 "-P ${peekmode}"
@@ -781,8 +801,11 @@ run_tests_peekmode()
 
 run_tests_mptfo()
 {
+       TEST_GROUP="MPTFO"
+
        if ! mptcp_lib_kallsyms_has "mptcp_fastopen_"; then
                echo "INFO: TFO not supported by the kernel: SKIP"
+               mptcp_lib_result_skip "${TEST_GROUP}"
                return
        fi
 
@@ -806,8 +829,11 @@ run_tests_disconnect()
        local old_cin=$cin
        local old_sin=$sin
 
+       TEST_GROUP="full disconnect"
+
        if ! mptcp_lib_kallsyms_has "mptcp_pm_data_reset$"; then
                echo "INFO: Full disconnect not supported: SKIP"
+               mptcp_lib_result_skip "${TEST_GROUP}"
                return
        fi
 
@@ -856,6 +882,7 @@ stop_if_error()
 {
        if ! log_if_error "${@}"; then
                display_time
+               mptcp_lib_result_print_all_tap
                exit ${final_ret}
        fi
 }
@@ -886,6 +913,8 @@ for sender in "$ns1" "$ns2" "$ns3" "$ns4";do
        do_ping "$ns4" $sender dead:beef:3::1
 done
 
+mptcp_lib_result_code "${ret}" "ping tests"
+
 stop_if_error "Could not even run ping tests"
 
 [ -n "$tc_loss" ] && tc -net "$ns2" qdisc add dev ns2eth3 root netem loss random $tc_loss delay ${tc_delay}ms
@@ -915,12 +944,15 @@ echo "on ns3eth4"
 
 tc -net "$ns3" qdisc add dev ns3eth4 root netem delay ${reorder_delay}ms $tc_reorder
 
+TEST_GROUP="loopback v4"
 run_tests_lo "$ns1" "$ns1" 10.0.1.1 1
 stop_if_error "Could not even run loopback test"
 
+TEST_GROUP="loopback v6"
 run_tests_lo "$ns1" "$ns1" dead:beef:1::1 1
 stop_if_error "Could not even run loopback v6 test"
 
+TEST_GROUP="multihosts"
 for sender in $ns1 $ns2 $ns3 $ns4;do
        # ns1<->ns2 is not subject to reordering/tc delays. Use it to test
        # mptcp syncookie support.
@@ -966,4 +998,5 @@ run_tests_disconnect
 log_if_error "Tests of the full disconnection have failed"
 
 display_time
+mptcp_lib_result_print_all_tap
 exit ${final_ret}