rdma: Use rdma link instead of /sys/class/infiniband/*/parent

The approach of verifying whether or not an RDMA interface is associated
with the rdma_rxe interface by looking up its parent device is deprecated
and will be removed soon from the Linux kernel. Hence this patch that uses
the rdma link command instead.

Cc: Jason Gunthorpe <jgg@nvidia.com>
Cc: Yi Zhang <yi.zhang@redhat.com>
Signed-off-by: Bart Van Assche <bvanassche@acm.org>

diff --git a/common/multipath-over-rdma b/common/multipath-over-rdma
index 9d9d2b27af8359fde7caeb190b9a213748170e91..9e68189b56fd1cfae02b786f0b9eb0995aa75bf4 100644 (file)
--- a/common/multipath-over-rdma
+++ b/common/multipath-over-rdma
@@ -57,6 +57,9 @@ _multipathd_version_ge() {
 }
 
 get_ipv4_addr() {
+       if [ ! -e "/sys/class/net/$1" ]; then
+               echo "get_ipv4_addr(): $1 is not a network interface" 1>&2
+       fi
        ip -4 -o addr show dev "$1" |
                sed -n 's/.*[[:blank:]]inet[[:blank:]]*\([^[:blank:]/]*\).*/\1/p'
 }
@@ -76,41 +79,11 @@ is_number() {
        [ "$1" -eq "0$1" ] 2>/dev/null
 }
 
-# Check whether a device is an RDMA device. An example argument:
-# /sys/devices/pci0000:00/0000:00:03.0/0000:04:00.0
-is_rdma_device() {
-       local d i inode1 inode2
-
-       inode1=$(stat -c %i "$1")
-       # echo "inode1 = $inode1"
-       for i in /sys/class/infiniband/*; do
-               d=/sys/class/infiniband/"$(readlink "$i")"
-               d=$(dirname "$(dirname "$d")")
-               inode2=$(stat -c %i "$d")
-               # echo "inode2 = $inode2"
-               if [ "$inode1" = "$inode2" ]; then
-                       return
-               fi
-       done
-       false
-}
-
 # Lists RDMA capable network interface names, e.g. ib0 ib1.
 rdma_network_interfaces() {
-       (
-               cd /sys/class/net &&
-                       for i in *; do
-                               [ -e "$i" ] || continue
-                               # Skip IPoIB (ARPHRD_INFINIBAND) network
-                               # interfaces.
-                               [ "$(<"$i"/type)" = 32 ] && continue
-                               [ -L "$i/device" ] || continue
-                               d=$(readlink "$i/device" 2>/dev/null)
-                               if [ -n "$d" ] && is_rdma_device "$i/$d"; then
-                                       echo "$i"
-                               fi
-                       done
-       )
+       rdma link show |
+               sed -n 's/^.*[[:blank:]]netdev[[:blank:]]\+\([^[:blank:]]*\)[[:blank:]]*/\1/p' |
+               sort -u
 }
 
 # Check whether any stacked block device holds block device $1. If so, echo
@@ -411,47 +384,36 @@ all_primary_gids() {
        done
 }
 
-# Check whether or not an rdma_rxe instance has been associated with network
-# interface $1.
-has_rdma_rxe() {
-       local f
-
-       for f in /sys/class/infiniband/*/parent; do
-               if [ -e "$f" ] && [ "$(<"$f")" = "$1" ]; then
-                       return 0
-               fi
-       done
-
-       return 1
+# Check whether or not an rdma_rxe or siw instance has been associated with
+# network interface $1.
+has_soft_rdma() {
+       rdma link | grep -q " netdev $1[[:blank:]]*\$"
 }
 
 # Load the rdma_rxe or siw kernel module and associate it with all network
 # interfaces.
 start_soft_rdma() {
+       local type
+
        {
        if [ -n "$use_siw" ]; then
                modprobe siw || return $?
-               (
-                       cd /sys/class/net &&
-                               for i in *; do
-                                       [ -e "$i" ] || continue
-                                       [ -e "/sys/class/infiniband/${i}_siw" ] && continue
-                                       rdma link add "${i}_siw" type siw netdev "$i" ||
-                                               echo "Failed to bind the siw driver to $i"
-                               done
-               )
+               type=siw
        else
                modprobe rdma_rxe || return $?
-               (
-                       cd /sys/class/net &&
-                               for i in *; do
-                                       if [ -e "$i" ] && ! has_rdma_rxe "$i"; then
-                                               echo "$i" > /sys/module/rdma_rxe/parameters/add ||
-                                                       echo "Failed to bind the rdma_rxe driver to $i"
-                                       fi
-                               done
-               )
+               type=rxe
        fi
+       (
+               cd /sys/class/net &&
+                       for i in *; do
+                               [ -e "$i" ] || continue
+                               [ "$i" = "lo" ] && continue
+                               [ "$(<"$i/addr_len")" = 6 ] || continue
+                               has_soft_rdma "$i" && continue
+                               rdma link add "${i}_$type" type $type netdev "$i" ||
+                               echo "Failed to bind the $type driver to $i"
+                       done
+       )
        } >>"$FULL"
 }
 
@@ -459,27 +421,16 @@ start_soft_rdma() {
 # unload the rdma_rxe kernel module.
 stop_soft_rdma() {
        {
-       (
-               cd /sys/class/net &&
-                       for i in *; do
-                               if [ -e "$i" ] && has_rdma_rxe "$i"; then
-                                       { echo "$i" > /sys/module/rdma_rxe/parameters/remove; } \
-                                               2>/dev/null
-                               fi
-                       done
-       )
+       rdma link |
+               sed -n 's,^link[[:blank:]]*\([^/]*\)/.* netdev .*,\1,p' |
+               while read -r i; do
+                     echo "$i ..."
+                     rdma link del "${i}" || echo "Failed to remove ${i}"
+               done
        if ! unload_module rdma_rxe 10; then
                echo "Unloading rdma_rxe failed"
                return 1
        fi
-       (
-               cd /sys/class/net &&
-                       for i in *_siw; do
-                               [ -e "$i" ] || continue
-                               rdma link del "${i}" ||
-                                       echo "Failed to unbind the siw driver from ${i%_siw}"
-                       done
-       )
        if ! unload_module siw 10; then
                echo "Unloading siw failed"
                return 1

diff --git a/tests/nvmeof-mp/rc b/tests/nvmeof-mp/rc
index ab7770f6aac0b77fe2d2a0788e81f9cf5f434dc2..0a12825e2c11fe0fb979ddc84da71e8d94dd4c1f 100755 (executable)
--- a/tests/nvmeof-mp/rc
+++ b/tests/nvmeof-mp/rc
@@ -42,7 +42,7 @@ and multipathing has been enabled in the nvme_core kernel module"
        )
        _have_modules "${required_modules[@]}" || return
 
-       for p in mkfs.ext4 mkfs.xfs multipath multipathd pidof; do
+       for p in mkfs.ext4 mkfs.xfs multipath multipathd pidof rdma; do
                _have_program "$p" || return
        done
 

diff --git a/tests/srp/rc b/tests/srp/rc
index 700cd71ea1550f4cb2e89ec882b9a7631b90d318..2986bfd5439dcc6f1e88ed940000b0632638a18b 100755 (executable)
--- a/tests/srp/rc
+++ b/tests/srp/rc
@@ -59,7 +59,8 @@ group_requires() {
        )
        _have_modules "${required_modules[@]}" || return
 
-       for p in mkfs.ext4 mkfs.xfs multipath multipathd pidof sg_reset; do
+       for p in mkfs.ext4 mkfs.xfs multipath multipathd pidof rdma \
+                sg_reset; do
                _have_program "$p" || return
        done
 
@@ -142,14 +143,7 @@ do_ib_cm_login() {
 }
 
 rdma_dev_to_net_dev() {
-       local b d rdma_dev=$1
-
-       b=/sys/class/infiniband/$rdma_dev/parent
-       if [ -e "$b" ]; then
-               echo "$(<"$b")"
-       else
-               echo "${rdma_dev%_siw}"
-       fi
+       rdma link show "$1/1" | sed 's/.* netdev //;s/[[:blank:]]*$//'
 }
 
 # Tell the SRP initiator to log in to an SRP target using the RDMA/CM.