From: Alexander Burmashev Date: Thu, 7 Feb 2019 15:40:30 +0000 (-0800) Subject: find-debuginfo.sh: backport parallel files procession X-Git-Tag: v4.1.12-124.31.3~278 X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=ff4a1850bbf59ac181c714d9f13b702be13240a1;p=users%2Fjedix%2Flinux-maple.git find-debuginfo.sh: backport parallel files procession Use bundled find-debuginfo.sh instead of copying one during the build rpm upstream commit 038bfe01796f751001e02de41c5d8678f511f366 find-debuginfo.sh: Split directory traversal and debuginfo extraction This siplifies the handling of hardlinks a bit and allows a later patch to parallelize the debuginfo extraction. Signed-off-by: Michal Marek rpm upstream commit 1b338aa84d4c67fefa957352a028eaca1a45d1f6 find-debuginfo.sh: Process files in parallel Add a -j option, which, when used, will spawn processes to do the debuginfo extraction in parallel. A pipe is used to dispatch the files among the processes. Signed-off-by: Michal Marek Orabug: 29323635 Signed-off-by: Alex Burmashev Reviewed-by: Somasundaram Krishnasamy Reviewed-by: Todd Vierling Signed-off-by: Brian Maly --- diff --git a/uek-rpm/ol6/find-debuginfo.sh b/uek-rpm/ol6/find-debuginfo.sh new file mode 100755 index 0000000000000..37c7e218ef670 --- /dev/null +++ b/uek-rpm/ol6/find-debuginfo.sh @@ -0,0 +1,364 @@ +#!/bin/bash +#find-debuginfo.sh - automagically generate debug info and file list +#for inclusion in an rpm spec file. +# +# Usage: find-debuginfo.sh [--strict-build-id] [-g] +# [-o debugfiles.list] +# [[-l filelist]... [-p 'pattern'] -o debuginfo.list] +# [builddir] +# +# The -g flag says to use strip -g instead of full strip on DSOs. +# The --strict-build-id flag says to exit with failure status if +# any ELF binary processed fails to contain a build-id note. +# +# A single -o switch before any -l or -p switches simply renames +# the primary output file from debugfiles.list to something else. +# A -o switch that follows a -p switch or some -l switches produces +# an additional output file with the debuginfo for the files in +# the -l filelist file, or whose names match the -p pattern. +# The -p argument is an egrep-style regexp matching the a file name, +# and must not use anchors (^ or $). +# +# All file names in switches are relative to builddir (. if not given). +# + +# With -g arg, pass it to strip on libraries. +strip_g=false + +# Barf on missing build IDs. +strict=false + +BUILDDIR=. +out=debugfiles.list +nout=0 +while [ $# -gt 0 ]; do + case "$1" in + --strict-build-id) + strict=true + ;; + -g) + strip_g=true + ;; + -o) + if [ -z "${lists[$nout]}" -a -z "${ptns[$nout]}" ]; then + out=$2 + else + outs[$nout]=$2 + ((nout++)) + fi + shift + ;; + -l) + lists[$nout]="${lists[$nout]} $2" + shift + ;; + -p) + ptns[$nout]=$2 + shift + ;; + *) + BUILDDIR=$1 + shift + break + ;; + esac + shift +done + +i=0 +while ((i < nout)); do + outs[$i]="$BUILDDIR/${outs[$i]}" + l='' + for f in ${lists[$i]}; do + l="$l $BUILDDIR/$f" + done + lists[$i]=$l + ((++i)) +done + +LISTFILE="$BUILDDIR/$out" +SOURCEFILE="$BUILDDIR/debugsources.list" +LINKSFILE="$BUILDDIR/debuglinks.list" + +> "$SOURCEFILE" +> "$LISTFILE" +> "$LINKSFILE" + +debugdir="${RPM_BUILD_ROOT}/usr/lib/debug" + +strip_to_debug() +{ + local g= + $strip_g && case "$(file -bi "$2")" in + application/x-sharedlib*) g=-g ;; + esac + eu-strip --remove-comment $g -f "$1" "$2" || exit + chmod 444 "$1" || exit +} + +# Make a relative symlink to $1 called $3$2 +shopt -s extglob +link_relative() +{ + local t="$1" f="$2" pfx="$3" + local fn="${f#/}" tn="${t#/}" + local fd td d + + while fd="${fn%%/*}"; td="${tn%%/*}"; [ "$fd" = "$td" ]; do + fn="${fn#*/}" + tn="${tn#*/}" + done + + d="${fn%/*}" + if [ "$d" != "$fn" ]; then + d="${d//+([!\/])/..}" + tn="${d}/${tn}" + fi + + mkdir -p "$(dirname "$pfx$f")" && ln -snf "$tn" "$pfx$f" +} + +# Make a symlink in /usr/lib/debug/$2 to $1 +debug_link() +{ + local l="/usr/lib/debug$2" + local t="$1" + echo >> "$LINKSFILE" "$l $t" + link_relative "$t" "$l" "$RPM_BUILD_ROOT" +} + +# Provide .2, .3, ... symlinks to all filename instances of this build-id. +make_id_dup_link() +{ + local id="$1" file="$2" idfile + + local n=1 + while true; do + idfile=".build-id/${id:0:2}/${id:2}.$n" + [ $# -eq 3 ] && idfile="${idfile}$3" + if [ ! -L "$RPM_BUILD_ROOT/usr/lib/debug/$idfile" ]; then + break + fi + n=$[$n+1] + done + debug_link "$file" "/$idfile" +} + +# Make a build-id symlink for id $1 with suffix $3 to file $2. +make_id_link() +{ + local id="$1" file="$2" + local idfile=".build-id/${id:0:2}/${id:2}" + [ $# -eq 3 ] && idfile="${idfile}$3" + local root_idfile="$RPM_BUILD_ROOT/usr/lib/debug/$idfile" + + if [ ! -L "$root_idfile" ]; then + debug_link "$file" "/$idfile" + return + fi + + make_id_dup_link "$@" + + [ $# -eq 3 ] && return 0 + + local other=$(readlink -m "$root_idfile") + other=${other#$RPM_BUILD_ROOT} + if cmp -s "$root_idfile" "$RPM_BUILD_ROOT$file" || + eu-elfcmp -q "$root_idfile" "$RPM_BUILD_ROOT$file" 2> /dev/null; then + # Two copies. Maybe one has to be setuid or something. + echo >&2 "*** WARNING: identical binaries are copied, not linked:" + echo >&2 " $file" + echo >&2 " and $other" + else + # This is pathological, break the build. + echo >&2 "*** ERROR: same build ID in nonidentical files!" + echo >&2 " $file" + echo >&2 " and $other" + exit 2 + fi +} + +get_debugfn() +{ + dn=$(dirname "${1#$RPM_BUILD_ROOT}") + bn=$(basename "$1" .debug).debug + + debugdn=${debugdir}${dn} + debugfn=${debugdn}/${bn} +} + +set -o pipefail + +strict_error=ERROR +$strict || strict_error=WARNING + +# Strip ELF binaries +find "$RPM_BUILD_ROOT" ! -path "${debugdir}/*.debug" -type f \ + \( -perm -0100 -or -perm -0010 -or -perm -0001 \) \ + -print | +file -N -f - | sed -n -e 's/^\(.*\):[ ]*.*ELF.*, not stripped/\1/p' | +xargs --no-run-if-empty stat -c '%h %D_%i %n' | +while read nlinks inum f; do + get_debugfn "$f" + [ -f "${debugfn}" ] && continue + + # If this file has multiple links, keep track and make + # the corresponding .debug files all links to one file too. + if [ $nlinks -gt 1 ]; then + eval linked=\$linked_$inum + if [ -n "$linked" ]; then + eval id=\$linkedid_$inum + make_id_dup_link "$id" "$dn/$(basename $f)" + make_id_dup_link "$id" "/usr/lib/debug$dn/$bn" .debug + link=$debugfn + get_debugfn "$linked" + echo "hard linked $link to $debugfn" + mkdir -p "$(dirname "$link")" && ln -nf "$debugfn" "$link" + continue + else + eval linked_$inum=\$f + echo "file $f has $[$nlinks - 1] other hard links" + fi + fi + + echo "extracting debug info from $f" + id=$(/usr/lib/rpm/debugedit -b "$RPM_BUILD_DIR" -d /usr/src/debug \ + -i -l "$SOURCEFILE" "$f") || exit + if [ $nlinks -gt 1 ]; then + eval linkedid_$inum=\$id + fi + if [ -z "$id" ]; then + echo >&2 "*** ${strict_error}: No build ID note found in $f" + $strict && exit 2 + fi + + [ -x /usr/bin/gdb-add-index ] && /usr/bin/gdb-add-index "$f" > /dev/null 2>&1 + + # A binary already copied into /usr/lib/debug doesn't get stripped, + # just has its file names collected and adjusted. + case "$dn" in + /usr/lib/debug/*) + [ -z "$id" ] || make_id_link "$id" "$dn/$(basename $f)" + continue ;; + esac + + mkdir -p "${debugdn}" + if test -w "$f"; then + strip_to_debug "${debugfn}" "$f" + else + chmod u+w "$f" + strip_to_debug "${debugfn}" "$f" + chmod u-w "$f" + fi + + if [ -n "$id" ]; then + make_id_link "$id" "$dn/$(basename $f)" + make_id_link "$id" "/usr/lib/debug$dn/$bn" .debug + fi +done || exit + +# For each symlink whose target has a .debug file, +# make a .debug symlink to that file. +find $RPM_BUILD_ROOT ! -path "${debugdir}/*" -type l -print | +while read f +do + t=$(readlink -m "$f").debug + f=${f#$RPM_BUILD_ROOT} + t=${t#$RPM_BUILD_ROOT} + if [ -f "$debugdir$t" ]; then + echo "symlinked /usr/lib/debug$t to /usr/lib/debug${f}.debug" + debug_link "/usr/lib/debug$t" "${f}.debug" + fi +done + +if [ -s "$SOURCEFILE" ]; then + mkdir -p "${RPM_BUILD_ROOT}/usr/src/debug" + LC_ALL=C sort -z -u "$SOURCEFILE" | egrep -v -z '(|)$' | + (cd "$RPM_BUILD_DIR"; cpio -pd0mL "${RPM_BUILD_ROOT}/usr/src/debug") + # stupid cpio creates new directories in mode 0700, fixup + find "${RPM_BUILD_ROOT}/usr/src/debug" -type d -print0 | + xargs --no-run-if-empty -0 chmod a+rx +fi + +if [ -d "${RPM_BUILD_ROOT}/usr/lib" -o -d "${RPM_BUILD_ROOT}/usr/src" ]; then + ((nout > 0)) || + test ! -d "${RPM_BUILD_ROOT}/usr/lib" || + (cd "${RPM_BUILD_ROOT}/usr/lib"; find debug -type d) | + sed 's,^,%dir /usr/lib/,' >> "$LISTFILE" + + (cd "${RPM_BUILD_ROOT}/usr" + test ! -d lib/debug || find lib/debug ! -type d + test ! -d src/debug || find src/debug -mindepth 1 -maxdepth 1 + ) | sed 's,^,/usr/,' >> "$LISTFILE" +fi + +# Append to $1 only the lines from stdin not already in the file. +append_uniq() +{ + fgrep -f "$1" -x -v >> "$1" +} + +# Helper to generate list of corresponding .debug files from a file list. +filelist_debugfiles() +{ + local extra="$1" + shift + sed 's/^%[a-z0-9_][a-z0-9_]*([^)]*) *// +s/^%[a-z0-9_][a-z0-9_]* *// +/^$/d +'"$extra" "$@" +} + +# Write an output debuginfo file list based on given input file lists. +filtered_list() +{ + local out="$1" + shift + test $# -gt 0 || return + fgrep -f <(filelist_debugfiles 's,^.*$,/usr/lib/debug&.debug,' "$@") \ + -x $LISTFILE >> $out + sed -n -f <(filelist_debugfiles 's/[\\.*+#]/\\&/g +h +s,^.*$,s# &$##p,p +g +s,^.*$,s# /usr/lib/debug&.debug$##p,p +' "$@") "$LINKSFILE" | append_uniq "$out" +} + +# Write an output debuginfo file list based on an egrep-style regexp. +pattern_list() +{ + local out="$1" ptn="$2" + test -n "$ptn" || return + egrep -x -e "$ptn" "$LISTFILE" >> "$out" + sed -n -r "\#^$ptn #s/ .*\$//p" "$LINKSFILE" | append_uniq "$out" +} + +# +# When given multiple -o switches, split up the output as directed. +# +i=0 +while ((i < nout)); do + > ${outs[$i]} + filtered_list ${outs[$i]} ${lists[$i]} + pattern_list ${outs[$i]} "${ptns[$i]}" + fgrep -vx -f ${outs[$i]} "$LISTFILE" > "${LISTFILE}.new" + mv "${LISTFILE}.new" "$LISTFILE" + ((++i)) +done +if ((nout > 0)); then + # Now add the right %dir lines to each output list. + (cd "${RPM_BUILD_ROOT}"; find usr/lib/debug -type d) | + sed 's#^.*$#\\@^/&/@{h;s@^.*$@%dir /&@p;g;}#' | + LC_ALL=C sort -ur > "${LISTFILE}.dirs.sed" + i=0 + while ((i < nout)); do + sed -n -f "${LISTFILE}.dirs.sed" "${outs[$i]}" | sort -u > "${outs[$i]}.new" + cat "${outs[$i]}" >> "${outs[$i]}.new" + mv -f "${outs[$i]}.new" "${outs[$i]}" + ((++i)) + done + sed -n -f "${LISTFILE}.dirs.sed" "${LISTFILE}" | sort -u > "${LISTFILE}.new" + cat "$LISTFILE" >> "${LISTFILE}.new" + mv "${LISTFILE}.new" "$LISTFILE" +fi diff --git a/uek-rpm/ol6/find-debuginfo.sh.parallel.diff b/uek-rpm/ol6/find-debuginfo.sh.parallel.diff new file mode 100644 index 0000000000000..91910867caa66 --- /dev/null +++ b/uek-rpm/ol6/find-debuginfo.sh.parallel.diff @@ -0,0 +1,160 @@ +--- find-debuginfo.sh.orig 2015-07-23 00:21:42.000000000 -0700 ++++ find-debuginfo.sh 2019-02-07 08:23:27.520239815 -0800 +@@ -28,6 +28,9 @@ + # Barf on missing build IDs. + strict=false + ++# Number of parallel jobs to spawn ++n_jobs=1 ++ + BUILDDIR=. + out=debugfiles.list + nout=0 +@@ -56,6 +59,13 @@ + ptns[$nout]=$2 + shift + ;; ++ -j) ++ n_jobs=$2 ++ shift ++ ;; ++ -j*) ++ n_jobs=${1#-j} ++ ;; + *) + BUILDDIR=$1 + shift +@@ -192,41 +202,41 @@ + strict_error=ERROR + $strict || strict_error=WARNING + +-# Strip ELF binaries ++temp=$(mktemp -d ${TMPDIR:-/tmp}/find-debuginfo.XXXXXX) ++trap 'rm -rf "$temp"' EXIT ++ ++# Build a list of unstripped ELF files and their hardlinks ++touch "$temp/primary" + find "$RPM_BUILD_ROOT" ! -path "${debugdir}/*.debug" -type f \ + \( -perm -0100 -or -perm -0010 -or -perm -0001 \) \ + -print | + file -N -f - | sed -n -e 's/^\(.*\):[ ]*.*ELF.*, not stripped/\1/p' | + xargs --no-run-if-empty stat -c '%h %D_%i %n' | + while read nlinks inum f; do +- get_debugfn "$f" +- [ -f "${debugfn}" ] && continue +- +- # If this file has multiple links, keep track and make +- # the corresponding .debug files all links to one file too. + if [ $nlinks -gt 1 ]; then +- eval linked=\$linked_$inum +- if [ -n "$linked" ]; then +- eval id=\$linkedid_$inum +- make_id_dup_link "$id" "$dn/$(basename $f)" +- make_id_dup_link "$id" "/usr/lib/debug$dn/$bn" .debug +- link=$debugfn +- get_debugfn "$linked" +- echo "hard linked $link to $debugfn" +- mkdir -p "$(dirname "$link")" && ln -nf "$debugfn" "$link" ++ var=seen_$inum ++ if test -n "${!var}"; then ++ echo "$inum $f" >>"$temp/linked" + continue + else +- eval linked_$inum=\$f +- echo "file $f has $[$nlinks - 1] other hard links" ++ read "$var" < <(echo 1) + fi + fi + ++ echo "$nlinks $inum $f" >>"$temp/primary" ++done ++ ++# Strip ELF binaries ++do_file() ++{ ++ local nlinks=$1 inum=$2 f=$3 id link linked ++ ++ get_debugfn "$f" ++ [ -f "${debugfn}" ] && return ++ + echo "extracting debug info from $f" + id=$(/usr/lib/rpm/debugedit -b "$RPM_BUILD_DIR" -d /usr/src/debug \ + -i -l "$SOURCEFILE" "$f") || exit +- if [ $nlinks -gt 1 ]; then +- eval linkedid_$inum=\$id +- fi + if [ -z "$id" ]; then + echo >&2 "*** ${strict_error}: No build ID note found in $f" + $strict && exit 2 +@@ -255,7 +265,70 @@ + make_id_link "$id" "$dn/$(basename $f)" + make_id_link "$id" "/usr/lib/debug$dn/$bn" .debug + fi +-done || exit ++ ++ # If this file has multiple links, make the corresponding .debug files ++ # all links to one file too. ++ if [ $nlinks -gt 1 ]; then ++ grep "^$inum " "$temp/linked" | while read inum linked; do ++ link=$debugfn ++ get_debugfn "$linked" ++ echo "hard linked $link to $debugfn" ++ mkdir -p "$(dirname "$debugfn")" && ln -nf "$link" "$debugfn" ++ done ++ fi ++} ++ ++# 16^6 - 1 or about 16 milion files ++FILENUM_DIGITS=6 ++run_job() ++{ ++ local jobid=$1 filenum ++ local SOURCEFILE=$temp/debugsources.$jobid ELFBINSFILE=$temp/elfbins.$jobid ++ ++ >"$SOURCEFILE" ++ >"$ELFBINSFILE" ++ # can't use read -n , because it reads bytes one by one, allowing for ++ # races ++ while :; do ++ filenum=$(dd bs=$(( FILENUM_DIGITS + 1 )) count=1 status=none) ++ if test -z "$filenum"; then ++ break ++ fi ++ do_file $(sed -n "$(( 0x$filenum )) p" "$temp/primary") ++ done ++ echo 0 >"$temp/res.$jobid" ++} ++ ++n_files=$(wc -l <"$temp/primary") ++if [ $n_jobs -gt $n_files ]; then ++ n_jobs=$n_files ++fi ++if [ $n_jobs -le 1 ]; then ++ while read nlinks inum f; do ++ do_file "$nlinks" "$inum" "$f" ++ done <"$temp/primary" ++else ++ for ((i = 1; i <= n_files; i++)); do ++ printf "%0${FILENUM_DIGITS}x\\n" $i ++ done | ( ++ exec 3<&0 ++ for ((i = 0; i < n_jobs; i++)); do ++ # The shell redirects stdin to /dev/null for background jobs. Work ++ # around this by duplicating fd 0 ++ run_job $i <&3 & ++ done ++ wait ++ ) ++ for f in "$temp"/res.*; do ++ res=$(< "$f") ++ if [ "$res" != "0" ]; then ++ exit 1 ++ fi ++ done ++ cat "$temp"/debugsources.* >"$SOURCEFILE" ++ cat "$temp"/elfbins.* >"$ELFBINSFILE" ++fi ++ + + # For each symlink whose target has a .debug file, + # make a .debug symlink to that file. diff --git a/uek-rpm/ol6/kernel-uek.spec b/uek-rpm/ol6/kernel-uek.spec index 6e2d3e8d5b0f3..d7ac8be50557c 100644 --- a/uek-rpm/ol6/kernel-uek.spec +++ b/uek-rpm/ol6/kernel-uek.spec @@ -561,6 +561,7 @@ Source20: x86_energy_perf_policy Source21: turbostat Source22: securebootca.cer Source23: secureboot.cer +Source24: find-debuginfo.sh Source1000: config-x86_64 Source1001: config-x86_64-debug @@ -577,6 +578,7 @@ Source202: ksplice_signing_key.x509 Source300: debuginfo-g1.diff Source301: find-debuginfo.sh.ol6.diff +Source302: find-debuginfo.sh.parallel.diff # Here should be only the patches up to the upstream canonical Linus tree. @@ -987,12 +989,13 @@ ApplyPatch %{stable_patch_00} ApplyPatch %{stable_patch_01} %endif -# Copy the RPM find-debuginfo.sh into the buildroot and patch it +# Copy bundled find-debuginfo.sh into the buildroot and patch it # to support -g1. (This is a patch of *RPM*, not of the kernel, # so it is not governed by nopatches.) -cp %{_rpmconfigdir}/find-debuginfo.sh %{_builddir} +cp %_sourcedir/find-debuginfo.sh %{_builddir} patch %{_builddir}/find-debuginfo.sh %{SOURCE300} && \ -patch %{_builddir}/find-debuginfo.sh %{SOURCE301} +patch %{_builddir}/find-debuginfo.sh %{SOURCE301} && \ +patch %{_builddir}/find-debuginfo.sh %{SOURCE302} chmod +x %{_builddir}/find-debuginfo.sh # only deal with configs if we are going to build for the arch @@ -1459,7 +1462,7 @@ find Documentation -type d | xargs chmod u+w %if %{with_debuginfo} %define __debug_install_post \ - %{_builddir}/find-debuginfo.sh %{debuginfo_args} -g1 %{_builddir}/%{?buildsubdir}\ + %{_builddir}/find-debuginfo.sh %{?_smp_mflags} %{debuginfo_args} -g1 %{_builddir}/%{?buildsubdir}\ %{nil} %ifnarch noarch diff --git a/uek-rpm/ol7/find-debuginfo.sh b/uek-rpm/ol7/find-debuginfo.sh new file mode 100755 index 0000000000000..65af3249d6c5b --- /dev/null +++ b/uek-rpm/ol7/find-debuginfo.sh @@ -0,0 +1,471 @@ +#!/bin/bash +#find-debuginfo.sh - automagically generate debug info and file list +#for inclusion in an rpm spec file. +# +# Usage: find-debuginfo.sh [--strict-build-id] [-g] [-r] [-m] +# [-o debugfiles.list] +# [--run-dwz] [--dwz-low-mem-die-limit N] +# [--dwz-max-die-limit N] +# [[-l filelist]... [-p 'pattern'] -o debuginfo.list] +# [builddir] +# +# The -g flag says to use strip -g instead of full strip on DSOs. +# The --strict-build-id flag says to exit with failure status if +# any ELF binary processed fails to contain a build-id note. +# The -r flag says to use eu-strip --reloc-debug-sections. +# +# A single -o switch before any -l or -p switches simply renames +# the primary output file from debugfiles.list to something else. +# A -o switch that follows a -p switch or some -l switches produces +# an additional output file with the debuginfo for the files in +# the -l filelist file, or whose names match the -p pattern. +# The -p argument is an grep -E -style regexp matching the a file name, +# and must not use anchors (^ or $). +# +# The --run-dwz flag instructs find-debuginfo.sh to run the dwz utility +# if available, and --dwz-low-mem-die-limit and --dwz-max-die-limit +# provide detailed limits. See dwz(1) -l and -L option for details. +# +# All file names in switches are relative to builddir (. if not given). +# + +# With -g arg, pass it to strip on libraries. +strip_g=false + +# with -r arg, pass --reloc-debug-sections to eu-strip. +strip_r=false + +# with -m arg, add minimal debuginfo to binary. +include_minidebug=false + +# Barf on missing build IDs. +strict=false + +# DWZ parameters. +run_dwz=false +dwz_low_mem_die_limit= +dwz_max_die_limit= + +BUILDDIR=. +out=debugfiles.list +nout=0 +while [ $# -gt 0 ]; do + case "$1" in + --strict-build-id) + strict=true + ;; + --run-dwz) + run_dwz=true + ;; + --dwz-low-mem-die-limit) + dwz_low_mem_die_limit=$2 + shift + ;; + --dwz-max-die-limit) + dwz_max_die_limit=$2 + shift + ;; + -g) + strip_g=true + ;; + -m) + include_minidebug=true + ;; + -o) + if [ -z "${lists[$nout]}" -a -z "${ptns[$nout]}" ]; then + out=$2 + else + outs[$nout]=$2 + ((nout++)) + fi + shift + ;; + -l) + lists[$nout]="${lists[$nout]} $2" + shift + ;; + -p) + ptns[$nout]=$2 + shift + ;; + -r) + strip_r=true + ;; + *) + BUILDDIR=$1 + shift + break + ;; + esac + shift +done + +i=0 +while ((i < nout)); do + outs[$i]="$BUILDDIR/${outs[$i]}" + l='' + for f in ${lists[$i]}; do + l="$l $BUILDDIR/$f" + done + lists[$i]=$l + ((++i)) +done + +LISTFILE="$BUILDDIR/$out" +SOURCEFILE="$BUILDDIR/debugsources.list" +LINKSFILE="$BUILDDIR/debuglinks.list" +ELFBINSFILE="$BUILDDIR/elfbins.list" + +> "$SOURCEFILE" +> "$LISTFILE" +> "$LINKSFILE" +> "$ELFBINSFILE" + +debugdir="${RPM_BUILD_ROOT}/usr/lib/debug" + +strip_to_debug() +{ + local g= + local r= + $strip_r && r=--reloc-debug-sections + $strip_g && case "$(file -bi "$2")" in + application/x-sharedlib*) g=-g ;; + esac + eu-strip --remove-comment $r $g -f "$1" "$2" || exit + chmod 444 "$1" || exit +} + +add_minidebug() +{ + local debuginfo="$1" + local binary="$2" + + local dynsyms=`mktemp` + local funcsyms=`mktemp` + local keep_symbols=`mktemp` + local mini_debuginfo=`mktemp` + + # Extract the dynamic symbols from the main binary, there is no need to also have these + # in the normal symbol table + nm -D "$binary" --format=posix --defined-only | awk '{ print $1 }' | sort > "$dynsyms" + # Extract all the text (i.e. function) symbols from the debuginfo + # Use format sysv to make sure we can match against the actual ELF FUNC + # symbol type. The binutils nm posix format symbol type chars are + # ambigous for architectures that might use function descriptors. + nm "$debuginfo" --format=sysv --defined-only | awk -F \| '{ if ($4 ~ "FUNC") print $1 }' | sort > "$funcsyms" + # Keep all the function symbols not already in the dynamic symbol table + comm -13 "$dynsyms" "$funcsyms" > "$keep_symbols" + # Copy the full debuginfo, keeping only a minumal set of symbols and removing some unnecessary sections + objcopy -S --remove-section .gdb_index --remove-section .comment --keep-symbols="$keep_symbols" "$debuginfo" "$mini_debuginfo" &> /dev/null + #Inject the compressed data into the .gnu_debugdata section of the original binary + xz "$mini_debuginfo" + mini_debuginfo="${mini_debuginfo}.xz" + objcopy --add-section .gnu_debugdata="$mini_debuginfo" "$binary" + rm -f "$dynsyms" "$funcsyms" "$keep_symbols" "$mini_debuginfo" +} + +# Make a relative symlink to $1 called $3$2 +shopt -s extglob +link_relative() +{ + local t="$1" f="$2" pfx="$3" + local fn="${f#/}" tn="${t#/}" + local fd td d + + while fd="${fn%%/*}"; td="${tn%%/*}"; [ "$fd" = "$td" ]; do + fn="${fn#*/}" + tn="${tn#*/}" + done + + d="${fn%/*}" + if [ "$d" != "$fn" ]; then + d="${d//+([!\/])/..}" + tn="${d}/${tn}" + fi + + mkdir -p "$(dirname "$pfx$f")" && ln -snf "$tn" "$pfx$f" +} + +# Make a symlink in /usr/lib/debug/$2 to $1 +debug_link() +{ + local l="/usr/lib/debug$2" + local t="$1" + echo >> "$LINKSFILE" "$l $t" + link_relative "$t" "$l" "$RPM_BUILD_ROOT" +} + +# Provide .2, .3, ... symlinks to all filename instances of this build-id. +make_id_dup_link() +{ + local id="$1" file="$2" idfile + + local n=1 + while true; do + idfile=".build-id/${id:0:2}/${id:2}.$n" + [ $# -eq 3 ] && idfile="${idfile}$3" + if [ ! -L "$RPM_BUILD_ROOT/usr/lib/debug/$idfile" ]; then + break + fi + n=$[$n+1] + done + debug_link "$file" "/$idfile" +} + +# Make a build-id symlink for id $1 with suffix $3 to file $2. +make_id_link() +{ + local id="$1" file="$2" + local idfile=".build-id/${id:0:2}/${id:2}" + [ $# -eq 3 ] && idfile="${idfile}$3" + local root_idfile="$RPM_BUILD_ROOT/usr/lib/debug/$idfile" + + if [ ! -L "$root_idfile" ]; then + debug_link "$file" "/$idfile" + return + fi + + make_id_dup_link "$@" + + [ $# -eq 3 ] && return 0 + + local other=$(readlink -m "$root_idfile") + other=${other#$RPM_BUILD_ROOT} + if cmp -s "$root_idfile" "$RPM_BUILD_ROOT$file" || + eu-elfcmp -q "$root_idfile" "$RPM_BUILD_ROOT$file" 2> /dev/null; then + # Two copies. Maybe one has to be setuid or something. + echo >&2 "*** WARNING: identical binaries are copied, not linked:" + echo >&2 " $file" + echo >&2 " and $other" + else + # This is pathological, break the build. + echo >&2 "*** ERROR: same build ID in nonidentical files!" + echo >&2 " $file" + echo >&2 " and $other" + exit 2 + fi +} + +get_debugfn() +{ + dn=$(dirname "${1#$RPM_BUILD_ROOT}") + bn=$(basename "$1" .debug).debug + + debugdn=${debugdir}${dn} + debugfn=${debugdn}/${bn} +} + +set -o pipefail + +strict_error=ERROR +$strict || strict_error=WARNING + +# Strip ELF binaries +find "$RPM_BUILD_ROOT" ! -path "${debugdir}/*.debug" -type f \ + \( -perm -0100 -or -perm -0010 -or -perm -0001 \) \ + -print | +file -N -f - | sed -n -e 's/^\(.*\):[ ]*.*ELF.*, not stripped.*/\1/p' | +xargs --no-run-if-empty stat -c '%h %D_%i %n' | +while read nlinks inum f; do + get_debugfn "$f" + [ -f "${debugfn}" ] && continue + + # If this file has multiple links, keep track and make + # the corresponding .debug files all links to one file too. + if [ $nlinks -gt 1 ]; then + eval linked=\$linked_$inum + if [ -n "$linked" ]; then + eval id=\$linkedid_$inum + make_id_dup_link "$id" "$dn/$(basename $f)" + make_id_dup_link "$id" "/usr/lib/debug$dn/$bn" .debug + link=$debugfn + get_debugfn "$linked" + echo "hard linked $link to $debugfn" + mkdir -p "$(dirname "$link")" && ln -nf "$debugfn" "$link" + continue + else + eval linked_$inum=\$f + echo "file $f has $[$nlinks - 1] other hard links" + fi + fi + + echo "extracting debug info from $f" + id=$(/usr/lib/rpm/debugedit -b "$RPM_BUILD_DIR" -d /usr/src/debug \ + -i -l "$SOURCEFILE" "$f") || exit + if [ $nlinks -gt 1 ]; then + eval linkedid_$inum=\$id + fi + if [ -z "$id" ]; then + echo >&2 "*** ${strict_error}: No build ID note found in $f" + $strict && exit 2 + fi + + [ -x /usr/bin/gdb-add-index ] && /usr/bin/gdb-add-index "$f" > /dev/null 2>&1 + + # A binary already copied into /usr/lib/debug doesn't get stripped, + # just has its file names collected and adjusted. + case "$dn" in + /usr/lib/debug/*) + [ -z "$id" ] || make_id_link "$id" "$dn/$(basename $f)" + continue ;; + esac + + mkdir -p "${debugdn}" + if test -w "$f"; then + strip_to_debug "${debugfn}" "$f" + else + chmod u+w "$f" + strip_to_debug "${debugfn}" "$f" + chmod u-w "$f" + fi + + $include_minidebug && add_minidebug "${debugfn}" "$f" + + echo "./${f#$RPM_BUILD_ROOT}" >> "$ELFBINSFILE" + + if [ -n "$id" ]; then + make_id_link "$id" "$dn/$(basename $f)" + make_id_link "$id" "/usr/lib/debug$dn/$bn" .debug + fi +done || exit + +# Invoke the DWARF Compressor utility. +if $run_dwz && type dwz >/dev/null 2>&1 \ + && [ -d "${RPM_BUILD_ROOT}/usr/lib/debug" ]; then + dwz_files="`cd "${RPM_BUILD_ROOT}/usr/lib/debug"; find -type f -name \*.debug`" + if [ -n "${dwz_files}" ]; then + dwz_multifile_name="${RPM_PACKAGE_NAME}-${RPM_PACKAGE_VERSION}-${RPM_PACKAGE_RELEASE}.${RPM_ARCH}" + dwz_multifile_suffix= + dwz_multifile_idx=0 + while [ -f "${RPM_BUILD_ROOT}/usr/lib/debug/.dwz/${dwz_multifile_name}${dwz_multifile_suffix}" ]; do + let ++dwz_multifile_idx + dwz_multifile_suffix=".${dwz_multifile_idx}" + done + dwz_multfile_name="${dwz_multifile_name}${dwz_multifile_suffix}" + dwz_opts="-h -q -r -m .dwz/${dwz_multifile_name}" + mkdir -p "${RPM_BUILD_ROOT}/usr/lib/debug/.dwz" + [ -n "${dwz_low_mem_die_limit}" ] \ + && dwz_opts="${dwz_opts} -l ${dwz_low_mem_die_limit}" + [ -n "${dwz_max_die_limit}" ] \ + && dwz_opts="${dwz_opts} -L ${dwz_max_die_limit}" + ( cd "${RPM_BUILD_ROOT}/usr/lib/debug" && dwz $dwz_opts $dwz_files ) + # Remove .dwz directory if empty + rmdir "${RPM_BUILD_ROOT}/usr/lib/debug/.dwz" 2>/dev/null + if [ -f "${RPM_BUILD_ROOT}/usr/lib/debug/.dwz/${dwz_multifile_name}" ]; then + id="`readelf -Wn "${RPM_BUILD_ROOT}/usr/lib/debug/.dwz/${dwz_multifile_name}" \ + 2>/dev/null | sed -n 's/^ Build ID: \([0-9a-f]\+\)/\1/p'`" + [ -n "$id" ] \ + && make_id_link "$id" "/usr/lib/debug/.dwz/${dwz_multifile_name}" .debug + fi + fi +fi + +# dwz invalidates .gnu_debuglink CRC32 in the main files. +cat "$ELFBINSFILE" | +(cd "$RPM_BUILD_ROOT"; xargs -d '\n' /usr/lib/rpm/sepdebugcrcfix usr/lib/debug) + +# For each symlink whose target has a .debug file, +# make a .debug symlink to that file. +find "$RPM_BUILD_ROOT" ! -path "${debugdir}/*" -type l -print | +while read f +do + t=$(readlink -m "$f").debug + f=${f#$RPM_BUILD_ROOT} + t=${t#$RPM_BUILD_ROOT} + if [ -f "$debugdir$t" ]; then + echo "symlinked /usr/lib/debug$t to /usr/lib/debug${f}.debug" + debug_link "/usr/lib/debug$t" "${f}.debug" + fi +done + +if [ -s "$SOURCEFILE" ]; then + mkdir -p "${RPM_BUILD_ROOT}/usr/src/debug" + LC_ALL=C sort -z -u "$SOURCEFILE" | grep -E -v -z '(|)$' | + (cd "$RPM_BUILD_DIR"; cpio -pd0mL "${RPM_BUILD_ROOT}/usr/src/debug") + # stupid cpio creates new directories in mode 0700, fixup + find "${RPM_BUILD_ROOT}/usr/src/debug" -type d -print0 | + xargs --no-run-if-empty -0 chmod a+rx +fi + +if [ -d "${RPM_BUILD_ROOT}/usr/lib" -o -d "${RPM_BUILD_ROOT}/usr/src" ]; then + ((nout > 0)) || + test ! -d "${RPM_BUILD_ROOT}/usr/lib" || + (cd "${RPM_BUILD_ROOT}/usr/lib"; find debug -type d) | + sed 's,^,%dir /usr/lib/,' >> "$LISTFILE" + + (cd "${RPM_BUILD_ROOT}/usr" + test ! -d lib/debug || find lib/debug ! -type d + test ! -d src/debug || find src/debug -mindepth 1 -maxdepth 1 + ) | sed 's,^,/usr/,' >> "$LISTFILE" +fi + +# Append to $1 only the lines from stdin not already in the file. +append_uniq() +{ + grep -F -f "$1" -x -v >> "$1" +} + +# Helper to generate list of corresponding .debug files from a file list. +filelist_debugfiles() +{ + local extra="$1" + shift + sed 's/^%[a-z0-9_][a-z0-9_]*([^)]*) *// +s/^%[a-z0-9_][a-z0-9_]* *// +/^$/d +'"$extra" "$@" +} + +# Write an output debuginfo file list based on given input file lists. +filtered_list() +{ + local out="$1" + shift + test $# -gt 0 || return + grep -F -f <(filelist_debugfiles 's,^.*$,/usr/lib/debug&.debug,' "$@") \ + -x $LISTFILE >> $out + sed -n -f <(filelist_debugfiles 's/[\\.*+#]/\\&/g +h +s,^.*$,s# &$##p,p +g +s,^.*$,s# /usr/lib/debug&.debug$##p,p +' "$@") "$LINKSFILE" | append_uniq "$out" +} + +# Write an output debuginfo file list based on an grep -E -style regexp. +pattern_list() +{ + local out="$1" ptn="$2" + test -n "$ptn" || return + grep -E -x -e "$ptn" "$LISTFILE" >> "$out" + sed -n -r "\#^$ptn #s/ .*\$//p" "$LINKSFILE" | append_uniq "$out" +} + +# +# When given multiple -o switches, split up the output as directed. +# +i=0 +while ((i < nout)); do + > ${outs[$i]} + filtered_list ${outs[$i]} ${lists[$i]} + pattern_list ${outs[$i]} "${ptns[$i]}" + grep -Fvx -f ${outs[$i]} "$LISTFILE" > "${LISTFILE}.new" + mv "${LISTFILE}.new" "$LISTFILE" + ((++i)) +done +if ((nout > 0)); then + # Now add the right %dir lines to each output list. + (cd "${RPM_BUILD_ROOT}"; find usr/lib/debug -type d) | + sed 's#^.*$#\\@^/&/@{h;s@^.*$@%dir /&@p;g;}#' | + LC_ALL=C sort -ur > "${LISTFILE}.dirs.sed" + i=0 + while ((i < nout)); do + sed -n -f "${LISTFILE}.dirs.sed" "${outs[$i]}" | sort -u > "${outs[$i]}.new" + cat "${outs[$i]}" >> "${outs[$i]}.new" + mv -f "${outs[$i]}.new" "${outs[$i]}" + ((++i)) + done + sed -n -f "${LISTFILE}.dirs.sed" "${LISTFILE}" | sort -u > "${LISTFILE}.new" + cat "$LISTFILE" >> "${LISTFILE}.new" + mv "${LISTFILE}.new" "$LISTFILE" +fi diff --git a/uek-rpm/ol7/find-debuginfo.sh.parallel.diff b/uek-rpm/ol7/find-debuginfo.sh.parallel.diff new file mode 100644 index 0000000000000..0cd2776e115d6 --- /dev/null +++ b/uek-rpm/ol7/find-debuginfo.sh.parallel.diff @@ -0,0 +1,158 @@ +--- find-debuginfo.sh.orig 2019-02-07 04:18:03.000000000 -0800 ++++ find-debuginfo.sh 2019-02-07 04:32:15.904597748 -0800 +@@ -46,6 +46,9 @@ + dwz_low_mem_die_limit= + dwz_max_die_limit= + ++# Number of parallel jobs to spawn ++n_jobs=1 ++ + BUILDDIR=. + out=debugfiles.list + nout=0 +@@ -94,6 +97,13 @@ + -r) + strip_r=true + ;; ++ -j) ++ n_jobs=$2 ++ shift ++ ;; ++ -j*) ++ n_jobs=${1#-j} ++ ;; + *) + BUILDDIR=$1 + shift +@@ -277,41 +287,40 @@ + strict_error=ERROR + $strict || strict_error=WARNING + +-# Strip ELF binaries ++temp=$(mktemp -d ${TMPDIR:-/tmp}/find-debuginfo.XXXXXX) ++trap 'rm -rf "$temp"' EXIT ++ ++# Build a list of unstripped ELF files and their hardlinks ++touch "$temp/primary" + find "$RPM_BUILD_ROOT" ! -path "${debugdir}/*.debug" -type f \ + \( -perm -0100 -or -perm -0010 -or -perm -0001 \) \ + -print | + file -N -f - | sed -n -e 's/^\(.*\):[ ]*.*ELF.*, not stripped.*/\1/p' | + xargs --no-run-if-empty stat -c '%h %D_%i %n' | + while read nlinks inum f; do +- get_debugfn "$f" +- [ -f "${debugfn}" ] && continue +- +- # If this file has multiple links, keep track and make +- # the corresponding .debug files all links to one file too. + if [ $nlinks -gt 1 ]; then +- eval linked=\$linked_$inum +- if [ -n "$linked" ]; then +- eval id=\$linkedid_$inum +- make_id_dup_link "$id" "$dn/$(basename $f)" +- make_id_dup_link "$id" "/usr/lib/debug$dn/$bn" .debug +- link=$debugfn +- get_debugfn "$linked" +- echo "hard linked $link to $debugfn" +- mkdir -p "$(dirname "$link")" && ln -nf "$debugfn" "$link" ++ var=seen_$inum ++ if test -n "${!var}"; then ++ echo "$inum $f" >>"$temp/linked" + continue + else +- eval linked_$inum=\$f +- echo "file $f has $[$nlinks - 1] other hard links" ++ read "$var" < <(echo 1) + fi + fi ++ echo "$nlinks $inum $f" >>"$temp/primary" ++done ++ ++# Strip ELF binaries ++do_file() ++{ ++ local nlinks=$1 inum=$2 f=$3 id link linked ++ ++ get_debugfn "$f" ++ [ -f "${debugfn}" ] && return + + echo "extracting debug info from $f" + id=$(/usr/lib/rpm/debugedit -b "$RPM_BUILD_DIR" -d /usr/src/debug \ + -i -l "$SOURCEFILE" "$f") || exit +- if [ $nlinks -gt 1 ]; then +- eval linkedid_$inum=\$id +- fi + if [ -z "$id" ]; then + echo >&2 "*** ${strict_error}: No build ID note found in $f" + $strict && exit 2 +@@ -344,7 +353,69 @@ + make_id_link "$id" "$dn/$(basename $f)" + make_id_link "$id" "/usr/lib/debug$dn/$bn" .debug + fi +-done || exit ++ ++ # If this file has multiple links, make the corresponding .debug files ++ # all links to one file too. ++ if [ $nlinks -gt 1 ]; then ++ grep "^$inum " "$temp/linked" | while read inum linked; do ++ link=$debugfn ++ get_debugfn "$linked" ++ echo "hard linked $link to $debugfn" ++ mkdir -p "$(dirname "$debugfn")" && ln -nf "$link" "$debugfn" ++ done ++ fi ++} ++ ++# 16^6 - 1 or about 16 milion files ++FILENUM_DIGITS=6 ++run_job() ++{ ++ local jobid=$1 filenum ++ local SOURCEFILE=$temp/debugsources.$jobid ELFBINSFILE=$temp/elfbins.$jobid ++ ++ >"$SOURCEFILE" ++ >"$ELFBINSFILE" ++ # can't use read -n , because it reads bytes one by one, allowing for ++ # races ++ while :; do ++ filenum=$(dd bs=$(( FILENUM_DIGITS + 1 )) count=1 status=none) ++ if test -z "$filenum"; then ++ break ++ fi ++ do_file $(sed -n "$(( 0x$filenum )) p" "$temp/primary") ++ done ++ echo 0 >"$temp/res.$jobid" ++} ++ ++n_files=$(wc -l <"$temp/primary") ++if [ $n_jobs -gt $n_files ]; then ++ n_jobs=$n_files ++fi ++if [ $n_jobs -le 1 ]; then ++ while read nlinks inum f; do ++ do_file "$nlinks" "$inum" "$f" ++ done <"$temp/primary" ++else ++ for ((i = 1; i <= n_files; i++)); do ++ printf "%0${FILENUM_DIGITS}x\\n" $i ++ done | ( ++ exec 3<&0 ++ for ((i = 0; i < n_jobs; i++)); do ++ # The shell redirects stdin to /dev/null for background jobs. Work ++ # around this by duplicating fd 0 ++ run_job $i <&3 & ++ done ++ wait ++ ) ++ for f in "$temp"/res.*; do ++ res=$(< "$f") ++ if [ "$res" != "0" ]; then ++ exit 1 ++ fi ++ done ++ cat "$temp"/debugsources.* >"$SOURCEFILE" ++ cat "$temp"/elfbins.* >"$ELFBINSFILE" ++fi + + # Invoke the DWARF Compressor utility. + if $run_dwz && type dwz >/dev/null 2>&1 \ diff --git a/uek-rpm/ol7/kernel-uek.spec b/uek-rpm/ol7/kernel-uek.spec index f11015c25ac2f..b769ce9885112 100644 --- a/uek-rpm/ol7/kernel-uek.spec +++ b/uek-rpm/ol7/kernel-uek.spec @@ -555,6 +555,7 @@ Source20: x86_energy_perf_policy Source21: securebootca.cer Source22: secureboot.cer Source23: turbostat +Source24: find-debuginfo.sh Source1000: config-x86_64 Source1001: config-x86_64-debug @@ -571,6 +572,7 @@ Source202: ksplice_signing_key.x509 Source300: debuginfo-g1.diff Source301: find-debuginfo.sh.ol7.diff +Source302: find-debuginfo.sh.parallel.diff # Here should be only the patches up to the upstream canonical Linus tree. @@ -981,12 +983,13 @@ ApplyPatch %{stable_patch_00} ApplyPatch %{stable_patch_01} %endif -# Copy the RPM find-debuginfo.sh into the buildroot and patch it +# Copy bundled find-debuginfo.sh into the buildroot and patch it # to support -g1. (This is a patch of *RPM*, not of the kernel, # so it is not governed by nopatches.) -cp %{_rpmconfigdir}/find-debuginfo.sh %{_builddir} +cp %_sourcedir/find-debuginfo.sh %{_builddir} patch %{_builddir}/find-debuginfo.sh %{SOURCE300} && \ -patch %{_builddir}/find-debuginfo.sh %{SOURCE301} +patch %{_builddir}/find-debuginfo.sh %{SOURCE301} && \ +patch %{_builddir}/find-debuginfo.sh %{SOURCE302} chmod +x %{_builddir}/find-debuginfo.sh # only deal with configs if we are going to build for the arch @@ -1432,7 +1435,7 @@ find Documentation -type d | xargs chmod u+w %if %{with_debuginfo} %define __debug_install_post \ - %{_builddir}/find-debuginfo.sh %{debuginfo_args} -g1 %{_builddir}/%{?buildsubdir}\ + %{_builddir}/find-debuginfo.sh %{?_smp_mflags} %{debuginfo_args} -g1 %{_builddir}/%{?buildsubdir}\ %{nil} %ifnarch noarch