From 714195e1eb7fafd73b1eaa4c24340cc580ebc05a47ec67096f5cbfd4937becbb Mon Sep 17 00:00:00 2001 From: Olaf Hering Date: Fri, 4 Mar 2022 23:16:27 +0000 Subject: [PATCH 1/4] baserev update by copy to link target OBS-URL: https://build.opensuse.org/package/show/openSUSE:Tools/build-compare?expand=0&rev=288 --- build-compare.changes | 13 ++ build-compare.spec | 2 +- functions.sh | 64 +++++-- pkg-diff.sh | 409 ++++++++++++++++++++---------------------- same-build-result.sh | 38 +++- srpm-check.sh | 61 +++---- 6 files changed, 319 insertions(+), 268 deletions(-) diff --git a/build-compare.changes b/build-compare.changes index 062b3de..03d45cd 100644 --- a/build-compare.changes +++ b/build-compare.changes @@ -1,3 +1,16 @@ +------------------------------------------------------------------- +Fri Mar 4 12:34:56 UTC 2022 - olaf@aepfle.de + +- remove Check time report from rpmlint.log +- ELF diffing performance improvements +- unpack packages in parallel +- Add zstd handling + +------------------------------------------------------------------- +Wed Feb 23 08:13:27 UTC 2022 - Stefan Seyfried + +- Add extra handling for KMP versions + ------------------------------------------------------------------- Tue Sep 21 19:19:19 UTC 2021 - Stephan Kulow diff --git a/build-compare.spec b/build-compare.spec index 5044494..fd3778b 100644 --- a/build-compare.spec +++ b/build-compare.spec @@ -21,7 +21,7 @@ Summary: Build Result Compare Script License: GPL-2.0+ Group: Development/Tools/Building Url: https://github.com/openSUSE/build-compare -Version: 20210921T214418.86d06d9 +Version: 20220305T001156.c33e802 Release: 0 Source1: COPYING Source2: same-build-result.sh diff --git a/functions.sh b/functions.sh index a0d228e..259c3cc 100644 --- a/functions.sh +++ b/functions.sh @@ -141,23 +141,38 @@ check_header() # - it is used as direntry below certain paths # - it is assigned to some variable in scripts, at the end of a line # - it is used in PROVIDES, at the end of a line +# - special-case KMP package: +# PROVIDES version_k.*-release at end of line, trim release +# [ 23s] -acpi_call-kmp-default 8 1.2.2_k5.17.0_rc5_1.ga9b2c1d-6.110 +# [ 23s] +acpi_call-kmp-default 8 1.2.2_k5.17.0_rc5_1.ga9b2c1d-6.111 # Trim name-version-release string: # - it is used in update-scripts which are called by libzypp +# - special-case KMP package: +# [ 64s] PREIN +# [ 64s] /bin/sh (none) /usr/lib/module-init-tools/kernel-scriptlets/kmp-pre --name "acpi_call-kmp-default" \ +# [ 64s] - --version "1.2.2_k5.17.0_rc5_1.ga9b2c1d" --release "6.112" --kernelrelease "5.17.0-rc5-1.ga9b2c1d" \ +# [ 64s] + --version "1.2.2_k5.17.0_rc5_1.ga9b2c1d" --release "6.113" --kernelrelease "5.17.0-rc5-1.ga9b2c1d" \ function trim_release_old() { + local rel_regex_l=${version_release_old_regex_l##*-} sed -e " /\(\/boot\|\/lib\/modules\|\/lib\/firmware\|\/usr\/src\|$version_release_old_regex_l\$\|$version_release_old_regex_l)\)/{s,$version_release_old_regex_l,@VERSION@-@RELEASE_LONG@,g;s,$version_release_old_regex_s,@VERSION@-@RELEASE_SHORT@,g} s/\(\/var\/adm\/update-scripts\/\)${name_ver_rel_old_regex_l}\([^[:blank:]]\+\)/\1@NAME_VER_REL@\2/g s/\(\/var\/adm\/update-messages\/\)${name_ver_rel_old_regex_l}\([^[:blank:]]\+\)/\1@NAME_VER_REL@\2/g + s/\(^[^[:blank:]].*-kmp-.*[[:blank:]].*_k.*-\)${rel_regex_l}$/\1@RELEASE_LONG@/g + s/--release \"${rel_regex_l}\" --kernel/--release \"@RELEASE_LONG@\" --kernel/g /\/usr\/lib\/\.build-id/d " } function trim_release_new() { + local rel_regex_l=${version_release_new_regex_l##*-} sed -e " /\(\/boot\|\/lib\/modules\|\/lib\/firmware\|\/usr\/src\|$version_release_new_regex_l\$\|$version_release_new_regex_l)\)/{s,$version_release_new_regex_l,@VERSION@-@RELEASE_LONG@,g;s,$version_release_new_regex_s,@VERSION@-@RELEASE_SHORT@,g} s/\(\/var\/adm\/update-scripts\/\)${name_ver_rel_new_regex_l}\([^[:blank:]]\+\)/\1@NAME_VER_REL@\2/g s/\(\/var\/adm\/update-messages\/\)${name_ver_rel_new_regex_l}\([^[:blank:]]\+\)/\1@NAME_VER_REL@\2/g + s/\(^[^[:blank:]].*-kmp-.*[[:blank:]].*_k.*-\)${rel_regex_l}$/\1@RELEASE_LONG@/g + s/--release \"${rel_regex_l}\" --kernel/--release \"@RELEASE_LONG@\" --kernel/g /\/usr\/lib\/\.build-id/d " } @@ -280,16 +295,15 @@ function set_regex() { function cmp_rpm_meta () { local RES - local file1 file2 local f local sh=$1 local oldrpm=$2 local newrpm=$3 - - file1=`mktemp` - file2=`mktemp` - rpm_meta_old=`mktemp` - rpm_meta_new=`mktemp` + local tmpdir="$(mktemp -d)" + local file1="$tmpdir/file1" + local file2="$tmpdir/file2" + local rpm_meta_old="$tmpdir/rpm-meta-old" + local rpm_meta_new="$tmpdir/rpm-meta-new" collect_rpm_querytags set_rpm_meta_global_variables $oldrpm @@ -303,6 +317,7 @@ function cmp_rpm_meta () else ls -l $rpm_meta_old $rpm_meta_new echo "empty 'rpm -qp' output..." + rm -rf "$tmpdir" return 1 fi @@ -317,17 +332,20 @@ function cmp_rpm_meta () # Check the whole spec file at first, return 0 immediately if they # are the same. - cat $rpm_meta_old | trim_release_old > $file1 - cat $rpm_meta_new | trim_release_new > $file2 + trim_release_old < $rpm_meta_old > $file1 + trim_release_new < $rpm_meta_new > $file2 echo "comparing the rpm tags of $name_new" - if diff --label old-rpm-tags --label new-rpm-tags -au $file1 $file2; then - rm $file1 $file2 $rpm_meta_old $rpm_meta_new + if diff --speed-large-files --label old-rpm-tags --label new-rpm-tags -au0 $file1 $file2; then + rm -rf "$tmpdir" return 0 fi get_value QF_TAGS $rpm_meta_old > $file1 get_value QF_TAGS $rpm_meta_new > $file2 - comp_file rpmtags $file1 $file2 $rpm_meta_old $rpm_meta_new || return 1 + if ! comp_file rpmtags $file1 $file2 $rpm_meta_old $rpm_meta_new; then + rm -rf "$tmpdir" + return 1 + fi # This might happen when?! echo "comparing RELEASE" @@ -337,6 +355,7 @@ function cmp_rpm_meta () # Make sure all kernel packages have the same %RELEASE echo "release prefix mismatch" if test -z "$check_all"; then + rm -rf "$tmpdir" return 1 fi difffound=1 @@ -348,16 +367,25 @@ function cmp_rpm_meta () get_value QF_PROVIDES $rpm_meta_old | trim_release_old | sort > $file1 get_value QF_PROVIDES $rpm_meta_new | trim_release_new | sort > $file2 - comp_file PROVIDES $file1 $file2 $rpm_meta_old $rpm_meta_new || return 1 + if ! comp_file PROVIDES $file1 $file2 $rpm_meta_old $rpm_meta_new; then + rm -rf "$tmpdir" + return 1 + fi get_value QF_SCRIPT $rpm_meta_old | trim_release_old > $file1 get_value QF_SCRIPT $rpm_meta_new | trim_release_new > $file2 - comp_file scripts $file1 $file2 $rpm_meta_old $rpm_meta_new || return 1 + if ! comp_file scripts $file1 $file2 $rpm_meta_old $rpm_meta_new; then + rm -rf "$tmpdir" + return 1 + fi # First check the file attributes and later the md5s get_value QF_FILELIST $rpm_meta_old | trim_release_old > $file1 get_value QF_FILELIST $rpm_meta_new | trim_release_new > $file2 - comp_file filelist $file1 $file2 $rpm_meta_old $rpm_meta_new || return 1 + if ! comp_file filelist $file1 $file2 $rpm_meta_old $rpm_meta_new; then + rm -rf "$tmpdir" + return 1 + fi # now the md5sums. if they are different, we check more detailed # if there are different filenames, we will already have aborted before @@ -392,8 +420,8 @@ function cmp_rpm_meta () echo mv -v \"new/${f}\" \"new/`echo ${f} | trim_release_new`\" done >> "${sh}" fi - # - rm $file1 $file2 + + rm -rf "$tmpdir" [ "$difffound" = 1 ] && RES=1 return $RES } @@ -404,9 +432,9 @@ function adjust_controlfile() { version_release_new="`sed -ne 's/^Version: \(.*\)/\1/p' $2/control`" name_ver_rel_new="`sed -n -e 's/^Package: \(.*\)/\1/p' $2/control`-`sed -n -e 's/^Version: \(.*\)/\1/p' $2/control`" set_regex - cat $1/control | trim_release_old > $1/control.fixed + trim_release_old < $1/control > $1/control.fixed mv $1/control.fixed $1/control - cat $2/control | trim_release_new > $2/control.fixed + trim_release_new < $2/control > $2/control.fixed mv $2/control.fixed $2/control } diff --git a/pkg-diff.sh b/pkg-diff.sh index 4eee363..3ad9b36 100644 --- a/pkg-diff.sh +++ b/pkg-diff.sh @@ -55,7 +55,7 @@ filter_disasm() s/[0-9a-f]\+ :/\1:/ s/<\(.*\)+0x[0-9a-f]\+>/<\1 + ofs>/ - ' + ' } filter_xenefi() { @@ -318,6 +318,13 @@ check_compressed_file() xz -d new/$file.xz & wait ;; + zst) + mv old/$file{,.zst} + mv new/$file{,.zst} + zstd -d old/$file.zst & + zstd -d new/$file.zst & + wait + ;; esac ftype=`/usr/bin/file old/$file | sed 's@^[^:]\+:[[:blank:]]*@@'` case $ftype in @@ -414,7 +421,7 @@ normalize_file() *.dvi) filter_generic dvi ;; - *png) + *.png) # Try to remove timestamps, only if convert from ImageMagick is installed if [[ $(type -p convert) ]]; then filter_generic png @@ -599,7 +606,7 @@ normalize_file() *.ps) filter_generic ps ;; - *pdf) + *.pdf) filter_generic pdf ;; */linuxrc.config) @@ -858,7 +865,7 @@ check_single_file() compare_archive "${file}" 'archive_squashfs' return $? ;; - *.tar|*.tar.bz2|*.tar.gz|*.tgz|*.tbz2) + *.tar|*.tar.bz2|*.tar.gz|*.tgz|*.tbz2|*.tar.zst) compare_archive "${file}" 'archive_tar' return $? ;; @@ -866,30 +873,36 @@ check_single_file() compare_archive "${file}" 'archive_zip' return $? ;; - *.bz2) - bunzip2 -c old/$file > old/${file/.bz2/} - bunzip2 -c new/$file > new/${file/.bz2/} - check_single_file ${file/.bz2/} - return $? - ;; - *.gz) - gunzip -c old/$file > old/${file/.gz/} - gunzip -c new/$file > new/${file/.gz/} - check_single_file ${file/.gz/} - return $? - ;; - *.rpm) - $self_script -a old/$file new/$file - return $? - ;; + *.bz2) + bunzip2 -c old/$file > old/${file/.bz2/} + bunzip2 -c new/$file > new/${file/.bz2/} + check_single_file ${file/.bz2/} + return $? + ;; + *.gz) + gunzip -c old/$file > old/${file/.gz/} + gunzip -c new/$file > new/${file/.gz/} + check_single_file ${file/.gz/} + return $? + ;; + *.zst) + zstd -dc old/$file > old/${file/.zst/} + zstd -dc new/$file > new/${file/.zst/} + check_single_file ${file/.zst/} + return $? + ;; + *.rpm) + $self_script -a old/$file new/$file + return $? + ;; esac ftype=`/usr/bin/file "old/$file" | sed -e 's@^[^:]\+:[[:blank:]]*@@' -e 's@[[:blank:]]*$@@'` case $ftype in - PE32\ executable*Mono\/\.Net\ assembly*) - wprint "PE32 Mono/.Net assembly: $file" - if [ -x /usr/bin/monodis ] ; then - monodis "old/$file" 2>/dev/null|sed -e 's/GUID = {.*}/GUID = { 42 }/;'> ${file1} + PE32\ executable*Mono\/\.Net\ assembly*) + wprint "PE32 Mono/.Net assembly: $file" + if [ -x /usr/bin/monodis ] ; then + monodis "old/$file" 2>/dev/null|sed -e 's/GUID = {.*}/GUID = { 42 }/;'> ${file1} monodis "new/$file" 2>/dev/null|sed -e 's/GUID = {.*}/GUID = { 42 }/;'> ${file2} if ! cmp -s "${file1}" "${file2}"; then wprint "$file differs ($ftype)" @@ -909,207 +922,182 @@ check_single_file() set?id\ ELF*[LM]SB\ shared\ object*|\ ELF*[LM]SB\ pie\ executable*|\ set?id\ ELF*[LM]SB\ pie\ executable*) - diff --speed-large-files --unified \ - --label "old $file (disasm)" \ - --label "new $file (disasm)" \ - <( $OBJDUMP -d --no-show-raw-insn old/$file | - filter_disasm | - sed -e "s,old/,," ; - echo "${PIPESTATUS[@]}" > $file1 - ) \ - <( $OBJDUMP -d --no-show-raw-insn new/$file | - filter_disasm | - sed -e "s,new/,," ; - echo "${PIPESTATUS[@]}" > $file2 - ) > $dfile - ret=$? - - failed= - read i < ${file1} - pipestatus=( $i ) - objdump_failed="${pipestatus[0]}" - i=0 - while test $i -lt ${#pipestatus[@]} - do - if test "${pipestatus[$i]}" != "0" - then - wprint "ELF: pipe command #$i failed with ${pipestatus[$i]} for old/$file" - failed='failed' - fi - : $(( i++ )) - done - read i < ${file2} - pipestatus=( $i ) - objdump_failed="${objdump_failed}${pipestatus[0]}" - i=0 - while test $i -lt ${#pipestatus[@]} - do - if test "${pipestatus[$i]}" != "0" - then - wprint "ELF: pipe command #$i failed with ${pipestatus[$i]} for new/$file" - failed='failed' - fi - : $(( i++ )) - done - - if test "${objdump_failed}" != "00" || test -n "${failed}" - then - # objdump had no idea how to handle it - if diff_two_files; then - return 0 - fi - return 1 - fi - - elfdiff= - if test "$ret" != "0" - then - wprint "$file differs in assembler output" - $buildcompare_head $dfile - elfdiff='elfdiff' - fi - - sections="$( - $OBJDUMP -s new/$file | + local sections=($( + $OBJDUMP -s new/$file | sed -n --regexp-extended -e ' /Contents of section .*:/ { - s,.* (.*):,\1,g + s,.* (.*):,-j \1,g /\.build-id/d /\.gnu_debuglink/d /\.gnu_debugdata/d p } - ' - )" - for section in $sections - do - diff --unified \ + ')) + ($OBJDUMP -s ${sections[@]} old/$file | + sed -e "s,old/,," ; echo "${PIPESTATUS[@]}" > $file1 ) > old/$file.objdump & + ($OBJDUMP -s ${sections[@]} new/$file | + sed -e "s,new/,," ; echo "${PIPESTATUS[@]}" > $file2 ) > new/$file.objdump & + wait + read i < ${file1} + pipestatus=( $i ) + objdump_failed="${pipestatus[0]}" + if [[ ${pipestatus[*]} =~ [1-9] ]] + then + wprint "ELF section: pipe command failed for old/$file" + elfdiff='failed' + fi + read i < ${file2} + pipestatus=( $i ) + objdump_failed="${objdump_failed}${pipestatus[0]}" + if [[ ${pipestatus[*]} =~ [1-9] ]] + then + wprint "ELF section: pipe command failed for new/$file" + elfdiff='failed' + fi + if test -z "${elfdiff}" + then + diff --speed-large-files --unified \ --label "old $file (objdump)" \ --label "new $file (objdump)" \ - <( $OBJDUMP -s -j $section old/$file | - sed -e "s,^old/,," ; - echo "${PIPESTATUS[@]}" > $file1) \ - <( $OBJDUMP -s -j $section new/$file | - sed -e "s,^new/,," ; - echo "${PIPESTATUS[@]}" > $file2 - ) > $dfile + old/$file.objdump new/$file.objdump > $dfile ret=$? - failed= - read i < ${file1} - pipestatus=( $i ) - objdump_failed="${pipestatus[0]}" - i=0 - while test $i -lt ${#pipestatus[@]} - do - if test "${pipestatus[$i]}" != "0" - then - wprint "ELF section: pipe command #$i failed with ${pipestatus[$i]} for old/$file" - failed='failed' - fi - : $(( i++ )) - done - read i < ${file2} - pipestatus=( $i ) - objdump_failed="${objdump_failed}${pipestatus[0]}" - i=0 - while test $i -lt ${#pipestatus[@]} - do - if test "${pipestatus[$i]}" != "0" - then - wprint "ELF section: pipe command #$i failed with ${pipestatus[$i]} for new/$file" - failed='failed' - fi - : $(( i++ )) - done - if test -n "${failed}" - then - elfdiff='elfdiff' - break - fi if test "$ret" != "0" then - wprint "$file differs in ELF section $section" + wprint "$file differs in ELF sections" $buildcompare_head $dfile elfdiff='elfdiff' - else - watchdog_touch fi - done - if test -n "$elfdiff"; then + fi + if test -n "$elfdiff" + then + rm old/$file.objdump new/$file.objdump & return 1 fi + watchdog_touch + ($OBJDUMP -d --no-show-raw-insn old/$file | filter_disasm | + sed -e "s,old/,," ; echo "${PIPESTATUS[@]}" > $file1 ) > old/$file.objdump & + ($OBJDUMP -d --no-show-raw-insn new/$file | filter_disasm | + sed -e "s,new/,," ; echo "${PIPESTATUS[@]}" > $file2 ) > new/$file.objdump & + wait + read i < ${file1} + pipestatus=( $i ) + objdump_failed="${objdump_failed}${pipestatus[0]}" + if [[ ${pipestatus[*]} =~ [1-9] ]] + then + wprint "ELF disassembly: pipe command failed for old/$file" + elfdiff='failed' + fi + read i < ${file2} + pipestatus=( $i ) + objdump_failed="${objdump_failed}${pipestatus[0]}" + if [[ ${pipestatus[*]} =~ [1-9] ]] + then + wprint "ELF disassembly: pipe command failed for new/$file" + elfdiff='failed' + fi + if test ${objdump_failed} -gt 0 || test -n "${elfdiff}" + then + # objdump had no idea how to handle it + rm old/$file.objdump new/$file.objdump & + if diff_two_files; then + return 0 + fi + return 1 + fi + diff --speed-large-files --unified \ + --label "old $file (disasm)" \ + --label "new $file (disasm)" \ + old/$file.objdump new/$file.objdump > $dfile + ret=$? + rm old/$file.objdump new/$file.objdump & + if test "$ret" != "0" + then + wprint "$file differs in assembler output" + $buildcompare_head $dfile + elfdiff='elfdiff' + else + watchdog_touch + fi + if test -n "$elfdiff" + then + return 1 + fi + ;; + *ASCII*|*text*) + if ! cmp -s "old/$file" "new/$file" ; then + wprint "$file differs ($ftype)" + diff -u "old/$file" "new/$file" | $buildcompare_head + return 1 + fi + ;; + directory|setuid\ directory|setuid,\ directory|sticky,\ directory) + # tar might package directories - ignore them here return 0 ;; - *ASCII*|*text*) - if ! cmp -s "old/$file" "new/$file"; then - wprint "$file differs ($ftype)" - diff -u "old/$file" "new/$file" | $buildcompare_head - return 1 - fi - ;; - directory|setuid\ directory|setuid,\ directory|sticky,\ directory) - # tar might package directories - ignore them here - return 0 - ;; - bzip2\ compressed\ data*) - if ! check_compressed_file "$file" "bz2"; then - return 1 - fi - ;; - gzip\ compressed\ data*) - if ! check_compressed_file "$file" "gzip"; then - return 1 - fi - ;; - XZ\ compressed\ data*) - if ! check_compressed_file "$file" "xz"; then - return 1 - fi - ;; + bzip2\ compressed\ data*) + if ! check_compressed_file "$file" "bz2" ; then + return 1 + fi + ;; + gzip\ compressed\ data*) + if ! check_compressed_file "$file" "gzip" ; then + return 1 + fi + ;; + XZ\ compressed\ data*) + if ! check_compressed_file "$file" "xz" ; then + return 1 + fi + ;; + Zstandard\ compressed\ data*) + if ! check_compressed_file "$file" "zst" ; then + return 1 + fi + ;; Zip\ archive\ data,*) if ! compare_archive "${file}" 'archive_zip' ; then return 1 fi ;; - POSIX\ tar\ archive) - mv old/$file{,.tar} - mv new/$file{,.tar} - if ! check_single_file ${file}.tar; then - return 1 - fi - ;; - cpio\ archive) - mv old/$file{,.cpio} - mv new/$file{,.cpio} - if ! check_single_file ${file}.cpio; then - return 1 - fi - ;; - Squashfs\ filesystem,*) - wprint "$file ($ftype)" - mv old/$file{,.squashfs} - mv new/$file{,.squashfs} - if ! check_single_file ${file}.squashfs; then - return 1 - fi - ;; - broken\ symbolic\ link\ to\ *|symbolic\ link\ to\ *) - readlink "old/$file" > $file1 - readlink "new/$file" > $file2 - if ! diff -u $file1 $file2; then - wprint "symlink target for $file differs" - return 1 - fi - ;; - block\ special\ *) - ;; - character\ special\ *) - ;; - *) - if ! diff_two_files; then - return 1 - fi - ;; + POSIX\ tar\ archive) + mv old/$file{,.tar} + mv new/$file{,.tar} + if ! check_single_file ${file}.tar; then + return 1 + fi + ;; + cpio\ archive) + mv old/$file{,.cpio} + mv new/$file{,.cpio} + if ! check_single_file ${file}.cpio; then + return 1 + fi + ;; + Squashfs\ filesystem,*) + wprint "$file ($ftype)" + mv old/$file{,.squashfs} + mv new/$file{,.squashfs} + if ! check_single_file ${file}.squashfs; then + return 1 + fi + ;; + broken\ symbolic\ link\ to\ *|symbolic\ link\ to\ *) + readlink "old/$file" > $file1 + readlink "new/$file" > $file2 + if ! diff -u $file1 $file2; then + wprint "symlink target for $file differs" + return 1 + fi + ;; + block\ special\ *) + ;; + character\ special\ *) + ;; + *) + if ! diff_two_files; then + return 1 + fi + ;; esac return 0 } @@ -1211,8 +1199,9 @@ case $oldpkg in esac wprint "Extracting packages" -unpackage $oldpkg $dir/old -unpackage $newpkg $dir/new +unpackage $oldpkg $dir/old & +unpackage $newpkg $dir/new & +wait case $oldpkg in *.deb|*.ipk) @@ -1241,12 +1230,12 @@ fi # preserve cmp_rpm_meta result for check_all runs ret=$RES for file in "${files[@]}"; do - if ! check_single_file "$file"; then - ret=1 - if test -z "$check_all"; then - break - fi - fi + if ! check_single_file "$file"; then + ret=1 + if test -z "$check_all"; then + break + fi + fi done if [ "$PROC_MOUNTED" -eq "1" ]; then @@ -1255,7 +1244,7 @@ if [ "$PROC_MOUNTED" -eq "1" ]; then fi if test "$ret" = 0; then - echo "Package content is identical" + echo "Package content is identical" fi exit $ret # vim: tw=666 ts=2 shiftwidth=2 et diff --git a/same-build-result.sh b/same-build-result.sh index 3c2868f..1bb31dd 100644 --- a/same-build-result.sh +++ b/same-build-result.sh @@ -21,6 +21,31 @@ _x() { } trap _x EXIT # +remove_check_time_report() { + local f=$1 + awk ' + BEGIN { + ctr_seen=0; + } + /Check time report .*:$/ { + ctr_seen=1; + next; + } + /TOTAL[[:blank:]]+[0-9]/ { + if (ctr_seen == 1) { + ctr_seen=0; + next; + } + } + { + if (ctr_seen == 1) { + next; + } + print $0; + } + ' < "${f}" +} +# check_all= if test "$1" = "-a" then @@ -134,12 +159,15 @@ else fi if test -n "$OTHERDIR"; then - if test -e $OLDDIR/rpmlint.log -a -e $OTHERDIR/rpmlint.log; then - echo "comparing $OLDDIR/rpmlint.log and $OTHERDIR/rpmlint.log" + old_log=$OLDDIR/rpmlint.log + new_log=$OTHERDIR/rpmlint.log + if test -e ${old_log} && test -e ${new_log} ; then + echo "comparing ${old_log} and ${new_log}" + # Remove --time-report from rpmlint # Sort the files first since the order of messages is not deterministic # Remove release from files - sort -u $OLDDIR/rpmlint.log|sed -e "s,$ver_rel1,@VERSION@-@RELEASE@,g" -e "s|/tmp/rpmlint\..*spec|.spec|g" > $file1 - sort -u $OTHERDIR/rpmlint.log|sed -e "s,$ver_rel2,@VERSION@-@RELEASE@,g" -e "s|/tmp/rpmlint\..*spec|.spec|g" > $file2 + remove_check_time_report ${old_log}|sort -u|sed -e "s,$ver_rel1,@VERSION@-@RELEASE@,g" -e "s|/tmp/rpmlint\..*spec|.spec|g" > $file1 + remove_check_time_report ${new_log}|sort -u|sed -e "s,$ver_rel2,@VERSION@-@RELEASE@,g" -e "s|/tmp/rpmlint\..*spec|.spec|g" > $file2 # Remove odd warning about not-hardlinked files # Remove odd warning about data and time, it comes and goes # Remove warning about python mtime mismatch, a republish will not help @@ -159,7 +187,7 @@ if test -n "$OTHERDIR"; then SUCCESS=0 fi rm $file1 $file2 - elif test -e $OTHERDIR/rpmlint.log; then + elif test -e ${new_log} ; then echo "rpmlint.log is new" SUCCESS=0 fi diff --git a/srpm-check.sh b/srpm-check.sh index 10880b5..05e5408 100644 --- a/srpm-check.sh +++ b/srpm-check.sh @@ -1,10 +1,11 @@ #!/bin/bash # # Copyright (c) 2009, 2010 SUSE Linux Product GmbH, Germany. +# Copyright (c) 2022 SUSE LLC # Licensed under GPL v2, see COPYING file for details. # -# Written by Michael Matz and Stephan Coolo -# Enhanced by Andreas Jaeger +# Written by Michael Matz and Stephan Kulow +# Enhanced by Andreas Jaeger and Dirk Müller # Compare two source RPMs @@ -18,21 +19,16 @@ case $1 in esac if test "$#" != 2; then - echo "usage: $0 [-a|--check-all] old.rpm new.rpm" - exit 1 + echo "usage: $0 [-a|--check-all] old.rpm new.rpm" + exit 1 fi source $FUNCTIONS -oldrpm=`readlink -f $1` -newrpm=`readlink -f $2` +oldrpm=$(readlink -f $1) +newrpm=$(readlink -f $2) rename_script= -# Get version-release from first RPM and keep for rpmlint check -# Remember to quote the "." for future regexes -ver_rel_old=$(rpm -qp --nodigest --nosignature --qf "%{RELEASE}" "${oldrpm}"|sed -e 's/\./\\./g') -ver_rel_new=$(rpm -qp --nodigest --nosignature --qf "%{RELEASE}" "${newrpm}"|sed -e 's/\./\\./g') - # For source RPMs, we can just check the metadata in the spec file # if those are not the same, the source RPM has changed and therefore # the resulting files are needed. @@ -59,41 +55,38 @@ esac # Now check that only the spec file has a changed release number and # nothing else -dir=`mktemp -d` -unpackage $oldrpm $dir/old -unpackage $newrpm $dir/new +dir=$(mktemp -d) +unpackage $oldrpm $dir/old & +unpackage $newrpm $dir/new & cd $dir +wait check_single_file() { local file=$1 case $file in *.spec) - sed -i -e 's,^Release:.*$,Release: @RELEASE@,' old/$file - sed -i -e 's,^Release:.*$,Release: @RELEASE@,' new/$file - if ! cmp -s old/$file new/$file; then - echo "$file differs (spec file)" - diff -u old/$file new/$file | head -n 20 - return 1 - fi - return 0 - ;; + sed -i -e 's,^Release:.*$,Release: @RELEASE@,' old/$file + sed -i -e 's,^Release:.*$,Release: @RELEASE@,' new/$file + diff --speed-large-files -su0 old/$file new/$file | head -n 20 + return "${PIPESTATUS[0]}" + ;; *) - echo "$file differs" - # Nothing else should be changed - ;; - esac - return 1 + echo "$file differs" + # Nothing else should be changed + ;; + esac + return 1 } ret=0 for file in "${files[@]}"; do - if ! check_single_file $file; then - ret=1 - if test -z "$check_all"; then - break - fi - fi + if ! check_single_file $file; then + ret=1 + if test -z "$check_all"; then + break + fi + fi done rm -rf $dir From 107f5e6e1c23bd66431011b792924552ff151d1ec3672215a8ee56e364921691 Mon Sep 17 00:00:00 2001 From: Olaf Hering Date: Sun, 6 Mar 2022 11:17:34 +0000 Subject: [PATCH 2/4] fix ELF comparision to really show disasm output OBS-URL: https://build.opensuse.org/package/show/openSUSE:Tools/build-compare?expand=0&rev=289 --- build-compare.spec | 2 +- pkg-diff.sh | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/build-compare.spec b/build-compare.spec index fd3778b..a8b5e0d 100644 --- a/build-compare.spec +++ b/build-compare.spec @@ -21,7 +21,7 @@ Summary: Build Result Compare Script License: GPL-2.0+ Group: Development/Tools/Building Url: https://github.com/openSUSE/build-compare -Version: 20220305T001156.c33e802 +Version: 20220306T121516.38e7800 Release: 0 Source1: COPYING Source2: same-build-result.sh diff --git a/pkg-diff.sh b/pkg-diff.sh index 3ad9b36..a8080ff 100644 --- a/pkg-diff.sh +++ b/pkg-diff.sh @@ -968,16 +968,17 @@ check_single_file() elfdiff='elfdiff' fi fi - if test -n "$elfdiff" + if test -z "$elfdiff" then rm old/$file.objdump new/$file.objdump & return 1 fi watchdog_touch + elfdiff= ($OBJDUMP -d --no-show-raw-insn old/$file | filter_disasm | - sed -e "s,old/,," ; echo "${PIPESTATUS[@]}" > $file1 ) > old/$file.objdump & + sed -e "s,^old/[^:]\+,," ; echo "${PIPESTATUS[@]}" > $file1 ) > old/$file.objdump & ($OBJDUMP -d --no-show-raw-insn new/$file | filter_disasm | - sed -e "s,new/,," ; echo "${PIPESTATUS[@]}" > $file2 ) > new/$file.objdump & + sed -e "s,^new/[^:]\+,," ; echo "${PIPESTATUS[@]}" > $file2 ) > new/$file.objdump & wait read i < ${file1} pipestatus=( $i ) @@ -1004,6 +1005,7 @@ check_single_file() fi return 1 fi + elfdiff= diff --speed-large-files --unified \ --label "old $file (disasm)" \ --label "new $file (disasm)" \ From f7c0d407ba80864f15f7db320d62776b24da3feaf9da01e5b32bb323523507b2 Mon Sep 17 00:00:00 2001 From: Olaf Hering Date: Sun, 6 Mar 2022 22:02:46 +0000 Subject: [PATCH 3/4] - remove count of checks and packages from rpmlint.log OBS-URL: https://build.opensuse.org/package/show/openSUSE:Tools/build-compare?expand=0&rev=290 --- build-compare.changes | 1 + build-compare.spec | 2 +- same-build-result.sh | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/build-compare.changes b/build-compare.changes index 03d45cd..07d82ea 100644 --- a/build-compare.changes +++ b/build-compare.changes @@ -1,6 +1,7 @@ ------------------------------------------------------------------- Fri Mar 4 12:34:56 UTC 2022 - olaf@aepfle.de +- remove count of checks and packages from rpmlint.log - remove Check time report from rpmlint.log - ELF diffing performance improvements - unpack packages in parallel diff --git a/build-compare.spec b/build-compare.spec index a8b5e0d..2d477ba 100644 --- a/build-compare.spec +++ b/build-compare.spec @@ -21,7 +21,7 @@ Summary: Build Result Compare Script License: GPL-2.0+ Group: Development/Tools/Building Url: https://github.com/openSUSE/build-compare -Version: 20220306T121516.38e7800 +Version: 20220306T230110.5c8c763 Release: 0 Source1: COPYING Source2: same-build-result.sh diff --git a/same-build-result.sh b/same-build-result.sh index 1bb31dd..08dbb86 100644 --- a/same-build-result.sh +++ b/same-build-result.sh @@ -173,6 +173,7 @@ if test -n "$OTHERDIR"; then # Remove warning about python mtime mismatch, a republish will not help # Remove odd warning about filenames, they contain VERSION-RELEASE # Remove durations from progress reports + # Remove odd output about number of checks and packages sed -i -e " /: W: files-duplicate /d /: W: file-contains-date-and-time /d @@ -180,6 +181,7 @@ if test -n "$OTHERDIR"; then /: W: filename-too-long-for-joliet /d /: I: \(filelist-initialization\|check-completed\) /s| [0-9]\+\.[0-9] s| x.x s| s/; has taken [0-9]\+\.[0-9] s/; has taken x.x s/ + /^checks: [0-9]\+, packages: [0-9]\+/d " $file1 $file2 if ! cmp -s $file1 $file2; then echo "rpmlint.log files differ:" From 3645d843bb3b068d75cf96e471eae85357683136d65cbf408e231de30bb2d61b Mon Sep 17 00:00:00 2001 From: Olaf Hering Date: Mon, 7 Mar 2022 10:57:31 +0000 Subject: [PATCH 4/4] - handle -a also in same-build-result.sh - Find rpmlint.log in more places - fix logic error in appstream comparison - rework exit handling in same-build-result.sh - Fix result in case no rpmlint.log exist OBS-URL: https://build.opensuse.org/package/show/openSUSE:Tools/build-compare?expand=0&rev=291 --- build-compare.changes | 5 +++++ build-compare.spec | 2 +- pkg-diff.sh | 2 +- same-build-result.sh | 52 +++++++++++++++++++++++++++++++------------ 4 files changed, 45 insertions(+), 16 deletions(-) diff --git a/build-compare.changes b/build-compare.changes index 07d82ea..6775c19 100644 --- a/build-compare.changes +++ b/build-compare.changes @@ -1,6 +1,11 @@ ------------------------------------------------------------------- Fri Mar 4 12:34:56 UTC 2022 - olaf@aepfle.de +- handle -a also in same-build-result.sh +- Find rpmlint.log in more places +- fix logic error in appstream comparison +- rework exit handling in same-build-result.sh +- Fix result in case no rpmlint.log exist - remove count of checks and packages from rpmlint.log - remove Check time report from rpmlint.log - ELF diffing performance improvements diff --git a/build-compare.spec b/build-compare.spec index 2d477ba..54fcbc5 100644 --- a/build-compare.spec +++ b/build-compare.spec @@ -21,7 +21,7 @@ Summary: Build Result Compare Script License: GPL-2.0+ Group: Development/Tools/Building Url: https://github.com/openSUSE/build-compare -Version: 20220306T230110.5c8c763 +Version: 20220307T115648.b5abdde Release: 0 Source1: COPYING Source2: same-build-result.sh diff --git a/pkg-diff.sh b/pkg-diff.sh index a8080ff..e208d0a 100644 --- a/pkg-diff.sh +++ b/pkg-diff.sh @@ -971,7 +971,7 @@ check_single_file() if test -z "$elfdiff" then rm old/$file.objdump new/$file.objdump & - return 1 + return 0 fi watchdog_touch elfdiff= diff --git a/same-build-result.sh b/same-build-result.sh index 08dbb86..b1d05d0 100644 --- a/same-build-result.sh +++ b/same-build-result.sh @@ -14,6 +14,11 @@ CMPSCRIPT=${0%/*}/pkg-diff.sh SCMPSCRIPT=${0%/*}/srpm-check.sh +declare -a exit_code +# exit_code[0]='' # binaries_differ +# exit_code[1]='' # rpmlint_differs +# exit_code[2]='' # appdata_differs +# exit_code[3]='' # srcrpm_differs file1=`mktemp` file2=`mktemp` _x() { @@ -90,7 +95,13 @@ if test ! -f "$nsrpm"; then fi echo "compare $osrpm $nsrpm" -bash $SCMPSCRIPT "$osrpm" "$nsrpm" || exit 1 +if bash $SCMPSCRIPT "$osrpm" "$nsrpm" +then + : src.rpm identical +else + test -z "${check_all}" && exit 1 + exit_code[3]='srcrpm_differs' +fi # technically we should not all exclude all -32bit but filter for different archs, # like done with -x86 @@ -121,7 +132,6 @@ NEWRPMS=($( sort --field-separator=/ --key=` sed -n '1s@[^/]@@gp' ${file2} | wc ver_rel1=$(rpm -qp --nodigest --nosignature --qf "%{VERSION}-%{RELEASE}" "${OLDRPMS[0]}"|sed -e 's/\./\\./g') ver_rel2=$(rpm -qp --nodigest --nosignature --qf "%{VERSION}-%{RELEASE}" "${NEWRPMS[0]}"|sed -e 's/\./\\./g') -SUCCESS=1 rpmqp='rpm -qp --qf %{NAME} --nodigest --nosignature ' for opac in ${OLDRPMS[*]}; do npac=${NEWRPMS[0]} @@ -138,7 +148,7 @@ for opac in ${OLDRPMS[*]}; do echo "skipping -debuginfo package" ;; *) - bash $CMPSCRIPT $check_all "$opac" "$npac" || SUCCESS=0 + bash $CMPSCRIPT $check_all "$opac" "$npac" || exit_code[0]='binaries_differ' ;; esac done @@ -148,14 +158,20 @@ if [ -n "${NEWRPMS[0]}" ]; then exit 1 fi +OTHERDIR= # Compare rpmlint.log files if test -d /home/abuild/rpmbuild/OTHER; then OTHERDIR=/home/abuild/rpmbuild/OTHER elif test -d /usr/src/packages/OTHER; then OTHERDIR=/usr/src/packages/OTHER else - echo "no OTHERDIR" - OTHERDIR= + for newdir in $NEWDIRS + do + test -f "${newdir}/rpmlint.log" || continue + OTHERDIR="${newdir}" + break + done + test -n "$OTHERDIR" || echo "no OTHERDIR" fi if test -n "$OTHERDIR"; then @@ -186,33 +202,41 @@ if test -n "$OTHERDIR"; then if ! cmp -s $file1 $file2; then echo "rpmlint.log files differ:" diff -u $file1 $file2 |head -n 20 - SUCCESS=0 + exit_code[1]='rpmlint_differs' fi rm $file1 $file2 - elif test -e ${new_log} ; then - echo "rpmlint.log is new" - SUCCESS=0 + else + if test -e "${new_log}" + then + exit_code[1]='rpmlint_new' + echo "rpmlint.log is new" + elif test -e "${old_log}" + then + exit_code[1]='rpmlint_old' + echo "rpmlint.log disappeared" + else + echo "No rpmlint.log available" + fi fi appdatas=$(cd $OTHERDIR && find . -name "*-appdata.xml") for xml in $appdatas; do # compare appstream data - if test -e $OLDDIR/$xml -a -e $OTHERDIR/$xml; then + if test -e $OLDDIR/$xml && test -e $OTHERDIR/$xml; then file1=$OLDDIR/$xml file2=$OTHERDIR/$xml if ! cmp -s $file1 $file2; then echo "$xml files differ:" diff -u0 $file1 $file2 |head -n 20 - SUCCESS=0 + exit_code[2]='appdata_differs' fi elif test -e $OTHERDIR/$xml; then echo "$xml is new" - SUCCESS=0 + exit_code[2]='appdata_new' fi done fi - -if test $SUCCESS -eq 0; then +if test -n "${exit_code[*]}"; then exit 1 fi echo 'compare validated build as identical !'