qemu/bundle2spec.sh
2019-09-18 08:58:50 +00:00

477 lines
18 KiB
Bash

#!/bin/bash
#
# Instead of a quilt workflow, we use a git tree that contains
# all the commits on top of a qemu source tarball.
#
# When updating this package, just either update the git tree
# below (use rebase!) or change the tree path and use your own
#
# That way we can easily rebase against the next stable release
# when it comes.
# TODO:
# * don't re-create bundles that are the same, and if no change
# don't re-create the bunde-of-bundles
# * somehow for the LATEST case, keep track of the patches which
# have come and gone until we apply the whole set to the changelog
# * also when that update comes, include the Referece: references as
# being included into the update
set -e
source ./config.sh
rm -rf $GIT_DIR
rm -rf $CMP_DIR
rm -rf $BUNDLE_DIR
rm -f checkpatch.log
rm -rf checkthese
if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then
# This is just a safety valve in case the above gets edited wrong:
if ! [ "$GIT_BRANCH" = "master" ]; then
echo "LATEST implies master branch, please fix configuration"
exit
fi
(cd ${LOCAL_REPO_MAP[0]} && git remote update upstream)
fi
BASE_RE="qemu-[[:digit:]]+(\.[[:digit:]]+){2}(-rc[[:digit:]])?"
EXTRA_RE="\+git\.[[:digit:]]+\.([[:xdigit:]]+)"
SUFFIX_RE="\.tar\.xz"
SIG_SUFFIX_RE="\.tar\.xz\.sig"
QEMU_TARBALL=($(find -maxdepth 1 -type f -regextype posix-extended -regex \
"\./$BASE_RE($EXTRA_RE)?$SUFFIX_RE" -printf "%f "))
QEMU_TARBALL_SIG=($(find -maxdepth 1 -type f -regextype posix-extended -regex \
"\./$BASE_RE($EXTRA_RE)?$SIG_SUFFIX_RE" -printf "%f "))
if [ ${#QEMU_TARBALL[@]} -gt 1 ]; then
echo "Multiple qemu tarballs detected. Please clean up"
exit
fi
if [ ${#QEMU_TARBALL_SIG[@]} -gt 1 ]; then
echo "Multiple qemu tarballs signature files detected. Please clean up"
exit
fi
# It's ok for either of these to be empty when using "LATEST"
OLD_SOURCE_VERSION_AND_EXTRA=$(echo $QEMU_TARBALL 2>/dev/null | head --bytes=-8\
| cut --bytes=6-)
VERSION_EXTRA=$(echo $OLD_SOURCE_VERSION_AND_EXTRA|awk -F+ '{if ($2) print \
"+"$2}')
if [ "$OLD_SOURCE_VERSION_AND_EXTRA" = "" ]; then
echo "Warning: No tarball found"
fi
mkdir -p $BUNDLE_DIR
# This tarball has git bundles stored in a directory structure which mimics the
# submodule locations in the containing git repo. Also at that same dir level
# is a file named repo which contains the one line git repo url (with git:// or
# http(s) prefix). The bundles are named as follows:
# "{path/}{git_sha}.{patch_prefix}.{bundle}", where {path/} isn't present for
# the top (qemu) bundle (ie it's for submodules).
tar xJf bundles.tar.xz -C $BUNDLE_DIR
# !!! The following may be overkill, since it seems that find does do a depth first, which is all we need
BUNDLE_FILES=$(find $BUNDLE_DIR -printf "%P\n"|grep "bundle$")
if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then
if [[ $QEMU_TARBALL =~ $BASE_RE$EXTRA_RE$SUFFIX_RE ]]; then
OLD_COMMIT_ISH=${BASH_REMATCH[3]}
else
#Assume release (or release candidate) tarball with equivalent tag:
OLD_COMMIT_ISH=$(cd ${LOCAL_REPO_MAP[0]} && git rev-list --abbrev-commit \
--abbrev=9 -1 v$OLD_SOURCE_VERSION_AND_EXTRA)
fi
if [ ${#QEMU_TARBALL_SIG[@]} -ne 0 ]; then
echo "INFO: Ignoring signature file: $QEMU_TARBALL_SIG"
QEMU_TARBALL_SIG=
fi
NEW_COMMIT_ISH=$(cd ${LOCAL_REPO_MAP[0]} && git rev-parse --short=9 \
upstream/$GIT_BRANCH)
NOW_SECONDS=$(date +%s)
git clone -ls ${LOCAL_REPO_MAP[0]} $GIT_DIR -b $GIT_BRANCH --single-branch &>/dev/null
if [ "$OLD_COMMIT_ISH" != "$NEW_COMMIT_ISH" ]; then
echo "Please wait..."
(cd $GIT_DIR && git remote add upstream \
git://git.qemu-project.org/qemu.git &>/dev/null)
(cd $GIT_DIR && git remote update upstream &>/dev/null)
(cd $GIT_DIR && git checkout $NEW_COMMIT_ISH &>/dev/null)
(cd $GIT_DIR && git submodule update --init --recursive &>/dev/null)
VERSION_EXTRA=+git.$NOW_SECONDS.$NEW_COMMIT_ISH
fi
QEMU_VERSION=$(cat $GIT_DIR/VERSION)
MAJOR_VERSION=$(echo $QEMU_VERSION|awk -F. '{print $1}')
MINOR_VERSION=$(echo $QEMU_VERSION|awk -F. '{print $2}')
X=$(echo $QEMU_VERSION|awk -F. '{print $3}')
# 0 = release, 50 = development cycle, 90..99 equate to release candidates
if [ "$X" != "0" -a "$X" != "50" ]; then
if [ "$NEXT_RELEASE_IS_MAJOR" = "0" ]; then
SOURCE_VERSION=$MAJOR_VERSION.$[$MINOR_VERSION+1].0-rc$[X-90]
else
SOURCE_VERSION=$[$MAJOR_VERSION+1].0.0-rc$[X-90]
fi
else
SOURCE_VERSION=$MAJOR_VERSION.$MINOR_VERSION.$X
fi
if [ "$OLD_COMMIT_ISH" != "$NEW_COMMIT_ISH" ]; then
if (cd ${LOCAL_REPO_MAP[0]} && git describe --exact-match $NEW_COMMIT_ISH \
&>/dev/null); then
if [ "$X" = "50" ]; then
echo "Ignoring non-standard tag"
else
# there is no VERSION_EXTRA
VERSION_EXTRA=
fi
fi
(cd $GIT_DIR/roms/seabios && git describe --tags --long --dirty > \
.version)
(cd $GIT_DIR/roms/skiboot && ./make_version.sh > .version)
echo "Almost there..."
tar --exclude=.git --transform "s,$GIT_DIR,qemu-$SOURCE_VERSION," \
-Pcf qemu-$SOURCE_VERSION$VERSION_EXTRA.tar $GIT_DIR
echo "New tarball created. Attempting rebase..."
if ! (cd $GIT_DIR && git rebase upstream/$GIT_BRANCH $GIT_BRANCH); then
echo "rebasing master on upstream/master needs human assistance." \
"Exiting"
(cd $GIT_DIR && git rebase --abort)
rm qemu-$SOURCE_VERSION$VERSION_EXTRA.tar
exit
fi
echo "WARNING: To rebase, master is being checked out"
if ! (cd ${LOCAL_REPO_MAP[0]} && git rebase upstream/$GIT_BRANCH \
$GIT_BRANCH); then
echo "WARNING: Script error? rebasing master on upstream/master" \
"succeeded in temp"
echo "dir but failed in local tree! Please investigate"
(cd ${LOCAL_REPO_MAP[0]} && git rebase --abort)
rm qemu-$SOURCE_VERSION$VERSION_EXTRA.tar
exit
fi
echo "Rebase successful"
osc rm --force qemu-$OLD_SOURCE_VERSION_AND_EXTRA.tar.xz &>/dev/null ||\
true
osc rm --force qemu-$OLD_SOURCE_VERSION_AND_EXTRA.tar.xz.sig \
&>/dev/null || true
unset QEMU_TARBALL_SIG
xz -T 0 qemu-$SOURCE_VERSION$VERSION_EXTRA.tar
osc add qemu-$SOURCE_VERSION$VERSION_EXTRA.tar.xz
fi
# We're done with GIT_UPSTREAM_COMMIT_ISH carrying the special value LATEST
GIT_UPSTREAM_COMMIT_ISH=$NEW_COMMIT_ISH
WRITE_LOG=0
else # not based on LATEST upstream master, rather any upstream commitish
if [ "$OLD_SOURCE_VERSION_AND_EXTRA" = "" ]; then
echo "Failure: tarball required which corresponds to commitish:" \
"$GIT_UPSTREAM_COMMITISH"
exit
fi
if [ -d "${LOCAL_REPO_MAP[0]}" ]; then
echo "Processing local git tree branch: $GIT_BRANCH, using commitish:"\
"$GIT_UPSTREAM_COMMIT_ISH"
if ! (cd ${LOCAL_REPO_MAP[0]} && git show-branch $GIT_BRANCH &>/dev/null)
then
echo "Error: Branch $GIT_BRANCH not found - please create a remote"\
"tracking branch of origin/$GIT_BRANCH"
exit
fi
for entry in ${BUNDLE_FILES[@]}; do
if [[ $entry =~ ^(.*)[/]*([a-f0-9]{40})[.]bundle$ ]]; then
SUBDIR=${BASH_REMATCH[1]}
GITREPO_COMMIT_ISH=${BASH_REMATCH[2]}
else
echo "ERROR! BAD BUNDLE CONTENT!"
exit
fi
for (( i=0; i <$REPO_COUNT; i++ )); do
if [[ "$SUBDIR" = "${PATCH_PATH_MAP[$i]}" ]]; then
PATCH_RANGE_INDEX=$i
break
fi
done
# !!!!! REVIEW WHERE THIS mkdir SHOULD BE HAPPENING (kind of replaces the clone operation)
mkdir -p $GIT_DIR/$SUBDIR
git -C $GIT_DIR/$SUBDIR init
git -C $GIT_DIR/$SUBDIR remote add origin file://$(readlink -f \
${LOCAL_REPO_MAP[$PATCH_RANGE_INDEX]})
git -C $GIT_DIR/$SUBDIR fetch origin $GIT_BRANCH
git -C $GIT_DIR/$SUBDIR reset --hard $GITREPO_COMMIT_ISH
git -C $GIT_DIR/$SUBDIR remote add bundle $BUNDLE_DIR/$entry
# depending on how created, the bundle's head is called HEAD or FETCH_HEAD
#git -C $GIT_DIR/$SUBDIR fetch bundle HEAD
git -C $GIT_DIR/$SUBDIR fetch bundle FETCH_HEAD
git -C $GIT_DIR/$SUBDIR format-patch -N --suffix= --no-renames -o $CMP_DIR -k --stat=72 \
--indent-heuristic --zero-commit --no-signature --full-index \
--src-prefix=a/$SUBDIR --dst-prefix=b/$SUBDIR \
--start-number=$(expr $PATCH_RANGE_INDEX \* $PATCH_RANGE) \
$GITREPO_COMMIT_ISH..FETCH_HEAD > /dev/null
done
# ( THIS ISNT WORKING - IS OLD HISTORY:)
else
echo "Processing $GIT_BRANCH branch of remote git tree, using"\
"commitish: $GIT_UPSTREAM_COMMIT_ISH"
echo "(For fast processing, consider establishing a local git tree"\
"at ${LOCAL_REPO_MAP[0]})"
# NYI - should be able to combine with local case for the most part
fi
QEMU_VERSION=$(cat $GIT_DIR/VERSION)
SOURCE_VERSION=$OLD_SOURCE_VERSION_AND_EXTRA
NEW_COMMIT_ISH=
WRITE_LOG=1
fi
rm -rf $GIT_DIR
rm -rf $BUNDLE_DIR
(
CHANGED_COUNT=0
UNCHANGED_COUNT=0
DELETED_COUNT=0
ADDED_COUNT=0
TOTAL_COUNT=0
shopt -s nullglob
for i in $CMP_DIR/*; do
# index line isn't consistent, so cut full index to normal line length
sed -E -i 's/(^index [a-f0-9]{28})[a-f0-9]{12}([.][.][a-f0-9]{28})[a-f0-9]{12}( [0-9]{6}$)/\1\2\3/' $i
BASENAME=$(basename $i)
if [ "$FIVE_DIGIT_POTENTIAL" = "1" ]; then
if [[ $BASENAME =~ [[:digit:]]{4}-.* ]]; then
BASENAME=0$BASENAME
fi
fi
if [[ "$NUMBERED_PATCHES" = "0" ]]; then
KEEP_COUNT=40+4+$FIVE_DIGIT_POTENTIAL+1
else
KEEP_COUNT=40
fi
tail -n +2 $i > $CMP_DIR/${BASENAME:0:$KEEP_COUNT}.patch
rm $i
done
if [[ "$NUMBERED_PATCHES" = "0" ]]; then
for i in [0-9]*.patch; do
osc rm --force $i
echo "calling osc rm on $i"
done
# we need to make sure that w/out the numbered prefixes, the patchnames are all unique
mkdir checkdir
for i in $CMP_DIR/*; do
BASENAME=$(basename $i)
FINALNAME=${BASENAME:4+$FIVE_DIGIT_POTENTIAL+1:40+1+5}
if [[ -e checkdir/$FINALNAME ]]; then
echo "ERROR! Patch name $FINALNAME is not unique! Please modify patch subject to achieve uniqueness"
exit 1
fi
cp $i checkdir/$FINALNAME
done
CHECK_DIR=checkdir
cp $CMP_DIR/*.patch .
else
CHECK_DIR=$CMP_DIR
fi
#step 0, and 0.1 are done above - question remains if the numbered case should use check dir
rm -f checkthese
if [ "$FIVE_DIGIT_POTENTIAL" = "0" ]; then
CHECK_PREFIX="0"
else
CHECK_PREFIX="00"
fi
for i in $CHECK_DIR/*; do
BASENAME=$(basename $i)
if [ -e $BASENAME ]; then
if cmp -s $i $BASENAME; then
touch --reference=$BASENAME $i
rm $BASENAME
let UNCHANGED_COUNT+=1
else
if [ "${BASENAME:0:1+$FIVE_DIGIT_POTENTIAL}" = "$CHECK_PREFIX" ]; then
echo "$BASENAME" >> checkthese
fi
rm $BASENAME
let CHANGED_COUNT+=1
let TOTAL_COUNT+=1
fi
else
echo " $BASENAME" >> qemu.changes.added
if [ "${BASENAME:0:1+$FIVE_DIGIT_POTENTIAL}" = "$CHECK_PREFIX" ]; then
echo "$BASENAME" >> checkthese
fi
let ADDED_COUNT+=1
let TOTAL_COUNT+=1
fi
done
if [ "$FIVE_DIGIT_POTENTIAL" = "0" ]; then
NUMBERED_PATCH_RE="^[[:digit:]]{4}-.*[.]patch$"
else
NUMBERED_PATCH_RE="^[[:digit:]]{5}-.*[.]patch$"
fi
#NEXT is #2 in lgorithm
for i in *.patch; do
if [[ $i =~ $NUMBERED_PATCH_RE ]]; then
if [[ "$NUMBERED_PATCHES" = "1" ]]; then
osc rm --force $i
echo "calling osc rm on $i"
echo " $i" >> qemu.changes.deleted
let DELETED_COUNT+=1
let TOTAL_COUNT+=1
fi
else
osc rm --force $i
echo "calling osc rm on $i"
echo " $i" >> qemu.changes.deleted
let DELETED_COUNT+=1
let TOTAL_COUNT+=1
fi
done
mv $CHECK_DIR/* .
if [ -e qemu.changes.added ]; then
xargs osc add < qemu.changes.added
echo "calling osc add on:"; cat qemu.changes.added
fi
# NYI do we need this check?
if [ ! -e checkpatch.pl ]; then
tar Jxf qemu-$SOURCE_VERSION$VERSION_EXTRA.tar.xz \
qemu-$SOURCE_VERSION/scripts/checkpatch.pl --strip-components=2
fi
if [[ -e checkthese ]]; then
for i in $(cat checkthese); do
./checkpatch.pl --no-tree --terse --no-summary --summary-file \
--patch $i >> checkpatch.log || true
done
fi
rm -f checkthese
rm -f checkpatch.pl
if [ -s checkpatch.log ]; then
echo "WARNING: Issues reported by qemu patch checker. Please handle" \
"ERROR items now:"
cat checkpatch.log
fi
rm -f checkpatch.log
if [ "$TOTAL_COUNT" != "0" -a "$VERSION_EXTRA" != "" -a "$OLD_COMMIT_ISH" =\
"$NEW_COMMIT_ISH" ]; then
# Patches changed, so update the version using current time
VERSION_EXTRA=+git.$NOW_SECONDS.$OLD_COMMIT_ISH
osc mv qemu-$OLD_SOURCE_VERSION_AND_EXTRA.tar.xz \
qemu-$SOURCE_VERSION$VERSION_EXTRA.tar.xz
osc add qemu-$SOURCE_VERSION$VERSION_EXTRA.tar.xz
fi
echo "QEMU version file: $QEMU_VERSION"
echo "QEMU source version: $SOURCE_VERSION"
echo "QEMU version extra: $VERSION_EXTRA"
SEABIOS_VERSION=$(tar JxfO qemu-$SOURCE_VERSION$VERSION_EXTRA.tar.xz \
qemu-$SOURCE_VERSION/roms/seabios/.version | cut -d '-' -f 2)
for package in qemu; do
while IFS= read -r line; do
if [ "$line" = "PATCH_FILES" ]; then
for i in [0-9]*-*.patch; do
NUM=${i%%-*}
DIV=$((10#$NUM/$PATCH_RANGE))
REM=$((10#$NUM%$PATCH_RANGE))
if [[ "$REM" = "0" ]]; then
if [[ "$DIV" = "0" ]]; then
echo "# Patches applied in base project:"
else
echo "# Patches applied in ${PATCH_PATH_MAP[$DIV]}:"
fi
fi
if [[ "$FIVE_DIGIT_POTENTIAL" != "0" ]]; then
if [[ "$NUMBERED_PATCHES" = "0" ]]; then
PATCH_NUMBER=${i%%-*}
echo -e "Patch$NUM: ${i:${#PATCH_NUMBER}+1:40+1+5}"
else
echo -e "Patch$NUM: $i"
fi
else
if [[ "$NUMBERED_PATCHES" = "0" ]]; then
PATCH_NUMBER=${i%%-*}
echo -e "Patch$NUM: ${i:${#PATCH_NUMBER}+1:40+1+5}"
else
echo -e "Patch$NUM: $i"
fi
fi
done
elif [ "$line" = "PATCH_EXEC" ]; then
for i in [0-9]*-*.patch; do
S=$(grep "^Include-If: " $i) || true
NUM=${i%%-*}
if [ "$S" != "" ]; then
echo "${S:12}"
echo "%patch$NUM -p1"
echo "%endif"
else
echo "%patch$NUM -p1"
fi
done
elif [ "$line" = "QEMU_VERSION" ]; then
echo "%define qemuver $QEMU_VERSION$VERSION_EXTRA"
echo "%define srcver $SOURCE_VERSION$VERSION_EXTRA"
echo "Version: %qemuver"
elif [[ "$line" =~ ^Source: ]]; then
echo "$line"
if [ ${#QEMU_TARBALL_SIG[@]} -eq 1 ]; then
# We assume the signature file corresponds - just add .sig
echo "$line.sig"|sed 's/^Source: /Source99:/'
fi
elif [ "$line" = "SEABIOS_VERSION" ]; then
echo "Version: $SEABIOS_VERSION"
else
echo "$line"
fi
done < $package.spec.in > $CMP_DIR/$package.spec
if cmp -s $package.spec $CMP_DIR/$package.spec; then
echo "$package.spec unchanged"
else
mv $CMP_DIR/$package.spec $package.spec
echo "$package.spec regenerated"
let PACKAGE_CHANGED_COUNT+=1
fi
if [ "$WRITE_LOG" = "1" ]; then
# Factory requires all deleted and added patches to be mentioned
if [ -e qemu.changes.deleted ] || [ -e qemu.changes.added ]; then
echo "Patch queue updated from ${GIT_TREE} ${GIT_BRANCH}" > \
$package.changes.proposed
fi
if [ -e qemu.changes.deleted ]; then
echo "* Patches dropped:" >> $package.changes.proposed
cat qemu.changes.deleted >> $package.changes.proposed
fi
if [ -e qemu.changes.added ]; then
echo "* Patches added:" >> $package.changes.proposed
cat qemu.changes.added >> $package.changes.proposed
fi
if [ -e $package.changes.proposed ]; then
osc vc --file=$package.changes.proposed $package
rm -f $package.changes.proposed
fi
fi
done
if [[ "$NUMBERED_PATCHES" = "0" ]]; then
rm -f [0-9]*-*.patch
fi
if [ -e qemu.changes.deleted ]; then
rm -f qemu.changes.deleted
fi
if [ -e qemu.changes.added ]; then
rm -f qemu.changes.added
fi
echo "git patch summary"
echo " unchanged: $UNCHANGED_COUNT"
echo " changed: $CHANGED_COUNT"
echo " deleted: $DELETED_COUNT"
echo " added: $ADDED_COUNT"
)
rm -rf $CMP_DIR
rm -rf checkdir
osc service localrun format_spec_file