SHA256
1
0
forked from pool/qemu

Accepting request 875464 from home:bfrogers:branches:Virtualization

- Added a few more usability improvements for our git packaging
  workflow

OBS-URL: https://build.opensuse.org/request/show/875464
OBS-URL: https://build.opensuse.org/package/show/Virtualization/qemu?expand=0&rev=620
This commit is contained in:
Bruce Rogers 2021-02-26 15:34:24 +00:00 committed by Git OBS Bridge
parent 698c429494
commit ba2fece093
4 changed files with 341 additions and 216 deletions

View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:4c62b4f7cb23a896ae882f3ce7871a36a65d2c36850932fdeaf1df1c5a8c4988
size 60624
oid sha256:534e2f464e862407578fc3bc358f589e3f4dd2f5fda8fc89b357ea9cb376bc0d
size 60640

View File

@ -4,6 +4,14 @@
# The next few VARIABLES are to be edited as required:
# Here is the git repo which tracks a separate upstream git based project
# We take this approach so we can have our own tags and branches, and store
# the patches in git for others to access outside of the bundle.
PACKAGE_MAIN_GIT_REPO=https://github.com/openSUSE/qemu.git
# This is the upstream for the PACKAGE_MAIN_GIT_REPO
UPSTREAM_GIT_REPO=https://gitlab.com/qemu-project/qemu.git
# The following specifies the upstream tag or commit upon which our patchqueue
# gets rebased. The special value LATEST may be used to "automatically" track
# the upstream development tree in the master branch
@ -28,9 +36,8 @@ BUNDLE_DIR=/dev/shm/qemu-factory-bundle-dir
NUMBERED_PATCHES=0
PATCH_RANGE=1000
REPO_COUNT=36
# This tracks the git submodule path within the superproject (1st entry)
# This array tracks all git submodule paths within the superproject (1st entry)
PATCH_PATH_MAP=(
""
"roms/seabios/"
@ -70,11 +77,7 @@ PATCH_PATH_MAP=(
"roms/edk2/MdeModulePkg/Library/BrotliCompress/brotli/research/libdivsufsort/"
)
# Perhaps we need to instead use the terminal local dirname as the index
# and store the ~/git/ as a separate VARIABLE
# This way, we only have one big array instead of two
# BUT STILL WE NEED TO START WITH THE DATA STORED SOMEWHERE!!!!!!
# (order must correspond to PATCH_PATH_MAP)
# (order and count must correspond to PATCH_PATH_MAP)
LOCAL_REPO_MAP=(
~/git/qemu-opensuse
~/git/qemu-seabios
@ -113,8 +116,3 @@ LOCAL_REPO_MAP=(
~/git/qemu-edk2-BrotliCompress-brotli-research-esaxx
~/git/qemu-edk2-BrotliCompress-brotli-research-libdivsufsort
)
# TEMPORARY! FOR NOW WE REQUIRE THESE LOCALLY TO DO WORK ON PACKAGE
REQUIRED_LOCAL_REPO_MAP=(
~/git/qemu-opensuse
)

View File

@ -1,3 +1,9 @@
-------------------------------------------------------------------
Fri Feb 26 15:31:46 UTC 2021 - Bruce Rogers <brogers@suse.com>
- Added a few more usability improvements for our git packaging
workflow
-------------------------------------------------------------------
Fri Feb 19 15:10:56 UTC 2021 - Bruce Rogers <brogers@suse.com>

View File

@ -12,10 +12,10 @@
# after ensuring current status of local repo is clean, incl submodules, we checkout master+submodules, then also ensure they're clean, then checkout the commit or tag corresponding to latest / stable-release + submodules (but don't bother to verift that's clean) - so this is a detached HEAD for stable-release and IS master (almost certainly) for LATEST. WOW - is that what we need to be doing!?!?! At least it seems to be working for the cases I've seen!!!!
# initbundle operates from the current checked out state of the local superproject, to get the submodule ids - CORRECT!!!!
# the LATEST processing of cloning the local repo, clones master - but perhaps it doesn't matter? because it adds upstream as a remote and probably gets most things from there? INVESTIGATE!!!!!!!!!!!!!!!!!!!1
# the LATEST processing of cloning the local repo, clones master - but perhaps it doesn't matter? because it adds upstream as a remote and probably gets most things from there? INVESTIGATE!!!!!!!!!!!!!!!!!!!
# bundle2local checks out master in local repo to ensure we're off the frombundle branch (doesn't seem needed the way the script currently is). It fetches the bundle's head (FETCH_HEAD) REQUIRING that the base commit be present (which it seems to be. Then it creates the frombundle branch, with the current FETCH_HEAD (SAME AS IN BUNDLE, RIGHT?)
# The LATEST's rebase loop checks out the frombundle branch (with force), so we are now OFF of the "correct checkout" that happened at the beginning, it DELETES the GIT_BRANCH (so in this case it DIDN'T MATTER WHAT WAS THERE WHEN SCRIPT STARTED !!!!!!! WARNING !!!!!, branches off of the frombundle branch (w/checkout), then rebases that (which came from bundle) onto the current superproject, or submodule commit id.
# At this point, if the GIT_BRANCH rebased ok, it's ready for making a patchqueue, out of, if the rebase failed, it's time to manually fix that.!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!1
# At this point, if the GIT_BRANCH rebased ok, it's ready for making a patchqueue, out of, if the rebase failed, it's time to manually fix that.!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# LATEST processing implies updated upstream/master IS the right thing
# HOW do we protect against a bad bundle being created (we do have the build service's tracking of previous files - is that sufficient?
@ -34,6 +34,12 @@ source ./config.sh
# TODO: Here we should validate the variables that should be set in config.sh
REPO_COUNT=${#PATCH_PATH_MAP[@]}
if [[ "$REPO_COUNT" != "${#LOCAL_REPO_MAP[@]}" ]]; then
echo "PATCH_PATH_MAP and LOCAL_REPO_MAP array sizes do not agree - please fix"
exit
fi
# Zero based numbering, so we subtract 1 here:
if (( (REPO_COUNT * PATCH_RANGE) - 1 > 9999 )); then
FIVE_DIGIT_POTENTIAL=1
@ -72,28 +78,6 @@ fi
#==============================================================================
TEMP_CHECK() {
# Validate that all the local repos that we currently have patches in are available
# TEMPORARY REQUIREMENT!
for entry in ${REQUIRED_LOCAL_REPO_MAP[@]}; do
if [[ -e $(readlink -f ${entry}) ]]; then
if $(git -C $entry branch| grep -F "$GIT_BRANCH" >/dev/null); then
:
else
echo "Didn't find the $GIT_BRANCH branch in repo at $entry"
exit
fi
else
echo "ERROR! For now, you need to have these local git repos available:"
echo ${REQUIRED_LOCAL_REPO_MAP[@]}
exit
fi
done
}
#==============================================================================
initbundle() {
# The bundle tarball has git bundles stored in a directory structure which mimics the
# submodule locations in the containing git repo. Also at that same dir level
@ -102,31 +86,46 @@ initbundle() {
# "{path/}{git_sha}.{bundle}", where {path/} isn't present for
# the top (qemu) bundle (ie it's for submodules).
SUBMODULE_COMMIT_IDS=($(git -C ${LOCAL_REPO_MAP[0]} submodule status --recursive|awk '{print $1}'))
SUBMODULE_DIRS=($(git -C ${LOCAL_REPO_MAP[0]} submodule status --recursive|awk '{print $2}'))
SUBMODULE_COUNT=${#SUBMODULE_COMMIT_IDS[@]}
# TODO: do this with simply math - ie: use (( ... ))
if [[ "$REPO_COUNT" != "$(expr $SUBMODULE_COUNT + 1)" ]]; then
echo "ERROR: submodule count doesn't match the REPO_COUNT variable in config.sh file!"
exit
fi
rm -rf $GIT_DIR
rm -rf $BUNDLE_DIR
mkdir -p $BUNDLE_DIR
for (( i=0; i <$SUBMODULE_COUNT; i++ )); do
mkdir -p $BUNDLE_DIR/${SUBMODULE_DIRS[$i]}
# what should this file be? for now use an extension of id
touch $BUNDLE_DIR/${SUBMODULE_DIRS[$i]}/${SUBMODULE_COMMIT_IDS[$i]}.id
done
if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then
GIT_UPSTREAM_COMMIT=$NEW_COMMIT_ISH_FULL
if [[ -e ${LOCAL_REPO_MAP[$i]}/.git/shallow ]]; then
if [[ -e bundles.tar.xz ]]; then
tar --extract --xz -f bundles.tar.xz -C $BUNDLE_DIR .
else
echo "ERROR: Superproject at ${LOCAL_REPO_MAP[$i]} is shallow (so we assume submodules aren't"
echo "recursively checked out), and there is not an existing bundle-of-bundles file, so we cannot"
echo "correctly initialize the packages bundle-of-bundles. Please fully initilize git superproject"
echo "before doing initbundle"
exit
fi
else
# TODO: make this smarter, or change something - works for tag, but not normal commit?
GIT_UPSTREAM_COMMIT=$(git -C ${LOCAL_REPO_MAP[0]} show-ref -d $GIT_UPSTREAM_COMMIT_ISH|grep -F "^{}"|awk '{print $1}')
if [[ -e bundles.tar.xz ]]; then
tar --extract --xz -f bundles.tar.xz -C $BUNDLE_DIR .
else
SUBMODULE_COMMIT_IDS=($(git -C ${LOCAL_REPO_MAP[0]} submodule status --recursive| cut -c 2- | awk '{print $1}'))
SUBMODULE_DIRS=($(git -C ${LOCAL_REPO_MAP[0]} submodule status --recursive| cut -c 2- |awk '{print $2}'))
SUBMODULE_COUNT=${#SUBMODULE_COMMIT_IDS[@]}
# TODO: do this with simply math - ie: use (( ... ))
if [[ "$REPO_COUNT" != "$(expr $SUBMODULE_COUNT + 1)" ]]; then
echo "ERROR: submodule count doesn't match what's in config.sh"
exit
fi
for (( i=0; i <$SUBMODULE_COUNT; i++ )); do
mkdir -p $BUNDLE_DIR/${SUBMODULE_DIRS[$i]}
touch $BUNDLE_DIR/${SUBMODULE_DIRS[$i]}/${SUBMODULE_COMMIT_IDS[$i]}.id
done
if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then
GIT_UPSTREAM_COMMIT=$NEW_COMMIT_ISH_FULL
else
# TODO: make this smarter, or change something - works for tag, but not normal commit?
GIT_UPSTREAM_COMMIT=$(git -C ${LOCAL_REPO_MAP[0]} show-ref -d $GIT_UPSTREAM_COMMIT_ISH|grep -F "^{}"|awk '{print $1}')
fi
touch $BUNDLE_DIR/$GIT_UPSTREAM_COMMIT.id
fi
fi
touch $BUNDLE_DIR/$GIT_UPSTREAM_COMMIT.id
# Now go through all the submodule local repos that are present and create a bundle file for the patches found there
# Now go through all the submodule local repos that are present with a $GIT_BRANCH and create a bundle file for the patches found
for (( i=0; i <$REPO_COUNT; i++ )); do
if [[ -e $(readlink -f ${LOCAL_REPO_MAP[$i]}) ]]; then
SUBDIR=${PATCH_PATH_MAP[$i]}
@ -140,7 +139,7 @@ for (( i=0; i <$REPO_COUNT; i++ )); do
git -C $GIT_DIR/$SUBDIR remote add origin file://$(readlink -f \
${LOCAL_REPO_MAP[$PATCH_RANGE_INDEX]})
if [[ $(git -C $GIT_DIR/$SUBDIR ls-remote --heads origin $GIT_BRANCH) ]]; then
git -C $GIT_DIR/$SUBDIR fetch origin $GIT_BRANCH
git -C $GIT_DIR/$SUBDIR fetch --update-shallow origin $GIT_BRANCH
if [[ $(git -C $GIT_DIR/$SUBDIR rev-list $GITREPO_COMMIT_ISH..FETCH_HEAD) ]]; then
git -C $GIT_DIR/$SUBDIR bundle create $BUNDLE_DIR/$SUBDIR$GITREPO_COMMIT_ISH.bundle $GITREPO_COMMIT_ISH..FETCH_HEAD
#TODO: post-process repo info to avoid un-needed diffs (eg git vs https)
@ -150,8 +149,9 @@ for (( i=0; i <$REPO_COUNT; i++ )); do
fi
fi
done
# keep diffs to a minimum - touch bundle files to "something common"
# parameters chosen to allow bundle tarball exact reproducibility
tar --format gnu --xz \
--sort=name \
--numeric-owner \
--owner=0 \
--group=0 \
@ -204,8 +204,7 @@ for entry in ${ID_FILES[@]}; do
else
if [ -e $BUNDLE_DIR/$SUBDIR/$GITREPO_COMMIT_ISH.bundle ]; then
# TODO: We should be able to handle this case with some more coding, but for now...
echo "No local repo $LOCAL_REPO available to process git bundle! Please create one"
exit
echo "No local repo $LOCAL_REPO available to process git bundle! Moving on..."
fi
fi
done
@ -225,7 +224,7 @@ rm -rf $GIT_DIR
git clone -ls ${LOCAL_REPO_MAP[0]} $GIT_DIR -b master --single-branch &>/dev/null
echo "Please wait..."
(cd $GIT_DIR && git remote add upstream \
git://git.qemu-project.org/qemu.git &>/dev/null)
$UPSTREAM_GIT_REPO &>/dev/null)
(cd $GIT_DIR && git remote update upstream &>/dev/null)
(cd $GIT_DIR && git checkout $NEW_COMMIT_ISH &>/dev/null)
# As an alternative, we could add a --recurse-submodules to the checkout instead here as well, right?
@ -322,10 +321,69 @@ bundle2spec() {
rm -f checkpatch.log
rm -f checkthese
rm -rf checkdir
rm -rf savedir
rm -rf $GIT_DIR
rm -rf $CMP_DIR
rm -rf $BUNDLE_DIR
mkdir -p $BUNDLE_DIR
mkdir savedir
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# Handle case of casual user missing local repos.
# To support this, we must grok the spec file's
# list of patches for "reuse" in case we don't
# have local repo to use for extracting bundle
# contents. (here we assume the existing bundle
# would then still correspond to the patches
# listed in spec file for that repo)
# WARNING:
# The following groking expects the patch section
# to be as this script lays it out, not modified!
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
declare -a PATCHES_BY_SUBMODULE_PATH
IN_PATCH_SECTION=0
INDEX=$REPO_COUNT # "invalid" since zero based index of objects < one based count of objects
while IFS= read -r line; do
if [[ "$line" = "# Patches applied in base project:" ]]; then
IN_PATCH_SECTION=1
INDEX=0 # base project is 0 by definition
continue
fi
if [[ "$line" =~ ^..Patches.applied.in.(.*)/:$ ]]; then
REPO="${BASH_REMATCH[1]}/"
IN_PATCH_SECTION=1
for (( i=0; i <$REPO_COUNT; i++ )); do
if [[ "$REPO" = "${PATCH_PATH_MAP[$i]}" ]]; then
if [[ "$INDEX" != "$REPO_COUNT" ]]; then
PATCHES_BY_SUBMODULE_PATH[$INDEX]=$ACCUMULATED_PATCHES
unset ACCUMULATED_PATCHES
fi
INDEX=$i
break
fi
done
if [[ "$INDEX" = "$REPO_COUNT" ]]; then
echo "ERROR: Failure groking spec file for patches!"
exit
fi
continue
fi
if [[ "$IN_PATCH_SECTION" = "0" ]]; then
continue
fi
if [[ "$line" =~ ^$ ]]; then
PATCHES_BY_SUBMODULE_PATH[$INDEX]=$ACCUMULATED_PATCHES
break;
fi
if [[ "$line" =~ ^Patch[0-9]*:[\ ]*(.*)$ ]]; then
PATCH="${BASH_REMATCH[1]}"
#echo "Patch is $PATCH"
ACCUMULATED_PATCHES="$ACCUMULATED_PATCHES $PATCH"
continue
fi
echo "ERROR: Failure groking spec file for patches!"
exit
done < qemu.spec
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# CONVERT BUNDLES INTO COMMITS AND FILL SPEC FILE
@ -349,19 +407,38 @@ for entry in ${BUNDLE_FILES[@]}; do
fi
done
mkdir -p $GIT_DIR/$SUBDIR
git -C $GIT_DIR/$SUBDIR init
git -C $GIT_DIR/$SUBDIR remote add origin file://$(readlink -f \
${LOCAL_REPO_MAP[$PATCH_RANGE_INDEX]})
git -C $GIT_DIR/$SUBDIR fetch origin $GIT_BRANCH
git -C $GIT_DIR/$SUBDIR reset --hard $GITREPO_COMMIT_ISH
git -C $GIT_DIR/$SUBDIR remote add bundle $BUNDLE_DIR/$entry
git -C $GIT_DIR/$SUBDIR fetch bundle FETCH_HEAD
git -C $GIT_DIR/$SUBDIR format-patch -N --suffix= --no-renames -o $CMP_DIR -k --stat=72 \
--indent-heuristic --zero-commit --no-signature --full-index \
--src-prefix=a/$SUBDIR --dst-prefix=b/$SUBDIR \
--start-number=$(expr $PATCH_RANGE_INDEX \* $PATCH_RANGE) \
$GITREPO_COMMIT_ISH..FETCH_HEAD > /dev/null
if [[ -e $(readlink -f ${LOCAL_REPO_MAP[$PATCH_RANGE_INDEX]}) ]]; then
mkdir -p $GIT_DIR/$SUBDIR
git -C $GIT_DIR/$SUBDIR init
git -C $GIT_DIR/$SUBDIR remote add origin file://$(readlink -f \
${LOCAL_REPO_MAP[$PATCH_RANGE_INDEX]})
# This tag reference, was added to resolve $GITREPO_COMMIT_ISH, which is tag as commit-id
#needed? git -C $GIT_DIR/$SUBDIR fetch --depth=1 origin $GIT_UPSTREAM_COMMIT_ISH
# Since origin may be shallow, we need to use the --update-shallow option
git -C $GIT_DIR/$SUBDIR fetch --update-shallow origin $GIT_BRANCH
#needed? git -C $GIT_DIR/$SUBDIR reset --hard $GITREPO_COMMIT_ISH
git -C $GIT_DIR/$SUBDIR remote add bundle $BUNDLE_DIR/$entry
git -C $GIT_DIR/$SUBDIR fetch bundle FETCH_HEAD
git -C $GIT_DIR/$SUBDIR format-patch -N --suffix= --no-renames -o $CMP_DIR -k --stat=72 \
--indent-heuristic --zero-commit --no-signature --full-index \
--src-prefix=a/$SUBDIR --dst-prefix=b/$SUBDIR \
--start-number=$(expr $PATCH_RANGE_INDEX \* $PATCH_RANGE) \
$GITREPO_COMMIT_ISH..FETCH_HEAD > /dev/null
PATCHES_BY_SUBMODULE_PATH[$PATCH_RANGE_INDEX]=""
else
# TODO: This doesn't handle numbered patches yet
COUNTER=$(expr $PATCH_RANGE_INDEX \* $PATCH_RANGE)
for patchname in ${PATCHES_BY_SUBMODULE_PATH[$PATCH_RANGE_INDEX]}; do
VALUE="0000"$COUNTER
if [ "$FIVE_DIGIT_POTENTIAL" = "1" ]; then
PREFIX=$(echo $VALUE|tail -c 6)
else
PREFIX=$(echo $VALUE|tail -c 5)
fi
cp $patchname savedir/$PREFIX-$patchname
let COUNTER+=1
done
fi
done
rm -rf $GIT_DIR
@ -379,22 +456,26 @@ rm -rf $BUNDLE_DIR
for i in $CMP_DIR/*; do
# index line isn't consistent, so cut full index to normal line length
sed -E -i 's/(^index [a-f0-9]{28})[a-f0-9]{12}([.][.][a-f0-9]{28})[a-f0-9]{12}( [0-9]{6}$)/\1\2\3/' $i
BASENAME=$(basename $i)
BASENAME=$(basename $i)
if [ "$FIVE_DIGIT_POTENTIAL" = "1" ]; then
if [[ "$BASENAME" =~ ^[[:digit:]]{5}.* ]]; then
:
else
BASENAME=0"$BASENAME"
fi
fi
fi
if [[ "$NUMBERED_PATCHES" = "0" ]]; then
KEEP_COUNT=40+4+$FIVE_DIGIT_POTENTIAL+1
KEEP_COUNT=40+4+$FIVE_DIGIT_POTENTIAL+1
else
KEEP_COUNT=40
fi
KEEP_COUNT=40
fi
tail -n +2 $i > $CMP_DIR/"${BASENAME:0:$KEEP_COUNT}".patch
rm $i
rm $i
done
cp savedir/* $CMP_DIR || true
rm -rf savedir
# 4 digit xxxx-name used in the dest (remember that if 5 digit potential, then if not now 5 digit, add a 0 in front)
if [[ "$NUMBERED_PATCHES" = "0" ]]; then
for i in [0-9][0-9][0-9][0-9]*-*.patch; do
osc rm --force "$i"
@ -403,15 +484,15 @@ rm -rf $BUNDLE_DIR
mkdir checkdir
for i in $CMP_DIR/*; do
BASENAME=$(basename $i)
FINALNAME="${BASENAME:4+$FIVE_DIGIT_POTENTIAL+1:40+1+5}"
if [[ -e checkdir/"$FINALNAME" ]]; then
echo "ERROR! Patch name $FINALNAME is not unique! Please modify patch subject to achieve uniqueness"
exit 1
FINALNAME="${BASENAME:4+$FIVE_DIGIT_POTENTIAL+1:40+1+5}"
if [[ -e checkdir/"$FINALNAME" ]]; then
echo "ERROR! Patch name $FINALNAME is not unique! Please modify patch subject to achieve uniqueness"
exit 1
fi
cp $i checkdir/"$FINALNAME"
cp $i checkdir/"$FINALNAME"
done
CHECK_DIR=checkdir
cp $CMP_DIR/*.patch .
cp $CMP_DIR/*.patch .
else
CHECK_DIR=$CMP_DIR
fi
@ -431,7 +512,7 @@ rm -rf $BUNDLE_DIR
let UNCHANGED_COUNT+=1
else
if [ "${BASENAME:0:1+$FIVE_DIGIT_POTENTIAL}" = "$CHECK_PREFIX" ]; then
echo "$BASENAME" >> checkthese
echo "$BASENAME" >> checkthese
fi
rm $BASENAME
let CHANGED_COUNT+=1
@ -440,20 +521,20 @@ rm -rf $BUNDLE_DIR
else
echo " $BASENAME" >> qemu.changes.added
if [ "${BASENAME:0:1+$FIVE_DIGIT_POTENTIAL}" = "$CHECK_PREFIX" ]; then
echo "$BASENAME" >> checkthese
echo "$BASENAME" >> checkthese
fi
let ADDED_COUNT+=1
let TOTAL_COUNT+=1
fi
done
for i in *.patch; do
if [[ "$i" =~ $NUMBERED_PATCH_RE ]]; then
if [[ "$i" =~ $NUMBERED_PATCH_RE ]]; then
if [[ "$NUMBERED_PATCHES" = "1" ]]; then
osc rm --force $i
echo " $i" >> qemu.changes.deleted
let DELETED_COUNT+=1
let TOTAL_COUNT+=1
fi
fi
else
osc rm --force $i
echo " $i" >> qemu.changes.deleted
@ -506,8 +587,8 @@ rm -rf $BUNDLE_DIR
# TODO: do this better!
for i in [0-9][0-9][0-9][0-9]*-*.patch; do
NUM=${i%%-*}
DIV=$((10#$NUM/$PATCH_RANGE))
REM=$((10#$NUM%$PATCH_RANGE))
DIV=$((10#$NUM/$PATCH_RANGE))
REM=$((10#$NUM%$PATCH_RANGE))
if [[ "$REM" = "0" ]]; then
if [[ "$DIV" = "0" ]]; then
echo "# Patches applied in base project:"
@ -519,24 +600,24 @@ rm -rf $BUNDLE_DIR
if [[ "$NUMBERED_PATCHES" = "0" ]]; then
PATCH_NUMBER=${i%%-*}
echo -e "Patch$NUM: ${i:${#PATCH_NUMBER}+1:40+1+5}"
else
else
echo -e "Patch$NUM: $i"
fi
else
else
if [[ "$NUMBERED_PATCHES" = "0" ]]; then
PATCH_NUMBER=${i%%-*}
echo -e "Patch$NUM: ${i:${#PATCH_NUMBER}+1:40+1+5}"
else
else
echo -e "Patch$NUM: $i"
fi
fi
done
elif [ "$line" = "PATCH_EXEC" ]; then
for i in [0-9][0-9][0-9][0-9]*-*.patch; do
S=$(grep "^Include-If: " $i) || true
S=$(grep "^Include-If: " $i) || true
NUM=${i%%-*}
if [ "$S" != "" ]; then
echo "${S:12}"
echo "${S:12}"
echo "%patch$NUM -p1"
echo "%endif"
else
@ -606,113 +687,16 @@ rm -rf checkdir
osc service localrun format_spec_file
# First, make the results of the older format_spec_file look like what I believe is the intended output
# And then change the somewhat broken output of the new format_spec_file to what I believe is
# the intended output
# And then change THE POSSIBLY BROKEN OUTPUT from the new format_spec_file look like what I
# believe is the intended output
sed -i 's/^# spec file for package qemu$/# spec file for package qemu%{name_suffix}/g' qemu.spec
sed -i 's/^# spec file for package qemu-linux-user$/# spec file for package qemu%{name_suffix}/g' qemu.spec
}
#==============================================================================
usage() {
echo "Usage:"
echo "bash ./git_update.sh <command>"
echo "description: package maintenance using a git-based workflow. Commands:"
echo " git2pkg (update package spec file and patches from git. Is default)"
echo " pkg2git (update git (frombundle branch) from the package "bundleofbundles")"
echo " refresh (refresh spec file from spec file template and "bundlofbundles")"
echo "(See script for details on doing 'LATEST' workflow)"
}
#==============================================================================
explain_setup() {
echo "Currently we require local git repos at these locations:"
echo "${REQUIRED_LOCAL_REPO_MAP[@]}"
echo "Where each has as it's remote the uri: https://github.com/opensuse/*.git"
echo "and where * is replaced by the qemu-whatever, and the remote is named origin"
echo "and the qemu or qemu submodule repos as remotes named upstream"
}
#==============================================================================
if [[ ! -e $(readlink -f ${LOCAL_REPO_MAP[0]}) ]]; then
echo "ERROR: Local QEMU related git repos not found. Please follow these setup instructions:"
explain_setup
exit
fi
# There are some req's on needing a recent git, and a recent osc (double chk the osc part - I guess it's related to the osc service )
# get the current state of the git superproject
# TODO: This sends output to stdout which we don't want to see
git -C ${LOCAL_REPO_MAP[0]} status --untracked-files=no --branch --porcelain=2 \
| awk '{print "var"NR"="$3}'
# $var1 is the current commit
# $var2 is the current branch or 'detached', if not on a branch
# $var3 is the current upstream branch (if set), as in eg 'origin/opensuse-5.0'
# $var4 is not of use to us
# TODO: What checks should be different between LATEST and non-LATEST?
# If we don't actually patch from the submodule repo, we shouldn't care about what's in the local one
# Does non-LATEST really require master?
echo "WARNING: Script using local git repos. Some operations may be time consuming..."
#TODO: Most of these checks are not necessary
for (( i=0; i <$REPO_COUNT; i++ )); do
if [[ -e $(readlink -f ${LOCAL_REPO_MAP[$i]}) ]]; then
if [[ -d ${LOCAL_REPO_MAP[$i]}/.git/rebase-merge || \
-d ${LOCAL_REPO_MAP[$i]}/.git/rebase-apply ]]; then
echo "ERROR! Rebase appears to be in progress in ${LOCAL_REPO_MAP[$i]}. Please resolve"
exit
fi
# !! Does this presume the branch as indicated in config is the current branch? (I believe that's been my modus operandi to date, so perhaps THAT should be enforced at this point?)
if ! git -C ${LOCAL_REPO_MAP[$i]} submodule update --init --recursive &> /dev/null; then
echo "Please clean up state of local repo ${LOCAL_REPO_MAP[$i]} before using script"
echo "(ensure git submodule update --init --recursive is successful)"
exit
fi
if [ "$(git -C ${LOCAL_REPO_MAP[$i]} status --porcelain)" ]; then
echo "Please clean up state of local repo ${LOCAL_REPO_MAP[$i]} before using script"
echo "(ensure git status --porcelain produces no output)"
exit
fi
if ! git -C ${LOCAL_REPO_MAP[$i]} checkout master --recurse-submodules -f &> /dev/null; then
echo "Please clean up state of local repo ${LOCAL_REPO_MAP[$i]} before using script"
echo "(cannot check out master, incl. it's submodules)"
exit
fi
if ! git -C ${LOCAL_REPO_MAP[$i]} submodule update --init --recursive &> /dev/null; then
echo "Please clean up state of local repo ${LOCAL_REPO_MAP[$i]} before using script"
echo "(cannot init and update master submodules)"
exit
fi
if [ "$(git -C ${LOCAL_REPO_MAP[$i]} status --porcelain)" ]; then
echo "Please clean up state of local repo ${LOCAL_REPO_MAP[$i]} before using script"
echo "(ensure git status --porcelain produces no output)"
exit
fi
fi
done
setup_common_vars() {
if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then
if [ "$1" = "continue" ]; then
CONTINUE_AFTER_REBASE=1
else
if [ "$1" = "pause" ]; then
PAUSE_BEFORE_BUNDLE_CREATION=1
else
if [ "$1" ]; then
echo "ERROR: unrecognized option '$1'. Script in LATEST mode only recognizes 'pause' and 'continue' options"
exit
fi
fi
fi
for (( i=0; i <$REPO_COUNT; i++ )); do
if [[ -e $(readlink -f ${LOCAL_REPO_MAP[$i]}) ]]; then
git -C ${LOCAL_REPO_MAP[$i]} remote update upstream &> /dev/null
fi
done
NEW_COMMIT_ISH_FULL=$(cd ${LOCAL_REPO_MAP[0]} && git rev-parse upstream/master)
NEW_COMMIT_ISH=${NEW_COMMIT_ISH_FULL:0:8}
git -C ${LOCAL_REPO_MAP[0]} checkout $NEW_COMMIT_ISH_FULL --recurse-submodules -f &> /dev/null
QEMU_VERSION=$(git -C ${LOCAL_REPO_MAP[0]} show upstream/master:VERSION)
MAJOR_VERSION=$(echo $QEMU_VERSION|awk -F. '{print $1}')
MINOR_VERSION=$(echo $QEMU_VERSION|awk -F. '{print $2}')
@ -738,14 +722,159 @@ if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then
fi
fi
fi
else
SOURCE_VERSION=$OLD_SOURCE_VERSION_AND_EXTRA
QEMU_VERSION=$(tar JxfO qemu-$SOURCE_VERSION$VERSION_EXTRA.tar.xz qemu-$SOURCE_VERSION/VERSION)
if [ ! "$QEMU_VERSION" = "$OLD_SOURCE_VERSION_AND_EXTRA" ]; then
echo "Tarball name (which we decode) doesn't correspond to the VERSION file contained therein"
exit
fi
MAJOR_VERSION=$(echo $QEMU_VERSION|awk -F. '{print $1}')
MINOR_VERSION=$(echo $QEMU_VERSION|awk -F. '{print $2}')
GIT_BRANCH=opensuse-$MAJOR_VERSION.$MINOR_VERSION
fi
}
#==============================================================================
usage() {
echo "Usage:"
echo "bash ./git_update.sh <command>"
echo "description: package maintenance using a git-based workflow. Commands:"
echo " git2pkg (update package spec file and patches from git. Is default)"
echo " pkg2git (update git (frombundle branch) from the package "bundleofbundles")"
echo " refresh (refresh spec file from spec file template and "bundlofbundles")"
echo "(See script for details on doing 'LATEST' workflow)"
}
#==============================================================================
if [[ ! -e $(readlink -f ${LOCAL_REPO_MAP[0]}) ]]; then
echo "No local repo found at ${LOCAL_REPO_MAP[0]}"
if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then
echo "Using LATEST config.sh setting is an expert mode. Set up local repos accordingly"
exit
fi
read -p "Would you like me to set that up for you? (Y/N)" -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
echo "Got an affirmative answer, proceeding..."
setup_common_vars
# TODO: The following doesn't really do what we need (we adjust later) FIX!!!
# git clone --depth 1 -b $GIT_BRANCH --single-branch $PACKAGE_MAIN_GIT_REPO ${LOCAL_REPO_MAP[0]}
git init ${LOCAL_REPO_MAP[0]}
git -C ${LOCAL_REPO_MAP[0]} remote add origin $PACKAGE_MAIN_GIT_REPO &>/dev/null
git -C ${LOCAL_REPO_MAP[0]} fetch origin +refs/tags/initial:refs/tags/initial --no-tags
git -C ${LOCAL_REPO_MAP[0]} reset --hard initial
GIT_UPSTREAM_COMMIT=$(git -C ${LOCAL_REPO_MAP[0]} ls-remote origin |grep -F "$GIT_UPSTREAM_COMMIT_ISH^{}"|awk '{print $1}')
# Here we use *COMMIT_ISH, not *_COMMIT - is that an issue?
git -C ${LOCAL_REPO_MAP[0]} fetch --depth=1 origin +refs/tags/$GIT_UPSTREAM_COMMIT_ISH:refs/tags/$GIT_UPSTREAM_COMMIT_ISH --no-tags
git -C ${LOCAL_REPO_MAP[0]} remote add upstream $UPSTREAM_GIT_REPO &>/dev/null
bundle2local
git -C ${LOCAL_REPO_MAP[0]} checkout -b $GIT_BRANCH frombundle
echo "We set up a shallow local repo of the package's git superproject at"
echo "${LOCAL_REPO_MAP[0]}, and initialized it from the bundle."
echo "(no options processed)"
echo "If you wish to make the repo complete for all qemu packaging work,"
echo "unshallow it first with git fetch --unshallow origin --all, then get"
echo "the submodules updated with git submodule update --init --recursive"
echo "Be aware that this downloads a LOT of data (that's why we didn't just"
echo "do that automatically. Then you may also fetch other branches from the"
echo "origin remote, and get the latest upstream patches from the upstream"
echo "remote. Refer to config.sh for submodule repos locations you can set"
echo "up to also work on patches for the superproject's submodules."
exit
else
echo "Script requires qemu superproject local git repo. Please provide in"
echo "order to use this script."
fi
exit
fi
# There are some req's on needing a recent git, and a recent osc (double chk the osc part - I guess it's related to the osc service )
# get the current state of the git superproject
# TODO: This sends output to stdout which we don't want to see
git -C ${LOCAL_REPO_MAP[0]} status --untracked-files=no --branch --porcelain=2 \
| awk '{print "var"NR"="$3}'
# $var1 is the current commit
# $var2 is the current branch or 'detached', if not on a branch
# $var3 is the current upstream branch (if set), as in eg 'origin/opensuse-5.0'
# $var4 is not of use to us
# TODO: What checks should be different between LATEST and non-LATEST?
# If we don't actually patch from the submodule repo, we shouldn't care about what's in the local one
# Does non-LATEST really require master?
echo "WARNING: Script using local git repos. Some operations may be time consuming..."
# TODO: Most of these checks are not necessary
for (( i=0; i <$REPO_COUNT; i++ )); do
if [[ -e $(readlink -f ${LOCAL_REPO_MAP[$i]}) ]]; then
if [[ -e ${LOCAL_REPO_MAP[$i]}/.git/shallow ]]; then
echo "${LOCAL_REPO_MAP[$i]} is shallow, skipping checks"
continue
fi
if [[ -d ${LOCAL_REPO_MAP[$i]}/.git/rebase-merge || \
-d ${LOCAL_REPO_MAP[$i]}/.git/rebase-apply ]]; then
echo "ERROR! Rebase appears to be in progress in ${LOCAL_REPO_MAP[$i]}. Please resolve"
exit
fi
# !! Does this presume the branch as indicated in config is the current branch? (I believe that's been my modus operandi to date, so perhaps THAT should be enforced at this point?)
if ! git -C ${LOCAL_REPO_MAP[$i]} submodule update --init --recursive &> /dev/null; then
echo "Please clean up state of local repo ${LOCAL_REPO_MAP[$i]} before using script"
echo "(ensure git submodule update --init --recursive is successful)"
exit
fi
if [ "$(git -C ${LOCAL_REPO_MAP[$i]} status --porcelain)" ]; then
echo "Please clean up state of local repo ${LOCAL_REPO_MAP[$i]} before using script"
echo "(ensure git status --porcelain produces no output)"
exit
fi
if ! git -C ${LOCAL_REPO_MAP[$i]} checkout master --recurse-submodules -f &> /dev/null; then
echo "Please clean up state of local repo ${LOCAL_REPO_MAP[$i]} before using script"
echo "(cannot check out master, incl. it's submodules)"
exit
fi
# This does additional setup now that we've possibly grabbed additional submodules
if ! git -C ${LOCAL_REPO_MAP[$i]} submodule update --init --recursive &> /dev/null; then
echo "Please clean up state of local repo ${LOCAL_REPO_MAP[$i]} before using script"
echo "(cannot init and update master submodules)"
exit
fi
if [ "$(git -C ${LOCAL_REPO_MAP[$i]} status --porcelain)" ]; then
echo "Please clean up state of local repo ${LOCAL_REPO_MAP[$i]} before using script"
echo "(ensure git status --porcelain produces no output)"
exit
fi
fi
done
if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then
if [ "$1" = "continue" ]; then
CONTINUE_AFTER_REBASE=1
else
if [ "$1" = "pause" ]; then
PAUSE_BEFORE_BUNDLE_CREATION=1
else
if [ "$1" ]; then
echo "ERROR: unrecognized option '$1'. Script in LATEST mode only recognizes 'pause' and 'continue' options"
exit
fi
fi
fi
for (( i=0; i <$REPO_COUNT; i++ )); do
if [[ -e $(readlink -f ${LOCAL_REPO_MAP[$i]}) ]]; then
git -C ${LOCAL_REPO_MAP[$i]} remote update upstream &> /dev/null
fi
done
NEW_COMMIT_ISH_FULL=$(cd ${LOCAL_REPO_MAP[0]} && git rev-parse upstream/master)
NEW_COMMIT_ISH=${NEW_COMMIT_ISH_FULL:0:8}
git -C ${LOCAL_REPO_MAP[0]} checkout $NEW_COMMIT_ISH_FULL --recurse-submodules -f &> /dev/null
setup_common_vars
WRITE_LOG=0
echo "Processing LATEST upstream changes"
echo "(If SUCCESS is not printed upon completion, see /tmp/latest.log for issues)"
TEMP_CHECK
if [[ $QEMU_TARBALL =~ $BASE_RE$EXTRA_RE$SUFFIX_RE ]]; then
OLD_COMMIT_ISH=${BASH_REMATCH[3]}
else
#Assume release (or release candidate) tarball with equivalent tag:
#Assume release (or release candidate) tarball with equivalent tag:
OLD_COMMIT_ISH=$(cd ${LOCAL_REPO_MAP[0]} && git rev-list --abbrev-commit \
--abbrev=8 -1 v$OLD_SOURCE_VERSION_AND_EXTRA)
fi
@ -757,7 +886,7 @@ if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then
if [ "$OLD_COMMIT_ISH" != "$NEW_COMMIT_ISH" ]; then
if [ "$CONTINUE_AFTER_REBASE" = "1" ]; then
echo "continue after rebase selected but tarball is out of date. Continuing not possible."
echo "If desired, save your rebase work (eg, branch $GIT_BRANCH), because otherwise it will"
echo "If desired, save your rebase work (eg, branch $GIT_BRANCH), because otherwise it will"
echo "be lost. Then run script again without the continue option"
exit
fi
@ -775,7 +904,7 @@ if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then
fi
if [[ "$PAUSE_BEFORE_BUNDLE_CREATION" = "1" ]]; then
echo "As requested, pausing before re-creating bundle of bundles for additional patch or specfile work"
echo "(using current 'ready to go' $GIT_BRANCH branch of local repos to produce patches.)"
echo "(using current 'ready to go' $GIT_BRANCH branch of local repos to produce patches.)"
echo "When changes are complete, finish the workflow by passing 'continue' to script"
exit
fi
@ -791,28 +920,22 @@ else # not LATEST
exit
fi
git -C ${LOCAL_REPO_MAP[0]} checkout $GIT_UPSTREAM_COMMIT_ISH --recurse-submodules -f &> /dev/null
setup_common_vars
NEW_COMMIT_ISH=
SOURCE_VERSION=$OLD_SOURCE_VERSION_AND_EXTRA
QEMU_VERSION=$(tar JxfO qemu-$SOURCE_VERSION$VERSION_EXTRA.tar.xz qemu-$SOURCE_VERSION/VERSION)
if [ ! "$QEMU_VERSION" = "$OLD_SOURCE_VERSION_AND_EXTRA" ]; then
echo "Tarball name (which we decode) doesn't correspond to the VERSION file contained therein"
exit
fi
MAJOR_VERSION=$(echo $QEMU_VERSION|awk -F. '{print $1}')
MINOR_VERSION=$(echo $QEMU_VERSION|awk -F. '{print $2}')
GIT_BRANCH=opensuse-$MAJOR_VERSION.$MINOR_VERSION
WRITE_LOG=1
if [ "$1" = "" ]; then
set -- git2pkg
fi
case $1 in
initbundle )
initbundle
echo "Updating the bundle using the $GIT_BRANCH branch of the local repos."
echo "(If SUCCESS is not printed upon completion, see /tmp/initbundle.log for issues)"
initbundle &> /tmp/initbundle.log
echo "SUCCESS"
;;
git2pkg )
echo "Updating the package using the $GIT_BRANCH branch of the local repos."
echo "(If SUCCESS is not printed upon completion, see /tmp/git2pkg.log for issues)"
TEMP_CHECK
initbundle &> /tmp/git2pkg.log
bundle2spec &>> /tmp/git2pkg.log
echo "SUCCESS"
@ -821,7 +944,6 @@ else # not LATEST
pkg2git )
echo "Exporting the package's git bundles to the local repo's frombundle branches..."
echo "(If SUCCESS is not printed upon completion, see /tmp/pkg2git.log for issues)"
TEMP_CHECK
bundle2local &> /tmp/pkg2git.log
echo "SUCCESS"
echo "To modify package patches, use the frombundle branch as the basis for updating"
@ -832,7 +954,6 @@ else # not LATEST
echo "Updating the spec file and patches from the spec file template and the bundle"
echo "of bundles (bundles.tar.xz)"
echo "(If SUCCESS is not printed upon completion, see /tmp/refresh.log for issues)"
TEMP_CHECK
bundle2spec &> /tmp/refresh.log
echo "SUCCESS"
tail -9 /tmp/refresh.log
@ -843,7 +964,7 @@ else # not LATEST
;;
help )
usage
;;
;;
esac
fi
exit