diff --git a/bundles.tar.xz b/bundles.tar.xz index f1f6be13..30a06382 100644 --- a/bundles.tar.xz +++ b/bundles.tar.xz @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:2769241dc2dbb205bd1e1fe5b54a5fc79b782c8ec0f1455a4649920e98a2247f -size 54868 +oid sha256:86afea4d0fcdfa578dd3b660ccbadd49e996057de08ca8078020cb5c10611f91 +size 56424 diff --git a/config.sh b/config.sh index 7aa48e12..fe40c7c1 100644 --- a/config.sh +++ b/config.sh @@ -4,19 +4,18 @@ # The next few VARIABLES may be edited (or uncommented) as required: -# The commit upon which our patchqueue gets rebased. The special value LATEST -# may be used to "automatically" track the upstream development tree in the -# master branch +# The following specifies the upstream tag or commit upon which our patchqueue +# gets rebased. The special value LATEST may be used to "automatically" track +# the upstream development tree in the master branch GIT_UPSTREAM_COMMIT_ISH=v4.1.0 -if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then - echo "Using LATEST upstream commit as base for tarball and patch queue" - GIT_BRANCH=master +if [[ "$GIT_UPSTREAM_COMMIT_ISH" != "LATEST" ]]; then + # This is the git branch used (otherwise it is computed) + GIT_BRANCH=opensuse-4.1 fi -# otherwise we specify the branch to use, eg: # WARNING: If transitioning from using LATEST to not, MANUALLY re-set the -# tarball present -GIT_BRANCH=opensuse-4.1 -# This is used for the automated development branch tracking +# tarball present. If transitioning TO LATEST, make sure that +# NEXT_RELEASE_IS_MAJOR is set correctly +# This is used to choose the version number when LATEST processing is active NEXT_RELEASE_IS_MAJOR=0 # The shared openSUSE specific git repo, on which $GIT_LOCAL_TREE is based diff --git a/coroutine-Add-qemu_co_mutex_assert_locke.patch b/coroutine-Add-qemu_co_mutex_assert_locke.patch new file mode 100644 index 00000000..6b9b25c1 --- /dev/null +++ b/coroutine-Add-qemu_co_mutex_assert_locke.patch @@ -0,0 +1,48 @@ +From: Kevin Wolf +Date: Thu, 24 Oct 2019 16:26:57 +0200 +Subject: coroutine: Add qemu_co_mutex_assert_locked() + +Git-commit: 944f3d5dd216fcd8cb007eddd4f82dced0a15b3d + +Some functions require that the caller holds a certain CoMutex for them +to operate correctly. Add a function so that they can assert the lock is +really held. + +Cc: qemu-stable@nongnu.org +Signed-off-by: Kevin Wolf +Tested-by: Michael Weiser +Reviewed-by: Michael Weiser +Reviewed-by: Vladimir Sementsov-Ogievskiy +Reviewed-by: Denis V. Lunev +Reviewed-by: Max Reitz +Signed-off-by: Bruce Rogers +--- + include/qemu/coroutine.h | 15 +++++++++++++++ + 1 file changed, 15 insertions(+) + +diff --git a/include/qemu/coroutine.h b/include/qemu/coroutine.h +index 9801e7f5a4978bf4a5a74d33aeb6..f4843b5f595eb0beb1ce2f3e4e39 100644 +--- a/include/qemu/coroutine.h ++++ b/include/qemu/coroutine.h +@@ -167,6 +167,21 @@ void coroutine_fn qemu_co_mutex_lock(CoMutex *mutex); + */ + void coroutine_fn qemu_co_mutex_unlock(CoMutex *mutex); + ++/** ++ * Assert that the current coroutine holds @mutex. ++ */ ++static inline coroutine_fn void qemu_co_mutex_assert_locked(CoMutex *mutex) ++{ ++ /* ++ * mutex->holder doesn't need any synchronisation if the assertion holds ++ * true because the mutex protects it. If it doesn't hold true, we still ++ * don't mind if another thread takes or releases mutex behind our back, ++ * because the condition will be false no matter whether we read NULL or ++ * the pointer for any other coroutine. ++ */ ++ assert(atomic_read(&mutex->locked) && ++ mutex->holder == qemu_coroutine_self()); ++} + + /** + * CoQueues are a mechanism to queue coroutines in order to continue executing diff --git a/qcow2-Fix-corruption-bug-in-qcow2_detect.patch b/qcow2-Fix-corruption-bug-in-qcow2_detect.patch new file mode 100644 index 00000000..2d965bbc --- /dev/null +++ b/qcow2-Fix-corruption-bug-in-qcow2_detect.patch @@ -0,0 +1,68 @@ +From: Kevin Wolf +Date: Thu, 24 Oct 2019 16:26:58 +0200 +Subject: qcow2: Fix corruption bug in qcow2_detect_metadata_preallocation() + +Git-commit: 5e9785505210e2477e590e61b1ab100d0ec22b01 + +qcow2_detect_metadata_preallocation() calls qcow2_get_refcount() which +requires s->lock to be taken to protect its accesses to the refcount +table and refcount blocks. However, nothing in this code path actually +took the lock. This could cause the same cache entry to be used by two +requests at the same time, for different tables at different offsets, +resulting in image corruption. + +As it would be preferable to base the detection on consistent data (even +though it's just heuristics), let's take the lock not only around the +qcow2_get_refcount() calls, but around the whole function. + +This patch takes the lock in qcow2_co_block_status() earlier and asserts +in qcow2_detect_metadata_preallocation() that we hold the lock. + +Fixes: 69f47505ee66afaa513305de0c1895a224e52c45 +Cc: qemu-stable@nongnu.org +Reported-by: Michael Weiser +Signed-off-by: Kevin Wolf +Tested-by: Michael Weiser +Reviewed-by: Michael Weiser +Reviewed-by: Vladimir Sementsov-Ogievskiy +Reviewed-by: Max Reitz +Signed-off-by: Bruce Rogers +--- + block/qcow2-refcount.c | 2 ++ + block/qcow2.c | 3 ++- + 2 files changed, 4 insertions(+), 1 deletion(-) + +diff --git a/block/qcow2-refcount.c b/block/qcow2-refcount.c +index ef965d78952637ed92d777db4e9a..0d64bf5a5e9c0359e5391185f6c5 100644 +--- a/block/qcow2-refcount.c ++++ b/block/qcow2-refcount.c +@@ -3455,6 +3455,8 @@ int qcow2_detect_metadata_preallocation(BlockDriverState *bs) + int64_t i, end_cluster, cluster_count = 0, threshold; + int64_t file_length, real_allocation, real_clusters; + ++ qemu_co_mutex_assert_locked(&s->lock); ++ + file_length = bdrv_getlength(bs->file->bs); + if (file_length < 0) { + return file_length; +diff --git a/block/qcow2.c b/block/qcow2.c +index 865839682cd639d1b7aba0cc328f..c0f5439dc8f10d7f920d9b4a29b1 100644 +--- a/block/qcow2.c ++++ b/block/qcow2.c +@@ -1899,6 +1899,8 @@ static int coroutine_fn qcow2_co_block_status(BlockDriverState *bs, + unsigned int bytes; + int status = 0; + ++ qemu_co_mutex_lock(&s->lock); ++ + if (!s->metadata_preallocation_checked) { + ret = qcow2_detect_metadata_preallocation(bs); + s->metadata_preallocation = (ret == 1); +@@ -1906,7 +1908,6 @@ static int coroutine_fn qcow2_co_block_status(BlockDriverState *bs, + } + + bytes = MIN(INT_MAX, count); +- qemu_co_mutex_lock(&s->lock); + ret = qcow2_get_cluster_offset(bs, offset, &bytes, &cluster_offset); + qemu_co_mutex_unlock(&s->lock); + if (ret < 0) { diff --git a/qemu.changes b/qemu.changes index 907ebc21..1661f17b 100644 --- a/qemu.changes +++ b/qemu.changes @@ -1,3 +1,13 @@ +------------------------------------------------------------------- +Sat Oct 26 03:07:00 UTC 2019 - Bruce Rogers + +- Address potential corruption when using qcow2 images + coroutine-Add-qemu_co_mutex_assert_locke.patch + qcow2-Fix-corruption-bug-in-qcow2_detect.patch +- Include more tweaks to our packaging workflow scripts - this will + continue as we refine the scripts +- Patch queue updated from git://github.com/openSUSE/qemu.git opensuse-4.1 + ------------------------------------------------------------------- Thu Oct 17 09:15:07 UTC 2019 - Ludwig Nussel diff --git a/qemu.spec b/qemu.spec index 7e210720..97c45192 100644 --- a/qemu.spec +++ b/qemu.spec @@ -147,46 +147,48 @@ Patch00021: s390-PCI-fix-IOMMU-region-init.patch Patch00022: hw-core-loader-Fix-possible-crash-in-rom.patch Patch00023: make-release-pull-in-edk2-submodules-so-.patch Patch00024: roms-Makefile.edk2-don-t-pull-in-submodu.patch -Patch00025: XXX-dont-dump-core-on-sigabort.patch -Patch00026: qemu-binfmt-conf-Modify-default-path.patch -Patch00027: qemu-cvs-gettimeofday.patch -Patch00028: qemu-cvs-ioctl_debug.patch -Patch00029: qemu-cvs-ioctl_nodirection.patch -Patch00030: linux-user-add-binfmt-wrapper-for-argv-0.patch -Patch00031: PPC-KVM-Disable-mmu-notifier-check.patch -Patch00032: linux-user-binfmt-support-host-binaries.patch -Patch00033: linux-user-Fake-proc-cpuinfo.patch -Patch00034: linux-user-use-target_ulong.patch -Patch00035: Make-char-muxer-more-robust-wrt-small-FI.patch -Patch00036: linux-user-lseek-explicitly-cast-non-set.patch -Patch00037: AIO-Reduce-number-of-threads-for-32bit-h.patch -Patch00038: xen_disk-Add-suse-specific-flush-disable.patch -Patch00039: qemu-bridge-helper-reduce-security-profi.patch -Patch00040: qemu-binfmt-conf-use-qemu-ARCH-binfmt.patch -Patch00041: linux-user-properly-test-for-infinite-ti.patch -Patch00042: roms-Makefile-pass-a-packaging-timestamp.patch -Patch00043: Raise-soft-address-space-limit-to-hard-l.patch -Patch00044: increase-x86_64-physical-bits-to-42.patch -Patch00045: vga-Raise-VRAM-to-16-MiB-for-pc-0.15-and.patch -Patch00046: i8254-Fix-migration-from-SLE11-SP2.patch -Patch00047: acpi_piix4-Fix-migration-from-SLE11-SP2.patch -Patch00048: Switch-order-of-libraries-for-mpath-supp.patch -Patch00049: Make-installed-scripts-explicitly-python.patch -Patch00050: hw-smbios-handle-both-file-formats-regar.patch -Patch00051: xen-add-block-resize-support-for-xen-dis.patch -Patch00052: tests-qemu-iotests-Triple-timeout-of-i-o.patch -Patch00053: tests-Fix-block-tests-to-be-compatible-w.patch -Patch00054: xen-ignore-live-parameter-from-xen-save-.patch -Patch00055: Conditionalize-ui-bitmap-installation-be.patch -Patch00056: tests-change-error-message-in-test-162.patch -Patch00057: hw-usb-hcd-xhci-Fix-GCC-9-build-warning.patch -Patch00058: hw-usb-dev-mtp-Fix-GCC-9-build-warning.patch -Patch00059: hw-intc-exynos4210_gic-provide-more-room.patch -Patch00060: configure-only-populate-roms-if-softmmu.patch -Patch00061: pc-bios-s390-ccw-net-avoid-warning-about.patch -Patch00062: roms-change-cross-compiler-naming-to-be-.patch -Patch00063: tests-Disable-some-block-tests-for-now.patch -Patch00064: test-add-mapping-from-arch-of-i686-to-qe.patch +Patch00025: coroutine-Add-qemu_co_mutex_assert_locke.patch +Patch00026: qcow2-Fix-corruption-bug-in-qcow2_detect.patch +Patch00027: XXX-dont-dump-core-on-sigabort.patch +Patch00028: qemu-binfmt-conf-Modify-default-path.patch +Patch00029: qemu-cvs-gettimeofday.patch +Patch00030: qemu-cvs-ioctl_debug.patch +Patch00031: qemu-cvs-ioctl_nodirection.patch +Patch00032: linux-user-add-binfmt-wrapper-for-argv-0.patch +Patch00033: PPC-KVM-Disable-mmu-notifier-check.patch +Patch00034: linux-user-binfmt-support-host-binaries.patch +Patch00035: linux-user-Fake-proc-cpuinfo.patch +Patch00036: linux-user-use-target_ulong.patch +Patch00037: Make-char-muxer-more-robust-wrt-small-FI.patch +Patch00038: linux-user-lseek-explicitly-cast-non-set.patch +Patch00039: AIO-Reduce-number-of-threads-for-32bit-h.patch +Patch00040: xen_disk-Add-suse-specific-flush-disable.patch +Patch00041: qemu-bridge-helper-reduce-security-profi.patch +Patch00042: qemu-binfmt-conf-use-qemu-ARCH-binfmt.patch +Patch00043: linux-user-properly-test-for-infinite-ti.patch +Patch00044: roms-Makefile-pass-a-packaging-timestamp.patch +Patch00045: Raise-soft-address-space-limit-to-hard-l.patch +Patch00046: increase-x86_64-physical-bits-to-42.patch +Patch00047: vga-Raise-VRAM-to-16-MiB-for-pc-0.15-and.patch +Patch00048: i8254-Fix-migration-from-SLE11-SP2.patch +Patch00049: acpi_piix4-Fix-migration-from-SLE11-SP2.patch +Patch00050: Switch-order-of-libraries-for-mpath-supp.patch +Patch00051: Make-installed-scripts-explicitly-python.patch +Patch00052: hw-smbios-handle-both-file-formats-regar.patch +Patch00053: xen-add-block-resize-support-for-xen-dis.patch +Patch00054: tests-qemu-iotests-Triple-timeout-of-i-o.patch +Patch00055: tests-Fix-block-tests-to-be-compatible-w.patch +Patch00056: xen-ignore-live-parameter-from-xen-save-.patch +Patch00057: Conditionalize-ui-bitmap-installation-be.patch +Patch00058: tests-change-error-message-in-test-162.patch +Patch00059: hw-usb-hcd-xhci-Fix-GCC-9-build-warning.patch +Patch00060: hw-usb-dev-mtp-Fix-GCC-9-build-warning.patch +Patch00061: hw-intc-exynos4210_gic-provide-more-room.patch +Patch00062: configure-only-populate-roms-if-softmmu.patch +Patch00063: pc-bios-s390-ccw-net-avoid-warning-about.patch +Patch00064: roms-change-cross-compiler-naming-to-be-.patch +Patch00065: tests-Disable-some-block-tests-for-now.patch +Patch00066: test-add-mapping-from-arch-of-i686-to-qe.patch # Patches applied in roms/seabios/: Patch01000: seabios-use-python2-explicitly-as-needed.patch Patch01001: seabios-switch-to-python3-as-needed.patch @@ -941,6 +943,8 @@ This package provides a service file for starting and stopping KSM. %patch00062 -p1 %patch00063 -p1 %patch00064 -p1 +%patch00065 -p1 +%patch00066 -p1 %patch01000 -p1 %patch01001 -p1 %patch01002 -p1 diff --git a/qemu.spec.in b/qemu.spec.in index d23d2733..1208a589 100644 --- a/qemu.spec.in +++ b/qemu.spec.in @@ -160,35 +160,19 @@ BuildRequires: bluez-devel BuildRequires: brlapi-devel %ifnarch %{ix86} aarch64 BuildRequires: cross-aarch64-binutils -%if 0%{suse_version} > 1500 -BuildRequires: cross-aarch64-gcc9 -%else -BuildRequires: cross-aarch64-gcc7 -%endif +BuildRequires: cross-aarch64-gcc%gcc_version %endif %ifnarch %{ix86} armv7hl BuildRequires: cross-arm-binutils -%if 0%{suse_version} > 1500 -BuildRequires: cross-arm-gcc9 -%else -BuildRequires: cross-arm-gcc7 -%endif +BuildRequires: cross-arm-gcc%gcc_version %endif %if %{build_x86_firmware_from_source} %ifnarch %{ix86} x86_64 # We must cross-compile on non-x86* BuildRequires: cross-i386-binutils -%if 0%{suse_version} > 1500 -BuildRequires: cross-i386-gcc9 -%else -BuildRequires: cross-i386-gcc7 -%endif +BuildRequires: cross-i386-gcc%gcc_version BuildRequires: cross-x86_64-binutils -%if 0%{suse_version} > 1500 -BuildRequires: cross-x86_64-gcc9 -%else -BuildRequires: cross-x86_64-gcc7 -%endif +BuildRequires: cross-x86_64-gcc%gcc_version %endif %endif BuildRequires: curl-devel diff --git a/update_git.sh b/update_git.sh index af20544d..adacb8e9 100644 --- a/update_git.sh +++ b/update_git.sh @@ -12,6 +12,8 @@ set -e source ./config.sh +declare -A COMMIT_IDS_BY_SUBMODULE_PATH + TEMP_CHECK() { # TEMPORARY! FOR NOW WE REQUIRE THESE LOCALLY TO DO WORK ON PACKAGE REQUIRED_LOCAL_REPO_MAP=( @@ -27,12 +29,12 @@ REQUIRED_LOCAL_REPO_MAP=( # TEMPORARY REQUIREMENT! for entry in ${REQUIRED_LOCAL_REPO_MAP[@]}; do if [[ -e $(readlink -f ${entry}) ]]; then - if $(git -C $entry branch| grep -F "$GIT_BRANCH" >/dev/null); then - : - else - echo "Didn't find the $GIT_BRANCH branch in repo at $entry" - exit - fi + if $(git -C $entry branch| grep -F "$GIT_BRANCH" >/dev/null); then + : + else + echo "Didn't find the $GIT_BRANCH branch in repo at $entry" + exit + fi else echo "ERROR! For now, you need to have these local git repos available:" echo ${REQUIRED_LOCAL_REPO_MAP[@]} @@ -43,12 +45,12 @@ done #============================================================================== initbundle() { -# What is needed to "start"? -# it all begins with an upstream repo, which may have submodules, incl. recursively (each represents another repo) -# To facilitate speedy work on this upstream repo, we want to have local clones of these repos -# Next we have a tarball, either that we created from the repo, or that upstream provided -# To alter the content of this tarball, lets use git to track these changes, and produce patches which can be included -# in the package spec file +# The bundle tarball has git bundles stored in a directory structure which mimics the +# submodule locations in the containing git repo. Also at that same dir level +# is a file named repo which contains the one line git repo url (with git:// or +# http(s) prefix). The bundles are named as follows: +# "{path/}{git_sha}.{bundle}", where {path/} isn't present for +# the top (qemu) bundle (ie it's for submodules). SUBMODULE_COMMIT_IDS=($(git -C ${LOCAL_REPO_MAP[0]} submodule status --recursive|awk '{print $1}')) SUBMODULE_DIRS=($(git -C ${LOCAL_REPO_MAP[0]} submodule status --recursive|awk '{print $2}')) @@ -58,6 +60,7 @@ if [[ "$REPO_COUNT" != "$(expr $SUBMODULE_COUNT + 1)" ]]; then echo "ERROR: submodule count doesn't match the REPO_COUNT variable in config.sh file!" exit fi +rm -rf $GIT_DIR rm -rf $BUNDLE_DIR mkdir -p $BUNDLE_DIR for (( i=0; i <$SUBMODULE_COUNT; i++ )); do @@ -65,29 +68,35 @@ for (( i=0; i <$SUBMODULE_COUNT; i++ )); do # what should this file be? for now use an extension of id touch $BUNDLE_DIR/${SUBMODULE_DIRS[$i]}/${SUBMODULE_COMMIT_IDS[$i]}.id done -# also handle the superproject (I need to make this smarter, or change something - works for tag, but not normal commit: - -GIT_UPSTREAM_COMMIT=$(git -C ${LOCAL_REPO_MAP[0]} show-ref -d $GIT_UPSTREAM_COMMIT_ISH|grep -F "^{}"|awk '{print $1}') +if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then + GIT_UPSTREAM_COMMIT=$(cd ${LOCAL_REPO_MAP[0]} && git rev-parse upstream/master) +else +# (I need to make this smarter, or change something - works for tag, but not normal commit?): + GIT_UPSTREAM_COMMIT=$(git -C ${LOCAL_REPO_MAP[0]} show-ref -d $GIT_UPSTREAM_COMMIT_ISH|grep -F "^{}"|awk '{print $1}') +fi touch $BUNDLE_DIR/$GIT_UPSTREAM_COMMIT.id # Now go through all the submodule local repos that are present and create a bundle file for the patches found there -rm -rf $GIT_DIR for (( i=0; i <$REPO_COUNT; i++ )); do if [[ -e $(readlink -f ${LOCAL_REPO_MAP[$i]}) ]]; then - SUBDIR=${PATCH_PATH_MAP[$i]} - GITREPO_COMMIT_ISH=($BUNDLE_DIR/$SUBDIR*.id) - if [[ $GITREPO_COMMIT_ISH =~ .*(.{40})[.]id ]]; then - GITREPO_COMMIT_ISH=${BASH_REMATCH[1]} - fi - echo "Using $GITREPO_COMMIT_ISH" - PATCH_RANGE_INDEX=$i + SUBDIR=${PATCH_PATH_MAP[$i]} + GITREPO_COMMIT_ISH=($BUNDLE_DIR/$SUBDIR*.id) + if [[ $GITREPO_COMMIT_ISH =~ .*(.{40})[.]id ]]; then + GITREPO_COMMIT_ISH=${BASH_REMATCH[1]} + echo "Using $GITREPO_COMMIT_ISH" + PATCH_RANGE_INDEX=$i mkdir -p $GIT_DIR/$SUBDIR git -C $GIT_DIR/$SUBDIR init git -C $GIT_DIR/$SUBDIR remote add origin file://$(readlink -f \ ${LOCAL_REPO_MAP[$PATCH_RANGE_INDEX]}) - git -C $GIT_DIR/$SUBDIR fetch origin $GIT_BRANCH - git -C $GIT_DIR/$SUBDIR bundle create $BUNDLE_DIR/$SUBDIR$GITREPO_COMMIT_ISH.bundle $GITREPO_COMMIT_ISH..FETCH_HEAD || true - git -C $(readlink -f ${LOCAL_REPO_MAP[$PATCH_RANGE_INDEX]}) remote get-url origin >$BUNDLE_DIR/$SUBDIR/repo + git -C $(readlink -f ${LOCAL_REPO_MAP[$PATCH_RANGE_INDEX]}) remote get-url origin >$BUNDLE_DIR/$SUBDIR/repo + if [[ $(git -C $GIT_DIR/$SUBDIR ls-remote --heads origin $GIT_BRANCH) ]]; then + git -C $GIT_DIR/$SUBDIR fetch origin $GIT_BRANCH + if [[ $(git -C $GIT_DIR/$SUBDIR rev-list $GITREPO_COMMIT_ISH..FETCH_HEAD) ]]; then + git -C $GIT_DIR/$SUBDIR bundle create $BUNDLE_DIR/$SUBDIR$GITREPO_COMMIT_ISH.bundle $GITREPO_COMMIT_ISH..FETCH_HEAD || true + fi + fi + fi fi done tar cJvf bundles.tar.xz -C $BUNDLE_DIR . @@ -113,30 +122,29 @@ for entry in ${BUNDLE_FILES[@]}; do fi for (( i=0; i <$REPO_COUNT; i++ )); do if [[ "$SUBDIR" = "${PATCH_PATH_MAP[$i]}" ]]; then - PATCH_RANGE_INDEX=$i - break + PATCH_RANGE_INDEX=$i + break fi done LOCAL_REPO=$(readlink -f ${LOCAL_REPO_MAP[$PATCH_RANGE_INDEX]}) if [ -e $LOCAL_REPO ]; then - echo "Found local repo $LOCAL_REPO corresponding to archived git bundle" - else - echo "No local repo $LOCAL_REPO corresponding to archived git bundle" - fi - git -C $LOCAL_REPO remote remove bundlerepo || true +# TODO: Detect if it's there before trying to remove! + git -C $LOCAL_REPO remote remove bundlerepo || true # git won't let you delete this branch if it's the current branch (returns 1) HOW TO HANDLE? # detect this case, and ask user to switch to another branch? or do it for them - switch to master killing any "state" for this branch - git -C $LOCAL_REPO branch -D frombundle || true - git -C $LOCAL_REPO remote add bundlerepo $BUNDLE_DIR/$entry - # in next, the head may be FETCH_HEAD or HEAD depending on how we created: - git -C $LOCAL_REPO fetch bundlerepo FETCH_HEAD - git -C $LOCAL_REPO branch frombundle FETCH_HEAD - git -C $LOCAL_REPO remote remove bundlerepo + git -C $LOCAL_REPO checkout master -f + git -C $LOCAL_REPO branch -D frombundle || true + git -C $LOCAL_REPO remote add bundlerepo $BUNDLE_DIR/$entry +# in next, the head may be FETCH_HEAD or HEAD depending on how we created: + git -C $LOCAL_REPO fetch bundlerepo FETCH_HEAD + git -C $LOCAL_REPO branch frombundle FETCH_HEAD + git -C $LOCAL_REPO remote remove bundlerepo + else + echo "No local repo $LOCAL_REPO corresponding to archived git bundle!" + exit + fi done -echo "For each local repo found, a branch named frombundle is created containing the" -echo "patches from the bundle. Use this as the starting point for making changes to" -echo "the $GIT_BRANCH, which is used when updating the bundle stored with the package." rm -rf $BUNDLE_DIR } @@ -148,14 +156,26 @@ rm -rf $CMP_DIR rm -rf $BUNDLE_DIR rm -f checkpatch.log rm -f checkthese +# there's probably a better place for the next: (only needed due to development failures?) +rm -rf checkdir if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then - # This is just a safety valve in case the above gets edited wrong: - if ! [ "$GIT_BRANCH" = "master" ]; then - echo "LATEST implies master branch, please fix configuration" - exit + for (( i=0; i <$REPO_COUNT; i++ )); do + if [[ -e $(readlink -f ${LOCAL_REPO_MAP[$i]}) ]]; then + git -C ${LOCAL_REPO_MAP[$i]} remote update upstream &> /dev/null + fi + done +#TODO: do we really want to checkout here? the code which gets the latest submodule commits doesnt rely on this !!! IN FACT master here isn't for latest upstream - that is the upstream branch! +# git -C ${LOCAL_REPO_MAP[0]} checkout master --recurse-submodules -f +# TODO: THE FOLLOWING NEEDS HELP + QEMU_VERSION=$(git -C ${LOCAL_REPO_MAP[0]} show origin:VERSION) + MAJOR_VERSION=$(echo $QEMU_VERSION|awk -F. '{print $1}') + MINOR_VERSION=$(echo $QEMU_VERSION|awk -F. '{print $2}') + if [ "$NEXT_RELEASE_IS_MAJOR" = "0" ]; then + GIT_BRANCH=opensuse-$MAJOR_VERSION.$[$MINOR_VERSION+1] + else + GIT_BRANCH=opensuse-$[$MAJOR_VERSION+1].0 fi - (cd ${LOCAL_REPO_MAP[0]} && git remote update upstream) fi BASE_RE="qemu-[[:digit:]]+(\.[[:digit:]]+){2}(-rc[[:digit:]])?" @@ -184,19 +204,11 @@ if [ "$OLD_SOURCE_VERSION_AND_EXTRA" = "" ]; then echo "Warning: No tarball found" fi -mkdir -p $BUNDLE_DIR # TODO: (repo file not yet done) -# This tarball has git bundles stored in a directory structure which mimics the -# submodule locations in the containing git repo. Also at that same dir level -# is a file named repo which contains the one line git repo url (with git:// or -# http(s) prefix). The bundles are named as follows: -# "{path/}{git_sha}.{bundle}", where {path/} isn't present for -# the top (qemu) bundle (ie it's for submodules). - -tar xJf bundles.tar.xz -C $BUNDLE_DIR -BUNDLE_FILES=$(find $BUNDLE_DIR -printf "%P\n"|grep "bundle$") - if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# DO TARBALL, GETTING ALL FROM UPSTREAM DIRECTLY +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! if [[ $QEMU_TARBALL =~ $BASE_RE$EXTRA_RE$SUFFIX_RE ]]; then OLD_COMMIT_ISH=${BASH_REMATCH[3]} else @@ -208,18 +220,26 @@ if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then echo "INFO: Ignoring signature file: $QEMU_TARBALL_SIG" QEMU_TARBALL_SIG= fi +# TODO: HERE WE REFERENCE MASTER - NEEDS FIXING + NEW_COMMIT_ISH_FULL=$(cd ${LOCAL_REPO_MAP[0]} && git rev-parse upstream/master) NEW_COMMIT_ISH=$(cd ${LOCAL_REPO_MAP[0]} && git rev-parse --short=9 \ - upstream/$GIT_BRANCH) + upstream/master) NOW_SECONDS=$(date +%s) - git clone -ls ${LOCAL_REPO_MAP[0]} $GIT_DIR -b $GIT_BRANCH --single-branch &>/dev/null +# TODO: HERE WE REFERENCE MASTER - NEEDS FIXING + git clone -ls ${LOCAL_REPO_MAP[0]} $GIT_DIR -b master --single-branch &>/dev/null if [ "$OLD_COMMIT_ISH" != "$NEW_COMMIT_ISH" ]; then echo "Please wait..." (cd $GIT_DIR && git remote add upstream \ git://git.qemu-project.org/qemu.git &>/dev/null) (cd $GIT_DIR && git remote update upstream &>/dev/null) (cd $GIT_DIR && git checkout $NEW_COMMIT_ISH &>/dev/null) - (cd $GIT_DIR && git submodule update --init --recursive &>/dev/null) +# As an alternative, we could add a --recurse-submodules to the checkout instead here as well, right? +#UPSTREAM DOESNT DO THIS (time takes 17 minutes!): +# (cd $GIT_DIR && git submodule update --init --recursive &>/dev/null) +#INSTEAD THESE NEXT TWO LINES ARE WHAT IS DONE (these take 9 minutes and 3 minutes respectively): + (cd $GIT_DIR && git submodule update --init &>/dev/null) + (cd $GIT_DIR/roms/edk2 && git submodule update --init &>/dev/null) VERSION_EXTRA=+git.$NOW_SECONDS.$NEW_COMMIT_ISH fi QEMU_VERSION=$(cat $GIT_DIR/VERSION) @@ -236,7 +256,7 @@ if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then else SOURCE_VERSION=$MAJOR_VERSION.$MINOR_VERSION.$X fi - if [ "$OLD_COMMIT_ISH" != "$NEW_COMMIT_ISH" ]; then + if [ "$OLD_COMMIT_ISH" != "$NEW_COMMIT_ISH" ]; then if (cd ${LOCAL_REPO_MAP[0]} && git describe --exact-match $NEW_COMMIT_ISH \ &>/dev/null); then if [ "$X" = "50" ]; then @@ -252,25 +272,6 @@ if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then echo "Almost there..." tar --exclude=.git --transform "s,$GIT_DIR,qemu-$SOURCE_VERSION," \ -Pcf qemu-$SOURCE_VERSION$VERSION_EXTRA.tar $GIT_DIR - echo "New tarball created. Attempting rebase..." - if ! (cd $GIT_DIR && git rebase upstream/$GIT_BRANCH $GIT_BRANCH); then - echo "rebasing master on upstream/master needs human assistance." \ - "Exiting" - (cd $GIT_DIR && git rebase --abort) - rm qemu-$SOURCE_VERSION$VERSION_EXTRA.tar - exit - fi - echo "WARNING: To rebase, master is being checked out" - if ! (cd ${LOCAL_REPO_MAP[0]} && git rebase upstream/$GIT_BRANCH \ - $GIT_BRANCH); then - echo "WARNING: Script error? rebasing master on upstream/master" \ - "succeeded in temp" - echo "dir but failed in local tree! Please investigate" - (cd ${LOCAL_REPO_MAP[0]} && git rebase --abort) - rm qemu-$SOURCE_VERSION$VERSION_EXTRA.tar - exit - fi - echo "Rebase successful" osc rm --force qemu-$OLD_SOURCE_VERSION_AND_EXTRA.tar.xz &>/dev/null ||\ true osc rm --force qemu-$OLD_SOURCE_VERSION_AND_EXTRA.tar.xz.sig \ @@ -278,10 +279,72 @@ if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then unset QEMU_TARBALL_SIG xz -T 0 qemu-$SOURCE_VERSION$VERSION_EXTRA.tar osc add qemu-$SOURCE_VERSION$VERSION_EXTRA.tar.xz +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# OK GET THE SUBMODULE COMMIT ID'S FROM THIS NEWLY MINTED QEMU CHECKOUT! WE'LL USE THAT WHEN WE REBASE OUR PATCHES +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +# !! We (perhaps temporarily) do MORE recursive submodules, since we are tracking ALL in these scripts, while upstream doesn't include all in tarball currently + (cd $GIT_DIR && git submodule update --init --recursive &>/dev/null) + SUBMODULE_COMMIT_IDS=($(git -C $GIT_DIR submodule status --recursive|awk '{print $1}')) + SUBMODULE_DIRS=($(git -C $GIT_DIR submodule status --recursive|awk '{print $2}')) + SUBMODULE_COUNT=${#SUBMODULE_COMMIT_IDS[@]} +# TODO: do this with simply math - ie: use (( ... )) + if [[ "$REPO_COUNT" != "$(expr $SUBMODULE_COUNT + 1)" ]]; then + echo "ERROR: submodule count doesn't match the REPO_COUNT variable in config.sh file!" + exit + fi +# We have the submodule commits, but not in the PATCH ORDER which our config.sh has (see $PATCH_PATH_MAP) + for (( i=0; i <$REPO_COUNT-1; i++ )); do + COMMIT_IDS_BY_SUBMODULE_PATH[${SUBMODULE_DIRS[$i]}/]=${SUBMODULE_COMMIT_IDS[$i]} + done + COMMIT_IDS_BY_SUBMODULE_PATH[SUPERPROJECT]=$NEW_COMMIT_ISH_FULL +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# MOVE BUNDLE COMMITS OVER TO LOCAL frombundle BRANCH +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + bundle2local + mkdir -p $BUNDLE_DIR + tar xJf bundles.tar.xz -C $BUNDLE_DIR +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# REBASE frombundle patches USING COMMIT_IDS_BY_SUBMODULE, ALSO USING OLD ID'S STORED IN OLD BUNDLE +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +# Now go through all the submodule local repos that are present and create a bundle file for the patches found there + for (( i=0; i <$REPO_COUNT; i++ )); do + if [[ -e $(readlink -f ${LOCAL_REPO_MAP[$i]}) ]]; then + if $(git -C ${LOCAL_REPO_MAP[$i]} branch | grep -F "frombundle" >/dev/null); then + SUBDIR=${PATCH_PATH_MAP[$i]} + GITREPO_COMMIT_ISH=($BUNDLE_DIR/$SUBDIR*.id) + if [[ $GITREPO_COMMIT_ISH =~ .*(.{40})[.]id ]]; then + GITREPO_COMMIT_ISH=${BASH_REMATCH[1]} + fi + git -C ${LOCAL_REPO_MAP[$i]} checkout frombundle -f + git -C ${LOCAL_REPO_MAP[$i]} branch -D $GIT_BRANCH + git -C ${LOCAL_REPO_MAP[$i]} checkout -b $GIT_BRANCH + if [[ "$SUBDIR" = "" ]]; then + SUBDIR=SUPERPROJECT + fi + if ! $(git -C ${LOCAL_REPO_MAP[$i]} rebase --onto ${COMMIT_IDS_BY_SUBMODULE_PATH[$SUBDIR]} $GITREPO_COMMIT_ISH >/dev/null); then +# TODO: record that this one needs manual help! + echo "Rebase of ${LOCAL_REPO_MAP[$i]}, branch $GIT_BRANCH needs manual help" + fi + fi + fi + done +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# CREATE BUNDLE FROM $GIT_BRANCH branch +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + initbundle +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# GET BUNDLE PATCHES FROM BUNDLE_DIR +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! fi + rm -rf $GIT_DIR # We're done with GIT_UPSTREAM_COMMIT_ISH carrying the special value LATEST GIT_UPSTREAM_COMMIT_ISH=$NEW_COMMIT_ISH WRITE_LOG=0 +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# DONE WITH LATEST WORK +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! else # not based on LATEST upstream master, rather any upstream commitish if [ "$OLD_SOURCE_VERSION_AND_EXTRA" = "" ]; then echo "Failure: tarball required which corresponds to commitish:" \ @@ -289,60 +352,62 @@ else # not based on LATEST upstream master, rather any upstream commitish exit fi if [ -d "${LOCAL_REPO_MAP[0]}" ]; then - echo "Processing local git tree branch: $GIT_BRANCH, using commitish:"\ + echo "Processing local git tree branch: master, using commitish:"\ "$GIT_UPSTREAM_COMMIT_ISH" - if ! (cd ${LOCAL_REPO_MAP[0]} && git show-branch $GIT_BRANCH &>/dev/null) + if ! (cd ${LOCAL_REPO_MAP[0]} && git show-branch master &>/dev/null) then - echo "Error: Branch $GIT_BRANCH not found - please create a remote"\ - "tracking branch of origin/$GIT_BRANCH" + echo "Error: Branch master not found - please create a remote"\ + "tracking branch of origin/master" exit fi - for entry in ${BUNDLE_FILES[@]}; do - if [[ $entry =~ ^(.*)[/]*([a-f0-9]{40})[.]bundle$ ]]; then - SUBDIR=${BASH_REMATCH[1]} - GITREPO_COMMIT_ISH=${BASH_REMATCH[2]} - else - echo "ERROR! BAD BUNDLE CONTENT!" - exit - fi - for (( i=0; i <$REPO_COUNT; i++ )); do - if [[ "$SUBDIR" = "${PATCH_PATH_MAP[$i]}" ]]; then - PATCH_RANGE_INDEX=$i - break - fi - done - -# !!!!! REVIEW WHERE THIS mkdir SHOULD BE HAPPENING (kind of replaces the clone operation) - mkdir -p $GIT_DIR/$SUBDIR - git -C $GIT_DIR/$SUBDIR init - git -C $GIT_DIR/$SUBDIR remote add origin file://$(readlink -f \ - ${LOCAL_REPO_MAP[$PATCH_RANGE_INDEX]}) - git -C $GIT_DIR/$SUBDIR fetch origin $GIT_BRANCH - git -C $GIT_DIR/$SUBDIR reset --hard $GITREPO_COMMIT_ISH - git -C $GIT_DIR/$SUBDIR remote add bundle $BUNDLE_DIR/$entry - # depending on how created, the bundle's head is called HEAD or FETCH_HEAD - #git -C $GIT_DIR/$SUBDIR fetch bundle HEAD - git -C $GIT_DIR/$SUBDIR fetch bundle FETCH_HEAD - git -C $GIT_DIR/$SUBDIR format-patch -N --suffix= --no-renames -o $CMP_DIR -k --stat=72 \ - --indent-heuristic --zero-commit --no-signature --full-index \ - --src-prefix=a/$SUBDIR --dst-prefix=b/$SUBDIR \ - --start-number=$(expr $PATCH_RANGE_INDEX \* $PATCH_RANGE) \ - $GITREPO_COMMIT_ISH..FETCH_HEAD > /dev/null - done - # ( THIS ISNT WORKING - IS OLD HISTORY:) else echo "Processing $GIT_BRANCH branch of remote git tree, using"\ "commitish: $GIT_UPSTREAM_COMMIT_ISH" echo "(For fast processing, consider establishing a local git tree"\ "at ${LOCAL_REPO_MAP[0]})" -# NYI - should be able to combine with local case for the most part fi - QEMU_VERSION=$(cat $GIT_DIR/VERSION) SOURCE_VERSION=$OLD_SOURCE_VERSION_AND_EXTRA + QEMU_VERSION=$(tar JxfO qemu-$SOURCE_VERSION$VERSION_EXTRA.tar.xz qemu-$SOURCE_VERSION/VERSION) NEW_COMMIT_ISH= WRITE_LOG=1 fi +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +# NOW PROCESS BUNDLES INTO COMMITS AND FILL SPEC FILE +#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! +mkdir -p $BUNDLE_DIR +tar xJf bundles.tar.xz -C $BUNDLE_DIR +BUNDLE_FILES=$(find $BUNDLE_DIR -printf "%P\n"|grep "bundle$") + +for entry in ${BUNDLE_FILES[@]}; do + if [[ $entry =~ ^(.*)[/]*([a-f0-9]{40})[.]bundle$ ]]; then + SUBDIR=${BASH_REMATCH[1]} + GITREPO_COMMIT_ISH=${BASH_REMATCH[2]} + else + echo "ERROR! BAD BUNDLE CONTENT!" + exit + fi + for (( i=0; i <$REPO_COUNT; i++ )); do + if [[ "$SUBDIR" = "${PATCH_PATH_MAP[$i]}" ]]; then + PATCH_RANGE_INDEX=$i + break + fi + done + + mkdir -p $GIT_DIR/$SUBDIR + git -C $GIT_DIR/$SUBDIR init + git -C $GIT_DIR/$SUBDIR remote add origin file://$(readlink -f \ + ${LOCAL_REPO_MAP[$PATCH_RANGE_INDEX]}) + git -C $GIT_DIR/$SUBDIR fetch origin $GIT_BRANCH + git -C $GIT_DIR/$SUBDIR reset --hard $GITREPO_COMMIT_ISH + git -C $GIT_DIR/$SUBDIR remote add bundle $BUNDLE_DIR/$entry + git -C $GIT_DIR/$SUBDIR fetch bundle FETCH_HEAD + git -C $GIT_DIR/$SUBDIR format-patch -N --suffix= --no-renames -o $CMP_DIR -k --stat=72 \ + --indent-heuristic --zero-commit --no-signature --full-index \ + --src-prefix=a/$SUBDIR --dst-prefix=b/$SUBDIR \ + --start-number=$(expr $PATCH_RANGE_INDEX \* $PATCH_RANGE) \ + $GITREPO_COMMIT_ISH..FETCH_HEAD > /dev/null +done rm -rf $GIT_DIR rm -rf $BUNDLE_DIR @@ -600,6 +665,17 @@ usage() { #============================================================================== +# LATEST processing currently doesn't expect cmdline params, so do it here, up front +if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then + echo "Processing latest upstream changes" + echo "(If SUCCESS is not printed upon completion, see /tmp/latest.log for issues)" + TEMP_CHECK + bundle2spec &> /tmp/latest.log + echo "SUCCESS" + tail -9 /tmp/latest.log + exit +fi + if [ "$1" = "" ]; then set -- git2pkg fi