pool/qemu
SHA256
4
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Accepting request 746405 from Virtualization

Browse Source 
OBS-URL: https://build.opensuse.org/request/show/746405
OBS-URL: https://build.opensuse.org/package/show/openSUSE:Factory/qemu?expand=0&rev=162
This commit is contained in:
Dominique Leuenberger 2019-11-08 14:23:56 +00:00 committed by Git OBS Bridge
commit 7b6f246840
13 changed files with 1397 additions and 314 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
_constraints
View File

@ -3,8 +3,8 @@
<overwrite>
<conditions>
<package>qemu</package>
<package>qemu-linux-user</package>
<package>qemu-testsuite</package>
<package>qemu:linux-user</package>
<package>qemu:testsuite</package>
</conditions>
<hardware>
<disk>
@ -30,7 +30,7 @@
<arch>i586</arch>
<arch>x86_64</arch>
<arch>ppc64le</arch>
<package>qemu-testsuite</package>
<package>qemu:testsuite</package>
</conditions>
<hardware>
<memory>
@ -42,7 +42,7 @@
<overwrite>
<conditions>
<arch>ppc64</arch>
<package>qemu-testsuite</package>
<package>qemu:testsuite</package>
</conditions>
<hardware>
<memory>

57
block-Add-bdrv_co_get_self_request.patch Normal file
View File

@ -0,0 +1,57 @@
From: Max Reitz <mreitz@redhat.com>
Date: Fri, 1 Nov 2019 16:25:09 +0100
Subject: block: Add bdrv_co_get_self_request()
Git-commit: c28107e9e55b11cd35cf3dc2505e3e69d10dcf13
Cc: qemu-stable@nongnu.org
Signed-off-by: Max Reitz <mreitz@redhat.com>
Message-id: 20191101152510.11719-3-mreitz@redhat.com
Signed-off-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Bruce Rogers <brogers@suse.com>
---
block/io.c | 18 ++++++++++++++++++
include/block/block_int.h | 1 +
2 files changed, 19 insertions(+)
diff --git a/block/io.c b/block/io.c
index d9f632f450b744515d3f91d2aa26..0366daf27f4a2133148716135d63 100644
--- a/block/io.c
+++ b/block/io.c
@@ -721,6 +721,24 @@ static bool is_request_serialising_and_aligned(BdrvTrackedRequest *req)
(req->bytes == req->overlap_bytes);
}
+/**
+ * Return the tracked request on @bs for the current coroutine, or
+ * NULL if there is none.
+ */
+BdrvTrackedRequest *coroutine_fn bdrv_co_get_self_request(BlockDriverState *bs)
+{
+ BdrvTrackedRequest *req;
+ Coroutine *self = qemu_coroutine_self();
+
+ QLIST_FOREACH(req, &bs->tracked_requests, list) {
+ if (req->co == self) {
+ return req;
+ }
+ }
+
+ return NULL;
+}
+
/**
* Round a region to cluster boundaries
*/
diff --git a/include/block/block_int.h b/include/block/block_int.h
index 4465b022424c23aea82942547cc3..05ee6b4866f84a9ab9ba0dcda5da 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -964,6 +964,7 @@ void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent);
bool coroutine_fn bdrv_wait_serialising_requests(BdrvTrackedRequest *self);
void bdrv_mark_request_serialising(BdrvTrackedRequest *req, uint64_t align);
+BdrvTrackedRequest *coroutine_fn bdrv_co_get_self_request(BlockDriverState *bs);
int get_tmp_filename(char *filename, int size);
BlockDriver *bdrv_probe_all(const uint8_t *buf, int buf_size,

126
block-Make-wait-mark-serialising-request.patch Normal file
View File

@ -0,0 +1,126 @@
From: Max Reitz <mreitz@redhat.com>
Date: Fri, 1 Nov 2019 16:25:08 +0100
Subject: block: Make wait/mark serialising requests public
Git-commit: 304d9d7f034ff7f5e1e66a65b7f720f63a72c57e
Make both bdrv_mark_request_serialising() and
bdrv_wait_serialising_requests() public so they can be used from block
drivers.
Cc: qemu-stable@nongnu.org
Signed-off-by: Max Reitz <mreitz@redhat.com>
Message-id: 20191101152510.11719-2-mreitz@redhat.com
Signed-off-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Bruce Rogers <brogers@suse.com>
---
block/io.c | 24 ++++++++++++------------
include/block/block_int.h | 3 +++
2 files changed, 15 insertions(+), 12 deletions(-)
diff --git a/block/io.c b/block/io.c
index bfb2653d8ee853e99bd4d55a1a87..d9f632f450b744515d3f91d2aa26 100644
--- a/block/io.c
+++ b/block/io.c
@@ -694,7 +694,7 @@ static void tracked_request_begin(BdrvTrackedRequest *req,
qemu_co_mutex_unlock(&bs->reqs_lock);
}
-static void mark_request_serialising(BdrvTrackedRequest *req, uint64_t align)
+void bdrv_mark_request_serialising(BdrvTrackedRequest *req, uint64_t align)
{
int64_t overlap_offset = req->offset & ~(align - 1);
uint64_t overlap_bytes = ROUND_UP(req->offset + req->bytes, align)
@@ -784,7 +784,7 @@ void bdrv_dec_in_flight(BlockDriverState *bs)
bdrv_wakeup(bs);
}
-static bool coroutine_fn wait_serialising_requests(BdrvTrackedRequest *self)
+bool coroutine_fn bdrv_wait_serialising_requests(BdrvTrackedRequest *self)
{
BlockDriverState *bs = self->bs;
BdrvTrackedRequest *req;
@@ -1340,14 +1340,14 @@ static int coroutine_fn bdrv_aligned_preadv(BdrvChild *child,
* with each other for the same cluster. For example, in copy-on-read
* it ensures that the CoR read and write operations are atomic and
* guest writes cannot interleave between them. */
- mark_request_serialising(req, bdrv_get_cluster_size(bs));
+ bdrv_mark_request_serialising(req, bdrv_get_cluster_size(bs));
}
/* BDRV_REQ_SERIALISING is only for write operation */
assert(!(flags & BDRV_REQ_SERIALISING));
if (!(flags & BDRV_REQ_NO_SERIALISING)) {
- wait_serialising_requests(req);
+ bdrv_wait_serialising_requests(req);
}
if (flags & BDRV_REQ_COPY_ON_READ) {
@@ -1736,10 +1736,10 @@ bdrv_co_write_req_prepare(BdrvChild *child, int64_t offset, uint64_t bytes,
assert(!(flags & ~BDRV_REQ_MASK));
if (flags & BDRV_REQ_SERIALISING) {
- mark_request_serialising(req, bdrv_get_cluster_size(bs));
+ bdrv_mark_request_serialising(req, bdrv_get_cluster_size(bs));
}
- waited = wait_serialising_requests(req);
+ waited = bdrv_wait_serialising_requests(req);
assert(!waited || !req->serialising ||
is_request_serialising_and_aligned(req));
@@ -1905,8 +1905,8 @@ static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child,
padding = bdrv_init_padding(bs, offset, bytes, &pad);
if (padding) {
- mark_request_serialising(req, align);
- wait_serialising_requests(req);
+ bdrv_mark_request_serialising(req, align);
+ bdrv_wait_serialising_requests(req);
bdrv_padding_rmw_read(child, req, &pad, true);
@@ -1993,8 +1993,8 @@ int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
}
if (bdrv_pad_request(bs, &qiov, &offset, &bytes, &pad)) {
- mark_request_serialising(&req, align);
- wait_serialising_requests(&req);
+ bdrv_mark_request_serialising(&req, align);
+ bdrv_wait_serialising_requests(&req);
bdrv_padding_rmw_read(child, &req, &pad, false);
}
@@ -3078,7 +3078,7 @@ static int coroutine_fn bdrv_co_copy_range_internal(
/* BDRV_REQ_SERIALISING is only for write operation */
assert(!(read_flags & BDRV_REQ_SERIALISING));
if (!(read_flags & BDRV_REQ_NO_SERIALISING)) {
- wait_serialising_requests(&req);
+ bdrv_wait_serialising_requests(&req);
}
ret = src->bs->drv->bdrv_co_copy_range_from(src->bs,
@@ -3205,7 +3205,7 @@ int coroutine_fn bdrv_co_truncate(BdrvChild *child, int64_t offset,
* new area, we need to make sure that no write requests are made to it
* concurrently or they might be overwritten by preallocation. */
if (new_bytes) {
- mark_request_serialising(&req, 1);
+ bdrv_mark_request_serialising(&req, 1);
}
if (bs->read_only) {
error_setg(errp, "Image is read-only");
diff --git a/include/block/block_int.h b/include/block/block_int.h
index 3aa1e832a8fdf32bf3f33d1e1508..4465b022424c23aea82942547cc3 100644
--- a/include/block/block_int.h
+++ b/include/block/block_int.h
@@ -962,6 +962,9 @@ extern unsigned int bdrv_drain_all_count;
void bdrv_apply_subtree_drain(BdrvChild *child, BlockDriverState *new_parent);
void bdrv_unapply_subtree_drain(BdrvChild *child, BlockDriverState *old_parent);
+bool coroutine_fn bdrv_wait_serialising_requests(BdrvTrackedRequest *self);
+void bdrv_mark_request_serialising(BdrvTrackedRequest *req, uint64_t align);
+
int get_tmp_filename(char *filename, int size);
BlockDriver *bdrv_probe_all(const uint8_t *buf, int buf_size,
const char *filename);

67
block-file-posix-Let-post-EOF-fallocate-.patch Normal file
View File

@ -0,0 +1,67 @@
From: Max Reitz <mreitz@redhat.com>
Date: Fri, 1 Nov 2019 16:25:10 +0100
Subject: block/file-posix: Let post-EOF fallocate serialize
Git-commit: 292d06b925b2787ee6f2430996b95651cae42fce
The XFS kernel driver has a bug that may cause data corruption for qcow2
images as of qemu commit c8bb23cbdbe32f. We can work around it by
treating post-EOF fallocates as serializing up until infinity (INT64_MAX
in practice).
Cc: qemu-stable@nongnu.org
Signed-off-by: Max Reitz <mreitz@redhat.com>
Message-id: 20191101152510.11719-4-mreitz@redhat.com
Signed-off-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Bruce Rogers <brogers@suse.com>
---
block/file-posix.c | 36 ++++++++++++++++++++++++++++++++++++
1 file changed, 36 insertions(+)
diff --git a/block/file-posix.c b/block/file-posix.c
index 992eb4a798b99fe02e93103028c6..c5df61b47735ee7e5201cebec46c 100644
--- a/block/file-posix.c
+++ b/block/file-posix.c
@@ -2623,6 +2623,42 @@ raw_do_pwrite_zeroes(BlockDriverState *bs, int64_t offset, int bytes,
RawPosixAIOData acb;
ThreadPoolFunc *handler;
+#ifdef CONFIG_FALLOCATE
+ if (offset + bytes > bs->total_sectors * BDRV_SECTOR_SIZE) {
+ BdrvTrackedRequest *req;
+ uint64_t end;
+
+ /*
+ * This is a workaround for a bug in the Linux XFS driver,
+ * where writes submitted through the AIO interface will be
+ * discarded if they happen beyond a concurrently running
+ * fallocate() that increases the file length (i.e., both the
+ * write and the fallocate() happen beyond the EOF).
+ *
+ * To work around it, we extend the tracked request for this
+ * zero write until INT64_MAX (effectively infinity), and mark
+ * it as serializing.
+ *
+ * We have to enable this workaround for all filesystems and
+ * AIO modes (not just XFS with aio=native), because for
+ * remote filesystems we do not know the host configuration.
+ */
+
+ req = bdrv_co_get_self_request(bs);
+ assert(req);
+ assert(req->type == BDRV_TRACKED_WRITE);
+ assert(req->offset <= offset);
+ assert(req->offset + req->bytes >= offset + bytes);
+
+ end = INT64_MAX & -(uint64_t)bs->bl.request_alignment;
+ req->bytes = end - req->offset;
+ req->overlap_bytes = req->bytes;
+
+ bdrv_mark_request_serialising(req, bs->bl.request_alignment);
+ bdrv_wait_serialising_requests(req);
+ }
+#endif
+
acb = (RawPosixAIOData) {
.bs = bs,
.aio_fildes = s->fd,

478
block-io-refactor-padding.patch Normal file
View File

@ -0,0 +1,478 @@
From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Date: Tue, 4 Jun 2019 19:15:05 +0300
Subject: block/io: refactor padding
Git-commit: 7a3f542fbdfd799be4fa6f8b96dc8c1e6933fce4
We have similar padding code in bdrv_co_pwritev,
bdrv_co_do_pwrite_zeroes and bdrv_co_preadv. Let's combine and unify
it.
[Squashed in Vladimir's qemu-iotests 077 fix
--Stefan]
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Acked-by: Stefan Hajnoczi <stefanha@redhat.com>
Message-id: 20190604161514.262241-4-vsementsov@virtuozzo.com
Message-Id: <20190604161514.262241-4-vsementsov@virtuozzo.com>
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
Signed-off-by: Bruce Rogers <brogers@suse.com>
---
block/io.c | 365 +++++++++++++++++++++++++++++------------------------
1 file changed, 200 insertions(+), 165 deletions(-)
diff --git a/block/io.c b/block/io.c
index 06305c6ea62efabf1efb43933bf6..bfb2653d8ee853e99bd4d55a1a87 100644
--- a/block/io.c
+++ b/block/io.c
@@ -1408,28 +1408,177 @@ out:
}
/*
- * Handle a read request in coroutine context
+ * Request padding
+ *
+ * |<---- align ----->| |<----- align ---->|
+ * |<- head ->|<------------- bytes ------------->|<-- tail -->|
+ * | | | | | |
+ * -*----------$-------*-------- ... --------*-----$------------*---
+ * | | | | | |
+ * | offset | | end |
+ * ALIGN_DOWN(offset) ALIGN_UP(offset) ALIGN_DOWN(end) ALIGN_UP(end)
+ * [buf ... ) [tail_buf )
+ *
+ * @buf is an aligned allocation needed to store @head and @tail paddings. @head
+ * is placed at the beginning of @buf and @tail at the @end.
+ *
+ * @tail_buf is a pointer to sub-buffer, corresponding to align-sized chunk
+ * around tail, if tail exists.
+ *
+ * @merge_reads is true for small requests,
+ * if @buf_len == @head + bytes + @tail. In this case it is possible that both
+ * head and tail exist but @buf_len == align and @tail_buf == @buf.
+ */
+typedef struct BdrvRequestPadding {
+ uint8_t *buf;
+ size_t buf_len;
+ uint8_t *tail_buf;
+ size_t head;
+ size_t tail;
+ bool merge_reads;
+ QEMUIOVector local_qiov;
+} BdrvRequestPadding;
+
+static bool bdrv_init_padding(BlockDriverState *bs,
+ int64_t offset, int64_t bytes,
+ BdrvRequestPadding *pad)
+{
+ uint64_t align = bs->bl.request_alignment;
+ size_t sum;
+
+ memset(pad, 0, sizeof(*pad));
+
+ pad->head = offset & (align - 1);
+ pad->tail = ((offset + bytes) & (align - 1));
+ if (pad->tail) {
+ pad->tail = align - pad->tail;
+ }
+
+ if ((!pad->head && !pad->tail) || !bytes) {
+ return false;
+ }
+
+ sum = pad->head + bytes + pad->tail;
+ pad->buf_len = (sum > align && pad->head && pad->tail) ? 2 * align : align;
+ pad->buf = qemu_blockalign(bs, pad->buf_len);
+ pad->merge_reads = sum == pad->buf_len;
+ if (pad->tail) {
+ pad->tail_buf = pad->buf + pad->buf_len - align;
+ }
+
+ return true;
+}
+
+static int bdrv_padding_rmw_read(BdrvChild *child,
+ BdrvTrackedRequest *req,
+ BdrvRequestPadding *pad,
+ bool zero_middle)
+{
+ QEMUIOVector local_qiov;
+ BlockDriverState *bs = child->bs;
+ uint64_t align = bs->bl.request_alignment;
+ int ret;
+
+ assert(req->serialising && pad->buf);
+
+ if (pad->head || pad->merge_reads) {
+ uint64_t bytes = pad->merge_reads ? pad->buf_len : align;
+
+ qemu_iovec_init_buf(&local_qiov, pad->buf, bytes);
+
+ if (pad->head) {
+ bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
+ }
+ if (pad->merge_reads && pad->tail) {
+ bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
+ }
+ ret = bdrv_aligned_preadv(child, req, req->overlap_offset, bytes,
+ align, &local_qiov, 0);
+ if (ret < 0) {
+ return ret;
+ }
+ if (pad->head) {
+ bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
+ }
+ if (pad->merge_reads && pad->tail) {
+ bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
+ }
+
+ if (pad->merge_reads) {
+ goto zero_mem;
+ }
+ }
+
+ if (pad->tail) {
+ qemu_iovec_init_buf(&local_qiov, pad->tail_buf, align);
+
+ bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
+ ret = bdrv_aligned_preadv(
+ child, req,
+ req->overlap_offset + req->overlap_bytes - align,
+ align, align, &local_qiov, 0);
+ if (ret < 0) {
+ return ret;
+ }
+ bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
+ }
+
+zero_mem:
+ if (zero_middle) {
+ memset(pad->buf + pad->head, 0, pad->buf_len - pad->head - pad->tail);
+ }
+
+ return 0;
+}
+
+static void bdrv_padding_destroy(BdrvRequestPadding *pad)
+{
+ if (pad->buf) {
+ qemu_vfree(pad->buf);
+ qemu_iovec_destroy(&pad->local_qiov);
+ }
+}
+
+/*
+ * bdrv_pad_request
+ *
+ * Exchange request parameters with padded request if needed. Don't include RMW
+ * read of padding, bdrv_padding_rmw_read() should be called separately if
+ * needed.
+ *
+ * All parameters except @bs are in-out: they represent original request at
+ * function call and padded (if padding needed) at function finish.
+ *
+ * Function always succeeds.
*/
+static bool bdrv_pad_request(BlockDriverState *bs, QEMUIOVector **qiov,
+ int64_t *offset, unsigned int *bytes,
+ BdrvRequestPadding *pad)
+{
+ if (!bdrv_init_padding(bs, *offset, *bytes, pad)) {
+ return false;
+ }
+
+ qemu_iovec_init_extended(&pad->local_qiov, pad->buf, pad->head,
+ *qiov, 0, *bytes,
+ pad->buf + pad->buf_len - pad->tail, pad->tail);
+ *bytes += pad->head + pad->tail;
+ *offset -= pad->head;
+ *qiov = &pad->local_qiov;
+
+ return true;
+}
+
int coroutine_fn bdrv_co_preadv(BdrvChild *child,
int64_t offset, unsigned int bytes, QEMUIOVector *qiov,
BdrvRequestFlags flags)
{
BlockDriverState *bs = child->bs;
- BlockDriver *drv = bs->drv;
BdrvTrackedRequest req;
-
- uint64_t align = bs->bl.request_alignment;
- uint8_t *head_buf = NULL;
- uint8_t *tail_buf = NULL;
- QEMUIOVector local_qiov;
- bool use_local_qiov = false;
+ BdrvRequestPadding pad;
int ret;
- trace_bdrv_co_preadv(child->bs, offset, bytes, flags);
-
- if (!drv) {
- return -ENOMEDIUM;
- }
+ trace_bdrv_co_preadv(bs, offset, bytes, flags);
ret = bdrv_check_byte_request(bs, offset, bytes);
if (ret < 0) {
@@ -1443,43 +1592,16 @@ int coroutine_fn bdrv_co_preadv(BdrvChild *child,
flags |= BDRV_REQ_COPY_ON_READ;
}
- /* Align read if necessary by padding qiov */
- if (offset & (align - 1)) {
- head_buf = qemu_blockalign(bs, align);
- qemu_iovec_init(&local_qiov, qiov->niov + 2);
- qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
- qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
- use_local_qiov = true;
-
- bytes += offset & (align - 1);
- offset = offset & ~(align - 1);
- }
-
- if ((offset + bytes) & (align - 1)) {
- if (!use_local_qiov) {
- qemu_iovec_init(&local_qiov, qiov->niov + 1);
- qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
- use_local_qiov = true;
- }
- tail_buf = qemu_blockalign(bs, align);
- qemu_iovec_add(&local_qiov, tail_buf,
- align - ((offset + bytes) & (align - 1)));
-
- bytes = ROUND_UP(bytes, align);
- }
+ bdrv_pad_request(bs, &qiov, &offset, &bytes, &pad);
tracked_request_begin(&req, bs, offset, bytes, BDRV_TRACKED_READ);
- ret = bdrv_aligned_preadv(child, &req, offset, bytes, align,
- use_local_qiov ? &local_qiov : qiov,
- flags);
+ ret = bdrv_aligned_preadv(child, &req, offset, bytes,
+ bs->bl.request_alignment,
+ qiov, flags);
tracked_request_end(&req);
bdrv_dec_in_flight(bs);
- if (use_local_qiov) {
- qemu_iovec_destroy(&local_qiov);
- qemu_vfree(head_buf);
- qemu_vfree(tail_buf);
- }
+ bdrv_padding_destroy(&pad);
return ret;
}
@@ -1775,44 +1897,34 @@ static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child,
BdrvTrackedRequest *req)
{
BlockDriverState *bs = child->bs;
- uint8_t *buf = NULL;
QEMUIOVector local_qiov;
uint64_t align = bs->bl.request_alignment;
- unsigned int head_padding_bytes, tail_padding_bytes;
int ret = 0;
+ bool padding;
+ BdrvRequestPadding pad;
- head_padding_bytes = offset & (align - 1);
- tail_padding_bytes = (align - (offset + bytes)) & (align - 1);
-
-
- assert(flags & BDRV_REQ_ZERO_WRITE);
- if (head_padding_bytes || tail_padding_bytes) {
- buf = qemu_blockalign(bs, align);
- qemu_iovec_init_buf(&local_qiov, buf, align);
- }
- if (head_padding_bytes) {
- uint64_t zero_bytes = MIN(bytes, align - head_padding_bytes);
-
- /* RMW the unaligned part before head. */
+ padding = bdrv_init_padding(bs, offset, bytes, &pad);
+ if (padding) {
mark_request_serialising(req, align);
wait_serialising_requests(req);
- bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
- ret = bdrv_aligned_preadv(child, req, offset & ~(align - 1), align,
- align, &local_qiov, 0);
- if (ret < 0) {
- goto fail;
- }
- bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
- memset(buf + head_padding_bytes, 0, zero_bytes);
- ret = bdrv_aligned_pwritev(child, req, offset & ~(align - 1), align,
- align, &local_qiov,
- flags & ~BDRV_REQ_ZERO_WRITE);
- if (ret < 0) {
- goto fail;
+ bdrv_padding_rmw_read(child, req, &pad, true);
+
+ if (pad.head || pad.merge_reads) {
+ int64_t aligned_offset = offset & ~(align - 1);
+ int64_t write_bytes = pad.merge_reads ? pad.buf_len : align;
+
+ qemu_iovec_init_buf(&local_qiov, pad.buf, write_bytes);
+ ret = bdrv_aligned_pwritev(child, req, aligned_offset, write_bytes,
+ align, &local_qiov,
+ flags & ~BDRV_REQ_ZERO_WRITE);
+ if (ret < 0 || pad.merge_reads) {
+ /* Error or all work is done */
+ goto out;
+ }
+ offset += write_bytes - pad.head;
+ bytes -= write_bytes - pad.head;
}
- offset += zero_bytes;
- bytes -= zero_bytes;
}
assert(!bytes || (offset & (align - 1)) == 0);
@@ -1822,7 +1934,7 @@ static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child,
ret = bdrv_aligned_pwritev(child, req, offset, aligned_bytes, align,
NULL, flags);
if (ret < 0) {
- goto fail;
+ goto out;
}
bytes -= aligned_bytes;
offset += aligned_bytes;
@@ -1830,26 +1942,17 @@ static int coroutine_fn bdrv_co_do_zero_pwritev(BdrvChild *child,
assert(!bytes || (offset & (align - 1)) == 0);
if (bytes) {
- assert(align == tail_padding_bytes + bytes);
- /* RMW the unaligned part after tail. */
- mark_request_serialising(req, align);
- wait_serialising_requests(req);
- bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
- ret = bdrv_aligned_preadv(child, req, offset, align,
- align, &local_qiov, 0);
- if (ret < 0) {
- goto fail;
- }
- bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
+ assert(align == pad.tail + bytes);
- memset(buf, 0, bytes);
+ qemu_iovec_init_buf(&local_qiov, pad.tail_buf, align);
ret = bdrv_aligned_pwritev(child, req, offset, align, align,
&local_qiov, flags & ~BDRV_REQ_ZERO_WRITE);
}
-fail:
- qemu_vfree(buf);
- return ret;
+out:
+ bdrv_padding_destroy(&pad);
+
+ return ret;
}
/*
@@ -1862,10 +1965,7 @@ int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
BlockDriverState *bs = child->bs;
BdrvTrackedRequest req;
uint64_t align = bs->bl.request_alignment;
- uint8_t *head_buf = NULL;
- uint8_t *tail_buf = NULL;
- QEMUIOVector local_qiov;
- bool use_local_qiov = false;
+ BdrvRequestPadding pad;
int ret;
trace_bdrv_co_pwritev(child->bs, offset, bytes, flags);
@@ -1892,86 +1992,21 @@ int coroutine_fn bdrv_co_pwritev(BdrvChild *child,
goto out;
}
- if (offset & (align - 1)) {
- QEMUIOVector head_qiov;
-
+ if (bdrv_pad_request(bs, &qiov, &offset, &bytes, &pad)) {
mark_request_serialising(&req, align);
wait_serialising_requests(&req);
-
- head_buf = qemu_blockalign(bs, align);
- qemu_iovec_init_buf(&head_qiov, head_buf, align);
-
- bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_HEAD);
- ret = bdrv_aligned_preadv(child, &req, offset & ~(align - 1), align,
- align, &head_qiov, 0);
- if (ret < 0) {
- goto fail;
- }
- bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_HEAD);
-
- qemu_iovec_init(&local_qiov, qiov->niov + 2);
- qemu_iovec_add(&local_qiov, head_buf, offset & (align - 1));
- qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
- use_local_qiov = true;
-
- bytes += offset & (align - 1);
- offset = offset & ~(align - 1);
-
- /* We have read the tail already if the request is smaller
- * than one aligned block.
- */
- if (bytes < align) {
- qemu_iovec_add(&local_qiov, head_buf + bytes, align - bytes);
- bytes = align;
- }
- }
-
- if ((offset + bytes) & (align - 1)) {
- QEMUIOVector tail_qiov;
- size_t tail_bytes;
- bool waited;
-
- mark_request_serialising(&req, align);
- waited = wait_serialising_requests(&req);
- assert(!waited || !use_local_qiov);
-
- tail_buf = qemu_blockalign(bs, align);
- qemu_iovec_init_buf(&tail_qiov, tail_buf, align);
-
- bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_TAIL);
- ret = bdrv_aligned_preadv(child, &req, (offset + bytes) & ~(align - 1),
- align, align, &tail_qiov, 0);
- if (ret < 0) {
- goto fail;
- }
- bdrv_debug_event(bs, BLKDBG_PWRITEV_RMW_AFTER_TAIL);
-
- if (!use_local_qiov) {
- qemu_iovec_init(&local_qiov, qiov->niov + 1);
- qemu_iovec_concat(&local_qiov, qiov, 0, qiov->size);
- use_local_qiov = true;
- }
-
- tail_bytes = (offset + bytes) & (align - 1);
- qemu_iovec_add(&local_qiov, tail_buf + tail_bytes, align - tail_bytes);
-
- bytes = ROUND_UP(bytes, align);
+ bdrv_padding_rmw_read(child, &req, &pad, false);
}
ret = bdrv_aligned_pwritev(child, &req, offset, bytes, align,
- use_local_qiov ? &local_qiov : qiov,
- flags);
+ qiov, flags);
-fail:
+ bdrv_padding_destroy(&pad);
- if (use_local_qiov) {
- qemu_iovec_destroy(&local_qiov);
- }
- qemu_vfree(head_buf);
- qemu_vfree(tail_buf);
out:
tracked_request_end(&req);
bdrv_dec_in_flight(bs);
+
return ret;
}

4
bundles.tar.xz
View File

@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:86afea4d0fcdfa578dd3b660ccbadd49e996057de08ca8078020cb5c10611f91
size 56424
oid sha256:4c674d3cdda7189a79c9f7babf6858dddc3bf621d19f43e80cacf8fb6c85cc12
size 60840

6
config.sh
View File

@ -2,16 +2,12 @@
# config.sh:
# The next few VARIABLES may be edited (or uncommented) as required:
# The next few VARIABLES are to be edited as required:
# The following specifies the upstream tag or commit upon which our patchqueue
# gets rebased. The special value LATEST may be used to "automatically" track
# the upstream development tree in the master branch
GIT_UPSTREAM_COMMIT_ISH=v4.1.0
if [[ "$GIT_UPSTREAM_COMMIT_ISH" != "LATEST" ]]; then
# This is the git branch used (otherwise it is computed)
GIT_BRANCH=opensuse-4.1
fi
# WARNING: If transitioning from using LATEST to not, MANUALLY re-set the
# tarball present. If transitioning TO LATEST, make sure that
# NEXT_RELEASE_IS_MAJOR is set correctly

33
qcow2-Fix-QCOW2_COMPRESSED_SECTOR_MASK.patch Normal file
View File

@ -0,0 +1,33 @@
From: Max Reitz <mreitz@redhat.com>
Date: Mon, 28 Oct 2019 17:18:40 +0100
Subject: qcow2: Fix QCOW2_COMPRESSED_SECTOR_MASK
Git-commit: 24552feb6ae2f615b76c2b95394af43901f75046
Masks for L2 table entries should have 64 bit.
Fixes: b6c246942b14d3e0dec46a6c5868ed84e7dbea19
Buglink: https://bugs.launchpad.net/qemu/+bug/1850000
Cc: qemu-stable@nongnu.org
Signed-off-by: Max Reitz <mreitz@redhat.com>
Message-id: 20191028161841.1198-2-mreitz@redhat.com
Reviewed-by: Alberto Garcia <berto@igalia.com>
Signed-off-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Bruce Rogers <brogers@suse.com>
---
block/qcow2.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/block/qcow2.h b/block/qcow2.h
index fc1b0d3c1e1e3110e86f36f0dcd9..359197f89fb9547d861d852cbf68 100644
--- a/block/qcow2.h
+++ b/block/qcow2.h
@@ -77,7 +77,7 @@
/* Defined in the qcow2 spec (compressed cluster descriptor) */
#define QCOW2_COMPRESSED_SECTOR_SIZE 512U
-#define QCOW2_COMPRESSED_SECTOR_MASK (~(QCOW2_COMPRESSED_SECTOR_SIZE - 1))
+#define QCOW2_COMPRESSED_SECTOR_MASK (~(QCOW2_COMPRESSED_SECTOR_SIZE - 1ULL))
/* Must be at least 2 to cover COW */
#define MIN_L2_CACHE_SIZE 2 /* cache entries */

66
qcow2-bitmap-Fix-uint64_t-left-shift-ove.patch Normal file
View File

@ -0,0 +1,66 @@
From: Tuguoyi <tu.guoyi@h3c.com>
Date: Fri, 1 Nov 2019 07:37:35 +0000
Subject: qcow2-bitmap: Fix uint64_t left-shift overflow
Git-commit: 570542ecb11e04b61ef4b3f4d0965a6915232a88
There are two issues in In check_constraints_on_bitmap(),
1) The sanity check on the granularity will cause uint64_t
integer left-shift overflow when cluster_size is 2M and the
granularity is BIGGER than 32K.
2) The way to calculate image size that the maximum bitmap
supported can map to is a bit incorrect.
This patch fix it by add a helper function to calculate the
number of bytes needed by a normal bitmap in image and compare
it to the maximum bitmap bytes supported by qemu.
Fixes: 5f72826e7fc62167cf3a
Signed-off-by: Guoyi Tu <tu.guoyi@h3c.com>
Message-id: 4ba40cd1e7ee4a708b40899952e49f22@h3c.com
Reviewed-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Cc: qemu-stable@nongnu.org
Signed-off-by: Max Reitz <mreitz@redhat.com>
Signed-off-by: Bruce Rogers <brogers@suse.com>
---
block/qcow2-bitmap.c | 14 +++++++++++---
1 file changed, 11 insertions(+), 3 deletions(-)
diff --git a/block/qcow2-bitmap.c b/block/qcow2-bitmap.c
index b2487101edec0bdde613c561da89..65034da1c0766603d273c200797d 100644
--- a/block/qcow2-bitmap.c
+++ b/block/qcow2-bitmap.c
@@ -142,6 +142,13 @@ static int check_table_entry(uint64_t entry, int cluster_size)
return 0;
}
+static int64_t get_bitmap_bytes_needed(int64_t len, uint32_t granularity)
+{
+ int64_t num_bits = DIV_ROUND_UP(len, granularity);
+
+ return DIV_ROUND_UP(num_bits, 8);
+}
+
static int check_constraints_on_bitmap(BlockDriverState *bs,
const char *name,
uint32_t granularity,
@@ -150,6 +157,7 @@ static int check_constraints_on_bitmap(BlockDriverState *bs,
BDRVQcow2State *s = bs->opaque;
int granularity_bits = ctz32(granularity);
int64_t len = bdrv_getlength(bs);
+ int64_t bitmap_bytes;
assert(granularity > 0);
assert((granularity & (granularity - 1)) == 0);
@@ -171,9 +179,9 @@ static int check_constraints_on_bitmap(BlockDriverState *bs,
return -EINVAL;
}
- if ((len > (uint64_t)BME_MAX_PHYS_SIZE << granularity_bits) ||
- (len > (uint64_t)BME_MAX_TABLE_SIZE * s->cluster_size <<
- granularity_bits))
+ bitmap_bytes = get_bitmap_bytes_needed(len, granularity);
+ if ((bitmap_bytes > (uint64_t)BME_MAX_PHYS_SIZE) ||
+ (bitmap_bytes > (uint64_t)BME_MAX_TABLE_SIZE * s->cluster_size))
{
error_setg(errp, "Too much space will be occupied by the bitmap. "
"Use larger granularity");

25
qemu.changes
View File

@ -1,3 +1,28 @@
-------------------------------------------------------------------
Thu Nov 7 19:02:39 UTC 2019 - Bruce Rogers <brogers@suse.com>
- Fix two issues with qcow2 image processing which could affect
disk integrity
qcow2-Fix-QCOW2_COMPRESSED_SECTOR_MASK.patch
qcow2-bitmap-Fix-uint64_t-left-shift-ove.patch
-------------------------------------------------------------------
Wed Nov 6 20:43:48 UTC 2019 - Bruce Rogers <brogers@suse.com>
- Work around a host kernel xfs bug which can result in qcow2 image
corruption
block-io-refactor-padding.patch
util-iov-introduce-qemu_iovec_init_exten.patch
block-Make-wait-mark-serialising-request.patch
block-Add-bdrv_co_get_self_request.patch
block-file-posix-Let-post-EOF-fallocate-.patch
- Patch queue updated from git://github.com/openSUSE/qemu.git opensuse-4.1
-------------------------------------------------------------------
Mon Nov 4 13:47:02 UTC 2019 - Stefan Brüns <stefan.bruens@rwth-aachen.de>
- Correct package names in _constraints after switch to multibuild.
-------------------------------------------------------------------
Sat Oct 26 03:07:00 UTC 2019 - Bruce Rogers <brogers@suse.com>

94
qemu.spec
View File

@ -149,46 +149,53 @@ Patch00023: make-release-pull-in-edk2-submodules-so-.patch
Patch00024: roms-Makefile.edk2-don-t-pull-in-submodu.patch
Patch00025: coroutine-Add-qemu_co_mutex_assert_locke.patch
Patch00026: qcow2-Fix-corruption-bug-in-qcow2_detect.patch
Patch00027: XXX-dont-dump-core-on-sigabort.patch
Patch00028: qemu-binfmt-conf-Modify-default-path.patch
Patch00029: qemu-cvs-gettimeofday.patch
Patch00030: qemu-cvs-ioctl_debug.patch
Patch00031: qemu-cvs-ioctl_nodirection.patch
Patch00032: linux-user-add-binfmt-wrapper-for-argv-0.patch
Patch00033: PPC-KVM-Disable-mmu-notifier-check.patch
Patch00034: linux-user-binfmt-support-host-binaries.patch
Patch00035: linux-user-Fake-proc-cpuinfo.patch
Patch00036: linux-user-use-target_ulong.patch
Patch00037: Make-char-muxer-more-robust-wrt-small-FI.patch
Patch00038: linux-user-lseek-explicitly-cast-non-set.patch
Patch00039: AIO-Reduce-number-of-threads-for-32bit-h.patch
Patch00040: xen_disk-Add-suse-specific-flush-disable.patch
Patch00041: qemu-bridge-helper-reduce-security-profi.patch
Patch00042: qemu-binfmt-conf-use-qemu-ARCH-binfmt.patch
Patch00043: linux-user-properly-test-for-infinite-ti.patch
Patch00044: roms-Makefile-pass-a-packaging-timestamp.patch
Patch00045: Raise-soft-address-space-limit-to-hard-l.patch
Patch00046: increase-x86_64-physical-bits-to-42.patch
Patch00047: vga-Raise-VRAM-to-16-MiB-for-pc-0.15-and.patch
Patch00048: i8254-Fix-migration-from-SLE11-SP2.patch
Patch00049: acpi_piix4-Fix-migration-from-SLE11-SP2.patch
Patch00050: Switch-order-of-libraries-for-mpath-supp.patch
Patch00051: Make-installed-scripts-explicitly-python.patch
Patch00052: hw-smbios-handle-both-file-formats-regar.patch
Patch00053: xen-add-block-resize-support-for-xen-dis.patch
Patch00054: tests-qemu-iotests-Triple-timeout-of-i-o.patch
Patch00055: tests-Fix-block-tests-to-be-compatible-w.patch
Patch00056: xen-ignore-live-parameter-from-xen-save-.patch
Patch00057: Conditionalize-ui-bitmap-installation-be.patch
Patch00058: tests-change-error-message-in-test-162.patch
Patch00059: hw-usb-hcd-xhci-Fix-GCC-9-build-warning.patch
Patch00060: hw-usb-dev-mtp-Fix-GCC-9-build-warning.patch
Patch00061: hw-intc-exynos4210_gic-provide-more-room.patch
Patch00062: configure-only-populate-roms-if-softmmu.patch
Patch00063: pc-bios-s390-ccw-net-avoid-warning-about.patch
Patch00064: roms-change-cross-compiler-naming-to-be-.patch
Patch00065: tests-Disable-some-block-tests-for-now.patch
Patch00066: test-add-mapping-from-arch-of-i686-to-qe.patch
Patch00027: block-io-refactor-padding.patch
Patch00028: util-iov-introduce-qemu_iovec_init_exten.patch
Patch00029: block-Make-wait-mark-serialising-request.patch
Patch00030: block-Add-bdrv_co_get_self_request.patch
Patch00031: block-file-posix-Let-post-EOF-fallocate-.patch
Patch00032: qcow2-bitmap-Fix-uint64_t-left-shift-ove.patch
Patch00033: qcow2-Fix-QCOW2_COMPRESSED_SECTOR_MASK.patch
Patch00034: XXX-dont-dump-core-on-sigabort.patch
Patch00035: qemu-binfmt-conf-Modify-default-path.patch
Patch00036: qemu-cvs-gettimeofday.patch
Patch00037: qemu-cvs-ioctl_debug.patch
Patch00038: qemu-cvs-ioctl_nodirection.patch
Patch00039: linux-user-add-binfmt-wrapper-for-argv-0.patch
Patch00040: PPC-KVM-Disable-mmu-notifier-check.patch
Patch00041: linux-user-binfmt-support-host-binaries.patch
Patch00042: linux-user-Fake-proc-cpuinfo.patch
Patch00043: linux-user-use-target_ulong.patch
Patch00044: Make-char-muxer-more-robust-wrt-small-FI.patch
Patch00045: linux-user-lseek-explicitly-cast-non-set.patch
Patch00046: AIO-Reduce-number-of-threads-for-32bit-h.patch
Patch00047: xen_disk-Add-suse-specific-flush-disable.patch
Patch00048: qemu-bridge-helper-reduce-security-profi.patch
Patch00049: qemu-binfmt-conf-use-qemu-ARCH-binfmt.patch
Patch00050: linux-user-properly-test-for-infinite-ti.patch
Patch00051: roms-Makefile-pass-a-packaging-timestamp.patch
Patch00052: Raise-soft-address-space-limit-to-hard-l.patch
Patch00053: increase-x86_64-physical-bits-to-42.patch
Patch00054: vga-Raise-VRAM-to-16-MiB-for-pc-0.15-and.patch
Patch00055: i8254-Fix-migration-from-SLE11-SP2.patch
Patch00056: acpi_piix4-Fix-migration-from-SLE11-SP2.patch
Patch00057: Switch-order-of-libraries-for-mpath-supp.patch
Patch00058: Make-installed-scripts-explicitly-python.patch
Patch00059: hw-smbios-handle-both-file-formats-regar.patch
Patch00060: xen-add-block-resize-support-for-xen-dis.patch
Patch00061: tests-qemu-iotests-Triple-timeout-of-i-o.patch
Patch00062: tests-Fix-block-tests-to-be-compatible-w.patch
Patch00063: xen-ignore-live-parameter-from-xen-save-.patch
Patch00064: Conditionalize-ui-bitmap-installation-be.patch
Patch00065: tests-change-error-message-in-test-162.patch
Patch00066: hw-usb-hcd-xhci-Fix-GCC-9-build-warning.patch
Patch00067: hw-usb-dev-mtp-Fix-GCC-9-build-warning.patch
Patch00068: hw-intc-exynos4210_gic-provide-more-room.patch
Patch00069: configure-only-populate-roms-if-softmmu.patch
Patch00070: pc-bios-s390-ccw-net-avoid-warning-about.patch
Patch00071: roms-change-cross-compiler-naming-to-be-.patch
Patch00072: tests-Disable-some-block-tests-for-now.patch
Patch00073: test-add-mapping-from-arch-of-i686-to-qe.patch
# Patches applied in roms/seabios/:
Patch01000: seabios-use-python2-explicitly-as-needed.patch
Patch01001: seabios-switch-to-python3-as-needed.patch
@ -945,6 +952,13 @@ This package provides a service file for starting and stopping KSM.
%patch00064 -p1
%patch00065 -p1
%patch00066 -p1
%patch00067 -p1
%patch00068 -p1
%patch00069 -p1
%patch00070 -p1
%patch00071 -p1
%patch00072 -p1
%patch00073 -p1
%patch01000 -p1
%patch01001 -p1
%patch01002 -p1

573
update_git.sh
View File

@ -14,6 +14,35 @@ source ./config.sh
declare -A COMMIT_IDS_BY_SUBMODULE_PATH
# Get version info from the packages' tarball - decode and do some checks
BASE_RE="qemu-[[:digit:]]+(\.[[:digit:]]+){2}(-rc[[:digit:]])?"
EXTRA_RE="\+git\.[[:digit:]]+\.([[:xdigit:]]+)"
SUFFIX_RE="\.tar\.xz"
SIG_SUFFIX_RE="\.tar\.xz\.sig"
QEMU_TARBALL=($(find -maxdepth 1 -type f -regextype posix-extended -regex \
"\./$BASE_RE($EXTRA_RE)?$SUFFIX_RE" -printf "%f "))
QEMU_TARBALL_SIG=($(find -maxdepth 1 -type f -regextype posix-extended -regex \
"\./$BASE_RE($EXTRA_RE)?$SIG_SUFFIX_RE" -printf "%f "))
if [ ${#QEMU_TARBALL[@]} -gt 1 ]; then
echo "Multiple qemu tarballs detected. Please clean up"
exit
fi
if [ ${#QEMU_TARBALL_SIG[@]} -gt 1 ]; then
echo "Multiple qemu tarballs signature files detected. Please clean up"
exit
fi
OLD_SOURCE_VERSION_AND_EXTRA=$(echo $QEMU_TARBALL 2>/dev/null | head --bytes=-8\
| cut --bytes=6-)
VERSION_EXTRA=$(echo $OLD_SOURCE_VERSION_AND_EXTRA|awk -F+ '{if ($2) print \
"+"$2}')
if [ "$OLD_SOURCE_VERSION_AND_EXTRA" = "" ]; then
echo "ERROR: No tarball found!"
exit
fi
#==============================================================================
TEMP_CHECK() {
# TEMPORARY! FOR NOW WE REQUIRE THESE LOCALLY TO DO WORK ON PACKAGE
REQUIRED_LOCAL_REPO_MAP=(
@ -38,6 +67,7 @@ for entry in ${REQUIRED_LOCAL_REPO_MAP[@]}; do
else
echo "ERROR! For now, you need to have these local git repos available:"
echo ${REQUIRED_LOCAL_REPO_MAP[@]}
exit
fi
done
}
@ -65,13 +95,13 @@ rm -rf $BUNDLE_DIR
mkdir -p $BUNDLE_DIR
for (( i=0; i <$SUBMODULE_COUNT; i++ )); do
mkdir -p $BUNDLE_DIR/${SUBMODULE_DIRS[$i]}
# what should this file be? for now use an extension of id
# what should this file be? for now use an extension of id
touch $BUNDLE_DIR/${SUBMODULE_DIRS[$i]}/${SUBMODULE_COMMIT_IDS[$i]}.id
done
if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then
GIT_UPSTREAM_COMMIT=$(cd ${LOCAL_REPO_MAP[0]} && git rev-parse upstream/master)
GIT_UPSTREAM_COMMIT=$NEW_COMMIT_ISH_FULL
else
# (I need to make this smarter, or change something - works for tag, but not normal commit?):
# TODO: make this smarter, or change something - works for tag, but not normal commit?
GIT_UPSTREAM_COMMIT=$(git -C ${LOCAL_REPO_MAP[0]} show-ref -d $GIT_UPSTREAM_COMMIT_ISH|grep -F "^{}"|awk '{print $1}')
fi
touch $BUNDLE_DIR/$GIT_UPSTREAM_COMMIT.id
@ -93,13 +123,15 @@ for (( i=0; i <$REPO_COUNT; i++ )); do
if [[ $(git -C $GIT_DIR/$SUBDIR ls-remote --heads origin $GIT_BRANCH) ]]; then
git -C $GIT_DIR/$SUBDIR fetch origin $GIT_BRANCH
if [[ $(git -C $GIT_DIR/$SUBDIR rev-list $GITREPO_COMMIT_ISH..FETCH_HEAD) ]]; then
git -C $GIT_DIR/$SUBDIR bundle create $BUNDLE_DIR/$SUBDIR$GITREPO_COMMIT_ISH.bundle $GITREPO_COMMIT_ISH..FETCH_HEAD || true
git -C $GIT_DIR/$SUBDIR bundle create $BUNDLE_DIR/$SUBDIR$GITREPO_COMMIT_ISH.bundle $GITREPO_COMMIT_ISH..FETCH_HEAD
fi
fi
fi
fi
done
tar cJvf bundles.tar.xz -C $BUNDLE_DIR .
# keep diffs to a minimum - touch bundle files to "something common" TODO: decide if there's something better
find $BUNDLE_DIR -exec touch -r qemu-$SOURCE_VERSION$VERSION_EXTRA.tar.xz {} \;
tar --format gnu --xz -cf bundles.tar.xz -C $BUNDLE_DIR .
rm -rf $BUNDLE_DIR
rm -rf $GIT_DIR
}
@ -129,14 +161,11 @@ for entry in ${BUNDLE_FILES[@]}; do
LOCAL_REPO=$(readlink -f ${LOCAL_REPO_MAP[$PATCH_RANGE_INDEX]})
if [ -e $LOCAL_REPO ]; then
# TODO: Detect if it's there before trying to remove!
git -C $LOCAL_REPO remote remove bundlerepo || true
# git won't let you delete this branch if it's the current branch (returns 1) HOW TO HANDLE?
# detect this case, and ask user to switch to another branch? or do it for them - switch to master killing any "state" for this branch
# git won't let you delete a branch we're on - so get onto master temporarily (TODO: is there a better approach?)
git -C $LOCAL_REPO checkout master -f
git -C $LOCAL_REPO branch -D frombundle || true
git -C $LOCAL_REPO remote add bundlerepo $BUNDLE_DIR/$entry
# in next, the head may be FETCH_HEAD or HEAD depending on how we created:
git -C $LOCAL_REPO fetch bundlerepo FETCH_HEAD
git -C $LOCAL_REPO branch frombundle FETCH_HEAD
git -C $LOCAL_REPO remote remove bundlerepo
@ -150,232 +179,122 @@ rm -rf $BUNDLE_DIR
#==============================================================================
bundle2spec() {
redo_tarball_and_rebase_patches() {
rm -rf $GIT_DIR
rm -rf $CMP_DIR
rm -rf $BUNDLE_DIR
rm -f checkpatch.log
rm -f checkthese
# there's probably a better place for the next: (only needed due to development failures?)
rm -rf checkdir
if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then
for (( i=0; i <$REPO_COUNT; i++ )); do
if [[ -e $(readlink -f ${LOCAL_REPO_MAP[$i]}) ]]; then
git -C ${LOCAL_REPO_MAP[$i]} remote update upstream &> /dev/null
fi
done
#TODO: do we really want to checkout here? the code which gets the latest submodule commits doesnt rely on this !!! IN FACT master here isn't for latest upstream - that is the upstream branch!
# git -C ${LOCAL_REPO_MAP[0]} checkout master --recurse-submodules -f
# TODO: THE FOLLOWING NEEDS HELP
QEMU_VERSION=$(git -C ${LOCAL_REPO_MAP[0]} show origin:VERSION)
MAJOR_VERSION=$(echo $QEMU_VERSION|awk -F. '{print $1}')
MINOR_VERSION=$(echo $QEMU_VERSION|awk -F. '{print $2}')
if [ "$NEXT_RELEASE_IS_MAJOR" = "0" ]; then
GIT_BRANCH=opensuse-$MAJOR_VERSION.$[$MINOR_VERSION+1]
else
GIT_BRANCH=opensuse-$[$MAJOR_VERSION+1].0
fi
fi
BASE_RE="qemu-[[:digit:]]+(\.[[:digit:]]+){2}(-rc[[:digit:]])?"
EXTRA_RE="\+git\.[[:digit:]]+\.([[:xdigit:]]+)"
SUFFIX_RE="\.tar\.xz"
SIG_SUFFIX_RE="\.tar\.xz\.sig"
QEMU_TARBALL=($(find -maxdepth 1 -type f -regextype posix-extended -regex \
"\./$BASE_RE($EXTRA_RE)?$SUFFIX_RE" -printf "%f "))
QEMU_TARBALL_SIG=($(find -maxdepth 1 -type f -regextype posix-extended -regex \
"\./$BASE_RE($EXTRA_RE)?$SIG_SUFFIX_RE" -printf "%f "))
if [ ${#QEMU_TARBALL[@]} -gt 1 ]; then
echo "Multiple qemu tarballs detected. Please clean up"
exit
fi
if [ ${#QEMU_TARBALL_SIG[@]} -gt 1 ]; then
echo "Multiple qemu tarballs signature files detected. Please clean up"
exit
fi
# It's ok for either of these to be empty when using "LATEST"
OLD_SOURCE_VERSION_AND_EXTRA=$(echo $QEMU_TARBALL 2>/dev/null | head --bytes=-8\
| cut --bytes=6-)
VERSION_EXTRA=$(echo $OLD_SOURCE_VERSION_AND_EXTRA|awk -F+ '{if ($2) print \
"+"$2}')
if [ "$OLD_SOURCE_VERSION_AND_EXTRA" = "" ]; then
echo "Warning: No tarball found"
fi
# TODO: (repo file not yet done)
if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# DO TARBALL, GETTING ALL FROM UPSTREAM DIRECTLY
# CREATE TARBALL, USING FRESH REPO - WE COULD RELY MORE ON LOCAL IF WE WERE MORE CAREFUL
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
if [[ $QEMU_TARBALL =~ $BASE_RE$EXTRA_RE$SUFFIX_RE ]]; then
OLD_COMMIT_ISH=${BASH_REMATCH[3]}
else
#Assume release (or release candidate) tarball with equivalent tag:
OLD_COMMIT_ISH=$(cd ${LOCAL_REPO_MAP[0]} && git rev-list --abbrev-commit \
--abbrev=9 -1 v$OLD_SOURCE_VERSION_AND_EXTRA)
fi
if [ ${#QEMU_TARBALL_SIG[@]} -ne 0 ]; then
echo "INFO: Ignoring signature file: $QEMU_TARBALL_SIG"
QEMU_TARBALL_SIG=
fi
# TODO: HERE WE REFERENCE MASTER - NEEDS FIXING
NEW_COMMIT_ISH_FULL=$(cd ${LOCAL_REPO_MAP[0]} && git rev-parse upstream/master)
NEW_COMMIT_ISH=$(cd ${LOCAL_REPO_MAP[0]} && git rev-parse --short=9 \
upstream/master)
NOW_SECONDS=$(date +%s)
# TODO: HERE WE REFERENCE MASTER - NEEDS FIXING
git clone -ls ${LOCAL_REPO_MAP[0]} $GIT_DIR -b master --single-branch &>/dev/null
if [ "$OLD_COMMIT_ISH" != "$NEW_COMMIT_ISH" ]; then
echo "Please wait..."
(cd $GIT_DIR && git remote add upstream \
git://git.qemu-project.org/qemu.git &>/dev/null)
(cd $GIT_DIR && git remote update upstream &>/dev/null)
(cd $GIT_DIR && git checkout $NEW_COMMIT_ISH &>/dev/null)
# TODO: WHAT IS THIS NEXT LINE EVEN DOING FOR US?? (OK, it's initing a repo, what do we rely on there?)
git clone -ls ${LOCAL_REPO_MAP[0]} $GIT_DIR -b master --single-branch &>/dev/null
echo "Please wait..."
(cd $GIT_DIR && git remote add upstream \
git://git.qemu-project.org/qemu.git &>/dev/null)
(cd $GIT_DIR && git remote update upstream &>/dev/null)
(cd $GIT_DIR && git checkout $NEW_COMMIT_ISH &>/dev/null)
# As an alternative, we could add a --recurse-submodules to the checkout instead here as well, right?
#UPSTREAM DOESNT DO THIS (time takes 17 minutes!):
# (cd $GIT_DIR && git submodule update --init --recursive &>/dev/null)
#INSTEAD THESE NEXT TWO LINES ARE WHAT IS DONE (these take 9 minutes and 3 minutes respectively):
(cd $GIT_DIR && git submodule update --init &>/dev/null)
(cd $GIT_DIR/roms/edk2 && git submodule update --init &>/dev/null)
VERSION_EXTRA=+git.$NOW_SECONDS.$NEW_COMMIT_ISH
fi
QEMU_VERSION=$(cat $GIT_DIR/VERSION)
MAJOR_VERSION=$(echo $QEMU_VERSION|awk -F. '{print $1}')
MINOR_VERSION=$(echo $QEMU_VERSION|awk -F. '{print $2}')
X=$(echo $QEMU_VERSION|awk -F. '{print $3}')
# 0 = release, 50 = development cycle, 90..99 equate to release candidates
if [ "$X" != "0" -a "$X" != "50" ]; then
if [ "$NEXT_RELEASE_IS_MAJOR" = "0" ]; then
SOURCE_VERSION=$MAJOR_VERSION.$[$MINOR_VERSION+1].0-rc$[X-90]
else
SOURCE_VERSION=$[$MAJOR_VERSION+1].0.0-rc$[X-90]
fi
(cd $GIT_DIR && git submodule update --init &>/dev/null)
(cd $GIT_DIR/roms/edk2 && git submodule update --init &>/dev/null)
VERSION_EXTRA=+git.$NOW_SECONDS.$NEW_COMMIT_ISH
if (cd ${LOCAL_REPO_MAP[0]} && git describe --exact-match $NEW_COMMIT_ISH \
&>/dev/null); then
if [ "$X" = "50" ]; then
echo "Ignoring non-standard tag"
else
SOURCE_VERSION=$MAJOR_VERSION.$MINOR_VERSION.$X
# there is no VERSION_EXTRA
VERSION_EXTRA=
fi
if [ "$OLD_COMMIT_ISH" != "$NEW_COMMIT_ISH" ]; then
if (cd ${LOCAL_REPO_MAP[0]} && git describe --exact-match $NEW_COMMIT_ISH \
&>/dev/null); then
if [ "$X" = "50" ]; then
echo "Ignoring non-standard tag"
else
# there is no VERSION_EXTRA
VERSION_EXTRA=
fi
fi
(cd $GIT_DIR/roms/seabios && git describe --tags --long --dirty > \
.version)
(cd $GIT_DIR/roms/skiboot && ./make_version.sh > .version)
echo "Almost there..."
tar --exclude=.git --transform "s,$GIT_DIR,qemu-$SOURCE_VERSION," \
-Pcf qemu-$SOURCE_VERSION$VERSION_EXTRA.tar $GIT_DIR
osc rm --force qemu-$OLD_SOURCE_VERSION_AND_EXTRA.tar.xz &>/dev/null ||\
true
osc rm --force qemu-$OLD_SOURCE_VERSION_AND_EXTRA.tar.xz.sig \
&>/dev/null || true
unset QEMU_TARBALL_SIG
xz -T 0 qemu-$SOURCE_VERSION$VERSION_EXTRA.tar
osc add qemu-$SOURCE_VERSION$VERSION_EXTRA.tar.xz
fi
(cd $GIT_DIR/roms/seabios && git describe --tags --long --dirty > \
.version)
(cd $GIT_DIR/roms/skiboot && ./make_version.sh > .version)
echo "Almost there..."
tar --exclude=.git --transform "s,$GIT_DIR,qemu-$SOURCE_VERSION," \
-Pcf qemu-$SOURCE_VERSION$VERSION_EXTRA.tar $GIT_DIR
osc rm --force qemu-$OLD_SOURCE_VERSION_AND_EXTRA.tar.xz &>/dev/null ||\
true
osc rm --force qemu-$OLD_SOURCE_VERSION_AND_EXTRA.tar.xz.sig \
&>/dev/null || true
unset QEMU_TARBALL_SIG
xz -T 0 qemu-$SOURCE_VERSION$VERSION_EXTRA.tar
osc add qemu-$SOURCE_VERSION$VERSION_EXTRA.tar.xz
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# OK GET THE SUBMODULE COMMIT ID'S FROM THIS NEWLY MINTED QEMU CHECKOUT! WE'LL USE THAT WHEN WE REBASE OUR PATCHES
# GET THE SUBMODULE COMMIT ID'S FROM THIS NEWLY MINTED QEMU CHECKOUT. WE'LL USE THAT WHEN WE REBASE OUR PATCHES
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# !! We (perhaps temporarily) do MORE recursive submodules, since we are tracking ALL in these scripts, while upstream doesn't include all in tarball currently
(cd $GIT_DIR && git submodule update --init --recursive &>/dev/null)
SUBMODULE_COMMIT_IDS=($(git -C $GIT_DIR submodule status --recursive|awk '{print $1}'))
SUBMODULE_DIRS=($(git -C $GIT_DIR submodule status --recursive|awk '{print $2}'))
SUBMODULE_COUNT=${#SUBMODULE_COMMIT_IDS[@]}
(cd $GIT_DIR && git submodule update --init --recursive &>/dev/null)
SUBMODULE_COMMIT_IDS=($(git -C $GIT_DIR submodule status --recursive|awk '{print $1}'))
SUBMODULE_DIRS=($(git -C $GIT_DIR submodule status --recursive|awk '{print $2}'))
SUBMODULE_COUNT=${#SUBMODULE_COMMIT_IDS[@]}
# TODO: do this with simply math - ie: use (( ... ))
if [[ "$REPO_COUNT" != "$(expr $SUBMODULE_COUNT + 1)" ]]; then
echo "ERROR: submodule count doesn't match the REPO_COUNT variable in config.sh file!"
exit
fi
if [[ "$REPO_COUNT" != "$(expr $SUBMODULE_COUNT + 1)" ]]; then
echo "ERROR: submodule count doesn't match the REPO_COUNT variable in config.sh file!"
exit
fi
# We have the submodule commits, but not in the PATCH ORDER which our config.sh has (see $PATCH_PATH_MAP)
for (( i=0; i <$REPO_COUNT-1; i++ )); do
COMMIT_IDS_BY_SUBMODULE_PATH[${SUBMODULE_DIRS[$i]}/]=${SUBMODULE_COMMIT_IDS[$i]}
done
COMMIT_IDS_BY_SUBMODULE_PATH[SUPERPROJECT]=$NEW_COMMIT_ISH_FULL
for (( i=0; i <$REPO_COUNT-1; i++ )); do
COMMIT_IDS_BY_SUBMODULE_PATH[${SUBMODULE_DIRS[$i]}/]=${SUBMODULE_COMMIT_IDS[$i]}
done
COMMIT_IDS_BY_SUBMODULE_PATH[SUPERPROJECT]=$NEW_COMMIT_ISH_FULL
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# MOVE BUNDLE COMMITS OVER TO LOCAL frombundle BRANCH
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
bundle2local
mkdir -p $BUNDLE_DIR
tar xJf bundles.tar.xz -C $BUNDLE_DIR
bundle2local
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# REBASE frombundle patches USING COMMIT_IDS_BY_SUBMODULE, ALSO USING OLD ID'S STORED IN OLD BUNDLE
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
mkdir -p $BUNDLE_DIR
tar xJf bundles.tar.xz -C $BUNDLE_DIR
# Now go through all the submodule local repos that are present and create a bundle file for the patches found there
for (( i=0; i <$REPO_COUNT; i++ )); do
if [[ -e $(readlink -f ${LOCAL_REPO_MAP[$i]}) ]]; then
if $(git -C ${LOCAL_REPO_MAP[$i]} branch | grep -F "frombundle" >/dev/null); then
SUBDIR=${PATCH_PATH_MAP[$i]}
GITREPO_COMMIT_ISH=($BUNDLE_DIR/$SUBDIR*.id)
if [[ $GITREPO_COMMIT_ISH =~ .*(.{40})[.]id ]]; then
GITREPO_COMMIT_ISH=${BASH_REMATCH[1]}
fi
git -C ${LOCAL_REPO_MAP[$i]} checkout frombundle -f
git -C ${LOCAL_REPO_MAP[$i]} branch -D $GIT_BRANCH
git -C ${LOCAL_REPO_MAP[$i]} checkout -b $GIT_BRANCH
if [[ "$SUBDIR" = "" ]]; then
SUBDIR=SUPERPROJECT
fi
if ! $(git -C ${LOCAL_REPO_MAP[$i]} rebase --onto ${COMMIT_IDS_BY_SUBMODULE_PATH[$SUBDIR]} $GITREPO_COMMIT_ISH >/dev/null); then
# TODO: record that this one needs manual help!
echo "Rebase of ${LOCAL_REPO_MAP[$i]}, branch $GIT_BRANCH needs manual help"
fi
fi
for (( i=0; i <$REPO_COUNT; i++ )); do
if [[ -e $(readlink -f ${LOCAL_REPO_MAP[$i]}) ]]; then
if $(git -C ${LOCAL_REPO_MAP[$i]} branch | grep -F "frombundle" >/dev/null); then
SUBDIR=${PATCH_PATH_MAP[$i]}
GITREPO_COMMIT_ISH=($BUNDLE_DIR/$SUBDIR*.id)
if [[ $GITREPO_COMMIT_ISH =~ .*(.{40})[.]id ]]; then
GITREPO_COMMIT_ISH=${BASH_REMATCH[1]}
fi
git -C ${LOCAL_REPO_MAP[$i]} checkout frombundle -f
git -C ${LOCAL_REPO_MAP[$i]} branch -D $GIT_BRANCH
git -C ${LOCAL_REPO_MAP[$i]} checkout -b $GIT_BRANCH
if [[ "$SUBDIR" = "" ]]; then
SUBDIR=SUPERPROJECT
fi
if ! $(git -C ${LOCAL_REPO_MAP[$i]} rebase --onto ${COMMIT_IDS_BY_SUBMODULE_PATH[$SUBDIR]} $GITREPO_COMMIT_ISH >/dev/null); then
# TODO: record that this one needs manual help!
echo "Rebase of ${LOCAL_REPO_MAP[$i]}, branch $GIT_BRANCH needs manual help"
REBASE_FAILS="${LOCAL_REPO_MAP[$i]} $REBASE_FAILS"
fi
done
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# CREATE BUNDLE FROM $GIT_BRANCH branch
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
initbundle
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# GET BUNDLE PATCHES FROM BUNDLE_DIR
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
fi
rm -rf $GIT_DIR
# We're done with GIT_UPSTREAM_COMMIT_ISH carrying the special value LATEST
GIT_UPSTREAM_COMMIT_ISH=$NEW_COMMIT_ISH
WRITE_LOG=0
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# DONE WITH LATEST WORK
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
else # not based on LATEST upstream master, rather any upstream commitish
if [ "$OLD_SOURCE_VERSION_AND_EXTRA" = "" ]; then
echo "Failure: tarball required which corresponds to commitish:" \
"$GIT_UPSTREAM_COMMITISH"
exit
fi
if [ -d "${LOCAL_REPO_MAP[0]}" ]; then
echo "Processing local git tree branch: master, using commitish:"\
"$GIT_UPSTREAM_COMMIT_ISH"
if ! (cd ${LOCAL_REPO_MAP[0]} && git show-branch master &>/dev/null)
then
echo "Error: Branch master not found - please create a remote"\
"tracking branch of origin/master"
exit
fi
# ( THIS ISNT WORKING - IS OLD HISTORY:)
else
echo "Processing $GIT_BRANCH branch of remote git tree, using"\
"commitish: $GIT_UPSTREAM_COMMIT_ISH"
echo "(For fast processing, consider establishing a local git tree"\
"at ${LOCAL_REPO_MAP[0]})"
fi
SOURCE_VERSION=$OLD_SOURCE_VERSION_AND_EXTRA
QEMU_VERSION=$(tar JxfO qemu-$SOURCE_VERSION$VERSION_EXTRA.tar.xz qemu-$SOURCE_VERSION/VERSION)
NEW_COMMIT_ISH=
WRITE_LOG=1
fi
done
}
#==============================================================================
bundle2spec() {
rm -f checkpatch.log
rm -f checkthese
rm -rf checkdir
rm -rf $GIT_DIR
rm -rf $CMP_DIR
rm -rf $BUNDLE_DIR
mkdir -p $BUNDLE_DIR
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# NOW PROCESS BUNDLES INTO COMMITS AND FILL SPEC FILE
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
mkdir -p $BUNDLE_DIR
tar xJf bundles.tar.xz -C $BUNDLE_DIR
BUNDLE_FILES=$(find $BUNDLE_DIR -printf "%P\n"|grep "bundle$")
@ -422,11 +341,13 @@ rm -rf $BUNDLE_DIR
shopt -s nullglob
for i in $CMP_DIR/*; do
# index line isn't consistent, so cut full index to normal line length
# index line isn't consistent, so cut full index to normal line length
sed -E -i 's/(^index [a-f0-9]{28})[a-f0-9]{12}([.][.][a-f0-9]{28})[a-f0-9]{12}( [0-9]{6}$)/\1\2\3/' $i
BASENAME=$(basename $i)
if [ "$FIVE_DIGIT_POTENTIAL" = "1" ]; then
if [[ $BASENAME =~ [[:digit:]]{4}-.* ]]; then
if [[ $BASENAME =~ [[:digit:]]{5}.* ]]; then
:
else
BASENAME=0$BASENAME
fi
fi
@ -442,7 +363,7 @@ rm -rf $BUNDLE_DIR
for i in [0-9]*.patch; do
osc rm --force $i
done
# we need to make sure that w/out the numbered prefixes, the patchnames are all unique
# make sure that w/out the numbered prefixes, the patchnames are all unique
mkdir checkdir
for i in $CMP_DIR/*; do
BASENAME=$(basename $i)
@ -530,7 +451,7 @@ rm -rf $BUNDLE_DIR
rm -f checkpatch.log
if [ "$TOTAL_COUNT" != "0" -a "$VERSION_EXTRA" != "" -a "$OLD_COMMIT_ISH" =\
"$NEW_COMMIT_ISH" ]; then
# Patches changed, so update the version using current time
# Only patches changed: update the version using current timestamp
VERSION_EXTRA=+git.$NOW_SECONDS.$OLD_COMMIT_ISH
osc mv qemu-$OLD_SOURCE_VERSION_AND_EXTRA.tar.xz \
qemu-$SOURCE_VERSION$VERSION_EXTRA.tar.xz
@ -593,7 +514,7 @@ rm -rf $BUNDLE_DIR
elif [[ "$line" =~ ^Source: ]]; then
echo "$line"
if [ ${#QEMU_TARBALL_SIG[@]} -eq 1 ]; then
# We assume the signature file corresponds - just add .sig
# We assume the signature file corresponds - just add .sig
echo "$line.sig"|sed 's/^Source: /Source99:/'
fi
elif [ "$line" = "SEABIOS_VERSION" ]; then
@ -611,7 +532,7 @@ rm -rf $BUNDLE_DIR
fi
if [ "$WRITE_LOG" = "1" ]; then
# Factory requires all deleted and added patches to be mentioned
# Factory requires all deleted and added patches to be mentioned
if [ -e qemu.changes.deleted ] || [ -e qemu.changes.added ]; then
echo "Patch queue updated from ${GIT_TREE} ${GIT_BRANCH}" > \
$package.changes.proposed
@ -639,6 +560,9 @@ rm -rf $BUNDLE_DIR
if [ -e qemu.changes.added ]; then
rm -f qemu.changes.added
fi
if [[ "0" = "$(expr $CHANGED_COUNT + $DELETED_COUNT + $ADDED_COUNT)" ]]; then
osc revert bundles.tar.xz
fi
echo "git patch summary"
echo " unchanged: $UNCHANGED_COUNT"
echo " changed: $CHANGED_COUNT"
@ -655,67 +579,190 @@ osc service localrun format_spec_file
#==============================================================================
usage() {
echo "Usage:"
echo "bash ./git_update.sh <command>
echo description: package maintenance using a git-based workflow. Commands:"
echo " git2pkg (update package spec file and patches from git. Is default)"
echo " pkg2git (update git (frombundle branch) from the package "bundleofbundles")"
echo " refresh (refresh spec file from spec file template and "bundlofbundles")"
echo "Usage:"
echo "bash ./git_update.sh <command>"
echo "description: package maintenance using a git-based workflow. Commands:"
echo " git2pkg (update package spec file and patches from git. Is default)"
echo " pkg2git (update git (frombundle branch) from the package "bundleofbundles")"
echo " refresh (refresh spec file from spec file template and "bundlofbundles")"
echo "(See script for details on doing 'LATEST' workflow)"
}
#==============================================================================
# LATEST processing currently doesn't expect cmdline params, so do it here, up front
echo "WARNING: Script using local git repos. Some operations may be time consuming..."
#TODO: Most of these checks are not necessary
for (( i=0; i <$REPO_COUNT; i++ )); do
if [[ -e $(readlink -f ${LOCAL_REPO_MAP[$i]}) ]]; then
if [[ -d ${LOCAL_REPO_MAP[$i]}/.git/rebase-merge || \
-d ${LOCAL_REPO_MAP[$i]}/.git/rebase-apply ]]; then
echo "ERROR! Rebase appears to be in progress in ${LOCAL_REPO_MAP[$i]}. Please resolve"
exit
fi
if ! git -C ${LOCAL_REPO_MAP[$i]} submodule update --init --recursive &> /dev/null; then
echo "Please clean up state of local repo ${LOCAL_REPO_MAP[$i]} before using script"
echo "(ensure git submodule update --init --recursive is successful)"
exit
fi
if [ "$(git -C ${LOCAL_REPO_MAP[$i]} status --porcelain)" ]; then
echo "Please clean up state of local repo ${LOCAL_REPO_MAP[$i]} before using script"
echo "(ensure git status --porcelain produces no output)"
exit
fi
if ! git -C ${LOCAL_REPO_MAP[$i]} checkout master --recurse-submodules -f &> /dev/null; then
echo "Please clean up state of local repo ${LOCAL_REPO_MAP[$i]} before using script"
echo "(cannot check out master, incl. it's submodules)"
exit
fi
if ! git -C ${LOCAL_REPO_MAP[$i]} submodule update --init --recursive &> /dev/null; then
echo "Please clean up state of local repo ${LOCAL_REPO_MAP[$i]} before using script"
echo "(cannot init and update master submodules)"
exit
fi
if [ "$(git -C ${LOCAL_REPO_MAP[$i]} status --porcelain)" ]; then
echo "Please clean up state of local repo ${LOCAL_REPO_MAP[$i]} before using script"
echo "(ensure git status --porcelain produces no output)"
exit
fi
fi
done
if [ "$GIT_UPSTREAM_COMMIT_ISH" = "LATEST" ]; then
echo "Processing latest upstream changes"
if [ "$1" = "continue" ]; then
CONTINUE_AFTER_REBASE=1
else
if [ "$1" = "pause" ]; then
PAUSE_BEFORE_BUNDLE_CREATION=1
else
if [ "$1" ]; then
echo "ERROR: unrecognized option '$1'. Script in LATEST mode only recognizes 'pause' and 'continue' options"
exit
fi
fi
fi
for (( i=0; i <$REPO_COUNT; i++ )); do
if [[ -e $(readlink -f ${LOCAL_REPO_MAP[$i]}) ]]; then
git -C ${LOCAL_REPO_MAP[$i]} remote update upstream &> /dev/null
fi
done
NEW_COMMIT_ISH_FULL=$(cd ${LOCAL_REPO_MAP[0]} && git rev-parse upstream/master)
NEW_COMMIT_ISH=${NEW_COMMIT_ISH_FULL:0:8}
git -C ${LOCAL_REPO_MAP[0]} checkout $NEW_COMMIT_ISH_FULL --recurse-submodules -f &> /dev/null
QEMU_VERSION=$(git -C ${LOCAL_REPO_MAP[0]} show upstream/master:VERSION)
MAJOR_VERSION=$(echo $QEMU_VERSION|awk -F. '{print $1}')
MINOR_VERSION=$(echo $QEMU_VERSION|awk -F. '{print $2}')
X=$(echo $QEMU_VERSION|awk -F. '{print $3}')
# 0 = release, 50 = development cycle, 90..99 equate to release candidates
if [ "$X" != "0" -a "$X" != "50" ]; then
if [ "$NEXT_RELEASE_IS_MAJOR" = "0" ]; then
SOURCE_VERSION=$MAJOR_VERSION.$[$MINOR_VERSION+1].0-rc$[X-90]
GIT_BRANCH=opensuse-$MAJOR_VERSION.$[$MINOR_VERSION+1]
else
SOURCE_VERSION=$[$MAJOR_VERSION+1].0.0-rc$[X-90]
GIT_BRANCH=opensuse-$[$MAJOR_VERSION+1].0
fi
else
SOURCE_VERSION=$MAJOR_VERSION.$MINOR_VERSION.$X
GIT_BRANCH=opensuse-$MAJOR_VERSION.$[$MINOR_VERSION+1]
fi
WRITE_LOG=0
echo "Processing LATEST upstream changes"
echo "(If SUCCESS is not printed upon completion, see /tmp/latest.log for issues)"
TEMP_CHECK
bundle2spec &> /tmp/latest.log
if [[ $QEMU_TARBALL =~ $BASE_RE$EXTRA_RE$SUFFIX_RE ]]; then
OLD_COMMIT_ISH=${BASH_REMATCH[3]}
else
#Assume release (or release candidate) tarball with equivalent tag:
OLD_COMMIT_ISH=$(cd ${LOCAL_REPO_MAP[0]} && git rev-list --abbrev-commit \
--abbrev=9 -1 v$OLD_SOURCE_VERSION_AND_EXTRA)
fi
if [ ${#QEMU_TARBALL_SIG[@]} -ne 0 ]; then
echo "INFO: Ignoring signature file: $QEMU_TARBALL_SIG"
QEMU_TARBALL_SIG=
fi
NOW_SECONDS=$(date +%s)
if [ "$OLD_COMMIT_ISH" != "$NEW_COMMIT_ISH" ]; then
if [ "$CONTINUE_AFTER_REBASE" = "1" ]; then
echo "continue after rebase selected but tarball is out of date. Continuing not possible."
echo "If desired, save your rebase work (eg, branch $GIT_BRANCH), because otherwise it will"
echo "be lost. Then run script again without the continue option"
exit
fi
redo_tarball_and_rebase_patches &> /tmp/latest.log
if [[ "$REBASE_FAILS" ]]; then
echo "ERROR! Rebase of the $GIT_BRANCH branch failed in the following local git repos:"
echo $REBASE_FAILS
echo "Manually resolve all these rebases, then finish the workflow by passing 'continue' to script"
if [[ "$PAUSE_BEFORE_BUNDLE_CREATION" = "1" ]]; then
echo "Feel free to also do the work now occasioned by the selected 'pause' option"
fi
exit
fi
CONTINUE_AFTER_REBASE=1
fi
if [[ "$PAUSE_BEFORE_BUNDLE_CREATION" = "1" ]]; then
echo "As requested, pausing before re-creating bundle of bundles for additional patch or specfile work"
echo "(using current 'ready to go' $GIT_BRANCH branch of local repos to produce patches.)"
echo "When changes are complete, finish the workflow by passing 'continue' to script"
exit
fi
if [ "$CONTINUE_AFTER_REBASE" = "1" ]; then
initbundle &>> /tmp/latest.log
fi
bundle2spec &>> /tmp/latest.log
echo "SUCCESS"
tail -9 /tmp/latest.log
exit
else # not LATEST
git -C ${LOCAL_REPO_MAP[0]} checkout $GIT_UPSTREAM_COMMIT_ISH --recurse-submodules -f &> /dev/null
NEW_COMMIT_ISH=
SOURCE_VERSION=$OLD_SOURCE_VERSION_AND_EXTRA
QEMU_VERSION=$(tar JxfO qemu-$SOURCE_VERSION$VERSION_EXTRA.tar.xz qemu-$SOURCE_VERSION/VERSION)
MAJOR_VERSION=$(echo $QEMU_VERSION|awk -F. '{print $1}')
MINOR_VERSION=$(echo $QEMU_VERSION|awk -F. '{print $2}')
GIT_BRANCH=opensuse-$MAJOR_VERSION.$MINOR_VERSION
WRITE_LOG=1
if [ "$1" = "" ]; then
set -- git2pkg
fi
case $1 in
initbundle )
initbundle
;;
git2pkg )
echo "Updating the package using the $GIT_BRANCH branch of the local repos."
echo "(If SUCCESS is not printed upon completion, see /tmp/git2pkg.log for issues)"
TEMP_CHECK
initbundle &> /tmp/git2pkg.log
bundle2spec &>> /tmp/git2pkg.log
echo "SUCCESS"
tail -9 /tmp/git2pkg.log
;;
pkg2git )
echo "Exporting the package's git bundles to the local repo's frombundle branches..."
echo "(If SUCCESS is not printed upon completion, see /tmp/pkg2git.log for issues)"
TEMP_CHECK
bundle2local &> /tmp/pkg2git.log
echo "SUCCESS"
echo "To modify package patches, use the frombundle branch as the basis for updating"
echo "the $GIT_BRANCH branch with the new patch queue."
echo "Then export the changes back to the package using update_git.sh git2pkg"
;;
refresh )
echo "Updating the spec file and patches from the spec file template and the bundle"
echo "of bundles (bundles.tar.xz)"
echo "(If SUCCESS is not printed upon completion, see /tmp/refresh.log for issues)"
TEMP_CHECK
bundle2spec &> /tmp/refresh.log
echo "SUCCESS"
tail -9 /tmp/refresh.log
;;
* )
echo "Unknown command"
usage
;;
help )
usage
;;
esac
fi
exit
if [ "$1" = "" ]; then
set -- git2pkg
fi
case $1 in
initbundle )
initbundle
;;
git2pkg )
echo "Updating the package from the $GIT_BRANCH branch of the local repos."
echo "(If SUCCESS is not printed upon completion, see /tmp/git2pkg.log for issues)"
TEMP_CHECK
initbundle &> /tmp/git2pkg.log
bundle2spec &>> /tmp/git2pkg.log
echo "SUCCESS"
tail -9 /tmp/git2pkg.log
;;
pkg2git )
echo "Exporting the package's git bundles to the local repo's frombundle branches..."
echo "(If SUCCESS is not printed upon completion, see /tmp/pkg2git.log for issues)"
TEMP_CHECK
bundle2local &> /tmp/pkg2git.log
echo "SUCCESS"
echo "To modify package patches, use the frombundle branch as the basis for updating"
echo "the $GIT_BRANCH branch with the new patch queue."
echo "Then export the changes back to the package using update_git.sh git2pkg"
;;
refresh )
echo "Updating the spec file and patches from the spec file template and the bundle"
echo "of bundles (bundles.tar.xz)"
echo "(If SUCCESS is not printed upon completion, see /tmp/refresh.log for issues)"
TEMP_CHECK
bundle2spec &> /tmp/refresh.log
echo "SUCCESS"
tail -9 /tmp/refresh.log
;;
* )
echo "Unknown command"
usage
;;
help )
usage
;;
esac

174
util-iov-introduce-qemu_iovec_init_exten.patch Normal file
View File

@ -0,0 +1,174 @@
From: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Date: Tue, 4 Jun 2019 19:15:03 +0300
Subject: util/iov: introduce qemu_iovec_init_extended
Git-commit: d953169d4840f312d3b9a54952f4a7ccfcb3b311
Introduce new initialization API, to create requests with padding. Will
be used in the following patch. New API uses qemu_iovec_init_buf if
resulting io vector has only one element, to avoid extra allocations.
So, we need to update qemu_iovec_destroy to support destroying such
QIOVs.
Signed-off-by: Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
Acked-by: Stefan Hajnoczi <stefanha@redhat.com>
Message-id: 20190604161514.262241-2-vsementsov@virtuozzo.com
Message-Id: <20190604161514.262241-2-vsementsov@virtuozzo.com>
Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>
Signed-off-by: Bruce Rogers <brogers@suse.com>
---
include/qemu/iov.h | 7 +++
util/iov.c | 112 +++++++++++++++++++++++++++++++++++++++++++--
2 files changed, 114 insertions(+), 5 deletions(-)
diff --git a/include/qemu/iov.h b/include/qemu/iov.h
index 48b45987b70ea28879af7989c31f..f3787a0cf768bd0ea1031913a038 100644
--- a/include/qemu/iov.h
+++ b/include/qemu/iov.h
@@ -199,6 +199,13 @@ static inline void *qemu_iovec_buf(QEMUIOVector *qiov)
void qemu_iovec_init(QEMUIOVector *qiov, int alloc_hint);
void qemu_iovec_init_external(QEMUIOVector *qiov, struct iovec *iov, int niov);
+void qemu_iovec_init_extended(
+ QEMUIOVector *qiov,
+ void *head_buf, size_t head_len,
+ QEMUIOVector *mid_qiov, size_t mid_offset, size_t mid_len,
+ void *tail_buf, size_t tail_len);
+void qemu_iovec_init_slice(QEMUIOVector *qiov, QEMUIOVector *source,
+ size_t offset, size_t len);
void qemu_iovec_add(QEMUIOVector *qiov, void *base, size_t len);
void qemu_iovec_concat(QEMUIOVector *dst,
QEMUIOVector *src, size_t soffset, size_t sbytes);
diff --git a/util/iov.c b/util/iov.c
index 74e6ca8ed7298c833e52257923c1..366ff9cdd1dee06c5d62712cb5ef 100644
--- a/util/iov.c
+++ b/util/iov.c
@@ -353,6 +353,103 @@ void qemu_iovec_concat(QEMUIOVector *dst,
qemu_iovec_concat_iov(dst, src->iov, src->niov, soffset, sbytes);
}
+/*
+ * qiov_find_iov
+ *
+ * Return pointer to iovec structure, where byte at @offset in original vector
+ * @iov exactly is.
+ * Set @remaining_offset to be offset inside that iovec to the same byte.
+ */
+static struct iovec *iov_skip_offset(struct iovec *iov, size_t offset,
+ size_t *remaining_offset)
+{
+ while (offset > 0 && offset >= iov->iov_len) {
+ offset -= iov->iov_len;
+ iov++;
+ }
+ *remaining_offset = offset;
+
+ return iov;
+}
+
+/*
+ * qiov_slice
+ *
+ * Find subarray of iovec's, containing requested range. @head would
+ * be offset in first iov (returned by the function), @tail would be
+ * count of extra bytes in last iovec (returned iov + @niov - 1).
+ */
+static struct iovec *qiov_slice(QEMUIOVector *qiov,
+ size_t offset, size_t len,
+ size_t *head, size_t *tail, int *niov)
+{
+ struct iovec *iov, *end_iov;
+
+ assert(offset + len <= qiov->size);
+
+ iov = iov_skip_offset(qiov->iov, offset, head);
+ end_iov = iov_skip_offset(iov, *head + len, tail);
+
+ if (*tail > 0) {
+ assert(*tail < end_iov->iov_len);
+ *tail = end_iov->iov_len - *tail;
+ end_iov++;
+ }
+
+ *niov = end_iov - iov;
+
+ return iov;
+}
+
+/*
+ * Compile new iovec, combining @head_buf buffer, sub-qiov of @mid_qiov,
+ * and @tail_buf buffer into new qiov.
+ */
+void qemu_iovec_init_extended(
+ QEMUIOVector *qiov,
+ void *head_buf, size_t head_len,
+ QEMUIOVector *mid_qiov, size_t mid_offset, size_t mid_len,
+ void *tail_buf, size_t tail_len)
+{
+ size_t mid_head, mid_tail;
+ int total_niov, mid_niov = 0;
+ struct iovec *p, *mid_iov;
+
+ if (mid_len) {
+ mid_iov = qiov_slice(mid_qiov, mid_offset, mid_len,
+ &mid_head, &mid_tail, &mid_niov);
+ }
+
+ total_niov = !!head_len + mid_niov + !!tail_len;
+ if (total_niov == 1) {
+ qemu_iovec_init_buf(qiov, NULL, 0);
+ p = &qiov->local_iov;
+ } else {
+ qiov->niov = qiov->nalloc = total_niov;
+ qiov->size = head_len + mid_len + tail_len;
+ p = qiov->iov = g_new(struct iovec, qiov->niov);
+ }
+
+ if (head_len) {
+ p->iov_base = head_buf;
+ p->iov_len = head_len;
+ p++;
+ }
+
+ if (mid_len) {
+ memcpy(p, mid_iov, mid_niov * sizeof(*p));
+ p[0].iov_base = (uint8_t *)p[0].iov_base + mid_head;
+ p[0].iov_len -= mid_head;
+ p[mid_niov - 1].iov_len -= mid_tail;
+ p += mid_niov;
+ }
+
+ if (tail_len) {
+ p->iov_base = tail_buf;
+ p->iov_len = tail_len;
+ }
+}
+
/*
* Check if the contents of the iovecs are all zero
*/
@@ -374,14 +471,19 @@ bool qemu_iovec_is_zero(QEMUIOVector *qiov)
return true;
}
+void qemu_iovec_init_slice(QEMUIOVector *qiov, QEMUIOVector *source,
+ size_t offset, size_t len)
+{
+ qemu_iovec_init_extended(qiov, NULL, 0, source, offset, len, NULL, 0);
+}
+
void qemu_iovec_destroy(QEMUIOVector *qiov)
{
- assert(qiov->nalloc != -1);
+ if (qiov->nalloc != -1) {
+ g_free(qiov->iov);
+ }
- qemu_iovec_reset(qiov);
- g_free(qiov->iov);
- qiov->nalloc = 0;
- qiov->iov = NULL;
+ memset(qiov, 0, sizeof(*qiov));
}
void qemu_iovec_reset(QEMUIOVector *qiov)