user: Olaf Hering date: Thu Mar 28 15:42:14 2013 +0100 files: docs/man/xl.pod.1 tools/libxc/Makefile tools/libxc/xc_domain_save.c tools/libxc/xc_nomigrate.c tools/libxc/xenguest.h tools/libxl/libxl.c tools/libxl/libxl.h tools/libxl/libxl_internal.h tools/libxl/libxl_save_callout.c tools/libxl/libxl_save_helper.c tools/libxl/xl_cmdimpl.c tools/libxl/xl_cmdtable.c tools/python/xen/lowlevel/checkpoint/libcheckpoint.c tools/python/xen/xend/XendCheckpoint.py tools/python/xen/xend/XendDomain.py tools/python/xen/xend/XendDomainInfo.py tools/python/xen/xm/migrate.py tools/xcutils/xc_save.c description: tools: set number of dirty pages during migration If a guest is really busy it will not reach the low number of remaining 50 dirty pages for the final suspend. As a result the guest is either suspendend for a long time during the final transfer, or if the number of iterations is increased the migration will take a long time. Add a new option xm/xl migrate --min_remaing to increase the default from command line. The default of 50 is 200kb, which is appearently an arbitrary number. With todays network speeds a larger block of memory can be transfered quickly without causing too much suspension time. This knob gives the admin the chance to adapt the suspension time to the given workload. The existing default of 50 pages is not altered by this change. Signed-off-by: Olaf Hering --- tools/libxc/xc_domain_save.c | 6 ++++-- tools/libxc/xc_nomigrate.c | 2 +- tools/libxc/xenguest.h | 2 +- tools/libxl/libxl_save_helper.c | 2 +- tools/python/xen/lowlevel/checkpoint/libcheckpoint.c | 2 +- tools/python/xen/xend/XendCheckpoint.py | 5 ++++- tools/python/xen/xend/XendDomain.py | 5 +++-- tools/python/xen/xend/XendDomainInfo.py | 8 +++++--- tools/python/xen/xm/migrate.py | 5 +++++ tools/xcutils/xc_save.c | 11 ++++++----- 11 files changed, 32 insertions(+), 18 deletions(-) Index: xen-4.4.0-testing/tools/libxc/xc_domain_save.c =================================================================== --- xen-4.4.0-testing.orig/tools/libxc/xc_domain_save.c +++ xen-4.4.0-testing/tools/libxc/xc_domain_save.c @@ -43,6 +43,7 @@ */ #define DEF_MAX_ITERS 29 /* limit us to 30 times round loop */ #define DEF_MAX_FACTOR 3 /* never send more than 3x p2m_size */ +#define DEF_MIN_REMAINING 50 /* low water mark of dirty pages */ struct save_ctx { unsigned long hvirt_start; /* virtual starting address of the hypervisor */ @@ -799,7 +800,7 @@ static int save_tsc_info(xc_interface *x } int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters, - uint32_t max_factor, uint32_t flags, + uint32_t max_factor, uint32_t min_remaining, uint32_t flags, struct save_callbacks* callbacks, int hvm, unsigned long vm_generationid_addr) { @@ -910,6 +911,7 @@ int xc_domain_save(xc_interface *xch, in /* If no explicit control parameters given, use defaults */ max_iters = max_iters ? : DEF_MAX_ITERS; max_factor = max_factor ? : DEF_MAX_FACTOR; + min_remaining = min_remaining ? : DEF_MIN_REMAINING; if ( !get_platform_info(xch, dom, &ctx->max_mfn, &ctx->hvirt_start, &ctx->pt_levels, &dinfo->guest_width) ) @@ -1533,7 +1535,7 @@ int xc_domain_save(xc_interface *xch, in if ( live ) { - int min_reached = sent_this_iter + skip_this_iter < 50; + int min_reached = sent_this_iter + skip_this_iter < min_remaining; if ( (iter >= max_iters) || min_reached || (total_sent > dinfo->p2m_size*max_factor) ) Index: xen-4.4.0-testing/tools/libxc/xc_nomigrate.c =================================================================== --- xen-4.4.0-testing.orig/tools/libxc/xc_nomigrate.c +++ xen-4.4.0-testing/tools/libxc/xc_nomigrate.c @@ -22,7 +22,7 @@ #include int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters, - uint32_t max_factor, uint32_t flags, + uint32_t max_factor, uint32_t min_remaining, uint32_t flags, struct save_callbacks* callbacks, int hvm, unsigned long vm_generationid_addr) { Index: xen-4.4.0-testing/tools/libxc/xenguest.h =================================================================== --- xen-4.4.0-testing.orig/tools/libxc/xenguest.h +++ xen-4.4.0-testing/tools/libxc/xenguest.h @@ -87,7 +87,7 @@ struct save_callbacks { * @return 0 on success, -1 on failure */ int xc_domain_save(xc_interface *xch, int io_fd, uint32_t dom, uint32_t max_iters, - uint32_t max_factor, uint32_t flags /* XCFLAGS_xxx */, + uint32_t max_factor, uint32_t min_remaining, uint32_t flags /* XCFLAGS_xxx */, struct save_callbacks* callbacks, int hvm, unsigned long vm_generationid_addr); Index: xen-4.4.0-testing/tools/libxl/libxl_save_helper.c =================================================================== --- xen-4.4.0-testing.orig/tools/libxl/libxl_save_helper.c +++ xen-4.4.0-testing/tools/libxl/libxl_save_helper.c @@ -235,7 +235,7 @@ int main(int argc, char **argv) helper_setcallbacks_save(&helper_save_callbacks, cbflags); startup("save"); - r = xc_domain_save(xch, io_fd, dom, max_iters, max_factor, flags, + r = xc_domain_save(xch, io_fd, dom, max_iters, max_factor, 0, flags, &helper_save_callbacks, hvm, genidad); complete(r); Index: xen-4.4.0-testing/tools/python/xen/lowlevel/checkpoint/libcheckpoint.c =================================================================== --- xen-4.4.0-testing.orig/tools/python/xen/lowlevel/checkpoint/libcheckpoint.c +++ xen-4.4.0-testing/tools/python/xen/lowlevel/checkpoint/libcheckpoint.c @@ -206,7 +206,7 @@ int checkpoint_start(checkpoint_state* s callbacks->switch_qemu_logdirty = noop_switch_logdirty; - rc = xc_domain_save(s->xch, fd, s->domid, 0, 0, flags, callbacks, hvm, + rc = xc_domain_save(s->xch, fd, s->domid, 0, 0, 0, flags, callbacks, hvm, vm_generationid_addr); if (hvm) Index: xen-4.4.0-testing/tools/python/xen/xend/XendCheckpoint.py =================================================================== --- xen-4.4.0-testing.orig/tools/python/xen/xend/XendCheckpoint.py +++ xen-4.4.0-testing/tools/python/xen/xend/XendCheckpoint.py @@ -120,19 +120,22 @@ def save(fd, dominfo, network, live, dst # more information. max_iters = dominfo.info.get('max_iters', "0") max_factor = dominfo.info.get('max_factor', "0") + min_remaining = dominfo.info.get('min_remaining', "0") abort_if_busy = dominfo.info.get('abort_if_busy', "0") log_save_progress = dominfo.info.get('log_save_progress', "0") if max_iters == "None": max_iters = "0" if max_factor == "None": max_factor = "0" + if min_remaining == "None": + min_remaining = "0" if abort_if_busy == "None": abort_if_busy = "0" if log_save_progress == "None": log_save_progress = "0" cmd = [xen.util.auxbin.pathTo(XC_SAVE), str(fd), str(dominfo.getDomid()), - max_iters, max_factor, + max_iters, max_factor, min_remaining, str( int(live) | (int(hvm) << 2) | (int(abort_if_busy) << 5) | (int(log_save_progress) << 6) ) ] log.debug("[xc_save]: %s", string.join(cmd)) Index: xen-4.4.0-testing/tools/python/xen/xend/XendDomain.py =================================================================== --- xen-4.4.0-testing.orig/tools/python/xen/xend/XendDomain.py +++ xen-4.4.0-testing/tools/python/xen/xend/XendDomain.py @@ -1832,18 +1832,19 @@ class XendDomain: log.exception(ex) raise XendError(str(ex)) - def domain_migrate_constraints_set(self, domid, max_iters, max_factor, abort_if_busy, log_save_progress): + def domain_migrate_constraints_set(self, domid, max_iters, max_factor, min_remaining, abort_if_busy, log_save_progress): """Set the Migrate Constraints of this domain. @param domid: Domain ID or Name @param max_iters: Number of iterations before final suspend @param max_factor: Max amount of memory to transfer before final suspend + @param min_remaining: Number of dirty pages before final suspend @param abort_if_busy: Abort migration instead of doing final suspend @param log_save_progress: Log progress of migrate to xend.log """ dominfo = self.domain_lookup_nr(domid) if not dominfo: raise XendInvalidDomain(str(domid)) - dominfo.setMigrateConstraints(max_iters, max_factor, abort_if_busy, log_save_progress) + dominfo.setMigrateConstraints(max_iters, max_factor, min_remaining, abort_if_busy, log_save_progress) def domain_maxmem_set(self, domid, mem): """Set the memory limit for a domain. Index: xen-4.4.0-testing/tools/python/xen/xend/XendDomainInfo.py =================================================================== --- xen-4.4.0-testing.orig/tools/python/xen/xend/XendDomainInfo.py +++ xen-4.4.0-testing/tools/python/xen/xend/XendDomainInfo.py @@ -1475,17 +1475,19 @@ class XendDomainInfo: break xen.xend.XendDomain.instance().managed_config_save(self) - def setMigrateConstraints(self, max_iters, max_factor, abort_if_busy, log_save_progress): + def setMigrateConstraints(self, max_iters, max_factor, min_remaining, abort_if_busy, log_save_progress): """Set the Migrate Constraints of this domain. @param max_iters: Number of iterations before final suspend @param max_factor: Max amount of memory to transfer before final suspend + @param min_remaining: Number of dirty pages before final suspend @param abort_if_busy: Abort migration instead of doing final suspend @param log_save_progress: Log progress of migrate to xend.log """ - log.debug("Setting migration constraints of domain %s (%s) to '%s' '%s' '%s'.", - self.info['name_label'], str(self.domid), max_iters, max_factor, abort_if_busy) + log.debug("Setting migration constraints of domain %s (%s) to '%s' '%s' '%s' '%s'.", + self.info['name_label'], str(self.domid), max_iters, max_factor, min_remaining, abort_if_busy) self.info['max_iters'] = str(max_iters) self.info['max_factor'] = str(max_factor) + self.info['min_remaining'] = str(min_remaining) self.info['abort_if_busy'] = str(abort_if_busy) self.info['log_save_progress'] = str(log_save_progress) Index: xen-4.4.0-testing/tools/python/xen/xm/migrate.py =================================================================== --- xen-4.4.0-testing.orig/tools/python/xen/xm/migrate.py +++ xen-4.4.0-testing/tools/python/xen/xm/migrate.py @@ -63,6 +63,10 @@ gopts.opt('max_factor', val='max_factor' fn=set_int, default=0, use="Max amount of memory to transfer before final suspend (default: 3*RAM).") +gopts.opt('min_remaining', val='min_remaining', + fn=set_int, default=0, + use="Number of dirty pages before final suspend (default: 50).") + gopts.opt('abort_if_busy', fn=set_true, default=0, use="Abort migration instead of doing final suspend.") @@ -99,6 +103,7 @@ def main(argv): server.xend.domain.migrate_constraints_set(dom, opts.vals.max_iters, opts.vals.max_factor, + opts.vals.min_remaining, opts.vals.abort_if_busy, opts.vals.log_progress) server.xend.domain.migrate(dom, dst, opts.vals.live, Index: xen-4.4.0-testing/tools/xcutils/xc_save.c =================================================================== --- xen-4.4.0-testing.orig/tools/xcutils/xc_save.c +++ xen-4.4.0-testing/tools/xcutils/xc_save.c @@ -166,20 +166,21 @@ static int switch_qemu_logdirty(int domi int main(int argc, char **argv) { - unsigned int maxit, max_f, lflags; + unsigned int maxit, max_f, min_r, lflags; int io_fd, ret, port; struct save_callbacks callbacks; xentoollog_level lvl; xentoollog_logger *l; - if (argc != 6) - errx(1, "usage: %s iofd domid maxit maxf flags", argv[0]); + if (argc != 7) + errx(1, "usage: %s iofd domid maxit maxf minr flags", argv[0]); io_fd = atoi(argv[1]); si.domid = atoi(argv[2]); maxit = atoi(argv[3]); max_f = atoi(argv[4]); - si.flags = atoi(argv[5]); + min_r = atoi(argv[5]); + si.flags = atoi(argv[6]); si.suspend_evtchn = -1; @@ -213,7 +214,7 @@ main(int argc, char **argv) memset(&callbacks, 0, sizeof(callbacks)); callbacks.suspend = suspend; callbacks.switch_qemu_logdirty = switch_qemu_logdirty; - ret = xc_domain_save(si.xch, io_fd, si.domid, maxit, max_f, si.flags, + ret = xc_domain_save(si.xch, io_fd, si.domid, maxit, max_f, min_r, si.flags, &callbacks, !!(si.flags & XCFLAGS_HVM), 0); if (si.suspend_evtchn > 0)