From a628ea8ccacd773f9a4380af7c10b8f18ecd596e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Tue, 3 Dec 2019 15:05:48 +0100 Subject: [PATCH 1/6] read debug packages from server side project configuration => avoids unresolvable errors when distro is not providing the extra/debug package --- NEWS | 2 ++ osc/build.py | 22 +++++++++++++++------- osc/commandline.py | 2 ++ osc/conf.py | 7 +++++-- 4 files changed, 24 insertions(+), 9 deletions(-) diff --git a/NEWS b/NEWS index 983154b7..c1523f1d 100644 --- a/NEWS +++ b/NEWS @@ -1,6 +1,8 @@ 0.167 - osc shell/chroot/wipe is now handled via build script (working for chroot and KVM only atm) - osc build --vm-type=qemu support for cross architecture builds + - osc build is reading debug packages from prjconf now + (extra-pkgs definition is therefore by default empty) 0.166.2 - Don't enforce password reuse (boo#1156501) diff --git a/osc/build.py b/osc/build.py index 28f97b54..6a33840f 100644 --- a/osc/build.py +++ b/osc/build.py @@ -797,8 +797,18 @@ def main(apiurl, opts, argv): if not os.path.isfile(bc_filename): raise oscerr.WrongOptions('--offline is not possible, no local buildconfig file') else: - print('Getting buildinfo from server and store to %s' % bi_filename) + print('Getting buildconfig from server and store to %s' % bc_filename) + bc = get_buildconfig(apiurl, prj, repo) + if not bc_file: + bc_file = open(bc_filename, 'w') + bc_file.write(decode_it(bc)) + bc_file.flush() + if os.path.exists('/usr/lib/build/queryconfig') and not opts.nodebugpackages: + debug_pkgs = decode_it(return_external('/usr/lib/build/queryconfig', '--dist', bc_filename, 'substitute', 'obs:cli_debug_packages')) + if len(debug_pkgs) > 0: + extra_pkgs += debug_pkgs.strip().split(" ") + print('Getting buildinfo from server and store to %s' % bi_filename) bi_text = decode_it(get_buildinfo(apiurl, prj, pac, @@ -814,12 +824,10 @@ def main(apiurl, opts, argv): kiwipath = None if build_type == 'kiwi': kiwipath = get_kiwipath_from_buildinfo(apiurl, bi_filename, prj, repo) - print('Getting buildconfig from server and store to %s' % bc_filename) - bc = get_buildconfig(apiurl, prj, repo, kiwipath) - if not bc_file: - bc_file = open(bc_filename, 'w') - bc_file.write(decode_it(bc)) - bc_file.flush() + bc = get_buildconfig(apiurl, prj, repo, kiwipath) + bc_file.seek(0) + bc_file.write(decode_it(bc)) + bc_file.flush() except HTTPError as e: if e.code == 404: # check what caused the 404 diff --git a/osc/commandline.py b/osc/commandline.py index 81c72de4..c2bea1bb 100644 --- a/osc/commandline.py +++ b/osc/commandline.py @@ -6263,6 +6263,8 @@ Please submit there instead, or use --nodevelproject to force direct submission. help='Do not run build checks on the resulting packages.') @cmdln.option('--no-verify', '--noverify', action='store_true', help='Skip signature verification (via pgp keys) of packages used for build. (Global config in oscrc: no_verify)') + @cmdln.option('--nodebugpackages', '--no-debug-packages', action='store_true', + help='Skip installation of additional debug packages for CLI builds') @cmdln.option('--noservice', '--no-service', action='store_true', help='Skip run of local source services as specified in _service file.') @cmdln.option('-p', '--prefer-pkgs', metavar='DIR', action='append', diff --git a/osc/conf.py b/osc/conf.py index b8cae34b..a289083c 100644 --- a/osc/conf.py +++ b/osc/conf.py @@ -278,8 +278,11 @@ apiurl = %(apiurl)s # extra packages to install when building packages locally (osc build) # this corresponds to osc build's -x option and can be overridden with that # -x '' can also be given on the command line to override this setting, or -# you can have an empty setting here. -#extra-pkgs = vim gdb strace +# you can have an empty setting here. This global setting may leads to +# dependency problems when the base distro is not providing the package. +# => using server side definition via cli_debug_packages substitute rule is +# recommended therefore. +#extra-pkgs = # build platform is used if the platform argument is omitted to osc build #build_repository = %(build_repository)s From eb7aab1cf7b30acd78380bad522f5f07765f7b1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Tue, 3 Dec 2019 15:08:41 +0100 Subject: [PATCH 2/6] osc build --vm-disk-size= switch support --- NEWS | 1 + osc/build.py | 7 +++++-- osc/commandline.py | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index c1523f1d..1f3f2a75 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,7 @@ - osc build --vm-type=qemu support for cross architecture builds - osc build is reading debug packages from prjconf now (extra-pkgs definition is therefore by default empty) + - osc build --vm-disk-size= switch support 0.166.2 - Don't enforce password reuse (boo#1156501) diff --git a/osc/build.py b/osc/build.py index 6a33840f..f2d2ae1c 100644 --- a/osc/build.py +++ b/osc/build.py @@ -531,6 +531,7 @@ def main(apiurl, opts, argv): cache_dir = None build_uid = '' vm_memory = config['build-memory'] + vm_disk_size = config['build-vmdisk-rootsize'] vm_type = config['build-type'] vm_telnet = None @@ -620,6 +621,8 @@ def main(apiurl, opts, argv): return 1 if opts.vm_memory: vm_memory = opts.vm_memory + if opts.vm_disk_size: + vm_disk_size = opts.vm_disk_size if opts.vm_type: vm_type = opts.vm_type if opts.vm_telnet: @@ -1225,9 +1228,9 @@ def main(apiurl, opts, argv): vm_options += [ '--vm-initrd=' + config['build-initrd'] ] build_root += '/.mount' + if vm_disk_size: + vm_options += [ '--vmdisk-rootsize=' + vm_disk_size ] - if config['build-vmdisk-rootsize']: - vm_options += [ '--vmdisk-rootsize=' + config['build-vmdisk-rootsize'] ] if config['build-vmdisk-swapsize']: vm_options += [ '--vmdisk-swapsize=' + config['build-vmdisk-swapsize'] ] if config['build-vmdisk-filesystem']: diff --git a/osc/commandline.py b/osc/commandline.py index c2bea1bb..313549e1 100644 --- a/osc/commandline.py +++ b/osc/commandline.py @@ -6305,7 +6305,9 @@ Please submit there instead, or use --nodevelproject to force direct submission. @cmdln.option('--linksources', action='store_true', help='use hard links instead of a deep copied source') @cmdln.option('--vm-memory', metavar='MEMORY', - help='use given MB for VM') + help='amount of memory for VM defined in MB') + @cmdln.option('--vm-disk-size', metavar='DISKSIZE', + help='size for newly created disk image in MB') @cmdln.option('--vm-type', metavar='TYPE', help='use VM type TYPE (e.g. kvm)') @cmdln.option('--vm-telnet', metavar='TELNET', From f4bdd3c8763005cc96c00092d61cc4504f8f33e0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Tue, 3 Dec 2019 15:17:38 +0100 Subject: [PATCH 3/6] avoid implicit --noinit if extra packages are specified --- osc/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osc/build.py b/osc/build.py index f2d2ae1c..9771acd8 100644 --- a/osc/build.py +++ b/osc/build.py @@ -676,7 +676,7 @@ def main(apiurl, opts, argv): if opts.shell: buildargs.append("--shell") - if os.path.exists(build_root) and not opts.clean: + if os.path.exists(build_root) and not opts.clean and not opts.extra_pkgs: opts.noinit = True opts.offline = True From 47e346dde85089723aeafc6259ed7145eeda2a52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Wed, 4 Dec 2019 08:41:15 +0100 Subject: [PATCH 4/6] avoid source service run for "osc shell" this also drops the warning line if service disabled. It seems not to be important enough to me. --- osc/build.py | 45 ++++++++++++++++++++++++++++----------------- osc/commandline.py | 12 ------------ 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/osc/build.py b/osc/build.py index 9771acd8..6d991a2b 100644 --- a/osc/build.py +++ b/osc/build.py @@ -674,15 +674,42 @@ def main(apiurl, opts, argv): except: pass + # define buildinfo & config local cache + bi_file = None + bc_file = None + bi_filename = '_buildinfo-%s-%s.xml' % (repo, arch) + bc_filename = '_buildconfig-%s-%s' % (repo, arch) + if is_package_dir('.') and os.access(osc.core.store, os.W_OK): + bi_filename = os.path.join(os.getcwd(), osc.core.store, bi_filename) + bc_filename = os.path.join(os.getcwd(), osc.core.store, bc_filename) + elif not os.access('.', os.W_OK): + bi_file = NamedTemporaryFile(prefix=bi_filename) + bi_filename = bi_file.name + bc_file = NamedTemporaryFile(prefix=bc_filename) + bc_filename = bc_file.name + else: + bi_filename = os.path.abspath(bi_filename) + bc_filename = os.path.abspath(bc_filename) + if opts.shell: buildargs.append("--shell") - if os.path.exists(build_root) and not opts.clean and not opts.extra_pkgs: + if os.path.exists(build_root) and os.path.exists(bi_filename) and not opts.clean and not opts.extra_pkgs: opts.noinit = True opts.offline = True + # we should check if the service did run before and only skip it then, + # but we have no save point for this atm + opts.noservice = True if opts.noinit: buildargs.append('--noinit') + # check for source services + if not opts.offline and not opts.noservice: + p = Package('.') + r = p.run_source_services(verbose=True) + if r: + raise oscerr.ServiceRuntimeError('Source service run failed!') + cache_dir = config['packagecachedir'] % {'apihost': apihost} extra_pkgs = [] @@ -769,22 +796,6 @@ def main(apiurl, opts, argv): raise oscerr.WrongOptions('--overlay %s is no valid directory!' % opts.overlay) specialcmdopts += ['--overlay='+myoverlay] - bi_file = None - bc_file = None - bi_filename = '_buildinfo-%s-%s.xml' % (repo, arch) - bc_filename = '_buildconfig-%s-%s' % (repo, arch) - if is_package_dir('.') and os.access(osc.core.store, os.W_OK): - bi_filename = os.path.join(os.getcwd(), osc.core.store, bi_filename) - bc_filename = os.path.join(os.getcwd(), osc.core.store, bc_filename) - elif not os.access('.', os.W_OK): - bi_file = NamedTemporaryFile(prefix=bi_filename) - bi_filename = bi_file.name - bc_file = NamedTemporaryFile(prefix=bc_filename) - bc_filename = bc_file.name - else: - bi_filename = os.path.abspath(bi_filename) - bc_filename = os.path.abspath(bc_filename) - try: if opts.noinit: if not os.path.isfile(bi_filename): diff --git a/osc/commandline.py b/osc/commandline.py index 313549e1..6c38e5c1 100644 --- a/osc/commandline.py +++ b/osc/commandline.py @@ -6441,18 +6441,6 @@ Please submit there instead, or use --nodevelproject to force direct submission. args = self.parse_repoarchdescr(args, opts.noinit or opts.offline, opts.alternative_project, False, opts.vm_type, opts.multibuild_package) - # check for source services - if not opts.offline and not opts.noservice: - p = Package('.') - r = p.run_source_services(verbose=True) - if r: - print('Source service run failed!', file=sys.stderr) - sys.exit(1) - else: - msg = ('WARNING: source services from package or project will not' - 'be executed. This may not be the same build as on server!') - print(msg) - if not opts.local_package: try: package = store_read_package(os.curdir) From b329b748162899d6ed837b62efc9d3f14567b843 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Wed, 4 Dec 2019 09:45:17 +0100 Subject: [PATCH 5/6] build environement parameter caching repository, architecture and vm_type from last build is automatically reused if not specified otherwise. --- NEWS | 11 +++++++---- osc/build.py | 5 ++++- osc/commandline.py | 11 ++++++++++- osc/core.py | 16 +++++++++++++++- 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/NEWS b/NEWS index 1f3f2a75..869b25d0 100644 --- a/NEWS +++ b/NEWS @@ -1,9 +1,12 @@ 0.167 - osc shell/chroot/wipe is now handled via build script (working for chroot and KVM only atm) - - osc build --vm-type=qemu support for cross architecture builds - - osc build is reading debug packages from prjconf now - (extra-pkgs definition is therefore by default empty) - - osc build --vm-disk-size= switch support + - osc build/shell features: + * --vm-type=qemu support for cross architecture builds + * additional debug packages are read from build config now + (extra-pkgs definition is therefore by default empty now) + * --vm-disk-size= switch supported (has only an effect on clean builds) + * repository, architecture and vm_type from last build is automatically + reused if not specified otherwise. 0.166.2 - Don't enforce password reuse (boo#1156501) diff --git a/osc/build.py b/osc/build.py index 6d991a2b..75fd9777 100644 --- a/osc/build.py +++ b/osc/build.py @@ -705,7 +705,7 @@ def main(apiurl, opts, argv): # check for source services if not opts.offline and not opts.noservice: - p = Package('.') + p = osc.core.Package(os.curdir) r = p.run_source_services(verbose=True) if r: raise oscerr.ServiceRuntimeError('Source service run failed!') @@ -1275,6 +1275,9 @@ def main(apiurl, opts, argv): if hostarch != bi.buildarch and bi.buildarch in change_personality: cmd = [ change_personality[bi.buildarch] ] + cmd + # record our settings for later builds + osc.core.store_write_last_buildroot(os.curdir, repo, arch, vm_type) + try: rc = run_external(cmd[0], *cmd[1:]) if rc: diff --git a/osc/commandline.py b/osc/commandline.py index 6c38e5c1..04704b81 100644 --- a/osc/commandline.py +++ b/osc/commandline.py @@ -6115,7 +6115,8 @@ Please submit there instead, or use --nodevelproject to force direct submission. # it seems to be an architecture in general arg_arch = arg if not (arg == osc.build.hostarch or arg in osc.build.can_also_build.get(osc.build.hostarch, [])): - print("WARNING: native compile is not possible, an emulator must be configured!") + if not (vm_type == 'qemu' or vm_type == 'emulator'): + print("WARNING: native compile is not possible, a emulator via binfmt misc handler must be configured!") elif not arg_repository: arg_repository = arg else: @@ -6439,6 +6440,14 @@ Please submit there instead, or use --nodevelproject to force direct submission. if project == opts.alternative_project: opts.alternative_project = None + if len(args) == 0: + # build env not specified, just read from last build attempt + lastbuildroot = store_read_last_buildroot(os.curdir) + if lastbuildroot: + args = [ lastbuildroot[0], lastbuildroot[1] ] + if not opts.vm_type: + opts.vm_type = lastbuildroot[2] + args = self.parse_repoarchdescr(args, opts.noinit or opts.offline, opts.alternative_project, False, opts.vm_type, opts.multibuild_package) if not opts.local_package: diff --git a/osc/core.py b/osc/core.py index b1289e48..361bae5e 100644 --- a/osc/core.py +++ b/osc/core.py @@ -1140,7 +1140,7 @@ class Package: REQ_STOREFILES = ('_project', '_package', '_apiurl', '_files', '_osclib_version') OPT_STOREFILES = ('_to_be_added', '_to_be_deleted', '_in_conflict', '_in_update', '_in_commit', '_meta', '_meta_mode', '_frozenlink', '_pulled', '_linkrepair', - '_size_limit', '_commit_msg') + '_size_limit', '_commit_msg', '_last_buildroot') def __init__(self, workingdir, progress_obj=None, size_limit=None, wc_check=True): global store @@ -6508,6 +6508,17 @@ def store_read_apiurl(dir, defaulturl=True): apiurl = conf.config['apiurl'] return apiurl +def store_read_last_buildroot(dir): + global store + + fname = os.path.join(dir, store, '_last_buildroot') + if os.path.exists(fname): + lines = open(fname).read().splitlines() + if len(lines) == 3: + return lines + + return + def store_write_string(dir, file, string, subdir=''): global store @@ -6532,6 +6543,9 @@ def store_write_project(dir, project): def store_write_apiurl(dir, apiurl): store_write_string(dir, '_apiurl', apiurl + '\n') +def store_write_last_buildroot(dir, repo, arch, vm_type): + store_write_string(dir, '_last_buildroot', repo + '\n' + arch + '\n' + vm_type + '\n') + def store_unlink_file(dir, file): global store From b08bd8a43703d28228147ea13bda243d73cb4fe7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adrian=20Schr=C3=B6ter?= Date: Wed, 4 Dec 2019 14:37:31 +0100 Subject: [PATCH 6/6] support building for kiwi products using obsrepositories:/ --- NEWS | 1 + osc/build.py | 21 ++++++++++++++++++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/NEWS b/NEWS index 869b25d0..ea0818c5 100644 --- a/NEWS +++ b/NEWS @@ -7,6 +7,7 @@ * --vm-disk-size= switch supported (has only an effect on clean builds) * repository, architecture and vm_type from last build is automatically reused if not specified otherwise. + * support building for kiwi products using obsrepositories:/ 0.166.2 - Don't enforce password reuse (boo#1156501) diff --git a/osc/build.py b/osc/build.py index 75fd9777..5c70d063 100644 --- a/osc/build.py +++ b/osc/build.py @@ -1094,11 +1094,26 @@ def main(apiurl, opts, argv): print(open(build_descr).read(), file=sys.stderr) sys.exit(1) root = tree.getroot() + # product for xml in root.findall('instsource'): - if xml.find('instrepo').find('source').get('path') == 'obsrepositories:/': - print("obsrepositories:/ for product builds is not yet supported in osc!") - sys.exit(1) + found_obsrepositories = 0 + for node in xml.findall('instrepo'): + if node and node.find('source').get('path') == 'obsrepositories:/': + found_obsrepositories = found_obsrepositories + 1 + for path in bi.pathes: + new_node = ET.SubElement(xml, 'instrepo') + new_node.set('name', node.get('name') + "_" + str(found_obsrepositories)) + new_node.set('priority', node.get('priority')) + new_node.set('local', 'true') + new_source_node = ET.SubElement(new_node, 'source') + new_source_node.set('path', "obs://" + path) + xml.remove(node) + + if found_obsrepositories > 0: + build_descr = '_service:' + build_descr.rsplit('/', 1)[-1] + tree.write(open(build_descr, 'wb')) + # appliance expand_obsrepos=None for xml in root.findall('repository'):