1
0
mirror of https://github.com/openSUSE/osc.git synced 2024-12-25 17:36:13 +01:00

Merge pull request #694 from adrianschroeter/vm_build_fixes

VM build fixes
This commit is contained in:
Marco Strigl 2019-12-05 10:26:40 +01:00 committed by GitHub
commit 8ec18e99a3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 112 additions and 47 deletions

9
NEWS
View File

@ -1,7 +1,14 @@
0.167
- Added --lastsucceeded option for logfile display (requires OBS 2.11)
- osc shell/chroot/wipe is now handled via build script (working for chroot and KVM only atm)
- osc build --vm-type=qemu support for cross architecture builds
- osc build/shell features:
* --vm-type=qemu support for cross architecture builds
* additional debug packages are read from build config now
(extra-pkgs definition is therefore by default empty now)
* --vm-disk-size= switch supported (has only an effect on clean builds)
* repository, architecture and vm_type from last build is automatically
reused if not specified otherwise.
* support building for kiwi products using obsrepositories:/
0.166.2
- Don't enforce password reuse (boo#1156501)

View File

@ -531,6 +531,7 @@ def main(apiurl, opts, argv):
cache_dir = None
build_uid = ''
vm_memory = config['build-memory']
vm_disk_size = config['build-vmdisk-rootsize']
vm_type = config['build-type']
vm_telnet = None
@ -620,6 +621,8 @@ def main(apiurl, opts, argv):
return 1
if opts.vm_memory:
vm_memory = opts.vm_memory
if opts.vm_disk_size:
vm_disk_size = opts.vm_disk_size
if opts.vm_type:
vm_type = opts.vm_type
if opts.vm_telnet:
@ -671,15 +674,42 @@ def main(apiurl, opts, argv):
except:
pass
# define buildinfo & config local cache
bi_file = None
bc_file = None
bi_filename = '_buildinfo-%s-%s.xml' % (repo, arch)
bc_filename = '_buildconfig-%s-%s' % (repo, arch)
if is_package_dir('.') and os.access(osc.core.store, os.W_OK):
bi_filename = os.path.join(os.getcwd(), osc.core.store, bi_filename)
bc_filename = os.path.join(os.getcwd(), osc.core.store, bc_filename)
elif not os.access('.', os.W_OK):
bi_file = NamedTemporaryFile(prefix=bi_filename)
bi_filename = bi_file.name
bc_file = NamedTemporaryFile(prefix=bc_filename)
bc_filename = bc_file.name
else:
bi_filename = os.path.abspath(bi_filename)
bc_filename = os.path.abspath(bc_filename)
if opts.shell:
buildargs.append("--shell")
if os.path.exists(build_root) and not opts.clean:
if os.path.exists(build_root) and os.path.exists(bi_filename) and not opts.clean and not opts.extra_pkgs:
opts.noinit = True
opts.offline = True
# we should check if the service did run before and only skip it then,
# but we have no save point for this atm
opts.noservice = True
if opts.noinit:
buildargs.append('--noinit')
# check for source services
if not opts.offline and not opts.noservice:
p = osc.core.Package(os.curdir)
r = p.run_source_services(verbose=True)
if r:
raise oscerr.ServiceRuntimeError('Source service run failed!')
cache_dir = config['packagecachedir'] % {'apihost': apihost}
extra_pkgs = []
@ -766,22 +796,6 @@ def main(apiurl, opts, argv):
raise oscerr.WrongOptions('--overlay %s is no valid directory!' % opts.overlay)
specialcmdopts += ['--overlay='+myoverlay]
bi_file = None
bc_file = None
bi_filename = '_buildinfo-%s-%s.xml' % (repo, arch)
bc_filename = '_buildconfig-%s-%s' % (repo, arch)
if is_package_dir('.') and os.access(osc.core.store, os.W_OK):
bi_filename = os.path.join(os.getcwd(), osc.core.store, bi_filename)
bc_filename = os.path.join(os.getcwd(), osc.core.store, bc_filename)
elif not os.access('.', os.W_OK):
bi_file = NamedTemporaryFile(prefix=bi_filename)
bi_filename = bi_file.name
bc_file = NamedTemporaryFile(prefix=bc_filename)
bc_filename = bc_file.name
else:
bi_filename = os.path.abspath(bi_filename)
bc_filename = os.path.abspath(bc_filename)
try:
if opts.noinit:
if not os.path.isfile(bi_filename):
@ -797,8 +811,18 @@ def main(apiurl, opts, argv):
if not os.path.isfile(bc_filename):
raise oscerr.WrongOptions('--offline is not possible, no local buildconfig file')
else:
print('Getting buildinfo from server and store to %s' % bi_filename)
print('Getting buildconfig from server and store to %s' % bc_filename)
bc = get_buildconfig(apiurl, prj, repo)
if not bc_file:
bc_file = open(bc_filename, 'w')
bc_file.write(decode_it(bc))
bc_file.flush()
if os.path.exists('/usr/lib/build/queryconfig') and not opts.nodebugpackages:
debug_pkgs = decode_it(return_external('/usr/lib/build/queryconfig', '--dist', bc_filename, 'substitute', 'obs:cli_debug_packages'))
if len(debug_pkgs) > 0:
extra_pkgs += debug_pkgs.strip().split(" ")
print('Getting buildinfo from server and store to %s' % bi_filename)
bi_text = decode_it(get_buildinfo(apiurl,
prj,
pac,
@ -814,12 +838,10 @@ def main(apiurl, opts, argv):
kiwipath = None
if build_type == 'kiwi':
kiwipath = get_kiwipath_from_buildinfo(apiurl, bi_filename, prj, repo)
print('Getting buildconfig from server and store to %s' % bc_filename)
bc = get_buildconfig(apiurl, prj, repo, kiwipath)
if not bc_file:
bc_file = open(bc_filename, 'w')
bc_file.write(decode_it(bc))
bc_file.flush()
bc = get_buildconfig(apiurl, prj, repo, kiwipath)
bc_file.seek(0)
bc_file.write(decode_it(bc))
bc_file.flush()
except HTTPError as e:
if e.code == 404:
# check what caused the 404
@ -1072,11 +1094,26 @@ def main(apiurl, opts, argv):
print(open(build_descr).read(), file=sys.stderr)
sys.exit(1)
root = tree.getroot()
# product
for xml in root.findall('instsource'):
if xml.find('instrepo').find('source').get('path') == 'obsrepositories:/':
print("obsrepositories:/ for product builds is not yet supported in osc!")
sys.exit(1)
found_obsrepositories = 0
for node in xml.findall('instrepo'):
if node and node.find('source').get('path') == 'obsrepositories:/':
found_obsrepositories = found_obsrepositories + 1
for path in bi.pathes:
new_node = ET.SubElement(xml, 'instrepo')
new_node.set('name', node.get('name') + "_" + str(found_obsrepositories))
new_node.set('priority', node.get('priority'))
new_node.set('local', 'true')
new_source_node = ET.SubElement(new_node, 'source')
new_source_node.set('path', "obs://" + path)
xml.remove(node)
if found_obsrepositories > 0:
build_descr = '_service:' + build_descr.rsplit('/', 1)[-1]
tree.write(open(build_descr, 'wb'))
# appliance
expand_obsrepos=None
for xml in root.findall('repository'):
@ -1217,9 +1254,9 @@ def main(apiurl, opts, argv):
vm_options += [ '--vm-initrd=' + config['build-initrd'] ]
build_root += '/.mount'
if vm_disk_size:
vm_options += [ '--vmdisk-rootsize=' + vm_disk_size ]
if config['build-vmdisk-rootsize']:
vm_options += [ '--vmdisk-rootsize=' + config['build-vmdisk-rootsize'] ]
if config['build-vmdisk-swapsize']:
vm_options += [ '--vmdisk-swapsize=' + config['build-vmdisk-swapsize'] ]
if config['build-vmdisk-filesystem']:
@ -1253,6 +1290,9 @@ def main(apiurl, opts, argv):
if hostarch != bi.buildarch and bi.buildarch in change_personality:
cmd = [ change_personality[bi.buildarch] ] + cmd
# record our settings for later builds
osc.core.store_write_last_buildroot(os.curdir, repo, arch, vm_type)
try:
rc = run_external(cmd[0], *cmd[1:])
if rc:

View File

@ -6117,7 +6117,8 @@ Please submit there instead, or use --nodevelproject to force direct submission.
# it seems to be an architecture in general
arg_arch = arg
if not (arg == osc.build.hostarch or arg in osc.build.can_also_build.get(osc.build.hostarch, [])):
print("WARNING: native compile is not possible, an emulator must be configured!")
if not (vm_type == 'qemu' or vm_type == 'emulator'):
print("WARNING: native compile is not possible, a emulator via binfmt misc handler must be configured!")
elif not arg_repository:
arg_repository = arg
else:
@ -6265,6 +6266,8 @@ Please submit there instead, or use --nodevelproject to force direct submission.
help='Do not run build checks on the resulting packages.')
@cmdln.option('--no-verify', '--noverify', action='store_true',
help='Skip signature verification (via pgp keys) of packages used for build. (Global config in oscrc: no_verify)')
@cmdln.option('--nodebugpackages', '--no-debug-packages', action='store_true',
help='Skip installation of additional debug packages for CLI builds')
@cmdln.option('--noservice', '--no-service', action='store_true',
help='Skip run of local source services as specified in _service file.')
@cmdln.option('-p', '--prefer-pkgs', metavar='DIR', action='append',
@ -6305,7 +6308,9 @@ Please submit there instead, or use --nodevelproject to force direct submission.
@cmdln.option('--linksources', action='store_true',
help='use hard links instead of a deep copied source')
@cmdln.option('--vm-memory', metavar='MEMORY',
help='use given MB for VM')
help='amount of memory for VM defined in MB')
@cmdln.option('--vm-disk-size', metavar='DISKSIZE',
help='size for newly created disk image in MB')
@cmdln.option('--vm-type', metavar='TYPE',
help='use VM type TYPE (e.g. kvm)')
@cmdln.option('--vm-telnet', metavar='TELNET',
@ -6437,19 +6442,15 @@ Please submit there instead, or use --nodevelproject to force direct submission.
if project == opts.alternative_project:
opts.alternative_project = None
args = self.parse_repoarchdescr(args, opts.noinit or opts.offline, opts.alternative_project, False, opts.vm_type, opts.multibuild_package)
if len(args) == 0:
# build env not specified, just read from last build attempt
lastbuildroot = store_read_last_buildroot(os.curdir)
if lastbuildroot:
args = [ lastbuildroot[0], lastbuildroot[1] ]
if not opts.vm_type:
opts.vm_type = lastbuildroot[2]
# check for source services
if not opts.offline and not opts.noservice:
p = Package('.')
r = p.run_source_services(verbose=True)
if r:
print('Source service run failed!', file=sys.stderr)
sys.exit(1)
else:
msg = ('WARNING: source services from package or project will not'
'be executed. This may not be the same build as on server!')
print(msg)
args = self.parse_repoarchdescr(args, opts.noinit or opts.offline, opts.alternative_project, False, opts.vm_type, opts.multibuild_package)
if not opts.local_package:
try:

View File

@ -278,8 +278,11 @@ apiurl = %(apiurl)s
# extra packages to install when building packages locally (osc build)
# this corresponds to osc build's -x option and can be overridden with that
# -x '' can also be given on the command line to override this setting, or
# you can have an empty setting here.
#extra-pkgs = vim gdb strace
# you can have an empty setting here. This global setting may leads to
# dependency problems when the base distro is not providing the package.
# => using server side definition via cli_debug_packages substitute rule is
# recommended therefore.
#extra-pkgs =
# build platform is used if the platform argument is omitted to osc build
#build_repository = %(build_repository)s

View File

@ -1140,7 +1140,7 @@ class Package:
REQ_STOREFILES = ('_project', '_package', '_apiurl', '_files', '_osclib_version')
OPT_STOREFILES = ('_to_be_added', '_to_be_deleted', '_in_conflict', '_in_update',
'_in_commit', '_meta', '_meta_mode', '_frozenlink', '_pulled', '_linkrepair',
'_size_limit', '_commit_msg')
'_size_limit', '_commit_msg', '_last_buildroot')
def __init__(self, workingdir, progress_obj=None, size_limit=None, wc_check=True):
global store
@ -6510,6 +6510,17 @@ def store_read_apiurl(dir, defaulturl=True):
apiurl = conf.config['apiurl']
return apiurl
def store_read_last_buildroot(dir):
global store
fname = os.path.join(dir, store, '_last_buildroot')
if os.path.exists(fname):
lines = open(fname).read().splitlines()
if len(lines) == 3:
return lines
return
def store_write_string(dir, file, string, subdir=''):
global store
@ -6534,6 +6545,9 @@ def store_write_project(dir, project):
def store_write_apiurl(dir, apiurl):
store_write_string(dir, '_apiurl', apiurl + '\n')
def store_write_last_buildroot(dir, repo, arch, vm_type):
store_write_string(dir, '_last_buildroot', repo + '\n' + arch + '\n' + vm_type + '\n')
def store_unlink_file(dir, file):
global store