Merge pull request #694 from adrianschroeter/vm_build_fixes

VM build fixes

NEWS

@@ -1,7 +1,14 @@
 0.167
   - Added --lastsucceeded option for logfile display (requires OBS 2.11)
   - osc shell/chroot/wipe is now handled via build script (working for chroot and KVM only atm)
-  - osc build --vm-type=qemu support for cross architecture builds
+  - osc build/shell features:
+    * --vm-type=qemu support for cross architecture builds
+    * additional debug packages are read from build config now
+      (extra-pkgs definition is therefore by default empty now)
+    * --vm-disk-size=  switch supported (has only an effect on clean builds)
+    * repository, architecture and vm_type from last build is automatically
+      reused if not specified otherwise.
+    * support building for kiwi products using obsrepositories:/
 
 0.166.2
   - Don't enforce password reuse (boo#1156501)

osc/build.py

@@ -531,6 +531,7 @@ def main(apiurl, opts, argv):
     cache_dir  = None
     build_uid = ''
     vm_memory = config['build-memory']
+    vm_disk_size = config['build-vmdisk-rootsize']
     vm_type = config['build-type']
     vm_telnet = None
 
@@ -620,6 +621,8 @@ def main(apiurl, opts, argv):
             return 1
     if opts.vm_memory:
         vm_memory = opts.vm_memory
+    if opts.vm_disk_size:
+        vm_disk_size = opts.vm_disk_size
     if opts.vm_type:
         vm_type = opts.vm_type
     if opts.vm_telnet:
@@ -671,15 +674,42 @@ def main(apiurl, opts, argv):
         except:
             pass
 
+    # define buildinfo & config local cache
+    bi_file = None
+    bc_file = None
+    bi_filename = '_buildinfo-%s-%s.xml' % (repo, arch)
+    bc_filename = '_buildconfig-%s-%s' % (repo, arch)
+    if is_package_dir('.') and os.access(osc.core.store, os.W_OK):
+        bi_filename = os.path.join(os.getcwd(), osc.core.store, bi_filename)
+        bc_filename = os.path.join(os.getcwd(), osc.core.store, bc_filename)
+    elif not os.access('.', os.W_OK):
+        bi_file = NamedTemporaryFile(prefix=bi_filename)
+        bi_filename = bi_file.name
+        bc_file = NamedTemporaryFile(prefix=bc_filename)
+        bc_filename = bc_file.name
+    else:
+        bi_filename = os.path.abspath(bi_filename)
+        bc_filename = os.path.abspath(bc_filename)
+
     if opts.shell:
         buildargs.append("--shell")
-        if os.path.exists(build_root) and not opts.clean:
+        if os.path.exists(build_root) and os.path.exists(bi_filename) and not opts.clean and not opts.extra_pkgs:
             opts.noinit = True
             opts.offline = True
+            # we should check if the service did run before and only skip it then,
+            # but we have no save point for this atm
+            opts.noservice = True
 
     if opts.noinit:
         buildargs.append('--noinit')
 
+    # check for source services
+    if not opts.offline and not opts.noservice:
+        p = osc.core.Package(os.curdir)
+        r = p.run_source_services(verbose=True)
+        if r:
+            raise oscerr.ServiceRuntimeError('Source service run failed!')
+
     cache_dir = config['packagecachedir'] % {'apihost': apihost}
 
     extra_pkgs = []
@@ -766,22 +796,6 @@ def main(apiurl, opts, argv):
             raise oscerr.WrongOptions('--overlay %s is no valid directory!' % opts.overlay)
         specialcmdopts += ['--overlay='+myoverlay]
 
-    bi_file = None
-    bc_file = None
-    bi_filename = '_buildinfo-%s-%s.xml' % (repo, arch)
-    bc_filename = '_buildconfig-%s-%s' % (repo, arch)
-    if is_package_dir('.') and os.access(osc.core.store, os.W_OK):
-        bi_filename = os.path.join(os.getcwd(), osc.core.store, bi_filename)
-        bc_filename = os.path.join(os.getcwd(), osc.core.store, bc_filename)
-    elif not os.access('.', os.W_OK):
-        bi_file = NamedTemporaryFile(prefix=bi_filename)
-        bi_filename = bi_file.name
-        bc_file = NamedTemporaryFile(prefix=bc_filename)
-        bc_filename = bc_file.name
-    else:
-        bi_filename = os.path.abspath(bi_filename)
-        bc_filename = os.path.abspath(bc_filename)
-
     try:
         if opts.noinit:
             if not os.path.isfile(bi_filename):
@@ -797,8 +811,18 @@ def main(apiurl, opts, argv):
             if not os.path.isfile(bc_filename):
                 raise oscerr.WrongOptions('--offline is not possible, no local buildconfig file')
         else:
-            print('Getting buildinfo from server and store to %s' % bi_filename)
+            print('Getting buildconfig from server and store to %s' % bc_filename)
+            bc = get_buildconfig(apiurl, prj, repo)
+            if not bc_file:
+                bc_file = open(bc_filename, 'w')
+            bc_file.write(decode_it(bc))
+            bc_file.flush()
+            if os.path.exists('/usr/lib/build/queryconfig') and not opts.nodebugpackages:
+                debug_pkgs = decode_it(return_external('/usr/lib/build/queryconfig', '--dist', bc_filename, 'substitute', 'obs:cli_debug_packages'))
+                if len(debug_pkgs) > 0:
+                    extra_pkgs += debug_pkgs.strip().split(" ")
 
+            print('Getting buildinfo from server and store to %s' % bi_filename)
             bi_text = decode_it(get_buildinfo(apiurl,
                                     prj,
                                     pac,
@@ -814,10 +838,8 @@ def main(apiurl, opts, argv):
             kiwipath = None
             if build_type == 'kiwi':
                 kiwipath = get_kiwipath_from_buildinfo(apiurl, bi_filename, prj, repo)
-            print('Getting buildconfig from server and store to %s' % bc_filename)
                 bc = get_buildconfig(apiurl, prj, repo, kiwipath)
-            if not bc_file:
+                bc_file.seek(0)
-                bc_file = open(bc_filename, 'w')
                 bc_file.write(decode_it(bc))
                 bc_file.flush()
     except HTTPError as e:
@@ -1072,11 +1094,26 @@ def main(apiurl, opts, argv):
             print(open(build_descr).read(), file=sys.stderr)
             sys.exit(1)
         root = tree.getroot()
+
         # product
         for xml in root.findall('instsource'):
-            if xml.find('instrepo').find('source').get('path') == 'obsrepositories:/':
+            found_obsrepositories = 0
-                print("obsrepositories:/ for product builds is not yet supported in osc!")
+            for node in xml.findall('instrepo'):
-                sys.exit(1)
+                if node and node.find('source').get('path') == 'obsrepositories:/':
+                   found_obsrepositories = found_obsrepositories + 1
+                   for path in bi.pathes:
+                       new_node = ET.SubElement(xml, 'instrepo')
+                       new_node.set('name', node.get('name') + "_" + str(found_obsrepositories))
+                       new_node.set('priority', node.get('priority'))
+                       new_node.set('local', 'true')
+                       new_source_node = ET.SubElement(new_node, 'source')
+                       new_source_node.set('path', "obs://" + path)
+                   xml.remove(node)
+
+            if found_obsrepositories > 0:
+               build_descr = '_service:' + build_descr.rsplit('/', 1)[-1]
+               tree.write(open(build_descr, 'wb'))
+
         # appliance
         expand_obsrepos=None
         for xml in root.findall('repository'):
@@ -1217,9 +1254,9 @@ def main(apiurl, opts, argv):
                     vm_options += [ '--vm-initrd=' + config['build-initrd'] ]
 
             build_root += '/.mount'
+        if vm_disk_size:
+            vm_options += [ '--vmdisk-rootsize=' + vm_disk_size ]
 
-        if config['build-vmdisk-rootsize']:
-            vm_options += [ '--vmdisk-rootsize=' + config['build-vmdisk-rootsize'] ]
         if config['build-vmdisk-swapsize']:
             vm_options += [ '--vmdisk-swapsize=' + config['build-vmdisk-swapsize'] ]
         if config['build-vmdisk-filesystem']:
@@ -1253,6 +1290,9 @@ def main(apiurl, opts, argv):
     if hostarch != bi.buildarch and bi.buildarch in change_personality:
         cmd = [ change_personality[bi.buildarch] ] + cmd
 
+    # record our settings for later builds
+    osc.core.store_write_last_buildroot(os.curdir, repo, arch, vm_type)
+
     try:
         rc = run_external(cmd[0], *cmd[1:])
         if rc:

osc/commandline.py

@@ -6117,7 +6117,8 @@ Please submit there instead, or use --nodevelproject to force direct submission.
                         # it seems to be an architecture in general
                         arg_arch = arg
                         if not (arg == osc.build.hostarch or arg in osc.build.can_also_build.get(osc.build.hostarch, [])):
-                            print("WARNING: native compile is not possible, an emulator must be configured!")
+                            if not (vm_type == 'qemu' or vm_type == 'emulator'):
+                               print("WARNING: native compile is not possible, a emulator via binfmt misc handler must be configured!")
                     elif not arg_repository:
                         arg_repository = arg
                     else:
@@ -6265,6 +6266,8 @@ Please submit there instead, or use --nodevelproject to force direct submission.
                   help='Do not run build checks on the resulting packages.')
     @cmdln.option('--no-verify', '--noverify', action='store_true',
                   help='Skip signature verification (via pgp keys) of packages used for build. (Global config in oscrc: no_verify)')
+    @cmdln.option('--nodebugpackages', '--no-debug-packages', action='store_true',
+                  help='Skip installation of additional debug packages for CLI builds')
     @cmdln.option('--noservice', '--no-service', action='store_true',
                   help='Skip run of local source services as specified in _service file.')
     @cmdln.option('-p', '--prefer-pkgs', metavar='DIR', action='append',
@@ -6305,7 +6308,9 @@ Please submit there instead, or use --nodevelproject to force direct submission.
     @cmdln.option('--linksources', action='store_true',
                   help='use hard links instead of a deep copied source')
     @cmdln.option('--vm-memory', metavar='MEMORY',
-                  help='use given MB for VM')
+                  help='amount of memory for VM defined in MB')
+    @cmdln.option('--vm-disk-size', metavar='DISKSIZE',
+                  help='size for newly created disk image in MB')
     @cmdln.option('--vm-type', metavar='TYPE',
                   help='use VM type TYPE (e.g. kvm)')
     @cmdln.option('--vm-telnet', metavar='TELNET',
@@ -6437,19 +6442,15 @@ Please submit there instead, or use --nodevelproject to force direct submission.
         if project == opts.alternative_project:
             opts.alternative_project = None
 
-        args = self.parse_repoarchdescr(args, opts.noinit or opts.offline, opts.alternative_project, False, opts.vm_type, opts.multibuild_package)
+        if len(args) == 0:
+            # build env not specified, just read from last build attempt
+            lastbuildroot = store_read_last_buildroot(os.curdir)
+            if lastbuildroot:
+               args = [ lastbuildroot[0], lastbuildroot[1] ]
+               if not opts.vm_type:
+                  opts.vm_type = lastbuildroot[2]
 
-        # check for source services
+        args = self.parse_repoarchdescr(args, opts.noinit or opts.offline, opts.alternative_project, False, opts.vm_type, opts.multibuild_package)
-        if not opts.offline and not opts.noservice:
-            p = Package('.')
-            r = p.run_source_services(verbose=True)
-            if r:
-                print('Source service run failed!', file=sys.stderr)
-                sys.exit(1)
-        else:
-            msg = ('WARNING: source services from package or project will not'
-                   'be executed. This may not be the same build as on server!')
-            print(msg)
 
         if not opts.local_package:
             try:

osc/conf.py

@@ -278,8 +278,11 @@ apiurl = %(apiurl)s
 # extra packages to install when building packages locally (osc build)
 # this corresponds to osc build's -x option and can be overridden with that
 # -x '' can also be given on the command line to override this setting, or
-# you can have an empty setting here.
+# you can have an empty setting here. This global setting may leads to 
-#extra-pkgs = vim gdb strace
+# dependency problems when the base distro is not providing the package.
+# => using server side definition via cli_debug_packages substitute rule is
+#    recommended therefore.
+#extra-pkgs = 
 
 # build platform is used if the platform argument is omitted to osc build
 #build_repository = %(build_repository)s

osc/core.py

@@ -1140,7 +1140,7 @@ class Package:
     REQ_STOREFILES = ('_project', '_package', '_apiurl', '_files', '_osclib_version')
     OPT_STOREFILES = ('_to_be_added', '_to_be_deleted', '_in_conflict', '_in_update',
         '_in_commit', '_meta', '_meta_mode', '_frozenlink', '_pulled', '_linkrepair',
-        '_size_limit', '_commit_msg')
+        '_size_limit', '_commit_msg', '_last_buildroot')
 
     def __init__(self, workingdir, progress_obj=None, size_limit=None, wc_check=True):
         global store
@@ -6510,6 +6510,17 @@ def store_read_apiurl(dir, defaulturl=True):
         apiurl = conf.config['apiurl']
     return apiurl
 
+def store_read_last_buildroot(dir):
+    global store
+
+    fname = os.path.join(dir, store, '_last_buildroot')
+    if os.path.exists(fname):
+        lines = open(fname).read().splitlines()
+        if len(lines) == 3:
+            return lines
+
+    return
+
 def store_write_string(dir, file, string, subdir=''):
     global store
 
@@ -6534,6 +6545,9 @@ def store_write_project(dir, project):
 def store_write_apiurl(dir, apiurl):
     store_write_string(dir, '_apiurl', apiurl + '\n')
 
+def store_write_last_buildroot(dir, repo, arch, vm_type):
+    store_write_string(dir, '_last_buildroot', repo + '\n' + arch + '\n' + vm_type + '\n')
+
 def store_unlink_file(dir, file):
     global store