Add support for git project handling

Automatically switches over from OBS api calls to git
when specifing a git repository on CLI

pkglistgen/cli.py

@@ -45,6 +45,7 @@ class CommandLineInterface(ToolBase.CommandLineInterface):
     @cmdln.option('-f', '--force', action='store_true', help='continue even if build is in progress')
     @cmdln.option('-d', '--dry', help='no modifications uploaded')
     @cmdln.option('-p', '--project', help='target project')
+    @cmdln.option('-g', '--git-url', help='git repository for target project')
     @cmdln.option('-s', '--scope', help='scope on which to operate ({}, staging:$letter)'.format(', '.join(SCOPES)))
     @cmdln.option('--no-checkout', action='store_true', help='reuse checkout in cache')
     @cmdln.option('--stop-after-solve', action='store_true', help='only create group files')
@@ -93,13 +94,13 @@ class CommandLineInterface(ToolBase.CommandLineInterface):
                 self.tool.reset()
                 self.tool.dry_run = self.options.dry
                 return self.tool.update_and_solve_target(api, target_project, target_config, main_repo,
-                                                         project=project, scope=scope, force=opts.force,
+                                                         git_url=opts.git_url, project=project, scope=scope,
-                                                         no_checkout=opts.no_checkout,
+                                                         force=opts.force, no_checkout=opts.no_checkout,
                                                          only_release_packages=opts.only_release_packages,
                                                          stop_after_solve=opts.stop_after_solve,
                                                          custom_cache_tag=opts.custom_cache_tag)
             except MismatchedRepoException:
-                logging.error("Failed to create weakremovers.inc due to mismatch in repos - project most likey started building again.")
+                logging.error("Failed to create weakremovers.inc due to mismatch in repos - project most likey started building again!")
                 # for stagings we have to be strict on the exit value
                 if scope == 'staging':
                     return 1

pkglistgen/group.py

@@ -330,11 +330,15 @@ class Group(object):
                 continue
             if name in missing:
                 if ignore_broken and name not in self.required:
-                    content += prefix + "#- " + name + " is missing\n"
+                    msg = ' {} not found on {}'.format(name, ','.join(sorted(missing[name])))
+                    content += prefix + "#- " + msg + "\n"
+                    self.logger.error(msg)
                 continue
             if name in unresolvable:
                 if ignore_broken and name not in self.required:
-                    content += prefix + "#- " + name + " is unresolvable: " + unresolvable[name] + "\n"
+                    msg = ' {} uninstallable: {}'.format(name, unresolvable[name])
+                    content += prefix + "#- " + msg + "\n"
+                    self.logger.error(msg)
                 continue
             content += prefix + "- " + name
             if comment:

pkglistgen/tool.py

@@ -37,6 +37,7 @@ from pkglistgen.group import Group
 SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__))
 
 PRODUCT_SERVICE = '/usr/lib/obs/service/create_single_product'
+PRODUCT_COMPOSE_SEPERATOR_LINE = '# The following is generated by openSUSE-release-tools'
 
 # share header cache with repochecker
 CACHEDIR = CacheManager.directory('repository-meta')
@@ -141,7 +142,7 @@ class PkgListGen(ToolBase.ToolBase):
         archs = ['*'] + self.all_architectures
         # a single file covering all builds via multibuild flavors
         with open(os.path.join(self.productcompose_dir, 'default.productcompose'), 'a') as opfh:
-            opfh.write("# The following is generated by openSUSE-release-tools\n")
+            opfh.write(PRODUCT_COMPOSE_SEPERATOR_LINE + "\n")
             for name in self.groups:
                 group = self.groups[name]
                 if not group.solved:
@@ -656,6 +657,7 @@ class PkgListGen(ToolBase.ToolBase):
         target_project: str,
         target_config: Mapping[str, Any],
         main_repo: str,
+        git_url: str,
         project: str,
         scope: str,
         force: bool,
@@ -675,32 +677,6 @@ class PkgListGen(ToolBase.ToolBase):
         release = target_config.get('pkglistgen-release', '000release-packages')
         oldrepos = target_config.get('pkglistgen-repos', '000update-repos')
 
-        url = api.makeurl(['source', project])
-        packages = ET.parse(http_GET(url)).getroot()
-        if packages.find('entry[@name="{}"]'.format(product)) is None:
-            if not self.dry_run:
-                undelete_package(api.apiurl, project, product, 'revive')
-            # TODO disable build.
-            logging.info('{} undeleted, skip dvd until next cycle'.format(product))
-            return
-        elif not force:
-            root = ET.fromstringlist(show_results_meta(api.apiurl, project, product,
-                                                       repository=[main_repo], multibuild=True))
-            if len(root.xpath('result[@state="building"]')) or len(root.xpath('result[@state="dirty"]')):
-                logging.info('{}/{} build in progress'.format(project, product))
-                return
-
-        drop_list = api.item_exists(project, oldrepos)
-        checkout_list = [group, product, release]
-        if drop_list and not only_release_packages:
-            checkout_list.append(oldrepos)
-
-        if packages.find('entry[@name="{}"]'.format(release)) is None:
-            if not self.dry_run:
-                undelete_package(api.apiurl, project, release, 'revive')
-            logging.info('{} undeleted, skip dvd until next cycle'.format(release))
-            return
-
         # Cache dir specific to hostname and project.
         host = urlparse(api.apiurl).hostname
         prefix_dir = 'pkglistgen'
@@ -708,10 +684,53 @@ class PkgListGen(ToolBase.ToolBase):
             prefix_dir += '-' + custom_cache_tag
         cache_dir = CacheManager.directory(prefix_dir, host, project)
 
-        if not no_checkout:
+        drop_list = []
-            if os.path.exists(cache_dir):
+        checkout_list = [group, product, release]
-                shutil.rmtree(cache_dir)
+        if not force:
-            os.makedirs(cache_dir)
+          root = ET.fromstringlist(show_results_meta(api.apiurl, project, product,
+                                                     repository=[main_repo], multibuild=True))
+          if len(root.xpath('result[@state="building"]')) or len(root.xpath('result[@state="dirty"]')):
+              logging.info('{}/{} build in progress'.format(project, product))
+              return
+        if git_url:
+            if os.path.exists(cache_dir + "/.git"):
+                # reset and update existing clone
+                logging.debug(subprocess.check_output(
+                    ['git', 'reset', '--hard'], cwd=cache_dir, encoding='utf-8'))
+                logging.debug(subprocess.check_output(
+                    ['git', 'pull', '--rebase'], cwd=cache_dir, encoding='utf-8'))
+            else:
+                if os.path.exists(cache_dir):
+                    shutil.rmtree(cache_dir)
+                logging.debug(subprocess.check_output(
+                    ['git', 'clone', '--recurse-submodules', git_url, cache_dir], encoding='utf-8'))
+            if os.path.exists(cache_dir + '/' + '000update-repos'):
+                logging.error('No support for 000update-repos in git projects atm')
+                return
+        else:
+            url = api.makeurl(['source', project])
+            packages = ET.parse(http_GET(url)).getroot()
+            if packages.find('entry[@name="{}"]'.format(product)) is None:
+                if not self.dry_run:
+                    undelete_package(api.apiurl, project, product, 'revive')
+                # TODO disable build.
+                logging.info('{} undeleted, skip dvd until next cycle'.format(product))
+                return
+
+            drop_list = api.item_exists(project, oldrepos)
+            if drop_list and not only_release_packages:
+                checkout_list.append(oldrepos)
+
+            if packages.find('entry[@name="{}"]'.format(release)) is None:
+                if not self.dry_run:
+                    undelete_package(api.apiurl, project, release, 'revive')
+                logging.info('{} undeleted, skip dvd until next cycle'.format(release))
+                return
+
+            if not no_checkout:
+                if os.path.exists(cache_dir):
+                    shutil.rmtree(cache_dir)
+                os.makedirs(cache_dir)
 
         group_dir = os.path.join(cache_dir, group)
         product_dir = os.path.join(cache_dir, product)
@@ -722,36 +741,41 @@ class PkgListGen(ToolBase.ToolBase):
         self.input_dir = group_dir
         self.output_dir = product_dir
 
-        for package in checkout_list:
+        if not no_checkout and not git_url:
-            if no_checkout:
+            logging.debug('Skipping checkout of {}'.format(project))
-                logging.debug('Skipping checkout of {}/{}'.format(project, package))
+            for package in checkout_list:
-                continue
+                checkout_package(api.apiurl, project, package, expand_link=True,
-            checkout_package(api.apiurl, project, package, expand_link=True,
+                                 prj_dir=cache_dir, outdir=os.path.join(cache_dir, package))
-                             prj_dir=cache_dir, outdir=os.path.join(cache_dir, package))
 
         file_utils.unlink_all_except(release_dir, ['weakremovers.inc', '*.changes'])
         if not only_release_packages:
             file_utils.unlink_all_except(product_dir)
-        ignore_list = ['supportstatus.txt', 'summary-staging.txt', 'package-groups.changes']
+        ignore_list = ['supportstatus.txt', 'summary-staging.txt', 'package-groups.changes',
+                       'default.productcompose.in']
         ignore_list += self.group_input_files()
         file_utils.copy_directory_contents(group_dir, product_dir, ignore_list)
         file_utils.change_extension(product_dir, '.spec.in', '.spec')
         file_utils.change_extension(product_dir, '.product.in', '.product')
-        fn = os.path.join(group_dir, 'default.productcompose')
+        fn = os.path.join(group_dir, 'default.productcompose.in')
         if os.path.isfile(fn):
             if not os.path.isdir(self.productcompose_dir):
-                os.mkdir(self.productcompose_dir)   # currently always creating it local at least
+                raise Exception('default.productcompose.in exists, but output directory is missing in git!')
             lines = open(fn).readlines()
             new_lines = []
             for line in lines:
-                if line.startswith('# The following is generated by openSUSE-release-tools'):
+                if line.startswith(PRODUCT_COMPOSE_SEPERATOR_LINE):
                     break
                 new_lines.append(line)
             open(os.path.join(self.productcompose_dir, 'default.productcompose'), 'w').write(''.join(new_lines))
+<<<<<<< HEAD
             if git_url:
                 logging.debug(subprocess.check_output(
                     ['git', 'add', self.productcompose_dir, f"{self.productcompose_dir}/default.productcompose"],
                     cwd=cache_dir, encoding='utf-8'))
+=======
+            if os.path.isfile(os.path.join(group_dir, 'supportstatus.txt')):
+                shutil.copy(os.path.join(group_dir, 'supportstatus.txt'), self.productcompose_dir)
+>>>>>>> 8c7c16b8 (git cleanup)
             self.skip_productcompose = False
         else:
             # No template file, so we don't create one either
@@ -834,7 +858,11 @@ class PkgListGen(ToolBase.ToolBase):
                     "- automatically generated by openSUSE-release-tools/pkglistgen\n\n"
                 )
 
-        self.commit_package(release_dir)
+        if git_url:
+            logging.debug(subprocess.check_output(
+                'git add *.spec', cwd=release_dir, shell=True, encoding='utf-8'))
+        else:
+            self.commit_package(release_dir)
 
         if only_release_packages:
             return
@@ -854,7 +882,27 @@ class PkgListGen(ToolBase.ToolBase):
                 for line in sorted(output):
                     f.write(line + '\n')
 
-        self.commit_package(product_dir)
+        if git_url:
+<<<<<<< HEAD
+            if os.path.isfile(f"{productcompose_dir}/default.productcompose"):
+                logging.debug(subprocess.check_output(
+                    ['git', 'add', productcompose_dir, f"{productcompose_dir}/default.productcompose"],
+                    cwd=cache_dir, encoding='utf-8'))
+            if os.path.isdir(product_dir):
+=======
+            if product_dir and os.path.isdir(product_dir):
+>>>>>>> 8c7c16b8 (git cleanup)
+                logging.debug(subprocess.check_output(
+                    ['git', 'add', productcompose_dir, product_dir],
+                    cwd=cache_dir, encoding='utf-8'))
+
+            logging.debug(subprocess.check_output(
+                'git commit -m "Update by pkglistgen of openSUSE-release-tool"', cwd=cache_dir, shell=True, encoding='utf-8'))
+            if not self.dry_run:
+                logging.debug(subprocess.check_output(
+                    'git push', cwd=cache_dir, shell=True, encoding='utf-8'))
+        elif not self.dry_run:
+            self.commit_package(product_dir)
 
         if os.path.isfile(reference_summary):
             return self.comment.handle_package_diff(project, reference_summary, summary_file)