yudaike/openSUSE-release-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for git project handling

Browse Source 
Automatically switches over from OBS api calls to git
when specifing a git repository on CLI
This commit is contained in:
Adrian Schröter 2024-02-13 16:17:05 +01:00
parent 9b449f91cb
commit cf36ceb5b8
No known key found for this signature in database
GPG Key ID: 918D8C954C08DB67
3 changed files with 101 additions and 48 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
pkglistgen/cli.py
View File

@ -45,6 +45,7 @@ class CommandLineInterface(ToolBase.CommandLineInterface):
@cmdln.option('-f', '--force', action='store_true', help='continue even if build is in progress')
@cmdln.option('-d', '--dry', help='no modifications uploaded')
@cmdln.option('-p', '--project', help='target project')
@cmdln.option('-g', '--git-url', help='git repository for target project')
@cmdln.option('-s', '--scope', help='scope on which to operate ({}, staging:$letter)'.format(', '.join(SCOPES)))
@cmdln.option('--no-checkout', action='store_true', help='reuse checkout in cache')
@cmdln.option('--stop-after-solve', action='store_true', help='only create group files')
@ -93,13 +94,13 @@ class CommandLineInterface(ToolBase.CommandLineInterface):
self.tool.reset()
self.tool.dry_run = self.options.dry
return self.tool.update_and_solve_target(api, target_project, target_config, main_repo,
project=project, scope=scope, force=opts.force,
no_checkout=opts.no_checkout,
git_url=opts.git_url, project=project, scope=scope,
force=opts.force, no_checkout=opts.no_checkout,
only_release_packages=opts.only_release_packages,
stop_after_solve=opts.stop_after_solve,
custom_cache_tag=opts.custom_cache_tag)
except MismatchedRepoException:
logging.error("Failed to create weakremovers.inc due to mismatch in repos - project most likey started building again.")
logging.error("Failed to create weakremovers.inc due to mismatch in repos - project most likey started building again!")
# for stagings we have to be strict on the exit value
if scope == 'staging':
return 1

8
pkglistgen/group.py
View File

@ -330,11 +330,15 @@ class Group(object):
continue
if name in missing:
if ignore_broken and name not in self.required:
content += prefix + "#- " + name + " is missing\n"
msg = ' {} not found on {}'.format(name, ','.join(sorted(missing[name])))
content += prefix + "#- " + msg + "\n"
self.logger.error(msg)
continue
if name in unresolvable:
if ignore_broken and name not in self.required:
content += prefix + "#- " + name + " is unresolvable: " + unresolvable[name] + "\n"
msg = ' {} uninstallable: {}'.format(name, unresolvable[name])
content += prefix + "#- " + msg + "\n"
self.logger.error(msg)
continue
content += prefix + "- " + name
if comment:

134
pkglistgen/tool.py
View File

@ -37,6 +37,7 @@ from pkglistgen.group import Group
SCRIPT_PATH = os.path.dirname(os.path.realpath(__file__))
PRODUCT_SERVICE = '/usr/lib/obs/service/create_single_product'
PRODUCT_COMPOSE_SEPERATOR_LINE = '# The following is generated by openSUSE-release-tools'
# share header cache with repochecker
CACHEDIR = CacheManager.directory('repository-meta')
@ -141,7 +142,7 @@ class PkgListGen(ToolBase.ToolBase):
archs = ['*'] + self.all_architectures
# a single file covering all builds via multibuild flavors
with open(os.path.join(self.productcompose_dir, 'default.productcompose'), 'a') as opfh:
opfh.write("# The following is generated by openSUSE-release-tools\n")
opfh.write(PRODUCT_COMPOSE_SEPERATOR_LINE + "\n")
for name in self.groups:
group = self.groups[name]
if not group.solved:
@ -656,6 +657,7 @@ class PkgListGen(ToolBase.ToolBase):
target_project: str,
target_config: Mapping[str, Any],
main_repo: str,
git_url: str,
project: str,
scope: str,
force: bool,
@ -675,32 +677,6 @@ class PkgListGen(ToolBase.ToolBase):
release = target_config.get('pkglistgen-release', '000release-packages')
oldrepos = target_config.get('pkglistgen-repos', '000update-repos')
url = api.makeurl(['source', project])
packages = ET.parse(http_GET(url)).getroot()
if packages.find('entry[@name="{}"]'.format(product)) is None:
if not self.dry_run:
undelete_package(api.apiurl, project, product, 'revive')
# TODO disable build.
logging.info('{} undeleted, skip dvd until next cycle'.format(product))
return
elif not force:
root = ET.fromstringlist(show_results_meta(api.apiurl, project, product,
repository=[main_repo], multibuild=True))
if len(root.xpath('result[@state="building"]')) or len(root.xpath('result[@state="dirty"]')):
logging.info('{}/{} build in progress'.format(project, product))
return
drop_list = api.item_exists(project, oldrepos)
checkout_list = [group, product, release]
if drop_list and not only_release_packages:
checkout_list.append(oldrepos)
if packages.find('entry[@name="{}"]'.format(release)) is None:
if not self.dry_run:
undelete_package(api.apiurl, project, release, 'revive')
logging.info('{} undeleted, skip dvd until next cycle'.format(release))
return
# Cache dir specific to hostname and project.
host = urlparse(api.apiurl).hostname
prefix_dir = 'pkglistgen'
@ -708,10 +684,53 @@ class PkgListGen(ToolBase.ToolBase):
prefix_dir += '-' + custom_cache_tag
cache_dir = CacheManager.directory(prefix_dir, host, project)
if not no_checkout:
if os.path.exists(cache_dir):
shutil.rmtree(cache_dir)
os.makedirs(cache_dir)
drop_list = []
checkout_list = [group, product, release]
if not force:
root = ET.fromstringlist(show_results_meta(api.apiurl, project, product,
repository=[main_repo], multibuild=True))
if len(root.xpath('result[@state="building"]')) or len(root.xpath('result[@state="dirty"]')):
logging.info('{}/{} build in progress'.format(project, product))
return
if git_url:
if os.path.exists(cache_dir + "/.git"):
# reset and update existing clone
logging.debug(subprocess.check_output(
['git', 'reset', '--hard'], cwd=cache_dir, encoding='utf-8'))
logging.debug(subprocess.check_output(
['git', 'pull', '--rebase'], cwd=cache_dir, encoding='utf-8'))
else:
if os.path.exists(cache_dir):
shutil.rmtree(cache_dir)
logging.debug(subprocess.check_output(
['git', 'clone', '--recurse-submodules', git_url, cache_dir], encoding='utf-8'))
if os.path.exists(cache_dir + '/' + '000update-repos'):
logging.error('No support for 000update-repos in git projects atm')
return
else:
url = api.makeurl(['source', project])
packages = ET.parse(http_GET(url)).getroot()
if packages.find('entry[@name="{}"]'.format(product)) is None:
if not self.dry_run:
undelete_package(api.apiurl, project, product, 'revive')
# TODO disable build.
logging.info('{} undeleted, skip dvd until next cycle'.format(product))
return
drop_list = api.item_exists(project, oldrepos)
if drop_list and not only_release_packages:
checkout_list.append(oldrepos)
if packages.find('entry[@name="{}"]'.format(release)) is None:
if not self.dry_run:
undelete_package(api.apiurl, project, release, 'revive')
logging.info('{} undeleted, skip dvd until next cycle'.format(release))
return
if not no_checkout:
if os.path.exists(cache_dir):
shutil.rmtree(cache_dir)
os.makedirs(cache_dir)
group_dir = os.path.join(cache_dir, group)
product_dir = os.path.join(cache_dir, product)
@ -722,36 +741,41 @@ class PkgListGen(ToolBase.ToolBase):
self.input_dir = group_dir
self.output_dir = product_dir
for package in checkout_list:
if no_checkout:
logging.debug('Skipping checkout of {}/{}'.format(project, package))
continue
checkout_package(api.apiurl, project, package, expand_link=True,
prj_dir=cache_dir, outdir=os.path.join(cache_dir, package))
if not no_checkout and not git_url:
logging.debug('Skipping checkout of {}'.format(project))
for package in checkout_list:
checkout_package(api.apiurl, project, package, expand_link=True,
prj_dir=cache_dir, outdir=os.path.join(cache_dir, package))
file_utils.unlink_all_except(release_dir, ['weakremovers.inc', '*.changes'])
if not only_release_packages:
file_utils.unlink_all_except(product_dir)
ignore_list = ['supportstatus.txt', 'summary-staging.txt', 'package-groups.changes']
ignore_list = ['supportstatus.txt', 'summary-staging.txt', 'package-groups.changes',
'default.productcompose.in']
ignore_list += self.group_input_files()
file_utils.copy_directory_contents(group_dir, product_dir, ignore_list)
file_utils.change_extension(product_dir, '.spec.in', '.spec')
file_utils.change_extension(product_dir, '.product.in', '.product')
fn = os.path.join(group_dir, 'default.productcompose')
fn = os.path.join(group_dir, 'default.productcompose.in')
if os.path.isfile(fn):
if not os.path.isdir(self.productcompose_dir):
os.mkdir(self.productcompose_dir) # currently always creating it local at least
raise Exception('default.productcompose.in exists, but output directory is missing in git!')
lines = open(fn).readlines()
new_lines = []
for line in lines:
if line.startswith('# The following is generated by openSUSE-release-tools'):
if line.startswith(PRODUCT_COMPOSE_SEPERATOR_LINE):
break
new_lines.append(line)
open(os.path.join(self.productcompose_dir, 'default.productcompose'), 'w').write(''.join(new_lines))
<<<<<<< HEAD
if git_url:
logging.debug(subprocess.check_output(
['git', 'add', self.productcompose_dir, f"{self.productcompose_dir}/default.productcompose"],
cwd=cache_dir, encoding='utf-8'))
=======
if os.path.isfile(os.path.join(group_dir, 'supportstatus.txt')):
shutil.copy(os.path.join(group_dir, 'supportstatus.txt'), self.productcompose_dir)
>>>>>>> 8c7c16b8 (git cleanup)
self.skip_productcompose = False
else:
# No template file, so we don't create one either
@ -834,7 +858,11 @@ class PkgListGen(ToolBase.ToolBase):
"- automatically generated by openSUSE-release-tools/pkglistgen\n\n"
)
self.commit_package(release_dir)
if git_url:
logging.debug(subprocess.check_output(
'git add *.spec', cwd=release_dir, shell=True, encoding='utf-8'))
else:
self.commit_package(release_dir)
if only_release_packages:
return
@ -854,7 +882,27 @@ class PkgListGen(ToolBase.ToolBase):
for line in sorted(output):
f.write(line + '\n')
self.commit_package(product_dir)
if git_url:
<<<<<<< HEAD
if os.path.isfile(f"{productcompose_dir}/default.productcompose"):
logging.debug(subprocess.check_output(
['git', 'add', productcompose_dir, f"{productcompose_dir}/default.productcompose"],
cwd=cache_dir, encoding='utf-8'))
if os.path.isdir(product_dir):
=======
if product_dir and os.path.isdir(product_dir):
>>>>>>> 8c7c16b8 (git cleanup)
logging.debug(subprocess.check_output(
['git', 'add', productcompose_dir, product_dir],
cwd=cache_dir, encoding='utf-8'))
logging.debug(subprocess.check_output(
'git commit -m "Update by pkglistgen of openSUSE-release-tool"', cwd=cache_dir, shell=True, encoding='utf-8'))
if not self.dry_run:
logging.debug(subprocess.check_output(
'git push', cwd=cache_dir, shell=True, encoding='utf-8'))
elif not self.dry_run:
self.commit_package(product_dir)
if os.path.isfile(reference_summary):
return self.comment.handle_package_diff(project, reference_summary, summary_file)