yudaike/openSUSE-release-tools
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

pkglistgen: Automatically merge old update repos

Browse Source 
Too many update repos file slow down the drop list generation by a lot,
so max them per channel to 10 - and collapse them automatically if
needed
This commit is contained in:
Stephan Kulow 2021-06-18 11:19:41 +02:00
parent 8c963bf3eb
commit b452cfba69
1 changed files with 20 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
pkglistgen/update_repo_handler.py
View File

@ -209,6 +209,7 @@ def merge_susetags(output, files):
print(dep, file=output_file) print(dep, file=output_file)
print('-Prv:', file=output_file) print('-Prv:', file=output_file)
def update_project(apiurl, project): def update_project(apiurl, project):
# Cache dir specific to hostname and project. # Cache dir specific to hostname and project.
host = urlparse(apiurl).hostname host = urlparse(apiurl).hostname
@ -224,6 +225,8 @@ def update_project(apiurl, project):
osc.core.checkout_package(apiurl, project, '000update-repos', expand_link=True, prj_dir=cache_dir) osc.core.checkout_package(apiurl, project, '000update-repos', expand_link=True, prj_dir=cache_dir)
package = osc.core.Package(repo_dir)
root = yaml.safe_load(open(os.path.join(repo_dir, 'config.yml'))) root = yaml.safe_load(open(os.path.join(repo_dir, 'config.yml')))
for item in root: for item in root:
key = list(item)[0] key = list(item)[0]
@ -240,6 +243,20 @@ def update_project(apiurl, project):
path = key + '.packages' path = key + '.packages'
packages_file = os.path.join(repo_dir, path) packages_file = os.path.join(repo_dir, path)
if opts.get('refresh', False):
oldfiles = glob.glob(os.path.join(repo_dir, '{}_*.packages.xz'.format(key)))
if len(oldfiles) > 10:
oldest = oldfiles[-1]
if oldest.count('and_before') > 1:
raise Exception('The oldest is already a compated file')
oldest = oldest.replace('.packages.xz', '_and_before.packages')
merge_susetags(oldest, oldfiles)
for file in oldfiles:
os.unlink(file)
package.delete_file(os.path.basename(file))
subprocess.check_call(['xz', oldest])
package.addfile(os.path.basename(oldest) + ".xz")
if os.path.exists(packages_file + '.xz'): if os.path.exists(packages_file + '.xz'):
print(path, 'already exists') print(path, 'already exists')
continue continue
@ -267,11 +284,7 @@ def update_project(apiurl, project):
subprocess.call(['xz', '-9', packages_file]) subprocess.call(['xz', '-9', packages_file])
os.unlink(solv_file) os.unlink(solv_file)
url = osc.core.makeurl(apiurl, ['source', project, '000update-repos', path + '.xz']) package.addfile(os.path.basename(path + '.xz'))
try:
osc.core.http_PUT(url, data=open(packages_file + '.xz', 'rb').read())
except HTTPError:
logger.error(f"Failed to upload to {url}")
sys.exit(1)
del pool del pool
package.commit('Automatic update')