From a6cd8cb3453d3463c76d6cec808b1766ead939e9 Mon Sep 17 00:00:00 2001 From: Max Lin Date: Tue, 10 Sep 2024 15:57:34 +0800 Subject: [PATCH] Add slfo-packagelist-uploader.py for uploading SLFO packagelist to meta package slfo-packagelist-uploader.py is be able to checkout pacakgelist from src.opensue.org/products/SLFO_main and uplaoding the packagelist to project's pseudometa package. --- .github/workflows/ci-test.yml | 2 +- dist/ci/testenv-tumbleweed/Dockerfile | 2 +- dist/package/openSUSE-release-tools.spec | 15 ++++ slfo-packagelist-uploader.py | 93 ++++++++++++++++++++++++ 4 files changed, 110 insertions(+), 2 deletions(-) create mode 100755 slfo-packagelist-uploader.py diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index b4471a9c..0db95a8a 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -83,7 +83,7 @@ jobs: run: | for f in $(find . -maxdepth 1 -type f -executable -print); do # skip completely broken scripts or those without --help - [[ " ./checknewer.py ./repo2fileprovides.py ./openqa-maintenance.py ./docker_publisher.py ./publish_distro ./findfileconflicts ./write_repo_susetags_file.pl ./issue-diff.py ./generate-release-packages ./verify-build-and-generatelists " =~ "$f" ]] || "$f" --help + [[ " ./checknewer.py ./repo2fileprovides.py ./openqa-maintenance.py ./docker_publisher.py ./publish_distro ./findfileconflicts ./write_repo_susetags_file.pl ./issue-diff.py ./generate-release-packages ./verify-build-and-generatelists ./slfo-packagelist-uploader.py" =~ "$f" ]] || "$f" --help done linters: diff --git a/dist/ci/testenv-tumbleweed/Dockerfile b/dist/ci/testenv-tumbleweed/Dockerfile index 88c89696..4348f754 100644 --- a/dist/ci/testenv-tumbleweed/Dockerfile +++ b/dist/ci/testenv-tumbleweed/Dockerfile @@ -10,7 +10,7 @@ RUN zypper in -y osc python3-pytest python3-httpretty python3-pyxdg python3-PyYA python3-influxdb python3-pytest-cov libxml2-tools curl python3-flake8 python3-requests \ shadow vim vim-data strace git sudo patch unzip which cpio gawk openSUSE-release openSUSE-release-ftp \ perl-Net-SSLeay perl-Text-Diff perl-XML-Simple perl-XML-Parser build \ - obs-service-download_files obs-service-format_spec_file obs-scm-bridge + obs-service-download_files obs-service-format_spec_file obs-scm-bridge python3-GitPython RUN useradd tester -d /code/tests/home COPY run_as_tester /usr/bin diff --git a/dist/package/openSUSE-release-tools.spec b/dist/package/openSUSE-release-tools.spec index 0de5f9ce..7eb22008 100644 --- a/dist/package/openSUSE-release-tools.spec +++ b/dist/package/openSUSE-release-tools.spec @@ -238,6 +238,10 @@ Requires: osclib = %{version} Requires: python3-requests Requires: python3-solv Requires: zstd +%if 0%{?suse_version} > 1500 +# slfo-packagelist-uploader.py +Requires: python3-GitPython +%endif # we use the same user as repo-checker PreReq: openSUSE-release-tools-repo-checker BuildArch: noarch @@ -316,6 +320,12 @@ OSC plugin for the staging workflow, see `osc staging --help`. %prep %setup -q +# slfo-packagelist-uploader requires python-GitPython but 15.6 providing +# python311-GitPython only, therefore do not ship slfo-packagelist-uploader +# in 15.6 since python3-GitPython is not available +%if 0%{?suse_version} <= 1500 +rm slfo-packagelist-uploader.py +%endif %build %make_build @@ -428,6 +438,7 @@ exit 0 %exclude %{_datadir}/%{source_dir}/project-installcheck.py %exclude %{_datadir}/%{source_dir}/suppkg_rebuild.py %exclude %{_datadir}/%{source_dir}/skippkg-finder.py +%exclude %{_datadir}/%{source_dir}/slfo-packagelist-uploader.py %exclude %{_datadir}/%{source_dir}/osclib %exclude %{_datadir}/%{source_dir}/osc-cycle.py %exclude %{_datadir}/%{source_dir}/osc-origin.py @@ -456,6 +467,10 @@ exit 0 %files check-source %{_bindir}/osrt-check_source +%if 0%{?suse_version} > 1500 +%{_bindir}/osrt-slfo-packagelist-uploader +%{_datadir}/%{source_dir}/slfo-packagelist-uploader.py +%endif %{_datadir}/%{source_dir}/check_source.py %files docker-publisher diff --git a/slfo-packagelist-uploader.py b/slfo-packagelist-uploader.py new file mode 100755 index 00000000..a1cfd1b3 --- /dev/null +++ b/slfo-packagelist-uploader.py @@ -0,0 +1,93 @@ +#!/usr/bin/python3 + +import argparse +import logging +import sys +import os +import shutil +import git + +from lxml import etree as ET + +import osc.conf +from osclib.core import source_file_ensure +from osclib.core import project_pseudometa_package + +META_FILE = 'SLFO_Packagelist.group' + + +class PackagelistUploader(object): + def __init__(self, project, print_only, verbose): + self.project = project + self.print_only = print_only + self.verbose = verbose + self.apiurl = osc.conf.config['apiurl'] + self.debug = osc.conf.config['debug'] + + def create_packagelist(self, packages=[]): + packagelist = ET.Element('packagelist') + for pkg in sorted(packages): + if not self.print_only and self.verbose: + print(pkg) + attr = {'name': pkg} + ET.SubElement(packagelist, 'package', attr) + + return ET.tostring(packagelist, pretty_print=True, encoding='unicode') + + def crawl(self): + """Main method""" + cwd = os.getcwd() + filepath = os.path.join(cwd, 'SLFO_main') + if os.path.isdir(filepath): + shutil.rmtree(filepath) + gitpath = 'https://src.opensuse.org/products/SLFO_main.git' + git.Repo.clone_from(gitpath, filepath, branch='main') + packages = [] + files = os.listdir(filepath) + for f in files: + if f.startswith("."): + continue + fullpath = os.path.join(filepath, f) + if os.path.isdir(fullpath): + packages.append(f) + + slfo_packagelist = self.create_packagelist(packages) + pseudometa_project, pseudometa_package = project_pseudometa_package(self.apiurl, self.project) + if not self.print_only: + source_file_ensure(self.apiurl, pseudometa_project, pseudometa_package, META_FILE, + slfo_packagelist, 'Update SLFO packagelist') + else: + print(slfo_packagelist) + + +def main(args): + osc.conf.get_config(override_apiurl=args.apiurl) + osc.conf.config['debug'] = args.debug + + if args.project is None: + print("Please pass --project argument. See usage with --help.") + quit() + + uc = PackagelistUploader(args.project, args.print_only, args.verbose) + uc.crawl() + + +if __name__ == '__main__': + description = 'Upload SLFO packagelist from src.opensuse.org to staging meta package.' + parser = argparse.ArgumentParser(description=description) + parser.add_argument('-A', '--apiurl', metavar='URL', help='API URL') + parser.add_argument('-d', '--debug', action='store_true', + help='print info useful for debuging') + parser.add_argument('-p', '--project', dest='project', metavar='PROJECT', + help='Target project on buildservice') + parser.add_argument('-o', '--print-only', action='store_true', + help='show the result instead of the uploading') + parser.add_argument('-v', '--verbose', action='store_true', + help='show the diff') + + args = parser.parse_args() + + logging.basicConfig(level=logging.DEBUG if args.debug + else logging.INFO) + + sys.exit(main(args))