From d6d501ad99137d3d17d3950e48ea768f0c0848d6d48fbb0f3cf05769ea7285c0 Mon Sep 17 00:00:00 2001 From: Nicolas Belouin Date: Thu, 20 Feb 2025 11:05:47 +0100 Subject: [PATCH] Sync metadata, revamp PR jobs Signed-off-by: Nicolas Belouin --- .gitea/workflows/pr_project.yaml | 40 ++++++++++++++++ .gitea/workflows/sync_config.yaml | 7 ++- .gitea/workflows/sync_meta.yaml | 41 ++++++++++++++++ .obs/create_projects.sh | 32 +++++++++++++ .obs/render_meta.py | 62 ++++++++++++++++++++++++ .obs/wait_obs.py | 79 +++++++++++++++++++++++++++++++ _meta | 79 +++++++++++++++++++++++++++++++ 7 files changed, 336 insertions(+), 4 deletions(-) create mode 100644 .gitea/workflows/pr_project.yaml create mode 100644 .gitea/workflows/sync_meta.yaml create mode 100644 .obs/create_projects.sh create mode 100644 .obs/render_meta.py create mode 100644 .obs/wait_obs.py create mode 100644 _meta diff --git a/.gitea/workflows/pr_project.yaml b/.gitea/workflows/pr_project.yaml new file mode 100644 index 0000000..b0aae8e --- /dev/null +++ b/.gitea/workflows/pr_project.yaml @@ -0,0 +1,40 @@ +name: Synchronize Project Config +on: + pull_request_target: + types: + - opened + - reopened + - synchronize + - closed + branches-ignore: + - "devel" + +jobs: + sync-prjconf: + runs-on: tumbleweed + steps: + - run: | + zypper in -y python3-jinja2 + mkdir -p ~/.config/osc + cat >~/.config/osc/oscrc <~/.config/osc/oscrc </dev/null | sha256sum)"; then + new_meta="$(python3 .obs/render_meta.py)" + if [ "${meta}" != "$(echo "${new_meta}" | sha256sum)"]; then + echo "${new_meta}" | osc meta prj "${PROJECT}" -F - + fi + else + # Create the projects + bash .obs/create_projects.sh + fi diff --git a/.obs/create_projects.sh b/.obs/create_projects.sh new file mode 100644 index 0000000..86604b1 --- /dev/null +++ b/.obs/create_projects.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +show_help() { + echo "Usage: $(basename $0) [--internal]" + echo "options:" + echo "-h, --help display this help and exit" + echo "-i, --internal create project as internal" + exit 0 +} + +while [[ "$#" -gt 0 ]]; do + case $1 in + -h|--help) show_help;; + -i|--internal) internal="--internal" ;; + *) echo "Unknown parameter passed: $1";show_help ;; + esac + shift +done + +PROJECT="$(grep PROJECT .obs/common.py | sed 's/PROJECT = "\(.*\)"/\1/')" +if [ -n "$internal" ]; then + PROJECT="ISV${PROJECT:3}" +fi + +python3 .obs/render_meta.py ${internal} ToTest | osc meta prj "${PROJECT}:ToTest" -F - +python3 .obs/render_meta.py ${internal} | osc meta prj "${PROJECT}" -F - +osc meta prjconf "${PROJECT}:ToTest" -F _config +osc meta prjconf "${PROJECT}" -F _config + +if [ -z "$internal" ]; then + python3 .obs/sync_packages.py +fi \ No newline at end of file diff --git a/.obs/render_meta.py b/.obs/render_meta.py new file mode 100644 index 0000000..3ed31ec --- /dev/null +++ b/.obs/render_meta.py @@ -0,0 +1,62 @@ +import argparse + +from jinja2 import Template +from common import PROJECT + +def render(base_project, subproject, internal, scm_url=None): + version = base_project.rsplit(':', 1)[-1] + context = { + "base_project": subproject == "", + "title": f"SUSE Edge {version} {subproject}".rstrip(), + } + if subproject == "ToTest": + context["project"] = f"{base_project}:ToTest" + context["description"] = ( + f"This project doesn't build, it stores a snapshot of SUSE Edge {version} " + "project currently going through the automated test layer" + ) + if "Factory" in base_project or internal: + context["release_project"] = f"{base_project}:Snapshot" + elif subproject == "Snapshot": + context["project"] = f"{base_project}:Snapshot" + context["release_project"] = f"{base_project.rsplit(':', 1)[0]}:Containers" + context["for_release"] = True + context["description"] = ( + f"This project doesn't build, it stores a snapshot of SUSE Edge {version} " + "project that passed automated test layer" + ) + elif subproject == "": + context["project"] = base_project + context["release_project"] = f"{base_project}:ToTest" + else: # PR case direct python call + context["base_project"] = True + context["project"] = f"{base_project}:{subproject}" + if scm_url is not None: + context["scm_url"] = scm_url + + with open("_meta") as meta: + template = Template(meta.read()) + return template.render(context) + +def main(): + parser = argparse.ArgumentParser( + prog='ProgramName', + description='What the program does', + epilog='Text at the bottom of help') + parser.add_argument("subproject", default="", choices=["", "ToTest", "Snapshot"], nargs="?") + parser.add_argument("--internal", action="store_true") + parser.add_argument("--pr") + parser.add_argument("--scm-url") + args = parser.parse_args() + base_project = PROJECT.replace("isv", "ISV", 1) if args.internal else PROJECT + + print(render( + base_project=base_project, + subproject=args.subproject if args.pr is None else f"Staging:PR-{args.pr}", + internal=args.internal, + scm_url=args.scm_url, + )) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/.obs/wait_obs.py b/.obs/wait_obs.py new file mode 100644 index 0000000..8f4ea1d --- /dev/null +++ b/.obs/wait_obs.py @@ -0,0 +1,79 @@ +import xml.etree.ElementTree as ET +import subprocess +import time +import os + +from collections import Counter + +def get_buildstatus(project: str) -> ET.Element: + for _ in range(5): + try: + output = subprocess.check_output(["osc", "pr", "--xml", project]) + return ET.fromstring(output) + except subprocess.CalledProcessError: + continue + +def do_wait(project:str, commit:str) -> ET.Element: + last_state = None + while True: + time.sleep(5) + status = get_buildstatus(project) + if last_state == status.get("state"): + continue + else: + last_state = status.get("state") + + scminfo = { e.text for e in status.findall(".//scminfo") } + if len(scminfo) != 1 or scminfo.pop() != commit: + print("Waiting for OBS to sync with SCM") + continue + + if not all([ e.get('state') == "published" and e.get('dirty') is None for e in status.findall("./result")]): + print("Waiting for OBS to finish building") + continue + + return status + +def print_results(status: ET.Element): + results = {} + failed = [] + for e in status.findall("./result"): + repo = results.get(e.get("repository"), {}) + repo[e.get("arch")] = e + results[e.get("repository")] = repo + + for repo in results.keys(): + print(f"{repo}:") + depth=1 + for arch in results[repo].keys(): + counts = Counter() + if repo != "charts": + print(f"\t{arch}:") + depth=2 + for package in results[repo][arch].findall("./status"): + if package.get("code") in ["excluded", "disabled"]: + continue + if package.get("code") in ["failed", "unresolvable", "broken"]: + details = package.findtext("details") + if details: + failed.append(f"{package.get('package')} ({arch}): {details}") + else: + failed.append(f"{package.get('package')} ({arch})") + counts[package.get("code")] += 1 + for (code, count) in counts.items(): + print("\t"*depth, f"{code}: {count}") + + failed.sort() + if failed: + print("\nPackages failing: ") + for fail in failed: + print("\t", fail) + +def main(): + project = os.environ.get("OBS_PROJECT") + sha = os.environ.get("GITEA_SHA") + status = do_wait(project, sha) + print_results(status) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/_meta b/_meta new file mode 100644 index 0000000..6d7e8b9 --- /dev/null +++ b/_meta @@ -0,0 +1,79 @@ +{#- + This template is rendered by the render_meta.py script + it is not automatically enforced by OBS +-#} +{%- set maintainers = [ + "adinov", + "dirkmueller", + "dprodanov", + "edge-engineering", + "jtomasek", + "kzhelyazkov", + "nbelouin", + "steven.hardy", +] -%} +{%- set maintainers = [ + "nbelouin", +] -%} + + {{ title }} + {%- if description is defined %} + {{ description }} + {%- else %} + + {%- endif %} + {%- if scm_url is defined %} + {{ scm_url }} + {%- endif %} +{%- for maintainer in maintainers %} + +{%- endfor %} +{%- if not base_project %} + + + + + + + + + + x86_64 + +{%- endif %} +{%- for repository in ["images", "test_manifest_images"] %} + + {%- if release_project is defined and repository == "images" %} + + {%- endif %} + + + + x86_64 + aarch64 + +{%- endfor %} + + {%- if release_project is defined and not for_release %} + + {%- endif %} + + + x86_64 + aarch64 + + + {%- if release_project is defined and not for_release %} + + {%- endif %} + + x86_64 + + {%- if for_release %} + + + + x86_64 + + {%- endif %} +