Sync metadata, revamp PR jobs

Signed-off-by: Nicolas Belouin <nicolas.belouin@suse.com>

.gitea/workflows/pr_project.yaml

@@ -0,0 +1,40 @@
+name: Synchronize Project Config
+on:
+  pull_request_target:
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - closed
+    branches-ignore:
+      - "devel"
+
+jobs:
+  sync-prjconf:
+    runs-on: tumbleweed
+    steps:
+      - run: |
+          zypper in -y python3-jinja2
+          mkdir -p ~/.config/osc
+          cat >~/.config/osc/oscrc <<EOF
+          [general]
+          apiurl = https://api.opensuse.org
+
+          [https://api.opensuse.org]
+          user=${{ vars.OBS_USERNAME }}
+          pass=${{ secrets.OBS_PASSWORD }}
+          EOF
+    # Waiting on PR to get merged for support in upstream action/checkout action
+      - uses: 'https://github.com/yangskyboxlabs/action-checkout@sha256'
+        with:
+          object-format: 'sha256'
+      - if: gitea.event.action == "closed"
+        run: |
+          PROJECT="$(grep PROJECT .obs/common.py | sed 's/PROJECT = "\(.*\)"/\1/')"
+          osc rdelete "${PROJECT}:Staging:PR-${{ gitea.event.number }}"
+      - if: gitea.event.action != "closed"
+        env:
+          SCM_URL: ${{ gitea.event.pull_request.head.repo.clone_url }}#${{ gitea.head_ref }}
+        run: |
+          PROJECT="$(grep PROJECT .obs/common.py | sed 's/PROJECT = "\(.*\)"/\1/')"
+          python3 .obs/render_meta.py --pr ${{ gitea.event.number }} --scm-url "${SCM_URL}" | osc meta prj "${PROJECT}:Staging:PR-${{ gitea.event.number }}" -F -

.gitea/workflows/sync_config.yaml

@@ -1,9 +1,8 @@
 name: Synchronize Project Config
 on:
   push:
-    branches:
-      - "main"
-      - "3.*"
+    branches-ignore:
+      - "devel"
     paths:
       - "_config"
       - ".gitea/workflows/sync_config.yaml"
@@ -28,6 +27,6 @@ jobs:
           object-format: 'sha256'
       - run: |
           PROJECT="$(grep PROJECT .obs/common.py | sed 's/PROJECT = "\(.*\)"/\1/')"
-          if [ "$(osc meta prjconf "${PROJECT}" | sha256sum)" = "$(cat _config | sha256sum)" ] ; then
+          if [ "$(osc meta prjconf "${PROJECT}" | sha256sum)" != "$(cat _config | sha256sum)" ] ; then
             osc meta prjconf "${PROJECT}" -F _config
           fi

.gitea/workflows/sync_meta.yaml

@@ -0,0 +1,41 @@
+name: Synchronize Project Metadata
+on:
+  push:
+    branches-ignore:
+      - "devel"
+    paths:
+      - "_meta"
+      - ".gitea/workflows/sync_meta.yaml"
+      - ".obs/common.py"
+
+jobs:
+  sync-prjconf:
+    runs-on: tumbleweed
+    steps:
+      - run: |
+          zypper in -y python3-jinja2
+          mkdir -p ~/.config/osc
+          cat >~/.config/osc/oscrc <<EOF
+          [general]
+          apiurl = https://api.opensuse.org
+
+          [https://api.opensuse.org]
+          user=${{ vars.OBS_USERNAME }}
+          pass=${{ secrets.OBS_PASSWORD }}
+          EOF
+    # Waiting on PR to get merged for support in upstream action/checkout action
+      - uses: 'https://github.com/yangskyboxlabs/action-checkout@sha256'
+        with:
+          object-format: 'sha256'
+      - run: |
+          PROJECT="$(grep PROJECT .obs/common.py | sed 's/PROJECT = "\(.*\)"/\1/')"
+          set -o pipefail
+          if meta="$(osc meta prj "${PROJECT}" 2>/dev/null | sha256sum)"; then
+            new_meta="$(python3 .obs/render_meta.py)"
+            if [ "${meta}" != "$(echo "${new_meta}" | sha256sum)"]; then
+              echo "${new_meta}" | osc meta prj "${PROJECT}" -F -
+            fi
+          else
+            # Create the projects
+            bash .obs/create_projects.sh
+          fi

.obs/create_projects.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+show_help() {
+    echo "Usage: $(basename $0) [--internal]"
+    echo "options:"
+    echo "-h, --help       display this help and exit"
+    echo "-i, --internal   create project as internal"
+    exit 0
+}
+
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        -h|--help) show_help;;
+        -i|--internal) internal="--internal" ;;
+        *) echo "Unknown parameter passed: $1";show_help ;;
+    esac
+    shift
+done
+
+PROJECT="$(grep PROJECT .obs/common.py | sed 's/PROJECT = "\(.*\)"/\1/')"
+if [ -n "$internal" ]; then
+    PROJECT="ISV${PROJECT:3}"
+fi
+
+python3 .obs/render_meta.py ${internal} ToTest | osc meta prj "${PROJECT}:ToTest" -F -
+python3 .obs/render_meta.py ${internal} | osc meta prj "${PROJECT}" -F -
+osc meta prjconf "${PROJECT}:ToTest" -F _config
+osc meta prjconf "${PROJECT}" -F _config
+
+if [ -z "$internal" ]; then
+    python3 .obs/sync_packages.py
+fi

.obs/render_meta.py

@@ -0,0 +1,62 @@
+import argparse
+
+from jinja2 import Template
+from common import PROJECT
+
+def render(base_project, subproject, internal, scm_url=None):
+    version = base_project.rsplit(':', 1)[-1]
+    context = {
+        "base_project": subproject == "",
+        "title": f"SUSE Edge {version} {subproject}".rstrip(),
+    }
+    if subproject == "ToTest":
+        context["project"] = f"{base_project}:ToTest"
+        context["description"] = (
+            f"This project doesn't build, it stores a snapshot of SUSE Edge {version} "
+            "project currently going through the automated test layer"
+        )
+        if "Factory" in base_project or internal:
+            context["release_project"] = f"{base_project}:Snapshot"
+    elif subproject == "Snapshot":
+        context["project"] = f"{base_project}:Snapshot"
+        context["release_project"] = f"{base_project.rsplit(':', 1)[0]}:Containers"
+        context["for_release"] = True
+        context["description"] = (
+            f"This project doesn't build, it stores a snapshot of SUSE Edge {version} "
+            "project that passed automated test layer"
+        )
+    elif subproject == "":
+        context["project"] = base_project
+        context["release_project"] = f"{base_project}:ToTest"
+    else: # PR case direct python call
+        context["base_project"] = True
+        context["project"] = f"{base_project}:{subproject}"
+        if scm_url is not None:
+            context["scm_url"] = scm_url
+
+    with open("_meta") as meta:
+        template = Template(meta.read())
+    return template.render(context)
+
+def main():
+    parser = argparse.ArgumentParser(
+                    prog='ProgramName',
+                    description='What the program does',
+                    epilog='Text at the bottom of help')
+    parser.add_argument("subproject", default="", choices=["", "ToTest", "Snapshot"], nargs="?")
+    parser.add_argument("--internal", action="store_true")
+    parser.add_argument("--pr")
+    parser.add_argument("--scm-url")
+    args = parser.parse_args()
+    base_project = PROJECT.replace("isv", "ISV", 1) if args.internal else PROJECT
+
+    print(render(
+        base_project=base_project,
+        subproject=args.subproject if args.pr is None else f"Staging:PR-{args.pr}",
+        internal=args.internal,
+        scm_url=args.scm_url,
+    ))
+
+
+if __name__ == "__main__":
+    main()

.obs/wait_obs.py

@@ -0,0 +1,79 @@
+import xml.etree.ElementTree as ET
+import subprocess
+import time
+import os
+
+from collections import Counter
+
+def get_buildstatus(project: str) -> ET.Element:
+    for _ in range(5):
+        try:
+            output = subprocess.check_output(["osc", "pr", "--xml", project])
+            return ET.fromstring(output)
+        except subprocess.CalledProcessError:
+            continue
+
+def do_wait(project:str, commit:str) -> ET.Element:
+    last_state = None
+    while True:
+        time.sleep(5)
+        status = get_buildstatus(project)
+        if last_state == status.get("state"):
+            continue
+        else:
+            last_state = status.get("state")
+
+        scminfo = { e.text for e in status.findall(".//scminfo") }
+        if len(scminfo) != 1 or scminfo.pop() != commit:
+            print("Waiting for OBS to sync with SCM")
+            continue
+
+        if not all([ e.get('state') == "published" and e.get('dirty') is None for e in status.findall("./result")]):
+            print("Waiting for OBS to finish building")
+            continue
+
+        return status
+        
+def print_results(status: ET.Element):
+    results = {}
+    failed = []
+    for e in status.findall("./result"):
+        repo = results.get(e.get("repository"), {})
+        repo[e.get("arch")] = e
+        results[e.get("repository")] = repo
+    
+    for repo in results.keys():
+        print(f"{repo}:")
+        depth=1
+        for arch in results[repo].keys():
+            counts = Counter()
+            if repo != "charts":
+                print(f"\t{arch}:")
+                depth=2
+            for package in results[repo][arch].findall("./status"):
+                if package.get("code") in ["excluded", "disabled"]:
+                    continue
+                if package.get("code") in ["failed", "unresolvable", "broken"]:
+                    details = package.findtext("details")
+                    if details:
+                        failed.append(f"{package.get('package')} ({arch}): {details}")
+                    else:
+                        failed.append(f"{package.get('package')} ({arch})")
+                counts[package.get("code")] += 1
+            for (code, count) in counts.items():
+                print("\t"*depth, f"{code}: {count}")
+    
+    failed.sort()
+    if failed:
+        print("\nPackages failing: ")
+    for fail in failed:
+        print("\t", fail)
+
+def main():
+    project = os.environ.get("OBS_PROJECT")
+    sha = os.environ.get("GITEA_SHA")
+    status = do_wait(project, sha)
+    print_results(status)
+
+if __name__ == "__main__":
+    main()

_meta

@@ -0,0 +1,79 @@
+{#- 
+  This template is rendered by the render_meta.py script
+  it is not automatically enforced by OBS
+-#}
+{%- set maintainers = [
+    "adinov",
+    "dirkmueller",
+    "dprodanov",
+    "edge-engineering",
+    "jtomasek",
+    "kzhelyazkov",
+    "nbelouin",
+    "steven.hardy",
+] -%}
+{%- set maintainers = [
+    "nbelouin",
+] -%}
+<project name="{{ project }}">
+  <title>{{ title }}</title>
+  {%- if description is defined %}
+  <description>{{ description }}</description>
+  {%- else %}
+  <description/>
+  {%- endif %}
+  {%- if scm_url is defined %}
+  <scmsync>{{ scm_url }}</scmsync>
+  {%- endif %}
+{%- for maintainer in maintainers %}
+  <person userid="{{ maintainer }}" role="maintainer"/>
+{%- endfor %}
+{%- if not base_project %}
+  <build>
+    <disable/>
+    <enable repository="charts"/>
+    <enable repository="test_manifest_images"/>
+  </build>
+  <publish>
+    <disable repository="phantomcharts"/>
+  </publish>
+  <repository name="phantomcharts">
+    <arch>x86_64</arch>
+  </repository>
+{%- endif %}
+{%- for repository in ["images", "test_manifest_images"] %}
+  <repository name="{{ repository }}">
+    {%- if release_project is defined and repository == "images" %}
+    <releasetarget project="{{ release_project }}" repository="images" trigger="manual"/>
+    {%- endif %}
+    <path project="SUSE:Registry" repository="standard"/>
+    <path project="SUSE:CA" repository="SLE_15_SP6"/>
+    <path project="{{ project }}" repository="standard"/>
+    <arch>x86_64</arch>
+    <arch>aarch64</arch>
+  </repository>
+{%- endfor %}
+  <repository name="standard" block="local">
+    {%- if release_project is defined and not for_release %}
+    <releasetarget project="{{ release_project }}" repository="standard" trigger="manual"/>
+    {%- endif %}
+    <path project="Cloud:OpenStack:2024.2" repository="15.6"/>
+    <path project="SUSE:SLE-15-SP6:Update" repository="standard"/>
+    <arch>x86_64</arch>
+    <arch>aarch64</arch>
+  </repository>
+  <repository name="charts"{{ ' rebuild="local"' if not base_project }}>
+    {%- if release_project is defined and not for_release %}
+    <releasetarget project="{{ release_project }}" repository="phantomcharts" trigger="manual"/>
+    {%- endif %}
+    <path project="{{ project }}" repository="standard"/>
+    <arch>x86_64</arch>
+  </repository>
+  {%- if for_release %}
+  <repository name="releasecharts" rebuild="local">
+    <releasetarget project="{{ release_project }}" repository="charts" trigger="manual"/>
+    <path project="{{ project }}" repository="standard"/>
+    <arch>x86_64</arch>
+  </repository>
+  {%- endif %}
+</project>