Synchronize metadata from template #76

Merged
nbelouin merged 6 commits from nbelouin/Factory:sync_meta into main 2025-02-21 15:04:46 +01:00
11 changed files with 344 additions and 263 deletions

View File

@@ -0,0 +1,45 @@
name: Synchronize Project Config
on:
pull_request_target:
types:
- opened
- reopened
- synchronize
- closed
branches-ignore:
- "devel"
jobs:
sync-prjconf:
runs-on: tumbleweed
steps:
- run: |
zypper in -y python3-jinja2
mkdir -p ~/.config/osc
cat >~/.config/osc/oscrc <<'EOF'
[general]
apiurl = https://api.opensuse.org
[https://api.opensuse.org]
user=${{ vars.OBS_USERNAME }}
pass=${{ secrets.OBS_PASSWORD }}
EOF
# Waiting on PR to get merged for support in upstream action/checkout action
- uses: 'https://github.com/yangskyboxlabs/action-checkout@sha256'
with:
object-format: 'sha256'
- if: gitea.event.action == "closed"
run: |
PROJECT="$(grep PROJECT .obs/common.py | sed 's/PROJECT = "\(.*\)"/\1/')"
osc rdelete "${PROJECT}:Staging:PR-${{ gitea.event.number }}"
- if: gitea.event.action != "closed"
env:
SCM_URL: ${{ gitea.event.pull_request.head.repo.clone_url }}#${{ gitea.head_ref }}
run: |
PROJECT="$(grep PROJECT .obs/common.py | sed 's/PROJECT = "\(.*\)"/\1/')"
python3 .obs/render_meta.py --pr ${{ gitea.event.number }} --scm-url "${SCM_URL}" | osc meta prj "${PROJECT}:Staging:PR-${{ gitea.event.number }}" -F -
- if: gitea.event.action != "closed"
run: |
PROJECT="$(grep PROJECT .obs/common.py | sed 's/PROJECT = "\(.*\)"/\1/')"
export OBS_PROJECT="${PROJECT}:Staging:PR-${{ gitea.event.number }}"
python3 .obs/wait_obs.py

View File

@@ -1,9 +1,8 @@
name: Synchronize Project Config
on:
push:
branches:
- "main"
- "3.*"
branches-ignore:
- "devel"
paths:
- "_config"
- ".gitea/workflows/sync_config.yaml"
@@ -14,7 +13,7 @@ jobs:
steps:
- run: |
mkdir -p ~/.config/osc
cat >~/.config/osc/oscrc <<EOF
cat >~/.config/osc/oscrc <<'EOF'
[general]
apiurl = https://api.opensuse.org
@@ -28,6 +27,6 @@ jobs:
object-format: 'sha256'
- run: |
PROJECT="$(grep PROJECT .obs/common.py | sed 's/PROJECT = "\(.*\)"/\1/')"
if [ "$(osc meta prjconf "${PROJECT}" | sha256sum)" = "$(cat _config | sha256sum)" ] ; then
if [ "$(osc meta prjconf "${PROJECT}" | sha256sum)" != "$(cat _config | sha256sum)" ] ; then
osc meta prjconf "${PROJECT}" -F _config
fi

View File

@@ -0,0 +1,42 @@
name: Synchronize Project Metadata
on:
push:
branches-ignore:
- "devel"
paths:
- "*" # Will trigger on new directories and changes to files in root of repository
- ".gitea/workflows/sync_meta.yaml"
- ".obs/common.py"
jobs:
sync-prjconf:
runs-on: tumbleweed
steps:
- run: |
zypper in -y python3-jinja2
mkdir -p ~/.config/osc
cat >~/.config/osc/oscrc <<'EOF'
[general]
apiurl = https://api.opensuse.org
[https://api.opensuse.org]
user=${{ vars.OBS_USERNAME }}
pass=${{ secrets.OBS_PASSWORD }}
EOF
# Waiting on PR to get merged for support in upstream action/checkout action
- uses: 'https://github.com/yangskyboxlabs/action-checkout@sha256'
with:
object-format: 'sha256'
- run: |
PROJECT="$(grep PROJECT .obs/common.py | sed 's/PROJECT = "\(.*\)"/\1/')"
set -o pipefail
if meta="$(osc meta prj "${PROJECT}" 2>/dev/null | sha256sum)"; then
new_meta="$(python3 .obs/render_meta.py)"
if [ "${meta}" != "$(echo "${new_meta}" | sha256sum)"]; then
echo "${new_meta}" | osc meta prj "${PROJECT}" -F -
fi
python3 .obs/sync_packages.py
else
# Create the projects
bash .obs/create_projects.sh
fi

View File

@@ -1,5 +1,4 @@
#!/usr/bin/env python3
import yaml
import subprocess
import argparse
import os
@@ -7,30 +6,6 @@ import os.path
from common import PROJECT, REPOSITORY, BRANCH
def add_package_to_workflow(name: str):
modified = False
with open(".obs/workflows.yml", "r") as wf_file:
workflows = yaml.safe_load(wf_file)
if not any(
x
for x in workflows["staging_build"]["steps"]
if x["branch_package"]["source_package"] == name
):
workflows["staging_build"]["steps"].append(
{
"branch_package": {
"source_project": PROJECT,
"target_project": f"{PROJECT}:Staging",
"source_package": name,
}
}
)
modified = True
if modified:
with open(".obs/workflows.yml", "w") as wf_file:
yaml.dump(workflows, wf_file)
def add_package_to_project(name: str):
package_meta = f"""<package name="{name}" project="{PROJECT}">
<title/>
@@ -53,7 +28,6 @@ def add_package(package_name: str):
os.exit(1)
add_package_to_project(package_name)
add_package_to_workflow(package_name)
def main():
@@ -65,7 +39,7 @@ def main():
add_package(args.package)
print("Package created in OBS, you can now push the modified workflow file")
print("Package created in OBS !")
if __name__ == '__main__':

37
.obs/create_projects.sh Normal file
View File

@@ -0,0 +1,37 @@
#!/bin/bash
show_help() {
echo "Usage: $(basename $0) [--internal]"
echo "options:"
echo "-h, --help display this help and exit"
echo "-i, --internal create project as internal"
exit 0
}
while [[ "$#" -gt 0 ]]; do
case $1 in
-h|--help) show_help;;
-i|--internal) internal="--internal" ;;
*) echo "Unknown parameter passed: $1";show_help ;;
esac
shift
done
PROJECT="$(grep PROJECT .obs/common.py | sed 's/PROJECT = "\(.*\)"/\1/')"
EXTRA_OSC_ARGS=""
if [ -n "$internal" ]; then
PROJECT="ISV${PROJECT:3}"
EXTRA_OSC_ARGS="-A https://api.suse.de"
python3 .obs/render_meta.py ${internal} Snapshot | osc ${EXTRA_OSC_ARGS} meta prj "${PROJECT}:Snapshot" -F -
osc ${EXTRA_OSC_ARGS} meta prjconf "${PROJECT}:Snapshot" -F _config
fi
python3 .obs/render_meta.py ${internal} ToTest | osc ${EXTRA_OSC_ARGS} meta prj "${PROJECT}:ToTest" -F -
python3 .obs/render_meta.py ${internal} | osc ${EXTRA_OSC_ARGS} meta prj "${PROJECT}" -F -
osc ${EXTRA_OSC_ARGS} meta prjconf "${PROJECT}:ToTest" -F _config
osc ${EXTRA_OSC_ARGS} meta prjconf "${PROJECT}" -F _config
if [ -z "$internal" ]; then
python3 .obs/sync_packages.py
fi

View File

@@ -1,5 +1,4 @@
#!/usr/bin/env python3
import yaml
import subprocess
import argparse
import os
@@ -8,18 +7,6 @@ import os.path
from common import PROJECT
def delete_package_from_workflow(name: str):
with open(".obs/workflows.yml", "r") as wf_file:
workflows = yaml.safe_load(wf_file)
workflows["staging_build"]["steps"] = [
x
for x in workflows["staging_build"]["steps"]
if x["branch_package"]["source_package"] != name
]
with open(".obs/workflows.yml", "w") as wf_file:
yaml.dump(workflows, wf_file)
def delete_package_from_project(name: str):
p = subprocess.run(["osc", "rdelete", PROJECT, name, "-m \"Deleted via delete_package.py\"" ], stdout=subprocess.PIPE)
print(p.stdout)
@@ -33,7 +20,6 @@ def delete_package(package_name: str):
os.exit(1)
delete_package_from_project(package_name)
delete_package_from_workflow(package_name)
def main():

62
.obs/render_meta.py Normal file
View File

@@ -0,0 +1,62 @@
import argparse
from jinja2 import Template
from common import PROJECT
def render(base_project, subproject, internal, scm_url=None):
version = base_project.rsplit(':', 1)[-1]
context = {
"base_project": subproject == "",
"title": f"SUSE Edge {version} {subproject}".rstrip(),
}
if subproject == "ToTest":
context["project"] = f"{base_project}:ToTest"
context["description"] = (
f"This project doesn't build, it stores a snapshot of SUSE Edge {version} "
"project currently going through the automated test layer"
)
if "Factory" in base_project or internal:
context["release_project"] = f"{base_project}:Snapshot"
elif subproject == "Snapshot":
context["project"] = f"{base_project}:Snapshot"
context["release_project"] = f"{base_project.rsplit(':', 1)[0]}:Containers"
context["for_release"] = True
context["description"] = (
f"This project doesn't build, it stores a snapshot of SUSE Edge {version} "
"project that passed automated test layer"
)
elif subproject == "":
context["project"] = base_project
context["release_project"] = f"{base_project}:ToTest"
else: # PR case direct python call
context["base_project"] = True
context["project"] = f"{base_project}:{subproject}"
if scm_url is not None:
context["scm_url"] = scm_url
with open("_meta") as meta:
template = Template(meta.read())
return template.render(context)
def main():
parser = argparse.ArgumentParser(
prog='ProgramName',
description='What the program does',
epilog='Text at the bottom of help')
parser.add_argument("subproject", default="", choices=["", "ToTest", "Snapshot"], nargs="?")
parser.add_argument("--internal", action="store_true")
parser.add_argument("--pr")
parser.add_argument("--scm-url")
args = parser.parse_args()
base_project = PROJECT.replace("isv", "ISV", 1) if args.internal else PROJECT
print(render(
base_project=base_project,
subproject=args.subproject if args.pr is None else f"Staging:PR-{args.pr}",
internal=args.internal,
scm_url=args.scm_url,
))
if __name__ == "__main__":
main()

81
.obs/wait_obs.py Normal file
View File

@@ -0,0 +1,81 @@
import xml.etree.ElementTree as ET
import subprocess
import time
import os
from collections import Counter
def get_buildstatus(project: str) -> ET.Element:
for _ in range(5):
try:
output = subprocess.check_output(["osc", "pr", "--xml", project])
return ET.fromstring(output)
except subprocess.CalledProcessError:
continue
def do_wait(project:str, commit:str) -> ET.Element:
last_state = None
while True:
time.sleep(5)
status = get_buildstatus(project)
if last_state == status.get("state"):
continue
else:
last_state = status.get("state")
scminfo = { e.text for e in status.findall(".//scminfo") }
if len(scminfo) != 1 or scminfo.pop() != commit:
print("Waiting for OBS to sync with SCM")
continue
if not all([ e.get('state') == "published" and e.get('dirty') is None for e in status.findall("./result")]):
print("Waiting for OBS to finish building")
continue
return status
def print_results(status: ET.Element) -> bool:
results = {}
failed = []
for e in status.findall("./result"):
repo = results.get(e.get("repository"), {})
repo[e.get("arch")] = e
results[e.get("repository")] = repo
for repo in results.keys():
print(f"{repo}:")
depth=1
for arch in results[repo].keys():
counts = Counter()
if repo != "charts":
print(f"\t{arch}:")
depth=2
for package in results[repo][arch].findall("./status"):
if package.get("code") in ["excluded", "disabled"]:
continue
if package.get("code") in ["failed", "unresolvable", "broken"]:
details = package.findtext("details")
if details:
failed.append(f"{package.get('package')} ({arch}): {details}")
else:
failed.append(f"{package.get('package')} ({arch})")
counts[package.get("code")] += 1
for (code, count) in counts.items():
print("\t"*depth, f"{code}: {count}")
failed.sort()
if failed:
print("\nPackages failing: ")
for fail in failed:
print("\t", fail)
return len(failed) != 0
def main():
project = os.environ.get("OBS_PROJECT")
sha = os.environ.get("GITEA_SHA")
status = do_wait(project, sha)
if print_results(status):
return 1
if __name__ == "__main__":
main()

View File

@@ -1,216 +0,0 @@
staging_build:
filters:
event: pull_request
steps:
- branch_package:
source_package: endpoint-copier-operator
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: endpoint-copier-operator-image
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: endpoint-copier-operator-chart
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: akri
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: akri-agent-image
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: akri-chart
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: akri-controller-image
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: akri-dashboard-extension-chart
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: akri-debug-echo-discovery-handler-image
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: akri-onvif-discovery-handler-image
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: akri-opcua-discovery-handler-image
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: akri-udev-discovery-handler-image
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: akri-webhook-configuration-image
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: obs-service-set_version
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: cosign
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: frr-k8s
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: kubectl
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: upgrade-controller
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: nm-configurator
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: kube-rbac-proxy
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: edge-image-builder
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: metallb
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: hauler
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: baremetal-operator
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: cdi-chart
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: metallb-chart
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: sriov-crd-chart
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: upgrade-controller-chart
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: edge-image-builder-image
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: ironic-ipa-downloader-image
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: upgrade-controller-image
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: metal3-chart
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: baremetal-operator-image
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: sriov-network-operator-chart
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: metallb-controller-image
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: metallb-speaker-image
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: ironic-image
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: cri-tools
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: crudini
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: fakeroot
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: ipcalc
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: autoconf
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: rancher-turtles-airgap-resources-chart
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: rancher-turtles-chart
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: kube-rbac-proxy-image
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: ironic-ipa-ramdisk
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: kubevirt-dashboard-extension-chart
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: kiwi-builder-image
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: kubevirt-chart
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: release-manifest-image
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: frr-image
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: kubectl-image
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging
- branch_package:
source_package: frr-k8s-image
source_project: isv:SUSE:Edge:Factory
target_project: isv:SUSE:Edge:Factory:Staging

69
_meta Normal file
View File

@@ -0,0 +1,69 @@
{#-
This template is rendered by the render_meta.py script
it is not automatically enforced by OBS
-#}
{%- set maintainers = [
"edge-engineering",
] -%}
<project name="{{ project }}">
<title>{{ title }}</title>
{%- if description is defined %}
<description>{{ description }}</description>
{%- else %}
<description/>
{%- endif %}
{%- if scm_url is defined %}
<scmsync>{{ scm_url }}</scmsync>
{%- endif %}
{%- for maintainer in maintainers %}
<person userid="{{ maintainer }}" role="maintainer"/>
{%- endfor %}
{%- if not base_project %}
<build>
<disable/>
<enable repository="charts"/>
<enable repository="test_manifest_images"/>
</build>
<publish>
<disable repository="phantomcharts"/>
</publish>
<repository name="phantomcharts">
<arch>x86_64</arch>
</repository>
{%- endif %}
{%- for repository in ["images", "test_manifest_images"] %}
<repository name="{{ repository }}">
{%- if release_project is defined and repository == "images" %}
<releasetarget project="{{ release_project }}" repository="images" trigger="manual"/>
{%- endif %}
<path project="SUSE:Registry" repository="standard"/>
<path project="SUSE:CA" repository="SLE_15_SP6"/>
<path project="{{ project }}" repository="standard"/>
<arch>x86_64</arch>
<arch>aarch64</arch>
</repository>
{%- endfor %}
<repository name="standard" block="local">
{%- if release_project is defined and not for_release %}
<releasetarget project="{{ release_project }}" repository="standard" trigger="manual"/>
{%- endif %}
<path project="Cloud:OpenStack:2024.2" repository="15.6"/>
<path project="SUSE:SLE-15-SP6:Update" repository="standard"/>
<arch>x86_64</arch>
<arch>aarch64</arch>
</repository>
<repository name="charts"{{ ' rebuild="local"' if not base_project }}>
{%- if release_project is defined and not for_release %}
<releasetarget project="{{ release_project }}" repository="phantomcharts" trigger="manual"/>
{%- endif %}
<path project="{{ project }}" repository="standard"/>
<arch>x86_64</arch>
</repository>
{%- if for_release %}
<repository name="releasecharts" rebuild="local">
<releasetarget project="{{ release_project }}" repository="charts" trigger="manual"/>
<path project="{{ project }}" repository="standard"/>
<arch>x86_64</arch>
</repository>
{%- endif %}
</project>

View File

@@ -10,7 +10,9 @@
<service name="cargo_vendor" mode="manual">
<param name="srcdir">akri</param>
</service>
<service name="tar" mode="buildtime" />
<service name="tar" mode="buildtime">
<param name="obsinfo">akri.obsinfo</param>
Review

Is this related to the rest of the changes?

Is this related to the rest of the changes?
Review

No, introduced it in my branch for testing purpose and did not remove, it is a real fix to the package though, so if you don't mind it sneaking in this PR then I'll leave it there.

No, introduced it in my branch for testing purpose and did not remove, it is a real fix to the package though, so if you don't mind it sneaking in this PR then I'll leave it there.
</service>
<service name="set_version" mode="buildtime" >
<param name="fromfile">version.txt</param>
<param name="regex">^(.*)$</param>