diff --git a/container_cleaner.py b/container_cleaner.py new file mode 100755 index 00000000..f63edad9 --- /dev/null +++ b/container_cleaner.py @@ -0,0 +1,151 @@ +#!/usr/bin/python2 +# (c) 2019 fvogt@suse.de +# GPLv3-only + +from __future__ import print_function + +import osc.conf +import osc.core +import logging +import ToolBase +import sys +import re +from lxml import etree as xml + + +class ContainerCleaner(ToolBase.ToolBase): + def __init__(self): + ToolBase.ToolBase.__init__(self) + self.logger = logging.getLogger(__name__) + + def getDirEntries(self, path): + url = self.makeurl(path) + directory = xml.parse(self.retried_GET(url)) + return directory.xpath("entry/@name") + + def getDirBinaries(self, path): + url = self.makeurl(path) + directory = xml.parse(self.retried_GET(url)) + return directory.xpath("binary/@filename") + + def findSourcepkgsToDelete(self, project): + # Get a list of all images + srccontainers = self.getDirEntries(["source", project]) + + # Sort them into buckets for each package: + # {"opensuse-tumbleweed-image": ["opensuse-tumbleweed-image.20190402134201", ...]} + buckets = {} + regex_maintenance_release = re.compile(R"^(.+)\.[0-9]+$") + for srccontainer in srccontainers: + # Get the right bucket + match = regex_maintenance_release.match(srccontainer) + if match: + # Maintenance release + package = match.group(1) + else: + # Not renamed + package = srccontainer + + if package not in buckets: + buckets[package] = [] + + buckets[package] += [srccontainer] + + for package in buckets: + # Sort each bucket: Newest provider first + buckets[package].sort(reverse=True) + logging.debug("Found %d providers of %s", len(buckets[package]), package) + + # Get a hash for sourcecontainer -> arch with binaries + # {"opensuse-tumbleweed-image.20190309164844": ["aarch64", "armv7l", "armv6l"], + # "kubic-pause-image.20190306124139": ["x86_64", "i586"], ... } + srccontainerarchs = {} + + archs = self.getDirEntries(["build", project, "containers"]) + regex_srccontainer = re.compile(R"^([^:]+)(:[^:]+)?$") + for arch in archs: + buildcontainers = self.getDirEntries(["build", project, "containers", arch]) + for buildcontainer in buildcontainers: + bins = self.getDirBinaries(["build", project, "containers", arch, buildcontainer]) + if len(bins) > 0: + match = regex_srccontainer.match(buildcontainer) + if not match: + raise Exception("Could not map %s to source container" % buildcontainer) + + srccontainer = match.group(1) + if srccontainer not in srccontainers: + raise Exception("Mapped %s to wrong source container (%s)" % (buildcontainer, srccontainer)) + + if srccontainer not in srccontainerarchs: + srccontainerarchs[srccontainer] = [] + + logging.debug("%s provides binaries for %s", srccontainer, arch) + srccontainerarchs[srccontainer] += [arch] + + # Now go through each bucket and find out what doesn't contribute to the newest five + can_delete = [] + for package in buckets: + # {"x86_64": 1, "aarch64": 2, ...} + archs_found = {} + for arch in archs: + archs_found[arch] = 0 + + for srccontainer in buckets[package]: + contributes = False + if srccontainer in srccontainerarchs: + for arch in srccontainerarchs[srccontainer]: + if archs_found[arch] < 5: + archs_found[arch] += 1 + contributes = True + + if contributes: + logging.debug("%s contributes to %s", srccontainer, package) + else: + logging.info("%s does not contribute", srccontainer) + if len([count for count in archs_found.values() if count > 0]) == 0: + # If there are A, B, C and D, with only C and D providing binaries, + # A and B aren't deleted because they have newer sources. This is + # to avoid deleting something due to unforeseen circumstances, e.g. + # OBS didn't copy the binaries yet. + logging.info("No newer provider found either, ignoring") + else: + can_delete += [srccontainer] + + return can_delete + + def run(self, project): + packages = self.findSourcepkgsToDelete(project) + + for package in packages: + url = self.makeurl(["source", project, package]) + if self.dryrun: + logging.info("DELETE %s", url) + else: + osc.core.http_DELETE(url) + + +class CommandLineInterface(ToolBase.CommandLineInterface): + def __init__(self, *args, **kwargs): + ToolBase.CommandLineInterface.__init__(self, args, kwargs) + + def setup_tool(self): + tool = ContainerCleaner() + if self.options.debug: + logging.basicConfig(level=logging.DEBUG) + elif self.options.verbose: + logging.basicConfig(level=logging.INFO) + + return tool + + def do_run(self, subcmd, opts, project): + """${cmd_name}: run the Container cleaner for the specified project + + ${cmd_usage} + ${cmd_option_list} + """ + + self.tool.run(project) + +if __name__ == "__main__": + cli = CommandLineInterface() + sys.exit(cli.main()) diff --git a/dist/package/openSUSE-release-tools.spec b/dist/package/openSUSE-release-tools.spec index 60cda1e6..bd9b040c 100644 --- a/dist/package/openSUSE-release-tools.spec +++ b/dist/package/openSUSE-release-tools.spec @@ -431,6 +431,7 @@ exit 0 %{_bindir}/osrt-check_source_in_factory %{_bindir}/osrt-check_tags_in_requests %{_bindir}/osrt-compare_pkglist +%{_bindir}/osrt-container_cleaner %{_bindir}/osrt-deptool %{_bindir}/osrt-fcc_submitter %{_bindir}/osrt-findfileconflicts diff --git a/gocd/container-cleaner.gocd.yaml b/gocd/container-cleaner.gocd.yaml new file mode 100644 index 00000000..4f96c186 --- /dev/null +++ b/gocd/container-cleaner.gocd.yaml @@ -0,0 +1,23 @@ +format_version: 3 +pipelines: + ContainerCleaner.Tumbleweed: + group: openSUSE.Checkers + lock_behavior: unlockWhenFinished + environment_variables: + OSC_CONFIG: /home/go/config/oscrc-totest-manager + materials: + script: + git: https://github.com/openSUSE/openSUSE-release-tools.git + branch: master + destination: scripts + timer: + spec: 0 0 0-23/2 ? * * + only_on_changes: false + stages: + - Run: + approval: manual + resources: + - staging-bot + tasks: + - script: |- + scripts/container_cleaner.py --verbose run openSUSE:Containers:Tumbleweed diff --git a/tests/container_cleaner_tests.py b/tests/container_cleaner_tests.py new file mode 100644 index 00000000..9792ef4d --- /dev/null +++ b/tests/container_cleaner_tests.py @@ -0,0 +1,129 @@ +import unittest + +from container_cleaner import ContainerCleaner + +class MockedContainerCleaner(ContainerCleaner): + def __init__(self, container_arch_map): + self.container_arch_map = container_arch_map + + def getDirEntries(self, path): + """Mock certain OBS APIs returning directory entries""" + if path == ["source", "mock:prj"]: + srccontainers = [a.split(":")[0] for a in self.container_arch_map.keys()] + return list(set(srccontainers)) # Remove duplicates + elif path == ["build", "mock:prj", "containers"]: + all_archs = [] + for archs in self.container_arch_map.values(): + all_archs += archs + + return list(set(all_archs)) + elif path[0:3] == ["build", "mock:prj", "containers"] and len(path) == 4: + arch = path[3] + ret = [] + for srccontainer in self.container_arch_map: + ret += [srccontainer] + + return ret + else: + raise RuntimeError("Path %s not expected" % path) + + def getDirBinaries(self, path): + """Mock certain OBS APIs returning a list of binaries""" + if path[0:3] == ["build", "mock:prj", "containers"] and len(path) == 5: + arch = path[3] + srccontainer = path[4] + if arch in self.container_arch_map[srccontainer]: + return ["A binary"] + + return [] + else: + raise RuntimeError("Path %s not expected" % path) + + +class TestContainerCleaner(unittest.TestCase): + def doTest(self, container_arch_map, to_be_deleted_exp): + cleaner = MockedContainerCleaner(container_arch_map) + to_be_deleted = cleaner.findSourcepkgsToDelete("mock:prj") + to_be_deleted.sort() + self.assertEqual(to_be_deleted, to_be_deleted_exp) + + def test_empty(self): + """Empty project, do nothing""" + container_arch_map = {} + + to_be_deleted_exp = [] + + return self.doTest(container_arch_map, to_be_deleted_exp) + + def test_nothingToDo(self): + """Non-empty project, still do nothing""" + container_arch_map = {"c": ["i586", "x86_64"], + "c.01": ["i586"], + "c.02": ["x86_64"], + "c.04": ["i586", "x86_64"], + "c.06": ["i586"], + "c.07": ["x86_64"], + "c.08": ["i586", "x86_64"], + "c.11": [], + "d.42": [], "d.43": []} + + to_be_deleted_exp = [] + + return self.doTest(container_arch_map, to_be_deleted_exp) + + def test_multiplePackages(self): + """Multiple packages in one project""" + container_arch_map = {"c": ["i586", "x86_64"], + "c.01": ["i586"], + "c.02": ["x86_64"], + "c.03": [], + "c.04": ["i586", "x86_64"], + "c.05": ["i586", "x86_64"], + "c.06": ["i586"], + "c.07": ["x86_64"], + "c.08": ["i586", "x86_64"], + "c.09": ["i586", "x86_64"], + "c.10": ["i586", "x86_64"], + "c.11": [], + "d.42": [], "d.43": [], + "e.51": ["i586"], + "e.52": ["aarch64"], + "e.53": ["i586"], + "e.54": ["i586"], + "e.55": ["i586"], + "e.56": ["i586"], + "e.57": ["i586"]} + + to_be_deleted_exp = ["c", "c.01", "c.02", "c.03", "c.04", + "e.51"] + + return self.doTest(container_arch_map, to_be_deleted_exp) + + def test_multibuild(self): + """Packages using _multbuild. + There is no special handling for _multibuild - It's enough if any flavor has binaries.""" + container_arch_map = {"c:docker": ["i586", "x86_64"], + "c.01:docker": ["i586"], + "c.02:lxc": ["x86_64"], + "c.03:docker": [], + "c.04": ["i586", "x86_64"], + "c.05:docker": ["i586", "x86_64"], + "c.06:docker": ["i586"], + "c.07:docker": ["x86_64"], + "c.08:docker": ["i586", "x86_64"], + "c.09:docker": ["i586", "x86_64"], + "c.10:docker": ["i586", "x86_64"], + "c.11:docker": [], + "d.42:lxc": [], "d.43": [], + "e.51": ["i586"], + "e.52": ["aarch64"], + "e.53": ["i586"], + "e.54:docker": ["i586"], + "e.55:docker": ["i586"], + "e.56": ["i586"], + "e.57": ["i586"]} + + to_be_deleted_exp = ["c", "c.01", "c.02", "c.03", "c.04", + "e.51"] + + return self.doTest(container_arch_map, to_be_deleted_exp)