From a66d40fe3a7c68463f9dd41dd5de5ba59ceb5141 Mon Sep 17 00:00:00 2001 From: Daniel Mach Date: Mon, 16 Jan 2023 10:19:28 +0100 Subject: [PATCH] behave: Speed running tests up by preparing containers in advance --- .github/workflows/tests.yaml | 2 +- behave/features/environment.py | 9 ++- behave/features/steps/osc.py | 4 +- behave/features/steps/podman.py | 138 ++++++++++++++++++++++++++++++-- 4 files changed, 141 insertions(+), 12 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index b96ebd4a..04bf1641 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -108,4 +108,4 @@ jobs: - name: "Run tests" run: | cd behave - behave -Dosc=../osc-wrapper.py + behave -Dosc=../osc-wrapper.py -Dmax_podman_containers=2 diff --git a/behave/features/environment.py b/behave/features/environment.py index 93fa77df..ee8c216a 100644 --- a/behave/features/environment.py +++ b/behave/features/environment.py @@ -22,7 +22,7 @@ def after_scenario(context, scenario): # start a new container after a destructive test # we must use an existing podman instance defined in `before_all` due to context attribute life-cycle: # https://behave.readthedocs.io/en/stable/context_attributes.html - context.podman.restart() + context.podman.new_container() context.osc.clear() common.check_exit_code(context) @@ -47,7 +47,12 @@ def before_all(context): # absolute path to .../behave/fixtures context.fixtures = os.path.join(os.path.dirname(__file__), "..", "fixtures") - context.podman = podman.Podman(context) + podman_max_containers = context.config.userdata.get("podman_max_containers", None) + if podman_max_containers: + podman_max_containers = int(podman_max_containers) + context.podman = podman.ThreadedPodman(context, container_name_prefix="osc-behave-", max_containers=podman_max_containers) + else: + context.podman = podman.Podman(context, container_name="osc-behave") context.osc = osc.Osc(context) diff --git a/behave/features/steps/osc.py b/behave/features/steps/osc.py index 86a03f47..cd3bd1f4 100644 --- a/behave/features/steps/osc.py +++ b/behave/features/steps/osc.py @@ -35,7 +35,7 @@ class Osc: with open(self.oscrc, "w") as f: f.write("[general]\n") f.write("\n") - f.write(f"[https://localhost:{self.context.podman.port}]\n") + f.write(f"[https://localhost:{self.context.podman.container.port}]\n") f.write("user=Admin\n") f.write("pass=opensuse\n") f.write("credentials_mgr_class=osc.credentials.PlaintextConfigFileCredentialsManager\n") @@ -48,7 +48,7 @@ class Osc: osc_cmd = self.context.config.userdata.get("osc", "osc") cmd = [osc_cmd] cmd += ["--config", self.oscrc] - cmd += ["-A", f"https://localhost:{self.context.podman.port}"] + cmd += ["-A", f"https://localhost:{self.context.podman.container.port}"] return cmd diff --git a/behave/features/steps/podman.py b/behave/features/steps/podman.py index ae9bc5ff..7614492e 100644 --- a/behave/features/steps/podman.py +++ b/behave/features/steps/podman.py @@ -1,13 +1,126 @@ +import queue import subprocess +import threading from steps.common import debug class Podman: - def __init__(self, context): + def __init__(self, context, container_name): self.context = context - debug(context, "Podman.__init__()") + self.container_name = container_name + self.container = None + debug(self.context, "Podman.__init__()") + + self.new_container() + + def __del__(self): + debug(self.context, "Podman.__del__()") + try: + self.kill() + except Exception: + pass + + def kill(self): + debug(self.context, "Podman.kill()") + if not self.container: + return + self.container.kill() + self.container = None + + def new_container(self): + debug(self.context, "Podman.new_container()") + # no need to stop the running container + # becuse the new container replaces an old container with the identical name + self.container = Container(self.context, name=self.container_name) + + +class ThreadedPodman: + def __init__(self, context, container_name_prefix, max_containers=1): + self.context = context + self.container = None + debug(self.context, "ThreadedPodman.__init__()") + + self.max_containers = max_containers + self.container_name_prefix = container_name_prefix + self.container_name_num = 0 + + # produce new containers + self.container_producer_queue = queue.Queue(maxsize=self.max_containers) + self.container_producer_queue_is_stopping = threading.Event() + self.container_producer_queue_is_stopped = threading.Event() + self.container_producer_thread = threading.Thread(target=self.container_producer, daemon=True) + self.container_producer_thread.start() + + # consume (kill) used containers + self.container_consumer_queue = queue.Queue() + self.container_consumer_thread = threading.Thread(target=self.container_consumer, daemon=True) + self.container_consumer_thread.start() + + self.new_container() + + def __del__(self): + debug(self.context, "ThreadedPodman.__del__()") + try: + self.kill() + except Exception: + pass + + def kill(self): + debug(self.context, "ThreadedPodman.kill()") + self.container_producer_queue_is_stopping.set() + + container = getattr(self, "container", None) + if container: + self.container_consumer_queue.put(container) + self.container = None + + while not self.container_producer_queue_is_stopped.is_set(): + try: + container = self.container_producer_queue.get(block=True, timeout=1) + self.container_consumer_queue.put(container) + except queue.Empty: + continue + + # 'None' is a signal to finish processing the queue + self.container_consumer_queue.put(None) + + self.container_producer_thread.join() + self.container_consumer_thread.join() + + def container_producer(self): + while not self.container_producer_queue_is_stopping.is_set(): + if self.container_name_prefix: + self.container_name_num += 1 + container_name = f"{self.container_name_prefix}{self.container_name_num}" + else: + container_name = None + container = Container(self.context, name=container_name) + self.container_producer_queue.put(container, block=True) + self.container_producer_queue_is_stopped.set() + + def container_consumer(self): + while True: + container = self.container_consumer_queue.get(block=True) + if container is None: + break + container.kill() + + def new_container(self): + debug(self.context, "ThreadedPodman.new_container()") + if getattr(self, "container", None): + self.container_consumer_queue.put(self.container) + self.container = self.container_producer_queue.get(block=True) + debug(self.context, f"> {self.container}") + + +class Container: + def __init__(self, context, name=None): + self.context = context + debug(self.context, "Container.__init__()") + self.container_name = name self.container_id = None + self.port = None self.start() def __del__(self): @@ -16,6 +129,11 @@ class Podman: except Exception: pass + def __repr__(self): + result = super().__repr__() + result += f"(port:{self.port}, id:{self.container_id}, name:{self.container_name})" + return result + def _run(self, args, check=True): cmd = ["podman"] + args debug(self.context, "Running command:", cmd) @@ -32,12 +150,18 @@ class Podman: return proc def start(self): - debug(self.context, "Podman.start()") + debug(self.context, "Container.start()") args = [ "run", - "--name", "obs-server-behave", "--hostname", "obs-server-behave", - "--replace", + ] + if self.container_name: + args += [ + "--name", self.container_name, + "--replace", + "--stop-signal", "SIGKILL", + ] + args += [ "--rm", "--detach", "--interactive", @@ -54,13 +178,13 @@ class Podman: def kill(self): if not self.container_id: return - debug(self.context, "Podman.kill()") + debug(self.context, "Container.kill()") args = ["kill", self.container_id] self._run(args) self.container_id = None def restart(self): - debug(self.context, "Podman.restart()") + debug(self.context, "Container.restart()") self.kill() self.start()