Merge pull request #1212 from dmach/behave-parallel

behave: Speed running tests up by preparing containers in advance

.github/workflows/tests.yaml

@@ -108,4 +108,4 @@ jobs:
       - name: "Run tests"
         run: |
           cd behave
-          behave -Dosc=../osc-wrapper.py
+          behave -Dosc=../osc-wrapper.py -Dmax_podman_containers=2

behave/features/environment.py

@@ -22,7 +22,7 @@ def after_scenario(context, scenario):
         # start a new container after a destructive test
         # we must use an existing podman instance defined in `before_all` due to context attribute life-cycle:
         # https://behave.readthedocs.io/en/stable/context_attributes.html
-        context.podman.restart()
+        context.podman.new_container()
     context.osc.clear()
     common.check_exit_code(context)
 
@@ -47,7 +47,12 @@ def before_all(context):
     # absolute path to .../behave/fixtures
     context.fixtures = os.path.join(os.path.dirname(__file__), "..", "fixtures")
 
-    context.podman = podman.Podman(context)
+    podman_max_containers = context.config.userdata.get("podman_max_containers", None)
+    if podman_max_containers:
+        podman_max_containers = int(podman_max_containers)
+        context.podman = podman.ThreadedPodman(context, container_name_prefix="osc-behave-", max_containers=podman_max_containers)
+    else:
+        context.podman = podman.Podman(context, container_name="osc-behave")
     context.osc = osc.Osc(context)
 
 

behave/features/steps/osc.py

@@ -35,7 +35,7 @@ class Osc:
         with open(self.oscrc, "w") as f:
             f.write("[general]\n")
             f.write("\n")
-            f.write(f"[https://localhost:{self.context.podman.port}]\n")
+            f.write(f"[https://localhost:{self.context.podman.container.port}]\n")
             f.write("user=Admin\n")
             f.write("pass=opensuse\n")
             f.write("credentials_mgr_class=osc.credentials.PlaintextConfigFileCredentialsManager\n")
@@ -48,7 +48,7 @@ class Osc:
         osc_cmd = self.context.config.userdata.get("osc", "osc")
         cmd = [osc_cmd]
         cmd += ["--config", self.oscrc]
-        cmd += ["-A", f"https://localhost:{self.context.podman.port}"]
+        cmd += ["-A", f"https://localhost:{self.context.podman.container.port}"]
         return cmd
 
 

behave/features/steps/podman.py

@@ -1,13 +1,126 @@
+import queue
 import subprocess
+import threading
 
 from steps.common import debug
 
 
 class Podman:
-    def __init__(self, context):
+    def __init__(self, context, container_name):
         self.context = context
-        debug(context, "Podman.__init__()")
+        self.container_name = container_name
+        self.container = None
+        debug(self.context, "Podman.__init__()")
+
+        self.new_container()
+
+    def __del__(self):
+        debug(self.context, "Podman.__del__()")
+        try:
+            self.kill()
+        except Exception:
+            pass
+
+    def kill(self):
+        debug(self.context, "Podman.kill()")
+        if not self.container:
+            return
+        self.container.kill()
+        self.container = None
+
+    def new_container(self):
+        debug(self.context, "Podman.new_container()")
+        # no need to stop the running container
+        # becuse the new container replaces an old container with the identical name
+        self.container = Container(self.context, name=self.container_name)
+
+
+class ThreadedPodman:
+    def __init__(self, context, container_name_prefix, max_containers=1):
+        self.context = context
+        self.container = None
+        debug(self.context, "ThreadedPodman.__init__()")
+
+        self.max_containers = max_containers
+        self.container_name_prefix = container_name_prefix
+        self.container_name_num = 0
+
+        # produce new containers
+        self.container_producer_queue = queue.Queue(maxsize=self.max_containers)
+        self.container_producer_queue_is_stopping = threading.Event()
+        self.container_producer_queue_is_stopped = threading.Event()
+        self.container_producer_thread = threading.Thread(target=self.container_producer, daemon=True)
+        self.container_producer_thread.start()
+
+        # consume (kill) used containers
+        self.container_consumer_queue = queue.Queue()
+        self.container_consumer_thread = threading.Thread(target=self.container_consumer, daemon=True)
+        self.container_consumer_thread.start()
+
+        self.new_container()
+
+    def __del__(self):
+        debug(self.context, "ThreadedPodman.__del__()")
+        try:
+            self.kill()
+        except Exception:
+            pass
+
+    def kill(self):
+        debug(self.context, "ThreadedPodman.kill()")
+        self.container_producer_queue_is_stopping.set()
+
+        container = getattr(self, "container", None)
+        if container:
+            self.container_consumer_queue.put(container)
+            self.container = None
+
+        while not self.container_producer_queue_is_stopped.is_set():
+            try:
+                container = self.container_producer_queue.get(block=True, timeout=1)
+                self.container_consumer_queue.put(container)
+            except queue.Empty:
+                continue
+
+        # 'None' is a signal to finish processing the queue
+        self.container_consumer_queue.put(None)
+
+        self.container_producer_thread.join()
+        self.container_consumer_thread.join()
+
+    def container_producer(self):
+        while not self.container_producer_queue_is_stopping.is_set():
+            if self.container_name_prefix:
+                self.container_name_num += 1
+                container_name = f"{self.container_name_prefix}{self.container_name_num}"
+            else:
+                container_name = None
+            container = Container(self.context, name=container_name)
+            self.container_producer_queue.put(container, block=True)
+        self.container_producer_queue_is_stopped.set()
+
+    def container_consumer(self):
+        while True:
+            container = self.container_consumer_queue.get(block=True)
+            if container is None:
+                break
+            container.kill()
+
+    def new_container(self):
+        debug(self.context, "ThreadedPodman.new_container()")
+        if getattr(self, "container", None):
+            self.container_consumer_queue.put(self.container)
+        self.container = self.container_producer_queue.get(block=True)
+        debug(self.context, f"> {self.container}")
+
+
+class Container:
+    def __init__(self, context, name=None):
+        self.context = context
+        debug(self.context, "Container.__init__()")
+        self.container_name = name
         self.container_id = None
+        self.port = None
         self.start()
 
     def __del__(self):
@@ -16,6 +129,11 @@ class Podman:
         except Exception:
             pass
 
+    def __repr__(self):
+        result = super().__repr__()
+        result += f"(port:{self.port}, id:{self.container_id}, name:{self.container_name})"
+        return result
+
     def _run(self, args, check=True):
         cmd = ["podman"] + args
         debug(self.context, "Running command:", cmd)
@@ -32,12 +150,18 @@ class Podman:
         return proc
 
     def start(self):
-        debug(self.context, "Podman.start()")
+        debug(self.context, "Container.start()")
         args = [
             "run",
-            "--name", "obs-server-behave",
             "--hostname", "obs-server-behave",
-            "--replace",
+        ]
+        if self.container_name:
+            args += [
+                "--name", self.container_name,
+                "--replace",
+                "--stop-signal", "SIGKILL",
+            ]
+        args += [
             "--rm",
             "--detach",
             "--interactive",
@@ -54,13 +178,13 @@ class Podman:
     def kill(self):
         if not self.container_id:
             return
-        debug(self.context, "Podman.kill()")
+        debug(self.context, "Container.kill()")
         args = ["kill", self.container_id]
         self._run(args)
         self.container_id = None
 
     def restart(self):
-        debug(self.context, "Podman.restart()")
+        debug(self.context, "Container.restart()")
         self.kill()
         self.start()