Compare commits
1 Commits
staging-co
...
t-refactor
| Author | SHA256 | Date | |
|---|---|---|---|
|
430dfc7296 |
@@ -40,10 +40,12 @@ jobs:
|
||||
run: make down
|
||||
working-directory: ./autogits/integration
|
||||
- name: Start images
|
||||
run: make up
|
||||
run: |
|
||||
make up
|
||||
make wait_healthy
|
||||
working-directory: ./autogits/integration
|
||||
- name: Run tests
|
||||
run: py.test-3.11 -v tests
|
||||
run: make pytest
|
||||
working-directory: ./autogits/integration
|
||||
- name: Make sure the pod is down
|
||||
if: always()
|
||||
|
||||
@@ -4,7 +4,7 @@ ENV container=podman
|
||||
|
||||
ENV LANG=en_US.UTF-8
|
||||
|
||||
RUN zypper -vvvn install podman podman-compose vim make python3-pytest python3-requests python3-pytest-dependency
|
||||
RUN zypper -vvvn install podman podman-compose vim make python3-pytest python3-requests python3-pytest-dependency python3-pytest-httpserver
|
||||
|
||||
COPY . /opt/project/
|
||||
|
||||
@@ -1,51 +1,19 @@
|
||||
# We want to be able to test in two **modes**:
|
||||
# A. bots are used from official packages as defined in */Dockerfile.package
|
||||
# B. bots are just picked up from binaries that are placed in corresponding parent directory.
|
||||
|
||||
# The topology is defined in podman-compose file and can be spawned in two ways:
|
||||
# 1. Privileged container (needs no additional dependancies)
|
||||
# 2. podman-compose on a local machine (needs dependencies as defined in the Dockerfile)
|
||||
|
||||
# 1. podman-compose on a local machine (needs dependencies as defined in the Dockerfile)
|
||||
# 2. pytest in a dedicated container (recommended)
|
||||
|
||||
# Typical workflow:
|
||||
# A1: - run 'make test_package'
|
||||
# B1: - run 'make test_local' (make sure that the go binaries in parent folder are built)
|
||||
# A2:
|
||||
# 1. 'make build_package' - prepares images (recommended, otherwise there might be surprises if image fails to build during `make up`)
|
||||
# 2. 'make up' - spawns podman-compose
|
||||
# 3. 'pytest -v tests/*' - run tests
|
||||
# 4. 'make down' - once the containers are not needed
|
||||
# B2: (make sure the go binaries in the parent folder are built)
|
||||
# 1. 'make build_local' - prepared images (recommended, otherwise there might be surprises if image fails to build during `make up`)
|
||||
# 2. 'make up' - spawns podman-compose
|
||||
# 3. 'pytest -v tests/*' - run tests
|
||||
# 4. 'make down' - once the containers are not needed
|
||||
|
||||
# 1. 'make build' - prepares images
|
||||
# 2. 'make up' - spawns podman-compose
|
||||
# 3. 'make pytest' - run tests inside the tester container
|
||||
# 4. 'make down' - once the containers are not needed
|
||||
#
|
||||
# OR just run 'make test' to do it all at once.
|
||||
|
||||
AUTO_DETECT_MODE := $(shell if test -e ../workflow-pr/workflow-pr; then echo .local; else echo .package; fi)
|
||||
|
||||
# try to detect mode B1, otherwise mode A1
|
||||
test: GIWTF_IMAGE_SUFFIX=$(AUTO_DETECT_MODE)
|
||||
test: build_container test_container
|
||||
|
||||
# mode A1
|
||||
test_package: GIWTF_IMAGE_SUFFIX=.package
|
||||
test_package: build_container test_container
|
||||
|
||||
# mode B1
|
||||
test_local: GIWTF_IMAGE_SUFFIX=.local
|
||||
test_local: build_container test_container
|
||||
|
||||
MODULES := gitea-events-rabbitmq-publisher obs-staging-bot workflow-pr
|
||||
|
||||
# Prepare topology 1
|
||||
build_container:
|
||||
podman build ../ -f integration/Dockerfile -t autogits_integration
|
||||
|
||||
# Run tests in topology 1
|
||||
test_container:
|
||||
podman run --rm --privileged -t -e GIWTF_IMAGE_SUFFIX=$(GIWTF_IMAGE_SUFFIX) autogits_integration /usr/bin/bash -c "make build && make up && sleep 25 && pytest -v tests/*"
|
||||
|
||||
# Default test target
|
||||
test: test_b
|
||||
|
||||
build_local: AUTO_DETECT_MODE=.local
|
||||
build_local: build
|
||||
@@ -53,16 +21,66 @@ build_local: build
|
||||
build_package: AUTO_DETECT_MODE=.package
|
||||
build_package: build
|
||||
|
||||
# parse all service images from podman-compose and build them (topology 2)
|
||||
# parse all service images from podman-compose and build them
|
||||
# mode B with pytest in container
|
||||
test_b: AUTO_DETECT_MODE=.local
|
||||
test_b: build up wait_healthy pytest
|
||||
|
||||
# Complete cycle for CI
|
||||
test-ci: test_b down
|
||||
|
||||
wait_healthy:
|
||||
@echo "Waiting for services to be healthy..."
|
||||
@echo "Waiting for gitea (max 2m)..."
|
||||
@start_time=$$(date +%s); \
|
||||
until podman exec gitea-test curl -f -s http://localhost:3000/api/v1/version >/dev/null 2>&1; do \
|
||||
current_time=$$(date +%s); \
|
||||
elapsed=$$((current_time - start_time)); \
|
||||
if [ $$elapsed -gt 120 ]; then \
|
||||
echo "ERROR: Gitea failed to start within 2 minutes."; \
|
||||
echo "--- Troubleshooting Info ---"; \
|
||||
echo "Diagnostics output (curl):"; \
|
||||
podman exec gitea-test curl -v http://localhost:3000/api/v1/version || true; \
|
||||
echo "--- Container Logs ---"; \
|
||||
podman logs gitea-test --tail 20; \
|
||||
echo "--- Container Status ---"; \
|
||||
podman inspect gitea-test --format '{{.State.Status}}'; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
sleep 2; \
|
||||
done
|
||||
@echo "Waiting for rabbitmq (max 2m)..."
|
||||
@start_time=$$(date +%s); \
|
||||
until podman exec rabbitmq-test rabbitmq-diagnostics check_running -q >/dev/null 2>&1; do \
|
||||
current_time=$$(date +%s); \
|
||||
elapsed=$$((current_time - start_time)); \
|
||||
if [ $$elapsed -gt 120 ]; then \
|
||||
echo "ERROR: RabbitMQ failed to start within 2 minutes."; \
|
||||
echo "--- Troubleshooting Info ---"; \
|
||||
echo "Diagnostics output:"; \
|
||||
podman exec rabbitmq-test rabbitmq-diagnostics check_running || true; \
|
||||
echo "--- Container Logs ---"; \
|
||||
podman logs rabbitmq-test --tail 20; \
|
||||
echo "--- Container Status ---"; \
|
||||
podman inspect rabbitmq-test --format '{{.State.Status}}'; \
|
||||
exit 1; \
|
||||
fi; \
|
||||
sleep 2; \
|
||||
done
|
||||
@echo "All services are healthy!"
|
||||
|
||||
pytest:
|
||||
podman-compose exec tester pytest -v tests/*
|
||||
|
||||
build:
|
||||
podman pull docker.io/library/rabbitmq:3.13.7-management
|
||||
for i in $$(grep -A 1000 services: podman-compose.yml | grep -oE '^ [^: ]+'); do GIWTF_IMAGE_SUFFIX=$(AUTO_DETECT_MODE) podman-compose build $$i || exit 1; done
|
||||
|
||||
# this will spawn prebuilt containers (topology 2)
|
||||
# this will spawn prebuilt containers
|
||||
up:
|
||||
podman-compose up -d
|
||||
|
||||
# tear down (topology 2)
|
||||
# tear down
|
||||
down:
|
||||
podman-compose down
|
||||
|
||||
@@ -73,4 +91,3 @@ up-bots-package:
|
||||
# mode B
|
||||
up-bots-local:
|
||||
GIWTF_IMAGE_SUFFIX=.local podman-compose up -d
|
||||
|
||||
|
||||
52
integration/Makefile.md
Normal file
52
integration/Makefile.md
Normal file
@@ -0,0 +1,52 @@
|
||||
# Makefile Targets
|
||||
|
||||
This document describes the targets available in the `integration/Makefile`.
|
||||
|
||||
## Primary Workflow
|
||||
|
||||
### `test` (or `test_b`)
|
||||
- **Action**: Performs a complete build-and-test cycle.
|
||||
- **Steps**:
|
||||
1. `build`: Prepares all container images.
|
||||
2. `up`: Starts all services via `podman-compose`.
|
||||
3. `wait_healthy`: Polls Gitea and RabbitMQ until they are ready.
|
||||
4. `pytest`: Executes the test suite inside the `tester` container.
|
||||
- **Outcome**: The environment remains active for fast iteration.
|
||||
|
||||
### `test-ci`
|
||||
- **Action**: Performs the full `test` cycle followed by teardown.
|
||||
- **Steps**: `test_b` -> `down`
|
||||
- **Purpose**: Ideal for CI environments where a clean state is required after testing.
|
||||
|
||||
---
|
||||
|
||||
## Individual Targets
|
||||
|
||||
### `build`
|
||||
- **Action**: Pulls external images (RabbitMQ) and builds all local service images defined in `podman-compose.yml`.
|
||||
- **Note**: Use `build_local` or `build_package` to specify bot source mode.
|
||||
|
||||
### `up`
|
||||
- **Action**: Starts the container topology in detached mode.
|
||||
|
||||
### `wait_healthy`
|
||||
- **Action**: Polls the health status of `gitea-test` and `rabbitmq-test` containers.
|
||||
- **Purpose**: Ensures infrastructure is stable before test execution.
|
||||
|
||||
### `pytest`
|
||||
- **Action**: Runs `pytest -v tests/*` inside the running `tester` container.
|
||||
- **Requirement**: The environment must already be started via `up`.
|
||||
|
||||
### `down`
|
||||
- **Action**: Stops and removes all containers and networks defined in the compose file.
|
||||
|
||||
---
|
||||
|
||||
## Configuration Modes
|
||||
|
||||
The Makefile supports two deployment modes via `GIWTF_IMAGE_SUFFIX`:
|
||||
|
||||
- **.local** (Default): Uses binaries built from the local source (requires `make build` in project root).
|
||||
- **.package**: Uses official pre-built packages for the bots.
|
||||
|
||||
Targets like `build_local`, `build_package`, `up-bots-local`, and `up-bots-package` allow for explicit mode selection.
|
||||
@@ -1,57 +0,0 @@
|
||||
+-------------------------------------------------------------------------------------------------+
|
||||
| Makefile Targets |
|
||||
+-------------------------------------------------------------------------------------------------+
|
||||
| |
|
||||
| [Default Test Workflow] |
|
||||
| test (Auto-detects mode: .local or .package) |
|
||||
| └─> build_container |
|
||||
| └─> test_container |
|
||||
| |
|
||||
| [Specific Test Workflows - Topology 1: Privileged Container] |
|
||||
| test_package (Mode A1: Bots from official packages) |
|
||||
| └─> build_container |
|
||||
| └─> test_container |
|
||||
| |
|
||||
| test_local (Mode B1: Bots from local binaries) |
|
||||
| └─> build_container |
|
||||
| └─> test_container |
|
||||
| |
|
||||
| build_container |
|
||||
| - Action: Builds the `autogits_integration` privileged container image. |
|
||||
| - Purpose: Prepares an environment for running tests within a single container. |
|
||||
| |
|
||||
| test_container |
|
||||
| - Action: Runs `autogits_integration` container, executes `make build`, `make up`, and |
|
||||
| `pytest -v tests/*` inside it. |
|
||||
| - Purpose: Executes the full test suite in Topology 1 (privileged container). |
|
||||
| |
|
||||
| [Build & Orchestration Workflows - Topology 2: podman-compose] |
|
||||
| |
|
||||
| build_package (Mode A: Builds service images from official packages) |
|
||||
| └─> build |
|
||||
| |
|
||||
| build_local (Mode B: Builds service images from local binaries) |
|
||||
| └─> build |
|
||||
| |
|
||||
| build |
|
||||
| - Action: Pulls `rabbitmq` image and iterates through `podman-compose.yml` services |
|
||||
| to build each one. |
|
||||
| - Purpose: Prepares all necessary service images for Topology 2 deployment. |
|
||||
| |
|
||||
| up |
|
||||
| - Action: Starts all services defined in `podman-compose.yml` in detached mode. |
|
||||
| - Purpose: Deploys the application topology (containers) for testing or development. |
|
||||
| |
|
||||
| down |
|
||||
| - Action: Stops and removes all services started by `up`. |
|
||||
| - Purpose: Cleans up the deployed application topology. |
|
||||
| |
|
||||
| up-bots-package (Mode A: Spawns Topology 2 with official package bots) |
|
||||
| - Action: Calls `podman-compose up -d` with `GIWTF_IMAGE_SUFFIX=.package`. |
|
||||
| - Purpose: Specifically brings up the environment using official package bots. |
|
||||
| |
|
||||
| up-bots-local (Mode B: Spawns Topology 2 with local binaries) |
|
||||
| - Action: Calls `podman-compose up -d` with `GIWTF_IMAGE_SUFFIX=.local`. |
|
||||
| - Purpose: Specifically brings up the environment using local binaries. |
|
||||
| |
|
||||
+-------------------------------------------------------------------------------------------------+
|
||||
@@ -1,14 +0,0 @@
|
||||
# Use a base Python image
|
||||
FROM registry.suse.com/bci/python:3.11
|
||||
|
||||
# Set the working directory
|
||||
WORKDIR /app
|
||||
|
||||
# Copy the server script
|
||||
COPY server.py .
|
||||
|
||||
# Expose the port the server will run on
|
||||
EXPOSE 8080
|
||||
|
||||
# Command to run the server
|
||||
CMD ["python3", "-u", "server.py"]
|
||||
@@ -1,18 +0,0 @@
|
||||
<project name="openSUSE:Leap:16.0:PullRequest">
|
||||
<title>Leap 16.0 PullRequest area</title>
|
||||
<description>Base project to define the pull request builds</description>
|
||||
<person userid="autogits_obs_staging_bot" role="maintainer"/>
|
||||
<person userid="maxlin_factory" role="maintainer"/>
|
||||
<group groupid="maintenance-opensuse.org" role="maintainer"/>
|
||||
<debuginfo>
|
||||
<enable/>
|
||||
</debuginfo>
|
||||
<repository name="standard">
|
||||
<path project="openSUSE:Leap:16.0" repository="standard"/>
|
||||
<arch>x86_64</arch>
|
||||
<arch>i586</arch>
|
||||
<arch>aarch64</arch>
|
||||
<arch>ppc64le</arch>
|
||||
<arch>s390x</arch>
|
||||
</repository>
|
||||
</project>
|
||||
@@ -1,59 +0,0 @@
|
||||
<project name="openSUSE:Leap:16.0">
|
||||
<title>openSUSE Leap 16.0 based on SLFO</title>
|
||||
<description>Leap 16.0 based on SLES 16.0 (specifically SLFO:1.2)</description>
|
||||
<link project="openSUSE:Backports:SLE-16.0"/>
|
||||
<scmsync>http://gitea-test:3000/myproducts/mySLFO#staging-main</scmsync>
|
||||
<person userid="dimstar_suse" role="maintainer"/>
|
||||
<person userid="lkocman-factory" role="maintainer"/>
|
||||
<person userid="maxlin_factory" role="maintainer"/>
|
||||
<person userid="factory-auto" role="reviewer"/>
|
||||
<person userid="licensedigger" role="reviewer"/>
|
||||
<group groupid="autobuild-team" role="maintainer"/>
|
||||
<group groupid="factory-maintainers" role="maintainer"/>
|
||||
<group groupid="maintenance-opensuse.org" role="maintainer"/>
|
||||
<group groupid="factory-staging" role="reviewer"/>
|
||||
<build>
|
||||
<disable repository="ports"/>
|
||||
</build>
|
||||
<debuginfo>
|
||||
<enable/>
|
||||
</debuginfo>
|
||||
<repository name="standard" rebuild="local">
|
||||
<path project="openSUSE:Backports:SLE-16.0" repository="standard"/>
|
||||
<path project="SUSE:SLFO:1.2" repository="standard"/>
|
||||
<arch>local</arch>
|
||||
<arch>i586</arch>
|
||||
<arch>x86_64</arch>
|
||||
<arch>aarch64</arch>
|
||||
<arch>ppc64le</arch>
|
||||
<arch>s390x</arch>
|
||||
</repository>
|
||||
<repository name="product">
|
||||
<releasetarget project="openSUSE:Leap:16.0:ToTest" repository="product" trigger="manual"/>
|
||||
<path project="openSUSE:Leap:16.0:NonFree" repository="standard"/>
|
||||
<path project="openSUSE:Leap:16.0" repository="images"/>
|
||||
<path project="openSUSE:Leap:16.0" repository="standard"/>
|
||||
<path project="openSUSE:Backports:SLE-16.0" repository="standard"/>
|
||||
<path project="SUSE:SLFO:1.2" repository="standard"/>
|
||||
<arch>local</arch>
|
||||
<arch>i586</arch>
|
||||
<arch>x86_64</arch>
|
||||
<arch>aarch64</arch>
|
||||
<arch>ppc64le</arch>
|
||||
<arch>s390x</arch>
|
||||
</repository>
|
||||
<repository name="ports">
|
||||
<arch>armv7l</arch>
|
||||
</repository>
|
||||
<repository name="images">
|
||||
<releasetarget project="openSUSE:Leap:16.0:ToTest" repository="images" trigger="manual"/>
|
||||
<path project="openSUSE:Leap:16.0" repository="standard"/>
|
||||
<path project="openSUSE:Backports:SLE-16.0" repository="standard"/>
|
||||
<path project="SUSE:SLFO:1.2" repository="standard"/>
|
||||
<arch>i586</arch>
|
||||
<arch>x86_64</arch>
|
||||
<arch>aarch64</arch>
|
||||
<arch>ppc64le</arch>
|
||||
<arch>s390x</arch>
|
||||
</repository>
|
||||
</project>
|
||||
@@ -1,140 +0,0 @@
|
||||
import http.server
|
||||
import socketserver
|
||||
import os
|
||||
import logging
|
||||
import signal
|
||||
import sys
|
||||
import threading
|
||||
import fnmatch
|
||||
|
||||
PORT = 8080
|
||||
RESPONSE_DIR = "/app/responses"
|
||||
STATE_DIR = "/tmp/mock_obs_state"
|
||||
|
||||
class MockOBSHandler(http.server.SimpleHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
logging.info(f"GET request for: {self.path}")
|
||||
path_without_query = self.path.split('?')[0]
|
||||
|
||||
# Check for state stored by a PUT request first
|
||||
sanitized_put_path = 'PUT' + path_without_query.replace('/', '_')
|
||||
state_file_path = os.path.join(STATE_DIR, sanitized_put_path)
|
||||
if os.path.exists(state_file_path):
|
||||
logging.info(f"Found stored PUT state for {self.path} at {state_file_path}")
|
||||
self.send_response(200)
|
||||
self.send_header("Content-type", "application/xml")
|
||||
file_size = os.path.getsize(state_file_path)
|
||||
self.send_header("Content-Length", str(file_size))
|
||||
self.end_headers()
|
||||
with open(state_file_path, 'rb') as f:
|
||||
self.wfile.write(f.read())
|
||||
return
|
||||
|
||||
# If no PUT state file, fall back to the glob/exact match logic
|
||||
self.handle_request('GET')
|
||||
|
||||
def do_PUT(self):
|
||||
logging.info(f"PUT request for: {self.path}")
|
||||
logging.info(f"Headers: {self.headers}")
|
||||
path_without_query = self.path.split('?')[0]
|
||||
|
||||
body = b''
|
||||
if self.headers.get('Transfer-Encoding', '').lower() == 'chunked':
|
||||
logging.info("Chunked transfer encoding detected")
|
||||
while True:
|
||||
line = self.rfile.readline().strip()
|
||||
if not line:
|
||||
break
|
||||
chunk_length = int(line, 16)
|
||||
if chunk_length == 0:
|
||||
self.rfile.readline()
|
||||
break
|
||||
body += self.rfile.read(chunk_length)
|
||||
self.rfile.read(2) # Read the trailing CRLF
|
||||
else:
|
||||
content_length = int(self.headers.get('Content-Length', 0))
|
||||
body = self.rfile.read(content_length)
|
||||
|
||||
logging.info(f"Body: {body.decode('utf-8')}")
|
||||
sanitized_path = 'PUT' + path_without_query.replace('/', '_')
|
||||
state_file_path = os.path.join(STATE_DIR, sanitized_path)
|
||||
|
||||
logging.info(f"Saving state for {self.path} to {state_file_path}")
|
||||
os.makedirs(os.path.dirname(state_file_path), exist_ok=True)
|
||||
with open(state_file_path, 'wb') as f:
|
||||
f.write(body)
|
||||
|
||||
self.send_response(200)
|
||||
self.send_header("Content-type", "text/plain")
|
||||
response_body = b"OK"
|
||||
self.send_header("Content-Length", str(len(response_body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(response_body)
|
||||
|
||||
def do_POST(self):
|
||||
logging.info(f"POST request for: {self.path}")
|
||||
self.handle_request('POST')
|
||||
|
||||
def do_DELETE(self):
|
||||
logging.info(f"DELETE request for: {self.path}")
|
||||
self.handle_request('DELETE')
|
||||
|
||||
def handle_request(self, method):
|
||||
path_without_query = self.path.split('?')[0]
|
||||
sanitized_request_path = method + path_without_query.replace('/', '_')
|
||||
logging.info(f"Handling request, looking for match for: {sanitized_request_path}")
|
||||
|
||||
response_file = None
|
||||
# Check for glob match first
|
||||
if os.path.exists(RESPONSE_DIR):
|
||||
for filename in os.listdir(RESPONSE_DIR):
|
||||
if fnmatch.fnmatch(sanitized_request_path, filename):
|
||||
response_file = os.path.join(RESPONSE_DIR, filename)
|
||||
logging.info(f"Found matching response file (glob): {response_file}")
|
||||
break
|
||||
|
||||
# Fallback to exact match if no glob match
|
||||
if response_file is None:
|
||||
exact_file = os.path.join(RESPONSE_DIR, sanitized_request_path)
|
||||
if os.path.exists(exact_file):
|
||||
response_file = exact_file
|
||||
logging.info(f"Found matching response file (exact): {response_file}")
|
||||
|
||||
if response_file:
|
||||
logging.info(f"Serving content from {response_file}")
|
||||
self.send_response(200)
|
||||
self.send_header("Content-type", "application/xml")
|
||||
file_size = os.path.getsize(response_file)
|
||||
self.send_header("Content-Length", str(file_size))
|
||||
self.end_headers()
|
||||
with open(response_file, 'rb') as f:
|
||||
self.wfile.write(f.read())
|
||||
else:
|
||||
logging.info(f"Response file not found for {sanitized_request_path}. Sending 404.")
|
||||
self.send_response(404)
|
||||
self.send_header("Content-type", "text/plain")
|
||||
body = f"Mock response not found for {sanitized_request_path}".encode('utf-8')
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')
|
||||
|
||||
if not os.path.exists(STATE_DIR):
|
||||
logging.info(f"Creating state directory: {STATE_DIR}")
|
||||
os.makedirs(STATE_DIR)
|
||||
if not os.path.exists(RESPONSE_DIR):
|
||||
os.makedirs(RESPONSE_DIR)
|
||||
|
||||
with socketserver.TCPServer(("", PORT), MockOBSHandler) as httpd:
|
||||
logging.info(f"Serving mock OBS API on port {PORT}")
|
||||
|
||||
def graceful_shutdown(sig, frame):
|
||||
logging.info("Received SIGTERM, shutting down gracefully...")
|
||||
threading.Thread(target=httpd.shutdown).start()
|
||||
|
||||
signal.signal(signal.SIGTERM, graceful_shutdown)
|
||||
|
||||
httpd.serve_forever()
|
||||
logging.info("Server has shut down.")
|
||||
64
integration/podman-compose.md
Normal file
64
integration/podman-compose.md
Normal file
@@ -0,0 +1,64 @@
|
||||
# Podman-Compose Services Architecture
|
||||
|
||||
This document describes the services defined in `podman-compose.yml` used for integration testing.
|
||||
|
||||
## Network
|
||||
- **gitea-network**: A bridge network that enables communication between all services.
|
||||
|
||||
## Services
|
||||
|
||||
### gitea
|
||||
- **Description**: Self-hosted Git service, serving as the central hub for repositories.
|
||||
- **Container Name**: `gitea-test`
|
||||
- **Image**: Built from `./gitea/Dockerfile`
|
||||
- **Ports**: `3000` (HTTP), `3022` (SSH)
|
||||
- **Volumes**: `./gitea-data` (persistent data), `./gitea-logs` (logs)
|
||||
- **Healthcheck**: Monitors the Gitea API version endpoint.
|
||||
|
||||
### rabbitmq
|
||||
- **Description**: Message broker for asynchronous communication between services.
|
||||
- **Container Name**: `rabbitmq-test`
|
||||
- **Image**: `rabbitmq:3.13.7-management`
|
||||
- **Ports**: `5671` (AMQP with TLS), `15672` (Management UI)
|
||||
- **Volumes**: `./rabbitmq-data`, `./rabbitmq-config/certs`, `./rabbitmq-config/rabbitmq.conf`, `./rabbitmq-config/definitions.json`
|
||||
- **Healthcheck**: Ensures the broker is running and ready to accept connections.
|
||||
|
||||
### gitea-publisher
|
||||
- **Description**: Publishes events from Gitea webhooks to the RabbitMQ message queue.
|
||||
- **Container Name**: `gitea-publisher`
|
||||
- **Dependencies**: `gitea` (started), `rabbitmq` (healthy)
|
||||
- **Topic Domain**: `suse`
|
||||
|
||||
### workflow-pr
|
||||
- **Description**: Manages pull request workflows, synchronizing between ProjectGit and PackageGit.
|
||||
- **Container Name**: `workflow-pr`
|
||||
- **Dependencies**: `gitea` (started), `rabbitmq` (healthy)
|
||||
- **Environment**: Configured via `AUTOGITS_*` variables.
|
||||
- **Volumes**: `./gitea-data` (read-only), `./workflow-pr/workflow-pr.json` (config), `./workflow-pr-repos` (working directories)
|
||||
|
||||
### tester
|
||||
- **Description**: The dedicated test runner container. It hosts the `pytest` suite and provides a mock OBS API using `pytest-httpserver`.
|
||||
- **Container Name**: `tester`
|
||||
- **Image**: Built from `./Dockerfile.tester`
|
||||
- **Mock API**: Listens on port `8080` within the container network to simulate OBS.
|
||||
- **Volumes**: Project root mounted at `/opt/project` for source access.
|
||||
|
||||
### obs-staging-bot
|
||||
- **Description**: Interacts with Gitea and the OBS API (mocked by `tester`) to manage staging projects.
|
||||
- **Container Name**: `obs-staging-bot`
|
||||
- **Dependencies**: `gitea` (started), `tester` (started)
|
||||
- **Environment**:
|
||||
- `AUTOGITS_STAGING_BOT_POLL_INTERVAL`: Set to `2s` for fast integration testing.
|
||||
- **Mock Integration**: Points to `http://tester:8080` for both OBS API and Web hosts.
|
||||
|
||||
---
|
||||
|
||||
## Testing Workflow
|
||||
|
||||
1. **Build**: `make build` (root) then `make build` (integration).
|
||||
2. **Up**: `make up` starts all services.
|
||||
3. **Wait**: `make wait_healthy` ensures infrastructure is ready.
|
||||
4. **Test**: `make pytest` runs the suite inside the `tester` container.
|
||||
5. **Down**: `make down` stops and removes containers.
|
||||
|
||||
Use `make test` to perform steps 1-4 automatically.
|
||||
@@ -1,77 +0,0 @@
|
||||
+-------------------------------------------------------------------------------------------------+
|
||||
| Podman-Compose Services Diagram |
|
||||
+-------------------------------------------------------------------------------------------------+
|
||||
| |
|
||||
| [Network] |
|
||||
| gitea-network (Bridge network for inter-service communication) |
|
||||
| |
|
||||
|-------------------------------------------------------------------------------------------------|
|
||||
| |
|
||||
| [Service: gitea] |
|
||||
| Description: Self-hosted Git service, central hub for repositories and code management. |
|
||||
| Container Name: gitea-test |
|
||||
| Image: Built from ./gitea Dockerfile |
|
||||
| Ports: 3000 (HTTP), 3022 (SSH) |
|
||||
| Volumes: ./gitea-data (for persistent data), ./gitea-logs (for logs) |
|
||||
| Network: gitea-network |
|
||||
| |
|
||||
|-------------------------------------------------------------------------------------------------|
|
||||
| |
|
||||
| [Service: rabbitmq] |
|
||||
| Description: Message broker for asynchronous communication between services. |
|
||||
| Container Name: rabbitmq-test |
|
||||
| Image: rabbitmq:3.13.7-management |
|
||||
| Ports: 5671 (AMQP), 15672 (Management UI) |
|
||||
| Volumes: ./rabbitmq-data (for persistent data), ./rabbitmq-config/certs (TLS certs), |
|
||||
| ./rabbitmq-config/rabbitmq.conf (config), ./rabbitmq-config/definitions.json (exchanges)|
|
||||
| Healthcheck: Ensures RabbitMQ is running and healthy. |
|
||||
| Network: gitea-network |
|
||||
| |
|
||||
|-------------------------------------------------------------------------------------------------|
|
||||
| |
|
||||
| [Service: gitea-publisher] |
|
||||
| Description: Publishes events from Gitea to the RabbitMQ message queue. |
|
||||
| Container Name: gitea-publisher |
|
||||
| Image: Built from ../gitea-events-rabbitmq-publisher/Dockerfile (local/package) |
|
||||
| Dependencies: gitea (started), rabbitmq (healthy) |
|
||||
| Environment: RABBITMQ_HOST, RABBITMQ_USERNAME, RABBITMQ_PASSWORD, SSL_CERT_FILE |
|
||||
| Command: Listens for Gitea events, publishes to 'suse' topic, debug enabled. |
|
||||
| Network: gitea-network |
|
||||
| |
|
||||
|-------------------------------------------------------------------------------------------------|
|
||||
| |
|
||||
| [Service: workflow-pr] |
|
||||
| Description: Manages pull request workflows, likely consuming events from RabbitMQ and |
|
||||
| interacting with Gitea. |
|
||||
| Container Name: workflow-pr |
|
||||
| Image: Built from ../workflow-pr/Dockerfile (local/package) |
|
||||
| Dependencies: gitea (started), rabbitmq (healthy) |
|
||||
| Environment: AMQP_USERNAME, AMQP_PASSWORD, SSL_CERT_FILE |
|
||||
| Volumes: ./gitea-data (read-only), ./workflow-pr/workflow-pr.json (config), |
|
||||
| ./workflow-pr-repos (for repositories) |
|
||||
| Command: Configures Gitea/RabbitMQ URLs, enables debug, manages repositories. |
|
||||
| Network: gitea-network |
|
||||
| |
|
||||
|-------------------------------------------------------------------------------------------------|
|
||||
| |
|
||||
| [Service: mock-obs] |
|
||||
| Description: A mock (simulated) service for the Open Build Service (OBS) for testing. |
|
||||
| Container Name: mock-obs |
|
||||
| Image: Built from ./mock-obs Dockerfile |
|
||||
| Ports: 8080 |
|
||||
| Volumes: ./mock-obs/responses (for mock API responses) |
|
||||
| Network: gitea-network |
|
||||
| |
|
||||
|-------------------------------------------------------------------------------------------------|
|
||||
| |
|
||||
| [Service: obs-staging-bot] |
|
||||
| Description: A bot that interacts with Gitea and the mock OBS, likely for staging processes. |
|
||||
| Container Name: obs-staging-bot |
|
||||
| Image: Built from ../obs-staging-bot/Dockerfile (local/package) |
|
||||
| Dependencies: gitea (started), mock-obs (started) |
|
||||
| Environment: OBS_USER, OBS_PASSWORD |
|
||||
| Volumes: ./gitea-data (read-only) |
|
||||
| Command: Configures Gitea/OBS URLs, enables debug. |
|
||||
| Network: gitea-network |
|
||||
| |
|
||||
+-------------------------------------------------------------------------------------------------+
|
||||
@@ -29,11 +29,6 @@ services:
|
||||
image: rabbitmq:3.13.7-management
|
||||
container_name: rabbitmq-test
|
||||
init: true
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "check_running", "-q"]
|
||||
interval: 30s
|
||||
timeout: 30s
|
||||
retries: 3
|
||||
networks:
|
||||
- gitea-network
|
||||
ports:
|
||||
@@ -104,17 +99,21 @@ services:
|
||||
]
|
||||
restart: unless-stopped
|
||||
|
||||
mock-obs:
|
||||
build: ./mock-obs
|
||||
container_name: mock-obs
|
||||
tester:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.tester
|
||||
container_name: tester
|
||||
init: true
|
||||
dns_search: .
|
||||
networks:
|
||||
- gitea-network
|
||||
ports:
|
||||
- "8080:8080"
|
||||
environment:
|
||||
- PYTEST_HTTPSERVER_HOST=0.0.0.0
|
||||
- PYTEST_HTTPSERVER_PORT=8080
|
||||
volumes:
|
||||
- ./mock-obs/responses:/app/responses:z # Use :z for shared SELinux label
|
||||
restart: unless-stopped
|
||||
- ..:/opt/project:z
|
||||
command: sleep infinity
|
||||
|
||||
obs-staging-bot:
|
||||
build:
|
||||
@@ -127,16 +126,17 @@ services:
|
||||
depends_on:
|
||||
gitea:
|
||||
condition: service_started
|
||||
mock-obs:
|
||||
tester:
|
||||
condition: service_started
|
||||
environment:
|
||||
- OBS_USER=mock
|
||||
- OBS_PASSWORD=mock-long-password
|
||||
- AUTOGITS_STAGING_BOT_POLL_INTERVAL=2s
|
||||
volumes:
|
||||
- ./gitea-data:/gitea-data:ro,z
|
||||
command:
|
||||
- "-debug"
|
||||
- "-gitea-url=http://gitea-test:3000"
|
||||
- "-obs=http://mock-obs:8080"
|
||||
- "-obs-web=http://mock-obs:8080"
|
||||
- "-obs=http://tester:8080"
|
||||
- "-obs-web=http://tester:8080"
|
||||
restart: unless-stopped
|
||||
|
||||
@@ -8,8 +8,74 @@ import time
|
||||
import os
|
||||
import json
|
||||
import base64
|
||||
import re
|
||||
from tests.lib.common_test_utils import GiteaAPIClient
|
||||
|
||||
class ObsMockState:
|
||||
def __init__(self):
|
||||
self.build_results = {} # project -> (package, code)
|
||||
self.project_metas = {} # project -> scmsync
|
||||
self.default_build_result = None
|
||||
|
||||
@pytest.fixture
|
||||
def obs_mock_state():
|
||||
return ObsMockState()
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def default_obs_handlers(httpserver, obs_mock_state):
|
||||
"""
|
||||
Sets up default handlers for OBS API to avoid 404s.
|
||||
"""
|
||||
def project_meta_handler(request):
|
||||
project = request.path.split("/")[2]
|
||||
scmsync = obs_mock_state.project_metas.get(project, "http://gitea-test:3000/myproducts/mySLFO.git")
|
||||
return f'<project name="{project}"><scmsync>{scmsync}</scmsync></project>'
|
||||
|
||||
def build_result_handler(request):
|
||||
project = request.path.split("/")[2]
|
||||
res = obs_mock_state.build_results.get(project) or obs_mock_state.default_build_result
|
||||
|
||||
if not res:
|
||||
return '<resultlist></resultlist>'
|
||||
|
||||
package_name, code = res
|
||||
|
||||
# We'll use a simple hardcoded XML here to avoid re-parsing template every time
|
||||
# or we can use the template. For simplicity, let's use a basic one.
|
||||
xml_template = f"""<resultlist state="mock">
|
||||
<result project="{project}" repository="standard" arch="x86_64" code="unpublished" state="unpublished">
|
||||
<scmsync>http://gitea-test:3000/myproducts/mySLFO.git?onlybuild={package_name}#sha</scmsync>
|
||||
<status package="{package_name}" code="{code}"/>
|
||||
</result>
|
||||
</resultlist>"""
|
||||
return xml_template
|
||||
|
||||
# Register handlers
|
||||
httpserver.expect_request(re.compile(r"/source/[^/]+/_meta$"), method="GET").respond_with_handler(project_meta_handler)
|
||||
httpserver.expect_request(re.compile(r"/build/[^/]+/_result"), method="GET").respond_with_handler(build_result_handler)
|
||||
httpserver.expect_request(re.compile(r"/source/[^/]+/_meta$"), method="PUT").respond_with_data("OK")
|
||||
httpserver.expect_request(re.compile(r"/source/[^/]+$"), method="DELETE").respond_with_data("OK")
|
||||
|
||||
@pytest.fixture
|
||||
def mock_build_result(obs_mock_state):
|
||||
"""
|
||||
Fixture to set up mock build results.
|
||||
"""
|
||||
def _setup_mock(package_name: str, code: str, project: str = None):
|
||||
if project:
|
||||
obs_mock_state.build_results[project] = (package_name, code)
|
||||
else:
|
||||
# If no project specified, we can't easily know which one to set
|
||||
# but usually it's the one the bot will request.
|
||||
# We'll use a special key to signify "all" or we can just wait for the request.
|
||||
# For now, let's assume we want to match openSUSE:Leap:16.0:PullRequest:*
|
||||
# The test will call it with specific project if needed.
|
||||
# In test_pr_workflow, it doesn't know the PR number yet.
|
||||
# So we'll make the handler fallback to this if project not found.
|
||||
obs_mock_state.default_build_result = (package_name, code)
|
||||
|
||||
return _setup_mock
|
||||
|
||||
BRANCH_CONFIG_COMMON = {
|
||||
"workflow.config": {
|
||||
"Workflows": ["pr"],
|
||||
@@ -163,8 +229,8 @@ def gitea_env():
|
||||
"""
|
||||
Global fixture to set up the Gitea environment for all tests.
|
||||
"""
|
||||
gitea_url = "http://127.0.0.1:3000"
|
||||
admin_token_path = "./gitea-data/admin.token"
|
||||
gitea_url = "http://gitea-test:3000"
|
||||
admin_token_path = os.path.join(os.path.dirname(__file__), "..", "gitea-data", "admin.token")
|
||||
|
||||
admin_token = None
|
||||
try:
|
||||
@@ -255,10 +321,6 @@ def gitea_env():
|
||||
# Setup users (using configs from this branch)
|
||||
setup_users_from_config(client, merged_configs.get("workflow.config", {}), merged_configs.get("_maintainership.json", {}))
|
||||
|
||||
if restart_needed:
|
||||
client.restart_service("workflow-pr")
|
||||
time.sleep(2) # Give it time to pick up changes
|
||||
|
||||
print("--- Gitea Global Setup Complete ---")
|
||||
yield client
|
||||
|
||||
|
||||
@@ -7,42 +7,6 @@ import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
import base64
|
||||
import subprocess
|
||||
|
||||
TEST_DATA_DIR = Path(__file__).parent.parent / "data"
|
||||
BUILD_RESULT_TEMPLATE = TEST_DATA_DIR / "build_result.xml.template"
|
||||
MOCK_RESPONSES_DIR = Path(__file__).parent.parent.parent / "mock-obs" / "responses"
|
||||
MOCK_BUILD_RESULT_FILE = (
|
||||
MOCK_RESPONSES_DIR / "GET_build_openSUSE:Leap:16.0:PullRequest:*__result"
|
||||
)
|
||||
MOCK_BUILD_RESULT_FILE1 = MOCK_RESPONSES_DIR / "GET_build_openSUSE:Leap:16.0__result"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_build_result():
|
||||
"""
|
||||
Fixture to create a mock build result file from the template.
|
||||
Returns a factory function that the test can call with parameters.
|
||||
"""
|
||||
|
||||
def _create_result_file(package_name: str, code: str):
|
||||
tree = ET.parse(BUILD_RESULT_TEMPLATE)
|
||||
root = tree.getroot()
|
||||
for status_tag in root.findall(".//status"):
|
||||
status_tag.set("package", package_name)
|
||||
status_tag.set("code", code)
|
||||
|
||||
MOCK_RESPONSES_DIR.mkdir(exist_ok=True)
|
||||
tree.write(MOCK_BUILD_RESULT_FILE)
|
||||
tree.write(MOCK_BUILD_RESULT_FILE1)
|
||||
return str(MOCK_BUILD_RESULT_FILE)
|
||||
|
||||
yield _create_result_file
|
||||
|
||||
if MOCK_BUILD_RESULT_FILE.exists():
|
||||
MOCK_BUILD_RESULT_FILE.unlink()
|
||||
MOCK_BUILD_RESULT_FILE1.unlink()
|
||||
|
||||
|
||||
class GiteaAPIClient:
|
||||
def __init__(self, base_url, token, sudo=None):
|
||||
@@ -117,18 +81,6 @@ class GiteaAPIClient:
|
||||
print(f"Organization '{org_name}' created.")
|
||||
else:
|
||||
raise
|
||||
print(f"--- Checking organization: {org_name} ---")
|
||||
try:
|
||||
self._request("GET", f"orgs/{org_name}")
|
||||
print(f"Organization '{org_name}' already exists.")
|
||||
except requests.exceptions.HTTPError as e:
|
||||
if e.response.status_code == 404:
|
||||
print(f"Creating organization '{org_name}'...")
|
||||
data = {"username": org_name, "full_name": org_name}
|
||||
self._request("POST", "orgs", json=data)
|
||||
print(f"Organization '{org_name}' created.")
|
||||
else:
|
||||
raise
|
||||
|
||||
def create_repo(self, org_name, repo_name):
|
||||
print(f"--- Checking repository: {org_name}/{repo_name} ---")
|
||||
@@ -347,8 +299,6 @@ index 0000000..{pkg_b_sha}
|
||||
raise
|
||||
raise Exception(f"Timeout waiting for branch {branch} in {owner}/{repo}")
|
||||
|
||||
|
||||
|
||||
def modify_gitea_pr(self, repo_full_name: str, pr_number: int, diff_content: str, message: str):
|
||||
owner, repo = repo_full_name.split("/")
|
||||
|
||||
@@ -503,16 +453,6 @@ index 0000000..{pkg_b_sha}
|
||||
time.sleep(1) # give a chance to avoid possible concurrency issues with reviews request/approval
|
||||
reviewer_client.create_review(repo_full_name, pr_number, event="APPROVED", body="Approving requested review")
|
||||
|
||||
def restart_service(self, service_name: str):
|
||||
print(f"--- Restarting service: {service_name} ---")
|
||||
try:
|
||||
# Assumes podman-compose.yml is in the parent directory of tests/lib
|
||||
subprocess.run(["podman-compose", "restart", service_name], check=True, cwd=os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)))
|
||||
print(f"Service {service_name} restarted successfully.")
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"Error restarting service {service_name}: {e}")
|
||||
raise
|
||||
|
||||
def wait_for_project_pr(self, package_pr_repo, package_pr_number, project_pr_repo="myproducts/mySLFO", timeout=60):
|
||||
print(f"Polling {package_pr_repo} PR #{package_pr_number} timeline for forwarded PR event in {project_pr_repo}...")
|
||||
for _ in range(timeout):
|
||||
@@ -554,4 +494,3 @@ index 0000000..{pkg_b_sha}
|
||||
|
||||
time.sleep(1)
|
||||
return package_merged, project_merged
|
||||
|
||||
|
||||
@@ -1,12 +1,9 @@
|
||||
import pytest
|
||||
import re
|
||||
import time
|
||||
import subprocess
|
||||
import requests
|
||||
from pathlib import Path
|
||||
from tests.lib.common_test_utils import (
|
||||
GiteaAPIClient,
|
||||
mock_build_result,
|
||||
)
|
||||
|
||||
# =============================================================================
|
||||
@@ -21,8 +18,6 @@ def test_pr_workflow_succeeded(staging_main_env, mock_build_result):
|
||||
pr = gitea_env.create_gitea_pr("mypool/pkgA", diff, "Test PR - should succeed", False, base_branch=merge_branch_name)
|
||||
initial_pr_number = pr["number"]
|
||||
|
||||
compose_dir = Path(__file__).parent.parent
|
||||
|
||||
forwarded_pr_number = gitea_env.wait_for_project_pr("mypool/pkgA", initial_pr_number)
|
||||
assert (
|
||||
forwarded_pr_number is not None
|
||||
@@ -43,17 +38,10 @@ def test_pr_workflow_succeeded(staging_main_env, mock_build_result):
|
||||
assert reviewer_added, "Staging bot was not added as a reviewer."
|
||||
print("Staging bot has been added as a reviewer.")
|
||||
|
||||
mock_build_result(package_name="pkgA", code="succeeded")
|
||||
|
||||
print("Restarting obs-staging-bot...")
|
||||
subprocess.run(
|
||||
["podman-compose", "restart", "obs-staging-bot"],
|
||||
cwd=compose_dir,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
mock_build_result(package_name="pkgA", code="succeeded")
|
||||
|
||||
print(f"Polling myproducts/mySLFO PR #{forwarded_pr_number} for final status...")
|
||||
|
||||
status_comment_found = False
|
||||
for _ in range(20):
|
||||
time.sleep(1)
|
||||
@@ -75,8 +63,6 @@ def test_pr_workflow_failed(staging_main_env, mock_build_result):
|
||||
pr = gitea_env.create_gitea_pr("mypool/pkgA", diff, "Test PR - should fail", False, base_branch=merge_branch_name)
|
||||
initial_pr_number = pr["number"]
|
||||
|
||||
compose_dir = Path(__file__).parent.parent
|
||||
|
||||
forwarded_pr_number = gitea_env.wait_for_project_pr("mypool/pkgA", initial_pr_number)
|
||||
assert (
|
||||
forwarded_pr_number is not None
|
||||
@@ -99,14 +85,6 @@ def test_pr_workflow_failed(staging_main_env, mock_build_result):
|
||||
|
||||
mock_build_result(package_name="pkgA", code="failed")
|
||||
|
||||
print("Restarting obs-staging-bot...")
|
||||
subprocess.run(
|
||||
["podman-compose", "restart", "obs-staging-bot"],
|
||||
cwd=compose_dir,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
print(f"Polling myproducts/mySLFO PR #{forwarded_pr_number} for final status...")
|
||||
status_comment_found = False
|
||||
for _ in range(20):
|
||||
|
||||
Reference in New Issue
Block a user