Compare commits

..

1 Commits

Author SHA256 Message Date
1fc0be5f60 make: refactor build target into per-module dependencies
Some checks failed
Integration tests / t (pull_request) Has been cancelled
- Replace shell loop in build with explicit module targets
- Add .PHONY for build and module targets
- Keep go build -C <module> -buildmode=pie behavior unchanged
- Enable parallel builds via make -jN build
2026-03-06 11:33:32 +01:00
24 changed files with 716 additions and 717 deletions

View File

@@ -40,12 +40,10 @@ jobs:
run: make down
working-directory: ./autogits/integration
- name: Start images
run: |
make up
make wait_healthy
run: make up
working-directory: ./autogits/integration
- name: Run tests
run: make pytest
run: py.test-3.11 -v tests
working-directory: ./autogits/integration
- name: Make sure the pod is down
if: always()

View File

@@ -1,4 +1,8 @@
MODULES := devel-importer utils/hujson utils/maintainer-update gitea-events-rabbitmq-publisher gitea_status_proxy group-review obs-forward-bot obs-staging-bot obs-status-service workflow-direct workflow-pr
build:
for m in $(MODULES); do go build -C $$m -buildmode=pie || exit 1 ; done
.PHONY: build $(MODULES)
build: $(MODULES)
$(MODULES):
go build -C $@ -buildmode=pie

View File

@@ -23,6 +23,7 @@ import (
"errors"
"fmt"
"io"
"log"
"os"
"slices"
"strings"
@@ -204,21 +205,16 @@ func ReadWorkflowConfig(gitea GiteaFileContentAndRepoFetcher, git_project string
func ResolveWorkflowConfigs(gitea GiteaFileContentAndRepoFetcher, config *ConfigFile) (AutogitConfigs, error) {
configs := make([]*AutogitConfig, 0, len(config.GitProjectNames))
var errs []error
for _, git_project := range config.GitProjectNames {
c, err := ReadWorkflowConfig(gitea, git_project)
if err != nil {
// can't sync, so ignore for now
errs = append(errs, err)
log.Println(err)
} else {
configs = append(configs, c)
}
}
if len(errs) > 0 {
return configs, errors.Join(errs...)
}
return configs, nil
}

View File

@@ -327,7 +327,6 @@ func main() {
interval := flag.Int64("interval", 10, "Notification polling interval in minutes (min 1 min)")
configFile := flag.String("config", "", "PrjGit listing config file")
logging := flag.String("logging", "info", "Logging level: [none, error, info, debug]")
exitOnConfigError := flag.Bool("exit-on-config-error", false, "Exit if any repository in configuration cannot be resolved")
flag.BoolVar(&common.IsDryRun, "dry", false, "Dry run, no effect. For debugging")
flag.Parse()
@@ -383,10 +382,8 @@ func main() {
giteaTransport := common.AllocateGiteaTransport(*giteaUrl)
configs, err := common.ResolveWorkflowConfigs(giteaTransport, configData)
if err != nil {
common.LogError("Failed to resolve some configuration repositories:", err)
if *exitOnConfigError {
return
}
common.LogError("Cannot parse workflow configs:", err)
return
}
reviewer, err := giteaTransport.GetCurrentUser()

View File

@@ -4,7 +4,7 @@ ENV container=podman
ENV LANG=en_US.UTF-8
RUN zypper -vvvn install podman podman-compose vim make python3-pytest python3-requests python3-pytest-dependency python3-pytest-httpserver
RUN zypper -vvvn install podman podman-compose vim make python3-pytest python3-requests python3-pytest-dependency
COPY . /opt/project/

View File

@@ -1,19 +1,51 @@
# We want to be able to test in two **modes**:
# A. bots are used from official packages as defined in */Dockerfile.package
# B. bots are just picked up from binaries that are placed in corresponding parent directory.
# The topology is defined in podman-compose file and can be spawned in two ways:
# 1. podman-compose on a local machine (needs dependencies as defined in the Dockerfile)
# 2. pytest in a dedicated container (recommended)
# 1. Privileged container (needs no additional dependancies)
# 2. podman-compose on a local machine (needs dependencies as defined in the Dockerfile)
# Typical workflow:
# 1. 'make build' - prepares images
# 2. 'make up' - spawns podman-compose
# 3. 'make pytest' - run tests inside the tester container
# 4. 'make down' - once the containers are not needed
#
# OR just run 'make test' to do it all at once.
# A1: - run 'make test_package'
# B1: - run 'make test_local' (make sure that the go binaries in parent folder are built)
# A2:
# 1. 'make build_package' - prepares images (recommended, otherwise there might be surprises if image fails to build during `make up`)
# 2. 'make up' - spawns podman-compose
# 3. 'pytest -v tests/*' - run tests
# 4. 'make down' - once the containers are not needed
# B2: (make sure the go binaries in the parent folder are built)
# 1. 'make build_local' - prepared images (recommended, otherwise there might be surprises if image fails to build during `make up`)
# 2. 'make up' - spawns podman-compose
# 3. 'pytest -v tests/*' - run tests
# 4. 'make down' - once the containers are not needed
AUTO_DETECT_MODE := $(shell if test -e ../workflow-pr/workflow-pr; then echo .local; else echo .package; fi)
# Default test target
test: test_b
# try to detect mode B1, otherwise mode A1
test: GIWTF_IMAGE_SUFFIX=$(AUTO_DETECT_MODE)
test: build_container test_container
# mode A1
test_package: GIWTF_IMAGE_SUFFIX=.package
test_package: build_container test_container
# mode B1
test_local: GIWTF_IMAGE_SUFFIX=.local
test_local: build_container test_container
MODULES := gitea-events-rabbitmq-publisher obs-staging-bot workflow-pr
# Prepare topology 1
build_container:
podman build ../ -f integration/Dockerfile -t autogits_integration
# Run tests in topology 1
test_container:
podman run --rm --privileged -t -e GIWTF_IMAGE_SUFFIX=$(GIWTF_IMAGE_SUFFIX) autogits_integration /usr/bin/bash -c "make build && make up && sleep 25 && pytest -v tests/*"
build_local: AUTO_DETECT_MODE=.local
build_local: build
@@ -21,66 +53,16 @@ build_local: build
build_package: AUTO_DETECT_MODE=.package
build_package: build
# parse all service images from podman-compose and build them
# mode B with pytest in container
test_b: AUTO_DETECT_MODE=.local
test_b: build up wait_healthy pytest
# Complete cycle for CI
test-ci: test_b down
wait_healthy:
@echo "Waiting for services to be healthy..."
@echo "Waiting for gitea (max 2m)..."
@start_time=$$(date +%s); \
until podman exec gitea-test curl -f -s http://localhost:3000/api/v1/version >/dev/null 2>&1; do \
current_time=$$(date +%s); \
elapsed=$$((current_time - start_time)); \
if [ $$elapsed -gt 120 ]; then \
echo "ERROR: Gitea failed to start within 2 minutes."; \
echo "--- Troubleshooting Info ---"; \
echo "Diagnostics output (curl):"; \
podman exec gitea-test curl -v http://localhost:3000/api/v1/version || true; \
echo "--- Container Logs ---"; \
podman logs gitea-test --tail 20; \
echo "--- Container Status ---"; \
podman inspect gitea-test --format '{{.State.Status}}'; \
exit 1; \
fi; \
sleep 2; \
done
@echo "Waiting for rabbitmq (max 2m)..."
@start_time=$$(date +%s); \
until podman exec rabbitmq-test rabbitmq-diagnostics check_running -q >/dev/null 2>&1; do \
current_time=$$(date +%s); \
elapsed=$$((current_time - start_time)); \
if [ $$elapsed -gt 120 ]; then \
echo "ERROR: RabbitMQ failed to start within 2 minutes."; \
echo "--- Troubleshooting Info ---"; \
echo "Diagnostics output:"; \
podman exec rabbitmq-test rabbitmq-diagnostics check_running || true; \
echo "--- Container Logs ---"; \
podman logs rabbitmq-test --tail 20; \
echo "--- Container Status ---"; \
podman inspect rabbitmq-test --format '{{.State.Status}}'; \
exit 1; \
fi; \
sleep 2; \
done
@echo "All services are healthy!"
pytest:
podman-compose exec tester pytest -v tests/*
# parse all service images from podman-compose and build them (topology 2)
build:
podman pull docker.io/library/rabbitmq:3.13.7-management
for i in $$(grep -A 1000 services: podman-compose.yml | grep -oE '^ [^: ]+'); do GIWTF_IMAGE_SUFFIX=$(AUTO_DETECT_MODE) podman-compose build $$i || exit 1; done
# this will spawn prebuilt containers
# this will spawn prebuilt containers (topology 2)
up:
podman-compose up -d
# tear down
# tear down (topology 2)
down:
podman-compose down
@@ -91,3 +73,4 @@ up-bots-package:
# mode B
up-bots-local:
GIWTF_IMAGE_SUFFIX=.local podman-compose up -d

View File

@@ -1,52 +0,0 @@
# Makefile Targets
This document describes the targets available in the `integration/Makefile`.
## Primary Workflow
### `test` (or `test_b`)
- **Action**: Performs a complete build-and-test cycle.
- **Steps**:
1. `build`: Prepares all container images.
2. `up`: Starts all services via `podman-compose`.
3. `wait_healthy`: Polls Gitea and RabbitMQ until they are ready.
4. `pytest`: Executes the test suite inside the `tester` container.
- **Outcome**: The environment remains active for fast iteration.
### `test-ci`
- **Action**: Performs the full `test` cycle followed by teardown.
- **Steps**: `test_b` -> `down`
- **Purpose**: Ideal for CI environments where a clean state is required after testing.
---
## Individual Targets
### `build`
- **Action**: Pulls external images (RabbitMQ) and builds all local service images defined in `podman-compose.yml`.
- **Note**: Use `build_local` or `build_package` to specify bot source mode.
### `up`
- **Action**: Starts the container topology in detached mode.
### `wait_healthy`
- **Action**: Polls the health status of `gitea-test` and `rabbitmq-test` containers.
- **Purpose**: Ensures infrastructure is stable before test execution.
### `pytest`
- **Action**: Runs `pytest -v tests/*` inside the running `tester` container.
- **Requirement**: The environment must already be started via `up`.
### `down`
- **Action**: Stops and removes all containers and networks defined in the compose file.
---
## Configuration Modes
The Makefile supports two deployment modes via `GIWTF_IMAGE_SUFFIX`:
- **.local** (Default): Uses binaries built from the local source (requires `make build` in project root).
- **.package**: Uses official pre-built packages for the bots.
Targets like `build_local`, `build_package`, `up-bots-local`, and `up-bots-package` allow for explicit mode selection.

57
integration/Makefile.txt Normal file
View File

@@ -0,0 +1,57 @@
+-------------------------------------------------------------------------------------------------+
| Makefile Targets |
+-------------------------------------------------------------------------------------------------+
| |
| [Default Test Workflow] |
| test (Auto-detects mode: .local or .package) |
| > build_container |
| > test_container |
| |
| [Specific Test Workflows - Topology 1: Privileged Container] |
| test_package (Mode A1: Bots from official packages) |
| > build_container |
| > test_container |
| |
| test_local (Mode B1: Bots from local binaries) |
| > build_container |
| > test_container |
| |
| build_container |
| - Action: Builds the `autogits_integration` privileged container image. |
| - Purpose: Prepares an environment for running tests within a single container. |
| |
| test_container |
| - Action: Runs `autogits_integration` container, executes `make build`, `make up`, and |
| `pytest -v tests/*` inside it. |
| - Purpose: Executes the full test suite in Topology 1 (privileged container). |
| |
| [Build & Orchestration Workflows - Topology 2: podman-compose] |
| |
| build_package (Mode A: Builds service images from official packages) |
| > build |
| |
| build_local (Mode B: Builds service images from local binaries) |
| > build |
| |
| build |
| - Action: Pulls `rabbitmq` image and iterates through `podman-compose.yml` services |
| to build each one. |
| - Purpose: Prepares all necessary service images for Topology 2 deployment. |
| |
| up |
| - Action: Starts all services defined in `podman-compose.yml` in detached mode. |
| - Purpose: Deploys the application topology (containers) for testing or development. |
| |
| down |
| - Action: Stops and removes all services started by `up`. |
| - Purpose: Cleans up the deployed application topology. |
| |
| up-bots-package (Mode A: Spawns Topology 2 with official package bots) |
| - Action: Calls `podman-compose up -d` with `GIWTF_IMAGE_SUFFIX=.package`. |
| - Purpose: Specifically brings up the environment using official package bots. |
| |
| up-bots-local (Mode B: Spawns Topology 2 with local binaries) |
| - Action: Calls `podman-compose up -d` with `GIWTF_IMAGE_SUFFIX=.local`. |
| - Purpose: Specifically brings up the environment using local binaries. |
| |
+-------------------------------------------------------------------------------------------------+

View File

@@ -0,0 +1,14 @@
# Use a base Python image
FROM registry.suse.com/bci/python:3.11
# Set the working directory
WORKDIR /app
# Copy the server script
COPY server.py .
# Expose the port the server will run on
EXPOSE 8080
# Command to run the server
CMD ["python3", "-u", "server.py"]

View File

@@ -0,0 +1,18 @@
<project name="openSUSE:Leap:16.0:PullRequest">
<title>Leap 16.0 PullRequest area</title>
<description>Base project to define the pull request builds</description>
<person userid="autogits_obs_staging_bot" role="maintainer"/>
<person userid="maxlin_factory" role="maintainer"/>
<group groupid="maintenance-opensuse.org" role="maintainer"/>
<debuginfo>
<enable/>
</debuginfo>
<repository name="standard">
<path project="openSUSE:Leap:16.0" repository="standard"/>
<arch>x86_64</arch>
<arch>i586</arch>
<arch>aarch64</arch>
<arch>ppc64le</arch>
<arch>s390x</arch>
</repository>
</project>

View File

@@ -0,0 +1,59 @@
<project name="openSUSE:Leap:16.0">
<title>openSUSE Leap 16.0 based on SLFO</title>
<description>Leap 16.0 based on SLES 16.0 (specifically SLFO:1.2)</description>
<link project="openSUSE:Backports:SLE-16.0"/>
<scmsync>http://gitea-test:3000/myproducts/mySLFO#staging-main</scmsync>
<person userid="dimstar_suse" role="maintainer"/>
<person userid="lkocman-factory" role="maintainer"/>
<person userid="maxlin_factory" role="maintainer"/>
<person userid="factory-auto" role="reviewer"/>
<person userid="licensedigger" role="reviewer"/>
<group groupid="autobuild-team" role="maintainer"/>
<group groupid="factory-maintainers" role="maintainer"/>
<group groupid="maintenance-opensuse.org" role="maintainer"/>
<group groupid="factory-staging" role="reviewer"/>
<build>
<disable repository="ports"/>
</build>
<debuginfo>
<enable/>
</debuginfo>
<repository name="standard" rebuild="local">
<path project="openSUSE:Backports:SLE-16.0" repository="standard"/>
<path project="SUSE:SLFO:1.2" repository="standard"/>
<arch>local</arch>
<arch>i586</arch>
<arch>x86_64</arch>
<arch>aarch64</arch>
<arch>ppc64le</arch>
<arch>s390x</arch>
</repository>
<repository name="product">
<releasetarget project="openSUSE:Leap:16.0:ToTest" repository="product" trigger="manual"/>
<path project="openSUSE:Leap:16.0:NonFree" repository="standard"/>
<path project="openSUSE:Leap:16.0" repository="images"/>
<path project="openSUSE:Leap:16.0" repository="standard"/>
<path project="openSUSE:Backports:SLE-16.0" repository="standard"/>
<path project="SUSE:SLFO:1.2" repository="standard"/>
<arch>local</arch>
<arch>i586</arch>
<arch>x86_64</arch>
<arch>aarch64</arch>
<arch>ppc64le</arch>
<arch>s390x</arch>
</repository>
<repository name="ports">
<arch>armv7l</arch>
</repository>
<repository name="images">
<releasetarget project="openSUSE:Leap:16.0:ToTest" repository="images" trigger="manual"/>
<path project="openSUSE:Leap:16.0" repository="standard"/>
<path project="openSUSE:Backports:SLE-16.0" repository="standard"/>
<path project="SUSE:SLFO:1.2" repository="standard"/>
<arch>i586</arch>
<arch>x86_64</arch>
<arch>aarch64</arch>
<arch>ppc64le</arch>
<arch>s390x</arch>
</repository>
</project>

View File

@@ -0,0 +1,140 @@
import http.server
import socketserver
import os
import logging
import signal
import sys
import threading
import fnmatch
PORT = 8080
RESPONSE_DIR = "/app/responses"
STATE_DIR = "/tmp/mock_obs_state"
class MockOBSHandler(http.server.SimpleHTTPRequestHandler):
def do_GET(self):
logging.info(f"GET request for: {self.path}")
path_without_query = self.path.split('?')[0]
# Check for state stored by a PUT request first
sanitized_put_path = 'PUT' + path_without_query.replace('/', '_')
state_file_path = os.path.join(STATE_DIR, sanitized_put_path)
if os.path.exists(state_file_path):
logging.info(f"Found stored PUT state for {self.path} at {state_file_path}")
self.send_response(200)
self.send_header("Content-type", "application/xml")
file_size = os.path.getsize(state_file_path)
self.send_header("Content-Length", str(file_size))
self.end_headers()
with open(state_file_path, 'rb') as f:
self.wfile.write(f.read())
return
# If no PUT state file, fall back to the glob/exact match logic
self.handle_request('GET')
def do_PUT(self):
logging.info(f"PUT request for: {self.path}")
logging.info(f"Headers: {self.headers}")
path_without_query = self.path.split('?')[0]
body = b''
if self.headers.get('Transfer-Encoding', '').lower() == 'chunked':
logging.info("Chunked transfer encoding detected")
while True:
line = self.rfile.readline().strip()
if not line:
break
chunk_length = int(line, 16)
if chunk_length == 0:
self.rfile.readline()
break
body += self.rfile.read(chunk_length)
self.rfile.read(2) # Read the trailing CRLF
else:
content_length = int(self.headers.get('Content-Length', 0))
body = self.rfile.read(content_length)
logging.info(f"Body: {body.decode('utf-8')}")
sanitized_path = 'PUT' + path_without_query.replace('/', '_')
state_file_path = os.path.join(STATE_DIR, sanitized_path)
logging.info(f"Saving state for {self.path} to {state_file_path}")
os.makedirs(os.path.dirname(state_file_path), exist_ok=True)
with open(state_file_path, 'wb') as f:
f.write(body)
self.send_response(200)
self.send_header("Content-type", "text/plain")
response_body = b"OK"
self.send_header("Content-Length", str(len(response_body)))
self.end_headers()
self.wfile.write(response_body)
def do_POST(self):
logging.info(f"POST request for: {self.path}")
self.handle_request('POST')
def do_DELETE(self):
logging.info(f"DELETE request for: {self.path}")
self.handle_request('DELETE')
def handle_request(self, method):
path_without_query = self.path.split('?')[0]
sanitized_request_path = method + path_without_query.replace('/', '_')
logging.info(f"Handling request, looking for match for: {sanitized_request_path}")
response_file = None
# Check for glob match first
if os.path.exists(RESPONSE_DIR):
for filename in os.listdir(RESPONSE_DIR):
if fnmatch.fnmatch(sanitized_request_path, filename):
response_file = os.path.join(RESPONSE_DIR, filename)
logging.info(f"Found matching response file (glob): {response_file}")
break
# Fallback to exact match if no glob match
if response_file is None:
exact_file = os.path.join(RESPONSE_DIR, sanitized_request_path)
if os.path.exists(exact_file):
response_file = exact_file
logging.info(f"Found matching response file (exact): {response_file}")
if response_file:
logging.info(f"Serving content from {response_file}")
self.send_response(200)
self.send_header("Content-type", "application/xml")
file_size = os.path.getsize(response_file)
self.send_header("Content-Length", str(file_size))
self.end_headers()
with open(response_file, 'rb') as f:
self.wfile.write(f.read())
else:
logging.info(f"Response file not found for {sanitized_request_path}. Sending 404.")
self.send_response(404)
self.send_header("Content-type", "text/plain")
body = f"Mock response not found for {sanitized_request_path}".encode('utf-8')
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(message)s')
if not os.path.exists(STATE_DIR):
logging.info(f"Creating state directory: {STATE_DIR}")
os.makedirs(STATE_DIR)
if not os.path.exists(RESPONSE_DIR):
os.makedirs(RESPONSE_DIR)
with socketserver.TCPServer(("", PORT), MockOBSHandler) as httpd:
logging.info(f"Serving mock OBS API on port {PORT}")
def graceful_shutdown(sig, frame):
logging.info("Received SIGTERM, shutting down gracefully...")
threading.Thread(target=httpd.shutdown).start()
signal.signal(signal.SIGTERM, graceful_shutdown)
httpd.serve_forever()
logging.info("Server has shut down.")

View File

@@ -1,64 +0,0 @@
# Podman-Compose Services Architecture
This document describes the services defined in `podman-compose.yml` used for integration testing.
## Network
- **gitea-network**: A bridge network that enables communication between all services.
## Services
### gitea
- **Description**: Self-hosted Git service, serving as the central hub for repositories.
- **Container Name**: `gitea-test`
- **Image**: Built from `./gitea/Dockerfile`
- **Ports**: `3000` (HTTP), `3022` (SSH)
- **Volumes**: `./gitea-data` (persistent data), `./gitea-logs` (logs)
- **Healthcheck**: Monitors the Gitea API version endpoint.
### rabbitmq
- **Description**: Message broker for asynchronous communication between services.
- **Container Name**: `rabbitmq-test`
- **Image**: `rabbitmq:3.13.7-management`
- **Ports**: `5671` (AMQP with TLS), `15672` (Management UI)
- **Volumes**: `./rabbitmq-data`, `./rabbitmq-config/certs`, `./rabbitmq-config/rabbitmq.conf`, `./rabbitmq-config/definitions.json`
- **Healthcheck**: Ensures the broker is running and ready to accept connections.
### gitea-publisher
- **Description**: Publishes events from Gitea webhooks to the RabbitMQ message queue.
- **Container Name**: `gitea-publisher`
- **Dependencies**: `gitea` (started), `rabbitmq` (healthy)
- **Topic Domain**: `suse`
### workflow-pr
- **Description**: Manages pull request workflows, synchronizing between ProjectGit and PackageGit.
- **Container Name**: `workflow-pr`
- **Dependencies**: `gitea` (started), `rabbitmq` (healthy)
- **Environment**: Configured via `AUTOGITS_*` variables.
- **Volumes**: `./gitea-data` (read-only), `./workflow-pr/workflow-pr.json` (config), `./workflow-pr-repos` (working directories)
### tester
- **Description**: The dedicated test runner container. It hosts the `pytest` suite and provides a mock OBS API using `pytest-httpserver`.
- **Container Name**: `tester`
- **Image**: Built from `./Dockerfile.tester`
- **Mock API**: Listens on port `8080` within the container network to simulate OBS.
- **Volumes**: Project root mounted at `/opt/project` for source access.
### obs-staging-bot
- **Description**: Interacts with Gitea and the OBS API (mocked by `tester`) to manage staging projects.
- **Container Name**: `obs-staging-bot`
- **Dependencies**: `gitea` (started), `tester` (started)
- **Environment**:
- `AUTOGITS_STAGING_BOT_POLL_INTERVAL`: Set to `2s` for fast integration testing.
- **Mock Integration**: Points to `http://tester:8080` for both OBS API and Web hosts.
---
## Testing Workflow
1. **Build**: `make build` (root) then `make build` (integration).
2. **Up**: `make up` starts all services.
3. **Wait**: `make wait_healthy` ensures infrastructure is ready.
4. **Test**: `make pytest` runs the suite inside the `tester` container.
5. **Down**: `make down` stops and removes containers.
Use `make test` to perform steps 1-4 automatically.

View File

@@ -0,0 +1,77 @@
+-------------------------------------------------------------------------------------------------+
| Podman-Compose Services Diagram |
+-------------------------------------------------------------------------------------------------+
| |
| [Network] |
| gitea-network (Bridge network for inter-service communication) |
| |
|-------------------------------------------------------------------------------------------------|
| |
| [Service: gitea] |
| Description: Self-hosted Git service, central hub for repositories and code management. |
| Container Name: gitea-test |
| Image: Built from ./gitea Dockerfile |
| Ports: 3000 (HTTP), 3022 (SSH) |
| Volumes: ./gitea-data (for persistent data), ./gitea-logs (for logs) |
| Network: gitea-network |
| |
|-------------------------------------------------------------------------------------------------|
| |
| [Service: rabbitmq] |
| Description: Message broker for asynchronous communication between services. |
| Container Name: rabbitmq-test |
| Image: rabbitmq:3.13.7-management |
| Ports: 5671 (AMQP), 15672 (Management UI) |
| Volumes: ./rabbitmq-data (for persistent data), ./rabbitmq-config/certs (TLS certs), |
| ./rabbitmq-config/rabbitmq.conf (config), ./rabbitmq-config/definitions.json (exchanges)|
| Healthcheck: Ensures RabbitMQ is running and healthy. |
| Network: gitea-network |
| |
|-------------------------------------------------------------------------------------------------|
| |
| [Service: gitea-publisher] |
| Description: Publishes events from Gitea to the RabbitMQ message queue. |
| Container Name: gitea-publisher |
| Image: Built from ../gitea-events-rabbitmq-publisher/Dockerfile (local/package) |
| Dependencies: gitea (started), rabbitmq (healthy) |
| Environment: RABBITMQ_HOST, RABBITMQ_USERNAME, RABBITMQ_PASSWORD, SSL_CERT_FILE |
| Command: Listens for Gitea events, publishes to 'suse' topic, debug enabled. |
| Network: gitea-network |
| |
|-------------------------------------------------------------------------------------------------|
| |
| [Service: workflow-pr] |
| Description: Manages pull request workflows, likely consuming events from RabbitMQ and |
| interacting with Gitea. |
| Container Name: workflow-pr |
| Image: Built from ../workflow-pr/Dockerfile (local/package) |
| Dependencies: gitea (started), rabbitmq (healthy) |
| Environment: AMQP_USERNAME, AMQP_PASSWORD, SSL_CERT_FILE |
| Volumes: ./gitea-data (read-only), ./workflow-pr/workflow-pr.json (config), |
| ./workflow-pr-repos (for repositories) |
| Command: Configures Gitea/RabbitMQ URLs, enables debug, manages repositories. |
| Network: gitea-network |
| |
|-------------------------------------------------------------------------------------------------|
| |
| [Service: mock-obs] |
| Description: A mock (simulated) service for the Open Build Service (OBS) for testing. |
| Container Name: mock-obs |
| Image: Built from ./mock-obs Dockerfile |
| Ports: 8080 |
| Volumes: ./mock-obs/responses (for mock API responses) |
| Network: gitea-network |
| |
|-------------------------------------------------------------------------------------------------|
| |
| [Service: obs-staging-bot] |
| Description: A bot that interacts with Gitea and the mock OBS, likely for staging processes. |
| Container Name: obs-staging-bot |
| Image: Built from ../obs-staging-bot/Dockerfile (local/package) |
| Dependencies: gitea (started), mock-obs (started) |
| Environment: OBS_USER, OBS_PASSWORD |
| Volumes: ./gitea-data (read-only) |
| Command: Configures Gitea/OBS URLs, enables debug. |
| Network: gitea-network |
| |
+-------------------------------------------------------------------------------------------------+

View File

@@ -95,8 +95,7 @@ services:
- ./workflow-pr/workflow-pr.json:/etc/workflow-pr.json:ro,z
- ./workflow-pr-repos:/var/lib/workflow-pr/repos:Z
command: [
"-check-on-start",
"-exit-on-config-error",
"-check-on-start",
"-debug",
"-gitea-url", "http://gitea-test:3000",
"-url", "amqps://rabbitmq-test:5671",
@@ -105,21 +104,17 @@ services:
]
restart: unless-stopped
tester:
build:
context: .
dockerfile: Dockerfile.tester
container_name: tester
mock-obs:
build: ./mock-obs
container_name: mock-obs
init: true
dns_search: .
networks:
- gitea-network
environment:
- PYTEST_HTTPSERVER_HOST=0.0.0.0
- PYTEST_HTTPSERVER_PORT=8080
ports:
- "8080:8080"
volumes:
- ..:/opt/project:z
command: sleep infinity
- ./mock-obs/responses:/app/responses:z # Use :z for shared SELinux label
restart: unless-stopped
obs-staging-bot:
build:
@@ -132,17 +127,16 @@ services:
depends_on:
gitea:
condition: service_started
tester:
mock-obs:
condition: service_started
environment:
- OBS_USER=mock
- OBS_PASSWORD=mock-long-password
- AUTOGITS_STAGING_BOT_POLL_INTERVAL=2s
volumes:
- ./gitea-data:/gitea-data:ro,z
command:
- "-debug"
- "-gitea-url=http://gitea-test:3000"
- "-obs=http://tester:8080"
- "-obs-web=http://tester:8080"
- "-obs=http://mock-obs:8080"
- "-obs-web=http://mock-obs:8080"
restart: unless-stopped

View File

@@ -8,83 +8,7 @@ import time
import os
import json
import base64
import re
from tests.lib.common_test_utils import GiteaAPIClient, vprint
import tests.lib.common_test_utils as common_utils
@pytest.fixture(autouse=True)
def is_test_run():
common_utils.IS_TEST_RUN = True
yield
common_utils.IS_TEST_RUN = False
if os.environ.get("AUTOGITS_PRINT_FIXTURES") is None:
print("--- Fixture messages are suppressed. Set AUTOGITS_PRINT_FIXTURES=1 to enable them. ---")
class ObsMockState:
def __init__(self):
self.build_results = {} # project -> (package, code)
self.project_metas = {} # project -> scmsync
self.default_build_result = None
@pytest.fixture
def obs_mock_state():
return ObsMockState()
@pytest.fixture(autouse=True)
def default_obs_handlers(httpserver, obs_mock_state):
"""
Sets up default handlers for OBS API to avoid 404s.
"""
def project_meta_handler(request):
project = request.path.split("/")[2]
scmsync = obs_mock_state.project_metas.get(project, "http://gitea-test:3000/myproducts/mySLFO.git")
return f'<project name="{project}"><scmsync>{scmsync}</scmsync></project>'
def build_result_handler(request):
project = request.path.split("/")[2]
res = obs_mock_state.build_results.get(project) or obs_mock_state.default_build_result
if not res:
return '<resultlist></resultlist>'
package_name, code = res
# We'll use a simple hardcoded XML here to avoid re-parsing template every time
# or we can use the template. For simplicity, let's use a basic one.
xml_template = f"""<resultlist state="mock">
<result project="{project}" repository="standard" arch="x86_64" code="unpublished" state="unpublished">
<scmsync>http://gitea-test:3000/myproducts/mySLFO.git?onlybuild={package_name}#sha</scmsync>
<status package="{package_name}" code="{code}"/>
</result>
</resultlist>"""
return xml_template
# Register handlers
httpserver.expect_request(re.compile(r"/source/[^/]+/_meta$"), method="GET").respond_with_handler(project_meta_handler)
httpserver.expect_request(re.compile(r"/build/[^/]+/_result"), method="GET").respond_with_handler(build_result_handler)
httpserver.expect_request(re.compile(r"/source/[^/]+/_meta$"), method="PUT").respond_with_data("OK")
httpserver.expect_request(re.compile(r"/source/[^/]+$"), method="DELETE").respond_with_data("OK")
@pytest.fixture
def mock_build_result(obs_mock_state):
"""
Fixture to set up mock build results.
"""
def _setup_mock(package_name: str, code: str, project: str = None):
if project:
obs_mock_state.build_results[project] = (package_name, code)
else:
# If no project specified, we can't easily know which one to set
# but usually it's the one the bot will request.
# We'll use a special key to signify "all" or we can just wait for the request.
# For now, let's assume we want to match openSUSE:Leap:16.0:PullRequest:*
# The test will call it with specific project if needed.
# In test_pr_workflow, it doesn't know the PR number yet.
# So we'll make the handler fallback to this if project not found.
obs_mock_state.default_build_result = (package_name, code)
return _setup_mock
from tests.lib.common_test_utils import GiteaAPIClient
BRANCH_CONFIG_COMMON = {
"workflow.config": {
@@ -172,7 +96,7 @@ _CREATED_USERS = set()
_CREATED_LABELS = set()
_ADDED_COLLABORATORS = set() # format: (org_repo, username)
def setup_users_from_config(client: GiteaAPIClient, wf: dict, mt: dict, stats: dict = None, handled: dict = None):
def setup_users_from_config(client: GiteaAPIClient, wf: dict, mt: dict):
"""
Parses workflow.config and _maintainership.json, creates users, and adds them as collaborators.
"""
@@ -192,19 +116,13 @@ def setup_users_from_config(client: GiteaAPIClient, wf: dict, mt: dict, stats: d
# Create all users
for username in all_users:
new_user = client.create_user(username, "password123", f"{username}@example.com")
_CREATED_USERS.add(username)
if stats and handled and username not in handled["users"]:
handled["users"].add(username)
if new_user: stats["users"]["new"] += 1
else: stats["users"]["reused"] += 1
if username not in _CREATED_USERS:
client.create_user(username, "password123", f"{username}@example.com")
_CREATED_USERS.add(username)
new_coll = client.add_collaborator("myproducts", "mySLFO", username, "write")
_ADDED_COLLABORATORS.add(("myproducts/mySLFO", username))
if stats and handled and ("myproducts/mySLFO", username) not in handled["collaborators"]:
handled["collaborators"].add(("myproducts/mySLFO", username))
if new_coll: stats["collaborators"]["new"] += 1
else: stats["collaborators"]["reused"] += 1
if ("myproducts/mySLFO", username) not in _ADDED_COLLABORATORS:
client.add_collaborator("myproducts", "mySLFO", username, "write")
_ADDED_COLLABORATORS.add(("myproducts/mySLFO", username))
# Set specific repository permissions based on maintainership
for pkg, users in mt.items():
@@ -212,34 +130,20 @@ def setup_users_from_config(client: GiteaAPIClient, wf: dict, mt: dict, stats: d
for username in users:
if not repo_name:
for r in ["pkgA", "pkgB"]:
new_coll = client.add_collaborator("mypool", r, username, "write")
_ADDED_COLLABORATORS.add((f"mypool/{r}", username))
if stats and handled and (f"mypool/{r}", username) not in handled["collaborators"]:
handled["collaborators"].add((f"mypool/{r}", username))
if new_coll: stats["collaborators"]["new"] += 1
else: stats["collaborators"]["reused"] += 1
if (f"mypool/{r}", username) not in _ADDED_COLLABORATORS:
client.add_collaborator("mypool", r, username, "write")
_ADDED_COLLABORATORS.add((f"mypool/{r}", username))
else:
new_coll = client.add_collaborator("mypool", repo_name, username, "write")
_ADDED_COLLABORATORS.add((f"mypool/{repo_name}", username))
if stats and handled and (f"mypool/{repo_name}", username) not in handled["collaborators"]:
handled["collaborators"].add((f"mypool/{repo_name}", username))
if new_coll: stats["collaborators"]["new"] += 1
else: stats["collaborators"]["reused"] += 1
if (f"mypool/{repo_name}", username) not in _ADDED_COLLABORATORS:
client.add_collaborator("mypool", repo_name, username, "write")
_ADDED_COLLABORATORS.add((f"mypool/{repo_name}", username))
def ensure_config_file(client: GiteaAPIClient, owner: str, repo: str, branch: str, file_name: str, expected_content_dict: dict, existing_files: list = None):
def ensure_config_file(client: GiteaAPIClient, owner: str, repo: str, branch: str, file_name: str, expected_content_dict: dict):
"""
Checks if a config file exists and has the correct content.
Returns True if a change was made, False otherwise.
"""
file_info = None
if existing_files is not None:
if file_name not in [f["path"] for f in existing_files]:
pass # File definitely doesn't exist
else:
file_info = client.get_file_info(owner, repo, file_name, branch=branch)
else:
file_info = client.get_file_info(owner, repo, file_name, branch=branch)
file_info = client.get_file_info(owner, repo, file_name, branch=branch)
expected_content = json.dumps(expected_content_dict, indent=4)
if file_info:
@@ -259,28 +163,8 @@ def gitea_env():
"""
Global fixture to set up the Gitea environment for all tests.
"""
setup_start_time = time.time()
stats = {
"orgs": {"new": 0, "reused": 0},
"repos": {"new": 0, "reused": 0},
"users": {"new": 0, "reused": 0},
"labels": {"new": 0, "reused": 0},
"collaborators": {"new": 0, "reused": 0},
"branches": {"new": 0, "reused": 0},
"webhooks": {"new": 0, "reused": 0},
}
handled_in_session = {
"orgs": set(),
"repos": set(),
"users": set(),
"labels": set(),
"collaborators": set(),
"branches": set(),
"webhooks": set(),
}
gitea_url = "http://gitea-test:3000"
admin_token_path = os.path.join(os.path.dirname(__file__), "..", "gitea-data", "admin.token")
gitea_url = "http://127.0.0.1:3000"
admin_token_path = "./gitea-data/admin.token"
admin_token = None
try:
@@ -290,55 +174,35 @@ def gitea_env():
raise Exception(f"Admin token file not found at {admin_token_path}.")
client = GiteaAPIClient(base_url=gitea_url, token=admin_token)
client.use_cache = True
# Wait for Gitea
for i in range(10):
try:
resp, dur = client._request("GET", "version")
if resp.status_code == 200:
vprint(f"DEBUG: Gitea connection successful (duration: {dur:.3f}s)")
if client._request("GET", "version").status_code == 200:
break
except Exception as e:
vprint(f"DEBUG: Gitea connection attempt {i+1} failed: {e}")
except:
pass
time.sleep(1)
else: raise Exception("Gitea not available.")
else:
raise Exception("Gitea not available.")
vprint("--- Starting Gitea Global Setup ---")
print("--- Starting Gitea Global Setup ---")
for org in ["myproducts", "mypool"]:
new_org = client.create_org(org)
_CREATED_ORGS.add(org)
if org not in handled_in_session["orgs"]:
handled_in_session["orgs"].add(org)
if new_org: stats["orgs"]["new"] += 1
else: stats["orgs"]["reused"] += 1
if org not in _CREATED_ORGS:
client.create_org(org)
_CREATED_ORGS.add(org)
for org, repo in [("myproducts", "mySLFO"), ("mypool", "pkgA"), ("mypool", "pkgB")]:
new_repo = client.create_repo(org, repo)
client.update_repo_settings(org, repo)
repo_full = f"{org}/{repo}"
_CREATED_REPOS.add(repo_full)
if repo_full not in handled_in_session["repos"]:
handled_in_session["repos"].add(repo_full)
if new_repo: stats["repos"]["new"] += 1
else: stats["repos"]["reused"] += 1
# Create webhook for publisher
new_hook = client.create_webhook(org, repo, "http://gitea-publisher:8002/rabbitmq-forwarder")
if repo_full not in handled_in_session["webhooks"]:
handled_in_session["webhooks"].add(repo_full)
if new_hook: stats["webhooks"]["new"] += 1
else: stats["webhooks"]["reused"] += 1
if f"{org}/{repo}" not in _CREATED_REPOS:
client.create_repo(org, repo)
client.update_repo_settings(org, repo)
_CREATED_REPOS.add(f"{org}/{repo}")
# Create labels
for name, color in [("staging/Backlog", "#0000ff"), ("review/Pending", "#ffff00")]:
new_label = client.create_label("myproducts", "mySLFO", name, color=color)
_CREATED_LABELS.add(("myproducts/mySLFO", name))
if ("myproducts/mySLFO", name) not in handled_in_session["labels"]:
handled_in_session["labels"].add(("myproducts/mySLFO", name))
if new_label: stats["labels"]["new"] += 1
else: stats["labels"]["reused"] += 1
if ("myproducts/mySLFO", name) not in _CREATED_LABELS:
client.create_label("myproducts", "mySLFO", name, color=color)
_CREATED_LABELS.add(("myproducts/mySLFO", name))
# Submodules in mySLFO
client.add_submodules("myproducts", "mySLFO")
@@ -347,51 +211,24 @@ def gitea_env():
("myproducts/mySLFO", "workflow-pr"),
("mypool/pkgA", "workflow-pr"),
("mypool/pkgB", "workflow-pr")]:
org_part, repo_part = repo_full.split("/")
new_coll = client.add_collaborator(org_part, repo_part, bot, "write")
_ADDED_COLLABORATORS.add((repo_full, bot))
if (repo_full, bot) not in handled_in_session["collaborators"]:
handled_in_session["collaborators"].add((repo_full, bot))
if new_coll: stats["collaborators"]["new"] += 1
else: stats["collaborators"]["reused"] += 1
# Collect all users from all configurations first to do setup once
all_setup_users_wf = {}
all_setup_users_mt = {}
if (repo_full, bot) not in _ADDED_COLLABORATORS:
org_part, repo_part = repo_full.split("/")
client.add_collaborator(org_part, repo_part, bot, "write")
_ADDED_COLLABORATORS.add((repo_full, bot))
restart_needed = False
# Setup all branches and configs
repo_list = [("mypool", "pkgA"), ("mypool", "pkgB"), ("myproducts", "mySLFO")]
repo_branches = {}
for owner, repo in repo_list:
resp, _ = client._request("GET", f"repos/{owner}/{repo}/branches")
repo_branches[(owner, repo)] = {b["name"] for b in resp.json()}
for branch_name, custom_configs in BRANCH_CONFIG_CUSTOM.items():
# Ensure branch exists in all 3 repos
for owner, repo in repo_list:
for owner, repo in [("myproducts", "mySLFO"), ("mypool", "pkgA"), ("mypool", "pkgB")]:
if branch_name != "main":
if branch_name not in repo_branches[(owner, repo)]:
try:
resp, _ = client._request("GET", f"repos/{owner}/{repo}/branches/main")
main_sha = resp.json()["commit"]["id"]
new_branch = client.create_branch(owner, repo, branch_name, main_sha)
repo_branches[(owner, repo)].add(branch_name)
if (f"{owner}/{repo}", branch_name) not in handled_in_session["branches"]:
handled_in_session["branches"].add((f"{owner}/{repo}", branch_name))
if new_branch: stats["branches"]["new"] += 1
else: stats["branches"]["reused"] += 1
except Exception as e:
if "already exists" not in str(e).lower():
raise
else:
if (f"{owner}/{repo}", branch_name) not in handled_in_session["branches"]:
handled_in_session["branches"].add((f"{owner}/{repo}", branch_name))
stats["branches"]["reused"] += 1
else:
# main branch always exists, but let's track it as reused if not handled
if (f"{owner}/{repo}", "main") not in handled_in_session["branches"]:
handled_in_session["branches"].add((f"{owner}/{repo}", "main"))
stats["branches"]["reused"] += 1
try:
main_sha = client._request("GET", f"repos/{owner}/{repo}/branches/main").json()["commit"]["id"]
client.create_branch(owner, repo, branch_name, main_sha)
except Exception as e:
if "already exists" not in str(e).lower():
raise
# Merge configs
merged_configs = {}
@@ -410,40 +247,19 @@ def gitea_env():
else:
merged_configs[file_name] = custom_content
# Pre-fetch existing files in this branch to avoid 404s in ensure_config_file
try:
resp, _ = client._request("GET", f"repos/myproducts/mySLFO/contents?ref={branch_name}")
existing_files = resp.json()
except:
existing_files = []
# Ensure config files in myproducts/mySLFO
for file_name, content_dict in merged_configs.items():
ensure_config_file(client, "myproducts", "mySLFO", branch_name, file_name, content_dict, existing_files=existing_files)
if ensure_config_file(client, "myproducts", "mySLFO", branch_name, file_name, content_dict):
restart_needed = True
# Collect configs for user setup
wf_cfg = merged_configs.get("workflow.config", {})
mt_cfg = merged_configs.get("_maintainership.json", {})
# Simple merge for user collection
if "Reviewers" in wf_cfg:
all_setup_users_wf.setdefault("Reviewers", []).extend(wf_cfg["Reviewers"])
for k, v in mt_cfg.items():
all_setup_users_mt.setdefault(k, []).extend(v)
# Setup users (using configs from this branch)
setup_users_from_config(client, merged_configs.get("workflow.config", {}), merged_configs.get("_maintainership.json", {}))
# Dedup and setup users once
if "Reviewers" in all_setup_users_wf:
all_setup_users_wf["Reviewers"] = list(set(all_setup_users_wf["Reviewers"]))
for k in all_setup_users_mt:
all_setup_users_mt[k] = list(set(all_setup_users_mt[k]))
setup_users_from_config(client, all_setup_users_wf, all_setup_users_mt, stats=stats, handled=handled_in_session)
if restart_needed:
client.restart_service("workflow-pr")
time.sleep(2) # Give it time to pick up changes
setup_duration = time.time() - setup_start_time
print(f"--- Gitea Global Setup Complete (took {setup_duration:.2f}s) ---\n"
f"Objects created: {stats['orgs']['new']} orgs, {stats['repos']['new']} repos, {stats['branches']['new']} branches, {stats['webhooks']['new']} webhooks, {stats['users']['new']} users, {stats['labels']['new']} labels, {stats['collaborators']['new']} collaborators\n"
f"Objects reused: {stats['orgs']['reused']} orgs, {stats['repos']['reused']} repos, {stats['branches']['reused']} branches, {stats['webhooks']['reused']} webhooks, {stats['users']['reused']} users, {stats['labels']['reused']} labels, {stats['collaborators']['reused']} collaborators")
client.use_cache = False
print("--- Gitea Global Setup Complete ---")
yield client
@pytest.fixture(scope="session")

View File

@@ -7,12 +7,42 @@ import re
import xml.etree.ElementTree as ET
from pathlib import Path
import base64
import subprocess
IS_TEST_RUN = False
TEST_DATA_DIR = Path(__file__).parent.parent / "data"
BUILD_RESULT_TEMPLATE = TEST_DATA_DIR / "build_result.xml.template"
MOCK_RESPONSES_DIR = Path(__file__).parent.parent.parent / "mock-obs" / "responses"
MOCK_BUILD_RESULT_FILE = (
MOCK_RESPONSES_DIR / "GET_build_openSUSE:Leap:16.0:PullRequest:*__result"
)
MOCK_BUILD_RESULT_FILE1 = MOCK_RESPONSES_DIR / "GET_build_openSUSE:Leap:16.0__result"
@pytest.fixture
def mock_build_result():
"""
Fixture to create a mock build result file from the template.
Returns a factory function that the test can call with parameters.
"""
def _create_result_file(package_name: str, code: str):
tree = ET.parse(BUILD_RESULT_TEMPLATE)
root = tree.getroot()
for status_tag in root.findall(".//status"):
status_tag.set("package", package_name)
status_tag.set("code", code)
MOCK_RESPONSES_DIR.mkdir(exist_ok=True)
tree.write(MOCK_BUILD_RESULT_FILE)
tree.write(MOCK_BUILD_RESULT_FILE1)
return str(MOCK_BUILD_RESULT_FILE)
yield _create_result_file
if MOCK_BUILD_RESULT_FILE.exists():
MOCK_BUILD_RESULT_FILE.unlink()
MOCK_BUILD_RESULT_FILE1.unlink()
def vprint(*args, **kwargs):
if IS_TEST_RUN or os.environ.get("AUTOGITS_PRINT_FIXTURES") == "1":
print(*args, **kwargs)
class GiteaAPIClient:
def __init__(self, base_url, token, sudo=None):
@@ -20,45 +50,24 @@ class GiteaAPIClient:
self.headers = {"Authorization": f"token {token}", "Content-Type": "application/json"}
if sudo:
self.headers["Sudo"] = sudo
self._cache = {}
self.use_cache = False
def _request(self, method, path, **kwargs):
# Very basic cache for GET requests to speed up setup
cache_key = (method, path, json.dumps(kwargs, sort_keys=True))
if self.use_cache and method == "GET" and cache_key in self._cache:
return self._cache[cache_key], 0.0
url = f"{self.base_url}/api/v1/{path}"
start_time = time.time()
response = requests.request(method, url, headers=self.headers, **kwargs)
try:
response = requests.request(method, url, headers=self.headers, **kwargs)
duration = time.time() - start_time
response.raise_for_status()
if self.use_cache:
if method == "GET":
self._cache[cache_key] = response
else:
self._cache.clear()
return response, duration
except requests.exceptions.HTTPError as e:
duration = time.time() - start_time
vprint(f"[{duration:.3f}s] HTTPError in _request: {e}")
vprint(f"Response Content: {e.response.text}")
raise
except requests.exceptions.RequestException as e:
duration = time.time() - start_time
vprint(f"[{duration:.3f}s] Request failed: {e}")
print(f"HTTPError in _request: {e}")
print(f"Response Content: {e.response.text}")
raise
return response
def get_file_info(self, owner: str, repo: str, file_path: str, branch: str = "main"):
url = f"repos/{owner}/{repo}/contents/{file_path}"
if branch and branch != "main":
url += f"?ref={branch}"
try:
response, duration = self._request("GET", url)
response = self._request("GET", url)
return response.json()
except requests.exceptions.HTTPError as e:
if e.response.status_code == 404:
@@ -66,7 +75,7 @@ class GiteaAPIClient:
raise
def create_user(self, username, password, email):
vprint(f"--- Creating user: {username} ---")
print(f"--- Creating user: {username} ---")
data = {
"username": username,
"password": password,
@@ -75,20 +84,18 @@ class GiteaAPIClient:
"send_notify": False
}
try:
response, duration = self._request("POST", "admin/users", json=data)
vprint(f"[{duration:.3f}s] User '{username}' created.")
return True
self._request("POST", "admin/users", json=data)
print(f"User '{username}' created.")
except requests.exceptions.HTTPError as e:
if e.response.status_code == 422: # Already exists
vprint(f"User '{username}' already exists. Updating password...")
print(f"User '{username}' already exists. Updating password...")
# Update password to be sure it matches our expectation
response, duration = self._request("PATCH", f"admin/users/{username}", json={"password": password, "login_name": username})
return False
self._request("PATCH", f"admin/users/{username}", json={"password": password, "login_name": username})
else:
raise
def get_user_token(self, username, password, token_name="test-token"):
vprint(f"--- Getting token for user: {username} ---")
print(f"--- Getting token for user: {username} ---")
url = f"{self.base_url}/api/v1/users/{username}/tokens"
# Create new token using Basic Auth
@@ -98,30 +105,39 @@ class GiteaAPIClient:
response.raise_for_status()
def create_org(self, org_name):
vprint(f"--- Checking organization: {org_name} ---")
print(f"--- Checking organization: {org_name} ---")
try:
response, duration = self._request("GET", f"orgs/{org_name}")
vprint(f"[{duration:.3f}s] Organization '{org_name}' already exists.")
return False
self._request("GET", f"orgs/{org_name}")
print(f"Organization '{org_name}' already exists.")
except requests.exceptions.HTTPError as e:
if e.response.status_code == 404:
vprint(f"Creating organization '{org_name}'...")
print(f"Creating organization '{org_name}'...")
data = {"username": org_name, "full_name": org_name}
response, duration = self._request("POST", "orgs", json=data)
vprint(f"[{duration:.3f}s] Organization '{org_name}' created.")
return True
self._request("POST", "orgs", json=data)
print(f"Organization '{org_name}' created.")
else:
raise
print(f"--- Checking organization: {org_name} ---")
try:
self._request("GET", f"orgs/{org_name}")
print(f"Organization '{org_name}' already exists.")
except requests.exceptions.HTTPError as e:
if e.response.status_code == 404:
print(f"Creating organization '{org_name}'...")
data = {"username": org_name, "full_name": org_name}
self._request("POST", "orgs", json=data)
print(f"Organization '{org_name}' created.")
else:
raise
def create_repo(self, org_name, repo_name):
vprint(f"--- Checking repository: {org_name}/{repo_name} ---")
print(f"--- Checking repository: {org_name}/{repo_name} ---")
try:
response, duration = self._request("GET", f"repos/{org_name}/{repo_name}")
vprint(f"[{duration:.3f}s] Repository '{org_name}/{repo_name}' already exists.")
return False
self._request("GET", f"repos/{org_name}/{repo_name}")
print(f"Repository '{org_name}/{repo_name}' already exists.")
except requests.exceptions.HTTPError as e:
if e.response.status_code == 404:
vprint(f"Creating repository '{org_name}/{repo_name}'...")
print(f"Creating repository '{org_name}/{repo_name}'...")
data = {
"name": repo_name,
"auto_init": True,
@@ -131,48 +147,34 @@ class GiteaAPIClient:
"private": False,
"readme": "Default"
}
response, duration = self._request("POST", f"orgs/{org_name}/repos", json=data)
vprint(f"[{duration:.3f}s] Repository '{org_name}/{repo_name}' created with a README.")
self._request("POST", f"orgs/{org_name}/repos", json=data)
print(f"Repository '{org_name}/{repo_name}' created with a README.")
time.sleep(0.1) # Added delay to allow Git operations to become available
return True
else:
raise
def add_collaborator(self, org_name, repo_name, collaborator_name, permission="write"):
vprint(f"--- Adding {collaborator_name} as a collaborator to {org_name}/{repo_name} with '{permission}' permission ---")
# Check if already a collaborator to provide accurate stats
try:
self._request("GET", f"repos/{org_name}/{repo_name}/collaborators/{collaborator_name}")
vprint(f"{collaborator_name} is already a collaborator of {org_name}/{repo_name}.")
return False
except requests.exceptions.HTTPError as e:
if e.response.status_code != 404:
raise
print(f"--- Adding {collaborator_name} as a collaborator to {org_name}/{repo_name} with '{permission}' permission ---")
data = {"permission": permission}
# Gitea API returns 204 No Content on success and doesn't fail if already present.
response, duration = self._request("PUT", f"repos/{org_name}/{repo_name}/collaborators/{collaborator_name}", json=data)
vprint(f"[{duration:.3f}s] Added {collaborator_name} to {org_name}/{repo_name}.")
return True
self._request("PUT", f"repos/{org_name}/{repo_name}/collaborators/{collaborator_name}", json=data)
print(f"Attempted to add {collaborator_name} to {org_name}/{repo_name}.")
def add_submodules(self, org_name, repo_name):
vprint(f"--- Adding submodules to {org_name}/{repo_name} using diffpatch ---")
print(f"--- Adding submodules to {org_name}/{repo_name} using diffpatch ---")
parent_repo_path = f"repos/{org_name}/{repo_name}"
try:
response, duration = self._request("GET", f"{parent_repo_path}/contents/.gitmodules")
vprint(f"[{duration:.3f}s] Submodules appear to be already added. Skipping.")
self._request("GET", f"{parent_repo_path}/contents/.gitmodules")
print("Submodules appear to be already added. Skipping.")
return
except requests.exceptions.HTTPError as e:
if e.response.status_code != 404:
raise
# Get latest commit SHAs for the submodules
response_a, duration_a = self._request("GET", "repos/mypool/pkgA/branches/main")
pkg_a_sha = response_a.json()["commit"]["id"]
response_b, duration_b = self._request("GET", "repos/mypool/pkgB/branches/main")
pkg_b_sha = response_b.json()["commit"]["id"]
pkg_a_sha = self._request("GET", "repos/mypool/pkgA/branches/main").json()["commit"]["id"]
pkg_b_sha = self._request("GET", "repos/mypool/pkgB/branches/main").json()["commit"]["id"]
if not pkg_a_sha or not pkg_b_sha:
raise Exception("Error: Could not get submodule commit SHAs. Cannot apply patch.")
@@ -210,81 +212,34 @@ index 0000000..{pkg_b_sha}
"content": diff_content,
"message": message
}
vprint(f"Applying submodule patch to {org_name}/{repo_name}...")
response, duration = self._request("POST", f"{parent_repo_path}/diffpatch", json=data)
vprint(f"[{duration:.3f}s] Submodule patch applied.")
print(f"Applying submodule patch to {org_name}/{repo_name}...")
self._request("POST", f"{parent_repo_path}/diffpatch", json=data)
print("Submodule patch applied.")
def update_repo_settings(self, org_name, repo_name):
vprint(f"--- Updating repository settings for: {org_name}/{repo_name} ---")
response, duration = self._request("GET", f"repos/{org_name}/{repo_name}")
repo_data = response.json()
print(f"--- Updating repository settings for: {org_name}/{repo_name} ---")
repo_data = self._request("GET", f"repos/{org_name}/{repo_name}").json()
# Ensure these are boolean values, not string
repo_data["allow_manual_merge"] = True
repo_data["autodetect_manual_merge"] = True
response, duration = self._request("PATCH", f"repos/{org_name}/{repo_name}", json=repo_data)
vprint(f"[{duration:.3f}s] Repository settings for '{org_name}/{repo_name}' updated.")
def create_webhook(self, owner: str, repo: str, target_url: str):
vprint(f"--- Checking webhook for {owner}/{repo} -> {target_url} ---")
url = f"repos/{owner}/{repo}/hooks"
try:
response, duration = self._request("GET", url)
hooks = response.json()
for hook in hooks:
if hook["config"]["url"] == target_url:
vprint(f"Webhook for {owner}/{repo} already exists with correct URL.")
return False
elif "gitea-publisher" in hook["config"]["url"] or "10.89.0." in hook["config"]["url"]:
vprint(f"Found old webhook {hook['id']} with URL {hook['config']['url']}. Deleting...")
self._request("DELETE", f"{url}/{hook['id']}")
except requests.exceptions.HTTPError:
pass
vprint(f"--- Creating webhook for {owner}/{repo} -> {target_url} ---")
data = {
"type": "gitea",
"config": {
"url": target_url,
"content_type": "json"
},
"events": ["push", "pull_request", "pull_request_review", "issue_comment"],
"active": True
}
response, duration = self._request("POST", url, json=data)
vprint(f"[{duration:.3f}s] Webhook created for {owner}/{repo}.")
return True
self._request("PATCH", f"repos/{org_name}/{repo_name}", json=repo_data)
print(f"Repository settings for '{org_name}/{repo_name}' updated.")
def create_label(self, owner: str, repo: str, name: str, color: str = "#abcdef"):
vprint(f"--- Checking label '{name}' in {owner}/{repo} ---")
print(f"--- Creating label '{name}' in {owner}/{repo} ---")
url = f"repos/{owner}/{repo}/labels"
# Check if label exists first
try:
response, duration = self._request("GET", url)
labels = response.json()
for label in labels:
if label["name"] == name:
vprint(f"Label '{name}' already exists in {owner}/{repo}.")
return False
except requests.exceptions.HTTPError:
pass
vprint(f"--- Creating label '{name}' in {owner}/{repo} ---")
data = {
"name": name,
"color": color
}
try:
response, duration = self._request("POST", url, json=data)
vprint(f"[{duration:.3f}s] Label '{name}' created.")
return True
self._request("POST", url, json=data)
print(f"Label '{name}' created.")
except requests.exceptions.HTTPError as e:
if e.response.status_code == 422: # Already exists (race condition or other reason)
vprint(f"Label '{name}' already exists.")
return False
if e.response.status_code == 422: # Already exists
print(f"Label '{name}' already exists.")
else:
raise
@@ -298,7 +253,7 @@ index 0000000..{pkg_b_sha}
}
if file_info:
vprint(f"--- Updating file {file_path} in {owner}/{repo} ---")
print(f"--- Updating file {file_path} in {owner}/{repo} ---")
# Re-fetch file_info to get the latest SHA right before update
latest_file_info = self.get_file_info(owner, repo, file_path, branch=branch)
if not latest_file_info:
@@ -307,12 +262,12 @@ index 0000000..{pkg_b_sha}
data["message"] = f"Update {file_path}"
method = "PUT"
else:
vprint(f"--- Creating file {file_path} in {owner}/{repo} ---")
print(f"--- Creating file {file_path} in {owner}/{repo} ---")
method = "POST"
url = f"repos/{owner}/{repo}/contents/{file_path}"
response, duration = self._request(method, url, json=data)
vprint(f"[{duration:.3f}s] File {file_path} {'updated' if file_info else 'created'} in {owner}/{repo}.")
self._request(method, url, json=data)
print(f"File {file_path} {'updated' if file_info else 'created'} in {owner}/{repo}.")
def create_gitea_pr(self, repo_full_name: str, diff_content: str, title: str, use_fork: bool, base_branch: str = "main", body: str = ""):
owner, repo = repo_full_name.split("/")
@@ -325,20 +280,20 @@ index 0000000..{pkg_b_sha}
head_owner = sudo_user
head_repo = repo
vprint(f"--- Forking {repo_full_name} ---")
print(f"--- Forking {repo_full_name} ---")
try:
response, duration = self._request("POST", f"repos/{owner}/{repo}/forks", json={})
vprint(f"[{duration:.3f}s] --- Forked to {head_owner}/{head_repo} ---")
self._request("POST", f"repos/{owner}/{repo}/forks", json={})
print(f"--- Forked to {head_owner}/{head_repo} ---")
time.sleep(0.5) # Give more time for fork to be ready
except requests.exceptions.HTTPError as e:
if e.response.status_code == 409: # Already forked
vprint(f"--- Already forked to {head_owner}/{head_repo} ---")
print(f"--- Already forked to {head_owner}/{head_repo} ---")
else:
raise
# Apply the diff using diffpatch and create the new branch automatically
vprint(f"--- Applying diff to {head_owner}/{head_repo} from {base_branch} to new branch {new_branch_name} ---")
response, duration = self._request("POST", f"repos/{head_owner}/{head_repo}/diffpatch", json={
print(f"--- Applying diff to {head_owner}/{head_repo} from {base_branch} to new branch {new_branch_name} ---")
self._request("POST", f"repos/{head_owner}/{head_repo}/diffpatch", json={
"branch": base_branch,
"new_branch": new_branch_name,
"content": diff_content,
@@ -353,59 +308,59 @@ index 0000000..{pkg_b_sha}
"body": body,
"allow_maintainer_edit": True
}
vprint(f"--- Creating PR in {repo_full_name} from {data['head']} ---")
response, duration = self._request("POST", f"repos/{owner}/{repo}/pulls", json=data)
print(f"--- Creating PR in {repo_full_name} from {data['head']} ---")
response = self._request("POST", f"repos/{owner}/{repo}/pulls", json=data)
return response.json()
def create_branch(self, owner: str, repo: str, new_branch_name: str, old_ref: str):
vprint(f"--- Checking branch '{new_branch_name}' in {owner}/{repo} ---")
print(f"--- Checking branch '{new_branch_name}' in {owner}/{repo} ---")
try:
response, duration = self._request("GET", f"repos/{owner}/{repo}/branches/{new_branch_name}")
vprint(f"[{duration:.3f}s] Branch '{new_branch_name}' already exists.")
return False
self._request("GET", f"repos/{owner}/{repo}/branches/{new_branch_name}")
print(f"Branch '{new_branch_name}' already exists.")
return
except requests.exceptions.HTTPError as e:
if e.response.status_code != 404:
raise # Re-raise other HTTP errors
vprint(f"--- Creating branch '{new_branch_name}' in {owner}/{repo} from {old_ref} ---")
print(f"--- Creating branch '{new_branch_name}' in {owner}/{repo} from {old_ref} ---")
url = f"repos/{owner}/{repo}/branches"
data = {
"new_branch_name": new_branch_name,
"old_ref": old_ref
}
response, duration = self._request("POST", url, json=data)
vprint(f"[{duration:.3f}s] Branch '{new_branch_name}' created in {owner}/{repo}.")
return True
self._request("POST", url, json=data)
print(f"Branch '{new_branch_name}' created in {owner}/{repo}.")
def ensure_branch_exists(self, owner: str, repo: str, branch: str = "main", timeout: int = 10):
vprint(f"--- Ensuring branch '{branch}' exists in {owner}/{repo} ---")
print(f"--- Ensuring branch '{branch}' exists in {owner}/{repo} ---")
start_time = time.time()
while time.time() - start_time < timeout:
try:
response, duration = self._request("GET", f"repos/{owner}/{repo}/branches/{branch}")
vprint(f"[{duration:.3f}s] Branch '{branch}' confirmed in {owner}/{repo}.")
self._request("GET", f"repos/{owner}/{repo}/branches/{branch}")
print(f"Branch '{branch}' confirmed in {owner}/{repo}.")
return
except requests.exceptions.HTTPError as e:
if e.response.status_code == 404:
vprint(f"Branch '{branch}' not found yet in {owner}/{repo}. Retrying...")
print(f"Branch '{branch}' not found yet in {owner}/{repo}. Retrying...")
time.sleep(1)
continue
raise
raise Exception(f"Timeout waiting for branch {branch} in {owner}/{repo}")
def modify_gitea_pr(self, repo_full_name: str, pr_number: int, diff_content: str, message: str):
owner, repo = repo_full_name.split("/")
# Get PR details to find the head branch AND head repo
response, duration = self._request("GET", f"repos/{owner}/{repo}/pulls/{pr_number}")
pr_details = response.json()
pr_details = self._request("GET", f"repos/{owner}/{repo}/pulls/{pr_number}").json()
head_branch = pr_details["head"]["ref"]
head_repo_owner = pr_details["head"]["repo"]["owner"]["login"]
head_repo_name = pr_details["head"]["repo"]["name"]
# Apply the diff using diffpatch
vprint(f"--- Modifying PR #{pr_number} in {head_repo_owner}/{head_repo_name} branch {head_branch} ---")
response, duration = self._request("POST", f"repos/{head_repo_owner}/{head_repo_name}/diffpatch", json={
print(f"--- Modifying PR #{pr_number} in {head_repo_owner}/{head_repo_name} branch {head_branch} ---")
self._request("POST", f"repos/{head_repo_owner}/{head_repo_name}/diffpatch", json={
"branch": head_branch,
"content": diff_content,
"message": message
@@ -414,15 +369,15 @@ index 0000000..{pkg_b_sha}
def update_gitea_pr_properties(self, repo_full_name: str, pr_number: int, **kwargs):
owner, repo = repo_full_name.split("/")
url = f"repos/{owner}/{repo}/pulls/{pr_number}"
response, duration = self._request("PATCH", url, json=kwargs)
response = self._request("PATCH", url, json=kwargs)
return response.json()
def create_issue_comment(self, repo_full_name: str, issue_number: int, body: str):
owner, repo = repo_full_name.split("/")
url = f"repos/{owner}/{repo}/issues/{issue_number}/comments"
data = {"body": body}
vprint(f"--- Creating comment on {repo_full_name} issue #{issue_number} ---")
response, duration = self._request("POST", url, json=data)
print(f"--- Creating comment on {repo_full_name} issue #{issue_number} ---")
response = self._request("POST", url, json=data)
return response.json()
def get_timeline_events(self, repo_full_name: str, pr_number: int):
@@ -432,15 +387,15 @@ index 0000000..{pkg_b_sha}
# Retry logic for timeline events
for i in range(10): # Try up to 10 times
try:
response, duration = self._request("GET", url)
response = self._request("GET", url)
timeline_events = response.json()
if timeline_events: # Check if timeline_events list is not empty
return timeline_events
vprint(f"Attempt {i+1}: Timeline for PR {pr_number} is empty. Retrying in 1 seconds...")
print(f"Attempt {i+1}: Timeline for PR {pr_number} is empty. Retrying in 1 seconds...")
time.sleep(1)
except requests.exceptions.HTTPError as e:
if e.response.status_code == 404:
vprint(f"Attempt {i+1}: Timeline for PR {pr_number} not found yet. Retrying in 1 seconds...")
print(f"Attempt {i+1}: Timeline for PR {pr_number} not found yet. Retrying in 1 seconds...")
time.sleep(1)
else:
raise # Re-raise other HTTP errors
@@ -453,16 +408,16 @@ index 0000000..{pkg_b_sha}
# Retry logic for comments
for i in range(10): # Try up to 10 times
try:
response, duration = self._request("GET", url)
response = self._request("GET", url)
comments = response.json()
vprint(f"[{duration:.3f}s] Attempt {i+1}: Comments for PR {pr_number} received: {comments}") # Added debug print
print(f"Attempt {i+1}: Comments for PR {pr_number} received: {comments}") # Added debug print
if comments: # Check if comments list is not empty
return comments
vprint(f"Attempt {i+1}: Comments for PR {pr_number} are empty. Retrying in 1 seconds...")
print(f"Attempt {i+1}: Comments for PR {pr_number} are empty. Retrying in 1 seconds...")
time.sleep(1)
except requests.exceptions.HTTPError as e:
if e.response.status_code == 404:
vprint(f"Attempt {i+1}: Comments for PR {pr_number} not found yet. Retrying in 1 seconds...")
print(f"Attempt {i+1}: Comments for PR {pr_number} not found yet. Retrying in 1 seconds...")
time.sleep(1)
else:
raise # Re-raise other HTTP errors
@@ -471,7 +426,7 @@ index 0000000..{pkg_b_sha}
def get_pr_details(self, repo_full_name: str, pr_number: int):
owner, repo = repo_full_name.split("/")
url = f"repos/{owner}/{repo}/pulls/{pr_number}"
response, duration = self._request("GET", url)
response = self._request("GET", url)
return response.json()
def create_review(self, repo_full_name: str, pr_number: int, event: str = "APPROVED", body: str = "LGTM"):
@@ -482,7 +437,7 @@ index 0000000..{pkg_b_sha}
existing_reviews = self.list_reviews(repo_full_name, pr_number)
for r in existing_reviews:
if r["user"]["login"] == current_user and r["state"] == "APPROVED" and event == "APPROVED":
vprint(f"User {current_user} already has an APPROVED review for {repo_full_name} PR #{pr_number}")
print(f"User {current_user} already has an APPROVED review for {repo_full_name} PR #{pr_number}")
return r
url = f"repos/{owner}/{repo}/pulls/{pr_number}/reviews"
@@ -490,13 +445,13 @@ index 0000000..{pkg_b_sha}
"event": event,
"body": body
}
vprint(f"--- Creating and submitting review ({event}) for {repo_full_name} PR #{pr_number} as {current_user} ---")
print(f"--- Creating and submitting review ({event}) for {repo_full_name} PR #{pr_number} as {current_user} ---")
try:
response, duration = self._request("POST", url, json=data)
response = self._request("POST", url, json=data)
review = response.json()
except requests.exceptions.HTTPError as e:
# If it fails with 422, it might be because a review is already pending or something else
vprint(f"Failed to create review: {e.response.text}")
print(f"Failed to create review: {e.response.text}")
# Try to find a pending review to submit
existing_reviews = self.list_reviews(repo_full_name, pr_number)
pending_review = next((r for r in existing_reviews if r["user"]["login"] == current_user and r["state"] == "PENDING"), None)
@@ -514,11 +469,11 @@ index 0000000..{pkg_b_sha}
"body": body
}
try:
response, duration = self._request("POST", submit_url, json=submit_data)
vprint(f"[{duration:.3f}s] --- Review {review_id} submitted ---")
self._request("POST", submit_url, json=submit_data)
print(f"--- Review {review_id} submitted ---")
except requests.exceptions.HTTPError as e:
if "already" in e.response.text.lower() or "stay pending" in e.response.text.lower():
vprint(f"Review {review_id} could not be submitted further: {e.response.text}")
print(f"Review {review_id} could not be submitted further: {e.response.text}")
else:
raise
@@ -527,29 +482,39 @@ index 0000000..{pkg_b_sha}
def list_reviews(self, repo_full_name: str, pr_number: int):
owner, repo = repo_full_name.split("/")
url = f"repos/{owner}/{repo}/pulls/{pr_number}/reviews"
response, duration = self._request("GET", url)
response = self._request("GET", url)
return response.json()
def approve_requested_reviews(self, repo_full_name: str, pr_number: int):
vprint(f"--- Checking for REQUEST_REVIEW state in {repo_full_name} PR #{pr_number} ---")
print(f"--- Checking for REQUEST_REVIEW state in {repo_full_name} PR #{pr_number} ---")
reviews = self.list_reviews(repo_full_name, pr_number)
requested_reviews = [r for r in reviews if r["state"] == "REQUEST_REVIEW"]
if not requested_reviews:
vprint(f"No reviews in REQUEST_REVIEW state found for {repo_full_name} PR #{pr_number}")
print(f"No reviews in REQUEST_REVIEW state found for {repo_full_name} PR #{pr_number}")
return
admin_token = self.headers["Authorization"].split(" ")[1]
for r in requested_reviews:
reviewer_username = r["user"]["login"]
vprint(f"Reacting on REQUEST_REVIEW for user {reviewer_username} by approving...")
print(f"Reacting on REQUEST_REVIEW for user {reviewer_username} by approving...")
reviewer_client = GiteaAPIClient(base_url=self.base_url, token=admin_token, sudo=reviewer_username)
time.sleep(1) # give a chance to avoid possible concurrency issues with reviews request/approval
reviewer_client.create_review(repo_full_name, pr_number, event="APPROVED", body="Approving requested review")
def restart_service(self, service_name: str):
print(f"--- Restarting service: {service_name} ---")
try:
# Assumes podman-compose.yml is in the parent directory of tests/lib
subprocess.run(["podman-compose", "restart", service_name], check=True, cwd=os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)))
print(f"Service {service_name} restarted successfully.")
except subprocess.CalledProcessError as e:
print(f"Error restarting service {service_name}: {e}")
raise
def wait_for_project_pr(self, package_pr_repo, package_pr_number, project_pr_repo="myproducts/mySLFO", timeout=60):
vprint(f"Polling {package_pr_repo} PR #{package_pr_number} timeline for forwarded PR event in {project_pr_repo}...")
print(f"Polling {package_pr_repo} PR #{package_pr_number} timeline for forwarded PR event in {project_pr_repo}...")
for _ in range(timeout):
time.sleep(1)
timeline_events = self.get_timeline_events(package_pr_repo, package_pr_number)
@@ -564,7 +529,7 @@ index 0000000..{pkg_b_sha}
return None
def approve_and_wait_merge(self, package_pr_repo, package_pr_number, project_pr_number, project_pr_repo="myproducts/mySLFO", timeout=30):
vprint(f"Approving reviews and verifying both PRs are merged ({package_pr_repo}#{package_pr_number} and {project_pr_repo}#{project_pr_number})...")
print(f"Approving reviews and verifying both PRs are merged ({package_pr_repo}#{package_pr_number} and {project_pr_repo}#{project_pr_number})...")
package_merged = False
project_merged = False
@@ -576,16 +541,17 @@ index 0000000..{pkg_b_sha}
pkg_details = self.get_pr_details(package_pr_repo, package_pr_number)
if pkg_details.get("merged"):
package_merged = True
vprint(f"Package PR {package_pr_repo}#{package_pr_number} merged.")
print(f"Package PR {package_pr_repo}#{package_pr_number} merged.")
if not project_merged:
prj_details = self.get_pr_details(project_pr_repo, project_pr_number)
if prj_details.get("merged"):
project_merged = True
vprint(f"Project PR {project_pr_repo}#{project_pr_number} merged.")
print(f"Project PR {project_pr_repo}#{project_pr_number} merged.")
if package_merged and project_merged:
return True, True
time.sleep(1)
return package_merged, project_merged

View File

@@ -1,9 +1,12 @@
import pytest
import re
import time
import subprocess
import requests
from pathlib import Path
from tests.lib.common_test_utils import (
GiteaAPIClient,
mock_build_result,
)
# =============================================================================
@@ -18,6 +21,8 @@ def test_pr_workflow_succeeded(staging_main_env, mock_build_result):
pr = gitea_env.create_gitea_pr("mypool/pkgA", diff, "Test PR - should succeed", False, base_branch=merge_branch_name)
initial_pr_number = pr["number"]
compose_dir = Path(__file__).parent.parent
forwarded_pr_number = gitea_env.wait_for_project_pr("mypool/pkgA", initial_pr_number)
assert (
forwarded_pr_number is not None
@@ -38,10 +43,17 @@ def test_pr_workflow_succeeded(staging_main_env, mock_build_result):
assert reviewer_added, "Staging bot was not added as a reviewer."
print("Staging bot has been added as a reviewer.")
mock_build_result(package_name="pkgA", code="succeeded")
mock_build_result(package_name="pkgA", code="succeeded")
print("Restarting obs-staging-bot...")
subprocess.run(
["podman-compose", "restart", "obs-staging-bot"],
cwd=compose_dir,
check=True,
capture_output=True,
)
print(f"Polling myproducts/mySLFO PR #{forwarded_pr_number} for final status...")
status_comment_found = False
for _ in range(20):
time.sleep(1)
@@ -63,6 +75,8 @@ def test_pr_workflow_failed(staging_main_env, mock_build_result):
pr = gitea_env.create_gitea_pr("mypool/pkgA", diff, "Test PR - should fail", False, base_branch=merge_branch_name)
initial_pr_number = pr["number"]
compose_dir = Path(__file__).parent.parent
forwarded_pr_number = gitea_env.wait_for_project_pr("mypool/pkgA", initial_pr_number)
assert (
forwarded_pr_number is not None
@@ -85,6 +99,14 @@ def test_pr_workflow_failed(staging_main_env, mock_build_result):
mock_build_result(package_name="pkgA", code="failed")
print("Restarting obs-staging-bot...")
subprocess.run(
["podman-compose", "restart", "obs-staging-bot"],
cwd=compose_dir,
check=True,
capture_output=True,
)
print(f"Polling myproducts/mySLFO PR #{forwarded_pr_number} for final status...")
status_comment_found = False
for _ in range(20):

View File

@@ -36,7 +36,7 @@ index 0000000..e69de29
print("Both PRs merged successfully.")
@pytest.mark.t002
def test_002_manual_merge(manual_merge_env, test_user_client, usera_client, staging_bot_client, ownerA_client):
def test_002_manual_merge(manual_merge_env, test_user_client, usera_client, staging_bot_client):
"""
Test scenario TC-MERGE-002:
1. Create a PackageGit PR with ManualMergeOnly set to true.
@@ -52,7 +52,7 @@ new file mode 100644
index 0000000..e69de29
"""
print(f"--- Creating package PR in mypool/pkgA on branch {merge_branch_name} ---")
package_pr = gitea_env.create_gitea_pr("mypool/pkgA", diff, "Test Manual Merge Fixture", False, base_branch=merge_branch_name)
package_pr = test_user_client.create_gitea_pr("mypool/pkgA", diff, "Test Manual Merge Fixture", False, base_branch=merge_branch_name)
package_pr_number = package_pr["number"]
print(f"Created package PR mypool/pkgA#{package_pr_number}")
@@ -62,14 +62,13 @@ index 0000000..e69de29
print(f"Found project PR: myproducts/mySLFO#{project_pr_number}")
# 3. Approve reviews and verify NOT merged
print("Waiting for required review requests and approving them...")
print("Waiting for all expected review requests and approving them...")
# Expected reviewers based on manual-merge branch config and pkgA maintainership
mandatory_reviewers = {"usera", "userb"}
maintainers = {"ownerA", "ownerX", "ownerY"}
expected_reviewers = {"usera", "userb", "ownerA", "ownerX", "ownerY"}
# ManualMergeOnly still requires regular reviews to be satisfied.
# We poll until required reviewers have approved.
all_approved = False
# We poll until all expected reviewers are requested, then approve them.
all_requested = False
for _ in range(30):
# Trigger approvals for whatever is already requested
gitea_env.approve_requested_reviews("mypool/pkgA", package_pr_number)
@@ -81,17 +80,20 @@ index 0000000..e69de29
print("Staging bot has a pending/requested review. Approving...")
staging_bot_client.create_review("myproducts/mySLFO", project_pr_number, event="APPROVED", body="Staging bot approves")
# Check if mandatory reviewers and at least one maintainer have approved
# Check if all expected reviewers have at least one review record (any state)
pkg_reviews = gitea_env.list_reviews("mypool/pkgA", package_pr_number)
approved_reviewers = {r["user"]["login"] for r in pkg_reviews if r["state"] == "APPROVED"}
current_reviewers = {r["user"]["login"] for r in pkg_reviews}
if mandatory_reviewers.issubset(approved_reviewers) and any(m in approved_reviewers for m in maintainers):
# And check project PR for bot approval
prj_approved = any(r["user"]["login"] == "autogits_obs_staging_bot" and r["state"] == "APPROVED" for r in prj_reviews)
if prj_approved:
all_approved = True
print(f"Required reviewers approved: mandatory={mandatory_reviewers}, maintainer={[m for m in maintainers if m in approved_reviewers]}, staging_bot=True")
break
if expected_reviewers.issubset(current_reviewers):
# Also ensure they are all approved (not just requested)
approved_reviewers = {r["user"]["login"] for r in pkg_reviews if r["state"] == "APPROVED"}
if expected_reviewers.issubset(approved_reviewers):
# And check project PR for bot approval
prj_approved = any(r["user"]["login"] == "autogits_obs_staging_bot" and r["state"] == "APPROVED" for r in prj_reviews)
if prj_approved:
all_requested = True
print(f"All expected reviewers {expected_reviewers} and staging bot have approved.")
break
pkg_details = gitea_env.get_pr_details("mypool/pkgA", package_pr_number)
prj_details = gitea_env.get_pr_details("myproducts/mySLFO", project_pr_number)
@@ -101,12 +103,12 @@ index 0000000..e69de29
time.sleep(2)
assert all_approved, f"Timed out waiting for required approvals. Mandatory: {mandatory_reviewers}, Maintainers: {maintainers}. Current approved: {approved_reviewers}"
assert all_requested, f"Timed out waiting for all expected reviewers {expected_reviewers} to approve. Current: {current_reviewers}"
print("Both PRs have all required approvals but are not merged (as expected with ManualMergeOnly).")
# 4. Comment "merge ok" from a requested reviewer (ownerA)
print("Commenting 'merge ok' on package PR from a maintainer...")
ownerA_client.create_issue_comment("mypool/pkgA", package_pr_number, "merge ok")
# 4. Comment "merge ok" from a requested reviewer (usera)
print("Commenting 'merge ok' on package PR...")
usera_client.create_issue_comment("mypool/pkgA", package_pr_number, "merge ok")
# 5. Verify both PRs are merged
print("Polling for PR merge status...")
@@ -162,14 +164,13 @@ index 0000000..e69de29
print(f"Found project PR: myproducts/mySLFO#{project_pr_number}")
# 3. Approve reviews and verify NOT merged
print("Waiting for required review requests and approving them...")
print("Waiting for all expected review requests and approving them...")
# Expected reviewers based on manual-merge branch config and pkgA maintainership
mandatory_reviewers = {"usera", "userb"}
maintainers = {"ownerA", "ownerX", "ownerY"}
expected_reviewers = {"usera", "userb", "ownerA", "ownerX", "ownerY"}
# ManualMergeOnly still requires regular reviews to be satisfied.
# We poll until required reviewers have approved.
all_approved = False
# We poll until all expected reviewers are requested, then approve them.
all_requested = False
for _ in range(30):
# Trigger approvals for whatever is already requested
gitea_env.approve_requested_reviews("mypool/pkgA", package_pr_number)
@@ -181,17 +182,20 @@ index 0000000..e69de29
print("Staging bot has a pending/requested review. Approving...")
staging_bot_client.create_review("myproducts/mySLFO", project_pr_number, event="APPROVED", body="Staging bot approves")
# Check if mandatory reviewers and at least one maintainer have approved
# Check if all expected reviewers have at least one review record (any state)
pkg_reviews = gitea_env.list_reviews("mypool/pkgA", package_pr_number)
approved_reviewers = {r["user"]["login"] for r in pkg_reviews if r["state"] == "APPROVED"}
current_reviewers = {r["user"]["login"] for r in pkg_reviews}
if mandatory_reviewers.issubset(approved_reviewers) and any(m in approved_reviewers for m in maintainers):
# And check project PR for bot approval
prj_approved = any(r["user"]["login"] == "autogits_obs_staging_bot" and r["state"] == "APPROVED" for r in prj_reviews)
if prj_approved:
all_approved = True
print(f"Required reviewers approved: mandatory={mandatory_reviewers}, maintainer={[m for m in maintainers if m in approved_reviewers]}, staging_bot=True")
break
if expected_reviewers.issubset(current_reviewers):
# Also ensure they are all approved (not just requested)
approved_reviewers = {r["user"]["login"] for r in pkg_reviews if r["state"] == "APPROVED"}
if expected_reviewers.issubset(approved_reviewers):
# And check project PR for bot approval
prj_approved = any(r["user"]["login"] == "autogits_obs_staging_bot" and r["state"] == "APPROVED" for r in prj_reviews)
if prj_approved:
all_requested = True
print(f"All expected reviewers {expected_reviewers} and staging bot have approved.")
break
pkg_details = gitea_env.get_pr_details("mypool/pkgA", package_pr_number)
prj_details = gitea_env.get_pr_details("myproducts/mySLFO", project_pr_number)
@@ -201,7 +205,7 @@ index 0000000..e69de29
time.sleep(2)
assert all_approved, f"Timed out waiting for required approvals. Mandatory: {mandatory_reviewers}, Maintainers: {maintainers}. Current approved: {approved_reviewers}"
assert all_requested, f"Timed out waiting for all expected reviewers {expected_reviewers} to approve. Current: {current_reviewers}"
print("Both PRs have all required approvals but are not merged (as expected with ManualMergeOnly).")
# 4. Comment "merge ok" from a requested reviewer (ownerB)
@@ -394,12 +398,10 @@ index 0000000..e69de29
print("Replace merge successful.")
# Verify that the project branch HEAD is a merge commit
resp, _ = gitea_env._request("GET", f"repos/myproducts/mySLFO/branches/{merge_branch_name}")
branch_info = resp.json()
branch_info = gitea_env._request("GET", f"repos/myproducts/mySLFO/branches/{merge_branch_name}").json()
new_head_sha = branch_info["commit"]["id"]
resp, _ = gitea_env._request("GET", f"repos/myproducts/mySLFO/git/commits/{new_head_sha}")
commit_details = resp.json()
commit_details = gitea_env._request("GET", f"repos/myproducts/mySLFO/git/commits/{new_head_sha}").json()
assert len(commit_details["parents"]) > 1, f"Project branch {merge_branch_name} HEAD should be a merge commit but has {len(commit_details['parents'])} parents"
# Verify that pkgA submodule points to the correct SHA
@@ -439,13 +441,11 @@ index 0000000..e69de29
print("Devel FF merge successful.")
# Verify that the package base branch HEAD is the same as the PR head (FF)
resp, _ = gitea_env._request("GET", f"repos/mypool/pkgA/branches/{merge_branch_name}")
branch_info = resp.json()
branch_info = gitea_env._request("GET", f"repos/mypool/pkgA/branches/{merge_branch_name}").json()
new_head_sha = branch_info["commit"]["id"]
assert new_head_sha == pkg_head_sha, f"Package branch {merge_branch_name} HEAD should be {pkg_head_sha} but is {new_head_sha}"
resp, _ = gitea_env._request("GET", f"repos/mypool/pkgA/git/commits/{new_head_sha}")
commit_details = resp.json()
commit_details = gitea_env._request("GET", f"repos/mypool/pkgA/git/commits/{new_head_sha}").json()
assert len(commit_details["parents"]) == 1, f"Package branch {merge_branch_name} HEAD should have 1 parent but has {len(commit_details['parents'])}"
@pytest.mark.t013
@@ -481,12 +481,10 @@ index 0000000..e69de29
print("Replace FF merge successful.")
# Verify that the package base branch HEAD is the same as the PR head (FF)
resp, _ = gitea_env._request("GET", f"repos/mypool/pkgA/branches/{merge_branch_name}")
branch_info = resp.json()
branch_info = gitea_env._request("GET", f"repos/mypool/pkgA/branches/{merge_branch_name}").json()
new_head_sha = branch_info["commit"]["id"]
assert new_head_sha == pkg_head_sha, f"Package branch {merge_branch_name} HEAD should be {pkg_head_sha} but is {new_head_sha}"
resp, _ = gitea_env._request("GET", f"repos/mypool/pkgA/git/commits/{new_head_sha}")
commit_details = resp.json()
commit_details = gitea_env._request("GET", f"repos/mypool/pkgA/git/commits/{new_head_sha}").json()
assert len(commit_details["parents"]) == 1, f"Package branch {merge_branch_name} HEAD should have 1 parent but has {len(commit_details['parents'])}"

View File

@@ -23,19 +23,12 @@ echo "Waiting for workflow.config in myproducts/mySLFO..."
API_URL="http://gitea-test:3000/api/v1/repos/myproducts/mySLFO/contents/workflow.config"
HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token $GITEA_TOKEN" "$API_URL")
WAITED=false
while [ "$HTTP_STATUS" != "200" ]; do
WAITED=true
echo "workflow.config not found yet (HTTP Status: $HTTP_STATUS). Retrying in 5s..."
sleep 5
HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token $GITEA_TOKEN" "$API_URL")
done
if [ "$WAITED" = true ]; then
echo "workflow.config found. Sleeping 15s to let other configurations settle..."
sleep 15
fi
# Wait for the shared SSH key to be generated by the gitea setup script
echo "Waiting for /var/lib/gitea/ssh-keys/id_ed25519..."
while [ ! -f /var/lib/gitea/ssh-keys/id_ed25519 ]; do
@@ -70,5 +63,4 @@ package=$(rpm -qa | grep autogits-workflow-pr) || :
echo "!!!!!!!!!!!!!!!! using binary $exe; installed package: $package"
which strings > /dev/null 2>&1 && strings "$exe" | grep -A 2 vcs.revision= | head -4 || :
set -x
exec "$exe" "$@"

View File

@@ -1171,7 +1171,6 @@ var IsDryRun bool
var ProcessPROnly string
var ObsClient common.ObsClientInterface
var BotUser string
var PollInterval = 5 * time.Minute
func ObsWebHostFromApiHost(apihost string) string {
u, err := url.Parse(apihost)
@@ -1194,18 +1193,9 @@ func main() {
flag.StringVar(&ObsApiHost, "obs", "", "API for OBS instance")
flag.StringVar(&ObsWebHost, "obs-web", "", "Web OBS instance, if not derived from the obs config")
flag.BoolVar(&IsDryRun, "dry", false, "Dry-run, don't actually create any build projects or review changes")
pollIntervalStr := flag.String("poll-interval", common.GetEnvOverrideString(os.Getenv("AUTOGITS_STAGING_BOT_POLL_INTERVAL"), ""), "Polling interval for notifications (e.g. 5m, 10s)")
debug := flag.Bool("debug", false, "Turns on debug logging")
flag.Parse()
if len(*pollIntervalStr) > 0 {
if d, err := time.ParseDuration(*pollIntervalStr); err == nil {
PollInterval = d
} else {
common.LogError("Invalid poll interval:", err)
}
}
if *debug {
common.SetLoggingLevel(common.LogLevelDebug)
} else {
@@ -1274,6 +1264,6 @@ func main() {
for {
PollWorkNotifications(ObsClient, gitea)
common.LogInfo("Poll cycle finished")
time.Sleep(PollInterval)
time.Sleep(5 * time.Minute)
}
}

View File

@@ -503,10 +503,7 @@ func updateConfiguration(configFilename string, orgs *[]string) {
os.Exit(4)
}
configs, err := common.ResolveWorkflowConfigs(gitea, configFile)
if err != nil {
common.LogError("Failed to resolve some configuration repositories:", err)
}
configs, _ := common.ResolveWorkflowConfigs(gitea, configFile)
configuredRepos = make(map[string][]*common.AutogitConfig)
*orgs = make([]string, 0, 1)
for _, c := range configs {

View File

@@ -58,7 +58,6 @@ func main() {
checkOnStart := flag.Bool("check-on-start", common.GetEnvOverrideBool(os.Getenv("AUTOGITS_CHECK_ON_START"), false), "Check all repositories for consistency on start, without delays")
checkIntervalHours := flag.Float64("check-interval", 5, "Check interval (+-random delay) for repositories for consitency, in hours")
flag.BoolVar(&ListPROnly, "list-prs-only", false, "Only lists PRs without acting on them")
exitOnConfigError := flag.Bool("exit-on-config-error", false, "Exit if any repository in configuration cannot be resolved")
flag.Int64Var(&PRID, "id", -1, "Process only the specific ID and ignore the rest. Use for debugging")
basePath := flag.String("repo-path", common.GetEnvOverrideString(os.Getenv("AUTOGITS_REPO_PATH"), ""), "Repository path. Default is temporary directory")
pr := flag.String("only-pr", "", "Only specific PR to process. For debugging")
@@ -98,10 +97,8 @@ func main() {
configs, err := common.ResolveWorkflowConfigs(Gitea, config)
if err != nil {
common.LogError("Failed to resolve some configuration repositories:", err)
if *exitOnConfigError {
return
}
common.LogError("Cannot resolve config files:", err)
return
}
for _, c := range configs {