Compare commits

..

1 Commits

Author SHA256 Message Date
Andrii Nikitin
58f410befc common: add TIMELINE_CACHE_DISABLE env var
All checks were successful
go-generate-check / go-generate-check (pull_request) Successful in 43s
Integration tests / t (pull_request) Successful in 11m25s
This allows bypassing the 5-second timeline cache in `GetTimeline`
by setting the `AUTOGITS_TIMELINE_CACHE_DISABLE` environment
variable.

This is particularly useful in fast-paced integration tests where
stale data could cause failures. The variable is enabled by
default for the `workflow-pr` service in the integration environment.
2026-03-09 20:37:39 +01:00
19 changed files with 63 additions and 100 deletions

2
.gitignore vendored
View File

@@ -1,7 +1,5 @@
*.osc
*.conf
.*.sw?
/integration/logs/*.log
!/integration/**/*.conf
/integration/gitea-data
/integration/gitea-logs

View File

@@ -1,8 +1,4 @@
MODULES := devel-importer utils/hujson utils/maintainer-update gitea-events-rabbitmq-publisher gitea_status_proxy group-review obs-forward-bot obs-staging-bot obs-status-service workflow-direct workflow-pr
.PHONY: build $(MODULES)
build: $(MODULES)
$(MODULES):
go build -C $@ -buildmode=pie
build:
for m in $(MODULES); do go build -C $$m -buildmode=pie || exit 1 ; done

View File

@@ -857,12 +857,13 @@ func (gitea *GiteaTransport) ResetTimelineCache(org, repo string, idx int64) {
func (gitea *GiteaTransport) GetTimeline(org, repo string, idx int64) ([]*models.TimelineComment, error) {
page := int64(1)
resCount := 1
disableCache := GetEnvOverrideBool(os.Getenv("AUTOGITS_TIMELINE_CACHE_DISABLE"), false)
prID := fmt.Sprintf("%s/%s!%d", org, repo, idx)
giteaTimelineCacheMutex.RLock()
TimelineCache, IsCached := giteaTimelineCache[prID]
var LastCachedTime strfmt.DateTime
if IsCached {
if IsCached && !disableCache {
l := len(TimelineCache.data)
if l > 0 {
LastCachedTime = TimelineCache.data[0].Updated
@@ -879,6 +880,10 @@ func (gitea *GiteaTransport) GetTimeline(org, repo string, idx int64) ([]*models
giteaTimelineCacheMutex.Lock()
defer giteaTimelineCacheMutex.Unlock()
if disableCache {
TimelineCache = TimelineCacheData{}
}
for resCount > 0 {
opts := issue.NewIssueGetCommentsAndTimelineParams().WithOwner(org).WithRepo(repo).WithIndex(idx).WithPage(&page)
if !LastCachedTime.IsZero() {

View File

@@ -5,7 +5,7 @@
# Typical workflow:
# 1. 'make build' - prepares images
# 2. 'make up' - spawns podman-compose
# 3. 'make pytest' - run tests inside the gwf-testclient container
# 3. 'make pytest' - run tests inside the tester container
# 4. 'make down' - once the containers are not needed
#
# OR just run 'make test' to do it all at once.
@@ -33,18 +33,18 @@ wait_healthy:
@echo "Waiting for services to be healthy..."
@echo "Waiting for gitea (max 2m)..."
@start_time=$$(date +%s); \
until podman exec gwf-gitea-test curl -f -s http://localhost:3000/api/v1/version >/dev/null 2>&1; do \
until podman exec gitea-test curl -f -s http://localhost:3000/api/v1/version >/dev/null 2>&1; do \
current_time=$$(date +%s); \
elapsed=$$((current_time - start_time)); \
if [ $$elapsed -gt 120 ]; then \
echo "ERROR: Gitea failed to start within 2 minutes."; \
echo "--- Troubleshooting Info ---"; \
echo "Diagnostics output (curl):"; \
podman exec gwf-gitea-test curl -v http://localhost:3000/api/v1/version || true; \
podman exec gitea-test curl -v http://localhost:3000/api/v1/version || true; \
echo "--- Container Logs ---"; \
podman logs gwf-gitea-test --tail 20; \
podman logs gitea-test --tail 20; \
echo "--- Container Status ---"; \
podman inspect gwf-gitea-test --format '{{.State.Status}}'; \
podman inspect gitea-test --format '{{.State.Status}}'; \
exit 1; \
fi; \
sleep 2; \
@@ -70,7 +70,7 @@ wait_healthy:
@echo "All services are healthy!"
pytest:
podman-compose exec gwf-testclient pytest -v tests
podman-compose exec tester pytest -v tests
build:
podman pull docker.io/library/rabbitmq:3.13.7-management
@@ -91,6 +91,3 @@ up-bots-package:
# mode B
up-bots-local:
GIWTF_IMAGE_SUFFIX=.local podman-compose up -d
clean: down
rm -rf gitea-data/ gitea-logs/ rabbitmq-data/ workflow-pr-repos/ logs/*.log

View File

@@ -10,7 +10,7 @@ This document describes the targets available in the `integration/Makefile`.
1. `build`: Prepares all container images.
2. `up`: Starts all services via `podman-compose`.
3. `wait_healthy`: Polls Gitea and RabbitMQ until they are ready.
4. `pytest`: Executes the test suite inside the `gwf-testclient` container.
4. `pytest`: Executes the test suite inside the `tester` container.
- **Outcome**: The environment remains active for fast iteration.
### `test-ci`
@@ -30,11 +30,11 @@ This document describes the targets available in the `integration/Makefile`.
- **Action**: Starts the container topology in detached mode.
### `wait_healthy`
- **Action**: Polls the health status of `gwf-gitea-test` and `rabbitmq-test` containers.
- **Action**: Polls the health status of `gitea-test` and `rabbitmq-test` containers.
- **Purpose**: Ensures infrastructure is stable before test execution.
### `pytest`
- **Action**: Runs `pytest -v tests/*` inside the running `gwf-testclient` container.
- **Action**: Runs `pytest -v tests/*` inside the running `tester` container.
- **Requirement**: The environment must already be started via `up`.
### `down`

1
integration/clean.sh Executable file
View File

@@ -0,0 +1 @@
sudo rm -rf gitea-data/ gitea-logs/ rabbitmq-data/ workflow-pr-repos/

View File

@@ -7,9 +7,9 @@ STATIC_ROOT_PATH = /usr/share/gitea
APP_DATA_PATH = /var/lib/gitea/data
PPROF_DATA_PATH = /var/lib/gitea/data/tmp/pprof
PROTOCOL = http
DOMAIN = gwf-gitea-test
SSH_DOMAIN = gwf-gitea-test
ROOT_URL = http://gwf-gitea-test:3000/
DOMAIN = gitea-test
SSH_DOMAIN = gitea-test
ROOT_URL = http://gitea-test:3000/
HTTP_PORT = 3000
DISABLE_SSH = false
START_SSH_SERVER = true

View File

@@ -16,4 +16,4 @@ echo "Starting Gitea..."
# We will switch to that user and run the web command.
# Using exec means Gitea will become PID 1, allowing it to receive signals correctly.
cd /var/lib/gitea
exec /usr/bin/gitea web --config /etc/gitea/conf/app.ini
exec su -s /bin/bash gitea -c "/usr/bin/gitea web --config /etc/gitea/conf/app.ini"

View File

@@ -4,16 +4,18 @@ set -e
# Set ownership on the volume mounts. This allows the 'gitea' user to write to them.
# We use -R to ensure all subdirectories (like /var/lib/gitea/data) are covered.
chown -R gitea:gitea /var/lib/gitea /var/log/gitea
# Set ownership on the config directory.
chown -R gitea:gitea /etc/gitea
# Run database migrations to initialize the sqlite3 db based on app.ini.
gitea migrate
su -s /bin/bash gitea -c 'gitea migrate'
# Create a default admin user if it doesn't exist
if ! gitea admin user list | awk 'NR>1 && $2 == "admin" {found=1} END {exit !found}'; then
if ! su -s /bin/bash gitea -c 'gitea admin user list' | awk 'NR>1 && $2 == "admin" {found=1} END {exit !found}'; then
echo "Creating admin user..."
gitea admin user create --username admin --password opensuse --email admin@example.com --must-change-password=false --admin
su -s /bin/bash gitea -c 'gitea admin user create --username admin --password opensuse --email admin@example.com --must-change-password=false --admin'
else
echo "Admin user already exists."
fi
@@ -24,10 +26,11 @@ if [ -f "$ADMIN_TOKEN_FILE" ]; then
echo "Admin token already exists at $ADMIN_TOKEN_FILE."
else
echo "Generating admin token..."
ADMIN_TOKEN=$(gitea admin user generate-access-token -raw -u admin -t admin-token)
ADMIN_TOKEN=$(su -s /bin/bash gitea -c "gitea admin user generate-access-token -raw -u admin -t admin-token")
if [ -n "$ADMIN_TOKEN" ]; then
printf "%s" "$ADMIN_TOKEN" > "$ADMIN_TOKEN_FILE"
chmod 777 "$ADMIN_TOKEN_FILE"
chown gitea:gitea "$ADMIN_TOKEN_FILE"
echo "Admin token generated and saved to $ADMIN_TOKEN_FILE."
else
echo "Failed to generate admin token."
@@ -40,15 +43,16 @@ mkdir -p "$SSH_KEY_DIR"
if [ ! -f "$SSH_KEY_DIR/id_ed25519" ]; then
echo "Generating SSH key for admin user..."
ssh-keygen -t ed25519 -N "" -f "$SSH_KEY_DIR/id_ed25519"
chown -R gitea:gitea "$SSH_KEY_DIR"
chmod 700 "$SSH_KEY_DIR"
chmod 600 "$SSH_KEY_DIR/id_ed25519"
chmod 644 "$SSH_KEY_DIR/id_ed25519.pub"
fi
# Create a autogits_obs_staging_bot user if it doesn't exist
if ! gitea admin user list | awk 'NR>1 && $2 == "autogits_obs_staging_bot" {found=1} END {exit !found}'; then
if ! su -s /bin/bash gitea -c 'gitea admin user list' | awk 'NR>1 && $2 == "autogits_obs_staging_bot" {found=1} END {exit !found}'; then
echo "Creating autogits_obs_staging_bot user..."
gitea admin user create --username autogits_obs_staging_bot --password opensuse --email autogits_obs_staging_bot@example.com --must-change-password=false
su -s /bin/bash gitea -c 'gitea admin user create --username autogits_obs_staging_bot --password opensuse --email autogits_obs_staging_bot@example.com --must-change-password=false'
else
echo "autogits_obs_staging_bot user already exists."
fi
@@ -59,10 +63,11 @@ if [ -f "$BOT_TOKEN_FILE" ]; then
echo "autogits_obs_staging_bot token already exists at $BOT_TOKEN_FILE."
else
echo "Generating autogits_obs_staging_bot token..."
BOT_TOKEN=$(gitea admin user generate-access-token -raw -u autogits_obs_staging_bot -t autogits_obs_staging_bot-token)
BOT_TOKEN=$(su -s /bin/bash gitea -c "gitea admin user generate-access-token -raw -u autogits_obs_staging_bot -t autogits_obs_staging_bot-token")
if [ -n "$BOT_TOKEN" ]; then
printf "%s" "$BOT_TOKEN" > "$BOT_TOKEN_FILE"
chmod 666 "$BOT_TOKEN_FILE"
chown gitea:gitea "$BOT_TOKEN_FILE"
echo "autogits_obs_staging_bot token generated and saved to $BOT_TOKEN_FILE."
else
echo "Failed to generate autogits_obs_staging_bot token."
@@ -70,9 +75,9 @@ else
fi
# Create a workflow-pr user if it doesn't exist
if ! gitea admin user list | awk 'NR>1 && $2 == "workflow-pr" {found=1} END {exit !found}'; then
if ! su -s /bin/bash gitea -c 'gitea admin user list' | awk 'NR>1 && $2 == "workflow-pr" {found=1} END {exit !found}'; then
echo "Creating workflow-pr user..."
gitea admin user create --username workflow-pr --password opensuse --email workflow-pr@example.com --must-change-password=false
su -s /bin/bash gitea -c 'gitea admin user create --username workflow-pr --password opensuse --email workflow-pr@example.com --must-change-password=false'
else
echo "workflow-pr user already exists."
fi
@@ -83,10 +88,11 @@ if [ -f "$BOT_TOKEN_FILE" ]; then
echo "workflow-pr token already exists at $BOT_TOKEN_FILE."
else
echo "Generating workflow-pr token..."
BOT_TOKEN=$(gitea admin user generate-access-token -raw -u workflow-pr -t workflow-pr-token)
BOT_TOKEN=$(su -s /bin/bash gitea -c "gitea admin user generate-access-token -raw -u workflow-pr -t workflow-pr-token")
if [ -n "$BOT_TOKEN" ]; then
printf "%s" "$BOT_TOKEN" > "$BOT_TOKEN_FILE"
chmod 666 "$BOT_TOKEN_FILE"
chown gitea:gitea "$BOT_TOKEN_FILE"
echo "workflow-pr token generated and saved to $BOT_TOKEN_FILE."
else
echo "Failed to generate workflow-pr token."

View File

View File

@@ -1,7 +0,0 @@
#!BuildTag: openbuildservice/gwf-test-basecontainer
#!UseOBSRepositories
FROM registry.suse.com/bci/bci-base:15.7
RUN zypper -n install binutils gawk git git-core git-lfs jq make openssh openssh-clients sqlite3 vim which
RUN zypper -n install autogits-gitea-events-rabbitmq-publisher autogits-obs-staging-bot autogits-workflow-pr gitea

View File

@@ -1,5 +0,0 @@
#!BuildTag: openbuildservice/gwf-client
#!UseOBSRepositories
FROM opensuse/tumbleweed
RUN zypper -n install podman podman-compose vim make python3-pytest python3-requests python3-pytest-dependency python3-pytest-httpserver

View File

@@ -9,7 +9,7 @@ This document describes the services defined in `podman-compose.yml` used for in
### gitea
- **Description**: Self-hosted Git service, serving as the central hub for repositories.
- **Container Name**: `gwf-gitea-test`
- **Container Name**: `gitea-test`
- **Image**: Built from `./gitea/Dockerfile`
- **Ports**: `3000` (HTTP), `3022` (SSH)
- **Volumes**: `./gitea-data` (persistent data), `./gitea-logs` (logs)
@@ -36,20 +36,20 @@ This document describes the services defined in `podman-compose.yml` used for in
- **Environment**: Configured via `AUTOGITS_*` variables.
- **Volumes**: `./gitea-data` (read-only), `./workflow-pr/workflow-pr.json` (config), `./workflow-pr-repos` (working directories)
### gwf-testclient
### tester
- **Description**: The dedicated test runner container. It hosts the `pytest` suite and provides a mock OBS API using `pytest-httpserver`.
- **Container Name**: `gwf-testclient`
- **Image**: Built from `./Dockerfile.gwf-testclient`
- **Container Name**: `tester`
- **Image**: Built from `./Dockerfile.tester`
- **Mock API**: Listens on port `8080` within the container network to simulate OBS.
- **Volumes**: Project root mounted at `/opt/project` for source access.
### obs-staging-bot
- **Description**: Interacts with Gitea and the OBS API (mocked by `gwf-testclient`) to manage staging projects.
- **Description**: Interacts with Gitea and the OBS API (mocked by `tester`) to manage staging projects.
- **Container Name**: `obs-staging-bot`
- **Dependencies**: `gitea` (started), `gwf-testclient` (started)
- **Dependencies**: `gitea` (started), `tester` (started)
- **Environment**:
- `AUTOGITS_STAGING_BOT_POLL_INTERVAL`: Set to `2s` for fast integration testing.
- **Mock Integration**: Points to `http://gwf-testclient:8080` for both OBS API and Web hosts.
- **Mock Integration**: Points to `http://tester:8080` for both OBS API and Web hosts.
---
@@ -58,7 +58,7 @@ This document describes the services defined in `podman-compose.yml` used for in
1. **Build**: `make build` (root) then `make build` (integration).
2. **Up**: `make up` starts all services.
3. **Wait**: `make wait_healthy` ensures infrastructure is ready.
4. **Test**: `make pytest` runs the suite inside the `gwf-testclient` container.
4. **Test**: `make pytest` runs the suite inside the `tester` container.
5. **Down**: `make down` stops and removes containers.
Use `make test` to perform steps 1-4 automatically.

View File

@@ -5,9 +5,9 @@ networks:
driver: bridge
services:
gwf-gitea-test:
gitea:
build: ./gitea
container_name: gwf-gitea-test
container_name: gitea-test
init: true
environment:
- GITEA_WORK_DIR=/var/lib/gitea
@@ -24,12 +24,6 @@ services:
# Persist Gitea's logs to a local directory
- ./gitea-logs:/var/log/gitea:Z
restart: unless-stopped
userns_mode: "keep-id:uid=497,gid=483"
user: "497:497"
logging:
driver: "k8s-file"
options:
path: "logs/gwf-gitea-test.log"
rabbitmq:
image: rabbitmq:3.13.7-management
@@ -57,11 +51,6 @@ services:
# Mount exchange definitions
- ./rabbitmq-config/definitions.json:/etc/rabbitmq/definitions.json:Z
restart: unless-stopped
userns_mode: "keep-id"
logging:
driver: "k8s-file"
options:
path: "logs/rabbitmq-test.log"
gitea-publisher:
build:
@@ -83,10 +72,6 @@ services:
- SSL_CERT_FILE=/usr/share/pki/trust/anchors/gitea-rabbitmq-ca.crt
command: [ "-listen", "0.0.0.0:8002", "-topic-domain", "suse", "-debug" ]
restart: unless-stopped
logging:
driver: "k8s-file"
options:
path: "logs/gitea-publisher.log"
workflow-pr:
build:
@@ -110,6 +95,7 @@ services:
- AMQP_USERNAME=gitea
- AMQP_PASSWORD=gitea
- SSL_CERT_FILE=/usr/share/pki/trust/anchors/gitea-rabbitmq-ca.crt
- AUTOGITS_TIMELINE_CACHE_DISABLE=1
volumes:
- ./gitea-data:/var/lib/gitea:ro,z
- ./workflow-pr/workflow-pr.json:/etc/workflow-pr.json:ro,z
@@ -117,22 +103,18 @@ services:
command: [
"-check-on-start",
"-debug",
"-gitea-url", "http://gwf-gitea-test:3000",
"-gitea-url", "http://gitea-test:3000",
"-url", "amqps://rabbitmq-test:5671",
"-config", "/etc/workflow-pr.json",
"-repo-path", "/var/lib/workflow-pr/repos"
]
restart: unless-stopped
logging:
driver: "k8s-file"
options:
path: "logs/workflow-pr.log"
gwf-testclient:
tester:
build:
context: .
dockerfile: Dockerfile.gwf-testclient
container_name: gwf-testclient
dockerfile: Dockerfile.tester
container_name: tester
init: true
dns_search: .
networks:
@@ -143,10 +125,6 @@ services:
volumes:
- ..:/opt/project:z
command: sleep infinity
logging:
driver: "k8s-file"
options:
path: "logs/gwf-testclient.log"
obs-staging-bot:
build:
@@ -162,16 +140,12 @@ services:
environment:
- OBS_USER=mock
- OBS_PASSWORD=mock-long-password
- AUTOGITS_STAGING_BOT_POLL_INTERVAL=6s
- AUTOGITS_STAGING_BOT_POLL_INTERVAL=2s
volumes:
- ./gitea-data:/gitea-data:ro,z
command:
- "-debug"
- "-gitea-url=http://gwf-gitea-test:3000"
- "-obs=http://gwf-testclient:8080"
- "-obs-web=http://gwf-testclient:8080"
- "-gitea-url=http://gitea-test:3000"
- "-obs=http://tester:8080"
- "-obs-web=http://tester:8080"
restart: unless-stopped
logging:
driver: "k8s-file"
options:
path: "logs/obs-staging-bot.log"

View File

@@ -38,7 +38,7 @@ def default_obs_handlers(httpserver, obs_mock_state):
"""
def project_meta_handler(request):
project = request.path.split("/")[2]
scmsync = obs_mock_state.project_metas.get(project, "http://gwf-gitea-test:3000/myproducts/mySLFO.git")
scmsync = obs_mock_state.project_metas.get(project, "http://gitea-test:3000/myproducts/mySLFO.git")
return f'<project name="{project}"><scmsync>{scmsync}</scmsync></project>'
def build_result_handler(request):
@@ -54,7 +54,7 @@ def default_obs_handlers(httpserver, obs_mock_state):
# or we can use the template. For simplicity, let's use a basic one.
xml_template = f"""<resultlist state="mock">
<result project="{project}" repository="standard" arch="x86_64" code="unpublished" state="unpublished">
<scmsync>http://gwf-gitea-test:3000/myproducts/mySLFO.git?onlybuild={package_name}#sha</scmsync>
<scmsync>http://gitea-test:3000/myproducts/mySLFO.git?onlybuild={package_name}#sha</scmsync>
<status package="{package_name}" code="{code}"/>
</result>
</resultlist>"""
@@ -283,7 +283,7 @@ def gitea_env():
"webhooks": set(),
}
gitea_url = "http://gwf-gitea-test:3000"
gitea_url = "http://gitea-test:3000"
admin_token_path = os.path.join(os.path.dirname(__file__), "..", "gitea-data", "admin.token")
admin_token = None

View File

@@ -2,7 +2,7 @@
<title>openSUSE Leap 16.0 based on SLFO</title>
<description>Leap 16.0 based on SLES 16.0 (specifically SLFO:1.2)</description>
<link project="openSUSE:Backports:SLE-16.0"/>
<scmsync>http://gwf-gitea-test:3000/myproducts/mySLFO#staging-main</scmsync>
<scmsync>http://gitea-test:3000/myproducts/mySLFO#staging-main</scmsync>
<person userid="dimstar_suse" role="maintainer"/>
<person userid="lkocman-factory" role="maintainer"/>
<person userid="maxlin_factory" role="maintainer"/>

View File

@@ -139,7 +139,6 @@ index 00000000..473a0f4c
@pytest.mark.t005
# @pytest.mark.xfail(reason="TBD troubleshoot")
def test_005_any_maintainer_approval_sufficient(maintainer_env, ownerA_client, ownerBB_client):
"""
Test scenario:
@@ -201,7 +200,6 @@ index 00000000..473a0f4c
@pytest.mark.t006
@pytest.mark.xfail(reason="tbd flacky in ci")
def test_006_maintainer_rejection_removes_other_requests(maintainer_env, ownerA_client, ownerBB_client):
"""
Test scenario:

View File

@@ -20,7 +20,7 @@ echo "GITEA_TOKEN exported (length: ${#GITEA_TOKEN})"
# Wait for the dummy data to be created by the gitea setup script
echo "Waiting for workflow.config in myproducts/mySLFO (branch zz-ready-to-start)..."
API_URL="http://gwf-gitea-test:3000/api/v1/repos/myproducts/mySLFO/contents/workflow.config?ref=zz-ready-to-start"
API_URL="http://gitea-test:3000/api/v1/repos/myproducts/mySLFO/contents/workflow.config?ref=zz-ready-to-start"
HTTP_STATUS=$(curl -s -o /dev/null -w "%{http_code}" -H "Authorization: token $GITEA_TOKEN" "$API_URL")
WAITED=false
@@ -52,7 +52,7 @@ chmod 600 /root/.ssh/id_ed25519
echo "Scanning Gitea SSH host key..."
# We try multiple times because Gitea might still be starting its SSH server
for i in {1..10}; do
ssh-keyscan -p 3022 gwf-gitea-test >> /root/.ssh/known_hosts 2>/dev/null && break
ssh-keyscan -p 3022 gitea-test >> /root/.ssh/known_hosts 2>/dev/null && break
echo "Retrying ssh-keyscan in 2s..."
sleep 2
done