ci: convert integration test to multi-cycle benchmarking tool

Update .gitea/workflows/t.yaml to perform four sequential test cycles:
   1. Baseline (original state)
   2. After cherry-picking 572e3311 (CP1)
   3. After cherry-picking 8fa732e6 (CP1 + CP2)
   4. After discarding previous CPs and cherry-picking 58f410be

   Each cycle runs test_005_any_maintainer_approval_sufficient 30 times
   to gather pass/fail statistics for performance and stability comparison.

.gitea/workflows/t.yaml

@@ -30,31 +30,167 @@ jobs:
           git fetch origin ${{ gitea.ref }}
           git checkout FETCH_HEAD
         working-directory: ./autogits
-      - name: Prepare binaries
+      - name: Prepare binaries (baseline)
         run: make build
         working-directory: ./autogits
-      - name: Prepare images
+      - name: Prepare images (baseline)
         run: |
           make build
           podman rmi $(podman images -f "dangling=true" -q)
         working-directory: ./autogits/integration
-      - name: Make sure the pod is down
+      - name: Make sure the pod is down (1)
         run: make down
         working-directory: ./autogits/integration
-      - name: Start images
+      - name: Start images (baseline)
         run: |
           make up
           make wait_healthy
           podman ps
           sleep 5
         working-directory: ./autogits/integration
-      - name: Run tests
-        run: make pytest
+      - name: Run tests 30 times (baseline)
+        run: |
+          pass=0
+          fail=0
+          for i in $(seq 1 30); do
+            echo "Iteration $i/30..."
+            if podman exec -t tester pytest -v tests/workflow_pr_review_test.py::test_005_any_maintainer_approval_sufficient; then
+              pass=$((pass + 1))
+            else
+              fail=$((fail + 1))
+            fi
+          done
+          echo "Summary (baseline): $pass passes, $fail failures"
         working-directory: ./autogits/integration
-      - name: Make sure the pod is down
+      - name: Make sure the pod is down (2)
+        run: |
+          podman ps
+          make down
+        working-directory: ./autogits/integration
+      - name: Cherry-pick 1
+        run: |
+          git config user.email "bot@example.com"
+          git config user.name "Bot"
+          git cherry-pick 572e33111bd72518f33ec4f7c93a7222282f43999afafac948e1e3da5c3453a0
+        working-directory: ./autogits
+      - name: Prepare binaries (after CP1)
+        run: make build
+        working-directory: ./autogits
+      - name: Prepare images (after CP1)
+        run: |
+          make build
+          podman rmi $(podman images -f "dangling=true" -q)
+        working-directory: ./autogits/integration
+      - name: Make sure the pod is down (3)
+        run: make down
+        working-directory: ./autogits/integration
+      - name: Start images (after CP1)
+        run: |
+          make up
+          make wait_healthy
+          podman ps
+          sleep 5
+        working-directory: ./autogits/integration
+      - name: Run tests 30 times (after CP1)
+        run: |
+          pass=0
+          fail=0
+          for i in $(seq 1 30); do
+            echo "Iteration $i/30..."
+            if podman exec -t tester pytest -v tests/workflow_pr_review_test.py::test_005_any_maintainer_approval_sufficient; then
+              pass=$((pass + 1))
+            else
+              fail=$((fail + 1))
+            fi
+          done
+          echo "Summary (after CP1): $pass passes, $fail failures"
+        working-directory: ./autogits/integration
+      - name: Make sure the pod is down (4)
+        run: |
+          podman ps
+          make down
+        working-directory: ./autogits/integration
+      - name: Cherry-pick 2
+        run: |
+          git cherry-pick 8fa732e67518769c9a962e6d12c2e70b38f7bc06e26332fb007ac666fa5e38c1
+        working-directory: ./autogits
+      - name: Prepare binaries (after CP2)
+        run: make build
+        working-directory: ./autogits
+      - name: Prepare images (after CP2)
+        run: |
+          make build
+          podman rmi $(podman images -f "dangling=true" -q)
+        working-directory: ./autogits/integration
+      - name: Make sure the pod is down (5)
+        run: make down
+        working-directory: ./autogits/integration
+      - name: Start images (after CP2)
+        run: |
+          make up
+          make wait_healthy
+          podman ps
+          sleep 5
+        working-directory: ./autogits/integration
+      - name: Run tests 30 times (after CP2)
+        run: |
+          pass=0
+          fail=0
+          for i in $(seq 1 30); do
+            echo "Iteration $i/30..."
+            if podman exec -t tester pytest -v tests/workflow_pr_review_test.py::test_005_any_maintainer_approval_sufficient; then
+              pass=$((pass + 1))
+            else
+              fail=$((fail + 1))
+            fi
+          done
+          echo "Summary (after CP2): $pass passes, $fail failures"
+        working-directory: ./autogits/integration
+      - name: Make sure the pod is down (6)
+        run: |
+          podman ps
+          make down
+        working-directory: ./autogits/integration
+      - name: Discard and Cherry-pick 3
+        run: |
+          git reset --hard FETCH_HEAD
+          git cherry-pick 58f410befce4da40f3ebc27e21ac81a55b6425dd4214e08eb59359d54322a29d
+        working-directory: ./autogits
+      - name: Prepare binaries (after CP3)
+        run: make build
+        working-directory: ./autogits
+      - name: Prepare images (after CP3)
+        run: |
+          make build
+          podman rmi $(podman images -f "dangling=true" -q)
+        working-directory: ./autogits/integration
+      - name: Make sure the pod is down (7)
+        run: make down
+        working-directory: ./autogits/integration
+      - name: Start images (after CP3)
+        run: |
+          make up
+          make wait_healthy
+          podman ps
+          sleep 5
+        working-directory: ./autogits/integration
+      - name: Run tests 30 times (after CP3)
+        run: |
+          pass=0
+          fail=0
+          for i in $(seq 1 30); do
+            echo "Iteration $i/30..."
+            if podman exec -t tester pytest -v tests/workflow_pr_review_test.py::test_005_any_maintainer_approval_sufficient; then
+              pass=$((pass + 1))
+            else
+              fail=$((fail + 1))
+            fi
+          done
+          echo "Summary (after CP3): $pass passes, $fail failures"
+        working-directory: ./autogits/integration
+      - name: Final cleanup
         if: always()
         run: |
           podman ps
           make down
         working-directory: ./autogits/integration
-

Makefile

@@ -1,4 +1,8 @@
 MODULES := devel-importer utils/hujson utils/maintainer-update gitea-events-rabbitmq-publisher gitea_status_proxy group-review obs-forward-bot obs-staging-bot obs-status-service workflow-direct workflow-pr
 
-build:
-	for m in $(MODULES); do go build -C $$m -buildmode=pie || exit 1 ; done
+.PHONY: build $(MODULES)
+
+build: $(MODULES)
+
+$(MODULES):
+	go build -C $@ -buildmode=pie

integration/podman-compose.yml

@@ -100,6 +100,7 @@ services:
       - ./workflow-pr/workflow-pr.json:/etc/workflow-pr.json:ro,z
       - ./workflow-pr-repos:/var/lib/workflow-pr/repos:Z
     command: [ 
+      "-check-on-start",
       "-debug", 
       "-gitea-url", "http://gitea-test:3000", 
       "-url", "amqps://rabbitmq-test:5671", 

integration/tests/lib/common_test_utils.py

@@ -532,20 +532,21 @@ index 00000000..{pkg_b_sha}
         return response.json()
 
     def approve_requested_reviews(self, repo_full_name: str, pr_number: int):
+        vprint(f"--- Checking for REQUEST_REVIEW state in {repo_full_name} PR #{pr_number} ---")
         reviews = self.list_reviews(repo_full_name, pr_number)
         
         requested_reviews = [r for r in reviews if r["state"] == "REQUEST_REVIEW"]
         if not requested_reviews:
+            vprint(f"No reviews in REQUEST_REVIEW state found for {repo_full_name} PR #{pr_number}")
             return
 
-        vprint(f"--- Found {len(requested_reviews)} REQUEST_REVIEW state(s) in {repo_full_name} PR #{pr_number} ---")
         admin_token = self.headers["Authorization"].split(" ")[1]
         for r in requested_reviews:
             reviewer_username = r["user"]["login"]
-            vprint(f"Approving requested review for user {reviewer_username}...")
+            vprint(f"Reacting on REQUEST_REVIEW for user {reviewer_username} by approving...")
             
             reviewer_client = GiteaAPIClient(base_url=self.base_url, token=admin_token, sudo=reviewer_username)
-            time.sleep(0.5) # reduced from 1s
+            time.sleep(1) # give a chance to avoid possible concurrency issues with reviews request/approval
             reviewer_client.create_review(repo_full_name, pr_number, event="APPROVED", body="Approving requested review")
 
     def wait_for_project_pr(self, package_pr_repo, package_pr_number, project_pr_repo="myproducts/mySLFO", timeout=60):
@@ -568,7 +569,7 @@ index 00000000..{pkg_b_sha}
         package_merged = False
         project_merged = False
         
-        for i in range(int(timeout / 0.5)): # Poll with 0.5s interval
+        for i in range(timeout):
             self.approve_requested_reviews(package_pr_repo, package_pr_number)
             self.approve_requested_reviews(project_pr_repo, project_pr_number)
             
@@ -587,5 +588,5 @@ index 00000000..{pkg_b_sha}
             if package_merged and project_merged:
                 return True, True
                 
-            time.sleep(0.5)
+            time.sleep(1)
         return package_merged, project_merged

integration/tests/workflow_pr_merge_test.py

@@ -4,80 +4,6 @@ import time
 from pathlib import Path
 from tests.lib.common_test_utils import GiteaAPIClient
 
-def wait_for_manual_merge_approvals(gitea_env, staging_bot_client, package_pr_number, project_pr_number):
-    """
-    Wait for required approvals on both package and project PRs for manual merge tests.
-    """
-    print("Waiting for required review requests and approving them...")
-    # Expected reviewers based on manual-merge branch config and pkgA maintainership
-    mandatory_reviewers = {"usera", "userb"}
-    maintainers = {"ownerA", "ownerX", "ownerY"}
-    
-    # ManualMergeOnly still requires regular reviews to be satisfied.
-    # We poll until required reviewers have approved.
-    all_approved = False
-    approved_reviewers = set()
-    for _ in range(60): # Poll for up to 60 seconds (1s interval)
-        # Trigger approvals for whatever is already requested
-        gitea_env.approve_requested_reviews("mypool/pkgA", package_pr_number)
-        gitea_env.approve_requested_reviews("myproducts/mySLFO", project_pr_number)
-        
-        # Explicitly handle staging bot if it is requested or pending
-        prj_reviews = gitea_env.list_reviews("myproducts/mySLFO", project_pr_number)
-        if any(r["user"]["login"] == "autogits_obs_staging_bot" and r["state"] in ["REQUEST_REVIEW", "PENDING"] for r in prj_reviews):
-            print("Staging bot has a pending/requested review. Approving...")
-            staging_bot_client.create_review("myproducts/mySLFO", project_pr_number, event="APPROVED", body="Staging bot approves")
-
-        # Check if mandatory reviewers and at least one maintainer have approved
-        pkg_reviews = gitea_env.list_reviews("mypool/pkgA", package_pr_number)
-        approved_reviewers = {r["user"]["login"] for r in pkg_reviews if r["state"] == "APPROVED"}
-        
-        if mandatory_reviewers.issubset(approved_reviewers) and any(m in approved_reviewers for m in maintainers):
-            # And check project PR for bot approval
-            prj_approved = any(r["user"]["login"] == "autogits_obs_staging_bot" and r["state"] == "APPROVED" for r in prj_reviews)
-            if prj_approved:
-                all_approved = True
-                print(f"Required reviewers approved: mandatory={mandatory_reviewers}, maintainer={[m for m in maintainers if m in approved_reviewers]}, staging_bot=True")
-                break
-        
-        pkg_details = gitea_env.get_pr_details("mypool/pkgA", package_pr_number)
-        prj_details = gitea_env.get_pr_details("myproducts/mySLFO", project_pr_number)
-        
-        assert not pkg_details.get("merged"), "Package PR merged prematurely (ManualMergeOnly ignored?)"
-        assert not prj_details.get("merged"), "Project PR merged prematurely (ManualMergeOnly ignored?)"
-        
-        time.sleep(1)
-
-    assert all_approved, f"Timed out waiting for required approvals. Mandatory: {mandatory_reviewers}, Maintainers: {maintainers}. Current approved: {approved_reviewers}"
-    print("Both PRs have all required approvals but are not merged (as expected with ManualMergeOnly).")
-
-def wait_for_merge_status(gitea_env, package_pr_number, project_pr_number, timeout_seconds=20):
-    """
-    Poll for PR merge status on both package and project PRs.
-    """
-    print("Polling for PR merge status...")
-    package_merged = False
-    project_merged = False
-    
-    for i in range(int(timeout_seconds / 0.5)): # Poll with 0.5s interval
-        if not package_merged:
-            pkg_details = gitea_env.get_pr_details("mypool/pkgA", package_pr_number)
-            if pkg_details.get("merged"):
-                package_merged = True
-                print(f"Package PR mypool/pkgA#{package_pr_number} merged.")
-        
-        if not project_merged:
-            prj_details = gitea_env.get_pr_details("myproducts/mySLFO", project_pr_number)
-            if prj_details.get("merged"):
-                project_merged = True
-                print(f"Project PR myproducts/mySLFO#{project_pr_number} merged.")
-        
-        if package_merged and project_merged:
-            break
-            
-        time.sleep(0.5)
-    return package_merged, project_merged
-
 @pytest.mark.t001
 def test_001_automerge(automerge_env, test_user_client):
     """
@@ -136,14 +62,74 @@ index 00000000..473a0f4c
     print(f"Found project PR: myproducts/mySLFO#{project_pr_number}")
 
     # 3. Approve reviews and verify NOT merged
-    wait_for_manual_merge_approvals(gitea_env, staging_bot_client, package_pr_number, project_pr_number)
+    print("Waiting for required review requests and approving them...")
+    # Expected reviewers based on manual-merge branch config and pkgA maintainership
+    mandatory_reviewers = {"usera", "userb"}
+    maintainers = {"ownerA", "ownerX", "ownerY"}
+    
+    # ManualMergeOnly still requires regular reviews to be satisfied.
+    # We poll until required reviewers have approved.
+    all_approved = False
+    for _ in range(30):
+        # Trigger approvals for whatever is already requested
+        gitea_env.approve_requested_reviews("mypool/pkgA", package_pr_number)
+        gitea_env.approve_requested_reviews("myproducts/mySLFO", project_pr_number)
+        
+        # Explicitly handle staging bot if it is requested or pending
+        prj_reviews = gitea_env.list_reviews("myproducts/mySLFO", project_pr_number)
+        if any(r["user"]["login"] == "autogits_obs_staging_bot" and r["state"] in ["REQUEST_REVIEW", "PENDING"] for r in prj_reviews):
+            print("Staging bot has a pending/requested review. Approving...")
+            staging_bot_client.create_review("myproducts/mySLFO", project_pr_number, event="APPROVED", body="Staging bot approves")
+
+        # Check if mandatory reviewers and at least one maintainer have approved
+        pkg_reviews = gitea_env.list_reviews("mypool/pkgA", package_pr_number)
+        approved_reviewers = {r["user"]["login"] for r in pkg_reviews if r["state"] == "APPROVED"}
+        
+        if mandatory_reviewers.issubset(approved_reviewers) and any(m in approved_reviewers for m in maintainers):
+            # And check project PR for bot approval
+            prj_approved = any(r["user"]["login"] == "autogits_obs_staging_bot" and r["state"] == "APPROVED" for r in prj_reviews)
+            if prj_approved:
+                all_approved = True
+                print(f"Required reviewers approved: mandatory={mandatory_reviewers}, maintainer={[m for m in maintainers if m in approved_reviewers]}, staging_bot=True")
+                break
+        
+        pkg_details = gitea_env.get_pr_details("mypool/pkgA", package_pr_number)
+        prj_details = gitea_env.get_pr_details("myproducts/mySLFO", project_pr_number)
+        
+        assert not pkg_details.get("merged"), "Package PR merged prematurely (ManualMergeOnly ignored?)"
+        assert not prj_details.get("merged"), "Project PR merged prematurely (ManualMergeOnly ignored?)"
+        
+        time.sleep(2)
+
+    assert all_approved, f"Timed out waiting for required approvals. Mandatory: {mandatory_reviewers}, Maintainers: {maintainers}. Current approved: {approved_reviewers}"
+    print("Both PRs have all required approvals but are not merged (as expected with ManualMergeOnly).")
 
     # 4. Comment "merge ok" from a requested reviewer (ownerA)
     print("Commenting 'merge ok' on package PR from a maintainer...")
     ownerA_client.create_issue_comment("mypool/pkgA", package_pr_number, "merge ok")
 
     # 5. Verify both PRs are merged
-    package_merged, project_merged = wait_for_merge_status(gitea_env, package_pr_number, project_pr_number)
+    print("Polling for PR merge status...")
+    package_merged = False
+    project_merged = False
+    
+    for i in range(20): # Poll for up to 20 seconds
+        if not package_merged:
+            pkg_details = gitea_env.get_pr_details("mypool/pkgA", package_pr_number)
+            if pkg_details.get("merged"):
+                package_merged = True
+                print(f"Package PR mypool/pkgA#{package_pr_number} merged.")
+        
+        if not project_merged:
+            prj_details = gitea_env.get_pr_details("myproducts/mySLFO", project_pr_number)
+            if prj_details.get("merged"):
+                project_merged = True
+                print(f"Project PR myproducts/mySLFO#{project_pr_number} merged.")
+        
+        if package_merged and project_merged:
+            break
+            
+        time.sleep(1)
             
     assert package_merged, f"Package PR mypool/pkgA#{package_pr_number} was not merged after 'merge ok'."
     assert project_merged, f"Project PR myproducts/mySLFO#{project_pr_number} was not merged after 'merge ok'."
@@ -176,14 +162,74 @@ index 00000000..473a0f4c
     print(f"Found project PR: myproducts/mySLFO#{project_pr_number}")
 
     # 3. Approve reviews and verify NOT merged
-    wait_for_manual_merge_approvals(gitea_env, staging_bot_client, package_pr_number, project_pr_number)
+    print("Waiting for required review requests and approving them...")
+    # Expected reviewers based on manual-merge branch config and pkgA maintainership
+    mandatory_reviewers = {"usera", "userb"}
+    maintainers = {"ownerA", "ownerX", "ownerY"}
+    
+    # ManualMergeOnly still requires regular reviews to be satisfied.
+    # We poll until required reviewers have approved.
+    all_approved = False
+    for _ in range(30):
+        # Trigger approvals for whatever is already requested
+        gitea_env.approve_requested_reviews("mypool/pkgA", package_pr_number)
+        gitea_env.approve_requested_reviews("myproducts/mySLFO", project_pr_number)
+        
+        # Explicitly handle staging bot if it is requested or pending
+        prj_reviews = gitea_env.list_reviews("myproducts/mySLFO", project_pr_number)
+        if any(r["user"]["login"] == "autogits_obs_staging_bot" and r["state"] in ["REQUEST_REVIEW", "PENDING"] for r in prj_reviews):
+            print("Staging bot has a pending/requested review. Approving...")
+            staging_bot_client.create_review("myproducts/mySLFO", project_pr_number, event="APPROVED", body="Staging bot approves")
+
+        # Check if mandatory reviewers and at least one maintainer have approved
+        pkg_reviews = gitea_env.list_reviews("mypool/pkgA", package_pr_number)
+        approved_reviewers = {r["user"]["login"] for r in pkg_reviews if r["state"] == "APPROVED"}
+        
+        if mandatory_reviewers.issubset(approved_reviewers) and any(m in approved_reviewers for m in maintainers):
+            # And check project PR for bot approval
+            prj_approved = any(r["user"]["login"] == "autogits_obs_staging_bot" and r["state"] == "APPROVED" for r in prj_reviews)
+            if prj_approved:
+                all_approved = True
+                print(f"Required reviewers approved: mandatory={mandatory_reviewers}, maintainer={[m for m in maintainers if m in approved_reviewers]}, staging_bot=True")
+                break
+        
+        pkg_details = gitea_env.get_pr_details("mypool/pkgA", package_pr_number)
+        prj_details = gitea_env.get_pr_details("myproducts/mySLFO", project_pr_number)
+        
+        assert not pkg_details.get("merged"), "Package PR merged prematurely (ManualMergeOnly ignored?)"
+        assert not prj_details.get("merged"), "Project PR merged prematurely (ManualMergeOnly ignored?)"
+        
+        time.sleep(2)
+
+    assert all_approved, f"Timed out waiting for required approvals. Mandatory: {mandatory_reviewers}, Maintainers: {maintainers}. Current approved: {approved_reviewers}"
+    print("Both PRs have all required approvals but are not merged (as expected with ManualMergeOnly).")
 
     # 4. Comment "merge ok" from a requested reviewer (ownerB)
     print("Commenting 'merge ok' on package PR as user ownerB ...")
     ownerB_client.create_issue_comment("mypool/pkgA", package_pr_number, "merge ok")
 
     # 5. Verify both PRs are merged
-    package_merged, project_merged = wait_for_merge_status(gitea_env, package_pr_number, project_pr_number)
+    print("Polling for PR merge status...")
+    package_merged = False
+    project_merged = False
+    
+    for i in range(20): # Poll for up to 20 seconds
+        if not package_merged:
+            pkg_details = gitea_env.get_pr_details("mypool/pkgA", package_pr_number)
+            if pkg_details.get("merged"):
+                package_merged = True
+                print(f"Package PR mypool/pkgA#{package_pr_number} merged.")
+        
+        if not project_merged:
+            prj_details = gitea_env.get_pr_details("myproducts/mySLFO", project_pr_number)
+            if prj_details.get("merged"):
+                project_merged = True
+                print(f"Project PR myproducts/mySLFO#{project_pr_number} merged.")
+        
+        if package_merged and project_merged:
+            break
+            
+        time.sleep(1)
             
     assert not package_merged, f"Package PR mypool/pkgA#{package_pr_number} was merged after 'merge ok'."
     assert not project_merged, f"Project PR myproducts/mySLFO#{project_pr_number} was merged after 'merge ok'."
@@ -254,10 +300,10 @@ index 00000000..473a0f4c
     
     # Approve again and verify it is NOT merged
     print("Approving again and verifying PR is NOT merged (because it's not FF)...")
-    for i in range(30): # Poll with 0.5s interval
+    for i in range(15):
         gitea_env.approve_requested_reviews("mypool/pkgA", package_pr_number)
         gitea_env.approve_requested_reviews("myproducts/mySLFO", project_pr_number)
-        time.sleep(0.5)
+        time.sleep(1)
         
     pkg_details = gitea_env.get_pr_details("mypool/pkgA", package_pr_number)
     assert not pkg_details.get("merged"), "Package PR merged despite NOT being FF-mergeable!"