t: add staging-bot br tests

- Replace shell loop in build with explicit module targets
- Add .PHONY for build and module targets
- Keep go build -C <module> -buildmode=pie behavior unchanged
- Enable parallel builds via make -jN build

Makefile

@@ -1,4 +1,8 @@
 MODULES := devel-importer utils/hujson utils/maintainer-update gitea-events-rabbitmq-publisher gitea_status_proxy group-review obs-forward-bot obs-staging-bot obs-status-service workflow-direct workflow-pr
 
-build:
-	for m in $(MODULES); do go build -C $$m -buildmode=pie || exit 1 ; done
+.PHONY: build $(MODULES)
+
+build: $(MODULES)
+
+$(MODULES):
+	go build -C $@ -buildmode=pie

integration/podman-compose.yml

@@ -100,6 +100,7 @@ services:
       - ./workflow-pr/workflow-pr.json:/etc/workflow-pr.json:ro,z
       - ./workflow-pr-repos:/var/lib/workflow-pr/repos:Z
     command: [ 
+      "-check-on-start",
       "-debug", 
       "-gitea-url", "http://gitea-test:3000", 
       "-url", "amqps://rabbitmq-test:5671", 

integration/tests/lib/common_test_utils.py

@@ -65,6 +65,12 @@ class GiteaAPIClient:
                 return None
             raise
 
+    def get_submodule_sha(self, owner: str, repo: str, submodule_path: str, ref: str = "main"):
+        info = self.get_file_info(owner, repo, submodule_path, branch=ref)
+        if info and info.get("type") == "submodule":
+            return info.get("sha")
+        return None
+
     def create_user(self, username, password, email):
         vprint(f"--- Creating user: {username} ---")
         data = {
@@ -525,6 +531,14 @@ index 00000000..{pkg_b_sha}
 
         return review
 
+    def request_reviewers(self, repo_full_name: str, pr_number: int, reviewers: list):
+        owner, repo = repo_full_name.split("/")
+        url = f"repos/{owner}/{repo}/pulls/{pr_number}/requested_reviewers"
+        data = {"reviewers": reviewers}
+        vprint(f"--- Requesting reviewers for {repo_full_name} PR #{pr_number}: {reviewers} ---")
+        response, duration = self._request("POST", url, json=data)
+        return response.json()
+
     def list_reviews(self, repo_full_name: str, pr_number: int):
         owner, repo = repo_full_name.split("/")
         url = f"repos/{owner}/{repo}/pulls/{pr_number}/reviews"
@@ -532,20 +546,21 @@ index 00000000..{pkg_b_sha}
         return response.json()
 
     def approve_requested_reviews(self, repo_full_name: str, pr_number: int):
+        vprint(f"--- Checking for REQUEST_REVIEW state in {repo_full_name} PR #{pr_number} ---")
         reviews = self.list_reviews(repo_full_name, pr_number)
         
         requested_reviews = [r for r in reviews if r["state"] == "REQUEST_REVIEW"]
         if not requested_reviews:
+            vprint(f"No reviews in REQUEST_REVIEW state found for {repo_full_name} PR #{pr_number}")
             return
 
-        vprint(f"--- Found {len(requested_reviews)} REQUEST_REVIEW state(s) in {repo_full_name} PR #{pr_number} ---")
         admin_token = self.headers["Authorization"].split(" ")[1]
         for r in requested_reviews:
             reviewer_username = r["user"]["login"]
-            vprint(f"Approving requested review for user {reviewer_username}...")
+            vprint(f"Reacting on REQUEST_REVIEW for user {reviewer_username} by approving...")
             
             reviewer_client = GiteaAPIClient(base_url=self.base_url, token=admin_token, sudo=reviewer_username)
-            time.sleep(0.5) # reduced from 1s
+            time.sleep(1) # give a chance to avoid possible concurrency issues with reviews request/approval
             reviewer_client.create_review(repo_full_name, pr_number, event="APPROVED", body="Approving requested review")
 
     def wait_for_project_pr(self, package_pr_repo, package_pr_number, project_pr_repo="myproducts/mySLFO", timeout=60):
@@ -568,7 +583,7 @@ index 00000000..{pkg_b_sha}
         package_merged = False
         project_merged = False
         
-        for i in range(int(timeout / 0.5)): # Poll with 0.5s interval
+        for i in range(timeout):
             self.approve_requested_reviews(package_pr_repo, package_pr_number)
             self.approve_requested_reviews(project_pr_repo, project_pr_number)
             
@@ -587,5 +602,5 @@ index 00000000..{pkg_b_sha}
             if package_merged and project_merged:
                 return True, True
                 
-            time.sleep(0.5)
+            time.sleep(1)
         return package_merged, project_merged

integration/tests/staging_br_test.py

@@ -0,0 +1,521 @@
+import pytest
+import time
+import re
+from tests.lib.common_test_utils import GiteaAPIClient, vprint
+
+# Shared methods
+
+def wait_for_staging_bot_reviewer(gitea_env, prj_pr_number, timeout=30):
+    """Wait for staging-bot to be added as a reviewer."""
+    for _ in range(timeout):
+        time.sleep(1)
+        pr_details = gitea_env.get_pr_details("myproducts/mySLFO", prj_pr_number)
+        if any(
+            r.get("login") == "autogits_obs_staging_bot"
+            for r in pr_details.get("requested_reviewers", [])
+        ):
+            return True
+    return False
+
+def wait_for_comment(gitea_env, repo_full_name, pr_number, text_to_find, subtext=None, timeout=60):
+    """Wait for a specific comment in the PR timeline."""
+    vprint(f"Waiting for comment '{text_to_find}'" + (f" with subtext '{subtext}'" if subtext else "") + f" in {repo_full_name} PR #{pr_number}...")
+    for _ in range(timeout):
+        time.sleep(1)
+        try:
+            events = gitea_env.get_timeline_events(repo_full_name, pr_number)
+        except Exception:
+            continue
+        for event in events:
+            body = event.get("body", "")
+            if not body:
+                continue
+            if subtext:
+                if text_to_find in body and subtext in body:
+                    return True
+            else:
+                if text_to_find == body.strip():
+                    return True
+    return False
+
+def setup_obs_mock(httpserver, project_name, build_result_handler):
+    """Setup OBS mock handlers."""
+    def general_project_meta_handler(request):
+        project = request.path.split("/")[2]
+        return f'<project name="{project}"><scmsync>http://gitea-test:3000/myproducts/mySLFO.git</scmsync></project>'
+
+    httpserver.clear()
+    httpserver.expect_request(re.compile(r"/source/[^/]+/_meta$"), method="GET").respond_with_handler(general_project_meta_handler)
+    httpserver.expect_request(re.compile(f"/build/{project_name}/_result"), method="GET").respond_with_handler(build_result_handler)
+    httpserver.expect_request(re.compile(r"/source/[^/]+/_meta$"), method="PUT").respond_with_data("OK")
+    httpserver.expect_request(re.compile(r"/source/[^/]+$"), method="DELETE").respond_with_data("OK")
+
+def create_build_result_xml(project_name, repo_arch_package_status):
+    """Create the XML response for OBS build results."""
+    results_xml = ""
+    # Group statuses by (repo, arch)
+    grouped = {}
+    for (repo, arch, package), (repo_code, pkg_code) in repo_arch_package_status.items():
+        if (repo, arch) not in grouped:
+            grouped[(repo, arch)] = {"repo_code": repo_code, "packages": []}
+        grouped[(repo, arch)]["packages"].append((package, pkg_code))
+    
+    for (repo, arch), data in grouped.items():
+        repo_code = data["repo_code"]
+        pkg_statuses = "".join([f'\n    <status package="{p}" code="{c}"/>' for p, c in data["packages"]])
+        results_xml += f"""
+  <result project="{project_name}" repository="{repo}" arch="{arch}" code="{repo_code}" state="{repo_code}">{pkg_statuses}
+  </result>"""
+    return f'<resultlist state="mock">{results_xml}\n</resultlist>'
+
+def create_submodule_diff(submodule_name, old_sha, new_sha):
+    """Create a git diff string for updating a submodule commit ID."""
+    return f"""diff --git a/{submodule_name} b/{submodule_name}
+index {old_sha[:7]}..{new_sha[:7]} 160000
+--- a/{submodule_name}
++++ b/{submodule_name}
+@@ -1 +1 @@
+-Subproject commit {old_sha}
++Subproject commit {new_sha}
+"""
+
+@pytest.mark.t001
+def test_001_build_on_all_archs_success(staging_main_env, httpserver):
+    gitea_env, test_full_repo_name, merge_branch_name = staging_main_env
+    
+    # 1. Create a package PR.
+    diff = "diff --git a/test_br.txt b/test_br.txt\nnew file mode 100644\nindex 00000000..473a0f4c\n"
+    pr = gitea_env.create_gitea_pr("mypool/pkgA", diff, "Test BR PR", False, base_branch=merge_branch_name)
+    pkg_pr_number = pr["number"]
+
+    # 2. Wait for the workflow-pr bot to create the related project PR.
+    prj_pr_number = gitea_env.wait_for_project_pr("mypool/pkgA", pkg_pr_number)
+    assert prj_pr_number is not None, "Workflow bot did not create a project PR."
+
+    # 3. Wait for staging-bot to be added as a reviewer.
+    assert wait_for_staging_bot_reviewer(gitea_env, prj_pr_number), "Staging bot was not added as a reviewer."
+
+    # 4. Mock the OBS result list.
+    project_name = f"openSUSE:Leap:16.0:PullRequest:{prj_pr_number}"
+    repos = ["repo1", "repo2"]
+    archs = ["x86_64", "aarch64"]
+    repo_arch_package_status = {}
+    for r in repos:
+        for a in archs:
+            repo_arch_package_status[(r, a, "pkgA")] = ("building", "building")
+
+    def build_result_handler(request):
+        return create_build_result_xml(project_name, repo_arch_package_status)
+
+    setup_obs_mock(httpserver, project_name, build_result_handler)
+
+    # Expected Result 1: "Build is started in..."
+    assert wait_for_comment(gitea_env, "myproducts/mySLFO", prj_pr_number, "Build is started in", project_name), "Staging bot did not post 'Build is started' comment."
+
+    # 6. Transition all to success.
+    for r in repos:
+        for a in archs:
+            repo_arch_package_status[(r, a, "pkgA")] = ("published", "succeeded")
+
+    # Expected Result 2: "Build successful" on project PR.
+    assert wait_for_comment(gitea_env, "myproducts/mySLFO", prj_pr_number, "Build successful"), "Staging bot did not post 'Build successful' comment on project PR."
+
+    # Expected Result 3: "Build successful..." on package PR.
+    assert wait_for_comment(gitea_env, "mypool/pkgA", pkg_pr_number, "Build successful, for more information go in", project_name), "Staging bot did not post 'Build successful' comment on package PR."
+
+@pytest.mark.t002
+def test_002_build_on_all_archs_mix(staging_main_env, httpserver):
+    gitea_env, test_full_repo_name, merge_branch_name = staging_main_env
+    
+    # 1. Create a package PR.
+    diff = "diff --git a/test_br_mix.txt b/test_br_mix.txt\nnew file mode 100644\nindex 00000000..473a0f4c\n"
+    pr = gitea_env.create_gitea_pr("mypool/pkgA", diff, "Test BR Mix PR", False, base_branch=merge_branch_name)
+    pkg_pr_number = pr["number"]
+
+    # 2. Wait for the workflow-pr bot to create the related project PR.
+    prj_pr_number = gitea_env.wait_for_project_pr("mypool/pkgA", pkg_pr_number)
+    assert prj_pr_number is not None, "Workflow bot did not create a project PR."
+
+    # 3. Wait for staging-bot to be added as a reviewer.
+    assert wait_for_staging_bot_reviewer(gitea_env, prj_pr_number), "Staging bot was not added as a reviewer."
+
+    # 4. Mock the OBS result list with 2 repositories and 2 architectures.
+    project_name = f"openSUSE:Leap:16.0:PullRequest:{prj_pr_number}"
+    repos = ["repo1", "repo2"]
+    archs = ["x86_64", "aarch64"]
+    repo_arch_package_status = {}
+    
+    # 5. Set all repository, architecture, and package statuses to "in progress" (e.g., code="building").
+    for r in repos:
+        for a in archs:
+            repo_arch_package_status[(r, a, "pkgA")] = ("building", "building")
+
+    def build_result_handler(request):
+        return create_build_result_xml(project_name, repo_arch_package_status)
+
+    setup_obs_mock(httpserver, project_name, build_result_handler)
+
+    # Expected Result 1: "Build is started in..."
+    assert wait_for_comment(gitea_env, "myproducts/mySLFO", prj_pr_number, "Build is started in", project_name), "Staging bot did not post 'Build is started' comment."
+
+    # 6. Transition the first repository (all architectures) to "finished" mode and set its package statuses to "success" (e.g., code="succeeded").
+    for a in archs:
+        repo_arch_package_status[("repo1", a, "pkgA")] = ("published", "succeeded")
+
+    # 7. Transition the second repository (all architectures) to "finished" mode and set its package statuses to "failed" (e.g., code="failed").
+    for a in archs:
+        repo_arch_package_status[("repo2", a, "pkgA")] = ("finished", "failed")
+
+    # Expected Result 2: "Build failed" on project PR.
+    assert wait_for_comment(gitea_env, "myproducts/mySLFO", prj_pr_number, "Build failed"), "Staging bot did not post 'Build failed' comment on project PR."
+
+    # Expected Result 3: "Build failed, for more information go in..." on package PR.
+    assert wait_for_comment(gitea_env, "mypool/pkgA", pkg_pr_number, "Build failed, for more information go in", project_name), "Staging bot did not post 'Build failed' comment on package PR."
+
+@pytest.mark.t003
+def test_003_build_on_some_archs_failed(staging_main_env, httpserver):
+    gitea_env, test_full_repo_name, merge_branch_name = staging_main_env
+    
+    # 1. Create a package PR.
+    diff = "diff --git a/test_br_some_fail.txt b/test_br_some_fail.txt\nnew file mode 100644\nindex 00000000..473a0f4c\n"
+    pr = gitea_env.create_gitea_pr("mypool/pkgA", diff, "Test BR Some Fail PR", False, base_branch=merge_branch_name)
+    pkg_pr_number = pr["number"]
+
+    # 2. Wait for the workflow-pr bot to create the related project PR.
+    prj_pr_number = gitea_env.wait_for_project_pr("mypool/pkgA", pkg_pr_number)
+    assert prj_pr_number is not None, "Workflow bot did not create a project PR."
+
+    # 3. Wait for staging-bot to be added as a reviewer.
+    assert wait_for_staging_bot_reviewer(gitea_env, prj_pr_number), "Staging bot was not added as a reviewer."
+
+    # 4. Mock the OBS result list with 2 repositories and 2 architectures.
+    project_name = f"openSUSE:Leap:16.0:PullRequest:{prj_pr_number}"
+    repos = ["repo1", "repo2"]
+    archs = ["x86_64", "aarch64"]
+    repo_arch_package_status = {}
+    
+    # 5. Set all repository, architecture, and package statuses to "in progress" (e.g., code="building").
+    for r in repos:
+        for a in archs:
+            repo_arch_package_status[(r, a, "pkgA")] = ("building", "building")
+
+    def build_result_handler(request):
+        return create_build_result_xml(project_name, repo_arch_package_status)
+
+    setup_obs_mock(httpserver, project_name, build_result_handler)
+
+    # Expected Result 1: "Build is started in..."
+    assert wait_for_comment(gitea_env, "myproducts/mySLFO", prj_pr_number, "Build is started in", project_name), "Staging bot did not post 'Build is started' comment."
+
+    # 6. Transition the first repository (all architectures) to "finished" mode and set its package statuses to "success" (e.g., code="succeeded").
+    for a in archs:
+        repo_arch_package_status[("repo1", a, "pkgA")] = ("published", "succeeded")
+
+    # 7. Transition the second repository to "finished" mode, setting the first architecture to "failed" (code="failed") and the second architecture to "success" (code="succeeded").
+    repo_arch_package_status[("repo2", archs[0], "pkgA")] = ("finished", "failed")
+    repo_arch_package_status[("repo2", archs[1], "pkgA")] = ("finished", "succeeded")
+
+    # Expected Result 2: "Build failed" on project PR.
+    assert wait_for_comment(gitea_env, "myproducts/mySLFO", prj_pr_number, "Build failed"), "Staging bot did not post 'Build failed' comment on project PR."
+
+    # Expected Result 3: "Build failed, for more information go in..." on package PR.
+    assert wait_for_comment(gitea_env, "mypool/pkgA", pkg_pr_number, "Build failed, for more information go in", project_name), "Staging bot did not post 'Build failed' comment on package PR."
+
+@pytest.mark.t004
+def test_004_build_multiple_packages_success(staging_main_env, httpserver):
+    gitea_env, test_full_repo_name, merge_branch_name = staging_main_env
+    
+    # 1. Create package PRs.
+    diff = "diff --git a/test_br_multi_pkgA.txt b/test_br_multi_pkgA.txt\nnew file mode 100644\nindex 00000000..473a0f4c\n"
+    pr1 = gitea_env.create_gitea_pr("mypool/pkgA", diff, "Test BR Multi PR A", False, base_branch=merge_branch_name)
+    pkg_pr_number1 = pr1["number"]
+    pkg_head_sha1 = pr1["head"]["sha"]
+
+    diff2 = "diff --git a/test_br_multi_pkgB.txt b/test_br_multi_pkgB.txt\nnew file mode 100644\nindex 00000000..473a0f4c\n"
+    pr2 = gitea_env.create_gitea_pr("mypool/pkgB", diff2, "Test BR Multi PR B", False, base_branch=merge_branch_name)
+    pkg_pr_number2 = pr2["number"]
+    pkg_head_sha2 = pr2["head"]["sha"]
+
+    # 2. Create a project PR mentioning both packages in description and UPDATING SUBMODULES.
+    old_sha1 = gitea_env.get_submodule_sha("myproducts", "mySLFO", "pkgA", ref=merge_branch_name)
+    old_sha2 = gitea_env.get_submodule_sha("myproducts", "mySLFO", "pkgB", ref=merge_branch_name)
+    
+    prj_diff = create_submodule_diff("pkgA", old_sha1, pkg_head_sha1)
+    prj_diff += create_submodule_diff("pkgB", old_sha2, pkg_head_sha2)
+
+    body = f"PR: mypool/pkgA!{pkg_pr_number1}\nPR: mypool/pkgB!{pkg_pr_number2}"
+    prj_pr = gitea_env.create_gitea_pr("myproducts/mySLFO", prj_diff, "Test Project PR Multi", False, base_branch=merge_branch_name, body=body)
+    prj_pr_number = prj_pr["number"]
+
+    # 3. Add staging_bot as a reviewer.
+    gitea_env.request_reviewers("myproducts/mySLFO", prj_pr_number, ["autogits_obs_staging_bot"])
+
+    # Wait for staging-bot to be added (verification)
+    assert wait_for_staging_bot_reviewer(gitea_env, prj_pr_number), "Staging bot was not added as a reviewer."
+
+    # 4. Mock the OBS result list with 2 repositories and 2 architectures for each package.
+    project_name = f"openSUSE:Leap:16.0:PullRequest:{prj_pr_number}"
+    repos = ["repo1", "repo2"]
+    archs = ["x86_64", "aarch64"]
+    packages = ["pkgA", "pkgB"]
+    repo_arch_package_status = {}
+    
+    # 5. Set all repository, architecture, and package statuses to "in progress" (e.g., code="building").
+    for r in repos:
+        for a in archs:
+            for p in packages:
+                repo_arch_package_status[(r, a, p)] = ("building", "building")
+
+    def build_result_handler(request):
+        return create_build_result_xml(project_name, repo_arch_package_status)
+
+    setup_obs_mock(httpserver, project_name, build_result_handler)
+
+    # Expected Result 1: "Build is started in..."
+    assert wait_for_comment(gitea_env, "myproducts/mySLFO", prj_pr_number, "Build is started in", project_name), "Staging bot did not post 'Build is started' comment."
+
+    # 6. Transition all repositories and architectures for the first package to "finished" mode and set its package statuses to "success" (e.g., code="succeeded").
+    for r in repos:
+        for a in archs:
+            repo_arch_package_status[(r, a, "pkgA")] = ("published", "succeeded")
+
+    # Verify "Build successful" is NOT yet posted
+    time.sleep(5)
+    events = gitea_env.get_timeline_events("myproducts/mySLFO", prj_pr_number)
+    assert not any(e.get("body") == "Build successful" for e in events), "Build successful posted prematurely."
+
+    # 7. Transition all repositories and architectures for the second package to "finished" mode and set its package statuses to "success" (e.g., code="succeeded").
+    for r in repos:
+        for a in archs:
+            repo_arch_package_status[(r, a, "pkgB")] = ("published", "succeeded")
+
+    # Expected Result 2: "Build successful" on project PR.
+    assert wait_for_comment(gitea_env, "myproducts/mySLFO", prj_pr_number, "Build successful"), "Staging bot did not post 'Build successful' comment on project PR."
+
+    # Expected Result 3: "Build successful..." on each package PR.
+    assert wait_for_comment(gitea_env, "mypool/pkgA", pkg_pr_number1, "Build successful, for more information go in", project_name), "Staging bot did not post 'Build successful' comment on package PR A."
+    assert wait_for_comment(gitea_env, "mypool/pkgB", pkg_pr_number2, "Build successful, for more information go in", project_name), "Staging bot did not post 'Build successful' comment on package PR B."
+
+@pytest.mark.t005
+def test_005_build_multiple_packages_mix(staging_main_env, httpserver):
+    gitea_env, test_full_repo_name, merge_branch_name = staging_main_env
+    
+    # 1. Create package PRs.
+    diff = "diff --git a/test_br_mix_multi_pkgA.txt b/test_br_mix_multi_pkgA.txt\nnew file mode 100644\nindex 00000000..473a0f4c\n"
+    pr1 = gitea_env.create_gitea_pr("mypool/pkgA", diff, "Test BR Mix Multi PR A", False, base_branch=merge_branch_name)
+    pkg_pr_number1 = pr1["number"]
+    pkg_head_sha1 = pr1["head"]["sha"]
+
+    diff2 = "diff --git a/test_br_mix_multi_pkgB.txt b/test_br_mix_multi_pkgB.txt\nnew file mode 100644\nindex 00000000..473a0f4c\n"
+    pr2 = gitea_env.create_gitea_pr("mypool/pkgB", diff2, "Test BR Mix Multi PR B", False, base_branch=merge_branch_name)
+    pkg_pr_number2 = pr2["number"]
+    pkg_head_sha2 = pr2["head"]["sha"]
+
+    # 2. Create a project PR mentioning both packages in description and UPDATING SUBMODULES.
+    old_sha1 = gitea_env.get_submodule_sha("myproducts", "mySLFO", "pkgA", ref=merge_branch_name)
+    old_sha2 = gitea_env.get_submodule_sha("myproducts", "mySLFO", "pkgB", ref=merge_branch_name)
+    
+    prj_diff = create_submodule_diff("pkgA", old_sha1, pkg_head_sha1)
+    prj_diff += create_submodule_diff("pkgB", old_sha2, pkg_head_sha2)
+
+    body = f"PR: mypool/pkgA!{pkg_pr_number1}\nPR: mypool/pkgB!{pkg_pr_number2}"
+    prj_pr = gitea_env.create_gitea_pr("myproducts/mySLFO", prj_diff, "Test Project PR Multi Mix", False, base_branch=merge_branch_name, body=body)
+    prj_pr_number = prj_pr["number"]
+
+    # 3. Add staging_bot as a reviewer.
+    gitea_env.request_reviewers("myproducts/mySLFO", prj_pr_number, ["autogits_obs_staging_bot"])
+
+    # Wait for staging-bot to be added (verification)
+    assert wait_for_staging_bot_reviewer(gitea_env, prj_pr_number), "Staging bot was not added as a reviewer."
+
+    # 4. Mock the OBS result list with 2 repositories and 2 architectures for each package.
+    project_name = f"openSUSE:Leap:16.0:PullRequest:{prj_pr_number}"
+    repos = ["repo1", "repo2"]
+    archs = ["x86_64", "aarch64"]
+    packages = ["pkgA", "pkgB"]
+    repo_arch_package_status = {}
+    
+    # 5. Set all repository, architecture, and package statuses to "in progress" (e.g., code="building").
+    for r in repos:
+        for a in archs:
+            for p in packages:
+                repo_arch_package_status[(r, a, p)] = ("building", "building")
+
+    def build_result_handler(request):
+        return create_build_result_xml(project_name, repo_arch_package_status)
+
+    setup_obs_mock(httpserver, project_name, build_result_handler)
+
+    # Expected Result 1: "Build is started in..."
+    assert wait_for_comment(gitea_env, "myproducts/mySLFO", prj_pr_number, "Build is started in", project_name), "Staging bot did not post 'Build is started' comment."
+
+    # 6. Transition the first package (all repositories and architectures) to "finished" mode and set its package statuses to "success" (e.g., code="succeeded").
+    for r in repos:
+        for a in archs:
+            repo_arch_package_status[(r, a, "pkgA")] = ("published", "succeeded")
+
+    # 7. Transition the second package (all repositories and architectures) to "finished" mode and set its package statuses to "failed" (e.g., code="failed").
+    for r in repos:
+        for a in archs:
+            repo_arch_package_status[(r, a, "pkgB")] = ("finished", "failed")
+
+    # Expected Result 2: "Build failed" on project PR.
+    assert wait_for_comment(gitea_env, "myproducts/mySLFO", prj_pr_number, "Build failed"), "Staging bot did not post 'Build failed' comment on project PR."
+
+    # Expected Result 3: "Build successful..." on the successful package's PR.
+    assert wait_for_comment(gitea_env, "mypool/pkgA", pkg_pr_number1, "Build successful, for more information go in", project_name), "Staging bot did not post 'Build successful' comment on package PR A."
+
+    # Expected Result 4: "Build failed..." on the failed package's PR.
+    assert wait_for_comment(gitea_env, "mypool/pkgB", pkg_pr_number2, "Build failed, for more information go in", project_name), "Staging bot did not post 'Build failed' comment on package PR B."
+
+@pytest.mark.t006
+def test_006_build_multiple_packages_partial_fail(staging_main_env, httpserver):
+    gitea_env, test_full_repo_name, merge_branch_name = staging_main_env
+    
+    # 1. Create package PRs.
+    diff = "diff --git a/test_br_partial_multi_pkgA.txt b/test_br_partial_multi_pkgA.txt\nnew file mode 100644\nindex 00000000..473a0f4c\n"
+    pr1 = gitea_env.create_gitea_pr("mypool/pkgA", diff, "Test BR Partial Multi PR A", False, base_branch=merge_branch_name)
+    pkg_pr_number1 = pr1["number"]
+    pkg_head_sha1 = pr1["head"]["sha"]
+
+    diff2 = "diff --git a/test_br_partial_multi_pkgB.txt b/test_br_partial_multi_pkgB.txt\nnew file mode 100644\nindex 00000000..473a0f4c\n"
+    pr2 = gitea_env.create_gitea_pr("mypool/pkgB", diff2, "Test BR Partial Multi PR B", False, base_branch=merge_branch_name)
+    pkg_pr_number2 = pr2["number"]
+    pkg_head_sha2 = pr2["head"]["sha"]
+
+    # 2. Create a project PR mentioning both packages in description and UPDATING SUBMODULES.
+    old_sha1 = gitea_env.get_submodule_sha("myproducts", "mySLFO", "pkgA", ref=merge_branch_name)
+    old_sha2 = gitea_env.get_submodule_sha("myproducts", "mySLFO", "pkgB", ref=merge_branch_name)
+    
+    prj_diff = create_submodule_diff("pkgA", old_sha1, pkg_head_sha1)
+    prj_diff += create_submodule_diff("pkgB", old_sha2, pkg_head_sha2)
+
+    body = f"PR: mypool/pkgA!{pkg_pr_number1}\nPR: mypool/pkgB!{pkg_pr_number2}"
+    prj_pr = gitea_env.create_gitea_pr("myproducts/mySLFO", prj_diff, "Test Project PR Multi Partial", False, base_branch=merge_branch_name, body=body)
+    prj_pr_number = prj_pr["number"]
+
+    # 3. Add staging_bot as a reviewer.
+    gitea_env.request_reviewers("myproducts/mySLFO", prj_pr_number, ["autogits_obs_staging_bot"])
+
+    # Wait for staging-bot to be added (verification)
+    assert wait_for_staging_bot_reviewer(gitea_env, prj_pr_number), "Staging bot was not added as a reviewer."
+
+    # 4. Mock the OBS result list with 2 repositories and 2 architectures for each package.
+    project_name = f"openSUSE:Leap:16.0:PullRequest:{prj_pr_number}"
+    repos = ["repo1", "repo2"]
+    archs = ["x86_64", "aarch64"]
+    packages = ["pkgA", "pkgB"]
+    repo_arch_package_status = {}
+    
+    # 5. Set all repository, architecture, and package statuses to "in progress" (e.g., code="building").
+    for r in repos:
+        for a in archs:
+            for p in packages:
+                repo_arch_package_status[(r, a, p)] = ("building", "building")
+
+    def build_result_handler(request):
+        return create_build_result_xml(project_name, repo_arch_package_status)
+
+    setup_obs_mock(httpserver, project_name, build_result_handler)
+
+    # Expected Result 1: "Build is started in..."
+    assert wait_for_comment(gitea_env, "myproducts/mySLFO", prj_pr_number, "Build is started in", project_name), "Staging bot did not post 'Build is started' comment."
+
+    # 6. Transition the first package (all repositories and architectures) to "finished" mode and set its package statuses to "success" (e.g., code="succeeded").
+    for r in repos:
+        for a in archs:
+            repo_arch_package_status[(r, a, "pkgA")] = ("published", "succeeded")
+
+    # 7. Transition the second package:
+    # Repository 1 (all architectures) to "finished" mode with "success" (e.g., code="succeeded").
+    for a in archs:
+        repo_arch_package_status[("repo1", a, "pkgB")] = ("published", "succeeded")
+    
+    # Repository 2 (all architectures) to "finished" mode with "failed" (e.g., code="failed").
+    for a in archs:
+        repo_arch_package_status[("repo2", a, "pkgB")] = ("finished", "failed")
+
+    # Expected Result 2: "Build failed" on project PR.
+    assert wait_for_comment(gitea_env, "myproducts/mySLFO", prj_pr_number, "Build failed"), "Staging bot did not post 'Build failed' comment on project PR."
+
+    # Expected Result 3: "Build successful..." on the first (successful) package's PR.
+    assert wait_for_comment(gitea_env, "mypool/pkgA", pkg_pr_number1, "Build successful, for more information go in", project_name), "Staging bot did not post 'Build successful' comment on package PR A."
+
+    # Expected Result 4: "Build failed..." on the second (failed) package's PR.
+    assert wait_for_comment(gitea_env, "mypool/pkgB", pkg_pr_number2, "Build failed, for more information go in", project_name), "Staging bot did not post 'Build failed' comment on package PR B."
+
+@pytest.mark.t007
+def test_007_build_multiple_packages_arch_fail(staging_main_env, httpserver):
+    gitea_env, test_full_repo_name, merge_branch_name = staging_main_env
+    
+    # 1. Create package PRs.
+    diff = "diff --git a/test_br_arch_multi_pkgA.txt b/test_br_arch_multi_pkgA.txt\nnew file mode 100644\nindex 00000000..473a0f4c\n"
+    pr1 = gitea_env.create_gitea_pr("mypool/pkgA", diff, "Test BR Arch Multi PR A", False, base_branch=merge_branch_name)
+    pkg_pr_number1 = pr1["number"]
+    pkg_head_sha1 = pr1["head"]["sha"]
+
+    diff2 = "diff --git a/test_br_arch_multi_pkgB.txt b/test_br_arch_multi_pkgB.txt\nnew file mode 100644\nindex 00000000..473a0f4c\n"
+    pr2 = gitea_env.create_gitea_pr("mypool/pkgB", diff2, "Test BR Arch Multi PR B", False, base_branch=merge_branch_name)
+    pkg_pr_number2 = pr2["number"]
+    pkg_head_sha2 = pr2["head"]["sha"]
+
+    # 2. Create a project PR mentioning both packages in description and UPDATING SUBMODULES.
+    old_sha1 = gitea_env.get_submodule_sha("myproducts", "mySLFO", "pkgA", ref=merge_branch_name)
+    old_sha2 = gitea_env.get_submodule_sha("myproducts", "mySLFO", "pkgB", ref=merge_branch_name)
+    
+    prj_diff = create_submodule_diff("pkgA", old_sha1, pkg_head_sha1)
+    prj_diff += create_submodule_diff("pkgB", old_sha2, pkg_head_sha2)
+
+    body = f"PR: mypool/pkgA!{pkg_pr_number1}\nPR: mypool/pkgB!{pkg_pr_number2}"
+    prj_pr = gitea_env.create_gitea_pr("myproducts/mySLFO", prj_diff, "Test Project PR Multi Arch Fail", False, base_branch=merge_branch_name, body=body)
+    prj_pr_number = prj_pr["number"]
+
+    # 3. Add staging_bot as a reviewer.
+    gitea_env.request_reviewers("myproducts/mySLFO", prj_pr_number, ["autogits_obs_staging_bot"])
+
+    # Wait for staging-bot to be added (verification)
+    assert wait_for_staging_bot_reviewer(gitea_env, prj_pr_number), "Staging bot was not added as a reviewer."
+
+    # 4. Mock the OBS result list with 2 repositories and 2 architectures for each package.
+    project_name = f"openSUSE:Leap:16.0:PullRequest:{prj_pr_number}"
+    repos = ["repo1", "repo2"]
+    archs = ["x86_64", "aarch64"]
+    packages = ["pkgA", "pkgB"]
+    repo_arch_package_status = {}
+    
+    # 5. Set all repository, architecture, and package statuses to "in progress" (e.g., code="building").
+    for r in repos:
+        for a in archs:
+            for p in packages:
+                repo_arch_package_status[(r, a, p)] = ("building", "building")
+
+    def build_result_handler(request):
+        return create_build_result_xml(project_name, repo_arch_package_status)
+
+    setup_obs_mock(httpserver, project_name, build_result_handler)
+
+    # Expected Result 1: "Build is started in..."
+    assert wait_for_comment(gitea_env, "myproducts/mySLFO", prj_pr_number, "Build is started in", project_name), "Staging bot did not post 'Build is started' comment."
+
+    # 6. Transition the first package (all repositories and architectures) to "finished" mode and set its package statuses to "success" (e.g., code="succeeded").
+    for r in repos:
+        for a in archs:
+            repo_arch_package_status[(r, a, "pkgA")] = ("published", "succeeded")
+
+    # 7. Transition the second package:
+    # Architecture 1 (all repositories) to "finished" mode with "success" (e.g., code="succeeded").
+    for r in repos:
+        repo_arch_package_status[(r, archs[0], "pkgB")] = ("published", "succeeded")
+    
+    # Architecture 2 (all repositories) to "finished" mode with "failed" (e.g., code="failed").
+    for r in repos:
+        repo_arch_package_status[(r, archs[1], "pkgB")] = ("finished", "failed")
+
+    # Expected Result 2: "Build failed" on project PR.
+    assert wait_for_comment(gitea_env, "myproducts/mySLFO", prj_pr_number, "Build failed"), "Staging bot did not post 'Build failed' comment on project PR."
+
+    # Expected Result 3: "Build successful..." on the first (successful) package's PR.
+    assert wait_for_comment(gitea_env, "mypool/pkgA", pkg_pr_number1, "Build successful, for more information go in", project_name), "Staging bot did not post 'Build successful' comment on package PR A."
+
+    # Expected Result 4: "Build failed..." on the second (failed) package's PR.
+    assert wait_for_comment(gitea_env, "mypool/pkgB", pkg_pr_number2, "Build failed, for more information go in", project_name), "Staging bot did not post 'Build failed' comment on package PR B."

integration/tests/workflow_pr_merge_test.py

@@ -4,80 +4,6 @@ import time
 from pathlib import Path
 from tests.lib.common_test_utils import GiteaAPIClient
 
-def wait_for_manual_merge_approvals(gitea_env, staging_bot_client, package_pr_number, project_pr_number):
-    """
-    Wait for required approvals on both package and project PRs for manual merge tests.
-    """
-    print("Waiting for required review requests and approving them...")
-    # Expected reviewers based on manual-merge branch config and pkgA maintainership
-    mandatory_reviewers = {"usera", "userb"}
-    maintainers = {"ownerA", "ownerX", "ownerY"}
-    
-    # ManualMergeOnly still requires regular reviews to be satisfied.
-    # We poll until required reviewers have approved.
-    all_approved = False
-    approved_reviewers = set()
-    for _ in range(60): # Poll for up to 60 seconds (1s interval)
-        # Trigger approvals for whatever is already requested
-        gitea_env.approve_requested_reviews("mypool/pkgA", package_pr_number)
-        gitea_env.approve_requested_reviews("myproducts/mySLFO", project_pr_number)
-        
-        # Explicitly handle staging bot if it is requested or pending
-        prj_reviews = gitea_env.list_reviews("myproducts/mySLFO", project_pr_number)
-        if any(r["user"]["login"] == "autogits_obs_staging_bot" and r["state"] in ["REQUEST_REVIEW", "PENDING"] for r in prj_reviews):
-            print("Staging bot has a pending/requested review. Approving...")
-            staging_bot_client.create_review("myproducts/mySLFO", project_pr_number, event="APPROVED", body="Staging bot approves")
-
-        # Check if mandatory reviewers and at least one maintainer have approved
-        pkg_reviews = gitea_env.list_reviews("mypool/pkgA", package_pr_number)
-        approved_reviewers = {r["user"]["login"] for r in pkg_reviews if r["state"] == "APPROVED"}
-        
-        if mandatory_reviewers.issubset(approved_reviewers) and any(m in approved_reviewers for m in maintainers):
-            # And check project PR for bot approval
-            prj_approved = any(r["user"]["login"] == "autogits_obs_staging_bot" and r["state"] == "APPROVED" for r in prj_reviews)
-            if prj_approved:
-                all_approved = True
-                print(f"Required reviewers approved: mandatory={mandatory_reviewers}, maintainer={[m for m in maintainers if m in approved_reviewers]}, staging_bot=True")
-                break
-        
-        pkg_details = gitea_env.get_pr_details("mypool/pkgA", package_pr_number)
-        prj_details = gitea_env.get_pr_details("myproducts/mySLFO", project_pr_number)
-        
-        assert not pkg_details.get("merged"), "Package PR merged prematurely (ManualMergeOnly ignored?)"
-        assert not prj_details.get("merged"), "Project PR merged prematurely (ManualMergeOnly ignored?)"
-        
-        time.sleep(1)
-
-    assert all_approved, f"Timed out waiting for required approvals. Mandatory: {mandatory_reviewers}, Maintainers: {maintainers}. Current approved: {approved_reviewers}"
-    print("Both PRs have all required approvals but are not merged (as expected with ManualMergeOnly).")
-
-def wait_for_merge_status(gitea_env, package_pr_number, project_pr_number, timeout_seconds=20):
-    """
-    Poll for PR merge status on both package and project PRs.
-    """
-    print("Polling for PR merge status...")
-    package_merged = False
-    project_merged = False
-    
-    for i in range(int(timeout_seconds / 0.5)): # Poll with 0.5s interval
-        if not package_merged:
-            pkg_details = gitea_env.get_pr_details("mypool/pkgA", package_pr_number)
-            if pkg_details.get("merged"):
-                package_merged = True
-                print(f"Package PR mypool/pkgA#{package_pr_number} merged.")
-        
-        if not project_merged:
-            prj_details = gitea_env.get_pr_details("myproducts/mySLFO", project_pr_number)
-            if prj_details.get("merged"):
-                project_merged = True
-                print(f"Project PR myproducts/mySLFO#{project_pr_number} merged.")
-        
-        if package_merged and project_merged:
-            break
-            
-        time.sleep(0.5)
-    return package_merged, project_merged
-
 @pytest.mark.t001
 def test_001_automerge(automerge_env, test_user_client):
     """
@@ -136,14 +62,74 @@ index 00000000..473a0f4c
     print(f"Found project PR: myproducts/mySLFO#{project_pr_number}")
 
     # 3. Approve reviews and verify NOT merged
-    wait_for_manual_merge_approvals(gitea_env, staging_bot_client, package_pr_number, project_pr_number)
+    print("Waiting for required review requests and approving them...")
+    # Expected reviewers based on manual-merge branch config and pkgA maintainership
+    mandatory_reviewers = {"usera", "userb"}
+    maintainers = {"ownerA", "ownerX", "ownerY"}
+    
+    # ManualMergeOnly still requires regular reviews to be satisfied.
+    # We poll until required reviewers have approved.
+    all_approved = False
+    for _ in range(30):
+        # Trigger approvals for whatever is already requested
+        gitea_env.approve_requested_reviews("mypool/pkgA", package_pr_number)
+        gitea_env.approve_requested_reviews("myproducts/mySLFO", project_pr_number)
+        
+        # Explicitly handle staging bot if it is requested or pending
+        prj_reviews = gitea_env.list_reviews("myproducts/mySLFO", project_pr_number)
+        if any(r["user"]["login"] == "autogits_obs_staging_bot" and r["state"] in ["REQUEST_REVIEW", "PENDING"] for r in prj_reviews):
+            print("Staging bot has a pending/requested review. Approving...")
+            staging_bot_client.create_review("myproducts/mySLFO", project_pr_number, event="APPROVED", body="Staging bot approves")
+
+        # Check if mandatory reviewers and at least one maintainer have approved
+        pkg_reviews = gitea_env.list_reviews("mypool/pkgA", package_pr_number)
+        approved_reviewers = {r["user"]["login"] for r in pkg_reviews if r["state"] == "APPROVED"}
+        
+        if mandatory_reviewers.issubset(approved_reviewers) and any(m in approved_reviewers for m in maintainers):
+            # And check project PR for bot approval
+            prj_approved = any(r["user"]["login"] == "autogits_obs_staging_bot" and r["state"] == "APPROVED" for r in prj_reviews)
+            if prj_approved:
+                all_approved = True
+                print(f"Required reviewers approved: mandatory={mandatory_reviewers}, maintainer={[m for m in maintainers if m in approved_reviewers]}, staging_bot=True")
+                break
+        
+        pkg_details = gitea_env.get_pr_details("mypool/pkgA", package_pr_number)
+        prj_details = gitea_env.get_pr_details("myproducts/mySLFO", project_pr_number)
+        
+        assert not pkg_details.get("merged"), "Package PR merged prematurely (ManualMergeOnly ignored?)"
+        assert not prj_details.get("merged"), "Project PR merged prematurely (ManualMergeOnly ignored?)"
+        
+        time.sleep(2)
+
+    assert all_approved, f"Timed out waiting for required approvals. Mandatory: {mandatory_reviewers}, Maintainers: {maintainers}. Current approved: {approved_reviewers}"
+    print("Both PRs have all required approvals but are not merged (as expected with ManualMergeOnly).")
 
     # 4. Comment "merge ok" from a requested reviewer (ownerA)
     print("Commenting 'merge ok' on package PR from a maintainer...")
     ownerA_client.create_issue_comment("mypool/pkgA", package_pr_number, "merge ok")
 
     # 5. Verify both PRs are merged
-    package_merged, project_merged = wait_for_merge_status(gitea_env, package_pr_number, project_pr_number)
+    print("Polling for PR merge status...")
+    package_merged = False
+    project_merged = False
+    
+    for i in range(20): # Poll for up to 20 seconds
+        if not package_merged:
+            pkg_details = gitea_env.get_pr_details("mypool/pkgA", package_pr_number)
+            if pkg_details.get("merged"):
+                package_merged = True
+                print(f"Package PR mypool/pkgA#{package_pr_number} merged.")
+        
+        if not project_merged:
+            prj_details = gitea_env.get_pr_details("myproducts/mySLFO", project_pr_number)
+            if prj_details.get("merged"):
+                project_merged = True
+                print(f"Project PR myproducts/mySLFO#{project_pr_number} merged.")
+        
+        if package_merged and project_merged:
+            break
+            
+        time.sleep(1)
             
     assert package_merged, f"Package PR mypool/pkgA#{package_pr_number} was not merged after 'merge ok'."
     assert project_merged, f"Project PR myproducts/mySLFO#{project_pr_number} was not merged after 'merge ok'."
@@ -176,14 +162,74 @@ index 00000000..473a0f4c
     print(f"Found project PR: myproducts/mySLFO#{project_pr_number}")
 
     # 3. Approve reviews and verify NOT merged
-    wait_for_manual_merge_approvals(gitea_env, staging_bot_client, package_pr_number, project_pr_number)
+    print("Waiting for required review requests and approving them...")
+    # Expected reviewers based on manual-merge branch config and pkgA maintainership
+    mandatory_reviewers = {"usera", "userb"}
+    maintainers = {"ownerA", "ownerX", "ownerY"}
+    
+    # ManualMergeOnly still requires regular reviews to be satisfied.
+    # We poll until required reviewers have approved.
+    all_approved = False
+    for _ in range(30):
+        # Trigger approvals for whatever is already requested
+        gitea_env.approve_requested_reviews("mypool/pkgA", package_pr_number)
+        gitea_env.approve_requested_reviews("myproducts/mySLFO", project_pr_number)
+        
+        # Explicitly handle staging bot if it is requested or pending
+        prj_reviews = gitea_env.list_reviews("myproducts/mySLFO", project_pr_number)
+        if any(r["user"]["login"] == "autogits_obs_staging_bot" and r["state"] in ["REQUEST_REVIEW", "PENDING"] for r in prj_reviews):
+            print("Staging bot has a pending/requested review. Approving...")
+            staging_bot_client.create_review("myproducts/mySLFO", project_pr_number, event="APPROVED", body="Staging bot approves")
+
+        # Check if mandatory reviewers and at least one maintainer have approved
+        pkg_reviews = gitea_env.list_reviews("mypool/pkgA", package_pr_number)
+        approved_reviewers = {r["user"]["login"] for r in pkg_reviews if r["state"] == "APPROVED"}
+        
+        if mandatory_reviewers.issubset(approved_reviewers) and any(m in approved_reviewers for m in maintainers):
+            # And check project PR for bot approval
+            prj_approved = any(r["user"]["login"] == "autogits_obs_staging_bot" and r["state"] == "APPROVED" for r in prj_reviews)
+            if prj_approved:
+                all_approved = True
+                print(f"Required reviewers approved: mandatory={mandatory_reviewers}, maintainer={[m for m in maintainers if m in approved_reviewers]}, staging_bot=True")
+                break
+        
+        pkg_details = gitea_env.get_pr_details("mypool/pkgA", package_pr_number)
+        prj_details = gitea_env.get_pr_details("myproducts/mySLFO", project_pr_number)
+        
+        assert not pkg_details.get("merged"), "Package PR merged prematurely (ManualMergeOnly ignored?)"
+        assert not prj_details.get("merged"), "Project PR merged prematurely (ManualMergeOnly ignored?)"
+        
+        time.sleep(2)
+
+    assert all_approved, f"Timed out waiting for required approvals. Mandatory: {mandatory_reviewers}, Maintainers: {maintainers}. Current approved: {approved_reviewers}"
+    print("Both PRs have all required approvals but are not merged (as expected with ManualMergeOnly).")
 
     # 4. Comment "merge ok" from a requested reviewer (ownerB)
     print("Commenting 'merge ok' on package PR as user ownerB ...")
     ownerB_client.create_issue_comment("mypool/pkgA", package_pr_number, "merge ok")
 
     # 5. Verify both PRs are merged
-    package_merged, project_merged = wait_for_merge_status(gitea_env, package_pr_number, project_pr_number)
+    print("Polling for PR merge status...")
+    package_merged = False
+    project_merged = False
+    
+    for i in range(20): # Poll for up to 20 seconds
+        if not package_merged:
+            pkg_details = gitea_env.get_pr_details("mypool/pkgA", package_pr_number)
+            if pkg_details.get("merged"):
+                package_merged = True
+                print(f"Package PR mypool/pkgA#{package_pr_number} merged.")
+        
+        if not project_merged:
+            prj_details = gitea_env.get_pr_details("myproducts/mySLFO", project_pr_number)
+            if prj_details.get("merged"):
+                project_merged = True
+                print(f"Project PR myproducts/mySLFO#{project_pr_number} merged.")
+        
+        if package_merged and project_merged:
+            break
+            
+        time.sleep(1)
             
     assert not package_merged, f"Package PR mypool/pkgA#{package_pr_number} was merged after 'merge ok'."
     assert not project_merged, f"Project PR myproducts/mySLFO#{project_pr_number} was merged after 'merge ok'."
@@ -254,10 +300,10 @@ index 00000000..473a0f4c
     
     # Approve again and verify it is NOT merged
     print("Approving again and verifying PR is NOT merged (because it's not FF)...")
-    for i in range(30): # Poll with 0.5s interval
+    for i in range(15):
         gitea_env.approve_requested_reviews("mypool/pkgA", package_pr_number)
         gitea_env.approve_requested_reviews("myproducts/mySLFO", project_pr_number)
-        time.sleep(0.5)
+        time.sleep(1)
         
     pkg_details = gitea_env.get_pr_details("mypool/pkgA", package_pr_number)
     assert not pkg_details.get("merged"), "Package PR merged despite NOT being FF-mergeable!"