6aaff89179
- Add TC-MERGE-008 to 013 for testing MergeMode of workflow-pr - Synchronize integration/test-plan.md with the actual test implementations
326 lines
15 KiB
Python
326 lines
15 KiB
Python
import pytest
|
|
import re
|
|
import time
|
|
import base64
|
|
from pathlib import Path
|
|
from tests.lib.common_test_utils import GiteaAPIClient
|
|
|
|
@pytest.mark.t001
|
|
def test_001_review_requests_matching_config(automerge_env, ownerA_client):
|
|
"""
|
|
Test scenario:
|
|
1. The package PR for pkgB is opened by ownerA (who is not a maintainer of pkgB).
|
|
2. Check that review request comes to ownerB and ownerBB (package maintainers)
|
|
AND usera and userb (from workflow.config).
|
|
"""
|
|
gitea_env, test_full_repo_name, branch_name = automerge_env
|
|
|
|
# 1. Create a package PR for mypool/pkgB as ownerA
|
|
ts = int(time.time() * 1000)
|
|
filename = f"pkgB_test_{ts}.txt"
|
|
diff = f"""diff --git a/{filename} b/{filename}
|
|
new file mode 100644
|
|
index 0000000..e69de29
|
|
"""
|
|
print(f"--- Creating package PR in mypool/pkgB on branch {branch_name} as ownerA ---")
|
|
package_pr = ownerA_client.create_gitea_pr("mypool/pkgB", diff, "Test Review Requests Config", True, base_branch=branch_name)
|
|
package_pr_number = package_pr["number"]
|
|
print(f"Created package PR mypool/pkgB#{package_pr_number}")
|
|
|
|
# 2. Check that review requests came to ownerB, ownerBB, usera, and userb
|
|
print("Checking for review requests from maintainers and workflow.config...")
|
|
reviewers_requested = set()
|
|
expected_reviewers = {"ownerB", "ownerBB", "usera", "userb"}
|
|
for _ in range(30):
|
|
reviews = gitea_env.list_reviews("mypool/pkgB", package_pr_number)
|
|
reviewers_requested = {r["user"]["login"] for r in reviews if r["state"] == "REQUEST_REVIEW"}
|
|
if expected_reviewers.issubset(reviewers_requested):
|
|
break
|
|
time.sleep(1)
|
|
|
|
for reviewer in expected_reviewers:
|
|
assert reviewer in reviewers_requested, f"{reviewer} was not requested for review. Requested: {reviewers_requested}"
|
|
|
|
print(f"Confirmed: {expected_reviewers} were requested for review.")
|
|
|
|
|
|
@pytest.mark.t004
|
|
def test_004_maintainer(maintainer_env, ownerA_client):
|
|
"""
|
|
Test scenario:
|
|
1. workflow.config will not have users with '+' sign.
|
|
2. The package PR is opened by the package maintainer (ownerA for pkgA).
|
|
3. Do not submit any review approval.
|
|
4. Check that both PRs are automatically merged anyway.
|
|
"""
|
|
gitea_env, test_full_repo_name, branch_name = maintainer_env
|
|
|
|
# 0. Smoke test ownerA_client
|
|
print(f"--- Smoke testing ownerA_client ---")
|
|
ownerA_client._request("GET", "users/admin")
|
|
print(f"ownerA_client smoke test passed")
|
|
|
|
# 0.1 Verify all users from config exist
|
|
print("--- Verifying all users from config exist ---")
|
|
import json
|
|
wf_file = gitea_env.get_file_info("myproducts", "mySLFO", "workflow.config", branch=branch_name)
|
|
wf = json.loads(base64.b64decode(wf_file["content"]).decode("utf-8"))
|
|
mt_file = gitea_env.get_file_info("myproducts", "mySLFO", "_maintainership.json", branch=branch_name)
|
|
mt = json.loads(base64.b64decode(mt_file["content"]).decode("utf-8"))
|
|
|
|
expected_users = set()
|
|
for r in wf.get("Reviewers", []):
|
|
username = r.lstrip("+-")
|
|
if username and username not in ["autogits_obs_staging_bot", "workflow-pr"]:
|
|
expected_users.add(username)
|
|
for pkg_users in mt.values():
|
|
for username in pkg_users:
|
|
expected_users.add(username)
|
|
|
|
for username in expected_users:
|
|
gitea_env._request("GET", f"users/{username}")
|
|
print(f"Verified user exists: {username}")
|
|
|
|
# 1. Create a package PR as ownerA
|
|
ts = int(time.time() * 1000)
|
|
filename = f"maintainer_test_{ts}.txt"
|
|
diff = f"""diff --git a/{filename} b/{filename}
|
|
new file mode 100644
|
|
index 0000000..e69de29
|
|
"""
|
|
print(f"--- Creating package PR in mypool/pkgA on branch {branch_name} as ownerA ---")
|
|
package_pr = ownerA_client.create_gitea_pr("mypool/pkgA", diff, "Test Maintainer Merge", True, base_branch=branch_name)
|
|
package_pr_number = package_pr["number"]
|
|
print(f"Created package PR mypool/pkgA#{package_pr_number}")
|
|
|
|
# 2. Make sure the workflow-pr service created related project PR
|
|
project_pr_number = gitea_env.wait_for_project_pr("mypool/pkgA", package_pr_number)
|
|
assert project_pr_number is not None, "Workflow bot did not create a project PR."
|
|
print(f"Found project PR: myproducts/mySLFO#{project_pr_number}")
|
|
|
|
# 3. Make sure both PRs are merged automatically WITHOUT manual approvals
|
|
print("Polling for PR merge status (only bot approval allowed)...")
|
|
package_merged = False
|
|
project_merged = False
|
|
|
|
for i in range(15): # Poll for up to 15 seconds
|
|
# Package PR
|
|
if not package_merged:
|
|
pkg_details = gitea_env.get_pr_details("mypool/pkgA", package_pr_number)
|
|
if pkg_details.get("merged"):
|
|
package_merged = True
|
|
print(f"Package PR mypool/pkgA#{package_pr_number} merged.")
|
|
else:
|
|
# Approve ONLY bot if requested
|
|
reviews = gitea_env.list_reviews("mypool/pkgA", package_pr_number)
|
|
if any(r["state"] == "REQUEST_REVIEW" and r["user"]["login"] == "autogits_obs_staging_bot" for r in reviews):
|
|
gitea_env.approve_requested_reviews("mypool/pkgA", package_pr_number)
|
|
|
|
# Project PR
|
|
if not project_merged:
|
|
prj_details = gitea_env.get_pr_details("myproducts/mySLFO", project_pr_number)
|
|
if prj_details.get("merged"):
|
|
project_merged = True
|
|
print(f"Project PR myproducts/mySLFO#{project_pr_number} merged.")
|
|
else:
|
|
# Approve ONLY bot if requested
|
|
reviews = gitea_env.list_reviews("myproducts/mySLFO", project_pr_number)
|
|
if any(r["state"] == "REQUEST_REVIEW" and r["user"]["login"] == "autogits_obs_staging_bot" for r in reviews):
|
|
gitea_env.approve_requested_reviews("myproducts/mySLFO", project_pr_number)
|
|
|
|
if package_merged and project_merged:
|
|
break
|
|
|
|
time.sleep(1)
|
|
|
|
assert package_merged, f"Package PR mypool/pkgA#{package_pr_number} was not merged automatically."
|
|
assert project_merged, f"Project PR myproducts/mySLFO#{project_pr_number} was not merged automatically."
|
|
print("Both PRs merged successfully by maintainer rule.")
|
|
|
|
|
|
@pytest.mark.t005
|
|
# @pytest.mark.xfail(reason="TBD troubleshoot")
|
|
def test_005_any_maintainer_approval_sufficient(maintainer_env, ownerA_client, ownerBB_client):
|
|
"""
|
|
Test scenario:
|
|
1. The package PR for pkgB is opened by ownerA (who is not a maintainer of pkgB).
|
|
2. Check that review request comes to both ownerB and ownerBB.
|
|
3. ownerB doesn't leave review.
|
|
4. check that review from ownerBB was enough to get both PRs merged.
|
|
"""
|
|
gitea_env, test_full_repo_name, branch_name = maintainer_env
|
|
|
|
# 1. Create a package PR for mypool/pkgB as ownerA
|
|
ts = int(time.time() * 1000)
|
|
filename = f"pkgB_test_{ts}.txt"
|
|
diff = f"""diff --git a/{filename} b/{filename}
|
|
new file mode 100644
|
|
index 0000000..e69de29
|
|
"""
|
|
print(f"--- Creating package PR in mypool/pkgB on branch {branch_name} as ownerA ---")
|
|
package_pr = ownerA_client.create_gitea_pr("mypool/pkgB", diff, "Test Single Maintainer Merge", True, base_branch=branch_name)
|
|
package_pr_number = package_pr["number"]
|
|
print(f"Created package PR mypool/pkgB#{package_pr_number}")
|
|
|
|
# 2. Make sure the workflow-pr service created related project PR
|
|
project_pr_number = gitea_env.wait_for_project_pr("mypool/pkgB", package_pr_number)
|
|
assert project_pr_number is not None, "Workflow bot did not create a project PR."
|
|
print(f"Found project PR: myproducts/mySLFO#{project_pr_number}")
|
|
|
|
# 3. Check that review requests came to ownerB and ownerBB
|
|
print("Checking for review requests from ownerB and ownerBB...")
|
|
reviewers_requested = set()
|
|
for _ in range(20):
|
|
reviews = gitea_env.list_reviews("mypool/pkgB", package_pr_number)
|
|
reviewers_requested = {r["user"]["login"] for r in reviews if r["state"] == "REQUEST_REVIEW"}
|
|
if "ownerB" in reviewers_requested and "ownerBB" in reviewers_requested:
|
|
break
|
|
time.sleep(1)
|
|
|
|
assert "ownerB" in reviewers_requested, f"ownerB was not requested for review. Requested: {reviewers_requested}"
|
|
assert "ownerBB" in reviewers_requested, f"ownerBB was not requested for review. Requested: {reviewers_requested}"
|
|
print(f"Confirmed: ownerB and ownerBB were requested for review.")
|
|
|
|
# 4. ownerBB leaves review, ownerB does not.
|
|
print("ownerBB approving the PR...")
|
|
ownerBB_client.create_review("mypool/pkgB", package_pr_number, event="APPROVED", body="Approval from ownerBB")
|
|
|
|
# 5. Check that review request for ownerB is removed
|
|
print("Polling for ownerB review request removal...")
|
|
ownerB_removed = False
|
|
for _ in range(30):
|
|
time.sleep(1)
|
|
reviews = gitea_env.list_reviews("mypool/pkgB", package_pr_number)
|
|
reviewers_requested = {r["user"]["login"] for r in reviews if r["state"] == "REQUEST_REVIEW"}
|
|
if "ownerB" not in reviewers_requested:
|
|
ownerB_removed = True
|
|
break
|
|
|
|
assert ownerB_removed, f"ownerB review request was not removed after ownerBB approval. Current requested: {reviewers_requested}"
|
|
print("Confirmed: ownerB review request removed after single maintainer approval.")
|
|
|
|
|
|
@pytest.mark.t006
|
|
@pytest.mark.xfail(reason="tbd flacky in ci")
|
|
def test_006_maintainer_rejection_removes_other_requests(maintainer_env, ownerA_client, ownerBB_client):
|
|
"""
|
|
Test scenario:
|
|
1. The package PR for pkgB is opened by ownerA (who is not a maintainer of pkgB).
|
|
2. Check that review request comes to both ownerB and ownerBB.
|
|
3. ownerBB rejects the PR (REQUEST_CHANGES).
|
|
4. Check that review request for ownerB is removed.
|
|
"""
|
|
gitea_env, test_full_repo_name, branch_name = maintainer_env
|
|
|
|
# 1. Create a package PR for mypool/pkgB as ownerA
|
|
ts = int(time.time() * 1000)
|
|
filename = f"pkgB_rejection_test_{ts}.txt"
|
|
diff = f"""diff --git a/{filename} b/{filename}
|
|
new file mode 100644
|
|
index 0000000..e69de29
|
|
"""
|
|
print(f"--- Creating package PR in mypool/pkgB on branch {branch_name} as ownerA ---")
|
|
package_pr = ownerA_client.create_gitea_pr("mypool/pkgB", diff, "Test Maintainer Rejection", True, base_branch=branch_name)
|
|
package_pr_number = package_pr["number"]
|
|
print(f"Created package PR mypool/pkgB#{package_pr_number}")
|
|
|
|
# 2. Check that review requests came to ownerB and ownerBB
|
|
print("Checking for review requests from ownerB and ownerBB...")
|
|
for _ in range(20):
|
|
reviews = gitea_env.list_reviews("mypool/pkgB", package_pr_number)
|
|
reviewers_requested = {r["user"]["login"] for r in reviews if r["state"] == "REQUEST_REVIEW"}
|
|
if "ownerB" in reviewers_requested and "ownerBB" in reviewers_requested:
|
|
break
|
|
time.sleep(1)
|
|
else:
|
|
reviews = gitea_env.list_reviews("mypool/pkgB", package_pr_number)
|
|
reviewers_requested = {r["user"]["login"] for r in reviews if r["state"] == "REQUEST_REVIEW"}
|
|
pytest.fail(f"ownerB and ownerBB were not both requested. Got: {reviewers_requested}")
|
|
|
|
# 3. ownerBB rejects the PR
|
|
print("ownerBB rejecting the PR...")
|
|
ownerBB_client.create_review("mypool/pkgB", package_pr_number, event="REQUEST_CHANGES", body="Rejecting from ownerBB")
|
|
|
|
# 4. Check that review request for ownerB is removed
|
|
print("Checking if ownerB's review request is removed...")
|
|
for _ in range(20):
|
|
reviews = gitea_env.list_reviews("mypool/pkgB", package_pr_number)
|
|
reviewers_requested = {r["user"]["login"] for r in reviews if r["state"] == "REQUEST_REVIEW"}
|
|
if "ownerB" not in reviewers_requested:
|
|
print("Confirmed: ownerB's review request was removed.")
|
|
break
|
|
time.sleep(1)
|
|
else:
|
|
pytest.fail("ownerB's review request was not removed after ownerBB rejection.")
|
|
|
|
|
|
@pytest.mark.t007
|
|
@pytest.mark.xfail(reason="TBD troubleshoot")
|
|
def test_007_review_required_needs_all_approvals(review_required_env, ownerA_client, ownerBB_client):
|
|
"""
|
|
Test scenario:
|
|
1. it uses new fixture with "ReviewRequired = true" in the workflow.config.
|
|
2. Package PR for pkgB opened by ownerA.
|
|
3. Check review request comes to both ownerB and ownerBB.
|
|
4. ownerBB approves.
|
|
5. make sure that review is not merged automatically and the request for ownerB is not removed.
|
|
"""
|
|
gitea_env, test_full_repo_name, branch_name = review_required_env
|
|
|
|
# 0. Smoke test ownerA_client
|
|
print(f"--- Smoke testing ownerA_client ---")
|
|
ownerA_client._request("GET", "users/admin")
|
|
print(f"ownerA_client smoke test passed")
|
|
|
|
# 1. Create a package PR for mypool/pkgB as ownerA
|
|
ts = int(time.time() * 1000)
|
|
filename = f"pkgB_review_required_test_{ts}.txt"
|
|
diff = f"""diff --git a/{filename} b/{filename}
|
|
new file mode 100644
|
|
index 0000000..e69de29
|
|
"""
|
|
print(f"--- Creating package PR in mypool/pkgB on branch {branch_name} as ownerA ---")
|
|
package_pr = ownerA_client.create_gitea_pr("mypool/pkgB", diff, "Test Review Required", True, base_branch=branch_name)
|
|
package_pr_number = package_pr["number"]
|
|
print(f"Created package PR mypool/pkgB#{package_pr_number}")
|
|
|
|
# 2. Make sure the workflow-pr service created related project PR
|
|
project_pr_number = gitea_env.wait_for_project_pr("mypool/pkgB", package_pr_number)
|
|
assert project_pr_number is not None, "Workflow bot did not create a project PR."
|
|
print(f"Found project PR: myproducts/mySLFO#{project_pr_number}")
|
|
|
|
# 3. Check that review requests came to ownerB and ownerBB
|
|
print("Checking for review requests from ownerB and ownerBB...")
|
|
for _ in range(20):
|
|
reviews = gitea_env.list_reviews("mypool/pkgB", package_pr_number)
|
|
reviewers_requested = {r["user"]["login"] for r in reviews if r["state"] == "REQUEST_REVIEW"}
|
|
if "ownerB" in reviewers_requested and "ownerBB" in reviewers_requested:
|
|
break
|
|
time.sleep(1)
|
|
else:
|
|
reviews = gitea_env.list_reviews("mypool/pkgB", package_pr_number)
|
|
reviewers_requested = {r["user"]["login"] for r in reviews if r["state"] == "REQUEST_REVIEW"}
|
|
pytest.fail(f"ownerB and ownerBB were not both requested. Got: {reviewers_requested}")
|
|
|
|
# 4. ownerBB leaves review, ownerB does not.
|
|
print("ownerBB approving the PR...")
|
|
ownerBB_client.create_review("mypool/pkgB", package_pr_number, event="APPROVED", body="Approval from ownerBB")
|
|
|
|
# 5. Check that the PR is NOT merged automatically and ownerB request remains
|
|
print("Waiting to ensure PR is NOT merged and ownerB request remains...")
|
|
for i in range(10):
|
|
pkg_details = gitea_env.get_pr_details("mypool/pkgB", package_pr_number)
|
|
reviews = gitea_env.list_reviews("mypool/pkgB", package_pr_number)
|
|
review_states = [(r["user"]["login"], r["state"]) for r in reviews]
|
|
print(f"Attempt {i+1}: Merged={pkg_details.get('merged')}, Reviews={review_states}")
|
|
time.sleep(2)
|
|
|
|
pkg_details = gitea_env.get_pr_details("mypool/pkgB", package_pr_number)
|
|
assert not pkg_details.get("merged"), "Package PR was merged automatically but it should NOT have been (ReviewRequired=true)."
|
|
|
|
reviews = gitea_env.list_reviews("mypool/pkgB", package_pr_number)
|
|
reviewers_requested = {r["user"]["login"] for r in reviews if r["state"] == "REQUEST_REVIEW"}
|
|
assert "ownerB" in reviewers_requested, f"ownerB's review request was removed, but it should have remained. All reviews: {[(r['user']['login'], r['state']) for r in reviews]}"
|
|
|
|
print("Confirmed: PR not merged and ownerB review request remains as expected.")
|