diff --git a/.github/workflows/linters.yaml b/.github/workflows/linters.yaml index ecac06a5..2361d102 100644 --- a/.github/workflows/linters.yaml +++ b/.github/workflows/linters.yaml @@ -42,7 +42,10 @@ jobs: - name: 'Install packages' run: | sudo apt-get -y update - sudo apt-get -y --no-install-recommends install pylint python3-rpm python3-ruamel.yaml + sudo apt-get -y --no-install-recommends install python3-rpm python3-ruamel.yaml + # we're using the latest pylint from pypi + sudo pip3 config set global.break-system-packages 1 + sudo pip3 install pylint - uses: actions/checkout@v3 @@ -68,7 +71,10 @@ jobs: - name: 'Install packages' run: | sudo apt-get -y update - sudo apt-get -y --no-install-recommends install diffutils pylint python3-pip python3-rpm python3-ruamel.yaml + sudo apt-get -y --no-install-recommends install diffutils python3-pip python3-rpm python3-ruamel.yaml + # we're using the latest pylint from pypi + sudo pip3 config set global.break-system-packages 1 + sudo pip3 install pylint - uses: actions/checkout@v3 with: diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index f23cf756..92b75655 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -146,6 +146,11 @@ jobs: run: | podman pull ghcr.io/suse-autobuild/obs-server:latest + - name: "Configure git" + run: | + git config --global user.email "admin@example.com" + git config --global user.name "Admin" + - name: "Run tests" run: | cd behave diff --git a/behave/features/environment.py b/behave/features/environment.py index ff0525cf..60b0f00c 100644 --- a/behave/features/environment.py +++ b/behave/features/environment.py @@ -17,14 +17,14 @@ def after_step(context, step): def before_scenario(context, scenario): # truncate the logs before running any commands - proc = context.podman.container.exec(["bash", "-c", "find /srv/www/obs/api/log/ /srv/obs/log/ -name '*.log' -exec truncate --size=0 {} \\;"]) + proc = context.podman.container.exec(["bash", "-c", "find /srv/www/obs/api/log/ /srv/obs/log/ /var/log/gitea/ -name '*.log' -exec truncate --size=0 {} \\;"]) def after_scenario(context, scenario): if scenario.status == Status.failed: # the scenario has failed, dump server logs print("===== BEGIN: server logs ======") - proc = context.podman.container.exec(["bash", "-c", "tail -n +1 /srv/www/obs/api/log/*.log /srv/obs/log/*.log"]) + proc = context.podman.container.exec(["bash", "-c", "tail -n +1 /srv/www/obs/api/log/*.log /srv/obs/log/*.log /var/log/gitea/*.log"]) print(proc.stdout) print("===== END: server logs ======") diff --git a/behave/features/git-pr.feature b/behave/features/git-pr.feature new file mode 100644 index 00000000..e5793811 --- /dev/null +++ b/behave/features/git-pr.feature @@ -0,0 +1,115 @@ +Feature: `git-obs pr` command + + +Background: + Given I set working directory to "{context.osc.temp}" + And I execute git-obs with args "repo fork pool/test-GitPkgA" + And I execute git-obs with args "repo clone Admin/test-GitPkgA --no-ssh-strict-host-key-checking" + And I set working directory to "{context.osc.temp}/test-GitPkgA" + And I execute "sed -i 's@^\(Version.*\)@\1.1@' *.spec" + And I execute "git commit -m 'Change version' -a" + And I execute "git push" + And I execute git-obs with args "pr create --title 'Change version' --description='some text'" + + +@destructive +Scenario: List pull requests + When I execute git-obs with args "pr list pool/test-GitPkgA" + Then the exit code is 0 + And stdout matches + """ + ID : pool/test-GitPkgA#1 + URL : http://localhost:{context.podman.container.ports[gitea_http]}/pool/test-GitPkgA/pulls/1 + Title : Change version + State : open + Draft : no + Merged : no + Author : Admin \(admin@example.com\) + Source : Admin/test-GitPkgA, branch: factory, commit: .* + Description : some text + """ + And stderr is + """ + Using the following Gitea settings: + * Config path: {context.git_obs.config} + * Login (name of the entry in the config file): admin + * URL: http://localhost:{context.podman.container.ports[gitea_http]} + * User: Admin + + + Total entries: 1 + """ + + +@destructive +Scenario: Search pull requests + When I execute git-obs with args "pr search" + Then the exit code is 0 + And stdout matches + """ + ID : pool/test-GitPkgA#1 + URL : http://localhost:{context.podman.container.ports[gitea_http]}/pool/test-GitPkgA/pulls/1 + Title : Change version + State : open + Author : Admin \(admin@example.com\) + Description : some text + """ + And stderr is + """ + Using the following Gitea settings: + * Config path: {context.git_obs.config} + * Login (name of the entry in the config file): admin + * URL: http://localhost:{context.podman.container.ports[gitea_http]} + * User: Admin + + + Total entries: 1 + """ + + +@destructive +Scenario: Get a pull request + When I execute git-obs with args "pr get pool/test-GitPkgA#1" + Then the exit code is 0 + And stdout matches + """ + ID : pool/test-GitPkgA#1 + URL : http://localhost:{context.podman.container.ports[gitea_http]}/pool/test-GitPkgA/pulls/1 + Title : Change version + State : open + Draft : no + Merged : no + Author : Admin \(admin@example.com\) + Source : Admin/test-GitPkgA, branch: factory, commit: .* + Description : some text + """ + And stderr is + """ + Using the following Gitea settings: + * Config path: {context.git_obs.config} + * Login (name of the entry in the config file): admin + * URL: http://localhost:{context.podman.container.ports[gitea_http]} + * User: Admin + + Total entries: 1 + """ + + +@destructive +Scenario: Get a pull request that doesn't exist + When I execute git-obs with args "pr get does-not/exist#1" + Then the exit code is 1 + And stdout matches + """ + """ + And stderr is + """ + Using the following Gitea settings: + * Config path: {context.git_obs.config} + * Login (name of the entry in the config file): admin + * URL: http://localhost:{context.podman.container.ports[gitea_http]} + * User: Admin + + Total entries: 0 + ERROR: Couldn't retrieve the following pull requests: does-not/exist#1 + """ diff --git a/behave/features/git-repo-clone.feature b/behave/features/git-repo-clone.feature index ae2235b0..dc0cf6ca 100644 --- a/behave/features/git-repo-clone.feature +++ b/behave/features/git-repo-clone.feature @@ -7,7 +7,7 @@ Background: @destructive Scenario: Clone a git repo - When I execute git-obs with args "repo clone pool test-GitPkgA --no-ssh-strict-host-key-checking" + When I execute git-obs with args "repo clone pool/test-GitPkgA --no-ssh-strict-host-key-checking" Then the exit code is 0 And stdout is """ @@ -20,5 +20,8 @@ Scenario: Clone a git repo * URL: http://localhost:{context.podman.container.ports[gitea_http]} * User: Admin + Cloning git repo pool/test-GitPkgA ... Cloning into 'test-GitPkgA'... + + Total cloned repos: 1 """ diff --git a/behave/features/git-repo-fork.feature b/behave/features/git-repo-fork.feature index 516f7398..1edba94b 100644 --- a/behave/features/git-repo-fork.feature +++ b/behave/features/git-repo-fork.feature @@ -7,7 +7,7 @@ Background: @destructive Scenario: Fork a git repo - When I execute git-obs with args "repo fork pool test-GitPkgA" + When I execute git-obs with args "repo fork pool/test-GitPkgA" Then the exit code is 0 And stdout is """ @@ -22,12 +22,14 @@ Scenario: Fork a git repo Forking git repo pool/test-GitPkgA ... * Fork created: Admin/test-GitPkgA + + Total forked repos: 1 """ @destructive Scenario: Fork a git repo twice under different names - When I execute git-obs with args "repo fork pool test-GitPkgA" + When I execute git-obs with args "repo fork pool/test-GitPkgA" Then the exit code is 0 And stdout is """ @@ -42,8 +44,10 @@ Scenario: Fork a git repo twice under different names Forking git repo pool/test-GitPkgA ... * Fork created: Admin/test-GitPkgA + + Total forked repos: 1 """ - When I execute git-obs with args "repo fork pool test-GitPkgA --new-repo-name=new-package" + When I execute git-obs with args "repo fork pool/test-GitPkgA --new-repo-name=new-package" Then the exit code is 0 And stdout is """ @@ -59,12 +63,14 @@ Scenario: Fork a git repo twice under different names Forking git repo pool/test-GitPkgA ... * Fork already exists: Admin/test-GitPkgA * WARNING: Using an existing fork with a different name than requested + + Total forked repos: 1 """ @destructive Scenario: Fork a git repo from pool and fork someone else's fork of the same repo - When I execute git-obs with args "repo fork pool test-GitPkgA" + When I execute git-obs with args "repo fork pool/test-GitPkgA" Then the exit code is 0 And stdout is """ @@ -79,8 +85,10 @@ Scenario: Fork a git repo from pool and fork someone else's fork of the same rep Forking git repo pool/test-GitPkgA ... * Fork created: Admin/test-GitPkgA + + Total forked repos: 1 """ - When I execute git-obs with args "repo fork -G alice pool test-GitPkgA --new-repo-name=test-GitPkgA-alice" + When I execute git-obs with args "repo fork -G alice pool/test-GitPkgA --new-repo-name=test-GitPkgA-alice" Then the exit code is 0 And stdout is """ @@ -95,9 +103,11 @@ Scenario: Fork a git repo from pool and fork someone else's fork of the same rep Forking git repo pool/test-GitPkgA ... * Fork created: Alice/test-GitPkgA-alice + + Total forked repos: 1 """ # this succeeds with 202 and the requested fork is NOT created - When I execute git-obs with args "repo fork Alice test-GitPkgA-alice" + When I execute git-obs with args "repo fork Alice/test-GitPkgA-alice" Then the exit code is 0 And stdout is """ @@ -112,6 +122,8 @@ Scenario: Fork a git repo from pool and fork someone else's fork of the same rep Forking git repo Alice/test-GitPkgA-alice ... * Fork created: Admin/test-GitPkgA-alice + + Total forked repos: 1 """ - When I execute git-obs with args "repo clone Admin test-GitPkgA-alice --no-ssh-strict-host-key-checking" + When I execute git-obs with args "repo clone Admin/test-GitPkgA-alice --no-ssh-strict-host-key-checking" Then the exit code is 0 diff --git a/behave/features/steps/common.py b/behave/features/steps/common.py index 2eac831f..6e81af77 100644 --- a/behave/features/steps/common.py +++ b/behave/features/steps/common.py @@ -83,6 +83,12 @@ def run_in_context(context, cmd, can_fail=False, **run_args): raise AssertionError('Running command "%s" failed: %s' % (cmd, context.cmd_exitcode)) +@behave.step("I execute \"{command}\"") +def step_impl(context, command): + command = command.format(context=context) + run_in_context(context, command, can_fail=True) + + @behave.step("stdout contains \"{text}\"") def step_impl(context, text): if re.search(text.format(context=context), context.cmd_stdout): diff --git a/osc/commandline_git.py b/osc/commandline_git.py index f3f59686..4dcb10c7 100644 --- a/osc/commandline_git.py +++ b/osc/commandline_git.py @@ -1,4 +1,6 @@ +import argparse import os +import subprocess import sys import osc.commandline_common @@ -7,6 +9,36 @@ from . import oscerr from .output import print_msg +class OwnerRepoAction(argparse.Action): + def __call__(self, parser, namespace, value, option_string=None): + from . import gitea_api + + try: + if isinstance(value, list): + namespace_value = [gitea_api.Repo.split_id(i) for i in value] + else: + namespace_value = gitea_api.Repo.split_id(value) + except ValueError as e: + raise argparse.ArgumentError(self, str(e)) + + setattr(namespace, self.dest, namespace_value) + + +class OwnerRepoPullAction(argparse.Action): + def __call__(self, parser, namespace, value, option_string=None): + from . import gitea_api + + try: + if isinstance(value, list): + namespace_value = [gitea_api.PullRequest.split_id(i) for i in value] + else: + namespace_value = gitea_api.PullRequest.split_id(value) + except ValueError as e: + raise argparse.ArgumentError(self, str(e)) + + setattr(namespace, self.dest, namespace_value) + + class GitObsCommand(osc.commandline_common.Command): @property def gitea_conf(self): @@ -28,16 +60,20 @@ class GitObsCommand(osc.commandline_common.Command): print(f" * User: {self.gitea_login.user}", file=sys.stderr) print("", file=sys.stderr) - def add_argument_owner(self): + def add_argument_owner_repo(self, **kwargs): self.add_argument( - "owner", - help="Name of the repository owner (login, org)", + "owner_repo", + action=OwnerRepoAction, + help="Owner and repo: (format: /)", + **kwargs, ) - def add_argument_repo(self): + def add_argument_owner_repo_pull(self, **kwargs): self.add_argument( - "repo", - help="Name of the repository", + "owner_repo_pull", + action=OwnerRepoPullAction, + help="Owner, repo and pull request number (format: /#)", + **kwargs, ) def add_argument_new_repo_name(self): @@ -132,7 +168,9 @@ def main(): except oscerr.OscBaseError as e: print_msg(str(e), print_to="error") sys.exit(1) - + except subprocess.CalledProcessError as e: + print_msg(str(e), print_to="error") + sys.exit(1) if __name__ == "__main__": main() diff --git a/osc/commands_git/pr.py b/osc/commands_git/pr.py new file mode 100644 index 00000000..eeb85df7 --- /dev/null +++ b/osc/commands_git/pr.py @@ -0,0 +1,19 @@ +import osc.commandline_git + + +# we decided not to use the command name 'pull' because that could be confused +# with the completely unrelated 'git pull' command + + +class PullRequestCommand(osc.commandline_git.GitObsCommand): + """ + Manage pull requests + """ + + name = "pr" + + def init_arguments(self): + pass + + def run(self, args): + self.parser.print_help() diff --git a/osc/commands_git/pr_create.py b/osc/commands_git/pr_create.py new file mode 100644 index 00000000..bb187b1a --- /dev/null +++ b/osc/commands_git/pr_create.py @@ -0,0 +1,212 @@ +import os +import re +import subprocess +import sys +from typing import Optional + +import osc.commandline_git + + +def get_editor() -> str: + import shutil + + editor = os.getenv("EDITOR", None) + if editor: + candidates = [editor] + else: + candidates = ["vim", "vi"] + + editor_path = None + for i in candidates: + editor_path = shutil.which(i) + if editor_path: + break + + if not editor_path: + raise RuntimeError(f"Unable to start editor '{candidates[0]}'") + + return editor_path + + +def run_editor(file_path: str): + cmd = [get_editor(), file_path] + subprocess.run(cmd) + + +def edit_message(template: Optional[str] = None) -> str: + import tempfile + + with tempfile.NamedTemporaryFile(mode="w+", encoding="utf-8", prefix="git_obs_message_") as f: + if template: + f.write(template) + f.flush() + + run_editor(f.name) + + f.seek(0) + return f.read() + + +NEW_PULL_REQUEST_TEMPLATE = """ +# title +{title} + +# description +{description} + +# +# Please enter pull request title and description in the following format: +# +# <blank line> +# <description> +# +# Lines starting with '#' will be ignored, and an empty message aborts the operation. +# +# Creating {source_owner}/{source_repo}#{source_branch} -> {target_owner}/{target_repo}#{target_branch} +# +""".lstrip() + + +class PullRequestCreateCommand(osc.commandline_git.GitObsCommand): + """ + Create a pull request + """ + + name = "create" + parent = "PullRequestCommand" + + def init_arguments(self): + self.add_argument( + "--title", + metavar="TEXT", + help="Pull request title", + ) + self.add_argument( + "--description", + metavar="TEXT", + help="Pull request description (body)", + ) + self.add_argument( + "--source-owner", + metavar="OWNER", + help="Owner of the source repo (default: derived from remote URL in local git repo)", + ) + self.add_argument( + "--source-repo", + metavar="REPO", + help="Name of the source repo (default: derived from remote URL in local git repo)", + ) + self.add_argument( + "--source-branch", + metavar="BRANCH", + help="Source branch (default: the current branch in local git repo)", + ) + self.add_argument( + "--target-branch", + metavar="BRANCH", + help="Target branch (default: derived from the current branch in local git repo)", + ) + + def run(self, args): + from osc import gitea_api + + # the source args are optional, but if one of them is used, the others must be used too + source_args = (args.source_owner, args.source_repo, args.source_branch) + if sum((int(i is not None) for i in source_args)) not in (0, len(source_args)): + self.parser.error("All of the following options must be used together: --source-owner, --source-repo, --source-branch") + + self.print_gitea_settings() + + use_local_git = args.source_owner is None + + if use_local_git: + # local git repo + git = gitea_api.Git(".") + local_owner, local_repo = git.get_owner_repo() + local_branch = git.current_branch + local_rev = git.get_branch_head(local_branch) + + # remote git repo - source + if use_local_git: + source_owner = local_owner + source_repo = local_repo + source_branch = local_branch + else: + source_owner = args.source_owner + source_repo = args.source_repo + source_branch = args.source_branch + source_repo_data = gitea_api.Repo.get(self.gitea_conn, source_owner, source_repo).json() + source_branch_data = gitea_api.Branch.get(self.gitea_conn, source_owner, source_repo, source_branch).json() + source_rev = source_branch_data["commit"]["id"] + + # remote git repo - target + target_owner, target_repo = source_repo_data["parent"]["full_name"].split("/") + + if source_branch.startswith("for/"): + # source branch name format: for/<target-branch>/<what-the-branch-name-would-normally-be> + target_branch = source_branch.split("/")[1] + else: + target_branch = source_branch + + target_branch_data = gitea_api.Branch.get(self.gitea_conn, target_owner, target_repo, target_branch).json() + target_rev = target_branch_data["commit"]["id"] + + print("Creating a pull request ...", file=sys.stderr) + if use_local_git: + print(f" * Local git: branch: {local_branch}, rev: {local_rev}", file=sys.stderr) + print(f" * Source: {source_owner}/{source_repo}, branch: {source_branch}, rev: {source_rev}", file=sys.stderr) + print(f" * Target: {target_owner}/{target_repo}, branch: {target_branch}, rev: {target_rev}", file=sys.stderr) + + if use_local_git and local_rev != source_rev: + from osc.output import tty + print(f"{tty.colorize('ERROR', 'red,bold')}: Local commit doesn't correspond with the latest commit in the remote source branch") + sys.exit(1) + + if source_rev == target_rev: + from osc.output import tty + print(f"{tty.colorize('ERROR', 'red,bold')}: Source and target are identical, make and push changes to the remote source repo first") + sys.exit(1) + + title = args.title or "" + description = args.description or "" + + if not title or not description: + # TODO: add list of commits and list of changed files to the template; requires local git repo + message = edit_message(template=NEW_PULL_REQUEST_TEMPLATE.format(**locals())) + + # remove comments + message = "\n".join([i for i in message.splitlines() if not i.startswith("#")]) + + # strip leading and trailing spaces + message = message.strip() + + if not message: + raise RuntimeError("Aborting operation due to empty title and description.") + + parts = re.split(r"\n\n", message, 1) + if len(parts) == 1: + # empty description + title = parts[0] + description = "" + else: + title = parts[0] + description = parts[1] + + title = title.strip() + description = description.strip() + + pull = gitea_api.PullRequest.create( + self.gitea_conn, + target_owner=target_owner, + target_repo=target_repo, + target_branch=target_branch, + source_owner=source_owner, + # source_repo is not required because the information lives in Gitea database + source_branch=source_branch, + title=title, + description=description, + ).json() + + print("", file=sys.stderr) + print("Pull request created:", file=sys.stderr) + print(gitea_api.PullRequest.to_human_readable_string(pull)) diff --git a/osc/commands_git/pr_get.py b/osc/commands_git/pr_get.py new file mode 100644 index 00000000..1db77abd --- /dev/null +++ b/osc/commands_git/pr_get.py @@ -0,0 +1,59 @@ +import sys + +import osc.commandline_git + + +class PullRequestGetCommand(osc.commandline_git.GitObsCommand): + """ + Get details about the specified pull requests + """ + + name = "get" + aliases = ["show"] # for compatibility with osc + parent = "PullRequestCommand" + + def init_arguments(self): + self.add_argument_owner_repo_pull(nargs="+") + self.add_argument( + "-p", + "--patch", + action="store_true", + help="Show patches associated with the pull requests", + ) + + def run(self, args): + from osc import gitea_api + from osc.core import highlight_diff + from osc.output import tty + + self.print_gitea_settings() + + num_entries = 0 + failed_entries = [] + for owner, repo, pull in args.owner_repo_pull: + try: + pr = gitea_api.PullRequest.get(self.gitea_conn, owner, repo, pull).json() + num_entries += 1 + except gitea_api.GiteaException as e: + if e.status == 404: + failed_entries.append(f"{owner}/{repo}#{pull}") + continue + raise + print(gitea_api.PullRequest.to_human_readable_string(pr)) + + if args.patch: + print("") + print(tty.colorize("Patch:", "bold")) + patch = gitea_api.PullRequest.get_patch(self.gitea_conn, owner, repo, pull).data + patch = highlight_diff(patch) + print(patch.decode("utf-8")) + + print() + + print(f"Total entries: {num_entries}", file=sys.stderr) + if failed_entries: + print( + f"{tty.colorize('ERROR', 'red,bold')}: Couldn't retrieve the following pull requests: {', '.join(failed_entries)}", + file=sys.stderr, + ) + sys.exit(1) diff --git a/osc/commands_git/pr_list.py b/osc/commands_git/pr_list.py new file mode 100644 index 00000000..3d3ed228 --- /dev/null +++ b/osc/commands_git/pr_list.py @@ -0,0 +1,38 @@ +import sys + +import osc.commandline_git + + +class PullRequestListCommand(osc.commandline_git.GitObsCommand): + """ + List pull requests in a repository + """ + + name = "list" + parent = "PullRequestCommand" + + def init_arguments(self): + self.add_argument_owner_repo(nargs="+") + self.add_argument( + "--state", + choices=["open", "closed", "all"], + default="open", + help="State of the pull requests (default: open)", + ) + + def run(self, args): + from osc import gitea_api + + self.print_gitea_settings() + + total_entries = 0 + for owner, repo in args.owner_repo: + data = gitea_api.PullRequest.list(self.gitea_conn, owner, repo, state=args.state).json() + total_entries += len(data) + + text = gitea_api.PullRequest.list_to_human_readable_string(data, sort=True) + if text: + print(text) + print("", file=sys.stderr) + + print(f"Total entries: {total_entries}", file=sys.stderr) diff --git a/osc/commands_git/pr_search.py b/osc/commands_git/pr_search.py new file mode 100644 index 00000000..b5a0c581 --- /dev/null +++ b/osc/commands_git/pr_search.py @@ -0,0 +1,79 @@ +import sys + +import osc.commandline_git + + +class PullRequestSearchCommand(osc.commandline_git.GitObsCommand): + """ + Search pull requests in the whole gitea instance + """ + + name = "search" + parent = "PullRequestCommand" + + def init_arguments(self): + self.add_argument( + "--state", + choices=["open", "closed"], + default="open", + help="Filter by state: open, closed (default: open)", + ) + self.add_argument( + "--title", + help="Filter by substring in title", + ) + self.add_argument( + "--owner", + help="Filter by owner of the repository associated with the pull requests", + ) + self.add_argument( + "--label", + dest="labels", + metavar="LABEL", + action="append", + help="Filter by associated labels. Non existent labels are discarded. Can be specified multiple times.", + ) + self.add_argument( + "--assigned", + action="store_true", + help="Filter pull requests assigned to you", + ) + self.add_argument( + "--created", + action="store_true", + help="Filter pull requests created by you", + ) + self.add_argument( + "--mentioned", + action="store_true", + help="Filter pull requests mentioning you", + ) + self.add_argument( + "--review-requested", + action="store_true", + help="Filter pull requests requesting your review", + ) + + def run(self, args): + from osc import gitea_api + + self.print_gitea_settings() + + data = gitea_api.PullRequest.search( + self.gitea_conn, + state=args.state, + title=args.title, + owner=args.owner, + labels=args.labels, + assigned=args.assigned, + created=args.created, + mentioned=args.mentioned, + review_requested=args.review_requested, + ).json() + + text = gitea_api.PullRequest.list_to_human_readable_string(data, sort=True) + if text: + print(text) + print("", file=sys.stderr) + + print(f"Total entries: {len(data)}", file=sys.stderr) diff --git a/osc/commands_git/repo_clone.py b/osc/commands_git/repo_clone.py index 80be23ec..9e9cd7d6 100644 --- a/osc/commands_git/repo_clone.py +++ b/osc/commands_git/repo_clone.py @@ -1,17 +1,22 @@ +import subprocess +import sys + import osc.commandline_git class RepoCloneCommand(osc.commandline_git.GitObsCommand): """ Clone a git repo + + NOTE: Some of the options may result in setting "core.sshCommand" + config option in the git repository." """ name = "clone" parent = "RepoCommand" def init_arguments(self): - self.add_argument_owner() - self.add_argument_repo() + self.add_argument_owner_repo(nargs="+") self.add_argument( "-a", @@ -42,15 +47,42 @@ class RepoCloneCommand(osc.commandline_git.GitObsCommand): def run(self, args): from osc import gitea_api + from osc.output import tty + self.print_gitea_settings() - gitea_api.Repo.clone( - self.gitea_conn, - args.owner, - args.repo, - directory=args.directory, - anonymous=args.anonymous, - add_remotes=True, - ssh_private_key_path=self.gitea_login.ssh_key or args.ssh_key, - ssh_strict_host_key_checking=not(args.no_ssh_strict_host_key_checking), - ) + if len(args.owner_repo) > 1 and args.directory: + self.parser.error("The --directory option cannot be used with multiple repos") + + num_entries = 0 + failed_entries = [] + for owner, repo in args.owner_repo: + print(f"Cloning git repo {owner}/{repo} ...", file=sys.stderr) + try: + gitea_api.Repo.clone( + self.gitea_conn, + owner, + repo, + directory=args.directory, + anonymous=args.anonymous, + add_remotes=True, + ssh_private_key_path=args.ssh_key or self.gitea_login.ssh_key, + ssh_strict_host_key_checking=not(args.no_ssh_strict_host_key_checking), + ) + num_entries += 1 + except gitea_api.GiteaException as e: + if e.status == 404: + print(f" * {tty.colorize('ERROR', 'red,bold')}: Repo doesn't exist: {owner}/{repo}", file=sys.stderr) + failed_entries.append(f"{owner}/{repo}") + continue + raise + except subprocess.CalledProcessError as e: + print(f" * {tty.colorize('ERROR', 'red,bold')}: git clone failed", file=sys.stderr) + failed_entries.append(f"{owner}/{repo}") + continue + + print("", file=sys.stderr) + print(f"Total cloned repos: {num_entries}", file=sys.stderr) + if failed_entries: + print(f"{tty.colorize('ERROR', 'red,bold')}: Couldn't clone the following repos: {', '.join(failed_entries)}", file=sys.stderr) + sys.exit(1) diff --git a/osc/commands_git/repo_fork.py b/osc/commands_git/repo_fork.py index b05e5516..8d90e2cd 100644 --- a/osc/commands_git/repo_fork.py +++ b/osc/commands_git/repo_fork.py @@ -12,8 +12,7 @@ class RepoForkCommand(osc.commandline_git.GitObsCommand): parent = "RepoCommand" def init_arguments(self): - self.add_argument_owner() - self.add_argument_repo() + self.add_argument_owner_repo(nargs="+") self.add_argument_new_repo_name() def run(self, args): @@ -22,15 +21,35 @@ class RepoForkCommand(osc.commandline_git.GitObsCommand): self.print_gitea_settings() - print(f"Forking git repo {args.owner}/{args.repo} ...", file=sys.stderr) - try: - response = gitea_api.Fork.create(self.gitea_conn, args.owner, args.repo, new_repo_name=args.new_repo_name) - repo = response.json() - fork_owner = repo["owner"]["login"] - fork_repo = repo["name"] - print(f" * Fork created: {fork_owner}/{fork_repo}", file=sys.stderr) - except gitea_api.ForkExists as e: - fork_owner = e.fork_owner - fork_repo = e.fork_repo - print(f" * Fork already exists: {fork_owner}/{fork_repo}", file=sys.stderr) - print(f" * {tty.colorize('WARNING', 'yellow,bold')}: Using an existing fork with a different name than requested", file=sys.stderr) + if len(args.owner_repo) > 1 and args.new_repo_name: + self.parser.error("The --new-repo-name option cannot be used with multiple repos") + + num_entries = 0 + failed_entries = [] + for owner, repo in args.owner_repo: + print(f"Forking git repo {owner}/{repo} ...", file=sys.stderr) + try: + response = gitea_api.Fork.create(self.gitea_conn, owner, repo, new_repo_name=args.new_repo_name) + repo = response.json() + fork_owner = repo["owner"]["login"] + fork_repo = repo["name"] + print(f" * Fork created: {fork_owner}/{fork_repo}", file=sys.stderr) + num_entries += 1 + except gitea_api.ForkExists as e: + fork_owner = e.fork_owner + fork_repo = e.fork_repo + print(f" * Fork already exists: {fork_owner}/{fork_repo}", file=sys.stderr) + print(f" * {tty.colorize('WARNING', 'yellow,bold')}: Using an existing fork with a different name than requested", file=sys.stderr) + num_entries += 1 + except gitea_api.GiteaException as e: + if e.status == 404: + print(f" * {tty.colorize('ERROR', 'red,bold')}: Repo doesn't exist: {owner}/{repo}", file=sys.stderr) + failed_entries.append(f"{owner}/{repo}") + continue + raise + + print("", file=sys.stderr) + print(f"Total forked repos: {num_entries}", file=sys.stderr) + if failed_entries: + print(f"{tty.colorize('ERROR', 'red,bold')}: Couldn't fork the following repos: {', '.join(failed_entries)}", file=sys.stderr) + sys.exit(1) diff --git a/osc/core.py b/osc/core.py index feb7592d..e64ee139 100644 --- a/osc/core.py +++ b/osc/core.py @@ -1935,7 +1935,7 @@ def get_default_editor(): def format_diff_line(line): - if line.startswith(b"+++") or line.startswith(b"---") or line.startswith(b"Index:"): + if line.startswith(b"+++ ") or line.startswith(b"--- ") or line.startswith(b"Index:"): line = b"\x1b[1m" + line + b"\x1b[0m" elif line.startswith(b"+"): line = b"\x1b[32m" + line + b"\x1b[0m" diff --git a/osc/gitea_api/__init__.py b/osc/gitea_api/__init__.py index 65058711..ff690db6 100644 --- a/osc/gitea_api/__init__.py +++ b/osc/gitea_api/__init__.py @@ -7,6 +7,8 @@ from .branch import Branch from .conf import Config from .conf import Login from .fork import Fork +from .git import Git +from .pr import PullRequest from .ssh_key import SSHKey from .repo import Repo from .user import User diff --git a/osc/gitea_api/connection.py b/osc/gitea_api/connection.py index 590d1064..e78d6ea7 100644 --- a/osc/gitea_api/connection.py +++ b/osc/gitea_api/connection.py @@ -1,18 +1,17 @@ import copy import http.client import json +import time import urllib.parse from typing import Optional import urllib3 +import urllib3.exceptions import urllib3.response from .conf import Login -# TODO: retry, backoff, connection pool? - - class GiteaHTTPResponse: """ A ``urllib3.response.HTTPResponse`` wrapper @@ -52,6 +51,16 @@ class Connection: self.port = alternative_port if alternative_port else parsed_url.port self.conn = ConnectionClass(host=self.host, port=self.port) + # retries; variables are named according to urllib3 + self.retry_count = 3 + self.retry_backoff_factor = 2 + self.retry_status_forcelist = ( + 500, # Internal Server Error + 502, # Bad Gateway + 503, # Service Unavailable + 504, # Gateway Timeout + ) + if hasattr(self.conn, "set_cert"): # needed to avoid: AttributeError: 'HTTPSConnection' object has no attribute 'assert_hostname'. Did you mean: 'server_hostname'? self.conn.set_cert() @@ -95,8 +104,27 @@ class Connection: body = json.dumps(json_data) if json_data else None - self.conn.request(method, url, body, headers) - response = self.conn.getresponse() + for retry in range(1 + self.retry_count): + # 1 regular request + ``self.retry_count`` retries + try: + self.conn.request(method, url, body, headers) + response = self.conn.getresponse() + + if response.status not in self.retry_status_forcelist: + # we are happy with the response status -> use the response + break + + if retry >= self.retry_count: + # we have reached maximum number of retries -> use the response + break + + except (urllib3.exceptions.HTTPError, ConnectionResetError): + if retry >= self.retry_count: + raise + + # {backoff factor} * (2 ** ({number of previous retries})) + time.sleep(self.retry_backoff_factor * (2 ** retry)) + self.conn.close() if isinstance(response, http.client.HTTPResponse): result = GiteaHTTPResponse(urllib3.response.HTTPResponse.from_httplib(response)) diff --git a/osc/gitea_api/fork.py b/osc/gitea_api/fork.py index 916ca5da..b59cc57e 100644 --- a/osc/gitea_api/fork.py +++ b/osc/gitea_api/fork.py @@ -56,8 +56,8 @@ class Fork: return conn.request("POST", url, json_data=json_data) except GiteaException as e: # use ForkExists exception to parse fork_owner and fork_repo from the response - fork_exists_exception = ForkExists(e.response, owner, repo) if e.status == 409: + fork_exists_exception = ForkExists(e.response, owner, repo) if exist_ok: from . import Repo return Repo.get(conn, fork_exists_exception.fork_owner, fork_exists_exception.fork_repo) diff --git a/osc/gitea_api/git.py b/osc/gitea_api/git.py new file mode 100644 index 00000000..9b17ec09 --- /dev/null +++ b/osc/gitea_api/git.py @@ -0,0 +1,31 @@ +import os +import subprocess +import urllib +from typing import Tuple + + +class Git: + def __init__(self, workdir): + self.abspath = os.path.abspath(workdir) + + def _run_git(self, args) -> str: + return subprocess.check_output(["git"] + args, encoding="utf-8", cwd=self.abspath).strip() + + @property + def current_branch(self) -> str: + return self._run_git(["branch", "--show-current"]) + + def get_branch_head(self, branch: str) -> str: + return self._run_git(["rev-parse", branch]) + + def get_remote_url(self, name: str = "origin") -> str: + return self._run_git(["remote", "get-url", name]) + + def get_owner_repo(self, remote: str = "origin") -> Tuple[str, str]: + remote_url = self.get_remote_url(name=remote) + parsed_remote_url = urllib.parse.urlparse(remote_url) + path = parsed_remote_url.path + if path.endswith(".git"): + path = path[:-4] + owner, repo = path.strip("/").split("/")[:2] + return owner, repo diff --git a/osc/gitea_api/pr.py b/osc/gitea_api/pr.py new file mode 100644 index 00000000..4b91fc15 --- /dev/null +++ b/osc/gitea_api/pr.py @@ -0,0 +1,212 @@ +import re +from typing import List +from typing import Optional +from typing import Tuple + +from .connection import Connection +from .connection import GiteaHTTPResponse + + +class PullRequest: + @classmethod + def cmp(cls, entry: dict): + if "base" in entry: + # a proper pull request + return entry["base"]["repo"]["full_name"], entry["number"] + else: + # an issue without pull request details + return entry["repository"]["full_name"], entry["number"] + + @classmethod + def split_id(cls, pr_id: str) -> Tuple[str, str, str]: + """ + Split <owner>/<repo>#<number> into individual components and return them in a tuple. + """ + match = re.match(r"^([^/]+)/([^/]+)#([0-9]+)$", pr_id) + if not match: + raise ValueError(f"Invalid pull request id: {pr_id}") + return match.groups() + + @classmethod + def to_human_readable_string(cls, entry: dict): + from osc.output import KeyValueTable + from . import User + + def yes_no(value): + return "yes" if value else "no" + + if "base" in entry: + # a proper pull request + entry_id = f"{entry['base']['repo']['full_name']}#{entry['number']}" + is_pull_request = True + else: + # an issue without pull request details + entry_id = f"{entry['repository']['full_name']}#{entry['number']}" + is_pull_request = False + + # HACK: search API returns issues, the URL needs to be transformed to a pull request URL + entry_url = entry["url"] + entry_url = re.sub(r"^(.*)/api/v1/repos/(.+/.+)/issues/([0-9]+)$", r"\1/\2/pulls/\3", entry_url) + + table = KeyValueTable() + table.add("ID", entry_id, color="bold") + table.add("URL", f"{entry_url}") + table.add("Title", f"{entry['title']}") + table.add("State", entry["state"]) + if is_pull_request: + table.add("Draft", yes_no(entry["draft"])) + table.add("Merged", yes_no(entry["merged"])) + table.add("Author", f"{User.to_login_full_name_email_string(entry['user'])}") + if is_pull_request: + table.add("Source", f"{entry['head']['repo']['full_name']}, branch: {entry['head']['ref']}, commit: {entry['head']['sha']}") + table.add("Description", entry["body"]) + + return str(table) + + @classmethod + def list_to_human_readable_string(cls, entries: List, sort: bool = False): + if sort: + entries = sorted(entries, key=cls.cmp) + result = [] + for entry in entries: + result.append(cls.to_human_readable_string(entry)) + return "\n\n".join(result) + + @classmethod + def create( + cls, + conn: Connection, + *, + target_owner: str, + target_repo: str, + target_branch: str, + source_owner: str, + source_branch: str, + title: str, + description: Optional[str] = None, + ) -> GiteaHTTPResponse: + """ + Create a pull request to ``owner``/``repo`` to the ``base`` branch. + The pull request comes from a fork. The fork repo name is determined from gitea database. + + :param conn: Gitea ``Connection`` instance. + :param target_owner: Owner of the target repo. + :param target_repo: Name of the target repo. + :param target_branch: Name of the target branch in the target repo. + :param source_owner: Owner of the source (forked) repo. + :param source_branch: Name of the source branch in the source (forked) repo. + :param title: Pull request title. + :param description: Pull request description. + """ + url = conn.makeurl("repos", target_owner, target_repo, "pulls") + data = { + "base": target_branch, + "head": f"{source_owner}:{source_branch}", + "title": title, + "body": description, + } + return conn.request("POST", url, json_data=data) + + @classmethod + def get( + cls, + conn: Connection, + owner: str, + repo: str, + number: str, + ) -> GiteaHTTPResponse: + """ + Get a pull request. + + :param conn: Gitea ``Connection`` instance. + :param owner: Owner of the repo. + :param repo: Name of the repo. + :param number: Number of the pull request in the repo. + """ + url = conn.makeurl("repos", owner, repo, "pulls", number) + return conn.request("GET", url) + + @classmethod + def list( + cls, + conn: Connection, + owner: str, + repo: str, + *, + state: Optional[str] = "open", + ) -> GiteaHTTPResponse: + """ + List pull requests in a repo. + + :param conn: Gitea ``Connection`` instance. + :param owner: Owner of the repo. + :param repo: Name of the repo. + :param state: Filter by state: open, closed, all. Defaults to open. + """ + if state == "all": + state = None + + q = { + "state": state, + } + url = conn.makeurl("repos", owner, repo, "pulls", query=q) + return conn.request("GET", url) + + @classmethod + def search( + cls, + conn: Connection, + *, + state: str = "open", + title: Optional[str] = None, + owner: Optional[str] = None, + labels: Optional[List[str]] = None, + assigned: bool = False, + created: bool = False, + mentioned: bool = False, + review_requested: bool = False, + ) -> GiteaHTTPResponse: + """ + Search pull requests. + :param conn: Gitea ``Connection`` instance. + :param state: Filter by state: open, closed. Defaults to open. + :param title: Filter by substring in title. + :param owner: Filter by owner of the repository associated with the pull requests. + :param labels: Filter by associated labels. Non existent labels are discarded. + :param assigned: Filter pull requests assigned to you. + :param created: Filter pull requests created by you. + :param mentioned: Filter pull requests mentioning you. + :param review_requested: Filter pull requests requesting your review. + """ + q = { + "type": "pulls", + "state": state, + "q": title, + "owner": owner, + "labels": ",".join(labels) if labels else None, + "assigned": assigned, + "created": created, + "mentioned": mentioned, + "review_requested": review_requested, + } + url = conn.makeurl("repos", "issues", "search", query=q) + return conn.request("GET", url) + + @classmethod + def get_patch( + cls, + conn: Connection, + owner: str, + repo: str, + number: str, + ) -> GiteaHTTPResponse: + """ + Get a patch associated with a pull request. + + :param conn: Gitea ``Connection`` instance. + :param owner: Owner of the repo. + :param repo: Name of the repo. + :param number: Number of the pull request in the repo. + """ + url = conn.makeurl("repos", owner, repo, "pulls", f"{number}.patch") + return conn.request("GET", url) diff --git a/osc/gitea_api/repo.py b/osc/gitea_api/repo.py index a2205113..10607322 100644 --- a/osc/gitea_api/repo.py +++ b/osc/gitea_api/repo.py @@ -1,6 +1,8 @@ import os +import re import subprocess from typing import Optional +from typing import Tuple from .connection import Connection from .connection import GiteaHTTPResponse @@ -8,6 +10,16 @@ from .user import User class Repo: + @classmethod + def split_id(cls, repo_id: str) -> Tuple[str, str]: + """ + Split <owner>/<repo> into individual components and return them in a tuple. + """ + match = re.match(r"^([^/]+)/([^/]+)$", repo_id) + if not match: + raise ValueError(f"Invalid repo id: {repo_id}") + return match.groups() + @classmethod def get( cls, @@ -77,22 +89,28 @@ class Repo: fork = forks[0] remotes["fork"] = fork["clone_url"] if anonymous else fork["ssh_url"] - env = os.environ.copy() ssh_args = [] + env = os.environ.copy() + if ssh_private_key_path: - ssh_args += [f"-i {shlex.quote(ssh_private_key_path)}"] + ssh_args += [ + # avoid guessing the ssh key, use the specified one + "-o IdentitiesOnly=yes", + f"-o IdentityFile={shlex.quote(ssh_private_key_path)}", + ] + if not ssh_strict_host_key_checking: ssh_args += [ "-o StrictHostKeyChecking=no", "-o UserKnownHostsFile=/dev/null", "-o LogLevel=ERROR", ] + if ssh_args: env["GIT_SSH_COMMAND"] = f"ssh {' '.join(ssh_args)}" # clone cmd = ["git", "clone", clone_url, directory] - subprocess.run(cmd, cwd=cwd, env=env, check=True) # setup remotes @@ -100,4 +118,10 @@ class Repo: cmd = ["git", "-C", directory_abspath, "remote", "add", name, url] subprocess.run(cmd, cwd=cwd, check=True) + # store used ssh args (GIT_SSH_COMMAND) in the local git config + # to allow seamlessly running ``git push`` and other commands + if ssh_args: + cmd = ["git", "-C", directory_abspath, "config", "core.sshCommand", f"echo 'Using core.sshCommand: {env['GIT_SSH_COMMAND']}' >&2; {env['GIT_SSH_COMMAND']}"] + subprocess.run(cmd, cwd=cwd, check=True) + return directory_abspath diff --git a/osc/gitea_api/user.py b/osc/gitea_api/user.py index e946c467..01c08678 100644 --- a/osc/gitea_api/user.py +++ b/osc/gitea_api/user.py @@ -3,6 +3,18 @@ from .connection import GiteaHTTPResponse class User: + @classmethod + def to_full_name_email_string(cls, data): + full_name = data["full_name"] + email = data["email"] + if full_name: + return f"{full_name} <{email}>" + return email + + @classmethod + def to_login_full_name_email_string(cls, data): + return f"{data['login']} ({cls.to_full_name_email_string(data)})" + @classmethod def get( cls,