diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 9788ccae..a16d60ce 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -58,7 +58,7 @@ jobs: zypper -n lr --details grep -qi tumbleweed /etc/os-release && zypper -n dist-upgrade || zypper -n patch || zypper -n patch zypper -n install git-lfs - zypper -n install diffstat diffutils python3 python3-cryptography python3-pip python3-rpm python3-setuptools python3-urllib3 + zypper -n install diffstat diffutils git-core python3 python3-cryptography python3-pip python3-rpm python3-setuptools python3-urllib3 - name: 'Install packages (Fedora/CentOS)' if: ${{ contains(matrix.container, '/fedora:') || contains(matrix.container, '/centos:') }} @@ -66,7 +66,7 @@ jobs: dnf -y makecache dnf -y distro-sync dnf -y install git-lfs - dnf -y install diffstat diffutils python3 python3-cryptography python3-pip python3-rpm python3-setuptools python3-urllib3 + dnf -y install diffstat diffutils git-core python3 python3-cryptography python3-pip python3-rpm python3-setuptools python3-urllib3 - name: 'Install packages (Debian/Ubuntu)' if: ${{ contains(matrix.container, '/debian:') || contains(matrix.container, '/ubuntu:') }} @@ -74,7 +74,7 @@ jobs: apt-get -y update apt-get -y upgrade apt-get -y --no-install-recommends install git-lfs - apt-get -y --no-install-recommends install diffstat diffutils python3 python3-cryptography python3-pip python3-rpm python3-setuptools python3-urllib3 + apt-get -y --no-install-recommends install diffstat diffutils git-core python3 python3-cryptography python3-pip python3-rpm python3-setuptools python3-urllib3 - uses: actions/checkout@v3 diff --git a/contrib/osc.spec b/contrib/osc.spec index 3e0ac9eb..3e065090 100644 --- a/contrib/osc.spec +++ b/contrib/osc.spec @@ -57,6 +57,8 @@ BuildRequires: %{use_python_pkg}-rpm BuildRequires: %{use_python_pkg}-setuptools BuildRequires: %{use_python_pkg}-urllib3 BuildRequires: diffstat +# needed for git scm tests +BuildRequires: git-core Requires: %{use_python_pkg}-cryptography Requires: %{use_python_pkg}-rpm @@ -78,6 +80,10 @@ Recommends: diffstat Recommends: powerpc32 Recommends: sudo +# needed for building from git +Recommends: git-core +Recommends: git-lfs + # needed for `osc add ` Recommends: obs-service-recompress Recommends: obs-service-download_files diff --git a/osc/commandline.py b/osc/commandline.py index ce6c2fdc..2cf69148 100644 --- a/osc/commandline.py +++ b/osc/commandline.py @@ -30,6 +30,7 @@ from . import build as osc_build from . import cmdln from . import commands as osc_commands from . import conf +from . import git_scm from . import oscerr from . import store as osc_store from .core import * @@ -6933,7 +6934,7 @@ Please submit there instead, or use --nodevelproject to force direct submission. if no_repo: raise oscerr.WrongArgs("Repository is missing. Cannot guess build description without repository") apiurl = self.get_api_url() - project = store_read_project('.') + project = alternative_project or store_read_project('.') # some distros like Debian rename and move build to obs-build if not os.path.isfile('/usr/lib/build/queryconfig') and os.path.isfile('/usr/lib/obs-build/queryconfig'): queryconfig = '/usr/lib/obs-build/queryconfig' @@ -7197,12 +7198,17 @@ Please submit there instead, or use --nodevelproject to force direct submission. if len(args) > 3: raise oscerr.WrongArgs('Too many arguments') - store = osc_store.Store(Path.cwd()) + store = osc_store.get_store(Path.cwd(), print_warnings=True) store.assert_is_package() if opts.alternative_project == store.project: opts.alternative_project = None + # HACK: avoid calling some underlying store_*() functions from parse_repoarchdescr() method + # We'll fix parse_repoarchdescr() later because it requires a larger change + if not opts.alternative_project and isinstance(store, git_scm.GitStore): + opts.alternative_project = store.project + if len(args) == 0 and store.is_package and store.last_buildroot: # build env not specified, just read from last build attempt args = [store.last_buildroot[0], store.last_buildroot[1]] diff --git a/osc/core.py b/osc/core.py index 713bdaf1..c5c81d55 100644 --- a/osc/core.py +++ b/osc/core.py @@ -50,6 +50,7 @@ from . import _private from . import conf from . import meter from . import oscerr +from . import store as osc_store from .connection import http_request, http_GET, http_POST, http_PUT, http_DELETE from .store import Store from .util.helper import decode_list, decode_it, raw_input, _html_escape @@ -1239,7 +1240,8 @@ class Package: self.dir = workingdir or "." self.absdir = os.path.abspath(self.dir) - self.store = Store(self.dir) + self.store = osc_store.get_store(self.dir) + self.store.assert_is_package() self.storedir = os.path.join(self.absdir, store) self.progress_obj = progress_obj self.size_limit = size_limit @@ -1247,10 +1249,8 @@ class Package: if size_limit and size_limit == 0: self.size_limit = None - check_store_version(self.dir) - - self.prjname = store_read_project(self.dir) - self.name = store_read_package(self.dir) + self.prjname = self.store.project + self.name = self.store.package self.apiurl = self.store.apiurl self.update_datastructs() diff --git a/osc/git_scm/README.md b/osc/git_scm/README.md new file mode 100644 index 00000000..94e970ac --- /dev/null +++ b/osc/git_scm/README.md @@ -0,0 +1,4 @@ +# Warning + +This module provides EXPERIMENTAL and UNSTABLE support for git scm such as https://src.opensuse.org/. +The code may change or disappear without a prior notice! diff --git a/osc/git_scm/__init__.py b/osc/git_scm/__init__.py new file mode 100644 index 00000000..69e5511f --- /dev/null +++ b/osc/git_scm/__init__.py @@ -0,0 +1,7 @@ +import sys + +from .store import GitStore + + +def warn_experimental(): + print("WARNING: Using EXPERIMENTAL support for git scm. The functionality may change or disappear without a prior notice!", file=sys.stderr) diff --git a/osc/git_scm/store.py b/osc/git_scm/store.py new file mode 100644 index 00000000..b6f50441 --- /dev/null +++ b/osc/git_scm/store.py @@ -0,0 +1,151 @@ +import json +import os +import subprocess +import urllib.parse +from pathlib import Path + +from .. import conf as osc_conf +from .. import oscerr + + +class GitStore: + + @classmethod + def is_project_dir(cls, path): + try: + store = cls(path) + except oscerr.NoWorkingCopy: + return False + return store.is_project + + @classmethod + def is_package_dir(cls, path): + try: + store = cls(path) + except oscerr.NoWorkingCopy: + return False + return store.is_package + + def __init__(self, path, check=True): + self.path = path + self.abspath = os.path.abspath(self.path) + + # TODO: how to determine if the current git repo contains a project or a package? + self.is_project = False + self.is_package = os.path.exists(os.path.join(self.abspath, ".git")) + + self._package = None + self._project = None + + if check and not any([self.is_project, self.is_package]): + msg = f"Directory '{self.path}' is not a GIT working copy" + raise oscerr.NoWorkingCopy(msg) + + # TODO: decide if we need explicit 'git lfs pull' or not + # self._run_git(["lfs", "pull"]) + + def assert_is_project(self): + if not self.is_project: + msg = f"Directory '{self.path}' is not a GIT working copy of a project" + raise oscerr.NoWorkingCopy(msg) + + def assert_is_package(self): + if not self.is_package: + msg = f"Directory '{self.path}' is not a GIT working copy of a package" + raise oscerr.NoWorkingCopy(msg) + + def _run_git(self, args): + return subprocess.check_output(["git"] + args, encoding="utf-8", cwd=self.abspath).strip() + + @property + def apiurl(self): + # HACK: we're using the currently configured apiurl + return osc_conf.config["apiurl"] + + @property + def project(self): + if self._project is None: + # get project from the branch name + branch = self._run_git(["branch", "--show-current"]) + + # HACK: replace hard-coded mapping with metadata from git or the build service + if branch == "factory": + self._project = "openSUSE:Factory" + else: + raise RuntimeError(f"Couldn't map git branch '{branch}' to a project") + return self._project + + @project.setter + def project(self, value): + self._project = value + + @property + def package(self): + if self._package is None: + origin = self._run_git(["remote", "get-url", "origin"]) + self._package = Path(urllib.parse.urlsplit(origin).path).stem + return self._package + + @package.setter + def package(self, value): + self._package = value + + def _get_option(self, name): + try: + result = self._run_git(["config", "--local", "--get", f"osc.{name}"]) + except subprocess.CalledProcessError: + result = None + return result + + def _check_type(self, name, value, expected_type): + if not isinstance(value, expected_type): + raise TypeError(f"The option '{name}' should be {expected_type.__name__}, not {type(value).__name__}") + + def _set_option(self, name, value): + self._run_git(["config", "--local", f"osc.{name}", value]) + + def _unset_option(self, name): + try: + self._run_git(["config", "--local", "--unset", f"osc.{name}"]) + except subprocess.CalledProcessError: + pass + + def _get_dict_option(self, name): + result = self._get_option(name) + if result is None: + return None + result = json.loads(result) + self._check_type(name, result, dict) + return result + + def _set_dict_option(self, name, value): + if value is None: + self._unset_option(name) + return + self._check_type(name, value, dict) + value = json.dumps(value) + self._set_option(name, value) + + @property + def last_buildroot(self): + self.assert_is_package() + result = self._get_dict_option("last-buildroot") + if result is not None: + result = (result["repo"], result["arch"], result["vm_type"]) + return result + + @last_buildroot.setter + def last_buildroot(self, value): + self.assert_is_package() + if len(value) != 3: + raise ValueError("A tuple with exactly 3 items is expected: (repo, arch, vm_type)") + value = { + "repo": value[0], + "arch": value[1], + "vm_type": value[2], + } + self._set_dict_option("last-buildroot", value) + + @property + def scmurl(self): + return self._run_git(["remote", "get-url", "origin"]) diff --git a/osc/store.py b/osc/store.py index a4f8f4fa..e81cae63 100644 --- a/osc/store.py +++ b/osc/store.py @@ -10,7 +10,7 @@ from xml.etree import ElementTree as ET from . import oscerr from ._private import api - +from . import git_scm class Store: STORE_DIR = ".osc" @@ -309,3 +309,22 @@ class Store: else: root = self.read_xml_node("_meta", "project").getroot() return root + + +def get_store(path, check=True, print_warnings=False): + """ + Return a store object that wraps SCM in given `path`: + - Store for OBS SCM + - GitStore for Git SCM + """ + try: + store = Store(path, check) + except oscerr.NoWorkingCopy as ex: + try: + store = git_scm.GitStore(path, check) + if print_warnings: + git_scm.warn_experimental() + except oscerr.NoWorkingCopy as ex_git: + # raise the original exception, do not inform that we've tried git working copy + raise ex from None + return store diff --git a/setup.cfg b/setup.cfg index 69a5c06f..0987f758 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,6 +35,7 @@ packages = osc osc._private osc.commands + osc.git_scm osc.output osc.util install_requires = diff --git a/tests/test_git_scm_store.py b/tests/test_git_scm_store.py new file mode 100644 index 00000000..e884f4b2 --- /dev/null +++ b/tests/test_git_scm_store.py @@ -0,0 +1,49 @@ +import os +import shutil +import subprocess +import tempfile +import unittest + +from osc.git_scm.store import GitStore + + +class TestGitStore(unittest.TestCase): + def setUp(self): + self.tmpdir = tempfile.mkdtemp(prefix="osc_test") + os.chdir(self.tmpdir) + subprocess.check_output(["git", "init", "-b", "factory"]) + subprocess.check_output(["git", "remote", "add", "origin", "https://example.com/packages/my-package.git"]) + + def tearDown(self): + try: + shutil.rmtree(self.tmpdir) + except OSError: + pass + + def test_package(self): + store = GitStore(self.tmpdir) + self.assertEqual(store.package, "my-package") + + def test_project(self): + store = GitStore(self.tmpdir) + self.assertEqual(store.project, "openSUSE:Factory") + + def test_last_buildroot(self): + store = GitStore(self.tmpdir) + self.assertEqual(store.last_buildroot, None) + store.last_buildroot = ("repo", "arch", "vm_type") + + store = GitStore(self.tmpdir) + self.assertEqual(store.last_buildroot, ("repo", "arch", "vm_type")) + + def test_scmurl(self): + store = GitStore(self.tmpdir) + self.assertEqual(store.scmurl, "https://example.com/packages/my-package.git") + + +if not shutil.which("git"): + TestGitStore = unittest.skip("The 'git' executable is not available")(TestGitStore) + + +if __name__ == "__main__": + unittest.main()