Merge pull request #1344 from dmach/build-from-git

Support building directly from git

.github/workflows/tests.yaml

@@ -58,7 +58,7 @@ jobs:
             zypper -n lr --details
             grep -qi tumbleweed /etc/os-release && zypper -n dist-upgrade || zypper -n patch || zypper -n patch
             zypper -n install git-lfs
-            zypper -n install diffstat diffutils python3 python3-cryptography python3-pip python3-rpm python3-setuptools python3-urllib3
+            zypper -n install diffstat diffutils git-core python3 python3-cryptography python3-pip python3-rpm python3-setuptools python3-urllib3
 
       - name: 'Install packages (Fedora/CentOS)'
         if: ${{ contains(matrix.container, '/fedora:') || contains(matrix.container, '/centos:') }}
@@ -66,7 +66,7 @@ jobs:
             dnf -y makecache
             dnf -y distro-sync
             dnf -y install git-lfs
-            dnf -y install diffstat diffutils python3 python3-cryptography python3-pip python3-rpm python3-setuptools python3-urllib3
+            dnf -y install diffstat diffutils git-core python3 python3-cryptography python3-pip python3-rpm python3-setuptools python3-urllib3
 
       - name: 'Install packages (Debian/Ubuntu)'
         if: ${{ contains(matrix.container, '/debian:') || contains(matrix.container, '/ubuntu:') }}
@@ -74,7 +74,7 @@ jobs:
             apt-get -y update
             apt-get -y upgrade
             apt-get -y --no-install-recommends install git-lfs
-            apt-get -y --no-install-recommends install diffstat diffutils python3 python3-cryptography python3-pip python3-rpm python3-setuptools python3-urllib3
+            apt-get -y --no-install-recommends install diffstat diffutils git-core python3 python3-cryptography python3-pip python3-rpm python3-setuptools python3-urllib3
 
       - uses: actions/checkout@v3
 

contrib/osc.spec

@@ -57,6 +57,8 @@ BuildRequires:  %{use_python_pkg}-rpm
 BuildRequires:  %{use_python_pkg}-setuptools
 BuildRequires:  %{use_python_pkg}-urllib3
 BuildRequires:  diffstat
+# needed for git scm tests
+BuildRequires:  git-core
 
 Requires:       %{use_python_pkg}-cryptography
 Requires:       %{use_python_pkg}-rpm
@@ -78,6 +80,10 @@ Recommends:     diffstat
 Recommends:     powerpc32
 Recommends:     sudo
 
+# needed for building from git
+Recommends:     git-core
+Recommends:     git-lfs
+
 # needed for `osc add <URL>`
 Recommends:     obs-service-recompress
 Recommends:     obs-service-download_files

osc/commandline.py

@@ -30,6 +30,7 @@ from . import build as osc_build
 from . import cmdln
 from . import commands as osc_commands
 from . import conf
+from . import git_scm
 from . import oscerr
 from . import store as osc_store
 from .core import *
@@ -6933,7 +6934,7 @@ Please submit there instead, or use --nodevelproject to force direct submission.
                 if no_repo:
                     raise oscerr.WrongArgs("Repository is missing. Cannot guess build description without repository")
                 apiurl = self.get_api_url()
-                project = store_read_project('.')
+                project = alternative_project or store_read_project('.')
                 # some distros like Debian rename and move build to obs-build
                 if not os.path.isfile('/usr/lib/build/queryconfig') and os.path.isfile('/usr/lib/obs-build/queryconfig'):
                     queryconfig = '/usr/lib/obs-build/queryconfig'
@@ -7197,12 +7198,17 @@ Please submit there instead, or use --nodevelproject to force direct submission.
         if len(args) > 3:
             raise oscerr.WrongArgs('Too many arguments')
 
-        store = osc_store.Store(Path.cwd())
+        store = osc_store.get_store(Path.cwd(), print_warnings=True)
         store.assert_is_package()
 
         if opts.alternative_project == store.project:
             opts.alternative_project = None
 
+        # HACK: avoid calling some underlying store_*() functions from parse_repoarchdescr() method
+        # We'll fix parse_repoarchdescr() later because it requires a larger change
+        if not opts.alternative_project and isinstance(store, git_scm.GitStore):
+            opts.alternative_project = store.project
+
         if len(args) == 0 and store.is_package and store.last_buildroot:
             # build env not specified, just read from last build attempt
             args = [store.last_buildroot[0], store.last_buildroot[1]]

osc/core.py

@@ -50,6 +50,7 @@ from . import _private
 from . import conf
 from . import meter
 from . import oscerr
+from . import store as osc_store
 from .connection import http_request, http_GET, http_POST, http_PUT, http_DELETE
 from .store import Store
 from .util.helper import decode_list, decode_it, raw_input, _html_escape
@@ -1239,7 +1240,8 @@ class Package:
 
         self.dir = workingdir or "."
         self.absdir = os.path.abspath(self.dir)
-        self.store = Store(self.dir)
+        self.store = osc_store.get_store(self.dir)
+        self.store.assert_is_package()
         self.storedir = os.path.join(self.absdir, store)
         self.progress_obj = progress_obj
         self.size_limit = size_limit
@@ -1247,10 +1249,8 @@ class Package:
         if size_limit and size_limit == 0:
             self.size_limit = None
 
-        check_store_version(self.dir)
-
-        self.prjname = store_read_project(self.dir)
-        self.name = store_read_package(self.dir)
+        self.prjname = self.store.project
+        self.name = self.store.package
         self.apiurl = self.store.apiurl
 
         self.update_datastructs()

osc/git_scm/README.md

@@ -0,0 +1,4 @@
+# Warning
+
+This module provides EXPERIMENTAL and UNSTABLE support for git scm such as https://src.opensuse.org/.
+The code may change or disappear without a prior notice!

osc/git_scm/__init__.py

@@ -0,0 +1,7 @@
+import sys
+
+from .store import GitStore
+
+
+def warn_experimental():
+    print("WARNING: Using EXPERIMENTAL support for git scm. The functionality may change or disappear without a prior notice!", file=sys.stderr)

osc/git_scm/store.py

@@ -0,0 +1,151 @@
+import json
+import os
+import subprocess
+import urllib.parse
+from pathlib import Path
+
+from .. import conf as osc_conf
+from .. import oscerr
+
+
+class GitStore:
+
+    @classmethod
+    def is_project_dir(cls, path):
+        try:
+            store = cls(path)
+        except oscerr.NoWorkingCopy:
+            return False
+        return store.is_project
+
+    @classmethod
+    def is_package_dir(cls, path):
+        try:
+            store = cls(path)
+        except oscerr.NoWorkingCopy:
+            return False
+        return store.is_package
+
+    def __init__(self, path, check=True):
+        self.path = path
+        self.abspath = os.path.abspath(self.path)
+
+        # TODO: how to determine if the current git repo contains a project or a package?
+        self.is_project = False
+        self.is_package = os.path.exists(os.path.join(self.abspath, ".git"))
+
+        self._package = None
+        self._project = None
+
+        if check and not any([self.is_project, self.is_package]):
+            msg = f"Directory '{self.path}' is not a GIT working copy"
+            raise oscerr.NoWorkingCopy(msg)
+
+        # TODO: decide if we need explicit 'git lfs pull' or not
+        # self._run_git(["lfs", "pull"])
+
+    def assert_is_project(self):
+        if not self.is_project:
+            msg = f"Directory '{self.path}' is not a GIT working copy of a project"
+            raise oscerr.NoWorkingCopy(msg)
+
+    def assert_is_package(self):
+        if not self.is_package:
+            msg = f"Directory '{self.path}' is not a GIT working copy of a package"
+            raise oscerr.NoWorkingCopy(msg)
+
+    def _run_git(self, args):
+        return subprocess.check_output(["git"] + args, encoding="utf-8", cwd=self.abspath).strip()
+
+    @property
+    def apiurl(self):
+        # HACK: we're using the currently configured apiurl
+        return osc_conf.config["apiurl"]
+
+    @property
+    def project(self):
+        if self._project is None:
+            # get project from the branch name
+            branch = self._run_git(["branch", "--show-current"])
+
+            # HACK: replace hard-coded mapping with metadata from git or the build service
+            if branch == "factory":
+                self._project = "openSUSE:Factory"
+            else:
+                raise RuntimeError(f"Couldn't map git branch '{branch}' to a project")
+        return self._project
+
+    @project.setter
+    def project(self, value):
+        self._project = value
+
+    @property
+    def package(self):
+        if self._package is None:
+            origin = self._run_git(["remote", "get-url", "origin"])
+            self._package = Path(urllib.parse.urlsplit(origin).path).stem
+        return self._package
+
+    @package.setter
+    def package(self, value):
+        self._package = value
+
+    def _get_option(self, name):
+        try:
+            result = self._run_git(["config", "--local", "--get", f"osc.{name}"])
+        except subprocess.CalledProcessError:
+            result = None
+        return result
+
+    def _check_type(self, name, value, expected_type):
+        if not isinstance(value, expected_type):
+            raise TypeError(f"The option '{name}' should be {expected_type.__name__}, not {type(value).__name__}")
+
+    def _set_option(self, name, value):
+        self._run_git(["config", "--local", f"osc.{name}", value])
+
+    def _unset_option(self, name):
+        try:
+            self._run_git(["config", "--local", "--unset", f"osc.{name}"])
+        except subprocess.CalledProcessError:
+            pass
+
+    def _get_dict_option(self, name):
+        result = self._get_option(name)
+        if result is None:
+            return None
+        result = json.loads(result)
+        self._check_type(name, result, dict)
+        return result
+
+    def _set_dict_option(self, name, value):
+        if value is None:
+            self._unset_option(name)
+            return
+        self._check_type(name, value, dict)
+        value = json.dumps(value)
+        self._set_option(name, value)
+
+    @property
+    def last_buildroot(self):
+        self.assert_is_package()
+        result = self._get_dict_option("last-buildroot")
+        if result is not None:
+            result = (result["repo"], result["arch"], result["vm_type"])
+        return result
+
+    @last_buildroot.setter
+    def last_buildroot(self, value):
+        self.assert_is_package()
+        if len(value) != 3:
+            raise ValueError("A tuple with exactly 3 items is expected: (repo, arch, vm_type)")
+        value = {
+            "repo": value[0],
+            "arch": value[1],
+            "vm_type": value[2],
+        }
+        self._set_dict_option("last-buildroot", value)
+
+    @property
+    def scmurl(self):
+        return self._run_git(["remote", "get-url", "origin"])

osc/store.py

@@ -10,7 +10,7 @@ from xml.etree import ElementTree as ET
 
 from . import oscerr
 from ._private import api
-
+from . import git_scm
 
 class Store:
     STORE_DIR = ".osc"
@@ -309,3 +309,22 @@ class Store:
         else:
             root = self.read_xml_node("_meta", "project").getroot()
         return root
+
+
+def get_store(path, check=True, print_warnings=False):
+    """
+    Return a store object that wraps SCM in given `path`:
+     - Store for OBS SCM
+     - GitStore for Git SCM
+    """
+    try:
+        store = Store(path, check)
+    except oscerr.NoWorkingCopy as ex:
+        try:
+            store = git_scm.GitStore(path, check)
+            if print_warnings:
+                git_scm.warn_experimental()
+        except oscerr.NoWorkingCopy as ex_git:
+            # raise the original exception, do not inform that we've tried git working copy
+            raise ex from None
+    return store

setup.cfg

@@ -35,6 +35,7 @@ packages =
     osc
     osc._private
     osc.commands
+    osc.git_scm
     osc.output
     osc.util
 install_requires =

tests/test_git_scm_store.py

@@ -0,0 +1,49 @@
+import os
+import shutil
+import subprocess
+import tempfile
+import unittest
+
+from osc.git_scm.store import GitStore
+
+
+class TestGitStore(unittest.TestCase):
+    def setUp(self):
+        self.tmpdir = tempfile.mkdtemp(prefix="osc_test")
+        os.chdir(self.tmpdir)
+        subprocess.check_output(["git", "init", "-b", "factory"])
+        subprocess.check_output(["git", "remote", "add", "origin", "https://example.com/packages/my-package.git"])
+
+    def tearDown(self):
+        try:
+            shutil.rmtree(self.tmpdir)
+        except OSError:
+            pass
+
+    def test_package(self):
+        store = GitStore(self.tmpdir)
+        self.assertEqual(store.package, "my-package")
+
+    def test_project(self):
+        store = GitStore(self.tmpdir)
+        self.assertEqual(store.project, "openSUSE:Factory")
+
+    def test_last_buildroot(self):
+        store = GitStore(self.tmpdir)
+        self.assertEqual(store.last_buildroot, None)
+        store.last_buildroot = ("repo", "arch", "vm_type")
+
+        store = GitStore(self.tmpdir)
+        self.assertEqual(store.last_buildroot, ("repo", "arch", "vm_type"))
+
+    def test_scmurl(self):
+        store = GitStore(self.tmpdir)
+        self.assertEqual(store.scmurl, "https://example.com/packages/my-package.git")
+
+
+if not shutil.which("git"):
+    TestGitStore = unittest.skip("The 'git' executable is not available")(TestGitStore)
+
+
+if __name__ == "__main__":
+    unittest.main()