Merge pull request #1012 from adrianschroeter/obs_git

initial obs-git support

NEWS

@@ -1,5 +1,8 @@
 0.178.0
-  - 
+  - EXPERIMENTAL: git repository handling
+    * init command is working inside of a git repository
+    * downloadassets command fetches references assets from build description
+    * checkout is cloning from git
 
 0.177.0
   - switch to python3 in osc-wrapper and make python3 explicit

osc/commandline.py

@@ -219,7 +219,7 @@ class Osc(cmdln.Cmdln):
             return help
         return cmdln.Cmdln._help_preprocess_cmd_usage(self, help, cmdname)
 
-    def do_init(self, subcmd, opts, project, package=None):
+    def do_init(self, subcmd, opts, project, package=None, scm_url=None):
         """${cmd_name}: Initialize a directory as working copy
 
         Initialize an existing directory to be a working copy of an
@@ -238,11 +238,25 @@ class Osc(cmdln.Cmdln):
         usage:
             osc init PRJ
             osc init PRJ PAC
+            osc init PRJ PAC SCM_URL
         ${cmd_option_list}
         """
 
         apiurl = self.get_api_url()
 
+        if not scm_url:
+            scm_url = show_scmsync(apiurl, project, package)
+
+        if scm_url:
+            if package:
+                Package.init_package(apiurl, project, package, os.curdir, scm_url=scm_url)
+                print('Initializing %s (Project: %s, Package: %s) as git repository' % (os.curdir, project, package))
+            else:
+                Project.init_project(apiurl, os.curdir, project, conf.config['do_package_tracking'],
+                                     getPackageList=False, scm_url=scm_url)
+                print('Initializing %s (Project: %s) as scm repository' % (os.curdir, project))
+            return
+
         if not package:
             Project.init_project(apiurl, os.curdir, project, conf.config['do_package_tracking'],
                                  getPackageList=False)
@@ -708,6 +722,20 @@ class Osc(cmdln.Cmdln):
         else:
             print(devprj)
 
+    @cmdln.alias('ca')
+    def do_cleanassets(self, subcmd, opts, *args):
+        """${cmd_name}: Clean all previous downloaded assets.
+
+        This is useful to prepare a new git commit.
+        """
+        clean_assets(".")
+
+    @cmdln.alias('da')
+    def do_downloadassets(self, subcmd, opts, *args):
+        """${cmd_name}: Download all assets referenced in the build descriptions
+        """
+        download_assets(".")
+
     @cmdln.alias('sdp')
     @cmdln.option('-u', '--unset', action='store_true',
                   help='remove devel project')
@@ -4682,9 +4710,19 @@ Please submit there instead, or use --nodevelproject to force direct submission.
             # check if the project does exist (show_project_meta will throw an exception)
             show_project_meta(apiurl, project)
 
-            Project.init_project(apiurl, prj_dir, project, conf.config['do_package_tracking'])
+            scm_url = show_scmsync(apiurl, project)
+            if scm_url != None:
+                if not os.path.isfile('/usr/lib/obs/service/obs_scm_bridge'):
+                    raise oscerr.OscIOError(None, 'Install the obs-scm-bridge package to work on packages managed in scm (git)!')
+                os.putenv("OSC_VERSION", get_osc_version())
+                run_external(['/usr/lib/obs/service/obs_scm_bridge', '--outdir', prj_dir, '--url', scm_url])
+
+            Project.init_project(apiurl, prj_dir, project, conf.config['do_package_tracking'], scm_url=scm_url)
             print(statfrmt('A', prj_dir))
 
+            if scm_url != None:
+                return
+
             # all packages
             for package in meta_get_packagelist(apiurl, project):
                 if opts.output_dir is not None:
@@ -6677,10 +6715,14 @@ Please submit there instead, or use --nodevelproject to force direct submission.
         project = None
         try:
             project = store_read_project(os.curdir)
-        except oscerr.NoWorkingCopy:
-            pass
             if project == opts.alternative_project:
                 opts.alternative_project = None
+        except oscerr.NoWorkingCopy:
+            # This may be a project managed entirely via git?
+            if os.path.isdir(os.curdir + "/../.osc") and os.path.isdir(os.curdir + "/../.git"):
+                project = store_read_project(os.curdir + "/..")
+                opts.alternative_project = project
+            pass
 
         if len(args) == 0 and is_package_dir(os.curdir):
             # build env not specified, just read from last build attempt

osc/conf.py

@@ -134,6 +134,7 @@ DEFAULTS = {'apiurl': 'https://api.opensuse.org',
             'build-vm-user': '',                # optional for VM builds
             'build-kernel': '',                 # optional for VM builds
             'build-initrd': '',                 # optional for VM builds
+            'download-assets-cmd': '/usr/lib/build/download_assets', # optional for scm/git based builds
 
             'build-jobs': str(_get_processors()),
             'builtin_signature_check': '1',     # by default use builtin check for verify pkgs

osc/core.py

@@ -685,6 +685,7 @@ class Project:
         self.progress_obj = progress_obj
 
         self.name = store_read_project(self.dir)
+        self.scm_url = store_read_scmurl(self.dir)
         self.apiurl = store_read_apiurl(self.dir, defaulturl=not wc_check)
 
         dirty_files = []
@@ -725,7 +726,7 @@ class Project:
         global store
         dirty_files = []
         req_storefiles = Project.REQ_STOREFILES
-        if conf.config['do_package_tracking']:
+        if conf.config['do_package_tracking'] and self.scm_url == None:
             req_storefiles += ('_packages',)
         for fname in req_storefiles:
             if not os.path.exists(os.path.join(self.absdir, store, fname)):
@@ -959,7 +960,11 @@ class Project:
                         p = Package(os.path.join(self.dir, pac), progress_obj=self.progress_obj)
                         rev = None
                         needs_update = True
-                        if expand_link and p.islink() and not p.isexpanded():
+                        if p.scm_url != None:
+                            # git managed.
+                            print("Skipping git managed package ", pac)
+                            continue
+                        elif expand_link and p.islink() and not p.isexpanded():
                             if p.haslinkerror():
                                 try:
                                     rev = show_upstream_xsrcmd5(p.apiurl, p.prjname, p.name, revision=p.rev)
@@ -1137,7 +1142,7 @@ class Project:
         return '\n'.join(r)
 
     @staticmethod
-    def init_project(apiurl, dir, project, package_tracking=True, getPackageList=True, progress_obj=None, wc_check=True):
+    def init_project(apiurl, dir, project, package_tracking=True, getPackageList=True, progress_obj=None, wc_check=True, scm_url=None):
         global store
 
         if not os.path.exists(dir):
@@ -1152,6 +1157,9 @@ class Project:
 
         store_write_project(dir, project)
         store_write_apiurl(dir, apiurl)
+        if scm_url:
+            store_write_string(dir, '_scm', scm_url + '\n')
+            package_tracking = None
         if package_tracking:
             store_write_initial_packages(dir, project, [])
         return Project(dir, getPackageList, progress_obj, wc_check)
@@ -1174,6 +1182,7 @@ class Package:
         self.storedir = os.path.join(self.absdir, store)
         self.progress_obj = progress_obj
         self.size_limit = size_limit
+        self.scm_url = store_read_scmurl(self.dir)
         if size_limit and size_limit == 0:
             self.size_limit = None
 
@@ -1198,6 +1207,8 @@ class Package:
 
     def wc_check(self):
         dirty_files = []
+        if self.scm_url:
+            return dirty_files
         for fname in self.filenamelist:
             if not os.path.exists(os.path.join(self.storedir, fname)) and not fname in self.skipped:
                 dirty_files.append(fname)
@@ -1774,6 +1785,20 @@ class Package:
         called).
         """
         import fnmatch
+        if self.scm_url:
+            self.filenamelist = []
+            self.filelist = []
+            self.skipped = []
+            self.to_be_added = []
+            self.to_be_deleted = []
+            self.in_conflict = []
+            self.linkrepair = None
+            self.rev = None
+            self.srcmd5 = None
+            self.linkinfo = None
+            self.serviceinfo = None
+            return
+
         files_tree = read_filemeta(self.dir)
         files_tree_root = files_tree.getroot()
 
@@ -1784,10 +1809,10 @@ class Package:
         self.linkinfo.read(files_tree_root.find('linkinfo'))
         self.serviceinfo = DirectoryServiceinfo()
         self.serviceinfo.read(files_tree_root.find('serviceinfo'))
-
         self.filenamelist = []
         self.filelist = []
         self.skipped = []
+
         for node in files_tree_root.findall('entry'):
             try:
                 f = File(node.get('name'),
@@ -2491,7 +2516,7 @@ rev: %s
             self.write_addlist()
 
     @staticmethod
-    def init_package(apiurl, project, package, dir, size_limit=None, meta=False, progress_obj=None):
+    def init_package(apiurl, project, package, dir, size_limit=None, meta=False, progress_obj=None, scm_url=None):
         global store
 
         if not os.path.exists(dir):
@@ -2509,6 +2534,9 @@ rev: %s
             store_write_string(dir, '_meta_mode', '')
         if size_limit:
             store_write_string(dir, '_size_limit', str(size_limit) + '\n')
+        if scm_url:
+            store_write_string(dir, '_scm', scm_url + '\n')
+        else:
             store_write_string(dir, '_files', '<directory />' + '\n')
         store_write_string(dir, '_osclib_version', __store_version__ + '\n')
         return Package(dir, progress_obj=progress_obj, size_limit=size_limit)
@@ -3255,6 +3283,8 @@ def read_filemeta(dir):
     filesmeta = os.path.join(dir, store, '_files')
     if not is_package_dir(dir):
         raise oscerr.NoWorkingCopy(msg)
+    if os.path.isfile(os.path.join(dir, store, '_scm')):
+        raise oscerr.NoWorkingCopy("Is managed via scm")
     if not os.path.isfile(filesmeta):
         raise oscerr.NoWorkingCopy('%s (%s does not exist)' % (msg, filesmeta))
 
@@ -3610,6 +3640,23 @@ def show_attribute_meta(apiurl, prj, pac, subpac, attribute, with_defaults, with
         raise
 
 
+def clean_assets(directory):
+    return run_external(conf.config['download-assets-cmd'], '--clean', directory)
+
+def download_assets(directory):
+    return run_external(conf.config['download-assets-cmd'], '--unpack', '--noassetdir', directory)
+
+def show_scmsync(apiurl, prj, pac=None):
+    if pac:
+        m = show_package_meta(apiurl, prj, pac)
+    else:
+        m = show_project_meta(apiurl, prj)
+    node = ET.fromstring(b''.join(m)).find('scmsync')
+    if node is None:
+        return None
+    else:
+        return node.text
+
 def show_devel_project(apiurl, prj, pac):
     m = show_package_meta(apiurl, prj, pac)
     node = ET.fromstring(b''.join(m)).find('devel')
@@ -5123,7 +5170,17 @@ def checkout_package(apiurl, project, package,
 
     # before we create directories and stuff, check if the package actually
     # exists
-    show_package_meta(apiurl, quote_plus(project), quote_plus(package), meta)
+    meta_data = b''.join(show_package_meta(apiurl, quote_plus(project), quote_plus(package)))
+    root = ET.fromstring(meta_data)
+    if root.find('scmsync') != None and root.find('scmsync').text != None:
+        if not os.path.isfile('/usr/lib/obs/service/obs_scm_bridge'):
+            raise oscerr.OscIOError(None, 'Install the obs-scm-bridge package to work on packages managed in scm (git)!')
+        scm_url = root.find('scmsync').text
+        directory = make_dir(apiurl, project, package, pathname, prj_dir, conf.config['do_package_tracking'], outdir)
+        os.putenv("OSC_VERSION", get_osc_version())
+        run_external(['/usr/lib/obs/service/obs_scm_bridge', '--outdir', directory, '--url', scm_url])
+        Package.init_package(apiurl, project, package, directory, size_limit, meta, progress_obj, scm_url)
+        return
 
     isfrozen = False
     if expand_link:
@@ -6583,6 +6640,21 @@ def store_read_package(dir):
         raise oscerr.NoWorkingCopy(msg)
     return p
 
+def store_read_scmurl(dir):
+    global store
+
+    url_file = os.path.join(dir, store, '_scm')
+    if not os.path.exists(url_file):
+        return
+    try:
+        p = open(url_file).readlines()[0].strip()
+    except IOError:
+        msg = 'Error: \'%s\' is not an osc package working copy' % os.path.abspath(dir)
+        if os.path.exists(os.path.join(dir, '.svn')):
+            msg += '\nTry svn instead of osc.'
+        raise oscerr.NoWorkingCopy(msg)
+    return p
+
 def store_read_apiurl(dir, defaulturl=True):
     global store