merged the package-tracking branch changes r2404:3491 into the trunk.

osc/commandline.py

@@ -786,7 +786,6 @@ class Osc(cmdln.Cmdln):
             osc co PROJECT [PACKAGE] [FILE]
         ${cmd_option_list}
         """
-
         args = slash_split(args)
         project = package = filename = None
         try: 
@@ -860,12 +859,21 @@ class Osc(cmdln.Cmdln):
         args = parseargs(args)
 
         pacpaths = []
+        prj = None
         for arg in args:
             # when 'status' is run inside a project dir, it should
             # stat all packages existing in the wc
             if is_project_dir(arg):
-                prj = Project(arg)
+                prj = Project(arg, False)
-                pacpaths += [arg + '/' + n for n in prj.pacs_have]
+
+                if conf.config['do_package_tracking']:
+                    for pac in prj.pacs_have:
+                        # we cannot create package objects if the dir does not exist
+                        if not pac in prj.pacs_broken:
+                            pacpaths.append(os.path.join(arg, pac))
+                else:
+                    pacpaths += [arg + '/' + n for n in prj.pacs_have]
+
             elif is_package_dir(arg):
                 pacpaths.append(arg)
             elif os.path.isfile(arg):
@@ -874,9 +882,20 @@ class Osc(cmdln.Cmdln):
                 print >>sys.stderr, 'osc: error: %s is neither a project or a package directory' % arg
                 return 1
             
-
         pacs = findpacs(pacpaths)
 
+        lines = []
+        if prj:
+
+            if conf.config['do_package_tracking']:
+                for data in prj.pacs_unvers:
+                    lines.append(statfrmt('?', os.path.normpath(os.path.join(prj.dir, data))))
+                for data in prj.pacs_broken:
+                    if prj.get_state(data) == 'D':
+                        lines.append(statfrmt('D', os.path.normpath(os.path.join(prj.dir, data))))
+                    else:
+                        lines.append(statfrmt('!', os.path.normpath(os.path.join(prj.dir, data))))
+
         for p in pacs:
 
             # no files given as argument? Take all files in current dir
@@ -884,7 +903,11 @@ class Osc(cmdln.Cmdln):
                 p.todo = p.filenamelist + p.filenamelist_unvers
             p.todo.sort()
 
-            lines = []
+            if prj and conf.config['do_package_tracking']:
+                state = prj.get_state(p.name)
+                if state != None and (state != ' ' or opts.verbose):
+                    lines.append(statfrmt(state, os.path.normpath(os.path.join(prj.dir, p.name))))
+
             for filename in p.todo:
                 if filename in p.excluded:
                     continue
@@ -894,12 +917,12 @@ class Osc(cmdln.Cmdln):
                 elif s != ' ' or (s == ' ' and opts.verbose):
                     lines.append(statfrmt(s, pathjoin(p.dir, filename)))
 
-            # arrange the lines in order: unknown files first
+        # arrange the lines in order: unknown files first
-            # filenames are already sorted
+        # filenames are already sorted
-            lines = [line for line in lines if line[0] == '?'] \
+        lines = [line for line in lines if line[0] == '?'] \
-                  + [line for line in lines if line[0] != '?']
+              + [line for line in lines if line[0] != '?']
-            if lines:
+        if lines:
-                print '\n'.join(lines)
+            print '\n'.join(lines)
 
 
     def do_add(self, subcmd, opts, *args):
@@ -915,24 +938,40 @@ class Osc(cmdln.Cmdln):
             return 2
 
         filenames = parseargs(args)
+        #print filenames
+        addFiles(filenames)
 
-        for filename in filenames:
+    
-            if not os.path.exists(filename):
+    def do_mkpac(self, subcmd, opts, *args):
-                print >>sys.stderr, "file '%s' does not exist" % filename
+        """${cmd_name}: Create a new package under version control
-                return 1
 
-        pacs = findpacs(filenames)
+        usage:
+            osc mkpac new_package
+        ${cmd_option_list}
+        """
+        if len(args) != 1:
+            print >>sys.stderr, 'wrong number of arguments!'
+            sys.exit(1)
 
-        for pac in pacs:
+        prj_dir, pac_dir = getPrjPacPaths(args[0])
-            for filename in pac.todo:
+        if is_project_dir(prj_dir):
-                if filename in pac.excluded:
+            if not os.path.exists(args[0]):
-                    continue
+                prj = Project(prj_dir, False)
-                if filename in pac.filenamelist:
+                if prj.addPackage(pac_dir):
-                    print >>sys.stderr, 'osc: warning: \'%s\' is already under version control' % filename
+                    os.mkdir(args[0])
-                    continue
+                    os.chdir(args[0])
-
+                    init_package_dir(prj.apiurl,
-                pac.addfile(filename)
+                                     prj.name,
-                print statfrmt('A', filename)
+                                     pac_dir, pac_dir, files=False)
+                    os.chdir(prj.absdir)
+                    print statfrmt('A', os.path.normpath(args[0]))
+            else:
+                print '\'%s\' already exists' % args[0]
+                sys.exit(1)
+        else:
+            print 'wrong number of arguments or ' \
+                  '\'%s\' is not a working copy' % prj_dir
+            sys.exit(1)
 
 
     def do_addremove(self, subcmd, opts, *args):
@@ -990,20 +1029,43 @@ class Osc(cmdln.Cmdln):
         ${cmd_usage}
         ${cmd_option_list}
         """
+        msg = ''
+        if opts.message:
+            msg = opts.message
+        elif opts.file:
+            try:
+                msg = open(opts.file).read()
+            except:
+                sys.exit('could not open file \'%s\'.' % opts.file)
 
         args = parseargs(args)
-        pacs = findpacs(args)
+        for arg in args:
+            if conf.config['do_package_tracking'] and is_project_dir(arg):
+                Project(arg).commit(msg=msg)
+                args.remove(arg)
 
-        for p in pacs:
+        pacs = findpacs(args)
-            msg = ''
+        if conf.config['do_package_tracking'] and len(pacs) > 0:
-            if opts.message:
+            prj_paths = {}
-                msg = opts.message
+            single_paths = []
-            elif opts.file:
+            files = {}
-                try:
+            # it is possible to commit packages from different projects at the same
-                    msg = open(opts.file).read()
+            # time: iterate over all pacs and put each pac to the right project in the dict
-                except:
+            for pac in pacs:
-                    sys.exit('could not open file \'%s\'.' % opts.file)
+                path = os.path.normpath(os.path.join(pac.dir, os.pardir))
-            p.commit(msg)
+                if is_project_dir(path):
+                    pac_path = os.path.basename(os.path.normpath(pac.absdir))
+                    prj_paths.setdefault(path, []).append(pac_path)
+                    files[pac_path] = pac.todo
+                else:
+                    single_paths.append(pac.dir)
+            for prj, packages in prj_paths.iteritems():
+                Project(prj).commit(tuple(packages), msg, files)
+            for pac in single_paths:
+                Package(pac).commit(msg)
+        else:
+            for p in pacs:
+                p.commit(msg)
 
 
     @cmdln.option('-r', '--revision', metavar='rev',
@@ -1036,17 +1098,20 @@ class Osc(cmdln.Cmdln):
 
         for arg in args:
 
-            # when 'update' is run inside a project dir, it should...
             if is_project_dir(arg):
-
                 prj = Project(arg)
 
-                # (a) update all packages
+                if conf.config['do_package_tracking']:
-                args += prj.pacs_have
+                    prj.update()
-
+                    args.remove(arg)
-                # (b) fetch new packages
+                else:   
-                prj.checkout_missing_pacs()
+                    # if not tracking package, and 'update' is run inside a project dir, 
-                args.remove(arg)
+                    # it should do the following:
+                    # (a) update all packages
+                    args += prj.pacs_have
+                    # (b) fetch new packages
+                    prj.checkout_missing_pacs()
+                    args.remove(arg)
 
 
         pacs = findpacs(args)
@@ -1060,60 +1125,13 @@ class Osc(cmdln.Cmdln):
             rev = None
 
         for p in pacs:
-
             if len(pacs) > 1:
                 print 'Updating %s' % p.name
-            # save filelist and (modified) status before replacing the meta file
+            p.update(rev)
-            saved_filenames = p.filenamelist
+                   
-            saved_modifiedfiles = [ f for f in p.filenamelist if p.status(f) == 'M' ]
 
-            oldp = p
+    @cmdln.option('-f', '--force', action='store_true',
-            p.update_local_filesmeta(rev)
+                        help='forces removal of package')
-            p = Package(p.dir)
-
-            # which files do no longer exist upstream?
-            disappeared = [ f for f in saved_filenames if f not in p.filenamelist ]
-                
-
-            for filename in saved_filenames:
-                if filename in disappeared:
-                    print statfrmt('D', filename)
-                    # keep file if it has local modifications
-                    if oldp.status(filename) == ' ':
-                        p.delete_localfile(filename)
-                    p.delete_storefile(filename)
-                    continue
-
-            for filename in p.filenamelist:
-
-                state = p.status(filename)
-                if state == 'M' and p.findfilebyname(filename).md5 == oldp.findfilebyname(filename).md5:
-                    # no merge necessary... local file is changed, but upstream isn't
-                    pass
-                elif state == 'M' and filename in saved_modifiedfiles:
-                    status_after_merge = p.mergefile(filename)
-                    print statfrmt(status_after_merge, filename)
-                elif state == 'M':
-                    p.updatefile(filename, rev)
-                    print statfrmt('U', filename)
-                elif state == '!':
-                    p.updatefile(filename, rev)
-                    print 'Restored \'%s\'' % filename
-                elif state == 'F':
-                    p.updatefile(filename, rev)
-                    print statfrmt('A', filename)
-                elif state == ' ':
-                    pass
-
-
-            p.update_local_pacmeta()
-
-            #print ljust(p.name, 45), 'At revision %s.' % p.rev
-            print 'At revision %s.' % p.rev
-                    
-
-
-            
     @cmdln.alias('rm')
     @cmdln.alias('del')
     @cmdln.alias('remove')
@@ -1131,21 +1149,38 @@ class Osc(cmdln.Cmdln):
             return 2
 
         args = parseargs(args)
+        # check if args contains a package which was removed by
+        # a non-osc command and mark it with the 'D'-state
+        for i in args:
+            if not os.path.exists(i):
+                prj_dir, pac_dir = getPrjPacPaths(i)
+                if is_project_dir(prj_dir):
+                    prj = Project(prj_dir, False)
+                    if i in prj.pacs_broken:
+                        if prj.get_state(i) != 'A':
+                            prj.set_state(pac_dir, 'D')
+                        else:
+                            prj.del_package_node(i)
+                        print statfrmt('D', getTransActPath(i))
+                        args.remove(i)
+                        prj.write_packages()
         pacs = findpacs(args)
 
         for p in pacs:
-
+            if not p.todo:
-            for filename in p.todo:
+                prj_dir, pac_dir = getPrjPacPaths(p.absdir)
-                if filename not in p.filenamelist:
+                if conf.config['do_package_tracking'] and is_project_dir(prj_dir):
-                    sys.exit('\'%s\' is not under version control' % filename)
+                    prj = Project(prj_dir, False)
-                p.put_on_deletelist(filename)
+                    prj.delPackage(p, opts.force)
-                p.write_deletelist()
+            else:
-                try:
+                pathn = getTransActPath(p.dir)
-                    os.unlink(os.path.join(p.dir, filename))
+                for filename in p.todo:
-                    os.unlink(os.path.join(p.storedir, filename))
+                    if filename not in p.filenamelist:
-                except:
+                        sys.exit('\'%s\' is not under version control' % filename)
-                    pass
+                    p.put_on_deletelist(filename)
-                print statfrmt('D', filename)
+                    p.write_deletelist()
+                    p.delete_source_file(filename)
+                    print statfrmt('D', os.path.join(pathn, filename))
 
 
     def do_resolved(self, subcmd, opts, *args):
@@ -1821,6 +1856,13 @@ class Osc(cmdln.Cmdln):
         """
         import glob
 
+        if opts.delete_old_files and conf.config['do_package_tracking']:
+            # IMHO the --delete-old-files option doesn't really fit into our
+            # package tracking strategy
+            print >>sys.stderr, '--delete-old-files is not supported anymore'
+            print >>sys.stderr, 'when do_package_tracking is enabled'
+            sys.exit(1)
+
         if '://' in srpm:
             print 'trying to fetch', srpm
             import urlgrabber
@@ -1832,13 +1874,16 @@ class Osc(cmdln.Cmdln):
         if opts.project:
             project_dir = opts.project
         else:
-            project_dir = os.getcwd()
+            project_dir = os.curdir
 
         if not is_project_dir(project_dir):
             print >>sys.stderr, 'project dir \'%s\' does not exist' % opts.project
             sys.exit(1)
         else:
-            project = store_read_project(project_dir)
+            if conf.config['do_package_tracking']:
+                project = Project(project_dir)
+            else:
+                project = store_read_project(project_dir)
             # act as if run with -A `cat $project_dir/.osc/_apiurl`
 	    # to get apiurl and user right
             apiurl = store_read_apiurl(project_dir)
@@ -1865,39 +1910,50 @@ class Osc(cmdln.Cmdln):
                                 'The automatic detection failed'
             sys.exit(1)
 
+        olddir = os.getcwd()
         if not os.path.exists(os.path.join(project_dir, pac)):
             os.mkdir(os.path.join(project_dir, pac))
             os.chdir(os.path.join(project_dir, pac))
-            data = meta_exists(metatype='pkg',
+            if conf.config['do_package_tracking']:
-                               path_args=(quote_plus(project), quote_plus(pac)),
+                if project.addPackage(pac):
-                               template_args=({
+                    init_package_dir(conf.config['apiurl'], project.name, pac, os.path.join(project.dir, pac), files=False)
-                                   'name': pac,
+                else:
-                                   'user': conf.config['user']}))
+                    sys.exit(1)
-            if data:
-                data = ET.fromstring(''.join(data))
-                data.find('title').text = title
-                data.find('description').text = ''.join(descr)
-                data = ET.tostring(data)
             else:
-                print >>sys.stderr, 'error - cannot get meta data'
+                data = meta_exists(metatype='pkg',
-                sys.exit(1)
+                                   path_args=(quote_plus(project), quote_plus(pac)),
-            edit_meta(metatype='pkg',
+                                   template_args=({
-                      path_args=(quote_plus(project), quote_plus(pac)),
+                                       'name': pac,
-                      data = data)
+                                       'user': conf.config['user']}))
-            init_package_dir(conf.config['apiurl'], project, pac, os.path.join(project, pac))
+                if data:
+                    data = ET.fromstring(''.join(data))
+                    data.find('title').text = title
+                    data.find('description').text = ''.join(descr)
+                    data = ET.tostring(data)
+                else:
+                    print >>sys.stderr, 'error - cannot get meta data'
+                    sys.exit(1)
+                edit_meta(metatype='pkg',
+                          path_args=(quote_plus(project), quote_plus(pac)),
+                          data = data)
+                init_package_dir(conf.config['apiurl'], project, pac, os.path.join(project, pac))
             unpack_srcrpm(srpm, os.getcwd())
             p = Package(os.getcwd())
             if len(p.filenamelist) == 0 and opts.commit:
-                # TODO: moving this into the Package class
                 print 'Adding files to working copy...'
-                self.do_add(None, None, *glob.glob('*'))
+                addFiles(glob.glob('*'))
-                p.commit()
+                if conf.config['do_package_tracking']:
+                    os.chdir(olddir)
+                    project.commit((pac, ))
+                else:
+                    p.update_datastructs()
+                    p.commit()
             elif opts.commit and opts.delete_old_files:
-                delete_server_files(conf.config['apiurl'], project, pac, p.filenamelist)
+                for file in p.filenamelist:
+                    p.delete_remote_source_file(file)
                 p.update_local_filesmeta()
-                # TODO: moving this into the Package class
                 print 'Adding files to working copy...'
-                self.do_add(None, None, *glob.glob('*'))
+                addFiles(glob.glob('*'))
                 p.update_datastructs()
                 p.commit()
             else:

osc/conf.py

@@ -56,8 +56,10 @@ DEFAULTS = { 'apisrv': 'https://api.opensuse.org/',
 
              'http_debug': '0',
              'cookiejar': '~/.osc_cookiejar',
+             # disable project tracking by default
+             'do_package_tracking': '0',
 }
-boolean_opts = ['http_debug']
+boolean_opts = ['http_debug', 'do_package_tracking']
 
 new_conf_template = """
 [general]

osc/core.py

@@ -165,27 +165,338 @@ class File:
 
 class Project:
     """represent a project directory, holding packages"""
-    def __init__(self, dir):
+    def __init__(self, dir, getPackageList=True):
         self.dir = dir
         self.absdir = os.path.abspath(dir)
 
         self.name = store_read_project(self.dir)
         self.apiurl = store_read_apiurl(self.dir)
 
-        self.pacs_available = meta_get_packagelist(self.apiurl, self.name)
+        if getPackageList:
-
+            self.pacs_available = meta_get_packagelist(self.apiurl, self.name)
-        self.pacs_have = [ i for i in os.listdir(self.dir) if i in self.pacs_available ]
+        else:
+            self.pacs_available = []
+        
+        if conf.config['do_package_tracking']:
+            self.pac_root = self.read_packages().getroot()
+            self.pacs_have = [ pac.get('name') for pac in self.pac_root.findall('package') ]
+            self.pacs_unvers = [ i for i in os.listdir(self.dir) if i not in self.pacs_have and i not in exclude_stuff ]
+            # store all broken packages (e.g. packages which where removed by a non-osc cmd)
+            # in the self.pacs_broken list
+            self.pacs_broken = []
+            for p in self.pacs_have:
+                if not os.path.isdir(os.path.join(self.absdir, p)):
+                    # all states will be replaced with the '!'-state
+                    # (except it is already marked as deleted ('D'-state))
+                    self.pacs_broken.append(p)
+        else:
+            self.pacs_have = [ i for i in os.listdir(self.dir) if i in self.pacs_available ]
 
         self.pacs_missing = [ i for i in self.pacs_available if i not in self.pacs_have ]
 
     def checkout_missing_pacs(self):
         for pac in self.pacs_missing:
-            print 'checking out new package %s' % pac
+
+            if conf.config['do_package_tracking'] and pac in self.pacs_unvers:
+                # pac is not under version control but a local file/dir exists
+                print 'can\'t add package \'%s\': Object already exists' % pac
+                sys.exit(1)
+            else:
+                print 'checking out new package %s' % pac
+                olddir = os.getcwd()
+                #os.chdir(os.pardir)
+                os.chdir(os.path.join(self.absdir, os.pardir))
+                #checkout_package(self.apiurl, self.name, pac, pathname = os.path.normpath(os.path.join(self.dir, pac)))
+                checkout_package(self.apiurl, self.name, pac, pathname = getTransActPath(os.path.join(self.dir, pac)), prj_obj=self)
+                os.chdir(olddir)
+                #self.new_package_entry(pac, ' ')
+                #self.pacs_have.append(pac)
+
+    def set_state(self, pac, state):
+        node = self.get_package_node(pac)
+        if node == None:
+            self.new_package_entry(pac, state)
+        else:
+            node.attrib['state'] = state
+
+    def get_package_node(self, pac):
+        for node in self.pac_root.findall('package'):
+            if pac == node.get('name'):
+                return node
+        return None
+
+    def del_package_node(self, pac):
+        for node in self.pac_root.findall('package'):
+            if pac == node.get('name'):
+                self.pac_root.remove(node)
+
+    def get_state(self, pac):
+        node = self.get_package_node(pac)
+        if node != None:
+            return node.get('state')
+        else:
+            return None
+
+    def new_package_entry(self, name, state):
+        ET.SubElement(self.pac_root, 'package', name=name, state=state)
+
+    def read_packages(self):
+        if os.path.isfile(os.path.join(self.absdir, store, '_packages')):
+            return ET.parse(os.path.join(self.absdir, store, '_packages'))
+        else:
+            # scan project for existing packages and migrate them
+            cur_pacs = []
+            for data in os.listdir(self.dir):
+                pac_dir = os.path.join(self.absdir, data)
+                # we cannot use self.pacs_available because we cannot guarantee that the package list
+                # was fetched from the server
+                if data in meta_get_packagelist(self.apiurl, self.name) and is_package_dir(pac_dir) \
+                   and Package(pac_dir).name == data:
+                    cur_pacs.append(ET.Element('package', name=data, state=' '))
+            store_write_initial_packages(self.absdir, self.name, cur_pacs)
+            return ET.parse(os.path.join(self.absdir, store, '_packages'))
+
+    def write_packages(self):
+        # TODO: should we only modify the existing file instead of overwriting?
+        ET.ElementTree(self.pac_root).write(os.path.join(self.absdir, store, '_packages'))
+
+    def addPackage(self, pac):
+        state = self.get_state(pac)
+        if state == None or state == 'D':
+            self.new_package_entry(pac, 'A')
+            self.write_packages()
+            # sometimes the new pac doesn't exist in the list because
+            # it would take too much time to update all data structs regulary
+            if pac in self.pacs_unvers:
+                self.pacs_unvers.remove(pac)
+            return True
+        else:
+            print 'package \'%s\' is already under version control' % pac
+            return False
+
+    def delPackage(self, pac, force = False):
+        state = self.get_state(pac.name)
+        can_delete = True
+        if state == ' ' or state == 'D':
+            del_files = []
+            for file in pac.filenamelist + pac.filenamelist_unvers:
+                filestate = pac.status(file)
+                if filestate == 'M' or filestate == 'C' or \
+                   filestate == 'A' or filestate == '?':
+                    can_delete = False
+                else:    
+                    del_files.append(file)
+            if can_delete or force:
+                for file in del_files:
+                    pac.delete_localfile(file)
+                    if pac.status(file) != '?':
+                        pac.delete_storefile(file)
+                        # this is not really necessary
+                        pac.put_on_deletelist(file)
+                        print statfrmt('D', os.path.join(pac.dir, file))
+                #print os.path.dirname(pac.dir)
+                # some black path vodoo
+                print statfrmt('D', getTransActPath(os.path.join(pac.dir, os.pardir, pac.name)))
+                pac.write_deletelist()
+                self.set_state(pac.name, 'D')
+                self.write_packages()
+            else:
+                print 'package \'%s\' has local modifications (see osc st for details)' % pac.name
+        elif state == 'A':
+            if force:
+                delete_dir(pac.absdir)
+                self.del_package_node(pac.name)
+                self.write_packages()
+                print statfrmt('D', pac.name)
+            else:
+                print 'package \'%s\' has local modifications (see osc st for details)' % pac.name
+        elif state == None:
+            print 'package is not under version control'
+        else:
+            print 'unsupported state'
+
+    def update(self, pacs = ()):
+        if len(pacs):
+            for pac in pacs:
+                Package(os.path.join(self.dir, pac)).update()
+        else:
+            # update complete project
+            # packages which no longer exists upstream
+            upstream_del = [ pac for pac in self.pacs_have if not pac in self.pacs_available and self.get_state(pac) != 'A']
+
+            for pac in upstream_del:
+                p = Package(os.path.join(self.dir, pac))
+                self.delPackage(p, force = True)
+                delete_storedir(p.storedir)
+                try:
+                    os.rmdir(pac)
+                except:
+                    pass
+                self.pac_root.remove(self.get_package_node(p.name))
+                self.pacs_have.remove(pac)
+
+            for pac in self.pacs_have:
+                state = self.get_state(pac)
+                if pac in self.pacs_broken:
+                    if self.get_state(pac) != 'A':
+                        olddir = self.absdir
+                        os.chdir(os.path.join(self.absdir, os.pardir))
+                        checkout_package(self.apiurl, self.name, pac,
+                                         pathname=getTransActPath(os.path.join(self.dir, pac)), prj_obj=self)
+                        os.chdir(olddir)
+                elif state == ' ':
+                    # do a simple update
+                    Package(os.path.join(self.dir, pac)).update()
+                elif state == 'D':
+                    # TODO: Package::update has to fixed to behave like svn does
+                    if pac in self.pacs_broken:
+                        olddir = self.absdir
+                        os.chdir(os.path.join(self.absdir, os.pardir))
+                        checkout_package(self.apiurl, self.name, pac,
+                                         pathname=getTransActPath(os.path.join(self.dir, pac)), prj_obj=self)
+                        os.chdir(olddir)
+                    else:
+                        Package(os.path.join(self.dir, pac)).update()
+                elif state == 'A' and pac in self.pacs_available:
+                    # file/dir called pac already exists and is under version control
+                    print 'can\'t add package \'%s\': Object already exists' % pac
+                    sys.exit(1)
+                elif state == 'A':
+                    # do nothing
+                    pass
+                else:
+                    print 'unexpected state.. package \'%s\'' % pac
+
+            self.checkout_missing_pacs()
+            self.write_packages()
+
+    def commit(self, pacs = (), msg = '', files = {}):
+        if len(pacs):
+            for pac in pacs:
+                todo = []
+                if files.has_key(pac):
+                    todo = files[pac]
+                state = self.get_state(pac)
+                if state == 'A':
+                    self.commitNewPackage(pac, msg, todo)
+                elif state == 'D':
+                    self.commitDelPackage(pac)
+                elif state == ' ':
+                    # display the correct dir when sending the changes
+                    if os.path.samefile(os.path.join(self.dir, pac), os.getcwd()):
+                        p = Package('.')
+                    else:
+                        p = Package(os.path.join(self.dir, pac))
+                    p.todo = todo
+                    p.commit(msg)
+                elif pac in self.pacs_unvers and not is_package_dir(os.path.join(self.dir, pac)):
+                    print 'osc: \'%s\' is not under version control' % pac
+                elif pac in self.pacs_broken:
+                    print 'osc: \'%s\' package not found' % pac
+                elif state == None:
+                    self.commitExtPackage(pac, msg, todo)
+        else:
+            # if we have packages marked as '!' we cannot commit
+            for pac in self.pacs_broken:
+                if self.get_state(pac) != 'D':
+                    print 'commit failed: package \'%s\' is missing' % pac
+                    sys.exit(1)
+            for pac in self.pacs_have:
+                state = self.get_state(pac)
+                if state == ' ':
+                    # do a simple commit
+                    try:
+                        Package(os.path.join(self.dir, pac)).commit(msg)
+                    except SystemExit:
+                        pass
+                elif state == 'D':
+                    self.commitDelPackage(pac)
+                elif state == 'A':
+                    self.commitNewPackage(pac, msg)
+        self.write_packages()
+
+    def commitNewPackage(self, pac, msg = '', files = []):
+        """creates and commits a new package if it does not exist on the server"""
+        if pac in self.pacs_available:
+            print 'package \'%s\' already exists' % pac
+        else:
+            edit_meta(metatype='pkg',
+                      path_args=(quote_plus(self.name), quote_plus(pac)),
+		      template_args=({
+			      'name': pac,
+			      'user': conf.config['user']}),
+		      apiurl=self.apiurl)
+            # display the correct dir when sending the changes
             olddir = os.getcwd()
-            os.chdir(os.pardir)
+            if os.path.samefile(os.path.join(self.dir, pac), os.curdir):
-            checkout_package(self.apiurl, self.name, pac)
+                os.chdir(os.pardir)
+                p = Package(pac)
+            else:
+                p = Package(os.path.join(self.dir, pac))
+            p.todo = files
+            #print statfrmt('Sending', os.path.normpath(os.path.join(p.dir, os.pardir, pac)))
+            print statfrmt('Sending', os.path.normpath(p.dir))
+            try:
+                p.commit(msg)
+            except SystemExit:
+                pass
+            self.set_state(pac, ' ')
             os.chdir(olddir)
 
+    def commitDelPackage(self, pac):
+        """deletes a package on the server and in the working copy"""
+        try:
+            # display the correct dir when sending the changes
+            if os.path.samefile(os.path.join(self.dir, pac), os.curdir):
+                pac_dir = pac
+            else:
+                pac_dir = os.path.join(self.dir, pac)
+            p = Package(os.path.join(self.dir, pac))
+            #print statfrmt('Deleting', os.path.normpath(os.path.join(p.dir, os.pardir, pac)))
+            delete_storedir(p.storedir)
+            try:
+                os.rmdir(p.dir)
+            except:
+                pass
+        except SystemExit:
+            pass
+        except OSError:
+            pac_dir = os.path.join(self.dir, pac)
+        #print statfrmt('Deleting', getTransActPath(os.path.join(self.dir, pac)))
+        print statfrmt('Deleting', getTransActPath(pac_dir))
+        delete_package(self.apiurl, self.name, pac)
+        self.del_package_node(pac)
+
+    def commitExtPackage(self, pac, msg, files = []):
+        """commits a package from an external project"""
+        if os.path.samefile(os.path.join(self.dir, pac), os.getcwd()):
+            pac_path = '.'
+        else:
+            pac_path = os.path.join(self.dir, pac)
+ 
+        project = store_read_project(pac_path)
+        package = store_read_package(pac_path)
+        apiurl = store_read_apiurl(pac_path)
+        if meta_exists(metatype='pkg',
+                       path_args=(quote_plus(project), quote_plus(package)),
+                       template_args=None,
+                       create_new=False, apiurl=apiurl):
+            p = Package(pac_path)
+            p.todo = files
+            p.commit(msg)
+        else:
+            edit_meta(metatype='pkg',
+                      path_args=(quote_plus(project), quote_plus(package)),
+                      template_args=({
+			      'name': pac,
+			      'user': conf.config['user']}),
+		      apiurl=apiurl)
+            try:
+                p = Package(pac_path)
+                p.todo = files
+                p.commit(msg)
+            except SystemExit:
+                pass
 
     def __str__(self):
         r = []
@@ -279,13 +590,15 @@ class Package:
             f.close()
 
     def delete_source_file(self, n):
-        
+        """delete local a source file"""
-        u = makeurl(self.apiurl, ['source', self.prjname, self.name, pathname2url(n)])
-        http_DELETE(u)
-
         self.delete_localfile(n)
         self.delete_storefile(n)
 
+    def delete_remote_source_file(self, n):
+        """delete a remote source file (e.g. from the server)"""
+        u = makeurl(self.apiurl, ['source', self.prjname, self.name, pathname2url(n)])
+        http_DELETE(u)
+ 
     def put_source_file(self, n):
         
         # escaping '+' in the URL path (note: not in the URL query string) is 
@@ -307,15 +620,17 @@ class Package:
 
         if not self.todo:
             self.todo = self.filenamelist_unvers + self.filenamelist
+      
+        pathn = getTransActPath(self.dir)
 
         for filename in self.todo:
             st = self.status(filename)
             if st == 'A' or st == 'M':
                 self.todo_send.append(filename)
-                print 'Sending        %s' % filename
+                print statfrmt('Sending', os.path.join(pathn, filename))
             elif st == 'D':
                 self.todo_delete.append(filename)
-                print 'Deleting       %s' % filename
+                print statfrmt('Deleting',  os.path.join(pathn, filename))
 
         if not self.todo_send and not self.todo_delete:
             print 'nothing to do for package %s' % self.name
@@ -323,12 +638,15 @@ class Package:
 
         print 'Transmitting file data ', 
         for filename in self.todo_delete:
-            self.delete_source_file(filename)
+            # do not touch local files on commit --
+            # delete remotely instead
+            self.delete_remote_source_file(filename)
             self.to_be_deleted.remove(filename)
         for filename in self.todo_send:
             sys.stdout.write('.')
             sys.stdout.flush()
             self.put_source_file(filename)
+
         # all source files are committed - now comes the log
         query = []
         query.append('cmd=commit')
@@ -336,7 +654,7 @@ class Package:
         query.append('user=%s' % conf.config['user'])
         query.append('comment=%s' % quote_plus(msg))
         u = makeurl(self.apiurl, ['source', self.prjname, self.name], query=query)
-        #print u
+
         f = http_POST(u)
         root = ET.parse(f).getroot()
         self.rev = int(root.get('rev'))
@@ -644,6 +962,60 @@ rev: %s
 
         os.unlink(filename)
 
+    def update(self, rev = None):
+        # save filelist and (modified) status before replacing the meta file
+        saved_filenames = self.filenamelist
+        saved_modifiedfiles = [ f for f in self.filenamelist if self.status(f) == 'M' ]
+
+        oldp = self
+        self.update_local_filesmeta(rev)
+        self = Package(self.dir)
+
+        # which files do no longer exist upstream?
+        disappeared = [ f for f in saved_filenames if f not in self.filenamelist ]
+
+        pathn = getTransActPath(self.dir)
+
+        for filename in saved_filenames:
+            if filename in disappeared:
+                print statfrmt('D', os.path.join(pathn, filename))
+                # keep file if it has local modifications
+                if oldp.status(filename) == ' ':
+                    self.delete_localfile(filename)
+                self.delete_storefile(filename)
+                continue
+
+        for filename in self.filenamelist:
+
+            state = self.status(filename)
+            if state == 'M' and self.findfilebyname(filename).md5 == oldp.findfilebyname(filename).md5:
+                # no merge necessary... local file is changed, but upstream isn't
+                pass
+            elif state == 'M' and filename in saved_modifiedfiles:
+                status_after_merge = self.mergefile(filename)
+                print statfrmt(status_after_merge, os.path.join(pathn, filename))
+            elif state == 'M':
+                self.updatefile(filename, rev)
+                print statfrmt('U', os.path.join(pathn, filename))
+            elif state == '!':
+                self.updatefile(filename, rev)
+                print 'Restored \'%s\'' % os.path.join(pathn, filename)
+            elif state == 'F':
+                self.updatefile(filename, rev)
+                print statfrmt('A', os.path.join(pathn, filename))
+            elif state == 'D' and self.findfilebyname(filename).md5 != oldp.findfilebyname(filename).md5:
+                self.updatefile(filename, rev)
+                self.delete_storefile(filename)
+                print statfrmt('U', os.path.join(pathn, filename))
+            elif state == ' ':
+                pass
+
+
+        self.update_local_pacmeta()
+
+        #print ljust(p.name, 45), 'At revision %s.' % p.rev
+        print 'At revision %s.' % self.rev
+ 
 
 class RequestState:
     """for objects to represent the "state" of a request"""
@@ -936,8 +1308,10 @@ def init_project_dir(apiurl, dir, project):
 
     store_write_project(dir, project)
     store_write_apiurl(dir, apiurl)
+    if conf.config['do_package_tracking']:
+        store_write_initial_packages(dir, project, [])
 
-def init_package_dir(apiurl, project, package, dir, revision=None):
+def init_package_dir(apiurl, project, package, dir, revision=None, files=True):
     if not os.path.isdir(store):
         os.mkdir(store)
     os.chdir(store)
@@ -948,9 +1322,13 @@ def init_package_dir(apiurl, project, package, dir, revision=None):
     f.write(package + '\n')
     f.close
 
-    f = open('_files', 'w')
+    if files:
-    f.write(''.join(show_files_meta(apiurl, project, package, revision)))
+        f = open('_files', 'w')
-    f.close()
+        f.write(''.join(show_files_meta(apiurl, project, package, revision)))
+        f.close()
+    else:
+        # create dummy
+        ET.ElementTree(element=ET.Element('directory')).write('_files')
 
     f = open('_osclib_version', 'w')
     f.write(__version__ + '\n')
@@ -1181,18 +1559,22 @@ def edit_meta(metatype,
               data=None, 
               template_args=None, 
               edit=False,
-              change_is_required=False):
+              change_is_required=False,
+              apiurl=None):
 
+    if not apiurl:
+        apiurl = conf.config['apiurl']
     if not data:
         data = meta_exists(metatype,
                            path_args,
                            template_args,
-                           create_new=True)
+                           create_new=True,
+                           apiurl=apiurl)
 
     if edit:
         change_is_required = True
 
-    url = make_meta_url(metatype, path_args)
+    url = make_meta_url(metatype, path_args, apiurl)
     f=metafile(url, data, change_is_required)
 
     if edit:
@@ -1658,37 +2040,48 @@ def pretty_diff(apiurl,
     return f.read()
 
 
-def make_dir(apiurl, project, package):
+def make_dir(apiurl, project, package, pathname):
     #print "creating directory '%s'" % project
     if not os.path.exists(project):
         print statfrmt('A', project)
         init_project_dir(apiurl, project, project)
-
     #print "creating directory '%s/%s'" % (project, package)
+    if not pathname:
+        pathname = os.path.join(project, package)
     if not os.path.exists(os.path.join(project, package)):
-        print statfrmt('A', '%s/%s' % (project, package))    
+        print statfrmt('A', pathname)
         os.mkdir(os.path.join(project, package))
         os.mkdir(os.path.join(project, package, store))
 
     return(os.path.join(project, package))
 
 
-def checkout_package(apiurl, project, package, revision=None):
+def checkout_package(apiurl, project, package, revision=None, pathname=None, prj_obj = None):
     olddir = os.getcwd()
+ 
+    if not pathname:
+        pathname = os.path.join(project, package)
 
     path = (quote_plus(project), quote_plus(package))
     if meta_exists(metatype='pkg', path_args=path, create_new=False, apiurl=apiurl) == None:
-        print >>sys.stderr, 'error 404 - package or package does not exist'
+        print >>sys.stderr, 'error 404 - project or package does not exist'
         sys.exit(1)
  
-    os.chdir(make_dir(apiurl, project, package))
+    os.chdir(make_dir(apiurl, project, package, pathname))
     init_package_dir(apiurl, project, package, store, revision)
-    p = Package(os.curdir)
+    os.chdir(os.pardir)
+    p = Package(package)
 
     for filename in p.filenamelist:
         p.updatefile(filename, revision)
-        print 'A   ', os.path.join(project, package, filename)
+        #print 'A   ', os.path.join(project, package, filename)
-
+        print statfrmt('A', os.path.join(pathname, filename))
+    if conf.config['do_package_tracking']:
+        # check if we can re-use an existing project object
+        if prj_obj == None:
+            prj_obj = Project(os.getcwd())
+        prj_obj.set_state(p.name, ' ')
+        prj_obj.write_packages()
     os.chdir(olddir)
 
 
@@ -2162,6 +2555,13 @@ def store_write_apiurl(dir, apiurl):
     fname = os.path.join(dir, store, '_apiurl')
     open(fname, 'w').write(apiurl + '\n')
 
+def store_write_initial_packages(dir, project, subelements):
+    fname = os.path.join(dir, store, '_packages')
+    root = ET.Element('project', name=project)
+    for elem in subelements:
+        root.append(elem)
+    ET.ElementTree(root).write(fname)
+
 def get_osc_version():
     return __version__
 
@@ -2363,29 +2763,16 @@ def search(apiurl, search_list, kind, search_term, verbose = False, exact_matche
     else:
         return None
 
-def delete_tmpdir(tmpdir):
+def delete_dir(dir):
-    """
-    This method deletes a tempdir. This tempdir
-    must be located under /tmp/$DIR. If "tmpdir" is not
-    a valid tempdir it'll return False. If os.unlink() / os.rmdir()
-    throws an exception we will return False too - otherwise
-    True.
-    """
-
     # small security checks
-    if os.path.islink(tmpdir):
+    if os.path.islink(dir):
         return False
-    elif os.path.abspath(tmpdir) == '/':
+    elif os.path.abspath(dir) == '/':
         return False
-    
+    elif not os.path.isdir(dir):
-    head, tail = os.path.split(tmpdir)
-    if not head.startswith('/tmp') or not tail:
         return False
 
-    if not os.path.isdir(tmpdir):
+    for dirpath, dirnames, filenames in os.walk(dir, topdown=False):
-        return False
-
-    for dirpath, dirnames, filenames in os.walk(tmpdir, topdown=False):
         for file in filenames:
             try:
                 os.unlink(os.path.join(dirpath, file))
@@ -2397,11 +2784,36 @@ def delete_tmpdir(tmpdir):
             except:
                 return False
     try:
-        os.rmdir(tmpdir)
+        os.rmdir(dir)
     except:
         return False
     return True
 
+def delete_tmpdir(tmpdir):
+    """
+    This method deletes a tempdir. This tempdir
+    must be located under /tmp/$DIR. If "tmpdir" is not
+    a valid tempdir it'll return False. If os.unlink() / os.rmdir()
+    throws an exception we will return False too - otherwise
+    True.
+    """
+
+    head, tail = os.path.split(tmpdir)
+    if not head.startswith('/tmp') or not tail:
+        return False
+    else:
+        return delete_dir(tmpdir)
+
+def delete_storedir(store_dir):
+    """
+    This method deletes a store dir.
+    """
+    head, tail = os.path.split(store_dir)
+    if tail == '.osc':
+        return delete_dir(store_dir)
+    else:
+        return False
+
 def unpack_srcrpm(srpm, dir, *files):
     """
     This method unpacks the passed srpm into the
@@ -2568,3 +2980,88 @@ def delMaintainer(apiurl, prj, pac, user):
             print "user \'%s\' not found in \'%s\'" % (user, pac or prj)
     else:
         print "an error occured"
+
+def addFiles(filenames):
+    for filename in filenames:
+        if not os.path.exists(filename):
+            print >>sys.stderr, "file '%s' does not exist" % filename
+            return 1
+
+    # init a package dir if we have a normal dir in the "filenames"-list
+    # so that it will be find by findpacs() later
+    for filename in filenames:
+
+        prj_dir, pac_dir = getPrjPacPaths(filename)
+
+        if not is_package_dir(filename) and os.path.isdir(filename) and is_project_dir(prj_dir) \
+           and conf.config['do_package_tracking']:
+            old_dir = os.getcwd()
+            prj_name = store_read_project(prj_dir)
+            prj_apiurl = store_read_apiurl(prj_dir)
+            os.chdir(filename)
+            init_package_dir(prj_apiurl, prj_name, pac_dir, pac_dir, files=False)
+            os.chdir(old_dir)
+
+        elif is_package_dir(filename) and conf.config['do_package_tracking']:
+            print 'osc: warning: \'%s\' is already under version control' % filename
+            sys.exit(1)
+
+    pacs = findpacs(filenames)
+
+    for pac in pacs:
+        if conf.config['do_package_tracking'] and not pac.todo:
+            prj = Project(os.path.dirname(pac.absdir))
+            if pac.name in prj.pacs_unvers:
+                prj.addPackage(pac.name)
+                print statfrmt('A', getTransActPath(os.path.join(pac.dir, os.pardir, pac.name)))
+                for filename in pac.filenamelist_unvers:
+                    pac.todo.append(filename)
+            elif pac.name in prj.pacs_have:
+                print 'osc: warning: \'%s\' is already under version control' % pac.name
+        for filename in pac.todo:
+            if filename in pac.excluded:
+                continue
+            if filename in pac.filenamelist:
+                print >>sys.stderr, 'osc: warning: \'%s\' is already under version control' % filename
+                continue
+            if pac.dir != '.':
+                pathname = os.path.join(pac.dir, filename)
+            else:
+                pathname = filename
+            print statfrmt('A', pathname)
+            pac.addfile(filename)
+
+def getPrjPacPaths(path):
+    """
+    returns the path for a project and a package
+    from path. This is needed if you try to add
+    or delete packages:
+    Examples:
+        osc add pac1/: prj_dir = CWD;
+                       pac_dir = pac1
+        osc add /path/to/pac1:
+                       prj_dir = path/to;
+                       pac_dir = pac1
+        osc add /path/to/pac1/file
+                       => this would be an invalid path
+                          the caller has to validate the returned
+                          path!
+    """
+    # make sure we hddave a dir: osc add bar vs. osc add bar/; osc add /path/to/prj_dir/new_pack
+    # filename = os.path.join(tail, '')
+    prj_dir, pac_dir = os.path.split(os.path.normpath(path))
+    if prj_dir == '':
+        prj_dir = os.getcwd()
+    return (prj_dir, pac_dir)
+
+def getTransActPath(pac_dir):
+    """
+    returns the path for the commit and update operations/transactions.
+    Normally the "dir" attribute of a Package() object will be passed to
+    this method.
+    """
+    if pac_dir != '.':
+        pathn = os.path.normpath(pac_dir)
+    else:
+        pathn = ''
+    return pathn

tests.py

@@ -328,7 +328,7 @@ class TestOsc(unittest.TestCase):
 
         self.out, self.err = runosc('ci -m msg')
         self.assertEqual(self.err, '')
-        self.assertEqual(remove_revid(self.out), """Sending        foo1
+        self.assertEqual(remove_revid(self.out), """Sending    foo1
 Transmitting file data .
 Committed revision XX.
 """)
@@ -341,7 +341,7 @@ Committed revision XX.
 
         self.out, self.err = runosc('ci -m msg')
         self.assertEqual(self.err, '')
-        self.assertEqual(remove_revid(self.out), """Deleting       foo1
+        self.assertEqual(remove_revid(self.out), """Deleting    foo1
 Transmitting file data 
 Committed revision XX.
 """)
@@ -378,7 +378,7 @@ Committed revision XX.
         # check in a single argument
         self.out, self.err = runosc('ci -m msg foo2')
         self.assertEqual(self.err, '')
-        self.assertEqual(remove_revid(self.out), """Sending        foo2
+        self.assertEqual(remove_revid(self.out), """Sending    foo2
 Transmitting file data .
 Committed revision XX.
 """)
@@ -398,8 +398,8 @@ Committed revision XX.
         self.assertEqual(self.out, 'D    foo2\nA    bar1\n')
         self.out, self.err = runosc('ci')
         self.assertEqual(self.err, '')
-        self.assertEqual(remove_revid(self.out), """Sending        bar1
+        self.assertEqual(remove_revid(self.out), """Sending    bar1
-Deleting       foo2
+Deleting    foo2
 Transmitting file data .
 Committed revision XX.
 """)