added code to get the sha256 hashes of files

This is needed for a new validation of the source server.

The source server will 'ask' for the sha256 sum of files which are new or
modified and osc calculates the sha256 sums for those files and sends them
back to the server.

The server checks the sha256 sums and if dies if something is wrong.

osc/core.py

@@ -22,6 +22,7 @@ import re
 import socket
 import errno
 import shlex
+import hashlib
 
 try:
     from urllib.parse import urlsplit, urlunsplit, urlparse, quote_plus, urlencode, unquote
@@ -1393,7 +1394,7 @@ class Package:
             todo.append(n.get('name'))
         return todo
 
-    def __send_commitlog(self, msg, local_filelist):
+    def __send_commitlog(self, msg, local_filelist, validate=False):
         """send the commitlog and the local filelist to the server"""
         query = {}
         if self.islink() and self.isexpanded():
@@ -1405,6 +1406,8 @@ class Package:
                 query['linkrev'] = self.get_pulled_srcmd5()
         if self.islinkrepair():
             query['repairlink'] = '1'
+        if validate:
+            query['withvalidate'] = '1'
         return self.commit_filelist(self.apiurl, self.prjname, self.name,
                                     local_filelist, msg, **query)
 
@@ -1444,6 +1447,7 @@ class Package:
         todo_send = {}
         todo_delete = []
         real_send = []
+        sha256sums = {}
         for filename in self.filenamelist + [i for i in self.to_be_added if not i in self.filenamelist]:
             if filename.startswith('_service:') or filename.startswith('_service_'):
                 continue
@@ -1454,6 +1458,7 @@ class Package:
             elif filename in self.todo:
                 if st in ('A', 'R', 'M'):
                     todo_send[filename] = dgst(os.path.join(self.absdir, filename))
+                    sha256sums[filename] = sha256_dgst(os.path.join(self.absdir, filename))
                     real_send.append(filename)
                     print(statfrmt('Sending', os.path.join(pathn, filename)))
                 elif st in (' ', '!', 'S'):
@@ -1486,7 +1491,21 @@ class Package:
 
         print('Transmitting file data', end=' ')
         filelist = self.__generate_commitlist(todo_send)
-        sfilelist = self.__send_commitlog(msg, filelist)
+        sfilelist = self.__send_commitlog(msg, filelist, validate=True)
+        if sfilelist.get('error') and sfilelist.findall('.//entry[@hash]'):
+            name2elem = dict([(e.get('name'), e) for e in filelist.findall('entry')])
+            for entry in sfilelist.findall('.//entry[@hash]'):
+                filename = entry.get('name')
+                fileelem = name2elem.get(filename)
+                if filename not in sha256sums:
+                    msg = 'There is no sha256 sum for file %s.\n' \
+                          'This could be due to an outdated working copy.\n' \
+                          'Please update your working copy with osc update and\n' \
+                          'commit again afterwards.'
+                    print(msg % filename)
+                    return 1
+                fileelem.set('hash', 'sha256:%s' % sha256sums[filename])
+            sfilelist = self.__send_commitlog(msg, filelist)
         send = self.commit_get_missing(sfilelist)
         real_send = [i for i in real_send if not i in send]
         # abort after 3 tries
@@ -4629,6 +4648,18 @@ def dgst(file):
     f.close()
     return s.hexdigest()
 
+def sha256_dgst(file):
+
+    global BUFSIZE
+
+    f = open(file, 'rb')
+    s = hashlib.sha256()
+    while True:
+        buf = f.read(BUFSIZE)
+        if not buf: break
+        s.update(buf)
+    f.close()
+    return s.hexdigest()
 
 def binary(s):
     """return true if a string is binary data using diff's heuristic"""

tests/commit_fixtures/testAddedMissing_lfilelistwithSHA

@@ -0,0 +1 @@
+<directory><entry hash="sha256:aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f" md5="14758f1afd44c09b7992073ccf00b43d" name="bar" /><entry md5="0d62ceea6020d75154078a20d8c9f9ba" name="foo" /></directory>

tests/commit_fixtures/testAddedMissing_missingfilelistwithSHA

@@ -0,0 +1,3 @@
+<directory error="missing" name="added_missing">
+  <entry md5="14758f1afd44c09b7992073ccf00b43d" name="bar" hash="new"/>
+</directory>

tests/commit_fixtures/testAddedMissing_missingfilelistwithSHAsum

@@ -0,0 +1,3 @@
+<directory error="missing" name="added_missing">
+  <entry md5="14758f1afd44c09b7992073ccf00b43d" name="bar" hash="sha256:aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f"/>
+</directory>

tests/commit_fixtures/testSimple_lfilelistwithSHA

@@ -0,0 +1 @@
+<directory><entry md5="0d62ceea6020d75154078a20d8c9f9ba" name="foo" /><entry md5="17b9e9e1a032ed44e7a584dc6303ffa8" name="merge" /><entry hash="sha256:a531b3f2e3bb545ad9396dcfbb9973385b22e67bad2ac4c2bdf8f62ca05ecb02" md5="382588b92f5976de693f44c4d6df27b7" name="nochange" /></directory>

tests/commit_fixtures/testSimple_missingfilelistwithSHA

@@ -0,0 +1,3 @@
+<directory error="missing" name="simple">
+  <entry hash="missing" md5="c4eaea5dcaff13418e38e7fea151dd49" name="nochange" />
+</directory>

tests/commit_fixtures/testSimple_missingfilelistwithSHAsum

@@ -0,0 +1,3 @@
+<directory error="missing" name="simple">
+  <entry hash="sha256:a531b3f2e3bb545ad9396dcfbb9973385b22e67bad2ac4c2bdf8f62ca05ecb02" md5="c4eaea5dcaff13418e38e7fea151dd49" name="nochange" />
+</directory>

tests/test_commit.py

@@ -25,7 +25,7 @@ class TestCommit(OscTestCase):
     @GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
     @POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testSimple_missingfilelist', expfile='testSimple_lfilelist')
     @PUT('http://localhost/source/osctest/simple/nochange?rev=repository',
           exp='This file didn\'t change but\nis modified.\n', text=rev_dummy)
@@ -48,7 +48,7 @@ class TestCommit(OscTestCase):
     @GET('http://localhost/source/osctest/add?rev=latest', file='testAddfile_filesremote')
     @POST('http://localhost/source/osctest/add?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testAddfile_missingfilelist', expfile='testAddfile_lfilelist')
     @PUT('http://localhost/source/osctest/add/add?rev=repository',
          exp='added file\n', text=rev_dummy)
@@ -73,7 +73,7 @@ class TestCommit(OscTestCase):
     @GET('http://localhost/source/osctest/delete?rev=latest', file='testDeletefile_filesremote')
     @POST('http://localhost/source/osctest/delete?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/delete?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/delete?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testDeletefile_cfilesremote', expfile='testDeletefile_lfilelist')
     def test_deletefile(self):
         """delete a file"""
@@ -120,7 +120,7 @@ class TestCommit(OscTestCase):
     @GET('http://localhost/source/osctest/multiple?rev=latest', file='testMultiple_filesremote')
     @POST('http://localhost/source/osctest/multiple?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testMultiple_missingfilelist', expfile='testMultiple_lfilelist')
     @PUT('http://localhost/source/osctest/multiple/nochange?rev=repository', exp='This file did change.\n', text=rev_dummy)
     @PUT('http://localhost/source/osctest/multiple/add?rev=repository', exp='added file\n', text=rev_dummy)
@@ -149,7 +149,7 @@ class TestCommit(OscTestCase):
     @GET('http://localhost/source/osctest/multiple?rev=latest', file='testPartial_filesremote')
     @POST('http://localhost/source/osctest/multiple?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testPartial_missingfilelist', expfile='testPartial_lfilelist')
     @PUT('http://localhost/source/osctest/multiple/add?rev=repository', exp='added file\n', text=rev_dummy)
     @PUT('http://localhost/source/osctest/multiple/nochange?rev=repository', exp='This file did change.\n', text=rev_dummy)
@@ -176,7 +176,7 @@ class TestCommit(OscTestCase):
     @GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
     @POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testSimple_missingfilelist', expfile='testSimple_lfilelist')
     @PUT('http://localhost/source/osctest/simple/nochange?rev=repository', exp='This file didn\'t change but\nis modified.\n',
         exception=IOError('test exception'), text=rev_dummy)
@@ -194,7 +194,7 @@ class TestCommit(OscTestCase):
     @GET('http://localhost/source/osctest/allstates?rev=latest', file='testPartial_filesremote')
     @POST('http://localhost/source/osctest/allstates?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/allstates?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/allstates?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testAllStates_missingfilelist', expfile='testAllStates_lfilelist')
     @PUT('http://localhost/source/osctest/allstates/add?rev=repository', exp='added file\n', text=rev_dummy)
     @PUT('http://localhost/source/osctest/allstates/missing?rev=repository', exp='replaced\n', text=rev_dummy)
@@ -224,7 +224,7 @@ class TestCommit(OscTestCase):
     @GET('http://localhost/source/osctest/add?rev=latest', file='testAddfile_filesremote')
     @POST('http://localhost/source/osctest/add?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testAddfile_cfilesremote', expfile='testAddfile_lfilelist')
     def test_remoteexists(self):
         """file 'add' should be committed but already exists on the server"""
@@ -245,7 +245,7 @@ class TestCommit(OscTestCase):
     @GET('http://localhost/source/osctest/branch?rev=latest', file='testExpand_filesremote')
     @POST('http://localhost/source/osctest/branch?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/branch?comment=&cmd=commitfilelist&user=Admin&keeplink=1',
+    @POST('http://localhost/source/osctest/branch?comment=&cmd=commitfilelist&user=Admin&withvalidate=1&keeplink=1',
           file='testExpand_missingfilelist', expfile='testExpand_lfilelist')
     @PUT('http://localhost/source/osctest/branch/simple?rev=repository', exp='simple modified file.\n', text=rev_dummy)
     @POST('http://localhost/source/osctest/branch?comment=&cmd=commitfilelist&user=Admin&keeplink=1',
@@ -277,7 +277,7 @@ class TestCommit(OscTestCase):
     @GET('http://localhost/source/osctest/added_missing?rev=latest', file='testAddedMissing_filesremote')
     @POST('http://localhost/source/osctest/added_missing?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testAddedMissing_missingfilelist', expfile='testAddedMissing_lfilelist')
     @PUT('http://localhost/source/osctest/added_missing/bar?rev=repository', exp='foobar\n', text=rev_dummy)
     @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
@@ -296,7 +296,7 @@ class TestCommit(OscTestCase):
     @GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
     @POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
           exp='', text='<services />')
-    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
+    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
           file='testSimple_missingfilelist', expfile='testSimple_lfilelist')
     @PUT('http://localhost/source/osctest/simple/nochange?rev=repository',
           exp='This file didn\'t change but\nis modified.\n', text=rev_dummy)
@@ -312,6 +312,52 @@ class TestCommit(OscTestCase):
         self.assertEqual(sys.stdout.getvalue(), exp)
         self._check_status(p, 'nochange', 'M')
 
-if __name__ == '__main__':
+    @GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
+    @POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
+          exp='', text='<services />')
+    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
+          file='testSimple_missingfilelistwithSHA', expfile='testSimple_lfilelist')
+    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
+          file='testSimple_missingfilelistwithSHAsum', expfile='testSimple_lfilelistwithSHA')
+    @PUT('http://localhost/source/osctest/simple/nochange?rev=repository',
+          exp='This file didn\'t change but\nis modified.\n', text=rev_dummy)
+    @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
+          file='testSimple_cfilesremote', expfile='testSimple_lfilelistwithSHA')
+    def test_simple_sha256(self):
+        """a simple commit (only one modified file)"""
+        self._change_to_pkg('simple')
+        p = osc.core.Package('.')
+        p.commit()
+        exp = 'Sending    nochange\nTransmitting file data .\nCommitted revision 2.\n'
+        self.assertEqual(sys.stdout.getvalue(), exp)
+        self._check_digests('testSimple_cfilesremote')
+        self.assertTrue(os.path.exists('nochange'))
+        self.assertEqual(open('nochange', 'r').read(), open(os.path.join('.osc', 'nochange'), 'r').read())
+        self._check_status(p, 'nochange', ' ')
+        self._check_status(p, 'foo', ' ')
+        self._check_status(p, 'merge', ' ')
+
+    @GET('http://localhost/source/osctest/added_missing?rev=latest', file='testAddedMissing_filesremote')
+    @POST('http://localhost/source/osctest/added_missing?cmd=getprojectservices',
+          exp='', text='<services />')
+    @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
+          file='testAddedMissing_missingfilelistwithSHA', expfile='testAddedMissing_lfilelist')
+    @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
+          file='testAddedMissing_missingfilelistwithSHAsum', expfile='testAddedMissing_lfilelistwithSHA')
+    @PUT('http://localhost/source/osctest/added_missing/bar?rev=repository', exp='foobar\n', text=rev_dummy)
+    @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
+          file='testAddedMissing_cfilesremote', expfile='testAddedMissing_lfilelistwithSHA')
+    def test_added_missing2_sha256(self):
+        """commit an added file, another added file missing (but it's not part of the commit)"""
+        self._change_to_pkg('added_missing')
+        p = osc.core.Package('.')
+        p.todo = ['bar']
+        p.commit()
+        exp = 'Sending    bar\nTransmitting file data .\nCommitted revision 2.\n'
+        self.assertEqual(sys.stdout.getvalue(), exp)
+        self._check_status(p, 'add', '!')
+        self._check_status(p, 'bar', ' ')
+
+if  __name__ == '__main__':
     import unittest
     unittest.main()