From f0325eb0b58c266eb0905ccf827dc7eb864378a1 Mon Sep 17 00:00:00 2001 From: lethliel Date: Mon, 27 Nov 2017 16:11:32 +0100 Subject: [PATCH] added code to get the sha256 hashes of files This is needed for a new validation of the source server. The source server will 'ask' for the sha256 sum of files which are new or modified and osc calculates the sha256 sums for those files and sends them back to the server. The server checks the sha256 sums and if dies if something is wrong. --- osc/core.py | 35 +++++++++- .../testAddedMissing_lfilelistwithSHA | 1 + .../testAddedMissing_missingfilelistwithSHA | 3 + ...testAddedMissing_missingfilelistwithSHAsum | 3 + .../testSimple_lfilelistwithSHA | 1 + .../testSimple_missingfilelistwithSHA | 3 + .../testSimple_missingfilelistwithSHAsum | 3 + tests/test_commit.py | 70 +++++++++++++++---- 8 files changed, 105 insertions(+), 14 deletions(-) create mode 100644 tests/commit_fixtures/testAddedMissing_lfilelistwithSHA create mode 100644 tests/commit_fixtures/testAddedMissing_missingfilelistwithSHA create mode 100644 tests/commit_fixtures/testAddedMissing_missingfilelistwithSHAsum create mode 100644 tests/commit_fixtures/testSimple_lfilelistwithSHA create mode 100644 tests/commit_fixtures/testSimple_missingfilelistwithSHA create mode 100644 tests/commit_fixtures/testSimple_missingfilelistwithSHAsum diff --git a/osc/core.py b/osc/core.py index 6d02c690..df499f27 100644 --- a/osc/core.py +++ b/osc/core.py @@ -22,6 +22,7 @@ import re import socket import errno import shlex +import hashlib try: from urllib.parse import urlsplit, urlunsplit, urlparse, quote_plus, urlencode, unquote @@ -1393,7 +1394,7 @@ class Package: todo.append(n.get('name')) return todo - def __send_commitlog(self, msg, local_filelist): + def __send_commitlog(self, msg, local_filelist, validate=False): """send the commitlog and the local filelist to the server""" query = {} if self.islink() and self.isexpanded(): @@ -1405,6 +1406,8 @@ class Package: query['linkrev'] = self.get_pulled_srcmd5() if self.islinkrepair(): query['repairlink'] = '1' + if validate: + query['withvalidate'] = '1' return self.commit_filelist(self.apiurl, self.prjname, self.name, local_filelist, msg, **query) @@ -1444,6 +1447,7 @@ class Package: todo_send = {} todo_delete = [] real_send = [] + sha256sums = {} for filename in self.filenamelist + [i for i in self.to_be_added if not i in self.filenamelist]: if filename.startswith('_service:') or filename.startswith('_service_'): continue @@ -1454,6 +1458,7 @@ class Package: elif filename in self.todo: if st in ('A', 'R', 'M'): todo_send[filename] = dgst(os.path.join(self.absdir, filename)) + sha256sums[filename] = sha256_dgst(os.path.join(self.absdir, filename)) real_send.append(filename) print(statfrmt('Sending', os.path.join(pathn, filename))) elif st in (' ', '!', 'S'): @@ -1486,7 +1491,21 @@ class Package: print('Transmitting file data', end=' ') filelist = self.__generate_commitlist(todo_send) - sfilelist = self.__send_commitlog(msg, filelist) + sfilelist = self.__send_commitlog(msg, filelist, validate=True) + if sfilelist.get('error') and sfilelist.findall('.//entry[@hash]'): + name2elem = dict([(e.get('name'), e) for e in filelist.findall('entry')]) + for entry in sfilelist.findall('.//entry[@hash]'): + filename = entry.get('name') + fileelem = name2elem.get(filename) + if filename not in sha256sums: + msg = 'There is no sha256 sum for file %s.\n' \ + 'This could be due to an outdated working copy.\n' \ + 'Please update your working copy with osc update and\n' \ + 'commit again afterwards.' + print(msg % filename) + return 1 + fileelem.set('hash', 'sha256:%s' % sha256sums[filename]) + sfilelist = self.__send_commitlog(msg, filelist) send = self.commit_get_missing(sfilelist) real_send = [i for i in real_send if not i in send] # abort after 3 tries @@ -4629,6 +4648,18 @@ def dgst(file): f.close() return s.hexdigest() +def sha256_dgst(file): + + global BUFSIZE + + f = open(file, 'rb') + s = hashlib.sha256() + while True: + buf = f.read(BUFSIZE) + if not buf: break + s.update(buf) + f.close() + return s.hexdigest() def binary(s): """return true if a string is binary data using diff's heuristic""" diff --git a/tests/commit_fixtures/testAddedMissing_lfilelistwithSHA b/tests/commit_fixtures/testAddedMissing_lfilelistwithSHA new file mode 100644 index 00000000..408146c6 --- /dev/null +++ b/tests/commit_fixtures/testAddedMissing_lfilelistwithSHA @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/commit_fixtures/testAddedMissing_missingfilelistwithSHA b/tests/commit_fixtures/testAddedMissing_missingfilelistwithSHA new file mode 100644 index 00000000..4895b5ee --- /dev/null +++ b/tests/commit_fixtures/testAddedMissing_missingfilelistwithSHA @@ -0,0 +1,3 @@ + + + diff --git a/tests/commit_fixtures/testAddedMissing_missingfilelistwithSHAsum b/tests/commit_fixtures/testAddedMissing_missingfilelistwithSHAsum new file mode 100644 index 00000000..13a25531 --- /dev/null +++ b/tests/commit_fixtures/testAddedMissing_missingfilelistwithSHAsum @@ -0,0 +1,3 @@ + + + diff --git a/tests/commit_fixtures/testSimple_lfilelistwithSHA b/tests/commit_fixtures/testSimple_lfilelistwithSHA new file mode 100644 index 00000000..aac9cf3c --- /dev/null +++ b/tests/commit_fixtures/testSimple_lfilelistwithSHA @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/commit_fixtures/testSimple_missingfilelistwithSHA b/tests/commit_fixtures/testSimple_missingfilelistwithSHA new file mode 100644 index 00000000..1a7a99b9 --- /dev/null +++ b/tests/commit_fixtures/testSimple_missingfilelistwithSHA @@ -0,0 +1,3 @@ + + + diff --git a/tests/commit_fixtures/testSimple_missingfilelistwithSHAsum b/tests/commit_fixtures/testSimple_missingfilelistwithSHAsum new file mode 100644 index 00000000..077cd516 --- /dev/null +++ b/tests/commit_fixtures/testSimple_missingfilelistwithSHAsum @@ -0,0 +1,3 @@ + + + diff --git a/tests/test_commit.py b/tests/test_commit.py index 02baa855..349fc4b4 100644 --- a/tests/test_commit.py +++ b/tests/test_commit.py @@ -25,7 +25,7 @@ class TestCommit(OscTestCase): @GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote') @POST('http://localhost/source/osctest/simple?cmd=getprojectservices', exp='', text='') - @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin', + @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1', file='testSimple_missingfilelist', expfile='testSimple_lfilelist') @PUT('http://localhost/source/osctest/simple/nochange?rev=repository', exp='This file didn\'t change but\nis modified.\n', text=rev_dummy) @@ -48,7 +48,7 @@ class TestCommit(OscTestCase): @GET('http://localhost/source/osctest/add?rev=latest', file='testAddfile_filesremote') @POST('http://localhost/source/osctest/add?cmd=getprojectservices', exp='', text='') - @POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin', + @POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin&withvalidate=1', file='testAddfile_missingfilelist', expfile='testAddfile_lfilelist') @PUT('http://localhost/source/osctest/add/add?rev=repository', exp='added file\n', text=rev_dummy) @@ -73,7 +73,7 @@ class TestCommit(OscTestCase): @GET('http://localhost/source/osctest/delete?rev=latest', file='testDeletefile_filesremote') @POST('http://localhost/source/osctest/delete?cmd=getprojectservices', exp='', text='') - @POST('http://localhost/source/osctest/delete?comment=&cmd=commitfilelist&user=Admin', + @POST('http://localhost/source/osctest/delete?comment=&cmd=commitfilelist&user=Admin&withvalidate=1', file='testDeletefile_cfilesremote', expfile='testDeletefile_lfilelist') def test_deletefile(self): """delete a file""" @@ -120,7 +120,7 @@ class TestCommit(OscTestCase): @GET('http://localhost/source/osctest/multiple?rev=latest', file='testMultiple_filesremote') @POST('http://localhost/source/osctest/multiple?cmd=getprojectservices', exp='', text='') - @POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin', + @POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1', file='testMultiple_missingfilelist', expfile='testMultiple_lfilelist') @PUT('http://localhost/source/osctest/multiple/nochange?rev=repository', exp='This file did change.\n', text=rev_dummy) @PUT('http://localhost/source/osctest/multiple/add?rev=repository', exp='added file\n', text=rev_dummy) @@ -149,7 +149,7 @@ class TestCommit(OscTestCase): @GET('http://localhost/source/osctest/multiple?rev=latest', file='testPartial_filesremote') @POST('http://localhost/source/osctest/multiple?cmd=getprojectservices', exp='', text='') - @POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin', + @POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1', file='testPartial_missingfilelist', expfile='testPartial_lfilelist') @PUT('http://localhost/source/osctest/multiple/add?rev=repository', exp='added file\n', text=rev_dummy) @PUT('http://localhost/source/osctest/multiple/nochange?rev=repository', exp='This file did change.\n', text=rev_dummy) @@ -176,7 +176,7 @@ class TestCommit(OscTestCase): @GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote') @POST('http://localhost/source/osctest/simple?cmd=getprojectservices', exp='', text='') - @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin', + @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1', file='testSimple_missingfilelist', expfile='testSimple_lfilelist') @PUT('http://localhost/source/osctest/simple/nochange?rev=repository', exp='This file didn\'t change but\nis modified.\n', exception=IOError('test exception'), text=rev_dummy) @@ -194,7 +194,7 @@ class TestCommit(OscTestCase): @GET('http://localhost/source/osctest/allstates?rev=latest', file='testPartial_filesremote') @POST('http://localhost/source/osctest/allstates?cmd=getprojectservices', exp='', text='') - @POST('http://localhost/source/osctest/allstates?comment=&cmd=commitfilelist&user=Admin', + @POST('http://localhost/source/osctest/allstates?comment=&cmd=commitfilelist&user=Admin&withvalidate=1', file='testAllStates_missingfilelist', expfile='testAllStates_lfilelist') @PUT('http://localhost/source/osctest/allstates/add?rev=repository', exp='added file\n', text=rev_dummy) @PUT('http://localhost/source/osctest/allstates/missing?rev=repository', exp='replaced\n', text=rev_dummy) @@ -224,7 +224,7 @@ class TestCommit(OscTestCase): @GET('http://localhost/source/osctest/add?rev=latest', file='testAddfile_filesremote') @POST('http://localhost/source/osctest/add?cmd=getprojectservices', exp='', text='') - @POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin', + @POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin&withvalidate=1', file='testAddfile_cfilesremote', expfile='testAddfile_lfilelist') def test_remoteexists(self): """file 'add' should be committed but already exists on the server""" @@ -245,7 +245,7 @@ class TestCommit(OscTestCase): @GET('http://localhost/source/osctest/branch?rev=latest', file='testExpand_filesremote') @POST('http://localhost/source/osctest/branch?cmd=getprojectservices', exp='', text='') - @POST('http://localhost/source/osctest/branch?comment=&cmd=commitfilelist&user=Admin&keeplink=1', + @POST('http://localhost/source/osctest/branch?comment=&cmd=commitfilelist&user=Admin&withvalidate=1&keeplink=1', file='testExpand_missingfilelist', expfile='testExpand_lfilelist') @PUT('http://localhost/source/osctest/branch/simple?rev=repository', exp='simple modified file.\n', text=rev_dummy) @POST('http://localhost/source/osctest/branch?comment=&cmd=commitfilelist&user=Admin&keeplink=1', @@ -277,7 +277,7 @@ class TestCommit(OscTestCase): @GET('http://localhost/source/osctest/added_missing?rev=latest', file='testAddedMissing_filesremote') @POST('http://localhost/source/osctest/added_missing?cmd=getprojectservices', exp='', text='') - @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin', + @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin&withvalidate=1', file='testAddedMissing_missingfilelist', expfile='testAddedMissing_lfilelist') @PUT('http://localhost/source/osctest/added_missing/bar?rev=repository', exp='foobar\n', text=rev_dummy) @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin', @@ -296,7 +296,7 @@ class TestCommit(OscTestCase): @GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote') @POST('http://localhost/source/osctest/simple?cmd=getprojectservices', exp='', text='') - @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin', + @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1', file='testSimple_missingfilelist', expfile='testSimple_lfilelist') @PUT('http://localhost/source/osctest/simple/nochange?rev=repository', exp='This file didn\'t change but\nis modified.\n', text=rev_dummy) @@ -312,6 +312,52 @@ class TestCommit(OscTestCase): self.assertEqual(sys.stdout.getvalue(), exp) self._check_status(p, 'nochange', 'M') -if __name__ == '__main__': + @GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote') + @POST('http://localhost/source/osctest/simple?cmd=getprojectservices', + exp='', text='') + @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1', + file='testSimple_missingfilelistwithSHA', expfile='testSimple_lfilelist') + @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin', + file='testSimple_missingfilelistwithSHAsum', expfile='testSimple_lfilelistwithSHA') + @PUT('http://localhost/source/osctest/simple/nochange?rev=repository', + exp='This file didn\'t change but\nis modified.\n', text=rev_dummy) + @POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin', + file='testSimple_cfilesremote', expfile='testSimple_lfilelistwithSHA') + def test_simple_sha256(self): + """a simple commit (only one modified file)""" + self._change_to_pkg('simple') + p = osc.core.Package('.') + p.commit() + exp = 'Sending nochange\nTransmitting file data .\nCommitted revision 2.\n' + self.assertEqual(sys.stdout.getvalue(), exp) + self._check_digests('testSimple_cfilesremote') + self.assertTrue(os.path.exists('nochange')) + self.assertEqual(open('nochange', 'r').read(), open(os.path.join('.osc', 'nochange'), 'r').read()) + self._check_status(p, 'nochange', ' ') + self._check_status(p, 'foo', ' ') + self._check_status(p, 'merge', ' ') + + @GET('http://localhost/source/osctest/added_missing?rev=latest', file='testAddedMissing_filesremote') + @POST('http://localhost/source/osctest/added_missing?cmd=getprojectservices', + exp='', text='') + @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin&withvalidate=1', + file='testAddedMissing_missingfilelistwithSHA', expfile='testAddedMissing_lfilelist') + @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin', + file='testAddedMissing_missingfilelistwithSHAsum', expfile='testAddedMissing_lfilelistwithSHA') + @PUT('http://localhost/source/osctest/added_missing/bar?rev=repository', exp='foobar\n', text=rev_dummy) + @POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin', + file='testAddedMissing_cfilesremote', expfile='testAddedMissing_lfilelistwithSHA') + def test_added_missing2_sha256(self): + """commit an added file, another added file missing (but it's not part of the commit)""" + self._change_to_pkg('added_missing') + p = osc.core.Package('.') + p.todo = ['bar'] + p.commit() + exp = 'Sending bar\nTransmitting file data .\nCommitted revision 2.\n' + self.assertEqual(sys.stdout.getvalue(), exp) + self._check_status(p, 'add', '!') + self._check_status(p, 'bar', ' ') + +if __name__ == '__main__': import unittest unittest.main()