mirrors/github.com_openSUSE_osc
1
0
Fork 0
mirror of https://github.com/openSUSE/osc.git synced 2025-01-24 05:46:13 +01:00
Code

added code to get the sha256 hashes of files

Browse Source 
This is needed for a new validation of the source server.

The source server will 'ask' for the sha256 sum of files which are new or
modified and osc calculates the sha256 sums for those files and sends them
back to the server.

The server checks the sha256 sums and if dies if something is wrong.
This commit is contained in:
lethliel 2017-11-27 16:11:32 +01:00
parent 03c25eb8f9
commit f0325eb0b5
8 changed files with 105 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
osc/core.py
View File

@ -22,6 +22,7 @@ import re
import socket
import errno
import shlex
import hashlib
try:
from urllib.parse import urlsplit, urlunsplit, urlparse, quote_plus, urlencode, unquote
@ -1393,7 +1394,7 @@ class Package:
todo.append(n.get('name'))
return todo
def __send_commitlog(self, msg, local_filelist):
def __send_commitlog(self, msg, local_filelist, validate=False):
"""send the commitlog and the local filelist to the server"""
query = {}
if self.islink() and self.isexpanded():
@ -1405,6 +1406,8 @@ class Package:
query['linkrev'] = self.get_pulled_srcmd5()
if self.islinkrepair():
query['repairlink'] = '1'
if validate:
query['withvalidate'] = '1'
return self.commit_filelist(self.apiurl, self.prjname, self.name,
local_filelist, msg, **query)
@ -1444,6 +1447,7 @@ class Package:
todo_send = {}
todo_delete = []
real_send = []
sha256sums = {}
for filename in self.filenamelist + [i for i in self.to_be_added if not i in self.filenamelist]:
if filename.startswith('_service:') or filename.startswith('_service_'):
continue
@ -1454,6 +1458,7 @@ class Package:
elif filename in self.todo:
if st in ('A', 'R', 'M'):
todo_send[filename] = dgst(os.path.join(self.absdir, filename))
sha256sums[filename] = sha256_dgst(os.path.join(self.absdir, filename))
real_send.append(filename)
print(statfrmt('Sending', os.path.join(pathn, filename)))
elif st in (' ', '!', 'S'):
@ -1486,7 +1491,21 @@ class Package:
print('Transmitting file data', end=' ')
filelist = self.__generate_commitlist(todo_send)
sfilelist = self.__send_commitlog(msg, filelist)
sfilelist = self.__send_commitlog(msg, filelist, validate=True)
if sfilelist.get('error') and sfilelist.findall('.//entry[@hash]'):
name2elem = dict([(e.get('name'), e) for e in filelist.findall('entry')])
for entry in sfilelist.findall('.//entry[@hash]'):
filename = entry.get('name')
fileelem = name2elem.get(filename)
if filename not in sha256sums:
msg = 'There is no sha256 sum for file %s.\n' \
'This could be due to an outdated working copy.\n' \
'Please update your working copy with osc update and\n' \
'commit again afterwards.'
print(msg % filename)
return 1
fileelem.set('hash', 'sha256:%s' % sha256sums[filename])
sfilelist = self.__send_commitlog(msg, filelist)
send = self.commit_get_missing(sfilelist)
real_send = [i for i in real_send if not i in send]
# abort after 3 tries
@ -4629,6 +4648,18 @@ def dgst(file):
f.close()
return s.hexdigest()
def sha256_dgst(file):
global BUFSIZE
f = open(file, 'rb')
s = hashlib.sha256()
while True:
buf = f.read(BUFSIZE)
if not buf: break
s.update(buf)
f.close()
return s.hexdigest()
def binary(s):
"""return true if a string is binary data using diff's heuristic"""

1
tests/commit_fixtures/testAddedMissing_lfilelistwithSHA Normal file
View File

@ -0,0 +1 @@
<directory><entry hash="sha256:aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f" md5="14758f1afd44c09b7992073ccf00b43d" name="bar" /><entry md5="0d62ceea6020d75154078a20d8c9f9ba" name="foo" /></directory>

3
tests/commit_fixtures/testAddedMissing_missingfilelistwithSHA Normal file
View File

@ -0,0 +1,3 @@
<directory error="missing" name="added_missing">
<entry md5="14758f1afd44c09b7992073ccf00b43d" name="bar" hash="new"/>
</directory>

3
tests/commit_fixtures/testAddedMissing_missingfilelistwithSHAsum Normal file
View File

@ -0,0 +1,3 @@
<directory error="missing" name="added_missing">
<entry md5="14758f1afd44c09b7992073ccf00b43d" name="bar" hash="sha256:aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f"/>
</directory>

1
tests/commit_fixtures/testSimple_lfilelistwithSHA Normal file
View File

@ -0,0 +1 @@
<directory><entry md5="0d62ceea6020d75154078a20d8c9f9ba" name="foo" /><entry md5="17b9e9e1a032ed44e7a584dc6303ffa8" name="merge" /><entry hash="sha256:a531b3f2e3bb545ad9396dcfbb9973385b22e67bad2ac4c2bdf8f62ca05ecb02" md5="382588b92f5976de693f44c4d6df27b7" name="nochange" /></directory>

3
tests/commit_fixtures/testSimple_missingfilelistwithSHA Normal file
View File

@ -0,0 +1,3 @@
<directory error="missing" name="simple">
<entry hash="missing" md5="c4eaea5dcaff13418e38e7fea151dd49" name="nochange" />
</directory>

3
tests/commit_fixtures/testSimple_missingfilelistwithSHAsum Normal file
View File

@ -0,0 +1,3 @@
<directory error="missing" name="simple">
<entry hash="sha256:a531b3f2e3bb545ad9396dcfbb9973385b22e67bad2ac4c2bdf8f62ca05ecb02" md5="c4eaea5dcaff13418e38e7fea151dd49" name="nochange" />
</directory>

70
tests/test_commit.py
View File

@ -25,7 +25,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
@POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
@POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testSimple_missingfilelist', expfile='testSimple_lfilelist')
@PUT('http://localhost/source/osctest/simple/nochange?rev=repository',
exp='This file didn\'t change but\nis modified.\n', text=rev_dummy)
@ -48,7 +48,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/add?rev=latest', file='testAddfile_filesremote')
@POST('http://localhost/source/osctest/add?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin',
@POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testAddfile_missingfilelist', expfile='testAddfile_lfilelist')
@PUT('http://localhost/source/osctest/add/add?rev=repository',
exp='added file\n', text=rev_dummy)
@ -73,7 +73,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/delete?rev=latest', file='testDeletefile_filesremote')
@POST('http://localhost/source/osctest/delete?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/delete?comment=&cmd=commitfilelist&user=Admin',
@POST('http://localhost/source/osctest/delete?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testDeletefile_cfilesremote', expfile='testDeletefile_lfilelist')
def test_deletefile(self):
"""delete a file"""
@ -120,7 +120,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/multiple?rev=latest', file='testMultiple_filesremote')
@POST('http://localhost/source/osctest/multiple?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin',
@POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testMultiple_missingfilelist', expfile='testMultiple_lfilelist')
@PUT('http://localhost/source/osctest/multiple/nochange?rev=repository', exp='This file did change.\n', text=rev_dummy)
@PUT('http://localhost/source/osctest/multiple/add?rev=repository', exp='added file\n', text=rev_dummy)
@ -149,7 +149,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/multiple?rev=latest', file='testPartial_filesremote')
@POST('http://localhost/source/osctest/multiple?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin',
@POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testPartial_missingfilelist', expfile='testPartial_lfilelist')
@PUT('http://localhost/source/osctest/multiple/add?rev=repository', exp='added file\n', text=rev_dummy)
@PUT('http://localhost/source/osctest/multiple/nochange?rev=repository', exp='This file did change.\n', text=rev_dummy)
@ -176,7 +176,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
@POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
@POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testSimple_missingfilelist', expfile='testSimple_lfilelist')
@PUT('http://localhost/source/osctest/simple/nochange?rev=repository', exp='This file didn\'t change but\nis modified.\n',
exception=IOError('test exception'), text=rev_dummy)
@ -194,7 +194,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/allstates?rev=latest', file='testPartial_filesremote')
@POST('http://localhost/source/osctest/allstates?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/allstates?comment=&cmd=commitfilelist&user=Admin',
@POST('http://localhost/source/osctest/allstates?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testAllStates_missingfilelist', expfile='testAllStates_lfilelist')
@PUT('http://localhost/source/osctest/allstates/add?rev=repository', exp='added file\n', text=rev_dummy)
@PUT('http://localhost/source/osctest/allstates/missing?rev=repository', exp='replaced\n', text=rev_dummy)
@ -224,7 +224,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/add?rev=latest', file='testAddfile_filesremote')
@POST('http://localhost/source/osctest/add?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin',
@POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testAddfile_cfilesremote', expfile='testAddfile_lfilelist')
def test_remoteexists(self):
"""file 'add' should be committed but already exists on the server"""
@ -245,7 +245,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/branch?rev=latest', file='testExpand_filesremote')
@POST('http://localhost/source/osctest/branch?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/branch?comment=&cmd=commitfilelist&user=Admin&keeplink=1',
@POST('http://localhost/source/osctest/branch?comment=&cmd=commitfilelist&user=Admin&withvalidate=1&keeplink=1',
file='testExpand_missingfilelist', expfile='testExpand_lfilelist')
@PUT('http://localhost/source/osctest/branch/simple?rev=repository', exp='simple modified file.\n', text=rev_dummy)
@POST('http://localhost/source/osctest/branch?comment=&cmd=commitfilelist&user=Admin&keeplink=1',
@ -277,7 +277,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/added_missing?rev=latest', file='testAddedMissing_filesremote')
@POST('http://localhost/source/osctest/added_missing?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
@POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testAddedMissing_missingfilelist', expfile='testAddedMissing_lfilelist')
@PUT('http://localhost/source/osctest/added_missing/bar?rev=repository', exp='foobar\n', text=rev_dummy)
@POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
@ -296,7 +296,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
@POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
@POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testSimple_missingfilelist', expfile='testSimple_lfilelist')
@PUT('http://localhost/source/osctest/simple/nochange?rev=repository',
exp='This file didn\'t change but\nis modified.\n', text=rev_dummy)
@ -312,6 +312,52 @@ class TestCommit(OscTestCase):
self.assertEqual(sys.stdout.getvalue(), exp)
self._check_status(p, 'nochange', 'M')
if __name__ == '__main__':
@GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
@POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testSimple_missingfilelistwithSHA', expfile='testSimple_lfilelist')
@POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
file='testSimple_missingfilelistwithSHAsum', expfile='testSimple_lfilelistwithSHA')
@PUT('http://localhost/source/osctest/simple/nochange?rev=repository',
exp='This file didn\'t change but\nis modified.\n', text=rev_dummy)
@POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
file='testSimple_cfilesremote', expfile='testSimple_lfilelistwithSHA')
def test_simple_sha256(self):
"""a simple commit (only one modified file)"""
self._change_to_pkg('simple')
p = osc.core.Package('.')
p.commit()
exp = 'Sending nochange\nTransmitting file data .\nCommitted revision 2.\n'
self.assertEqual(sys.stdout.getvalue(), exp)
self._check_digests('testSimple_cfilesremote')
self.assertTrue(os.path.exists('nochange'))
self.assertEqual(open('nochange', 'r').read(), open(os.path.join('.osc', 'nochange'), 'r').read())
self._check_status(p, 'nochange', ' ')
self._check_status(p, 'foo', ' ')
self._check_status(p, 'merge', ' ')
@GET('http://localhost/source/osctest/added_missing?rev=latest', file='testAddedMissing_filesremote')
@POST('http://localhost/source/osctest/added_missing?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testAddedMissing_missingfilelistwithSHA', expfile='testAddedMissing_lfilelist')
@POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
file='testAddedMissing_missingfilelistwithSHAsum', expfile='testAddedMissing_lfilelistwithSHA')
@PUT('http://localhost/source/osctest/added_missing/bar?rev=repository', exp='foobar\n', text=rev_dummy)
@POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
file='testAddedMissing_cfilesremote', expfile='testAddedMissing_lfilelistwithSHA')
def test_added_missing2_sha256(self):
"""commit an added file, another added file missing (but it's not part of the commit)"""
self._change_to_pkg('added_missing')
p = osc.core.Package('.')
p.todo = ['bar']
p.commit()
exp = 'Sending bar\nTransmitting file data .\nCommitted revision 2.\n'
self.assertEqual(sys.stdout.getvalue(), exp)
self._check_status(p, 'add', '!')
self._check_status(p, 'bar', ' ')
if __name__ == '__main__':
import unittest
unittest.main()