mirrors/github.com_openSUSE_osc
1
0
Fork 0
mirror of https://github.com/openSUSE/osc.git synced 2024-11-10 22:56:15 +01:00
Code

Merge branch 'check_hash_for_file' of https://github.com/lethliel/osc

Browse Source 
Validate the sha256 hashes for new, modified, and replaced files, if the
srcserver supports it.
This commit is contained in:
Marcus Huewe 2017-11-27 19:14:00 +01:00
commit a39b44a53b
8 changed files with 105 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
osc/core.py
View File

@ -22,6 +22,7 @@ import re
import socket
import errno
import shlex
import hashlib
try:
from urllib.parse import urlsplit, urlunsplit, urlparse, quote_plus, urlencode, unquote
@ -1393,7 +1394,7 @@ class Package:
todo.append(n.get('name'))
return todo
def __send_commitlog(self, msg, local_filelist):
def __send_commitlog(self, msg, local_filelist, validate=False):
"""send the commitlog and the local filelist to the server"""
query = {}
if self.islink() and self.isexpanded():
@ -1405,6 +1406,8 @@ class Package:
query['linkrev'] = self.get_pulled_srcmd5()
if self.islinkrepair():
query['repairlink'] = '1'
if validate:
query['withvalidate'] = '1'
return self.commit_filelist(self.apiurl, self.prjname, self.name,
local_filelist, msg, **query)
@ -1444,6 +1447,7 @@ class Package:
todo_send = {}
todo_delete = []
real_send = []
sha256sums = {}
for filename in self.filenamelist + [i for i in self.to_be_added if not i in self.filenamelist]:
if filename.startswith('_service:') or filename.startswith('_service_'):
continue
@ -1454,6 +1458,7 @@ class Package:
elif filename in self.todo:
if st in ('A', 'R', 'M'):
todo_send[filename] = dgst(os.path.join(self.absdir, filename))
sha256sums[filename] = sha256_dgst(os.path.join(self.absdir, filename))
real_send.append(filename)
print(statfrmt('Sending', os.path.join(pathn, filename)))
elif st in (' ', '!', 'S'):
@ -1486,7 +1491,21 @@ class Package:
print('Transmitting file data', end=' ')
filelist = self.__generate_commitlist(todo_send)
sfilelist = self.__send_commitlog(msg, filelist)
sfilelist = self.__send_commitlog(msg, filelist, validate=True)
if sfilelist.get('error') and sfilelist.findall('.//entry[@hash]'):
name2elem = dict([(e.get('name'), e) for e in filelist.findall('entry')])
for entry in sfilelist.findall('.//entry[@hash]'):
filename = entry.get('name')
fileelem = name2elem.get(filename)
if filename not in sha256sums:
msg = 'There is no sha256 sum for file %s.\n' \
'This could be due to an outdated working copy.\n' \
'Please update your working copy with osc update and\n' \
'commit again afterwards.'
print(msg % filename)
return 1
fileelem.set('hash', 'sha256:%s' % sha256sums[filename])
sfilelist = self.__send_commitlog(msg, filelist)
send = self.commit_get_missing(sfilelist)
real_send = [i for i in real_send if not i in send]
# abort after 3 tries
@ -4629,6 +4648,18 @@ def dgst(file):
f.close()
return s.hexdigest()
def sha256_dgst(file):
global BUFSIZE
f = open(file, 'rb')
s = hashlib.sha256()
while True:
buf = f.read(BUFSIZE)
if not buf: break
s.update(buf)
f.close()
return s.hexdigest()
def binary(s):
"""return true if a string is binary data using diff's heuristic"""

1
tests/commit_fixtures/testAddedMissing_lfilelistwithSHA Normal file
View File

@ -0,0 +1 @@
<directory><entry hash="sha256:aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f" md5="14758f1afd44c09b7992073ccf00b43d" name="bar" /><entry md5="0d62ceea6020d75154078a20d8c9f9ba" name="foo" /></directory>

3
tests/commit_fixtures/testAddedMissing_missingfilelistwithSHA Normal file
View File

@ -0,0 +1,3 @@
<directory error="missing" name="added_missing">
<entry md5="14758f1afd44c09b7992073ccf00b43d" name="bar" hash="new"/>
</directory>

3
tests/commit_fixtures/testAddedMissing_missingfilelistwithSHAsum Normal file
View File

@ -0,0 +1,3 @@
<directory error="missing" name="added_missing">
<entry md5="14758f1afd44c09b7992073ccf00b43d" name="bar" hash="sha256:aec070645fe53ee3b3763059376134f058cc337247c978add178b6ccdfb0019f"/>
</directory>

1
tests/commit_fixtures/testSimple_lfilelistwithSHA Normal file
View File

@ -0,0 +1 @@
<directory><entry md5="0d62ceea6020d75154078a20d8c9f9ba" name="foo" /><entry md5="17b9e9e1a032ed44e7a584dc6303ffa8" name="merge" /><entry hash="sha256:a531b3f2e3bb545ad9396dcfbb9973385b22e67bad2ac4c2bdf8f62ca05ecb02" md5="382588b92f5976de693f44c4d6df27b7" name="nochange" /></directory>

3
tests/commit_fixtures/testSimple_missingfilelistwithSHA Normal file
View File

@ -0,0 +1,3 @@
<directory error="missing" name="simple">
<entry hash="missing" md5="c4eaea5dcaff13418e38e7fea151dd49" name="nochange" />
</directory>

3
tests/commit_fixtures/testSimple_missingfilelistwithSHAsum Normal file
View File

@ -0,0 +1,3 @@
<directory error="missing" name="simple">
<entry hash="sha256:a531b3f2e3bb545ad9396dcfbb9973385b22e67bad2ac4c2bdf8f62ca05ecb02" md5="c4eaea5dcaff13418e38e7fea151dd49" name="nochange" />
</directory>

70
tests/test_commit.py
View File

@ -25,7 +25,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
@POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
@POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testSimple_missingfilelist', expfile='testSimple_lfilelist')
@PUT('http://localhost/source/osctest/simple/nochange?rev=repository',
exp='This file didn\'t change but\nis modified.\n', text=rev_dummy)
@ -48,7 +48,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/add?rev=latest', file='testAddfile_filesremote')
@POST('http://localhost/source/osctest/add?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin',
@POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testAddfile_missingfilelist', expfile='testAddfile_lfilelist')
@PUT('http://localhost/source/osctest/add/add?rev=repository',
exp='added file\n', text=rev_dummy)
@ -73,7 +73,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/delete?rev=latest', file='testDeletefile_filesremote')
@POST('http://localhost/source/osctest/delete?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/delete?comment=&cmd=commitfilelist&user=Admin',
@POST('http://localhost/source/osctest/delete?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testDeletefile_cfilesremote', expfile='testDeletefile_lfilelist')
def test_deletefile(self):
"""delete a file"""
@ -120,7 +120,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/multiple?rev=latest', file='testMultiple_filesremote')
@POST('http://localhost/source/osctest/multiple?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin',
@POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testMultiple_missingfilelist', expfile='testMultiple_lfilelist')
@PUT('http://localhost/source/osctest/multiple/nochange?rev=repository', exp='This file did change.\n', text=rev_dummy)
@PUT('http://localhost/source/osctest/multiple/add?rev=repository', exp='added file\n', text=rev_dummy)
@ -149,7 +149,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/multiple?rev=latest', file='testPartial_filesremote')
@POST('http://localhost/source/osctest/multiple?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin',
@POST('http://localhost/source/osctest/multiple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testPartial_missingfilelist', expfile='testPartial_lfilelist')
@PUT('http://localhost/source/osctest/multiple/add?rev=repository', exp='added file\n', text=rev_dummy)
@PUT('http://localhost/source/osctest/multiple/nochange?rev=repository', exp='This file did change.\n', text=rev_dummy)
@ -176,7 +176,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
@POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
@POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testSimple_missingfilelist', expfile='testSimple_lfilelist')
@PUT('http://localhost/source/osctest/simple/nochange?rev=repository', exp='This file didn\'t change but\nis modified.\n',
exception=IOError('test exception'), text=rev_dummy)
@ -194,7 +194,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/allstates?rev=latest', file='testPartial_filesremote')
@POST('http://localhost/source/osctest/allstates?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/allstates?comment=&cmd=commitfilelist&user=Admin',
@POST('http://localhost/source/osctest/allstates?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testAllStates_missingfilelist', expfile='testAllStates_lfilelist')
@PUT('http://localhost/source/osctest/allstates/add?rev=repository', exp='added file\n', text=rev_dummy)
@PUT('http://localhost/source/osctest/allstates/missing?rev=repository', exp='replaced\n', text=rev_dummy)
@ -224,7 +224,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/add?rev=latest', file='testAddfile_filesremote')
@POST('http://localhost/source/osctest/add?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin',
@POST('http://localhost/source/osctest/add?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testAddfile_cfilesremote', expfile='testAddfile_lfilelist')
def test_remoteexists(self):
"""file 'add' should be committed but already exists on the server"""
@ -245,7 +245,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/branch?rev=latest', file='testExpand_filesremote')
@POST('http://localhost/source/osctest/branch?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/branch?comment=&cmd=commitfilelist&user=Admin&keeplink=1',
@POST('http://localhost/source/osctest/branch?comment=&cmd=commitfilelist&user=Admin&withvalidate=1&keeplink=1',
file='testExpand_missingfilelist', expfile='testExpand_lfilelist')
@PUT('http://localhost/source/osctest/branch/simple?rev=repository', exp='simple modified file.\n', text=rev_dummy)
@POST('http://localhost/source/osctest/branch?comment=&cmd=commitfilelist&user=Admin&keeplink=1',
@ -277,7 +277,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/added_missing?rev=latest', file='testAddedMissing_filesremote')
@POST('http://localhost/source/osctest/added_missing?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
@POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testAddedMissing_missingfilelist', expfile='testAddedMissing_lfilelist')
@PUT('http://localhost/source/osctest/added_missing/bar?rev=repository', exp='foobar\n', text=rev_dummy)
@POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
@ -296,7 +296,7 @@ class TestCommit(OscTestCase):
@GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
@POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
@POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testSimple_missingfilelist', expfile='testSimple_lfilelist')
@PUT('http://localhost/source/osctest/simple/nochange?rev=repository',
exp='This file didn\'t change but\nis modified.\n', text=rev_dummy)
@ -312,6 +312,52 @@ class TestCommit(OscTestCase):
self.assertEqual(sys.stdout.getvalue(), exp)
self._check_status(p, 'nochange', 'M')
if __name__ == '__main__':
@GET('http://localhost/source/osctest/simple?rev=latest', file='testSimple_filesremote')
@POST('http://localhost/source/osctest/simple?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testSimple_missingfilelistwithSHA', expfile='testSimple_lfilelist')
@POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
file='testSimple_missingfilelistwithSHAsum', expfile='testSimple_lfilelistwithSHA')
@PUT('http://localhost/source/osctest/simple/nochange?rev=repository',
exp='This file didn\'t change but\nis modified.\n', text=rev_dummy)
@POST('http://localhost/source/osctest/simple?comment=&cmd=commitfilelist&user=Admin',
file='testSimple_cfilesremote', expfile='testSimple_lfilelistwithSHA')
def test_simple_sha256(self):
"""a simple commit (only one modified file)"""
self._change_to_pkg('simple')
p = osc.core.Package('.')
p.commit()
exp = 'Sending nochange\nTransmitting file data .\nCommitted revision 2.\n'
self.assertEqual(sys.stdout.getvalue(), exp)
self._check_digests('testSimple_cfilesremote')
self.assertTrue(os.path.exists('nochange'))
self.assertEqual(open('nochange', 'r').read(), open(os.path.join('.osc', 'nochange'), 'r').read())
self._check_status(p, 'nochange', ' ')
self._check_status(p, 'foo', ' ')
self._check_status(p, 'merge', ' ')
@GET('http://localhost/source/osctest/added_missing?rev=latest', file='testAddedMissing_filesremote')
@POST('http://localhost/source/osctest/added_missing?cmd=getprojectservices',
exp='', text='<services />')
@POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin&withvalidate=1',
file='testAddedMissing_missingfilelistwithSHA', expfile='testAddedMissing_lfilelist')
@POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
file='testAddedMissing_missingfilelistwithSHAsum', expfile='testAddedMissing_lfilelistwithSHA')
@PUT('http://localhost/source/osctest/added_missing/bar?rev=repository', exp='foobar\n', text=rev_dummy)
@POST('http://localhost/source/osctest/added_missing?comment=&cmd=commitfilelist&user=Admin',
file='testAddedMissing_cfilesremote', expfile='testAddedMissing_lfilelistwithSHA')
def test_added_missing2_sha256(self):
"""commit an added file, another added file missing (but it's not part of the commit)"""
self._change_to_pkg('added_missing')
p = osc.core.Package('.')
p.todo = ['bar']
p.commit()
exp = 'Sending bar\nTransmitting file data .\nCommitted revision 2.\n'
self.assertEqual(sys.stdout.getvalue(), exp)
self._check_status(p, 'add', '!')
self._check_status(p, 'bar', ' ')
if __name__ == '__main__':
import unittest
unittest.main()