From 1d866351b900d4186a4599a43eda4b8e2198b8ee Mon Sep 17 00:00:00 2001 From: lethliel Date: Tue, 23 Jul 2019 17:37:12 +0200 Subject: [PATCH 1/6] fix oscssl.py not verifying TLS cert verification use own implementation of HTTPSConnection (myHTTPSConnection) instead the one provided by M2Crypto (httpslib.HTTPConnection) And in proxy case use myProxyHTTPSConnection. all credits go to wfrisch --- osc/oscssl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osc/oscssl.py b/osc/oscssl.py index 312c236e..32c548d0 100644 --- a/osc/oscssl.py +++ b/osc/oscssl.py @@ -199,13 +199,13 @@ class myHTTPSHandler(M2Crypto.m2urllib2.HTTPSHandler): if target_host != host: request_uri = urldefrag(full_url)[0] - h = httpslib.ProxyHTTPSConnection(host=host, ssl_context=self.ctx) + h = myProxyHTTPSConnection(host=host, ssl_context=self.ctx) else: try: # up to python-3.2 request_uri = req.get_selector() except AttributeError: # from python-3.3 request_uri = req.selector - h = httpslib.HTTPSConnection(host=host, ssl_context=self.ctx) + h = myHTTPSConnection(host=host, ssl_context=self.ctx) # End our change h.set_debuglevel(self._debuglevel) From 837a63e9d67ead0ddb20aa1080061610b2822622 Mon Sep 17 00:00:00 2001 From: lethliel Date: Wed, 24 Jul 2019 08:23:50 +0200 Subject: [PATCH 2/6] updated NEWS --- NEWS | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index bd291887..bd36915e 100644 --- a/NEWS +++ b/NEWS @@ -1,5 +1,7 @@ 0.165.3 - - + - fix osc ls -lb handling empty size and mtime + - removed decoding on osc api command. + - fixed broken TLS certificate handling (boo#1142518, CVE-2019-3685) 0.165.2 - support different token operations (runservice, release and rebuild) (requires OBS 2.10) From 2b47be6b1e1eeb17957aba1f45c1af4433b6bdec Mon Sep 17 00:00:00 2001 From: lethliel Date: Wed, 17 Jul 2019 10:34:20 +0200 Subject: [PATCH 3/6] switch to difflib.diff_bytes in python3 case. The files are now opened as rb for diffing. In python2 nothing changes. In python3 the returned diff is bytes now. The following changes were made: * commandline.py: The returned diff is now bytes * get_diff now returs the diff as a bytes-like object * run_pager writes with sys.stdout.buffer.write if message is not a string * for the commit message the returned diff needs to be decoded now. Otherwise it will just producce garbage. For the commit message the diff on decoded bytes-objects is ok. (nothing harmfull can happen here) * fixed submit_action_diff * fixed request_interactive_review --- osc/commandline.py | 43 +++++++++------ osc/core.py | 112 ++++++++++++++++++++++------------------ tests/test_difffiles.py | 3 +- 3 files changed, 89 insertions(+), 69 deletions(-) diff --git a/osc/commandline.py b/osc/commandline.py index 8ea37ba3..d5be6f47 100644 --- a/osc/commandline.py +++ b/osc/commandline.py @@ -1405,16 +1405,19 @@ Please submit there instead, or use --nodevelproject to force direct submission. rdiff = None if opts.diff or not opts.message: try: - rdiff = 'old: %s/%s\nnew: %s/%s rev %s\n' % (dst_project, dst_package, src_project, src_package, rev) - rdiff += decode_it(server_diff(apiurl, - dst_project, dst_package, None, - src_project, src_package, rev, True)) + rdiff = b'old: %s/%s\nnew: %s/%s rev %s\n' % (dst_project.encode(), dst_package.encode(), src_project.encode(), src_package.encode(), str(rev).encode()) + rdiff += server_diff(apiurl, + dst_project, dst_package, None, + src_project, src_package, rev, True) except: - rdiff = '' + rdiff = b'' if opts.diff: run_pager(rdiff) return + if rdiff is not None: + rdiff = decode_it(rdiff) + supersede_existing = False reqs = [] if not opts.supersede: @@ -1510,13 +1513,13 @@ Please submit there instead, or use --nodevelproject to force direct submission. sys.exit("Please fix this first") t = linkinfo.get('project') if t: - rdiff = '' + rdiff = b'' try: rdiff = server_diff(apiurl, t, p, opts.revision, project, p, None, True) except: - rdiff = '' + rdiff = b'' - if rdiff != '': + if rdiff != b'': targetprojects.append(t) pac.append(p) else: @@ -2495,10 +2498,10 @@ Please submit there instead, or use --nodevelproject to force direct submission. if not r.get_actions('submit') and not r.get_actions('maintenance_incident') and not r.get_actions('maintenance_release'): raise oscerr.WrongOptions('\'--diff\' not possible (request has no supported actions)') for action in sr_actions: - diff += 'old: %s/%s\nnew: %s/%s\n' % (action.src_project, action.src_package, - action.tgt_project, action.tgt_package) + diff += b'old: %s/%s\nnew: %s/%s\n' % (action.src_project.encode(), action.src_package.encode(), + action.tgt_project.encode(), action.tgt_package.encode()) diff += submit_action_diff(apiurl, action) - diff += '\n\n' + diff += b'\n\n' run_pager(decode_it(diff), tmp_suffix='') # checkout @@ -3864,15 +3867,15 @@ Please submit there instead, or use --nodevelproject to force direct submission. return else: rev1, rev2 = parseRevisionOption(opts.revision) - diff = '' + diff = b'' for pac in pacs: if not rev2: for i in pac.get_diff(rev1): - diff += ''.join(i) + diff += b''.join(i) else: - diff += decode_it(server_diff_noex(pac.apiurl, pac.prjname, pac.name, rev1, - pac.prjname, pac.name, rev2, - not opts.plain, opts.missingok, opts.meta, not opts.unexpand)) + diff += server_diff_noex(pac.apiurl, pac.prjname, pac.name, rev1, + pac.prjname, pac.name, rev2, + not opts.plain, opts.missingok, opts.meta, not opts.unexpand) run_pager(diff) @@ -4156,7 +4159,13 @@ Please submit there instead, or use --nodevelproject to force direct submission. print("".join(decode_it(x) for x in p.stdout.readlines())) elif opts.unified: print() - print(decode_it(rdiff)) + if isinstance(rdiff, str): + print(rdiff) + else: + try: + sys.stdout.buffer.write(rdiff) + except AttributeError as e: + print(decode_it(rdiff)) #run_pager(rdiff) def _prdiff_output_matching_requests(self, opts, requests, diff --git a/osc/core.py b/osc/core.py index 23ca55f6..df497d4f 100644 --- a/osc/core.py +++ b/osc/core.py @@ -1963,26 +1963,30 @@ class Package: def get_diff(self, revision=None, ignoreUnversioned=False): import tempfile - diff_hdr = 'Index: %s\n' - diff_hdr += '===================================================================\n' + diff_hdr = b'Index: %s\n' + diff_hdr += b'===================================================================\n' kept = [] added = [] deleted = [] def diff_add_delete(fname, add, revision): diff = [] - diff.append(diff_hdr % fname) + diff.append(diff_hdr % fname.encode()) tmpfile = None origname = fname if add: - diff.append('--- %s\t(revision 0)\n' % fname) + diff.append(b'--- %s\t(revision 0)\n' % fname.encode()) rev = 'revision 0' if revision and not fname in self.to_be_added: rev = 'working copy' - diff.append('+++ %s\t(%s)\n' % (fname, rev)) + diff.append(b'+++ %s\t(%s)\n' % (fname.encode(), rev.encode())) fname = os.path.join(self.absdir, fname) else: - diff.append('--- %s\t(revision %s)\n' % (fname, revision or self.rev)) - diff.append('+++ %s\t(working copy)\n' % fname) + if revision: + b_revision = str(revision).encode() + else: + b_revision = self.rev.encode() + diff.append(b'--- %s\t(revision %s)\n' % (fname.encode(), b_revision)) + diff.append(b'+++ %s\t(working copy)\n' % fname.encode()) fname = os.path.join(self.storedir, fname) try: @@ -1991,22 +1995,22 @@ class Package: get_source_file(self.apiurl, self.prjname, self.name, origname, tmpfile, revision) fname = tmpfile if binary_file(fname): - what = 'added' + what = b'added' if not add: - what = 'deleted' + what = b'deleted' diff = diff[:1] - diff.append('Binary file \'%s\' %s.\n' % (origname, what)) + diff.append(b'Binary file \'%s\' %s.\n' % (origname.encode(), what)) return diff - tmpl = '+%s' - ltmpl = '@@ -0,0 +1,%d @@\n' + tmpl = b'+%s' + ltmpl = b'@@ -0,0 +1,%d @@\n' if not add: - tmpl = '-%s' - ltmpl = '@@ -1,%d +0,0 @@\n' - lines = [tmpl % i for i in open(fname, 'r').readlines()] + tmpl = b'-%s' + ltmpl = b'@@ -1,%d +0,0 @@\n' + lines = [tmpl % i for i in open(fname, 'rb').readlines()] if len(lines): diff.append(ltmpl % len(lines)) - if not lines[-1].endswith('\n'): - lines.append('\n\\ No newline at end of file\n') + if not lines[-1].endswith(b'\n'): + lines.append(b'\n\\ No newline at end of file\n') diff.extend(lines) finally: if tmpfile is not None: @@ -2051,7 +2055,7 @@ class Package: continue elif revision and self.findfilebyname(f.name).md5 == f.md5 and state != 'M': continue - yield [diff_hdr % f.name] + yield [diff_hdr % f.name.encode()] if revision is None: yield get_source_file_diff(self.absdir, f.name, self.rev) else: @@ -4053,7 +4057,7 @@ def run_pager(message, tmp_suffix=''): if isinstance(message, str): print(message) else: - print(decode_it(message)) + sys.stdout.buffer.write(message) else: tmpfile = tempfile.NamedTemporaryFile(suffix=tmp_suffix) if isinstance(message, str): @@ -4778,15 +4782,15 @@ def get_source_file_diff(dir, filename, rev, oldfilename = None, olddir = None, file1 = os.path.join(olddir, oldfilename) # old/stored original file2 = os.path.join(dir, filename) # working copy if binary_file(file1) or binary_file(file2): - return ['Binary file \'%s\' has changed.\n' % origfilename] + return [b'Binary file \'%s\' has changed.\n' % origfilename.encode()] f1 = f2 = None try: - f1 = open(file1, 'rt') + f1 = open(file1, 'rb') s1 = f1.readlines() f1.close() - f2 = open(file2, 'rt') + f2 = open(file2, 'rb') s2 = f2.readlines() f2.close() finally: @@ -4794,23 +4798,31 @@ def get_source_file_diff(dir, filename, rev, oldfilename = None, olddir = None, f1.close() if f2: f2.close() + + from_file = b'%s\t(revision %s)' % (origfilename.encode(), str(rev).encode()) + to_file = b'%s\t(working copy)' % origfilename.encode() - d = difflib.unified_diff(s1, s2, - fromfile = '%s\t(revision %s)' % (origfilename, rev), \ - tofile = '%s\t(working copy)' % origfilename) + if sys.version_info < (3,0): + d = difflib.unified_diff(s1, s2, + fromfile = from_file, \ + tofile = to_file) + else: + d = difflib.diff_bytes(difflib.unified_diff, s1, s2, \ + fromfile = from_file, \ + tofile = to_file) d = list(d) # python2.7's difflib slightly changed the format # adapt old format to the new format if len(d) > 1: - d[0] = d[0].replace(' \n', '\n') - d[1] = d[1].replace(' \n', '\n') + d[0] = d[0].replace(b' \n', b'\n') + d[1] = d[1].replace(b' \n', b'\n') # if file doesn't end with newline, we need to append one in the diff result for i, line in enumerate(d): - if not line.endswith('\n'): - d[i] += '\n\\ No newline at end of file' + if not line.endswith(b'\n'): + d[i] += b'\n\\ No newline at end of file' if i+1 != len(d): - d[i] += '\n' + d[i] += b'\n' return d def server_diff(apiurl, @@ -4866,14 +4878,14 @@ def server_diff_noex(apiurl, msg = None body = None try: - body = decode_it(e.read()) - if not 'bad link' in body: - return '# diff failed: ' + body + body = e.read() + if not b'bad link' in body: + return b'# diff failed: ' + body except: - return '# diff failed with unknown error' + return b'# diff failed with unknown error' if expand: - rdiff = "## diff on expanded link not possible, showing unexpanded version\n" + rdiff = b"## diff on expanded link not possible, showing unexpanded version\n" try: rdiff += server_diff_noex(apiurl, old_project, old_package, old_revision, @@ -4884,7 +4896,7 @@ def server_diff_noex(apiurl, summary = '' if not elm is None: summary = elm.text - return 'error: diffing failed: %s' % summary + return b'error: diffing failed: %s' % summary.encode() return rdiff @@ -4933,10 +4945,10 @@ def submit_action_diff(apiurl, action): if e.code != 404: raise e root = ET.fromstring(e.read()) - return 'error: \'%s\' does not exist' % root.find('summary').text + return b'error: \'%s\' does not exist' % root.find('summary').text.encode() elif e.code == 404: root = ET.fromstring(e.read()) - return 'error: \'%s\' does not exist' % root.find('summary').text + return b'error: \'%s\' does not exist' % root.find('summary').text.encode() raise e def make_dir(apiurl, project, package, pathname=None, prj_dir=None, package_tracking=True, pkg_path=None): @@ -7247,13 +7259,11 @@ def get_commit_message_template(pac): if pac.status(filename) == 'M': diff += get_source_file_diff(pac.absdir, filename, pac.rev) elif pac.status(filename) == 'A': - f = open(os.path.join(pac.absdir, filename), 'r') - for line in f: - diff += '+' + line - f.close() + with open(os.path.join(pac.absdir, filename), 'rb') as f: + diff.extend((b'+' + line for line in f)) if diff: - template = parse_diff_for_commit_message(''.join(diff)) + template = parse_diff_for_commit_message(''.join(decode_list(diff))) return template @@ -7288,7 +7298,7 @@ def get_commit_msg(wc_dir, pacs): if changed: footer += changed footer.append('\nDiff for working copy: %s' % p.dir) - footer.extend([''.join(i) for i in p.get_diff(ignoreUnversioned=True)]) + footer.extend([''.join(decode_list(i)) for i in p.get_diff(ignoreUnversioned=True)]) lines.extend(get_commit_message_template(p)) if template is None: if lines and lines[0] == '': @@ -7438,21 +7448,21 @@ def request_interactive_review(apiurl, request, initial_cmd='', group=None, tmpfile.close() tmpfile = None if tmpfile is None: - tmpfile = tempfile.NamedTemporaryFile(suffix='.diff', mode='r+') - tmpfile.write(req_summary) - tmpfile.write(issues) + tmpfile = tempfile.NamedTemporaryFile(suffix='.diff', mode='rb+') + tmpfile.write(req_summary.encode()) + tmpfile.write(issues.encode()) try: diff = request_diff(apiurl, request.reqid) - tmpfile.write(decode_it(diff)) + tmpfile.write(diff) except HTTPError as e: if e.code != 400: raise # backward compatible diff for old apis for action in src_actions: - diff = 'old: %s/%s\nnew: %s/%s\n' % (action.src_project, action.src_package, - action.tgt_project, action.tgt_package) + diff = b'old: %s/%s\nnew: %s/%s\n' % (action.src_project.encode(), action.src_package.encode(), + action.tgt_project.encode(), action.tgt_package.encode()) diff += submit_action_diff(apiurl, action) - diff += '\n\n' + diff += b'\n\n' tmpfile.write(diff) tmpfile.flush() run_editor(tmpfile.name) diff --git a/tests/test_difffiles.py b/tests/test_difffiles.py index 43c8afe1..3c430d8b 100644 --- a/tests/test_difffiles.py +++ b/tests/test_difffiles.py @@ -1,5 +1,6 @@ import osc.core import osc.oscerr +from osc.util.helper import decode_list import os import re from common import GET, OscTestCase @@ -298,7 +299,7 @@ Binary file 'binary' has changed. def __check_diff(self, p, exp, revision=None): got = '' for i in p.get_diff(revision): - got += ''.join(i) + got += ''.join(decode_list(i)) # When a hunk header refers to a single line in the "from" # file and/or the "to" file, e.g. From 26214f7dcdda43006790c91e4ca4cf6bf8ecec3b Mon Sep 17 00:00:00 2001 From: lethliel Date: Wed, 24 Jul 2019 15:13:39 +0200 Subject: [PATCH 4/6] release 0.165.3 --- NEWS | 2 ++ osc/core.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index bd36915e..16f4ef6d 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,6 @@ 0.165.3 + - switch to difflib.diff_bytes and sys.stdout.buffer.write for diffing. + This will fix all decoding issues with osc diff, osc ci and osc rq -d - fix osc ls -lb handling empty size and mtime - removed decoding on osc api command. - fixed broken TLS certificate handling (boo#1142518, CVE-2019-3685) diff --git a/osc/core.py b/osc/core.py index df497d4f..295597e8 100644 --- a/osc/core.py +++ b/osc/core.py @@ -5,7 +5,7 @@ from __future__ import print_function -__version__ = '0.165.3.git' +__version__ = '0.165.3' # __store_version__ is to be incremented when the format of the working copy # "store" changes in an incompatible way. Please add any needed migration From 5e39cdf6838f4af6eb4e19aa458e755650b0b99e Mon Sep 17 00:00:00 2001 From: lethliel Date: Wed, 24 Jul 2019 15:15:45 +0200 Subject: [PATCH 5/6] open 0.166 development --- NEWS | 3 +++ osc/core.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 16f4ef6d..ff7bae27 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,6 @@ +0.166 + - + 0.165.3 - switch to difflib.diff_bytes and sys.stdout.buffer.write for diffing. This will fix all decoding issues with osc diff, osc ci and osc rq -d diff --git a/osc/core.py b/osc/core.py index 295597e8..18b906b5 100644 --- a/osc/core.py +++ b/osc/core.py @@ -5,7 +5,7 @@ from __future__ import print_function -__version__ = '0.165.3' +__version__ = '0.166.git' # __store_version__ is to be incremented when the format of the working copy # "store" changes in an incompatible way. Please add any needed migration From 0a99f351263ab829f9077f1a1f1424a1527be65c Mon Sep 17 00:00:00 2001 From: Matthias Gerstner Date: Thu, 25 Jul 2019 10:58:56 +0200 Subject: [PATCH 6/6] babysitter: fix RPMError fallback when running in Python3 After (successfully) running an 'osc vc' the following exception trace comes up, when no rpm python module is available: ``` no changes made Traceback (most recent call last): File "/home/mgerstner/.local/lib64/python3.6/site-packages/osc/babysitter.py", line 62, in run return prg.main(argv) File "/home/mgerstner/.local/lib64/python3.6/site-packages/osc/cmdln.py", line 344, in main return self.cmd(args) File "/home/mgerstner/.local/lib64/python3.6/site-packages/osc/cmdln.py", line 367, in cmd retval = self.onecmd(argv) File "/home/mgerstner/.local/lib64/python3.6/site-packages/osc/cmdln.py", line 501, in onecmd return self._dispatch_cmd(handler, argv) File "/home/mgerstner/.local/lib64/python3.6/site-packages/osc/cmdln.py", line 1232, in _dispatch_cmd return handler(argv[0], opts, *args) File "/home/mgerstner/.local/lib64/python3.6/site-packages/osc/commandline.py", line 8924, in do_vc sys.exit(vc.returncode) SystemExit: 0 During handling of the above exception, another exception occurred: Traceback (most recent call last): File "/home/mgerstner/bin/osc", line 41, in r = babysitter.run(osccli) File "/home/mgerstner/.local/lib64/python3.6/site-packages/osc/babysitter.py", line 172, in run except RPMError as e: TypeError: catching classes that do not inherit from BaseException is not allowed ``` To fix this change the fallback RPMError from None to an actual Exception-derived type. --- osc/babysitter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/osc/babysitter.py b/osc/babysitter.py index 6ca92531..9a314cd1 100644 --- a/osc/babysitter.py +++ b/osc/babysitter.py @@ -30,7 +30,8 @@ try: from rpm import error as RPMError except: # if rpm-python isn't installed (we might be on a debian system): - RPMError = None + class RPMError(Exception): + pass try: from http.client import HTTPException, BadStatusLine