From 6bb8ca437b59b758c40bca4665d9b65e038907f6 Mon Sep 17 00:00:00 2001 From: Ruediger Meier Date: Thu, 22 May 2014 14:33:55 +0000 Subject: [PATCH 1/3] core: style, de-duplicate read() calls Signed-off-by: Ruediger Meier --- osc/core.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/osc/core.py b/osc/core.py index 226615df..9739e2ea 100644 --- a/osc/core.py +++ b/osc/core.py @@ -5366,14 +5366,17 @@ def streamfile(url, http_meth = http_GET, bufsize=8192, data=None, progress_obj= if progress_obj: basename = os.path.basename(urlsplit(url)[2]) progress_obj.start(basename=basename, text=text, size=cl) - data = f.read(bufsize) - read = len(data) - while len(data): + + read = 0 + while True: + data = f.read(bufsize) + if not len(data): + break + read += len(data) if progress_obj: progress_obj.update(read) yield data - data = f.read(bufsize) - read += len(data) + if progress_obj: progress_obj.end(read) f.close() From 1d7b9540226d46e74218b5b2ef84dc477aaae2e9 Mon Sep 17 00:00:00 2001 From: Ruediger Meier Date: Thu, 22 May 2014 14:52:59 +0000 Subject: [PATCH 2/3] core: streamfile() can read line by line Signed-off-by: Ruediger Meier --- osc/core.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/osc/core.py b/osc/core.py index 9739e2ea..1ab2103e 100644 --- a/osc/core.py +++ b/osc/core.py @@ -5341,7 +5341,7 @@ def streamfile(url, http_meth = http_GET, bufsize=8192, data=None, progress_obj= """ performs http_meth on url and read bufsize bytes from the response until EOF is reached. After each read bufsize bytes are yielded to the - caller. + caller. A spezial usage is bufsize="line" to read line by line (text). """ cl = '' retries = 0 @@ -5367,9 +5367,15 @@ def streamfile(url, http_meth = http_GET, bufsize=8192, data=None, progress_obj= basename = os.path.basename(urlsplit(url)[2]) progress_obj.start(basename=basename, text=text, size=cl) + if bufsize == "line": + bufsize = 8192 + xread = f.readline + else: + xread = f.read + read = 0 while True: - data = f.read(bufsize) + data = xread(bufsize) if not len(data): break read += len(data) From 7fc5936fafce2d846d0ef92c6e41b2ab4fc442bd Mon Sep 17 00:00:00 2001 From: Ruediger Meier Date: Thu, 22 May 2014 14:55:06 +0000 Subject: [PATCH 3/3] core: fix, buildlog --strip-time failed to remove time If a time field is not complete within the same read block then it can't be found by time_regex in buildlog_strip_time(). Fixed by simply reading line by line. I couldn't measure any performance difference neither for real nor user time. IMO no need to optimize for more lines per data chunk. Maybe it's even more fluent now for interactive users. BTW we can safely simplify time_regex. Signed-off-by: Ruediger Meier --- osc/core.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/osc/core.py b/osc/core.py index 1ab2103e..d7f40846 100644 --- a/osc/core.py +++ b/osc/core.py @@ -5393,7 +5393,7 @@ def streamfile(url, http_meth = http_GET, bufsize=8192, data=None, progress_obj= def buildlog_strip_time(data): """Strips the leading build time from the log""" - time_regex = re.compile('^\[\s{0,5}\d+s\]\s', re.M) + time_regex = re.compile('^\[[^\]]*\] ', re.M) return time_regex.sub('', data) @@ -5412,7 +5412,7 @@ def print_buildlog(apiurl, prj, package, repository, arch, offset=0, strip_time= query['start'] = offset start_offset = offset u = makeurl(apiurl, ['build', prj, repository, arch, package, '_log'], query=query) - for data in streamfile(u): + for data in streamfile(u, bufsize="line"): offset += len(data) if strip_time: data = buildlog_strip_time(data)