mirror of
https://gitlab.gnome.org/GNOME/glib.git
synced 2025-10-31 16:32:18 +01:00
52c831099e
In order to distinguish reports generated from a specific revision of a projects it would be good if gtester-report handled a <revision> node in the XML. The payload is free-form, just like for the other nodes under <info>. https://bugzilla.gnome.org/show_bug.cgi?id=631980
493 lines
18 KiB
Python
Executable File
493 lines
18 KiB
Python
Executable File
#! /usr/bin/env python
|
|
# GLib Testing Framework Utility -*- Mode: python; -*-
|
|
# Copyright (C) 2007 Imendio AB
|
|
# Authors: Tim Janik
|
|
#
|
|
# This library is free software; you can redistribute it and/or
|
|
# modify it under the terms of the GNU Lesser General Public
|
|
# License as published by the Free Software Foundation; either
|
|
# version 2 of the License, or (at your option) any later version.
|
|
#
|
|
# This library is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
# Lesser General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Lesser General Public
|
|
# License along with this library; if not, write to the
|
|
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
|
|
# Boston, MA 02111-1307, USA.
|
|
import datetime
|
|
import optparse
|
|
import sys, re, xml.dom.minidom
|
|
|
|
try:
|
|
import subunit
|
|
from subunit import iso8601
|
|
from testtools.content import Content, ContentType
|
|
mime_utf8 = ContentType('text', 'plain', {'charset': 'utf8'})
|
|
except ImportError:
|
|
subunit = None
|
|
|
|
|
|
pkginstall_configvars = {
|
|
#@PKGINSTALL_CONFIGVARS_IN24LINES@ # configvars are substituted upon script installation
|
|
}
|
|
|
|
# xml utilities
|
|
def find_child (node, child_name):
|
|
for child in node.childNodes:
|
|
if child.nodeName == child_name:
|
|
return child
|
|
return None
|
|
def list_children (node, child_name):
|
|
rlist = []
|
|
for child in node.childNodes:
|
|
if child.nodeName == child_name:
|
|
rlist += [ child ]
|
|
return rlist
|
|
def find_node (node, name = None):
|
|
if not node or node.nodeName == name or not name:
|
|
return node
|
|
for child in node.childNodes:
|
|
c = find_node (child, name)
|
|
if c:
|
|
return c
|
|
return None
|
|
def node_as_text (node, name = None):
|
|
if name:
|
|
node = find_node (node, name)
|
|
txt = ''
|
|
if node:
|
|
if node.nodeValue:
|
|
txt += node.nodeValue
|
|
for child in node.childNodes:
|
|
txt += node_as_text (child)
|
|
return txt
|
|
def attribute_as_text (node, aname, node_name = None):
|
|
node = find_node (node, node_name)
|
|
if not node:
|
|
return ''
|
|
attr = node.attributes.get (aname, '')
|
|
if hasattr (attr, 'value'):
|
|
return attr.value
|
|
return ''
|
|
|
|
# HTML utilities
|
|
def html_indent_string (n):
|
|
uncollapsible_space = ' ' # HTML won't compress alternating sequences of ' ' and ' '
|
|
string = ''
|
|
for i in range (0, (n + 1) / 2):
|
|
string += uncollapsible_space
|
|
return string
|
|
|
|
# TestBinary object, instantiated per test binary in the log file
|
|
class TestBinary:
|
|
def __init__ (self, name):
|
|
self.name = name
|
|
self.testcases = []
|
|
self.duration = 0
|
|
self.success_cases = 0
|
|
self.skipped_cases = 0
|
|
self.file = '???'
|
|
self.random_seed = ''
|
|
|
|
# base class to handle processing/traversion of XML nodes
|
|
class TreeProcess:
|
|
def __init__ (self):
|
|
self.nest_level = 0
|
|
def trampoline (self, node):
|
|
name = node.nodeName
|
|
if name == '#text':
|
|
self.handle_text (node)
|
|
else:
|
|
try: method = getattr (self, 'handle_' + re.sub ('[^a-zA-Z0-9]', '_', name))
|
|
except: method = None
|
|
if method:
|
|
return method (node)
|
|
else:
|
|
return self.process_recursive (name, node)
|
|
def process_recursive (self, node_name, node):
|
|
self.process_children (node)
|
|
def process_children (self, node):
|
|
self.nest_level += 1
|
|
for child in node.childNodes:
|
|
self.trampoline (child)
|
|
self.nest_level += 1
|
|
|
|
# test report reader, this class collects some statistics and merges duplicate test binary runs
|
|
class ReportReader (TreeProcess):
|
|
def __init__ (self):
|
|
TreeProcess.__init__ (self)
|
|
self.binary_names = []
|
|
self.binaries = {}
|
|
self.last_binary = None
|
|
self.info = {}
|
|
def binary_list (self):
|
|
lst = []
|
|
for name in self.binary_names:
|
|
lst += [ self.binaries[name] ]
|
|
return lst
|
|
def get_info (self):
|
|
return self.info
|
|
def handle_info (self, node):
|
|
dn = find_child (node, 'package')
|
|
self.info['package'] = node_as_text (dn)
|
|
dn = find_child (node, 'version')
|
|
self.info['version'] = node_as_text (dn)
|
|
dn = find_child (node, 'revision')
|
|
if dn is not None:
|
|
self.info['revision'] = node_as_text (dn)
|
|
def handle_testcase (self, node):
|
|
self.last_binary.testcases += [ node ]
|
|
result = attribute_as_text (node, 'result', 'status')
|
|
if result == 'success':
|
|
self.last_binary.success_cases += 1
|
|
if bool (int (attribute_as_text (node, 'skipped') + '0')):
|
|
self.last_binary.skipped_cases += 1
|
|
def handle_text (self, node):
|
|
pass
|
|
def handle_testbinary (self, node):
|
|
path = node.attributes.get ('path', None).value
|
|
if self.binaries.get (path, -1) == -1:
|
|
self.binaries[path] = TestBinary (path)
|
|
self.binary_names += [ path ]
|
|
self.last_binary = self.binaries[path]
|
|
dn = find_child (node, 'duration')
|
|
dur = node_as_text (dn)
|
|
try: dur = float (dur)
|
|
except: dur = 0
|
|
if dur:
|
|
self.last_binary.duration += dur
|
|
bin = find_child (node, 'binary')
|
|
if bin:
|
|
self.last_binary.file = attribute_as_text (bin, 'file')
|
|
rseed = find_child (node, 'random-seed')
|
|
if rseed:
|
|
self.last_binary.random_seed = node_as_text (rseed)
|
|
self.process_children (node)
|
|
|
|
|
|
class ReportWriter(object):
|
|
"""Base class for reporting."""
|
|
|
|
def __init__(self, binary_list):
|
|
self.binaries = binary_list
|
|
|
|
def _error_text(self, node):
|
|
"""Get a string representing the error children of node."""
|
|
rlist = list_children(node, 'error')
|
|
txt = ''
|
|
for enode in rlist:
|
|
txt += node_as_text (enode)
|
|
if txt and txt[-1] != '\n':
|
|
txt += '\n'
|
|
return txt
|
|
|
|
|
|
class HTMLReportWriter(ReportWriter):
|
|
# Javascript/CSS snippet to toggle element visibility
|
|
cssjs = r'''
|
|
<style type="text/css" media="screen">
|
|
.VisibleSection { }
|
|
.HiddenSection { display: none; }
|
|
</style>
|
|
<script language="javascript" type="text/javascript"><!--
|
|
function toggle_display (parentid, tagtype, idmatch, keymatch) {
|
|
ptag = document.getElementById (parentid);
|
|
tags = ptag.getElementsByTagName (tagtype);
|
|
for (var i = 0; i < tags.length; i++) {
|
|
tag = tags[i];
|
|
var key = tag.getAttribute ("keywords");
|
|
if (tag.id.indexOf (idmatch) == 0 && key && key.match (keymatch)) {
|
|
if (tag.className.indexOf ("HiddenSection") >= 0)
|
|
tag.className = "VisibleSection";
|
|
else
|
|
tag.className = "HiddenSection";
|
|
}
|
|
}
|
|
}
|
|
message_array = Array();
|
|
function view_testlog (wname, file, random_seed, tcase, msgtitle, msgid) {
|
|
txt = message_array[msgid];
|
|
var w = window.open ("", // URI
|
|
wname,
|
|
"resizable,scrollbars,status,width=790,height=400");
|
|
var doc = w.document;
|
|
doc.write ("<h2>File: " + file + "</h2>\n");
|
|
doc.write ("<h3>Case: " + tcase + "</h3>\n");
|
|
doc.write ("<strong>Random Seed:</strong> <code>" + random_seed + "</code> <br /><br />\n");
|
|
doc.write ("<strong>" + msgtitle + "</strong><br />\n");
|
|
doc.write ("<pre>");
|
|
doc.write (txt);
|
|
doc.write ("</pre>\n");
|
|
doc.write ("<a href=\'javascript:window.close()\'>Close Window</a>\n");
|
|
doc.close();
|
|
}
|
|
--></script>
|
|
'''
|
|
def __init__ (self, info, binary_list):
|
|
ReportWriter.__init__(self, binary_list)
|
|
self.info = info
|
|
self.bcounter = 0
|
|
self.tcounter = 0
|
|
self.total_tcounter = 0
|
|
self.total_fcounter = 0
|
|
self.total_duration = 0
|
|
self.indent_depth = 0
|
|
self.lastchar = ''
|
|
def oprint (self, message):
|
|
sys.stdout.write (message)
|
|
if message:
|
|
self.lastchar = message[-1]
|
|
def handle_info (self):
|
|
self.oprint ('<h3>Package: %(package)s, version: %(version)s</h3>\n' % self.info)
|
|
if self.info['revision']:
|
|
self.oprint ('<h5>Report generated from: %(revision)s</h5>\n' % self.info)
|
|
def handle_text (self, node):
|
|
self.oprint (node.nodeValue)
|
|
def handle_testcase (self, node, binary):
|
|
skipped = bool (int (attribute_as_text (node, 'skipped') + '0'))
|
|
if skipped:
|
|
return # skipped tests are uninteresting for HTML reports
|
|
path = attribute_as_text (node, 'path')
|
|
duration = node_as_text (node, 'duration')
|
|
result = attribute_as_text (node, 'result', 'status')
|
|
rcolor = {
|
|
'success': 'bgcolor="lightgreen"',
|
|
'failed': 'bgcolor="red"',
|
|
}.get (result, '')
|
|
if result != 'success':
|
|
duration = '-' # ignore bogus durations
|
|
self.oprint ('<tr id="b%u_t%u_" keywords="%s all" class="HiddenSection">\n' % (self.bcounter, self.tcounter, result))
|
|
self.oprint ('<td>%s %s</td> <td align="right">%s</td> \n' % (html_indent_string (4), path, duration))
|
|
perflist = list_children (node, 'performance')
|
|
if result != 'success':
|
|
txt = self._error_text(node)
|
|
txt = re.sub (r'"', r'\\"', txt)
|
|
txt = re.sub (r'\n', r'\\n', txt)
|
|
txt = re.sub (r'&', r'&', txt)
|
|
txt = re.sub (r'<', r'<', txt)
|
|
self.oprint ('<script language="javascript" type="text/javascript">message_array["b%u_t%u_"] = "%s";</script>\n' % (self.bcounter, self.tcounter, txt))
|
|
self.oprint ('<td align="center"><a href="javascript:view_testlog (\'%s\', \'%s\', \'%s\', \'%s\', \'Output:\', \'b%u_t%u_\')">Details</a></td>\n' %
|
|
('TestResultWindow', binary.file, binary.random_seed, path, self.bcounter, self.tcounter))
|
|
elif perflist:
|
|
presults = []
|
|
for perf in perflist:
|
|
pmin = bool (int (attribute_as_text (perf, 'minimize')))
|
|
pmax = bool (int (attribute_as_text (perf, 'maximize')))
|
|
pval = float (attribute_as_text (perf, 'value'))
|
|
txt = node_as_text (perf)
|
|
txt = re.sub (r'&', r'&', txt)
|
|
txt = re.sub (r'<', r'>', txt)
|
|
txt = '<strong>Performance(' + (pmin and '<em>minimized</em>' or '<em>maximized</em>') + '):</strong> ' + txt.strip() + '<br />\n'
|
|
txt = re.sub (r'"', r'\\"', txt)
|
|
txt = re.sub (r'\n', r'\\n', txt)
|
|
presults += [ (pval, txt) ]
|
|
presults.sort()
|
|
ptxt = ''.join ([e[1] for e in presults])
|
|
self.oprint ('<script language="javascript" type="text/javascript">message_array["b%u_t%u_"] = "%s";</script>\n' % (self.bcounter, self.tcounter, ptxt))
|
|
self.oprint ('<td align="center"><a href="javascript:view_testlog (\'%s\', \'%s\', \'%s\', \'%s\', \'Test Results:\', \'b%u_t%u_\')">Details</a></td>\n' %
|
|
('TestResultWindow', binary.file, binary.random_seed, path, self.bcounter, self.tcounter))
|
|
else:
|
|
self.oprint ('<td align="center">-</td>\n')
|
|
self.oprint ('<td align="right" %s>%s</td>\n' % (rcolor, result))
|
|
self.oprint ('</tr>\n')
|
|
self.tcounter += 1
|
|
self.total_tcounter += 1
|
|
self.total_fcounter += result != 'success'
|
|
def handle_binary (self, binary):
|
|
self.tcounter = 1
|
|
self.bcounter += 1
|
|
self.total_duration += binary.duration
|
|
self.oprint ('<tr><td><strong>%s</strong></td><td align="right">%f</td> <td align="center">\n' % (binary.name, binary.duration))
|
|
erlink, oklink = ('', '')
|
|
real_cases = len (binary.testcases) - binary.skipped_cases
|
|
if binary.success_cases < real_cases:
|
|
erlink = 'href="javascript:toggle_display (\'ResultTable\', \'tr\', \'b%u_\', \'failed\')"' % self.bcounter
|
|
if binary.success_cases:
|
|
oklink = 'href="javascript:toggle_display (\'ResultTable\', \'tr\', \'b%u_\', \'success\')"' % self.bcounter
|
|
if real_cases != 0:
|
|
self.oprint ('<a %s>ER</a>\n' % erlink)
|
|
self.oprint ('<a %s>OK</a>\n' % oklink)
|
|
self.oprint ('</td>\n')
|
|
perc = binary.success_cases * 100.0 / real_cases
|
|
pcolor = {
|
|
100 : 'bgcolor="lightgreen"',
|
|
0 : 'bgcolor="red"',
|
|
}.get (int (perc), 'bgcolor="yellow"')
|
|
self.oprint ('<td align="right" %s>%.2f%%</td>\n' % (pcolor, perc))
|
|
self.oprint ('</tr>\n')
|
|
else:
|
|
self.oprint ('Empty\n')
|
|
self.oprint ('</td>\n')
|
|
self.oprint ('</tr>\n')
|
|
for tc in binary.testcases:
|
|
self.handle_testcase (tc, binary)
|
|
def handle_totals (self):
|
|
self.oprint ('<tr>')
|
|
self.oprint ('<td><strong>Totals:</strong> %u Binaries, %u Tests, %u Failed, %u Succeeded</td>' %
|
|
(self.bcounter, self.total_tcounter, self.total_fcounter, self.total_tcounter - self.total_fcounter))
|
|
self.oprint ('<td align="right">%f</td>\n' % self.total_duration)
|
|
self.oprint ('<td align="center">-</td>\n')
|
|
if self.total_tcounter != 0:
|
|
perc = (self.total_tcounter - self.total_fcounter) * 100.0 / self.total_tcounter
|
|
else:
|
|
perc = 0.0
|
|
pcolor = {
|
|
100 : 'bgcolor="lightgreen"',
|
|
0 : 'bgcolor="red"',
|
|
}.get (int (perc), 'bgcolor="yellow"')
|
|
self.oprint ('<td align="right" %s>%.2f%%</td>\n' % (pcolor, perc))
|
|
self.oprint ('</tr>\n')
|
|
def printout (self):
|
|
self.oprint ('<html><head>\n')
|
|
self.oprint ('<title>GTester Unit Test Report</title>\n')
|
|
self.oprint (self.cssjs)
|
|
self.oprint ('</head>\n')
|
|
self.oprint ('<body>\n')
|
|
self.oprint ('<h2>GTester Unit Test Report</h2>\n')
|
|
self.handle_info ()
|
|
self.oprint ('<table id="ResultTable" width="100%" border="1">\n<tr>\n')
|
|
self.oprint ('<th>Program / Testcase </th>\n')
|
|
self.oprint ('<th style="width:8em">Duration (sec)</th>\n')
|
|
self.oprint ('<th style="width:5em">View</th>\n')
|
|
self.oprint ('<th style="width:5em">Result</th>\n')
|
|
self.oprint ('</tr>\n')
|
|
for tb in self.binaries:
|
|
self.handle_binary (tb)
|
|
self.handle_totals()
|
|
self.oprint ('</table>\n')
|
|
self.oprint ('</body>\n')
|
|
self.oprint ('</html>\n')
|
|
|
|
|
|
class SubunitWriter(ReportWriter):
|
|
"""Reporter to output a subunit stream."""
|
|
|
|
def printout(self):
|
|
reporter = subunit.TestProtocolClient(sys.stdout)
|
|
for binary in self.binaries:
|
|
for tc in binary.testcases:
|
|
test = GTestCase(tc, binary)
|
|
test.run(reporter)
|
|
|
|
|
|
class GTestCase(object):
|
|
"""A representation of a gtester test result as a pyunit TestCase."""
|
|
|
|
def __init__(self, case, binary):
|
|
"""Create a GTestCase for case `case` from binary program `binary`."""
|
|
self._case = case
|
|
self._binary = binary
|
|
# the name of the case - e.g. /dbusmenu/glib/objects/menuitem/props_boolstr
|
|
self._path = attribute_as_text(self._case, 'path')
|
|
|
|
def id(self):
|
|
"""What test is this? Returns the gtester path for the testcase."""
|
|
return self._path
|
|
|
|
def _get_details(self):
|
|
"""Calculate a details dict for the test - attachments etc."""
|
|
details = {}
|
|
result = attribute_as_text(self._case, 'result', 'status')
|
|
details['filename'] = Content(mime_utf8, lambda:[self._binary.file])
|
|
details['random_seed'] = Content(mime_utf8,
|
|
lambda:[self._binary.random_seed])
|
|
if self._get_outcome() == 'addFailure':
|
|
# Extract the error details. Skips have no details because its not
|
|
# skip like unittest does, instead the runner just bypasses N test.
|
|
txt = self._error_text(self._case)
|
|
details['error'] = Content(mime_utf8, lambda:[txt])
|
|
if self._get_outcome() == 'addSuccess':
|
|
# Sucessful tests may have performance metrics.
|
|
perflist = list_children(self._case, 'performance')
|
|
if perflist:
|
|
presults = []
|
|
for perf in perflist:
|
|
pmin = bool (int (attribute_as_text (perf, 'minimize')))
|
|
pmax = bool (int (attribute_as_text (perf, 'maximize')))
|
|
pval = float (attribute_as_text (perf, 'value'))
|
|
txt = node_as_text (perf)
|
|
txt = 'Performance(' + (pmin and 'minimized' or 'maximized'
|
|
) + '): ' + txt.strip() + '\n'
|
|
presults += [(pval, txt)]
|
|
presults.sort()
|
|
perf_details = [e[1] for e in presults]
|
|
details['performance'] = Content(mime_utf8, lambda:perf_details)
|
|
return details
|
|
|
|
def _get_outcome(self):
|
|
if int(attribute_as_text(self._case, 'skipped') + '0'):
|
|
return 'addSkip'
|
|
outcome = attribute_as_text(self._case, 'result', 'status')
|
|
if outcome == 'success':
|
|
return 'addSuccess'
|
|
else:
|
|
return 'addFailure'
|
|
|
|
def run(self, result):
|
|
time = datetime.datetime.utcnow().replace(tzinfo=iso8601.Utc())
|
|
result.time(time)
|
|
result.startTest(self)
|
|
try:
|
|
outcome = self._get_outcome()
|
|
details = self._get_details()
|
|
# Only provide a duration IFF outcome == 'addSuccess' - the main
|
|
# parser claims bogus results otherwise: in that case emit time as
|
|
# zero perhaps.
|
|
if outcome == 'addSuccess':
|
|
duration = float(node_as_text(self._case, 'duration'))
|
|
duration = duration * 1000000
|
|
timedelta = datetime.timedelta(0, 0, duration)
|
|
time = time + timedelta
|
|
result.time(time)
|
|
getattr(result, outcome)(self, details=details)
|
|
finally:
|
|
result.stopTest(self)
|
|
|
|
|
|
|
|
# main program handling
|
|
def parse_opts():
|
|
"""Parse program options.
|
|
|
|
:return: An options object and the program arguments.
|
|
"""
|
|
parser = optparse.OptionParser()
|
|
parser.version = pkginstall_configvars.get ('glib-version', '0.0-uninstalled')
|
|
parser.usage = "%prog [OPTIONS] <gtester-log.xml>"
|
|
parser.description = "Generate HTML reports from the XML log files generated by gtester."
|
|
parser.epilog = "gtester-report (GLib utils) version %s."% (parser.version,)
|
|
parser.add_option("-v", "--version", action="store_true", dest="version", default=False,
|
|
help="Show program version.")
|
|
parser.add_option("-s", "--subunit", action="store_true", dest="subunit", default=False,
|
|
help="Output subunit [See https://launchpad.net/subunit/"
|
|
" Needs python-subunit]")
|
|
options, files = parser.parse_args()
|
|
if options.version:
|
|
print parser.epilog
|
|
return None, None
|
|
if len(files) != 1:
|
|
parser.error("Must supply a log file to parse.")
|
|
if options.subunit and subunit is None:
|
|
parser.error("python-subunit is not installed.")
|
|
return options, files
|
|
|
|
|
|
def main():
|
|
options, files = parse_opts()
|
|
if options is None:
|
|
return 0
|
|
xd = xml.dom.minidom.parse (files[0])
|
|
rr = ReportReader()
|
|
rr.trampoline (xd)
|
|
if not options.subunit:
|
|
HTMLReportWriter(rr.get_info(), rr.binary_list()).printout()
|
|
else:
|
|
SubunitWriter(rr.get_info(), rr.binary_list()).printout()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|