This is merely a historical archive of years 2008-2021, before the migration to mailman3.
A maintained and still updated list archive can be found at https://lists.osmocom.org/hyperkitty/list/gerrit-log@lists.osmocom.org/.
Pau Espin Pedrol gerrit-no-reply at lists.osmocom.org
Review at https://gerrit.osmocom.org/2672
Add JUnit XML reports; refactor test reporting
* Add Junit output file support
* Differentiate between an expected failure test and an error in the
test, as described in JUnit.
* In case of an error/exception during test, record and attach it to the
Test object and continue running the tests, and show it at the end
during the trial report.
Change-Id: Iedf6d912b3cce3333a187a4ac6d5c6b70fe9d5c5
---
M .gitignore
M selftest/suite_test.ok
M selftest/suite_test.py
A selftest/suite_test/test_suite/test_fail.py
A selftest/suite_test/test_suite/test_fail_raise.py
M selftest/trial_test.ok
M src/osmo-gsm-tester.py
M src/osmo_gsm_tester/suite.py
M src/osmo_gsm_tester/test.py
M src/osmo_gsm_tester/trial.py
A suites/debug/error.py
M suites/debug/fail.py
A suites/debug/fail_raise.py
A suites/debug/pass.py
14 files changed, 265 insertions(+), 89 deletions(-)
git pull ssh://gerrit.osmocom.org:29418/osmo-gsm-tester refs/changes/72/2672/1
diff --git a/.gitignore b/.gitignore
index 4119e7f..1ab84d9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,4 @@
test_work
state
*.pyc
+selftest/trial_test/
diff --git a/selftest/suite_test.ok b/selftest/suite_test.ok
index fda77dc..30c6915 100644
--- a/selftest/suite_test.ok
+++ b/selftest/suite_test.ok
@@ -59,15 +59,30 @@
tst hello_world.py:[LINENR]: two [test_suite↪hello_world.py:[LINENR]]
tst hello_world.py:[LINENR]: three [test_suite↪hello_world.py:[LINENR]]
tst hello_world.py:[LINENR] PASS [test_suite↪hello_world.py]
-pass: all 1 tests passed.
+tst test_suite: PASS
+pass: all 6 tests passed (5 skipped).
- a test with an error
tst test_suite: Suite run start [suite.py:[LINENR]]
tst test_error.py:[LINENR] START [test_suite↪test_error.py] [suite.py:[LINENR]]
tst test_error.py:[LINENR]: I am 'test_suite' / 'test_error.py:[LINENR]' [test_suite↪test_error.py:[LINENR]] [test_error.py:[LINENR]]
-tst test_error.py:[LINENR]: FAIL [test_suite↪test_error.py:[LINENR]] [suite.py:[LINENR]]
-tst test_error.py:[LINENR]: ERR: AssertionError: [test_suite↪test_error.py:[LINENR]] [test_error.py:[LINENR]: assert False]
-FAIL: 1 of 1 tests failed:
- test_error.py
+tst test_error.py:[LINENR]: ERR: AssertionError: [test_error.py:[LINENR]: assert False]
+tst test_error.py:[LINENR] FAIL (AssertionError) [test_suite↪test_error.py] [suite.py:[LINENR]]
+tst test_suite: FAIL [suite.py:[LINENR]]
+
+- a test with a failure
+tst test_suite: Suite run start [suite.py:[LINENR]]
+tst test_fail.py:[LINENR] START [test_suite↪test_fail.py] [suite.py:[LINENR]]
+tst test_fail.py:[LINENR]: I am 'test_suite' / 'test_fail.py:[LINENR]' [test_suite↪test_fail.py:[LINENR]] [test_fail.py:[LINENR]]
+tst test_fail.py:[LINENR] FAIL (EpicFail) [test_suite↪test_fail.py] [suite.py:[LINENR]]
+tst test_suite: FAIL [suite.py:[LINENR]]
+
+- a test with a raised failure
+tst test_suite: Suite run start [suite.py:[LINENR]]
+tst test_fail_raise.py:[LINENR] START [test_suite↪test_fail_raise.py] [suite.py:[LINENR]]
+tst test_fail_raise.py:[LINENR]: I am 'test_suite' / 'test_fail_raise.py:[LINENR]' [test_suite↪test_fail_raise.py:[LINENR]] [test_fail_raise.py:[LINENR]]
+tst test_fail_raise.py:[LINENR]: ERR: Failure: ('EpicFail', 'This failure is expected') [test_fail_raise.py:[LINENR]: raise Failure('EpicFail', 'This failure is expected')]
+tst test_fail_raise.py:[LINENR] FAIL (EpicFail) [test_suite↪test_fail_raise.py] [suite.py:[LINENR]]
+tst test_suite: FAIL [suite.py:[LINENR]]
- graceful exit.
diff --git a/selftest/suite_test.py b/selftest/suite_test.py
index 315c683..3750a94 100755
--- a/selftest/suite_test.py
+++ b/selftest/suite_test.py
@@ -22,13 +22,30 @@
print('- run hello world test')
s = suite.SuiteRun(None, 'test_suite', s_def)
results = s.run_tests('hello_world.py')
-print(str(results))
+print(str(s))
log.style_change(src=True)
#log.style_change(trace=True)
print('\n- a test with an error')
results = s.run_tests('test_error.py')
-print(str(results))
+assert 'FAIL: [test_suite] 1 failed ' in str(s)
+assert 'FAIL: [test_error.py]' in str(s)
+assert "type:'AssertionError' message: AssertionError()" in str(s)
+assert 'assert False' in str(s)
+
+print('\n- a test with a failure')
+results = s.run_tests('test_fail.py')
+assert 'FAIL: [test_suite] 1 failed ' in str(s)
+assert 'FAIL: [test_fail.py]' in str(s)
+assert "type:'EpicFail' message: This failure is expected" in str(s)
+assert "test.set_fail('EpicFail', 'This failure is expected')" in str(s)
+
+print('\n- a test with a raised failure')
+results = s.run_tests('test_fail_raise.py')
+assert 'FAIL: [test_suite] 1 failed ' in str(s)
+assert 'FAIL: [test_fail_raise.py]' in str(s)
+assert "type:'EpicFail' message: This failure is expected" in str(s)
+assert "raise Failure('EpicFail', 'This failure is expected')" in str(s)
print('\n- graceful exit.')
# vim: expandtab tabstop=4 shiftwidth=4
diff --git a/selftest/suite_test/test_suite/test_fail.py b/selftest/suite_test/test_suite/test_fail.py
new file mode 100755
index 0000000..6880c81
--- /dev/null
+++ b/selftest/suite_test/test_suite/test_fail.py
@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+from osmo_gsm_tester.test import *
+
+print('I am %r / %r' % (suite.name(), test.name()))
+
+test.set_fail('EpicFail', 'This failure is expected')
diff --git a/selftest/suite_test/test_suite/test_fail_raise.py b/selftest/suite_test/test_suite/test_fail_raise.py
new file mode 100755
index 0000000..a7b0b61
--- /dev/null
+++ b/selftest/suite_test/test_suite/test_fail_raise.py
@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+from osmo_gsm_tester.test import *
+
+print('I am %r / %r' % (suite.name(), test.name()))
+
+raise Failure('EpicFail', 'This failure is expected')
diff --git a/selftest/trial_test.ok b/selftest/trial_test.ok
index 6ad39a9..7434a10 100644
--- a/selftest/trial_test.ok
+++ b/selftest/trial_test.ok
@@ -4,7 +4,7 @@
[TMP]/third
- fetch trial dirs in order
first
-['taken']
+['last_run', 'run.[TIMESTAMP]', 'taken']
second
third
- no more trial dirs left
diff --git a/src/osmo-gsm-tester.py b/src/osmo-gsm-tester.py
index 0a04708..1473d6d 100755
--- a/src/osmo-gsm-tester.py
+++ b/src/osmo-gsm-tester.py
@@ -169,44 +169,33 @@
t.verify()
trials.append(t)
- trials_passed = []
- trials_failed = []
+ trials_run = []
+ any_failed = False
for current_trial in trials:
try:
with current_trial:
- suites_passed = []
- suites_failed = []
for suite_scenario_str, suite_def, scenarios in suite_scenarios:
log.large_separator(current_trial.name(), suite_scenario_str)
suite_run = suite.SuiteRun(current_trial, suite_scenario_str, suite_def, scenarios)
- result = suite_run.run_tests(test_names)
- if result.all_passed:
- suites_passed.append(suite_scenario_str)
- suite_run.log('PASS')
- else:
- suites_failed.append(suite_scenario_str)
- suite_run.err('FAIL')
- if not suites_failed:
- current_trial.log('PASS')
- trials_passed.append(current_trial.name())
- else:
- current_trial.err('FAIL')
- trials_failed.append((current_trial.name(), suites_passed, suites_failed))
+ current_trial.add_suite(suite_run)
+
+ status = current_trial.run_suites(test_names)
+
+ if status == trial.Trial.FAIL:
+ any_failed = True
+ trials_run.append(current_trial)
except:
current_trial.log_exn()
sys.stderr.flush()
sys.stdout.flush()
- log.large_separator()
- if trials_passed:
- print('Trials passed:\n ' + ('\n '.join(trials_passed)))
- if trials_failed:
- print('Trials failed:')
- for trial_name, suites_passed, suites_failed in trials_failed:
- print(' %s (%d of %d suite runs failed)' % (trial_name, len(suites_failed), len(suites_failed) + len(suites_passed)))
- for suite_failed in suites_failed:
- print(' FAIL:', suite_failed)
+ if not any_failed:
+ log.large_separator('All trials passed:\n ' + ('\n '.join(mytrial.name() for mytrial in trials_run)))
+ else:
+ for mytrial in trials_run:
+ log.large_separator('Trial Report for %s' % mytrial.name())
+ mytrial.log_report()
exit(1)
if __name__ == '__main__':
diff --git a/src/osmo_gsm_tester/suite.py b/src/osmo_gsm_tester/suite.py
index 43e55af..0d7f931 100644
--- a/src/osmo_gsm_tester/suite.py
+++ b/src/osmo_gsm_tester/suite.py
@@ -21,11 +21,19 @@
import sys
import time
import copy
+import traceback
+import xml.etree.ElementTree as et
+from datetime import datetime
from . import config, log, template, util, resource, schema, ofono_client, osmo_nitb
from . import test
class Timeout(Exception):
pass
+
+class Failure(Exception):
+ def __init__(self, fail_type='', fail_msg=''):
+ self.fail_type = fail_type
+ self.fail_msg = fail_msg
class SuiteDefinition(log.Origin):
'''A test suite reserves resources for a number of tests.
@@ -78,9 +86,11 @@
raise ValueError('add_test(): test already belongs to another suite')
self.tests.append(test)
-
-
class Test(log.Origin):
+ UNKNOWN = 'UNKNOWN'
+ SKIP = 'SKIP'
+ PASS = 'PASS'
+ FAIL = 'FAIL'
def __init__(self, suite, test_basename):
self.suite = suite
@@ -89,26 +99,45 @@
super().__init__(self.path)
self.set_name(self.basename)
self.set_log_category(log.C_TST)
+ self.status = Test.UNKNOWN
+ self.ts_start = 0
+ self.ts_end = 0
+ self.time = 0
+ self.fail_type = ''
+ self.fail_message = ''
def run(self, suite_run):
assert self.suite is suite_run.definition
- with self:
- test.setup(suite_run, self, ofono_client, sys.modules[__name__])
- success = False
- try:
+ try:
+ with self:
+ self.status = Test.UNKNOWN
+ self.ts_start = time.time()
+ test.setup(suite_run, self, ofono_client, sys.modules[__name__])
self.log('START')
with self.redirect_stdout():
util.run_python_file('%s.%s' % (self.suite.name(), self.name()),
self.path)
- success = True
- except resource.NoResourceExn:
- self.err('Current resource state:\n', repr(suite_run.reserved_resources))
- raise
- finally:
- if success:
- self.log('PASS')
+ if self.status == Test.UNKNOWN:
+ self.set_pass()
+ except Exception as e:
+ self.log_exn()
+ if self.status == Test.UNKNOWN:
+ if isinstance(e, Failure):
+ ftype = e.fail_type
+ fmsg = e.fail_msg + '\n' + traceback.format_exc().rstrip()
else:
- self.log('FAIL')
+ ftype = type(e).__name__
+ fmsg = repr(e) + '\n' + traceback.format_exc().rstrip()
+ if isinstance(e, resource.NoResourceExn):
+ msg += '\n' + 'Current resource state:\n' + repr(suite_run.reserved_resources)
+ self.set_fail(ftype, fmsg, False)
+
+ finally:
+ if self.status == Test.PASS or self.status == Test.SKIP:
+ self.log(self.status)
+ else:
+ self.log('%s (%s)' % (self.status, self.fail_type))
+ return self.status
def name(self):
l = log.get_line_for_src(self.path)
@@ -116,7 +145,50 @@
return '%s:%s' % (self._name, l)
return super().name()
+ def set_fail(self, fail_type='', fail_message='', tb=True):
+ self.status = Test.FAIL
+ self.ts_end = time.time()
+ self.time = round(self.ts_end - self.ts_start)
+ self.fail_type = fail_type
+ self.fail_message = fail_message
+ if tb:
+ self.fail_message += '\n' + ''.join(traceback.format_stack()[:-1]).rstrip()
+
+ def set_pass(self):
+ self.status = Test.PASS
+ self.ts_end = time.time()
+ self.time = round(self.ts_end - self.ts_start)
+
+ def set_skip(self):
+ self.status = Test.SKIP
+ self.ts_end = time.time()
+ self.time = 0
+
+ def to_junit(self):
+ testcase = et.Element('testcase')
+ testcase.set('name', self.name())
+ testcase.set('time', str(self.time))
+ if self.status == Test.SKIP:
+ skip = et.SubElement(testcase, 'skipped')
+ elif self.status == Test.FAIL:
+ failure = et.SubElement(testcase, 'failure')
+ failure.set('type', self.fail_type)
+ failure.text = self.fail_message
+ return testcase
+
+ def __str__(self):
+ ret = "%s: [%s]" % (self.status, self.name())
+ if self.status != Test.SKIP:
+ ret += " (%s, %d sec)" % (datetime.fromtimestamp(round(self.ts_start)).isoformat(), self.time)
+ if self.status == Test.FAIL:
+ ret += " type:'%s' message: %s" % (self.fail_type, self.fail_message.replace('\n', '\n '))
+ return ret
+
+
class SuiteRun(log.Origin):
+ UNKNOWN = 'UNKNOWN'
+ PASS = 'PASS'
+ FAIL = 'FAIL'
trial = None
resources_pool = None
@@ -132,6 +204,16 @@
self.set_name(suite_scenario_str)
self.set_log_category(log.C_TST)
self.resources_pool = resource.ResourcesPool()
+ self.init_test_vars()
+
+ def init_test_vars(self):
+ self.tests = []
+ self.ts_start = time.time()
+ self.ts_end = time.time()
+ self.time = 0
+ self.test_failed_ctr = 0
+ self.test_skipped_ctr = 0
+ self.status = SuiteRun.UNKNOWN
def combined(self, conf_name):
self.dbg(combining=conf_name)
@@ -157,32 +239,6 @@
self._config = self.combined('config')
return self._config
- class Results:
- def __init__(self):
- self.passed = []
- self.failed = []
- self.all_passed = None
-
- def add_pass(self, test):
- self.passed.append(test)
-
- def add_fail(self, test):
- self.failed.append(test)
-
- def conclude(self):
- self.all_passed = bool(self.passed) and not bool(self.failed)
- return self
-
- def __str__(self):
- if self.failed:
- return 'FAIL: %d of %d tests failed:\n %s' % (
- len(self.failed),
- len(self.failed) + len(self.passed),
- '\n '.join([t.name() for t in self.failed]))
- if not self.passed:
- return 'no tests were run.'
- return 'pass: all %d tests passed.' % len(self.passed)
-
def reserve_resources(self):
if self.reserved_resources:
raise RuntimeError('Attempt to reserve resources twice for a SuiteRun')
@@ -192,24 +248,28 @@
def run_tests(self, names=None):
self.log('Suite run start')
+ self.init_test_vars()
if not self.reserved_resources:
self.reserve_resources()
- results = SuiteRun.Results()
for test in self.definition.tests:
if names and not test.name() in names:
+ test.set_skip()
+ self.test_skipped_ctr += 1
+ self.tests.append(test)
continue
- self._run_test(test, results)
- self.stop_processes()
- return results.conclude()
-
- def _run_test(self, test, results):
- try:
with self:
- test.run(self)
- results.add_pass(test)
- except:
- results.add_fail(test)
- self.log_exn()
+ if test.run(self) == Test.FAIL:
+ self.test_failed_ctr += 1
+ self.tests.append(test)
+ self.stop_processes()
+ self.ts_end = time.time()
+ self.time = round(self.ts_end - self.ts_start)
+ if self.test_failed_ctr:
+ self.status = SuiteRun.FAIL
+ else:
+ self.status = SuiteRun.PASS
+ self.log(self.status)
+ return self.status
def remember_to_stop(self, process):
if self._processes is None:
@@ -222,6 +282,29 @@
for process in self._processes:
process.terminate()
+ def to_junit(self):
+ testsuite = et.Element('testsuite')
+ testsuite.set('name', self.name())
+ testsuite.set('hostname', 'localhost')
+ testsuite.set('timestamp', datetime.fromtimestamp(round(self.ts_start)).isoformat())
+ testsuite.set('time', str(self.time))
+ testsuite.set('tests', str(len(self.tests)))
+ testsuite.set('failures', str(self.test_failed_ctr))
+ for test in self.tests:
+ testcase = test.to_junit()
+ testsuite.append(testcase)
+ return testsuite
+
+ def __str__(self):
+ if self.test_failed_ctr:
+ return 'FAIL: [%s] %d failed out of %d tests run (%d skipped):\n %s' % (
+ self.name(), self.test_failed_ctr, len(self.tests), self.test_skipped_ctr,
+ '\n '.join([str(t) for t in self.tests]))
+ if not self.tests:
+ return 'no tests were run.'
+ return 'pass: all %d tests passed (%d skipped).' % (len(self.tests), self.test_skipped_ctr)
+
+
def nitb_iface(self):
return self.reserved_resources.get(resource.R_NITB_IFACE)
diff --git a/src/osmo_gsm_tester/test.py b/src/osmo_gsm_tester/test.py
index 871e3ae..f584c92 100644
--- a/src/osmo_gsm_tester/test.py
+++ b/src/osmo_gsm_tester/test.py
@@ -32,9 +32,10 @@
poll = None
prompt = None
Timeout = None
+Failure = None
def setup(suite_run, _test, ofono_client, suite_module):
- global trial, suite, test, resources, log, dbg, err, wait, sleep, poll, prompt, Timeout
+ global trial, suite, test, resources, log, dbg, err, wait, sleep, poll, prompt, Failure, Timeout
trial = suite_run.trial
suite = suite_run
test = _test
@@ -46,6 +47,7 @@
sleep = suite_run.sleep
poll = suite_run.poll
prompt = suite_run.prompt
+ Failure = suite_module.Failure
Timeout = suite_module.Timeout
# vim: expandtab tabstop=4 shiftwidth=4
diff --git a/src/osmo_gsm_tester/trial.py b/src/osmo_gsm_tester/trial.py
index 171061d..c00e51b 100644
--- a/src/osmo_gsm_tester/trial.py
+++ b/src/osmo_gsm_tester/trial.py
@@ -21,8 +21,9 @@
import time
import shutil
import tarfile
+import xml.etree.ElementTree as et
-from . import log, util
+from . import log, util, suite
FILE_MARK_TAKEN = 'taken'
FILE_CHECKSUMS = 'checksums.md5'
@@ -32,6 +33,10 @@
FILE_LOG_BRIEF = 'log_brief'
class Trial(log.Origin):
+ UNKNOWN = 'UNKNOWN'
+ PASS = 'PASS'
+ FAIL = 'FAIL'
+
path = None
dir = None
_run_dir = None
@@ -58,6 +63,9 @@
self.dir = util.Dir(self.path)
self.inst_dir = util.Dir(self.dir.child('inst'))
self.bin_tars = []
+ self.suites = []
+ self.junitfname = self.get_run_dir().new_file(self.name()+'.xml')
+ self.status = Trial.UNKNOWN
def __repr__(self):
return self.name()
@@ -176,4 +184,38 @@
except:
pass
+ def add_suite(self, suite_run):
+ self.suites.append(suite_run)
+
+ def run_suites(self, names=None):
+ self.status = Trial.UNKNOWN
+ for suite_run in self.suites:
+ if suite_run.run_tests(names) == suite.SuiteRun.FAIL:
+ self.status = Trial.FAIL
+ elif self.status == suite.SuiteRun.UNKNOWN:
+ self.status = Trial.PASS
+ self.log(self.status)
+ self.write_junit_report()
+ return self.status
+
+ def to_junit(self):
+ testsuites = et.Element('testsuites')
+ for suite in self.suites:
+ testsuite = suite.to_junit()
+ testsuites.append(testsuite)
+ return testsuites
+
+ def write_junit_report(self):
+ self.log("Storing JUnit report in ", self.junitfname)
+ elements = et.ElementTree(element=self.to_junit())
+ elements.write(self.junitfname)
+
+ def log_report(self):
+ msg = '\n%s [%s]\n ' % (self.status, self.name())
+ msg += '\n '.join(str(result) for result in self.suites)
+ self.log(msg)
+
+
+
+
# vim: expandtab tabstop=4 shiftwidth=4
diff --git a/suites/debug/error.py b/suites/debug/error.py
new file mode 100644
index 0000000..8e146fa
--- /dev/null
+++ b/suites/debug/error.py
@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+from osmo_gsm_tester.test import *
+
+# This can be used to verify that a test error is reported properly.
+assert False
diff --git a/suites/debug/fail.py b/suites/debug/fail.py
index 1b412b5..fcd56e0 100644
--- a/suites/debug/fail.py
+++ b/suites/debug/fail.py
@@ -2,4 +2,4 @@
from osmo_gsm_tester.test import *
# This can be used to verify that a test failure is reported properly.
-assert False
+test.set_fail('EpicFail', 'This failure is expected')
diff --git a/suites/debug/fail_raise.py b/suites/debug/fail_raise.py
new file mode 100644
index 0000000..22fb940
--- /dev/null
+++ b/suites/debug/fail_raise.py
@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+from osmo_gsm_tester.test import *
+
+# This can be used to verify that a test failure is reported properly.
+raise Failure('EpicFail', 'This failure is expected')
diff --git a/suites/debug/pass.py b/suites/debug/pass.py
new file mode 100644
index 0000000..42c1f25
--- /dev/null
+++ b/suites/debug/pass.py
@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+from osmo_gsm_tester.test import *
+
+# This can be used to verify that a test passes correctly.
+pass
--
To view, visit https://gerrit.osmocom.org/2672
To unsubscribe, visit https://gerrit.osmocom.org/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Iedf6d912b3cce3333a187a4ac6d5c6b70fe9d5c5
Gerrit-PatchSet: 1
Gerrit-Project: osmo-gsm-tester
Gerrit-Branch: report
Gerrit-Owner: Pau Espin Pedrol <pespin at sysmocom.de>