This is merely a historical archive of years 2008-2021, before the migration to mailman3.
A maintained and still updated list archive can be found at https://lists.osmocom.org/hyperkitty/list/gerrit-log@lists.osmocom.org/.
Pau Espin Pedrol gerrit-no-reply at lists.osmocom.orgHello Neels Hofmeyr, Harald Welte, Jenkins Builder, I'd like you to reexamine a change. Please visit https://gerrit.osmocom.org/4764 to look at the new patch set (#2). Move Test class to its own test.py module Make the code more reachable for newcomers, as well as more organized for people who work a lot with it. SuiteRun in suite.py is already quite big, and having the Test class in there make it unnecessarily more big, and makes it difficult to find stuff. At the same time, having a test.py which does actually not contain the Test class but other stuff, makes it even more confusing. Change-Id: I9c8d67f598466ba52a4827ff77027b9eae85929a --- M selftest/suite_test.ok M src/osmo_gsm_tester/report.py M src/osmo_gsm_tester/suite.py A src/osmo_gsm_tester/test.py 4 files changed, 149 insertions(+), 123 deletions(-) git pull ssh://gerrit.osmocom.org:29418/osmo-gsm-tester refs/changes/64/4764/2 diff --git a/selftest/suite_test.ok b/selftest/suite_test.ok index cd5a9e7..79c37cc 100644 --- a/selftest/suite_test.ok +++ b/selftest/suite_test.ok @@ -110,7 +110,7 @@ ---------------------------------------------- tst test_error.py:[LINENR]: I am 'test_suite' / 'test_error.py:[LINENR]' [test_suite↪test_error.py:[LINENR]] [test_error.py:[LINENR]] tst test_error.py:[LINENR]: ERR: AssertionError: test_error.py:[LINENR]: assert False [test_suite↪test_error.py:[LINENR]] [test_error.py:[LINENR]: assert False] -tst test_error.py:[LINENR]: Test FAILED (N.N sec) [test_suite↪test_error.py:[LINENR]] [suite.py:[LINENR]] +tst test_error.py:[LINENR]: Test FAILED (N.N sec) [test_suite↪test_error.py:[LINENR]] [test.py:[LINENR]] --------------------------------------------------------------------- trial test_suite FAIL --------------------------------------------------------------------- @@ -133,7 +133,7 @@ ---------------------------------------------- tst test_fail.py:[LINENR]: I am 'test_suite' / 'test_fail.py:[LINENR]' [test_suite↪test_fail.py:[LINENR]] [test_fail.py:[LINENR]] tst test_fail.py:[LINENR]: ERR: EpicFail: This failure is expected [test_suite↪test_fail.py:[LINENR]] [test_fail.py:[LINENR]] -tst test_fail.py:[LINENR]: Test FAILED (N.N sec) [test_suite↪test_fail.py:[LINENR]] [suite.py:[LINENR]] +tst test_fail.py:[LINENR]: Test FAILED (N.N sec) [test_suite↪test_fail.py:[LINENR]] [test.py:[LINENR]] --------------------------------------------------------------------- trial test_suite FAIL --------------------------------------------------------------------- @@ -155,7 +155,7 @@ trial test_suite test_fail_raise.py ---------------------------------------------- tst test_fail_raise.py:[LINENR]: ERR: ExpectedFail: This failure is expected [test_suite↪test_fail_raise.py:[LINENR]] [test_fail_raise.py:[LINENR]: raise ExpectedFail('This failure is expected')] -tst test_fail_raise.py:[LINENR]: Test FAILED (N.N sec) [test_suite↪test_fail_raise.py:[LINENR]] [suite.py:[LINENR]] +tst test_fail_raise.py:[LINENR]: Test FAILED (N.N sec) [test_suite↪test_fail_raise.py:[LINENR]] [test.py:[LINENR]] --------------------------------------------------------------------- trial test_suite FAIL --------------------------------------------------------------------- @@ -233,7 +233,7 @@ tst hello_world.py:[LINENR]: one [test_suite↪hello_world.py:[LINENR]] [hello_world.py:[LINENR]] tst hello_world.py:[LINENR]: two [test_suite↪hello_world.py:[LINENR]] [hello_world.py:[LINENR]] tst hello_world.py:[LINENR]: three [test_suite↪hello_world.py:[LINENR]] [hello_world.py:[LINENR]] -tst hello_world.py:[LINENR] Test passed (N.N sec) [test_suite↪hello_world.py] [suite.py:[LINENR]] +tst hello_world.py:[LINENR] Test passed (N.N sec) [test_suite↪hello_world.py] [test.py:[LINENR]] --------------------------------------------------------------------- trial test_suite PASS --------------------------------------------------------------------- @@ -311,7 +311,7 @@ tst hello_world.py:[LINENR]: one [test_suite↪hello_world.py:[LINENR]] [hello_world.py:[LINENR]] tst hello_world.py:[LINENR]: two [test_suite↪hello_world.py:[LINENR]] [hello_world.py:[LINENR]] tst hello_world.py:[LINENR]: three [test_suite↪hello_world.py:[LINENR]] [hello_world.py:[LINENR]] -tst hello_world.py:[LINENR] Test passed (N.N sec) [test_suite↪hello_world.py] [suite.py:[LINENR]] +tst hello_world.py:[LINENR] Test passed (N.N sec) [test_suite↪hello_world.py] [test.py:[LINENR]] --------------------------------------------------------------------- trial test_suite PASS --------------------------------------------------------------------- diff --git a/src/osmo_gsm_tester/report.py b/src/osmo_gsm_tester/report.py index 82b2f13..a53504b 100644 --- a/src/osmo_gsm_tester/report.py +++ b/src/osmo_gsm_tester/report.py @@ -21,7 +21,7 @@ import math from datetime import datetime import xml.etree.ElementTree as et -from . import log, suite +from . import log, suite, test def trial_to_junit_write(trial, junit_path): elements = et.ElementTree(element=trial_to_junit(trial)) @@ -48,20 +48,20 @@ testsuite.append(testcase) return testsuite -def test_to_junit(test): +def test_to_junit(t): testcase = et.Element('testcase') - testcase.set('name', test.name()) - testcase.set('time', str(math.ceil(test.duration))) - if test.status == suite.Test.SKIP: + testcase.set('name', t.name()) + testcase.set('time', str(math.ceil(t.duration))) + if t.status == test.Test.SKIP: skip = et.SubElement(testcase, 'skipped') - elif test.status == suite.Test.FAIL: + elif t.status == test.Test.FAIL: failure = et.SubElement(testcase, 'failure') - failure.set('type', test.fail_type or 'failure') - failure.text = test.fail_message - if test.fail_tb: + failure.set('type', t.fail_type or 'failure') + failure.text = t.fail_message + if t.fail_tb: system_err = et.SubElement(testcase, 'system-err') - system_err.text = test.fail_tb - elif test.status != suite.Test.PASS: + system_err.text = t.fail_tb + elif t.status != test.Test.PASS: error = et.SubElement(testcase, 'error') error.text = 'could not run' return testcase @@ -102,12 +102,12 @@ msgs.extend([test_to_text(t) for t in suite.tests]) return '\n '.join(msgs) -def test_to_text(test): - msgs = ['%s: %s' % (test.status, test.name())] - if test.start_timestamp: - msgs.append('(%.1f sec)' % test.duration) - if test.status == suite.Test.FAIL: - msgs.append('%s: %s' % (test.fail_type, test.fail_message)) +def test_to_text(t): + msgs = ['%s: %s' % (t.status, t.name())] + if t.start_timestamp: + msgs.append('(%.1f sec)' % t.duration) + if t.status == test.Test.FAIL: + msgs.append('%s: %s' % (t.fail_type, t.fail_message)) return ' '.join(msgs) # vim: expandtab tabstop=4 shiftwidth=4 diff --git a/src/osmo_gsm_tester/suite.py b/src/osmo_gsm_tester/suite.py index 5b9df76..25aef35 100644 --- a/src/osmo_gsm_tester/suite.py +++ b/src/osmo_gsm_tester/suite.py @@ -20,11 +20,9 @@ import os import sys import time -import traceback import pprint -from . import config, log, template, util, resource, schema, event_loop -from . import osmo_nitb, osmo_hlr, osmo_mgcpgw, osmo_mgw, osmo_msc, osmo_bsc, osmo_stp, modem, esme, sms -from . import testenv +from . import config, log, template, util, resource, schema, event_loop, test +from . import osmo_nitb, osmo_hlr, osmo_mgcpgw, osmo_mgw, osmo_msc, osmo_bsc, osmo_stp, modem, esme class Timeout(Exception): pass @@ -59,94 +57,6 @@ continue self.test_basenames.append(basename) - -class Test(log.Origin): - UNKNOWN = 'UNKNOWN' - SKIP = 'skip' - PASS = 'pass' - FAIL = 'FAIL' - - _run_dir = None - - def __init__(self, suite_run, test_basename): - self.basename = test_basename - super().__init__(log.C_TST, self.basename) - self.suite_run = suite_run - self.path = os.path.join(self.suite_run.definition.suite_dir, self.basename) - self.status = Test.UNKNOWN - self.start_timestamp = 0 - self.duration = 0 - self.fail_type = None - self.fail_message = None - - def get_run_dir(self): - if self._run_dir is None: - self._run_dir = util.Dir(self.suite_run.get_run_dir().new_dir(self._name)) - return self._run_dir - - def run(self): - try: - log.large_separator(self.suite_run.trial.name(), self.suite_run.name(), self.name(), sublevel=3) - self.status = Test.UNKNOWN - self.start_timestamp = time.time() - testenv.setup(self.suite_run, self, sys.modules[__name__], event_loop, sms) - with self.redirect_stdout(): - util.run_python_file('%s.%s' % (self.suite_run.definition.name(), self.basename), - self.path) - if self.status == Test.UNKNOWN: - self.set_pass() - except Exception as e: - if hasattr(e, 'msg'): - msg = e.msg - else: - msg = str(e) - if isinstance(e, AssertionError): - # AssertionError lacks further information on what was - # asserted. Find the line where the code asserted: - msg += log.get_src_from_exc_info(sys.exc_info()) - # add source file information to failure report - if hasattr(e, 'origins'): - msg += ' [%s]' % e.origins - tb_str = traceback.format_exc() - if isinstance(e, resource.NoResourceExn): - tb_str += self.suite_run.resource_status_str() - self.set_fail(type(e).__name__, msg, tb_str, log.get_src_from_exc_info()) - except BaseException as e: - # when the program is aborted by a signal (like Ctrl-C), escalate to abort all. - self.err('TEST RUN ABORTED: %s' % type(e).__name__) - raise - - def name(self): - l = log.get_line_for_src(self.path) - if l is not None: - return '%s:%s' % (self._name, l) - return super().name() - - def set_fail(self, fail_type, fail_message, tb_str=None, src=4): - self.status = Test.FAIL - self.duration = time.time() - self.start_timestamp - self.fail_type = fail_type - self.fail_message = fail_message - - if tb_str is None: - # populate an exception-less call to set_fail() with traceback info - tb_str = ''.join(traceback.format_stack()[:-1]) - - self.fail_tb = tb_str - self.err('%s: %s' % (self.fail_type, self.fail_message), _src=src) - if self.fail_tb: - self.log(self.fail_tb, _level=log.L_TRACEBACK) - self.log('Test FAILED (%.1f sec)' % self.duration) - - def set_pass(self): - self.status = Test.PASS - self.duration = time.time() - self.start_timestamp - self.log('Test passed (%.1f sec)' % self.duration) - - def set_skip(self): - self.status = Test.SKIP - self.duration = 0 - class SuiteRun(log.Origin): UNKNOWN = 'UNKNOWN' PASS = 'PASS' @@ -176,7 +86,7 @@ def load_tests(self): self.tests = [] for test_basename in self.definition.test_basenames: - self.tests.append(Test(self, test_basename)) + self.tests.append(test.Test(self, test_basename)) def register_for_cleanup(self, *obj): assert all([hasattr(o, 'cleanup') for o in obj]) @@ -243,12 +153,12 @@ event_loop.register_poll_func(self.poll) if not self.reserved_resources: self.reserve_resources() - for test in self.tests: - if names and not test.name() in names: - test.set_skip() + for t in self.tests: + if names and not t.name() in names: + t.set_skip() continue - self.current_test = test - test.run() + self.current_test = t + t.run() self.stop_processes() self.objects_cleanup() self.reserved_resources.put_all() @@ -284,10 +194,10 @@ passed = 0 skipped = 0 failed = 0 - for test in self.tests: - if test.status == Test.PASS: + for t in self.tests: + if t.status == test.Test.PASS: passed += 1 - elif test.status == Test.FAIL: + elif t.status == test.Test.FAIL: failed += 1 else: skipped += 1 diff --git a/src/osmo_gsm_tester/test.py b/src/osmo_gsm_tester/test.py new file mode 100644 index 0000000..82b290f --- /dev/null +++ b/src/osmo_gsm_tester/test.py @@ -0,0 +1,116 @@ +# osmo_gsm_tester: test class +# +# Copyright (C) 2017 by sysmocom - s.f.m.c. GmbH +# +# Author: Pau Espin Pedrol <pespin at sysmocom.de> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see <http://www.gnu.org/licenses/>. + +import os +import sys +import time +import traceback +from . import testenv + +from . import log, util, resource + +class Test(log.Origin): + UNKNOWN = 'UNKNOWN' + SKIP = 'skip' + PASS = 'pass' + FAIL = 'FAIL' + + _run_dir = None + + def __init__(self, suite_run, test_basename): + self.basename = test_basename + super().__init__(log.C_TST, self.basename) + self.suite_run = suite_run + self.path = os.path.join(self.suite_run.definition.suite_dir, self.basename) + self.status = Test.UNKNOWN + self.start_timestamp = 0 + self.duration = 0 + self.fail_type = None + self.fail_message = None + + def get_run_dir(self): + if self._run_dir is None: + self._run_dir = util.Dir(self.suite_run.get_run_dir().new_dir(self._name)) + return self._run_dir + + def run(self): + try: + log.large_separator(self.suite_run.trial.name(), self.suite_run.name(), self.name(), sublevel=3) + self.status = Test.UNKNOWN + self.start_timestamp = time.time() + from . import suite, event_loop, sms + testenv.setup(self.suite_run, self, suite, event_loop, sms) + with self.redirect_stdout(): + util.run_python_file('%s.%s' % (self.suite_run.definition.name(), self.basename), + self.path) + if self.status == Test.UNKNOWN: + self.set_pass() + except Exception as e: + if hasattr(e, 'msg'): + msg = e.msg + else: + msg = str(e) + if isinstance(e, AssertionError): + # AssertionError lacks further information on what was + # asserted. Find the line where the code asserted: + msg += log.get_src_from_exc_info(sys.exc_info()) + # add source file information to failure report + if hasattr(e, 'origins'): + msg += ' [%s]' % e.origins + tb_str = traceback.format_exc() + if isinstance(e, resource.NoResourceExn): + tb_str += self.suite_run.resource_status_str() + self.set_fail(type(e).__name__, msg, tb_str, log.get_src_from_exc_info()) + except BaseException as e: + # when the program is aborted by a signal (like Ctrl-C), escalate to abort all. + self.err('TEST RUN ABORTED: %s' % type(e).__name__) + raise + + def name(self): + l = log.get_line_for_src(self.path) + if l is not None: + return '%s:%s' % (self._name, l) + return super().name() + + def set_fail(self, fail_type, fail_message, tb_str=None, src=4): + self.status = Test.FAIL + self.duration = time.time() - self.start_timestamp + self.fail_type = fail_type + self.fail_message = fail_message + + if tb_str is None: + # populate an exception-less call to set_fail() with traceback info + tb_str = ''.join(traceback.format_stack()[:-1]) + + self.fail_tb = tb_str + self.err('%s: %s' % (self.fail_type, self.fail_message), _src=src) + if self.fail_tb: + self.log(self.fail_tb, _level=log.L_TRACEBACK) + self.log('Test FAILED (%.1f sec)' % self.duration) + + def set_pass(self): + self.status = Test.PASS + self.duration = time.time() - self.start_timestamp + self.log('Test passed (%.1f sec)' % self.duration) + + def set_skip(self): + self.status = Test.SKIP + self.duration = 0 + +# vim: expandtab tabstop=4 shiftwidth=4 -- To view, visit https://gerrit.osmocom.org/4764 To unsubscribe, visit https://gerrit.osmocom.org/settings Gerrit-MessageType: newpatchset Gerrit-Change-Id: I9c8d67f598466ba52a4827ff77027b9eae85929a Gerrit-PatchSet: 2 Gerrit-Project: osmo-gsm-tester Gerrit-Branch: master Gerrit-Owner: Pau Espin Pedrol <pespin at sysmocom.de> Gerrit-Reviewer: Harald Welte <laforge at gnumonks.org> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: Neels Hofmeyr <nhofmeyr at sysmocom.de> Gerrit-Reviewer: Pau Espin Pedrol <pespin at sysmocom.de>