[M] Change in osmo-ttcn3-hacks[master]: testenv: add run --until-nok

osmith has submitted this change. ( https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/38776?usp=email ) ( 1 is the latest approved patch-set. No files were changed between the latest approved patch-set and the submitted one. )Change subject: testenv: add run --until-nok ...................................................................... testenv: add run --until-nok Add an argument to run a specific test (if using --test) or a whole testsuite until it fails with "failure" or "error". This helped me in reproducing a race condition in the mgw testsuite (related issue). Related: OS#3849 Change-Id: I17e1ebcc5d6ff1b6a087c4d4c9405a02798212f1 --- M _testenv/testenv.py M _testenv/testenv/__init__.py M _testenv/testenv/testdir.py M _testenv/testenv/testenv_cfg.py M _testenv/testenv/testsuite.py 5 files changed, 76 insertions(+), 23 deletions(-) Approvals: pespin: Looks good to me, but someone else must approve Jenkins Builder: Verified fixeria: Looks good to me, approved laforge: Looks good to me, but someone else must approve diff --git a/_testenv/testenv.py b/_testenv/testenv.py index 81b3aac..b9de4e8 100755 --- a/_testenv/testenv.py +++ b/_testenv/testenv.py @@ -16,6 +16,21 @@ import testenv.testsuite +def loop_continue_cond(loop_count): + if loop_count == 0: + return True + + if testenv.args.until_nok: + logging.info("Checking testsuite logs for failures and errors") + for match_str in ["failures='0'", "errors='0'"]: + if not testenv.testsuite.check_junit_logs_have(loop_count - 1, match_str): + logging.critical("Stopping the loop") + return False + return True + else: + return False + + def run(): testenv.testenv_cfg.init() @@ -45,26 +60,34 @@ testenv.osmo_dev.make(cfg) # Run the components + testsuite - cfg_count = 0 - for cfg_name, cfg in testenv.testenv_cfg.cfgs.items(): - # Restart podman container before running with another config - if testenv.args.podman and cfg_count: + loop_count = 0 + while loop_continue_cond(loop_count): + # Restart podman container before running again + if testenv.args.podman and loop_count: testenv.podman.stop(True) - testenv.testenv_cfg.set_current(cfg_name) + cfg_count = 0 + for cfg_name, cfg in testenv.testenv_cfg.cfgs.items(): + # Restart podman container before running with another config + if testenv.args.podman and cfg_count: + testenv.podman.stop(True) - if testenv.args.binary_repo: - testenv.podman.enable_binary_repo() - testenv.podman_install.packages(cfg, cfg_name) + testenv.testenv_cfg.set_current(cfg_name, loop_count) - testenv.testdir.prepare(cfg_name, cfg) - testenv.daemons.start(cfg) - testenv.testsuite.run(cfg) - testenv.daemons.stop() - testenv.testdir.clean_run_scripts("finished") + if testenv.args.binary_repo: + testenv.podman.enable_binary_repo() + testenv.podman_install.packages(cfg, cfg_name) - cfg_count += 1 - testenv.set_log_prefix("[testenv]") + testenv.testdir.prepare(cfg_name, cfg, loop_count) + testenv.daemons.start(cfg) + testenv.testsuite.run(cfg) + testenv.daemons.stop() + testenv.testdir.clean_run_scripts("finished") + + cfg_count += 1 + testenv.set_log_prefix("[testenv]") + + loop_count += 1 # Show test results testenv.testsuite.cat_junit_logs() diff --git a/_testenv/testenv/__init__.py b/_testenv/testenv/__init__.py index eb687f3..c1991ca 100644 --- a/_testenv/testenv/__init__.py +++ b/_testenv/testenv/__init__.py @@ -93,6 +93,14 @@ help="use binary packages from this Osmocom OBS project instead (e.g. osmocom:nightly)", ) + group = sub_run.add_argument_group("loop options", "Run the testsuite / a single test multiple times.") + group.add_argument( + "-u", + "--until-nok", + action="store_true", + help="run until there was either a failure or error", + ) + group = sub_run.add_argument_group( "QEMU options", "For some tests, the SUT can or must run in QEMU, typically to use kernel GTP-U.", diff --git a/_testenv/testenv/testdir.py b/_testenv/testenv/testdir.py index 88a2dca..b18efd1 100644 --- a/_testenv/testenv/testdir.py +++ b/_testenv/testenv/testdir.py @@ -51,14 +51,19 @@ testenv.cmd.run(["ln", "-sf", testdir_topdir, "/tmp/logs"], no_podman=True) -def prepare(cfg_name, cfg): +def prepare(cfg_name, cfg, loop_count): global testdir global clean_scripts + topdir = testdir_topdir + + if testenv.args.until_nok: + topdir = os.path.join(topdir, f"loop-{loop_count}") + if len(testenv.testenv_cfg.cfgs) == 1: - testdir = testdir_topdir + testdir = topdir else: - testdir = os.path.join(testdir_topdir, cfg_name.replace("testenv_", "").replace(".cfg", "")) + testdir = os.path.join(topdir, cfg_name.replace("testenv_", "").replace(".cfg", "")) logging.info(f"Preparing testdir: {testdir}") testsuite_dir = os.path.join(testenv.testsuite.ttcn3_hacks_dir, testenv.args.testsuite) diff --git a/_testenv/testenv/testenv_cfg.py b/_testenv/testenv/testenv_cfg.py index fefaf5c..4044225 100644 --- a/_testenv/testenv/testenv_cfg.py +++ b/_testenv/testenv/testenv_cfg.py @@ -13,16 +13,20 @@ current = None -def set_current(cfg_name): +def set_current(cfg_name, loop_count=0): global current current = cfg_name + loop_str = "" + + if testenv.args.until_nok: + loop_str = f"[loop-{loop_count}]" if cfg_name == "testenv.cfg": - testenv.set_log_prefix("[testenv]") + testenv.set_log_prefix(f"[testenv]{loop_str}") else: cfg_name = cfg_name.replace("testenv_", "") cfg_name = cfg_name.replace(".cfg", "") - testenv.set_log_prefix(f"[testenv][{cfg_name}]") + testenv.set_log_prefix(f"[testenv]{loop_str}[{cfg_name}]") def exit_error_readme(): diff --git a/_testenv/testenv/testsuite.py b/_testenv/testenv/testsuite.py index 1878b5c..0fa42ab 100644 --- a/_testenv/testenv/testsuite.py +++ b/_testenv/testenv/testsuite.py @@ -138,6 +138,11 @@ testenv.cmd.run(cmd, cwd=cwd) +def get_junit_logs(topdir): + pattern = os.path.join(topdir, "**", "junit-*.log") + return glob.glob(pattern, recursive=True) + + def cat_junit_logs(): tool = "cat" @@ -145,13 +150,21 @@ colors = os.environ.get("TESTENV_SOURCE_HIGHLIGHT_COLORS", "esc256") tool = f"source-highlight -f {shlex.quote(colors)} -s xml -i" - pattern = os.path.join(testenv.testdir.testdir_topdir, "**", "junit-*.log") - for path in glob.glob(pattern, recursive=True): + for path in get_junit_logs(testenv.testdir.testdir_topdir): cmd = f"echo && {tool} {shlex.quote(path)} && echo" logging.info(f"Showing {os.path.relpath(path, testenv.testdir.testdir_topdir)}") testenv.cmd.run(cmd) +def check_junit_logs_have(loop_count, match_str): + topdir = os.path.join(testenv.testdir.testdir_topdir, f"loop-{loop_count}") + for path in get_junit_logs(topdir): + cmd = ["grep", "-q", match_str, path] + if testenv.cmd.run(cmd, check=False).returncode: + return False + return True + + def run(cfg): global testsuite_proc -- To view, visit https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/38776?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: osmo-ttcn3-hacks Gerrit-Branch: master Gerrit-Change-Id: I17e1ebcc5d6ff1b6a087c4d4c9405a02798212f1 Gerrit-Change-Number: 38776 Gerrit-PatchSet: 2 Gerrit-Owner: osmith <osmith(a)sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: daniel <dwillmann(a)sysmocom.de> Gerrit-Reviewer: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Reviewer: laforge <laforge(a)osmocom.org> Gerrit-Reviewer: osmith <osmith(a)sysmocom.de> Gerrit-Reviewer: pespin <pespin(a)sysmocom.de>