osmith has uploaded this change for review.
testenv: add run --until-nok
Add an argument to run a specific test (if using --test) or a whole
testsuite until it fails with "failure" or "error". This helped me in
reproducing a race condition in the mgw testsuite (related issue).
Related: OS#3849
Change-Id: I17e1ebcc5d6ff1b6a087c4d4c9405a02798212f1
---
M _testenv/testenv.py
M _testenv/testenv/__init__.py
M _testenv/testenv/testdir.py
M _testenv/testenv/testenv_cfg.py
M _testenv/testenv/testsuite.py
5 files changed, 76 insertions(+), 23 deletions(-)
git pull ssh://gerrit.osmocom.org:29418/osmo-ttcn3-hacks refs/changes/76/38776/1
diff --git a/_testenv/testenv.py b/_testenv/testenv.py
index 81b3aac..b9de4e8 100755
--- a/_testenv/testenv.py
+++ b/_testenv/testenv.py
@@ -16,6 +16,21 @@
import testenv.testsuite
+def loop_continue_cond(loop_count):
+ if loop_count == 0:
+ return True
+
+ if testenv.args.until_nok:
+ logging.info("Checking testsuite logs for failures and errors")
+ for match_str in ["failures='0'", "errors='0'"]:
+ if not testenv.testsuite.check_junit_logs_have(loop_count - 1, match_str):
+ logging.critical("Stopping the loop")
+ return False
+ return True
+ else:
+ return False
+
+
def run():
testenv.testenv_cfg.init()
@@ -45,26 +60,34 @@
testenv.osmo_dev.make(cfg)
# Run the components + testsuite
- cfg_count = 0
- for cfg_name, cfg in testenv.testenv_cfg.cfgs.items():
- # Restart podman container before running with another config
- if testenv.args.podman and cfg_count:
+ loop_count = 0
+ while loop_continue_cond(loop_count):
+ # Restart podman container before running again
+ if testenv.args.podman and loop_count:
testenv.podman.stop(True)
- testenv.testenv_cfg.set_current(cfg_name)
+ cfg_count = 0
+ for cfg_name, cfg in testenv.testenv_cfg.cfgs.items():
+ # Restart podman container before running with another config
+ if testenv.args.podman and cfg_count:
+ testenv.podman.stop(True)
- if testenv.args.binary_repo:
- testenv.podman.enable_binary_repo()
- testenv.podman_install.packages(cfg, cfg_name)
+ testenv.testenv_cfg.set_current(cfg_name, loop_count)
- testenv.testdir.prepare(cfg_name, cfg)
- testenv.daemons.start(cfg)
- testenv.testsuite.run(cfg)
- testenv.daemons.stop()
- testenv.testdir.clean_run_scripts("finished")
+ if testenv.args.binary_repo:
+ testenv.podman.enable_binary_repo()
+ testenv.podman_install.packages(cfg, cfg_name)
- cfg_count += 1
- testenv.set_log_prefix("[testenv]")
+ testenv.testdir.prepare(cfg_name, cfg, loop_count)
+ testenv.daemons.start(cfg)
+ testenv.testsuite.run(cfg)
+ testenv.daemons.stop()
+ testenv.testdir.clean_run_scripts("finished")
+
+ cfg_count += 1
+ testenv.set_log_prefix("[testenv]")
+
+ loop_count += 1
# Show test results
testenv.testsuite.cat_junit_logs()
diff --git a/_testenv/testenv/__init__.py b/_testenv/testenv/__init__.py
index eb687f3..c1991ca 100644
--- a/_testenv/testenv/__init__.py
+++ b/_testenv/testenv/__init__.py
@@ -93,6 +93,14 @@
help="use binary packages from this Osmocom OBS project instead (e.g. osmocom:nightly)",
)
+ group = sub_run.add_argument_group("loop options", "Run the testsuite / a single test multiple times.")
+ group.add_argument(
+ "-u",
+ "--until-nok",
+ action="store_true",
+ help="run until there was either a failure or error",
+ )
+
group = sub_run.add_argument_group(
"QEMU options",
"For some tests, the SUT can or must run in QEMU, typically to use kernel GTP-U.",
diff --git a/_testenv/testenv/testdir.py b/_testenv/testenv/testdir.py
index 88a2dca..b18efd1 100644
--- a/_testenv/testenv/testdir.py
+++ b/_testenv/testenv/testdir.py
@@ -51,14 +51,19 @@
testenv.cmd.run(["ln", "-sf", testdir_topdir, "/tmp/logs"], no_podman=True)
-def prepare(cfg_name, cfg):
+def prepare(cfg_name, cfg, loop_count):
global testdir
global clean_scripts
+ topdir = testdir_topdir
+
+ if testenv.args.until_nok:
+ topdir = os.path.join(topdir, f"loop-{loop_count}")
+
if len(testenv.testenv_cfg.cfgs) == 1:
- testdir = testdir_topdir
+ testdir = topdir
else:
- testdir = os.path.join(testdir_topdir, cfg_name.replace("testenv_", "").replace(".cfg", ""))
+ testdir = os.path.join(topdir, cfg_name.replace("testenv_", "").replace(".cfg", ""))
logging.info(f"Preparing testdir: {testdir}")
testsuite_dir = os.path.join(testenv.testsuite.ttcn3_hacks_dir, testenv.args.testsuite)
diff --git a/_testenv/testenv/testenv_cfg.py b/_testenv/testenv/testenv_cfg.py
index fefaf5c..4044225 100644
--- a/_testenv/testenv/testenv_cfg.py
+++ b/_testenv/testenv/testenv_cfg.py
@@ -13,16 +13,20 @@
current = None
-def set_current(cfg_name):
+def set_current(cfg_name, loop_count=0):
global current
current = cfg_name
+ loop_str = ""
+
+ if testenv.args.until_nok:
+ loop_str = f"[loop-{loop_count}]"
if cfg_name == "testenv.cfg":
- testenv.set_log_prefix("[testenv]")
+ testenv.set_log_prefix(f"[testenv]{loop_str}")
else:
cfg_name = cfg_name.replace("testenv_", "")
cfg_name = cfg_name.replace(".cfg", "")
- testenv.set_log_prefix(f"[testenv][{cfg_name}]")
+ testenv.set_log_prefix(f"[testenv]{loop_str}[{cfg_name}]")
def exit_error_readme():
diff --git a/_testenv/testenv/testsuite.py b/_testenv/testenv/testsuite.py
index 1878b5c..0fa42ab 100644
--- a/_testenv/testenv/testsuite.py
+++ b/_testenv/testenv/testsuite.py
@@ -138,6 +138,11 @@
testenv.cmd.run(cmd, cwd=cwd)
+def get_junit_logs(topdir):
+ pattern = os.path.join(topdir, "**", "junit-*.log")
+ return glob.glob(pattern, recursive=True)
+
+
def cat_junit_logs():
tool = "cat"
@@ -145,13 +150,21 @@
colors = os.environ.get("TESTENV_SOURCE_HIGHLIGHT_COLORS", "esc256")
tool = f"source-highlight -f {shlex.quote(colors)} -s xml -i"
- pattern = os.path.join(testenv.testdir.testdir_topdir, "**", "junit-*.log")
- for path in glob.glob(pattern, recursive=True):
+ for path in get_junit_logs(testenv.testdir.testdir_topdir):
cmd = f"echo && {tool} {shlex.quote(path)} && echo"
logging.info(f"Showing {os.path.relpath(path, testenv.testdir.testdir_topdir)}")
testenv.cmd.run(cmd)
+def check_junit_logs_have(loop_count, match_str):
+ topdir = os.path.join(testenv.testdir.testdir_topdir, f"loop-{loop_count}")
+ for path in get_junit_logs(topdir):
+ cmd = ["grep", "-q", match_str, path]
+ if testenv.cmd.run(cmd, check=False).returncode:
+ return False
+ return True
+
+
def run(cfg):
global testsuite_proc
To view, visit change 38776. To unsubscribe, or for help writing mail filters, visit settings.