laforge has submitted this change. ( https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/38135?usp=email ) Change subject: testenv: on crash, wait until test stopped ...................................................................... testenv: on crash, wait until test stopped When the IUT (or other test component) crashes, wait until ttcn3-tcpdump-stop.sh has closed the pcap, as otherwise the last buffered data may be lost. Add a timeout of 2 minutes in case the test doesn't stop on its own. I considered making this feature optional, but impatient users can still immediately kill everything with ^C. Change-Id: If49263869b1d46103813e9d06deff47f8ba72896 --- M _testenv/testenv/daemons.py M _testenv/testenv/testsuite.py M ttcn3-tcpdump-start.sh M ttcn3-tcpdump-stop.sh 4 files changed, 47 insertions(+), 3 deletions(-) Approvals: laforge: Looks good to me, but someone else must approve Jenkins Builder: Verified fixeria: Looks good to me, approved pespin: Looks good to me, but someone else must approve diff --git a/_testenv/testenv/daemons.py b/_testenv/testenv/daemons.py index c211cd7..c7e8722 100644 --- a/_testenv/testenv/daemons.py +++ b/_testenv/testenv/daemons.py @@ -9,6 +9,7 @@ import testenv import testenv.testdir import time +import sys daemons = {} run_shell_on_stop = False @@ -100,6 +101,24 @@ kill_process_tree(pid, ppids) +def check_if_crashed(): + crashed = False + for daemon_name, daemon_proc in daemons.items(): + if not testenv.testsuite.is_running(daemon_proc.pid): + crashed = True + break + if not crashed: + return + + current_test = testenv.testsuite.get_current_test() + if current_test: + logging.error(f"{daemon_name} crashed during {current_test}!") + testenv.testsuite.wait_until_test_stopped() + else: + logging.error(f"{daemon_name} crashed!") + sys.exit(1) + + def stop(): global daemons global run_shell_on_stop diff --git a/_testenv/testenv/testsuite.py b/_testenv/testenv/testsuite.py index d14d36f..7a42216 100644 --- a/_testenv/testenv/testsuite.py +++ b/_testenv/testenv/testsuite.py @@ -200,9 +200,7 @@ stop() break - for daemon_name, daemon_proc in testenv.daemons.daemons.items(): - if not is_running(daemon_proc.pid): - raise testenv.NoTraceException(f"{daemon_name} crashed!") + testenv.daemons.check_if_crashed() merge_log_files(cfg) format_log_files(cfg) @@ -217,6 +215,29 @@ testenv.cmd.run(section_data["prepare"]) +def get_current_test(): + path = os.path.join(testenv.testdir.testdir, "testsuite/.current_test") + try: + with open(path, "r") as h: + return h.readline().rstrip() + except: + # File may not exist, e.g. if test was stopped + return None + + +def wait_until_test_stopped(): + path = os.path.join(testenv.testdir.testdir, "testsuite/.current_test") + + logging.debug("Waiting until test has stopped...") + + for i in range(0, 1200): + time.sleep(0.1) + if not os.path.exists(path): + return + + raise testenv.NoTraceError("Timeout in wait_until_test_stopped()") + + def stop(): global testsuite_proc diff --git a/ttcn3-tcpdump-start.sh b/ttcn3-tcpdump-start.sh index 4068b26..9575a68 100755 --- a/ttcn3-tcpdump-start.sh +++ b/ttcn3-tcpdump-start.sh @@ -120,3 +120,5 @@ fi done kill $PID + +echo "$TESTCASE" > "$TTCN3_PCAP_PATH/.current_test" diff --git a/ttcn3-tcpdump-stop.sh b/ttcn3-tcpdump-stop.sh index 0ac1b5e..def10d8 100755 --- a/ttcn3-tcpdump-stop.sh +++ b/ttcn3-tcpdump-stop.sh @@ -83,3 +83,5 @@ PCAP_FILENAME="$TTCN3_PCAP_PATH/$TESTCASE.$i.pcap" fi gzip -f "$PCAP_FILENAME" + +rm -f "$TTCN3_PCAP_PATH/.current_test" -- To view, visit https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/38135?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: osmo-ttcn3-hacks Gerrit-Branch: master Gerrit-Change-Id: If49263869b1d46103813e9d06deff47f8ba72896 Gerrit-Change-Number: 38135 Gerrit-PatchSet: 2 Gerrit-Owner: osmith <osmith(a)sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Reviewer: laforge <laforge(a)osmocom.org> Gerrit-Reviewer: pespin <pespin(a)sysmocom.de>