osmith has uploaded this change for review.

View Change

testenv: on crash, wait until test stopped

When the IUT (or other test component) crashes, wait until
ttcn3-tcpdump-stop.sh has closed the pcap, as otherwise the last
buffered data may be lost.

Add a timeout of 2 minutes in case the test doesn't stop on its own.

I considered making this feature optional, but impatient users can still
immediately kill everything with ^C.

Change-Id: If49263869b1d46103813e9d06deff47f8ba72896
---
M _testenv/testenv/daemons.py
M _testenv/testenv/testsuite.py
M ttcn3-tcpdump-start.sh
M ttcn3-tcpdump-stop.sh
4 files changed, 47 insertions(+), 3 deletions(-)

git pull ssh://gerrit.osmocom.org:29418/osmo-ttcn3-hacks refs/changes/35/38135/1

diff --git a/_testenv/testenv/daemons.py b/_testenv/testenv/daemons.py
index c211cd7..c7e8722 100644
--- a/_testenv/testenv/daemons.py
+++ b/_testenv/testenv/daemons.py
@@ -9,6 +9,7 @@
 import testenv
 import testenv.testdir
 import time
+import sys
 
 daemons = {}
 run_shell_on_stop = False
@@ -100,6 +101,24 @@
     kill_process_tree(pid, ppids)
 
 
+def check_if_crashed():
+    crashed = False
+    for daemon_name, daemon_proc in daemons.items():
+        if not testenv.testsuite.is_running(daemon_proc.pid):
+            crashed = True
+            break
+    if not crashed:
+        return
+
+    current_test = testenv.testsuite.get_current_test()
+    if current_test:
+        logging.error(f"{daemon_name} crashed during {current_test}!")
+        testenv.testsuite.wait_until_test_stopped()
+    else:
+        logging.error(f"{daemon_name} crashed!")
+    sys.exit(1)
+
+
 def stop():
     global daemons
     global run_shell_on_stop
diff --git a/_testenv/testenv/testsuite.py b/_testenv/testenv/testsuite.py
index d14d36f..7a42216 100644
--- a/_testenv/testenv/testsuite.py
+++ b/_testenv/testenv/testsuite.py
@@ -200,9 +200,7 @@
             stop()
             break
 
-        for daemon_name, daemon_proc in testenv.daemons.daemons.items():
-            if not is_running(daemon_proc.pid):
-                raise testenv.NoTraceException(f"{daemon_name} crashed!")
+        testenv.daemons.check_if_crashed()
 
     merge_log_files(cfg)
     format_log_files(cfg)
@@ -217,6 +215,29 @@
     testenv.cmd.run(section_data["prepare"])
 
 
+def get_current_test():
+    path = os.path.join(testenv.testdir.testdir, "testsuite/.current_test")
+    try:
+        with open(path, "r") as h:
+            return h.readline().rstrip()
+    except:
+        # File may not exist, e.g. if test was stopped
+        return None
+
+
+def wait_until_test_stopped():
+    path = os.path.join(testenv.testdir.testdir, "testsuite/.current_test")
+
+    logging.debug("Waiting until test has stopped...")
+
+    for i in range(0, 1200):
+        time.sleep(0.1)
+        if not os.path.exists(path):
+            return
+
+    raise testenv.NoTraceError("Timeout in wait_until_test_stopped()")
+
+
 def stop():
     global testsuite_proc
 
diff --git a/ttcn3-tcpdump-start.sh b/ttcn3-tcpdump-start.sh
index 4068b26..9575a68 100755
--- a/ttcn3-tcpdump-start.sh
+++ b/ttcn3-tcpdump-start.sh
@@ -120,3 +120,5 @@
 	fi
 done
 kill $PID
+
+echo "$TESTCASE" > "$TTCN3_PCAP_PATH/.current_test"
diff --git a/ttcn3-tcpdump-stop.sh b/ttcn3-tcpdump-stop.sh
index 0ac1b5e..def10d8 100755
--- a/ttcn3-tcpdump-stop.sh
+++ b/ttcn3-tcpdump-stop.sh
@@ -83,3 +83,5 @@
        PCAP_FILENAME="$TTCN3_PCAP_PATH/$TESTCASE.$i.pcap"
 fi
 gzip -f "$PCAP_FILENAME"
+
+rm -f "$TTCN3_PCAP_PATH/.current_test"

To view, visit change 38135. To unsubscribe, or for help writing mail filters, visit settings.