[L] Change in docker-playground[master]: nplab-{m3ua,sua}-test: Record pcap of all testsuite run
laforge has submitted this change. ( https://gerrit.osmocom.org/c/docker-playground/+/39770?usp=email )
Change subject: nplab-{m3ua,sua}-test: Record pcap of all testsuite run ......................................................................
nplab-{m3ua,sua}-test: Record pcap of all testsuite run
Scripts are cherry-picked from osmo-ttcn3-hacks.git/ttcn3-tcpdump-{start,stop}.sh, 8fc1b2152ff7615f610e7a040cb3b5135dfc5351.
Change-Id: If4ea74c07d5aaca1278811ce8398dff23c386c10 --- A common/tcpdump-start.sh A common/tcpdump-stop.sh M debian-buster-build/Dockerfile M nplab-m3ua-test/Dockerfile M nplab-m3ua-test/jenkins.sh M nplab-m3ua-test/osmo-stp.cfg A nplab-m3ua-test/test.sh M nplab-sua-test/Dockerfile M nplab-sua-test/jenkins.sh M nplab-sua-test/osmo-stp.cfg A nplab-sua-test/test.sh 11 files changed, 239 insertions(+), 44 deletions(-)
Approvals: osmith: Looks good to me, approved Jenkins Builder: Verified
diff --git a/common/tcpdump-start.sh b/common/tcpdump-start.sh new file mode 100755 index 0000000..889c171 --- /dev/null +++ b/common/tcpdump-start.sh @@ -0,0 +1,124 @@ +#!/bin/sh + +PIDFILE_PCAP=/tmp/pcap.pid +TCPDUMP=$(command -v tcpdump) +DUMPCAP=$(command -v dumpcap) + +PIDFILE_NETCAT=/tmp/netcat.pid +NETCAT=$(command -v nc) +GSMTAP_PORT=4729 + +TESTCASE=$1 + + +SUDOSTR="" +if ! [ "$(id -u)" = "0" ]; then + SUDOSTR="sudo -n" + # Otherwise, if sudo /usr/bin/kill, sudo /usr/bin/tcpdump cannot be run without a password prompt, + # and this script will hang indefinitely +fi + +kill_rm_pidfile() { + # NOTE: This requires you to be root or something like + # "laforge ALL=NOPASSWD: /usr/sbin/tcpdump, /bin/kill" in your sudoers file + if [ -e "$1" ]; then + if [ -s "$1" ]; then + $SUDOSTR kill "$(cat "$1")" 2>&1 | grep -v "No such process" + fi + rm $1 + fi +} + +echo "------ $TESTCASE ------" +date + +if [ "z$TTCN3_PCAP_PATH" = "z" ]; then + TTCN3_PCAP_PATH=/tmp +fi + +kill_rm_pidfile $PIDFILE_NETCAT +kill_rm_pidfile $PIDFILE_PCAP + +CMD="$SUDOSTR $TCPDUMP -U" + +if [ -x "$DUMPCAP" ]; then + CAP_ERR="1" + if [ -x /sbin/setcap ]; then + # N. B: this check requires libcap2-bin package + /sbin/setcap -q -v 'cap_net_admin,cap_net_raw=pie' $DUMPCAP + CAP_ERR="$?" + fi + + # did we implicitly inherit all those caps because we're root in a netns? + if [ -u $DUMPCAP -o "$CAP_ERR" = "1" ]; then + getpcaps 0 2>&1 | grep -e cap_net_admin | grep -q -e cap_net_raw + CAP_ERR="$?" + fi + + # did we implicitly inherit all those caps because we're root in a netns? + if [ -u $DUMPCAP -o "$CAP_ERR" = "1" ]; then + getpcaps 0 2>&1 | grep -q -e " =ep" # all perms + CAP_ERR="$?" + fi + + if [ -u $DUMPCAP -o "$CAP_ERR" = "0" ]; then + # dumpcap, *after dropping permissions*, needs to be able to write to the directory to create the pcap file: + if [ "$(stat -L -c "%u" "$TTCN3_PCAP_PATH")" = "$(id -u)" ] && [ "$(stat -L -c "%A" "$TTCN3_PCAP_PATH" | head -c 4)" = "drwx" ]; then + CMD="$DUMPCAP -q" + else + echo "NOTE: unable to use dumpcap due to missing permissions in $TTCN3_PCAP_PATH" + fi + else + echo "NOTE: unable to use dumpcap due to missing capabilities or suid bit" + fi +fi + +# Create a dummy sink for GSMTAP packets +$NETCAT -l -u -k -p $GSMTAP_PORT >/dev/null 2>$TESTCASE.netcat.stderr & +PID=$! +echo $PID > $PIDFILE_NETCAT + +CMD_OUTFILE=$TTCN3_PCAP_PATH/$TESTCASE.pcap.stdout +CMD_OUTFILE_ERR=$TTCN3_PCAP_PATH/$TESTCASE.pcap.stderr +FIFO=/tmp/cmderr +if ! [ -e $FIFO ]; then + mkfifo $FIFO +else + echo "Warning: Named pipe already exists: $FIFO" +fi + +# Log stderr to CMD_OUTFILE and a dedicated error log file +tee $CMD_OUTFILE < $FIFO > $CMD_OUTFILE_ERR & +CMD_STR="$CMD -s 1520 -n -i any -w "$TTCN3_PCAP_PATH/$TESTCASE.pcap" >$CMD_OUTFILE 2>$FIFO &" +echo "$CMD_STR" +eval $CMD_STR +# $CMD -s 1520 -n -i any -w "$TTCN3_PCAP_PATH/$TESTCASE.pcap" >$CMD_OUTFILE & +PID=$! +echo $PID > $PIDFILE_PCAP +if [ -f $CMD_OUTFILE_ERR ] && [ $(wc -l $CMD_OUTFILE_ERR | awk '{print $1}') -ne 0 ]; then + echo "Warnings or error messages from command:" >&2 + echo " $CMD_STR" >&2 + echo "Message:" >&2 + echo "$(cat $CMD_OUTFILE_ERR)" | sed 's/^/\t/' >&2 +fi + +# Wait until packet dumper creates the pcap file and starts recording. +# We generate some traffic until we see packet dumper catches it. +# Timeout is 10 seconds. +ping 127.0.0.1 >/dev/null 2>&1 & +PID=$! +i=0 +while [ ! -f "$TTCN3_PCAP_PATH/$TESTCASE.pcap" ] || + [ "$(stat -c '%s' "$TTCN3_PCAP_PATH/$TESTCASE.pcap")" -eq 32 ] +do + echo "Waiting for packet dumper to start... $i" + sleep 1 + i=$((i+1)) + if [ $i -eq 10 ]; then + echo "Packet dumper didn't start filling pcap file after $i seconds!!!" + break + fi +done +kill $PID + +echo "$TESTCASE" > "$TTCN3_PCAP_PATH/.current_test" diff --git a/common/tcpdump-stop.sh b/common/tcpdump-stop.sh new file mode 100755 index 0000000..c9226a0 --- /dev/null +++ b/common/tcpdump-stop.sh @@ -0,0 +1,83 @@ +#!/bin/sh + +PIDFILE_PCAP=/tmp/pcap.pid +PIDFILE_NETCAT=/tmp/netcat.pid +FIFO=/tmp/cmderr +TESTCASE=$1 + +SUDOSTR="" +if ! [ "$(id -u)" = "0" ]; then + SUDOSTR="sudo -n" + # Otherwise, if sudo /usr/bin/kill, sudo /usr/bin/tcpdump cannot be run without a password prompt, + # and this script will hang indefinitely +fi + +kill_rm_pidfile() { + # NOTE: This requires you to be root or something like + # "laforge ALL=NOPASSWD: /usr/sbin/tcpdump, /bin/kill" in your sudoers file + if [ -e "$1" ]; then + if [ -s "$1" ]; then + $SUDOSTR kill "$(cat "$1")" 2>&1 | grep -v "No such process" + fi + rm $1 + fi +} + +date + +echo "------ $TESTCASE ------" + +if [ "z$TTCN3_PCAP_PATH" = "z" ]; then + TTCN3_PCAP_PATH=/tmp +fi + +# Order the SUT to print a talloc report +if [ "z$OSMO_SUT_HOST" != "z" ] && [ "z$OSMO_SUT_PORT" != "z" ]; then + if [ -x "$(command -v osmo_interact_vty.py)" ]; then + echo "Saving talloc report from $OSMO_SUT_HOST:$OSMO_SUT_PORT to $TESTCASE.talloc" + if ! timeout 5 osmo_interact_vty.py \ + -H $OSMO_SUT_HOST -p $OSMO_SUT_PORT \ + -c "en; show talloc-context application full" \ + > "$TTCN3_PCAP_PATH/$TESTCASE.talloc"; then + echo + echo "ERROR: failed to get talloc report via vty" + echo + fi + else + echo "Missing osmo_interact_vty.py from osmo-python-tests!" + echo " -> Unable to obtain talloc report from the SUT" + fi +fi + +# Wait for up to 2 seconds if we keep receiving traffinc from packet dumper, +# otherwise we might lose last packets from test. +i=0 +prev_count=-1 +count=$(stat --format="%s" "$TTCN3_PCAP_PATH/$TESTCASE.pcap") +while [ $count -gt $prev_count ] && [ $i -lt 2 ] +do + echo "Waiting for packet dumper to finish... $i (prev_count=$prev_count, count=$count)" + sleep 1 + prev_count=$count + count=$(stat --format="%s" "$TTCN3_PCAP_PATH/$TESTCASE.pcap") + i=$((i+1)) +done + +kill_rm_pidfile "$PIDFILE_PCAP" +kill_rm_pidfile "$PIDFILE_NETCAT" +rm $FIFO + +# Add a numeral suffix to subsequent runs of the same test: +PCAP_FILENAME=$TTCN3_PCAP_PATH/$TESTCASE.pcap +if [ -f "$TTCN3_PCAP_PATH/$TESTCASE.pcap.gz" ]; then + i=1 + while [ -f "$TTCN3_PCAP_PATH/$TESTCASE.$i.pcap.gz" ]; + do i=$((i+1)) + done + mv "$PCAP_FILENAME" "$TTCN3_PCAP_PATH/$TESTCASE.$i.pcap" + PCAP_FILENAME="$TTCN3_PCAP_PATH/$TESTCASE.$i.pcap" +fi +gzip -f "$PCAP_FILENAME" +mv "${PCAP_FILENAME}.gz" /data/ + +rm -f "$TTCN3_PCAP_PATH/.current_test" diff --git a/debian-buster-build/Dockerfile b/debian-buster-build/Dockerfile index 0d2b5b7..1059d3a 100644 --- a/debian-buster-build/Dockerfile +++ b/debian-buster-build/Dockerfile @@ -54,10 +54,12 @@ libusb-1.0-0-dev \ libusb-dev \ make \ + netcat-openbsd \ pkg-config \ sqlite3 \ stow \ telnet \ + tcpdump \ wget && \ apt-get clean
diff --git a/nplab-m3ua-test/Dockerfile b/nplab-m3ua-test/Dockerfile index 2f6ad93..9e4f7c4 100644 --- a/nplab-m3ua-test/Dockerfile +++ b/nplab-m3ua-test/Dockerfile @@ -10,6 +10,9 @@ cp runtest-junitxml.py /usr/local/bin/
COPY dotguile /root/.guile +COPY .common/tcpdump-start.sh /usr/local/bin/tcpdump-start.sh +COPY .common/tcpdump-stop.sh /usr/local/bin/tcpdump-stop.sh +COPY test.sh /usr/local/bin/test.sh
RUN mkdir /data
diff --git a/nplab-m3ua-test/jenkins.sh b/nplab-m3ua-test/jenkins.sh index 3db8745..5614346 100755 --- a/nplab-m3ua-test/jenkins.sh +++ b/nplab-m3ua-test/jenkins.sh @@ -36,4 +36,5 @@ --ulimit core=-1 \ -v $VOL_BASE_DIR/m3ua-tester:/data \ --name ${BUILD_TAG}-m3ua-test \ - $REPO_USER/nplab-m3ua-test > $VOL_BASE_DIR/m3ua-tester/junit-xml-m3ua.log + $REPO_USER/nplab-m3ua-test \ + /usr/local/bin/test.sh diff --git a/nplab-m3ua-test/osmo-stp.cfg b/nplab-m3ua-test/osmo-stp.cfg index f315d77..5206412 100644 --- a/nplab-m3ua-test/osmo-stp.cfg +++ b/nplab-m3ua-test/osmo-stp.cfg @@ -2,6 +2,9 @@ ! osmo-stp (0.0.6.3.179-b248) configuration saved from vty !! ! +log gsmtap 172.18.19.2 + logging level set-all debug + logging filter all 1 log stderr logging filter all 1 logging color 1 @@ -23,27 +26,6 @@ logging level lsccp debug logging level lsua debug logging level lm3ua debug -log file /data/osmo-stp.log - logging filter all 1 - logging color 0 - logging print category 1 - logging timestamp 1 - logging level lglobal notice - logging level llapd notice - logging level linp debug - logging level lmux notice - logging level lmi notice - logging level lmib notice - logging level lsms notice - logging level lctrl notice - logging level lgtp notice - logging level lstats notice - logging level lgsup notice - logging level loap notice - logging level lss7 debug - logging level lsccp debug - logging level lsua debug - logging level lm3ua debug ! line vty no login diff --git a/nplab-m3ua-test/test.sh b/nplab-m3ua-test/test.sh new file mode 100755 index 0000000..58ef8dd --- /dev/null +++ b/nplab-m3ua-test/test.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +/usr/local/bin/tcpdump-start.sh nplab-m3ua-test + +/usr/local/bin/runtest-junitxml.py -s 0.1 -t 10 -d /root /data/all-sgp-tests.txt > /data/junit-xml-m3ua.log + +/usr/local/bin/tcpdump-stop.sh nplab-m3ua-test diff --git a/nplab-sua-test/Dockerfile b/nplab-sua-test/Dockerfile index 6bb3ad2..5957c26 100644 --- a/nplab-sua-test/Dockerfile +++ b/nplab-sua-test/Dockerfile @@ -10,6 +10,9 @@ cp runtest-junitxml.py /usr/local/bin/
COPY dotguile /root/.guile +COPY .common/tcpdump-start.sh /usr/local/bin/tcpdump-start.sh +COPY .common/tcpdump-stop.sh /usr/local/bin/tcpdump-stop.sh +COPY test.sh /usr/local/bin/test.sh
RUN mkdir /data
diff --git a/nplab-sua-test/jenkins.sh b/nplab-sua-test/jenkins.sh index a9fbadf..b2ea725 100755 --- a/nplab-sua-test/jenkins.sh +++ b/nplab-sua-test/jenkins.sh @@ -36,4 +36,5 @@ --ulimit core=-1 \ -v $VOL_BASE_DIR/sua-tester:/data \ --name ${BUILD_TAG}-sua-test \ - $REPO_USER/nplab-sua-test > $VOL_BASE_DIR/sua-tester/junit-xml-sua.log + $REPO_USER/nplab-sua-test \ + /usr/local/bin/test.sh diff --git a/nplab-sua-test/osmo-stp.cfg b/nplab-sua-test/osmo-stp.cfg index b9e754b..4799ab1 100644 --- a/nplab-sua-test/osmo-stp.cfg +++ b/nplab-sua-test/osmo-stp.cfg @@ -2,6 +2,9 @@ ! osmo-stp (0.0.6.3.179-b248) configuration saved from vty !! ! +log gsmtap 172.18.19.3 + logging level set-all debug + logging filter all 1 log stderr logging filter all 1 logging color 1 @@ -23,27 +26,6 @@ logging level lsccp debug logging level lsua debug logging level lm3ua debug -log file /data/osmo-stp.log - logging filter all 1 - logging color 0 - logging print category 1 - logging timestamp 1 - logging level lglobal notice - logging level llapd notice - logging level linp debug - logging level lmux notice - logging level lmi notice - logging level lmib notice - logging level lsms notice - logging level lctrl notice - logging level lgtp notice - logging level lstats notice - logging level lgsup notice - logging level loap notice - logging level lss7 debug - logging level lsccp debug - logging level lsua debug - logging level lm3ua debug ! line vty no login diff --git a/nplab-sua-test/test.sh b/nplab-sua-test/test.sh new file mode 100755 index 0000000..728c01d --- /dev/null +++ b/nplab-sua-test/test.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +/usr/local/bin/tcpdump-start.sh nplab-sua-test + +/usr/local/bin/runtest-junitxml.py -s 0.1 -t 10 -d /root /data/some-sua-sgp-tests.txt > /data/junit-xml-m3ua.log + +/usr/local/bin/tcpdump-stop.sh nplab-sua-test
-
laforge