pespin has uploaded this change for review. ( https://gerrit.osmocom.org/c/docker-playground/+/39770?usp=email ) Change subject: nplab-m3ua-test: Record pcap of all testsuite run ...................................................................... nplab-m3ua-test: Record pcap of all testsuite run Change-Id: If4ea74c07d5aaca1278811ce8398dff23c386c10 --- M debian-buster-build/Dockerfile M nplab-m3ua-test/Dockerfile M nplab-m3ua-test/jenkins.sh A nplab-m3ua-test/tcpdump-start.sh A nplab-m3ua-test/tcpdump-stop.sh A nplab-m3ua-test/test.sh 6 files changed, 219 insertions(+), 1 deletion(-) git pull ssh://gerrit.osmocom.org:29418/docker-playground refs/changes/70/39770/1 diff --git a/debian-buster-build/Dockerfile b/debian-buster-build/Dockerfile index 0d2b5b7..49ee3c1 100644 --- a/debian-buster-build/Dockerfile +++ b/debian-buster-build/Dockerfile @@ -58,6 +58,7 @@ sqlite3 \ stow \ telnet \ + tcpdump \ wget && \ apt-get clean diff --git a/nplab-m3ua-test/Dockerfile b/nplab-m3ua-test/Dockerfile index 2f6ad93..23e885b 100644 --- a/nplab-m3ua-test/Dockerfile +++ b/nplab-m3ua-test/Dockerfile @@ -10,6 +10,9 @@ cp runtest-junitxml.py /usr/local/bin/ COPY dotguile /root/.guile +COPY test.sh /usr/local/bin/test.sh +COPY tcpdump-start.sh /usr/local/bin/tcpdump-start.sh +COPY tcpdump-stop.sh /usr/local/bin/tcpdump-stop.sh RUN mkdir /data diff --git a/nplab-m3ua-test/jenkins.sh b/nplab-m3ua-test/jenkins.sh index 3db8745..96e5854 100755 --- a/nplab-m3ua-test/jenkins.sh +++ b/nplab-m3ua-test/jenkins.sh @@ -36,4 +36,4 @@ --ulimit core=-1 \ -v $VOL_BASE_DIR/m3ua-tester:/data \ --name ${BUILD_TAG}-m3ua-test \ - $REPO_USER/nplab-m3ua-test > $VOL_BASE_DIR/m3ua-tester/junit-xml-m3ua.log + $REPO_USER/nplab-m3ua-test /usr/local/bin/test.sh diff --git a/nplab-m3ua-test/tcpdump-start.sh b/nplab-m3ua-test/tcpdump-start.sh new file mode 100755 index 0000000..9575a68 --- /dev/null +++ b/nplab-m3ua-test/tcpdump-start.sh @@ -0,0 +1,124 @@ +#!/bin/sh + +PIDFILE_PCAP=/tmp/pcap.pid +TCPDUMP=$(command -v tcpdump) +DUMPCAP=$(command -v dumpcap) + +PIDFILE_NETCAT=/tmp/netcat.pid +NETCAT=$(command -v nc) +GSMTAP_PORT=4729 + +TESTCASE=$1 + + +SUDOSTR="" +if ! [ "$(id -u)" = "0" ]; then + SUDOSTR="sudo -n" + # Otherwise, if sudo /usr/bin/kill, sudo /usr/bin/tcpdump cannot be run without a password prompt, + # and this script will hang indefinitely +fi + +kill_rm_pidfile() { + # NOTE: This requires you to be root or something like + # "laforge ALL=NOPASSWD: /usr/sbin/tcpdump, /bin/kill" in your sudoers file + if [ -e "$1" ]; then + if [ -s "$1" ]; then + $SUDOSTR kill "$(cat "$1")" 2>&1 | grep -v "No such process" + fi + rm $1 + fi +} + +echo "------ $TESTCASE ------" +date + +if [ "z$TTCN3_PCAP_PATH" = "z" ]; then + TTCN3_PCAP_PATH=/tmp +fi + +kill_rm_pidfile $PIDFILE_NETCAT +kill_rm_pidfile $PIDFILE_PCAP + +CMD="$SUDOSTR $TCPDUMP -U" + +if [ -x "$DUMPCAP" ]; then + CAP_ERR="1" + if [ -x /sbin/setcap ]; then + # N. B: this check requires libcap2-bin package + /sbin/setcap -q -v 'cap_net_admin,cap_net_raw=pie' $DUMPCAP + CAP_ERR="$?" + fi + + # did we implicitly inherit all those caps because we're root in a netns? + if [ -u $DUMPCAP -o "$CAP_ERR" = "1" ]; then + getpcaps 0 2>&1 | grep -e cap_net_admin | grep -q -e cap_net_raw + CAP_ERR="$?" + fi + + # did we implicitly inherit all those caps because we're root in a netns? + if [ -u $DUMPCAP -o "$CAP_ERR" = "1" ]; then + getpcaps 0 2>&1 | grep -q -e " =ep" # all perms + CAP_ERR="$?" + fi + + if [ -u $DUMPCAP -o "$CAP_ERR" = "0" ]; then + # dumpcap, *after dropping permissions*, needs to be able to write to the directory to create the pcap file: + if [ "$(stat -L -c "%u" "$TTCN3_PCAP_PATH")" = "$(id -u)" ] && [ "$(stat -L -c "%A" "$TTCN3_PCAP_PATH" | head -c 4)" = "drwx" ]; then + CMD="$DUMPCAP -q" + else + echo "NOTE: unable to use dumpcap due to missing permissions in $TTCN3_PCAP_PATH" + fi + else + echo "NOTE: unable to use dumpcap due to missing capabilities or suid bit" + fi +fi + +# Create a dummy sink for GSMTAP packets +$NETCAT -l -u -k -p $GSMTAP_PORT >/dev/null 2>$TESTCASE.netcat.stderr & +PID=$! +echo $PID > $PIDFILE_NETCAT + +CMD_OUTFILE=$TTCN3_PCAP_PATH/$TESTCASE.pcap.stdout +CMD_OUTFILE_ERR=$TTCN3_PCAP_PATH/$TESTCASE.pcap.stderr +FIFO=/tmp/cmderr +if ! [ -e $FIFO ]; then + mkfifo $FIFO +else + echo "Warning: Named pipe already exists: $FIFO" +fi + +# Log stderr to CMD_OUTFILE and a dedicated error log file +tee $CMD_OUTFILE < $FIFO > $CMD_OUTFILE_ERR & +CMD_STR="$CMD -s 1520 -n -i any -w \"$TTCN3_PCAP_PATH/$TESTCASE.pcap\" >$CMD_OUTFILE 2>$FIFO &" +echo "$CMD_STR" +eval $CMD_STR +# $CMD -s 1520 -n -i any -w \"$TTCN3_PCAP_PATH/$TESTCASE.pcap\" >$CMD_OUTFILE & +PID=$! +echo $PID > $PIDFILE_PCAP +if [ -f $CMD_OUTFILE_ERR ] && [ $(wc -l $CMD_OUTFILE_ERR | awk '{print $1}') -ne 0 ]; then + echo "Warnings or error messages from command:" >&2 + echo " $CMD_STR" >&2 + echo "Message:" >&2 + echo "$(cat $CMD_OUTFILE_ERR)" | sed 's/^/\t/' >&2 +fi + +# Wait until packet dumper creates the pcap file and starts recording. +# We generate some traffic until we see packet dumper catches it. +# Timeout is 10 seconds. +ping 127.0.0.1 >/dev/null 2>&1 & +PID=$! +i=0 +while [ ! -f "$TTCN3_PCAP_PATH/$TESTCASE.pcap" ] || + [ "$(stat -c '%s' "$TTCN3_PCAP_PATH/$TESTCASE.pcap")" -eq 32 ] +do + echo "Waiting for packet dumper to start... $i" + sleep 1 + i=$((i+1)) + if [ $i -eq 10 ]; then + echo "Packet dumper didn't start filling pcap file after $i seconds!!!" + break + fi +done +kill $PID + +echo "$TESTCASE" > "$TTCN3_PCAP_PATH/.current_test" diff --git a/nplab-m3ua-test/tcpdump-stop.sh b/nplab-m3ua-test/tcpdump-stop.sh new file mode 100755 index 0000000..c9226a0 --- /dev/null +++ b/nplab-m3ua-test/tcpdump-stop.sh @@ -0,0 +1,83 @@ +#!/bin/sh + +PIDFILE_PCAP=/tmp/pcap.pid +PIDFILE_NETCAT=/tmp/netcat.pid +FIFO=/tmp/cmderr +TESTCASE=$1 + +SUDOSTR="" +if ! [ "$(id -u)" = "0" ]; then + SUDOSTR="sudo -n" + # Otherwise, if sudo /usr/bin/kill, sudo /usr/bin/tcpdump cannot be run without a password prompt, + # and this script will hang indefinitely +fi + +kill_rm_pidfile() { + # NOTE: This requires you to be root or something like + # "laforge ALL=NOPASSWD: /usr/sbin/tcpdump, /bin/kill" in your sudoers file + if [ -e "$1" ]; then + if [ -s "$1" ]; then + $SUDOSTR kill "$(cat "$1")" 2>&1 | grep -v "No such process" + fi + rm $1 + fi +} + +date + +echo "------ $TESTCASE ------" + +if [ "z$TTCN3_PCAP_PATH" = "z" ]; then + TTCN3_PCAP_PATH=/tmp +fi + +# Order the SUT to print a talloc report +if [ "z$OSMO_SUT_HOST" != "z" ] && [ "z$OSMO_SUT_PORT" != "z" ]; then + if [ -x "$(command -v osmo_interact_vty.py)" ]; then + echo "Saving talloc report from $OSMO_SUT_HOST:$OSMO_SUT_PORT to $TESTCASE.talloc" + if ! timeout 5 osmo_interact_vty.py \ + -H $OSMO_SUT_HOST -p $OSMO_SUT_PORT \ + -c "en; show talloc-context application full" \ + > "$TTCN3_PCAP_PATH/$TESTCASE.talloc"; then + echo + echo "ERROR: failed to get talloc report via vty" + echo + fi + else + echo "Missing osmo_interact_vty.py from osmo-python-tests!" + echo " -> Unable to obtain talloc report from the SUT" + fi +fi + +# Wait for up to 2 seconds if we keep receiving traffinc from packet dumper, +# otherwise we might lose last packets from test. +i=0 +prev_count=-1 +count=$(stat --format="%s" "$TTCN3_PCAP_PATH/$TESTCASE.pcap") +while [ $count -gt $prev_count ] && [ $i -lt 2 ] +do + echo "Waiting for packet dumper to finish... $i (prev_count=$prev_count, count=$count)" + sleep 1 + prev_count=$count + count=$(stat --format="%s" "$TTCN3_PCAP_PATH/$TESTCASE.pcap") + i=$((i+1)) +done + +kill_rm_pidfile "$PIDFILE_PCAP" +kill_rm_pidfile "$PIDFILE_NETCAT" +rm $FIFO + +# Add a numeral suffix to subsequent runs of the same test: +PCAP_FILENAME=$TTCN3_PCAP_PATH/$TESTCASE.pcap +if [ -f "$TTCN3_PCAP_PATH/$TESTCASE.pcap.gz" ]; then + i=1 + while [ -f "$TTCN3_PCAP_PATH/$TESTCASE.$i.pcap.gz" ]; + do i=$((i+1)) + done + mv "$PCAP_FILENAME" "$TTCN3_PCAP_PATH/$TESTCASE.$i.pcap" + PCAP_FILENAME="$TTCN3_PCAP_PATH/$TESTCASE.$i.pcap" +fi +gzip -f "$PCAP_FILENAME" +mv "${PCAP_FILENAME}.gz" /data/ + +rm -f "$TTCN3_PCAP_PATH/.current_test" diff --git a/nplab-m3ua-test/test.sh b/nplab-m3ua-test/test.sh new file mode 100755 index 0000000..58ef8dd --- /dev/null +++ b/nplab-m3ua-test/test.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +/usr/local/bin/tcpdump-start.sh nplab-m3ua-test + +/usr/local/bin/runtest-junitxml.py -s 0.1 -t 10 -d /root /data/all-sgp-tests.txt > /data/junit-xml-m3ua.log + +/usr/local/bin/tcpdump-stop.sh nplab-m3ua-test -- To view, visit https://gerrit.osmocom.org/c/docker-playground/+/39770?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: newchange Gerrit-Project: docker-playground Gerrit-Branch: master Gerrit-Change-Id: If4ea74c07d5aaca1278811ce8398dff23c386c10 Gerrit-Change-Number: 39770 Gerrit-PatchSet: 1 Gerrit-Owner: pespin &lt;pespin(a)sysmocom.de&gt;