osmith has submitted this change. ( https://gerrit.osmocom.org/c/docker-playground/+/41720?usp=email ) Change subject: ttcn3-bts-test: run bpftrace scripts ...................................................................... ttcn3-bts-test: run bpftrace scripts Add logic to run bpftrace scripts attached to osmo-bts-* while the testsuite runs. The initial script is from Vadim: https://gitea.osmocom.org/osmocom/bpftrace-scripts/src/branch/master/udp_sen... Change-Id: I5c22cdbb08fe797be936ca174f65ade626fa4895 --- M README.md M osmo-bts-latest/Dockerfile M osmo-bts-master/Dockerfile A ttcn3-bts-test/bpftrace/udp_sendmsg_delay.bt M ttcn3-bts-test/jenkins.sh 5 files changed, 82 insertions(+), 0 deletions(-) Approvals: Jenkins Builder: Verified pespin: Looks good to me, but someone else must approve fixeria: Looks good to me, approved diff --git a/README.md b/README.md index eeb9272..3f55e3e 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ * `TEST_CONFIGS`: for tests that can run with multiple config sets (e.g. `ttcn3-bts-test`), run only some of them. See `TEST_CONFIGS_ALL` in the `jenkins.sh` for possible values. +* `RUN_BPFTRACE`: when set to `1`, run bpftrace scripts in `ttcn3-bts-test`. ### Run only one test diff --git a/osmo-bts-latest/Dockerfile b/osmo-bts-latest/Dockerfile index ad79cb3..6449829 100644 --- a/osmo-bts-latest/Dockerfile +++ b/osmo-bts-latest/Dockerfile @@ -9,6 +9,7 @@ debian*) \ apt-get update && \ apt-get install -y --no-install-recommends \ + bpftrace \ osmo-bts-trx \ osmo-bts-virtual && \ apt-get clean \ diff --git a/osmo-bts-master/Dockerfile b/osmo-bts-master/Dockerfile index 2e1b628..e081471 100644 --- a/osmo-bts-master/Dockerfile +++ b/osmo-bts-master/Dockerfile @@ -8,6 +8,7 @@ debian*) \ apt-get update && \ apt-get install -y --no-install-recommends \ + bpftrace \ libosmocore-dev \ libosmo-abis-dev \ libosmo-netif-dev && \ diff --git a/ttcn3-bts-test/bpftrace/udp_sendmsg_delay.bt b/ttcn3-bts-test/bpftrace/udp_sendmsg_delay.bt new file mode 100644 index 0000000..8525413 --- /dev/null +++ b/ttcn3-bts-test/bpftrace/udp_sendmsg_delay.bt @@ -0,0 +1,42 @@ +#!/usr/bin/env bpftrace + +/* Script to obtain udp_sendmsg() latency histograms. + * It will measure time between entry and exit and plot one histogram + * for each destination port number, indicating the amount of time spent. + * + * Implemented 2025 by Vadim Yanitskiy <fixeria@osmocom.org> + * SPDX-License-Identifier: CC0 + * + * As no filtering on PID etc. is done, you will likely use this with 'bpftrace -p' like + * + * bpftrace ./udp_sendmsg_delay.bt -p PID + * + * (where PID is the PID of the process, like osmo-bts-trx) + */ + +BEGIN +{ + printf("Tracing udp_sendmsg() latency... Hit Ctrl-C to end.\n"); +} + +kprobe:udp_sendmsg +{ + $sk = (struct sock *)arg0; + /* poor man's ntohs() */ + $dport = ($sk->__sk_common.skc_dport & 0xff) << 8; + $dport |= ($sk->__sk_common.skc_dport >> 8) & 0xff; + + @start[tid] = nsecs; + @dport[tid] = $dport; +} + +kretprobe:udp_sendmsg / @start[tid] / +{ + $delta_us = (nsecs - @start[tid]) / 1000; + @latency[@dport[tid]] = hist($delta_us); + + delete(@start[tid]); + delete(@dport[tid]); +} + +/* vim:set ts=2 sw=2 et: */ diff --git a/ttcn3-bts-test/jenkins.sh b/ttcn3-bts-test/jenkins.sh index 49d5cf3..24a91a5 100755 --- a/ttcn3-bts-test/jenkins.sh +++ b/ttcn3-bts-test/jenkins.sh @@ -44,8 +44,17 @@ start_bts() { local variant="$1" local sleep_time_respawn="$2" + local extra_args="" echo Starting container with BTS + if [ "$RUN_BPFTRACE" = "1" ]; then + if ! mount | grep -q /sys/kernel/tracing; then + echo "ERROR: mount /sys/kernel/tracing first!" + echo "https://www.kernel.org/doc/html/latest/trace/ftrace.html#the-file-system" + exit 1 + fi + extra_args="--ulimit memlock=-1 --privileged" + fi if [ -z "$variant" ]; then echo ERROR: You have to specify a BTS variant exit 23 @@ -55,13 +64,41 @@ --cap-add=SYS_ADMIN \ --ulimit rtprio=99 \ --ulimit core=-1 \ + -v /sys/kernel/tracing:/sys/kernel/tracing \ -v $VOL_BASE_DIR/bts:/data \ -v $VOL_BASE_DIR/unix:/data/unix \ -e "SLEEP_BEFORE_RESPAWN=$sleep_time_respawn" \ --name ${BUILD_TAG}-bts -d \ $DOCKER_ARGS \ + $extra_args \ $REPO_USER/osmo-bts-$IMAGE_SUFFIX \ /bin/sh -c "/usr/local/bin/respawn.sh osmo-bts-$variant -c /data/osmo-bts.gen.cfg >>/data/osmo-bts.log 2>&1" + + # Run bpftrace scripts (OS#6794#note-16) + if [ "$RUN_BPFTRACE" = "1" ]; then + local script + mkdir -p "$VOL_BASE_DIR"/bts/bpftrace + for script in bpftrace/*.bt; do + if ! [ -e "$script" ]; then + # Avoid unexpected behavior when there are no scripts + continue + fi + + local logfile="$VOL_BASE_DIR"/bts/bpftrace/$(basename "$script" | sed s/\.bt/.log/) + local startscript="$VOL_BASE_DIR"/bts/bpftrace/$(basename "$script" | sed s/\.bt/.sh/) + + cp "$script" "$VOL_BASE_DIR"/bts/bpftrace + + ( echo "#!/bin/sh -ex" + echo "bpftrace /data/bpftrace/$(basename "$script") -p \$(pidof osmo-bts-$variant)" ) >"$startscript" + chmod +x "$startscript" + + docker exec \ + "${BUILD_TAG}-bts" \ + /usr/local/bin/respawn.sh /data/bpftrace/"$(basename "$startscript")" \ + >>"$logfile" 2>&1 & + done + fi } start_fake_trx() { -- To view, visit https://gerrit.osmocom.org/c/docker-playground/+/41720?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: docker-playground Gerrit-Branch: master Gerrit-Change-Id: I5c22cdbb08fe797be936ca174f65ade626fa4895 Gerrit-Change-Number: 41720 Gerrit-PatchSet: 1 Gerrit-Owner: osmith <osmith@sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: fixeria <vyanitskiy@sysmocom.de> Gerrit-Reviewer: osmith <osmith@sysmocom.de> Gerrit-Reviewer: pespin <pespin@sysmocom.de>