[L] Change in osmo-hnbgw[master]: per-HNB GTP-U traffic counters via nft

neels has uploaded this change for review. ( https://gerrit.osmocom.org/c/osmo-hnbgw/+/36385?usp=email ) Change subject: per-HNB GTP-U traffic counters via nft ...................................................................... per-HNB GTP-U traffic counters via nft Add external dependency libnftables. When an hNodeB registers, set up netfilter rules to count GTP-U packets (UDP port 2152) to and from that hNodeB's address -- we are assuming that it is the same address that Iuh is connecting from. This is a "workaround" to get performance indicators per hNodeB, without needing a UPF that supports URR. Related: SYS#6773 Change-Id: I35b7e97fd039e36633dfde1317170527c82f9f68 --- M configure.ac M include/osmocom/hnbgw/hnbgw.h A include/osmocom/hnbgw/nft_kpi.h M src/osmo-hnbgw/Makefile.am M src/osmo-hnbgw/hnbgw.c M src/osmo-hnbgw/hnbgw_hnbap.c M src/osmo-hnbgw/hnbgw_vty.c A src/osmo-hnbgw/nft_kpi.c M src/osmo-hnbgw/tdefs.c 9 files changed, 447 insertions(+), 2 deletions(-) git pull ssh://gerrit.osmocom.org:29418/osmo-hnbgw refs/changes/85/36385/1 diff --git a/configure.ac b/configure.ac index bb8d29e..bea9a76 100644 --- a/configure.ac +++ b/configure.ac @@ -59,6 +59,7 @@ PKG_CHECK_MODULES(LIBOSMORANAP, libosmo-ranap >= 1.5.0) PKG_CHECK_MODULES(LIBOSMOHNBAP, libosmo-hnbap >= 1.5.0) PKG_CHECK_MODULES(LIBOSMOMGCPCLIENT, libosmo-mgcp-client >= 1.12.0) +PKG_CHECK_MODULES(LIBNFTABLES, libnftables >= 1.0.2) # Enable PFCP support for GTP tunnel mapping via UPF AC_ARG_ENABLE([pfcp], [AS_HELP_STRING([--enable-pfcp], [Build with PFCP support, for GTP tunnel mapping via UPF])], diff --git a/include/osmocom/hnbgw/hnbgw.h b/include/osmocom/hnbgw/hnbgw.h index b3ba3b2..fd36168 100644 --- a/include/osmocom/hnbgw/hnbgw.h +++ b/include/osmocom/hnbgw/hnbgw.h @@ -6,6 +6,7 @@ #include <osmocom/core/write_queue.h> #include <osmocom/core/timer.h> #include <osmocom/core/rate_ctr.h> +#include <osmocom/core/sockaddr_str.h> #include <osmocom/gsm/gsm23003.h> #include <osmocom/sigtran/sccp_sap.h> #include <osmocom/sigtran/osmo_ss7.h> @@ -18,6 +19,8 @@ #include <osmocom/mgcp_client/mgcp_client.h> #include <osmocom/mgcp_client/mgcp_client_pool.h> +#include <osmocom/hnbgw/nft_kpi.h> + #define STORE_UPTIME_INTERVAL 10 /* seconds */ #define HNB_STORE_RAB_DURATIONS_INTERVAL 1 /* seconds */ @@ -29,6 +32,7 @@ DMGW, DHNB, DCN, + DNFT, }; extern const struct log_info hnbgw_log_info; @@ -132,6 +136,11 @@ HNB_CTR_CS_PAGING_ATTEMPTED, HNB_CTR_RAB_ACTIVE_MILLISECONDS_TOTAL, + + HNB_CTR_GTPU_DOWNLOAD_PACKETS, + HNB_CTR_GTPU_DOWNLOAD_GTP_BYTES, + HNB_CTR_GTPU_UPLOAD_PACKETS, + HNB_CTR_GTPU_UPLOAD_GTP_BYTES, }; enum hnb_stat { @@ -353,6 +362,14 @@ struct rate_ctr_group *ctrs; struct osmo_stat_item_group *statg; + + struct { + struct osmo_sockaddr_str addr_remote; + struct { + struct nft_kpi_val rx; + struct nft_kpi_val tx; + } last; + } nft_kpi; }; struct ue_context { @@ -450,7 +467,9 @@ void hnb_context_release_ue_state(struct hnb_context *ctx); struct hnb_persistent *hnb_persistent_alloc(const struct umts_cell_id *id); -struct hnb_persistent *hnb_persistent_find_by_id(const struct umts_cell_id *id); +struct hnb_persistent *hnb_persistent_find_by_id(const struct umts_cell_id *id_str); +struct hnb_persistent *hnb_persistent_find_by_id_str(const char *id); +void hnb_persistent_update_addr(struct hnb_persistent *hnbp, int new_fd); void hnb_persistent_free(struct hnb_persistent *hnbp); void hnbgw_vty_init(void); diff --git a/include/osmocom/hnbgw/nft_kpi.h b/include/osmocom/hnbgw/nft_kpi.h new file mode 100644 index 0000000..d56946f --- /dev/null +++ b/include/osmocom/hnbgw/nft_kpi.h @@ -0,0 +1,14 @@ +#pragma once +#include <stdint.h> + +struct hnb_persistent; + +struct nft_kpi_val { + uint64_t packets; + uint64_t bytes; +}; + +int hnb_nft_kpi_start(struct hnb_persistent *hnbp); +int hnb_nft_kpi_end(struct hnb_persistent *hnbp); + +const char *nft_kpi_read_counters(void); diff --git a/src/osmo-hnbgw/Makefile.am b/src/osmo-hnbgw/Makefile.am index 0727f30..1ee6e44 100644 --- a/src/osmo-hnbgw/Makefile.am +++ b/src/osmo-hnbgw/Makefile.am @@ -20,6 +20,7 @@ $(LIBOSMORANAP_CFLAGS) \ $(LIBOSMOHNBAP_CFLAGS) \ $(LIBOSMOMGCPCLIENT_CFLAGS) \ + $(LIBNFTABLES_CFLAGS) \ $(NULL) AM_LDFLAGS = \ @@ -46,6 +47,7 @@ mgw_fsm.c \ kpi_ranap.c \ tdefs.c \ + nft_kpi.c \ $(NULL) libhnbgw_la_LIBADD = \ @@ -62,6 +64,7 @@ $(LIBOSMOHNBAP_LIBS) \ $(LIBSCTP_LIBS) \ $(LIBOSMOMGCPCLIENT_LIBS) \ + $(LIBNFTABLES_LIBS) \ $(NULL) if ENABLE_PFCP diff --git a/src/osmo-hnbgw/hnbgw.c b/src/osmo-hnbgw/hnbgw.c index fff900a..1e27ee2 100644 --- a/src/osmo-hnbgw/hnbgw.c +++ b/src/osmo-hnbgw/hnbgw.c @@ -340,8 +340,10 @@ } /* remove back reference from hnb_persistent to context */ - if (ctx->persistent) + if (ctx->persistent) { + hnb_nft_kpi_end(ctx->persistent); ctx->persistent->ctx = NULL; + } talloc_free(ctx); } @@ -455,6 +457,24 @@ [HNB_CTR_RAB_ACTIVE_MILLISECONDS_TOTAL] = { "rab:cs:active_milliseconds:total", "Cumulative number of milliseconds of CS RAB activity" }, + + [HNB_CTR_GTPU_DOWNLOAD_PACKETS] = { + "gtpu:dl:packets", + "Count of total GTP-U packets sent to the HNB.", + }, + [HNB_CTR_GTPU_DOWNLOAD_GTP_BYTES] = { + "gtpu:dl:gtpbytes", + "Count of total GTP-U bytes sent to the HNB, including the GTP-U header.", + }, + [HNB_CTR_GTPU_UPLOAD_PACKETS] = { + "gtpu:ul:packets", + "Count of total GTP-U packets received from the HNB.", + }, + [HNB_CTR_GTPU_UPLOAD_GTP_BYTES] = { + "gtpu:ul:gtpbytes", + "Count of total GTP-U bytes received from the HNB, including the GTP-U header.", + }, + }; const struct rate_ctr_group_desc hnb_ctrg_desc = { @@ -517,9 +537,49 @@ return NULL; } +struct hnb_persistent *hnb_persistent_find_by_id_str(const char *id_str) +{ + struct hnb_persistent *hnbp; + llist_for_each_entry(hnbp, &g_hnbgw->hnb_persistent_list, list) { + if (strcmp(hnbp->id_str, id_str)) + continue; + return hnbp; + } + return NULL; +} + +void hnb_persistent_update_addr(struct hnb_persistent *hnbp, int new_fd) +{ + int rc; + socklen_t socklen; + struct osmo_sockaddr osa; + struct osmo_sockaddr_str remote_str; + + socklen = sizeof(struct osmo_sockaddr); + rc = getpeername(new_fd, &osa.u.sa, &socklen); + if (!rc) + rc = osmo_sockaddr_str_from_osa(&remote_str, &osa); + if (rc < 0) { + LOGP(DHNB, LOGL_ERROR, "cannot read remote hNodeB address from Iuh file descriptor\n"); + return; + } + + if (!osmo_sockaddr_str_cmp(&remote_str, &hnbp->nft_kpi.addr_remote)) { + /* The remote address is unchanged, no need to update the nft probe */ + return; + } + + /* The remote address has changed. Cancel previous probe, if any, and start a new one. */ + if (osmo_sockaddr_str_is_nonzero(&hnbp->nft_kpi.addr_remote)) + hnb_nft_kpi_end(hnbp); + hnbp->nft_kpi.addr_remote = remote_str; + hnb_nft_kpi_start(hnbp); +} + void hnb_persistent_free(struct hnb_persistent *hnbp) { /* FIXME: check if in use? */ + hnb_nft_kpi_end(hnbp); llist_del(&hnbp->list); talloc_free(hnbp); } @@ -845,6 +905,11 @@ .color = OSMO_LOGCOLOR_DARKYELLOW, .description = "Core Network side (via SCCP)", }, + [DNFT] = { + .name = "DNFT", .loglevel = LOGL_NOTICE, .enabled = 1, + .color = OSMO_LOGCOLOR_BLUE, + .description = "netfilter counters for per-HNB traffic counters", + }, }; const struct log_info hnbgw_log_info = { diff --git a/src/osmo-hnbgw/hnbgw_hnbap.c b/src/osmo-hnbgw/hnbgw_hnbap.c index b2ff911..4a2e1b9 100644 --- a/src/osmo-hnbgw/hnbgw_hnbap.c +++ b/src/osmo-hnbgw/hnbgw_hnbap.c @@ -494,6 +494,8 @@ ctx->hnb_registered = true; + hnb_persistent_update_addr(ctx->persistent, osmo_stream_srv_get_fd(ctx->conn)); + /* Send HNBRegisterAccept */ rc = hnbgw_tx_hnb_register_acc(ctx); hnbap_free_hnbregisterrequesties(&ies); diff --git a/src/osmo-hnbgw/hnbgw_vty.c b/src/osmo-hnbgw/hnbgw_vty.c index c5af249..832ffb6 100644 --- a/src/osmo-hnbgw/hnbgw_vty.c +++ b/src/osmo-hnbgw/hnbgw_vty.c @@ -35,6 +35,7 @@ #include <osmocom/hnbgw/hnbgw_cn.h> #include <osmocom/hnbgw/context_map.h> #include <osmocom/hnbgw/tdefs.h> +#include <osmocom/hnbgw/nft_kpi.h> #include <osmocom/sigtran/protocol/sua.h> #include <osmocom/sigtran/sccp_helpers.h> #include <osmocom/netif/stream.h> @@ -741,6 +742,14 @@ return CMD_SUCCESS; } +DEFUN(show_nft, show_nft_cmd, + "show nft", + SHOW_STR "nft counters") +{ + vty_out(vty, "%s%s", nft_kpi_read_counters(), VTY_NEWLINE); + return CMD_SUCCESS; +} + /* Hidden since it exists only for use by ttcn3 tests */ DEFUN_HIDDEN(cnpool_roundrobin_next, cnpool_roundrobin_next_cmd, "cnpool roundrobin next (msc|sgsn) " CNLINK_NR_RANGE, @@ -1164,4 +1173,6 @@ install_element_ve(&show_nri_cmd); install_element(ENABLE_NODE, &cnpool_roundrobin_next_cmd); install_element(ENABLE_NODE, &cnlink_ranap_reset_cmd); + + install_element_ve(&show_nft_cmd); } diff --git a/src/osmo-hnbgw/nft_kpi.c b/src/osmo-hnbgw/nft_kpi.c new file mode 100644 index 0000000..3360bcb --- /dev/null +++ b/src/osmo-hnbgw/nft_kpi.c @@ -0,0 +1,310 @@ +/* Set up and read internet traffic counters using netfilter */ +/* Copyright (C) 2024 by sysmocom - s.f.m.c. GmbH &lt;info(a)sysmocom.de&gt; + * All Rights Reserved + * + * Author: Neels Janosch Hofmeyr &lt;nhofmeyr(a)sysmocom.de&gt; + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + */ + +#include <stdbool.h> +#include <ctype.h> + +#include <nftables/libnftables.h> + +#include <osmocom/core/logging.h> +#include <osmocom/core/timer.h> +#include <osmocom/hnbgw/nft_kpi.h> +#include <osmocom/hnbgw/hnbgw.h> +#include <osmocom/hnbgw/tdefs.h> + +struct nft_kpi_state { + struct { + struct nft_ctx *nft_ctx; + char *table_name; + bool table_created; + } nft; + struct osmo_timer_list period; +}; + +static struct nft_kpi_state g_nft_kpi_state = {}; + +static struct nft_ctx *g_nft_ctx(void) +{ + struct nft_kpi_state *s = &g_nft_kpi_state; + + if (s->nft.nft_ctx) + return s->nft.nft_ctx; + + s->nft.nft_ctx = nft_ctx_new(NFT_CTX_DEFAULT); + if (!s->nft.nft_ctx) { + LOGP(DNFT, LOGL_ERROR, "cannot allocate libnftables nft_ctx\n"); + OSMO_ASSERT(false); + } + + return s->nft.nft_ctx; +} + +static int nft_run_now(const char *buffer) +{ + int rc; + const int logmax = 256; + + rc = nft_run_cmd_from_buffer(g_nft_ctx(), buffer); + if (rc < 0) { + LOGP(DNFT, LOGL_ERROR, "error running nft buffer: rc=%d buffer=%s\n", + rc, osmo_quote_str_c(OTC_SELECT, buffer, -1)); + return -EIO; + } + + if (log_check_level(DNFT, LOGL_DEBUG)) { + size_t l = strlen(buffer); + LOGP(DNFT, LOGL_DEBUG, "ran nft buffer, %zu chars: \"%s%s\"\n", + l, + osmo_escape_cstr_c(OTC_SELECT, buffer, OSMO_MIN(logmax, l)), + l > logmax ? "..." : ""); + } + + return 0; +} + +static void nft_kpi_period_cb(void *data); + +static void nft_kpi_period_schedule(void) +{ + unsigned long period = osmo_tdef_get(hnbgw_T_defs, -34, OSMO_TDEF_S, 10); + if (period < 1) + period = 1; + osmo_timer_setup(&g_nft_kpi_state.period, nft_kpi_period_cb, NULL); + osmo_timer_schedule(&g_nft_kpi_state.period, period, 0); +} + +static int nft_kpi_init(void) +{ + struct nft_kpi_state *s = &g_nft_kpi_state; + + if (s->nft.table_created) + return 0; + + if (!s->nft.table_name || !*s->nft.table_name) + s->nft.table_name = talloc_strdup(g_hnbgw, "osmo-hnbgw"); + + if (nft_run_now(talloc_asprintf(OTC_SELECT, "add table inet %s { flags owner; };\n", s->nft.table_name))) + return -EIO; + + s->nft.table_created = true; + nft_kpi_period_schedule(); + return 0; +} + +/* Set up counters for the hNodeB's remote address */ +int hnb_nft_kpi_start(struct hnb_persistent *hnbp) +{ + struct nft_kpi_state *s = &g_nft_kpi_state; + char *cmd; + + nft_kpi_init(); + + hnbp->nft_kpi.last.rx = (struct nft_kpi_val){}; + hnbp->nft_kpi.last.tx = (struct nft_kpi_val){}; + + cmd = talloc_asprintf(OTC_SELECT, + "add chain inet %s hnb-rx-%s {" + " type filter hook input priority filter;" + " ip protocol udp udp sport 2152 udp dport 2152 ip saddr %s counter;" + "};\n" + "add chain inet %s hnb-tx-%s {" + " type filter hook output priority filter;" + " ip protocol udp udp sport 2152 udp dport 2152 ip daddr %s counter;" + "};\n", + s->nft.table_name, + hnbp->id_str, + hnbp->nft_kpi.addr_remote.ip, + s->nft.table_name, + hnbp->id_str, + hnbp->nft_kpi.addr_remote.ip); + return nft_run_now(cmd); +} + +int hnb_nft_kpi_end(struct hnb_persistent *hnbp) +{ + struct nft_kpi_state *s = &g_nft_kpi_state; + char *cmd; + + if (!s->nft.table_created) + return 0; + + if (!osmo_sockaddr_str_is_nonzero(&hnbp->nft_kpi.addr_remote)) + return 0; + hnbp->nft_kpi.addr_remote = (struct osmo_sockaddr_str){}; + + cmd = talloc_asprintf(OTC_SELECT, + "delete chain inet %s hnb-rx-%s;\n" + "delete chain inet %s hnb-tx-%s;\n", + s->nft.table_name, + hnbp->id_str, + s->nft.table_name, + hnbp->id_str); + return nft_run_now(cmd); +} + +static void update_ctr(struct rate_ctr_group *cg, int cgidx, uint64_t *last_val, uint64_t new_val) +{ + if (new_val <= *last_val) + return; + rate_ctr_add2(cg, cgidx, new_val - *last_val); + *last_val = new_val; +} + +static void hnb_update_counters(struct hnb_persistent *hnbp, bool rx, int64_t packets, int64_t bytes) +{ + update_ctr(hnbp->ctrs, + rx ? HNB_CTR_GTPU_UPLOAD_PACKETS : HNB_CTR_GTPU_DOWNLOAD_PACKETS, + rx ? &hnbp->nft_kpi.last.rx.packets : &hnbp->nft_kpi.last.tx.packets, + packets); + update_ctr(hnbp->ctrs, + rx ? HNB_CTR_GTPU_UPLOAD_GTP_BYTES : HNB_CTR_GTPU_DOWNLOAD_GTP_BYTES, + rx ? &hnbp->nft_kpi.last.rx.bytes : &hnbp->nft_kpi.last.tx.bytes, + bytes); +} + +const char *nft_kpi_read_counters(void) +{ + int rc; + const int logmax = 256; + struct nft_kpi_state *s = &g_nft_kpi_state; + struct nft_ctx *nft = s->nft.nft_ctx; + char *cmd; + const char *output = NULL; + const char *pos; + char buf[128]; + + if (!nft) + return NULL; + + cmd = talloc_asprintf(OTC_SELECT, "list table inet %s", s->nft.table_name); + + rc = nft_ctx_buffer_output(nft); + if (rc) { + LOGP(DNFT, LOGL_ERROR, "error: nft_ctx_buffer_output() returned failure: rc=%d cmd=%s\n", + rc, osmo_quote_str_c(OTC_SELECT, cmd, -1)); + goto unbuffer_and_exit; + } + rc = nft_run_cmd_from_buffer(nft, cmd); + if (rc < 0) { + LOGP(DNFT, LOGL_ERROR, "error running nft cmd: rc=%d cmd=%s\n", + rc, osmo_quote_str_c(OTC_SELECT, cmd, -1)); + goto unbuffer_and_exit; + } + if (log_check_level(DNFT, LOGL_DEBUG)) { + size_t l = strlen(cmd); + LOGP(DNFT, LOGL_DEBUG, "ran nft request, %zu chars: \"%s%s\"\n", + l, + osmo_escape_cstr_c(OTC_SELECT, cmd, OSMO_MIN(logmax, l)), + l > logmax ? "..." : ""); + } + + output = nft_ctx_get_output_buffer(nft); + + pos = output; + while (*pos) { + const char *id, *id_end; + const char *chain_end; + const char *counter; + const char *counter_end; + int64_t packets; + int64_t bytes; + bool rx; + struct hnb_persistent *hnbp; + + id = strstr(pos, "chain hnb-"); + if (!id) + break; + id += 10; + + if (osmo_str_startswith(id, "rx-")) + rx = true; + else if (osmo_str_startswith(id, "tx-")) + rx = false; + else + break; + id += 3; + + id_end = id; + while (*id_end && *id_end != ' ' && *id_end != '{') + id_end++; + + osmo_strlcpy(buf, id, OSMO_MIN(sizeof(buf), id_end - id + 1)); + + hnbp = hnb_persistent_find_by_id_str(buf); + if (!hnbp) + break; + + if (!osmo_str_startswith(id_end, " {")) + break; + chain_end = id_end + 2; + while (*chain_end && *chain_end != '}') + chain_end++; + + counter = strstr(id_end + 2, "counter packets "); + if (!counter) + break; + counter += 16; + if (counter > chain_end) + break; + if (!isdigit(*counter)) + break; + + counter_end = counter; + while (isdigit(*counter_end)) + counter_end++; + if (counter_end > chain_end) + break; + osmo_strlcpy(buf, counter, OSMO_MIN(sizeof(buf), counter_end - counter + 1)); + if (osmo_str_to_int64(&packets, buf, 10, 0, INT64_MAX)) + break; + if (packets < 0) + break; + + counter = strstr(counter_end, " bytes "); + if (!counter) + break; + counter += 7; + if (counter > chain_end) + break; + if (!isdigit(*counter)) + break; + counter_end = counter; + while (isdigit(*counter_end)) + counter_end++; + if (counter_end > chain_end) + break; + osmo_strlcpy(buf, counter, OSMO_MIN(sizeof(buf), counter_end - counter + 1)); + if (osmo_str_to_int64(&bytes, buf, 10, 0, INT64_MAX)) + break; + if (bytes < 0) + break; + + hnb_update_counters(hnbp, rx, packets, bytes); + + pos = chain_end + 1; + } + +unbuffer_and_exit: + nft_ctx_unbuffer_output(nft); + return output; +} + +static void nft_kpi_period_cb(void *data) +{ + nft_kpi_read_counters(); + nft_kpi_period_schedule(); +} diff --git a/src/osmo-hnbgw/tdefs.c b/src/osmo-hnbgw/tdefs.c index af09d17..bb54049 100644 --- a/src/osmo-hnbgw/tdefs.c +++ b/src/osmo-hnbgw/tdefs.c @@ -36,6 +36,7 @@ {.T = 4, .default_val = 5, .desc = "Timeout to receive RANAP RESET ACKNOWLEDGE from an MSC/SGSN" }, {.T = -31, .default_val = 15, .desc = "Timeout for establishing and releasing context maps (RUA <-> SCCP)" }, {.T = -1002, .default_val = 10, .desc = "Timeout for the HNB to respond to PS RAB Assignment Request" }, + {.T = -34, .default_val = 10, .desc = "Period to query network traffic stats from netfilter" }, { } }; -- To view, visit https://gerrit.osmocom.org/c/osmo-hnbgw/+/36385?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-hnbgw Gerrit-Branch: master Gerrit-Change-Id: I35b7e97fd039e36633dfde1317170527c82f9f68 Gerrit-Change-Number: 36385 Gerrit-PatchSet: 1 Gerrit-Owner: neels &lt;nhofmeyr(a)sysmocom.de&gt; Gerrit-MessageType: newchange