fixeria has uploaded this change for review. ( https://gerrit.osmocom.org/c/erlang/osmo-s1gw/+/40648?usp=email ) Change subject: s1ap_proxy: implement per-eNB counters ...................................................................... s1ap_proxy: implement per-eNB counters Change-Id: Ib138a0843c783c5a21ec2646c8a671cce7e689ad --- M src/s1ap_proxy.erl M src/s1gw_metrics.erl 2 files changed, 94 insertions(+), 41 deletions(-) git pull ssh://gerrit.osmocom.org:29418/erlang/osmo-s1gw refs/changes/48/40648/1 diff --git a/src/s1ap_proxy.erl b/src/s1ap_proxy.erl index 2e2f464..c918329 100644 --- a/src/s1ap_proxy.erl +++ b/src/s1ap_proxy.erl @@ -1,4 +1,4 @@ -%% Copyright (C) 2024 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de> +%% Copyright (C) 2024-2025 by sysmocom - s.f.m.c. GmbH <info@sysmocom.de> %% Author: Vadim Yanitskiy <vyanitskiy@sysmocom.de> %% %% All Rights Reserved @@ -237,10 +237,12 @@ parse_enb_id({'long-macroENB-ID', << ID:21 >>}) -> ID. --spec genb_id_str(proxy_state()) -> string(). +-spec genb_id_str(proxy_state()) -> string() | none. genb_id_str(#proxy_state{plmn_id = {MCC, MNC}, enb_id = ENBId}) -> - MCC ++ "-" ++ MNC ++ "-" ++ integer_to_list(ENBId). + MCC ++ "-" ++ MNC ++ "-" ++ integer_to_list(ENBId); + +genb_id_str(#proxy_state{}) -> none. -spec tla_str(binary()) -> string(). @@ -249,6 +251,43 @@ inet:ntoa(TLA1). +%% register a single per-eNB counter +-spec ctr_reg(C, GlobalENBId) -> C + when C :: [ctr | _], + GlobalENBId :: string(). +ctr_reg([ctr, s1ap, proxy | _] = C0, GlobalENBId) -> + C1 = s1gw_metrics:enb_metric(C0, GlobalENBId), + %% counter may already exist, so catch exceptions here + catch exometer:new(C1, counter), + C0; + +ctr_reg(C0, _GlobalENBId) -> C0. + + +%% register all per-eNB counters +-spec ctr_reg_all(string()) -> list(). +ctr_reg_all(GlobalENBId) -> + Ctrs = s1gw_metrics:ctr_list(), + lists:map(fun(Name) -> ctr_reg(Name, GlobalENBId) end, Ctrs). + + +%% increment the global and/or per-eNB counters +-spec ctr_inc(C0, S) -> term() + when C0 :: [ctr | _], + S :: proxy_state(). +ctr_inc([ctr | _] = C0, S) -> + case genb_id_str(S) of + none -> + %% increment the global counter only + s1gw_metrics:ctr_inc(C0); + GlobalENBId -> + %% increment both the global and per-eNB counters + C1 = s1gw_metrics:enb_metric(C0, GlobalENBId), + s1gw_metrics:ctr_inc(C0), + s1gw_metrics:ctr_inc(C1) + end. + + %% Encode an S1AP PDU -spec encode_pdu(s1ap_pdu()) -> {ok, binary()} | {error, {asn1, tuple()}}. @@ -266,7 +305,7 @@ %% Process an S1AP PDU (binary) -spec handle_pdu_bin(binary(), proxy_state()) -> {process_pdu_result(), proxy_state()}. handle_pdu_bin(OrigData, S0) -> - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ALL), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ALL, S0), try decode_pdu(OrigData) of {ok, PDU} -> ?LOG_DEBUG("Rx S1AP PDU: ~p", [PDU]), @@ -276,36 +315,36 @@ ?LOG_DEBUG("Tx (~p) S1AP PDU: ~p", [Action, NewPDU]), case Action of forward -> - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_ALL), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_PROC); + ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_ALL, S1), + ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_PROC, S1); reply -> - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_REPLY_ALL) + ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_REPLY_ALL, S1) end, {{Action, NewData}, S1}; {forward, S1} -> ?LOG_DEBUG("Tx (forward) S1AP PDU unmodified"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_ALL), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_UNMODIFIED), + ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_ALL, S1), + ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_UNMODIFIED, S1), {{forward, OrigData}, S1} catch Exception:Reason:StackTrace -> ?LOG_ERROR("An exception occurred: ~p, ~p, ~p", [Exception, Reason, StackTrace]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_EXCEPTION), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_UNMODIFIED), + ctr_inc(?S1GW_CTR_S1AP_PROXY_EXCEPTION, S0), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR, S0), + ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_UNMODIFIED, S0), {{forward, OrigData}, S0} %% XXX: proxy as-is or drop? end; {error, {asn1, Error}} -> ?LOG_ERROR("S1AP PDU decoding failed: ~p", [Error]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_DECODE_ERROR), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_UNMODIFIED), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_DECODE_ERROR, S0), + ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_UNMODIFIED, S0), {{forward, OrigData}, S0} %% XXX: forward as-is or drop? catch Exception:Reason:StackTrace -> ?LOG_ERROR("An exception occurred: ~p, ~p, ~p", [Exception, Reason, StackTrace]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_EXCEPTION), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_DECODE_ERROR), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_UNMODIFIED), + ctr_inc(?S1GW_CTR_S1AP_PROXY_EXCEPTION, S0), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_DECODE_ERROR, S0), + ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_UNMODIFIED, S0), {{forward, OrigData}, S0} %% XXX: proxy as-is or drop? end. @@ -339,7 +378,7 @@ #'InitiatingMessage'{procedureCode = ?'id-E-RABSetup', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing E-RAB SETUP REQUEST"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_SETUP_REQ), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_SETUP_REQ, S0), case handle_ies(?'id-E-RABToBeSetupListBearerSUReq', C0#'E-RABSetupRequest'.protocolIEs, S0) of {{ok, IEs}, S1} -> @@ -348,9 +387,9 @@ {{forward, PDU}, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process E-RAB SETUP REQUEST: ~p", [Reason]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR, S1), PDU = build_erab_setup_response_failure(S1), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_REPLY_ERAB_SETUP_RSP), + ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_REPLY_ERAB_SETUP_RSP, S1), {{reply, PDU}, S1} %% reply PDU back to sender end; @@ -359,7 +398,7 @@ #'SuccessfulOutcome'{procedureCode = ?'id-E-RABSetup', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing E-RAB SETUP RESPONSE"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_SETUP_RSP), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_SETUP_RSP, S0), case handle_ies(?'id-E-RABSetupListBearerSURes', C0#'E-RABSetupResponse'.protocolIEs, S0) of {{ok, IEs}, S1} -> @@ -368,7 +407,7 @@ {{forward, PDU}, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process E-RAB SETUP RESPONSE: ~p", [Reason]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR, S1), {forward, S1} %% XXX: forward as-is or drop? end; @@ -377,7 +416,7 @@ #'InitiatingMessage'{procedureCode = ?'id-E-RABModify', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing E-RAB MODIFY REQUEST"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_MODIFY_REQ), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_MODIFY_REQ, S0), case handle_ies(?'id-E-RABToBeModifiedListBearerModReq', C0#'E-RABModifyRequest'.protocolIEs, S0) of {{ok, IEs}, S1} -> @@ -386,7 +425,7 @@ {{forward, PDU}, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process E-RAB MODIFY REQUEST: ~p", [Reason]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR, S1), {forward, S1} %% XXX: forward as-is or drop? end; @@ -395,7 +434,7 @@ #'SuccessfulOutcome'{procedureCode = ?'id-E-RABModify', value = C0}}, S0) -> ?LOG_DEBUG("Processing E-RAB MODIFY RESPONSE"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_MODIFY_RSP), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_MODIFY_RSP, S0), %% there's nothing to patch in this PDU, so we forward it as-is %% TODO: check result of handle_ies(), inc. ?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR {_, S1} = handle_ies(?'id-E-RABModifyListBearerModRes', @@ -409,7 +448,7 @@ #'InitiatingMessage'{procedureCode = ?'id-E-RABRelease', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing E-RAB RELEASE COMMAND"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_RELEASE_CMD), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_RELEASE_CMD, S0), case handle_ies(?'id-E-RABToBeReleasedList', C0#'E-RABReleaseCommand'.protocolIEs, S0) of {{ok, IEs}, S1} -> @@ -418,7 +457,7 @@ {{forward, PDU}, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process E-RAB RELEASE COMMAND: ~p", [Reason]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR, S1), {forward, S1} %% XXX: forward as-is or drop? end; @@ -427,7 +466,7 @@ #'SuccessfulOutcome'{procedureCode = ?'id-E-RABRelease', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing E-RAB RELEASE RESPONSE"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_RELEASE_RSP), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_RELEASE_RSP, S0), case handle_ies(?'id-E-RABReleaseListBearerRelComp', C0#'E-RABReleaseResponse'.protocolIEs, S0) of {{ok, IEs}, S1} -> @@ -436,7 +475,7 @@ {{forward, PDU}, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process E-RAB RELEASE RESPONSE: ~p", [Reason]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR, S1), {forward, S1} %% XXX: forward as-is or drop? end; @@ -445,7 +484,7 @@ #'InitiatingMessage'{procedureCode = ?'id-E-RABReleaseIndication', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing E-RAB RELEASE INDICATION"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_RELEASE_IND), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_RELEASE_IND, S0), case handle_ies(?'id-E-RABReleasedList', C0#'E-RABReleaseIndication'.protocolIEs, S0) of {{ok, IEs}, S1} -> @@ -454,7 +493,7 @@ {{forward, PDU}, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process E-RAB RELEASE INDICATION: ~p", [Reason]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR, S1), {forward, S1} %% XXX: forward as-is or drop? end; @@ -463,7 +502,7 @@ #'InitiatingMessage'{procedureCode = ?'id-E-RABModificationIndication', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing E-RAB MODIFICATION INDICATION"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_MOD_IND), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_MOD_IND, S0), IEs0 = C0#'E-RABModificationIndication'.protocolIEs, %% E-RAB to be Modified List %% TODO: handle {error, Reason} @@ -480,7 +519,7 @@ #'SuccessfulOutcome'{procedureCode = ?'id-E-RABModificationIndication', value = C0}}, S0) -> ?LOG_DEBUG("Processing E-RAB MODIFICATION CONFIRM"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_MOD_CNF), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_MOD_CNF, S0), IEs = C0#'E-RABModificationConfirm'.protocolIEs, %% E-RAB Modify List %% TODO: handle {error, Reason} @@ -499,7 +538,7 @@ #'InitiatingMessage'{procedureCode = ?'id-InitialContextSetup', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing INITIAL CONTEXT SETUP REQUEST"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_INIT_CTX_REQ), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_INIT_CTX_REQ, S0), case handle_ies(?'id-E-RABToBeSetupListCtxtSUReq', C0#'InitialContextSetupRequest'.protocolIEs, S0) of {{ok, IEs}, S1} -> @@ -508,7 +547,7 @@ {{forward, PDU}, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process INITIAL CONTEXT SETUP REQUEST: ~p", [Reason]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR, S1), {forward, S1} %% XXX: forward as-is or drop? end; @@ -517,7 +556,7 @@ #'SuccessfulOutcome'{procedureCode = ?'id-InitialContextSetup', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing INITIAL CONTEXT SETUP RESPONSE"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_INIT_CTX_RSP), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_INIT_CTX_RSP, S0), %% TODO: handle optional E-RAB Failed to Setup List IE case handle_ies(?'id-E-RABSetupListCtxtSURes', C0#'InitialContextSetupResponse'.protocolIEs, S0) of @@ -527,7 +566,7 @@ {{forward, PDU}, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process INITIAL CONTEXT SETUP RESPONSE: ~p", [Reason]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR, S1), {forward, S1} %% XXX: forward as-is or drop? end; @@ -537,7 +576,7 @@ value = #'UEContextReleaseRequest'{protocolIEs = IEs}}}, #proxy_state{erabs = ERABs} = S) -> ?LOG_DEBUG("Processing UE CONTEXT RELEASE REQUEST"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_RELEASE_CTX_REQ), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_RELEASE_CTX_REQ, S), %% fetch {MME,eNB}-UE-S1AP-ID values (mandatory IEs) #'ProtocolIE-Field'{id = ?'id-MME-UE-S1AP-ID', value = MMEUEId} = lists:nth(1, IEs), @@ -555,7 +594,7 @@ value = #'UEContextReleaseCommand'{protocolIEs = IEs}}}, #proxy_state{erabs = ERABs} = S) -> ?LOG_DEBUG("Processing UE CONTEXT RELEASE COMMAND"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_RELEASE_CTX_CMD), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_RELEASE_CTX_CMD, S), #'ProtocolIE-Field'{id = ?'id-UE-S1AP-IDs', value = S1APIDs} = lists:nth(1, IEs), case S1APIDs of @@ -579,7 +618,7 @@ value = #'UEContextReleaseComplete'{protocolIEs = IEs}}}, #proxy_state{erabs = ERABs} = S) -> ?LOG_DEBUG("Processing UE CONTEXT RELEASE COMPLETE"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_RELEASE_CTX_COMPL), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_RELEASE_CTX_COMPL, S), %% fetch {MME,eNB}-UE-S1AP-ID values (mandatory IEs) #'ProtocolIE-Field'{id = ?'id-MME-UE-S1AP-ID', value = MMEUEId} = lists:nth(1, IEs), @@ -622,7 +661,10 @@ [S1#proxy_state.plmn_id, S1#proxy_state.enb_id]), %% use that as a context for logging - set_logging_prefix("eNB " ++ genb_id_str(S1)), + GlobalENBId = genb_id_str(S1), + set_logging_prefix("eNB " ++ GlobalENBId), + %% register per-eNB metrics + ctr_reg_all(GlobalENBId), {{ok, C}, S1}; %% E-RAB SETUP REQUEST related IEs diff --git a/src/s1gw_metrics.erl b/src/s1gw_metrics.erl index 00e603d..361269c 100644 --- a/src/s1gw_metrics.erl +++ b/src/s1gw_metrics.erl @@ -35,6 +35,7 @@ -module(s1gw_metrics). -export([init/0, + enb_metric/2, ctr_reset/1, ctr_inc/1, ctr_inc/2, @@ -99,6 +100,7 @@ -type counter() :: [ctr | _]. -type gauge() :: [gauge | _]. +-type metric() :: counter() | gauge(). -spec register_all(Type, List) -> Result @@ -126,6 +128,15 @@ [] = register_all(counter, ?S1GW_COUNTERS), [] = register_all(gauge, ?S1GW_GAUGES). + +-spec enb_metric(M0, GlobalENBId) -> M1 + when M0 :: metric(), + M1 :: metric(), + GlobalENBId :: string(). +enb_metric([Type | Name], GlobalENBId) -> + [Type, enb, GlobalENBId | Name]. + + %%%%%%%%%%%%% %% CTR APIs %%%%%%%%%%%%% -- To view, visit https://gerrit.osmocom.org/c/erlang/osmo-s1gw/+/40648?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: newchange Gerrit-Project: erlang/osmo-s1gw Gerrit-Branch: master Gerrit-Change-Id: Ib138a0843c783c5a21ec2646c8a671cce7e689ad Gerrit-Change-Number: 40648 Gerrit-PatchSet: 1 Gerrit-Owner: fixeria <vyanitskiy@sysmocom.de>