fixeria has submitted this change. ( https://gerrit.osmocom.org/c/erlang/osmo-s1gw/+/40648?usp=email ) Change subject: s1ap_proxy: implement per-eNB counters ...................................................................... s1ap_proxy: implement per-eNB counters Change-Id: Ib138a0843c783c5a21ec2646c8a671cce7e689ad Related: SYS#7065 --- M src/s1ap_proxy.erl M src/s1gw_metrics.erl 2 files changed, 92 insertions(+), 41 deletions(-) Approvals: Jenkins Builder: Verified laforge: Looks good to me, approved pespin: Looks good to me, but someone else must approve diff --git a/src/s1ap_proxy.erl b/src/s1ap_proxy.erl index 3073370..f2fd722 100644 --- a/src/s1ap_proxy.erl +++ b/src/s1ap_proxy.erl @@ -1,4 +1,4 @@ -%% Copyright (C) 2024 by sysmocom - s.f.m.c. GmbH &lt;info(a)sysmocom.de&gt; +%% Copyright (C) 2024-2025 by sysmocom - s.f.m.c. GmbH &lt;info(a)sysmocom.de&gt; %% Author: Vadim Yanitskiy &lt;vyanitskiy(a)sysmocom.de&gt; %% %% All Rights Reserved @@ -233,10 +233,49 @@ parse_enb_id({'long-macroENB-ID', << ID:21 >>}) -> ID. --spec genb_id_str(proxy_state()) -> string(). +-spec genb_id_str(proxy_state()) -> string() | none. genb_id_str(#proxy_state{plmn_id = {MCC, MNC}, enb_id = ENBId}) -> - MCC ++ "-" ++ MNC ++ "-" ++ integer_to_list(ENBId). + MCC ++ "-" ++ MNC ++ "-" ++ integer_to_list(ENBId); + +genb_id_str(#proxy_state{}) -> none. + + +%% register a single per-eNB counter +-spec ctr_reg(C, GlobalENBId) -> C + when C :: [ctr | _], + GlobalENBId :: string(). +ctr_reg([ctr, s1ap, proxy | _] = C0, GlobalENBId) -> + C1 = s1gw_metrics:enb_metric(C0, GlobalENBId), + %% counter may already exist, so catch exceptions here + catch exometer:new(C1, counter), + C0; + +ctr_reg(C0, _GlobalENBId) -> C0. + + +%% register all per-eNB counters +-spec ctr_reg_all(string()) -> list(). +ctr_reg_all(GlobalENBId) -> + Ctrs = s1gw_metrics:ctr_list(), + lists:map(fun(Name) -> ctr_reg(Name, GlobalENBId) end, Ctrs). + + +%% increment the global and/or per-eNB counters +-spec ctr_inc(C0, S) -> term() + when C0 :: [ctr | _], + S :: proxy_state(). +ctr_inc([ctr | _] = C0, S) -> + case genb_id_str(S) of + none -> + %% increment the global counter only + s1gw_metrics:ctr_inc(C0); + GlobalENBId -> + %% increment both the global and per-eNB counters + C1 = s1gw_metrics:enb_metric(C0, GlobalENBId), + s1gw_metrics:ctr_inc(C0), + s1gw_metrics:ctr_inc(C1) + end. %% Encode an S1AP PDU @@ -256,7 +295,7 @@ %% Process an S1AP PDU (binary) -spec handle_pdu_bin(binary(), proxy_state()) -> {process_pdu_result(), proxy_state()}. handle_pdu_bin(OrigData, S0) -> - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ALL), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ALL, S0), try decode_pdu(OrigData) of {ok, PDU} -> ?LOG_DEBUG("Rx S1AP PDU: ~p", [PDU]), @@ -266,36 +305,36 @@ ?LOG_DEBUG("Tx (~p) S1AP PDU: ~p", [Action, NewPDU]), case Action of forward -> - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_ALL), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_PROC); + ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_ALL, S1), + ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_PROC, S1); reply -> - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_REPLY_ALL) + ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_REPLY_ALL, S1) end, {{Action, NewData}, S1}; {forward, S1} -> ?LOG_DEBUG("Tx (forward) S1AP PDU unmodified"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_ALL), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_UNMODIFIED), + ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_ALL, S1), + ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_UNMODIFIED, S1), {{forward, OrigData}, S1} catch Exception:Reason:StackTrace -> ?LOG_ERROR("An exception occurred: ~p, ~p, ~p", [Exception, Reason, StackTrace]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_EXCEPTION), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_UNMODIFIED), + ctr_inc(?S1GW_CTR_S1AP_PROXY_EXCEPTION, S0), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR, S0), + ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_UNMODIFIED, S0), {{forward, OrigData}, S0} %% XXX: proxy as-is or drop? end; {error, {asn1, Error}} -> ?LOG_ERROR("S1AP PDU decoding failed: ~p", [Error]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_DECODE_ERROR), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_UNMODIFIED), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_DECODE_ERROR, S0), + ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_UNMODIFIED, S0), {{forward, OrigData}, S0} %% XXX: forward as-is or drop? catch Exception:Reason:StackTrace -> ?LOG_ERROR("An exception occurred: ~p, ~p, ~p", [Exception, Reason, StackTrace]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_EXCEPTION), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_DECODE_ERROR), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_UNMODIFIED), + ctr_inc(?S1GW_CTR_S1AP_PROXY_EXCEPTION, S0), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_DECODE_ERROR, S0), + ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_UNMODIFIED, S0), {{forward, OrigData}, S0} %% XXX: proxy as-is or drop? end. @@ -320,7 +359,7 @@ #'InitiatingMessage'{procedureCode = ?'id-E-RABSetup', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing E-RAB SETUP REQUEST"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_SETUP_REQ), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_SETUP_REQ, S0), case handle_ies(?'id-E-RABToBeSetupListBearerSUReq', C0#'E-RABSetupRequest'.protocolIEs, S0) of {{ok, IEs}, S1} -> @@ -329,9 +368,9 @@ {{forward, PDU}, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process E-RAB SETUP REQUEST: ~p", [Reason]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR, S1), PDU = build_erab_setup_response_failure(S1), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_REPLY_ERAB_SETUP_RSP), + ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_REPLY_ERAB_SETUP_RSP, S1), {{reply, PDU}, S1} %% reply PDU back to sender end; @@ -340,7 +379,7 @@ #'SuccessfulOutcome'{procedureCode = ?'id-E-RABSetup', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing E-RAB SETUP RESPONSE"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_SETUP_RSP), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_SETUP_RSP, S0), case handle_ies(?'id-E-RABSetupListBearerSURes', C0#'E-RABSetupResponse'.protocolIEs, S0) of {{ok, IEs}, S1} -> @@ -349,7 +388,7 @@ {{forward, PDU}, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process E-RAB SETUP RESPONSE: ~p", [Reason]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR, S1), {forward, S1} %% XXX: forward as-is or drop? end; @@ -358,7 +397,7 @@ #'InitiatingMessage'{procedureCode = ?'id-E-RABModify', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing E-RAB MODIFY REQUEST"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_MODIFY_REQ), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_MODIFY_REQ, S0), case handle_ies(?'id-E-RABToBeModifiedListBearerModReq', C0#'E-RABModifyRequest'.protocolIEs, S0) of {{ok, IEs}, S1} -> @@ -367,7 +406,7 @@ {{forward, PDU}, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process E-RAB MODIFY REQUEST: ~p", [Reason]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR, S1), {forward, S1} %% XXX: forward as-is or drop? end; @@ -376,7 +415,7 @@ #'SuccessfulOutcome'{procedureCode = ?'id-E-RABModify', value = C0}}, S0) -> ?LOG_DEBUG("Processing E-RAB MODIFY RESPONSE"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_MODIFY_RSP), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_MODIFY_RSP, S0), %% there's nothing to patch in this PDU, so we forward it as-is %% TODO: check result of handle_ies(), inc. ?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR {_, S1} = handle_ies(?'id-E-RABModifyListBearerModRes', @@ -390,7 +429,7 @@ #'InitiatingMessage'{procedureCode = ?'id-E-RABRelease', value = C0}}, S0) -> ?LOG_DEBUG("Processing E-RAB RELEASE COMMAND"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_RELEASE_CMD), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_RELEASE_CMD, S0), case handle_ies(?'id-E-RABToBeReleasedList', C0#'E-RABReleaseCommand'.protocolIEs, S0) of {{ok, _}, S1} -> @@ -398,7 +437,7 @@ {forward, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process E-RAB RELEASE COMMAND: ~p", [Reason]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR, S1), {forward, S1} %% XXX: forward as-is or drop? end; @@ -407,7 +446,7 @@ #'SuccessfulOutcome'{procedureCode = ?'id-E-RABRelease', value = C0}}, S0) -> ?LOG_DEBUG("Processing E-RAB RELEASE RESPONSE"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_RELEASE_RSP), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_RELEASE_RSP, S0), case handle_ies(?'id-E-RABReleaseListBearerRelComp', C0#'E-RABReleaseResponse'.protocolIEs, S0) of {{ok, _}, S1} -> @@ -415,7 +454,7 @@ {forward, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process E-RAB RELEASE RESPONSE: ~p", [Reason]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR, S1), {forward, S1} %% XXX: forward as-is or drop? end; @@ -424,7 +463,7 @@ #'InitiatingMessage'{procedureCode = ?'id-E-RABReleaseIndication', value = C0}}, S0) -> ?LOG_DEBUG("Processing E-RAB RELEASE INDICATION"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_RELEASE_IND), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_RELEASE_IND, S0), case handle_ies(?'id-E-RABReleasedList', C0#'E-RABReleaseIndication'.protocolIEs, S0) of {{ok, _}, S1} -> @@ -432,7 +471,7 @@ {forward, S1}; {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process E-RAB RELEASE INDICATION: ~p", [Reason]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR, S1), {forward, S1} %% XXX: forward as-is or drop? end; @@ -441,7 +480,7 @@ #'InitiatingMessage'{procedureCode = ?'id-E-RABModificationIndication', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing E-RAB MODIFICATION INDICATION"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_MOD_IND), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_MOD_IND, S0), IEs0 = C0#'E-RABModificationIndication'.protocolIEs, %% E-RAB to be Modified List %% TODO: handle {error, Reason} @@ -458,7 +497,7 @@ #'SuccessfulOutcome'{procedureCode = ?'id-E-RABModificationIndication', value = C0}}, S0) -> ?LOG_DEBUG("Processing E-RAB MODIFICATION CONFIRM"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_MOD_CNF), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_MOD_CNF, S0), IEs = C0#'E-RABModificationConfirm'.protocolIEs, %% E-RAB Modify List %% TODO: handle {error, Reason} @@ -477,7 +516,7 @@ #'InitiatingMessage'{procedureCode = ?'id-InitialContextSetup', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing INITIAL CONTEXT SETUP REQUEST"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_INIT_CTX_REQ), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_INIT_CTX_REQ, S0), case handle_ies(?'id-E-RABToBeSetupListCtxtSUReq', C0#'InitialContextSetupRequest'.protocolIEs, S0) of {{ok, IEs}, S1} -> @@ -486,7 +525,7 @@ {{forward, PDU}, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process INITIAL CONTEXT SETUP REQUEST: ~p", [Reason]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR, S1), {forward, S1} %% XXX: forward as-is or drop? end; @@ -495,7 +534,7 @@ #'SuccessfulOutcome'{procedureCode = ?'id-InitialContextSetup', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing INITIAL CONTEXT SETUP RESPONSE"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_INIT_CTX_RSP), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_INIT_CTX_RSP, S0), %% TODO: handle optional E-RAB Failed to Setup List IE case handle_ies(?'id-E-RABSetupListCtxtSURes', C0#'InitialContextSetupResponse'.protocolIEs, S0) of @@ -505,7 +544,7 @@ {{forward, PDU}, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process INITIAL CONTEXT SETUP RESPONSE: ~p", [Reason]), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR, S1), {forward, S1} %% XXX: forward as-is or drop? end; @@ -515,7 +554,7 @@ value = #'UEContextReleaseRequest'{protocolIEs = IEs}}}, #proxy_state{erabs = ERABs} = S) -> ?LOG_DEBUG("Processing UE CONTEXT RELEASE REQUEST"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_RELEASE_CTX_REQ), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_RELEASE_CTX_REQ, S), %% fetch {MME,eNB}-UE-S1AP-ID values (mandatory IEs) #'ProtocolIE-Field'{id = ?'id-MME-UE-S1AP-ID', value = MMEUEId} = lists:nth(1, IEs), @@ -533,7 +572,7 @@ value = #'UEContextReleaseCommand'{protocolIEs = IEs}}}, #proxy_state{erabs = ERABs} = S) -> ?LOG_DEBUG("Processing UE CONTEXT RELEASE COMMAND"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_RELEASE_CTX_CMD), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_RELEASE_CTX_CMD, S), #'ProtocolIE-Field'{id = ?'id-UE-S1AP-IDs', value = S1APIDs} = lists:nth(1, IEs), case S1APIDs of @@ -557,7 +596,7 @@ value = #'UEContextReleaseComplete'{protocolIEs = IEs}}}, #proxy_state{erabs = ERABs} = S) -> ?LOG_DEBUG("Processing UE CONTEXT RELEASE COMPLETE"), - s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_RELEASE_CTX_COMPL), + ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_RELEASE_CTX_COMPL, S), %% fetch {MME,eNB}-UE-S1AP-ID values (mandatory IEs) #'ProtocolIE-Field'{id = ?'id-MME-UE-S1AP-ID', value = MMEUEId} = lists:nth(1, IEs), @@ -600,7 +639,10 @@ [S1#proxy_state.plmn_id, S1#proxy_state.enb_id]), %% use that as a context for logging - set_logging_prefix("eNB " ++ genb_id_str(S1)), + GlobalENBId = genb_id_str(S1), + set_logging_prefix("eNB " ++ GlobalENBId), + %% register per-eNB metrics + ctr_reg_all(GlobalENBId), {{ok, C}, S1}; %% E-RAB SETUP REQUEST related IEs diff --git a/src/s1gw_metrics.erl b/src/s1gw_metrics.erl index 317012d..9272033 100644 --- a/src/s1gw_metrics.erl +++ b/src/s1gw_metrics.erl @@ -36,6 +36,7 @@ -export([init/0, get_current_value/1, + enb_metric/2, ctr_reset/1, ctr_inc/1, ctr_inc/2, @@ -129,6 +130,14 @@ Value. +-spec enb_metric(M0, GlobalENBId) -> M1 + when M0 :: metric(), + M1 :: metric(), + GlobalENBId :: string(). +enb_metric([Type | Name], GlobalENBId) -> + [Type, enb, GlobalENBId | Name]. + + %%%%%%%%%%%%% %% CTR APIs %%%%%%%%%%%%% -- To view, visit https://gerrit.osmocom.org/c/erlang/osmo-s1gw/+/40648?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: erlang/osmo-s1gw Gerrit-Branch: master Gerrit-Change-Id: Ib138a0843c783c5a21ec2646c8a671cce7e689ad Gerrit-Change-Number: 40648 Gerrit-PatchSet: 3 Gerrit-Owner: fixeria &lt;vyanitskiy(a)sysmocom.de&gt; Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: fixeria &lt;vyanitskiy(a)sysmocom.de&gt; Gerrit-Reviewer: laforge &lt;laforge(a)osmocom.org&gt; Gerrit-Reviewer: pespin &lt;pespin(a)sysmocom.de&gt;