fixeria has submitted this change. ( https://gerrit.osmocom.org/c/erlang/osmo-s1gw/+/37955?usp=email )
Change subject: Introduce initial metrics support ......................................................................
Introduce initial metrics support
Metrics are implemented with the "exometer" library. A reference config to export the metrics through statsd is provided.
Change-Id: I952e198238384dca4be94f91a01d7cfff0a1471f Related: SYS#7065 --- M config/sys.config A include/s1gw_metrics.hrl M rebar.config M rebar.lock M src/osmo_s1gw.app.src M src/osmo_s1gw_sup.erl M src/pfcp_peer.erl M src/s1ap_proxy.erl A src/s1gw_metrics.erl M src/sctp_proxy.erl M src/sctp_server.erl A test/exometer_mock.erl M test/s1ap_proxy_test.erl 13 files changed, 357 insertions(+), 8 deletions(-)
Approvals: Jenkins Builder: Verified pespin: Looks good to me, approved
diff --git a/config/sys.config b/config/sys.config index 45a14cb..d405db8 100644 --- a/config/sys.config +++ b/config/sys.config @@ -35,6 +35,54 @@ {prefix, [" ", prefix, " ::"], ""}, " ", msg, {mfa, [" (", mfa, ":", line, ")"], ""}, - reset, "\n"]}}}}]}]}]. + reset, "\n"]}}}}]} + ] + }, + {exometer_core, + [{predefined, + [%{[erlang, memory], {function, erlang, memory, [], value, []}, []}, + %{[erlang, system_info], {function, erlang, system_info, ['$dp'], value, [process_count]}, []}, + %{[erlang, statistics], {function, erlang, statistics, ['$dp'], value, [run_queue]}, []}, + %{[erlang, io], {function, erlang, statistics, [io], match, {{'_', input}, {'_', output}}}, []} + ] + }, + {report, + [{reporters, + [%%{exometer_report_tty, []}, + {exometer_report_statsd, + [{hostname, "127.0.4.10"}, + {port, 8125}, + {prefix, "s1gw"}, + {type_map, []} + ] + } + ] + }, + {subscribers, + [%%{select, {[{ {['_' | '_'],'_','_'}, [], ['$_']}], + %% exometer_report_tty, value, 1000, true}} + {select, {[{ {['_' | '_'],counter,'_'}, [], ['$_']}], + exometer_report_statsd, + value, + 1000, + true, + [{report_type, counter}] + } + }, + {select, {[{ {['_' | '_'],gauge,'_'}, [], ['$_']}], + exometer_report_statsd, + value, + 1000, + true, + [{report_type, gauge}] + } + } + ] + } + ] + } + ] + } +].
%% vim:set ts=2 sw=2 et: diff --git a/include/s1gw_metrics.hrl b/include/s1gw_metrics.hrl new file mode 100644 index 0000000..551c136 --- /dev/null +++ b/include/s1gw_metrics.hrl @@ -0,0 +1,32 @@ +-define(S1GW_CTR_PFCP_ASSOC_SETUP_REQ_TX, [ctr, pfcp, assoc_setup_req, tx]). +-define(S1GW_CTR_PFCP_ASSOC_SETUP_REQ_TIMEOUT, [ctr, pfcp, assoc_setup_req, timeout]). +-define(S1GW_CTR_PFCP_ASSOC_SETUP_RESP_RX, [ctr, pfcp, assoc_setup_resp, rx]). +-define(S1GW_CTR_PFCP_ASSOC_SETUP_RESP_RX_ACK, [ctr, pfcp, assoc_setup_resp, rx_ack]). +-define(S1GW_CTR_PFCP_ASSOC_SETUP_RESP_RX_NACK, [ctr, pfcp, assoc_setup_resp, rx_nack]). +-define(S1GW_CTR_S1AP_ENB_ALL_RX, [ctr, s1ap, enb, all, rx]). +-define(S1GW_CTR_S1AP_ENB_ALL_RX_UNKNOWN_ENB, [ctr, s1ap, enb, all, rx_unknown_enb]). +-define(S1GW_CTR_S1AP_PROXY_UPLINK_PACKETS_QUEUED, [ctr, s1ap, proxy, uplink_packets_queued]). +-define(S1GW_CTR_S1AP_PROXY_IN_PKT_ALL, [ctr, s1ap, proxy, in_pkt, all]). +-define(S1GW_CTR_S1AP_PROXY_IN_PKT_DECODE_ERROR, [ctr, s1ap, proxy, in_pkt, decode_error]). +-define(S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR, [ctr, s1ap, proxy, in_pkt, proc_error]). +-define(S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_SETUP_REQ, [ctr, s1ap, proxy, in_pkt, erab_setup_req]). +-define(S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_SETUP_RSP, [ctr, s1ap, proxy, in_pkt, erab_setup_rsp]). +-define(S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_RELEASE_CMD, [ctr, s1ap, proxy, in_pkt, erab_release_cmd]). +-define(S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_RELEASE_RSP, [ctr, s1ap, proxy, in_pkt, erab_release_rsp]). +-define(S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_RELEASE_IND, [ctr, s1ap, proxy, in_pkt, erab_release_ind]). +-define(S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_MOD_IND, [ctr, s1ap, proxy, in_pkt, erab_mod_ind]). +-define(S1GW_CTR_S1AP_PROXY_IN_PKT_INIT_CTX_REQ, [ctr, s1ap, proxy, in_pkt, init_ctx_req]). +-define(S1GW_CTR_S1AP_PROXY_IN_PKT_INIT_CTX_RSP, [ctr, s1ap, proxy, in_pkt, init_ctx_rsp]). +-define(S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_ALL, [ctr, s1ap, proxy, out_pkt, forward, all]). +-define(S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_PROC, [ctr, s1ap, proxy, out_pkt, forward, proc]). +-define(S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_UNMODIFIED, [ctr, s1ap, proxy, out_pkt, forward, unmodified]). +-define(S1GW_CTR_S1AP_PROXY_OUT_PKT_REPLY_ALL, [ctr, s1ap, proxy, out_pkt, reply, all]). +-define(S1GW_CTR_S1AP_PROXY_OUT_PKT_REPLY_ERAB_SETUP_RSP, [ctr, s1ap, proxy, out_pkt, reply, erab_setup_rsp]). + +-define(S1GW_GAUGE_PFCP_ASSOCIATED, [gauge, pfcp, associated]). +-define(S1GW_GAUGE_S1AP_ENB_NUM_SCTP_CONNECTIONS, [gauge, s1ap, enb, num_sctp_connections]). +-define(S1GW_GAUGE_S1AP_PROXY_UPLINK_PACKETS_QUEUED, [gauge, s1ap, proxy, uplink_packets_queued]). + +%% NOTE: Remember to add new entries to s1gw_metric:init()! + +%% vim:set ts=4 sw=4 et: diff --git a/rebar.config b/rebar.config index 91bd759..ef3931d 100644 --- a/rebar.config +++ b/rebar.config @@ -8,7 +8,11 @@ {deps, [{logger_color_formatter, {git, "https://github.com/rlipscombe/logger_color_formatter.git", {tag, "0.5.0"}}}, {pfcplib, - {git, "https://github.com/travelping/pfcplib.git", {branch, "master"}}} + {git, "https://github.com/travelping/pfcplib.git", {branch, "master"}}}, + {exometer_core, + {git, "https://github.com/Feuerlabs/exometer_core.git", {branch, "master"}}}, + {exometer_report_statsd, + {git, "https://github.com/esl/exometer_report_statsd.git", {branch, "master"}}} ]}.
%% test deps diff --git a/rebar.lock b/rebar.lock index 3b99c22..c52d677 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,16 +1,33 @@ {"1.2.0", [{<<"cut">>,{pkg,<<"cut">>,<<"1.0.3">>},1}, + {<<"exometer_core">>, + {git,"https://github.com/Feuerlabs/exometer_core.git", + {ref,"f9c7abc095edc893c9354a3d5f061715de1d9e79"}}, + 0}, + {<<"exometer_report_statsd">>, + {git,"https://github.com/esl/exometer_report_statsd.git", + {ref,"f1c369becb6e57871f1c7b0e491f6c3a302a65ee"}}, + 0}, + {<<"hut">>,{pkg,<<"hut">>,<<"1.3.0">>},1}, {<<"logger_color_formatter">>, {git,"https://github.com/rlipscombe/logger_color_formatter.git", {ref,"f1c96f979e6350f8cd787d27fe9ff003cbf3416b"}}, 0}, + {<<"parse_trans">>,{pkg,<<"parse_trans">>,<<"3.4.1">>},1}, {<<"pfcplib">>, {git,"https://github.com/travelping/pfcplib.git", {ref,"e505c0a4c05f5f14a2c40c7ebf36db62cc911d93"}}, - 0}]}. + 0}, + {<<"setup">>,{pkg,<<"setup">>,<<"2.1.0">>},1}]}. [ {pkg_hash,[ - {<<"cut">>, <<"1577F2F3BC0F2BF3B97903B7426F8A3D79523687B6A444D0F59A095EF69A0E81">>}]}, + {<<"cut">>, <<"1577F2F3BC0F2BF3B97903B7426F8A3D79523687B6A444D0F59A095EF69A0E81">>}, + {<<"hut">>, <<"71F2F054E657C03F959CF1ACC43F436EA87580696528CA2A55C8AFB1B06C85E7">>}, + {<<"parse_trans">>, <<"6E6AA8167CB44CC8F39441D05193BE6E6F4E7C2946CB2759F015F8C56B76E5FF">>}, + {<<"setup">>, <<"05F69185A5EB71474C9BC6BA892565651EC7507791F85632B7B914DBFE130510">>}]}, {pkg_hash_ext,[ - {<<"cut">>, <<"1A4A25DB2B7C5565FD28B314A4EEB898B1ED3CAFFA1AB09149345FB5731ED04B">>}]} + {<<"cut">>, <<"1A4A25DB2B7C5565FD28B314A4EEB898B1ED3CAFFA1AB09149345FB5731ED04B">>}, + {<<"hut">>, <<"7E15D28555D8A1F2B5A3A931EC120AF0753E4853A4C66053DB354F35BF9AB563">>}, + {<<"parse_trans">>, <<"620A406CE75DADA827B82E453C19CF06776BE266F5A67CFF34E1EF2CBB60E49A">>}, + {<<"setup">>, <<"EFD072578F0CF85BEA96CAAFFC7ADB0992398272522660A136E10567377071C5">>}]} ]. diff --git a/src/osmo_s1gw.app.src b/src/osmo_s1gw.app.src index 8bb89de..363d6d6 100644 --- a/src/osmo_s1gw.app.src +++ b/src/osmo_s1gw.app.src @@ -8,7 +8,9 @@ kernel, stdlib, logger_color_formatter, - pfcplib + pfcplib, + exometer_core, + exometer_report_statsd ]}, {modules, []}, {mod, {osmo_s1gw_app, []}}, diff --git a/src/osmo_s1gw_sup.erl b/src/osmo_s1gw_sup.erl index c6adb55..32272bf 100644 --- a/src/osmo_s1gw_sup.erl +++ b/src/osmo_s1gw_sup.erl @@ -81,6 +81,7 @@ worker, [pfcp_peer]},
+ s1gw_metrics:init(), {ok, {{one_for_one, 5, 10}, [SctpServer, PfcpPeer]}}.
diff --git a/src/pfcp_peer.erl b/src/pfcp_peer.erl index aa33e3a..d0da4bb 100644 --- a/src/pfcp_peer.erl +++ b/src/pfcp_peer.erl @@ -50,6 +50,7 @@
-include_lib("kernel/include/logger.hrl"). -include_lib("pfcplib/include/pfcp_packet.hrl"). +-include("s1gw_metrics.hrl").
%% 3GPP TS 29.244, section 4.2 "UDP Header and Port Numbers" -define(PFCP_PORT, 8805). @@ -136,6 +137,7 @@
init([LocAddr, RemAddr]) -> process_flag(trap_exit, true), + s1gw_metrics:gauge_set(?S1GW_GAUGE_PFCP_ASSOCIATED, 0), {ok, Sock} = gen_udp:open(?PFCP_PORT, [binary, {ip, LocAddr}, {reuseaddr, true}, @@ -158,6 +160,7 @@ connecting(enter, OldState, #peer_state{} = S0) -> ?LOG_INFO("State change: ~p -> ~p", [OldState, ?FUNCTION_NAME]), + s1gw_metrics:gauge_set(?S1GW_GAUGE_PFCP_ASSOCIATED, 0), %% Tx PFCP Association Setup {ok, S1} = send_assoc_setup(S0), {keep_state, S1, [{state_timeout, 2_000, assoc_setup_timeout}]}; @@ -166,6 +169,7 @@ connecting(state_timeout, assoc_setup_timeout, S) -> % Re-start sending PFCP Association Setup above: ?LOG_NOTICE("PFCP Association Setup timeout, UPF may be down, retrying..."), + s1gw_metrics:ctr_inc(?S1GW_CTR_PFCP_ASSOC_SETUP_REQ_TIMEOUT), {repeat_state, S};
%% Handle incoming PFCP PDU(s) @@ -177,11 +181,15 @@ ie = #{pfcp_cause := 'Request accepted', recovery_time_stamp := #recovery_time_stamp{time = RRTS}}} -> ?LOG_INFO("Rx Association Setup Response (Request accepted)"), + s1gw_metrics:ctr_inc(?S1GW_CTR_PFCP_ASSOC_SETUP_RESP_RX), + s1gw_metrics:ctr_inc(?S1GW_CTR_PFCP_ASSOC_SETUP_RESP_RX_ACK), {next_state, connected, S#peer_state{rem_rts = RRTS}}; #pfcp{type = association_setup_response, ie = #{pfcp_cause := Cause}} -> ?LOG_ERROR("Rx Association Setup Response (~p)", [Cause]), + s1gw_metrics:ctr_inc(?S1GW_CTR_PFCP_ASSOC_SETUP_RESP_RX), + s1gw_metrics:ctr_inc(?S1GW_CTR_PFCP_ASSOC_SETUP_RESP_RX_NACK), {stop, {shutdown, assoc_setup_nack}}; %% 3GPP TS 29.244, 6.2.2.2 Heartbeat Request %% A CP function or UP function shall be prepared to receive a Heartbeat Request @@ -201,6 +209,7 @@ %% CONNECTED state connected(enter, OldState, S) -> ?LOG_INFO("State change: ~p -> ~p", [OldState, ?FUNCTION_NAME]), + s1gw_metrics:gauge_set(?S1GW_GAUGE_PFCP_ASSOCIATED, 1), {keep_state, S};
connected({call, From}, @@ -272,7 +281,8 @@ ?LOG_NOTICE("Terminating in state ~p, reason ~p", [State, Reason]), case State of connected -> - send_assoc_release(S); + send_assoc_release(S), + s1gw_metrics:gauge_set(?S1GW_GAUGE_PFCP_ASSOCIATED, 0); _ -> nop end, @@ -395,6 +405,7 @@ send_assoc_setup(#peer_state{loc_rts = LRTS} = S) -> IEs = #{node_id => #node_id{id = get_node_id(S)}, recovery_time_stamp => #recovery_time_stamp{time = LRTS}}, + s1gw_metrics:ctr_inc(?S1GW_CTR_PFCP_ASSOC_SETUP_REQ_TX), send_pdu({association_setup_request, IEs}, S).
diff --git a/src/s1ap_proxy.erl b/src/s1ap_proxy.erl index 4758484..68d0d0e 100644 --- a/src/s1ap_proxy.erl +++ b/src/s1ap_proxy.erl @@ -43,6 +43,8 @@
-include_lib("kernel/include/logger.hrl").
+-include("s1gw_metrics.hrl"). + -include("S1AP-PDU-Descriptions.hrl"). -include("S1AP-PDU-Contents.hrl"). -include("S1AP-Containers.hrl"). @@ -90,6 +92,7 @@ %% Process an S1AP PDU -spec process_pdu(binary(), proxy_state()) -> {{proxy_action(), binary()}, proxy_state()}. process_pdu(OrigData, S0) -> + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ALL), case decode_pdu(OrigData) of {ok, PDU} -> ?LOG_DEBUG("Rx S1AP PDU: ~p", [PDU]), @@ -97,13 +100,24 @@ {{Action, NewPDU}, S1} -> {ok, NewData} = encode_pdu(NewPDU), ?LOG_DEBUG("Tx (~p) S1AP PDU: ~p", [Action, NewPDU]), + case Action of + forward -> + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_ALL), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_PROC); + reply -> + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_REPLY_ALL) + end, {{Action, NewData}, S1}; {forward, S1} -> ?LOG_DEBUG("Tx (forward) S1AP PDU unmodified"), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_ALL), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_UNMODIFIED), {{forward, OrigData}, S1} end; {error, {asn1, Error}} -> ?LOG_ERROR("S1AP PDU decoding failed: ~p", [Error]), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_DECODE_ERROR), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_UNMODIFIED), {{forward, OrigData}, S0} %% XXX: forward as-is or drop? end.
@@ -148,6 +162,7 @@ #'InitiatingMessage'{procedureCode = ?'id-E-RABSetup', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing E-RAB SETUP REQUEST"), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_SETUP_REQ), case handle_ies(C0#'E-RABSetupRequest'.protocolIEs, ?'id-E-RABToBeSetupListBearerSUReq', S0) of {{ok, IEs}, S1} -> @@ -156,7 +171,9 @@ {{forward, PDU}, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process E-RAB SETUP REQUEST: ~p", [Reason]), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), PDU = build_erab_setup_response_failure(S1), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_OUT_PKT_REPLY_ERAB_SETUP_RSP), {{reply, PDU}, S1} %% reply PDU back to sender end;
@@ -165,6 +182,7 @@ #'SuccessfulOutcome'{procedureCode = ?'id-E-RABSetup', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing E-RAB SETUP RESPONSE"), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_SETUP_RSP), case handle_ies(C0#'E-RABSetupResponse'.protocolIEs, ?'id-E-RABSetupListBearerSURes', S0) of {{ok, IEs}, S1} -> @@ -173,6 +191,7 @@ {{forward, PDU}, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process E-RAB SETUP RESPONSE: ~p", [Reason]), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), {forward, S1} %% XXX: forward as-is or drop? end;
@@ -183,6 +202,7 @@ #'InitiatingMessage'{procedureCode = ?'id-E-RABRelease', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing E-RAB RELEASE COMMAND"), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_RELEASE_CMD), case handle_ies(C0#'E-RABReleaseCommand'.protocolIEs, ?'id-E-RABToBeReleasedList', S0) of {{ok, IEs}, S1} -> @@ -191,6 +211,7 @@ {{forward, PDU}, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process E-RAB RELEASE COMMAND: ~p", [Reason]), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), {forward, S1} %% XXX: forward as-is or drop? end;
@@ -199,6 +220,7 @@ #'SuccessfulOutcome'{procedureCode = ?'id-E-RABRelease', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing E-RAB RELEASE RESPONSE"), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_RELEASE_RSP), case handle_ies(C0#'E-RABReleaseResponse'.protocolIEs, ?'id-E-RABReleaseListBearerRelComp', S0) of {{ok, IEs}, S1} -> @@ -207,6 +229,7 @@ {{forward, PDU}, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process E-RAB RELEASE RESPONSE: ~p", [Reason]), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), {forward, S1} %% XXX: forward as-is or drop? end;
@@ -215,6 +238,7 @@ #'InitiatingMessage'{procedureCode = ?'id-E-RABReleaseIndication', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing E-RAB RELEASE INDICATION"), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_RELEASE_IND), case handle_ies(C0#'E-RABReleaseIndication'.protocolIEs, ?'id-E-RABReleasedList', S0) of {{ok, IEs}, S1} -> @@ -223,6 +247,7 @@ {{forward, PDU}, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process E-RAB RELEASE INDICATION: ~p", [Reason]), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), {forward, S1} %% XXX: forward as-is or drop? end;
@@ -231,6 +256,7 @@ #'InitiatingMessage'{procedureCode = ?'id-E-RABModificationIndication', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing E-RAB MODIFICATION INDICATION"), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_MOD_IND), IEs0 = C0#'E-RABModificationIndication'.protocolIEs, %% E-RAB to be Modified List %% TODO: handle {error, Reason} @@ -247,6 +273,7 @@ #'InitiatingMessage'{procedureCode = ?'id-InitialContextSetup', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing INITIAL CONTEXT SETUP REQUEST"), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_INIT_CTX_REQ), case handle_ies(C0#'InitialContextSetupRequest'.protocolIEs, ?'id-E-RABToBeSetupListCtxtSUReq', S0) of {{ok, IEs}, S1} -> @@ -255,6 +282,7 @@ {{forward, PDU}, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process INITIAL CONTEXT SETUP REQUEST: ~p", [Reason]), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), {forward, S1} %% XXX: forward as-is or drop? end;
@@ -263,6 +291,7 @@ #'SuccessfulOutcome'{procedureCode = ?'id-InitialContextSetup', value = C0} = Msg}, S0) -> ?LOG_DEBUG("Processing INITIAL CONTEXT SETUP RESPONSE"), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_INIT_CTX_RSP), case handle_ies(C0#'InitialContextSetupResponse'.protocolIEs, ?'id-E-RABSetupListCtxtSURes', S0) of {{ok, IEs}, S1} -> @@ -271,6 +300,7 @@ {{forward, PDU}, S1}; %% forward patched PDU {{error, Reason}, S1} -> ?LOG_NOTICE("Failed to process INITIAL CONTEXT SETUP RESPONSE: ~p", [Reason]), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR), {forward, S1} %% XXX: forward as-is or drop? end;
diff --git a/src/s1gw_metrics.erl b/src/s1gw_metrics.erl new file mode 100644 index 0000000..6b078e5 --- /dev/null +++ b/src/s1gw_metrics.erl @@ -0,0 +1,168 @@ +%% Copyright (C) 2024 by sysmocom - s.f.m.c. GmbH info@sysmocom.de +%% Author: Pau Espin Pedrol pespin@sysmocom.de +%% +%% All Rights Reserved +%% +%% SPDX-License-Identifier: AGPL-3.0-or-later +%% +%% This program is free software; you can redistribute it and/or modify +%% it under the terms of the GNU Affero General Public License as +%% published by the Free Software Foundation; either version 3 of the +%% License, or (at your option) any later version. +%% +%% This program is distributed in the hope that it will be useful, +%% but WITHOUT ANY WARRANTY; without even the implied warranty of +%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +%% GNU General Public License for more details. +%% +%% You should have received a copy of the GNU Affero General Public License +%% along with this program. If not, see https://www.gnu.org/licenses/. +%% +%% Additional Permission under GNU AGPL version 3 section 7: +%% +%% If you modify this Program, or any covered work, by linking or +%% combining it with runtime libraries of Erlang/OTP as released by +%% Ericsson on https://www.erlang.org (or a modified version of these +%% libraries), containing parts covered by the terms of the Erlang Public +%% License (https://www.erlang.org/EPLICENSE), the licensors of this +%% Program grant you additional permission to convey the resulting work +%% without the need to license the runtime libraries of Erlang/OTP under +%% the GNU Affero General Public License. Corresponding Source for a +%% non-source form of such a combination shall include the source code +%% for the parts of the runtime libraries of Erlang/OTP used as well as +%% that of the covered work. + +-module(s1gw_metrics). + +-export([init/0, + ctr_reset/1, + ctr_inc/1, + ctr_inc/2, + gauge_reset/1, + gauge_set/2, + gauge_inc/1, + gauge_inc/2, + gauge_dec/1]). + +-include_lib("kernel/include/logger.hrl"). +-include("s1gw_metrics.hrl"). + +-define(S1GW_COUNTERS, [ + ?S1GW_CTR_PFCP_ASSOC_SETUP_REQ_TX, + ?S1GW_CTR_PFCP_ASSOC_SETUP_REQ_TIMEOUT, + ?S1GW_CTR_PFCP_ASSOC_SETUP_RESP_RX, + ?S1GW_CTR_PFCP_ASSOC_SETUP_RESP_RX_ACK, + ?S1GW_CTR_PFCP_ASSOC_SETUP_RESP_RX_NACK, + ?S1GW_CTR_S1AP_ENB_ALL_RX, + ?S1GW_CTR_S1AP_ENB_ALL_RX_UNKNOWN_ENB, + ?S1GW_CTR_S1AP_PROXY_UPLINK_PACKETS_QUEUED, + %% s1ap_proxy: INcoming PDU counters + ?S1GW_CTR_S1AP_PROXY_IN_PKT_ALL, %% received total + ?S1GW_CTR_S1AP_PROXY_IN_PKT_DECODE_ERROR, %% failed to decode + ?S1GW_CTR_S1AP_PROXY_IN_PKT_PROC_ERROR, %% failed to process + ?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_SETUP_REQ, %% E-RAB SETUP.req PDUs + ?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_SETUP_RSP, %% E-RAB SETUP.rsp PDUs + ?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_RELEASE_CMD, %% E-RAB RELEASE.cmd PDUs + ?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_RELEASE_RSP, %% E-RAB RELEASE.rsp PDUs + ?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_RELEASE_IND, %% E-RAB RELEASE.ind PDUs + ?S1GW_CTR_S1AP_PROXY_IN_PKT_ERAB_MOD_IND, %% E-RAB MODIFY.ind PDUs + ?S1GW_CTR_S1AP_PROXY_IN_PKT_INIT_CTX_REQ, %% INITIAL CONTEXT SETUP.req PDUs + ?S1GW_CTR_S1AP_PROXY_IN_PKT_INIT_CTX_RSP, %% INITIAL CONTEXT SETUP.rsp PDUs + %% s1ap_proxy: OUTgoing PDU counters + ?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_ALL, %% forwarded: total + ?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_PROC, %% forwarded: processed + ?S1GW_CTR_S1AP_PROXY_OUT_PKT_FWD_UNMODIFIED, %% forwarded: unmodified + ?S1GW_CTR_S1AP_PROXY_OUT_PKT_REPLY_ALL, %% replied: total + ?S1GW_CTR_S1AP_PROXY_OUT_PKT_REPLY_ERAB_SETUP_RSP %% replied: E-RAB SETUP.rsp +]). + +-define(S1GW_GAUGES, [ + ?S1GW_GAUGE_PFCP_ASSOCIATED, + ?S1GW_GAUGE_S1AP_ENB_NUM_SCTP_CONNECTIONS, + ?S1GW_GAUGE_S1AP_PROXY_UPLINK_PACKETS_QUEUED +]). + +-spec new_ctr(list()) -> ok. +new_ctr(Name) -> + %%?LOG_INFO("New counter ~p", [Name]), + ok = exometer:new(Name, counter). + +-spec new_ctrs(list()) -> ok. +new_ctrs([]) -> + ok; +new_ctrs([Name | MoreNames]) -> + new_ctr(Name), + new_ctrs(MoreNames). + +-spec new_gauge(list()) -> ok. +new_gauge(Name) -> + %%?LOG_INFO("New gauge ~p", [Name]), + ok = exometer:new(Name, gauge). + +-spec new_gauges(list()) -> ok. +new_gauges([]) -> + ok; +new_gauges([Name | MoreNames]) -> + new_gauge(Name), + new_gauges(MoreNames). + +-spec get_current_value(list()) -> integer(). +get_current_value(Name) -> + Result = exometer:get_value(Name, value), + {ok, [{value, PrevVal}]} = Result, + PrevVal. + +%% ------------------------------------------------------------------ +%% public API +%% ------------------------------------------------------------------ + +init() -> + ?LOG_INFO("Initiating metrics"), + new_ctrs(?S1GW_COUNTERS), + new_gauges(?S1GW_GAUGES). + +%%%%%%%%%%%%% +%% CTR APIs +%%%%%%%%%%%%% +-spec ctr_reset(list()) -> ok | {error, any()}. +ctr_reset(Name) -> + ?LOG_DEBUG("ctr_reset(~p)", [Name]), + exometer:reset(Name). + +-spec ctr_inc(list(), integer()) -> ok | {error, any()}. +ctr_inc(Name, Value) -> + ?LOG_DEBUG("ctr_inc(~p, ~p)", [Name, Value]), + exometer:update(Name, Value). + +-spec ctr_inc(list()) -> ok | {error, any()}. +ctr_inc(Name) -> + ctr_inc(Name, 1). + +%%%%%%%%%%%%% +%% GAUGE APIs +%%%%%%%%%%%%% +-spec gauge_reset(list()) -> ok | {error, any()}. +gauge_reset(Name) -> + ?LOG_DEBUG("gauge_reset(~p)", [Name]), + exometer:reset(Name). + +-spec gauge_set(list(), integer()) -> ok | {error, any()}. +gauge_set(Name, Value) -> + exometer:update(Name, Value). + +-spec gauge_inc(list(), integer()) -> ok | {error, any()}. +gauge_inc(Name, Value) -> + PrevVal = get_current_value(Name), + ?LOG_DEBUG("gauge_inc(~p, ~p): pre_val=~p", [Name, Value, PrevVal]), + exometer:update(Name, Value + PrevVal). + +-spec gauge_inc(list()) -> ok | {error, any()}. +gauge_inc(Name) -> + gauge_inc(Name, 1). + +-spec gauge_dec(list()) -> ok | {error, any()}. +gauge_dec(Name) -> + gauge_inc(Name, -1). + + +%% vim:set ts=4 sw=4 et: diff --git a/src/sctp_proxy.erl b/src/sctp_proxy.erl index b37bead..280e408 100644 --- a/src/sctp_proxy.erl +++ b/src/sctp_proxy.erl @@ -49,6 +49,8 @@ -include_lib("kernel/include/inet.hrl"). -include_lib("kernel/include/inet_sctp.hrl").
+-include("s1gw_metrics.hrl"). + %% ------------------------------------------------------------------ %% public API %% ------------------------------------------------------------------ @@ -104,6 +106,8 @@ %% Handle an eNB -> MME data forwarding request (queue) connecting(cast, {send_data, Data}, #{tx_queue := Pending} = S) -> + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_PROXY_UPLINK_PACKETS_QUEUED), + s1gw_metrics:gauge_inc(?S1GW_GAUGE_S1AP_PROXY_UPLINK_PACKETS_QUEUED), {keep_state, S#{tx_queue := [Data | Pending]}};
%% Handle an #sctp_assoc_change event (connection state) @@ -254,6 +258,7 @@
sctp_send_pending([Data | Pending], S0) -> S1 = sctp_send(Data, S0), + s1gw_metrics:gauge_dec(?S1GW_CTR_S1AP_PROXY_UPLINK_PACKETS_QUEUED), sctp_send_pending(Pending, S1);
sctp_send_pending([], S) -> diff --git a/src/sctp_server.erl b/src/sctp_server.erl index 8ea311f..58d91ca 100644 --- a/src/sctp_server.erl +++ b/src/sctp_server.erl @@ -48,6 +48,7 @@ -include_lib("kernel/include/inet.hrl"). -include_lib("kernel/include/inet_sctp.hrl").
+-include("s1gw_metrics.hrl"). -include("s1ap.hrl").
-record(server_state, {sock, clients, mme_addr_port}). @@ -170,12 +171,14 @@ sctp_recv(State, {FromAddr, FromPort, [#sctp_sndrcvinfo{assoc_id = Aid}], Data}) -> ?LOG_DEBUG("eNB connection (id=~p, ~p:~p) -> MME: ~p", [Aid, FromAddr, FromPort, Data]), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_ENB_ALL_RX), case dict:find(Aid, State#server_state.clients) of {ok, #client_state{pid = Pid}} -> sctp_proxy:send_data(Pid, Data); error -> ?LOG_ERROR("eNB connection (id=~p, ~p:~p) is not known to us?!?", - [Aid, FromAddr, FromPort]) + [Aid, FromAddr, FromPort]), + s1gw_metrics:ctr_inc(?S1GW_CTR_S1AP_ENB_ALL_RX_UNKNOWN_ENB) end, State;
@@ -190,6 +193,7 @@ client_add(Clients, Aid, FromAddr, FromPort, {MmeAddr, MmePort}) -> {ok, Pid} = sctp_proxy:start_link(Aid, MmeAddr, MmePort), NewClient = #client_state{addr_port = {FromAddr, FromPort}, pid = Pid}, + s1gw_metrics:gauge_inc(?S1GW_GAUGE_S1AP_ENB_NUM_SCTP_CONNECTIONS), dict:store(Aid, NewClient, Clients).
@@ -200,6 +204,7 @@ %% the proxy process might be already dead, so we guard %% against exceptions like noproc or {nodedown,Node}. catch sctp_proxy:shutdown(Client#client_state.pid), + s1gw_metrics:gauge_dec(?S1GW_GAUGE_S1AP_ENB_NUM_SCTP_CONNECTIONS), dict:erase(Aid, Clients); error -> Clients diff --git a/test/exometer_mock.erl b/test/exometer_mock.erl new file mode 100644 index 0000000..5a09692 --- /dev/null +++ b/test/exometer_mock.erl @@ -0,0 +1,24 @@ +-module(exometer_mock). + +-export([mock_all/0, + unmock_all/0]). + + +%% ------------------------------------------------------------------ +%% public API +%% ------------------------------------------------------------------ + +%% mock all pfcp_peer module functions +mock_all() -> + meck:new(exometer), + meck:expect(exometer, new, fun(_Name, _Type) -> ok end), + meck:expect(exometer, reset, fun(_Name) -> ok end), + meck:expect(exometer, update, fun(_Name, _Value) -> ok end), + meck:expect(exometer, get_value, fun(_Name, DataPoint) -> {ok,[{DataPoint,0}]} end). + +%% unmock all pfcp_peer module functions +unmock_all() -> + true = meck:validate(exometer), + meck:unload(exometer). + +%% vim:set ts=4 sw=4 et: diff --git a/test/s1ap_proxy_test.erl b/test/s1ap_proxy_test.erl index 04d52df..28fe209 100644 --- a/test/s1ap_proxy_test.erl +++ b/test/s1ap_proxy_test.erl @@ -15,10 +15,12 @@
start() -> pfcp_mock:mock_all(), + exometer_mock:mock_all(), s1ap_proxy:init().
stop(S) -> + exometer_mock:unmock_all(), pfcp_mock:unmock_all(), s1ap_proxy:deinit(S).
-
fixeria