[L] Change in ...osmo-s1gw[master]: enft_kpi: support different UL/DL GTP-U addresses

fixeria has uploaded this change for review. ( https://gerrit.osmocom.org/c/erlang/osmo-s1gw/+/40437?usp=email ) Change subject: enft_kpi: support different UL/DL GTP-U addresses ...................................................................... enft_kpi: support different UL/DL GTP-U addresses Change-Id: Icd4a17790062bfcaf2bccb01fa94dcdb65c0872c Related: SYS#7307 --- M src/enft_kpi.erl M src/s1ap_proxy.erl 2 files changed, 156 insertions(+), 113 deletions(-) git pull ssh://gerrit.osmocom.org:29418/erlang/osmo-s1gw refs/changes/37/40437/1 diff --git a/src/enft_kpi.erl b/src/enft_kpi.erl index 015956c..9b4caf1 100644 --- a/src/enft_kpi.erl +++ b/src/enft_kpi.erl @@ -72,14 +72,22 @@ -type counters() :: dict:dict(K :: string(), V :: counter()). +-type uldl() :: ul | dl. + +-type uldl_addr() :: {ULDL :: uldl(), + Addr :: string()}. + +-type enb_uldl_state() :: #{uldl => uldl(), + addr => string(), + handle => integer(), + ctr => counter() + }. + -type enb_state() :: #{pid => pid(), - addr => string(), genb_id => string(), mon_ref => reference(), - handle_ul => integer(), - handle_dl => integer(), - ctr_ul => counter(), %% UL counters - ctr_dl => counter() %% DL counters + ul => enb_uldl_state(), %% Uplink data + dl => enb_uldl_state() %% Downlink data }. -type registry() :: dict:dict(K :: pid(), @@ -109,10 +117,11 @@ gen_server:call(?MODULE, {?FUNCTION_NAME, GlobalENBId}). --spec enb_set_addr(Addr) -> ok | {error, term()} - when Addr :: string(). -enb_set_addr(Addr) -> - gen_server:call(?MODULE, {?FUNCTION_NAME, Addr}). +-spec enb_set_addr(ULDLAddr) -> ok | {error, term()} + when ULDLAddr :: uldl_addr(). +enb_set_addr(ULDLAddr) -> + %% TODO: use cast here to avoid blocking + gen_server:call(?MODULE, {?FUNCTION_NAME, ULDLAddr}). -spec enb_unregister() -> ok | {error, term()}. @@ -194,15 +203,30 @@ {reply, ok, S#state{registry = R1}} end; -handle_call({enb_set_addr, Addr}, {Pid, _Ref}, +handle_call({enb_set_addr, {ULDL, Addr}}, {Pid, _Ref}, #state{registry = R} = S0) -> case dict:find(Pid, R) of - {ok, EnbState} -> - {Reply, S1} = enb_set_addr(Addr, EnbState, S0), + %% the given UL/DL Addr is already known + {ok, #{genb_id := GlobalENBId, + ULDL := #{addr := Addr}}} -> + ?LOG_DEBUG("eNB (pid ~p, ~p): ~p address ~p is already known", + [Pid, GlobalENBId, ULDL, Addr]), + {reply, ok, S0}; + %% the given UL/DL Addr differs from stored Addr + {ok, #{genb_id := GlobalENBId, + ULDL := #{addr := OldAddr}}} -> + ?LOG_ERROR("eNB (pid ~p, ~p): ~p address ~p -> ~p change?!?", + [Pid, GlobalENBId, ULDL, OldAddr, Addr]), + {reply, {error, addr_mismatch}, S0}; + %% UL/DL state is missing => create it + {ok, #{genb_id := GlobalENBId} = ES} -> + ?LOG_DEBUG("eNB (pid ~p, ~p): ~p address ~p indicated, " + "creating NFT counters and rules", + [Pid, GlobalENBId, ULDL, Addr]), + {Reply, S1} = enb_set_addr({ULDL, Addr}, ES, S0), {reply, Reply, S1}; error -> - ?LOG_ERROR("eNB @ ~p (pid ~p) is *not* registered", - [Addr, Pid]), + ?LOG_ERROR("eNB (pid ~p) is *not* registered", [Pid]), {reply, {error, not_registered}, S0} end; @@ -303,107 +327,108 @@ enb_add_metrics(GlobalENBId, ul), enb_add_metrics(GlobalENBId, dl). -enb_add_metrics(GlobalENBId, UDL) -> +enb_add_metrics(GlobalENBId, ULDL) -> %% counters may already exist, so catch exceptions here - catch exometer:new(?S1GW_CTR_GTPU_PACKETS(GlobalENBId, UDL), counter), - catch exometer:new(?S1GW_CTR_GTPU_BYTES_UE(GlobalENBId, UDL), counter), - catch exometer:new(?S1GW_CTR_GTPU_BYTES_TOTAL(GlobalENBId, UDL), counter). + catch exometer:new(?S1GW_CTR_GTPU_PACKETS(GlobalENBId, ULDL), counter), + catch exometer:new(?S1GW_CTR_GTPU_BYTES_UE(GlobalENBId, ULDL), counter), + catch exometer:new(?S1GW_CTR_GTPU_BYTES_TOTAL(GlobalENBId, ULDL), counter). --spec enb_set_addr(Addr, ES, S0) -> {Reply, S1} - when Addr :: string(), +-spec enb_set_addr(ULDLAddr, ES, S0) -> {Reply, S1} + when ULDLAddr :: uldl_addr(), ES :: enb_state(), S0 :: #state{}, S1 :: #state{}, Reply :: ok | {error, term()}. -enb_set_addr(Addr, %% given Addr matches stored Addr +enb_set_addr({ULDL, Addr}, #{genb_id := GlobalENBId, - addr := Addr, - pid := Pid}, S) -> - ?LOG_DEBUG("eNB @ ~p (pid ~p, ~p): address is already known", - [Addr, Pid, GlobalENBId]), - {ok, S}; - -enb_set_addr(NewAddr, %% given NewAddr differs from stored Addr - #{genb_id := GlobalENBId, - addr := Addr, - pid := Pid}, S) -> - ?LOG_ERROR("eNB @ ~p (pid ~p, ~p): address (~p) mismatch?!?", - [Addr, Pid, GlobalENBId, NewAddr]), - {{error, addr_mismatch}, S}; - -enb_set_addr(Addr, #{genb_id := GlobalENBId, - pid := Pid} = ES0, + pid := Pid} = ES0, #state{cfg = Cfg, registry = R0} = S) -> - ?LOG_DEBUG("eNB @ ~p (pid ~p, ~p): " - "address indicated, creating NFT rules/counters", - [Addr, Pid, GlobalENBId]), - case enb_add_nft_counters(ES0#{addr => Addr}, Cfg) of - {ok, ES1} -> + case enb_add_nft_counter({ULDL, Addr}, GlobalENBId, Cfg) of + {ok, Handle} -> + ?LOG_INFO("eNB (pid ~p, ~p): NFT rules/counters created for ~p", + [Pid, GlobalENBId, ULDL]), + %% store an updated eNB state to the registry + ES1 = ES0#{ULDL => #{uldl => ULDL, + addr => Addr, + handle => Handle, + ctr => #ctr{}}}, R1 = dict:store(Pid, ES1, R0), - ?LOG_INFO("eNB @ ~p (pid ~p, ~p): " - "NFT rules/counters created (KPI ready)", - [Addr, Pid, GlobalENBId]), {ok, S#state{registry = R1}}; {error, Error} -> - ?LOG_ERROR("eNB @ ~p (pid ~p, ~p): " - "creating NFT rules/counters failed: ~p", - [Addr, Pid, GlobalENBId, Error]), + ?LOG_ERROR("eNB (pid ~p, ~p): creating NFT rules/counters failed: ~p", + [Pid, GlobalENBId, Error]), {{error, Error}, S} end. --spec enb_add_nft_counters(ES0, Cfg) -> {ok, ES1} | {error, term()} - when ES0 :: enb_state(), - ES1 :: enb_state(), - Cfg :: cfg(). -enb_add_nft_counters(#{addr := Addr, - genb_id := GlobalENBId} = ES0, - #{table_name := TName}) -> - RUL = [nft_expr_match_ip_saddr(Addr, ?OP_EQ), - nft_expr_counter("ul-" ++ GlobalENBId)], - RDL = [nft_expr_match_ip_daddr(Addr, ?OP_EQ), - nft_expr_counter("dl-" ++ GlobalENBId)], - Cmds = [nft_cmd_add_counter(TName, "ul-" ++ GlobalENBId), - nft_cmd_add_counter(TName, "dl-" ++ GlobalENBId), - nft_cmd_add_rule(TName, "gtpu-ul", RUL), - nft_cmd_add_rule(TName, "gtpu-dl", RDL) +enb_nft_counter_name(ul, GlobalENBId) -> "ul-" ++ GlobalENBId; +enb_nft_counter_name(dl, GlobalENBId) -> "dl-" ++ GlobalENBId. + +enb_nft_rule_name(ul) -> "gtpu-ul"; +enb_nft_rule_name(dl) -> "gtpu-dl". + + +-spec enb_add_nft_counter(ULDLAddr, GlobalENBId, Cfg) -> {ok, Handle} | {error, term()} + when ULDLAddr :: uldl_addr(), + GlobalENBId :: string(), + Cfg :: cfg(), + Handle :: integer(). +enb_add_nft_counter({ULDL, Addr}, GlobalENBId, + #{table_name := TName}) -> + CName = enb_nft_counter_name(ULDL, GlobalENBId), + RName = enb_nft_rule_name(ULDL), + Rule = [nft_expr_match_ip_addr({ULDL, Addr}), + nft_expr_counter(CName)], + Cmds = [nft_cmd_add_counter(TName, CName), + nft_cmd_add_rule(TName, RName, Rule) ], case nft_exec(Cmds) of ok -> - ULH = nft_chain_last_handle(TName, "gtpu-ul"), - DLH = nft_chain_last_handle(TName, "gtpu-dl"), - %% update the eNB state with new info - ES1 = ES0#{handle_ul => ULH, - handle_dl => DLH, - ctr_ul => #ctr{}, - ctr_dl => #ctr{}}, - {ok, ES1}; + Handle = nft_chain_last_handle(TName, RName), + {ok, Handle}; {error, Error} -> ?LOG_ERROR("~p() failed: ~p", [?FUNCTION_NAME, Error]), {error, Error} end. --spec enb_del_nft_counters(enb_state(), cfg()) -> ok | {error, term()}. -enb_del_nft_counters(#{genb_id := GlobalENBId, - handle_ul := ULH, - handle_dl := DLH}, - #{table_name := TName}) -> - Cmds = [nft_cmd_del_rule(TName, "gtpu-ul", ULH), - nft_cmd_del_rule(TName, "gtpu-dl", DLH), - nft_cmd_del_counter(TName, "ul-" ++ GlobalENBId), - nft_cmd_del_counter(TName, "dl-" ++ GlobalENBId) +-spec enb_del_nft_counters(ES0, Cfg) -> ES1 + when Cfg :: cfg(), + ES0 :: enb_state(), + ES1 :: enb_state(). +enb_del_nft_counters(#{ul := ULS} = ES, Cfg) -> + enb_del_nft_counter(ULS, ES, Cfg), + enb_del_nft_counters(maps:remove(ul, ES), Cfg); + +enb_del_nft_counters(#{dl := DLS} = ES, Cfg) -> + enb_del_nft_counter(DLS, ES, Cfg), + enb_del_nft_counters(maps:remove(dl, ES), Cfg); + +%% no UL/DL state => nothing to delete +enb_del_nft_counters(ES, _Cfg) -> ES. + + +-spec enb_del_nft_counter(S, ES, Cfg) -> ok | {error, term()} + when S :: enb_uldl_state(), + ES :: enb_state(), + Cfg :: cfg(). +enb_del_nft_counter(#{uldl := ULDL, handle := Handle}, + #{genb_id := GlobalENBId, pid := Pid}, + #{table_name := TName}) -> + ?LOG_DEBUG("eNB (pid ~p, ~p): deleting NFT counter for ~p", + [Pid, GlobalENBId, ULDL]), + CName = enb_nft_counter_name(ULDL, GlobalENBId), + RName = enb_nft_rule_name(ULDL), + Cmds = [nft_cmd_del_rule(TName, RName, Handle), + nft_cmd_del_counter(TName, CName) ], case nft_exec(Cmds) of ok -> ok; {error, Error} -> ?LOG_ERROR("~p() failed: ~p", [?FUNCTION_NAME, Error]), {error, Error} - end; - -%% missing ULH/DLH => nothing to delete -enb_del_nft_counters(_ES, _Cfg) -> ok. + end. %% Parse the given list of NFT counters (result of nft_cmd_list_counters()). @@ -437,51 +462,59 @@ R1 :: registry(). %% (new) registry of eNBs report_nft_counters(Ctrs, R0) -> %% for each registered eNB, look-up and report UL/DL counters - dict:map(fun(_Pid, EnbState) -> enb_report_nft_counters(Ctrs, EnbState) end, R0). + dict:map(fun(_Pid, ES) -> enb_report_nft_counters(Ctrs, ES) end, R0). -spec enb_report_nft_counters(counters(), enb_state()) -> enb_state(). -enb_report_nft_counters(Ctrs, #{genb_id := GlobalENBId, - ctr_ul := ULC, - ctr_dl := DLC} = ES) -> - ?LOG_DEBUG("Reporting NFT counters for eNB ~p", [GlobalENBId]), - ES#{ctr_ul => report_nft_counter(Ctrs, {ul, GlobalENBId}, ULC), - ctr_dl => report_nft_counter(Ctrs, {dl, GlobalENBId}, DLC)}; +enb_report_nft_counters(Ctrs, #{ul := ULS, + dl := DLS} = ES) -> + %% report both UL and DL counters + ES#{ul => enb_report_nft_counters(Ctrs, ES, ULS), + dl => enb_report_nft_counters(Ctrs, ES, DLS)}; -%% missing ULH/DLH => nothing to report +enb_report_nft_counters(Ctrs, #{ul := ULS} = ES) -> + %% report UL counters only + ES#{ul => enb_report_nft_counters(Ctrs, ES, ULS)}; + +enb_report_nft_counters(Ctrs, #{dl := DLS} = ES) -> + %% report DL counters only + ES#{dl => enb_report_nft_counters(Ctrs, ES, DLS)}; + +%% no UL/DL state => nothing to report enb_report_nft_counters(_Ctrs, ES) -> ES. --spec report_nft_counter(Ctrs, {UDL, GlobalENBId}, C0) -> C1 +-spec enb_report_nft_counters(Ctrs, ES, S0) -> S1 when Ctrs :: counters(), - UDL :: ul | dl, - GlobalENBId :: string(), - C0 :: counter(), - C1 :: counter(). -report_nft_counter(Ctrs, {UDL, GlobalENBId}, C0) -> - CtrName = atom_to_list(UDL) ++ "-" ++ GlobalENBId, + ES :: enb_state(), + S0 :: enb_uldl_state(), + S1 :: enb_uldl_state(). +enb_report_nft_counters(Ctrs, + #{genb_id := GlobalENBId, pid := Pid}, + #{uldl := ULDL, ctr := C0} = S) -> + CtrName = enb_nft_counter_name(ULDL, GlobalENBId), case dict:find(CtrName, Ctrs) of {ok, C0} -> %% no diff, nothing to report - ?LOG_DEBUG("NFT counters (~p) for eNB ~p: ~p", - [UDL, GlobalENBId, C0]), - C0; + ?LOG_DEBUG("NFT counters (~p) for eNB (pid ~p, ~p): ~p", + [ULDL, Pid, GlobalENBId, C0]), + S; {ok, C1} -> %% XXX: assuming C1 (new) values >= C0 (cached) values - ?LOG_DEBUG("NFT counters (~p) for eNB ~p: ~p -> ~p", - [UDL, GlobalENBId, C0, C1]), - s1gw_metrics:ctr_inc(?S1GW_CTR_GTPU_PACKETS(GlobalENBId, UDL), + ?LOG_DEBUG("NFT counters (~p) for eNB (pid ~p, ~p): ~p -> ~p", + [ULDL, Pid, GlobalENBId, C0, C1]), + s1gw_metrics:ctr_inc(?S1GW_CTR_GTPU_PACKETS(GlobalENBId, ULDL), C1#ctr.packets - C0#ctr.packets), - s1gw_metrics:ctr_inc(?S1GW_CTR_GTPU_BYTES_UE(GlobalENBId, UDL), + s1gw_metrics:ctr_inc(?S1GW_CTR_GTPU_BYTES_UE(GlobalENBId, ULDL), C1#ctr.bytes_ue - C0#ctr.bytes_ue), - s1gw_metrics:ctr_inc(?S1GW_CTR_GTPU_BYTES_TOTAL(GlobalENBId, UDL), + s1gw_metrics:ctr_inc(?S1GW_CTR_GTPU_BYTES_TOTAL(GlobalENBId, ULDL), C1#ctr.bytes_total - C0#ctr.bytes_total), - C1; + S#{ctr => C1}; error -> %% no counters for this eNB (yet?) - ?LOG_DEBUG("NFT counters (~p) for eNB ~p: nope", - [UDL, GlobalENBId]), - C0 + ?LOG_DEBUG("NFT counters (~p) for eNB (pid ~p, ~p): nope", + [ULDL, Pid, GlobalENBId]), + S end. @@ -591,6 +624,14 @@ nft_expr_match_payload({"udp", "dport"}, Port, Op). +-spec nft_expr_match_ip_addr(uldl_addr()) -> map(). +nft_expr_match_ip_addr({ul, Addr}) -> + nft_expr_match_ip_saddr(Addr, ?OP_EQ); + +nft_expr_match_ip_addr({dl, Addr}) -> + nft_expr_match_ip_daddr(Addr, ?OP_EQ). + + nft_expr_accept() -> #{accept => null}. diff --git a/src/s1ap_proxy.erl b/src/s1ap_proxy.erl index 85a2e80..6f63e4c 100644 --- a/src/s1ap_proxy.erl +++ b/src/s1ap_proxy.erl @@ -652,7 +652,8 @@ 'transportLayerAddress' = TLA_In, 'gTP-TEID' = << TEID_In:32/big >>} = C0, S) -> %% indicate eNB's address to the enft_kpi module - enft_kpi:enb_set_addr(tla_str(TLA_In)), + enft_kpi:enb_set_addr({ul, tla_str(TLA_In)}), + enft_kpi:enb_set_addr({dl, tla_str(TLA_In)}), %% poke E-RAB FSM case erab_fsm_find(ERABId, S) of {ok, Pid} -> @@ -941,7 +942,8 @@ 'transportLayerAddress' = TLA_In, 'gTP-TEID' = << TEID_In:32/big >>} = C0, S) -> %% indicate eNB's address to the enft_kpi module - enft_kpi:enb_set_addr(tla_str(TLA_In)), + enft_kpi:enb_set_addr({ul, tla_str(TLA_In)}), + enft_kpi:enb_set_addr({dl, tla_str(TLA_In)}), %% poke E-RAB FSM case erab_fsm_find(ERABId, S) of {ok, Pid} -> -- To view, visit https://gerrit.osmocom.org/c/erlang/osmo-s1gw/+/40437?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: newchange Gerrit-Project: erlang/osmo-s1gw Gerrit-Branch: master Gerrit-Change-Id: Icd4a17790062bfcaf2bccb01fa94dcdb65c0872c Gerrit-Change-Number: 40437 Gerrit-PatchSet: 1 Gerrit-Owner: fixeria &lt;vyanitskiy(a)sysmocom.de&gt;