fixeria has submitted this change. ( https://gerrit.osmocom.org/c/erlang/osmo-s1gw/+/42299?usp=email )
Change subject: enb_registry: rework handling of eNB property updates ......................................................................
enb_registry: rework handling of eNB property updates
The idea of an enb_event/0 containing the current MME connection state and the associated information (Global-eNB-ID, eNB/MME conn info) looked promising initially, however turned out to be impractical in light of ongoing MME pooling related changes.
* remove type enb_state/0 * rename type enb_event/0 -> enb_prop/0 * rename function enb_event/2 -> enb_update/2 (now private) * enb_prop/0: separate the state from other properties * enb_update/2: accept a list of enb_prop/0 - enb_proplist/0 * add typed notify_*() helpers that group related property updates, ensuring consistency and serving as the sole public call sites * notify_mme_connecting(): explicitly clears mme_conn_info * openapi: EnbItem.state reflects the actual enb_proxy FSM state
Change-Id: Ib5c5dedce729151cd7350390987fdd008b254ef4 Related: SYS#7052 --- M contrib/openapi.yaml M doc/osmo-s1gw-cli.md M priv/openapi.json M src/enb_proxy.erl M src/enb_registry.erl 5 files changed, 104 insertions(+), 63 deletions(-)
Approvals: pespin: Looks good to me, but someone else must approve Jenkins Builder: Verified fixeria: Looks good to me, approved osmith: Looks good to me, but someone else must approve
diff --git a/contrib/openapi.yaml b/contrib/openapi.yaml index 307ae1c..7f81608 100644 --- a/contrib/openapi.yaml +++ b/contrib/openapi.yaml @@ -292,8 +292,8 @@ $ref: '#/components/schemas/Pid' state: type: string - enum: [connecting, connected, s1setup] - description: Connection state + enum: [unknown, wait_s1setup_req, connecting, wait_s1setup_rsp, connected] + description: eNB proxy state uptime: type: integer description: Uptime (in seconds) diff --git a/doc/osmo-s1gw-cli.md b/doc/osmo-s1gw-cli.md index 4a0bd22..be5ffc3 100644 --- a/doc/osmo-s1gw-cli.md +++ b/doc/osmo-s1gw-cli.md @@ -174,18 +174,18 @@
``` OsmoS1GW# enb_list -| eNB handle | PID | Global-eNB-ID | State | eNB addr:port (aid) | MME addr:port (aid) | Uptime (s) | # E-RABs | -|--------------|-----------|-----------------|---------|-------------------------|-------------------------|--------------|------------| -| 0 | <0.699.0> | 001-01-0 | s1setup | 127.0.1.10:56767 (5706) | 127.0.2.10:36412 (5707) | 418 | 0 | -| 1 | <0.701.0> | 001-01-1 | s1setup | 127.0.1.10:54140 (5710) | 127.0.2.10:36412 (5711) | 33 | 3 | -| 2 | <0.703.0> | 001-01-2 | s1setup | 127.0.1.10:34076 (5714) | 127.0.2.10:36412 (5715) | 3600 | 20 | -| 3 | <0.705.0> | 001-01-3 | s1setup | 127.0.1.10:46501 (5718) | 127.0.2.10:36412 (5719) | 869 | 13 | -| 4 | <0.707.0> | 001-01-4 | s1setup | 127.0.1.10:35610 (5722) | 127.0.2.10:36412 (5723) | 18 | 0 | -| 5 | <0.709.0> | 001-01-5 | s1setup | 127.0.1.10:37610 (5726) | 127.0.2.10:36412 (5727) | 933 | 129 | -| 6 | <0.711.0> | 001-01-6 | s1setup | 127.0.1.10:58447 (5730) | 127.0.2.10:36412 (5741) | 535 | 6 | -| 7 | <0.713.0> | 001-01-7 | s1setup | 127.0.1.10:35842 (5732) | 127.0.2.10:36412 (5743) | 736 | 8 | -| 8 | <0.715.0> | 001-01-8 | s1setup | 127.0.1.10:57362 (5734) | 127.0.2.10:36412 (5745) | 521 | 99 | -| 9 | <0.717.0> | 001-01-9 | s1setup | 127.0.1.10:50955 (5736) | 127.0.2.10:36412 (5747) | 33 | 1 | +| eNB handle | PID | Global-eNB-ID | State | eNB addr:port (aid) | MME addr:port (aid) | Uptime (s) | # E-RABs | +|--------------|-----------|-----------------|-----------|-------------------------|-------------------------|--------------|------------| +| 0 | <0.699.0> | 001-01-0 | connected | 127.0.1.10:56767 (5706) | 127.0.2.10:36412 (5707) | 418 | 0 | +| 1 | <0.701.0> | 001-01-1 | connected | 127.0.1.10:54140 (5710) | 127.0.2.10:36412 (5711) | 33 | 3 | +| 2 | <0.703.0> | 001-01-2 | connected | 127.0.1.10:34076 (5714) | 127.0.2.10:36412 (5715) | 3600 | 20 | +| 3 | <0.705.0> | 001-01-3 | connected | 127.0.1.10:46501 (5718) | 127.0.2.10:36412 (5719) | 869 | 13 | +| 4 | <0.707.0> | 001-01-4 | connected | 127.0.1.10:35610 (5722) | 127.0.2.10:36412 (5723) | 18 | 0 | +| 5 | <0.709.0> | 001-01-5 | connected | 127.0.1.10:37610 (5726) | 127.0.2.10:36412 (5727) | 933 | 129 | +| 6 | <0.711.0> | 001-01-6 | connected | 127.0.1.10:58447 (5730) | 127.0.2.10:36412 (5741) | 535 | 6 | +| 7 | <0.713.0> | 001-01-7 | connected | 127.0.1.10:35842 (5732) | 127.0.2.10:36412 (5743) | 736 | 8 | +| 8 | <0.715.0> | 001-01-8 | connected | 127.0.1.10:57362 (5734) | 127.0.2.10:36412 (5745) | 521 | 99 | +| 9 | <0.717.0> | 001-01-9 | connected | 127.0.1.10:50955 (5736) | 127.0.2.10:36412 (5747) | 33 | 1 | ```
### `enb_info` @@ -219,7 +219,7 @@ | eNB handle | 8 | | PID | <0.715.0> | | Global-eNB-ID | 001-01-8 | -| State | s1setup | +| State | connected | | eNB addr:port (aid) | 127.0.1.10:57362 (5734) | | MME addr:port (aid) | 127.0.2.10:36412 (5745) | | Uptime (s) | 521 | @@ -235,7 +235,7 @@ | eNB handle | 8 | | PID | <0.715.0> | | Global-eNB-ID | 001-01-8 | -| State | s1setup | +| State | connected | | eNB addr:port (aid) | 127.0.1.10:57362 (5734) | | MME addr:port (aid) | 127.0.2.10:36412 (5745) | | Uptime (s) | 521 | diff --git a/priv/openapi.json b/priv/openapi.json index c13d20c..6bcefdb 100644 --- a/priv/openapi.json +++ b/priv/openapi.json @@ -432,11 +432,13 @@ "state": { "type": "string", "enum": [ + "unknown", + "wait_s1setup_req", "connecting", - "connected", - "s1setup" + "wait_s1setup_rsp", + "connected" ], - "description": "Connection state" + "description": "eNB proxy state" }, "uptime": { "type": "integer", diff --git a/src/enb_proxy.erl b/src/enb_proxy.erl index 4495e31..0aa00db 100644 --- a/src/enb_proxy.erl +++ b/src/enb_proxy.erl @@ -58,8 +58,8 @@ -include("S1AP-Constants.hrl").
--type conn_info() :: #{state := atom(), - handler := pid(), +%% XXX: replace/merge with mme_registry:mme_info()? +-type conn_info() :: #{handler := pid(), mme_addr := inet:ip_address(), mme_port := inet:port_number(), enb_aid := gen_sctp:assoc_id(), @@ -115,7 +115,6 @@
init([EnbConnInfo, MmeConnCfg]) -> {ok, EnbHandle} = enb_registry:enb_register(), - enb_registry:enb_event(EnbHandle, {connecting, EnbConnInfo}), {ok, Pid} = s1ap_proxy:start_link(self()), {ok, wait_s1setup_req, #state{enb_aid = maps:get(aid, EnbConnInfo), @@ -132,6 +131,8 @@ %% WAIT_S1SETUP_REQ state wait_s1setup_req(enter, _OldState, S) -> ?LOG_INFO("State enter: ~p", [?FUNCTION_NAME]), + enb_registry:notify_enb_connected(S#state.enb_handle, + S#state.enb_conn_info), {next_state, ?FUNCTION_NAME, S, [{state_timeout, 5_000, s1setup_req_timeout}]};
@@ -158,7 +159,7 @@ S#state{genb_id_str = GlobalENBId}), %% signal the Global-eNB-ID to other modules s1ap_proxy:set_genb_id(S#state.handler, GlobalENBId), - enb_registry:enb_event(S#state.enb_handle, {s1setup, GENBId}), + enb_registry:notify_genb_id(S#state.enb_handle, GENBId), gtpu_kpi_enb_register(S#state{genb_id_str = GlobalENBId}), {next_state, connecting, S#state{s1setup_req = Data, @@ -180,6 +181,7 @@ connecting(enter, OldState, #state{mme_conn_cfg = MmeConnCfg} = S) -> ?LOG_INFO("State change: ~p -> ~p", [OldState, ?FUNCTION_NAME]), + enb_registry:notify_mme_connecting(S#state.enb_handle), %% Initiate connection establishment with the MME {ok, Sock} = sctp_client:connect(MmeConnCfg), %% loop transition to enable state_timeout @@ -222,6 +224,7 @@ %% WAIT_S1SETUP_RSP state wait_s1setup_rsp(enter, OldState, S) -> ?LOG_INFO("State change: ~p -> ~p", [OldState, ?FUNCTION_NAME]), + enb_registry:notify_mme_wait_rsp(S#state.enb_handle), {next_state, ?FUNCTION_NAME, S, [{state_timeout, 5_000, s1setup_rsp_timeout}]};
@@ -288,11 +291,9 @@
%% CONNECTED state -connected(enter, OldState, - #state{enb_handle = Handle} = S) -> +connected(enter, OldState, S) -> ?LOG_INFO("State change: ~p -> ~p", [OldState, ?FUNCTION_NAME]), - MmeConnInfo = conn_info(?FUNCTION_NAME, S), - ok = enb_registry:enb_event(Handle, {?FUNCTION_NAME, MmeConnInfo}), + enb_registry:notify_mme_connected(S#state.enb_handle, conn_info(S)), {keep_state, S};
%% Handle an eNB -> MME data forwarding request (forward) @@ -338,9 +339,8 @@
%% Event handler for all states -handle_event(State, {call, From}, fetch_info, S) -> - Info = conn_info(State, S), - {keep_state_and_data, {reply, From, Info}}; +handle_event(_State, {call, From}, fetch_info, S) -> + {keep_state_and_data, {reply, From, conn_info(S)}};
handle_event(State, {call, From}, EventData, _S) -> ?LOG_ERROR("Unexpected call in state ~p: ~p", [State, EventData]), @@ -425,10 +425,9 @@ end.
--spec conn_info(atom(), state()) -> conn_info(). -conn_info(State, #state{mme_conn_cfg = MmeConnCfg} = S) -> - Info = #{state => State, - handler => S#state.handler, +-spec conn_info(state()) -> conn_info(). +conn_info(#state{mme_conn_cfg = MmeConnCfg} = S) -> + Info = #{handler => S#state.handler, mme_addr => maps:get(raddr, MmeConnCfg), mme_port => maps:get(rport, MmeConnCfg), enb_aid => S#state.enb_aid, diff --git a/src/enb_registry.erl b/src/enb_registry.erl index 3c9e8ef..61f0c59 100644 --- a/src/enb_registry.erl +++ b/src/enb_registry.erl @@ -43,7 +43,11 @@ -export([start_link/0, enb_register/0, enb_unregister/1, - enb_event/2, + notify_enb_connected/2, + notify_genb_id/2, + notify_mme_connecting/1, + notify_mme_wait_rsp/1, + notify_mme_connected/2, fetch_enb_info/1, fetch_enb_list/0, fetch_enb_list/1, @@ -59,13 +63,11 @@
-type enb_handle() :: non_neg_integer().
--type enb_state() :: connecting %% S1GW -> MME connection in progress - | connected %% S1GW -> MME connection established - | s1setup. %% S1 SETUP procedure completed - --type enb_event() :: {connecting, sctp_server:conn_info()} - | {connected, enb_proxy:conn_info()} - | {s1setup, s1ap_utils:genb_id()}. +-type enb_prop() :: {state, atom()} + | {enb_conn_info, sctp_server:conn_info()} + | {mme_conn_info, undefined | enb_proxy:conn_info()} + | {genb_id, s1ap_utils:genb_id()}. +-type enb_proplist() :: [enb_prop()].
-type enb_filter() :: {genb_id_str, string()} | {enb_sctp_aid, gen_sctp:assoc_id()} @@ -76,7 +78,7 @@ -type enb_info() :: #{handle := enb_handle(), %% unique identifier of this eNB pid := pid(), %% pid() of the registrant mon_ref := reference(), %% monitor() reference - state := enb_state(), %% connection state + state := atom(), %% enb_proxy FSM state reg_time := integer(), %% registration time (monotonic) uptime := non_neg_integer(), %% seconds since reg_time genb_id => s1ap_utils:genb_id(), %% Global-eNB-ID @@ -91,7 +93,6 @@ }).
-export_type([enb_handle/0, - enb_state/0, enb_info/0]).
@@ -114,9 +115,34 @@ gen_server:call(?MODULE, {?FUNCTION_NAME, Handle}).
--spec enb_event(enb_handle(), enb_event()) -> ok. -enb_event(Handle, Event) -> - gen_server:cast(?MODULE, {?FUNCTION_NAME, Handle, Event}). +%% eNB SCTP connection established; waiting for S1 SETUP REQUEST +-spec notify_enb_connected(enb_handle(), sctp_server:conn_info()) -> ok. +notify_enb_connected(Handle, ConnInfo) -> + enb_update(Handle, [{state, wait_s1setup_req}, {enb_conn_info, ConnInfo}]). + + +%% Global-eNB-ID is now known (from S1 SETUP REQUEST) +-spec notify_genb_id(enb_handle(), s1ap_utils:genb_id()) -> ok. +notify_genb_id(Handle, GENBId) -> + enb_update(Handle, [{genb_id, GENBId}]). + + +%% MME connection attempt in progress; clears any stale mme_conn_info +-spec notify_mme_connecting(enb_handle()) -> ok. +notify_mme_connecting(Handle) -> + enb_update(Handle, [{state, connecting}, {mme_conn_info, undefined}]). + + +%% MME SCTP link established; S1 SETUP REQUEST forwarded, awaiting response +-spec notify_mme_wait_rsp(enb_handle()) -> ok. +notify_mme_wait_rsp(Handle) -> + enb_update(Handle, [{state, wait_s1setup_rsp}]). + + +%% S1 SETUP complete; eNB is fully operational +-spec notify_mme_connected(enb_handle(), enb_proxy:conn_info()) -> ok. +notify_mme_connected(Handle, ConnInfo) -> + enb_update(Handle, [{state, connected}, {mme_conn_info, ConnInfo}]).
-spec fetch_enb_info(enb_handle() | pid()) -> {ok, enb_info()} | error. @@ -168,7 +194,7 @@ EnbInfo = #{handle => Handle, pid => Pid, mon_ref => MonRef, - state => connecting, + state => unknown, reg_time => erlang:monotonic_time(), uptime => 0}, ?LOG_INFO("eNB (handle=~p, pid ~p) registered", [Handle, Pid]), @@ -229,12 +255,12 @@ {reply, {error, not_implemented}, S}.
-handle_cast({enb_event, Handle, Event}, +handle_cast({enb_update, Handle, PropList}, #state{enbs = ENBs} = S) -> case maps:find(Handle, ENBs) of {ok, EnbInfo0} -> - ?LOG_INFO("eNB (handle=~p) event: ~p", [Handle, Event]), - EnbInfo1 = enb_handle_event(EnbInfo0, Event), + ?LOG_DEBUG("eNB (handle=~p) update: ~p", [Handle, PropList]), + EnbInfo1 = enb_update_proplist(EnbInfo0, PropList), {noreply, S#state{enbs = maps:update(Handle, EnbInfo1, ENBs)}}; error -> ?LOG_ERROR("eNB (handle=~p) is *not* registered", [Handle]), @@ -281,6 +307,11 @@ %% private API %% ------------------------------------------------------------------
+-spec enb_update(enb_handle(), enb_proplist()) -> ok. +enb_update(Handle, PropList) -> + gen_server:cast(?MODULE, {?FUNCTION_NAME, Handle, PropList}). + + -spec enb_metrics_register(string()) -> term(). enb_metrics_register(GlobalENBId) -> catch exometer:new(?S1GW_CTR_ENB_UPTIME(GlobalENBId), counter). @@ -293,24 +324,33 @@ enb_metrics_reset(_) -> ok.
--spec enb_handle_event(enb_info(), enb_event()) -> enb_info(). -enb_handle_event(EnbInfo, {connecting, ConnInfo}) -> - EnbInfo#{state => connecting, - enb_conn_info => ConnInfo}; +-spec enb_update_proplist(enb_info(), enb_proplist()) -> enb_info(). +enb_update_proplist(EnbInfo, PropList) -> + lists:foldl(fun enb_update_prop/2, EnbInfo, PropList).
-enb_handle_event(EnbInfo, {connected, ConnInfo}) -> - EnbInfo#{state => connected, - mme_conn_info => ConnInfo};
-enb_handle_event(EnbInfo, {s1setup, GENBId}) -> +-spec enb_update_prop(enb_info(), enb_prop()) -> enb_info(). +enb_update_prop({state, State}, EnbInfo) -> + EnbInfo#{state => State}; + +enb_update_prop({enb_conn_info, ConnInfo}, EnbInfo) -> + EnbInfo#{enb_conn_info => ConnInfo}; + +enb_update_prop({mme_conn_info, undefined}, EnbInfo) -> + maps:remove(mme_conn_info, EnbInfo); + +enb_update_prop({mme_conn_info, ConnInfo}, EnbInfo) -> + EnbInfo#{mme_conn_info => ConnInfo}; + +enb_update_prop({genb_id, GENBId}, EnbInfo) -> GlobalENBId = s1ap_utils:genb_id_str(GENBId), enb_metrics_register(GlobalENBId), - EnbInfo#{state => s1setup, - genb_id => GENBId, + EnbInfo#{genb_id => GENBId, genb_id_str => GlobalENBId};
-enb_handle_event(EnbInfo, Event) -> - ?LOG_ERROR("Unhandled event: ~p", [Event]), +enb_update_prop(Prop, EnbInfo) -> + ?LOG_ERROR("eNB (handle=~p) update: unknown property ~p", + [maps:get(handle, EnbInfo), Prop]), EnbInfo.