Change in osmo-bsc[master]: lchan: try to recover from ST_BORKEN after timeout

osmith has uploaded this change for review. ( https://gerrit.osmocom.org/c/osmo-bsc/+/30252 ) Change subject: lchan: try to recover from ST_BORKEN after timeout ...................................................................... lchan: try to recover from ST_BORKEN after timeout Change-Id: Ic4728b3efe843ea63e2a0b54b1ea8a925347484a --- M include/osmocom/bsc/lchan.h M include/osmocom/bsc/lchan_fsm.h M src/osmo-bsc/lchan_fsm.c M src/osmo-bsc/net_init.c M tests/timer.vty 5 files changed, 110 insertions(+), 0 deletions(-) git pull ssh://gerrit.osmocom.org:29418/osmo-bsc refs/changes/52/30252/1 diff --git a/include/osmocom/bsc/lchan.h b/include/osmocom/bsc/lchan.h index 4fcfa20..7f25acf 100644 --- a/include/osmocom/bsc/lchan.h +++ b/include/osmocom/bsc/lchan.h @@ -359,6 +359,8 @@ /* Timestamps and markers to track active state duration. */ struct timespec active_start; struct timespec active_stored; + /* How many times we tried to recover from ST_BORKEN, gets reset to 0 on success */ + uint8_t borken_recovery_attempts; }; #define GSM_LCHAN_SI(lchan, i) (void *)((lchan)->si.buf[i][0]) diff --git a/include/osmocom/bsc/lchan_fsm.h b/include/osmocom/bsc/lchan_fsm.h index cc231dc..f47c417 100644 --- a/include/osmocom/bsc/lchan_fsm.h +++ b/include/osmocom/bsc/lchan_fsm.h @@ -33,6 +33,8 @@ LCHAN_ST_WAIT_RF_RELEASE_ACK, LCHAN_ST_WAIT_AFTER_ERROR, LCHAN_ST_BORKEN, + LCHAN_ST_RECOVER_WAIT_ACTIV_ACK, /*< Attempt to recover from BORKEN: first try to activate the lchan */ + LCHAN_ST_RECOVER_WAIT_RF_RELEASE_ACK, /*< Attempt to recover from BORKEN: then try to release it */ }; enum lchan_fsm_event { diff --git a/src/osmo-bsc/lchan_fsm.c b/src/osmo-bsc/lchan_fsm.c index b1cef5d..f7ceec2 100644 --- a/src/osmo-bsc/lchan_fsm.c +++ b/src/osmo-bsc/lchan_fsm.c @@ -43,6 +43,8 @@ #include <osmocom/bsc/bsc_stats.h> #include <osmocom/bsc/lchan.h> +#define BORKEN_RECOVERY_ATTEMPTS_MAX 5 + static struct osmo_fsm lchan_fsm; struct gsm_lchan *lchan_fi_lchan(struct osmo_fsm_inst *fi) @@ -291,6 +293,9 @@ [LCHAN_ST_WAIT_AFTER_ERROR] = { .T=-3111 }, [LCHAN_ST_WAIT_RR_CHAN_MODE_MODIFY_ACK] = { .T=-13 }, [LCHAN_ST_WAIT_RSL_CHAN_MODE_MODIFY_ACK] = { .T=-14 }, + [LCHAN_ST_BORKEN] = { .T=-15 }, + [LCHAN_ST_RECOVER_WAIT_ACTIV_ACK] = { .T=-6 }, + [LCHAN_ST_RECOVER_WAIT_RF_RELEASE_ACK] = { .T=3111 }, }; /* Transition to a state, using the T timer defined in lchan_fsm_timeouts. @@ -337,6 +342,8 @@ [LCHAN_ST_BORKEN] = LCHAN_ST_BORKEN, [LCHAN_ST_WAIT_RR_CHAN_MODE_MODIFY_ACK] = LCHAN_ST_WAIT_RF_RELEASE_ACK, [LCHAN_ST_WAIT_RSL_CHAN_MODE_MODIFY_ACK] = LCHAN_ST_WAIT_RF_RELEASE_ACK, + [LCHAN_ST_RECOVER_WAIT_ACTIV_ACK] = LCHAN_ST_BORKEN, + [LCHAN_ST_RECOVER_WAIT_RF_RELEASE_ACK] = LCHAN_ST_BORKEN, }; #define lchan_fail(fmt, args...) lchan_fail_to(lchan_fsm_on_error[fi->state], fmt, ## args) @@ -1494,6 +1501,10 @@ if (prev_state != LCHAN_ST_BORKEN) osmo_stat_item_inc(osmo_stat_item_group_get_item(bts->bts_statg, BTS_STAT_LCHAN_BORKEN), 1); + if (lchan->borken_recovery_attempts == BORKEN_RECOVERY_ATTEMPTS_MAX) + LOG_LCHAN(lchan, LOGL_ERROR, "Reached BORKEN_RECOVERY_ATTEMPTS_MAX=%d, giving up\n", + BORKEN_RECOVERY_ATTEMPTS_MAX); + /* The actual action besides all the beancounting above */ lchan_reset(lchan); chan_counts_ts_update(lchan->ts); @@ -1552,6 +1563,69 @@ } } +static void lchan_fsm_recover_wait_activ_ack_onenter(struct osmo_fsm_inst *fi, uint32_t prev_state) +{ + int rc; + struct gsm_lchan *lchan = lchan_fi_lchan(fi); + + lchan->borken_recovery_attempts++; + LOG_LCHAN(lchan, LOGL_NOTICE, "attempting to recover from BORKEN lchan (%d/%d)\n", + lchan->borken_recovery_attempts, BORKEN_RECOVERY_ATTEMPTS_MAX); + + rc = rsl_tx_chan_activ(lchan, RSL_ACT_INTRA_NORM_ASS, 0); + if (rc) + lchan_fail("Tx Chan Activ failed: %s (%d)", strerror(-rc), rc); +} + +static void lchan_fsm_recover_wait_activ_ack(struct osmo_fsm_inst *fi, uint32_t event, void *data) +{ + struct gsm_lchan *lchan = lchan_fi_lchan(fi); + + switch (event) { + + case LCHAN_EV_RSL_CHAN_ACTIV_ACK: + lchan_fsm_state_chg(LCHAN_ST_RECOVER_WAIT_RF_RELEASE_ACK); + break; + + case LCHAN_EV_RSL_CHAN_ACTIV_NACK: + /* If an earlier lchan activ got through to the BTS, but the + * ACK did not get back to the BSC, it may still be active on + * the BTS side. Proceed to release it. */ + LOG_LCHAN(lchan, LOGL_NOTICE, "received NACK for activation of BORKEN lchan, assuming still active\n"); + lchan_fsm_state_chg(LCHAN_ST_RECOVER_WAIT_RF_RELEASE_ACK); + break; + + default: + OSMO_ASSERT(false); + } +} + +static void lchan_fsm_recover_wait_rf_release_ack_onenter(struct osmo_fsm_inst *fi, uint32_t prev_state) +{ + int rc; + struct gsm_lchan *lchan = lchan_fi_lchan(fi); + + rc = rsl_tx_rf_chan_release(lchan); + if (rc) + lchan_fail("Tx RSL RF Channel Release failed: %s (%d)\n", strerror(-rc), rc); +} + +static void lchan_fsm_recover_wait_rf_release_ack(struct osmo_fsm_inst *fi, uint32_t event, void *data) +{ + struct gsm_lchan *lchan = lchan_fi_lchan(fi); + switch (event) { + + case LCHAN_EV_RSL_RF_CHAN_REL_ACK: + LOG_LCHAN(lchan, LOGL_NOTICE, "successfully recovered BORKEN lchan\n"); + lchan->borken_recovery_attempts = 0; + lchan_fsm_state_chg(LCHAN_ST_UNUSED); + break; + + default: + OSMO_ASSERT(false); + } +} + #define S(x) (1 << (x)) static const struct osmo_fsm_state lchan_fsm_states[] = { @@ -1743,6 +1817,30 @@ | S(LCHAN_ST_WAIT_AFTER_ERROR) , }, + [LCHAN_ST_RECOVER_WAIT_ACTIV_ACK] { + .name = "RECOVER_WAIT_ACTIV_ACK", + .onenter = lchan_fsm_recover_wait_activ_ack_onenter, + .action = lchan_fsm_recover_wait_activ_ack, + .in_event_mask = 0 + | S(LCHAN_EV_RSL_CHAN_ACTIV_ACK) + | S(LCHAN_EV_RSL_CHAN_ACTIV_NACK) + , + .out_state_mask = 0 + | S(LCHAN_ST_BORKEN) + | S(LCHAN_ST_RECOVER_WAIT_RF_RELEASE_ACK) + , + }, + [LCHAN_ST_RECOVER_WAIT_RF_RELEASE_ACK] { + .name = "RECOVER_WAIT_RF_RELEASE_ACK", + .onenter = lchan_fsm_recover_wait_rf_release_ack_onenter, + .action = lchan_fsm_recover_wait_rf_release_ack, + .in_event_mask = 0 + | S(LCHAN_EV_RSL_RF_CHAN_REL_ACK) + , + .out_state_mask = 0 + | S(LCHAN_ST_UNUSED) + , + }, }; static const struct value_string lchan_fsm_event_names[] = { @@ -1814,6 +1912,11 @@ lchan_fsm_state_chg(LCHAN_ST_UNUSED); return 0; + case LCHAN_ST_BORKEN: + if (lchan->borken_recovery_attempts < BORKEN_RECOVERY_ATTEMPTS_MAX) + lchan_fsm_state_chg(LCHAN_ST_RECOVER_WAIT_ACTIV_ACK); + return 0; + default: lchan->release.in_error = true; lchan->release.rsl_error_cause = RSL_ERR_INTERWORKING; diff --git a/src/osmo-bsc/net_init.c b/src/osmo-bsc/net_init.c index 8f2b26c..28ce175 100644 --- a/src/osmo-bsc/net_init.c +++ b/src/osmo-bsc/net_init.c @@ -58,6 +58,7 @@ { .T=-12, .default_val=5, .desc="Timeout for obtaining TA after BSSLAP TA Request" }, { .T=-13, .default_val=5, .desc="Timeout for RR Channel Mode Modify ACK (BSC <-> MS)" }, { .T=-14, .default_val=5, .desc="Timeout for RSL Channel Mode Modify ACK (BSC <-> BTS)" }, + { .T=-15, .default_val=30, .desc="Wait time before trying to use a BORKEN timeslot again" }, { .T = -16, .default_val = 1000, .unit = OSMO_TDEF_MS, .desc = "Granularity for all_allocated:* rate counters: amount of milliseconds that one counter increment" " represents. See also X17, X18" }, diff --git a/tests/timer.vty b/tests/timer.vty index 04c9872..16baaf2 100644 --- a/tests/timer.vty +++ b/tests/timer.vty @@ -30,6 +30,7 @@ net: X12 = 5 s Timeout for obtaining TA after BSSLAP TA Request (default: 5 s) net: X13 = 5 s Timeout for RR Channel Mode Modify ACK (BSC <-> MS) (default: 5 s) net: X14 = 5 s Timeout for RSL Channel Mode Modify ACK (BSC <-> BTS) (default: 5 s) +net: X15 = 30 s Wait time before trying to use a BORKEN timeslot again (default: 30 s) net: X16 = 1000 ms Granularity for all_allocated:* rate counters: amount of milliseconds that one counter increment represents. See also X17, X18 (default: 1000 ms) net: X17 = 0 ms Rounding threshold for all_allocated:* rate counters: round up to the next counter increment after this many milliseconds. If set to half of X16 (or 0), employ the usual round() behavior: round up after half of a granularity period. If set to 1, behave like ceil(): already increment the counter immediately when all channels are allocated. If set >= X16, behave like floor(): only increment after a full X16 period of all channels being occupied. See also X16, X18 (default: 0 ms) net: X18 = 60000 ms Forget-sum period for all_allocated:* rate counters: after this amount of idle time, forget internally cumulated time remainders. Zero to always keep remainders. See also X16, X17. (default: 60000 ms) @@ -84,6 +85,7 @@ net: X12 = 5 s Timeout for obtaining TA after BSSLAP TA Request (default: 5 s) net: X13 = 5 s Timeout for RR Channel Mode Modify ACK (BSC <-> MS) (default: 5 s) net: X14 = 5 s Timeout for RSL Channel Mode Modify ACK (BSC <-> BTS) (default: 5 s) +net: X15 = 30 s Wait time before trying to use a BORKEN timeslot again (default: 30 s) net: X16 = 1000 ms Granularity for all_allocated:* rate counters: amount of milliseconds that one counter increment represents. See also X17, X18 (default: 1000 ms) net: X17 = 0 ms Rounding threshold for all_allocated:* rate counters: round up to the next counter increment after this many milliseconds. If set to half of X16 (or 0), employ the usual round() behavior: round up after half of a granularity period. If set to 1, behave like ceil(): already increment the counter immediately when all channels are allocated. If set >= X16, behave like floor(): only increment after a full X16 period of all channels being occupied. See also X16, X18 (default: 0 ms) net: X18 = 60000 ms Forget-sum period for all_allocated:* rate counters: after this amount of idle time, forget internally cumulated time remainders. Zero to always keep remainders. See also X16, X17. (default: 60000 ms) -- To view, visit https://gerrit.osmocom.org/c/osmo-bsc/+/30252 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-bsc Gerrit-Branch: master Gerrit-Change-Id: Ic4728b3efe843ea63e2a0b54b1ea8a925347484a Gerrit-Change-Number: 30252 Gerrit-PatchSet: 1 Gerrit-Owner: osmith &lt;osmith(a)sysmocom.de&gt; Gerrit-MessageType: newchange