[M] Change in osmo-ttcn3-hacks[master]: StatsD_Checker: Allow running without VTY support

pespin has uploaded this change for review. ( https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/37980?usp=email ) Change subject: StatsD_Checker: Allow running without VTY support ...................................................................... StatsD_Checker: Allow running without VTY support New features are added to the public API of StatsD_Checker which make it possible to use it... * without "stats reset": Feature to take snapshots (f_statsd_snapshot()) which can later be used to validate expectancies with values relative to the snapshot, using API f_statsd_expect_from_snapshot(). This way, one can do: """ var StatsDExpects statsd_exp := { /* relative expectancies here... */ }; var StatsDMetrics statsd_snapshot := f_statsd_snapshot(f_statsd_keys_from_expect(statsd_exp)); /* do some test stuff here changing the state of the IUT... */ f_statsd_expect_from_snapshot(statsd_exp, snapshot := statsd_snapshot); """ * without polling ("stats report"), aka with periodict reporting: New parameter wait_converge in f_statsd_expect(), which allows overcoming race conditions with StatsD server processing older incoming metrics due to periodic reporting. This feature also allows a test to wait until a state changes in the IUT. Change-Id: Ie1180a5b674504864309c3b9b11bfcf5256d9178 --- M library/StatsD_Checker.ttcnpp M library/StatsD_Types.ttcn 2 files changed, 217 insertions(+), 30 deletions(-) git pull ssh://gerrit.osmocom.org:29418/osmo-ttcn3-hacks refs/changes/80/37980/1 diff --git a/library/StatsD_Checker.ttcnpp b/library/StatsD_Checker.ttcnpp index 8f0feba..ecadee3 100644 --- a/library/StatsD_Checker.ttcnpp +++ b/library/StatsD_Checker.ttcnpp @@ -44,13 +44,23 @@ boolean mp_enable_stats := true; } +type record StatsDMetricKey { + MetricName name, + MetricType mtype +}; +type set of StatsDMetricKey StatsDMetricKeys; + +template (value) StatsDMetricKey ts_StatsDMetricKey(template (value) MetricName name, template (value) MetricType mtype) := { + name := name, + mtype := mtype +} + type record StatsDExpect { MetricName name, MetricType mtype, MetricValue min, MetricValue max }; - type set of StatsDExpect StatsDExpects; type enumerated StatsDResultType { @@ -78,10 +88,11 @@ } signature STATSD_reset(); -signature STATSD_expect(in StatsDExpects expects) return boolean; +signature STATSD_snapshot(in StatsDMetricKeys keys, in boolean since_last_snapshot) return StatsDMetrics; +signature STATSD_expect(in StatsDExpects expects, in boolean wait_converge, in boolean use_snapshot, in StatsDMetrics snapshot) return boolean; type port STATSD_PROC_PT procedure { - inout STATSD_reset, STATSD_expect; + inout STATSD_reset, STATSD_snapshot, STATSD_expect; } with {extension "internal"}; /* Expect templates and functions */ @@ -90,7 +101,12 @@ /* StatsD checker component */ function main(charstring statsd_host, integer statsd_port) runs on StatsD_Checker_CT { var StatsD_ConnHdlr vc_conn; + var StatsDMetricKeys keys; + var boolean since_last_snapshot; var StatsDExpects expects; + var boolean wait_converge; + var boolean use_snapshot; + var StatsDMetrics snapshot; var Result res; while (not mp_enable_stats) { @@ -126,26 +142,130 @@ "STATSD_reset not supported, StatsD_Checker was built without VTY support"); #endif } - [] STATSD_PROC.getcall(STATSD_expect:{?}) -> param(expects) sender vc_conn { - var boolean success := f_statsd_checker_expect(expects); - STATSD_PROC.reply(STATSD_expect:{expects} value success) to vc_conn; + [] STATSD_PROC.getcall(STATSD_snapshot:{?, ?}) -> param(keys, since_last_snapshot) sender vc_conn { + snapshot := f_statsd_checker_snapshot(keys, since_last_snapshot); + STATSD_PROC.reply(STATSD_snapshot:{keys, since_last_snapshot} value snapshot) to vc_conn; + } + [] STATSD_PROC.getcall(STATSD_expect:{?, ?, ?, ?}) -> param(expects, wait_converge, use_snapshot, snapshot) sender vc_conn { + var boolean success := f_statsd_checker_expect(expects, wait_converge, use_snapshot, snapshot); + STATSD_PROC.reply(STATSD_expect:{expects, wait_converge, use_snapshot, snapshot} value success) to vc_conn; } } } } - -/* Return false if the expectation doesn't match the metric, otherwise return true */ -private function f_compare_expect(StatsDMetric metric, StatsDExpect expect) return boolean { - if ((metric.name == expect.name) and (metric.mtype == expect.mtype) - and (metric.val >= expect.min) and (metric.val <= expect.max)) { - return true; - } else { - return false; +/* Updates "metrics" & "seen" with content from "it". Returns true if the metric becomes known (for first time) as a result. */ +private function f_statsd_metrics_update_value(inout StatsDMetrics metrics, inout Booleans seen, StatsDMetric it) return boolean +{ + for (var integer i := 0; i < lengthof(metrics); i := i + 1) { + log("Rx new metric ", it); + if (it.name != metrics[i].name or it.mtype != metrics[i].mtype) { + log("PESPIN: metric[",i,"] ", it, " doesn't match ", metrics[i]); + continue; + } + metrics[i] := it; + log("PESPIN: metric[",i,"] seen =", seen[i]); + if (seen[i]) { + return false; + } else { + seen[i] := true; + return true; + } } + return false; } -private function f_statsd_checker_metric_expects(StatsDExpects expects, StatsDMetric metric) + +private function f_statsd_checker_snapshot(StatsDMetricKeys keys, boolean since_last_snapshot := true) runs on StatsD_Checker_CT return StatsDMetrics { + var default t; + var StatsDMessage msg; + var StatsDMetrics metrics := {}; + var Booleans seen := {}; + var integer seen_remain := 0; + + for (var integer i := 0; i < lengthof(keys); i := i + 1) { + metrics := metrics & {valueof(ts_StatsDMetric(keys[i].name, 0, keys[i].mtype))}; + seen := seen & {false}; + seen_remain := seen_remain + 1; + } + + if (not since_last_snapshot) { + STATS.clear; + } + + T_statsd.start; + while (seen_remain > 0) { + var StatsD_RecvFrom rf; + alt { + [] STATS.receive(tr_StatsD_RecvFrom(?, ?)) -> value rf { + msg := rf.msg; + } + [] T_statsd.timeout { + for (var integer i := 0; i < lengthof(metrics); i := i + 1) { + /* We're still missing some expects, keep looking */ + if (not seen[i]) { + log("Timeout waiting for ", metrics[i].name); + } + } + Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail, + log2str("Timeout waiting for metrics: ", keys, seen)); + } + } + + for (var integer i := 0; i < lengthof(msg); i := i + 1) { + var StatsDMetric metric := msg[i]; + if (f_statsd_metrics_update_value(metrics, seen, metric)) { + seen_remain := seen_remain - 1; + } + } + } + T_statsd.stop; + + return metrics; +} + +private function get_val_from_snapshot(inout integer val, StatsDMetric metric, StatsDMetrics snapshot) return boolean +{ + for (var integer i := 0; i < lengthof(snapshot); i := i + 1) { + if (metric.name != snapshot[i].name or metric.mtype != snapshot[i].mtype) { + continue; + } + val := snapshot[i].val; + return true; + } + return false; +} + +/* Return false if the expectation doesn't match the metric, otherwise return true */ +private function f_compare_expect(StatsDMetric metric, + StatsDExpect expect, + boolean use_snapshot := false, + StatsDMetrics snapshot := {}) return boolean { + var integer val := 0; + if ((metric.name != expect.name) or (metric.mtype != expect.mtype)) { + return false; + } + if (use_snapshot) { + var integer prev_val := 0; + if (not get_val_from_snapshot(prev_val, metric, snapshot)) { + Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail, + log2str("Metric ", metric.name, " not found in snapshot ", snapshot)); + } + val := metric.val - prev_val; + } else { + val := metric.val; + } + + if ((val < expect.min) or (val > expect.max)) { + return false; + } + return true; +} + +private function f_statsd_checker_metric_expects(StatsDExpects expects, + StatsDMetric metric, + boolean use_snapshot := false, + StatsDMetrics snapshot := {}) return StatsDExpectResult { var StatsDExpectResult result := { kind := e_NotFound, @@ -157,15 +277,15 @@ if (exp.name != metric.name) { continue; } - if (not f_compare_expect(metric, exp)) { - log("EXP mismatch: ", metric, " vs ", exp); + if (not f_compare_expect(metric, exp, use_snapshot, snapshot)) { + log("EXP mismatch: ", metric, " vs exp ", exp, " | use_snapshot=", use_snapshot, ", snapshot=", snapshot); result := { kind := e_Mismatched, idx := i }; break; } else { - log("EXP match: ", metric, " vs ", exp); + log("EXP match: ", metric, " vs exp ", exp); result := { kind := e_Matched, idx := i @@ -176,7 +296,10 @@ return result; } -private function f_statsd_checker_expect(StatsDExpects expects) runs on StatsD_Checker_CT return boolean { +private function f_statsd_checker_expect(StatsDExpects expects, + boolean wait_converge := false, + boolean use_snapshot := false, + StatsDMetrics snapshot := {}) runs on StatsD_Checker_CT return boolean { var default t; var StatsDMessage msg; var StatsDExpectResult res; @@ -191,11 +314,13 @@ /* Dismiss any messages we might have skipped from the last report */ STATS.clear; + if (not use_snapshot) { #ifdef STATSD_HAVE_VTY - f_vty_transceive(STATSVTY, "stats report"); + f_vty_transceive(STATSVTY, "stats report"); #else - /* Assume caller knows previous state, eg. gauges may have been 0 due to IUT being reset */ + /* Assume caller knows previous state, eg. gauges may have been 0 due to IUT being reset */ #endif + } T_statsd.start; while (matched_remain > 0) { @@ -219,12 +344,15 @@ for (var integer i := 0; i < lengthof(msg); i := i + 1) { var StatsDMetric metric := msg[i]; - - res := f_statsd_checker_metric_expects(expects, metric); + res := f_statsd_checker_metric_expects(expects, metric, use_snapshot, snapshot); if (res.kind == e_NotFound) { continue; } if (res.kind == e_Mismatched) { + if (wait_converge and not matched[res.idx]) { + log("Waiting convergence: Ignoring metric mismatch metric=", metric, " expect=", expects[res.idx]) + continue; + } log("Metric: ", metric); log("Expect: ", expects[res.idx]); setverdict(fail, "Metric failed expectation ", metric, " vs ", expects[res.idx]); @@ -239,7 +367,6 @@ } } } - T_statsd.stop; return true; } @@ -263,15 +390,53 @@ } } -function f_statsd_expect(StatsDExpects expects) runs on StatsD_ConnHdlr return boolean { +/* Useful to automatically generate param for f_statsd_snapshot() from StatsDExpects used in f_statsd_expect_from_snapshot() */ +function f_statsd_keys_from_expect(StatsDExpects expects) return StatsDMetricKeys +{ + var StatsDMetricKeys keys := {} + for (var integer i := 0; i < lengthof(expects); i := i + 1) { + keys := keys & {valueof(ts_StatsDMetricKey(expects[i].name, expects[i].mtype))} + } + return keys; +} + +/* Retrieve current values obtained at statsd server. +* If since_last_snapshot is false, then clear the received packets in port. */ +function f_statsd_snapshot(StatsDMetricKeys keys, boolean since_last_snapshot := true) runs on StatsD_ConnHdlr return StatsDMetrics { + var StatsDMetrics snapshot; + if (not mp_enable_stats) { + return {}; + } + + STATSD_PROC.call(STATSD_snapshot:{keys, since_last_snapshot}) { + [] STATSD_PROC.getreply(STATSD_snapshot:{keys, since_last_snapshot}) -> value snapshot; + } + return snapshot; +} + +function f_statsd_expect(StatsDExpects expects, boolean wait_converge := false) runs on StatsD_ConnHdlr return boolean { var boolean res; if (not mp_enable_stats) { return true; } - STATSD_PROC.call(STATSD_expect:{expects}) { - [] STATSD_PROC.getreply(STATSD_expect:{expects}) -> value res; + STATSD_PROC.call(STATSD_expect:{expects, wait_converge, false, {}}) { + [] STATSD_PROC.getreply(STATSD_expect:{expects, wait_converge, false, {}}) -> value res; + } + return res; +} + +function f_statsd_expect_from_snapshot(StatsDExpects expects, boolean wait_converge := false, + StatsDMetrics snapshot := {}) runs on StatsD_ConnHdlr return boolean { + var boolean res; + + if (not mp_enable_stats) { + return true; + } + + STATSD_PROC.call(STATSD_expect:{expects, wait_converge, true, snapshot}) { + [] STATSD_PROC.getreply(STATSD_expect:{expects, wait_converge, true, snapshot}) -> value res; } return res; } diff --git a/library/StatsD_Types.ttcn b/library/StatsD_Types.ttcn index 8938571..e60b8e8 100644 --- a/library/StatsD_Types.ttcn +++ b/library/StatsD_Types.ttcn @@ -33,6 +33,7 @@ MetricType mtype, MetricSampleRate srate optional }; +type set of StatsDMetric StatsDMetrics; type record of StatsDMetric StatsDMessage with { variant "SEPARATOR('\n')"; @@ -48,7 +49,8 @@ [0] := metric } -template (present) StatsDMetric tr_StatsDMetric(template (present) MetricName name, template (present) MetricValue val := ?, +template (present) StatsDMetric tr_StatsDMetric(template (present) MetricName name, + template (present) MetricValue val := ?, template (present) MetricType mtype) := { name := name, val := val, @@ -56,10 +58,30 @@ srate := * } -template (present) StatsDMetric tr_StatsDMetricCounter(template (present) MetricName name, template (present) MetricValue val := ?) := +template (present) StatsDMetric tr_StatsDMetricCounter(template (present) MetricName name, + template (present) MetricValue val := ?) := tr_StatsDMetric(name, val, "c"); -template (present) StatsDMetric tr_StatsDMetricGauge(template (present) MetricName name, template (present) MetricValue val := ?) := +template (present) StatsDMetric tr_StatsDMetricGauge(template (present) MetricName name, + template (present) MetricValue val := ?) := tr_StatsDMetric(name, val, "g"); +template (value) StatsDMetric ts_StatsDMetric(template (value) MetricName name, + template (value) MetricValue val := 0, + template (value) MetricType mtype := "c", + template (omit) MetricSampleRate srate := omit) := { + name := name, + val := val, + mtype := mtype, + srate := srate +} + +template (value) StatsDMetric ts_StatsDMetricCounter(template (value) MetricName name, + template (value) MetricValue val := 0) := + ts_StatsDMetric(name, val, "c"); + +template (value) StatsDMetric ts_StatsDMetricGauge(template (value) MetricName name, + template (value) MetricValue val := 0) := + ts_StatsDMetric(name, val, "g"); + } with { encode "TEXT" } -- To view, visit https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/37980?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: newchange Gerrit-Project: osmo-ttcn3-hacks Gerrit-Branch: master Gerrit-Change-Id: Ie1180a5b674504864309c3b9b11bfcf5256d9178 Gerrit-Change-Number: 37980 Gerrit-PatchSet: 1 Gerrit-Owner: pespin &lt;pespin(a)sysmocom.de&gt;