pespin has uploaded this change for review.
StatsD_Checker: Allow running without VTY support
New features are added to the public API of StatsD_Checker which make it
possible to use it...
* without "stats reset":
Feature to take snapshots (f_statsd_snapshot()) which can later be
used to validate expectancies with values relative to the snapshot,
using API f_statsd_expect_from_snapshot().
This way, one can do:
"""
var StatsDExpects statsd_exp := { /* relative expectancies here... */ };
var StatsDMetrics statsd_snapshot := f_statsd_snapshot(f_statsd_keys_from_expect(statsd_exp));
/* do some test stuff here changing the state of the IUT... */
f_statsd_expect_from_snapshot(statsd_exp, snapshot := statsd_snapshot);
"""
* without polling ("stats report"), aka with periodict reporting:
New parameter wait_converge in f_statsd_expect(), which allows
overcoming race conditions with StatsD server processing older incoming
metrics due to periodic reporting.
This feature also allows a test to wait until a state changes in the
IUT.
Change-Id: Ie1180a5b674504864309c3b9b11bfcf5256d9178
---
M library/StatsD_Checker.ttcnpp
M library/StatsD_Types.ttcn
2 files changed, 217 insertions(+), 30 deletions(-)
git pull ssh://gerrit.osmocom.org:29418/osmo-ttcn3-hacks refs/changes/80/37980/1
diff --git a/library/StatsD_Checker.ttcnpp b/library/StatsD_Checker.ttcnpp
index 8f0feba..ecadee3 100644
--- a/library/StatsD_Checker.ttcnpp
+++ b/library/StatsD_Checker.ttcnpp
@@ -44,13 +44,23 @@
boolean mp_enable_stats := true;
}
+type record StatsDMetricKey {
+ MetricName name,
+ MetricType mtype
+};
+type set of StatsDMetricKey StatsDMetricKeys;
+
+template (value) StatsDMetricKey ts_StatsDMetricKey(template (value) MetricName name, template (value) MetricType mtype) := {
+ name := name,
+ mtype := mtype
+}
+
type record StatsDExpect {
MetricName name,
MetricType mtype,
MetricValue min,
MetricValue max
};
-
type set of StatsDExpect StatsDExpects;
type enumerated StatsDResultType {
@@ -78,10 +88,11 @@
}
signature STATSD_reset();
-signature STATSD_expect(in StatsDExpects expects) return boolean;
+signature STATSD_snapshot(in StatsDMetricKeys keys, in boolean since_last_snapshot) return StatsDMetrics;
+signature STATSD_expect(in StatsDExpects expects, in boolean wait_converge, in boolean use_snapshot, in StatsDMetrics snapshot) return boolean;
type port STATSD_PROC_PT procedure {
- inout STATSD_reset, STATSD_expect;
+ inout STATSD_reset, STATSD_snapshot, STATSD_expect;
} with {extension "internal"};
/* Expect templates and functions */
@@ -90,7 +101,12 @@
/* StatsD checker component */
function main(charstring statsd_host, integer statsd_port) runs on StatsD_Checker_CT {
var StatsD_ConnHdlr vc_conn;
+ var StatsDMetricKeys keys;
+ var boolean since_last_snapshot;
var StatsDExpects expects;
+ var boolean wait_converge;
+ var boolean use_snapshot;
+ var StatsDMetrics snapshot;
var Result res;
while (not mp_enable_stats) {
@@ -126,26 +142,130 @@
"STATSD_reset not supported, StatsD_Checker was built without VTY support");
#endif
}
- [] STATSD_PROC.getcall(STATSD_expect:{?}) -> param(expects) sender vc_conn {
- var boolean success := f_statsd_checker_expect(expects);
- STATSD_PROC.reply(STATSD_expect:{expects} value success) to vc_conn;
+ [] STATSD_PROC.getcall(STATSD_snapshot:{?, ?}) -> param(keys, since_last_snapshot) sender vc_conn {
+ snapshot := f_statsd_checker_snapshot(keys, since_last_snapshot);
+ STATSD_PROC.reply(STATSD_snapshot:{keys, since_last_snapshot} value snapshot) to vc_conn;
+ }
+ [] STATSD_PROC.getcall(STATSD_expect:{?, ?, ?, ?}) -> param(expects, wait_converge, use_snapshot, snapshot) sender vc_conn {
+ var boolean success := f_statsd_checker_expect(expects, wait_converge, use_snapshot, snapshot);
+ STATSD_PROC.reply(STATSD_expect:{expects, wait_converge, use_snapshot, snapshot} value success) to vc_conn;
}
}
}
}
-
-/* Return false if the expectation doesn't match the metric, otherwise return true */
-private function f_compare_expect(StatsDMetric metric, StatsDExpect expect) return boolean {
- if ((metric.name == expect.name) and (metric.mtype == expect.mtype)
- and (metric.val >= expect.min) and (metric.val <= expect.max)) {
- return true;
- } else {
- return false;
+/* Updates "metrics" & "seen" with content from "it". Returns true if the metric becomes known (for first time) as a result. */
+private function f_statsd_metrics_update_value(inout StatsDMetrics metrics, inout Booleans seen, StatsDMetric it) return boolean
+{
+ for (var integer i := 0; i < lengthof(metrics); i := i + 1) {
+ log("Rx new metric ", it);
+ if (it.name != metrics[i].name or it.mtype != metrics[i].mtype) {
+ log("PESPIN: metric[",i,"] ", it, " doesn't match ", metrics[i]);
+ continue;
+ }
+ metrics[i] := it;
+ log("PESPIN: metric[",i,"] seen =", seen[i]);
+ if (seen[i]) {
+ return false;
+ } else {
+ seen[i] := true;
+ return true;
+ }
}
+ return false;
}
-private function f_statsd_checker_metric_expects(StatsDExpects expects, StatsDMetric metric)
+
+private function f_statsd_checker_snapshot(StatsDMetricKeys keys, boolean since_last_snapshot := true) runs on StatsD_Checker_CT return StatsDMetrics {
+ var default t;
+ var StatsDMessage msg;
+ var StatsDMetrics metrics := {};
+ var Booleans seen := {};
+ var integer seen_remain := 0;
+
+ for (var integer i := 0; i < lengthof(keys); i := i + 1) {
+ metrics := metrics & {valueof(ts_StatsDMetric(keys[i].name, 0, keys[i].mtype))};
+ seen := seen & {false};
+ seen_remain := seen_remain + 1;
+ }
+
+ if (not since_last_snapshot) {
+ STATS.clear;
+ }
+
+ T_statsd.start;
+ while (seen_remain > 0) {
+ var StatsD_RecvFrom rf;
+ alt {
+ [] STATS.receive(tr_StatsD_RecvFrom(?, ?)) -> value rf {
+ msg := rf.msg;
+ }
+ [] T_statsd.timeout {
+ for (var integer i := 0; i < lengthof(metrics); i := i + 1) {
+ /* We're still missing some expects, keep looking */
+ if (not seen[i]) {
+ log("Timeout waiting for ", metrics[i].name);
+ }
+ }
+ Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail,
+ log2str("Timeout waiting for metrics: ", keys, seen));
+ }
+ }
+
+ for (var integer i := 0; i < lengthof(msg); i := i + 1) {
+ var StatsDMetric metric := msg[i];
+ if (f_statsd_metrics_update_value(metrics, seen, metric)) {
+ seen_remain := seen_remain - 1;
+ }
+ }
+ }
+ T_statsd.stop;
+
+ return metrics;
+}
+
+private function get_val_from_snapshot(inout integer val, StatsDMetric metric, StatsDMetrics snapshot) return boolean
+{
+ for (var integer i := 0; i < lengthof(snapshot); i := i + 1) {
+ if (metric.name != snapshot[i].name or metric.mtype != snapshot[i].mtype) {
+ continue;
+ }
+ val := snapshot[i].val;
+ return true;
+ }
+ return false;
+}
+
+/* Return false if the expectation doesn't match the metric, otherwise return true */
+private function f_compare_expect(StatsDMetric metric,
+ StatsDExpect expect,
+ boolean use_snapshot := false,
+ StatsDMetrics snapshot := {}) return boolean {
+ var integer val := 0;
+ if ((metric.name != expect.name) or (metric.mtype != expect.mtype)) {
+ return false;
+ }
+ if (use_snapshot) {
+ var integer prev_val := 0;
+ if (not get_val_from_snapshot(prev_val, metric, snapshot)) {
+ Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail,
+ log2str("Metric ", metric.name, " not found in snapshot ", snapshot));
+ }
+ val := metric.val - prev_val;
+ } else {
+ val := metric.val;
+ }
+
+ if ((val < expect.min) or (val > expect.max)) {
+ return false;
+ }
+ return true;
+}
+
+private function f_statsd_checker_metric_expects(StatsDExpects expects,
+ StatsDMetric metric,
+ boolean use_snapshot := false,
+ StatsDMetrics snapshot := {})
return StatsDExpectResult {
var StatsDExpectResult result := {
kind := e_NotFound,
@@ -157,15 +277,15 @@
if (exp.name != metric.name) {
continue;
}
- if (not f_compare_expect(metric, exp)) {
- log("EXP mismatch: ", metric, " vs ", exp);
+ if (not f_compare_expect(metric, exp, use_snapshot, snapshot)) {
+ log("EXP mismatch: ", metric, " vs exp ", exp, " | use_snapshot=", use_snapshot, ", snapshot=", snapshot);
result := {
kind := e_Mismatched,
idx := i
};
break;
} else {
- log("EXP match: ", metric, " vs ", exp);
+ log("EXP match: ", metric, " vs exp ", exp);
result := {
kind := e_Matched,
idx := i
@@ -176,7 +296,10 @@
return result;
}
-private function f_statsd_checker_expect(StatsDExpects expects) runs on StatsD_Checker_CT return boolean {
+private function f_statsd_checker_expect(StatsDExpects expects,
+ boolean wait_converge := false,
+ boolean use_snapshot := false,
+ StatsDMetrics snapshot := {}) runs on StatsD_Checker_CT return boolean {
var default t;
var StatsDMessage msg;
var StatsDExpectResult res;
@@ -191,11 +314,13 @@
/* Dismiss any messages we might have skipped from the last report */
STATS.clear;
+ if (not use_snapshot) {
#ifdef STATSD_HAVE_VTY
- f_vty_transceive(STATSVTY, "stats report");
+ f_vty_transceive(STATSVTY, "stats report");
#else
- /* Assume caller knows previous state, eg. gauges may have been 0 due to IUT being reset */
+ /* Assume caller knows previous state, eg. gauges may have been 0 due to IUT being reset */
#endif
+ }
T_statsd.start;
while (matched_remain > 0) {
@@ -219,12 +344,15 @@
for (var integer i := 0; i < lengthof(msg); i := i + 1) {
var StatsDMetric metric := msg[i];
-
- res := f_statsd_checker_metric_expects(expects, metric);
+ res := f_statsd_checker_metric_expects(expects, metric, use_snapshot, snapshot);
if (res.kind == e_NotFound) {
continue;
}
if (res.kind == e_Mismatched) {
+ if (wait_converge and not matched[res.idx]) {
+ log("Waiting convergence: Ignoring metric mismatch metric=", metric, " expect=", expects[res.idx])
+ continue;
+ }
log("Metric: ", metric);
log("Expect: ", expects[res.idx]);
setverdict(fail, "Metric failed expectation ", metric, " vs ", expects[res.idx]);
@@ -239,7 +367,6 @@
}
}
}
-
T_statsd.stop;
return true;
}
@@ -263,15 +390,53 @@
}
}
-function f_statsd_expect(StatsDExpects expects) runs on StatsD_ConnHdlr return boolean {
+/* Useful to automatically generate param for f_statsd_snapshot() from StatsDExpects used in f_statsd_expect_from_snapshot() */
+function f_statsd_keys_from_expect(StatsDExpects expects) return StatsDMetricKeys
+{
+ var StatsDMetricKeys keys := {}
+ for (var integer i := 0; i < lengthof(expects); i := i + 1) {
+ keys := keys & {valueof(ts_StatsDMetricKey(expects[i].name, expects[i].mtype))}
+ }
+ return keys;
+}
+
+/* Retrieve current values obtained at statsd server.
+* If since_last_snapshot is false, then clear the received packets in port. */
+function f_statsd_snapshot(StatsDMetricKeys keys, boolean since_last_snapshot := true) runs on StatsD_ConnHdlr return StatsDMetrics {
+ var StatsDMetrics snapshot;
+ if (not mp_enable_stats) {
+ return {};
+ }
+
+ STATSD_PROC.call(STATSD_snapshot:{keys, since_last_snapshot}) {
+ [] STATSD_PROC.getreply(STATSD_snapshot:{keys, since_last_snapshot}) -> value snapshot;
+ }
+ return snapshot;
+}
+
+function f_statsd_expect(StatsDExpects expects, boolean wait_converge := false) runs on StatsD_ConnHdlr return boolean {
var boolean res;
if (not mp_enable_stats) {
return true;
}
- STATSD_PROC.call(STATSD_expect:{expects}) {
- [] STATSD_PROC.getreply(STATSD_expect:{expects}) -> value res;
+ STATSD_PROC.call(STATSD_expect:{expects, wait_converge, false, {}}) {
+ [] STATSD_PROC.getreply(STATSD_expect:{expects, wait_converge, false, {}}) -> value res;
+ }
+ return res;
+}
+
+function f_statsd_expect_from_snapshot(StatsDExpects expects, boolean wait_converge := false,
+ StatsDMetrics snapshot := {}) runs on StatsD_ConnHdlr return boolean {
+ var boolean res;
+
+ if (not mp_enable_stats) {
+ return true;
+ }
+
+ STATSD_PROC.call(STATSD_expect:{expects, wait_converge, true, snapshot}) {
+ [] STATSD_PROC.getreply(STATSD_expect:{expects, wait_converge, true, snapshot}) -> value res;
}
return res;
}
diff --git a/library/StatsD_Types.ttcn b/library/StatsD_Types.ttcn
index 8938571..e60b8e8 100644
--- a/library/StatsD_Types.ttcn
+++ b/library/StatsD_Types.ttcn
@@ -33,6 +33,7 @@
MetricType mtype,
MetricSampleRate srate optional
};
+type set of StatsDMetric StatsDMetrics;
type record of StatsDMetric StatsDMessage with {
variant "SEPARATOR('\n')";
@@ -48,7 +49,8 @@
[0] := metric
}
-template (present) StatsDMetric tr_StatsDMetric(template (present) MetricName name, template (present) MetricValue val := ?,
+template (present) StatsDMetric tr_StatsDMetric(template (present) MetricName name,
+ template (present) MetricValue val := ?,
template (present) MetricType mtype) := {
name := name,
val := val,
@@ -56,10 +58,30 @@
srate := *
}
-template (present) StatsDMetric tr_StatsDMetricCounter(template (present) MetricName name, template (present) MetricValue val := ?) :=
+template (present) StatsDMetric tr_StatsDMetricCounter(template (present) MetricName name,
+ template (present) MetricValue val := ?) :=
tr_StatsDMetric(name, val, "c");
-template (present) StatsDMetric tr_StatsDMetricGauge(template (present) MetricName name, template (present) MetricValue val := ?) :=
+template (present) StatsDMetric tr_StatsDMetricGauge(template (present) MetricName name,
+ template (present) MetricValue val := ?) :=
tr_StatsDMetric(name, val, "g");
+template (value) StatsDMetric ts_StatsDMetric(template (value) MetricName name,
+ template (value) MetricValue val := 0,
+ template (value) MetricType mtype := "c",
+ template (omit) MetricSampleRate srate := omit) := {
+ name := name,
+ val := val,
+ mtype := mtype,
+ srate := srate
+}
+
+template (value) StatsDMetric ts_StatsDMetricCounter(template (value) MetricName name,
+ template (value) MetricValue val := 0) :=
+ ts_StatsDMetric(name, val, "c");
+
+template (value) StatsDMetric ts_StatsDMetricGauge(template (value) MetricName name,
+ template (value) MetricValue val := 0) :=
+ ts_StatsDMetric(name, val, "g");
+
} with { encode "TEXT" }
To view, visit change 37980. To unsubscribe, or for help writing mail filters, visit settings.