[L] Change in osmo-ttcn3-hacks[master]: Introduce Prometheus_Checker module

pespin has uploaded this change for review. ( https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/38026?usp=email ) Change subject: Introduce Prometheus_Checker module ...................................................................... Introduce Prometheus_Checker module This module serves the same purpose as the existing StatsD_Checker. It will be used in open5gs, which so far exports its metrics using Prometheus. Change-Id: Iec5544ba74978918f1bddba12971f69a1824683e --- A library/Prometheus_Checker.ttcn 1 file changed, 408 insertions(+), 0 deletions(-) git pull ssh://gerrit.osmocom.org:29418/osmo-ttcn3-hacks refs/changes/26/38026/1 diff --git a/library/Prometheus_Checker.ttcn b/library/Prometheus_Checker.ttcn new file mode 100644 index 0000000..061b0fd --- /dev/null +++ b/library/Prometheus_Checker.ttcn @@ -0,0 +1,408 @@ +module Prometheus_Checker { + +/* (C) 2024 by sysmocom s.f.m.c. GmbH &lt;info(a)sysmocom.de&gt; + * All rights reserved. + * + * Author: Pau Espin Pedrol &lt;pespin(a)sysmocom.de&gt; + * + * Released under the terms of GNU General Public License, Version 2 or + * (at your option) any later version. + * SPDX-License-Identifier: GPL-2.0-or-later + */ + +import from Misc_Helpers all; +import from Socket_API_Definitions all; + +import from General_Types all; +import from Osmocom_Types all; + +import from HTTP_Adapter all; +import from HTTPmsg_Types all; + +const integer c_prometheus_default_http_port := 9090; + +type enumerated PrometheusMetricType { + COUNTER, + GAUGE +}; + +type record PrometheusMetricKey { + charstring name, + PrometheusMetricType mtype +}; +type set of PrometheusMetricKey PrometheusMetricKeys; + +type record PrometheusMetric { + PrometheusMetricKey key, + integer val +}; +type set of PrometheusMetric PrometheusMetrics; + +type record PrometheusExpect { + PrometheusMetricKey key, + integer min, + integer max +}; +type set of PrometheusExpect PrometheusExpects; + +modulepar { + boolean mp_enable_stats := true +} + +type enumerated PrometheusResultType { + e_Matched, + e_Mismatched, + e_NotFound +} + +type record PrometheusExpectResult { + PrometheusResultType kind, + integer idx +} + +type component Prometheus_Checker_CT extends http_CT { + var float g_tout_http := 5.0; +}; + +template (value) PrometheusMetricKey +ts_PrometheusMetricKey(template (value) charstring name, + template (value) PrometheusMetricType mtype) := { + name := name, + mtype := mtype +}; + +template (value) PrometheusMetric +ts_PrometheusMetric(template (value) charstring name, + template (value) PrometheusMetricType mtype, + template (value) integer val := 0) := { + key := ts_PrometheusMetricKey(name, mtype), + val := val +}; + +template (value) PrometheusExpect +ts_PrometheusExpect(template (value) charstring name, + template (value) PrometheusMetricType mtype, + template (value) integer min, + template (value) integer max) := { + key := ts_PrometheusMetricKey(name, mtype), + min := min, + max := max +}; + +function f_prometheus_init(charstring http_host, integer http_port := c_prometheus_default_http_port) runs on Prometheus_Checker_CT { + var HTTP_Adapter_Params http_adapter_pars := { + http_host := http_host, + http_port := http_port, + use_ssl := false + }; + f_http_init(http_adapter_pars); +} + +private function f_prometheus_metric_mtype_from_string(charstring str) return PrometheusMetricType +{ + var PrometheusMetricType mtype; + if (str == "counter") { + mtype := COUNTER; + } else if (str == "gauge") { + mtype:= GAUGE; + } else { + Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail, + log2str("Unknown Prometheus metric type: ", str)); + } + return mtype; +} + +private function f_prometheus_parse_http_response(charstring body) return PrometheusMetrics +{ + var PrometheusMetrics metrics := {}; + var Misc_Helpers.ro_charstring lines := f_str_split(body, "\n"); + for (var integer i := 0; i + 2 < lengthof(lines); i := i + 3) { + var PrometheusMetric it; + /* HELP line, example: "# HELP cx_rx_unknown Received Cx unknown messages" */ + if (not f_str_startswith(lines[i], "# HELP ")) { + Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail, + log2str("Failed parsing Prometheus HTTP response line: ", lines[i])); + } + + /* TYPE line, example: "# TYPE cx_rx_unknown counter" */ + if (not f_str_startswith(lines[i + 1], "# TYPE ")) { + Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail, + log2str("Failed parsing Prometheus HTTP response line: ", lines[i + 1])); + } + var Misc_Helpers.ro_charstring type_tokens := f_str_split(lines[i + 1], " "); + if (lengthof(type_tokens) < 4) { + Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail, + log2str("Failed parsing Prometheus HTTP response line: ", type_tokens)); + } + it.key.name := type_tokens[2]; + it.key.mtype := f_prometheus_metric_mtype_from_string(type_tokens[3]); + + /* Value line, example: "cx_rx_unknown 0" */ + if (not f_str_startswith(lines[i + 2], it.key.name)) { + Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail, + log2str("Failed parsing Prometheus HTTP response line: ", lines[i + 2])); + } + var Misc_Helpers.ro_charstring value_tokens := f_str_split(lines[i + 2], " "); + it.val := str2int(value_tokens[1]); + + metrics := metrics & { it }; + } + return metrics; +} + +private function f_prometheus_get_http_metrics() runs on Prometheus_Checker_CT return charstring +{ + var HTTPMessage http_resp; + f_http_tx_request(url := "/metrics", method := "GET", tout := g_tout_http); + http_resp := f_http_rx_response(tr_HTTP_Resp(200), tout := g_tout_http); + return http_resp.response.body; +} + +private function f_prometheus_get_metrics() runs on Prometheus_Checker_CT return PrometheusMetrics +{ + var PrometheusMetrics metrics; + var charstring str; + str := f_prometheus_get_http_metrics(); + metrics := f_prometheus_parse_http_response(str); + return metrics; +} + +/* Updates "metrics" & "seen" with content from "it". Returns true if the metric becomes known (for first time) as a result. */ +private function f_prometheus_metrics_update_value(inout PrometheusMetrics metrics, inout Booleans seen, PrometheusMetric it) return boolean +{ + for (var integer i := 0; i < lengthof(metrics); i := i + 1) { + if (it.key.name != metrics[i].key.name or it.key.mtype != metrics[i].key.mtype) { + continue; + } + metrics[i] := it; + if (seen[i]) { + return false; + } else { + seen[i] := true; + return true; + } + } + return false; +} + +/* Useful to automatically generate param for f_statsd_snapshot() from StatsDExpects used in f_statsd_expect_from_snapshot() */ +function f_prometheus_keys_from_expect(PrometheusExpects expects) return PrometheusMetricKeys +{ + var PrometheusMetricKeys keys := {} + for (var integer i := 0; i < lengthof(expects); i := i + 1) { + keys := keys & { expects[i].key } + } + return keys; +} + +function f_prometheus_snapshot(PrometheusMetricKeys keys, float time_out := 10.0) runs on Prometheus_Checker_CT return PrometheusMetrics { + var PrometheusMetrics rx_metrics; + var PrometheusMetrics metrics := {}; + var Booleans seen := {}; + var integer seen_remain := 0; + timer T_snapshot := time_out; + + if (not mp_enable_stats) { + return metrics; + } + + for (var integer i := 0; i < lengthof(keys); i := i + 1) { + metrics := metrics & {valueof(ts_PrometheusMetric(keys[i].name, keys[i].mtype, 0))}; + seen := seen & {false}; + seen_remain := seen_remain + 1; + } + + T_snapshot.start; + while (seen_remain > 0) { + if (not T_snapshot.running) { + for (var integer i := 0; i < lengthof(metrics); i := i + 1) { + /* We're still missing some expects, keep looking */ + if (not seen[i]) { + log("Timeout waiting for ", metrics[i].key.name); + } + } + Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail, + log2str("Timeout waiting for metrics: ", keys, seen)); + } + + rx_metrics := f_prometheus_get_metrics(); + + for (var integer i := 0; i < lengthof(rx_metrics); i := i + 1) { + var PrometheusMetric metric := rx_metrics[i]; + if (f_prometheus_metrics_update_value(metrics, seen, metric)) { + seen_remain := seen_remain - 1; + } + } + + if (seen_remain > 0) { + /* Wait 1 second before retrieving stats again: */ + f_sleep(1.0); + } + } + T_snapshot.stop; + + return metrics; +} + +private function f_compare_PrometheusMetricKey(PrometheusMetricKey a, PrometheusMetricKey b) return boolean { + return a.name == b.name and a.mtype == b.mtype; +} + +private function get_val_from_snapshot(inout integer val, PrometheusMetric metric, PrometheusMetrics snapshot) return boolean +{ + for (var integer i := 0; i < lengthof(snapshot); i := i + 1) { + if (not f_compare_PrometheusMetricKey(metric.key, snapshot[i].key)) { + continue; + } + val := snapshot[i].val; + return true; + } + return false; +} + + +/* Return false if the expectation doesn't match the metric, otherwise return true */ +private function f_compare_expect(PrometheusMetric metric, + PrometheusExpect expect, + boolean use_snapshot := false, + PrometheusMetrics snapshot := {}) return boolean { + var integer val := 0; + if (not f_compare_PrometheusMetricKey(metric.key, expect.key)) { + return false; + } + if (use_snapshot) { + var integer prev_val := 0; + if (not get_val_from_snapshot(prev_val, metric, snapshot)) { + Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail, + log2str("Metric ", metric.key, " not found in snapshot ", snapshot)); + } + val := metric.val - prev_val; + } else { + val := metric.val; + } + + if ((val < expect.min) or (val > expect.max)) { + return false; + } + return true; +} + + +private function f_prometheus_metric_expects(PrometheusExpects expects, + PrometheusMetric metric, + boolean use_snapshot := false, + PrometheusMetrics snapshot := {}) +return PrometheusExpectResult { + var PrometheusExpectResult result := { + kind := e_NotFound, + idx := -1 + }; + + for (var integer i := 0; i < lengthof(expects); i := i + 1) { + var PrometheusExpect exp := expects[i]; + if (exp.key.name != metric.key.name) { + continue; + } + if (not f_compare_expect(metric, exp, use_snapshot, snapshot)) { + log("EXP mismatch: ", metric, " vs exp ", exp, " | use_snapshot=", use_snapshot, ", snapshot=", snapshot); + result := { + kind := e_Mismatched, + idx := i + }; + break; + } else { + log("EXP match: ", metric, " vs exp ", exp); + result := { + kind := e_Matched, + idx := i + }; + break; + } + } + return result; +} + +private function f_prometheus_expect_ext(PrometheusExpects expects, + boolean wait_converge := false, + boolean use_snapshot := false, + PrometheusMetrics snapshot := {}, + float time_out := 10.0) +runs on Prometheus_Checker_CT return boolean { + var PrometheusMetrics rx_metrics; + var PrometheusExpectResult res; + var Booleans matched := {}; + var integer matched_remain := 0; + timer T_expect := time_out; + + for (var integer i := 0; i < lengthof(expects); i := i + 1) { + matched := matched & {false}; + matched_remain := matched_remain + 1; + } + + T_expect.start; + while (matched_remain > 0) { + if (not T_expect.running) { + for (var integer i := 0; i < lengthof(expects); i := i + 1) { + /* We're still missing some expects, keep looking */ + if (not matched[i]) { + log("Timeout waiting for ", expects[i].key, + " (min: ", expects[i].min, ", max: ", expects[i].max, ")"); + } + } + setverdict(fail, "Timeout waiting for metrics ", expects, matched); + return false; + } + + rx_metrics := f_prometheus_get_metrics(); + + for (var integer i := 0; i < lengthof(rx_metrics); i := i + 1) { + var PrometheusMetric metric := rx_metrics[i]; + res := f_prometheus_metric_expects(expects, metric, use_snapshot, snapshot); + if (res.kind == e_NotFound) { + continue; + } + if (res.kind == e_Mismatched) { + if (wait_converge and not matched[res.idx]) { + log("Waiting convergence: Ignoring metric mismatch metric=", metric, " expect=", expects[res.idx]) + continue; + } + log("Metric: ", metric); + log("Expect: ", expects[res.idx]); + setverdict(fail, "Metric failed expectation ", metric, " vs ", expects[res.idx]); + return false; + } + if (res.kind == e_Matched) { + if (not matched[res.idx]) { + matched[res.idx] := true; + matched_remain := matched_remain - 1; + } + continue; + } + } + + if (matched_remain > 0) { + /* Wait 1 second before retrieving stats again: */ + f_sleep(1.0); + } + } + + T_expect.stop; + return true; +} + +function f_prometheus_expect(PrometheusExpects expects, + boolean wait_converge := false, + float time_out := 10.0) +runs on Prometheus_Checker_CT return boolean { + return f_prometheus_expect_ext(expects, wait_converge, false, {}, time_out); +} + +function f_prometheus_expect_from_snapshot(PrometheusExpects expects, + boolean wait_converge := false, + PrometheusMetrics snapshot := {}, + float time_out := 10.0) +runs on Prometheus_Checker_CT return boolean { + return f_prometheus_expect_ext(expects, wait_converge, true, snapshot, time_out); +} + +} -- To view, visit https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/38026?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: newchange Gerrit-Project: osmo-ttcn3-hacks Gerrit-Branch: master Gerrit-Change-Id: Iec5544ba74978918f1bddba12971f69a1824683e Gerrit-Change-Number: 38026 Gerrit-PatchSet: 1 Gerrit-Owner: pespin &lt;pespin(a)sysmocom.de&gt;