lists.osmocom.org
Sign In Sign Up
Manage this list Sign In Sign Up

Keyboard Shortcuts

Thread View

  • j: Next unread message
  • k: Previous unread message
  • j a: Jump to all threads
  • j l: Jump to MailingList overview

gerrit-log

Thread Start a new thread
Download
Threads by month
  • ----- 2026 -----
  • November
  • October
  • September
  • August
  • July
  • June
  • May
  • April
  • March
  • February
  • January
  • ----- 2025 -----
  • December
  • November
  • October
  • September
  • August
  • July
  • June
  • May
  • April
  • March
  • February
  • January
  • ----- 2024 -----
  • December
  • November
  • October
  • September
  • August
  • July
  • June
  • May
  • April
  • March
  • February
  • January
  • ----- 2023 -----
  • December
  • November
  • October
  • September
  • August
  • July
  • June
  • May
  • April
  • March
  • February
  • January
  • ----- 2022 -----
  • December
  • November
  • October
  • September
  • August
  • July
  • June
  • May
  • April
  • March
  • February
  • January
gerrit-log@lists.osmocom.org

October 2026

  • 1 participants
  • 1426 discussions
[M] Change in osmo-ttcn3-hacks[master]: smf: Close all HTTP2_Server_Emulation tcp conns before exiting
by pespin 02 Oct '26

02 Oct '26
pespin has submitted this change. ( https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/43749?usp=email ) Change subject: smf: Close all HTTP2_Server_Emulation tcp conns before exiting ...................................................................... smf: Close all HTTP2_Server_Emulation tcp conns before exiting The IUT may leave the HTTP2 conn towards our emulated SCP (HTTP2 server) opened for a while in order to reuse it to send further HTTP2 requests. If we don't explicitly close the TCP conn when finishing the test, the IUT HTTP2 stack doesn't know the conn doesn't exist anymore, and upon next test it will try to re-use the same conn which will be answered by the system with a TCP RST to indicate the conn doesn't exist anymore. As a result of that TCP RST, the request fails and the test also fails. Change-Id: I6b4c40fa5a8ae494a0064bb492ffe0185a394659 --- M library/HTTP2_Server_Emulation.ttcn M smf/SMF_Tests.ttcn 2 files changed, 58 insertions(+), 1 deletion(-) Approvals: pespin: Looks good to me, approved fixeria: Looks good to me, but someone else must approve Jenkins Builder: Verified diff --git a/library/HTTP2_Server_Emulation.ttcn b/library/HTTP2_Server_Emulation.ttcn index 6c45287..ec167bb 100644 --- a/library/HTTP2_Server_Emulation.ttcn +++ b/library/HTTP2_Server_Emulation.ttcn @@ -72,6 +72,19 @@ f_HTTP2_Server_Conn_init(conn); } +private function f_HTTP2_Server_Conn_close_release(inout HTTP2_Server_Conn conn) +runs on HTTP2_Server_Emulation_CT { + var IPL4asp_Types.Result res; + + /* Announce to the peer what we handled until now and tell it we are exiting: */ + f_HTTP2_Server_Conn_tx(conn, ts_HTTP2_Frame_Goaway(conn.last_stream_id, 0 /* NO_ERROR */)); + + /* Close the socket, sends TCP FIN */ + res := HTTP2_CodecPort_CtrlFunct.f_IPL4_close(HTTP2, conn.conn_id, {tcp := {}}); + + f_HTTP2_Server_Conn_release(conn); +} + private function f_HTTP2_Server_Conn_str(in HTTP2_Server_Conn conn) return charstring { return "HTTP2_Server_Conn(" & int2str(conn.conn_id) & "," & log2str(conn.state) & ")"; } @@ -186,6 +199,16 @@ return f_conn_table_find_by_conn_id(-1); } +private function f_conn_table_close_conn_all() +runs on HTTP2_Server_Emulation_CT { + var integer i; + for (i := 0; i < sizeof(conn_table); i := i + 1) { + if (conn_table[i].conn_id != -1) { + f_HTTP2_Server_Conn_close_release(conn_table[i]); + } + } +} + private function f_vc_ConnHdlr_table_add(HTTP2_ConnHdlr vc_conn, charstring path_pattern) runs on HTTP2_Server_Emulation_CT { var integer i; @@ -512,6 +535,11 @@ f_vc_ConnHdlr_table_del(vc_conn, path_pattern); CONN_PROC.reply(HTTP2EM_unregister:{path_pattern}) to vc_conn; } + [] CONN_PROC.getcall(HTTP2EM_close_conn_all:{}) -> sender vc_conn { + log("HTTP2_Server: Closing all conns, requested by ", vc_conn); + f_conn_table_close_conn_all(); + CONN_PROC.reply(HTTP2EM_close_conn_all:{}) to vc_conn; + } } } @@ -528,10 +556,12 @@ signature HTTP2EM_register(in charstring path_pattern); signature HTTP2EM_unregister(in charstring path_pattern); +signature HTTP2EM_close_conn_all(); type port HTTP2_CONN_PROC_PT procedure { inout HTTP2EM_register; inout HTTP2EM_unregister; + inout HTTP2EM_close_conn_all; } with { extension "internal" }; @@ -559,4 +589,10 @@ } } +function f_http2_server_close_conn_all(integer idx := 0) runs on HTTP2_ConnHdlr { + HTTP2_CONN_PROC[idx].call(HTTP2EM_close_conn_all:{}) { + [] HTTP2_CONN_PROC[idx].getreply(HTTP2EM_close_conn_all:{}); + } +} + } diff --git a/smf/SMF_Tests.ttcn b/smf/SMF_Tests.ttcn index 4501520..861004c 100644 --- a/smf/SMF_Tests.ttcn +++ b/smf/SMF_Tests.ttcn @@ -68,7 +68,7 @@ /* main component, we typically have one per testcase */ -type component SMF_Test_CT extends GTP2_ConnHdlr, PFCP_ConnHdlr, HTTP2_CT { +type component SMF_Test_CT extends GTP2_ConnHdlr, PFCP_ConnHdlr, HTTP2_CT, HTTP2_ConnHdlr { var GTPv2_Emulation_CT vc_GTP2; port GTP2EM_PT TEIC0; var GTPv1U_Emulation_CT vc_GTP1U; @@ -252,8 +252,11 @@ vc_HTTP2_SRV_SCP := HTTP2_Server_Emulation_CT.create(id) alive; vc_HTTP2_SRV_SCP.start(HTTP2_Server_Emulation.main(http2_srv_cfg)); + + connect(self:HTTP2_CONN_PROC[HTTP2_CONN_SCP], vc_HTTP2_SRV_SCP:CONN_PROC); } + private function f_init(float guard_timeout := 60.0) runs on SMF_Test_CT { T_guard.start(guard_timeout); activate(as_Tguard()); @@ -270,6 +273,11 @@ } } +private function f_shutdown() runs on SMF_Test_CT { + f_http2_server_close_conn_all(); + setverdict(pass); +} + function f_start_handler(void_fn fn, template (omit) SessionPars pars_tmpl := omit) runs on SMF_Test_CT return SMF_Session_CT { var charstring id := testcasename(); @@ -346,6 +354,7 @@ f_init(); vc_conn := f_start_handler(refers(f_TC_createSession_deleteSession), pars); vc_conn.done; + f_shutdown(); } /* create an IPv6 session, then delete it again */ @@ -356,6 +365,7 @@ f_init(); vc_conn := f_start_handler(refers(f_TC_createSession_deleteSession), pars); vc_conn.done; + f_shutdown(); } /* create an IPv4v6 session, then delete it again */ @@ -366,6 +376,7 @@ f_init(); vc_conn := f_start_handler(refers(f_TC_createSession_deleteSession), pars); vc_conn.done; + f_shutdown(); } function f_TC_createSession_deleteSession_250(INT3b pdn_type) runs on SMF_Test_CT { @@ -382,6 +393,7 @@ for (i := 0; i < sizeof(vc_conn); i := i + 1) { vc_conn[i].done; } + f_shutdown(); } testcase TC_createSession_deleteSession_250() runs on SMF_Test_CT { @@ -408,6 +420,7 @@ f_init(); vc_conn := f_start_handler(refers(f_TC_deleteSession_unknown), pars); vc_conn.done; + f_shutdown(); } /* Test charging over Gy interface. */ @@ -462,6 +475,7 @@ f_init(); vc_conn := f_start_handler(refers(f_TC_gy_charging_cc_time), pars); vc_conn.done; + f_shutdown(); } /* Test Gy CCR rejected with CCA Result-Code DIAMETER_AUTHORIZATION_REJECTED (5003) */ @@ -499,6 +513,7 @@ f_init(); vc_conn := f_start_handler(refers(f_TC_gy_ccr_update_rejected), pars); vc_conn.done; + f_shutdown(); } /* create a session, expect it to succeed */ @@ -515,6 +530,7 @@ f_init(); vc_conn := f_start_handler(refers(f_TC_s2b_createSession_v4_noapco), pars); vc_conn.done; + f_shutdown(); } private function f_TC_s2b_createSession_v4_apco() runs on SMF_Session_CT { @@ -535,6 +551,7 @@ f_init(); vc_conn := f_start_handler(refers(f_TC_s2b_createSession_v4_apco), pars); vc_conn.done; + f_shutdown(); } /* create an IPv4 session from 5G AMF, then delete it again */ @@ -551,6 +568,7 @@ f_init(); vc_conn := f_start_handler(refers(f_TC_n11_createSession_deleteSession), pars); vc_conn.done; + f_shutdown(); } /* create an IPv6 session from 5G AMF, then delete it again */ @@ -562,6 +580,7 @@ f_init(); vc_conn := f_start_handler(refers(f_TC_n11_createSession_deleteSession), pars); vc_conn.done; + f_shutdown(); } /* create an IPv4v6 session from 5G AMF, then delete it again */ @@ -573,6 +592,7 @@ f_init(); vc_conn := f_start_handler(refers(f_TC_n11_createSession_deleteSession), pars); vc_conn.done; + f_shutdown(); } function f_TC_n11_createSession_deleteSession_250(INT3b pdn_type) runs on SMF_Test_CT { @@ -589,6 +609,7 @@ for (i := 0; i < sizeof(vc_conn); i := i + 1) { vc_conn[i].done; } + f_shutdown(); } testcase TC_n11_createSession_deleteSession_250() runs on SMF_Test_CT { -- To view, visit https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/43749?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: osmo-ttcn3-hacks Gerrit-Branch: master Gerrit-Change-Id: I6b4c40fa5a8ae494a0064bb492ffe0185a394659 Gerrit-Change-Number: 43749 Gerrit-PatchSet: 1 Gerrit-Owner: pespin <pespin(a)sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Reviewer: jolly <andreas(a)eversberg.eu> Gerrit-Reviewer: pespin <pespin(a)sysmocom.de>
1 0
0 0
[S] Change in osmo-ttcn3-hacks[master]: Misc_Helpers: Fix f_str_(oct_)split() skipping empty strings
by pespin 02 Oct '26

02 Oct '26
pespin has submitted this change. ( https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/43750?usp=email ) Change subject: Misc_Helpers: Fix f_str_(oct_)split() skipping empty strings ...................................................................... Misc_Helpers: Fix f_str_(oct_)split() skipping empty strings Default behavior should be to return proper splitting including empty chunks too, otherwise it's impossible under some scenarios to discern between input string where number of delimiters matter. A new skip_empty_chunks param is added (disabled by default) to allow skipping empty strings, since this is helpful under some situations and more efficient than splitting then re-creating the returned list removing empty strings (or skipping them in a loop). Change-Id: I69a20b4659e3c701bd3c9f7c72ea8127feddb006 --- M library/HTTP2_Functions.ttcn M library/Misc_Helpers.ttcn M library/Osmocom_VTY_Functions.ttcn M library/Prometheus_Checker.ttcn 4 files changed, 22 insertions(+), 13 deletions(-) Approvals: pespin: Looks good to me, approved Jenkins Builder: Verified fixeria: Looks good to me, but someone else must approve diff --git a/library/HTTP2_Functions.ttcn b/library/HTTP2_Functions.ttcn index 307938b..14f03ce 100644 --- a/library/HTTP2_Functions.ttcn +++ b/library/HTTP2_Functions.ttcn @@ -55,12 +55,9 @@ var HTTP2_header_list li := {}; var charstring data_str := oct2char(data); var charstring delim := "\r\n"; - var ro_charstring sp := f_str_split(data_str, delim); + var ro_charstring sp := f_str_split(data_str, delim, skip_empty_chunks := true); var charstring line_regexp := "([A-Za-z][A-Za-z0-9-]+):\s#()(?+)\s#()"; for (var integer i := 0; i < lengthof(sp); i := i + 1) { - if (sp[i] == "") { - continue; - } var charstring header_name := regexp(sp[i], line_regexp, 0); var charstring header_value := regexp(sp[i], line_regexp, 1); li := li & { valueof(ts_HTTP2_header_field(header_name, header_value)) }; @@ -77,6 +74,11 @@ /* 0..off is headers, off+lengthof(delim)..end is the data. * Furthermore, data contains an extra "\r\n" at the end. */ var integer off := f_strstr_oct(data, delim, 0); + if (off < 0) { + Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail, + log2str("f_strstr_oct(data=",data, ", delim=", delim, ", 0) returned negative")); + + } var octetstring header := substr(data, 0, off); off := off + lengthof(delim); @@ -110,8 +112,8 @@ log2str("Unexpected parsing multipart ", data, " into ", parts)); } - /* Parse each part and put it in the list: */ - for (var integer i := 0; i < lengthof(parts) - 1; i := i + 1) { + /* Parse each part and put it in the list: Start after 1st boundary, and avoid after last one (see check above) */ + for (var integer i := 1; i < lengthof(parts) - 1; i := i + 1) { li := li & { f_HTTP2_multipart_msg_parse_part(parts[i]) }; } return li; diff --git a/library/Misc_Helpers.ttcn b/library/Misc_Helpers.ttcn index 24d0590..eb63a2c 100644 --- a/library/Misc_Helpers.ttcn +++ b/library/Misc_Helpers.ttcn @@ -114,8 +114,11 @@ return new_str; } +/* Return a list of the substrings in string "str", using "delim" as the delimiter string. + * If skip_empty_chunks is true, empty substrings resulting from 2 consecutive "delim" tokens + * are NOT added to the returned list. This is useful for instance to skip empty lines while splitting. */ type record of charstring ro_charstring; -function f_str_split(charstring str, charstring delim := "\n") return ro_charstring +function f_str_split(charstring str, charstring delim := "\n", boolean skip_empty_chunks := false) return ro_charstring { var integer pos := 0; var ro_charstring parts := {}; @@ -128,6 +131,8 @@ } if (delim_pos > pos) { parts := parts & { substr(str, pos, delim_pos - pos) }; + } else if (not skip_empty_chunks and (delim_pos == pos)) { + parts := parts & { "" }; } pos := delim_pos + lengthof(delim); } @@ -136,7 +141,7 @@ /* Same as f_str_split() but using octetstring */ type record of octetstring ro_octetstring; -function f_str_oct_split(octetstring str, octetstring delim := ''O) return ro_octetstring +function f_str_oct_split(octetstring str, octetstring delim := ''O, boolean skip_empty_chunks := false) return ro_octetstring { var integer pos := 0; var ro_octetstring parts := {}; @@ -149,6 +154,8 @@ } if (delim_pos > pos) { parts := parts & { substr(str, pos, delim_pos - pos) }; + } else if (not skip_empty_chunks and (delim_pos == pos)) { + parts := parts & { ''O }; } pos := delim_pos + lengthof(delim); } diff --git a/library/Osmocom_VTY_Functions.ttcn b/library/Osmocom_VTY_Functions.ttcn index 7dd5c59..f99a13b 100644 --- a/library/Osmocom_VTY_Functions.ttcn +++ b/library/Osmocom_VTY_Functions.ttcn @@ -288,7 +288,7 @@ attempts := attempts - 1; var charstring ret := f_vty_transceive_ret(pt, show_cmd); - var Misc_Helpers.ro_charstring lines := f_str_split(ret); + var Misc_Helpers.ro_charstring lines := f_str_split(ret, skip_empty_chunks := true); var boolean ok := true; for (var integer i := 0; i < lengthof(object_strs); i := i + 1) { @@ -301,7 +301,7 @@ if (f_strstr(line, object_str) < 0) { continue; } - var Misc_Helpers.ro_charstring tokens := f_str_split(line, " "); + var Misc_Helpers.ro_charstring tokens := f_str_split(line, " ", skip_empty_chunks := true); if (lengthof(tokens) < 4) { continue; } diff --git a/library/Prometheus_Checker.ttcn b/library/Prometheus_Checker.ttcn index 061b0fd..870a728 100644 --- a/library/Prometheus_Checker.ttcn +++ b/library/Prometheus_Checker.ttcn @@ -115,7 +115,7 @@ private function f_prometheus_parse_http_response(charstring body) return PrometheusMetrics { var PrometheusMetrics metrics := {}; - var Misc_Helpers.ro_charstring lines := f_str_split(body, "\n"); + var Misc_Helpers.ro_charstring lines := f_str_split(body, "\n", skip_empty_chunks := true); for (var integer i := 0; i + 2 < lengthof(lines); i := i + 3) { var PrometheusMetric it; /* HELP line, example: "# HELP cx_rx_unknown Received Cx unknown messages" */ @@ -129,7 +129,7 @@ Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail, log2str("Failed parsing Prometheus HTTP response line: ", lines[i + 1])); } - var Misc_Helpers.ro_charstring type_tokens := f_str_split(lines[i + 1], " "); + var Misc_Helpers.ro_charstring type_tokens := f_str_split(lines[i + 1], " ", skip_empty_chunks := true); if (lengthof(type_tokens) < 4) { Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail, log2str("Failed parsing Prometheus HTTP response line: ", type_tokens)); @@ -142,7 +142,7 @@ Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail, log2str("Failed parsing Prometheus HTTP response line: ", lines[i + 2])); } - var Misc_Helpers.ro_charstring value_tokens := f_str_split(lines[i + 2], " "); + var Misc_Helpers.ro_charstring value_tokens := f_str_split(lines[i + 2], " ", skip_empty_chunks := true); it.val := str2int(value_tokens[1]); metrics := metrics & { it }; -- To view, visit https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/43750?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: osmo-ttcn3-hacks Gerrit-Branch: master Gerrit-Change-Id: I69a20b4659e3c701bd3c9f7c72ea8127feddb006 Gerrit-Change-Number: 43750 Gerrit-PatchSet: 1 Gerrit-Owner: pespin <pespin(a)sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Reviewer: pespin <pespin(a)sysmocom.de>
1 0
0 0
[M] Change in osmo-ttcn3-hacks[master]: Prometheus_Checker: Support labeled keys
by pespin 02 Oct '26

02 Oct '26
pespin has submitted this change. ( https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/43751?usp=email ) Change subject: Prometheus_Checker: Support labeled keys ...................................................................... Prometheus_Checker: Support labeled keys open5gs-smfd defines some labeled metrics, eg: """ fivegs_smffunction_sm_pdusessioncreationreq{plmnid="001001",snssai="1"} 1 """ Update Prometheus_Checker logic to support parsing and matching agains them by adding a new PrometheusMetricLabels type to the key. Change-Id: I1d50666b677faa3fcfb53f179ce2f5f97d9253c6 --- M library/Prometheus_Checker.ttcn 1 file changed, 84 insertions(+), 26 deletions(-) Approvals: pespin: Looks good to me, approved fixeria: Looks good to me, but someone else must approve Jenkins Builder: Verified diff --git a/library/Prometheus_Checker.ttcn b/library/Prometheus_Checker.ttcn index 870a728..3214873 100644 --- a/library/Prometheus_Checker.ttcn +++ b/library/Prometheus_Checker.ttcn @@ -16,6 +16,8 @@ import from General_Types all; import from Osmocom_Types all; +import from TCCConversion_Functions all; + import from HTTP_Adapter all; import from HTTPmsg_Types all; @@ -26,9 +28,16 @@ GAUGE }; +type record PrometheusMetricLabel { + charstring name, + charstring val +}; +type record of PrometheusMetricLabel PrometheusMetricLabels; + type record PrometheusMetricKey { charstring name, - PrometheusMetricType mtype + PrometheusMetricType mtype, + PrometheusMetricLabels labels optional }; type set of PrometheusMetricKey PrometheusMetricKeys; @@ -66,25 +75,29 @@ template (value) PrometheusMetricKey ts_PrometheusMetricKey(template (value) charstring name, - template (value) PrometheusMetricType mtype) := { + template (value) PrometheusMetricType mtype, + template (omit) PrometheusMetricLabels labels := omit) := { name := name, - mtype := mtype + mtype := mtype, + labels := labels }; template (value) PrometheusMetric ts_PrometheusMetric(template (value) charstring name, template (value) PrometheusMetricType mtype, + template (omit) PrometheusMetricLabels labels := omit, template (value) integer val := 0) := { - key := ts_PrometheusMetricKey(name, mtype), + key := ts_PrometheusMetricKey(name, mtype, labels), val := val }; template (value) PrometheusExpect ts_PrometheusExpect(template (value) charstring name, template (value) PrometheusMetricType mtype, + template (omit) PrometheusMetricLabels labels := omit, template (value) integer min, template (value) integer max) := { - key := ts_PrometheusMetricKey(name, mtype), + key := ts_PrometheusMetricKey(name, mtype, labels), min := min, max := max }; @@ -112,40 +125,89 @@ return mtype; } +/* Parse labels from eg. "plmnid="001001",snssai="1"" */ +private function prometheus_parse_metric_labels(charstring str) return PrometheusMetricLabels +{ + var PrometheusMetricLabels labels := {}; + var Misc_Helpers.ro_charstring tokens := f_str_split(str, ","); + for (var integer i := 0; i < lengthof(tokens); i := i + 1) { + var PrometheusMetricLabel l; + l.name := regexp(tokens[i],"(?+)=\"(?+)\"", 0); + l.val := regexp(tokens[i],"(?+)=\"(?+)\"", 1); + labels := labels & { l }; + } + return labels; +} + +/* Value line, example: "cx_rx_unknown 0" or "fivegs_smffunction_sm_pdusessioncreationreq{plmnid="001001",snssai="1"} 1" */ +private function f_prometheus_parse_metric_val_line(inout PrometheusMetric metric, charstring line) +{ + var integer pos0, pos1; + if (not f_str_startswith(line, metric.key.name)) { + Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail, + log2str("Failed parsing Prometheus HTTP response line: ", line)); + } + var Misc_Helpers.ro_charstring tokens := f_str_split(line, " ", skip_empty_chunks := true); + if (lengthof(tokens) != 2) { + Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail, + log2str("Failed parsing Prometheus HTTP response line: ", line)); + } + + pos0 := f_strstr(tokens[0], "{", 0); + if (pos0 > 0) { + /* Labeled key, parse labels: */ + pos1 := f_strstr(tokens[0], "}", pos0); + if (pos1 < 0) { + Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail, + log2str("Failed parsing Prometheus HTTP response line: ", line)); + } + var charstring labels_str := substr(tokens[0], pos0, pos1 - pos0); + metric.key.labels := prometheus_parse_metric_labels(labels_str); + } else { + metric.key.labels := omit; + } + + /* Finally, fill in value: */ + metric.val := str2int(tokens[1]); +} + private function f_prometheus_parse_http_response(charstring body) return PrometheusMetrics { var PrometheusMetrics metrics := {}; var Misc_Helpers.ro_charstring lines := f_str_split(body, "\n", skip_empty_chunks := true); - for (var integer i := 0; i + 2 < lengthof(lines); i := i + 3) { + var integer i := 0; + while (i < lengthof(lines)) { var PrometheusMetric it; + /* HELP line, example: "# HELP cx_rx_unknown Received Cx unknown messages" */ if (not f_str_startswith(lines[i], "# HELP ")) { Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail, log2str("Failed parsing Prometheus HTTP response line: ", lines[i])); } + i := i + 1; + if (i >= lengthof(lines)) { + break; + } /* TYPE line, example: "# TYPE cx_rx_unknown counter" */ - if (not f_str_startswith(lines[i + 1], "# TYPE ")) { + if (not f_str_startswith(lines[i], "# TYPE ")) { Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail, - log2str("Failed parsing Prometheus HTTP response line: ", lines[i + 1])); + log2str("Failed parsing Prometheus HTTP response line: ", lines[i])); } - var Misc_Helpers.ro_charstring type_tokens := f_str_split(lines[i + 1], " ", skip_empty_chunks := true); + var Misc_Helpers.ro_charstring type_tokens := f_str_split(lines[i], " ", skip_empty_chunks := true); if (lengthof(type_tokens) < 4) { Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail, log2str("Failed parsing Prometheus HTTP response line: ", type_tokens)); } it.key.name := type_tokens[2]; it.key.mtype := f_prometheus_metric_mtype_from_string(type_tokens[3]); + i := i + 1; - /* Value line, example: "cx_rx_unknown 0" */ - if (not f_str_startswith(lines[i + 2], it.key.name)) { - Misc_Helpers.f_shutdown(__BFILE__, __LINE__, fail, - log2str("Failed parsing Prometheus HTTP response line: ", lines[i + 2])); + while (i < lengthof(lines) and f_str_startswith(lines[i], it.key.name)) { + f_prometheus_parse_metric_val_line(it, lines[i]); + metrics := metrics & { it }; + i := i + 1; } - var Misc_Helpers.ro_charstring value_tokens := f_str_split(lines[i + 2], " ", skip_empty_chunks := true); - it.val := str2int(value_tokens[1]); - - metrics := metrics & { it }; } return metrics; } @@ -171,7 +233,7 @@ private function f_prometheus_metrics_update_value(inout PrometheusMetrics metrics, inout Booleans seen, PrometheusMetric it) return boolean { for (var integer i := 0; i < lengthof(metrics); i := i + 1) { - if (it.key.name != metrics[i].key.name or it.key.mtype != metrics[i].key.mtype) { + if (not match(it.key, metrics[i].key)) { continue; } metrics[i] := it; @@ -207,7 +269,7 @@ } for (var integer i := 0; i < lengthof(keys); i := i + 1) { - metrics := metrics & {valueof(ts_PrometheusMetric(keys[i].name, keys[i].mtype, 0))}; + metrics := metrics & {valueof(ts_PrometheusMetric(keys[i].name, keys[i].mtype, keys[i].labels, 0))}; seen := seen & {false}; seen_remain := seen_remain + 1; } @@ -244,14 +306,10 @@ return metrics; } -private function f_compare_PrometheusMetricKey(PrometheusMetricKey a, PrometheusMetricKey b) return boolean { - return a.name == b.name and a.mtype == b.mtype; -} - private function get_val_from_snapshot(inout integer val, PrometheusMetric metric, PrometheusMetrics snapshot) return boolean { for (var integer i := 0; i < lengthof(snapshot); i := i + 1) { - if (not f_compare_PrometheusMetricKey(metric.key, snapshot[i].key)) { + if (not match(metric.key, snapshot[i].key)) { continue; } val := snapshot[i].val; @@ -267,7 +325,7 @@ boolean use_snapshot := false, PrometheusMetrics snapshot := {}) return boolean { var integer val := 0; - if (not f_compare_PrometheusMetricKey(metric.key, expect.key)) { + if (not match(metric.key, expect.key)) { return false; } if (use_snapshot) { @@ -300,7 +358,7 @@ for (var integer i := 0; i < lengthof(expects); i := i + 1) { var PrometheusExpect exp := expects[i]; - if (exp.key.name != metric.key.name) { + if (not match(metric.key, exp.key)) { continue; } if (not f_compare_expect(metric, exp, use_snapshot, snapshot)) { -- To view, visit https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/43751?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: osmo-ttcn3-hacks Gerrit-Branch: master Gerrit-Change-Id: I1d50666b677faa3fcfb53f179ce2f5f97d9253c6 Gerrit-Change-Number: 43751 Gerrit-PatchSet: 1 Gerrit-Owner: pespin <pespin(a)sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Reviewer: pespin <pespin(a)sysmocom.de>
1 0
0 0
[M] Change in osmo-ttcn3-hacks[master]: smf: Validate metrics
by pespin 02 Oct '26

02 Oct '26
pespin has submitted this change. ( https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/43752?usp=email ) Change subject: smf: Validate metrics ...................................................................... smf: Validate metrics Change-Id: Idba67f84164a74c5778237cfd6fe753f8939dfef --- M smf/SMF_Session_CT.ttcn M smf/SMF_Session_CT_5G.ttcn M smf/SMF_Tests.default M smf/SMF_Tests.ttcn M smf/gen_links.sh M smf/regen_makefile.sh 6 files changed, 63 insertions(+), 4 deletions(-) Approvals: Jenkins Builder: Verified pespin: Looks good to me, approved fixeria: Looks good to me, but someone else must approve diff --git a/smf/SMF_Session_CT.ttcn b/smf/SMF_Session_CT.ttcn index 2eae5c8..bb617cb 100644 --- a/smf/SMF_Session_CT.ttcn +++ b/smf/SMF_Session_CT.ttcn @@ -23,6 +23,8 @@ import from DIAMETER_Templates all; import from DIAMETER_Emulation all; +import from Prometheus_Checker all; + import from NG_NAS_TypeDefs all; import from PFCP_Types all; @@ -49,7 +51,8 @@ /* per-session component; we typically have 1..N per testcase */ type component SMF_Session_CT extends GTP2_ConnHdlr, GTP1U_ConnHdlr, DIAMETER_ConnHdlr, - PFCP_ConnHdlr, MutexCT, HTTP2_CT, HTTP2_ConnHdlr { + PFCP_ConnHdlr, MutexCT, HTTP2_CT, HTTP2_ConnHdlr, + Prometheus_Checker_CT { var SessionPars g_pars; /* Address allocation */ @@ -301,6 +304,8 @@ runs on SMF_Session_CT { g_pars := valueof(pars); + f_prometheus_init(mp_smf_hostname, c_prometheus_default_http_port); + /* No need to register g_pars.pfcp_pars.gtp1u_pars.access.local.teid since we don't expect any message on it */ f_gtp1u_register_teid(g_pars.pfcp_pars.gtp1u_pars.core.local.teid); diff --git a/smf/SMF_Session_CT_5G.ttcn b/smf/SMF_Session_CT_5G.ttcn index 0008235..61dfc0a 100644 --- a/smf/SMF_Session_CT_5G.ttcn +++ b/smf/SMF_Session_CT_5G.ttcn @@ -60,6 +60,7 @@ runs on SMF_Session_CT { /* Register SCP HTTP2 sever, where IUT will send all SBI requests: */ f_http2_server_emu_register(f_http2_server_path_match(), idx := HTTP2_CONN_SCP); + f_http2_adapter_init(); } function f_handler_cleanup_5G() diff --git a/smf/SMF_Tests.default b/smf/SMF_Tests.default index 4c0b502..3a78848 100644 --- a/smf/SMF_Tests.default +++ b/smf/SMF_Tests.default @@ -1,6 +1,8 @@ [LOGGING] [TESTPORT_PARAMETERS] +*.TCP.noDelay := "yes" // turn off nagle +*.HTTP.use_notification_ASPs := "yes" [MODULE_PARAMETERS] diff --git a/smf/SMF_Tests.ttcn b/smf/SMF_Tests.ttcn index 861004c..9341de4 100644 --- a/smf/SMF_Tests.ttcn +++ b/smf/SMF_Tests.ttcn @@ -23,6 +23,8 @@ import from DIAMETER_ts32_299_Templates all; import from DIAMETER_Emulation all; +import from Prometheus_Checker all; + import from PFCP_Types all; import from PFCP_Emulation all; import from PFCP_Templates all; @@ -344,8 +346,21 @@ /* create an IPv4 session, then delete it again */ private function f_TC_createSession_deleteSession() runs on SMF_Session_CT { + var PrometheusExpects expects0 := valueof({ + ts_PrometheusExpect("ues_active", GAUGE, min := 0, max := 0), + ts_PrometheusExpect("gtp2_sessions_active", GAUGE, min := 0, max := 0) + }); + var PrometheusExpects expects1 := valueof({ + ts_PrometheusExpect("ues_active", GAUGE, min := 1, max := 1), + ts_PrometheusExpect("gtp2_sessions_active", GAUGE, min := 1, max := 1) + }); + f_prometheus_expect(expects0, wait_converge := true); + f_s5s8_create_session(); + f_prometheus_expect(expects1, wait_converge := true); + f_s5s8_delete_session(omit, Request_accepted); + f_prometheus_expect(expects0, wait_converge := true); setverdict(pass); } testcase TC_createSession_deleteSession() runs on SMF_Test_CT { @@ -379,6 +394,11 @@ f_shutdown(); } +private function f_TC_createSession_deleteSession_250_Session() runs on SMF_Session_CT { + f_s5s8_create_session(); + f_s5s8_delete_session(omit, Request_accepted); + setverdict(pass); +} function f_TC_createSession_deleteSession_250(INT3b pdn_type) runs on SMF_Test_CT { var SMF_Session_CT vc_conn[250]; var integer i; @@ -387,7 +407,7 @@ for (i := 0; i < sizeof(vc_conn); i := i + 1) { var SessionPars pars := f_SessionPars(idx := i, pdn_type := pdn_type); - vc_conn[i] := f_start_handler(refers(f_TC_createSession_deleteSession), pars); + vc_conn[i] := f_start_handler(refers(f_TC_createSession_deleteSession_250_Session), pars); } for (i := 0; i < sizeof(vc_conn); i := i + 1) { @@ -556,9 +576,21 @@ /* create an IPv4 session from 5G AMF, then delete it again */ private function f_TC_n11_createSession_deleteSession() runs on SMF_Session_CT { - f_n11_init(); + var PrometheusExpects expects0 := valueof({ + ts_PrometheusExpect("ues_active", GAUGE, min := 0, max := 0), + ts_PrometheusExpect("pfcp_sessions_active", GAUGE, min := 0, max := 0) + }); + var PrometheusExpects expects1 := valueof({ + ts_PrometheusExpect("ues_active", GAUGE, min := 1, max := 1), + ts_PrometheusExpect("pfcp_sessions_active", GAUGE, min := 1, max := 1) + }); + f_prometheus_expect(expects0, wait_converge := true); + f_n11_create_session(); + f_prometheus_expect(expects1, wait_converge := true); + f_n11_delete_session(); + f_prometheus_expect(expects0, wait_converge := true); setverdict(pass); } testcase TC_n11_createSession_deleteSession() runs on SMF_Test_CT { @@ -595,6 +627,11 @@ f_shutdown(); } +private function f_TC_n11_createSession_deleteSession_250_Session() runs on SMF_Session_CT { + f_n11_create_session(); + f_n11_delete_session(); + setverdict(pass); +} function f_TC_n11_createSession_deleteSession_250(INT3b pdn_type) runs on SMF_Test_CT { var SMF_Session_CT vc_conn[250]; var integer i; @@ -603,7 +640,7 @@ for (i := 0; i < sizeof(vc_conn); i := i + 1) { var SessionPars pars := f_SessionPars(idx := i, pdn_type := pdn_type); - vc_conn[i] := f_start_handler(refers(f_TC_n11_createSession_deleteSession), pars); + vc_conn[i] := f_start_handler(refers(f_TC_n11_createSession_deleteSession_250_Session), pars); } for (i := 0; i < sizeof(vc_conn); i := i + 1) { diff --git a/smf/gen_links.sh b/smf/gen_links.sh index 061f69a..7b7e678 100755 --- a/smf/gen_links.sh +++ b/smf/gen_links.sh @@ -54,6 +54,16 @@ FILES="DIAMETER_EncDec.cc" gen_links $DIR $FILES +# Needed by HTTP_Adapter (Prometheus_Checker.ttcn metrics): +DIR=$BASEDIR/titan.TestPorts.Common_Components.Abstract_Socket/src +FILES="Abstract_Socket.cc Abstract_Socket.hh " +gen_links $DIR $FILES + +# Needed by HTTP_Adapter (Prometheus_Checker.ttcn metrics): +DIR=$BASEDIR/titan.TestPorts.HTTPmsg/src +FILES="HTTPmsg_MessageLen.ttcn HTTPmsg_MessageLen_Function.cc HTTPmsg_PT.cc HTTPmsg_PT.hh HTTPmsg_PortType.ttcn HTTPmsg_Types.ttcn " +gen_links $DIR $FILES + DIR=$BASEDIR/titan.ProtocolModules.HTTP2/src FILES="HTTP2_EncDec.cc HTTP2_Types.ttcn" gen_links $DIR $FILES @@ -126,6 +136,7 @@ FILES+="HTTP2_CodecPort.ttcn HTTP2_CodecPort_CtrlFunct.ttcn HTTP2_CodecPort_CtrlFunctDef.cc HTTP2_Templates.ttcn HTTP2_Functions.ttcn HTTP2_Msg.ttcn HTTP2_Adapter.ttcn HTTP2_Server_Emulation.ttcn " FILES+="TS29503_Nudm_UECM_Templates.ttcn TS29503_Nudm_SDM_Templates.ttcn TS29502_Nsmf_PDUSession_Templates.ttcn TS29512_Npcf_SMPolicyControl_Templates.ttcn TS29518_Namf_Communication_Templates.ttcn " FILES+="NG_NAS_Osmo_Types.ttcn NG_NAS_Osmo_Templates.ttcn NG_NAS_Functions.ttcn " +FILES+="HTTP_Adapter.ttcn Prometheus_Checker.ttcn " gen_links $DIR $FILES gen_links_finish diff --git a/smf/regen_makefile.sh b/smf/regen_makefile.sh index 08a5b4a..b3d5863 100755 --- a/smf/regen_makefile.sh +++ b/smf/regen_makefile.sh @@ -5,6 +5,7 @@ FILES=" *.asn *.ttcn *.ttcnpp + Abstract_Socket.cc BSSGP_EncDec.cc common_ext.cc DIAMETER_CodecPort_CtrlFunctDef.cc @@ -14,6 +15,8 @@ GTPv1C_CodecPort_CtrlFunctDef.cc GTPv1U_CodecPort_CtrlFunctDef.cc GTPv2_CodecPort_CtrlFunctDef.cc + HTTPmsg_MessageLen_Function.cc + HTTPmsg_PT.cc HTTP2_EncDec.cc HTTP2_CodecPort_CtrlFunctDef.cc ICMPv6_EncDec.cc -- To view, visit https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/43752?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: osmo-ttcn3-hacks Gerrit-Branch: master Gerrit-Change-Id: Idba67f84164a74c5778237cfd6fe753f8939dfef Gerrit-Change-Number: 43752 Gerrit-PatchSet: 1 Gerrit-Owner: pespin <pespin(a)sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Reviewer: pespin <pespin(a)sysmocom.de>
1 0
0 0
[M] Change in osmo-ttcn3-hacks[master]: smf: Validate metrics
by pespin 02 Oct '26

02 Oct '26
pespin has posted comments on this change by pespin. ( https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/43752?usp=email ) Change subject: smf: Validate metrics ...................................................................... Patch Set 1: Code-Review+2 -- To view, visit https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/43752?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: comment Gerrit-Project: osmo-ttcn3-hacks Gerrit-Branch: master Gerrit-Change-Id: Idba67f84164a74c5778237cfd6fe753f8939dfef Gerrit-Change-Number: 43752 Gerrit-PatchSet: 1 Gerrit-Owner: pespin <pespin(a)sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Reviewer: pespin <pespin(a)sysmocom.de> Gerrit-Comment-Date: Fri, 02 Oct 2026 11:10:00 +0000 Gerrit-HasComments: No Gerrit-Has-Labels: Yes
1 0
0 0
[M] Change in osmo-ttcn3-hacks[master]: Prometheus_Checker: Support labeled keys
by pespin 02 Oct '26

02 Oct '26
pespin has posted comments on this change by pespin. ( https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/43751?usp=email ) Change subject: Prometheus_Checker: Support labeled keys ...................................................................... Patch Set 1: Code-Review+2 -- To view, visit https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/43751?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: comment Gerrit-Project: osmo-ttcn3-hacks Gerrit-Branch: master Gerrit-Change-Id: I1d50666b677faa3fcfb53f179ce2f5f97d9253c6 Gerrit-Change-Number: 43751 Gerrit-PatchSet: 1 Gerrit-Owner: pespin <pespin(a)sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Reviewer: pespin <pespin(a)sysmocom.de> Gerrit-Comment-Date: Fri, 02 Oct 2026 11:09:58 +0000 Gerrit-HasComments: No Gerrit-Has-Labels: Yes
1 0
0 0
[S] Change in osmo-ttcn3-hacks[master]: Misc_Helpers: Fix f_str_(oct_)split() skipping empty strings
by pespin 02 Oct '26

02 Oct '26
pespin has posted comments on this change by pespin. ( https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/43750?usp=email ) Change subject: Misc_Helpers: Fix f_str_(oct_)split() skipping empty strings ...................................................................... Patch Set 1: Code-Review+2 -- To view, visit https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/43750?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: comment Gerrit-Project: osmo-ttcn3-hacks Gerrit-Branch: master Gerrit-Change-Id: I69a20b4659e3c701bd3c9f7c72ea8127feddb006 Gerrit-Change-Number: 43750 Gerrit-PatchSet: 1 Gerrit-Owner: pespin <pespin(a)sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Reviewer: pespin <pespin(a)sysmocom.de> Gerrit-Comment-Date: Fri, 02 Oct 2026 11:09:56 +0000 Gerrit-HasComments: No Gerrit-Has-Labels: Yes
1 0
0 0
[M] Change in osmo-ttcn3-hacks[master]: smf: Close all HTTP2_Server_Emulation tcp conns before exiting
by pespin 02 Oct '26

02 Oct '26
Attention is currently required from: jolly. pespin has posted comments on this change by pespin. ( https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/43749?usp=email ) Change subject: smf: Close all HTTP2_Server_Emulation tcp conns before exiting ...................................................................... Patch Set 1: Code-Review+2 -- To view, visit https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/43749?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: comment Gerrit-Project: osmo-ttcn3-hacks Gerrit-Branch: master Gerrit-Change-Id: I6b4c40fa5a8ae494a0064bb492ffe0185a394659 Gerrit-Change-Number: 43749 Gerrit-PatchSet: 1 Gerrit-Owner: pespin <pespin(a)sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Reviewer: jolly <andreas(a)eversberg.eu> Gerrit-Reviewer: pespin <pespin(a)sysmocom.de> Gerrit-Attention: jolly <andreas(a)eversberg.eu> Gerrit-Comment-Date: Fri, 02 Oct 2026 11:09:54 +0000 Gerrit-HasComments: No Gerrit-Has-Labels: Yes
1 0
0 0
[M] Change in osmo-ttcn3-hacks[master]: smf: Validate metrics
by fixeria 02 Oct '26

02 Oct '26
Attention is currently required from: pespin. fixeria has posted comments on this change by pespin. ( https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/43752?usp=email ) Change subject: smf: Validate metrics ...................................................................... Patch Set 1: Code-Review+1 -- To view, visit https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/43752?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: comment Gerrit-Project: osmo-ttcn3-hacks Gerrit-Branch: master Gerrit-Change-Id: Idba67f84164a74c5778237cfd6fe753f8939dfef Gerrit-Change-Number: 43752 Gerrit-PatchSet: 1 Gerrit-Owner: pespin <pespin(a)sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Attention: pespin <pespin(a)sysmocom.de> Gerrit-Comment-Date: Fri, 02 Oct 2026 11:09:06 +0000 Gerrit-HasComments: No Gerrit-Has-Labels: Yes
1 0
0 0
[M] Change in osmo-ttcn3-hacks[master]: Prometheus_Checker: Support labeled keys
by fixeria 02 Oct '26

02 Oct '26
Attention is currently required from: pespin. fixeria has posted comments on this change by pespin. ( https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/43751?usp=email ) Change subject: Prometheus_Checker: Support labeled keys ...................................................................... Patch Set 1: Code-Review+1 -- To view, visit https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/43751?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: comment Gerrit-Project: osmo-ttcn3-hacks Gerrit-Branch: master Gerrit-Change-Id: I1d50666b677faa3fcfb53f179ce2f5f97d9253c6 Gerrit-Change-Number: 43751 Gerrit-PatchSet: 1 Gerrit-Owner: pespin <pespin(a)sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Attention: pespin <pespin(a)sysmocom.de> Gerrit-Comment-Date: Fri, 02 Oct 2026 11:07:37 +0000 Gerrit-HasComments: No Gerrit-Has-Labels: Yes
1 0
0 0
  • ← Newer
  • 1
  • ...
  • 136
  • 137
  • 138
  • 139
  • 140
  • 141
  • 142
  • 143
  • Older →

HyperKitty Powered by HyperKitty version 1.3.12.