This is merely a historical archive of years 2008-2021, before the migration to mailman3.
A maintained and still updated list archive can be found at https://lists.osmocom.org/hyperkitty/list/gerrit-log@lists.osmocom.org/.
pespin gerrit-no-reply at lists.osmocom.orgpespin has uploaded this change for review. ( https://gerrit.osmocom.org/c/osmo-gsm-tester/+/18832 ) Change subject: Add per-test KPI support ...................................................................... Add per-test KPI support tests can now use 'tenv.test().set_kpis(some_dict)' to set any kind of data as KPIs, which will be presented in the junit report. The representation of KPIs in the xml file doesn't follow the junit format, mainly because it has no support for per-test properties. Change-Id: I00e976f65a202e82d440bf33708f06c8ce2643e2 --- M selftest/report_test/expected_junit_output.xml M selftest/report_test/report_test.ok M selftest/report_test/report_test.py M src/osmo_gsm_tester/core/report.py M src/osmo_gsm_tester/core/test.py 5 files changed, 98 insertions(+), 3 deletions(-) git pull ssh://gerrit.osmocom.org:29418/osmo-gsm-tester refs/changes/32/18832/1 diff --git a/selftest/report_test/expected_junit_output.xml b/selftest/report_test/expected_junit_output.xml index 5de0edf..9f6185c 100644 --- a/selftest/report_test/expected_junit_output.xml +++ b/selftest/report_test/expected_junit_output.xml @@ -1,4 +1,4 @@ -<testsuites errors="2" failures="1" name="trial" tests="10" time="102"> +<testsuites errors="2" failures="2" name="trial" tests="12" time="122"> <testsuite disabled="0" errors="0" failures="0" hostname="localhost" id="0" name="suiteA" skipped="0" tests="2"> <testcase classname="suiteA" name="suiteA-0" time="30"> <system-out>test log file not available</system-out> @@ -59,4 +59,34 @@ <property name="ref:orange" value="abcd"></property> </properties> </testsuite> -</testsuites> + <testsuite disabled="0" errors="0" failures="1" hostname="localhost" id="4" name="suiteE" skipped="0" tests="2"> + <testcase classname="suiteE" name="suiteE-0" time="12"> + <failure type="fake_fail_type">fake_fail_message</failure> + <system-err>system stderr fake content</system-err> + <kpis> + <kpi_node name="ueA"> + <property name="kpiA" value="30"></property> + <property name="kpiB" value="foobar"></property> + <kpi_node name="yet-another-level"> + <property name="foo" value="bar"></property> + </kpi_node> + </kpi_node> + <kpi_node name="enbD"> + <property name="foobar-boolean" value="True"></property> + </kpi_node> + <property name="somekpi" value="someval"></property> + </kpis> + <system-out>test log file not available</system-out> + </testcase> + <testcase classname="suiteE" name="suiteE-1" time="10"> + <kpis> + <property name="abcd" value="abcdval"></property> + </kpis> + <system-out>test log file not available</system-out> + </testcase> + <properties> + <property name="ref:foobar/potato" value="1234"></property> + <property name="ref:orange" value="abcd"></property> + </properties> + </testsuite> +</testsuites> \ No newline at end of file diff --git a/selftest/report_test/report_test.ok b/selftest/report_test/report_test.ok index 87092ea..442d343 100644 --- a/selftest/report_test/report_test.ok +++ b/selftest/report_test/report_test.ok @@ -15,3 +15,5 @@ tst {combining_scenarios='config'}: DBG: {definition_conf={}} [suiteC↪{combining_scenarios='config'}] tst suiteD: DBG: {combining='config'} tst {combining_scenarios='config'}: DBG: {definition_conf={}} [suiteD↪{combining_scenarios='config'}] +tst suiteE: DBG: {combining='config'} +tst {combining_scenarios='config'}: DBG: {definition_conf={}} [suiteE↪{combining_scenarios='config'}] diff --git a/selftest/report_test/report_test.py b/selftest/report_test/report_test.py index 57e3a89..5888bf3 100755 --- a/selftest/report_test/report_test.py +++ b/selftest/report_test/report_test.py @@ -39,11 +39,13 @@ self.suite_dir = util.Dir(example_trial_dir).new_child('suitedef' + name) -def fake_run_test(test_obj, status, duration, sysout=None): +def fake_run_test(test_obj, status, duration, sysout=None, kpis=None): test_obj.status = status test_obj.duration = duration if sysout is not None: test_obj.set_report_stdout(sysout) + if kpis is not None: + test_obj.set_kpis(kpis) if status == test.Test.FAIL: test_obj.fail_type = 'fake_fail_type' test_obj.fail_message = 'fake_fail_message' @@ -92,6 +94,14 @@ fake_run_test(s.tests[1], test.Test.PASS, 10) fake_run_suite(s, 20) +# Test adding KPIs +s_def = FakeSuiteDefinition('suiteE', 2) +s = suite.SuiteRun(trial, s_def.name(), s_def) +trial.suites.append(s) +fake_run_test(s.tests[0], test.Test.FAIL, 12, kpis={'ueA': {'kpiA': 30, 'kpiB': 'foobar', 'yet-another-level': {'foo': 'bar'}}, 'enbD': {'foobar-boolean': True }, 'somekpi': 'someval'}) +fake_run_test(s.tests[1], test.Test.PASS, 10, kpis={'abcd': 'abcdval'}) +fake_run_suite(s, 20) + element = report.trial_to_junit(trial) def indent(elem, level=0): @@ -126,6 +136,9 @@ with open(exp_path, 'r') as f: exp = f.read().rstrip() udiff(exp, got, exp_path) + # Uncomment to update exp_path: + #with open(exp_path, 'w') as f: + # f.write(got) #deleting generated tmp trial dir: shutil.rmtree(example_trial_dir, ignore_errors=True) diff --git a/src/osmo_gsm_tester/core/report.py b/src/osmo_gsm_tester/core/report.py index 5014bf5..d2c68c5 100644 --- a/src/osmo_gsm_tester/core/report.py +++ b/src/osmo_gsm_tester/core/report.py @@ -53,6 +53,46 @@ prop.set('name', 'ref:' + key) prop.set('value', val) +def dict_to_junit(parent, d): + for key, val in d.items(): + if isinstance(val, dict): + node = et.SubElement(parent, 'kpi_node') + node.set('name', key) + dict_to_junit(node, val) + continue + if isinstance(val, (tuple, list)): + node = et.SubElement(parent, 'kpi_node') + node.set('name', key) + list_to_junit(node, val) + continue + # scalar: + node = et.SubElement(parent, 'property') + node.set('name', key) + node.set('value', str(val)) + +def list_to_junit(parent, li): + for i in range(len(li)): + if isinstance(li[i], dict): + node = et.SubElement(parent, 'kpi_node') + node.set('name', str(i)) + dict_to_junit(node, li[i]) + continue + if isinstance(val, (tuple, list)): + node = et.SubElement(parent, 'kpi_node') + node.set('name', str(i)) + list_to_junit(node, li[i]) + continue + # scalar: + node = et.SubElement(parent, 'property') + node.set('name', str(i)) + node.set('value', str(li[i])) + +def kpis_to_junit(parent, kpis): + if not kpis: + return + assert isinstance(kpis, dict) + knode = et.SubElement(parent, 'kpis') + dict_to_junit(knode, kpis) def trial_to_junit_write(trial, junit_path): elements = et.ElementTree(element=trial_to_junit(trial)) @@ -118,6 +158,7 @@ elif t.status != test.Test.PASS: error = et.SubElement(testcase, 'error') error.text = 'could not run' + kpis_to_junit(testcase, t.kpis()) sout = et.SubElement(testcase, 'system-out') sout.text = escape_xml_invalid_characters(t.report_stdout()) return testcase diff --git a/src/osmo_gsm_tester/core/test.py b/src/osmo_gsm_tester/core/test.py index c6d88e6..dfbd169 100644 --- a/src/osmo_gsm_tester/core/test.py +++ b/src/osmo_gsm_tester/core/test.py @@ -49,6 +49,7 @@ self.fail_message = None self.log_targets = [] self._report_stdout = None + self._kpis = None self.timeout = int(config_test_specific['timeout']) if 'timeout' in config_test_specific else None def module_name(self): @@ -139,6 +140,14 @@ def config_test_specific(self): return self._config_test_specific + def set_kpis(self, kpis): + if not isinstance(kpis, dict): + raise log.Error('Expected dictionary in toplevel kpis') + self._kpis = kpis + + def kpis(self): + return self._kpis + def set_report_stdout(self, text): 'Overwrite stdout text stored in report from inside a test' self._report_stdout = text -- To view, visit https://gerrit.osmocom.org/c/osmo-gsm-tester/+/18832 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-gsm-tester Gerrit-Branch: master Gerrit-Change-Id: I00e976f65a202e82d440bf33708f06c8ce2643e2 Gerrit-Change-Number: 18832 Gerrit-PatchSet: 1 Gerrit-Owner: pespin <pespin at sysmocom.de> Gerrit-MessageType: newchange -------------- next part -------------- An HTML attachment was scrubbed... URL: <http://lists.osmocom.org/pipermail/gerrit-log/attachments/20200615/bb4623f2/attachment.htm>