Change in osmo-gsm-tester[master]: add test.report_fragment()

This is merely a historical archive of years 2008-2021, before the migration to mailman3.

A maintained and still updated list archive can be found at https://lists.osmocom.org/hyperkitty/list/gerrit-log@lists.osmocom.org/.

neels gerrit-no-reply at lists.osmocom.org
Thu Dec 3 23:56:24 UTC 2020


neels has uploaded this change for review. ( https://gerrit.osmocom.org/c/osmo-gsm-tester/+/21516 )


Change subject: add test.report_fragment()
......................................................................

add test.report_fragment()

Allow enriching the junit output with arbitrary subtasks within a test.

The current aim is, for handover tests, to not just show that a test
failed, but to show exactly which steps worked and which didn't, e.g.:

 handover.py__01_bts0_started PASSED
 handover.py__02.1_ms0_attach PASSED
 handover.py__02.2_ms1_attach PASSED
 handover.py__02.3_subscribed_in_msc PASSED
 handover.py__03_call_established PASSED
 handover.py__04.1_bts1_started FAILED

In this case it is immediately obvious from looking at the jenkins
results analyzer that bts1 is the cause of the test failure, and it is
visible which parts of the test are flaky, over time.

First user Will be the upcoming handover_2G suite, in
I0b2671304165a1aaae2b386af46fbd8b098e3bd8.

Change-Id: I4ca9100b6f8db24d1f7e0a09b3b7ba88b8ae3b59
---
M selftest/suite_test/suite_test.ok.ign
M selftest/suite_test/suite_test.py
A selftest/suite_test/suitedirA/test_suite/test_report_fragment.py
M src/osmo_gsm_tester/core/report.py
M src/osmo_gsm_tester/core/test.py
5 files changed, 102 insertions(+), 5 deletions(-)



  git pull ssh://gerrit.osmocom.org:29418/osmo-gsm-tester refs/changes/16/21516/1

diff --git a/selftest/suite_test/suite_test.ok.ign b/selftest/suite_test/suite_test.ok.ign
index 460da92..7fb57c6 100644
--- a/selftest/suite_test/suite_test.ok.ign
+++ b/selftest/suite_test/suite_test.ok.ign
@@ -4,3 +4,8 @@
 {combining_scenarios='resources', scenario='foo'}:.*	{combining_scenarios='resources', scenario='foo'}: [RESOURCE_DICT]
 test_suite-[0-9]*-[0-9]*	test_suite-[ID_NUM]-[ID_NUM]
 suiteC-[0-9]*-[0-9]*	suiteC-[ID_NUM]-[ID_NUM]
+line [0-9]+	line [LINENR]
+[0-9][0-9]:[0-9][0-9]:[0-9][0-9]\.[0-9]+	[TIMESTAMP]
+time="[0-9]+"	time="[VAL]"
+  File "[^"]*"	  File "[FILE]"
+timestamp="[^"]+"	timestamp="[TIMESTAMP]"
diff --git a/selftest/suite_test/suite_test.py b/selftest/suite_test/suite_test.py
index 9708037..75eb712 100755
--- a/selftest/suite_test/suite_test.py
+++ b/selftest/suite_test/suite_test.py
@@ -11,6 +11,8 @@
 from osmo_gsm_tester.core import suite
 from osmo_gsm_tester.core.schema import generate_schemas, get_all_schema
 
+import xml.etree.ElementTree as et
+
 config.override_conf = os.path.join(os.path.dirname(sys.argv[0]), 'paths.conf')
 
 example_trial_dir = os.path.join('test_trial_tmp')
@@ -51,6 +53,11 @@
 results = s.run_tests('hello_world.py')
 print(report.suite_to_text(s))
 
+print('- run report fragment test')
+results = s.run_tests('test_report_fragment.py')
+print(report.suite_to_text(s))
+print('\njunit XML:\n' + et.tostring(report.suite_to_junit(s)).decode('utf-8') + '\n\n')
+
 log.style_change(src=True)
 #log.style_change(trace=True)
 print('\n- a test with an error')
diff --git a/selftest/suite_test/suitedirA/test_suite/test_report_fragment.py b/selftest/suite_test/suitedirA/test_suite/test_report_fragment.py
new file mode 100644
index 0000000..06ff37d
--- /dev/null
+++ b/selftest/suite_test/suitedirA/test_suite/test_report_fragment.py
@@ -0,0 +1,11 @@
+from osmo_gsm_tester.testenv import *
+
+with test.report_fragment('fragment1'):
+    print('a step in the first fragment')
+
+with test.report_fragment('fragment2'):
+    print('a step in the second fragment')
+
+with test.report_fragment('fragment3'):
+    print('a step in the third fragment')
+    raise Exception('failure in the third fragment')
diff --git a/src/osmo_gsm_tester/core/report.py b/src/osmo_gsm_tester/core/report.py
index c3390fe..35327ce 100644
--- a/src/osmo_gsm_tester/core/report.py
+++ b/src/osmo_gsm_tester/core/report.py
@@ -132,14 +132,37 @@
         testsuite.set('time', str(math.ceil(suite.duration)))
     testsuite.set('tests', str(len(suite.tests)))
     passed, skipped, failed, errors = suite.count_test_results()
-    testsuite.set('errors', str(errors))
-    testsuite.set('failures', str(failed))
-    testsuite.set('skipped', str(skipped))
-    testsuite.set('disabled', str(skipped))
     for suite_test in suite.tests:
         testcase = test_to_junit(suite_test)
         testcase.set('classname', suite.name())
         testsuite.append(testcase)
+
+        for report_fragment in suite_test.report_fragments:
+            full_name = '%s__%s' % (suite_test.name(), report_fragment.name)
+            el = et.Element('testcase')
+            el.set('name', full_name)
+            el.set('time', str(math.ceil(report_fragment.duration)))
+            if report_fragment.result == test.Test.SKIP:
+                et.SubElement(el, 'skipped')
+                skipped += 1
+            elif report_fragment.result == test.Test.FAIL:
+                failure = et.SubElement(el, 'failure')
+                failure.set('type', suite_test.fail_type or 'failure')
+                failed += 1
+            elif report_fragment.result != test.Test.PASS:
+                error = et.SubElement(el, 'error')
+                error.text = 'could not run'
+                errors += 1
+
+            if report_fragment.output:
+                sout = et.SubElement(el, 'system-out')
+                sout.text = escape_xml_invalid_characters(report_fragment.output)
+            testsuite.append(el)
+
+    testsuite.set('errors', str(errors))
+    testsuite.set('failures', str(failed))
+    testsuite.set('skipped', str(skipped))
+    testsuite.set('disabled', str(skipped))
     return testsuite
 
 def test_to_junit(t):
diff --git a/src/osmo_gsm_tester/core/test.py b/src/osmo_gsm_tester/core/test.py
index adfc444..3847e48 100644
--- a/src/osmo_gsm_tester/core/test.py
+++ b/src/osmo_gsm_tester/core/test.py
@@ -43,6 +43,7 @@
         self._config_test_specific = config_test_specific
         self.path = os.path.join(self.suite_run.definition.suite_dir, self.basename)
         self.status = Test.UNKNOWN
+        self.report_fragments = []
         self.start_timestamp = 0
         self.duration = 0
         self.fail_type = None
@@ -167,7 +168,7 @@
 
     def log_file(self):
         for lt in self.log_targets:
-            if isinstance(lt, FileLogTarget):
+            if isinstance(lt, log.FileLogTarget):
                 return lt
         return None
 
@@ -183,4 +184,54 @@
             return ''
         return lt.get_output(since_mark)
 
+    def report_fragment(self, name, result=None, **kwargs):
+        return Test.ReportFragment(parent_test=self, name=name, result=result, **kwargs)
+
+    class ReportFragment:
+        '''Add additional test results in junit XML.
+           Convenient method that includes a test log:
+             with test.report_fragment('foo'):
+                 do_test_steps()
+
+           Or manually add a report fragment directly:
+             test.report_fragment('foo', result = test.PASS if worked else test.FAIL)
+        '''
+
+        def __init__(self, parent_test, name, result=None, output=None, since_mark=None, start_time=0.0):
+            self.parent_test = parent_test
+            self.name = name
+            self.result = Test.UNKNOWN
+            self.duration = 0.0
+            self.output = output
+            self.start_time = start_time
+            self.log_mark = since_mark
+            assert name not in (x.name for x in self.parent_test.report_fragments)
+            self.parent_test.report_fragments.append(self)
+            if result is not None:
+                self.got_result(result)
+
+        def __str__(self):
+            return '%s %s: %s (%.1fs)' % (self.parent_test.name(), self.name, self.result, self.duration)
+
+        def __enter__(self):
+            self.start_time = self.parent_test.elapsed_time()
+            self.log_mark = self.parent_test.get_log_mark()
+
+        def __exit__(self, *exc_info):
+            self.got_result(self.parent_test.PASS if exc_info[0] is None else self.parent_test.FAIL,
+                            exc_info=exc_info)
+
+        def got_result(self, result, exc_info=None):
+            self.result = result
+            self.duration = self.parent_test.elapsed_time() - self.start_time
+            if self.log_mark is not None and self.output is None:
+                self.output = self.parent_test.get_log_output(since_mark=self.log_mark)
+            if exc_info is not None and exc_info[0] is not None:
+                o = []
+                if self.output:
+                    o.append(self.output)
+                o.extend(traceback.format_exception(*exc_info))
+                self.output = '\n'.join(o)
+            self.parent_test.log('----- Report fragment:', self)
+
 # vim: expandtab tabstop=4 shiftwidth=4

-- 
To view, visit https://gerrit.osmocom.org/c/osmo-gsm-tester/+/21516
To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings

Gerrit-Project: osmo-gsm-tester
Gerrit-Branch: master
Gerrit-Change-Id: I4ca9100b6f8db24d1f7e0a09b3b7ba88b8ae3b59
Gerrit-Change-Number: 21516
Gerrit-PatchSet: 1
Gerrit-Owner: neels <nhofmeyr at sysmocom.de>
Gerrit-MessageType: newchange
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.osmocom.org/pipermail/gerrit-log/attachments/20201203/5a02f319/attachment.htm>


More information about the gerrit-log mailing list