Change in osmo-gsm-tester[master]: nitb_netreg_mass: Add code to declare a run successful/failure

Holger Freyther gerrit-no-reply at lists.osmocom.org
Tue Mar 5 14:25:13 UTC 2019


Holger Freyther has uploaded this change for review. ( https://gerrit.osmocom.org/13130


Change subject: nitb_netreg_mass: Add code to declare a run successful/failure
......................................................................

nitb_netreg_mass: Add code to declare a run successful/failure

Change-Id: If26047f0635db8c7209af143fbd1da4c9b2312d1
---
M src/osmo_gsm_tester/ms_driver.py
M src/osmo_ms_driver/location_update_test.py
M suites/nitb_netreg_mass/register_default_mass.py
3 files changed, 71 insertions(+), 7 deletions(-)



  git pull ssh://gerrit.osmocom.org:29418/osmo-gsm-tester refs/changes/30/13130/1

diff --git a/src/osmo_gsm_tester/ms_driver.py b/src/osmo_gsm_tester/ms_driver.py
index 8ee93b7..96b907a 100644
--- a/src/osmo_gsm_tester/ms_driver.py
+++ b/src/osmo_gsm_tester/ms_driver.py
@@ -118,6 +118,18 @@
         """
         self._test_case.print_stats()
 
+    def get_stats(self):
+        """
+        Returns a statistical summary of the test.
+        """
+        return self._test_case.get_stats()
+
+    def get_result_values(self):
+        """
+        Returns the raw result values of the test run in any order.
+        """
+        return self._test_case.get_result_values()
+
     def cleanup(self):
         """
         Cleans up the driver (e.g. AF_UNIX files).
diff --git a/src/osmo_ms_driver/location_update_test.py b/src/osmo_ms_driver/location_update_test.py
index 2e3a648..5ff2199 100644
--- a/src/osmo_ms_driver/location_update_test.py
+++ b/src/osmo_ms_driver/location_update_test.py
@@ -23,9 +23,11 @@
 
 from datetime import timedelta
 
+import collections
 import time
 
 class LUResult(Results):
+    """Representation of a Location Updating Result."""
 
     def __init__(self, name):
         super().__init__(name)
@@ -44,6 +46,10 @@
     def lu_delay(self):
         return self.lu_time() - self.start_time()
 
+
+LUStats = collections.namedtuple("LUStats", ["num_attempted", "num_completed",
+                                 "min_latency", "max_latency"])
+
 class MassUpdateLocationTest(log.Origin):
     """
     A test to launch a configurable amount of MS and make them
@@ -219,9 +225,24 @@
                 max_value = result.lu_delay()
         return min_value, max_value
 
+    def get_result_values(self):
+        """
+        Returns the raw result values of the test run in any order.
+        """
+        return self._results.values()
+
+    def get_stats(self):
+        """
+        Returns a statistical summary of the test.
+        """
+        attempted = len(self._subscribers)
+        completed = attempted - self._outstanding
+        min_latency, max_latency = self.find_min_max(filter(lambda x: x.has_lu_time(), self._results.values()))
+        return LUStats(attempted, completed, min_latency, max_latency)
+
     def print_stats(self):
-        all_completed = self.all_completed()
-        min_value, max_value = self.find_min_max(filter(lambda x: x.has_lu_time(), self._results.values()))
+        stats = self.get_stats()
+        all_completed = stats.num_attempted == stats.num_completed
 
         self.log("Tests done", all_completed=all_completed,
-                    min=min_value, max=max_value)
+                    min=stats.min_latency, max=stats.max_latency)
diff --git a/suites/nitb_netreg_mass/register_default_mass.py b/suites/nitb_netreg_mass/register_default_mass.py
index 1033608..028e7e5 100644
--- a/suites/nitb_netreg_mass/register_default_mass.py
+++ b/suites/nitb_netreg_mass/register_default_mass.py
@@ -1,17 +1,18 @@
 #!/usr/bin/env python3
 """
-Run a network registration with a 'massive' amount of MS
+Runs a network registration with a 'massive' amount of MS
 using the ms_driver infrastructure.
 """
 from osmo_gsm_tester.testenv import *
+from datetime import timedelta
 
-print('use resources...')
+print('Claiming resources for the test')
 nitb = suite.nitb()
 bts = suite.bts()
 ms_driver = suite.ms_driver()
 modems = suite.all_modems()
 
-print('start nitb and bts...')
+print('Launching a simple network')
 nitb.bts_add(bts)
 nitb.start()
 bts.start()
@@ -25,5 +26,35 @@
 # Run the base test.
 ms_driver.run_test()
 
-# Print stats
+# Print the stats of the run.
 ms_driver.print_stats()
+
+# Evaluate if this run was successful or not. Our initial acceptance criteria
+# is quite basic but it should allow us to scale to a larger number of MS and
+# reasons (e.g. have a full BCCH).
+#
+# 99% of LUs should complete
+# 99% of successful LUs should complete within 10s.
+stats = ms_driver.get_stats()
+if len(modems) > 0 and stats.num_completed < 1:
+    raise Exception("No run completed.")
+completion_ratio = stats.num_attempted / stats.num_completed
+
+# Verify that 99% of LUs completed.
+if completion_ratio < 0.99:
+    raise Exception("Completion ratio of %f%% lower than threshold." % (completion_ratio * 100.0))
+
+# Check how many results are below our threshold.
+acceptable_delay = timedelta(seconds=20)
+results = ms_driver.get_result_values()
+quick_enough = 0
+for result in results:
+    if not result.has_lu_time():
+        continue
+    if timedelta(seconds=result.lu_delay()) >= acceptable_delay:
+        continue
+    quick_enough = quick_enough + 1
+
+latency_ratio = quick_enough / len(results)
+if latency_ratio < 0.99:
+    raise Exception("Latency ratio of %f%% lower than threshold." % (latency_ratio * 100.0))

-- 
To view, visit https://gerrit.osmocom.org/13130
To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings

Gerrit-Project: osmo-gsm-tester
Gerrit-Branch: master
Gerrit-MessageType: newchange
Gerrit-Change-Id: If26047f0635db8c7209af143fbd1da4c9b2312d1
Gerrit-Change-Number: 13130
Gerrit-PatchSet: 1
Gerrit-Owner: Holger Freyther <holger at freyther.de>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.osmocom.org/pipermail/gerrit-log/attachments/20190305/7d17f82c/attachment.html>


More information about the gerrit-log mailing list