osmith has submitted this change. ( https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/42814?usp=email )
Change subject: testenv: introduce testsrcdir.cfg ......................................................................
testenv: introduce testsrcdir.cfg
Source directories can have more than one testenv.cfg file. Some options have been added to testenv.cfg that are not really specific to a single testsuite or how it gets executed, but to the whole source directory.
This is not ideal, because we need to have additional code that ensures these options have the same value across all testenv.cfg files in the same source directory, and we need functions that just pick the value from the first of these configs. When we change such a value, we also need to potentially make the change in multiple files.
Resolve this by introducing a new config file testsrcdir.cfg, that can be optionally present in the source directory, and has options that count for all testsuites in the same source directory.
Move max_jobs_per_gb_ram= as first option to the new file. The following patches will move all other source directory specific options to testsrcdir.cfg.
I have also considered naming the new file testdir.cfg, but the name "testdir" is already used in the source code for the place where we execute the individual testsuites after copying configs into that directory.
Change-Id: I8eceea7b874ce1352e2cc9780b77d2a8e694cd28 --- M 5gc/testenv.cfg A 5gc/testsrcdir.cfg M _testenv/README.md M _testenv/testenv.py M _testenv/testenv/testenv_cfg.py A _testenv/testenv/testsrcdir_cfg.py M _testenv/testenv/testsuite.py 7 files changed, 105 insertions(+), 70 deletions(-)
Approvals: pespin: Looks good to me, but someone else must approve osmith: Looks good to me, approved Jenkins Builder: Verified
diff --git a/5gc/testenv.cfg b/5gc/testenv.cfg index 876aacf..d70c135 100644 --- a/5gc/testenv.cfg +++ b/5gc/testenv.cfg @@ -1,6 +1,5 @@ [testsuite] titan_min=11.1.0 -max_jobs_per_gb_ram=0.3 program=C5G_Tests config=C5G_Tests.cfg copy=testsuite_prepare.sh diff --git a/5gc/testsrcdir.cfg b/5gc/testsrcdir.cfg new file mode 100644 index 0000000..7f19289 --- /dev/null +++ b/5gc/testsrcdir.cfg @@ -0,0 +1,2 @@ +[testsrcdir] +max_jobs_per_gb_ram=0.3 diff --git a/_testenv/README.md b/_testenv/README.md index 196af94..9c5c35a 100644 --- a/_testenv/README.md +++ b/_testenv/README.md @@ -89,11 +89,6 @@ depending on when the script runs. The script will not run on crash if podman is used, as the container gets shutdown beforehand.
-* `max_jobs_per_gb_ram=`: optional value that can be set to reduce the amount - of parallel jobs when compiling the testsuite. This is set in the 5gc - testsuite to avoid consuming the whole RAM and freezing (or possibly getting - killed from an out-of-memory daemon). - * `podman_extra=`: optional value that can be set to add extra parameters to podman. For example, it can be used to mount an additional volume to pass- through USB devices. Use spaces to separate parameters, use quotes to include @@ -171,6 +166,27 @@ * `bts/testenv_hopping.cfg` * `bts/testenv_oml.cfg`
+## testsrcdir.cfg + +Directories can optionally also have a `testsrcdir.cfg` file next to the +`testenv*.cfg` files. These `testsrcdir.cfg` files set configurations for all +testsuites in that directory. The file has one `[testsrcdir]` section. + +### Example + +```ini +[testsrcdir] +max_jobs_per_gb_ram=0.3 +``` + +### Keys + +* `max_jobs_per_gb_ram=`: optional value that can be set to reduce the amount + of parallel jobs when compiling testsuites from this directory and their + dependency sources. This is set in the 5gc testsuite to avoid consuming the + whole RAM and freezing (or possibly getting killed from an out-of-memory + daemon). + ## Environment variables
### Environment variables read by testenv diff --git a/_testenv/testenv.py b/_testenv/testenv.py index 55d7896..9450c8f 100755 --- a/_testenv/testenv.py +++ b/_testenv/testenv.py @@ -13,6 +13,7 @@ import testenv.podman_install import testenv.requirements import testenv.testdir +import testenv.testsrcdir_cfg import testenv.testenv_cfg import testenv.testsuite
@@ -31,6 +32,7 @@
def run(): testenv.requirements.check() + testenv.testsrcdir_cfg.init() testenv.testenv_cfg.init()
if not testenv.args.binary_repo: diff --git a/_testenv/testenv/testenv_cfg.py b/_testenv/testenv/testenv_cfg.py index eb9e030..ec65ac0 100644 --- a/_testenv/testenv/testenv_cfg.py +++ b/_testenv/testenv/testenv_cfg.py @@ -5,11 +5,9 @@ import fnmatch import glob import logging -import math import os.path import sys import testenv -import traceback
cfgs = {} current = None @@ -149,64 +147,6 @@ return get_titan_version(cfg)
-def verify_max_jobs_per_gb_ram(cfgs_all): - error = False - max_jobs_per_gb_ram = None - - for cfg_name, cfg in cfgs_all.items(): - cfg_max = cfg["testsuite"].get("max_jobs_per_gb_ram", None) - if cfg_max is not None: - if max_jobs_per_gb_ram is not None: - if cfg_max != max_jobs_per_gb_ram: - error = True - break - else: - max_jobs_per_gb_ram = cfg_max - elif max_jobs_per_gb_ram: - error = True - break - - if not error: - return - - logging.error("Found different max_jobs_per_gb_ram= values in testenv.cfg files of the same directory.") - logging.error("This is not supported, please fix it.") - sys.exit(1) - - -def get_titan_make_job_count(): - _, cfg = next(iter(cfgs.items())) - max_jobs_per_gb_ram = cfg["testsuite"].get("max_jobs_per_gb_ram", None) - max_jobs = None - - if max_jobs_per_gb_ram: - try: - gb_ram = 0 - with open("/proc/meminfo") as f: - line = f.readline() - # Parse e.g. "MemTotal: 15571604 kB" - if line.startswith("MemTotal:"): - gb_ram = int(line.split(" ")[-2]) / 1024 / 1024 - logging.debug(f"Building with {round(gb_ram, 2)} GB of RAM") - max_jobs = math.floor(gb_ram * float(max_jobs_per_gb_ram)) - if max_jobs < 1: - raise RuntimeError(f"max_jobs is invalid: max_jobs={max_jobs}, gb_ram={gb_ram}") - - except Exception as ex: - traceback.print_exception(type(ex), ex, ex.__traceback__) - logging.error(f"Calculating max jobs with max_jobs_per_gb_ram={max_jobs_per_gb_ram} failed, assuming 4") - max_jobs = 4 - - if max_jobs and max_jobs < testenv.args.jobs: - logging.info( - f"Using only {max_jobs} jobs instead of {testenv.args.jobs} because of" - f" max_jobs_per_gb_ram={max_jobs_per_gb_ram} in testenv.cfg" - ) - return max_jobs - - return testenv.args.jobs - - def get_podman_extra_first_cfg(): _, cfg = next(iter(cfgs.items())) return cfg["testsuite"].get("podman_extra", None) @@ -259,7 +199,6 @@ "prepare", "program", "titan_min", - "max_jobs_per_gb_ram", "podman_extra", ] keys_valid_component = [ @@ -283,6 +222,9 @@ "copy", "package", ] + keys_moved_to_testsrcdir_cfg = [ + "max_jobs_per_gb_ram", + ]
if "testsuite" not in cfg: logging.error(f"{path}: missing [testsuite] section") @@ -315,6 +257,8 @@ msg = f"{path}: [{section}]: {key}= is invalid" if key in keys_invalid and keys_invalid[key] in valid: msg += f", did you mean {keys_invalid[key]}=?" + if key in keys_moved_to_testsrcdir_cfg: + msg += " (this key has been moved to testsrcdir.cfg)"
logging.error(msg) exit_error_readme() @@ -413,8 +357,6 @@
cfgs_all[basename] = cfg
- verify_max_jobs_per_gb_ram(cfgs_all) - # Select configs based on --config argument(s) for config_arg in testenv.args.config: if config_arg == "all": diff --git a/_testenv/testenv/testsrcdir_cfg.py b/_testenv/testenv/testsrcdir_cfg.py new file mode 100644 index 0000000..d1d560d --- /dev/null +++ b/_testenv/testenv/testsrcdir_cfg.py @@ -0,0 +1,73 @@ +# Copyright 2026 sysmocom - s.f.m.c. GmbH +# SPDX-License-Identifier: GPL-3.0-or-later +import configparser +import logging +import math +import os +import sys +import testenv +import traceback + +cfg = { + "max_jobs_per_gb_ram": None, +} + + +def get_titan_make_job_count(): + max_jobs_per_gb_ram = cfg["max_jobs_per_gb_ram"] + max_jobs = None + + if max_jobs_per_gb_ram: + try: + gb_ram = 0 + with open("/proc/meminfo") as f: + line = f.readline() + # Parse e.g. "MemTotal: 15571604 kB" + if line.startswith("MemTotal:"): + gb_ram = int(line.split(" ")[-2]) / 1024 / 1024 + logging.debug(f"Building with {round(gb_ram, 2)} GB of RAM") + max_jobs = math.floor(gb_ram * float(max_jobs_per_gb_ram)) + if max_jobs < 1: + raise RuntimeError(f"max_jobs is invalid: max_jobs={max_jobs}, gb_ram={gb_ram}") + + except Exception as ex: + traceback.print_exception(type(ex), ex, ex.__traceback__) + logging.error(f"Calculating max jobs with max_jobs_per_gb_ram={max_jobs_per_gb_ram} failed, assuming 4") + max_jobs = 4 + + if max_jobs and max_jobs < testenv.args.jobs: + logging.info( + f"Using only {max_jobs} jobs instead of {testenv.args.jobs} because of" + f" max_jobs_per_gb_ram={max_jobs_per_gb_ram} in testsrcdir.cfg" + ) + return max_jobs + + return testenv.args.jobs + + +def init(): + global cfg + + cfg_path = os.path.join(testenv.ttcn3_hacks_dir, testenv.args.testsuite, "testsrcdir.cfg") + if not os.path.exists(cfg_path): + return + + parser = configparser.ConfigParser() + parser.read(cfg_path) + + if "testsrcdir" not in parser: + logging.error(f"Missing section [testsrcdir] in {cfg_path}") + sys.exit(1) + + for section in parser: + if section == "DEFAULT": + continue + if section != "testsrcdir": + logging.error(f"Invalid section [{section}] in {cfg_path}") + sys.exit(1) + + for key in parser["testsrcdir"]: + if key not in cfg: + logging.error("Invalid key {key}= in {cfg_path}") + sys.exit(1) + cfg[key] = parser["testsrcdir"][key] diff --git a/_testenv/testenv/testsuite.py b/_testenv/testenv/testsuite.py index 3fdafa8..cdea60e 100644 --- a/_testenv/testenv/testsuite.py +++ b/_testenv/testenv/testsuite.py @@ -12,6 +12,7 @@ import testenv import testenv.cmd import testenv.testenv_cfg +import testenv.testsrcdir_cfg import time
builddir_env = {} @@ -58,7 +59,7 @@ logging.info(f"Building testsuite (eclipse-titan {titan_version}, {titan_reason})")
env = copy.copy(builddir_env) - env["PARALLEL_MAKE"] = f"-j{testenv.testenv_cfg.get_titan_make_job_count()}" + env["PARALLEL_MAKE"] = f"-j{testenv.testsrcdir_cfg.get_titan_make_job_count()}"
testenv.cmd.run(["make", testenv.args.testsuite], cwd=testenv.ttcn3_hacks_dir, env=env)
-
osmith