osmith has uploaded this change for review. ( https://gerrit.osmocom.org/c/osmo-ci/+/28905 )
Change subject: obs: build nightly-asan with --enable-sanitize ......................................................................
obs: build nightly-asan with --enable-sanitize
Related: OS#5301 Change-Id: Ife826b2c8e1173a2faf292e76815be6f0bd0a62c --- M scripts/obs/build_srcpkg.py M scripts/obs/lib/__init__.py M scripts/obs/lib/config.py M scripts/obs/lib/debian.py M scripts/obs/lib/metapkg.py M scripts/obs/lib/srcpkg.py M scripts/obs/update_obs_project.py 7 files changed, 156 insertions(+), 24 deletions(-)
git pull ssh://gerrit.osmocom.org:29418/osmo-ci refs/changes/05/28905/1
diff --git a/scripts/obs/build_srcpkg.py b/scripts/obs/build_srcpkg.py index ccf82d7..0e259b5 100755 --- a/scripts/obs/build_srcpkg.py +++ b/scripts/obs/build_srcpkg.py @@ -33,7 +33,7 @@
lib.check_required_programs() if args.package: - lib.check_package(args.package) + lib.check_packages(args.feed, [args.package]) lib.remove_temp()
if args.meta: diff --git a/scripts/obs/lib/__init__.py b/scripts/obs/lib/__init__.py index 193c248..bb990df 100644 --- a/scripts/obs/lib/__init__.py +++ b/scripts/obs/lib/__init__.py @@ -66,15 +66,28 @@ exit(1)
-def check_package(package): - if package in lib.config.projects_osmocom: - return - if package in lib.config.projects_other: - return +def get_packages_for_feed(feed): + ret = []
- print(f"ERROR: unknown package: {package}") - print("See packages_osmocom and packages_other in obs/lib/config.py") - exit(1) + if feed == "nightly-asan": + for project in lib.config.projects_osmocom: + if project not in lib.config.projects_osmocom_exclude_asan: + ret += [project] + return ret + + ret += lib.config.projects_osmocom + ret += lib.config.projects_other + return ret + + +def check_packages(feed, packages): + packages_feed = get_packages_for_feed(feed) + + for package in packages: + if package not in packages_feed: + print(f"ERROR: unknown package: {package}") + print("See packages_osmocom and packages_other in obs/lib/config.py") + exit(1)
def exit_error_cmd(completed, error_msg): diff --git a/scripts/obs/lib/config.py b/scripts/obs/lib/config.py index 22c9c05..efaecd4 100644 --- a/scripts/obs/lib/config.py +++ b/scripts/obs/lib/config.py @@ -32,6 +32,10 @@ "2022q2", "latest", "nightly", + "nightly-asan", # OS#5301 +] +feeds_no_rpm_spec = [ + "nightly-asan", ]
# Osmocom projects: generated source packages will depend on a meta package, @@ -80,6 +84,15 @@ "open5gs", ]
+# Do not build these for osmocom:nightly:asan +projects_osmocom_exclude_asan = [ + "erlang/osmo_dia2gsup", + "libosmo-dsp", + "osmo-gsm-manuals", + "python/osmo-python-tests", + "rtl-sdr", +] + git_url_default = "https://gerrit.osmocom.org" # /project gets appended git_url_other = { "libosmo-dsp": "https://gitea.osmocom.org/sdr/libosmo-dsp", diff --git a/scripts/obs/lib/debian.py b/scripts/obs/lib/debian.py index 89153fb..9109b29 100644 --- a/scripts/obs/lib/debian.py +++ b/scripts/obs/lib/debian.py @@ -87,6 +87,10 @@ """ Adjust the changelog if the version in the changelog is different from the given version. """ version_changelog = get_last_version_from_changelog(project) + + if feed.endswith("-asan"): + version = f"{version}~asan0" + if version_changelog == version: return
@@ -95,6 +99,100 @@ changelog_add_entry(project, feed, version)
+def add_configure_arg(project, arg): + """ Add a configure option like --enable-sanitize to the dh_auto_configure + line, also add the override_dh_auto_configure block if missing. """ + print(f"{project}: adding {arg} to debian/rules") + rules = f"{lib.git.get_repo_path(project)}/debian/rules" + + override_found = False + replaced = False + + with open(rules, "r") as f: + lines = f.readlines() + + for i in range(len(lines)): + line = lines[i] + if line.startswith("override_dh_auto_configure:"): + override_found = True + continue + + if "dh_auto_configure" not in line: + continue + + assert override_found + assert " -- " in line.replace("\t", " ") + + lines[i] = line.replace(" --", f" -- {arg}", 1) + replaced = True + break + + if not override_found: + lines += ["\n", + "override_dh_auto_configure:\n", + f"\tdh_auto_configure -- {arg}\n"] + + with open(rules, "w") as f: + f.writelines(lines) + + +def disable_tests(project): + """ Add or replace an existing override_dh_auto_test block with one that + disables the tests. As of writing we need this for osmocom:nightly:asan + because OBS has ulimit -v hardcoded (OS#5301). """ + print(f"{project}: disabling tests in debian/rules") + rules = f"{lib.git.get_repo_path(project)}/debian/rules" + + override_found = False + replaced = False + + with open(rules, "r") as f: + lines = f.readlines() + + for i in range(len(lines)): + line = lines[i] + if line.startswith("override_dh_auto_test:"): + override_found = True + continue + + if not override_found: + continue + + # End of override_dh_auto_test block + if line != "\n": + lines[i] = "\n" + continue + + replaced = True + break + + if not override_found: + lines += ["\n", + "override_dh_auto_test:\n"] + + with open(rules, "w") as f: + f.writelines(lines) + + +def disable_manuals(project): + """ For osmocom:nightly:asan we need to disable manuals, as the binaries + built with sanitizer flags can't run in OBS and they would be running + during generation of VTY references (OS#5301). """ + print(f"{project}: disabling manuals") + debian = f"{lib.git.get_repo_path(project)}/debian" + + # Remove osmo-gsm-manuals dep + lib.run_cmd(["sed", "-i", "/osmo-gsm-manuals-dev/d", f"{debian}/control"]) + + # Remove debian/*-doc.install + lib.run_cmd(f"rm -rf {shlex.quote(debian)}/*-doc.install", shell=True) + + # debian/rules: remove --enable-manuals/doxygen, add --disable-doxygen + lib.run_cmd(["sed", "-i", "s/--enable-manuals//g", f"{debian}/rules"]) + lib.run_cmd(["sed", "-i", "s/--enable-doxygen//g", f"{debian}/rules"]) + add_configure_arg(project, "--disable-doxygen") + + def build_source_package(project): fix_source_format(project) print(f"{project}: building debian source package") diff --git a/scripts/obs/lib/metapkg.py b/scripts/obs/lib/metapkg.py index dd261de..fcb1f91 100644 --- a/scripts/obs/lib/metapkg.py +++ b/scripts/obs/lib/metapkg.py @@ -97,8 +97,9 @@ lib.debian.build_source_package(pkgname) lib.debian.move_files_to_output(pkgname)
- generate_rpm_spec(feed, version) - lib.rpm_spec.copy_to_output(pkgname) + if feed not in lib.config.feeds_no_rpm_spec: + generate_rpm_spec(feed, version) + lib.rpm_spec.copy_to_output(pkgname)
lib.remove_cache_extra_files() return version diff --git a/scripts/obs/lib/srcpkg.py b/scripts/obs/lib/srcpkg.py index 468a240..55831be 100644 --- a/scripts/obs/lib/srcpkg.py +++ b/scripts/obs/lib/srcpkg.py @@ -11,7 +11,7 @@ """ checkout a commit, either latest tag or master or 20YY branch """ if feed == "latest": lib.git.checkout_latest_tag(project) - elif feed == "nightly": + elif feed == "nightly" or feed.startswith("nightly-"): lib.git.checkout_default_branch(project) else: # 2022q1 etc lib.git.checkout(project, feed) @@ -118,6 +118,13 @@ f.write(f"{version}\n")
+def check_has_rpm_spec(project, feed): + if feed in lib.config.feeds_no_rpm_spec: + return False + + return lib.rpm_spec.get_spec_in_path(project) is not None + + def build(project, feed, conflict_version, fetch): lib.git.clone(project, fetch) lib.git.clean(project) @@ -125,7 +132,7 @@ version = get_version_for_feed(project, feed, conflict_version) epoch = get_epoch(project) version_epoch = f"{epoch}:{version}" if epoch else version - has_rpm_spec = lib.rpm_spec.get_spec_in_path(project) is not None + has_rpm_spec = check_has_rpm_spec(project, feed)
print(f"{project}: building source package {version_epoch}") write_tarball_version(project, version_epoch) @@ -146,6 +153,11 @@ print(f"{project}: running {project_specific_func}") globals()[project_specific_func]()
+ if feed.endswith("-asan"): + lib.debian.add_configure_arg(project, "--enable-sanitize") + lib.debian.disable_tests(project) + lib.debian.disable_manuals(project) + lib.debian.build_source_package(project) lib.debian.move_files_to_output(project)
diff --git a/scripts/obs/update_obs_project.py b/scripts/obs/update_obs_project.py index 759acfe..4f84edf 100755 --- a/scripts/obs/update_obs_project.py +++ b/scripts/obs/update_obs_project.py @@ -19,17 +19,12 @@ srcpkgs_updated = [] # list of pkgnames
-def parse_packages(packages_arg): - if packages_arg: - for package in packages_arg: - lib.check_package(package) - return packages_arg +def parse_packages(feed, packages): + if packages: + lib.check_packages(feed, packages) + return packages
- # Default to all - ret = [] - ret += lib.config.projects_osmocom - ret += lib.config.projects_other - return ret + return lib.get_packages_for_feed(feed)
def build_srcpkg(feed, package, conflict_version, fetch, is_meta_pkg): @@ -191,7 +186,7 @@ args = parser.parse_args() proj = args.obs_project feed = args.feed - packages = parse_packages(args.package) + packages = parse_packages(feed, args.package)
lib.set_cmds_verbose(args.verbose)