osmith has submitted this change. ( https://gerrit.osmocom.org/c/osmo-ci/+/29603 )
Change subject: obs: build_binpkg: optimize osmo-gsm-manuals-dev ......................................................................
obs: build_binpkg: optimize osmo-gsm-manuals-dev
Installing osmo-gsm-manuals-dev plus depends takes a long time. Don't do this for every build, instead do it once when building a second docker container and then use that.
Related: OS#2385 Change-Id: I8475bd954352b572197795ad4cd9461e39896d48 --- M scripts/obs/build_binpkg.py A scripts/obs/data/build_binpkg_manuals.Dockerfile M scripts/obs/lib/docker.py M scripts/obs/lib/srcpkg.py 4 files changed, 57 insertions(+), 4 deletions(-)
Approvals: Jenkins Builder: Verified laforge: Looks good to me, approved pespin: Looks good to me, but someone else must approve
diff --git a/scripts/obs/build_binpkg.py b/scripts/obs/build_binpkg.py index b670af9..27817e2 100755 --- a/scripts/obs/build_binpkg.py +++ b/scripts/obs/build_binpkg.py @@ -59,9 +59,17 @@ script_path = "data/build_rpm.sh"
if args.docker: + image_type = "build_binpkg" + + # Optimization: use docker container with osmo-gsm-manuals-dev already + # installed if it is in build depends + if distro.startswith("debian:") \ + and lib.srcpkg.requires_osmo_gsm_manuals_dev(args.package): + image_type += "_manuals" + env["BUILDUSER"] = "user" lib.docker.run_in_docker_and_exit(script_path, - image_type="build_binpkg", + image_type=image_type, distro=distro, pass_argv=False, env=env) else: diff --git a/scripts/obs/data/build_binpkg_manuals.Dockerfile b/scripts/obs/data/build_binpkg_manuals.Dockerfile new file mode 100644 index 0000000..fd4c709 --- /dev/null +++ b/scripts/obs/data/build_binpkg_manuals.Dockerfile @@ -0,0 +1,20 @@ +# Optimization: installing osmo-gsm-manuals-dev and its many, many dependencies +# takes quite a long time - sometimes longer than building the package itself +# (related: OS#4132). Instead of doing this every time before starting a build, +# here is a second docker container that already has it installed. This gets +# used by build_binpkg.py in case the package to build depends on +# osmo-gsm-manuals-dev and the build is done for Debian. Note that right now we +# don't build the manuals for rpm-based distributions. +ARG DISTRO_FROM +FROM ${DISTRO_FROM} +ARG DISTRO + +RUN case "$DISTRO" in \ + debian*) \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + osmo-gsm-manuals-dev \ + && \ + apt-get clean \ + ;; \ + esac diff --git a/scripts/obs/lib/docker.py b/scripts/obs/lib/docker.py index 81464bb..136aeeb 100644 --- a/scripts/obs/lib/docker.py +++ b/scripts/obs/lib/docker.py @@ -14,7 +14,11 @@ return ret
-def get_distro_from(distro): +def get_distro_from(distro, image_type): + # Manuals: depend on regular image (data/build_binpkg_manuals.Dockerfile) + if image_type.endswith("_manuals"): + return get_image_name(distro, image_type.replace("_manuals", "")) + # CentOS 8 is EOL (SYS#5818) if distro == "centos:8": return "almalinux:8" @@ -24,7 +28,7 @@
def build_image(distro, image_type): image_name = get_image_name(distro, image_type) - distro_from = get_distro_from(distro) + distro_from = get_distro_from(distro, image_type)
print(f"docker: building image {image_name}")
@@ -75,10 +79,15 @@ if add_oscrc: oscrc = get_oscrc()
- # Build the docker image. Unless it is up-to-date, this will take a few + # Unless the docker image is up-to-date, building will take a few # minutes or so, therefore print the output. No need to restore # set_cmds_verbose, as we use subprocess.run() below and exit afterwards. lib.set_cmds_verbose(True) + + # Manuals: build regular image first (data/build_binpkg_manuals.Dockerfile) + if image_type.endswith("_manuals"): + build_image(distro, image_type.replace("_manuals","")) + build_image(distro, image_type)
cmd = ["docker", "run", diff --git a/scripts/obs/lib/srcpkg.py b/scripts/obs/lib/srcpkg.py index 1711f70..b37b8b3 100644 --- a/scripts/obs/lib/srcpkg.py +++ b/scripts/obs/lib/srcpkg.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-or-later # Copyright 2022 sysmocom - s.f.m.c. GmbH info@sysmocom.de +import glob import os import pathlib import lib.config @@ -173,3 +174,18 @@
lib.remove_cache_extra_files() return version_epoch + + +def requires_osmo_gsm_manuals_dev(project): + """ Check if an already built source package has osmo-gsm-manuals-dev in + Build-Depends of the .dsc file """ + path_dsc = glob.glob(f"{lib.get_output_path(project)}/*.dsc") + assert len(path_dsc) == 1, f"failed to get dsc path for {project}" + + with open(path_dsc[0], "r") as handle: + for line in handle.readlines(): + if line.startswith("Build-Depends:") \ + and "osmo-gsm-manuals-dev" in line: + return True + + return False
-
osmith