Change in osmo-ci[master]: obs: add build_binpkg.py

osmith has uploaded this change for review. ( https://gerrit.osmocom.org/c/osmo-ci/+/29601 ) Change subject: obs: add build_binpkg.py ...................................................................... obs: add build_binpkg.py Add a script to build deb/rpm packages, as it would be done on obs.osmocom.org. This will be used by jenkins to verify deb and rpm builds for each submitted gerrit patch. I have attempted to use 'osc build' instead of directly calling apt-get build-dep and dpkg-buildpackage (and rpm equivalents). Using 'osc build' would have the advantage that the build works as close to the OBS build as possible. However it would try to install dependencies with sudo, so we would need to have sudo available in the docker container that builds the untrusted code from gerrit. Let's not do that. Related: OS#2385 Change-Id: I4c6b5d61af35df98cbc70d9ddc8ad36d38a9ce18 --- A scripts/obs/build_binpkg.py A scripts/obs/data/build_binpkg.Dockerfile A scripts/obs/data/build_deb.sh A scripts/obs/data/build_rpm.sh M scripts/obs/lib/__init__.py A scripts/obs/lib/binpkg_deb.py A scripts/obs/lib/binpkg_rpm.py M scripts/obs/lib/config.py M scripts/obs/lib/docker.py 9 files changed, 275 insertions(+), 3 deletions(-) git pull ssh://gerrit.osmocom.org:29418/osmo-ci refs/changes/01/29601/1 diff --git a/scripts/obs/build_binpkg.py b/scripts/obs/build_binpkg.py new file mode 100755 index 0000000..b670af9 --- /dev/null +++ b/scripts/obs/build_binpkg.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright 2022 sysmocom - s.f.m.c. GmbH <info(a)sysmocom.de> +import argparse +import multiprocessing +import os +import lib +import lib.binpkg_deb +import lib.config +import lib.docker +import lib.git +import lib.metapkg +import lib.srcpkg + + +def main(): + distro_default = lib.config.docker_distro_default + distro_choices = [distro_default] + lib.config.docker_distro_other + jobs_default = multiprocessing.cpu_count() + 1 + + parser = argparse.ArgumentParser( + description="Build a deb or rpm package as it would be done on" + " obs.osmocom.org. Use after building a source package" + " with build_srcpkg.py." + f" Output dir: {lib.config.path_temp}/binpkgs") + parser.add_argument("-d", "--docker", choices=distro_choices, + const=distro_default, nargs="?", + help="build the package in docker for a specific" + f" distro (default: {distro_default})") + parser.add_argument("-j", "--jobs", type=int, default=jobs_default, + help=f"parallel running jobs (default: {jobs_default})") + parser.add_argument("-v", "--verbose", action="store_true", + help="always print shell commands and their output," + " instead of only printing them on error") + parser.add_argument("package", + help="package name, e.g. libosmocore") + args = parser.parse_args() + + lib.set_cmds_verbose(args.verbose) + + srcdir = f"{lib.config.path_temp}/srcpkgs/{args.package}" + if not os.path.exists(srcdir): + print(f"ERROR: {args.package}: no srcpkg found, run build_srcpkg.py" + " first!") + exit(1) + + bindir = f"{lib.config.path_temp}/binpkgs" + lib.run_cmd(["rm", "-rf", bindir]) + os.makedirs(bindir) + + distro = args.docker if args.docker else distro_default + + env = {"JOBS": str(args.jobs), + "PACKAGE": args.package, + "BUILDUSER": os.environ["USER"]} + + script_path = "data/build_deb.sh" + if not distro.startswith("debian:"): + script_path = "data/build_rpm.sh" + + if args.docker: + env["BUILDUSER"] = "user" + lib.docker.run_in_docker_and_exit(script_path, + image_type="build_binpkg", + distro=distro, + pass_argv=False, env=env) + else: + lib.run_cmd(["sudo", "-E", script_path], env=env, + cwd=lib.config.path_top) + +if __name__ == "__main__": + main() diff --git a/scripts/obs/data/build_binpkg.Dockerfile b/scripts/obs/data/build_binpkg.Dockerfile new file mode 100644 index 0000000..feacbc9 --- /dev/null +++ b/scripts/obs/data/build_binpkg.Dockerfile @@ -0,0 +1,72 @@ +ARG DISTRO_FROM +FROM ${DISTRO_FROM} +ARG DISTRO +ARG UID + +COPY Release.key /tmp/Release.key + +RUN useradd --uid=${UID} -m user + +# Only install build-essential here, and what's needed to add the Osmocom +# repository. Everything else must be defined as dependency in the package +# build recipe. For rpm-based distributions, there is no build-essential or +# similar package. Instead add relevant packages from prjconf, e.g.: +# https://build.opensuse.org/projects/CentOS:CentOS-8/prjconf +RUN case "$DISTRO" in \ + debian*) \ + apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + fakeroot \ + git \ + gnupg2 \ + && \ + apt-get clean \ + ;; \ + centos*) \ + dnf -y install \ + autoconf \ + automake \ + binutils \ + dnf-utils \ + gcc \ + gcc-c++ \ + glibc-devel \ + libtool \ + make \ + redhat-rpm-config \ + rpm-build \ + rpmdevtools \ + wget && \ + yum config-manager --set-enabled powertools && \ + su user -c rpmdev-setuptree \ + ;; \ + esac + +# Add master repository, where packages immediately get updated after merging +# patches to master. +RUN case "$DISTRO" in \ + debian:11) \ + apt-key add /tmp/Release.key && \ + rm /tmp/Release.key && \ + echo "deb https://downloads.osmocom.org/packages/osmocom:/master/Debian_11/ ./" \ + > /etc/apt/sources.list.d/osmocom-master.list \ + ;; \ + centos:8) \ + { echo "[network_osmocom_master]"; \ + echo "name=Nightly packages of the Osmocom project (CentOS_8)"; \ + echo "type=rpm-md"; \ + echo "baseurl=https://downloads.osmocom.org/packages/osmocom:/master/CentOS_8/"; \ + echo "gpgcheck=1"; \ + echo "gpgkey=https://downloads.osmocom.org/packages/osmocom:/master/CentOS_8/repodata/repomd.xml.key"; \ + echo "enabled=1"; \ + } > /etc/yum.repos.d/network:osmocom:master.repo \ + ;; \ + *) \ + echo "can't install repo for $DISTRO" && \ + exit 1 \ + ;; \ + esac + +WORKDIR /obs/ diff --git a/scripts/obs/data/build_deb.sh b/scripts/obs/data/build_deb.sh new file mode 100755 index 0000000..0030278 --- /dev/null +++ b/scripts/obs/data/build_deb.sh @@ -0,0 +1,14 @@ +#!/bin/sh -ex + +apt_get="apt-get" +if [ -n "$INSIDE_DOCKER" ]; then + export DEBIAN_FRONTEND=noninteractive + apt_get="apt-get -y" +fi + +su "$BUILDUSER" -c "tar -C _temp/binpkgs -xvf _temp/srcpkgs/$PACKAGE/*.tar.*" +cd _temp/binpkgs/* + +$apt_get update +$apt_get build-dep . +su "$BUILDUSER" -c "dpkg-buildpackage -us -uc -j$JOBS" diff --git a/scripts/obs/data/build_rpm.sh b/scripts/obs/data/build_rpm.sh new file mode 100755 index 0000000..39f49ee --- /dev/null +++ b/scripts/obs/data/build_rpm.sh @@ -0,0 +1,30 @@ +#!/bin/sh -ex + +if ! [ -d /home/$BUILDUSER/rpmbuild/SOURCES ]; then + set +x + echo "ERROR: rpmdev-setuptree did not run" + echo "If this is an rpm based system and you want to build the package" + echo "here, run rpmdev-setuptree. Otherwise consider building the" + echo "package in docker (-d)." + exit 1 +fi + +yum_builddep="yum-builddep" +if [ -n "$INSIDE_DOCKER" ]; then + yum_builddep="yum-builddep -y" +fi + +spec="$(basename "$(find _temp/srcpkgs/"$PACKAGE" -name '*.spec')")" + +su "$BUILDUSER" -c "cp _temp/srcpkgs/$PACKAGE/$spec ~/rpmbuild/SPECS" +su "$BUILDUSER" -c "cp _temp/srcpkgs/$PACKAGE/*.tar.* ~/rpmbuild/SOURCES" +su "$BUILDUSER" -c "cp _temp/srcpkgs/$PACKAGE/rpmlintrc ~/rpmbuild/SOURCES" + +$yum_builddep "/home/$BUILDUSER/rpmbuild/SPECS/$spec" + +su "$BUILDUSER" -c "rpmbuild -bb ~/rpmbuild/SPECS/$spec" + +# Make built rpms available outside of docker +if [ -n "$INSIDE_DOCKER" ]; then + su "$BUILDUSER" -c "mv ~/rpmbuild/RPMS/*/*.rpm _temp/binpkgs/" +fi diff --git a/scripts/obs/lib/__init__.py b/scripts/obs/lib/__init__.py index 8f52dba..5979623 100644 --- a/scripts/obs/lib/__init__.py +++ b/scripts/obs/lib/__init__.py @@ -14,6 +14,7 @@ def add_shared_arguments(parser): + """ Arguments shared between build_srcpkg.py and update_obs_project.py. """ parser.add_argument("-f", "--feed", help="package feed (default: nightly). The feed" " determines the git revision to be built:" diff --git a/scripts/obs/lib/binpkg_deb.py b/scripts/obs/lib/binpkg_deb.py new file mode 100644 index 0000000..b26623c --- /dev/null +++ b/scripts/obs/lib/binpkg_deb.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright 2022 sysmocom - s.f.m.c. GmbH <info(a)sysmocom.de> +import os +import glob +import lib.config + + +def extract_source(srcdir, bindir): + tarball = glob.glob(f"{srcdir}/*.tar.*")[0] + + print(f"extracting {os.path.basename(tarball)}") + lib.run_cmd(["tar", "-xf", tarball], cwd=bindir) + + return glob.glob(f"{bindir}/*/")[0] + + +def build(srcdir, jobs): + bindir = f"{lib.config.path_temp}/binpkg" + extractdir = extract_source(srcdir, bindir) + + lib.set_cmds_verbose(True) + + # install deps + lib.run_cmd(["apt-get", "-y", "build-dep", "."], cwd=extractdir) + + print("running dpkg-buildpackage") + lib.run_cmd(["dpkg-buildpackage", "-us", "-uc", f"-j{jobs}"], + cwd=extractdir) diff --git a/scripts/obs/lib/binpkg_rpm.py b/scripts/obs/lib/binpkg_rpm.py new file mode 100644 index 0000000..b26623c --- /dev/null +++ b/scripts/obs/lib/binpkg_rpm.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright 2022 sysmocom - s.f.m.c. GmbH <info(a)sysmocom.de> +import os +import glob +import lib.config + + +def extract_source(srcdir, bindir): + tarball = glob.glob(f"{srcdir}/*.tar.*")[0] + + print(f"extracting {os.path.basename(tarball)}") + lib.run_cmd(["tar", "-xf", tarball], cwd=bindir) + + return glob.glob(f"{bindir}/*/")[0] + + +def build(srcdir, jobs): + bindir = f"{lib.config.path_temp}/binpkg" + extractdir = extract_source(srcdir, bindir) + + lib.set_cmds_verbose(True) + + # install deps + lib.run_cmd(["apt-get", "-y", "build-dep", "."], cwd=extractdir) + + print("running dpkg-buildpackage") + lib.run_cmd(["dpkg-buildpackage", "-us", "-uc", f"-j{jobs}"], + cwd=extractdir) diff --git a/scripts/obs/lib/config.py b/scripts/obs/lib/config.py index 52b84fb..393251b 100644 --- a/scripts/obs/lib/config.py +++ b/scripts/obs/lib/config.py @@ -112,3 +112,6 @@ } docker_distro_default = "debian:11" +docker_distro_other = [ + "centos:8", +] diff --git a/scripts/obs/lib/docker.py b/scripts/obs/lib/docker.py index 2944cb3..81464bb 100644 --- a/scripts/obs/lib/docker.py +++ b/scripts/obs/lib/docker.py @@ -14,11 +14,23 @@ return ret +def get_distro_from(distro): + # CentOS 8 is EOL (SYS#5818) + if distro == "centos:8": + return "almalinux:8" + + return distro + + def build_image(distro, image_type): - image_name = get_image_name(distro, image_type), + image_name = get_image_name(distro, image_type) + distro_from = get_distro_from(distro) + print(f"docker: building image {image_name}") + lib.run_cmd(["docker", "build", "--build-arg", f"DISTRO={distro}", + "--build-arg", f"DISTRO_FROM={distro_from}", "--build-arg", f"UID={os.getuid()}", "-t", image_name, "-f", f"{lib.config.path_top}/data/{image_type}.Dockerfile", @@ -38,12 +50,15 @@ def run_in_docker_and_exit(script_path, add_oscrc=False, - image_type="build_srcpkg", distro=None): + image_type="build_srcpkg", distro=None, + pass_argv=True, env={}): """ :param script_path: what to run inside docker, relative to scripts/obs/ :param add_oscrc: put user's oscrc in docker (contains obs credentials!) :param image_type: which Dockerfile to use (data/{image_type}.Dockerfile) :param distro: which Linux distribution to use, e.g. "debian:11" + :param pass_argv: pass arguments from sys.argv to the script + :param env: dict of environment variables """ if "INSIDE_DOCKER" in os.environ: return @@ -72,10 +87,17 @@ "-e", "PYTHONUNBUFFERED=1", "-v", f"{lib.config.path_top}:/obs"] + + for env_key, env_val in env.items(): + cmd += ["-e", f"{env_key}={env_val}"] + if oscrc: cmd += ["-v", f"{oscrc}:/home/user/.oscrc"] - cmd += [image_name, f"/obs/{script_path}"] + sys.argv[1:] + cmd += [image_name, f"/obs/{script_path}"] + + if pass_argv: + cmd += sys.argv[1:] print(f"docker: running: {script_path} inside docker") ret = subprocess.run(cmd) -- To view, visit https://gerrit.osmocom.org/c/osmo-ci/+/29601 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-ci Gerrit-Branch: master Gerrit-Change-Id: I4c6b5d61af35df98cbc70d9ddc8ad36d38a9ce18 Gerrit-Change-Number: 29601 Gerrit-PatchSet: 1 Gerrit-Owner: osmith <osmith(a)sysmocom.de> Gerrit-MessageType: newchange