osmith has uploaded this change for review. ( https://gerrit.osmocom.org/c/osmo-ci/+/41370?usp=email )
Change subject: OBS: support building pyhss ......................................................................
OBS: support building pyhss
Build source packages for the python project pyhss similar to how we do it for erlang projects: by downloading all dependencies ahead of time and vendoring them in the source package.
Related: SYS#6819 Change-Id: I321090e811f7c5c142bf973c616d83cd5b2219ab --- M scripts/obs/data/build_srcpkg.Dockerfile A scripts/obs/data/pyhss_download_deps.sh M scripts/obs/lib/__init__.py M scripts/obs/lib/srcpkg.py 4 files changed, 113 insertions(+), 0 deletions(-)
git pull ssh://gerrit.osmocom.org:29418/osmo-ci refs/changes/70/41370/1
diff --git a/scripts/obs/data/build_srcpkg.Dockerfile b/scripts/obs/data/build_srcpkg.Dockerfile index 8b57931..e0368fb 100644 --- a/scripts/obs/data/build_srcpkg.Dockerfile +++ b/scripts/obs/data/build_srcpkg.Dockerfile @@ -3,13 +3,19 @@ FROM ${DISTRO_FROM} ARG UID
+# default-libmysqlclient-dev: needed for fetching the source package +# "mysqlclient" with pip that PyHSS depends on. Pip actually compiles the +# package to figure out its dependency tree and aborts if libmysqlclient-dev is +# missing (https://github.com/pypa/pip/issues/1884). RUN apt-get update && \ apt-get upgrade -y && \ apt-get install -y --no-install-recommends \ ca-certificates \ colordiff \ debhelper \ + default-libmysqlclient-dev \ dh-python \ + dh-virtualenv \ dpkg-dev \ erlang-nox \ fakeroot \ @@ -20,7 +26,9 @@ lsb-release \ meson \ osc \ + pkgconf \ python3-packaging \ + python3-pip \ python3-setuptools \ quilt \ sed \ diff --git a/scripts/obs/data/pyhss_download_deps.sh b/scripts/obs/data/pyhss_download_deps.sh new file mode 100644 index 0000000..7dfd5d9 --- /dev/null +++ b/scripts/obs/data/pyhss_download_deps.sh @@ -0,0 +1,97 @@ +#!/bin/sh -e +# Copyright 2025 sysmocom - s.f.m.c. GmbH +# SPDX-License-Identifier: GPL-3.0-or-later +# This script downloads all dependencies of PyHSS from the python package +# index, either as binary package for all python versions and CPU architectures +# we care about, or as source package depending on what is available. + +check_cwd() { + if ! [ -e services/hssService.py ]; then + echo "ERROR: run this script from the PyHSS directory" + exit 1 + fi + if [ -d debian/deps ]; then + echo "ERROR: debian/deps exists already!" + exit 1 + fi +} + +download_deps() { + local srcpkgs=_temp/requirements-source.txt + local binpkgs=_temp/requirements-binary.txt + local py_ver + local python_versions=" + 3.11 + 3.12 + 3.13 + " + # See e.g. https://pypi.org/project/SQLAlchemy/#files + local platform + local platforms=" + manylinux_2_17_aarch64 + manylinux_2_17_x86_64 + " + + rm -rf _temp + mkdir _temp + + while IFS= read -r line; do + case "$line" in + # These packages are only available as sources + mongo*|pymongo*|mysqlclient*|pysctp*|pycryptodome=*) + echo "$line" >>"$srcpkgs" + ;; + # The rest is available as binary packages. This is + # preferred as dependencies for building some of these + # are not always available in the target distributions + # (e.g. pydantic-core is written in rust and tooling + # for building python + rust is not in debian 12). + *) + echo "$line" >>"$binpkgs" + ;; + esac + done < "requirements.txt" + + # Build system dependencies must also be installed as we will build + # offline with --no-index and pip won't use system libraries: + # https://github.com/pypa/pip/issues/5696 + echo "setuptools" >>"$binpkgs" + echo "wheel" >>"$binpkgs" + echo "hatchling" >>"$binpkgs" + + echo ":: Downloading source packages" + pip download \ + --dest debian/deps \ + --no-binary=:all: \ + -r "$srcpkgs" + + for py_ver in $python_versions; do + for platform in $platforms; do + echo ":: Downloading binary packages (python $py_ver, platform $platform)" + local binpkgs_extra="" + + # Redis depends on async-timeout, which has been upstreamed + # into Python 3.11+. This means "pip download" may not download + # it if it runs with a more recent python version, but older + # distros (debian 12) will need it. + if [ "$py_ver" = "3.11" ]; then + binpkgs_extra="async-timeout" + fi + + pip download \ + --dest debian/deps \ + --python-version "$py_ver" \ + --platform "$platform" \ + --only-binary=:all: \ + -r "$binpkgs" \ + $binpkgs_extra + done + done + + rm -r _temp +} + +check_cwd +download_deps + +echo ":: Success" diff --git a/scripts/obs/lib/__init__.py b/scripts/obs/lib/__init__.py index 66090f2..4d0d624 100644 --- a/scripts/obs/lib/__init__.py +++ b/scripts/obs/lib/__init__.py @@ -16,6 +16,8 @@ # Print output of commands as they run, not only on error cmds_verbose = False
+ci_obs_dir = os.path.realpath(f"{__file__}/../..") +
def add_shared_arguments(parser): """ Arguments shared between build_srcpkg.py and update_obs_project.py. """ diff --git a/scripts/obs/lib/srcpkg.py b/scripts/obs/lib/srcpkg.py index 78e49d1..1b31f84 100644 --- a/scripts/obs/lib/srcpkg.py +++ b/scripts/obs/lib/srcpkg.py @@ -141,6 +141,12 @@ )
+def prepare_project_pyhss(): + repo_path = lib.git.get_repo_path("pyhss") + script = os.path.join(lib.ci_obs_dir, "data/pyhss_download_deps.sh") + lib.run_cmd(["sh", "-e", script], cwd=repo_path) + + def run_generate_build_dep(project): """Run contrib/generate_build_dep.sh if it exists in the given project, to to download sources for dependencies (see e.g. osmo_dia2gsup.git)."""