osmith has uploaded this change for review. ( https://gerrit.osmocom.org/c/osmo-dev/+/41213?usp=email )

Change subject: gen_makefile: support building PyHSS ......................................................................

gen_makefile: support building PyHSS

Support cloning and building PyHSS and "interesting" dependencies (that we might want to patch while hacking on PyHSS). Other dependencies such as sqlalchemy get installed via pip into a venv that is currently shared by all python projects that osmo-dev can build. We can change this later on to use multiple venvs or change the dependencies that get built from source, if needed.

Python projects get built with "python3 -m build" into a whl file, and then installed into the venv with "pip install".

The above works with projects that have a pyproject.toml. PyHSS currently doesn't have this yet, but this patchset adds one: https://github.com/nickvsnetworking/pyhss/pull/258

python-venv-requirements.txt in this patch is a combination of the relevant dependencies from PyHSS and dependencies that get built from source.

Related: OS#6862 Change-Id: If40c9e8ea07c9f6c7d379f6d5ff659e95165e4ae --- M all.buildsystems M all.deps M all.urls M gen_makefile.py A python-venv-requirements.txt 5 files changed, 75 insertions(+), 2 deletions(-)

git pull ssh://gerrit.osmocom.org:29418/osmo-dev refs/changes/13/41213/1

diff --git a/all.buildsystems b/all.buildsystems index 7b5a573..63eab23 100644 --- a/all.buildsystems +++ b/all.buildsystems @@ -4,3 +4,9 @@ osmo-s1gw erlang osmo-epdg erlang osmo_dia2gsup erlang + +comp128-python python +gsm0338 python +pyosmocom python +pysctp python +pyhss python diff --git a/all.deps b/all.deps index a49eb87..3439de7 100644 --- a/all.deps +++ b/all.deps @@ -60,3 +60,10 @@ strongswan-epdg libosmocore

upf-benchmark libosmocore libosmo-pfcp + +# PyHSS + deps +comp128-python +gsm0338 +pyosmocom gsm0338 +pysctp +pyhss comp128-python pyosmocom pysctp diff --git a/all.urls b/all.urls index ef1b0ab..507399e 100644 --- a/all.urls +++ b/all.urls @@ -5,3 +5,8 @@ open5gs https://github.com/open5gs/open5gs strongswan-epdg https://gitea.osmocom.org/ims-volte-vowifi/strongswan-epdg osmo-gsm-shark https://gitea.osmocom.org/nhofmeyr/osmo-gsm-shark + +comp128-python https://github.com/Takuto88/comp128-python +gsm0338 https://github.com/dsch/gsm0338 +pysctp https://github.com/P1sec/pysctp +pyhss https://github.com/nickvsnetworking/pyhss diff --git a/gen_makefile.py b/gen_makefile.py index e5f07ab..cfc681b 100755 --- a/gen_makefile.py +++ b/gen_makefile.py @@ -55,6 +55,7 @@ topdir = os.path.dirname(os.path.realpath(__file__)) all_deps_file = os.path.join(topdir, "all.deps") all_urls_file = os.path.join(topdir, "all.urls") +venv_req_file = os.path.join(topdir, "python-venv-requirements.txt") all_buildsystems_file = os.path.join(topdir, "all.buildsystems") parser = argparse.ArgumentParser(epilog=__doc__, formatter_class=argparse.RawTextHelpFormatter)

@@ -218,6 +219,9 @@ ret += f"{short}: {' '.join(full)}\n" return ret

+def gen_venv_activate(): + return f". {shlex.quote(args.install_prefix)}/venv/bin/activate" + def filter_projects_deps_targets(): if not args.targets: return projects_deps @@ -345,7 +349,7 @@ sync touch $@ ''' - elif buildsystem in ["meson", "erlang"]: + elif buildsystem in ["meson", "erlang", "python"]: return "" else: assert False, f"unknown buildsystem: {buildsystem}" @@ -381,7 +385,7 @@ sync touch $@ ''' - elif buildsystem == "erlang": + elif buildsystem in ["erlang", "python"]: return f''' .make.{proj}.configure: .make.{proj}.clone {deps_installed} touch $@ @@ -426,6 +430,19 @@ sync touch $@ ''' + elif buildsystem == "python": + return f''' +.make.{proj}.build: .make.{proj}.configure .make.{proj}.clone $({proj}_files) + @echo "\n\n\n===== $@\n" + rm -f {build_proj}/*.whl + python3 \ + -m build \ + --no-isolation \ + {src_proj} \ + --outdir {build_proj} + sync + touch $@ + ''' else: assert False, f"unknown buildsystem: {buildsystem}"

@@ -474,6 +491,14 @@ sync touch $@ ''' + elif buildsystem == "python": + return f''' +.make.{proj}.install: .make.venv .make.{proj}.build + @echo "\n\n\n===== $@\n" + {gen_venv_activate()} && pip install {shlex.quote(build_proj)}/*.whl --force-reinstall + sync + touch $@ + ''' else: assert False, f"unknown buildsystem: {buildsystem}"

@@ -556,6 +581,7 @@ \( \ -name "*.[hc]" \ -or -name "*.py" \ + -or -name "pyproject.toml" \ -or -name "*.cpp" \ -or -name "*.tpl" \ -or -name "*.map" \ @@ -651,6 +677,12 @@ $(MAKE) --dry-run -d all | grep "is newer than target" $(MAKE) all

+# one shared venv for python projects +.make.venv: {venv_req_file} + python3 -m venv {args.install_prefix}/venv + {gen_venv_activate()} && pip install -r {venv_req_file} + touch $@ + # regenerate this Makefile, in case the deps or opts changed .PHONY: regen regen: diff --git a/python-venv-requirements.txt b/python-venv-requirements.txt new file mode 100644 index 0000000..aba74c7 --- /dev/null +++ b/python-venv-requirements.txt @@ -0,0 +1,23 @@ +# These dependencies will be installed into one shared venv in the make dir +# when building python projects such as pyhss. Dependencies that are built from +# source are in all.deps. + +Flask==2.2.3 +Jinja2==3.1.2 +PyYAML +Requests==2.31.0 +SQLAlchemy==2.0.9 +SQLAlchemy_Utils==0.41.1 +Werkzeug==2.2.3 +aiohttp +alembic +comp128==1.0.0 +flask_restx==1.1.0 +prometheus_flask_exporter +pycryptodome==3.17 +pydantic==2.8.2 +pydantic_core==2.20.1 +pysnmp==4.4.12 +redis==5.0.0 +tzlocal==4.3 +xmltodict==0.14.2