osmith has uploaded this change for review.
gen_makefile: support building PyHSS
Support cloning and building PyHSS and "interesting" dependencies (that
we might want to patch while hacking on PyHSS). Other dependencies such
as sqlalchemy get installed via pip into a venv that is currently shared
by all python projects that osmo-dev can build. We can change this later
on to use multiple venvs or change the dependencies that get built from
source, if needed.
Python projects get built with "python3 -m build" into a whl file, and
then installed into the venv with "pip install".
The above works with projects that have a pyproject.toml. PyHSS
currently doesn't have this yet, but this patchset adds one:
https://github.com/nickvsnetworking/pyhss/pull/258
python-venv-requirements.txt in this patch is a combination of the
relevant dependencies from PyHSS and dependencies that get built from
source.
Related: OS#6862
Change-Id: If40c9e8ea07c9f6c7d379f6d5ff659e95165e4ae
---
M all.buildsystems
M all.deps
M all.urls
M gen_makefile.py
A python-venv-requirements.txt
5 files changed, 75 insertions(+), 2 deletions(-)
git pull ssh://gerrit.osmocom.org:29418/osmo-dev refs/changes/13/41213/1
diff --git a/all.buildsystems b/all.buildsystems
index 7b5a573..63eab23 100644
--- a/all.buildsystems
+++ b/all.buildsystems
@@ -4,3 +4,9 @@
osmo-s1gw erlang
osmo-epdg erlang
osmo_dia2gsup erlang
+
+comp128-python python
+gsm0338 python
+pyosmocom python
+pysctp python
+pyhss python
diff --git a/all.deps b/all.deps
index a49eb87..3439de7 100644
--- a/all.deps
+++ b/all.deps
@@ -60,3 +60,10 @@
strongswan-epdg libosmocore
upf-benchmark libosmocore libosmo-pfcp
+
+# PyHSS + deps
+comp128-python
+gsm0338
+pyosmocom gsm0338
+pysctp
+pyhss comp128-python pyosmocom pysctp
diff --git a/all.urls b/all.urls
index ef1b0ab..507399e 100644
--- a/all.urls
+++ b/all.urls
@@ -5,3 +5,8 @@
open5gs https://github.com/open5gs/open5gs
strongswan-epdg https://gitea.osmocom.org/ims-volte-vowifi/strongswan-epdg
osmo-gsm-shark https://gitea.osmocom.org/nhofmeyr/osmo-gsm-shark
+
+comp128-python https://github.com/Takuto88/comp128-python
+gsm0338 https://github.com/dsch/gsm0338
+pysctp https://github.com/P1sec/pysctp
+pyhss https://github.com/nickvsnetworking/pyhss
diff --git a/gen_makefile.py b/gen_makefile.py
index e5f07ab..cfc681b 100755
--- a/gen_makefile.py
+++ b/gen_makefile.py
@@ -55,6 +55,7 @@
topdir = os.path.dirname(os.path.realpath(__file__))
all_deps_file = os.path.join(topdir, "all.deps")
all_urls_file = os.path.join(topdir, "all.urls")
+venv_req_file = os.path.join(topdir, "python-venv-requirements.txt")
all_buildsystems_file = os.path.join(topdir, "all.buildsystems")
parser = argparse.ArgumentParser(epilog=__doc__, formatter_class=argparse.RawTextHelpFormatter)
@@ -218,6 +219,9 @@
ret += f"{short}: {' '.join(full)}\n"
return ret
+def gen_venv_activate():
+ return f". {shlex.quote(args.install_prefix)}/venv/bin/activate"
+
def filter_projects_deps_targets():
if not args.targets:
return projects_deps
@@ -345,7 +349,7 @@
sync
touch $@
'''
- elif buildsystem in ["meson", "erlang"]:
+ elif buildsystem in ["meson", "erlang", "python"]:
return ""
else:
assert False, f"unknown buildsystem: {buildsystem}"
@@ -381,7 +385,7 @@
sync
touch $@
'''
- elif buildsystem == "erlang":
+ elif buildsystem in ["erlang", "python"]:
return f'''
.make.{proj}.configure: .make.{proj}.clone {deps_installed}
touch $@
@@ -426,6 +430,19 @@
sync
touch $@
'''
+ elif buildsystem == "python":
+ return f'''
+.make.{proj}.build: .make.{proj}.configure .make.{proj}.clone $({proj}_files)
+ @echo "\\n\\n\\n===== $@\\n"
+ rm -f {build_proj}/*.whl
+ python3 \
+ -m build \
+ --no-isolation \
+ {src_proj} \
+ --outdir {build_proj}
+ sync
+ touch $@
+ '''
else:
assert False, f"unknown buildsystem: {buildsystem}"
@@ -474,6 +491,14 @@
sync
touch $@
'''
+ elif buildsystem == "python":
+ return f'''
+.make.{proj}.install: .make.venv .make.{proj}.build
+ @echo "\\n\\n\\n===== $@\\n"
+ {gen_venv_activate()} && pip install {shlex.quote(build_proj)}/*.whl --force-reinstall
+ sync
+ touch $@
+ '''
else:
assert False, f"unknown buildsystem: {buildsystem}"
@@ -556,6 +581,7 @@
\\( \\
-name "*.[hc]" \\
-or -name "*.py" \\
+ -or -name "pyproject.toml" \\
-or -name "*.cpp" \\
-or -name "*.tpl" \\
-or -name "*.map" \\
@@ -651,6 +677,12 @@
$(MAKE) --dry-run -d all | grep "is newer than target"
$(MAKE) all
+# one shared venv for python projects
+.make.venv: {venv_req_file}
+ python3 -m venv {args.install_prefix}/venv
+ {gen_venv_activate()} && pip install -r {venv_req_file}
+ touch $@
+
# regenerate this Makefile, in case the deps or opts changed
.PHONY: regen
regen:
diff --git a/python-venv-requirements.txt b/python-venv-requirements.txt
new file mode 100644
index 0000000..aba74c7
--- /dev/null
+++ b/python-venv-requirements.txt
@@ -0,0 +1,23 @@
+# These dependencies will be installed into one shared venv in the make dir
+# when building python projects such as pyhss. Dependencies that are built from
+# source are in all.deps.
+
+Flask==2.2.3
+Jinja2==3.1.2
+PyYAML
+Requests==2.31.0
+SQLAlchemy==2.0.9
+SQLAlchemy_Utils==0.41.1
+Werkzeug==2.2.3
+aiohttp
+alembic
+comp128==1.0.0
+flask_restx==1.1.0
+prometheus_flask_exporter
+pycryptodome==3.17
+pydantic==2.8.2
+pydantic_core==2.20.1
+pysnmp==4.4.12
+redis==5.0.0
+tzlocal==4.3
+xmltodict==0.14.2
To view, visit change 41213. To unsubscribe, or for help writing mail filters, visit settings.