2 is the latest approved patch-set.
No files were changed between the latest approved patch-set and the submitted one.


buildsystem: build out-of-tree

The buildsystem used to create symlinks to dependency source files in
the testsuite directories, and then building inside that source
directory. This lead to many unrelated files being in the source
directory.

Change the logic to create symlinks to all sources in a separate
$BUILDDIR instead (default: _build) and do the build there.

Advantages:
* Source directories are not cluttered with other files anymore.
* Clean up logic becomes much simpler and faster (rm -rf _build instead
  of generating a Makefile and running "make clean" in every testsuite
  directory).
* No need to generate gitignore files on the fly anymore.
* Using a separate $BUILDDIR is now possible, this will be used by
  testenv in a follow-up patch when running with podman, to make sure
  that build artifacts from podman and not using podman are not mixed as
  they are incompatible.

Related: OS#6599
Change-Id: If18aaf2a2a0d55bb617e5cb1b73f6ee4b1952494
---
M .gitignore
M Makefile
M _buildsystem/gen_links.inc.sh
M _buildsystem/regen_makefile.inc.sh
D _buildsystem/rmlinks.sh
M _testenv/testenv/testsuite.py
M start-testsuite.sh
7 files changed, 89 insertions(+), 58 deletions(-)

diff --git a/.gitignore b/.gitignore
index d1a9de5..b14e63c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,3 +23,4 @@
 __pycache__
 tags
 .linux
+_build
diff --git a/Makefile b/Makefile
index 2355d9a..e490df8 100644
--- a/Makefile
+++ b/Makefile
@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+BUILDDIR ?= _build
+
 SUBDIRS= \
 	asterisk \
 	bsc \
@@ -87,43 +89,46 @@
 deps-update: .make.deps
 
 compile: $(foreach dir,$(SUBDIRS),$(dir)/compile)
-clean: $(foreach dir,$(SUBDIRS),$(dir)/clean)
+clean-old: $(foreach dir,$(SUBDIRS),$(dir)/clean-old)
+	@rm -rfv ~/.cache/osmo-ttcn3-testenv/podman/osmo-ttcn3-hacks
 all: $(foreach dir,$(SUBDIRS),$(dir)/all)
 
 define DIR_Makefile_template
-$(1)/Makefile: $(1)/gen_links.sh $(1)/regen_makefile.sh
+$(BUILDDIR)/$(1)/Makefile: $(1)/gen_links.sh $(1)/regen_makefile.sh
 	(cd $(1) && ./gen_links.sh && ./regen_makefile.sh)
 endef
 
 define DIR_compile_template
 .PHONY: $(1)/compile
-$(1)/compile: deps $(1)/Makefile
-	$(MAKE) -C $(1) compile
+$(1)/compile: deps $(BUILDDIR)/$(1)/Makefile
+	$(MAKE) -C $(BUILDDIR)/$(1) compile
 endef
 
-define DIR_clean_template
-.PHONY: $(1)/clean
-$(1)/clean: $(1)/Makefile
-	$(MAKE) -C $(1) clean
-	(cd $(1) && ../_buildsystem/rmlinks.sh && rm Makefile)
+# Remove left-over files from before the buildsystem was changed to do
+# out-of-tree builds. _buildsystem/gen_links.inc.sh tells the users to run this
+# if needed.
+define DIR_clean_old_template
+.PHONY: $(1)/clean-old
+$(1)/clean-old:
+	@git clean -fx "$(1)"
 endef
 
 define DIR_all_template
 $(1): $(1)/all
 .PHONY: $(1)/all
-$(1)/all: deps $(1)/Makefile
-	$(MAKE) -C $(1) compile
-	$(MAKE) $(PARALLEL_MAKE) -C $(1)
+$(1)/all: deps $(BUILDDIR)/$(1)/Makefile
+	$(MAKE) -C $(BUILDDIR)/$(1) compile
+	$(MAKE) $(PARALLEL_MAKE) -C $(BUILDDIR)/$(1)
 endef
 
 $(foreach dir,$(SUBDIRS), \
 	$(eval $(call DIR_Makefile_template,$(dir)))	\
 	$(eval $(call DIR_compile_template,$(dir)))	\
-	$(eval $(call DIR_clean_template,$(dir)))	\
+	$(eval $(call DIR_clean_old_template,$(dir)))	\
 	$(eval $(call DIR_all_template,$(dir)))		\
 	)
 
-.PHONY: tags regen-diameter-types-ttcn
+.PHONY: tags regen-diameter-types-ttcn clean
 tags:
 	find $(shell pwd) \
 		-type f -name "*.ttcn" -o \
@@ -132,3 +137,9 @@
 
 regen-diameter-types-ttcn:
 	(cd library/ && ./regen-DIAMETER_Types_ttcn.sh)
+
+# Intentionally not using $(BUILDDIR) here to avoid user errors leading to
+# unintentional removal of other files. If $(BUILDDIR) is changed, it is
+# trivial to clean up the builddir manually.
+clean:
+	rm -rf _build
diff --git a/_buildsystem/gen_links.inc.sh b/_buildsystem/gen_links.inc.sh
index 5701f0f..161754c 100644
--- a/_buildsystem/gen_links.inc.sh
+++ b/_buildsystem/gen_links.inc.sh
@@ -1,45 +1,57 @@
-#!dont_run_this
+# Copyright 2017 Harald Welte
+# Copyright 2018-2025 sysmocom - s.f.m.c. GmbH
+# SPDX-License-Identifier: Apache-2.0
 # This file is sourced by */gen_links.sh
 
-# Copyright 2017 Harald Welte
-# Copyright 2018 sysmocom - s.f.m.c. GmbH
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+if [ -e Makefile ]; then
+	echo
+	echo "ERROR: found old build artefacts in source tree"
+	echo
+	echo "osmo-ttcn3-hacks does out-of-tree builds now. Make sure that you"
+	echo "have comitted all code changes and run 'make clean-old' once"
+	echo "to clean up your source tree, then try again."
+	echo
+	exit 1
+fi
 
-rm -f .gitignore
+TOPDIR="$(realpath "$(dirname "$0")/..")"
+PROJECTDIR=$(realpath . --relative-to "$TOPDIR")  # e.g. "msc", "library/rua"
+BUILDDIR="${BUILDDIR:-$TOPDIR/_build}"
 
-gen_link() {
-	src="$1"
-	f="$2"
-	echo "Linking $f"
-	ln -sf "$src" "$f"
-	echo "$f" >> .gitignore
-}
+mkdir -p "$BUILDDIR/$PROJECTDIR"
 
 gen_links() {
-	DIR=$1
+	local f
+	local dir="$1"
 	shift
-	FILES=$*
-	for f in $FILES; do
-		gen_link "$DIR/$f" "$f"
+	local files="$*"
+
+	for f in $files; do
+		(ln -sf \
+			"$(realpath "$TOPDIR/$PROJECTDIR/$dir/$f")" \
+			"$BUILDDIR/$PROJECTDIR/$f") &
 	done
 }
 
 gen_links_finish() {
-	# Avoid using the pattern itself if no file is found in globbing below:
-	shopt -s nullglob
-	for pp in *.ttcnpp; do
-		ttcn_file="$(echo $pp | sed 's/pp$//')"
-		echo "$ttcn_file" >> .gitignore
+	local f
+	local patterns="
+		*.asn
+		*.c
+		*.cc
+		*.h
+		*.hh
+		*.ttcn
+		*.ttcnpp
+	"
+	for f in $patterns; do
+		if ! [ -e "$f" ]; then
+			continue
+		fi
+
+		ln -sf \
+			"$TOPDIR/$PROJECTDIR/$f" \
+			"$BUILDDIR/$PROJECTDIR/$f" &
 	done
+	wait
 }
diff --git a/_buildsystem/regen_makefile.inc.sh b/_buildsystem/regen_makefile.inc.sh
index 5534ff5..497fa09 100644
--- a/_buildsystem/regen_makefile.inc.sh
+++ b/_buildsystem/regen_makefile.inc.sh
@@ -18,6 +18,12 @@
 	exit 1
 fi
 
+TOPDIR="$(realpath "$(dirname "$0")/..")"
+BUILDDIR="${BUILDDIR:-$TOPDIR/_build}"
+PROJECTDIR=$(realpath . --relative-to "$TOPDIR")  # e.g. "msc", "library/rua"
+
+cd "$BUILDDIR/$PROJECTDIR"
+
 ttcn3_makefilegen -g -p -l -U 8 -f -e "$NAME" $FILES
 
 sed -i -e 's/# TTCN3_DIR = /TTCN3_DIR = \/usr/' Makefile
diff --git a/_buildsystem/rmlinks.sh b/_buildsystem/rmlinks.sh
deleted file mode 100755
index 65ee03c..0000000
--- a/_buildsystem/rmlinks.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#!/bin/sh
-find . -type l -delete
diff --git a/_testenv/testenv/testsuite.py b/_testenv/testenv/testsuite.py
index 9b28668..c1d048b 100644
--- a/_testenv/testenv/testsuite.py
+++ b/_testenv/testenv/testsuite.py
@@ -3,7 +3,6 @@
 import atexit
 import glob
 import logging
-import multiprocessing
 import os
 import os.path
 import shlex
@@ -99,11 +98,7 @@
 
 def build():
     logging.info("Building testsuite")
-    testsuite_dir = f"{ttcn3_hacks_dir}/{testenv.args.testsuite}"
-    testenv.cmd.run(["make", "compile"], cwd=testsuite_dir)
-
-    jobs = multiprocessing.cpu_count() + 1
-    testenv.cmd.run(["make", "-j", f"{jobs}"], cwd=testsuite_dir)
+    testenv.cmd.run(["make", testenv.args.testsuite], cwd=ttcn3_hacks_dir)
 
 
 def is_running(pid):
@@ -173,6 +168,7 @@
     cwd = os.path.join(testenv.testdir.testdir, "testsuite")
     start_testsuite = os.path.join(ttcn3_hacks_dir, "start-testsuite.sh")
     suite = os.path.join(ttcn3_hacks_dir, testenv.args.testsuite, section_data["program"])
+    suite = os.path.relpath(suite, ttcn3_hacks_dir)
 
     env = {
         "TTCN3_PCAP_PATH": os.path.join(testenv.testdir.testdir, "testsuite"),
diff --git a/start-testsuite.sh b/start-testsuite.sh
index 4b88501..8c55779 100755
--- a/start-testsuite.sh
+++ b/start-testsuite.sh
@@ -27,8 +27,11 @@
 	exit 1
 fi
 
+TOPDIR="$(realpath "$(dirname "$0")")"
+BUILDDIR="${BUILDDIR:-$TOPDIR/_build}"
+
 SUITE=$1
-SUITE_DIR="$(dirname "$SUITE")"
+SUITE_DIR="$(basename "$(dirname "$SUITE")")"
 SUITE_NAME="$(basename "$SUITE")"
 CFG="$SUITE_NAME.cfg"
 if [ $# -gt 1 ]; then
@@ -52,9 +55,13 @@
 # below is for the debian packages
 TTCN3_BIN_DIR="${TTCN3_BIN_DIR:-/usr/bin}"
 TITAN_LIBRARY_PATH="${TITAN_LIBRARY_PATH:-/usr/lib/titan:/usr/ttcn3/lib}"
-LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$SUITE_DIR:$TITAN_LIBRARY_PATH" "$TTCN3_BIN_DIR/ttcn3_start" $SUITE $CFG $TEST
+LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$BUILDDIR/$SUITE_DIR:$TITAN_LIBRARY_PATH" \
+	"$TTCN3_BIN_DIR/ttcn3_start" \
+	"$BUILDDIR/$SUITE_DIR/$SUITE_NAME" \
+	"$CFG" \
+	"$TEST"
 
-expected="$SUITE_DIR/expected-results.xml"
+expected="$TOPDIR/$SUITE_DIR/expected-results.xml"
 if [ ! -f "$expected" ]; then
   echo "No expected results found, not comparing outcome. ($expected)"
   exit 0
@@ -67,7 +74,7 @@
   exit 1
 fi
 
-compare="$SUITE_DIR/../compare-results.py"
+compare="$TOPDIR/compare-results.py"
 if [ ! -x "$compare" ]; then
   echo "ERROR: cannot find $compare"
   exit 1

