osmith has submitted this change. ( https://gerrit.osmocom.org/c/osmo-dev/+/39404?usp=email ) Change subject: gen_makefile: add --autoreconf-in-src-copy arg ...................................................................... gen_makefile: add --autoreconf-in-src-copy arg When this new argument is set, do not run "autoreconf -fi" directly in the source directory. Instead, make a copy of all relevant sources in the git repository (everything except .git and files that are matched by gitignore) right before working with the source dir. Then use that copy instead. This avoids the problem that ./configure refuses to run if it has already been executed in the source dir. (It aborts and asks the user to run "make distclean" first, which is especially annoying if it has to be done in multiple source dirs before being able to build successfully.) Copying the sources is very fast due to generating a list of files that need to be copied first, and due to using rsync which skips the copy if the file size and timestamp already match. On my machine the copying is not noticable except for the additional prints: updating src_copy: /home/user/.cache/osmo-ttcn3-testenv/host/make2/src_copy/osmo-mgw/ updating version: 1.13.1.38-78ee9 Change-Id: I18ac50e3441df81e1fe7d8d5321df7e80ab9c650 --- M .gitignore M gen_makefile.py A src/_update_src_copy.sh 3 files changed, 164 insertions(+), 14 deletions(-) Approvals: pespin: Looks good to me, but someone else must approve osmith: Verified neels: Looks good to me, approved diff --git a/.gitignore b/.gitignore index 770ca29..0241046 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ !src/gits !src/grd !src/osmo-add-gerrit-hooks.sh +!src/_update_src_copy.sh !src/README net/* !net/templates* diff --git a/gen_makefile.py b/gen_makefile.py index e1baa5b..3130940 100755 --- a/gen_makefile.py +++ b/gen_makefile.py @@ -122,6 +122,9 @@ parser.add_argument('-i', '--install-prefix', default='/usr/local', help='''install there instead of /usr/local''') +parser.add_argument('-A', '--autoreconf-in-src-copy', action='store_true', + help="run autoreconf in a copy of the source dir, avoids 'run make distclean' errors") + args = parser.parse_args() class listdict(dict): @@ -175,12 +178,13 @@ return {} return dict(read_projects_deps(path)) -def gen_makefile_clone(proj, src, src_proj, url, push_url): +def gen_makefile_clone(proj, src, src_proj, url, push_url, update_src_copy_cmd): if proj == "osmocom-bb_layer23": return f''' .make.{proj}.clone: .make.osmocom-bb.clone @echo "\\n\\n\\n===== $@\\n" test -L {src_proj} || ln -s osmocom-bb/src/host/layer23 {src_proj} + {update_src_copy_cmd} touch $@ ''' @@ -189,6 +193,7 @@ .make.{proj}.clone: .make.osmocom-bb.clone @echo "\\n\\n\\n===== $@\\n" test -L {src_proj} || ln -s osmocom-bb/src/host/virt_phy {src_proj} + {update_src_copy_cmd} touch $@ ''' @@ -197,6 +202,7 @@ .make.{proj}.clone: .make.osmocom-bb.clone @echo "\\n\\n\\n===== $@\\n" test -L {src_proj} || ln -s osmocom-bb/src/host/trxcon {src_proj} + {update_src_copy_cmd} touch $@ ''' @@ -205,6 +211,7 @@ .make.{proj}.clone: .make.simtrace2.clone @echo "\\n\\n\\n===== $@\\n" test -L {src_proj} || ln -s simtrace2/host {src_proj} + {update_src_copy_cmd} touch $@ ''' @@ -240,19 +247,22 @@ ( {cmd_clone} && {cmd_set_push_url} ); \\ fi + {update_src_copy_cmd} sync touch $@ ''' -def gen_makefile_autoconf(proj, src_proj, distclean_cond): +def gen_makefile_autoconf(proj, src_proj, src_proj_copy, distclean_cond, update_src_copy_cmd): buildsystem = projects_buildsystems.get(proj, "autotools") + if buildsystem == "autotools": return f''' .make.{proj}.autoconf: .make.{proj}.clone {src_proj}/configure.ac if {distclean_cond}; then $(MAKE) {proj}-distclean; fi @echo "\\n\\n\\n===== $@\\n" - -rm -f {src_proj}/.version - cd {src_proj}; autoreconf -fi + {update_src_copy_cmd} + -rm -f {src_proj_copy}/.version + cd {src_proj_copy}; autoreconf -fi sync touch $@ ''' @@ -263,13 +273,15 @@ def gen_makefile_configure(proj, deps_installed, distclean_cond, build_proj, - cflags, docker_cmd, build_to_src, configure_opts): + cflags, docker_cmd, build_to_src, configure_opts, + update_src_copy_cmd): buildsystem = projects_buildsystems.get(proj, "autotools") if buildsystem == "autotools": return f''' .make.{proj}.configure: .make.{proj}.autoconf {deps_installed} $({proj}_configure_files) if {distclean_cond}; then $(MAKE) {proj}-distclean .make.{proj}.autoconf; fi @echo "\\n\\n\\n===== $@\\n" + {update_src_copy_cmd} -chmod -R ug+w {build_proj} -rm -rf {build_proj} mkdir -p {build_proj} @@ -297,7 +309,7 @@ assert False, f"unknown buildsystem: {buildsystem}" def gen_makefile_build(proj, distclean_cond, build_proj, docker_cmd, jobs, - check, src_proj): + check, src_proj, update_src_copy_cmd): buildsystem = projects_buildsystems.get(proj, "autotools") if buildsystem == "autotools": @@ -305,6 +317,7 @@ .make.{proj}.build: .make.{proj}.configure $({proj}_files) if {distclean_cond}; then $(MAKE) {proj}-distclean .make.{proj}.configure; fi @echo "\\n\\n\\n===== $@\\n" + {update_src_copy_cmd} {docker_cmd}$(MAKE) -C {build_proj} -j {jobs} {check} sync touch $@ @@ -396,17 +409,46 @@ $(MAKE) -C {src_proj} distclean ''' +def is_src_copy_needed(proj): + if not args.autoreconf_in_src_copy: + return False + + # The rsync workaround to not write to the source dir during "autoreconf -fi" + # is only needed for autotools based projects + buildsystem = projects_buildsystems.get(proj, "autotools") + if buildsystem != "autotools": + return False + + return True + +def gen_update_src_copy_cmd(proj, src_dir, make_dir): + if not is_src_copy_needed(proj): + return "" + ret = "@sh -e " + ret += os.path.join(os.path.relpath(args.src_dir, make_dir), "_update_src_copy.sh") + ret += f" {shlex.quote(src_dir)}" + ret += f" {shlex.quote(proj)}" + ret += " $(TIME_START)" + return ret + +def gen_src_proj_copy(src_proj, make_dir, proj): + if not is_src_copy_needed(proj): + return src_proj + return os.path.join(make_dir, "src_copy", proj) + def gen_make(proj, deps, configure_opts, jobs, make_dir, src_dir, build_dir, url, push_url, sudo_make_install, no_ldconfig, ldconfig_without_sudo, make_check): src_proj = os.path.join(src_dir, proj) if proj == 'openbsc': src_proj = os.path.join(src_proj, 'openbsc') - build_proj = os.path.join(build_dir, proj) - build_to_src = os.path.relpath(src_proj, build_proj) - build_proj = os.path.relpath(build_proj, make_dir) - src = os.path.relpath(src_dir, make_dir) src_proj = os.path.relpath(src_proj, make_dir) + src_proj_copy = gen_src_proj_copy(src_proj, make_dir, proj) + + build_proj = os.path.join(build_dir, proj) + build_to_src = os.path.relpath(src_proj_copy, build_proj) + build_proj = os.path.relpath(build_proj, make_dir) + push_url = push_url or url if configure_opts: @@ -423,6 +465,7 @@ no_ldconfig = '#' if no_ldconfig else '' sudo_ldconfig = '' if ldconfig_without_sudo else 'sudo ' sudo_make_install = 'sudo ' if sudo_make_install else '' + update_src_copy_cmd = gen_update_src_copy_cmd(proj, src_dir, make_dir) return f''' ### {proj} ### @@ -443,9 +486,18 @@ \\) \\ -and -not -name "config.h" 2>/dev/null) -{gen_makefile_clone(proj, src, src_proj, url, push_url)} +{gen_makefile_clone(proj, + src, + src_proj, + url, + push_url, + update_src_copy_cmd)} -{gen_makefile_autoconf(proj, src_proj, distclean_cond)} +{gen_makefile_autoconf(proj, + src_proj, + src_proj_copy, + distclean_cond, + update_src_copy_cmd)} {gen_makefile_configure(proj, deps_installed, @@ -454,7 +506,8 @@ cflags, docker_cmd, build_to_src, - configure_opts_str)} + configure_opts_str, + update_src_copy_cmd)} {gen_makefile_build(proj, distclean_cond, @@ -462,7 +515,8 @@ docker_cmd, jobs, check, - src_proj)} + src_proj_copy, + update_src_copy_cmd)} {gen_makefile_install(proj, docker_cmd, @@ -606,8 +660,19 @@ content += " --build-debug \\\n" if args.auto_distclean: content += " --auto-distclean \\\n" +if args.autoreconf_in_src_copy: + content += " --autoreconf-in-src-copy \\\n" content += " $(NULL)\n" +if args.autoreconf_in_src_copy: + content += """ + +# --autoreconf-in-src-copy: get current time to avoid running rsync more than +# once within this Makefile per project +TIME_START := $(shell date +%s%N) + +""" + # convenience target: clone all repositories first content += 'clone: \\\n\t' + ' \\\n\t'.join([ '.make.%s.clone' % p for p, d in projects_deps ]) + '\n\n' diff --git a/src/_update_src_copy.sh b/src/_update_src_copy.sh new file mode 100644 index 0000000..d80b5fa --- /dev/null +++ b/src/_update_src_copy.sh @@ -0,0 +1,84 @@ +#!/bin/sh -e +# Update the src_copy dir with all relevant files from the original git +# repository (+ submodules). Used by gen_makefile.py --autoreconf-in-src-copy. + +MAKE_DIR="$PWD" +SRC_DIR="$1" +PROJ="$2" +TIME_START="$3" +MARKER="$MAKE_DIR/.make.$PROJ.src_copy" + +# Don't run more than once per "make" call +if [ "$(cat "$MARKER" 2>/dev/null)" = "$TIME_START" ]; then + exit 0 +fi + +DEST_DIR_PROJ="$MAKE_DIR/src_copy/$PROJ" +COPY_LIST="$(mktemp --suffix=-osmo-dev-rsync-copylist)" + +update_git_dir() { + local src="$1" + local dest="$DEST_DIR_PROJ/$(echo "$src" | cut -c 3-)" # cut: remove './' + + echo "updating src_copy: $dest" + + # Get list of files (everything that is not in gitignore) + git \ + -C "$src" \ + ls-files \ + --others \ + --cached \ + --exclude-standard \ + > "$COPY_LIST" + + mkdir -p "$dest" + + # Copy files that changed + rsync \ + --archive \ + --files-from="$COPY_LIST" \ + "$src" \ + "$dest" +} + +# The "git-version-gen" script only works correctly when running inside a git +# repository. Run it in the original git repo and copy its output to +# ".tarball-version", so when configure.ac runs it again the version gets read +# from there. +run_git_version_gen() { + if ! [ -e "$src/git-version-gen" ]; then + return + fi + + local version_file="$DEST_DIR_PROJ/$src/.tarball-version" + rm -f "$version_file" + + # git-version-gen does not run without the tarball-version argument + version_new="$(cd "$src" && ./git-version-gen "$version_file")" + + echo "updating version: $version_new" + echo "$version_new" >"$version_file" +} + +update_git_dirs_all() { + local git_dirs="$(find . -name '.git')" + + if [ -z "$git_dirs" ]; then + echo "_update_read_only_src_copy: find failed" + exit 1 + fi + + for git_dir in $git_dirs; do + local src="$(dirname "$git_dir")" + update_git_dir "$src" + run_git_version_gen "$src" + done +} + +cd "$SRC_DIR/$PROJ" +update_git_dirs_all +rm "$COPY_LIST" +echo "$TIME_START" >"$MARKER" + +# Output an empty line when done to make the Makefile output more readable +echo -- To view, visit https://gerrit.osmocom.org/c/osmo-dev/+/39404?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: osmo-dev Gerrit-Branch: master Gerrit-Change-Id: I18ac50e3441df81e1fe7d8d5321df7e80ab9c650 Gerrit-Change-Number: 39404 Gerrit-PatchSet: 2 Gerrit-Owner: osmith <osmith(a)sysmocom.de> Gerrit-Reviewer: fixeria <vyanitskiy(a)sysmocom.de> Gerrit-Reviewer: lynxis lazus <lynxis(a)fe80.eu> Gerrit-Reviewer: neels <nhofmeyr(a)sysmocom.de> Gerrit-Reviewer: osmith <osmith(a)sysmocom.de> Gerrit-Reviewer: pespin <pespin(a)sysmocom.de>