osmith has submitted this change. ( https://gerrit.osmocom.org/c/osmo-dev/+/41514?usp=email ) Change subject: src/_update_src_copy: don't copy deleted files ...................................................................... src/_update_src_copy: don't copy deleted files Fix that _update_src_copy.sh is failing with the following error on git repos with deleted but not yet commited files: updating src_copy: /home/lynxis/.cache/osmo-ttcn3-testenv/podman/make4-asan-debian-trixie/src_copy/libosmo-netif/ rsync: [sender] link_stat "/home/lynxis/projects/osmocom/repos/libosmo-netif/doc/twrtp/README" failed: No such file or directory (2) rsync: [sender] link_stat "/home/lynxis/projects/osmocom/repos/libosmo-netif/doc/twrtp/twrtp-guide.t" failed: No such file or directory (2) rsync error: some files/attrs were not transferred (see previous errors) (code 23) at main.c(1338) [sender=3.4.1] I have implemented this with "comm" to substract the list of deleted files from the list of relevant files to copy, as there was no option for "git ls-files" to do this in one step, and as using "--exclude-from" with this file didn't work in combination with the already passed allowlist with neither "git ls-files --exclude-from" nor "rsync --exclude-from". Fixes: OS#6889 Change-Id: I232263bc5bd26c90f1e73b115480269165a1e1f8 --- M src/_update_src_copy.sh A tests/test_update_src_copy.py 2 files changed, 94 insertions(+), 4 deletions(-) Approvals: Jenkins Builder: Verified pespin: Looks good to me, but someone else must approve fixeria: Looks good to me, approved diff --git a/src/_update_src_copy.sh b/src/_update_src_copy.sh index f13a357..941c3a2 100644 --- a/src/_update_src_copy.sh +++ b/src/_update_src_copy.sh @@ -1,4 +1,6 @@ #!/bin/sh -e +# Copyright 2025 sysmocom - s.f.m.c. GmbH +# SPDX-License-Identifier: GPL-3.0-or-later # Update the src_copy dir with all relevant files from the original git # repository (+ submodules). Used by gen_makefile.py --autoreconf-in-src-copy. @@ -8,6 +10,13 @@ echo "updating src_copy: $dest" + # Get files that are deleted, but not committed yet + git \ + -C "$src" \ + ls-files \ + --deleted \ + | sort >"$LIST_DELETED" + # Get list of files (everything that is not in gitignore) git \ -C "$src" \ @@ -15,14 +24,21 @@ --others \ --cached \ --exclude-standard \ - > "$COPY_LIST" + | sort >"$LIST_COPY" + + # Remove the deleted files from the list + comm \ + -3 \ + "$LIST_COPY" \ + "$LIST_DELETED" \ + >"$LIST_COPY_WITHOUT_DELETED" mkdir -p "$dest" # Copy files that changed rsync \ --archive \ - --files-from="$COPY_LIST" \ + --files-from="$LIST_COPY_WITHOUT_DELETED" \ "$src" \ "$dest" } @@ -114,11 +130,16 @@ DEST_DIR_PROJ="$MAKE_DIR/src_copy/$PROJ" handle_symlink_proj -COPY_LIST="$(mktemp --suffix=-osmo-dev-rsync-copylist)" +LIST_COPY="$(mktemp --suffix=-osmo-dev-rsync-list-copy)" +LIST_DELETED="$(mktemp --suffix=-osmo-dev-rsync-list-deleted)" +LIST_COPY_WITHOUT_DELETED="$(mktemp --suffix=-osmo-dev-rsync-list-copy-without-deleted)" cd "$SRC_DIR/$PROJ" update_git_dirs_all -rm "$COPY_LIST" +rm \ + "$LIST_COPY" \ + "$LIST_DELETED" \ + "$LIST_COPY_WITHOUT_DELETED" echo "$TIME_START" >"$MARKER" # Output an empty line when done to make the Makefile output more readable diff --git a/tests/test_update_src_copy.py b/tests/test_update_src_copy.py new file mode 100644 index 0000000..bb573ae --- /dev/null +++ b/tests/test_update_src_copy.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +# Copyright 2025 sysmocom - s.f.m.c. GmbH +# SPDX-License-Identifier: GPL-3.0-or-later +import os +import subprocess + +osmo_dev_path = os.path.realpath(os.path.join(__file__, "../../")) +script = os.path.join(osmo_dev_path, "src/_update_src_copy.sh") + + +def run_cmd(cmd, *args, **kwargs): + print(f"+ {cmd}") + return subprocess.run(cmd, check=True, *args, **kwargs) + + +def test_update_git_dir(tmp_path): + proj = "testproj" + src_dir = os.path.join(tmp_path, "src") + proj_dir = os.path.join(src_dir, proj) + make_dir = os.path.join(tmp_path, "make") + dest_dir_proj = os.path.join(make_dir, "src_copy", proj) + + run_cmd(["mkdir", "-p", proj_dir, make_dir]) + + # Create git repo + os.chdir(proj_dir) + run_cmd(["git", "init", "."]) + run_cmd(["git", "config", "user.email", "osmo-dev@test"]) + run_cmd(["git", "config", "user.name", "osmo-dev-test"]) + + # Make initial commit + run_cmd(["touch", "1.c", "2.c", "3.c"]) + run_cmd("echo '*.o' >.gitignore", shell=True) + run_cmd(["git", "add", "-A"]) + run_cmd(["git", "commit", "-m", "initial"]) + + # Make uncommitted changes + run_cmd(["touch", "1.o", "4.c", "5.c"]) + run_cmd(["rm", "1.c", "2.c"]) + + # Stage half of the changes + run_cmd(["git", "add", "1.c", "4.c"]) + + # Run _update_src_copy.sh + os.chdir(make_dir) + time_start = "1" + run_cmd(["sh", "-ex", script, src_dir, proj, time_start]) + assert sorted(os.listdir(dest_dir_proj)) == [ + ".gitignore", + "3.c", + "4.c", + "5.c", + ] + + # Commit changes + os.chdir(proj_dir) + run_cmd(["git", "add", "-A"]) + run_cmd(["git", "commit", "-m", "test"]) + + # Run _update_src_copy.sh again (this time LIST_DELETED is empty) + os.chdir(make_dir) + time_start = "2" + run_cmd(["sh", "-ex", script, src_dir, proj, time_start]) + assert sorted(os.listdir(dest_dir_proj)) == [ + ".gitignore", + "3.c", + "4.c", + "5.c", + ] -- To view, visit https://gerrit.osmocom.org/c/osmo-dev/+/41514?usp=email To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings?usp=email Gerrit-MessageType: merged Gerrit-Project: osmo-dev Gerrit-Branch: master Gerrit-Change-Id: I232263bc5bd26c90f1e73b115480269165a1e1f8 Gerrit-Change-Number: 41514 Gerrit-PatchSet: 4 Gerrit-Owner: osmith <osmith@sysmocom.de> Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: fixeria <vyanitskiy@sysmocom.de> Gerrit-Reviewer: lynxis lazus <lynxis@fe80.eu> Gerrit-Reviewer: osmith <osmith@sysmocom.de> Gerrit-Reviewer: pespin <pespin@sysmocom.de>