osmith has uploaded this change for review. ( https://gerrit.osmocom.org/c/osmo-ci/+/42095?usp=email )
Change subject: scripts/kernel/linux-shallow-clone: new script ......................................................................
scripts/kernel/linux-shallow-clone: new script
Add a script that maintains a linux repository in one place on jenkins nodes, so we need less git clones from git.kernel.org and less disk space. All jobs that need a kernel tree can now clone the relevant branch directly from the jenkins node.
Follow up patches will add a jenkins job that runs the script daily and adjust the existing jobs to make use of this instead of doing their own clones.
Currently this script produces a 396M bare git repository.
Related: OS#6938 Change-Id: Id3aadb46813047ecac3b80868192809b272dba0e --- A scripts/kernel/linux-shallow-clone.sh 1 file changed, 97 insertions(+), 0 deletions(-)
git pull ssh://gerrit.osmocom.org:29418/osmo-ci refs/changes/95/42095/1
diff --git a/scripts/kernel/linux-shallow-clone.sh b/scripts/kernel/linux-shallow-clone.sh new file mode 100755 index 0000000..3aa9267 --- /dev/null +++ b/scripts/kernel/linux-shallow-clone.sh @@ -0,0 +1,97 @@ +#!/bin/sh -ex +# Create a bare git repository and add/update shallow clones of linux.git +# branches that are relevant for our CI jobs. Jobs can then quickly clone a +# branch from this git repository and discard it afterwards. This saves disk +# space on our jenkins nodes, and keeps the traffic to git.kernel.org minimal. + +set_dest() { + if [ -n "$1" ]; then + DEST="$1" + else + DEST=/tmp/linux-shallow + fi + + # Check if an existing repository is from last month, and if that is + # the case then start with a new repository. This is done because the + # repository size increases with each fetch: we start with a shallow + # clone but can't truncate the history after follow-up fetches. + if [ -e "$DEST/.month" ] && [ "$(cat "$DEST/.month")" != "$(date +%m)" ]; then + DEST_OLD="$DEST" + DEST="$DEST-new" + fi +} + +init() { + if [ -d "$DEST" ]; then + return + fi + + mkdir -p "$DEST" + cd "$DEST" + git init . --bare + + # Garbage collect in foreground + git config gc.autoDetach false + + git remote add torvalds "https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git" + git remote add stable "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git" + git remote add net-next "https://git.kernel.org/pub/scm/linux/kernel/git/netdev/net-next.git" + + echo "$(date +%m)" > "$DEST/.month" +} + +fetch_branch() { + local remote="$1" + local branch="$2" + local head + + git fetch --no-tags --depth=1 "$remote" "$branch" + head="$(git log -n1 --pretty=format:"%H" "$remote"/"$branch")" + git branch --force "$remote-$branch" "$head" + + # Pretty print the current commit for the log + git -c color.ui=always log -1 --oneline "$remote-$branch" +} + +fetch_branches() { + fetch_branch torvalds "master" + + fetch_branch stable "linux-rolling-stable" + fetch_branch stable "linux-4.19.y" + fetch_branch stable "linux-5.10.y" + fetch_branch stable "linux-6.1.y" + fetch_branch stable "linux-6.12.y" + + fetch_branch net-next "main" +} + +collect_garbage() { + git gc --no-cruft --prune=now +} + +# Replace last month's repository if needed, see the comment in set_dest +replace_old() { + if [ -z "$DEST_OLD" ]; then + return + fi + + mv "$DEST_OLD" "$DEST"-old + mv "$DEST" "$DEST_OLD" + rm -rf "$DEST"-old + + DEST="$DEST_OLD" + DEST_OLD="" + cd "$DEST" +} + +show_size() { + du -h -s "$DEST" +} + +set_dest "$@" +init +cd "$DEST" +fetch_branches +collect_garbage +replace_old +show_size