osmith has submitted this change. ( https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/43170?usp=email )
Change subject: deps: fetch/clone: retry with backoff time ......................................................................
deps: fetch/clone: retry with backoff time
Replace the previous logic of running the whole Makefile again on any failure, with wrapping all git clone and fetch commands in a new retry_with_backoff_time function that actually sleeps before retrying (a random amount of seconds to make less requests at once), and retries up to 5 times, each time with likely more sleep time.
With this change it is more likely to succeed and we have less confusing output as deps/Makefile will not run twice if it fails for any reason (e.g. a syntax error).
Change-Id: I317c0357ff330a0626a622dadd1e44ba65b99545 --- M Makefile M deps/update.sh 2 files changed, 29 insertions(+), 6 deletions(-)
Approvals: Jenkins Builder: Verified laforge: Looks good to me, approved pespin: Looks good to me, but someone else must approve
diff --git a/Makefile b/Makefile index 9cafa4c..b58fcf3 100644 --- a/Makefile +++ b/Makefile @@ -77,10 +77,8 @@
default: deps all
-# Eclipse GitLab has rate limiting and sometimes to many concurrent conns fail. -# If -jN fails, retry with -j1. .make.deps: deps/Makefile - ($(MAKE) $(PARALLEL_MAKE) -C deps || $(MAKE) -j1 -C deps) + $(MAKE) $(PARALLEL_MAKE) -C deps touch $@
.PHONY: deps diff --git a/deps/update.sh b/deps/update.sh index 64c5b54..1307045 100755 --- a/deps/update.sh +++ b/deps/update.sh @@ -14,6 +14,31 @@ esac }
+# Eclipse GitLab has rate limiting and sometimes too many concurrent +# connections fail. If that happens, sleep and try again in a few (random) +# seconds, to give less concurrent load to the server. +retry_with_backoff_time() { + local max=5 + local sec + local i + + for i in $(seq 1 $max); do + if "$@"; then + return + fi + + if [ $i -lt $max ]; then + sec=$(($i * $(shuf -i 1-10 -n1))) + echo "[$DIR] Failed ($i/$max), retrying in ${sec}s..." + sleep $sec + else + echo "[$DIR] Failed ($i/$max), giving up!" + exit 1 + fi + echo "[$DIR] Retrying: $@" + done +} + update_url() { local current="$(git -C "$DIR" remote get-url origin)" local full_url="$(get_full_url)" @@ -21,7 +46,7 @@ if [ "$current" != "$full_url" ]; then echo "[$DIR] Updating URL to $full_url" git -C "$DIR" remote set-url origin "$full_url" - git -C "$DIR" fetch + retry_with_backoff_time git -C "$DIR" fetch fi }
@@ -29,7 +54,7 @@ update_url else echo "[$DIR] Initial git clone" - git clone -q "$(get_full_url)" + retry_with_backoff_time git clone -q "$(get_full_url)" fi
cd "$DIR" @@ -41,7 +66,7 @@
if ! git cat-file -e "$COMMIT"; then echo "[$DIR] Missing $COMMIT, fetching git repository" - git fetch + retry_with_backoff_time git fetch fi
if git rev-parse -q "origin/$COMMIT" 1>/dev/null 2>&1; then
-
osmith