[PATCH] osmo-ci[master]: Introduce artifacts holding dependencies to speed up builds.

[blobb]

Introduce artifacts holding dependencies to speed up builds.

Basically, osmo-build.sh holds logic to check whether the necessary
artifact is available. If so it fetches artifact, unpacks it and
triggers the actual build. In case the necessary artifact is not
available osmo-build.sh simply builds all dependencies from source
by using osmo-build-dep.sh and archives deps to the ARTIFACT_STORE
afterwards.

The necessary functions to determine the artifact name from remote and
local repositories as well as the handling of artifact files live in
osmo-artifacts.sh, which is sourced by osmo-build.sh.

osmo-build.sh will be sourced by the contrib/jenkins.sh build script
inside each git repository and invoked via 'build'.
See jenkins-openBsc.sh [1] for more details.

Artifacts will be stored as follows:

	$ARTIFACT_STORE/$JOB_NAME/<dep_1>.<branch_1>.<rev_1>_...
		..._<dep_n>.<branch_n>.<rev_n>.tar.gz

Furthermore, ARTIFACT_STORE environment variable has to be set on all
jenkins slaves. The JOB_NAME variables is injected to each jenkins job
by jenkins.

[1] https://github.com/blobbsen/diy-artifacts/blob/master/jenkins-openBSC.sh

Change-Id: Ifee0a2f837d23b19aa5326f810234d5452e47484
---
A scripts/osmo-artifacts.sh
A scripts/osmo-build.sh
2 files changed, 134 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.osmocom.org:29418/osmo-ci refs/changes/65/2465/4

diff --git a/scripts/osmo-artifacts.sh b/scripts/osmo-artifacts.sh
new file mode 100755
index 0000000..ceea623
--- /dev/null
+++ b/scripts/osmo-artifacts.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+
+# determining artifact name (local & remote)
+getArtifactNameByLocalRepos(){
+	genericDeps "getBranchAndRevByLocalRepo"
+	cd "$base"
+}
+
+getArtifactNameByRemoteRepos() {
+	genericDeps "getBranchAndRevByRemoteRepo"
+}
+
+getBranchAndRevByLocalRepo() {
+	cd "$deps/$1"
+	rev=$(git rev-parse --short HEAD)
+	branch=$(git rev-parse --abbrev-ref HEAD)
+	echo "$1.${branch//\//#}.${rev}"
+}
+
+getBranchAndRevByRemoteRepo() {
+	if [ -z "${2+x}" ]; then branch="master"; else branch="$2"; fi
+	rev=$(git ls-remote "https://git.osmocom.org/$1" "refs/heads/$branch")
+	echo "$1.${branch//\//#}.${rev:0:7}"
+}
+
+# file handling
+archiveArtifact() {
+	set +x
+	echo
+	echo "[INFO] Archiving artifact to artifactStore."
+	echo
+	set -x
+
+	cd "$base"
+	artifact="$(getArtifactNameByLocalRepos)"
+	tempJobStore="$ARTIFACT_STORE/tmp/$jobName/"
+	jobStore="$ARTIFACT_STORE/$jobName/"
+
+	if [ ! -f "$tempJobStore/$artifact" ]; then
+		mkdir -p "$jobStore" "$tempJobStore"
+		rm -f "$jobStore/*"
+		tar czf "$tempJobStore/$artifact" "deps"
+		mv -n "$tempJobStore/$artifact" "$jobStore/$artifact"
+		rm -f "$tempJobStore"
+
+		generateArtifactHashes "$jobStore/$artifact"
+	fi
+}
+
+fetchArtifact() {
+	set +x
+	echo
+	echo "[INFO] Fetching artifact from artifactStore."
+	echo
+	set -x
+
+	generateArtifactHashes "$1"
+	tar xzf "$1"
+}
+
+generateArtifactHashes() {
+	set +x
+	echo
+	echo "[INFO] name: $1"
+	echo "[INFO] md5: $(md5sum "$1" | cut -d ' ' -f1)"
+	echo "[INFO] sha1: $(sha1sum "$1" | cut -d ' ' -f1)"
+	echo "[INFO] sha256: $(sha256sum "$1" | cut -d ' ' -f1)"
+	echo
+	set -x
+	sleep 1
+}
diff --git a/scripts/osmo-build.sh b/scripts/osmo-build.sh
new file mode 100644
index 0000000..a3c1e74
--- /dev/null
+++ b/scripts/osmo-build.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+
+source osmo-artifacts.sh
+
+initBuild() {
+
+	if [ -z "$JOB_NAME" ]; then
+		set +x
+		echo
+		echo "[ERROR] JOB_NAME variable is not set, running in Jenkins?"
+		echo
+		set -x
+		exit 1
+	fi
+
+	base="$(pwd)"
+	deps="$base/deps"
+	inst="$deps/install"
+
+	project=$(git config --get --local remote.origin.url \
+		| cut -d '/' -f4 | cut -d '.' -f1)
+
+	jobName="${JOB_NAME//\//#}"
+
+	export base deps inst project jobName
+	export PKG_CONFIG_PATH="$inst/lib/pkgconfig:$PKG_CONFIG_PATH"
+	export LD_LIBRARY_PATH="$inst/lib"
+}
+
+buildDeps() {
+	set +x
+	echo
+	echo "[INFO] Compile $project dependencies from source."
+	echo
+	set -x
+
+	mkdir -p "$deps"
+	rm -rf "$inst"
+	genericDeps "osmo-build-dep.sh"
+}
+
+build() {
+
+	initBuild
+
+	neededArtifact="$(getArtifactNameByRemoteRepos)"
+	pathOfNeededArtifact="$ARTIFACT_STORE/$jobName/$neededArtifact"
+
+	if [ -f "$pathOfNeededArtifact" ]; then
+		fetchArtifact "$pathOfNeededArtifact"
+	else
+		buildDeps
+		archiveArtifact
+	fi
+
+	set +x
+	echo
+	echo " ============================= $project =========================="
+	echo
+	set -x
+
+	buildProject
+}

-- 
To view, visit https://gerrit.osmocom.org/2465
To unsubscribe, visit https://gerrit.osmocom.org/settings

Gerrit-MessageType: newpatchset
Gerrit-Change-Id: Ifee0a2f837d23b19aa5326f810234d5452e47484
Gerrit-PatchSet: 4
Gerrit-Project: osmo-ci
Gerrit-Branch: master
Gerrit-Owner: blobb <dr.blobb at gmail.com>
Gerrit-Reviewer: Holger Freyther <holger at freyther.de>
Gerrit-Reviewer: neels <nhofmeyr at sysmocom.de>