[PATCH] osmo-ci[master]: Introduce artifacts holding dependencies to speed up builds.

[blobb]

Introduce artifacts holding dependencies to speed up builds.

Basically, osmo-build.sh holds logic to check whether the necessary artifact
is available. If so it fetches artifact, unpacks it and triggers the actual build.
In case the necessary artifact is not available osmo-build.sh simply builds all
dependencies from source by using osmo-build-dep.sh and archives deps to the
ARTIFACT_STORE afterwards.

The necessary functions to determine the artifact name from remote and local
repositories as well as the handling of artifact files live in osmo-artifacts.sh,
which is sourced by osmo-build.sh.

osmo-build.sh will be sourced by the contrib/jenkins.sh build script inside
each git repository and invoked via 'build'. See jenkins-openBsc [1] for
more details.

Artifacts will be stored as follows:

	$ARTIFACT_STORE/$JOB_NAME/<dep_1>.<branch_1>.<rev_1>_...
		..._<dep_n>.<branch_n>.<rev_n>.tar.gz

Furthermore, ARTIFACT_STORE environment variable has to be set on all jenkins
slaves. The JOB_NAME variables is injected to each jenkins job by jenkins.

[1] https://github.com/blobbsen/diy-artifacts/blob/master/jenkins-openBSC.sh

Change-Id: Ifee0a2f837d23b19aa5326f810234d5452e47484
---
A scripts/osmo-artifacts.sh
A scripts/osmo-build.sh
2 files changed, 134 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.osmocom.org:29418/osmo-ci refs/changes/65/2465/2

diff --git a/scripts/osmo-artifacts.sh b/scripts/osmo-artifacts.sh
new file mode 100755
index 0000000..ceea623
--- /dev/null
+++ b/scripts/osmo-artifacts.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+
+# determining artifact name (local & remote)
+getArtifactNameByLocalRepos(){
+	genericDeps "getBranchAndRevByLocalRepo"
+	cd "$base"
+}
+
+getArtifactNameByRemoteRepos() {
+	genericDeps "getBranchAndRevByRemoteRepo"
+}
+
+getBranchAndRevByLocalRepo() {
+	cd "$deps/$1"
+	rev=$(git rev-parse --short HEAD)
+	branch=$(git rev-parse --abbrev-ref HEAD)
+	echo "$1.${branch//\//#}.${rev}"
+}
+
+getBranchAndRevByRemoteRepo() {
+	if [ -z "${2+x}" ]; then branch="master"; else branch="$2"; fi
+	rev=$(git ls-remote "https://git.osmocom.org/$1" "refs/heads/$branch")
+	echo "$1.${branch//\//#}.${rev:0:7}"
+}
+
+# file handling
+archiveArtifact() {
+	set +x
+	echo
+	echo "[INFO] Archiving artifact to artifactStore."
+	echo
+	set -x
+
+	cd "$base"
+	artifact="$(getArtifactNameByLocalRepos)"
+	tempJobStore="$ARTIFACT_STORE/tmp/$jobName/"
+	jobStore="$ARTIFACT_STORE/$jobName/"
+
+	if [ ! -f "$tempJobStore/$artifact" ]; then
+		mkdir -p "$jobStore" "$tempJobStore"
+		rm -f "$jobStore/*"
+		tar czf "$tempJobStore/$artifact" "deps"
+		mv -n "$tempJobStore/$artifact" "$jobStore/$artifact"
+		rm -f "$tempJobStore"
+
+		generateArtifactHashes "$jobStore/$artifact"
+	fi
+}
+
+fetchArtifact() {
+	set +x
+	echo
+	echo "[INFO] Fetching artifact from artifactStore."
+	echo
+	set -x
+
+	generateArtifactHashes "$1"
+	tar xzf "$1"
+}
+
+generateArtifactHashes() {
+	set +x
+	echo
+	echo "[INFO] name: $1"
+	echo "[INFO] md5: $(md5sum "$1" | cut -d ' ' -f1)"
+	echo "[INFO] sha1: $(sha1sum "$1" | cut -d ' ' -f1)"
+	echo "[INFO] sha256: $(sha256sum "$1" | cut -d ' ' -f1)"
+	echo
+	set -x
+	sleep 1
+}
diff --git a/scripts/osmo-build.sh b/scripts/osmo-build.sh
new file mode 100644
index 0000000..a3c1e74
--- /dev/null
+++ b/scripts/osmo-build.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+
+source osmo-artifacts.sh
+
+initBuild() {
+
+	if [ -z "$JOB_NAME" ]; then
+		set +x
+		echo
+		echo "[ERROR] JOB_NAME variable is not set, running in Jenkins?"
+		echo
+		set -x
+		exit 1
+	fi
+
+	base="$(pwd)"
+	deps="$base/deps"
+	inst="$deps/install"
+
+	project=$(git config --get --local remote.origin.url \
+		| cut -d '/' -f4 | cut -d '.' -f1)
+
+	jobName="${JOB_NAME//\//#}"
+
+	export base deps inst project jobName
+	export PKG_CONFIG_PATH="$inst/lib/pkgconfig:$PKG_CONFIG_PATH"
+	export LD_LIBRARY_PATH="$inst/lib"
+}
+
+buildDeps() {
+	set +x
+	echo
+	echo "[INFO] Compile $project dependencies from source."
+	echo
+	set -x
+
+	mkdir -p "$deps"
+	rm -rf "$inst"
+	genericDeps "osmo-build-dep.sh"
+}
+
+build() {
+
+	initBuild
+
+	neededArtifact="$(getArtifactNameByRemoteRepos)"
+	pathOfNeededArtifact="$ARTIFACT_STORE/$jobName/$neededArtifact"
+
+	if [ -f "$pathOfNeededArtifact" ]; then
+		fetchArtifact "$pathOfNeededArtifact"
+	else
+		buildDeps
+		archiveArtifact
+	fi
+
+	set +x
+	echo
+	echo " ============================= $project =========================="
+	echo
+	set -x
+
+	buildProject
+}

-- 
To view, visit https://gerrit.osmocom.org/2465
To unsubscribe, visit https://gerrit.osmocom.org/settings

Gerrit-MessageType: newpatchset
Gerrit-Change-Id: Ifee0a2f837d23b19aa5326f810234d5452e47484
Gerrit-PatchSet: 2
Gerrit-Project: osmo-ci
Gerrit-Branch: master
Gerrit-Owner: blobb <dr.blobb at gmail.com>