[PATCH] osmo-ci[master]: Introduce artifacts holding dependencies to speed up builds.

This is merely a historical archive of years 2008-2021, before the migration to mailman3.

A maintained and still updated list archive can be found at https://lists.osmocom.org/hyperkitty/list/gerrit-log@lists.osmocom.org/.

blobb gerrit-no-reply at lists.osmocom.org
Tue May 2 19:06:08 UTC 2017


Introduce artifacts holding dependencies to speed up builds.

Basically, osmo-build.sh holds logic to check whether the necessary 
artifact is available. If so it fetches artifact, unpacks it and 
triggers the actual build. In case the necessary artifact is not 
available osmo-build.sh simply builds all dependencies from source 
by using osmo-build-dep.sh and archives deps to the ARTIFACT_STORE 
afterwards.

The necessary functions to determine the artifact name from remote and 
local repositories as well as the handling of artifact files live in 
osmo-artifacts.sh, which is sourced by osmo-build.sh.

osmo-build.sh will be sourced by the contrib/jenkins.sh build script 
inside each git repository and invoked via 'build'. 
See jenkins-openBsc.sh [1] for more details.

Artifacts will be stored as follows:

	$ARTIFACT_STORE/$JOB_NAME/<dep_1>.<branch_1>.<rev_1>_...
		..._<dep_n>.<branch_n>.<rev_n>.tar.gz

Furthermore, ARTIFACT_STORE environment variable has to be set on all 
jenkins slaves. The JOB_NAME variables is injected to each jenkins job 
by jenkins.

[1] https://github.com/blobbsen/diy-artifacts/blob/master/jenkins-openBSC.sh

Change-Id: Ifee0a2f837d23b19aa5326f810234d5452e47484
---
A scripts/osmo-artifacts.sh
A scripts/osmo-build.sh
2 files changed, 134 insertions(+), 0 deletions(-)


  git pull ssh://gerrit.osmocom.org:29418/osmo-ci refs/changes/65/2465/3

diff --git a/scripts/osmo-artifacts.sh b/scripts/osmo-artifacts.sh
new file mode 100755
index 0000000..ceea623
--- /dev/null
+++ b/scripts/osmo-artifacts.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+
+# determining artifact name (local & remote)
+getArtifactNameByLocalRepos(){
+	genericDeps "getBranchAndRevByLocalRepo"
+	cd "$base"
+}
+
+getArtifactNameByRemoteRepos() {
+	genericDeps "getBranchAndRevByRemoteRepo"
+}
+
+getBranchAndRevByLocalRepo() {
+	cd "$deps/$1"
+	rev=$(git rev-parse --short HEAD)
+	branch=$(git rev-parse --abbrev-ref HEAD)
+	echo "$1.${branch//\//#}.${rev}"
+}
+
+getBranchAndRevByRemoteRepo() {
+	if [ -z "${2+x}" ]; then branch="master"; else branch="$2"; fi
+	rev=$(git ls-remote "https://git.osmocom.org/$1" "refs/heads/$branch")
+	echo "$1.${branch//\//#}.${rev:0:7}"
+}
+
+# file handling
+archiveArtifact() {
+	set +x
+	echo
+	echo "[INFO] Archiving artifact to artifactStore."
+	echo
+	set -x
+
+	cd "$base"
+	artifact="$(getArtifactNameByLocalRepos)"
+	tempJobStore="$ARTIFACT_STORE/tmp/$jobName/"
+	jobStore="$ARTIFACT_STORE/$jobName/"
+
+	if [ ! -f "$tempJobStore/$artifact" ]; then
+		mkdir -p "$jobStore" "$tempJobStore"
+		rm -f "$jobStore/*"
+		tar czf "$tempJobStore/$artifact" "deps"
+		mv -n "$tempJobStore/$artifact" "$jobStore/$artifact"
+		rm -f "$tempJobStore"
+
+		generateArtifactHashes "$jobStore/$artifact"
+	fi
+}
+
+fetchArtifact() {
+	set +x
+	echo
+	echo "[INFO] Fetching artifact from artifactStore."
+	echo
+	set -x
+
+	generateArtifactHashes "$1"
+	tar xzf "$1"
+}
+
+generateArtifactHashes() {
+	set +x
+	echo
+	echo "[INFO] name: $1"
+	echo "[INFO] md5: $(md5sum "$1" | cut -d ' ' -f1)"
+	echo "[INFO] sha1: $(sha1sum "$1" | cut -d ' ' -f1)"
+	echo "[INFO] sha256: $(sha256sum "$1" | cut -d ' ' -f1)"
+	echo
+	set -x
+	sleep 1
+}
diff --git a/scripts/osmo-build.sh b/scripts/osmo-build.sh
new file mode 100644
index 0000000..a3c1e74
--- /dev/null
+++ b/scripts/osmo-build.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+
+source osmo-artifacts.sh
+
+initBuild() {
+
+	if [ -z "$JOB_NAME" ]; then
+		set +x
+		echo
+		echo "[ERROR] JOB_NAME variable is not set, running in Jenkins?"
+		echo
+		set -x
+		exit 1
+	fi
+
+	base="$(pwd)"
+	deps="$base/deps"
+	inst="$deps/install"
+
+	project=$(git config --get --local remote.origin.url \
+		| cut -d '/' -f4 | cut -d '.' -f1)
+
+	jobName="${JOB_NAME//\//#}"
+
+	export base deps inst project jobName
+	export PKG_CONFIG_PATH="$inst/lib/pkgconfig:$PKG_CONFIG_PATH"
+	export LD_LIBRARY_PATH="$inst/lib"
+}
+
+buildDeps() {
+	set +x
+	echo
+	echo "[INFO] Compile $project dependencies from source."
+	echo
+	set -x
+
+	mkdir -p "$deps"
+	rm -rf "$inst"
+	genericDeps "osmo-build-dep.sh"
+}
+
+build() {
+
+	initBuild
+
+	neededArtifact="$(getArtifactNameByRemoteRepos)"
+	pathOfNeededArtifact="$ARTIFACT_STORE/$jobName/$neededArtifact"
+
+	if [ -f "$pathOfNeededArtifact" ]; then
+		fetchArtifact "$pathOfNeededArtifact"
+	else
+		buildDeps
+		archiveArtifact
+	fi
+
+	set +x
+	echo
+	echo " ============================= $project =========================="
+	echo
+	set -x
+
+	buildProject
+}

-- 
To view, visit https://gerrit.osmocom.org/2465
To unsubscribe, visit https://gerrit.osmocom.org/settings

Gerrit-MessageType: newpatchset
Gerrit-Change-Id: Ifee0a2f837d23b19aa5326f810234d5452e47484
Gerrit-PatchSet: 3
Gerrit-Project: osmo-ci
Gerrit-Branch: master
Gerrit-Owner: blobb <dr.blobb at gmail.com>
Gerrit-Reviewer: Holger Freyther <holger at freyther.de>
Gerrit-Reviewer: neels <nhofmeyr at sysmocom.de>



More information about the gerrit-log mailing list