[MERGED] osmo-gsm-tester[master]: jenkins: extract common parts of build scripts to separate file

This is merely a historical archive of years 2008-2021, before the migration to mailman3.

A maintained and still updated list archive can be found at https://lists.osmocom.org/hyperkitty/list/gerrit-log@lists.osmocom.org/.

Neels Hofmeyr gerrit-no-reply at lists.osmocom.org
Fri Jun 2 21:09:01 UTC 2017


Neels Hofmeyr has submitted this change and it was merged.

Change subject: jenkins: extract common parts of build scripts to separate file
......................................................................


jenkins: extract common parts of build scripts to separate file

Have all complexity in one common shell script, greatly simplify the individual
scripts.

This allows to provide a specific branch or git hash to build instead of
current master. Some scripts allowed to provide branch names before, this now
also allows using git hashes directly.

Environment variables can be used to override the git hash/branch to use for
specific repositories.

Motivation for this patch: we need this to investigate failure causes more
easily.

Change-Id: I5ac2f90d006a1b2f6c246976346d852a70c89089
---
A contrib/jenkins-build-common.sh
M contrib/jenkins-build-osmo-bts-sysmo.sh
M contrib/jenkins-build-osmo-bts-trx.sh
M contrib/jenkins-build-osmo-hlr.sh
M contrib/jenkins-build-osmo-msc.sh
M contrib/jenkins-build-osmo-nitb.sh
6 files changed, 190 insertions(+), 364 deletions(-)

Approvals:
  Neels Hofmeyr: Looks good to me, approved
  Jenkins Builder: Verified



diff --git a/contrib/jenkins-build-common.sh b/contrib/jenkins-build-common.sh
new file mode 100644
index 0000000..921434d
--- /dev/null
+++ b/contrib/jenkins-build-common.sh
@@ -0,0 +1,152 @@
+#!source_this_file
+
+# Common parts for osmo-gsm-tester jenkins build scripts. Use like in below example:
+#
+#--------------
+# #!/bin/sh
+# set -e -x
+# base="$PWD"
+# name="osmo-name"
+# . "$(dirname "$0")/jenkins-build-common.sh"
+#
+# build_repo libosmocore --configure --opts
+# build_repo libosmo-foo special_branch --configure --opts
+# build_repo osmo-bar
+#
+# create_bin_tgz
+#--------------
+#
+# Some explanations:
+#
+# To allow calling from arbitrary working directories, other scripts should
+# source this file like shown above.
+#
+# Sourcing scripts must provide some variables/functions, see above.
+# In addition, these values can optionally be passed to override:
+# git_url, prefix, prefix_real, BUILD_NUMBER
+#
+# CONFIGURE_FLAGS may contain flags that should be passed to all builds'
+# ./configure steps (useful e.g. for building in the sysmobts SDK).
+#
+# For each built repository, a specific git branch or hash can be provided by
+# environment variable: OSMO_GSM_TESTER_BUILD_$repo="<git-hash>"
+# NOTE: convert $repo's dashes to underscore. For example:
+#  OSMO_GSM_TESTER_BUILD_osmo_hlr="f001234abc"
+#  OSMO_GSM_TESTER_BUILD_libosmocore="my/branch"
+# ("origin/" is prepended to branch names automatically)
+
+if [ -z "$name" -o -z "$base" ]; then
+  set +x
+  echo "Some environment variables are not provided as required by jenkins-build-common.sh. Error."
+  exit 1
+fi
+
+git_url="${git_url-"git://git.osmocom.org"}"
+prefix="${prefix-"$base/inst-$name"}"
+# prefix_real is usually identical with prefix, except when installing to a
+# different $DESTDIR than /, which is the case for example when building
+# osmo-bts within the sysmoBTS SDK
+prefix_real="${prefix_real-"$prefix"}"
+
+export PKG_CONFIG_PATH="$prefix_real/lib/pkgconfig:$PKG_CONFIG_PATH"
+export LD_LIBRARY_PATH="$prefix_real/lib:$LD_LIBRARY_PATH"
+
+# Show current environment. Sometimes the LESS_ vars have ansi colors making a
+# mess, so exclude those.
+env | grep -v "^LESS" | sort
+
+# clean the workspace
+rm -f "$base/${name}"*.tgz rm -f "$base/${name}"*.md5
+rm -rf "$prefix_real"
+mkdir -p "$prefix_real"
+
+have_repo() {
+  repo="$1"
+  branch="${2-master}"
+
+  # Evaluate environment for instructions to build a specific git hash.
+  # Using a hash as $branch above unfortunately doesn't work.
+  branch_override_var="$(echo "OSMO_GSM_TESTER_BUILD_$repo" | sed 's/-/_/g')"
+  branch_override="$(eval "echo \$$branch_override_var")"
+  if [ -n "$branch_override" ]; then
+    branch="$branch_override"
+  fi
+
+  cd "$base"
+  if [ ! -d "$repo" ]; then
+    git clone "$git_url/$repo" "$repo"
+  fi
+  cd "$repo"
+  rm -rf *
+  git fetch origin
+
+  # Figure out whether we need to prepend origin/ to find branches in upstream
+  if ! git rev-parse "$branch"; then
+    branch="origin/$branch"
+  fi
+
+  git reset --hard "$branch"
+
+  git rev-parse HEAD
+
+  cd "$base"
+}
+
+build_repo() {
+  # usage: build_repo <name> [<branch>] [--configure-opts [...]]
+  dep="$1"
+  branch="master"
+  if [ -z "$(echo "$2" | grep '^-')" ]; then
+    # second arg does not start with a dash, it's empty or a branch
+    branch="$2"
+    if [ -n "$branch" ]; then
+      # we had a branch arg, need to shift once more to get config options
+      shift
+    else
+      branch="master"
+    fi
+  fi
+  shift
+  configure_opts="$@"
+
+  set +x; echo "
+
+====================== $dep
+
+"; set -x
+
+
+  have_repo "$dep" "$branch"
+
+  cd "$dep"
+
+  echo "$(git rev-parse HEAD) $dep" >> "$prefix_real/${name}_git_hashes.txt"
+
+  # special shim: we know the openbsc.git needs to be built in the openbsc/ subdir.
+  if [ "$dep" = "openbsc" ]; then
+    cd openbsc
+  fi
+
+  set +x; echo; echo; set -x
+  autoreconf -fi
+  set +x; echo; echo; set -x
+  ./configure --prefix="$prefix" $CONFIGURE_FLAGS $configure_opts
+  set +x; echo; echo; set -x
+  make -j8 || make  # libsmpp34 can't build in parallel
+  set +x; echo; echo; set -x
+  make install
+}
+
+create_bin_tgz() {
+  # don't package documentation -- the libosmocore docs can be up to 16 Mb large,
+  # a significant amount compared to the binaries
+  rm -rf "$prefix_real/share/doc/{libosmocore,libosmo-sccp}" || true
+
+  # build the archive that is going to be copied to the tester
+  cd "$prefix_real"
+  this="$name.build-${BUILD_NUMBER-$(date +%Y-%m-%d_%H_%M_%S)}"
+  tar="${this}.tgz"
+  tar czf "$base/$tar" *
+  cd "$base"
+  md5sum "$tar" > "${this}.md5"
+}
diff --git a/contrib/jenkins-build-osmo-bts-sysmo.sh b/contrib/jenkins-build-osmo-bts-sysmo.sh
index 9838834..dad2151 100755
--- a/contrib/jenkins-build-osmo-bts-sysmo.sh
+++ b/contrib/jenkins-build-osmo-bts-sysmo.sh
@@ -1,92 +1,23 @@
 #!/bin/sh
 set -e -x
 
-deps="
-libosmocore
-libosmo-abis
-osmo-bts
-"
+poky="/opt/poky/1.5.4"
+. "$poky/environment-setup-armv5te-poky-linux-gnueabi"
+
+# Cross-compilation: all installations need to be put in the sysmo SDK sysroot
+export DESTDIR="$poky/sysroots/armv5te-poky-linux-gnueabi"
 
 base="$PWD"
-rm -f "$base/osmo-bts-sysmo.*.tgz"
-
-have_repo() {
-	repo="$1"
-	cd "$base"
-	if [ ! -d "$repo" ]; then
-		git clone "git://git.osmocom.org/$repo" "$repo"
-	fi
-	cd "$repo"
-	git clean -dxf
-	git fetch origin
-	git reset --hard origin/master
-	git rev-parse HEAD
-	cd "$base"
-}
-
-for dep in $deps; do
-    have_repo "$dep"
-done
+name="osmo-bts-sysmo"
+prefix="/usr/local/jenkins-build/inst-$name"
+prefix_real="$DESTDIR$prefix"
+. "$(dirname "$0")/jenkins-build-common.sh"
 
 # for gsm_data_shared.h
 have_repo openbsc
 
-. /opt/poky/1.5.4/environment-setup-armv5te-poky-linux-gnueabi
+build_repo libosmocore --disable-pcsc
+build_repo libosmo-abis
+build_repo osmo-bts --enable-sysmocom-bts --with-openbsc=$base/openbsc/openbsc/include
 
-# Cross-compilation: all installations need to be put in the sysmo SDK sysroot
-export DESTDIR="/opt/poky/1.5.4/sysroots/armv5te-poky-linux-gnueabi"
-
-prefix_base="/usr/local/jenkins-build"
-prefix_base_real="$DESTDIR$prefix_base"
-rm -rf "$prefix_base_real"
-
-prefix="$prefix_base/inst-osmo-bts-sysmo"
-prefix_real="$DESTDIR$prefix"
-mkdir -p "$prefix_real"
-
-# Installation in non-system dir, but keep the PKG_CONFIG_PATH from the SDK:
-export PKG_CONFIG_PATH="$prefix_real/lib/pkgconfig:$PKG_CONFIG_PATH"
-
-env
-
-for dep in $deps; do
-	set +x; echo "
-
-====================== $dep
-
-"; set -x
-
-        cd "$base/$dep"
-        rm -rf *
-        git checkout .
-
-        echo "$(git rev-parse HEAD) $dep" >> "$prefix_real/osmo-bts-sysmo_git_hashes.txt"
-
-        autoreconf -fi
-
-        config_opts=""
-        case "$dep" in
-        'libosmocore')    config_opts="--disable-pcsc" ;;
-        'osmo-bts')       config_opts="--enable-sysmocom-bts --with-openbsc=$base/openbsc/openbsc/include" ;;
-        esac
-
-	set +x;	echo; echo; set -x
-        ./configure --prefix="$prefix" $CONFIGURE_FLAGS $config_opts
-	set +x;	echo; echo; set -x
-        make -j8
-	set +x;	echo; echo; set -x
-        make install
-done
-
-# don't package documentation -- the libosmocore docs can be up to 16 Mb large,
-# a significant amount compared to the binaries
-rm -rf "$prefix_real/share/doc"
-
-# build the archive that is going to be copied to the tester and then to the BTS
-rm "$base"/*.tgz "$base"/*.md5 || true
-cd "$prefix_real"
-this="osmo-bts-sysmo.build-${BUILD_NUMBER}"
-tar="${this}.tgz"
-tar czf "$base/$tar" *
-cd "$base"
-md5sum "$tar" > "${this}.md5"
+create_bin_tgz
diff --git a/contrib/jenkins-build-osmo-bts-trx.sh b/contrib/jenkins-build-osmo-bts-trx.sh
index 1b1d94e..3e61b70 100755
--- a/contrib/jenkins-build-osmo-bts-trx.sh
+++ b/contrib/jenkins-build-osmo-bts-trx.sh
@@ -1,82 +1,15 @@
 #!/bin/sh
-set -x -e
-
+set -e -x
 base="$PWD"
-prefix="$base/inst-osmo-bts-trx"
-
-rm -f "$base/osmo-bts-trx*.tgz"
-
-deps="
-libosmocore
-libosmo-abis
-osmo-trx
-osmo-bts
-"
-
-have_repo() {
-	repo="$1"
-	cd "$base"
-	if [ ! -d "$repo" ]; then
-		git clone "git://git.osmocom.org/$repo" "$repo"
-	fi
-	cd "$repo"
-	git clean -dxf
-	git fetch origin
-	git reset --hard origin/master
-	git rev-parse HEAD
-	cd "$base"
-}
+name="osmo-bts-trx"
+. "$(dirname "$0")/jenkins-build-common.sh"
 
 # for gsm_data_shared.*
 have_repo openbsc
 
+build_repo libosmocore
+build_repo libosmo-abis
+build_repo osmo-trx --without-sse
+build_repo osmo-bts --enable-trx --with-openbsc=$base/openbsc/openbsc/include
 
-rm -rf "$prefix"
-mkdir -p "$prefix"
-
-export PKG_CONFIG_PATH="$prefix/lib/pkgconfig"
-export LD_LIBRARY_PATH="$prefix/lib"
-
-for dep in $deps; do
-	set +x; echo "
-
-====================== $dep
-
-"; set -x
-
-	have_repo "$dep"
-	cd "$dep"
-	rm -rf *
-	git checkout .
-
-	echo "$(git rev-parse HEAD) $dep" >> "$prefix/osmo-bts-trx_osmo-trx_git_hashes.txt"
-
-	autoreconf -fi
-
-	config_opts=""
-
-	case "$repo" in
-	'osmo-bts') config_opts="--enable-trx --with-openbsc=$base/openbsc/openbsc/include" ;;
-	'osmo-trx') config_opts="--without-sse" ;;
-	esac
-
-	set +x;	echo; echo; set -x
-	./configure --prefix="$prefix" $config_opts
-	set +x;	echo; echo; set -x
-	make -j8
-	set +x;	echo; echo; set -x
-	make install
-done
-
-# don't package documentation -- the libosmocore docs can be up to 16 Mb large,
-# a significant amount compared to the binaries
-rm -rf "$prefix/share/doc"
-
-# build the archive that is going to be copied to the tester
-rm "$base"/*.tgz "$base"/*.md5 || true
-cd "$prefix"
-this="osmo-bts-trx.build-${BUILD_NUMBER}"
-tar="${this}.tgz"
-tar czf "$base/$tar" *
-cd "$base"
-md5sum "$tar" > "${this}.md5"
+create_bin_tgz
diff --git a/contrib/jenkins-build-osmo-hlr.sh b/contrib/jenkins-build-osmo-hlr.sh
index 6fbebab..8207451 100755
--- a/contrib/jenkins-build-osmo-hlr.sh
+++ b/contrib/jenkins-build-osmo-hlr.sh
@@ -1,69 +1,11 @@
 #!/bin/sh
 set -e -x
-
 base="$PWD"
-prefix="$base/inst-osmo-hlr"
-
-rm -f "$base/osmo-hlr*.tgz"
-
-git_url="git://git.osmocom.org"
-
-have_repo() {
-	repo="$1"
-	branch="${2-master}"
-
-	cd "$base"
-	if [ ! -d "$repo" ]; then
-		git clone "$git_url/$repo" -b "$branch" "$repo"
-	fi
-	cd "$repo"
-	rm -rf *
-	git fetch origin
-	git checkout .
-	git checkout "$branch"
-	git reset --hard origin/"$branch"
-	git rev-parse HEAD
-
-	cd "$base"
-}
-
-build_repo() {
-	dep="$1"
-	branch="${2-master}"
-
-	have_repo "$dep" "$branch"
-
-	cd "$dep"
-
-	echo "$(git rev-parse HEAD) $dep" >> "$prefix/osmo-hlr_git_hashes.txt"
-
-	config_opts=""
-
-	autoreconf -fi
-	./configure --prefix="$prefix" $config_opts
-	make -j8
-	make install
-}
-
-rm -rf "$prefix"
-mkdir -p "$prefix"
-
-export PKG_CONFIG_PATH="$prefix/lib/pkgconfig"
-export LD_LIBRARY_PATH="$prefix/lib"
+name="osmo-hlr"
+. "$(dirname "$0")/jenkins-build-common.sh"
 
 build_repo libosmocore
 build_repo libosmo-abis
 build_repo osmo-hlr
 
-# don't package documentation -- the libosmocore docs can be up to 16 Mb large,
-# a significant amount compared to the binaries
-rm -rf "$prefix/share/doc/libosmocore"
-
-# build the archive that is going to be copied to the tester
-rm "$base"/*.tgz "$base"/*.md5 || true
-cd "$prefix"
-this="osmo-hlr.build-${BUILD_NUMBER-$(date +%Y-%m-%d_%H_%M_%S)}"
-tar="${this}.tgz"
-tar czf "$base/$tar" *
-cd "$base"
-md5sum "$tar" > "${this}.md5"
+create_bin_tgz
diff --git a/contrib/jenkins-build-osmo-msc.sh b/contrib/jenkins-build-osmo-msc.sh
index f7808ff..53a23fe 100755
--- a/contrib/jenkins-build-osmo-msc.sh
+++ b/contrib/jenkins-build-osmo-msc.sh
@@ -1,62 +1,8 @@
 #!/bin/sh
 set -e -x
-
 base="$PWD"
-prefix="$base/inst-osmo-msc"
-
-rm -f "$base/osmo-msc*.tgz"
-
-git_url="git://git.osmocom.org"
-
-have_repo() {
-	repo="$1"
-	branch="${2-master}"
-
-	cd "$base"
-	if [ ! -d "$repo" ]; then
-		git clone "$git_url/$repo" -b "$branch" "$repo"
-	fi
-	cd "$repo"
-	rm -rf *
-	git fetch origin
-	git checkout .
-	git checkout "$branch"
-	git reset --hard origin/"$branch"
-	git rev-parse HEAD
-
-	cd "$base"
-}
-
-build_repo() {
-	dep="$1"
-	branch="${2-master}"
-
-	have_repo "$dep" "$branch"
-
-	cd "$dep"
-
-	echo "$(git rev-parse HEAD) $dep" >> "$prefix/osmo-msc_git_hashes.txt"
-
-	config_opts=""
-
-	case "$dep" in
-	'openbsc')
-		config_opts="$config_opts --enable-smpp --enable-osmo-bsc --enable-nat --enable-iu"
-		cd openbsc/
-	;;
-	esac
-
-	autoreconf -fi
-	./configure --prefix="$prefix" $config_opts
-	make -j8 || make  # libsmpp34 can't build in parallel
-	make install
-}
-
-rm -rf "$prefix"
-mkdir -p "$prefix"
-
-export PKG_CONFIG_PATH="$prefix/lib/pkgconfig"
-export LD_LIBRARY_PATH="$prefix/lib"
+name="osmo-msc"
+. "$(dirname "$0")/jenkins-build-common.sh"
 
 build_repo libosmocore
 build_repo libosmo-abis
@@ -66,17 +12,6 @@
 build_repo libosmo-sccp neels/aoip # TEMPORARY BRANCH
 build_repo libasn1c
 build_repo osmo-iuh neels/sigtran # TEMPORARY BRANCH
-build_repo openbsc aoip
+build_repo openbsc aoip --enable-smpp --enable-osmo-bsc --enable-nat --enable-iu
 
-# don't package documentation -- the libosmocore docs can be up to 16 Mb large,
-# a significant amount compared to the binaries
-rm -rf "$prefix/share/doc"
-
-# build the archive that is going to be copied to the tester
-rm "$base"/*.tgz "$base"/*.md5 || true
-cd "$prefix"
-this="osmo-msc.build-${BUILD_NUMBER-$(date +%Y-%m-%d_%H_%M_%S)}"
-tar="${this}.tgz"
-tar czf "$base/$tar" *
-cd "$base"
-md5sum "$tar" > "${this}.md5"
+create_bin_tgz
diff --git a/contrib/jenkins-build-osmo-nitb.sh b/contrib/jenkins-build-osmo-nitb.sh
index 86d787c..76fd6ba 100755
--- a/contrib/jenkins-build-osmo-nitb.sh
+++ b/contrib/jenkins-build-osmo-nitb.sh
@@ -1,82 +1,15 @@
 #!/bin/sh
 set -e -x
-
 base="$PWD"
-prefix="$base/inst-osmo-nitb"
+name="osmo-nitb"
+. "$(dirname "$0")/jenkins-build-common.sh"
 
-rm -f "$base/osmo-nitb*.tgz"
+build_repo libosmocore
+build_repo libosmo-abis
+build_repo libosmo-netif
+build_repo openggsn
+build_repo libsmpp34
+build_repo libosmo-sccp
+build_repo openbsc --enable-smpp --enable-osmo-bsc --enable-nat
 
-deps="
-libosmocore
-libosmo-abis
-libosmo-netif
-openggsn
-libsmpp34
-libosmo-sccp
-openbsc
-"
-
-have_repo() {
-	repo="$1"
-	cd "$base"
-	if [ ! -d "$repo" ]; then
-		git clone "git://git.osmocom.org/$repo" "$repo"
-	fi
-	cd "$repo"
-	git clean -dxf
-	git fetch origin
-	git reset --hard origin/master
-	git rev-parse HEAD
-	cd "$base"
-}
-
-rm -rf "$prefix"
-mkdir -p "$prefix"
-
-export PKG_CONFIG_PATH="$prefix/lib/pkgconfig"
-export LD_LIBRARY_PATH="$prefix/lib"
-
-for dep in $deps; do
-	set +x; echo "
-
-====================== $dep
-
-"; set -x
-
-	have_repo "$dep"
-	cd "$dep"
-	rm -rf *
-	git checkout .
-
-	echo "$(git rev-parse HEAD) $dep" >> "$prefix/osmo-nitb_git_hashes.txt"
-
-	config_opts=""
-
-	case "$dep" in
-	'openbsc')
-		config_opts="$config_opts --enable-smpp --enable-osmo-bsc --enable-nat"
-		cd openbsc/
-	;;
-	esac
-
-	autoreconf -fi
-	set +x;	echo; echo; set -x
-	./configure --prefix="$prefix" $config_opts
-	set +x;	echo; echo; set -x
-	make -j8 || make  # libsmpp34 can't build in parallel
-	set +x;	echo; echo; set -x
-	make install
-done
-
-# don't package documentation -- the libosmocore docs can be up to 16 Mb large,
-# a significant amount compared to the binaries
-rm -rf "$prefix/share/doc"
-
-# build the archive that is going to be copied to the tester
-rm "$base"/*.tgz "$base"/*.md5 || true
-cd "$prefix"
-this="osmo-nitb.build-${BUILD_NUMBER}"
-tar="${this}.tgz"
-tar czf "$base/$tar" *
-cd "$base"
-md5sum "$tar" > "${this}.md5"
+create_bin_tgz

-- 
To view, visit https://gerrit.osmocom.org/2828
To unsubscribe, visit https://gerrit.osmocom.org/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: I5ac2f90d006a1b2f6c246976346d852a70c89089
Gerrit-PatchSet: 4
Gerrit-Project: osmo-gsm-tester
Gerrit-Branch: master
Gerrit-Owner: Neels Hofmeyr <nhofmeyr at sysmocom.de>
Gerrit-Reviewer: Jenkins Builder
Gerrit-Reviewer: Neels Hofmeyr <nhofmeyr at sysmocom.de>



More information about the gerrit-log mailing list