osmith submitted this change.
scripts/osmo-build.sh / ARTIFACT_STORE: remove
This script was supposed to cache already built Osmocom libraries. But
it was only used with openbsc, didn't work as expected anymore (looking
at openbsc-master, it just builds all deps from source without caching)
and will be replaced with logic that allows using ccache in a future
patch.
Using ccache has the big advantage that we can cache all build objects
where the source files did not change, and not just do it on a
dependency level. Oftentimes only few source files change in our
patches, so we can really cache everything else that is not affected
within any git repository and not just a dependency that didn't change.
ccache also does automatic cache size management to ensure a cache stays
within a given size limit.
Related: OS#5848
Depends: openbsc Ib3272feec76b30412ca60dec204255b64e33831b
Change-Id: Id94d6126b476077d57839e4a884621b8c034f0c6
---
M jobs/gerrit-verifications.yml
M jobs/master-builds.yml
D scripts/osmo-build.sh
3 files changed, 0 insertions(+), 238 deletions(-)
diff --git a/jobs/gerrit-verifications.yml b/jobs/gerrit-verifications.yml
index b83b5aa..9d1acb9 100644
--- a/jobs/gerrit-verifications.yml
+++ b/jobs/gerrit-verifications.yml
@@ -48,11 +48,8 @@
# Documentation for variables (keep in sync!):
# https://osmocom.org/projects/osmocom-servers/wiki/Jenkins_build_verification_jobs
docker_run: |
- ARTIFACT_STORE="$HOME/jenkins_build_artifact_store"
- mkdir -p "$ARTIFACT_STORE"
docker run --rm=true \
--cap-add SYS_PTRACE \
- -e ARTIFACT_STORE=/artifact_store \
-e ASCIIDOC_WARNINGS_CHECK="1" \
-e HOME=/build \
-e JOB_NAME="$JOB_NAME" \
@@ -63,7 +60,6 @@
-e WITH_MANUALS="1" \
-w /build -i -u build \
-v "$PWD:/build" \
- -v "$ARTIFACT_STORE:/artifact_store" \
docker_img: '$USER/debian-buster-jenkins'
docker_img_erlang: '$USER/debian-bullseye-erlang'
timeout_cmd: '/usr/bin/timeout 30m'
diff --git a/jobs/master-builds.yml b/jobs/master-builds.yml
index 0f22a77..7d1e6da 100644
--- a/jobs/master-builds.yml
+++ b/jobs/master-builds.yml
@@ -22,11 +22,8 @@
# Documentation for variables (keep in sync!):
# https://osmocom.org/projects/osmocom-servers/wiki/Jenkins_build_verification_jobs
docker_run: |
- ARTIFACT_STORE="$HOME/jenkins_build_artifact_store"
- mkdir -p "$ARTIFACT_STORE"
docker run --rm=true \
--cap-add SYS_PTRACE \
- -e ARTIFACT_STORE=/artifact_store \
-e ASCIIDOC_WARNINGS_CHECK="1" \
-e HOME=/build \
-e JOB_NAME="$JOB_NAME" \
@@ -40,7 +37,6 @@
-w /build -i -u build \
-v $(readlink -f $SSH_AUTH_SOCK):/ssh-agent \
-v "$PWD:/build" \
- -v "$ARTIFACT_STORE:/artifact_store" \
-v "$HOME/.ssh:/home/build/.ssh:ro" \
docker_img: '$USER/debian-buster-jenkins'
timeout_cmd: '/usr/bin/timeout 30m'
diff --git a/scripts/osmo-build.sh b/scripts/osmo-build.sh
deleted file mode 100644
index 43413cf..0000000
--- a/scripts/osmo-build.sh
+++ /dev/null
@@ -1,230 +0,0 @@
-#!/bin/sh
-#
-# This script enables artifacts holding dependencies on a jenkins job level to
-# speed up builds. Basically, it holds logic to check whether the necessary artifact
-# is available. If so it fetches artifact, unpacks it and if cp/tar succeeded
-# it triggers the actual build.
-#
-# Otherwise it simply builds all dependencies from source by using osmo-build-dep.sh
-# and archives deps to the ARTIFACT_STORE afterwards. Revisions of locally built
-# dependencies are detrmined after dependencies are built to ensure catching new
-# changes in dep_n+1 meanwhile building dep_n.
-#
-# Furthermore, ARTIFACT_STORE environment variable has to be set on all jenkins slaves.
-# The JOB_NAME variable will be injected to each jenkins' job by jenkins itself.
-# When using script within a docker container one must inject jenkins' JOB_NAME variable
-# to the container and ensure that ARTIFACT_STORE is mounted to the container's
-# internal ARTIFACT_STORE.
-#
-# Artifacts will be stored as follows:
-#
-# $ARTIFACT_STORE/$JOB_NAME/<dep_1>.<branch_1>.<rev_1>_...
-# ..._<dep_n>.<tag_n>.tar.gz
-#
-# Note: each matrix-build has its own directory inside ARTIFACT_STORE.
-#
-# In order to make use of osmo-build.sh one needs to source it, e.g. from
-# ./contrib/jenkins.sh. Furthermore, jenkins should check out the git tree of
-# the project to be built in the workspace root. Following functions needs to be
-# declared within a build script that sources osmo-build.sh:
-#
-# - artifact_name()
-# - build_deps()
-# - build_project()
-#
-# This is an example for building "libosmo-netif" which depends on "libosmocore"
-# and "libosmo-abis".
-#
-# #!/bin/sh
-#
-# artifact_deps() {
-# # $1 will be one of folllowing functions:
-# # - artifact_name_by_local_repo()
-# # - artifact_name_by_remote_repo()
-# # osmo-build.sh takes care about which function to use
-#
-# x="$($1 libosmocore)"
-# x="${x}_$($1 libosmo-abis)"
-#
-# echo "${x}.tar.gz"
-# }
-#
-# build_deps() {
-# # all commands to build necessary dependencies
-# osmo-build-dep.sh libosmocore master ac_cv_path_DOXYGEN=false
-# "$deps"/libosmocore/contrib/verify_value_string_arrays_are_terminated.py $(find . -name "*.[hc]")
-# osmo-build-dep.sh libosmo-abis
-# }
-#
-# build_project() {
-# # Necessary commands to build the project, expecting all dependencies have
-# # been built or fetched. Commands within build_project() will be executed
-# # in jenkins' $WORKSPACE.
-#
-# autoreconf --install --force
-# ./configure --enable-sanitize
-# $MAKE $PARALLEL_MAKE
-# $MAKE distcheck || cat-testlogs.sh
-# }
-#
-##
-# # source osmo-build.sh to fire the build
-# . osmo-build.sh
-
-log() {
- set +x
- echo
- echo "[INFO] $1"
- echo
- set -x
-}
-
-# SOURCING SANITY
-log "source sanity check to ensure that sourcing script holds necessary functions"
-type artifact_deps
-type build_deps
-type build_project
-log "check whether necessary dependency build scripts are in PATH"
-type osmo-build-dep.sh
-type osmo-deps.sh
-
-# BUILD FUNCTIONS
-init_build() {
-
- if [ -z "$JOB_NAME" ]; then
- log "[ERROR] JOB_NAME variable is not set, running in Jenkins?"
- exit 1
- fi
-
- if [ -z "$ARTIFACT_STORE" ]; then
- log "[ERROR] ARTIFACT_STORE variable is not set on this build slave"
- exit 1
- fi
-
- base="$(pwd)"
- deps="$base/deps"
- inst="$deps/install"
- rm -rf "$deps" || true
-
- # obtain the project name from the git clone found in the workspace root
- project=$(git config --get --local remote.origin.url \
- | cut -d '/' -f4 | cut -d '.' -f1)
-
- # replace invalid char for dirs in $JOB_NAME (jenkins variable)
- # ( '/' separates job name and matrix-axis)
- job_name="$( echo "$JOB_NAME" | tr '/' '_')"
-
- export base deps inst project job_name
- export PKG_CONFIG_PATH="$inst/lib/pkgconfig:$PKG_CONFIG_PATH"
- export LD_LIBRARY_PATH="$inst/lib"
-
- log "$project build initialized"
-}
-
-build() {
-
- init_build
-
- artifact_name="$(artifact_name)"
-
- if [ -f "$ARTIFACT_STORE/$job_name/$artifact_name" ]; then
- fetch_artifact "$ARTIFACT_STORE/$job_name" "$artifact_name"
- else
- log "Compile $project dependencies from source."
- mkdir -p "$deps"
- rm -rf "$inst"
-
- build_deps
- archive_artifact
- fi
-
- log "building $project"
- build_project
-}
-
-# ARTIFACT FUNCTIONS
-artifact_name() {
- # in case deps is empty or does not exist we
- if [ -d "$deps" ]; then
- artifact_deps "branch_and_rev_of_local_repo"
- cd "$base"
- else
- artifact_deps "branch_and_rev_of_remote_repo"
- fi
-}
-
-branch_and_rev_of_local_repo() {
- cd "$deps/$1"
- rev="$(git rev-parse --short HEAD)"
- branch="$(git rev-parse --abbrev-ref HEAD)"
-
- # check whether it is a tag
- if [ "$branch" = "HEAD" ]; then
- tag="$(git describe --tags HEAD)"
- tag="$(echo "$tag" | tr '/' '_')"
- echo "$1.$tag"
- else
- branch="$( echo "$branch" | tr '/' '_')"
- echo "$1.$branch.$rev"
- fi
-}
-
-branch_and_rev_of_remote_repo() {
- if [ -z "${2+x}" ]; then branch="master"; else branch="$2"; fi
- branch="$( echo "$branch" | tr '/' '_')"
- rev="$(git ls-remote "https://gerrit.osmocom.org/$1" "refs/heads/$branch")"
-
- # check whether branch is a tag
- if [ "$rev" = "" ]; then
- echo "$1.$branch"
- else
- rev="$(echo "$rev" | cut -c 1-7)"
- echo "$1.$branch.$rev"
- fi
-}
-
-archive_artifact() {
- log "Archiving artifact to artifactStore."
-
- cd "$base"
- artifact="$(artifact_name)"
- # temp_job_store is necessary to atomically move it to production.
- temp_job_store="$ARTIFACT_STORE/tmp/$job_name/"
- job_store="$ARTIFACT_STORE/$job_name/"
-
- if [ ! -f "$temp_job_store/$artifact" ]; then
- mkdir -p "$job_store" "$temp_job_store"
- # remove outdated artifact first to avoid temporarily
- # doubling of artifact storage consumption
- rm -f "$job_store/*"
- tar czf "$temp_job_store/$artifact" "deps"
- mv -n "$temp_job_store/$artifact" "$job_store/$artifact"
- rm -rf "$temp_job_store"
-
- log_artifact_hash "$job_store/$artifact"
- fi
-}
-
-fetch_artifact() {
- log "Fetching artifact from artifactStore."
-
- log_artifact_hash "$1/$2"
- cp "$1/$2" .
- log_artifact_hash "$2"
- tar xzf "$2"
-
- if [ $? -gt 0 ]; then
- log "Artifact could not be fetched, triggering build_deps()"
- build_deps
- else
- log "Artifact successfully fetched, triggering $project compilation"
- fi
-}
-
-# checksum is not used by script itself,
-# but might be handy in logs when debugging.
-log_artifact_hash() {
- log "name: $1 \n sha256: $(sha256sum "$1" | cut -d ' ' -f1)"
-}
-
-build
To view, visit change 31041. To unsubscribe, or for help writing mail filters, visit settings.