Change in osmo-ci[master]: scripts/osmo-build.sh / ARTIFACT_STORE: remove

osmith has uploaded this change for review. ( https://gerrit.osmocom.org/c/osmo-ci/+/31041 ) Change subject: scripts/osmo-build.sh / ARTIFACT_STORE: remove ...................................................................... scripts/osmo-build.sh / ARTIFACT_STORE: remove This script was supposed to cache already built Osmocom libraries. But it was only used with openbsc, didn't work as expected anymore (looking at openbsc-master, it just builds all deps from source without caching) and will be replaced with logic that allows using ccache in a future patch. Using ccache has the big advantage that we can cache all build objects where the source files did not change, and not just do it on a dependency level. Oftentimes only few source files change in our patches, so we can really cache everything else that is not affected within any git repository and not just a dependency that didn't change. ccache also does automatic cache size management to ensure a cache stays within a given size limit. Related: OS#5848 Depends: openbsc Ib3272feec76b30412ca60dec204255b64e33831b Change-Id: Id94d6126b476077d57839e4a884621b8c034f0c6 --- M jobs/gerrit-verifications.yml M jobs/master-builds.yml D scripts/osmo-build.sh 3 files changed, 0 insertions(+), 238 deletions(-) git pull ssh://gerrit.osmocom.org:29418/osmo-ci refs/changes/41/31041/1 diff --git a/jobs/gerrit-verifications.yml b/jobs/gerrit-verifications.yml index b83b5aa..9d1acb9 100644 --- a/jobs/gerrit-verifications.yml +++ b/jobs/gerrit-verifications.yml @@ -48,11 +48,8 @@ # Documentation for variables (keep in sync!): # https://osmocom.org/projects/osmocom-servers/wiki/Jenkins_build_verificatio… docker_run: | - ARTIFACT_STORE="$HOME/jenkins_build_artifact_store" - mkdir -p "$ARTIFACT_STORE" docker run --rm=true \ --cap-add SYS_PTRACE \ - -e ARTIFACT_STORE=/artifact_store \ -e ASCIIDOC_WARNINGS_CHECK="1" \ -e HOME=/build \ -e JOB_NAME="$JOB_NAME" \ @@ -63,7 +60,6 @@ -e WITH_MANUALS="1" \ -w /build -i -u build \ -v "$PWD:/build" \ - -v "$ARTIFACT_STORE:/artifact_store" \ docker_img: '$USER/debian-buster-jenkins' docker_img_erlang: '$USER/debian-bullseye-erlang' timeout_cmd: '/usr/bin/timeout 30m' diff --git a/jobs/master-builds.yml b/jobs/master-builds.yml index 0f22a77..7d1e6da 100644 --- a/jobs/master-builds.yml +++ b/jobs/master-builds.yml @@ -22,11 +22,8 @@ # Documentation for variables (keep in sync!): # https://osmocom.org/projects/osmocom-servers/wiki/Jenkins_build_verificatio… docker_run: | - ARTIFACT_STORE="$HOME/jenkins_build_artifact_store" - mkdir -p "$ARTIFACT_STORE" docker run --rm=true \ --cap-add SYS_PTRACE \ - -e ARTIFACT_STORE=/artifact_store \ -e ASCIIDOC_WARNINGS_CHECK="1" \ -e HOME=/build \ -e JOB_NAME="$JOB_NAME" \ @@ -40,7 +37,6 @@ -w /build -i -u build \ -v $(readlink -f $SSH_AUTH_SOCK):/ssh-agent \ -v "$PWD:/build" \ - -v "$ARTIFACT_STORE:/artifact_store" \ -v "$HOME/.ssh:/home/build/.ssh:ro" \ docker_img: '$USER/debian-buster-jenkins' timeout_cmd: '/usr/bin/timeout 30m' diff --git a/scripts/osmo-build.sh b/scripts/osmo-build.sh deleted file mode 100644 index 43413cf..0000000 --- a/scripts/osmo-build.sh +++ /dev/null @@ -1,230 +0,0 @@ -#!/bin/sh -# -# This script enables artifacts holding dependencies on a jenkins job level to -# speed up builds. Basically, it holds logic to check whether the necessary artifact -# is available. If so it fetches artifact, unpacks it and if cp/tar succeeded -# it triggers the actual build. -# -# Otherwise it simply builds all dependencies from source by using osmo-build-dep.sh -# and archives deps to the ARTIFACT_STORE afterwards. Revisions of locally built -# dependencies are detrmined after dependencies are built to ensure catching new -# changes in dep_n+1 meanwhile building dep_n. -# -# Furthermore, ARTIFACT_STORE environment variable has to be set on all jenkins slaves. -# The JOB_NAME variable will be injected to each jenkins' job by jenkins itself. -# When using script within a docker container one must inject jenkins' JOB_NAME variable -# to the container and ensure that ARTIFACT_STORE is mounted to the container's -# internal ARTIFACT_STORE. -# -# Artifacts will be stored as follows: -# -# $ARTIFACT_STORE/$JOB_NAME/<dep_1>.<branch_1>.<rev_1>_... -# ..._<dep_n>.<tag_n>.tar.gz -# -# Note: each matrix-build has its own directory inside ARTIFACT_STORE. -# -# In order to make use of osmo-build.sh one needs to source it, e.g. from -# ./contrib/jenkins.sh. Furthermore, jenkins should check out the git tree of -# the project to be built in the workspace root. Following functions needs to be -# declared within a build script that sources osmo-build.sh: -# -# - artifact_name() -# - build_deps() -# - build_project() -# -# This is an example for building "libosmo-netif" which depends on "libosmocore" -# and "libosmo-abis". -# -# #!/bin/sh -# -# artifact_deps() { -# # $1 will be one of folllowing functions: -# # - artifact_name_by_local_repo() -# # - artifact_name_by_remote_repo() -# # osmo-build.sh takes care about which function to use -# -# x="$($1 libosmocore)" -# x="${x}_$($1 libosmo-abis)" -# -# echo "${x}.tar.gz" -# } -# -# build_deps() { -# # all commands to build necessary dependencies -# osmo-build-dep.sh libosmocore master ac_cv_path_DOXYGEN=false -# "$deps"/libosmocore/contrib/verify_value_string_arrays_are_terminated.py $(find . -name "*.[hc]") -# osmo-build-dep.sh libosmo-abis -# } -# -# build_project() { -# # Necessary commands to build the project, expecting all dependencies have -# # been built or fetched. Commands within build_project() will be executed -# # in jenkins' $WORKSPACE. -# -# autoreconf --install --force -# ./configure --enable-sanitize -# $MAKE $PARALLEL_MAKE -# $MAKE distcheck || cat-testlogs.sh -# } -# -## -# # source osmo-build.sh to fire the build -# . osmo-build.sh - -log() { - set +x - echo - echo "[INFO] $1" - echo - set -x -} - -# SOURCING SANITY -log "source sanity check to ensure that sourcing script holds necessary functions" -type artifact_deps -type build_deps -type build_project -log "check whether necessary dependency build scripts are in PATH" -type osmo-build-dep.sh -type osmo-deps.sh - -# BUILD FUNCTIONS -init_build() { - - if [ -z "$JOB_NAME" ]; then - log "[ERROR] JOB_NAME variable is not set, running in Jenkins?" - exit 1 - fi - - if [ -z "$ARTIFACT_STORE" ]; then - log "[ERROR] ARTIFACT_STORE variable is not set on this build slave" - exit 1 - fi - - base="$(pwd)" - deps="$base/deps" - inst="$deps/install" - rm -rf "$deps" || true - - # obtain the project name from the git clone found in the workspace root - project=$(git config --get --local remote.origin.url \ - | cut -d '/' -f4 | cut -d '.' -f1) - - # replace invalid char for dirs in $JOB_NAME (jenkins variable) - # ( '/' separates job name and matrix-axis) - job_name="$( echo "$JOB_NAME" | tr '/' '_')" - - export base deps inst project job_name - export PKG_CONFIG_PATH="$inst/lib/pkgconfig:$PKG_CONFIG_PATH" - export LD_LIBRARY_PATH="$inst/lib" - - log "$project build initialized" -} - -build() { - - init_build - - artifact_name="$(artifact_name)" - - if [ -f "$ARTIFACT_STORE/$job_name/$artifact_name" ]; then - fetch_artifact "$ARTIFACT_STORE/$job_name" "$artifact_name" - else - log "Compile $project dependencies from source." - mkdir -p "$deps" - rm -rf "$inst" - - build_deps - archive_artifact - fi - - log "building $project" - build_project -} - -# ARTIFACT FUNCTIONS -artifact_name() { - # in case deps is empty or does not exist we - if [ -d "$deps" ]; then - artifact_deps "branch_and_rev_of_local_repo" - cd "$base" - else - artifact_deps "branch_and_rev_of_remote_repo" - fi -} - -branch_and_rev_of_local_repo() { - cd "$deps/$1" - rev="$(git rev-parse --short HEAD)" - branch="$(git rev-parse --abbrev-ref HEAD)" - - # check whether it is a tag - if [ "$branch" = "HEAD" ]; then - tag="$(git describe --tags HEAD)" - tag="$(echo "$tag" | tr '/' '_')" - echo "$1.$tag" - else - branch="$( echo "$branch" | tr '/' '_')" - echo "$1.$branch.$rev" - fi -} - -branch_and_rev_of_remote_repo() { - if [ -z "${2+x}" ]; then branch="master"; else branch="$2"; fi - branch="$( echo "$branch" | tr '/' '_')" - rev="$(git ls-remote "https://gerrit.osmocom.org/$1" "refs/heads/$branch")" - - # check whether branch is a tag - if [ "$rev" = "" ]; then - echo "$1.$branch" - else - rev="$(echo "$rev" | cut -c 1-7)" - echo "$1.$branch.$rev" - fi -} - -archive_artifact() { - log "Archiving artifact to artifactStore." - - cd "$base" - artifact="$(artifact_name)" - # temp_job_store is necessary to atomically move it to production. - temp_job_store="$ARTIFACT_STORE/tmp/$job_name/" - job_store="$ARTIFACT_STORE/$job_name/" - - if [ ! -f "$temp_job_store/$artifact" ]; then - mkdir -p "$job_store" "$temp_job_store" - # remove outdated artifact first to avoid temporarily - # doubling of artifact storage consumption - rm -f "$job_store/*" - tar czf "$temp_job_store/$artifact" "deps" - mv -n "$temp_job_store/$artifact" "$job_store/$artifact" - rm -rf "$temp_job_store" - - log_artifact_hash "$job_store/$artifact" - fi -} - -fetch_artifact() { - log "Fetching artifact from artifactStore." - - log_artifact_hash "$1/$2" - cp "$1/$2" . - log_artifact_hash "$2" - tar xzf "$2" - - if [ $? -gt 0 ]; then - log "Artifact could not be fetched, triggering build_deps()" - build_deps - else - log "Artifact successfully fetched, triggering $project compilation" - fi -} - -# checksum is not used by script itself, -# but might be handy in logs when debugging. -log_artifact_hash() { - log "name: $1 \n sha256: $(sha256sum "$1" | cut -d ' ' -f1)" -} - -build -- To view, visit https://gerrit.osmocom.org/c/osmo-ci/+/31041 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-ci Gerrit-Branch: master Gerrit-Change-Id: Id94d6126b476077d57839e4a884621b8c034f0c6 Gerrit-Change-Number: 31041 Gerrit-PatchSet: 1 Gerrit-Owner: osmith &lt;osmith(a)sysmocom.de&gt; Gerrit-MessageType: newchange