[M] Change in osmo-ci[master]: jobs/gerrit-binpkgs: don't hardcode distros

osmith has submitted this change. ( https://gerrit.osmocom.org/c/osmo-ci/+/33988 ) Change subject: jobs/gerrit-binpkgs: don't hardcode distros ...................................................................... jobs/gerrit-binpkgs: don't hardcode distros Make it possible to set a list of distributions to check in gerrit-verifications.yml instead of only having one boolean for testing two hardcoded rpm and deb distributions. Change-Id: I59487e3dc2f55057de1b6a322f088fff0d18654c --- M jobs/gerrit-binpkgs.yml M jobs/gerrit-verifications.yml M scripts/jenkins-gerrit/comment_generate.py 3 files changed, 109 insertions(+), 61 deletions(-) Approvals: laforge: Looks good to me, approved pespin: Looks good to me, but someone else must approve Jenkins Builder: Verified diff --git a/jobs/gerrit-binpkgs.yml b/jobs/gerrit-binpkgs.yml index 7a4e9e9..ecfbd52 100644 --- a/jobs/gerrit-binpkgs.yml +++ b/jobs/gerrit-binpkgs.yml @@ -5,13 +5,10 @@ - project: name: gerrit-binpkgs jobs: - - 'gerrit-binpkgs-{type}' - type: - - 'deb' - - 'rpm' + - 'gerrit-binpkgs' -- job-template: - name: 'gerrit-binpkgs-{type}' +- job: + name: 'gerrit-binpkgs' project-type: freestyle node: osmocom-gerrit concurrent: true @@ -21,7 +18,7 @@ days-to-keep: 30 num-to-keep: 1000 description: | - {type} package job of CI for patches sent to + binary packages job (deb/rpm) of CI for patches sent to <a href="https://gerrit.osmocom.org">gerrit</a> </br></br> Related issue: <a href="https://osmocom.org/issues/2385">OS#2385</a> @@ -44,6 +41,9 @@ - string: name: PROJECT_NAME description: project to test, e.g. 'libosmocore' (set by pipeline) + - string: + name: DISTRO + description: distribution to build for, e.g. 'debian:12' (set by pipeline) scm: - git: @@ -62,14 +62,13 @@ builders: - shell: |- - # Skip rpm build if .spec.in file is missing - if [ "{type}" = "rpm" ] && [ -z "$(find code-from-gerrit -name '*.spec.in')" ]; then - set +x - echo - echo "No *.spec.in file found, skipping rpm build." - echo - exit 0 - fi + # Make obvious which distro we build for, for humans and for parsing + # by scripts/jenkins-gerrit/comment_generate.py + set +x + echo + echo "Building binary packages for distro: '$DISTRO'" + echo + set -x # Clone osmo-ci.git rm -rf osmo-ci @@ -80,13 +79,6 @@ osmo-ci git -C osmo-ci log --oneline - # Get distro from type - case "{type}" in - deb) distro="debian:11" ;; - rpm) distro="almalinux:8" ;; - *) echo "unknown package format: '{type}'"; exit 1 ;; - esac - # Move code from gerrit to build_srcpkg.py's git cache cache_dir=osmo-ci/scripts/obs/_cache mkdir -p $cache_dir @@ -106,7 +98,7 @@ # Build binary package ./build_binpkg.py \ - --docker "$distro" \ + --docker "$DISTRO" \ "$PROJECT_NAME" wrappers: - ansicolor: diff --git a/jobs/gerrit-verifications.yml b/jobs/gerrit-verifications.yml index b77b0ca..f2c8bec 100644 --- a/jobs/gerrit-verifications.yml +++ b/jobs/gerrit-verifications.yml @@ -92,20 +92,21 @@ # Which jobs to run in the pipeline pipeline_build: true pipeline_lint: true - pipeline_binpkgs: true + pipeline_binpkgs: "almalinux:8 debian:11" pipeline_endianness: true # in alphabetical order repos: - asn1c: - pipeline_binpkgs: false + pipeline_binpkgs: "" pipeline_lint: false - - cellmgr-ng + - cellmgr-ng: + pipeline_binpkgs: "debian:11" - docker-playground: pipeline_build: false - pipeline_binpkgs: false + pipeline_binpkgs: "" pipeline_endianness: false - libasn1c @@ -125,7 +126,7 @@ - libsmpp34 - libtelnet: - pipeline_binpkgs: false + pipeline_binpkgs: "" - libusrp @@ -143,7 +144,7 @@ -e SMPP="$SMPP" \ -e MGCP="$MGCP" \ {docker_img} {timeout_cmd} /build/contrib/jenkins.sh - pipeline_binpkgs: false + pipeline_binpkgs: "" - openggsn: a1_name: GTP @@ -190,6 +191,7 @@ gerrit_project: 'erlang/{repos}' cmd: '{docker_run} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh' pipeline_endianness: false + pipeline_binpkgs: "debian:11" - osmo-e1-hardware: a2_name: JOB_TYPE @@ -206,11 +208,11 @@ -e JOB_TYPE="$JOB_TYPE" \ "$DOCKER_IMG" \ {timeout_cmd} /build/contrib/jenkins.sh - pipeline_binpkgs: false + pipeline_binpkgs: "" - osmo-ci: pipeline_build: false - pipeline_binpkgs: false + pipeline_binpkgs: "" pipeline_endianness: false - osmo-ggsn: @@ -229,12 +231,13 @@ - osmo-gsm-manuals: pipeline_endianness: false + pipeline_binpkgs: "debian:11" - osmo_gsup: repos_url: '{gerrit_url}/erlang/{repos}' gerrit_project: 'erlang/{repos}' cmd: '{docker_run_ccache} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh' - pipeline_binpkgs: false + pipeline_binpkgs: "" - osmo-hlr - osmo-iuh @@ -290,6 +293,7 @@ repos_url: '{gerrit_url}/python/{repos}' gerrit_project: 'python/{repos}' pipeline_endianness: false + pipeline_binpkgs: "debian:11" - osmo-sgsn: a1_name: IU @@ -309,7 +313,7 @@ repos_url: '{gerrit_url}/erlang/{repos}' gerrit_project: 'erlang/{repos}' cmd: '{docker_run} {docker_img_erlang} {timeout_cmd} /build/contrib/jenkins.sh' - pipeline_binpkgs: false + pipeline_binpkgs: "" pipeline_endianness: false - osmo-sip-connector @@ -336,39 +340,41 @@ "$DOCKER_IMG" {timeout_cmd} /build/contrib/jenkins.sh - osmocom-bb: - pipeline_binpkgs: false + pipeline_binpkgs: "" - osmo-tetra: - pipeline_binpkgs: false + pipeline_binpkgs: "" - - osmo-sysmon + - osmo-sysmon: + pipeline_binpkgs: "debian:11" + - osmo-remsim - simtrace2 - osmo-opencm3-projects: cmd: '{docker_run} {docker_img} {timeout_cmd} /build/contrib/jenkins.sh' - pipeline_binpkgs: false + pipeline_binpkgs: "" - osmo-asf4-dfu: - pipeline_binpkgs: false + pipeline_binpkgs: "" - osmo-ccid-firmware: - pipeline_binpkgs: false + pipeline_binpkgs: "" - osmo-e1d - osmo-cbc - osmo-e1-recorder: docker_img: '$USER/debian-bullseye-jenkins' # needs python2 (OS#5950) - pipeline_binpkgs: false + pipeline_binpkgs: "" - gapk: - pipeline_binpkgs: false + pipeline_binpkgs: "" - osmo-uecups - osmo-el2tpd: - pipeline_binpkgs: false + pipeline_binpkgs: "" - osmo-smlc: docker_img: '$USER/debian-bullseye-jenkins' # needs python2 (OS#5950) @@ -413,7 +419,7 @@ {timeout_cmd} /build/contrib/jenkins.sh ;; esac - pipeline_binpkgs: false + pipeline_binpkgs: "" pipeline_endianness: false - osmo-ttcn3-hacks: @@ -431,12 +437,12 @@ su build -c 'make clean'; su build -c 'make compile'; " - pipeline_binpkgs: false + pipeline_binpkgs: "" pipeline_endianness: false - dahdi-tools: pipeline_lint: false - pipeline_binpkgs: false + pipeline_binpkgs: "" jobs: - 'gerrit-{repos}' @@ -459,21 +465,26 @@ name: REPO_URL description: Clone URL, to be passed to jobs started in the pipeline default: '{obj:repos_url}' - - bool: + - string: name: PIPELINE_BINPKGS - description: Enable the binpkgs job (build deb/rpm packages) + description: | + Enable the binpkgs job (build deb/rpm packages) for the given space + separated distributions. Empty list disables the binpkgs job. The + distributions must have binary packages in the osmocom:master + repository. default: '{obj:pipeline_binpkgs}' - bool: name: PIPELINE_ENDIANNESS description: Run struct_endianness.py from libosmocore.git default: '{obj:pipeline_endianness}' dsl: | - def run_job(display_name, job_name, comment_type=null) {{ + def run_job(display_name, job_name, comment_type=null, distro=null) {{ try {{ echo "pipeline_${{display_name}}: running job ${{job_name}}" build job: job_name, parameters: [ string(name: "COMMENT_TYPE", value: comment_type), + string(name: "DISTRO", value: distro), string(name: "GERRIT_BRANCH", value: "${{env.GERRIT_BRANCH}}"), string(name: "GERRIT_CHANGE_NUMBER", value: "${{env.GERRIT_CHANGE_NUMBER}}"), string(name: "GERRIT_HOST", value: "${{env.GERRIT_HOST}}"), @@ -530,24 +541,42 @@ }} }} - stage("deb") {{ + // Generating binpkgs stages is not possible if we want to run + // them in parallel and not have another job that just waits + // until they complete. + + stage("binpkgs-0") {{ when {{ - expression {{ params.PIPELINE_BINPKGS }} + expression {{ params.PIPELINE_BINPKGS.split().size() > 0 }} }} steps {{ script {{ - run_job("deb", "gerrit-binpkgs-deb") + distro = params.PIPELINE_BINPKGS.split()[0] + run_job(distro, "gerrit-binpkgs", null, distro) }} }} }} - stage("rpm") {{ + stage("binpkgs-1") {{ when {{ - expression {{ params.PIPELINE_BINPKGS }} + expression {{ params.PIPELINE_BINPKGS.split().size() > 1 }} }} steps {{ script {{ - run_job("rpm", "gerrit-binpkgs-rpm") + distro = params.PIPELINE_BINPKGS.split()[1] + run_job(distro, "gerrit-binpkgs", null, distro) + }} + }} + }} + + stage("binpkgs-2") {{ + when {{ + expression {{ params.PIPELINE_BINPKGS.split().size() > 2 }} + }} + steps {{ + script {{ + distro = params.PIPELINE_BINPKGS.split()[2] + run_job(distro, "gerrit-binpkgs", null, distro) }} }} }} diff --git a/scripts/jenkins-gerrit/comment_generate.py b/scripts/jenkins-gerrit/comment_generate.py index 6689ee4..e96893c 100755 --- a/scripts/jenkins-gerrit/comment_generate.py +++ b/scripts/jenkins-gerrit/comment_generate.py @@ -9,8 +9,9 @@ jenkins_url = "https://jenkins.osmocom.org" re_start_build = re.compile("Starting building: gerrit-[a-zA-Z-_0-9]* #[0-9]*") -re_result = re.compile("^pipeline_([a-zA-Z-_0-9]*): (SUCCESS|FAILED)$") +re_result = re.compile("^pipeline_([a-zA-Z-_0-9:]*): (SUCCESS|FAILED)$") re_job_type = re.compile("JOB_TYPE=([a-zA-Z-_0-9]*),") +re_distro = re.compile("Building binary packages for distro: '([a-zA-Z0-9:].*)'") def parse_args(): @@ -31,16 +32,29 @@ return parser.parse_args() -def stage_from_job_name(job_name): +def stage_binpkgs_from_url(job_url): + """ Multiple gerrit-binpkgs jobs may be started to build binary packages + for multiple distributions. It is not clear from the job name / URL of + a job for which distro it is building, so read it from the log output. + :returns: a distro like "debian:12" """ + global re_distro + + url = f"{job_url}/consoleText" + with urllib.request.urlopen(url) as response: + content = response.read().decode("utf-8") + match = re_distro.search(content) + assert match, f"couldn't find distro name in log: {url}" + return match.group(1) + + +def stage_from_job_name(job_name, job_url): if job_name == "gerrit-verifications-comment": # The job that runs this script. Don't include it in the summary. return None if job_name == "gerrit-lint": return "lint" - if job_name == "gerrit-binpkgs-deb": - return "deb" - if job_name == "gerrit-binpkgs-rpm": - return "rpm" + if job_name == "gerrit-binpkgs": + return stage_binpkgs_from_url(job_url) if job_name == "gerrit-pipeline-endianness": return "endianness" if job_name.endswith("-build"): @@ -69,7 +83,7 @@ job_name = match.split(" ")[2] job_id = int(match.split(" ")[3].replace("#", "")) job_url = f"{jenkins_url}/jenkins/job/{job_name}/{job_id}" - stage = stage_from_job_name(job_name) + stage = stage_from_job_name(job_name, job_url) if stage: ret[stage] = {"url": job_url, "name": job_name, "id": job_id} -- To view, visit https://gerrit.osmocom.org/c/osmo-ci/+/33988 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-ci Gerrit-Branch: master Gerrit-Change-Id: I59487e3dc2f55057de1b6a322f088fff0d18654c Gerrit-Change-Number: 33988 Gerrit-PatchSet: 2 Gerrit-Owner: osmith &lt;osmith(a)sysmocom.de&gt; Gerrit-Reviewer: Jenkins Builder Gerrit-Reviewer: laforge &lt;laforge(a)osmocom.org&gt; Gerrit-Reviewer: osmith &lt;osmith(a)sysmocom.de&gt; Gerrit-Reviewer: pespin &lt;pespin(a)sysmocom.de&gt; Gerrit-MessageType: merged