osmith has uploaded this change for review. ( https://gerrit.osmocom.org/c/osmo-ci/+/35223?usp=email )

Change subject: OBS: add jenkins job to check for new distros ......................................................................

OBS: add jenkins job to check for new distros

Add a new jenkins job to notify us about new versions of Debian, Raspbian and Ubuntu becoming available in the openSUSE OBS. This allows us to consistently follow along and make the repositories available in the Osmocom OBS. I've decided to check the openSUSE OBS instead of checking somewhere upstream, because it takes time between a new release of a distribution and the version becoming available in openSUSE OBS, and we have another job that syncs the configuration with that OBS instance.

Related: OS#6163 Change-Id: I0abc49a95197da55f7d83ed4fd1c4ebb6bd14b1e --- M jobs/README.adoc A jobs/osmocom-obs-check-new-distros.yml A scripts/obs/check_new_distros.py M scripts/obs/lib/config.py 4 files changed, 181 insertions(+), 0 deletions(-)

git pull ssh://gerrit.osmocom.org:29418/osmo-ci refs/changes/23/35223/1

diff --git a/jobs/README.adoc b/jobs/README.adoc index 86fc485..c37b3bb 100644 --- a/jobs/README.adoc +++ b/jobs/README.adoc @@ -123,6 +123,7 @@

18:00 - 21:00 OBS related 18:XX osmocom-obs-sync (sync Debian:12 etc. with openSUSE OBS) + 19:XX osmocom-obs-check-new-distros 19:XX osmocom-obs-wireshark 20:XX osmocom-obs (new binary packages start building on OBS!)

diff --git a/jobs/osmocom-obs-check-new-distros.yml b/jobs/osmocom-obs-check-new-distros.yml new file mode 100644 index 0000000..760479a --- /dev/null +++ b/jobs/osmocom-obs-check-new-distros.yml @@ -0,0 +1,39 @@ +--- +- job: + name: 'Osmocom_OBS_check_new_distros' + project-type: freestyle + concurrent: false + defaults: global + description: | + Check whether new distributions (Debian, Raspyberry Pi OS, Ubuntu) are + available in openSUSE's OBS that we need to add to the Osmocom OBS. + properties: + - build-discarder: + days-to-keep: 30 + num-to-keep: 30 + node: obs + parameters: + - string: + name: CI_BRANCH + description: osmo-ci.git branch + default: 'master' + scm: + - git: + branches: + - '$CI_BRANCH' + url: https://gerrit.osmocom.org/osmo-ci + git-config-name: 'Jenkins Builder' + git-config-email: 'jenkins@osmocom.org' + builders: + - shell: | + export PYTHONUNBUFFERED=1 + ./scripts/obs/check_new_distros.py --docker + triggers: + - timed: "H 19 * * *" + wrappers: + - ansicolor: + colormap: xterm + publishers: + - email: + notify-every-unstable-build: true + recipients: 'jenkins-notifications@lists.osmocom.org' diff --git a/scripts/obs/check_new_distros.py b/scripts/obs/check_new_distros.py new file mode 100755 index 0000000..962d547 --- /dev/null +++ b/scripts/obs/check_new_distros.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright 2023 sysmocom - s.f.m.c. GmbH info@sysmocom.de +import argparse +import lib.docker +import lib.osc +import sys + +projects_opensuse = None +projects_osmocom = None + + +def parse_args(): + parser = argparse.ArgumentParser(description="Check for new distribution" + " projects on the openSUSE OBS, that we want to configure in the" + " Osmocom OBS as soon as they are available") + parser.add_argument("-d", "--docker", + help="run in docker to avoid installing required pkgs", + action="store_true") + parser.add_argument("-v", "--verbose", action="store_true", + help="always print shell commands and their output," + " instead of only printing them on error") + + advanced = parser.add_argument_group("advanced options") + advanced.add_argument("-A", "--apiurl", help="source OBS API URL" + " (default: https://api.opensuse.org)", + default="https://api.opensuse.org") + advanced.add_argument("-p", "--prefix", default="openSUSE.org-mirror", + help="destination OBS prefix" + " (default: openSUSE.org-mirror)") + advanced.add_argument("-t", "--to-apiurl", help="destination OBS API URL" + " (default: https://obs.osmocom.org)", + default="https://obs.osmocom.org") + + args = parser.parse_args() + lib.set_args(args) + + lib.osc.check_oscrc() + + if args.docker: + lib.docker.run_in_docker_and_exit("check_new_distros.py", add_oscrc=True) + + +def find_highest_distro_project(distro): + highest = None + for project in projects_opensuse: + if not project.startswith(f"{distro}:"): + continue + + num = project[len(distro) + 1:] + if not lib.config.check_new_distros_version_regex.match(num): + if lib.args.verbose: + print(f"ignoring {distro}:{num} (doesn't match version regex)") + continue + + if not highest or float(num) > float(highest): + highest = num + + if highest: + return f"{distro}:{highest}" + + return None + + +def check_distro(distro): + highest = find_highest_distro_project(distro) + if not highest: + print(f"ERROR: {distro}: not found in {lib.args.apiurl}") + return False + + # check if it is in the osmo obs + exp = f"{lib.args.prefix}:{highest}" + if exp in projects_osmocom: + print(f"{exp}: OK") + return True + + print() + print(f"ERROR: {exp} not found") + print() + print("Follow this guide to add it to the Osmocom OBS:") + print("https://osmocom.org/projects/cellular-infrastructure/wiki/Add_a_new_distribu...") + print() + + return False + + +def main(): + global projects_opensuse + global projects_osmocom + + parse_args() + ret = 0 + + # Get list of projects from Osmocom OBS + lib.osc.set_apiurl(lib.args.to_apiurl) + projects_osmocom = lib.osc.get_projects() + + # Get list of projects from openSUSE OBS + lib.osc.set_apiurl(lib.args.apiurl) + projects_opensuse = lib.osc.get_projects() + + # Check for missing distros in Osmocom OBS + for distro in lib.config.check_new_distros: + if not check_distro(distro): + ret = 1 + + sys.exit(ret) + +if __name__ == "__main__": + main() diff --git a/scripts/obs/lib/config.py b/scripts/obs/lib/config.py index 4779d77..ce2917e 100644 --- a/scripts/obs/lib/config.py +++ b/scripts/obs/lib/config.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright 2022 sysmocom - s.f.m.c. GmbH info@sysmocom.de import os +import re

# Lists are ordered alphabetically.

@@ -136,3 +137,14 @@ sync_set_maintainers = [ "osmocom-jenkins", ] + +# Distributions for which we want to make sure we add the latest release as +# soon as it is available in openSUSE's OBS +# https://osmocom.org/projects/cellular-infrastructure/wiki/Linux_Distribution... +check_new_distros = [ + "Debian", + "Raspbian", + "Ubuntu", +] + +check_new_distros_version_regex = re.compile(r'[0-9.]+$')