osmith has uploaded this change for review. ( https://gerrit.osmocom.org/c/osmo-ci/+/29594 ) Change subject: obs: update_obs_project: add master feed ...................................................................... obs: update_obs_project: add master feed Add a new master feed, where packages are updated as soon as patches get merged to master. Upload a commit_$COMMIT.txt file for each package in this feed and use it to determine if the package needs to be updated or not. In most packages the commit is already part of the version in the dsc file, e.g. "libosmocore_1.7.0.38.c3b90.dsc", but that's harder to parse and is more likely to have a hash collision (just 5 characters). Related: OS#2385 Change-Id: I3b0b4f4876b8c1eeb61f20d903a6f2cac6e99638 --- M scripts/obs/lib/__init__.py M scripts/obs/lib/config.py M scripts/obs/lib/git.py M scripts/obs/lib/osc.py M scripts/obs/lib/srcpkg.py M scripts/obs/update_obs_project.py 6 files changed, 58 insertions(+), 10 deletions(-) git pull ssh://gerrit.osmocom.org:29418/osmo-ci refs/changes/94/29594/1 diff --git a/scripts/obs/lib/__init__.py b/scripts/obs/lib/__init__.py index f94c0a8..8f52dba 100644 --- a/scripts/obs/lib/__init__.py +++ b/scripts/obs/lib/__init__.py @@ -17,7 +17,7 @@ parser.add_argument("-f", "--feed", help="package feed (default: nightly). The feed" " determines the git revision to be built:" - " 'nightly' builds 'origin/master'," + " 'nightly' and 'master' build 'origin/master'," " 'latest' builds the last signed tag," " other feeds build their respective branch.", metavar="FEED", default="nightly", diff --git a/scripts/obs/lib/config.py b/scripts/obs/lib/config.py index c9df74a..79a7849 100644 --- a/scripts/obs/lib/config.py +++ b/scripts/obs/lib/config.py @@ -32,6 +32,7 @@ "2022q1", "2022q2", "latest", + "master", "nightly", ] diff --git a/scripts/obs/lib/git.py b/scripts/obs/lib/git.py index aab0533..8faf5b2 100644 --- a/scripts/obs/lib/git.py +++ b/scripts/obs/lib/git.py @@ -76,6 +76,28 @@ checkout(project, f"origin/{branch}") +def get_head(project): + repo_path = get_repo_path(project) + ret = lib.run_cmd(["git", "rev-parse", "HEAD"], cwd=repo_path) + return ret.output.rstrip() + + +def get_head_remote(project, branch): + if not branch: + branch = get_default_branch(project) + repo_url = get_repo_url(project) + + print(f"{project}: getting head from git remote for {branch}") + ls_remote = lib.run_cmd(["git", "ls-remote", repo_url, branch]) + + ret = ls_remote.output.split("\t")[0] + if not ret: + lib.exit_error_cmd(ls_remote, f"failed to find head commit for" + "{project} in output") + + return ret + + def get_latest_tag(project): pattern_str = get_latest_tag_pattern(project) pattern = re.compile(pattern_str) diff --git a/scripts/obs/lib/osc.py b/scripts/obs/lib/osc.py index 51e2f5f..df5c34c 100644 --- a/scripts/obs/lib/osc.py +++ b/scripts/obs/lib/osc.py @@ -62,7 +62,7 @@ return ret.output.rstrip().split("\n") -def get_package_version(proj, package): +def get_package_version(proj, package, feed): print(f"{package}: getting OBS version") ret = run_osc(["list", proj, os.path.basename(package)]) @@ -70,15 +70,21 @@ if ret.output == '\n': return "0" - # Extract the version from the dsc filename + # Extract the version from the file list for line in ret.output.split('\n'): line = line.rstrip() - if line.endswith(".dsc"): - return line.split("_")[-1][:-4] + if feed == "master" and package != "osmocom-master": + # Use commit_*.txt + if line.startswith("commit_") and line.endswith(".txt"): + return line.split("_")[1].split(".")[0] + else: + # Use *.dsc + if line.endswith(".dsc"): + return line.split("_")[-1][:-4] lib.exit_error_cmd(ret, "failed to find package version on OBS by" - " extracting the version from the .dsc filename") + " extracting the version from the file list") def create_package(proj, package): diff --git a/scripts/obs/lib/srcpkg.py b/scripts/obs/lib/srcpkg.py index 159f4ed..1711f70 100644 --- a/scripts/obs/lib/srcpkg.py +++ b/scripts/obs/lib/srcpkg.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright 2022 sysmocom - s.f.m.c. GmbH <info(a)sysmocom.de> import os +import pathlib import lib.config import lib.debian import lib.rpm_spec @@ -13,7 +14,7 @@ lib.git.checkout(project, f"origin/{branch}") elif feed == "latest": lib.git.checkout_latest_tag(project) - elif feed == "nightly": + elif feed in ["master", "nightly"]: lib.git.checkout_default_branch(project) else: # 2022q1 etc lib.git.checkout(project, f"origin/{feed}") @@ -117,6 +118,18 @@ f.write(f"{version}\n") +def write_commit_txt(project): + """ Write the current git commit to commit_$commit.txt file, so it gets + uploaded to OBS along with the rest of the source package. This allows + figuring out if the source package is still up-to-date or not for the + master feed. """ + output_path = lib.get_output_path(project) + commit = lib.git.get_head(project) + + print(f"{project}: adding commit_{commit}.txt") + pathlib.Path(f"{output_path}/commit_{commit}.txt").touch() + + def build(project, feed, branch, conflict_version, fetch, gerrit_id=0): lib.git.clone(project, fetch) lib.git.clean(project) @@ -155,5 +168,8 @@ lib.rpm_spec.generate(project, version, epoch) lib.rpm_spec.copy_to_output(project) + if feed == "master": + write_commit_txt(project) + lib.remove_cache_extra_files() return version_epoch diff --git a/scripts/obs/update_obs_project.py b/scripts/obs/update_obs_project.py index 168193d..9bd46ff 100755 --- a/scripts/obs/update_obs_project.py +++ b/scripts/obs/update_obs_project.py @@ -66,13 +66,16 @@ fetch, is_meta_pkg, skip_up_to_date): global srcpkgs_skipped - if feed == "latest": + if feed in ["master", "latest"]: """ Check if we can skip this package by comparing the OBS version with the git remote. """ if is_meta_pkg: latest_version = conflict_version if conflict_version else "1.0.0" else: - latest_version = lib.git.get_latest_tag_remote(package) + if feed == "master": + latest_version = lib.git.get_head_remote(package, branch) + else: + latest_version = lib.git.get_latest_tag_remote(package) if latest_version is None: print(f"{package}: skipping (no git tag found)") @@ -82,7 +85,7 @@ if os.path.basename(package) not in pkgs_remote: print(f"{package}: building source package (not in OBS)") else: - obs_version = lib.osc.get_package_version(proj, package) + obs_version = lib.osc.get_package_version(proj, package, feed) if is_up_to_date(obs_version, latest_version): if skip_up_to_date: print(f"{package}: skipping ({obs_version} is up-to-date)") -- To view, visit https://gerrit.osmocom.org/c/osmo-ci/+/29594 To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings Gerrit-Project: osmo-ci Gerrit-Branch: master Gerrit-Change-Id: I3b0b4f4876b8c1eeb61f20d903a6f2cac6e99638 Gerrit-Change-Number: 29594 Gerrit-PatchSet: 1 Gerrit-Owner: osmith <osmith(a)sysmocom.de> Gerrit-MessageType: newchange