Change in osmo-ci[master]: osmo-depcheck: don't use /tmp, better git code

This is merely a historical archive of years 2008-2021, before the migration to mailman3.

A maintained and still updated list archive can be found at https://lists.osmocom.org/hyperkitty/list/gerrit-log@lists.osmocom.org/.

osmith gerrit-no-reply at lists.osmocom.org
Fri Sep 21 13:00:52 UTC 2018


osmith has uploaded this change for review. ( https://gerrit.osmocom.org/11053


Change subject: osmo-depcheck: don't use /tmp, better git code
......................................................................

osmo-depcheck: don't use /tmp, better git code

* replace --gitdir with --workdir and give it a new folder structure:
  * git/$repo: downloaded source code
  * build/$repo: files created during the build process
  * install/: installation prefix
* adjust the jenkins job to use --workdir
* fetch --tags when source exists already
* readable error message for failed git checkout

Change-Id: I06589277b9d54a2af177451cfab2ca1a658b4058
Relates: OS#2642
---
M jobs/osmocom-depcheck.yml
M scripts/osmo-depcheck/buildstack.py
M scripts/osmo-depcheck/dependencies.py
M scripts/osmo-depcheck/osmo-depcheck.py
M scripts/osmo-depcheck/parse.py
5 files changed, 96 insertions(+), 68 deletions(-)



  git pull ssh://gerrit.osmocom.org:29418/osmo-ci refs/changes/53/11053/1

diff --git a/jobs/osmocom-depcheck.yml b/jobs/osmocom-depcheck.yml
index f13d4b7..fd2432e 100644
--- a/jobs/osmocom-depcheck.yml
+++ b/jobs/osmocom-depcheck.yml
@@ -52,13 +52,13 @@
           # Build the arguments
           args="$PROJECTS"
           args="$args -j 5"
-          args="$args -g $PWD/DEPCHECK_GITDIR"
+          args="$args -w $PWD/DEPCHECK_WORKDIR"
           args="$args -u $GIT_URL_PREFIX"
           [ "$BUILD" = "true" ] && args="$args -b"
           [ "$PRINT_OLD_DEPENDS" = "true" ] && args="$args -o"
 
           # Run osmo-depcheck
-          mkdir DEPCHECK_GITDIR
+          mkdir DEPCHECK_WORKDIR
           export PYTHONUNBUFFERED=1
           scripts/osmo-depcheck/osmo-depcheck.py $args
     scm:
diff --git a/scripts/osmo-depcheck/buildstack.py b/scripts/osmo-depcheck/buildstack.py
index 87210ab..0a9a011 100644
--- a/scripts/osmo-depcheck/buildstack.py
+++ b/scripts/osmo-depcheck/buildstack.py
@@ -70,29 +70,15 @@
         print(" * " + program + ":" + version)
 
 
-def temp_install_folder():
-    """ Generate a temporary installation folder
-
-        It will be used as configure prefix, so when running 'make install',
-        the files will get copied in there instead of "/usr/local/". The folder
-        will get removed when the script has finished.
-
-        :returns: the path to the temporary folder """
-    ret = tempfile.mkdtemp(prefix="depcheck_")
-    atexit.register(shutil.rmtree, ret)
-    print("Temporary install folder: " + ret)
-    return ret
-
-
-def set_environment(jobs, tempdir):
+def set_environment(jobs, prefix):
     """ Configure the environment variables before running configure, make etc.
 
         :param jobs: parallel build jobs (for make)
-        :param tempdir: temporary installation dir (see temp_install_folder())
+        :param prefix: installation folder
     """
-    # Add tempdir to PKG_CONFIG_PATH and LD_LIBRARY_PATH
-    extend = {"PKG_CONFIG_PATH": tempdir + "/lib/pkgconfig",
-              "LD_LIBRARY_PATH": tempdir + "/lib"}
+    # Add prefix to PKG_CONFIG_PATH and LD_LIBRARY_PATH
+    extend = {"PKG_CONFIG_PATH": prefix + "/lib/pkgconfig",
+              "LD_LIBRARY_PATH": prefix + "/lib"}
     for env_var, folder in extend.items():
         old = os.environ[env_var] if env_var in os.environ else ""
         os.environ[env_var] = old + ":" + folder
@@ -101,10 +87,10 @@
     os.environ["JOBS"] = str(jobs)
 
 
-def build(gitdir, jobs, stack):
+def build(workdir, jobs, stack):
     """ Build one program with all its dependencies.
 
-        :param gitdir: folder to which the sources will be cloned
+        :param workdir: path to where all data (git, build, install) is stored
         :param jobs: parallel build jobs (for make)
         :param stack: the build stack as returned by generate() above
 
@@ -122,18 +108,23 @@
           anymore in case they decide to compile the code again manually from
           the source folder. """
     # Prepare the install folder and environment
-    tempdir = temp_install_folder()
-    unitdir = tempdir + "/lib/systemd/system/"
-    set_environment(jobs, tempdir)
+    prefix = workdir + "/install"
+    unitdir = prefix + "/lib/systemd/system/"
+    set_environment(jobs, prefix)
 
     # Iterate over stack
     for program, version in stack.items():
         print("Building " + program + ":" + version)
-        os.chdir(gitdir + "/" + program)
+
+        # Create and enter the build folder
+        builddir = workdir + "/build/" + program
+        os.mkdir(builddir)
+        os.chdir(builddir)
 
         # Run the build commands
-        commands = [["autoreconf", "-fi"],
-                    ["./configure", "--prefix", tempdir,
+        gitdir = workdir + "/git/" + program
+        commands = [["autoreconf", "-fi", gitdir],
+                    [gitdir + "/configure", "--prefix", prefix,
                      "--with-systemdsystemunitdir=" + unitdir],
                     ["make", "clean"],
                     ["make"],
diff --git a/scripts/osmo-depcheck/dependencies.py b/scripts/osmo-depcheck/dependencies.py
index 78cf4a0..9b5187d 100644
--- a/scripts/osmo-depcheck/dependencies.py
+++ b/scripts/osmo-depcheck/dependencies.py
@@ -10,37 +10,55 @@
 import parse
 
 
-def git_clone(gitdir, prefix, repository, version):
+def git_clone(workdir, prefix, cache_git_fetch, repository, version):
     """ Clone a missing git repository and checkout a specific version tag.
 
-        :param gitdir: folder to which the sources will be cloned
+        :param workdir: path to where all data (git, build, install) is stored
         :param prefix: git url prefix (e.g. "git://git.osmocom.org/")
+        :param cache_git_fetch: list of repositories that have already been
+                                fetched in this run of osmo-depcheck
         :param repository: Osmocom git repository name (e.g. "libosmo-abis")
         :param version: "master" or a version tag like "0.11.0" """
-    # Clone when needed
-    if not os.path.exists(gitdir + "/" + repository):
-        url = prefix + repository
-        print("Cloning git repo: " + url)
-        try:
-            subprocess.run(["git", "-C", gitdir, "clone", "-q", url],
+    repodir = workdir + "/git/" + repository
+    if repository not in cache_git_fetch:
+        if os.path.exists(repodir):
+            # Fetch tags for existing source
+            print("Fetching tags...")
+            subprocess.run(["git", "-C", repodir, "fetch", "--tags", "-q"],
                            check=True)
-        except subprocess.CalledProcessError:
-            print("NOTE: if '" + repository + "' is part of a git repository"
-                  " with a different name, please add it to the mapping in"
-                  " 'config.py' and try again.")
-            sys.exit(1)
+        else:
+            # Clone the source
+            url = prefix + repository
+            print("Cloning git repo: " + url)
+            try:
+                subprocess.run(["git", "-C", workdir + "/git", "clone", "-q",
+                                url], check=True)
+            except subprocess.CalledProcessError:
+                print("NOTE: if '" + repository + "' is part of a git"
+                      " repository with a different name, please add it to the"
+                      " mapping in 'config.py' and try again.")
+                sys.exit(1)
+
+        # Only fetch the same repository once per session
+        cache_git_fetch.append(repository)
 
     # Checkout the version tag
-    subprocess.run(["git", "-C", gitdir + "/" + repository, "checkout",
-                    version, "-q"], check=True)
+    try:
+        subprocess.run(["git", "-C", repodir, "checkout", version, "-q"],
+                       check=True)
+    except subprocess.CalledProcessError:
+        print("ERROR: git checkout failed! Invalid version specified?")
+        sys.exit(1)
 
 
-def generate(gitdir, prefix, initial, rev):
+def generate(workdir, prefix, cache_git_fetch, initial, rev):
     """ Generate the dependency graph of an Osmocom program by cloning the git
         repository, parsing the "configure.ac" file, and recursing.
 
-        :param gitdir: folder to which the sources will be cloned
+        :param workdir: path to where all data (git, build, install) is stored
         :param prefix: git url prefix (e.g. "git://git.osmocom.org/")
+        :param cache_git_fetch: list of repositories that have already been
+                                fetched in this run of osmo-depcheck
         :param initial: the first program to look at (e.g. "osmo-bts")
         :param rev: the git revision to check out ("master", "0.1.0", ...)
         :returns: a dictionary like the following:
@@ -65,8 +83,8 @@
 
         # Add the programs dependencies to the stack
         print("Looking at " + program + ":" + version)
-        git_clone(gitdir, prefix, program, version)
-        depends = parse.configure_ac(gitdir, program)
+        git_clone(workdir, prefix, cache_git_fetch, program, version)
+        depends = parse.configure_ac(workdir, program)
         stack.update(depends)
 
         # Add the program to the ret
@@ -86,28 +104,28 @@
         print(" * " + program + ":" + version + " depends: " + str(depends))
 
 
-def git_latest_tag(gitdir, repository):
+def git_latest_tag(workdir, repository):
     """ Get the last release string by asking git for the latest tag.
 
-        :param gitdir: folder to which the sources will be cloned
+        :param workdir: path to where all data (git, build, install) is stored
         :param repository: Osmocom git repository name (e.g. "libosmo-abis")
         :returns: the latest git tag (e.g. "1.0.2") """
-    dir = gitdir + "/" + repository
+    dir = workdir + "/git/" + repository
     complete = subprocess.run(["git", "-C", dir, "describe", "--abbrev=0",
                                "master"], check=True, stdout=subprocess.PIPE)
     return complete.stdout.decode().rstrip()
 
 
-def print_old(gitdir, depends):
+def print_old(workdir, depends):
     """ Print dependencies tied to an old release tag
 
-        :param gitdir: folder to which the sources will be cloned
+        :param workdir: path to where all data (git, build, install) is stored
         :param depends: return value from generate() above """
     print("Dependencies on old releases:")
 
     for program, data in depends.items():
         for depend, version in data["depends"].items():
-            latest = git_latest_tag(gitdir, depend)
+            latest = git_latest_tag(workdir, depend)
             if latest == version:
                 continue
             print(" * " + program + ":" + data["version"] + " -> " +
diff --git a/scripts/osmo-depcheck/osmo-depcheck.py b/scripts/osmo-depcheck/osmo-depcheck.py
index 92c0ce6..7f701d6 100755
--- a/scripts/osmo-depcheck/osmo-depcheck.py
+++ b/scripts/osmo-depcheck/osmo-depcheck.py
@@ -4,6 +4,7 @@
 
 import argparse
 import os
+import shutil
 import sys
 
 # Same folder
@@ -17,16 +18,16 @@
     description = ("This script verifies that Osmocom programs really build"
                    " with the dependency versions they claim to support in"
                    " configure.ac. In order to do that, it clones the"
-                   " dependency repositories if they don't exist in gitdir"
+                   " dependency repositories if they don't exist in workdir"
                    " already, and checks out the minimum version tag. This"
                    " happens recursively for their dependencies as well.")
     parser = argparse.ArgumentParser(description=description)
 
     # Git sources folder
-    gitdir_default = os.path.expanduser("~") + "/code"
-    parser.add_argument("-g", "--gitdir", default=gitdir_default,
+    workdir_default = os.path.expanduser("~") + "/osmo-depcheck-work"
+    parser.add_argument("-w", "--workdir", default=workdir_default,
                         help="folder to which the sources will be cloned"
-                             " (default: " + gitdir_default + ")")
+                             " (default: " + workdir_default + ")")
 
     # Build switch
     parser.add_argument("-b", "--build", action="store_true",
@@ -55,17 +56,33 @@
                              " revision is 'master')",
                         metavar="project[:revision]")
 
-    # Gitdir must exist
+    # Workdir must exist
     ret = parser.parse_args()
-    if not os.path.exists(ret.gitdir):
-        print("ERROR: gitdir does not exist: " + ret.gitdir)
+    if not os.path.exists(ret.workdir):
+        print("ERROR: workdir does not exist: " + ret.workdir)
         sys.exit(1)
     return ret
 
 
+def workdir_prepare(workdir):
+    """ Delete old binaries and create the subfolders in workdir
+        :param workdir: path to where all data is stored """
+    # Delete folders with binaries from previous runs
+    for subfolder in ("build", "install"):
+        full = workdir + "/" + subfolder
+        if os.path.exists(full):
+            shutil.rmtree(full)
+
+    # Create all subfolders
+    for subfolder in ("build", "install", "git"):
+        os.makedirs(workdir + "/" + subfolder, exist_ok=True)
+
+
 def main():
-    # Iterate over projects
     args = parse_arguments()
+
+    # Iterate over projects
+    cache_git_fetch = []
     for project_rev in args.projects_revs:
         # Split the git revision from the project name
         project = project_rev
@@ -74,7 +91,9 @@
             project, rev = project_rev.split(":", 1)
 
         # Clone and parse the repositories
-        depends = dependencies.generate(args.gitdir, args.prefix, project, rev)
+        workdir_prepare(args.workdir)
+        depends = dependencies.generate(args.workdir, args.prefix,
+                                        cache_git_fetch, project, rev)
         print("---")
         dependencies.print_dict(depends)
         stack = buildstack.generate(depends)
@@ -84,12 +103,12 @@
         # Old versions
         if args.old:
             print("---")
-            dependencies.print_old(args.gitdir, depends)
+            dependencies.print_old(args.workdir, depends)
 
         # Build
         if args.build:
             print("---")
-            buildstack.build(args.gitdir, args.jobs, stack)
+            buildstack.build(args.workdir, args.jobs, stack)
 
         # Success
         print("---")
diff --git a/scripts/osmo-depcheck/parse.py b/scripts/osmo-depcheck/parse.py
index c6297d6..06f932e 100644
--- a/scripts/osmo-depcheck/parse.py
+++ b/scripts/osmo-depcheck/parse.py
@@ -84,16 +84,16 @@
                   operator + "'")
 
 
-def configure_ac(gitdir, repo):
+def configure_ac(workdir, repo):
     """ Parse the PKG_CHECK_MODULES statements of a configure.ac file.
 
-        :param gitdir: parent folder of all locally cloned git repositories
+        :param workdir: path to where all data (git, build, install) is stored
         :param repo: the repository to look at (e.g. "osmo-bts")
         :returns: a dictionary like the following:
                   {"libosmocore": "0.11.0",
                    "libosmo-abis": "0.5.0"} """
     # Read configure.ac
-    path = gitdir + "/" + repo + "/configure.ac"
+    path = workdir + "/git/" + repo + "/configure.ac"
     with open(path) as handle:
         lines = handle.readlines()
 

-- 
To view, visit https://gerrit.osmocom.org/11053
To unsubscribe, or for help writing mail filters, visit https://gerrit.osmocom.org/settings

Gerrit-Project: osmo-ci
Gerrit-Branch: master
Gerrit-MessageType: newchange
Gerrit-Change-Id: I06589277b9d54a2af177451cfab2ca1a658b4058
Gerrit-Change-Number: 11053
Gerrit-PatchSet: 1
Gerrit-Owner: osmith <osmith at sysmocom.de>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.osmocom.org/pipermail/gerrit-log/attachments/20180921/b6a1b5f9/attachment.htm>


More information about the gerrit-log mailing list