From 566bd6e8c19c1df64e8ecbc6ef08ac390f964c83 Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <maxim.cournoyer@savoirfairelinux.com>
Date: Fri, 21 May 2021 12:16:09 -0400
Subject: [PATCH] ci: Spread builds across multiple nodes.
When a .tarball-version file containing the version string is present,
it is taken for granted that the source tarball is present. This
allows to lift the requirement on a Git checkout to use the source
tarball. This makes it easy and cheap to sync this tarball and
accompanying .tarball-version file across different Jenkins agents to
distribute the packaging jobs, bring the total build time from about 1
h 40 to 40 minutes.
* Makefile (TARBALL_VERSION): New variable. When defined, do not do
not invoke Git to derive the version string, simply use it.
[RELEASE_TARBALL_FILENAME]: Do not compute prerequisites when
TARBALL_VERSION is defined; assume the tarball is present.
* .gitignore: Ignore .tarball-version.
* Jenkinsfile: (Generate release tarball): Generate a .tarball-version
file, and stash it along the release tarball. Explicit this should
run on a 'guix' agent.
(Build packages): Forward to any 'linux-builder' agent, clean the
workspace, unstash and extract the tarball and build.
GitLab: https://git.jami.net/savoirfairelinux/jami-packaging/-/issues/98
Change-Id: I13088b75c9ccd5166662a84b7c5f8d4e8b7b65da
---
.gitignore | 1 +
Jenkinsfile | 16 +++++++++++++---
Makefile | 19 +++++++++++++++++++
3 files changed, 33 insertions(+), 3 deletions(-)
diff --git a/.gitignore b/.gitignore
index ee1732cf..985d96a2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,6 +11,7 @@ repositories
tarballs.manifest
manual-download
.docker-image-*
+.tarball-version
# QtCreator
/build-*
diff --git a/Jenkinsfile b/Jenkinsfile
index f810811b..08c453d6 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -52,8 +52,10 @@ See https://wiki.savoirfairelinux.com/wiki/Jenkins.jami.net#Configuration"
// environment variables used by Guix.
sh '''#!/usr/bin/env bash
test -f $HOME/.bashrc && . $HOME/.bashrc
- make portable-release-tarball
+ make portable-release-tarball .tarball-version
'''
+ stash(includes: '*.tar.gz, .tarball-version',
+ name: 'release-tarball')
}
}
@@ -81,8 +83,16 @@ See https://wiki.savoirfairelinux.com/wiki/Jenkins.jami.net#Configuration"
// Note: The stage calls are wrapped in closures, to
// delay their execution.
stages["${target}"] = {
- stage("stage: ${target}") {
- sh "make ${target}"
+ stage("${target}") {
+ // Offload builds to different agents.
+ node('linux-builder') {
+ cleanWs()
+ unstash 'release-tarball'
+ sh """
+ tar xf *.tar.gz --strip-components=1
+ make ${target}
+ """
+ }
}
}
}
diff --git a/Makefile b/Makefile
index 4773d241..d44c5940 100644
--- a/Makefile
+++ b/Makefile
@@ -19,6 +19,9 @@
##############################
## Version number variables ##
##############################
+TARBALL_VERSION := $(shell cat $(CURDIR)/.tarball-version 2> /dev/null)
+
+ifeq ($(TARBALL_VERSION),)
# YYYY-MM-DD
LAST_COMMIT_DATE:=$(shell git log -1 --format=%cd --date=short)
@@ -32,6 +35,10 @@ LAST_COMMIT_DATE_SHORT:=$(shell echo $(LAST_COMMIT_DATE) | sed -s 's/-//g')
COMMIT_ID:=$(shell git rev-parse --short HEAD)
RELEASE_VERSION:=$(LAST_COMMIT_DATE_SHORT).$(NUMBER_OF_COMMITS).$(COMMIT_ID)
+else
+$(warning Using version from the .tarball-version file: $(TARBALL_VERSION))
+RELEASE_VERSION:=$(TARBALL_VERSION)
+endif
RELEASE_TARBALL_FILENAME:=jami_$(RELEASE_VERSION).tar.gz
# Debian versions
@@ -59,6 +66,12 @@ CURRENT_GID:=$(shell id -g)
#############################
.PHONY: release-tarball purge-release-tarballs portable-release-tarball
+# This file can be used when not wanting to invoke the tarball
+# producing machinery (which depends on the Git checkout), nor its
+# prerequisites. It is used to set the TARBALL_VERSION Make variable.
+.tarball-version:
+ echo $(RELEASE_VERSION) > $@
+
purge-release-tarballs:
rm -f jami_*.tar.* tarballs.manifest
@@ -96,6 +109,7 @@ tarballs.manifest: daemon/contrib/native/Makefile
$(MAKE) fetch -j && \
$(MAKE) --no-print-directory --silent list-tarballs > "$(CURDIR)/$@"
+ifeq ($(TARBALL_VERSION),)
# Generate the release tarball. To regenerate a fresh tarball
# manually clear the tarballs.manifest file.
$(RELEASE_TARBALL_FILENAME): tarballs.manifest
@@ -117,6 +131,11 @@ $(RELEASE_TARBALL_FILENAME): tarballs.manifest
gzip $(TMPDIR)/ring-project.tar
mv $(TMPDIR)/ring-project.tar.gz "$@"
rm -rf $(TMPDIR)
+else
+# If TARBALL_VERSION is defined, assume it's already been generated,
+# without doing any checks, which would require Git.
+$(RELEASE_TARBALL_FILENAME):
+endif
#######################
## Packaging targets ##
--
GitLab