ci: Use NFS for the contrib tarballs and cached Qt packages.

Change-Id: I99a20bcf357032a6fcf689265c93670dcf65b8b7

Jenkinsfile

@@ -68,7 +68,7 @@ pipeline {
     }
 
     environment {
-        TARBALLS = '/opt/ring-contrib' // set the cache directory
+        TARBALLS = '/var/cache/jami' // set the cache directory
     }
 
     stages {
@@ -79,6 +79,13 @@ pipeline {
                         error "The ${TARBALLS} directory does not exist. \
 See https://wiki.savoirfairelinux.com/wiki/Jenkins.jami.net#Configuration"
                     }
+
+                    mountType = sh(script: "findmnt ${TARBALLS} -o fstype -n",
+                                   returnStdout: true)
+                    if (!(mountType =~ /^nfs/)) {
+                        error "The ${TARBALLS} directory is not mounted on NFS storage. \
+See https://wiki.savoirfairelinux.com/wiki/Jenkins.jami.net#Configuration_client_NFS"
+                    }
                 }
             }
         }

Makefile

@@ -16,6 +16,9 @@
 #
 .DEFAULT_GOAL := package-all
 
+# Default caching directory
+export TARBALLS ?= /var/cache/jami
+
 ##############################
 ## Version number variables ##
 ##############################

client-android @ ba4b6348

-Subproject commit 195212d0b3a9351a7ca9808cf0ec987c22d8f96f
+Subproject commit ba4b63485d94c1bcca760cc8c30351f6779affb1

client-gnome @ 920c4b9b

-Subproject commit 0ace8ef7222ea96ac054072ccfc910175c41912e
+Subproject commit 920c4b9bf09d2c06978d9bba2fd366c9c4111db9

client-ios @ a68da39c

-Subproject commit 31ab0ba7c54384730bcda89cfac0698f846a3649
+Subproject commit a68da39c3efce4d5068d8e3e756f55ef3ea71eac

client-macosx @ d7aa59f2

-Subproject commit 5f30cfbb1f853095e8f4d4e6af68d2c2a59259f9
+Subproject commit d7aa59f247a21930a760f81e6c8f9d22defc3a05

client-qt @ c2be09f5

-Subproject commit a2858a883b73707ab2a226975201c8558d6a991d
+Subproject commit c2be09f55aee1f0d4f65ed7751fe3412f0ea0375

client-uwp @ 7c248e9d

-Subproject commit 4b639e9a903931bab09292b4d93a4de6d531ae4c
+Subproject commit 7c248e9d48ee899ad58156e9cc92e4cc4a02f6bc

daemon @ 24f54f16

-Subproject commit 73926a7662910501a3f420e3e595ba862ac5f321
+Subproject commit 24f54f161ca9a2ee195783ce2361c4161a84b41b

guix/minimal-manifest.scm

@@ -10,6 +10,7 @@
 
 (specifications->manifest
  (list
+  "bash"
   "coreutils"
   "gcc-toolchain"
   "git-minimal"
@@ -21,5 +22,6 @@
   "python"
   "sed"
   "tar"
+  "util-linux"
   "wget"
   "xz"))

lrc @ 110c8d8b

-Subproject commit 1a8509ec1408bf28ce0ec0c71fb5f4a5845faab7
+Subproject commit 110c8d8bbd98097b8a2ef20ab76c109e301d2696

plugins @ 17c0bc64

-Subproject commit 1bbc7cc2a5de65e03676a2f372be7c603df86887
+Subproject commit 17c0bc64a374d601012943a0a79ace65734dd922

scripts/build-package-debian.sh

@@ -33,69 +33,78 @@ if grep -q "raspbian_10_armhf" <<< "${DISTRIBUTION}"; then
     DPKG_BUILD_OPTIONS="${DPKG_BUILD_OPTIONS} -a armhf"
 fi
 
-cache_packaging=/opt/cache-packaging/${DISTRIBUTION}
+cache_packaging=$TARBALLS/$DISTRIBUTION
+mkdir -p "$cache_packaging"
+
 deb_arch=$(dpkg --print-architecture)
 qt_deb_name=libqt-jami_${DEBIAN_QT_VERSION}_${deb_arch}.deb
 qt_deb_path=${cache_packaging}/${qt_deb_name}
 
 if [ ! -f "${qt_deb_path}" ] || [ "${FORCE_REBUILD_QT}" = "true" ]; then
-    # we need to build Qt
-
-    cache_dir=/opt/ring-contrib
-    temp_dir=$(mktemp -d)
+    (
+        flock 9                 # block until the lock file is gone
+        test -f "${qt_deb_path}" && exit 0 # check again
 
+        # Build Qt.
         mkdir /opt/libqt-jami-build
         cd /opt/libqt-jami-build
 
         qt_version=${QT_MAJOR}.${QT_MINOR}.${QT_PATCH}
         tarball_name=qt-everywhere-src-${qt_version}.tar.xz
-    cached_tarball="${cache_dir}/${tarball_name}"
-    qt_base_url=https://download.qt.io/archive/qt/${QT_MAJOR}.${QT_MINOR}/${qt_version}/single
+        cached_tarball=$TARBALLS/$tarball_name
+        qt_base_url=https://download.qt.io/archive/qt/\
+${QT_MAJOR}.${QT_MINOR}/${qt_version}/single
 
-    if [ ! -d "${cache_dir}" ] || [ ! -w "${cache_dir}" ]; then
-        echo "error: $cache_dir does not exist or is not writable"
+        if [ ! -d "${TARBALLS}" ] || [ ! -w "${TARBALLS}" ]; then
+            echo "error: $TARBALLS does not exist or is not writable"
             exit 1
         fi
 
         if [ ! -f "${cached_tarball}" ]; then
             (
+                flock 8         # block until the lock file is gone
+                test -f "${cached_tarball}" && exit 0 # check again
+
+                temp_dir=$(mktemp -d)
                 cd "${temp_dir}"
                 wget "${qt_base_url}/${tarball_name}"
                 echo -n "${QT_TARBALL_CHECKSUM}  ${tarball_name}" | sha256sum -c - || \
                     (echo "Qt tarball checksum mismatch; quitting" && exit 1)
-            flock "${cached_tarball}.lock" mv "${tarball_name}" "${cached_tarball}"
-        )
+                mv "${tarball_name}" "${cached_tarball}"
                 rm -rf "${temp_dir}"
+            ) 8>"${cached_tarball}.lock"
+
         fi
 
-    cp "${cached_tarball}" libqt-jami_${qt_version}.orig.tar.xz
-    tar xvf libqt-jami_${qt_version}.orig.tar.xz
-    mv qt-everywhere-src-${qt_version} libqt-jami-${qt_version}
-    cd libqt-jami-${qt_version}
+        cp "${cached_tarball}" "libqt-jami_${qt_version}.orig.tar.xz"
+        tar xvf "libqt-jami_${qt_version}.orig.tar.xz"
+        mv "qt-everywhere-src-${qt_version}" "libqt-jami-${qt_version}"
+        cd "libqt-jami-${qt_version}"
 
-    # import the debian folder
+        # Import the debian folder.
         cp --verbose -r /opt/ring-project-ro/packaging/rules/debian-qt debian
 
-    # create changelog file
+        # Create the changelog file.
         DEBEMAIL="The Jami project <jami@gnu.org>" dch --create \
                 --package libqt-jami \
-            --newversion ${DEBIAN_QT_VERSION} "New libqt-jami release"
+                --newversion "${DEBIAN_QT_VERSION}" "New libqt-jami release"
         DEBEMAIL="The Jami project <jami@gnu.org>" dch --release \
                 --distribution "unstable" debian/changelog
 
-    # build and package qt
+        # Build and package Qt.
         dpkg-buildpackage -uc -us ${DPKG_BUILD_OPTIONS}
 
-    # copy the built deb to cache
-    mkdir -p ${cache_packaging}/
-    cp ../${qt_deb_name} ${qt_deb_path}
+        # Cache the built .deb.
+        mv "../${qt_deb_name}" "${qt_deb_path}"
+
+    ) 9>"${qt_deb_path}.lock"
 fi
 
 # install libqt-jami from cache
-apt-get install -y ${qt_deb_path}
+apt-get install -y "${qt_deb_path}"
 
 # copy libqt-jami to output
-cp ${qt_deb_path} /opt/output/
+cp "${qt_deb_path}" /opt/output/
 
 # Set up work directory.
 mkdir -p /jami/work && cd /jami/work
@@ -103,7 +112,7 @@ mkdir -p /jami/work && cd /jami/work
 # Create a changelog file, required by dpkg-buildpackage.
 mkdir debian
 DEBEMAIL="The Jami project <jami@gnu.org>" dch --create --package jami \
-        --newversion ${DEBIAN_VERSION} "Automatic nightly release"
+        --newversion "${DEBIAN_VERSION}" "Automatic nightly release"
 DEBEMAIL="The Jami project <jami@gnu.org>" dch --release \
         --distribution "unstable" debian/changelog
 
@@ -120,4 +129,4 @@ dpkg-buildpackage -b -uc -us ${DPKG_BUILD_OPTIONS}
 
 # Move the artifacts to the output.
 mv ../*deb /opt/output        # .deb and .ddeb packages
-chown -R ${CURRENT_UID}:${CURRENT_UID} /opt/output
+chown -R "${CURRENT_UID}:${CURRENT_UID}" /opt/output

scripts/build-package-rpm.sh

@@ -21,14 +21,13 @@
 # This script is used in the packaging containers to build packages on
 # rpm-based distros.
 #
-
 set -e
 
 # Import the spec file.
 mkdir -p /opt/ring-project
 cd /opt/ring-project
 cp /opt/ring-project-ro/packaging/rules/rpm/* .
-rm jami-libqt.spec
+rm -f jami-libqt.spec
 
 # Prepare the build tree.
 rpmdev-setuptree
@@ -53,30 +52,42 @@ $QT_MAJOR_MINOR_PATCH/single/qt-everywhere-src-$QT_MAJOR_MINOR_PATCH.tar.xz
 
 QT_TARBALL_SHA256="3a530d1b243b5dec00bc54937455471aaa3e56849d2593edb8ded07228202240"
 QT_TARBALL_FILE_NAME=$(basename "$QT_TARBALL_URL")
-CACHED_QT_TARBALL=/opt/ring-contrib/$QT_TARBALL_FILE_NAME
+CACHED_QT_TARBALL=$TARBALLS/$QT_TARBALL_FILE_NAME
 
 if [[ "${DISTRIBUTION:0:4}" == "rhel" \
    || "${DISTRIBUTION:0:13}" == "opensuse-leap" ]]; then
 
-    RPM_PATH=/opt/cache-packaging/${DISTRIBUTION}/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.x86_64.rpm
+    mkdir -p "$TARBALLS/$DISTRIBUTION"
+    RPM_PATH=$TARBALLS/$DISTRIBUTION/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.x86_64.rpm
     if [[ "${DISTRIBUTION:0:4}" == "rhel" ]]; then
-        RPM_PATH=/opt/cache-packaging/${DISTRIBUTION}/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.el8.x86_64.rpm
+        RPM_PATH=$TARBALLS/${DISTRIBUTION}/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.el8.x86_64.rpm
     fi
 
     if [ ! -f "${RPM_PATH}" ]; then
+        # The following block will only run on one build machine at a
+        # time, thanks to flock.
+        (
+            flock 9             # block until the lock is available
+            test -f "$RPM_PATH" && exit 0 # check again
+
             mkdir /opt/qt-jami-build
             cd /opt/qt-jami-build
             cp /opt/ring-project-ro/packaging/rules/rpm/jami-libqt.spec .
 
             # Fetch and cache the tarball, if not already available.
             if [ ! -f "$CACHED_QT_TARBALL" ]; then
+                (
+                    flock 8     # block until the lock file is gone
+                    test -f "$CACHED_QT_TARBALL" && exit 0 # check again
+
                     wget "$QT_TARBALL_URL"
                     if ! echo -n ${QT_TARBALL_SHA256} "$QT_TARBALL_FILE_NAME" | sha256sum -c -
                     then
                         echo "qt tarball checksum mismatch; quitting"
                         exit 1
                     fi
-            flock "$CACHED_QT_TARBALL" mv "$QT_TARBALL_FILE_NAME" "$CACHED_QT_TARBALL"
+                    mv "$QT_TARBALL_FILE_NAME" "$CACHED_QT_TARBALL"
+                ) 8>"${CACHED_QT_TARBALL}.lock"
             fi
 
             cp "$CACHED_QT_TARBALL" "/root/rpmbuild/SOURCES/jami-qtlib_$QT_MAJOR_MINOR_PATCH.tar.xz"
@@ -85,13 +96,15 @@ if [[ "${DISTRIBUTION:0:4}" == "rhel" \
                             --userstring="Jenkins <jami@lists.savoirfairelinux.net>" jami-libqt.spec
 
             rpmbuild -ba jami-libqt.spec
-        mkdir -p /opt/cache-packaging/${DISTRIBUTION}/
+            mkdir -p "$TARBALLS/${DISTRIBUTION}"
 
+            # Cache the built Qt RPM package.
             if [[ "${DISTRIBUTION:0:4}" == "rhel" ]]; then
-            cp "/root/rpmbuild/RPMS/x86_64/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.el8.x86_64.rpm" "${RPM_PATH}"
+                mv "/root/rpmbuild/RPMS/x86_64/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.el8.x86_64.rpm" "${RPM_PATH}"
             else
-            cp /root/rpmbuild/RPMS/x86_64/jami-libqt-*.rpm "${RPM_PATH}"
+                mv /root/rpmbuild/RPMS/x86_64/jami-libqt-*.rpm "${RPM_PATH}"
             fi
+        ) 9>"${RPM_PATH}.lock"
     fi
     rpm --install "${RPM_PATH}"
     cp "${RPM_PATH}" /opt/output
@@ -99,9 +112,9 @@ if [[ "${DISTRIBUTION:0:4}" == "rhel" \
 fi
 
 # Set the version and associated comment.
-sed -i "s/RELEASE_VERSION/${RELEASE_VERSION}/g" *.spec
+sed -i "s/RELEASE_VERSION/${RELEASE_VERSION}/g" ./*.spec
 rpmdev-bumpspec --comment="Automatic nightly release" \
-                --userstring="Jenkins <jami@lists.savoirfairelinux.net>" *.spec
+                --userstring="Jenkins <jami@lists.savoirfairelinux.net>" ./*.spec
 
 # Build the daemon and install it.
 rpmbuild -ba jami-daemon.spec
@@ -117,7 +130,7 @@ rpmbuild -ba jami-gnome.spec jami-qt.spec
 # Move the built packages to the output directory.
 mv /root/rpmbuild/RPMS/*/* /opt/output
 touch /opt/output/.packages-built
-chown -R ${CURRENT_UID}:${CURRENT_UID} /opt/output
+chown -R "$CURRENT_UID:$CURRENT_UID" /opt/output
 
 # TODO: One click install: create a package that combines the already
 # built package into one.

scripts/make-packaging-target.py

@@ -53,11 +53,11 @@ PACKAGE_%(distribution)s_DOCKER_RUN_COMMAND = docker run \\
     -e CURRENT_UID=$(CURRENT_UID) \\
     -e CURRENT_GID=$(CURRENT_GID) \\
     -e DISTRIBUTION=%(distribution)s \\
+    -e TARBALLS=$(TARBALLS) \\
     -v $(CURDIR)/$(RELEASE_TARBALL_FILENAME):/src/$(RELEASE_TARBALL_FILENAME) \\
     -v $(CURDIR):/opt/ring-project-ro:ro \\
     -v $(CURDIR)/packages/%(distribution)s:/opt/output \\
-    -v /opt/cache-packaging:/opt/cache-packaging \\
-    -v /opt/ring-contrib:/opt/ring-contrib \\
+    -v $(TARBALLS):$(TARBALLS) \\
     -t $(and $(IS_SHELL_INTERACTIVE),-i) %(options)s \\
     $(DOCKER_RUN_EXTRA_ARGS) \\
     $(PACKAGE_%(distribution)s_DOCKER_IMAGE_NAME)
@@ -97,7 +97,6 @@ DPKG_BASED_SYSTEMS_DOCKER_RUN_OPTIONS = (
     '-e QT_PATCH=$(QT_PATCH) '
     '-e QT_TARBALL_CHECKSUM=$(QT_TARBALL_CHECKSUM) '
     '-e FORCE_REBUILD_QT=$(FORCE_REBUILD_QT) '
-    '-v /opt/ring-contrib:/opt/ring-contrib '
     '--privileged '
     '--security-opt apparmor=docker-default ')