From bfb8823877137248f7fdaec2b0f4c5c04f09716b Mon Sep 17 00:00:00 2001
From: Maxim Cournoyer <maxim.cournoyer@gmail.com>
Date: Thu, 5 Aug 2021 16:59:29 -0400
Subject: [PATCH] ci: Use NFS for the contrib tarballs and cached Qt packages.

Change-Id: I99a20bcf357032a6fcf689265c93670dcf65b8b7
---
 Jenkinsfile                      |   9 ++-
 Makefile                         |   3 +
 client-android                   |   2 +-
 client-gnome                     |   2 +-
 client-ios                       |   2 +-
 client-macosx                    |   2 +-
 client-qt                        |   2 +-
 client-uwp                       |   2 +-
 daemon                           |   2 +-
 guix/minimal-manifest.scm        |   2 +
 lrc                              |   2 +-
 plugins                          |   2 +-
 scripts/build-package-debian.sh  | 119 +++++++++++++++++--------------
 scripts/build-package-rpm.sh     |  83 ++++++++++++---------
 scripts/make-packaging-target.py |   5 +-
 15 files changed, 136 insertions(+), 103 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 7e28db69..715925fe 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -68,7 +68,7 @@ pipeline {
     }
 
     environment {
-        TARBALLS = '/opt/ring-contrib' // set the cache directory
+        TARBALLS = '/var/cache/jami' // set the cache directory
     }
 
     stages {
@@ -79,6 +79,13 @@ pipeline {
                         error "The ${TARBALLS} directory does not exist. \
 See https://wiki.savoirfairelinux.com/wiki/Jenkins.jami.net#Configuration"
                     }
+
+                    mountType = sh(script: "findmnt ${TARBALLS} -o fstype -n",
+                                   returnStdout: true)
+                    if (!(mountType =~ /^nfs/)) {
+                        error "The ${TARBALLS} directory is not mounted on NFS storage. \
+See https://wiki.savoirfairelinux.com/wiki/Jenkins.jami.net#Configuration_client_NFS"
+                    }
                 }
             }
         }
diff --git a/Makefile b/Makefile
index 20b511e0..71a907b7 100644
--- a/Makefile
+++ b/Makefile
@@ -16,6 +16,9 @@
 #
 .DEFAULT_GOAL := package-all
 
+# Default caching directory
+export TARBALLS ?= /var/cache/jami
+
 ##############################
 ## Version number variables ##
 ##############################
diff --git a/client-android b/client-android
index 195212d0..ba4b6348 160000
--- a/client-android
+++ b/client-android
@@ -1 +1 @@
-Subproject commit 195212d0b3a9351a7ca9808cf0ec987c22d8f96f
+Subproject commit ba4b63485d94c1bcca760cc8c30351f6779affb1
diff --git a/client-gnome b/client-gnome
index 0ace8ef7..920c4b9b 160000
--- a/client-gnome
+++ b/client-gnome
@@ -1 +1 @@
-Subproject commit 0ace8ef7222ea96ac054072ccfc910175c41912e
+Subproject commit 920c4b9bf09d2c06978d9bba2fd366c9c4111db9
diff --git a/client-ios b/client-ios
index 31ab0ba7..a68da39c 160000
--- a/client-ios
+++ b/client-ios
@@ -1 +1 @@
-Subproject commit 31ab0ba7c54384730bcda89cfac0698f846a3649
+Subproject commit a68da39c3efce4d5068d8e3e756f55ef3ea71eac
diff --git a/client-macosx b/client-macosx
index 5f30cfbb..d7aa59f2 160000
--- a/client-macosx
+++ b/client-macosx
@@ -1 +1 @@
-Subproject commit 5f30cfbb1f853095e8f4d4e6af68d2c2a59259f9
+Subproject commit d7aa59f247a21930a760f81e6c8f9d22defc3a05
diff --git a/client-qt b/client-qt
index a2858a88..c2be09f5 160000
--- a/client-qt
+++ b/client-qt
@@ -1 +1 @@
-Subproject commit a2858a883b73707ab2a226975201c8558d6a991d
+Subproject commit c2be09f55aee1f0d4f65ed7751fe3412f0ea0375
diff --git a/client-uwp b/client-uwp
index 4b639e9a..7c248e9d 160000
--- a/client-uwp
+++ b/client-uwp
@@ -1 +1 @@
-Subproject commit 4b639e9a903931bab09292b4d93a4de6d531ae4c
+Subproject commit 7c248e9d48ee899ad58156e9cc92e4cc4a02f6bc
diff --git a/daemon b/daemon
index 73926a76..24f54f16 160000
--- a/daemon
+++ b/daemon
@@ -1 +1 @@
-Subproject commit 73926a7662910501a3f420e3e595ba862ac5f321
+Subproject commit 24f54f161ca9a2ee195783ce2361c4161a84b41b
diff --git a/guix/minimal-manifest.scm b/guix/minimal-manifest.scm
index f9c1f14a..c970c696 100644
--- a/guix/minimal-manifest.scm
+++ b/guix/minimal-manifest.scm
@@ -10,6 +10,7 @@
 
 (specifications->manifest
  (list
+  "bash"
   "coreutils"
   "gcc-toolchain"
   "git-minimal"
@@ -21,5 +22,6 @@
   "python"
   "sed"
   "tar"
+  "util-linux"
   "wget"
   "xz"))
diff --git a/lrc b/lrc
index 1a8509ec..110c8d8b 160000
--- a/lrc
+++ b/lrc
@@ -1 +1 @@
-Subproject commit 1a8509ec1408bf28ce0ec0c71fb5f4a5845faab7
+Subproject commit 110c8d8bbd98097b8a2ef20ab76c109e301d2696
diff --git a/plugins b/plugins
index 1bbc7cc2..17c0bc64 160000
--- a/plugins
+++ b/plugins
@@ -1 +1 @@
-Subproject commit 1bbc7cc2a5de65e03676a2f372be7c603df86887
+Subproject commit 17c0bc64a374d601012943a0a79ace65734dd922
diff --git a/scripts/build-package-debian.sh b/scripts/build-package-debian.sh
index bfb7a51c..ef1833df 100755
--- a/scripts/build-package-debian.sh
+++ b/scripts/build-package-debian.sh
@@ -33,69 +33,78 @@ if grep -q "raspbian_10_armhf" <<< "${DISTRIBUTION}"; then
     DPKG_BUILD_OPTIONS="${DPKG_BUILD_OPTIONS} -a armhf"
 fi
 
-cache_packaging=/opt/cache-packaging/${DISTRIBUTION}
+cache_packaging=$TARBALLS/$DISTRIBUTION
+mkdir -p "$cache_packaging"
+
 deb_arch=$(dpkg --print-architecture)
 qt_deb_name=libqt-jami_${DEBIAN_QT_VERSION}_${deb_arch}.deb
 qt_deb_path=${cache_packaging}/${qt_deb_name}
 
 if [ ! -f "${qt_deb_path}" ] || [ "${FORCE_REBUILD_QT}" = "true" ]; then
-    # we need to build Qt
-
-    cache_dir=/opt/ring-contrib
-    temp_dir=$(mktemp -d)
-
-    mkdir /opt/libqt-jami-build
-    cd /opt/libqt-jami-build
-
-    qt_version=${QT_MAJOR}.${QT_MINOR}.${QT_PATCH}
-    tarball_name=qt-everywhere-src-${qt_version}.tar.xz
-    cached_tarball="${cache_dir}/${tarball_name}"
-    qt_base_url=https://download.qt.io/archive/qt/${QT_MAJOR}.${QT_MINOR}/${qt_version}/single
-
-    if [ ! -d "${cache_dir}" ] || [ ! -w "${cache_dir}" ]; then
-        echo "error: $cache_dir does not exist or is not writable"
-        exit 1
-    fi
-
-    if [ ! -f "${cached_tarball}" ]; then
-        (
-            cd "${temp_dir}"
-            wget "${qt_base_url}/${tarball_name}"
-            echo -n "${QT_TARBALL_CHECKSUM}  ${tarball_name}" | sha256sum -c - || \
-                (echo "Qt tarball checksum mismatch; quitting" && exit 1)
-            flock "${cached_tarball}.lock" mv "${tarball_name}" "${cached_tarball}"
-        )
-        rm -rf "${temp_dir}"
-    fi
-
-    cp "${cached_tarball}" libqt-jami_${qt_version}.orig.tar.xz
-    tar xvf libqt-jami_${qt_version}.orig.tar.xz
-    mv qt-everywhere-src-${qt_version} libqt-jami-${qt_version}
-    cd libqt-jami-${qt_version}
-
-    # import the debian folder
-    cp --verbose -r /opt/ring-project-ro/packaging/rules/debian-qt debian
-
-    # create changelog file
-    DEBEMAIL="The Jami project <jami@gnu.org>" dch --create \
-            --package libqt-jami \
-            --newversion ${DEBIAN_QT_VERSION} "New libqt-jami release"
-    DEBEMAIL="The Jami project <jami@gnu.org>" dch --release \
-            --distribution "unstable" debian/changelog
-
-    # build and package qt
-    dpkg-buildpackage -uc -us ${DPKG_BUILD_OPTIONS}
-
-    # copy the built deb to cache
-    mkdir -p ${cache_packaging}/
-    cp ../${qt_deb_name} ${qt_deb_path}
+    (
+        flock 9                 # block until the lock file is gone
+        test -f "${qt_deb_path}" && exit 0 # check again
+
+        # Build Qt.
+        mkdir /opt/libqt-jami-build
+        cd /opt/libqt-jami-build
+
+        qt_version=${QT_MAJOR}.${QT_MINOR}.${QT_PATCH}
+        tarball_name=qt-everywhere-src-${qt_version}.tar.xz
+        cached_tarball=$TARBALLS/$tarball_name
+        qt_base_url=https://download.qt.io/archive/qt/\
+${QT_MAJOR}.${QT_MINOR}/${qt_version}/single
+
+        if [ ! -d "${TARBALLS}" ] || [ ! -w "${TARBALLS}" ]; then
+            echo "error: $TARBALLS does not exist or is not writable"
+            exit 1
+        fi
+
+        if [ ! -f "${cached_tarball}" ]; then
+            (
+                flock 8         # block until the lock file is gone
+                test -f "${cached_tarball}" && exit 0 # check again
+
+                temp_dir=$(mktemp -d)
+                cd "${temp_dir}"
+                wget "${qt_base_url}/${tarball_name}"
+                echo -n "${QT_TARBALL_CHECKSUM}  ${tarball_name}" | sha256sum -c - || \
+                    (echo "Qt tarball checksum mismatch; quitting" && exit 1)
+                mv "${tarball_name}" "${cached_tarball}"
+                rm -rf "${temp_dir}"
+            ) 8>"${cached_tarball}.lock"
+
+        fi
+
+        cp "${cached_tarball}" "libqt-jami_${qt_version}.orig.tar.xz"
+        tar xvf "libqt-jami_${qt_version}.orig.tar.xz"
+        mv "qt-everywhere-src-${qt_version}" "libqt-jami-${qt_version}"
+        cd "libqt-jami-${qt_version}"
+
+        # Import the debian folder.
+        cp --verbose -r /opt/ring-project-ro/packaging/rules/debian-qt debian
+
+        # Create the changelog file.
+        DEBEMAIL="The Jami project <jami@gnu.org>" dch --create \
+                --package libqt-jami \
+                --newversion "${DEBIAN_QT_VERSION}" "New libqt-jami release"
+        DEBEMAIL="The Jami project <jami@gnu.org>" dch --release \
+                --distribution "unstable" debian/changelog
+
+        # Build and package Qt.
+        dpkg-buildpackage -uc -us ${DPKG_BUILD_OPTIONS}
+
+        # Cache the built .deb.
+        mv "../${qt_deb_name}" "${qt_deb_path}"
+
+    ) 9>"${qt_deb_path}.lock"
 fi
 
 # install libqt-jami from cache
-apt-get install -y ${qt_deb_path}
+apt-get install -y "${qt_deb_path}"
 
 # copy libqt-jami to output
-cp ${qt_deb_path} /opt/output/
+cp "${qt_deb_path}" /opt/output/
 
 # Set up work directory.
 mkdir -p /jami/work && cd /jami/work
@@ -103,7 +112,7 @@ mkdir -p /jami/work && cd /jami/work
 # Create a changelog file, required by dpkg-buildpackage.
 mkdir debian
 DEBEMAIL="The Jami project <jami@gnu.org>" dch --create --package jami \
-        --newversion ${DEBIAN_VERSION} "Automatic nightly release"
+        --newversion "${DEBIAN_VERSION}" "Automatic nightly release"
 DEBEMAIL="The Jami project <jami@gnu.org>" dch --release \
         --distribution "unstable" debian/changelog
 
@@ -120,4 +129,4 @@ dpkg-buildpackage -b -uc -us ${DPKG_BUILD_OPTIONS}
 
 # Move the artifacts to the output.
 mv ../*deb /opt/output        # .deb and .ddeb packages
-chown -R ${CURRENT_UID}:${CURRENT_UID} /opt/output
+chown -R "${CURRENT_UID}:${CURRENT_UID}" /opt/output
diff --git a/scripts/build-package-rpm.sh b/scripts/build-package-rpm.sh
index 5d079f05..313415e8 100755
--- a/scripts/build-package-rpm.sh
+++ b/scripts/build-package-rpm.sh
@@ -21,14 +21,13 @@
 # This script is used in the packaging containers to build packages on
 # rpm-based distros.
 #
-
 set -e
 
 # Import the spec file.
 mkdir -p /opt/ring-project
 cd /opt/ring-project
 cp /opt/ring-project-ro/packaging/rules/rpm/* .
-rm jami-libqt.spec
+rm -f jami-libqt.spec
 
 # Prepare the build tree.
 rpmdev-setuptree
@@ -53,45 +52,59 @@ $QT_MAJOR_MINOR_PATCH/single/qt-everywhere-src-$QT_MAJOR_MINOR_PATCH.tar.xz
 
 QT_TARBALL_SHA256="3a530d1b243b5dec00bc54937455471aaa3e56849d2593edb8ded07228202240"
 QT_TARBALL_FILE_NAME=$(basename "$QT_TARBALL_URL")
-CACHED_QT_TARBALL=/opt/ring-contrib/$QT_TARBALL_FILE_NAME
+CACHED_QT_TARBALL=$TARBALLS/$QT_TARBALL_FILE_NAME
 
 if [[ "${DISTRIBUTION:0:4}" == "rhel" \
    || "${DISTRIBUTION:0:13}" == "opensuse-leap" ]]; then
 
-    RPM_PATH=/opt/cache-packaging/${DISTRIBUTION}/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.x86_64.rpm
+    mkdir -p "$TARBALLS/$DISTRIBUTION"
+    RPM_PATH=$TARBALLS/$DISTRIBUTION/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.x86_64.rpm
     if [[ "${DISTRIBUTION:0:4}" == "rhel" ]]; then
-        RPM_PATH=/opt/cache-packaging/${DISTRIBUTION}/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.el8.x86_64.rpm
+        RPM_PATH=$TARBALLS/${DISTRIBUTION}/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.el8.x86_64.rpm
     fi
 
     if [ ! -f "${RPM_PATH}" ]; then
-        mkdir /opt/qt-jami-build
-        cd /opt/qt-jami-build
-        cp /opt/ring-project-ro/packaging/rules/rpm/jami-libqt.spec .
-
-        # Fetch and cache the tarball, if not already available.
-        if [ ! -f "$CACHED_QT_TARBALL" ]; then
-            wget "$QT_TARBALL_URL"
-            if ! echo -n ${QT_TARBALL_SHA256} "$QT_TARBALL_FILE_NAME" | sha256sum -c -
-            then
-                echo "qt tarball checksum mismatch; quitting"
-                exit 1
+        # The following block will only run on one build machine at a
+        # time, thanks to flock.
+        (
+            flock 9             # block until the lock is available
+            test -f "$RPM_PATH" && exit 0 # check again
+
+            mkdir /opt/qt-jami-build
+            cd /opt/qt-jami-build
+            cp /opt/ring-project-ro/packaging/rules/rpm/jami-libqt.spec .
+
+            # Fetch and cache the tarball, if not already available.
+            if [ ! -f "$CACHED_QT_TARBALL" ]; then
+                (
+                    flock 8     # block until the lock file is gone
+                    test -f "$CACHED_QT_TARBALL" && exit 0 # check again
+
+                    wget "$QT_TARBALL_URL"
+                    if ! echo -n ${QT_TARBALL_SHA256} "$QT_TARBALL_FILE_NAME" | sha256sum -c -
+                    then
+                        echo "qt tarball checksum mismatch; quitting"
+                        exit 1
+                    fi
+                    mv "$QT_TARBALL_FILE_NAME" "$CACHED_QT_TARBALL"
+                ) 8>"${CACHED_QT_TARBALL}.lock"
+            fi
+
+            cp "$CACHED_QT_TARBALL" "/root/rpmbuild/SOURCES/jami-qtlib_$QT_MAJOR_MINOR_PATCH.tar.xz"
+            sed -i "s/RELEASE_VERSION/$QT_MAJOR_MINOR_PATCH/g" jami-libqt.spec
+            rpmdev-bumpspec --comment="Automatic nightly release" \
+                            --userstring="Jenkins <jami@lists.savoirfairelinux.net>" jami-libqt.spec
+
+            rpmbuild -ba jami-libqt.spec
+            mkdir -p "$TARBALLS/${DISTRIBUTION}"
+
+            # Cache the built Qt RPM package.
+            if [[ "${DISTRIBUTION:0:4}" == "rhel" ]]; then
+                mv "/root/rpmbuild/RPMS/x86_64/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.el8.x86_64.rpm" "${RPM_PATH}"
+            else
+                mv /root/rpmbuild/RPMS/x86_64/jami-libqt-*.rpm "${RPM_PATH}"
             fi
-            flock "$CACHED_QT_TARBALL" mv "$QT_TARBALL_FILE_NAME" "$CACHED_QT_TARBALL"
-        fi
-
-        cp "$CACHED_QT_TARBALL" "/root/rpmbuild/SOURCES/jami-qtlib_$QT_MAJOR_MINOR_PATCH.tar.xz"
-        sed -i "s/RELEASE_VERSION/$QT_MAJOR_MINOR_PATCH/g" jami-libqt.spec
-        rpmdev-bumpspec --comment="Automatic nightly release" \
-                        --userstring="Jenkins <jami@lists.savoirfairelinux.net>" jami-libqt.spec
-
-        rpmbuild -ba jami-libqt.spec
-        mkdir -p /opt/cache-packaging/${DISTRIBUTION}/
-
-        if [[ "${DISTRIBUTION:0:4}" == "rhel" ]]; then
-            cp "/root/rpmbuild/RPMS/x86_64/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.el8.x86_64.rpm" "${RPM_PATH}"
-        else
-            cp /root/rpmbuild/RPMS/x86_64/jami-libqt-*.rpm "${RPM_PATH}"
-        fi
+        ) 9>"${RPM_PATH}.lock"
     fi
     rpm --install "${RPM_PATH}"
     cp "${RPM_PATH}" /opt/output
@@ -99,9 +112,9 @@ if [[ "${DISTRIBUTION:0:4}" == "rhel" \
 fi
 
 # Set the version and associated comment.
-sed -i "s/RELEASE_VERSION/${RELEASE_VERSION}/g" *.spec
+sed -i "s/RELEASE_VERSION/${RELEASE_VERSION}/g" ./*.spec
 rpmdev-bumpspec --comment="Automatic nightly release" \
-                --userstring="Jenkins <jami@lists.savoirfairelinux.net>" *.spec
+                --userstring="Jenkins <jami@lists.savoirfairelinux.net>" ./*.spec
 
 # Build the daemon and install it.
 rpmbuild -ba jami-daemon.spec
@@ -117,7 +130,7 @@ rpmbuild -ba jami-gnome.spec jami-qt.spec
 # Move the built packages to the output directory.
 mv /root/rpmbuild/RPMS/*/* /opt/output
 touch /opt/output/.packages-built
-chown -R ${CURRENT_UID}:${CURRENT_UID} /opt/output
+chown -R "$CURRENT_UID:$CURRENT_UID" /opt/output
 
 # TODO: One click install: create a package that combines the already
 # built package into one.
diff --git a/scripts/make-packaging-target.py b/scripts/make-packaging-target.py
index 3f933ab9..50cd3bce 100755
--- a/scripts/make-packaging-target.py
+++ b/scripts/make-packaging-target.py
@@ -53,11 +53,11 @@ PACKAGE_%(distribution)s_DOCKER_RUN_COMMAND = docker run \\
     -e CURRENT_UID=$(CURRENT_UID) \\
     -e CURRENT_GID=$(CURRENT_GID) \\
     -e DISTRIBUTION=%(distribution)s \\
+    -e TARBALLS=$(TARBALLS) \\
     -v $(CURDIR)/$(RELEASE_TARBALL_FILENAME):/src/$(RELEASE_TARBALL_FILENAME) \\
     -v $(CURDIR):/opt/ring-project-ro:ro \\
     -v $(CURDIR)/packages/%(distribution)s:/opt/output \\
-    -v /opt/cache-packaging:/opt/cache-packaging \\
-    -v /opt/ring-contrib:/opt/ring-contrib \\
+    -v $(TARBALLS):$(TARBALLS) \\
     -t $(and $(IS_SHELL_INTERACTIVE),-i) %(options)s \\
     $(DOCKER_RUN_EXTRA_ARGS) \\
     $(PACKAGE_%(distribution)s_DOCKER_IMAGE_NAME)
@@ -97,7 +97,6 @@ DPKG_BASED_SYSTEMS_DOCKER_RUN_OPTIONS = (
     '-e QT_PATCH=$(QT_PATCH) '
     '-e QT_TARBALL_CHECKSUM=$(QT_TARBALL_CHECKSUM) '
     '-e FORCE_REBUILD_QT=$(FORCE_REBUILD_QT) '
-    '-v /opt/ring-contrib:/opt/ring-contrib '
     '--privileged '
     '--security-opt apparmor=docker-default ')
 
-- 
GitLab