From bfb8823877137248f7fdaec2b0f4c5c04f09716b Mon Sep 17 00:00:00 2001 From: Maxim Cournoyer <maxim.cournoyer@gmail.com> Date: Thu, 5 Aug 2021 16:59:29 -0400 Subject: [PATCH] ci: Use NFS for the contrib tarballs and cached Qt packages. Change-Id: I99a20bcf357032a6fcf689265c93670dcf65b8b7 --- Jenkinsfile | 9 ++- Makefile | 3 + client-android | 2 +- client-gnome | 2 +- client-ios | 2 +- client-macosx | 2 +- client-qt | 2 +- client-uwp | 2 +- daemon | 2 +- guix/minimal-manifest.scm | 2 + lrc | 2 +- plugins | 2 +- scripts/build-package-debian.sh | 119 +++++++++++++++++-------------- scripts/build-package-rpm.sh | 83 ++++++++++++--------- scripts/make-packaging-target.py | 5 +- 15 files changed, 136 insertions(+), 103 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7e28db69..715925fe 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -68,7 +68,7 @@ pipeline { } environment { - TARBALLS = '/opt/ring-contrib' // set the cache directory + TARBALLS = '/var/cache/jami' // set the cache directory } stages { @@ -79,6 +79,13 @@ pipeline { error "The ${TARBALLS} directory does not exist. \ See https://wiki.savoirfairelinux.com/wiki/Jenkins.jami.net#Configuration" } + + mountType = sh(script: "findmnt ${TARBALLS} -o fstype -n", + returnStdout: true) + if (!(mountType =~ /^nfs/)) { + error "The ${TARBALLS} directory is not mounted on NFS storage. \ +See https://wiki.savoirfairelinux.com/wiki/Jenkins.jami.net#Configuration_client_NFS" + } } } } diff --git a/Makefile b/Makefile index 20b511e0..71a907b7 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,9 @@ # .DEFAULT_GOAL := package-all +# Default caching directory +export TARBALLS ?= /var/cache/jami + ############################## ## Version number variables ## ############################## diff --git a/client-android b/client-android index 195212d0..ba4b6348 160000 --- a/client-android +++ b/client-android @@ -1 +1 @@ -Subproject commit 195212d0b3a9351a7ca9808cf0ec987c22d8f96f +Subproject commit ba4b63485d94c1bcca760cc8c30351f6779affb1 diff --git a/client-gnome b/client-gnome index 0ace8ef7..920c4b9b 160000 --- a/client-gnome +++ b/client-gnome @@ -1 +1 @@ -Subproject commit 0ace8ef7222ea96ac054072ccfc910175c41912e +Subproject commit 920c4b9bf09d2c06978d9bba2fd366c9c4111db9 diff --git a/client-ios b/client-ios index 31ab0ba7..a68da39c 160000 --- a/client-ios +++ b/client-ios @@ -1 +1 @@ -Subproject commit 31ab0ba7c54384730bcda89cfac0698f846a3649 +Subproject commit a68da39c3efce4d5068d8e3e756f55ef3ea71eac diff --git a/client-macosx b/client-macosx index 5f30cfbb..d7aa59f2 160000 --- a/client-macosx +++ b/client-macosx @@ -1 +1 @@ -Subproject commit 5f30cfbb1f853095e8f4d4e6af68d2c2a59259f9 +Subproject commit d7aa59f247a21930a760f81e6c8f9d22defc3a05 diff --git a/client-qt b/client-qt index a2858a88..c2be09f5 160000 --- a/client-qt +++ b/client-qt @@ -1 +1 @@ -Subproject commit a2858a883b73707ab2a226975201c8558d6a991d +Subproject commit c2be09f55aee1f0d4f65ed7751fe3412f0ea0375 diff --git a/client-uwp b/client-uwp index 4b639e9a..7c248e9d 160000 --- a/client-uwp +++ b/client-uwp @@ -1 +1 @@ -Subproject commit 4b639e9a903931bab09292b4d93a4de6d531ae4c +Subproject commit 7c248e9d48ee899ad58156e9cc92e4cc4a02f6bc diff --git a/daemon b/daemon index 73926a76..24f54f16 160000 --- a/daemon +++ b/daemon @@ -1 +1 @@ -Subproject commit 73926a7662910501a3f420e3e595ba862ac5f321 +Subproject commit 24f54f161ca9a2ee195783ce2361c4161a84b41b diff --git a/guix/minimal-manifest.scm b/guix/minimal-manifest.scm index f9c1f14a..c970c696 100644 --- a/guix/minimal-manifest.scm +++ b/guix/minimal-manifest.scm @@ -10,6 +10,7 @@ (specifications->manifest (list + "bash" "coreutils" "gcc-toolchain" "git-minimal" @@ -21,5 +22,6 @@ "python" "sed" "tar" + "util-linux" "wget" "xz")) diff --git a/lrc b/lrc index 1a8509ec..110c8d8b 160000 --- a/lrc +++ b/lrc @@ -1 +1 @@ -Subproject commit 1a8509ec1408bf28ce0ec0c71fb5f4a5845faab7 +Subproject commit 110c8d8bbd98097b8a2ef20ab76c109e301d2696 diff --git a/plugins b/plugins index 1bbc7cc2..17c0bc64 160000 --- a/plugins +++ b/plugins @@ -1 +1 @@ -Subproject commit 1bbc7cc2a5de65e03676a2f372be7c603df86887 +Subproject commit 17c0bc64a374d601012943a0a79ace65734dd922 diff --git a/scripts/build-package-debian.sh b/scripts/build-package-debian.sh index bfb7a51c..ef1833df 100755 --- a/scripts/build-package-debian.sh +++ b/scripts/build-package-debian.sh @@ -33,69 +33,78 @@ if grep -q "raspbian_10_armhf" <<< "${DISTRIBUTION}"; then DPKG_BUILD_OPTIONS="${DPKG_BUILD_OPTIONS} -a armhf" fi -cache_packaging=/opt/cache-packaging/${DISTRIBUTION} +cache_packaging=$TARBALLS/$DISTRIBUTION +mkdir -p "$cache_packaging" + deb_arch=$(dpkg --print-architecture) qt_deb_name=libqt-jami_${DEBIAN_QT_VERSION}_${deb_arch}.deb qt_deb_path=${cache_packaging}/${qt_deb_name} if [ ! -f "${qt_deb_path}" ] || [ "${FORCE_REBUILD_QT}" = "true" ]; then - # we need to build Qt - - cache_dir=/opt/ring-contrib - temp_dir=$(mktemp -d) - - mkdir /opt/libqt-jami-build - cd /opt/libqt-jami-build - - qt_version=${QT_MAJOR}.${QT_MINOR}.${QT_PATCH} - tarball_name=qt-everywhere-src-${qt_version}.tar.xz - cached_tarball="${cache_dir}/${tarball_name}" - qt_base_url=https://download.qt.io/archive/qt/${QT_MAJOR}.${QT_MINOR}/${qt_version}/single - - if [ ! -d "${cache_dir}" ] || [ ! -w "${cache_dir}" ]; then - echo "error: $cache_dir does not exist or is not writable" - exit 1 - fi - - if [ ! -f "${cached_tarball}" ]; then - ( - cd "${temp_dir}" - wget "${qt_base_url}/${tarball_name}" - echo -n "${QT_TARBALL_CHECKSUM} ${tarball_name}" | sha256sum -c - || \ - (echo "Qt tarball checksum mismatch; quitting" && exit 1) - flock "${cached_tarball}.lock" mv "${tarball_name}" "${cached_tarball}" - ) - rm -rf "${temp_dir}" - fi - - cp "${cached_tarball}" libqt-jami_${qt_version}.orig.tar.xz - tar xvf libqt-jami_${qt_version}.orig.tar.xz - mv qt-everywhere-src-${qt_version} libqt-jami-${qt_version} - cd libqt-jami-${qt_version} - - # import the debian folder - cp --verbose -r /opt/ring-project-ro/packaging/rules/debian-qt debian - - # create changelog file - DEBEMAIL="The Jami project <jami@gnu.org>" dch --create \ - --package libqt-jami \ - --newversion ${DEBIAN_QT_VERSION} "New libqt-jami release" - DEBEMAIL="The Jami project <jami@gnu.org>" dch --release \ - --distribution "unstable" debian/changelog - - # build and package qt - dpkg-buildpackage -uc -us ${DPKG_BUILD_OPTIONS} - - # copy the built deb to cache - mkdir -p ${cache_packaging}/ - cp ../${qt_deb_name} ${qt_deb_path} + ( + flock 9 # block until the lock file is gone + test -f "${qt_deb_path}" && exit 0 # check again + + # Build Qt. + mkdir /opt/libqt-jami-build + cd /opt/libqt-jami-build + + qt_version=${QT_MAJOR}.${QT_MINOR}.${QT_PATCH} + tarball_name=qt-everywhere-src-${qt_version}.tar.xz + cached_tarball=$TARBALLS/$tarball_name + qt_base_url=https://download.qt.io/archive/qt/\ +${QT_MAJOR}.${QT_MINOR}/${qt_version}/single + + if [ ! -d "${TARBALLS}" ] || [ ! -w "${TARBALLS}" ]; then + echo "error: $TARBALLS does not exist or is not writable" + exit 1 + fi + + if [ ! -f "${cached_tarball}" ]; then + ( + flock 8 # block until the lock file is gone + test -f "${cached_tarball}" && exit 0 # check again + + temp_dir=$(mktemp -d) + cd "${temp_dir}" + wget "${qt_base_url}/${tarball_name}" + echo -n "${QT_TARBALL_CHECKSUM} ${tarball_name}" | sha256sum -c - || \ + (echo "Qt tarball checksum mismatch; quitting" && exit 1) + mv "${tarball_name}" "${cached_tarball}" + rm -rf "${temp_dir}" + ) 8>"${cached_tarball}.lock" + + fi + + cp "${cached_tarball}" "libqt-jami_${qt_version}.orig.tar.xz" + tar xvf "libqt-jami_${qt_version}.orig.tar.xz" + mv "qt-everywhere-src-${qt_version}" "libqt-jami-${qt_version}" + cd "libqt-jami-${qt_version}" + + # Import the debian folder. + cp --verbose -r /opt/ring-project-ro/packaging/rules/debian-qt debian + + # Create the changelog file. + DEBEMAIL="The Jami project <jami@gnu.org>" dch --create \ + --package libqt-jami \ + --newversion "${DEBIAN_QT_VERSION}" "New libqt-jami release" + DEBEMAIL="The Jami project <jami@gnu.org>" dch --release \ + --distribution "unstable" debian/changelog + + # Build and package Qt. + dpkg-buildpackage -uc -us ${DPKG_BUILD_OPTIONS} + + # Cache the built .deb. + mv "../${qt_deb_name}" "${qt_deb_path}" + + ) 9>"${qt_deb_path}.lock" fi # install libqt-jami from cache -apt-get install -y ${qt_deb_path} +apt-get install -y "${qt_deb_path}" # copy libqt-jami to output -cp ${qt_deb_path} /opt/output/ +cp "${qt_deb_path}" /opt/output/ # Set up work directory. mkdir -p /jami/work && cd /jami/work @@ -103,7 +112,7 @@ mkdir -p /jami/work && cd /jami/work # Create a changelog file, required by dpkg-buildpackage. mkdir debian DEBEMAIL="The Jami project <jami@gnu.org>" dch --create --package jami \ - --newversion ${DEBIAN_VERSION} "Automatic nightly release" + --newversion "${DEBIAN_VERSION}" "Automatic nightly release" DEBEMAIL="The Jami project <jami@gnu.org>" dch --release \ --distribution "unstable" debian/changelog @@ -120,4 +129,4 @@ dpkg-buildpackage -b -uc -us ${DPKG_BUILD_OPTIONS} # Move the artifacts to the output. mv ../*deb /opt/output # .deb and .ddeb packages -chown -R ${CURRENT_UID}:${CURRENT_UID} /opt/output +chown -R "${CURRENT_UID}:${CURRENT_UID}" /opt/output diff --git a/scripts/build-package-rpm.sh b/scripts/build-package-rpm.sh index 5d079f05..313415e8 100755 --- a/scripts/build-package-rpm.sh +++ b/scripts/build-package-rpm.sh @@ -21,14 +21,13 @@ # This script is used in the packaging containers to build packages on # rpm-based distros. # - set -e # Import the spec file. mkdir -p /opt/ring-project cd /opt/ring-project cp /opt/ring-project-ro/packaging/rules/rpm/* . -rm jami-libqt.spec +rm -f jami-libqt.spec # Prepare the build tree. rpmdev-setuptree @@ -53,45 +52,59 @@ $QT_MAJOR_MINOR_PATCH/single/qt-everywhere-src-$QT_MAJOR_MINOR_PATCH.tar.xz QT_TARBALL_SHA256="3a530d1b243b5dec00bc54937455471aaa3e56849d2593edb8ded07228202240" QT_TARBALL_FILE_NAME=$(basename "$QT_TARBALL_URL") -CACHED_QT_TARBALL=/opt/ring-contrib/$QT_TARBALL_FILE_NAME +CACHED_QT_TARBALL=$TARBALLS/$QT_TARBALL_FILE_NAME if [[ "${DISTRIBUTION:0:4}" == "rhel" \ || "${DISTRIBUTION:0:13}" == "opensuse-leap" ]]; then - RPM_PATH=/opt/cache-packaging/${DISTRIBUTION}/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.x86_64.rpm + mkdir -p "$TARBALLS/$DISTRIBUTION" + RPM_PATH=$TARBALLS/$DISTRIBUTION/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.x86_64.rpm if [[ "${DISTRIBUTION:0:4}" == "rhel" ]]; then - RPM_PATH=/opt/cache-packaging/${DISTRIBUTION}/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.el8.x86_64.rpm + RPM_PATH=$TARBALLS/${DISTRIBUTION}/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.el8.x86_64.rpm fi if [ ! -f "${RPM_PATH}" ]; then - mkdir /opt/qt-jami-build - cd /opt/qt-jami-build - cp /opt/ring-project-ro/packaging/rules/rpm/jami-libqt.spec . - - # Fetch and cache the tarball, if not already available. - if [ ! -f "$CACHED_QT_TARBALL" ]; then - wget "$QT_TARBALL_URL" - if ! echo -n ${QT_TARBALL_SHA256} "$QT_TARBALL_FILE_NAME" | sha256sum -c - - then - echo "qt tarball checksum mismatch; quitting" - exit 1 + # The following block will only run on one build machine at a + # time, thanks to flock. + ( + flock 9 # block until the lock is available + test -f "$RPM_PATH" && exit 0 # check again + + mkdir /opt/qt-jami-build + cd /opt/qt-jami-build + cp /opt/ring-project-ro/packaging/rules/rpm/jami-libqt.spec . + + # Fetch and cache the tarball, if not already available. + if [ ! -f "$CACHED_QT_TARBALL" ]; then + ( + flock 8 # block until the lock file is gone + test -f "$CACHED_QT_TARBALL" && exit 0 # check again + + wget "$QT_TARBALL_URL" + if ! echo -n ${QT_TARBALL_SHA256} "$QT_TARBALL_FILE_NAME" | sha256sum -c - + then + echo "qt tarball checksum mismatch; quitting" + exit 1 + fi + mv "$QT_TARBALL_FILE_NAME" "$CACHED_QT_TARBALL" + ) 8>"${CACHED_QT_TARBALL}.lock" + fi + + cp "$CACHED_QT_TARBALL" "/root/rpmbuild/SOURCES/jami-qtlib_$QT_MAJOR_MINOR_PATCH.tar.xz" + sed -i "s/RELEASE_VERSION/$QT_MAJOR_MINOR_PATCH/g" jami-libqt.spec + rpmdev-bumpspec --comment="Automatic nightly release" \ + --userstring="Jenkins <jami@lists.savoirfairelinux.net>" jami-libqt.spec + + rpmbuild -ba jami-libqt.spec + mkdir -p "$TARBALLS/${DISTRIBUTION}" + + # Cache the built Qt RPM package. + if [[ "${DISTRIBUTION:0:4}" == "rhel" ]]; then + mv "/root/rpmbuild/RPMS/x86_64/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.el8.x86_64.rpm" "${RPM_PATH}" + else + mv /root/rpmbuild/RPMS/x86_64/jami-libqt-*.rpm "${RPM_PATH}" fi - flock "$CACHED_QT_TARBALL" mv "$QT_TARBALL_FILE_NAME" "$CACHED_QT_TARBALL" - fi - - cp "$CACHED_QT_TARBALL" "/root/rpmbuild/SOURCES/jami-qtlib_$QT_MAJOR_MINOR_PATCH.tar.xz" - sed -i "s/RELEASE_VERSION/$QT_MAJOR_MINOR_PATCH/g" jami-libqt.spec - rpmdev-bumpspec --comment="Automatic nightly release" \ - --userstring="Jenkins <jami@lists.savoirfairelinux.net>" jami-libqt.spec - - rpmbuild -ba jami-libqt.spec - mkdir -p /opt/cache-packaging/${DISTRIBUTION}/ - - if [[ "${DISTRIBUTION:0:4}" == "rhel" ]]; then - cp "/root/rpmbuild/RPMS/x86_64/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.el8.x86_64.rpm" "${RPM_PATH}" - else - cp /root/rpmbuild/RPMS/x86_64/jami-libqt-*.rpm "${RPM_PATH}" - fi + ) 9>"${RPM_PATH}.lock" fi rpm --install "${RPM_PATH}" cp "${RPM_PATH}" /opt/output @@ -99,9 +112,9 @@ if [[ "${DISTRIBUTION:0:4}" == "rhel" \ fi # Set the version and associated comment. -sed -i "s/RELEASE_VERSION/${RELEASE_VERSION}/g" *.spec +sed -i "s/RELEASE_VERSION/${RELEASE_VERSION}/g" ./*.spec rpmdev-bumpspec --comment="Automatic nightly release" \ - --userstring="Jenkins <jami@lists.savoirfairelinux.net>" *.spec + --userstring="Jenkins <jami@lists.savoirfairelinux.net>" ./*.spec # Build the daemon and install it. rpmbuild -ba jami-daemon.spec @@ -117,7 +130,7 @@ rpmbuild -ba jami-gnome.spec jami-qt.spec # Move the built packages to the output directory. mv /root/rpmbuild/RPMS/*/* /opt/output touch /opt/output/.packages-built -chown -R ${CURRENT_UID}:${CURRENT_UID} /opt/output +chown -R "$CURRENT_UID:$CURRENT_UID" /opt/output # TODO: One click install: create a package that combines the already # built package into one. diff --git a/scripts/make-packaging-target.py b/scripts/make-packaging-target.py index 3f933ab9..50cd3bce 100755 --- a/scripts/make-packaging-target.py +++ b/scripts/make-packaging-target.py @@ -53,11 +53,11 @@ PACKAGE_%(distribution)s_DOCKER_RUN_COMMAND = docker run \\ -e CURRENT_UID=$(CURRENT_UID) \\ -e CURRENT_GID=$(CURRENT_GID) \\ -e DISTRIBUTION=%(distribution)s \\ + -e TARBALLS=$(TARBALLS) \\ -v $(CURDIR)/$(RELEASE_TARBALL_FILENAME):/src/$(RELEASE_TARBALL_FILENAME) \\ -v $(CURDIR):/opt/ring-project-ro:ro \\ -v $(CURDIR)/packages/%(distribution)s:/opt/output \\ - -v /opt/cache-packaging:/opt/cache-packaging \\ - -v /opt/ring-contrib:/opt/ring-contrib \\ + -v $(TARBALLS):$(TARBALLS) \\ -t $(and $(IS_SHELL_INTERACTIVE),-i) %(options)s \\ $(DOCKER_RUN_EXTRA_ARGS) \\ $(PACKAGE_%(distribution)s_DOCKER_IMAGE_NAME) @@ -97,7 +97,6 @@ DPKG_BASED_SYSTEMS_DOCKER_RUN_OPTIONS = ( '-e QT_PATCH=$(QT_PATCH) ' '-e QT_TARBALL_CHECKSUM=$(QT_TARBALL_CHECKSUM) ' '-e FORCE_REBUILD_QT=$(FORCE_REBUILD_QT) ' - '-v /opt/ring-contrib:/opt/ring-contrib ' '--privileged ' '--security-opt apparmor=docker-default ') -- GitLab