Commit bfb88238 authored by Maxim Cournoyer's avatar Maxim Cournoyer Committed by Amin Bandali
Browse files

ci: Use NFS for the contrib tarballs and cached Qt packages.

Change-Id: I99a20bcf357032a6fcf689265c93670dcf65b8b7
parent 6b53758f
......@@ -68,7 +68,7 @@ pipeline {
}
environment {
TARBALLS = '/opt/ring-contrib' // set the cache directory
TARBALLS = '/var/cache/jami' // set the cache directory
}
stages {
......@@ -79,6 +79,13 @@ pipeline {
error "The ${TARBALLS} directory does not exist. \
See https://wiki.savoirfairelinux.com/wiki/Jenkins.jami.net#Configuration"
}
mountType = sh(script: "findmnt ${TARBALLS} -o fstype -n",
returnStdout: true)
if (!(mountType =~ /^nfs/)) {
error "The ${TARBALLS} directory is not mounted on NFS storage. \
See https://wiki.savoirfairelinux.com/wiki/Jenkins.jami.net#Configuration_client_NFS"
}
}
}
}
......
......@@ -16,6 +16,9 @@
#
.DEFAULT_GOAL := package-all
# Default caching directory
export TARBALLS ?= /var/cache/jami
##############################
## Version number variables ##
##############################
......
client-android @ ba4b6348
Subproject commit 195212d0b3a9351a7ca9808cf0ec987c22d8f96f
Subproject commit ba4b63485d94c1bcca760cc8c30351f6779affb1
client-gnome @ 920c4b9b
Subproject commit 0ace8ef7222ea96ac054072ccfc910175c41912e
Subproject commit 920c4b9bf09d2c06978d9bba2fd366c9c4111db9
client-ios @ a68da39c
Subproject commit 31ab0ba7c54384730bcda89cfac0698f846a3649
Subproject commit a68da39c3efce4d5068d8e3e756f55ef3ea71eac
client-macosx @ d7aa59f2
Subproject commit 5f30cfbb1f853095e8f4d4e6af68d2c2a59259f9
Subproject commit d7aa59f247a21930a760f81e6c8f9d22defc3a05
client-qt @ c2be09f5
Subproject commit a2858a883b73707ab2a226975201c8558d6a991d
Subproject commit c2be09f55aee1f0d4f65ed7751fe3412f0ea0375
client-uwp @ 7c248e9d
Subproject commit 4b639e9a903931bab09292b4d93a4de6d531ae4c
Subproject commit 7c248e9d48ee899ad58156e9cc92e4cc4a02f6bc
daemon @ 24f54f16
Subproject commit 73926a7662910501a3f420e3e595ba862ac5f321
Subproject commit 24f54f161ca9a2ee195783ce2361c4161a84b41b
......@@ -10,6 +10,7 @@
(specifications->manifest
(list
"bash"
"coreutils"
"gcc-toolchain"
"git-minimal"
......@@ -21,5 +22,6 @@
"python"
"sed"
"tar"
"util-linux"
"wget"
"xz"))
lrc @ 110c8d8b
Subproject commit 1a8509ec1408bf28ce0ec0c71fb5f4a5845faab7
Subproject commit 110c8d8bbd98097b8a2ef20ab76c109e301d2696
plugins @ 17c0bc64
Subproject commit 1bbc7cc2a5de65e03676a2f372be7c603df86887
Subproject commit 17c0bc64a374d601012943a0a79ace65734dd922
......@@ -33,69 +33,78 @@ if grep -q "raspbian_10_armhf" <<< "${DISTRIBUTION}"; then
DPKG_BUILD_OPTIONS="${DPKG_BUILD_OPTIONS} -a armhf"
fi
cache_packaging=/opt/cache-packaging/${DISTRIBUTION}
cache_packaging=$TARBALLS/$DISTRIBUTION
mkdir -p "$cache_packaging"
deb_arch=$(dpkg --print-architecture)
qt_deb_name=libqt-jami_${DEBIAN_QT_VERSION}_${deb_arch}.deb
qt_deb_path=${cache_packaging}/${qt_deb_name}
if [ ! -f "${qt_deb_path}" ] || [ "${FORCE_REBUILD_QT}" = "true" ]; then
# we need to build Qt
cache_dir=/opt/ring-contrib
temp_dir=$(mktemp -d)
mkdir /opt/libqt-jami-build
cd /opt/libqt-jami-build
qt_version=${QT_MAJOR}.${QT_MINOR}.${QT_PATCH}
tarball_name=qt-everywhere-src-${qt_version}.tar.xz
cached_tarball="${cache_dir}/${tarball_name}"
qt_base_url=https://download.qt.io/archive/qt/${QT_MAJOR}.${QT_MINOR}/${qt_version}/single
if [ ! -d "${cache_dir}" ] || [ ! -w "${cache_dir}" ]; then
echo "error: $cache_dir does not exist or is not writable"
exit 1
fi
if [ ! -f "${cached_tarball}" ]; then
(
cd "${temp_dir}"
wget "${qt_base_url}/${tarball_name}"
echo -n "${QT_TARBALL_CHECKSUM} ${tarball_name}" | sha256sum -c - || \
(echo "Qt tarball checksum mismatch; quitting" && exit 1)
flock "${cached_tarball}.lock" mv "${tarball_name}" "${cached_tarball}"
)
rm -rf "${temp_dir}"
fi
cp "${cached_tarball}" libqt-jami_${qt_version}.orig.tar.xz
tar xvf libqt-jami_${qt_version}.orig.tar.xz
mv qt-everywhere-src-${qt_version} libqt-jami-${qt_version}
cd libqt-jami-${qt_version}
# import the debian folder
cp --verbose -r /opt/ring-project-ro/packaging/rules/debian-qt debian
# create changelog file
DEBEMAIL="The Jami project <jami@gnu.org>" dch --create \
--package libqt-jami \
--newversion ${DEBIAN_QT_VERSION} "New libqt-jami release"
DEBEMAIL="The Jami project <jami@gnu.org>" dch --release \
--distribution "unstable" debian/changelog
# build and package qt
dpkg-buildpackage -uc -us ${DPKG_BUILD_OPTIONS}
# copy the built deb to cache
mkdir -p ${cache_packaging}/
cp ../${qt_deb_name} ${qt_deb_path}
(
flock 9 # block until the lock file is gone
test -f "${qt_deb_path}" && exit 0 # check again
# Build Qt.
mkdir /opt/libqt-jami-build
cd /opt/libqt-jami-build
qt_version=${QT_MAJOR}.${QT_MINOR}.${QT_PATCH}
tarball_name=qt-everywhere-src-${qt_version}.tar.xz
cached_tarball=$TARBALLS/$tarball_name
qt_base_url=https://download.qt.io/archive/qt/\
${QT_MAJOR}.${QT_MINOR}/${qt_version}/single
if [ ! -d "${TARBALLS}" ] || [ ! -w "${TARBALLS}" ]; then
echo "error: $TARBALLS does not exist or is not writable"
exit 1
fi
if [ ! -f "${cached_tarball}" ]; then
(
flock 8 # block until the lock file is gone
test -f "${cached_tarball}" && exit 0 # check again
temp_dir=$(mktemp -d)
cd "${temp_dir}"
wget "${qt_base_url}/${tarball_name}"
echo -n "${QT_TARBALL_CHECKSUM} ${tarball_name}" | sha256sum -c - || \
(echo "Qt tarball checksum mismatch; quitting" && exit 1)
mv "${tarball_name}" "${cached_tarball}"
rm -rf "${temp_dir}"
) 8>"${cached_tarball}.lock"
fi
cp "${cached_tarball}" "libqt-jami_${qt_version}.orig.tar.xz"
tar xvf "libqt-jami_${qt_version}.orig.tar.xz"
mv "qt-everywhere-src-${qt_version}" "libqt-jami-${qt_version}"
cd "libqt-jami-${qt_version}"
# Import the debian folder.
cp --verbose -r /opt/ring-project-ro/packaging/rules/debian-qt debian
# Create the changelog file.
DEBEMAIL="The Jami project <jami@gnu.org>" dch --create \
--package libqt-jami \
--newversion "${DEBIAN_QT_VERSION}" "New libqt-jami release"
DEBEMAIL="The Jami project <jami@gnu.org>" dch --release \
--distribution "unstable" debian/changelog
# Build and package Qt.
dpkg-buildpackage -uc -us ${DPKG_BUILD_OPTIONS}
# Cache the built .deb.
mv "../${qt_deb_name}" "${qt_deb_path}"
) 9>"${qt_deb_path}.lock"
fi
# install libqt-jami from cache
apt-get install -y ${qt_deb_path}
apt-get install -y "${qt_deb_path}"
# copy libqt-jami to output
cp ${qt_deb_path} /opt/output/
cp "${qt_deb_path}" /opt/output/
# Set up work directory.
mkdir -p /jami/work && cd /jami/work
......@@ -103,7 +112,7 @@ mkdir -p /jami/work && cd /jami/work
# Create a changelog file, required by dpkg-buildpackage.
mkdir debian
DEBEMAIL="The Jami project <jami@gnu.org>" dch --create --package jami \
--newversion ${DEBIAN_VERSION} "Automatic nightly release"
--newversion "${DEBIAN_VERSION}" "Automatic nightly release"
DEBEMAIL="The Jami project <jami@gnu.org>" dch --release \
--distribution "unstable" debian/changelog
......@@ -120,4 +129,4 @@ dpkg-buildpackage -b -uc -us ${DPKG_BUILD_OPTIONS}
# Move the artifacts to the output.
mv ../*deb /opt/output # .deb and .ddeb packages
chown -R ${CURRENT_UID}:${CURRENT_UID} /opt/output
chown -R "${CURRENT_UID}:${CURRENT_UID}" /opt/output
......@@ -21,14 +21,13 @@
# This script is used in the packaging containers to build packages on
# rpm-based distros.
#
set -e
# Import the spec file.
mkdir -p /opt/ring-project
cd /opt/ring-project
cp /opt/ring-project-ro/packaging/rules/rpm/* .
rm jami-libqt.spec
rm -f jami-libqt.spec
# Prepare the build tree.
rpmdev-setuptree
......@@ -53,45 +52,59 @@ $QT_MAJOR_MINOR_PATCH/single/qt-everywhere-src-$QT_MAJOR_MINOR_PATCH.tar.xz
QT_TARBALL_SHA256="3a530d1b243b5dec00bc54937455471aaa3e56849d2593edb8ded07228202240"
QT_TARBALL_FILE_NAME=$(basename "$QT_TARBALL_URL")
CACHED_QT_TARBALL=/opt/ring-contrib/$QT_TARBALL_FILE_NAME
CACHED_QT_TARBALL=$TARBALLS/$QT_TARBALL_FILE_NAME
if [[ "${DISTRIBUTION:0:4}" == "rhel" \
|| "${DISTRIBUTION:0:13}" == "opensuse-leap" ]]; then
RPM_PATH=/opt/cache-packaging/${DISTRIBUTION}/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.x86_64.rpm
mkdir -p "$TARBALLS/$DISTRIBUTION"
RPM_PATH=$TARBALLS/$DISTRIBUTION/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.x86_64.rpm
if [[ "${DISTRIBUTION:0:4}" == "rhel" ]]; then
RPM_PATH=/opt/cache-packaging/${DISTRIBUTION}/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.el8.x86_64.rpm
RPM_PATH=$TARBALLS/${DISTRIBUTION}/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.el8.x86_64.rpm
fi
if [ ! -f "${RPM_PATH}" ]; then
mkdir /opt/qt-jami-build
cd /opt/qt-jami-build
cp /opt/ring-project-ro/packaging/rules/rpm/jami-libqt.spec .
# Fetch and cache the tarball, if not already available.
if [ ! -f "$CACHED_QT_TARBALL" ]; then
wget "$QT_TARBALL_URL"
if ! echo -n ${QT_TARBALL_SHA256} "$QT_TARBALL_FILE_NAME" | sha256sum -c -
then
echo "qt tarball checksum mismatch; quitting"
exit 1
# The following block will only run on one build machine at a
# time, thanks to flock.
(
flock 9 # block until the lock is available
test -f "$RPM_PATH" && exit 0 # check again
mkdir /opt/qt-jami-build
cd /opt/qt-jami-build
cp /opt/ring-project-ro/packaging/rules/rpm/jami-libqt.spec .
# Fetch and cache the tarball, if not already available.
if [ ! -f "$CACHED_QT_TARBALL" ]; then
(
flock 8 # block until the lock file is gone
test -f "$CACHED_QT_TARBALL" && exit 0 # check again
wget "$QT_TARBALL_URL"
if ! echo -n ${QT_TARBALL_SHA256} "$QT_TARBALL_FILE_NAME" | sha256sum -c -
then
echo "qt tarball checksum mismatch; quitting"
exit 1
fi
mv "$QT_TARBALL_FILE_NAME" "$CACHED_QT_TARBALL"
) 8>"${CACHED_QT_TARBALL}.lock"
fi
cp "$CACHED_QT_TARBALL" "/root/rpmbuild/SOURCES/jami-qtlib_$QT_MAJOR_MINOR_PATCH.tar.xz"
sed -i "s/RELEASE_VERSION/$QT_MAJOR_MINOR_PATCH/g" jami-libqt.spec
rpmdev-bumpspec --comment="Automatic nightly release" \
--userstring="Jenkins <jami@lists.savoirfairelinux.net>" jami-libqt.spec
rpmbuild -ba jami-libqt.spec
mkdir -p "$TARBALLS/${DISTRIBUTION}"
# Cache the built Qt RPM package.
if [[ "${DISTRIBUTION:0:4}" == "rhel" ]]; then
mv "/root/rpmbuild/RPMS/x86_64/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.el8.x86_64.rpm" "${RPM_PATH}"
else
mv /root/rpmbuild/RPMS/x86_64/jami-libqt-*.rpm "${RPM_PATH}"
fi
flock "$CACHED_QT_TARBALL" mv "$QT_TARBALL_FILE_NAME" "$CACHED_QT_TARBALL"
fi
cp "$CACHED_QT_TARBALL" "/root/rpmbuild/SOURCES/jami-qtlib_$QT_MAJOR_MINOR_PATCH.tar.xz"
sed -i "s/RELEASE_VERSION/$QT_MAJOR_MINOR_PATCH/g" jami-libqt.spec
rpmdev-bumpspec --comment="Automatic nightly release" \
--userstring="Jenkins <jami@lists.savoirfairelinux.net>" jami-libqt.spec
rpmbuild -ba jami-libqt.spec
mkdir -p /opt/cache-packaging/${DISTRIBUTION}/
if [[ "${DISTRIBUTION:0:4}" == "rhel" ]]; then
cp "/root/rpmbuild/RPMS/x86_64/jami-libqt-$QT_MAJOR_MINOR_PATCH-1.el8.x86_64.rpm" "${RPM_PATH}"
else
cp /root/rpmbuild/RPMS/x86_64/jami-libqt-*.rpm "${RPM_PATH}"
fi
) 9>"${RPM_PATH}.lock"
fi
rpm --install "${RPM_PATH}"
cp "${RPM_PATH}" /opt/output
......@@ -99,9 +112,9 @@ if [[ "${DISTRIBUTION:0:4}" == "rhel" \
fi
# Set the version and associated comment.
sed -i "s/RELEASE_VERSION/${RELEASE_VERSION}/g" *.spec
sed -i "s/RELEASE_VERSION/${RELEASE_VERSION}/g" ./*.spec
rpmdev-bumpspec --comment="Automatic nightly release" \
--userstring="Jenkins <jami@lists.savoirfairelinux.net>" *.spec
--userstring="Jenkins <jami@lists.savoirfairelinux.net>" ./*.spec
# Build the daemon and install it.
rpmbuild -ba jami-daemon.spec
......@@ -117,7 +130,7 @@ rpmbuild -ba jami-gnome.spec jami-qt.spec
# Move the built packages to the output directory.
mv /root/rpmbuild/RPMS/*/* /opt/output
touch /opt/output/.packages-built
chown -R ${CURRENT_UID}:${CURRENT_UID} /opt/output
chown -R "$CURRENT_UID:$CURRENT_UID" /opt/output
# TODO: One click install: create a package that combines the already
# built package into one.
......@@ -53,11 +53,11 @@ PACKAGE_%(distribution)s_DOCKER_RUN_COMMAND = docker run \\
-e CURRENT_UID=$(CURRENT_UID) \\
-e CURRENT_GID=$(CURRENT_GID) \\
-e DISTRIBUTION=%(distribution)s \\
-e TARBALLS=$(TARBALLS) \\
-v $(CURDIR)/$(RELEASE_TARBALL_FILENAME):/src/$(RELEASE_TARBALL_FILENAME) \\
-v $(CURDIR):/opt/ring-project-ro:ro \\
-v $(CURDIR)/packages/%(distribution)s:/opt/output \\
-v /opt/cache-packaging:/opt/cache-packaging \\
-v /opt/ring-contrib:/opt/ring-contrib \\
-v $(TARBALLS):$(TARBALLS) \\
-t $(and $(IS_SHELL_INTERACTIVE),-i) %(options)s \\
$(DOCKER_RUN_EXTRA_ARGS) \\
$(PACKAGE_%(distribution)s_DOCKER_IMAGE_NAME)
......@@ -97,7 +97,6 @@ DPKG_BASED_SYSTEMS_DOCKER_RUN_OPTIONS = (
'-e QT_PATCH=$(QT_PATCH) '
'-e QT_TARBALL_CHECKSUM=$(QT_TARBALL_CHECKSUM) '
'-e FORCE_REBUILD_QT=$(FORCE_REBUILD_QT) '
'-v /opt/ring-contrib:/opt/ring-contrib '
'--privileged '
'--security-opt apparmor=docker-default ')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment