build/linux: Make Snap on GNOME runners (not on Launchpad)
Now can make snaps 3x faster on CI without snapcraft remote-build.
This commit is contained in:
Bruno Lopes
parent
b960be9735
commit
5da4b89002
4 changed files with 101 additions and 76 deletions
|
|
@ -439,19 +439,14 @@ gimp-flatpak:
|
|||
- if: '$GIMP_CI_SNAP != null || "$[[ inputs.distribution_pipeline ]]" =~ /.*GIMP_CI_SNAP.*/'
|
||||
parallel:
|
||||
matrix:
|
||||
- DPKG_ARCH: [arm64, amd64]
|
||||
- RUNNER: [aarch64, x86_64_v3]
|
||||
tags:
|
||||
- x86_64_v2
|
||||
image:
|
||||
name: $CI_REGISTRY_IMAGE:build-snap-${SNAPCRAFT_CORE_VERSION}
|
||||
entrypoint: [""]
|
||||
- $RUNNER
|
||||
image: $CI_REGISTRY_IMAGE:build-snap-${SNAPCRAFT_BASE_VERSION}-${RUNNER}
|
||||
variables:
|
||||
SNAPCRAFT_CORE_VERSION: "8_core24"
|
||||
#@brunvonlope private credentials. We are using these for now
|
||||
#until we don't get acess to 'gimp' entry on Snap Store: https://github.com/snapcrafters/gimp/issues/447
|
||||
LAUNCHPAD_CREDENTIALS_ACCESS_TOKEN: pPs367Km5glpfXXK9BQl
|
||||
LAUNCHPAD_CREDENTIALS_ACCESS_SECRET: PQ0r1JGdHQRWQCS255jNZdglCcK4KDmsGRSZxl4CZwWvMF00GLRFbz1185L08cjh5zWqhQddsvmgLg7l
|
||||
timeout: 120m
|
||||
SNAPCRAFT_BASE_VERSION: "8_core24"
|
||||
RUNNER: x86_64_v3
|
||||
timeout: 40m
|
||||
|
||||
deps-snap:
|
||||
extends: .snap
|
||||
|
|
@ -460,7 +455,30 @@ deps-snap:
|
|||
name: gcr.io/kaniko-project/executor:debug
|
||||
entrypoint: [""]
|
||||
script:
|
||||
- sh build/linux/snap/1_build-deps-snapcraft.sh
|
||||
- echo "FROM ghcr.io/canonical/snapcraft:${SNAPCRAFT_BASE_VERSION}" > Dockerfile
|
||||
- echo "ENTRYPOINT [\"\"]" >> Dockerfile
|
||||
- echo "WORKDIR $CI_PROJECT_DIR" >> Dockerfile
|
||||
- echo "ENV GITLAB_CI=1" >> Dockerfile
|
||||
- echo "ENV RUNNER=$RUNNER" >> Dockerfile
|
||||
- echo "RUN apt-get update -y" >> Dockerfile
|
||||
- echo "RUN apt-get install -y curl jq squashfs-tools sudo" >> Dockerfile
|
||||
# https://github.com/canonical/snapcraft-rocks/issues/37
|
||||
- echo "RUN cp -r /usr/lib/python3.12/site-packages/extensions/* /usr/share/snapcraft/extensions/" >> Dockerfile
|
||||
# https://github.com/canonical/snapcraft-rocks/issues/33
|
||||
- echo "RUN ln -s /usr/libexec/snapcraft/craftctl /usr/bin/craftctl" >> Dockerfile
|
||||
- echo "RUN sh build/linux/snap/1_build-deps-snapcraft.sh --install-snaps" >> Dockerfile;
|
||||
# Build babl and GEGL
|
||||
- echo "FROM $CI_REGISTRY_IMAGE:build-snap-${SNAPCRAFT_BASE_VERSION}-${RUNNER}" > Dockerfile2;
|
||||
- echo "RUN sh build/linux/snap/1_build-deps-snapcraft.sh" >> Dockerfile2;
|
||||
- /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $CI_REGISTRY_IMAGE:build-snap-${SNAPCRAFT_BASE_VERSION}-${RUNNER} --cache=true --cache-ttl=120h --image-fs-extract-retry 1 --verbosity=warn
|
||||
- if [ -f 'Dockerfile2' ]; then /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile2 --destination build-snap-${SNAPCRAFT_BASE_VERSION}-${RUNNER}-temp --cache=false --no-push --verbosity=warn; fi
|
||||
artifacts:
|
||||
paths:
|
||||
- _install-$RUNNER.tar
|
||||
- _build-$RUNNER.tar
|
||||
- babl-meson-log.tar
|
||||
- gegl-meson-log.tar
|
||||
expire_in: 2 hours
|
||||
|
||||
gimp-snap:
|
||||
extends: .snap
|
||||
|
|
@ -475,7 +493,8 @@ gimp-snap:
|
|||
artifacts:
|
||||
paths:
|
||||
- temp*.snap
|
||||
- snapcraft*.txt
|
||||
- gimp-snapcraft.log
|
||||
- gimp-meson-log.tar
|
||||
expire_in: 2 days
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -15,55 +15,56 @@ elif [ $(basename "$PWD") = 'snap' ]; then
|
|||
fi
|
||||
|
||||
|
||||
# Install part of the deps (snapcraft)
|
||||
if [ -z "$GITLAB_CI" ]; then
|
||||
#Only snapcraft snap is available locally
|
||||
if ! which snapcraft >/dev/null 2>&1; then
|
||||
sudo apt update
|
||||
sudo apt install snapd
|
||||
sudo snap install snapd
|
||||
sudo snap install snapcraft --classic
|
||||
fi
|
||||
else
|
||||
#Only snapcraft docker image is available in CI
|
||||
echo "FROM ghcr.io/canonical/snapcraft:${SNAPCRAFT_CORE_VERSION}" > Dockerfile
|
||||
echo "ENTRYPOINT [\"\"]" >> Dockerfile
|
||||
echo "RUN apt-get update -y" >> Dockerfile
|
||||
echo "RUN apt-get install -y git" >> Dockerfile
|
||||
export SNAPCRAFT_CREDENTIALS_PATH="$HOME/.local/share/snapcraft/launchpad-credentials"
|
||||
echo "RUN mkdir -p $(dirname $SNAPCRAFT_CREDENTIALS_PATH)" >> Dockerfile
|
||||
echo "RUN printf '[1]\n' > $SNAPCRAFT_CREDENTIALS_PATH" >> Dockerfile
|
||||
echo "RUN printf 'consumer_key = System-wide:\ Ubuntu (Ubuntu-vai)\n' >> $SNAPCRAFT_CREDENTIALS_PATH" >> Dockerfile && sed 's|\\ Ubuntu| Ubuntu|g' -i Dockerfile
|
||||
echo "RUN printf 'consumer_secret = \n' >> $SNAPCRAFT_CREDENTIALS_PATH" >> Dockerfile
|
||||
echo "RUN printf 'access_token = %s\n' $LAUNCHPAD_CREDENTIALS_ACCESS_TOKEN >> $SNAPCRAFT_CREDENTIALS_PATH" >> Dockerfile
|
||||
echo "RUN printf 'access_secret = %s\n' $LAUNCHPAD_CREDENTIALS_ACCESS_SECRET >> $SNAPCRAFT_CREDENTIALS_PATH" >> Dockerfile
|
||||
/kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/Dockerfile --destination $CI_REGISTRY_IMAGE:build-snap-${SNAPCRAFT_CORE_VERSION} --cache=true --cache-ttl=120h --image-fs-extract-retry 1 --verbosity=warn
|
||||
fi
|
||||
# Install part of the deps
|
||||
if ! which snapcraft >/dev/null 2>&1; then
|
||||
printf '(INFO): installing snapcraft\n'
|
||||
sudo apt update -y
|
||||
sudo apt install -y snapd
|
||||
sudo snap install snapd
|
||||
sudo snap install snapcraft --classic
|
||||
fi #End of check
|
||||
|
||||
if [ "$GITLAB_CI" ] && [ "$1" = '--install-snaps' ]; then
|
||||
#snapd can not be used to install snaps on CI since it is a daemon so we manually "install" them
|
||||
GNOME_SDK=$(grep '^_SDK_SNAP' $(sudo find $(dirname $(which snapcraft))/.. -name gnome.py | grep -i extensions) | sed -n "s/.*\"$(sed -n 's/^base:[[:space:]]*//p' build/linux/snap/snapcraft.yaml)\": *\"\\([^\"]*\\)\".*/\\1/p")
|
||||
gnome_runtime=$(echo $GNOME_SDK | sed 's/-sdk//')
|
||||
mesa_runtime="mesa-$(echo $GNOME_SDK | sed -n 's/.*-\([0-9]\+\)-sdk/\1/p')"
|
||||
for snap in $GNOME_SDK $gnome_runtime $mesa_runtime; do
|
||||
if [ ! -d /snap/$snap/ ]; then
|
||||
curl --progress-bar -L -o ${snap}.snap $(curl --progress-bar -H 'Snap-Device-Series: 16' https://api.snapcraft.io/v2/snaps/info/$snap | jq -r --arg arch "$(dpkg --print-architecture)" '.["channel-map"][] | select(.channel.architecture == $arch and .channel.track == "latest" and .channel.risk == "stable") | .download.url')
|
||||
mkdir -p /snap/$snap/current
|
||||
unsquashfs -d /snap/$snap/current ${snap}.snap
|
||||
fi
|
||||
done
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
||||
# Prepare env
|
||||
if [ -z "$GITLAB_CI" ]; then
|
||||
printf "\e[0Ksection_start:`date +%s`:snap_environ[collapsed=true]\r\e[0KPreparing build environment\n"
|
||||
## portable environment which works on CI (unlike lxd) and on VMs (unlike multipass)
|
||||
export SNAPCRAFT_BUILD_ENVIRONMENT=host
|
||||
build_environment_option='--destructive-mode'
|
||||
## (snapcraft does not allow to freely set the .yaml path, so let's just temporarely copy it)
|
||||
# Prepare env (snapcraft does not allow to freely set the .yaml path)
|
||||
cp build/linux/snap/snapcraft.yaml .
|
||||
printf "\e[0Ksection_end:`date +%s`:snap_environ\r\e[0K\n"
|
||||
|
||||
|
||||
# Build babl and gegl
|
||||
# Build babl and GEGL
|
||||
printf "\e[0Ksection_start:`date +%s`:deps_install[collapsed=true]\r\e[0KInstalling dependencies provided by Ubuntu\n"
|
||||
sudo snapcraft pull $build_environment_option --build-for=${DPKG_ARCH:-$(dpkg --print-architecture)} --verbosity=verbose
|
||||
sudo snapcraft pull --destructive-mode --build-for=$(dpkg --print-architecture) --verbosity=verbose
|
||||
printf "\e[0Ksection_end:`date +%s`:deps_install\r\e[0K\n"
|
||||
|
||||
|
||||
printf "\e[0Ksection_start:`date +%s`:babl_build[collapsed=true]\r\e[0KBuilding babl\n"
|
||||
sudo snapcraft stage babl $build_environment_option --build-for=${DPKG_ARCH:-$(dpkg --print-architecture)} --verbosity=verbose
|
||||
sudo snapcraft stage babl --destructive-mode --build-for=$(dpkg --print-architecture) --verbosity=verbose
|
||||
if [ "$GITLAB_CI" ]; then
|
||||
tar cf babl-meson-log.tar /root/parts/babl/build/meson-logs/meson-log.txt >/dev/null 2>&1
|
||||
fi
|
||||
printf "\e[0Ksection_end:`date +%s`:babl_build\r\e[0K\n"
|
||||
|
||||
printf "\e[0Ksection_start:`date +%s`:gegl_build[collapsed=true]\r\e[0KBuilding gegl\n"
|
||||
sudo snapcraft stage gegl $build_environment_option --build-for=${DPKG_ARCH:-$(dpkg --print-architecture)} --verbosity=verbose
|
||||
printf "\e[0Ksection_end:`date +%s`:gegl_build\r\e[0K\n"
|
||||
sudo snapcraft stage gegl --destructive-mode --build-for=$(dpkg --print-architecture) --verbosity=verbose
|
||||
if [ "$GITLAB_CI" ]; then
|
||||
tar cf gegl-meson-log.tar /root/parts/gegl/build/meson-logs/meson-log.txt >/dev/null 2>&1
|
||||
printf "\e[0Ksection_end:`date +%s`:gegl_build\r\e[0K\n"
|
||||
|
||||
## Save built deps for 'gimp-snap' job
|
||||
tar cf _install-$RUNNER.tar /root/stage/ >/dev/null 2>&1
|
||||
tar cf _build-$RUNNER.tar /root/parts/ >/dev/null 2>&1
|
||||
fi
|
||||
|
||||
rm snapcraft.yaml
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -18,33 +18,38 @@ if [ -z "$GITLAB_CI" ]; then
|
|||
fi
|
||||
|
||||
|
||||
# Prepare env
|
||||
printf "\e[0Ksection_start:`date +%s`:snap_environ[collapsed=true]\r\e[0KPreparing build environment\n"
|
||||
## portable environment which works on CI (unlike lxd) and on VMs (unlike multipass)
|
||||
export SNAPCRAFT_BUILD_ENVIRONMENT=host
|
||||
build_environment_option='--destructive-mode'
|
||||
## (snapcraft does not allow to freely set the .yaml path, so let's just temporarely copy it)
|
||||
# Install part of the deps
|
||||
eval "$(sed -n '/Install part/,/End of check/p' build/linux/snap/1_build-deps-snapcraft.sh)"
|
||||
|
||||
if [ "$GITLAB_CI" ]; then
|
||||
# Extract deps from previous job
|
||||
tar xf _install-$RUNNER.tar -C /
|
||||
tar xf _build-$RUNNER.tar -C /
|
||||
fi
|
||||
|
||||
|
||||
# Prepare env (snapcraft does not allow to freely set the .yaml path)
|
||||
cp build/linux/snap/snapcraft.yaml .
|
||||
printf "\e[0Ksection_end:`date +%s`:snap_environ\r\e[0K\n"
|
||||
|
||||
|
||||
# Build (babl, gegl and) GIMP
|
||||
printf "\e[0Ksection_start:`date +%s`:gimp_build[collapsed=true]\r\e[0KBuilding (babl, gegl and) GIMP\n"
|
||||
if [ -z "$GITLAB_CI" ]; then
|
||||
sudo snapcraft stage gimp $build_environment_option --build-for=${DPKG_ARCH:-$(dpkg --print-architecture)} --verbosity=verbose
|
||||
else
|
||||
## (snapcraft remote-build does not allow building in parts, so we need to build and pack everything in one go)
|
||||
snapcraft remote-build --launchpad-accept-public-upload --build-for=$DPKG_ARCH --launchpad-timeout=7200 --verbosity=verbose
|
||||
# Build GIMP
|
||||
printf "\e[0Ksection_start:`date +%s`:gimp_build[collapsed=true]\r\e[0KBuilding GIMP\n"
|
||||
sudo snapcraft stage gimp --destructive-mode --build-for=$(dpkg --print-architecture) --verbosity=verbose > gimp-snapcraft.log 2>&1 || { cat gimp-snapcraft.log; exit 1; }
|
||||
if [ "$GITLAB_CI" ]; then
|
||||
tar cf gimp-meson-log.tar /root/parts/gimp/build/meson-logs/meson-log.txt >/dev/null 2>&1
|
||||
fi
|
||||
printf "\e[0Ksection_end:`date +%s`:gimp_build\r\e[0K\n"
|
||||
|
||||
|
||||
# Bundle
|
||||
printf "\e[0Ksection_start:`date +%s`:gimp_bundle[collapsed=true]\r\e[0KBundling GIMP\n"
|
||||
if [ -z "$GITLAB_CI" ]; then
|
||||
sudo snapcraft prime $build_environment_option --build-for=${DPKG_ARCH:-$(dpkg --print-architecture)} --verbosity=verbose
|
||||
else
|
||||
# Rename .snap bundle file to avoid confusion (the distribution of the .snap is done only in 3_dist-gimp-snapcraft.sh)
|
||||
mv *.snap temp_$(echo *.snap)
|
||||
printf "\e[0Ksection_start:`date +%s`:gimp_bundle[collapsed=true]\r\e[0KCreating prime dir\n"
|
||||
sudo mkdir /tmp/craft-state
|
||||
sudo snapcraft prime --destructive-mode --build-for=$(dpkg --print-architecture) --verbosity=trace >> gimp-snapcraft.log 2>&1 || { cat gimp-snapcraft.log; exit 1; }
|
||||
|
||||
if [ "$GITLAB_CI" ]; then
|
||||
## On CI, make the .snap prematurely on each runner since snapcraft does not allow multiple prime/ directories
|
||||
eval "$(sed -n -e '/GIMP_VERSION=/,/NAME=/ { s/ //; p }' build/linux/snap/3_dist-gimp-snapcraft.sh)"
|
||||
eval "$(grep 'snapcraft pack' build/linux/snap/3_dist-gimp-snapcraft.sh | sed 's/ //')"
|
||||
fi
|
||||
printf "\e[0Ksection_end:`date +%s`:gimp_bundle\r\e[0K\n"
|
||||
|
||||
|
|
|
|||
|
|
@ -23,8 +23,8 @@ printf "\e[0Ksection_end:`date +%s`:snap_tlkt\r\e[0K\n"
|
|||
|
||||
# Global info
|
||||
printf "\e[0Ksection_start:`date +%s`:snap_info\r\e[0KGetting snap global info\n"
|
||||
#(we do not use config.h like other scripts because the info is on snapcraft.yaml too
|
||||
#and `snapcraft remote-build` from previous job does not get config.h from launchpad)
|
||||
#(we do not use config.h like other scripts because the info needs to be on snapcraft.yaml too
|
||||
#so taking such info from the .yaml ensures we not forget to manually update it)
|
||||
cp build/linux/snap/snapcraft.yaml .
|
||||
|
||||
## Get info about GIMP version
|
||||
|
|
@ -63,8 +63,8 @@ for SNAP in $supported_archs; do
|
|||
SNAP=$(echo "$SNAP" | sed 's|^\./temp_||')
|
||||
printf "\e[0Ksection_start:`date +%s`:${SNAP}_making[collapsed=true]\r\e[0KFinishing ${SNAP}\n"
|
||||
if [ -z "$GITLAB_CI" ]; then
|
||||
#as explained in 2_build-gimp-snapcraft.sh, we can only make snaps this way locally due to remote-build limitations
|
||||
sudo snapcraft pack --destructive-mode --output temp_${SNAP}
|
||||
#as explained in 2_build-gimp-snapcraft.sh, we can only make snaps this way locally due to snapcraft design
|
||||
sudo snapcraft pack --destructive-mode --output temp_${NAME}_${GIMP_VERSION}_$(dpkg --print-architecture).snap
|
||||
fi
|
||||
mv temp_${SNAP} ${SNAP}
|
||||
printf "(INFO): Suceeded. To test this build, install it from the artifact with: sudo snap install --dangerous ${SNAP}\n"
|
||||
|
|
|
|||
Loading…
Reference in a new issue