Merge branch 'adapt-linux-repository-generation-to-encompass-mullvad-des-866'

6 files changed, 431 insertions, 0 deletions

diff --git a/ci/linux-repository-builder/build-linux-repositories-config.sh b/ci/linux-repository-builder/build-linux-repositories-config.sh
new file mode 100644
index 0000000000..bed015f5c0
--- /dev/null
+++ b/ci/linux-repository-builder/build-linux-repositories-config.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# The directory to use as inbox. This is where .src files are read
+#export LINUX_REPOSITORY_INBOX_DIR_BASE="PLEASE CONFIGURE ME"
+
+# GPG key to sign the repositories with
+export CODE_SIGNING_KEY_FINGERPRINT="A1198702FC3E0A09A9AE5B75D5A1D4F266DE8DDF"
+
+# Debian codenames we support.
+SUPPORTED_DEB_CODENAMES=("sid" "testing" "bookworm" "bullseye")
+# Ubuntu codenames we support. Latest two LTS. But when adding a new
+# don't immediately remove the oldest one. Allow for some transition period
+# with the last three.
+SUPPORTED_DEB_CODENAMES+=("noble" "jammy" "focal")
+# ... + latest non-LTS Ubuntu. We officially only support the latest non-LTS.
+# But to not cause too much disturbance just when Ubuntu is released, we keep
+# the last two codenames working in the repository.
+SUPPORTED_DEB_CODENAMES+=("mantic")
+export SUPPORTED_DEB_CODENAMES
+
+export SUPPORTED_RPM_ARCHITECTURES=("x86_64" "aarch64")
+
+export REPOSITORIES=("stable" "beta")
+
+export PRODUCTION_LINUX_REPOSITORY_PUBLIC_URL="https://repository.mullvad.net"
+export STAGING_LINUX_REPOSITORY_PUBLIC_URL="https://repository.stagemole.eu"
+export DEV_LINUX_REPOSITORY_PUBLIC_URL="https://repository.devmole.eu"
+
+# Servers to upload Linux deb/rpm repositories to
+export PRODUCTION_REPOSITORY_SERVER="cdn.mullvad.net"
+export STAGING_REPOSITORY_SERVER="cdn.stagemole.eu"
+export DEV_REPOSITORY_SERVER="cdn.devmole.eu"
diff --git a/ci/linux-repository-builder/build-linux-repositories.sh b/ci/linux-repository-builder/build-linux-repositories.sh
new file mode 100755
index 0000000000..418183485b
--- /dev/null
+++ b/ci/linux-repository-builder/build-linux-repositories.sh
@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+#
+# Builds linux deb and rpm repositories and uploads them to a repository server.
+# One instance of this script targets *one* server environment.
+# This means that if you want to publish to development, staging and production servers,
+# you need to run three instances of this script.
+#
+# This script works on an $inbox_dir. In this directory it expects one directory per repository.
+# For each repository it will read all files having the .src extension 
+# These files are expected to contain a single line, a path to some directory where
+# it can read new artifacts for that product.
+# All deb and rpm files from that directory will be signed and moved over to a folder with
+# the same name as the .src file, but with a .latest extension instead.
+# So artifacts read from `app.src` will be moved to `app.latest/`.
+#
+# Then the deb and rpm repositories will be generated and all deb and rpm files in
+# $inbox_dir/$repository/*.latest/ will be added to the repository. These repositories are then synced
+# to `$repository_server_upload_domain respectively.
+
+set -eu
+# nullglob is needed to produce expected results when globing an empty directory
+shopt -s nullglob
+
+function usage() {
+    echo "Usage: $0 <environment>"
+    echo
+    echo "Example usage: ./build-linux-repositories.sh production"
+    echo
+    echo "This script reads an inbox folder for the corresponding server environment."
+    echo "It then generates and uploads new Linux repositories for all"
+    echo "the latest artifacts"
+    echo
+    echo "Options:"
+    echo "  -h | --help		Show this help message and exit."
+    exit 1
+}
+
+if [[ "$#" == 0 || $1 == "-h" || $1 == "--help" ]]; then
+    usage
+fi
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# shellcheck source=ci/linux-repository-builder/build-linux-repositories-config.sh
+source "$SCRIPT_DIR/build-linux-repositories-config.sh"
+
+environment="$1"
+case "$environment" in
+    "production")
+        repository_server_upload_domain="$PRODUCTION_REPOSITORY_SERVER"
+        repository_server_public_url="$PRODUCTION_LINUX_REPOSITORY_PUBLIC_URL"
+        ;;
+    "staging")
+        repository_server_upload_domain="$STAGING_REPOSITORY_SERVER"
+        repository_server_public_url="$STAGING_LINUX_REPOSITORY_PUBLIC_URL"
+        ;;
+    "dev")
+        repository_server_upload_domain="$DEV_REPOSITORY_SERVER"
+        repository_server_public_url="$DEV_LINUX_REPOSITORY_PUBLIC_URL"
+        ;;
+    *)
+        echo "Unknown environment. Specify production, staging or dev" >&2
+        exit 1
+        ;;
+esac
+
+inbox_dir="$LINUX_REPOSITORY_INBOX_DIR_BASE/$environment"
+
+if [[ ! -d "$inbox_dir" ]]; then
+    echo "Inbox $inbox_dir does not exist" 1>&2
+    exit 1
+fi
+
+# Process all .src files in the given inbox dir.
+# Signs and moves all found artifacts into .latest directories
+# Returns 0 if everything went well and there are new artifacts for a product.
+# Returns 1 if no new artifacts were found
+function process_inbox {
+    local inbox_dir=$1
+    echo "[#] Processing inbox at $inbox_dir"
+
+    local found_new_artifacts="false"
+    # Read all notify files and move the artifacts they point to into a local .latest copy
+    for notify_file in "$inbox_dir"/*.src; do
+        if [[ ! -f "$notify_file" ]]; then
+            echo "Ignoring non-file $notify_file" 1>&2
+            continue
+        fi
+        echo "Processing notify file $notify_file"
+
+        local src_dir
+        src_dir=$(cat "$notify_file")
+        if [[ ! -d "$src_dir" ]]; then
+            echo "Artifact source dir $src_dir from notify file $notify_file does not exist" 1>&2
+            continue
+        fi
+
+        # Removing the file before we move the artifacts out of where it points.
+        # This ensures we don't get stuck in a loop processing it over and over
+        # if the signing and moving fails.
+        rm "$notify_file"
+
+        local artifact_dir=${notify_file/%.src/.latest}
+        # Recreate the artifact dir, cleaning it
+        rm -rf "$artifact_dir" && mkdir -p "$artifact_dir" || exit 1
+
+        # The fact that we have processed one .src file is enough tro trigger a repository
+        # rebuild. Because if a product would like to publish "no artifacts" they should
+        # be able to create a .src file pointing to an empty directory
+        found_new_artifacts="true"
+
+        echo "Moving artifacts from $src_dir to $artifact_dir"
+        # Move all deb and rpm files into the .latest dir
+        for src_deb in "$src_dir"/*.deb; do
+            echo "Signing and moving $src_deb into $artifact_dir/"
+            dpkg-sig --sign builder "$src_deb"
+            mv "$src_deb" "$artifact_dir/"
+        done
+        for src_rpm in "$src_dir"/*.rpm; do
+            echo "Signing and moving $src_rpm into $artifact_dir/"
+            rpm --addsign "$src_rpm"
+            mv "$src_rpm" "$artifact_dir/"
+        done
+        rm -r "$src_dir" || echo "Failed to remove src dir $src_dir"
+    done
+
+    if [[ $found_new_artifacts == "false" ]]; then
+        return 1
+    fi
+    return 0
+}
+
+function rsync_repo {
+    local local_repo_dir=$1
+    local remote_repo_dir=$2
+
+    echo "Syncing to $repository_server_upload_domain:$remote_repo_dir"
+    rsync -av --delete --mkpath --rsh='ssh -p 1122' \
+        "$local_repo_dir"/ \
+        build@"$repository_server_upload_domain":"$remote_repo_dir"
+}
+
+for repository in "${REPOSITORIES[@]}"; do
+    deb_remote_repo_dir="deb/$repository"
+    rpm_remote_repo_dir="rpm/$repository"
+
+    repository_inbox_dir="$inbox_dir/$repository"
+    if ! process_inbox "$repository_inbox_dir"; then
+        echo "Nothing new in inbox at $repository_inbox_dir"
+        continue
+    fi
+
+    # Read all .latest artifact dirs into array
+    readarray -d '' artifact_dirs < <(find "$repository_inbox_dir" -maxdepth 1 -name "*.latest" -type d -print0)
+    if [ "${#artifact_dirs[@]}" -lt 1 ]; then
+        echo "No artifact directories in $repository_inbox_dir to generate repositories from" >&2
+        continue
+    fi
+
+    echo "Generating repositories from these artifact directories: ${artifact_dirs[*]}"
+
+    # Generate deb repository from all the .latest artifacts
+
+    deb_repo_dir="$repository_inbox_dir/repos/deb"
+    rm -rf "$deb_repo_dir" && mkdir -p "$deb_repo_dir" || exit 1
+    "$SCRIPT_DIR/prepare-apt-repository.sh" "$deb_repo_dir" "${artifact_dirs[@]}"
+
+    # Generate rpm repository from all the .latest artifacts
+
+    rpm_repo_dir="$repository_inbox_dir/repos/rpm"
+    rm -rf "$rpm_repo_dir" && mkdir -p "$rpm_repo_dir" || exit 1
+    "$SCRIPT_DIR/prepare-rpm-repository.sh" "$rpm_repo_dir" \
+        "$repository_server_public_url" "$rpm_remote_repo_dir" "$repository" "${artifact_dirs[@]}"
+
+    # rsync repositories to repository server
+
+    echo "[#] Syncing deb repository to $deb_remote_repo_dir"
+    rsync_repo "$deb_repo_dir" "$deb_remote_repo_dir"
+    echo "[#] Syncing rpm repository to $rpm_remote_repo_dir"
+    rsync_repo "$rpm_repo_dir" "$rpm_remote_repo_dir"
+
+done
diff --git a/ci/linux-repository-builder/build-linux-repositories@.service b/ci/linux-repository-builder/build-linux-repositories@.service
new file mode 100644
index 0000000000..e4b8d78a23
--- /dev/null
+++ b/ci/linux-repository-builder/build-linux-repositories@.service
@@ -0,0 +1,24 @@
+# Create separate services with instance names being set to "$server_environment".
+# See the build-linux-repositories.sh script for details.
+#
+# For example:
+# * build-linux-repositories@production.service
+# * build-linux-repositories@staging.service
+# * build-linux-repositories@dev.service
+#
+# To install this service, do the following:
+#
+# * Log in to the user with `sudo machinectl shell <build account>@` to get a proper shell
+# * Copy this service file and corresponding timer file to ~/.config/systemd/user
+# * Edit `ExecStart` path to point to absolute path to script
+# * Reload systemd: `systemctl --user daemon-reload`
+# * Start the timers:
+#   $ systemctl --user enable --now build-linux-repositories@production.timer
+#   ... Repeat for other environments.
+
+[Unit]
+Description=Mullvad Linux repository generation and upload service
+
+[Service]
+Type=oneshot
+ExecStart=./build-linux-repositories.sh %i
diff --git a/ci/linux-repository-builder/build-linux-repositories@.timer b/ci/linux-repository-builder/build-linux-repositories@.timer
new file mode 100644
index 0000000000..b4f4aaa202
--- /dev/null
+++ b/ci/linux-repository-builder/build-linux-repositories@.timer
@@ -0,0 +1,9 @@
+[Unit]
+Description=Run Mullvad Linux repository generation and upload service periodically
+
+[Timer]
+OnCalendar=*-*-* *:*:30
+Persistent=true
+
+[Install]
+WantedBy=timers.target
diff --git a/ci/linux-repository-builder/prepare-apt-repository.sh b/ci/linux-repository-builder/prepare-apt-repository.sh
new file mode 100755
index 0000000000..18a2d5bdf9
--- /dev/null
+++ b/ci/linux-repository-builder/prepare-apt-repository.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+set -eu
+# nullglob is needed to produce expected results when globing an empty directory
+shopt -s nullglob
+
+function usage() {
+    echo "Usage: $0 <repository dir> <artifact dirs...>"
+    echo
+    echo "Will create a deb repository in <repository dir> and add all .deb files from all <artifact dirs>"
+    echo
+    echo "Options:"
+    echo "  -h | --help		Show this help message and exit."
+    exit 1
+}
+
+if [[ "$#" == 0 || $1 == "-h" || $1 == "--help" ]]; then
+    usage
+fi
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# shellcheck source=ci/linux-repository-builder/build-linux-repositories-config.sh
+source "$SCRIPT_DIR/build-linux-repositories-config.sh"
+
+repo_dir=${1:?"Specify the output repository directory as the first argument"}
+
+artifact_dirs=()
+while [ "$#" -gt 1 ]; do
+    artifact_dirs+=("$2")
+    shift
+done
+
+if [ "${#artifact_dirs[@]}" -lt 1 ]; then
+    echo "No artifact directories given" >&2
+    exit 1
+fi
+
+function generate_repository_configuration {
+    local codename=$1
+
+    echo -e "Origin: repository.mullvad.net
+Label: Mullvad apt repository
+Description: Mullvad package repository for Debian/Ubuntu
+Codename: $codename
+Architectures: amd64 arm64
+Components: main
+SignWith: $CODE_SIGNING_KEY_FINGERPRINT"
+}
+
+function generate_deb_distributions_content {
+    local distributions=""
+    for codename in "${SUPPORTED_DEB_CODENAMES[@]}"; do
+        distributions+=$(generate_repository_configuration "$codename")$'\n'$'\n'
+        distributions+=$(generate_repository_configuration "$codename"-testing)$'\n'$'\n'
+    done
+    echo "$distributions"
+}
+
+function add_deb_to_repo {
+    local deb_path=$1
+    local codename=$2
+    echo "Adding $deb_path to repository $codename"
+    reprepro -V --basedir "$repo_dir" --component main includedeb "$codename" "$deb_path"
+}
+
+echo "Generating deb repository into $repo_dir/"
+mkdir -p "$repo_dir/conf"
+
+echo "Writing repository configuration to $repo_dir/conf/distributions"
+generate_deb_distributions_content > "$repo_dir/conf/distributions"
+echo ""
+
+for artifact_dir in "${artifact_dirs[@]}"; do
+    for deb_path in "$artifact_dir"/*.deb; do
+        for codename in "${SUPPORTED_DEB_CODENAMES[@]}"; do
+            add_deb_to_repo "$deb_path" "$codename"
+            echo ""
+        done
+        echo ""
+    done
+done
diff --git a/ci/linux-repository-builder/prepare-rpm-repository.sh b/ci/linux-repository-builder/prepare-rpm-repository.sh
new file mode 100755
index 0000000000..4e258dedb4
--- /dev/null
+++ b/ci/linux-repository-builder/prepare-rpm-repository.sh
@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+
+set -eu
+# nullglob is needed to produce expected results when globing an empty directory
+shopt -s nullglob
+
+function usage() {
+    echo "Usage: $0 <repository dir> <repository server url> <remote repo dir> <stable_or_beta> <artifact dirs...>"
+    echo
+    echo "Example: $0 ./repos/rpm https://cdn.mullvad.net/ rpm/stable stable inbox/app.latest inbox/browser.latest"
+    echo
+    echo "Will create an rpm repository in <repository dir> and add all .rpm files from all <artifact dirs> to it."
+    echo
+    echo "Options:"
+    echo "  -h | --help		Show this help message and exit."
+    exit 1
+}
+
+if [[ "$#" == 0 || $1 == "-h" || $1 == "--help" ]]; then
+    usage
+fi
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+# shellcheck source=ci/linux-repository-builder/build-linux-repositories-config.sh
+source "$SCRIPT_DIR/build-linux-repositories-config.sh"
+
+repo_dir=${1:?"Specify the output repository directory as the first argument"}
+repository_server_url=${2:?"Give repository server URL as second argument (needed for repository metadata generation)"}
+remote_repo_dir=${3:?"Give remote repo dir as third argument"}
+stable_or_beta=${4:?"Give 'stable' or 'beta' as fourth argument"}
+shift 4
+
+artifact_dirs=()
+while [ "$#" -gt 0 ]; do
+    artifact_dirs+=("$1")
+    shift
+done
+
+if [[ "$stable_or_beta" != "beta" && "$stable_or_beta" != "stable" ]]; then
+    echo "<stable or beta> argument must be 'stable' or 'beta'"
+    exit 1
+fi
+if [ "${#artifact_dirs[@]}" -lt 1 ]; then
+    echo "No artifact directories given" >&2
+    exit 1
+fi
+
+# Writes the mullvad.repo config file to the repository root.
+# This needs to contain the absolute url and path to the repository.
+# As such, it depends on what server we upload to as well as if it's stable or beta.
+function generate_rpm_repository_configuration {
+    local repository_dir=$1
+    local repository_server_url=$2
+    local remote_repo_dir=$3
+    local stable_or_beta=$4
+
+    local repository_name="Mullvad VPN"
+    if [[ "$stable_or_beta" == "beta" ]]; then
+        repository_name+=" (beta)"
+    fi
+
+    echo -e "[mullvad-$stable_or_beta]
+name=$repository_name
+baseurl=$repository_server_url/$remote_repo_dir/\$basearch
+type=rpm
+enabled=1
+gpgcheck=1
+gpgkey=$repository_server_url/rpm/mullvad-keyring.asc" > "$repository_dir/mullvad.repo"
+}
+
+for artifact_dir in "${artifact_dirs[@]}"; do
+    for arch in "${SUPPORTED_RPM_ARCHITECTURES[@]}"; do
+        arch_repo_dir="$repo_dir/$arch"
+        for rpm_path in "$artifact_dir"/*"$arch".rpm; do
+            mkdir -p "$arch_repo_dir"
+            echo "[#] Copying $rpm_path to $arch_repo_dir/"
+            cp "$rpm_path" "$arch_repo_dir"/
+        done
+    done
+done
+
+for arch in "${SUPPORTED_RPM_ARCHITECTURES[@]}"; do
+    arch_repo_dir="$repo_dir/$arch"
+    if [[ ! -d "$arch_repo_dir" ]]; then
+        echo "No $arch repository in $repo_dir" >&2
+        continue
+    fi
+
+    echo "Generating architecture repository metadata for $arch_repo_dir"
+
+    # Generate repository metadata files (containing among other things checksums
+    # for the rpm files in it)
+    createrepo_c "$arch_repo_dir"
+
+    # Sign repository metadata (created by createrepo_c above)
+    gpg --detach-sign --armor "$arch_repo_dir/repodata/repomd.xml"
+done
+
+echo "Generating global repository configuration in $repo_dir"
+generate_rpm_repository_configuration "$repo_dir" "$repository_server_url" "$remote_repo_dir" "$stable_or_beta"
+