Merge branch 're-use-builds-of-test-manager-and-test-runner-for-multiple-des-1141'

11 files changed, 512 insertions, 187 deletions

diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml
index 2165308d76..274107a29b 100644
--- a/.github/workflows/desktop-e2e.yml
+++ b/.github/workflows/desktop-e2e.yml
@@ -106,8 +106,9 @@ jobs:
           echo "inner_container_image=$(cat ./building/linux-container-image.txt)" >> $GITHUB_ENV
     outputs:
       container_image: ${{ env.inner_container_image }}
-  build-linux:
-    name: Build Linux
+
+  build-linux-app:
+    name: Build Linux App
     needs: prepare-linux
     runs-on: ubuntu-latest
     container:
@@ -123,15 +124,26 @@ jobs:
       - name: Checkout submodules
         run: |
           git config --global --add safe.directory '*'
-          git submodule update --init --depth=1 dist-assets/binaries
-          git submodule update --init wireguard-go-rs/libwg/wireguard-go
+          git submodule update --init --depth=1
+      - uses: actions/cache@v4
+        id: cache-app-cargo-artifacts
+        with:
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            target/
+          key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
       - name: Build app
-        env:
-          USE_MOLD: false
-        run: ./build.sh
+        run: |
+          export CARGO_TARGET_DIR=target/
+          ./build.sh --optimize
       - name: Build test executable
         run: ./desktop/packages/mullvad-vpn/scripts/build-test-executable.sh
-      - uses: actions/upload-artifact@v4
+      - name: Upload app
+        uses: actions/upload-artifact@v4
         if: '!cancelled()'
         with:
           name: linux-build
@@ -140,9 +152,83 @@ jobs:
             ./dist/*.deb
             ./dist/app-e2e-*
 
+  build-mullvad-version-linux:
+    name: Build mullvad-version
+    needs: prepare-linux
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ needs.prepare-linux.outputs.container_image }}
+    steps:
+      # Fix for HOME path overridden by GH runners when building in containers, see:
+      # https://github.com/actions/runner/issues/863
+      - name: Fix HOME path
+        run: echo "HOME=/root" >> $GITHUB_ENV
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Build mullvad-version
+        run: |
+          cargo build --package mullvad-version --release
+          # Move `mullvad-version` to a known location. This is needed in the coming `upload-artifact` step.
+          mkdir bin
+          mv -t ./bin/ "$CARGO_TARGET_DIR/release/mullvad-version"
+        shell: bash
+      - name: Upload mullvad-version
+        uses: actions/upload-artifact@v4
+        with:
+          name: mullvad-version-linux
+          path: ./bin/mullvad-version
+          if-no-files-found: error
+
+  build-test-manager-linux:
+    name: Build Test Manager
+    needs: prepare-linux
+    # Note: libssl-dev is installed on the test server, so build test-manager there for the sake of simplicity
+    runs-on: [self-hosted, desktop-test, Linux] # app-test-linux
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - uses: actions-rust-lang/setup-rust-toolchain@v1
+      - name: Build test-manager
+        run: ./test/scripts/container-run.sh cargo build --package test-manager --release
+      - uses: actions/upload-artifact@v4
+        if: '!cancelled()'
+        with:
+          name: linux-test-manager-build
+          path: |
+            ./test/target/release/test-manager
+          if-no-files-found: error
+      - name: Clean up Cargo artifacts
+        run: |
+          cargo clean
+
+  build-test-runner-binaries-linux:
+    name: Build Test Runner Binaries
+    needs: prepare-linux
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ needs.prepare-linux.outputs.container_image }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Build binaries
+        run: |
+          # Move test runner binaries to a known location. This is needed in the coming `upload-artifact` step.
+          mkdir bin
+          test/scripts/build/test-runner.sh linux
+          mv -t ./bin/ \
+            "$CARGO_TARGET_DIR/x86_64-unknown-linux-gnu/release/test-runner" \
+            "$CARGO_TARGET_DIR/x86_64-unknown-linux-gnu/release/connection-checker"
+      - uses: actions/upload-artifact@v4
+        if: '!cancelled()'
+        with:
+          name: linux-test-runner-binaries
+          path: bin/*
+          if-no-files-found: error
+
   e2e-test-linux:
     name: Linux end-to-end tests
-    needs: [prepare-matrices, build-linux]
+    # yamllint disable-line rule:line-length
+    needs: [prepare-matrices, build-linux-app, build-mullvad-version-linux, build-test-manager-linux, build-test-runner-binaries-linux]
     if: |
       !cancelled() &&
       needs.prepare-matrices.outputs.linux_matrix != '[]' &&
@@ -154,20 +240,59 @@ jobs:
       matrix:
         os: ${{ fromJSON(needs.prepare-matrices.outputs.linux_matrix) }}
     steps:
-      - uses: actions/download-artifact@v4
-        if: ${{ needs.build-linux.result == 'success' }}
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Create binaries directory & add to PATH
+        shell: bash -ieo pipefail {0}
+        run: |
+          # Put all binaries in a known folder: test-runner, connection-checker, mullvad-version
+          mkdir "${{ github.workspace }}/bin"
+          echo "${{ github.workspace }}/bin/" >> "$GITHUB_PATH"
+      - name: Download Test Manager
+        uses: actions/download-artifact@v4
+        if: ${{ needs.build-test-manager-linux.result == 'success' }}
+        with:
+          name: linux-test-manager-build
+          path: ${{ github.workspace }}/bin
+      - name: Download mullvad-version
+        uses: actions/download-artifact@v4
+        if: ${{ needs.build-test-manager-linux.result == 'success' }}
+        with:
+          name: mullvad-version-linux
+          path: ${{ github.workspace }}/bin
+      - name: Download Test Runner binaries
+        uses: actions/download-artifact@v4
+        if: ${{ needs.build-test-runner-binaries-linux.result == 'success' }}
+        with:
+          name: linux-test-runner-binaries
+          path: ${{ github.workspace }}/bin
+      - name: chmod binaries
+        run: |
+          chmod +x ${{ github.workspace }}/bin/*
+        shell: bash
+      - name: Check binaries
+        run: |
+          ls -la ${{ github.workspace }}/bin
+        shell: bash
+      - name: Download App
+        uses: actions/download-artifact@v4
+        if: ${{ needs.build-linux-app.result == 'success' }}
         with:
           name: linux-build
           path: ~/.cache/mullvad-test/packages
-      - name: Checkout repository
-        uses: actions/checkout@v4
       - name: Run end-to-end tests
         shell: bash -ieo pipefail {0}
         run: |
+          # A directory with all the binaries is required to run test-manager.
+          # The test scripts which runs in CI expects this folder to be available as the `TEST_DIST_DIR` variable.
+          export TEST_DIST_DIR="${{ github.workspace }}/bin/"
           git fetch --tags --prune-tags --force
           export TEST_FILTERS="${{ github.event.inputs.tests }}"
-          ./test/scripts/ci-runtests.sh ${{ matrix.os }}
-      - uses: actions/upload-artifact@v4
+          ls -la "$TEST_DIST_DIR"
+          ${{ github.workspace }}/bin/mullvad-version
+          ./test/scripts/run/ci.sh ${{ matrix.os }}
+      - name: Upload test report
+        uses: actions/upload-artifact@v4
         if: '!cancelled()'
         with:
           name: ${{ matrix.os }}_report
@@ -183,11 +308,10 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-        with:
-          submodules: true
       - name: Checkout submodules
         run: |
-          git submodule update --init wireguard-go-rs/libwg/wireguard-go
+          git config --global --add safe.directory '*'
+          git submodule update --init --depth=1
       - name: Install Protoc
         uses: arduino/setup-protoc@v3
         with:
@@ -242,7 +366,8 @@ jobs:
       matrix:
         os: ${{ fromJSON(needs.prepare-matrices.outputs.windows_matrix) }}
     steps:
-      - uses: actions/download-artifact@v4
+      - name: Download App
+        uses: actions/download-artifact@v4
         if: ${{ needs.build-windows.result == 'success' }}
         with:
           name: windows-build
@@ -255,7 +380,8 @@ jobs:
           git fetch --tags --prune-tags --force
           export TEST_FILTERS="${{ github.event.inputs.tests }}"
           ./test/scripts/ci-runtests.sh ${{ matrix.os }}
-      - uses: actions/upload-artifact@v4
+      - name: Upload test report
+        uses: actions/upload-artifact@v4
         if: '!cancelled()'
         with:
           name: ${{ matrix.os }}_report
@@ -274,7 +400,7 @@ jobs:
       - name: Checkout submodules
         run: |
           git config --global --add safe.directory '*'
-          git submodule update --init wireguard-go-rs/libwg/wireguard-go
+          git submodule update --init --depth=1
       - name: Install Go
         uses: actions/setup-go@v3
         with:
@@ -314,7 +440,8 @@ jobs:
       matrix:
         os: ${{ fromJSON(needs.prepare-matrices.outputs.macos_matrix) }}
     steps:
-      - uses: actions/download-artifact@v4
+      - name: Download App
+        uses: actions/download-artifact@v4
         if: ${{ needs.build-macos.result == 'success' }}
         with:
           name: macos-build
@@ -327,7 +454,8 @@ jobs:
           git fetch --tags --prune-tags --force
           export TEST_FILTERS="${{ github.event.inputs.tests }}"
           ./test/scripts/ci-runtests.sh ${{ matrix.os }}
-      - uses: actions/upload-artifact@v4
+      - name: Upload test report
+        uses: actions/upload-artifact@v4
         if: '!cancelled()'
         with:
           name: ${{ matrix.os }}_report
@@ -337,25 +465,38 @@ jobs:
     name: Result matrix
     needs: [e2e-test-linux, e2e-test-windows, e2e-test-macos]
     if: '!cancelled()'
-    runs-on: [self-hosted, desktop-test, Linux]
-    timeout-minutes: 240
-    strategy:
-      fail-fast: false
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ needs.prepare-linux.outputs.container_image }}
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-      - uses: actions/download-artifact@v4
+      - name: Download test report
+        uses: actions/download-artifact@v4
         with:
-          path: ./test/.ci-logs/artifacts
+          pattern: '*_report'
+          merge-multiple: true
+      - name: Create binaries directory
+        shell: bash -ieo pipefail {0}
+        run: |
+          mkdir "${{ github.workspace }}/bin"
+      - name: Download report compiler
+        uses: actions/download-artifact@v4
+        with:
+          name: linux-test-manager-build
+          path: ${{ github.workspace }}/bin
+      - name: chmod binaries
+        run: |
+          chmod +x ${{ github.workspace }}/bin/*
+      - name: Check binaries
+        run: |
+          ls -la ${{ github.workspace }}/bin
+        shell: bash
       - name: Generate test result matrix
         shell: bash -ieo pipefail {0}
         run: |
-          cd test
-          # "Unpack" the downloaded report artifacts: https://github.com/actions/download-artifact/issues/141
-          cp ./.ci-logs/artifacts/*_report/*_report ./.ci-logs/
-          cargo run --bin test-manager format-test-reports ./.ci-logs/*_report \
-            | tee summary.html >> $GITHUB_STEP_SUMMARY
+          ${{ github.workspace }}/bin/test-manager \
+          format-test-reports ${{ github.workspace }}/*_report \
+          | tee summary.html >> $GITHUB_STEP_SUMMARY
       - uses: actions/upload-artifact@v4
         with:
           name: summary.html
-          path: test/summary.html
+          path: summary.html
diff --git a/test/README.md b/test/README.md
index 112edb65d9..66ba136ddf 100644
--- a/test/README.md
+++ b/test/README.md
@@ -107,33 +107,33 @@ Next: build the `test-runner`
 
 ### Building the test runner
 
-Building the `test-runner` binary is done with the `build-runner.sh` script.
+Building the `test-runner` binary is done with the `build/test-runner.sh` script.
 Currently, only `x86_64` platforms are supported for Windows/Linux and `ARM64` (Apple Silicon) for macOS.
 
 For example, building `test-runner` for Windows would look like this:
 
 ``` bash
-./scripts/container-run.sh ./scripts/build-runner.sh windows
+./scripts/container-run.sh ./scripts/build/test-runner.sh windows
 ```
 
 #### Linux
 Using `podman` is the recommended way to build the `test-runner`. See the [Linux section under Prerequisities](#prerequisites) for more details.
 
 ``` bash
-./scripts/container-run.sh ./scripts/build-runner.sh linux
+./scripts/container-run.sh ./scripts/build/test-runner.sh linux
 ```
 
 #### macOS
 
 ``` bash
-./scripts/build-runner.sh macos
+./scripts/build/test-runner.sh macos
 ```
 
 #### Windows
 The `test-runner` binary for Windows may be cross-compiled from a Linux host.
 
 ``` bash
-./scripts/container-run.sh ./scripts/build-runner.sh windows
+./scripts/container-run.sh ./scripts/build/test-runner.sh windows
 ```
 
 ### Running the tests
@@ -164,6 +164,6 @@ cargo run --bin test-manager run-tests --vm macos-ventura \
     --app-package-to-upgrade-from 2023.2
 ```
 
-## Note on `scripts/ci-runtests.sh`
+## Note on `scripts/run/ci.sh`
 
-`scripts/ci-runtests.sh` is the script that GitHub actions uses to invokes the `test-manager`, with similar functionality as `test-by-version.sh`. Note that account numbers are read (newline-delimited) from the path specified by the environment variable `ACCOUNT_TOKENS`. Round robin is used to select an account for each VM.
+`scripts/run/ci.sh` is the script that GitHub actions uses to invokes the `test-manager`, with similar functionality as `test-by-version.sh`. Note that account numbers are read (newline-delimited) from the path specified by the environment variable `ACCOUNT_TOKENS`. Round robin is used to select an account for each VM.
diff --git a/test/scripts/build.sh b/test/scripts/build.sh
index 7cfaede311..99b7b34942 100755
--- a/test/scripts/build.sh
+++ b/test/scripts/build.sh
@@ -13,11 +13,11 @@ REPO_ROOT="$SCRIPT_DIR/../.."
 build_linux() {
     mkdir -p "$TEST_FRAMEWORK_ROOT/dist"
     # Build the test manager
-    "$SCRIPT_DIR/build-manager.sh" linux
+    "$SCRIPT_DIR/build/test-manager.sh" linux
     cp "$TEST_FRAMEWORK_ROOT/target/release/test-manager" "$TEST_FRAMEWORK_ROOT/dist/"
 
     # Build the test runner
-    "$SCRIPT_DIR/build-runner.sh" linux
+    "$SCRIPT_DIR/build/test-runner.sh" linux
     cp "$TEST_FRAMEWORK_ROOT/target/x86_64-unknown-linux-gnu/release/test-runner" "$TEST_FRAMEWORK_ROOT/dist/"
     cp "$TEST_FRAMEWORK_ROOT/target/x86_64-unknown-linux-gnu/release/connection-checker" "$TEST_FRAMEWORK_ROOT/dist/"
 
diff --git a/test/scripts/build-runner-image.sh b/test/scripts/build/runner-image.sh
index 512dc93a4c..47c41b2e38 100755
--- a/test/scripts/build-runner-image.sh
+++ b/test/scripts/build/runner-image.sh
@@ -4,6 +4,9 @@
 
 set -eu
 
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+TEST_FRAMEWORK_ROOT="$SCRIPT_DIR/../.."
+
 TEST_RUNNER_IMAGE_SIZE_MB=5000
 
 case $TARGET in
@@ -20,10 +23,8 @@ echo "************************************************************"
 echo "* Preparing test runner image: $TARGET"
 echo "************************************************************"
 
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
-mkdir -p "${SCRIPT_DIR}/../testrunner-images/"
-TEST_RUNNER_IMAGE_PATH="${SCRIPT_DIR}/../testrunner-images/${TEST_RUNNER_IMAGE_FILENAME}"
+mkdir -p "${TEST_FRAMEWORK_REPO}/testrunner-images"
+TEST_RUNNER_IMAGE_PATH="${TEST_FRAMEWORK_REPO}/testrunner-images/${TEST_RUNNER_IMAGE_FILENAME}"
 
 case $TARGET in
 
@@ -32,8 +33,8 @@ case $TARGET in
         mformat -F -i "${TEST_RUNNER_IMAGE_PATH}" "::"
         mcopy \
             -i "${TEST_RUNNER_IMAGE_PATH}" \
-            "${SCRIPT_DIR}/../target/$TARGET/release/test-runner.exe" \
-            "${SCRIPT_DIR}/../target/$TARGET/release/connection-checker.exe" \
+            "${TEST_FRAMEWORK_ROOT}/target/$TARGET/release/test-runner.exe" \
+            "${TEST_FRAMEWORK_ROOT}/target/$TARGET/release/connection-checker.exe" \
             "${PACKAGE_DIR}/"*.exe \
             "::"
         mdir -i "${TEST_RUNNER_IMAGE_PATH}"
diff --git a/test/scripts/build-manager.sh b/test/scripts/build/test-manager.sh
index 60cf5da98c..517bdf310f 100755
--- a/test/scripts/build-manager.sh
+++ b/test/scripts/build/test-manager.sh
@@ -4,7 +4,7 @@ set -eu
 
 # Build `test-manager`
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-TEST_FRAMEWORK_ROOT="$SCRIPT_DIR/.."
+TEST_FRAMEWORK_ROOT="$SCRIPT_DIR/../.."
 REPO_DIR="$TEST_FRAMEWORK_ROOT/.."
 
 # shellcheck disable=SC1091
diff --git a/test/scripts/build-runner.sh b/test/scripts/build/test-runner.sh
index ffdb2b861a..05d2132679 100755
--- a/test/scripts/build-runner.sh
+++ b/test/scripts/build/test-runner.sh
@@ -3,8 +3,10 @@
 set -eu
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-REPO_DIR="$SCRIPT_DIR/../.."
-cd "$SCRIPT_DIR"
+TEST_FRAMEWORK_ROOT="$SCRIPT_DIR/../.."
+REPO_DIR="$TEST_FRAMEWORK_ROOT/.."
+
+pushd "$SCRIPT_DIR"
 
 # shellcheck disable=SC1091
 source "$REPO_DIR/scripts/utils/log"
@@ -35,5 +37,22 @@ cargo build \
 
 # Only build runner image for Windows
 if [[ $TARGET == x86_64-pc-windows-gnu ]]; then
-    TARGET="$TARGET" ./build-runner-image.sh
+    TARGET="$TARGET" ./runner-image.sh
 fi
+
+popd
+
+while [[ "$#" -gt 0 ]]; do
+    case $1 in
+        # Optionally move binaries to some known location
+        --output)
+            ARTIFACTS_DIR="$TEST_FRAMEWORK_ROOT/target/$TARGET/release"
+            mv -t "$1" "$ARTIFACTS_DIR/test-runner" "$ARTIFACTS_DIR/connection-checker"
+            ;;
+        *)
+            log_error "Unknown parameter: $1"
+            exit 1
+            ;;
+    esac
+    shift
+done
diff --git a/test/scripts/ci-runtests.sh b/test/scripts/run/ci.sh
index 88a8944164..b1aff3a6e4 100755
--- a/test/scripts/ci-runtests.sh
+++ b/test/scripts/run/ci.sh
@@ -1,11 +1,17 @@
 #!/usr/bin/env bash
 
+# TODO: Break this down into multiple, smaller scripts and compose them in this file.
+
 set -eu
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+TEST_FRAMEWORK_ROOT="$SCRIPT_DIR/../.."
+TEST_DIR="$TEST_FRAMEWORK_ROOT"
+
 cd "$SCRIPT_DIR"
-TEST_DIR="$SCRIPT_DIR/.."
 
+# Parse arguments
+# TODO: Add support for either passing in --account-tokens or reading from env variable.
 if [[ "$#" -lt 1 ]]; then
     echo "usage: $0 TEST_OS" 1>&2
     exit 1
@@ -19,15 +25,17 @@ if [[ "$(uname -s)" == "Darwin" ]]; then
     rustup update
 fi
 
-# shellcheck source=test/scripts/test-utils.sh
-source "test-utils.sh"
+# shellcheck source=test/scripts/utils/lib.sh
+source "../utils/lib.sh"
+# shellcheck source=test/scripts/utils/download.sh
+source "../utils/download.sh" # TODO: Do not source it, call it instead.
 
 echo "**********************************"
 echo "* Version to upgrade from: $LATEST_STABLE_RELEASE"
 echo "* Version to test: $CURRENT_VERSION"
 echo "**********************************"
 
-
+# TODO: Add support for either passing in --account-tokens or reading from env variable.
 if [[ -z "${ACCOUNT_TOKENS+x}" ]]; then
     echo "'ACCOUNT_TOKENS' must be specified" 1>&2
     exit 1
@@ -36,11 +44,15 @@ if ! readarray -t tokens < "${ACCOUNT_TOKENS}"; then
     echo "Specify account numbers in 'ACCOUNT_TOKENS' file" 1>&2
     exit 1
 fi
+
+# TODO: Can we get rid of this? Seemse excessive / leaves a trail
 CI_LOGS_DIR="$TEST_DIR/.ci-logs"
 mkdir -p "$CI_LOGS_DIR"
 echo "$CURRENT_VERSION" > "$CI_LOGS_DIR/last-version.log"
 
 
+# TODO: This should def be it's own step in the GitHub actions workflow
+
 echo "**********************************"
 echo "* Downloading app packages"
 echo "**********************************"
@@ -50,16 +62,14 @@ nice_time download_app_package "$LATEST_STABLE_RELEASE" "$TEST_OS"
 nice_time download_app_package "$CURRENT_VERSION" "$TEST_OS"
 nice_time download_e2e_executable "$CURRENT_VERSION" "$TEST_OS"
 
-echo "**********************************"
-echo "* Building test manager"
-echo "**********************************"
-
-cargo build -p test-manager
+# TODO: This should def be it's own step in the GitHub actions workflow
 
 echo "**********************************"
 echo "* Running tests"
 echo "**********************************"
 
+# TODO: Should we really care about logging in this script?
+
 mkdir -p "$CI_LOGS_DIR/os/"
 export TEST_REPORT="$CI_LOGS_DIR/${TEST_OS}_report"
 rm -f "$TEST_REPORT"
diff --git a/test/scripts/utils/download.sh b/test/scripts/utils/download.sh
new file mode 100755
index 0000000000..b3e73d579c
--- /dev/null
+++ b/test/scripts/utils/download.sh
@@ -0,0 +1,219 @@
+#!/usr/bin/env bash
+
+set -eu
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+TEST_FRAMEWORK_ROOT="$SCRIPT_DIR/../.."
+REPO_ROOT="$TEST_FRAMEWORK_ROOT/.."
+
+export BUILD_RELEASE_REPOSITORY="https://releases.mullvad.net/desktop/releases"
+export BUILD_DEV_REPOSITORY="https://releases.mullvad.net/desktop/builds"
+
+function executable_not_found_in_dist_error {
+    1>&2 echo "Executable \"$1\" not found in specified dist dir. Exiting."
+    exit 1
+}
+
+# Returns the directory of the lib.sh script
+function get_test_utils_dir {
+    echo "$SCRIPT_DIR"
+}
+
+# Infer stable version from GitHub repo
+RELEASES=$(curl -sf https://api.github.com/repos/mullvad/mullvadvpn-app/releases | jq -r '[.[] | select(((.tag_name|(startswith("android") or startswith("ios"))) | not))]')
+LATEST_STABLE_RELEASE=$(jq -r '[.[] | select(.prerelease==false)] | .[0].tag_name' <<<"$RELEASES")
+
+function get_current_version {
+    local app_dir
+    app_dir="$REPO_ROOT"
+    if [ -n "${TEST_DIST_DIR+x}" ]; then
+        if [ ! -x "${TEST_DIST_DIR%/}/mullvad-version" ]; then
+            executable_not_found_in_dist_error mullvad-version
+        fi
+        "${TEST_DIST_DIR%/}/mullvad-version"
+    else
+        cargo run -q --manifest-path="$app_dir/Cargo.toml" --bin mullvad-version
+    fi
+}
+
+CURRENT_VERSION=$(get_current_version)
+commit=$(git rev-parse HEAD^\{commit\})
+commit=${commit:0:6}
+
+TAG=$(git describe --exact-match HEAD 2>/dev/null || echo "")
+
+if [[ -n "$TAG" && ${CURRENT_VERSION} =~ -dev- ]]; then
+    # Remove disallowed version characters from the tag
+    CURRENT_VERSION+="+${TAG//[^0-9a-z_-]/}"
+fi
+
+export CURRENT_VERSION
+export LATEST_STABLE_RELEASE
+
+function print_available_releases {
+    for release in $(jq -r '.[].tag_name' <<<"$RELEASES"); do
+        echo "$release"
+    done
+}
+
+function get_package_dir {
+    local package_dir
+    if [[ -n "${PACKAGE_DIR+x}" ]]; then
+        # Resolve the package dir to an absolute path since cargo must be invoked from the test directory
+        package_dir=$(realpath "$PACKAGE_DIR")
+    elif [[ ("$(uname -s)" == "Darwin") ]]; then
+        package_dir="$HOME/Library/Caches/mullvad-test/packages"
+    elif [[ ("$(uname -s)" == "Linux") ]]; then
+        package_dir="$HOME/.cache/mullvad-test/packages"
+    else
+        echo "Unsupported OS" 1>&2
+        exit 1
+    fi
+
+    mkdir -p "$package_dir" || exit 1
+    # Clean up old packages
+    find "$package_dir" -type f -mtime +5 -delete || true
+
+    echo "$package_dir"
+    return 0
+}
+
+function nice_time {
+    SECONDS=0
+    if "$@"; then
+        result=0
+    else
+        result=$?
+    fi
+    s=$SECONDS
+    echo "\"$*\" completed in $((s / 60))m:$((s % 60))s"
+    return $result
+}
+# Matches $1 with a build version string and sets the following exported variables:
+# - BUILD_VERSION: The version part of the build string (e.g., "2024.3-beta1-dev-").
+# - COMMIT_HASH: The commit hash part of the build string (e.g., "abcdef").
+# - TAG: The tag part of the build string (e.g., "+tag").
+function parse_build_version {
+    if [[ "$1" =~ (^[0-9.]+(-beta[0-9]+)?-dev-)([0-9a-z]+)(\+[0-9a-z|-]+)?$ ]]; then
+        BUILD_VERSION="${BASH_REMATCH[1]}"
+        COMMIT_HASH="${BASH_REMATCH[3]}"
+        TAG="${BASH_REMATCH[4]}"
+        return 0
+    fi
+    return 1
+}
+
+# Returns 0 if $1 is a development build.
+function is_dev_version {
+    if [[ "$1" == *"-dev-"* ]]; then
+        return 0
+    fi
+    return 1
+}
+
+function get_app_filename {
+    local version=$1
+    local os=$2
+    if is_dev_version "$version"; then
+        parse_build_version "$version"
+        version="${BUILD_VERSION}${COMMIT_HASH}${TAG:-}"
+    fi
+    case $os in
+    debian* | ubuntu*)
+        echo "MullvadVPN-${version}_amd64.deb"
+        ;;
+    fedora*)
+        echo "MullvadVPN-${version}_x86_64.rpm"
+        ;;
+    windows*)
+        echo "MullvadVPN-${version}.exe"
+        ;;
+    macos*)
+        echo "MullvadVPN-${version}.pkg"
+        ;;
+    *)
+        echo "Unsupported target: $os" 1>&2
+        return 1
+        ;;
+    esac
+}
+
+function download_app_package {
+    local version=$1
+    local os=$2
+    local package_repo=""
+
+    if is_dev_version "$version"; then
+        package_repo="${BUILD_DEV_REPOSITORY}"
+    else
+        package_repo="${BUILD_RELEASE_REPOSITORY}"
+    fi
+
+    local filename
+    filename=$(get_app_filename "$version" "$os")
+    local url="${package_repo}/$version/$filename"
+
+    local package_dir
+    package_dir=$(get_package_dir)
+    if [[ ! -f "$package_dir/$filename" ]]; then
+        echo "Downloading build for $version ($os) from $url"
+        if ! curl -sf -o "$package_dir/$filename" "$url"; then
+            echo "Failed to download package from $url (hint: build may not exist, check the url)" 1>&2
+            exit 1
+        fi
+    else
+        echo "App package for version $version ($os) already exists at $package_dir/$filename, skipping download"
+    fi
+}
+
+function get_e2e_filename {
+    local version=$1
+    local os=$2
+    if is_dev_version "$version"; then
+        parse_build_version "$version"
+        version="${BUILD_VERSION}${COMMIT_HASH}"
+    fi
+    case $os in
+    debian* | ubuntu* | fedora*)
+        echo "app-e2e-tests-${version}-x86_64-unknown-linux-gnu"
+        ;;
+    windows*)
+        echo "app-e2e-tests-${version}-x86_64-pc-windows-msvc.exe"
+        ;;
+    macos*)
+        echo "app-e2e-tests-${version}-aarch64-apple-darwin"
+        ;;
+    *)
+        echo "Unsupported target: $os" 1>&2
+        return 1
+        ;;
+    esac
+}
+
+function download_e2e_executable {
+    local version=${1:?Error: version not set}
+    local os=${2:?Error: os not set}
+    local package_repo
+
+    if is_dev_version "$version"; then
+        package_repo="${BUILD_DEV_REPOSITORY}"
+    else
+        package_repo="${BUILD_RELEASE_REPOSITORY}"
+    fi
+
+    local filename
+    filename=$(get_e2e_filename "$version" "$os")
+    local url="${package_repo}/$version/additional-files/$filename"
+
+    local package_dir
+    package_dir=$(get_package_dir)
+    if [[ ! -f "$package_dir/$filename" ]]; then
+        echo "Downloading e2e executable for $version ($os) from $url"
+        if ! curl -sf -o "$package_dir/$filename" "$url"; then
+            echo "Failed to download package from $url (hint: build may not exist, check the url)" 1>&2
+            exit 1
+        fi
+    else
+        echo "GUI e2e executable for version $version ($os) already exists at $package_dir/$filename, skipping download"
+    fi
+}
diff --git a/test/scripts/test-utils.sh b/test/scripts/utils/lib.sh
index be826a6ef0..dd1630dc33 100755
--- a/test/scripts/test-utils.sh
+++ b/test/scripts/utils/lib.sh
@@ -2,33 +2,30 @@
 
 set -eu
 
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+TEST_FRAMEWORK_ROOT="$SCRIPT_DIR/../.."
+REPO_ROOT="$TEST_FRAMEWORK_ROOT/.."
+
+export BUILD_RELEASE_REPOSITORY="https://releases.mullvad.net/desktop/releases"
+export BUILD_DEV_REPOSITORY="https://releases.mullvad.net/desktop/builds"
+
 function executable_not_found_in_dist_error {
     1>&2 echo "Executable \"$1\" not found in specified dist dir. Exiting."
     exit 1
 }
 
-# Returns the directory of the test-utils.sh script
-function get_test_utls_dir {
-    local script_path="${BASH_SOURCE[0]}"
-    local script_dir
-    if [[ -n "$script_path" ]]; then
-        script_dir="$(cd "$(dirname "$script_path")" >/dev/null && pwd)"
-    else
-        script_dir="$(cd "$(dirname "$0")" >/dev/null && pwd)"
-    fi
-    echo "$script_dir"
+# Returns the directory of the lib.sh script
+function get_test_utils_dir {
+    echo "$SCRIPT_DIR"
 }
 
-export BUILD_RELEASE_REPOSITORY="https://releases.mullvad.net/desktop/releases"
-export BUILD_DEV_REPOSITORY="https://releases.mullvad.net/desktop/builds"
-
 # Infer stable version from GitHub repo
 RELEASES=$(curl -sf https://api.github.com/repos/mullvad/mullvadvpn-app/releases | jq -r '[.[] | select(((.tag_name|(startswith("android") or startswith("ios"))) | not))]')
 LATEST_STABLE_RELEASE=$(jq -r '[.[] | select(.prerelease==false)] | .[0].tag_name' <<<"$RELEASES")
 
 function get_current_version {
     local app_dir
-    app_dir="$(get_test_utls_dir)/../.."
+    app_dir="$REPO_ROOT"
     if [ -n "${TEST_DIST_DIR+x}" ]; then
         if [ ! -x "${TEST_DIST_DIR%/}/mullvad-version" ]; then
             executable_not_found_in_dist_error mullvad-version
@@ -114,25 +111,22 @@ function is_dev_version {
     return 1
 }
 
-function get_app_filename {
+function get_e2e_filename {
     local version=$1
     local os=$2
     if is_dev_version "$version"; then
         parse_build_version "$version"
-        version="${BUILD_VERSION}${COMMIT_HASH}${TAG:-}"
+        version="${BUILD_VERSION}${COMMIT_HASH}"
     fi
     case $os in
-    debian* | ubuntu*)
-        echo "MullvadVPN-${version}_amd64.deb"
-        ;;
-    fedora*)
-        echo "MullvadVPN-${version}_x86_64.rpm"
+    debian* | ubuntu* | fedora*)
+        echo "app-e2e-tests-${version}-x86_64-unknown-linux-gnu"
         ;;
     windows*)
-        echo "MullvadVPN-${version}.exe"
+        echo "app-e2e-tests-${version}-x86_64-pc-windows-msvc.exe"
         ;;
     macos*)
-        echo "MullvadVPN-${version}.pkg"
+        echo "app-e2e-tests-${version}-aarch64-apple-darwin"
         ;;
     *)
         echo "Unsupported target: $os" 1>&2
@@ -141,50 +135,25 @@ function get_app_filename {
     esac
 }
 
-function download_app_package {
-    local version=$1
-    local os=$2
-    local package_repo=""
-
-    if is_dev_version "$version"; then
-        package_repo="${BUILD_DEV_REPOSITORY}"
-    else
-        package_repo="${BUILD_RELEASE_REPOSITORY}"
-    fi
-
-    local filename
-    filename=$(get_app_filename "$version" "$os")
-    local url="${package_repo}/$version/$filename"
-
-    local package_dir
-    package_dir=$(get_package_dir)
-    if [[ ! -f "$package_dir/$filename" ]]; then
-        echo "Downloading build for $version ($os) from $url"
-        if ! curl -sf -o "$package_dir/$filename" "$url"; then
-            echo "Failed to download package from $url (hint: build may not exist, check the url)" 1>&2
-            exit 1
-        fi
-    else
-        echo "App package for version $version ($os) already exists at $package_dir/$filename, skipping download"
-    fi
-}
-
-function get_e2e_filename {
+function get_app_filename {
     local version=$1
     local os=$2
     if is_dev_version "$version"; then
         parse_build_version "$version"
-        version="${BUILD_VERSION}${COMMIT_HASH}"
+        version="${BUILD_VERSION}${COMMIT_HASH}${TAG:-}"
     fi
     case $os in
-    debian* | ubuntu* | fedora*)
-        echo "app-e2e-tests-${version}-x86_64-unknown-linux-gnu"
+    debian* | ubuntu*)
+        echo "MullvadVPN-${version}_amd64.deb"
+        ;;
+    fedora*)
+        echo "MullvadVPN-${version}_x86_64.rpm"
         ;;
     windows*)
-        echo "app-e2e-tests-${version}-x86_64-pc-windows-msvc.exe"
+        echo "MullvadVPN-${version}.exe"
         ;;
     macos*)
-        echo "app-e2e-tests-${version}-aarch64-apple-darwin"
+        echo "MullvadVPN-${version}.pkg"
         ;;
     *)
         echo "Unsupported target: $os" 1>&2
@@ -193,44 +162,16 @@ function get_e2e_filename {
     esac
 }
 
-function download_e2e_executable {
-    local version=${1:?Error: version not set}
-    local os=${2:?Error: os not set}
-    local package_repo
-
-    if is_dev_version "$version"; then
-        package_repo="${BUILD_DEV_REPOSITORY}"
-    else
-        package_repo="${BUILD_RELEASE_REPOSITORY}"
-    fi
-
-    local filename
-    filename=$(get_e2e_filename "$version" "$os")
-    local url="${package_repo}/$version/additional-files/$filename"
-
-    local package_dir
-    package_dir=$(get_package_dir)
-    if [[ ! -f "$package_dir/$filename" ]]; then
-        echo "Downloading e2e executable for $version ($os) from $url"
-        if ! curl -sf -o "$package_dir/$filename" "$url"; then
-            echo "Failed to download package from $url (hint: build may not exist, check the url)" 1>&2
-            exit 1
-        fi
-    else
-        echo "GUI e2e executable for version $version ($os) already exists at $package_dir/$filename, skipping download"
-    fi
-}
-
 function build_test_runner {
     local script_dir
-    script_dir=$(get_test_utls_dir)
+    script_dir="$(get_test_utils_dir)/../"
     local test_os=${1:?Error: test os not set}
     if [[ "${test_os}" =~ "debian"|"ubuntu"|"fedora" ]]; then
-        "$script_dir"/container-run.sh scripts/build-runner.sh linux || exit 1
+        "$script_dir"/container-run.sh scripts/build/test-runner.sh linux || exit 1
     elif [[ "${test_os}" =~ "windows" ]]; then
-        "$script_dir"/container-run.sh scripts/build-runner.sh windows || exit 1
+        "$script_dir"/container-run.sh scripts/build/test-runner.sh windows || exit 1
     elif [[ "${test_os}" =~ "macos" ]]; then
-        "$script_dir"/build-runner.sh macos || exit 1
+        "$script_dir"/build/test-runner.sh macos || exit 1
     fi
 }
 
@@ -279,7 +220,7 @@ function run_tests_for_os {
     local package_dir
     package_dir=$(get_package_dir)
     local test_dir
-    test_dir=$(get_test_utls_dir)/..
+    test_dir=$(get_test_utils_dir)/../..
     read -ra test_filters_arg <<<"${TEST_FILTERS:-}" # Split the string by words into an array
     pushd "$test_dir"
     if [ -n "${TEST_DIST_DIR+x}" ]; then
@@ -289,7 +230,8 @@ function run_tests_for_os {
         test_manager="${TEST_DIST_DIR%/}/test-manager"
         runner_dir_flag=("--runner-dir" "$TEST_DIST_DIR")
     else
-        test_manager="cargo run --bin test-manager"
+        # Assume test-manager is in path.
+        test_manager="$(which test-manager)"
         runner_dir_flag=()
     fi
 
@@ -321,7 +263,7 @@ function run_tests_for_os {
 # Currently unused, but may be useful in the future
 function build_current_version {
     local app_dir
-    app_dir="$(get_test_utls_dir)/../.."
+    app_dir="$REPO_ROOT"
     local app_filename
     # TODO: TEST_OS must be set to local OS manually, should be set automatically
     app_filename=$(get_app_filename "$CURRENT_VERSION" "${TEST_OS:?Error: TEST_OS not set}")
diff --git a/test/test-by-version.sh b/test/test-by-version.sh
index f409a214cc..3b97d45461 100755
--- a/test/test-by-version.sh
+++ b/test/test-by-version.sh
@@ -21,8 +21,8 @@ usage() {
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 cd "$SCRIPT_DIR"
 
-# shellcheck source=test/scripts/test-utils.sh
-source "scripts/test-utils.sh"
+# shellcheck source=test/scripts/utils/lib.sh
+source "scripts/utils/lib.sh"
 
 if [[ ("$*" == "--help") || "$*" == "-h" ]]; then
     usage
diff --git a/test/test-manager/src/main.rs b/test/test-manager/src/main.rs
index 2a043b4276..c64d8d4165 100644
--- a/test/test-manager/src/main.rs
+++ b/test/test-manager/src/main.rs
@@ -183,25 +183,12 @@ enum VmConfig {
     List,
 }
 
-#[cfg(target_os = "linux")]
-impl Args {
-    fn get_vnc_port(&self) -> Option<u16> {
-        match self.cmd {
-            Commands::RunTests { vnc, .. } | Commands::RunVm { vnc, .. } => vnc,
-            _ => None,
-        }
-    }
-}
-
 #[tokio::main]
 async fn main() -> Result<()> {
     logging::Logger::get_or_init();
 
     let args = Args::parse();
 
-    #[cfg(target_os = "linux")]
-    container::relaunch_with_rootlesskit(args.get_vnc_port()).await;
-
     let mut config = config::ConfigFile::load_or_default()
         .await
         .context("Failed to load config")?;
@@ -250,11 +237,29 @@ async fn main() -> Result<()> {
                 }
             },
         },
+        Commands::ListTests => {
+            println!("priority\tname");
+            for test in tests::get_test_descriptions() {
+                println!(
+                    "{priority:8}\t{name}",
+                    name = test.name,
+                    priority = test.priority.unwrap_or(0),
+                );
+            }
+            Ok(())
+        }
+        Commands::FormatTestReports { reports } => {
+            summary::print_summary_table(&reports).await;
+            Ok(())
+        }
         Commands::RunVm {
             vm,
             vnc,
             keep_changes,
         } => {
+            #[cfg(target_os = "linux")]
+            container::relaunch_with_rootlesskit(vnc).await;
+
             let mut config = config.clone();
             config.runtime_opts.keep_changes = keep_changes;
             config.runtime_opts.display = if vnc.is_some() {
@@ -269,17 +274,6 @@ async fn main() -> Result<()> {
 
             Ok(())
         }
-        Commands::ListTests => {
-            println!("priority\tname");
-            for test in tests::get_test_descriptions() {
-                println!(
-                    "{priority:8}\t{name}",
-                    name = test.name,
-                    priority = test.priority.unwrap_or(0),
-                );
-            }
-            Ok(())
-        }
         Commands::RunTests {
             vm,
             display,
@@ -296,6 +290,9 @@ async fn main() -> Result<()> {
             test_report,
             runner_dir,
         } => {
+            #[cfg(target_os = "linux")]
+            container::relaunch_with_rootlesskit(vnc).await;
+
             let mut config = config.clone();
             config.runtime_opts.display = match (display, vnc.is_some()) {
                 (false, false) => config::Display::None,
@@ -410,10 +407,6 @@ async fn main() -> Result<()> {
             // Propagate any error from the test run if applicable
             result?.anyhow()
         }
-        Commands::FormatTestReports { reports } => {
-            summary::print_summary_table(&reports).await;
-            Ok(())
-        }
         Commands::Update { name } => {
             let vm_config = vm::get_vm_config(&config, &name).context("Cannot get VM config")?;