Merge branch 'test-manager-fixes-misc'

24 files changed, 703 insertions, 403 deletions

diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml
index 03e2959be1..23a612f9ac 100644
--- a/.github/workflows/desktop-e2e.yml
+++ b/.github/workflows/desktop-e2e.yml
@@ -151,7 +151,7 @@ jobs:
         shell: bash -ieo pipefail {0}
         run: |
           git fetch --tags --force
-          ./test/ci-runtests.sh ${{ matrix.os }}
+          ./test/scripts/ci-runtests.sh ${{ matrix.os }}
       - uses: actions/upload-artifact@v3
         if: '!cancelled()'
         with:
@@ -226,7 +226,7 @@ jobs:
         shell: bash -ieo pipefail {0}
         run: |
           git fetch --tags --force
-          ./test/ci-runtests.sh ${{ matrix.os }}
+          ./test/scripts/ci-runtests.sh ${{ matrix.os }}
       - uses: actions/upload-artifact@v3
         if: '!cancelled()'
         with:
@@ -297,7 +297,7 @@ jobs:
         shell: bash -ieo pipefail {0}
         run: |
           git fetch --tags --force
-          ./test/ci-runtests.sh ${{ matrix.os }}
+          ./test/scripts/ci-runtests.sh ${{ matrix.os }}
       - uses: actions/upload-artifact@v3
         if: '!cancelled()'
         with:
diff --git a/test/README.md b/test/README.md
index b490166ef5..6aaff56d2f 100644
--- a/test/README.md
+++ b/test/README.md
@@ -1,31 +1,34 @@
-# Project structure
+# Mullvad VPN end to end test framework
 
-## test-manager
+## Project structure
+
+### test-manager
 
 The client part of the testing environment. This program runs on the host and connects over a
 virtual serial port to the `test-runner`.
 
 The tests themselves are defined in this package, using the interface provided by `test-runner`.
 
-## test-runner
+### test-runner
 
 The server part of the testing environment. This program runs in guest VMs and provides the
 `test-manager` with the building blocks (RPCs) needed to create tests.
 
-## test-rpc
+### test-rpc
 
 A support library for the other two packages. Defines an RPC interface, transports, shared types,
 etc.
 
-# Prerequisities
+
+## Prerequisites
 
 For macOS, the host machine must be macOS. All other platforms assume that the host is Linux.
 
-## All platforms
+### All platforms
 
 * Get the latest stable Rust from https://rustup.rs/.
 
-## macOS
+### macOS
 
 Normally, you would use Tart here. It can be installed with Homebrew. You'll also need
 `wireguard-tools`, a protobuf compiler, and OpenSSL:
@@ -34,7 +37,7 @@ Normally, you would use Tart here. It can be installed with Homebrew. You'll als
 brew install cirruslabs/cli/tart wireguard-tools pkg-config openssl protobuf
 ```
 
-### Wireshark
+#### Wireshark
 
 Wireshark is also required. More specifically, you'll need `wireshark-chmodbpf`, which can be found
 in the Wireshark installer here: https://www.wireshark.org/download.html
@@ -47,7 +50,7 @@ dseditgroup -o edit -a THISUSER -t user access_bpf
 
 This lets us monitor traffic on network interfaces without root access.
 
-## Linux
+### Linux
 
 For running tests on Linux and Windows guests, you will need these tools and libraries:
 
@@ -58,7 +61,51 @@ dnf install git gcc protobuf-devel libpcap-devel qemu \
     wireguard-tools
 ```
 
-# Building the test runner
+## Setting up testing environment
+
+First you need to build the images for running tests on, see [`BUILD_OS_IMAGE.md`](./docs/BUILD_OS_IMAGE.md). The `test-manager` then needs to be configured to use the image.
+
+Here is an example of how to create a new OS configuration for Linux and macOS:
+
+### Linux
+
+```bash
+# Create or edit configuration
+# The image is assumed to contain a test runner service set up as described in ./docs/BUILD_OS_IMAGE.md
+cargo run --bin test-manager set debian11 qemu ./os-images/debian11.qcow2 linux \
+    --package-type deb --architecture x64 \
+    --provisioner ssh --ssh-user test --ssh-password test
+
+# Try it out to see if it works - you should reach the VM's graphical desktop environment
+cargo run --bin test-manager run-vm debian11
+```
+
+### macOS
+
+
+```bash
+# Download some VM image
+tart clone ghcr.io/cirruslabs/macos-ventura-base:latest ventura-base
+
+# Create or edit configuration
+# Use SSH to deploy the test runner since the image doesn't contain a runner
+cargo run --bin test-manager set macos-ventura tart ventura-base macos \
+    --architecture aarch64 \
+    --provisioner ssh --ssh-user admin --ssh-password admin
+
+# Try it out to see if it works
+cargo run -p test-manager run-vm macos-ventura
+```
+
+## Testing the app
+
+To automatically download and test a pre-built version of the app, use the `test-by-version.sh` script, see `test-by-version.sh --help` for instructions.
+
+To manually invoke `test-manager`, start by checking out the desired git version of this repo. Next, [build the app](../BuildInstructions.md) for the target platform then build the GUI test binary using `$(cd ../gui && npm run build-test-executable)`. The newly built packages will be located in the `../dist` folder by default.
+
+Next: build the `test-runner`
+
+### Building the test runner
 
 Building the `test-runner` binary is done with the `build-runner.sh` script.
 Currently, only `x86_64` platforms are supported for Windows/Linux and `ARM64` (Apple Silicon) for macOS.
@@ -66,86 +113,57 @@ Currently, only `x86_64` platforms are supported for Windows/Linux and `ARM64` (
 For example, building `test-runner` for Windows would look like this:
 
 ``` bash
-./container-run.sh ./build-runner.sh windows
+./scripts/container-run.sh ./scripts/build-runner.sh windows
 ```
 
-## Linux
-Using `podman` is the recommended way to build the `test-runner`. See the [Linux section under Prerequisities](#Prerequisities) for more details.
+#### Linux
+Using `podman` is the recommended way to build the `test-runner`. See the [Linux section under Prerequisities](#prerequisites) for more details.
 
 ``` bash
-./container-run.sh ./build-runner.sh linux
+./scripts/container-run.sh ./scripts/build-runner.sh linux
 ```
 
-## macOS
+#### macOS
 
 ``` bash
-./build-runner.sh macos
+./scripts/build-runner.sh macos
 ```
 
-## Windows
+#### Windows
 The `test-runner` binary for Windows may be cross-compiled from a Linux host.
 
 ``` bash
-./container-run.sh ./build-runner.sh windows
+./scripts/container-run.sh ./scripts/build-runner.sh windows
 ```
 
-# Building base images
-
-See [`BUILD_OS_IMAGE.md`](./docs/BUILD_OS_IMAGE.md) for how to build images for running tests on.
-
-# Running tests
+### Running the tests
 
-See `cargo run --bin test-manager` for details.
+After configuring the VM image using `test-manager set` and building the required packages (see [previous step](#setting-up-testing-environment)), `test-manager run-tests` is used to launch the tests. See `cargo run --bin test-manager -- run-tests --help` for details.
 
-## Linux
+Here is an example of how to run all tests using the Linux/macOS VM we set up earlier:
 
-Here is an example of how to create a new OS configuration and then run all tests:
+#### Linux
 
 ```bash
-# Create or edit configuration
-# The image is assumed to contain a test runner service set up as described in ./docs/BUILD_OS_IMAGE.md
-cargo run --bin test-manager set debian11 qemu ./os-images/debian11.qcow2 linux \
-    --package-type deb --architecture x64 \
-    --provisioner ssh --ssh-user test --ssh-password test
-
-# Try it out to see if it works - you should reach the VM's graphical desktop environment
-cargo run --bin test-manager run-vm debian11
-
 # Run all tests
-cargo run --bin test-manager run-tests debian11 \
+cargo run --bin test-manager run-tests --vm debian11 \
     --display \
     --account 0123456789 \
     --app-package <git hash or tag> \
     --app-package-to-upgrade-from 2023.2
 ```
 
-## macOS
-
-Here is an example of how to create a new OS configuration (on Apple Silicon) and then run all
-tests:
+#### macOS
 
 ```bash
-# Download some VM image
-tart clone ghcr.io/cirruslabs/macos-ventura-base:latest ventura-base
-
-# Create or edit configuration
-# Use SSH to deploy the test runner since the image doesn't contain a runner
-cargo run --bin test-manager set macos-ventura tart ventura-base macos \
-    --architecture aarch64 \
-    --provisioner ssh --ssh-user admin --ssh-password admin
-
-# Try it out to see if it works
-#cargo run -p test-manager run-vm macos-ventura
-
 # Run all tests
-cargo run --bin test-manager run-tests macos-ventura \
+cargo run --bin test-manager run-tests --vm macos-ventura \
     --display \
     --account 0123456789 \
     --app-package <git hash or tag> \
     --app-package-to-upgrade-from 2023.2
 ```
 
-## Note on `ci-runtests.sh`
+## Note on `scripts/ci-runtests.sh`
 
-Account tokens are read (newline-delimited) from the path specified by the environment variable
-`ACCOUNT_TOKENS`. Round robin is used to select an account for each VM.
+`scripts/ci-runtests.sh` is the script that GitHub actions uses to invokes the `test-manager`, with similar functionality as `test-by-version.sh`. Note that account tokens are read (newline-delimited) from the path specified by the environment variable `ACCOUNT_TOKENS`. Round robin is used to select an account for each VM.
diff --git a/test/ci-runtests.sh b/test/ci-runtests.sh
deleted file mode 100755
index e6b7ba7b18..0000000000
--- a/test/ci-runtests.sh
+++ /dev/null
@@ -1,246 +0,0 @@
-#!/usr/bin/env bash
-
-set -eu
-
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-APP_DIR="$SCRIPT_DIR/../"
-cd "$SCRIPT_DIR"
-
-BUILD_RELEASE_REPOSITORY="https://releases.mullvad.net/desktop/releases"
-BUILD_DEV_REPOSITORY="https://releases.mullvad.net/desktop/builds"
-
-if [[ ("$(uname -s)" == "Darwin") ]]; then
-    export PACKAGES_DIR=$HOME/Library/Caches/mullvad-test/packages
-elif [[ ("$(uname -s)" == "Linux") ]]; then
-    export PACKAGES_DIR=$HOME/.cache/mullvad-test/packages
-else
-    echo "Unsupported OS" 1>&2
-    exit 1
-fi
-
-if [[ "$#" -lt 1 ]]; then
-    echo "usage: $0 TEST_OS" 1>&2
-    exit 1
-fi
-
-TEST_OS=$1
-
-# Infer stable version from GitHub repo
-RELEASES=$(curl -sf https://api.github.com/repos/mullvad/mullvadvpn-app/releases | jq -r '[.[] | select(((.tag_name|(startswith("android") or startswith("ios"))) | not))]')
-OLD_APP_VERSION=$(jq -r '[.[] | select(.prerelease==false)] | .[0].tag_name' <<<"$RELEASES")
-
-NEW_APP_VERSION=$(cargo run -q --manifest-path="$APP_DIR/Cargo.toml" --bin mullvad-version)
-commit=$(git rev-parse HEAD^\{commit\})
-commit=${commit:0:6}
-
-TAG=$(git describe --exact-match HEAD 2>/dev/null || echo "")
-
-if [[ -n "$TAG" && ${NEW_APP_VERSION} =~ -dev- ]]; then
-    NEW_APP_VERSION+="+${TAG}"
-fi
-
-echo "**********************************"
-echo "* Version to upgrade from: $OLD_APP_VERSION"
-echo "* Version to test: $NEW_APP_VERSION"
-echo "**********************************"
-
-
-if [[ -z "${ACCOUNT_TOKENS+x}" ]]; then
-    echo "'ACCOUNT_TOKENS' must be specified" 1>&2
-    exit 1
-fi
-if ! readarray -t tokens < "${ACCOUNT_TOKENS}"; then
-    echo "Specify account tokens in 'ACCOUNT_TOKENS' file" 1>&2
-    exit 1
-fi
-
-mkdir -p "$SCRIPT_DIR/.ci-logs"
-echo "$NEW_APP_VERSION" > "$SCRIPT_DIR/.ci-logs/last-version.log"
-
-function nice_time {
-    SECONDS=0
-    if "$@"; then
-        result=0
-    else
-        result=$?
-    fi
-    s=$SECONDS
-    echo "\"$*\" completed in $((s/60))m:$((s%60))s"
-    return $result
-}
-
-# Returns 0 if $1 is a development build. `BASH_REMATCH` contains match groups
-# if that is the case.
-function is_dev_version {
-    local pattern="(^[0-9.]+(-beta[0-9]+)?-dev-)([0-9a-z]+)(\+[0-9a-z|-]+)?$"
-    if [[ "$1" =~ $pattern ]]; then
-        return 0
-    fi
-    return 1
-}
-
-function get_app_filename {
-    local version=$1
-    local os=$2
-    if is_dev_version "$version"; then
-        # only save 6 chars of the hash
-        local commit="${BASH_REMATCH[3]}"
-        version="${BASH_REMATCH[1]}${commit}"
-        # If the dev-version includes a tag, we need to append it to the app filename
-        if [[ -n ${BASH_REMATCH[4]} ]]; then
-            version="${version}${BASH_REMATCH[4]}"
-        fi
-    fi
-    case $os in
-        debian*|ubuntu*)
-            echo "MullvadVPN-${version}_amd64.deb"
-            ;;
-        fedora*)
-            echo "MullvadVPN-${version}_x86_64.rpm"
-            ;;
-        windows*)
-            echo "MullvadVPN-${version}.exe"
-            ;;
-        macos*)
-            echo "MullvadVPN-${version}.pkg"
-            ;;
-        *)
-            echo "Unsupported target: $os" 1>&2
-            return 1
-            ;;
-    esac
-}
-
-function download_app_package {
-    local version=$1
-    local os=$2
-    local package_repo=""
-
-    if is_dev_version "$version"; then
-        package_repo="${BUILD_DEV_REPOSITORY}"
-    else
-        package_repo="${BUILD_RELEASE_REPOSITORY}"
-    fi
-
-    local filename
-    filename=$(get_app_filename "$version" "$os")
-    local url="${package_repo}/$version/$filename"
-
-    mkdir -p "$PACKAGES_DIR"
-    if [[ ! -f "$PACKAGES_DIR/$filename" ]]; then
-        echo "Downloading build for $version ($os) from $url"
-        curl -sf -o "$PACKAGES_DIR/$filename" "$url"
-    else
-        echo "Found build for $version ($os)"
-    fi
-}
-
-function get_e2e_filename {
-    local version=$1
-    local os=$2
-    if is_dev_version "$version"; then
-        # only save 6 chars of the hash
-        local commit="${BASH_REMATCH[3]}"
-        version="${BASH_REMATCH[1]}${commit}"
-    fi
-    case $os in
-        debian*|ubuntu*|fedora*)
-            echo "app-e2e-tests-${version}-x86_64-unknown-linux-gnu"
-            ;;
-        windows*)
-            echo "app-e2e-tests-${version}-x86_64-pc-windows-msvc.exe"
-            ;;
-        macos*)
-            echo "app-e2e-tests-${version}-aarch64-apple-darwin"
-            ;;
-        *)
-            echo "Unsupported target: $os" 1>&2
-            return 1
-            ;;
-    esac
-}
-
-function download_e2e_executable {
-    local version=$1
-    local os=$2
-    local package_repo=""
-
-    if is_dev_version "$version"; then
-        package_repo="${BUILD_DEV_REPOSITORY}"
-    else
-        package_repo="${BUILD_RELEASE_REPOSITORY}"
-    fi
-
-    local filename
-    filename=$(get_e2e_filename "$version" "$os")
-    local url="${package_repo}/$version/additional-files/$filename"
-
-    mkdir -p "$PACKAGES_DIR"
-    if [[ ! -f "$PACKAGES_DIR/$filename" ]]; then
-        echo "Downloading e2e executable for $version ($os) from $url"
-        curl -sf -o "$PACKAGES_DIR/$filename" "$url"
-    else
-        echo "Found e2e executable for $version ($os)"
-    fi
-}
-
-function run_tests_for_os {
-    local os=$1
-
-    local prev_filename
-    prev_filename=$(get_app_filename "$OLD_APP_VERSION" "$os")
-    local cur_filename
-    cur_filename=$(get_app_filename "$NEW_APP_VERSION" "$os")
-
-    rm -f "$SCRIPT_DIR/.ci-logs/${os}_report"
-
-    RUST_LOG=debug cargo run --bin test-manager \
-        run-tests \
-        --account "${ACCOUNT_TOKEN:?Error: ACCOUNT_TOKEN not set}" \
-        --app-package "${cur_filename}" \
-        --app-package-to-upgrade-from "${prev_filename}" \
-        --package-folder "$PACKAGES_DIR" \
-        --test-report "$SCRIPT_DIR/.ci-logs/${os}_report" \
-        "$os" 2>&1 | sed "s/${ACCOUNT_TOKEN}/\{ACCOUNT_TOKEN\}/g"
-}
-
-echo "**********************************"
-echo "* Downloading app packages"
-echo "**********************************"
-
-find "$PACKAGES_DIR" -type f -mtime +5 -delete || true
-
-mkdir -p "$PACKAGES_DIR"
-nice_time download_app_package "$OLD_APP_VERSION" "$TEST_OS"
-nice_time download_app_package "$NEW_APP_VERSION" "$TEST_OS"
-nice_time download_e2e_executable "$NEW_APP_VERSION" "$TEST_OS"
-
-echo "**********************************"
-echo "* Building test runner"
-echo "**********************************"
-
-function build_test_runner {
-    if [[ "${TEST_OS}" =~ "debian"|"ubuntu"|"fedora" ]]; then
-        ./container-run.sh ./build-runner.sh linux
-    elif [[ "${TEST_OS}" =~ "windows" ]]; then
-        ./container-run.sh ./build-runner.sh windows
-    elif [[ "${TEST_OS}" =~ "macos" ]]; then
-        ./build-runner.sh macos
-    fi
-}
-
-nice_time build_test_runner
-
-echo "**********************************"
-echo "* Building test manager"
-echo "**********************************"
-
-cargo build -p test-manager
-
-echo "**********************************"
-echo "* Running tests"
-echo "**********************************"
-
-mkdir -p "$SCRIPT_DIR/.ci-logs/os/"
-set -o pipefail
-ACCOUNT_TOKEN=${tokens[0]} nice_time run_tests_for_os "${TEST_OS}"
diff --git a/test/Dockerfile b/test/scripts/Dockerfile
index 9e6fe20f80..9e6fe20f80 100644
--- a/test/Dockerfile
+++ b/test/scripts/Dockerfile
diff --git a/test/scripts/build-runner-image.sh b/test/scripts/build-runner-image.sh
index 4d8b39267d..4aec7b0439 100755
--- a/test/scripts/build-runner-image.sh
+++ b/test/scripts/build-runner-image.sh
@@ -34,7 +34,7 @@ case $TARGET in
             -i "${TEST_RUNNER_IMAGE_PATH}" \
             "${SCRIPT_DIR}/../target/$TARGET/release/test-runner.exe" \
             "${SCRIPT_DIR}/../target/$TARGET/release/connection-checker.exe" \
-            "${PACKAGES_DIR}/"*.exe \
+            "${PACKAGE_DIR}/"*.exe \
             "${SCRIPT_DIR}/../openvpn.ca.crt" \
             "::"
         mdir -i "${TEST_RUNNER_IMAGE_PATH}"
diff --git a/test/build-runner.sh b/test/scripts/build-runner.sh
index ef25d001c3..ffdb2b861a 100755
--- a/test/build-runner.sh
+++ b/test/scripts/build-runner.sh
@@ -3,9 +3,10 @@
 set -eu
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-REPO_DIR="$SCRIPT_DIR/.."
+REPO_DIR="$SCRIPT_DIR/../.."
 cd "$SCRIPT_DIR"
 
+# shellcheck disable=SC1091
 source "$REPO_DIR/scripts/utils/log"
 
 case ${1-:""} in
@@ -34,5 +35,5 @@ cargo build \
 
 # Only build runner image for Windows
 if [[ $TARGET == x86_64-pc-windows-gnu ]]; then
-    TARGET="$TARGET" ./scripts/build-runner-image.sh
+    TARGET="$TARGET" ./build-runner-image.sh
 fi
diff --git a/test/scripts/ci-runtests.sh b/test/scripts/ci-runtests.sh
new file mode 100755
index 0000000000..3e64894e74
--- /dev/null
+++ b/test/scripts/ci-runtests.sh
@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+set -eu
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd "$SCRIPT_DIR"
+TEST_DIR="$SCRIPT_DIR/.."
+
+if [[ "$#" -lt 1 ]]; then
+    echo "usage: $0 TEST_OS" 1>&2
+    exit 1
+fi
+
+TEST_OS=$1
+
+# shellcheck source=test/scripts/test-utils.sh
+source "test-utils.sh"
+
+echo "**********************************"
+echo "* Version to upgrade from: $LATEST_STABLE_RELEASE"
+echo "* Version to test: $CURRENT_VERSION"
+echo "**********************************"
+
+
+if [[ -z "${ACCOUNT_TOKENS+x}" ]]; then
+    echo "'ACCOUNT_TOKENS' must be specified" 1>&2
+    exit 1
+fi
+if ! readarray -t tokens < "${ACCOUNT_TOKENS}"; then
+    echo "Specify account tokens in 'ACCOUNT_TOKENS' file" 1>&2
+    exit 1
+fi
+CI_LOGS_DIR="$TEST_DIR/.ci-logs"
+mkdir -p "$CI_LOGS_DIR"
+echo "$CURRENT_VERSION" > "$CI_LOGS_DIR/last-version.log"
+
+
+echo "**********************************"
+echo "* Downloading app packages"
+echo "**********************************"
+
+
+nice_time download_app_package "$LATEST_STABLE_RELEASE" "$TEST_OS"
+nice_time download_app_package "$CURRENT_VERSION" "$TEST_OS"
+nice_time download_e2e_executable "$CURRENT_VERSION" "$TEST_OS"
+
+echo "**********************************"
+echo "* Building test manager"
+echo "**********************************"
+
+cargo build -p test-manager
+
+echo "**********************************"
+echo "* Running tests"
+echo "**********************************"
+
+mkdir -p "$CI_LOGS_DIR/os/"
+export TEST_REPORT="$CI_LOGS_DIR/${TEST_OS}_report"
+rm -f "$TEST_REPORT"
+
+set -o pipefail
+
+APP_PACKAGE=$(get_app_filename "$CURRENT_VERSION" "$TEST_OS")
+export APP_PACKAGE
+APP_PACKAGE_TO_UPGRADE_FROM=$(get_app_filename "$LATEST_STABLE_RELEASE" "$TEST_OS")
+export APP_PACKAGE_TO_UPGRADE_FROM
+ACCOUNT_TOKEN=${tokens[0]} RUST_LOG=debug nice_time run_tests_for_os "${TEST_OS}"
diff --git a/test/container-run.sh b/test/scripts/container-run.sh
index e4f24a29d9..4f87655123 100755
--- a/test/container-run.sh
+++ b/test/scripts/container-run.sh
@@ -4,16 +4,17 @@ set -eu
 
 CARGO_REGISTRY_VOLUME_NAME=${CARGO_REGISTRY_VOLUME_NAME:-"cargo-registry"}
 CONTAINER_RUNNER=${CONTAINER_RUNNER:-"podman"}
-PACKAGES_DIR=${PACKAGES_DIR:-"$HOME/.cache/mullvad-test/packages"}
+PACKAGE_DIR=${PACKAGE_DIR:-"$HOME/.cache/mullvad-test/packages"}
 
-if [ ! -d "$PACKAGES_DIR" ]; then
-  echo "$PACKAGES_DIR does not exist. It is needed to build the test bundle, so please go ahead and create the directory and re-run this script."
+if [ ! -d "$PACKAGE_DIR" ]; then
+  echo "$PACKAGE_DIR does not exist. It is needed to build the test bundle, so please go ahead and create the directory and re-run this script."
 fi
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-REPO_DIR="$SCRIPT_DIR/.."
+REPO_DIR="$SCRIPT_DIR/../.."
 cd "$SCRIPT_DIR"
 
+# shellcheck disable=SC1091
 source "$REPO_DIR/scripts/utils/log"
 
 if [[ "$(uname -s)" != "Linux" ]]; then
@@ -24,13 +25,12 @@ fi
 container_image=$(cat "$REPO_DIR/building/linux-container-image.txt")
 podman build -t mullvadvpn-app-tests --build-arg IMAGE="${container_image}" .
 
-set -x
 exec "$CONTAINER_RUNNER" run --rm -it \
     -v "${CARGO_REGISTRY_VOLUME_NAME}":/root/.cargo/registry:Z \
     -v "${REPO_DIR}":/build:z \
     -w "/build/test" \
     -e CARGO_TARGET_DIR=/build/test/target \
-    -v "${PACKAGES_DIR}":/packages:Z \
-    -e PACKAGES_DIR=/packages \
+    -v "${PACKAGE_DIR}":/packages:Z \
+    -e PACKAGE_DIR=/packages \
     mullvadvpn-app-tests \
     /bin/bash -c "$*"
diff --git a/test/scripts/test-utils.sh b/test/scripts/test-utils.sh
new file mode 100755
index 0000000000..c73c79e52a
--- /dev/null
+++ b/test/scripts/test-utils.sh
@@ -0,0 +1,312 @@
+#!/usr/bin/env bash
+
+set -eu
+
+# Returns the directory of the test-utils.sh script
+function get_test_utls_dir {
+    local script_path="${BASH_SOURCE[0]}"
+    local script_dir
+    if [[ -n "$script_path" ]]; then
+        script_dir="$(cd "$(dirname "$script_path")" > /dev/null && pwd)"
+    else
+        script_dir="$(cd "$(dirname "$0")" > /dev/null && pwd)"
+    fi
+    echo "$script_dir"
+}
+
+export BUILD_RELEASE_REPOSITORY="https://releases.mullvad.net/desktop/releases"
+export BUILD_DEV_REPOSITORY="https://releases.mullvad.net/desktop/builds"
+
+# Infer stable version from GitHub repo
+RELEASES=$(curl -sf https://api.github.com/repos/mullvad/mullvadvpn-app/releases | jq -r '[.[] | select(((.tag_name|(startswith("android") or startswith("ios"))) | not))]')
+LATEST_STABLE_RELEASE=$(jq -r '[.[] | select(.prerelease==false)] | .[0].tag_name' <<<"$RELEASES")
+
+function get_current_version {
+    local app_dir
+    app_dir="$(get_test_utls_dir)/../.."
+    cargo run -q --manifest-path="$app_dir/Cargo.toml" --bin mullvad-version
+}
+
+CURRENT_VERSION=$(get_current_version)
+commit=$(git rev-parse HEAD^\{commit\})
+commit=${commit:0:6}
+
+TAG=$(git describe --exact-match HEAD 2>/dev/null || echo "")
+
+if [[ -n "$TAG" && ${CURRENT_VERSION} =~ -dev- ]]; then
+    CURRENT_VERSION+="+${TAG}"
+fi
+
+export CURRENT_VERSION
+export LATEST_STABLE_RELEASE
+
+function print_available_releases {
+    for release in $(jq -r '.[].tag_name'<<<"$RELEASES"); do
+        echo "$release"
+    done
+}
+
+function get_package_dir {
+    local package_dir
+    if [[ -n "${PACKAGE_DIR+x}" ]]; then
+        # Resolve the package dir to an absolute path since cargo must be invoked from the test directory
+        package_dir=$(cd "$PACKAGE_DIR" > /dev/null && pwd)
+    elif [[ ("$(uname -s)" == "Darwin") ]]; then
+        package_dir="$HOME/Library/Caches/mullvad-test/packages"
+    elif [[ ("$(uname -s)" == "Linux") ]]; then
+        package_dir="$HOME/.cache/mullvad-test/packages"
+    else
+        echo "Unsupported OS" 1>&2
+        exit 1
+    fi
+
+    mkdir -p  "$package_dir" || exit 1
+    # Clean up old packages
+    find "$package_dir" -type f -mtime +5 -delete || true
+
+    echo "$package_dir"
+    return 0
+}
+
+function nice_time {
+    SECONDS=0
+    if "$@"; then
+        result=0
+    else
+        result=$?
+    fi
+    s=$SECONDS
+    echo "\"$*\" completed in $((s/60))m:$((s%60))s"
+    return $result
+}
+
+# Returns 0 if $1 is a development build. `BASH_REMATCH` contains match groups
+# if that is the case.
+function is_dev_version {
+    local pattern="(^[0-9.]+(-beta[0-9]+)?-dev-)([0-9a-z]+)(\+[0-9a-z|-]+)?$"
+    if [[ "$1" =~ $pattern ]]; then
+        return 0
+    fi
+    return 1
+}
+
+function get_app_filename {
+    local version=$1
+    local os=$2
+    if is_dev_version "$version"; then
+        # only save 6 chars of the hash
+        local commit="${BASH_REMATCH[3]}"
+        version="${BASH_REMATCH[1]}${commit}"
+        # If the dev-version includes a tag, we need to append it to the app filename
+        if [[ -n ${BASH_REMATCH[4]} ]]; then
+            version="${version}${BASH_REMATCH[4]}"
+        fi
+    fi
+    case $os in
+        debian*|ubuntu*)
+            echo "MullvadVPN-${version}_amd64.deb"
+            ;;
+        fedora*)
+            echo "MullvadVPN-${version}_x86_64.rpm"
+            ;;
+        windows*)
+            echo "MullvadVPN-${version}.exe"
+            ;;
+        macos*)
+            echo "MullvadVPN-${version}.pkg"
+            ;;
+        *)
+            echo "Unsupported target: $os" 1>&2
+            return 1
+            ;;
+    esac
+}
+
+function download_app_package {
+    local version=$1
+    local os=$2
+    local package_repo=""
+
+    if is_dev_version "$version"; then
+        package_repo="${BUILD_DEV_REPOSITORY}"
+    else
+        package_repo="${BUILD_RELEASE_REPOSITORY}"
+    fi
+
+    local filename
+    filename=$(get_app_filename "$version" "$os")
+    local url="${package_repo}/$version/$filename"
+
+    local package_dir
+    package_dir=$(get_package_dir)
+    if [[ ! -f "$package_dir/$filename" ]]; then
+        echo "Downloading build for $version ($os) from $url"
+        if ! curl -sf -o "$package_dir/$filename" "$url"; then
+            echo "Failed to download package from $url (hint: build may not exist, check the url)" 1>&2
+            exit 1
+        fi
+    else
+        echo "App package for version $version ($os) already exists at $package_dir/$filename, skipping download"
+    fi
+}
+
+function get_e2e_filename {
+    local version=$1
+    local os=$2
+    if is_dev_version "$version"; then
+        # only save 6 chars of the hash
+        local commit="${BASH_REMATCH[3]}"
+        version="${BASH_REMATCH[1]}${commit}"
+    fi
+    case $os in
+        debian*|ubuntu*|fedora*)
+            echo "app-e2e-tests-${version}-x86_64-unknown-linux-gnu"
+            ;;
+        windows*)
+            echo "app-e2e-tests-${version}-x86_64-pc-windows-msvc.exe"
+            ;;
+        macos*)
+            echo "app-e2e-tests-${version}-aarch64-apple-darwin"
+            ;;
+        *)
+            echo "Unsupported target: $os" 1>&2
+            return 1
+            ;;
+    esac
+}
+
+function download_e2e_executable {
+    local version=${1:?Error: version not set}
+    local os=${2:?Error: os not set}
+    local package_repo
+
+    if is_dev_version "$version"; then
+        package_repo="${BUILD_DEV_REPOSITORY}"
+    else
+        package_repo="${BUILD_RELEASE_REPOSITORY}"
+    fi
+
+    local filename
+    filename=$(get_e2e_filename "$version" "$os")
+    local url="${package_repo}/$version/additional-files/$filename"
+
+    local package_dir
+    package_dir=$(get_package_dir)
+    if [[ ! -f "$package_dir/$filename" ]]; then
+        echo "Downloading e2e executable for $version ($os) from $url"
+        if ! curl -sf -o "$package_dir/$filename" "$url"; then
+            echo "Failed to download package from $url (hint: build may not exist, check the url)" 1>&2
+            exit 1
+        fi
+    else
+        echo "GUI e2e executable for version $version ($os) already exists at $package_dir/$filename, skipping download"
+    fi
+}
+
+function build_test_runner {
+    local script_dir
+    script_dir=$(get_test_utls_dir)
+    local test_os=${1:?Error: test os not set}
+    if [[ "${test_os}" =~ "debian"|"ubuntu"|"fedora" ]]; then
+        "$script_dir"/container-run.sh scripts/build-runner.sh linux || exit 1
+    elif [[ "${test_os}" =~ "windows" ]]; then
+        "$script_dir"/container-run.sh scripts/build-runner.sh windows || exit 1
+    elif [[ "${test_os}" =~ "macos" ]]; then
+        "$script_dir"/build-runner.sh macos || exit 1
+    fi
+}
+
+function run_tests_for_os {
+    local vm=$1
+
+    if [[ -z "${ACCOUNT_TOKEN+x}" ]]; then
+        echo "'ACCOUNT_TOKEN' must be specified" 1>&2
+        exit 1
+    fi
+
+    echo "**********************************"
+    echo "* Building test runner"
+    echo "**********************************"
+
+    nice_time build_test_runner "$vm"
+
+
+    echo "**********************************"
+    echo "* Running tests"
+    echo "**********************************"
+
+    local upgrade_package_arg
+    if [[ -z "${APP_PACKAGE_TO_UPGRADE_FROM+x}" ]]; then
+        echo "'APP_PACKAGE_TO_UPGRADE_FROM' env not set, not testing upgrades"
+        upgrade_package_arg=()
+    else
+        upgrade_package_arg=(--app-package-to-upgrade-from "${APP_PACKAGE_TO_UPGRADE_FROM}")
+    fi
+
+    if [[ -z "${TEST_REPORT+x}" ]]; then
+        echo "'TEST_REPORT' env not set, not saving test report"
+        test_report_arg=()
+    else
+        test_report_arg=(--test-report "${TEST_REPORT}")
+    fi
+
+    local package_dir
+    package_dir=$(get_package_dir)
+    local test_dir
+    test_dir=$(get_test_utls_dir)/..
+    pushd "$test_dir"
+        if ! RUST_LOG_STYLE=always cargo run --bin test-manager \
+            run-tests \
+            --account "${ACCOUNT_TOKEN:?Error: ACCOUNT_TOKEN not set}" \
+            --app-package "${APP_PACKAGE:?Error: APP_PACKAGE not set}" \
+            "${upgrade_package_arg[@]}" \
+            "${test_report_arg[@]}" \
+            --package-dir "${package_dir}" \
+            --vm "$vm" \
+            "${TEST_FILTERS:-}" \
+            2>&1 | sed -r "s/${ACCOUNT_TOKEN}/\{ACCOUNT_TOKEN\}/g"; then
+            echo "Test run failed"
+            exit 1
+        fi
+    popd
+}
+
+# Build the current version of the app and move the package to the package folder
+# Currently unused, but may be useful in the future
+function build_current_version {
+    local app_dir
+    app_dir="$(get_test_utls_dir)/../.."
+    local app_filename
+    # TODO: TEST_OS must be set to local OS manually, should be set automatically
+    app_filename=$(get_app_filename "$CURRENT_VERSION" "${TEST_OS:?Error: TEST_OS not set}") 
+    local package_dir
+    package_dir=$(get_package_dir)
+    local app_package="$package_dir"/"$app_filename"
+
+    local gui_test_filename
+    gui_test_filename=$(get_e2e_filename "$CURRENT_VERSION" "$TEST_OS")
+    local gui_test_bin="$package_dir"/"$gui_test_filename"
+
+    if [ ! -f "$app_package" ]; then
+        pushd "$app_dir"
+            if [[ $(git diff --quiet) ]]; then
+                echo "WARNING: the app repository contains uncommitted changes, this script will only rebuild the app package when the git hash changes"
+            fi
+            ./build.sh
+        popd
+        echo "Moving '$(realpath "$app_dir/dist/$app_filename")' to '$(realpath "$app_package")'"
+        mv -n "$app_dir"/dist/"$app_filename" "$app_package"
+    else
+        echo "App package for current version already exists at $app_package, skipping build"
+    fi
+
+    if [ ! -f "$gui_test_bin" ]; then
+        pushd "$app_dir"/gui
+            npm run build-test-executable
+        popd
+        echo "Moving '$(realpath "$app_dir/dist/$gui_test_filename")' to '$(realpath "$gui_test_bin")'"
+        mv -n "$app_dir"/dist/"$gui_test_filename" "$gui_test_bin"
+    else
+        echo "GUI e2e executable for current version already exists at $gui_test_bin, skipping build"
+    fi
+}
\ No newline at end of file
diff --git a/test/test-by-version.sh b/test/test-by-version.sh
new file mode 100755
index 0000000000..a86dac8ced
--- /dev/null
+++ b/test/test-by-version.sh
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+set -eu
+
+usage() {
+    echo "This script downloads and tests the given app version from the build repositories."
+    echo
+    echo "Required environment variables:"
+    echo "  - ACCOUNT_TOKEN: Valid MullvadVPN account token"
+    echo "  - TEST_OS: Name of the VM configuration to use. List available configurations with 'cargo run --bin test-manager list'"
+    echo "Optional environment variables:"
+    echo "  - APP_VERSION: The version of the app to test (defaults to the latest stable release)"
+    echo "  - APP_PACKAGE_TO_UPGRADE_FROM: The package version to upgrade from (defaults to none)"
+    echo "  - TEST_FILTERS: specifies which tests to run (defaults to all)"
+    echo "  - TEST_REPORT : path to save the test results in a structured format"
+}
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd "$SCRIPT_DIR"
+
+# shellcheck source=test/scripts/test-utils.sh
+source "scripts/test-utils.sh"
+
+if [[ ( "$*" == "--help") ||  "$*" == "-h" ]]; then
+    usage
+    exit 0
+fi
+
+if [[ -z "${ACCOUNT_TOKEN+x}" ]]; then
+    echo "'ACCOUNT_TOKEN' must be specified" 1>&2
+    echo
+    usage
+    exit 1
+fi
+
+if [[ -z "${TEST_OS+x}" ]]; then
+    echo "'TEST_OS' must be specified" 1>&2
+    echo
+    usage
+    exit 1
+fi
+
+if [[ -z "${APP_VERSION+x}" ]]; then
+    echo "'APP_VERSION' not set, using latest build from the list of GitHub releases:"
+    print_available_releases
+    echo "For a full list of available releases you can choose from, see the stable build repository: $BUILD_RELEASE_REPOSITORY"
+    echo "and the dev build repository: $BUILD_DEV_REPOSITORY"
+    APP_VERSION=$LATEST_STABLE_RELEASE
+fi
+
+echo "**********************************"
+echo "* Version to test: $APP_VERSION"
+echo "**********************************"
+
+echo "**********************************"
+echo "* Downloading app packages"
+echo "**********************************"
+
+download_app_package "$APP_VERSION" "$TEST_OS"
+download_e2e_executable "$APP_VERSION" "$TEST_OS"
+
+if [[ -n "${APP_PACKAGE_TO_UPGRADE_FROM+x}" ]]; then
+    download_app_package "$APP_PACKAGE_TO_UPGRADE_FROM" "$TEST_OS"
+fi
+
+set -o pipefail
+APP_PACKAGE=$(get_app_filename "$APP_VERSION" "$TEST_OS")
+export APP_PACKAGE
+run_tests_for_os "${TEST_OS}"
diff --git a/test/test-manager/src/logging.rs b/test/test-manager/src/logging.rs
index 3d1aa13156..2bc5ea90d1 100644
--- a/test/test-manager/src/logging.rs
+++ b/test/test-manager/src/logging.rs
@@ -30,6 +30,8 @@ impl Logger {
             logger.filter_module("tower", log::LevelFilter::Info);
             logger.filter_module("hyper", log::LevelFilter::Info);
             logger.filter_module("rustls", log::LevelFilter::Info);
+            logger.filter_module("tarpc", log::LevelFilter::Warn);
+            logger.filter_module("mio_serial", log::LevelFilter::Warn);
             logger.filter_level(log::LevelFilter::Info);
             logger.parse_env(env_logger::DEFAULT_FILTER_ENV);
 
diff --git a/test/test-manager/src/main.rs b/test/test-manager/src/main.rs
index 79d239f4d0..daad04983b 100644
--- a/test/test-manager/src/main.rs
+++ b/test/test-manager/src/main.rs
@@ -28,27 +28,27 @@ struct Args {
 enum Commands {
     /// Create or edit a VM config
     Set {
-        /// Name of the config
-        name: String,
+        /// Name of the VM config
+        vm: String,
 
         /// VM config
         #[clap(flatten)]
         config: config::VmConfig,
     },
 
-    /// Remove specified configuration
+    /// Remove specified VM config
     Remove {
-        /// Name of the config
-        name: String,
+        /// Name of the VM config, run `test-manager list` to see available configs
+        vm: String,
     },
 
-    /// List available configurations
+    /// List available VM configurations
     List,
 
     /// Spawn a runner instance without running any tests
     RunVm {
-        /// Name of the runner config
-        name: String,
+        /// Name of the VM config, run `test-manager list` to see available configs
+        vm: String,
 
         /// Run VNC server on a specified port
         #[arg(long)]
@@ -64,8 +64,9 @@ enum Commands {
 
     /// Spawn a runner instance and run tests
     RunTests {
-        /// Name of the runner config
-        name: String,
+        /// Name of the VM config, run `test-manager list` to see available configs
+        #[arg(long)]
+        vm: String,
 
         /// Show display of guest
         #[arg(long, group = "display_args")]
@@ -81,7 +82,7 @@ enum Commands {
 
         /// App package to test. Can be a path to the package, just the package file name, git hash
         /// or tag. If the direct path is not given, the package is assumed to be in the directory
-        /// specified by the `--package-folder` argument.
+        /// specified by the `--package-dir` argument.
         ///
         /// # Note
         ///
@@ -99,9 +100,15 @@ enum Commands {
         #[arg(long)]
         app_package_to_upgrade_from: Option<String>,
 
+        /// Package used for GUI tests. Parsed the same way as `--app-package`.
+        /// If not specified, will look for a package matching the version of the app package. If
+        /// no such package is found, the GUI tests will fail.
+        #[arg(long)]
+        gui_package: Option<String>,
+
         /// Folder to search for packages. Defaults to current directory.
         #[arg(long, value_name = "DIR")]
-        package_folder: Option<PathBuf>,
+        package_dir: Option<PathBuf>,
 
         /// Only run tests matching substrings
         test_filters: Vec<String>,
@@ -110,8 +117,8 @@ enum Commands {
         #[arg(long, short)]
         verbose: bool,
 
-        /// Output test results in a structured format.
-        #[arg(long)]
+        /// Path to output test results in a structured format
+        #[arg(long, value_name = "PATH")]
         test_report: Option<PathBuf>,
     },
 
@@ -127,7 +134,7 @@ enum Commands {
     /// to have `provisioner` set to `ssh`, `ssh_user` & `ssh_password` set and
     /// the `ssh_user` should be able to execute commands with sudo/ as root.
     Update {
-        /// Name of the runner config
+        /// Name of the VM config
         name: String,
     },
 }
@@ -156,34 +163,34 @@ async fn main() -> Result<()> {
         .context("Failed to load config")?;
     match args.cmd {
         Commands::Set {
-            name,
+            vm,
             config: vm_config,
-        } => vm::set_config(&mut config, &name, vm_config)
+        } => vm::set_config(&mut config, &vm, vm_config)
             .await
             .context("Failed to edit or create VM config"),
-        Commands::Remove { name } => {
-            if config.get_vm(&name).is_none() {
+        Commands::Remove { vm } => {
+            if config.get_vm(&vm).is_none() {
                 println!("No such configuration");
                 return Ok(());
             }
             config
                 .edit(|config| {
-                    config.vms.remove_entry(&name);
+                    config.vms.remove_entry(&vm);
                 })
                 .await
                 .context("Failed to remove config entry")?;
-            println!("Removed configuration \"{name}\"");
+            println!("Removed configuration \"{vm}\"");
             Ok(())
         }
         Commands::List => {
             println!("Available configurations:");
-            for name in config.vms.keys() {
-                println!("{}", name);
+            for (vm, config) in config.vms.iter() {
+                println!("{vm}: {config:#?}");
             }
             Ok(())
         }
         Commands::RunVm {
-            name,
+            vm,
             vnc,
             keep_changes,
         } => {
@@ -195,9 +202,7 @@ async fn main() -> Result<()> {
                 config::Display::Local
             };
 
-            let mut instance = vm::run(&config, &name)
-                .await
-                .context("Failed to start VM")?;
+            let mut instance = vm::run(&config, &vm).await.context("Failed to start VM")?;
 
             instance.wait().await;
 
@@ -215,13 +220,14 @@ async fn main() -> Result<()> {
             Ok(())
         }
         Commands::RunTests {
-            name,
+            vm,
             display,
             vnc,
             account,
             app_package,
             app_package_to_upgrade_from,
-            package_folder,
+            gui_package,
+            package_dir,
             test_filters,
             verbose,
             test_report,
@@ -240,12 +246,12 @@ async fn main() -> Result<()> {
                 .unwrap_or(DEFAULT_MULLVAD_HOST.to_owned());
             log::debug!("Mullvad host: {mullvad_host}");
 
-            let vm_config = vm::get_vm_config(&config, &name).context("Cannot get VM config")?;
+            let vm_config = vm::get_vm_config(&config, &vm).context("Cannot get VM config")?;
 
             let summary_logger = match test_report {
                 Some(path) => Some(
                     summary::SummaryLogger::new(
-                        &name,
+                        &vm,
                         test_rpc::meta::Os::from(vm_config.os_type),
                         &path,
                     )
@@ -259,14 +265,13 @@ async fn main() -> Result<()> {
                 vm_config,
                 app_package,
                 app_package_to_upgrade_from,
-                package_folder,
+                gui_package,
+                package_dir,
             )
             .context("Could not find the specified app packages")?;
 
-            let mut instance = vm::run(&config, &name)
-                .await
-                .context("Failed to start VM")?;
-            let artifacts_dir = vm::provision(&config, &name, &*instance, &manifest)
+            let mut instance = vm::run(&config, &vm).await.context("Failed to start VM")?;
+            let artifacts_dir = vm::provision(&config, &vm, &*instance, &manifest)
                 .await
                 .context("Failed to run provisioning for VM")?;
 
@@ -293,7 +298,7 @@ async fn main() -> Result<()> {
                         .app_package_to_upgrade_from_path
                         .map(|path| path.file_name().unwrap().to_string_lossy().into_owned()),
                     ui_e2e_tests_filename: manifest
-                        .ui_e2e_tests_path
+                        .gui_package_path
                         .map(|path| path.file_name().unwrap().to_string_lossy().into_owned()),
                     mullvad_host,
                     #[cfg(target_os = "macos")]
diff --git a/test/test-manager/src/mullvad_daemon.rs b/test/test-manager/src/mullvad_daemon.rs
index 115a09e049..251ed3d1fe 100644
--- a/test/test-manager/src/mullvad_daemon.rs
+++ b/test/test-manager/src/mullvad_daemon.rs
@@ -69,7 +69,7 @@ impl RpcClientProvider {
         // FIXME: Ugly workaround to ensure that we don't receive stuff from a
         // previous RPC session.
         tokio::time::sleep(std::time::Duration::from_millis(500)).await;
-        log::debug!("Mullvad daemon: connecting");
+        log::trace!("Mullvad daemon: connecting");
         let channel = tonic::transport::Endpoint::from_static("serial://placeholder")
             .timeout(GRPC_REQUEST_TIMEOUT)
             .connect_with_connector(self.service.clone())
diff --git a/test/test-manager/src/network_monitor.rs b/test/test-manager/src/network_monitor.rs
index ddd35edc7e..1a5ef5e463 100644
--- a/test/test-manager/src/network_monitor.rs
+++ b/test/test-manager/src/network_monitor.rs
@@ -59,7 +59,7 @@ impl PacketCodec for Codec {
             EtherTypes::Ipv4 => Self::parse_ipv4(frame.payload()),
             EtherTypes::Ipv6 => Self::parse_ipv6(frame.payload()),
             ethertype => {
-                log::debug!("Ignoring unknown ethertype: {ethertype}");
+                log::trace!("Ignoring unknown ethertype: {ethertype}");
                 None
             }
         }
@@ -288,11 +288,11 @@ async fn start_packet_monitor_for_interface(
                         Some(Ok(packet))=> {
                             if let Some(packet) = packet {
                                 if !filter_fn(&packet) {
-                                    log::debug!("{interface} \"{packet:?}\" does not match closure conditions");
+                                    log::trace!("{interface} \"{packet:?}\" does not match closure conditions");
                                     monitor_result.discarded_packets =
                                         monitor_result.discarded_packets.saturating_add(1);
                                 } else {
-                                    log::debug!("{interface} \"{packet:?}\" matches closure conditions");
+                                    log::trace!("{interface} \"{packet:?}\" matches closure conditions");
 
                                     let should_continue = should_continue_fn(&packet);
 
diff --git a/test/test-manager/src/package.rs b/test/test-manager/src/package.rs
index f3fd535673..34fb30419a 100644
--- a/test/test-manager/src/package.rs
+++ b/test/test-manager/src/package.rs
@@ -1,17 +1,15 @@
 use crate::config::{Architecture, OsType, PackageType, VmConfig};
 use anyhow::{Context, Result};
+use itertools::Itertools;
 use once_cell::sync::Lazy;
 use regex::Regex;
 use std::path::{Path, PathBuf};
 
-static VERSION_REGEX: Lazy<Regex> =
-    Lazy::new(|| Regex::new(r"\d{4}\.\d+(-beta\d+)?(-dev)?-([0-9a-z])+").unwrap());
-
 #[derive(Debug, Clone)]
 pub struct Manifest {
     pub app_package_path: PathBuf,
     pub app_package_to_upgrade_from_path: Option<PathBuf>,
-    pub ui_e2e_tests_path: Option<PathBuf>,
+    pub gui_package_path: Option<PathBuf>,
 }
 
 /// Obtain app packages and their filenames
@@ -22,40 +20,72 @@ pub fn get_app_manifest(
     config: &VmConfig,
     app_package: String,
     app_package_to_upgrade_from: Option<String>,
-    package_folder: Option<PathBuf>,
+    gui_package: Option<String>,
+    package_dir: Option<PathBuf>,
 ) -> Result<Manifest> {
     let package_type = (config.os_type, config.package_type, config.architecture);
 
-    let app_package_path = find_app(&app_package, false, package_type, package_folder.as_ref())?;
+    let app_package_path = find_app(&app_package, false, package_type, package_dir.as_ref())?;
     log::info!("App package: {}", app_package_path.display());
 
     let app_package_to_upgrade_from_path = app_package_to_upgrade_from
-        .map(|app| find_app(&app, false, package_type, package_folder.as_ref()))
+        .map(|app| find_app(&app, false, package_type, package_dir.as_ref()))
         .transpose()?;
     log::info!("App package to upgrade from: {app_package_to_upgrade_from_path:?}");
 
-    let capture = VERSION_REGEX
-        .captures(app_package_path.to_str().unwrap())
-        .with_context(|| format!("Cannot parse version: {}", app_package_path.display()))?
-        .get(0)
-        .map(|c| c.as_str())
-        .expect("Could not parse version from package name: {app_package}");
+    // Automatically try to find the UI e2e tests based on the app package
+
+    // Search the specified package folder, or same folder as the app package if missing
+    let ui_e2e_package_dir = package_dir.unwrap_or(
+        app_package_path
+            .parent()
+            .expect("Path to app package should have parent")
+            .into(),
+    );
 
-    let ui_e2e_tests_path = find_app(capture, true, package_type, package_folder.as_ref()).ok();
-    log::info!("GUI e2e test binary: {ui_e2e_tests_path:?}");
+    let app_version = get_version_from_path(&app_package_path)?;
+    let gui_package_path = find_app(
+        match &gui_package {
+            Some(gui_package) => gui_package,
+            None => &app_version,
+        },
+        true,
+        package_type,
+        Some(&ui_e2e_package_dir),
+    );
+
+    // Don't allow the UI/e2e test binary to missing if it's flag was specified
+    let gui_package_path = match gui_package {
+        Some(_) => Some(gui_package_path.context("Could not find specified UI/e2e test binary")?),
+        None => gui_package_path.ok(),
+    };
+
+    log::info!("GUI e2e test binary: {gui_package_path:?}");
 
     Ok(Manifest {
         app_package_path,
         app_package_to_upgrade_from_path,
-        ui_e2e_tests_path,
+        gui_package_path,
     })
 }
 
+fn get_version_from_path(app_package_path: &Path) -> Result<String, anyhow::Error> {
+    static VERSION_REGEX: Lazy<Regex> =
+        Lazy::new(|| Regex::new(r"\d{4}\.\d+((-beta\d+)?(-dev)?-([0-9a-z])+)?").unwrap());
+
+    VERSION_REGEX
+        .captures(app_package_path.to_str().unwrap())
+        .with_context(|| format!("Cannot parse version: {}", app_package_path.display()))?
+        .get(0)
+        .map(|c| c.as_str().to_owned())
+        .context("Could not parse version from package name: {app_package}")
+}
+
 fn find_app(
     app: &str,
     e2e_bin: bool,
     package_type: (OsType, Option<PackageType>, Option<Architecture>),
-    package_folder: Option<&PathBuf>,
+    package_dir: Option<&PathBuf>,
 ) -> Result<PathBuf> {
     // If it's a path, use that path
     let app_path = Path::new(app);
@@ -68,7 +98,7 @@ fn find_app(
     app.make_ascii_lowercase();
 
     let current_dir = std::env::current_dir().expect("Unable to get current directory");
-    let packages_dir = package_folder.unwrap_or(&current_dir);
+    let packages_dir = package_dir.unwrap_or(&current_dir);
     std::fs::create_dir_all(packages_dir)?;
     let dir = std::fs::read_dir(packages_dir.clone()).context("Failed to list packages")?;
 
@@ -95,6 +125,7 @@ fn find_app(
             // Skip for non-Linux, because there's only one package
             !linux || matching_ident
         }) // Skip file if it doesn't match the architecture
+        .sorted_unstable_by_key(|(_path, u8_path)| u8_path.len())
         .find(|(_path, u8_path)| u8_path.contains(&app)) //  Find match
         .map(|(path, _)| path).context(if e2e_bin {
             format!(
@@ -126,3 +157,27 @@ fn get_os_name(package_type: (OsType, Option<PackageType>, Option<Architecture>)
         OsType::Linux => "linux",
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_version_regex() {
+        let path = Path::new("../some/path/MullvadVPN-2024.4-beta1-dev-f7df8e_amd64.deb");
+        let capture = get_version_from_path(path).unwrap();
+        assert_eq!(capture, "2024.4-beta1-dev-f7df8e");
+
+        let path = Path::new("../some/path/MullvadVPN-2024.4-beta1-f7df8e_amd64.deb");
+        let capture = get_version_from_path(path).unwrap();
+        assert_eq!(capture, "2024.4-beta1-f7df8e");
+
+        let path = Path::new("../some/path/MullvadVPN-2024.4-dev-f7df8e_amd64.deb");
+        let capture = get_version_from_path(path).unwrap();
+        assert_eq!(capture, "2024.4-dev-f7df8e");
+
+        let path = Path::new("../some/path/MullvadVPN-2024.3_amd64.deb");
+        let capture = get_version_from_path(path).unwrap();
+        assert_eq!(capture, "2024.3");
+    }
+}
diff --git a/test/test-manager/src/run_tests.rs b/test/test-manager/src/run_tests.rs
index 73c72ebc2e..4449a06a6f 100644
--- a/test/test-manager/src/run_tests.rs
+++ b/test/test-manager/src/run_tests.rs
@@ -169,6 +169,7 @@ pub async fn run(
     }
 
     // wait for cleanup
+    drop(client);
     drop(test_context);
     let _ = tokio::time::timeout(Duration::from_secs(5), completion_handle).await;
 
diff --git a/test/test-manager/src/summary.rs b/test/test-manager/src/summary.rs
index 30b71948cf..9706e351a1 100644
--- a/test/test-manager/src/summary.rs
+++ b/test/test-manager/src/summary.rs
@@ -68,7 +68,7 @@ pub struct SummaryLogger {
 impl SummaryLogger {
     /// Create a new logger and log to `path`. If `path` does not exist, it will be created. If it
     /// already exists, it is truncated and overwritten.
-    pub async fn new(name: &str, os: Os, path: &Path) -> Result<SummaryLogger, Error> {
+    pub async fn new(vm: &str, os: Os, path: &Path) -> Result<SummaryLogger, Error> {
         let mut file = fs::OpenOptions::new()
             .create(true)
             .write(true)
@@ -77,9 +77,7 @@ impl SummaryLogger {
             .await
             .map_err(|err| Error::Open(err, path.to_path_buf()))?;
 
-        file.write_all(name.as_bytes())
-            .await
-            .map_err(Error::Write)?;
+        file.write_all(vm.as_bytes()).await.map_err(Error::Write)?;
         file.write_u8(b'\n').await.map_err(Error::Write)?;
         file.write_all(&serde_json::to_vec(&os).map_err(Error::Serialize)?)
             .await
diff --git a/test/test-manager/src/tests/account.rs b/test/test-manager/src/tests/account.rs
index 2cc7c5a95e..c67e1bed19 100644
--- a/test/test-manager/src/tests/account.rs
+++ b/test/test-manager/src/tests/account.rs
@@ -97,10 +97,13 @@ pub async fn test_too_many_devices(
     log::info!("Log in with too many devices");
     let login_result = login_with_retries(&mut mullvad_client).await;
 
-    assert!(matches!(
-        login_result,
-        Err(mullvad_management_interface::Error::TooManyDevices)
-    ));
+    assert!(
+        matches!(
+            login_result,
+            Err(mullvad_management_interface::Error::TooManyDevices)
+        ),
+        "Expected too many devices error, got {login_result:?}"
+    );
 
     // Run UI test
     let ui_result = ui::run_test_env(
diff --git a/test/test-manager/src/tests/tunnel_state.rs b/test/test-manager/src/tests/tunnel_state.rs
index 5a94c60bc6..031a669db4 100644
--- a/test/test-manager/src/tests/tunnel_state.rs
+++ b/test/test-manager/src/tests/tunnel_state.rs
@@ -366,7 +366,7 @@ pub async fn test_connected_state(
                     tunnel_type: TunnelType::Wireguard,
                     quantum_resistant: _,
                     proxy: None,
-                    obfuscation: None,
+                    obfuscation: _,
                     entry_endpoint: None,
                     tunnel_interface: _,
                     daita: _,
@@ -440,13 +440,11 @@ pub async fn test_connecting_state_when_corrupted_state_cache(
     // side-effect of this is that no network traffic is allowed to leak.
     log::info!("Starting the app back up again");
     rpc.start_mullvad_daemon().await?;
-    wait_for_tunnel_state(mullvad_client.clone(), |state| !state.is_disconnected())
-        .await
-        .map_err(|err| {
+    wait_for_tunnel_state(mullvad_client.clone(), |state| !state.is_disconnected()).await
+        .inspect_err(|_| {
             log::error!("App did not start in an expected state. \
                         App is not in either `Connecting` or `Connected` state after starting with corrupt state cache! \
                         There is a possibility of leaks during app startup ");
-            err
         })?;
     log::info!("App successfully recovered from a corrupt tunnel state cache.");
 
diff --git a/test/test-manager/src/tests/ui.rs b/test/test-manager/src/tests/ui.rs
index f4b9af04f3..f2b266a4e6 100644
--- a/test/test-manager/src/tests/ui.rs
+++ b/test/test-manager/src/tests/ui.rs
@@ -82,7 +82,7 @@ pub async fn run_test_env<
         let stdout = std::str::from_utf8(&result.stdout).unwrap_or("invalid utf8");
         let stderr = std::str::from_utf8(&result.stderr).unwrap_or("invalid utf8");
 
-        log::debug!("UI test failed:\n\nstdout:\n\n{stdout}\n\n{stderr}\n");
+        log::error!("UI test failed:\n\nstdout:\n\n{stdout}\n\n{stderr}\n");
     }
 
     Ok(result)
diff --git a/test/test-manager/src/vm/network/linux.rs b/test/test-manager/src/vm/network/linux.rs
index 1a4e994a52..12095138f6 100644
--- a/test/test-manager/src/vm/network/linux.rs
+++ b/test/test-manager/src/vm/network/linux.rs
@@ -106,7 +106,7 @@ pub async fn setup_test_network() -> Result<NetworkHandle> {
 
     let test_subnet = TEST_SUBNET.to_string();
 
-    log::info!("Create bridge network: dev {BRIDGE_NAME}, net {test_subnet}");
+    log::debug!("Create bridge network: dev {BRIDGE_NAME}, net {test_subnet}");
 
     run_ip_cmd(["link", "add", BRIDGE_NAME, "type", "bridge"]).await?;
     run_ip_cmd(["addr", "add", "dev", BRIDGE_NAME, &test_subnet]).await?;
diff --git a/test/test-manager/src/vm/provision.rs b/test/test-manager/src/vm/provision.rs
index f440cbe9c8..0b7b88e766 100644
--- a/test/test-manager/src/vm/provision.rs
+++ b/test/test-manager/src/vm/provision.rs
@@ -18,7 +18,7 @@ pub async fn provision(
 ) -> Result<String> {
     match config.provisioner {
         Provisioner::Ssh => {
-            log::info!("SSH provisioning");
+            log::debug!("SSH provisioning");
 
             let (user, password) = config.get_ssh_options().context("missing SSH config")?;
             ssh(
@@ -125,9 +125,9 @@ fn blocking_ssh(
     } else {
         log::warn!("No previous app to send to remote")
     }
-    if let Some(ui_e2e_tests_path) = &local_app_manifest.ui_e2e_tests_path {
-        ssh_send_file_path(&session, ui_e2e_tests_path, temp_dir)
-            .context("Failed to send ui_e2e_tests_path to remote")?;
+    if let Some(gui_package_path) = &local_app_manifest.gui_package_path {
+        ssh_send_file_path(&session, gui_package_path, temp_dir)
+            .context("Failed to send gui_package_path to remote")?;
     } else {
         log::warn!("No UI e2e test to send to remote")
     }
@@ -166,13 +166,13 @@ fn blocking_ssh(
         .app_package_to_upgrade_from_path
         .map(|path| path.file_name().unwrap().to_string_lossy().into_owned())
         .unwrap_or_default();
-    let ui_e2e_tests_path = local_app_manifest
-        .ui_e2e_tests_path
+    let gui_package_path = local_app_manifest
+        .gui_package_path
         .map(|path| path.file_name().unwrap().to_string_lossy().into_owned())
         .unwrap_or_default();
 
     let cmd = format!(
-        "sudo {} {remote_dir} \"{app_package_path}\" \"{app_package_to_upgrade_from_path}\" \"{ui_e2e_tests_path}\"",
+        "sudo {} {remote_dir} \"{app_package_path}\" \"{app_package_to_upgrade_from_path}\" \"{gui_package_path}\"",
         dest.display()
     );
     log::debug!("Running setup script on remote, cmd: {cmd}");
@@ -218,6 +218,7 @@ fn ssh_send_file<R: Read>(
 fn ssh_exec(session: &Session, command: &str) -> Result<String> {
     let mut channel = session.channel_session()?;
     channel.exec(command)?;
+    let mut stderr_handle = channel.stderr();
     let mut output = String::new();
     channel.read_to_string(&mut output)?;
     channel.send_eof()?;
@@ -228,7 +229,9 @@ fn ssh_exec(session: &Session, command: &str) -> Result<String> {
         .exit_status()
         .context("Failed to obtain exit status")?;
     if exit_status != 0 {
-        log::error!("command failed: {command}\n{output}");
+        let mut stderr = String::new();
+        stderr_handle.read_to_string(&mut stderr).unwrap();
+        log::error!("Command failed: command: {command}\n\noutput:\n{output}\n\nstderr: {stderr}");
         bail!("command failed: {exit_status}");
     }
 
diff --git a/test/test-runner/src/main.rs b/test/test-runner/src/main.rs
index 3fe91fb723..59fc83672d 100644
--- a/test/test-runner/src/main.rs
+++ b/test/test-runner/src/main.rs
@@ -422,7 +422,7 @@ impl Service for TestServer {
                     match stderr.read_line(&mut line).await {
                         Ok(0) => break,
                         Ok(_) => {
-                            let trimmed = line.trim_end_matches(&['\r', '\n']);
+                            let trimmed = line.trim_end_matches(['\r', '\n']);
                             log::info!("child stderr (pid={pid}): {trimmed}");
                             line.clear();
                         }
diff --git a/test/test-runner/src/sys.rs b/test/test-runner/src/sys.rs
index a53396d08f..e1060124d7 100644
--- a/test/test-runner/src/sys.rs
+++ b/test/test-runner/src/sys.rs
@@ -434,7 +434,7 @@ impl EnvVar {
 
 #[cfg(target_os = "linux")]
 pub async fn set_daemon_environment(env: HashMap<String, String>) -> Result<(), test_rpc::Error> {
-    use std::fmt::Write;
+    use std::{fmt::Write, ops::Not};
 
     let mut override_content = String::new();
     override_content.push_str("[Service]\n");
@@ -459,17 +459,31 @@ pub async fn set_daemon_environment(env: HashMap<String, String>) -> Result<(),
         .await
         .map_err(|e| test_rpc::Error::Service(e.to_string()))?;
 
-    tokio::process::Command::new("systemctl")
+    if tokio::process::Command::new("systemctl")
         .args(["daemon-reload"])
         .status()
         .await
-        .map_err(|e| test_rpc::Error::Service(e.to_string()))?;
+        .map_err(|e| test_rpc::Error::Io(e.to_string()))?
+        .success()
+        .not()
+    {
+        return Err(test_rpc::Error::Service(
+            "Daemon service could not be reloaded".to_owned(),
+        ));
+    };
 
-    tokio::process::Command::new("systemctl")
+    if tokio::process::Command::new("systemctl")
         .args(["restart", "mullvad-daemon"])
         .status()
         .await
-        .map_err(|e| test_rpc::Error::Service(e.to_string()))?;
+        .map_err(|e| test_rpc::Error::Io(e.to_string()))?
+        .success()
+        .not()
+    {
+        return Err(test_rpc::Error::Service(
+            "Daemon service could not be restarted".to_owned(),
+        ));
+    };
 
     wait_for_service_state(ServiceState::Running).await?;
     Ok(())