Restructure test scripts

Split functionality `ci-runtest.sh` into multiple scripts.
`test-by-version.sh` can be used to test against any version
of the app available on the build servers. `test-utils.sh` contains
shared logic.

Rename `PACKAGES_DIR` env `PACKAGE_DIR`,
it's more consistent with the new CLI flag.

12 files changed, 478 insertions, 275 deletions

diff --git a/.github/workflows/desktop-e2e.yml b/.github/workflows/desktop-e2e.yml
index 03e2959be1..23a612f9ac 100644
--- a/.github/workflows/desktop-e2e.yml
+++ b/.github/workflows/desktop-e2e.yml
@@ -151,7 +151,7 @@ jobs:
         shell: bash -ieo pipefail {0}
         run: |
           git fetch --tags --force
-          ./test/ci-runtests.sh ${{ matrix.os }}
+          ./test/scripts/ci-runtests.sh ${{ matrix.os }}
       - uses: actions/upload-artifact@v3
         if: '!cancelled()'
         with:
@@ -226,7 +226,7 @@ jobs:
         shell: bash -ieo pipefail {0}
         run: |
           git fetch --tags --force
-          ./test/ci-runtests.sh ${{ matrix.os }}
+          ./test/scripts/ci-runtests.sh ${{ matrix.os }}
       - uses: actions/upload-artifact@v3
         if: '!cancelled()'
         with:
@@ -297,7 +297,7 @@ jobs:
         shell: bash -ieo pipefail {0}
         run: |
           git fetch --tags --force
-          ./test/ci-runtests.sh ${{ matrix.os }}
+          ./test/scripts/ci-runtests.sh ${{ matrix.os }}
       - uses: actions/upload-artifact@v3
         if: '!cancelled()'
         with:
diff --git a/test/README.md b/test/README.md
index b490166ef5..c8ff94c425 100644
--- a/test/README.md
+++ b/test/README.md
@@ -66,27 +66,27 @@ Currently, only `x86_64` platforms are supported for Windows/Linux and `ARM64` (
 For example, building `test-runner` for Windows would look like this:
 
 ``` bash
-./container-run.sh ./build-runner.sh windows
+./scripts/container-run.sh ./scripts/build-runner.sh windows
 ```
 
 ## Linux
 Using `podman` is the recommended way to build the `test-runner`. See the [Linux section under Prerequisities](#Prerequisities) for more details.
 
 ``` bash
-./container-run.sh ./build-runner.sh linux
+./scripts/container-run.sh ./scripts/build-runner.sh linux
 ```
 
 ## macOS
 
 ``` bash
-./build-runner.sh macos
+./scripts/build-runner.sh macos
 ```
 
 ## Windows
 The `test-runner` binary for Windows may be cross-compiled from a Linux host.
 
 ``` bash
-./container-run.sh ./build-runner.sh windows
+./scripts/container-run.sh ./scripts/build-runner.sh windows
 ```
 
 # Building base images
@@ -145,7 +145,7 @@ cargo run --bin test-manager run-tests macos-ventura \
     --app-package-to-upgrade-from 2023.2
 ```
 
-## Note on `ci-runtests.sh`
+## Note on `scripts/ci-runtests.sh`
 
 Account tokens are read (newline-delimited) from the path specified by the environment variable
 `ACCOUNT_TOKENS`. Round robin is used to select an account for each VM.
diff --git a/test/ci-runtests.sh b/test/ci-runtests.sh
deleted file mode 100755
index 0346044e7b..0000000000
--- a/test/ci-runtests.sh
+++ /dev/null
@@ -1,246 +0,0 @@
-#!/usr/bin/env bash
-
-set -eu
-
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-APP_DIR="$SCRIPT_DIR/../"
-cd "$SCRIPT_DIR"
-
-BUILD_RELEASE_REPOSITORY="https://releases.mullvad.net/desktop/releases"
-BUILD_DEV_REPOSITORY="https://releases.mullvad.net/desktop/builds"
-
-if [[ ("$(uname -s)" == "Darwin") ]]; then
-    export PACKAGES_DIR=$HOME/Library/Caches/mullvad-test/packages
-elif [[ ("$(uname -s)" == "Linux") ]]; then
-    export PACKAGES_DIR=$HOME/.cache/mullvad-test/packages
-else
-    echo "Unsupported OS" 1>&2
-    exit 1
-fi
-
-if [[ "$#" -lt 1 ]]; then
-    echo "usage: $0 TEST_OS" 1>&2
-    exit 1
-fi
-
-TEST_OS=$1
-
-# Infer stable version from GitHub repo
-RELEASES=$(curl -sf https://api.github.com/repos/mullvad/mullvadvpn-app/releases | jq -r '[.[] | select(((.tag_name|(startswith("android") or startswith("ios"))) | not))]')
-OLD_APP_VERSION=$(jq -r '[.[] | select(.prerelease==false)] | .[0].tag_name' <<<"$RELEASES")
-
-NEW_APP_VERSION=$(cargo run -q --manifest-path="$APP_DIR/Cargo.toml" --bin mullvad-version)
-commit=$(git rev-parse HEAD^\{commit\})
-commit=${commit:0:6}
-
-TAG=$(git describe --exact-match HEAD 2>/dev/null || echo "")
-
-if [[ -n "$TAG" && ${NEW_APP_VERSION} =~ -dev- ]]; then
-    NEW_APP_VERSION+="+${TAG}"
-fi
-
-echo "**********************************"
-echo "* Version to upgrade from: $OLD_APP_VERSION"
-echo "* Version to test: $NEW_APP_VERSION"
-echo "**********************************"
-
-
-if [[ -z "${ACCOUNT_TOKENS+x}" ]]; then
-    echo "'ACCOUNT_TOKENS' must be specified" 1>&2
-    exit 1
-fi
-if ! readarray -t tokens < "${ACCOUNT_TOKENS}"; then
-    echo "Specify account tokens in 'ACCOUNT_TOKENS' file" 1>&2
-    exit 1
-fi
-
-mkdir -p "$SCRIPT_DIR/.ci-logs"
-echo "$NEW_APP_VERSION" > "$SCRIPT_DIR/.ci-logs/last-version.log"
-
-function nice_time {
-    SECONDS=0
-    if "$@"; then
-        result=0
-    else
-        result=$?
-    fi
-    s=$SECONDS
-    echo "\"$*\" completed in $((s/60))m:$((s%60))s"
-    return $result
-}
-
-# Returns 0 if $1 is a development build. `BASH_REMATCH` contains match groups
-# if that is the case.
-function is_dev_version {
-    local pattern="(^[0-9.]+(-beta[0-9]+)?-dev-)([0-9a-z]+)(\+[0-9a-z|-]+)?$"
-    if [[ "$1" =~ $pattern ]]; then
-        return 0
-    fi
-    return 1
-}
-
-function get_app_filename {
-    local version=$1
-    local os=$2
-    if is_dev_version "$version"; then
-        # only save 6 chars of the hash
-        local commit="${BASH_REMATCH[3]}"
-        version="${BASH_REMATCH[1]}${commit}"
-        # If the dev-version includes a tag, we need to append it to the app filename
-        if [[ -n ${BASH_REMATCH[4]} ]]; then
-            version="${version}${BASH_REMATCH[4]}"
-        fi
-    fi
-    case $os in
-        debian*|ubuntu*)
-            echo "MullvadVPN-${version}_amd64.deb"
-            ;;
-        fedora*)
-            echo "MullvadVPN-${version}_x86_64.rpm"
-            ;;
-        windows*)
-            echo "MullvadVPN-${version}.exe"
-            ;;
-        macos*)
-            echo "MullvadVPN-${version}.pkg"
-            ;;
-        *)
-            echo "Unsupported target: $os" 1>&2
-            return 1
-            ;;
-    esac
-}
-
-function download_app_package {
-    local version=$1
-    local os=$2
-    local package_repo=""
-
-    if is_dev_version "$version"; then
-        package_repo="${BUILD_DEV_REPOSITORY}"
-    else
-        package_repo="${BUILD_RELEASE_REPOSITORY}"
-    fi
-
-    local filename
-    filename=$(get_app_filename "$version" "$os")
-    local url="${package_repo}/$version/$filename"
-
-    mkdir -p "$PACKAGES_DIR"
-    if [[ ! -f "$PACKAGES_DIR/$filename" ]]; then
-        echo "Downloading build for $version ($os) from $url"
-        curl -sf -o "$PACKAGES_DIR/$filename" "$url"
-    else
-        echo "Found build for $version ($os)"
-    fi
-}
-
-function get_e2e_filename {
-    local version=$1
-    local os=$2
-    if is_dev_version "$version"; then
-        # only save 6 chars of the hash
-        local commit="${BASH_REMATCH[3]}"
-        version="${BASH_REMATCH[1]}${commit}"
-    fi
-    case $os in
-        debian*|ubuntu*|fedora*)
-            echo "app-e2e-tests-${version}-x86_64-unknown-linux-gnu"
-            ;;
-        windows*)
-            echo "app-e2e-tests-${version}-x86_64-pc-windows-msvc.exe"
-            ;;
-        macos*)
-            echo "app-e2e-tests-${version}-aarch64-apple-darwin"
-            ;;
-        *)
-            echo "Unsupported target: $os" 1>&2
-            return 1
-            ;;
-    esac
-}
-
-function download_e2e_executable {
-    local version=$1
-    local os=$2
-    local package_repo=""
-
-    if is_dev_version "$version"; then
-        package_repo="${BUILD_DEV_REPOSITORY}"
-    else
-        package_repo="${BUILD_RELEASE_REPOSITORY}"
-    fi
-
-    local filename
-    filename=$(get_e2e_filename "$version" "$os")
-    local url="${package_repo}/$version/additional-files/$filename"
-
-    mkdir -p "$PACKAGES_DIR"
-    if [[ ! -f "$PACKAGES_DIR/$filename" ]]; then
-        echo "Downloading e2e executable for $version ($os) from $url"
-        curl -sf -o "$PACKAGES_DIR/$filename" "$url"
-    else
-        echo "Found e2e executable for $version ($os)"
-    fi
-}
-
-function run_tests_for_os {
-    local os=$1
-
-    local prev_filename
-    prev_filename=$(get_app_filename "$OLD_APP_VERSION" "$os")
-    local cur_filename
-    cur_filename=$(get_app_filename "$NEW_APP_VERSION" "$os")
-
-    rm -f "$SCRIPT_DIR/.ci-logs/${os}_report"
-
-    RUST_LOG=debug cargo run --bin test-manager \
-        run-tests \
-        --account "${ACCOUNT_TOKEN:?Error: ACCOUNT_TOKEN not set}" \
-        --app-package "${cur_filename}" \
-        --app-package-to-upgrade-from "${prev_filename}" \
-        --package-folder "$PACKAGES_DIR" \
-        --test-report "$SCRIPT_DIR/.ci-logs/${os}_report" \
-        --vm "$os" 2>&1 | sed "s/${ACCOUNT_TOKEN}/\{ACCOUNT_TOKEN\}/g"
-}
-
-echo "**********************************"
-echo "* Downloading app packages"
-echo "**********************************"
-
-find "$PACKAGES_DIR" -type f -mtime +5 -delete || true
-
-mkdir -p "$PACKAGES_DIR"
-nice_time download_app_package "$OLD_APP_VERSION" "$TEST_OS"
-nice_time download_app_package "$NEW_APP_VERSION" "$TEST_OS"
-nice_time download_e2e_executable "$NEW_APP_VERSION" "$TEST_OS"
-
-echo "**********************************"
-echo "* Building test runner"
-echo "**********************************"
-
-function build_test_runner {
-    if [[ "${TEST_OS}" =~ "debian"|"ubuntu"|"fedora" ]]; then
-        ./container-run.sh ./build-runner.sh linux
-    elif [[ "${TEST_OS}" =~ "windows" ]]; then
-        ./container-run.sh ./build-runner.sh windows
-    elif [[ "${TEST_OS}" =~ "macos" ]]; then
-        ./build-runner.sh macos
-    fi
-}
-
-nice_time build_test_runner
-
-echo "**********************************"
-echo "* Building test manager"
-echo "**********************************"
-
-cargo build -p test-manager
-
-echo "**********************************"
-echo "* Running tests"
-echo "**********************************"
-
-mkdir -p "$SCRIPT_DIR/.ci-logs/os/"
-set -o pipefail
-ACCOUNT_TOKEN=${tokens[0]} nice_time run_tests_for_os "${TEST_OS}"
diff --git a/test/Dockerfile b/test/scripts/Dockerfile
index 9e6fe20f80..9e6fe20f80 100644
--- a/test/Dockerfile
+++ b/test/scripts/Dockerfile
diff --git a/test/scripts/build-runner-image.sh b/test/scripts/build-runner-image.sh
index 4d8b39267d..4aec7b0439 100755
--- a/test/scripts/build-runner-image.sh
+++ b/test/scripts/build-runner-image.sh
@@ -34,7 +34,7 @@ case $TARGET in
             -i "${TEST_RUNNER_IMAGE_PATH}" \
             "${SCRIPT_DIR}/../target/$TARGET/release/test-runner.exe" \
             "${SCRIPT_DIR}/../target/$TARGET/release/connection-checker.exe" \
-            "${PACKAGES_DIR}/"*.exe \
+            "${PACKAGE_DIR}/"*.exe \
             "${SCRIPT_DIR}/../openvpn.ca.crt" \
             "::"
         mdir -i "${TEST_RUNNER_IMAGE_PATH}"
diff --git a/test/build-runner.sh b/test/scripts/build-runner.sh
index ef25d001c3..ffdb2b861a 100755
--- a/test/build-runner.sh
+++ b/test/scripts/build-runner.sh
@@ -3,9 +3,10 @@
 set -eu
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-REPO_DIR="$SCRIPT_DIR/.."
+REPO_DIR="$SCRIPT_DIR/../.."
 cd "$SCRIPT_DIR"
 
+# shellcheck disable=SC1091
 source "$REPO_DIR/scripts/utils/log"
 
 case ${1-:""} in
@@ -34,5 +35,5 @@ cargo build \
 
 # Only build runner image for Windows
 if [[ $TARGET == x86_64-pc-windows-gnu ]]; then
-    TARGET="$TARGET" ./scripts/build-runner-image.sh
+    TARGET="$TARGET" ./build-runner-image.sh
 fi
diff --git a/test/scripts/ci-runtests.sh b/test/scripts/ci-runtests.sh
new file mode 100755
index 0000000000..3e64894e74
--- /dev/null
+++ b/test/scripts/ci-runtests.sh
@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+set -eu
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd "$SCRIPT_DIR"
+TEST_DIR="$SCRIPT_DIR/.."
+
+if [[ "$#" -lt 1 ]]; then
+    echo "usage: $0 TEST_OS" 1>&2
+    exit 1
+fi
+
+TEST_OS=$1
+
+# shellcheck source=test/scripts/test-utils.sh
+source "test-utils.sh"
+
+echo "**********************************"
+echo "* Version to upgrade from: $LATEST_STABLE_RELEASE"
+echo "* Version to test: $CURRENT_VERSION"
+echo "**********************************"
+
+
+if [[ -z "${ACCOUNT_TOKENS+x}" ]]; then
+    echo "'ACCOUNT_TOKENS' must be specified" 1>&2
+    exit 1
+fi
+if ! readarray -t tokens < "${ACCOUNT_TOKENS}"; then
+    echo "Specify account tokens in 'ACCOUNT_TOKENS' file" 1>&2
+    exit 1
+fi
+CI_LOGS_DIR="$TEST_DIR/.ci-logs"
+mkdir -p "$CI_LOGS_DIR"
+echo "$CURRENT_VERSION" > "$CI_LOGS_DIR/last-version.log"
+
+
+echo "**********************************"
+echo "* Downloading app packages"
+echo "**********************************"
+
+
+nice_time download_app_package "$LATEST_STABLE_RELEASE" "$TEST_OS"
+nice_time download_app_package "$CURRENT_VERSION" "$TEST_OS"
+nice_time download_e2e_executable "$CURRENT_VERSION" "$TEST_OS"
+
+echo "**********************************"
+echo "* Building test manager"
+echo "**********************************"
+
+cargo build -p test-manager
+
+echo "**********************************"
+echo "* Running tests"
+echo "**********************************"
+
+mkdir -p "$CI_LOGS_DIR/os/"
+export TEST_REPORT="$CI_LOGS_DIR/${TEST_OS}_report"
+rm -f "$TEST_REPORT"
+
+set -o pipefail
+
+APP_PACKAGE=$(get_app_filename "$CURRENT_VERSION" "$TEST_OS")
+export APP_PACKAGE
+APP_PACKAGE_TO_UPGRADE_FROM=$(get_app_filename "$LATEST_STABLE_RELEASE" "$TEST_OS")
+export APP_PACKAGE_TO_UPGRADE_FROM
+ACCOUNT_TOKEN=${tokens[0]} RUST_LOG=debug nice_time run_tests_for_os "${TEST_OS}"
diff --git a/test/container-run.sh b/test/scripts/container-run.sh
index e4f24a29d9..4f87655123 100755
--- a/test/container-run.sh
+++ b/test/scripts/container-run.sh
@@ -4,16 +4,17 @@ set -eu
 
 CARGO_REGISTRY_VOLUME_NAME=${CARGO_REGISTRY_VOLUME_NAME:-"cargo-registry"}
 CONTAINER_RUNNER=${CONTAINER_RUNNER:-"podman"}
-PACKAGES_DIR=${PACKAGES_DIR:-"$HOME/.cache/mullvad-test/packages"}
+PACKAGE_DIR=${PACKAGE_DIR:-"$HOME/.cache/mullvad-test/packages"}
 
-if [ ! -d "$PACKAGES_DIR" ]; then
-  echo "$PACKAGES_DIR does not exist. It is needed to build the test bundle, so please go ahead and create the directory and re-run this script."
+if [ ! -d "$PACKAGE_DIR" ]; then
+  echo "$PACKAGE_DIR does not exist. It is needed to build the test bundle, so please go ahead and create the directory and re-run this script."
 fi
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-REPO_DIR="$SCRIPT_DIR/.."
+REPO_DIR="$SCRIPT_DIR/../.."
 cd "$SCRIPT_DIR"
 
+# shellcheck disable=SC1091
 source "$REPO_DIR/scripts/utils/log"
 
 if [[ "$(uname -s)" != "Linux" ]]; then
@@ -24,13 +25,12 @@ fi
 container_image=$(cat "$REPO_DIR/building/linux-container-image.txt")
 podman build -t mullvadvpn-app-tests --build-arg IMAGE="${container_image}" .
 
-set -x
 exec "$CONTAINER_RUNNER" run --rm -it \
     -v "${CARGO_REGISTRY_VOLUME_NAME}":/root/.cargo/registry:Z \
     -v "${REPO_DIR}":/build:z \
     -w "/build/test" \
     -e CARGO_TARGET_DIR=/build/test/target \
-    -v "${PACKAGES_DIR}":/packages:Z \
-    -e PACKAGES_DIR=/packages \
+    -v "${PACKAGE_DIR}":/packages:Z \
+    -e PACKAGE_DIR=/packages \
     mullvadvpn-app-tests \
     /bin/bash -c "$*"
diff --git a/test/scripts/test-utils.sh b/test/scripts/test-utils.sh
new file mode 100755
index 0000000000..c73c79e52a
--- /dev/null
+++ b/test/scripts/test-utils.sh
@@ -0,0 +1,312 @@
+#!/usr/bin/env bash
+
+set -eu
+
+# Returns the directory of the test-utils.sh script
+function get_test_utls_dir {
+    local script_path="${BASH_SOURCE[0]}"
+    local script_dir
+    if [[ -n "$script_path" ]]; then
+        script_dir="$(cd "$(dirname "$script_path")" > /dev/null && pwd)"
+    else
+        script_dir="$(cd "$(dirname "$0")" > /dev/null && pwd)"
+    fi
+    echo "$script_dir"
+}
+
+export BUILD_RELEASE_REPOSITORY="https://releases.mullvad.net/desktop/releases"
+export BUILD_DEV_REPOSITORY="https://releases.mullvad.net/desktop/builds"
+
+# Infer stable version from GitHub repo
+RELEASES=$(curl -sf https://api.github.com/repos/mullvad/mullvadvpn-app/releases | jq -r '[.[] | select(((.tag_name|(startswith("android") or startswith("ios"))) | not))]')
+LATEST_STABLE_RELEASE=$(jq -r '[.[] | select(.prerelease==false)] | .[0].tag_name' <<<"$RELEASES")
+
+function get_current_version {
+    local app_dir
+    app_dir="$(get_test_utls_dir)/../.."
+    cargo run -q --manifest-path="$app_dir/Cargo.toml" --bin mullvad-version
+}
+
+CURRENT_VERSION=$(get_current_version)
+commit=$(git rev-parse HEAD^\{commit\})
+commit=${commit:0:6}
+
+TAG=$(git describe --exact-match HEAD 2>/dev/null || echo "")
+
+if [[ -n "$TAG" && ${CURRENT_VERSION} =~ -dev- ]]; then
+    CURRENT_VERSION+="+${TAG}"
+fi
+
+export CURRENT_VERSION
+export LATEST_STABLE_RELEASE
+
+function print_available_releases {
+    for release in $(jq -r '.[].tag_name'<<<"$RELEASES"); do
+        echo "$release"
+    done
+}
+
+function get_package_dir {
+    local package_dir
+    if [[ -n "${PACKAGE_DIR+x}" ]]; then
+        # Resolve the package dir to an absolute path since cargo must be invoked from the test directory
+        package_dir=$(cd "$PACKAGE_DIR" > /dev/null && pwd)
+    elif [[ ("$(uname -s)" == "Darwin") ]]; then
+        package_dir="$HOME/Library/Caches/mullvad-test/packages"
+    elif [[ ("$(uname -s)" == "Linux") ]]; then
+        package_dir="$HOME/.cache/mullvad-test/packages"
+    else
+        echo "Unsupported OS" 1>&2
+        exit 1
+    fi
+
+    mkdir -p  "$package_dir" || exit 1
+    # Clean up old packages
+    find "$package_dir" -type f -mtime +5 -delete || true
+
+    echo "$package_dir"
+    return 0
+}
+
+function nice_time {
+    SECONDS=0
+    if "$@"; then
+        result=0
+    else
+        result=$?
+    fi
+    s=$SECONDS
+    echo "\"$*\" completed in $((s/60))m:$((s%60))s"
+    return $result
+}
+
+# Returns 0 if $1 is a development build. `BASH_REMATCH` contains match groups
+# if that is the case.
+function is_dev_version {
+    local pattern="(^[0-9.]+(-beta[0-9]+)?-dev-)([0-9a-z]+)(\+[0-9a-z|-]+)?$"
+    if [[ "$1" =~ $pattern ]]; then
+        return 0
+    fi
+    return 1
+}
+
+function get_app_filename {
+    local version=$1
+    local os=$2
+    if is_dev_version "$version"; then
+        # only save 6 chars of the hash
+        local commit="${BASH_REMATCH[3]}"
+        version="${BASH_REMATCH[1]}${commit}"
+        # If the dev-version includes a tag, we need to append it to the app filename
+        if [[ -n ${BASH_REMATCH[4]} ]]; then
+            version="${version}${BASH_REMATCH[4]}"
+        fi
+    fi
+    case $os in
+        debian*|ubuntu*)
+            echo "MullvadVPN-${version}_amd64.deb"
+            ;;
+        fedora*)
+            echo "MullvadVPN-${version}_x86_64.rpm"
+            ;;
+        windows*)
+            echo "MullvadVPN-${version}.exe"
+            ;;
+        macos*)
+            echo "MullvadVPN-${version}.pkg"
+            ;;
+        *)
+            echo "Unsupported target: $os" 1>&2
+            return 1
+            ;;
+    esac
+}
+
+function download_app_package {
+    local version=$1
+    local os=$2
+    local package_repo=""
+
+    if is_dev_version "$version"; then
+        package_repo="${BUILD_DEV_REPOSITORY}"
+    else
+        package_repo="${BUILD_RELEASE_REPOSITORY}"
+    fi
+
+    local filename
+    filename=$(get_app_filename "$version" "$os")
+    local url="${package_repo}/$version/$filename"
+
+    local package_dir
+    package_dir=$(get_package_dir)
+    if [[ ! -f "$package_dir/$filename" ]]; then
+        echo "Downloading build for $version ($os) from $url"
+        if ! curl -sf -o "$package_dir/$filename" "$url"; then
+            echo "Failed to download package from $url (hint: build may not exist, check the url)" 1>&2
+            exit 1
+        fi
+    else
+        echo "App package for version $version ($os) already exists at $package_dir/$filename, skipping download"
+    fi
+}
+
+function get_e2e_filename {
+    local version=$1
+    local os=$2
+    if is_dev_version "$version"; then
+        # only save 6 chars of the hash
+        local commit="${BASH_REMATCH[3]}"
+        version="${BASH_REMATCH[1]}${commit}"
+    fi
+    case $os in
+        debian*|ubuntu*|fedora*)
+            echo "app-e2e-tests-${version}-x86_64-unknown-linux-gnu"
+            ;;
+        windows*)
+            echo "app-e2e-tests-${version}-x86_64-pc-windows-msvc.exe"
+            ;;
+        macos*)
+            echo "app-e2e-tests-${version}-aarch64-apple-darwin"
+            ;;
+        *)
+            echo "Unsupported target: $os" 1>&2
+            return 1
+            ;;
+    esac
+}
+
+function download_e2e_executable {
+    local version=${1:?Error: version not set}
+    local os=${2:?Error: os not set}
+    local package_repo
+
+    if is_dev_version "$version"; then
+        package_repo="${BUILD_DEV_REPOSITORY}"
+    else
+        package_repo="${BUILD_RELEASE_REPOSITORY}"
+    fi
+
+    local filename
+    filename=$(get_e2e_filename "$version" "$os")
+    local url="${package_repo}/$version/additional-files/$filename"
+
+    local package_dir
+    package_dir=$(get_package_dir)
+    if [[ ! -f "$package_dir/$filename" ]]; then
+        echo "Downloading e2e executable for $version ($os) from $url"
+        if ! curl -sf -o "$package_dir/$filename" "$url"; then
+            echo "Failed to download package from $url (hint: build may not exist, check the url)" 1>&2
+            exit 1
+        fi
+    else
+        echo "GUI e2e executable for version $version ($os) already exists at $package_dir/$filename, skipping download"
+    fi
+}
+
+function build_test_runner {
+    local script_dir
+    script_dir=$(get_test_utls_dir)
+    local test_os=${1:?Error: test os not set}
+    if [[ "${test_os}" =~ "debian"|"ubuntu"|"fedora" ]]; then
+        "$script_dir"/container-run.sh scripts/build-runner.sh linux || exit 1
+    elif [[ "${test_os}" =~ "windows" ]]; then
+        "$script_dir"/container-run.sh scripts/build-runner.sh windows || exit 1
+    elif [[ "${test_os}" =~ "macos" ]]; then
+        "$script_dir"/build-runner.sh macos || exit 1
+    fi
+}
+
+function run_tests_for_os {
+    local vm=$1
+
+    if [[ -z "${ACCOUNT_TOKEN+x}" ]]; then
+        echo "'ACCOUNT_TOKEN' must be specified" 1>&2
+        exit 1
+    fi
+
+    echo "**********************************"
+    echo "* Building test runner"
+    echo "**********************************"
+
+    nice_time build_test_runner "$vm"
+
+
+    echo "**********************************"
+    echo "* Running tests"
+    echo "**********************************"
+
+    local upgrade_package_arg
+    if [[ -z "${APP_PACKAGE_TO_UPGRADE_FROM+x}" ]]; then
+        echo "'APP_PACKAGE_TO_UPGRADE_FROM' env not set, not testing upgrades"
+        upgrade_package_arg=()
+    else
+        upgrade_package_arg=(--app-package-to-upgrade-from "${APP_PACKAGE_TO_UPGRADE_FROM}")
+    fi
+
+    if [[ -z "${TEST_REPORT+x}" ]]; then
+        echo "'TEST_REPORT' env not set, not saving test report"
+        test_report_arg=()
+    else
+        test_report_arg=(--test-report "${TEST_REPORT}")
+    fi
+
+    local package_dir
+    package_dir=$(get_package_dir)
+    local test_dir
+    test_dir=$(get_test_utls_dir)/..
+    pushd "$test_dir"
+        if ! RUST_LOG_STYLE=always cargo run --bin test-manager \
+            run-tests \
+            --account "${ACCOUNT_TOKEN:?Error: ACCOUNT_TOKEN not set}" \
+            --app-package "${APP_PACKAGE:?Error: APP_PACKAGE not set}" \
+            "${upgrade_package_arg[@]}" \
+            "${test_report_arg[@]}" \
+            --package-dir "${package_dir}" \
+            --vm "$vm" \
+            "${TEST_FILTERS:-}" \
+            2>&1 | sed -r "s/${ACCOUNT_TOKEN}/\{ACCOUNT_TOKEN\}/g"; then
+            echo "Test run failed"
+            exit 1
+        fi
+    popd
+}
+
+# Build the current version of the app and move the package to the package folder
+# Currently unused, but may be useful in the future
+function build_current_version {
+    local app_dir
+    app_dir="$(get_test_utls_dir)/../.."
+    local app_filename
+    # TODO: TEST_OS must be set to local OS manually, should be set automatically
+    app_filename=$(get_app_filename "$CURRENT_VERSION" "${TEST_OS:?Error: TEST_OS not set}") 
+    local package_dir
+    package_dir=$(get_package_dir)
+    local app_package="$package_dir"/"$app_filename"
+
+    local gui_test_filename
+    gui_test_filename=$(get_e2e_filename "$CURRENT_VERSION" "$TEST_OS")
+    local gui_test_bin="$package_dir"/"$gui_test_filename"
+
+    if [ ! -f "$app_package" ]; then
+        pushd "$app_dir"
+            if [[ $(git diff --quiet) ]]; then
+                echo "WARNING: the app repository contains uncommitted changes, this script will only rebuild the app package when the git hash changes"
+            fi
+            ./build.sh
+        popd
+        echo "Moving '$(realpath "$app_dir/dist/$app_filename")' to '$(realpath "$app_package")'"
+        mv -n "$app_dir"/dist/"$app_filename" "$app_package"
+    else
+        echo "App package for current version already exists at $app_package, skipping build"
+    fi
+
+    if [ ! -f "$gui_test_bin" ]; then
+        pushd "$app_dir"/gui
+            npm run build-test-executable
+        popd
+        echo "Moving '$(realpath "$app_dir/dist/$gui_test_filename")' to '$(realpath "$gui_test_bin")'"
+        mv -n "$app_dir"/dist/"$gui_test_filename" "$gui_test_bin"
+    else
+        echo "GUI e2e executable for current version already exists at $gui_test_bin, skipping build"
+    fi
+}
\ No newline at end of file
diff --git a/test/test-by-version.sh b/test/test-by-version.sh
new file mode 100755
index 0000000000..a86dac8ced
--- /dev/null
+++ b/test/test-by-version.sh
@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+
+set -eu
+
+usage() {
+    echo "This script downloads and tests the given app version from the build repositories."
+    echo
+    echo "Required environment variables:"
+    echo "  - ACCOUNT_TOKEN: Valid MullvadVPN account token"
+    echo "  - TEST_OS: Name of the VM configuration to use. List available configurations with 'cargo run --bin test-manager list'"
+    echo "Optional environment variables:"
+    echo "  - APP_VERSION: The version of the app to test (defaults to the latest stable release)"
+    echo "  - APP_PACKAGE_TO_UPGRADE_FROM: The package version to upgrade from (defaults to none)"
+    echo "  - TEST_FILTERS: specifies which tests to run (defaults to all)"
+    echo "  - TEST_REPORT : path to save the test results in a structured format"
+}
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd "$SCRIPT_DIR"
+
+# shellcheck source=test/scripts/test-utils.sh
+source "scripts/test-utils.sh"
+
+if [[ ( "$*" == "--help") ||  "$*" == "-h" ]]; then
+    usage
+    exit 0
+fi
+
+if [[ -z "${ACCOUNT_TOKEN+x}" ]]; then
+    echo "'ACCOUNT_TOKEN' must be specified" 1>&2
+    echo
+    usage
+    exit 1
+fi
+
+if [[ -z "${TEST_OS+x}" ]]; then
+    echo "'TEST_OS' must be specified" 1>&2
+    echo
+    usage
+    exit 1
+fi
+
+if [[ -z "${APP_VERSION+x}" ]]; then
+    echo "'APP_VERSION' not set, using latest build from the list of GitHub releases:"
+    print_available_releases
+    echo "For a full list of available releases you can choose from, see the stable build repository: $BUILD_RELEASE_REPOSITORY"
+    echo "and the dev build repository: $BUILD_DEV_REPOSITORY"
+    APP_VERSION=$LATEST_STABLE_RELEASE
+fi
+
+echo "**********************************"
+echo "* Version to test: $APP_VERSION"
+echo "**********************************"
+
+echo "**********************************"
+echo "* Downloading app packages"
+echo "**********************************"
+
+download_app_package "$APP_VERSION" "$TEST_OS"
+download_e2e_executable "$APP_VERSION" "$TEST_OS"
+
+if [[ -n "${APP_PACKAGE_TO_UPGRADE_FROM+x}" ]]; then
+    download_app_package "$APP_PACKAGE_TO_UPGRADE_FROM" "$TEST_OS"
+fi
+
+set -o pipefail
+APP_PACKAGE=$(get_app_filename "$APP_VERSION" "$TEST_OS")
+export APP_PACKAGE
+run_tests_for_os "${TEST_OS}"
diff --git a/test/test-manager/src/main.rs b/test/test-manager/src/main.rs
index bd9954bc1e..daad04983b 100644
--- a/test/test-manager/src/main.rs
+++ b/test/test-manager/src/main.rs
@@ -82,7 +82,7 @@ enum Commands {
 
         /// App package to test. Can be a path to the package, just the package file name, git hash
         /// or tag. If the direct path is not given, the package is assumed to be in the directory
-        /// specified by the `--package-folder` argument.
+        /// specified by the `--package-dir` argument.
         ///
         /// # Note
         ///
@@ -108,7 +108,7 @@ enum Commands {
 
         /// Folder to search for packages. Defaults to current directory.
         #[arg(long, value_name = "DIR")]
-        package_folder: Option<PathBuf>,
+        package_dir: Option<PathBuf>,
 
         /// Only run tests matching substrings
         test_filters: Vec<String>,
@@ -227,7 +227,7 @@ async fn main() -> Result<()> {
             app_package,
             app_package_to_upgrade_from,
             gui_package,
-            package_folder,
+            package_dir,
             test_filters,
             verbose,
             test_report,
@@ -266,7 +266,7 @@ async fn main() -> Result<()> {
                 app_package,
                 app_package_to_upgrade_from,
                 gui_package,
-                package_folder,
+                package_dir,
             )
             .context("Could not find the specified app packages")?;
 
diff --git a/test/test-manager/src/package.rs b/test/test-manager/src/package.rs
index b6147d7dd8..34fb30419a 100644
--- a/test/test-manager/src/package.rs
+++ b/test/test-manager/src/package.rs
@@ -21,22 +21,22 @@ pub fn get_app_manifest(
     app_package: String,
     app_package_to_upgrade_from: Option<String>,
     gui_package: Option<String>,
-    package_folder: Option<PathBuf>,
+    package_dir: Option<PathBuf>,
 ) -> Result<Manifest> {
     let package_type = (config.os_type, config.package_type, config.architecture);
 
-    let app_package_path = find_app(&app_package, false, package_type, package_folder.as_ref())?;
+    let app_package_path = find_app(&app_package, false, package_type, package_dir.as_ref())?;
     log::info!("App package: {}", app_package_path.display());
 
     let app_package_to_upgrade_from_path = app_package_to_upgrade_from
-        .map(|app| find_app(&app, false, package_type, package_folder.as_ref()))
+        .map(|app| find_app(&app, false, package_type, package_dir.as_ref()))
         .transpose()?;
     log::info!("App package to upgrade from: {app_package_to_upgrade_from_path:?}");
 
     // Automatically try to find the UI e2e tests based on the app package
 
     // Search the specified package folder, or same folder as the app package if missing
-    let ui_e2e_package_folder = package_folder.unwrap_or(
+    let ui_e2e_package_dir = package_dir.unwrap_or(
         app_package_path
             .parent()
             .expect("Path to app package should have parent")
@@ -51,7 +51,7 @@ pub fn get_app_manifest(
         },
         true,
         package_type,
-        Some(&ui_e2e_package_folder),
+        Some(&ui_e2e_package_dir),
     );
 
     // Don't allow the UI/e2e test binary to missing if it's flag was specified
@@ -85,7 +85,7 @@ fn find_app(
     app: &str,
     e2e_bin: bool,
     package_type: (OsType, Option<PackageType>, Option<Architecture>),
-    package_folder: Option<&PathBuf>,
+    package_dir: Option<&PathBuf>,
 ) -> Result<PathBuf> {
     // If it's a path, use that path
     let app_path = Path::new(app);
@@ -98,7 +98,7 @@ fn find_app(
     app.make_ascii_lowercase();
 
     let current_dir = std::env::current_dir().expect("Unable to get current directory");
-    let packages_dir = package_folder.unwrap_or(&current_dir);
+    let packages_dir = package_dir.unwrap_or(&current_dir);
     std::fs::create_dir_all(packages_dir)?;
     let dir = std::fs::read_dir(packages_dir.clone()).context("Failed to list packages")?;