Restructure test scripts

Split functionality `ci-runtest.sh` into multiple scripts.
`test-by-version.sh` can be used to test against any version
of the app available on the build servers. `test-utils.sh` contains
shared logic.

Rename `PACKAGES_DIR` env `PACKAGE_DIR`,
it's more consistent with the new CLI flag.

6 files changed, 462 insertions, 1 deletions

diff --git a/test/scripts/Dockerfile b/test/scripts/Dockerfile
new file mode 100644
index 0000000000..9e6fe20f80
--- /dev/null
+++ b/test/scripts/Dockerfile
@@ -0,0 +1,7 @@
+ARG IMAGE=ghcr.io/mullvad/mullvadvpn-app-build:latest
+FROM $IMAGE
+
+RUN rustup target add x86_64-pc-windows-gnu
+
+RUN apt-get update && apt-get install -y \
+    mtools pkg-config libssl-dev libpcap-dev
diff --git a/test/scripts/build-runner-image.sh b/test/scripts/build-runner-image.sh
index 4d8b39267d..4aec7b0439 100755
--- a/test/scripts/build-runner-image.sh
+++ b/test/scripts/build-runner-image.sh
@@ -34,7 +34,7 @@ case $TARGET in
             -i "${TEST_RUNNER_IMAGE_PATH}" \
             "${SCRIPT_DIR}/../target/$TARGET/release/test-runner.exe" \
             "${SCRIPT_DIR}/../target/$TARGET/release/connection-checker.exe" \
-            "${PACKAGES_DIR}/"*.exe \
+            "${PACKAGE_DIR}/"*.exe \
             "${SCRIPT_DIR}/../openvpn.ca.crt" \
             "::"
         mdir -i "${TEST_RUNNER_IMAGE_PATH}"
diff --git a/test/scripts/build-runner.sh b/test/scripts/build-runner.sh
new file mode 100755
index 0000000000..ffdb2b861a
--- /dev/null
+++ b/test/scripts/build-runner.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -eu
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+REPO_DIR="$SCRIPT_DIR/../.."
+cd "$SCRIPT_DIR"
+
+# shellcheck disable=SC1091
+source "$REPO_DIR/scripts/utils/log"
+
+case ${1-:""} in
+    linux)
+        TARGET=x86_64-unknown-linux-gnu
+        shift
+    ;;
+    windows)
+        TARGET=x86_64-pc-windows-gnu
+        shift
+    ;;
+    macos)
+        # TODO: x86
+        TARGET=aarch64-apple-darwin
+        shift
+    ;;
+    *)
+        log_error "Invalid platform. Specify a valid platform as first argument"
+        exit 1
+esac
+
+cargo build \
+    --bin test-runner \
+    --bin connection-checker \
+    --release --target "${TARGET}"
+
+# Only build runner image for Windows
+if [[ $TARGET == x86_64-pc-windows-gnu ]]; then
+    TARGET="$TARGET" ./build-runner-image.sh
+fi
diff --git a/test/scripts/ci-runtests.sh b/test/scripts/ci-runtests.sh
new file mode 100755
index 0000000000..3e64894e74
--- /dev/null
+++ b/test/scripts/ci-runtests.sh
@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+set -eu
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+cd "$SCRIPT_DIR"
+TEST_DIR="$SCRIPT_DIR/.."
+
+if [[ "$#" -lt 1 ]]; then
+    echo "usage: $0 TEST_OS" 1>&2
+    exit 1
+fi
+
+TEST_OS=$1
+
+# shellcheck source=test/scripts/test-utils.sh
+source "test-utils.sh"
+
+echo "**********************************"
+echo "* Version to upgrade from: $LATEST_STABLE_RELEASE"
+echo "* Version to test: $CURRENT_VERSION"
+echo "**********************************"
+
+
+if [[ -z "${ACCOUNT_TOKENS+x}" ]]; then
+    echo "'ACCOUNT_TOKENS' must be specified" 1>&2
+    exit 1
+fi
+if ! readarray -t tokens < "${ACCOUNT_TOKENS}"; then
+    echo "Specify account tokens in 'ACCOUNT_TOKENS' file" 1>&2
+    exit 1
+fi
+CI_LOGS_DIR="$TEST_DIR/.ci-logs"
+mkdir -p "$CI_LOGS_DIR"
+echo "$CURRENT_VERSION" > "$CI_LOGS_DIR/last-version.log"
+
+
+echo "**********************************"
+echo "* Downloading app packages"
+echo "**********************************"
+
+
+nice_time download_app_package "$LATEST_STABLE_RELEASE" "$TEST_OS"
+nice_time download_app_package "$CURRENT_VERSION" "$TEST_OS"
+nice_time download_e2e_executable "$CURRENT_VERSION" "$TEST_OS"
+
+echo "**********************************"
+echo "* Building test manager"
+echo "**********************************"
+
+cargo build -p test-manager
+
+echo "**********************************"
+echo "* Running tests"
+echo "**********************************"
+
+mkdir -p "$CI_LOGS_DIR/os/"
+export TEST_REPORT="$CI_LOGS_DIR/${TEST_OS}_report"
+rm -f "$TEST_REPORT"
+
+set -o pipefail
+
+APP_PACKAGE=$(get_app_filename "$CURRENT_VERSION" "$TEST_OS")
+export APP_PACKAGE
+APP_PACKAGE_TO_UPGRADE_FROM=$(get_app_filename "$LATEST_STABLE_RELEASE" "$TEST_OS")
+export APP_PACKAGE_TO_UPGRADE_FROM
+ACCOUNT_TOKEN=${tokens[0]} RUST_LOG=debug nice_time run_tests_for_os "${TEST_OS}"
diff --git a/test/scripts/container-run.sh b/test/scripts/container-run.sh
new file mode 100755
index 0000000000..4f87655123
--- /dev/null
+++ b/test/scripts/container-run.sh
@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+set -eu
+
+CARGO_REGISTRY_VOLUME_NAME=${CARGO_REGISTRY_VOLUME_NAME:-"cargo-registry"}
+CONTAINER_RUNNER=${CONTAINER_RUNNER:-"podman"}
+PACKAGE_DIR=${PACKAGE_DIR:-"$HOME/.cache/mullvad-test/packages"}
+
+if [ ! -d "$PACKAGE_DIR" ]; then
+  echo "$PACKAGE_DIR does not exist. It is needed to build the test bundle, so please go ahead and create the directory and re-run this script."
+fi
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+REPO_DIR="$SCRIPT_DIR/../.."
+cd "$SCRIPT_DIR"
+
+# shellcheck disable=SC1091
+source "$REPO_DIR/scripts/utils/log"
+
+if [[ "$(uname -s)" != "Linux" ]]; then
+    log_error "$0 only works on Linux"
+    exit 1
+fi
+
+container_image=$(cat "$REPO_DIR/building/linux-container-image.txt")
+podman build -t mullvadvpn-app-tests --build-arg IMAGE="${container_image}" .
+
+exec "$CONTAINER_RUNNER" run --rm -it \
+    -v "${CARGO_REGISTRY_VOLUME_NAME}":/root/.cargo/registry:Z \
+    -v "${REPO_DIR}":/build:z \
+    -w "/build/test" \
+    -e CARGO_TARGET_DIR=/build/test/target \
+    -v "${PACKAGE_DIR}":/packages:Z \
+    -e PACKAGE_DIR=/packages \
+    mullvadvpn-app-tests \
+    /bin/bash -c "$*"
diff --git a/test/scripts/test-utils.sh b/test/scripts/test-utils.sh
new file mode 100755
index 0000000000..c73c79e52a
--- /dev/null
+++ b/test/scripts/test-utils.sh
@@ -0,0 +1,312 @@
+#!/usr/bin/env bash
+
+set -eu
+
+# Returns the directory of the test-utils.sh script
+function get_test_utls_dir {
+    local script_path="${BASH_SOURCE[0]}"
+    local script_dir
+    if [[ -n "$script_path" ]]; then
+        script_dir="$(cd "$(dirname "$script_path")" > /dev/null && pwd)"
+    else
+        script_dir="$(cd "$(dirname "$0")" > /dev/null && pwd)"
+    fi
+    echo "$script_dir"
+}
+
+export BUILD_RELEASE_REPOSITORY="https://releases.mullvad.net/desktop/releases"
+export BUILD_DEV_REPOSITORY="https://releases.mullvad.net/desktop/builds"
+
+# Infer stable version from GitHub repo
+RELEASES=$(curl -sf https://api.github.com/repos/mullvad/mullvadvpn-app/releases | jq -r '[.[] | select(((.tag_name|(startswith("android") or startswith("ios"))) | not))]')
+LATEST_STABLE_RELEASE=$(jq -r '[.[] | select(.prerelease==false)] | .[0].tag_name' <<<"$RELEASES")
+
+function get_current_version {
+    local app_dir
+    app_dir="$(get_test_utls_dir)/../.."
+    cargo run -q --manifest-path="$app_dir/Cargo.toml" --bin mullvad-version
+}
+
+CURRENT_VERSION=$(get_current_version)
+commit=$(git rev-parse HEAD^\{commit\})
+commit=${commit:0:6}
+
+TAG=$(git describe --exact-match HEAD 2>/dev/null || echo "")
+
+if [[ -n "$TAG" && ${CURRENT_VERSION} =~ -dev- ]]; then
+    CURRENT_VERSION+="+${TAG}"
+fi
+
+export CURRENT_VERSION
+export LATEST_STABLE_RELEASE
+
+function print_available_releases {
+    for release in $(jq -r '.[].tag_name'<<<"$RELEASES"); do
+        echo "$release"
+    done
+}
+
+function get_package_dir {
+    local package_dir
+    if [[ -n "${PACKAGE_DIR+x}" ]]; then
+        # Resolve the package dir to an absolute path since cargo must be invoked from the test directory
+        package_dir=$(cd "$PACKAGE_DIR" > /dev/null && pwd)
+    elif [[ ("$(uname -s)" == "Darwin") ]]; then
+        package_dir="$HOME/Library/Caches/mullvad-test/packages"
+    elif [[ ("$(uname -s)" == "Linux") ]]; then
+        package_dir="$HOME/.cache/mullvad-test/packages"
+    else
+        echo "Unsupported OS" 1>&2
+        exit 1
+    fi
+
+    mkdir -p  "$package_dir" || exit 1
+    # Clean up old packages
+    find "$package_dir" -type f -mtime +5 -delete || true
+
+    echo "$package_dir"
+    return 0
+}
+
+function nice_time {
+    SECONDS=0
+    if "$@"; then
+        result=0
+    else
+        result=$?
+    fi
+    s=$SECONDS
+    echo "\"$*\" completed in $((s/60))m:$((s%60))s"
+    return $result
+}
+
+# Returns 0 if $1 is a development build. `BASH_REMATCH` contains match groups
+# if that is the case.
+function is_dev_version {
+    local pattern="(^[0-9.]+(-beta[0-9]+)?-dev-)([0-9a-z]+)(\+[0-9a-z|-]+)?$"
+    if [[ "$1" =~ $pattern ]]; then
+        return 0
+    fi
+    return 1
+}
+
+function get_app_filename {
+    local version=$1
+    local os=$2
+    if is_dev_version "$version"; then
+        # only save 6 chars of the hash
+        local commit="${BASH_REMATCH[3]}"
+        version="${BASH_REMATCH[1]}${commit}"
+        # If the dev-version includes a tag, we need to append it to the app filename
+        if [[ -n ${BASH_REMATCH[4]} ]]; then
+            version="${version}${BASH_REMATCH[4]}"
+        fi
+    fi
+    case $os in
+        debian*|ubuntu*)
+            echo "MullvadVPN-${version}_amd64.deb"
+            ;;
+        fedora*)
+            echo "MullvadVPN-${version}_x86_64.rpm"
+            ;;
+        windows*)
+            echo "MullvadVPN-${version}.exe"
+            ;;
+        macos*)
+            echo "MullvadVPN-${version}.pkg"
+            ;;
+        *)
+            echo "Unsupported target: $os" 1>&2
+            return 1
+            ;;
+    esac
+}
+
+function download_app_package {
+    local version=$1
+    local os=$2
+    local package_repo=""
+
+    if is_dev_version "$version"; then
+        package_repo="${BUILD_DEV_REPOSITORY}"
+    else
+        package_repo="${BUILD_RELEASE_REPOSITORY}"
+    fi
+
+    local filename
+    filename=$(get_app_filename "$version" "$os")
+    local url="${package_repo}/$version/$filename"
+
+    local package_dir
+    package_dir=$(get_package_dir)
+    if [[ ! -f "$package_dir/$filename" ]]; then
+        echo "Downloading build for $version ($os) from $url"
+        if ! curl -sf -o "$package_dir/$filename" "$url"; then
+            echo "Failed to download package from $url (hint: build may not exist, check the url)" 1>&2
+            exit 1
+        fi
+    else
+        echo "App package for version $version ($os) already exists at $package_dir/$filename, skipping download"
+    fi
+}
+
+function get_e2e_filename {
+    local version=$1
+    local os=$2
+    if is_dev_version "$version"; then
+        # only save 6 chars of the hash
+        local commit="${BASH_REMATCH[3]}"
+        version="${BASH_REMATCH[1]}${commit}"
+    fi
+    case $os in
+        debian*|ubuntu*|fedora*)
+            echo "app-e2e-tests-${version}-x86_64-unknown-linux-gnu"
+            ;;
+        windows*)
+            echo "app-e2e-tests-${version}-x86_64-pc-windows-msvc.exe"
+            ;;
+        macos*)
+            echo "app-e2e-tests-${version}-aarch64-apple-darwin"
+            ;;
+        *)
+            echo "Unsupported target: $os" 1>&2
+            return 1
+            ;;
+    esac
+}
+
+function download_e2e_executable {
+    local version=${1:?Error: version not set}
+    local os=${2:?Error: os not set}
+    local package_repo
+
+    if is_dev_version "$version"; then
+        package_repo="${BUILD_DEV_REPOSITORY}"
+    else
+        package_repo="${BUILD_RELEASE_REPOSITORY}"
+    fi
+
+    local filename
+    filename=$(get_e2e_filename "$version" "$os")
+    local url="${package_repo}/$version/additional-files/$filename"
+
+    local package_dir
+    package_dir=$(get_package_dir)
+    if [[ ! -f "$package_dir/$filename" ]]; then
+        echo "Downloading e2e executable for $version ($os) from $url"
+        if ! curl -sf -o "$package_dir/$filename" "$url"; then
+            echo "Failed to download package from $url (hint: build may not exist, check the url)" 1>&2
+            exit 1
+        fi
+    else
+        echo "GUI e2e executable for version $version ($os) already exists at $package_dir/$filename, skipping download"
+    fi
+}
+
+function build_test_runner {
+    local script_dir
+    script_dir=$(get_test_utls_dir)
+    local test_os=${1:?Error: test os not set}
+    if [[ "${test_os}" =~ "debian"|"ubuntu"|"fedora" ]]; then
+        "$script_dir"/container-run.sh scripts/build-runner.sh linux || exit 1
+    elif [[ "${test_os}" =~ "windows" ]]; then
+        "$script_dir"/container-run.sh scripts/build-runner.sh windows || exit 1
+    elif [[ "${test_os}" =~ "macos" ]]; then
+        "$script_dir"/build-runner.sh macos || exit 1
+    fi
+}
+
+function run_tests_for_os {
+    local vm=$1
+
+    if [[ -z "${ACCOUNT_TOKEN+x}" ]]; then
+        echo "'ACCOUNT_TOKEN' must be specified" 1>&2
+        exit 1
+    fi
+
+    echo "**********************************"
+    echo "* Building test runner"
+    echo "**********************************"
+
+    nice_time build_test_runner "$vm"
+
+
+    echo "**********************************"
+    echo "* Running tests"
+    echo "**********************************"
+
+    local upgrade_package_arg
+    if [[ -z "${APP_PACKAGE_TO_UPGRADE_FROM+x}" ]]; then
+        echo "'APP_PACKAGE_TO_UPGRADE_FROM' env not set, not testing upgrades"
+        upgrade_package_arg=()
+    else
+        upgrade_package_arg=(--app-package-to-upgrade-from "${APP_PACKAGE_TO_UPGRADE_FROM}")
+    fi
+
+    if [[ -z "${TEST_REPORT+x}" ]]; then
+        echo "'TEST_REPORT' env not set, not saving test report"
+        test_report_arg=()
+    else
+        test_report_arg=(--test-report "${TEST_REPORT}")
+    fi
+
+    local package_dir
+    package_dir=$(get_package_dir)
+    local test_dir
+    test_dir=$(get_test_utls_dir)/..
+    pushd "$test_dir"
+        if ! RUST_LOG_STYLE=always cargo run --bin test-manager \
+            run-tests \
+            --account "${ACCOUNT_TOKEN:?Error: ACCOUNT_TOKEN not set}" \
+            --app-package "${APP_PACKAGE:?Error: APP_PACKAGE not set}" \
+            "${upgrade_package_arg[@]}" \
+            "${test_report_arg[@]}" \
+            --package-dir "${package_dir}" \
+            --vm "$vm" \
+            "${TEST_FILTERS:-}" \
+            2>&1 | sed -r "s/${ACCOUNT_TOKEN}/\{ACCOUNT_TOKEN\}/g"; then
+            echo "Test run failed"
+            exit 1
+        fi
+    popd
+}
+
+# Build the current version of the app and move the package to the package folder
+# Currently unused, but may be useful in the future
+function build_current_version {
+    local app_dir
+    app_dir="$(get_test_utls_dir)/../.."
+    local app_filename
+    # TODO: TEST_OS must be set to local OS manually, should be set automatically
+    app_filename=$(get_app_filename "$CURRENT_VERSION" "${TEST_OS:?Error: TEST_OS not set}") 
+    local package_dir
+    package_dir=$(get_package_dir)
+    local app_package="$package_dir"/"$app_filename"
+
+    local gui_test_filename
+    gui_test_filename=$(get_e2e_filename "$CURRENT_VERSION" "$TEST_OS")
+    local gui_test_bin="$package_dir"/"$gui_test_filename"
+
+    if [ ! -f "$app_package" ]; then
+        pushd "$app_dir"
+            if [[ $(git diff --quiet) ]]; then
+                echo "WARNING: the app repository contains uncommitted changes, this script will only rebuild the app package when the git hash changes"
+            fi
+            ./build.sh
+        popd
+        echo "Moving '$(realpath "$app_dir/dist/$app_filename")' to '$(realpath "$app_package")'"
+        mv -n "$app_dir"/dist/"$app_filename" "$app_package"
+    else
+        echo "App package for current version already exists at $app_package, skipping build"
+    fi
+
+    if [ ! -f "$gui_test_bin" ]; then
+        pushd "$app_dir"/gui
+            npm run build-test-executable
+        popd
+        echo "Moving '$(realpath "$app_dir/dist/$gui_test_filename")' to '$(realpath "$gui_test_bin")'"
+        mv -n "$app_dir"/dist/"$gui_test_filename" "$gui_test_bin"
+    else
+        echo "GUI e2e executable for current version already exists at $gui_test_bin, skipping build"
+    fi
+}
\ No newline at end of file