diff options
Diffstat (limited to 'test/scripts')
| -rw-r--r-- | test/scripts/Dockerfile | 7 | ||||
| -rwxr-xr-x | test/scripts/build-runner-image.sh | 2 | ||||
| -rwxr-xr-x | test/scripts/build-runner.sh | 39 | ||||
| -rwxr-xr-x | test/scripts/ci-runtests.sh | 67 | ||||
| -rwxr-xr-x | test/scripts/container-run.sh | 36 | ||||
| -rwxr-xr-x | test/scripts/test-utils.sh | 312 |
6 files changed, 462 insertions, 1 deletions
diff --git a/test/scripts/Dockerfile b/test/scripts/Dockerfile new file mode 100644 index 0000000000..9e6fe20f80 --- /dev/null +++ b/test/scripts/Dockerfile @@ -0,0 +1,7 @@ +ARG IMAGE=ghcr.io/mullvad/mullvadvpn-app-build:latest +FROM $IMAGE + +RUN rustup target add x86_64-pc-windows-gnu + +RUN apt-get update && apt-get install -y \ + mtools pkg-config libssl-dev libpcap-dev diff --git a/test/scripts/build-runner-image.sh b/test/scripts/build-runner-image.sh index 4d8b39267d..4aec7b0439 100755 --- a/test/scripts/build-runner-image.sh +++ b/test/scripts/build-runner-image.sh @@ -34,7 +34,7 @@ case $TARGET in -i "${TEST_RUNNER_IMAGE_PATH}" \ "${SCRIPT_DIR}/../target/$TARGET/release/test-runner.exe" \ "${SCRIPT_DIR}/../target/$TARGET/release/connection-checker.exe" \ - "${PACKAGES_DIR}/"*.exe \ + "${PACKAGE_DIR}/"*.exe \ "${SCRIPT_DIR}/../openvpn.ca.crt" \ "::" mdir -i "${TEST_RUNNER_IMAGE_PATH}" diff --git a/test/scripts/build-runner.sh b/test/scripts/build-runner.sh new file mode 100755 index 0000000000..ffdb2b861a --- /dev/null +++ b/test/scripts/build-runner.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -eu + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +REPO_DIR="$SCRIPT_DIR/../.." +cd "$SCRIPT_DIR" + +# shellcheck disable=SC1091 +source "$REPO_DIR/scripts/utils/log" + +case ${1-:""} in + linux) + TARGET=x86_64-unknown-linux-gnu + shift + ;; + windows) + TARGET=x86_64-pc-windows-gnu + shift + ;; + macos) + # TODO: x86 + TARGET=aarch64-apple-darwin + shift + ;; + *) + log_error "Invalid platform. Specify a valid platform as first argument" + exit 1 +esac + +cargo build \ + --bin test-runner \ + --bin connection-checker \ + --release --target "${TARGET}" + +# Only build runner image for Windows +if [[ $TARGET == x86_64-pc-windows-gnu ]]; then + TARGET="$TARGET" ./build-runner-image.sh +fi diff --git a/test/scripts/ci-runtests.sh b/test/scripts/ci-runtests.sh new file mode 100755 index 0000000000..3e64894e74 --- /dev/null +++ b/test/scripts/ci-runtests.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +set -eu + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd "$SCRIPT_DIR" +TEST_DIR="$SCRIPT_DIR/.." + +if [[ "$#" -lt 1 ]]; then + echo "usage: $0 TEST_OS" 1>&2 + exit 1 +fi + +TEST_OS=$1 + +# shellcheck source=test/scripts/test-utils.sh +source "test-utils.sh" + +echo "**********************************" +echo "* Version to upgrade from: $LATEST_STABLE_RELEASE" +echo "* Version to test: $CURRENT_VERSION" +echo "**********************************" + + +if [[ -z "${ACCOUNT_TOKENS+x}" ]]; then + echo "'ACCOUNT_TOKENS' must be specified" 1>&2 + exit 1 +fi +if ! readarray -t tokens < "${ACCOUNT_TOKENS}"; then + echo "Specify account tokens in 'ACCOUNT_TOKENS' file" 1>&2 + exit 1 +fi +CI_LOGS_DIR="$TEST_DIR/.ci-logs" +mkdir -p "$CI_LOGS_DIR" +echo "$CURRENT_VERSION" > "$CI_LOGS_DIR/last-version.log" + + +echo "**********************************" +echo "* Downloading app packages" +echo "**********************************" + + +nice_time download_app_package "$LATEST_STABLE_RELEASE" "$TEST_OS" +nice_time download_app_package "$CURRENT_VERSION" "$TEST_OS" +nice_time download_e2e_executable "$CURRENT_VERSION" "$TEST_OS" + +echo "**********************************" +echo "* Building test manager" +echo "**********************************" + +cargo build -p test-manager + +echo "**********************************" +echo "* Running tests" +echo "**********************************" + +mkdir -p "$CI_LOGS_DIR/os/" +export TEST_REPORT="$CI_LOGS_DIR/${TEST_OS}_report" +rm -f "$TEST_REPORT" + +set -o pipefail + +APP_PACKAGE=$(get_app_filename "$CURRENT_VERSION" "$TEST_OS") +export APP_PACKAGE +APP_PACKAGE_TO_UPGRADE_FROM=$(get_app_filename "$LATEST_STABLE_RELEASE" "$TEST_OS") +export APP_PACKAGE_TO_UPGRADE_FROM +ACCOUNT_TOKEN=${tokens[0]} RUST_LOG=debug nice_time run_tests_for_os "${TEST_OS}" diff --git a/test/scripts/container-run.sh b/test/scripts/container-run.sh new file mode 100755 index 0000000000..4f87655123 --- /dev/null +++ b/test/scripts/container-run.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +set -eu + +CARGO_REGISTRY_VOLUME_NAME=${CARGO_REGISTRY_VOLUME_NAME:-"cargo-registry"} +CONTAINER_RUNNER=${CONTAINER_RUNNER:-"podman"} +PACKAGE_DIR=${PACKAGE_DIR:-"$HOME/.cache/mullvad-test/packages"} + +if [ ! -d "$PACKAGE_DIR" ]; then + echo "$PACKAGE_DIR does not exist. It is needed to build the test bundle, so please go ahead and create the directory and re-run this script." +fi + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +REPO_DIR="$SCRIPT_DIR/../.." +cd "$SCRIPT_DIR" + +# shellcheck disable=SC1091 +source "$REPO_DIR/scripts/utils/log" + +if [[ "$(uname -s)" != "Linux" ]]; then + log_error "$0 only works on Linux" + exit 1 +fi + +container_image=$(cat "$REPO_DIR/building/linux-container-image.txt") +podman build -t mullvadvpn-app-tests --build-arg IMAGE="${container_image}" . + +exec "$CONTAINER_RUNNER" run --rm -it \ + -v "${CARGO_REGISTRY_VOLUME_NAME}":/root/.cargo/registry:Z \ + -v "${REPO_DIR}":/build:z \ + -w "/build/test" \ + -e CARGO_TARGET_DIR=/build/test/target \ + -v "${PACKAGE_DIR}":/packages:Z \ + -e PACKAGE_DIR=/packages \ + mullvadvpn-app-tests \ + /bin/bash -c "$*" diff --git a/test/scripts/test-utils.sh b/test/scripts/test-utils.sh new file mode 100755 index 0000000000..c73c79e52a --- /dev/null +++ b/test/scripts/test-utils.sh @@ -0,0 +1,312 @@ +#!/usr/bin/env bash + +set -eu + +# Returns the directory of the test-utils.sh script +function get_test_utls_dir { + local script_path="${BASH_SOURCE[0]}" + local script_dir + if [[ -n "$script_path" ]]; then + script_dir="$(cd "$(dirname "$script_path")" > /dev/null && pwd)" + else + script_dir="$(cd "$(dirname "$0")" > /dev/null && pwd)" + fi + echo "$script_dir" +} + +export BUILD_RELEASE_REPOSITORY="https://releases.mullvad.net/desktop/releases" +export BUILD_DEV_REPOSITORY="https://releases.mullvad.net/desktop/builds" + +# Infer stable version from GitHub repo +RELEASES=$(curl -sf https://api.github.com/repos/mullvad/mullvadvpn-app/releases | jq -r '[.[] | select(((.tag_name|(startswith("android") or startswith("ios"))) | not))]') +LATEST_STABLE_RELEASE=$(jq -r '[.[] | select(.prerelease==false)] | .[0].tag_name' <<<"$RELEASES") + +function get_current_version { + local app_dir + app_dir="$(get_test_utls_dir)/../.." + cargo run -q --manifest-path="$app_dir/Cargo.toml" --bin mullvad-version +} + +CURRENT_VERSION=$(get_current_version) +commit=$(git rev-parse HEAD^\{commit\}) +commit=${commit:0:6} + +TAG=$(git describe --exact-match HEAD 2>/dev/null || echo "") + +if [[ -n "$TAG" && ${CURRENT_VERSION} =~ -dev- ]]; then + CURRENT_VERSION+="+${TAG}" +fi + +export CURRENT_VERSION +export LATEST_STABLE_RELEASE + +function print_available_releases { + for release in $(jq -r '.[].tag_name'<<<"$RELEASES"); do + echo "$release" + done +} + +function get_package_dir { + local package_dir + if [[ -n "${PACKAGE_DIR+x}" ]]; then + # Resolve the package dir to an absolute path since cargo must be invoked from the test directory + package_dir=$(cd "$PACKAGE_DIR" > /dev/null && pwd) + elif [[ ("$(uname -s)" == "Darwin") ]]; then + package_dir="$HOME/Library/Caches/mullvad-test/packages" + elif [[ ("$(uname -s)" == "Linux") ]]; then + package_dir="$HOME/.cache/mullvad-test/packages" + else + echo "Unsupported OS" 1>&2 + exit 1 + fi + + mkdir -p "$package_dir" || exit 1 + # Clean up old packages + find "$package_dir" -type f -mtime +5 -delete || true + + echo "$package_dir" + return 0 +} + +function nice_time { + SECONDS=0 + if "$@"; then + result=0 + else + result=$? + fi + s=$SECONDS + echo "\"$*\" completed in $((s/60))m:$((s%60))s" + return $result +} + +# Returns 0 if $1 is a development build. `BASH_REMATCH` contains match groups +# if that is the case. +function is_dev_version { + local pattern="(^[0-9.]+(-beta[0-9]+)?-dev-)([0-9a-z]+)(\+[0-9a-z|-]+)?$" + if [[ "$1" =~ $pattern ]]; then + return 0 + fi + return 1 +} + +function get_app_filename { + local version=$1 + local os=$2 + if is_dev_version "$version"; then + # only save 6 chars of the hash + local commit="${BASH_REMATCH[3]}" + version="${BASH_REMATCH[1]}${commit}" + # If the dev-version includes a tag, we need to append it to the app filename + if [[ -n ${BASH_REMATCH[4]} ]]; then + version="${version}${BASH_REMATCH[4]}" + fi + fi + case $os in + debian*|ubuntu*) + echo "MullvadVPN-${version}_amd64.deb" + ;; + fedora*) + echo "MullvadVPN-${version}_x86_64.rpm" + ;; + windows*) + echo "MullvadVPN-${version}.exe" + ;; + macos*) + echo "MullvadVPN-${version}.pkg" + ;; + *) + echo "Unsupported target: $os" 1>&2 + return 1 + ;; + esac +} + +function download_app_package { + local version=$1 + local os=$2 + local package_repo="" + + if is_dev_version "$version"; then + package_repo="${BUILD_DEV_REPOSITORY}" + else + package_repo="${BUILD_RELEASE_REPOSITORY}" + fi + + local filename + filename=$(get_app_filename "$version" "$os") + local url="${package_repo}/$version/$filename" + + local package_dir + package_dir=$(get_package_dir) + if [[ ! -f "$package_dir/$filename" ]]; then + echo "Downloading build for $version ($os) from $url" + if ! curl -sf -o "$package_dir/$filename" "$url"; then + echo "Failed to download package from $url (hint: build may not exist, check the url)" 1>&2 + exit 1 + fi + else + echo "App package for version $version ($os) already exists at $package_dir/$filename, skipping download" + fi +} + +function get_e2e_filename { + local version=$1 + local os=$2 + if is_dev_version "$version"; then + # only save 6 chars of the hash + local commit="${BASH_REMATCH[3]}" + version="${BASH_REMATCH[1]}${commit}" + fi + case $os in + debian*|ubuntu*|fedora*) + echo "app-e2e-tests-${version}-x86_64-unknown-linux-gnu" + ;; + windows*) + echo "app-e2e-tests-${version}-x86_64-pc-windows-msvc.exe" + ;; + macos*) + echo "app-e2e-tests-${version}-aarch64-apple-darwin" + ;; + *) + echo "Unsupported target: $os" 1>&2 + return 1 + ;; + esac +} + +function download_e2e_executable { + local version=${1:?Error: version not set} + local os=${2:?Error: os not set} + local package_repo + + if is_dev_version "$version"; then + package_repo="${BUILD_DEV_REPOSITORY}" + else + package_repo="${BUILD_RELEASE_REPOSITORY}" + fi + + local filename + filename=$(get_e2e_filename "$version" "$os") + local url="${package_repo}/$version/additional-files/$filename" + + local package_dir + package_dir=$(get_package_dir) + if [[ ! -f "$package_dir/$filename" ]]; then + echo "Downloading e2e executable for $version ($os) from $url" + if ! curl -sf -o "$package_dir/$filename" "$url"; then + echo "Failed to download package from $url (hint: build may not exist, check the url)" 1>&2 + exit 1 + fi + else + echo "GUI e2e executable for version $version ($os) already exists at $package_dir/$filename, skipping download" + fi +} + +function build_test_runner { + local script_dir + script_dir=$(get_test_utls_dir) + local test_os=${1:?Error: test os not set} + if [[ "${test_os}" =~ "debian"|"ubuntu"|"fedora" ]]; then + "$script_dir"/container-run.sh scripts/build-runner.sh linux || exit 1 + elif [[ "${test_os}" =~ "windows" ]]; then + "$script_dir"/container-run.sh scripts/build-runner.sh windows || exit 1 + elif [[ "${test_os}" =~ "macos" ]]; then + "$script_dir"/build-runner.sh macos || exit 1 + fi +} + +function run_tests_for_os { + local vm=$1 + + if [[ -z "${ACCOUNT_TOKEN+x}" ]]; then + echo "'ACCOUNT_TOKEN' must be specified" 1>&2 + exit 1 + fi + + echo "**********************************" + echo "* Building test runner" + echo "**********************************" + + nice_time build_test_runner "$vm" + + + echo "**********************************" + echo "* Running tests" + echo "**********************************" + + local upgrade_package_arg + if [[ -z "${APP_PACKAGE_TO_UPGRADE_FROM+x}" ]]; then + echo "'APP_PACKAGE_TO_UPGRADE_FROM' env not set, not testing upgrades" + upgrade_package_arg=() + else + upgrade_package_arg=(--app-package-to-upgrade-from "${APP_PACKAGE_TO_UPGRADE_FROM}") + fi + + if [[ -z "${TEST_REPORT+x}" ]]; then + echo "'TEST_REPORT' env not set, not saving test report" + test_report_arg=() + else + test_report_arg=(--test-report "${TEST_REPORT}") + fi + + local package_dir + package_dir=$(get_package_dir) + local test_dir + test_dir=$(get_test_utls_dir)/.. + pushd "$test_dir" + if ! RUST_LOG_STYLE=always cargo run --bin test-manager \ + run-tests \ + --account "${ACCOUNT_TOKEN:?Error: ACCOUNT_TOKEN not set}" \ + --app-package "${APP_PACKAGE:?Error: APP_PACKAGE not set}" \ + "${upgrade_package_arg[@]}" \ + "${test_report_arg[@]}" \ + --package-dir "${package_dir}" \ + --vm "$vm" \ + "${TEST_FILTERS:-}" \ + 2>&1 | sed -r "s/${ACCOUNT_TOKEN}/\{ACCOUNT_TOKEN\}/g"; then + echo "Test run failed" + exit 1 + fi + popd +} + +# Build the current version of the app and move the package to the package folder +# Currently unused, but may be useful in the future +function build_current_version { + local app_dir + app_dir="$(get_test_utls_dir)/../.." + local app_filename + # TODO: TEST_OS must be set to local OS manually, should be set automatically + app_filename=$(get_app_filename "$CURRENT_VERSION" "${TEST_OS:?Error: TEST_OS not set}") + local package_dir + package_dir=$(get_package_dir) + local app_package="$package_dir"/"$app_filename" + + local gui_test_filename + gui_test_filename=$(get_e2e_filename "$CURRENT_VERSION" "$TEST_OS") + local gui_test_bin="$package_dir"/"$gui_test_filename" + + if [ ! -f "$app_package" ]; then + pushd "$app_dir" + if [[ $(git diff --quiet) ]]; then + echo "WARNING: the app repository contains uncommitted changes, this script will only rebuild the app package when the git hash changes" + fi + ./build.sh + popd + echo "Moving '$(realpath "$app_dir/dist/$app_filename")' to '$(realpath "$app_package")'" + mv -n "$app_dir"/dist/"$app_filename" "$app_package" + else + echo "App package for current version already exists at $app_package, skipping build" + fi + + if [ ! -f "$gui_test_bin" ]; then + pushd "$app_dir"/gui + npm run build-test-executable + popd + echo "Moving '$(realpath "$app_dir/dist/$gui_test_filename")' to '$(realpath "$gui_test_bin")'" + mv -n "$app_dir"/dist/"$gui_test_filename" "$gui_test_bin" + else + echo "GUI e2e executable for current version already exists at $gui_test_bin, skipping build" + fi +}
\ No newline at end of file |