diff options
8 files changed, 93 insertions, 27 deletions
diff --git a/.github/workflows/android-app.yml b/.github/workflows/android-app.yml index 7135304007..f800fbe138 100644 --- a/.github/workflows/android-app.yml +++ b/.github/workflows/android-app.yml @@ -28,6 +28,7 @@ on: branches: [master] jobs: prepare: + name: Prepare runs-on: ubuntu-latest steps: - name: Checkout repository @@ -47,6 +48,7 @@ jobs: container_image: ${{ env.inner_container_image }} build: + name: Build app and run unit tests needs: prepare runs-on: ubuntu-latest container: @@ -92,7 +94,7 @@ jobs: STRIPPED_LIB_PATH="./android/app/build/extraJni/$ABI/libmullvad_jni.so" NDK_TOOLCHAIN_STRIP_TOOL="$NDK_TOOLCHAIN_DIR/x86_64-linux-android-strip" ./wireguard/build-wireguard-go.sh --android --no-docker - cargo build --target $TARGET --verbose --package mullvad-jni + cargo build --target $TARGET --verbose --package mullvad-jni --features api-override cargo run --bin relay_list > build/relays.json $NDK_TOOLCHAIN_STRIP_TOOL --strip-debug --strip-unneeded -o "$STRIPPED_LIB_PATH" "$UNSTRIPPED_LIB_PATH" @@ -113,7 +115,7 @@ jobs: build-root-directory: android execution-only-caches: true - - name: Assemble instrumented test apk + - name: Assemble instrumented test apk (app) uses: burrunan/gradle-cache-action@v1 with: job-id: jdk11 @@ -122,21 +124,34 @@ jobs: build-root-directory: android execution-only-caches: true + - name: Assemble instrumented test apk (mockapi) + uses: burrunan/gradle-cache-action@v1 + with: + job-id: jdk11 + arguments: :test:mockapi:assemble + gradle-version: wrapper + build-root-directory: android + execution-only-caches: true + - name: Upload apks uses: actions/upload-artifact@v3 with: name: apks - path: android/app/build/outputs/apk + path: | + android/app/build/outputs/apk + android/test/mockapi/build/outputs/apk if-no-files-found: error retention-days: 1 instrumented-tests: - name: Instrumented tests + name: Run instrumented tests runs-on: [self-hosted, android-emulator] timeout-minutes: 30 needs: [build] strategy: fail-fast: false + matrix: + test-type: [app, mockapi] steps: - name: Checkout repository uses: actions/checkout@v3 @@ -144,8 +159,20 @@ jobs: - uses: actions/download-artifact@v3 with: name: apks - path: android/app/build/outputs/apk + path: android - - name: Run Android instrumented tests + - name: Run instrumented test script shell: bash -ieo pipefail {0} - run: ./android/scripts/run-instrumented-tests.sh app + env: + AUTO_FETCH_TEST_HELPER_APKS: true + run: | + ./android/scripts/run-instrumented-tests.sh ${{ matrix.test-type }} + + - name: Upload instrumentation report (${{ matrix.test-type }}) + uses: actions/upload-artifact@v3 + if: always() + with: + name: ${{ matrix.test-type }}-instrumentation-report + path: /tmp/mullvad-${{ matrix.test-type }}-instrumentation-report + if-no-files-found: ignore + retention-days: 1 diff --git a/android/scripts/run-instrumented-tests.sh b/android/scripts/run-instrumented-tests.sh index 461a9e9d59..5e855ca242 100755 --- a/android/scripts/run-instrumented-tests.sh +++ b/android/scripts/run-instrumented-tests.sh @@ -5,8 +5,14 @@ set -eu SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "$SCRIPT_DIR" +AUTO_FETCH_TEST_HELPER_APKS=${AUTO_FETCH_TEST_HELPER_APKS:-"false"} + APK_BASE_DIR=${APK_BASE_DIR:-"$SCRIPT_DIR/.."} LOG_FAILURE_MESSAGE="FAILURES!!!" +DEFAULT_ORCHESTRATOR_APK_PATH=/tmp/orchestrator.apk +ORCHESTRATOR_URL=https://dl.google.com/android/maven2/androidx/test/orchestrator/1.4.2/orchestrator-1.4.2.apk +DEFAULT_TEST_SERVICES_APK_PATH=/tmp/test-services.apk +TEST_SERVICES_URL=https://dl.google.com/android/maven2/androidx/test/services/test-services/1.4.2/test-services-1.4.2.apk while [[ "$#" -gt 0 ]]; do case $1 in @@ -52,25 +58,41 @@ if [[ -z ${TEST_TYPE-} ]]; then exit 1 fi -if [[ "${USE_ORCHESTRATOR-}" == "true" ]]; then - if [[ -z ${ORCHESTRATOR_APK_PATH-} ]]; then - echo "The variable ORCHESTRATOR_APK_PATH is not set." - exit 1 - fi - if [[ -z ${TEST_SERVICES_APK_PATH-} ]]; then - echo "The variable TEST_SERVICES_APK_PATH is not set." - exit 1 - fi -fi +LOCAL_TMP_REPORT_PATH="/tmp/mullvad-$TEST_TYPE-instrumentation-report" +INSTRUMENTATION_LOG_FILE_PATH="$LOCAL_TMP_REPORT_PATH/instrumentation-log.txt" +LOGCAT_FILE_PATH="$LOCAL_TMP_REPORT_PATH/logcat.txt" +LOCAL_SCREENSHOT_PATH="$LOCAL_TMP_REPORT_PATH/screenshots" +DEVICE_SCREENSHOT_PATH="/sdcard/Pictures/mullvad-$TEST_TYPE" -LOG_FILE_NAME="mullvad-$TEST_TYPE.txt" -LOG_FILE_PATH="/tmp/$LOG_FILE_NAME" +echo "Preparing to run tests of type: $TEST_TYPE" +echo "" -echo "Starting instrumented tests of type: $TEST_TYPE" +echo "### Ensure clean report structure ###" +rm -rf "$LOCAL_TMP_REPORT_PATH" || echo "No report path" +adb logcat --clear +adb shell rm -rf "$DEVICE_SCREENSHOT_PATH" +mkdir "$LOCAL_TMP_REPORT_PATH" echo "" -echo "### Clean up previous logs ###" -rm "$LOG_FILE_PATH" +if [[ "${USE_ORCHESTRATOR-}" == "true" ]]; then + if [[ "${AUTO_FETCH_TEST_HELPER_APKS-}" == "true" ]]; then + echo "### Fetching orchestrator and test services apks ###" + ORCHESTRATOR_APK_PATH=$DEFAULT_ORCHESTRATOR_APK_PATH + TEST_SERVICES_APK_PATH=$DEFAULT_TEST_SERVICES_APK_PATH + curl -sL "$ORCHESTRATOR_URL" -o "$ORCHESTRATOR_APK_PATH" + curl -sL "$TEST_SERVICES_URL" -o "$TEST_SERVICES_APK_PATH" + echo "" + else + if [[ -z ${ORCHESTRATOR_APK_PATH-} ]]; then + echo "The variable ORCHESTRATOR_APK_PATH is not set." + exit 1 + fi + if [[ -z ${TEST_SERVICES_APK_PATH-} ]]; then + echo "The variable TEST_SERVICES_APK_PATH is not set." + exit 1 + fi + fi +fi echo "### Ensure that packages are not previously installed ###" adb uninstall net.mullvad.mullvadvpn || echo "App package not installed" @@ -79,11 +101,16 @@ adb uninstall androidx.test.services || echo "Test services package not installe adb uninstall androidx.test.orchestrator || echo "Test orchestrator package not installed" echo "" +echo "Starting instrumented tests of type: $TEST_TYPE" +echo "" + echo "### Install packages ###" adb install -t "$APK_BASE_DIR/app/build/outputs/apk/debug/app-debug.apk" adb install "$TEST_APK" if [[ "$USE_ORCHESTRATOR" == "true" ]]; then + echo "Using ORCHESTRATOR_APK_PATH: $ORCHESTRATOR_APK_PATH" adb install "$ORCHESTRATOR_APK_PATH" + echo "Using TEST_SERVICES_APK_PATH: $TEST_SERVICES_APK_PATH" adb install "$TEST_SERVICES_APK_PATH" fi echo "" @@ -102,7 +129,7 @@ else am instrument -w \ $TEST_PACKAGE/androidx.test.runner.AndroidJUnitRunner" fi -adb shell "$INSTRUMENTATION_COMMAND" | tee "$LOG_FILE_PATH" +adb shell "$INSTRUMENTATION_COMMAND" | tee "$INSTRUMENTATION_LOG_FILE_PATH" echo "" echo "### Ensure that packages are uninstalled ###" @@ -113,8 +140,11 @@ adb uninstall androidx.test.orchestrator || echo "Test orchestrator package not echo "" echo "### Checking logs for failures ###" -if grep -q "$LOG_FAILURE_MESSAGE" "$LOG_FILE_PATH"; then +if grep -q "$LOG_FAILURE_MESSAGE" "$INSTRUMENTATION_LOG_FILE_PATH"; then echo "One or more tests failed, see logs for more details." + echo "Collecting report..." + adb pull "$DEVICE_SCREENSHOT_PATH" "$LOCAL_SCREENSHOT_PATH" || echo "No screenshots" + adb logcat -d > "$LOGCAT_FILE_PATH" exit 1 else echo "No failures!" diff --git a/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/constant/TimeoutConstants.kt b/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/constant/TimeoutConstants.kt index 0da1d02aaf..6e4b6fa733 100644 --- a/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/constant/TimeoutConstants.kt +++ b/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/constant/TimeoutConstants.kt @@ -5,4 +5,5 @@ const val CONNECTION_TIMEOUT = 30000L const val DEFAULT_INTERACTION_TIMEOUT = 3000L const val LOGIN_TIMEOUT = 30000L const val LOGIN_FAILURE_TIMEOUT = 60000L +const val LOGIN_PROMPT_TIMEOUT = 30000L const val WEB_TIMEOUT = 30000L diff --git a/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/interactor/AppInteractor.kt b/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/interactor/AppInteractor.kt index 1d6e9358a8..d4ddb37fba 100644 --- a/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/interactor/AppInteractor.kt +++ b/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/interactor/AppInteractor.kt @@ -10,6 +10,7 @@ import net.mullvad.mullvadvpn.lib.endpoint.CustomApiEndpointConfiguration import net.mullvad.mullvadvpn.lib.endpoint.putApiEndpointConfigurationExtra import net.mullvad.mullvadvpn.test.common.constant.APP_LAUNCH_TIMEOUT import net.mullvad.mullvadvpn.test.common.constant.CONNECTION_TIMEOUT +import net.mullvad.mullvadvpn.test.common.constant.LOGIN_PROMPT_TIMEOUT import net.mullvad.mullvadvpn.test.common.constant.LOGIN_TIMEOUT import net.mullvad.mullvadvpn.test.common.constant.MULLVAD_PACKAGE import net.mullvad.mullvadvpn.test.common.constant.SETTINGS_COG_ID @@ -53,7 +54,6 @@ class AppInteractor( } fun attemptLogin(accountToken: String) { - device.findObjectWithTimeout(By.text("Login")) val loginObject = device.findObjectWithTimeout(By.clazz("android.widget.EditText")) .apply { text = accountToken } loginObject.parent.findObject(By.clazz(ImageButton::class.java)).click() @@ -83,6 +83,12 @@ class AppInteractor( device.findObjectWithTimeout(By.text(text)).click() } + fun waitForLoginPrompt( + timeout: Long = LOGIN_PROMPT_TIMEOUT + ) { + device.findObjectWithTimeout(By.text("Login"), timeout) + } + private fun String.extractIpAddress(): String { return split(" ")[1].split(" ")[0] } diff --git a/android/test/e2e/build.gradle.kts b/android/test/e2e/build.gradle.kts index 6310ca5b12..13f1bcaf46 100644 --- a/android/test/e2e/build.gradle.kts +++ b/android/test/e2e/build.gradle.kts @@ -43,7 +43,6 @@ android { testInstrumentationRunnerArguments += mutableMapOf<String, String>().apply { put("clearPackageData", "true") - put("useTestStorageService", "true") addOptionalPropertyAsArgument("valid_test_account_token") addOptionalPropertyAsArgument("invalid_test_account_token") } diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt index 9e106d45a2..07c1101bc4 100644 --- a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt @@ -26,6 +26,7 @@ class LoginTest : EndToEndTest() { // When app.launch() device.clickAllowOnNotificationPermissionPromptIfApiLevel31AndAbove() + app.waitForLoginPrompt() app.attemptLogin(invalidDummyAccountToken) // Then diff --git a/android/test/mockapi/build.gradle.kts b/android/test/mockapi/build.gradle.kts index 7fd7634e63..cbdd8c0295 100644 --- a/android/test/mockapi/build.gradle.kts +++ b/android/test/mockapi/build.gradle.kts @@ -17,7 +17,6 @@ android { testInstrumentationRunnerArguments.putAll( mapOf( "clearPackageData" to "true", - "useTestStorageService" to "true" ) ) } diff --git a/android/test/mockapi/src/main/kotlin/net/mullvad/mullvadvpn/test/mockapi/LoginMockApiTest.kt b/android/test/mockapi/src/main/kotlin/net/mullvad/mullvadvpn/test/mockapi/LoginMockApiTest.kt index 2a72b41373..51d0ac7ee6 100644 --- a/android/test/mockapi/src/main/kotlin/net/mullvad/mullvadvpn/test/mockapi/LoginMockApiTest.kt +++ b/android/test/mockapi/src/main/kotlin/net/mullvad/mullvadvpn/test/mockapi/LoginMockApiTest.kt @@ -24,6 +24,7 @@ class LoginMockApiTest : MockApiTest() { // Act device.clickAllowOnNotificationPermissionPromptIfApiLevel31AndAbove() + app.waitForLoginPrompt() app.attemptLogin(validAccountToken) // Assert @@ -43,6 +44,7 @@ class LoginMockApiTest : MockApiTest() { // Act app.launch(endpoint) device.clickAllowOnNotificationPermissionPromptIfApiLevel31AndAbove() + app.waitForLoginPrompt() app.attemptLogin(validAccountToken) // Assert @@ -62,6 +64,7 @@ class LoginMockApiTest : MockApiTest() { // Act app.launch(endpoint) device.clickAllowOnNotificationPermissionPromptIfApiLevel31AndAbove() + app.waitForLoginPrompt() app.attemptLogin(validAccountToken) // Assert |