Merge branch 'run-mockapi-tests-in-gh-actions'

8 files changed, 93 insertions, 27 deletions

diff --git a/.github/workflows/android-app.yml b/.github/workflows/android-app.yml
index 7135304007..f800fbe138 100644
--- a/.github/workflows/android-app.yml
+++ b/.github/workflows/android-app.yml
@@ -28,6 +28,7 @@ on:
     branches: [master]
 jobs:
   prepare:
+    name: Prepare
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -47,6 +48,7 @@ jobs:
       container_image: ${{ env.inner_container_image }}
 
   build:
+    name: Build app and run unit tests
     needs: prepare
     runs-on: ubuntu-latest
     container:
@@ -92,7 +94,7 @@ jobs:
           STRIPPED_LIB_PATH="./android/app/build/extraJni/$ABI/libmullvad_jni.so"
           NDK_TOOLCHAIN_STRIP_TOOL="$NDK_TOOLCHAIN_DIR/x86_64-linux-android-strip"
           ./wireguard/build-wireguard-go.sh --android --no-docker
-          cargo build --target $TARGET --verbose --package mullvad-jni
+          cargo build --target $TARGET --verbose --package mullvad-jni --features api-override
           cargo run --bin relay_list > build/relays.json
           $NDK_TOOLCHAIN_STRIP_TOOL --strip-debug --strip-unneeded -o "$STRIPPED_LIB_PATH" "$UNSTRIPPED_LIB_PATH"
 
@@ -113,7 +115,7 @@ jobs:
           build-root-directory: android
           execution-only-caches: true
 
-      - name: Assemble instrumented test apk
+      - name: Assemble instrumented test apk (app)
         uses: burrunan/gradle-cache-action@v1
         with:
           job-id: jdk11
@@ -122,21 +124,34 @@ jobs:
           build-root-directory: android
           execution-only-caches: true
 
+      - name: Assemble instrumented test apk (mockapi)
+        uses: burrunan/gradle-cache-action@v1
+        with:
+          job-id: jdk11
+          arguments: :test:mockapi:assemble
+          gradle-version: wrapper
+          build-root-directory: android
+          execution-only-caches: true
+
       - name: Upload apks
         uses: actions/upload-artifact@v3
         with:
           name: apks
-          path: android/app/build/outputs/apk
+          path: |
+            android/app/build/outputs/apk
+            android/test/mockapi/build/outputs/apk
           if-no-files-found: error
           retention-days: 1
 
   instrumented-tests:
-    name: Instrumented tests
+    name: Run instrumented tests
     runs-on: [self-hosted, android-emulator]
     timeout-minutes: 30
     needs: [build]
     strategy:
       fail-fast: false
+      matrix:
+        test-type: [app, mockapi]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -144,8 +159,20 @@ jobs:
       - uses: actions/download-artifact@v3
         with:
           name: apks
-          path: android/app/build/outputs/apk
+          path: android
 
-      - name: Run Android instrumented tests
+      - name: Run instrumented test script
         shell: bash -ieo pipefail {0}
-        run: ./android/scripts/run-instrumented-tests.sh app
+        env:
+          AUTO_FETCH_TEST_HELPER_APKS: true
+        run: |
+          ./android/scripts/run-instrumented-tests.sh ${{ matrix.test-type }}
+
+      - name: Upload instrumentation report (${{ matrix.test-type }})
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: ${{ matrix.test-type }}-instrumentation-report
+          path: /tmp/mullvad-${{ matrix.test-type }}-instrumentation-report
+          if-no-files-found: ignore
+          retention-days: 1
diff --git a/android/scripts/run-instrumented-tests.sh b/android/scripts/run-instrumented-tests.sh
index 461a9e9d59..5e855ca242 100755
--- a/android/scripts/run-instrumented-tests.sh
+++ b/android/scripts/run-instrumented-tests.sh
@@ -5,8 +5,14 @@ set -eu
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 cd "$SCRIPT_DIR"
 
+AUTO_FETCH_TEST_HELPER_APKS=${AUTO_FETCH_TEST_HELPER_APKS:-"false"}
+
 APK_BASE_DIR=${APK_BASE_DIR:-"$SCRIPT_DIR/.."}
 LOG_FAILURE_MESSAGE="FAILURES!!!"
+DEFAULT_ORCHESTRATOR_APK_PATH=/tmp/orchestrator.apk
+ORCHESTRATOR_URL=https://dl.google.com/android/maven2/androidx/test/orchestrator/1.4.2/orchestrator-1.4.2.apk
+DEFAULT_TEST_SERVICES_APK_PATH=/tmp/test-services.apk
+TEST_SERVICES_URL=https://dl.google.com/android/maven2/androidx/test/services/test-services/1.4.2/test-services-1.4.2.apk
 
 while [[ "$#" -gt 0 ]]; do
     case $1 in
@@ -52,25 +58,41 @@ if [[ -z ${TEST_TYPE-} ]]; then
     exit 1
 fi
 
-if [[ "${USE_ORCHESTRATOR-}" == "true" ]]; then
-    if [[ -z ${ORCHESTRATOR_APK_PATH-} ]]; then
-        echo "The variable ORCHESTRATOR_APK_PATH is not set."
-        exit 1
-    fi
-    if [[ -z ${TEST_SERVICES_APK_PATH-} ]]; then
-        echo "The variable TEST_SERVICES_APK_PATH is not set."
-        exit 1
-    fi
-fi
+LOCAL_TMP_REPORT_PATH="/tmp/mullvad-$TEST_TYPE-instrumentation-report"
+INSTRUMENTATION_LOG_FILE_PATH="$LOCAL_TMP_REPORT_PATH/instrumentation-log.txt"
+LOGCAT_FILE_PATH="$LOCAL_TMP_REPORT_PATH/logcat.txt"
+LOCAL_SCREENSHOT_PATH="$LOCAL_TMP_REPORT_PATH/screenshots"
+DEVICE_SCREENSHOT_PATH="/sdcard/Pictures/mullvad-$TEST_TYPE"
 
-LOG_FILE_NAME="mullvad-$TEST_TYPE.txt"
-LOG_FILE_PATH="/tmp/$LOG_FILE_NAME"
+echo "Preparing to run tests of type: $TEST_TYPE"
+echo ""
 
-echo "Starting instrumented tests of type: $TEST_TYPE"
+echo "### Ensure clean report structure ###"
+rm -rf "$LOCAL_TMP_REPORT_PATH" || echo "No report path"
+adb logcat --clear
+adb shell rm -rf "$DEVICE_SCREENSHOT_PATH"
+mkdir "$LOCAL_TMP_REPORT_PATH"
 echo ""
 
-echo "### Clean up previous logs ###"
-rm "$LOG_FILE_PATH"
+if [[ "${USE_ORCHESTRATOR-}" == "true" ]]; then
+    if [[ "${AUTO_FETCH_TEST_HELPER_APKS-}" == "true" ]]; then
+        echo "### Fetching orchestrator and test services apks ###"
+        ORCHESTRATOR_APK_PATH=$DEFAULT_ORCHESTRATOR_APK_PATH
+        TEST_SERVICES_APK_PATH=$DEFAULT_TEST_SERVICES_APK_PATH
+        curl -sL "$ORCHESTRATOR_URL" -o "$ORCHESTRATOR_APK_PATH"
+        curl -sL "$TEST_SERVICES_URL" -o "$TEST_SERVICES_APK_PATH"
+        echo ""
+    else
+        if [[ -z ${ORCHESTRATOR_APK_PATH-} ]]; then
+            echo "The variable ORCHESTRATOR_APK_PATH is not set."
+            exit 1
+        fi
+        if [[ -z ${TEST_SERVICES_APK_PATH-} ]]; then
+            echo "The variable TEST_SERVICES_APK_PATH is not set."
+            exit 1
+        fi
+    fi
+fi
 
 echo "### Ensure that packages are not previously installed ###"
 adb uninstall net.mullvad.mullvadvpn || echo "App package not installed"
@@ -79,11 +101,16 @@ adb uninstall androidx.test.services || echo "Test services package not installe
 adb uninstall androidx.test.orchestrator || echo "Test orchestrator package not installed"
 echo ""
 
+echo "Starting instrumented tests of type: $TEST_TYPE"
+echo ""
+
 echo "### Install packages ###"
 adb install -t "$APK_BASE_DIR/app/build/outputs/apk/debug/app-debug.apk"
 adb install "$TEST_APK"
 if [[ "$USE_ORCHESTRATOR" == "true" ]]; then
+    echo "Using ORCHESTRATOR_APK_PATH: $ORCHESTRATOR_APK_PATH"
     adb install "$ORCHESTRATOR_APK_PATH"
+    echo "Using TEST_SERVICES_APK_PATH: $TEST_SERVICES_APK_PATH"
     adb install "$TEST_SERVICES_APK_PATH"
 fi
 echo ""
@@ -102,7 +129,7 @@ else
     am instrument -w \
     $TEST_PACKAGE/androidx.test.runner.AndroidJUnitRunner"
 fi
-adb shell "$INSTRUMENTATION_COMMAND" | tee "$LOG_FILE_PATH"
+adb shell "$INSTRUMENTATION_COMMAND" | tee "$INSTRUMENTATION_LOG_FILE_PATH"
 echo ""
 
 echo "### Ensure that packages are uninstalled ###"
@@ -113,8 +140,11 @@ adb uninstall androidx.test.orchestrator || echo "Test orchestrator package not
 echo ""
 
 echo "### Checking logs for failures ###"
-if grep -q "$LOG_FAILURE_MESSAGE" "$LOG_FILE_PATH"; then
+if grep -q "$LOG_FAILURE_MESSAGE" "$INSTRUMENTATION_LOG_FILE_PATH"; then
     echo "One or more tests failed, see logs for more details."
+    echo "Collecting report..."
+    adb pull "$DEVICE_SCREENSHOT_PATH" "$LOCAL_SCREENSHOT_PATH" || echo "No screenshots"
+    adb logcat -d > "$LOGCAT_FILE_PATH"
     exit 1
 else
     echo "No failures!"
diff --git a/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/constant/TimeoutConstants.kt b/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/constant/TimeoutConstants.kt
index 0da1d02aaf..6e4b6fa733 100644
--- a/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/constant/TimeoutConstants.kt
+++ b/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/constant/TimeoutConstants.kt
@@ -5,4 +5,5 @@ const val CONNECTION_TIMEOUT = 30000L
 const val DEFAULT_INTERACTION_TIMEOUT = 3000L
 const val LOGIN_TIMEOUT = 30000L
 const val LOGIN_FAILURE_TIMEOUT = 60000L
+const val LOGIN_PROMPT_TIMEOUT = 30000L
 const val WEB_TIMEOUT = 30000L
diff --git a/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/interactor/AppInteractor.kt b/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/interactor/AppInteractor.kt
index 1d6e9358a8..d4ddb37fba 100644
--- a/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/interactor/AppInteractor.kt
+++ b/android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/interactor/AppInteractor.kt
@@ -10,6 +10,7 @@ import net.mullvad.mullvadvpn.lib.endpoint.CustomApiEndpointConfiguration
 import net.mullvad.mullvadvpn.lib.endpoint.putApiEndpointConfigurationExtra
 import net.mullvad.mullvadvpn.test.common.constant.APP_LAUNCH_TIMEOUT
 import net.mullvad.mullvadvpn.test.common.constant.CONNECTION_TIMEOUT
+import net.mullvad.mullvadvpn.test.common.constant.LOGIN_PROMPT_TIMEOUT
 import net.mullvad.mullvadvpn.test.common.constant.LOGIN_TIMEOUT
 import net.mullvad.mullvadvpn.test.common.constant.MULLVAD_PACKAGE
 import net.mullvad.mullvadvpn.test.common.constant.SETTINGS_COG_ID
@@ -53,7 +54,6 @@ class AppInteractor(
     }
 
     fun attemptLogin(accountToken: String) {
-        device.findObjectWithTimeout(By.text("Login"))
         val loginObject = device.findObjectWithTimeout(By.clazz("android.widget.EditText"))
             .apply { text = accountToken }
         loginObject.parent.findObject(By.clazz(ImageButton::class.java)).click()
@@ -83,6 +83,12 @@ class AppInteractor(
         device.findObjectWithTimeout(By.text(text)).click()
     }
 
+    fun waitForLoginPrompt(
+        timeout: Long = LOGIN_PROMPT_TIMEOUT
+    ) {
+        device.findObjectWithTimeout(By.text("Login"), timeout)
+    }
+
     private fun String.extractIpAddress(): String {
         return split("  ")[1].split(" ")[0]
     }
diff --git a/android/test/e2e/build.gradle.kts b/android/test/e2e/build.gradle.kts
index 6310ca5b12..13f1bcaf46 100644
--- a/android/test/e2e/build.gradle.kts
+++ b/android/test/e2e/build.gradle.kts
@@ -43,7 +43,6 @@ android {
 
         testInstrumentationRunnerArguments += mutableMapOf<String, String>().apply {
             put("clearPackageData", "true")
-            put("useTestStorageService", "true")
             addOptionalPropertyAsArgument("valid_test_account_token")
             addOptionalPropertyAsArgument("invalid_test_account_token")
         }
diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt
index 9e106d45a2..07c1101bc4 100644
--- a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt
+++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt
@@ -26,6 +26,7 @@ class LoginTest : EndToEndTest() {
         // When
         app.launch()
         device.clickAllowOnNotificationPermissionPromptIfApiLevel31AndAbove()
+        app.waitForLoginPrompt()
         app.attemptLogin(invalidDummyAccountToken)
 
         // Then
diff --git a/android/test/mockapi/build.gradle.kts b/android/test/mockapi/build.gradle.kts
index 7fd7634e63..cbdd8c0295 100644
--- a/android/test/mockapi/build.gradle.kts
+++ b/android/test/mockapi/build.gradle.kts
@@ -17,7 +17,6 @@ android {
         testInstrumentationRunnerArguments.putAll(
             mapOf(
                 "clearPackageData" to "true",
-                "useTestStorageService" to "true"
             )
         )
     }
diff --git a/android/test/mockapi/src/main/kotlin/net/mullvad/mullvadvpn/test/mockapi/LoginMockApiTest.kt b/android/test/mockapi/src/main/kotlin/net/mullvad/mullvadvpn/test/mockapi/LoginMockApiTest.kt
index 2a72b41373..51d0ac7ee6 100644
--- a/android/test/mockapi/src/main/kotlin/net/mullvad/mullvadvpn/test/mockapi/LoginMockApiTest.kt
+++ b/android/test/mockapi/src/main/kotlin/net/mullvad/mullvadvpn/test/mockapi/LoginMockApiTest.kt
@@ -24,6 +24,7 @@ class LoginMockApiTest : MockApiTest() {
 
         // Act
         device.clickAllowOnNotificationPermissionPromptIfApiLevel31AndAbove()
+        app.waitForLoginPrompt()
         app.attemptLogin(validAccountToken)
 
         // Assert
@@ -43,6 +44,7 @@ class LoginMockApiTest : MockApiTest() {
         // Act
         app.launch(endpoint)
         device.clickAllowOnNotificationPermissionPromptIfApiLevel31AndAbove()
+        app.waitForLoginPrompt()
         app.attemptLogin(validAccountToken)
 
         // Assert
@@ -62,6 +64,7 @@ class LoginMockApiTest : MockApiTest() {
         // Act
         app.launch(endpoint)
         device.clickAllowOnNotificationPermissionPromptIfApiLevel31AndAbove()
+        app.waitForLoginPrompt()
         app.attemptLogin(validAccountToken)
 
         // Assert