diff options
| author | Albin <albin@mullvad.net> | 2022-11-23 17:54:09 +0100 |
|---|---|---|
| committer | Albin <albin@mullvad.net> | 2023-01-10 15:32:27 +0100 |
| commit | bf287ad5153bb3687afb03370cdea1014b3cef75 (patch) | |
| tree | 38fb21e35c8108f973853a03fe20273f5dc4c7e7 /android/test/e2e | |
| parent | 14c536c8cf902894188a72c65301659b7cd8256b (diff) | |
| download | mullvadvpn-bf287ad5153bb3687afb03370cdea1014b3cef75.tar.xz mullvadvpn-bf287ad5153bb3687afb03370cdea1014b3cef75.zip | |
Move :e2e project to :test:e2e
Also changes source directory from "java" to "kotlin" which is
supported since upgrading the project from AGP 3.x to 7.x.
Diffstat (limited to 'android/test/e2e')
24 files changed, 964 insertions, 0 deletions
diff --git a/android/test/e2e/README.md b/android/test/e2e/README.md new file mode 100644 index 0000000000..7c1271ad97 --- /dev/null +++ b/android/test/e2e/README.md @@ -0,0 +1,56 @@ +# End-to-end (e2e) test module +## Overview +The tests in this module are end-to-end tests that rely on the publicly accessible Mullvad infrastucture and APIs. It's therefore required to provide a valid account token (not expired) that can be used to login, connect etc. It's also required to provide an invalid account token which for example is used for negative tests of the login flow. The invalid account token should not exist in the Mullvad infrastucture, however it must be at least 9 characters for some tests to properly run due to input validation. + +## How to run the tests +### Locally +Set tokens in the below command and then execute the command in the `android` directory to run the tests on a local device: +``` +./gradlew :test:e2e:connectedDebugAndroidTest \ + -Pvalid_test_account_token=XXXX \ + -Pinvalid_test_account_token=XXXX +``` + +For convenience, the tokens can also be set in `<REPO-ROOT>/android/local.properties` in the following way: +``` +valid_test_account_token=XXXX +invalid_test_account_token=XXXX +``` + +It's also possible to provide the tokens to the test runner during test execution. However note that this requires [the APKs to be installed manually](https://developer.android.com/training/testing/instrumented-tests/androidx-test-libraries/runner#architecture). +``` +adb shell 'CLASSPATH=$(pm path androidx.test.services) app_process / \ + androidx.test.services.shellexecutor.ShellMain am instrument -w \ + -e clearPackageData true \ + -e valid_test_account_token XXXX \ + -e invalid_test_account_token XXXX \ + -e targetInstrumentation net.mullvad.mullvadvpn.test.e2e/androidx.test.runner.AndroidJUnitRunner \ + androidx.test.orchestrator/.AndroidTestOrchestrator' +``` + +### Firebase Test Lab +Firebase Test Lab can be used to run the tests on vast collection of physical and virtual devices. + +1. Setup the gcloud CLI by following the [official documentation](https://firebase.google.com/docs/test-lab/android/command-line). + +2. Set tokens in the below command and then execute the command in the `android` directory to run the tests (on a Pixel 5e): +``` +gcloud firebase test android run \ + --type instrumentation \ + --app ./android/app/build/outputs/apk/debug/app-debug.apk \ + --test ./android/test/e2e/build/outputs/apk/debug/e2e-debug.apk \ + --device model=redfin,version=30,locale=en,orientation=portrait \ + --use-orchestrator \ + --environment-variables clearPackageData=true,valid_test_account_token=XXXX,invalid_test_account_token=XXXX +``` + +If using gcloud via the docker image, the following can be executed in the `android` directory to run the tests (on a Pixel 5e): +``` +docker run --rm --volumes-from gcloud-config -v ${PWD}:/android gcr.io/google.com/cloudsdktool/google-cloud-cli gcloud firebase test android run \ + --type instrumentation \ + --app ./android/app/build/outputs/apk/debug/app-debug.apk \ + --test ./android/test/e2e/build/outputs/apk/debug/e2e-debug.apk \ + --device model=redfin,version=30,locale=en,orientation=portrait \ + --use-orchestrator \ + --environment-variables clearPackageData=true,valid_test_account_token=XXXX,invalid_test_account_token=XXXX +``` diff --git a/android/test/e2e/build.gradle.kts b/android/test/e2e/build.gradle.kts new file mode 100644 index 0000000000..8c9082c6ef --- /dev/null +++ b/android/test/e2e/build.gradle.kts @@ -0,0 +1,118 @@ +import com.android.build.gradle.internal.cxx.configure.gradleLocalProperties +import java.util.Properties + +plugins { + id(Dependencies.Plugin.androidTestId) + id(Dependencies.Plugin.kotlinAndroidId) +} + +android { + namespace = "net.mullvad.mullvadvpn.test.e2e" + compileSdk = Versions.Android.compileSdkVersion + + defaultConfig { + minSdk = Versions.Android.minSdkVersion + targetSdk = Versions.Android.targetSdkVersion + testApplicationId = "net.mullvad.mullvadvpn.test.e2e" + testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" + targetProjectPath = ":app" + + fun Properties.addRequiredPropertyAsBuildConfigField(name: String) { + val value = getProperty(name) ?: throw GradleException("Missing property: $name") + buildConfigField( + type = "String", + name = name, + value = "\"$value\"" + ) + } + + Properties().apply { + load(project.file("e2e.properties").inputStream()) + addRequiredPropertyAsBuildConfigField("API_BASE_URL") + addRequiredPropertyAsBuildConfigField("API_VERSION") + } + + fun MutableMap<String, String>.addOptionalPropertyAsArgument(name: String) { + val value = rootProject.properties.getOrDefault(name, null) as? String + ?: gradleLocalProperties(rootProject.projectDir).getProperty(name) + + if (value != null) { + put(name, value) + } + } + + testInstrumentationRunnerArguments += mutableMapOf<String, String>().apply { + put("clearPackageData", "true") + addOptionalPropertyAsArgument("valid_test_account_token") + addOptionalPropertyAsArgument("invalid_test_account_token") + } + } + + testOptions { + execution = "ANDROIDX_TEST_ORCHESTRATOR" + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + } + + kotlinOptions { + jvmTarget = Versions.jvmTarget + } +} + +val localScreenshotPath = "$buildDir/reports/androidTests/connected/screenshots" +val deviceScreenshotPath = "/sdcard/Pictures/Screenshots" + +tasks.register("createDeviceScreenshotDir", Exec::class) { + executable = android.adbExecutable.toString() + args = listOf("shell", "mkdir", "-p", deviceScreenshotPath) +} + +tasks.register("createLocalScreenshotDir", Exec::class) { + executable = "mkdir" + args = listOf("-p", localScreenshotPath) +} + +tasks.register("clearDeviceScreenshots", Exec::class) { + executable = android.adbExecutable.toString() + args = listOf("shell", "rm", "-r", deviceScreenshotPath) +} + +tasks.register("fetchScreenshots", Exec::class) { + executable = android.adbExecutable.toString() + args = listOf("pull", "$deviceScreenshotPath/.", localScreenshotPath) + + dependsOn(tasks.getByName("createLocalScreenshotDir")) + finalizedBy(tasks.getByName("clearDeviceScreenshots")) +} + +tasks.whenTaskAdded { + if (name == "connectedDebugAndroidTest") { + dependsOn(tasks.getByName("createDeviceScreenshotDir")) + finalizedBy(tasks.getByName("fetchScreenshots")) + } +} + +configure<org.owasp.dependencycheck.gradle.extension.DependencyCheckExtension> { + // Skip the lintClassPath configuration, which relies on many dependencies that has been flagged + // to have CVEs, as it's related to the lint tooling rather than the project's compilation class + // path. The alternative would be to suppress specific CVEs, however that could potentially + // result in suppressed CVEs in project compilation class path. + skipConfigurations = listOf("lintClassPath") + suppressionFile = "$projectDir/e2e-suppression.xml" +} + +dependencies { + implementation(Dependencies.AndroidX.testCore) + // Fixes: https://github.com/android/android-test/issues/1589 + implementation(Dependencies.AndroidX.testMonitor) + implementation(Dependencies.AndroidX.testOrchestrator) + implementation(Dependencies.AndroidX.testRunner) + implementation(Dependencies.AndroidX.testRules) + implementation(Dependencies.AndroidX.testUiAutomator) + implementation(Dependencies.androidVolley) + implementation(Dependencies.junit) + implementation(Dependencies.Kotlin.stdlib) +} diff --git a/android/test/e2e/e2e-suppression.xml b/android/test/e2e/e2e-suppression.xml new file mode 100644 index 0000000000..2b57bc13e8 --- /dev/null +++ b/android/test/e2e/e2e-suppression.xml @@ -0,0 +1,95 @@ +<?xml version="1.0" encoding="UTF-8"?> +<suppressions xmlns="https://jeremylong.github.io/DependencyCheck/dependency-suppression.1.3.xsd"> + <!-- + CVEs in the e2e project are deemed less severe than CVEs in the main projects as CVEs in the e2e + project doesn't affect release or debug versions of the app. + --> + <suppress until="2023-05-01Z"> + <notes><![CDATA[ + This CVE is tracked externally and is therefore suppressed in the automatic audit checks. + ]]></notes> + <packageUrl regex="true">^pkg:maven/com\.google\.protobuf/protobuf\-java@.*$</packageUrl> + <cve>CVE-2022-3171</cve> + <cve>CVE-2022-3509</cve> + <cve>CVE-2022-3510</cve> + <cve>CVE-2021-22569</cve> + </suppress> + <suppress until="2023-05-01Z"> + <notes><![CDATA[ + These CVEs affects the Apache Commons Net's FTP client that this app doesn't use. + https://www.openwall.com/lists/oss-security/2022/12/03/1 + + File names: + - commons-beanutils-1.9.4.jar + - commons-collections-3.2.2.jar + - commons-digester-2.1.jar + - commons-logging-1.2.jar + - commons-validator-1.7.jar + ]]></notes> + <packageUrl regex="true">^pkg:maven/commons\-.*/commons\-.*@.*$</packageUrl> + <cve>CVE-2021-37533</cve> + </suppress> + <suppress until="2023-05-01Z"> + <notes><![CDATA[ + This CVE is tracked externally and is therefore suppressed in the automatic audit checks. + https://nvd.nist.gov/vuln/detail/CVE-2021-29425 + + File name: commons-io-2.4.jar + ]]></notes> + <packageUrl regex="true">^pkg:maven/commons\-io/commons\-io@.*$</packageUrl> + <cve>CVE-2021-29425</cve> + </suppress> + <suppress until="2023-05-01Z"> + <notes><![CDATA[ + These CVEs are tracked externally and is therefore suppressed in the automatic audit checks. + ]]></notes> + <packageUrl regex="true">^pkg:maven/io\.netty/netty\-.*@.*$</packageUrl> + <cve>CVE-2021-37136</cve> + <cve>CVE-2021-37137</cve> + <cve>CVE-2021-43797</cve> + <cve>CVE-2021-21295</cve> + <cve>CVE-2021-21409</cve> + <cve>CVE-2021-21290</cve> + <cve>CVE-2022-24823</cve> + <cve>CVE-2022-41881</cve> + <cve>CVE-2022-41915</cve> + </suppress> + <suppress until="2023-05-01Z"> + <notes><![CDATA[ + This CVE is tracked externally and is therefore suppressed in the automatic audit checks. + https://nvd.nist.gov/vuln/detail/CVE-2022-25647 + + File name: gson-2.8.6.jar + ]]></notes> + <packageUrl regex="true">^pkg:maven/com\.google\.code\.gson/gson@.*$</packageUrl> + <cve>CVE-2022-25647</cve> + </suppress> + <suppress until="2023-05-01Z"> + <notes><![CDATA[ + This CVE only affect Multiplatform Gradle Projects, which this project is not. + https://nvd.nist.gov/vuln/detail/CVE-2022-24329 + ]]></notes> + <packageUrl regex="true">^pkg:maven/org\.jetbrains\.kotlin/kotlin\-stdlib.*@.*$</packageUrl> + <cve>CVE-2022-24329</cve> + </suppress> + <suppress until="2023-06-01Z"> + <notes><![CDATA[ + This CVE is limited to processing of screenshots, which this app doesn't use. + https://nvd.nist.gov/vuln/detail/CVE-2021-4277 + + File name: legacy-support-core-utils-1.0.0.aar + ]]></notes> + <packageUrl regex="true">^pkg:maven/androidx\.legacy/legacy\-support\-core\-utils@.*$</packageUrl> + <cve>CVE-2021-4277</cve> + </suppress> + <suppress until="2023-06-01Z"> + <notes><![CDATA[ + This CVE is limited to processing of screenshots, which this app doesn't use. + https://nvd.nist.gov/vuln/detail/CVE-2021-4277 + + File name: common-30.3.1.jar + ]]></notes> + <packageUrl regex="true">^pkg:maven/com\.android\.tools/common@.*$</packageUrl> + <cve>CVE-2021-4277</cve> + </suppress> +</suppressions> diff --git a/android/test/e2e/e2e.properties b/android/test/e2e/e2e.properties new file mode 100644 index 0000000000..b02f6a9381 --- /dev/null +++ b/android/test/e2e/e2e.properties @@ -0,0 +1,2 @@ +API_BASE_URL=https://api.mullvad.net +API_VERSION=v1-beta1 diff --git a/android/test/e2e/src/main/AndroidManifest.xml b/android/test/e2e/src/main/AndroidManifest.xml new file mode 100644 index 0000000000..931f79d291 --- /dev/null +++ b/android/test/e2e/src/main/AndroidManifest.xml @@ -0,0 +1,7 @@ +<manifest xmlns:android="http://schemas.android.com/apk/res/android"> + + <uses-permission android:name="android.permission.INTERNET" /> + <instrumentation + android:name="androidx.test.runner.AndroidJUnitRunner" + android:targetPackage="net.mullvad.mullvadvpn" /> +</manifest> diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/ConnectionTest.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/ConnectionTest.kt new file mode 100644 index 0000000000..feabbc5335 --- /dev/null +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/ConnectionTest.kt @@ -0,0 +1,32 @@ +package net.mullvad.mullvadvpn.test.e2e + +import androidx.test.uiautomator.By +import junit.framework.Assert.assertEquals +import net.mullvad.mullvadvpn.test.e2e.extension.findObjectWithTimeout +import net.mullvad.mullvadvpn.test.e2e.interactor.WebViewInteractor +import net.mullvad.mullvadvpn.test.e2e.misc.CleanupAccountTestRule +import org.junit.Rule +import org.junit.Test + +class ConnectionTest : EndToEndTest() { + + @Rule + @JvmField + val cleanupAccountTestRule = CleanupAccountTestRule() + + @Test + fun testConnectAndVerifyWithConnectionCheck() { + // Given + app.launchAndEnsureLoggedIn() + + // When + device.findObjectWithTimeout(By.text("Secure my connection")).click() + device.findObjectWithTimeout(By.text("OK")).click() + device.findObjectWithTimeout(By.text("SECURE CONNECTION")) + val expected = WebViewInteractor.ConnCheckState(true, app.extractIpAddress()) + + // Then + val result = web.launchAndExtractConnCheckState() + assertEquals(expected, result) + } +} diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/EndToEndTest.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/EndToEndTest.kt new file mode 100644 index 0000000000..8adf065663 --- /dev/null +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/EndToEndTest.kt @@ -0,0 +1,53 @@ +package net.mullvad.mullvadvpn.test.e2e + +import android.content.Context +import androidx.test.platform.app.InstrumentationRegistry +import androidx.test.runner.AndroidJUnit4 +import androidx.test.uiautomator.UiDevice +import net.mullvad.mullvadvpn.test.e2e.constant.INVALID_TEST_ACCOUNT_TOKEN_ARGUMENT_KEY +import net.mullvad.mullvadvpn.test.e2e.constant.VALID_TEST_ACCOUNT_TOKEN_ARGUMENT_KEY +import net.mullvad.mullvadvpn.test.e2e.extension.getRequiredArgument +import net.mullvad.mullvadvpn.test.e2e.interactor.AppInteractor +import net.mullvad.mullvadvpn.test.e2e.interactor.WebViewInteractor +import net.mullvad.mullvadvpn.test.e2e.misc.CaptureScreenshotOnFailedTestRule +import org.junit.Before +import org.junit.Rule +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +abstract class EndToEndTest { + + @Rule + @JvmField + val rule = CaptureScreenshotOnFailedTestRule() + + lateinit var device: UiDevice + lateinit var targetContext: Context + lateinit var app: AppInteractor + lateinit var web: WebViewInteractor + lateinit var validTestAccountToken: String + lateinit var invalidTestAccountToken: String + + @Before + fun setup() { + device = UiDevice.getInstance(InstrumentationRegistry.getInstrumentation()) + targetContext = InstrumentationRegistry.getInstrumentation().targetContext + + validTestAccountToken = InstrumentationRegistry.getArguments() + .getRequiredArgument(VALID_TEST_ACCOUNT_TOKEN_ARGUMENT_KEY) + invalidTestAccountToken = InstrumentationRegistry.getArguments() + .getRequiredArgument(INVALID_TEST_ACCOUNT_TOKEN_ARGUMENT_KEY) + + app = AppInteractor( + device, + targetContext, + validTestAccountToken, + invalidTestAccountToken + ) + + web = WebViewInteractor( + targetContext, + device + ) + } +} diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LaunchAppTest.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LaunchAppTest.kt new file mode 100644 index 0000000000..64f534990c --- /dev/null +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LaunchAppTest.kt @@ -0,0 +1,13 @@ +package net.mullvad.mullvadvpn.test.e2e + +import androidx.test.runner.AndroidJUnit4 +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class LaunchAppTest : EndToEndTest() { + @Test + fun testLaunchApp() { + app.launch() + } +} diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt new file mode 100644 index 0000000000..d16795e5dd --- /dev/null +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt @@ -0,0 +1,58 @@ +package net.mullvad.mullvadvpn.test.e2e + +import androidx.test.runner.AndroidJUnit4 +import androidx.test.uiautomator.By +import junit.framework.Assert.assertNotNull +import net.mullvad.mullvadvpn.test.e2e.constant.LOGIN_FAILURE_TIMEOUT +import net.mullvad.mullvadvpn.test.e2e.extension.findObjectWithTimeout +import net.mullvad.mullvadvpn.test.e2e.misc.CleanupAccountTestRule +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class LoginTest : EndToEndTest() { + + @Rule + @JvmField + val cleanupAccountTestRule = CleanupAccountTestRule() + + @Test + fun testLoginWithInvalidCredentials() { + // Given + val invalidDummyAccountToken = invalidTestAccountToken + + // When + app.launch() + app.attemptLogin(invalidDummyAccountToken) + + // Then + device.findObjectWithTimeout(By.text("Login failed"), LOGIN_FAILURE_TIMEOUT) + } + + @Test + fun testLoginWithValidCredentials() { + // Given + val token = validTestAccountToken + + // When + app.launchAndEnsureLoggedIn(token) + + // Then + app.ensureLoggedIn() + } + + @Test + fun testLogout() { + // Given + app.launchAndEnsureLoggedIn() + + // When + app.clickSettingsCog() + app.clickListItemByText("Account") + app.clickActionButtonByText("Log out") + + // Then + assertNotNull(device.findObjectWithTimeout(By.text("Login"))) + } +} diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/WebLinkTest.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/WebLinkTest.kt new file mode 100644 index 0000000000..a6a04bf094 --- /dev/null +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/WebLinkTest.kt @@ -0,0 +1,21 @@ +package net.mullvad.mullvadvpn.test.e2e + +import androidx.test.uiautomator.By +import net.mullvad.mullvadvpn.test.e2e.extension.findObjectWithTimeout +import org.junit.Test + +class WebLinkTest : EndToEndTest() { + @Test + fun testOpenFaqFromApp() { + // Given + app.launch() + + // When + device.findObjectWithTimeout(By.text("Login")) + app.clickSettingsCog() + app.clickListItemByText("FAQs & Guides") + + // Then + device.findObjectWithTimeout(By.text("Mullvad help center")) + } +} diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/Constants.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/Constants.kt new file mode 100644 index 0000000000..23f29fca3c --- /dev/null +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/Constants.kt @@ -0,0 +1,6 @@ +package net.mullvad.mullvadvpn.test.e2e.constant + +const val LOG_TAG = "mullvad-e2e" +const val CONN_CHECK_URL = "https://mullvad.net/en/check/" +const val VALID_TEST_ACCOUNT_TOKEN_ARGUMENT_KEY = "valid_test_account_token" +const val INVALID_TEST_ACCOUNT_TOKEN_ARGUMENT_KEY = "invalid_test_account_token" diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/PackageConstants.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/PackageConstants.kt new file mode 100644 index 0000000000..5476c37687 --- /dev/null +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/PackageConstants.kt @@ -0,0 +1,3 @@ +package net.mullvad.mullvadvpn.test.e2e.constant + +const val MULLVAD_PACKAGE = "net.mullvad.mullvadvpn" diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/ResourceConstants.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/ResourceConstants.kt new file mode 100644 index 0000000000..7b27a581ad --- /dev/null +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/ResourceConstants.kt @@ -0,0 +1,5 @@ +package net.mullvad.mullvadvpn.test.e2e.constant + +const val SETTINGS_COG_ID = "net.mullvad.mullvadvpn:id/settings" +const val TUNNEL_INFO_ID = "net.mullvad.mullvadvpn:id/tunnel_info" +const val TUNNEL_OUT_ADDRESS_ID = "net.mullvad.mullvadvpn:id/out_address" diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/TextConstants.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/TextConstants.kt new file mode 100644 index 0000000000..cfc4080ea4 --- /dev/null +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/TextConstants.kt @@ -0,0 +1,4 @@ +package net.mullvad.mullvadvpn.test.e2e.constant + +const val CONNECTION_CHECK_IS_CONNECTED = "Using Mullvad VPN" +const val CONNECTION_CHECK_IS_NOT_CONNECTED = "Not using Mullvad VPN" diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/TimeoutConstants.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/TimeoutConstants.kt new file mode 100644 index 0000000000..6599a26bcd --- /dev/null +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/TimeoutConstants.kt @@ -0,0 +1,7 @@ +package net.mullvad.mullvadvpn.test.e2e.constant + +const val APP_LAUNCH_TIMEOUT = 5000L +const val CONNECTION_TIMEOUT = 30000L +const val DEFAULT_INTERACTION_TIMEOUT = 3000L +const val LOGIN_TIMEOUT = 30000L +const val LOGIN_FAILURE_TIMEOUT = 60000L diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/extension/BundleExtensions.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/extension/BundleExtensions.kt new file mode 100644 index 0000000000..c96c28bb09 --- /dev/null +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/extension/BundleExtensions.kt @@ -0,0 +1,8 @@ +package net.mullvad.mullvadvpn.test.e2e.extension + +import android.os.Bundle + +fun Bundle.getRequiredArgument(argument: String): String { + return getString(argument) + ?: throw IllegalArgumentException("Missing required argument: $argument") +} diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/extension/UiAutomatorExtensions.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/extension/UiAutomatorExtensions.kt new file mode 100644 index 0000000000..5d96f4cc45 --- /dev/null +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/extension/UiAutomatorExtensions.kt @@ -0,0 +1,55 @@ +package net.mullvad.mullvadvpn.test.e2e.extension + +import androidx.test.uiautomator.By +import androidx.test.uiautomator.BySelector +import androidx.test.uiautomator.UiDevice +import androidx.test.uiautomator.UiObject2 +import androidx.test.uiautomator.Until +import java.util.regex.Pattern +import net.mullvad.mullvadvpn.test.e2e.constant.DEFAULT_INTERACTION_TIMEOUT + +fun UiDevice.findObjectByCaseInsensitiveText(text: String): UiObject2 { + return findObjectWithTimeout(By.text(Pattern.compile(text, Pattern.CASE_INSENSITIVE))) +} + +fun UiObject2.findObjectByCaseInsensitiveText(text: String): UiObject2 { + return findObjectWithTimeout(By.text(Pattern.compile(text, Pattern.CASE_INSENSITIVE))) +} + +fun UiDevice.findObjectWithTimeout( + selector: BySelector, + timeout: Long = DEFAULT_INTERACTION_TIMEOUT +): UiObject2 { + + wait( + Until.hasObject(selector), + timeout + ) + + return try { + findObject(selector) + } catch (e: NullPointerException) { + throw IllegalArgumentException( + "No matches for selector within timeout ($timeout): $selector" + ) + } +} + +fun UiObject2.findObjectWithTimeout( + selector: BySelector, + timeout: Long = DEFAULT_INTERACTION_TIMEOUT +): UiObject2 { + + wait( + Until.hasObject(selector), + timeout + ) + + return try { + findObject(selector) + } catch (e: NullPointerException) { + throw IllegalArgumentException( + "No matches for selector within timeout ($timeout): $selector" + ) + } +} diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/interactor/AppInteractor.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/interactor/AppInteractor.kt new file mode 100644 index 0000000000..a6739b91e5 --- /dev/null +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/interactor/AppInteractor.kt @@ -0,0 +1,83 @@ +package net.mullvad.mullvadvpn.test.e2e.interactor + +import android.content.Context +import android.content.Intent +import android.widget.ImageButton +import androidx.test.uiautomator.By +import androidx.test.uiautomator.UiDevice +import androidx.test.uiautomator.Until +import net.mullvad.mullvadvpn.test.e2e.constant.APP_LAUNCH_TIMEOUT +import net.mullvad.mullvadvpn.test.e2e.constant.CONNECTION_TIMEOUT +import net.mullvad.mullvadvpn.test.e2e.constant.LOGIN_TIMEOUT +import net.mullvad.mullvadvpn.test.e2e.constant.MULLVAD_PACKAGE +import net.mullvad.mullvadvpn.test.e2e.constant.SETTINGS_COG_ID +import net.mullvad.mullvadvpn.test.e2e.constant.TUNNEL_INFO_ID +import net.mullvad.mullvadvpn.test.e2e.constant.TUNNEL_OUT_ADDRESS_ID +import net.mullvad.mullvadvpn.test.e2e.extension.findObjectWithTimeout + +class AppInteractor( + private val device: UiDevice, + private val targetContext: Context, + private val validTestAccountToken: String, + private val invalidTestAccountToken: String +) { + fun launch() { + device.pressHome() + // Wait for launcher + device.wait( + Until.hasObject(By.pkg(device.launcherPackageName).depth(0)), + APP_LAUNCH_TIMEOUT + ) + val intent = + targetContext.packageManager.getLaunchIntentForPackage(MULLVAD_PACKAGE)?.apply { + // Clear out any previous instances + addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK) + } + targetContext.startActivity(intent) + device.wait( + Until.hasObject(By.pkg(MULLVAD_PACKAGE).depth(0)), + APP_LAUNCH_TIMEOUT + ) + } + + fun launchAndEnsureLoggedIn(accountToken: String = validTestAccountToken) { + launch() + attemptLogin(accountToken) + ensureLoggedIn() + } + + fun attemptLogin(accountToken: String = validTestAccountToken) { + device.findObjectWithTimeout(By.text("Login")) + val loginObject = device.findObjectWithTimeout(By.clazz("android.widget.EditText")) + .apply { text = accountToken } + loginObject.parent.findObject(By.clazz(ImageButton::class.java)).click() + } + + fun ensureLoggedIn() { + device.findObjectWithTimeout(By.text("UNSECURED CONNECTION"), LOGIN_TIMEOUT) + } + + fun extractIpAddress(): String { + device.findObjectWithTimeout(By.res(TUNNEL_INFO_ID)).click() + return device.findObjectWithTimeout( + By.res(TUNNEL_OUT_ADDRESS_ID), + CONNECTION_TIMEOUT + ).text.extractIpAddress() + } + + fun clickSettingsCog() { + device.findObjectWithTimeout(By.res(SETTINGS_COG_ID)).click() + } + + fun clickListItemByText(text: String) { + device.findObjectWithTimeout(By.text(text)).click() + } + + fun clickActionButtonByText(text: String) { + device.findObjectWithTimeout(By.text(text)).click() + } + + private fun String.extractIpAddress(): String { + return split(" ")[1].split(" ")[0] + } +} diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/interactor/MullvadAccountInteractor.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/interactor/MullvadAccountInteractor.kt new file mode 100644 index 0000000000..8f3f55166f --- /dev/null +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/interactor/MullvadAccountInteractor.kt @@ -0,0 +1,12 @@ +package net.mullvad.mullvadvpn.test.e2e.interactor + +import net.mullvad.mullvadvpn.test.e2e.misc.SimpleMullvadHttpClient + +class MullvadAccountInteractor( + private val httpClient: SimpleMullvadHttpClient, + private val testAccountToken: String +) { + fun cleanupAccount() { + httpClient.removeAllDevices(testAccountToken) + } +} diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/interactor/SystemSettingsInteractor.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/interactor/SystemSettingsInteractor.kt new file mode 100644 index 0000000000..bbe941d488 --- /dev/null +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/interactor/SystemSettingsInteractor.kt @@ -0,0 +1,36 @@ +package net.mullvad.mullvadvpn.test.e2e.interactor + +import android.content.ComponentName +import android.content.Context +import android.content.Intent +import androidx.test.uiautomator.By +import androidx.test.uiautomator.UiDevice +import net.mullvad.mullvadvpn.test.e2e.extension.findObjectByCaseInsensitiveText + +class SystemSettingsInteractor( + private val uiDevice: UiDevice, + private val context: Context +) { + fun openVpnSettings() { + val intent = Intent("com.intent.MAIN").apply { + addFlags(Intent.FLAG_ACTIVITY_CLEAR_TASK) + addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + } + intent.component = ComponentName.unflattenFromString( + "com.android.settings/.Settings\$VpnSettingsActivity" + ) + context.startActivity(intent) + Thread.sleep(1000) + } + + fun removeAllVpnPermissions() { + openVpnSettings() + uiDevice.findObjects(By.descContains("Settings")).forEach { + it.click() + Thread.sleep(1000) + uiDevice.findObjectByCaseInsensitiveText("forget vpn").click() + Thread.sleep(1000) + uiDevice.findObjectByCaseInsensitiveText("forget").click() + } + } +} diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/interactor/WebViewInteractor.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/interactor/WebViewInteractor.kt new file mode 100644 index 0000000000..fd809b13dd --- /dev/null +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/interactor/WebViewInteractor.kt @@ -0,0 +1,47 @@ +package net.mullvad.mullvadvpn.test.e2e.interactor + +import android.content.Context +import android.content.Intent +import android.view.View +import android.webkit.WebView +import androidx.test.uiautomator.By +import androidx.test.uiautomator.UiDevice +import net.mullvad.mullvadvpn.TestActivity +import net.mullvad.mullvadvpn.test.e2e.constant.CONNECTION_CHECK_IS_CONNECTED +import net.mullvad.mullvadvpn.test.e2e.constant.CONN_CHECK_URL +import net.mullvad.mullvadvpn.test.e2e.extension.findObjectByCaseInsensitiveText +import net.mullvad.mullvadvpn.test.e2e.extension.findObjectWithTimeout + +class WebViewInteractor( + private val context: Context, + private val device: UiDevice +) { + fun launchWebView(context: Context, url: String) { + val intent = Intent(context, TestActivity::class.java).apply { + addFlags(Intent.FLAG_ACTIVITY_NEW_TASK) + putExtra("url", url) + } + context.startActivity(intent) + } + + fun launchAndExtractConnCheckState(): ConnCheckState { + launchWebView(context, CONN_CHECK_URL) + val webView = device.findObjectWithTimeout(By.clazz(WebView::class.java)) + val stateText = device.findObjectByCaseInsensitiveText("using Mullvad VPN").apply { + click() + } + + // Wait for view to expand after click. + Thread.sleep(1000) + + val wireGuardIpv4ConnectionRow = webView.findObjects(By.clazz(View::class.java)) + .first { it.text?.endsWith("(WireGuard)") == true } + val wireGuardIpv4Address = wireGuardIpv4ConnectionRow.text.split(" ")[0].trim() + return ConnCheckState(stateText.text == CONNECTION_CHECK_IS_CONNECTED, wireGuardIpv4Address) + } + + data class ConnCheckState( + val isConnected: Boolean, + val ipAddress: String + ) +} diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/CaptureScreenshotOnFailedTestRule.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/CaptureScreenshotOnFailedTestRule.kt new file mode 100644 index 0000000000..5be65f7ec8 --- /dev/null +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/CaptureScreenshotOnFailedTestRule.kt @@ -0,0 +1,31 @@ +package net.mullvad.mullvadvpn.test.e2e.misc + +import android.util.Log +import androidx.test.runner.screenshot.BasicScreenCaptureProcessor +import androidx.test.runner.screenshot.ScreenCaptureProcessor +import androidx.test.runner.screenshot.Screenshot +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter +import net.mullvad.mullvadvpn.test.e2e.constant.LOG_TAG +import org.junit.rules.TestWatcher +import org.junit.runner.Description + +class CaptureScreenshotOnFailedTestRule : TestWatcher() { + override fun failed(e: Throwable?, description: Description?) { + Log.d(LOG_TAG, "Capturing screenshot of failed test: " + description?.methodName) + val timestamp = DateTimeFormatter.ISO_DATE_TIME.format(LocalDateTime.now()).replace(":", "") + val screenshotName = "$timestamp-${description?.methodName}" + captureScreenshot(screenshotName) + } + + private fun captureScreenshot(screenShotName: String) { + try { + val screenCapture = Screenshot.capture().apply { name = screenShotName } + val processorSet: MutableSet<ScreenCaptureProcessor> = HashSet() + processorSet.add(BasicScreenCaptureProcessor()) + screenCapture.process(processorSet) + } catch (ex: Exception) { + Log.d(LOG_TAG, "Error capturing screenshot: " + ex.message) + } + } +} diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/CleanupAccountTestRule.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/CleanupAccountTestRule.kt new file mode 100644 index 0000000000..4ed5e72512 --- /dev/null +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/CleanupAccountTestRule.kt @@ -0,0 +1,21 @@ +package net.mullvad.mullvadvpn.test.e2e.misc + +import android.util.Log +import androidx.test.platform.app.InstrumentationRegistry +import net.mullvad.mullvadvpn.test.e2e.constant.LOG_TAG +import net.mullvad.mullvadvpn.test.e2e.constant.VALID_TEST_ACCOUNT_TOKEN_ARGUMENT_KEY +import net.mullvad.mullvadvpn.test.e2e.extension.getRequiredArgument +import net.mullvad.mullvadvpn.test.e2e.interactor.MullvadAccountInteractor +import org.junit.rules.TestWatcher +import org.junit.runner.Description + +class CleanupAccountTestRule : TestWatcher() { + override fun starting(description: Description?) { + Log.d(LOG_TAG, "Cleaning up account before test: ${description?.methodName}") + val targetContext = InstrumentationRegistry.getInstrumentation().targetContext + val validTestAccountToken = InstrumentationRegistry.getArguments() + .getRequiredArgument(VALID_TEST_ACCOUNT_TOKEN_ARGUMENT_KEY) + MullvadAccountInteractor(SimpleMullvadHttpClient(targetContext), validTestAccountToken) + .cleanupAccount() + } +} diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt new file mode 100644 index 0000000000..b06ecee780 --- /dev/null +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt @@ -0,0 +1,191 @@ +package net.mullvad.mullvadvpn.test.e2e.misc + +import android.content.Context +import android.util.Log +import androidx.test.services.events.TestEventException +import com.android.volley.Request +import com.android.volley.toolbox.JsonArrayRequest +import com.android.volley.toolbox.JsonObjectRequest +import com.android.volley.toolbox.RequestFuture +import com.android.volley.toolbox.StringRequest +import com.android.volley.toolbox.Volley +import net.mullvad.mullvadvpn.test.e2e.BuildConfig +import net.mullvad.mullvadvpn.test.e2e.constant.LOG_TAG +import org.json.JSONArray +import org.json.JSONObject + +class SimpleMullvadHttpClient(context: Context) { + + private val queue = Volley.newRequestQueue(context) + + fun removeAllDevices(accountToken: String) { + Log.v(LOG_TAG, "Remove all devices") + val token = login(accountToken) + val devices = getDeviceList(token) + devices.forEach { + removeDevice(token, it) + } + Log.v(LOG_TAG, "All devices removed") + } + + fun login(accountToken: String): String { + Log.v(LOG_TAG, "Attempt login with account token: $accountToken") + val json = JSONObject().apply { + put("account_number", accountToken) + } + return sendSimpleSynchronousRequest(Request.Method.POST, AUTH_URL, json)!!.let { response -> + response.getString("access_token").also { accessToken -> + Log.v(LOG_TAG, "Successfully logged in and received access token: $accessToken") + } + } + } + + fun getDeviceList(accessToken: String): List<String> { + Log.v(LOG_TAG, "Get devices") + + val response = sendSimpleSynchronousRequestArray( + Request.Method.GET, + DEVICE_LIST_URL, + token = accessToken + ) + + return response!!.iterator<JSONObject>().asSequence().toList() + .also { + it + .map { jsonObject -> + jsonObject.getString("name") + } + .also { deviceNames -> + Log.v(LOG_TAG, "Devices received: $deviceNames") + } + } + .map { it.getString("id") } + .toList() + } + + fun removeDevice(token: String, deviceId: String) { + Log.v(LOG_TAG, "Remove device: $deviceId") + sendSimpleSynchronousRequestString( + Request.Method.DELETE, + "$DEVICE_LIST_URL/$deviceId", + token = token + ) + } + + private fun sendSimpleSynchronousRequest( + method: Int, + url: String, + body: JSONObject? = null, + token: String? = null + ): JSONObject? { + val future = RequestFuture.newFuture<JSONObject>() + val request = object : JsonObjectRequest( + method, + url, + body, + future, + future + ) { + override fun getHeaders(): MutableMap<String, String> { + val headers = HashMap<String, String>() + if (body != null) { + headers.put("Content-Type", "application/json") + } + if (token != null) { + headers.put("Authorization", "Bearer $token") + } + return headers + } + } + queue.add(request) + return try { + future.get().also { response -> + Log.v(LOG_TAG, "Json object request response: $response") + } + } catch (e: Exception) { + Log.v(LOG_TAG, "Json object request error: ${e.message}") + throw TestEventException(REQUEST_ERROR_MESSAGE) + } + } + + private fun sendSimpleSynchronousRequestString( + method: Int, + url: String, + body: String? = null, + token: String? = null + ): String? { + val future = RequestFuture.newFuture<String>() + val request = object : StringRequest( + method, + url, + future, + future + ) { + override fun getHeaders(): MutableMap<String, String> { + val headers = HashMap<String, String>() + if (body != null) { + headers.put("Content-Type", "application/json") + } + if (token != null) { + headers.put("Authorization", "Bearer $token") + } + return headers + } + } + queue.add(request) + return try { + future.get().also { response -> + Log.v(LOG_TAG, "String request response: $response") + } + } catch (e: Exception) { + Log.v(LOG_TAG, "String request error: ${e.message}") + throw TestEventException(REQUEST_ERROR_MESSAGE) + } + } + + private fun sendSimpleSynchronousRequestArray( + method: Int, + url: String, + body: JSONArray? = null, + token: String? = null + ): JSONArray? { + val future = RequestFuture.newFuture<JSONArray>() + val request = object : JsonArrayRequest( + method, + url, + null, + future, + future + ) { + override fun getHeaders(): MutableMap<String, String> { + val headers = HashMap<String, String>() + headers.put("Content-Type", "application/json") + if (token != null) { + headers.put("Authorization", "Bearer $token") + } + return headers + } + } + queue.add(request) + return try { + future.get().also { response -> + Log.v(LOG_TAG, "Json array request response: $response") + } + } catch (e: Exception) { + Log.v(LOG_TAG, "Json array request error: ${e.message}") + throw TestEventException(REQUEST_ERROR_MESSAGE) + } + } + + operator fun <T> JSONArray.iterator(): Iterator<T> = + (0 until this.length()).asSequence().map { this.get(it) as T }.iterator() + + companion object { + private const val AUTH_URL = + "${BuildConfig.API_BASE_URL}/auth/${BuildConfig.API_VERSION}/token" + private const val DEVICE_LIST_URL = + "${BuildConfig.API_BASE_URL}/accounts/${BuildConfig.API_VERSION}/devices" + private const val REQUEST_ERROR_MESSAGE = + "Unable to verify account due to invalid account or connectivity issues." + } +} |
