Move iOS end to end tests to staging environment

3 files changed, 58 insertions, 6 deletions

diff --git a/.github/actions/ios-end-to-end-tests/action.yml b/.github/actions/ios-end-to-end-tests/action.yml
index 9d771d151c..fa2d198a44 100644
--- a/.github/actions/ios-end-to-end-tests/action.yml
+++ b/.github/actions/ios-end-to-end-tests/action.yml
@@ -19,10 +19,22 @@ inputs:
   xcode_test_plan:
     description: 'Xcode Test Plan to run'
     required: true
+  partner_api_token:
+    description: 'Partner API Token'
+    required: true
+  test_name:
+    description: 'Test case/suite name. Will run all tests in the test plan if not provided.'
+    required: false
 
 runs:
   using: 'composite'
   steps:
+    - name: Make sure app is not installed
+      run: ios-deploy --id $IOS_TEST_DEVICE_UDID --uninstall_only --bundle_id net.mullvad.MullvadVPN
+      shell: bash
+      env:
+        IOS_TEST_DEVICE_UDID: ${{ inputs.test_device_udid }}
+
     - name: Configure Xcode project
       run: |
         for file in *.xcconfig.template ; do cp $file ${file//.template/} ; done
@@ -34,8 +46,12 @@ runs:
           sed -i "" \
             "/TEST_DEVICE_IDENTIFIER_UUID =/ s/= .*/= $TEST_DEVICE_IDENTIFIER_UUID/" \
             UITests.xcconfig
-          echo -e "\nHAS_TIME_ACCOUNT_NUMBER = $HAS_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig
-          echo "NO_TIME_ACCOUNT_NUMBER = $NO_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig
+          sed -i "" \
+            "/PARTNER_API_TOKEN =/ s#= .*#= $PARTNER_API_TOKEN#" \
+            UITests.xcconfig
+          sed -i "" \
+            "/ATTACH_APP_LOGS_ON_FAILURE =/ s#= .*#= 1#" \
+            UITests.xcconfig
       shell: bash
       working-directory: ios/Configurations
       env:
@@ -43,17 +59,31 @@ runs:
         TEST_DEVICE_IDENTIFIER_UUID: ${{ inputs.test_device_identifier_uuid }}
         HAS_TIME_ACCOUNT_NUMBER: ${{ inputs.has_time_account_number }}
         NO_TIME_ACCOUNT_NUMBER: ${{ inputs.no_time_account_number }}
+        PARTNER_API_TOKEN: ${{ inputs.partner_api_token }}
 
     - name: Run end-to-end-tests
       run: |
+        if [ -n "$TEST_NAME" ]; then
+          TEST_NAME_ARGUMENT=" -only-testing $TEST_NAME"
+        else
+          TEST_NAME_ARGUMENT=""
+        fi
         set -o pipefail && env NSUnbufferedIO=YES xcodebuild \
           -project MullvadVPN.xcodeproj \
           -scheme MullvadVPNUITests \
-          -testPlan $XCODE_TEST_PLAN \
+          -testPlan $XCODE_TEST_PLAN $TEST_NAME_ARGUMENT \
+          -resultBundlePath xcode-test-report \
           -destination "platform=iOS,id=$TEST_DEVICE_UDID" \
-          clean test 2>&1 | xcbeautify --report junit --report-path test-report
+          clean test 2>&1 | xcbeautify --report junit --report-path junit-test-report
       shell: bash
       working-directory: ios/
       env:
         XCODE_TEST_PLAN: ${{ inputs.xcode_test_plan }}
         TEST_DEVICE_UDID: ${{ inputs.test_device_udid }}
+        TEST_NAME: ${{ inputs.test_name }}
+
+    - name: Uninstall app if still installed
+      run: ios-deploy --id $IOS_TEST_DEVICE_UDID --uninstall_only --bundle_id net.mullvad.MullvadVPN
+      shell: bash
+      env:
+        IOS_TEST_DEVICE_UDID: ${{ inputs.test_device_udid }}
diff --git a/.github/workflows/ios-end-to-end-tests-settings-migration.yml b/.github/workflows/ios-end-to-end-tests-settings-migration.yml
index a5a27fd645..2ff8e1a2ba 100644
--- a/.github/workflows/ios-end-to-end-tests-settings-migration.yml
+++ b/.github/workflows/ios-end-to-end-tests-settings-migration.yml
@@ -42,6 +42,7 @@ jobs:
           no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
           test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
           xcode_test_plan: 'MullvadVPNUITestsChangeDNSSettings'
+          partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }}
 
       - name: Store test report for changing DNS settings
         uses: actions/upload-artifact@v4
@@ -62,6 +63,7 @@ jobs:
           has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
           no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
           test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
+          partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }}
           xcode_test_plan: 'MullvadVPNUITestsVerifyDNSSettingsChanged'
 
       - name: Store test report for verifying DNS settings
@@ -85,6 +87,7 @@ jobs:
           has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
           no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
           test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
+          partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }}
           xcode_test_plan: 'MullvadVPNUITestsChangeSettings'
 
       - name: Store test report for changing all settings
@@ -106,6 +109,7 @@ jobs:
           has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
           no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
           test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
+          partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }}
           xcode_test_plan: 'MullvadVPNUITestsVerifySettingsChanged'
 
       - name: Store test report for verifying all other settings
diff --git a/.github/workflows/ios-end-to-end-tests.yml b/.github/workflows/ios-end-to-end-tests.yml
index 64d494b8ab..8d1c8178d0 100644
--- a/.github/workflows/ios-end-to-end-tests.yml
+++ b/.github/workflows/ios-end-to-end-tests.yml
@@ -14,6 +14,12 @@ on:
       - .github/workflows/ios-end-to-end-tests.yml
       - ios/**
   workflow_dispatch:
+    inputs:
+      # Optionally specify a test case or suite to run.
+      # Must be in the format MullvadVPNUITest/<test-suite-name>/<test-case-name> where test case name is optional.
+      test_name:
+        description: 'Only run test case/suite'
+        required: false
   schedule:
     # At midnight every day.
     # Notifications for scheduled workflows are sent to the user who last modified the cron
@@ -26,6 +32,7 @@ jobs:
     if: github.event.pull_request.merged || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
     name: End to end tests
     runs-on: [self-hosted, macOS, ios-test]
+    timeout-minutes: 60
     steps:
       - name: Configure Rust
         uses: actions-rs/toolchain@v1
@@ -50,11 +57,13 @@ jobs:
         uses: ./.github/actions/ios-end-to-end-tests
         with:
           xcode_test_plan: ${{ env.XCODE_TEST_PLAN }}
+          test_name: ${{ github.event.inputs.test_name }}
           ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }}
           test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
           has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
           no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
           test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
+          partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }}
 
       - name: Comment PR on test failure
         if: failure() && github.event_name == 'pull_request'
@@ -76,5 +85,14 @@ jobs:
         if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: test-report
-          path: ios/test-report/junit.xml
+          name: test-results
+          path: |
+            ios/junit-test-report/junit.xml
+            ios/xcode-test-report.xcresult
+
+      - name: Store app log artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: app-logs
+          path: ios/xcode-test-report/**/app-log-*.log