Add settings migration tests for iOS

3 files changed, 207 insertions, 37 deletions

diff --git a/.github/actions/ios-end-to-end-tests/action.yml b/.github/actions/ios-end-to-end-tests/action.yml
new file mode 100644
index 0000000000..9d771d151c
--- /dev/null
+++ b/.github/actions/ios-end-to-end-tests/action.yml
@@ -0,0 +1,59 @@
+name: 'iOS end to end tests action'
+description: 'Prepares and runs end to end tests on iOS device'
+inputs:
+  ios_device_pin_code:
+    description: 'iOS Device Pin Code'
+    required: true
+  test_device_identifier_uuid:
+    description: 'Test Device Identifier UUID'
+    required: true
+  has_time_account_number:
+    description: 'Has Time Account Number'
+    required: true
+  no_time_account_number:
+    description: 'No Time Account Number'
+    required: true
+  test_device_udid:
+    description: 'Test Device UDID'
+    required: true
+  xcode_test_plan:
+    description: 'Xcode Test Plan to run'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    - name: Configure Xcode project
+      run: |
+        for file in *.xcconfig.template ; do cp $file ${file//.template/} ; done
+          sed -i "" "/^HAS_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig
+          sed -i "" "/^NO_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig
+          sed -i "" \
+            "/IOS_DEVICE_PIN_CODE =/ s/= .*/= $IOS_DEVICE_PIN_CODE/" \
+            UITests.xcconfig
+          sed -i "" \
+            "/TEST_DEVICE_IDENTIFIER_UUID =/ s/= .*/= $TEST_DEVICE_IDENTIFIER_UUID/" \
+            UITests.xcconfig
+          echo -e "\nHAS_TIME_ACCOUNT_NUMBER = $HAS_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig
+          echo "NO_TIME_ACCOUNT_NUMBER = $NO_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig
+      shell: bash
+      working-directory: ios/Configurations
+      env:
+        IOS_DEVICE_PIN_CODE: ${{ inputs.ios_device_pin_code }}
+        TEST_DEVICE_IDENTIFIER_UUID: ${{ inputs.test_device_identifier_uuid }}
+        HAS_TIME_ACCOUNT_NUMBER: ${{ inputs.has_time_account_number }}
+        NO_TIME_ACCOUNT_NUMBER: ${{ inputs.no_time_account_number }}
+
+    - name: Run end-to-end-tests
+      run: |
+        set -o pipefail && env NSUnbufferedIO=YES xcodebuild \
+          -project MullvadVPN.xcodeproj \
+          -scheme MullvadVPNUITests \
+          -testPlan $XCODE_TEST_PLAN \
+          -destination "platform=iOS,id=$TEST_DEVICE_UDID" \
+          clean test 2>&1 | xcbeautify --report junit --report-path test-report
+      shell: bash
+      working-directory: ios/
+      env:
+        XCODE_TEST_PLAN: ${{ inputs.xcode_test_plan }}
+        TEST_DEVICE_UDID: ${{ inputs.test_device_udid }}
diff --git a/.github/workflows/ios-end-to-end-tests-settings-migration.yml b/.github/workflows/ios-end-to-end-tests-settings-migration.yml
new file mode 100644
index 0000000000..180f454cec
--- /dev/null
+++ b/.github/workflows/ios-end-to-end-tests-settings-migration.yml
@@ -0,0 +1,116 @@
+---
+name: iOS settings migration tests
+permissions:
+  contents: read
+on:
+  workflow_dispatch:
+  schedule:
+    # At midnight every day.
+    # Notifications for scheduled workflows are sent to the user who last modified the cron
+    # syntax in the workflow file. If you update this you must have notifications for
+    # Github Actions enabled, so these don't go unnoticed.
+    # https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/notifications-for-workflow-runs
+    - cron: '0 0 * * *'
+jobs:
+  test:
+    name: Settings migration end to end tests
+    runs-on: [self-hosted, macOS, ios-test]
+    env:
+      OLD_APP_COMMIT_HASH: f741bf159d5719301e4d1fd099b3d353457532fd
+    steps:
+      - name: Configure Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
+          target: aarch64-apple-ios
+
+      - name: Uninstall app
+        run: ios-deploy --id ${{ secrets.IOS_TEST_DEVICE_UDID }} --uninstall_only --bundle_id net.mullvad.MullvadVPN
+
+      - name: Checkout old repository version
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.OLD_APP_COMMIT_HASH }}
+
+      - name: Change DNS settings on old app version
+        uses: ./.github/actions/ios-end-to-end-tests
+        with:
+          ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }}
+          test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
+          has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
+          no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
+          test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
+          xcode_test_plan: 'MullvadVPNUITestsChangeDNSSettings'
+
+      - name: Store test report for changing DNS settings
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: test-report-change-dns-settings
+          path: ios/test-report/junit.xml
+
+      - name: Checkout repository to get the current app version
+        uses: actions/checkout@v4
+
+      - name: Verify DNS settings still changed on current app version
+        uses: ./.github/actions/ios-end-to-end-tests
+        if: always()
+        with:
+          ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }}
+          test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
+          has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
+          no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
+          test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
+          xcode_test_plan: 'MullvadVPNUITestsVerifyDNSSettingsChanged'
+
+      - name: Store test report for verifying DNS settings
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: test-report-verify-dns-settings
+          path: ios/test-report/junit.xml
+
+      - name: Checkout old repository version
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.OLD_APP_COMMIT_HASH }}
+
+      - name: Change all other settings on old app version
+        uses: ./.github/actions/ios-end-to-end-tests
+        if: always()
+        with:
+          ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }}
+          test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
+          has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
+          no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
+          test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
+          xcode_test_plan: 'MullvadVPNUITestsChangeSettings'
+
+      - name: Store test report for changing all settings
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: test-report-change-all-other-settings
+          path: ios/test-report/junit.xml
+
+      - name: Checkout repository to get the current app version
+        uses: actions/checkout@v4
+
+      - name: Verify all other settings still changed on current app version
+        uses: ./.github/actions/ios-end-to-end-tests
+        if: always()
+        with:
+          ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }}
+          test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
+          has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
+          no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
+          test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
+          xcode_test_plan: 'MullvadVPNUITestsVerifySettingsChanged'
+
+      - name: Store test report for verifying all other settings
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: test-report-verify-all-other-settings
+          path: ios/test-report/junit.xml
diff --git a/.github/workflows/ios-end-to-end-tests.yml b/.github/workflows/ios-end-to-end-tests.yml
index 6f953cd346..b44665649f 100644
--- a/.github/workflows/ios-end-to-end-tests.yml
+++ b/.github/workflows/ios-end-to-end-tests.yml
@@ -14,56 +14,51 @@ on:
       - .github/workflows/ios-end-to-end-tests.yml
       - ios/**
   workflow_dispatch:
+  schedule:
+    # At midnight every day.
+    # Notifications for scheduled workflows are sent to the user who last modified the cron
+    # syntax in the workflow file. If you update this you must have notifications for
+    # Github Actions enabled, so these don't go unnoticed.
+    # https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/notifications-for-workflow-runs
+    - cron: '0 0 * * *'
 jobs:
   test:
     if: github.event.pull_request.merged || github.event_name == 'workflow_dispatch'
     name: End to end tests
     runs-on: [self-hosted, macOS, ios-test]
-    env:
-      IOS_DEVICE_PIN_CODE: ${{ secrets.IOS_DEVICE_PIN_CODE }}
-      TEST_DEVICE_IDENTIFIER_UUID: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
-      TEST_DEVICE_UDID: ${{ secrets.IOS_TEST_DEVICE_UDID }}
-      HAS_TIME_ACCOUNT_NUMBER: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
-      NO_TIME_ACCOUNT_NUMBER: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
     steps:
+      - name: Configure Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
+          target: aarch64-apple-ios
+
       - name: Checkout repository
         uses: actions/checkout@v4
 
-      - name: Configure Rust
-        run: |
-          rustup default stable
-          rustup update stable
-          rustup target add aarch64-apple-ios aarch64-apple-ios-sim
-
-      - name: Configure Xcode project
+      - name: Select test plan to execute
         run: |
-          for file in *.xcconfig.template ; do cp $file ${file//.template/} ; done
-          sed -i "" "/^HAS_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig
-          sed -i "" "/^NO_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig
-          sed -i "" \
-            "/IOS_DEVICE_PIN_CODE =/ s/= .*/= $IOS_DEVICE_PIN_CODE/" \
-            UITests.xcconfig
-          sed -i "" \
-            "/TEST_DEVICE_IDENTIFIER_UUID =/ s/= .*/= $TEST_DEVICE_IDENTIFIER_UUID/" \
-            UITests.xcconfig
-          echo -e "\nHAS_TIME_ACCOUNT_NUMBER = $HAS_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig
-          echo "NO_TIME_ACCOUNT_NUMBER = $NO_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig
-        working-directory: ios/Configurations
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            echo "XCODE_TEST_PLAN=MullvadVPNUITestsSmoke" >> $GITHUB_ENV
+          elif [[ "${{ github.event_name }}" == "schedule" ]]; then
+            echo "XCODE_TEST_PLAN=MullvadVPNUITestsAll" >> $GITHUB_ENV
+          elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            echo "XCODE_TEST_PLAN=MullvadVPNUITestsAll" >> $GITHUB_ENV
+          fi
 
-      - name: Run end-to-end-tests
-        run: |
-          set -o pipefail && env NSUnbufferedIO=YES xcodebuild \
-            -project MullvadVPN.xcodeproj \
-            -scheme MullvadVPNUITests \
-            -configuration Debug \
-            -testPlan MullvadVPNUITestsSmoke \
-            -destination "platform=iOS,id=$TEST_DEVICE_UDID" \
-            -disableAutomaticPackageResolution \
-            test 2>&1 | xcbeautify --report junit --report-path test-report
-        working-directory: ios/
+      - name: iOS end to end tests action
+        uses: ./.github/actions/ios-end-to-end-tests
+        with:
+          xcode_test_plan: ${{ env.XCODE_TEST_PLAN }}
+          ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }}
+          test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
+          has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
+          no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
+          test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
 
       - name: Comment PR on test failure
-        if: failure() && github.event_name != 'workflow_dispatch'
+        if: failure() && github.event_name == 'pull_request'
         uses: actions/github-script@v7
         with:
           github-token: ${{secrets.GITHUB_TOKEN}}