[e2e] Android Detox Test Run #3688
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # Detox e2e tests in CI | |
| # | |
| name: '[e2e] Android Detox Test Run' | |
| on: | |
| push: | |
| branches: | |
| - 'dependabot/**' | |
| pull_request: | |
| types: synchronize | |
| paths: | |
| - 'VAMobile/src/**' | |
| workflow_dispatch: | |
| inputs: | |
| run_full_test: | |
| description: 'Run full e2e test?' | |
| type: boolean | |
| required: false | |
| tests_to_run: | |
| description: 'List tests to test in ["test name", "test name"] format. Leave blank if running full e2e test.' | |
| type: string | |
| required: false | |
| testRail_name: | |
| description: 'Name testRail run (will default to date if blank)' | |
| type: string | |
| required: false | |
| run_testRail: | |
| description: 'Add results to testRail?' | |
| type: boolean | |
| required: false | |
| device_specific: | |
| description: 'Is your testRail run Android only? (only check if adding results to testRail)' | |
| type: boolean | |
| required: false | |
| schedule: | |
| - cron: '0 4 * * 1,2,3,4,5' | |
| workflow_run: | |
| workflows: ['[Release] New Release Issue'] | |
| types: | |
| - in_progress | |
| workflow_call: | |
| secrets: | |
| TEST_RAIL_USER: | |
| description: "TestRail robot userid" | |
| required: true | |
| TEST_RAIL_KEY: | |
| description: "TestRail api key" | |
| required: true | |
| concurrency: | |
| group: android-detox-${{ github.ref }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| working-directory: VAMobile | |
| env: | |
| # IAM staging app client secret | |
| APP_CLIENT_SECRET: ${{ secrets.APP_CLIENT_SECRET }} | |
| # IAM production app client secret | |
| APP_CLIENT_SECRET_PROD: ${{ secrets.APP_CLIENT_SECRET_PROD }} | |
| # Android Key Store Key Alias | |
| ANDROID_KS_KEY_ALIAS: ${{ secrets.ANDROID_KS_KEY_ALIAS }} | |
| # Android Key Store Key Alias Password | |
| ANDROID_KS_KEY_PW: ${{ secrets.ANDROID_KS_KEY_PW }} | |
| # Android Key Store Key Password | |
| ANDROID_KS_PW: ${{ secrets.ANDROID_KS_PW }} | |
| # App ID for Android project in Firebase | |
| FIREBASE_ANDROID_APP_ID: ${{ secrets.FIREBASE_ANDROID_APP_ID }} | |
| # Filepath for firebase distribution key. Also used by fastlane | |
| FIREBASE_DIST_KEY_FILEPATH: ${{ secrets.FIREBASE_DIST_KEY_FILEPATH }} | |
| # Slack API token | |
| SLACK_API_TOKEN: ${{ secrets.SLACK_API_TOKEN }} | |
| jobs: | |
| start_slack_thread: | |
| if: github.event_name == 'schedule' | |
| name: Start Slack thread | |
| uses: ./.github/workflows/start_slack_thread.yml | |
| secrets: inherit | |
| with: | |
| channel_name: va-mobile-build-alerts | |
| message: 'Starting E2E Android tests. Please see :thread: for results. This process may take a while.' | |
| find_detox_tests_to_run: | |
| if: github.event_name == 'pull_request' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' | |
| uses: ./.github/workflows/e2e_detox_mapping.yml | |
| output_detox_tests_to_run: | |
| runs-on: macos-latest-xl | |
| needs: find_detox_tests_to_run | |
| outputs: | |
| output1: ${{ steps.matrix_value.outputs.matrix }} | |
| output2: ${{ steps.matrix_value.outputs.individual_matrix}} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: 'Get Matrix Value' | |
| id: matrix_value | |
| run: | | |
| if [[ "${{ inputs.run_full_test }}" == "true" ]] || [[ ${{ github.event_name }} == 'schedule' ]] || [[ "${{ inputs.tests_to_run}}" != "" ]]; then | |
| if [[ "${{ inputs.run_full_test }}" == "true" ]] || [[ ${{ github.event_name }} == 'schedule' ]]; then | |
| e2eNames=$(gh api repos/department-of-veterans-affairs/va-mobile-app/contents/VAMobile/e2e/tests | jq --compact-output 'del(.[] | select(.name == "utils.ts")) | [.[].name]') | |
| echo "matrix=$e2eNames" >> "$GITHUB_OUTPUT" | |
| echo "individual_matrix=" >> "$GITHUB_OUTPUT" | |
| else | |
| if [[ "${{ inputs.tests_to_run}}" != "" ]]; then | |
| echo "matrix=${{ toJson(inputs.tests_to_run)}}" >> "$GITHUB_OUTPUT" | |
| echo "individual_matrix=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "matrix=" >> "$GITHUB_OUTPUT" | |
| fi | |
| fi | |
| else | |
| if [[ "${{ needs.find_detox_tests_to_run.outputs.test_run }}" != "" ]]; then | |
| echo "${{needs.find_detox_tests_to_run.outputs.test_matrix}}" | |
| if [[ "${{needs.find_detox_tests_to_run.outputs.test_matrix}}" == "[]" ]]; then | |
| if [[ "${{ inputs.run_full_test }}" == "true" ]] || [[ ${{ github.event_name }} == 'schedule' ]]; then | |
| e2eNames=$(gh api repos/department-of-veterans-affairs/va-mobile-app/contents/VAMobile/e2e/tests | jq --compact-output 'del(.[] | select(.name == "utils.ts")) | [.[].name]') | |
| echo "matrix=$e2eNames" >> "$GITHUB_OUTPUT" | |
| echo "individual_matrix=" >> "$GITHUB_OUTPUT" | |
| else | |
| if [[ "${{ inputs.tests_to_run}}" != "" ]]; then | |
| echo "matrix=${{ toJson(inputs.tests_to_run)}}" >> "$GITHUB_OUTPUT" | |
| echo "individual_matrix=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "matrix=" >> "$GITHUB_OUTPUT" | |
| fi | |
| fi | |
| else | |
| resp=${{toJson(needs.find_detox_tests_to_run.outputs.test_matrix)}} | |
| echo "matrix=$resp" >> "$GITHUB_OUTPUT" | |
| fi | |
| fi | |
| fi | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| matrix-e2e-android: | |
| if: (!cancelled()) && needs.output_detox_tests_to_run.outputs.output1 != '' | |
| runs-on: macos-latest-xl | |
| needs: [start_slack_thread, output_detox_tests_to_run] | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| testsuite: ${{fromJson(needs.output_detox_tests_to_run.outputs.output1)}} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: Set git config | |
| run: | | |
| git config --global user.name 'VA Automation Bot' | |
| git config --global user.email 'va-mobileapp@adhocteam.us' | |
| - name: Decode base64 encoded keys | |
| working-directory: VAMobile/android | |
| run: | | |
| mkdir keys | |
| cd keys | |
| echo "${{ secrets.GOOGLE_KS }}" | base64 --decode > vamobile | |
| echo "${{ secrets.GOOGLE_SA_JSON }}" | base64 --decode > service-account.json | |
| echo "${{ secrets.FIREBASE_DIST_FILE_BASE64 }}" | base64 --decode > firebase-dist.json | |
| cd ../app | |
| echo "${{ secrets.GOOGLE_SERVICES_JSON }}" | base64 --decode > google-services.json | |
| - name: Setup ruby and restore bundler cache | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: '3.0.2' | |
| bundler-cache: true | |
| working-directory: VAMobile/android | |
| - name: Set up JDK | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: 17 | |
| distribution: 'microsoft' | |
| - name: Setup node and restore yarn cache | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version-file: 'VAMobile/.nvmrc' | |
| cache: 'yarn' | |
| cache-dependency-path: './VAMobile/yarn.lock' | |
| - name: Install npm dependencies | |
| run: yarn install --frozen-lockfile --non-interactive | |
| - name: Set app environment variables | |
| run: yarn env:staging | |
| - name: Bundle Android app | |
| run: yarn e2e:android-build | |
| - name: Run e2e tests for Android - Full Test | |
| id: run_e2e_tests | |
| if: ${{needs.output_detox_tests_to_run.outputs.output2 == ''}} | |
| uses: reactivecircus/android-emulator-runner@v2 | |
| with: | |
| working-directory: VAMobile | |
| api-level: 28 | |
| profile: pixel_6_pro | |
| force-avd-creation: false | |
| disable-animations: true | |
| arch: x86_64 | |
| avd-name: Pixel_4_XL_API_28 | |
| script: yarn e2e:android-test /e2e/tests/${{matrix.testsuite}} --updateSnapshot | |
| continue-on-error: true | |
| - name: Run e2e tests for Android | |
| id: run_e2e_tests_nav_AF | |
| if: ${{needs.output_detox_tests_to_run.outputs.output2 != ''}} | |
| uses: reactivecircus/android-emulator-runner@v2 | |
| with: | |
| working-directory: VAMobile | |
| api-level: 28 | |
| profile: pixel_6_pro | |
| force-avd-creation: false | |
| disable-animations: true | |
| arch: x86_64 | |
| avd-name: Pixel_4_XL_API_28 | |
| script: yarn e2e:android-test /e2e/tests/Navigation.e2e AvailabilityFramework.e2e ${{matrix.testsuite}}.e2e --updateSnapshot | |
| - name: Run e2e tests for Android on failure | |
| if: failure() && steps.run_e2e_tests_nav_AF.outcome == 'failure' | |
| uses: reactivecircus/android-emulator-runner@v2 | |
| with: | |
| working-directory: VAMobile | |
| api-level: 28 | |
| profile: pixel_6_pro | |
| force-avd-creation: false | |
| disable-animations: true | |
| arch: x86_64 | |
| avd-name: Pixel_4_XL_API_28 | |
| script: yarn e2e:android-test /e2e/tests/Navigation.e2e AvailabilityFramework.e2e ${{matrix.testsuite}}.e2e --updateSnapshot | |
| - name: Upload e2e-junit | |
| if: failure() || success() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{matrix.testsuite}}-e2e-junit | |
| path: VAMobile/e2e/test_reports/e2e-junit.xml | |
| - name: Upload artifacts on failure | |
| if: failure() || steps.run_e2e_tests.outcome == 'failure' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: detox-artifacts-${{ runner.os }}-${{ github.run_id }}-${{matrix.testsuite}} | |
| path: VAMobile/artifacts | |
| retention-days: 1 | |
| - name: Fail workflow if needed(View e2e step for details) | |
| if: steps.run_e2e_tests.outcome == 'failure' | |
| run: exit 1 | |
| output-slack-results-and-update-detox-failure-ticket: | |
| if: (!cancelled()) && github.event_name == 'schedule' | |
| needs: [matrix-e2e-android, start_slack_thread] | |
| runs-on: macos-latest-xl | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: Download results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: VAMobile/e2e/test_reports | |
| - name: Inform Slack | |
| id: inform_slack | |
| shell: bash | |
| run: | | |
| if grep -rq "<failure>" .; then | |
| echo SLACK_THREAD_TS=${ts} >> $GITHUB_OUTPUT | |
| ts=$(curl -X POST -H 'Authorization: Bearer '"$SLACK_API_TOKEN"' ' \ | |
| -H 'Content-type: application/json' \ | |
| --data '{"channel":"'${{needs.start_slack_thread.outputs.channel_id}}'","thread_ts":"'${{needs.start_slack_thread.outputs.thread_ts}}'","text":"Failed! Please check workflow for results: https://github.com/department-of-veterans-affairs/va-mobile-app/actions"}' \ | |
| https://slack.com/api/chat.postMessage| | |
| jq -r '.ts') | |
| echo SLACK_THREAD_TS=${ts} >> $GITHUB_OUTPUT | |
| echo "TEST_FAILURE=true" >> "$GITHUB_OUTPUT" | |
| else | |
| ts=$(curl -X POST -H 'Authorization: Bearer '"$SLACK_API_TOKEN"' ' \ | |
| -H 'Content-type: application/json' \ | |
| --data '{"channel":"'${{needs.start_slack_thread.outputs.channel_id}}'","thread_ts":"'${{needs.start_slack_thread.outputs.thread_ts}}'","text":"Success! All E2E tests have passed"}' \ | |
| https://slack.com/api/chat.postMessage| | |
| jq -r '.ts') | |
| fi | |
| - name: Find failing tests | |
| id: failing_tests | |
| shell: bash | |
| run: | | |
| if grep -rq "<failure>" .; then | |
| failing_tests_list=$(grep -rl "<failure>") | |
| failing_tests="" | |
| for i in ${failing_tests_list[@]}; do | |
| remove_before=${i%.ts*} | |
| remove_after=${remove_before#*test_reports/} | |
| echo "$remove_after" | |
| if [ -z "${failing_tests}" ]; then | |
| failing_tests=$(echo $failing_tests"$remove_after") | |
| else | |
| failing_tests=$(echo $failing_tests", ""$remove_after") | |
| fi | |
| done | |
| echo "FAILING_TEST=$failing_tests" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Github CLI Authentication | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" >> token.txt | |
| gh auth login --with-token < token.txt | |
| - name: Update a ticket on failure (if needed) | |
| if: steps.inform_slack.outputs.TEST_FAILURE != '' | |
| id: find_if_ticket_already_exists | |
| run: | | |
| old_body=$(gh issue list --json body --jq '.[] | .body' --state open --search "Bug - Detox - Fix Overnight Failures in:title") | |
| if [[ "$old_body" != "" ]]; then | |
| ticket_number=$(gh issue list --json number --jq '.[] | .number' --state open --search "Bug - Detox - Fix Overnight Failures in:title") | |
| new_body=$(echo "$old_body"$'\n'"- "${{ env.DATE_OF_ISSUE}}" on Android: "${{steps.failing_tests.outputs.FAILING_TEST}}) | |
| echo "$new_body" | |
| echo "TICKET_EXISTS=true" >> "$GITHUB_OUTPUT" | |
| gh issue edit $ticket_number -b "$new_body" | |
| else | |
| echo "TICKET_EXISTS=" >> "$GITHUB_OUTPUT" | |
| fi | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Make a ticket on failure (if needed) | |
| if: steps.inform_slack.outputs.TEST_FAILURE != '' && steps.find_if_ticket_already_exists.outputs.TICKET_EXISTS == '' | |
| uses: JasonEtco/create-an-issue@v2 | |
| with: | |
| filename: .github/ISSUE_TEMPLATE/detox_nightly_build_failure.md | |
| search_existing: open | |
| env: | |
| dateOfIssue: ${{ env.DATE_OF_ISSUE }} | |
| OS: "Android" | |
| issues: ${{steps.failing_tests.outputs.FAILING_TEST}} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| id: create-issue | |
| matrix_send_test_results_to_testrail: | |
| if: (!cancelled()) && github.event.inputs.run_testRail == 'true' | |
| needs: [matrix-e2e-android, output_detox_tests_to_run] | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| testsuite: ${{ fromJSON(needs.output_detox_tests_to_run.outputs.output1) }} | |
| name: Update testRail Results | |
| uses: ./.github/workflows/update_testrail_results.yml | |
| with: | |
| test_names: "${{matrix.testsuite}}" | |
| testRail_name: ${{ inputs.testRail_name }} | |
| test_specific_OS_needed: ${{ inputs.device_specific}} | |
| test_OS_name: "Android" | |
| secrets: inherit | |
| update-test-names: | |
| if: (!cancelled()) && github.event.inputs.run_testRail == 'true' | |
| needs: [matrix-e2e-android, output_detox_tests_to_run, matrix_send_test_results_to_testrail] | |
| continue-on-error: false | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: update-test-names | |
| run: | | |
| for ((i=0; ; i+=250)); do | |
| echo "i = $i" | |
| getTestCaseNewName=$(curl -X GET -H 'Content-Type: application/json' \ | |
| -u "${{secrets.TEST_RAIL_USER}}:${{secrets.TEST_RAIL_KEY}}" \ | |
| "https://dsvavsp.testrail.io//index.php?/api/v2/get_tests/${{ needs.matrix_send_test_results_to_testrail.outputs.test_run_id_number }}&offset=$i") | |
| echo $(jq --compact-output '[.tests[] | .title |= sub("(]).*$"; "]")]' <<< "$getTestCaseNewName") >> input.json | |
| if [[ $(jq '._links.next == null' <<< "$getTestCaseNewName") == 'true' ]]; | |
| then | |
| break | |
| fi | |
| done | |
| getTestCaseNewName=$(jq --compact-output -s 'add' input.json) | |
| groupTestCasesByTitle=$(echo $getTestCaseNewName | jq --compact-output '[group_by(.title)[] | [.[0].title, .[0].status_id, map(.id)[]]]') | |
| getFirstTestNewName=$(echo $groupTestCasesByTitle | jq --compact-output '[.[] | .[0]]') | |
| getCasesID=$(curl -X GET -H 'Content-Type: application/json' \ | |
| -u "${{secrets.TEST_RAIL_USER}}:${{secrets.TEST_RAIL_KEY}}" \ | |
| "https://dsvavsp.testrail.io//index.php?/api/v2/get_cases/29") | |
| updateRunWithNewCases=$(curl -X POST -H 'Content-Type: application/json' \ | |
| -u "${{secrets.TEST_RAIL_USER}}:${{secrets.TEST_RAIL_KEY}}" \ | |
| -d '{"suite_id": 92, "include_all": false, "case_ids": '$(echo $getCasesID | jq --argjson testRailNames "$getFirstTestNewName" --compact-output '.cases | map(select(.title == $testRailNames[])) | map(.id)')'}' \ | |
| "https://dsvavsp.testrail.io//index.php?/api/v2/update_run/${{needs.matrix_send_test_results_to_testrail.outputs.test_run_id_number}}") | |
| getTestCases=$(curl -X GET -H 'Content-Type: application/json' \ | |
| -u "${{secrets.TEST_RAIL_USER}}:${{secrets.TEST_RAIL_KEY}}" \ | |
| "https://dsvavsp.testrail.io//index.php?/api/v2/get_tests/${{needs.matrix_send_test_results_to_testrail.outputs.test_run_id_number}}") | |
| for i in $(echo $getTestCases | jq -r '.tests[].id'); do | |
| getTestName=$(echo $getTestCases | jq --argjson testIDs "$i" -r '.tests | map(select(.id == $testIDs)) | .[0].title') | |
| getFailedTests=$(echo $getTestCaseNewName | jq --compact-output 'map(select(.status_id == 5)) | map(.title)') | |
| isTestFailed=$(echo $getFailedTests | jq 'any(. == "'"$getTestName"'")') | |
| getRetestTests=$(echo $getTestCaseNewName | jq --compact-output 'map(select(.status_id == 4)) | map(.title)') | |
| isTestRetest=$(echo $getRetestTests | jq 'any(. == "'"$getTestName"'")') | |
| getPassedTests=$(echo $getTestCaseNewName | jq --compact-output 'map(select(.status_id == 1)) | map(.title)') | |
| isTestPassed=$(echo $getPassedTests | jq 'any(. == "'"$getTestName"'")') | |
| if [[ "$isTestFailed" == "true" ]]; then | |
| resp5=$(curl -X POST -H 'Content-Type: application/json' \ | |
| -u "${{secrets.TEST_RAIL_USER}}:${{secrets.TEST_RAIL_KEY}}" \ | |
| -d '{"status_id": 5}' \ | |
| "https://dsvavsp.testrail.io//index.php?/api/v2/add_result/"$i"") | |
| elif [[ "$isTestRetest" == "true" ]]; then | |
| resp5=$(curl -X POST -H 'Content-Type: application/json' \ | |
| -u "${{secrets.TEST_RAIL_USER}}:${{secrets.TEST_RAIL_KEY}}" \ | |
| -d '{"status_id": 4}' \ | |
| "https://dsvavsp.testrail.io//index.php?/api/v2/add_result/"$i"") | |
| elif [[ "$isTestPassed" == "true" ]]; then | |
| resp5=$(curl -X POST -H 'Content-Type: application/json' \ | |
| -u "${{secrets.TEST_RAIL_USER}}:${{secrets.TEST_RAIL_KEY}}" \ | |
| -d '{"status_id": 1}' \ | |
| "https://dsvavsp.testrail.io//index.php?/api/v2/add_result/"$i"") | |
| fi | |
| done |