integration-tests.yml

name: Integration Tests
run-name: Integration Tests ${{ inputs.distinct_run_name && inputs.distinct_run_name || '' }}
on:
  merge_group:
  pull_request:
  push:
    tags:
      - "*"
  workflow_dispatch:
    inputs:
      cl_ref:
        description: 'The ref to checkout, defaults to the calling branch'
        required: false
        type: string
      evm-ref:
        description: 'The sha of the chainlink-evm commit to use if wanted'
        required: false
        type: string
      run_solana:
        description: 'Run solana tests'
        required: false
        type: string
        default: 'false'
      distinct_run_name:
        description: 'A unique identifier for this run, only use from other repos'
        required: false
        type: string

# Only run 1 of this workflow at a time per PR
concurrency:
  group: ${{ github.ref }}-${{ github.repository }}-${{ github.event_name }}--e2e-tests-${{ inputs.distinct_run_name }}
  cancel-in-progress: true

env:
  # for run-test variables and environment
  ENV_JOB_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-tests:${{ inputs.evm-ref || github.sha }}
  CHAINLINK_IMAGE: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink
  TEST_SUITE: smoke
  TEST_ARGS: -test.timeout 12m
  INTERNAL_DOCKER_REPO: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com
  MOD_CACHE_VERSION: 2
  COLLECTION_ID: chainlink-e2e-tests

jobs:
  enforce-ctf-version:
    name: Enforce CTF Version
    runs-on: ubuntu-latest
    # We don't directly merge dependabot PRs, so let's not waste the resources
    if: github.actor != 'dependabot[bot]'
    steps:
      - run: echo "${{github.event_name}}"
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: smartcontractkit/chainlink
          ref: ${{ inputs.cl_ref }}
      - name: Check Merge Group Condition
        id: condition-check
        run: |
          echo "Checking event condition..."
          SHOULD_ENFORCE="false"
          if [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then
              echo "We are in a merge_group event, now check if we are on the develop branch"
              target_branch=$(cat $GITHUB_EVENT_PATH | jq -r .merge_group.base_ref)
              if [[ "$target_branch" == "refs/heads/develop" ]]; then
                  echo "We are on the develop branch, we should enforce ctf version"
                  SHOULD_ENFORCE="true"
              fi
          fi
          echo "should we enforce ctf version = $SHOULD_ENFORCE"
          echo "should-enforce=$SHOULD_ENFORCE" >> $GITHUB_OUTPUT
      - name: Enforce CTF Version
        if: steps.condition-check.outputs.should-enforce == 'true'
        uses: smartcontractkit/.github/actions/ctf-check-mod-version@21b0189c5fdca0318617d259634b1a91e6d80262 # ctf-check-mod-version@0.0.0
        with:
          go-project-path: ./integration-tests
          module-name: github.com/smartcontractkit/chainlink-testing-framework/lib
          enforce-semantic-tag: "true"
  changes:
    environment: integration
    name: Check Paths That Require Tests To Run
    runs-on: ubuntu-latest
    # We don't directly merge dependabot PRs, so let's not waste the resources
    if: github.actor != 'dependabot[bot]'
    steps:
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: smartcontractkit/chainlink
          ref: ${{ inputs.cl_ref }}
      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
        id: changes
        with:
          filters: |
            github_ci_changes:
              - '.github/workflows/integration-tests.yml'
              - '.github/workflows/run-e2e-tests-reusable-workflow.yml'
              - '.github/e2e-tests.yml'
            core_changes:
              - '**/*.go'
              - '**/*go.sum'
              - '**/*go.mod'
              - '**/*Dockerfile'
              - 'core/**/migrations/*.sql'
              - 'core/**/config/**/*.toml'
              - 'integration-tests/**/*.toml'
            ccip_changes:
              - '**/*ccip*'
              - '**/*ccip*/**'
      - name: Ignore Filter On Workflow Dispatch
        if: ${{ github.event_name == 'workflow_dispatch' }}
        id: ignore-filter
        run: echo "changes=true" >> $GITHUB_OUTPUT
      - name: Collect Metrics
        if: always()
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
        with:
          id: ${{ env.COLLECTION_ID }}-check-paths
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Check Paths That Require Tests To Run
        continue-on-error: true
    outputs:
      github_ci_changes: ${{ steps.ignore-filter.outputs.changes || steps.changes.outputs.github_ci_changes }}
      core_changes: ${{ steps.ignore-filter.outputs.changes || steps.changes.outputs.core_changes }}
      ccip_changes: ${{ steps.ignore-filter.outputs.changes || steps.changes.outputs.ccip_changes }}

  lint-integration-tests:
    name: Lint ${{ matrix.project.name }}
    runs-on: ubuntu22.04-8cores-32GB
    # We don't directly merge dependabot PRs, so let's not waste the resources
    if: github.actor != 'dependabot[bot]'
    strategy:
      matrix:
        project:
          - name: integration-tests
            id: e2e-tests
            path: ./integration-tests
            cache_id: e2e-tests
          - name: load
            id: load
            path: ./integration-tests/load
            cache_id: load
    steps:
      - name: Collect Metrics
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
        with:
          id: ${{ env.COLLECTION_ID }}-build-lint-${{ matrix.project.id }}
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Lint ${{ matrix.project.name }}
        continue-on-error: true
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: smartcontractkit/chainlink
          ref: ${{ inputs.cl_ref }}
      - name: Setup Go
        uses: smartcontractkit/.github/actions/ctf-setup-go@b0d756c57fcdbcff187e74166562a029fdd5d1b9 # ctf-setup-go@0.0.0
        with:
          test_download_vendor_packages_command: cd ${{ matrix.project.path }} && go mod download
          go_mod_path: ${{ matrix.project.path }}/go.mod
          cache_key_id: ${{ matrix.project.cache_id }}
          cache_restore_only: "true"
      - name: Lint Go
        uses: golangci/golangci-lint-action@3cfe3a4abbb849e10058ce4af15d205b6da42804 # v4.0.0
        with:
          version: v1.59.1
          # We already cache these directories in setup-go
          skip-pkg-cache: true
          skip-build-cache: true
          # only-new-issues is only applicable to PRs, otherwise it is always set to false
          only-new-issues: false # disabled for PRs due to unreliability
          args: --out-format colored-line-number,checkstyle:golangci-lint-report.xml
          working-directory: ${{ matrix.project.path }}

  build-chainlink:
    environment: integration
    permissions:
      id-token: write
      contents: read
    strategy:
      matrix:
        image:
          - name: ""
            dockerfile: core/chainlink.Dockerfile
            tag-suffix: ""
          - name: (plugins)
            dockerfile: plugins/chainlink.Dockerfile
            tag-suffix: -plugins
    name: Build Chainlink Image ${{ matrix.image.name }}
    runs-on: ubuntu22.04-8cores-32GB
    needs: [changes, enforce-ctf-version]
    steps:
      - name: Collect Metrics
        if: needs.changes.outputs.core_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true' || github.event_name == 'workflow_dispatch'
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
        with:
          id: ${{ env.COLLECTION_ID }}-build-chainlink
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Build Chainlink Image ${{ matrix.image.name }}
        continue-on-error: true
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: smartcontractkit/chainlink
          ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Setup Github Token
        if: ${{ inputs.evm-ref }}
        id: get-gh-token
        uses: smartcontractkit/.github/actions/setup-github-token@ef78fa97bf3c77de6563db1175422703e9e6674f # setup-github-token@0.2.1
        with:
          aws-role-arn: ${{ secrets.AWS_OIDC_GLOBAL_READ_ONLY_TOKEN_ISSUER_ROLE_ARN }}
          aws-lambda-url: ${{ secrets.AWS_INFRA_RELENG_TOKEN_ISSUER_LAMBDA_URL }}
          aws-region: ${{ secrets.AWS_REGION }}
          set-git-config: "true"
      - name: Build Chainlink Image
        if: needs.changes.outputs.core_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true' || github.event_name == 'workflow_dispatch'
        uses: ./.github/actions/build-chainlink-image
        with:
          tag_suffix: ${{ matrix.image.tag-suffix }}
          dockerfile: ${{ matrix.image.dockerfile }}
          git_commit_sha: ${{ inputs.evm-ref || github.sha }}
          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          dep_evm_sha: ${{ inputs.evm-ref }}

  run-core-e2e-tests-for-pr:
    name: Run Core E2E Tests For PR
    permissions:
      actions: read
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    needs: [build-chainlink, changes]
    if: github.event_name == 'pull_request' && ( needs.changes.outputs.core_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true')
    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
    with:
      workflow_name: Run Core E2E Tests For PR
      chainlink_version: ${{ inputs.evm-ref || github.sha }}
      chainlink_upgrade_version: ${{ github.sha }}
      test_workflow: PR E2E Core Tests
      upload_cl_node_coverage_artifact: true
      upload_cl_node_coverage_artifact_prefix: cl_node_coverage_data_
      enable_otel_traces_for_ocr2_plugins: ${{ contains(join(github.event.pull_request.labels.*.name, ' '), 'enable tracing') }}
    secrets:
      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
      LOKI_URL: ${{ secrets.LOKI_URL }}
      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
      SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}

  run-core-e2e-tests-for-merge-queue:
    name: Run Core E2E Tests For Merge Queue
    permissions:
      actions: read
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    needs: [build-chainlink, changes]
    if: github.event_name == 'merge_group' && ( needs.changes.outputs.core_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true')
    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
    with:
      workflow_name: Run Core E2E Tests For Merge Queue
      chainlink_version: ${{ inputs.evm-ref || github.sha }}
      chainlink_upgrade_version: ${{ github.sha }}
      test_workflow: Merge Queue E2E Core Tests
      upload_cl_node_coverage_artifact: true
      upload_cl_node_coverage_artifact_prefix: cl_node_coverage_data_
      enable_otel_traces_for_ocr2_plugins: ${{ contains(join(github.event.pull_request.labels.*.name, ' '), 'enable tracing') }}
      # Notify Test Tooling team in slack when merge queue tests fail
      slack_notification_after_tests: on_failure
      slack_notification_after_tests_channel_id: "#team-test-tooling-internal"
      slack_notification_after_tests_name: Core E2E Tests In Merge Queue
    secrets:
      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
      LOKI_URL: ${{ secrets.LOKI_URL }}
      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
      SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
              
  run-ccip-e2e-tests-for-pr:
    name: Run CCIP E2E Tests For PR
    permissions:
      actions: read
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    needs: [build-chainlink, changes]
    if: github.event_name == 'pull_request' && (needs.changes.outputs.ccip_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true')
    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
    with:
      workflow_name: Run CCIP E2E Tests For PR
      chainlink_version: ${{ inputs.evm-ref || github.sha }}
      chainlink_upgrade_version: ${{ github.sha }}
      test_workflow: PR E2E CCIP Tests
      upload_cl_node_coverage_artifact: true
      upload_cl_node_coverage_artifact_prefix: cl_node_coverage_data_
      enable_otel_traces_for_ocr2_plugins: ${{ contains(join(github.event.pull_request.labels.*.name, ' '), 'enable tracing') }}
    secrets:
      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
      LOKI_URL: ${{ secrets.LOKI_URL }}
      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
      SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}      
              
  run-ccip-e2e-tests-for-merge-queue:
    name: Run CCIP E2E Tests For Merge Queue
    permissions:
      actions: read
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    needs: [build-chainlink, changes]
    if: github.event_name == 'merge_group' && (needs.changes.outputs.ccip_changes == 'true' || needs.changes.outputs.github_ci_changes == 'true')
    uses: ./.github/workflows/run-e2e-tests-reusable-workflow.yml
    with:
      workflow_name: Run CCIP E2E Tests For Merge Queue
      chainlink_version: ${{ inputs.evm-ref || github.sha }}
      chainlink_upgrade_version: ${{ github.sha }}
      test_workflow: Merge Queue E2E CCIP Tests
      upload_cl_node_coverage_artifact: true
      upload_cl_node_coverage_artifact_prefix: cl_node_coverage_data_
      enable_otel_traces_for_ocr2_plugins: ${{ contains(join(github.event.pull_request.labels.*.name, ' '), 'enable tracing') }}
    secrets:
      QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
      QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
      QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
      QA_PYROSCOPE_INSTANCE: ${{ secrets.QA_PYROSCOPE_INSTANCE }}
      QA_PYROSCOPE_KEY: ${{ secrets.QA_PYROSCOPE_KEY }}
      QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
      GRAFANA_INTERNAL_TENANT_ID: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
      GRAFANA_INTERNAL_BASIC_AUTH: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
      GRAFANA_INTERNAL_HOST: ${{ secrets.GRAFANA_INTERNAL_HOST }}
      GRAFANA_INTERNAL_URL_SHORTENER_TOKEN: ${{ secrets.GRAFANA_INTERNAL_URL_SHORTENER_TOKEN }}
      LOKI_TENANT_ID: ${{ secrets.LOKI_TENANT_ID }}
      LOKI_URL: ${{ secrets.LOKI_URL }}
      LOKI_BASIC_AUTH: ${{ secrets.LOKI_BASIC_AUTH }}
      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      AWS_REGION: ${{ secrets.QA_AWS_REGION }}
      AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN: ${{ secrets.AWS_OIDC_IAM_ROLE_VALIDATION_PROD_ARN }}
      AWS_API_GW_HOST_GRAFANA: ${{ secrets.AWS_API_GW_HOST_GRAFANA }}
      SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}

  check-e2e-test-results:
    if: always()
    name: ETH Smoke Tests
    runs-on: ubuntu-latest
    needs: [lint-integration-tests, run-core-e2e-tests-for-pr, run-ccip-e2e-tests-for-pr, run-core-e2e-tests-for-merge-queue, run-ccip-e2e-tests-for-merge-queue]
    steps:
      - name: Check Core test results
        id: check_core_results
        run: |
          results='${{ needs.run-core-e2e-tests-for-pr.outputs.test_results }}'
          echo "Core test results:"
          echo "$results" | jq .
          
          node_migration_tests_failed=$(echo $results | jq '[.[] | select(.id == "integration-tests/migration/upgrade_version_test.go:*" ) | select(.result != "success")] | length > 0')
          echo "node_migration_tests_failed=$node_migration_tests_failed" >> $GITHUB_OUTPUT

      - name: Check CCIP test results
        id: check_ccip_results
        run: |
          if [[ '${{ needs.run-ccip-e2e-tests-for-pr.result }}' != 'skipped' ]]; then
            results='${{ needs.run-ccip-e2e-tests-for-pr.outputs.test_results }}'
            echo "CCIP test results:"
            echo "$results" | jq .
          else
            echo "CCIP tests were skipped."
          fi

      - name: Send slack notification for failed migration tests
        if: steps.check_core_results.outputs.node_migration_tests_failed == 'true' && github.event_name != 'workflow_dispatch'
        uses: slackapi/slack-github-action@6c661ce58804a1a20f6dc5fbee7f0381b469e001 # v1.25.0
        env:
          SLACK_BOT_TOKEN: ${{ secrets.QA_SLACK_API_KEY }}
        with:
          channel-id: "#team-test-tooling-internal"
          slack-message: ":x: :mild-panic-intensifies: Node Migration Tests Failed: \n${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }}\n${{ format('Notifying <!subteam^{0}|{0}>', secrets.GUARDIAN_SLACK_NOTIFICATION_HANDLE) }}"

      - name: Fail the job if core tests in PR not successful
        if: always() && needs.run-core-e2e-tests-for-pr.result == 'failure'
        run: exit 1

      - name: Fail the job if core tests in merge queue not successful
        if: always() && needs.run-core-e2e-tests-for-merge-queue.result == 'failure'
        run: exit 1        

      - name: Fail the job if lint not successful
        if: always() && needs.lint-integration-tests.result == 'failure'
        run: exit 1

  cleanup:
    name: Clean up integration environment deployments
    if: always()
    needs: [run-core-e2e-tests-for-pr, run-ccip-e2e-tests-for-pr, run-core-e2e-tests-for-merge-queue, run-ccip-e2e-tests-for-merge-queue]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repo
        if: ${{ github.event_name == 'pull_request' }}
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: smartcontractkit/chainlink
          ref: ${{ inputs.cl_ref }}

      - name: 🧼 Clean up Environment
        if: ${{ github.event_name == 'pull_request' }}
        uses: ./.github/actions/delete-deployments
        with:
          environment: integration
          ref: ${{ github.head_ref }} # See https://github.com/github/docs/issues/15319#issuecomment-1476705663

      - name: Collect Metrics
        if: ${{ github.event_name == 'pull_request' }}
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
        with:
          id: ${{ env.COLLECTION_ID }}-env-cleanup
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Clean up integration environment deployments
        continue-on-error: true

  show-chainlink-node-coverage:
    name: Show Chainlink Node Go Coverage
    if: always()
    needs: [run-core-e2e-tests-for-pr, run-ccip-e2e-tests-for-pr, run-core-e2e-tests-for-merge-queue, run-ccip-e2e-tests-for-merge-queue]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: smartcontractkit/chainlink
          ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Download All Artifacts
        uses: actions/download-artifact@9c19ed7fe5d278cd354c7dfd5d3b88589c7e2395 # v4.1.6
        with:
          path: cl_node_coverage_data
          pattern: cl_node_coverage_data_*
          merge-multiple: true
      - name: Show Coverage
        run: go run ./integration-tests/scripts/show_coverage.go "${{ github.workspace }}/cl_node_coverage_data/*/merged"

  ## Solana Section
  get_solana_sha:
    # We don't directly merge dependabot PRs, so let's not waste the resources
    if: ${{ github.actor != 'dependabot[bot]' && inputs.run_solana != 'false' }}
    name: Get Solana Sha From Go Mod
    environment: Integration
    runs-on: ubuntu-latest
    outputs:
      sha: ${{ steps.getsha.outputs.sha }}
    steps:
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: smartcontractkit/chainlink
          ref: ${{ inputs.cl_ref || github.event.pull_request.head.sha || github.event.merge_group.head_sha }}
      - name: Setup Go
        uses: ./.github/actions/setup-go
        with:
          only-modules: "true"
      - name: Get the sha from go mod
        id: getshortsha
        run: |
          sol_ver=$(go list -m -json github.com/smartcontractkit/chainlink-solana  | jq -r .Version)
          if [ -z "${sol_ver}" ]; then
              echo "Error: could not get the solana version from the go.mod file, look above for error(s)"
              exit 1
          fi
          short_sha="${sol_ver##*-}"
          echo "short sha is: ${short_sha}"
          echo "short_sha=${short_sha}" >> "$GITHUB_OUTPUT"
      - name: Checkout solana
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: smartcontractkit/chainlink-solana
          ref: develop
          fetch-depth: 0
          path: solanapath
      - name: Get long sha
        id: getsha
        run: |
          cd solanapath
          full_sha=$(git rev-parse ${{steps.getshortsha.outputs.short_sha}}^{}) # additional suffix allows handling tagged versions as well
          if [ -z "${full_sha}" ]; then
              echo "Error: could not get the full sha from the short sha using git, look above for error(s)"
              exit 1
          fi
          echo "sha is: ${full_sha}"
          echo "sha=${full_sha}" >> "$GITHUB_OUTPUT"

  get_projectserum_version:
    name: Get ProjectSerum Version
    environment: integration
    runs-on: ubuntu-latest
    needs: [get_solana_sha]
    outputs:
      projectserum_version: ${{ steps.psversion.outputs.projectserum_version }}
    steps:
      - name: Checkout the solana repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: smartcontractkit/chainlink-solana
          ref: ${{ needs.get_solana_sha.outputs.sha }}
      - name: Get ProjectSerum Version
        id: psversion
        uses: smartcontractkit/chainlink-solana/.github/actions/projectserum_version@4b971869e26b79c7ce3fb7c98005cc2e3f350915 # stable action on Oct 12 2022

  solana-test-image-exists:
    environment: integration
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    name: Check If Solana Test Image Exists
    runs-on: ubuntu-latest
    needs: [get_solana_sha]
    outputs:
      exists: ${{ steps.check-image.outputs.exists }}
    steps:
      - name: Check if image exists
        id: check-image
        uses: smartcontractkit/chainlink-github-actions/docker/image-exists@0ce1e67b254a4f041e03cc6f0e3afc987b47c7bd # v2.3.30
        with:
          repository: chainlink-solana-tests
          tag: ${{ needs.get_solana_sha.outputs.sha }}
          AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}

  solana-build-contracts:
    environment: integration
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    name: Solana Build Artifacts
    runs-on: ubuntu22.04-8cores-32GB
    needs:
      [
        changes,
        get_projectserum_version,
        solana-test-image-exists,
        get_solana_sha,
      ]
    steps:
      - name: Collect Metrics
        if: needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch'
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
        with:
          id: ${{ env.COLLECTION_ID }}-solana-build-contracts
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Solana Build Artifacts
        continue-on-error: true
      - name: Checkout the solana repo
        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
        with:
          repository: smartcontractkit/chainlink-solana
          ref: ${{ needs.get_solana_sha.outputs.sha }}
      - name: Build contracts
        if: (needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false'
        uses: smartcontractkit/chainlink-solana/.github/actions/build_contract_artifacts@46b1311a5a83f33d08ffa8e1e0ab04f9ad51665d # node20 update on may 10, 2024
        with:
          ref: ${{ needs.get_solana_sha.outputs.sha }}
          image: backpackapp/build
          image-version: ${{ needs.get_projectserum_version.outputs.projectserum_version }}

  solana-build-test-image:
    environment: integration
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    name: Solana Build Test Image
    runs-on: ubuntu22.04-8cores-32GB
    needs:
      [
        solana-build-contracts,
        solana-test-image-exists,
        changes,
        get_solana_sha,
      ]
    env:
      CONTRACT_ARTIFACTS_PATH: contracts/target/deploy
    steps:
      - name: Collect Metrics
        if: (needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false'
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
        with:
          id: ${{ env.COLLECTION_ID }}-solana-build-test-image
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Solana Build Test Image
        continue-on-error: true
      - name: Checkout the repo
        if: (needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false'
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: smartcontractkit/chainlink-solana
          ref: ${{ needs.get_solana_sha.outputs.sha }}
      - name: Build Test Image
        if: (needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch') && needs.solana-test-image-exists.outputs.exists == 'false'
        uses: smartcontractkit/.github/actions/ctf-build-test-image@a5e4f4c8fbb8e15ab2ad131552eca6ac83c4f4b3 # ctf-build-test-image@0.1.0
        with:
          tag: ${{ needs.get_solana_sha.outputs.sha }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ACCOUNT_NUMBER: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
      - run: echo "this exists so we don't have to run anything else if the build is skipped"
        if: needs.changes.outputs.core_changes == 'false' || needs.solana-test-image-exists.outputs.exists == 'true'

  solana-smoke-tests:
    if: ${{ !contains(join(github.event.pull_request.labels.*.name, ' '), 'skip-smoke-tests') }}
    environment: integration
    permissions:
      checks: write
      pull-requests: write
      id-token: write
      contents: read
    name: Solana Smoke Tests
    runs-on: ubuntu22.04-8cores-32GB
    needs:
      [
        build-chainlink,
        solana-build-contracts,
        solana-build-test-image,
        changes,
        get_solana_sha,
      ]
    env:
      CHAINLINK_COMMIT_SHA: ${{ inputs.evm-ref || github.sha }}
      CHAINLINK_ENV_USER: ${{ github.actor }}
      TEST_LOG_LEVEL: debug
      CONTRACT_ARTIFACTS_PATH: contracts/target/deploy
    steps:
      - name: Collect Metrics
        if: needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch'
        id: collect-gha-metrics
        uses: smartcontractkit/push-gha-metrics-action@d9da21a2747016b3e13de58c7d4115a3d5c97935 # v3.0.1
        with:
          id: ${{ env.COLLECTION_ID }}-solana-e2e-tests
          org-id: ${{ secrets.GRAFANA_INTERNAL_TENANT_ID }}
          basic-auth: ${{ secrets.GRAFANA_INTERNAL_BASIC_AUTH }}
          hostname: ${{ secrets.GRAFANA_INTERNAL_HOST }}
          this-job-name: Solana Smoke Tests
          test-results-file: '{"testType":"go","filePath":"/tmp/gotest.log"}'
        continue-on-error: true
      - name: Checkout the repo
        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
        with:
          repository: smartcontractkit/chainlink-solana
          ref: ${{ needs.get_solana_sha.outputs.sha }}
      - name: Run Setup
        if: needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch'
        uses: smartcontractkit/.github/actions/ctf-setup-run-tests-environment@49cb1613e96c9ce17f7290e4dabd38f43aa9bd4d # ctf-setup-run-tests-environment@0.0.0
        with:
          go_mod_path: ./integration-tests/go.mod
          cache_restore_only: true
          cache_key_id: core-solana-e2e-${{ env.MOD_CACHE_VERSION }}
          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
          dockerhub_username: ${{ secrets.DOCKERHUB_READONLY_USERNAME }}
          dockerhub_password: ${{ secrets.DOCKERHUB_READONLY_PASSWORD }}
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_KUBECONFIG: ${{ secrets.QA_KUBECONFIG }}
      - name: Pull Artifacts
        if: needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch'
        run: |
          IMAGE_NAME=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-solana-tests:${{ needs.get_solana_sha.outputs.sha }}
          # Pull the Docker image
          docker pull "$IMAGE_NAME"

          # Create a container without starting it
          CONTAINER_ID=$(docker create "$IMAGE_NAME")

          # Copy the artifacts from the container
          mkdir -p ./${{env.CONTRACT_ARTIFACTS_PATH}}/
          docker cp "$CONTAINER_ID:/go/testdir/${{env.CONTRACT_ARTIFACTS_PATH}}/" "./${{env.CONTRACT_ARTIFACTS_PATH}}/../"

          # Remove the created container
          docker rm "$CONTAINER_ID"
      - name: Install Solana CLI # required for ensuring the local test validator is configured correctly
        run: ./scripts/install-solana-ci.sh

      - name: Install gauntlet
        run: |
          yarn --cwd ./gauntlet install --frozen-lockfile
          yarn --cwd ./gauntlet build
          yarn --cwd ./gauntlet gauntlet
      - name: Generate config overrides
        run: | # https://github.com/smartcontractkit/chainlink-testing-framework/lib/blob/main/config/README.md
          cat << EOF > config.toml
          [ChainlinkImage]
          version="${{ inputs.evm-ref || github.sha }}"
          [Common]
          user="${{ github.actor }}"
          internal_docker_repo = "${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com"
          EOF
          # shellcheck disable=SC2002
          BASE64_CONFIG_OVERRIDE=$(cat config.toml | base64 -w 0)
          # shellcheck disable=SC2086
          echo ::add-mask::$BASE64_CONFIG_OVERRIDE
          # shellcheck disable=SC2086
          echo "BASE64_CONFIG_OVERRIDE=$BASE64_CONFIG_OVERRIDE" >> $GITHUB_ENV
      - name: Run Tests
        if: needs.changes.outputs.core_changes == 'true' || github.event_name == 'workflow_dispatch'
        uses: smartcontractkit/.github/actions/ctf-run-tests@b8731364b119e88983e94b0c4da87fc27ddb41b8 # ctf-run-tests@0.0.0
        with:
          test_command_to_run: export ENV_JOB_IMAGE=${{ secrets.QA_AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ secrets.QA_AWS_REGION }}.amazonaws.com/chainlink-solana-tests:${{ needs.get_solana_sha.outputs.sha }} && make test_smoke
          test_config_override_base64: ${{ env.BASE64_CONFIG_OVERRIDE }}
          cl_repo: ${{ env.CHAINLINK_IMAGE }}
          cl_image_tag: ${{ inputs.evm-ref || github.sha }}
          publish_check_name: Solana Smoke Test Results
          go_mod_path: ./integration-tests/go.mod
          cache_key_id: core-solana-e2e-${{ env.MOD_CACHE_VERSION }}
          token: ${{ secrets.GITHUB_TOKEN }}
          aws_registries: ${{ secrets.QA_AWS_ACCOUNT_NUMBER }}
          artifacts_name: solana-test-artifacts
          artifacts_location: |
            ./integration-tests/smoke/logs
            ./integration-tests/smoke/db_dumps
            ./integration-tests/smoke/seth_artifacts
            /tmp/gotest.log
          QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }}
          QA_AWS_ROLE_TO_ASSUME: ${{ secrets.QA_AWS_ROLE_TO_ASSUME }}
          QA_KUBECONFIG: ""
          run_setup: false
          go_coverage_src_dir: /var/tmp/go-coverage
          go_coverage_dest_dir: ${{ github.workspace }}/.covdata
        env:
          E2E_TEST_CHAINLINK_IMAGE: ${{ env.CHAINLINK_IMAGE }}
          E2E_TEST_SOLANA_SECRET: thisisatestingonlysecret

      - name: Upload Coverage Data
        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
        timeout-minutes: 2
        continue-on-error: true
        with:
          name: cl_node_coverage_data_solana_tests
          path: .covdata
          retention-days: 1