jet-tests.yml

.jet_common:
  stage: jet
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /Run tests/'
    - if: $JET_CUSTOM_FILTER != "" && $CI_PIPELINE_SOURCE != 'merge_request_event'
    - when: never

include:
  - project: dl/jet/gitlab-templates
    ref: main
    file: downstreams.yml

jet-setup:
  extends: [ .jet_common ]
  tags: 
    - os/linux
  script:
    - set -x
    - JET_FILTER=${JET_CUSTOM_FILTER:-False}
    - echo "_JET_FILTER=$JET_FILTER" | tee -a config.env
  artifacts:
    reports:
      dotenv: config.env

jet-configure:
  image: gitlab-master.nvidia.com:5005/adlr/megatron-lm/ci_yq:v1
  extends: [.jet_common, .jet-configure]
  tags:
    - os/linux
  script:
    - cd tests/functional_tests/jet_recipes
    - |
      if [[ $CI_PIPELINE_SOURCE == "merge_request_event" ]]; then
        yq e ".spec.source.ref = \"merge-requests/${CI_MERGE_REQUEST_IID}/head\"" -i build-pyt.yaml
      else
        yq e ".spec.source.ref = \"${CI_COMMIT_REF_NAME}\"" -i build-pyt.yaml
      fi
  artifacts:
    paths:
      - tests/functional_tests/jet_recipes

jet-trigger:
  stage: jet
  extends: [.jet_common, .jet-trigger]
  needs:  [ jet-configure, jet-setup ]
  trigger:
    project: dl/jet/ci
    branch: $JET_CLUSTER_BRANCH
    strategy: depend
  inherit:
    variables:
      - JET_CUSTOM_FILTER
      - JET_CLUSTER_BRANCH
  variables:
    JET_WORKLOADS_FILTER: "$_JET_FILTER"


jet-results-summary:
  stage: jet
  image: gitlab-master.nvidia.com:5005/dl/jet/api:latest
  tags:
    - os/linux
  needs: [ jet-trigger ]
  before_script:
    - jet secrets jwt-login jwt/nvidia/gitlab-master adlr-megatron-lm-ci $CI_JOB_JWT
  script: 
    - python -m pip install -U --no-cache-dir prettytable
    - rc=0
    - python tests/functional_tests/python_test_utils/jet_test_pipeline.py ${CI_PIPELINE_ID} --test exit --artifact_links $CI_JOB_ID || rc=$?
    - python tests/functional_tests/python_test_utils/jet_test_pipeline.py ${CI_PIPELINE_ID} --download_scripts_dir ./scripts || rc=$?
    - exit $rc
  rules:
    - if: '$CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /Run tests/'
      when: always
    - if: $JET_CUSTOM_FILTER != "" && $CI_PIPELINE_SOURCE != 'merge_request_event'
      when: always
    - when: never
  artifacts:
    when: always
    paths:
      - scripts