AzureML Tests

.github/workflows/azureml-nightly.yml

@@ -0,0 +1,103 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# ---------------------------------------------------------
+
+name: azureml-nightly
+
+on:
+  #          ┌───────────── minute (0 - 59)
+  #          │ ┌───────────── hour (0 - 23)
+  #          │ │ ┌───────────── day of the month (1 - 31)
+  #          │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
+  #          │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)                                  
+  #          │ │ │ │ │
+  #          │ │ │ │ │
+  schedule: 
+    - cron: '0 0 * * *'  # basically running everyday at 12AM
+    # cron works with default branch (main) only: # https://gh.neting.ccmunity/t/on-schedule-per-branch/17525/2
+
+  push:
+    # because we can't schedule runs for non-main branches,
+    # to ensure we are running the build on the staging branch, we can add push policy for it
+    branches: [staging, pradjoshi/aml_tests]
+    # branches: [staging]
+
+  # enable manual trigger
+  workflow_dispatch:
+    input:
+      tags:
+        description: 'Tags to label this manual run (optional)'
+        default: 'Anything to describe this manual run'
+
+env:
+  CLUSTER_NAME: "nightly-cluster"
+  EXP_NAME: "nightly_tests"
+  RG: "pradjoshi-aml-rg"
+  WS: "pradjoshi-aml-ws"
+  TEST_LOGS_PATH: '"test_logs.log"'
+  PYTEST_EXIT_CODE: "pytest_exit_code.log"
+
+jobs:
+  nightly-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['"python=3.7"']
+        test-group: ['"group_cpu_001"', '"group_cpu_002"', '"group_cpu_003"', '"group_gpu_001"', '"group_gpu_002"', '"group_gpu_003"', '"group_gpu_004"', '"group_gpu_005"', '"group_gpu_006"', '"group_gpu_007"', '"group_gpu_008"', '"group_gpu_009"', '"group_gpu_010"', '"group_spark_001"']
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with: 
+          python-version: "3.8"
+      - name: Install azureml-core and azure-cli on a GitHub hosted server
+        run: pip install azureml-core azure-cli
+      - name: Log in to Azure
+        uses: azure/login@v1
+        with:
+          creds: ${{secrets.AML_TEST_CREDENTIALS}}
+      - name: Install wheel package
+        run: pip install wheel
+      - name: Create wheel from setup.py
+        run: python setup.py bdist_wheel
+      - name: Extract branch name
+        shell: bash
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: extract_branch
+      - name: Submit CPU tests to AzureML
+        if: contains(matrix.test-group, 'cpu')
+        run: >-
+            python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{env.CLUSTER_NAME}}
+            --subid ${{secrets.AML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }}
+            --rg ${{env.RG}} --wsname ${{env.WS}} --expname ${{env.EXP_NAME}}_${{matrix.test-group}}
+            --testlogs ${{env.TEST_LOGS_PATH}} --testkind "nightly"
+            --conda_pkg_python ${{matrix.python-version}} --testgroup ${{matrix.test-group}}
+      - name: Submit GPU tests to AML
+        if: contains(matrix.test-group, 'gpu')
+        run: >-
+            python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{env.CLUSTER_NAME}}
+            --subid ${{secrets.AML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }}
+            --rg ${{env.RG}} --wsname ${{env.WS}} --expname ${{env.EXP_NAME}}_${{matrix.test-group}}
+            --testlogs ${{env.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind "nightly"
+            --conda_pkg_python ${{matrix.python-version}} --testgroup ${{matrix.test-group}}
+      - name: Submit PySpark tests to AML
+        if: contains(matrix.test-group, 'spark')
+        run: >-
+            python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{env.CLUSTER_NAME}}
+            --subid ${{secrets.AML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }}
+            --rg ${{env.RG}} --wsname ${{env.WS}} --expname ${{env.EXP_NAME}}_${{matrix.test-group}}
+            --testlogs ${{env.TEST_LOGS_PATH}} --add_spark_dependencies --testkind "nightly"
+            --conda_pkg_python ${{matrix.python-version}} --testgroup ${{matrix.test-group}}
+      - name: Print test logs
+        run: cat ${{env.TEST_LOGS_PATH}}
+      - name: Get exit status
+        id: exit_status
+        run: echo ::set-output name=code::$(cat ${{env.PYTEST_EXIT_CODE}})
+      - name: Check Success/Failure
+        if: ${{ steps.exit_status.outputs.code != 0 }}
+        uses: actions/github-script@v3
+        with:
+          script: |
+              core.setFailed('All tests did not pass!')

.github/workflows/azureml-unit-tests.yml

@@ -0,0 +1,92 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+# ---------------------------------------------------------
+
+name: azureml-unit-tests
+
+on:
+  # pull_request:
+  #   branches: [ staging, main ]
+
+  # push:
+  #   branches: [pradjoshi/aml_tests]
+
+  # enable manual trigger
+  workflow_dispatch:
+    input:
+      tags:
+        description: 'Tags to label this manual run (optional)'
+        default: 'Anything to describe this manual run'
+
+env:
+  CLUSTER_NAME: "nightly-cluster"
+  EXP_NAME: "unit_tests"
+  RG: "pradjoshi-aml-rg"
+  WS: "pradjoshi-aml-ws"
+  TEST_LOGS_PATH: '"test_logs.log"'
+  PYTEST_EXIT_CODE: "pytest_exit_code.log"
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['"python=3.7"']
+        test-group: ['"group_cpu_001"', '"group_notebooks_cpu_001"', '"group_notebooks_cpu_002"', '"group_pyspark_001"', '"group_pyspark_002"', '"group_notebooks_pyspark_001"', '"group_notebooks_pyspark_002"', '"group_notebooks_pyspark_003"', '"group_gpu_001"', '"group_gpu_002"', '"group_gpu_003"', '"group_notebooks_gpu_001"', '"group_notebooks_gpu_002"', '"group_notebooks_gpu_003"']
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Setup python
+        uses: actions/setup-python@v2
+        with: 
+          python-version: "3.8"
+      - name: Install azureml-core and azure-cli on a GitHub hosted server
+        run: pip install azureml-core azure-cli
+      - name: Log in to Azure
+        uses: azure/login@v1
+        with:
+          creds: ${{secrets.AML_TEST_CREDENTIALS}}
+      - name: Install wheel package
+        run: pip install wheel
+      - name: Create wheel from setup.py
+        run: python setup.py bdist_wheel
+      - name: Extract branch name
+        shell: bash
+        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
+        id: extract_branch
+      - name: Submit CPU tests to AzureML
+        if: contains(matrix.test-group, 'cpu')
+        run: >-
+            python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{env.CLUSTER_NAME}}
+            --subid ${{secrets.AML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }}
+            --rg ${{env.RG}} --wsname ${{env.WS}} --expname ${{env.EXP_NAME}}_${{matrix.test-group}}
+            --testlogs ${{env.TEST_LOGS_PATH}} --testkind "unit"
+            --conda_pkg_python ${{matrix.python-version}} --testgroup ${{matrix.test-group}}
+      - name: Submit GPU tests to AzureML
+        if: contains(matrix.test-group, 'gpu')
+        run: >-
+            python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{env.CLUSTER_NAME}}
+            --subid ${{secrets.AML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }}
+            --rg ${{env.RG}} --wsname ${{env.WS}} --expname ${{env.EXP_NAME}}_${{matrix.test-group}}
+            --testlogs ${{env.TEST_LOGS_PATH}} --add_gpu_dependencies --testkind "unit"
+            --conda_pkg_python ${{matrix.python-version}} --testgroup ${{matrix.test-group}}
+      - name: Submit PySpark tests to AzureML
+        if: contains(matrix.test-group, 'spark')
+        run: >-
+            python tests/ci/azureml_tests/submit_groupwise_azureml_pytest.py --clustername ${{env.CLUSTER_NAME}}
+            --subid ${{secrets.AML_TEST_SUBID}} --reponame "recommenders" --branch ${{ steps.extract_branch.outputs.branch }}
+            --rg ${{env.RG}} --wsname ${{env.WS}} --expname ${{env.EXP_NAME}}_${{matrix.test-group}}
+            --testlogs ${{env.TEST_LOGS_PATH}} --add_spark_dependencies --testkind "unit"
+            --conda_pkg_python ${{matrix.python-version}} --testgroup ${{matrix.test-group}}
+      - name: Print test logs
+        run: cat ${{env.TEST_LOGS_PATH}}
+      - name: Get exit status
+        id: exit_status
+        run: echo ::set-output name=code::$(cat ${{env.PYTEST_EXIT_CODE}})
+      - name: Check Success/Failure
+        if: ${{ steps.exit_status.outputs.code != 0 }}
+        uses: actions/github-script@v3
+        with:
+          script: |
+              core.setFailed('All tests did not pass!')

tests/ci/azureml_tests/run_groupwise_pytest.py

@@ -0,0 +1,74 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+"""
+run_pytest.py is the script submitted to Azure ML that runs pytest.
+pytest runs all tests in the specified test folder unless parameters
+are set otherwise.
+"""
+
+import logging
+import os
+import sys
+from azureml.core import Run
+import pytest
+import json
+import argparse
+import glob
+from test_groups import nightly_test_groups, unit_test_groups
+
+if __name__ == "__main__":
+
+    logger = logging.getLogger("submit_groupwise_azureml_pytest.py")
+    logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
+    parser = argparse.ArgumentParser(description="Process inputs")
+    parser.add_argument(
+        "--testkind",
+        "-k",
+        action="store",
+        default="unit",
+        help="Test kind - nightly or unit",
+    )
+    parser.add_argument(
+        "--testgroup",
+        "-g",
+        action="store",
+        default="group_cpu_001",
+        help="Group name for the tests",
+    )
+    args = parser.parse_args()
+
+    if args.testkind == "nightly":
+        test_group = nightly_test_groups[args.testgroup]
+    else:
+        test_group = unit_test_groups[args.testgroup]
+
+    logger.info("Tests to be executed")
+    logger.info(str(test_group))
+
+    # Run.get_context() is needed to save context as pytest causes corruption
+    # of env vars
+    run = Run.get_context()
+
+    logger.info("Python version ")
+    logger.info(str(sys.version))
+    logger.info("Executing tests now...")
+
+    # execute pytest command
+    pytest_exit_code = pytest.main(test_group)
+
+    logger.info("Test execution completed!")
+
+    # log pytest exit code as a metric
+    # to be used to indicate success/failure in github workflow
+    run.log("pytest_exit_code", pytest_exit_code.value)
+
+    # #
+    # # Leveraged code from this  notebook:
+    # # https://msdata.visualstudio.com/Vienna/_search?action=contents&text=upload_folder&type=code&lp=code-Project&filters=ProjectFilters%7BVienna%7DRepositoryFilters%7BAzureMlCli%7D&pageSize=25&sortOptions=%5B%7B%22field%22%3A%22relevance%22%2C%22sortOrder%22%3A%22desc%22%7D%5D&result=DefaultCollection%2FVienna%2FAzureMlCli%2FGBmaster%2F%2Fsrc%2Fazureml-core%2Fazureml%2Fcore%2Frun.py
+    # logger.info("os.listdir files {}".format(os.listdir(".")))
+
+    # upload pytest stdout file
+    logs_path = glob.glob('**/70_driver_log.txt', recursive=True)[0]
+    run.upload_file(name='test_logs', path_or_stream=logs_path)