Examples as Tests #447
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Examples as Tests | |
| on: | |
| schedule: | |
| # * is a special character in YAML so you have to quote this string | |
| # Format: minute hour day-of-month month day-of-week(starts on sunday) | |
| # Scheduled for 2 am, everyday | |
| - cron: '0 10 * * *' | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} | |
| cancel-in-progress: true | |
| jobs: | |
| generate_example_tests: | |
| name: Generate Example Tests | |
| runs-on: [self-hosted, linux, gpu, dataset-enabled] | |
| defaults: | |
| run: | |
| shell: micromamba run -n omnigibson /bin/bash -leo pipefail {0} | |
| steps: | |
| - name: Checkout source | |
| uses: actions/checkout@v2 | |
| with: | |
| submodules: true | |
| path: omnigibson-src | |
| # These are temporarily disabled due to some errors they cause where some | |
| # unknown important dependency has its version changed and Isaac Sim will | |
| # no longer launch properly. The result is that if dependencies change, | |
| # the tests will stop working. This is a temporary fix until we can figure | |
| # out how to fix this issue. | |
| # - name: Install dev requirements | |
| # working-directory: omnigibson-src | |
| # run: pip install -r requirements-dev.txt | |
| # - name: Install | |
| # working-directory: omnigibson-src | |
| # run: pip install -e . | |
| - name: Generate example tests | |
| working-directory: omnigibson-src | |
| run: python tests/create_tests_of_examples.py | |
| - name: Get list of generated tests | |
| id: get-test-list | |
| working-directory: omnigibson-src | |
| run: | | |
| echo "example_tests=$(cat tests/example_tests.json)" >> $GITHUB_OUTPUT | |
| outputs: | |
| example_tests: ${{ steps.get-test-list.outputs.example_tests }} | |
| run_test: | |
| name: Run Example Tests | |
| needs: [generate_example_tests] | |
| runs-on: [self-hosted, linux, gpu, dataset-enabled] | |
| strategy: | |
| matrix: | |
| test_file: | |
| - ${{ needs.generate_example_tests.outputs.example_tests != '' && fromJson(needs.generate_example_tests.outputs.example_tests) }} | |
| fail-fast: true | |
| defaults: | |
| run: | |
| shell: micromamba run -n omnigibson /bin/bash -leo pipefail {0} | |
| steps: | |
| - name: Check for generated tests | |
| if: ${{ needs.generate_example_tests.outputs.example_tests == '' }} | |
| run: | | |
| echo "No tests were generated. Failing the job." | |
| exit 1 | |
| - name: Fix home | |
| run: echo "HOME=/root" >> $GITHUB_ENV | |
| - name: Checkout source | |
| uses: actions/checkout@v2 | |
| with: | |
| submodules: true | |
| path: omnigibson-src | |
| - name: Install dev requirements | |
| working-directory: omnigibson-src | |
| run: pip install -r requirements-dev.txt | |
| - name: Install | |
| working-directory: omnigibson-src | |
| run: pip install -e . | |
| - name: Run tests | |
| working-directory: omnigibson-src | |
| run: pytest -s tests/tests_of_examples/${{ matrix.test_file }}.py --junitxml=${{ matrix.test_file }}.xml && cp ${{ matrix.test_file }}.xml ${GITHUB_WORKSPACE}/ | |
| - name: Deploy artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ github.run_id }}-tests-${{ matrix.test_file }} | |
| path: ${{ matrix.test_file }}.xml | |
| - name: Fail on failure or error | |
| run: grep -Eq "<failure|error" ${{ matrix.test_file }}.xml; if [ $? -eq 0 ]; then exit 1; else exit 0; fi | |
| upload_report: | |
| name: Compile Example Test Report | |
| runs-on: [self-hosted, linux] | |
| defaults: | |
| run: | |
| shell: micromamba run -n omnigibson /bin/bash -leo pipefail {0} | |
| needs: [run_test] | |
| steps: | |
| - name: Checkout source | |
| uses: actions/checkout@v2 | |
| with: | |
| submodules: true | |
| path: omnigibson-src | |
| - name: Pull reports | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: omnigibson-src | |
| - name: Example Test Report0 | |
| uses: dorny/test-reporter@v1 | |
| with: | |
| name: Example Test Results | |
| working-directory: omnigibson-src | |
| path: ${{ github.run_id }}-tests-*/*_test.xml | |
| reporter: java-junit | |
| fail-on-error: 'true' | |
| fail-on-empty: 'true' | |
| # - name: Upload coverage to Codecov | |
| # uses: codecov/codecov-action@v2.1.0 | |
| # with: | |
| # token: ${{ secrets.CODECOV_TOKEN }} |