From 1d8a97ce1a279d8163375e4da4562367a316db2c Mon Sep 17 00:00:00 2001 From: wugeshui <106943115+wugeshui@users.noreply.github.com> Date: Wed, 20 Dec 2023 15:38:47 +0800 Subject: [PATCH] run on sco (#745) * add run on sco --- .github/actions/checkout-code/action.yml | 45 ++++++++++++++ .github/workflows/data-cron.yml | 3 +- .github/workflows/runs-on-sco.yml | 79 ++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 1 deletion(-) create mode 100644 .github/actions/checkout-code/action.yml create mode 100644 .github/workflows/runs-on-sco.yml diff --git a/.github/actions/checkout-code/action.yml b/.github/actions/checkout-code/action.yml new file mode 100644 index 000000000..5eb40e06e --- /dev/null +++ b/.github/actions/checkout-code/action.yml @@ -0,0 +1,45 @@ +name: Checkout Code + +description: checkout code + +inputs: + machine: + description: If set to other value, the job need ssh + required: false + default: 'local' + working_path: + description: ci work home + required: false + default: '/mnt/cache/share/deeplinkci/github/${{ github.repository }}/${{ github.run_number }}' + +runs: + using: composite + steps: + - name : checkout + env: + MACHINE: ${{ inputs.machine }} + shell: bash + run: | + function checkout_code(){ + source ~/.bashrc + mkdir -p ${{ inputs.working_path }} && cd ${{ inputs.working_path }} && rm -rf source + if [ -n "${{ github.event.pull_request.head.repo.full_name }}" ] && [[ ! "${{ github.event.pull_request.head.repo.full_name }}" == "${{ github.repository }}" ]]; then + git clone ${{ github.event.pull_request.head.repo.clone_url }} source + cd source && git checkout ${{ github.event.pull_request.head.sha }} + git remote add mainrepo https://github.com/${{ github.repository }}.git + git fetch mainrepo && git merge --no-edit mainrepo/${{ github.base_ref }} + else + git clone ${{ github.event.repository.clone_url }} source && cd source + if [ "${{ github.event_name }}" == "pull_request" ]; then + git checkout ${{ github.event.pull_request.head.sha }} && git merge --no-edit ${{ github.base_ref }} + else + git checkout ${{ github.sha }} + fi + fi + git submodule update --init --recursive + } + if [ ${MACHINE} != "local" ];then + ssh ${MACHINE} " $(typeset -f checkout_code); checkout_code " + else + $(typeset -f checkout_code); checkout_code + fi diff --git a/.github/workflows/data-cron.yml b/.github/workflows/data-cron.yml index dd754f122..7f9339add 100644 --- a/.github/workflows/data-cron.yml +++ b/.github/workflows/data-cron.yml @@ -5,7 +5,7 @@ on: # branches: # - "wgs/*" schedule: - - cron: '01 02 * * *' + - cron: '01 22 * * *' concurrency: group: data-${{ github.head_ref || github.ref }} @@ -75,6 +75,7 @@ jobs: else echo "true" > to_gen_data fi + cp new_hash pre_hash """ export file_path="/home/autolink/rsync/sourcecode" rsync -a --delete ${CLUSTER_1988}:${DATA_DIR}/to_gen_data ${file_path}/ diff --git a/.github/workflows/runs-on-sco.yml b/.github/workflows/runs-on-sco.yml new file mode 100644 index 000000000..5a2f1cf21 --- /dev/null +++ b/.github/workflows/runs-on-sco.yml @@ -0,0 +1,79 @@ +name: runs on sco + +on: + workflow_dispatch: + push: + branches: + - main + pull_request: + paths-ignore: + - "**.md" + - ".github/ISSUE_TEMPLATE/**" + - ".git*" + - "CODE_OF_CONDUCT**" + +concurrency: + group: sco-${{ github.head_ref || github.ref }} + cancel-in-progress: true + +env: + DEEPLINK_PATH: '/mnt/cache/share/deeplinkci/github/${{ github.repository }}' + ENV_PATH: '/mnt/cache/share/platform/env' + BUILD_TEST1: build_test + GEN_DATA: diopi_test/python/cache + ALL_COVERAGE: ${{ (contains( github.ref, 'main') || startsWith(github.ref, 'refs/heads/v') || startsWith(github.ref, 'refs/heads/dev')) && 'ON' || 'OFF' }} + REQUIRE_COVERAGE: ${{ vars.REQUIRE_COVERAGE != '' && vars.REQUIRE_COVERAGE || '40' }} + +jobs: + checkout_code: + name: checkout code + runs-on: tps-sco-ci + steps: + - name: Checkout Code + uses: DeepLink-org/DIOPI.dev/.github/actions/checkout-code@wgs/run_on_sco + + Build-Nvidia: + name: Build-Nvidia + runs-on: tps-sco-ci + needs: [checkout_code] + steps: + - name: build + run: | + set -e + cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER} && rm -rf ${BUILD_TEST1} && cp -R source ${BUILD_TEST1} && cd ${BUILD_TEST1} + srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1} \ + && source ${ENV_PATH}/pt2.0_diopi_nodeps \ + && cd impl && bash scripts/build_impl.sh torch" || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/ && rm -rf ${BUILD_TEST1} && exit 1 ) + + Gen-Data: + name: Gen-Data + runs-on: tps-sco-ci + needs: [Build-Nvidia] + steps: + - name: gen-test-data + run: | + set -e + srun --job-name=${GITHUB_JOB} bash -c "cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1}/diopi_test/python \ + && source ${ENV_PATH}/pt2.0_diopi_nodeps \ + && python main.py --mode gen_data" \ + || ( cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1} && git clean -xdf ${GEN_DATA} && exit 1 ) + + Op-Test-Nvidia: + name: Op-Test-Nvidia + runs-on: tps-sco-ci + needs: Gen-Data + env: + run_test: "coverage run -p" + steps: + - name: NHWC-32bit-test + run: | + set -e + srun --job-name=${GITHUB_JOB} bash -c "export USE_COVERAGE=ON && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1} \ + && source ${ENV_PATH}/pt2.0_diopi_nodeps && cd diopi_test/python && python main.py --mode gen_case --fname batch_norm --nhwc && + python main.py --mode gen_case --fname index_select --four_bytes && python main.py --mode gen_case --fname arange --four_bytes && ${run_test} main.py --mode run_test" + - name: test + run: | + set -e + srun --job-name=${GITHUB_JOB} bash -c "export CI=true && export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1}/impl/lib \ + && source ${ENV_PATH}/pt2.0_diopi_nodeps && cd ${DEEPLINK_PATH}/${GITHUB_RUN_NUMBER}/${BUILD_TEST1} \ + && cd diopi_test/python && python main.py --mode gen_case && python main.py --mode run_test" \ No newline at end of file