Release #143
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Release | |
| on: | |
| push: | |
| branches: | |
| - "release/*" | |
| schedule: | |
| # beijing: 2 am. | |
| # utc: 6 pm. | |
| - cron: "0 18 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| placeholder: | |
| description: "update .github/workflows/release.yml to config your build" | |
| required: false | |
| concurrency: | |
| group: release-${{ github.ref }} | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} | |
| env: | |
| ONEFLOW_SRC: . | |
| jobs: | |
| generate-build-matrix: | |
| name: "Generate build matrix" | |
| runs-on: ubuntu-latest | |
| env: | |
| ONEFLOW_SRC: . | |
| outputs: | |
| matrix: ${{ steps.find-cache.outputs.matrix }} | |
| formatted_date: ${{ steps.date.outputs.formatted_date }} | |
| steps: | |
| - name: Checkout Oneflow-Inc/oneflow | |
| uses: actions/checkout@v2 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| repository: ${{github.event.pull_request.head.repo.full_name}} | |
| - uses: Oneflow-Inc/get-oneflow/cache-complete/matrix/build@add-nightly-date | |
| name: find cache | |
| id: find-cache | |
| timeout-minutes: 5 | |
| with: | |
| delete-cache: ${{ contains(github.event.pull_request.labels.*.name, 'need-clean-ccache') }} | |
| runner-labels: | | |
| self-hosted | |
| linux | |
| release | |
| oneflow-src: ${{ env.ONEFLOW_SRC }} | |
| entries: | | |
| cu118 | |
| cu117 | |
| cu116 | |
| cpu | |
| - name: Get current date | |
| id: date | |
| run: echo "formatted_date=$(date +'%Y%m%d')" >> $GITHUB_OUTPUT | |
| staging_release: | |
| env: | |
| MANYLINUX_CACHE_DIR: ~/manylinux-cache-dir/release/${{ matrix.entry }} | |
| WHEELHOUSE_DIR: manylinux_wheelhouse | |
| OSS_DIR: branch/${{ github.ref_name }}/${{ matrix.entry }}/${{ github.sha }} | |
| needs: [generate-build-matrix] | |
| name: Staging Release | |
| timeout-minutes: 180 | |
| runs-on: [self-hosted, linux, release] | |
| if: github.repository == 'Oneflow-Inc/oneflow' | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 5 | |
| matrix: ${{ fromJson(needs.generate-build-matrix.outputs.matrix) }} | |
| steps: | |
| - name: Fix permissions | |
| run: | | |
| docker run --rm -v $PWD:/p -w /p busybox rm -rf * | |
| - name: Install dependencies | |
| run: | | |
| python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple | |
| python3 -m pip install -U setuptools wheel --user | |
| python3 -m pip install oss2 --user | |
| - uses: actions/checkout@v2 | |
| - uses: Oneflow-Inc/get-oneflow@add-nightly-date | |
| name: Build ${{ matrix.entry }} | |
| if: ${{ matrix.entry !='cpu' }} | |
| with: | |
| cmake-init-cache: ${{ env.ONEFLOW_SRC }}/cmake/caches/ci/release/cuda.cmake | |
| build-script: ${{ env.ONEFLOW_SRC }}/ci/manylinux/build-gcc9.sh | |
| oneflow-src: ${{ env.ONEFLOW_SRC }} | |
| oneflow-build-env: manylinux | |
| wheelhouse-dir: ${{ env.WHEELHOUSE_DIR }} | |
| clear-wheelhouse-dir: true | |
| self-hosted: true | |
| compute-platform: ${{ matrix.entry }} | |
| manylinux-cache-dir: ${{ env.MANYLINUX_CACHE_DIR }} | |
| docker-run-use-system-http-proxy: false | |
| docker-run-use-lld: false | |
| retry-failed-build: true | |
| clean-ccache: true | |
| nightly: ${{ github.event_name == 'schedule' || github.ref == 'refs/heads/release/add_nightly_date_index'}} | |
| nightly-date: ${{ needs.generate-build-matrix.outputs.formatted_date }} | |
| use-nvidia-wheels: ${{ matrix.entry !='cu112' }} | |
| python-versions: | | |
| 3.7 | |
| 3.8 | |
| 3.9 | |
| 3.10 | |
| - uses: Oneflow-Inc/get-oneflow@add-nightly-date | |
| name: Build ${{ matrix.entry }} | |
| if: ${{ matrix.entry =='cpu' }} | |
| with: | |
| cmake-init-cache: ${{ env.ONEFLOW_SRC }}/cmake/caches/ci/release/cpu.cmake | |
| build-script: ${{ env.ONEFLOW_SRC }}/ci/manylinux/build.sh | |
| oneflow-src: ${{ env.ONEFLOW_SRC }} | |
| oneflow-build-env: manylinux | |
| wheelhouse-dir: ${{ env.WHEELHOUSE_DIR }} | |
| clear-wheelhouse-dir: true | |
| self-hosted: true | |
| compute-platform: ${{ matrix.entry }} | |
| manylinux-cache-dir: ${{ env.MANYLINUX_CACHE_DIR }} | |
| docker-run-use-system-http-proxy: false | |
| docker-run-use-lld: false | |
| retry-failed-build: true | |
| clean-ccache: false | |
| nightly: ${{ github.event_name == 'schedule' || github.ref == 'refs/heads/release/add_nightly_date_index'}} | |
| nightly-date: ${{ needs.generate-build-matrix.outputs.formatted_date }} | |
| python-versions: | | |
| 3.7 | |
| 3.8 | |
| 3.9 | |
| 3.10 | |
| - name: Upload wheel | |
| uses: ./.github/actions/upload_oss | |
| with: | |
| src_path: ${{ env.WHEELHOUSE_DIR }} | |
| oss_dst_path: oss://oneflow-staging/${{ env.OSS_DIR }} | |
| oss_access_key_id: ${{ secrets.OSS_ACCESS_KEY_ID }} | |
| oss_access_key_secret: ${{ secrets.OSS_ACCESS_KEY_SECRET }} | |
| - name: Update pip index | |
| env: | |
| OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }} | |
| OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }} | |
| run: | | |
| python3 -m pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple | |
| python3 -m pip install oss2 beautifulsoup4 --user | |
| python3 tools/create_pip_index.py --dir_key ${{ env.OSS_DIR }} -b oneflow-staging \ | |
| --index_key=branch/${{ github.ref_name }}/${{ matrix.entry }}/index.html \ | |
| --index_key=branch/${{ github.ref_name }}/date/${{ needs.generate-build-matrix.outputs.formatted_date }}/${{ matrix.entry }}/index.html \ | |
| --index_key=${{ env.OSS_DIR }}/index.html \ | |
| --index_key=commit/${{ github.sha }}/${{ matrix.entry }}/index.html | |
| - name: Update API docs | |
| if: github.ref == 'refs/heads/master' && matrix.entry == 'cpu' | |
| env: | |
| READTHEDOCS_TOKEN: ${{ secrets.READTHEDOCS_TOKEN }} | |
| run: | | |
| curl -X POST -d "branches=master" -d "token=${READTHEDOCS_TOKEN}" https://readthedocs.org/api/v2/webhook/oneflow/135376/ | |
| pack_src: | |
| name: Pack source code | |
| if: github.ref == 'refs/heads/master' && github.repository == 'Oneflow-Inc/oneflow' | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Pack OneFlow source code | |
| env: | |
| OSS_ACCESS_KEY_ID: ${{ secrets.OSS_ACCESS_KEY_ID }} | |
| OSS_ACCESS_KEY_SECRET: ${{ secrets.OSS_ACCESS_KEY_SECRET }} | |
| run: | | |
| git reset --hard | |
| git clean -f | |
| git archive --prefix oneflow/ --format zip HEAD > oneflow-src.zip | |
| curl http://gosspublic.alicdn.com/ossutil/1.6.19/ossutil64 -o $HOME/ossutil64 | |
| chmod 755 $HOME/ossutil64 | |
| $HOME/ossutil64 config -e oss-cn-beijing.aliyuncs.com -i ${OSS_ACCESS_KEY_ID} -k ${OSS_ACCESS_KEY_SECRET} -L EN -c $HOME/.ossutilconfig | |
| $HOME/ossutil64 cp --update oneflow-src.zip oss://oneflow-public/oneflow-src.zip |