From 0820eb987ff9555b48a0e1704f5a8644ea4ab087 Mon Sep 17 00:00:00 2001 From: Raphael Taylor-Davies <1781103+tustvold@users.noreply.github.com> Date: Tue, 24 Jan 2023 19:01:34 +0000 Subject: [PATCH] Cleanup CI (#5040) (#5047) * Cleanup CI (#5040) * Update job name * Split out examples --- .github/actions/setup-builder/action.yaml | 9 ++ .github/workflows/cancel.yml | 46 ------- .github/workflows/dev.yml | 4 + .github/workflows/dev_pr.yml | 4 + .github/workflows/rust.yml | 148 +++++++++++----------- 5 files changed, 90 insertions(+), 121 deletions(-) delete mode 100644 .github/workflows/cancel.yml diff --git a/.github/actions/setup-builder/action.yaml b/.github/actions/setup-builder/action.yaml index 13a3008b74bc..41b91113540b 100644 --- a/.github/actions/setup-builder/action.yaml +++ b/.github/actions/setup-builder/action.yaml @@ -32,8 +32,17 @@ runs: apt-get install -y protobuf-compiler - name: Setup Rust toolchain shell: bash + # rustfmt is needed for the substrait build script run: | echo "Installing ${{ inputs.rust-version }}" rustup toolchain install ${{ inputs.rust-version }} rustup default ${{ inputs.rust-version }} rustup component add rustfmt + - name: Disable debuginfo generation + # Disable full debug symbol generation to speed up CI build and keep memory down + # "1" means line tables only, which is useful for panic tracebacks. + shell: bash + run: echo "RUSTFLAGS=-C debuginfo=1" >> $GITHUB_ENV + - name: Enable backtraces + shell: bash + run: echo "RUST_BACKTRACE=1" >> $GITHUB_ENV diff --git a/.github/workflows/cancel.yml b/.github/workflows/cancel.yml deleted file mode 100644 index e1c6ed98ae8b..000000000000 --- a/.github/workflows/cancel.yml +++ /dev/null @@ -1,46 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Cancel stale runs - -on: - workflow_run: - # The name of another workflow (whichever one) that always runs on PRs - workflows: ['Dev'] - types: ['requested'] - -jobs: - cancel-stale-workflow-runs: - name: "Cancel stale workflow runs" - runs-on: ubuntu-latest - steps: - # Unfortunately, we need to define a separate cancellation step for - # each workflow where we want to cancel stale runs. - - uses: potiuk/cancel-workflow-runs@master - name: "Cancel stale Dev runs" - with: - cancelMode: allDuplicates - token: ${{ secrets.GITHUB_TOKEN }} - workflowFileName: dev.yml - skipEventTypes: '["push", "schedule"]' - - uses: potiuk/cancel-workflow-runs@master - name: "Cancel stale Rust runs" - with: - cancelMode: allDuplicates - token: ${{ secrets.GITHUB_TOKEN }} - workflowFileName: rust.yml - skipEventTypes: '["push", "schedule"]' diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml index f8786adad68b..8854f53a40bc 100644 --- a/.github/workflows/dev.yml +++ b/.github/workflows/dev.yml @@ -18,6 +18,10 @@ name: Dev on: [push, pull_request] +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + jobs: rat: name: Release Audit Tool (RAT) diff --git a/.github/workflows/dev_pr.yml b/.github/workflows/dev_pr.yml index 4bd3344b61e1..7499239a9d12 100644 --- a/.github/workflows/dev_pr.yml +++ b/.github/workflows/dev_pr.yml @@ -17,6 +17,10 @@ name: Labeler +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + on: pull_request_target: types: diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e73e52cba1ea..c6e6fc244a04 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -17,6 +17,10 @@ name: Rust +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + on: # always trigger on PR push: @@ -32,10 +36,6 @@ jobs: runs-on: ubuntu-latest container: image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" steps: - uses: actions/checkout@v3 - name: Cache Cargo @@ -49,18 +49,17 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable - - name: Check workspace in debug mode - run: | - cargo check - - name: Check workspace in release mode - run: | - cargo check --release + - name: Check workspace without default features - run: | - cargo check --no-default-features -p datafusion + run: cargo check --no-default-features -p datafusion + + - name: Check workspace in debug mode + run: cargo check + + # Note: this does not include dictionary_expressions to reduce codegen - name: Check workspace with all features - run: | - cargo check --workspace --benches --features avro,jit,scheduler,json + run: cargo check --workspace --benches --features avro,jit,scheduler,json + - name: Check Cargo.lock for datafusion-cli run: | # If this test fails, try running `cargo update` in the `datafusion-cli` directory @@ -70,14 +69,10 @@ jobs: # test the crate linux-test: name: cargo test (amd64) - needs: [linux-build-lib] + needs: [ linux-build-lib ] runs-on: ubuntu-latest container: image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" steps: - uses: actions/checkout@v3 with: @@ -92,17 +87,27 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable - - name: Build tests - run: | - export PATH=$PATH:$HOME/d/protoc/bin - cargo test --features avro,jit,scheduler,json,dictionary_expressions --no-run - - name: Run tests - run: | - export PATH=$PATH:$HOME/d/protoc/bin - cargo test --features avro,jit,scheduler,json,dictionary_expressions + - name: Run tests (excluding doctests) + run: cargo test --lib --tests --bins --features avro,jit,scheduler,json,dictionary_expressions + - name: Verify Working Directory Clean + run: git diff --exit-code + + linux-test-example: + name: cargo examples (amd64) + needs: [ linux-build-lib ] + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable - name: Run examples run: | - export PATH=$PATH:$HOME/d/protoc/bin # test datafusion-sql examples cargo run --example sql # test datafusion-examples @@ -122,17 +127,34 @@ jobs: - name: Verify Working Directory Clean run: git diff --exit-code + # Run doc tests + linux-test-doc: + name: cargo doctest (amd64) + needs: [ linux-build-lib ] + runs-on: ubuntu-latest + container: + image: amd64/rust + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable + # Note: this does not include dictionary_expressions to reduce codegen + - name: Run doctests + run: cargo test --doc --features avro,jit,scheduler,json + - name: Verify Working Directory Clean + run: git diff --exit-code + # verify that the benchmark queries return the correct results verify-benchmark-results: name: verify benchmark results (amd64) - needs: [linux-build-lib] + needs: [ linux-build-lib ] runs-on: ubuntu-latest container: image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" steps: - uses: actions/checkout@v3 with: @@ -166,7 +188,7 @@ jobs: integration-test: name: "Compare to postgres" - needs: [linux-build-lib] + needs: [ linux-build-lib ] runs-on: ubuntu-latest services: postgres: @@ -230,7 +252,7 @@ jobs: sqllogictest-postgres: name: "Run sqllogictest with Postgres runner" - needs: [linux-build-lib] + needs: [ linux-build-lib ] runs-on: ubuntu-latest services: postgres: @@ -327,14 +349,10 @@ jobs: test-datafusion-pyarrow: name: cargo test pyarrow (amd64) - needs: [linux-build-lib] + needs: [ linux-build-lib ] runs-on: ubuntu-20.04 container: image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" steps: - uses: actions/checkout@v3 with: @@ -356,10 +374,9 @@ jobs: uses: ./.github/actions/setup-builder with: rust-version: stable - - name: Run tests - run: | - cd datafusion - cargo test --features=pyarrow + - name: Run datafusion-common tests + run: cargo test -p datafusion-common --features=pyarrow + check-fmt: name: Check cargo fmt @@ -368,11 +385,10 @@ jobs: image: amd64/rust steps: - uses: actions/checkout@v3 - - name: Setup toolchain - run: | - rustup toolchain install stable - rustup default stable - rustup component add rustfmt + - name: Setup Rust toolchain + uses: ./.github/actions/setup-builder + with: + rust-version: stable - name: Run run: | echo '' > datafusion/proto/src/generated/datafusion.rs @@ -422,14 +438,10 @@ jobs: clippy: name: clippy - needs: [linux-build-lib] + needs: [ linux-build-lib ] runs-on: ubuntu-latest container: image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" steps: - uses: actions/checkout@v3 with: @@ -445,22 +457,17 @@ jobs: with: rust-version: stable - name: Install Clippy - run: | - rustup component add clippy + run: rustup component add clippy - name: Run clippy run: ci/scripts/rust_clippy.sh # Check answers are correct when hash values collide hash-collisions: name: cargo test hash collisions (amd64) - needs: [linux-build-lib] + needs: [ linux-build-lib ] runs-on: ubuntu-latest container: image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" steps: - uses: actions/checkout@v3 with: @@ -478,19 +485,14 @@ jobs: - name: Run tests run: | cd datafusion - # Force all hash values to collide - cargo test --all --features=force_hash_collisions + cargo test --lib --tests --features=force_hash_collisions cargo-toml-formatting-checks: name: check Cargo.toml formatting - needs: [linux-build-lib] + needs: [ linux-build-lib ] runs-on: ubuntu-latest container: image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" steps: - uses: actions/checkout@v3 with: @@ -506,8 +508,8 @@ jobs: with: rust-version: stable - name: Install cargo-tomlfmt - run: | - which cargo-tomlfmt || cargo install cargo-tomlfmt + run: which cargo-tomlfmt || cargo install cargo-tomlfmt + - name: Check Cargo.toml formatting run: | # if you encounter error, try rerun the command below, finally run 'git diff' to @@ -519,14 +521,10 @@ jobs: config-docs-check: name: check configs.md is up-to-date - needs: [linux-build-lib] + needs: [ linux-build-lib ] runs-on: ubuntu-latest container: image: amd64/rust - env: - # Disable full debug symbol generation to speed up CI build and keep memory down - # "1" means line tables only, which is useful for panic tracebacks. - RUSTFLAGS: "-C debuginfo=1" steps: - uses: actions/checkout@v3 with: