Skip to content

Commit

Permalink
chore: ci reorganize dag (#2529)
Browse files Browse the repository at this point in the history
  • Loading branch information
@tychoish
tychoish authored Jan 30, 2024
1 parent 8cb7d94 commit cb4becc
Show file tree
Hide file tree
Showing 3 changed files with 91 additions and 220 deletions.
140 changes: 87 additions & 53 deletions .github/workflows/ci.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,23 +25,24 @@ jobs:
- uses: actions/cache@v4
name: toolchain cache
with:
path: |
~/.rustup/toolchains/
path: ~/.rustup/toolchains/
key: ${{ runner.os }}-toolchain-${{ hashFiles('**/rust-toolchain.toml') }}
- uses: actions/cache@v4
name: cargo cache
with:
path: |
~/.cargo/
path: ~/.cargo/
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- uses: actions/cache@v4
name: workspace cache
name: build cache
with:
path: |
target/
!target/**/glaredb
!target/**/pgprototest
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
key: ${{ runner.os }}-cargo-build-${{ github.ref_name }}
restore-keys: |
${{ runner.os }}-cargo-glaredb-build-${{ hashFiles('**/Cargo.lock') }}
${{ runner.os }}-cargo-glaredb-build-
- run: just build
- run: cargo build --bin pgprototest
- uses: actions/cache/save@v4
Expand All @@ -58,8 +59,8 @@ jobs:

fmt:
name: Format (rustfmt +nightly)
runs-on: ubuntu-latest
needs: ["build"]
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@v4
Expand All @@ -69,8 +70,7 @@ jobs:
- uses: actions/cache@v4
name: nightly toolchain cache
with:
path: |
~/.rustup/toolchains/
path: ~/.rustup/toolchains/
key: ${{ runner.os }}-toolchain-${{ hashFiles('**/rustfmt.toml') }}
- run: rustup install nightly
- run: rustup component add rustfmt --toolchain nightly
Expand All @@ -81,7 +81,7 @@ jobs:
name: Lint (clippy)
# emperically runtimes are the same for big/small hosts:
runs-on: ubuntu-latest
needs: ["fmt"]
needs: ["build"]
steps:
- name: checkout
uses: actions/checkout@v4
Expand All @@ -91,21 +91,27 @@ jobs:
- uses: actions/cache/restore@v4
name: toolchain cache
with:
path: |
~/.rustup/toolchains/
path: ~/.rustup/toolchains/
key: ${{ runner.os }}-toolchain-${{ hashFiles('**/rust-toolchain.toml') }}
- uses: actions/cache/restore@v4
name: cargo cache
with:
path: |
~/.cargo/
path: ~/.cargo/
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- uses: actions/cache/restore@v4
name: build cache
with:
path: |
target/
!target/**/glaredb
!target/**/pgprototest
key: ${{ runner.os }}-cargo-glaredb-${{ github.ref_name }}
- run: just clippy


unit-tests:
name: Unit Tests
runs-on: ubuntu-latest-8-cores
python-binding-tests:
name: Python Binding Tests
runs-on: ubuntu-latest-4-cores
needs: ["build"]
steps:
- name: checkout
Expand All @@ -116,30 +122,34 @@ jobs:
- uses: actions/cache/restore@v4
name: toolchain cache
with:
path: |
~/.rustup/toolchains/
path: ~/.rustup/toolchains/
key: ${{ runner.os }}-toolchain-${{ hashFiles('**/rust-toolchain.toml') }}
- uses: actions/cache/restore@v4
- uses: actions/cache@v4
name: cargo cache
with:
path: |
~/.cargo/
path: ~/.cargo/
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- uses: actions/cache/restore@v4
name: glaredb cache
- uses: actions/cache@v4
name: build cache
with:
path: target/debug/glaredb
key: ${{ github.run_id }}
- run: just unit-tests
path: |
target/
!target/**/glaredb
!target/**/pgprototest
key: ${{ runner.os }}-cargo-build-${{ github.ref_name }}
- run: just python build
- run: just python test


python-binding-tests:
name: Python Binding Tests
runs-on: ubuntu-latest-4-cores
nodejs-bindings-tests:
name: Node.js Binding Tests
runs-on: ubuntu-latest-8-cores
needs: ["build"]
steps:
- name: checkout
uses: actions/checkout@v4
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
- uses: extractions/setup-just@v1
with:
just-version: "1.23.0"
Expand All @@ -156,51 +166,76 @@ jobs:
~/.cargo/
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- uses: actions/cache/restore@v4
name: workspace cache
name: build cache
with:
path: |
target/
!target/**/glaredb
!target/**/pgprototest
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- run: just python build
- run: just python test
key: ${{ runner.os }}-cargo-build-${{ github.ref_name }}
- run: just js build-debug
- run: just js test


nodejs-bindings-tests:
name: Node.js Binding Tests
unit-tests:
name: Unit Tests
runs-on: ubuntu-latest-8-cores
needs: ["build"]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-node@v4
with:
node-version: 20
- name: checkout
uses: actions/checkout@v4
- uses: extractions/setup-just@v1
with:
just-version: "1.23.0"
- uses: actions/cache/restore@v4
name: toolchain cache
with:
path: |
~/.rustup/toolchains/
path: ~/.rustup/toolchains/
key: ${{ runner.os }}-toolchain-${{ hashFiles('**/rust-toolchain.toml') }}
- uses: actions/cache@v4
name: cargo cache
with:
path: |
~/.cargo/
path: ~/.cargo/
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- uses: actions/cache@v4
name: build cache
with:
path: |
target/
!target/**/glaredb
!target/**/pgprototest
key: ${{ runner.os }}-cargo-build-${{ github.ref_name }}
- uses: actions/cache/restore@v4
name: workspace cache
name: glaredb cache
with:
path: target/debug/glaredb
key: ${{ github.run_id }}
- run: just unit-tests


cache:
# update a cache that can be used between builds, but do it
# outside of the hot path.
name: cache helper
needs: ["build", "python-binding-tests", "unit-tests"]
runs-on: ubuntu-latest
steps:
- uses: actions/cache/restore@v4
name: build cache
with:
path: |
target/
!target/**/glaredb
!target/**/pgprototest
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- run: just js build-debug
- run: just js test
key: ${{ runner.os }}-cargo-build-${{ github.ref_name }}
- uses: actions/cache/save@v4
name: build cache
with:
path: |
target/
!target/**/glaredb
!target/**/pgprototest
key: ${{ runner.os }}-cargo-glaredb-build-${{ hashFiles('**/Cargo.lock') }}


pg-protocol:
Expand Down Expand Up @@ -289,9 +324,8 @@ jobs:
- uses: actions/cache@v4
name: py cache
with:
path: |
tests/.venv/
key: ${{ runner.os }}-poetry-${{ hashFiles('**/Cargo.lock') }}
path: tests/.venv/
key: ${{ runner.os }}-poetry-${{ hashFiles('tests/poetry.lock') }}
- run: just venv
- run: just pytest

Expand Down
5 changes: 3 additions & 2 deletions justfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,8 +128,9 @@ venv:


# Runs pytest in the tests directory.
pytest *args:
{{VENV_BIN}}/poetry -C tests run pytest --rootdir={{invocation_directory()}}/tests {{ if args == "" {'tests'} else {args} }}
pytest *args:
{{VENV_BIN}}/poetry -C tests lock --no-update
{{VENV_BIN}}/poetry -C tests run pytest --rootdir={{invocation_directory()}}/tests {{ if args == "" {'tests'} else {args} }}

# private helpers below
# ---------------------
Expand Down
Loading

0 comments on commit cb4becc

Please sign in to comment.