Skip to content
build

build #21331

Sign in to view logs

build

build #21331

Workflow file for this run

.github/workflows/ci.yml at ec0cb13
name: build
on:
push:
schedule:
- cron: '10 * * * *'
jobs:
build:
name: Build
runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
- name: Rust Build Cache
uses: Swatinem/rust-cache@v2
with:
key: cfg-v2
- name: Install system dependencies
run: |
set -e
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-clients_7.1.17-1_amd64.deb --output fdb-client.deb
sudo dpkg -i fdb-client.deb
sudo apt update && sudo apt install -y libfuse3-dev execstack
- name: Build (mvstore)
run: cargo build --locked --release -p mvstore
- name: Build (mvsqlite)
run: cargo build --locked --release -p mvsqlite
- name: Build (mvsqlite-fuse)
run: cargo build --locked --release -p mvsqlite-fuse
- name: Build (shared libraries)
run: make -C ./mvsqlite-preload build-preload && make -C ./mvsqlite-preload build-lib
- name: Build (patched libsqlite3)
run: |
set -e
curl -fL -o sqlite.zip https://www.sqlite.org/2023/sqlite-amalgamation-3410000.zip
unzip sqlite.zip
cp sqlite-amalgamation-3410000/sqlite3.c ./mvsqlite-preload/
cd mvsqlite-preload
patch -p1 ./sqlite3.c < ./sqlite-3410000.patch
make build-patched-sqlite3
- name: Build (mvstore-stress)
run: cargo build --locked --release -p mvstore-stress
- name: Collect binaries
run: |
set -e
mkdir build
cd build
cp ../target/release/mvstore ./
cp ../target/release/mvstore-stress ./
cp ../target/release/mvsqlite-fuse ./
cp ../mvsqlite-preload/libmvsqlite_preload.so ./
cp ../mvsqlite-preload/libmvsqlite.a ./
cp ../mvsqlite-preload/libsqlite3.so ./
find . -type f -exec sha256sum '{}' ';'
- name: Check binaries
run: |
set -e
execstack -q ./build/libmvsqlite_preload.so | tee execstack.log
grep -v '^X ' execstack.log
- name: Push binaries
uses: actions/upload-artifact@v3
with:
name: build
path: ./build
build-deb:
name: Build DEB
runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
profile: minimal
toolchain: stable
- name: Install system dependencies
run: |
set -e
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-clients_7.1.17-1_amd64.deb --output fdb-client.deb
sudo dpkg -i fdb-client.deb
cargo install cargo-deb
sudo apt update && sudo apt install -y libfuse3-dev # stop cargo from complaining, not really needed
- name: Build mvstore
run: cd mvstore && cargo deb && cd .. && mkdir deb && cp ./target/debian/*.deb ./deb/
- name: Push
uses: actions/upload-artifact@v3
with:
name: deb
path: ./deb
stress:
name: mvstore stress test
runs-on: ubuntu-20.04
needs:
- build
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install system dependencies
run: |
set -e
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-clients_7.1.17-1_amd64.deb --output fdb-client.deb
sudo dpkg -i fdb-client.deb
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-server_7.1.17-1_amd64.deb --output fdb-server.deb
sudo dpkg -i fdb-server.deb
- name: Fetch binaries
uses: actions/download-artifact@v3
with:
name: build
path: ./build
- name: Run it
run: |
set -e
chmod +x ./build/mvstore ./build/mvstore-stress
export RUST_LOG=info
./build/mvstore --data-plane 127.0.0.1:7000 --admin-api 127.0.0.1:7001 --metadata-prefix mvstore-test --raw-data-prefix m &
sleep 1
curl http://localhost:7001/api/create_namespace -d '{"key":"stress","metadata":""}'
RUST_LOG=error,mvstore_stress=info ./build/mvstore-stress --concurrency 50 --data-plane http://localhost:7000 --admin-api http://localhost:7001 --iterations 1000 --ns-key stress --pages 1000
stress-auto-create-namespace:
name: mvstore stress test
runs-on: ubuntu-20.04
needs:
- build
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install system dependencies
run: |
set -e
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-clients_7.1.17-1_amd64.deb --output fdb-client.deb
sudo dpkg -i fdb-client.deb
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-server_7.1.17-1_amd64.deb --output fdb-server.deb
sudo dpkg -i fdb-server.deb
- name: Fetch binaries
uses: actions/download-artifact@v3
with:
name: build
path: ./build
- name: Run it
run: |
set -e
chmod +x ./build/mvstore ./build/mvstore-stress
export RUST_LOG=info
./build/mvstore --data-plane 127.0.0.1:7000 --admin-api 127.0.0.1:7001 --metadata-prefix mvstore-test --raw-data-prefix m --auto-create-namespace &
sleep 1
RUST_LOG=error,mvstore_stress=info ./build/mvstore-stress --concurrency 50 --data-plane http://localhost:7000 --admin-api http://localhost:7001 --iterations 1000 --ns-key stress --pages 1000
stress-disable-read-set:
name: mvstore stress test (disable read set)
runs-on: ubuntu-20.04
needs:
- build
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install system dependencies
run: |
set -e
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-clients_7.1.17-1_amd64.deb --output fdb-client.deb
sudo dpkg -i fdb-client.deb
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-server_7.1.17-1_amd64.deb --output fdb-server.deb
sudo dpkg -i fdb-server.deb
- name: Fetch binaries
uses: actions/download-artifact@v3
with:
name: build
path: ./build
- name: Run it
run: |
set -e
chmod +x ./build/mvstore ./build/mvstore-stress
export RUST_LOG=info
./build/mvstore --data-plane 127.0.0.1:7000 --admin-api 127.0.0.1:7001 --metadata-prefix mvstore-test --raw-data-prefix m &
sleep 1
curl http://localhost:7001/api/create_namespace -d '{"key":"stress","metadata":""}'
RUST_LOG=error,mvstore_stress=info ./build/mvstore-stress --concurrency 50 --data-plane http://localhost:7000 --admin-api http://localhost:7001 --iterations 1000 --ns-key stress --pages 1000 --disable-read-set
stress-small-gc-batch:
name: mvstore stress test (small gc batch)
runs-on: ubuntu-20.04
needs:
- build
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install system dependencies
run: |
set -e
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-clients_7.1.17-1_amd64.deb --output fdb-client.deb
sudo dpkg -i fdb-client.deb
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-server_7.1.17-1_amd64.deb --output fdb-server.deb
sudo dpkg -i fdb-server.deb
- name: Fetch binaries
uses: actions/download-artifact@v3
with:
name: build
path: ./build
- name: Run it
run: |
set -e
chmod +x ./build/mvstore ./build/mvstore-stress
export RUST_LOG=info
./build/mvstore --data-plane 127.0.0.1:7000 --admin-api 127.0.0.1:7001 --metadata-prefix mvstore-test --raw-data-prefix m \
--knob-gc-scan-batch-size 50 &
sleep 1
curl http://localhost:7001/api/create_namespace -d '{"key":"stress","metadata":""}'
RUST_LOG=error,mvstore_stress=info ./build/mvstore-stress --concurrency 50 --data-plane http://localhost:7000 --admin-api http://localhost:7001 --iterations 1000 --ns-key stress --pages 1000
stress-gc-nofresh:
name: mvstore stress test (gc without fresh time check)
runs-on: ubuntu-20.04
needs:
- build
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install system dependencies
run: |
set -e
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-clients_7.1.17-1_amd64.deb --output fdb-client.deb
sudo dpkg -i fdb-client.deb
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-server_7.1.17-1_amd64.deb --output fdb-server.deb
sudo dpkg -i fdb-server.deb
- name: Fetch binaries
uses: actions/download-artifact@v3
with:
name: build
path: ./build
- name: Run it
run: |
set -e
chmod +x ./build/mvstore ./build/mvstore-stress
export RUST_LOG=info
./build/mvstore --data-plane 127.0.0.1:7000 --admin-api 127.0.0.1:7001 --metadata-prefix mvstore-test --raw-data-prefix m \
--knob-gc-fresh-page-ttl-secs 0 &
sleep 1
curl http://localhost:7001/api/create_namespace -d '{"key":"stress","metadata":""}'
RUST_LOG=error,mvstore_stress=info ./build/mvstore-stress --concurrency 50 --data-plane http://localhost:7000 --admin-api http://localhost:7001 --iterations 1000 --ns-key stress --pages 1000 \
--disable-ryw --permit-410
stress-commit-multi-phase:
name: mvstore stress test (gc without fresh time check + multi phase commit)
runs-on: ubuntu-20.04
needs:
- build
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install system dependencies
run: |
set -e
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-clients_7.1.17-1_amd64.deb --output fdb-client.deb
sudo dpkg -i fdb-client.deb
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-server_7.1.17-1_amd64.deb --output fdb-server.deb
sudo dpkg -i fdb-server.deb
- name: Fetch binaries
uses: actions/download-artifact@v3
with:
name: build
path: ./build
- name: Run it
run: |
set -e
chmod +x ./build/mvstore ./build/mvstore-stress
export RUST_LOG=info
./build/mvstore --data-plane 127.0.0.1:7000 --admin-api 127.0.0.1:7001 --metadata-prefix mvstore-test --raw-data-prefix m \
--knob-gc-fresh-page-ttl-secs 0 --knob-commit-multi-phase-threshold 0 --knob-gc-scan-batch-size 50 &
sleep 1
curl http://localhost:7001/api/create_namespace -d '{"key":"stress","metadata":""}'
RUST_LOG=error,mvstore_stress=info ./build/mvstore-stress --concurrency 50 --data-plane http://localhost:7000 --admin-api http://localhost:7001 --iterations 1000 --ns-key stress --pages 1000 \
--disable-ryw --permit-410
stress-buggify:
name: mvstore stress test (buggify)
runs-on: ubuntu-20.04
needs:
- build
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install system dependencies
run: |
set -e
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-clients_7.1.17-1_amd64.deb --output fdb-client.deb
sudo dpkg -i fdb-client.deb
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-server_7.1.17-1_amd64.deb --output fdb-server.deb
sudo dpkg -i fdb-server.deb
- name: Fetch binaries
uses: actions/download-artifact@v3
with:
name: build
path: ./build
- name: Run it
run: |
set -e
chmod +x ./build/mvstore ./build/mvstore-stress
export RUST_LOG=info
./build/mvstore --data-plane 127.0.0.1:7000 --admin-api 127.0.0.1:7001 --metadata-prefix mvstore-test --raw-data-prefix m &
sleep 1
curl http://localhost:7001/api/create_namespace -d '{"key":"stress","metadata":""}'
killall -9 mvstore
sleep 1
./build/mvstore --data-plane 127.0.0.1:7000 --admin-api 127.0.0.1:7001 --metadata-prefix mvstore-test --raw-data-prefix m --fdb-buggify &
sleep 1
RUST_LOG=error,mvstore_stress=info ./build/mvstore-stress --concurrency 50 --data-plane http://localhost:7000 --admin-api http://localhost:7001 --iterations 500 --ns-key stress --pages 1000
sqlite-speedtest1-default-page-size:
name: SQLite benchmark (speedtest1), default page size
runs-on: ubuntu-20.04
needs:
- build
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install system dependencies
run: |
set -e
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-clients_7.1.17-1_amd64.deb --output fdb-client.deb
sudo dpkg -i fdb-client.deb
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-server_7.1.17-1_amd64.deb --output fdb-server.deb
sudo dpkg -i fdb-server.deb
- name: Fetch binaries
uses: actions/download-artifact@v3
with:
name: build
path: ./build
- name: Run it
run: |
set -e
chmod +x ./build/mvstore
export RUST_LOG=info
./build/mvstore --data-plane 127.0.0.1:7000 --admin-api 127.0.0.1:7001 --metadata-prefix mvstore-test --raw-data-prefix m &
sleep 1
curl http://localhost:7001/api/create_namespace -d '{"key":"test","metadata":""}'
git clone https://github.com/sqlite/sqlite
cd sqlite
git checkout version-3.31.1 # ubuntu 20.04
git apply ../res/ci/sqlite.patch
gcc -O2 -o speedtest1 test/speedtest1.c -lsqlite3
./speedtest1 ./test.db
LD_PRELOAD=../build/libmvsqlite_preload.so MVSQLITE_DATA_PLANE=http://localhost:7000 RUST_LOG=error ./speedtest1 test
sqlite-speedtest1-16k-page:
name: SQLite benchmark (speedtest1), 16K pages
runs-on: ubuntu-20.04
needs:
- build
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install system dependencies
run: |
set -e
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-clients_7.1.17-1_amd64.deb --output fdb-client.deb
sudo dpkg -i fdb-client.deb
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-server_7.1.17-1_amd64.deb --output fdb-server.deb
sudo dpkg -i fdb-server.deb
- name: Fetch binaries
uses: actions/download-artifact@v3
with:
name: build
path: ./build
- name: Run it
run: |
set -e
chmod +x ./build/mvstore
export RUST_LOG=info
./build/mvstore --data-plane 127.0.0.1:7000 --admin-api 127.0.0.1:7001 --metadata-prefix mvstore-test --raw-data-prefix m &
sleep 1
curl http://localhost:7001/api/create_namespace -d '{"key":"test","metadata":""}'
git clone https://github.com/sqlite/sqlite
cd sqlite
git checkout version-3.31.1 # ubuntu 20.04
git apply ../res/ci/sqlite.patch
gcc -O2 -o speedtest1 test/speedtest1.c -lsqlite3
./speedtest1 ./test.db
LD_PRELOAD=../build/libmvsqlite_preload.so MVSQLITE_DATA_PLANE=http://localhost:7000 MVSQLITE_SECTOR_SIZE=16384 RUST_LOG=error ./speedtest1 test
sqlite-speedtest1-default-page-size-fuse:
name: SQLite benchmark (speedtest1), default page size, fuse
runs-on: ubuntu-20.04
needs:
- build
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install system dependencies
run: |
set -e
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-clients_7.1.17-1_amd64.deb --output fdb-client.deb
sudo dpkg -i fdb-client.deb
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-server_7.1.17-1_amd64.deb --output fdb-server.deb
sudo dpkg -i fdb-server.deb
sudo apt update && sudo apt install -y libfuse3-dev
- name: Fetch binaries
uses: actions/download-artifact@v3
with:
name: build
path: ./build
- name: Run it
run: |
set -e
chmod +x ./build/mvstore
export RUST_LOG=info
./build/mvstore --data-plane 127.0.0.1:7000 --admin-api 127.0.0.1:7001 --metadata-prefix mvstore-test --raw-data-prefix m &
sleep 1
curl http://localhost:7001/api/create_namespace -d '{"key":"test","metadata":""}'
git clone https://github.com/sqlite/sqlite
cd sqlite
git checkout version-3.31.1 # ubuntu 20.04
git apply ../res/ci/sqlite.patch
gcc -O2 -o speedtest1 test/speedtest1.c -lsqlite3
./speedtest1 ./test.db
mkdir fusemnt
chmod +x ../build/mvsqlite-fuse
RUST_LOG=error ../build/mvsqlite-fuse --data-plane http://localhost:7000 \
--mountpoint ./fusemnt \
--namespaces "db=test" &
sleep 1
set +e
./speedtest1 ./fusemnt/db 2>&1 | tee log
set -e
grep "SQL error: no such table: t1" log
echo "Got the expected error."
build-ycsb:
name: Build ycsb
runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Build
run: |
git clone https://github.com/losfair/go-ycsb
cd go-ycsb
git checkout e6bad4a1af10fbd4d921c5f578d7d57cf80b08ea
make
- name: Push binaries
uses: actions/upload-artifact@v3
with:
name: go-ycsb
path: ./go-ycsb/bin/go-ycsb
ycsb:
name: ycsb
runs-on: ubuntu-20.04
needs:
- build
- build-ycsb
strategy:
matrix:
workload: [workloadb]
content_cache_size: ["0", "10000"]
wire_zstd: ["", "--wire-zstd"]
include:
- workload: workloada
content_cache_size: "0"
wire_zstd: ""
- workload: workloadc
content_cache_size: "0"
wire_zstd: ""
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Install system dependencies
run: |
set -e
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-clients_7.1.17-1_amd64.deb --output fdb-client.deb
sudo dpkg -i fdb-client.deb
curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-server_7.1.17-1_amd64.deb --output fdb-server.deb
sudo dpkg -i fdb-server.deb
sudo apt update && sudo apt install -y libfuse3-3
- name: Fetch binaries
uses: actions/download-artifact@v2
with:
name: build
path: ./build
- name: Fetch go-ycsb
uses: actions/download-artifact@v2
with:
name: go-ycsb
path: ./go-ycsb
- name: Run it
run: |
set -e
chmod +x ./build/mvstore
chmod +x ./go-ycsb/go-ycsb
RUST_LOG=error ./build/mvstore --data-plane 127.0.0.1:7000 --admin-api 127.0.0.1:7001 --metadata-prefix mvstore-test --raw-data-prefix m \
--content-cache-size "${{ matrix.content_cache_size }}" \
${{ matrix.wire_zstd }} &
sleep 1
# Preload mode
curl -f http://localhost:7001/api/create_namespace -d '{"key":"ycsb","metadata":""}'
PRELOAD_PATH="$PWD/build/libmvsqlite_preload.so" \
MVSQLITE_PAGE_CACHE_SIZE=50000 \
MVSQLITE_DATA_PLANE=http://localhost:7000 \
RUST_LOG=off \
./res/ci/ycsb.sh ycsb "${{ matrix.workload }}"
release:
name: Release
needs:
- build
- stress
- stress-buggify
- stress-small-gc-batch
- stress-gc-nofresh
- stress-commit-multi-phase
- stress-disable-read-set
- build-deb
if: startsWith(github.ref, 'refs/tags/')
runs-on: ubuntu-20.04
steps:
- uses: actions/download-artifact@v3
with:
name: build
path: ./build
- uses: actions/download-artifact@v3
with:
name: deb
path: ./deb
- uses: softprops/action-gh-release@v1
with:
files: |
build/mvstore
build/mvsqlite-fuse
build/libmvsqlite_preload.so
build/libmvsqlite.a
build/libsqlite3.so
deb/*.deb
prerelease: true