feat: introduce new kafka topic consumer respecting WAL index #18811

Workflow file for this run

.github/workflows/develop.yml at c5291c4

	on:
	merge_group:
	pull_request:
	types: [ opened, synchronize, reopened, ready_for_review ]
	paths-ignore:
	- 'docs/**'
	- 'config/**'
	- '**.md'
	- '.dockerignore'
	- 'docker/**'
	- '.gitignore'
	- 'grafana/**'
	push:
	branches:
	- main
	paths-ignore:
	- 'docs/**'
	- 'config/**'
	- '**.md'
	- '.dockerignore'
	- 'docker/**'
	- '.gitignore'
	- 'grafana/**'
	workflow_dispatch:

	name: CI

	concurrency:
	group: ${{ github.workflow }}-${{ github.head_ref \|\| github.run_id }}
	cancel-in-progress: true

	env:
	RUST_TOOLCHAIN: nightly-2024-04-20

	jobs:
	check-typos-and-docs:
	name: Check typos and docs
	runs-on: ubuntu-20.04
	steps:
	- uses: actions/checkout@v4
	- uses: crate-ci/typos@master
	- name: Check the config docs
	run: \|
	make config-docs && \
	git diff --name-only --exit-code ./config/config.md \
	\|\| (echo "'config/config.md' is not up-to-date, please run 'make config-docs'." && exit 1)

	license-header-check:
	runs-on: ubuntu-20.04
	name: Check License Header
	steps:
	- uses: actions/checkout@v4
	- uses: korandoru/hawkeye@v5

	check:
	name: Check
	runs-on: ${{ matrix.os }}
	strategy:
	matrix:
	os: [ windows-2022, ubuntu-20.04 ]
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@v4
	- uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- uses: dtolnay/rust-toolchain@master
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN }}
	- name: Rust Cache
	uses: Swatinem/rust-cache@v2
	with:
	# Shares across multiple jobs
	# Shares with `Clippy` job
	shared-key: "check-lint"
	- name: Run cargo check
	run: cargo check --locked --workspace --all-targets

	toml:
	name: Toml Check
	runs-on: ubuntu-20.04
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@v4
	- uses: dtolnay/rust-toolchain@master
	with:
	toolchain: stable
	- name: Rust Cache
	uses: Swatinem/rust-cache@v2
	with:
	# Shares across multiple jobs
	shared-key: "check-toml"
	- name: Install taplo
	run: cargo +stable install taplo-cli --version ^0.9 --locked
	- name: Run taplo
	run: taplo format --check

	build:
	name: Build GreptimeDB binaries
	runs-on: ${{ matrix.os }}
	strategy:
	matrix:
	os: [ ubuntu-20.04 ]
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@v4
	- uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- uses: dtolnay/rust-toolchain@master
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN }}
	- uses: Swatinem/rust-cache@v2
	with:
	# Shares across multiple jobs
	shared-key: "build-binaries"
	- name: Install cargo-gc-bin
	shell: bash
	run: cargo install cargo-gc-bin
	- name: Build greptime binaries
	shell: bash
	# `cargo gc` will invoke `cargo build` with specified args
	run: cargo gc -- --bin greptime --bin sqlness-runner
	- name: Pack greptime binaries
	shell: bash
	run: \|
	mkdir bins && \
	mv ./target/debug/greptime bins && \
	mv ./target/debug/sqlness-runner bins
	- name: Print greptime binaries info
	run: ls -lh bins
	- name: Upload artifacts
	uses: ./.github/actions/upload-artifacts
	with:
	artifacts-dir: bins
	version: current

	fuzztest:
	name: Fuzz Test
	needs: build
	runs-on: ubuntu-latest
	timeout-minutes: 60
	strategy:
	matrix:
	target: [ "fuzz_create_table", "fuzz_alter_table", "fuzz_create_database", "fuzz_create_logical_table", "fuzz_alter_logical_table", "fuzz_insert", "fuzz_insert_logical_table" ]
	steps:
	- uses: actions/checkout@v4
	- uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- uses: dtolnay/rust-toolchain@master
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN }}
	- name: Rust Cache
	uses: Swatinem/rust-cache@v2
	with:
	# Shares across multiple jobs
	shared-key: "fuzz-test-targets"
	- name: Set Rust Fuzz
	shell: bash
	run: \|
	sudo apt-get install -y libfuzzer-14-dev
	rustup install nightly
	cargo +nightly install cargo-fuzz cargo-gc-bin
	- name: Download pre-built binaries
	uses: actions/download-artifact@v4
	with:
	name: bins
	path: .
	- name: Unzip binaries
	run: \|
	tar -xvf ./bins.tar.gz
	rm ./bins.tar.gz
	- name: Run GreptimeDB
	run: \|
	./bins/greptime standalone start&
	- name: Fuzz Test
	uses: ./.github/actions/fuzz-test
	env:
	CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
	GT_MYSQL_ADDR: 127.0.0.1:4002
	with:
	target: ${{ matrix.target }}
	max-total-time: 120

	unstable-fuzztest:
	name: Unstable Fuzz Test
	needs: build-greptime-ci
	runs-on: ubuntu-latest
	timeout-minutes: 60
	strategy:
	matrix:
	target: [ "unstable_fuzz_create_table_standalone" ]
	steps:
	- uses: actions/checkout@v4
	- uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- uses: dtolnay/rust-toolchain@master
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN }}
	- name: Rust Cache
	uses: Swatinem/rust-cache@v2
	with:
	# Shares across multiple jobs
	shared-key: "fuzz-test-targets"
	- name: Set Rust Fuzz
	shell: bash
	run: \|
	sudo apt update && sudo apt install -y libfuzzer-14-dev
	cargo install cargo-fuzz cargo-gc-bin
	- name: Download pre-built binariy
	uses: actions/download-artifact@v4
	with:
	name: bin
	path: .
	- name: Unzip bianry
	run: \|
	tar -xvf ./bin.tar.gz
	rm ./bin.tar.gz
	- name: Run Fuzz Test
	uses: ./.github/actions/fuzz-test
	env:
	CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
	GT_MYSQL_ADDR: 127.0.0.1:4002
	GT_FUZZ_BINARY_PATH: ./bin/greptime
	GT_FUZZ_INSTANCE_ROOT_DIR: /tmp/unstable-greptime/
	with:
	target: ${{ matrix.target }}
	max-total-time: 120
	unstable: 'true'
	- name: Upload unstable fuzz test logs
	if: failure()
	uses: actions/upload-artifact@v4
	with:
	name: unstable-fuzz-logs
	path: /tmp/unstable-greptime/
	retention-days: 3

	build-greptime-ci:
	name: Build GreptimeDB binary (profile-CI)
	runs-on: ${{ matrix.os }}
	strategy:
	matrix:
	os: [ ubuntu-20.04 ]
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@v4
	- uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- uses: dtolnay/rust-toolchain@master
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN }}
	- uses: Swatinem/rust-cache@v2
	with:
	# Shares across multiple jobs
	shared-key: "build-greptime-ci"
	- name: Install cargo-gc-bin
	shell: bash
	run: cargo install cargo-gc-bin
	- name: Build greptime bianry
	shell: bash
	# `cargo gc` will invoke `cargo build` with specified args
	run: cargo gc --profile ci -- --bin greptime
	- name: Pack greptime binary
	shell: bash
	run: \|
	mkdir bin && \
	mv ./target/ci/greptime bin
	- name: Print greptime binaries info
	run: ls -lh bin
	- name: Upload artifacts
	uses: ./.github/actions/upload-artifacts
	with:
	artifacts-dir: bin
	version: current

	distributed-fuzztest:
	name: Fuzz Test (Distributed, ${{ matrix.mode.name }}, ${{ matrix.target }})
	runs-on: ubuntu-latest
	needs: build-greptime-ci
	timeout-minutes: 60
	strategy:
	matrix:
	target: [ "fuzz_create_table", "fuzz_alter_table", "fuzz_create_database", "fuzz_create_logical_table", "fuzz_alter_logical_table", "fuzz_insert", "fuzz_insert_logical_table" ]
	mode:
	- name: "Disk"
	minio: false
	kafka: false
	values: "with-disk.yaml"
	- name: "Minio"
	minio: true
	kafka: false
	values: "with-minio.yaml"
	- name: "Minio with Cache"
	minio: true
	kafka: false
	values: "with-minio-and-cache.yaml"
	- name: "Remote WAL"
	minio: true
	kafka: true
	values: "with-remote-wal.yaml"
	steps:
	- uses: actions/checkout@v4
	- name: Setup Kind
	uses: ./.github/actions/setup-kind
	- if: matrix.mode.minio
	name: Setup Minio
	uses: ./.github/actions/setup-minio
	- if: matrix.mode.kafka
	name: Setup Kafka cluser
	uses: ./.github/actions/setup-kafka-cluster
	- name: Setup Etcd cluser
	uses: ./.github/actions/setup-etcd-cluster
	# Prepares for fuzz tests
	- uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- uses: dtolnay/rust-toolchain@master
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN }}
	- name: Rust Cache
	uses: Swatinem/rust-cache@v2
	with:
	# Shares across multiple jobs
	shared-key: "fuzz-test-targets"
	- name: Set Rust Fuzz
	shell: bash
	run: \|
	sudo apt-get install -y libfuzzer-14-dev
	rustup install nightly
	cargo +nightly install cargo-fuzz cargo-gc-bin
	# Downloads ci image
	- name: Download pre-built binariy
	uses: actions/download-artifact@v4
	with:
	name: bin
	path: .
	- name: Unzip binary
	run: \|
	tar -xvf ./bin.tar.gz
	rm ./bin.tar.gz
	- name: Build and push GreptimeDB image
	uses: ./.github/actions/build-and-push-ci-image
	- name: Wait for etcd
	run: \|
	kubectl wait \
	--for=condition=Ready \
	pod -l app.kubernetes.io/instance=etcd \
	--timeout=120s \
	-n etcd-cluster
	- if: matrix.mode.minio
	name: Wait for minio
	run: \|
	kubectl wait \
	--for=condition=Ready \
	pod -l app=minio \
	--timeout=120s \
	-n minio
	- if: matrix.mode.kafka
	name: Wait for kafka
	run: \|
	kubectl wait \
	--for=condition=Ready \
	pod -l app.kubernetes.io/instance=kafka \
	--timeout=120s \
	-n kafka-cluster
	- name: Print etcd info
	shell: bash
	run: kubectl get all --show-labels -n etcd-cluster
	# Setup cluster for test
	- name: Setup GreptimeDB cluster
	uses: ./.github/actions/setup-greptimedb-cluster
	with:
	image-registry: localhost:5001
	values-filename: ${{ matrix.mode.values }}
	- name: Port forward (mysql)
	run: \|
	kubectl port-forward service/my-greptimedb-frontend 4002:4002 -n my-greptimedb&
	- name: Fuzz Test
	uses: ./.github/actions/fuzz-test
	env:
	CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
	GT_MYSQL_ADDR: 127.0.0.1:4002
	with:
	target: ${{ matrix.target }}
	max-total-time: 120
	- name: Describe Nodes
	if: failure()
	shell: bash
	run: \|
	kubectl describe nodes
	- name: Export kind logs
	if: failure()
	shell: bash
	run: \|
	kind export logs /tmp/kind
	- name: Upload logs
	if: failure()
	uses: actions/upload-artifact@v4
	with:
	name: fuzz-tests-kind-logs-${{ matrix.mode.name }}-${{ matrix.target }}
	path: /tmp/kind
	retention-days: 3
	- name: Delete cluster
	if: success()
	shell: bash
	run: \|
	kind delete cluster
	docker stop $(docker ps -a -q)
	docker rm $(docker ps -a -q)
	docker system prune -f
	# FIXME(weny): wait for the chart to release a new version.
	# distributed-fuzztest-with-chaos:
	# name: Fuzz Test with Chaos (Distributed, ${{ matrix.mode.name }}, ${{ matrix.target }})
	# runs-on: ubuntu-latest
	# needs: build-greptime-ci
	# timeout-minutes: 60
	# strategy:
	# matrix:
	# target: ["fuzz_migrate_mito_regions", "fuzz_failover_mito_regions", "fuzz_failover_metric_regions"]
	# mode:
	# - name: "Remote WAL"
	# minio: true
	# kafka: true
	# values: "with-remote-wal.yaml"
	# steps:
	# - uses: actions/checkout@v4
	# - name: Setup Kind
	# uses: ./.github/actions/setup-kind
	# - name: Setup Chaos Mesh
	# uses: ./.github/actions/setup-chaos
	# - if: matrix.mode.minio
	# name: Setup Minio
	# uses: ./.github/actions/setup-minio
	# - if: matrix.mode.kafka
	# name: Setup Kafka cluser
	# uses: ./.github/actions/setup-kafka-cluster
	# - name: Setup Etcd cluser
	# uses: ./.github/actions/setup-etcd-cluster
	# # Prepares for fuzz tests
	# - uses: arduino/setup-protoc@v3
	# with:
	# repo-token: ${{ secrets.GITHUB_TOKEN }}
	# - uses: dtolnay/rust-toolchain@master
	# with:
	# toolchain: ${{ env.RUST_TOOLCHAIN }}
	# - name: Rust Cache
	# uses: Swatinem/rust-cache@v2
	# with:
	# # Shares across multiple jobs
	# shared-key: "fuzz-test-targets"
	# - name: Set Rust Fuzz
	# shell: bash
	# run: \|
	# sudo apt-get install -y libfuzzer-14-dev
	# rustup install nightly
	# cargo +nightly install cargo-fuzz cargo-gc-bin
	# # Downloads ci image
	# - name: Download pre-built binariy
	# uses: actions/download-artifact@v4
	# with:
	# name: bin
	# path: .
	# - name: Unzip binary
	# run: \|
	# tar -xvf ./bin.tar.gz
	# rm ./bin.tar.gz
	# - name: Build and push GreptimeDB image
	# uses: ./.github/actions/build-and-push-ci-image
	# - name: Wait for etcd
	# run: \|
	# kubectl wait \
	# --for=condition=Ready \
	# pod -l app.kubernetes.io/instance=etcd \
	# --timeout=120s \
	# -n etcd-cluster
	# - if: matrix.mode.minio
	# name: Wait for minio
	# run: \|
	# kubectl wait \
	# --for=condition=Ready \
	# pod -l app=minio \
	# --timeout=120s \
	# -n minio
	# - if: matrix.mode.kafka
	# name: Wait for kafka
	# run: \|
	# kubectl wait \
	# --for=condition=Ready \
	# pod -l app.kubernetes.io/instance=kafka \
	# --timeout=120s \
	# -n kafka-cluster
	# - name: Print etcd info
	# shell: bash
	# run: kubectl get all --show-labels -n etcd-cluster
	# # Setup cluster for test
	# - name: Setup GreptimeDB cluster
	# uses: ./.github/actions/setup-greptimedb-cluster
	# with:
	# image-registry: localhost:5001
	# values-filename: ${{ matrix.mode.values }}
	# enable-region-failover: true
	# - name: Port forward (mysql)
	# run: \|
	# kubectl port-forward service/my-greptimedb-frontend 4002:4002 -n my-greptimedb&
	# - name: Fuzz Test
	# uses: ./.github/actions/fuzz-test
	# env:
	# CUSTOM_LIBFUZZER_PATH: /usr/lib/llvm-14/lib/libFuzzer.a
	# GT_MYSQL_ADDR: 127.0.0.1:4002
	# with:
	# target: ${{ matrix.target }}
	# max-total-time: 120
	# - name: Describe Nodes
	# if: failure()
	# shell: bash
	# run: \|
	# kubectl describe nodes
	# - name: Export kind logs
	# if: failure()
	# shell: bash
	# run: \|
	# kind export logs /tmp/kind
	# - name: Upload logs
	# if: failure()
	# uses: actions/upload-artifact@v4
	# with:
	# name: fuzz-tests-kind-logs-${{ matrix.mode.name }}-${{ matrix.target }}
	# path: /tmp/kind
	# retention-days: 3
	# - name: Delete cluster
	# if: success()
	# shell: bash
	# run: \|
	# kind delete cluster
	# docker stop $(docker ps -a -q)
	# docker rm $(docker ps -a -q)
	# docker system prune -f

	sqlness:
	name: Sqlness Test (${{ matrix.mode.name }})
	needs: build
	runs-on: ${{ matrix.os }}
	strategy:
	matrix:
	os: [ ubuntu-20.04 ]
	mode:
	- name: "Basic"
	opts: ""
	kafka: false
	- name: "Remote WAL"
	opts: "-w kafka -k 127.0.0.1:9092"
	kafka: true
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@v4
	- name: Download pre-built binaries
	uses: actions/download-artifact@v4
	with:
	name: bins
	path: .
	- name: Unzip binaries
	run: tar -xvf ./bins.tar.gz
	- if: matrix.mode.kafka
	name: Setup kafka server
	working-directory: tests-integration/fixtures/kafka
	run: docker compose -f docker-compose-standalone.yml up -d --wait
	- name: Run sqlness
	run: RUST_BACKTRACE=1 ./bins/sqlness-runner ${{ matrix.mode.opts }} -c ./tests/cases --bins-dir ./bins --preserve-state
	- name: Upload sqlness logs
	if: failure()
	uses: actions/upload-artifact@v4
	with:
	name: sqlness-logs-${{ matrix.mode.name }}
	path: /tmp/sqlness*
	retention-days: 3

	fmt:
	name: Rustfmt
	runs-on: ubuntu-20.04
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@v4
	- uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- uses: dtolnay/rust-toolchain@master
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN }}
	components: rustfmt
	- name: Rust Cache
	uses: Swatinem/rust-cache@v2
	with:
	# Shares across multiple jobs
	shared-key: "check-rust-fmt"
	- name: Run cargo fmt
	run: cargo fmt --all -- --check

	clippy:
	name: Clippy
	runs-on: ubuntu-20.04
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@v4
	- uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- uses: dtolnay/rust-toolchain@master
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN }}
	components: clippy
	- name: Rust Cache
	uses: Swatinem/rust-cache@v2
	with:
	# Shares across multiple jobs
	# Shares with `Check` job
	shared-key: "check-lint"
	- name: Run cargo clippy
	run: make clippy

	coverage:
	if: github.event.pull_request.draft == false
	runs-on: ubuntu-20.04-8-cores
	timeout-minutes: 60
	steps:
	- uses: actions/checkout@v4
	- uses: arduino/setup-protoc@v3
	with:
	repo-token: ${{ secrets.GITHUB_TOKEN }}
	- uses: KyleMayes/install-llvm-action@v1
	with:
	version: "14.0"
	- name: Install toolchain
	uses: dtolnay/rust-toolchain@master
	with:
	toolchain: ${{ env.RUST_TOOLCHAIN }}
	components: llvm-tools-preview
	- name: Rust Cache
	uses: Swatinem/rust-cache@v2
	with:
	# Shares cross multiple jobs
	shared-key: "coverage-test"
	- name: Docker Cache
	uses: ScribeMD/docker-cache@0.3.7
	with:
	key: docker-${{ runner.os }}-coverage
	- name: Install latest nextest release
	uses: taiki-e/install-action@nextest
	- name: Install cargo-llvm-cov
	uses: taiki-e/install-action@cargo-llvm-cov
	- name: Install Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.10'
	- name: Install PyArrow Package
	run: pip install pyarrow
	- name: Setup etcd server
	working-directory: tests-integration/fixtures/etcd
	run: docker compose -f docker-compose-standalone.yml up -d --wait
	- name: Setup kafka server
	working-directory: tests-integration/fixtures/kafka
	run: docker compose -f docker-compose-standalone.yml up -d --wait
	- name: Setup minio
	working-directory: tests-integration/fixtures/minio
	run: docker compose -f docker-compose-standalone.yml up -d --wait
	- name: Run nextest cases
	run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F pyo3_backend -F dashboard
	env:
	CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=lld"
	RUST_BACKTRACE: 1
	CARGO_INCREMENTAL: 0
	GT_S3_BUCKET: ${{ vars.AWS_CI_TEST_BUCKET }}
	GT_S3_ACCESS_KEY_ID: ${{ secrets.AWS_CI_TEST_ACCESS_KEY_ID }}
	GT_S3_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }}
	GT_S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }}
	GT_MINIO_BUCKET: greptime
	GT_MINIO_ACCESS_KEY_ID: superpower_ci_user
	GT_MINIO_ACCESS_KEY: superpower_password
	GT_MINIO_REGION: us-west-2
	GT_MINIO_ENDPOINT_URL: http://127.0.0.1:9000
	GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
	GT_KAFKA_ENDPOINTS: 127.0.0.1:9092
	UNITTEST_LOG_DIR: "__unittest_logs"
	- name: Codecov upload
	uses: codecov/codecov-action@v4
	with:
	token: ${{ secrets.CODECOV_TOKEN }}
	files: ./lcov.info
	flags: rust
	fail_ci_if_error: false
	verbose: true

	# compat:
	# name: Compatibility Test
	# needs: build
	# runs-on: ubuntu-20.04
	# timeout-minutes: 60
	# steps:
	# - uses: actions/checkout@v4
	# - name: Download pre-built binaries
	# uses: actions/download-artifact@v4
	# with:
	# name: bins
	# path: .
	# - name: Unzip binaries
	# run: \|
	# mkdir -p ./bins/current
	# tar -xvf ./bins.tar.gz --strip-components=1 -C ./bins/current
	# - run: ./tests/compat/test-compat.sh 0.6.0

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

feat: introduce new kafka topic consumer respecting WAL index #18811

Workflow file

feat: introduce new kafka topic consumer respecting WAL index #18811

Jobs

Run details

Workflow file for this run