Feature:Online Query and Dynamic Modification of Table-level RocksDB Options #1633

Workflow file for this run

.github/workflows/lint_and_test_cpp.yaml at 4a17c4d

	# Licensed to the Apache Software Foundation (ASF) under one
	# or more contributor license agreements. See the NOTICE file
	# distributed with this work for additional information
	# regarding copyright ownership. The ASF licenses this file
	# to you under the Apache License, Version 2.0 (the
	# "License"); you may not use this file except in compliance
	# with the License. You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing,
	# software distributed under the License is distributed on an
	# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
	# KIND, either express or implied. See the License for the
	# specific language governing permissions and limitations
	# under the License.
	---
	# Developer Notes:
	#
	# This config is for github actions. Before merging your changes of this file,
	# it's recommended to create a PR against the ci-test branch to test if it works
	# as expected.

	name: Cpp CI

	on:
	# run on each pull request
	pull_request:
	types: [ synchronize, opened, reopened ]
	branches:
	- master
	- 'v[0-9]+.*' # release branch
	- ci-test # testing branch for github action
	- '*dev'
	paths:
	- '.github/workflows/lint_and_test_cpp.yaml'
	- '.github/workflows/thirdparty-regular-push.yml'
	- 'docker/thirdparties-src/**'
	- 'docker/thirdparties-bin/**'
	- 'CMakeLists.txt'
	- 'run.sh'
	- 'cmake_modules/**'
	- 'src/**'
	- 'thirdparty/**'

	# for manually triggering workflow
	workflow_dispatch:

	env:
	# Update the options to reduce the consumption of the disk space
	ONEBOX_OPTS: disk_min_available_space_ratio=5
	TEST_OPTS: throttle_test_medium_value_kb=10,throttle_test_large_value_kb=25

	jobs:
	cpp_clang_format_linter:
	name: Lint
	runs-on: ubuntu-latest
	container:
	image: apache/pegasus:clang-format-3.9
	steps:
	- uses: actions/checkout@v3
	- name: clang-format
	run: ./scripts/run-clang-format.py --clang-format-executable clang-format-3.9 -e ./src/shell/linenoise -e ./src/shell/sds -e ./thirdparty -r .

	iwyu:
	name: IWYU
	needs: cpp_clang_format_linter
	runs-on: ubuntu-latest
	container:
	image: apache/pegasus:thirdparties-bin-ubuntu2204-${{ github.base_ref }}
	steps:
	- uses: actions/checkout@v3
	- uses: dorny/paths-filter@v2
	id: changes
	with:
	filters: \|
	thirdparty:
	- '.github/workflows/thirdparty-regular-push.yml'
	- 'docker/thirdparties-src/**'
	- 'docker/thirdparties-bin/**'
	- 'thirdparty/**'
	- name: Unpack prebuilt third-parties
	if: steps.changes.outputs.thirdparty == 'false'
	run: unzip /root/thirdparties-bin.zip -d ./thirdparty
	- name: Rebuild third-parties
	if: steps.changes.outputs.thirdparty == 'true'
	working-directory: thirdparty
	# Build thirdparties and leave some necessary libraries and source
	run: \|
	mkdir build
	cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
	cmake --build build/ -j $(nproc)
	rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
	../scripts/download_hadoop.sh hadoop-bin
	../scripts/download_zk.sh zookeeper-bin
	- name: Build IWYU binary
	run: \|
	mkdir iwyu && cd iwyu && git clone https://github.com/include-what-you-use/include-what-you-use.git
	cd include-what-you-use && git checkout clang_14
	cd .. && mkdir build && cd build && cmake -G "Unix Makefiles" -DCMAKE_PREFIX_PATH=/usr/lib/llvm-14 ../include-what-you-use
	make -j $(nproc)
	- name: Check IWYU
	run: \|
	PWD=`pwd`
	./run.sh build --test --compiler clang-14,clang++-14 -j $(nproc) -t release --iwyu ${PWD}/iwyu/build/bin/include-what-you-use --skip_thirdparty
	python3 iwyu/include-what-you-use/fix_includes.py --blank_lines --reorder --nocomments --nosafe_headers --separate_project_includes="<tld>" --ignore_re="thrift-gen\|rrdb_types\|layer2_types\|metrics_test.cpp" --dry_run < build/latest/iwyu.out
	python3 iwyu/include-what-you-use/fix_includes.py --blank_lines --reorder --nocomments --nosafe_headers --separate_project_includes="<tld>" --ignore_re="thrift-gen\|rrdb_types\|layer2_types\|metrics_test.cpp" --dry_run < build/latest/iwyu.out \| egrep "IWYU edited 0 files on your behalf"

	build_Release:
	name: Build Release
	needs: cpp_clang_format_linter
	runs-on: ubuntu-latest
	container:
	image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
	steps:
	- uses: actions/checkout@v3
	- name: Setup cache
	uses: actions/cache@v3
	with:
	path: \|
	/github/home/.ccache
	key: release_ccache
	- uses: dorny/paths-filter@v2
	id: changes
	with:
	filters: \|
	thirdparty:
	- '.github/workflows/thirdparty-regular-push.yml'
	- 'docker/thirdparties-src/**'
	- 'docker/thirdparties-bin/**'
	- 'thirdparty/**'
	- name: Unpack prebuilt third-parties
	if: steps.changes.outputs.thirdparty == 'false'
	run: \|
	unzip /root/thirdparties-bin.zip -d ./thirdparty
	rm -f /root/thirdparties-bin.zip
	- name: Rebuild third-parties
	if: steps.changes.outputs.thirdparty == 'true'
	working-directory: thirdparty
	# Build thirdparties and leave some necessary libraries and source
	run: \|
	mkdir build
	cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
	cmake --build build/ -j $(nproc)
	rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
	../scripts/download_hadoop.sh hadoop-bin
	../scripts/download_zk.sh zookeeper-bin
	- name: Compilation
	run: \|
	ccache -p
	ccache -z
	./run.sh build --test --skip_thirdparty -j $(nproc) -t release
	ccache -s
	- name: Pack Server
	run: \|
	./run.sh pack_server
	rm -rf pegasus-server-*
	- name: Pack Tools
	run: \|
	./run.sh pack_tools
	rm -rf pegasus-tools-*
	- name: Tar files
	run: \|
	mv thirdparty/hadoop-bin ./
	mv thirdparty/zookeeper-bin ./
	rm -rf thirdparty
	find ./build/latest/src/ -name 'CMakeFiles' -type d -exec rm -rf "{}" +
	tar -zcvhf release__builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin
	- name: Upload Artifact
	uses: actions/upload-artifact@v3
	with:
	name: release_artifact_${{ github.sha }}
	path: release__builder.tar

	test_Release:
	name: Test Release
	strategy:
	fail-fast: false
	matrix:
	test_module:
	- backup_restore_test
	- base_api_test
	- base_test
	- bulk_load_test
	- detect_hotspot_test
	- dsn_aio_test
	- dsn_block_service_test
	- dsn_client_test
	- dsn.failure_detector.tests
	- dsn_http_test
	- dsn_meta_state_tests
	- dsn.meta.test
	- dsn_nfs_test
	- dsn_perf_counter_test
	- dsn_replica_backup_test
	- dsn_replica_bulk_load_test
	- dsn_replica_dup_test
	- dsn_replica_split_test
	- dsn.replica.test
	- dsn_replication_common_test
	- dsn.replication.simple_kv
	- dsn.rep_tests.simple_kv
	- dsn_runtime_tests
	- dsn_utils_tests
	- dsn.zookeeper.tests
	# TODO(yingchun): Disable it because we find it's too flaky, we will re-enable it after
	# it has been optimized.
	# - partition_split_test
	- pegasus_geo_test
	- pegasus_rproxy_test
	- pegasus_unit_test
	- recovery_test
	- restore_test
	- throttle_test
	needs: build_Release
	runs-on: ubuntu-latest
	container:
	image: apache/pegasus:thirdparties-bin-test-ubuntu2204-${{ github.base_ref }}
	options: --cap-add=SYS_PTRACE
	steps:
	- uses: actions/checkout@v3
	- name: Unpack prebuilt third-parties
	run: \|
	unzip /root/thirdparties-bin.zip -d ./thirdparty
	rm -f /root/thirdparties-bin.zip
	- name: Download Artifact
	uses: actions/download-artifact@v3
	with:
	name: release_artifact_${{ github.sha }}
	path: .
	- name: Tar files
	run: \|
	tar -zxvf release__builder.tar
	- name: Unit Testing
	run: \|
	export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
	ulimit -s unlimited
	. ./scripts/config_hdfs.sh
	./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }}

	build_ASAN:
	name: Build ASAN
	needs: cpp_clang_format_linter
	runs-on: ubuntu-latest
	container:
	image: apache/pegasus:thirdparties-bin-test-ubuntu1804-${{ github.base_ref }}
	steps:
	- uses: actions/checkout@v3
	- name: Setup cache
	uses: actions/cache@v3
	with:
	path: \|
	/github/home/.ccache
	key: asan_ccache
	- uses: dorny/paths-filter@v2
	id: changes
	with:
	filters: \|
	thirdparty:
	- '.github/workflows/thirdparty-regular-push.yml'
	- 'docker/thirdparties-src/**'
	- 'docker/thirdparties-bin/**'
	- 'thirdparty/**'
	- name: Unpack prebuilt third-parties
	if: steps.changes.outputs.thirdparty == 'false'
	run: \|
	unzip /root/thirdparties-bin.zip -d ./thirdparty
	rm -f /root/thirdparties-bin.zip
	- name: Rebuild third-parties
	if: steps.changes.outputs.thirdparty == 'true'
	working-directory: thirdparty
	# Build thirdparties and leave some necessary libraries and source
	run: \|
	mkdir build
	cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
	cmake --build build/ -j $(nproc)
	rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
	../scripts/download_hadoop.sh hadoop-bin
	../scripts/download_zk.sh zookeeper-bin
	- name: Compilation
	run: \|
	ccache -p
	ccache -z
	./run.sh build --test --sanitizer address --skip_thirdparty --disable_gperf -j $(nproc)
	ccache -s
	- name: Tar files
	run: \|
	mv thirdparty/hadoop-bin ./
	mv thirdparty/zookeeper-bin ./
	rm -rf thirdparty
	find ./build/latest/src/ -name 'CMakeFiles' -type d -exec rm -rf "{}" +
	tar -zcvhf release_address_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin
	- name: Upload Artifact
	uses: actions/upload-artifact@v3
	with:
	name: release_address_artifact_${{ github.sha }}
	path: release_address_builder.tar

	test_ASAN:
	name: Test ASAN
	strategy:
	fail-fast: false
	matrix:
	test_module:
	- backup_restore_test
	- base_api_test
	- base_test
	- bulk_load_test
	- detect_hotspot_test
	- dsn_aio_test
	- dsn_block_service_test
	- dsn_client_test
	- dsn.failure_detector.tests
	- dsn_http_test
	- dsn_meta_state_tests
	- dsn.meta.test
	- dsn_nfs_test
	- dsn_perf_counter_test
	- dsn_replica_backup_test
	- dsn_replica_bulk_load_test
	- dsn_replica_dup_test
	- dsn_replica_split_test
	- dsn.replica.test
	- dsn_replication_common_test
	- dsn.replication.simple_kv
	- dsn.rep_tests.simple_kv
	- dsn_runtime_tests
	- dsn_utils_tests
	- dsn.zookeeper.tests
	# TODO(yingchun): Disable it because we find it's too flaky, we will re-enable it after
	# it has been optimized.
	# - partition_split_test
	- pegasus_geo_test
	- pegasus_rproxy_test
	- pegasus_unit_test
	- recovery_test
	- restore_test
	# TODO(yingchun): Disable it because we find it's too flaky, we will re-enable it after
	# it has been optimized.
	# - throttle_test
	needs: build_ASAN
	runs-on: ubuntu-latest
	container:
	image: apache/pegasus:thirdparties-bin-test-ubuntu1804-${{ github.base_ref }}
	options: --cap-add=SYS_PTRACE
	steps:
	- uses: actions/checkout@v3
	- name: Unpack prebuilt third-parties
	run: \|
	unzip /root/thirdparties-bin.zip -d ./thirdparty
	rm -f /root/thirdparties-bin.zip
	- name: Download Artifact
	uses: actions/download-artifact@v3
	with:
	name: release_address_artifact_${{ github.sha }}
	path: .
	- name: Tar files
	run: \|
	tar -zxvf release_address_builder.tar
	- name: Unit Testing
	run: \|
	export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
	ulimit -s unlimited
	. ./scripts/config_hdfs.sh
	./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }}

	# TODO(yingchun): Build and test UBSAN version would cost a very long time, we will run these tests
	# when we are going to release a stable version. So we disable them in regular CI
	# before we find any way to reduce the time cost.
	# build_UBSAN:
	# name: Build UBSAN
	# needs: cpp_clang_format_linter
	# runs-on: ubuntu-latest
	# container:
	# image: apache/pegasus:thirdparties-bin-test-ubuntu1804-${{ github.base_ref }}
	# steps:
	# - uses: actions/checkout@v3
	# - name: Setup cache
	# uses: actions/cache@v3
	# with:
	# path: \|
	# /github/home/.ccache
	# key: ubsan_ccache
	# - uses: dorny/paths-filter@v2
	# id: changes
	# with:
	# filters: \|
	# thirdparty:
	# - '.github/workflows/thirdparty-regular-push.yml'
	# - 'docker/thirdparties-src/**'
	# - 'docker/thirdparties-bin/**'
	# - 'thirdparty/**'
	# - name: Unpack prebuilt third-parties
	# if: steps.changes.outputs.thirdparty == 'false'
	# run: \|
	# unzip /root/thirdparties-bin.zip -d ./thirdparty
	# rm -f /root/thirdparties-bin.zip
	# - name: Rebuild third-parties
	# if: steps.changes.outputs.thirdparty == 'true'
	# working-directory: thirdparty
	# # Build thirdparties and leave some necessary libraries and source
	# run: \|
	# mkdir build
	# cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
	# cmake --build build/ -j $(nproc)
	# rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
	# ../scripts/download_hadoop.sh hadoop-bin
	# ../scripts/download_zk.sh zookeeper-bin
	# - name: Compilation
	# run: \|
	# ccache -p
	# ccache -z
	# ./run.sh build --test --sanitizer undefined --skip_thirdparty --disable_gperf -j $(nproc)
	# ccache -s
	# - name: Tar files
	# run: \|
	# mv thirdparty/hadoop-bin ./
	# mv thirdparty/zookeeper-bin ./
	# rm -rf thirdparty
	# find ./build/latest/src/ -name 'CMakeFiles' -type d -exec rm -rf "{}" +
	# tar -zcvhf release_undefined_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin
	# - name: Upload Artifact
	# uses: actions/upload-artifact@v3
	# with:
	# name: release_undefined_artifact_${{ github.sha }}
	# path: release_undefined_builder.tar
	#
	# test_UBSAN:
	# name: Test UBSAN
	# strategy:
	# fail-fast: false
	# matrix:
	# test_module:
	# - backup_restore_test
	# - base_api_test
	# - base_test
	# - bulk_load_test
	# - detect_hotspot_test
	# - dsn_aio_test
	# - dsn_block_service_test
	# - dsn_client_test
	# - dsn.failure_detector.tests
	# - dsn_http_test
	# - dsn_meta_state_tests
	# - dsn.meta.test
	# - dsn_nfs_test
	# - dsn_perf_counter_test
	# - dsn_replica_backup_test
	# - dsn_replica_bulk_load_test
	# - dsn_replica_dup_test
	# - dsn_replica_split_test
	# - dsn.replica.test
	# - dsn_replication_common_test
	# - dsn.replication.simple_kv
	# - dsn.rep_tests.simple_kv
	# - dsn_runtime_tests
	# - dsn_utils_tests
	# - dsn.zookeeper.tests
	# - partition_split_test
	# - pegasus_geo_test
	# - pegasus_rproxy_test
	# - pegasus_unit_test
	# - recovery_test
	# - restore_test
	# - throttle_test
	# needs: build_UBSAN
	# runs-on: ubuntu-latest
	# container:
	# image: apache/pegasus:thirdparties-bin-test-ubuntu1804-${{ github.base_ref }}
	# options: --cap-add=SYS_PTRACE
	# steps:
	# - uses: actions/checkout@v3
	# - name: Unpack prebuilt third-parties
	# run: \|
	# unzip /root/thirdparties-bin.zip -d ./thirdparty
	# rm -f /root/thirdparties-bin.zip
	# - name: Download Artifact
	# uses: actions/download-artifact@v3
	# with:
	# name: release_undefined_artifact_${{ github.sha }}
	# path: .
	# - name: Tar files
	# run: \|
	# tar -zxvf release_undefined_builder.tar
	# - name: Unit Testing
	# run: \|
	# export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
	# ulimit -s unlimited
	# . ./scripts/config_hdfs.sh
	# ./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }}

	build_with_jemalloc:
	name: Build with jemalloc
	needs: cpp_clang_format_linter
	runs-on: ubuntu-latest
	container:
	image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu2204-${{ github.base_ref }}
	steps:
	- uses: actions/checkout@v3
	- name: Setup cache
	uses: actions/cache@v3
	with:
	path: \|
	/github/home/.ccache
	key: jemalloc_ccache
	- uses: dorny/paths-filter@v2
	id: changes
	with:
	filters: \|
	thirdparty:
	- '.github/workflows/thirdparty-regular-push.yml'
	- 'docker/thirdparties-src/**'
	- 'docker/thirdparties-bin/**'
	- 'thirdparty/**'
	- name: Unpack prebuilt third-parties
	if: steps.changes.outputs.thirdparty == 'false'
	run: \|
	unzip /root/thirdparties-bin.zip -d ./thirdparty
	rm -f /root/thirdparties-bin.zip
	- name: Rebuild third-parties
	if: steps.changes.outputs.thirdparty == 'true'
	working-directory: thirdparty
	# Build thirdparties and leave some necessary libraries and source
	run: \|
	mkdir build
	cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -DUSE_JEMALLOC=ON -B build/
	cmake --build build/ -j $(nproc)
	rm -rf build/Build build/Download/[a-y]* build/Source/[a-g]* build/Source/[i-q]* build/Source/[s-z]*
	../scripts/download_hadoop.sh hadoop-bin
	../scripts/download_zk.sh zookeeper-bin
	- name: Compilation
	run: \|
	ccache -p
	ccache -z
	./run.sh build --test --skip_thirdparty -j $(nproc) -t release --use_jemalloc
	ccache -s
	- name: Pack Server
	run: ./run.sh pack_server -j
	- name: Pack Tools
	run: ./run.sh pack_tools -j
	- name: Tar files
	run: \|
	mv thirdparty/hadoop-bin ./
	mv thirdparty/zookeeper-bin ./
	rm -rf thirdparty
	find ./build/latest/src/ -name 'CMakeFiles' -type d -exec rm -rf "{}" +
	tar -zcvhf release_jemalloc_builder.tar build/latest/output build/latest/bin build/latest/src/server/test/config.ini hadoop-bin zookeeper-bin
	- name: Upload Artifact
	uses: actions/upload-artifact@v3
	with:
	name: release_jemalloc_artifact_${{ github.sha }}
	path: release_jemalloc_builder.tar

	test_with_jemalloc:
	name: Test with jemallc
	strategy:
	fail-fast: false
	matrix:
	test_module:
	- dsn_utils_tests
	needs: build_with_jemalloc
	runs-on: ubuntu-latest
	container:
	image: apache/pegasus:thirdparties-bin-test-jemallc-ubuntu2204-${{ github.base_ref }}
	options: --cap-add=SYS_PTRACE
	steps:
	- uses: actions/checkout@v3
	- name: Unpack prebuilt third-parties
	run: \|
	unzip /root/thirdparties-bin.zip -d ./thirdparty
	rm -f /root/thirdparties-bin.zip
	- name: Download Artifact
	uses: actions/download-artifact@v3
	with:
	name: release_jemalloc_artifact_${{ github.sha }}
	path: .
	- name: Tar files
	run: \|
	tar -zxvf release_jemalloc_builder.tar
	- name: Unit Testing
	run: \|
	export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
	ulimit -s unlimited
	. ./scripts/config_hdfs.sh
	./run.sh test --onebox_opts "$ONEBOX_OPTS" --test_opts "$TEST_OPTS" -m ${{ matrix.test_module }}

	build_pegasus_on_macos:
	name: macOS
	needs: cpp_clang_format_linter
	runs-on: macos-12
	env:
	OPENSSL_ROOT_DIR: /usr/local/opt/openssl
	steps:
	- name: Install Softwares
	run: \|
	# Preinstalled softwares: https://github.com/actions/virtual-environments/blob/main/images/macos/macos-12-Readme.md
	brew install ccache
	brew install snappy
	brew install lz4
	brew install zstd
	brew install openssl@1.1
	- uses: actions/checkout@v3
	- name: Setup cache
	uses: actions/cache@v3
	with:
	path: \|
	/Users/runner/Library/Caches/ccache
	key: macos_build_ccache
	restore-keys: \|
	macos_build_ccache
	macos-12-thirdparty-bins
	- name: Build thirdparty
	working-directory: thirdparty
	run: \|
	export JAVA_HOME="${JAVA_HOME_8_X64}"
	mkdir -p build
	cmake -DCMAKE_BUILD_TYPE=Release -B build/ -DMACOS_OPENSSL_ROOT_DIR=${OPENSSL_ROOT_DIR}
	cmake --build build/ -j $(sysctl -n hw.physicalcpu)
	- name: Compilation
	run: \|
	ccache -p
	ccache -z
	./run.sh build --test --skip_thirdparty -j $(sysctl -n hw.physicalcpu)
	ccache -s

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Feature:Online Query and Dynamic Modification of Table-level RocksDB Options #1633

Workflow file

Feature:Online Query and Dynamic Modification of Table-level RocksDB Options #1633

Jobs

Run details

Workflow file for this run