Release 2.2.0

[neverchanje]

PR (72 TOTAL, 0 PICKED)	TITLE
#736	fix: fix logo missing
#730	doc: update README.md fix a typo
#715	doc: Grammar fixes
#718	fix: overload dump_write_request shouldn't auto-reply to client
#708	fix(script): oprimizations for pegasus_rolling_update.sh
#711	fix: build failure on clang9
#716	fix: shell app_stat removes unexpected columns
#710	fix(hotkey): fix input and output of detect_hotkey in shell
#707	fix(restore): make 'old_policy_name' an optional parameter
#706	fix: do not get master while raise a new pull request
#705	feat: add read rate limiter perfcounter in info collector
#700	fix(scripts): skip packaging irrelevant libs
#697	fix: package script on ubuntu1804
#676	fix: pack without libaio
#690	chore(ci): migrate pegasus ci to github action
#684	fix(split): add flush_memtable for copy_checkpoint_to_dir
#686	feat: remove the dependency of tls memory for redis_parser
#685	fix(asan): repeated memory release in pegasus_server_write_test.batch_writes
#675	fix: compilation error in pegasus_utils
#680	refactor: reimplement pegasus_server_write::on_batched_write_requests
#679	feat(hotkey): dynamic change some parameters in hotkey detection
#678	refactor: use rocksdb_wrapper to reimplement batch operations
#673	ci: update action-semantic-pull-request to v2.2.0
#665	feat(hotkey): add a function test of hotkey detection
#674	refactor: remove unused unit tests in pegasus_write_service_impl_test
#671	refactor: move func ingestion_files to rocksdb_wrapper
#672	refactor: use rocksdb_wrapper to reimplement check_and_mutate
#670	refactor: use rocksdb_wrapper to reimplement empty_put
#669	refactor: move write_batch_delete into rocksdb_wrapper
#666	refactor: use rocksdb_wrapper::write_batch_put to reimplement check_and_set
#657	docs: add rfc for meta-proxy
#667	refactor: use rocksdb_wrapper::write_batch_put to reimplement incr
#656	refactor: use rocksdb_wrapper::write_batch_put to reimplement check_and_set
#663	feat: add query app data_version
#664	feat(security): add thirdparty lib when pack server
#662	feat(hotkey): add an interface to query hotkey on the specified partition
#658	chore: add config_hdfs.sh
#660	feat(bulk_load): add remote_file_root for start_bulk_load_request
#655	refactor: use rocksdb_wrapper::get to reimplement check_and_mutate
#654	refactor: use rocksdb_wrapper::get to reimplement check_and_set
#646	feat: support costing memtable and index and filter blocks to block c…
#651	refactor: use rocksdb_wrapper::get to reimplement incr
#648	feat(hotkey): add unit test of hotkey collector
#652	fix: fix log bug in multi_get
#647	feat(backup): support using hdfs to backup/restore pegasus tables
#632	fix(hotkey): fix two hidden dangers in hotkey_collector
#639	feat(hotkey): add unit test of internal_collector
#641	fix(hotspot): replace partition_resolver to ddl_client
#644	refactor(shell): delete the disk info query command
#640	feat(hotkey): add thread_safe method to clear collector
#631	feat(hotkey): capture data part3 - declare fine collector
#624	feat(hotkey): capture data part2 - declare coarse collector
#627	fix: plog aio tasks are assigned replica long thread pool will cause close app block
#625	feat(hotkey): collector can be terminated by timeout
#614	docs: propose a rfc of pegasus data version v3
#621	feat(hotkey): implement the RPC handler in hotkey_collector
#618	feat(hotkey): capture data part1 - declare internal collector
#619	fix: fix unit test of capacity_unit_calculator
#616	feat(hotkey): add a state machine of hotkey_collector
#615	feat(hotkey): add an interface of hotkey capture
#603	feat(hotkey): build a fundamental framework of hotkey detection
#613	chore: move github pullrequests to commits mail-list
#611	feat(shell): add a util to validate the input ip string with list_nodes
#612	refactor(hotspot): remove hotkey_detect_request in rrdb.thrift
#605	feat(hotspot): add a function to start hotkey detecting in shell commands
#608	fix: bugfix on incr
#607	fix(hotspot): using dsn::make_unique instead of std::make_unique
#604	feat(hotspot): calculator auto detect hotkey in the hot partition
#592	refactor: split read/write hotspot of hotspot_partition_calculator
#601	feat(hotspot): add a function to start hotkey detecting in hotspot_partition_calculator
#598	refactor: use db_get to implement incr
#597	refactor(collector): sort out the structure of partition hotspot detection

PR (175 TOTAL, 0 PICKED)	TITLE
XiaoMi/rdsn#835	fix: fix a compile error on clang
XiaoMi/rdsn#837	fix: update bulk load download thread pool
XiaoMi/rdsn#832	feat: add remote command to get tcmalloc status
XiaoMi/rdsn#818	fix: support retry when io write incompletely
XiaoMi/rdsn#781	fix: update bool app envs
XiaoMi/rdsn#768	fix(security): doesn't reject request even if it's rejected by access controller
XiaoMi/rdsn#829	feat(bulk_load): meta server adds bulk load ingestion concurrent count restriction
XiaoMi/rdsn#827	feat(bulk_load): add bulk load max rollback count
XiaoMi/rdsn#828	fix(bulk_load): remove possible old directory when start bulk load
XiaoMi/rdsn#826	fix(bulk_load): redownload file if local file damaged
XiaoMi/rdsn#823	fix(bulk_load): fix remove_local_bulk_load_dir()
XiaoMi/rdsn#824	fix: add /version http api
XiaoMi/rdsn#807	fix(backup_policy): do not try again when got fs errors
XiaoMi/rdsn#808	fix: incorrect help info of meta/replica remote commands
XiaoMi/rdsn#806	fix: compilation errors on clang-9
XiaoMi/rdsn#805	feat(http): update query manual compaction interface
XiaoMi/rdsn#803	build: fix compilation errors on clang-10
XiaoMi/rdsn#801	feat(http): update query data version
XiaoMi/rdsn#800	fix: modify the ouput format for configs http api
XiaoMi/rdsn#799	fix: remove read size throttling
XiaoMi/rdsn#780	fix: fix compilation errors
XiaoMi/rdsn#368	refactor: move primary's learning preparation of cache into another function
XiaoMi/rdsn#770	fix: enable config runtime-update using flag for disk_cleaner
XiaoMi/rdsn#760	refactor: separate files from replication.thrift
XiaoMi/rdsn#668	feat(disk_balance): close origin replica and update replica dir
XiaoMi/rdsn#754	refactor: use shared_ptr to replace raw pointer
XiaoMi/rdsn#759	refactor: remove thrift generated files of fd/simplekv
XiaoMi/rdsn#758	refactor: remove generated thrift srcs from repo
XiaoMi/rdsn#753	fix: do not call unregister_rpc_handler when exiting progress
XiaoMi/rdsn#752	fix: make disk_engine destructed after service_engine destructed
XiaoMi/rdsn#750	fix(perf_counter_wrapper): avoid using invalid perf_counters::instance() to remove counter
XiaoMi/rdsn#678	fix: cluster_name of ddl client from zk root
XiaoMi/rdsn#742	feat(split): supplement some case after implementing pause cancel split
XiaoMi/rdsn#757	fix: memory leak in nfs test
XiaoMi/rdsn#756	fix: delete unused flags rocksdb_target_file_size_base and rocksdb_write_buffer_size
XiaoMi/rdsn#755	fix: set burst_size of _write_token_bucket in fds_service to double max
XiaoMi/rdsn#747	feat(split): replica add validate_partition_hash
XiaoMi/rdsn#749	refactor: remove the dependency of tls memory for message_ex
XiaoMi/rdsn#751	fix(asan): memory leak in command_manager
XiaoMi/rdsn#748	fix: fix log print
XiaoMi/rdsn#734	fix(command_manager): avoid using std::remove to erase elements from std::vector
XiaoMi/rdsn#746	fix: error reported in build process of pegasus-docker
XiaoMi/rdsn#745	feat(split): add app_env split_validate_partition_hash
XiaoMi/rdsn#744	fix: fix lock in notify_stop_split
XiaoMi/rdsn#726	feat(split): add query split status interface
XiaoMi/rdsn#729	fix(split): add flush_memtable for copy_checkpoint_to_dir
XiaoMi/rdsn#711	refactor: reimplement join point with less code
XiaoMi/rdsn#731	feat(split): add notify_stop_split rpc
XiaoMi/rdsn#739	refactor: remove the dependency of tls memory for some member functions in message_ex
XiaoMi/rdsn#728	feat: add throttling for read size and read qps
XiaoMi/rdsn#738	fix(asan): heap-use-after-free in rpc_read_stream
XiaoMi/rdsn#737	feat: remove transient_object which is implemented by tls memory
XiaoMi/rdsn#736	fix(hdfs): do not call hdfsDisconnect() after jvm exited
XiaoMi/rdsn#735	fix(perf_counter): remove static map from counter_info
XiaoMi/rdsn#730	fix: fix the usage of service_engine::get_all_nodes()
XiaoMi/rdsn#733	fix(asan): heap-use-after-free caused by replica_test_base::create_test_mutation
XiaoMi/rdsn#681	feat(split): replica server handle pause and cancel status
XiaoMi/rdsn#725	fix(split): message header should not be empty
XiaoMi/rdsn#679	feat(split): add meta control split
XiaoMi/rdsn#715	fix: multiple asan errors
XiaoMi/rdsn#722	refactor: don't resend message if it is rejected by server
XiaoMi/rdsn#723	refactor: delete find package libaio dependency when build
XiaoMi/rdsn#721	refactor: refactor backup_test
XiaoMi/rdsn#720	feat: add read limiter for hdfs
XiaoMi/rdsn#719	feat: add http interface to get a specified config
XiaoMi/rdsn#717	refactor(backup): refactor add_backup_policy
XiaoMi/rdsn#718	fix(hotkey): add replication_enums of detect_action::QUERY
XiaoMi/rdsn#704	feat: add http interface to get all configs
XiaoMi/rdsn#716	BREAKING CHANGE(backup): remove 'policy_name' from backup files' path
XiaoMi/rdsn#712	fix(security): make mandatory_auth to be a server side config
XiaoMi/rdsn#714	feat: add validation for value updating through http api
XiaoMi/rdsn#713	feat: update query partition data version
XiaoMi/rdsn#710	feat(hdfs_service): make hdfs_read_batch_size_bytes/hdfs_write_batch_size_bytes mutable
XiaoMi/rdsn#709	fix(test): add gtest framework to meta tests
XiaoMi/rdsn#696	feat(bulk_load): add query compaction status http interface
XiaoMi/rdsn#690	feat: support to build a portable rocksdb binary
XiaoMi/rdsn#699	refactor: fix typo and add comments to security
XiaoMi/rdsn#703	fix: block service download file return error when file exists
XiaoMi/rdsn#707	feat: add update rocksdb scenario interface
XiaoMi/rdsn#708	refactor(security): remove mandatory on client side
XiaoMi/rdsn#695	feat(bulk_load): add update compaction envs http interface
XiaoMi/rdsn#706	feat: add query app data_version interface
XiaoMi/rdsn#702	feat(security): make enable_acl and mandatory_auth mutable
XiaoMi/rdsn#705	fix(scripts): fix zk version in zk scripts
XiaoMi/rdsn#700	feat(hotkey): Change RPC to query hotkey
XiaoMi/rdsn#698	fix: add hash for flag_tag to compatible with gcc < 6.1
XiaoMi/rdsn#692	fix(block_service): download file when local file is different from remote file
XiaoMi/rdsn#682	feat: support to modify configs without restart
XiaoMi/rdsn#693	feat(bulk_load): add start bulk load http interface
XiaoMi/rdsn#694	feat(security): treat negotiation succeed if server is old version
XiaoMi/rdsn#691	fix(hdfs): fix 'libjvm.so not found' error
XiaoMi/rdsn#689	fix: zk compile cflag no warn format errors
XiaoMi/rdsn#686	feat(bulk-load): support user-define remote storage root path
XiaoMi/rdsn#688	fix: add #pragma once to defer.h
XiaoMi/rdsn#685	fix: compile error of zookeeper 3.4.10 on gcc9
XiaoMi/rdsn#687	feat(bulk-load): add query bulk load http interface
XiaoMi/rdsn#683	feat(security): enable access replica when appenv replica_access_controller.allowed_users is empty
XiaoMi/rdsn#680	refactor: remove support_write_vec in aio_task
XiaoMi/rdsn#676	feat(split): child copy mutation asynchronously
XiaoMi/rdsn#675	feat(split): secondary start split
XiaoMi/rdsn#677	refactor: use THREAD_POOL_BLOCK_SERVICE ro replace THREAD_POOL_FDS_SERVICE/THREAD_POOL_LOCAL_SERVICE
XiaoMi/rdsn#672	feat(security): implement reset interface for replica access controller
XiaoMi/rdsn#673	fix: downgrade hadoop version to 2.8.4
XiaoMi/rdsn#674	fix: fmtlib incompatibility problems from 7.1.3 to 5.3.0
XiaoMi/rdsn#653	feat(split): add splitting_replicas while on_config_sync
XiaoMi/rdsn#664	feat(disk_balance): add do_disk_migrate_replica to support migrate origin data
XiaoMi/rdsn#665	fix: time utils overflow, meta_split_service lock
XiaoMi/rdsn#671	fix: update thirdparty fmtlib version
XiaoMi/rdsn#670	feat(security): implement replica server access controller
XiaoMi/rdsn#655	feat(security): implement meta server access controller
XiaoMi/rdsn#647	feat(backup): add hdfs as an alternative backup provider
XiaoMi/rdsn#666	fix: local_service compile error loading file metadata
XiaoMi/rdsn#646	refactor: refactor app operation unit tests
XiaoMi/rdsn#660	feat(disk_balance): support and validate disk migration rpc
XiaoMi/rdsn#663	fix(http): fix parse path from message
XiaoMi/rdsn#662	fix(http): fix message buffers size check
XiaoMi/rdsn#661	fix(backup): use existing block_service_manager when add backup policy
XiaoMi/rdsn#651	refactor: update detail disk info
XiaoMi/rdsn#658	chore: migrate ci from travis to github actions
XiaoMi/rdsn#652	fix(security): fix bug in negotiation_service::on_negotiation_request when rpc_session is closed
XiaoMi/rdsn#654	feat(split): parent group update partition count
XiaoMi/rdsn#648	fix(split): fix split register child unit test
XiaoMi/rdsn#645	feat(split): add update_child_group_partition_count
XiaoMi/rdsn#649	fix: plog aio tasks are assigned replica long thread pool will cause close app block
XiaoMi/rdsn#631	refactor(backup): straighten the execution path of download_file
XiaoMi/rdsn#644	refactor: reduce netprovider unit test executing time
XiaoMi/rdsn#643	feat(split): update register_child with split_status
XiaoMi/rdsn#633	perf: optimizing write latency using independent IO queues replace of libaio
XiaoMi/rdsn#641	feat(split): add split_status
XiaoMi/rdsn#642	feat(split): implement child_handle_split_error
XiaoMi/rdsn#640	feat(split): add start_partition_split interface
XiaoMi/rdsn#637	refactor: refactor replica split unit tests
XiaoMi/rdsn#638	refactor(split): refactor meta start partition split
XiaoMi/rdsn#636	feat(split): small refactor and fix of replica split
XiaoMi/rdsn#639	refactor(security): make send/recv message allowed when mandatory_auth is false
XiaoMi/rdsn#635	fix: fix fds_service_test
XiaoMi/rdsn#634	feat(hotkey): Add an interface of on_detect_hotkey
XiaoMi/rdsn#632	feat: send hotkey_detect_request in ddl_client
XiaoMi/rdsn#630	refactor(block_service): remove rebundant functions
XiaoMi/rdsn#629	refactor(security): add join point to filter the sending message
XiaoMi/rdsn#628	refactor(security): add join point to filter received message
XiaoMi/rdsn#626	refactor(security): add join point to start negotiation
XiaoMi/rdsn#624	refactor(split): move replica split functions into replica_split_manager class
XiaoMi/rdsn#627	refactor(fds_service): clean up duplicate code
XiaoMi/rdsn#625	refactor(backup): move backup rpc handling to replica_backup_server
XiaoMi/rdsn#623	refactor(backup): move common utils out to backup_utils
XiaoMi/rdsn#621	fix: set the tracer type when init
XiaoMi/rdsn#622	refactor(security): use blob instead of std::string as the type for msg member of negotiation_request
XiaoMi/rdsn#620	refactor(security): refactor the send func of client_negotiation
XiaoMi/rdsn#619	refactor(backup): move cold_backup_context out to standalone file
XiaoMi/rdsn#615	refactor(http): add http handler without creating http service
XiaoMi/rdsn#618	feat(security): server_negotiation handle challenge response
XiaoMi/rdsn#617	feat(security): client_negotiation handle challenge
XiaoMi/rdsn#616	fix(security): fix bug in building cyrus-sasl
XiaoMi/rdsn#613	feat(security): server_negotiation handle SASL_INITIATE request
XiaoMi/rdsn#612	feat(security): client_negotiation handle mechanism selected response
XiaoMi/rdsn#610	refactor: hide http_server from http api
XiaoMi/rdsn#608	feat(security): add unit tests for client_negotiation
XiaoMi/rdsn#611	feat(dup): clear data on meta when duplication removed
XiaoMi/rdsn#604	feat(security): implement sasl_wrapper which is a wrapper to operate sasl
XiaoMi/rdsn#607	feat(security): add unit tests for server_negotiation
XiaoMi/rdsn#609	feat(security): add unit test for negotiation_service
XiaoMi/rdsn#594	refactor(http): replace service/method with simple HTTP path
XiaoMi/rdsn#606	refactor(security): close the connection when something is wrong with sever_negotiation
XiaoMi/rdsn#511	feat: add point for write using latency_tracer tool
XiaoMi/rdsn#599	feat(security): init sasl
XiaoMi/rdsn#598	refactor: use member func instead of macro RPC_CHECK_STATUS
XiaoMi/rdsn#600	feat(security): select mechanism
XiaoMi/rdsn#595	feat(security): receive mechanisms which are supported by server_negotiation
XiaoMi/rdsn#589	feat(security): make network support security negotiation
XiaoMi/rdsn#585	feat(security): init kerberos
XiaoMi/rdsn#591	build: use nodejs/http_parser as a external dep
XiaoMi/rdsn#588	feat(security): handle SASL_LIST_MECHANISMS by server_negotiation
XiaoMi/rdsn#593	feat(http): support parsing content-type of HTTP request
XiaoMi/rdsn#592	refactor: move http_server.h to dsn/http

Configuration updates

[pegasus.server]
+ coarse_data_variance_threshold = 3
+ data_capture_hash_bucket_num = 37
+ max_seconds_to_detect_hotkey = 150

+ rocksdb_enable_write_buffer_manager = false
+ rocksdb_total_size_across_write_buffer = 0
+ rocksdb_max_open_files = -1

+ rocksdb_index_type = binary_search | hash_search | two_level_index_search | binary_search_with_first_key
+ rocksdb_partition_filters = false
+ rocksdb_metadata_block_size = 4096
+ rocksdb_cache_index_and_filter_blocks = false
+ rocksdb_pin_top_level_index_and_filter = true
+ rocksdb_cache_index_and_filter_blocks_with_high_priority = true
+ rocksdb_pin_l0_filter_and_index_blocks_in_cache = false

[pegasus.collector]
- hotspot_detect_algorithm
+ enable_hotkey_auto_detect = false
+ hot_partition_threshold = 3
+ occurrence_threshold = 100
+ hotkey_analyse_time_interval_s = 10

[apps.replica]
- pools = THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_FD,THREAD_POOL_LOCAL_APP,THREAD_POOL_FDS_SERVICE,THREAD_POOL_COMPACT,THREAD_POOL_INGESTION,THREAD_POOL_SLOG
+ pools = THREAD_POOL_DEFAULT,THREAD_POOL_REPLICATION_LONG,THREAD_POOL_REPLICATION,THREAD_POOL_FD,THREAD_POOL_LOCAL_APP,THREAD_POOL_BLOCK_SERVICE,THREAD_POOL_COMPACT,THREAD_POOL_INGESTION,THREAD_POOL_SLOG,THREAD_POOL_PLOG

[apps.meta]
- pools = THREAD_POOL_DEFAULT,THREAD_POOL_META_SERVER,THREAD_POOL_META_STATE,THREAD_POOL_FD,THREAD_POOL_DLOCK,THREAD_POOL_FDS_SERVICE
+ pools = THREAD_POOL_DEFAULT,THREAD_POOL_META_SERVER,THREAD_POOL_META_STATE,THREAD_POOL_FD,THREAD_POOL_DLOCK,THREAD_POOL_BLOCK_SERVICE

+[threadpool.THREAD_POOL_PLOG]
+ name = plog
+ partitioned = true
+ worker_count = 4

-[threadpool.THREAD_POOL_FDS_SERVICE]
- name = fds_service
+[threadpool.THREAD_POOL_BLOCK_SERVICE]
+ name = block_service

[replication]
- bulk_load_provider_root = bulk_load_root
+ hdfs_read_limit_rate_megabytes = 200
+ gc_disk_migration_tmp_replica_interval_seconds = 86400 #1day
+ gc_disk_migration_origin_replica_interval_seconds = 604800 #7day
+ hdfs_read_batch_size_bytes = 67108864 # 64MB
+ hdfs_write_batch_size_bytes = 67108864 # 64MB

+[security]
+ enable_acl = false
+ super_users = 
+ meta_acl_rpc_allow_list = 
+ service_name = 
+ service_fqdn = 
+ sasl_plugin_path = 
+ mandatory_auth = false
+ krb5_keytab =
+ krb5_config =
+ krb5_principal =
+ enable_auth = false