diff --git a/build.sh b/build.sh index 5768bef..5be6114 100644 --- a/build.sh +++ b/build.sh @@ -33,7 +33,7 @@ cd $BUILD_PATH && wget https://github.com/facebook/zstd/archive/v${zstd_version} -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DZSTD_ZLIB_SUPPORT=ON -DZSTD_LZMA_SUPPORT=OFF -DCMAKE_BUILD_TYPE=Release .. && make -j$(nproc) install && \ cd $BUILD_PATH && rm -rf * && ldconfig -rocksdb_version="7.2.2" +rocksdb_version="7.3.1" cd $BUILD_PATH && wget https://github.com/facebook/rocksdb/archive/v${rocksdb_version}.tar.gz && tar xzf v${rocksdb_version}.tar.gz && cd rocksdb-${rocksdb_version}/ && \ mkdir -p build_place && cd build_place && cmake -DCMAKE_BUILD_TYPE=Release $CMAKE_REQUIRED_PARAMS -DCMAKE_PREFIX_PATH=$INSTALL_PREFIX -DWITH_TESTS=OFF -DWITH_GFLAGS=OFF \ -DWITH_BENCHMARK_TOOLS=OFF -DWITH_TOOLS=OFF -DWITH_MD_LIBRARY=OFF -DWITH_RUNTIME_DEBUG=OFF -DROCKSDB_BUILD_SHARED=OFF -DWITH_SNAPPY=ON -DWITH_LZ4=ON -DWITH_ZLIB=ON -DWITH_LIBURING=OFF \ diff --git a/c.h b/c.h index aae4d34..cbc7e1c 100644 --- a/c.h +++ b/c.h @@ -63,6 +63,7 @@ extern "C" { #endif #include +#include #include #include @@ -438,6 +439,38 @@ extern ROCKSDB_LIBRARY_API void rocksdb_multi_get_cf( const size_t* keys_list_sizes, char** values_list, size_t* values_list_sizes, char** errs); +// The MultiGet API that improves performance by batching operations +// in the read path for greater efficiency. Currently, only the block based +// table format with full filters are supported. Other table formats such +// as plain table, block based table with block based filters and +// partitioned indexes will still work, but will not get any performance +// benefits. +// +// Note that all the keys passed to this API are restricted to a single +// column family. +// +// Parameters - +// db - the RocksDB instance. +// options - ReadOptions +// column_family - ColumnFamilyHandle* that the keys belong to. All the keys +// passed to the API are restricted to a single column family +// num_keys - Number of keys to lookup +// keys_list - Pointer to C style array of keys with num_keys elements +// keys_list_sizes - Pointer to C style array of the size of corresponding key +// in key_list with num_keys elements. +// values - Pointer to C style array of PinnableSlices with num_keys elements +// statuses - Pointer to C style array of Status with num_keys elements +// sorted_input - If true, it means the input keys are already sorted by key +// order, so the MultiGet() API doesn't have to sort them +// again. If false, the keys will be copied and sorted +// internally by the API - the input array will not be +// modified +extern ROCKSDB_LIBRARY_API void rocksdb_batched_multi_get_cf( + rocksdb_t* db, const rocksdb_readoptions_t* options, + rocksdb_column_family_handle_t* column_family, size_t num_keys, + const char* const* keys_list, const size_t* keys_list_sizes, + rocksdb_pinnableslice_t** values, char** errs, const bool sorted_input); + // The value is only allocated (using malloc) and returned if it is found and // value_found isn't NULL. In that case the user is responsible for freeing it. extern ROCKSDB_LIBRARY_API unsigned char rocksdb_key_may_exist( @@ -1008,6 +1041,12 @@ extern ROCKSDB_LIBRARY_API int rocksdb_options_get_compression_options_zstd_max_train_bytes( rocksdb_options_t* opt); extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_compression_options_use_zstd_dict_trainer( + rocksdb_options_t*, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_compression_options_use_zstd_dict_trainer( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_compression_options_parallel_threads(rocksdb_options_t*, int); extern ROCKSDB_LIBRARY_API int @@ -1026,6 +1065,12 @@ extern ROCKSDB_LIBRARY_API void rocksdb_options_set_bottommost_compression_options_zstd_max_train_bytes( rocksdb_options_t*, int, unsigned char); extern ROCKSDB_LIBRARY_API void +rocksdb_options_set_bottommost_compression_options_use_zstd_dict_trainer( + rocksdb_options_t*, unsigned char, unsigned char); +extern ROCKSDB_LIBRARY_API unsigned char +rocksdb_options_get_bottommost_compression_options_use_zstd_dict_trainer( + rocksdb_options_t* opt); +extern ROCKSDB_LIBRARY_API void rocksdb_options_set_bottommost_compression_options_max_dict_buffer_bytes( rocksdb_options_t*, uint64_t, unsigned char); extern ROCKSDB_LIBRARY_API void rocksdb_options_set_prefix_extractor( @@ -1508,7 +1553,8 @@ enum { rocksdb_env_lock_file_nanos, rocksdb_env_unlock_file_nanos, rocksdb_env_new_logger_nanos, - rocksdb_total_metric_count = 68 + rocksdb_number_async_seek, + rocksdb_total_metric_count = 69 }; extern ROCKSDB_LIBRARY_API void rocksdb_set_perf_level(int); @@ -1766,6 +1812,8 @@ extern ROCKSDB_LIBRARY_API void rocksdb_lru_cache_options_set_memory_allocator( extern ROCKSDB_LIBRARY_API rocksdb_cache_t* rocksdb_cache_create_lru( size_t capacity); +extern ROCKSDB_LIBRARY_API rocksdb_cache_t* +rocksdb_cache_create_lru_with_strict_capacity_limit(size_t capacity); extern ROCKSDB_LIBRARY_API rocksdb_cache_t* rocksdb_cache_create_lru_opts( rocksdb_lru_cache_options_t*); extern ROCKSDB_LIBRARY_API void rocksdb_cache_destroy(rocksdb_cache_t* cache); diff --git a/options.go b/options.go index c38f386..beba9b3 100644 --- a/options.go +++ b/options.go @@ -579,6 +579,27 @@ func (opts *Options) GetCompressionOptionsZstdMaxTrainBytes() int { return int(C.rocksdb_options_get_compression_options_zstd_max_train_bytes(opts.c)) } +// SetCompressionOptionsZstdDictTrainer uses/not use zstd trainer to generate dictionaries. +// When this option is set to true, zstd_max_train_bytes of training data sampled from +// max_dict_buffer_bytes buffered data will be passed to zstd dictionary trainer to generate a +// dictionary of size max_dict_bytes. +// +// When this option is false, zstd's API ZDICT_finalizeDictionary() will be +// called to generate dictionaries. zstd_max_train_bytes of training sampled +// data will be passed to this API. Using this API should save CPU time on +// dictionary training, but the compression ratio may not be as good as using +// a dictionary trainer. +// +// Default: true +func (opts *Options) SetCompressionOptionsZstdDictTrainer(enabled bool) { + C.rocksdb_options_set_compression_options_use_zstd_dict_trainer(opts.c, boolToChar(enabled)) +} + +// GetCompressionOptionsZstdDictTrainer returns if zstd dict trainer is used or not. +func (opts *Options) GetCompressionOptionsZstdDictTrainer() bool { + return charToBool(C.rocksdb_options_get_compression_options_use_zstd_dict_trainer(opts.c)) +} + // SetCompressionOptionsParallelThreads sets number of threads for // parallel compression. Parallel compression is enabled only if threads > 1. // @@ -677,6 +698,28 @@ func (opts *Options) SetBottommostCompressionOptionsMaxDictBufferBytes(value uin ) } +// // SetBottommostCompressionOptionsZstdDictTrainer uses/not use zstd trainer to generate dictionaries. +// // When this option is set to true, zstd_max_train_bytes of training data sampled from +// // max_dict_buffer_bytes buffered data will be passed to zstd dictionary trainer to generate a +// // dictionary of size max_dict_bytes. +// // +// // When this option is false, zstd's API ZDICT_finalizeDictionary() will be +// // called to generate dictionaries. zstd_max_train_bytes of training sampled +// // data will be passed to this API. Using this API should save CPU time on +// // dictionary training, but the compression ratio may not be as good as using +// // a dictionary trainer. +// // +// // Default: true +// func (opts *Options) SetBottommostCompressionOptionsZstdDictTrainer(enabled bool) { +// c := boolToChar(enabled) +// C.rocksdb_options_set_bottommost_compression_options_use_zstd_dict_trainer(opts.c, c, c) +// } + +// // GetBottommostCompressionOptionsZstdDictTrainer returns if zstd dict trainer is used or not. +// func (opts *Options) GetBottommostCompressionOptionsZstdDictTrainer() bool { +// return charToBool(C.rocksdb_options_get_bottommost_compression_options_use_zstd_dict_trainer(opts.c)) +// } + // SetMinLevelToCompress sets the start level to use compression. func (opts *Options) SetMinLevelToCompress(value int) { C.rocksdb_options_set_min_level_to_compress(opts.c, C.int(value)) diff --git a/options_test.go b/options_test.go index d036133..bc8e980 100644 --- a/options_test.go +++ b/options_test.go @@ -353,6 +353,14 @@ func TestOptions(t *testing.T) { opts.SetWALCompression(LZ4Compression) require.EqualValues(t, LZ4Compression, opts.GetWALCompression()) + require.True(t, opts.GetCompressionOptionsZstdDictTrainer()) + opts.SetCompressionOptionsZstdDictTrainer(false) + require.False(t, opts.GetCompressionOptionsZstdDictTrainer()) + + // require.True(t, opts.GetBottommostCompressionOptionsZstdDictTrainer()) + // opts.SetCompressionOptionsZstdDictTrainer(false) + // require.False(t, opts.GetBottommostCompressionOptionsZstdDictTrainer()) + // cloning cl := opts.Clone() require.EqualValues(t, 5, cl.GetTableCacheNumshardbits())