Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adapt rocksdb 7.3.1 #75

Merged
merged 1 commit into from
Jul 19, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ cd $BUILD_PATH && wget https://github.com/facebook/zstd/archive/v${zstd_version}
-DCMAKE_POSITION_INDEPENDENT_CODE=ON -DZSTD_ZLIB_SUPPORT=ON -DZSTD_LZMA_SUPPORT=OFF -DCMAKE_BUILD_TYPE=Release .. && make -j$(nproc) install && \
cd $BUILD_PATH && rm -rf * && ldconfig

rocksdb_version="7.2.2"
rocksdb_version="7.3.1"
cd $BUILD_PATH && wget https://github.com/facebook/rocksdb/archive/v${rocksdb_version}.tar.gz && tar xzf v${rocksdb_version}.tar.gz && cd rocksdb-${rocksdb_version}/ && \
mkdir -p build_place && cd build_place && cmake -DCMAKE_BUILD_TYPE=Release $CMAKE_REQUIRED_PARAMS -DCMAKE_PREFIX_PATH=$INSTALL_PREFIX -DWITH_TESTS=OFF -DWITH_GFLAGS=OFF \
-DWITH_BENCHMARK_TOOLS=OFF -DWITH_TOOLS=OFF -DWITH_MD_LIBRARY=OFF -DWITH_RUNTIME_DEBUG=OFF -DROCKSDB_BUILD_SHARED=OFF -DWITH_SNAPPY=ON -DWITH_LZ4=ON -DWITH_ZLIB=ON -DWITH_LIBURING=OFF \
Expand Down
50 changes: 49 additions & 1 deletion c.h
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ extern "C" {
#endif

#include <stdarg.h>
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>

Expand Down Expand Up @@ -438,6 +439,38 @@ extern ROCKSDB_LIBRARY_API void rocksdb_multi_get_cf(
const size_t* keys_list_sizes, char** values_list,
size_t* values_list_sizes, char** errs);

// The MultiGet API that improves performance by batching operations
// in the read path for greater efficiency. Currently, only the block based
// table format with full filters are supported. Other table formats such
// as plain table, block based table with block based filters and
// partitioned indexes will still work, but will not get any performance
// benefits.
//
// Note that all the keys passed to this API are restricted to a single
// column family.
//
// Parameters -
// db - the RocksDB instance.
// options - ReadOptions
// column_family - ColumnFamilyHandle* that the keys belong to. All the keys
// passed to the API are restricted to a single column family
// num_keys - Number of keys to lookup
// keys_list - Pointer to C style array of keys with num_keys elements
// keys_list_sizes - Pointer to C style array of the size of corresponding key
// in key_list with num_keys elements.
// values - Pointer to C style array of PinnableSlices with num_keys elements
// statuses - Pointer to C style array of Status with num_keys elements
// sorted_input - If true, it means the input keys are already sorted by key
// order, so the MultiGet() API doesn't have to sort them
// again. If false, the keys will be copied and sorted
// internally by the API - the input array will not be
// modified
extern ROCKSDB_LIBRARY_API void rocksdb_batched_multi_get_cf(
rocksdb_t* db, const rocksdb_readoptions_t* options,
rocksdb_column_family_handle_t* column_family, size_t num_keys,
const char* const* keys_list, const size_t* keys_list_sizes,
rocksdb_pinnableslice_t** values, char** errs, const bool sorted_input);

// The value is only allocated (using malloc) and returned if it is found and
// value_found isn't NULL. In that case the user is responsible for freeing it.
extern ROCKSDB_LIBRARY_API unsigned char rocksdb_key_may_exist(
Expand Down Expand Up @@ -1008,6 +1041,12 @@ extern ROCKSDB_LIBRARY_API int
rocksdb_options_get_compression_options_zstd_max_train_bytes(
rocksdb_options_t* opt);
extern ROCKSDB_LIBRARY_API void
rocksdb_options_set_compression_options_use_zstd_dict_trainer(
rocksdb_options_t*, unsigned char);
extern ROCKSDB_LIBRARY_API unsigned char
rocksdb_options_get_compression_options_use_zstd_dict_trainer(
rocksdb_options_t* opt);
extern ROCKSDB_LIBRARY_API void
rocksdb_options_set_compression_options_parallel_threads(rocksdb_options_t*,
int);
extern ROCKSDB_LIBRARY_API int
Expand All @@ -1026,6 +1065,12 @@ extern ROCKSDB_LIBRARY_API void
rocksdb_options_set_bottommost_compression_options_zstd_max_train_bytes(
rocksdb_options_t*, int, unsigned char);
extern ROCKSDB_LIBRARY_API void
rocksdb_options_set_bottommost_compression_options_use_zstd_dict_trainer(
rocksdb_options_t*, unsigned char, unsigned char);
extern ROCKSDB_LIBRARY_API unsigned char
rocksdb_options_get_bottommost_compression_options_use_zstd_dict_trainer(
rocksdb_options_t* opt);
extern ROCKSDB_LIBRARY_API void
rocksdb_options_set_bottommost_compression_options_max_dict_buffer_bytes(
rocksdb_options_t*, uint64_t, unsigned char);
extern ROCKSDB_LIBRARY_API void rocksdb_options_set_prefix_extractor(
Expand Down Expand Up @@ -1508,7 +1553,8 @@ enum {
rocksdb_env_lock_file_nanos,
rocksdb_env_unlock_file_nanos,
rocksdb_env_new_logger_nanos,
rocksdb_total_metric_count = 68
rocksdb_number_async_seek,
rocksdb_total_metric_count = 69
};

extern ROCKSDB_LIBRARY_API void rocksdb_set_perf_level(int);
Expand Down Expand Up @@ -1766,6 +1812,8 @@ extern ROCKSDB_LIBRARY_API void rocksdb_lru_cache_options_set_memory_allocator(

extern ROCKSDB_LIBRARY_API rocksdb_cache_t* rocksdb_cache_create_lru(
size_t capacity);
extern ROCKSDB_LIBRARY_API rocksdb_cache_t*
rocksdb_cache_create_lru_with_strict_capacity_limit(size_t capacity);
extern ROCKSDB_LIBRARY_API rocksdb_cache_t* rocksdb_cache_create_lru_opts(
rocksdb_lru_cache_options_t*);
extern ROCKSDB_LIBRARY_API void rocksdb_cache_destroy(rocksdb_cache_t* cache);
Expand Down
43 changes: 43 additions & 0 deletions options.go
Original file line number Diff line number Diff line change
Expand Up @@ -579,6 +579,27 @@ func (opts *Options) GetCompressionOptionsZstdMaxTrainBytes() int {
return int(C.rocksdb_options_get_compression_options_zstd_max_train_bytes(opts.c))
}

// SetCompressionOptionsZstdDictTrainer uses/not use zstd trainer to generate dictionaries.
// When this option is set to true, zstd_max_train_bytes of training data sampled from
// max_dict_buffer_bytes buffered data will be passed to zstd dictionary trainer to generate a
// dictionary of size max_dict_bytes.
//
// When this option is false, zstd's API ZDICT_finalizeDictionary() will be
// called to generate dictionaries. zstd_max_train_bytes of training sampled
// data will be passed to this API. Using this API should save CPU time on
// dictionary training, but the compression ratio may not be as good as using
// a dictionary trainer.
//
// Default: true
func (opts *Options) SetCompressionOptionsZstdDictTrainer(enabled bool) {
C.rocksdb_options_set_compression_options_use_zstd_dict_trainer(opts.c, boolToChar(enabled))
}

// GetCompressionOptionsZstdDictTrainer returns if zstd dict trainer is used or not.
func (opts *Options) GetCompressionOptionsZstdDictTrainer() bool {
return charToBool(C.rocksdb_options_get_compression_options_use_zstd_dict_trainer(opts.c))
}

// SetCompressionOptionsParallelThreads sets number of threads for
// parallel compression. Parallel compression is enabled only if threads > 1.
//
Expand Down Expand Up @@ -677,6 +698,28 @@ func (opts *Options) SetBottommostCompressionOptionsMaxDictBufferBytes(value uin
)
}

// // SetBottommostCompressionOptionsZstdDictTrainer uses/not use zstd trainer to generate dictionaries.
// // When this option is set to true, zstd_max_train_bytes of training data sampled from
// // max_dict_buffer_bytes buffered data will be passed to zstd dictionary trainer to generate a
// // dictionary of size max_dict_bytes.
// //
// // When this option is false, zstd's API ZDICT_finalizeDictionary() will be
// // called to generate dictionaries. zstd_max_train_bytes of training sampled
// // data will be passed to this API. Using this API should save CPU time on
// // dictionary training, but the compression ratio may not be as good as using
// // a dictionary trainer.
// //
// // Default: true
// func (opts *Options) SetBottommostCompressionOptionsZstdDictTrainer(enabled bool) {
// c := boolToChar(enabled)
// C.rocksdb_options_set_bottommost_compression_options_use_zstd_dict_trainer(opts.c, c, c)
// }

// // GetBottommostCompressionOptionsZstdDictTrainer returns if zstd dict trainer is used or not.
// func (opts *Options) GetBottommostCompressionOptionsZstdDictTrainer() bool {
// return charToBool(C.rocksdb_options_get_bottommost_compression_options_use_zstd_dict_trainer(opts.c))
// }

// SetMinLevelToCompress sets the start level to use compression.
func (opts *Options) SetMinLevelToCompress(value int) {
C.rocksdb_options_set_min_level_to_compress(opts.c, C.int(value))
Expand Down
8 changes: 8 additions & 0 deletions options_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -353,6 +353,14 @@ func TestOptions(t *testing.T) {
opts.SetWALCompression(LZ4Compression)
require.EqualValues(t, LZ4Compression, opts.GetWALCompression())

require.True(t, opts.GetCompressionOptionsZstdDictTrainer())
opts.SetCompressionOptionsZstdDictTrainer(false)
require.False(t, opts.GetCompressionOptionsZstdDictTrainer())

// require.True(t, opts.GetBottommostCompressionOptionsZstdDictTrainer())
// opts.SetCompressionOptionsZstdDictTrainer(false)
// require.False(t, opts.GetBottommostCompressionOptionsZstdDictTrainer())

// cloning
cl := opts.Clone()
require.EqualValues(t, 5, cl.GetTableCacheNumshardbits())
Expand Down