Adapt rocksdb 7.3.1 (#75)

build.sh

@@ -33,7 +33,7 @@ cd $BUILD_PATH && wget https://github.com/facebook/zstd/archive/v${zstd_version}
     -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DZSTD_ZLIB_SUPPORT=ON -DZSTD_LZMA_SUPPORT=OFF -DCMAKE_BUILD_TYPE=Release .. && make -j$(nproc) install && \
     cd $BUILD_PATH && rm -rf * && ldconfig
 
-rocksdb_version="7.2.2"
+rocksdb_version="7.3.1"
 cd $BUILD_PATH && wget https://github.com/facebook/rocksdb/archive/v${rocksdb_version}.tar.gz && tar xzf v${rocksdb_version}.tar.gz && cd rocksdb-${rocksdb_version}/ && \
     mkdir -p build_place && cd build_place && cmake -DCMAKE_BUILD_TYPE=Release $CMAKE_REQUIRED_PARAMS -DCMAKE_PREFIX_PATH=$INSTALL_PREFIX -DWITH_TESTS=OFF -DWITH_GFLAGS=OFF \
     -DWITH_BENCHMARK_TOOLS=OFF -DWITH_TOOLS=OFF -DWITH_MD_LIBRARY=OFF -DWITH_RUNTIME_DEBUG=OFF -DROCKSDB_BUILD_SHARED=OFF -DWITH_SNAPPY=ON -DWITH_LZ4=ON -DWITH_ZLIB=ON -DWITH_LIBURING=OFF \

c.h

@@ -63,6 +63,7 @@ extern "C" {
 #endif
 
 #include <stdarg.h>
+#include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
 
@@ -438,6 +439,38 @@ extern ROCKSDB_LIBRARY_API void rocksdb_multi_get_cf(
     const size_t* keys_list_sizes, char** values_list,
     size_t* values_list_sizes, char** errs);
 
+// The MultiGet API that improves performance by batching operations
+// in the read path for greater efficiency. Currently, only the block based
+// table format with full filters are supported. Other table formats such
+// as plain table, block based table with block based filters and
+// partitioned indexes will still work, but will not get any performance
+// benefits.
+//
+// Note that all the keys passed to this API are restricted to a single
+// column family.
+//
+// Parameters -
+// db - the RocksDB instance.
+// options - ReadOptions
+// column_family - ColumnFamilyHandle* that the keys belong to. All the keys
+//                 passed to the API are restricted to a single column family
+// num_keys - Number of keys to lookup
+// keys_list - Pointer to C style array of keys with num_keys elements
+// keys_list_sizes - Pointer to C style array of the size of corresponding key
+//   in key_list with num_keys elements.
+// values - Pointer to C style array of PinnableSlices with num_keys elements
+// statuses - Pointer to C style array of Status with num_keys elements
+// sorted_input - If true, it means the input keys are already sorted by key
+//                order, so the MultiGet() API doesn't have to sort them
+//                again. If false, the keys will be copied and sorted
+//                internally by the API - the input array will not be
+//                modified
+extern ROCKSDB_LIBRARY_API void rocksdb_batched_multi_get_cf(
+    rocksdb_t* db, const rocksdb_readoptions_t* options,
+    rocksdb_column_family_handle_t* column_family, size_t num_keys,
+    const char* const* keys_list, const size_t* keys_list_sizes,
+    rocksdb_pinnableslice_t** values, char** errs, const bool sorted_input);
+
 // The value is only allocated (using malloc) and returned if it is found and
 // value_found isn't NULL. In that case the user is responsible for freeing it.
 extern ROCKSDB_LIBRARY_API unsigned char rocksdb_key_may_exist(
@@ -1008,6 +1041,12 @@ extern ROCKSDB_LIBRARY_API int
 rocksdb_options_get_compression_options_zstd_max_train_bytes(
     rocksdb_options_t* opt);
 extern ROCKSDB_LIBRARY_API void
+rocksdb_options_set_compression_options_use_zstd_dict_trainer(
+    rocksdb_options_t*, unsigned char);
+extern ROCKSDB_LIBRARY_API unsigned char
+rocksdb_options_get_compression_options_use_zstd_dict_trainer(
+    rocksdb_options_t* opt);
+extern ROCKSDB_LIBRARY_API void
 rocksdb_options_set_compression_options_parallel_threads(rocksdb_options_t*,
                                                          int);
 extern ROCKSDB_LIBRARY_API int
@@ -1026,6 +1065,12 @@ extern ROCKSDB_LIBRARY_API void
 rocksdb_options_set_bottommost_compression_options_zstd_max_train_bytes(
     rocksdb_options_t*, int, unsigned char);
 extern ROCKSDB_LIBRARY_API void
+rocksdb_options_set_bottommost_compression_options_use_zstd_dict_trainer(
+    rocksdb_options_t*, unsigned char, unsigned char);
+extern ROCKSDB_LIBRARY_API unsigned char
+rocksdb_options_get_bottommost_compression_options_use_zstd_dict_trainer(
+    rocksdb_options_t* opt);
+extern ROCKSDB_LIBRARY_API void
 rocksdb_options_set_bottommost_compression_options_max_dict_buffer_bytes(
     rocksdb_options_t*, uint64_t, unsigned char);
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_prefix_extractor(
@@ -1508,7 +1553,8 @@ enum {
   rocksdb_env_lock_file_nanos,
   rocksdb_env_unlock_file_nanos,
   rocksdb_env_new_logger_nanos,
-  rocksdb_total_metric_count = 68
+  rocksdb_number_async_seek,
+  rocksdb_total_metric_count = 69
 };
 
 extern ROCKSDB_LIBRARY_API void rocksdb_set_perf_level(int);
@@ -1766,6 +1812,8 @@ extern ROCKSDB_LIBRARY_API void rocksdb_lru_cache_options_set_memory_allocator(
 
 extern ROCKSDB_LIBRARY_API rocksdb_cache_t* rocksdb_cache_create_lru(
     size_t capacity);
+extern ROCKSDB_LIBRARY_API rocksdb_cache_t*
+rocksdb_cache_create_lru_with_strict_capacity_limit(size_t capacity);
 extern ROCKSDB_LIBRARY_API rocksdb_cache_t* rocksdb_cache_create_lru_opts(
     rocksdb_lru_cache_options_t*);
 extern ROCKSDB_LIBRARY_API void rocksdb_cache_destroy(rocksdb_cache_t* cache);

options.go

@@ -579,6 +579,27 @@ func (opts *Options) GetCompressionOptionsZstdMaxTrainBytes() int {
 	return int(C.rocksdb_options_get_compression_options_zstd_max_train_bytes(opts.c))
 }
 
+// SetCompressionOptionsZstdDictTrainer uses/not use zstd trainer to generate dictionaries.
+// When this option is set to true, zstd_max_train_bytes of training data sampled from
+// max_dict_buffer_bytes buffered data will be passed to zstd dictionary trainer to generate a
+// dictionary of size max_dict_bytes.
+//
+// When this option is false, zstd's API ZDICT_finalizeDictionary() will be
+// called to generate dictionaries. zstd_max_train_bytes of training sampled
+// data will be passed to this API. Using this API should save CPU time on
+// dictionary training, but the compression ratio may not be as good as using
+// a dictionary trainer.
+//
+// Default: true
+func (opts *Options) SetCompressionOptionsZstdDictTrainer(enabled bool) {
+	C.rocksdb_options_set_compression_options_use_zstd_dict_trainer(opts.c, boolToChar(enabled))
+}
+
+// GetCompressionOptionsZstdDictTrainer returns if zstd dict trainer is used or not.
+func (opts *Options) GetCompressionOptionsZstdDictTrainer() bool {
+	return charToBool(C.rocksdb_options_get_compression_options_use_zstd_dict_trainer(opts.c))
+}
+
 // SetCompressionOptionsParallelThreads sets number of threads for
 // parallel compression. Parallel compression is enabled only if threads > 1.
 //
@@ -677,6 +698,28 @@ func (opts *Options) SetBottommostCompressionOptionsMaxDictBufferBytes(value uin
 	)
 }
 
+// // SetBottommostCompressionOptionsZstdDictTrainer uses/not use zstd trainer to generate dictionaries.
+// // When this option is set to true, zstd_max_train_bytes of training data sampled from
+// // max_dict_buffer_bytes buffered data will be passed to zstd dictionary trainer to generate a
+// // dictionary of size max_dict_bytes.
+// //
+// // When this option is false, zstd's API ZDICT_finalizeDictionary() will be
+// // called to generate dictionaries. zstd_max_train_bytes of training sampled
+// // data will be passed to this API. Using this API should save CPU time on
+// // dictionary training, but the compression ratio may not be as good as using
+// // a dictionary trainer.
+// //
+// // Default: true
+// func (opts *Options) SetBottommostCompressionOptionsZstdDictTrainer(enabled bool) {
+// 	c := boolToChar(enabled)
+// 	C.rocksdb_options_set_bottommost_compression_options_use_zstd_dict_trainer(opts.c, c, c)
+// }
+
+// // GetBottommostCompressionOptionsZstdDictTrainer returns if zstd dict trainer is used or not.
+// func (opts *Options) GetBottommostCompressionOptionsZstdDictTrainer() bool {
+// 	return charToBool(C.rocksdb_options_get_bottommost_compression_options_use_zstd_dict_trainer(opts.c))
+// }
+
 // SetMinLevelToCompress sets the start level to use compression.
 func (opts *Options) SetMinLevelToCompress(value int) {
 	C.rocksdb_options_set_min_level_to_compress(opts.c, C.int(value))

options_test.go

@@ -353,6 +353,14 @@ func TestOptions(t *testing.T) {
 	opts.SetWALCompression(LZ4Compression)
 	require.EqualValues(t, LZ4Compression, opts.GetWALCompression())
 
+	require.True(t, opts.GetCompressionOptionsZstdDictTrainer())
+	opts.SetCompressionOptionsZstdDictTrainer(false)
+	require.False(t, opts.GetCompressionOptionsZstdDictTrainer())
+
+	// require.True(t, opts.GetBottommostCompressionOptionsZstdDictTrainer())
+	// opts.SetCompressionOptionsZstdDictTrainer(false)
+	// require.False(t, opts.GetBottommostCompressionOptionsZstdDictTrainer())
+
 	// cloning
 	cl := opts.Clone()
 	require.EqualValues(t, 5, cl.GetTableCacheNumshardbits())