Adapt rocksdb 6.7.3 (#14)

build.sh

@@ -28,10 +28,10 @@ cd $BUILD_PATH && wget https://github.com/facebook/zstd/releases/download/v1.4.4
     $CMAKE_REQUIRED_PARAMS -DZSTD_ZLIB_SUPPORT=ON -DZSTD_LZMA_SUPPORT=OFF -DCMAKE_BUILD_TYPE=Release .. && make -j16 install && \
     cd $BUILD_PATH && rm -rf *
 
-cd $BUILD_PATH && wget https://github.com/facebook/rocksdb/archive/v6.6.4.tar.gz && tar xzf v6.6.4.tar.gz && cd rocksdb-6.6.4/ && \
-    cp $DIRECTORY/CMakeLists.txt ./ && \
-    mkdir -p build_place && cd build_place && cmake -DCMAKE_BUILD_TYPE=Release $CMAKE_REQUIRED_PARAMS -DCMAKE_PREFIX_PATH=$INSTALL_PREFIX -DWITH_TESTS=OFF -DWITH_BENCHMARK_TOOLS=OFF -DWITH_TOOLS=OFF \
-    -DWITH_MD_LIBRARY=OFF -DWITH_RUNTIME_DEBUG=OFF -DROCKSDB_BUILD_SHARED=OFF -DWITH_SNAPPY=ON -DWITH_LZ4=ON -DWITH_ZLIB=ON -DWITH_ZSTD=ON -DWITH_BZ2=OFF -WITH_GFLAGS=OFF .. && make -j16 install/strip && \
+cd $BUILD_PATH && wget https://github.com/facebook/rocksdb/archive/v6.7.3.tar.gz && tar xzf v6.7.3.tar.gz && cd rocksdb-6.7.3/ && \
+    mkdir -p build_place && cd build_place && cmake -DCMAKE_BUILD_TYPE=Release $CMAKE_REQUIRED_PARAMS -DCMAKE_PREFIX_PATH=$INSTALL_PREFIX -DWITH_TESTS=OFF -DWITH_GFLAGS=OFF \
+    -DWITH_BENCHMARK_TOOLS=OFF -DWITH_TOOLS=OFF -DWITH_MD_LIBRARY=OFF -DWITH_RUNTIME_DEBUG=OFF -DROCKSDB_BUILD_SHARED=OFF -DWITH_SNAPPY=ON -DWITH_LZ4=ON -DWITH_ZLIB=ON \
+    -DWITH_ZSTD=ON -DWITH_BZ2=OFF -WITH_GFLAGS=OFF .. && make -j16 install/strip && \
     cd $BUILD_PATH && rm -rf *
 
 rm -rf $INSTALL_PREFIX/bin $INSTALL_PREFIX/share $INSTALL_PREFIX/lib/cmake $INSTALL_PREFIX/lib64/cmake $INSTALL_PREFIX/lib/pkgconfig $INSTALL_PREFIX/lib64/pkgconfig

cf_test.go

@@ -134,29 +134,49 @@ func TestColumnFamilyPutGetDelete(t *testing.T) {
 	givenKey1 := []byte("hello1")
 	givenVal1 := []byte("world1")
 
-	ensure.Nil(t, db.PutCF(wo, cfh[0], givenKey0, givenVal0))
-	actualVal0, err := db.GetCF(ro, cfh[0], givenKey0)
-	defer actualVal0.Free()
-	ensure.Nil(t, err)
-	ensure.DeepEqual(t, actualVal0.Data(), givenVal0)
-
-	ensure.Nil(t, db.PutCF(wo, cfh[1], givenKey1, givenVal1))
-	actualVal1, err := db.GetCF(ro, cfh[1], givenKey1)
-	defer actualVal1.Free()
-	ensure.Nil(t, err)
-	ensure.DeepEqual(t, actualVal1.Data(), givenVal1)
-
-	actualVal, err := db.GetCF(ro, cfh[0], givenKey1)
-	ensure.Nil(t, err)
-	ensure.DeepEqual(t, actualVal.Size(), 0)
-	actualVal, err = db.GetCF(ro, cfh[1], givenKey0)
-	ensure.Nil(t, err)
-	ensure.DeepEqual(t, actualVal.Size(), 0)
+	{
+		ensure.Nil(t, db.PutCF(wo, cfh[0], givenKey0, givenVal0))
+		actualVal0, err := db.GetCF(ro, cfh[0], givenKey0)
+		defer actualVal0.Free()
+		ensure.Nil(t, err)
+		ensure.DeepEqual(t, actualVal0.Data(), givenVal0)
+
+		ensure.Nil(t, db.PutCF(wo, cfh[1], givenKey1, givenVal1))
+		actualVal1, err := db.GetCF(ro, cfh[1], givenKey1)
+		defer actualVal1.Free()
+		ensure.Nil(t, err)
+		ensure.DeepEqual(t, actualVal1.Data(), givenVal1)
+
+		actualVal, err := db.GetCF(ro, cfh[0], givenKey1)
+		ensure.Nil(t, err)
+		ensure.DeepEqual(t, actualVal.Size(), 0)
+		actualVal, err = db.GetCF(ro, cfh[1], givenKey0)
+		ensure.Nil(t, err)
+		ensure.DeepEqual(t, actualVal.Size(), 0)
+
+		ensure.Nil(t, db.DeleteCF(wo, cfh[0], givenKey0))
+		actualVal, err = db.GetCF(ro, cfh[0], givenKey0)
+		ensure.Nil(t, err)
+		ensure.DeepEqual(t, actualVal.Size(), 0)
+	}
 
-	ensure.Nil(t, db.DeleteCF(wo, cfh[0], givenKey0))
-	actualVal, err = db.GetCF(ro, cfh[0], givenKey0)
-	ensure.Nil(t, err)
-	ensure.DeepEqual(t, actualVal.Size(), 0)
+	{
+		ensure.Nil(t, db.PutCF(wo, cfh[0], givenKey0, givenVal0))
+		actualVal0, err := db.GetCF(ro, cfh[0], givenKey0)
+		defer actualVal0.Free()
+		ensure.Nil(t, err)
+		ensure.DeepEqual(t, actualVal0.Data(), givenVal0)
+
+		ensure.Nil(t, db.DeleteRangeCF(wo, cfh[0], givenKey0, givenKey1))
+		actualVal, err := db.GetCF(ro, cfh[0], givenKey0)
+		ensure.Nil(t, err)
+		ensure.DeepEqual(t, actualVal.Size(), 0)
+
+		actualVal1, err := db.GetCF(ro, cfh[1], givenKey1)
+		defer actualVal1.Free()
+		ensure.Nil(t, err)
+		ensure.DeepEqual(t, actualVal1.Data(), givenVal1)
+	}
 }
 
 func newTestDBCF(t *testing.T, name string) (db *DB, cfh []*ColumnFamilyHandle, cleanup func()) {

db.go

@@ -570,6 +570,20 @@ func (db *DB) DeleteCF(opts *WriteOptions, cf *ColumnFamilyHandle, key []byte) (
 	return
 }
 
+// DeleteRangeCF deletes keys that are between [startKey, endKey)
+func (db *DB) DeleteRangeCF(opts *WriteOptions, cf *ColumnFamilyHandle, startKey []byte, endKey []byte) (err error) {
+	var (
+		cErr      *C.char
+		cStartKey = byteToChar(startKey)
+		cEndKey   = byteToChar(endKey)
+	)
+
+	C.rocksdb_delete_range_cf(db.c, opts.c, cf.c, cStartKey, C.size_t(len(startKey)), cEndKey, C.size_t(len(endKey)), &cErr)
+	err = fromCError(cErr)
+
+	return
+}
+
 // Merge merges the data associated with the key with the actual data in the database.
 func (db *DB) Merge(opts *WriteOptions, key []byte, value []byte) (err error) {
 	var (

dist/darwin_amd64/include/rocksdb/c.h

@@ -210,23 +210,23 @@ extern ROCKSDB_LIBRARY_API void rocksdb_checkpoint_object_destroy(
 
 extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_column_families(
     const rocksdb_options_t* options, const char* name, int num_column_families,
-    const char** column_family_names,
-    const rocksdb_options_t** column_family_options,
+    const char* const* column_family_names,
+    const rocksdb_options_t* const* column_family_options,
     rocksdb_column_family_handle_t** column_family_handles, char** errptr);
 
 extern ROCKSDB_LIBRARY_API rocksdb_t*
 rocksdb_open_for_read_only_column_families(
     const rocksdb_options_t* options, const char* name, int num_column_families,
-    const char** column_family_names,
-    const rocksdb_options_t** column_family_options,
+    const char* const* column_family_names,
+    const rocksdb_options_t* const* column_family_options,
     rocksdb_column_family_handle_t** column_family_handles,
     unsigned char error_if_log_file_exist, char** errptr);
 
 extern ROCKSDB_LIBRARY_API rocksdb_t* rocksdb_open_as_secondary_column_families(
     const rocksdb_options_t* options, const char* name,
     const char* secondary_path, int num_column_families,
-    const char** column_family_names,
-    const rocksdb_options_t** column_family_options,
+    const char* const* column_family_names,
+    const rocksdb_options_t* const* column_family_options,
     rocksdb_column_family_handle_t** colummn_family_handles, char** errptr);
 
 extern ROCKSDB_LIBRARY_API char** rocksdb_list_column_families(
@@ -267,6 +267,12 @@ extern ROCKSDB_LIBRARY_API void rocksdb_delete_cf(
     rocksdb_column_family_handle_t* column_family, const char* key,
     size_t keylen, char** errptr);
 
+extern ROCKSDB_LIBRARY_API void rocksdb_delete_range_cf(
+    rocksdb_t* db, const rocksdb_writeoptions_t* options,
+    rocksdb_column_family_handle_t* column_family, const char* start_key,
+    size_t start_key_len, const char* end_key, size_t end_key_len,
+    char** errptr);
+
 extern ROCKSDB_LIBRARY_API void rocksdb_merge(
     rocksdb_t* db, const rocksdb_writeoptions_t* options, const char* key,
     size_t keylen, const char* val, size_t vallen, char** errptr);
@@ -706,6 +712,14 @@ enum {
 };
 extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_index_type(
     rocksdb_block_based_table_options_t*, int);  // uses one of the above enums
+enum {
+  rocksdb_block_based_table_data_block_index_type_binary_search = 0,
+  rocksdb_block_based_table_data_block_index_type_binary_search_and_hash = 1,
+};
+extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_data_block_index_type(
+    rocksdb_block_based_table_options_t*, int);  // uses one of the above enums
+extern ROCKSDB_LIBRARY_API void rocksdb_block_based_options_set_data_block_hash_ratio(
+    rocksdb_block_based_table_options_t* options, double v);
 extern ROCKSDB_LIBRARY_API void
 rocksdb_block_based_options_set_hash_index_allow_collision(
     rocksdb_block_based_table_options_t*, unsigned char);
@@ -1023,6 +1037,8 @@ extern ROCKSDB_LIBRARY_API void rocksdb_options_set_fifo_compaction_options(
     rocksdb_options_t* opt, rocksdb_fifo_compaction_options_t* fifo);
 extern ROCKSDB_LIBRARY_API void rocksdb_options_set_ratelimiter(
     rocksdb_options_t* opt, rocksdb_ratelimiter_t* limiter);
+extern ROCKSDB_LIBRARY_API void rocksdb_options_set_atomic_flush(
+    rocksdb_options_t* opt, unsigned char);
 
 /* RateLimiter */
 extern ROCKSDB_LIBRARY_API rocksdb_ratelimiter_t* rocksdb_ratelimiter_create(
@@ -1498,8 +1514,8 @@ extern ROCKSDB_LIBRARY_API rocksdb_transactiondb_t* rocksdb_transactiondb_open(
 rocksdb_transactiondb_t* rocksdb_transactiondb_open_column_families(
     const rocksdb_options_t* options,
     const rocksdb_transactiondb_options_t* txn_db_options, const char* name,
-    int num_column_families, const char** column_family_names,
-    const rocksdb_options_t** column_family_options,
+    int num_column_families, const char* const* column_family_names,
+    const rocksdb_options_t* const* column_family_options,
     rocksdb_column_family_handle_t** column_family_handles, char** errptr);
 
 extern ROCKSDB_LIBRARY_API const rocksdb_snapshot_t*
@@ -1647,8 +1663,8 @@ rocksdb_optimistictransactiondb_open(const rocksdb_options_t* options,
 extern ROCKSDB_LIBRARY_API rocksdb_optimistictransactiondb_t*
 rocksdb_optimistictransactiondb_open_column_families(
     const rocksdb_options_t* options, const char* name, int num_column_families,
-    const char** column_family_names,
-    const rocksdb_options_t** column_family_options,
+    const char* const* column_family_names,
+    const rocksdb_options_t* const* column_family_options,
     rocksdb_column_family_handle_t** column_family_handles, char** errptr);
 
 extern ROCKSDB_LIBRARY_API rocksdb_t*

dist/darwin_amd64/include/rocksdb/compaction_filter.h

@@ -45,6 +45,8 @@ class CompactionFilter {
     kRemoveAndSkipUntil,
   };
 
+  enum class BlobDecision { kKeep, kChangeValue, kCorruption, kIOError };
+
   // Context information of a compaction run
   struct Context {
     // Does this compaction run include all data files
@@ -173,6 +175,13 @@ class CompactionFilter {
     return Decision::kKeep;
   }
 
+  // Internal (BlobDB) use only. Do not override in application code.
+  virtual BlobDecision PrepareBlobOutput(const Slice& /* key */,
+                                         const Slice& /* existing_value */,
+                                         std::string* /* new_value */) const {
+    return BlobDecision::kKeep;
+  }
+
   // This function is deprecated. Snapshots will always be ignored for
   // compaction filters, because we realized that not ignoring snapshots doesn't
   // provide the gurantee we initially thought it would provide. Repeatable

dist/darwin_amd64/include/rocksdb/db.h

@@ -57,6 +57,7 @@ class TraceWriter;
 #ifdef ROCKSDB_LITE
 class CompactionJobInfo;
 #endif
+class FileSystem;
 
 extern const std::string kDefaultColumnFamilyName;
 extern const std::string kPersistentStatsColumnFamilyName;
@@ -715,6 +716,10 @@ class DB {
     //      timestamp of oldest unreleased snapshot.
     static const std::string kOldestSnapshotTime;
 
+    //  "rocksdb.oldest-snapshot-sequence" - returns number representing
+    //      sequence number of oldest unreleased snapshot.
+    static const std::string kOldestSnapshotSequence;
+
     //  "rocksdb.num-live-versions" - returns number of live versions. `Version`
     //      is an internal data structure. See version_set.h for details. More
     //      live versions often mean more SST files are held from being deleted,
@@ -1067,6 +1072,8 @@ class DB {
   // Get Env object from the DB
   virtual Env* GetEnv() const = 0;
 
+  virtual FileSystem* GetFileSystem() const;
+
   // Get DB Options that we use.  During the process of opening the
   // column family, the options provided when calling DB::Open() or
   // DB::CreateColumnFamily() will have been "sanitized" and transformed