forked from google/leveldb
-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Closes google#320 During compaction it was possible that records from a block b1=(l1,u1) would be pushed down from level i to level i+1. If there is a block b2=(l2,u2) at level i with k1 = user_key(u1) = user_key(l2) then a subsequent search for k1 will yield the record l2 which has a smaller sequence number than u1 because the sort order for records sorts increasing by user key but decreaing by sequence number. This change add a call to a new function AddBoundaryInputs to SetupOtherInputs. AddBoundaryInputs searches for a block b2 matching the criteria above and adds it to the set of files to be compacted. Whenever AddBoundaryInputs is called it is important that the compaction fileset in level i+1 (known as c->inputs_[1] in the code) be recomputed. Each call to AddBoundaryInputs is followed by a call to GetOverlappingInputs. SetupOtherInputs is called on both manual and automated compaction passes. It is called for both level zero and for levels greater than 0. Testing Done: A test program (issue320_test) was constructed that performs mutations while snapshots are active. issue320_test fails without this bug fix after 64k writes. It passes with this bug fix. It was run with 200M writes and passed. Unit tests were written for the new function that was added to the code. Make check was run and seen to pass. Signed-off-by: Richard Cole <richcole@amazon.com>
- Loading branch information
1 parent
2d0320a
commit cedb600
Showing
4 changed files
with
340 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,130 @@ | ||
| #include <iostream> | ||
| #include <sstream> | ||
| #include <string> | ||
| #include <map> | ||
| #include <vector> | ||
| #include <memory> | ||
|
|
||
| #include <math.h> | ||
|
|
||
| #include <leveldb/db.h> | ||
| #include <leveldb/write_batch.h> | ||
|
|
||
| using namespace std; | ||
|
|
||
| unsigned int random(unsigned int max) { | ||
| return rand() % max; | ||
| } | ||
|
|
||
| string newString(int32_t index) { | ||
| const unsigned int len = 1024; | ||
| char bytes[len]; | ||
| unsigned int i = 0; | ||
| while (i < 8) { | ||
| bytes[i] = 'a' + ((index >> (4 * i)) & 0xf); | ||
| ++i; | ||
| } | ||
| while (i < sizeof(bytes)) { | ||
| bytes[i] = 'a' + random(26); | ||
| ++i; | ||
| } | ||
| return string(bytes, sizeof(bytes)); | ||
| } | ||
|
|
||
| int main(int argc, char** argv) { | ||
| srandom(0); | ||
|
|
||
| bool delete_before_put = false; | ||
| bool keep_snapshots = true; | ||
|
|
||
| vector<pair<string, string> *> test_map(10000, NULL); | ||
| vector<leveldb::Snapshot const*> snapshots(100, NULL); | ||
|
|
||
| leveldb::DB* db; | ||
| leveldb::Options options; | ||
| options.create_if_missing = true; | ||
| int system_status; | ||
|
|
||
| system_status = system("rm -rf ./issue320_test_db"); | ||
| leveldb::Status status = leveldb::DB::Open(options, "./issue320_test_db", | ||
| &db); | ||
|
|
||
| if (false == status.ok()) { | ||
| cerr << "Unable to open/create test database './issue320_test_db'" << endl; | ||
| cerr << status.ToString() << endl; | ||
| return -1; | ||
| } | ||
|
|
||
| unsigned int target_size = 10000; | ||
| unsigned int num_items = 0; | ||
| unsigned long count = 0; | ||
| string key; | ||
| string value, old_value; | ||
|
|
||
| leveldb::WriteOptions writeOptions; | ||
| leveldb::ReadOptions readOptions; | ||
| while (count < 200000) { | ||
| if ((++count % 1000) == 0) { | ||
| cout << "count: " << count << endl; | ||
| } | ||
|
|
||
| unsigned int index = random(test_map.size()); | ||
| leveldb::WriteBatch batch; | ||
|
|
||
| if (test_map[index] == NULL) { | ||
| num_items++; | ||
| test_map[index] = new pair<string, string>(newString(index), | ||
| newString(index)); | ||
| batch.Put(test_map[index]->first, test_map[index]->second); | ||
| } else { | ||
| if (!db->Get(readOptions, test_map[index]->first, &old_value).ok()) { | ||
| cout << "read failed" << endl; | ||
| abort(); | ||
| } | ||
| if (old_value != test_map[index]->second) { | ||
| cout << "ERROR incorrect value returned by Get" << endl; | ||
| cout << " count=" << count << endl; | ||
| cout << " old value=" << old_value << endl; | ||
| cout << " test_map[index]->second=" << test_map[index]->second << endl; | ||
| cout << " test_map[index]->first=" << test_map[index]->first << endl; | ||
| cout << " index=" << index << endl; | ||
| exit(-1); | ||
| } | ||
|
|
||
| if (num_items >= target_size && random(100) > 30) { | ||
| batch.Delete(test_map[index]->first); | ||
| delete test_map[index]; | ||
| test_map[index] = NULL; | ||
| --num_items; | ||
| } else { | ||
| test_map[index]->second = newString(index); | ||
| if (delete_before_put) | ||
| batch.Delete(test_map[index]->first); | ||
| batch.Put(test_map[index]->first, test_map[index]->second); | ||
| } | ||
| } | ||
|
|
||
| if (!db->Write(writeOptions, &batch).ok()) { | ||
| cout << "write failed" << endl; | ||
| abort(); | ||
| } | ||
|
|
||
| if (keep_snapshots && random(10) == 0) { | ||
| unsigned int i = random(snapshots.size()); | ||
| if (snapshots[i] != NULL) { | ||
| db->ReleaseSnapshot(snapshots[i]); | ||
| } | ||
| snapshots[i] = db->GetSnapshot(); | ||
| } | ||
| } | ||
|
|
||
| for (size_t i = 0; i < test_map.size(); ++i) { | ||
| if (test_map[i] != NULL) { | ||
| delete test_map[i]; | ||
| test_map[i] = NULL; | ||
| } | ||
| } | ||
|
|
||
| system_status = system("rm -rf ./issue320_test_db"); | ||
| return 0; | ||
| } |