Merge pull request #9 from losfair/feat/gc

Garbage collection

.github/workflows/ci.yml

@@ -97,7 +97,37 @@ jobs:
           ./build/mvstore --data-plane 127.0.0.1:7000 --admin-api 127.0.0.1:7001 --metadata-prefix mvstore-test --raw-data-prefix m &
           sleep 1
           curl http://localhost:7001/api/create_namespace -d '{"key":"stress","metadata":""}'
-          RUST_LOG=error ./build/mvstore-stress --concurrency 50 --data-plane http://localhost:7000 --iterations 1000 --ns-key stress --pages 1000
+          RUST_LOG=error,mvstore_stress=info ./build/mvstore-stress --concurrency 50 --data-plane http://localhost:7000 --admin-api http://localhost:7001 --iterations 1000 --ns-key stress --pages 1000
+  stress-exotic:
+    name: mvstore stress test (exotic knobs)
+    runs-on: ubuntu-20.04
+    needs:
+    - build
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Install system dependencies
+        run: |
+          set -e
+          curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-clients_7.1.17-1_amd64.deb --output fdb-client.deb
+          sudo dpkg -i fdb-client.deb
+          curl -L https://github.com/apple/foundationdb/releases/download/7.1.17/foundationdb-server_7.1.17-1_amd64.deb --output fdb-server.deb
+          sudo dpkg -i fdb-server.deb
+      - name: Fetch binaries
+        uses: actions/download-artifact@v2
+        with:
+          name: build
+          path: ./build
+      - name: Run it
+        run: |
+          set -e
+          chmod +x ./build/mvstore ./build/mvstore-stress
+          export RUST_LOG=info
+          ./build/mvstore --data-plane 127.0.0.1:7000 --admin-api 127.0.0.1:7001 --metadata-prefix mvstore-test --raw-data-prefix m \
+            --knob-gc-scan-batch-size 50 &
+          sleep 1
+          curl http://localhost:7001/api/create_namespace -d '{"key":"stress","metadata":""}'
+          RUST_LOG=error,mvstore_stress=info ./build/mvstore-stress --concurrency 50 --data-plane http://localhost:7000 --admin-api http://localhost:7001 --iterations 1000 --ns-key stress --pages 1000
   stress-buggify:
     name: mvstore stress test (buggify)
     runs-on: ubuntu-20.04
@@ -130,13 +160,14 @@ jobs:
           sleep 1
           ./build/mvstore --data-plane 127.0.0.1:7000 --admin-api 127.0.0.1:7001 --metadata-prefix mvstore-test --raw-data-prefix m --fdb-buggify &
           sleep 1
-          RUST_LOG=error ./build/mvstore-stress --concurrency 50 --data-plane http://localhost:7000 --iterations 1000 --ns-key stress --pages 1000
+          RUST_LOG=error,mvstore_stress=info ./build/mvstore-stress --concurrency 50 --data-plane http://localhost:7000 --admin-api http://localhost:7001 --iterations 1000 --ns-key stress --pages 1000
   release:
     name: Release
     needs:
     - build
     - stress
     - stress-buggify
+    - stress-exotic
     - build-deb
     if: startsWith(github.ref, 'refs/tags/')
     runs-on: ubuntu-20.04

README.md

@@ -13,9 +13,8 @@ Distributed, MVCC SQLite that runs on top of [FoundationDB](https://github.com/a
   - [Caveats](#caveats)
     - [The "database is locked" error](#the-database-is-locked-error)
     - [No ABA-style idempotency](#no-aba-style-idempotency)
-  - [Limits](#limits)
+    - [Transaction size and time limit](#transaction-size-and-time-limit)
     - [Read latency](#read-latency)
-    - [Not yet implemented: garbage collection](#not-yet-implemented-garbage-collection)
 
 ## Features
 
@@ -128,14 +127,12 @@ This means that, in a very rare circumstance as described below:
 
 The first client will get a commit conflict and abort. This is the expected behavior, since unbounded idempotency requires too much overhead.
 
-## Limits
+### Transaction size and time limit
+
+Currently the max transaction size in mvsqlite is ~1GB and the time limit is 1 hour.
 
 ### Read latency
 
 SQLite does synchronous "disk" I/O. While we can (and do) concurrently execute write operations, reads from FoundationDB block the SQLite thread.
 
 This is probably fine if you don't expect to get very I/O intensive on a single database, but you may want to enable `coroutine` in the `IoEngine` config if you have an event loop outside, so that network I/O won't block the thread.
-
-### Not yet implemented: garbage collection
-
-Currently history versions will be kept in the database forever. There is no garbage collection yet. In a future version this will be fixed.

mvclient/src/lib.rs

@@ -468,7 +468,9 @@ async fn request_and_check_returning_status(
         };
         Ok(Some((headers, body)))
     } else if res.status().is_server_error() {
-        tracing::error!(status = %res.status(), "server error");
+        let status = res.status();
+        let text = res.text().await.unwrap_or_default();
+        tracing::error!(status = %status, text = text, "server error");
         Ok(None)
     } else {
         let status = res.status();

mvstore-stress/src/inmem.rs

@@ -4,10 +4,10 @@ use rand::{thread_rng, Rng};
 use rpds::RedBlackTreeMapSync;
 
 pub struct Inmem {
-    versions: BTreeMap<String, RedBlackTreeMapSync<u32, [u8; 32]>>,
+    pub versions: BTreeMap<String, RedBlackTreeMapSync<u32, [u8; 32]>>,
     inflight: BTreeMap<u64, RedBlackTreeMapSync<u32, [u8; 32]>>,
     next_inflight_id: u64,
-    version_list: Vec<String>,
+    pub version_list: Vec<String>,
 }
 
 impl Inmem {
@@ -49,11 +49,11 @@ impl Inmem {
         version.insert_mut(index, *hash.as_bytes());
     }
 
-    pub fn verify_page(&self, id: u64, index: u32, data: &[u8]) {
+    pub fn verify_page(&self, id: u64, index: u32, data: &[u8], desc: &str) {
         let version = self.inflight.get(&id).expect("inflight not found");
 
         if data.is_empty() {
-            assert!(version.get(&index).is_none(), "page should not exist");
+            assert!(version.get(&index).is_none(), "page should not exist ({})", desc);
         } else {
             let computed_hash = blake3::hash(data);
             let stored_hash = *version.get(&index).expect("page not found");

mvstore-stress/src/main.rs

@@ -14,6 +14,10 @@ struct Opt {
     /// Data plane URL.
     #[structopt(long)]
     data_plane: String,
+
+    /// Admin API URL.
+    #[structopt(long)]
+    admin_api: String,
 
     /// Output log in JSON format.
     #[structopt(long)]
@@ -62,7 +66,7 @@ async fn main() -> Result<()> {
         data_plane: opt.data_plane.parse()?,
         ns_key: opt.ns_key.clone(),
     })?;
-    let t = Tester::new(client.clone(), opt.pages);
+    let t = Tester::new(client.clone(), opt.admin_api.clone(), opt.pages);
     t.run(opt.concurrency as _, opt.iterations as _).await;
     println!("Test succeeded.");
     Ok(())