[core] Implement a vacuum strategy for the offline database

Enable `PRAGMA auto_vacuum = INCREMENTAL`, and perform a `PRAGMA incremental_vacuum` when deleting an offline region.

.gitignore

@@ -26,6 +26,7 @@ offline.db
 /test/fixtures/api/1.png
 /test/fixtures/api/2.png
 /test/fixtures/database/*.db
+/test/fixtures/offline/v3.db
 /test/fixtures/**/actual.png
 /test/fixtures/**/diff.png
 /test/output

platform/default/mbgl/storage/offline_database.cpp

@@ -14,9 +14,6 @@ namespace mbgl {
 
 using namespace mapbox::sqlite;
 
-// If you change the schema you must write a migration from the previous version.
-static const uint32_t schemaVersion = 2;
-
 OfflineDatabase::Statement::~Statement() {
     stmt.reset();
     stmt.clearBindings();
@@ -50,15 +47,12 @@ void OfflineDatabase::ensureSchema() {
         try {
             connect(ReadWrite);
 
-            {
-                auto userVersionStmt = db->prepare("PRAGMA user_version");
-                userVersionStmt.run();
-                switch (userVersionStmt.get<int>(0)) {
-                case 0: break; // cache-only database; ok to delete
-                case 1: break; // cache-only database; ok to delete
-                case 2: return;
-                default: throw std::runtime_error("unknown schema version");
-                }
+            switch (userVersion()) {
+            case 0: break; // cache-only database; ok to delete
+            case 1: break; // cache-only database; ok to delete
+            case 2: migrateToVersion3(); // fall through
+            case 3: return;
+            default: throw std::runtime_error("unknown schema version");
             }
 
             removeExisting();
@@ -76,8 +70,17 @@ void OfflineDatabase::ensureSchema() {
     #include "offline_schema.cpp.include"
 
     connect(ReadWrite | Create);
+
+    // If you change the schema you must write a migration from the previous version.
+    db->exec("PRAGMA auto_vacuum = INCREMENTAL");
     db->exec(schema);
-    db->exec("PRAGMA user_version = " + util::toString(schemaVersion));
+    db->exec("PRAGMA user_version = 3");
+}
+
+int OfflineDatabase::userVersion() {
+    auto stmt = db->prepare("PRAGMA user_version");
+    stmt.run();
+    return stmt.get<int>(0);
 }
 
 void OfflineDatabase::removeExisting() {
@@ -92,6 +95,11 @@ void OfflineDatabase::removeExisting() {
     }
 }
 
+void OfflineDatabase::migrateToVersion3() {
+    db->exec("PRAGMA auto_vacuum = INCREMENTAL");
+    db->exec("VACUUM");
+}
+
 OfflineDatabase::Statement OfflineDatabase::getStatement(const char * sql) {
     auto it = statements.find(sql);
 
@@ -458,6 +466,7 @@ void OfflineDatabase::deleteRegion(OfflineRegion&& region) {
     stmt->run();
 
     evict(0);
+    db->exec("PRAGMA incremental_vacuum");
 
     // Ensure that the cached offlineTileCount value is recalculated.
     offlineMapboxTileCount = {};
@@ -614,10 +623,6 @@ bool OfflineDatabase::evict(uint64_t neededFreeSize) {
     uint64_t pageSize = getPragma<int64_t>("PRAGMA page_size");
     uint64_t pageCount = getPragma<int64_t>("PRAGMA page_count");
 
-    if (pageSize * pageCount > maximumCacheSize) {
-        Log::Warning(mbgl::Event::Database, "Current size is larger than the maximum size. Database won't get truncated.");
-    }
-
     auto usedSize = [&] {
         return pageSize * (pageCount - getPragma<int64_t>("PRAGMA freelist_count"));
     };

platform/default/mbgl/storage/offline_database.hpp

@@ -58,8 +58,10 @@ class OfflineDatabase : private util::noncopyable {
 
 private:
     void connect(int flags);
+    int userVersion();
     void ensureSchema();
     void removeExisting();
+    void migrateToVersion3();
 
     class Statement {
     public:

test/storage/offline_database.cpp

@@ -7,6 +7,7 @@
 #include <mbgl/util/string.hpp>
 
 #include <gtest/gtest.h>
+#include <sqlite3.hpp>
 #include <sqlite3.h>
 #include <thread>
 #include <random>
@@ -660,3 +661,30 @@ TEST(OfflineDatabase, OfflineMapboxTileCount) {
     db.deleteRegion(std::move(region1));
     EXPECT_EQ(0, db.getOfflineMapboxTileCount());
 }
+
+static int databasePageCount(const std::string& path) {
+    mapbox::sqlite::Database db(path, mapbox::sqlite::ReadOnly);
+    mapbox::sqlite::Statement stmt = db.prepare("pragma page_count");
+    stmt.run();
+    return stmt.get<int>(0);
+}
+
+TEST(OfflineDatabase, MigrateFromV2Schema) {
+    using namespace mbgl;
+
+    // v2.db is a v2 database containing a single offline region with a small number of resources.
+
+    deleteFile("test/fixtures/offline/v3.db");
+    writeFile("test/fixtures/offline/v3.db", util::read_file("test/fixtures/offline/v2.db"));
+
+    {
+        OfflineDatabase db("test/fixtures/offline/v3.db", 0);
+        auto regions = db.listRegions();
+        for (auto& region : regions) {
+            db.deleteRegion(std::move(region));
+        }
+    }
+
+    EXPECT_LT(databasePageCount("test/fixtures/offline/v3.db"),
+              databasePageCount("test/fixtures/offline/v2.db"));
+}