From 978f7c6f09e3b89187aa15842060f291c672fd1d Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Wed, 2 Oct 2024 16:45:25 -0400 Subject: [PATCH 01/10] WIP stale Signed-off-by: Cole Miller --- include/dqlite.h | 4 ++ src/gateway.c | 83 ++++++++++++++++++++++++++---------- src/leader.c | 9 +++- src/leader.h | 3 +- test/lib/leader.h | 2 +- test/unit/test_gateway.c | 1 + test/unit/test_replication.c | 6 +-- 7 files changed, 79 insertions(+), 29 deletions(-) diff --git a/include/dqlite.h b/include/dqlite.h index d0dbe153f..df1ce9f1e 100644 --- a/include/dqlite.h +++ b/include/dqlite.h @@ -220,6 +220,10 @@ enum { DQLITE_NOMEM /* A malloc() failed */ }; +enum { + DQLITE_ALLOW_STALE = 1<<0, +}; + /** * Dqlite node handle. * diff --git a/src/gateway.c b/src/gateway.c index e827e7ca6..42d5b7e02 100644 --- a/src/gateway.c +++ b/src/gateway.c @@ -126,12 +126,6 @@ void gateway__close(struct gateway *g) } \ } -#define CHECK_LEADER(REQ) \ - if (raft_state(g->raft) != RAFT_LEADER) { \ - failure(REQ, SQLITE_IOERR_NOT_LEADER, "not leader"); \ - return 0; \ - } - #define SUCCESS(LOWER, UPPER, RESP, SCHEMA) \ { \ size_t _n = response_##LOWER##__sizeof(&RESP); \ @@ -206,6 +200,26 @@ static void failure(struct handle *req, int code, const char *message) req->cb(req, 0, DQLITE_RESPONSE_FAILURE, 0); } +static bool check_leader_weak(const struct gateway *g, struct handle *req) +{ + const struct leader *l = g->leader; + bool ok = raft_state(g->raft) == RAFT_LEADER || + (l != NULL && (l->flags & DQLITE_ALLOW_STALE)); + if (!ok) { + failure(req, SQLITE_IOERR_NOT_LEADER, "not leader"); + } + return ok; +} + +static bool check_leader_strong(const struct gateway *g, struct handle *req) +{ + bool ok = raft_state(g->raft) == RAFT_LEADER; + if (!ok) { + failure(req, SQLITE_IOERR_NOT_LEADER, "not leader"); + } + return ok; +} + static void emptyRows(struct handle *req) { char *cursor = buffer__advance(req->buffer, 8 + 8); @@ -297,7 +311,7 @@ static int handle_open(struct gateway *g, struct handle *req) tracef("malloc failed"); return DQLITE_NOMEM; } - rc = leader__init(g->leader, db, g->raft); + rc = leader_init(g->leader, db, request.flags, g->raft); if (rc != 0) { tracef("leader init failed %d", rc); sqlite3_free(g->leader); @@ -399,7 +413,9 @@ static int handle_prepare(struct gateway *g, struct handle *req) return rc; } - CHECK_LEADER(req); + if (!check_leader_weak(g, req)) { + return 0; + } LOOKUP_DB(request.db_id); rc = stmt__registry_add(&g->stmts, &stmt); if (rc != 0) { @@ -501,7 +517,9 @@ static int handle_exec(struct gateway *g, struct handle *req) return rv; } - CHECK_LEADER(req); + if (!check_leader_strong(g, req)) { + return 0; + } LOOKUP_DB(request.db_id); LOOKUP_STMT(request.stmt_id); FAIL_IF_CHECKPOINTING; @@ -621,7 +639,7 @@ static int handle_query(struct gateway *g, struct handle *req) struct stmt *stmt; struct request_query request = { 0 }; int tuple_format; - bool is_readonly; + bool readonly; uint64_t req_id; int rv; @@ -645,10 +663,16 @@ static int handle_query(struct gateway *g, struct handle *req) return rv; } - CHECK_LEADER(req); + if (!check_leader_weak(g, req)) { + return 0; + } LOOKUP_DB(request.db_id); LOOKUP_STMT(request.stmt_id); FAIL_IF_CHECKPOINTING; + readonly = (bool)sqlite3_stmt_readonly(stmt->stmt); + if (!(readonly || check_leader_strong(g, req))) { + return 0; + } rv = bind__params(stmt->stmt, cursor, tuple_format); if (rv != 0) { tracef("handle query bind failed %d", rv); @@ -658,8 +682,7 @@ static int handle_query(struct gateway *g, struct handle *req) req->stmt_id = stmt->id; g->req = req; - is_readonly = (bool)sqlite3_stmt_readonly(stmt->stmt); - if (is_readonly) { + if (readonly) { rv = leader__barrier(g->leader, &g->barrier, query_barrier_cb); } else { req_id = idNext(&g->random_state); @@ -826,7 +849,9 @@ static int handle_exec_sql(struct gateway *g, struct handle *req) return rc; } - CHECK_LEADER(req); + if (!check_leader_strong(g, req)) { + return 0; + } LOOKUP_DB(request.db_id); FAIL_IF_CHECKPOINTING; req->sql = request.sql; @@ -873,7 +898,7 @@ static void querySqlBarrierCb(struct barrier *barrier, int status) const char *tail; sqlite3_stmt *tail_stmt; int tuple_format; - bool is_readonly; + bool readonly; uint64_t req_id; int rv; @@ -902,7 +927,10 @@ static void querySqlBarrierCb(struct barrier *barrier, int status) failure(req, SQLITE_ERROR, "nonempty statement tail"); return; } - + readonly = (bool)sqlite3_stmt_readonly(stmt); + if (!(readonly || check_leader_strong(g, req))) { + return; + } switch (req->schema) { case DQLITE_REQUEST_PARAMS_SCHEMA_V0: tuple_format = TUPLE__PARAMS; @@ -925,8 +953,7 @@ static void querySqlBarrierCb(struct barrier *barrier, int status) req->stmt = stmt; g->req = req; - is_readonly = (bool)sqlite3_stmt_readonly(stmt); - if (is_readonly) { + if (readonly) { query_batch(g); } else { req_id = idNext(&g->random_state); @@ -959,7 +986,9 @@ static int handle_query_sql(struct gateway *g, struct handle *req) return rv; } - CHECK_LEADER(req); + if (!check_leader_weak(g, req)) { + return 0; + } LOOKUP_DB(request.db_id); FAIL_IF_CHECKPOINTING; req->sql = request.sql; @@ -1021,7 +1050,9 @@ static int handle_add(struct gateway *g, struct handle *req) START_V0(add, empty); (void)response; - CHECK_LEADER(req); + if (!check_leader_strong(g, req)) { + return 0; + } r = sqlite3_malloc(sizeof *r); if (r == NULL) { @@ -1057,7 +1088,9 @@ static int handle_promote_or_assign(struct gateway *g, struct handle *req) START_V0(promote_or_assign, empty); (void)response; - CHECK_LEADER(req); + if (!check_leader_strong(g, req)) { + return 0; + } /* Detect if this is an assign role request, instead of the former * promote request. */ @@ -1103,7 +1136,9 @@ static int handle_remove(struct gateway *g, struct handle *req) START_V0(remove, empty); (void)response; - CHECK_LEADER(req); + if (!check_leader_strong(g, req)) { + return 0; + } r = sqlite3_malloc(sizeof *r); if (r == NULL) { @@ -1359,7 +1394,9 @@ static int handle_transfer(struct gateway *g, struct handle *req) START_V0(transfer, empty); (void)response; - CHECK_LEADER(req); + if (!check_leader_strong(g, req)) { + return 0; + } r = sqlite3_malloc(sizeof *r); if (r == NULL) { diff --git a/src/leader.c b/src/leader.c index 4f1e4b5e4..41fac2685 100644 --- a/src/leader.c +++ b/src/leader.c @@ -127,11 +127,15 @@ static bool needsBarrier(struct leader *l) raft_last_applied(l->raft) < raft_last_index(l->raft); } -int leader__init(struct leader *l, struct db *db, struct raft *raft) +int leader_init(struct leader *l, + struct db *db, + uint64_t flags, + struct raft *raft) { tracef("leader init"); int rc; l->db = db; + l->flags = flags; l->raft = raft; rc = openConnection(db->path, db->config->name, db->config->page_size, &l->conn); @@ -482,6 +486,9 @@ int leader__barrier(struct leader *l, struct barrier *barrier, barrier_cb cb) tracef("not needed"); cb(barrier, 0); return 0; + } else if (l->flags & DQLITE_ALLOW_STALE) { + cb(barrier, 0); + return 0; } barrier->cb = cb; barrier->leader = l; diff --git a/src/leader.h b/src/leader.h index 9d022d3e9..c77c26486 100644 --- a/src/leader.h +++ b/src/leader.h @@ -38,6 +38,7 @@ struct apply { struct leader { struct db *db; /* Database the connection. */ + uint64_t flags; sqlite3 *conn; /* Underlying SQLite connection. */ struct raft *raft; /* Raft instance. */ struct exec *exec; /* Exec request in progress, if any. */ @@ -74,7 +75,7 @@ struct exec { * transfering control back to main coroutine and then opening a new leader * connection against the given database. */ -int leader__init(struct leader *l, struct db *db, struct raft *raft); +int leader_init(struct leader *l, struct db *db, uint64_t flags, struct raft *raft); void leader__close(struct leader *l); diff --git a/test/lib/leader.h b/test/lib/leader.h index 17d8709ad..38828abd5 100644 --- a/test/lib/leader.h +++ b/test/lib/leader.h @@ -15,7 +15,7 @@ int rv; \ rv = registry__db_get(&f->registry, "test.db", &db); \ munit_assert_int(rv, ==, 0); \ - rv = leader__init(&f->leader, db, &f->raft); \ + rv = leader_init(&f->leader, db, 0, &f->raft); \ munit_assert_int(rv, ==, 0); \ } #define TEAR_DOWN_LEADER leader__close(&f->leader) diff --git a/test/unit/test_gateway.c b/test/unit/test_gateway.c index d0e1fb7c6..b87a0f37a 100644 --- a/test/unit/test_gateway.c +++ b/test/unit/test_gateway.c @@ -180,6 +180,7 @@ static void handleCb(struct handle *req, { \ struct request_open open; \ open.filename = "test"; \ + open.flags = 0; \ open.vfs = ""; \ ENCODE(&open, open); \ HANDLE(OPEN); \ diff --git a/test/unit/test_replication.c b/test/unit/test_replication.c index 00fdd9ba7..02f3134f3 100644 --- a/test/unit/test_replication.c +++ b/test/unit/test_replication.c @@ -34,7 +34,7 @@ TEST_MODULE(replication_v1); int rc2; \ rc2 = registry__db_get(registry, "test.db", &db); \ munit_assert_int(rc2, ==, 0); \ - rc2 = leader__init(leader, db, CLUSTER_RAFT(I)); \ + rc2 = leader_init(leader, db, 0, CLUSTER_RAFT(I)); \ munit_assert_int(rc2, ==, 0); \ } while (0) @@ -130,7 +130,7 @@ TEST_MODULE(replication_v1); /****************************************************************************** * - * leader__init + * leader_init * ******************************************************************************/ @@ -286,7 +286,7 @@ TEST_CASE(exec, checkpoint_read_lock, NULL) /* Initialize another leader. */ rv = registry__db_get(registry, "test.db", &db); munit_assert_int(rv, ==, 0); - leader__init(&leader2, db, CLUSTER_RAFT(0)); + leader_init(&leader2, db, 0, CLUSTER_RAFT(0)); /* Start a read transaction in the other leader. */ rv = sqlite3_exec(leader2.conn, "BEGIN", NULL, NULL, &errmsg); From 923d4a96e8cedf15a1630b03aa78f2e9070fb630 Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Wed, 2 Oct 2024 17:37:12 -0400 Subject: [PATCH 02/10] Fix Signed-off-by: Cole Miller --- src/gateway.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/gateway.c b/src/gateway.c index 42d5b7e02..a0bd92d03 100644 --- a/src/gateway.c +++ b/src/gateway.c @@ -929,6 +929,7 @@ static void querySqlBarrierCb(struct barrier *barrier, int status) } readonly = (bool)sqlite3_stmt_readonly(stmt); if (!(readonly || check_leader_strong(g, req))) { + sqlite3_finalize(stmt); return; } switch (req->schema) { From 8806b1bfa3c0fcc398b8409e5b44b50b698a65e1 Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Wed, 2 Oct 2024 17:37:31 -0400 Subject: [PATCH 03/10] Tests Signed-off-by: Cole Miller --- test/unit/test_gateway.c | 168 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) diff --git a/test/unit/test_gateway.c b/test/unit/test_gateway.c index b87a0f37a..54da44d45 100644 --- a/test/unit/test_gateway.c +++ b/test/unit/test_gateway.c @@ -187,6 +187,17 @@ static void handleCb(struct handle *req, ASSERT_CALLBACK(0, DB); \ } +#define OPEN_ALLOWSTALE \ + { \ + struct request_open open; \ + open.filename = "test"; \ + open.flags = DQLITE_ALLOW_STALE; \ + open.vfs = ""; \ + ENCODE(&open, open); \ + HANDLE(OPEN); \ + ASSERT_CALLBACK(0, DB); \ + } + /* Prepare a statement. The ID will be saved in stmt_id. */ #define PREPARE(SQL) \ { \ @@ -584,6 +595,7 @@ TEST_CASE(prepare, non_leader, NULL) CLUSTER_ELECT(0); SELECT(1); + OPEN; f->request.db_id = 0; f->request.sql = "CREATE TABLE test (n INT)"; ENCODE(&f->request, prepare); @@ -673,6 +685,24 @@ TEST_CASE(prepare, nonempty_tail_v1, NULL) return MUNIT_OK; } +/* Successfully prepare a statement on a non-leader when this is requested. */ +TEST_CASE(prepare, non_leader_ok, NULL) +{ + struct prepare_fixture *f = data; + (void)params; + + CLUSTER_ELECT(0); + SELECT(1); + OPEN_ALLOWSTALE; + f->request.db_id = 0; + f->request.sql = "CREATE TABLE test (n INT)"; + ENCODE(&f->request, prepare); + HANDLE(PREPARE); + WAIT; + ASSERT_CALLBACK(0, STMT); + return MUNIT_OK; +} + /****************************************************************************** * * exec @@ -1198,6 +1228,25 @@ TEST_CASE(exec, unexpectedRow, NULL) return MUNIT_OK; } +TEST_CASE(exec, not_leader_never_okay, NULL) +{ + struct exec_fixture *f = data; + uint64_t stmt_id; + (void)params; + CLUSTER_ELECT(0); + SELECT(1); + OPEN_ALLOWSTALE; + PREPARE("CREATE TABLE test (n INT)"); + f->request.db_id = 0; + f->request.stmt_id = stmt_id; + ENCODE(&f->request, exec); + HANDLE(EXEC); + WAIT; + ASSERT_CALLBACK(0, FAILURE); + ASSERT_FAILURE(SQLITE_IOERR_NOT_LEADER, "not leader"); + return MUNIT_OK; +} + /****************************************************************************** * * query @@ -1682,6 +1731,51 @@ TEST_CASE(query, close_while_in_flight, NULL) return MUNIT_OK; } +TEST_CASE(query, allow_stale, NULL) +{ + (void)params; + struct query_fixture *f = data; + uint64_t stmt_id; + uint64_t n; + const char *column; + + CLUSTER_APPLIED(4); + SELECT(1); + OPEN_ALLOWSTALE; + PREPARE("SELECT n FROM test"); + f->request.db_id = 0; + f->request.stmt_id = stmt_id; + ENCODE(&f->request, query); + HANDLE(QUERY); + ASSERT_CALLBACK(0, ROWS); + uint64__decode(f->cursor, &n); + munit_assert_int(n, ==, 1); + text__decode(f->cursor, &column); + munit_assert_string_equal(column, "n"); + DECODE(&f->response, rows); + munit_assert_ullong(f->response.eof, ==, DQLITE_RESPONSE_ROWS_DONE); + return MUNIT_OK; +} + +TEST_CASE(query, allow_stale_modifying, NULL) +{ + (void)params; + struct query_fixture *f = data; + uint64_t stmt_id; + + CLUSTER_APPLIED(4); + SELECT(1); + OPEN_ALLOWSTALE; + PREPARE("INSERT INTO test (n) VALUES (17)"); + f->request.db_id = 0; + f->request.stmt_id = stmt_id; + ENCODE(&f->request, query); + HANDLE(QUERY); + ASSERT_CALLBACK(0, FAILURE); + ASSERT_FAILURE(SQLITE_IOERR_NOT_LEADER, "not leader"); + return MUNIT_OK; +} + /****************************************************************************** * * finalize @@ -1725,6 +1819,23 @@ TEST_CASE(finalize, success, NULL) return MUNIT_OK; } +TEST_CASE(finalize, not_leader, NULL) +{ + uint64_t stmt_id; + struct finalize_fixture *f = data; + (void)params; + CLUSTER_ELECT(0); + SELECT(1); + OPEN_ALLOWSTALE; + PREPARE("CREATE TABLE test (n INT)"); + f->request.db_id = 0; + f->request.stmt_id = stmt_id; + ENCODE(&f->request, finalize); + HANDLE(FINALIZE); + ASSERT_CALLBACK(0, EMPTY); + return MUNIT_OK; +} + /****************************************************************************** * * exec_sql @@ -1915,6 +2026,22 @@ TEST_CASE(exec_sql, manyParams, NULL) return MUNIT_OK; } +TEST_CASE(exec_sql, not_leader_never_ok, NULL) +{ + struct exec_sql_fixture *f = data; + (void)params; + + SELECT(1); + OPEN_ALLOWSTALE; + f->request.db_id = 0; + f->request.sql = "CREATE TABLE test (n INT)"; + ENCODE(&f->request, exec_sql); + HANDLE(EXEC_SQL); + ASSERT_CALLBACK(0, FAILURE); + ASSERT_FAILURE(SQLITE_IOERR_NOT_LEADER, "not leader"); + return MUNIT_OK; +} + /****************************************************************************** * * query_sql @@ -2302,6 +2429,47 @@ TEST_CASE(query_sql, nonemptyTail, NULL) return MUNIT_OK; } +TEST_CASE(query_sql, allow_stale, NULL) +{ + (void)params; + struct query_sql_fixture *f = data; + uint64_t n; + const char *column; + + CLUSTER_APPLIED(4); + SELECT(1); + OPEN_ALLOWSTALE; + f->request.db_id = 0; + f->request.sql = "SELECT * FROM test"; + ENCODE(&f->request, query_sql); + HANDLE(QUERY_SQL); + ASSERT_CALLBACK(0, ROWS); + uint64__decode(f->cursor, &n); + munit_assert_int(n, ==, 1); + text__decode(f->cursor, &column); + munit_assert_string_equal(column, "n"); + DECODE(&f->response, rows); + munit_assert_ullong(f->response.eof, ==, DQLITE_RESPONSE_ROWS_DONE); + return MUNIT_OK; +} + +TEST_CASE(query_sql, allow_stale_modifying, NULL) +{ + (void)params; + struct query_sql_fixture *f = data; + + CLUSTER_APPLIED(4); + SELECT(1); + OPEN_ALLOWSTALE; + f->request.db_id = 0; + f->request.sql = "INSERT INTO test (n) VALUES (17)"; + ENCODE(&f->request, query_sql); + HANDLE(QUERY_SQL); + ASSERT_CALLBACK(0, FAILURE); + ASSERT_FAILURE(SQLITE_IOERR_NOT_LEADER, "not leader"); + return MUNIT_OK; +} + /****************************************************************************** * * cluster From a7e01e2b054d5df3fb1b010a056a5256281c48c0 Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Tue, 8 Oct 2024 12:35:00 -0400 Subject: [PATCH 04/10] No side effects in if predicates Signed-off-by: Cole Miller --- src/gateway.c | 61 +++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 44 insertions(+), 17 deletions(-) diff --git a/src/gateway.c b/src/gateway.c index a0bd92d03..970d48635 100644 --- a/src/gateway.c +++ b/src/gateway.c @@ -401,6 +401,7 @@ static int handle_prepare(struct gateway *g, struct handle *req) struct cursor *cursor = &req->cursor; struct stmt *stmt; struct request_prepare request = { 0 }; + bool ok; int rc; if (req->schema != DQLITE_PREPARE_STMT_SCHEMA_V0 && @@ -413,7 +414,8 @@ static int handle_prepare(struct gateway *g, struct handle *req) return rc; } - if (!check_leader_weak(g, req)) { + ok = check_leader_weak(g, req); + if (!ok) { return 0; } LOOKUP_DB(request.db_id); @@ -495,6 +497,7 @@ static int handle_exec(struct gateway *g, struct handle *req) struct request_exec request = { 0 }; int tuple_format; uint64_t req_id; + bool ok; int rv; switch (req->schema) { @@ -517,7 +520,8 @@ static int handle_exec(struct gateway *g, struct handle *req) return rv; } - if (!check_leader_strong(g, req)) { + ok = check_leader_strong(g, req); + if (!ok) { return 0; } LOOKUP_DB(request.db_id); @@ -639,7 +643,7 @@ static int handle_query(struct gateway *g, struct handle *req) struct stmt *stmt; struct request_query request = { 0 }; int tuple_format; - bool readonly; + bool ok; uint64_t req_id; int rv; @@ -663,15 +667,20 @@ static int handle_query(struct gateway *g, struct handle *req) return rv; } - if (!check_leader_weak(g, req)) { + + ok = check_leader_weak(g, req); + if (!ok) { return 0; } LOOKUP_DB(request.db_id); LOOKUP_STMT(request.stmt_id); FAIL_IF_CHECKPOINTING; readonly = (bool)sqlite3_stmt_readonly(stmt->stmt); - if (!(readonly || check_leader_strong(g, req))) { - return 0; + if (!sqlite3_stmt_readonly(stmt->stmt)) { + ok = check_leader_strong(g, req); + if (!ok) { + return 0; + } } rv = bind__params(stmt->stmt, cursor, tuple_format); if (rv != 0) { @@ -833,6 +842,7 @@ static int handle_exec_sql(struct gateway *g, struct handle *req) tracef("handle exec sql schema:%" PRIu8, req->schema); struct cursor *cursor = &req->cursor; struct request_exec_sql request = { 0 }; + bool ok; int rc; /* Fail early if the schema version isn't recognized, even though we @@ -849,7 +859,8 @@ static int handle_exec_sql(struct gateway *g, struct handle *req) return rc; } - if (!check_leader_strong(g, req)) { + ok = check_leader_strong(g, req); + if (!ok) { return 0; } LOOKUP_DB(request.db_id); @@ -898,7 +909,7 @@ static void querySqlBarrierCb(struct barrier *barrier, int status) const char *tail; sqlite3_stmt *tail_stmt; int tuple_format; - bool readonly; + bool ok; uint64_t req_id; int rv; @@ -927,10 +938,12 @@ static void querySqlBarrierCb(struct barrier *barrier, int status) failure(req, SQLITE_ERROR, "nonempty statement tail"); return; } - readonly = (bool)sqlite3_stmt_readonly(stmt); - if (!(readonly || check_leader_strong(g, req))) { - sqlite3_finalize(stmt); - return; + if (!sqlite3_stmt_readonly(stmt)) { + ok = check_leader_strong(g, req); + if (!ok) { + sqlite3_finalize(stmt); + return; + } } switch (req->schema) { case DQLITE_REQUEST_PARAMS_SCHEMA_V0: @@ -973,6 +986,7 @@ static int handle_query_sql(struct gateway *g, struct handle *req) tracef("handle query sql schema:%" PRIu8, req->schema); struct cursor *cursor = &req->cursor; struct request_query_sql request = { 0 }; + bool ok; int rv; /* Fail early if the schema version isn't recognized. */ @@ -987,7 +1001,8 @@ static int handle_query_sql(struct gateway *g, struct handle *req) return rv; } - if (!check_leader_weak(g, req)) { + ok = check_leader_weak(g, req); + if (!ok) { return 0; } LOOKUP_DB(request.db_id); @@ -1047,11 +1062,14 @@ static int handle_add(struct gateway *g, struct handle *req) struct cursor *cursor = &req->cursor; struct change *r; uint64_t req_id; + bool ok; int rv; + START_V0(add, empty); (void)response; - if (!check_leader_strong(g, req)) { + ok = check_leader_strong(g, req); + if (!ok) { return 0; } @@ -1085,11 +1103,14 @@ static int handle_promote_or_assign(struct gateway *g, struct handle *req) struct change *r; uint64_t role = DQLITE_VOTER; uint64_t req_id; + bool ok; int rv; + START_V0(promote_or_assign, empty); (void)response; - if (!check_leader_strong(g, req)) { + ok = check_leader_strong(g, req); + if (!ok) { return 0; } @@ -1133,11 +1154,14 @@ static int handle_remove(struct gateway *g, struct handle *req) struct cursor *cursor = &req->cursor; struct change *r; uint64_t req_id; + bool ok; int rv; + START_V0(remove, empty); (void)response; - if (!check_leader_strong(g, req)) { + ok = check_leader_strong(g, req); + if (!ok) { return 0; } @@ -1391,11 +1415,14 @@ static int handle_transfer(struct gateway *g, struct handle *req) tracef("handle transfer"); struct cursor *cursor = &req->cursor; struct raft_transfer *r; + bool ok; int rv; + START_V0(transfer, empty); (void)response; - if (!check_leader_strong(g, req)) { + ok = check_leader_strong(g, req); + if (!ok) { return 0; } From 2c8b76dad5e9c795be6d9ef0d6605e75854e9912 Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Tue, 8 Oct 2024 12:42:48 -0400 Subject: [PATCH 05/10] Cleanup Signed-off-by: Cole Miller --- include/dqlite.h | 4 ---- src/gateway.c | 11 +++++++---- src/leader.c | 2 +- src/protocol.h | 10 ++++++++++ test/unit/test_gateway.c | 20 ++++++++++---------- 5 files changed, 28 insertions(+), 19 deletions(-) diff --git a/include/dqlite.h b/include/dqlite.h index df1ce9f1e..d0dbe153f 100644 --- a/include/dqlite.h +++ b/include/dqlite.h @@ -220,10 +220,6 @@ enum { DQLITE_NOMEM /* A malloc() failed */ }; -enum { - DQLITE_ALLOW_STALE = 1<<0, -}; - /** * Dqlite node handle. * diff --git a/src/gateway.c b/src/gateway.c index 970d48635..06e1001c3 100644 --- a/src/gateway.c +++ b/src/gateway.c @@ -204,7 +204,7 @@ static bool check_leader_weak(const struct gateway *g, struct handle *req) { const struct leader *l = g->leader; bool ok = raft_state(g->raft) == RAFT_LEADER || - (l != NULL && (l->flags & DQLITE_ALLOW_STALE)); + (l != NULL && (l->flags & DQLITE_OPEN_ALLOW_STALE)); if (!ok) { failure(req, SQLITE_IOERR_NOT_LEADER, "not leader"); } @@ -643,6 +643,7 @@ static int handle_query(struct gateway *g, struct handle *req) struct stmt *stmt; struct request_query request = { 0 }; int tuple_format; + bool readonly; bool ok; uint64_t req_id; int rv; @@ -675,8 +676,8 @@ static int handle_query(struct gateway *g, struct handle *req) LOOKUP_DB(request.db_id); LOOKUP_STMT(request.stmt_id); FAIL_IF_CHECKPOINTING; - readonly = (bool)sqlite3_stmt_readonly(stmt->stmt); - if (!sqlite3_stmt_readonly(stmt->stmt)) { + readonly = sqlite3_stmt_readonly(stmt->stmt); + if (!readonly) { ok = check_leader_strong(g, req); if (!ok) { return 0; @@ -909,6 +910,7 @@ static void querySqlBarrierCb(struct barrier *barrier, int status) const char *tail; sqlite3_stmt *tail_stmt; int tuple_format; + bool readonly; bool ok; uint64_t req_id; int rv; @@ -938,7 +940,8 @@ static void querySqlBarrierCb(struct barrier *barrier, int status) failure(req, SQLITE_ERROR, "nonempty statement tail"); return; } - if (!sqlite3_stmt_readonly(stmt)) { + readonly = sqlite3_stmt_readonly(stmt); + if (!readonly) { ok = check_leader_strong(g, req); if (!ok) { sqlite3_finalize(stmt); diff --git a/src/leader.c b/src/leader.c index 41fac2685..e95608b48 100644 --- a/src/leader.c +++ b/src/leader.c @@ -486,7 +486,7 @@ int leader__barrier(struct leader *l, struct barrier *barrier, barrier_cb cb) tracef("not needed"); cb(barrier, 0); return 0; - } else if (l->flags & DQLITE_ALLOW_STALE) { + } else if (l->flags & DQLITE_OPEN_ALLOW_STALE) { cb(barrier, 0); return 0; } diff --git a/src/protocol.h b/src/protocol.h index 60bb2a8b3..2f6523eef 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -84,4 +84,14 @@ enum { DQLITE_RESPONSE_METADATA }; +/** + * Flags for the OPEN request. + */ +enum { + /* When set, gives permission for the server to execute readonly QUERY + * and QUERY_SQL requests against this database even if it is not the + * leader. */ + DQLITE_OPEN_ALLOW_STALE = 1 << 0, +}; + #endif /* DQLITE_PROTOCOL_H_ */ diff --git a/test/unit/test_gateway.c b/test/unit/test_gateway.c index 54da44d45..f4595cfcb 100644 --- a/test/unit/test_gateway.c +++ b/test/unit/test_gateway.c @@ -187,11 +187,11 @@ static void handleCb(struct handle *req, ASSERT_CALLBACK(0, DB); \ } -#define OPEN_ALLOWSTALE \ +#define OPEN_ALLOW_STALE \ { \ struct request_open open; \ open.filename = "test"; \ - open.flags = DQLITE_ALLOW_STALE; \ + open.flags = DQLITE_OPEN_ALLOW_STALE; \ open.vfs = ""; \ ENCODE(&open, open); \ HANDLE(OPEN); \ @@ -693,7 +693,7 @@ TEST_CASE(prepare, non_leader_ok, NULL) CLUSTER_ELECT(0); SELECT(1); - OPEN_ALLOWSTALE; + OPEN_ALLOW_STALE; f->request.db_id = 0; f->request.sql = "CREATE TABLE test (n INT)"; ENCODE(&f->request, prepare); @@ -1235,7 +1235,7 @@ TEST_CASE(exec, not_leader_never_okay, NULL) (void)params; CLUSTER_ELECT(0); SELECT(1); - OPEN_ALLOWSTALE; + OPEN_ALLOW_STALE; PREPARE("CREATE TABLE test (n INT)"); f->request.db_id = 0; f->request.stmt_id = stmt_id; @@ -1741,7 +1741,7 @@ TEST_CASE(query, allow_stale, NULL) CLUSTER_APPLIED(4); SELECT(1); - OPEN_ALLOWSTALE; + OPEN_ALLOW_STALE; PREPARE("SELECT n FROM test"); f->request.db_id = 0; f->request.stmt_id = stmt_id; @@ -1765,7 +1765,7 @@ TEST_CASE(query, allow_stale_modifying, NULL) CLUSTER_APPLIED(4); SELECT(1); - OPEN_ALLOWSTALE; + OPEN_ALLOW_STALE; PREPARE("INSERT INTO test (n) VALUES (17)"); f->request.db_id = 0; f->request.stmt_id = stmt_id; @@ -1826,7 +1826,7 @@ TEST_CASE(finalize, not_leader, NULL) (void)params; CLUSTER_ELECT(0); SELECT(1); - OPEN_ALLOWSTALE; + OPEN_ALLOW_STALE; PREPARE("CREATE TABLE test (n INT)"); f->request.db_id = 0; f->request.stmt_id = stmt_id; @@ -2032,7 +2032,7 @@ TEST_CASE(exec_sql, not_leader_never_ok, NULL) (void)params; SELECT(1); - OPEN_ALLOWSTALE; + OPEN_ALLOW_STALE; f->request.db_id = 0; f->request.sql = "CREATE TABLE test (n INT)"; ENCODE(&f->request, exec_sql); @@ -2438,7 +2438,7 @@ TEST_CASE(query_sql, allow_stale, NULL) CLUSTER_APPLIED(4); SELECT(1); - OPEN_ALLOWSTALE; + OPEN_ALLOW_STALE; f->request.db_id = 0; f->request.sql = "SELECT * FROM test"; ENCODE(&f->request, query_sql); @@ -2460,7 +2460,7 @@ TEST_CASE(query_sql, allow_stale_modifying, NULL) CLUSTER_APPLIED(4); SELECT(1); - OPEN_ALLOWSTALE; + OPEN_ALLOW_STALE; f->request.db_id = 0; f->request.sql = "INSERT INTO test (n) VALUES (17)"; ENCODE(&f->request, query_sql); From 368a4252df47e047bd5514b2ba4f04c1dd2df4eb Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Tue, 8 Oct 2024 12:49:44 -0400 Subject: [PATCH 06/10] Explain Signed-off-by: Cole Miller --- test/unit/test_gateway.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test/unit/test_gateway.c b/test/unit/test_gateway.c index f4595cfcb..bcc63e6e5 100644 --- a/test/unit/test_gateway.c +++ b/test/unit/test_gateway.c @@ -1731,6 +1731,8 @@ TEST_CASE(query, close_while_in_flight, NULL) return MUNIT_OK; } +/* A non-leader serves readonly QUERY requests for a database that was opened + * with the ALLOW_STALE flag. */ TEST_CASE(query, allow_stale, NULL) { (void)params; @@ -1757,6 +1759,8 @@ TEST_CASE(query, allow_stale, NULL) return MUNIT_OK; } +/* A non-leader will not serve a QUERY request that modifies the database, even + * if opened with the ALLOW_STALE flag. */ TEST_CASE(query, allow_stale_modifying, NULL) { (void)params; @@ -2429,6 +2433,8 @@ TEST_CASE(query_sql, nonemptyTail, NULL) return MUNIT_OK; } +/* A non-leader serves readonly QUERY_SQL requests for a database that was + * opened with the ALLOW_STALE flag. */ TEST_CASE(query_sql, allow_stale, NULL) { (void)params; @@ -2453,6 +2459,8 @@ TEST_CASE(query_sql, allow_stale, NULL) return MUNIT_OK; } +/* A non-leader will not serve a QUERY_SQL request that modifies the database, + * even if opened with the ALLOW_STALE flag. */ TEST_CASE(query_sql, allow_stale_modifying, NULL) { (void)params; From fe5cccc70821a5ec2cc5b07a3507c328a8e2d1fc Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Tue, 8 Oct 2024 17:10:26 -0400 Subject: [PATCH 07/10] Don't shut down ALLOW_STALE connections when leadership is lost Signed-off-by: Cole Miller --- src/gateway.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/gateway.c b/src/gateway.c index 06e1001c3..aa9a0e933 100644 --- a/src/gateway.c +++ b/src/gateway.c @@ -36,15 +36,22 @@ void gateway__init(struct gateway *g, g->random_state = seed; } -/* FIXME: This function becomes unsound when using the new thread pool, since - * the request callbacks will race with operations running in the pool. */ void gateway__leader_close(struct gateway *g, int reason) { + bool allow_stale; + if (g == NULL || g->leader == NULL) { tracef("gateway:%p or gateway->leader are NULL", g); return; } + /* If the client has opted into reading potentially stale data, don't + * shut down the connection unnecessarily when we lost leadership. */ + allow_stale = g->leader->flags & DQLITE_OPEN_ALLOW_STALE; + if (allow_stale && reason == RAFT_LEADERSHIPLOST) { + return; + } + if (g->req != NULL) { if (g->leader->inflight != NULL) { tracef("finish inflight apply request"); From d9fe637c344107969ad2203cf95504fd2d2b0021 Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Wed, 9 Oct 2024 15:36:22 -0400 Subject: [PATCH 08/10] WIP Signed-off-by: Cole Miller --- src/gateway.c | 188 +++++++++++++++++++++++++------------------------- 1 file changed, 94 insertions(+), 94 deletions(-) diff --git a/src/gateway.c b/src/gateway.c index aa9a0e933..ffc29ada3 100644 --- a/src/gateway.c +++ b/src/gateway.c @@ -150,15 +150,6 @@ void gateway__close(struct gateway *g) * using schema version 0. */ #define SUCCESS_V0(LOWER, UPPER) SUCCESS(LOWER, UPPER, response, 0) -/* Lookup the database with the given ID. - * - * TODO: support more than one database per connection? */ -#define LOOKUP_DB(ID) \ - if (ID != 0 || g->leader == NULL) { \ - failure(req, SQLITE_NOTFOUND, "no database opened"); \ - return 0; \ - } - /* Lookup the statement with the given ID. */ #define LOOKUP_STMT(ID) \ stmt = stmt__registry_get(&g->stmts, ID); \ @@ -207,24 +198,45 @@ static void failure(struct handle *req, int code, const char *message) req->cb(req, 0, DQLITE_RESPONSE_FAILURE, 0); } -static bool check_leader_weak(const struct gateway *g, struct handle *req) +static struct leader *gw_get_readonly(const struct gateway *g, + uint64_t db_id, + struct handle *req) { - const struct leader *l = g->leader; - bool ok = raft_state(g->raft) == RAFT_LEADER || - (l != NULL && (l->flags & DQLITE_OPEN_ALLOW_STALE)); - if (!ok) { + struct leader *l = g->leader; + bool opened_readonly; + + if (db_id != 0 || l == NULL) { + failure(req, SQLITE_NOTFOUND, "no database opened"); + return NULL; + } + opened_readonly = l->flags & DQLITE_OPEN_ALLOW_STALE; + if (raft_state(g->raft) != RAFT_LEADER && !opened_readonly) { failure(req, SQLITE_IOERR_NOT_LEADER, "not leader"); + return NULL; } - return ok; + return l; } -static bool check_leader_strong(const struct gateway *g, struct handle *req) +static struct leader *gw_get_readwrite(const struct gateway *g, + uint64_t db_id, + struct handle *req) { - bool ok = raft_state(g->raft) == RAFT_LEADER; - if (!ok) { + struct leader *l = g->leader; + bool opened_readonly; + + if (db_id != 0 || l == NULL) { + failure(req, SQLITE_NOTFOUND, "no database opened"); + return NULL; + } + opened_readonly = l->flags & DQLITE_OPEN_ALLOW_STALE; + if (raft_state(g->raft) != RAFT_LEADER) { failure(req, SQLITE_IOERR_NOT_LEADER, "not leader"); + return NULL; + } else if (opened_readonly) { + failure(req, SQLITE_READONLY, "dqlite connection is readonly"); + return NULL; } - return ok; + return l; } static void emptyRows(struct handle *req) @@ -408,7 +420,7 @@ static int handle_prepare(struct gateway *g, struct handle *req) struct cursor *cursor = &req->cursor; struct stmt *stmt; struct request_prepare request = { 0 }; - bool ok; + struct leader *l; int rc; if (req->schema != DQLITE_PREPARE_STMT_SCHEMA_V0 && @@ -421,11 +433,11 @@ static int handle_prepare(struct gateway *g, struct handle *req) return rc; } - ok = check_leader_weak(g, req); - if (!ok) { + l = gw_get_readonly(g, request.db_id, req); + if (l == NULL) { return 0; } - LOOKUP_DB(request.db_id); + rc = stmt__registry_add(&g->stmts, &stmt); if (rc != 0) { tracef("handle prepare registry add failed %d", rc); @@ -438,7 +450,7 @@ static int handle_prepare(struct gateway *g, struct handle *req) req->stmt_id = stmt->id; req->sql = request.sql; g->req = req; - rc = leader__barrier(g->leader, &g->barrier, prepareBarrierCb); + rc = leader__barrier(l, &g->barrier, prepareBarrierCb); if (rc != 0) { tracef("handle prepare barrier failed %d", rc); stmt__registry_del(&g->stmts, stmt); @@ -503,8 +515,8 @@ static int handle_exec(struct gateway *g, struct handle *req) struct stmt *stmt; struct request_exec request = { 0 }; int tuple_format; + struct leader *l; uint64_t req_id; - bool ok; int rv; switch (req->schema) { @@ -527,11 +539,11 @@ static int handle_exec(struct gateway *g, struct handle *req) return rv; } - ok = check_leader_strong(g, req); - if (!ok) { + l = gw_get_readwrite(g, request.db_id, req); + if (l == NULL) { return 0; } - LOOKUP_DB(request.db_id); + LOOKUP_STMT(request.stmt_id); FAIL_IF_CHECKPOINTING; rv = bind__params(stmt->stmt, cursor, tuple_format); @@ -543,8 +555,7 @@ static int handle_exec(struct gateway *g, struct handle *req) req->stmt_id = stmt->id; g->req = req; req_id = idNext(&g->random_state); - rv = leader__exec(g->leader, &g->exec, stmt->stmt, req_id, - leader_exec_cb); + rv = leader__exec(l, &g->exec, stmt->stmt, req_id, leader_exec_cb); if (rv != 0) { tracef("handle exec leader exec failed %d", rv); g->req = NULL; @@ -650,8 +661,9 @@ static int handle_query(struct gateway *g, struct handle *req) struct stmt *stmt; struct request_query request = { 0 }; int tuple_format; - bool readonly; - bool ok; + struct leader *l; + bool conn_readonly; + bool stmt_readonly; uint64_t req_id; int rv; @@ -675,21 +687,24 @@ static int handle_query(struct gateway *g, struct handle *req) return rv; } - - ok = check_leader_weak(g, req); - if (!ok) { - return 0; - } - LOOKUP_DB(request.db_id); - LOOKUP_STMT(request.stmt_id); - FAIL_IF_CHECKPOINTING; - readonly = sqlite3_stmt_readonly(stmt->stmt); - if (!readonly) { - ok = check_leader_strong(g, req); - if (!ok) { + conn_readonly = false; + l = gw_get_readwrite(g, request.db_id, NULL); + if (l == NULL) { + conn_readonly = true; + l = gw_get_readonly(g, request.db_id, req); + if (l == NULL) { return 0; } } + LOOKUP_STMT(request.stmt_id); + stmt_readonly = sqlite3_stmt_readonly(stmt->stmt); + if (!ERGO(conn_readonly, stmt_readonly)) { + failure(req, SQLITE_READONLY, "dqlite connection is readonly"); + return 0; + } + + FAIL_IF_CHECKPOINTING; + rv = bind__params(stmt->stmt, cursor, tuple_format); if (rv != 0) { tracef("handle query bind failed %d", rv); @@ -699,11 +714,13 @@ static int handle_query(struct gateway *g, struct handle *req) req->stmt_id = stmt->id; g->req = req; - if (readonly) { - rv = leader__barrier(g->leader, &g->barrier, query_barrier_cb); + if (conn_readonly) { + rv = eager_query_stub(); + } else if (stmt_readonly) { + rv = leader__barrier(l, &g->barrier, query_barrier_cb); } else { req_id = idNext(&g->random_state); - rv = leader__exec(g->leader, &g->exec, stmt->stmt, req_id, + rv = leader__exec(l, &g->exec, stmt->stmt, req_id, leaderModifyingQueryCb); } if (rv != 0) { @@ -717,10 +734,15 @@ static int handle_finalize(struct gateway *g, struct handle *req) { tracef("handle finalize"); struct cursor *cursor = &req->cursor; + struct leader *l; struct stmt *stmt; int rv; + START_V0(finalize, empty); - LOOKUP_DB(request.db_id); + l = gw_get_readonly(g, request.db_id, req); + if (l == NULL) { + return 0; + } LOOKUP_STMT(request.stmt_id); rv = stmt__registry_del(&g->stmts, stmt); if (rv != 0) { @@ -850,7 +872,7 @@ static int handle_exec_sql(struct gateway *g, struct handle *req) tracef("handle exec sql schema:%" PRIu8, req->schema); struct cursor *cursor = &req->cursor; struct request_exec_sql request = { 0 }; - bool ok; + struct leader *l; int rc; /* Fail early if the schema version isn't recognized, even though we @@ -867,16 +889,16 @@ static int handle_exec_sql(struct gateway *g, struct handle *req) return rc; } - ok = check_leader_strong(g, req); - if (!ok) { + l = gw_get_readwrite(g, request.db_id, req); + if (l == NULL) { return 0; } - LOOKUP_DB(request.db_id); + FAIL_IF_CHECKPOINTING; req->sql = request.sql; req->exec_count = 0; g->req = req; - rc = leader__barrier(g->leader, &g->barrier, execSqlBarrierCb); + rc = leader__barrier(l, &g->barrier, execSqlBarrierCb); if (rc != 0) { tracef("handle exec sql barrier failed %d", rc); g->req = NULL; @@ -917,8 +939,6 @@ static void querySqlBarrierCb(struct barrier *barrier, int status) const char *tail; sqlite3_stmt *tail_stmt; int tuple_format; - bool readonly; - bool ok; uint64_t req_id; int rv; @@ -947,14 +967,6 @@ static void querySqlBarrierCb(struct barrier *barrier, int status) failure(req, SQLITE_ERROR, "nonempty statement tail"); return; } - readonly = sqlite3_stmt_readonly(stmt); - if (!readonly) { - ok = check_leader_strong(g, req); - if (!ok) { - sqlite3_finalize(stmt); - return; - } - } switch (req->schema) { case DQLITE_REQUEST_PARAMS_SCHEMA_V0: tuple_format = TUPLE__PARAMS; @@ -977,7 +989,7 @@ static void querySqlBarrierCb(struct barrier *barrier, int status) req->stmt = stmt; g->req = req; - if (readonly) { + if (sqlite3_stmt_readonly(stmt)) { query_batch(g); } else { req_id = idNext(&g->random_state); @@ -996,7 +1008,8 @@ static int handle_query_sql(struct gateway *g, struct handle *req) tracef("handle query sql schema:%" PRIu8, req->schema); struct cursor *cursor = &req->cursor; struct request_query_sql request = { 0 }; - bool ok; + struct leader *l; + bool conn_readonly; int rv; /* Fail early if the schema version isn't recognized. */ @@ -1011,15 +1024,26 @@ static int handle_query_sql(struct gateway *g, struct handle *req) return rv; } - ok = check_leader_weak(g, req); - if (!ok) { - return 0; + conn_readonly = false; + l = gw_get_readwrite(g, request.db_id, NULL); + if (l == NULL) { + conn_readonly = true; + l = gw_get_readonly(g, request.db_id, req); + if (l == NULL) { + return 0; + } } - LOOKUP_DB(request.db_id); + FAIL_IF_CHECKPOINTING; + + if (conn_readonly) { + prepare_transient_and_query_readonly_stub(); + return 0; + } + req->sql = request.sql; g->req = req; - rv = leader__barrier(g->leader, &g->barrier, querySqlBarrierCb); + rv = leader__barrier(l, &g->barrier, querySqlBarrierCb); if (rv != 0) { tracef("handle query sql barrier failed %d", rv); g->req = NULL; @@ -1072,17 +1096,11 @@ static int handle_add(struct gateway *g, struct handle *req) struct cursor *cursor = &req->cursor; struct change *r; uint64_t req_id; - bool ok; int rv; START_V0(add, empty); (void)response; - ok = check_leader_strong(g, req); - if (!ok) { - return 0; - } - r = sqlite3_malloc(sizeof *r); if (r == NULL) { return DQLITE_NOMEM; @@ -1113,17 +1131,11 @@ static int handle_promote_or_assign(struct gateway *g, struct handle *req) struct change *r; uint64_t role = DQLITE_VOTER; uint64_t req_id; - bool ok; int rv; START_V0(promote_or_assign, empty); (void)response; - ok = check_leader_strong(g, req); - if (!ok) { - return 0; - } - /* Detect if this is an assign role request, instead of the former * promote request. */ if (cursor->cap > 0) { @@ -1164,17 +1176,11 @@ static int handle_remove(struct gateway *g, struct handle *req) struct cursor *cursor = &req->cursor; struct change *r; uint64_t req_id; - bool ok; int rv; START_V0(remove, empty); (void)response; - ok = check_leader_strong(g, req); - if (!ok) { - return 0; - } - r = sqlite3_malloc(sizeof *r); if (r == NULL) { tracef("malloc failed"); @@ -1425,17 +1431,11 @@ static int handle_transfer(struct gateway *g, struct handle *req) tracef("handle transfer"); struct cursor *cursor = &req->cursor; struct raft_transfer *r; - bool ok; int rv; START_V0(transfer, empty); (void)response; - ok = check_leader_strong(g, req); - if (!ok) { - return 0; - } - r = sqlite3_malloc(sizeof *r); if (r == NULL) { tracef("malloc failed"); From 48ccd7a358d774b08cd131b83e80760ff47160ae Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Tue, 15 Oct 2024 17:02:28 -0400 Subject: [PATCH 09/10] Fixes Signed-off-by: Cole Miller --- src/gateway.c | 42 +++++++++++++++++++++++++++++------- src/vfs.c | 1 - test/unit/test_concurrency.c | 1 + test/unit/test_gateway.c | 2 +- 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/src/gateway.c b/src/gateway.c index 90e918126..a07926f84 100644 --- a/src/gateway.c +++ b/src/gateway.c @@ -180,6 +180,7 @@ void gateway__close(struct gateway *g) /* Encode fa failure response and invoke the request callback */ static void failure(struct handle *req, int code, const char *message) { + tracef("failure %s", message); struct response_failure failure; size_t n; char *cursor; @@ -227,10 +228,14 @@ static struct leader *gw_get_readwrite(const struct gateway *g, } opened_readonly = l->flags & DQLITE_OPEN_ALLOW_STALE; if (raft_state(g->raft) != RAFT_LEADER) { - failure(req, SQLITE_IOERR_NOT_LEADER, "not leader"); + if (req != NULL) { + failure(req, SQLITE_IOERR_NOT_LEADER, "not leader"); + } return NULL; } else if (opened_readonly) { - failure(req, SQLITE_READONLY, "dqlite connection is readonly"); + if (req != NULL) { + failure(req, SQLITE_READONLY, "dqlite connection is readonly"); + } return NULL; } return l; @@ -418,6 +423,7 @@ static int handle_prepare(struct gateway *g, struct handle *req) struct stmt *stmt; struct request_prepare request = { 0 }; struct leader *l; + bool conn_readonly; int rc; if (req->schema != DQLITE_PREPARE_STMT_SCHEMA_V0 && @@ -430,9 +436,14 @@ static int handle_prepare(struct gateway *g, struct handle *req) return rc; } - l = gw_get_readonly(g, request.db_id, req); + conn_readonly = false; + l = gw_get_readwrite(g, request.db_id, NULL); if (l == NULL) { - return 0; + conn_readonly = true; + l = gw_get_readonly(g, request.db_id, req); + if (l == NULL) { + return 0; + } } rc = stmt__registry_add(&g->stmts, &stmt); @@ -447,6 +458,12 @@ static int handle_prepare(struct gateway *g, struct handle *req) req->stmt_id = stmt->id; req->sql = request.sql; g->req = req; + if (conn_readonly) { + /* XXX */ + g->barrier.data = g; + prepare_barrier_cb(&g->barrier, 0); + return 0; + } rc = leader_barrier_v2(g->leader, &g->barrier, prepare_barrier_cb); if (rc == LEADER_NOT_ASYNC) { prepare_barrier_cb(&g->barrier, 0); @@ -713,7 +730,10 @@ static int handle_query(struct gateway *g, struct handle *req) g->req = req; if (conn_readonly) { - rv = eager_query_stub(); + /* XXX */ + g->barrier.data = g; + query_barrier_cb(&g->barrier, 0); + rv = 0; } else if (stmt_readonly) { rv = leader_barrier_v2(g->leader, &g->barrier, query_barrier_cb); if (rv == LEADER_NOT_ASYNC) { @@ -949,6 +969,7 @@ static void query_sql_barrier_cb(struct barrier *barrier, int status) const char *tail; sqlite3_stmt *tail_stmt; int tuple_format; + struct leader *l; int rv; if (status != 0) { @@ -1001,6 +1022,10 @@ static void query_sql_barrier_cb(struct barrier *barrier, int status) if (sqlite3_stmt_readonly(stmt)) { query_batch(g); } else { + l = gw_get_readwrite(g, req->db_id, req); + if (l == NULL) { + return; + } rv = leader_exec_v2(g->leader, &g->exec, stmt, modifying_query_sql_exec_cb); if (rv == LEADER_NOT_ASYNC) { @@ -1046,13 +1071,14 @@ static int handle_query_sql(struct gateway *g, struct handle *req) FAIL_IF_CHECKPOINTING; + req->sql = request.sql; + g->req = req; if (conn_readonly) { - prepare_transient_and_query_readonly_stub(); + g->barrier.data = g; + query_sql_barrier_cb(&g->barrier, 0); return 0; } - req->sql = request.sql; - g->req = req; rv = leader_barrier_v2(g->leader, &g->barrier, query_sql_barrier_cb); if (rv == LEADER_NOT_ASYNC) { query_sql_barrier_cb(&g->barrier, 0); diff --git a/src/vfs.c b/src/vfs.c index fbb1b8361..a1a4f1fc0 100644 --- a/src/vfs.c +++ b/src/vfs.c @@ -1685,7 +1685,6 @@ static int vfsFileShmLock(sqlite3_file *file, int ofst, int n, int flags) /* When releasing the write lock, if we find a pending * uncommitted transaction then a rollback must have occurred. * In that case we delete the pending transaction. */ - tracef("ROLLBACK TIME"); if (flags == (SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE)) { vfsWalRollbackIfUncommitted(wal); } diff --git a/test/unit/test_concurrency.c b/test/unit/test_concurrency.c index bf4d66375..313aca748 100644 --- a/test/unit/test_concurrency.c +++ b/test/unit/test_concurrency.c @@ -56,6 +56,7 @@ struct connection { rc = buffer__init(&c->response); \ munit_assert_int(rc, ==, 0); \ open.filename = "test"; \ + open.flags = 0; \ open.vfs = ""; \ ENCODE(c, &open, open); \ HANDLE(c, OPEN); \ diff --git a/test/unit/test_gateway.c b/test/unit/test_gateway.c index c7db7cf5a..5d589ce30 100644 --- a/test/unit/test_gateway.c +++ b/test/unit/test_gateway.c @@ -1776,7 +1776,7 @@ TEST_CASE(query, allow_stale_modifying, NULL) ENCODE(&f->request, query); HANDLE(QUERY); ASSERT_CALLBACK(0, FAILURE); - ASSERT_FAILURE(SQLITE_IOERR_NOT_LEADER, "not leader"); + ASSERT_FAILURE(SQLITE_READONLY, "dqlite connection is readonly"); return MUNIT_OK; } From 384712fd1e5aba1a4c5ab46da6b96cfa4b97dd5e Mon Sep 17 00:00:00 2001 From: Cole Miller Date: Tue, 15 Oct 2024 19:09:22 -0400 Subject: [PATCH 10/10] Refactor Signed-off-by: Cole Miller --- src/gateway.c | 157 ++++++++++++++------------------------- src/leader.c | 8 +- src/leader.h | 2 +- src/protocol.h | 11 ++- test/unit/test_gateway.c | 46 ++++++------ 5 files changed, 89 insertions(+), 135 deletions(-) diff --git a/src/gateway.c b/src/gateway.c index a07926f84..da4738526 100644 --- a/src/gateway.c +++ b/src/gateway.c @@ -35,8 +35,6 @@ void gateway__init(struct gateway *g, void gateway__leader_close(struct gateway *g, int reason) { - bool allow_stale; - if (g == NULL || g->leader == NULL) { tracef("gateway:%p or gateway->leader are NULL", g); return; @@ -44,8 +42,7 @@ void gateway__leader_close(struct gateway *g, int reason) /* If the client has opted into reading potentially stale data, don't * shut down the connection unnecessarily when we lost leadership. */ - allow_stale = g->leader->flags & DQLITE_OPEN_ALLOW_STALE; - if (allow_stale && reason == RAFT_LEADERSHIPLOST) { + if (g->leader->readonly && reason == RAFT_LEADERSHIPLOST) { return; } @@ -196,51 +193,22 @@ static void failure(struct handle *req, int code, const char *message) req->cb(req, 0, DQLITE_RESPONSE_FAILURE, 0); } -static struct leader *gw_get_readonly(const struct gateway *g, - uint64_t db_id, - struct handle *req) +static struct leader *gw_get_leader(const struct gateway *g, + uint64_t db_id, + struct handle *req) { struct leader *l = g->leader; - bool opened_readonly; if (db_id != 0 || l == NULL) { failure(req, SQLITE_NOTFOUND, "no database opened"); return NULL; - } - opened_readonly = l->flags & DQLITE_OPEN_ALLOW_STALE; - if (raft_state(g->raft) != RAFT_LEADER && !opened_readonly) { + } else if (!l->readonly && raft_state(g->raft) != RAFT_LEADER) { failure(req, SQLITE_IOERR_NOT_LEADER, "not leader"); return NULL; } return l; } -static struct leader *gw_get_readwrite(const struct gateway *g, - uint64_t db_id, - struct handle *req) -{ - struct leader *l = g->leader; - bool opened_readonly; - - if (db_id != 0 || l == NULL) { - failure(req, SQLITE_NOTFOUND, "no database opened"); - return NULL; - } - opened_readonly = l->flags & DQLITE_OPEN_ALLOW_STALE; - if (raft_state(g->raft) != RAFT_LEADER) { - if (req != NULL) { - failure(req, SQLITE_IOERR_NOT_LEADER, "not leader"); - } - return NULL; - } else if (opened_readonly) { - if (req != NULL) { - failure(req, SQLITE_READONLY, "dqlite connection is readonly"); - } - return NULL; - } - return l; -} - static void emptyRows(struct handle *req) { char *cursor = buffer__advance(req->buffer, 8 + 8); @@ -423,7 +391,6 @@ static int handle_prepare(struct gateway *g, struct handle *req) struct stmt *stmt; struct request_prepare request = { 0 }; struct leader *l; - bool conn_readonly; int rc; if (req->schema != DQLITE_PREPARE_STMT_SCHEMA_V0 && @@ -436,14 +403,9 @@ static int handle_prepare(struct gateway *g, struct handle *req) return rc; } - conn_readonly = false; - l = gw_get_readwrite(g, request.db_id, NULL); + l = gw_get_leader(g, request.db_id, req); if (l == NULL) { - conn_readonly = true; - l = gw_get_readonly(g, request.db_id, req); - if (l == NULL) { - return 0; - } + return 0; } rc = stmt__registry_add(&g->stmts, &stmt); @@ -458,13 +420,12 @@ static int handle_prepare(struct gateway *g, struct handle *req) req->stmt_id = stmt->id; req->sql = request.sql; g->req = req; - if (conn_readonly) { - /* XXX */ + if (l->readonly) { g->barrier.data = g; - prepare_barrier_cb(&g->barrier, 0); - return 0; + rc = LEADER_NOT_ASYNC; + } else { + rc = leader_barrier_v2(l, &g->barrier, prepare_barrier_cb); } - rc = leader_barrier_v2(g->leader, &g->barrier, prepare_barrier_cb); if (rc == LEADER_NOT_ASYNC) { prepare_barrier_cb(&g->barrier, 0); } else if (rc != 0) { @@ -554,9 +515,12 @@ static int handle_exec(struct gateway *g, struct handle *req) return rv; } - l = gw_get_readwrite(g, request.db_id, req); + l = gw_get_leader(g, request.db_id, req); if (l == NULL) { return 0; + } else if (l->readonly) { + failure(req, SQLITE_READONLY, "dqlite connection is readonly"); + return 0; } LOOKUP_STMT(request.stmt_id); @@ -678,8 +642,6 @@ static int handle_query(struct gateway *g, struct handle *req) struct request_query request = { 0 }; int tuple_format; struct leader *l; - bool conn_readonly; - bool stmt_readonly; int rv; switch (req->schema) { @@ -702,22 +664,11 @@ static int handle_query(struct gateway *g, struct handle *req) return rv; } - conn_readonly = false; - l = gw_get_readwrite(g, request.db_id, NULL); + l = gw_get_leader(g, request.db_id, req); if (l == NULL) { - conn_readonly = true; - l = gw_get_readonly(g, request.db_id, req); - if (l == NULL) { - return 0; - } - } - LOOKUP_STMT(request.stmt_id); - stmt_readonly = sqlite3_stmt_readonly(stmt->stmt); - if (!ERGO(conn_readonly, stmt_readonly)) { - failure(req, SQLITE_READONLY, "dqlite connection is readonly"); return 0; } - + LOOKUP_STMT(request.stmt_id); FAIL_IF_CHECKPOINTING; rv = bind__params(stmt->stmt, cursor, tuple_format); @@ -726,32 +677,35 @@ static int handle_query(struct gateway *g, struct handle *req) failure(req, rv, "bind parameters"); return 0; } + req->stmt_id = stmt->id; g->req = req; - if (conn_readonly) { - /* XXX */ - g->barrier.data = g; - query_barrier_cb(&g->barrier, 0); - rv = 0; - } else if (stmt_readonly) { - rv = leader_barrier_v2(g->leader, &g->barrier, query_barrier_cb); - if (rv == LEADER_NOT_ASYNC) { - query_barrier_cb(&g->barrier, 0); - rv = 0; + if (!sqlite3_stmt_readonly(stmt->stmt)) { + if (l->readonly) { + failure(req, SQLITE_READONLY, "dqlite connection is readonly"); + return 0; } - } else { - rv = leader_exec_v2(g->leader, &g->exec, stmt->stmt, - modifying_query_exec_cb); + rv = leader_exec_v2(l, &g->exec, stmt->stmt, modifying_query_exec_cb); if (rv == LEADER_NOT_ASYNC) { modifying_query_exec_cb(&g->exec, g->exec.status); - rv = 0; } + return 0; } - if (rv != 0) { + + if (l->readonly) { + g->barrier.data = g; + rv = LEADER_NOT_ASYNC; + } else { + rv = leader_barrier_v2(l, &g->barrier, query_barrier_cb); + } + if (rv == LEADER_NOT_ASYNC) { + query_barrier_cb(&g->barrier, 0); + } else if (rv != 0) { g->req = NULL; return rv; } + return 0; } @@ -764,7 +718,7 @@ static int handle_finalize(struct gateway *g, struct handle *req) int rv; START_V0(finalize, empty); - l = gw_get_readonly(g, request.db_id, req); + l = gw_get_leader(g, request.db_id, req); if (l == NULL) { return 0; } @@ -917,16 +871,19 @@ static int handle_exec_sql(struct gateway *g, struct handle *req) return rc; } - l = gw_get_readwrite(g, request.db_id, req); + l = gw_get_leader(g, request.db_id, req); if (l == NULL) { return 0; + } else if (l->readonly) { + failure(req, SQLITE_READONLY, "dqlite connection is readonly"); + return 0; } FAIL_IF_CHECKPOINTING; req->sql = request.sql; req->exec_count = 0; g->req = req; - rc = leader_barrier_v2(g->leader, &g->barrier, exec_sql_barrier_cb); + rc = leader_barrier_v2(l, &g->barrier, exec_sql_barrier_cb); if (rc == LEADER_NOT_ASYNC) { exec_sql_barrier_cb(&g->barrier, 0); } else if (rc != 0) { @@ -969,7 +926,6 @@ static void query_sql_barrier_cb(struct barrier *barrier, int status) const char *tail; sqlite3_stmt *tail_stmt; int tuple_format; - struct leader *l; int rv; if (status != 0) { @@ -1019,11 +975,9 @@ static void query_sql_barrier_cb(struct barrier *barrier, int status) req->stmt = stmt; g->req = req; - if (sqlite3_stmt_readonly(stmt)) { - query_batch(g); - } else { - l = gw_get_readwrite(g, req->db_id, req); - if (l == NULL) { + if (!sqlite3_stmt_readonly(stmt)) { + if (g->leader->readonly) { + failure(req, SQLITE_READONLY, "dqlite connection is readonly"); return; } rv = leader_exec_v2(g->leader, &g->exec, stmt, @@ -1035,7 +989,10 @@ static void query_sql_barrier_cb(struct barrier *barrier, int status) g->req = NULL; failure(req, rv, "leader exec"); } + return; } + + query_batch(g); } static int handle_query_sql(struct gateway *g, struct handle *req) @@ -1044,7 +1001,6 @@ static int handle_query_sql(struct gateway *g, struct handle *req) struct cursor *cursor = &req->cursor; struct request_query_sql request = { 0 }; struct leader *l; - bool conn_readonly; int rv; /* Fail early if the schema version isn't recognized. */ @@ -1059,27 +1015,22 @@ static int handle_query_sql(struct gateway *g, struct handle *req) return rv; } - conn_readonly = false; - l = gw_get_readwrite(g, request.db_id, NULL); + l = gw_get_leader(g, request.db_id, req); if (l == NULL) { - conn_readonly = true; - l = gw_get_readonly(g, request.db_id, req); - if (l == NULL) { - return 0; - } + return 0; } FAIL_IF_CHECKPOINTING; req->sql = request.sql; g->req = req; - if (conn_readonly) { + + if (l->readonly) { g->barrier.data = g; - query_sql_barrier_cb(&g->barrier, 0); - return 0; + rv = LEADER_NOT_ASYNC; + } else { + rv = leader_barrier_v2(g->leader, &g->barrier, query_sql_barrier_cb); } - - rv = leader_barrier_v2(g->leader, &g->barrier, query_sql_barrier_cb); if (rv == LEADER_NOT_ASYNC) { query_sql_barrier_cb(&g->barrier, 0); } else if (rv != 0) { diff --git a/src/leader.c b/src/leader.c index b2ee15f68..9be8fa784 100644 --- a/src/leader.c +++ b/src/leader.c @@ -123,14 +123,14 @@ static bool needsBarrier(struct leader *l) } int leader_init(struct leader *l, - struct db *db, - uint64_t flags, - struct raft *raft) + struct db *db, + uint64_t flags, + struct raft *raft) { tracef("leader init"); int rc; l->db = db; - l->flags = flags; + l->readonly = flags & DQLITE_OPEN_READONLY; l->raft = raft; rc = openConnection(db->path, db->config->name, db->config->page_size, &l->conn); diff --git a/src/leader.h b/src/leader.h index ec1a301bc..7c474b878 100644 --- a/src/leader.h +++ b/src/leader.h @@ -41,7 +41,7 @@ struct apply { struct leader { struct db *db; /* Database the connection. */ - uint64_t flags; + bool readonly; sqlite3 *conn; /* Underlying SQLite connection. */ struct raft *raft; /* Raft instance. */ struct exec *exec; /* Exec request in progress, if any. */ diff --git a/src/protocol.h b/src/protocol.h index 2f6523eef..c22fcde58 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -88,10 +88,13 @@ enum { * Flags for the OPEN request. */ enum { - /* When set, gives permission for the server to execute readonly QUERY - * and QUERY_SQL requests against this database even if it is not the - * leader. */ - DQLITE_OPEN_ALLOW_STALE = 1 << 0, + + /* This flag has two effects on the server: first, client requests to + * modify the target database will be denied; second, client requests + * to read the target database will be accepted even when the server is + * not the leader, and the connection will not be closed when the + * server loses leadership. */ + DQLITE_OPEN_READONLY = 1 << 0, }; #endif /* DQLITE_PROTOCOL_H_ */ diff --git a/test/unit/test_gateway.c b/test/unit/test_gateway.c index 5d589ce30..9bf68dfcf 100644 --- a/test/unit/test_gateway.c +++ b/test/unit/test_gateway.c @@ -186,11 +186,11 @@ static void handleCb(struct handle *req, ASSERT_CALLBACK(0, DB); \ } -#define OPEN_ALLOW_STALE \ +#define OPEN_READONLY \ { \ struct request_open open; \ open.filename = "test"; \ - open.flags = DQLITE_OPEN_ALLOW_STALE; \ + open.flags = DQLITE_OPEN_READONLY; \ open.vfs = ""; \ ENCODE(&open, open); \ HANDLE(OPEN); \ @@ -692,7 +692,7 @@ TEST_CASE(prepare, non_leader_ok, NULL) CLUSTER_ELECT(0); SELECT(1); - OPEN_ALLOW_STALE; + OPEN_READONLY; f->request.db_id = 0; f->request.sql = "CREATE TABLE test (n INT)"; ENCODE(&f->request, prepare); @@ -1228,14 +1228,14 @@ TEST_CASE(exec, unexpectedRow, NULL) return MUNIT_OK; } -TEST_CASE(exec, not_leader_never_okay, NULL) +TEST_CASE(exec, readonly, NULL) { struct exec_fixture *f = data; uint64_t stmt_id; (void)params; CLUSTER_ELECT(0); SELECT(1); - OPEN_ALLOW_STALE; + OPEN_READONLY; PREPARE("CREATE TABLE test (n INT)"); f->request.db_id = 0; f->request.stmt_id = stmt_id; @@ -1243,7 +1243,7 @@ TEST_CASE(exec, not_leader_never_okay, NULL) HANDLE(EXEC); WAIT; ASSERT_CALLBACK(0, FAILURE); - ASSERT_FAILURE(SQLITE_IOERR_NOT_LEADER, "not leader"); + ASSERT_FAILURE(SQLITE_READONLY, "dqlite connection is readonly"); return MUNIT_OK; } @@ -1732,8 +1732,8 @@ TEST_CASE(query, close_while_in_flight, NULL) } /* A non-leader serves readonly QUERY requests for a database that was opened - * with the ALLOW_STALE flag. */ -TEST_CASE(query, allow_stale, NULL) + * with the READONLY flag. */ +TEST_CASE(query, non_leader_readonly, NULL) { (void)params; struct query_fixture *f = data; @@ -1743,7 +1743,7 @@ TEST_CASE(query, allow_stale, NULL) CLUSTER_APPLIED(4); SELECT(1); - OPEN_ALLOW_STALE; + OPEN_READONLY; PREPARE("SELECT n FROM test"); f->request.db_id = 0; f->request.stmt_id = stmt_id; @@ -1760,8 +1760,8 @@ TEST_CASE(query, allow_stale, NULL) } /* A non-leader will not serve a QUERY request that modifies the database, even - * if opened with the ALLOW_STALE flag. */ -TEST_CASE(query, allow_stale_modifying, NULL) + * if opened with the READONLY flag. */ +TEST_CASE(query, modifying_on_readonly_conn, NULL) { (void)params; struct query_fixture *f = data; @@ -1769,7 +1769,7 @@ TEST_CASE(query, allow_stale_modifying, NULL) CLUSTER_APPLIED(4); SELECT(1); - OPEN_ALLOW_STALE; + OPEN_READONLY; PREPARE("INSERT INTO test (n) VALUES (17)"); f->request.db_id = 0; f->request.stmt_id = stmt_id; @@ -1830,7 +1830,7 @@ TEST_CASE(finalize, not_leader, NULL) (void)params; CLUSTER_ELECT(0); SELECT(1); - OPEN_ALLOW_STALE; + OPEN_READONLY; PREPARE("CREATE TABLE test (n INT)"); f->request.db_id = 0; f->request.stmt_id = stmt_id; @@ -2030,19 +2030,19 @@ TEST_CASE(exec_sql, manyParams, NULL) return MUNIT_OK; } -TEST_CASE(exec_sql, not_leader_never_ok, NULL) +TEST_CASE(exec_sql, readonly, NULL) { struct exec_sql_fixture *f = data; (void)params; SELECT(1); - OPEN_ALLOW_STALE; + OPEN_READONLY; f->request.db_id = 0; f->request.sql = "CREATE TABLE test (n INT)"; ENCODE(&f->request, exec_sql); HANDLE(EXEC_SQL); ASSERT_CALLBACK(0, FAILURE); - ASSERT_FAILURE(SQLITE_IOERR_NOT_LEADER, "not leader"); + ASSERT_FAILURE(SQLITE_READONLY, "dqlite connection is readonly"); return MUNIT_OK; } @@ -2434,8 +2434,8 @@ TEST_CASE(query_sql, nonemptyTail, NULL) } /* A non-leader serves readonly QUERY_SQL requests for a database that was - * opened with the ALLOW_STALE flag. */ -TEST_CASE(query_sql, allow_stale, NULL) + * opened with the READONLY flag. */ +TEST_CASE(query_sql, non_leader_readonly, NULL) { (void)params; struct query_sql_fixture *f = data; @@ -2444,7 +2444,7 @@ TEST_CASE(query_sql, allow_stale, NULL) CLUSTER_APPLIED(4); SELECT(1); - OPEN_ALLOW_STALE; + OPEN_READONLY; f->request.db_id = 0; f->request.sql = "SELECT * FROM test"; ENCODE(&f->request, query_sql); @@ -2460,21 +2460,21 @@ TEST_CASE(query_sql, allow_stale, NULL) } /* A non-leader will not serve a QUERY_SQL request that modifies the database, - * even if opened with the ALLOW_STALE flag. */ -TEST_CASE(query_sql, allow_stale_modifying, NULL) + * even if opened with the READONLY flag. */ +TEST_CASE(query_sql, modifying_on_readonly_conn, NULL) { (void)params; struct query_sql_fixture *f = data; CLUSTER_APPLIED(4); SELECT(1); - OPEN_ALLOW_STALE; + OPEN_READONLY; f->request.db_id = 0; f->request.sql = "INSERT INTO test (n) VALUES (17)"; ENCODE(&f->request, query_sql); HANDLE(QUERY_SQL); ASSERT_CALLBACK(0, FAILURE); - ASSERT_FAILURE(SQLITE_IOERR_NOT_LEADER, "not leader"); + ASSERT_FAILURE(SQLITE_READONLY, "dqlite connection is readonly"); return MUNIT_OK; }