diff --git a/rest/admin_api.go b/rest/admin_api.go index bf079b30d0..de1965676e 100644 --- a/rest/admin_api.go +++ b/rest/admin_api.go @@ -126,6 +126,9 @@ func (h *handler) handleCreateDB() error { // if it used to be corrupt we need to remove it from the invalid database map on server context and remove the old corrupt config from the bucket err = h.removeCorruptConfigIfExists(contextNoCancel.Ctx, bucket, h.server.Config.Bootstrap.ConfigGroupID, dbName) if err != nil { + // we cannot continue on with database creation with possibility of the corrupt database config in the bucket for this db + // thus we need to unload the requested database config to prevent the cluster being in an inconsistent state + h.server._removeDatabase(contextNoCancel.Ctx, dbName) return err } cas, err := h.server.BootstrapContext.InsertConfig(contextNoCancel.Ctx, bucket, h.server.Config.Bootstrap.ConfigGroupID, &persistedConfig) diff --git a/rest/config_test.go b/rest/config_test.go index a6ebeb08a0..64615d1b04 100644 --- a/rest/config_test.go +++ b/rest/config_test.go @@ -2997,3 +2997,72 @@ func TestInvalidDbConfigNoLongerPresentInBucket(t *testing.T) { resp = rt.CreateDatabase(dbName, dbConfig) RequireStatus(t, resp, http.StatusCreated) } + +// TestNotFoundOnInvalidDatabase: +// - Create rest tester with large config polling interval +// - Insert a bad dbConfig into the bucket +// - Manually fetch and load db from buckets +// - Assert that the bad config is tracked as invalid config +// - Delete the bad config manually and attempt to correct the db config through create db endpoint +// - Assert db is removed form invalid db's and is now a running database on server context +func TestNotFoundOnInvalidDatabase(t *testing.T) { + rt := NewRestTester(t, &RestTesterConfig{ + CustomTestBucket: base.GetTestBucket(t), + PersistentConfig: true, + MutateStartupConfig: func(config *StartupConfig) { + // configure the interval time to not run + config.Bootstrap.ConfigUpdateFrequency = base.NewConfigDuration(100 * time.Second) + }, + DatabaseConfig: nil, + }) + defer rt.Close() + realBucketName := rt.CustomTestBucket.GetName() + + // create a new invalid db config and persist to bucket + badName := "badBucketName" + dbConfig := rt.NewDbConfig() + dbConfig.Name = "db1" + + version, err := GenerateDatabaseConfigVersionID(rt.Context(), "", &dbConfig) + require.NoError(t, err) + metadataID, metadataIDError := rt.ServerContext().BootstrapContext.ComputeMetadataIDForDbConfig(base.TestCtx(t), &dbConfig) + require.NoError(t, metadataIDError) + + // insert the db config with bad bucket name + dbConfig.Bucket = &badName + persistedConfig := DatabaseConfig{ + Version: version, + MetadataID: metadataID, + DbConfig: dbConfig, + SGVersion: base.ProductVersion.String(), + } + rt.InsertDbConfigToBucket(&persistedConfig, rt.CustomTestBucket.GetName()) + + // manually fetch and load db configs from bucket + _, err = rt.ServerContext().fetchAndLoadConfigs(rt.Context(), false) + require.NoError(t, err) + + // assert the config is picked as invalid db config + require.EventuallyWithT(t, func(c *assert.CollectT) { + invalidDatabases := rt.ServerContext().AllInvalidDatabaseNames(t) + assert.Equal(c, 1, len(invalidDatabases)) + }, time.Second*10, time.Millisecond*100) + + resp := rt.SendAdminRequest(http.MethodGet, "/db1/", "") + RequireStatus(t, resp, http.StatusNotFound) + assert.Contains(t, resp.Body.String(), "You must update database config immediately") + + // delete the invalid db config to force the not found error + rt.DeleteDbConfigInBucket(dbConfig.Name, realBucketName) + + // fix the bucket name and try fix corrupt db through create db endpoint + dbConfig.Bucket = &realBucketName + RequireStatus(t, rt.CreateDatabase(dbConfig.Name, dbConfig), http.StatusCreated) + + // assert the config is remove the invalid config and we have a running db + require.EventuallyWithT(t, func(c *assert.CollectT) { + invalidDatabases := rt.ServerContext().AllInvalidDatabaseNames(t) + assert.Equal(c, 0, len(invalidDatabases)) + assert.Equal(c, 1, len(rt.ServerContext().dbConfigs)) + }, time.Second*10, time.Millisecond*100) +} diff --git a/rest/handler.go b/rest/handler.go index eaac91ad9e..666470f097 100644 --- a/rest/handler.go +++ b/rest/handler.go @@ -704,7 +704,7 @@ func (h *handler) removeCorruptConfigIfExists(ctx context.Context, bucket, confi } // remove the bad config from the bucket err := h.server.BootstrapContext.DeleteConfig(ctx, bucket, configGroupID, dbName) - if err != nil { + if err != nil && !base.IsDocNotFoundError(err) { return err } // delete the database name form the invalid database map on server context