From bc4d10fc4b6204afffa264049a6f0573f54b539b Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Wed, 26 Jun 2024 00:24:40 +0100 Subject: [PATCH 1/7] Wire up /db/_config/audit APIs --- base/audit_events.go | 22 +++++-- base/audit_types.go | 5 ++ rest/admin_api.go | 145 +++++++++++++++++++++++++++++++++++++------ rest/config.go | 16 +++++ 4 files changed, 164 insertions(+), 24 deletions(-) diff --git a/base/audit_events.go b/base/audit_events.go index b542f2ba23..c05319ad9c 100644 --- a/base/audit_events.go +++ b/base/audit_events.go @@ -77,12 +77,24 @@ var AuditEvents = events{ // DefaultAuditEventIDs is a list of audit event IDs that are enabled by default. var DefaultAuditEventIDs = buildDefaultAuditIDList(AuditEvents) -func buildDefaultAuditIDList(e events) []uint { - var ids []uint - for id, event := range e { - if event.EnabledByDefault { - ids = append(ids, uint(id)) +func buildDefaultAuditIDList(e events) (ids []uint) { + for k, v := range e { + if v.EnabledByDefault { + ids = append(ids, uint(k)) } } return ids } + +// NonFilterableEvents is a map of audit events that are not permitted to be filtered. +var NonFilterableEvents = buildNonFilterableEvents(AuditEvents) + +func buildNonFilterableEvents(e events) events { + nonFilterable := make(events) + for k, v := range e { + if !v.FilteringPermitted { + nonFilterable[k] = v + } + } + return nonFilterable +} diff --git a/base/audit_types.go b/base/audit_types.go index 703aaea4fc..05dd65bbc8 100644 --- a/base/audit_types.go +++ b/base/audit_types.go @@ -21,6 +21,11 @@ func (i AuditID) String() string { return strconv.FormatUint(uint64(i), 10) } +func ParseAuditID(s string) (AuditID, error) { + id, err := strconv.ParseUint(s, 10, 64) + return AuditID(id), err +} + // events is a map of audit event IDs to event descriptors. type events map[AuditID]EventDescriptor diff --git a/rest/admin_api.go b/rest/admin_api.go index 1459d41b4a..8718a706b4 100644 --- a/rest/admin_api.go +++ b/rest/admin_api.go @@ -668,7 +668,18 @@ func (h *handler) handlePutDbConfig() (err error) { h.server.dbConfigs[dbName].cfgCas = cas return base.HTTPErrorf(http.StatusCreated, "updated") +} + +type handleDbAuditConfigBody struct { + Enabled *bool `json:"enabled,omitempty"` + Events map[string]any `json:"events,omitempty"` +} +type handleDbAuditConfigBodyVerboseEvent struct { + Name *string `json:"name,omitempty"` + Description *string `json:"description,omitempty"` + Enabled *bool `json:"enabled,omitempty"` + Filterable *bool `json:"filterable,omitempty"` } // GET audit config for database @@ -678,46 +689,142 @@ func (h *handler) handleGetDbAuditConfig() error { showOnlyFilterable := h.getBoolQuery("filterable") verbose := h.getBoolQuery("verbose") - isEnabledFn := func(id base.AuditID) bool { - _, ok := h.db.Options.LoggingConfig.Audit.EnabledEvents[id] - return ok + var ( + etagVersion string + dbAuditEnabled bool + enabledEvents = make(map[base.AuditID]struct{}) + ) + + if h.server.BootstrapContext.Connection != nil { + found, dbConfig, err := h.server.fetchDatabase(h.ctx(), h.db.Name) + if err != nil { + return err + } + + if !found { + return base.HTTPErrorf(http.StatusNotFound, "database config not found") + } + + etagVersion = dbConfig.Version + + if dbConfig.Logging != nil && dbConfig.Logging.Audit != nil { + dbAuditEnabled = base.BoolDefault(dbConfig.Logging.Audit.Enabled, false) + for _, event := range dbConfig.Logging.Audit.EnabledEvents { + enabledEvents[base.AuditID(event)] = struct{}{} + } + } } - // TODO: Move to structs - events := make(map[string]interface{}, len(base.AuditEvents)) + events := make(map[string]any, len(base.AuditEvents)) for id, descriptor := range base.AuditEvents { if showOnlyFilterable && !descriptor.FilteringPermitted { continue } + idStr := id.String() + _, eventEnabled := enabledEvents[id] + if verbose { - events[idStr] = map[string]interface{}{ - "name": descriptor.Name, - "description": descriptor.Description, - "enabled": isEnabledFn(id), - "filterable": descriptor.FilteringPermitted, + events[idStr] = handleDbAuditConfigBodyVerboseEvent{ + Name: stringPtrOrNil(descriptor.Name), + Description: stringPtrOrNil(descriptor.Description), + Enabled: &eventEnabled, + Filterable: base.BoolPtr(descriptor.FilteringPermitted), } } else { - events[idStr] = isEnabledFn(id) + events[idStr] = &eventEnabled } } - h.writeJSON(events) + resp := handleDbAuditConfigBody{ + Enabled: &dbAuditEnabled, + Events: events, + } + h.setEtag(etagVersion) + h.writeJSON(resp) return nil } // PUT/POST audit config for database func (h *handler) handlePutDbAuditConfig() error { - h.assertAdminOnly() + return h.mutateDbConfig(func(config *DbConfig) error { + var body handleDbAuditConfigBody + if err := h.readJSONInto(&body); err != nil { + return err + } - // interface can be either bool or object for verbose-format - var body map[string]interface{} - if err := h.readJSONInto(&body); err != nil { - return err - } + // isReplace if the request is a PUT, and we want to overwrite existing config + isReplace := h.rq.Method == http.MethodPut - return nil + // This API endpoint takes audit config in a format that does not match the actual DbConfig stored, so translate the request here. + toChange := make(map[base.AuditID]bool, len(body.Events)) + var multiError *base.MultiError + for id, val := range body.Events { + // find the event + auditID, err := base.ParseAuditID(id) + if err != nil { + multiError = multiError.Append(fmt.Errorf("invalid audit event ID: %q", id)) + continue + } + _, ok := base.AuditEvents[auditID] + if !ok { + multiError = multiError.Append(fmt.Errorf("unknown audit event ID: %q", id)) + continue + } + + var eventEnabled bool + switch valT := val.(type) { + case bool: + eventEnabled = valT + case map[string]any: + // verbose format + eventEnabled = valT["enabled"].(bool) + } + + toChange[auditID] = eventEnabled + } + if err := multiError.ErrorOrNil(); err != nil { + return base.HTTPErrorf(http.StatusBadRequest, "couldn't update audit configuration: %s", err) + } + + if isReplace || body.Enabled != nil { + config.Logging.Audit.Enabled = body.Enabled + } + + if isReplace { + // we don't need to do anything to "disable" events, other than not enable them + config.Logging.Audit.EnabledEvents = func() []uint { + enabledEvents := make([]uint, 0) + for event, shouldEnable := range toChange { + if shouldEnable { + enabledEvents = append(enabledEvents, uint(event)) + } + } + return enabledEvents + }() + } else { + for i, event := range config.Logging.Audit.EnabledEvents { + if shouldEnable, ok := toChange[base.AuditID(event)]; ok { + if shouldEnable { + // already enabled + continue + } else { + // disable by removing + config.Logging.Audit.EnabledEvents = append(config.Logging.Audit.EnabledEvents[:i], config.Logging.Audit.EnabledEvents[i+1:]...) + } + // drop from toChange so we don't duplicate IDs + delete(toChange, base.AuditID(event)) + } + } + for id, enabled := range toChange { + if enabled { + config.Logging.Audit.EnabledEvents = append(config.Logging.Audit.EnabledEvents, uint(id)) + } + } + } + return nil + }) } // GET collection config sync function diff --git a/rest/config.go b/rest/config.go index a26195b986..db317b65d6 100644 --- a/rest/config.go +++ b/rest/config.go @@ -1065,6 +1065,21 @@ func (dbConfig *DbConfig) validateVersion(ctx context.Context, isEnterpriseEditi } } } + + if dbConfig.Logging != nil { + if dbConfig.Logging.Audit != nil { + if !isEnterpriseEdition && dbConfig.Logging.Audit.Enabled != nil { + base.WarnfCtx(ctx, eeOnlyWarningMsg, "logging.audit.enabled", *dbConfig.Logging.Audit.Enabled, false) + dbConfig.Logging.Audit.Enabled = nil + } + for _, id := range dbConfig.Logging.Audit.EnabledEvents { + if _, ok := base.AuditEvents[base.AuditID(id)]; !ok { + multiError = multiError.Append(fmt.Errorf("unknown audit event ID %q", id)) + } + } + } + } + return multiError.ErrorOrNil() } @@ -2197,6 +2212,7 @@ func (c *DbConfig) toDbLogConfig(ctx context.Context) *base.DbLogConfig { enabledEvents[base.AuditID(event)] = struct{}{} } aud = &base.DbAuditLogConfig{ + Enabled: base.BoolDefault(l.Audit.Enabled, false), EnabledEvents: enabledEvents, } } From 53fac2e8861cef28bd5bf327993a5536a9ee4ed1 Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Fri, 5 Jul 2024 19:49:32 +0100 Subject: [PATCH 2/7] nil log config panic fixes --- rest/admin_api.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/rest/admin_api.go b/rest/admin_api.go index 8718a706b4..564b88687f 100644 --- a/rest/admin_api.go +++ b/rest/admin_api.go @@ -788,6 +788,13 @@ func (h *handler) handlePutDbAuditConfig() error { return base.HTTPErrorf(http.StatusBadRequest, "couldn't update audit configuration: %s", err) } + if config.Logging == nil { + config.Logging = &DbLoggingConfig{} + } + if config.Logging.Audit == nil { + config.Logging.Audit = &DbAuditLoggingConfig{} + } + if isReplace || body.Enabled != nil { config.Logging.Audit.Enabled = body.Enabled } From 4065913917839e5722f57b1755c249feb89a63c2 Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Tue, 9 Jul 2024 13:23:32 +0100 Subject: [PATCH 3/7] wip --- rest/admin_api.go | 12 +++++++++--- rest/adminapitest/admin_api_test.go | 25 +++++++++++++++++++++++++ rest/config_database.go | 1 - 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/rest/admin_api.go b/rest/admin_api.go index 564b88687f..f8fe7a2509 100644 --- a/rest/admin_api.go +++ b/rest/admin_api.go @@ -707,9 +707,15 @@ func (h *handler) handleGetDbAuditConfig() error { etagVersion = dbConfig.Version - if dbConfig.Logging != nil && dbConfig.Logging.Audit != nil { - dbAuditEnabled = base.BoolDefault(dbConfig.Logging.Audit.Enabled, false) - for _, event := range dbConfig.Logging.Audit.EnabledEvents { + runtimeConfig, err := MergeDatabaseConfigWithDefaults(h.server.Config, &dbConfig.DbConfig) + if err != nil { + return err + } + + // grab runtime version of config, so that we can see what events would be enabled + if runtimeConfig.Logging != nil && runtimeConfig.Logging.Audit != nil { + dbAuditEnabled = base.BoolDefault(runtimeConfig.Logging.Audit.Enabled, false) + for _, event := range runtimeConfig.Logging.Audit.EnabledEvents { enabledEvents[base.AuditID(event)] = struct{}{} } } diff --git a/rest/adminapitest/admin_api_test.go b/rest/adminapitest/admin_api_test.go index 01225f8a70..eaf948c865 100644 --- a/rest/adminapitest/admin_api_test.go +++ b/rest/adminapitest/admin_api_test.go @@ -4263,3 +4263,28 @@ func TestDatabaseCreationWithEnvVariableWithBackticks(t *testing.T) { resp := rt.SendAdminRequestWithAuth(http.MethodPut, "/backticks/", string(input), rest.MobileSyncGatewayRole.RoleName, "password") rest.RequireStatus(t, resp, http.StatusCreated) } + +func TestDatabaseConfigAuditAPI(t *testing.T) { + rt := rest.NewRestTesterPersistentConfig(t) + defer rt.Close() + + // check default audit config + resp := rt.SendAdminRequest(http.MethodGet, "/db/_config/audit", "") + rest.RequireStatus(t, resp, http.StatusOK) + resp.DumpBody() + var responseBody map[string]interface{} + require.NoError(t, json.Unmarshal(resp.Body.Bytes(), &responseBody)) + assert.Equal(t, false, responseBody["enabled"].(bool)) + + // enable audit on the database + resp = rt.SendAdminRequest(http.MethodPost, "/db/_config/audit", `{"enabled":true}`) + rest.RequireStatus(t, resp, http.StatusOK) + + // check audit config + resp = rt.SendAdminRequest(http.MethodGet, "/db/_config/audit", "") + rest.RequireStatus(t, resp, http.StatusOK) + resp.DumpBody() + responseBody = nil + require.NoError(t, json.Unmarshal(resp.Body.Bytes(), &responseBody)) + assert.Equal(t, true, responseBody["enabled"].(bool)) +} diff --git a/rest/config_database.go b/rest/config_database.go index 3eb995dc9a..07283f62c5 100644 --- a/rest/config_database.go +++ b/rest/config_database.go @@ -192,7 +192,6 @@ func DefaultDbConfig(sc *StartupConfig, useXattrs bool) *DbConfig { if base.IsEnterpriseEdition() { dbConfig.ImportPartitions = base.Uint16Ptr(base.GetDefaultImportPartitions(sc.IsServerless())) } - } else { dbConfig.AutoImport = base.BoolPtr(false) } From 6604ec71bd444b31bb77079465b562371c4206a9 Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Tue, 9 Jul 2024 16:21:34 +0100 Subject: [PATCH 4/7] Make audit test EE only and fix DefaultPerDBLogging when bootstrap is not set --- rest/adminapitest/admin_api_test.go | 4 ++++ rest/config_database.go | 8 +++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/rest/adminapitest/admin_api_test.go b/rest/adminapitest/admin_api_test.go index eaf948c865..d72e7b4996 100644 --- a/rest/adminapitest/admin_api_test.go +++ b/rest/adminapitest/admin_api_test.go @@ -4265,6 +4265,10 @@ func TestDatabaseCreationWithEnvVariableWithBackticks(t *testing.T) { } func TestDatabaseConfigAuditAPI(t *testing.T) { + if !base.IsEnterpriseEdition() { + t.Skip("Audit logging is an EE-only feature") + } + rt := rest.NewRestTesterPersistentConfig(t) defer rt.Close() diff --git a/rest/config_database.go b/rest/config_database.go index 07283f62c5..9d820c5ba1 100644 --- a/rest/config_database.go +++ b/rest/config_database.go @@ -99,11 +99,9 @@ func DefaultPerDBLogging(bootstrapLoggingCnf base.LoggingConfig) *DbLoggingConfi } } } - if bootstrapLoggingCnf.Audit != nil { - dblc.Audit = &DbAuditLoggingConfig{ - Enabled: base.BoolPtr(false), - EnabledEvents: base.DefaultAuditEventIDs, - } + dblc.Audit = &DbAuditLoggingConfig{ + Enabled: base.BoolPtr(false), + EnabledEvents: base.DefaultAuditEventIDs, } return dblc } From c5f7cc4fb565c7559b194d0b6432afd69b299e78 Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Tue, 9 Jul 2024 16:45:51 +0100 Subject: [PATCH 5/7] Expanded coverage for audit API --- rest/adminapitest/admin_api_test.go | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/rest/adminapitest/admin_api_test.go b/rest/adminapitest/admin_api_test.go index d72e7b4996..9e913b2452 100644 --- a/rest/adminapitest/admin_api_test.go +++ b/rest/adminapitest/admin_api_test.go @@ -4272,15 +4272,17 @@ func TestDatabaseConfigAuditAPI(t *testing.T) { rt := rest.NewRestTesterPersistentConfig(t) defer rt.Close() - // check default audit config - resp := rt.SendAdminRequest(http.MethodGet, "/db/_config/audit", "") + // check default audit config - verbose to read event names, etc. + resp := rt.SendAdminRequest(http.MethodGet, "/db/_config/audit?verbose=true", "") rest.RequireStatus(t, resp, http.StatusOK) resp.DumpBody() var responseBody map[string]interface{} require.NoError(t, json.Unmarshal(resp.Body.Bytes(), &responseBody)) assert.Equal(t, false, responseBody["enabled"].(bool)) + // check we got the verbose output + assert.NotEmpty(t, responseBody["events"].(map[string]interface{})[base.AuditIDPublicUserAuthenticated.String()].(map[string]interface{})["description"].(string), "expected verbose output (event description, etc.)") - // enable audit on the database + // enable auditing on the database (upsert) resp = rt.SendAdminRequest(http.MethodPost, "/db/_config/audit", `{"enabled":true}`) rest.RequireStatus(t, resp, http.StatusOK) @@ -4291,4 +4293,21 @@ func TestDatabaseConfigAuditAPI(t *testing.T) { responseBody = nil require.NoError(t, json.Unmarshal(resp.Body.Bytes(), &responseBody)) assert.Equal(t, true, responseBody["enabled"].(bool)) + assert.False(t, responseBody["events"].(map[string]interface{})[base.AuditIDAuditEnabled.String()].(bool), "audit enabled event should be disabled by default") // TODO: This will change - replace with an actual non-default event. + assert.True(t, responseBody["events"].(map[string]interface{})[base.AuditIDPublicUserAuthenticated.String()].(bool), "public user authenticated event should be enabled by default") + + // do a PUT to completely replace the full config (events not declared here will be disabled) + // enable AuditEnabled event, but implicitly others + resp = rt.SendAdminRequest(http.MethodPost, "/db/_config/audit", fmt.Sprintf(`{"enabled":true,"events":{"%s":true}}`, base.AuditIDAuditEnabled)) + rest.RequireStatus(t, resp, http.StatusOK) + + // check audit config + resp = rt.SendAdminRequest(http.MethodGet, "/db/_config/audit", "") + rest.RequireStatus(t, resp, http.StatusOK) + resp.DumpBody() + responseBody = nil + require.NoError(t, json.Unmarshal(resp.Body.Bytes(), &responseBody)) + assert.Equal(t, true, responseBody["enabled"].(bool)) + assert.True(t, responseBody["events"].(map[string]interface{})[base.AuditIDAuditEnabled.String()].(bool), "audit enabled event should've been enabled via PUT") + assert.False(t, responseBody["events"].(map[string]interface{})[base.AuditIDPublicUserAuthenticated.String()].(bool), "public user authenticated event should've been disabled via PUT") } From e43a9054deedf709317f19807f0c3f2794f5e18f Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Tue, 9 Jul 2024 17:01:58 +0100 Subject: [PATCH 6/7] fix TestDBGetConfigNamesAndDefaultLogging --- rest/adminapitest/admin_api_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest/adminapitest/admin_api_test.go b/rest/adminapitest/admin_api_test.go index 9e913b2452..b5c65bd9c2 100644 --- a/rest/adminapitest/admin_api_test.go +++ b/rest/adminapitest/admin_api_test.go @@ -562,7 +562,7 @@ func TestDBGetConfigNamesAndDefaultLogging(t *testing.T) { require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &body)) assert.Equal(t, len(rt.DatabaseConfig.Users), len(body.Users)) - emptyCnf := &rest.DbLoggingConfig{} + emptyCnf := rest.DefaultPerDBLogging(rt.ServerContext().Config.Logging) assert.Equal(t, body.Logging, emptyCnf) for k, v := range body.Users { From a1ff675c990e42e9342d2bc60f9edcedefdd0258 Mon Sep 17 00:00:00 2001 From: Ben Brooks Date: Tue, 9 Jul 2024 18:13:18 +0100 Subject: [PATCH 7/7] Make handleGetDbAuditConfig only work in persistent config mode --- rest/admin_api.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rest/admin_api.go b/rest/admin_api.go index f8fe7a2509..451e4a5183 100644 --- a/rest/admin_api.go +++ b/rest/admin_api.go @@ -719,6 +719,8 @@ func (h *handler) handleGetDbAuditConfig() error { enabledEvents[base.AuditID(event)] = struct{}{} } } + } else { + return base.HTTPErrorf(http.StatusServiceUnavailable, "audit config not available in non-persistent mode") } events := make(map[string]any, len(base.AuditEvents))