CBG-3981: Wire up PUT/POST /db/_config/audit APIs (#6940)

* Wire up /db/_config/audit APIs

* nil log config panic fixes

* wip

* Make audit test EE only and fix DefaultPerDBLogging when bootstrap is not set

* Expanded coverage for audit API

* fix TestDBGetConfigNamesAndDefaultLogging

* Make handleGetDbAuditConfig only work in persistent config mode

base/audit_events.go

@@ -77,12 +77,24 @@ var AuditEvents = events{
 // DefaultAuditEventIDs is a list of audit event IDs that are enabled by default.
 var DefaultAuditEventIDs = buildDefaultAuditIDList(AuditEvents)
 
-func buildDefaultAuditIDList(e events) []uint {
-	var ids []uint
-	for id, event := range e {
-		if event.EnabledByDefault {
-			ids = append(ids, uint(id))
+func buildDefaultAuditIDList(e events) (ids []uint) {
+	for k, v := range e {
+		if v.EnabledByDefault {
+			ids = append(ids, uint(k))
 		}
 	}
 	return ids
 }
+
+// NonFilterableEvents is a map of audit events that are not permitted to be filtered.
+var NonFilterableEvents = buildNonFilterableEvents(AuditEvents)
+
+func buildNonFilterableEvents(e events) events {
+	nonFilterable := make(events)
+	for k, v := range e {
+		if !v.FilteringPermitted {
+			nonFilterable[k] = v
+		}
+	}
+	return nonFilterable
+}

base/audit_types.go

@@ -21,6 +21,11 @@ func (i AuditID) String() string {
 	return strconv.FormatUint(uint64(i), 10)
 }
 
+func ParseAuditID(s string) (AuditID, error) {
+	id, err := strconv.ParseUint(s, 10, 64)
+	return AuditID(id), err
+}
+
 // events is a map of audit event IDs to event descriptors.
 type events map[AuditID]EventDescriptor
 

rest/admin_api.go

@@ -668,7 +668,18 @@ func (h *handler) handlePutDbConfig() (err error) {
 	h.server.dbConfigs[dbName].cfgCas = cas
 
 	return base.HTTPErrorf(http.StatusCreated, "updated")
+}
+
+type handleDbAuditConfigBody struct {
+	Enabled *bool          `json:"enabled,omitempty"`
+	Events  map[string]any `json:"events,omitempty"`
+}
 
+type handleDbAuditConfigBodyVerboseEvent struct {
+	Name        *string `json:"name,omitempty"`
+	Description *string `json:"description,omitempty"`
+	Enabled     *bool   `json:"enabled,omitempty"`
+	Filterable  *bool   `json:"filterable,omitempty"`
 }
 
 // GET audit config for database
@@ -678,46 +689,157 @@ func (h *handler) handleGetDbAuditConfig() error {
 	showOnlyFilterable := h.getBoolQuery("filterable")
 	verbose := h.getBoolQuery("verbose")
 
-	isEnabledFn := func(id base.AuditID) bool {
-		_, ok := h.db.Options.LoggingConfig.Audit.EnabledEvents[id]
-		return ok
+	var (
+		etagVersion    string
+		dbAuditEnabled bool
+		enabledEvents  = make(map[base.AuditID]struct{})
+	)
+
+	if h.server.BootstrapContext.Connection != nil {
+		found, dbConfig, err := h.server.fetchDatabase(h.ctx(), h.db.Name)
+		if err != nil {
+			return err
+		}
+
+		if !found {
+			return base.HTTPErrorf(http.StatusNotFound, "database config not found")
+		}
+
+		etagVersion = dbConfig.Version
+
+		runtimeConfig, err := MergeDatabaseConfigWithDefaults(h.server.Config, &dbConfig.DbConfig)
+		if err != nil {
+			return err
+		}
+
+		// grab runtime version of config, so that we can see what events would be enabled
+		if runtimeConfig.Logging != nil && runtimeConfig.Logging.Audit != nil {
+			dbAuditEnabled = base.BoolDefault(runtimeConfig.Logging.Audit.Enabled, false)
+			for _, event := range runtimeConfig.Logging.Audit.EnabledEvents {
+				enabledEvents[base.AuditID(event)] = struct{}{}
+			}
+		}
+	} else {
+		return base.HTTPErrorf(http.StatusServiceUnavailable, "audit config not available in non-persistent mode")
 	}
 
-	// TODO: Move to structs
-	events := make(map[string]interface{}, len(base.AuditEvents))
+	events := make(map[string]any, len(base.AuditEvents))
 	for id, descriptor := range base.AuditEvents {
 		if showOnlyFilterable && !descriptor.FilteringPermitted {
 			continue
 		}
+
 		idStr := id.String()
+		_, eventEnabled := enabledEvents[id]
+
 		if verbose {
-			events[idStr] = map[string]interface{}{
-				"name":        descriptor.Name,
-				"description": descriptor.Description,
-				"enabled":     isEnabledFn(id),
-				"filterable":  descriptor.FilteringPermitted,
+			events[idStr] = handleDbAuditConfigBodyVerboseEvent{
+				Name:        stringPtrOrNil(descriptor.Name),
+				Description: stringPtrOrNil(descriptor.Description),
+				Enabled:     &eventEnabled,
+				Filterable:  base.BoolPtr(descriptor.FilteringPermitted),
 			}
 		} else {
-			events[idStr] = isEnabledFn(id)
+			events[idStr] = &eventEnabled
 		}
 	}
 
-	h.writeJSON(events)
+	resp := handleDbAuditConfigBody{
+		Enabled: &dbAuditEnabled,
+		Events:  events,
+	}
 
+	h.setEtag(etagVersion)
+	h.writeJSON(resp)
 	return nil
 }
 
 // PUT/POST audit config for database
 func (h *handler) handlePutDbAuditConfig() error {
-	h.assertAdminOnly()
+	return h.mutateDbConfig(func(config *DbConfig) error {
+		var body handleDbAuditConfigBody
+		if err := h.readJSONInto(&body); err != nil {
+			return err
+		}
 
-	// interface can be either bool or object for verbose-format
-	var body map[string]interface{}
-	if err := h.readJSONInto(&body); err != nil {
-		return err
-	}
+		// isReplace if the request is a PUT, and we want to overwrite existing config
+		isReplace := h.rq.Method == http.MethodPut
 
-	return nil
+		// This API endpoint takes audit config in a format that does not match the actual DbConfig stored, so translate the request here.
+		toChange := make(map[base.AuditID]bool, len(body.Events))
+		var multiError *base.MultiError
+		for id, val := range body.Events {
+			// find the event
+			auditID, err := base.ParseAuditID(id)
+			if err != nil {
+				multiError = multiError.Append(fmt.Errorf("invalid audit event ID: %q", id))
+				continue
+			}
+			_, ok := base.AuditEvents[auditID]
+			if !ok {
+				multiError = multiError.Append(fmt.Errorf("unknown audit event ID: %q", id))
+				continue
+			}
+
+			var eventEnabled bool
+			switch valT := val.(type) {
+			case bool:
+				eventEnabled = valT
+			case map[string]any:
+				// verbose format
+				eventEnabled = valT["enabled"].(bool)
+			}
+
+			toChange[auditID] = eventEnabled
+		}
+		if err := multiError.ErrorOrNil(); err != nil {
+			return base.HTTPErrorf(http.StatusBadRequest, "couldn't update audit configuration: %s", err)
+		}
+
+		if config.Logging == nil {
+			config.Logging = &DbLoggingConfig{}
+		}
+		if config.Logging.Audit == nil {
+			config.Logging.Audit = &DbAuditLoggingConfig{}
+		}
+
+		if isReplace || body.Enabled != nil {
+			config.Logging.Audit.Enabled = body.Enabled
+		}
+
+		if isReplace {
+			// we don't need to do anything to "disable" events, other than not enable them
+			config.Logging.Audit.EnabledEvents = func() []uint {
+				enabledEvents := make([]uint, 0)
+				for event, shouldEnable := range toChange {
+					if shouldEnable {
+						enabledEvents = append(enabledEvents, uint(event))
+					}
+				}
+				return enabledEvents
+			}()
+		} else {
+			for i, event := range config.Logging.Audit.EnabledEvents {
+				if shouldEnable, ok := toChange[base.AuditID(event)]; ok {
+					if shouldEnable {
+						// already enabled
+						continue
+					} else {
+						// disable by removing
+						config.Logging.Audit.EnabledEvents = append(config.Logging.Audit.EnabledEvents[:i], config.Logging.Audit.EnabledEvents[i+1:]...)
+					}
+					// drop from toChange so we don't duplicate IDs
+					delete(toChange, base.AuditID(event))
+				}
+			}
+			for id, enabled := range toChange {
+				if enabled {
+					config.Logging.Audit.EnabledEvents = append(config.Logging.Audit.EnabledEvents, uint(id))
+				}
+			}
+		}
+		return nil
+	})
 }
 
 // GET collection config sync function

rest/adminapitest/admin_api_test.go

@@ -562,7 +562,7 @@ func TestDBGetConfigNamesAndDefaultLogging(t *testing.T) {
 	require.NoError(t, base.JSONUnmarshal(response.Body.Bytes(), &body))
 
 	assert.Equal(t, len(rt.DatabaseConfig.Users), len(body.Users))
-	emptyCnf := &rest.DbLoggingConfig{}
+	emptyCnf := rest.DefaultPerDBLogging(rt.ServerContext().Config.Logging)
 	assert.Equal(t, body.Logging, emptyCnf)
 
 	for k, v := range body.Users {
@@ -4263,3 +4263,51 @@ func TestDatabaseCreationWithEnvVariableWithBackticks(t *testing.T) {
 	resp := rt.SendAdminRequestWithAuth(http.MethodPut, "/backticks/", string(input), rest.MobileSyncGatewayRole.RoleName, "password")
 	rest.RequireStatus(t, resp, http.StatusCreated)
 }
+
+func TestDatabaseConfigAuditAPI(t *testing.T) {
+	if !base.IsEnterpriseEdition() {
+		t.Skip("Audit logging is an EE-only feature")
+	}
+
+	rt := rest.NewRestTesterPersistentConfig(t)
+	defer rt.Close()
+
+	// check default audit config - verbose to read event names, etc.
+	resp := rt.SendAdminRequest(http.MethodGet, "/db/_config/audit?verbose=true", "")
+	rest.RequireStatus(t, resp, http.StatusOK)
+	resp.DumpBody()
+	var responseBody map[string]interface{}
+	require.NoError(t, json.Unmarshal(resp.Body.Bytes(), &responseBody))
+	assert.Equal(t, false, responseBody["enabled"].(bool))
+	// check we got the verbose output
+	assert.NotEmpty(t, responseBody["events"].(map[string]interface{})[base.AuditIDPublicUserAuthenticated.String()].(map[string]interface{})["description"].(string), "expected verbose output (event description, etc.)")
+
+	// enable auditing on the database (upsert)
+	resp = rt.SendAdminRequest(http.MethodPost, "/db/_config/audit", `{"enabled":true}`)
+	rest.RequireStatus(t, resp, http.StatusOK)
+
+	// check audit config
+	resp = rt.SendAdminRequest(http.MethodGet, "/db/_config/audit", "")
+	rest.RequireStatus(t, resp, http.StatusOK)
+	resp.DumpBody()
+	responseBody = nil
+	require.NoError(t, json.Unmarshal(resp.Body.Bytes(), &responseBody))
+	assert.Equal(t, true, responseBody["enabled"].(bool))
+	assert.False(t, responseBody["events"].(map[string]interface{})[base.AuditIDAuditEnabled.String()].(bool), "audit enabled event should be disabled by default") // TODO: This will change - replace with an actual non-default event.
+	assert.True(t, responseBody["events"].(map[string]interface{})[base.AuditIDPublicUserAuthenticated.String()].(bool), "public user authenticated event should be enabled by default")
+
+	// do a PUT to completely replace the full config (events not declared here will be disabled)
+	// enable AuditEnabled event, but implicitly others
+	resp = rt.SendAdminRequest(http.MethodPost, "/db/_config/audit", fmt.Sprintf(`{"enabled":true,"events":{"%s":true}}`, base.AuditIDAuditEnabled))
+	rest.RequireStatus(t, resp, http.StatusOK)
+
+	// check audit config
+	resp = rt.SendAdminRequest(http.MethodGet, "/db/_config/audit", "")
+	rest.RequireStatus(t, resp, http.StatusOK)
+	resp.DumpBody()
+	responseBody = nil
+	require.NoError(t, json.Unmarshal(resp.Body.Bytes(), &responseBody))
+	assert.Equal(t, true, responseBody["enabled"].(bool))
+	assert.True(t, responseBody["events"].(map[string]interface{})[base.AuditIDAuditEnabled.String()].(bool), "audit enabled event should've been enabled via PUT")
+	assert.False(t, responseBody["events"].(map[string]interface{})[base.AuditIDPublicUserAuthenticated.String()].(bool), "public user authenticated event should've been disabled via PUT")
+}

rest/config.go

@@ -1065,6 +1065,21 @@ func (dbConfig *DbConfig) validateVersion(ctx context.Context, isEnterpriseEditi
 			}
 		}
 	}
+
+	if dbConfig.Logging != nil {
+		if dbConfig.Logging.Audit != nil {
+			if !isEnterpriseEdition && dbConfig.Logging.Audit.Enabled != nil {
+				base.WarnfCtx(ctx, eeOnlyWarningMsg, "logging.audit.enabled", *dbConfig.Logging.Audit.Enabled, false)
+				dbConfig.Logging.Audit.Enabled = nil
+			}
+			for _, id := range dbConfig.Logging.Audit.EnabledEvents {
+				if _, ok := base.AuditEvents[base.AuditID(id)]; !ok {
+					multiError = multiError.Append(fmt.Errorf("unknown audit event ID %q", id))
+				}
+			}
+		}
+	}
+
 	return multiError.ErrorOrNil()
 }
 
@@ -2206,6 +2221,7 @@ func (c *DbConfig) toDbLogConfig(ctx context.Context) *base.DbLogConfig {
 			enabledEvents[base.AuditID(event)] = struct{}{}
 		}
 		aud = &base.DbAuditLogConfig{
+			Enabled:       base.BoolDefault(l.Audit.Enabled, false),
 			EnabledEvents: enabledEvents,
 		}
 	}

rest/config_database.go

@@ -99,11 +99,9 @@ func DefaultPerDBLogging(bootstrapLoggingCnf base.LoggingConfig) *DbLoggingConfi
 			}
 		}
 	}
-	if bootstrapLoggingCnf.Audit != nil {
-		dblc.Audit = &DbAuditLoggingConfig{
-			Enabled:       base.BoolPtr(false),
-			EnabledEvents: base.DefaultAuditEventIDs,
-		}
+	dblc.Audit = &DbAuditLoggingConfig{
+		Enabled:       base.BoolPtr(false),
+		EnabledEvents: base.DefaultAuditEventIDs,
 	}
 	return dblc
 }
@@ -192,7 +190,6 @@ func DefaultDbConfig(sc *StartupConfig, useXattrs bool) *DbConfig {
 		if base.IsEnterpriseEdition() {
 			dbConfig.ImportPartitions = base.Uint16Ptr(base.GetDefaultImportPartitions(sc.IsServerless()))
 		}
-
 	} else {
 		dbConfig.AutoImport = base.BoolPtr(false)
 	}