Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add acme clients internal data structures and adjust tests #26020

Merged
merged 4 commits into from
Mar 19, 2024
Merged

Add acme clients internal data structures and adjust tests #26020

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
907aa75
add acme clients internal data structures and adjust tests
raskchanky Mar 18, 2024
3e68ad3
fix another acme test
raskchanky Mar 18, 2024
cc3a80f
replace manual list with ActivityClientTypes
raskchanky Mar 19, 2024
d9987d6
add changelog
raskchanky Mar 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions builtin/logical/pki/acme_billing_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,10 +131,10 @@ func validateClientCount(t *testing.T, client *api.Client, mount string, expecte

require.NotNil(t, resp)
require.NotNil(t, resp.Data)
require.Contains(t, resp.Data, "non_entity_clients")
require.Contains(t, resp.Data, "acme_clients")
require.Contains(t, resp.Data, "months")

rawCount := resp.Data["non_entity_clients"].(json.Number)
rawCount := resp.Data["acme_clients"].(json.Number)
count, err := rawCount.Int64()
require.NoError(t, err, "failed to parse number as int64: "+rawCount.String())

Expand All @@ -158,8 +158,8 @@ func validateClientCount(t *testing.T, client *api.Client, mount string, expecte
// Validate this month's aggregate counts match the overall value.
require.Contains(t, monthlyInfo, "counts", "expected monthly info to contain a count key")
monthlyCounts := monthlyInfo["counts"].(map[string]interface{})
require.Contains(t, monthlyCounts, "non_entity_clients", "expected month[0].counts to contain a non_entity_clients key")
monthlyCountNonEntityRaw := monthlyCounts["non_entity_clients"].(json.Number)
require.Contains(t, monthlyCounts, "acme_clients", "expected month[0].counts to contain a non_entity_clients key")
monthlyCountNonEntityRaw := monthlyCounts["acme_clients"].(json.Number)
monthlyCountNonEntity, err := monthlyCountNonEntityRaw.Int64()
require.NoError(t, err, "failed to parse number as int64: "+monthlyCountNonEntityRaw.String())
require.Equal(t, count, monthlyCountNonEntity, "expected equal values for non entity client counts")
Expand Down Expand Up @@ -194,8 +194,8 @@ func validateClientCount(t *testing.T, client *api.Client, mount string, expecte
// This namespace must have a non-empty aggregate non-entity count.
require.Contains(t, namespace, "counts", "expected monthly.namespaces[%v] to contain a counts key", index)
namespaceCounts := namespace["counts"].(map[string]interface{})
require.Contains(t, namespaceCounts, "non_entity_clients", "expected namespace counts to contain a non_entity_clients key")
namespaceCountNonEntityRaw := namespaceCounts["non_entity_clients"].(json.Number)
require.Contains(t, namespaceCounts, "acme_clients", "expected namespace counts to contain a non_entity_clients key")
namespaceCountNonEntityRaw := namespaceCounts["acme_clients"].(json.Number)
namespaceCountNonEntity, err := namespaceCountNonEntityRaw.Int64()
require.NoError(t, err, "failed to parse number as int64: "+namespaceCountNonEntityRaw.String())
require.Greater(t, namespaceCountNonEntity, int64(0), "expected at least one non-entity client count value in the namespace")
Expand All @@ -217,8 +217,8 @@ func validateClientCount(t *testing.T, client *api.Client, mount string, expecte
// This mount must also have a non-empty non-entity client count.
require.Contains(t, mountInfo, "counts", "expected monthly.namespaces[%v].mounts[%v] to contain a counts key", index, mountIndex)
mountCounts := mountInfo["counts"].(map[string]interface{})
require.Contains(t, mountCounts, "non_entity_clients", "expected mount counts to contain a non_entity_clients key")
mountCountNonEntityRaw := mountCounts["non_entity_clients"].(json.Number)
require.Contains(t, mountCounts, "acme_clients", "expected mount counts to contain a non_entity_clients key")
mountCountNonEntityRaw := mountCounts["acme_clients"].(json.Number)
mountCountNonEntity, err := mountCountNonEntityRaw.Int64()
require.NoError(t, err, "failed to parse number as int64: "+mountCountNonEntityRaw.String())
require.Greater(t, mountCountNonEntity, int64(0), "expected at least one non-entity client count value in the mount")
Expand Down
4 changes: 3 additions & 1 deletion vault/activity/query.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,17 +24,19 @@ type NamespaceRecord struct {
NonEntityTokens uint64 `json:"non_entity_tokens"`
SecretSyncs uint64 `json:"secret_syncs"`
Mounts []*MountRecord `json:"mounts"`
ACMEClients uint64 `json:"acme_clients"`
}

type CountsRecord struct {
EntityClients int `json:"entity_clients"`
NonEntityClients int `json:"non_entity_clients"`
SecretSyncs int `json:"secret_syncs"`
ACMEClients int `json:"acme_clients"`
}

// HasCounts returns true when any of the record's fields have a non-zero value
func (c *CountsRecord) HasCounts() bool {
return c.EntityClients+c.NonEntityClients+c.SecretSyncs != 0
return c.EntityClients+c.NonEntityClients+c.SecretSyncs+c.ACMEClients != 0
}

type NewClientRecord struct {
Expand Down
19 changes: 7 additions & 12 deletions vault/activity_log.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ const (
secretSyncActivityType = "secret-sync"
)

var ActivityClientTypes = []string{nonEntityTokenActivityType, entityActivityType, secretSyncActivityType, ACMEActivityType}

type segmentInfo struct {
startTimestamp int64
currentClients *activity.EntityActivityLog
Expand Down Expand Up @@ -1573,6 +1575,7 @@ type ResponseCounts struct {
NonEntityClients int `json:"non_entity_clients" mapstructure:"non_entity_clients"`
Clients int `json:"clients"`
SecretSyncs int `json:"secret_syncs" mapstructure:"secret_syncs"`
ACMEClients int `json:"acme_clients" mapstructure:"acme_clients"`
}

// Add adds the new record's counts to the existing record
Expand All @@ -1585,6 +1588,7 @@ func (r *ResponseCounts) Add(newRecord *ResponseCounts) {
r.DistinctEntities += newRecord.DistinctEntities
r.NonEntityClients += newRecord.NonEntityClients
r.NonEntityTokens += newRecord.NonEntityTokens
r.ACMEClients += newRecord.ACMEClients
r.SecretSyncs += newRecord.SecretSyncs
}

Expand Down Expand Up @@ -2035,6 +2039,7 @@ func (p *processCounts) toCountsRecord() *activity.CountsRecord {
return &activity.CountsRecord{
EntityClients: p.countByType(entityActivityType),
NonEntityClients: p.countByType(nonEntityTokenActivityType),
ACMEClients: p.countByType(ACMEActivityType),
SecretSyncs: p.countByType(secretSyncActivityType),
}
}
Expand All @@ -2045,25 +2050,14 @@ func (p *processCounts) toCountsRecord() *activity.CountsRecord {
func (p *processCounts) countByType(typ string) int {
switch typ {
case nonEntityTokenActivityType:
return len(p.ClientsByType[nonEntityTokenActivityType]) + int(p.Tokens) + len(p.ClientsByType[ACMEActivityType])
return len(p.ClientsByType[nonEntityTokenActivityType]) + int(p.Tokens)
}
return len(p.ClientsByType[typ])
}

// clientsByType returns the set of client IDs with the given type.
// ACME clients are included in the non entity results
func (p *processCounts) clientsByType(typ string) clientIDSet {
switch typ {
case nonEntityTokenActivityType:
clients := make(clientIDSet)
for c := range p.ClientsByType[nonEntityTokenActivityType] {
clients[c] = struct{}{}
}
for c := range p.ClientsByType[ACMEActivityType] {
clients[c] = struct{}{}
}
return clients
}
return p.ClientsByType[typ]
}

Expand Down Expand Up @@ -2792,6 +2786,7 @@ func (a *ActivityLog) partialMonthClientCount(ctx context.Context) (map[string]i
responseData["non_entity_clients"] = totalCounts.NonEntityClients
responseData["clients"] = totalCounts.Clients
responseData["secret_syncs"] = totalCounts.SecretSyncs
responseData["acme_clients"] = totalCounts.ACMEClients

// The partialMonthClientCount should not have more than one month worth of data.
// If it does, something has gone wrong and we should warn that the activity log data
Expand Down
82 changes: 57 additions & 25 deletions vault/activity_log_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4219,14 +4219,14 @@ func TestActivityLog_partialMonthClientCountWithMultipleMountPaths(t *testing.T)
// TestActivityLog_processNewClients_delete ensures that the correct clients are deleted from a processNewClients struct
func TestActivityLog_processNewClients_delete(t *testing.T) {
mount := "mount"
namespace := "namespace"
ns := "namespace"
clientID := "client-id"
run := func(t *testing.T, clientType string) {
t.Helper()
isNonEntity := clientType == nonEntityTokenActivityType || clientType == ACMEActivityType
record := &activity.EntityRecord{
MountAccessor: mount,
NamespaceID: namespace,
NamespaceID: ns,
ClientID: clientID,
NonEntity: isNonEntity,
ClientType: clientType,
Expand All @@ -4235,17 +4235,17 @@ func TestActivityLog_processNewClients_delete(t *testing.T) {
newClients.add(record)

require.True(t, newClients.Counts.contains(record))
require.True(t, newClients.Namespaces[namespace].Counts.contains(record))
require.True(t, newClients.Namespaces[namespace].Mounts[mount].Counts.contains(record))
require.True(t, newClients.Namespaces[ns].Counts.contains(record))
require.True(t, newClients.Namespaces[ns].Mounts[mount].Counts.contains(record))

newClients.delete(record)

byNS := newClients.Namespaces
counts := newClients.Counts
for _, typ := range []string{nonEntityTokenActivityType, secretSyncActivityType, entityActivityType, ACMEActivityType} {
require.NotContains(t, counts.clientsByType(typ), clientID)
require.NotContains(t, byNS[namespace].Mounts[mount].Counts.clientsByType(typ), clientID)
require.NotContains(t, byNS[namespace].Counts.clientsByType(typ), clientID)
require.NotContains(t, byNS[ns].Mounts[mount].Counts.clientsByType(typ), clientID)
require.NotContains(t, byNS[ns].Counts.clientsByType(typ), clientID)
}
}
t.Run("entity", func(t *testing.T) {
Expand All @@ -4267,43 +4267,44 @@ func TestActivityLog_processNewClients_delete(t *testing.T) {
func TestActivityLog_processClientRecord(t *testing.T) {
startTime := time.Now()
mount := "mount"
namespace := "namespace"
ns := "namespace"
clientID := "client-id"

run := func(t *testing.T, clientType string) {
t.Helper()
isNonEntity := clientType == nonEntityTokenActivityType || clientType == ACMEActivityType
record := &activity.EntityRecord{
MountAccessor: mount,
NamespaceID: namespace,
NamespaceID: ns,
ClientID: clientID,
NonEntity: isNonEntity,
ClientType: clientType,
}
byNS := make(summaryByNamespace)
byMonth := make(summaryByMonth)
processClientRecord(record, byNS, byMonth, startTime)
require.Contains(t, byNS, namespace)
require.Contains(t, byNS[namespace].Mounts, mount)
require.Contains(t, byNS, ns)
require.Contains(t, byNS[ns].Mounts, mount)
monthIndex := timeutil.StartOfMonth(startTime).UTC().Unix()
require.Contains(t, byMonth, monthIndex)
require.Equal(t, byMonth[monthIndex].Namespaces, byNS)
require.Equal(t, byMonth[monthIndex].NewClients.Namespaces, byNS)

for _, typ := range []string{nonEntityTokenActivityType, secretSyncActivityType, entityActivityType} {
if clientType == typ || (clientType == ACMEActivityType && typ == nonEntityTokenActivityType) {
for _, typ := range []string{nonEntityTokenActivityType, secretSyncActivityType, entityActivityType, ACMEActivityType} {
raskchanky marked this conversation as resolved.
Show resolved Hide resolved
if clientType == typ {
require.Contains(t, byMonth[monthIndex].Counts.clientsByType(typ), clientID)
require.Contains(t, byMonth[monthIndex].NewClients.Counts.clientsByType(typ), clientID)
require.Contains(t, byNS[namespace].Mounts[mount].Counts.clientsByType(typ), clientID)
require.Contains(t, byNS[namespace].Counts.clientsByType(typ), clientID)
require.Contains(t, byNS[ns].Mounts[mount].Counts.clientsByType(typ), clientID)
require.Contains(t, byNS[ns].Counts.clientsByType(typ), clientID)
} else {
require.NotContains(t, byMonth[monthIndex].Counts.clientsByType(typ), clientID)
require.NotContains(t, byMonth[monthIndex].NewClients.Counts.clientsByType(typ), clientID)
require.NotContains(t, byNS[namespace].Mounts[mount].Counts.clientsByType(typ), clientID)
require.NotContains(t, byNS[namespace].Counts.clientsByType(typ), clientID)

require.NotContains(t, byNS[ns].Mounts[mount].Counts.clientsByType(typ), clientID)
require.NotContains(t, byNS[ns].Counts.clientsByType(typ), clientID)
}
}
}

t.Run("non entity", func(t *testing.T) {
run(t, nonEntityTokenActivityType)
})
Expand Down Expand Up @@ -4600,13 +4601,20 @@ func TestActivityLog_writePrecomputedQuery(t *testing.T) {
MountAccessor: "mnt-3",
ClientType: secretSyncActivityType,
}
acmeClient := &activity.EntityRecord{
ClientID: "id-4",
NamespaceID: "ns-4",
MountAccessor: "mnt-4",
ClientType: ACMEActivityType,
}

now := time.Now()

// add the 2 clients to the namespace and month summaries
processClientRecord(clientEntity, byNS, byMonth, now)
processClientRecord(clientNonEntity, byNS, byMonth, now)
processClientRecord(secretSync, byNS, byMonth, now)
processClientRecord(acmeClient, byNS, byMonth, now)

endTime := timeutil.EndOfMonth(now)
opts := pqOptions{
Expand All @@ -4624,8 +4632,8 @@ func TestActivityLog_writePrecomputedQuery(t *testing.T) {
require.Equal(t, now.UTC().Unix(), val.StartTime.UTC().Unix())
require.Equal(t, endTime.UTC().Unix(), val.EndTime.UTC().Unix())

// ns-1, ns-2, and ns-3 should both be present in the results
require.Len(t, val.Namespaces, 3)
// ns-1, ns-2, ns-3, ns-4 should all be present in the results
require.Len(t, val.Namespaces, 4)
require.Len(t, val.Months, 1)
resultByNS := make(map[string]*activity.NamespaceRecord)
for _, ns := range val.Namespaces {
Expand All @@ -4634,53 +4642,76 @@ func TestActivityLog_writePrecomputedQuery(t *testing.T) {
ns1 := resultByNS["ns-1"]
ns2 := resultByNS["ns-2"]
ns3 := resultByNS["ns-3"]
ns4 := resultByNS["ns-4"]

require.Equal(t, ns1.Entities, uint64(1))
require.Equal(t, ns1.NonEntityTokens, uint64(0))
require.Equal(t, ns1.SecretSyncs, uint64(0))
require.Equal(t, ns1.ACMEClients, uint64(0))
require.Equal(t, ns2.Entities, uint64(0))
require.Equal(t, ns2.NonEntityTokens, uint64(1))
require.Equal(t, ns2.SecretSyncs, uint64(0))
require.Equal(t, ns2.ACMEClients, uint64(0))
require.Equal(t, ns3.Entities, uint64(0))
require.Equal(t, ns3.NonEntityTokens, uint64(0))
require.Equal(t, ns3.SecretSyncs, uint64(1))
require.Equal(t, ns3.ACMEClients, uint64(0))
require.Equal(t, ns4.Entities, uint64(0))
require.Equal(t, ns4.NonEntityTokens, uint64(0))
require.Equal(t, ns4.SecretSyncs, uint64(0))
require.Equal(t, ns4.ACMEClients, uint64(1))

require.Len(t, ns1.Mounts, 1)
require.Len(t, ns2.Mounts, 1)
require.Len(t, ns3.Mounts, 1)
require.Len(t, ns4.Mounts, 1)

// ns-1 needs to have mnt-1
require.Contains(t, ns1.Mounts[0].MountPath, "mnt-1")
// ns-2 needs to have mnt-2
require.Contains(t, ns2.Mounts[0].MountPath, "mnt-2")
// ns-3 needs to have mnt-3
require.Contains(t, ns3.Mounts[0].MountPath, "mnt-3")
// ns-4 needs to have mnt-4
require.Contains(t, ns4.Mounts[0].MountPath, "mnt-4")

// ns1 only has an entity client
require.Equal(t, 1, ns1.Mounts[0].Counts.EntityClients)
require.Equal(t, 0, ns1.Mounts[0].Counts.NonEntityClients)
require.Equal(t, 0, ns1.Mounts[0].Counts.SecretSyncs)
require.Equal(t, 0, ns1.Mounts[0].Counts.ACMEClients)

// ns2 only has a non entity client
require.Equal(t, 0, ns2.Mounts[0].Counts.EntityClients)
require.Equal(t, 1, ns2.Mounts[0].Counts.NonEntityClients)
require.Equal(t, 0, ns2.Mounts[0].Counts.SecretSyncs)
require.Equal(t, 0, ns2.Mounts[0].Counts.ACMEClients)

// ns3 only has a secret sync association
require.Equal(t, 0, ns3.Mounts[0].Counts.EntityClients)
require.Equal(t, 0, ns3.Mounts[0].Counts.NonEntityClients)
require.Equal(t, 1, ns3.Mounts[0].Counts.SecretSyncs)
require.Equal(t, 0, ns3.Mounts[0].Counts.ACMEClients)

// ns4 only has an ACME client
require.Equal(t, 0, ns4.Mounts[0].Counts.EntityClients)
require.Equal(t, 0, ns4.Mounts[0].Counts.NonEntityClients)
require.Equal(t, 0, ns4.Mounts[0].Counts.SecretSyncs)
require.Equal(t, 1, ns4.Mounts[0].Counts.ACMEClients)

monthRecord := val.Months[0]
// there should only be one month present, since the clients were added with the same timestamp
require.Equal(t, monthRecord.Timestamp, timeutil.StartOfMonth(now).UTC().Unix())
require.Equal(t, 1, monthRecord.Counts.NonEntityClients)
require.Equal(t, 1, monthRecord.Counts.EntityClients)
require.Equal(t, 1, monthRecord.Counts.SecretSyncs)
require.Len(t, monthRecord.Namespaces, 3)
require.Len(t, monthRecord.NewClients.Namespaces, 3)
require.Equal(t, 1, monthRecord.Counts.ACMEClients)
require.Len(t, monthRecord.Namespaces, 4)
require.Len(t, monthRecord.NewClients.Namespaces, 4)
require.Equal(t, 1, monthRecord.NewClients.Counts.EntityClients)
require.Equal(t, 1, monthRecord.NewClients.Counts.NonEntityClients)
require.Equal(t, 1, monthRecord.NewClients.Counts.SecretSyncs)
require.Equal(t, 1, monthRecord.NewClients.Counts.ACMEClients)
}

type mockTimeNowClock struct {
Expand Down Expand Up @@ -4759,9 +4790,9 @@ func TestAddActivityToFragment(t *testing.T) {
a.SetEnable(true)

mount := "mount"
namespace := "root"
ns := "root"
id := "id1"
a.AddActivityToFragment(id, namespace, 0, entityActivityType, mount)
a.AddActivityToFragment(id, ns, 0, entityActivityType, mount)

testCases := []struct {
name string
Expand Down Expand Up @@ -4810,13 +4841,14 @@ func TestAddActivityToFragment(t *testing.T) {
isNonEntity: true,
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
a.fragmentLock.RLock()
numClientsBefore := len(a.fragment.Clients)
a.fragmentLock.RUnlock()

a.AddActivityToFragment(tc.id, namespace, 0, tc.activityType, mount)
a.AddActivityToFragment(tc.id, ns, 0, tc.activityType, mount)
a.fragmentLock.RLock()
defer a.fragmentLock.RUnlock()
numClientsAfter := len(a.fragment.Clients)
Expand All @@ -4830,7 +4862,7 @@ func TestAddActivityToFragment(t *testing.T) {
require.Contains(t, a.partialMonthClientTracker, tc.expectedID)
require.True(t, proto.Equal(&activity.EntityRecord{
ClientID: tc.expectedID,
NamespaceID: namespace,
NamespaceID: ns,
Timestamp: 0,
NonEntity: tc.isNonEntity,
MountAccessor: mount,
Expand Down
Loading
Loading