Skip to content

Commit

Permalink
VAULT-22642: Include secret syncs in activity log responses (#24710)
Browse files Browse the repository at this point in the history 
* refactor and include secret sync

* add secret sync tests

* changelog

* include secret syncs in clients

* pr comments

* add godocs
  • Loading branch information
@miagilepner
miagilepner authored Jan 15, 2024
1 parent bc9be55 commit caef593
Show file tree
Hide file tree
Showing 5 changed files with 264 additions and 103 deletions.
3 changes: 3 additions & 0 deletions changelog/24710.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:improvement
core/activity: Include secret_syncs in activity log responses
```
5 changes: 5 additions & 0 deletions vault/activity/query.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@ type CountsRecord struct {
SecretSyncs int `json:"secret_syncs"`
}

// HasCounts returns true when any of the record's fields have a non-zero value
func (c *CountsRecord) HasCounts() bool {
return c.EntityClients+c.NonEntityClients+c.SecretSyncs != 0
}

type NewClientRecord struct {
Counts *CountsRecord `json:"counts"`
Namespaces []*MonthlyNamespaceRecord `json:"namespaces"`
Expand Down
172 changes: 75 additions & 97 deletions vault/activity_log.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1578,6 +1578,20 @@ type ResponseCounts struct {
NonEntityTokens int `json:"non_entity_tokens" mapstructure:"non_entity_tokens"`
NonEntityClients int `json:"non_entity_clients" mapstructure:"non_entity_clients"`
Clients int `json:"clients"`
SecretSyncs int `json:"secret_syncs" mapstructure:"secret_syncs"`
}

// Add adds the new record's counts to the existing record
func (r *ResponseCounts) Add(newRecord *ResponseCounts) {
if newRecord == nil {
return
}
r.EntityClients += newRecord.EntityClients
r.Clients += newRecord.Clients
r.DistinctEntities += newRecord.DistinctEntities
r.NonEntityClients += newRecord.NonEntityClients
r.NonEntityTokens += newRecord.NonEntityTokens
r.SecretSyncs += newRecord.SecretSyncs
}

type ResponseNamespace struct {
Expand All @@ -1587,6 +1601,30 @@ type ResponseNamespace struct {
Mounts []*ResponseMount `json:"mounts"`
}

// Add adds the namespace counts to the existing record, then either adds the
// mount counts to the existing mount (if it exists) or appends the mount to the
// list of mounts
func (r *ResponseNamespace) Add(newRecord *ResponseNamespace) {
// Create a map of the existing mounts, so we don't duplicate them
mountMap := make(map[string]*ResponseCounts)
for _, erm := range r.Mounts {
mountMap[erm.MountPath] = erm.Counts
}

r.Counts.Add(&newRecord.Counts)

// Check the current month mounts against the existing mounts and if there are matches, update counts
// accordingly. If there is no match, append the new mount to the existing mounts, so it will be counted
// later.
for _, newRecordMount := range newRecord.Mounts {
if existingRecordMountCounts, ok := mountMap[newRecordMount.MountPath]; ok {
existingRecordMountCounts.Add(&newRecord.Counts)
} else {
r.Mounts = append(r.Mounts, newRecordMount)
}
}
}

type ResponseMonth struct {
Timestamp string `json:"timestamp"`
Counts *ResponseCounts `json:"counts"`
Expand Down Expand Up @@ -1681,7 +1719,7 @@ func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.T

// Calculate the namespace response breakdowns and totals for entities and tokens from the initial
// namespace data.
totalEntities, totalTokens, byNamespaceResponse, err := a.calculateByNamespaceResponseForQuery(ctx, pq.Namespaces)
totalCounts, byNamespaceResponse, err := a.calculateByNamespaceResponseForQuery(ctx, pq.Namespaces)
if err != nil {
return nil, err
}
Expand All @@ -1690,9 +1728,8 @@ func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.T
// breakdown for the current month as well.
var partialByMonth map[int64]*processMonth
var partialByNamespace map[string]*processByNamespace
var totalEntitiesCurrent int
var totalTokensCurrent int
var byNamespaceResponseCurrent []*ResponseNamespace
var totalCurrentCounts *ResponseCounts
if computePartial {
// Traverse through current month's activitylog data and group clients
// into months and namespaces
Expand All @@ -1705,9 +1742,9 @@ func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.T
// endpoints.
byNamespaceComputation := a.transformALNamespaceBreakdowns(partialByNamespace)

// Calculate the namespace response breakdowns and totals for entities and tokens from the initial
// namespace data.
totalEntitiesCurrent, totalTokensCurrent, byNamespaceResponseCurrent, err = a.calculateByNamespaceResponseForQuery(ctx, byNamespaceComputation)
// Calculate the namespace response breakdowns and totals for entities
// and tokens from current month namespace data.
totalCurrentCounts, byNamespaceResponseCurrent, err = a.calculateByNamespaceResponseForQuery(ctx, byNamespaceComputation)
if err != nil {
return nil, err
}
Expand All @@ -1723,34 +1760,7 @@ func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.T
// month counts, and append or update as necessary. We also want to account for mounts and their counts.
for _, nrc := range byNamespaceResponseCurrent {
if ndx, ok := nsrMap[nrc.NamespaceID]; ok {
existingRecord := byNamespaceResponse[ndx]

// Create a map of the existing mounts, so we don't duplicate them
mountMap := make(map[string]*ResponseCounts)
for _, erm := range existingRecord.Mounts {
mountMap[erm.MountPath] = erm.Counts
}

existingRecord.Counts.EntityClients += nrc.Counts.EntityClients
existingRecord.Counts.Clients += nrc.Counts.Clients
existingRecord.Counts.DistinctEntities += nrc.Counts.DistinctEntities
existingRecord.Counts.NonEntityClients += nrc.Counts.NonEntityClients
existingRecord.Counts.NonEntityTokens += nrc.Counts.NonEntityTokens

// Check the current month mounts against the existing mounts and if there are matches, update counts
// accordingly. If there is no match, append the new mount to the existing mounts, so it will be counted
// later.
for _, nrcMount := range nrc.Mounts {
if existingRecordMountCounts, ook := mountMap[nrcMount.MountPath]; ook {
existingRecordMountCounts.EntityClients += nrcMount.Counts.EntityClients
existingRecordMountCounts.Clients += nrcMount.Counts.Clients
existingRecordMountCounts.DistinctEntities += nrcMount.Counts.DistinctEntities
existingRecordMountCounts.NonEntityClients += nrcMount.Counts.NonEntityClients
existingRecordMountCounts.NonEntityTokens += nrcMount.Counts.NonEntityTokens
} else {
existingRecord.Mounts = append(existingRecord.Mounts, nrcMount)
}
}
byNamespaceResponse[ndx].Add(nrc)
} else {
byNamespaceResponse = append(byNamespaceResponse, nrc)
}
Expand All @@ -1761,10 +1771,10 @@ func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.T
a.sortALResponseNamespaces(byNamespaceResponse)

if limitNamespaces > 0 {
totalEntities, totalTokens, byNamespaceResponse = a.limitNamespacesInALResponse(byNamespaceResponse, limitNamespaces)
totalCounts, byNamespaceResponse = a.limitNamespacesInALResponse(byNamespaceResponse, limitNamespaces)
}

distinctEntitiesResponse := totalEntities
distinctEntitiesResponse := totalCounts.EntityClients
if computePartial {
currentMonth, err := a.computeCurrentMonthForBillingPeriod(ctx, partialByMonth, startTime, endTime)
if err != nil {
Expand Down Expand Up @@ -1808,13 +1818,9 @@ func (a *ActivityLog) handleQuery(ctx context.Context, startTime, endTime time.T
}

responseData["by_namespace"] = byNamespaceResponse
responseData["total"] = &ResponseCounts{
DistinctEntities: distinctEntitiesResponse,
EntityClients: totalEntities + totalEntitiesCurrent,
NonEntityTokens: totalTokens + totalTokensCurrent,
NonEntityClients: totalTokens + totalTokensCurrent,
Clients: totalEntities + totalEntitiesCurrent + totalTokens + totalTokensCurrent,
}
totalCounts.Add(totalCurrentCounts)
totalCounts.DistinctEntities = distinctEntitiesResponse
responseData["total"] = totalCounts

// Create and populate the month response structs based on the monthly breakdown.
months, err := a.prepareMonthsResponseForQuery(ctx, pq.Months)
Expand Down Expand Up @@ -2610,32 +2616,25 @@ func (a *ActivityLog) transformMonthBreakdowns(byMonth map[int64]*processMonth)
return monthly
}

func (a *ActivityLog) calculateByNamespaceResponseForQuery(ctx context.Context, byNamespace []*activity.NamespaceRecord) (int, int, []*ResponseNamespace, error) {
func (a *ActivityLog) calculateByNamespaceResponseForQuery(ctx context.Context, byNamespace []*activity.NamespaceRecord) (*ResponseCounts, []*ResponseNamespace, error) {
queryNS, err := namespace.FromContext(ctx)
if err != nil {
return 0, 0, nil, err
return nil, nil, err
}
byNamespaceResponse := make([]*ResponseNamespace, 0)
totalEntities := 0
totalTokens := 0
totalCounts := &ResponseCounts{}

for _, nsRecord := range byNamespace {
ns, err := NamespaceByID(ctx, nsRecord.NamespaceID, a.core)
if err != nil {
return 0, 0, nil, err
return nil, nil, err
}
if a.includeInResponse(queryNS, ns) {
mountResponse := make([]*ResponseMount, 0, len(nsRecord.Mounts))
for _, mountRecord := range nsRecord.Mounts {
mountResponse = append(mountResponse, &ResponseMount{
MountPath: mountRecord.MountPath,
Counts: &ResponseCounts{
DistinctEntities: int(mountRecord.Counts.EntityClients),
EntityClients: int(mountRecord.Counts.EntityClients),
NonEntityClients: int(mountRecord.Counts.NonEntityClients),
NonEntityTokens: int(mountRecord.Counts.NonEntityClients),
Clients: int(mountRecord.Counts.EntityClients + mountRecord.Counts.NonEntityClients),
},
Counts: a.countsRecordToCountsResponse(mountRecord.Counts, true),
})
}
// Sort the mounts in descending order of usage
Expand All @@ -2649,55 +2648,41 @@ func (a *ActivityLog) calculateByNamespaceResponseForQuery(ctx context.Context,
} else {
displayPath = ns.Path
}
nsCounts := a.namespaceRecordToCountsResponse(nsRecord)
byNamespaceResponse = append(byNamespaceResponse, &ResponseNamespace{
NamespaceID: nsRecord.NamespaceID,
NamespacePath: displayPath,
Counts: ResponseCounts{
DistinctEntities: int(nsRecord.Entities),
EntityClients: int(nsRecord.Entities),
NonEntityTokens: int(nsRecord.NonEntityTokens),
NonEntityClients: int(nsRecord.NonEntityTokens),
Clients: int(nsRecord.Entities + nsRecord.NonEntityTokens),
},
Mounts: mountResponse,
Counts: *nsCounts,
Mounts: mountResponse,
})
totalEntities += int(nsRecord.Entities)
totalTokens += int(nsRecord.NonEntityTokens)
totalCounts.Add(nsCounts)
}
}
return totalEntities, totalTokens, byNamespaceResponse, nil
return totalCounts, byNamespaceResponse, nil
}

func (a *ActivityLog) prepareMonthsResponseForQuery(ctx context.Context, byMonth []*activity.MonthRecord) ([]*ResponseMonth, error) {
months := make([]*ResponseMonth, 0, len(byMonth))
for _, monthsRecord := range byMonth {
newClientsResponse := &ResponseNewClients{}
if int(monthsRecord.NewClients.Counts.EntityClients+monthsRecord.NewClients.Counts.NonEntityClients) != 0 {
if monthsRecord.NewClients.Counts.HasCounts() {
newClientsNSResponse, err := a.prepareNamespaceResponse(ctx, monthsRecord.NewClients.Namespaces)
if err != nil {
return nil, err
}
newClientsResponse.Counts = &ResponseCounts{
EntityClients: int(monthsRecord.NewClients.Counts.EntityClients),
NonEntityClients: int(monthsRecord.NewClients.Counts.NonEntityClients),
Clients: int(monthsRecord.NewClients.Counts.EntityClients + monthsRecord.NewClients.Counts.NonEntityClients),
}
newClientsResponse.Counts = a.countsRecordToCountsResponse(monthsRecord.NewClients.Counts, false)
newClientsResponse.Namespaces = newClientsNSResponse
}

monthResponse := &ResponseMonth{
Timestamp: time.Unix(monthsRecord.Timestamp, 0).UTC().Format(time.RFC3339),
}
if int(monthsRecord.Counts.EntityClients+monthsRecord.Counts.NonEntityClients) != 0 {
if monthsRecord.Counts.HasCounts() {
nsResponse, err := a.prepareNamespaceResponse(ctx, monthsRecord.Namespaces)
if err != nil {
return nil, err
}
monthResponse.Counts = &ResponseCounts{
EntityClients: int(monthsRecord.Counts.EntityClients),
NonEntityClients: int(monthsRecord.Counts.NonEntityClients),
Clients: int(monthsRecord.Counts.EntityClients + monthsRecord.Counts.NonEntityClients),
}
monthResponse.Counts = a.countsRecordToCountsResponse(monthsRecord.Counts, false)
monthResponse.Namespaces = nsResponse
monthResponse.NewClients = newClientsResponse
months = append(months, monthResponse)
Expand All @@ -2715,7 +2700,7 @@ func (a *ActivityLog) prepareNamespaceResponse(ctx context.Context, nsRecords []
}
nsResponse := make([]*ResponseNamespace, 0, len(nsRecords))
for _, nsRecord := range nsRecords {
if int(nsRecord.Counts.EntityClients) == 0 && int(nsRecord.Counts.NonEntityClients) == 0 {
if !nsRecord.Counts.HasCounts() {
continue
}

Expand All @@ -2726,17 +2711,13 @@ func (a *ActivityLog) prepareNamespaceResponse(ctx context.Context, nsRecords []
if a.includeInResponse(queryNS, ns) {
mountResponse := make([]*ResponseMount, 0, len(nsRecord.Mounts))
for _, mountRecord := range nsRecord.Mounts {
if int(mountRecord.Counts.EntityClients) == 0 && int(mountRecord.Counts.NonEntityClients) == 0 {
if !mountRecord.Counts.HasCounts() {
continue
}

mountResponse = append(mountResponse, &ResponseMount{
MountPath: mountRecord.MountPath,
Counts: &ResponseCounts{
EntityClients: int(mountRecord.Counts.EntityClients),
NonEntityClients: int(mountRecord.Counts.NonEntityClients),
Clients: int(mountRecord.Counts.EntityClients + mountRecord.Counts.NonEntityClients),
},
Counts: a.countsRecordToCountsResponse(mountRecord.Counts, false),
})
}

Expand All @@ -2749,12 +2730,8 @@ func (a *ActivityLog) prepareNamespaceResponse(ctx context.Context, nsRecords []
nsResponse = append(nsResponse, &ResponseNamespace{
NamespaceID: nsRecord.NamespaceID,
NamespacePath: displayPath,
Counts: ResponseCounts{
EntityClients: int(nsRecord.Counts.EntityClients),
NonEntityClients: int(nsRecord.Counts.NonEntityClients),
Clients: int(nsRecord.Counts.EntityClients + nsRecord.Counts.NonEntityClients),
},
Mounts: mountResponse,
Counts: *a.countsRecordToCountsResponse(nsRecord.Counts, false),
Mounts: mountResponse,
})
}
}
Expand Down Expand Up @@ -2783,7 +2760,7 @@ func (a *ActivityLog) partialMonthClientCount(ctx context.Context) (map[string]i

// Calculate the namespace response breakdowns and totals for entities and tokens from the initial
// namespace data.
totalEntities, totalTokens, byNamespaceResponse, err := a.calculateByNamespaceResponseForQuery(ctx, byNamespaceComputation)
totalCounts, byNamespaceResponse, err := a.calculateByNamespaceResponseForQuery(ctx, byNamespaceComputation)
if err != nil {
return nil, err
}
Expand All @@ -2794,11 +2771,12 @@ func (a *ActivityLog) partialMonthClientCount(ctx context.Context) (map[string]i
// Now populate the response based on breakdowns.
responseData := make(map[string]interface{})
responseData["by_namespace"] = byNamespaceResponse
responseData["distinct_entities"] = totalEntities
responseData["entity_clients"] = totalEntities
responseData["non_entity_tokens"] = totalTokens
responseData["non_entity_clients"] = totalTokens
responseData["clients"] = totalEntities + totalTokens
responseData["distinct_entities"] = totalCounts.EntityClients
responseData["entity_clients"] = totalCounts.EntityClients
responseData["non_entity_tokens"] = totalCounts.NonEntityClients
responseData["non_entity_clients"] = totalCounts.NonEntityClients
responseData["clients"] = totalCounts.Clients
responseData["secret_syncs"] = totalCounts.SecretSyncs

// The partialMonthClientCount should not have more than one month worth of data.
// If it does, something has gone wrong and we should warn that the activity log data
Expand Down
Loading

0 comments on commit caef593

Please sign in to comment.