mirror of
https://github.com/hashicorp/vault.git
synced 2026-05-28 04:10:44 -04:00
Backport VAULT-43310: gcp kms counting of successful requests for billing into ce/main (#13947)
* no-op commit * cherry pick --------- Co-authored-by: Amir Aslamov <amir.aslamov@hashicorp.com>
This commit is contained in:
parent
ac4503cf69
commit
7f0c0dbdad
12 changed files with 336 additions and 17 deletions
|
|
@ -180,6 +180,10 @@ const billingOverviewResponse = `{
|
|||
{
|
||||
"type": "transform",
|
||||
"count": 50
|
||||
},
|
||||
{
|
||||
"type": "gcpkms",
|
||||
"count": 50
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
|
|||
2
go.mod
2
go.mod
|
|
@ -160,7 +160,7 @@ require (
|
|||
github.com/hashicorp/vault-plugin-secrets-alicloud v0.22.1
|
||||
github.com/hashicorp/vault-plugin-secrets-azure v0.25.0
|
||||
github.com/hashicorp/vault-plugin-secrets-gcp v0.24.0
|
||||
github.com/hashicorp/vault-plugin-secrets-gcpkms v0.23.0
|
||||
github.com/hashicorp/vault-plugin-secrets-gcpkms v0.24.0
|
||||
github.com/hashicorp/vault-plugin-secrets-kubernetes v0.13.1
|
||||
github.com/hashicorp/vault-plugin-secrets-kv v0.26.2
|
||||
github.com/hashicorp/vault-plugin-secrets-mongodbatlas v0.17.1
|
||||
|
|
|
|||
4
go.sum
4
go.sum
|
|
@ -893,8 +893,8 @@ github.com/hashicorp/vault-plugin-secrets-azure v0.25.0 h1:IYIGfFiw3ICLfVlF+YlPz
|
|||
github.com/hashicorp/vault-plugin-secrets-azure v0.25.0/go.mod h1:S4E5R3RVpGWnzR0bDGDwpV0HlyNsMAIcuYuNlgWv2zo=
|
||||
github.com/hashicorp/vault-plugin-secrets-gcp v0.24.0 h1:cQ94yUwvDRt7+pUHUfvSjE/ORgKF31hO2p+6AwIHjX8=
|
||||
github.com/hashicorp/vault-plugin-secrets-gcp v0.24.0/go.mod h1:ZqfNKh3d0O/brwj3z7K1NjqNx0Is60G7zlBedijXrfI=
|
||||
github.com/hashicorp/vault-plugin-secrets-gcpkms v0.23.0 h1:gXFfSYVpebgklMeLxWfMrBHOZJ5EdJtM80ir0NZNdMM=
|
||||
github.com/hashicorp/vault-plugin-secrets-gcpkms v0.23.0/go.mod h1:YdDoi8TIpbJ4lljL3fISJmZQxZqmJfHMdzxCS33pjBc=
|
||||
github.com/hashicorp/vault-plugin-secrets-gcpkms v0.24.0 h1:kh3ULsvsqYLmJxRO2EW55Hbh0QCdnDkNTtGZoDhl07w=
|
||||
github.com/hashicorp/vault-plugin-secrets-gcpkms v0.24.0/go.mod h1:RK7XvCOc4haR0YXkUi8Rm+O2zBpBWAL4MefK4BkZz0E=
|
||||
github.com/hashicorp/vault-plugin-secrets-kubernetes v0.13.1 h1:ug+5nibS3AulD3ElaQeD42N0VJsTUwTRVPgJSj0ovvM=
|
||||
github.com/hashicorp/vault-plugin-secrets-kubernetes v0.13.1/go.mod h1:t34JjbPLaLrhvwb7iKmvW9y72o7ZhxGfN0Q3yClsV8Y=
|
||||
github.com/hashicorp/vault-plugin-secrets-kv v0.26.2 h1:5ruO7aTfQqOKIuC+G6hXQbBKXZ6sPGDA3s2oCtyGtdU=
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ const (
|
|||
KmseHWMCountsHWM = "maxKmseCounts/"
|
||||
TransitDataProtectionCallCountsPrefix = "transitDataProtectionCallCounts/"
|
||||
TransformDataProtectionCallCountsPrefix = "transformDataProtectionCallCounts/"
|
||||
GcpKmsDataProtectionCallCountsPrefix = "gcpKmsDataProtectionCallCounts/"
|
||||
LocalPrefix = "local/"
|
||||
ThirdPartyPluginsPrefix = "thirdPartyPluginCounts/"
|
||||
KmipEnabledPrefix = "kmipEnabled/"
|
||||
|
|
@ -80,6 +81,7 @@ func GetMonthlyBillingPath(localPrefix string, now time.Time) string {
|
|||
type DataProtectionCallCounts struct {
|
||||
Transit *atomic.Uint64 `json:"transit,omitempty"`
|
||||
Transform *atomic.Uint64 `json:"transform,omitempty"`
|
||||
GcpKms *atomic.Uint64 `json:"gcpkms,omitempty"`
|
||||
}
|
||||
|
||||
var _ logical.ConsumptionBillingManager = (*ConsumptionBilling)(nil)
|
||||
|
|
@ -106,6 +108,14 @@ func (s *ConsumptionBilling) WriteBillingData(ctx context.Context, mountType str
|
|||
}
|
||||
|
||||
s.DataProtectionCallCounts.Transform.Add(val)
|
||||
case "gcpkms":
|
||||
val, ok := data["count"].(uint64)
|
||||
if !ok {
|
||||
err := fmt.Errorf("invalid value type for gcp kms")
|
||||
return err
|
||||
}
|
||||
|
||||
s.DataProtectionCallCounts.GcpKms.Add(val)
|
||||
default:
|
||||
err := fmt.Errorf("unknown metric type: %s", mountType)
|
||||
return err
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ func (c *Core) setupConsumptionBilling(ctx context.Context) error {
|
|||
DataProtectionCallCounts: billing.DataProtectionCallCounts{
|
||||
Transit: &atomic.Uint64{},
|
||||
Transform: &atomic.Uint64{},
|
||||
GcpKms: &atomic.Uint64{},
|
||||
},
|
||||
Logger: logger,
|
||||
}
|
||||
|
|
@ -157,6 +158,7 @@ func (c *Core) resetInMemoryBillingMetrics() error {
|
|||
defer c.consumptionBillingLock.Unlock()
|
||||
c.consumptionBilling.DataProtectionCallCounts.Transit.Store(0)
|
||||
c.consumptionBilling.DataProtectionCallCounts.Transform.Store(0)
|
||||
c.consumptionBilling.DataProtectionCallCounts.GcpKms.Store(0)
|
||||
c.consumptionBilling.KmipSeenEnabledThisMonth.Store(false)
|
||||
return nil
|
||||
}
|
||||
|
|
@ -257,5 +259,8 @@ func (c *Core) UpdateLocalAggregatedMetrics(ctx context.Context, currentMonth ti
|
|||
if _, err := c.UpdateTransformCallCounts(ctx, currentMonth); err != nil {
|
||||
return fmt.Errorf("could not store transform data protection call counts: %w", err)
|
||||
}
|
||||
if _, err := c.UpdateGcpKmsCallCounts(ctx, currentMonth); err != nil {
|
||||
return fmt.Errorf("could not store GCP KMS data protection call counts: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -179,6 +179,7 @@ func TestHandleEndOfMonthMetrics(t *testing.T) {
|
|||
}, localPathPrefix, month)
|
||||
core.storeMaxKvCountsLocked(context.Background(), 10, localPathPrefix, month)
|
||||
core.storeTransitCallCountsLocked(context.Background(), 10, localPathPrefix, month)
|
||||
core.storeGcpKmsCallCountsLocked(context.Background(), 10, localPathPrefix, month)
|
||||
core.storeThirdPartyPluginCountsLocked(context.Background(), localPathPrefix, month, 10)
|
||||
|
||||
// List the data paths to verify that the billing metrics have been stored
|
||||
|
|
@ -186,7 +187,7 @@ func TestHandleEndOfMonthMetrics(t *testing.T) {
|
|||
require.True(t, ok)
|
||||
paths, err := view.List(context.Background(), billing.GetMonthlyBillingPath(localPathPrefix, month))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 4, len(paths))
|
||||
require.Equal(t, 5, len(paths))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -206,11 +207,12 @@ func TestHandleEndOfMonthMetrics(t *testing.T) {
|
|||
require.True(t, ok)
|
||||
paths, err = view.List(context.Background(), billing.GetMonthlyBillingPath(localPathPrefix, previousMonth))
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 4, len(paths))
|
||||
require.Equal(t, 5, len(paths))
|
||||
}
|
||||
|
||||
require.Equal(t, uint64(0), core.GetInMemoryTransitDataProtectionCallCounts())
|
||||
require.Equal(t, uint64(0), core.GetInMemoryTransformDataProtectionCallCounts())
|
||||
require.Equal(t, uint64(0), core.GetInMemoryGcpKmsDataProtectionCallCounts())
|
||||
require.False(t, core.consumptionBilling.KmipSeenEnabledThisMonth.Load())
|
||||
}
|
||||
|
||||
|
|
@ -268,6 +270,9 @@ func TestConsumptionBillingMetricsWorkerWithCustomClock(t *testing.T) {
|
|||
transitCounts, err := core.GetStoredTransitCallCounts(context.Background(), month)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(10), transitCounts)
|
||||
gcpKmsCounts, err := core.GetStoredGcpKmsCallCounts(context.Background(), month)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(10), gcpKmsCounts)
|
||||
thirdPartyPluginCounts, err := core.GetStoredThirdPartyPluginCounts(context.Background(), month)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 10, thirdPartyPluginCounts)
|
||||
|
|
@ -277,6 +282,7 @@ func TestConsumptionBillingMetricsWorkerWithCustomClock(t *testing.T) {
|
|||
for _, month := range []time.Time{twoMonthsAgo, previousMonth} {
|
||||
|
||||
core.storeTransitCallCountsLocked(context.Background(), uint64(10), billing.LocalPrefix, month)
|
||||
core.storeGcpKmsCallCountsLocked(context.Background(), uint64(10), billing.LocalPrefix, month)
|
||||
core.storeThirdPartyPluginCountsLocked(context.Background(), billing.LocalPrefix, month, 10)
|
||||
|
||||
for _, localPathPrefix := range []string{billing.ReplicatedPrefix, billing.LocalPrefix} {
|
||||
|
|
@ -303,6 +309,8 @@ func TestConsumptionBillingMetricsWorkerWithCustomClock(t *testing.T) {
|
|||
if localPathPrefix == billing.LocalPrefix {
|
||||
transitCounts, _ := core.GetStoredTransitCallCounts(context.Background(), twoMonthsAgo)
|
||||
require.Equal(t, uint64(0), transitCounts)
|
||||
gcpKmsCounts, _ := core.GetStoredGcpKmsCallCounts(context.Background(), twoMonthsAgo)
|
||||
require.Equal(t, uint64(0), gcpKmsCounts)
|
||||
thirdPartyPluginCounts, _ := core.GetStoredThirdPartyPluginCounts(context.Background(), twoMonthsAgo)
|
||||
require.Equal(t, 0, thirdPartyPluginCounts)
|
||||
}
|
||||
|
|
@ -313,5 +321,6 @@ func TestConsumptionBillingMetricsWorkerWithCustomClock(t *testing.T) {
|
|||
|
||||
require.Equal(t, uint64(0), core.GetInMemoryTransitDataProtectionCallCounts())
|
||||
require.Equal(t, uint64(0), core.GetInMemoryTransformDataProtectionCallCounts())
|
||||
require.Equal(t, uint64(0), core.GetInMemoryGcpKmsDataProtectionCallCounts())
|
||||
require.False(t, core.consumptionBilling.KmipSeenEnabledThisMonth.Load())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -64,3 +64,24 @@ func (c *Core) SetInMemoryTransformDataProtectionCallCounts(count uint64) {
|
|||
cb.DataProtectionCallCounts.Transform.Store(count)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Core) SetInMemoryGcpKmsDataProtectionCallCounts(count uint64) {
|
||||
c.consumptionBillingLock.RLock()
|
||||
cb := c.consumptionBilling
|
||||
c.consumptionBillingLock.RUnlock()
|
||||
|
||||
if cb != nil {
|
||||
cb.DataProtectionCallCounts.GcpKms.Store(count)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Core) GetInMemoryGcpKmsDataProtectionCallCounts() uint64 {
|
||||
c.consumptionBillingLock.RLock()
|
||||
cb := c.consumptionBilling
|
||||
c.consumptionBillingLock.RUnlock()
|
||||
|
||||
if cb != nil {
|
||||
return cb.DataProtectionCallCounts.GcpKms.Load()
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
|
|
|||
|
|
@ -523,6 +523,83 @@ func (c *Core) UpdateTransitCallCounts(ctx context.Context, currentMonth time.Ti
|
|||
return transitCount, nil
|
||||
}
|
||||
|
||||
func (c *Core) UpdateGcpKmsCallCounts(ctx context.Context, currentMonth time.Time) (uint64, error) {
|
||||
c.consumptionBillingLock.RLock()
|
||||
cb := c.consumptionBilling
|
||||
c.consumptionBillingLock.RUnlock()
|
||||
|
||||
if cb == nil {
|
||||
return 0, ErrConsumptionBillingNotInitialized
|
||||
}
|
||||
cb.BillingStorageLock.Lock()
|
||||
defer cb.BillingStorageLock.Unlock()
|
||||
storedGcpKmsCount, err := c.getStoredGcpKmsCallCountsLocked(ctx, billing.LocalPrefix, currentMonth)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// Sum the current count with the stored count
|
||||
gcpKmsCount := cb.DataProtectionCallCounts.GcpKms.Swap(0) + storedGcpKmsCount
|
||||
|
||||
err = c.storeGcpKmsCallCountsLocked(ctx, gcpKmsCount, billing.LocalPrefix, currentMonth)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return gcpKmsCount, nil
|
||||
}
|
||||
|
||||
// storeGcpKmsCallCountsLocked must be called with BillingStorageLock held
|
||||
func (c *Core) storeGcpKmsCallCountsLocked(ctx context.Context, gcpKmsCount uint64, localPathPrefix string, month time.Time) error {
|
||||
// Store count for each data protection type separately because they are atomic counters
|
||||
billingPath := billing.GetMonthlyBillingMetricPath(localPathPrefix, month, billing.GcpKmsDataProtectionCallCountsPrefix)
|
||||
entry := &logical.StorageEntry{
|
||||
Key: billingPath,
|
||||
Value: []byte(strconv.FormatUint(gcpKmsCount, 10)),
|
||||
}
|
||||
view, ok := c.GetBillingSubView()
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
return view.Put(ctx, entry)
|
||||
}
|
||||
|
||||
// getStoredGcpKmsCallCountsLocked must be called with BillingStorageLock held
|
||||
func (c *Core) getStoredGcpKmsCallCountsLocked(ctx context.Context, localPathPrefix string, month time.Time) (uint64, error) {
|
||||
// Retrieve count for each data protection type separately because they are atomic counters
|
||||
billingPath := billing.GetMonthlyBillingMetricPath(localPathPrefix, month, billing.GcpKmsDataProtectionCallCountsPrefix)
|
||||
view, ok := c.GetBillingSubView()
|
||||
if !ok {
|
||||
return 0, nil
|
||||
}
|
||||
entry, err := view.Get(ctx, billingPath)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if entry == nil {
|
||||
return 0, nil
|
||||
}
|
||||
gcpKmsCount, err := strconv.ParseUint(string(entry.Value), 10, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return gcpKmsCount, nil
|
||||
}
|
||||
|
||||
func (c *Core) GetStoredGcpKmsCallCounts(ctx context.Context, month time.Time) (uint64, error) {
|
||||
c.consumptionBillingLock.RLock()
|
||||
cb := c.consumptionBilling
|
||||
c.consumptionBillingLock.RUnlock()
|
||||
|
||||
if cb == nil {
|
||||
return 0, ErrConsumptionBillingNotInitialized
|
||||
}
|
||||
|
||||
cb.BillingStorageLock.RLock()
|
||||
defer cb.BillingStorageLock.RUnlock()
|
||||
return c.getStoredGcpKmsCallCountsLocked(ctx, billing.LocalPrefix, month)
|
||||
}
|
||||
|
||||
func (c *Core) storeKmipEnabledLocked(ctx context.Context, localPathPrefix string, currentMonth time.Time, kmipEnabled bool) error {
|
||||
billingPath := billing.GetMonthlyBillingMetricPath(localPathPrefix, currentMonth, billing.KmipEnabledPrefix)
|
||||
entry, err := logical.StorageEntryJSON(billingPath, kmipEnabled)
|
||||
|
|
|
|||
|
|
@ -1185,3 +1185,69 @@ func deleteTotpKeyFromStorage(t *testing.T, ctx context.Context, core *Core, mou
|
|||
_, err := core.HandleRequest(ctx, req)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// TestGcpKmsDataProtectionCallCounts tests that we correctly store and track the GCP KMS data protection call counts
|
||||
func TestGcpKmsDataProtectionCallCounts(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
coreConfig := &CoreConfig{
|
||||
BillingConfig: billing.BillingConfig{
|
||||
MetricsUpdateCadence: 3 * time.Second,
|
||||
},
|
||||
}
|
||||
core, _, _, _ := TestCoreUnsealedWithMetricsAndConfig(t, coreConfig)
|
||||
|
||||
ctx := context.Background()
|
||||
currentMonth := time.Now()
|
||||
|
||||
// Simulate GCP KMS plugin writing billing data (this is what the plugin does when operations occur)
|
||||
// In a real scenario, this would be triggered by actual encrypt/decrypt/sign/verify operations
|
||||
err := core.consumptionBilling.WriteBillingData(ctx, "gcpkms", map[string]interface{}{
|
||||
"count": uint64(1),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify that the GCP KMS counter is incremented
|
||||
require.Equal(t, uint64(1), core.GetInMemoryGcpKmsDataProtectionCallCounts())
|
||||
|
||||
// Wait until the data protection calls are updated and verify that the value in storage is correct
|
||||
require.Eventually(t, func() bool {
|
||||
counts, err := core.GetStoredGcpKmsCallCounts(ctx, currentMonth)
|
||||
return err == nil && counts == 1
|
||||
}, 5*time.Second, 100*time.Millisecond)
|
||||
|
||||
// The in memory counter should be reset after the update
|
||||
require.Equal(t, uint64(0), core.GetInMemoryGcpKmsDataProtectionCallCounts())
|
||||
|
||||
// Simulate more operations
|
||||
err = core.consumptionBilling.WriteBillingData(ctx, "gcpkms", map[string]interface{}{
|
||||
"count": uint64(1),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
err = core.consumptionBilling.WriteBillingData(ctx, "gcpkms", map[string]interface{}{
|
||||
"count": uint64(1),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
// Verify that the GCP KMS counter is incremented
|
||||
require.Equal(t, uint64(2), core.GetInMemoryGcpKmsDataProtectionCallCounts())
|
||||
|
||||
// Wait until the data protection calls are updated and verify that the value in storage is correct
|
||||
require.Eventually(t, func() bool {
|
||||
counts, err := core.GetStoredGcpKmsCallCounts(ctx, currentMonth)
|
||||
return err == nil && counts == 3
|
||||
}, 5*time.Second, 100*time.Millisecond)
|
||||
|
||||
// The in memory counter should be reset after the update
|
||||
require.Equal(t, uint64(0), core.GetInMemoryGcpKmsDataProtectionCallCounts())
|
||||
|
||||
// Run update again and make sure the value in storage is still 3
|
||||
counts, err := core.UpdateGcpKmsCallCounts(ctx, currentMonth)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(3), counts)
|
||||
|
||||
// Verify the value in storage is still 3
|
||||
counts, err = core.GetStoredGcpKmsCallCounts(ctx, currentMonth)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(3), counts)
|
||||
}
|
||||
|
|
|
|||
106
vault/external_tests/billing/billing_test.go
Normal file
106
vault/external_tests/billing/billing_test.go
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package billing
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/vault/helper/namespace"
|
||||
"github.com/hashicorp/vault/vault"
|
||||
"github.com/hashicorp/vault/vault/billing"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestGcpKmsDataProtectionCallCounts tests that we correctly store and track
|
||||
// the GCP KMS data protection call counts by simulating billing operations.
|
||||
func TestGcpKmsDataProtectionCallCounts(t *testing.T) {
|
||||
coreConfig := &vault.CoreConfig{
|
||||
BillingConfig: billing.BillingConfig{
|
||||
MetricsUpdateCadence: 3 * time.Second,
|
||||
},
|
||||
}
|
||||
core, _, _, _ := vault.TestCoreUnsealedWithMetricsAndConfig(t, coreConfig)
|
||||
|
||||
currentMonth := time.Now()
|
||||
ctx := namespace.RootContext(context.Background())
|
||||
|
||||
// Get the consumption billing manager
|
||||
cbm := core.GetConsumptionBillingManager()
|
||||
require.NotNil(t, cbm)
|
||||
|
||||
// Simulate GCP KMS operations by directly calling the billing manager
|
||||
// This tests the Vault-side tracking without needing the actual plugin
|
||||
|
||||
// Simulate encrypt operation
|
||||
err := cbm.WriteBillingData(ctx, "gcpkms", map[string]interface{}{"count": uint64(1)})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(1), core.GetInMemoryGcpKmsDataProtectionCallCounts())
|
||||
|
||||
// Wait for storage update
|
||||
require.Eventually(t, func() bool {
|
||||
counts, err := core.GetStoredGcpKmsCallCounts(context.Background(), currentMonth)
|
||||
return err == nil && counts == 1
|
||||
}, 5*time.Second, 100*time.Millisecond)
|
||||
require.Equal(t, uint64(0), core.GetInMemoryGcpKmsDataProtectionCallCounts())
|
||||
|
||||
// Simulate decrypt operation
|
||||
err = cbm.WriteBillingData(ctx, "gcpkms", map[string]interface{}{"count": uint64(1)})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(1), core.GetInMemoryGcpKmsDataProtectionCallCounts())
|
||||
|
||||
// Wait for storage update
|
||||
require.Eventually(t, func() bool {
|
||||
counts, err := core.GetStoredGcpKmsCallCounts(context.Background(), currentMonth)
|
||||
return err == nil && counts == 2
|
||||
}, 5*time.Second, 100*time.Millisecond)
|
||||
require.Equal(t, uint64(0), core.GetInMemoryGcpKmsDataProtectionCallCounts())
|
||||
|
||||
// Simulate reencrypt operation
|
||||
err = cbm.WriteBillingData(ctx, "gcpkms", map[string]interface{}{"count": uint64(1)})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(1), core.GetInMemoryGcpKmsDataProtectionCallCounts())
|
||||
|
||||
// Wait for storage update
|
||||
require.Eventually(t, func() bool {
|
||||
counts, err := core.GetStoredGcpKmsCallCounts(context.Background(), currentMonth)
|
||||
return err == nil && counts == 3
|
||||
}, 5*time.Second, 100*time.Millisecond)
|
||||
require.Equal(t, uint64(0), core.GetInMemoryGcpKmsDataProtectionCallCounts())
|
||||
|
||||
// Simulate sign operation
|
||||
err = cbm.WriteBillingData(ctx, "gcpkms", map[string]interface{}{"count": uint64(1)})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(1), core.GetInMemoryGcpKmsDataProtectionCallCounts())
|
||||
|
||||
// Wait for storage update
|
||||
require.Eventually(t, func() bool {
|
||||
counts, err := core.GetStoredGcpKmsCallCounts(context.Background(), currentMonth)
|
||||
return err == nil && counts == 4
|
||||
}, 5*time.Second, 100*time.Millisecond)
|
||||
require.Equal(t, uint64(0), core.GetInMemoryGcpKmsDataProtectionCallCounts())
|
||||
|
||||
// Simulate verify operation
|
||||
err = cbm.WriteBillingData(ctx, "gcpkms", map[string]interface{}{"count": uint64(1)})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(1), core.GetInMemoryGcpKmsDataProtectionCallCounts())
|
||||
|
||||
// Wait for storage update
|
||||
require.Eventually(t, func() bool {
|
||||
counts, err := core.GetStoredGcpKmsCallCounts(context.Background(), currentMonth)
|
||||
return err == nil && counts == 5
|
||||
}, 5*time.Second, 100*time.Millisecond)
|
||||
require.Equal(t, uint64(0), core.GetInMemoryGcpKmsDataProtectionCallCounts())
|
||||
|
||||
// Run update again and make sure the value in storage is still 5
|
||||
counts, err := core.UpdateGcpKmsCallCounts(context.Background(), currentMonth)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(5), counts)
|
||||
|
||||
// Verify the value in storage is still 5
|
||||
counts, err = core.GetStoredGcpKmsCallCounts(context.Background(), currentMonth)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, uint64(5), counts)
|
||||
}
|
||||
|
|
@ -125,7 +125,7 @@ func (b *SystemBackend) buildMonthBillingData(ctx context.Context, month time.Ti
|
|||
return nil, err
|
||||
}
|
||||
|
||||
transitCounts, transformCounts, err := b.Core.getDataProtectionCounts(ctx, month)
|
||||
transitCounts, transformCounts, gcpKmsCounts, err := b.Core.getDataProtectionCounts(ctx, month)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -180,11 +180,14 @@ func (b *SystemBackend) buildMonthBillingData(ctx context.Context, month time.Ti
|
|||
if transformCounts > 0 {
|
||||
dataProtectionDetails = append(dataProtectionDetails, map[string]interface{}{"type": "transform", "count": transformCounts})
|
||||
}
|
||||
if gcpKmsCounts > 0 {
|
||||
dataProtectionDetails = append(dataProtectionDetails, map[string]interface{}{"type": "gcpkms", "count": gcpKmsCounts})
|
||||
}
|
||||
|
||||
usageMetrics = append(usageMetrics, map[string]interface{}{
|
||||
"metric_name": "data_protection_calls",
|
||||
"metric_data": map[string]interface{}{
|
||||
"total": transitCounts + transformCounts,
|
||||
"total": transitCounts + transformCounts + gcpKmsCounts,
|
||||
"metric_details": dataProtectionDetails,
|
||||
},
|
||||
})
|
||||
|
|
@ -457,20 +460,24 @@ func (c *Core) getKvCounts(ctx context.Context, month time.Time) (int, error) {
|
|||
return replicatedKvCounts + localKvCounts, nil
|
||||
}
|
||||
|
||||
// getDataProtectionCounts retrieves Transit and Transform call counts
|
||||
// getDataProtectionCounts retrieves Transit, Transform, and GCP KMS call counts
|
||||
// Data protection call counts are stored at local path only
|
||||
// Each cluster tracks its own total requests to avoid double counting
|
||||
func (c *Core) getDataProtectionCounts(ctx context.Context, month time.Time) (uint64, uint64, error) {
|
||||
func (c *Core) getDataProtectionCounts(ctx context.Context, month time.Time) (uint64, uint64, uint64, error) {
|
||||
transitCounts, err := c.GetStoredTransitCallCounts(ctx, month)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("error retrieving local transit call counts: %w", err)
|
||||
return 0, 0, 0, fmt.Errorf("error retrieving local transit call counts: %w", err)
|
||||
}
|
||||
transformCounts, err := c.GetStoredTransformCallCounts(ctx, month)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("error retrieving local transform call counts: %w", err)
|
||||
return 0, 0, 0, fmt.Errorf("error retrieving local transform call counts: %w", err)
|
||||
}
|
||||
gcpKmsCounts, err := c.GetStoredGcpKmsCallCounts(ctx, month)
|
||||
if err != nil {
|
||||
return 0, 0, 0, fmt.Errorf("error retrieving local GCP KMS call counts: %w", err)
|
||||
}
|
||||
|
||||
return transitCounts, transformCounts, nil
|
||||
return transitCounts, transformCounts, gcpKmsCounts, nil
|
||||
}
|
||||
|
||||
// getKmipStatus retrieves KMIP enabled status (always stored at local path)
|
||||
|
|
|
|||
|
|
@ -21,11 +21,11 @@ import (
|
|||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// TestSystemBackend_BillingOverview verifies that the billing overview endpoint
|
||||
// TestSystemBackend_BillingOverviewMonthFormat verifies that the billing overview endpoint
|
||||
// returns the correct response structure with current and previous month data.
|
||||
// It validates the response format, month strings (YYYY-MM), RFC3339 timestamps,
|
||||
// and ensures both months are present in the response.
|
||||
func TestSystemBackend_BillingOverview(t *testing.T) {
|
||||
func TestSystemBackend_BillingOverviewMonthFormat(t *testing.T) {
|
||||
_, b, _ := testCoreSystemBackend(t)
|
||||
ctx := namespace.RootContext(nil)
|
||||
|
||||
|
|
@ -164,9 +164,9 @@ func TestSystemBackend_BillingOverview_WithMetrics(t *testing.T) {
|
|||
require.True(t, foundStaticSecrets, "static_secrets metric should be present")
|
||||
}
|
||||
|
||||
// TestSystemBackend_BillingOverview_MetricFormats validates that different metric types
|
||||
// TestSystemBackend_BillingOverview_MetricTypeFormat validates that different metric types
|
||||
// in the billing overview response have the correct data structure.
|
||||
func TestSystemBackend_BillingOverview_MetricFormats(t *testing.T) {
|
||||
func TestSystemBackend_BillingOverview_MetricTypeFormat(t *testing.T) {
|
||||
c, _, root, _ := TestCoreUnsealedWithMetricsAndConfig(t, &CoreConfig{
|
||||
LogicalBackends: map[string]logical.Factory{
|
||||
pluginconsts.SecretEngineKV: logicalKv.Factory,
|
||||
|
|
@ -299,6 +299,12 @@ func TestSystemBackend_BillingOverview_MetricFormats(t *testing.T) {
|
|||
_, err = c.UpdateStoredSSHOTPCount(ctx, currentMonth, c.certCountManager.GetCounts().SSHIssuedOTPs)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Write GCP KMS count directly to storage
|
||||
c.consumptionBilling.BillingStorageLock.Lock()
|
||||
err = c.storeGcpKmsCallCountsLocked(ctx, uint64(5), billing.LocalPrefix, currentMonth)
|
||||
c.consumptionBilling.BillingStorageLock.Unlock()
|
||||
require.NoError(t, err)
|
||||
|
||||
// Make a request to the billing overview endpoint
|
||||
req = logical.TestRequest(t, logical.ReadOperation, "billing/overview")
|
||||
req.Data["refresh_data"] = true
|
||||
|
|
@ -391,7 +397,7 @@ func TestSystemBackend_BillingOverview_MetricFormats(t *testing.T) {
|
|||
require.Contains(t, metricData, "total")
|
||||
total, ok := metricData["total"].(uint64)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, total, uint64(1))
|
||||
require.Equal(t, total, uint64(6)) // 1 transit + 5 gcpkms
|
||||
|
||||
require.Contains(t, metricData, "metric_details")
|
||||
metricDetails, ok := metricData["metric_details"].([]map[string]interface{})
|
||||
|
|
@ -399,6 +405,7 @@ func TestSystemBackend_BillingOverview_MetricFormats(t *testing.T) {
|
|||
require.NotEmpty(t, metricDetails)
|
||||
|
||||
foundTransit := false
|
||||
foundGcpKms := false
|
||||
for _, detail := range metricDetails {
|
||||
if detail["type"] == "transit" {
|
||||
foundTransit = true
|
||||
|
|
@ -406,8 +413,15 @@ func TestSystemBackend_BillingOverview_MetricFormats(t *testing.T) {
|
|||
require.True(t, ok)
|
||||
require.Equal(t, count, uint64(1))
|
||||
}
|
||||
if detail["type"] == "gcpkms" {
|
||||
foundGcpKms = true
|
||||
count, ok := detail["count"].(uint64)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, count, uint64(5))
|
||||
}
|
||||
}
|
||||
require.True(t, foundTransit, "should have transit type in metric_details")
|
||||
require.True(t, foundGcpKms, "should have gcpkms type in metric_details")
|
||||
|
||||
case "pki_units":
|
||||
require.Contains(t, metricData, "total")
|
||||
|
|
|
|||
Loading…
Reference in a new issue