From 5f71e3734cb0825198ccf7bf49b7c216a9461641 Mon Sep 17 00:00:00 2001 From: Houssein Mnaouar Date: Sun, 14 Sep 2025 15:34:53 +0000 Subject: [PATCH 1/2] feat(gcs): support custom universe_domain via env/config and minimal test bucket location support Small, opt-in changes: - Accept GOOGLE_UNIVERSE_DOMAIN (or config value) when building the GCS client. - Add minimal support for GOOGLE_BUCKET_LOCATION in tests to create buckets in non-standard regions. - Tests updated accordingly. Default behaviour unchanged when no universe_domain provided. Signed-off-by: Houssein Mnaouar --- physical/gcs/gcs.go | 22 ++++++++++++++++++++-- physical/gcs/gcs_ha_test.go | 33 ++++++++++++++++++++++++++------- physical/gcs/gcs_test.go | 32 +++++++++++++++++++++++++++----- 3 files changed, 73 insertions(+), 14 deletions(-) diff --git a/physical/gcs/gcs.go b/physical/gcs/gcs.go index 8832bbb813..0d7cefe025 100644 --- a/physical/gcs/gcs.go +++ b/physical/gcs/gcs.go @@ -48,6 +48,10 @@ const ( // objectDelimiter is the string to use to delimit objects. objectDelimiter = "/" + + // envUniverseDomain is an optional env var used to override the universe + // domain (helpful for custom GCS-compatible endpoints). + envUniverseDomain = "GOOGLE_UNIVERSE_DOMAIN" ) var ( @@ -124,6 +128,20 @@ func NewBackend(c map[string]string, logger log.Logger) (physical.Backend, error // Values are specified as kb, but the API expects them as bytes. chunkSize = chunkSize * 1024 + // universe_domain: optional custom domain used by the GCS client + universeDomain := os.Getenv(envUniverseDomain) + if universeDomain == "" { + universeDomain = c["universe_domain"] + } + + // Build client options (we always set a user-agent). + clientOptions := []option.ClientOption{option.WithUserAgent(useragent.String())} + if universeDomain != "" { + // WithUniverseDomain is optional and only appended when provided. + clientOptions = append(clientOptions, option.WithUniverseDomain(universeDomain)) + logger.Debug("using custom universe_domain", "universe_domain", universeDomain) + } + // HA configuration haClient := (*storage.Client)(nil) haEnabled := false @@ -142,7 +160,7 @@ func NewBackend(c map[string]string, logger log.Logger) (physical.Backend, error logger.Debug("creating client") var err error ctx := context.Background() - haClient, err = storage.NewClient(ctx, option.WithUserAgent(useragent.String())) + haClient, err = storage.NewClient(ctx, clientOptions...) if err != nil { return nil, fmt.Errorf("failed to create HA storage client: %w", err) } @@ -163,7 +181,7 @@ func NewBackend(c map[string]string, logger log.Logger) (physical.Backend, error logger.Debug("creating client") ctx := context.Background() - client, err := storage.NewClient(ctx, option.WithUserAgent(useragent.String())) + client, err := storage.NewClient(ctx, clientOptions...) if err != nil { return nil, fmt.Errorf("failed to create storage client: %w", err) } diff --git a/physical/gcs/gcs_ha_test.go b/physical/gcs/gcs_ha_test.go index ab6ca888a6..96d6cf3249 100644 --- a/physical/gcs/gcs_ha_test.go +++ b/physical/gcs/gcs_ha_test.go @@ -15,6 +15,7 @@ import ( log "github.com/hashicorp/go-hclog" "github.com/hashicorp/vault/sdk/helper/logging" "github.com/hashicorp/vault/sdk/physical" + "google.golang.org/api/option" ) func TestHABackend(t *testing.T) { @@ -23,27 +24,45 @@ func TestHABackend(t *testing.T) { t.Skip("GOOGLE_PROJECT_ID not set") } + universeDomain := os.Getenv("GOOGLE_UNIVERSE_DOMAIN") + r := rand.New(rand.NewSource(time.Now().UnixNano())).Int() bucket := fmt.Sprintf("vault-gcs-testacc-%d", r) ctx := context.Background() - client, err := storage.NewClient(ctx) - if err != nil { - t.Fatal(err) + // Build client options: if a custom universe domain is provided in env, use it. + clientOpts := []option.ClientOption{} + if universeDomain != "" { + clientOpts = append(clientOpts, option.WithUniverseDomain(universeDomain)) } + client, err := storage.NewClient(ctx, clientOpts...) testCleanup(t, client, bucket) defer testCleanup(t, client, bucket) bh := client.Bucket(bucket) - if err := bh.Create(context.Background(), projectID, nil); err != nil { - t.Fatal(err) + // Support minimal for providers that require an explicit Location. + bucketLocation := os.Getenv("GOOGLE_BUCKET_LOCATION") + + if bucketLocation != "" { + // Create the bucket with the explicit location required by some universe domains. + if err := bh.Create(ctx, projectID, &storage.BucketAttrs{ + Location: bucketLocation, + }); err != nil { + t.Fatalf("failed to create bucket %q with location %q: %v", bucket, bucketLocation, err) + } + } else { + // Default behaviour (no explicit location). + if err := bh.Create(ctx, projectID, nil); err != nil { + t.Fatalf("failed to create bucket %q: %v", bucket, err) + } } logger := logging.NewVaultLogger(log.Trace) config := map[string]string{ - "bucket": bucket, - "ha_enabled": "true", + "bucket": bucket, + "ha_enabled": "true", + "universe_domain": universeDomain, } b, err := NewBackend(config, logger) diff --git a/physical/gcs/gcs_test.go b/physical/gcs/gcs_test.go index 6ee9ab432c..79b42035e4 100644 --- a/physical/gcs/gcs_test.go +++ b/physical/gcs/gcs_test.go @@ -17,6 +17,7 @@ import ( "github.com/hashicorp/vault/sdk/helper/logging" "github.com/hashicorp/vault/sdk/physical" "google.golang.org/api/googleapi" + "google.golang.org/api/option" ) func testCleanup(t testing.TB, client *storage.Client, bucket string) { @@ -36,11 +37,18 @@ func TestBackend(t *testing.T) { t.Skip("GOOGLE_PROJECT_ID not set") } + universeDomain := os.Getenv("GOOGLE_UNIVERSE_DOMAIN") + r := rand.New(rand.NewSource(time.Now().UnixNano())).Int() bucket := fmt.Sprintf("vault-gcs-testacc-%d", r) ctx := context.Background() - client, err := storage.NewClient(ctx) + // Build client options: if a custom universe domain is provided in env, use it. + clientOpts := []option.ClientOption{} + if universeDomain != "" { + clientOpts = append(clientOpts, option.WithUniverseDomain(universeDomain)) + } + client, err := storage.NewClient(ctx, clientOpts...) if err != nil { t.Fatal(err) } @@ -49,13 +57,27 @@ func TestBackend(t *testing.T) { defer testCleanup(t, client, bucket) b := client.Bucket(bucket) - if err := b.Create(context.Background(), projectID, nil); err != nil { - t.Fatal(err) + // Support minimal for providers that require an explicit Location. + bucketLocation := os.Getenv("GOOGLE_BUCKET_LOCATION") + + if bucketLocation != "" { + // Create the bucket with the explicit location required by some universe domains. + if err := b.Create(ctx, projectID, &storage.BucketAttrs{ + Location: bucketLocation, + }); err != nil { + t.Fatalf("failed to create bucket %q with location %q: %v", bucket, bucketLocation, err) + } + } else { + // Default behaviour (no explicit location). + if err := b.Create(ctx, projectID, nil); err != nil { + t.Fatalf("failed to create bucket %q: %v", bucket, err) + } } backend, err := NewBackend(map[string]string{ - "bucket": bucket, - "ha_enabled": "false", + "bucket": bucket, + "ha_enabled": "false", + "universe_domain": universeDomain, }, logging.NewVaultLogger(log.Trace)) if err != nil { t.Fatal(err) From d3d8e1006bb60014234d77b927a1cb5d37a8bf78 Mon Sep 17 00:00:00 2001 From: Houssein Mnaouar Date: Sun, 14 Sep 2025 19:03:41 +0000 Subject: [PATCH 2/2] changelog: add entry for GCS universe_domain support --- changelog/31550.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog/31550.txt diff --git a/changelog/31550.txt b/changelog/31550.txt new file mode 100644 index 0000000000..3ea2076e06 --- /dev/null +++ b/changelog/31550.txt @@ -0,0 +1,3 @@ +```release-note:feature +gcs: add support for the `universe_domain` parameter, allowing the backend to work with GCP sovereign cloud environments that require a custom API domain. +``` \ No newline at end of file