Backport [VAULT-39160] actions(hcp): add support for testing custom images on HCP into ce/main (#9433)

[VAULT-39160] actions(hcp): add support for testing custom images on HCP (#9345)

Add support for running the `cloud` scenario with a custom image in the
int HCP environment. We support two new tags that trigger new
functionality. If the `hcp/build-image` tag is present on a PR at the
time of `build`, we'll automatically trigger a custom build for the int
environment. If the `hcp/test` tag is present, we'll trigger a custom
build and run the `cloud` scenario with the resulting image.

* Fix a bug in our custom build pattern to handle prerelease versions.
* pipeline(hcp): add `--github-output` support to `show image` and
  `wait image` commands.
* enos(hcp/create_vault_cluster): use a unique identifier for HVN
  and vault clusters.
* actions(enos-cloud): add workflow to execute the `cloud` enos
  scenario.
* actions(build): add support for triggering a custom build and running
  the `enos-cloud` scenario.
* add more debug logging and query without a status
* add shim build-hcp-image for CE workflows

Signed-off-by: Ryan Cragun <me@ryan.ec>
Co-authored-by: Ryan Cragun <me@ryan.ec>

.github/workflows/build-hcp-image.yml

@@ -0,0 +1,24 @@
+name: build-hcp-image
+
+on:
+  workflow_call:
+    inputs:
+      pull-request:
+        type: number
+      create-azure-image:
+        type: boolean
+      create-aws-image:
+        type: boolean
+      hcp-environment:
+        type: string
+    outputs:
+      artifact:
+        value: shim
+      image:
+        value: shim
+
+jobs:
+  create-image:
+    runs-on: ubuntu-latest
+    steps:
+      - run: exit 0

.github/workflows/build.yml

@@ -291,6 +291,24 @@ jobs:
       web-ui-cache-key: ${{ needs.ui.outputs.cache-key }}
     secrets: inherit
 
+  hcp-image:
+    if: |
+      needs.setup.outputs.is-ent-branch == 'true' &&
+      needs.setup.outputs.workflow-trigger == 'pull_request' &&
+      (
+        contains(fromJSON(needs.setup.outputs.labels), 'hcp/build-image') ||
+        contains(fromJSON(needs.setup.outputs.labels), 'hcp/test')
+      )
+    needs:
+      - setup
+      - artifacts-ent
+    uses: ./.github/workflows/build-hcp-image.yml
+    with:
+      pull-request: ${{ github.event.pull_request.number }}
+      create-aws-image: true
+      create-azure-image: false
+      hcp-environment: int
+
   test:
     # Test all of the testable artifacts if our repo isn't a fork. We don't test when the PR is
     # created from a fork because secrets are not passed in and they are required.
@@ -347,6 +365,23 @@ jobs:
       vault-version: ${{ needs.setup.outputs.vault-version-metadata }}
     secrets: inherit
 
+  test-hcp-image:
+    # Test our custom HCP image if our image build was successful and we've
+    # been configured with the correct label.
+    if: |
+      needs.setup.outputs.is-ent-branch == 'true' &&
+      needs.setup.outputs.workflow-trigger == 'pull_request' &&
+      needs.artifacts-ent.result == 'success' &&
+      needs.hcp-image.result == 'success' &&
+      contains(fromJSON(needs.setup.outputs.labels), 'hcp/test')
+    needs:
+      - setup
+      - artifacts-ent
+      - hcp-image
+    uses: ./.github/workflows/test-run-enos-scenario-cloud.yml
+    with:
+      product-version: ${{ fromJSON(needs.hcp-image.outputs.image).product_version }}
+
   completed-successfully:
     # build/completed-successfully is the only build workflow that must pass in order to merge
     # a pull request. This workflow is used to determine the overall status of all the prior
@@ -369,8 +404,10 @@ jobs:
       - ui
       - artifacts-ce
       - artifacts-ent
+      - hcp-image
       - test
       - test-containers
+      - test-hcp-image
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
       - id: disallow-merge-on-ce

.github/workflows/test-run-enos-scenario-cloud.yml

@@ -0,0 +1,14 @@
+---
+name: enos-cloud
+
+on:
+  workflow_call:
+    inputs:
+      product-version:
+        type: string
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    steps:
+      - run: exit 0

enos/modules/hcp/create_vault_cluster/main.tf

@@ -21,18 +21,6 @@ variable "cloud_region" {
   default     = "us-west-2"
 }
 
-variable "cluster_id" {
-  description = "The ID of the HCP Vault cluster."
-  type        = string
-  default     = "enos"
-}
-
-variable "hvn_id" {
-  description = "The ID of the HCP HVN."
-  type        = string
-  default     = "default"
-}
-
 variable "maintenance_window_day" {
   description = "The maintenance window day"
   type        = string
@@ -48,13 +36,14 @@ variable "maintenance_window_time" {
 variable "min_vault_version" {
   description = "The minimum vault version. This also corresponds to the image id"
   type        = string
-  default     = "default"
+  default     = null
 }
 
 variable "tier" {
   description = "Tier of the HCP Vault cluster. Valid options for tiers."
   type        = string
-  default     = "dev"
+  // NOTE: we can't use dev for custom images
+  default = "plus_small"
 }
 
 variable "upgrade_type" {
@@ -65,17 +54,38 @@ variable "upgrade_type" {
 
 data "enos_environment" "localhost" {}
 
+resource "random_string" "id" {
+  length  = 4
+  lower   = true
+  upper   = false
+  numeric = false
+  special = false
+}
+
+locals {
+  // Generate a unique identifier for our scenario. If we've been given a
+  // min_vault_version we'll use that as it will likely be the version and
+  // a SHA of a custom image. Make sure it doesn't have special characters.
+  // Otherwise, just use a random string.
+  id = var.min_vault_version != null ? try(replace(var.min_vault_version, "/[^0-9A-Za-z]/", "-"), random_string.id.result) : random_string.id.result
+}
+
 resource "hcp_hvn" "default" {
-  hvn_id         = var.hvn_id
+  hvn_id         = local.id
   cloud_provider = var.cloud_provider
   region         = var.cloud_region
 }
 
 resource "hcp_vault_cluster" "enos" {
-  hvn_id          = hcp_hvn.default.id
-  cluster_id      = var.cluster_id
-  tier            = var.tier
-  public_endpoint = true
+  depends_on = [
+    hcp_hvn.default,
+  ]
+
+  hvn_id            = local.id
+  cluster_id        = "enos-${local.id}"
+  tier              = var.tier
+  public_endpoint   = true
+  min_vault_version = var.min_vault_version
 
   dynamic "ip_allowlist" {
     for_each = data.enos_environment.localhost.public_ipv4_addresses
@@ -84,11 +94,13 @@ resource "hcp_vault_cluster" "enos" {
     }
   }
 
+  /*
   major_version_upgrade_config {
     maintenance_window_day  = var.maintenance_window_day
     maintenance_window_time = var.maintenance_window_time
     upgrade_type            = var.upgrade_type
   }
+  */
 }
 
 output "cloud_provider" {

tools/pipeline/internal/cmd/hcp_show_image.go

@@ -11,7 +11,9 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var showHCPImageReq = &hcp.GetLatestProductVersionReq{}
+var showHCPImageReq = &hcp.ShowImageReq{
+	Req: &hcp.GetLatestProductVersionReq{},
+}
 
 func newHCPShowImageCmd() *cobra.Command {
 	availability := ""
@@ -21,18 +23,19 @@ func newHCPShowImageCmd() *cobra.Command {
 		Short: "Show details of an HCP image",
 		Long:  "Show details of an HCP image",
 		PersistentPreRun: func(cmd *cobra.Command, args []string) {
-			showHCPImageReq.Availability = hcp.GetLatestProductVersionAvailability(availability)
+			showHCPImageReq.Req.Availability = hcp.GetLatestProductVersionAvailability(availability)
 		},
 		RunE: runHCPImageShowLatestCmd,
 	}
 
-	showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.ProductName, "product-name", "p", "vault", "The product or component of the image")
-	showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.ProductVersionConstraint, "product-version-constraint", "v", "", "A comma seperated list of constraints. If left unset the latest will be returned")
-	showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.HostManagerVersionConstraint, "host-manager-version-constraint", "m", "", "A semver string. If left unset the latest will be used")
-	showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.CloudProvider, "cloud", "c", "aws", "The cloud provider you wish to search. E.g. aws, azure")
-	showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.CloudRegion, "region", "r", "us-west-2", "The cloud region you wish to search")
+	showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.Req.ProductName, "product-name", "p", "vault", "The product or component of the image")
+	showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.Req.ProductVersionConstraint, "product-version-constraint", "v", "", "A comma seperated list of constraints. If left unset the latest will be returned")
+	showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.Req.HostManagerVersionConstraint, "host-manager-version-constraint", "m", "", "A semver string. If left unset the latest will be used")
+	showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.Req.CloudProvider, "cloud", "c", "aws", "The cloud provider you wish to search. E.g. aws, azure")
+	showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.Req.CloudRegion, "region", "r", "us-west-2", "The cloud region you wish to search")
 	showHCPImage.PersistentFlags().StringVarP(&availability, "availability", "a", "public", "The image availability")
-	showHCPImage.PersistentFlags().BoolVarP(&showHCPImageReq.ExcludeReleaseCandidates, "exclude-release-candidates", "x", false, "Exclude release candidates")
+	showHCPImage.PersistentFlags().BoolVarP(&showHCPImageReq.Req.ExcludeReleaseCandidates, "exclude-release-candidates", "x", false, "Exclude release candidates")
+	showHCPImage.PersistentFlags().BoolVar(&showHCPImageReq.WriteToGithubOutput, "github-output", false, "Whether or not to write 'show-image' to $GITHUB_OUTPUT")
 
 	return showHCPImage
 }
@@ -47,17 +50,26 @@ func runHCPImageShowLatestCmd(cmd *cobra.Command, args []string) error {
 
 	switch rootCfg.format {
 	case "json":
-		b, err := res.ToJSON()
+		b, err := res.Res.ToJSON()
 		if err != nil {
 			return err
 		}
 		fmt.Println(string(b))
 	case "markdown":
-		tbl := res.ToTable()
+		tbl := res.Res.ToTable()
 		tbl.SetTitle("HCP Image")
 		fmt.Println(tbl.RenderMarkdown())
 	default:
-		fmt.Println(res.ToTable().Render())
+		fmt.Println(res.Res.ToTable().Render())
+	}
+
+	if showHCPImageReq.WriteToGithubOutput {
+		output, err := res.ToGithubOutput()
+		if err != nil {
+			return err
+		}
+
+		return writeToGithubOutput("show-image", output)
 	}
 
 	return nil

tools/pipeline/internal/cmd/hcp_wait_image.go

@@ -16,7 +16,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var waitForHCPImage = &hcp.WaitForImageReq{
+var waitForHCPImageReq = &hcp.WaitForImageReq{
 	Req: &hcp.GetLatestProductVersionReq{},
 }
 
@@ -24,12 +24,12 @@ func newHCPWaitForImageCmd() *cobra.Command {
 	availability := ""
 	var timeout time.Duration
 
-	imageGetLatestCmd := &cobra.Command{
+	waitHCPImage := &cobra.Command{
 		Use:   "image",
-		Short: "Show details of an HCP image",
-		Long:  "Show details of an HCP image",
+		Short: "Wait for an HCP image",
+		Long:  "Wait for an HCP image",
 		PersistentPreRun: func(cmd *cobra.Command, args []string) {
-			waitForHCPImage.Req.Availability = hcp.GetLatestProductVersionAvailability(availability)
+			waitForHCPImageReq.Req.Availability = hcp.GetLatestProductVersionAvailability(availability)
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			cmd.SilenceUsage = true // Don't spam the usage on failure
@@ -51,7 +51,7 @@ func newHCPWaitForImageCmd() *cobra.Command {
 				}
 			}()
 
-			res, err := waitForHCPImage.Run(ctx, hcpCmdState.client)
+			res, err := waitForHCPImageReq.Run(ctx, hcpCmdState.client)
 			if err != nil {
 				return fmt.Errorf("waiting for an HCP image: %w", err)
 			}
@@ -71,19 +71,29 @@ func newHCPWaitForImageCmd() *cobra.Command {
 				fmt.Println(res.Res.ToTable().Render())
 			}
 
+			if waitForHCPImageReq.WriteToGithubOutput {
+				output, err := res.ToGithubOutput()
+				if err != nil {
+					return err
+				}
+
+				return writeToGithubOutput("wait-image", output)
+			}
+
 			return nil
 		},
 	}
 
-	imageGetLatestCmd.PersistentFlags().StringVarP(&waitForHCPImage.Req.ProductName, "product-name", "p", "vault", "The product or component of the image")
-	imageGetLatestCmd.PersistentFlags().StringVarP(&waitForHCPImage.Req.ProductVersionConstraint, "product-version-constraint", "v", "", "A comma seperated list of constraints. If left unset the latest will be returned")
-	imageGetLatestCmd.PersistentFlags().StringVarP(&waitForHCPImage.Req.HostManagerVersionConstraint, "host-manager-version-constraint", "m", "", "A semver string. If left unset the latest will be used")
-	imageGetLatestCmd.PersistentFlags().StringVarP(&waitForHCPImage.Req.CloudProvider, "cloud", "c", "aws", "The cloud provider you wish to search. E.g. aws, azure")
-	imageGetLatestCmd.PersistentFlags().StringVarP(&waitForHCPImage.Req.CloudRegion, "region", "r", "us-west-2", "The cloud region you wish to search")
-	imageGetLatestCmd.PersistentFlags().StringVarP(&availability, "availability", "a", "public", "The image availability")
-	imageGetLatestCmd.PersistentFlags().BoolVarP(&waitForHCPImage.Req.ExcludeReleaseCandidates, "exclude-release-candidates", "x", false, "Exclude release candidates")
-	imageGetLatestCmd.PersistentFlags().DurationVarP(&waitForHCPImage.Delay, "delay", "d", 10*time.Second, "the time to wait in-between requests")
-	imageGetLatestCmd.PersistentFlags().DurationVarP(&timeout, "timeout", "t", 30*time.Minute, "the maximum duration to wait for the image")
+	waitHCPImage.PersistentFlags().StringVarP(&waitForHCPImageReq.Req.ProductName, "product-name", "p", "vault", "The product or component of the image")
+	waitHCPImage.PersistentFlags().StringVarP(&waitForHCPImageReq.Req.ProductVersionConstraint, "product-version-constraint", "v", "", "A comma seperated list of constraints. If left unset the latest will be returned")
+	waitHCPImage.PersistentFlags().StringVarP(&waitForHCPImageReq.Req.HostManagerVersionConstraint, "host-manager-version-constraint", "m", "", "A semver string. If left unset the latest will be used")
+	waitHCPImage.PersistentFlags().StringVarP(&waitForHCPImageReq.Req.CloudProvider, "cloud", "c", "aws", "The cloud provider you wish to search. E.g. aws, azure")
+	waitHCPImage.PersistentFlags().StringVarP(&waitForHCPImageReq.Req.CloudRegion, "region", "r", "us-west-2", "The cloud region you wish to search")
+	waitHCPImage.PersistentFlags().StringVarP(&availability, "availability", "a", "public", "The image availability")
+	waitHCPImage.PersistentFlags().BoolVarP(&waitForHCPImageReq.Req.ExcludeReleaseCandidates, "exclude-release-candidates", "x", false, "Exclude release candidates")
+	waitHCPImage.PersistentFlags().DurationVarP(&waitForHCPImageReq.Delay, "delay", "d", 10*time.Second, "the time to wait in-between requests")
+	waitHCPImage.PersistentFlags().DurationVarP(&timeout, "timeout", "t", 30*time.Minute, "the maximum duration to wait for the image")
+	waitHCPImage.PersistentFlags().BoolVar(&waitForHCPImageReq.WriteToGithubOutput, "github-output", false, "Whether or not to write 'wait-image' to $GITHUB_OUTPUT")
 
-	return imageGetLatestCmd
+	return waitHCPImage
 }

tools/pipeline/internal/pkg/github/find_workflow_artifact.go

@@ -80,7 +80,7 @@ func (r *FindWorkflowArtifactReq) Run(ctx context.Context, client *gh.Client) (*
 	}
 
 	if len(runs) < 1 {
-		return nil, fmt.Errorf("no matching workflow runs are associated with the pull request: %w", err)
+		return nil, fmt.Errorf("no matching workflow runs are associated with the pull request")
 	}
 
 	// In instances where we have more than one run we want to get the artifact

tools/pipeline/internal/pkg/github/workflows.go

@@ -62,13 +62,14 @@ func getWorkflowRuns(
 	// By default our status will be "success" which elimates in_progress runs.
 	// Instead, we'll try both so that we're sure to include what's actually
 	// running along with historical runs.
-	for _, status := range []string{"success", "in_progress"} {
+	for _, status := range []string{"", "success", "in_progress"} {
+		var runsForStatus []*WorkflowRun
 		for {
 			opts.Status = status
 			slog.Default().DebugContext(slogctx.Append(ctx,
 				slog.String("owner", owner),
 				slog.String("repo", repo),
-				slog.Int64("id", id),
+				slog.Int64("workflow-id", id),
 				slog.String("query-status", opts.Status),
 			), "getting github actions workflow runs")
 
@@ -78,10 +79,27 @@ func getWorkflowRuns(
 			}
 
 			for _, r := range wfrs.WorkflowRuns {
-				runs = append(runs, &WorkflowRun{Run: r})
+				runsForStatus = append(runsForStatus, &WorkflowRun{Run: r})
 			}
 
 			if res.NextPage == 0 {
+				if len(runsForStatus) > 0 {
+					slog.Default().DebugContext(slogctx.Append(ctx,
+						slog.String("owner", owner),
+						slog.String("repo", repo),
+						slog.Int64("workflow-id", id),
+						slog.String("query-status", opts.Status),
+						slog.Int("count", len(runsForStatus)),
+					), "found github actions workflow runs")
+				} else {
+					slog.Default().DebugContext(slogctx.Append(ctx,
+						slog.String("owner", owner),
+						slog.String("repo", repo),
+						slog.Int64("workflow-id", id),
+						slog.String("query-status", opts.Status),
+					), "no github actions workflow runs found for status")
+				}
+				runs = append(runs, runsForStatus...)
 				break
 			}
 
@@ -103,12 +121,29 @@ func getWorkflowRunArtifacts(
 	slog.Default().DebugContext(slogctx.Append(ctx,
 		slog.String("owner", owner),
 		slog.String("repo", repo),
-		slog.Int64("id", id),
+		slog.Int64("run-id", id),
 	), "getting github actions workflow run artifacts")
 
 	opts := &gh.ListOptions{PerPage: PerPageMax}
 	artifacts := gh.ArtifactList{}
 
+	defer func() {
+		if count := artifacts.GetTotalCount(); count > 0 {
+			slog.Default().DebugContext(slogctx.Append(ctx,
+				slog.String("owner", owner),
+				slog.String("repo", repo),
+				slog.Int64("run-id", id),
+				slog.Int64("count", count),
+			), "found workflow run artifacts")
+		} else {
+			slog.Default().DebugContext(slogctx.Append(ctx,
+				slog.String("owner", owner),
+				slog.String("repo", repo),
+				slog.Int64("run-id", id),
+			), "no workflow run artifacts found")
+		}
+	}()
+
 	for {
 		arts, res, err := client.Actions.ListWorkflowRunArtifacts(ctx, owner, repo, id, opts)
 		if err != nil {

tools/pipeline/internal/pkg/hcp/show_image.go

@@ -0,0 +1,54 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package hcp
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"log/slog"
+)
+
+// ShowImageReq is a request to wait for an image to be available in the
+// image service.
+type ShowImageReq struct {
+	Req                 *GetLatestProductVersionReq `json:"req,omitempty"`
+	WriteToGithubOutput bool                        `json:"write_to_github_output,omitempty"`
+}
+
+// ShowImageRes is a response to a ShowImageReq.
+type ShowImageRes struct {
+	Res *GetLatestProductVersionRes `json:"res,omitempty"`
+}
+
+// Run runs the wait for image request.
+func (r *ShowImageReq) Run(ctx context.Context, client *Client) (*ShowImageRes, error) {
+	select {
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	default:
+	}
+
+	slog.Default().DebugContext(ctx, "showing HCP image")
+
+	res := &ShowImageRes{}
+	var err error
+	res.Res, err = r.Req.Run(ctx, client)
+
+	return res, err
+}
+
+// ToGithubOutput marshals just the artifact response to JSON.
+func (r *ShowImageRes) ToGithubOutput() ([]byte, error) {
+	if r == nil || r.Res == nil {
+		return nil, fmt.Errorf("unable to marshal unitialized response to GITHUB_OUTPUT")
+	}
+
+	b, err := json.Marshal(r.Res.Image)
+	if err != nil {
+		return nil, fmt.Errorf("marshaling show-image to GITHUB_OUTPUT JSON: %w", err)
+	}
+
+	return b, nil
+}

tools/pipeline/internal/pkg/hcp/wait_for_image.go

@@ -5,6 +5,8 @@ package hcp
 
 import (
 	"context"
+	"encoding/json"
+	"fmt"
 	"log/slog"
 	"time"
 
@@ -15,8 +17,9 @@ import (
 // WaitForImageReq is a request to wait for an image to be available in the
 // image service.
 type WaitForImageReq struct {
-	Req   *GetLatestProductVersionReq `json:"req,omitempty"`
-	Delay time.Duration               `json:"delay,omitempty"`
+	Req                 *GetLatestProductVersionReq `json:"req,omitempty"`
+	Delay               time.Duration               `json:"delay,omitempty"`
+	WriteToGithubOutput bool                        `json:"write_to_github_output,omitempty"`
 }
 
 // WaitForImageRes is a response to a WaitForImageReq.
@@ -74,3 +77,17 @@ func (r *WaitForImageReq) Run(ctx context.Context, client *Client) (*WaitForImag
 
 	return res, err
 }
+
+// ToGithubOutput marshals just the artifact response to JSON.
+func (r *WaitForImageRes) ToGithubOutput() ([]byte, error) {
+	if r == nil || r.Res == nil {
+		return nil, fmt.Errorf("unable to marshal unitialized response to GITHUB_OUTPUT")
+	}
+
+	b, err := json.Marshal(r.Res.Image)
+	if err != nil {
+		return nil, fmt.Errorf("marshaling wait-image to GITHUB_OUTPUT JSON: %w", err)
+	}
+
+	return b, nil
+}