diff --git a/.github/workflows/build-hcp-image.yml b/.github/workflows/build-hcp-image.yml new file mode 100644 index 0000000000..1dc4b4a606 --- /dev/null +++ b/.github/workflows/build-hcp-image.yml @@ -0,0 +1,24 @@ +name: build-hcp-image + +on: + workflow_call: + inputs: + pull-request: + type: number + create-azure-image: + type: boolean + create-aws-image: + type: boolean + hcp-environment: + type: string + outputs: + artifact: + value: shim + image: + value: shim + +jobs: + create-image: + runs-on: ubuntu-latest + steps: + - run: exit 0 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 974f1f3c60..751882c173 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -291,6 +291,24 @@ jobs: web-ui-cache-key: ${{ needs.ui.outputs.cache-key }} secrets: inherit + hcp-image: + if: | + needs.setup.outputs.is-ent-branch == 'true' && + needs.setup.outputs.workflow-trigger == 'pull_request' && + ( + contains(fromJSON(needs.setup.outputs.labels), 'hcp/build-image') || + contains(fromJSON(needs.setup.outputs.labels), 'hcp/test') + ) + needs: + - setup + - artifacts-ent + uses: ./.github/workflows/build-hcp-image.yml + with: + pull-request: ${{ github.event.pull_request.number }} + create-aws-image: true + create-azure-image: false + hcp-environment: int + test: # Test all of the testable artifacts if our repo isn't a fork. We don't test when the PR is # created from a fork because secrets are not passed in and they are required. @@ -347,6 +365,23 @@ jobs: vault-version: ${{ needs.setup.outputs.vault-version-metadata }} secrets: inherit + test-hcp-image: + # Test our custom HCP image if our image build was successful and we've + # been configured with the correct label. + if: | + needs.setup.outputs.is-ent-branch == 'true' && + needs.setup.outputs.workflow-trigger == 'pull_request' && + needs.artifacts-ent.result == 'success' && + needs.hcp-image.result == 'success' && + contains(fromJSON(needs.setup.outputs.labels), 'hcp/test') + needs: + - setup + - artifacts-ent + - hcp-image + uses: ./.github/workflows/test-run-enos-scenario-cloud.yml + with: + product-version: ${{ fromJSON(needs.hcp-image.outputs.image).product_version }} + completed-successfully: # build/completed-successfully is the only build workflow that must pass in order to merge # a pull request. This workflow is used to determine the overall status of all the prior @@ -369,8 +404,10 @@ jobs: - ui - artifacts-ce - artifacts-ent + - hcp-image - test - test-containers + - test-hcp-image steps: - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - id: disallow-merge-on-ce diff --git a/.github/workflows/test-run-enos-scenario-cloud.yml b/.github/workflows/test-run-enos-scenario-cloud.yml new file mode 100644 index 0000000000..22e2b192da --- /dev/null +++ b/.github/workflows/test-run-enos-scenario-cloud.yml @@ -0,0 +1,14 @@ +--- +name: enos-cloud + +on: + workflow_call: + inputs: + product-version: + type: string + +jobs: + run: + runs-on: ubuntu-latest + steps: + - run: exit 0 diff --git a/enos/modules/hcp/create_vault_cluster/main.tf b/enos/modules/hcp/create_vault_cluster/main.tf index 39fbcee585..4cd5186c34 100644 --- a/enos/modules/hcp/create_vault_cluster/main.tf +++ b/enos/modules/hcp/create_vault_cluster/main.tf @@ -21,18 +21,6 @@ variable "cloud_region" { default = "us-west-2" } -variable "cluster_id" { - description = "The ID of the HCP Vault cluster." - type = string - default = "enos" -} - -variable "hvn_id" { - description = "The ID of the HCP HVN." - type = string - default = "default" -} - variable "maintenance_window_day" { description = "The maintenance window day" type = string @@ -48,13 +36,14 @@ variable "maintenance_window_time" { variable "min_vault_version" { description = "The minimum vault version. This also corresponds to the image id" type = string - default = "default" + default = null } variable "tier" { description = "Tier of the HCP Vault cluster. Valid options for tiers." type = string - default = "dev" + // NOTE: we can't use dev for custom images + default = "plus_small" } variable "upgrade_type" { @@ -65,17 +54,38 @@ variable "upgrade_type" { data "enos_environment" "localhost" {} +resource "random_string" "id" { + length = 4 + lower = true + upper = false + numeric = false + special = false +} + +locals { + // Generate a unique identifier for our scenario. If we've been given a + // min_vault_version we'll use that as it will likely be the version and + // a SHA of a custom image. Make sure it doesn't have special characters. + // Otherwise, just use a random string. + id = var.min_vault_version != null ? try(replace(var.min_vault_version, "/[^0-9A-Za-z]/", "-"), random_string.id.result) : random_string.id.result +} + resource "hcp_hvn" "default" { - hvn_id = var.hvn_id + hvn_id = local.id cloud_provider = var.cloud_provider region = var.cloud_region } resource "hcp_vault_cluster" "enos" { - hvn_id = hcp_hvn.default.id - cluster_id = var.cluster_id - tier = var.tier - public_endpoint = true + depends_on = [ + hcp_hvn.default, + ] + + hvn_id = local.id + cluster_id = "enos-${local.id}" + tier = var.tier + public_endpoint = true + min_vault_version = var.min_vault_version dynamic "ip_allowlist" { for_each = data.enos_environment.localhost.public_ipv4_addresses @@ -84,11 +94,13 @@ resource "hcp_vault_cluster" "enos" { } } + /* major_version_upgrade_config { maintenance_window_day = var.maintenance_window_day maintenance_window_time = var.maintenance_window_time upgrade_type = var.upgrade_type } + */ } output "cloud_provider" { diff --git a/tools/pipeline/internal/cmd/hcp_show_image.go b/tools/pipeline/internal/cmd/hcp_show_image.go index 1b2a315afa..7dbe5059ed 100644 --- a/tools/pipeline/internal/cmd/hcp_show_image.go +++ b/tools/pipeline/internal/cmd/hcp_show_image.go @@ -11,7 +11,9 @@ import ( "github.com/spf13/cobra" ) -var showHCPImageReq = &hcp.GetLatestProductVersionReq{} +var showHCPImageReq = &hcp.ShowImageReq{ + Req: &hcp.GetLatestProductVersionReq{}, +} func newHCPShowImageCmd() *cobra.Command { availability := "" @@ -21,18 +23,19 @@ func newHCPShowImageCmd() *cobra.Command { Short: "Show details of an HCP image", Long: "Show details of an HCP image", PersistentPreRun: func(cmd *cobra.Command, args []string) { - showHCPImageReq.Availability = hcp.GetLatestProductVersionAvailability(availability) + showHCPImageReq.Req.Availability = hcp.GetLatestProductVersionAvailability(availability) }, RunE: runHCPImageShowLatestCmd, } - showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.ProductName, "product-name", "p", "vault", "The product or component of the image") - showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.ProductVersionConstraint, "product-version-constraint", "v", "", "A comma seperated list of constraints. If left unset the latest will be returned") - showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.HostManagerVersionConstraint, "host-manager-version-constraint", "m", "", "A semver string. If left unset the latest will be used") - showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.CloudProvider, "cloud", "c", "aws", "The cloud provider you wish to search. E.g. aws, azure") - showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.CloudRegion, "region", "r", "us-west-2", "The cloud region you wish to search") + showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.Req.ProductName, "product-name", "p", "vault", "The product or component of the image") + showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.Req.ProductVersionConstraint, "product-version-constraint", "v", "", "A comma seperated list of constraints. If left unset the latest will be returned") + showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.Req.HostManagerVersionConstraint, "host-manager-version-constraint", "m", "", "A semver string. If left unset the latest will be used") + showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.Req.CloudProvider, "cloud", "c", "aws", "The cloud provider you wish to search. E.g. aws, azure") + showHCPImage.PersistentFlags().StringVarP(&showHCPImageReq.Req.CloudRegion, "region", "r", "us-west-2", "The cloud region you wish to search") showHCPImage.PersistentFlags().StringVarP(&availability, "availability", "a", "public", "The image availability") - showHCPImage.PersistentFlags().BoolVarP(&showHCPImageReq.ExcludeReleaseCandidates, "exclude-release-candidates", "x", false, "Exclude release candidates") + showHCPImage.PersistentFlags().BoolVarP(&showHCPImageReq.Req.ExcludeReleaseCandidates, "exclude-release-candidates", "x", false, "Exclude release candidates") + showHCPImage.PersistentFlags().BoolVar(&showHCPImageReq.WriteToGithubOutput, "github-output", false, "Whether or not to write 'show-image' to $GITHUB_OUTPUT") return showHCPImage } @@ -47,17 +50,26 @@ func runHCPImageShowLatestCmd(cmd *cobra.Command, args []string) error { switch rootCfg.format { case "json": - b, err := res.ToJSON() + b, err := res.Res.ToJSON() if err != nil { return err } fmt.Println(string(b)) case "markdown": - tbl := res.ToTable() + tbl := res.Res.ToTable() tbl.SetTitle("HCP Image") fmt.Println(tbl.RenderMarkdown()) default: - fmt.Println(res.ToTable().Render()) + fmt.Println(res.Res.ToTable().Render()) + } + + if showHCPImageReq.WriteToGithubOutput { + output, err := res.ToGithubOutput() + if err != nil { + return err + } + + return writeToGithubOutput("show-image", output) } return nil diff --git a/tools/pipeline/internal/cmd/hcp_wait_image.go b/tools/pipeline/internal/cmd/hcp_wait_image.go index 8054ae5c00..7e8f6c7e20 100644 --- a/tools/pipeline/internal/cmd/hcp_wait_image.go +++ b/tools/pipeline/internal/cmd/hcp_wait_image.go @@ -16,7 +16,7 @@ import ( "github.com/spf13/cobra" ) -var waitForHCPImage = &hcp.WaitForImageReq{ +var waitForHCPImageReq = &hcp.WaitForImageReq{ Req: &hcp.GetLatestProductVersionReq{}, } @@ -24,12 +24,12 @@ func newHCPWaitForImageCmd() *cobra.Command { availability := "" var timeout time.Duration - imageGetLatestCmd := &cobra.Command{ + waitHCPImage := &cobra.Command{ Use: "image", - Short: "Show details of an HCP image", - Long: "Show details of an HCP image", + Short: "Wait for an HCP image", + Long: "Wait for an HCP image", PersistentPreRun: func(cmd *cobra.Command, args []string) { - waitForHCPImage.Req.Availability = hcp.GetLatestProductVersionAvailability(availability) + waitForHCPImageReq.Req.Availability = hcp.GetLatestProductVersionAvailability(availability) }, RunE: func(cmd *cobra.Command, args []string) error { cmd.SilenceUsage = true // Don't spam the usage on failure @@ -51,7 +51,7 @@ func newHCPWaitForImageCmd() *cobra.Command { } }() - res, err := waitForHCPImage.Run(ctx, hcpCmdState.client) + res, err := waitForHCPImageReq.Run(ctx, hcpCmdState.client) if err != nil { return fmt.Errorf("waiting for an HCP image: %w", err) } @@ -71,19 +71,29 @@ func newHCPWaitForImageCmd() *cobra.Command { fmt.Println(res.Res.ToTable().Render()) } + if waitForHCPImageReq.WriteToGithubOutput { + output, err := res.ToGithubOutput() + if err != nil { + return err + } + + return writeToGithubOutput("wait-image", output) + } + return nil }, } - imageGetLatestCmd.PersistentFlags().StringVarP(&waitForHCPImage.Req.ProductName, "product-name", "p", "vault", "The product or component of the image") - imageGetLatestCmd.PersistentFlags().StringVarP(&waitForHCPImage.Req.ProductVersionConstraint, "product-version-constraint", "v", "", "A comma seperated list of constraints. If left unset the latest will be returned") - imageGetLatestCmd.PersistentFlags().StringVarP(&waitForHCPImage.Req.HostManagerVersionConstraint, "host-manager-version-constraint", "m", "", "A semver string. If left unset the latest will be used") - imageGetLatestCmd.PersistentFlags().StringVarP(&waitForHCPImage.Req.CloudProvider, "cloud", "c", "aws", "The cloud provider you wish to search. E.g. aws, azure") - imageGetLatestCmd.PersistentFlags().StringVarP(&waitForHCPImage.Req.CloudRegion, "region", "r", "us-west-2", "The cloud region you wish to search") - imageGetLatestCmd.PersistentFlags().StringVarP(&availability, "availability", "a", "public", "The image availability") - imageGetLatestCmd.PersistentFlags().BoolVarP(&waitForHCPImage.Req.ExcludeReleaseCandidates, "exclude-release-candidates", "x", false, "Exclude release candidates") - imageGetLatestCmd.PersistentFlags().DurationVarP(&waitForHCPImage.Delay, "delay", "d", 10*time.Second, "the time to wait in-between requests") - imageGetLatestCmd.PersistentFlags().DurationVarP(&timeout, "timeout", "t", 30*time.Minute, "the maximum duration to wait for the image") + waitHCPImage.PersistentFlags().StringVarP(&waitForHCPImageReq.Req.ProductName, "product-name", "p", "vault", "The product or component of the image") + waitHCPImage.PersistentFlags().StringVarP(&waitForHCPImageReq.Req.ProductVersionConstraint, "product-version-constraint", "v", "", "A comma seperated list of constraints. If left unset the latest will be returned") + waitHCPImage.PersistentFlags().StringVarP(&waitForHCPImageReq.Req.HostManagerVersionConstraint, "host-manager-version-constraint", "m", "", "A semver string. If left unset the latest will be used") + waitHCPImage.PersistentFlags().StringVarP(&waitForHCPImageReq.Req.CloudProvider, "cloud", "c", "aws", "The cloud provider you wish to search. E.g. aws, azure") + waitHCPImage.PersistentFlags().StringVarP(&waitForHCPImageReq.Req.CloudRegion, "region", "r", "us-west-2", "The cloud region you wish to search") + waitHCPImage.PersistentFlags().StringVarP(&availability, "availability", "a", "public", "The image availability") + waitHCPImage.PersistentFlags().BoolVarP(&waitForHCPImageReq.Req.ExcludeReleaseCandidates, "exclude-release-candidates", "x", false, "Exclude release candidates") + waitHCPImage.PersistentFlags().DurationVarP(&waitForHCPImageReq.Delay, "delay", "d", 10*time.Second, "the time to wait in-between requests") + waitHCPImage.PersistentFlags().DurationVarP(&timeout, "timeout", "t", 30*time.Minute, "the maximum duration to wait for the image") + waitHCPImage.PersistentFlags().BoolVar(&waitForHCPImageReq.WriteToGithubOutput, "github-output", false, "Whether or not to write 'wait-image' to $GITHUB_OUTPUT") - return imageGetLatestCmd + return waitHCPImage } diff --git a/tools/pipeline/internal/pkg/github/find_workflow_artifact.go b/tools/pipeline/internal/pkg/github/find_workflow_artifact.go index 90e88dcf07..0ee8001dbd 100644 --- a/tools/pipeline/internal/pkg/github/find_workflow_artifact.go +++ b/tools/pipeline/internal/pkg/github/find_workflow_artifact.go @@ -80,7 +80,7 @@ func (r *FindWorkflowArtifactReq) Run(ctx context.Context, client *gh.Client) (* } if len(runs) < 1 { - return nil, fmt.Errorf("no matching workflow runs are associated with the pull request: %w", err) + return nil, fmt.Errorf("no matching workflow runs are associated with the pull request") } // In instances where we have more than one run we want to get the artifact diff --git a/tools/pipeline/internal/pkg/github/workflows.go b/tools/pipeline/internal/pkg/github/workflows.go index e05b28d33c..52f2e5299f 100644 --- a/tools/pipeline/internal/pkg/github/workflows.go +++ b/tools/pipeline/internal/pkg/github/workflows.go @@ -62,13 +62,14 @@ func getWorkflowRuns( // By default our status will be "success" which elimates in_progress runs. // Instead, we'll try both so that we're sure to include what's actually // running along with historical runs. - for _, status := range []string{"success", "in_progress"} { + for _, status := range []string{"", "success", "in_progress"} { + var runsForStatus []*WorkflowRun for { opts.Status = status slog.Default().DebugContext(slogctx.Append(ctx, slog.String("owner", owner), slog.String("repo", repo), - slog.Int64("id", id), + slog.Int64("workflow-id", id), slog.String("query-status", opts.Status), ), "getting github actions workflow runs") @@ -78,10 +79,27 @@ func getWorkflowRuns( } for _, r := range wfrs.WorkflowRuns { - runs = append(runs, &WorkflowRun{Run: r}) + runsForStatus = append(runsForStatus, &WorkflowRun{Run: r}) } if res.NextPage == 0 { + if len(runsForStatus) > 0 { + slog.Default().DebugContext(slogctx.Append(ctx, + slog.String("owner", owner), + slog.String("repo", repo), + slog.Int64("workflow-id", id), + slog.String("query-status", opts.Status), + slog.Int("count", len(runsForStatus)), + ), "found github actions workflow runs") + } else { + slog.Default().DebugContext(slogctx.Append(ctx, + slog.String("owner", owner), + slog.String("repo", repo), + slog.Int64("workflow-id", id), + slog.String("query-status", opts.Status), + ), "no github actions workflow runs found for status") + } + runs = append(runs, runsForStatus...) break } @@ -103,12 +121,29 @@ func getWorkflowRunArtifacts( slog.Default().DebugContext(slogctx.Append(ctx, slog.String("owner", owner), slog.String("repo", repo), - slog.Int64("id", id), + slog.Int64("run-id", id), ), "getting github actions workflow run artifacts") opts := &gh.ListOptions{PerPage: PerPageMax} artifacts := gh.ArtifactList{} + defer func() { + if count := artifacts.GetTotalCount(); count > 0 { + slog.Default().DebugContext(slogctx.Append(ctx, + slog.String("owner", owner), + slog.String("repo", repo), + slog.Int64("run-id", id), + slog.Int64("count", count), + ), "found workflow run artifacts") + } else { + slog.Default().DebugContext(slogctx.Append(ctx, + slog.String("owner", owner), + slog.String("repo", repo), + slog.Int64("run-id", id), + ), "no workflow run artifacts found") + } + }() + for { arts, res, err := client.Actions.ListWorkflowRunArtifacts(ctx, owner, repo, id, opts) if err != nil { diff --git a/tools/pipeline/internal/pkg/hcp/show_image.go b/tools/pipeline/internal/pkg/hcp/show_image.go new file mode 100644 index 0000000000..39adb44699 --- /dev/null +++ b/tools/pipeline/internal/pkg/hcp/show_image.go @@ -0,0 +1,54 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package hcp + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" +) + +// ShowImageReq is a request to wait for an image to be available in the +// image service. +type ShowImageReq struct { + Req *GetLatestProductVersionReq `json:"req,omitempty"` + WriteToGithubOutput bool `json:"write_to_github_output,omitempty"` +} + +// ShowImageRes is a response to a ShowImageReq. +type ShowImageRes struct { + Res *GetLatestProductVersionRes `json:"res,omitempty"` +} + +// Run runs the wait for image request. +func (r *ShowImageReq) Run(ctx context.Context, client *Client) (*ShowImageRes, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + slog.Default().DebugContext(ctx, "showing HCP image") + + res := &ShowImageRes{} + var err error + res.Res, err = r.Req.Run(ctx, client) + + return res, err +} + +// ToGithubOutput marshals just the artifact response to JSON. +func (r *ShowImageRes) ToGithubOutput() ([]byte, error) { + if r == nil || r.Res == nil { + return nil, fmt.Errorf("unable to marshal unitialized response to GITHUB_OUTPUT") + } + + b, err := json.Marshal(r.Res.Image) + if err != nil { + return nil, fmt.Errorf("marshaling show-image to GITHUB_OUTPUT JSON: %w", err) + } + + return b, nil +} diff --git a/tools/pipeline/internal/pkg/hcp/wait_for_image.go b/tools/pipeline/internal/pkg/hcp/wait_for_image.go index 2e75f0b541..e5d454b800 100644 --- a/tools/pipeline/internal/pkg/hcp/wait_for_image.go +++ b/tools/pipeline/internal/pkg/hcp/wait_for_image.go @@ -5,6 +5,8 @@ package hcp import ( "context" + "encoding/json" + "fmt" "log/slog" "time" @@ -15,8 +17,9 @@ import ( // WaitForImageReq is a request to wait for an image to be available in the // image service. type WaitForImageReq struct { - Req *GetLatestProductVersionReq `json:"req,omitempty"` - Delay time.Duration `json:"delay,omitempty"` + Req *GetLatestProductVersionReq `json:"req,omitempty"` + Delay time.Duration `json:"delay,omitempty"` + WriteToGithubOutput bool `json:"write_to_github_output,omitempty"` } // WaitForImageRes is a response to a WaitForImageReq. @@ -74,3 +77,17 @@ func (r *WaitForImageReq) Run(ctx context.Context, client *Client) (*WaitForImag return res, err } + +// ToGithubOutput marshals just the artifact response to JSON. +func (r *WaitForImageRes) ToGithubOutput() ([]byte, error) { + if r == nil || r.Res == nil { + return nil, fmt.Errorf("unable to marshal unitialized response to GITHUB_OUTPUT") + } + + b, err := json.Marshal(r.Res.Image) + if err != nil { + return nil, fmt.Errorf("marshaling wait-image to GITHUB_OUTPUT JSON: %w", err) + } + + return b, nil +}