upstream/packer
1
0
Fork 0
mirror of https://github.com/hashicorp/packer.git synced 2026-06-05 06:42:30 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

hcp: extract all HCP-related code to hcp package

Browse source 
As part of the work to displace everything related to HCP from scattered
places around the Packer code, we move it all to an hcp package.

This in turn reduces the amount of code that the commands have to
integrate, and leaves the HCP details to its own enclave.
This commit is contained in:
Lucas Bajolet 2022-09-06 17:26:49 -04:00 committed by Lucas Bajolet
parent dad07c6097
commit 1cee460d0d
21 changed files with 417 additions and 671 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
acctest/testing.go
View file

@ -146,8 +146,8 @@ func Test(t TestT, c TestCase) {
},
Template: tpl,
})
err = core.Initialize()
if err != nil {
diags := core.Initialize(packer.InitializeOptions{})
if diags.HasErrors() {
t.Fatal(fmt.Sprintf("Failed to init core: %s", err))
return
}

121
command/build.go
View file

@ -3,6 +3,7 @@ package command
import (
"bytes"
"context"
"errors"
"fmt"
"log"
"math"
@ -13,6 +14,7 @@ import (
"github.com/hashicorp/hcl/v2"
packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
"github.com/hashicorp/packer/internal/hcp"
"github.com/hashicorp/packer/packer"
"golang.org/x/sync/semaphore"
@ -88,35 +90,21 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int
return ret
}
diags = TrySetupHCP(packerStarter)
hcpHandler, diags := hcp.GetOrchestrator(packerStarter)
ret = writeDiags(c.Ui, nil, diags)
if ret != 0 {
return ret
}
// This build currently enforces a 1:1 mapping that one publisher can be assigned to a single packer config file.
// It also requires that each config type implements this ConfiguredArtifactMetadataPublisher to return a configured bucket.
// TODO find an option that is not managed by a globally shared Publisher.
ArtifactMetadataPublisher, diags := packerStarter.ConfiguredArtifactMetadataPublisher()
if diags.HasErrors() {
return writeDiags(c.Ui, nil, diags)
}
// We need to create a bucket and an empty iteration before we retrieve builds
// so that we can add the iteration ID to the build's eval context
if ArtifactMetadataPublisher != nil {
if err := ArtifactMetadataPublisher.Initialize(buildCtx); err != nil {
diags := hcl.Diagnostics{
&hcl.Diagnostic{
Summary: "Failed to get HCP Packer Registry iteration",
Detail: fmt.Sprintf("Packer could not create an iteration or "+
"link the build to an existing iteration. Contact HCP Packer "+
"support for further assistance.\nError: %s", err),
Severity: hcl.DiagError,
},
}
return writeDiags(c.Ui, nil, diags)
}
err := hcpHandler.PopulateIteration(buildCtx)
if err != nil {
return writeDiags(c.Ui, nil, hcl.Diagnostics{
&hcl.Diagnostic{
Summary: "HCP: populating iteration failed",
Severity: hcl.DiagError,
Detail: err.Error(),
},
})
}
builds, hcpMap, diags := packerStarter.GetBuilds(packer.GetBuildsOptions{
@ -138,23 +126,6 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int
c.Ui.Say("Debug mode enabled. Builds will not be parallelized.")
}
// Now that builds have been retrieved, we can populate the iteration with
// the builds we expect to run.
if ArtifactMetadataPublisher != nil {
if err := ArtifactMetadataPublisher.PopulateIteration(buildCtx); err != nil {
diags := hcl.Diagnostics{
&hcl.Diagnostic{
Summary: "Failed to register builds to the HCP Packer registry iteration",
Detail: fmt.Sprintf("Packer could not register builds within your "+
"configuration to the iteration. Contact HCP Packer support "+
"for further assistance.\nError: %s", err),
Severity: hcl.DiagError,
},
}
return writeDiags(c.Ui, nil, diags)
}
}
// Compile all the UIs for the builds
colors := [5]packer.UiColor{
packer.UiColorGreen,
@ -215,7 +186,7 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int
m map[string][]packersdk.Artifact
}{m: make(map[string][]packersdk.Artifact)}
// Get the builds we care about
var errors = struct {
var errs = struct {
sync.RWMutex
m map[string]error
}{m: make(map[string]error)}
@ -231,9 +202,9 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int
ui := buildUis[b]
if err := limitParallel.Acquire(buildCtx, 1); err != nil {
ui.Error(fmt.Sprintf("Build '%s' failed to acquire semaphore: %s", name, err))
errors.Lock()
errors.m[name] = err
errors.Unlock()
errs.Lock()
errs.m[name] = err
errs.Unlock()
break
}
// Increment the waitgroup so we wait for this item to finish properly
@ -248,19 +219,13 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int
defer limitParallel.Release(1)
if ArtifactMetadataPublisher != nil {
err := ArtifactMetadataPublisher.BuildStart(buildCtx, hcpMap[name])
if err != nil {
msg := err.Error()
if strings.Contains(msg, "already done") {
ui.Say(fmt.Sprintf(
"Build %q already done for bucket %q, skipping to prevent drift: %q",
name,
ArtifactMetadataPublisher.Slug,
err))
return
}
err := hcpHandler.BuildStart(buildCtx, hcpMap[name])
if err != nil {
if errors.As(err, &hcp.BuildDone{}) {
ui.Say(fmt.Sprintf(
"skipping HCP-enabled build %q: already done.",
name))
return
}
}
@ -272,24 +237,28 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int
buildDuration := buildEnd.Sub(buildStart)
fmtBuildDuration := durafmt.Parse(buildDuration).LimitFirstN(2)
if ArtifactMetadataPublisher != nil {
runArtifacts, err = ArtifactMetadataPublisher.BuildDone(
buildCtx,
hcpMap[name],
runArtifacts,
err,
)
if err != nil {
ui.Error(fmt.Sprintf("failed to complete HCP build %q: %s",
name, err))
}
runArtifacts, hcperr := hcpHandler.BuildDone(
buildCtx,
hcpMap[name],
runArtifacts,
err)
if hcperr != nil {
writeDiags(c.Ui, nil, hcl.Diagnostics{
&hcl.Diagnostic{
Summary: fmt.Sprintf(
"failed to complete HCP-enabled build %q",
name),
Severity: hcl.DiagError,
Detail: hcperr.Error(),
},
})
}
if err != nil {
ui.Error(fmt.Sprintf("Build '%s' errored after %s: %s", name, fmtBuildDuration, err))
errors.Lock()
errors.m[name] = err
errors.Unlock()
errs.Lock()
errs.m[name] = err
errs.Unlock()
} else {
ui.Say(fmt.Sprintf("Build '%s' finished after %s.", name, fmtBuildDuration))
if runArtifacts != nil {
@ -327,11 +296,11 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int
return 1
}
if len(errors.m) > 0 {
c.Ui.Machine("error-count", strconv.FormatInt(int64(len(errors.m)), 10))
if len(errs.m) > 0 {
c.Ui.Machine("error-count", strconv.FormatInt(int64(len(errs.m)), 10))
c.Ui.Error("\n==> Some builds didn't complete successfully and had errors:")
for name, err := range errors.m {
for name, err := range errs.m {
// Create a UI for the machine readable stuff to be targeted
ui := &packer.TargetedUI{
Target: name,
@ -394,7 +363,7 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int
c.Ui.Say("\n==> Builds finished but no artifacts were created.")
}
if len(errors.m) > 0 {
if len(errs.m) > 0 {
// If any errors occurred, exit with a non-zero exit status
ret = 1
}

72
command/core_wrapper.go
View file

@ -1,72 +0,0 @@
package command
import (
"fmt"
"github.com/hashicorp/hcl/v2"
packerregistry "github.com/hashicorp/packer/internal/registry"
"github.com/hashicorp/packer/packer"
plugingetter "github.com/hashicorp/packer/packer/plugin-getter"
)
// CoreWrapper wraps a packer.Core in order to have it's Initialize func return
// a diagnostic.
type CoreWrapper struct {
*packer.Core
}
func (c *CoreWrapper) Initialize(_ packer.InitializeOptions) hcl.Diagnostics {
err := c.Core.Initialize()
if err != nil {
return hcl.Diagnostics{
&hcl.Diagnostic{
Detail: err.Error(),
Severity: hcl.DiagError,
},
}
}
return nil
}
func (c *CoreWrapper) PluginRequirements() (plugingetter.Requirements, hcl.Diagnostics) {
return nil, hcl.Diagnostics{
&hcl.Diagnostic{
Summary: "Packer plugins currently only works with HCL2 configuration templates",
Detail: "Please manually install plugins with the plugins command or use a HCL2 configuration that will do that for you.",
Severity: hcl.DiagError,
},
}
}
// ConfiguredArtifactMetadataPublisher returns a configured image bucket that can be used for publishing
// build image artifacts to a configured Packer Registry destination.
func (c *CoreWrapper) ConfiguredArtifactMetadataPublisher() (*packerregistry.Bucket, hcl.Diagnostics) {
bucket := c.Core.GetRegistryBucket()
// If at this point the bucket is nil, it means the HCP Packer registry is not enabled
if bucket == nil {
return nil, hcl.Diagnostics{
&hcl.Diagnostic{
Summary: "Publishing build artifacts to HCP Packer Registry not enabled",
Detail: "No Packer Registry configuration detected; skipping all publishing steps " +
"See publishing to a Packer registry for Packer configuration details",
Severity: hcl.DiagWarning,
},
}
}
err := bucket.Validate()
if err != nil {
return nil, hcl.Diagnostics{
&hcl.Diagnostic{
Summary: "Invalid HCP Packer configuration",
Detail: fmt.Sprintf("Packer could not validate the provided "+
"HCP Packer registry configuration. Check the error message for details "+
"or contact HCP Packer support for further assistance.\nError: %s", err),
Severity: hcl.DiagError,
},
}
}
return bucket, nil
}

4
command/hcl2_upgrade.go
View file

@ -159,8 +159,8 @@ func (c *HCL2UpgradeCommand) RunContext(_ context.Context, cla *HCL2UpgradeArgs)
return 1
}
core := hdl.(*CoreWrapper).Core
if err := core.Initialize(); err != nil {
core := hdl.(*packer.Core)
if err := core.Initialize(packer.InitializeOptions{}); err != nil {
c.Ui.Error(fmt.Sprintf("Ignoring following initialization error: %v", err))
}
tpl := core.Template

2
command/meta.go
View file

@ -169,5 +169,5 @@ func (m *Meta) GetConfigFromJSON(cla *MetaArgs) (packer.Handler, int) {
m.Ui.Error(err.Error())
ret = 1
}
return &CoreWrapper{core}, ret
return core, ret
}

292
command/registry_test.go
View file

@ -1,292 +0,0 @@
package command
import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/hashicorp/hcl/v2"
packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
"github.com/hashicorp/packer/builder/file"
"github.com/hashicorp/packer/hcl2template"
"github.com/hashicorp/packer/internal/registry"
"github.com/hashicorp/packer/packer"
)
type registryTestArgs struct {
name string
inputFilePath string
expectedBuilds []packersdk.Build
expectError bool
envvars map[string]string
}
var registryCmpOpts = cmp.Options{
cmpopts.IgnoreFields(
registry.Iteration{},
"Fingerprint",
),
cmpopts.IgnoreFields(
packer.CoreBuild{},
"TemplatePath",
"Variables",
),
cmpopts.IgnoreUnexported(
hcl2template.PackerConfig{},
hcl2template.Variable{},
hcl2template.SourceBlock{},
hcl2template.DatasourceBlock{},
hcl2template.ProvisionerBlock{},
hcl2template.PostProcessorBlock{},
packer.CoreBuild{},
hcl2template.HCL2Provisioner{},
hcl2template.HCL2PostProcessor{},
packer.CoreBuildPostProcessor{},
packer.CoreBuildProvisioner{},
packer.CoreBuildPostProcessor{},
file.Builder{},
registry.Bucket{},
registry.Iteration{},
),
}
func TestRegistrySetup(t *testing.T) {
tests := []registryTestArgs{
{
"HCL2 - hcp_registry_block and multiple sources",
"test-fixtures/hcp/multiple_sources.pkr.hcl",
[]packersdk.Build{
&packer.CoreBuild{
BuildName: "bucket-slug",
Type: "file.test",
Prepared: true,
Builder: &file.Builder{},
Provisioners: []packer.CoreBuildProvisioner{},
PostProcessors: [][]packer.CoreBuildPostProcessor{},
},
&packer.CoreBuild{
BuildName: "bucket-slug",
Type: "file.other",
Prepared: true,
Builder: &file.Builder{},
Provisioners: []packer.CoreBuildProvisioner{},
PostProcessors: [][]packer.CoreBuildPostProcessor{},
},
},
false,
map[string]string{},
},
{
"HCL2 - set slug in hcp packer registry block",
"test-fixtures/hcp/slug.pkr.hcl",
[]packersdk.Build{
&packer.CoreBuild{
BuildName: "bucket-slug",
Type: "file.test",
Prepared: true,
Builder: &file.Builder{},
Provisioners: []packer.CoreBuildProvisioner{},
PostProcessors: [][]packer.CoreBuildPostProcessor{},
},
},
false,
map[string]string{},
},
{
"HCL2 - hcp - use build description",
"test-fixtures/hcp/build-description.pkr.hcl",
[]packersdk.Build{
&packer.CoreBuild{
Type: "file.test",
Prepared: true,
BuildName: "bucket-slug",
Builder: &file.Builder{},
Provisioners: []packer.CoreBuildProvisioner{},
PostProcessors: [][]packer.CoreBuildPostProcessor{},
},
},
false,
map[string]string{},
},
{
"HCL2 - override build description with hcp packer registry description",
"test-fixtures/hcp/override-build-description.pkr.hcl",
[]packersdk.Build{
&packer.CoreBuild{
Type: "file.test",
Prepared: true,
Builder: &file.Builder{},
Provisioners: []packer.CoreBuildProvisioner{},
PostProcessors: [][]packer.CoreBuildPostProcessor{},
},
},
false,
map[string]string{},
},
{
"HCL2 - deprecated labels in hcp packer registry block",
"test-fixtures/hcp/deprecated_labels.pkr.hcl",
[]packersdk.Build{
&packer.CoreBuild{
Type: "file.test",
Prepared: true,
Builder: &file.Builder{},
Provisioners: []packer.CoreBuildProvisioner{},
PostProcessors: [][]packer.CoreBuildPostProcessor{},
},
},
true,
map[string]string{},
},
{
"HCL2 - duplicate hcp_registry blocks",
"test-fixtures/hcp/duplicate.pkr.hcl",
nil,
true,
map[string]string{},
},
{
"HCL2 - two build blocks with hcp_registry",
"test-fixtures/hcp/dup_build_blocks.pkr.hcl",
nil,
true,
map[string]string{},
},
{
"HCL2/JSON - hcp enabled build",
"test-fixtures/hcp/hcp_normal.pkr.json",
[]packersdk.Build{
&packer.CoreBuild{
Type: "file.test",
Prepared: true,
BuildName: "test-file",
Builder: &file.Builder{},
Provisioners: []packer.CoreBuildProvisioner{},
PostProcessors: [][]packer.CoreBuildPostProcessor{},
},
},
false,
map[string]string{
"HCP_PACKER_REGISTRY": "1",
"HCP_PACKER_BUCKET_NAME": "bucket-slug",
},
},
{
"Legacy JSON - hcp enabled build",
"test-fixtures/hcp/hcp_build.json",
[]packersdk.Build{
&packer.CoreBuild{
Type: "file",
Prepared: false,
BuildName: "",
BuilderType: "file",
BuilderConfig: map[string]interface{}{
"content": " ",
"target": "output",
},
Builder: &file.Builder{},
Provisioners: []packer.CoreBuildProvisioner{},
PostProcessors: [][]packer.CoreBuildPostProcessor{},
},
},
false,
map[string]string{
"HCP_PACKER_REGISTRY": "1",
"HCP_PACKER_BUCKET_NAME": "bucket-slug",
},
},
}
t.Setenv("HCP_CLIENT_ID", "test")
t.Setenv("HCP_CLIENT_SECRET", "test")
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
runRegistryTest(t, tt)
})
}
}
func runRegistryTest(t *testing.T, args registryTestArgs) {
for evar, val := range args.envvars {
t.Setenv(evar, val)
}
defaultMeta := TestMetaFile(t)
c := &BuildCommand{
Meta: defaultMeta,
}
cla := &BuildArgs{
MetaArgs: MetaArgs{
Path: args.inputFilePath,
},
}
packerStarter, ret := c.GetConfig(&cla.MetaArgs)
if ret != 0 {
t.Fatalf("failed to get packer config")
}
diags := packerStarter.Initialize(packer.InitializeOptions{})
if diagCheck(diags, t) {
return
}
diags = TrySetupHCP(packerStarter)
if diagCheck(diags, t) {
if !args.expectError {
t.Errorf("SetupRegistry unexpectedly failed")
}
return
}
builds, _, diags := packerStarter.GetBuilds(packer.GetBuildsOptions{
Only: cla.Only,
Except: cla.Except,
Debug: cla.Debug,
Force: cla.Force,
OnError: cla.OnError,
})
if diagCheck(diags, t) {
if !args.expectError {
t.Errorf("SetupRegistry unexpectedly failed")
}
return
}
diff := cmp.Diff(builds, args.expectedBuilds, registryCmpOpts...)
if diff != "" {
t.Error(diff)
}
}
func severityString(sev hcl.DiagnosticSeverity) string {
switch sev {
case hcl.DiagInvalid:
return "UNKNOWN"
case hcl.DiagError:
return "ERROR"
case hcl.DiagWarning:
return "WARNING"
}
panic("unknown severity")
}
// diagCheck errors for each diagnostic received and returns if a diag was processed
func diagCheck(diags hcl.Diagnostics, t *testing.T) bool {
if len(diags) == 0 {
return false
}
for _, d := range diags {
t.Logf(
"%s: %s - %s",
severityString(d.Severity),
d.Summary,
d.Detail,
)
}
return true
}

3
hcl2template/common_test.go
View file

@ -361,7 +361,8 @@ var cmpOpts = []cmp.Option{
packerregistry.Iteration{},
),
cmpopts.IgnoreFields(PackerConfig{},
"Cwd", // Cwd will change for every os type
"Cwd", // Cwd will change for every os type
"HCPVars", // HCPVars will not be filled-in during parsing
),
cmpopts.IgnoreFields(packerregistry.Iteration{},
"Fingerprint", // Fingerprint will change everytime

1
hcl2template/parser.go
View file

@ -137,6 +137,7 @@ func (p *Parser) Parse(filename string, varFiles []string, argVars map[string]st
Basedir: basedir,
Cwd: wd,
CorePackerVersionString: p.CorePackerVersionString,
HCPVars: map[string]cty.Value{},
parser: p,
files: files,
}

40
hcl2template/registry.go
View file

@ -1,40 +0,0 @@
package hcl2template
import (
"fmt"
"github.com/hashicorp/hcl/v2"
packerregistry "github.com/hashicorp/packer/internal/registry"
)
// ConfiguredArtifactMetadataPublisher returns a configured image bucket that can be used for publishing
// build image artifacts to a configured Packer Registry destination.
func (cfg *PackerConfig) ConfiguredArtifactMetadataPublisher() (*packerregistry.Bucket, hcl.Diagnostics) {
// If this was a PAR (HCP Packer registry) build either the env. variables are set, or if there is a hcp_packer_registry block
// defined we would have a non-nil bucket. So if nil assume we are not in a some sort of PAR mode.
if cfg.Bucket == nil {
return nil, hcl.Diagnostics{
&hcl.Diagnostic{
Summary: "Publishing build artifacts to HCP Packer Registry not enabled",
Detail: "No Packer Registry configuration detected; skipping all publishing steps " +
"See publishing to a Packer registry for Packer configuration details",
Severity: hcl.DiagWarning,
},
}
}
err := cfg.Bucket.Validate()
if err != nil {
return nil, hcl.Diagnostics{
&hcl.Diagnostic{
Summary: "Invalid HCP Packer configuration",
Detail: fmt.Sprintf("Packer could not validate the provided "+
"HCP Packer registry configuration. Check the error message for details "+
"or contact HCP Packer support for further assistance.\nError: %s", err),
Severity: hcl.DiagError,
},
}
}
return cfg.Bucket, nil
}

17
hcl2template/types.packer_config.go
View file

@ -7,12 +7,11 @@ import (
"strings"
"github.com/gobwas/glob"
"github.com/hashicorp/hcl/v2"
hcl "github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hcldec"
"github.com/hashicorp/hcl/v2/hclsyntax"
packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
pkrfunction "github.com/hashicorp/packer/hcl2template/function"
packerregistry "github.com/hashicorp/packer/internal/registry"
"github.com/hashicorp/packer/packer"
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/function"
@ -54,8 +53,8 @@ type PackerConfig struct {
// Builds is the list of Build blocks defined in the config files.
Builds Builds
// Represents registry bucket defined in the config files.
Bucket *packerregistry.Bucket
// HCPVars is the list of HCP-set variables for use later in a template
HCPVars map[string]cty.Value
parser *Parser
files []*hcl.File
@ -119,11 +118,13 @@ func (cfg *PackerConfig) EvalContext(ctx BlockContext, variables map[string]cty.
},
}
// Store the iteration_id, if it exists. Otherwise, it'll be "unknown"
if cfg.Bucket != nil {
iterID, ok := cfg.HCPVars["iterationID"]
if ok {
log.Printf("iterationID set: %q", iterID)
ectx.Variables[packerAccessor] = cty.ObjectVal(map[string]cty.Value{
"version": cty.StringVal(cfg.CorePackerVersionString),
"iterationID": cty.StringVal(cfg.Bucket.Iteration.ID),
"iterationID": iterID,
})
}
@ -576,7 +577,7 @@ func (cfg *PackerConfig) GetBuilds(opts packer.GetBuildsOptions) ([]packersdk.Bu
cfg.onError = opts.OnError
if len(cfg.Builds) == 0 {
return res, append(diags, &hcl.Diagnostic{
return res, hcpTranslationMap, append(diags, &hcl.Diagnostic{
Summary: "Missing build block",
Detail: "A build block with one or more sources is required for executing a build.",
Severity: hcl.DiagError,

84
internal/hcp/bucket_init.go Normal file
View file

@ -0,0 +1,84 @@
package hcp
import (
"fmt"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/packer/internal/registry"
"github.com/hashicorp/packer/internal/registry/env"
)
// HCPConfigMode types specify the mode in which HCP configuration
// is defined for a given Packer build execution.
type HCPConfigMode int
const (
// HCPConfigUnset mode is set when no HCP configuration has been found for the Packer execution.
HCPConfigUnset HCPConfigMode = iota
// HCPConfigEnabled mode is set when the HCP configuration is codified in the template.
HCPConfigEnabled
// HCPEnvEnabled mode is set when the HCP configuration is read from environment variables.
HCPEnvEnabled
)
type bucketConfigurationOpts func(*registry.Bucket) hcl.Diagnostics
// createConfiguredBucket returns a bucket that can be used for connecting to the HCP Packer registry.
// Configuration for the bucket is obtained from the base iteration setting and any addition configuration
// options passed in as opts. All errors during configuration are collected and returned as Diagnostics.
func createConfiguredBucket(templateDir string, opts ...bucketConfigurationOpts) (*registry.Bucket, hcl.Diagnostics) {
var diags hcl.Diagnostics
if !env.HasHCPCredentials() {
diags = append(diags, &hcl.Diagnostic{
Summary: "HCP authentication information required",
Detail: fmt.Sprintf("The client authentication requires both %s and %s environment "+
"variables to be set for authenticating with HCP.",
env.HCPClientID,
env.HCPClientSecret),
Severity: hcl.DiagError,
})
}
bucket := registry.NewBucketWithIteration()
for _, opt := range opts {
if optDiags := opt(bucket); optDiags.HasErrors() {
diags = append(diags, optDiags...)
}
}
if bucket.Slug == "" {
diags = append(diags, &hcl.Diagnostic{
Summary: "Image bucket name required",
Detail: "You must provide an image bucket name for HCP Packer builds. " +
"You can set the HCP_PACKER_BUCKET_NAME environment variable. " +
"For HCL2 templates, the registry either uses the name of your " +
"template's build block, or you can set the bucket_name argument " +
"in an hcp_packer_registry block.",
Severity: hcl.DiagError,
})
}
err := bucket.Iteration.Initialize(registry.IterationOptions{
TemplateBaseDir: templateDir,
})
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Summary: "Iteration initialization failed",
Detail: fmt.Sprintf("Initialization of the iteration failed with "+
"the following error message: %s", err),
Severity: hcl.DiagError,
})
}
return bucket, diags
}
func withPackerEnvConfiguration(bucket *registry.Bucket) hcl.Diagnostics {
// Add default values for Packer settings configured via EnvVars.
// TODO look to break this up to be more explicit on what is loaded here.
bucket.LoadDefaultSettingsFromEnv()
return nil
}

13
internal/hcp/errors.go Normal file
View file

@ -0,0 +1,13 @@
package hcp
import "fmt"
// BuildDone is the error retuned by an HCP handler when a build cannot be started since it's already marked as DONE.
type BuildDone struct {
Message string
}
// Error returns the message for the BuildDone type
func (b BuildDone) Error() string {
return fmt.Sprintf("BuildDone: %s", b.Message)
}

191
command/registry.go → internal/hcp/hcl.go
View file

@ -1,33 +1,25 @@
package command
package hcp
import (
"context"
"fmt"
"path/filepath"
"github.com/hashicorp/hcl/v2"
sdkpacker "github.com/hashicorp/packer-plugin-sdk/packer"
imgds "github.com/hashicorp/packer/datasource/hcp-packer-image"
iterds "github.com/hashicorp/packer/datasource/hcp-packer-iteration"
"github.com/hashicorp/packer/hcl2template"
"github.com/hashicorp/packer/internal/registry"
"github.com/hashicorp/packer/internal/registry/env"
"github.com/hashicorp/packer/packer"
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/gocty"
)
// HCPConfigMode types specify the mode in which HCP configuration
// is defined for a given Packer build execution.
type HCPConfigMode int
const (
// HCPConfigUnset mode is set when no HCP configuration has been found for the Packer execution.
HCPConfigUnset HCPConfigMode = iota
// HCPConfigEnabled mode is set when the HCP configuration is codified in the template.
HCPConfigEnabled
// HCPEnvEnabled mode is set when the HCP configuration is read from environment variables.
HCPEnvEnabled
)
// hclOrchestrator is a HCP handler made for handling HCL configurations
type hclOrchestrator struct {
configuration *hcl2template.PackerConfig
bucket *registry.Bucket
}
const (
// Known HCP Packer Image Datasource, whose id is the SourceImageId for some build.
@ -36,53 +28,65 @@ const (
buildLabel string = "build"
)
// TrySetupHCP attempts to configure the HCP-related structures if
// Packer has been configured to publish to a HCP Packer registry.
func TrySetupHCP(cfg packer.Handler) hcl.Diagnostics {
switch cfg := cfg.(type) {
case *hcl2template.PackerConfig:
return setupRegistryForPackerConfig(cfg)
case *CoreWrapper:
return setupRegistryForPackerCore(cfg)
// PopulateIteration creates the metadata on HCP for a build
func (h *hclOrchestrator) PopulateIteration(ctx context.Context) error {
err := h.bucket.Initialize(ctx)
if err != nil {
return err
}
return hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "unknown Handler type",
Detail: "TrySetupHCP called with an unknown Handler. " +
"This is a Packer bug and should be brought to the attention " +
"of the Packer team, please consider opening an issue for this.",
},
err = h.bucket.PopulateIteration(ctx)
if err != nil {
return err
}
iterationID := h.bucket.Iteration.ID
h.configuration.HCPVars["iterationID"] = cty.StringVal(iterationID)
return nil
}
func setupRegistryForPackerConfig(pc *hcl2template.PackerConfig) hcl.Diagnostics {
// BuildStart is invoked when one build for the configuration is starting to be processed
func (h *hclOrchestrator) BuildStart(ctx context.Context, buildName string) error {
return h.bucket.BuildStart(ctx, buildName)
}
// BuildDone is invoked when one build for the configuration has finished
func (h *hclOrchestrator) BuildDone(
ctx context.Context,
buildName string,
artifacts []sdkpacker.Artifact,
buildErr error,
) ([]sdkpacker.Artifact, error) {
return h.bucket.BuildDone(ctx, buildName, artifacts, buildErr)
}
func newHCLOrchestrator(config *hcl2template.PackerConfig) (Orchestrator, hcl.Diagnostics) {
// HCP_PACKER_REGISTRY is explicitly turned off
if env.IsHCPDisabled() {
return nil
return newNoopHandler(), nil
}
mode := HCPConfigUnset
for _, build := range pc.Builds {
for _, build := range config.Builds {
if build.HCPPackerRegistry != nil {
mode = HCPConfigEnabled
}
}
// HCP_PACKER_BUCKET_NAME is set or HCP_PACKER_REGISTRY not toggled off
// HCP_PACKER_BUCKET_NAME is set or HCP_PACKER_REGISTRY not toggled off
if mode == HCPConfigUnset && (env.HasPackerRegistryBucket() || env.IsHCPExplicitelyEnabled()) {
mode = HCPEnvEnabled
}
if mode == HCPConfigUnset {
return nil
return newNoopHandler(), nil
}
var diags hcl.Diagnostics
if len(pc.Builds) > 1 {
if len(config.Builds) > 1 {
diags = append(diags, &hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Multiple " + buildLabel + " blocks",
@ -92,7 +96,7 @@ func setupRegistryForPackerConfig(pc *hcl2template.PackerConfig) hcl.Diagnostics
"clear any HCP_PACKER_* environment variables."),
})
return diags
return nil, diags
}
withHCLBucketConfiguration := func(bb *hcl2template.BuildBlock) bucketConfigurationOpts {
@ -113,117 +117,34 @@ func setupRegistryForPackerConfig(pc *hcl2template.PackerConfig) hcl.Diagnostics
}
// Capture Datasource configuration data
vals, dsDiags := pc.Datasources.Values()
vals, dsDiags := config.Datasources.Values()
if dsDiags != nil {
diags = append(diags, dsDiags...)
}
build := pc.Builds[0]
pc.Bucket, diags = createConfiguredBucket(
pc.Basedir,
build := config.Builds[0]
bucket, bucketDiags := createConfiguredBucket(
config.Basedir,
withPackerEnvConfiguration,
withHCLBucketConfiguration(build),
withDatasourceConfiguration(vals),
)
if bucketDiags != nil {
diags = append(diags, bucketDiags...)
}
if diags.HasErrors() {
return diags
return nil, diags
}
for _, source := range build.Sources {
pc.Bucket.RegisterBuildForComponent(source.String())
bucket.RegisterBuildForComponent(source.String())
}
return diags
}
func setupRegistryForPackerCore(cfg *CoreWrapper) hcl.Diagnostics {
if env.IsHCPDisabled() {
return nil
}
if !env.HasPackerRegistryBucket() && !env.IsHCPExplicitelyEnabled() {
return nil
}
bucket, diags := createConfiguredBucket(
filepath.Dir(cfg.Core.Template.Path),
withPackerEnvConfiguration,
)
if diags.HasErrors() {
return diags
}
cfg.Core.Bucket = bucket
for _, b := range cfg.Core.Template.Builders {
// Get all builds slated within config ignoring any only or exclude flags.
cfg.Core.Bucket.RegisterBuildForComponent(packer.HCPName(b))
}
return diags
}
type bucketConfigurationOpts func(*registry.Bucket) hcl.Diagnostics
// createConfiguredBucket returns a bucket that can be used for connecting to the HCP Packer registry.
// Configuration for the bucket is obtained from the base iteration setting and any addition configuration
// options passed in as opts. All errors during configuration are collected and returned as Diagnostics.
func createConfiguredBucket(templateDir string, opts ...bucketConfigurationOpts) (*registry.Bucket, hcl.Diagnostics) {
var diags hcl.Diagnostics
if !env.HasHCPCredentials() {
diags = append(diags, &hcl.Diagnostic{
Summary: "HCP authentication information required",
Detail: fmt.Sprintf("The client authentication requires both %s and %s environment "+
"variables to be set for authenticating with HCP.",
env.HCPClientID,
env.HCPClientSecret),
Severity: hcl.DiagError,
})
}
bucket := registry.NewBucketWithIteration()
for _, opt := range opts {
if optDiags := opt(bucket); optDiags.HasErrors() {
diags = append(diags, optDiags...)
}
}
if bucket.Slug == "" {
diags = append(diags, &hcl.Diagnostic{
Summary: "Image bucket name required",
Detail: "You must provide an image bucket name for HCP Packer builds. " +
"You can set the HCP_PACKER_BUCKET_NAME environment variable. " +
"For HCL2 templates, the registry either uses the name of your " +
"template's build block, or you can set the bucket_name argument " +
"in an hcp_packer_registry block.",
Severity: hcl.DiagError,
})
}
err := bucket.Iteration.Initialize(registry.IterationOptions{
TemplateBaseDir: templateDir,
})
if err != nil {
diags = append(diags, &hcl.Diagnostic{
Summary: "Iteration initialization failed",
Detail: fmt.Sprintf("Initialization of the iteration failed with "+
"the following error message: %s", err),
Severity: hcl.DiagError,
})
}
return bucket, diags
}
func withPackerEnvConfiguration(bucket *registry.Bucket) hcl.Diagnostics {
// Add default values for Packer settings configured via EnvVars.
// TODO look to break this up to be more explicit on what is loaded here.
bucket.LoadDefaultSettingsFromEnv()
return nil
return &hclOrchestrator{
configuration: config,
bucket: bucket,
}, nil
}
func imageValueToDSOutput(imageVal map[string]cty.Value) imgds.DatasourceOutput {

83
internal/hcp/json.go Normal file
View file

@ -0,0 +1,83 @@
package hcp
import (
"context"
"path/filepath"
"github.com/hashicorp/hcl/v2"
sdkpacker "github.com/hashicorp/packer-plugin-sdk/packer"
"github.com/hashicorp/packer/internal/registry"
"github.com/hashicorp/packer/internal/registry/env"
"github.com/hashicorp/packer/packer"
)
// jsonOrchestrator is a HCP handler made to process legacy JSON templates
type jsonOrchestrator struct {
configuration *packer.Core
bucket *registry.Bucket
}
func newJSONOrchestrator(config *packer.Core) (Orchestrator, hcl.Diagnostics) {
if env.IsHCPDisabled() ||
(!env.HasPackerRegistryBucket() && !env.IsHCPExplicitelyEnabled()) {
return newNoopHandler(), nil
}
bucket, diags := createConfiguredBucket(
filepath.Dir(config.Template.Path),
withPackerEnvConfiguration,
)
if diags.HasErrors() {
return nil, diags
}
for _, b := range config.Template.Builders {
// Get all builds slated within config ignoring any only or exclude flags.
bucket.RegisterBuildForComponent(packer.HCPName(b))
}
return &jsonOrchestrator{
configuration: config,
bucket: bucket,
}, nil
}
// PopulateIteration creates the metadata on HCP for a build
func (h *jsonOrchestrator) PopulateIteration(ctx context.Context) error {
for _, b := range h.configuration.Template.Builders {
// Get all builds slated within config ignoring any only or exclude flags.
h.bucket.RegisterBuildForComponent(b.Name)
}
err := h.bucket.Validate()
if err != nil {
return err
}
err = h.bucket.Initialize(ctx)
if err != nil {
return err
}
err = h.bucket.PopulateIteration(ctx)
if err != nil {
return err
}
return nil
}
// BuildStart is invoked when one build for the configuration is starting to be processed
func (h *jsonOrchestrator) BuildStart(ctx context.Context, buildName string) error {
return h.bucket.BuildStart(ctx, buildName)
}
// BuildDone is invoked when one build for the configuration has finished
func (h *jsonOrchestrator) BuildDone(
ctx context.Context,
buildName string,
artifacts []sdkpacker.Artifact,
buildErr error,
) ([]sdkpacker.Artifact, error) {
return h.bucket.BuildDone(ctx, buildName, artifacts, buildErr)
}

31
internal/hcp/noop.go Normal file
View file

@ -0,0 +1,31 @@
package hcp
import (
"context"
sdkpacker "github.com/hashicorp/packer-plugin-sdk/packer"
)
// noopOrchestrator is a special handler that does nothing
type noopOrchestrator struct{}
func newNoopHandler() Orchestrator {
return noopOrchestrator{}
}
func (h noopOrchestrator) PopulateIteration(context.Context) error {
return nil
}
func (h noopOrchestrator) BuildStart(context.Context, string) error {
return nil
}
func (h noopOrchestrator) BuildDone(
ctx context.Context,
buildName string,
artifacts []sdkpacker.Artifact,
buildErr error,
) ([]sdkpacker.Artifact, error) {
return artifacts, nil
}

34
internal/hcp/orchestrator.go Normal file
View file

@ -0,0 +1,34 @@
package hcp
import (
"context"
"github.com/hashicorp/hcl/v2"
sdkpacker "github.com/hashicorp/packer-plugin-sdk/packer"
"github.com/hashicorp/packer/hcl2template"
"github.com/hashicorp/packer/packer"
)
// Orchestrator is an entity capable to orchestrate a Packer build and upload metadata to HCP
type Orchestrator interface {
PopulateIteration(context.Context) error
BuildStart(context.Context, string) error
BuildDone(ctx context.Context, buildName string, artifacts []sdkpacker.Artifact, buildErr error) ([]sdkpacker.Artifact, error)
}
// GetOrchestrator instanciates the appropriate handler for the configuration type given as parameter.
//
// If no HCP-related data is present, it will be a NoopHandler.
func GetOrchestrator(cfg packer.Handler) (Orchestrator, hcl.Diagnostics) {
var handler Orchestrator
var err hcl.Diagnostics
switch cfg := cfg.(type) {
case *hcl2template.PackerConfig:
handler, err = newHCLOrchestrator(cfg)
case *packer.Core:
handler, err = newJSONOrchestrator(cfg)
}
return handler, err
}

9
internal/registry/types.bucket.go
View file

@ -524,6 +524,7 @@ func (b *Bucket) BuildDone(
doneCh, ok := b.RunningBuilds[buildName]
if !ok {
log.Print("[ERROR] done build does not have an entry in the heartbeat table, state will be inconsistent.")
} else {
log.Printf("[TRACE] signal stopping heartbeats")
// Stop heartbeating
@ -547,13 +548,17 @@ func (b *Bucket) BuildDone(
ErrorUnused: false,
})
if err != nil {
return artifacts, fmt.Errorf("failed to create decoder for HCP Packer registry image: %w", err)
return artifacts, fmt.Errorf(
"failed to create decoder for HCP Packer registry image: %w",
err)
}
state := art.State(registryimage.ArtifactStateURI)
err = decoder.Decode(state)
if err != nil {
return artifacts, fmt.Errorf("failed to obtain HCP Packer registry image from post-processor artifact: %w", err)
return artifacts, fmt.Errorf(
"failed to obtain HCP Packer registry image from post-processor artifact: %w",
err)
}
log.Printf("[TRACE] updating artifacts for build %q", buildName)
err = b.UpdateImageForBuild(buildName, images...)

38
packer/core.go
View file

@ -14,11 +14,11 @@ import (
"github.com/google/go-cmp/cmp"
multierror "github.com/hashicorp/go-multierror"
version "github.com/hashicorp/go-version"
"github.com/hashicorp/hcl/v2"
hcl "github.com/hashicorp/hcl/v2"
packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
"github.com/hashicorp/packer-plugin-sdk/template"
"github.com/hashicorp/packer-plugin-sdk/template/interpolate"
packerregistry "github.com/hashicorp/packer/internal/registry"
plugingetter "github.com/hashicorp/packer/packer/plugin-getter"
packerversion "github.com/hashicorp/packer/version"
)
@ -27,8 +27,6 @@ import (
type Core struct {
Template *template.Template
Bucket *packerregistry.Bucket
components ComponentFinder
variables map[string]string
builds map[string]*template.Builder
@ -131,7 +129,20 @@ func NewCore(c *CoreConfig) *Core {
return core
}
func (core *Core) Initialize() error {
func (c *Core) Initialize(_ InitializeOptions) hcl.Diagnostics {
err := c.initialize()
if err != nil {
return hcl.Diagnostics{
&hcl.Diagnostic{
Detail: err.Error(),
Severity: hcl.DiagError,
},
}
}
return nil
}
func (core *Core) initialize() error {
if err := core.validate(); err != nil {
return err
}
@ -155,9 +166,20 @@ func (core *Core) Initialize() error {
core.builds[v] = b
}
return nil
}
func (c *Core) PluginRequirements() (plugingetter.Requirements, hcl.Diagnostics) {
return nil, hcl.Diagnostics{
&hcl.Diagnostic{
Summary: "Packer plugins currently only works with HCL2 configuration templates",
Detail: "Please manually install plugins with the plugins command or use a HCL2 configuration that will do that for you.",
Severity: hcl.DiagError,
},
}
}
// BuildNames returns the builds that are available in this configured core.
func (c *Core) BuildNames(only, except []string) []string {
@ -889,9 +911,3 @@ func (c *Core) init() error {
return nil
}
/// GetRegistryBucket returns a configured bucket that can be used for
// publishing build image artifacts to some HCP Packer Registry.
func (c *Core) GetRegistryBucket() *packerregistry.Bucket {
return c.Bucket
}

36
packer/core_test.go
View file

@ -43,9 +43,9 @@ func TestCoreBuildNames(t *testing.T) {
Template: tpl,
Variables: tc.Vars,
})
err = core.Initialize()
if err != nil {
t.Fatalf("err: %s\n\n%s", tc.File, err)
diags := core.Initialize(InitializeOptions{})
if diags.HasErrors() {
t.Fatalf("err: %s\n\n%s", tc.File, diags)
}
names := core.BuildNames(nil, nil)
@ -493,9 +493,9 @@ func TestCoreValidate(t *testing.T) {
Variables: tc.Vars,
Version: "1.0.0",
})
err = core.Initialize()
diags := core.Initialize(InitializeOptions{})
if (err != nil) != tc.Err {
if diags.HasErrors() != tc.Err {
t.Fatalf("err: %s\n\n%s", tc.File, err)
}
}
@ -541,13 +541,13 @@ func TestCore_InterpolateUserVars(t *testing.T) {
Template: tpl,
Version: "1.0.0",
})
err = ccf.Initialize()
diags := ccf.Initialize(InitializeOptions{})
if (err != nil) != tc.Err {
if diags.HasErrors() != tc.Err {
if tc.Err == false {
t.Fatalf("Error interpolating %s: Expected no error, but got: %s", tc.File, err)
t.Fatalf("Error interpolating %s: Expected no error, but got: %s", tc.File, diags)
} else {
t.Fatalf("Error interpolating %s: Expected an error, but got: %s", tc.File, err)
t.Fatalf("Error interpolating %s: Expected an error, but got: %s", tc.File, diags)
}
}
@ -614,10 +614,10 @@ func TestCore_InterpolateUserVars_VarFile(t *testing.T) {
Version: "1.0.0",
Variables: tc.Variables,
})
err = ccf.Initialize()
diags := ccf.Initialize(InitializeOptions{})
if (err != nil) != tc.Err {
t.Fatalf("err: %s\n\n%s", tc.File, err)
if diags.HasErrors() != tc.Err {
t.Fatalf("err: %s\n\n%s", tc.File, diags)
}
if !tc.Err {
for k, v := range ccf.variables {
@ -674,10 +674,10 @@ func TestSensitiveVars(t *testing.T) {
Variables: tc.Vars,
Version: "1.0.0",
})
err = ccf.Initialize()
diags := ccf.Initialize(InitializeOptions{})
if (err != nil) != tc.Err {
t.Fatalf("err: %s\n\n%s", tc.File, err)
if diags.HasErrors() != tc.Err {
t.Fatalf("err: %s\n\n%s", tc.File, diags)
}
// Check that filter correctly manipulates strings:
filtered := packersdk.LogSecretFilter.FilterString("the foo jumped over the bar_extra_sensitive_probably_a_password")
@ -756,7 +756,7 @@ func TestEnvAndFileVars(t *testing.T) {
"final_var": "{{user `env_1`}}/{{user `env_2`}}/{{user `env_4`}}{{user `env_3`}}-{{user `var_1`}}/vmware/{{user `var_2`}}.vmx",
},
})
err = ccf.Initialize()
diags := ccf.Initialize(InitializeOptions{})
expected := map[string]string{
"var_1": "partyparrot",
@ -767,8 +767,8 @@ func TestEnvAndFileVars(t *testing.T) {
"env_3": "/path/to/nowhere",
"env_4": "bananas",
}
if err != nil {
t.Fatalf("err: %s\n\n%s", "complex-recursed-env-user-var-file.json", err)
if diags.HasErrors() {
t.Fatalf("err: %s\n\n%s", "complex-recursed-env-user-var-file.json", diags)
}
for k, v := range ccf.variables {
if expected[k] != v {

11
packer/run_interfaces.go
View file

@ -1,9 +1,8 @@
package packer
import (
"github.com/hashicorp/hcl/v2"
hcl "github.com/hashicorp/hcl/v2"
packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
packerregistry "github.com/hashicorp/packer/internal/registry"
plugingetter "github.com/hashicorp/packer/packer/plugin-getter"
)
@ -51,14 +50,6 @@ type Handler interface {
BuildGetter
ConfigFixer
ConfigInspector
HCPHandler
}
// The HCPHandler handles Packer things needed for communicating with a HCP Packer Registry.
type HCPHandler interface {
// ConfiguredArtifactMetadataPublisher returns a configured Bucket that can be used to publish
// build artifacts to the said image bucket.
ConfiguredArtifactMetadataPublisher() (*packerregistry.Bucket, hcl.Diagnostics)
}
//go:generate enumer -type FixConfigMode

2
packer/testing.go
View file

@ -25,7 +25,7 @@ func TestCoreConfig(t *testing.T) *CoreConfig {
func TestCore(t *testing.T, c *CoreConfig) *Core {
core := NewCore(c)
err := core.Initialize()
err := core.Initialize(InitializeOptions{})
if err != nil {
t.Fatalf("err: %s", err)
}