upstream/packer
1
0
Fork 0
mirror of https://github.com/hashicorp/packer.git synced 2026-05-28 04:35:38 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

internal: reorganise registry/HCP code

Browse source 
This commit reorganises the code for both the registry/API and the
Orchestrator/Registry.

The main difference with the previous version is how stuff is exposed.
Now we only expose a Registry interface to the outside (previously named
Orchestrator), which has several implementations: null is the default,
and is returned if HCP is not enabled.

The other implementations being HCL/JSON, both private to the hcp
sub-package.

The api (previously `registry') is the set of functionality that
abstracts and calls the HCP API.
It was meant to be merged with the `hcp' package, but because of a
dependency loop with the datasources, both are separated for now.
This commit is contained in:
Wilken Rivera 2022-11-07 14:16:04 -05:00 committed by Lucas Bajolet
parent 1cee460d0d
commit 606e6c48f1
31 changed files with 301 additions and 289 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
command/build.go
View file

@ -14,7 +14,7 @@ import (
"github.com/hashicorp/hcl/v2"
packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
"github.com/hashicorp/packer/internal/hcp"
"github.com/hashicorp/packer/internal/hcp/registry"
"github.com/hashicorp/packer/packer"
"golang.org/x/sync/semaphore"
@ -90,13 +90,13 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int
return ret
}
hcpHandler, diags := hcp.GetOrchestrator(packerStarter)
hcpRegistry, diags := registry.New(packerStarter)
ret = writeDiags(c.Ui, nil, diags)
if ret != 0 {
return ret
}
err := hcpHandler.PopulateIteration(buildCtx)
err := hcpRegistry.PopulateIteration(buildCtx)
if err != nil {
return writeDiags(c.Ui, nil, hcl.Diagnostics{
&hcl.Diagnostic{
@ -219,14 +219,24 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int
defer limitParallel.Release(1)
err := hcpHandler.BuildStart(buildCtx, hcpMap[name])
err := hcpRegistry.StartBuild(buildCtx, hcpMap[name])
// Seems odd to require this error check here. Now that it is an error we can just exit with diag
if err != nil {
if errors.As(err, &hcp.BuildDone{}) {
ui.Say(fmt.Sprintf(
"skipping HCP-enabled build %q: already done.",
name))
// If the build is already done, we skip without a warning
if errors.As(err, &registry.ErrBuildAlreadyDone{}) {
ui.Say(fmt.Sprintf("skipping already done build %q", name))
return
}
writeDiags(c.Ui, nil, hcl.Diagnostics{
&hcl.Diagnostic{
Summary: fmt.Sprintf(
"hcp: failed to start build %q",
name),
Severity: hcl.DiagError,
Detail: err.Error(),
},
})
return
}
log.Printf("Starting build run: %s", name)
@ -237,7 +247,7 @@ func (c *BuildCommand) RunContext(buildCtx context.Context, cla *BuildArgs) int
buildDuration := buildEnd.Sub(buildStart)
fmtBuildDuration := durafmt.Parse(buildDuration).LimitFirstN(2)
runArtifacts, hcperr := hcpHandler.BuildDone(
runArtifacts, hcperr := hcpRegistry.CompleteBuild(
buildCtx,
hcpMap[name],
runArtifacts,

6
datasource/hcp-packer-image/data.go
View file

@ -17,7 +17,7 @@ import (
"github.com/hashicorp/packer-plugin-sdk/hcl2helper"
packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
"github.com/hashicorp/packer-plugin-sdk/template/config"
packerregistry "github.com/hashicorp/packer/internal/registry"
hcpapi "github.com/hashicorp/packer/internal/hcp/api"
)
type Datasource struct {
@ -139,7 +139,7 @@ func (d *Datasource) OutputSpec() hcldec.ObjectSpec {
func (d *Datasource) Execute() (cty.Value, error) {
ctx := context.TODO()
cli, err := packerregistry.NewClient()
cli, err := hcpapi.NewClient()
if err != nil {
return cty.NullVal(cty.EmptyObject), err
}
@ -150,7 +150,7 @@ func (d *Datasource) Execute() (cty.Value, error) {
log.Printf("[INFO] Reading info from HCP Packer registry (%s) [project_id=%s, organization_id=%s, iteration_id=%s]",
d.config.Bucket, cli.ProjectID, cli.OrganizationID, d.config.IterationID)
iter, err := cli.GetIteration(ctx, d.config.Bucket, packerregistry.GetIteration_byID(d.config.IterationID))
iter, err := cli.GetIteration(ctx, d.config.Bucket, hcpapi.GetIteration_byID(d.config.IterationID))
if err != nil {
return cty.NullVal(cty.EmptyObject), fmt.Errorf(
"error retrieving image iteration from HCP Packer registry: %s",

4
datasource/hcp-packer-iteration/data.go
View file

@ -15,7 +15,7 @@ import (
"github.com/hashicorp/packer-plugin-sdk/hcl2helper"
packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
"github.com/hashicorp/packer-plugin-sdk/template/config"
packerregistry "github.com/hashicorp/packer/internal/registry"
hcpapi "github.com/hashicorp/packer/internal/hcp/api"
)
type Datasource struct {
@ -99,7 +99,7 @@ func (d *Datasource) OutputSpec() hcldec.ObjectSpec {
func (d *Datasource) Execute() (cty.Value, error) {
ctx := context.TODO()
cli, err := packerregistry.NewClient()
cli, err := hcpapi.NewClient()
if err != nil {
return cty.NullVal(cty.EmptyObject), err
}

10
datasource/hcp-packer-iteration/data_acc_test.go
View file

@ -8,11 +8,7 @@ import (
"testing"
"github.com/hashicorp/packer-plugin-sdk/acctest"
)
const (
HCPClientID string = "HCP_CLIENT_ID"
HCPClientSecret string = "HCP_CLIENT_SECRET"
"github.com/hashicorp/packer/internal/hcp/env"
)
//go:embed test-fixtures/template.pkr.hcl
@ -31,8 +27,8 @@ var testDatasourceBasic string
// as defined above.
func TestAccDatasource_HCPPackerIteration(t *testing.T) {
if os.Getenv(HCPClientID) == "" && os.Getenv(HCPClientSecret) == "" {
t.Skip(fmt.Sprintf("Acceptance tests skipped unless envs %q and %q are set", HCPClientID, HCPClientSecret))
if os.Getenv(env.HCPClientID) == "" && os.Getenv(env.HCPClientSecret) == "" {
t.Skipf(fmt.Sprintf("Acceptance tests skipped unless envs %q and %q are set", env.HCPClientID, env.HCPClientSecret))
return
}

4
datasource/packer-image-iteration/data.go
View file

@ -16,7 +16,7 @@ import (
"github.com/hashicorp/packer-plugin-sdk/hcl2helper"
packersdk "github.com/hashicorp/packer-plugin-sdk/packer"
"github.com/hashicorp/packer-plugin-sdk/template/config"
packerregistry "github.com/hashicorp/packer/internal/registry"
hcpapi "github.com/hashicorp/packer/internal/hcp/api"
)
// Type for Packer datasource has been renamed temporarily to prevent it from being
@ -136,7 +136,7 @@ func (d *DeactivatedDatasource) OutputSpec() hcldec.ObjectSpec {
func (d *DeactivatedDatasource) Execute() (cty.Value, error) {
ctx := context.TODO()
cli, err := packerregistry.NewClient()
cli, err := hcpapi.NewClient()
if err != nil {
return cty.NullVal(cty.EmptyObject), err
}

9
hcl2template/common_test.go
View file

@ -15,7 +15,6 @@ import (
dnull "github.com/hashicorp/packer/datasource/null"
. "github.com/hashicorp/packer/hcl2template/internal"
hcl2template "github.com/hashicorp/packer/hcl2template/internal"
packerregistry "github.com/hashicorp/packer/internal/registry"
"github.com/hashicorp/packer/packer"
"github.com/zclconf/go-cty/cty"
)
@ -357,19 +356,11 @@ var cmpOpts = []cmp.Option{
packer.CoreBuildProvisioner{},
packer.CoreBuildPostProcessor{},
null.Builder{},
packerregistry.Bucket{},
packerregistry.Iteration{},
),
cmpopts.IgnoreFields(PackerConfig{},
"Cwd", // Cwd will change for every os type
"HCPVars", // HCPVars will not be filled-in during parsing
),
cmpopts.IgnoreFields(packerregistry.Iteration{},
"Fingerprint", // Fingerprint will change everytime
),
cmpopts.IgnoreFields(packerregistry.Bucket{},
"SourceImagesToParentIterations", // Requires execution of datasource at this time
),
cmpopts.IgnoreFields(VariableAssignment{},
"Expr", // its an interface
),

15
hcl2template/types.build.hcp_packer_registry.go
View file

@ -5,7 +5,6 @@ import (
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/gohcl"
packerregistry "github.com/hashicorp/packer/internal/registry"
)
type HCPPackerRegistryBlock struct {
@ -21,20 +20,6 @@ type HCPPackerRegistryBlock struct {
HCL2Ref
}
func (b *HCPPackerRegistryBlock) WriteToBucketConfig(bucket *packerregistry.Bucket) {
if b == nil {
return
}
bucket.Description = b.Description
bucket.BucketLabels = b.BucketLabels
bucket.BuildLabels = b.BuildLabels
// If there's already a Slug this was set from env variable.
// In Packer, env variable overrides config values so we keep it that way for consistency.
if bucket.Slug == "" && b.Slug != "" {
bucket.Slug = b.Slug
}
}
func (p *Parser) decodeHCPRegistry(block *hcl.Block, cfg *PackerConfig) (*HCPPackerRegistryBlock, hcl.Diagnostics) {
par := &HCPPackerRegistryBlock{}
body := block.Body

5
internal/registry/client.go → internal/hcp/api/client.go
View file

@ -1,4 +1,5 @@
package registry
// Package api provides access to the HCP Packer Registry API.
package api
import (
"fmt"
@ -8,7 +9,7 @@ import (
projectSvc "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/client/project_service"
rmmodels "github.com/hashicorp/hcp-sdk-go/clients/cloud-resource-manager/preview/2019-12-10/models"
"github.com/hashicorp/hcp-sdk-go/httpclient"
"github.com/hashicorp/packer/internal/registry/env"
"github.com/hashicorp/packer/internal/hcp/env"
"github.com/hashicorp/packer/version"
)

6
internal/registry/errors.go → internal/hcp/api/errors.go
View file

@ -1,4 +1,4 @@
package registry
package api
import (
"fmt"
@ -23,11 +23,11 @@ func (c *ClientError) Error() string {
return fmt.Sprintf("status %d: err %v", c.StatusCode, c.Err)
}
// checkErrorCode checks the error string for err for some code and returns true
// CheckErrorCode checks the error string for err for some code and returns true
// if the code is found. Ideally this function should use status.FromError
// https://pkg.go.dev/google.golang.org/grpc/status#pkg-functions but that
// doesn't appear to work for all of the Cloud Packer Service response errors.
func checkErrorCode(err error, code codes.Code) bool {
func CheckErrorCode(err error, code codes.Code) bool {
if err == nil {
return false
}

2
internal/registry/mock_service.go → internal/hcp/api/mock_service.go
View file

@ -1,4 +1,4 @@
package registry
package api
import (
"errors"

4
internal/registry/service.go → internal/hcp/api/service.go
View file

@ -1,4 +1,4 @@
package registry
package api
import (
"context"
@ -55,7 +55,7 @@ func (client *Client) UpsertBucket(
// Create bucket if exist we continue as is, eventually we want to treat
// this like an upsert
_, err := client.CreateBucket(ctx, bucketSlug, bucketDescription, bucketLabels)
if err != nil && !checkErrorCode(err, codes.AlreadyExists) {
if err != nil && !CheckErrorCode(err, codes.AlreadyExists) {
return err
}

1
internal/registry/env/env.go → internal/hcp/env/env.go vendored
View file

@ -1,3 +1,4 @@
// Package env provides HCP Packer environment variables.
package env
import (

0
internal/registry/env/env_test.go → internal/hcp/env/env_test.go vendored
View file

0
internal/registry/env/variables.go → internal/hcp/env/variables.go vendored
View file

13
internal/hcp/errors.go
View file

@ -1,13 +0,0 @@
package hcp
import "fmt"
// BuildDone is the error retuned by an HCP handler when a build cannot be started since it's already marked as DONE.
type BuildDone struct {
Message string
}
// Error returns the message for the BuildDone type
func (b BuildDone) Error() string {
return fmt.Sprintf("BuildDone: %s", b.Message)
}

31
internal/hcp/noop.go
View file

@ -1,31 +0,0 @@
package hcp
import (
"context"
sdkpacker "github.com/hashicorp/packer-plugin-sdk/packer"
)
// noopOrchestrator is a special handler that does nothing
type noopOrchestrator struct{}
func newNoopHandler() Orchestrator {
return noopOrchestrator{}
}
func (h noopOrchestrator) PopulateIteration(context.Context) error {
return nil
}
func (h noopOrchestrator) BuildStart(context.Context, string) error {
return nil
}
func (h noopOrchestrator) BuildDone(
ctx context.Context,
buildName string,
artifacts []sdkpacker.Artifact,
buildErr error,
) ([]sdkpacker.Artifact, error) {
return artifacts, nil
}

34
internal/hcp/orchestrator.go
View file

@ -1,34 +0,0 @@
package hcp
import (
"context"
"github.com/hashicorp/hcl/v2"
sdkpacker "github.com/hashicorp/packer-plugin-sdk/packer"
"github.com/hashicorp/packer/hcl2template"
"github.com/hashicorp/packer/packer"
)
// Orchestrator is an entity capable to orchestrate a Packer build and upload metadata to HCP
type Orchestrator interface {
PopulateIteration(context.Context) error
BuildStart(context.Context, string) error
BuildDone(ctx context.Context, buildName string, artifacts []sdkpacker.Artifact, buildErr error) ([]sdkpacker.Artifact, error)
}
// GetOrchestrator instanciates the appropriate handler for the configuration type given as parameter.
//
// If no HCP-related data is present, it will be a NoopHandler.
func GetOrchestrator(cfg packer.Handler) (Orchestrator, hcl.Diagnostics) {
var handler Orchestrator
var err hcl.Diagnostics
switch cfg := cfg.(type) {
case *hcl2template.PackerConfig:
handler, err = newHCLOrchestrator(cfg)
case *packer.Core:
handler, err = newJSONOrchestrator(cfg)
}
return handler, err
}

37
internal/hcp/registry/artifact.go Normal file
View file

@ -0,0 +1,37 @@
package registry
import (
"fmt"
)
const BuilderId = "packer.post-processor.hpc-packer-registry"
type registryArtifact struct {
BucketSlug string
IterationID string
BuildName string
}
func (a *registryArtifact) BuilderId() string {
return BuilderId
}
func (*registryArtifact) Id() string {
return ""
}
func (a *registryArtifact) Files() []string {
return []string{}
}
func (a *registryArtifact) String() string {
return fmt.Sprintf("Published metadata to HCP Packer registry packer/%s/iterations/%s", a.BucketSlug, a.IterationID)
}
func (*registryArtifact) State(name string) interface{} {
return nil
}
func (a *registryArtifact) Destroy() error {
return nil
}

11
internal/hcp/registry/errors.go Normal file
View file

@ -0,0 +1,11 @@
package registry
// ErrBuildAlreadyDone is the error returned by an HCP handler when a build cannot be started since it's already marked as DONE.
type ErrBuildAlreadyDone struct {
Message string
}
// Error returns the message for the ErrBuildAlreadyDone type
func (b ErrBuildAlreadyDone) Error() string {
return b.Message
}

76
internal/hcp/hcl.go → internal/hcp/registry/hcl.go
View file

@ -1,4 +1,4 @@
package hcp
package registry
import (
"context"
@ -6,19 +6,17 @@ import (
"github.com/hashicorp/hcl/v2"
sdkpacker "github.com/hashicorp/packer-plugin-sdk/packer"
imgds "github.com/hashicorp/packer/datasource/hcp-packer-image"
iterds "github.com/hashicorp/packer/datasource/hcp-packer-iteration"
"github.com/hashicorp/packer/hcl2template"
"github.com/hashicorp/packer/internal/registry"
"github.com/hashicorp/packer/internal/registry/env"
hcppackerimagedatasource "github.com/hashicorp/packer/internal/hcp/datasource/hcp-packer-image"
hcppackeriterationdatasource "github.com/hashicorp/packer/internal/hcp/datasource/hcp-packer-iteration"
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/gocty"
)
// hclOrchestrator is a HCP handler made for handling HCL configurations
type hclOrchestrator struct {
// HCLMetadataRegistry is a HCP handler made for handling HCL configurations
type HCLMetadataRegistry struct {
configuration *hcl2template.PackerConfig
bucket *registry.Bucket
bucket *Bucket
}
const (
@ -29,13 +27,13 @@ const (
)
// PopulateIteration creates the metadata on HCP for a build
func (h *hclOrchestrator) PopulateIteration(ctx context.Context) error {
func (h *HCLMetadataRegistry) PopulateIteration(ctx context.Context) error {
err := h.bucket.Initialize(ctx)
if err != nil {
return err
}
err = h.bucket.PopulateIteration(ctx)
err = h.bucket.populateIteration(ctx)
if err != nil {
return err
}
@ -47,44 +45,22 @@ func (h *hclOrchestrator) PopulateIteration(ctx context.Context) error {
return nil
}
// BuildStart is invoked when one build for the configuration is starting to be processed
func (h *hclOrchestrator) BuildStart(ctx context.Context, buildName string) error {
return h.bucket.BuildStart(ctx, buildName)
// StartBuild is invoked when one build for the configuration is starting to be processed
func (h *HCLMetadataRegistry) StartBuild(ctx context.Context, buildName string) error {
return h.bucket.startBuild(ctx, buildName)
}
// BuildDone is invoked when one build for the configuration has finished
func (h *hclOrchestrator) BuildDone(
// CompleteBuild is invoked when one build for the configuration has finished
func (h *HCLMetadataRegistry) CompleteBuild(
ctx context.Context,
buildName string,
artifacts []sdkpacker.Artifact,
buildErr error,
) ([]sdkpacker.Artifact, error) {
return h.bucket.BuildDone(ctx, buildName, artifacts, buildErr)
return h.bucket.completeBuild(ctx, buildName, artifacts, buildErr)
}
func newHCLOrchestrator(config *hcl2template.PackerConfig) (Orchestrator, hcl.Diagnostics) {
// HCP_PACKER_REGISTRY is explicitly turned off
if env.IsHCPDisabled() {
return newNoopHandler(), nil
}
mode := HCPConfigUnset
for _, build := range config.Builds {
if build.HCPPackerRegistry != nil {
mode = HCPConfigEnabled
}
}
// HCP_PACKER_BUCKET_NAME is set or HCP_PACKER_REGISTRY not toggled off
if mode == HCPConfigUnset && (env.HasPackerRegistryBucket() || env.IsHCPExplicitelyEnabled()) {
mode = HCPEnvEnabled
}
if mode == HCPConfigUnset {
return newNoopHandler(), nil
}
func NewHCLMetadataRegistry(config *hcl2template.PackerConfig) (*HCLMetadataRegistry, hcl.Diagnostics) {
var diags hcl.Diagnostics
if len(config.Builds) > 1 {
diags = append(diags, &hcl.Diagnostic{
@ -100,8 +76,8 @@ func newHCLOrchestrator(config *hcl2template.PackerConfig) (Orchestrator, hcl.Di
}
withHCLBucketConfiguration := func(bb *hcl2template.BuildBlock) bucketConfigurationOpts {
return func(bucket *registry.Bucket) hcl.Diagnostics {
bb.HCPPackerRegistry.WriteToBucketConfig(bucket)
return func(bucket *Bucket) hcl.Diagnostics {
bucket.ReadFromHCLBuildBlock(bb)
// If at this point the bucket.Slug is still empty,
// last try is to use the build.Name if present
if bucket.Slug == "" && bb.Name != "" {
@ -141,14 +117,14 @@ func newHCLOrchestrator(config *hcl2template.PackerConfig) (Orchestrator, hcl.Di
bucket.RegisterBuildForComponent(source.String())
}
return &hclOrchestrator{
return &HCLMetadataRegistry{
configuration: config,
bucket: bucket,
}, nil
}
func imageValueToDSOutput(imageVal map[string]cty.Value) imgds.DatasourceOutput {
dso := imgds.DatasourceOutput{}
func imageValueToDSOutput(imageVal map[string]cty.Value) hcppackerimagedatasource.DatasourceOutput {
dso := hcppackerimagedatasource.DatasourceOutput{}
for k, v := range imageVal {
switch k {
case "id":
@ -182,8 +158,8 @@ func imageValueToDSOutput(imageVal map[string]cty.Value) imgds.DatasourceOutput
return dso
}
func iterValueToDSOutput(iterVal map[string]cty.Value) iterds.DatasourceOutput {
dso := iterds.DatasourceOutput{}
func iterValueToDSOutput(iterVal map[string]cty.Value) hcppackeriterationdatasource.DatasourceOutput {
dso := hcppackeriterationdatasource.DatasourceOutput{}
for k, v := range iterVal {
switch k {
case "author_id":
@ -213,7 +189,7 @@ func iterValueToDSOutput(iterVal map[string]cty.Value) iterds.DatasourceOutput {
}
func withDatasourceConfiguration(vals map[string]cty.Value) bucketConfigurationOpts {
return func(bucket *registry.Bucket) hcl.Diagnostics {
return func(bucket *Bucket) hcl.Diagnostics {
var diags hcl.Diagnostics
imageDS, imageOK := vals[hcpImageDatasourceType]
@ -223,7 +199,7 @@ func withDatasourceConfiguration(vals map[string]cty.Value) bucketConfigurationO
return nil
}
iterations := map[string]iterds.DatasourceOutput{}
iterations := map[string]hcppackeriterationdatasource.DatasourceOutput{}
var err error
if iterOK {
@ -245,7 +221,7 @@ func withDatasourceConfiguration(vals map[string]cty.Value) bucketConfigurationO
}
}
images := map[string]imgds.DatasourceOutput{}
images := map[string]hcppackerimagedatasource.DatasourceOutput{}
if imageOK {
hcpData := map[string]cty.Value{}
@ -267,7 +243,7 @@ func withDatasourceConfiguration(vals map[string]cty.Value) bucketConfigurationO
}
for _, img := range images {
sourceIteration := registry.ParentIteration{}
sourceIteration := ParentIteration{}
sourceIteration.IterationID = img.IterationID

43
internal/hcp/bucket_init.go → internal/hcp/registry/hcp.go
View file

@ -1,11 +1,12 @@
package hcp
package registry
import (
"fmt"
"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/packer/internal/registry"
"github.com/hashicorp/packer/internal/registry/env"
"github.com/hashicorp/packer/hcl2template"
"github.com/hashicorp/packer/internal/hcp/env"
"github.com/hashicorp/packer/packer"
)
// HCPConfigMode types specify the mode in which HCP configuration
@ -21,12 +22,38 @@ const (
HCPEnvEnabled
)
type bucketConfigurationOpts func(*registry.Bucket) hcl.Diagnostics
type bucketConfigurationOpts func(*Bucket) hcl.Diagnostics
// IsHCPEnabled returns true if HCP integration is enabled for a build
func IsHCPEnabled(cfg packer.Handler) bool {
// HCP_PACKER_REGISTRY is explicitly turned off
if env.IsHCPDisabled() {
return false
}
mode := HCPConfigUnset
switch config := cfg.(type) {
case *hcl2template.PackerConfig:
for _, build := range config.Builds {
if build.HCPPackerRegistry != nil {
mode = HCPConfigEnabled
}
}
}
// HCP_PACKER_BUCKET_NAME is set or HCP_PACKER_REGISTRY not toggled off
if mode == HCPConfigUnset && (env.HasPackerRegistryBucket() || env.IsHCPExplicitelyEnabled()) {
mode = HCPEnvEnabled
}
return mode != HCPConfigUnset
}
// createConfiguredBucket returns a bucket that can be used for connecting to the HCP Packer registry.
// Configuration for the bucket is obtained from the base iteration setting and any addition configuration
// options passed in as opts. All errors during configuration are collected and returned as Diagnostics.
func createConfiguredBucket(templateDir string, opts ...bucketConfigurationOpts) (*registry.Bucket, hcl.Diagnostics) {
func createConfiguredBucket(templateDir string, opts ...bucketConfigurationOpts) (*Bucket, hcl.Diagnostics) {
var diags hcl.Diagnostics
if !env.HasHCPCredentials() {
@ -40,7 +67,7 @@ func createConfiguredBucket(templateDir string, opts ...bucketConfigurationOpts)
})
}
bucket := registry.NewBucketWithIteration()
bucket := NewBucketWithIteration()
for _, opt := range opts {
if optDiags := opt(bucket); optDiags.HasErrors() {
@ -60,7 +87,7 @@ func createConfiguredBucket(templateDir string, opts ...bucketConfigurationOpts)
})
}
err := bucket.Iteration.Initialize(registry.IterationOptions{
err := bucket.Iteration.Initialize(IterationOptions{
TemplateBaseDir: templateDir,
})
@ -75,7 +102,7 @@ func createConfiguredBucket(templateDir string, opts ...bucketConfigurationOpts)
return bucket, diags
}
func withPackerEnvConfiguration(bucket *registry.Bucket) hcl.Diagnostics {
func withPackerEnvConfiguration(bucket *Bucket) hcl.Diagnostics {
// Add default values for Packer settings configured via EnvVars.
// TODO look to break this up to be more explicit on what is loaded here.
bucket.LoadDefaultSettingsFromEnv()

35
internal/hcp/json.go → internal/hcp/registry/json.go
View file

@ -1,4 +1,4 @@
package hcp
package registry
import (
"context"
@ -6,23 +6,16 @@ import (
"github.com/hashicorp/hcl/v2"
sdkpacker "github.com/hashicorp/packer-plugin-sdk/packer"
"github.com/hashicorp/packer/internal/registry"
"github.com/hashicorp/packer/internal/registry/env"
"github.com/hashicorp/packer/packer"
)
// jsonOrchestrator is a HCP handler made to process legacy JSON templates
type jsonOrchestrator struct {
// JSONMetadataRegistry is a HCP handler made to process legacy JSON templates
type JSONMetadataRegistry struct {
configuration *packer.Core
bucket *registry.Bucket
bucket *Bucket
}
func newJSONOrchestrator(config *packer.Core) (Orchestrator, hcl.Diagnostics) {
if env.IsHCPDisabled() ||
(!env.HasPackerRegistryBucket() && !env.IsHCPExplicitelyEnabled()) {
return newNoopHandler(), nil
}
func NewJSONMetadataRegistry(config *packer.Core) (*JSONMetadataRegistry, hcl.Diagnostics) {
bucket, diags := createConfiguredBucket(
filepath.Dir(config.Template.Path),
withPackerEnvConfiguration,
@ -37,14 +30,14 @@ func newJSONOrchestrator(config *packer.Core) (Orchestrator, hcl.Diagnostics) {
bucket.RegisterBuildForComponent(packer.HCPName(b))
}
return &jsonOrchestrator{
return &JSONMetadataRegistry{
configuration: config,
bucket: bucket,
}, nil
}
// PopulateIteration creates the metadata on HCP for a build
func (h *jsonOrchestrator) PopulateIteration(ctx context.Context) error {
func (h *JSONMetadataRegistry) PopulateIteration(ctx context.Context) error {
for _, b := range h.configuration.Template.Builders {
// Get all builds slated within config ignoring any only or exclude flags.
h.bucket.RegisterBuildForComponent(b.Name)
@ -59,7 +52,7 @@ func (h *jsonOrchestrator) PopulateIteration(ctx context.Context) error {
return err
}
err = h.bucket.PopulateIteration(ctx)
err = h.bucket.populateIteration(ctx)
if err != nil {
return err
}
@ -67,17 +60,17 @@ func (h *jsonOrchestrator) PopulateIteration(ctx context.Context) error {
return nil
}
// BuildStart is invoked when one build for the configuration is starting to be processed
func (h *jsonOrchestrator) BuildStart(ctx context.Context, buildName string) error {
return h.bucket.BuildStart(ctx, buildName)
// StartBuild is invoked when one build for the configuration is starting to be processed
func (h *JSONMetadataRegistry) StartBuild(ctx context.Context, buildName string) error {
return h.bucket.startBuild(ctx, buildName)
}
// BuildDone is invoked when one build for the configuration has finished
func (h *jsonOrchestrator) BuildDone(
// CompleteBuild is invoked when one build for the configuration has finished
func (h *JSONMetadataRegistry) CompleteBuild(
ctx context.Context,
buildName string,
artifacts []sdkpacker.Artifact,
buildErr error,
) ([]sdkpacker.Artifact, error) {
return h.bucket.BuildDone(ctx, buildName, artifacts, buildErr)
return h.bucket.completeBuild(ctx, buildName, artifacts, buildErr)
}

27
internal/hcp/registry/null_registry.go Normal file
View file

@ -0,0 +1,27 @@
package registry
import (
"context"
sdkpacker "github.com/hashicorp/packer-plugin-sdk/packer"
)
// nullRegistry is a special handler that does nothing
type nullRegistry struct{}
func (r nullRegistry) PopulateIteration(context.Context) error {
return nil
}
func (r nullRegistry) StartBuild(context.Context, string) error {
return nil
}
func (r nullRegistry) CompleteBuild(
ctx context.Context,
buildName string,
artifacts []sdkpacker.Artifact,
buildErr error,
) ([]sdkpacker.Artifact, error) {
return artifacts, nil
}

45
internal/hcp/registry/registry.go Normal file
View file

@ -0,0 +1,45 @@
// Package registry provides access to the HCP registry.
package registry
import (
"context"
"github.com/hashicorp/hcl/v2"
sdkpacker "github.com/hashicorp/packer-plugin-sdk/packer"
"github.com/hashicorp/packer/hcl2template"
"github.com/hashicorp/packer/packer"
)
// Registry is an entity capable to orchestrate a Packer build and upload metadata to HCP
type Registry interface {
//Configure(packer.Handler)
PopulateIteration(context.Context) error
StartBuild(context.Context, string) error
CompleteBuild(ctx context.Context, buildName string, artifacts []sdkpacker.Artifact, buildErr error) ([]sdkpacker.Artifact, error)
}
// New instanciates the appropriate registry for the Packer configuration template type.
// A nullRegistry is returned for non-HCP Packer registry enabled templates.
func New(cfg packer.Handler) (Registry, hcl.Diagnostics) {
if !IsHCPEnabled(cfg) {
return &nullRegistry{}, nil
}
switch config := cfg.(type) {
case *hcl2template.PackerConfig:
// Maybe rename to what it represents....
return NewHCLMetadataRegistry(config)
case *packer.Core:
return NewJSONMetadataRegistry(config)
}
return nil, hcl.Diagnostics{
&hcl.Diagnostic{
Severity: hcl.DiagError,
Summary: "Unknown Config type",
Detail: "The config type %s does not match a Packer-known template type. " +
"This is a Packer error and should be brought up to the Packer " +
"team via a GitHub Issue.",
},
}
}

55
internal/registry/types.bucket.go → internal/hcp/registry/types.bucket.go
View file

@ -13,7 +13,9 @@ import (
"github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/stable/2021-04-30/models"
"github.com/hashicorp/packer-plugin-sdk/packer"
registryimage "github.com/hashicorp/packer-plugin-sdk/packer/registry/image"
"github.com/hashicorp/packer/internal/registry/env"
"github.com/hashicorp/packer/hcl2template"
"github.com/hashicorp/packer/internal/hcp/api"
"github.com/hashicorp/packer/internal/hcp/env"
"github.com/mitchellh/mapstructure"
"google.golang.org/grpc/codes"
)
@ -32,7 +34,7 @@ type Bucket struct {
SourceImagesToParentIterations map[string]ParentIteration
RunningBuilds map[string]chan struct{}
Iteration *Iteration
client *Client
client *api.Client
}
type ParentIteration struct {
@ -61,6 +63,27 @@ func (b *Bucket) Validate() error {
return nil
}
// ReadFromHCLBuildBlock reads the information for initialising a Bucket from a HCL2 build block
func (b *Bucket) ReadFromHCLBuildBlock(hcpBlock *hcl2template.BuildBlock) {
if b == nil {
return
}
b.Description = hcpBlock.Description
hcp := hcpBlock.HCPPackerRegistry
if hcp == nil {
return
}
b.BucketLabels = hcp.BucketLabels
b.BuildLabels = hcp.BuildLabels
// If there's already a Slug this was set from env variable.
// In Packer, env variable overrides config values so we keep it that way for consistency.
if b.Slug == "" && hcp.Slug != "" {
b.Slug = hcp.Slug
}
}
// connect initializes a client connection to a remote HCP Packer Registry service on HCP.
// Upon a successful connection the initialized client is persisted on the Bucket b for later usage.
func (b *Bucket) connect() error {
@ -68,7 +91,7 @@ func (b *Bucket) connect() error {
return nil
}
registryClient, err := NewClient()
registryClient, err := api.NewClient()
if err != nil {
return errors.New("Failed to create client connection to artifact registry: " + err.Error())
}
@ -186,10 +209,10 @@ func (b *Bucket) UpdateBuildStatus(ctx context.Context, name string, status mode
return nil
}
// CompleteBuild should be called to set a build on the HCP Packer registry to DONE.
// markBuildComplete should be called to set a build on the HCP Packer registry to DONE.
// Upon a successful call markBuildComplete will publish all images created by the named build,
// and set the registry build to done. A build with no images can not be set to DONE.
func (b *Bucket) CompleteBuild(ctx context.Context, name string) error {
func (b *Bucket) markBuildComplete(ctx context.Context, name string) error {
buildToUpdate, err := b.Iteration.Build(name)
if err != nil {
return err
@ -297,8 +320,8 @@ func (b *Bucket) createIteration() (*models.HashicorpCloudPackerIteration, error
func (b *Bucket) initializeIteration(ctx context.Context) error {
// load existing iteration using fingerprint.
iteration, err := b.client.GetIteration(ctx, b.Slug, GetIteration_byFingerprint(b.Iteration.Fingerprint))
if checkErrorCode(err, codes.Aborted) {
iteration, err := b.client.GetIteration(ctx, b.Slug, api.GetIteration_byFingerprint(b.Iteration.Fingerprint))
if api.CheckErrorCode(err, codes.Aborted) {
// probably means Iteration doesn't exist need a way to check the error
iteration, err = b.createIteration()
}
@ -325,11 +348,11 @@ func (b *Bucket) initializeIteration(ctx context.Context) error {
return nil
}
// PopulateIteration populates the bucket iteration with the details needed for tracking builds for a Packer run.
// populateIteration populates the bucket iteration with the details needed for tracking builds for a Packer run.
// If an existing Packer registry iteration exists for the said iteration fingerprint, calling initialize on iteration
// that doesn't yet exist will call createIteration to create the entry on the HCP packer registry for the given bucket.
// All build details will be created (if they don't exists) and added to b.Iteration.builds for tracking during runtime.
func (b *Bucket) PopulateIteration(ctx context.Context) error {
func (b *Bucket) populateIteration(ctx context.Context) error {
// list all this iteration's builds so we can figure out which ones
// we want to run against. TODO: pagination?
existingBuilds, err := b.client.ListBuilds(ctx, b.Slug, b.Iteration.ID)
@ -387,7 +410,7 @@ func (b *Bucket) PopulateIteration(ctx context.Context) error {
log.Printf("[TRACE] registering build with iteration for %q.", name)
err := b.CreateInitialBuildForIteration(ctx, name)
if checkErrorCode(err, codes.AlreadyExists) {
if api.CheckErrorCode(err, codes.AlreadyExists) {
log.Printf("[TRACE] build %s already exists in Packer registry, continuing...", name)
return
}
@ -473,9 +496,11 @@ func (b *Bucket) HeartbeatBuild(ctx context.Context, build string) (func(), erro
}, nil
}
func (b *Bucket) BuildStart(ctx context.Context, buildName string) error {
func (b *Bucket) startBuild(ctx context.Context, buildName string) error {
if !b.IsExpectingBuildForComponent(buildName) {
return fmt.Errorf("already done")
return &ErrBuildAlreadyDone{
Message: "build is already done",
}
}
err := b.UpdateBuildStatus(ctx, buildName, models.HashicorpCloudPackerBuildStatusRUNNING)
@ -515,7 +540,7 @@ func (b *Bucket) BuildStart(ctx context.Context, buildName string) error {
return nil
}
func (b *Bucket) BuildDone(
func (b *Bucket) completeBuild(
ctx context.Context,
buildName string,
artifacts []packer.Artifact,
@ -568,7 +593,7 @@ func (b *Bucket) BuildDone(
}
}
parErr := b.CompleteBuild(ctx, buildName)
parErr := b.markBuildComplete(ctx, buildName)
if parErr != nil {
return artifacts, fmt.Errorf(
"failed to update Packer registry with image artifacts for %q: %s",
@ -576,7 +601,7 @@ func (b *Bucket) BuildDone(
parErr)
}
return append(artifacts, &RegistryArtifact{
return append(artifacts, &registryArtifact{
BuildName: buildName,
BucketSlug: b.Slug,
IterationID: b.Iteration.ID,

37
internal/registry/types.bucket_service_test.go → internal/hcp/registry/types.bucket_service_test.go
View file

@ -5,15 +5,16 @@ import (
"testing"
"github.com/hashicorp/hcp-sdk-go/clients/cloud-packer-service/stable/2021-04-30/models"
"github.com/hashicorp/packer/internal/hcp/api"
)
func TestInitialize_NewBucketNewIteration(t *testing.T) {
t.Setenv("HCP_PACKER_BUILD_FINGERPRINT", "testnumber")
mockService := NewMockPackerClientService()
mockService := api.NewMockPackerClientService()
b := &Bucket{
Slug: "TestBucket",
client: &Client{
client: &api.Client{
Packer: mockService,
},
}
@ -47,7 +48,7 @@ func TestInitialize_NewBucketNewIteration(t *testing.T) {
t.Errorf("expected an iteration to created but it didn't")
}
err = b.PopulateIteration(context.TODO())
err = b.populateIteration(context.TODO())
if err != nil {
t.Errorf("unexpected failure: %v", err)
}
@ -63,12 +64,12 @@ func TestInitialize_NewBucketNewIteration(t *testing.T) {
func TestInitialize_ExistingBucketNewIteration(t *testing.T) {
t.Setenv("HCP_PACKER_BUILD_FINGERPRINT", "testnumber")
mockService := NewMockPackerClientService()
mockService := api.NewMockPackerClientService()
mockService.BucketAlreadyExist = true
b := &Bucket{
Slug: "TestBucket",
client: &Client{
client: &api.Client{
Packer: mockService,
},
}
@ -101,7 +102,7 @@ func TestInitialize_ExistingBucketNewIteration(t *testing.T) {
t.Errorf("expected an iteration to created but it didn't")
}
err = b.PopulateIteration(context.TODO())
err = b.populateIteration(context.TODO())
if err != nil {
t.Errorf("unexpected failure: %v", err)
}
@ -117,13 +118,13 @@ func TestInitialize_ExistingBucketNewIteration(t *testing.T) {
func TestInitialize_ExistingBucketExistingIteration(t *testing.T) {
t.Setenv("HCP_PACKER_BUILD_FINGERPRINT", "testnumber")
mockService := NewMockPackerClientService()
mockService := api.NewMockPackerClientService()
mockService.BucketAlreadyExist = true
mockService.IterationAlreadyExist = true
b := &Bucket{
Slug: "TestBucket",
client: &Client{
client: &api.Client{
Packer: mockService,
},
}
@ -141,7 +142,7 @@ func TestInitialize_ExistingBucketExistingIteration(t *testing.T) {
if err != nil {
t.Errorf("unexpected failure: %v", err)
}
err = b.PopulateIteration(context.TODO())
err = b.populateIteration(context.TODO())
if err != nil {
t.Errorf("unexpected failure: %v", err)
}
@ -170,7 +171,7 @@ func TestInitialize_ExistingBucketExistingIteration(t *testing.T) {
t.Errorf("expected an iteration to created but it didn't")
}
err = b.PopulateIteration(context.TODO())
err = b.populateIteration(context.TODO())
if err != nil {
t.Errorf("unexpected failure: %v", err)
}
@ -187,7 +188,7 @@ func TestInitialize_ExistingBucketExistingIteration(t *testing.T) {
func TestInitialize_ExistingBucketCompleteIteration(t *testing.T) {
t.Setenv("HCP_PACKER_BUILD_FINGERPRINT", "testnumber")
mockService := NewMockPackerClientService()
mockService := api.NewMockPackerClientService()
mockService.BucketAlreadyExist = true
mockService.IterationAlreadyExist = true
mockService.IterationCompleted = true
@ -195,7 +196,7 @@ func TestInitialize_ExistingBucketCompleteIteration(t *testing.T) {
b := &Bucket{
Slug: "TestBucket",
client: &Client{
client: &api.Client{
Packer: mockService,
},
}
@ -233,13 +234,13 @@ func TestInitialize_ExistingBucketCompleteIteration(t *testing.T) {
func TestUpdateBuildStatus(t *testing.T) {
t.Setenv("HCP_PACKER_BUILD_FINGERPRINT", "testnumber")
mockService := NewMockPackerClientService()
mockService := api.NewMockPackerClientService()
mockService.BucketAlreadyExist = true
mockService.IterationAlreadyExist = true
b := &Bucket{
Slug: "TestBucket",
client: &Client{
client: &api.Client{
Packer: mockService,
},
}
@ -256,7 +257,7 @@ func TestUpdateBuildStatus(t *testing.T) {
if err != nil {
t.Errorf("unexpected failure: %v", err)
}
err = b.PopulateIteration(context.TODO())
err = b.populateIteration(context.TODO())
if err != nil {
t.Errorf("unexpected failure: %v", err)
}
@ -287,13 +288,13 @@ func TestUpdateBuildStatus(t *testing.T) {
func TestUpdateBuildStatus_DONENoImages(t *testing.T) {
t.Setenv("HCP_PACKER_BUILD_FINGERPRINT", "testnumber")
mockService := NewMockPackerClientService()
mockService := api.NewMockPackerClientService()
mockService.BucketAlreadyExist = true
mockService.IterationAlreadyExist = true
b := &Bucket{
Slug: "TestBucket",
client: &Client{
client: &api.Client{
Packer: mockService,
},
}
@ -311,7 +312,7 @@ func TestUpdateBuildStatus_DONENoImages(t *testing.T) {
if err != nil {
t.Errorf("unexpected failure: %v", err)
}
err = b.PopulateIteration(context.TODO())
err = b.populateIteration(context.TODO())
if err != nil {
t.Errorf("unexpected failure: %v", err)
}

13
internal/registry/types.bucket_test.go → internal/hcp/registry/types.bucket_test.go
View file

@ -6,6 +6,7 @@ import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/hashicorp/packer/internal/hcp/api"
)
func createInitialTestBucket(t testing.TB) *Bucket {
@ -19,10 +20,10 @@ func createInitialTestBucket(t testing.TB) *Bucket {
return nil
}
mockService := NewMockPackerClientService()
mockService := api.NewMockPackerClientService()
mockService.TrackCalledServiceMethods = false
bucket.Slug = "TestBucket"
bucket.client = &Client{
bucket.client = &api.Client{
Packer: mockService,
}
@ -164,7 +165,7 @@ func TestBucket_UpdateLabelsForBuild_withMultipleBuilds(t *testing.T) {
secondComponent := "happycloud.image2"
bucket.RegisterBuildForComponent(secondComponent)
err := bucket.PopulateIteration(context.TODO())
err := bucket.populateIteration(context.TODO())
checkError(t, err)
err = bucket.UpdateLabelsForBuild(firstComponent, map[string]string{
@ -282,7 +283,7 @@ func TestBucket_PopulateIteration(t *testing.T) {
t.Setenv("HCP_PACKER_BUILD_FINGERPRINT", "test-run-"+strconv.Itoa(i))
mockService := NewMockPackerClientService()
mockService := api.NewMockPackerClientService()
mockService.BucketAlreadyExist = true
mockService.IterationAlreadyExist = true
mockService.BuildAlreadyDone = tt.buildCompleted
@ -294,7 +295,7 @@ func TestBucket_PopulateIteration(t *testing.T) {
}
bucket.Slug = "TestBucket"
bucket.client = &Client{
bucket.client = &api.Client{
Packer: mockService,
}
for k, v := range tt.bucketBuildLabels {
@ -307,7 +308,7 @@ func TestBucket_PopulateIteration(t *testing.T) {
mockService.ExistingBuilds = append(mockService.ExistingBuilds, componentName)
mockService.ExistingBuildLabels = tt.buildLabels
err = bucket.PopulateIteration(context.TODO())
err = bucket.populateIteration(context.TODO())
checkError(t, err)
if mockService.CreateBuildCalled {

0
internal/registry/types.builds.go → internal/hcp/registry/types.builds.go
View file

2
internal/registry/types.iterations.go → internal/hcp/registry/types.iterations.go
View file

@ -8,7 +8,7 @@ import (
git "github.com/go-git/go-git/v5"
registryimage "github.com/hashicorp/packer-plugin-sdk/packer/registry/image"
"github.com/hashicorp/packer/internal/registry/env"
"github.com/hashicorp/packer/internal/hcp/env"
)
type Iteration struct {

0
internal/registry/types.iterations_test.go → internal/hcp/registry/types.iterations_test.go
View file

37
internal/registry/registry_artifact.go
View file

@ -1,37 +0,0 @@
package registry
import (
"fmt"
)
const BuilderId = "packer.post-processor.packer-registry"
type RegistryArtifact struct {
BucketSlug string
IterationID string
BuildName string
}
func (a *RegistryArtifact) BuilderId() string {
return BuilderId
}
func (*RegistryArtifact) Id() string {
return ""
}
func (a *RegistryArtifact) Files() []string {
return []string{}
}
func (a *RegistryArtifact) String() string {
return fmt.Sprintf("Published metadata to HCP Packer registry packer/%s/iterations/%s", a.BucketSlug, a.IterationID)
}
func (*RegistryArtifact) State(name string) interface{} {
return nil
}
func (a *RegistryArtifact) Destroy() error {
return nil
}